@thor123141245r/ai-translate 0.0.0

package/dist/split.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.js","sourceRoot":"","sources":["../src/split.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAA;AAKzE,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAQpD,MAAM,kBAAkB,GAA6B;IACnD,GAAG,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC;IACzB,EAAE,EAAE,CAAC,KAAK,CAAC;IACX,IAAI,EAAE,CAAC,OAAO,CAAC;IACf,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC;IAClD,GAAG,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,CAAC,QAAQ,CAAC;IACjB,MAAM,EAAE,CAAC,KAAK,CAAC;IACf,GAAG,EAAE,CAAC,MAAM,CAAC;IACb,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,CAAC,KAAK,CAAC;IACb,KAAK,EAAE,CAAC,QAAQ,EAAE,KAAK,CAAC;IACxB,KAAK,EAAE,CAAC,QAAQ,CAAC;IACjB,QAAQ,EAAE,CAAC,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC;IACvC,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC;IACjC,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC;CACrE,CAAA;AAED,IAAI,iBAA4E,CAAA;AAEhF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAClC,OAAe,EAC4B,EAAE;IAC7C,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,iBAAiB,GAAG,EAAE,CAAA;QACtB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC/D,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBACzB,iBAAiB,CAAC,GAAG,CAAC,GAAG,GAAoC,CAAA;YAC/D,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAA;AACnC,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,OAA4B,EAAE,EAAE;IACzE,MAAM,EAAE,MAAM,GAAG,UAAU,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,IAAI,EAAE,CAAA;IACtD,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,CAAC;QAC3C,CAAC,CAAC,8BAA8B,CAAC,wBAAwB,CAAC,MAAM,CAAC;QACjE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;IAC3B,OAAO,IAAI,0BAA0B,CAAC;QACpC,SAAS,EAAE,IAAI;QACf,YAAY,EAAE,CAAC;QACf,UAAU;QACV,GAAG,IAAI;KACR,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,uFAAuF;AACvF,MAAM,OAAO,0BAA2B,SAAQ,8BAA8B;IACzD,QAAQ,CAAC,IAAc,EAAE,SAAiB;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACjC,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;IAClC,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,cAAc;IACnC,aAAa,CAAkB;IACxC,UAAU,CAAU;IACpB,OAAO,CAAQ;IAEvB,YAAY,OAA6C;QACvD,MAAM,EAAE,YAAY,EAAE,GAAG,OAAO,IAAI,EAAE,CAAA;QACtC,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,aAAa,GAAG,YAAY,IAAI,0BAA0B,EAAE,CAAA;QACjE,IAAI,CAAC,UAAU,GAAG,EAAE,CAAA;QACpB,IAAI,CAAC,OAAO,GAAG,EAAE,CAAA;IACnB,CAAC;IAES,KAAK,CAAC,eAAe,CAC7B,KAAsB,EACtB,QAAmC;QAEnC,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACrD,IAAI,CAAC,OAAO,IAAI,IAAI,CAAA;QAEpB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAClE,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAA;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAEnD,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;YACxC,IAAI,QAAQ,KAAK,SAAS;gBAAE,MAAK;YACjC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACjC,IAAI,IAAI,EAAE,CAAC;gBACT,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,CAAC,IAAoC;QACzC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACvC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACrB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACvB,IAAI,EAAE,CAAA;IACR,CAAC;CACF"}

package/dist/utils.d.ts

@@ -0,0 +1,7 @@
+export declare const isWhiteSpace: (text?: string) => boolean;
+export declare const whiteSpace: (text?: string) => {
+    head: string;
+    tail: string;
+};
+export declare const preserveWhiteSpace: (inp: string, out: string) => string;
+//# sourceMappingURL=utils.d.ts.map

package/dist/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,GAAI,aAAS,YAAuB,CAAA;AAE7D,eAAO,MAAM,UAAU,GAAI,aAAS;;;CAOnC,CAAA;AAED,eAAO,MAAM,kBAAkB,GAAI,KAAK,MAAM,EAAE,KAAK,MAAM,WAG1D,CAAA"}

package/dist/utils.js

@@ -0,0 +1,14 @@
+export const isWhiteSpace = (text = '') => text.trim() === '';
+export const whiteSpace = (text = '') => {
+    const head = /^\s{0,200}/.exec(text)?.[0] || '';
+    let tail = '';
+    if (head.length !== text.length) {
+        tail = /\s{0,200}$/.exec(text)?.[0] || '';
+    }
+    return { head, tail };
+};
+export const preserveWhiteSpace = (inp, out) => {
+    const { head, tail } = whiteSpace(inp);
+    return head + out.trim() + tail;
+};
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAI,GAAG,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAA;AAE7D,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAI,GAAG,EAAE,EAAE,EAAE;IACtC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC/C,IAAI,IAAI,GAAG,EAAE,CAAA;IACb,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC3C,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;AACvB,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,GAAW,EAAE,GAAW,EAAE,EAAE;IAC7D,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,CAAA;IACtC,OAAO,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,GAAG,IAAI,CAAA;AACjC,CAAC,CAAA"}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@thor123141245r/ai-translate",
+  "version": "0.0.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "bin": {
+    "ai-translate": "dist/bin/ai-translate.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "dev:cli": "tsx src/bin/ai-translate.ts",
+    "start": "node dist/bin/ai-translate.js",
+    "validate": "tsx scripts/validate-ai-translate.ts"
+  },
+  "dependencies": {
+    "@commenthol/app-config": "^0.1.1",
+    "@langchain/anthropic": "^0.3.15",
+    "@langchain/core": "^0.3.42",
+    "@langchain/deepseek": "^0.0.1",
+    "@langchain/mistralai": "^0.2.0",
+    "@langchain/ollama": "^0.2.0",
+    "@langchain/openai": "^0.4.4",
+    "@langchain/textsplitters": "^0.1.0",
+    "debug-level": "^4.1.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.2",
+    "tsx": "^4.19.2",
+    "typescript": "^5.5.3"
+  }
+}

package/src/asyncTransform.ts

@@ -0,0 +1,31 @@
+import { Transform } from 'node:stream'
+
+export class AsyncTransform extends Transform {
+  static toString(chunk: Buffer | object, encoding: BufferEncoding | 'buffer') {
+    const enc = encoding !== 'buffer' ? encoding : undefined
+    return Buffer.isBuffer(chunk) ? chunk.toString(enc) : String(chunk)
+  }
+
+  _transform(
+    chunk: Buffer | object,
+    enc: BufferEncoding | 'buffer',
+    cb: (error?: Error | null) => void
+  ) {
+    this._asyncTransform(chunk, enc)
+      .then((wait) => {
+        if (wait) {
+          this.once('drain', () => cb())
+        } else {
+          cb()
+        }
+      })
+      .catch((err) => this.emit('error', err))
+  }
+
+  protected async _asyncTransform(
+    _chunk: Buffer | object,
+    _enc: BufferEncoding | 'buffer'
+  ): Promise<boolean | undefined> {
+    throw new Error('needs implementation')
+  }
+}

package/src/bin/ai-translate.ts

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+
+import { cli } from '../cli.js'
+
+await cli()

package/src/cli.ts

@@ -0,0 +1,313 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { pipeline } from 'node:stream/promises'
+import { initProcLog } from 'debug-level'
+import { AppConfig, StringSchema, v } from '@commenthol/app-config'
+import type { SupportedTextSplitterLanguage } from '@langchain/textsplitters'
+import { logger } from './logger.js'
+import { modelFactory, AiTranslateTransform } from './model.js'
+import {
+  TextSplitterStream,
+  recursiveChunkTextSplitter,
+  getFormatByExtension
+} from './split.js'
+
+const log = logger('cli')
+
+const help: Record<string, string> = {}
+
+help._ = `
+AI Translator
+
+Usage:
+  ai-translate [flags]
+  ai-translate [command] [flags]
+
+Commands:
+  set                 set config value
+
+Flags:
+  -h, --help            Help for ai-translate
+  -v, --version         Show version information
+  -c, --config DIR      Use config file .ai-translate.json in DIR
+  -f, --from LANG       Source language
+  -t, --to LANG         Target language; LANG is English language name or
+                        supported BCP47 codes (ar, de, en, es, fr, ja, pt, ru,
+                        vi, zh-CN, zh-TW)
+  -i, --input FILE      input file
+  -o, --output FILE     output file
+      --format FORMAT   specify input format (cpp, go, java, js, php, proto,
+                        python, rst, ruby, rust, scala, swift, markdown, latex,
+                        html, sol)
+Examples:
+  Translate input.md from Spanish to output.md in English
+    ai-translate -f Spanish -t English -i input.md -o output.md
+
+  Pipe from stdin to stdout using the config in the local folder
+    echo "translate" | ai-translate -f en -t en -c .
+
+Use "ai-translate [command] --help" for more information about a command.
+`
+
+help.set = `
+Set ai-translate configuration
+
+Writes config to \`.ai-translate.json\`
+If --config flag is omitted then global config is used.
+
+Usage:
+  ai-translate [flags] set KEY VALUE
+
+Flags:
+  -c, --config DIR    Use config file .ai-translate.json in DIR
+
+Available KEYs:
+  provider      set provider (ollama, mistral, anthropic, openai, deepseek);
+                default="ollama"
+  model         set model from provider; default="qwen2.5:7b"
+  apiKey        set api key
+  baseUrl       baseUrl for model
+  temperature   model temperature; default=0.1
+  maxRetries    max. number of retries; default=10
+  chunkSize     number of chunks used in text-splitter; default=1000
+`
+
+const PACKAGE_JSON = '../package.json'
+const APP = 'ai-translate'
+const CONF_FILE = `.${APP}.json`
+const DEFAULT_LANG = 'en'
+
+const commands: Record<string, (cmd: Record<string, unknown>, argv: string[]) => void> = {
+  set: (c, argv) => {
+    const key = nextArg(argv)
+    const value = nextArg(argv)
+    c.set = [key, value]
+  }
+}
+
+const flags: Record<string, (opts: Record<string, unknown>, argv: string[]) => void> = {
+  '--help': (f) => (f.help = true),
+  '--version': (f) => (f.version = true),
+  '--config': (f, argv) => {
+    const dir = nextArg(argv)
+    if (dir) f.config = path.resolve(process.cwd(), dir)
+  },
+  '--from': (f, argv) => {
+    const lang = nextArg(argv)
+    f.sourceLanguage = lang || DEFAULT_LANG
+  },
+  '--to': (f, argv) => {
+    const lang = nextArg(argv)
+    f.targetLanguage = lang || DEFAULT_LANG
+  },
+  '--input': (f, argv) => {
+    const filename = nextArg(argv)
+    if (filename) f.input = path.resolve(process.cwd(), filename)
+  },
+  '--output': (f, argv) => {
+    const filename = nextArg(argv)
+    if (filename) f.output = path.resolve(process.cwd(), filename)
+  },
+  '--format': (f, argv) => {
+    const format = nextArg(argv)
+    if (format) f.format = format
+  }
+}
+flags['-h'] = flags['--help']
+flags['-v'] = flags['--version']
+flags['-c'] = flags['--config']
+flags['-f'] = flags['--from']
+flags['-t'] = flags['--to']
+flags['-i'] = flags['--input']
+flags['-o'] = flags['--output']
+
+export const argvParse = (args?: string[]) => {
+  initProcLog()
+  const argv = args || process.argv.slice(2)
+  log.debug(argv)
+  const opts: Record<string, unknown> = {
+    sourceLanguage: 'en',
+    targetLanguage: 'en'
+  }
+  const cmd: Record<string, unknown> = {}
+
+  while (argv.length) {
+    const arg = argv.shift()
+    if (!arg) continue
+    if (commands[arg]) {
+      commands[arg](cmd, argv)
+    } else if (flags[arg]) {
+      flags[arg](opts, argv)
+    }
+  }
+  return { cmd, opts }
+}
+
+const nextArg = (argv: string[]) => {
+  const next = argv[0] || ''
+  if (next.indexOf('-') === 0) {
+    return
+  }
+  return argv.shift()
+}
+
+let _console = console
+export const _injectConsole = (obj: typeof console) => {
+  _console = obj
+}
+
+const displayError = (msg: string) => {
+  _console.error(`ERROR: ${msg}`)
+}
+
+const display = (msg: string) => _console.log(msg)
+
+const version = () => {
+  const packageJson = new URL(PACKAGE_JSON, import.meta.url)
+  const { version: pkgVersion } = JSON.parse(
+    fs.readFileSync(packageJson, 'utf-8')
+  ) as { version: string }
+  display(pkgVersion)
+}
+
+const schema = {
+  provider: StringSchema,
+  model: StringSchema,
+  apiKey: StringSchema,
+  baseUrl: StringSchema,
+  temperature: v.pipe(
+    v.string(),
+    v.transform(Number),
+    v.minValue(0),
+    v.maxValue(2)
+  ),
+  maxRetries: v.pipe(
+    v.string(),
+    v.transform(Number),
+    v.integer(),
+    v.minValue(0),
+    v.maxValue(99)
+  ),
+  chunkSize: v.pipe(
+    v.string(),
+    v.transform(Number),
+    v.integer(),
+    v.minValue(100),
+    v.maxValue(200e3)
+  )
+}
+
+const PROVIDER_API_KEY_ENV: Record<string, string> = {
+  openai: 'OPENAI_API_KEY',
+  anthropic: 'ANTHROPIC_API_KEY',
+  mistral: 'MISTRAL_API_KEY',
+  deepseek: 'DEEPSEEK_API_KEY'
+}
+
+const PROVIDER_BASE_URL_ENV: Record<string, string> = {
+  openai: 'OPENAI_BASE_URL',
+  anthropic: 'ANTHROPIC_BASE_URL',
+  mistral: 'MISTRAL_BASE_URL',
+  deepseek: 'DEEPSEEK_BASE_URL',
+  ollama: 'OLLAMA_BASE_URL'
+}
+
+const readEnv = (key?: string) => {
+  if (!key) return
+  const value = process.env[key]
+  if (!value) return
+  const trimmed = value.trim()
+  return trimmed ? trimmed : undefined
+}
+
+const pickFirst = (...values: Array<string | undefined>) =>
+  values.find((value) => value && value.length > 0)
+
+const resolveRuntimeConfig = (config: Record<string, unknown>) => {
+  const provider = String(config.provider || 'ollama')
+  const envApiKey = pickFirst(
+    readEnv('AI_TRANSLATE_API_KEY'),
+    readEnv(PROVIDER_API_KEY_ENV[provider])
+  )
+  const envBaseUrl = pickFirst(
+    readEnv('AI_TRANSLATE_BASE_URL'),
+    readEnv(PROVIDER_BASE_URL_ENV[provider])
+  )
+
+  return {
+    ...config,
+    ...(envApiKey ? { apiKey: envApiKey } : {}),
+    ...(envBaseUrl ? { baseUrl: envBaseUrl } : {})
+  }
+}
+
+export const cli = async (args?: string[]) => {
+  const { cmd, opts } = argvParse(args)
+  const command = Object.keys(cmd)[0]
+  log.debug({ cmd, opts })
+
+  const filename = !opts.config
+    ? CONF_FILE
+    : fs.lstatSync(String(opts.config)).isDirectory()
+      ? path.resolve(String(opts.config), CONF_FILE)
+      : String(opts.config)
+
+  const appConf = new AppConfig({ appName: APP, schema, filename })
+  await appConf.read().catch(() => null)
+
+  try {
+    if (opts.version) {
+      version()
+      return
+    }
+    if (opts.help) {
+      const helpText = help[command || ''] || help._
+      display(helpText)
+      return
+    }
+    if (cmd.set) {
+      const [key, value] = cmd.set as [string | undefined, string | undefined]
+      if (key) {
+        appConf.set(key, value)
+        await appConf.write()
+      } else {
+        display(JSON.stringify(appConf.config, null, 2))
+      }
+      return
+    }
+
+    const sourceLanguage = String(opts.sourceLanguage || DEFAULT_LANG)
+    const targetLanguage = String(opts.targetLanguage || DEFAULT_LANG)
+
+    const reader = opts.input ? fs.createReadStream(String(opts.input)) : process.stdin
+    const writer = opts.output ? fs.createWriteStream(String(opts.output)) : process.stdout
+
+    const runtimeConfig = resolveRuntimeConfig(
+      appConf.config as Record<string, unknown>
+    )
+    const model = modelFactory(runtimeConfig)
+    const lcNamespace =
+      (model as unknown as { lc_namespace?: Record<string, unknown> }).lc_namespace || {}
+    const lcKwargs = (model as unknown as { lc_kwargs?: Record<string, unknown> }).lc_kwargs || {}
+    const { apiKey: _apiKey, ...modelParams } = { ...lcNamespace, ...lcKwargs } as Record<string, unknown>
+    log.debug(modelParams)
+
+    const format =
+      (opts.format as SupportedTextSplitterLanguage | undefined) ||
+      getFormatByExtension(path.extname(String(opts.input || '_.md')))
+    const chunkSize = (runtimeConfig as { chunkSize?: number }).chunkSize || 1000
+    const textSplitter = recursiveChunkTextSplitter({ chunkSize, format })
+    const splitter = new TextSplitterStream({ textSplitter })
+    const translator = new AiTranslateTransform({
+      ...(opts as Record<string, unknown>),
+      sourceLanguage,
+      targetLanguage,
+      model
+    })
+
+    await pipeline(reader, splitter, translator, writer)
+  } catch (err) {
+    log.debug(err)
+    displayError((err as Error).message)
+  }
+}

package/src/index.ts

@@ -0,0 +1,9 @@
+export { AiTranslateTransform, modelFactory } from './model.js'
+export { promptInvoke, languages } from './prompt.js'
+export {
+  TextSplitterStream,
+  recursiveChunkTextSplitter,
+  getFormatByExtension
+} from './split.js'
+export type { Metadata, ModelFactoryOptions } from './model.js'
+export type { TextSplitterParams, Separator } from './split.js'

package/src/logger.ts

@@ -0,0 +1,3 @@
+import { ProcLog } from 'debug-level'
+
+export const logger = (namespace: string) => new ProcLog(`ai-translate:${namespace}`)

package/src/model.ts

@@ -0,0 +1,139 @@
+import { ChatOllama } from '@langchain/ollama'
+import { ChatMistralAI } from '@langchain/mistralai'
+import { ChatAnthropic } from '@langchain/anthropic'
+import { ChatOpenAI } from '@langchain/openai'
+import { ChatDeepSeek } from '@langchain/deepseek'
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
+import { AsyncTransform } from './asyncTransform.js'
+import { promptInvoke, replaceMarkerSymbol } from './prompt.js'
+import { logger } from './logger.js'
+import { isWhiteSpace, preserveWhiteSpace } from './utils.js'
+
+const log = logger('model')
+
+const DEFAULT = {
+  temperature: 0.1,
+  maxRetries: 10,
+  maxConcurrency: 1
+}
+
+export type Metadata = {
+  inputTokens: number
+  outputTokens: number
+}
+
+export type ModelFactoryOptions = {
+  provider?: 'ollama' | 'mistral' | 'anthropic' | 'openai' | 'deepseek'
+  [key: string]: unknown
+}
+
+export const modelFactory = (modelOpts?: ModelFactoryOptions): BaseChatModel => {
+  const { provider = 'ollama', ...other } = modelOpts || {}
+  const apiKeyValue = typeof other.apiKey === 'string' ? other.apiKey.trim() : ''
+  const baseUrlValue = typeof other.baseUrl === 'string' ? other.baseUrl.trim() : ''
+  const apiKey = apiKeyValue ? apiKeyValue : undefined
+  const baseUrl = baseUrlValue ? baseUrlValue : undefined
+  const filtered = { ...other }
+  if ('apiKey' in filtered) {
+    delete filtered.apiKey
+  }
+  if ('baseUrl' in filtered) {
+    delete filtered.baseUrl
+  }
+
+  switch (provider) {
+    case 'ollama':
+      return new ChatOllama({
+        ...DEFAULT,
+        model: 'qwen2.5:7b',
+        ...(baseUrl ? { baseUrl } : {}),
+        ...filtered
+      })
+    case 'mistral':
+      return new ChatMistralAI({
+        ...DEFAULT,
+        model: 'ministral-8b',
+        ...(apiKey ? { apiKey } : {}),
+        ...filtered
+      })
+    case 'anthropic':
+      return new ChatAnthropic({
+        ...DEFAULT,
+        model: 'claude-3-5-haiku-20241022',
+        ...(apiKey ? { apiKey } : {}),
+        ...filtered
+      })
+    case 'openai':
+      return new ChatOpenAI({
+        ...DEFAULT,
+        model: 'gpt-4o-mini',
+        ...(apiKey ? { apiKey } : {}),
+        ...(baseUrl ? { configuration: { baseURL: baseUrl } } : {}),
+        ...filtered
+      })
+    case 'deepseek':
+      return new ChatDeepSeek({
+        ...DEFAULT,
+        model: 'deepseek-reasoner',
+        ...(apiKey ? { apiKey } : {}),
+        ...(baseUrl ? { configuration: { baseURL: baseUrl } } : {}),
+        ...filtered
+      })
+    default:
+      throw new Error(`unsupported provider=${provider}`)
+  }
+}
+
+export class AiTranslateTransform extends AsyncTransform {
+  private readonly _model: BaseChatModel
+  private readonly _promptOpts: {
+    format?: string
+    sourceLanguage: string
+    targetLanguage: string
+  }
+  private _metadata: Metadata = { inputTokens: 0, outputTokens: 0 }
+
+  constructor(options: {
+    model: BaseChatModel
+    format?: string
+    sourceLanguage: string
+    targetLanguage: string
+    [key: string]: unknown
+  }) {
+    const { model, format, sourceLanguage, targetLanguage, ...rest } = options
+    super(rest)
+    this._model = model
+    this._promptOpts = { format, sourceLanguage, targetLanguage }
+  }
+
+  getMetadata() {
+    return { ...this._metadata }
+  }
+
+  protected async _asyncTransform(
+    chunk: Buffer | object,
+    encoding: BufferEncoding | 'buffer'
+  ) {
+    const text = AsyncTransform.toString(chunk, encoding)
+    if (isWhiteSpace(text)) {
+      return !this.push(text)
+    }
+    log.debug('inp=%j', text)
+    const messages = await promptInvoke({ ...this._promptOpts, text })
+    const result = await this._model.invoke(messages)
+    log.debug('out=%j', result.content)
+
+    const usage = (result as { usage_metadata?: { input_tokens?: number; output_tokens?: number } })
+      .usage_metadata
+    const inputTokens = usage?.input_tokens ?? 0
+    const outputTokens = usage?.output_tokens ?? 0
+
+    this.emit('metadata', { inputTokens, outputTokens })
+    this._metadata.inputTokens += inputTokens
+    this._metadata.outputTokens += outputTokens
+
+    return !this.push(
+      preserveWhiteSpace(text, replaceMarkerSymbol(result.content))
+    )
+  }
+}

package/src/prompt.ts

@@ -0,0 +1,71 @@
+import { ChatPromptTemplate } from '@langchain/core/prompts'
+import type { ChatPromptValueInterface } from '@langchain/core/prompt_values'
+import type { MessageContent } from '@langchain/core/messages'
+import { logger } from './logger.js'
+
+const log = logger('prompt')
+
+export const languages: Record<string, string> = {
+  ar: 'Arabic',
+  de: 'German',
+  en: 'English',
+  es: 'Spanish',
+  fr: 'French',
+  ja: 'Japanese',
+  pt: 'Portuguese',
+  ru: 'Russian',
+  vi: 'Vietnamese',
+  'zh-CN': 'Chinese-simplified',
+  'zh-TW': 'Chinese-traditional'
+}
+
+export const getLanguageName = (lang: string) => languages[lang] || lang
+
+const systemPrompt =
+  'You are an AI-driven advanced translation system, specifically designed to ' +
+  'translate structured and technical documents. ' +
+  'You will receive a text snippet from a file formatted as "{format}"\n\n' +
+  'Your task is to **accurately translate** the text enclosed between the 🔤 ' +
+  'symbols from "{sourceLanguage}" to "{targetLanguage}". ' +
+  'Preserve the original formatting, sentence structure, and terminology. ' +
+  'Ensure that every word and sentence is translated as closely as possible to ' +
+  'the original meaning, without summarizing or omitting any part of the content. ' +
+  'The translation must be faithful, detailed, and maintain the original length ' +
+  'and complexity. ' +
+  'Deliver the translation exactly as required, without any additional ' +
+  'commentary or explanation, and ensure the 🔤 symbols are removed in the final output.\n\n' +
+  'Remember: your job is to **translate** the text exactly as it is, without ' +
+  'adding summaries or changing the content in any way. ' +
+  'Do not skip or modify any part of the text. Ensure that the output is a ' +
+  'direct translation, and that the original structure and meaning are preserved.'
+
+const promptTemplate = ChatPromptTemplate.fromMessages([
+  ['system', systemPrompt],
+  ['user', '🔤{text}🔤']
+])
+
+export const promptInvoke = ({
+  format = 'markdown',
+  sourceLanguage,
+  targetLanguage,
+  text = ''
+}: {
+  format?: string
+  sourceLanguage: string
+  targetLanguage: string
+  text: string
+}): Promise<ChatPromptValueInterface> =>
+  promptTemplate.invoke({
+    format,
+    sourceLanguage: getLanguageName(sourceLanguage),
+    targetLanguage: getLanguageName(targetLanguage),
+    text
+  })
+
+export const replaceMarkerSymbol = (text: MessageContent) => {
+  if (typeof text === 'string') {
+    return text.replace(/^\s*🔤/, '').replace(/🔤\s*$/, '')
+  }
+  log.error(text)
+  return ''
+}