@thor123141245r/ai-translate 0.0.0

package/dist/cli.js

@@ -0,0 +1,259 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import { initProcLog } from 'debug-level';
+import { AppConfig, StringSchema, v } from '@commenthol/app-config';
+import { logger } from './logger.js';
+import { modelFactory, AiTranslateTransform } from './model.js';
+import { TextSplitterStream, recursiveChunkTextSplitter, getFormatByExtension } from './split.js';
+const log = logger('cli');
+const help = {};
+help._ = `
+AI Translator
+
+Usage:
+  ai-translate [flags]
+  ai-translate [command] [flags]
+
+Commands:
+  set                 set config value
+
+Flags:
+  -h, --help            Help for ai-translate
+  -v, --version         Show version information
+  -c, --config DIR      Use config file .ai-translate.json in DIR
+  -f, --from LANG       Source language
+  -t, --to LANG         Target language; LANG is English language name or
+                        supported BCP47 codes (ar, de, en, es, fr, ja, pt, ru,
+                        vi, zh-CN, zh-TW)
+  -i, --input FILE      input file
+  -o, --output FILE     output file
+      --format FORMAT   specify input format (cpp, go, java, js, php, proto,
+                        python, rst, ruby, rust, scala, swift, markdown, latex,
+                        html, sol)
+Examples:
+  Translate input.md from Spanish to output.md in English
+    ai-translate -f Spanish -t English -i input.md -o output.md
+
+  Pipe from stdin to stdout using the config in the local folder
+    echo "translate" | ai-translate -f en -t en -c .
+
+Use "ai-translate [command] --help" for more information about a command.
+`;
+help.set = `
+Set ai-translate configuration
+
+Writes config to \`.ai-translate.json\`
+If --config flag is omitted then global config is used.
+
+Usage:
+  ai-translate [flags] set KEY VALUE
+
+Flags:
+  -c, --config DIR    Use config file .ai-translate.json in DIR
+
+Available KEYs:
+  provider      set provider (ollama, mistral, anthropic, openai, deepseek);
+                default="ollama"
+  model         set model from provider; default="qwen2.5:7b"
+  apiKey        set api key
+  baseUrl       baseUrl for model
+  temperature   model temperature; default=0.1
+  maxRetries    max. number of retries; default=10
+  chunkSize     number of chunks used in text-splitter; default=1000
+`;
+const PACKAGE_JSON = '../package.json';
+const APP = 'ai-translate';
+const CONF_FILE = `.${APP}.json`;
+const DEFAULT_LANG = 'en';
+const commands = {
+    set: (c, argv) => {
+        const key = nextArg(argv);
+        const value = nextArg(argv);
+        c.set = [key, value];
+    }
+};
+const flags = {
+    '--help': (f) => (f.help = true),
+    '--version': (f) => (f.version = true),
+    '--config': (f, argv) => {
+        const dir = nextArg(argv);
+        if (dir)
+            f.config = path.resolve(process.cwd(), dir);
+    },
+    '--from': (f, argv) => {
+        const lang = nextArg(argv);
+        f.sourceLanguage = lang || DEFAULT_LANG;
+    },
+    '--to': (f, argv) => {
+        const lang = nextArg(argv);
+        f.targetLanguage = lang || DEFAULT_LANG;
+    },
+    '--input': (f, argv) => {
+        const filename = nextArg(argv);
+        if (filename)
+            f.input = path.resolve(process.cwd(), filename);
+    },
+    '--output': (f, argv) => {
+        const filename = nextArg(argv);
+        if (filename)
+            f.output = path.resolve(process.cwd(), filename);
+    },
+    '--format': (f, argv) => {
+        const format = nextArg(argv);
+        if (format)
+            f.format = format;
+    }
+};
+flags['-h'] = flags['--help'];
+flags['-v'] = flags['--version'];
+flags['-c'] = flags['--config'];
+flags['-f'] = flags['--from'];
+flags['-t'] = flags['--to'];
+flags['-i'] = flags['--input'];
+flags['-o'] = flags['--output'];
+export const argvParse = (args) => {
+    initProcLog();
+    const argv = args || process.argv.slice(2);
+    log.debug(argv);
+    const opts = {
+        sourceLanguage: 'en',
+        targetLanguage: 'en'
+    };
+    const cmd = {};
+    while (argv.length) {
+        const arg = argv.shift();
+        if (!arg)
+            continue;
+        if (commands[arg]) {
+            commands[arg](cmd, argv);
+        }
+        else if (flags[arg]) {
+            flags[arg](opts, argv);
+        }
+    }
+    return { cmd, opts };
+};
+const nextArg = (argv) => {
+    const next = argv[0] || '';
+    if (next.indexOf('-') === 0) {
+        return;
+    }
+    return argv.shift();
+};
+let _console = console;
+export const _injectConsole = (obj) => {
+    _console = obj;
+};
+const displayError = (msg) => {
+    _console.error(`ERROR: ${msg}`);
+};
+const display = (msg) => _console.log(msg);
+const version = () => {
+    const packageJson = new URL(PACKAGE_JSON, import.meta.url);
+    const { version: pkgVersion } = JSON.parse(fs.readFileSync(packageJson, 'utf-8'));
+    display(pkgVersion);
+};
+const schema = {
+    provider: StringSchema,
+    model: StringSchema,
+    apiKey: StringSchema,
+    baseUrl: StringSchema,
+    temperature: v.pipe(v.string(), v.transform(Number), v.minValue(0), v.maxValue(2)),
+    maxRetries: v.pipe(v.string(), v.transform(Number), v.integer(), v.minValue(0), v.maxValue(99)),
+    chunkSize: v.pipe(v.string(), v.transform(Number), v.integer(), v.minValue(100), v.maxValue(200e3))
+};
+const PROVIDER_API_KEY_ENV = {
+    openai: 'OPENAI_API_KEY',
+    anthropic: 'ANTHROPIC_API_KEY',
+    mistral: 'MISTRAL_API_KEY',
+    deepseek: 'DEEPSEEK_API_KEY'
+};
+const PROVIDER_BASE_URL_ENV = {
+    openai: 'OPENAI_BASE_URL',
+    anthropic: 'ANTHROPIC_BASE_URL',
+    mistral: 'MISTRAL_BASE_URL',
+    deepseek: 'DEEPSEEK_BASE_URL',
+    ollama: 'OLLAMA_BASE_URL'
+};
+const readEnv = (key) => {
+    if (!key)
+        return;
+    const value = process.env[key];
+    if (!value)
+        return;
+    const trimmed = value.trim();
+    return trimmed ? trimmed : undefined;
+};
+const pickFirst = (...values) => values.find((value) => value && value.length > 0);
+const resolveRuntimeConfig = (config) => {
+    const provider = String(config.provider || 'ollama');
+    const envApiKey = pickFirst(readEnv('AI_TRANSLATE_API_KEY'), readEnv(PROVIDER_API_KEY_ENV[provider]));
+    const envBaseUrl = pickFirst(readEnv('AI_TRANSLATE_BASE_URL'), readEnv(PROVIDER_BASE_URL_ENV[provider]));
+    return {
+        ...config,
+        ...(envApiKey ? { apiKey: envApiKey } : {}),
+        ...(envBaseUrl ? { baseUrl: envBaseUrl } : {})
+    };
+};
+export const cli = async (args) => {
+    const { cmd, opts } = argvParse(args);
+    const command = Object.keys(cmd)[0];
+    log.debug({ cmd, opts });
+    const filename = !opts.config
+        ? CONF_FILE
+        : fs.lstatSync(String(opts.config)).isDirectory()
+            ? path.resolve(String(opts.config), CONF_FILE)
+            : String(opts.config);
+    const appConf = new AppConfig({ appName: APP, schema, filename });
+    await appConf.read().catch(() => null);
+    try {
+        if (opts.version) {
+            version();
+            return;
+        }
+        if (opts.help) {
+            const helpText = help[command || ''] || help._;
+            display(helpText);
+            return;
+        }
+        if (cmd.set) {
+            const [key, value] = cmd.set;
+            if (key) {
+                appConf.set(key, value);
+                await appConf.write();
+            }
+            else {
+                display(JSON.stringify(appConf.config, null, 2));
+            }
+            return;
+        }
+        const sourceLanguage = String(opts.sourceLanguage || DEFAULT_LANG);
+        const targetLanguage = String(opts.targetLanguage || DEFAULT_LANG);
+        const reader = opts.input ? fs.createReadStream(String(opts.input)) : process.stdin;
+        const writer = opts.output ? fs.createWriteStream(String(opts.output)) : process.stdout;
+        const runtimeConfig = resolveRuntimeConfig(appConf.config);
+        const model = modelFactory(runtimeConfig);
+        const lcNamespace = model.lc_namespace || {};
+        const lcKwargs = model.lc_kwargs || {};
+        const { apiKey: _apiKey, ...modelParams } = { ...lcNamespace, ...lcKwargs };
+        log.debug(modelParams);
+        const format = opts.format ||
+            getFormatByExtension(path.extname(String(opts.input || '_.md')));
+        const chunkSize = runtimeConfig.chunkSize || 1000;
+        const textSplitter = recursiveChunkTextSplitter({ chunkSize, format });
+        const splitter = new TextSplitterStream({ textSplitter });
+        const translator = new AiTranslateTransform({
+            ...opts,
+            sourceLanguage,
+            targetLanguage,
+            model
+        });
+        await pipeline(reader, splitter, translator, writer);
+    }
+    catch (err) {
+        log.debug(err);
+        displayError(err.message);
+    }
+};
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,wBAAwB,CAAA;AAEnE,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,oBAAoB,EACrB,MAAM,YAAY,CAAA;AAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;AAEzB,MAAM,IAAI,GAA2B,EAAE,CAAA;AAEvC,IAAI,CAAC,CAAC,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+BR,CAAA;AAED,IAAI,CAAC,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;;CAqBV,CAAA;AAED,MAAM,YAAY,GAAG,iBAAiB,CAAA;AACtC,MAAM,GAAG,GAAG,cAAc,CAAA;AAC1B,MAAM,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;AAChC,MAAM,YAAY,GAAG,IAAI,CAAA;AAEzB,MAAM,QAAQ,GAA2E;IACvF,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACf,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QACzB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC3B,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACtB,CAAC;CACF,CAAA;AAED,MAAM,KAAK,GAA4E;IACrF,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;IAChC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC;IACtC,UAAU,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,GAAG;YAAE,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAA;IACtD,CAAC;IACD,QAAQ,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACpB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC,CAAC,cAAc,GAAG,IAAI,IAAI,YAAY,CAAA;IACzC,CAAC;IACD,MAAM,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAClB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC1B,CAAC,CAAC,cAAc,GAAG,IAAI,IAAI,YAAY,CAAA;IACzC,CAAC;IACD,SAAS,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACrB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC9B,IAAI,QAAQ;YAAE,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAA;IAC/D,CAAC;IACD,UAAU,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC9B,IAAI,QAAQ;YAAE,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAA;IAChE,CAAC;IACD,UAAU,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAC5B,IAAI,MAAM;YAAE,CAAC,CAAC,MAAM,GAAG,MAAM,CAAA;IAC/B,CAAC;CACF,CAAA;AACD,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAA;AAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC,CAAA;AAChC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAA;AAC/B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAA;AAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAA;AAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,CAAA;AAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAA;AAE/B,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAe,EAAE,EAAE;IAC3C,WAAW,EAAE,CAAA;IACb,MAAM,IAAI,GAAG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAC1C,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACf,MAAM,IAAI,GAA4B;QACpC,cAAc,EAAE,IAAI;QACpB,cAAc,EAAE,IAAI;KACrB,CAAA;IACD,MAAM,GAAG,GAA4B,EAAE,CAAA;IAEvC,OAAO,IAAI,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;QACxB,IAAI,CAAC,GAAG;YAAE,SAAQ;QAClB,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAC1B,CAAC;aAAM,IAAI,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAA;AACtB,CAAC,CAAA;AAED,MAAM,OAAO,GAAG,CAAC,IAAc,EAAE,EAAE;IACjC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IAC1B,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAM;IACR,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;AACrB,CAAC,CAAA;AAED,IAAI,QAAQ,GAAG,OAAO,CAAA;AACtB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,GAAmB,EAAE,EAAE;IACpD,QAAQ,GAAG,GAAG,CAAA;AAChB,CAAC,CAAA;AAED,MAAM,YAAY,GAAG,CAAC,GAAW,EAAE,EAAE;IACnC,QAAQ,CAAC,KAAK,CAAC,UAAU,GAAG,EAAE,CAAC,CAAA;AACjC,CAAC,CAAA;AAED,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;AAElD,MAAM,OAAO,GAAG,GAAG,EAAE;IACnB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAC1D,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,KAAK,CACxC,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CACf,CAAA;IACxB,OAAO,CAAC,UAAU,CAAC,CAAA;AACrB,CAAC,CAAA;AAED,MAAM,MAAM,GAAG;IACb,QAAQ,EAAE,YAAY;IACtB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IACpB,OAAO,EAAE,YAAY;IACrB,WAAW,EAAE,CAAC,CAAC,IAAI,CACjB,CAAC,CAAC,MAAM,EAAE,EACV,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,EACnB,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EACb,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CACd;IACD,UAAU,EAAE,CAAC,CAAC,IAAI,CAChB,CAAC,CAAC,MAAM,EAAE,EACV,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,EACnB,CAAC,CAAC,OAAO,EAAE,EACX,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EACb,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CACf;IACD,SAAS,EAAE,CAAC,CAAC,IAAI,CACf,CAAC,CAAC,MAAM,EAAE,EACV,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,EACnB,CAAC,CAAC,OAAO,EAAE,EACX,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EACf,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAClB;CACF,CAAA;AAED,MAAM,oBAAoB,GAA2B;IACnD,MAAM,EAAE,gBAAgB;IACxB,SAAS,EAAE,mBAAmB;IAC9B,OAAO,EAAE,iBAAiB;IAC1B,QAAQ,EAAE,kBAAkB;CAC7B,CAAA;AAED,MAAM,qBAAqB,GAA2B;IACpD,MAAM,EAAE,iBAAiB;IACzB,SAAS,EAAE,oBAAoB;IAC/B,OAAO,EAAE,kBAAkB;IAC3B,QAAQ,EAAE,mBAAmB;IAC7B,MAAM,EAAE,iBAAiB;CAC1B,CAAA;AAED,MAAM,OAAO,GAAG,CAAC,GAAY,EAAE,EAAE;IAC/B,IAAI,CAAC,GAAG;QAAE,OAAM;IAChB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAM;IAClB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAC5B,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAA;AACtC,CAAC,CAAA;AAED,MAAM,SAAS,GAAG,CAAC,GAAG,MAAiC,EAAE,EAAE,CACzD,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;AAEnD,MAAM,oBAAoB,GAAG,CAAC,MAA+B,EAAE,EAAE;IAC/D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,SAAS,CACzB,OAAO,CAAC,sBAAsB,CAAC,EAC/B,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CACxC,CAAA;IACD,MAAM,UAAU,GAAG,SAAS,CAC1B,OAAO,CAAC,uBAAuB,CAAC,EAChC,OAAO,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CACzC,CAAA;IAED,OAAO;QACL,GAAG,MAAM;QACT,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/C,CAAA;AACH,CAAC,CAAA;AAED,MAAM,CAAC,MAAM,GAAG,GAAG,KAAK,EAAE,IAAe,EAAE,EAAE;IAC3C,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IACrC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IACnC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;IAExB,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,MAAM;QAC3B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE;YAC/C,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC;YAC9C,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAEzB,MAAM,OAAO,GAAG,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;IACjE,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;IAEtC,IAAI,CAAC;QACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,EAAE,CAAA;YACT,OAAM;QACR,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;YAC9C,OAAO,CAAC,QAAQ,CAAC,CAAA;YACjB,OAAM;QACR,CAAC;QACD,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,GAA+C,CAAA;YACxE,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;gBACvB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAA;YACvB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;YAClD,CAAC;YACD,OAAM;QACR,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,YAAY,CAAC,CAAA;QAClE,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,YAAY,CAAC,CAAA;QAElE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAA;QACnF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAA;QAEvF,MAAM,aAAa,GAAG,oBAAoB,CACxC,OAAO,CAAC,MAAiC,CAC1C,CAAA;QACD,MAAM,KAAK,GAAG,YAAY,CAAC,aAAa,CAAC,CAAA;QACzC,MAAM,WAAW,GACd,KAA+D,CAAC,YAAY,IAAI,EAAE,CAAA;QACrF,MAAM,QAAQ,GAAI,KAA4D,CAAC,SAAS,IAAI,EAAE,CAAA;QAC9F,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,WAAW,EAAE,GAAG,EAAE,GAAG,WAAW,EAAE,GAAG,QAAQ,EAA6B,CAAA;QACtG,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QAEtB,MAAM,MAAM,GACT,IAAI,CAAC,MAAoD;YAC1D,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC,CAAC,CAAC,CAAA;QAClE,MAAM,SAAS,GAAI,aAAwC,CAAC,SAAS,IAAI,IAAI,CAAA;QAC7E,MAAM,YAAY,GAAG,0BAA0B,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,IAAI,kBAAkB,CAAC,EAAE,YAAY,EAAE,CAAC,CAAA;QACzD,MAAM,UAAU,GAAG,IAAI,oBAAoB,CAAC;YAC1C,GAAI,IAAgC;YACpC,cAAc;YACd,cAAc;YACd,KAAK;SACN,CAAC,CAAA;QAEF,MAAM,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACd,YAAY,CAAE,GAAa,CAAC,OAAO,CAAC,CAAA;IACtC,CAAC;AACH,CAAC,CAAA"}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export { AiTranslateTransform, modelFactory } from './model.js';
+export { promptInvoke, languages } from './prompt.js';
+export { TextSplitterStream, recursiveChunkTextSplitter, getFormatByExtension } from './split.js';
+export type { Metadata, ModelFactoryOptions } from './model.js';
+export type { TextSplitterParams, Separator } from './split.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,oBAAoB,EACrB,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAC/D,YAAY,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA"}

package/dist/index.js

@@ -0,0 +1,4 @@
+export { AiTranslateTransform, modelFactory } from './model.js';
+export { promptInvoke, languages } from './prompt.js';
+export { TextSplitterStream, recursiveChunkTextSplitter, getFormatByExtension } from './split.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AACrD,OAAO,EACL,kBAAkB,EAClB,0BAA0B,EAC1B,oBAAoB,EACrB,MAAM,YAAY,CAAA"}

package/dist/logger.d.ts

@@ -0,0 +1,3 @@
+import { ProcLog } from 'debug-level';
+export declare const logger: (namespace: string) => ProcLog;
+//# sourceMappingURL=logger.d.ts.map

package/dist/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AAErC,eAAO,MAAM,MAAM,GAAI,WAAW,MAAM,YAA6C,CAAA"}

package/dist/logger.js

@@ -0,0 +1,3 @@
+import { ProcLog } from 'debug-level';
+export const logger = (namespace) => new ProcLog(`ai-translate:${namespace}`);
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AAErC,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,SAAiB,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAA"}

package/dist/model.d.ts

@@ -0,0 +1,29 @@
+import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import { AsyncTransform } from './asyncTransform.js';
+export type Metadata = {
+    inputTokens: number;
+    outputTokens: number;
+};
+export type ModelFactoryOptions = {
+    provider?: 'ollama' | 'mistral' | 'anthropic' | 'openai' | 'deepseek';
+    [key: string]: unknown;
+};
+export declare const modelFactory: (modelOpts?: ModelFactoryOptions) => BaseChatModel;
+export declare class AiTranslateTransform extends AsyncTransform {
+    private readonly _model;
+    private readonly _promptOpts;
+    private _metadata;
+    constructor(options: {
+        model: BaseChatModel;
+        format?: string;
+        sourceLanguage: string;
+        targetLanguage: string;
+        [key: string]: unknown;
+    });
+    getMetadata(): {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    protected _asyncTransform(chunk: Buffer | object, encoding: BufferEncoding | 'buffer'): Promise<boolean>;
+}
+//# sourceMappingURL=model.d.ts.map

package/dist/model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6CAA6C,CAAA;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAapD,MAAM,MAAM,QAAQ,GAAG;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;CACrB,CAAA;AAED,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,GAAG,UAAU,CAAA;IACrE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACvB,CAAA;AAED,eAAO,MAAM,YAAY,GAAI,YAAY,mBAAmB,KAAG,aAuD9D,CAAA;AAED,qBAAa,oBAAqB,SAAQ,cAAc;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAI3B;IACD,OAAO,CAAC,SAAS,CAAgD;gBAErD,OAAO,EAAE;QACnB,KAAK,EAAE,aAAa,CAAA;QACpB,MAAM,CAAC,EAAE,MAAM,CAAA;QACf,cAAc,EAAE,MAAM,CAAA;QACtB,cAAc,EAAE,MAAM,CAAA;QACtB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KACvB;IAOD,WAAW;qBAxFE,MAAM;sBACL,MAAM;;cA2FJ,eAAe,CAC7B,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,QAAQ,EAAE,cAAc,GAAG,QAAQ;CAwBtC"}

package/dist/model.js

@@ -0,0 +1,103 @@
+import { ChatOllama } from '@langchain/ollama';
+import { ChatMistralAI } from '@langchain/mistralai';
+import { ChatAnthropic } from '@langchain/anthropic';
+import { ChatOpenAI } from '@langchain/openai';
+import { ChatDeepSeek } from '@langchain/deepseek';
+import { AsyncTransform } from './asyncTransform.js';
+import { promptInvoke, replaceMarkerSymbol } from './prompt.js';
+import { logger } from './logger.js';
+import { isWhiteSpace, preserveWhiteSpace } from './utils.js';
+const log = logger('model');
+const DEFAULT = {
+    temperature: 0.1,
+    maxRetries: 10,
+    maxConcurrency: 1
+};
+export const modelFactory = (modelOpts) => {
+    const { provider = 'ollama', ...other } = modelOpts || {};
+    const apiKeyValue = typeof other.apiKey === 'string' ? other.apiKey.trim() : '';
+    const baseUrlValue = typeof other.baseUrl === 'string' ? other.baseUrl.trim() : '';
+    const apiKey = apiKeyValue ? apiKeyValue : undefined;
+    const baseUrl = baseUrlValue ? baseUrlValue : undefined;
+    const filtered = { ...other };
+    if ('apiKey' in filtered) {
+        delete filtered.apiKey;
+    }
+    if ('baseUrl' in filtered) {
+        delete filtered.baseUrl;
+    }
+    switch (provider) {
+        case 'ollama':
+            return new ChatOllama({
+                ...DEFAULT,
+                model: 'qwen2.5:7b',
+                ...(baseUrl ? { baseUrl } : {}),
+                ...filtered
+            });
+        case 'mistral':
+            return new ChatMistralAI({
+                ...DEFAULT,
+                model: 'ministral-8b',
+                ...(apiKey ? { apiKey } : {}),
+                ...filtered
+            });
+        case 'anthropic':
+            return new ChatAnthropic({
+                ...DEFAULT,
+                model: 'claude-3-5-haiku-20241022',
+                ...(apiKey ? { apiKey } : {}),
+                ...filtered
+            });
+        case 'openai':
+            return new ChatOpenAI({
+                ...DEFAULT,
+                model: 'gpt-4o-mini',
+                ...(apiKey ? { apiKey } : {}),
+                ...(baseUrl ? { configuration: { baseURL: baseUrl } } : {}),
+                ...filtered
+            });
+        case 'deepseek':
+            return new ChatDeepSeek({
+                ...DEFAULT,
+                model: 'deepseek-reasoner',
+                ...(apiKey ? { apiKey } : {}),
+                ...(baseUrl ? { configuration: { baseURL: baseUrl } } : {}),
+                ...filtered
+            });
+        default:
+            throw new Error(`unsupported provider=${provider}`);
+    }
+};
+export class AiTranslateTransform extends AsyncTransform {
+    _model;
+    _promptOpts;
+    _metadata = { inputTokens: 0, outputTokens: 0 };
+    constructor(options) {
+        const { model, format, sourceLanguage, targetLanguage, ...rest } = options;
+        super(rest);
+        this._model = model;
+        this._promptOpts = { format, sourceLanguage, targetLanguage };
+    }
+    getMetadata() {
+        return { ...this._metadata };
+    }
+    async _asyncTransform(chunk, encoding) {
+        const text = AsyncTransform.toString(chunk, encoding);
+        if (isWhiteSpace(text)) {
+            return !this.push(text);
+        }
+        log.debug('inp=%j', text);
+        const messages = await promptInvoke({ ...this._promptOpts, text });
+        const result = await this._model.invoke(messages);
+        log.debug('out=%j', result.content);
+        const usage = result
+            .usage_metadata;
+        const inputTokens = usage?.input_tokens ?? 0;
+        const outputTokens = usage?.output_tokens ?? 0;
+        this.emit('metadata', { inputTokens, outputTokens });
+        this._metadata.inputTokens += inputTokens;
+        this._metadata.outputTokens += outputTokens;
+        return !this.push(preserveWhiteSpace(text, replaceMarkerSymbol(result.content)));
+    }
+}
+//# sourceMappingURL=model.js.map

package/dist/model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.js","sourceRoot":"","sources":["../src/model.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAA;AAElD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAC/D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACpC,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAE7D,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAA;AAE3B,MAAM,OAAO,GAAG;IACd,WAAW,EAAE,GAAG;IAChB,UAAU,EAAE,EAAE;IACd,cAAc,EAAE,CAAC;CAClB,CAAA;AAYD,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,SAA+B,EAAiB,EAAE;IAC7E,MAAM,EAAE,QAAQ,GAAG,QAAQ,EAAE,GAAG,KAAK,EAAE,GAAG,SAAS,IAAI,EAAE,CAAA;IACzD,MAAM,WAAW,GAAG,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/E,MAAM,YAAY,GAAG,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IAClF,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAA;IACpD,MAAM,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAA;IACvD,MAAM,QAAQ,GAAG,EAAE,GAAG,KAAK,EAAE,CAAA;IAC7B,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACzB,OAAO,QAAQ,CAAC,MAAM,CAAA;IACxB,CAAC;IACD,IAAI,SAAS,IAAI,QAAQ,EAAE,CAAC;QAC1B,OAAO,QAAQ,CAAC,OAAO,CAAA;IACzB,CAAC;IAED,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ;YACX,OAAO,IAAI,UAAU,CAAC;gBACpB,GAAG,OAAO;gBACV,KAAK,EAAE,YAAY;gBACnB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/B,GAAG,QAAQ;aACZ,CAAC,CAAA;QACJ,KAAK,SAAS;YACZ,OAAO,IAAI,aAAa,CAAC;gBACvB,GAAG,OAAO;gBACV,KAAK,EAAE,cAAc;gBACrB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7B,GAAG,QAAQ;aACZ,CAAC,CAAA;QACJ,KAAK,WAAW;YACd,OAAO,IAAI,aAAa,CAAC;gBACvB,GAAG,OAAO;gBACV,KAAK,EAAE,2BAA2B;gBAClC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7B,GAAG,QAAQ;aACZ,CAAC,CAAA;QACJ,KAAK,QAAQ;YACX,OAAO,IAAI,UAAU,CAAC;gBACpB,GAAG,OAAO;gBACV,KAAK,EAAE,aAAa;gBACpB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,GAAG,QAAQ;aACZ,CAAC,CAAA;QACJ,KAAK,UAAU;YACb,OAAO,IAAI,YAAY,CAAC;gBACtB,GAAG,OAAO;gBACV,KAAK,EAAE,mBAAmB;gBAC1B,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,GAAG,QAAQ;aACZ,CAAC,CAAA;QACJ;YACE,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAA;IACvD,CAAC;AACH,CAAC,CAAA;AAED,MAAM,OAAO,oBAAqB,SAAQ,cAAc;IACrC,MAAM,CAAe;IACrB,WAAW,CAI3B;IACO,SAAS,GAAa,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAA;IAEjE,YAAY,OAMX;QACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAA;QAC1E,KAAK,CAAC,IAAI,CAAC,CAAA;QACX,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;QACnB,IAAI,CAAC,WAAW,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,CAAA;IAC/D,CAAC;IAED,WAAW;QACT,OAAO,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;IAC9B,CAAC;IAES,KAAK,CAAC,eAAe,CAC7B,KAAsB,EACtB,QAAmC;QAEnC,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;QACrD,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,CAAC;QACD,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAA;QACzB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAClE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QACjD,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,OAAO,CAAC,CAAA;QAEnC,MAAM,KAAK,GAAI,MAAiF;aAC7F,cAAc,CAAA;QACjB,MAAM,WAAW,GAAG,KAAK,EAAE,YAAY,IAAI,CAAC,CAAA;QAC5C,MAAM,YAAY,GAAG,KAAK,EAAE,aAAa,IAAI,CAAC,CAAA;QAE9C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,CAAC,CAAA;QACpD,IAAI,CAAC,SAAS,CAAC,WAAW,IAAI,WAAW,CAAA;QACzC,IAAI,CAAC,SAAS,CAAC,YAAY,IAAI,YAAY,CAAA;QAE3C,OAAO,CAAC,IAAI,CAAC,IAAI,CACf,kBAAkB,CAAC,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAC9D,CAAA;IACH,CAAC;CACF"}

package/dist/prompt.d.ts

@@ -0,0 +1,12 @@
+import type { ChatPromptValueInterface } from '@langchain/core/prompt_values';
+import type { MessageContent } from '@langchain/core/messages';
+export declare const languages: Record<string, string>;
+export declare const getLanguageName: (lang: string) => string;
+export declare const promptInvoke: ({ format, sourceLanguage, targetLanguage, text }: {
+    format?: string;
+    sourceLanguage: string;
+    targetLanguage: string;
+    text: string;
+}) => Promise<ChatPromptValueInterface>;
+export declare const replaceMarkerSymbol: (text: MessageContent) => string;
+//# sourceMappingURL=prompt.d.ts.map

package/dist/prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.d.ts","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAA;AAC7E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAK9D,eAAO,MAAM,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAY5C,CAAA;AAED,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,WAA4B,CAAA;AAyBxE,eAAO,MAAM,YAAY,GAAI,kDAK1B;IACD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;CACb,KAAG,OAAO,CAAC,wBAAwB,CAMhC,CAAA;AAEJ,eAAO,MAAM,mBAAmB,GAAI,MAAM,cAAc,WAMvD,CAAA"}

package/dist/prompt.js

@@ -0,0 +1,51 @@
+import { ChatPromptTemplate } from '@langchain/core/prompts';
+import { logger } from './logger.js';
+const log = logger('prompt');
+export const languages = {
+    ar: 'Arabic',
+    de: 'German',
+    en: 'English',
+    es: 'Spanish',
+    fr: 'French',
+    ja: 'Japanese',
+    pt: 'Portuguese',
+    ru: 'Russian',
+    vi: 'Vietnamese',
+    'zh-CN': 'Chinese-simplified',
+    'zh-TW': 'Chinese-traditional'
+};
+export const getLanguageName = (lang) => languages[lang] || lang;
+const systemPrompt = 'You are an AI-driven advanced translation system, specifically designed to ' +
+    'translate structured and technical documents. ' +
+    'You will receive a text snippet from a file formatted as "{format}"\n\n' +
+    'Your task is to **accurately translate** the text enclosed between the 🔤 ' +
+    'symbols from "{sourceLanguage}" to "{targetLanguage}". ' +
+    'Preserve the original formatting, sentence structure, and terminology. ' +
+    'Ensure that every word and sentence is translated as closely as possible to ' +
+    'the original meaning, without summarizing or omitting any part of the content. ' +
+    'The translation must be faithful, detailed, and maintain the original length ' +
+    'and complexity. ' +
+    'Deliver the translation exactly as required, without any additional ' +
+    'commentary or explanation, and ensure the 🔤 symbols are removed in the final output.\n\n' +
+    'Remember: your job is to **translate** the text exactly as it is, without ' +
+    'adding summaries or changing the content in any way. ' +
+    'Do not skip or modify any part of the text. Ensure that the output is a ' +
+    'direct translation, and that the original structure and meaning are preserved.';
+const promptTemplate = ChatPromptTemplate.fromMessages([
+    ['system', systemPrompt],
+    ['user', '🔤{text}🔤']
+]);
+export const promptInvoke = ({ format = 'markdown', sourceLanguage, targetLanguage, text = '' }) => promptTemplate.invoke({
+    format,
+    sourceLanguage: getLanguageName(sourceLanguage),
+    targetLanguage: getLanguageName(targetLanguage),
+    text
+});
+export const replaceMarkerSymbol = (text) => {
+    if (typeof text === 'string') {
+        return text.replace(/^\s*🔤/, '').replace(/🔤\s*$/, '');
+    }
+    log.error(text);
+    return '';
+};
+//# sourceMappingURL=prompt.js.map

package/dist/prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../src/prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAG5D,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEpC,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAA;AAE5B,MAAM,CAAC,MAAM,SAAS,GAA2B;IAC/C,EAAE,EAAE,QAAQ;IACZ,EAAE,EAAE,QAAQ;IACZ,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,QAAQ;IACZ,EAAE,EAAE,UAAU;IACd,EAAE,EAAE,YAAY;IAChB,EAAE,EAAE,SAAS;IACb,EAAE,EAAE,YAAY;IAChB,OAAO,EAAE,oBAAoB;IAC7B,OAAO,EAAE,qBAAqB;CAC/B,CAAA;AAED,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAA;AAExE,MAAM,YAAY,GAChB,6EAA6E;IAC7E,gDAAgD;IAChD,yEAAyE;IACzE,4EAA4E;IAC5E,yDAAyD;IACzD,yEAAyE;IACzE,8EAA8E;IAC9E,iFAAiF;IACjF,+EAA+E;IAC/E,kBAAkB;IAClB,sEAAsE;IACtE,2FAA2F;IAC3F,4EAA4E;IAC5E,uDAAuD;IACvD,0EAA0E;IAC1E,gFAAgF,CAAA;AAElF,MAAM,cAAc,GAAG,kBAAkB,CAAC,YAAY,CAAC;IACrD,CAAC,QAAQ,EAAE,YAAY,CAAC;IACxB,CAAC,MAAM,EAAE,YAAY,CAAC;CACvB,CAAC,CAAA;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAC3B,MAAM,GAAG,UAAU,EACnB,cAAc,EACd,cAAc,EACd,IAAI,GAAG,EAAE,EAMV,EAAqC,EAAE,CACtC,cAAc,CAAC,MAAM,CAAC;IACpB,MAAM;IACN,cAAc,EAAE,eAAe,CAAC,cAAc,CAAC;IAC/C,cAAc,EAAE,eAAe,CAAC,cAAc,CAAC;IAC/C,IAAI;CACL,CAAC,CAAA;AAEJ,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,IAAoB,EAAE,EAAE;IAC1D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IACzD,CAAC;IACD,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACf,OAAO,EAAE,CAAA;AACX,CAAC,CAAA"}

package/dist/split.d.ts

@@ -0,0 +1,27 @@
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import type { RecursiveCharacterTextSplitterParams, SupportedTextSplitterLanguage } from '@langchain/textsplitters';
+import { AsyncTransform } from './asyncTransform.js';
+export type TextSplitterFormat = {
+    format?: SupportedTextSplitterLanguage;
+};
+export type TextSplitterParams = Partial<RecursiveCharacterTextSplitterParams> & TextSplitterFormat;
+export type Separator = string | RegExp;
+export type TextSplitterLike = {
+    splitText(text: string): Promise<string[]>;
+};
+export declare const getFormatByExtension: (extname: string) => SupportedTextSplitterLanguage | undefined;
+export declare const recursiveChunkTextSplitter: (options?: TextSplitterParams) => RecursiveChunkTextSplitter;
+export declare class RecursiveChunkTextSplitter extends RecursiveCharacterTextSplitter {
+    protected joinDocs(docs: string[], separator: string): string | null;
+}
+export declare class TextSplitterStream extends AsyncTransform {
+    private readonly _textSplitter;
+    private _documents;
+    private _buffer;
+    constructor(options?: {
+        textSplitter?: TextSplitterLike;
+    });
+    protected _asyncTransform(chunk: Buffer | object, encoding: BufferEncoding | 'buffer'): Promise<boolean>;
+    _flush(done: (error?: Error | null) => void): void;
+}
+//# sourceMappingURL=split.d.ts.map

package/dist/split.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../src/split.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,8BAA8B,EAAE,MAAM,0BAA0B,CAAA;AACzE,OAAO,KAAK,EACV,oCAAoC,EACpC,6BAA6B,EAC9B,MAAM,0BAA0B,CAAA;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AAEpD,MAAM,MAAM,kBAAkB,GAAG;IAAE,MAAM,CAAC,EAAE,6BAA6B,CAAA;CAAE,CAAA;AAC3E,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,oCAAoC,CAAC,GAC5E,kBAAkB,CAAA;AACpB,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,CAAA;AACvC,MAAM,MAAM,gBAAgB,GAAG;IAAE,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CAAE,CAAA;AAsB7E,eAAO,MAAM,oBAAoB,GAC/B,SAAS,MAAM,KACd,6BAA6B,GAAG,SAUlC,CAAA;AAED,eAAO,MAAM,0BAA0B,GAAI,UAAU,kBAAkB,+BAWtE,CAAA;AAGD,qBAAa,0BAA2B,SAAQ,8BAA8B;cACzD,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;CAI9E;AAED,qBAAa,kBAAmB,SAAQ,cAAc;IACpD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkB;IAChD,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,OAAO,CAAQ;gBAEX,OAAO,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,gBAAgB,CAAA;KAAE;cAQzC,eAAe,CAC7B,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,QAAQ,EAAE,cAAc,GAAG,QAAQ;IAoBrC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI;CAO5C"}

package/dist/split.js

@@ -0,0 +1,87 @@
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import { AsyncTransform } from './asyncTransform.js';
+const extensionsByFormat = {
+    cpp: ['.h', '.c', '.cpp'],
+    go: ['.go'],
+    java: ['.java'],
+    js: ['.js', '.cjs', '.mjs', '.jsx', '.ts', '.tsx'],
+    php: ['.php', '.inc'],
+    proto: ['.proto'],
+    python: ['.py'],
+    rst: ['.rst'],
+    ruby: ['.rb', '.gem'],
+    rust: ['.rs'],
+    scala: ['.scala', '.sc'],
+    swift: ['.swift'],
+    markdown: ['.md', '.markdown', '.mkdn'],
+    latex: ['.tex', '.latex', '.ltx'],
+    html: ['.html', '.htm', '.xhtml', '.shtml', '.asp', '.aspx', '.jsp']
+};
+let formatByExtension;
+export const getFormatByExtension = (extname) => {
+    if (!formatByExtension) {
+        formatByExtension = {};
+        for (const [key, values] of Object.entries(extensionsByFormat)) {
+            for (const ext of values) {
+                formatByExtension[ext] = key;
+            }
+        }
+    }
+    return formatByExtension[extname];
+};
+export const recursiveChunkTextSplitter = (options) => {
+    const { format = 'markdown', ...rest } = options || {};
+    const separators = extensionsByFormat[format]
+        ? RecursiveCharacterTextSplitter.getSeparatorsForLanguage(format)
+        : ['\n\n', '\n', '.', ''];
+    return new RecursiveChunkTextSplitter({
+        chunkSize: 1000,
+        chunkOverlap: 0,
+        separators,
+        ...rest
+    });
+};
+// @ts-expect-error override joinDocs to preserve whitespace; upstream marks it private
+export class RecursiveChunkTextSplitter extends RecursiveCharacterTextSplitter {
+    joinDocs(docs, separator) {
+        const text = docs.join(separator);
+        return text === '' ? null : text;
+    }
+}
+export class TextSplitterStream extends AsyncTransform {
+    _textSplitter;
+    _documents;
+    _buffer;
+    constructor(options) {
+        const { textSplitter } = options || {};
+        super();
+        this._textSplitter = textSplitter || recursiveChunkTextSplitter();
+        this._documents = [];
+        this._buffer = '';
+    }
+    async _asyncTransform(chunk, encoding) {
+        const text = AsyncTransform.toString(chunk, encoding);
+        this._buffer += text;
+        const documents = await this._textSplitter.splitText(this._buffer);
+        this._buffer = documents.pop() ?? '';
+        this._documents = this._documents.concat(documents);
+        while (this._documents.length) {
+            const document = this._documents.shift();
+            if (document === undefined)
+                break;
+            const wait = !this.push(document);
+            if (wait) {
+                return true;
+            }
+        }
+        return false;
+    }
+    _flush(done) {
+        for (const document of this._documents) {
+            this.push(document);
+        }
+        this.push(this._buffer);
+        done();
+    }
+}
+//# sourceMappingURL=split.js.map