@thor123141245r/ai-translate 0.0.0 → 1.0.0

package/src/cli.ts

@@ -1,313 +0,0 @@
-import fs from 'node:fs'
-import path from 'node:path'
-import { pipeline } from 'node:stream/promises'
-import { initProcLog } from 'debug-level'
-import { AppConfig, StringSchema, v } from '@commenthol/app-config'
-import type { SupportedTextSplitterLanguage } from '@langchain/textsplitters'
-import { logger } from './logger.js'
-import { modelFactory, AiTranslateTransform } from './model.js'
-import {
-  TextSplitterStream,
-  recursiveChunkTextSplitter,
-  getFormatByExtension
-} from './split.js'
-
-const log = logger('cli')
-
-const help: Record<string, string> = {}
-
-help._ = `
-AI Translator
-
-Usage:
-  ai-translate [flags]
-  ai-translate [command] [flags]
-
-Commands:
-  set                 set config value
-
-Flags:
-  -h, --help            Help for ai-translate
-  -v, --version         Show version information
-  -c, --config DIR      Use config file .ai-translate.json in DIR
-  -f, --from LANG       Source language
-  -t, --to LANG         Target language; LANG is English language name or
-                        supported BCP47 codes (ar, de, en, es, fr, ja, pt, ru,
-                        vi, zh-CN, zh-TW)
-  -i, --input FILE      input file
-  -o, --output FILE     output file
-      --format FORMAT   specify input format (cpp, go, java, js, php, proto,
-                        python, rst, ruby, rust, scala, swift, markdown, latex,
-                        html, sol)
-Examples:
-  Translate input.md from Spanish to output.md in English
-    ai-translate -f Spanish -t English -i input.md -o output.md
-
-  Pipe from stdin to stdout using the config in the local folder
-    echo "translate" | ai-translate -f en -t en -c .
-
-Use "ai-translate [command] --help" for more information about a command.
-`
-
-help.set = `
-Set ai-translate configuration
-
-Writes config to \`.ai-translate.json\`
-If --config flag is omitted then global config is used.
-
-Usage:
-  ai-translate [flags] set KEY VALUE
-
-Flags:
-  -c, --config DIR    Use config file .ai-translate.json in DIR
-
-Available KEYs:
-  provider      set provider (ollama, mistral, anthropic, openai, deepseek);
-                default="ollama"
-  model         set model from provider; default="qwen2.5:7b"
-  apiKey        set api key
-  baseUrl       baseUrl for model
-  temperature   model temperature; default=0.1
-  maxRetries    max. number of retries; default=10
-  chunkSize     number of chunks used in text-splitter; default=1000
-`
-
-const PACKAGE_JSON = '../package.json'
-const APP = 'ai-translate'
-const CONF_FILE = `.${APP}.json`
-const DEFAULT_LANG = 'en'
-
-const commands: Record<string, (cmd: Record<string, unknown>, argv: string[]) => void> = {
-  set: (c, argv) => {
-    const key = nextArg(argv)
-    const value = nextArg(argv)
-    c.set = [key, value]
-  }
-}
-
-const flags: Record<string, (opts: Record<string, unknown>, argv: string[]) => void> = {
-  '--help': (f) => (f.help = true),
-  '--version': (f) => (f.version = true),
-  '--config': (f, argv) => {
-    const dir = nextArg(argv)
-    if (dir) f.config = path.resolve(process.cwd(), dir)
-  },
-  '--from': (f, argv) => {
-    const lang = nextArg(argv)
-    f.sourceLanguage = lang || DEFAULT_LANG
-  },
-  '--to': (f, argv) => {
-    const lang = nextArg(argv)
-    f.targetLanguage = lang || DEFAULT_LANG
-  },
-  '--input': (f, argv) => {
-    const filename = nextArg(argv)
-    if (filename) f.input = path.resolve(process.cwd(), filename)
-  },
-  '--output': (f, argv) => {
-    const filename = nextArg(argv)
-    if (filename) f.output = path.resolve(process.cwd(), filename)
-  },
-  '--format': (f, argv) => {
-    const format = nextArg(argv)
-    if (format) f.format = format
-  }
-}
-flags['-h'] = flags['--help']
-flags['-v'] = flags['--version']
-flags['-c'] = flags['--config']
-flags['-f'] = flags['--from']
-flags['-t'] = flags['--to']
-flags['-i'] = flags['--input']
-flags['-o'] = flags['--output']
-
-export const argvParse = (args?: string[]) => {
-  initProcLog()
-  const argv = args || process.argv.slice(2)
-  log.debug(argv)
-  const opts: Record<string, unknown> = {
-    sourceLanguage: 'en',
-    targetLanguage: 'en'
-  }
-  const cmd: Record<string, unknown> = {}
-
-  while (argv.length) {
-    const arg = argv.shift()
-    if (!arg) continue
-    if (commands[arg]) {
-      commands[arg](cmd, argv)
-    } else if (flags[arg]) {
-      flags[arg](opts, argv)
-    }
-  }
-  return { cmd, opts }
-}
-
-const nextArg = (argv: string[]) => {
-  const next = argv[0] || ''
-  if (next.indexOf('-') === 0) {
-    return
-  }
-  return argv.shift()
-}
-
-let _console = console
-export const _injectConsole = (obj: typeof console) => {
-  _console = obj
-}
-
-const displayError = (msg: string) => {
-  _console.error(`ERROR: ${msg}`)
-}
-
-const display = (msg: string) => _console.log(msg)
-
-const version = () => {
-  const packageJson = new URL(PACKAGE_JSON, import.meta.url)
-  const { version: pkgVersion } = JSON.parse(
-    fs.readFileSync(packageJson, 'utf-8')
-  ) as { version: string }
-  display(pkgVersion)
-}
-
-const schema = {
-  provider: StringSchema,
-  model: StringSchema,
-  apiKey: StringSchema,
-  baseUrl: StringSchema,
-  temperature: v.pipe(
-    v.string(),
-    v.transform(Number),
-    v.minValue(0),
-    v.maxValue(2)
-  ),
-  maxRetries: v.pipe(
-    v.string(),
-    v.transform(Number),
-    v.integer(),
-    v.minValue(0),
-    v.maxValue(99)
-  ),
-  chunkSize: v.pipe(
-    v.string(),
-    v.transform(Number),
-    v.integer(),
-    v.minValue(100),
-    v.maxValue(200e3)
-  )
-}
-
-const PROVIDER_API_KEY_ENV: Record<string, string> = {
-  openai: 'OPENAI_API_KEY',
-  anthropic: 'ANTHROPIC_API_KEY',
-  mistral: 'MISTRAL_API_KEY',
-  deepseek: 'DEEPSEEK_API_KEY'
-}
-
-const PROVIDER_BASE_URL_ENV: Record<string, string> = {
-  openai: 'OPENAI_BASE_URL',
-  anthropic: 'ANTHROPIC_BASE_URL',
-  mistral: 'MISTRAL_BASE_URL',
-  deepseek: 'DEEPSEEK_BASE_URL',
-  ollama: 'OLLAMA_BASE_URL'
-}
-
-const readEnv = (key?: string) => {
-  if (!key) return
-  const value = process.env[key]
-  if (!value) return
-  const trimmed = value.trim()
-  return trimmed ? trimmed : undefined
-}
-
-const pickFirst = (...values: Array<string | undefined>) =>
-  values.find((value) => value && value.length > 0)
-
-const resolveRuntimeConfig = (config: Record<string, unknown>) => {
-  const provider = String(config.provider || 'ollama')
-  const envApiKey = pickFirst(
-    readEnv('AI_TRANSLATE_API_KEY'),
-    readEnv(PROVIDER_API_KEY_ENV[provider])
-  )
-  const envBaseUrl = pickFirst(
-    readEnv('AI_TRANSLATE_BASE_URL'),
-    readEnv(PROVIDER_BASE_URL_ENV[provider])
-  )
-
-  return {
-    ...config,
-    ...(envApiKey ? { apiKey: envApiKey } : {}),
-    ...(envBaseUrl ? { baseUrl: envBaseUrl } : {})
-  }
-}
-
-export const cli = async (args?: string[]) => {
-  const { cmd, opts } = argvParse(args)
-  const command = Object.keys(cmd)[0]
-  log.debug({ cmd, opts })
-
-  const filename = !opts.config
-    ? CONF_FILE
-    : fs.lstatSync(String(opts.config)).isDirectory()
-      ? path.resolve(String(opts.config), CONF_FILE)
-      : String(opts.config)
-
-  const appConf = new AppConfig({ appName: APP, schema, filename })
-  await appConf.read().catch(() => null)
-
-  try {
-    if (opts.version) {
-      version()
-      return
-    }
-    if (opts.help) {
-      const helpText = help[command || ''] || help._
-      display(helpText)
-      return
-    }
-    if (cmd.set) {
-      const [key, value] = cmd.set as [string | undefined, string | undefined]
-      if (key) {
-        appConf.set(key, value)
-        await appConf.write()
-      } else {
-        display(JSON.stringify(appConf.config, null, 2))
-      }
-      return
-    }
-
-    const sourceLanguage = String(opts.sourceLanguage || DEFAULT_LANG)
-    const targetLanguage = String(opts.targetLanguage || DEFAULT_LANG)
-
-    const reader = opts.input ? fs.createReadStream(String(opts.input)) : process.stdin
-    const writer = opts.output ? fs.createWriteStream(String(opts.output)) : process.stdout
-
-    const runtimeConfig = resolveRuntimeConfig(
-      appConf.config as Record<string, unknown>
-    )
-    const model = modelFactory(runtimeConfig)
-    const lcNamespace =
-      (model as unknown as { lc_namespace?: Record<string, unknown> }).lc_namespace || {}
-    const lcKwargs = (model as unknown as { lc_kwargs?: Record<string, unknown> }).lc_kwargs || {}
-    const { apiKey: _apiKey, ...modelParams } = { ...lcNamespace, ...lcKwargs } as Record<string, unknown>
-    log.debug(modelParams)
-
-    const format =
-      (opts.format as SupportedTextSplitterLanguage | undefined) ||
-      getFormatByExtension(path.extname(String(opts.input || '_.md')))
-    const chunkSize = (runtimeConfig as { chunkSize?: number }).chunkSize || 1000
-    const textSplitter = recursiveChunkTextSplitter({ chunkSize, format })
-    const splitter = new TextSplitterStream({ textSplitter })
-    const translator = new AiTranslateTransform({
-      ...(opts as Record<string, unknown>),
-      sourceLanguage,
-      targetLanguage,
-      model
-    })
-
-    await pipeline(reader, splitter, translator, writer)
-  } catch (err) {
-    log.debug(err)
-    displayError((err as Error).message)
-  }
-}

package/src/index.ts

@@ -1,9 +0,0 @@
-export { AiTranslateTransform, modelFactory } from './model.js'
-export { promptInvoke, languages } from './prompt.js'
-export {
-  TextSplitterStream,
-  recursiveChunkTextSplitter,
-  getFormatByExtension
-} from './split.js'
-export type { Metadata, ModelFactoryOptions } from './model.js'
-export type { TextSplitterParams, Separator } from './split.js'

package/src/logger.ts

@@ -1,3 +0,0 @@
-import { ProcLog } from 'debug-level'
-
-export const logger = (namespace: string) => new ProcLog(`ai-translate:${namespace}`)

package/src/model.ts

@@ -1,139 +0,0 @@
-import { ChatOllama } from '@langchain/ollama'
-import { ChatMistralAI } from '@langchain/mistralai'
-import { ChatAnthropic } from '@langchain/anthropic'
-import { ChatOpenAI } from '@langchain/openai'
-import { ChatDeepSeek } from '@langchain/deepseek'
-import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
-import { AsyncTransform } from './asyncTransform.js'
-import { promptInvoke, replaceMarkerSymbol } from './prompt.js'
-import { logger } from './logger.js'
-import { isWhiteSpace, preserveWhiteSpace } from './utils.js'
-
-const log = logger('model')
-
-const DEFAULT = {
-  temperature: 0.1,
-  maxRetries: 10,
-  maxConcurrency: 1
-}
-
-export type Metadata = {
-  inputTokens: number
-  outputTokens: number
-}
-
-export type ModelFactoryOptions = {
-  provider?: 'ollama' | 'mistral' | 'anthropic' | 'openai' | 'deepseek'
-  [key: string]: unknown
-}
-
-export const modelFactory = (modelOpts?: ModelFactoryOptions): BaseChatModel => {
-  const { provider = 'ollama', ...other } = modelOpts || {}
-  const apiKeyValue = typeof other.apiKey === 'string' ? other.apiKey.trim() : ''
-  const baseUrlValue = typeof other.baseUrl === 'string' ? other.baseUrl.trim() : ''
-  const apiKey = apiKeyValue ? apiKeyValue : undefined
-  const baseUrl = baseUrlValue ? baseUrlValue : undefined
-  const filtered = { ...other }
-  if ('apiKey' in filtered) {
-    delete filtered.apiKey
-  }
-  if ('baseUrl' in filtered) {
-    delete filtered.baseUrl
-  }
-
-  switch (provider) {
-    case 'ollama':
-      return new ChatOllama({
-        ...DEFAULT,
-        model: 'qwen2.5:7b',
-        ...(baseUrl ? { baseUrl } : {}),
-        ...filtered
-      })
-    case 'mistral':
-      return new ChatMistralAI({
-        ...DEFAULT,
-        model: 'ministral-8b',
-        ...(apiKey ? { apiKey } : {}),
-        ...filtered
-      })
-    case 'anthropic':
-      return new ChatAnthropic({
-        ...DEFAULT,
-        model: 'claude-3-5-haiku-20241022',
-        ...(apiKey ? { apiKey } : {}),
-        ...filtered
-      })
-    case 'openai':
-      return new ChatOpenAI({
-        ...DEFAULT,
-        model: 'gpt-4o-mini',
-        ...(apiKey ? { apiKey } : {}),
-        ...(baseUrl ? { configuration: { baseURL: baseUrl } } : {}),
-        ...filtered
-      })
-    case 'deepseek':
-      return new ChatDeepSeek({
-        ...DEFAULT,
-        model: 'deepseek-reasoner',
-        ...(apiKey ? { apiKey } : {}),
-        ...(baseUrl ? { configuration: { baseURL: baseUrl } } : {}),
-        ...filtered
-      })
-    default:
-      throw new Error(`unsupported provider=${provider}`)
-  }
-}
-
-export class AiTranslateTransform extends AsyncTransform {
-  private readonly _model: BaseChatModel
-  private readonly _promptOpts: {
-    format?: string
-    sourceLanguage: string
-    targetLanguage: string
-  }
-  private _metadata: Metadata = { inputTokens: 0, outputTokens: 0 }
-
-  constructor(options: {
-    model: BaseChatModel
-    format?: string
-    sourceLanguage: string
-    targetLanguage: string
-    [key: string]: unknown
-  }) {
-    const { model, format, sourceLanguage, targetLanguage, ...rest } = options
-    super(rest)
-    this._model = model
-    this._promptOpts = { format, sourceLanguage, targetLanguage }
-  }
-
-  getMetadata() {
-    return { ...this._metadata }
-  }
-
-  protected async _asyncTransform(
-    chunk: Buffer | object,
-    encoding: BufferEncoding | 'buffer'
-  ) {
-    const text = AsyncTransform.toString(chunk, encoding)
-    if (isWhiteSpace(text)) {
-      return !this.push(text)
-    }
-    log.debug('inp=%j', text)
-    const messages = await promptInvoke({ ...this._promptOpts, text })
-    const result = await this._model.invoke(messages)
-    log.debug('out=%j', result.content)
-
-    const usage = (result as { usage_metadata?: { input_tokens?: number; output_tokens?: number } })
-      .usage_metadata
-    const inputTokens = usage?.input_tokens ?? 0
-    const outputTokens = usage?.output_tokens ?? 0
-
-    this.emit('metadata', { inputTokens, outputTokens })
-    this._metadata.inputTokens += inputTokens
-    this._metadata.outputTokens += outputTokens
-
-    return !this.push(
-      preserveWhiteSpace(text, replaceMarkerSymbol(result.content))
-    )
-  }
-}

package/src/prompt.ts

@@ -1,71 +0,0 @@
-import { ChatPromptTemplate } from '@langchain/core/prompts'
-import type { ChatPromptValueInterface } from '@langchain/core/prompt_values'
-import type { MessageContent } from '@langchain/core/messages'
-import { logger } from './logger.js'
-
-const log = logger('prompt')
-
-export const languages: Record<string, string> = {
-  ar: 'Arabic',
-  de: 'German',
-  en: 'English',
-  es: 'Spanish',
-  fr: 'French',
-  ja: 'Japanese',
-  pt: 'Portuguese',
-  ru: 'Russian',
-  vi: 'Vietnamese',
-  'zh-CN': 'Chinese-simplified',
-  'zh-TW': 'Chinese-traditional'
-}
-
-export const getLanguageName = (lang: string) => languages[lang] || lang
-
-const systemPrompt =
-  'You are an AI-driven advanced translation system, specifically designed to ' +
-  'translate structured and technical documents. ' +
-  'You will receive a text snippet from a file formatted as "{format}"\n\n' +
-  'Your task is to **accurately translate** the text enclosed between the 🔤 ' +
-  'symbols from "{sourceLanguage}" to "{targetLanguage}". ' +
-  'Preserve the original formatting, sentence structure, and terminology. ' +
-  'Ensure that every word and sentence is translated as closely as possible to ' +
-  'the original meaning, without summarizing or omitting any part of the content. ' +
-  'The translation must be faithful, detailed, and maintain the original length ' +
-  'and complexity. ' +
-  'Deliver the translation exactly as required, without any additional ' +
-  'commentary or explanation, and ensure the 🔤 symbols are removed in the final output.\n\n' +
-  'Remember: your job is to **translate** the text exactly as it is, without ' +
-  'adding summaries or changing the content in any way. ' +
-  'Do not skip or modify any part of the text. Ensure that the output is a ' +
-  'direct translation, and that the original structure and meaning are preserved.'
-
-const promptTemplate = ChatPromptTemplate.fromMessages([
-  ['system', systemPrompt],
-  ['user', '🔤{text}🔤']
-])
-
-export const promptInvoke = ({
-  format = 'markdown',
-  sourceLanguage,
-  targetLanguage,
-  text = ''
-}: {
-  format?: string
-  sourceLanguage: string
-  targetLanguage: string
-  text: string
-}): Promise<ChatPromptValueInterface> =>
-  promptTemplate.invoke({
-    format,
-    sourceLanguage: getLanguageName(sourceLanguage),
-    targetLanguage: getLanguageName(targetLanguage),
-    text
-  })
-
-export const replaceMarkerSymbol = (text: MessageContent) => {
-  if (typeof text === 'string') {
-    return text.replace(/^\s*🔤/, '').replace(/🔤\s*$/, '')
-  }
-  log.error(text)
-  return ''
-}

package/src/split.ts

@@ -1,111 +0,0 @@
-import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters'
-import type {
-  RecursiveCharacterTextSplitterParams,
-  SupportedTextSplitterLanguage
-} from '@langchain/textsplitters'
-import { AsyncTransform } from './asyncTransform.js'
-
-export type TextSplitterFormat = { format?: SupportedTextSplitterLanguage }
-export type TextSplitterParams = Partial<RecursiveCharacterTextSplitterParams> &
-  TextSplitterFormat
-export type Separator = string | RegExp
-export type TextSplitterLike = { splitText(text: string): Promise<string[]> }
-
-const extensionsByFormat: Record<string, string[]> = {
-  cpp: ['.h', '.c', '.cpp'],
-  go: ['.go'],
-  java: ['.java'],
-  js: ['.js', '.cjs', '.mjs', '.jsx', '.ts', '.tsx'],
-  php: ['.php', '.inc'],
-  proto: ['.proto'],
-  python: ['.py'],
-  rst: ['.rst'],
-  ruby: ['.rb', '.gem'],
-  rust: ['.rs'],
-  scala: ['.scala', '.sc'],
-  swift: ['.swift'],
-  markdown: ['.md', '.markdown', '.mkdn'],
-  latex: ['.tex', '.latex', '.ltx'],
-  html: ['.html', '.htm', '.xhtml', '.shtml', '.asp', '.aspx', '.jsp']
-}
-
-let formatByExtension: Record<string, SupportedTextSplitterLanguage> | undefined
-
-export const getFormatByExtension = (
-  extname: string
-): SupportedTextSplitterLanguage | undefined => {
-  if (!formatByExtension) {
-    formatByExtension = {}
-    for (const [key, values] of Object.entries(extensionsByFormat)) {
-      for (const ext of values) {
-        formatByExtension[ext] = key as SupportedTextSplitterLanguage
-      }
-    }
-  }
-  return formatByExtension[extname]
-}
-
-export const recursiveChunkTextSplitter = (options?: TextSplitterParams) => {
-  const { format = 'markdown', ...rest } = options || {}
-  const separators = extensionsByFormat[format]
-    ? RecursiveCharacterTextSplitter.getSeparatorsForLanguage(format)
-    : ['\n\n', '\n', '.', '']
-  return new RecursiveChunkTextSplitter({
-    chunkSize: 1000,
-    chunkOverlap: 0,
-    separators,
-    ...rest
-  })
-}
-
-// @ts-expect-error override joinDocs to preserve whitespace; upstream marks it private
-export class RecursiveChunkTextSplitter extends RecursiveCharacterTextSplitter {
-  protected override joinDocs(docs: string[], separator: string): string | null {
-    const text = docs.join(separator)
-    return text === '' ? null : text
-  }
-}
-
-export class TextSplitterStream extends AsyncTransform {
-  private readonly _textSplitter: TextSplitterLike
-  private _documents: string[]
-  private _buffer: string
-
-  constructor(options?: { textSplitter?: TextSplitterLike }) {
-    const { textSplitter } = options || {}
-    super()
-    this._textSplitter = textSplitter || recursiveChunkTextSplitter()
-    this._documents = []
-    this._buffer = ''
-  }
-
-  protected async _asyncTransform(
-    chunk: Buffer | object,
-    encoding: BufferEncoding | 'buffer'
-  ) {
-    const text = AsyncTransform.toString(chunk, encoding)
-    this._buffer += text
-
-    const documents = await this._textSplitter.splitText(this._buffer)
-    this._buffer = documents.pop() ?? ''
-    this._documents = this._documents.concat(documents)
-
-    while (this._documents.length) {
-      const document = this._documents.shift()
-      if (document === undefined) break
-      const wait = !this.push(document)
-      if (wait) {
-        return true
-      }
-    }
-    return false
-  }
-
-  _flush(done: (error?: Error | null) => void) {
-    for (const document of this._documents) {
-      this.push(document)
-    }
-    this.push(this._buffer)
-    done()
-  }
-}

package/src/utils.ts

@@ -1,15 +0,0 @@
-export const isWhiteSpace = (text = '') => text.trim() === ''
-
-export const whiteSpace = (text = '') => {
-  const head = /^\s{0,200}/.exec(text)?.[0] || ''
-  let tail = ''
-  if (head.length !== text.length) {
-    tail = /\s{0,200}$/.exec(text)?.[0] || ''
-  }
-  return { head, tail }
-}
-
-export const preserveWhiteSpace = (inp: string, out: string) => {
-  const { head, tail } = whiteSpace(inp)
-  return head + out.trim() + tail
-}

package/tsconfig.json

@@ -1,19 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "NodeNext",
-    "moduleResolution": "NodeNext",
-    "rootDir": "src",
-    "outDir": "dist",
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "esModuleInterop": true,
-    "resolveJsonModule": false,
-    "strict": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true
-  },
-  "include": ["src/**/*.ts"],
-  "exclude": ["node_modules", "dist"]
-}