@llm-translate/cli 1.0.0-next.3 → 1.0.0-next.5

package/dist/cli/index.js

@@ -8,6 +8,7 @@ import 'remark-parse';
 import 'remark-stringify';
 import 'remark-gfm';
 import 'unist-util-visit';
+import * as cheerio from 'cheerio';
 import { createAnthropic } from '@ai-sdk/anthropic';
 import { generateText, streamText } from 'ai';
 import { createOpenAI } from '@ai-sdk/openai';
@@ -1789,6 +1790,243 @@ var init_markdown = __esm({
   "src/parsers/markdown.ts"() {
   }
 });
+function parseHTML(content) {
+  const $ = cheerio.load(content, {
+    decodeEntities: false,
+    xmlMode: false
+  });
+  const isFullDocument = content.includes("<html") || content.includes("<body");
+  const sections = extractTranslatableSections($, isFullDocument);
+  return {
+    original: content,
+    $,
+    sections,
+    isFullDocument
+  };
+}
+function extractTranslatableSections($, isFullDocument) {
+  const sections = [];
+  let sectionId = 0;
+  const root = isFullDocument ? $("body") : $.root();
+  function processElement(element, parentSelector) {
+    let textNodeIndex = 0;
+    element.contents().each((_index, node) => {
+      if (node.type === "text") {
+        const textNode = node;
+        const text = textNode.data;
+        const currentTextIndex = textNodeIndex++;
+        if (!text || !text.trim()) return;
+        const parent = $(node).parent();
+        const tagName = parent[0]?.tagName?.toLowerCase() || "unknown";
+        if (SKIP_TAGS.has(tagName)) return;
+        const selector = buildSelector($, parent, parentSelector, currentTextIndex);
+        sections.push({
+          id: `section-${sectionId++}`,
+          content: text,
+          selector,
+          tagName,
+          tokenCount: estimateTokens(text),
+          translatable: true
+        });
+      } else if (node.type === "tag") {
+        const elem = node;
+        const tagName = elem.tagName?.toLowerCase();
+        if (SKIP_TAGS.has(tagName)) return;
+        const $elem = $(elem);
+        const selector = buildSelector($, $elem, parentSelector);
+        for (const attrName of TRANSLATABLE_ATTRIBUTES) {
+          const attrValue = $elem.attr(attrName);
+          if (attrValue && attrValue.trim()) {
+            sections.push({
+              id: `section-${sectionId++}`,
+              content: attrValue,
+              selector,
+              tagName,
+              isAttribute: true,
+              attributeName: attrName,
+              tokenCount: estimateTokens(attrValue),
+              translatable: true
+            });
+          }
+        }
+        processElement($elem, selector);
+      }
+    });
+  }
+  processElement(root, "");
+  return sections;
+}
+function buildSelector($, element, parentSelector, textIndex) {
+  const elem = element[0];
+  if (!elem || elem.type !== "tag") {
+    return parentSelector + (textIndex !== void 0 ? `::text(${textIndex})` : "");
+  }
+  const tagElem = elem;
+  const tagName = tagElem.tagName?.toLowerCase() || "unknown";
+  const id = $(elem).attr("id");
+  if (textIndex !== void 0) {
+    if (id) {
+      return `#${id}::text(${textIndex})`;
+    }
+    return parentSelector ? `${parentSelector}::text(${textIndex})` : `${tagName}::text(${textIndex})`;
+  }
+  if (id) {
+    return `#${id}`;
+  }
+  const parent = $(elem).parent();
+  const siblings = parent.children(tagName);
+  const index = siblings.index(elem);
+  let selector = tagName;
+  if (siblings.length > 1) {
+    selector += `:nth-of-type(${index + 1})`;
+  }
+  if (parentSelector) {
+    selector = `${parentSelector} > ${selector}`;
+  }
+  return selector;
+}
+function chunkHTMLSections(sections, options = {}) {
+  const config2 = { ...DEFAULT_HTML_CHUNKING, ...options };
+  const chunks = [];
+  const translatableSections = sections.filter((s) => s.translatable);
+  if (translatableSections.length === 0) {
+    return [];
+  }
+  let currentChunk = [];
+  let currentTokens = 0;
+  let chunkId = 0;
+  for (const section of translatableSections) {
+    const sectionTokens = section.tokenCount;
+    if (currentTokens + sectionTokens > config2.maxTokens && currentChunk.length > 0) {
+      chunks.push(createChunk(currentChunk, chunkId++));
+      currentChunk = [];
+      currentTokens = 0;
+    }
+    currentChunk.push(section);
+    currentTokens += sectionTokens;
+  }
+  if (currentChunk.length > 0) {
+    chunks.push(createChunk(currentChunk, chunkId));
+  }
+  return chunks;
+}
+function createChunk(sections, id) {
+  const lines = [];
+  for (const section of sections) {
+    if (section.isAttribute) {
+      lines.push(`[${section.id}:${section.attributeName}] ${section.content}`);
+    } else {
+      lines.push(`[${section.id}] ${section.content}`);
+    }
+  }
+  const content = lines.join("\n\n");
+  return {
+    id: `chunk-${id}`,
+    content,
+    sections,
+    tokenCount: estimateTokens(content)
+  };
+}
+function parseTranslatedChunk(chunk, translatedContent) {
+  const map = {};
+  const regex = /\[([^\]]+)\]\s*([^[]*?)(?=\n\n\[|\n*$)/gs;
+  let match;
+  while ((match = regex.exec(translatedContent)) !== null) {
+    const marker = match[1];
+    let translation = match[2]?.trim() || "";
+    const idMatch = marker?.match(/^(section-\d+)/);
+    if (idMatch) {
+      map[idMatch[1]] = translation;
+    }
+  }
+  if (Object.keys(map).length === 0 && chunk.sections.length === 1) {
+    map[chunk.sections[0].id] = translatedContent.trim();
+  }
+  return map;
+}
+function applyHTMLTranslations(document, translations) {
+  const $ = document.$;
+  for (const section of document.sections) {
+    const translation = translations[section.id];
+    if (!translation) continue;
+    try {
+      if (section.isAttribute && section.attributeName) {
+        const elem = $(section.selector.replace(/::text\(\d+\)$/, ""));
+        elem.attr(section.attributeName, translation);
+      } else {
+        const textMatch = section.selector.match(/^(.*)::text\((\d+)\)$/);
+        if (textMatch) {
+          const [, parentSelector, textIndexStr] = textMatch;
+          const textIndex = parseInt(textIndexStr, 10);
+          const parent = $(parentSelector);
+          let currentTextIndex = 0;
+          parent.contents().each((_i, node) => {
+            if (node.type === "text") {
+              if (currentTextIndex === textIndex) {
+                node.data = translation;
+                return false;
+              }
+              currentTextIndex++;
+            }
+          });
+        } else {
+          const elem = $(section.selector);
+          const contents = elem.contents();
+          let updated = false;
+          contents.each((_i, node) => {
+            if (node.type === "text" && !updated) {
+              node.data = translation;
+              updated = true;
+            }
+          });
+        }
+      }
+    } catch (error) {
+      console.warn(`Failed to apply translation for ${section.id}:`, error);
+    }
+  }
+  if (document.isFullDocument) {
+    return $.html();
+  } else {
+    return $("body").html() || $.html();
+  }
+}
+function getHTMLStats(document) {
+  const translatableSections = document.sections.filter((s) => s.translatable);
+  const attributeSections = document.sections.filter((s) => s.isAttribute);
+  const totalTokens = document.sections.reduce((sum, s) => sum + s.tokenCount, 0);
+  return {
+    totalSections: document.sections.length,
+    translatableSections: translatableSections.length,
+    attributeSections: attributeSections.length,
+    totalTokens,
+    avgTokensPerSection: document.sections.length > 0 ? Math.round(totalTokens / document.sections.length) : 0
+  };
+}
+var SKIP_TAGS, TRANSLATABLE_ATTRIBUTES, DEFAULT_HTML_CHUNKING;
+var init_html = __esm({
+  "src/parsers/html.ts"() {
+    init_tokens();
+    SKIP_TAGS = /* @__PURE__ */ new Set([
+      "script",
+      "style",
+      "code",
+      "pre",
+      "kbd",
+      "samp",
+      "var",
+      "noscript",
+      "template",
+      "svg",
+      "math"
+    ]);
+    TRANSLATABLE_ATTRIBUTES = ["alt", "title", "placeholder", "aria-label"];
+    DEFAULT_HTML_CHUNKING = {
+      maxTokens: 2048,
+      minTokensForChunk: 100
+    };
+  }
+});
 function mapFinishReason(reason) {
   switch (reason) {
     case "stop":
@@ -2949,6 +3187,7 @@ var init_engine = __esm({
     init_agent();
     init_chunker();
     init_markdown();
+    init_html();
     init_glossary();
     init_registry();
     init_logger();
@@ -3037,7 +3276,7 @@ var init_engine = __esm({
             result = await this.translateMarkdown(options, glossary);
             break;
           case "html":
-            result = await this.translatePlainText(options, glossary);
+            result = await this.translateHTML(options, glossary);
             break;
           case "text":
           default:
@@ -3176,6 +3415,157 @@ var init_engine = __esm({
           }
         };
       }
+      async translateHTML(options, glossary) {
+        const document = parseHTML(options.content);
+        if (this.verbose) {
+          const stats = getHTMLStats(document);
+          logger.info(`Parsed HTML: ${stats.translatableSections} translatable sections, ${stats.totalTokens} tokens`);
+        }
+        if (document.sections.length === 0) {
+          return {
+            content: options.content,
+            chunks: [],
+            metadata: {
+              totalTokensUsed: 0,
+              totalDuration: 0,
+              averageQuality: 100,
+              provider: this.provider.name,
+              model: this.config.provider.model ?? this.provider.defaultModel,
+              totalIterations: 0,
+              tokensUsed: { input: 0, output: 0 },
+              cache: { hits: 0, misses: 0 }
+            }
+          };
+        }
+        const chunks = chunkHTMLSections(document.sections, {
+          maxTokens: this.config.chunking.maxTokens
+        });
+        if (this.verbose) {
+          logger.info(`Chunked into ${chunks.length} translation units`);
+        }
+        const agent = createTranslationAgent({
+          provider: this.provider,
+          qualityThreshold: options.qualityThreshold ?? this.config.quality.threshold,
+          maxIterations: options.maxIterations ?? this.config.quality.maxIterations,
+          verbose: this.verbose,
+          strictQuality: options.strictQuality
+        });
+        const allTranslations = {};
+        const chunkResults = [];
+        let totalInputTokens = 0;
+        let totalOutputTokens = 0;
+        let totalIterations = 0;
+        for (let i = 0; i < chunks.length; i++) {
+          const chunk = chunks[i];
+          if (!chunk) continue;
+          if (this.verbose) {
+            logger.info(`Translating HTML chunk ${i + 1}/${chunks.length} (${chunk.sections.length} sections)...`);
+          }
+          const glossaryString = glossary ? JSON.stringify(glossary.terms.map((t) => ({ s: t.source, t: t.target }))) : void 0;
+          const cacheKey = {
+            content: chunk.content,
+            sourceLang: options.sourceLang,
+            targetLang: options.targetLang,
+            glossary: glossaryString,
+            provider: this.provider.name,
+            model: this.config.provider.model ?? this.provider.defaultModel
+          };
+          const cacheResult = this.cache.get(cacheKey);
+          if (cacheResult.hit && cacheResult.entry) {
+            this.cacheHits++;
+            if (this.verbose) {
+              logger.info(`  \u21B3 Cache hit (quality: ${cacheResult.entry.qualityScore})`);
+            }
+            const chunkTranslations = parseTranslatedChunk(chunk, cacheResult.entry.translation);
+            Object.assign(allTranslations, chunkTranslations);
+            chunkResults.push({
+              original: chunk.content,
+              translated: cacheResult.entry.translation,
+              startOffset: 0,
+              endOffset: chunk.content.length,
+              qualityScore: cacheResult.entry.qualityScore,
+              iterations: 0,
+              tokensUsed: { input: 0, output: 0, cacheRead: 1 },
+              cached: true
+            });
+            continue;
+          }
+          this.cacheMisses++;
+          const resolvedStyleInstruction = options.styleInstruction ?? this.config.languages.styles?.[options.targetLang];
+          const request = {
+            content: chunk.content,
+            sourceLang: options.sourceLang,
+            targetLang: options.targetLang,
+            format: "html",
+            glossary,
+            context: {
+              documentPurpose: options.context,
+              styleInstruction: resolvedStyleInstruction,
+              documentSummary: "HTML document with structured sections. Preserve the [section-N] markers exactly as they appear. Translate only the text after each marker."
+            }
+          };
+          try {
+            const result = await agent.translate(request);
+            const chunkTranslations = parseTranslatedChunk(chunk, result.content);
+            Object.assign(allTranslations, chunkTranslations);
+            this.cache.set(cacheKey, result.content, result.metadata.qualityScore);
+            chunkResults.push({
+              original: chunk.content,
+              translated: result.content,
+              startOffset: 0,
+              endOffset: chunk.content.length,
+              qualityScore: result.metadata.qualityScore,
+              iterations: result.metadata.iterations,
+              tokensUsed: result.metadata.tokensUsed
+            });
+            if (result.metadata.tokensUsed) {
+              totalInputTokens += result.metadata.tokensUsed.input;
+              totalOutputTokens += result.metadata.tokensUsed.output;
+            }
+            totalIterations += result.metadata.iterations;
+          } catch (error) {
+            logger.error(`Failed to translate HTML chunk ${i + 1}: ${error}`);
+            for (const section of chunk.sections) {
+              allTranslations[section.id] = section.content;
+            }
+            chunkResults.push({
+              original: chunk.content,
+              translated: chunk.content,
+              startOffset: 0,
+              endOffset: chunk.content.length,
+              qualityScore: 0,
+              iterations: 0,
+              tokensUsed: { input: 0, output: 0 }
+            });
+          }
+        }
+        const finalContent = applyHTMLTranslations(document, allTranslations);
+        const qualityScores = chunkResults.filter((r) => r.qualityScore > 0).map((r) => r.qualityScore);
+        const averageQuality = qualityScores.length > 0 ? qualityScores.reduce((a, b) => a + b, 0) / qualityScores.length : 0;
+        const cacheHits = chunkResults.filter((r) => r.cached).length;
+        const cacheMisses = chunkResults.filter((r) => !r.cached && r.qualityScore > 0).length;
+        return {
+          content: finalContent,
+          chunks: chunkResults,
+          metadata: {
+            totalTokensUsed: totalInputTokens + totalOutputTokens,
+            totalDuration: 0,
+            // Will be set by caller
+            averageQuality,
+            provider: this.provider.name,
+            model: this.config.provider.model ?? this.provider.defaultModel,
+            totalIterations,
+            tokensUsed: {
+              input: totalInputTokens,
+              output: totalOutputTokens
+            },
+            cache: {
+              hits: cacheHits,
+              misses: cacheMisses
+            }
+          }
+        };
+      }
       async translatePlainText(options, glossary) {
         const chunks = chunkContent(options.content, {
           maxTokens: this.config.chunking.maxTokens,
@@ -4368,8 +4758,9 @@ function createApp(options) {
     json: options.jsonLogging ?? false
   }));
   if (options.enableCors) {
+    const corsOrigin = options.corsOrigins ?? "*";
     app.use("*", cors({
-      origin: "*",
+      origin: corsOrigin,
       allowMethods: ["GET", "POST", "OPTIONS"],
       allowHeaders: ["Content-Type", "Authorization", "X-API-Key"],
       exposeHeaders: ["X-Request-Id"],
@@ -4429,7 +4820,12 @@ llm-translate server started`);
   console.log(`  - Health: http://${options.host}:${options.port}/health`);
   console.log(`  - Translate: http://${options.host}:${options.port}/translate`);
   console.log(`  - Auth: ${options.enableAuth ? "enabled" : "disabled"}`);
-  console.log(`  - CORS: ${options.enableCors ? "enabled" : "disabled"}`);
+  if (options.enableCors) {
+    const corsInfo = options.corsOrigins ? Array.isArray(options.corsOrigins) ? options.corsOrigins.join(", ") : options.corsOrigins : "all origins";
+    console.log(`  - CORS: enabled (${corsInfo})`);
+  } else {
+    console.log(`  - CORS: disabled`);
+  }
   console.log(`  - Cache: ${options.cachePath ?? "disabled"}`);
   console.log("");
   const shutdown = (signal) => {
@@ -4458,7 +4854,7 @@ var serveCommand = new Command("serve").description("Start the translation API s
   "-p, --port <number>",
   "Server port (env: TRANSLATE_PORT)",
   process.env["TRANSLATE_PORT"] ?? "3000"
-).option("-H, --host <string>", "Host to bind", "0.0.0.0").option("--no-auth", "Disable API key authentication").option("--cors", "Enable CORS for browser clients").option("--json", "Use JSON logging format (for containers)").option(
+).option("-H, --host <string>", "Host to bind", "0.0.0.0").option("--no-auth", "Disable API key authentication").option("--cors [origins]", "Enable CORS (optionally specify allowed origins, comma-separated)").option("--json", "Use JSON logging format (for containers)").option(
   "--cache-dir <path>",
   "Cache directory path (env: TRANSLATE_CACHE_DIR)",
   process.env["TRANSLATE_CACHE_DIR"]
@@ -4478,11 +4874,17 @@ var serveCommand = new Command("serve").description("Start the translation API s
       "Set TRANSLATE_API_KEY environment variable to enable authentication.\n"
     );
   }
+  const enableCors = options.cors !== void 0 && options.cors !== false;
+  let corsOrigins;
+  if (typeof options.cors === "string") {
+    corsOrigins = options.cors.includes(",") ? options.cors.split(",").map((o) => o.trim()) : options.cors;
+  }
   startServer({
     port,
     host,
     enableAuth,
-    enableCors: options.cors ?? false,
+    enableCors,
+    corsOrigins,
     apiKey: process.env["TRANSLATE_API_KEY"],
     jsonLogging: options.json ?? false,
     cachePath: options.cacheDir