@llm-translate/cli 1.0.0-next.2 → 1.0.0-next.4

package/Dockerfile

@@ -38,9 +38,13 @@ COPY --from=builder --chown=llmtranslate:nodejs /app/node_modules ./node_modules
 COPY --from=builder --chown=llmtranslate:nodejs /app/dist ./dist
 COPY --from=builder --chown=llmtranslate:nodejs /app/package.json ./
 
+# Create cache directory with correct ownership
+RUN mkdir -p /app/cache && chown llmtranslate:nodejs /app/cache
+
 # Environment
 ENV NODE_ENV=production
 ENV TRANSLATE_PORT=3000
+ENV TRANSLATE_CACHE_DIR=/app/cache
 
 # Switch to non-root user
 USER llmtranslate
@@ -51,5 +55,6 @@ EXPOSE 3000
 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
   CMD node -e "fetch('http://localhost:3000/health/live').then(r => r.ok ? process.exit(0) : process.exit(1)).catch(() => process.exit(1))"
 
-# Start server with JSON logging for container environments
-CMD ["node", "dist/cli/index.js", "serve", "--json", "--cors", "--no-auth", "--cache-dir", "./.translate-cache/server"]
+# ENTRYPOINT for CLI, CMD for default arguments
+ENTRYPOINT ["node", "dist/cli/index.js"]
+CMD ["serve", "--json", "--cors", "--no-auth", "--cache-dir", "/app/cache"]

package/dist/cli/index.js

@@ -8,6 +8,7 @@ import 'remark-parse';
 import 'remark-stringify';
 import 'remark-gfm';
 import 'unist-util-visit';
+import * as cheerio from 'cheerio';
 import { createAnthropic } from '@ai-sdk/anthropic';
 import { generateText, streamText } from 'ai';
 import { createOpenAI } from '@ai-sdk/openai';
@@ -1789,6 +1790,243 @@ var init_markdown = __esm({
   "src/parsers/markdown.ts"() {
   }
 });
+function parseHTML(content) {
+  const $ = cheerio.load(content, {
+    decodeEntities: false,
+    xmlMode: false
+  });
+  const isFullDocument = content.includes("<html") || content.includes("<body");
+  const sections = extractTranslatableSections($, isFullDocument);
+  return {
+    original: content,
+    $,
+    sections,
+    isFullDocument
+  };
+}
+function extractTranslatableSections($, isFullDocument) {
+  const sections = [];
+  let sectionId = 0;
+  const root = isFullDocument ? $("body") : $.root();
+  function processElement(element, parentSelector) {
+    let textNodeIndex = 0;
+    element.contents().each((_index, node) => {
+      if (node.type === "text") {
+        const textNode = node;
+        const text = textNode.data;
+        const currentTextIndex = textNodeIndex++;
+        if (!text || !text.trim()) return;
+        const parent = $(node).parent();
+        const tagName = parent[0]?.tagName?.toLowerCase() || "unknown";
+        if (SKIP_TAGS.has(tagName)) return;
+        const selector = buildSelector($, parent, parentSelector, currentTextIndex);
+        sections.push({
+          id: `section-${sectionId++}`,
+          content: text,
+          selector,
+          tagName,
+          tokenCount: estimateTokens(text),
+          translatable: true
+        });
+      } else if (node.type === "tag") {
+        const elem = node;
+        const tagName = elem.tagName?.toLowerCase();
+        if (SKIP_TAGS.has(tagName)) return;
+        const $elem = $(elem);
+        const selector = buildSelector($, $elem, parentSelector);
+        for (const attrName of TRANSLATABLE_ATTRIBUTES) {
+          const attrValue = $elem.attr(attrName);
+          if (attrValue && attrValue.trim()) {
+            sections.push({
+              id: `section-${sectionId++}`,
+              content: attrValue,
+              selector,
+              tagName,
+              isAttribute: true,
+              attributeName: attrName,
+              tokenCount: estimateTokens(attrValue),
+              translatable: true
+            });
+          }
+        }
+        processElement($elem, selector);
+      }
+    });
+  }
+  processElement(root, "");
+  return sections;
+}
+function buildSelector($, element, parentSelector, textIndex) {
+  const elem = element[0];
+  if (!elem || elem.type !== "tag") {
+    return parentSelector + (textIndex !== void 0 ? `::text(${textIndex})` : "");
+  }
+  const tagElem = elem;
+  const tagName = tagElem.tagName?.toLowerCase() || "unknown";
+  const id = $(elem).attr("id");
+  if (textIndex !== void 0) {
+    if (id) {
+      return `#${id}::text(${textIndex})`;
+    }
+    return parentSelector ? `${parentSelector}::text(${textIndex})` : `${tagName}::text(${textIndex})`;
+  }
+  if (id) {
+    return `#${id}`;
+  }
+  const parent = $(elem).parent();
+  const siblings = parent.children(tagName);
+  const index = siblings.index(elem);
+  let selector = tagName;
+  if (siblings.length > 1) {
+    selector += `:nth-of-type(${index + 1})`;
+  }
+  if (parentSelector) {
+    selector = `${parentSelector} > ${selector}`;
+  }
+  return selector;
+}
+function chunkHTMLSections(sections, options = {}) {
+  const config2 = { ...DEFAULT_HTML_CHUNKING, ...options };
+  const chunks = [];
+  const translatableSections = sections.filter((s) => s.translatable);
+  if (translatableSections.length === 0) {
+    return [];
+  }
+  let currentChunk = [];
+  let currentTokens = 0;
+  let chunkId = 0;
+  for (const section of translatableSections) {
+    const sectionTokens = section.tokenCount;
+    if (currentTokens + sectionTokens > config2.maxTokens && currentChunk.length > 0) {
+      chunks.push(createChunk(currentChunk, chunkId++));
+      currentChunk = [];
+      currentTokens = 0;
+    }
+    currentChunk.push(section);
+    currentTokens += sectionTokens;
+  }
+  if (currentChunk.length > 0) {
+    chunks.push(createChunk(currentChunk, chunkId));
+  }
+  return chunks;
+}
+function createChunk(sections, id) {
+  const lines = [];
+  for (const section of sections) {
+    if (section.isAttribute) {
+      lines.push(`[${section.id}:${section.attributeName}] ${section.content}`);
+    } else {
+      lines.push(`[${section.id}] ${section.content}`);
+    }
+  }
+  const content = lines.join("\n\n");
+  return {
+    id: `chunk-${id}`,
+    content,
+    sections,
+    tokenCount: estimateTokens(content)
+  };
+}
+function parseTranslatedChunk(chunk, translatedContent) {
+  const map = {};
+  const regex = /\[([^\]]+)\]\s*([^[]*?)(?=\n\n\[|\n*$)/gs;
+  let match;
+  while ((match = regex.exec(translatedContent)) !== null) {
+    const marker = match[1];
+    let translation = match[2]?.trim() || "";
+    const idMatch = marker?.match(/^(section-\d+)/);
+    if (idMatch) {
+      map[idMatch[1]] = translation;
+    }
+  }
+  if (Object.keys(map).length === 0 && chunk.sections.length === 1) {
+    map[chunk.sections[0].id] = translatedContent.trim();
+  }
+  return map;
+}
+function applyHTMLTranslations(document, translations) {
+  const $ = document.$;
+  for (const section of document.sections) {
+    const translation = translations[section.id];
+    if (!translation) continue;
+    try {
+      if (section.isAttribute && section.attributeName) {
+        const elem = $(section.selector.replace(/::text\(\d+\)$/, ""));
+        elem.attr(section.attributeName, translation);
+      } else {
+        const textMatch = section.selector.match(/^(.*)::text\((\d+)\)$/);
+        if (textMatch) {
+          const [, parentSelector, textIndexStr] = textMatch;
+          const textIndex = parseInt(textIndexStr, 10);
+          const parent = $(parentSelector);
+          let currentTextIndex = 0;
+          parent.contents().each((_i, node) => {
+            if (node.type === "text") {
+              if (currentTextIndex === textIndex) {
+                node.data = translation;
+                return false;
+              }
+              currentTextIndex++;
+            }
+          });
+        } else {
+          const elem = $(section.selector);
+          const contents = elem.contents();
+          let updated = false;
+          contents.each((_i, node) => {
+            if (node.type === "text" && !updated) {
+              node.data = translation;
+              updated = true;
+            }
+          });
+        }
+      }
+    } catch (error) {
+      console.warn(`Failed to apply translation for ${section.id}:`, error);
+    }
+  }
+  if (document.isFullDocument) {
+    return $.html();
+  } else {
+    return $("body").html() || $.html();
+  }
+}
+function getHTMLStats(document) {
+  const translatableSections = document.sections.filter((s) => s.translatable);
+  const attributeSections = document.sections.filter((s) => s.isAttribute);
+  const totalTokens = document.sections.reduce((sum, s) => sum + s.tokenCount, 0);
+  return {
+    totalSections: document.sections.length,
+    translatableSections: translatableSections.length,
+    attributeSections: attributeSections.length,
+    totalTokens,
+    avgTokensPerSection: document.sections.length > 0 ? Math.round(totalTokens / document.sections.length) : 0
+  };
+}
+var SKIP_TAGS, TRANSLATABLE_ATTRIBUTES, DEFAULT_HTML_CHUNKING;
+var init_html = __esm({
+  "src/parsers/html.ts"() {
+    init_tokens();
+    SKIP_TAGS = /* @__PURE__ */ new Set([
+      "script",
+      "style",
+      "code",
+      "pre",
+      "kbd",
+      "samp",
+      "var",
+      "noscript",
+      "template",
+      "svg",
+      "math"
+    ]);
+    TRANSLATABLE_ATTRIBUTES = ["alt", "title", "placeholder", "aria-label"];
+    DEFAULT_HTML_CHUNKING = {
+      maxTokens: 2048,
+      minTokensForChunk: 100
+    };
+  }
+});
 function mapFinishReason(reason) {
   switch (reason) {
     case "stop":
@@ -2949,6 +3187,7 @@ var init_engine = __esm({
     init_agent();
     init_chunker();
     init_markdown();
+    init_html();
     init_glossary();
     init_registry();
     init_logger();
@@ -3037,7 +3276,7 @@ var init_engine = __esm({
             result = await this.translateMarkdown(options, glossary);
             break;
           case "html":
-            result = await this.translatePlainText(options, glossary);
+            result = await this.translateHTML(options, glossary);
             break;
           case "text":
           default:
@@ -3176,6 +3415,157 @@ var init_engine = __esm({
           }
         };
       }
+      async translateHTML(options, glossary) {
+        const document = parseHTML(options.content);
+        if (this.verbose) {
+          const stats = getHTMLStats(document);
+          logger.info(`Parsed HTML: ${stats.translatableSections} translatable sections, ${stats.totalTokens} tokens`);
+        }
+        if (document.sections.length === 0) {
+          return {
+            content: options.content,
+            chunks: [],
+            metadata: {
+              totalTokensUsed: 0,
+              totalDuration: 0,
+              averageQuality: 100,
+              provider: this.provider.name,
+              model: this.config.provider.model ?? this.provider.defaultModel,
+              totalIterations: 0,
+              tokensUsed: { input: 0, output: 0 },
+              cache: { hits: 0, misses: 0 }
+            }
+          };
+        }
+        const chunks = chunkHTMLSections(document.sections, {
+          maxTokens: this.config.chunking.maxTokens
+        });
+        if (this.verbose) {
+          logger.info(`Chunked into ${chunks.length} translation units`);
+        }
+        const agent = createTranslationAgent({
+          provider: this.provider,
+          qualityThreshold: options.qualityThreshold ?? this.config.quality.threshold,
+          maxIterations: options.maxIterations ?? this.config.quality.maxIterations,
+          verbose: this.verbose,
+          strictQuality: options.strictQuality
+        });
+        const allTranslations = {};
+        const chunkResults = [];
+        let totalInputTokens = 0;
+        let totalOutputTokens = 0;
+        let totalIterations = 0;
+        for (let i = 0; i < chunks.length; i++) {
+          const chunk = chunks[i];
+          if (!chunk) continue;
+          if (this.verbose) {
+            logger.info(`Translating HTML chunk ${i + 1}/${chunks.length} (${chunk.sections.length} sections)...`);
+          }
+          const glossaryString = glossary ? JSON.stringify(glossary.terms.map((t) => ({ s: t.source, t: t.target }))) : void 0;
+          const cacheKey = {
+            content: chunk.content,
+            sourceLang: options.sourceLang,
+            targetLang: options.targetLang,
+            glossary: glossaryString,
+            provider: this.provider.name,
+            model: this.config.provider.model ?? this.provider.defaultModel
+          };
+          const cacheResult = this.cache.get(cacheKey);
+          if (cacheResult.hit && cacheResult.entry) {
+            this.cacheHits++;
+            if (this.verbose) {
+              logger.info(`  \u21B3 Cache hit (quality: ${cacheResult.entry.qualityScore})`);
+            }
+            const chunkTranslations = parseTranslatedChunk(chunk, cacheResult.entry.translation);
+            Object.assign(allTranslations, chunkTranslations);
+            chunkResults.push({
+              original: chunk.content,
+              translated: cacheResult.entry.translation,
+              startOffset: 0,
+              endOffset: chunk.content.length,
+              qualityScore: cacheResult.entry.qualityScore,
+              iterations: 0,
+              tokensUsed: { input: 0, output: 0, cacheRead: 1 },
+              cached: true
+            });
+            continue;
+          }
+          this.cacheMisses++;
+          const resolvedStyleInstruction = options.styleInstruction ?? this.config.languages.styles?.[options.targetLang];
+          const request = {
+            content: chunk.content,
+            sourceLang: options.sourceLang,
+            targetLang: options.targetLang,
+            format: "html",
+            glossary,
+            context: {
+              documentPurpose: options.context,
+              styleInstruction: resolvedStyleInstruction,
+              documentSummary: "HTML document with structured sections. Preserve the [section-N] markers exactly as they appear. Translate only the text after each marker."
+            }
+          };
+          try {
+            const result = await agent.translate(request);
+            const chunkTranslations = parseTranslatedChunk(chunk, result.content);
+            Object.assign(allTranslations, chunkTranslations);
+            this.cache.set(cacheKey, result.content, result.metadata.qualityScore);
+            chunkResults.push({
+              original: chunk.content,
+              translated: result.content,
+              startOffset: 0,
+              endOffset: chunk.content.length,
+              qualityScore: result.metadata.qualityScore,
+              iterations: result.metadata.iterations,
+              tokensUsed: result.metadata.tokensUsed
+            });
+            if (result.metadata.tokensUsed) {
+              totalInputTokens += result.metadata.tokensUsed.input;
+              totalOutputTokens += result.metadata.tokensUsed.output;
+            }
+            totalIterations += result.metadata.iterations;
+          } catch (error) {
+            logger.error(`Failed to translate HTML chunk ${i + 1}: ${error}`);
+            for (const section of chunk.sections) {
+              allTranslations[section.id] = section.content;
+            }
+            chunkResults.push({
+              original: chunk.content,
+              translated: chunk.content,
+              startOffset: 0,
+              endOffset: chunk.content.length,
+              qualityScore: 0,
+              iterations: 0,
+              tokensUsed: { input: 0, output: 0 }
+            });
+          }
+        }
+        const finalContent = applyHTMLTranslations(document, allTranslations);
+        const qualityScores = chunkResults.filter((r) => r.qualityScore > 0).map((r) => r.qualityScore);
+        const averageQuality = qualityScores.length > 0 ? qualityScores.reduce((a, b) => a + b, 0) / qualityScores.length : 0;
+        const cacheHits = chunkResults.filter((r) => r.cached).length;
+        const cacheMisses = chunkResults.filter((r) => !r.cached && r.qualityScore > 0).length;
+        return {
+          content: finalContent,
+          chunks: chunkResults,
+          metadata: {
+            totalTokensUsed: totalInputTokens + totalOutputTokens,
+            totalDuration: 0,
+            // Will be set by caller
+            averageQuality,
+            provider: this.provider.name,
+            model: this.config.provider.model ?? this.provider.defaultModel,
+            totalIterations,
+            tokensUsed: {
+              input: totalInputTokens,
+              output: totalOutputTokens
+            },
+            cache: {
+              hits: cacheHits,
+              misses: cacheMisses
+            }
+          }
+        };
+      }
       async translatePlainText(options, glossary) {
         const chunks = chunkContent(options.content, {
           maxTokens: this.config.chunking.maxTokens,