bctranslate 1.0.0 → 1.0.2

package/src/bridges/python.js

@@ -1,6 +1,7 @@
 import { spawn } from 'child_process';
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
+import { shieldInterpolations, unshieldInterpolations } from '../utils.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -8,24 +9,20 @@ const PYTHON_SCRIPT = join(__dirname, '..', '..', 'python', 'translator.py');
 
 let cachedPythonCmd = null;
 
-/**
- * Find the correct python command on this system.
- */
 async function findPython() {
   if (cachedPythonCmd) return cachedPythonCmd;
-
   for (const cmd of ['python3', 'python']) {
     try {
-      const result = await execSimple(cmd, ['--version']);
-      if (result.includes('Python 3')) {
+      const out = await execSimple(cmd, ['--version']);
+      if (out.includes('Python 3')) {
         cachedPythonCmd = cmd;
         return cmd;
       }
     } catch { /* try next */ }
   }
   throw new Error(
-    'Python 3 not found. Install Python 3.8+ and ensure it is on your PATH.\n' +
-    '  Also install argostranslate: pip install argostranslate'
+    'Python 3 not found. Install Python 3.8+ and add it to your PATH.\n' +
+    '  Then run: pip install argostranslate'
   );
 }
 
@@ -45,13 +42,21 @@ function execSimple(cmd, args) {
 }
 
 /**
- * Check that Python and argostranslate are available,
- * and that the required language pair is installed.
+ * Install argostranslate via pip.
+ * Called from `init` and from checkPythonBridge when the package is missing.
+ */
+export async function installArgostranslate() {
+  const py = await findPython();
+  return execSimple(py, ['-m', 'pip', 'install', '--quiet', 'argostranslate']);
+}
+
+/**
+ * Check Python + argostranslate availability and download the language pair
+ * model if not already installed.
  */
 export async function checkPythonBridge(from, to) {
   const py = await findPython();
 
-  // Check argostranslate is installed and language pair available
   const checkScript = `
 import sys, json
 try:
@@ -62,41 +67,34 @@ except ImportError:
     sys.exit(0)
 
 from_code = "${from}"
-to_code = "${to}"
+to_code   = "${to}"
 
 installed = argostranslate.translate.get_installed_languages()
-from_lang = None
-to_lang = None
-for lang in installed:
-    if lang.code == from_code:
-        from_lang = lang
-    if lang.code == to_code:
-        to_lang = lang
-
-if not from_lang or not to_lang:
-    available_codes = [l.code for l in installed]
-    # Try to auto-download
+from_lang = next((l for l in installed if l.code == from_code), None)
+to_lang   = next((l for l in installed if l.code == to_code),   None)
+
+if from_lang and to_lang and from_lang.get_translation(to_lang):
+    print(json.dumps({"status": "ready"}))
+else:
     try:
         argostranslate.package.update_package_index()
-        available_packages = argostranslate.package.get_available_packages()
-        pkg = next((p for p in available_packages if p.from_code == from_code and p.to_code == to_code), None)
+        available = argostranslate.package.get_available_packages()
+        pkg = next((p for p in available if p.from_code == from_code and p.to_code == to_code), None)
         if pkg:
             print(json.dumps({"status": "downloading", "pair": f"{from_code}->{to_code}"}))
             argostranslate.package.install_from_path(pkg.download())
             print(json.dumps({"status": "ready"}))
         else:
+            available_codes = [l.code for l in installed]
             print(json.dumps({"error": f"Language pair {from_code}->{to_code} not available. Installed: {available_codes}"}))
     except Exception as e:
-        print(json.dumps({"error": f"Failed to download language pair: {str(e)}. Installed: {available_codes}"}))
-else:
-    print(json.dumps({"status": "ready"}))
+        print(json.dumps({"error": f"Failed to download language pair: {str(e)}"}))
 `;
 
   const result = await execSimple(py, ['-c', checkScript]);
 
-  // Parse last JSON line (might have multiple from downloading)
-  const lines = result.split('\n').filter(l => l.trim());
-  for (const line of lines.reverse()) {
+  const lines = result.split('\n').filter((l) => l.trim());
+  for (const line of [...lines].reverse()) {
     try {
       const parsed = JSON.parse(line);
       if (parsed.error) throw new Error(parsed.error);
@@ -105,31 +103,38 @@ else:
       if (e.message && !e.message.includes('Unexpected')) throw e;
     }
   }
-
   return true;
 }
 
 /**
  * Translate a batch of strings using argostranslate via Python.
- * Sends all strings at once for efficiency (model loads once).
  *
- * @param {Array<{key: string, text: string}>} batch - Strings to translate
- * @param {string} from - Source language code
- * @param {string} to - Target language code
- * @returns {Promise<Object<string, string>>} Map of key -> translated text
+ * Interpolation variables like {{ name }}, {count}, ${val} are shielded
+ * before sending and restored after — Argos never sees them.
+ *
+ * @param {Array<{key: string, text: string}>} batch
+ * @param {string} from  Source language code
+ * @param {string} to    Target language code
+ * @returns {Promise<Record<string, string>>}  key → translated text
  */
 export async function translateBatch(batch, from, to) {
   if (batch.length === 0) return {};
 
   const py = await findPython();
 
+  // ── Shield interpolations so Argos never mangles {name} / {{ expr }} ────────
+  const shielded = batch.map((item) => {
+    const { shielded: text, tokens } = shieldInterpolations(item.text);
+    return { key: item.key, text, tokens };
+  });
+
   return new Promise((resolve, reject) => {
     const proc = spawn(py, [PYTHON_SCRIPT, from, to], {
       stdio: ['pipe', 'pipe', 'pipe'],
       env: { ...process.env, PYTHONIOENCODING: 'utf-8' },
     });
 
-    const input = JSON.stringify(batch);
+    const input = JSON.stringify(shielded.map(({ key, text }) => ({ key, text })));
     let stdout = '';
     let stderr = '';
 
@@ -142,34 +147,35 @@ export async function translateBatch(batch, from, to) {
       }
 
       try {
-        // Find the JSON output line (ignore any debug/warning output)
+        // Find the JSON output line (skip debug/warning lines)
         const lines = stdout.split('\n');
-        let result = null;
+        let raw = null;
         for (let i = lines.length - 1; i >= 0; i--) {
           const line = lines[i].trim();
-          if (line.startsWith('{') || line.startsWith('[')) {
-            try {
-              result = JSON.parse(line);
-              break;
-            } catch { /* try previous line */ }
+          if (line.startsWith('[') || line.startsWith('{')) {
+            try { raw = JSON.parse(line); break; } catch { /* try previous */ }
           }
         }
 
-        if (!result) {
+        if (!raw) {
           return reject(new Error(`No valid JSON in Python output: ${stdout.slice(0, 500)}`));
         }
 
-        // Convert array to map
-        const map = {};
-        if (Array.isArray(result)) {
-          for (const item of result) {
-            map[item.key] = item.text;
-          }
+        // Convert array → map, then unshield interpolations
+        const rawMap = {};
+        if (Array.isArray(raw)) {
+          for (const item of raw) rawMap[item.key] = item.text;
         } else {
-          Object.assign(map, result);
+          Object.assign(rawMap, raw);
         }
 
-        resolve(map);
+        const result = {};
+        for (const { key, tokens } of shielded) {
+          const translated = rawMap[key] ?? batch.find((b) => b.key === key)?.text ?? '';
+          result[key] = unshieldInterpolations(translated, tokens);
+        }
+
+        resolve(result);
       } catch (err) {
         reject(new Error(`Failed to parse Python output: ${err.message}\nOutput: ${stdout.slice(0, 500)}`));
       }
@@ -180,4 +186,4 @@ export async function translateBatch(batch, from, to) {
     proc.stdin.write(input);
     proc.stdin.end();
   });
-}
+}

package/src/index.js

@@ -8,148 +8,209 @@ import { parseJson, applyJsonTranslations } from './parsers/json.js';
 import { translateBatch } from './bridges/python.js';
 import { getLocaleDir, saveLocale, loadLocale } from './generators/locales.js';
 
-/**
- * Process a single file: extract strings, translate only untranslated ones,
- * write results. Smart: skips strings already present in the target locale.
- */
-export async function translateFile(filePath, opts) {
-  const { from, to, dryRun, outdir, project, cwd, verbose, jsonMode } = opts;
-  const ext = extname(filePath).toLowerCase();
-  const source = readFileSync(filePath, 'utf-8');
-  const relativePath = relative(cwd, filePath);
+// ── Route source file to the correct parser ──────────────────────────────────
 
-  // Route to appropriate parser
-  let result;
+function routeParser(ext, source, filePath, project, jsonMode) {
   switch (ext) {
-    case '.vue':
-      result = parseVue(source, filePath);
-      break;
+    case '.vue':  return parseVue(source, filePath);
     case '.jsx':
-    case '.tsx':
-      result = parseReact(source, filePath);
-      break;
+    case '.tsx':  return parseReact(source, filePath);
     case '.html':
-    case '.htm':
-      result = parseHtml(source, filePath, project);
-      break;
+    case '.htm':  return parseHtml(source, filePath, project);
     case '.js':
     case '.ts':
       if (source.includes('React') || source.includes('jsx') || /<\w+[\s>]/.test(source)) {
-        result = parseReact(source, filePath);
-      } else {
-        result = parseJs(source, filePath, project);
+        return parseReact(source, filePath);
       }
-      break;
-    case '.json':
-      result = parseJson(source, filePath, { jsonMode });
-      break;
-    default:
-      return { count: 0, skipped: 0, relativePath };
+      return parseJs(source, filePath, project);
+    case '.json': return parseJson(source, filePath, { jsonMode });
+    default:      return null;
   }
+}
+
+/**
+ * Parse a file — no translation, no disk writes.
+ * Returns extracted strings + modified source for later application.
+ */
+export function parseFileOnly(filePath, opts) {
+  const { project, jsonMode = 'values' } = opts;
+  const ext    = extname(filePath).toLowerCase();
+  const source = readFileSync(filePath, 'utf-8');
 
-  if (!result.extracted || result.extracted.length === 0) {
-    return { count: 0, skipped: 0, relativePath };
+  const result = routeParser(ext, source, filePath, project, jsonMode);
+  if (!result || !result.extracted?.length) {
+    return { source, modified: source, extracted: [], ext, filePath };
   }
+  return {
+    source,
+    modified:  result.source,
+    extracted: result.extracted,
+    ext,
+    filePath,
+    jsonData: result.jsonData,
+  };
+}
 
-  // Deduplicate by key (same text = same hash key)
-  const seen = new Set();
-  const uniqueBatch = result.extracted
-    .map((item) => ({ key: item.key, text: item.text }))
-    .filter((item) => {
-      if (seen.has(item.key)) return false;
-      seen.add(item.key);
-      return true;
-    });
+/**
+ * Apply a completed translations map to one parsed file and write to disk.
+ * The original file is NOT touched until an atomic rename succeeds.
+ */
+export async function writeFileResult(parseResult, translations, opts) {
+  const { source, modified, extracted, ext, filePath, jsonData } = parseResult;
+  const { from, to, dryRun, outdir, project, cwd, verbose, localesDir } = opts;
 
-  // ── Smart half-translation: load existing locale, skip already-done keys ──
-  const localeDir = opts.localesDir || getLocaleDir(cwd, project);
-  const existingTarget = loadLocale(localeDir, to);
+  const relativePath = relative(cwd, filePath);
+  if (!extracted?.length) return { count: 0, skipped: 0, relativePath };
 
-  const needsTranslation = uniqueBatch.filter((item) => !(item.key in existingTarget));
-  const skippedCount = uniqueBatch.length - needsTranslation.length;
+  // Deduplicate extracted items by key
+  const seen   = new Set();
+  const unique = extracted.filter(({ key }) => {
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
 
-  // Start with existing translations
-  let translations = { ...existingTarget };
+  const resolvedLocaleDir = localesDir || getLocaleDir(cwd, project);
+  const existingTarget    = loadLocale(resolvedLocaleDir, to);
 
-  if (needsTranslation.length > 0) {
-    try {
-      const newTranslations = await translateBatch(needsTranslation, from, to);
-      Object.assign(translations, newTranslations);
-    } catch (err) {
-      if (verbose) {
-        console.error(`    Translation error: ${err.message}`);
-      }
-      // Fallback: use original text
-      for (const item of needsTranslation) {
-        translations[item.key] = item.text;
-      }
-    }
-  }
+  const newCount = unique.filter(({ key }) => !(key in existingTarget)).length;
+  const skipped  = unique.length - newCount;
 
-  // Build locale entries for all extracted strings
+  // Build locale entry maps
   const fromEntries = {};
-  const toEntries = {};
-  for (const item of uniqueBatch) {
-    fromEntries[item.key] = item.text;
-    toEntries[item.key] = translations[item.key] || item.text;
+  const toEntries   = {};
+  for (const { key, text } of unique) {
+    fromEntries[key] = text;
+    toEntries[key]   = translations[key] ?? existingTarget[key] ?? text;
   }
 
   if (!dryRun) {
-    // Write locale files (merge with existing — won't overwrite manual edits)
-    saveLocale(localeDir, from, fromEntries);
-    saveLocale(localeDir, to, toEntries);
+    saveLocale(resolvedLocaleDir, from, fromEntries);
+    saveLocale(resolvedLocaleDir, to,   toEntries);
 
     if (ext === '.json') {
-      const translatedData = applyJsonTranslations(result.jsonData, translations);
+      const translatedData = applyJsonTranslations(jsonData, toEntries);
       const outName = basename(filePath, ext) + `.${to}${ext}`;
       const outPath = outdir
         ? join(outdir, outName)
         : join(dirname(filePath), outName);
-
       mkdirSync(dirname(outPath), { recursive: true });
       writeFileSync(outPath, JSON.stringify(translatedData, null, 2) + '\n', 'utf-8');
-    } else {
-      // Only rewrite source if it actually changed
-      if (result.source !== source) {
-        const outPath = outdir ? join(outdir, basename(filePath)) : filePath;
-        if (outdir) mkdirSync(outdir, { recursive: true });
-
-        const tmpPath = outPath + '.bctmp';
-        writeFileSync(tmpPath, result.source, 'utf-8');
-        const { renameSync } = await import('fs');
-        renameSync(tmpPath, outPath);
-      }
+
+    } else if (modified !== source) {
+      // Only rewrite source when content actually changed
+      // Atomic: write .bctmp first, then rename — original untouched until swap
+      const outPath  = outdir ? join(outdir, basename(filePath)) : filePath;
+      const tmpPath  = outPath + '.bctmp';
+      if (outdir) mkdirSync(outdir, { recursive: true });
+      writeFileSync(tmpPath, modified, 'utf-8');
+      const { renameSync } = await import('fs');
+      renameSync(tmpPath, outPath);
     }
   }
 
-  // count = newly translated strings; skipped = already had translations
   return {
-    count: needsTranslation.length,
-    skipped: skippedCount,
+    count:  newCount,
+    skipped,
     relativePath,
-    diff: dryRun ? generateDiff(source, result.source) : null,
+    diff: dryRun || verbose ? generateDiff(source, modified, relativePath) : null,
   };
 }
 
 /**
- * Simple line-diff preview for dry-run mode.
+ * Translate ALL files in one Python invocation — model loads exactly once.
+ *
+ *  Phase 1: Parse every file (pure CPU — no network/Python)
+ *  Phase 2: Collect all unique keys not already in the target locale
+ *  Phase 3: translateBatch() once → Python starts, model loads, all strings translated
+ *  Phase 4: Write each file
+ */
+export async function translateAllFiles(files, opts) {
+  const { from, to, cwd, project, verbose, localesDir } = opts;
+
+  const resolvedLocaleDir = localesDir || getLocaleDir(cwd, project);
+  const existingTarget    = loadLocale(resolvedLocaleDir, to);
+
+  // Phase 1 — parse
+  const parsed = [];
+  for (const file of files) {
+    try {
+      const result = parseFileOnly(file, opts);
+      if (result.extracted.length > 0) parsed.push(result);
+    } catch (err) {
+      if (verbose) console.error(`  Parse error ${file}: ${err.message}`);
+    }
+  }
+
+  if (parsed.length === 0) return [];
+
+  // Phase 2 — deduplicate across all files, skip already-translated keys
+  const seenKeys = new Set(Object.keys(existingTarget));
+  const needed   = [];
+  for (const { extracted } of parsed) {
+    for (const { key, text } of extracted) {
+      if (!seenKeys.has(key)) {
+        seenKeys.add(key);
+        needed.push({ key, text });
+      }
+    }
+  }
+
+  // Phase 3 — translate once (Python spawned exactly once, model loads once)
+  let newTranslations = {};
+  if (needed.length > 0) {
+    newTranslations = await translateBatch(needed, from, to);
+  }
+
+  const allTranslations = { ...existingTarget, ...newTranslations };
+
+  // Phase 4 — write files
+  const results = [];
+  for (const parseResult of parsed) {
+    try {
+      const r = await writeFileResult(parseResult, allTranslations, {
+        ...opts,
+        localesDir: resolvedLocaleDir,
+      });
+      results.push(r);
+    } catch (err) {
+      if (verbose) console.error(`  Write error ${parseResult.filePath}: ${err.message}`);
+      results.push({ count: 0, skipped: 0, relativePath: relative(cwd, parseResult.filePath) });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Single-file convenience wrapper (for backward compatibility).
  */
-function generateDiff(original, modified) {
-  if (original === modified) return '';
+export async function translateFile(filePath, opts) {
+  const results = await translateAllFiles([filePath], opts);
+  return results[0] ?? { count: 0, skipped: 0, relativePath: relative(opts.cwd, filePath) };
+}
+
+// ── Diff display ─────────────────────────────────────────────────────────────
+
+export function generateDiff(original, modified, label = '') {
+  if (!modified || original === modified) return '';
 
   const origLines = original.split('\n');
-  const modLines = modified.split('\n');
-  const diffs = [];
+  const modLines  = modified.split('\n');
+  const out       = label ? [`--- ${label} ---`] : [];
 
   const maxLines = Math.max(origLines.length, modLines.length);
   for (let i = 0; i < maxLines; i++) {
-    const orig = origLines[i] || '';
-    const mod = modLines[i] || '';
-    if (orig !== mod) {
-      diffs.push(`    ${i + 1}: - ${orig.trim()}`);
-      diffs.push(`    ${i + 1}: + ${mod.trim()}`);
+    const a = origLines[i] ?? '';
+    const b = modLines[i]  ?? '';
+    if (a !== b) {
+      out.push(`  ${String(i + 1).padStart(4)} - ${a.trimEnd()}`);
+      out.push(`  ${String(i + 1).padStart(4)} + ${b.trimEnd()}`);
     }
   }
 
-  return diffs.slice(0, 40).join('\n') + (diffs.length > 40 ? '\n    ... (truncated)' : '');
+  if (out.length > 80) {
+    return out.slice(0, 80).join('\n') + `\n  ... (${out.length - 80} more lines truncated)`;
+  }
+  return out.join('\n');
 }

package/src/parsers/html.js

@@ -1,6 +1,6 @@
 import MagicString from 'magic-string';
 import { parse, NodeTypes } from '@vue/compiler-dom';
-import { hashKey, isTranslatable } from '../utils.js';
+import { contextKey, isTranslatable } from '../utils.js';
 
 const ATTR_WHITELIST = new Set([
   'title',
@@ -82,7 +82,7 @@ export function parseHtml(source, filePath, project) {
       const marker = `data-i18n-${name}`;
       if (openTagText.includes(marker)) continue;
 
-      const key = hashKey(value);
+      const key = contextKey(value, filePath);
       s.appendRight(prop.loc.end.offset, ` ${marker}="${key}"`);
       extracted.push({ key, text: value, context: `html-attr-${name}` });
     }
@@ -98,7 +98,7 @@ export function parseHtml(source, filePath, project) {
 
         const marker = tag === 'title' ? 'data-i18n-title' : 'data-i18n';
         if (!openTagText.includes(marker) && isTranslatable(plain)) {
-          const key = hashKey(text);
+          const key = contextKey(text, filePath);
           s.appendLeft(openTagEnd, ` ${marker}="${key}"`);
           extracted.push({ key, text, context: `html-inner-${tag}` });
         }

package/src/parsers/js.js

@@ -2,7 +2,7 @@ import * as babelParser from '@babel/parser';
 import _traverse from '@babel/traverse';
 import * as t from '@babel/types';
 import MagicString from 'magic-string';
-import { hashKey, isTranslatable } from '../utils.js';
+import { contextKey, isTranslatable } from '../utils.js';
 
 const traverse = _traverse.default || _traverse;
 
@@ -55,7 +55,7 @@ export function parseJs(source, filePath, project) {
 
       if (!translatableKeys.has(keyName)) return;
 
-      const key = hashKey(valueNode.value);
+      const key = contextKey(valueNode.value, filePath);
       const tFunc = project.type === 'vue' ? `this.$t('${key}')` : `t('${key}')`;
 
       s.overwrite(valueNode.start, valueNode.end, tFunc);
@@ -80,7 +80,7 @@ export function parseJs(source, filePath, project) {
       if (['alert', 'confirm', 'prompt'].includes(calleeName)) {
         const arg = path.node.arguments[0];
         if (arg && arg.type === 'StringLiteral' && isTranslatable(arg.value)) {
-          const key = hashKey(arg.value);
+          const key = contextKey(arg.value, filePath);
           const tFunc = project.type === 'vue' ? `this.$t('${key}')` : `t('${key}')`;
           s.overwrite(arg.start, arg.end, tFunc);
           extracted.push({ key, text: arg.value, context: 'js-call' });
@@ -106,7 +106,7 @@ export function parseJs(source, filePath, project) {
         ]);
 
         if (domProps.has(propName)) {
-          const key = hashKey(right.value);
+          const key = contextKey(right.value, filePath);
           const tFunc = project.type === 'vanilla'
             ? `i18n.t('${key}')`
             : project.type === 'vue'

package/src/parsers/json.js

@@ -1,4 +1,4 @@
-import { hashKey, isTranslatable } from '../utils.js';
+import { isTranslatable } from '../utils.js';
 
 /**
  * Parse a JSON file and extract translatable string values.