locale-ai 0.6.6 → 0.6.8

package/README.md

@@ -35,7 +35,7 @@ Download 577 MB? (Y/n)
 - **Incremental, and edit-aware**: only missing or changed keys are translated; untouched translations are kept, so re-runs are no-ops. A small `locale-ai.lock.json` (commit it, like a package lock) remembers a hash of each source string and of the output last written for it per target locale — so editing a source string gets it re-translated automatically (it isn't enough for the target to already hold *some* text), while editing a target string by hand is detected and reported for review instead of being silently overwritten on the next run. `--overwrite` bypasses the lock and re-translates everything.
 - **Structure**: key order, numbers, booleans, arrays, indentation and trailing newline are preserved.
 
-**Format notes**: for `.po`, `msgid` is the stable key (never changed) and `msgstr` is the translated value, as gettext expects — the English `.po` file's `msgstr` must hold the source text (most generators already fill it; an empty `msgstr` is treated as empty, not inherited from `msgid`). `msgid_plural`/`msgstr[n]` plural entries are translated too (the singular and plural source forms), then expanded to however many `msgstr[n]` the target language's plural rules need (6 for Arabic, 1 for Japanese, ...) — not full grammatical inflection for in-between categories (`few`/`many`/...), which get the plural-sourced translation verbatim, the same simplification `addPluralForms` makes for i18next. `Plural-Forms` in the header is computed for the target language, not copied from the source; an existing target file's header is otherwise left as you last edited it (a hand-fixed `Plural-Forms`, `Last-Translator`, ...) instead of being regenerated on every rerun. For `.arb`, `@key` metadata and `@@locale` are never translated; `@@locale` is set to the target locale on output.
+**Format notes**: for `.po`, `msgid` is the stable key (never changed) and `msgstr` is the translated value, as gettext expects — the English `.po` file's `msgstr` must hold the source text (most generators already fill it; an empty `msgstr` is treated as empty, not inherited from `msgid`). `msgid_plural`/`msgstr[n]` plural entries are translated too (the singular and plural source forms), then expanded to however many `msgstr[n]` the target language's plural rules need (6 for Arabic, 1 for Japanese, ...) — not full grammatical inflection for in-between categories (`few`/`many`/...), which get the plural-sourced translation verbatim, the same simplification `addPluralForms` makes for i18next. `Plural-Forms` in the header is computed for the target language, not copied from the source; an existing target file's header is otherwise left as you last edited it (a hand-fixed `Plural-Forms`, `Last-Translator`, ...) instead of being regenerated on every rerun. For `.arb`, `@key` metadata and `@@locale` are never translated; `@@locale` is set to the target locale on output. For `.strings`, a file saved as UTF-16 (Xcode's traditional default) is read like a UTF-8 one, an unquoted key (`title = "Settings";`, without quotes around `title`) is read the same as a quoted one, and `\U00e9`-style escapes are decoded — output is always written as UTF-8 with quoted keys, regardless of how the source was saved.
 
 ## Why locale-ai?
 
@@ -51,6 +51,7 @@ Download 577 MB? (Y/n)
 ```bash
 locale-ai translate --to fr,pt-BR [--from en] [--input dir] [--dry-run] [--overwrite] [--fast] [--report file] [-y]
 locale-ai check [--from en] [--input dir]   # CI: fails if a translated locale has missing keys or placeholder mismatches
+                                             # (also warns, without failing, about strings left identical to the source)
 locale-ai models list [--from en]     # supported language pairs
 locale-ai models installed
 locale-ai models install en-de        # pre-download (e.g. in CI)
@@ -62,6 +63,8 @@ locale-ai models convert <hf-marian-model> --out <dir>   # bring your own model
 
 Models are cached once per machine in `~/.cache/locale-ai` (`LOCALE_AI_CACHE` to change). In CI, pass `--yes`.
 
+Installing trims the `onnxruntime-node` dependency down to the one CPU binary locale-ai actually loads (it ships binaries for every platform/arch plus GPU providers locale-ai never uses) — but only when nothing else in your `node_modules` also depends on it, so a shared copy another package needs for GPU inference is left alone. Set `LOCALE_AI_SKIP_TRIM=1` to disable this entirely.
+
 ## Project config (optional)
 
 `locale-ai.config.json` in the project root:

package/dist/cli.js

@@ -10,9 +10,9 @@ import { translateStrings } from "./engine.js";
 import { CONFIG_FILE, glossaryLookup, loadConfig, protectRegex } from "./glossary.js";
 import { intlCode, parseLocale } from "./lang.js";
 import { editedPaths, isSourceStale, LOCK_FILENAME, loadLock, recordEntries, saveLock } from "./lock.js";
-import { addPluralForms, buildTarget, detectLayout, existingTargets, flattenStrings, keyPathProtect, pendingStrings, pluralOneStrings, readLocaleDoc, sourceFiles, targetFile, } from "./locales.js";
+import { addPluralForms, buildTarget, detectLayout, existingTargets, flattenStrings, keyPathProtect, pendingStrings, pluralOneStrings, readLocaleDoc, sourceFiles, targetFile, usesI18nextPlurals, } from "./locales.js";
 import { cacheRoot, install, isInstalled, loadTranslator, localModelProblem, planInstall, readManifest, uninstall, } from "./models.js";
-import { mask } from "./placeholders.js";
+import { hasTranslatableText, mask } from "./placeholders.js";
 /** Read from package.json so `npm version` is the single source of truth. */
 const VERSION = JSON.parse(fs.readFileSync(fileURLToPath(new URL("../package.json", import.meta.url)), "utf8")).version;
 const HELP = `locale-ai ${VERSION} — translate i18n locale files locally, on CPU, for free.
@@ -258,16 +258,16 @@ async function cmdTranslate(argv) {
                 }
             }
             const built = buildTarget(f.doc.data, f.existing, (s) => translations.get(s) ?? s, values.overwrite, isProtectedKey, isStale);
-            const { doc, added } = addPluralForms(built, intlCode(job.locale));
-            pluralsAdded += added;
+            const plural = usesI18nextPlurals(f.source) ? addPluralForms(built, intlCode(job.locale)) : { doc: built, added: 0 };
+            pluralsAdded += plural.added;
             const before = fs.existsSync(f.target) ? fs.readFileSync(f.target, "utf8") : undefined;
-            const text = f.doc.serialize(doc, job.locale.tag, before);
+            const text = f.doc.serialize(plural.doc, job.locale.tag, before);
             if (text !== before) {
                 fs.mkdirSync(path.dirname(f.target), { recursive: true });
                 fs.writeFileSync(f.target, text);
                 written++;
             }
-            const outFlat = new Map(flattenStrings(doc).map((e) => [e.path, e.value]));
+            const outFlat = new Map(flattenStrings(plural.doc).map((e) => [e.path, e.value]));
             recordEntries(lock, lockKey, job.locale.tag, flattenStrings(f.doc.data).map(({ path: p, value }) => ({ path: p, source: value, written: outFlat.get(p) ?? "" })));
         }
         const secs = ((Date.now() - started) / 1000).toFixed(1);
@@ -424,9 +424,11 @@ async function cmdCheck(argv) {
         return 0;
     }
     let problems = 0;
+    let identicalTotal = 0;
     for (const to of targets) {
         let missing = 0;
         let mismatched = 0;
+        let identical = 0;
         for (const { file, doc } of sources) {
             const target = targetFile(layout, from.tag, to, file);
             if (!fs.existsSync(target)) {
@@ -443,19 +445,27 @@ async function cmdCheck(argv) {
                     log(`✗ ${to}: ${rel(target)} missing key "${p}"`);
                     continue;
                 }
-                const srcTokens = mask(srcVal, "natural").tokens.toSorted();
+                const srcMasked = mask(srcVal, "natural");
                 const tgtTokens = mask(tgtVal, "natural").tokens.toSorted();
-                if (srcTokens.join("\u0001") !== tgtTokens.join("\u0001")) {
+                if (srcMasked.tokens.toSorted().join("\u0001") !== tgtTokens.join("\u0001")) {
                     mismatched++;
-                    log(`✗ ${to}: ${rel(target)} "${p}" placeholders differ: source [${srcTokens.join(", ")}], target [${tgtTokens.join(", ")}]`);
+                    log(`✗ ${to}: ${rel(target)} "${p}" placeholders differ: source [${srcMasked.tokens.toSorted().join(", ")}], target [${tgtTokens.join(", ")}]`);
+                    continue;
+                }
+                if (tgtVal === srcVal && hasTranslatableText(srcMasked.text)) {
+                    identical++;
+                    log(`⚠ ${to}: ${rel(target)} "${p}" is untouched, identical to the ${from.tag} source`);
                 }
             }
         }
+        identicalTotal += identical;
         if (missing || mismatched)
             problems++;
         else
-            log(`✓ ${to}`);
+            log(`✓ ${to}${identical ? ` (${identical} left identical to ${from.tag})` : ""}`);
     }
+    if (identicalTotal)
+        log(`\n${identicalTotal} string${identicalTotal === 1 ? "" : "s"} left identical to the source — not a failure, but worth a look.`);
     return problems ? 1 : 0;
 }
 async function main() {

package/dist/locales.js

@@ -1,6 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
+import { parseLocale } from "./lang.js";
 const EXTENSIONS = [".json", ".yaml", ".yml", ".po", ".arb", ".strings"];
 const CANDIDATES = [
     "locales",
@@ -82,6 +83,16 @@ export function targetFile(layout, from, to, src) {
         return path.join(layout.root, `${to}${layout.ext}`);
     return path.join(layout.root, to, path.relative(path.join(layout.root, from), src));
 }
+/** True when `tag` parses as a locale (e.g. "fr", "pt-BR") — rejects names like "__mocks__" or "schemas". */
+function looksLikeLocale(tag) {
+    try {
+        parseLocale(tag);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 /** Locale tags already translated alongside `from` in this layout (for `locale-ai check`). */
 export function existingTargets(layout, from) {
     if (layout.kind === "file") {
@@ -90,12 +101,14 @@ export function existingTargets(layout, from) {
             .readdirSync(layout.root)
             .filter((f) => f.endsWith(ext) && f.slice(0, -ext.length) !== from)
             .map((f) => f.slice(0, -ext.length))
+            .filter(looksLikeLocale)
             .sort();
     }
     return fs
         .readdirSync(layout.root, { withFileTypes: true })
         .filter((e) => e.isDirectory() && !e.name.startsWith(".") && e.name !== "node_modules" && e.name !== from)
         .map((e) => e.name)
+        .filter(looksLikeLocale)
         .filter((name) => listLocaleFiles(path.join(layout.root, name)).length > 0)
         .sort();
 }
@@ -138,19 +151,28 @@ function readYamlDoc(file, text, tag) {
 }
 /** Apple .strings: `"key" = "value";`, optionally preceded by a `/* ... *\/` comment. */
 function unescapeC(s) {
-    return s.replace(/\\(.)/g, (_, c) => ({ n: "\n", t: "\t", r: "\r", '"': '"', "\\": "\\" })[c] ?? c);
+    return s.replace(/\\(?:U([0-9a-fA-F]{4})|(.))/g, (_, hex, c) => {
+        // \Uxxxx is a UTF-16 code unit; two in a row combine into one character via
+        // ordinary JS string concatenation when they're a surrogate pair.
+        if (hex !== undefined)
+            return String.fromCharCode(parseInt(hex, 16));
+        return ({ n: "\n", t: "\t", r: "\r", '"': '"', "\\": "\\" })[c] ?? c;
+    });
 }
 function escapeC(s) {
     return s.replace(/[\\"\n\t\r]/g, (c) => ({ "\\": "\\\\", '"': '\\"', "\n": "\\n", "\t": "\\t", "\r": "\\r" })[c]);
 }
 function readStringsDoc(file, text) {
     // A key's preceding comment is either a /* ... */ block, or one or more consecutive // lines.
-    const PAIR = /(?:((?:\/\*[\s\S]*?\*\/)|(?:\/\/[^\r\n]*(?:\r?\n[ \t]*\/\/[^\r\n]*)*))[ \t]*\r?\n+)?[ \t]*"((?:\\.|[^"\\])*)"\s*=\s*"((?:\\.|[^"\\])*)"\s*;/g;
+    // The old NeXT/Apple plist-string format allows a bare, unquoted key (no spaces,
+    // "=" or ";") as well as a quoted one — Xcode itself only ever writes quoted keys,
+    // but hand-edited or third-party-exported .strings files sometimes have both.
+    const PAIR = /(?:((?:\/\*[\s\S]*?\*\/)|(?:\/\/[^\r\n]*(?:\r?\n[ \t]*\/\/[^\r\n]*)*))[ \t]*\r?\n+)?[ \t]*(?:"((?:\\.|[^"\\])*)"|([^\s"=;]+))\s*=\s*"((?:\\.|[^"\\])*)"\s*;/g;
     const data = newObj();
     const entries = [];
     for (const m of text.matchAll(PAIR)) {
-        const key = unescapeC(m[2]);
-        data[key] = unescapeC(m[3]);
+        const key = m[2] !== undefined ? unescapeC(m[2]) : m[3];
+        data[key] = unescapeC(m[4]);
         entries.push({ key, comment: m[1] });
     }
     if (entries.length === 0 && text.trim() !== "") {
@@ -418,9 +440,28 @@ function readPoDoc(file, text) {
     };
     return { data, serialize };
 }
+/**
+ * Decodes a locale file's raw bytes to text, detecting a UTF-16 BOM first — Xcode's
+ * traditional default for `.strings` files, which `fs.readFileSync(file, "utf8")`
+ * would otherwise mangle into unreadable text (and then "no entries found").
+ */
+function readText(file) {
+    const buf = fs.readFileSync(file);
+    if (buf.length >= 2 && buf[0] === 0xff && buf[1] === 0xfe)
+        return buf.subarray(2).toString("utf16le");
+    if (buf.length >= 2 && buf[0] === 0xfe && buf[1] === 0xff) {
+        const swapped = Buffer.alloc(buf.length - 2);
+        for (let i = 2; i + 1 < buf.length; i += 2) {
+            swapped[i - 2] = buf[i + 1];
+            swapped[i - 1] = buf[i];
+        }
+        return swapped.toString("utf16le");
+    }
+    return buf.toString("utf8").replace(/^/, "");
+}
 /** `tag` is the locale this file is for (the source tag, or an existing target's own tag). */
 export function readLocaleDoc(file, tag) {
-    const text = fs.readFileSync(file, "utf8").replace(/^/, "");
+    const text = readText(file);
     const ext = path.extname(file).toLowerCase();
     if (ext === ".yaml" || ext === ".yml")
         return readYamlDoc(file, text, tag);
@@ -538,6 +579,15 @@ export function pluralOneStrings(data, out = new Set()) {
     visit(data);
     return out;
 }
+/**
+ * True for the file extensions that use i18next's `key_one`/`key_other` plural-key
+ * convention (flat JSON/YAML locale trees). A `.po`, `.arb` or `.strings` file might
+ * happen to have a key ending the same way without meaning it as an i18next plural,
+ * so addPluralForms should only ever run on these.
+ */
+export function usesI18nextPlurals(file) {
+    return /\.(json|ya?ml)$/i.test(file);
+}
 /**
  * i18next stores plurals as `key_one` / `key_other`. Languages such as Arabic,
  * Russian or Polish need more forms (`_few`, `_many` ...) and i18next does not fall

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locale-ai",
-  "version": "0.6.6",
+  "version": "0.6.8",
   "description": "Translate your i18n locale files locally on CPU. Free, offline, no API keys.",
   "author": "Taha KHABOUSS",
   "repository": {

package/scripts/trim-onnxruntime.cjs

@@ -9,6 +9,15 @@
 // onnxruntime-node has finished installing, shrinking a ~550 MB unpacked
 // dependency down to the one binary locale-ai actually loads.
 //
+// Node's resolution can hoist onnxruntime-node to a node_modules shared with
+// other packages, so this only trims when nothing else next to it in that
+// same node_modules also depends on it — another consumer there might need a
+// GPU provider, or run on a different platform than this machine (e.g. a
+// node_modules folder built here and later mounted into a container). Set
+// LOCALE_AI_SKIP_TRIM=1 to disable this entirely, e.g. if node_modules is
+// going to be reused somewhere this check can't see (a different machine, a
+// Docker image built from it later).
+//
 // Best-effort: never fails the install, even if onnxruntime-node changes
 // its layout in a future version.
 
@@ -23,9 +32,44 @@ function rm(p) {
   }
 }
 
+/** Any OTHER package in `nodeModulesDir` that itself depends on onnxruntime-node. */
+function hasOtherConsumer(nodeModulesDir) {
+  let entries;
+  try {
+    entries = fs.readdirSync(nodeModulesDir, { withFileTypes: true });
+  } catch {
+    return false;
+  }
+  for (const e of entries) {
+    if (!e.isDirectory() || e.name === 'onnxruntime-node' || e.name === '.bin') continue;
+    const names = e.name.startsWith('@')
+      ? fs.readdirSync(path.join(nodeModulesDir, e.name), { withFileTypes: true })
+          .filter((s) => s.isDirectory())
+          .map((s) => `${e.name}/${s.name}`)
+      : [e.name];
+    for (const name of names) {
+      if (name === 'locale-ai') continue;
+      try {
+        const pkg = JSON.parse(fs.readFileSync(path.join(nodeModulesDir, name, 'package.json'), 'utf8'));
+        const deps = { ...pkg.dependencies, ...pkg.optionalDependencies, ...pkg.peerDependencies };
+        if (deps && 'onnxruntime-node' in deps) return true;
+      } catch {
+        // Not a real package (or unreadable) — ignore.
+      }
+    }
+  }
+  return false;
+}
+
+if (process.env.LOCALE_AI_SKIP_TRIM) process.exit(0);
+
 try {
   const pkgJson = require.resolve('onnxruntime-node/package.json');
-  const binRoot = path.join(path.dirname(pkgJson), 'bin', 'napi-v6');
+  const onnxRoot = path.dirname(pkgJson);
+  const nodeModulesDir = path.dirname(onnxRoot);
+  if (hasOtherConsumer(nodeModulesDir)) process.exit(0);
+
+  const binRoot = path.join(onnxRoot, 'bin', 'napi-v6');
   if (!fs.existsSync(binRoot)) process.exit(0);
 
   const platform = process.platform;