locale-ai 0.6.2 → 0.6.3

package/README.md

@@ -15,7 +15,7 @@ npx locale-ai translate --from en --to fr,es,de
 ```
 
 ```
-locale-ai 0.2.0
+locale-ai 0.6.3
 Found 3 en files in locales/en/
 Models to download (one time, then cached for every project):
   en-fr         345 MB  khabouss/opus-mt-tc-big-en-fr-onnx
@@ -102,24 +102,24 @@ then declare it in `locale-ai.config.json`:
 - `prefix` is required for multi-target models (`convert` lists the model's `>>xx<<` tokens).
 - Project models take priority over built-ins for their language pairs and can be used for pivoting (e.g. `ja → en → tr`).
 
-**Recommended for quality**: the `opus-mt-tc-big-en-*` models (es, it, pt, ar, tr, and `zle` for ru/uk). They are ~3x larger (270–370 MB int8) and ~1.5x slower, and clearly better. `en-fr` already uses one of these as its built-in model (see [Languages](#languages)), hosted at [khabouss/opus-mt-tc-big-en-fr-onnx](https://huggingface.co/khabouss/opus-mt-tc-big-en-fr-onnx) — no conversion needed for it.
+**Already built-in, no conversion needed**: `fr`, `es`, `it`, `pt`, `ar` and `tr` all use the larger, clearly-better `opus-mt-tc-big-en-*` checkpoints as their built-in model (~3x larger, 270–370 MB int8, ~1.5x slower than the plain OPUS-MT default); `ru`/`uk` use the tc-big `zle` multi-target checkpoint, and `pl` uses Helsinki-NLP's multi-target Slavic model (no tc-big `en-pl` checkpoint exists). The section above is for adding a language or model that *isn't* already built in.
 
-| Target | Built-in model, chrF / BLEU | tc-big model, chrF / BLEU |
+| Target | Previous (plain OPUS-MT), chrF / BLEU | Built-in today (tc-big / better), chrF / BLEU |
 |---|---|---|
+| fr | 68.9 / 53.7 | **72.5 / 55.9** |
 | es | 72.9 / 56.9 | **73.6 / 57.4** |
 | it | 73.8 / 54.9 | **75.3 / 58.2** |
 | pt | 68.3 / 44.4 | **70.6 / 47.7** (`>>por<<`, or `>>pob<<` for pt-BR) |
 | ru | 65.7 / 46.0 | **68.2 / 48.1** (`>>rus<<`) |
 | ar | 47.8 / 16.6 | **49.7 / 19.9** (`>>ara<<`, with `--fast`) |
-| tr | — | **67.4 / 42.1** |
+| tr | — | **67.4 / 42.1** (not supported before) |
+| pl | — | **62.6 / 40.3** (`>>pol<<`, not supported before) |
 
 300 Tatoeba test sentences per language, int8, beam 4 unless noted; reproduce with `bench/` (see `bench/README.md`). `opus-mt-tc-big-en-ko` is broken upstream (garbage output even in PyTorch) and is not recommended.
 
-Also new: **pl** (chrF 62.6 / BLEU 40.3, `>>pol<<`), hosted at [khabouss/opus-mt-en-sla-onnx](https://huggingface.co/khabouss/opus-mt-en-sla-onnx) — no tc-big `en-pl` checkpoint exists, so this uses Helsinki-NLP's multi-target Slavic model instead.
-
 ## Languages
 
-From English: af, ar, ca, cs, da, de, es, fi, fr, gl, hi, hu, id, it, nl, oc, pl, pt (pt-BR, pt-PT), ro, ru, sv, uk, vi, xh, zh (zh-CN, zh-TW). Other source languages are supported directly or through English (`locale-ai models list --from fr`).
+From English: af, ar, ca, cs, da, de, es, fi, fr, gl, hi, hu, id, it, nl, oc, pl, pt (pt-BR, pt-PT), ro, ru, sv, tr, uk, vi, xh, zh (zh-CN, zh-TW). Other source languages are supported directly or through English (`locale-ai models list --from fr`).
 
 Measured on a sample app (see `test/fixtures`):
 
@@ -129,7 +129,7 @@ Measured on a sample app (see `test/fixtures`):
 | Usable — review before shipping | it, ar, ru, pl, zh-CN, zh-TW |
 | Untested | the other pairs in `locale-ai models list` |
 
-`en-fr` is already on the tc-big model (chrF 72.5 / BLEU 55.9) as a built-in, no config needed. With the other tc-big models above, it, ar and ru move up a tier, and tr and pl become available. Not available yet: ja, ko and others; he works with a converted `opus-mt-en-he` (weaker). Moroccan Arabic (Darija): the tc-big en-ar model has a `>>ary<<` token, but its output is close to Modern Standard Arabic — not supported.
+`fr`, `es`, `it`, `pt`, `ar`, `ru`/`uk`, `tr` and `pl` all ship on the tc-big (or better) models above as built-ins already — no config needed. Not available yet: ja, ko and others; he works with a converted `opus-mt-en-he` (weaker). Moroccan Arabic (Darija): the tc-big en-ar model has a `>>ary<<` token, but its output is close to Modern Standard Arabic — not supported.
 
 ## How it works
 

package/dist/cli.js

@@ -49,16 +49,52 @@ const endStatus = () => {
 };
 const mb = (bytes) => `${Math.round(bytes / 1e6)} MB`;
 const rel = (p) => path.relative(process.cwd(), p) || ".";
+/**
+ * `Number("abc")` is NaN, and NaN silently poisons everything downstream (an
+ * all-garbage translation batch, or a raw ONNX crash deep in the model) instead of
+ * a clear error at the CLI boundary where the bad flag value actually came from.
+ */
+function positiveInt(raw, flag, fallback) {
+    if (raw === undefined) {
+        if (fallback === undefined)
+            throw new Error(`Missing --${flag}.`);
+        return fallback;
+    }
+    const n = Number(raw);
+    if (!Number.isInteger(n) || n < 1)
+        throw new Error(`Invalid --${flag} "${raw}": expected a positive integer.`);
+    return n;
+}
 function bar(done, total, width = 24) {
     const ratio = total ? Math.min(1, done / total) : 1;
     const filled = Math.round(ratio * width);
     return `[${"█".repeat(filled)}${"░".repeat(width - filled)}] ${Math.round(ratio * 100)}%`;
 }
+/**
+ * A long download or translation batch can run for a minute or more. The `\r`-redrawn
+ * bar only ever reaches a real terminal, so a non-TTY run (CI logs, piped output) would
+ * otherwise print nothing at all between the start and the final result, indistinguishable
+ * from a hang. This prints one plain line per 10% there instead.
+ */
+function makeProgress(render) {
+    let lastPct = -1;
+    return (done, total) => {
+        if (isTTY) {
+            status(render(done, total));
+            return;
+        }
+        const pct = total ? Math.floor((done / total) * 100) : 100;
+        if (pct >= lastPct + 10 || (done >= total && lastPct < 100)) {
+            lastPct = done >= total ? 100 : pct;
+            log(render(done, total));
+        }
+    };
+}
 async function confirm(question, yes) {
     if (yes)
         return true;
     if (!process.stdin.isTTY) {
-        throw new Error(`${question.trim()} Re-run with --yes to allow downloads in non-interactive mode.`);
+        throw new Error(`Cannot ask "${question.trim()}" in a non-interactive shell. Re-run with --yes to allow downloads.`);
     }
     const rl = createInterface({ input: process.stdin, output: process.stderr });
     const answer = (await rl.question(`${question} (Y/n) `)).trim().toLowerCase();
@@ -86,7 +122,7 @@ async function ensureInstalled(models, yes) {
     if (!(await confirm(`Download ${mb(total)}?`, yes)))
         throw new Error("Download declined.");
     for (const p of plans) {
-        await install(p, (d, t) => status(`Downloading ${p.model.id} ${bar(d, t)} ${mb(d)}/${mb(t)}`));
+        await install(p, makeProgress((d, t) => `Downloading ${p.model.id} ${bar(d, t)} ${mb(d)}/${mb(t)}`));
         endStatus();
         log(`✓ Installed ${p.model.id}`);
     }
@@ -105,10 +141,6 @@ async function cmdTranslate(argv) {
             beams: { type: "string" },
             report: { type: "string" },
             config: { type: "string" },
-            out: { type: "string" },
-            source: { type: "string" },
-            target: { type: "string" },
-            python: { type: "string" },
             yes: { type: "boolean", short: "y", default: false },
         },
     });
@@ -116,13 +148,13 @@ async function cmdTranslate(argv) {
         throw new Error("Missing --to. Example: locale-ai translate --from en --to fr,es");
     const from = parseLocale(values.from);
     const targets = values.to.split(",").map((t) => t.trim()).filter(Boolean).map(parseLocale);
-    const batchSize = Math.max(1, Number(values["batch-size"]));
-    const beams = Math.max(1, Number(values.beams ?? (values.fast ? 1 : 4)));
+    const batchSize = positiveInt(values["batch-size"], "batch-size");
+    const beams = positiveInt(values.beams, "beams", values.fast ? 1 : 4);
     const config = loadConfig(process.cwd(), values.config);
     const catalog = catalogFor(config);
     const protect = protectRegex(config);
     const layout = detectLayout(process.cwd(), from.tag, values.input);
-    const sources = sourceFiles(layout, from.tag).map((f) => ({ file: f, doc: readLocaleDoc(f) }));
+    const sources = sourceFiles(layout, from.tag).map((f) => ({ file: f, doc: readLocaleDoc(f, from.tag) }));
     log(`locale-ai ${VERSION}`);
     log(`Found ${sources.length} ${from.tag} file${sources.length === 1 ? "" : "s"} in ${rel(layout.root)}/` +
         (layout.kind === "dir" ? `${from.tag}/` : ""));
@@ -139,7 +171,7 @@ async function cmdTranslate(argv) {
         }
         const files = sources.map(({ file, doc }) => {
             const target = targetFile(layout, from.tag, locale.tag, file);
-            const existing = fs.existsSync(target) ? readLocaleDoc(target).data : undefined;
+            const existing = fs.existsSync(target) ? readLocaleDoc(target, locale.tag).data : undefined;
             return { source: file, target, doc, existing, pending: pendingStrings(doc.data, existing, values.overwrite) };
         });
         const unique = new Set(files.flatMap((f) => f.pending)).size;
@@ -182,7 +214,7 @@ async function cmdTranslate(argv) {
                 batchSize,
                 protect,
                 lookup: glossaryLookup(from, job.locale, config),
-                onProgress: (d, t) => status(`${job.locale.tag}: translating ${bar(d, t)} ${d}/${t}`),
+                onProgress: makeProgress((d, t) => `${job.locale.tag}: translating ${bar(d, t)} ${d}/${t}`),
             });
             endStatus();
             translations = result.translations;
@@ -201,8 +233,8 @@ async function cmdTranslate(argv) {
             const built = buildTarget(f.doc.data, f.existing, (s) => translations.get(s) ?? s, values.overwrite);
             const { doc, added } = addPluralForms(built, intlCode(job.locale));
             pluralsAdded += added;
-            const text = f.doc.serialize(doc, job.locale.tag);
             const before = fs.existsSync(f.target) ? fs.readFileSync(f.target, "utf8") : undefined;
+            const text = f.doc.serialize(doc, job.locale.tag, before);
             if (text !== before) {
                 fs.mkdirSync(path.dirname(f.target), { recursive: true });
                 fs.writeFileSync(f.target, text);

package/dist/engine.js

@@ -137,21 +137,34 @@ async function attempt(units, style, hops, opts, progress, issues) {
         }
         let ok = true;
         const found = [];
-        const joined = pieces
-            .map((p) => {
+        const parts = await Promise.all(pieces.map(async (p) => {
             // No spaces between sentences in Chinese, Japanese, Thai...
             if (!p.translate)
                 return NO_SPACES.test(opts.targetLang) ? p.value.replace(/^[ \t]+$/, "") : p.value;
-            const out = results.get(prepare(p.value)) ?? "";
-            if (isDegenerate(p.value, out))
+            let out = results.get(prepare(p.value)) ?? "";
+            const suspicionKind = (o) => o.length > 3 * p.value.length + 20 ? "suspicious-length" : /(\b\p{L}+\b)(?:\W+\1\b){3,}/u.test(o) ? "repetition" : undefined;
+            let degenerate = isDegenerate(p.value, out);
+            let kind = degenerate ? undefined : suspicionKind(out);
+            // Quantizing a batched tensor computes scale factors from the whole batch, so one
+            // row's output can depend on what else shared its batch. Retrying the same input
+            // alone (batch of 1) often clears a hallucination that batch composition caused.
+            if (degenerate || kind !== undefined) {
+                const retry = await runModels(new Set([prepare(p.value)]), hops, opts, progress);
+                const retryOut = retry.get(prepare(p.value)) ?? out;
+                const retryDegenerate = isDegenerate(p.value, retryOut);
+                if (!retryDegenerate && suspicionKind(retryOut) === undefined) {
+                    out = retryOut;
+                    degenerate = false;
+                    kind = undefined;
+                }
+            }
+            if (degenerate)
                 ok = false;
-            else if (out.length > 3 * p.value.length + 20)
-                found.push({ source: unit.owner, kind: "suspicious-length", output: out });
-            else if (/(\b\p{L}+\b)(?:\W+\1\b){3,}/u.test(out))
-                found.push({ source: unit.owner, kind: "repetition", output: out });
+            else if (kind)
+                found.push({ source: unit.owner, kind, output: out });
             return postprocess(p.value, out, opts.targetLang);
-        })
-            .join("");
+        }));
+        const joined = parts.join("");
         const restored = ok ? unmask(joined, masked) : undefined;
         if (restored === undefined)
             failed.push(unit);

package/dist/locales.js

@@ -96,17 +96,28 @@ function readJsonDoc(file, text) {
     const keepNewline = finalNewline(text);
     return { data, serialize: (d) => withNewline(JSON.stringify(d, null, indent), keepNewline) };
 }
-function readYamlDoc(file, text) {
-    let data;
+/**
+ * Rails and some other frameworks nest an entire YAML locale file under a single
+ * top-level key matching the locale tag itself (`en:\n  hello: Hi`), so the file is
+ * valid wherever it's loaded from. Detect that wrapper and unwrap it before handing
+ * `data` to the rest of the pipeline (which otherwise treats "en" as just another
+ * translatable key), then re-wrap under the target's own tag on serialize.
+ */
+function readYamlDoc(file, text, tag) {
+    let raw;
     try {
-        data = parseYaml(text) ?? {};
+        raw = parseYaml(text) ?? {};
     }
     catch (e) {
         throw new Error(`Invalid YAML in ${file}: ${e.message}`);
     }
     const indent = Number(/\n( +)\S/.exec(text)?.[1].length) || 2;
     const keepNewline = finalNewline(text);
-    return { data, serialize: (d) => withNewline(stringifyYaml(d, { indent }), keepNewline) };
+    const keys = isObj(raw) ? Object.keys(raw) : [];
+    const wrapped = keys.length === 1 && keys[0] === tag;
+    const data = wrapped ? raw[tag] : raw;
+    const serialize = (d, targetTag) => withNewline(stringifyYaml(wrapped ? { [targetTag]: d } : d, { indent }), keepNewline);
+    return { data, serialize };
 }
 /** Apple .strings: `"key" = "value";`, optionally preceded by a `/* ... *\/` comment. */
 function unescapeC(s) {
@@ -116,7 +127,8 @@ function escapeC(s) {
     return s.replace(/[\\"\n\t\r]/g, (c) => ({ "\\": "\\\\", '"': '\\"', "\n": "\\n", "\t": "\\t", "\r": "\\r" })[c]);
 }
 function readStringsDoc(file, text) {
-    const PAIR = /(?:(\/\*[\s\S]*?\*\/)[ \t]*\r?\n+)?[ \t]*"((?:\\.|[^"\\])*)"\s*=\s*"((?:\\.|[^"\\])*)"\s*;/g;
+    // A key's preceding comment is either a /* ... */ block, or one or more consecutive // lines.
+    const PAIR = /(?:((?:\/\*[\s\S]*?\*\/)|(?:\/\/[^\r\n]*(?:\r?\n[ \t]*\/\/[^\r\n]*)*))[ \t]*\r?\n+)?[ \t]*"((?:\\.|[^"\\])*)"\s*=\s*"((?:\\.|[^"\\])*)"\s*;/g;
     const data = newObj();
     const entries = [];
     for (const m of text.matchAll(PAIR)) {
@@ -182,6 +194,54 @@ function readArbDoc(file, text) {
     return { data, serialize };
 }
 const poKey = (e) => (e.msgctxt ?? "") + "\u0004" + e.msgid;
+/**
+ * gettext's well-established Plural-Forms formulas, keyed by language. Copying the
+ * source header's line verbatim (as a naive patch does) is wrong whenever the target
+ * needs a different count — Arabic needs 6 categories, an English source's line says
+ * 2, and anything consuming the .po indexes msgstr[] short or drops forms silently.
+ */
+const PLURAL_FORMS = {
+    ja: "nplurals=1; plural=0;",
+    zh: "nplurals=1; plural=0;",
+    ko: "nplurals=1; plural=0;",
+    th: "nplurals=1; plural=0;",
+    vi: "nplurals=1; plural=0;",
+    id: "nplurals=1; plural=0;",
+    ms: "nplurals=1; plural=0;",
+    fr: "nplurals=2; plural=(n > 1);",
+    pt: "nplurals=2; plural=(n != 1);",
+    cs: "nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;",
+    sk: "nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;",
+    pl: "nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);",
+    ru: "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);",
+    uk: "nplurals=3; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2);",
+    ro: "nplurals=3; plural=(n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < 20)) ? 1 : 2);",
+    ar: "nplurals=6; plural=(n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5);",
+};
+/**
+ * Falls back to CLDR's plural category count (via Intl.PluralRules) for a language
+ * not in the table above: not necessarily linguistically exact (it buckets purely by
+ * small integer value), but it keeps nplurals correct, which is the part that breaks
+ * gettext consumers when wrong.
+ */
+function pluralFormsHeader(lang) {
+    const known = PLURAL_FORMS[lang];
+    if (known)
+        return known;
+    let n;
+    try {
+        n = new Intl.PluralRules(lang).resolvedOptions().pluralCategories.length;
+    }
+    catch {
+        n = 2;
+    }
+    if (n <= 1)
+        return "nplurals=1; plural=0;";
+    if (n === 2)
+        return "nplurals=2; plural=(n != 1);";
+    const branches = Array.from({ length: n - 1 }, (_, i) => `n==${i + 1} ? ${i} :`).join(" ");
+    return `nplurals=${n}; plural=(${branches} ${n - 1});`;
+}
 function parsePo(text) {
     const entries = [];
     let leading = [];
@@ -265,17 +325,24 @@ function readPoDoc(file, text) {
         if (!e.isHeader && e.msgidPlural === undefined)
             data[poKey(e)] = e.msgstr;
     const keepNewline = finalNewline(text);
-    const serialize = (d, targetTag) => {
+    const serialize = (d, targetTag, existingText) => {
         const obj = isObj(d) ? d : newObj();
         const seen = new Set();
         const blocks = leading.length ? [leading.join("\n")] : [];
+        // A previous run may have left a header a translator then hand-fixed (a corrected
+        // Plural-Forms, an added Last-Translator, ...). Keep it as-is rather than
+        // regenerating from the source every time; only a brand-new target file gets a
+        // freshly computed header.
+        const existingHeader = existingText && parsePo(existingText).entries.find((e) => e.isHeader);
         for (const e of entries) {
             const lines = [...e.comments];
             if (e.msgctxt !== undefined)
                 lines.push(`msgctxt "${escapeC(e.msgctxt)}"`);
             if (e.isHeader) {
                 lines.push(`msgid ""`);
-                const header = e.msgstr.replace(/^Language:.*$/m, `Language: ${targetTag}`);
+                const header = existingHeader
+                    ? existingHeader.msgstr
+                    : e.msgstr.replace(/^Language:.*$/m, `Language: ${targetTag}`).replace(/^Plural-Forms:.*$/m, `Plural-Forms: ${pluralFormsHeader(targetTag)}`);
                 lines.push(`msgstr "${escapeC(header)}"`);
             }
             else if (e.msgidPlural !== undefined) {
@@ -304,11 +371,12 @@ function readPoDoc(file, text) {
     };
     return { data, serialize };
 }
-export function readLocaleDoc(file) {
+/** `tag` is the locale this file is for (the source tag, or an existing target's own tag). */
+export function readLocaleDoc(file, tag) {
     const text = fs.readFileSync(file, "utf8").replace(/^/, "");
     const ext = path.extname(file).toLowerCase();
     if (ext === ".yaml" || ext === ".yml")
-        return readYamlDoc(file, text);
+        return readYamlDoc(file, text, tag);
     if (ext === ".po")
         return readPoDoc(file, text);
     if (ext === ".arb")

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locale-ai",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Translate your i18n locale files locally on CPU. Free, offline, no API keys.",
   "author": "Taha KHABOUSS",
   "repository": {