locale-ai 0.6.4 → 0.6.5

package/README.md

@@ -31,7 +31,7 @@ Download 577 MB? (Y/n)
 - **Layouts**: `locales/en/*.json` (nested folders too) → `locales/fr/...`, or `messages/en.json` → `messages/fr.json`. `.json`, `.yaml`, `.yml`, `.po` (gettext), `.arb` (Flutter) and `.strings` (Apple) are all supported, mixed freely in the same project. Common folders (`locales`, `src/locales`, `public/locales`, `src/i18n`, `messages`, `lang`, ...) are found automatically, or pass `--input`.
 - **Placeholders are never translated or lost**: `{name}`, `{{count}}`, `%s`, `%1$s`, `%(name)s`, `%{name}`, `${name}`, `:attribute`, `$t(key)`, `@:key`, `<b>…</b>`, `<0>…</0>`, `&nbsp;`, URLs, e-mails.
 - **ICU MessageFormat**: `{count, plural, one {# file} other {# files}}`, `select`, nesting. Only the text is translated.
-- **i18next plurals**: adds the forms the target language needs (`_few`, `_many` for Arabic, Russian, Polish...), so the app doesn't fall back to the English string — each added form copies the `_other` translation verbatim rather than inflecting it, so review and adjust the wording of `_zero`/`_two`/`_few`/`_many` by hand for languages where those forms use a different noun/adjective form.
+- **i18next plurals**: adds the forms the target language needs (`_few`, `_many` for Arabic, Russian, Polish...), so the app doesn't fall back to the English string — each added form copies the `_other` translation verbatim rather than inflecting it, so review and adjust the wording of `_zero`/`_two`/`_few`/`_many` by hand for languages where those forms use a different noun/adjective form. The `_one` form is translated with its `{{count}}` temporarily replaced by a literal "1", since the model otherwise sometimes picks the target's plural noun form despite the English source being grammatically singular.
 - **Incremental**: only missing keys are translated, existing translations are kept, re-runs are no-ops. `--overwrite` re-translates everything.
 - **Structure**: key order, numbers, booleans, arrays, indentation and trailing newline are preserved.
 
@@ -69,6 +69,7 @@ Models are cached once per machine in `~/.cache/locale-ai` (`LOCALE_AI_CACHE` to
 ```json
 {
   "doNotTranslate": ["Acme", "Acme Pro"],
+  "doNotTranslateKeys": ["brand.name", "config.statusEnum"],
   "glossary": {
     "fr": { "Workspace": "Espace de travail" },
     "de": { "Sign in": "Einloggen" }
@@ -78,6 +79,8 @@ Models are cached once per machine in `~/.cache/locale-ai` (`LOCALE_AI_CACHE` to
 
 `glossary` entries replace whole strings. `models` adds your own translation models (next section). A built-in glossary covers ~60 standard UI labels ("Sign in", "Settings", "Log out", ...) for fr, es, de, it, pt-BR, nl and ar, because one-to-three word labels are where sentence-level MT is weakest.
 
+`doNotTranslate` protects a term everywhere it appears in any string. `doNotTranslateKeys` instead protects a key path (dot/`[i]` notation, e.g. `brand.name`, `items[0]`) — the whole value at that path, and everything nested under it if it's an object or array, is copied to every target verbatim. Use it for an enum array (`"statusEnum": ["ACTIVE", "PENDING"]`, which you'd otherwise have to list value-by-value in `doNotTranslate`) or a field a model might respell or inflect even when the word is listed in `doNotTranslate` elsewhere in the same string.
+
 ## Add a language, or a better model
 
 Any [MarianMT / OPUS-MT](https://huggingface.co/Helsinki-NLP) model on Hugging Face can be used. Convert it once (needs Python):

package/dist/cli.js

@@ -9,7 +9,7 @@ import { CATALOG, customEntries, findById, reachableTargets, resolveRoute, } fro
 import { translateStrings } from "./engine.js";
 import { CONFIG_FILE, glossaryLookup, loadConfig, protectRegex } from "./glossary.js";
 import { intlCode, parseLocale } from "./lang.js";
-import { addPluralForms, buildTarget, detectLayout, existingTargets, flattenStrings, pendingStrings, readLocaleDoc, sourceFiles, targetFile, } from "./locales.js";
+import { addPluralForms, buildTarget, detectLayout, existingTargets, flattenStrings, keyPathProtect, pendingStrings, pluralOneStrings, readLocaleDoc, sourceFiles, targetFile, } from "./locales.js";
 import { cacheRoot, install, isInstalled, loadTranslator, localModelProblem, planInstall, readManifest, uninstall, } from "./models.js";
 import { mask } from "./placeholders.js";
 /** Read from package.json so `npm version` is the single source of truth. */
@@ -158,8 +158,10 @@ async function cmdTranslate(argv) {
     const config = loadConfig(process.cwd(), values.config);
     const catalog = catalogFor(config);
     const protect = protectRegex(config);
+    const isProtectedKey = keyPathProtect(config.doNotTranslateKeys);
     const layout = detectLayout(process.cwd(), from.tag, values.input);
     const sources = sourceFiles(layout, from.tag).map((f) => ({ file: f, doc: readLocaleDoc(f, from.tag) }));
+    const singularHints = new Set(sources.flatMap(({ doc }) => [...pluralOneStrings(doc.data)]));
     log(`locale-ai ${VERSION}`);
     log(`Found ${sources.length} ${from.tag} file${sources.length === 1 ? "" : "s"} in ${rel(layout.root)}/` +
         (layout.kind === "dir" ? `${from.tag}/` : ""));
@@ -177,7 +179,13 @@ async function cmdTranslate(argv) {
         const files = sources.map(({ file, doc }) => {
             const target = targetFile(layout, from.tag, locale.tag, file);
             const existing = fs.existsSync(target) ? readLocaleDoc(target, locale.tag).data : undefined;
-            return { source: file, target, doc, existing, pending: pendingStrings(doc.data, existing, values.overwrite) };
+            return {
+                source: file,
+                target,
+                doc,
+                existing,
+                pending: pendingStrings(doc.data, existing, values.overwrite, isProtectedKey),
+            };
         });
         const unique = new Set(files.flatMap((f) => f.pending)).size;
         jobs.push({ locale, route, files, unique });
@@ -219,6 +227,7 @@ async function cmdTranslate(argv) {
                 batchSize,
                 protect,
                 lookup: glossaryLookup(from, job.locale, config),
+                singularHints,
                 onProgress: makeProgress((d, t) => `${job.locale.tag}: translating ${bar(d, t)} ${d}/${t}`),
             });
             endStatus();
@@ -235,7 +244,7 @@ async function cmdTranslate(argv) {
         let written = 0;
         let pluralsAdded = 0;
         for (const f of job.files) {
-            const built = buildTarget(f.doc.data, f.existing, (s) => translations.get(s) ?? s, values.overwrite);
+            const built = buildTarget(f.doc.data, f.existing, (s) => translations.get(s) ?? s, values.overwrite, isProtectedKey);
             const { doc, added } = addPluralForms(built, intlCode(job.locale));
             pluralsAdded += added;
             const before = fs.existsSync(f.target) ? fs.readFileSync(f.target, "utf8") : undefined;

package/dist/engine.js

@@ -208,6 +208,36 @@ async function splitFallback(units, hops, opts, progress, issues) {
         issues.push({ source: unit.owner, kind: "placeholder-fallback", output: unit.result });
     }
 }
+const COUNT_PLACEHOLDER = /\{\{\s*count\s*\}\}/;
+/**
+ * A "_one" key's source is grammatically singular ("You have {{count}} file"), but the
+ * masked marker that stands in for {{count}} carries no number of its own, so the model
+ * sometimes still produces the target's PLURAL noun form (confirmed: en→fr "fileCount_one"
+ * came back "Vous avez {{count}} fichiers"). Retranslating with a literal "1" in place of
+ * {{count}} gives the model the real grammatical cue; only applied when {{count}} is the
+ * string's only placeholder (anything else needs the normal masking pipeline to survive),
+ * and only kept when the "1" survives translation intact enough to restore {{count}}.
+ */
+async function fixSingularGrammar(translations, singularHints, remaining, hops, opts, progress) {
+    for (const s of singularHints) {
+        if (!remaining.has(s) || translations.get(s) === undefined)
+            continue;
+        const tokens = mask(s, "natural").tokens;
+        if (tokens.length !== 1 || !COUNT_PLACEHOLDER.test(tokens[0]))
+            continue;
+        const m = COUNT_PLACEHOLDER.exec(s);
+        const pinned = s.slice(0, m.index) + "1" + s.slice(m.index + m[0].length);
+        const prepared = prepare(pinned);
+        const out = (await runModels(new Set([prepared]), hops, opts, progress)).get(prepared) ?? "";
+        if (isDegenerate(pinned, out))
+            continue;
+        const post = postprocess(pinned, out, opts.targetLang);
+        const restored = /(?<![\d.])1(?![\d.])/.exec(post);
+        if (!restored)
+            continue; // can't confirm the "1" survived: keep the original translation
+        translations.set(s, post.slice(0, restored.index) + m[0] + post.slice(restored.index + 1));
+    }
+}
 export async function translateStrings(strings, hops, opts) {
     const issues = [];
     const progress = { done: 0, total: 0 };
@@ -236,5 +266,7 @@ export async function translateStrings(strings, hops, opts) {
         const parts = p.units.map((u) => u.result ?? u.text);
         translations.set(p.source, p.split ? p.split.rebuild(parts) : parts[0]);
     }
+    if (opts.singularHints?.size)
+        await fixSingularGrammar(translations, opts.singularHints, new Set(remaining), hops, opts, progress);
     return { translations, issues };
 }

package/dist/glossary.js

@@ -84,6 +84,8 @@ export function loadConfig(cwd, file) {
         const cfg = JSON.parse(fs.readFileSync(p, "utf8"));
         if (cfg.doNotTranslate && !Array.isArray(cfg.doNotTranslate))
             throw new Error('"doNotTranslate" must be an array');
+        if (cfg.doNotTranslateKeys && !Array.isArray(cfg.doNotTranslateKeys))
+            throw new Error('"doNotTranslateKeys" must be an array');
         if (cfg.models && !Array.isArray(cfg.models))
             throw new Error('"models" must be an array');
         return { ...cfg, dir: path.dirname(p) };

package/dist/locales.js

@@ -448,47 +448,93 @@ export function flattenStrings(data, prefix = "", out = []) {
             flattenStrings(data[k], prefix ? `${prefix}.${k}` : k, out);
     return out;
 }
-export function pendingStrings(source, existing, overwrite, out = []) {
-    if (typeof source === "string") {
-        if (needsTranslation(existing, overwrite))
-            out.push(source);
-    }
-    else if (Array.isArray(source)) {
-        source.forEach((v, i) => pendingStrings(v, Array.isArray(existing) ? existing[i] : undefined, overwrite, out));
-    }
-    else if (isObj(source)) {
-        for (const k of Object.keys(source))
-            pendingStrings(source[k], isObj(existing) ? existing[k] : undefined, overwrite, out);
-    }
+/**
+ * True when `path` is, or is nested under, one of `doNotTranslateKeys` — the same
+ * dot/`[i]` notation flattenStrings produces (e.g. "brand.name", "config.statusEnum").
+ * Protecting a path that points at an object or array protects every string nested
+ * under it too, which is what makes an enum array ("statusEnum": ["ACTIVE", ...])
+ * protectable without enumerating each value as a doNotTranslate term.
+ */
+export function keyPathProtect(doNotTranslateKeys) {
+    const keys = (doNotTranslateKeys ?? []).filter(Boolean);
+    if (!keys.length)
+        return undefined;
+    return (path) => keys.some((k) => path === k || path.startsWith(`${k}.`) || path.startsWith(`${k}[`));
+}
+export function pendingStrings(source, existing, overwrite, isProtected) {
+    const out = [];
+    const visit = (s, e, path) => {
+        if (typeof s === "string") {
+            if (!isProtected?.(path) && needsTranslation(e, overwrite))
+                out.push(s);
+        }
+        else if (Array.isArray(s)) {
+            s.forEach((v, i) => visit(v, Array.isArray(e) ? e[i] : undefined, `${path}[${i}]`));
+        }
+        else if (isObj(s)) {
+            for (const k of Object.keys(s))
+                visit(s[k], isObj(e) ? e[k] : undefined, path ? `${path}.${k}` : k);
+        }
+    };
+    visit(source, existing, "");
     return out;
 }
 /**
  * Builds the target document: source structure and key order, existing
  * translations kept (unless overwrite), new strings from `translate`, and keys
- * that only exist in the target preserved at the end.
+ * that only exist in the target preserved at the end. A string at a protected
+ * path (see keyPathProtect) is always copied from the source verbatim.
  */
-export function buildTarget(source, existing, translate, overwrite) {
-    if (typeof source === "string") {
-        return needsTranslation(existing, overwrite) ? translate(source) : existing;
-    }
-    if (Array.isArray(source)) {
-        return source.map((v, i) => buildTarget(v, Array.isArray(existing) ? existing[i] : undefined, translate, overwrite));
-    }
-    if (isObj(source)) {
-        const out = newObj();
-        for (const k of Object.keys(source)) {
-            out[k] = buildTarget(source[k], isObj(existing) ? existing[k] : undefined, translate, overwrite);
+export function buildTarget(source, existing, translate, overwrite, isProtected) {
+    const visit = (s, e, path) => {
+        if (typeof s === "string") {
+            if (isProtected?.(path))
+                return s;
+            return needsTranslation(e, overwrite) ? translate(s) : e;
         }
-        if (isObj(existing))
-            for (const k of Object.keys(existing))
-                if (!(k in out))
-                    out[k] = existing[k];
-        return out;
-    }
-    return source;
+        if (Array.isArray(s))
+            return s.map((v, i) => visit(v, Array.isArray(e) ? e[i] : undefined, `${path}[${i}]`));
+        if (isObj(s)) {
+            const out = newObj();
+            for (const k of Object.keys(s))
+                out[k] = visit(s[k], isObj(e) ? e[k] : undefined, path ? `${path}.${k}` : k);
+            if (isObj(e))
+                for (const k of Object.keys(e))
+                    if (!(k in out))
+                        out[k] = e[k];
+            return out;
+        }
+        return s;
+    };
+    return visit(source, existing, "");
 }
 const CATEGORIES = ["zero", "one", "two", "few", "many", "other"];
 const PLURAL_KEY = /^(.+?)(_ordinal)?_(zero|one|two|few|many|other)$/;
+/**
+ * Source strings at a cardinal i18next "_one" plural key ("itemCount_one": "{{count}}
+ * item"). These are grammatically singular in English, but the model sometimes infers
+ * the TARGET's plural form anyway because {{count}} reads as an opaque marker rather
+ * than a concrete number — the engine uses this set to retranslate with a real "1" in
+ * place of {{count}} and recover the correct grammatical number (see engine.ts).
+ */
+export function pluralOneStrings(data, out = new Set()) {
+    const visit = (v) => {
+        if (Array.isArray(v)) {
+            v.forEach(visit);
+            return;
+        }
+        if (!isObj(v))
+            return;
+        for (const k of Object.keys(v)) {
+            const m = PLURAL_KEY.exec(k);
+            if (m && !m[2] && m[3] === "one" && typeof v[k] === "string")
+                out.add(v[k]);
+            visit(v[k]);
+        }
+    };
+    visit(data);
+    return out;
+}
 /**
  * i18next stores plurals as `key_one` / `key_other`. Languages such as Arabic,
  * Russian or Polish need more forms (`_few`, `_many` ...) and i18next does not fall

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locale-ai",
-  "version": "0.6.4",
+  "version": "0.6.5",
   "description": "Translate your i18n locale files locally on CPU. Free, offline, no API keys.",
   "author": "Taha KHABOUSS",
   "repository": {