locale-ai 0.6.13 → 0.6.15

package/README.md

@@ -28,7 +28,7 @@ Download 577 MB? (Y/n)
 
 ## What it handles
 
-- **Layouts**: `locales/en/*.json` (nested folders too) → `locales/fr/...`, or `messages/en.json` → `messages/fr.json`. `.json`, `.yaml`, `.yml`, `.po`/`.pot` (gettext), `.arb` (Flutter) and `.strings` (Apple) are all supported, mixed freely in the same project. Common folders (`locales`, `src/locales`, `public/locales`, `src/i18n`, `messages`, `lang`, ...) are found automatically, or pass `--input`.
+- **Layouts**: `locales/en/*.json` (nested folders too) → `locales/fr/...`, or `messages/en.json` → `messages/fr.json`. `.json`, `.yaml`, `.yml`, `.po`/`.pot` (gettext), `.arb` (Flutter) and `.strings` (Apple) are all supported, mixed freely in the same project. Common folders (`locales`, `src/locales`, `public/locales`, `src/i18n`, `messages`, `lang`, `config/locales` (Rails), ...) are found automatically, or pass `--input`. Framework-specific naming is also recognized: `flutter gen-l10n`'s `app_en.arb`/`app_fr.arb` filename prefix, and Apple's `en.lproj`/`fr.lproj` directory-suffix convention (`--input en.lproj` or just the project root both work), on both the source and the generated target side.
 - **Placeholders are never translated or lost**: `{name}`, `{{count}}`, `%s`, `%1$s`, `%(name)s`, `%{name}`, `${name}`, `:attribute`, `$t(key)`, `@:key`, `<b>…</b>`, `<0>…</0>`, `&nbsp;`, URLs, e-mails.
 - **ICU MessageFormat**: `{count, plural, one {# file} other {# files}}`, `select`, nesting. Only the text is translated.
 - **i18next plurals**: adds the forms the target language needs (`_few`, `_many` for Arabic, Russian, Polish...), so the app doesn't fall back to the English string — each added form copies the `_other` translation verbatim rather than inflecting it, so review and adjust the wording of `_zero`/`_two`/`_few`/`_many` by hand for languages where those forms use a different noun/adjective form. The `_one` form is translated with its `{{count}}` temporarily replaced by a literal "1", since the model otherwise sometimes picks the target's plural noun form despite the English source being grammatically singular. Rails-style nested plurals (`item:` with `one:`/`other:` children instead of `item_one`/`item_other`) get the same missing-forms treatment.
@@ -140,7 +140,7 @@ Measured on a sample app (see `test/fixtures`):
 
 ## How it works
 
-- **Models**: [OPUS-MT](https://huggingface.co/Helsinki-NLP) (MarianMT) in ONNX, int8-quantized (int8 vs fp32 measured equal: en-fr chrF 68.9 vs 69.0, en-ar 47.8 vs 47.8). Downloads are pinned to a commit and verified (sha256 / git blob hash). Every model is OPUS-MT by Helsinki-NLP (Tiedemann & Thottingal, 2020 — cite this if you use the models); each is licensed individually (most CC-BY 4.0, some Apache 2.0 — check the model's own card). The larger `tc-big` and multi-target checkpoints are re-hosted as ONNX conversions under [khabouss/*](https://huggingface.co/khabouss), each model card carrying the original's license, crediting and citing it, and listing only the ONNX conversion and quantization as locale-ai's own addition.
+- **Models**: [OPUS-MT](https://huggingface.co/Helsinki-NLP) (MarianMT) in ONNX, int8-quantized (int8 vs fp32 measured equal: en-fr chrF 68.9 vs 69.0, en-ar 47.8 vs 47.8). Downloads are pinned to a commit and verified (sha256 / git blob hash) — third-party repos (Xenova's OPUS-MT conversions) to a commit captured when that model was added, so an upstream change there never silently changes your translations; this project's own `khabouss/*` repos stay on their latest commit, since we only ever change those ourselves. Every model is OPUS-MT by Helsinki-NLP (Tiedemann & Thottingal, 2020 — cite this if you use the models); each is licensed individually (most CC-BY 4.0, some Apache 2.0 — check the model's own card). The larger `tc-big` and multi-target checkpoints are re-hosted as ONNX conversions under [khabouss/*](https://huggingface.co/khabouss), each model card carrying the original's license, crediting and citing it, and listing only the ONNX conversion and quantization as locale-ai's own addition.
 - **Own runtime** on [onnxruntime](https://onnxruntime.ai) (`src/marian.ts`), no transformers.js:
   - a SentencePiece Unigram tokenizer including the model's precompiled normalisation map, identical to Python `sentencepiece` on 3,577/3,577 test sentences across 5 models (`src/spm.ts`);
   - real beam search with Hugging Face semantics, identical to `transformers` PyTorch beam search on 599/600 sentences; log-softmax + top-k run natively in a tiny embedded ONNX graph (`scripts/make-topk-graph.py`);

package/dist/catalog.js

@@ -17,7 +17,90 @@ const DIRECT = [
     "th-en", "tr-en", "uk-en", "uk-ru", "vi-en", "xh-en", "zh-en",
 ];
 const ROMANCE = ["es", "pt", "ca", "gl", "oc", "it", "ro", "fr"];
-export const CATALOG = [
+/**
+ * Every third-party repo this catalog downloads from, pinned to the commit that was
+ * current when each pair was added — so every install gets the exact same files
+ * instead of silently picking up whatever Xenova's "main" branch has moved to by
+ * then, with no changelog entry or version bump on our side to explain why
+ * translations changed. This project's own `khabouss/*` repos are deliberately left
+ * out: we only ever change those ourselves, so there's no silent-drift risk to
+ * guard against, and pinning them would just mean remembering to bump a SHA by hand
+ * every time a better conversion is published.
+ */
+const PINNED_REVISIONS = {
+    "Xenova/opus-mt-ROMANCE-en": "f60feaabe8bb435a5871e7702b25f53a01e770f8",
+    "Xenova/opus-mt-af-en": "0eba4e40af86c6ad316a58fc9a7ed1134b05fd90",
+    "Xenova/opus-mt-ar-en": "09c7556866400dcc53562ba8d7035119e7d9a2c1",
+    "Xenova/opus-mt-cs-en": "c9f7336d1cf49ba5b7a412cba3b86999fa781716",
+    "Xenova/opus-mt-da-de": "3784087953108e4ee9957c8f9eff11c0bf0cabda",
+    "Xenova/opus-mt-da-en": "6496d5e47a016552357be8f06294096609256ac4",
+    "Xenova/opus-mt-de-en": "399dfd68706739fffd503f876093e455ae268a06",
+    "Xenova/opus-mt-de-es": "d339419ee3bda301bdcd00249d4436b74664ee7e",
+    "Xenova/opus-mt-de-fr": "e783da03d247c4d16778e938530e9b5d7bd1523d",
+    "Xenova/opus-mt-en-ROMANCE": "9d2ba69ac80c8e8453c3d9a1e2323a0e7b8ca3cd",
+    "Xenova/opus-mt-en-af": "3d48312617d092d77cb5aef9f9e9636ad639dd8d",
+    "Xenova/opus-mt-en-cs": "ed163ca109a93297e2f055eb3397bd33230141ad",
+    "Xenova/opus-mt-en-da": "33862dc071a5cc43e821e60742553c501c587e77",
+    "Xenova/opus-mt-en-de": "1ca130c44c4c5441ef16d48aae521a424ab644f7",
+    "Xenova/opus-mt-en-es": "4b002a4c7edd54a7ced58877258b87f7efd3f892",
+    "Xenova/opus-mt-en-fi": "4e5799273dbd7cce94c1e99cc9d3acb9f191b8eb",
+    "Xenova/opus-mt-en-hi": "7aad72006a1588a9c3485ed6203aed99eebbc872",
+    "Xenova/opus-mt-en-hu": "1b7df958c1bd8f094eff1ce76d536468923ed7e3",
+    "Xenova/opus-mt-en-id": "b3b41f654c2fb3286d007dee1da7b0f47fd27d82",
+    "Xenova/opus-mt-en-it": "075406e3c8c2c30634d4a1bd8f00c21d9e162011",
+    "Xenova/opus-mt-en-nl": "c3a0541231a7329b4492809fc2bd7ab7d2ceb0bc",
+    "Xenova/opus-mt-en-ro": "3d816891778276930df2a45cbfb70689b3fe8981",
+    "Xenova/opus-mt-en-sv": "33d522d806c041cdebecd549c558d22a2b9b4ea4",
+    "Xenova/opus-mt-en-vi": "30bcd461c0d7338ad106b36314d3eedc2d4e998c",
+    "Xenova/opus-mt-en-xh": "1ab9c4f4400ea4805f220ff5d551d3d0169b7a96",
+    "Xenova/opus-mt-en-zh": "046f55aec303cdee3e0318604406d4df20f1e8ea",
+    "Xenova/opus-mt-es-de": "7f493e4b81b684478396f37de4c1ebc28ecd5855",
+    "Xenova/opus-mt-es-en": "eadfd7c658a9d8929ac3b8e996b68a68e2c7d480",
+    "Xenova/opus-mt-es-fr": "3bef5f612825d503f6fb85e0527a1fe30f2578c2",
+    "Xenova/opus-mt-es-it": "596c67cda7cb919f0d67221f3a31bd7df79f19d0",
+    "Xenova/opus-mt-es-ru": "268ccd60bde3cabde17068d482ca6930512f1a85",
+    "Xenova/opus-mt-et-en": "58104952b730220eb7a9cdbbe19533a064f40dc0",
+    "Xenova/opus-mt-fi-de": "6d93339597e19574f543259315d2400939067c1e",
+    "Xenova/opus-mt-fi-en": "a623e4812687be788e3ab720c8a950ab64704793",
+    "Xenova/opus-mt-fr-de": "7eea1996b4480d073cf560fcba6d4480aaf9a2bb",
+    "Xenova/opus-mt-fr-en": "6b166a182780e118c997879d0ad5be4b53671644",
+    "Xenova/opus-mt-fr-es": "962338be51033b548c13dbb6f4da3ddb8c8aba0e",
+    "Xenova/opus-mt-fr-ro": "d2b0db550987291ebb35cdeb3a5394d176ecdcc6",
+    "Xenova/opus-mt-fr-ru": "c2ec732b045ae9f0e1ae1ea9267840145a8c03fe",
+    "Xenova/opus-mt-hi-en": "99853100ed9d932987a6816e8cf7b1b6742a6116",
+    "Xenova/opus-mt-hu-en": "c9e8a7617fb4cb4fca3785c5bb22cfe457cde00b",
+    "Xenova/opus-mt-id-en": "c38ef36c843b71177da903442835c3c870975fd6",
+    "Xenova/opus-mt-it-en": "fd0b89b9c052adc1f2f64152f555aa17353728be",
+    "Xenova/opus-mt-it-es": "0eba27fe6da2208231ea5e7047b0f62241ffc2bb",
+    "Xenova/opus-mt-it-fr": "7843e3b7137d19face19cb32e27256888d26fe84",
+    "Xenova/opus-mt-ja-en": "1a906cfaaf7c8f4193f67f5885c082aa6dbd9d16",
+    "Xenova/opus-mt-ko-en": "d9fa1ac6008242100fecb4fff9b0a5917a1be81f",
+    "Xenova/opus-mt-nl-en": "82c42d95d1508037cbe7172d85fd4e940a2f3584",
+    "Xenova/opus-mt-nl-fr": "6c8ebb68522ffd80cf3474b2dc44f4b5f741e9cd",
+    "Xenova/opus-mt-no-de": "5ed60d1d245fe19d3ee80a172de6b38724642eb3",
+    "Xenova/opus-mt-pl-en": "aeaf0e003b045248c28a559a5ce6027b54ebdaba",
+    "Xenova/opus-mt-ro-fr": "48d206ed25c6f5b869a49a79207906c401334727",
+    "Xenova/opus-mt-ru-en": "afe8c6c738ec81b6d033fd8f44f9678a639a7c67",
+    "Xenova/opus-mt-ru-es": "9cb5fb9fae4d5c0c731dcc3ba74c4e0a87f15eb0",
+    "Xenova/opus-mt-ru-fr": "c7275f468246036d24bfe14ce8465e94e1037af9",
+    "Xenova/opus-mt-ru-uk": "6d8a744ed6eafd9003bcc2b052c4575de0485e1e",
+    "Xenova/opus-mt-sv-en": "63a40256869b6a2b29a84e47aa7f03524fa5f3a2",
+    "Xenova/opus-mt-th-en": "7f0406b972f742985cfbe3be8f61dead9f1bc0e3",
+    "Xenova/opus-mt-tr-en": "78247f41f354af036d24e6f0f8a3d02c6806f96e",
+    "Xenova/opus-mt-uk-en": "b2ad6afa424aaeff8913ff350860b7a7b7c0a607",
+    "Xenova/opus-mt-uk-ru": "cfb76ccd207734d7847679664fa4538cda5ed597",
+    "Xenova/opus-mt-vi-en": "541dfd72ec07e8f1563795e0224408ad8d729e80",
+    "Xenova/opus-mt-xh-en": "f2950c1503ae17ba532ea858914ea13838bc2ab1",
+    "Xenova/opus-mt-zh-en": "39d480d52a9ea3065a1f117adfe4dbc55de10e6f",
+};
+/** Applies PINNED_REVISIONS by the entry's final `repo` (after any override), not by where `pair()` first pointed it. */
+function pinned(entries) {
+    return entries.map((m) => ({
+        ...m,
+        revision: m.repo && !m.repo.startsWith("khabouss/") ? PINNED_REVISIONS[m.repo] : undefined,
+    }));
+}
+export const CATALOG = pinned([
     ...DIRECT.map((p) => {
         const [s, t] = p.split("-");
         return pair(s, t);
@@ -88,7 +171,7 @@ export const CATALOG = [
         sources: ROMANCE,
         targets: ["en"],
     },
-];
+]);
 const asList = (v) => (Array.isArray(v) ? v : [v]).map((x) => parseLocale(x).lang);
 /**
  * A Hugging Face repo id is always exactly "namespace/name". `models.ts` joins it

package/dist/cli.js

@@ -173,7 +173,7 @@ async function cmdTranslate(argv) {
     const singularHints = new Set(sources.flatMap(({ doc }) => [...pluralOneStrings(doc.data)]));
     log(`locale-ai ${VERSION}`);
     log(`Found ${sources.length} ${from.tag} file${sources.length === 1 ? "" : "s"} in ${rel(layout.root)}/` +
-        (layout.kind === "dir" ? `${from.tag}/` : ""));
+        (layout.kind === "dir" ? `${from.tag}${layout.dirSuffix ?? ""}/` : ""));
     let failures = 0;
     const jobs = [];
     for (const locale of targets) {

package/dist/locales.js

@@ -23,6 +23,7 @@ const CANDIDATES = [
     "src/locale",
     "app/i18n",
     "static/locales",
+    "config/locales",
     ".",
 ];
 const isDir = (p) => fs.existsSync(p) && fs.statSync(p).isDirectory();
@@ -31,25 +32,60 @@ const isFile = (p) => fs.existsSync(p) && fs.statSync(p).isFile();
 function fileExtAt(root, from) {
     return EXTENSIONS.find((ext) => isFile(path.join(root, `${from}${ext}`)));
 }
+/**
+ * A prefixed source-locale file, if `<root>/<prefix>_<from><ext>` exists for one of
+ * EXTENSIONS — the convention `flutter gen-l10n` uses by default ("app_en.arb").
+ */
+function prefixedFileAt(root, from) {
+    if (!isDir(root))
+        return undefined;
+    for (const entry of fs.readdirSync(root)) {
+        if (!isFile(path.join(root, entry)))
+            continue;
+        for (const ext of EXTENSIONS) {
+            const suffix = `_${from}${ext}`;
+            if (entry.endsWith(suffix) && entry.length > suffix.length) {
+                return { ext, prefix: entry.slice(0, -suffix.length) };
+            }
+        }
+    }
+    return undefined;
+}
 function layoutAt(root, from) {
     const dir = path.join(root, from);
     if (isDir(dir) && listLocaleFiles(dir).length > 0)
         return { kind: "dir", root };
+    const lprojDir = path.join(root, `${from}.lproj`);
+    if (isDir(lprojDir) && listLocaleFiles(lprojDir).length > 0)
+        return { kind: "dir", root, dirSuffix: ".lproj" };
     const ext = fileExtAt(root, from);
     if (ext)
         return { kind: "file", root, ext };
+    const prefixed = prefixedFileAt(root, from);
+    if (prefixed)
+        return { kind: "file", root, ext: prefixed.ext, prefix: prefixed.prefix };
     return undefined;
 }
+const dirNameFor = (layout, tag) => `${tag}${layout.dirSuffix ?? ""}`;
+const fileNameFor = (layout, tag, ext) => `${layout.prefix ? `${layout.prefix}_` : ""}${tag}${ext}`;
 export function detectLayout(cwd, from, input) {
     if (input) {
         const abs = path.resolve(cwd, input);
         // Accept pointing at the source locale itself: --input locales/en or locales/en.json
         if (path.basename(abs) === from && isDir(abs))
             return { kind: "dir", root: path.dirname(abs) };
+        if (path.basename(abs) === `${from}.lproj` && isDir(abs)) {
+            return { kind: "dir", root: path.dirname(abs), dirSuffix: ".lproj" };
+        }
         const base = path.basename(abs);
         const directExt = EXTENSIONS.find((ext) => base === `${from}${ext}`);
         if (directExt && isFile(abs))
             return { kind: "file", root: path.dirname(abs), ext: directExt };
+        const prefixedExt = EXTENSIONS.find((ext) => base.endsWith(`_${from}${ext}`) && base.length > `_${from}${ext}`.length);
+        if (prefixedExt && isFile(abs)) {
+            const suffix = `_${from}${prefixedExt}`;
+            return { kind: "file", root: path.dirname(abs), ext: prefixedExt, prefix: base.slice(0, -suffix.length) };
+        }
         const l = layoutAt(abs, from);
         if (!l)
             throw new Error(`No "${from}" locale found in ${input} (looked for ${from}/* and ${from}${EXTENSIONS.join("/")}).`);
@@ -76,16 +112,18 @@ function listLocaleFiles(dir) {
     return out.sort();
 }
 export function sourceFiles(layout, from) {
-    return layout.kind === "dir" ? listLocaleFiles(path.join(layout.root, from)) : [path.join(layout.root, `${from}${layout.ext}`)];
+    return layout.kind === "dir"
+        ? listLocaleFiles(path.join(layout.root, dirNameFor(layout, from)))
+        : [path.join(layout.root, fileNameFor(layout, from, layout.ext))];
 }
 /** A translated file is a real PO, never a template, regardless of what the source was. */
 const outputExt = (ext) => (ext === ".pot" ? ".po" : ext);
 export function targetFile(layout, from, to, src) {
     if (layout.kind === "file")
-        return path.join(layout.root, `${to}${outputExt(layout.ext)}`);
-    const rel = path.relative(path.join(layout.root, from), src);
+        return path.join(layout.root, fileNameFor(layout, to, outputExt(layout.ext)));
+    const rel = path.relative(path.join(layout.root, dirNameFor(layout, from)), src);
     const ext = path.extname(rel);
-    return path.join(layout.root, to, ext === ".pot" ? `${rel.slice(0, -ext.length)}.po` : rel);
+    return path.join(layout.root, dirNameFor(layout, to), ext === ".pot" ? `${rel.slice(0, -ext.length)}.po` : rel);
 }
 /** True when `tag` parses as a locale (e.g. "fr", "pt-BR") — rejects names like "__mocks__" or "schemas". */
 function looksLikeLocale(tag) {
@@ -101,19 +139,22 @@ function looksLikeLocale(tag) {
 export function existingTargets(layout, from) {
     if (layout.kind === "file") {
         const ext = layout.ext;
+        const prefix = layout.prefix ? `${layout.prefix}_` : "";
         return fs
             .readdirSync(layout.root)
-            .filter((f) => f.endsWith(ext) && f.slice(0, -ext.length) !== from)
-            .map((f) => f.slice(0, -ext.length))
+            .filter((f) => f.startsWith(prefix) && f.endsWith(ext) && f.slice(prefix.length, -ext.length) !== from)
+            .map((f) => f.slice(prefix.length, -ext.length))
             .filter(looksLikeLocale)
             .sort();
     }
+    const suffix = layout.dirSuffix ?? "";
     return fs
         .readdirSync(layout.root, { withFileTypes: true })
-        .filter((e) => e.isDirectory() && !e.name.startsWith(".") && e.name !== "node_modules" && e.name !== from)
-        .map((e) => e.name)
+        .filter((e) => e.isDirectory() && !e.name.startsWith(".") && e.name !== "node_modules" && e.name.endsWith(suffix))
+        .map((e) => (suffix ? e.name.slice(0, -suffix.length) : e.name))
+        .filter((name) => name !== from)
         .filter(looksLikeLocale)
-        .filter((name) => listLocaleFiles(path.join(layout.root, name)).length > 0)
+        .filter((name) => listLocaleFiles(path.join(layout.root, dirNameFor(layout, name))).length > 0)
         .sort();
 }
 const finalNewline = (text) => text === "" || text.endsWith("\n");

package/dist/models.js

@@ -125,7 +125,7 @@ async function fetchWithRetry(url, attempts = 3) {
 export async function planInstall(m) {
     if (!m.repo)
         throw new Error(`${m.id} is a local model; nothing to download.`);
-    const info = (await getJson(`${HF}/api/models/${m.repo}/revision/main`));
+    const info = (await getJson(`${HF}/api/models/${m.repo}/revision/${m.revision ?? "main"}`));
     const tree = (await getJson(`${HF}/api/models/${m.repo}/tree/${info.sha}?recursive=true`));
     const byPath = new Map(tree.map((e) => [e.path, e]));
     const pick = (names) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locale-ai",
-  "version": "0.6.13",
+  "version": "0.6.15",
   "description": "Translate your i18n locale files locally on CPU. Free, offline, no API keys.",
   "author": "Taha KHABOUSS",
   "repository": {