locale-ai 0.3.0 → 0.4.0

package/README.md

@@ -6,7 +6,7 @@
 [![license](https://img.shields.io/npm/l/locale-ai.svg)](LICENSE)
 [![node](https://img.shields.io/node/v/locale-ai.svg)](package.json)
 
-Translate your i18n JSON locale files **locally, on CPU, for free**. No API keys, no Python, nothing leaves your machine.
+Translate your i18n JSON/YAML locale files **locally, on CPU, for free**. No API keys, no Python, nothing leaves your machine.
 
 ```bash
 npx locale-ai translate --from en --to fr,es,de
@@ -26,7 +26,7 @@ Download 577 MB? (Y/n)
 
 ## What it handles
 
-- **Layouts**: `locales/en/*.json` (nested folders too) → `locales/fr/...`, or `messages/en.json` → `messages/fr.json`. Common folders (`locales`, `src/locales`, `public/locales`, `src/i18n`, `messages`, `lang`, ...) are found automatically, or pass `--input`.
+- **Layouts**: `locales/en/*.json` (nested folders too) → `locales/fr/...`, or `messages/en.json` → `messages/fr.json`. `.json`, `.yaml` and `.yml` are all supported, mixed freely in the same project. Common folders (`locales`, `src/locales`, `public/locales`, `src/i18n`, `messages`, `lang`, ...) are found automatically, or pass `--input`.
 - **Placeholders are never translated or lost**: `{name}`, `{{count}}`, `%s`, `%1$s`, `%(name)s`, `%{name}`, `${name}`, `:attribute`, `$t(key)`, `@:key`, `<b>…</b>`, `<0>…</0>`, `&nbsp;`, URLs, e-mails.
 - **ICU MessageFormat**: `{count, plural, one {# file} other {# files}}`, `select`, nesting. Only the text is translated.
 - **i18next plurals**: adds the forms the target language needs (`_few`, `_many` for Arabic, Russian, Polish...).
@@ -135,6 +135,7 @@ Measured on a sample app (see `test/fixtures`):
 - **Sentence splitting**: OPUS-MT drops sentences from multi-sentence inputs, so each sentence is translated separately.
 - **Placeholder masking** with three marker styles tried in order (`{0}` / `<0>…</0>`, then numbers `900`, then `_P0_`), validating every marker; numbers survive models that mangle braces (en-zh, en-ru). A last-resort fallback translates around placeholders, and anything that needed it is listed for review.
 - **Speed** (20-core CPU, ~1 GB RAM): sample app (43 strings) in ~3 s per language with beam search, ~2 s with `--fast`; full sentences at ~18/s (built-in, beam 4), ~12/s (tc-big, beam 4), ~33/s (tc-big, `--fast`).
+- **Install size**: `onnxruntime-node` bundles binaries for every platform, and on linux/x64 downloads a ~260 MB CUDA provider by default — locale-ai only ever runs on CPU (`executionProviders: ["cpu"]`), so a `postinstall` script ([`scripts/trim-onnxruntime.cjs`](scripts/trim-onnxruntime.cjs)) removes the unused platforms and GPU providers after install, cutting it from ~550 MB to ~45 MB on your current platform.
 
 ## Development
 
@@ -149,7 +150,5 @@ npm run build       # → dist/
 
 ## Next steps
 
-1. **Host the tc-big models** (converted ONNX) on Hugging Face so they can become the built-in defaults instead of a manual conversion. `en-fr` is done ([khabouss/opus-mt-tc-big-en-fr-onnx](https://huggingface.co/khabouss/opus-mt-tc-big-en-fr-onnx)); es, it, pt, ar, ru/uk, tr remain.
-2. **Smaller install.** `onnxruntime-node` is ~550 MB unpacked: it ships binaries for every platform plus a 260 MB CUDA provider we never use.
-3. **More languages**: ja, ko, pl have no usable OPUS-MT model yet; a different model family would be needed.
-4. **More formats**: YAML, `.po`, ARB, Apple `.strings`.
+1. **More languages**: ja, ko, pl have no usable OPUS-MT model yet; a different model family would be needed.
+2. **More formats**: `.po` (gettext), ARB (Flutter), Apple `.strings`. (JSON and YAML are supported today.)

package/dist/cli.js

@@ -9,11 +9,11 @@ import { CATALOG, customEntries, findById, reachableTargets, resolveRoute, } fro
 import { translateStrings } from "./engine.js";
 import { CONFIG_FILE, glossaryLookup, loadConfig, protectRegex } from "./glossary.js";
 import { intlCode, parseLocale } from "./lang.js";
-import { addPluralForms, buildTarget, detectLayout, pendingStrings, readJson, sourceFiles, targetFile, } from "./locales.js";
+import { addPluralForms, buildTarget, detectLayout, pendingStrings, readLocaleDoc, sourceFiles, targetFile, } from "./locales.js";
 import { cacheRoot, install, isInstalled, loadTranslator, localModelProblem, planInstall, readManifest, uninstall, } from "./models.js";
 /** Read from package.json so `npm version` is the single source of truth. */
 const VERSION = JSON.parse(fs.readFileSync(fileURLToPath(new URL("../package.json", import.meta.url)), "utf8")).version;
-const HELP = `locale-ai ${VERSION} — translate i18n JSON locale files locally, on CPU, for free.
+const HELP = `locale-ai ${VERSION} — translate i18n JSON/YAML locale files locally, on CPU, for free.
 
 Usage:
   locale-ai translate --to <locales> [options]
@@ -27,7 +27,7 @@ Usage:
 Translate options:
   --from <locale>       Source locale (default: en)
   --to <locales>        Target locale(s), comma separated: fr,es,pt-BR
-  --input <dir>         Folder containing <from>/ or <from>.json (auto-detected)
+  --input <dir>         Folder containing <from>/ or <from>.json/.yaml/.yml (auto-detected)
   --overwrite           Re-translate strings that already have a translation
   --dry-run             Show what would be translated without downloading or writing
   --fast                Greedy decoding: ~2x faster, slightly lower quality
@@ -122,7 +122,7 @@ async function cmdTranslate(argv) {
     const catalog = catalogFor(config);
     const protect = protectRegex(config);
     const layout = detectLayout(process.cwd(), from.tag, values.input);
-    const sources = sourceFiles(layout, from.tag).map((f) => ({ file: f, doc: readJson(f) }));
+    const sources = sourceFiles(layout, from.tag).map((f) => ({ file: f, doc: readLocaleDoc(f) }));
     log(`locale-ai ${VERSION}`);
     log(`Found ${sources.length} ${from.tag} file${sources.length === 1 ? "" : "s"} in ${rel(layout.root)}/` +
         (layout.kind === "dir" ? `${from.tag}/` : ""));
@@ -139,7 +139,7 @@ async function cmdTranslate(argv) {
         }
         const files = sources.map(({ file, doc }) => {
             const target = targetFile(layout, from.tag, locale.tag, file);
-            const existing = fs.existsSync(target) ? readJson(target).data : undefined;
+            const existing = fs.existsSync(target) ? readLocaleDoc(target).data : undefined;
             return { source: file, target, doc, existing, pending: pendingStrings(doc.data, existing, values.overwrite) };
         });
         const unique = new Set(files.flatMap((f) => f.pending)).size;
@@ -201,7 +201,7 @@ async function cmdTranslate(argv) {
             const built = buildTarget(f.doc.data, f.existing, (s) => translations.get(s) ?? s, values.overwrite);
             const { doc, added } = addPluralForms(built, intlCode(job.locale));
             pluralsAdded += added;
-            const text = JSON.stringify(doc, null, f.doc.indent) + (f.doc.finalNewline ? "\n" : "");
+            const text = f.doc.serialize(doc);
             const before = fs.existsSync(f.target) ? fs.readFileSync(f.target, "utf8") : undefined;
             if (text !== before) {
                 fs.mkdirSync(path.dirname(f.target), { recursive: true });

package/dist/locales.js

@@ -1,5 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
+import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
+const EXTENSIONS = [".json", ".yaml", ".yml"];
 const CANDIDATES = [
     "locales",
     "src/locales",
@@ -24,12 +26,17 @@ const CANDIDATES = [
 ];
 const isDir = (p) => fs.existsSync(p) && fs.statSync(p).isDirectory();
 const isFile = (p) => fs.existsSync(p) && fs.statSync(p).isFile();
+/** The source-locale file's extension, if `<root>/<from><ext>` exists for one of EXTENSIONS. */
+function fileExtAt(root, from) {
+    return EXTENSIONS.find((ext) => isFile(path.join(root, `${from}${ext}`)));
+}
 function layoutAt(root, from) {
     const dir = path.join(root, from);
-    if (isDir(dir) && listJson(dir).length > 0)
+    if (isDir(dir) && listLocaleFiles(dir).length > 0)
         return { kind: "dir", root };
-    if (isFile(path.join(root, `${from}.json`)))
-        return { kind: "file", root };
+    const ext = fileExtAt(root, from);
+    if (ext)
+        return { kind: "file", root, ext };
     return undefined;
 }
 export function detectLayout(cwd, from, input) {
@@ -38,11 +45,13 @@ export function detectLayout(cwd, from, input) {
         // Accept pointing at the source locale itself: --input locales/en or locales/en.json
         if (path.basename(abs) === from && isDir(abs))
             return { kind: "dir", root: path.dirname(abs) };
-        if (path.basename(abs) === `${from}.json` && isFile(abs))
-            return { kind: "file", root: path.dirname(abs) };
+        const base = path.basename(abs);
+        const directExt = EXTENSIONS.find((ext) => base === `${from}${ext}`);
+        if (directExt && isFile(abs))
+            return { kind: "file", root: path.dirname(abs), ext: directExt };
         const l = layoutAt(abs, from);
         if (!l)
-            throw new Error(`No "${from}" locale found in ${input} (looked for ${from}/*.json and ${from}.json).`);
+            throw new Error(`No "${from}" locale found in ${input} (looked for ${from}/*.json and ${from}.json/.yaml/.yml).`);
         return l;
     }
     for (const c of CANDIDATES) {
@@ -50,31 +59,32 @@ export function detectLayout(cwd, from, input) {
         if (l)
             return l;
     }
-    throw new Error(`Could not find "${from}" locale files. Use --input <dir> to point at the folder containing ${from}/ or ${from}.json.`);
+    throw new Error(`Could not find "${from}" locale files. Use --input <dir> to point at the folder containing ${from}/ or ${from}.json/.yaml/.yml.`);
 }
-function listJson(dir) {
+function listLocaleFiles(dir) {
     const out = [];
     for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
         if (e.name.startsWith(".") || e.name === "node_modules")
             continue;
         const p = path.join(dir, e.name);
         if (e.isDirectory())
-            out.push(...listJson(p));
-        else if (e.isFile() && e.name.endsWith(".json"))
+            out.push(...listLocaleFiles(p));
+        else if (e.isFile() && EXTENSIONS.includes(path.extname(e.name).toLowerCase()))
             out.push(p);
     }
     return out.sort();
 }
 export function sourceFiles(layout, from) {
-    return layout.kind === "dir" ? listJson(path.join(layout.root, from)) : [path.join(layout.root, `${from}.json`)];
+    return layout.kind === "dir" ? listLocaleFiles(path.join(layout.root, from)) : [path.join(layout.root, `${from}${layout.ext}`)];
 }
 export function targetFile(layout, from, to, src) {
     if (layout.kind === "file")
-        return path.join(layout.root, `${to}.json`);
+        return path.join(layout.root, `${to}${layout.ext}`);
     return path.join(layout.root, to, path.relative(path.join(layout.root, from), src));
 }
-export function readJson(file) {
-    const text = fs.readFileSync(file, "utf8").replace(/^/, "");
+const finalNewline = (text) => text === "" || text.endsWith("\n");
+const withNewline = (text, keep) => text.replace(/\n*$/, keep ? "\n" : "");
+function readJsonDoc(file, text) {
     let data;
     try {
         data = JSON.parse(text);
@@ -83,14 +93,24 @@ export function readJson(file) {
         throw new Error(`Invalid JSON in ${file}: ${e.message}`);
     }
     const indent = /^[{[]\r?\n([ \t]+)/.exec(text)?.[1] ?? (text.includes("\n") ? 2 : 0);
-    return { data, indent, finalNewline: text.endsWith("\n") };
+    const keepNewline = finalNewline(text);
+    return { data, serialize: (d) => withNewline(JSON.stringify(d, null, indent), keepNewline) };
 }
-export function writeJson(file, data, style) {
-    fs.mkdirSync(path.dirname(file), { recursive: true });
-    const text = JSON.stringify(data, null, style.indent) + (style.finalNewline ? "\n" : "");
-    const tmp = `${file}.tmp-${process.pid}`;
-    fs.writeFileSync(tmp, text);
-    fs.renameSync(tmp, file);
+function readYamlDoc(file, text) {
+    let data;
+    try {
+        data = parseYaml(text) ?? {};
+    }
+    catch (e) {
+        throw new Error(`Invalid YAML in ${file}: ${e.message}`);
+    }
+    const indent = Number(/\n( +)\S/.exec(text)?.[1].length) || 2;
+    const keepNewline = finalNewline(text);
+    return { data, serialize: (d) => withNewline(stringifyYaml(d, { indent }), keepNewline) };
+}
+export function readLocaleDoc(file) {
+    const text = fs.readFileSync(file, "utf8").replace(/^/, "");
+    return /\.ya?ml$/i.test(file) ? readYamlDoc(file, text) : readJsonDoc(file, text);
 }
 const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
 /** Null-prototype object so keys such as "__proto__" are stored as plain data. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "locale-ai",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Translate your i18n locale files locally on CPU. Free, offline, no API keys.",
   "author": "Taha KHABOUSS",
   "repository": {
@@ -17,7 +17,8 @@
   },
   "files": [
     "dist",
-    "scripts/convert-marian.py"
+    "scripts/convert-marian.py",
+    "scripts/trim-onnxruntime.cjs"
   ],
   "engines": {
     "node": ">=22.18"
@@ -28,7 +29,8 @@
     "dev": "node src/cli.ts",
     "test": "node --test test/*.test.ts",
     "test:e2e": "LOCALE_AI_E2E=1 node --test test/e2e.test.ts",
-    "prepublishOnly": "npm run build && npm test"
+    "prepublishOnly": "npm run build && npm test",
+    "postinstall": "node scripts/trim-onnxruntime.cjs"
   },
   "keywords": [
     "i18n",
@@ -43,10 +45,14 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "onnxruntime-node": "^1.30.0"
+    "onnxruntime-node": "^1.30.0",
+    "yaml": "^2.9.1"
   },
   "devDependencies": {
     "@types/node": "^26.6.6",
     "typescript": "^7.0.2"
+  },
+  "allowScripts": {
+    "onnxruntime-node@1.30.0": true
   }
 }

package/scripts/trim-onnxruntime.cjs

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+'use strict';
+
+// locale-ai only ever runs onnxruntime-node with executionProviders: ["cpu"]
+// (see src/marian.ts) and only ever runs on one platform/arch per install.
+// onnxruntime-node still bundles binaries for every platform, and on
+// linux/x64 its own postinstall downloads a 260 MB CUDA provider we never
+// use (see its script/install-metadata.js). This trims all of that after
+// onnxruntime-node has finished installing, shrinking a ~550 MB unpacked
+// dependency down to the one binary locale-ai actually loads.
+//
+// Best-effort: never fails the install, even if onnxruntime-node changes
+// its layout in a future version.
+
+const fs = require('fs');
+const path = require('path');
+
+function rm(p) {
+  try {
+    fs.rmSync(p, { recursive: true, force: true });
+  } catch {
+    // ignore — best-effort cleanup only
+  }
+}
+
+try {
+  const pkgJson = require.resolve('onnxruntime-node/package.json');
+  const binRoot = path.join(path.dirname(pkgJson), 'bin', 'napi-v6');
+  if (!fs.existsSync(binRoot)) process.exit(0);
+
+  const platform = process.platform;
+  const arch = process.arch;
+
+  for (const entry of fs.readdirSync(binRoot)) {
+    const platformDir = path.join(binRoot, entry);
+    if (entry !== platform) {
+      rm(platformDir);
+      continue;
+    }
+    // Same platform: drop every other architecture.
+    for (const archEntry of fs.readdirSync(platformDir)) {
+      if (archEntry !== arch) rm(path.join(platformDir, archEntry));
+    }
+    // Drop GPU execution providers locale-ai never requests.
+    const archDir = path.join(platformDir, arch);
+    for (const name of fs.readdirSync(archDir)) {
+      if (/^libonnxruntime_providers_(cuda|tensorrt|rocm|dml)\./.test(name)) {
+        rm(path.join(archDir, name));
+      }
+    }
+  }
+} catch {
+  // onnxruntime-node not installed, or its layout changed — nothing to do.
+}