@fisharmy100/auto-i18n-cli 1.0.0

package/README.md

@@ -0,0 +1,210 @@
+# auto-i18n-cli
+
+A command-line tool for automatically generating translation databases for use with the [`react-auto-i18n`](https://github.com/your-org/react-auto-i18n) library. It scans your TypeScript source files for all `__t()` and `__tv()` calls and produces a JSON translation database.
+
+**NOTE:** You must have a compatible version of python installed `^3.12`
+
+---
+
+## Installation
+
+```bash
+npm install -D auto-i18n-cli
+```
+
+Then run it directly with `npx`:
+
+```bash
+npx auto-i18n-cli [options]
+```
+
+---
+
+## Usage
+
+```
+auto-i18n-cli -i <input> -b <backend> [options]
+```
+
+### Example — Azure (remote) backend
+
+```bash
+npx auto-i18n-cli \
+  -i "./src" \
+  -o "./assets/translations.json" \
+  -s eng_Latn \
+  -l spa_Latn fra_Latn deu_Latn \
+  -b azure \
+  --azureKey "YOUR_AZURE_KEY"
+```
+
+### Example — NLLB (local) backend
+
+```bash
+npx auto-i18n-cli \
+  -i "./src" \
+  -o "./assets/translations.json" \
+  -l spa_Latn jpn_Jpan \
+  -b nllb
+```
+
+---
+
+## Options
+
+| Flag | Alias | Required | Description |
+|---|---|---|---|
+| `--input` | `-i` | ✅ | Path to a TypeScript file or project directory to scan |
+| `--backend` | `-b` | ✅ | Translation backend: `azure` or `nllb` |
+| `--languages` | `-l` | | One or more target language codes to translate into (e.g. `spa_Latn fra_Latn`) |
+| `--out` | `-o` | | Output file path for the translation JSON (default: `out.json`) |
+| `--sourceLang` | `-s` | | The language your source strings are written in (default: `eng_Latn`) |
+| `--maxTokens` | `-t` | | Maximum tokens per translation chunk — must be ≥ 100 (default: `250`) |
+| `--azureKey` | | ✅ (Azure) | Azure Translator API key |
+| `--azureRegion` | | | Azure Translator region (default: `eastus`) |
+| `--azureEndpoint` | | | Custom Azure Translator endpoint URL |
+| `--nllbModel` | | | NLLB model name (default: `facebook/nllb-200-distilled-1.3B`) |
+| `--multiFile` | `-m` | | Separate each language into its own file with a manifest (default: `false`) |
+| `--help` | `-h` | | Show help message |
+
+---
+
+## Backends
+
+### Azure Translator
+
+Uses Microsoft's Azure Cognitive Services Translator API. Requires an API key.
+
+```bash
+npx auto-i18n-cli -i "./src" -b azure --azureKey "YOUR_KEY" -l spa_Latn
+```
+
+Note: not all NLLB language codes have an Azure equivalent. The CLI will report an error if you specify an unsupported language for the Azure backend.
+
+### NLLB (Local)
+
+Uses Meta's [NLLB-200](https://ai.meta.com/research/no-language-left-behind/) model, running locally via Python. No API key required, but requires a compatible Python environment with the model available.
+
+```bash
+npx auto-i18n-cli -i "./src" -b nllb -l fra_Latn
+```
+
+You can specify a custom model with `--nllbModel`:
+
+```bash
+npx auto-i18n-cli -i "./src" -b nllb --nllbModel "facebook/nllb-200-1.3B" -l fra_Latn
+```
+
+---
+
+## Output
+
+### Single File Format (default)
+
+The CLI produces a JSON file structured as an `I18nDatabase`, with one top-level key per language:
+
+```json
+{
+  "eng_Latn": {
+    "greeting": "Hello!",
+    "farewell": "Goodbye!"
+  },
+  "spa_Latn": {
+    "greeting": "¡Hola!",
+    "farewell": "¡Adiós!"
+  },
+  "fra_Latn": {
+    "greeting": "Bonjour!",
+    "farewell": "Au revoir!"
+  }
+}
+```
+
+This file can be imported directly and passed to `I18nProvider` in your React app:
+
+```tsx
+import db from './assets/translations.json'
+import { I18nProvider, SimpleI18nDb } from 'react-auto-i18n'
+
+<I18nProvider defaultLang="eng_Latn" db={new SimpleI18nDb(db)}>
+  <App />
+</I18nProvider>
+```
+
+Or, you can also load the file if it is in your public folder.
+
+```tsx
+import { I18nFileProvider } from 'react-auto-i18n'
+
+<I18nFileProvider defaultLang="eng_Latn" path="./translations.json">
+  <App />
+</I18nFileProvider>
+```
+
+### Multi-File Format (`--multiFile`)
+
+When using the `--multiFile` (or `-m`) flag, translations are split into separate JSON files organized in a directory with a manifest:
+
+```
+translations/
+├── manifest.json
+├── eng_Latn.json
+├── spa_Latn.json
+└── fra_Latn.json
+```
+
+The `manifest.json` file contains:
+
+```json
+{
+  "langs": ["eng_Latn", "spa_Latn", "fra_Latn"]
+}
+```
+
+Each language file contains only that language's translations:
+
+```json
+{
+  "greeting": "¡Hola!",
+  "farewell": "¡Adiós!"
+}
+```
+
+You can use `I18nMultiFileProvider` with the multi-file format:
+
+```tsx
+import { I18nMultiFileProvider } from 'react-auto-i18n'
+
+<I18nMultiFileProvider 
+  defaultLang="eng_Latn" 
+  path="./translations"
+>
+  <App />
+</I18nMultiFileProvider>
+```
+
+---
+
+## How It Works
+
+1. The CLI scans your TypeScript source files (or project directory) for all `__t()` and `__tv()` calls.
+2. It extracts the key and message string from each call.
+3. It sends those strings to the configured translation backend.
+4. It writes the resulting translations to the output JSON file.
+
+Because extraction is based on static analysis, both arguments to `__t` and `__tv` **must be raw string literals** — not variables or expressions.
+
+```ts
+// ✅ Valid — static string literals
+__t("greeting", "Hello!")
+
+// ❌ Invalid — cannot be statically extracted
+const msg = "Hello!"
+__t("greeting", msg)
+```
+
+---
+
+## Language Codes
+
+Language codes follow the `{lang}_{Script}` format used by the NLLB-200 model (e.g. `eng_Latn`, `spa_Latn`, `cmn_Hans`). See the `react-auto-i18n` documentation for the full list of supported codes.

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/index.js

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+import { runPython } from "./python_ffi.js";
+import { parse } from 'ts-command-line-args';
+import * as fs from 'fs';
+import * as path from 'path';
+import pkg from "../package.json" with { type: 'json' };
+import { nllbToAzure, stringToLanguageCode } from "./langs.js";
+import { parseTSFiles } from "./parser.js";
+import chalk from "chalk";
+import { logError, logMessage } from "./utils.js";
+import { SingleBar } from "cli-progress";
+const DEFAULT_MAX_TOKENS = 250;
+async function main() {
+    let options;
+    try {
+        options = parse({
+            input: {
+                type: String,
+                alias: 'i',
+                description: 'The input typescript file or typescript project directory'
+            },
+            languages: {
+                type: String,
+                alias: 'l',
+                multiple: true,
+                optional: true,
+                description: 'The languages to generate translations for'
+            },
+            out: {
+                type: String,
+                alias: 'o',
+                optional: true,
+                description: 'The output file path'
+            },
+            sourceLang: {
+                type: String,
+                alias: 's',
+                optional: true,
+                description: 'The source language that all __t use. Defaults to English'
+            },
+            maxTokens: {
+                type: Number,
+                alias: 't',
+                optional: true,
+                description: `The maximum number of tokens. Defaults to ${DEFAULT_MAX_TOKENS}`
+            },
+            backend: {
+                type: String,
+                alias: 'b',
+                description: 'Translation backend to use: "nllb" or "azure"',
+            },
+            azureKey: {
+                type: String,
+                optional: true,
+                description: 'Azure Translator API key (required when backend is "azure")',
+            },
+            azureRegion: {
+                type: String,
+                optional: true,
+                description: 'Azure Translator region (default: eastus)',
+            },
+            azureEndpoint: {
+                type: String,
+                optional: true,
+                description: 'Azure Translator endpoint URL',
+            },
+            nllbModel: {
+                type: String,
+                optional: true,
+                description: 'NLLB model name (default: facebook/nllb-200-distilled-1.3B)',
+            },
+            multiFile: {
+                type: Boolean,
+                optional: true,
+                alias: 'm',
+                description: 'If used will separate each language into its own file with a manifest'
+            },
+            help: {
+                type: Boolean,
+                optional: true,
+                alias: 'h',
+                description: 'Show this help message'
+            },
+        }, {
+            helpArg: 'help',
+            headerContentSections: [
+                {
+                    header: 'auto-i18n-cli',
+                    content: `A translation database generator library for the react-auto-i18n library\n\nVersion: ${pkg.version}`
+                }
+            ],
+        });
+    }
+    catch (error) {
+        logError("Console Error:\n");
+        console.log(error);
+        return;
+    }
+    runWithOptions(options);
+}
+async function runWithOptions(options) {
+    logMessage("Running command...");
+    const validated = validateProgramOptions(options);
+    if (!validated)
+        return;
+    const { input, output, translateArgs } = validated;
+    logMessage("Parsing TS files...");
+    const result = await parseTSFiles(input);
+    if (result.type === "error") {
+        logError("Error:\n" + result.value.join("\n\n"));
+        return;
+    }
+    const segments = Object.values(result.value)
+        .filter(v => v !== undefined)
+        .reduce((acc, v) => {
+        acc[v.key] = v.message;
+        return acc;
+    }, {});
+    translateArgs.segments = segments;
+    logMessage("Translating...");
+    let success = await generateTranslationFile(translateArgs, output, options.multiFile ?? false);
+    if (!success) {
+        logError("Translation Failed");
+        return;
+    }
+    const fullOutPath = path.resolve(output);
+    logMessage(`Extraction & Translation complete! Translations located at: ${fullOutPath}`);
+}
+function validateProgramOptions(options) {
+    const inputPath = path.resolve(options.input);
+    if (!fs.existsSync(inputPath)) {
+        logError(`Error: file path '${chalk.bold(inputPath)}' does not exist`);
+        return null;
+    }
+    if (options.backend !== "nllb" && options.backend !== "azure") {
+        logError(`Error: backend must be ${chalk.bold("nllb")} or ${chalk.bold("azure")}`);
+        return null;
+    }
+    const langs = [];
+    const option_langs = options.languages ?? [];
+    for (let lang of option_langs) {
+        const code = stringToLanguageCode(lang);
+        if (!code) {
+            logError(`Error: language code ${chalk.bold(lang)} is not valid`);
+            return null;
+        }
+        if (options.backend === "azure" && nllbToAzure(code) === null) {
+            logError(`Error: language code ${chalk.bold(lang)} has no Azure equivalent`);
+            return null;
+        }
+        langs.push(code);
+    }
+    let src_lang = "eng_Latn";
+    if (options.sourceLang) {
+        const code = stringToLanguageCode(options.sourceLang);
+        if (!code) {
+            logError(`Error: language code ${chalk.bold(options.sourceLang)} is not valid`);
+            return null;
+        }
+        src_lang = code;
+    }
+    let max_tokens = options.maxTokens;
+    if (max_tokens === undefined) {
+        max_tokens = DEFAULT_MAX_TOKENS;
+    }
+    if (max_tokens < 100) {
+        logError(`Error: maxTokens cannot be less than 100`);
+        return null;
+    }
+    let translateArgs;
+    if (options.backend === "azure") {
+        if (!options.azureKey) {
+            logError(`Error: ${chalk.bold("--azureKey")} is required when using the Azure backend`);
+            return null;
+        }
+        translateArgs = {
+            langs,
+            src_lang,
+            segments: {},
+            max_tokens,
+            backend: "azure",
+            azure_key: options.azureKey,
+            ...(options.azureRegion && { azure_region: options.azureRegion }),
+            ...(options.azureEndpoint && { azure_endpoint: options.azureEndpoint }),
+        };
+    }
+    else {
+        translateArgs = {
+            langs,
+            src_lang,
+            segments: {},
+            max_tokens,
+            backend: "nllb",
+            ...(options.nllbModel && { nllb_model: options.nllbModel }),
+        };
+    }
+    const output = options.out ?? (options.multiFile ? "translations" : "translations.json");
+    return { input: inputPath, output, translateArgs };
+}
+async function generateTranslationFile(args, out, multiFile) {
+    const bar = new SingleBar({
+        format: chalk.green(`Generation Progress |${chalk.bold("{bar}")}| {percentage}% || {value}/{total} Chunks || ETA: {eta_formatted}`),
+        barCompleteChar: '\u2588',
+        barCompleteString: '\u2591',
+        hideCursor: true,
+    });
+    let bar_started = false;
+    let is_error = false;
+    await runPython(args, (p) => {
+        if (!bar_started) {
+            bar.start(args.langs.length * Object.values(args.segments).length, 0);
+            bar_started = true;
+        }
+        bar.update(p.current);
+    })
+        .then(o => {
+        if (multiFile) {
+            fs.mkdirSync(out, { recursive: true });
+            Object.entries(o.values).forEach(([k, v]) => {
+                const filePath = path.join(out, `${k}.json`);
+                fs.writeFileSync(filePath, JSON.stringify(v, null, 2));
+            });
+            const langs = Object.keys(o.values);
+            const manifestData = { langs };
+            const manifestPath = path.join(out, "manifest.json");
+            fs.writeFileSync(manifestPath, JSON.stringify(manifestData, null, 2));
+        }
+        else {
+            fs.mkdirSync(path.dirname(out), { recursive: true });
+            fs.writeFileSync(out, JSON.stringify(o.values, null, 2));
+        }
+    })
+        .catch(e => {
+        is_error = true;
+        bar.stop();
+        logError(e);
+    })
+        .finally(() => {
+        bar.stop();
+    });
+    return !is_error;
+}
+main();

package/dist/langs.d.ts

@@ -0,0 +1,8 @@
+export declare const LANGUAGE_CODES: readonly ["ace_Arab", "ace_Latn", "acm_Arab", "acq_Arab", "aeb_Arab", "afr_Latn", "als_Latn", "amh_Ethi", "apc_Arab", "arb_Arab", "arb_Latn", "arg_Latn", "ars_Arab", "ary_Arab", "arz_Arab", "asm_Beng", "ast_Latn", "awa_Deva", "ayr_Latn", "azb_Arab", "azj_Latn", "bak_Cyrl", "bam_Latn", "ban_Latn", "bel_Cyrl", "bem_Latn", "ben_Beng", "bho_Deva", "bjn_Arab", "bjn_Latn", "bod_Tibt", "bos_Latn", "brx_Deva", "bug_Latn", "bul_Cyrl", "cat_Latn", "ceb_Latn", "ces_Latn", "chv_Cyrl", "cjk_Latn", "ckb_Arab", "cmn_Hans", "cmn_Hant", "crh_Latn", "cym_Latn", "dan_Latn", "dar_Cyrl", "deu_Latn", "dgo_Deva", "dik_Latn", "dyu_Latn", "dzo_Tibt", "ekk_Latn", "ell_Grek", "eng_Latn", "epo_Latn", "eus_Latn", "ewe_Latn", "fao_Latn", "fij_Latn", "fil_Latn", "fin_Latn", "fon_Latn", "fra_Latn", "fur_Latn", "fuv_Latn", "gaz_Latn", "gla_Latn", "gle_Latn", "glg_Latn", "gom_Deva", "gug_Latn", "guj_Gujr", "hat_Latn", "hau_Latn", "heb_Hebr", "hin_Deva", "hne_Deva", "hrv_Latn", "hun_Latn", "hye_Armn", "ibo_Latn", "ilo_Latn", "ind_Latn", "isl_Latn", "ita_Latn", "jav_Latn", "jpn_Jpan", "kaa_Latn", "kab_Latn", "kac_Latn", "kam_Latn", "kan_Knda", "kas_Arab", "kas_Deva", "kat_Geor", "kaz_Cyrl", "kbp_Latn", "kea_Latn", "khk_Cyrl", "khm_Khmr", "kik_Latn", "kin_Latn", "kir_Cyrl", "kmb_Latn", "kmr_Latn", "knc_Arab", "knc_Latn", "kor_Hang", "ktu_Latn", "lao_Laoo", "lij_Latn", "lim_Latn", "lin_Latn", "lit_Latn", "lld_Latn", "lmo_Latn", "ltg_Latn", "ltz_Latn", "lua_Latn", "lug_Latn", "luo_Latn", "lus_Latn", "lvs_Latn", "mag_Deva", "mai_Deva", "mal_Mlym", "mar_Deva", "mfe_Latn", "mhr_Cyrl", "min_Arab", "min_Latn", "mkd_Cyrl", "mlt_Latn", "mni_Beng", "mni_Mtei", "mos_Latn", "mri_Latn", "mya_Mymr", "myv_Cyrl", "nld_Latn", "nno_Latn", "nob_Latn", "npi_Deva", "nqo_Nkoo", "nso_Latn", "nus_Latn", "nya_Latn", "oci_Latn", "ory_Orya", "pag_Latn", "pan_Guru", "pap_Latn", "pbt_Arab", "pes_Arab", "plt_Latn", "pol_Latn", "por_Latn", "prs_Arab", "quy_Latn", "ron_Latn", "run_Latn", "rus_Cyrl", "sag_Latn", "san_Deva", "sat_Olck", "scn_Latn", "shn_Mymr", "sin_Sinh", "slk_Latn", "slv_Latn", "smo_Latn", "sna_Latn", "snd_Arab", "snd_Deva", "som_Latn", "sot_Latn", "spa_Latn", "srd_Latn", "srp_Cyrl", "ssw_Latn", "sun_Latn", "swe_Latn", "swh_Latn", "szl_Latn", "tam_Taml", "taq_Latn", "taq_Tfng", "tat_Cyrl", "tel_Telu", "tgk_Cyrl", "tha_Thai", "tir_Ethi", "tpi_Latn", "tsn_Latn", "tso_Latn", "tuk_Latn", "tum_Latn", "tur_Latn", "twi_Latn", "tyv_Cyrl", "uig_Arab", "ukr_Cyrl", "umb_Latn", "urd_Arab", "uzn_Latn", "uzs_Arab", "vec_Latn", "vie_Latn", "vmw_Latn", "war_Latn", "wol_Latn", "wuu_Hans", "xho_Latn", "ydd_Hebr", "yor_Latn", "yue_Hant", "zgh_Tfng", "zsm_Latn", "zul_Latn"];
+export type LanguageCode = typeof LANGUAGE_CODES[number];
+export declare function stringToLanguageCode(str: string): LanguageCode | null;
+/**
+ * Converts an NLLB LanguageCode to an Azure Translator BCP-47 language tag.
+ * Returns null if there is no Azure equivalent for the given code.
+ */
+export declare function nllbToAzure(code: LanguageCode): string | null;