@liberfi.io/i18n 0.1.231 → 0.1.233

package/script/translateLocales.mjs

@@ -0,0 +1,239 @@
+// One-shot locale translation script for @liberfi.io/i18n.
+//
+// Translates the canonical English bundle (locales/en.json) into the target
+// languages using an OpenAI-compatible chat endpoint (default: arouter.ai /
+// gpt-4o, matching the backend i18n pipeline choice in the worldcup i18n plan).
+//
+// Design notes:
+// - The API key is read from env (I18N_LLM_API_KEY / OPENAI_API_KEY); it is
+//   NEVER hardcoded. The key is injected at run time, same as the backend.
+// - Resumable: an existing locales/<lang>.json is loaded and only MISSING keys
+//   are translated, so re-runs are cheap and partial progress is preserved.
+// - Batched: strings are translated in chunks (JSON array in -> array out).
+//   On a length mismatch the batch is split in half and retried, so a single
+//   bad response never corrupts the whole file.
+// - Placeholders ({{x}}, {x}, <...>, %s, \n, numbers) must be preserved; the
+//   prompt enforces this and we keep the English key order.
+//
+// Usage:
+//   I18N_LLM_API_KEY=... node script/translateLocales.mjs
+//   I18N_LLM_API_KEY=... LANGS=ja,ko node script/translateLocales.mjs
+//   I18N_LLM_API_KEY=... PREFIXES=predict,scaffold,common node script/translateLocales.mjs
+
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const localesDir = join(__dirname, "..", "locales");
+
+const API_KEY = process.env.I18N_LLM_API_KEY || process.env.OPENAI_API_KEY;
+const BASE_URL = process.env.I18N_LLM_BASE_URL || "https://api.arouter.ai/v1";
+const MODEL = process.env.I18N_LLM_MODEL || "gpt-4o";
+const BATCH_SIZE = Number(process.env.BATCH_SIZE || 30);
+const CONCURRENCY = Number(process.env.CONCURRENCY || 4);
+const MAX_RETRIES = Number(process.env.MAX_RETRIES || 4);
+
+// Target languages (product set minus en; zh-Hans is a copy of zh.json and is
+// not regenerated here). Display names guide the model's dialect/script.
+const DEFAULT_LANGS = {
+  "zh-Hant": "Traditional Chinese (Taiwan/Hong Kong conventions, 繁體中文)",
+  ja: "Japanese",
+  ko: "Korean",
+  th: "Thai",
+  vi: "Vietnamese",
+  fr: "French",
+  de: "German",
+  it: "Italian",
+  es: "Spanish",
+  pt: "Portuguese (Brazil)",
+  ru: "Russian",
+};
+
+if (!API_KEY) {
+  console.error(
+    "[translateLocales] Missing I18N_LLM_API_KEY (or OPENAI_API_KEY). Aborting.",
+  );
+  process.exit(1);
+}
+
+const langFilter = (process.env.LANGS || "")
+  .split(",")
+  .map((s) => s.trim())
+  .filter(Boolean);
+const prefixFilter = (process.env.PREFIXES || "")
+  .split(",")
+  .map((s) => s.trim())
+  .filter(Boolean);
+
+const langs = langFilter.length
+  ? Object.fromEntries(
+      Object.entries(DEFAULT_LANGS).filter(([code]) =>
+        langFilter.includes(code),
+      ),
+    )
+  : DEFAULT_LANGS;
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+function loadJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+function stripFences(text) {
+  let t = text.trim();
+  if (t.startsWith("```")) {
+    t = t.replace(/^```[a-zA-Z]*\n?/, "").replace(/```$/, "").trim();
+  }
+  return t;
+}
+
+async function chat(messages) {
+  const res = await fetch(`${BASE_URL}/chat/completions`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${API_KEY}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ model: MODEL, temperature: 0, messages }),
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => "");
+    throw new Error(`HTTP ${res.status}: ${body.slice(0, 300)}`);
+  }
+  const data = await res.json();
+  return data.choices?.[0]?.message?.content ?? "";
+}
+
+function buildPrompt(languageName, values) {
+  const system = [
+    "You are a professional UI string localizer for a crypto trading and",
+    "prediction-market web app. Translate from English to",
+    `${languageName}.`,
+    "Rules:",
+    "- Return ONLY a JSON array of strings, same length and order as the input.",
+    "- Do not add, drop, merge, reorder, or explain anything.",
+    "- Preserve ALL placeholders verbatim: {{var}}, {var}, %s, %d, <tag>...</tag>,",
+    "  HTML entities, line breaks (\\n), URLs, and leading/trailing whitespace.",
+    "- Do not translate brand names, ticker symbols, or numbers.",
+    "- Keep the tone concise and idiomatic for app UI labels.",
+  ].join(" ");
+  const user = JSON.stringify(values);
+  return [
+    { role: "system", content: system },
+    { role: "user", content: user },
+  ];
+}
+
+async function translateBatch(languageName, values, depth = 0) {
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      const content = await chat(buildPrompt(languageName, values));
+      const parsed = JSON.parse(stripFences(content));
+      if (Array.isArray(parsed) && parsed.length === values.length) {
+        return parsed.map((v, i) => (typeof v === "string" ? v : values[i]));
+      }
+      throw new Error(
+        `length mismatch: got ${Array.isArray(parsed) ? parsed.length : "non-array"}, want ${values.length}`,
+      );
+    } catch (err) {
+      const backoff = 500 * Math.pow(2, attempt);
+      console.warn(
+        `[translateLocales]   retry ${attempt + 1}/${MAX_RETRIES} (${languageName}, n=${values.length}): ${err.message}`,
+      );
+      await sleep(backoff);
+    }
+  }
+  // Split-and-retry to isolate the bad chunk; single items fall back to English.
+  if (values.length > 1 && depth < 6) {
+    const mid = Math.floor(values.length / 2);
+    const left = await translateBatch(languageName, values.slice(0, mid), depth + 1);
+    const right = await translateBatch(languageName, values.slice(mid), depth + 1);
+    return [...left, ...right];
+  }
+  console.error(
+    `[translateLocales]   giving up on ${values.length} item(s) for ${languageName}; keeping English.`,
+  );
+  return values;
+}
+
+async function runPool(tasks, concurrency) {
+  const results = new Array(tasks.length);
+  let next = 0;
+  async function worker() {
+    while (next < tasks.length) {
+      const i = next++;
+      results[i] = await tasks[i]();
+    }
+  }
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, tasks.length) }, worker),
+  );
+  return results;
+}
+
+async function translateLang(code, languageName, en) {
+  const outPath = join(localesDir, `${code}.json`);
+  const existing = existsSync(outPath) ? loadJson(outPath) : {};
+
+  let keys = Object.keys(en);
+  if (prefixFilter.length) {
+    keys = keys.filter((k) => prefixFilter.some((p) => k.startsWith(p)));
+  }
+  const missing = keys.filter(
+    (k) => typeof existing[k] !== "string" || existing[k] === "",
+  );
+
+  console.log(
+    `[translateLocales] ${code} (${languageName}): ${missing.length} missing / ${keys.length} keys`,
+  );
+  if (!missing.length) {
+    return { code, translated: 0, total: keys.length };
+  }
+
+  const batches = [];
+  for (let i = 0; i < missing.length; i += BATCH_SIZE) {
+    batches.push(missing.slice(i, i + BATCH_SIZE));
+  }
+
+  let done = 0;
+  const tasks = batches.map((batchKeys) => async () => {
+    const values = batchKeys.map((k) => en[k]);
+    const out = await translateBatch(languageName, values);
+    batchKeys.forEach((k, i) => {
+      existing[k] = out[i];
+    });
+    done += batchKeys.length;
+    process.stdout.write(
+      `\r[translateLocales]   ${code}: ${done}/${missing.length}   `,
+    );
+  });
+
+  await runPool(tasks, CONCURRENCY);
+  process.stdout.write("\n");
+
+  // Write keys in the English file's order for stable diffs.
+  const ordered = {};
+  for (const k of Object.keys(en)) {
+    if (typeof existing[k] === "string") ordered[k] = existing[k];
+  }
+  writeFileSync(outPath, JSON.stringify(ordered, null, 2) + "\n", "utf8");
+  console.log(`[translateLocales] wrote ${outPath}`);
+  return { code, translated: missing.length, total: keys.length };
+}
+
+async function main() {
+  const en = loadJson(join(localesDir, "en.json"));
+  console.log(
+    `[translateLocales] model=${MODEL} base=${BASE_URL} batch=${BATCH_SIZE} concurrency=${CONCURRENCY}`,
+  );
+  for (const [code, name] of Object.entries(langs)) {
+    await translateLang(code, name, en);
+  }
+  console.log("[translateLocales] done.");
+}
+
+main().catch((err) => {
+  console.error("[translateLocales] fatal:", err);
+  process.exit(1);
+});