@remvst/localization 1.0.2 → 2.0.0

package/.prettierignore

@@ -0,0 +1 @@
+testOut/

package/README.md

@@ -14,13 +14,16 @@ Create a JSON file with your localized strings (`my-localization.json`):
 
 ```json
 {
-    "ok": { "en": "Okay" },
-    "confirm": { "en": "Confirm" },
-    "cancel": { "en": "Cancel" },
-    "back": { "en": "Back" },
-    "play": { "en": "Play" },
-    "time": { "en": "Time" },
-    "backToMainMenu": { "en": "Back To Main Menu", "fr": "Retour au menu principal" }
+  "ok": { "en": "Okay" },
+  "confirm": { "en": "Confirm" },
+  "cancel": { "en": "Cancel" },
+  "back": { "en": "Back" },
+  "play": { "en": "Play" },
+  "time": { "en": "Time" },
+  "backToMainMenu": {
+    "en": "Back To Main Menu",
+    "fr": "Retour au menu principal"
+  }
 }
 ```
 

package/lib/commands/find-strings.d.ts

@@ -0,0 +1,5 @@
+export declare function findStringsCommand(options: {
+    in: string;
+    out: string;
+    localizeFunctionName: string;
+}): Promise<void>;

package/lib/commands/find-strings.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.findStringsCommand = void 0;
+const fs_1 = require("fs");
+const glob_1 = require("glob");
+async function findStringsCommand(options) {
+    const jsfiles = await (0, glob_1.glob)(options.in);
+    const strings = new Set();
+    for (const file of jsfiles) {
+        const content = await fs_1.promises.readFile(file, "utf-8");
+        const matches = [
+            ...content.matchAll(new RegExp(`${options.localizeFunctionName}\\('([^']+)'\\)`, "g")),
+            ...content.matchAll(new RegExp(`${options.localizeFunctionName}\\("([^"]+)"\\)`, "g")),
+            ...content.matchAll(new RegExp(`${options.localizeFunctionName}\\(\`([^"]+)\`\\)`, "g")),
+        ];
+        for (const [_, string] of matches) {
+            strings.add(string);
+        }
+    }
+    const outJson = {};
+    for (const string of Array.from(strings).sort()) {
+        outJson[string] = string;
+    }
+    await fs_1.promises.writeFile(options.out, JSON.stringify(outJson, null, 4));
+}
+exports.findStringsCommand = findStringsCommand;

package/lib/commands/localize.d.ts

@@ -0,0 +1,9 @@
+export declare function googleTranslate(text: string, from: string, to: string): Promise<string>;
+export declare function localize(options: {
+    sourceJson: string;
+    sourceLocale: string;
+    destinationJson: string;
+    destinationLocale: string;
+    context?: string;
+    engine?: string;
+}): Promise<void>;

package/lib/commands/localize.js

@@ -0,0 +1,67 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.localize = exports.googleTranslate = void 0;
+const dotenv_1 = require("dotenv");
+const openai_1 = __importDefault(require("openai"));
+const fs_1 = require("fs");
+const google_translate_api_1 = require("@vitalets/google-translate-api");
+(0, dotenv_1.config)();
+async function googleTranslate(text, from, to) {
+    const result = await (0, google_translate_api_1.translate)(text, { from, to });
+    return result.text;
+}
+exports.googleTranslate = googleTranslate;
+async function gptTranslate(text, from, to, context) {
+    const openai = new openai_1.default({ apiKey: process.env.OPENAI_API_KEY });
+    const prompt = `Translate the following text from ${from} to ${to}.` +
+        (context ? ` Context: ${context}` : "") +
+        `\nText: ${text}`;
+    const completion = await openai.chat.completions.create({
+        model: "gpt-3.5-turbo",
+        messages: [
+            {
+                role: "system",
+                content: `
+          You are a helpful translation assistant.
+          Only return the string that would be used for translation, so it can be put in the game as is (verbatim).
+          `.trim(),
+            },
+            { role: "user", content: prompt },
+        ],
+        max_tokens: 1000,
+        temperature: 0.3,
+    });
+    return completion.choices?.[0]?.message?.content?.trim() || "";
+}
+async function localize(options) {
+    let existingJson = {};
+    try {
+        existingJson = JSON.parse(await fs_1.promises.readFile(options.destinationJson, "utf-8"));
+    }
+    catch (error) { }
+    const allStrings = JSON.parse(await fs_1.promises.readFile(options.sourceJson, "utf-8"));
+    const outJson = {};
+    console.log("Translating strings from", options.sourceLocale, "to", options.destinationLocale);
+    try {
+        for (const key of Object.keys(allStrings).sort()) {
+            if (existingJson[key]) {
+                outJson[key] = existingJson[key];
+                continue;
+            }
+            if (options.engine === "gpt") {
+                outJson[key] = await gptTranslate(key, options.sourceLocale, options.destinationLocale, options.context);
+            }
+            else {
+                outJson[key] = await googleTranslate(key, options.sourceLocale, options.destinationLocale);
+            }
+            console.log(`${key} => ${outJson[key]}`);
+        }
+    }
+    finally {
+        await fs_1.promises.writeFile(options.destinationJson, JSON.stringify(outJson, null, 4));
+    }
+}
+exports.localize = localize;

package/lib/commands/to-typescript.d.ts

@@ -1,4 +1,5 @@
 export declare function toTypescriptCommand(options: {
-    in: string;
+    defaultLocalization: string;
     out: string;
+    localizeFunctionName: string;
 }): Promise<void>;

package/lib/commands/to-typescript.js

@@ -1,16 +1,21 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.toTypescriptCommand = void 0;
-const path_1 = __importDefault(require("path"));
-const translation_set_1 = require("../model/translation-set");
 const fs_1 = require("fs");
 async function toTypescriptCommand(options) {
-    const sourceJson = JSON.parse(await fs_1.promises.readFile(options.in, 'utf-8'));
-    const outTranslations = translation_set_1.TranslationSet.fromJSON(sourceJson);
-    await fs_1.promises.mkdir(path_1.default.dirname(options.out), { recursive: true });
-    await fs_1.promises.writeFile(options.out, outTranslations.toTypeScript());
+    const allStringsJson = JSON.parse(await fs_1.promises.readFile(options.defaultLocalization, "utf-8"));
+    let ts = "";
+    ts += `export type LocalizedKey = ${Object.keys(allStringsJson)
+        .map((key) => JSON.stringify(key))
+        .join(" | ")};\n\n`;
+    ts += `export type Localization = {[key in LocalizedKey]: string};\n\n`;
+    ts += `let LOCALIZATION: Localization = ${JSON.stringify(allStringsJson)};\n\n`;
+    ts += `export function setLocalization(localization: Localization) {\n`;
+    ts += `    LOCALIZATION = localization;\n`;
+    ts += `}\n\n`;
+    ts += `export function ${options.localizeFunctionName}(key: string) {\n`;
+    ts += `    return LOCALIZATION[key] || key;\n`;
+    ts += `}\n`;
+    await fs_1.promises.writeFile(options.out, ts);
 }
 exports.toTypescriptCommand = toTypescriptCommand;

package/lib/index.js

@@ -4,120 +4,89 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const helpers_1 = require("yargs/helpers");
 const yargs_1 = __importDefault(require("yargs"));
-const combine_json_1 = require("./commands/combine-json");
-const google_translate_1 = require("./commands/google-translate");
-const parse_csv_1 = require("./commands/parse-csv");
+const helpers_1 = require("yargs/helpers");
+const find_strings_1 = require("./commands/find-strings");
+const localize_1 = require("./commands/localize");
 const to_typescript_1 = require("./commands/to-typescript");
 async function main() {
     await (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
-        .command('parse-csv', 'convert a CSV into a JSON', async (yargs) => {
+        .command("find-strings", "Finds all localizable strings in a source file", async (yargs) => {
         const argv = await yargs
-            .option('in', {
-            type: 'string',
-            alias: 'i',
-            describe: 'Path to CSV to read from',
+            .option("in", {
+            type: "string",
+            alias: "i",
+            describe: "Folder to read from",
         })
-            .option('out', {
-            type: 'string',
-            alias: 'o',
-            describe: 'Output file',
+            .options("out", {
+            type: "string",
+            alias: "o",
+            describe: "Output JSON file",
         })
-            .option('languagesLineIndex', {
-            type: 'number',
-            alias: 'l',
-            describe: 'Index of the row containing the languages',
+            .options("localize-function-name", {
+            type: "string",
+            describe: "Name of the function used for localization",
         })
-            .demandOption('in')
-            .demandOption('out')
-            .demandOption('languagesLineIndex')
-            .argv;
-        await (0, parse_csv_1.parseCsvCommand)(argv);
+            .demandOption("in")
+            .demandOption("out")
+            .demandOption("localize-function-name").argv;
+        await (0, find_strings_1.findStringsCommand)(argv);
     })
-        .command('combine-json', 'combine multiple JSON files', async (yargs) => {
+        .command("localize", "Creates a localization JSON file", async (yargs) => {
         const argv = await yargs
-            .option('main', {
-            type: 'string',
-            alias: 'i',
-            describe: 'JSON file to read from',
-        })
-            .option('out', {
-            type: 'string',
-            alias: 'o',
-            describe: 'Output JSON file',
+            .option("source-json", {
+            type: "string",
+            describe: "File that contains all strings in the original language",
         })
-            .option('fallbackLocale', {
-            type: 'string',
-            describe: 'Default locale to backfill translations',
-            default: 'en',
+            .option("source-locale", {
+            type: "string",
+            describe: "Locale to translate from",
         })
-            .option('fallback', {
-            type: 'string',
-            alias: 'f',
-            describe: 'Fallback JSON file to read from',
+            .options("destination-json", {
+            type: "string",
+            alias: "o",
+            describe: "Output JSON file",
         })
-            .option('locale', {
-            type: 'string',
-            alias: 'l',
-            describe: 'Locale to include',
+            .option("destination-locale", {
+            type: "string",
+            describe: "Locale to translate to",
         })
-            .array('fallback')
-            .array('locale')
-            .demandOption('main')
-            .demandOption('out')
-            .demandOption('locale')
-            .demandOption('fallback')
-            .argv;
-        await (0, combine_json_1.combineJsonCommand)(argv);
-    })
-        .command('to-typescript', 'converts a JSON file into a TypeScript file', async (yargs) => {
-        const argv = await yargs
-            .option('in', {
-            type: 'string',
-            alias: 'i',
-            describe: 'JSON file to read from',
+            .option("context", {
+            type: "string",
+            describe: "Context to provide to the translation engine",
         })
-            .options('out', {
-            type: 'string',
-            alias: 'o',
-            describe: 'Output Typescript file',
+            .option("engine", {
+            type: "string",
+            choices: ["gpt", "google"],
+            default: "google",
+            describe: "Translation engine to use",
         })
-            .demandOption('in')
-            .demandOption('out')
-            .argv;
-        await (0, to_typescript_1.toTypescriptCommand)(argv);
+            .demandOption("source-json")
+            .demandOption("source-locale")
+            .demandOption("destination-json")
+            .demandOption("destination-locale").argv;
+        await (0, localize_1.localize)(argv);
     })
-        .command('google-translate', 'adds missing translations to a JSON file using Google translate', async (yargs) => {
+        .command("to-typescript", "Creates a TypeScript file that can be then imported to localize a project", async (yargs) => {
         const argv = await yargs
-            .option('in', {
-            type: 'string',
-            alias: 'i',
-            describe: 'JSON file to read from',
-        })
-            .option('out', {
-            type: 'string',
-            alias: 'o',
-            describe: 'Output JSON file',
+            .option("default-localization", {
+            type: "string",
+            alias: "i",
+            describe: "Default localization JSON file",
         })
-            .option('fallbackLocale', {
-            type: 'string',
-            describe: 'Default locale to backfill translations',
-            default: 'en',
+            .options("out", {
+            type: "string",
+            alias: "o",
+            describe: "Output Typescript file",
         })
-            .option('locale', {
-            type: 'string',
-            alias: 'l',
-            describe: 'Locale to include',
+            .options("localize-function-name", {
+            type: "string",
+            describe: "Name of the function used for localization",
         })
-            .array('fallback')
-            .array('locale')
-            .demandOption('in')
-            .demandOption('out')
-            .demandOption('locale')
-            .argv;
-        await (0, google_translate_1.googleTranslateCommand)(argv);
-    })
-        .argv;
+            .demandOption("default-localization")
+            .demandOption("out")
+            .demandOption("localize-function-name").argv;
+        await (0, to_typescript_1.toTypescriptCommand)(argv);
+    }).argv;
 }
 main();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remvst/localization",
-  "version": "1.0.2",
+  "version": "2.0.0",
   "description": "",
   "main": "lib/index.js",
   "bin": {
@@ -9,21 +9,26 @@
   "scripts": {
     "build": "rm -rf lib && tsc",
     "prepublishOnly": "npm i && npm run build",
-    "test:polyglot": "ts-node src/index.ts parse-csv -i test/polyglot.csv -o testOut/polyglot.json -l 1",
-    "test:combine-json": "ts-node src/index.ts combine-json --main=test/localization.json --out=testOut/localization-combined.json --fallback=testOut/polyglot.json -l en -l fr -l es",
-    "test:google-translate": "ts-node src/index.ts google-translate --in=testOut/localization-combined.json --out=testOut/localization-backfilled-google-translate.json -l en -l fr -l es",
-    "test:to-typescript": "ts-node src/index.ts to-typescript -i testOut/localization-combined.json -o testOut/localization.ts && tsc testOut/localization.ts",
-    "test": "npm run test:polyglot && npm run test:combine-json && npm run test:to-typescript"
+    "test:find-strings": "ts-node src/index.ts find-strings -i test/**/*.js -o testOut/localization-en.json --localize-function-name=i18n",
+    "test:localize": "ts-node src/index.ts localize --source-json=testOut/localization-en.json --source-locale=en --destination-locale=fr --destination-json=testOut/localization-fr.json",
+    "test:to-typescript": "ts-node src/index.ts to-typescript --default-localization=testOut/localization-en.json --out=testOut/localization.ts --localize-function-name=i18n && tsc testOut/localization.ts",
+    "test": "npm run test:find-strings && npm run test:localize && npm run test:to-typescript && npm run prettier:check",
+    "prettier:check": "prettier --check .",
+    "prettier:fix": "prettier --write ."
   },
   "author": "Rémi Vansteelandt",
   "license": "UNLICENSED",
   "dependencies": {
+    "openai": "^4.30.0",
     "@vitalets/google-translate-api": "^9.2.0",
+    "dotenv": "^16.4.5",
+    "glob": "^13.0.6",
     "yargs": "^17.7.2"
   },
   "devDependencies": {
     "@types/node": "^18.11.5",
     "@types/yargs": "^17.0.32",
+    "prettier": "3.8.1",
     "ts-node": "^10.9.1",
     "typescript": "^5.2.2"
   }

package/src/commands/find-strings.ts

@@ -0,0 +1,38 @@
+import { promises as fs } from "fs";
+import { glob } from "glob";
+
+export async function findStringsCommand(options: {
+  in: string;
+  out: string;
+  localizeFunctionName: string;
+}) {
+  const jsfiles = await glob(options.in);
+
+  const strings = new Set<string>();
+  for (const file of jsfiles) {
+    const content = await fs.readFile(file, "utf-8");
+
+    const matches = [
+      ...content.matchAll(
+        new RegExp(`${options.localizeFunctionName}\\('([^']+)'\\)`, "g"),
+      ),
+      ...content.matchAll(
+        new RegExp(`${options.localizeFunctionName}\\("([^"]+)"\\)`, "g"),
+      ),
+      ...content.matchAll(
+        new RegExp(`${options.localizeFunctionName}\\(\`([^"]+)\`\\)`, "g"),
+      ),
+    ];
+
+    for (const [_, string] of matches) {
+      strings.add(string);
+    }
+  }
+
+  const outJson: Record<string, string> = {};
+  for (const string of Array.from(strings).sort()) {
+    outJson[string] = string;
+  }
+
+  await fs.writeFile(options.out, JSON.stringify(outJson, null, 4));
+}

package/src/commands/localize.ts

@@ -0,0 +1,104 @@
+import { config } from "dotenv";
+import OpenAI from "openai";
+import { promises as fs } from "fs";
+import { translate } from "@vitalets/google-translate-api";
+
+config();
+
+export async function googleTranslate(
+  text: string,
+  from: string,
+  to: string,
+): Promise<string> {
+  const result = await translate(text, { from, to });
+  return result.text;
+}
+
+async function gptTranslate(
+  text: string,
+  from: string,
+  to: string,
+  context?: string,
+): Promise<string> {
+  const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+
+  const prompt =
+    `Translate the following text from ${from} to ${to}.` +
+    (context ? ` Context: ${context}` : "") +
+    `\nText: ${text}`;
+  const completion = await openai.chat.completions.create({
+    model: "gpt-3.5-turbo",
+    messages: [
+      {
+        role: "system",
+        content: `
+          You are a helpful translation assistant.
+          Only return the string that would be used for translation, so it can be put in the game as is (verbatim).
+          `.trim(),
+      },
+      { role: "user", content: prompt },
+    ],
+    max_tokens: 1000,
+    temperature: 0.3,
+  });
+  return completion.choices?.[0]?.message?.content?.trim() || "";
+}
+
+export async function localize(options: {
+  sourceJson: string;
+  sourceLocale: string;
+  destinationJson: string;
+  destinationLocale: string;
+  context?: string;
+  engine?: string;
+}) {
+  let existingJson: Record<string, string> = {};
+  try {
+    existingJson = JSON.parse(
+      await fs.readFile(options.destinationJson, "utf-8"),
+    );
+  } catch (error) {}
+
+  const allStrings: Record<string, string> = JSON.parse(
+    await fs.readFile(options.sourceJson, "utf-8"),
+  );
+
+  const outJson: Record<string, string> = {};
+
+  console.log(
+    "Translating strings from",
+    options.sourceLocale,
+    "to",
+    options.destinationLocale,
+  );
+
+  try {
+    for (const key of Object.keys(allStrings).sort()) {
+      if (existingJson[key]) {
+        outJson[key] = existingJson[key];
+        continue;
+      }
+
+      if (options.engine === "gpt") {
+        outJson[key] = await gptTranslate(
+          key,
+          options.sourceLocale,
+          options.destinationLocale,
+          options.context,
+        );
+      } else {
+        outJson[key] = await googleTranslate(
+          key,
+          options.sourceLocale,
+          options.destinationLocale,
+        );
+      }
+      console.log(`${key} => ${outJson[key]}`);
+    }
+  } finally {
+    await fs.writeFile(
+      options.destinationJson,
+      JSON.stringify(outJson, null, 4),
+    );
+  }
+}

package/src/commands/to-typescript.ts

@@ -1,14 +1,30 @@
-import path from "path";
-import { TranslationSet } from "../model/translation-set";
-import { promises as fs } from 'fs';
+import { promises as fs } from "fs";
 
 export async function toTypescriptCommand(options: {
-    in: string,
-    out: string,
+  defaultLocalization: string;
+  out: string;
+  localizeFunctionName: string;
 }) {
-    const sourceJson = JSON.parse(await fs.readFile(options.in, 'utf-8'));
-    const outTranslations = TranslationSet.fromJSON(sourceJson);
+  const allStringsJson = JSON.parse(
+    await fs.readFile(options.defaultLocalization, "utf-8"),
+  );
 
-    await fs.mkdir(path.dirname(options.out), { recursive: true });
-    await fs.writeFile(options.out, outTranslations.toTypeScript());
+  let ts = "";
+
+  ts += `export type LocalizedKey = ${Object.keys(allStringsJson)
+    .map((key) => JSON.stringify(key))
+    .join(" | ")};\n\n`;
+  ts += `export type Localization = {[key in LocalizedKey]: string};\n\n`;
+
+  ts += `let LOCALIZATION: Localization = ${JSON.stringify(allStringsJson)};\n\n`;
+
+  ts += `export function setLocalization(localization: Localization) {\n`;
+  ts += `    LOCALIZATION = localization;\n`;
+  ts += `}\n\n`;
+
+  ts += `export function ${options.localizeFunctionName}(key: string) {\n`;
+  ts += `    return LOCALIZATION[key] || key;\n`;
+  ts += `}\n`;
+
+  await fs.writeFile(options.out, ts);
 }