@remvst/localization 1.0.0

package/.github/workflows/check.yaml

@@ -0,0 +1,25 @@
+name: Check
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: check-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Install dependencies
+        run: npm install
+      - name: Create testOut
+        run: mkdir testOut
+      - name: Run tests
+        shell: bash
+        run: npm test

package/README.md

@@ -0,0 +1,67 @@
+# @remvst/localization
+
+Localization utils for TypeScript.
+
+## Installation
+
+```sh
+npm install --save-dev @remvst/localization
+```
+
+## Usage
+
+Create a JSON file with your localized strings (`my-localization.json`):
+
+```json
+{
+    "ok": { "en": "Okay" },
+    "confirm": { "en": "Confirm" },
+    "cancel": { "en": "Cancel" },
+    "back": { "en": "Back" },
+    "play": { "en": "Play" },
+    "time": { "en": "Time" },
+    "backToMainMenu": { "en": "Back To Main Menu", "fr": "Retour au menu principal" }
+}
+```
+
+Parse the Polyglot CSV file into a JSON file:
+
+```sh
+npx @remvst/localize parse-csv \
+    --in=polyglot.csv \
+    --out=polyglot.json \
+    --languages-line-index=1
+```
+
+Backfill missing translations using a fallback JSON:
+
+```sh
+npx @remvst/localize combine-json \
+    --main=localization.json \
+    --fallback=polyglot.json \
+    --out=localization-combined.json \
+    --fallback-locale=en \
+    --locale=en \
+    --locale=fr \
+    --locale=es
+```
+
+Backfill missing translations using Google translate:
+
+```sh
+npx @remvst/localize google-translate \
+    --in=localization-combined.json \
+    --out=localization-full.json \
+    --fallback-locale=en \
+    --locale=en \
+    --locale=fr \
+    --locale=es
+```
+
+Export to Typescript:
+
+```sh
+npx @remvst/localize to-typescript \
+    --in=localization-full.json \
+    --out=localization-full.ts
+```

package/lib/commands/combine-json.d.ts

@@ -0,0 +1,7 @@
+export declare function combineJsonCommand(options: {
+    main: string;
+    out: string;
+    locale: string[];
+    fallbackLocale: string;
+    fallback: string[];
+}): Promise<void>;

package/lib/commands/combine-json.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.combineJsonCommand = void 0;
+const fs_1 = require("fs");
+const translation_set_1 = require("../model/translation-set");
+async function combineJsonCommand(options) {
+    const fallbackTranslations = [];
+    for (const fallbackPath of options.fallback || []) {
+        const sourceJson = JSON.parse(await fs_1.promises.readFile(fallbackPath, 'utf-8'));
+        const translationSet = translation_set_1.TranslationSet.fromJSON(sourceJson);
+        fallbackTranslations.push(translationSet);
+    }
+    const mainJson = JSON.parse(await fs_1.promises.readFile(options.main, 'utf-8'));
+    const outTranslations = translation_set_1.TranslationSet.fromJSON(mainJson);
+    await outTranslations.applyFallbacks(options.locale || ['en'], async (key, locale, item) => {
+        const translationInFallbackLocale = item.get(options.fallbackLocale);
+        if (!translationInFallbackLocale)
+            return null;
+        for (const fallbackSet of fallbackTranslations) {
+            if (!translationInFallbackLocale)
+                continue;
+            const fromFallbackLocale = fallbackSet.fromLocalization(options.fallbackLocale, translationInFallbackLocale)?.get(locale);
+            if (fromFallbackLocale)
+                return fromFallbackLocale;
+        }
+        return null;
+    });
+    await fs_1.promises.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
+}
+exports.combineJsonCommand = combineJsonCommand;

package/lib/commands/google-translate.d.ts

@@ -0,0 +1,6 @@
+export declare function googleTranslateCommand(options: {
+    in: string;
+    out: string;
+    locale: string[];
+    fallbackLocale: string;
+}): Promise<void>;

package/lib/commands/google-translate.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.googleTranslateCommand = void 0;
+const fs_1 = require("fs");
+const translation_set_1 = require("../model/translation-set");
+const google_translate_api_1 = require("@vitalets/google-translate-api");
+async function googleTranslateCommand(options) {
+    const mainJson = JSON.parse(await fs_1.promises.readFile(options.in, 'utf-8'));
+    const outTranslations = translation_set_1.TranslationSet.fromJSON(mainJson);
+    await outTranslations.applyFallbacks(options.locale || ['en'], async (_, locale, item) => {
+        const translationInFallbackLocale = item.get(options.fallbackLocale);
+        if (!translationInFallbackLocale)
+            return null;
+        console.log(`Translating ${JSON.stringify(translationInFallbackLocale)} to ${locale}`);
+        const translationResult = await (0, google_translate_api_1.translate)(translationInFallbackLocale, { from: options.fallbackLocale, to: locale });
+        return translationResult?.text;
+    });
+    await fs_1.promises.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
+}
+exports.googleTranslateCommand = googleTranslateCommand;

package/lib/commands/parse-csv.d.ts

@@ -0,0 +1,7 @@
+import { TranslationSet } from "../model/translation-set";
+export declare function parseCsv(path: string, languagesLine: number): Promise<TranslationSet>;
+export declare function parseCsvCommand(options: {
+    in: string;
+    out: string;
+    languagesLineIndex: number;
+}): Promise<void>;

package/lib/commands/parse-csv.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseCsvCommand = exports.parseCsv = void 0;
+const fs_1 = require("fs");
+const translation_set_1 = require("../model/translation-set");
+async function parseCsv(path, languagesLine) {
+    const csvContent = await fs_1.promises.readFile(path, 'utf-8');
+    const csvLines = csvContent.split('\n');
+    const res = new translation_set_1.TranslationSet();
+    const ietfLine = csvLines[languagesLine];
+    const languagesIndices = new Map();
+    const ietfColumns = ietfLine.split(',');
+    for (let i = 2; i < ietfColumns.length; i++) {
+        const locale = ietfColumns[i].slice(0, 2); // only grab the first two chars to identify the language
+        languagesIndices.set(locale, i);
+    }
+    for (const line of csvLines) {
+        const columns = line.split(',');
+        const key = columns[0];
+        for (const [locale, columnIndex] of languagesIndices.entries()) {
+            res.add(key, locale, columns[columnIndex]);
+        }
+    }
+    return res;
+}
+exports.parseCsv = parseCsv;
+async function parseCsvCommand(options) {
+    const polyglotTranslations = await parseCsv(options.in, options.languagesLineIndex);
+    await fs_1.promises.writeFile(options.out, JSON.stringify(polyglotTranslations, null, 4));
+}
+exports.parseCsvCommand = parseCsvCommand;

package/lib/commands/to-typescript.d.ts

@@ -0,0 +1,4 @@
+export declare function toTypescriptCommand(options: {
+    in: string;
+    out: string;
+}): Promise<void>;

package/lib/commands/to-typescript.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toTypescriptCommand = void 0;
+const translation_set_1 = require("../model/translation-set");
+const fs_1 = require("fs");
+async function toTypescriptCommand(options) {
+    const sourceJson = JSON.parse(await fs_1.promises.readFile(options.in, 'utf-8'));
+    const outTranslations = translation_set_1.TranslationSet.fromJSON(sourceJson);
+    await fs_1.promises.writeFile(options.out, outTranslations.toTypeScript());
+}
+exports.toTypescriptCommand = toTypescriptCommand;

package/lib/index.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/index.js

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const helpers_1 = require("yargs/helpers");
+const yargs_1 = __importDefault(require("yargs"));
+const combine_json_1 = require("./commands/combine-json");
+const google_translate_1 = require("./commands/google-translate");
+const parse_csv_1 = require("./commands/parse-csv");
+const to_typescript_1 = require("./commands/to-typescript");
+async function main() {
+    await (0, yargs_1.default)((0, helpers_1.hideBin)(process.argv))
+        .command('parse-csv', 'convert a CSV into a JSON', async (yargs) => {
+        const argv = await yargs
+            .option('in', {
+            type: 'string',
+            alias: 'i',
+            describe: 'Path to CSV to read from',
+        })
+            .option('out', {
+            type: 'string',
+            alias: 'o',
+            describe: 'Output file',
+        })
+            .option('languagesLineIndex', {
+            type: 'number',
+            alias: 'l',
+            describe: 'Index of the row containing the languages',
+        })
+            .demandOption('in')
+            .demandOption('out')
+            .demandOption('languagesLineIndex')
+            .argv;
+        await (0, parse_csv_1.parseCsvCommand)(argv);
+    })
+        .command('combine-json', 'combine multiple JSON files', async (yargs) => {
+        const argv = await yargs
+            .option('main', {
+            type: 'string',
+            alias: 'i',
+            describe: 'JSON file to read from',
+        })
+            .option('out', {
+            type: 'string',
+            alias: 'o',
+            describe: 'Output JSON file',
+        })
+            .option('fallbackLocale', {
+            type: 'string',
+            describe: 'Default locale to backfill translations',
+            default: 'en',
+        })
+            .option('fallback', {
+            type: 'string',
+            alias: 'f',
+            describe: 'Fallback JSON file to read from',
+        })
+            .option('locale', {
+            type: 'string',
+            alias: 'l',
+            describe: 'Locale to include',
+        })
+            .array('fallback')
+            .array('locale')
+            .demandOption('main')
+            .demandOption('out')
+            .demandOption('locale')
+            .demandOption('fallback')
+            .argv;
+        await (0, combine_json_1.combineJsonCommand)(argv);
+    })
+        .command('to-typescript', 'converts a JSON file into a TypeScript file', async (yargs) => {
+        const argv = await yargs
+            .option('in', {
+            type: 'string',
+            alias: 'i',
+            describe: 'JSON file to read from',
+        })
+            .options('out', {
+            type: 'string',
+            alias: 'o',
+            describe: 'Output Typescript file',
+        })
+            .demandOption('in')
+            .demandOption('out')
+            .argv;
+        await (0, to_typescript_1.toTypescriptCommand)(argv);
+    })
+        .command('google-translate', 'adds missing translations to a JSON file using Google translate', async (yargs) => {
+        const argv = await yargs
+            .option('in', {
+            type: 'string',
+            alias: 'i',
+            describe: 'JSON file to read from',
+        })
+            .option('out', {
+            type: 'string',
+            alias: 'o',
+            describe: 'Output JSON file',
+        })
+            .option('fallbackLocale', {
+            type: 'string',
+            describe: 'Default locale to backfill translations',
+            default: 'en',
+        })
+            .option('locale', {
+            type: 'string',
+            alias: 'l',
+            describe: 'Locale to include',
+        })
+            .array('fallback')
+            .array('locale')
+            .demandOption('in')
+            .demandOption('out')
+            .demandOption('locale')
+            .argv;
+        await (0, google_translate_1.googleTranslateCommand)(argv);
+    })
+        .argv;
+}
+main();

package/lib/model/translation-set.d.ts

@@ -0,0 +1,21 @@
+export type Key = string;
+export type Locale = string;
+export type LocalizedItem = Map<Locale, string>;
+export declare class TranslationSet {
+    private readonly locales;
+    private readonly translations;
+    add(key: Key, locale: Locale, localization: string): void;
+    fromLocalization(language: Locale, localization: string): LocalizedItem | null;
+    toJSON(): {
+        [key: string]: {
+            [key: string]: string;
+        };
+    };
+    static fromJSON(json: {
+        [key: string]: {
+            [key: string]: string;
+        };
+    }): TranslationSet;
+    toTypeScript(): string;
+    applyFallbacks(locales: Locale[], fallback: (key: Key, locale: Locale, localizedItem: LocalizedItem) => Promise<string | null>): Promise<void>;
+}

package/lib/model/translation-set.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TranslationSet = void 0;
+class TranslationSet {
+    constructor() {
+        this.locales = new Set();
+        this.translations = new Map();
+    }
+    add(key, locale, localization) {
+        this.locales.add(locale);
+        if (!this.translations.has(key)) {
+            this.translations.set(key, new Map());
+        }
+        this.translations.get(key).set(locale, localization);
+    }
+    fromLocalization(language, localization) {
+        const search = localization.toLowerCase();
+        for (const localizedItem of this.translations.values()) {
+            if (localizedItem.get(language)?.toLowerCase() === search) {
+                return localizedItem;
+            }
+        }
+        return null;
+    }
+    toJSON() {
+        const res = {};
+        for (const [key, localizedItem] of this.translations.entries()) {
+            res[key] = {};
+            for (const [locale, translation] of localizedItem.entries()) {
+                res[key][locale] = translation;
+            }
+        }
+        return res;
+    }
+    static fromJSON(json) {
+        const res = new TranslationSet();
+        for (const [key, localizedItem] of Object.entries(json)) {
+            for (const [locale, translation] of Object.entries(localizedItem)) {
+                res.add(key, locale, translation);
+            }
+        }
+        return res;
+    }
+    toTypeScript() {
+        let generatedFileContent = '';
+        generatedFileContent += `export type Locale = ${Array.from(this.locales).map(key => JSON.stringify(key)).join(' | ')};\n\n`;
+        generatedFileContent += `export type LocalizedKey = ${Array.from(this.translations.keys()).map(key => JSON.stringify(key)).join(' | ')};\n\n`;
+        generatedFileContent += `export type LocalizationItem = {[key in Locale]?: string};\n\n`;
+        generatedFileContent += 'export const LOCALIZATION: {[key in LocalizedKey]: LocalizationItem} = {\n';
+        for (const [key, localizedItem] of this.translations.entries()) {
+            generatedFileContent += `    ${key}: {\n`;
+            for (const [locale, translation] of localizedItem.entries()) {
+                if (!translation)
+                    continue;
+                generatedFileContent += `        ${locale}: ${JSON.stringify(translation)},\n`;
+            }
+            generatedFileContent += `    },\n`;
+        }
+        generatedFileContent += '};\n\n';
+        generatedFileContent += `let LOCALE: Locale = 'en';\n\n`;
+        generatedFileContent += `export function setLocale(locale: string) {\n`;
+        generatedFileContent += `    LOCALE = locale.split('-')[0] as Locale\n`;
+        generatedFileContent += `}\n\n`;
+        generatedFileContent += `export function localize(key: LocalizedKey) {\n`;
+        generatedFileContent += `    const localizationItem: LocalizationItem = LOCALIZATION[key];\n`;
+        generatedFileContent += `    return localizationItem[LOCALE] || localizationItem.en;\n`;
+        generatedFileContent += `}\n`;
+        return generatedFileContent;
+    }
+    async applyFallbacks(locales, fallback) {
+        for (const [key, localizedItem] of this.translations.entries()) {
+            for (const locale of locales) {
+                let translation = localizedItem.get(locale) || null;
+                if (translation)
+                    continue;
+                try {
+                    translation = await fallback(key, locale, localizedItem);
+                }
+                catch (err) {
+                    console.error(err);
+                    continue;
+                }
+                if (!translation) {
+                    console.warn(`Unable to find fallback for ${key}`);
+                    continue;
+                }
+                this.add(key, locale, translation);
+            }
+        }
+    }
+}
+exports.TranslationSet = TranslationSet;

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@remvst/localization",
+  "version": "1.0.0",
+  "description": "",
+  "main": "lib/index.js",
+  "bin": {
+    "localization": "./lib/index.js"
+  },
+  "scripts": {
+    "build": "rm -rf lib && tsc",
+    "prepublishOnly": "npm i && npm run build",
+    "test:polyglot": "ts-node src/index.ts index -i test/polyglot.csv -o testOut/polyglot.json -l 1",
+    "test:backfill-json": "ts-node src/index.ts combine-json --main=test/localization.json --out=testOut/localization-backfilled.json --fallback=testOut/polyglot.json -l en -l fr -l es",
+    "test:backfill-google-translate": "ts-node src/index.ts google-translate --in=testOut/localization-backfilled.json --out=testOut/localization-backfilled-google-translate.json -l en -l fr -l es",
+    "test:to-typescript": "ts-node src/index.ts to-typescript -i testOut/localization-backfilled.json -o testOut/localization.ts",
+    "test": "npm run test:polyglot && npm run test:backfill-json && npm run test:to-typescript"
+  },
+  "author": "Rémi Vansteelandt",
+  "license": "UNLICENSED",
+  "dependencies": {
+    "@vitalets/google-translate-api": "^9.2.0",
+    "yargs": "^17.7.2"
+  },
+  "devDependencies": {
+    "@types/node": "^18.11.5",
+    "@types/yargs": "^17.0.32",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.2.2"
+  }
+}

package/src/commands/combine-json.ts

@@ -0,0 +1,39 @@
+import { promises as fs } from 'fs';
+import { TranslationSet, Key, Locale, LocalizedItem } from '../model/translation-set';
+
+export async function combineJsonCommand(options: {
+    main: string,
+    out: string,
+    locale: string[],
+    fallbackLocale: string,
+    fallback: string[],
+}) {
+    const fallbackTranslations: TranslationSet[] = [];
+
+    for (const fallbackPath of options.fallback || []) {
+        const sourceJson = JSON.parse(await fs.readFile(fallbackPath, 'utf-8'));
+        const translationSet = TranslationSet.fromJSON(sourceJson);
+        fallbackTranslations.push(translationSet);
+    }
+
+    const mainJson = JSON.parse(await fs.readFile(options.main, 'utf-8'));
+    const outTranslations = TranslationSet.fromJSON(mainJson);
+
+    await outTranslations.applyFallbacks(
+        options.locale || ['en'],
+        async (key: Key, locale: Locale, item: LocalizedItem) => {
+            const translationInFallbackLocale = item.get(options.fallbackLocale);
+            if (!translationInFallbackLocale) return null;
+
+            for (const fallbackSet of fallbackTranslations) {
+                if (!translationInFallbackLocale) continue;
+                const fromFallbackLocale = fallbackSet.fromLocalization(options.fallbackLocale, translationInFallbackLocale)?.get(locale);
+                if (fromFallbackLocale) return fromFallbackLocale;
+            }
+
+            return null;
+        }
+    );
+
+    await fs.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
+}

package/src/commands/google-translate.ts

@@ -0,0 +1,32 @@
+import { promises as fs } from 'fs';
+import { Key, Locale, LocalizedItem, TranslationSet } from "../model/translation-set";
+import { translate } from '@vitalets/google-translate-api';
+
+export async function googleTranslateCommand(options: {
+    in: string,
+    out: string,
+    locale: string[],
+    fallbackLocale: string,
+}) {
+    const mainJson = JSON.parse(await fs.readFile(options.in, 'utf-8'));
+    const outTranslations = TranslationSet.fromJSON(mainJson);
+
+    await outTranslations.applyFallbacks(
+        options.locale || ['en'],
+        async (_: Key, locale: Locale, item: LocalizedItem) => {
+            const translationInFallbackLocale = item.get(options.fallbackLocale);
+            if (!translationInFallbackLocale) return null;
+
+            console.log(`Translating ${JSON.stringify(translationInFallbackLocale)} to ${locale}`);
+
+            const translationResult = await translate(
+                translationInFallbackLocale,
+                { from: options.fallbackLocale, to: locale },
+            );
+
+            return translationResult?.text;
+        }
+    );
+
+    await fs.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
+}

package/src/commands/parse-csv.ts

@@ -0,0 +1,41 @@
+import { promises as fs } from 'fs';
+import { TranslationSet } from "../model/translation-set";
+
+export async function parseCsv(
+    path: string,
+    languagesLine: number,
+): Promise<TranslationSet> {
+    const csvContent = await fs.readFile(path, 'utf-8');
+    const csvLines = csvContent.split('\n');
+
+    const res = new TranslationSet();
+
+    const ietfLine = csvLines[languagesLine];
+    const languagesIndices = new Map<string, number>();
+    const ietfColumns = ietfLine.split(',');
+    for (let i = 2 ; i < ietfColumns.length ; i++) {
+        const locale = ietfColumns[i].slice(0, 2); // only grab the first two chars to identify the language
+        languagesIndices.set(locale, i);
+    }
+
+    for (const line of csvLines) {
+        const columns = line.split(',');
+
+        const key = columns[0];
+
+        for (const [locale, columnIndex] of languagesIndices.entries()) {
+            res.add(key, locale, columns[columnIndex]);
+        }
+    }
+
+    return res;
+}
+
+export async function parseCsvCommand(options: {
+    in: string,
+    out: string,
+    languagesLineIndex: number,
+}) {
+    const polyglotTranslations = await parseCsv(options.in!, options.languagesLineIndex);
+    await fs.writeFile(options.out!, JSON.stringify(polyglotTranslations, null, 4));
+}

package/src/commands/to-typescript.ts

@@ -0,0 +1,12 @@
+import { TranslationSet } from "../model/translation-set";
+import { promises as fs } from 'fs';
+
+export async function toTypescriptCommand(options: {
+    in: string,
+    out: string,
+}) {
+    const sourceJson = JSON.parse(await fs.readFile(options.in, 'utf-8'));
+    const outTranslations = TranslationSet.fromJSON(sourceJson);
+
+    await fs.writeFile(options.out, outTranslations.toTypeScript());
+}