@remvst/localization 1.0.2 → 2.0.0

package/src/index.ts

@@ -1,125 +1,101 @@
 #!/usr/bin/env node
 
-import { hideBin } from 'yargs/helpers';
-import yargs from 'yargs';
-import { combineJsonCommand } from './commands/combine-json';
-import { googleTranslateCommand } from './commands/google-translate';
-import { parseCsvCommand } from './commands/parse-csv';
-import { toTypescriptCommand } from './commands/to-typescript';
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { findStringsCommand } from "./commands/find-strings";
+import { localize } from "./commands/localize";
+import { toTypescriptCommand } from "./commands/to-typescript";
 
 async function main() {
-    await yargs(hideBin(process.argv))
-        .command('parse-csv', 'convert a CSV into a JSON', async (yargs) => {
-            const argv = await yargs
-                .option('in', {
-                    type: 'string',
-                    alias: 'i',
-                    describe: 'Path to CSV to read from',
-                })
-                .option('out', {
-                    type: 'string',
-                    alias: 'o',
-                    describe: 'Output file',
-                })
-                .option('languagesLineIndex', {
-                    type: 'number',
-                    alias: 'l',
-                    describe: 'Index of the row containing the languages',
-                })
-                .demandOption('in')
-                .demandOption('out')
-                .demandOption('languagesLineIndex')
-                .argv
+  await yargs(hideBin(process.argv))
+    .command(
+      "find-strings",
+      "Finds all localizable strings in a source file",
+      async (yargs) => {
+        const argv = await yargs
+          .option("in", {
+            type: "string",
+            alias: "i",
+            describe: "Folder to read from",
+          })
+          .options("out", {
+            type: "string",
+            alias: "o",
+            describe: "Output JSON file",
+          })
+          .options("localize-function-name", {
+            type: "string",
+            describe: "Name of the function used for localization",
+          })
+          .demandOption("in")
+          .demandOption("out")
+          .demandOption("localize-function-name").argv;
 
-            await parseCsvCommand(argv);
+        await findStringsCommand(argv);
+      },
+    )
+    .command("localize", "Creates a localization JSON file", async (yargs) => {
+      const argv = await yargs
+        .option("source-json", {
+          type: "string",
+          describe: "File that contains all strings in the original language",
         })
-        .command('combine-json', 'combine multiple JSON files', async (yargs) => {
-            const argv = await yargs
-                .option('main', {
-                    type: 'string',
-                    alias: 'i',
-                    describe: 'JSON file to read from',
-                })
-                .option('out', {
-                    type: 'string',
-                    alias: 'o',
-                    describe: 'Output JSON file',
-                })
-                .option('fallbackLocale', {
-                    type: 'string',
-                    describe: 'Default locale to backfill translations',
-                    default: 'en',
-                })
-                .option('fallback', {
-                    type: 'string',
-                    alias: 'f',
-                    describe: 'Fallback JSON file to read from',
-                })
-                .option('locale', {
-                    type: 'string',
-                    alias: 'l',
-                    describe: 'Locale to include',
-                })
-                .array('fallback')
-                .array('locale')
-                .demandOption('main')
-                .demandOption('out')
-                .demandOption('locale')
-                .demandOption('fallback')
-                .argv
-
-            await combineJsonCommand(argv);
+        .option("source-locale", {
+          type: "string",
+          describe: "Locale to translate from",
         })
-        .command('to-typescript', 'converts a JSON file into a TypeScript file', async (yargs) => {
-            const argv = await yargs
-                .option('in', {
-                    type: 'string',
-                    alias: 'i',
-                    describe: 'JSON file to read from',
-                })
-                .options('out', {
-                    type: 'string',
-                    alias: 'o',
-                    describe: 'Output Typescript file',
-                })
-                .demandOption('in')
-                .demandOption('out')
-                .argv
-
-            await toTypescriptCommand(argv);
+        .options("destination-json", {
+          type: "string",
+          alias: "o",
+          describe: "Output JSON file",
         })
-        .command('google-translate', 'adds missing translations to a JSON file using Google translate', async (yargs) => {
-            const argv = await yargs
-                .option('in', {
-                    type: 'string',
-                    alias: 'i',
-                    describe: 'JSON file to read from',
-                })
-                .option('out', {
-                    type: 'string',
-                    alias: 'o',
-                    describe: 'Output JSON file',
-                })
-                .option('fallbackLocale', {
-                    type: 'string',
-                    describe: 'Default locale to backfill translations',
-                    default: 'en',
-                })
-                .option('locale', {
-                    type: 'string',
-                    alias: 'l',
-                    describe: 'Locale to include',
-                })
-                .array('fallback')
-                .array('locale')
-                .demandOption('in')
-                .demandOption('out')
-                .demandOption('locale')
-                .argv
-
-            await googleTranslateCommand(argv);
+        .option("destination-locale", {
+          type: "string",
+          describe: "Locale to translate to",
+        })
+        .option("context", {
+          type: "string",
+          describe: "Context to provide to the translation engine",
         })
-        .argv;
+        .option("engine", {
+          type: "string",
+          choices: ["gpt", "google"],
+          default: "google",
+          describe: "Translation engine to use",
+        })
+        .demandOption("source-json")
+        .demandOption("source-locale")
+        .demandOption("destination-json")
+        .demandOption("destination-locale").argv;
+
+      await localize(argv);
+    })
+    .command(
+      "to-typescript",
+      "Creates a TypeScript file that can be then imported to localize a project",
+      async (yargs) => {
+        const argv = await yargs
+          .option("default-localization", {
+            type: "string",
+            alias: "i",
+            describe: "Default localization JSON file",
+          })
+          .options("out", {
+            type: "string",
+            alias: "o",
+            describe: "Output Typescript file",
+          })
+          .options("localize-function-name", {
+            type: "string",
+            describe: "Name of the function used for localization",
+          })
+          .demandOption("default-localization")
+          .demandOption("out")
+          .demandOption("localize-function-name").argv;
+
+        await toTypescriptCommand(argv);
+      },
+    ).argv;
 }
 
 main();

package/test/my-script.js

@@ -0,0 +1,3 @@
+console.log(i18n("hello world"));
+console.log(i18n("how is it going?"));
+console.log(i18n(`how do you do?`));

package/tsconfig.json

@@ -1,14 +1,14 @@
 {
-    "compilerOptions": {
-        "target": "es2020",
-        "module": "commonjs",
-        "declaration": true,
-        "strict": true,
-        "esModuleInterop": true,
-        "allowJs": false,
-        "outDir": "./lib",
-        "skipLibCheck": true
-    },
-    "include": ["./src/**/*"],
-    "exclude": ["node_modules"]
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "commonjs",
+    "declaration": true,
+    "strict": true,
+    "esModuleInterop": true,
+    "allowJs": false,
+    "outDir": "./lib",
+    "skipLibCheck": true
+  },
+  "include": ["./src/**/*"],
+  "exclude": ["node_modules"]
 }

package/lib/commands/combine-json.d.ts

@@ -1,7 +0,0 @@
-export declare function combineJsonCommand(options: {
-    main: string;
-    out: string;
-    locale: string[];
-    fallbackLocale: string;
-    fallback: string[];
-}): Promise<void>;

package/lib/commands/combine-json.js

@@ -1,35 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.combineJsonCommand = void 0;
-const fs_1 = require("fs");
-const translation_set_1 = require("../model/translation-set");
-const path_1 = __importDefault(require("path"));
-async function combineJsonCommand(options) {
-    const fallbackTranslations = [];
-    for (const fallbackPath of options.fallback || []) {
-        const sourceJson = JSON.parse(await fs_1.promises.readFile(fallbackPath, 'utf-8'));
-        const translationSet = translation_set_1.TranslationSet.fromJSON(sourceJson);
-        fallbackTranslations.push(translationSet);
-    }
-    const mainJson = JSON.parse(await fs_1.promises.readFile(options.main, 'utf-8'));
-    const outTranslations = translation_set_1.TranslationSet.fromJSON(mainJson);
-    await outTranslations.applyFallbacks(options.locale || ['en'], async (key, locale, item) => {
-        const translationInFallbackLocale = item.get(options.fallbackLocale);
-        if (!translationInFallbackLocale)
-            return null;
-        for (const fallbackSet of fallbackTranslations) {
-            if (!translationInFallbackLocale)
-                continue;
-            const fromFallbackLocale = fallbackSet.fromLocalization(options.fallbackLocale, translationInFallbackLocale)?.get(locale);
-            if (fromFallbackLocale)
-                return fromFallbackLocale;
-        }
-        return null;
-    });
-    await fs_1.promises.mkdir(path_1.default.dirname(options.out), { recursive: true });
-    await fs_1.promises.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
-}
-exports.combineJsonCommand = combineJsonCommand;

package/lib/commands/google-translate.d.ts

@@ -1,6 +0,0 @@
-export declare function googleTranslateCommand(options: {
-    in: string;
-    out: string;
-    locale: string[];
-    fallbackLocale: string;
-}): Promise<void>;

package/lib/commands/google-translate.js

@@ -1,25 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.googleTranslateCommand = void 0;
-const fs_1 = require("fs");
-const translation_set_1 = require("../model/translation-set");
-const google_translate_api_1 = require("@vitalets/google-translate-api");
-const path_1 = __importDefault(require("path"));
-async function googleTranslateCommand(options) {
-    const mainJson = JSON.parse(await fs_1.promises.readFile(options.in, 'utf-8'));
-    const outTranslations = translation_set_1.TranslationSet.fromJSON(mainJson);
-    await outTranslations.applyFallbacks(options.locale || ['en'], async (_, locale, item) => {
-        const translationInFallbackLocale = item.get(options.fallbackLocale);
-        if (!translationInFallbackLocale)
-            return null;
-        console.log(`Translating ${JSON.stringify(translationInFallbackLocale)} to ${locale}`);
-        const translationResult = await (0, google_translate_api_1.translate)(translationInFallbackLocale, { from: options.fallbackLocale, to: locale });
-        return translationResult?.text;
-    });
-    await fs_1.promises.mkdir(path_1.default.dirname(options.out), { recursive: true });
-    await fs_1.promises.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
-}
-exports.googleTranslateCommand = googleTranslateCommand;

package/lib/commands/parse-csv.d.ts

@@ -1,7 +0,0 @@
-import { TranslationSet } from "../model/translation-set";
-export declare function parseCsv(path: string, languagesLine: number): Promise<TranslationSet>;
-export declare function parseCsvCommand(options: {
-    in: string;
-    out: string;
-    languagesLineIndex: number;
-}): Promise<void>;

package/lib/commands/parse-csv.js

@@ -1,36 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseCsvCommand = exports.parseCsv = void 0;
-const fs_1 = require("fs");
-const translation_set_1 = require("../model/translation-set");
-const path_1 = __importDefault(require("path"));
-async function parseCsv(path, languagesLine) {
-    const csvContent = await fs_1.promises.readFile(path, 'utf-8');
-    const csvLines = csvContent.split('\n');
-    const res = new translation_set_1.TranslationSet();
-    const ietfLine = csvLines[languagesLine];
-    const languagesIndices = new Map();
-    const ietfColumns = ietfLine.split(',');
-    for (let i = 2; i < ietfColumns.length; i++) {
-        const locale = ietfColumns[i].slice(0, 2); // only grab the first two chars to identify the language
-        languagesIndices.set(locale, i);
-    }
-    for (const line of csvLines) {
-        const columns = line.split(',');
-        const key = columns[0];
-        for (const [locale, columnIndex] of languagesIndices.entries()) {
-            res.add(key, locale, columns[columnIndex]);
-        }
-    }
-    return res;
-}
-exports.parseCsv = parseCsv;
-async function parseCsvCommand(options) {
-    const polyglotTranslations = await parseCsv(options.in, options.languagesLineIndex);
-    await fs_1.promises.mkdir(path_1.default.dirname(options.out), { recursive: true });
-    await fs_1.promises.writeFile(options.out, JSON.stringify(polyglotTranslations, null, 4));
-}
-exports.parseCsvCommand = parseCsvCommand;

package/lib/model/translation-set.d.ts

@@ -1,21 +0,0 @@
-export type Key = string;
-export type Locale = string;
-export type LocalizedItem = Map<Locale, string>;
-export declare class TranslationSet {
-    private readonly locales;
-    private readonly translations;
-    add(key: Key, locale: Locale, localization: string): void;
-    fromLocalization(language: Locale, localization: string): LocalizedItem | null;
-    toJSON(): {
-        [key: string]: {
-            [key: string]: string;
-        };
-    };
-    static fromJSON(json: {
-        [key: string]: {
-            [key: string]: string;
-        };
-    }): TranslationSet;
-    toTypeScript(): string;
-    applyFallbacks(locales: Locale[], fallback: (key: Key, locale: Locale, localizedItem: LocalizedItem) => Promise<string | null>): Promise<void>;
-}

package/lib/model/translation-set.js

@@ -1,93 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.TranslationSet = void 0;
-class TranslationSet {
-    constructor() {
-        this.locales = new Set();
-        this.translations = new Map();
-    }
-    add(key, locale, localization) {
-        this.locales.add(locale);
-        if (!this.translations.has(key)) {
-            this.translations.set(key, new Map());
-        }
-        this.translations.get(key).set(locale, localization);
-    }
-    fromLocalization(language, localization) {
-        const search = localization.toLowerCase();
-        for (const localizedItem of this.translations.values()) {
-            if (localizedItem.get(language)?.toLowerCase() === search) {
-                return localizedItem;
-            }
-        }
-        return null;
-    }
-    toJSON() {
-        const res = {};
-        for (const [key, localizedItem] of this.translations.entries()) {
-            res[key] = {};
-            for (const [locale, translation] of localizedItem.entries()) {
-                res[key][locale] = translation;
-            }
-        }
-        return res;
-    }
-    static fromJSON(json) {
-        const res = new TranslationSet();
-        for (const [key, localizedItem] of Object.entries(json)) {
-            for (const [locale, translation] of Object.entries(localizedItem)) {
-                res.add(key, locale, translation);
-            }
-        }
-        return res;
-    }
-    toTypeScript() {
-        let generatedFileContent = '';
-        generatedFileContent += `export type Locale = ${Array.from(this.locales).map(key => JSON.stringify(key)).join(' | ')};\n\n`;
-        generatedFileContent += `export type LocalizedKey = ${Array.from(this.translations.keys()).map(key => JSON.stringify(key)).join(' | ')};\n\n`;
-        generatedFileContent += `export type LocalizationItem = {[key in Locale]?: string};\n\n`;
-        generatedFileContent += 'export const LOCALIZATION: {[key in LocalizedKey]: LocalizationItem} = {\n';
-        for (const [key, localizedItem] of this.translations.entries()) {
-            generatedFileContent += `    ${JSON.stringify(key)}: {\n`;
-            for (const [locale, translation] of localizedItem.entries()) {
-                if (!translation)
-                    continue;
-                generatedFileContent += `        ${JSON.stringify(locale)}: ${JSON.stringify(translation)},\n`;
-            }
-            generatedFileContent += `    },\n`;
-        }
-        generatedFileContent += '};\n\n';
-        generatedFileContent += `let LOCALE: Locale = 'en';\n\n`;
-        generatedFileContent += `export function setLocale(locale: string) {\n`;
-        generatedFileContent += `    LOCALE = locale.split('-')[0] as Locale\n`;
-        generatedFileContent += `}\n\n`;
-        generatedFileContent += `export function localize(key: LocalizedKey) {\n`;
-        generatedFileContent += `    const localizationItem: LocalizationItem = LOCALIZATION[key];\n`;
-        generatedFileContent += `    if (!localizationItem) return key;\n`;
-        generatedFileContent += `    return localizationItem[LOCALE] || localizationItem.en;\n`;
-        generatedFileContent += `}\n`;
-        return generatedFileContent;
-    }
-    async applyFallbacks(locales, fallback) {
-        for (const [key, localizedItem] of this.translations.entries()) {
-            for (const locale of locales) {
-                let translation = localizedItem.get(locale) || null;
-                if (translation)
-                    continue;
-                try {
-                    translation = await fallback(key, locale, localizedItem);
-                }
-                catch (err) {
-                    console.error(err);
-                    continue;
-                }
-                if (!translation) {
-                    console.warn(`Unable to find fallback for ${key}`);
-                    continue;
-                }
-                this.add(key, locale, translation);
-            }
-        }
-    }
-}
-exports.TranslationSet = TranslationSet;

package/src/commands/combine-json.ts

@@ -1,41 +0,0 @@
-import { promises as fs } from 'fs';
-import { TranslationSet, Key, Locale, LocalizedItem } from '../model/translation-set';
-import path from 'path';
-
-export async function combineJsonCommand(options: {
-    main: string,
-    out: string,
-    locale: string[],
-    fallbackLocale: string,
-    fallback: string[],
-}) {
-    const fallbackTranslations: TranslationSet[] = [];
-
-    for (const fallbackPath of options.fallback || []) {
-        const sourceJson = JSON.parse(await fs.readFile(fallbackPath, 'utf-8'));
-        const translationSet = TranslationSet.fromJSON(sourceJson);
-        fallbackTranslations.push(translationSet);
-    }
-
-    const mainJson = JSON.parse(await fs.readFile(options.main, 'utf-8'));
-    const outTranslations = TranslationSet.fromJSON(mainJson);
-
-    await outTranslations.applyFallbacks(
-        options.locale || ['en'],
-        async (key: Key, locale: Locale, item: LocalizedItem) => {
-            const translationInFallbackLocale = item.get(options.fallbackLocale);
-            if (!translationInFallbackLocale) return null;
-
-            for (const fallbackSet of fallbackTranslations) {
-                if (!translationInFallbackLocale) continue;
-                const fromFallbackLocale = fallbackSet.fromLocalization(options.fallbackLocale, translationInFallbackLocale)?.get(locale);
-                if (fromFallbackLocale) return fromFallbackLocale;
-            }
-
-            return null;
-        }
-    );
-
-    await fs.mkdir(path.dirname(options.out), { recursive: true });
-    await fs.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
-}

package/src/commands/google-translate.ts

@@ -1,34 +0,0 @@
-import { promises as fs } from 'fs';
-import { Key, Locale, LocalizedItem, TranslationSet } from "../model/translation-set";
-import { translate } from '@vitalets/google-translate-api';
-import path from 'path';
-
-export async function googleTranslateCommand(options: {
-    in: string,
-    out: string,
-    locale: string[],
-    fallbackLocale: string,
-}) {
-    const mainJson = JSON.parse(await fs.readFile(options.in, 'utf-8'));
-    const outTranslations = TranslationSet.fromJSON(mainJson);
-
-    await outTranslations.applyFallbacks(
-        options.locale || ['en'],
-        async (_: Key, locale: Locale, item: LocalizedItem) => {
-            const translationInFallbackLocale = item.get(options.fallbackLocale);
-            if (!translationInFallbackLocale) return null;
-
-            console.log(`Translating ${JSON.stringify(translationInFallbackLocale)} to ${locale}`);
-
-            const translationResult = await translate(
-                translationInFallbackLocale,
-                { from: options.fallbackLocale, to: locale },
-            );
-
-            return translationResult?.text;
-        }
-    );
-
-    await fs.mkdir(path.dirname(options.out), { recursive: true });
-    await fs.writeFile(options.out, JSON.stringify(outTranslations.toJSON(), null, 4));
-}

package/src/commands/parse-csv.ts

@@ -1,43 +0,0 @@
-import { promises as fs } from 'fs';
-import { TranslationSet } from "../model/translation-set";
-import path from 'path';
-
-export async function parseCsv(
-    path: string,
-    languagesLine: number,
-): Promise<TranslationSet> {
-    const csvContent = await fs.readFile(path, 'utf-8');
-    const csvLines = csvContent.split('\n');
-
-    const res = new TranslationSet();
-
-    const ietfLine = csvLines[languagesLine];
-    const languagesIndices = new Map<string, number>();
-    const ietfColumns = ietfLine.split(',');
-    for (let i = 2 ; i < ietfColumns.length ; i++) {
-        const locale = ietfColumns[i].slice(0, 2); // only grab the first two chars to identify the language
-        languagesIndices.set(locale, i);
-    }
-
-    for (const line of csvLines) {
-        const columns = line.split(',');
-
-        const key = columns[0];
-
-        for (const [locale, columnIndex] of languagesIndices.entries()) {
-            res.add(key, locale, columns[columnIndex]);
-        }
-    }
-
-    return res;
-}
-
-export async function parseCsvCommand(options: {
-    in: string,
-    out: string,
-    languagesLineIndex: number,
-}) {
-    const polyglotTranslations = await parseCsv(options.in!, options.languagesLineIndex);
-    await fs.mkdir(path.dirname(options.out), { recursive: true });
-    await fs.writeFile(options.out!, JSON.stringify(polyglotTranslations, null, 4));
-}