@theia/localization-manager 1.23.0-next.8 → 1.23.0

package/src/index.ts

@@ -1,17 +1,19 @@
-/********************************************************************************
- * Copyright (C) 2021 TypeFox and others.
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License v. 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0.
- *
- * This Source Code may also be made available under the following Secondary
- * Licenses when the conditions for such availability set forth in the Eclipse
- * Public License v. 2.0 are satisfied: GNU General Public License, version 2
- * with the GNU Classpath Exception which is available at
- * https://www.gnu.org/software/classpath/license.html.
- *
- * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
- ********************************************************************************/
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
 
+export * from './common';
 export * from './localization-extractor';
+export * from './localization-manager';

package/src/localization-extractor.spec.ts

@@ -1,23 +1,24 @@
-/********************************************************************************
- * Copyright (C) 2021 TypeFox and others.
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License v. 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0.
- *
- * This Source Code may also be made available under the following Secondary
- * Licenses when the conditions for such availability set forth in the Eclipse
- * Public License v. 2.0 are satisfied: GNU General Public License, version 2
- * with the GNU Classpath Exception which is available at
- * https://www.gnu.org/software/classpath/license.html.
- *
- * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
- ********************************************************************************/
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
 
 import * as assert from 'assert';
-import { extractFromFile } from './localization-extractor';
+import { extractFromFile, ExtractionOptions } from './localization-extractor';
 
 const TEST_FILE = 'test.ts';
+const quiet: ExtractionOptions = { quiet: true };
 
 describe('correctly extracts from file content', () => {
 
@@ -93,7 +94,7 @@ describe('correctly extracts from file content', () => {
     it('should return an error when resolving is not successful', async () => {
         const content = "nls.localize(a, 'value')";
         const errors: string[] = [];
-        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors), {});
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {});
         assert.deepStrictEqual(errors, [
             "test.ts(1,14): Could not resolve reference to 'a'"
         ]);
@@ -102,7 +103,7 @@ describe('correctly extracts from file content', () => {
     it('should return an error when resolving from an expression', async () => {
         const content = "nls.localize(test.value, 'value');";
         const errors: string[] = [];
-        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors), {});
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {});
         assert.deepStrictEqual(errors, [
             "test.ts(1,14): 'test.value' is not a string constant"
         ]);
@@ -114,7 +115,7 @@ describe('correctly extracts from file content', () => {
         nls.localize('key/nested', 'value');
         `.trim();
         const errors: string[] = [];
-        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors), {
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {
             'key': 'value'
         });
         assert.deepStrictEqual(errors, [
@@ -128,7 +129,7 @@ describe('correctly extracts from file content', () => {
         nls.localize('key', 'value');
         `.trim();
         const errors: string[] = [];
-        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors), {
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {
             'key': {
                 'nested': 'value'
             }
@@ -141,7 +142,7 @@ describe('correctly extracts from file content', () => {
     it('should show error for template literals', async () => {
         const content = 'nls.localize("key", `template literal value`)';
         const errors: string[] = [];
-        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors), {});
+        assert.deepStrictEqual(await extractFromFile(TEST_FILE, content, errors, quiet), {});
         assert.deepStrictEqual(errors, [
             "test.ts(1,20): Template literals are not supported for localization. Please use the additional arguments of the 'nls.localize' function to format strings"
         ]);

package/src/localization-extractor.ts

@@ -1,18 +1,18 @@
-/********************************************************************************
- * Copyright (C) 2021 TypeFox and others.
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License v. 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0.
- *
- * This Source Code may also be made available under the following Secondary
- * Licenses when the conditions for such availability set forth in the Eclipse
- * Public License v. 2.0 are satisfied: GNU General Public License, version 2
- * with the GNU Classpath Exception which is available at
- * https://www.gnu.org/software/classpath/license.html.
- *
- * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
- ********************************************************************************/
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
 
 import * as fs from 'fs-extra';
 import * as ts from 'typescript';
@@ -21,21 +21,19 @@ import * as path from 'path';
 import { glob } from 'glob';
 import { promisify } from 'util';
 import deepmerge = require('deepmerge');
+import { Localization, sortLocalization } from './common';
 
 const globPromise = promisify(glob);
 
-export interface Localization {
-    [key: string]: string | Localization
-}
-
 export interface ExtractionOptions {
-    root: string
-    output: string
+    root?: string
+    output?: string
     exclude?: string
     logs?: string
     /** List of globs matching the files to extract from. */
     files?: string[]
-    merge: boolean
+    merge?: boolean
+    quiet?: boolean
 }
 
 class SingleFileServiceHost implements ts.LanguageServiceHost {
@@ -74,9 +72,9 @@ const tsOptions: ts.CompilerOptions = {
 };
 
 export async function extract(options: ExtractionOptions): Promise<void> {
-    const cwd = path.resolve(process.cwd(), options.root);
+    const cwd = path.resolve(process.env.INIT_CWD || process.cwd(), options.root ?? '');
     const files: string[] = [];
-    await Promise.all((options.files ?? ['**/src/**/*.ts']).map(
+    await Promise.all((options.files ?? ['**/src/**/*.{ts,tsx}']).map(
         async pattern => files.push(...await globPromise(pattern, { cwd }))
     ));
     let localization: Localization = {};
@@ -90,13 +88,15 @@ export async function extract(options: ExtractionOptions): Promise<void> {
     if (errors.length > 0 && options.logs) {
         await fs.writeFile(options.logs, errors.join(os.EOL));
     }
-    const output = path.resolve(process.cwd(), options.output);
-    if (options.merge && await fs.pathExists(output)) {
-        const existing = await fs.readJson(output);
+    const out = path.resolve(process.env.INIT_CWD || process.cwd(), options.output ?? '');
+    if (options.merge && await fs.pathExists(out)) {
+        const existing = await fs.readJson(out);
         localization = deepmerge(existing, localization);
     }
-    await fs.writeJson(options.output, localization, {
-        spaces: 4
+    localization = sortLocalization(localization);
+    await fs.mkdirs(path.dirname(out));
+    await fs.writeJson(out, localization, {
+        spaces: 2
     });
 }
 
@@ -108,20 +108,22 @@ export async function extractFromFile(file: string, content: string, errors?: st
     const localizationCalls = collect(sourceFile, node => isLocalizeCall(node));
     for (const call of localizationCalls) {
         try {
-            const extracted = extractFromLocalizeCall(call);
-            if (!isExcluded(options, extracted[0])) {
+            const extracted = extractFromLocalizeCall(call, options);
+            if (extracted) {
                 insert(localization, extracted);
             }
         } catch (err) {
             const tsError = err as Error;
             errors?.push(tsError.message);
-            console.log(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
         }
     }
     const localizedCommands = collect(sourceFile, node => isCommandLocalizeUtility(node));
     for (const command of localizedCommands) {
         try {
-            const extracted = extractFromLocalizedCommandCall(command);
+            const extracted = extractFromLocalizedCommandCall(command, errors, options);
             const label = extracted.label;
             const category = extracted.category;
             if (!isExcluded(options, label[0])) {
@@ -133,7 +135,9 @@ export async function extractFromFile(file: string, content: string, errors?: st
         } catch (err) {
             const tsError = err as Error;
             errors?.push(tsError.message);
-            console.log(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
         }
     }
     return localization;
@@ -194,7 +198,7 @@ function isLocalizeCall(node: ts.Node): boolean {
     return node.expression.getText() === 'nls.localize';
 }
 
-function extractFromLocalizeCall(node: ts.Node): [string, string, ts.Node] {
+function extractFromLocalizeCall(node: ts.Node, options?: ExtractionOptions): [string, string, ts.Node] | undefined {
     if (!ts.isCallExpression(node)) {
         throw new TypeScriptError('Invalid node type', node);
     }
@@ -206,10 +210,18 @@ function extractFromLocalizeCall(node: ts.Node): [string, string, ts.Node] {
 
     const key = extractString(args[0]);
     const value = extractString(args[1]);
+
+    if (isExcluded(options, key)) {
+        return undefined;
+    }
+
     return [key, value, args[1]];
 }
 
-function extractFromLocalizedCommandCall(node: ts.Node): { label: [string, string, ts.Node], category?: [string, string, ts.Node] } {
+function extractFromLocalizedCommandCall(node: ts.Node, errors?: string[], options?: ExtractionOptions): {
+    label: [string, string, ts.Node],
+    category?: [string, string, ts.Node]
+} {
     if (!ts.isCallExpression(node)) {
         throw new TypeScriptError('Invalid node type', node);
     }
@@ -248,8 +260,16 @@ function extractFromLocalizedCommandCall(node: ts.Node): { label: [string, strin
             labelNode = property.initializer;
         }
 
-        const value = extractString(property.initializer);
-        propertyMap.set(name, value);
+        try {
+            const value = extractString(property.initializer);
+            propertyMap.set(name, value);
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
+        }
     }
 
     let labelKey = propertyMap.get('id');
@@ -258,17 +278,33 @@ function extractFromLocalizedCommandCall(node: ts.Node): { label: [string, strin
 
     // We have an explicit label translation key
     if (args.length > 1) {
-        const labelOverrideKey = extractStringOrUndefined(args[1]);
-        if (labelOverrideKey) {
-            labelKey = labelOverrideKey;
-            labelNode = args[1];
+        try {
+            const labelOverrideKey = extractStringOrUndefined(args[1]);
+            if (labelOverrideKey) {
+                labelKey = labelOverrideKey;
+                labelNode = args[1];
+            }
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
         }
     }
 
     // We have an explicit category translation key
     if (args.length > 2) {
-        categoryKey = extractStringOrUndefined(args[2]);
-        categoryNode = args[2];
+        try {
+            categoryKey = extractStringOrUndefined(args[2]);
+            categoryNode = args[2];
+        } catch (err) {
+            const tsError = err as Error;
+            errors?.push(tsError.message);
+            if (!options?.quiet) {
+                console.log(tsError.message);
+            }
+        }
     }
 
     if (!labelKey) {

package/src/localization-manager.spec.ts

@@ -0,0 +1,80 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as assert from 'assert';
+import { DeeplParameters, DeeplResponse } from './deepl-api';
+import { LocalizationManager, LocalizationOptions } from './localization-manager';
+
+describe('localization-manager#translateLanguage', () => {
+
+    async function mockLocalization(parameters: DeeplParameters): Promise<DeeplResponse> {
+        return {
+            translations: parameters.text.map(value => ({
+                detected_source_language: '',
+                text: `[${value}]`
+            }))
+        };
+    }
+
+    const manager = new LocalizationManager(mockLocalization);
+    const defaultOptions: LocalizationOptions = {
+        authKey: '',
+        freeApi: false,
+        sourceFile: '',
+        targetLanguages: ['EN']
+    };
+
+    it('should translate a single value', async () => {
+        const input = {
+            key: 'value'
+        };
+        const target = {};
+        await manager.translateLanguage(input, target, 'EN', defaultOptions);
+        assert.deepStrictEqual(target, {
+            key: '[value]'
+        });
+    });
+
+    it('should translate nested values', async () => {
+        const input = {
+            a: {
+                b: 'b'
+            },
+            c: 'c'
+        };
+        const target = {};
+        await manager.translateLanguage(input, target, 'EN', defaultOptions);
+        assert.deepStrictEqual(target, {
+            a: {
+                b: '[b]'
+            },
+            c: '[c]'
+        });
+    });
+
+    it('should not override existing targets', async () => {
+        const input = {
+            a: 'a'
+        };
+        const target = {
+            a: 'b'
+        };
+        await manager.translateLanguage(input, target, 'EN', defaultOptions);
+        assert.deepStrictEqual(target, {
+            a: 'b'
+        });
+    });
+});

package/src/localization-manager.ts

@@ -0,0 +1,151 @@
+// *****************************************************************************
+// Copyright (C) 2021 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as chalk from 'chalk';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { sortLocalization } from '.';
+import { Localization } from './common';
+import { deepl, DeeplLanguage, DeeplParameters, isSupportedLanguage, supportedLanguages } from './deepl-api';
+
+export interface LocalizationOptions {
+    freeApi: Boolean
+    authKey: string
+    sourceFile: string
+    sourceLanguage?: string
+    targetLanguages: string[]
+}
+
+export type LocalizationFunction = (parameters: DeeplParameters) => Promise<string[]>;
+
+export class LocalizationManager {
+
+    constructor(private localizationFn = deepl) { }
+
+    async localize(options: LocalizationOptions): Promise<void> {
+        let source: Localization = {};
+        const cwd = process.env.INIT_CWD || process.cwd();
+        const sourceFile = path.resolve(cwd, options.sourceFile);
+        try {
+            source = await fs.readJson(sourceFile);
+        } catch {
+            console.log(chalk.red(`Could not read file "${options.sourceFile}"`));
+            process.exit(1);
+        }
+        const languages: string[] = [];
+        for (const targetLanguage of options.targetLanguages) {
+            if (!isSupportedLanguage(targetLanguage)) {
+                console.log(chalk.yellow(`Language "${targetLanguage}" is not supported for automatic localization`));
+            } else {
+                languages.push(targetLanguage);
+            }
+        }
+        if (languages.length !== options.targetLanguages.length) {
+            console.log('Supported languages: ' + supportedLanguages.join(', '));
+        }
+        const existingTranslations: Map<string, Localization> = new Map();
+        for (const targetLanguage of languages) {
+            try {
+                const targetPath = this.translationFileName(sourceFile, targetLanguage);
+                existingTranslations.set(targetLanguage, await fs.readJson(targetPath));
+            } catch {
+                existingTranslations.set(targetLanguage, {});
+            }
+        }
+        await Promise.all(languages.map(language => this.translateLanguage(source, existingTranslations.get(language)!, language, options)));
+
+        for (const targetLanguage of languages) {
+            const targetPath = this.translationFileName(sourceFile, targetLanguage);
+            try {
+                const translation = existingTranslations.get(targetLanguage)!;
+                await fs.writeJson(targetPath, sortLocalization(translation), { spaces: 2 });
+            } catch {
+                console.error(chalk.red(`Error writing translated file to '${targetPath}'`));
+            }
+        }
+    }
+
+    protected translationFileName(original: string, language: string): string {
+        const directory = path.dirname(original);
+        const fileName = path.basename(original, '.json');
+        return path.join(directory, `${fileName}.${language.toLowerCase()}.json`);
+    }
+
+    async translateLanguage(source: Localization, target: Localization, targetLanguage: string, options: LocalizationOptions): Promise<void> {
+        const map = this.buildLocalizationMap(source, target);
+        if (map.text.length > 0) {
+            try {
+                const translationResponse = await this.localizationFn({
+                    auth_key: options.authKey,
+                    free_api: options.freeApi,
+                    target_lang: targetLanguage.toUpperCase() as DeeplLanguage,
+                    source_lang: options.sourceLanguage?.toUpperCase() as DeeplLanguage,
+                    text: map.text
+                });
+                translationResponse.translations.forEach(({ text }, i) => {
+                    map.localize(i, text);
+                });
+                console.log(chalk.green(`Successfully translated ${map.text.length} value${map.text.length > 1 ? 's' : ''} for language "${targetLanguage}"`));
+            } catch (e) {
+                console.log(chalk.red(`Could not translate into language "${targetLanguage}"`), e);
+            }
+        } else {
+            console.log(`No translation necessary for language "${targetLanguage}"`);
+        }
+    }
+
+    protected buildLocalizationMap(source: Localization, target: Localization): LocalizationMap {
+        const functionMap = new Map<number, (value: string) => void>();
+        const text: string[] = [];
+        const process = (s: Localization, t: Localization) => {
+            // Delete all extra keys in the target translation first
+            for (const key of Object.keys(t)) {
+                if (!(key in s)) {
+                    delete t[key];
+                }
+            }
+            for (const [key, value] of Object.entries(s)) {
+                if (!(key in t)) {
+                    if (typeof value === 'string') {
+                        functionMap.set(text.length, translation => t[key] = translation);
+                        text.push(value);
+                    } else {
+                        const newLocalization: Localization = {};
+                        t[key] = newLocalization;
+                        process(value, newLocalization);
+                    }
+                } else if (typeof value === 'object') {
+                    if (typeof t[key] === 'string') {
+                        t[key] = {};
+                    }
+                    process(value, t[key] as Localization);
+                }
+            }
+        };
+
+        process(source, target);
+
+        return {
+            text,
+            localize: (index, value) => functionMap.get(index)!(value)
+        };
+    }
+}
+
+export interface LocalizationMap {
+    text: string[]
+    localize: (index: number, value: string) => void
+}