@diplodoc/cli 4.13.6 → 4.13.8

package/src/cmd/translate/handler.ts

@@ -1,21 +1,23 @@
+import axios from 'axios';
 import {Arguments} from 'yargs';
 import {ArgvService} from '../../services';
 import {logger} from '../../utils';
 import {ok} from 'assert';
-import {basename, dirname, join, resolve} from 'path';
-import glob from 'glob';
-import {getYandexOAuthToken} from '../../packages/credentials';
+import {dirname, extname, join, resolve} from 'path';
+import {mkdir} from 'fs/promises';
+import {getYandexAuth} from './yandex/auth';
 import {asyncify, eachLimit, retry} from 'async';
-import {Session} from '@yandex-cloud/nodejs-sdk/dist/session';
-import {TranslationServiceClient} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/service_clients';
-import {
-    TranslateRequest_Format as Format,
-    TranslateRequest,
-} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/ai/translate/v2/translation_service';
-import {mkdir, readFile, writeFile} from 'fs/promises';
 
-// @ts-ignore
-import {compose, extract} from '@diplodoc/markdown-translation';
+import {
+    Defer,
+    TranslateParams,
+    bytes,
+    compose,
+    dumpFile,
+    extract,
+    loadFile,
+    normalizeParams,
+} from './utils';
 
 const REQUESTS_LIMIT = 20;
 const BYTES_LIMIT = 10000;
@@ -31,96 +33,75 @@ class TranslatorError extends Error {
     }
 }
 
-type TranslateConfig = {
-    input: string;
-    output?: string;
-    sourceLanguage: string;
-    targetLanguage: string;
-    folderId?: string;
-    glossary?: string;
-    include?: string[];
-    exclude?: string[];
-};
+class RequestError extends Error {
+    code: string;
+
+    constructor(error: Error) {
+        super(error?.message || String(error));
+        this.code = 'REQUEST_ERROR';
+    }
+}
 
 export async function handler(args: Arguments<any>) {
-    ArgvService.init({
+    const params = normalizeParams({
         ...(args.translate || {}),
         ...args,
     });
 
-    const {
-        folderId,
-        // yandexCloudTranslateGlossaryPairs,
-        sourceLanguage,
-        targetLanguage,
-        exclude = [],
-    } = ArgvService.getConfig() as unknown as TranslateConfig;
+    ArgvService.init(params);
 
-    let {input, output, include = []} = ArgvService.getConfig() as unknown as TranslateConfig;
-
-    logger.info(
-        input,
-        `translating documentation from ${sourceLanguage} to ${targetLanguage} language`,
-    );
+    const {input, output, auth, folder, source, targets, files, dryRun} =
+        ArgvService.getConfig() as unknown as TranslateParams;
 
-    output = output || input;
-
-    ok(input, 'Required param input is not configured');
-    ok(sourceLanguage, 'Required param sourceLanguage is not configured');
-    ok(targetLanguage, 'Required param targetLanguage is not configured');
+    ok(auth, 'Required param auth is not configured');
+    ok(folder, 'Required param folder is not configured');
+    ok(source, `Required param source is not configured`);
+    ok(targets.length, `Required param target is not configured`);
 
     try {
-        if (input.endsWith('.md')) {
-            include = [basename(input)];
-            input = dirname(input);
-        } else if (!include.length) {
-            include.push('**/*');
-        }
-
-        const files = ([] as string[]).concat(
-            ...include.map((match) =>
-                glob.sync(match, {
-                    cwd: join(input, sourceLanguage),
-                    ignore: exclude,
+        const authInfo = getYandexAuth(auth);
+
+        for (const target of targets) {
+            const translatorParams = {
+                input,
+                output,
+                sourceLanguage: source[0],
+                targetLanguage: target[0],
+                // yandexCloudTranslateGlossaryPairs,
+                folderId: folder,
+                auth: authInfo,
+                dryRun,
+            };
+
+            const cache = new Map<string, Defer>();
+            const request = requester(translatorParams, cache);
+            const split = splitter(request, cache);
+            const translate = translator(translatorParams, split);
+
+            await eachLimit(
+                files,
+                REQUESTS_LIMIT,
+                asyncify(async function (file: string) {
+                    try {
+                        await translate(file);
+                    } catch (error: any) {
+                        logger.error(file, error.message);
+                    }
                 }),
-            ),
-        );
-        const found = [...new Set(files)];
-
-        const oauthToken = await getYandexOAuthToken();
-
-        const translatorParams = {
-            input,
-            output,
-            sourceLanguage,
-            targetLanguage,
-            // yandexCloudTranslateGlossaryPairs,
-            folderId,
-            oauthToken,
-        };
-
-        const translateFn = translator(translatorParams);
+            );
 
-        await eachLimit(found, REQUESTS_LIMIT, asyncify(translateFn));
-    } catch (err) {
-        if (err instanceof Error || err instanceof TranslatorError) {
-            const message = err.message;
+            console.log('PROCESSED', `bytes: ${request.stat.bytes} chunks: ${request.stat.chunks}`);
+        }
+    } catch (error: any) {
+        const message = error.message;
 
-            const file = err instanceof TranslatorError ? err.path : '';
+        const file = error instanceof TranslatorError ? error.path : '';
 
-            logger.error(file, message);
-        }
+        logger.error(file, message);
     }
-
-    logger.info(
-        output,
-        `translated documentation from ${sourceLanguage} to ${targetLanguage} language`,
-    );
 }
 
-export type TranslatorParams = {
-    oauthToken: string;
-    folderId: string | undefined;
+type TranslatorParams = {
     input: string;
     output: string;
     sourceLanguage: string;
@@ -128,157 +109,176 @@ export type TranslatorParams = {
     // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
 };
 
-function translator(params: TranslatorParams) {
-    const {
-        oauthToken,
-        folderId,
-        input,
-        output,
-        sourceLanguage,
-        targetLanguage,
-        // yandexCloudTranslateGlossaryPairs,
-    } = params;
-
-    const tmap = new Map<string, Defer>();
-    const session = new Session({oauthToken});
-    const client = session.client(TranslationServiceClient);
-    const request = (texts: string[]) => () => {
-        return client
-            .translate(
-                TranslateRequest.fromPartial({
-                    texts,
+type RequesterParams = {
+    auth: string;
+    folderId: string | undefined;
+    sourceLanguage: string;
+    targetLanguage: string;
+    dryRun: boolean;
+};
+
+type Request = {
+    (texts: string[]): () => Promise<string[]>;
+    stat: {
+        bytes: number;
+        chunks: number;
+    };
+};
+
+type Split = (path: string, texts: string[]) => Promise<string>[];
+
+type Cache = Map<string, Defer>;
+
+type Translations = {
+    translations: {
+        text: string;
+    }[];
+};
+
+function requester(params: RequesterParams, cache: Cache) {
+    const {auth, folderId, sourceLanguage, targetLanguage, dryRun} = params;
+    const resolve = (text: string, index: number, texts: string[]) => {
+        const defer = cache.get(texts[index]);
+        if (defer) {
+            defer.resolve(text);
+        }
+        return text;
+    };
+
+    const request = function request(texts: string[]) {
+        request.stat.bytes += bytes(texts);
+        request.stat.chunks++;
+
+        return async function () {
+            if (dryRun) {
+                return texts.map(resolve);
+            }
+
+            return axios({
+                method: 'POST',
+                url: 'https://translate.api.cloud.yandex.net/translate/v2/translate',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: auth,
+                },
+                data: {
                     folderId,
+                    texts,
                     sourceLanguageCode: sourceLanguage,
                     targetLanguageCode: targetLanguage,
-                    // glossaryConfig: {
-                    //     glossaryData: {
-                    //         glossaryPairs: yandexCloudTranslateGlossaryPairs,
-                    //     },
-                    // },
-                    format: Format.HTML,
-                }),
-            )
-            .then((results) => {
-                return results.translations.map(({text}, index) => {
-                    const defer = tmap.get(texts[index]);
-                    if (defer) {
-                        defer.resolve([text]);
+                    format: 'HTML',
+                },
+            })
+                .then(({data, status}) => {
+                    if (status === 200) {
+                        return data;
+                    } else {
+                        throw new Error(data.message);
                     }
-
-                    return text;
+                })
+                .then((result: Translations) => {
+                    return result.translations.map(({text}, index) => {
+                        return resolve(text, index, texts);
+                    });
+                })
+                .catch((error) => {
+                    console.error(error);
+                    throw new RequestError(error);
                 });
-            });
+        };
+    };
+
+    request.stat = {
+        bytes: 0,
+        chunks: 0,
     };
 
-    return async (mdPath: string) => {
-        if (!mdPath.endsWith('.md')) {
+    return request;
+}
+
+function translator(params: TranslatorParams, split: Split) {
+    const {input, output, sourceLanguage, targetLanguage} = params;
+    const inputRoot = resolve(input);
+    const outputRoot = resolve(output);
+
+    return async (path: string) => {
+        const ext = extname(path);
+        if (!['.yaml', '.json', '.md'].includes(ext)) {
             return;
         }
 
-        try {
-            logger.info(mdPath, 'translating');
+        const inputPath = join(inputRoot, path);
+        const outputPath = join(outputRoot, path.replace(sourceLanguage, targetLanguage));
+        const content = await loadFile(inputPath);
 
-            const inputPath = resolve(input, sourceLanguage, mdPath);
-            const outputPath = resolve(output, targetLanguage, mdPath);
-            const md = await readFile(inputPath, {encoding: 'utf-8'});
+        await mkdir(dirname(outputPath), {recursive: true});
 
-            await mkdir(dirname(outputPath), {recursive: true});
+        if (!content) {
+            await dumpFile(outputPath, content);
+            return;
+        }
 
-            if (!md) {
-                await writeFile(outputPath, md);
-                return;
-            }
+        const {units, skeleton} = extract(content, {
+            source: {
+                language: sourceLanguage,
+                locale: 'RU',
+            },
+            target: {
+                language: targetLanguage,
+                locale: 'US',
+            },
+        });
 
-            const {units, skeleton} = extract({
-                source: {
-                    language: sourceLanguage,
-                    locale: 'RU',
-                },
-                target: {
-                    language: targetLanguage,
-                    locale: 'US',
-                },
-                markdown: md,
-                markdownPath: mdPath,
-                skeletonPath: '',
-            });
-
-            if (!units.length) {
-                await writeFile(outputPath, md);
-                return;
-            }
+        if (!units.length) {
+            await dumpFile(outputPath, content);
+            return;
+        }
 
-            const parts = await Promise.all(
-                (units as string[]).reduce(
-                    (
-                        {
-                            promises,
-                            buffer,
-                            bufferSize,
-                        }: {
-                            promises: Promise<string[]>[];
-                            buffer: string[];
-                            bufferSize: number;
-                        },
-                        text,
-                        index,
-                    ) => {
-                        if (text.length >= BYTES_LIMIT) {
-                            logger.warn(
-                                mdPath,
-                                'Skip document part for translation. Part is too big.',
-                            );
-                            promises.push(Promise.resolve([text]));
-                            return {promises, buffer, bufferSize};
-                        }
-
-                        const defer = tmap.get(text);
-                        if (defer) {
-                            console.log('SKIPPED', text);
-                            promises.push(defer.promise);
-                            return {promises, buffer, bufferSize};
-                        }
-
-                        if (bufferSize + text.length > BYTES_LIMIT) {
-                            promises.push(backoff(request(buffer)));
-                            buffer = [];
-                            bufferSize = 0;
-                        }
-
-                        buffer.push(text);
-                        bufferSize += text.length;
-                        tmap.set(text, new Defer());
-
-                        if (index === units.length - 1) {
-                            promises.push(backoff(request(buffer)));
-                        }
-
-                        return {promises, buffer, bufferSize};
-                    },
-                    {
-                        promises: [],
-                        buffer: [],
-                        bufferSize: 0,
-                    },
-                ).promises,
-            );
+        const parts = await Promise.all(split(path, units));
+        const composed = compose(skeleton, parts, {useSource: true});
+
+        await dumpFile(outputPath, composed);
+    };
+}
 
-            const translations = ([] as string[]).concat(...parts);
+function splitter(request: Request, cache: Cache): Split {
+    return function (path: string, texts: string[]) {
+        const promises: Promise<string>[] = [];
+        let buffer: string[] = [];
+        let bufferSize = 0;
 
-            const composed = await compose({
-                useSource: true,
-                units: translations,
-                skeleton,
-            });
+        const release = () => {
+            backoff(request(buffer));
+            buffer = [];
+            bufferSize = 0;
+        };
 
-            await writeFile(outputPath, composed);
+        for (const text of texts) {
+            if (text.length >= BYTES_LIMIT) {
+                logger.warn(path, 'Skip document part for translation. Part is too big.');
+                promises.push(Promise.resolve(text));
+            } else {
+                const defer = cache.get(text) || new Defer();
+                promises.push(defer.promise);
+
+                if (!cache.get(text)) {
+                    if (bufferSize + text.length > BYTES_LIMIT) {
+                        release();
+                    }
 
-            logger.info(outputPath, 'finished translating');
-        } catch (err) {
-            if (err instanceof Error) {
-                throw new TranslatorError(err.toString(), mdPath);
+                    buffer.push(text);
+                    bufferSize += text.length;
+                }
+
+                cache.set(text, defer);
             }
         }
+
+        if (bufferSize) {
+            release();
+        }
+
+        return promises;
     };
 }
 
@@ -286,26 +286,8 @@ function backoff(action: () => Promise<string[]>): Promise<string[]> {
     return retry(
         {
             times: RETRY_LIMIT,
-            interval: (count: number) => {
-                // eslint-disable-next-line no-bitwise
-                return (1 << count) * 1000;
-            },
+            interval: (count: number) => Math.pow(2, count) * 1000,
         },
         asyncify(action),
     );
 }
-
-class Defer {
-    resolve!: (text: string[]) => void;
-
-    reject!: (error: any) => void;
-
-    promise: Promise<string[]>;
-
-    constructor() {
-        this.promise = new Promise((resolve, reject) => {
-            this.resolve = resolve;
-            this.reject = reject;
-        });
-    }
-}

package/src/cmd/translate/index.ts

@@ -1,5 +1,5 @@
-import {extract} from '../xliff/extract';
-import {compose} from '../xliff/compose';
+import {extract} from './extract';
+import {compose} from './compose';
 import {handler} from './handler';
 
 import {Argv} from 'yargs';
@@ -18,32 +18,55 @@ const translate = {
 };
 
 function builder<T>(argv: Argv<T>) {
-    return argv
-        .command(extract)
-        .command(compose)
-        .option('folder-id', {
-            describe: 'folder id',
-            type: 'string',
-        })
-        .option('source-language', {
-            alias: 'sl',
-            describe: 'source language code',
-            type: 'string',
-        })
-        .option('target-language', {
-            alias: 'tl',
-            describe: 'target language code',
-            type: 'string',
-        })
-        .option('include', {
-            describe: 'relative to input globs to include in processing',
-            type: 'string',
-        })
-        .option('exclude', {
-            describe: 'relative to input globs to exclude from processing',
-            type: 'string',
-        })
-        .check(argvValidator);
+    return (
+        argv
+            // @ts-ignore
+            .command(extract)
+            // @ts-ignore
+            .command(compose)
+            .option('input', {
+                alias: 'i',
+                describe: 'input folder with markdown files',
+                type: 'string',
+                default: process.cwd(),
+            })
+            .option('output', {
+                alias: 'o',
+                describe: 'output folder to store xliff and skeleton files',
+                type: 'string',
+            })
+            .option('auth', {
+                describe: 'auth credentials path',
+                type: 'string',
+            })
+            .option('folder', {
+                describe: 'folder',
+                type: 'string',
+            })
+            .option('source', {
+                alias: ['sl', 'sll', 'source-language', 'source-language-locale'],
+                describe: 'source language and locale',
+                type: 'string',
+            })
+            .option('target', {
+                alias: ['tl', 'tll', 'target-language', 'target-language-locale'],
+                describe: 'target language and locale',
+                type: 'string',
+            })
+            .option('include', {
+                describe: 'relative to input globs to include in processing',
+                type: 'string',
+            })
+            .option('exclude', {
+                describe: 'relative to input globs to exclude from processing',
+                type: 'string',
+            })
+            .option('dry-run', {
+                describe: 'check command usage',
+                type: 'boolean',
+            })
+            .check(argvValidator)
+    );
 }
 
 export {translate};

package/src/cmd/translate/utils/fs.ts

@@ -0,0 +1,52 @@
+import {JSONValue, resolveRefs} from '@diplodoc/translation';
+import {extname} from 'path';
+import {dump, load} from 'js-yaml';
+import {readFile, writeFile} from 'fs/promises';
+
+function parseFile(text: string, path: string): JSONValue | string {
+    if (typeof text !== 'string') {
+        return text;
+    }
+
+    switch (extname(path)) {
+        case '.yaml':
+            return load(text) as object;
+        case '.json':
+            return JSON.parse(text);
+        default:
+            return text;
+    }
+}
+
+function stringifyFile(content: JSONValue | string, path: string): string {
+    if (typeof content === 'string') {
+        return content;
+    }
+
+    switch (extname(path)) {
+        case '.yaml':
+            return dump(content);
+        case '.json':
+            return JSON.stringify(content);
+        default:
+            return content as unknown as string;
+    }
+}
+
+export async function loadFile<T = string | JSONValue>(path: string): Promise<T> {
+    const text = await readFile(path, 'utf8');
+
+    let content = parseFile(text, path);
+
+    if (content && typeof content === 'object') {
+        content = await resolveRefs(content, path, parseFile);
+    }
+
+    return content as T;
+}
+
+export async function dumpFile(path: string, content: string | JSONValue) {
+    const text = stringifyFile(content, path);
+
+    await writeFile(path, text, 'utf8');
+}