@diplodoc/cli 4.0.0

package/src/cmd/translate/index.ts

@@ -0,0 +1,261 @@
+import {
+    eachLimit,
+    retry,
+    asyncify,
+} from 'async';
+
+import {dirname, resolve, join} from 'path';
+import {readFile, writeFile, mkdir} from 'fs/promises';
+import {XMLParser} from 'fast-xml-parser';
+
+import {Session} from '@yandex-cloud/nodejs-sdk/dist/session';
+import {TranslationServiceClient} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/service_clients';
+import {
+    TranslateRequest,
+    TranslateRequest_Format as Format,
+} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/ai/translate/v2/translation_service';
+
+const yfm2xliff = require('@doc-tools/yfm2xliff/lib/cjs');
+
+import {ArgvService} from '../../services';
+import {getYandexOAuthToken} from '../../packages/credentials';
+import {glob, logger} from '../../utils';
+
+import {Argv, Arguments} from 'yargs';
+
+import {YandexCloudTranslateGlossaryPair} from '../../models';
+
+const composer = async (xliff: string, skeleton: string): Promise<string> => new Promise((res, rej) =>
+    yfm2xliff.compose(xliff, skeleton, (err: Error, composed: string) => {
+        if (err) {
+            rej(err);
+        }
+
+        return res(composed);
+    }));
+
+const command = 'translate';
+
+const description = 'translate documentation with Yandex.Cloud Translator API';
+
+const translate = {
+    command,
+    description,
+    handler,
+    builder,
+};
+
+const MD_GLOB = '**/*.md';
+const REQUESTS_LIMIT = 20;
+const RETRY_LIMIT = 8;
+const MTRANS_LOCALE = 'MTRANS';
+
+function builder<T>(argv: Argv<T>) {
+    return argv
+        .option('source-language', {
+            alias: 'sl',
+            describe: 'source language code',
+            type: 'string',
+        })
+        .option('target-language', {
+            alias: 'tl',
+            describe: 'target language code',
+            type: 'string',
+        })
+        .demandOption(
+            ['source-language', 'target-language'],
+            'command requires to specify source and target languages');
+}
+
+class TranslatorError extends Error {
+    path: string;
+
+    constructor(message: string, path: string) {
+        super(message);
+
+        this.path = path;
+    }
+}
+
+/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+async function handler(args: Arguments<any>) {
+    ArgvService.init({
+        ...args,
+    });
+
+    const {input,
+        output,
+        yandexCloudTranslateFolderId,
+        yandexCloudTranslateGlossaryPairs,
+        sl: sourceLanguage,
+        tl: targetLanguage,
+    } = args;
+
+    logger.info(input, `translating documentation from ${sourceLanguage} to ${targetLanguage} language`);
+
+    try {
+        let found = [];
+
+        ({state: {found}} = await glob(join(input, MD_GLOB), {
+            nosort: true,
+        }));
+
+        const oauthToken = await getYandexOAuthToken();
+
+        const translatorParams = {
+            input,
+            output,
+            sourceLanguage,
+            targetLanguage,
+            yandexCloudTranslateGlossaryPairs,
+            folderId: yandexCloudTranslateFolderId,
+            oauthToken,
+        };
+
+        const translateFn = translator(translatorParams);
+
+        await eachLimit(found, REQUESTS_LIMIT, asyncify(translateFn));
+    } catch (err) {
+        if (err instanceof Error || err instanceof TranslatorError) {
+            const message = err.message;
+
+            const file = err instanceof TranslatorError ? err.path : '';
+
+            logger.error(file, message);
+        }
+    }
+
+    logger.info(output, `translated documentation from ${sourceLanguage} to ${targetLanguage} language`);
+}
+
+export type TranslatorParams = {
+    oauthToken: string;
+    folderId: string;
+    input: string;
+    output: string;
+    sourceLanguage: string;
+    targetLanguage: string;
+    yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
+};
+
+function translator(params: TranslatorParams) {
+    const {
+        oauthToken,
+        folderId,
+        input,
+        output,
+        sourceLanguage,
+        targetLanguage,
+        yandexCloudTranslateGlossaryPairs,
+    } = params;
+
+    const session = new Session({oauthToken});
+    const client = session.client(TranslationServiceClient);
+
+    return async (mdPath: string) => {
+        try {
+            logger.info(mdPath, 'translating');
+
+            const md = await readFile(resolve(mdPath), {encoding: 'utf-8'});
+
+            const extracted = await yfm2xliff.extract({
+                md,
+                mdPath,
+                source: sourceLanguage,
+                target: targetLanguage,
+                sklPath: '',
+                xlfPath: '',
+            });
+
+            const texts = parseSourcesFromXLIFF(extracted.xliff);
+
+            const machineTranslateParams = TranslateRequest.fromPartial({
+                texts,
+                folderId,
+                sourceLanguageCode: sourceLanguage,
+                targetLanguageCode: targetLanguage,
+                glossaryConfig: {
+                    glossaryData: {
+                        glossaryPairs: yandexCloudTranslateGlossaryPairs,
+                    },
+                },
+                format: Format.PLAIN_TEXT,
+            });
+
+            const translations = await retry({times: RETRY_LIMIT, interval: (count: number) => {
+                // eslint-disable-next-line no-bitwise
+                return (1 << count) * 1000;
+            }}, asyncify(async () =>
+                await client.translate(machineTranslateParams)
+                    .then((results: {translations: {text: string}[]}) =>
+                        results.translations.map(({text}: {text: string}) => text)),
+            ));
+
+            const createXLIFFDocumentParams = {
+                sourceLanguage: sourceLanguage + '-' + MTRANS_LOCALE,
+                targetLanguage: targetLanguage + '-' + MTRANS_LOCALE,
+                sources: texts,
+                targets: translations as string[],
+            };
+
+            const translatedXLIFF = createXLIFFDocument(createXLIFFDocumentParams);
+
+            const composed = await composer(translatedXLIFF, extracted.skeleton);
+
+            const outputPath = mdPath.replace(input, output);
+
+            await mkdir(dirname(outputPath), {recursive: true});
+            await writeFile(outputPath, composed);
+
+            logger.info(outputPath, 'finished translating');
+        } catch (err) {
+            if (err instanceof Error) {
+                throw new TranslatorError(err.toString(), mdPath);
+            }
+        }
+    };
+}
+
+function parseSourcesFromXLIFF(xliff: string) {
+    const parser = new XMLParser();
+
+    const inputs = parser.parse(xliff)?.xliff?.file?.body['trans-unit'] ?? [];
+
+    return Array.isArray(inputs)
+        ? inputs.map(({source}: {source: string}) => source)
+        : [inputs.source];
+}
+
+export type CreateXLIFFDocumentParams = {
+    sourceLanguage: string;
+    targetLanguage: string;
+    sources: string[];
+    targets: string[];
+};
+
+function createXLIFFDocument(params: CreateXLIFFDocumentParams) {
+    const {sourceLanguage, targetLanguage, sources, targets} = params;
+
+    const unit = (text: string, i: number): string => `
+<trans-unit id="${i + 1}">
+    <source xml:lang="${sourceLanguage}">${sources[i]}</source>
+    <target xml:lang="${targetLanguage}">${text}</target>
+</trans-unit>`;
+
+    const doc = `
+<?xml version="1.0" encoding="UTF-8"?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" version="1.2">
+    <file original="" source-language="${sourceLanguage}" target-language="${targetLanguage}">
+        <header>
+            <skl><external-file href="" /></skl>
+        </header>
+        <body>${targets.map(unit)}</body>
+    </file>
+</xliff>`;
+
+    return doc;
+}
+
+export {translate};
+
+export default {translate};

package/src/cmd/xliff/compose.ts

@@ -0,0 +1,222 @@
+const {promises: {readFile, writeFile, mkdir}} = require('fs');
+import {join, extname, dirname} from 'path';
+
+import markdownTranslation, {ComposeParameters} from '@diplodoc/markdown-translation';
+import {Arguments, Argv} from 'yargs';
+import {eachLimit} from 'async';
+
+import {ArgvService} from '../../services';
+import {glob, logger} from '../../utils';
+
+const command = 'compose';
+
+const description = 'compose xliff and skeleton into documentation';
+
+const compose = {command, description, handler, builder};
+
+const SKL_MD_GLOB = '**/*.skl.md';
+const XLF_GLOB = '**/*.xliff';
+const MAX_CONCURRENCY = 50;
+
+class ComposeError extends Error {
+    path: string;
+
+    constructor(message: string, path: string) {
+        super(message);
+
+        this.path = path;
+    }
+}
+
+const USAGE = 'yfm xliff compose \
+--input <folder-with-xliff-and-skeleton> \
+--ouput <folder-to-store-translated-markdown>';
+
+function builder<T>(argv: Argv<T>) {
+    return argv
+        .option('input', {
+            alias: 'i',
+            describe: 'input folder with xliff and skeleton files',
+            type: 'string',
+        }).option('output', {
+            alias: 'o',
+            describe: 'output folder where translated markdown will be stored',
+            type: 'string',
+        }).demandOption(['input', 'output'], USAGE);
+}
+
+/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+async function handler(args: Arguments<any>) {
+    ArgvService.init({
+        ...args,
+    });
+
+    const {input, output} = args;
+
+    let cache = {};
+    let skeletonPaths: string[] = [];
+    let xliffPaths: string[] = [];
+
+    try {
+        ({state: {found: skeletonPaths, cache}} = await glob(join(input, SKL_MD_GLOB), {
+            nosort: false,
+            cache,
+        }));
+
+        ({state: {found: xliffPaths, cache}} = await glob(join(input, XLF_GLOB), {
+            nosort: false,
+            cache,
+        }));
+
+        if (xliffPaths.length !== skeletonPaths.length) {
+            throw new ComposeError('number of xliff and skeleton files does\'not match', input);
+        }
+    } catch (err) {
+        if (err instanceof Error || err instanceof ComposeError) {
+            const file = err instanceof ComposeError ? err.path : input;
+
+            logger.error(file, err.message);
+        }
+    }
+
+    const pipelineParameters = {input, output};
+    const configuredPipeline = pipeline(pipelineParameters);
+
+    try {
+        logger.info(input, 'staring translated markdown composition pipeline');
+
+        await eachLimit(xliffPaths, MAX_CONCURRENCY, configuredPipeline);
+
+        logger.info(input, 'finished translated markdown composition pipeline');
+    } catch (err) {
+        if (err instanceof Error || err instanceof ComposeError) {
+            const file = err instanceof ComposeError ? err.path : input;
+
+            logger.error(file, err.message);
+        }
+    }
+}
+
+export type PipelineParameters = {
+    input: string;
+    output: string;
+};
+
+function pipeline(params: PipelineParameters) {
+    const {input, output} = params;
+
+    return async (xliffPath: string) => {
+        const extension = extname(xliffPath);
+        const extensionLessPath = xliffPath.replace(extension, '');
+        const skeletonPath = extensionLessPath + '.skl.md';
+
+        const readerParameters = {xliffPath, skeletonPath};
+        const read = await reader(readerParameters);
+
+        const composerParameters = {
+            ...read,
+            skeletonPath,
+            xliffPath,
+        };
+        const {markdown} = await composer(composerParameters);
+
+        const inputRelativePath = extensionLessPath.slice(input.length);
+        const markdownPath = join(output, inputRelativePath) + '.md';
+
+        const writerParameters = {
+            markdown,
+            markdownPath,
+        };
+        await writer(writerParameters);
+    };
+}
+
+export type ReaderParameters = {
+    skeletonPath: string;
+    xliffPath: string;
+};
+
+async function reader(params: ReaderParameters) {
+    const {skeletonPath, xliffPath} = params;
+
+    let skeleton;
+    let xlf;
+
+    try {
+        logger.info(skeletonPath, 'reading skeleton file');
+
+        skeleton = await readFile(skeletonPath, {encoding: 'utf-8'});
+
+        logger.info(skeletonPath, 'finished reading skeleton file');
+    } catch (err) {
+        if (err instanceof Error) {
+            throw new ComposeError(err.message, skeletonPath);
+        }
+    }
+
+    try {
+        logger.info(xliffPath, 'reading xliff file');
+
+        xlf = await readFile(xliffPath, {encoding: 'utf-8'});
+
+        logger.info(xliffPath, 'finished reading xliff file');
+    } catch (err) {
+        if (err instanceof Error) {
+            throw new ComposeError(err.message, xliffPath);
+        }
+    }
+
+    return {skeleton, xlf};
+}
+
+export type ComposerParameters = {
+    skeletonPath: string;
+    xliffPath: string;
+} & ComposeParameters;
+
+async function composer(params: ComposerParameters) {
+    const {skeletonPath, xliffPath} = params;
+    let markdown;
+
+    try {
+        logger.info(skeletonPath, 'composing markdown from xliff and skeleton');
+        logger.info(xliffPath, 'composing markdown from xliff and skeleton');
+
+        markdown = markdownTranslation.compose(params);
+
+        logger.info(skeletonPath, 'finished composing markdown from xliff and skeleton');
+        logger.info(xliffPath, 'finished composing markdown from xliff and skeleton');
+    } catch (err) {
+        if (err instanceof Error) {
+            throw new ComposeError(err.message, `${xliffPath} ${skeletonPath}`);
+        }
+    }
+
+    return {markdown};
+}
+
+export type WriterParameters = {
+    markdown: string;
+    markdownPath: string;
+};
+
+async function writer(params: WriterParameters) {
+    const {markdown, markdownPath} = params;
+
+    try {
+        logger.info(markdownPath, 'writing markdown file');
+
+        await mkdir(dirname(markdownPath), {recursive: true});
+        await writeFile(markdownPath, markdown);
+
+        logger.info(markdownPath, 'finished writing markdown file');
+    } catch (err) {
+        if (err instanceof Error) {
+            throw new ComposeError(err.message, markdownPath);
+        }
+    }
+}
+
+export {compose};
+
+export default {compose};

package/src/cmd/xliff/extract.ts

@@ -0,0 +1,237 @@
+const {promises: {readFile, writeFile, mkdir}} = require('fs');
+import {join, dirname, extname} from 'path';
+
+import markdownTranslation, {ExtractParameters} from '@diplodoc/markdown-translation';
+import {Arguments, Argv} from 'yargs';
+import {eachLimit, asyncify} from 'async';
+
+import {ArgvService} from '../../services';
+import {glob, logger} from '../../utils';
+
+const command = 'extract';
+
+const description = 'extract xliff and skeleton from yfm documentation';
+
+const extract = {command, description, handler, builder};
+
+const MD_GLOB = '**/*.md';
+
+const MAX_CONCURRENCY = 50;
+
+class ExtractError extends Error {
+    path: string;
+
+    constructor(message: string, path: string) {
+        super(message);
+
+        this.path = path;
+    }
+}
+
+const USAGE = `yfm xliff extract \
+--input <folder-with-markdown> \
+--output <folder-to-store-xlff-and-skeleton> \
+--sll <source-language>-<source-locale> \
+--tll <target-language>-<target-locale>
+
+where <source/target-language> is the language code, as described in ISO 639-1.
+
+where <source/target-locale> is the locale code in alpha-2 format, as described in ISO 3166-1`;
+
+function builder<T>(argv: Argv<T>) {
+    return argv
+        .option('source-language-locale', {
+            alias: 'sll',
+            describe: 'source language and locale',
+            type: 'string',
+        }).option('target-language-locale', {
+            alias: 'tll',
+            describe: 'target language and locale',
+            type: 'string',
+        }).option('input', {
+            alias: 'i',
+            describe: 'input folder with markdown files',
+            type: 'string',
+        }).option('output', {
+            alias: 'o',
+            describe: 'output folder to store xliff and skeleton files',
+            type: 'string',
+        }).demandOption(['source-language-locale', 'target-language-locale', 'input', 'output'], USAGE);
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+async function handler(args: Arguments<any>) {
+    ArgvService.init({
+        ...args,
+    });
+
+    const {input, output, sourceLanguageLocale, targetLanguageLocale} = args;
+
+    let source;
+    let target;
+
+    try {
+        source = parseLanguageLocale(sourceLanguageLocale);
+        target = parseLanguageLocale(targetLanguageLocale);
+    } catch (err) {
+        if (err instanceof Error) {
+            logger.error(input, err.message);
+        }
+    }
+
+    let cache = {};
+    let found: string[] = [];
+
+    try {
+        ({state: {found, cache}} = await glob(join(input, MD_GLOB), {
+            nosort: true,
+            cache,
+        }));
+    } catch (err) {
+        if (err instanceof Error) {
+            logger.error(input, err.message);
+        }
+    }
+
+    const pipelineParameters = {source, target, input, output};
+    const configuredPipeline = pipeline(pipelineParameters);
+
+    try {
+        logger.info(input, 'starting xliff and skeleton generation pipeline');
+
+        await eachLimit(found, MAX_CONCURRENCY, asyncify(configuredPipeline));
+
+        logger.info(input, 'finished xliff and skeleton generation pipeline');
+    } catch (err) {
+        if (err instanceof Error || err instanceof ExtractError) {
+            const file = err instanceof ExtractError ? err.path : input;
+
+            logger.error(file, err.message);
+        }
+    }
+}
+
+function parseLanguageLocale(languageLocale: string) {
+    const [language, locale] = languageLocale.split('-');
+    if (language?.length && locale?.length) {
+        return {language, locale};
+    }
+
+    throw new Error('invalid language-locale string');
+}
+
+export type PipelineParameters = {
+    input: string;
+    output: string;
+    source: ExtractParameters['source'];
+    target: ExtractParameters['target'];
+};
+
+function pipeline(params: PipelineParameters) {
+    const {input, output, source, target} = params;
+
+    return async (markdownPath: string) => {
+        const markdown = await reader({path: markdownPath});
+        const extension = extname(markdownPath);
+
+        const outputRelativePath = markdownPath
+            .replace(extension, '')
+            .slice(input.length);
+
+        const outputPath = join(output, outputRelativePath);
+        const xlfPath = outputPath + '.xliff';
+        const skeletonPath = outputPath + '.skl.md';
+
+        const extractParameters = {
+            markdownPath,
+            skeletonPath,
+            markdown,
+            source,
+            target,
+        };
+
+        const extracted = await extractor(extractParameters);
+
+        const writerParameters = {
+            ...extracted,
+            xlfPath,
+            skeletonPath,
+        };
+
+        await writer(writerParameters);
+    };
+}
+
+export type ReaderParameters = {
+    path: string;
+};
+
+async function reader(params: ReaderParameters) {
+    const {path} = params;
+
+    let markdown;
+    try {
+        logger.info(path, 'reading markdown file');
+
+        markdown = await readFile(path, {encoding: 'utf-8'});
+
+        logger.info(path, 'finished reading markdown file');
+    } catch (err) {
+        if (err instanceof Error) {
+            throw new ExtractError(err.message, path);
+        }
+    }
+
+    return markdown;
+}
+
+export type ExtractorParameters = {
+    source: ExtractParameters['source'];
+    target: ExtractParameters['target'];
+    skeletonPath: string;
+    markdownPath: string;
+    markdown: string;
+};
+
+async function extractor(params: ExtractorParameters) {
+    let extracted;
+
+    logger.info(params.markdownPath, 'generating skeleton and xliff from markdown');
+
+    try {
+        extracted = markdownTranslation.extract(params);
+    } catch (err) {
+        if (err instanceof Error) {
+            throw new ExtractError(err.message, params.markdownPath);
+        }
+    }
+
+    logger.info(params.markdownPath, 'finished generating skeleton and xliff from markdown');
+
+    return extracted;
+}
+
+export type WriterParameters = {
+    skeletonPath: string;
+    skeleton: string;
+    xlfPath: string;
+    xlf: string;
+};
+
+async function writer(params: WriterParameters) {
+    const {xlfPath, skeletonPath, xlf, skeleton} = params;
+
+    logger.info(params.xlfPath, 'writing xliff file');
+    logger.info(params.skeletonPath, 'writing skeleton file');
+
+    await mkdir(dirname(xlfPath), {recursive: true});
+
+    await Promise.all([writeFile(skeletonPath, skeleton), writeFile(xlfPath, xlf)]);
+
+    logger.info(params.xlfPath, 'finished writing xliff file');
+    logger.info(params.skeletonPath, 'finished writing skeleton file');
+}
+
+export {extract};
+
+export default {extract};