@diplodoc/cli 4.13.4 → 4.13.6

package/src/cmd/xliff/compose.ts

@@ -1,14 +1,18 @@
+import {ok} from 'assert';
+import {argvValidator} from '../../validator';
+import glob from 'glob';
+
 const {
     promises: {readFile, writeFile, mkdir},
 } = require('fs');
-import {dirname, extname, join} from 'path';
+import {dirname, join} from 'path';
 
-import markdownTranslation, {ComposeParameters} from '@diplodoc/markdown-translation';
+// @ts-ignore
+import {ComposeParams, compose as composeMD} from '@diplodoc/markdown-translation';
 import {Arguments, Argv} from 'yargs';
 import {eachLimit} from 'async';
 
 import {ArgvService} from '../../services';
-import {glob, logger} from '../../utils';
 
 const command = 'compose';
 
@@ -16,25 +20,8 @@ const description = 'compose xliff and skeleton into documentation';
 
 const compose = {command, description, handler, builder};
 
-const SKL_MD_GLOB = '**/*.skl.md';
-const XLF_GLOB = '**/*.xliff';
 const MAX_CONCURRENCY = 50;
 
-class ComposeError extends Error {
-    path: string;
-
-    constructor(message: string, path: string) {
-        super(message);
-
-        this.path = path;
-    }
-}
-
-const USAGE =
-    'yfm xliff compose \
---input <folder-with-xliff-and-skeleton> \
---ouput <folder-to-store-translated-markdown>';
-
 function builder<T>(argv: Argv<T>) {
     return argv
         .option('input', {
@@ -47,183 +34,90 @@ function builder<T>(argv: Argv<T>) {
             describe: 'output folder where translated markdown will be stored',
             type: 'string',
         })
-        .demandOption(['input', 'output'], USAGE);
+        .option('target-language', {
+            alias: 'tl',
+            describe: 'target language',
+            type: 'string',
+        })
+        .option('use-source', {
+            describe: 'for debug',
+            type: 'boolean',
+        })
+        .check(argvValidator);
 }
 
+type HandlerParams = {
+    input: string;
+    output: string;
+    targetLanguage: string;
+    exclude?: string[];
+    useSource?: boolean;
+};
+
 /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
-async function handler(args: Arguments<any>) {
+async function handler(args: Arguments<HandlerParams>) {
+    args = Object.assign({}, args.translate || {}, args);
+    delete args.translate;
+
     ArgvService.init({
         ...args,
     });
 
-    const {input, output} = args;
-
-    let cache = {};
-    let skeletonPaths: string[] = [];
-    let xliffPaths: string[] = [];
-
-    try {
-        ({
-            state: {found: skeletonPaths, cache},
-        } = await glob(join(input, SKL_MD_GLOB), {
-            nosort: false,
-            cache,
-        }));
-
-        ({
-            state: {found: xliffPaths, cache},
-        } = await glob(join(input, XLF_GLOB), {
-            nosort: false,
-            cache,
-        }));
-
-        if (xliffPaths.length !== skeletonPaths.length) {
-            throw new ComposeError("number of xliff and skeleton files does'not match", input);
-        }
-    } catch (err) {
-        if (err instanceof Error || err instanceof ComposeError) {
-            const file = err instanceof ComposeError ? err.path : input;
-
-            logger.error(file, err.message);
-        }
-    }
-
-    const pipelineParameters = {input, output};
-    const configuredPipeline = pipeline(pipelineParameters);
-
-    try {
-        logger.info(input, 'staring translated markdown composition pipeline');
+    const {
+        input,
+        output,
+        exclude = [],
+        targetLanguage,
+        useSource = false,
+    } = ArgvService.getConfig() as unknown as HandlerParams;
+
+    ok(input);
+    ok(output);
+    ok(targetLanguage);
+
+    const skeletons = glob.sync('**/*.skl', {
+        cwd: join(input, targetLanguage),
+        ignore: exclude,
+    });
+    const xliffs = glob.sync('**/*.xliff', {
+        cwd: join(input, targetLanguage),
+        ignore: exclude,
+    });
 
-        await eachLimit(xliffPaths, MAX_CONCURRENCY, configuredPipeline);
+    ok(xliffs.length === skeletons.length, 'Inconsistent number of xliff and skeleton files.');
 
-        logger.info(input, 'finished translated markdown composition pipeline');
-    } catch (err) {
-        if (err instanceof Error || err instanceof ComposeError) {
-            const file = err instanceof ComposeError ? err.path : input;
+    const pipelineParameters = {input, output, targetLanguage, useSource};
+    const configuredPipeline = pipeline(pipelineParameters);
 
-            logger.error(file, err.message);
-        }
-    }
+    await eachLimit(skeletons, MAX_CONCURRENCY, configuredPipeline);
 }
 
-export type PipelineParameters = {
-    input: string;
-    output: string;
-};
+export type PipelineParameters = ComposeParams;
 
 function pipeline(params: PipelineParameters) {
-    const {input, output} = params;
-
-    return async (xliffPath: string) => {
-        const extension = extname(xliffPath);
-        const extensionLessPath = xliffPath.replace(extension, '');
-        const skeletonPath = extensionLessPath + '.skl.md';
-
-        const readerParameters = {xliffPath, skeletonPath};
-        const read = await reader(readerParameters);
-
-        const composerParameters = {
-            ...read,
+    const {input, output, targetLanguage, useSource} = params;
+
+    return async (skeletonPath: string) => {
+        const fileName = skeletonPath.split('.').slice(0, -1).join('.');
+        const xliffPath = fileName + '.xliff';
+
+        const [skeleton, xliff] = await Promise.all<string[]>([
+            readFile(join(input, targetLanguage, skeletonPath), 'utf-8'),
+            readFile(join(input, targetLanguage, xliffPath), 'utf-8'),
+        ]);
+        const markdown = composeMD({
+            skeleton,
+            xliff,
             skeletonPath,
             xliffPath,
-        };
-        const {markdown} = await composer(composerParameters);
+            useSource,
+        });
 
-        const inputRelativePath = extensionLessPath.slice(input.length);
-        const markdownPath = join(output, inputRelativePath) + '.md';
-
-        const writerParameters = {
-            markdown,
-            markdownPath,
-        };
-        await writer(writerParameters);
-    };
-}
-
-export type ReaderParameters = {
-    skeletonPath: string;
-    xliffPath: string;
-};
-
-async function reader(params: ReaderParameters) {
-    const {skeletonPath, xliffPath} = params;
-
-    let skeleton;
-    let xlf;
-
-    try {
-        logger.info(skeletonPath, 'reading skeleton file');
-
-        skeleton = await readFile(skeletonPath, {encoding: 'utf-8'});
-
-        logger.info(skeletonPath, 'finished reading skeleton file');
-    } catch (err) {
-        if (err instanceof Error) {
-            throw new ComposeError(err.message, skeletonPath);
-        }
-    }
-
-    try {
-        logger.info(xliffPath, 'reading xliff file');
-
-        xlf = await readFile(xliffPath, {encoding: 'utf-8'});
-
-        logger.info(xliffPath, 'finished reading xliff file');
-    } catch (err) {
-        if (err instanceof Error) {
-            throw new ComposeError(err.message, xliffPath);
-        }
-    }
-
-    return {skeleton, xlf};
-}
-
-export type ComposerParameters = {
-    skeletonPath: string;
-    xliffPath: string;
-} & ComposeParameters;
-
-async function composer(params: ComposerParameters) {
-    const {skeletonPath, xliffPath} = params;
-    let markdown;
-
-    try {
-        logger.info(skeletonPath, 'composing markdown from xliff and skeleton');
-        logger.info(xliffPath, 'composing markdown from xliff and skeleton');
-
-        markdown = markdownTranslation.compose(params);
-
-        logger.info(skeletonPath, 'finished composing markdown from xliff and skeleton');
-        logger.info(xliffPath, 'finished composing markdown from xliff and skeleton');
-    } catch (err) {
-        if (err instanceof Error) {
-            throw new ComposeError(err.message, `${xliffPath} ${skeletonPath}`);
-        }
-    }
-
-    return {markdown};
-}
-
-export type WriterParameters = {
-    markdown: string;
-    markdownPath: string;
-};
-
-async function writer(params: WriterParameters) {
-    const {markdown, markdownPath} = params;
-
-    try {
-        logger.info(markdownPath, 'writing markdown file');
+        const markdownPath = join(output, targetLanguage, fileName);
 
         await mkdir(dirname(markdownPath), {recursive: true});
         await writeFile(markdownPath, markdown);
-
-        logger.info(markdownPath, 'finished writing markdown file');
-    } catch (err) {
-        if (err instanceof Error) {
-            throw new ComposeError(err.message, markdownPath);
-        }
-    }
+    };
 }
 
 export {compose};

package/src/cmd/xliff/extract.ts

@@ -1,14 +1,18 @@
+import {ok} from 'assert';
+import glob from 'glob';
+import {argvValidator} from '../../validator';
+
 const {
     promises: {readFile, writeFile, mkdir},
 } = require('fs');
-import {dirname, extname, join} from 'path';
+import {basename, dirname, join, resolve} from 'path';
 
-import markdownTranslation, {ExtractParameters} from '@diplodoc/markdown-translation';
+// @ts-ignore
+import {ExtractParams, extract as extractMD} from '@diplodoc/markdown-translation';
 import {Arguments, Argv} from 'yargs';
 import {asyncify, eachLimit} from 'async';
 
 import {ArgvService} from '../../services';
-import {glob, logger} from '../../utils';
 
 const command = 'extract';
 
@@ -16,30 +20,8 @@ const description = 'extract xliff and skeleton from yfm documentation';
 
 const extract = {command, description, handler, builder};
 
-const MD_GLOB = '**/*.md';
-
 const MAX_CONCURRENCY = 50;
 
-class ExtractError extends Error {
-    path: string;
-
-    constructor(message: string, path: string) {
-        super(message);
-
-        this.path = path;
-    }
-}
-
-const USAGE = `yfm xliff extract \
---input <folder-with-markdown> \
---output <folder-to-store-xlff-and-skeleton> \
---sll <source-language>-<source-locale> \
---tll <target-language>-<target-locale>
-
-where <source/target-language> is the language code, as described in ISO 639-1.
-
-where <source/target-locale> is the locale code in alpha-2 format, as described in ISO 3166-1`;
-
 function builder<T>(argv: Argv<T>) {
     return argv
         .option('source-language-locale', {
@@ -62,185 +44,132 @@ function builder<T>(argv: Argv<T>) {
             describe: 'output folder to store xliff and skeleton files',
             type: 'string',
         })
-        .demandOption(
-            ['source-language-locale', 'target-language-locale', 'input', 'output'],
-            USAGE,
-        );
+        .check(argvValidator);
 }
 
+type HandlerParams = {
+    input: string;
+    output: string;
+    include?: string[];
+    exclude?: string[];
+    sourceLanguage?: string;
+    sourceLocale?: string;
+    sourceLanguageLocale?: string;
+    targetLanguage?: string;
+    targetLocale?: string;
+    targetLanguageLocale?: string;
+};
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-async function handler(args: Arguments<any>) {
+async function handler(args: Arguments<HandlerParams>) {
+    args = Object.assign({}, args.translate || {}, args);
+    delete args.translate;
+
     ArgvService.init({
         ...args,
     });
 
-    const {input, output, sourceLanguageLocale, targetLanguageLocale} = args;
-
-    let source;
-    let target;
-
-    try {
-        source = parseLanguageLocale(sourceLanguageLocale);
-        target = parseLanguageLocale(targetLanguageLocale);
-    } catch (err) {
-        if (err instanceof Error) {
-            logger.error(input, err.message);
-        }
-    }
-
-    let cache = {};
-    let found: string[] = [];
-
-    try {
-        ({
-            state: {found, cache},
-        } = await glob(join(input, MD_GLOB), {
-            nosort: true,
-            cache,
-        }));
-    } catch (err) {
-        if (err instanceof Error) {
-            logger.error(input, err.message);
-        }
+    const {
+        output,
+        exclude = [],
+        sourceLanguage,
+        sourceLocale,
+        targetLanguage,
+        targetLocale,
+    } = ArgvService.getConfig() as HandlerParams;
+
+    let {
+        input,
+        include = [],
+        sourceLanguageLocale,
+        targetLanguageLocale,
+    } = ArgvService.getConfig() as HandlerParams;
+
+    ok(input);
+    ok(output);
+
+    ok(
+        sourceLanguageLocale || (sourceLanguage && sourceLocale),
+        'Source language and locale should be configured',
+    );
+
+    ok(
+        targetLanguageLocale || (targetLanguage && targetLocale),
+        'Source language and locale should be configured',
+    );
+
+    sourceLanguageLocale = sourceLanguageLocale || sourceLanguage + '-' + sourceLocale;
+    targetLanguageLocale = targetLanguageLocale || targetLanguage + '-' + targetLocale;
+
+    const source = parseLanguageLocale(sourceLanguageLocale);
+    const target = parseLanguageLocale(targetLanguageLocale);
+
+    if (input.endsWith('.md')) {
+        include = [basename(input)];
+        input = dirname(input);
+    } else if (!include.length) {
+        include.push('**/*');
     }
 
-    const pipelineParameters = {source, target, input, output};
-    const configuredPipeline = pipeline(pipelineParameters);
-
-    try {
-        logger.info(input, 'starting xliff and skeleton generation pipeline');
-
-        await eachLimit(found, MAX_CONCURRENCY, asyncify(configuredPipeline));
-
-        logger.info(input, 'finished xliff and skeleton generation pipeline');
-    } catch (err) {
-        if (err instanceof Error || err instanceof ExtractError) {
-            const file = err instanceof ExtractError ? err.path : input;
-
-            logger.error(file, err.message);
-        }
-    }
+    const files = ([] as string[]).concat(
+        ...include.map((match) =>
+            glob.sync(match, {
+                cwd: join(input, source.language),
+                ignore: exclude,
+            }),
+        ),
+    );
+    const found = [...new Set(files)];
+    const configuredPipeline = pipeline({source, target, input, output});
+
+    await eachLimit(found, MAX_CONCURRENCY, asyncify(configuredPipeline));
 }
 
 function parseLanguageLocale(languageLocale: string) {
     const [language, locale] = languageLocale.split('-');
-    if (language?.length && locale?.length) {
-        return {language, locale};
+    if (!language?.length || !locale?.length) {
+        throw new Error('invalid language-locale string');
     }
 
-    throw new Error('invalid language-locale string');
+    return {language, locale};
 }
 
 export type PipelineParameters = {
     input: string;
     output: string;
-    source: ExtractParameters['source'];
-    target: ExtractParameters['target'];
+    source: ExtractParams['source'];
+    target: ExtractParams['target'];
 };
 
 function pipeline(params: PipelineParameters) {
     const {input, output, source, target} = params;
+    const inputRoot = resolve(input, source.language);
+    const outputRoot = resolve(output, target.language);
 
-    return async (markdownPath: string) => {
-        const markdown = await reader({path: markdownPath});
-        const extension = extname(markdownPath);
+    return async (path: string) => {
+        if (!path.endsWith('.md')) {
+            return;
+        }
 
-        const outputRelativePath = markdownPath.replace(extension, '').slice(input.length);
+        const inputPath = join(inputRoot, path);
+        const xliffPath = join(outputRoot, path + '.xliff');
+        const skeletonPath = join(outputRoot, path + '.skl');
+        const markdown = await readFile(inputPath, 'utf-8');
 
-        const outputPath = join(output, outputRelativePath);
-        const xlfPath = outputPath + '.xliff';
-        const skeletonPath = outputPath + '.skl.md';
+        await mkdir(dirname(xliffPath), {recursive: true});
 
-        const extractParameters = {
-            markdownPath,
+        const {xliff, skeleton} = await extractMD({
+            markdownPath: path,
             skeletonPath,
             markdown,
             source,
             target,
-        };
-
-        const extracted = await extractor(extractParameters);
+        });
 
-        const writerParameters = {
-            ...extracted,
-            xlfPath,
-            skeletonPath,
-        };
-
-        await writer(writerParameters);
+        await Promise.all([writeFile(skeletonPath, skeleton), writeFile(xliffPath, xliff)]);
     };
 }
 
-export type ReaderParameters = {
-    path: string;
-};
-
-async function reader(params: ReaderParameters) {
-    const {path} = params;
-
-    let markdown;
-    try {
-        logger.info(path, 'reading markdown file');
-
-        markdown = await readFile(path, {encoding: 'utf-8'});
-
-        logger.info(path, 'finished reading markdown file');
-    } catch (err) {
-        if (err instanceof Error) {
-            throw new ExtractError(err.message, path);
-        }
-    }
-
-    return markdown;
-}
-
-export type ExtractorParameters = {
-    source: ExtractParameters['source'];
-    target: ExtractParameters['target'];
-    skeletonPath: string;
-    markdownPath: string;
-    markdown: string;
-};
-
-async function extractor(params: ExtractorParameters) {
-    let extracted;
-
-    logger.info(params.markdownPath, 'generating skeleton and xliff from markdown');
-
-    try {
-        extracted = markdownTranslation.extract(params);
-    } catch (err) {
-        if (err instanceof Error) {
-            throw new ExtractError(err.message, params.markdownPath);
-        }
-    }
-
-    logger.info(params.markdownPath, 'finished generating skeleton and xliff from markdown');
-
-    return extracted;
-}
-
-export type WriterParameters = {
-    skeletonPath: string;
-    skeleton: string;
-    xlfPath: string;
-    xlf: string;
-};
-
-async function writer(params: WriterParameters) {
-    const {xlfPath, skeletonPath, xlf, skeleton} = params;
-
-    logger.info(params.xlfPath, 'writing xliff file');
-    logger.info(params.skeletonPath, 'writing skeleton file');
-
-    await mkdir(dirname(xlfPath), {recursive: true});
-
-    await Promise.all([writeFile(skeletonPath, skeleton), writeFile(xlfPath, xlf)]);
-
-    logger.info(params.xlfPath, 'finished writing xliff file');
-    logger.info(params.skeletonPath, 'finished writing skeleton file');
-}
-
 export {extract};
 
 export default {extract};