@diplodoc/cli 4.13.6 → 4.13.7

package/src/cmd/translate/utils/index.ts

@@ -0,0 +1,165 @@
+import {ok} from 'assert';
+import {basename, dirname, extname, resolve} from 'path';
+import {readFileSync} from 'node:fs';
+import glob from 'glob';
+
+export {dumpFile, loadFile} from './fs';
+export {extract, compose} from './translate';
+
+type TranslateArgs = {
+    input: string;
+    output?: string;
+    source?: string;
+    sourceLanguage?: string;
+    sourceLanguageLocale?: string;
+    target?: string;
+    targetLanguage?: string;
+    targetLanguageLocale?: string;
+    auth?: string;
+    folder?: string;
+    glossary?: string;
+    include?: string[] | string;
+    exclude?: string[] | string;
+    dryRun?: boolean;
+    useSource?: boolean;
+} & {
+    [prop: string]: any;
+};
+
+export type TranslateParams = {
+    input: string;
+    output: string;
+    source: [string, string];
+    targets: [string, string][];
+    auth?: string;
+    folder?: string;
+    glossary?: string;
+    files: string[];
+    dryRun: boolean;
+    useSource: boolean;
+};
+
+export function normalizeParams(params: TranslateArgs, exts = EXTS): TranslateParams {
+    const source = normalizeLocale(params, 'source')[0];
+    const targets = normalizeLocale(params, 'target');
+    const {input, files} = normalizeInput(params, source[0], exts);
+    const {output, auth, folder, dryRun, useSource} = params;
+
+    ok(input, 'Required param input is not configured');
+
+    return {
+        input,
+        output: output || input,
+        auth,
+        folder,
+        source,
+        targets,
+        files: [...new Set(files)],
+        dryRun: Boolean(dryRun),
+        useSource: Boolean(useSource),
+    };
+}
+
+function normalizeLocale(params: TranslateArgs, scope: 'source' | 'target'): [string, string][] {
+    const _scopeLanguage = params[scope + 'Language'];
+    const _scopeLanguageLocale = params[scope + 'LanguageLocale'];
+    const _scope = params[scope] || _scopeLanguageLocale || _scopeLanguage;
+
+    if (!_scope) {
+        return [['', '']];
+    }
+
+    if (Array.isArray(_scope)) {
+        return _scope.map((_scope) => _scope.split('-'));
+    }
+
+    return [_scope.split('-') as [string, string]];
+}
+
+const EXTS = ['.md', '.yaml', '.json'];
+
+function normalizeInput(params: TranslateArgs, language: string, exts: string[]) {
+    let {input, include = [], exclude = []} = params;
+
+    include = ([] as string[]).concat(include);
+    exclude = ([] as string[]).concat(exclude);
+
+    let files: string[] | null = null;
+    if (extname(input) === '.list') {
+        const list = readFileSync(input, 'utf8').split('\n');
+        input = dirname(input);
+        files = list.map((file) => {
+            const absPath = resolve(input, file);
+
+            if (!absPath.startsWith(input)) {
+                throw new Error(`Insecure access to file out of project scope. (file: ${absPath})`);
+            }
+
+            if (!exts.includes(extname(file))) {
+                throw new Error(`Unhandles file extension. (file: ${absPath})`);
+            }
+
+            return absPath;
+        });
+    } else {
+        if (exts.includes(extname(input))) {
+            files = [basename(input)];
+            input = dirname(input);
+        }
+
+        if (!include.length) {
+            include.push('...');
+        }
+
+        include = include.reduce((acc, item) => {
+            if (item === '...') {
+                acc.push(...exts.map((ext) => (language || '.') + '/**/*' + ext));
+            } else {
+                acc.push(item);
+            }
+
+            return acc;
+        }, [] as string[]);
+
+        files =
+            files ||
+            ([] as string[]).concat(
+                ...include.map((match) =>
+                    glob.sync(match, {
+                        cwd: input,
+                        ignore: exclude,
+                        nodir: true,
+                    }),
+                ),
+            );
+    }
+
+    return {input, files};
+}
+
+export function resolveSchemas(_path: string) {
+    return null;
+}
+
+export function flat<T>(parts: T[][]) {
+    return ([] as T[]).concat(...parts);
+}
+
+export class Defer {
+    resolve!: (text: string[]) => void;
+
+    reject!: (error: any) => void;
+
+    promise: Promise<string[]>;
+
+    constructor() {
+        this.promise = new Promise((resolve, reject) => {
+            this.resolve = resolve;
+            this.reject = reject;
+        });
+    }
+}
+
+export function bytes(texts: string[]) {
+    return texts.reduce((sum, text) => sum + text.length, 0);
+}

package/src/cmd/translate/utils/translate.ts

@@ -0,0 +1,43 @@
+import type {ComposeOptions, ExtractOptions} from '@diplodoc/translation';
+import {compose as _compose, extract as _extract} from '@diplodoc/translation';
+
+class ExtractError extends Error {
+    code: string;
+
+    constructor(error: Error) {
+        super(error?.message || String(error));
+        this.code = 'EXTRACT_ERROR';
+    }
+}
+
+class ComposeError extends Error {
+    code: string;
+
+    constructor(error: Error) {
+        super(error?.message || String(error));
+        this.code = 'COMPOSE_ERROR';
+    }
+}
+
+type Content = Parameters<typeof _extract>[0];
+
+export function extract(content: Content, options: ExtractOptions) {
+    try {
+        const {units, skeleton} = _extract(content, options);
+
+        return {units, skeleton};
+    } catch (error: any) {
+        throw new ExtractError(error);
+    }
+}
+
+type Skeleton = Parameters<typeof _compose>[0];
+type Xliff = Parameters<typeof _compose>[1];
+
+export function compose(skeleton: Skeleton, xliff: Xliff, options: ComposeOptions) {
+    try {
+        return _compose(skeleton, xliff, options);
+    } catch (error: any) {
+        throw new ComposeError(error);
+    }
+}

package/src/cmd/translate/yandex/auth.ts

@@ -0,0 +1,41 @@
+import {readFileSync} from 'fs';
+
+type ServiceAccauntInfo = {
+    id: string;
+    service_account_id: string;
+    public_key: string;
+    private_key: string;
+};
+
+export type AuthInfo = ReturnType<typeof getYandexAuth>;
+
+export function getYandexAuth(path: string) {
+    if (path.startsWith('y0_')) {
+        return {
+            oauthToken: path,
+        };
+    }
+
+    const data = readFileSync(path, 'utf8');
+    try {
+        const json = JSON.parse(data);
+
+        if (isServeseAccount(json)) {
+            return {
+                serviceAccountJson: {
+                    serviceAccountId: json.service_account_id,
+                    accessKeyId: json.id,
+                    privateKey: json.private_key,
+                },
+            };
+        }
+    } catch {}
+
+    return {
+        oauthToken: data,
+    };
+}
+
+function isServeseAccount(json: any): json is ServiceAccauntInfo {
+    return 'private_key' in json;
+}

package/src/index.ts

@@ -5,14 +5,13 @@ import 'threads/register';
 
 import {MAIN_TIMER_ID} from './constants';
 
-import {build, publish, translate, xliff} from './cmd';
+import {build, publish, translate} from './cmd';
 
 console.time(MAIN_TIMER_ID);
 
 yargs
     .command(build)
     .command(publish)
-    .command(xliff)
     .command(translate)
     .option('config', {
         alias: 'c',

package/src/models.ts

@@ -47,8 +47,6 @@ interface YfmConfig {
     buildDisabled: boolean;
     lintConfig: LintConfig;
     resources?: Resources;
-    yandexCloudTranslateFolderId: string;
-    yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
     needToSanitizeHtml: boolean;
 }
 

package/src/cmd/xliff/compose.ts

@@ -1,125 +0,0 @@
-import {ok} from 'assert';
-import {argvValidator} from '../../validator';
-import glob from 'glob';
-
-const {
-    promises: {readFile, writeFile, mkdir},
-} = require('fs');
-import {dirname, join} from 'path';
-
-// @ts-ignore
-import {ComposeParams, compose as composeMD} from '@diplodoc/markdown-translation';
-import {Arguments, Argv} from 'yargs';
-import {eachLimit} from 'async';
-
-import {ArgvService} from '../../services';
-
-const command = 'compose';
-
-const description = 'compose xliff and skeleton into documentation';
-
-const compose = {command, description, handler, builder};
-
-const MAX_CONCURRENCY = 50;
-
-function builder<T>(argv: Argv<T>) {
-    return argv
-        .option('input', {
-            alias: 'i',
-            describe: 'input folder with xliff and skeleton files',
-            type: 'string',
-        })
-        .option('output', {
-            alias: 'o',
-            describe: 'output folder where translated markdown will be stored',
-            type: 'string',
-        })
-        .option('target-language', {
-            alias: 'tl',
-            describe: 'target language',
-            type: 'string',
-        })
-        .option('use-source', {
-            describe: 'for debug',
-            type: 'boolean',
-        })
-        .check(argvValidator);
-}
-
-type HandlerParams = {
-    input: string;
-    output: string;
-    targetLanguage: string;
-    exclude?: string[];
-    useSource?: boolean;
-};
-
-/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
-async function handler(args: Arguments<HandlerParams>) {
-    args = Object.assign({}, args.translate || {}, args);
-    delete args.translate;
-
-    ArgvService.init({
-        ...args,
-    });
-
-    const {
-        input,
-        output,
-        exclude = [],
-        targetLanguage,
-        useSource = false,
-    } = ArgvService.getConfig() as unknown as HandlerParams;
-
-    ok(input);
-    ok(output);
-    ok(targetLanguage);
-
-    const skeletons = glob.sync('**/*.skl', {
-        cwd: join(input, targetLanguage),
-        ignore: exclude,
-    });
-    const xliffs = glob.sync('**/*.xliff', {
-        cwd: join(input, targetLanguage),
-        ignore: exclude,
-    });
-
-    ok(xliffs.length === skeletons.length, 'Inconsistent number of xliff and skeleton files.');
-
-    const pipelineParameters = {input, output, targetLanguage, useSource};
-    const configuredPipeline = pipeline(pipelineParameters);
-
-    await eachLimit(skeletons, MAX_CONCURRENCY, configuredPipeline);
-}
-
-export type PipelineParameters = ComposeParams;
-
-function pipeline(params: PipelineParameters) {
-    const {input, output, targetLanguage, useSource} = params;
-
-    return async (skeletonPath: string) => {
-        const fileName = skeletonPath.split('.').slice(0, -1).join('.');
-        const xliffPath = fileName + '.xliff';
-
-        const [skeleton, xliff] = await Promise.all<string[]>([
-            readFile(join(input, targetLanguage, skeletonPath), 'utf-8'),
-            readFile(join(input, targetLanguage, xliffPath), 'utf-8'),
-        ]);
-        const markdown = composeMD({
-            skeleton,
-            xliff,
-            skeletonPath,
-            xliffPath,
-            useSource,
-        });
-
-        const markdownPath = join(output, targetLanguage, fileName);
-
-        await mkdir(dirname(markdownPath), {recursive: true});
-        await writeFile(markdownPath, markdown);
-    };
-}
-
-export {compose};
-
-export default {compose};

package/src/cmd/xliff/extract.ts

@@ -1,175 +0,0 @@
-import {ok} from 'assert';
-import glob from 'glob';
-import {argvValidator} from '../../validator';
-
-const {
-    promises: {readFile, writeFile, mkdir},
-} = require('fs');
-import {basename, dirname, join, resolve} from 'path';
-
-// @ts-ignore
-import {ExtractParams, extract as extractMD} from '@diplodoc/markdown-translation';
-import {Arguments, Argv} from 'yargs';
-import {asyncify, eachLimit} from 'async';
-
-import {ArgvService} from '../../services';
-
-const command = 'extract';
-
-const description = 'extract xliff and skeleton from yfm documentation';
-
-const extract = {command, description, handler, builder};
-
-const MAX_CONCURRENCY = 50;
-
-function builder<T>(argv: Argv<T>) {
-    return argv
-        .option('source-language-locale', {
-            alias: 'sll',
-            describe: 'source language and locale',
-            type: 'string',
-        })
-        .option('target-language-locale', {
-            alias: 'tll',
-            describe: 'target language and locale',
-            type: 'string',
-        })
-        .option('input', {
-            alias: 'i',
-            describe: 'input folder with markdown files',
-            type: 'string',
-        })
-        .option('output', {
-            alias: 'o',
-            describe: 'output folder to store xliff and skeleton files',
-            type: 'string',
-        })
-        .check(argvValidator);
-}
-
-type HandlerParams = {
-    input: string;
-    output: string;
-    include?: string[];
-    exclude?: string[];
-    sourceLanguage?: string;
-    sourceLocale?: string;
-    sourceLanguageLocale?: string;
-    targetLanguage?: string;
-    targetLocale?: string;
-    targetLanguageLocale?: string;
-};
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-async function handler(args: Arguments<HandlerParams>) {
-    args = Object.assign({}, args.translate || {}, args);
-    delete args.translate;
-
-    ArgvService.init({
-        ...args,
-    });
-
-    const {
-        output,
-        exclude = [],
-        sourceLanguage,
-        sourceLocale,
-        targetLanguage,
-        targetLocale,
-    } = ArgvService.getConfig() as HandlerParams;
-
-    let {
-        input,
-        include = [],
-        sourceLanguageLocale,
-        targetLanguageLocale,
-    } = ArgvService.getConfig() as HandlerParams;
-
-    ok(input);
-    ok(output);
-
-    ok(
-        sourceLanguageLocale || (sourceLanguage && sourceLocale),
-        'Source language and locale should be configured',
-    );
-
-    ok(
-        targetLanguageLocale || (targetLanguage && targetLocale),
-        'Source language and locale should be configured',
-    );
-
-    sourceLanguageLocale = sourceLanguageLocale || sourceLanguage + '-' + sourceLocale;
-    targetLanguageLocale = targetLanguageLocale || targetLanguage + '-' + targetLocale;
-
-    const source = parseLanguageLocale(sourceLanguageLocale);
-    const target = parseLanguageLocale(targetLanguageLocale);
-
-    if (input.endsWith('.md')) {
-        include = [basename(input)];
-        input = dirname(input);
-    } else if (!include.length) {
-        include.push('**/*');
-    }
-
-    const files = ([] as string[]).concat(
-        ...include.map((match) =>
-            glob.sync(match, {
-                cwd: join(input, source.language),
-                ignore: exclude,
-            }),
-        ),
-    );
-    const found = [...new Set(files)];
-    const configuredPipeline = pipeline({source, target, input, output});
-
-    await eachLimit(found, MAX_CONCURRENCY, asyncify(configuredPipeline));
-}
-
-function parseLanguageLocale(languageLocale: string) {
-    const [language, locale] = languageLocale.split('-');
-    if (!language?.length || !locale?.length) {
-        throw new Error('invalid language-locale string');
-    }
-
-    return {language, locale};
-}
-
-export type PipelineParameters = {
-    input: string;
-    output: string;
-    source: ExtractParams['source'];
-    target: ExtractParams['target'];
-};
-
-function pipeline(params: PipelineParameters) {
-    const {input, output, source, target} = params;
-    const inputRoot = resolve(input, source.language);
-    const outputRoot = resolve(output, target.language);
-
-    return async (path: string) => {
-        if (!path.endsWith('.md')) {
-            return;
-        }
-
-        const inputPath = join(inputRoot, path);
-        const xliffPath = join(outputRoot, path + '.xliff');
-        const skeletonPath = join(outputRoot, path + '.skl');
-        const markdown = await readFile(inputPath, 'utf-8');
-
-        await mkdir(dirname(xliffPath), {recursive: true});
-
-        const {xliff, skeleton} = await extractMD({
-            markdownPath: path,
-            skeletonPath,
-            markdown,
-            source,
-            target,
-        });
-
-        await Promise.all([writeFile(skeletonPath, skeleton), writeFile(xliffPath, xliff)]);
-    };
-}
-
-export {extract};
-
-export default {extract};

package/src/cmd/xliff/index.ts

@@ -1,30 +0,0 @@
-import {extract} from './extract';
-import {compose} from './compose';
-
-import {Argv} from 'yargs';
-
-const command = 'xliff';
-
-const description =
-    'extract xliff and skeleton from documentation files\ncompose xliff and skeleton into documentation';
-
-const xliff = {
-    command,
-    description,
-    handler: () => {},
-    builder,
-};
-
-function builder<T>(argv: Argv<T>) {
-    return argv
-        .command(extract)
-        .command(compose)
-        .demandCommand(
-            1,
-            `provide one of the folowing ${command} commands: ${extract.command}, ${compose.command}`,
-        );
-}
-
-export {xliff};
-
-export default {xliff};