@diplodoc/cli 4.1.0 → 4.2.1

package/package.json

@@ -3,7 +3,7 @@
   "author": "Yandex Data UI Team <data-ui@yandex-team.ru>",
   "description": "Make documentation using yfm-docs in Markdown and HTML formats",
   "license": "MIT",
-  "version": "4.1.0",
+  "version": "4.2.1",
   "repository": {
     "type": "git",
     "url": "git@github.com:diplodoc-platform/cli.git"
@@ -21,20 +21,21 @@
     "start": "node build/index.js",
     "lint": "eslint \"src/**/*.{js,jsx,ts,tsx}\"",
     "lint:fix": "npm run lint -- --fix",
-    "prepublishOnly": "npm run lint && npm run build"
+    "prepublishOnly": "npm run lint && npm run build",
+    "git:head": "git checkout master && git pull"
   },
   "engines": {
-    "node": "14.*"
+    "node": "18.*",
+    "npm:": "9.*"
   },
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.0",
     "@aws-sdk/client-s3": "^3.369.0",
-    "@diplodoc/client": "^0.0.14",
-    "@diplodoc/markdown-translation": "^1.0.0-beta.1",
+    "@diplodoc/client": "^1.0.2",
+    "@diplodoc/markdown-translation": "^1.0.0-beta.4",
     "@diplodoc/mermaid-extension": "^1.1.1",
     "@diplodoc/openapi-extension": "^1.3.3",
     "@diplodoc/transform": "^4.0.0",
-    "@doc-tools/yfm2xliff": "0.0.5",
     "@octokit/core": "4.2.4",
     "@types/glob": "^8.0.0",
     "@types/json-stringify-safe": "^5.0.0",
@@ -65,6 +66,9 @@
     "yargs": "17.7.2"
   },
   "devDependencies": {
+    "@diplodoc/eslint-config": "^1.0.14",
+    "@diplodoc/prettier-config": "^1.0.0",
+    "@diplodoc/tsconfig": "^1.0.2",
     "@types/async": "^3.2.15",
     "@types/chalk": "2.2.0",
     "@types/html-escaper": "^3.0.0",
@@ -75,15 +79,12 @@
     "@types/shelljs": "0.8.12",
     "@types/tar-stream": "^2.2.2",
     "@types/yargs": "17.0.24",
-    "@typescript-eslint/eslint-plugin": "2.34.0",
-    "@typescript-eslint/parser": "2.34.0",
-    "babel-eslint": "10.1.0",
     "esbuild": "^0.17.17",
-    "eslint": "6.8.0",
-    "eslint-plugin-security": "1.7.1",
+    "eslint": "^8.0.0",
     "husky": "8.0.3",
     "lint-staged": "13.2.2",
     "lodash": "4.17.21",
+    "prettier": "^3.0.3",
     "ts-jest": "29.1.0",
     "ts-node": "^10.9.1",
     "typescript": "4.9.5"

package/src/cmd/build/index.ts

@@ -1,6 +1,7 @@
 import {Arguments, Argv} from 'yargs';
 import {
-    BUNDLE_FOLDER, LINT_CONFIG_FILENAME,
+    BUNDLE_FOLDER,
+    LINT_CONFIG_FILENAME,
     REDIRECTS_FILENAME,
     Stage,
     TMP_INPUT_FOLDER,
@@ -15,7 +16,8 @@ import {
     initLinterWorkers,
     processAssets,
     processExcludedFiles,
-    processLinter, processLogs,
+    processLinter,
+    processLogs,
     processPages,
     processServiceFiles,
 } from '../../steps';
@@ -166,7 +168,10 @@ function builder<T>(argv: Argv<T>) {
         })
         .check(argvValidator)
         .example('yfm -i ./input -o ./output', '')
-        .demandOption(['input', 'output'], 'Please provide input and output arguments to work with this tool');
+        .demandOption(
+            ['input', 'output'],
+            'Please provide input and output arguments to work with this tool',
+        );
 }
 
 async function handler(args: Arguments<any>) {
@@ -235,8 +240,12 @@ async function handler(args: Arguments<any>) {
                         const resourcePaths: string[] = [];
 
                         // collect paths of all resources
-                        Object.keys(resources).forEach((type) =>
-                            resources[type as keyof Resources]?.forEach((path: string) => resourcePaths.push(path)));
+                        Object.keys(resources).forEach(
+                            (type) =>
+                                resources[type as keyof Resources]?.forEach((path: string) =>
+                                    resourcePaths.push(path),
+                                ),
+                        );
 
                         //copy resources
                         copyFiles(args.input, tmpOutputFolder, resourcePaths);
@@ -247,7 +256,11 @@ async function handler(args: Arguments<any>) {
             }
 
             // Copy all generated files to user' output folder
-            shell.cp('-r', [join(tmpOutputFolder, '*'), join(tmpOutputFolder, '.*')], userOutputFolder);
+            shell.cp(
+                '-r',
+                [join(tmpOutputFolder, '*'), join(tmpOutputFolder, '.*')],
+                userOutputFolder,
+            );
 
             if (publish) {
                 const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
@@ -292,13 +305,14 @@ function preparingTemporaryFolders(userOutputFolder: string) {
     shell.mkdir(args.input, args.output);
     shell.chmod('-R', 'u+w', args.input);
 
-    copyFiles(args.rootInput, args.input, glob.sync('**', {
-        cwd: args.rootInput,
-        nodir: true,
-        follow: true,
-        ignore: [
-            'node_modules/**',
-            '*/node_modules/**',
-        ],
-    }));
+    copyFiles(
+        args.rootInput,
+        args.input,
+        glob.sync('**', {
+            cwd: args.rootInput,
+            nodir: true,
+            follow: true,
+            ignore: ['node_modules/**', '*/node_modules/**'],
+        }),
+    );
 }

package/src/cmd/publish/index.ts

@@ -2,7 +2,7 @@ import {join} from 'path';
 import {ArgvService} from '../../services';
 import {logger} from '../../utils';
 
-import {Argv, Arguments} from 'yargs';
+import {Arguments, Argv} from 'yargs';
 
 import {upload} from './upload';
 
@@ -73,12 +73,7 @@ async function handler(args: Arguments<any>) {
         ...args,
     });
 
-    const {
-        input,
-        endpoint,
-        bucket,
-        prefix,
-    } = ArgvService.getConfig();
+    const {input, endpoint, bucket, prefix} = ArgvService.getConfig();
 
     logger.info('', `Upload artifacts from ${input} to ${join(endpoint, bucket, prefix)}`);
 

package/src/cmd/publish/upload.ts

@@ -1,7 +1,7 @@
 import {createReadStream} from 'fs';
 import walkSync from 'walk-sync';
-import {resolve, join} from 'path';
-import {S3Client, PutObjectCommand} from '@aws-sdk/client-s3';
+import {join, resolve} from 'path';
+import {PutObjectCommand, S3Client} from '@aws-sdk/client-s3';
 import mime from 'mime-types';
 
 import {convertBackSlashToSlash, logger} from '../../utils';
@@ -42,20 +42,26 @@ export async function upload(props: UploadProps): Promise<void> {
         ignore,
     });
 
-    await mapLimit(filesToPublish, 100, asyncify(async (pathToFile: string) => {
-        const mimeType = mime.lookup(pathToFile);
-
-        logger.upload(pathToFile);
-
-        try {
-            await s3Client.send(new PutObjectCommand({
-                ContentType: mimeType ? mimeType : undefined,
-                Bucket: bucket,
-                Key: convertBackSlashToSlash(join(prefix, pathToFile)),
-                Body: createReadStream(resolve(input, pathToFile)),
-            }));
-        } catch (error) {
-            logger.error(pathToFile, error.message);
-        }
-    }));
+    await mapLimit(
+        filesToPublish,
+        100,
+        asyncify(async (pathToFile: string) => {
+            const mimeType = mime.lookup(pathToFile);
+
+            logger.upload(pathToFile);
+
+            try {
+                await s3Client.send(
+                    new PutObjectCommand({
+                        ContentType: mimeType ? mimeType : undefined,
+                        Bucket: bucket,
+                        Key: convertBackSlashToSlash(join(prefix, pathToFile)),
+                        Body: createReadStream(resolve(input, pathToFile)),
+                    }),
+                );
+            } catch (error) {
+                logger.error(pathToFile, error.message);
+            }
+        }),
+    );
 }

package/src/cmd/translate/index.ts

@@ -1,39 +1,26 @@
-import {
-    eachLimit,
-    retry,
-    asyncify,
-} from 'async';
+import {asyncify, eachLimit, retry} from 'async';
 
-import {dirname, resolve, join} from 'path';
-import {readFile, writeFile, mkdir} from 'fs/promises';
+import {dirname, join, resolve} from 'path';
+import {mkdir, readFile, writeFile} from 'fs/promises';
 import {XMLParser} from 'fast-xml-parser';
 
 import {Session} from '@yandex-cloud/nodejs-sdk/dist/session';
 import {TranslationServiceClient} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/service_clients';
 import {
-    TranslateRequest,
     TranslateRequest_Format as Format,
+    TranslateRequest,
 } from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/ai/translate/v2/translation_service';
 
-const yfm2xliff = require('@doc-tools/yfm2xliff/lib/cjs');
+import markdownTranslation from '@diplodoc/markdown-translation';
 
 import {ArgvService} from '../../services';
 import {getYandexOAuthToken} from '../../packages/credentials';
 import {glob, logger} from '../../utils';
 
-import {Argv, Arguments} from 'yargs';
+import {Arguments, Argv} from 'yargs';
 
 import {YandexCloudTranslateGlossaryPair} from '../../models';
 
-const composer = async (xliff: string, skeleton: string): Promise<string> => new Promise((res, rej) =>
-    yfm2xliff.compose(xliff, skeleton, (err: Error, composed: string) => {
-        if (err) {
-            rej(err);
-        }
-
-        return res(composed);
-    }));
-
 const command = 'translate';
 
 const description = 'translate documentation with Yandex.Cloud Translator API';
@@ -64,7 +51,8 @@ function builder<T>(argv: Argv<T>) {
         })
         .demandOption(
             ['source-language', 'target-language'],
-            'command requires to specify source and target languages');
+            'command requires to specify source and target languages',
+        );
 }
 
 class TranslatorError extends Error {
@@ -83,7 +71,8 @@ async function handler(args: Arguments<any>) {
         ...args,
     });
 
-    const {input,
+    const {
+        input,
         output,
         yandexCloudTranslateFolderId,
         yandexCloudTranslateGlossaryPairs,
@@ -91,12 +80,17 @@ async function handler(args: Arguments<any>) {
         tl: targetLanguage,
     } = args;
 
-    logger.info(input, `translating documentation from ${sourceLanguage} to ${targetLanguage} language`);
+    logger.info(
+        input,
+        `translating documentation from ${sourceLanguage} to ${targetLanguage} language`,
+    );
 
     try {
         let found = [];
 
-        ({state: {found}} = await glob(join(input, MD_GLOB), {
+        ({
+            state: {found},
+        } = await glob(join(input, MD_GLOB), {
             nosort: true,
         }));
 
@@ -125,7 +119,10 @@ async function handler(args: Arguments<any>) {
         }
     }
 
-    logger.info(output, `translated documentation from ${sourceLanguage} to ${targetLanguage} language`);
+    logger.info(
+        output,
+        `translated documentation from ${sourceLanguage} to ${targetLanguage} language`,
+    );
 }
 
 export type TranslatorParams = {
@@ -158,16 +155,21 @@ function translator(params: TranslatorParams) {
 
             const md = await readFile(resolve(mdPath), {encoding: 'utf-8'});
 
-            const extracted = await yfm2xliff.extract({
-                md,
-                mdPath,
-                source: sourceLanguage,
-                target: targetLanguage,
-                sklPath: '',
-                xlfPath: '',
+            const {xlf, skeleton} = markdownTranslation.extract({
+                source: {
+                    language: sourceLanguage,
+                    locale: sourceLanguage.toUpperCase(),
+                },
+                target: {
+                    language: targetLanguage,
+                    locale: targetLanguage.toUpperCase(),
+                },
+                markdown: md,
+                markdownPath: mdPath,
+                skeletonPath: '',
             });
 
-            const texts = parseSourcesFromXLIFF(extracted.xliff);
+            const texts = parseSourcesFromXLIFF(xlf);
 
             const machineTranslateParams = TranslateRequest.fromPartial({
                 texts,
@@ -182,14 +184,23 @@ function translator(params: TranslatorParams) {
                 format: Format.PLAIN_TEXT,
             });
 
-            const translations = await retry({times: RETRY_LIMIT, interval: (count: number) => {
-                // eslint-disable-next-line no-bitwise
-                return (1 << count) * 1000;
-            }}, asyncify(async () =>
-                await client.translate(machineTranslateParams)
-                    .then((results: {translations: {text: string}[]}) =>
-                        results.translations.map(({text}: {text: string}) => text)),
-            ));
+            const translations = await retry(
+                {
+                    times: RETRY_LIMIT,
+                    interval: (count: number) => {
+                        // eslint-disable-next-line no-bitwise
+                        return (1 << count) * 1000;
+                    },
+                },
+                asyncify(
+                    async () =>
+                        await client
+                            .translate(machineTranslateParams)
+                            .then((results: {translations: {text: string}[]}) =>
+                                results.translations.map(({text}: {text: string}) => text),
+                            ),
+                ),
+            );
 
             const createXLIFFDocumentParams = {
                 sourceLanguage: sourceLanguage + '-' + MTRANS_LOCALE,
@@ -200,7 +211,10 @@ function translator(params: TranslatorParams) {
 
             const translatedXLIFF = createXLIFFDocument(createXLIFFDocumentParams);
 
-            const composed = await composer(translatedXLIFF, extracted.skeleton);
+            const composed = await markdownTranslation.compose({
+                xlf: translatedXLIFF,
+                skeleton,
+            });
 
             const outputPath = mdPath.replace(input, output);
 

package/src/cmd/xliff/compose.ts

@@ -1,5 +1,7 @@
-const {promises: {readFile, writeFile, mkdir}} = require('fs');
-import {join, extname, dirname} from 'path';
+const {
+    promises: {readFile, writeFile, mkdir},
+} = require('fs');
+import {dirname, extname, join} from 'path';
 
 import markdownTranslation, {ComposeParameters} from '@diplodoc/markdown-translation';
 import {Arguments, Argv} from 'yargs';
@@ -28,7 +30,8 @@ class ComposeError extends Error {
     }
 }
 
-const USAGE = 'yfm xliff compose \
+const USAGE =
+    'yfm xliff compose \
 --input <folder-with-xliff-and-skeleton> \
 --ouput <folder-to-store-translated-markdown>';
 
@@ -38,11 +41,13 @@ function builder<T>(argv: Argv<T>) {
             alias: 'i',
             describe: 'input folder with xliff and skeleton files',
             type: 'string',
-        }).option('output', {
+        })
+        .option('output', {
             alias: 'o',
             describe: 'output folder where translated markdown will be stored',
             type: 'string',
-        }).demandOption(['input', 'output'], USAGE);
+        })
+        .demandOption(['input', 'output'], USAGE);
 }
 
 /* eslint-disable-next-line @typescript-eslint/no-explicit-any */
@@ -58,18 +63,22 @@ async function handler(args: Arguments<any>) {
     let xliffPaths: string[] = [];
 
     try {
-        ({state: {found: skeletonPaths, cache}} = await glob(join(input, SKL_MD_GLOB), {
+        ({
+            state: {found: skeletonPaths, cache},
+        } = await glob(join(input, SKL_MD_GLOB), {
             nosort: false,
             cache,
         }));
 
-        ({state: {found: xliffPaths, cache}} = await glob(join(input, XLF_GLOB), {
+        ({
+            state: {found: xliffPaths, cache},
+        } = await glob(join(input, XLF_GLOB), {
             nosort: false,
             cache,
         }));
 
         if (xliffPaths.length !== skeletonPaths.length) {
-            throw new ComposeError('number of xliff and skeleton files does\'not match', input);
+            throw new ComposeError("number of xliff and skeleton files does'not match", input);
         }
     } catch (err) {
         if (err instanceof Error || err instanceof ComposeError) {

package/src/cmd/xliff/extract.ts

@@ -1,9 +1,11 @@
-const {promises: {readFile, writeFile, mkdir}} = require('fs');
-import {join, dirname, extname} from 'path';
+const {
+    promises: {readFile, writeFile, mkdir},
+} = require('fs');
+import {dirname, extname, join} from 'path';
 
 import markdownTranslation, {ExtractParameters} from '@diplodoc/markdown-translation';
 import {Arguments, Argv} from 'yargs';
-import {eachLimit, asyncify} from 'async';
+import {asyncify, eachLimit} from 'async';
 
 import {ArgvService} from '../../services';
 import {glob, logger} from '../../utils';
@@ -44,19 +46,26 @@ function builder<T>(argv: Argv<T>) {
             alias: 'sll',
             describe: 'source language and locale',
             type: 'string',
-        }).option('target-language-locale', {
+        })
+        .option('target-language-locale', {
             alias: 'tll',
             describe: 'target language and locale',
             type: 'string',
-        }).option('input', {
+        })
+        .option('input', {
             alias: 'i',
             describe: 'input folder with markdown files',
             type: 'string',
-        }).option('output', {
+        })
+        .option('output', {
             alias: 'o',
             describe: 'output folder to store xliff and skeleton files',
             type: 'string',
-        }).demandOption(['source-language-locale', 'target-language-locale', 'input', 'output'], USAGE);
+        })
+        .demandOption(
+            ['source-language-locale', 'target-language-locale', 'input', 'output'],
+            USAGE,
+        );
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -83,7 +92,9 @@ async function handler(args: Arguments<any>) {
     let found: string[] = [];
 
     try {
-        ({state: {found, cache}} = await glob(join(input, MD_GLOB), {
+        ({
+            state: {found, cache},
+        } = await glob(join(input, MD_GLOB), {
             nosort: true,
             cache,
         }));
@@ -134,9 +145,7 @@ function pipeline(params: PipelineParameters) {
         const markdown = await reader({path: markdownPath});
         const extension = extname(markdownPath);
 
-        const outputRelativePath = markdownPath
-            .replace(extension, '')
-            .slice(input.length);
+        const outputRelativePath = markdownPath.replace(extension, '').slice(input.length);
 
         const outputPath = join(output, outputRelativePath);
         const xlfPath = outputPath + '.xliff';

package/src/cmd/xliff/index.ts

@@ -19,7 +19,10 @@ function builder<T>(argv: Argv<T>) {
     return argv
         .command(extract)
         .command(compose)
-        .demandCommand(1, `provide one of the folowing ${command} commands: ${extract.command}, ${compose.command}`);
+        .demandCommand(
+            1,
+            `provide one of the folowing ${command} commands: ${extract.command}, ${compose.command}`,
+        );
 }
 
 export {xliff};

package/src/constants.ts

@@ -59,7 +59,7 @@ export enum Platforms {
 export enum IncludeMode {
     ROOT_MERGE = 'root_merge',
     MERGE = 'merge',
-    LINK = 'link'
+    LINK = 'link',
 }
 
 export enum ResourceType {
@@ -95,8 +95,8 @@ export const PROCESSING_FINISHED = 'Processing finished:';
 export const LINTING_FINISHED = 'Linting finished:';
 export const GETTING_ALL_CONTRIBUTORS = 'Getting all contributors.';
 export const ALL_CONTRIBUTORS_RECEIVED = 'All contributors received.';
-export const getMsgСonfigurationMustBeProvided =
-    (repo: string) => `Сonfiguration must be provided for ${repo} like env variables or in .yfm file`;
+export const getMsgСonfigurationMustBeProvided = (repo: string) =>
+    `Сonfiguration must be provided for ${repo} like env variables or in .yfm file`;
 
 export const FIRST_COMMIT_FROM_ROBOT_IN_GITHUB = '2dce14271359cd20d7e874956d604de087560cf4';
 
@@ -117,6 +117,6 @@ export const REGEXP_INCLUDE_FILE_PATH = /(?<=[(]).+(?=[)])/g;
 export const REGEXP_AUTHOR = /(?<=author:\s).+(?=\r?\n)/g;
 
 export const MIN_CHUNK_SIZE = Number(process.env.MIN_CHUNK_SIZE) || 1000;
-export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || (os.cpus().length - 1);
+export const WORKERS_COUNT = Number(process.env.WORKERS_COUNT) || os.cpus().length - 1;
 
 export const metadataBorder = '---';

package/src/index.ts

@@ -5,7 +5,7 @@ import 'threads/register';
 
 import {MAIN_TIMER_ID} from './constants';
 
-import {build, xliff, translate, publish} from './cmd';
+import {build, publish, translate, xliff} from './cmd';
 
 console.time(MAIN_TIMER_ID);
 
@@ -28,7 +28,7 @@ yargs
     .option('quiet', {
         alias: 'q',
         default: false,
-        describe: 'Run in quiet mode. Don\'t write logs to stdout',
+        describe: "Run in quiet mode. Don't write logs to stdout",
         type: 'boolean',
     })
     .group(['config', 'strict', 'quiet', 'help', 'version'], 'Common options:')
@@ -44,7 +44,7 @@ yargs
 
         const {warn, error} = log.get();
 
-        if (strict && warn.length || error.length) {
+        if ((strict && warn.length) || error.length) {
             process.exit(1);
         }
 

package/src/models.ts

@@ -2,10 +2,10 @@ import {Logger} from '@diplodoc/transform/lib/log';
 import {LintConfig} from '@diplodoc/transform/lib/yfmlint';
 
 import {FileContributors, VCSConnector, VCSConnectorConfig} from './vcs-connector/connector-models';
-import {Lang, Stage, IncludeMode, ResourceType} from './constants';
+import {IncludeMode, Lang, ResourceType, Stage} from './constants';
 import {ChangelogItem} from '@diplodoc/transform/lib/plugins/changelog/types';
 
-export type VarsPreset = 'internal'|'external';
+export type VarsPreset = 'internal' | 'external';
 
 export type YfmPreset = Record<string, string>;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -13,7 +13,10 @@ export type Metadata = Record<string, any>;
 
 export type ExternalAuthorByPathFunction = (path: string) => Contributor | null;
 export type ContributorsByPathFunction = (path: string) => Promise<FileContributors>;
-export type NestedContributorsForPathFunction = (path: string, nestedContributors: Contributors) => void;
+export type NestedContributorsForPathFunction = (
+    path: string,
+    nestedContributors: Contributors,
+) => void;
 export type UserByLoginFunction = (login: string) => Promise<Contributor | null>;
 export type CollectionOfPluginsFunction = (output: string, options: PluginOptions) => string;
 
@@ -96,7 +99,7 @@ export type YfmTocIncluder = {
 
 export const includersNames = ['sourcedocs', 'openapi', 'generic', 'unarchive'] as const;
 
-export type YfmTocIncluderName = typeof includersNames[number];
+export type YfmTocIncluderName = (typeof includersNames)[number];
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type Includer<FnParams = any> = {
@@ -104,7 +107,9 @@ export type Includer<FnParams = any> = {
     includerFunction: IncluderFunction<FnParams>;
 };
 
-export type IncluderFunction<PassedParams> = (args: IncluderFunctionParams<PassedParams>) => Promise<void>;
+export type IncluderFunction<PassedParams> = (
+    args: IncluderFunctionParams<PassedParams>,
+) => Promise<void>;
 
 export type IncluderFunctionParams<PassedParams> = {
     // item that contains include that uses includer
@@ -148,7 +153,7 @@ export interface LeadingPageLinks extends Filter {
 }
 
 export interface Filter {
-    when?: boolean|string;
+    when?: boolean | string;
     [key: string]: unknown;
 }
 
@@ -239,8 +244,8 @@ export type Resources = {
 };
 
 export type YandexCloudTranslateGlossaryPair = {
-  sourceText: string;
-  translatedText: string;
+    sourceText: string;
+    translatedText: string;
 };
 
 export type CommitInfo = {