@diplodoc/cli 4.6.3 → 4.6.5

package/package.json

@@ -3,7 +3,7 @@
   "author": "Yandex Data UI Team <data-ui@yandex-team.ru>",
   "description": "Make documentation using yfm-docs in Markdown and HTML formats",
   "license": "MIT",
-  "version": "4.6.3",
+  "version": "4.6.5",
   "repository": {
     "type": "git",
     "url": "git@github.com:diplodoc-platform/cli.git"
@@ -29,23 +29,17 @@
     "npm:": ">=9.*"
   },
   "dependencies": {
-    "@apidevtools/swagger-parser": "^10.1.0",
     "@aws-sdk/client-s3": "^3.369.0",
-    "@diplodoc/client": "^1.0.2",
-    "@diplodoc/markdown-translation": "^1.0.1",
-    "@diplodoc/mermaid-extension": "^1.1.3",
-    "@diplodoc/openapi-extension": "^1.4.4",
-    "@diplodoc/transform": "^4.3.0",
+    "@diplodoc/client": "^1.2.0",
+    "@diplodoc/markdown-translation": "^1.0.2",
+    "@diplodoc/mermaid-extension": "^1.2.1",
+    "@diplodoc/openapi-extension": "^1.4.10",
+    "@diplodoc/transform": "^4.7.3",
     "@octokit/core": "4.2.4",
-    "@types/glob": "^8.0.0",
-    "@types/json-stringify-safe": "^5.0.0",
     "@yandex-cloud/nodejs-sdk": "^2.2.2",
     "ajv": "^8.11.0",
     "async": "^3.2.4",
-    "aws-sdk": "^2.1425.0",
-    "bem-cn-lite": "^4.1.0",
     "chalk": "4.1.2",
-    "dotenv": "^16.0.3",
     "fast-xml-parser": "^4.0.11",
     "glob": "^8.0.3",
     "highlight.js": "^11.7.0",
@@ -66,13 +60,15 @@
     "yargs": "17.7.2"
   },
   "devDependencies": {
-    "@diplodoc/eslint-config": "^1.0.14",
-    "@diplodoc/prettier-config": "^1.0.0",
+    "@diplodoc/eslint-config": "^2.0.0",
+    "@diplodoc/prettier-config": "^2.0.0",
     "@diplodoc/tsconfig": "^1.0.2",
     "@types/async": "^3.2.15",
     "@types/chalk": "2.2.0",
+    "@types/glob": "^8.1.0",
     "@types/html-escaper": "^3.0.0",
     "@types/js-yaml": "4.0.5",
+    "@types/json-stringify-safe": "^5.0.3",
     "@types/lodash": "4.14.195",
     "@types/mime-types": "2.1.1",
     "@types/node": "14.*",
@@ -80,21 +76,11 @@
     "@types/tar-stream": "^2.2.2",
     "@types/yargs": "17.0.24",
     "esbuild": "^0.17.17",
-    "eslint": "^8.0.0",
     "husky": "8.0.3",
     "lint-staged": "13.2.2",
     "lodash": "4.17.21",
-    "prettier": "^3.0.3",
-    "ts-jest": "29.1.0",
-    "ts-node": "^10.9.1",
     "typescript": "4.9.5"
   },
-  "overrides": {
-    "react-sortable-hoc": {
-      "react": "^18.2.0",
-      "react-dom": "^18.2.0"
-    }
-  },
   "husky": {
     "hooks": {
       "pre-commit": "lint-staged"
@@ -129,5 +115,8 @@
     "tool",
     "tools",
     "generator"
-  ]
+  ],
+  "overrides": {
+    "markdown-it": "^13.0.2"
+  }
 }

package/src/cmd/translate/index.ts

@@ -34,7 +34,8 @@ const translate = {
 
 const MD_GLOB = '**/*.md';
 const REQUESTS_LIMIT = 20;
-const RETRY_LIMIT = 8;
+const BYTES_LIMIT = 10000;
+const RETRY_LIMIT = 3;
 const MTRANS_LOCALE = 'MTRANS';
 
 function builder<T>(argv: Argv<T>) {
@@ -148,6 +149,23 @@ function translator(params: TranslatorParams) {
 
     const session = new Session({oauthToken});
     const client = session.client(TranslationServiceClient);
+    const request = (texts: string[]) => () =>
+        client
+            .translate(
+                TranslateRequest.fromPartial({
+                    texts,
+                    folderId,
+                    sourceLanguageCode: sourceLanguage,
+                    targetLanguageCode: targetLanguage,
+                    glossaryConfig: {
+                        glossaryData: {
+                            glossaryPairs: yandexCloudTranslateGlossaryPairs,
+                        },
+                    },
+                    format: Format.PLAIN_TEXT,
+                }),
+            )
+            .then((results) => results.translations.map(({text}) => text));
 
     return async (mdPath: string) => {
         try {
@@ -171,45 +189,57 @@ function translator(params: TranslatorParams) {
 
             const texts = parseSourcesFromXLIFF(xlf);
 
-            const machineTranslateParams = TranslateRequest.fromPartial({
-                texts,
-                folderId,
-                sourceLanguageCode: sourceLanguage,
-                targetLanguageCode: targetLanguage,
-                glossaryConfig: {
-                    glossaryData: {
-                        glossaryPairs: yandexCloudTranslateGlossaryPairs,
+            const parts = await Promise.all(
+                texts.reduce(
+                    (
+                        {
+                            promises,
+                            buffer,
+                            bufferSize,
+                        }: {
+                            promises: Promise<string[]>[];
+                            buffer: string[];
+                            bufferSize: number;
+                        },
+                        text,
+                        index,
+                    ) => {
+                        if (text.length >= BYTES_LIMIT) {
+                            logger.warn(
+                                mdPath,
+                                'Skip document part for translation. Part is too big.',
+                            );
+                            promises.push(Promise.resolve([text]));
+                            return {promises, buffer, bufferSize};
+                        }
+
+                        if (bufferSize + text.length > BYTES_LIMIT || index === texts.length - 1) {
+                            promises.push(backoff(request(buffer)));
+                            buffer = [];
+                            bufferSize = 0;
+                        }
+
+                        buffer.push(text);
+                        bufferSize += text.length;
+
+                        return {promises, buffer, bufferSize};
                     },
-                },
-                format: Format.PLAIN_TEXT,
-            });
-
-            const translations = await retry(
-                {
-                    times: RETRY_LIMIT,
-                    interval: (count: number) => {
-                        // eslint-disable-next-line no-bitwise
-                        return (1 << count) * 1000;
+                    {
+                        promises: [],
+                        buffer: [],
+                        bufferSize: 0,
                     },
-                },
-                asyncify(
-                    async () =>
-                        await client
-                            .translate(machineTranslateParams)
-                            .then((results: {translations: {text: string}[]}) =>
-                                results.translations.map(({text}: {text: string}) => text),
-                            ),
-                ),
+                ).promises,
             );
 
-            const createXLIFFDocumentParams = {
+            const translations = ([] as string[]).concat(...parts);
+
+            const translatedXLIFF = createXLIFFDocument({
                 sourceLanguage: sourceLanguage + '-' + MTRANS_LOCALE,
                 targetLanguage: targetLanguage + '-' + MTRANS_LOCALE,
                 sources: texts,
-                targets: translations as string[],
-            };
-
-            const translatedXLIFF = createXLIFFDocument(createXLIFFDocumentParams);
+                targets: translations,
+            });
 
             const composed = await markdownTranslation.compose({
                 xlf: translatedXLIFF,
@@ -230,7 +260,20 @@ function translator(params: TranslatorParams) {
     };
 }
 
-function parseSourcesFromXLIFF(xliff: string) {
+function backoff(action: () => Promise<string[]>): Promise<string[]> {
+    return retry(
+        {
+            times: RETRY_LIMIT,
+            interval: (count: number) => {
+                // eslint-disable-next-line no-bitwise
+                return (1 << count) * 1000;
+            },
+        },
+        asyncify(action),
+    );
+}
+
+function parseSourcesFromXLIFF(xliff: string): string[] {
     const parser = new XMLParser();
 
     const inputs = parser.parse(xliff)?.xliff?.file?.body['trans-unit'] ?? [];

package/src/steps/publishFilesToS3.ts

@@ -1,51 +0,0 @@
-import {readFileSync} from 'fs';
-import walkSync from 'walk-sync';
-import {join, resolve} from 'path';
-import S3 from 'aws-sdk/clients/s3';
-import mime from 'mime-types';
-
-import {ArgvService} from '../services';
-import {convertBackSlashToSlash, logger} from '../utils';
-
-const DEFAULT_PREFIX = process.env.YFM_STORAGE_PREFIX ?? '';
-
-export async function publishFilesToS3(): Promise<void> {
-    const {
-        output: outputFolderPath,
-        ignore = [],
-        storageEndpoint: endpoint,
-        storageBucket: bucket,
-        storagePrefix: prefix = DEFAULT_PREFIX,
-        storageKeyId: accessKeyId,
-        storageSecretKey: secretAccessKey,
-    } = ArgvService.getConfig();
-
-    const s3Client = new S3({
-        endpoint,
-        accessKeyId,
-        secretAccessKey,
-    });
-
-    const filesToPublish: string[] = walkSync(resolve(outputFolderPath), {
-        directories: false,
-        includeBasePath: false,
-        ignore,
-    });
-
-    await Promise.all(
-        filesToPublish.map(async (pathToFile) => {
-            const mimeType = mime.lookup(pathToFile);
-
-            const params: S3.Types.PutObjectRequest = {
-                ContentType: mimeType ? mimeType : undefined,
-                Bucket: bucket,
-                Key: convertBackSlashToSlash(join(prefix, pathToFile)),
-                Body: readFileSync(resolve(outputFolderPath, pathToFile)),
-            };
-
-            logger.upload(pathToFile);
-
-            return s3Client.upload(params).promise();
-        }),
-    );
-}