@diplodoc/cli 4.13.8 → 4.13.9-alpha-1

package/package.json

@@ -3,7 +3,7 @@
   "author": "Yandex Data UI Team <data-ui@yandex-team.ru>",
   "description": "Make documentation using yfm-docs in Markdown and HTML formats",
   "license": "MIT",
-  "version": "4.13.8",
+  "version": "4.13.9-alpha-1",
   "repository": {
     "type": "git",
     "url": "git@github.com:diplodoc-platform/cli.git"
@@ -31,7 +31,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.369.0",
-    "@diplodoc/client": "^2.0.5",
+    "@diplodoc/client": "^2.1.1",
     "@diplodoc/latex-extension": "^1.1.0",
     "@diplodoc/mermaid-extension": "^1.2.1",
     "@diplodoc/openapi-extension": "^1.4.13",

package/src/cmd/translate/handler.ts

@@ -1,4 +1,5 @@
-import axios from 'axios';
+import axios, {AxiosError, AxiosResponse} from 'axios';
+import {green, red} from 'chalk';
 import {Arguments} from 'yargs';
 import {ArgvService} from '../../services';
 import {logger} from '../../utils';
@@ -6,10 +7,14 @@ import {ok} from 'assert';
 import {dirname, extname, join, resolve} from 'path';
 import {mkdir} from 'fs/promises';
 import {getYandexAuth} from './yandex/auth';
-import {asyncify, eachLimit, retry} from 'async';
+import {asyncify, eachLimit} from 'async';
 
 import {
+    AuthError,
     Defer,
+    LimitExceed,
+    RequestError,
+    TranslateError,
     TranslateParams,
     bytes,
     compose,
@@ -23,24 +28,39 @@ const REQUESTS_LIMIT = 20;
 const BYTES_LIMIT = 10000;
 const RETRY_LIMIT = 3;
 
-class TranslatorError extends Error {
-    path: string;
+type TranslatorParams = {
+    input: string;
+    output: string;
+    sourceLanguage: string;
+    targetLanguage: string;
+    // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
+};
 
-    constructor(message: string, path: string) {
-        super(message);
+type RequesterParams = {
+    auth: string;
+    folderId: string | undefined;
+    sourceLanguage: string;
+    targetLanguage: string;
+    dryRun: boolean;
+};
 
-        this.path = path;
-    }
-}
+type Request = {
+    (texts: string[]): () => Promise<void>;
+    stat: {
+        bytes: number;
+        chunks: number;
+    };
+};
 
-class RequestError extends Error {
-    code: string;
+type Split = (path: string, texts: string[]) => Promise<string[]>;
 
-    constructor(error: Error) {
-        super(error?.message || String(error));
-        this.code = 'REQUEST_ERROR';
-    }
-}
+type Cache = Map<string, Defer>;
+
+type Translations = {
+    translations: {
+        text: string;
+    }[];
+};
 
 export async function handler(args: Arguments<any>) {
     const params = normalizeParams({
@@ -85,106 +105,130 @@ export async function handler(args: Arguments<any>) {
                     try {
                         await translate(file);
                     } catch (error: any) {
-                        logger.error(file, error.message);
+                        if (error instanceof TranslateError) {
+                            logger.error(file, `${error.message}`, error.code);
+
+                            if (error.fatal) {
+                                process.exit(1);
+                            }
+                        } else {
+                            logger.error(file, error.message);
+                        }
                     }
                 }),
             );
 
-            console.log('PROCESSED', `bytes: ${request.stat.bytes} chunks: ${request.stat.chunks}`);
+            console.log(
+                green('PROCESSED'),
+                `bytes: ${request.stat.bytes} chunks: ${request.stat.chunks}`,
+            );
         }
     } catch (error: any) {
-        const message = error.message;
-
-        const file = error instanceof TranslatorError ? error.path : '';
+        if (error instanceof TranslateError) {
+            console.error(red(error.code), error.message);
+        } else {
+            console.error(error);
+        }
 
-        logger.error(file, message);
+        process.exit(1);
     }
 }
 
-type TranslatorParams = {
-    input: string;
-    output: string;
-    sourceLanguage: string;
-    targetLanguage: string;
-    // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
-};
+function scheduler(limit: number, interval: number) {
+    const scheduled: Defer<void>[] = [];
 
-type RequesterParams = {
-    auth: string;
-    folderId: string | undefined;
-    sourceLanguage: string;
-    targetLanguage: string;
-    dryRun: boolean;
-};
+    let processing = 0;
 
-type Request = {
-    (texts: string[]): () => Promise<string[]>;
-    stat: {
-        bytes: number;
-        chunks: number;
-    };
-};
+    function idle() {
+        const defer = new Defer<void>();
 
-type Split = (path: string, texts: string[]) => Promise<string>[];
+        scheduled.push(defer);
 
-type Cache = Map<string, Defer>;
+        return defer.promise;
+    }
 
-type Translations = {
-    translations: {
-        text: string;
-    }[];
-};
+    async function queue() {
+        processing++;
+        await wait(interval);
+        processing--;
+        unqueue();
+    }
 
-function requester(params: RequesterParams, cache: Cache) {
-    const {auth, folderId, sourceLanguage, targetLanguage, dryRun} = params;
-    const resolve = (text: string, index: number, texts: string[]) => {
-        const defer = cache.get(texts[index]);
-        if (defer) {
-            defer.resolve(text);
+    async function unqueue() {
+        scheduled.shift()?.resolve();
+    }
+
+    return async function <R>(action: Function): Promise<R> {
+        if (processing >= limit) {
+            await idle();
         }
-        return text;
+
+        queue();
+
+        return action();
     };
+}
+
+function requester(params: RequesterParams, cache: Cache) {
+    const {auth, folderId, sourceLanguage, targetLanguage, dryRun} = params;
+    const schedule = scheduler(REQUESTS_LIMIT, 1000);
 
     const request = function request(texts: string[]) {
+        const resolve = (text: string, index: number) => {
+            const defer = cache.get(texts[index]);
+            if (defer) {
+                defer.resolve(text);
+            }
+        };
+
         request.stat.bytes += bytes(texts);
         request.stat.chunks++;
 
         return async function () {
             if (dryRun) {
-                return texts.map(resolve);
+                texts.forEach(resolve);
             }
 
-            return axios({
-                method: 'POST',
-                url: 'https://translate.api.cloud.yandex.net/translate/v2/translate',
-                headers: {
-                    'Content-Type': 'application/json',
-                    Authorization: auth,
-                },
-                data: {
-                    folderId,
-                    texts,
-                    sourceLanguageCode: sourceLanguage,
-                    targetLanguageCode: targetLanguage,
-                    format: 'HTML',
-                },
-            })
-                .then(({data, status}) => {
-                    if (status === 200) {
-                        return data;
-                    } else {
-                        throw new Error(data.message);
+            try {
+                const {data} = await schedule<AxiosResponse<Translations>>(() =>
+                    axios({
+                        method: 'POST',
+                        url: 'https://translate.api.cloud.yandex.net/translate/v2/translate',
+                        timeout: 5000,
+                        maxRedirects: 0,
+                        headers: {
+                            Authorization: auth,
+                            'Content-Type': 'application/json',
+                            'User-Agent': 'github.com/diplodoc-platform/cli',
+                        },
+                        data: {
+                            folderId,
+                            texts,
+                            sourceLanguageCode: sourceLanguage,
+                            targetLanguageCode: targetLanguage,
+                            format: 'HTML',
+                        },
+                    }),
+                );
+
+                return data.translations.map(({text}) => text).forEach(resolve);
+            } catch (error: any) {
+                if (error instanceof AxiosError) {
+                    const {response} = error;
+                    const {status, statusText, data} = response as AxiosResponse;
+
+                    switch (true) {
+                        case LimitExceed.is(data.message):
+                            throw new LimitExceed(data.message);
+                        case AuthError.is(data.message):
+                            throw new AuthError(data.message);
+                        default:
+                            throw new RequestError(status, statusText, data);
                     }
-                })
-                .then((result: Translations) => {
-                    return result.translations.map(({text}, index) => {
-                        return resolve(text, index, texts);
-                    });
-                })
-                .catch((error) => {
-                    console.error(error);
-                    throw new RequestError(error);
-                });
+                }
+
+                throw new RequestError(0, error.message, {fatal: true});
+            }
         };
     };
 
@@ -234,7 +278,7 @@ function translator(params: TranslatorParams, split: Split) {
             return;
         }
 
-        const parts = await Promise.all(split(path, units));
+        const parts = await split(path, units);
         const composed = compose(skeleton, parts, {useSource: true});
 
         await dumpFile(outputPath, composed);
@@ -242,13 +286,14 @@ function translator(params: TranslatorParams, split: Split) {
 }
 
 function splitter(request: Request, cache: Cache): Split {
-    return function (path: string, texts: string[]) {
+    return async function (path: string, texts: string[]) {
         const promises: Promise<string>[] = [];
+        const requests: Promise<void>[] = [];
         let buffer: string[] = [];
         let bufferSize = 0;
 
         const release = () => {
-            backoff(request(buffer));
+            requests.push(backoff(request(buffer)));
             buffer = [];
             bufferSize = 0;
         };
@@ -278,16 +323,30 @@ function splitter(request: Request, cache: Cache): Split {
             release();
         }
 
-        return promises;
+        await Promise.all(requests);
+
+        return Promise.all(promises);
     };
 }
 
-function backoff(action: () => Promise<string[]>): Promise<string[]> {
-    return retry(
-        {
-            times: RETRY_LIMIT,
-            interval: (count: number) => Math.pow(2, count) * 1000,
-        },
-        asyncify(action),
-    );
+function wait(interval: number) {
+    const defer = new Defer<void>();
+    setTimeout(() => defer.resolve(), interval);
+    return defer.promise;
+}
+
+async function backoff(action: () => Promise<void>): Promise<void> {
+    let retry = 0;
+
+    while (++retry < RETRY_LIMIT) {
+        try {
+            await action();
+        } catch (error: any) {
+            if (RequestError.canRetry(error)) {
+                await wait(Math.pow(2, retry) * 1000);
+            } else {
+                throw error;
+            }
+        }
+    }
 }

package/src/cmd/translate/utils/errors.ts

@@ -0,0 +1,97 @@
+export class TranslateError extends Error {
+    code: string;
+
+    fatal: boolean;
+
+    constructor(message: string, code: string, fatal = false) {
+        super(message);
+
+        this.code = code;
+        this.fatal = fatal;
+    }
+}
+
+export class RequestError extends TranslateError {
+    static canRetry(error: any) {
+        if (error instanceof RequestError) {
+            switch (true) {
+                case error.status === 429:
+                    return true;
+                case error.status === 500:
+                    return true;
+                case error.status === 503:
+                    return true;
+                case error.status === 504:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+
+        return false;
+    }
+
+    status: number;
+
+    constructor(
+        status: number,
+        statusText: string,
+        info: {code?: number; message?: string; fatal?: boolean} = {},
+    ) {
+        super(`${statusText}\n${info.message || ''}`, 'REQUEST_ERROR', info.fatal);
+
+        this.status = status;
+    }
+}
+
+const INACTIVE_CLOUD = /^The cloud .*? is inactive/;
+const WRONG_APIKEY = /^Unknown api key/;
+const WRONG_TOKEN = /^The token is invalid/;
+
+export class AuthError extends TranslateError {
+    static is(message: string) {
+        return Boolean(AuthError.reason(message));
+    }
+
+    static reason(message: string) {
+        switch (true) {
+            case INACTIVE_CLOUD.test(message):
+                return 'INACTIVE_CLOUD';
+            case WRONG_APIKEY.test(message):
+                return 'WRONG_APIKEY';
+            case WRONG_TOKEN.test(message):
+                return 'WRONG_TOKEN';
+            default:
+                return null;
+        }
+    }
+
+    constructor(message: string) {
+        super(message, AuthError.reason(message) || 'AUTH_ERROR', true);
+    }
+}
+
+const LIMIT_EXCEED_RX = /^limit on units was exceeded. (.*)$/;
+
+export class LimitExceed extends TranslateError {
+    static is(message: string) {
+        return Boolean(LIMIT_EXCEED_RX.test(message));
+    }
+
+    constructor(message: string) {
+        const [, desc] = LIMIT_EXCEED_RX.exec(message) || [];
+        super(desc, 'TRANSLATE_LIMIT_EXCEED', true);
+    }
+}
+
+export class ExtractError extends TranslateError {
+    constructor(error: Error) {
+        super('EXTRACT_ERROR', error?.message || String(error));
+    }
+}
+
+export class ComposeError extends TranslateError {
+    constructor(error: Error) {
+        super('COMPOSE_ERROR', error?.message || String(error));
+    }
+}

package/src/cmd/translate/utils/index.ts

@@ -5,6 +5,7 @@ import glob from 'glob';
 
 export {dumpFile, loadFile} from './fs';
 export {extract, compose} from './translate';
+export {TranslateError, LimitExceed, RequestError, AuthError} from './errors';
 
 type TranslateArgs = {
     input: string;
@@ -141,12 +142,12 @@ export function resolveSchemas(_path: string) {
     return null;
 }
 
-export class Defer {
-    resolve!: (text: string) => void;
+export class Defer<T = string> {
+    resolve!: (text: T) => void;
 
     reject!: (error: any) => void;
 
-    promise: Promise<string>;
+    promise: Promise<T>;
 
     constructor() {
         this.promise = new Promise((resolve, reject) => {

package/src/cmd/translate/utils/translate.ts

@@ -1,23 +1,6 @@
 import type {ComposeOptions, ExtractOptions} from '@diplodoc/translation';
 import {compose as _compose, extract as _extract} from '@diplodoc/translation';
-
-class ExtractError extends Error {
-    code: string;
-
-    constructor(error: Error) {
-        super(error?.message || String(error));
-        this.code = 'EXTRACT_ERROR';
-    }
-}
-
-class ComposeError extends Error {
-    code: string;
-
-    constructor(error: Error) {
-        super(error?.message || String(error));
-        this.code = 'COMPOSE_ERROR';
-    }
-}
+import {ComposeError, ExtractError} from './errors';
 
 type Content = Parameters<typeof _extract>[0];
 

package/src/cmd/translate/yandex/auth.ts

@@ -1,6 +1,8 @@
 import {readFileSync} from 'fs';
 
 const resolveKey = (data: string) => {
+    data = data.trim();
+
     switch (true) {
         case data.startsWith('y0_'):
             return 'Bearer ' + data;

package/src/utils/logger.ts

@@ -32,8 +32,8 @@ export const logger = {
 
         log.warn(`file: ${pathToFile} ${extraMessage}`);
     },
-    error: function (pathToFile: string, extraMessage: string) {
-        const message = `${red('ERROR')} file: ${pathToFile} error: ${extraMessage}`;
+    error: function (pathToFile: string, extraMessage: string, reason?: string) {
+        const message = `${red(reason || 'ERROR')} file: ${pathToFile} error: ${extraMessage}`;
 
         writeLog(message, true);
 

package/src/packages/credentials/index.ts

@@ -1 +0,0 @@
-export * from './yandex-oauth';

package/src/packages/credentials/yandex-oauth.ts

@@ -1,42 +0,0 @@
-import {readFile} from 'fs/promises';
-import {env} from 'process';
-import {homedir} from 'os';
-import {join} from 'path';
-
-import {logger} from '../../utils';
-
-const YANDEX_OAUTH_TOKEN_FILENAME = '.ya_oauth_token';
-
-async function getYandexOAuthToken() {
-    const {YANDEX_OAUTH_TOKEN} = env;
-
-    return YANDEX_OAUTH_TOKEN ?? getYandexOAuthTokenFromHomeDir();
-}
-
-async function getYandexOAuthTokenFromHomeDir() {
-    const error = 'failed reading yandex oauth token';
-
-    const path = join(homedir(), YANDEX_OAUTH_TOKEN_FILENAME);
-
-    let token;
-
-    try {
-        token = await readFile(path, {encoding: 'utf8'});
-
-        token = token.trim();
-
-        if (!token?.length) {
-            throw new Error(error);
-        }
-    } catch (err) {
-        logger.error(error);
-
-        throw err;
-    }
-
-    return token;
-}
-
-export {getYandexOAuthToken};
-
-export default {getYandexOAuthToken};