@diplodoc/cli 4.13.7 → 4.13.9-alpha-1

package/package.json

@@ -3,7 +3,7 @@
   "author": "Yandex Data UI Team <data-ui@yandex-team.ru>",
   "description": "Make documentation using yfm-docs in Markdown and HTML formats",
   "license": "MIT",
-  "version": "4.13.7",
+  "version": "4.13.9-alpha-1",
   "repository": {
     "type": "git",
     "url": "git@github.com:diplodoc-platform/cli.git"
@@ -31,16 +31,16 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.369.0",
-    "@diplodoc/client": "^2.0.5",
+    "@diplodoc/client": "^2.1.1",
     "@diplodoc/latex-extension": "^1.1.0",
     "@diplodoc/mermaid-extension": "^1.2.1",
     "@diplodoc/openapi-extension": "^1.4.13",
     "@diplodoc/transform": "^4.8.2",
-    "@diplodoc/translation": "^1.0.6",
+    "@diplodoc/translation": "^1.0.7",
     "@octokit/core": "4.2.4",
-    "@yandex-cloud/nodejs-sdk": "^2.2.2",
     "ajv": "^8.11.0",
     "async": "^3.2.4",
+    "axios": "^1.6.7",
     "chalk": "4.1.2",
     "glob": "^8.0.3",
     "highlight.js": "^11.7.0",

package/src/cmd/translate/compose.ts

@@ -28,7 +28,6 @@ function builder<T>(argv: Argv<T>) {
             alias: 'o',
             describe: 'output folder where translated markdown will be stored',
             type: 'string',
-            default: process.cwd(),
         })
         .option('use-source', {
             describe: 'for debug',

package/src/cmd/translate/extract.ts

@@ -29,7 +29,6 @@ function builder<T>(argv: Argv<T>) {
             alias: 'o',
             describe: 'output folder where translated markdown will be stored',
             type: 'string',
-            default: process.cwd(),
         })
         .option('source', {
             alias: ['sll', 'source-language-locale'],

package/src/cmd/translate/handler.ts

@@ -1,26 +1,25 @@
+import axios, {AxiosError, AxiosResponse} from 'axios';
+import {green, red} from 'chalk';
 import {Arguments} from 'yargs';
 import {ArgvService} from '../../services';
 import {logger} from '../../utils';
 import {ok} from 'assert';
-import {dirname, extname, resolve} from 'path';
+import {dirname, extname, join, resolve} from 'path';
 import {mkdir} from 'fs/promises';
-import {AuthInfo, getYandexAuth} from './yandex/auth';
-import {asyncify, eachLimit, retry} from 'async';
-import {Session} from '@yandex-cloud/nodejs-sdk/dist/session';
-import {TranslationServiceClient} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/service_clients';
-import {
-    TranslateRequest_Format as Format,
-    TranslateRequest,
-} from '@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/ai/translate/v2/translation_service';
+import {getYandexAuth} from './yandex/auth';
+import {asyncify, eachLimit} from 'async';
 
 import {
+    AuthError,
     Defer,
+    LimitExceed,
+    RequestError,
+    TranslateError,
     TranslateParams,
     bytes,
     compose,
     dumpFile,
     extract,
-    flat,
     loadFile,
     normalizeParams,
 } from './utils';
@@ -29,24 +28,39 @@ const REQUESTS_LIMIT = 20;
 const BYTES_LIMIT = 10000;
 const RETRY_LIMIT = 3;
 
-class TranslatorError extends Error {
-    path: string;
+type TranslatorParams = {
+    input: string;
+    output: string;
+    sourceLanguage: string;
+    targetLanguage: string;
+    // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
+};
 
-    constructor(message: string, path: string) {
-        super(message);
+type RequesterParams = {
+    auth: string;
+    folderId: string | undefined;
+    sourceLanguage: string;
+    targetLanguage: string;
+    dryRun: boolean;
+};
 
-        this.path = path;
-    }
-}
+type Request = {
+    (texts: string[]): () => Promise<void>;
+    stat: {
+        bytes: number;
+        chunks: number;
+    };
+};
 
-class RequestError extends Error {
-    code: string;
+type Split = (path: string, texts: string[]) => Promise<string[]>;
 
-    constructor(error: Error) {
-        super(error?.message || String(error));
-        this.code = 'REQUEST_ERROR';
-    }
-}
+type Cache = Map<string, Defer>;
+
+type Translations = {
+    translations: {
+        text: string;
+    }[];
+};
 
 export async function handler(args: Arguments<any>) {
     const params = normalizeParams({
@@ -91,95 +105,130 @@ export async function handler(args: Arguments<any>) {
                     try {
                         await translate(file);
                     } catch (error: any) {
-                        logger.error(file, error.message);
+                        if (error instanceof TranslateError) {
+                            logger.error(file, `${error.message}`, error.code);
+
+                            if (error.fatal) {
+                                process.exit(1);
+                            }
+                        } else {
+                            logger.error(file, error.message);
+                        }
                     }
                 }),
             );
 
-            console.log('PROCESSED', `bytes: ${request.stat.bytes} chunks: ${request.stat.chunks}`);
+            console.log(
+                green('PROCESSED'),
+                `bytes: ${request.stat.bytes} chunks: ${request.stat.chunks}`,
+            );
         }
     } catch (error: any) {
-        const message = error.message;
-
-        const file = error instanceof TranslatorError ? error.path : '';
+        if (error instanceof TranslateError) {
+            console.error(red(error.code), error.message);
+        } else {
+            console.error(error);
+        }
 
-        logger.error(file, message);
+        process.exit(1);
     }
 }
 
-type TranslatorParams = {
-    input: string;
-    output: string;
-    sourceLanguage: string;
-    targetLanguage: string;
-    // yandexCloudTranslateGlossaryPairs: YandexCloudTranslateGlossaryPair[];
-};
+function scheduler(limit: number, interval: number) {
+    const scheduled: Defer<void>[] = [];
 
-type RequesterParams = {
-    auth: AuthInfo;
-    folderId: string | undefined;
-    sourceLanguage: string;
-    targetLanguage: string;
-    dryRun: boolean;
-};
+    let processing = 0;
 
-type Request = {
-    (texts: string[]): () => Promise<string[]>;
-    stat: {
-        bytes: number;
-        chunks: number;
-    };
-};
+    function idle() {
+        const defer = new Defer<void>();
 
-type Split = (path: string, texts: string[]) => Promise<string[]>[];
+        scheduled.push(defer);
 
-type Cache = Map<string, Defer>;
+        return defer.promise;
+    }
 
-function requester(params: RequesterParams, cache: Cache) {
-    const {auth, folderId, sourceLanguage, targetLanguage, dryRun} = params;
-    const session = new Session(auth);
-    const client = session.client(TranslationServiceClient);
-    const resolve = (text: string, index: number, texts: string[]) => {
-        const defer = cache.get(texts[index]);
-        if (defer) {
-            defer.resolve([text]);
+    async function queue() {
+        processing++;
+        await wait(interval);
+        processing--;
+        unqueue();
+    }
+
+    async function unqueue() {
+        scheduled.shift()?.resolve();
+    }
+
+    return async function <R>(action: Function): Promise<R> {
+        if (processing >= limit) {
+            await idle();
         }
-        return text;
+
+        queue();
+
+        return action();
     };
+}
+
+function requester(params: RequesterParams, cache: Cache) {
+    const {auth, folderId, sourceLanguage, targetLanguage, dryRun} = params;
+    const schedule = scheduler(REQUESTS_LIMIT, 1000);
 
     const request = function request(texts: string[]) {
+        const resolve = (text: string, index: number) => {
+            const defer = cache.get(texts[index]);
+            if (defer) {
+                defer.resolve(text);
+            }
+        };
+
         request.stat.bytes += bytes(texts);
         request.stat.chunks++;
 
         return async function () {
             if (dryRun) {
-                return texts.map(resolve);
+                texts.forEach(resolve);
             }
 
-            return client
-                .translate(
-                    TranslateRequest.fromPartial({
-                        texts,
-                        folderId,
-                        sourceLanguageCode: sourceLanguage,
-                        targetLanguageCode: targetLanguage,
-                        // glossaryConfig: {
-                        //     glossaryData: {
-                        //         glossaryPairs: yandexCloudTranslateGlossaryPairs,
-                        //     },
-                        // },
-                        format: Format.HTML,
+            try {
+                const {data} = await schedule<AxiosResponse<Translations>>(() =>
+                    axios({
+                        method: 'POST',
+                        url: 'https://translate.api.cloud.yandex.net/translate/v2/translate',
+                        timeout: 5000,
+                        maxRedirects: 0,
+                        headers: {
+                            Authorization: auth,
+                            'Content-Type': 'application/json',
+                            'User-Agent': 'github.com/diplodoc-platform/cli',
+                        },
+                        data: {
+                            folderId,
+                            texts,
+                            sourceLanguageCode: sourceLanguage,
+                            targetLanguageCode: targetLanguage,
+                            format: 'HTML',
+                        },
                     }),
-                )
-                .then((results) => {
-                    return results.translations.map(({text}, index) => {
-                        return resolve(text, index, texts);
-                    });
-                })
-                .catch((error) => {
-                    console.error(error);
-                    throw new RequestError(error);
-                });
+                );
+
+                return data.translations.map(({text}) => text).forEach(resolve);
+            } catch (error: any) {
+                if (error instanceof AxiosError) {
+                    const {response} = error;
+                    const {status, statusText, data} = response as AxiosResponse;
+
+                    switch (true) {
+                        case LimitExceed.is(data.message):
+                            throw new LimitExceed(data.message);
+                        case AuthError.is(data.message):
+                            throw new AuthError(data.message);
+                        default:
+                            throw new RequestError(status, statusText, data);
+                    }
+                }
+
+                throw new RequestError(0, error.message, {fatal: true});
+            }
         };
     };
 
@@ -193,6 +242,8 @@ function requester(params: RequesterParams, cache: Cache) {
 
 function translator(params: TranslatorParams, split: Split) {
     const {input, output, sourceLanguage, targetLanguage} = params;
+    const inputRoot = resolve(input);
+    const outputRoot = resolve(output);
 
     return async (path: string) => {
         const ext = extname(path);
@@ -200,8 +251,8 @@ function translator(params: TranslatorParams, split: Split) {
             return;
         }
 
-        const inputPath = resolve(input, path);
-        const outputPath = resolve(output, path);
+        const inputPath = join(inputRoot, path);
+        const outputPath = join(outputRoot, path.replace(sourceLanguage, targetLanguage));
         const content = await loadFile(inputPath);
 
         await mkdir(dirname(outputPath), {recursive: true});
@@ -227,7 +278,7 @@ function translator(params: TranslatorParams, split: Split) {
             return;
         }
 
-        const parts = flat(await Promise.all(split(path, units)));
+        const parts = await split(path, units);
         const composed = compose(skeleton, parts, {useSource: true});
 
         await dumpFile(outputPath, composed);
@@ -235,50 +286,67 @@ function translator(params: TranslatorParams, split: Split) {
 }
 
 function splitter(request: Request, cache: Cache): Split {
-    return function (path: string, texts: string[]) {
-        const promises: Promise<string[]>[] = [];
+    return async function (path: string, texts: string[]) {
+        const promises: Promise<string>[] = [];
+        const requests: Promise<void>[] = [];
         let buffer: string[] = [];
         let bufferSize = 0;
 
         const release = () => {
-            promises.push(backoff(request(buffer)));
+            requests.push(backoff(request(buffer)));
             buffer = [];
             bufferSize = 0;
         };
 
         for (const text of texts) {
-            const defer = cache.get(text);
-
-            if (defer) {
-                promises.push(defer.promise);
-            } else if (text.length >= BYTES_LIMIT) {
+            if (text.length >= BYTES_LIMIT) {
                 logger.warn(path, 'Skip document part for translation. Part is too big.');
-                promises.push(Promise.resolve([text]));
+                promises.push(Promise.resolve(text));
             } else {
-                if (bufferSize + text.length > BYTES_LIMIT) {
-                    release();
+                const defer = cache.get(text) || new Defer();
+                promises.push(defer.promise);
+
+                if (!cache.get(text)) {
+                    if (bufferSize + text.length > BYTES_LIMIT) {
+                        release();
+                    }
+
+                    buffer.push(text);
+                    bufferSize += text.length;
                 }
 
-                buffer.push(text);
-                bufferSize += text.length;
-                cache.set(text, new Defer());
+                cache.set(text, defer);
             }
         }
 
         if (bufferSize) {
-            promises.push(backoff(request(buffer)));
+            release();
         }
 
-        return promises;
+        await Promise.all(requests);
+
+        return Promise.all(promises);
     };
 }
 
-function backoff(action: () => Promise<string[]>): Promise<string[]> {
-    return retry(
-        {
-            times: RETRY_LIMIT,
-            interval: (count: number) => Math.pow(2, count) * 1000,
-        },
-        asyncify(action),
-    );
+function wait(interval: number) {
+    const defer = new Defer<void>();
+    setTimeout(() => defer.resolve(), interval);
+    return defer.promise;
+}
+
+async function backoff(action: () => Promise<void>): Promise<void> {
+    let retry = 0;
+
+    while (++retry < RETRY_LIMIT) {
+        try {
+            await action();
+        } catch (error: any) {
+            if (RequestError.canRetry(error)) {
+                await wait(Math.pow(2, retry) * 1000);
+            } else {
+                throw error;
+            }
+        }
+    }
 }

package/src/cmd/translate/utils/errors.ts

@@ -0,0 +1,97 @@
+export class TranslateError extends Error {
+    code: string;
+
+    fatal: boolean;
+
+    constructor(message: string, code: string, fatal = false) {
+        super(message);
+
+        this.code = code;
+        this.fatal = fatal;
+    }
+}
+
+export class RequestError extends TranslateError {
+    static canRetry(error: any) {
+        if (error instanceof RequestError) {
+            switch (true) {
+                case error.status === 429:
+                    return true;
+                case error.status === 500:
+                    return true;
+                case error.status === 503:
+                    return true;
+                case error.status === 504:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+
+        return false;
+    }
+
+    status: number;
+
+    constructor(
+        status: number,
+        statusText: string,
+        info: {code?: number; message?: string; fatal?: boolean} = {},
+    ) {
+        super(`${statusText}\n${info.message || ''}`, 'REQUEST_ERROR', info.fatal);
+
+        this.status = status;
+    }
+}
+
+const INACTIVE_CLOUD = /^The cloud .*? is inactive/;
+const WRONG_APIKEY = /^Unknown api key/;
+const WRONG_TOKEN = /^The token is invalid/;
+
+export class AuthError extends TranslateError {
+    static is(message: string) {
+        return Boolean(AuthError.reason(message));
+    }
+
+    static reason(message: string) {
+        switch (true) {
+            case INACTIVE_CLOUD.test(message):
+                return 'INACTIVE_CLOUD';
+            case WRONG_APIKEY.test(message):
+                return 'WRONG_APIKEY';
+            case WRONG_TOKEN.test(message):
+                return 'WRONG_TOKEN';
+            default:
+                return null;
+        }
+    }
+
+    constructor(message: string) {
+        super(message, AuthError.reason(message) || 'AUTH_ERROR', true);
+    }
+}
+
+const LIMIT_EXCEED_RX = /^limit on units was exceeded. (.*)$/;
+
+export class LimitExceed extends TranslateError {
+    static is(message: string) {
+        return Boolean(LIMIT_EXCEED_RX.test(message));
+    }
+
+    constructor(message: string) {
+        const [, desc] = LIMIT_EXCEED_RX.exec(message) || [];
+        super(desc, 'TRANSLATE_LIMIT_EXCEED', true);
+    }
+}
+
+export class ExtractError extends TranslateError {
+    constructor(error: Error) {
+        super('EXTRACT_ERROR', error?.message || String(error));
+    }
+}
+
+export class ComposeError extends TranslateError {
+    constructor(error: Error) {
+        super('COMPOSE_ERROR', error?.message || String(error));
+    }
+}

package/src/cmd/translate/utils/index.ts

@@ -5,6 +5,7 @@ import glob from 'glob';
 
 export {dumpFile, loadFile} from './fs';
 export {extract, compose} from './translate';
+export {TranslateError, LimitExceed, RequestError, AuthError} from './errors';
 
 type TranslateArgs = {
     input: string;
@@ -141,16 +142,12 @@ export function resolveSchemas(_path: string) {
     return null;
 }
 
-export function flat<T>(parts: T[][]) {
-    return ([] as T[]).concat(...parts);
-}
-
-export class Defer {
-    resolve!: (text: string[]) => void;
+export class Defer<T = string> {
+    resolve!: (text: T) => void;
 
     reject!: (error: any) => void;
 
-    promise: Promise<string[]>;
+    promise: Promise<T>;
 
     constructor() {
         this.promise = new Promise((resolve, reject) => {

package/src/cmd/translate/utils/translate.ts

@@ -1,23 +1,6 @@
 import type {ComposeOptions, ExtractOptions} from '@diplodoc/translation';
 import {compose as _compose, extract as _extract} from '@diplodoc/translation';
-
-class ExtractError extends Error {
-    code: string;
-
-    constructor(error: Error) {
-        super(error?.message || String(error));
-        this.code = 'EXTRACT_ERROR';
-    }
-}
-
-class ComposeError extends Error {
-    code: string;
-
-    constructor(error: Error) {
-        super(error?.message || String(error));
-        this.code = 'COMPOSE_ERROR';
-    }
-}
+import {ComposeError, ExtractError} from './errors';
 
 type Content = Parameters<typeof _extract>[0];
 

package/src/cmd/translate/yandex/auth.ts

@@ -1,41 +1,32 @@
 import {readFileSync} from 'fs';
 
-type ServiceAccauntInfo = {
-    id: string;
-    service_account_id: string;
-    public_key: string;
-    private_key: string;
-};
+const resolveKey = (data: string) => {
+    data = data.trim();
 
-export type AuthInfo = ReturnType<typeof getYandexAuth>;
+    switch (true) {
+        case data.startsWith('y0_'):
+            return 'Bearer ' + data;
+        case data.startsWith('t1.'):
+            return 'Bearer ' + data;
+        case data.startsWith('AQVN'):
+            return 'Api-Key ' + data;
+        default:
+            return null;
+    }
+};
 
 export function getYandexAuth(path: string) {
-    if (path.startsWith('y0_')) {
-        return {
-            oauthToken: path,
-        };
-    }
+    let auth = resolveKey(path);
 
-    const data = readFileSync(path, 'utf8');
-    try {
-        const json = JSON.parse(data);
+    if (auth !== null) {
+        return auth;
+    }
 
-        if (isServeseAccount(json)) {
-            return {
-                serviceAccountJson: {
-                    serviceAccountId: json.service_account_id,
-                    accessKeyId: json.id,
-                    privateKey: json.private_key,
-                },
-            };
-        }
-    } catch {}
+    auth = resolveKey(readFileSync(path, 'utf8'));
 
-    return {
-        oauthToken: data,
-    };
-}
+    if (auth === null) {
+        throw new Error('No Auth');
+    }
 
-function isServeseAccount(json: any): json is ServiceAccauntInfo {
-    return 'private_key' in json;
+    return auth;
 }

package/src/utils/logger.ts

@@ -32,8 +32,8 @@ export const logger = {
 
         log.warn(`file: ${pathToFile} ${extraMessage}`);
     },
-    error: function (pathToFile: string, extraMessage: string) {
-        const message = `${red('ERROR')} file: ${pathToFile} error: ${extraMessage}`;
+    error: function (pathToFile: string, extraMessage: string, reason?: string) {
+        const message = `${red(reason || 'ERROR')} file: ${pathToFile} error: ${extraMessage}`;
 
         writeLog(message, true);
 

package/src/packages/credentials/index.ts

@@ -1 +0,0 @@
-export * from './yandex-oauth';