@diplodoc/cli 4.6.4 → 4.6.5

package/build/index.js

@@ -3905,7 +3905,8 @@ var translate = {
 };
 var MD_GLOB3 = "**/*.md";
 var REQUESTS_LIMIT = 20;
-var RETRY_LIMIT = 8;
+var BYTES_LIMIT = 1e4;
+var RETRY_LIMIT = 3;
 var MTRANS_LOCALE = "MTRANS";
 function builder6(argv) {
   return argv.option("source-language", {
@@ -3986,6 +3987,20 @@ function translator(params) {
   } = params;
   const session = new import_session.Session({ oauthToken });
   const client3 = session.client(import_service_clients.TranslationServiceClient);
+  const request = (texts) => () => client3.translate(
+    import_translation_service.TranslateRequest.fromPartial({
+      texts,
+      folderId,
+      sourceLanguageCode: sourceLanguage,
+      targetLanguageCode: targetLanguage,
+      glossaryConfig: {
+        glossaryData: {
+          glossaryPairs: yandexCloudTranslateGlossaryPairs
+        }
+      },
+      format: import_translation_service.TranslateRequest_Format.PLAIN_TEXT
+    })
+  ).then((results) => results.translations.map(({ text }) => text));
   return (mdPath) => __async(this, null, function* () {
     try {
       logger.info(mdPath, "translating");
@@ -4004,40 +4019,44 @@ function translator(params) {
         skeletonPath: ""
       });
       const texts = parseSourcesFromXLIFF(xlf);
-      const machineTranslateParams = import_translation_service.TranslateRequest.fromPartial({
-        texts,
-        folderId,
-        sourceLanguageCode: sourceLanguage,
-        targetLanguageCode: targetLanguage,
-        glossaryConfig: {
-          glossaryData: {
-            glossaryPairs: yandexCloudTranslateGlossaryPairs
-          }
-        },
-        format: import_translation_service.TranslateRequest_Format.PLAIN_TEXT
-      });
-      const translations = yield (0, import_async5.retry)(
-        {
-          times: RETRY_LIMIT,
-          interval: (count) => {
-            return (1 << count) * 1e3;
+      const parts = yield Promise.all(
+        texts.reduce(
+          ({
+            promises,
+            buffer,
+            bufferSize
+          }, text, index) => {
+            if (text.length >= BYTES_LIMIT) {
+              logger.warn(
+                mdPath,
+                "Skip document part for translation. Part is too big."
+              );
+              promises.push(Promise.resolve([text]));
+              return { promises, buffer, bufferSize };
+            }
+            if (bufferSize + text.length > BYTES_LIMIT || index === texts.length - 1) {
+              promises.push(backoff(request(buffer)));
+              buffer = [];
+              bufferSize = 0;
+            }
+            buffer.push(text);
+            bufferSize += text.length;
+            return { promises, buffer, bufferSize };
+          },
+          {
+            promises: [],
+            buffer: [],
+            bufferSize: 0
           }
-        },
-        (0, import_async5.asyncify)(
-          () => __async(this, null, function* () {
-            return yield client3.translate(machineTranslateParams).then(
-              (results) => results.translations.map(({ text }) => text)
-            );
-          })
-        )
+        ).promises
       );
-      const createXLIFFDocumentParams = {
+      const translations = [].concat(...parts);
+      const translatedXLIFF = createXLIFFDocument({
         sourceLanguage: sourceLanguage + "-" + MTRANS_LOCALE,
         targetLanguage: targetLanguage + "-" + MTRANS_LOCALE,
         sources: texts,
         targets: translations
-      };
-      const translatedXLIFF = createXLIFFDocument(createXLIFFDocumentParams);
+      });
       const composed = yield import_markdown_translation3.default.compose({
         xlf: translatedXLIFF,
         skeleton
@@ -4053,6 +4072,17 @@ function translator(params) {
     }
   });
 }
+function backoff(action) {
+  return (0, import_async5.retry)(
+    {
+      times: RETRY_LIMIT,
+      interval: (count) => {
+        return (1 << count) * 1e3;
+      }
+    },
+    (0, import_async5.asyncify)(action)
+  );
+}
 function parseSourcesFromXLIFF(xliff2) {
   var _a, _b, _c, _d;
   const parser = new import_fast_xml_parser.XMLParser();
@@ -4095,7 +4125,7 @@ import_yargs.default.command(build).command(publish).command(xliff).command(tran
   default: false,
   describe: "Run in quiet mode. Don't write logs to stdout",
   type: "boolean"
-}).group(["config", "strict", "quiet", "help", "version"], "Common options:").version(true ? "4.6.4" : "").help().parse((0, import_helpers.hideBin)(process.argv), {}, (err, { strict }, output) => {
+}).group(["config", "strict", "quiet", "help", "version"], "Common options:").version(true ? "4.6.5" : "").help().parse((0, import_helpers.hideBin)(process.argv), {}, (err, { strict }, output) => {
   console.timeEnd(MAIN_TIMER_ID);
   if (err) {
     console.error(err);