@diplodoc/cli 4.13.2 → 4.13.3

package/build/index.js

@@ -3975,14 +3975,15 @@ function builder5(argv) {
 }
 
 // src/cmd/translate/index.ts
+var import_glob4 = __toESM(require("glob"));
+var import_assert = require("assert");
 var import_async5 = require("async");
 var import_path33 = require("path");
 var import_promises4 = require("fs/promises");
-var import_fast_xml_parser = require("fast-xml-parser");
 var import_session = require("@yandex-cloud/nodejs-sdk/dist/session");
 var import_service_clients = require("@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/service_clients");
 var import_translation_service = require("@yandex-cloud/nodejs-sdk/dist/generated/yandex/cloud/ai/translate/v2/translation_service");
-var import_markdown_translation3 = __toESM(require("@diplodoc/markdown-translation"));
+var import_markdown_translation3 = require("@diplodoc/markdown-translation");
 
 // src/packages/credentials/yandex-oauth.ts
 var import_promises3 = require("fs/promises");
@@ -4024,13 +4025,14 @@ var translate = {
   handler: handler5,
   builder: builder6
 };
-var MD_GLOB3 = "**/*.md";
 var REQUESTS_LIMIT = 20;
 var BYTES_LIMIT = 1e4;
 var RETRY_LIMIT = 3;
-var MTRANS_LOCALE = "MTRANS";
 function builder6(argv) {
-  return argv.option("source-language", {
+  return argv.option("folder-id", {
+    describe: "folder id",
+    type: "string"
+  }).option("source-language", {
     alias: "sl",
     describe: "source language code",
     type: "string"
@@ -4038,10 +4040,7 @@ function builder6(argv) {
     alias: "tl",
     describe: "target language code",
     type: "string"
-  }).demandOption(
-    ["source-language", "target-language"],
-    "command requires to specify source and target languages"
-  );
+  }).check(argvValidator);
 }
 var TranslatorError = class extends Error {
   constructor(message, path) {
@@ -4051,34 +4050,47 @@ var TranslatorError = class extends Error {
 };
 function handler5(args) {
   return __async(this, null, function* () {
-    argv_default.init(__spreadValues({}, args));
+    argv_default.init(__spreadValues(__spreadValues({}, args.translate || {}), args));
     const {
-      input,
-      output,
-      yandexCloudTranslateFolderId,
-      yandexCloudTranslateGlossaryPairs,
-      sl: sourceLanguage,
-      tl: targetLanguage
-    } = args;
+      folderId,
+      // yandexCloudTranslateGlossaryPairs,
+      sourceLanguage,
+      targetLanguage,
+      exclude = []
+    } = argv_default.getConfig();
+    let { input, output, include = [] } = argv_default.getConfig();
     logger.info(
       input,
       `translating documentation from ${sourceLanguage} to ${targetLanguage} language`
     );
+    output = output || input;
+    (0, import_assert.ok)(input, "Required param input is not configured");
+    (0, import_assert.ok)(sourceLanguage, "Required param sourceLanguage is not configured");
+    (0, import_assert.ok)(targetLanguage, "Required param targetLanguage is not configured");
     try {
-      let found = [];
-      ({
-        state: { found }
-      } = yield glob((0, import_path33.join)(input, MD_GLOB3), {
-        nosort: true
-      }));
+      if (input.endsWith(".md")) {
+        include = [(0, import_path33.basename)(input)];
+        input = (0, import_path33.dirname)(input);
+      } else if (!include.length) {
+        include.push("**/*");
+      }
+      const files = [].concat(
+        ...include.map(
+          (match) => import_glob4.default.sync(match, {
+            cwd: (0, import_path33.join)(input, sourceLanguage),
+            ignore: exclude
+          })
+        )
+      );
+      const found = [...new Set(files)];
       const oauthToken = yield getYandexOAuthToken();
       const translatorParams = {
         input,
         output,
         sourceLanguage,
         targetLanguage,
-        yandexCloudTranslateGlossaryPairs,
-        folderId: yandexCloudTranslateFolderId,
+        // yandexCloudTranslateGlossaryPairs,
+        folderId,
         oauthToken
       };
       const translateFn = translator(translatorParams);
@@ -4103,45 +4115,69 @@ function translator(params) {
     input,
     output,
     sourceLanguage,
-    targetLanguage,
-    yandexCloudTranslateGlossaryPairs
+    targetLanguage
+    // yandexCloudTranslateGlossaryPairs,
   } = params;
+  const tmap = /* @__PURE__ */ new Map();
   const session = new import_session.Session({ oauthToken });
   const client = session.client(import_service_clients.TranslationServiceClient);
-  const request = (texts) => () => client.translate(
-    import_translation_service.TranslateRequest.fromPartial({
-      texts,
-      folderId,
-      sourceLanguageCode: sourceLanguage,
-      targetLanguageCode: targetLanguage,
-      glossaryConfig: {
-        glossaryData: {
-          glossaryPairs: yandexCloudTranslateGlossaryPairs
+  const request = (texts) => () => {
+    return client.translate(
+      import_translation_service.TranslateRequest.fromPartial({
+        texts,
+        folderId,
+        sourceLanguageCode: sourceLanguage,
+        targetLanguageCode: targetLanguage,
+        // glossaryConfig: {
+        //     glossaryData: {
+        //         glossaryPairs: yandexCloudTranslateGlossaryPairs,
+        //     },
+        // },
+        format: import_translation_service.TranslateRequest_Format.HTML
+      })
+    ).then((results) => {
+      return results.translations.map(({ text }, index) => {
+        const defer = tmap.get(texts[index]);
+        if (defer) {
+          defer.resolve([text]);
         }
-      },
-      format: import_translation_service.TranslateRequest_Format.PLAIN_TEXT
-    })
-  ).then((results) => results.translations.map(({ text }) => text));
+        return text;
+      });
+    });
+  };
   return (mdPath) => __async(this, null, function* () {
+    if (!mdPath.endsWith(".md")) {
+      return;
+    }
     try {
       logger.info(mdPath, "translating");
-      const md = yield (0, import_promises4.readFile)((0, import_path33.resolve)(mdPath), { encoding: "utf-8" });
-      const { xlf, skeleton } = import_markdown_translation3.default.extract({
+      const inputPath = (0, import_path33.resolve)(input, sourceLanguage, mdPath);
+      const outputPath = (0, import_path33.resolve)(output, targetLanguage, mdPath);
+      const md = yield (0, import_promises4.readFile)(inputPath, { encoding: "utf-8" });
+      yield (0, import_promises4.mkdir)((0, import_path33.dirname)(outputPath), { recursive: true });
+      if (!md) {
+        yield (0, import_promises4.writeFile)(outputPath, md);
+        return;
+      }
+      const { units, skeleton } = (0, import_markdown_translation3.extract)({
         source: {
           language: sourceLanguage,
-          locale: sourceLanguage.toUpperCase()
+          locale: "RU"
         },
         target: {
           language: targetLanguage,
-          locale: targetLanguage.toUpperCase()
+          locale: "US"
         },
         markdown: md,
         markdownPath: mdPath,
         skeletonPath: ""
       });
-      const texts = parseSourcesFromXLIFF(xlf);
+      if (!units.length) {
+        yield (0, import_promises4.writeFile)(outputPath, md);
+        return;
+      }
       const parts = yield Promise.all(
-        texts.reduce(
+        units.reduce(
           ({
             promises,
             buffer,
@@ -4155,13 +4191,23 @@ function translator(params) {
               promises.push(Promise.resolve([text]));
               return { promises, buffer, bufferSize };
             }
-            if (bufferSize + text.length > BYTES_LIMIT || index === texts.length - 1) {
+            const defer = tmap.get(text);
+            if (defer) {
+              console.log("SKIPPED", text);
+              promises.push(defer.promise);
+              return { promises, buffer, bufferSize };
+            }
+            if (bufferSize + text.length > BYTES_LIMIT) {
               promises.push(backoff(request(buffer)));
               buffer = [];
               bufferSize = 0;
             }
             buffer.push(text);
             bufferSize += text.length;
+            tmap.set(text, new Defer());
+            if (index === units.length - 1) {
+              promises.push(backoff(request(buffer)));
+            }
             return { promises, buffer, bufferSize };
           },
           {
@@ -4172,18 +4218,11 @@ function translator(params) {
         ).promises
       );
       const translations = [].concat(...parts);
-      const translatedXLIFF = createXLIFFDocument({
-        sourceLanguage: sourceLanguage + "-" + MTRANS_LOCALE,
-        targetLanguage: targetLanguage + "-" + MTRANS_LOCALE,
-        sources: texts,
-        targets: translations
-      });
-      const composed = yield import_markdown_translation3.default.compose({
-        xlf: translatedXLIFF,
+      const composed = yield (0, import_markdown_translation3.compose)({
+        useSource: true,
+        units: translations,
         skeleton
       });
-      const outputPath = mdPath.replace(input, output);
-      yield (0, import_promises4.mkdir)((0, import_path33.dirname)(outputPath), { recursive: true });
       yield (0, import_promises4.writeFile)(outputPath, composed);
       logger.info(outputPath, "finished translating");
     } catch (err) {
@@ -4204,31 +4243,14 @@ function backoff(action) {
     (0, import_async5.asyncify)(action)
   );
 }
-function parseSourcesFromXLIFF(xliff2) {
-  var _a, _b, _c, _d;
-  const parser = new import_fast_xml_parser.XMLParser();
-  const inputs = (_d = (_c = (_b = (_a = parser.parse(xliff2)) == null ? void 0 : _a.xliff) == null ? void 0 : _b.file) == null ? void 0 : _c.body["trans-unit"]) != null ? _d : [];
-  return Array.isArray(inputs) ? inputs.map(({ source }) => source) : [inputs.source];
-}
-function createXLIFFDocument(params) {
-  const { sourceLanguage, targetLanguage, sources, targets } = params;
-  const unit = (text, i) => `
-<trans-unit id="${i + 1}">
-    <source xml:lang="${sourceLanguage}">${sources[i]}</source>
-    <target xml:lang="${targetLanguage}">${text}</target>
-</trans-unit>`;
-  const doc = `
-<?xml version="1.0" encoding="UTF-8"?>
-<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" version="1.2">
-    <file original="" source-language="${sourceLanguage}" target-language="${targetLanguage}">
-        <header>
-            <skl><external-file href="" /></skl>
-        </header>
-        <body>${targets.map(unit)}</body>
-    </file>
-</xliff>`;
-  return doc;
-}
+var Defer = class {
+  constructor() {
+    this.promise = new Promise((resolve18, reject) => {
+      this.resolve = resolve18;
+      this.reject = reject;
+    });
+  }
+};
 
 // src/index.ts
 console.time(MAIN_TIMER_ID);
@@ -4246,7 +4268,7 @@ import_yargs.default.command(build).command(publish).command(xliff).command(tran
   default: false,
   describe: "Run in quiet mode. Don't write logs to stdout",
   type: "boolean"
-}).group(["config", "strict", "quiet", "help", "version"], "Common options:").version(true ? "4.13.2" : "").help().parse((0, import_helpers.hideBin)(process.argv), {}, (err, { strict }, output) => {
+}).group(["config", "strict", "quiet", "help", "version"], "Common options:").version(true ? "4.13.3" : "").help().parse((0, import_helpers.hideBin)(process.argv), {}, (err, { strict }, output) => {
   console.timeEnd(MAIN_TIMER_ID);
   if (err) {
     console.error(err);