lingo.dev 0.92.13 → 0.92.14

package/build/cli.mjs

@@ -1025,7 +1025,7 @@ import {
 } from "@lingo.dev/_spec";
 import { Command as Command6 } from "interactive-commander";
 import Z3 from "zod";
-import _25 from "lodash";
+import _26 from "lodash";
 import Ora5 from "ora";
 
 // src/cli/loaders/_utils.ts
@@ -1249,7 +1249,7 @@ function createTextFileLoader(pathPattern) {
       const trimmedResult = result.trim();
       return trimmedResult;
     },
-    async push(locale, data, _29, originalLocale) {
+    async push(locale, data, _30, originalLocale) {
       const draftPath = pathPattern.replaceAll("[locale]", locale);
       const finalPath = path10.resolve(draftPath);
       const dirPath = path10.dirname(finalPath);
@@ -1778,7 +1778,7 @@ function createPropertiesLoader() {
       return result;
     },
     async push(locale, payload) {
-      const result = Object.entries(payload).filter(([_29, value]) => value != null).map(([key, value]) => `${key}=${value}`).join("\n");
+      const result = Object.entries(payload).filter(([_30, value]) => value != null).map(([key, value]) => `${key}=${value}`).join("\n");
       return result;
     }
   });
@@ -2045,10 +2045,10 @@ function createUnlocalizableLoader(isCacheRestore = false, returnUnlocalizedKeys
           }
         }
         return false;
-      }).map(([key, _29]) => key);
-      const result = _10.omitBy(input2, (_29, key) => passthroughKeys.includes(key));
+      }).map(([key, _30]) => key);
+      const result = _10.omitBy(input2, (_30, key) => passthroughKeys.includes(key));
       if (returnUnlocalizedKeys) {
-        result.unlocalizable = _10.omitBy(input2, (_29, key) => !passthroughKeys.includes(key));
+        result.unlocalizable = _10.omitBy(input2, (_30, key) => !passthroughKeys.includes(key));
       }
       return result;
     },
@@ -4005,8 +4005,24 @@ function createLingoLocalizer(params) {
 
 // src/cli/processor/basic.ts
 import { generateText } from "ai";
+import _24 from "lodash";
 function createBasicTranslator(model, systemPrompt) {
   return async (input2, onProgress) => {
+    const chunks = extractPayloadChunks(input2.processableData);
+    const subResults = [];
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i];
+      const result2 = await doJob({
+        ...input2,
+        processableData: chunk
+      });
+      subResults.push(result2);
+      onProgress(i / chunks.length * 100, chunk, result2);
+    }
+    const result = _24.merge({}, ...subResults);
+    return result;
+  };
+  async function doJob(input2) {
     if (!Object.keys(input2.processableData).length) {
       return input2.processableData;
     }
@@ -4052,7 +4068,41 @@ function createBasicTranslator(model, systemPrompt) {
     });
     const result = JSON.parse(response.text);
     return result?.data || {};
-  };
+  }
+}
+function extractPayloadChunks(payload) {
+  const idealBatchItemSize = 250;
+  const batchSize = 25;
+  const result = [];
+  let currentChunk = {};
+  let currentChunkItemCount = 0;
+  const payloadEntries = Object.entries(payload);
+  for (let i = 0; i < payloadEntries.length; i++) {
+    const [key, value] = payloadEntries[i];
+    currentChunk[key] = value;
+    currentChunkItemCount++;
+    const currentChunkSize = countWordsInRecord(currentChunk);
+    if (currentChunkSize > idealBatchItemSize || currentChunkItemCount >= batchSize || i === payloadEntries.length - 1) {
+      result.push(currentChunk);
+      currentChunk = {};
+      currentChunkItemCount = 0;
+    }
+  }
+  return result;
+}
+function countWordsInRecord(payload) {
+  if (Array.isArray(payload)) {
+    return payload.reduce((acc, item) => acc + countWordsInRecord(item), 0);
+  } else if (typeof payload === "object" && payload !== null) {
+    return Object.values(payload).reduce(
+      (acc, item) => acc + countWordsInRecord(item),
+      0
+    );
+  } else if (typeof payload === "string") {
+    return payload.trim().split(/\s+/).filter(Boolean).length;
+  } else {
+    return 0;
+  }
 }
 
 // src/cli/processor/index.ts
@@ -4178,7 +4228,7 @@ async function trackEvent(distinctId, event, properties) {
 }
 
 // src/cli/utils/delta.ts
-import _24 from "lodash";
+import _25 from "lodash";
 import z from "zod";
 
 // src/cli/utils/fs.ts
@@ -4227,9 +4277,9 @@ function createDeltaProcessor(fileKey) {
       return checkIfFileExists(lockfilePath);
     },
     async calculateDelta(params) {
-      let added = _24.difference(Object.keys(params.sourceData), Object.keys(params.targetData));
-      let removed = _24.difference(Object.keys(params.targetData), Object.keys(params.sourceData));
-      const updated = _24.filter(Object.keys(params.sourceData), (key) => {
+      let added = _25.difference(Object.keys(params.sourceData), Object.keys(params.targetData));
+      let removed = _25.difference(Object.keys(params.targetData), Object.keys(params.sourceData));
+      const updated = _25.filter(Object.keys(params.sourceData), (key) => {
         return md5(params.sourceData[key]) !== params.checksums[key] && params.checksums[key];
       });
       const renamed = [];
@@ -4278,7 +4328,7 @@ function createDeltaProcessor(fileKey) {
       await this.saveLock(lockfileData);
     },
     async createChecksums(sourceData) {
-      const checksums = _24.mapValues(sourceData, (value) => md5(value));
+      const checksums = _25.mapValues(sourceData, (value) => md5(value));
       return checksums;
     }
   };
@@ -4450,7 +4500,7 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
           const deltaProcessor = createDeltaProcessor(bucketPath.pathPattern);
           const sourceChecksums = await deltaProcessor.createChecksums(sourceData);
           const savedChecksums = await deltaProcessor.loadChecksums();
-          const updatedSourceData = _25.pickBy(
+          const updatedSourceData = _26.pickBy(
             sourceData,
             (value, key) => sourceChecksums[key] !== savedChecksums[key]
           );
@@ -4464,15 +4514,15 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
               bucketPath.delimiter
             );
             const { unlocalizable: targetUnlocalizable, ...targetData } = await bucketLoader.pull(targetLocale);
-            const missingKeys = _25.difference(
+            const missingKeys = _26.difference(
               Object.keys(sourceData),
               Object.keys(targetData)
             );
-            const extraKeys = _25.difference(
+            const extraKeys = _26.difference(
               Object.keys(targetData),
               Object.keys(sourceData)
             );
-            const unlocalizableDataDiff = !_25.isEqual(
+            const unlocalizableDataDiff = !_26.isEqual(
               sourceUnlocalizable,
               targetUnlocalizable
             );
@@ -4554,13 +4604,13 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
                 targetData,
                 checksums: checksums2
               });
-              let processableData = _25.chain(sourceData).entries().filter(
+              let processableData = _26.chain(sourceData).entries().filter(
                 ([key, value]) => delta.added.includes(key) || delta.updated.includes(key) || !!flags.force
               ).fromPairs().value();
               if (flags.key) {
-                processableData = _25.pickBy(
+                processableData = _26.pickBy(
                   processableData,
-                  (_29, key) => key === flags.key
+                  (_30, key) => key === flags.key
                 );
               }
               if (flags.verbose) {
@@ -4593,13 +4643,13 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
               if (flags.verbose) {
                 bucketOra.info(JSON.stringify(processedTargetData, null, 2));
               }
-              let finalTargetData = _25.merge(
+              let finalTargetData = _26.merge(
                 {},
                 sourceData,
                 targetData,
                 processedTargetData
               );
-              finalTargetData = _25.chain(finalTargetData).entries().map(([key, value]) => {
+              finalTargetData = _26.chain(finalTargetData).entries().map(([key, value]) => {
                 const renaming = delta.renamed.find(
                   ([oldKey, newKey]) => oldKey === key
                 );
@@ -4623,7 +4673,7 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
                   `Applying changes to ${bucketPath} (${targetLocale})`
                 );
               }
-              const finalDiffSize = _25.chain(finalTargetData).omitBy((value, key) => value === targetData[key]).size().value();
+              const finalDiffSize = _26.chain(finalTargetData).omitBy((value, key) => value === targetData[key]).size().value();
               await bucketLoader.push(targetLocale, finalTargetData);
               if (finalDiffSize > 0 || flags.force) {
                 bucketOra.succeed(
@@ -4794,7 +4844,7 @@ Reviewing changes for ${chalk2.blue(args.pathPattern)} (${chalk2.yellow(args.tar
     return args.currentData;
   }
   const customData = { ...args.currentData };
-  const changes = _25.reduce(
+  const changes = _26.reduce(
     args.proposedData,
     (result, value, key) => {
       if (args.currentData[key] !== value) {
@@ -4867,7 +4917,7 @@ import path14 from "path";
 import Z4 from "zod";
 import YAML5 from "yaml";
 import { MD5 as MD52 } from "object-hash";
-import _26 from "lodash";
+import _27 from "lodash";
 function createLockfileHelper() {
   return {
     isLockfileExists: () => {
@@ -4877,23 +4927,23 @@ function createLockfileHelper() {
     registerSourceData: (pathPattern, sourceData) => {
       const lockfile = _loadLockfile();
       const sectionKey = MD52(pathPattern);
-      const sectionChecksums = _26.mapValues(sourceData, (value) => MD52(value));
+      const sectionChecksums = _27.mapValues(sourceData, (value) => MD52(value));
       lockfile.checksums[sectionKey] = sectionChecksums;
       _saveLockfile(lockfile);
     },
     registerPartialSourceData: (pathPattern, partialSourceData) => {
       const lockfile = _loadLockfile();
       const sectionKey = MD52(pathPattern);
-      const sectionChecksums = _26.mapValues(partialSourceData, (value) => MD52(value));
-      lockfile.checksums[sectionKey] = _26.merge({}, lockfile.checksums[sectionKey] ?? {}, sectionChecksums);
+      const sectionChecksums = _27.mapValues(partialSourceData, (value) => MD52(value));
+      lockfile.checksums[sectionKey] = _27.merge({}, lockfile.checksums[sectionKey] ?? {}, sectionChecksums);
       _saveLockfile(lockfile);
     },
     extractUpdatedData: (pathPattern, sourceData) => {
       const lockfile = _loadLockfile();
       const sectionKey = MD52(pathPattern);
-      const currentChecksums = _26.mapValues(sourceData, (value) => MD52(value));
+      const currentChecksums = _27.mapValues(sourceData, (value) => MD52(value));
       const savedChecksums = lockfile.checksums[sectionKey] || {};
-      const updatedData = _26.pickBy(sourceData, (value, key) => savedChecksums[key] !== currentChecksums[key]);
+      const updatedData = _27.pickBy(sourceData, (value, key) => savedChecksums[key] !== currentChecksums[key]);
       return updatedData;
     }
   };
@@ -4963,7 +5013,7 @@ var flagsSchema = Z5.object({
 // src/cli/cmd/cleanup.ts
 import { resolveOverriddenLocale as resolveOverriddenLocale5 } from "@lingo.dev/_spec";
 import { Command as Command8 } from "interactive-commander";
-import _27 from "lodash";
+import _28 from "lodash";
 import Ora7 from "ora";
 var cleanup_default = new Command8().command("cleanup").description("Remove keys from target files that do not exist in the source file").helpOption("-h, --help", "Show help").option("--locale <locale>", "Specific locale to cleanup").option("--bucket <bucket>", "Specific bucket to cleanup").option("--dry-run", "Show what would be removed without making changes").option(
   "--verbose",
@@ -4999,7 +5049,7 @@ var cleanup_default = new Command8().command("cleanup").description("Remove keys
           try {
             const targetData = await bucketLoader.pull(targetLocale);
             const targetKeys = Object.keys(targetData);
-            const keysToRemove = _27.difference(targetKeys, sourceKeys);
+            const keysToRemove = _28.difference(targetKeys, sourceKeys);
             if (keysToRemove.length === 0) {
               bucketOra.succeed(`[${targetLocale}] No keys to remove`);
               continue;
@@ -5008,7 +5058,7 @@ var cleanup_default = new Command8().command("cleanup").description("Remove keys
               bucketOra.info(`[${targetLocale}] Keys to remove: ${JSON.stringify(keysToRemove, null, 2)}`);
             }
             if (!options.dryRun) {
-              const cleanedData = _27.pick(targetData, sourceKeys);
+              const cleanedData = _28.pick(targetData, sourceKeys);
               await bucketLoader.push(targetLocale, cleanedData);
               bucketOra.succeed(`[${targetLocale}] Removed ${keysToRemove.length} keys`);
             } else {
@@ -5063,7 +5113,7 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import Z6 from "zod";
 import { ReplexicaEngine } from "@lingo.dev/_sdk";
-var mcp_default = new Command9().command("mcp").description("Use Lingo.dev model context provider with your AI agent").helpOption("-h, --help", "Show help").action(async (_29, program) => {
+var mcp_default = new Command9().command("mcp").description("Use Lingo.dev model context provider with your AI agent").helpOption("-h, --help", "Show help").action(async (_30, program) => {
   const apiKey = program.args[0];
   const settings = getSettings(apiKey);
   if (!settings.auth.apiKey) {
@@ -6317,7 +6367,7 @@ async function renderHero() {
 // package.json
 var package_default = {
   name: "lingo.dev",
-  version: "0.92.13",
+  version: "0.92.14",
   description: "Lingo.dev CLI",
   private: false,
   publishConfig: {
@@ -6885,7 +6935,7 @@ async function plan(input2) {
 import chalk10 from "chalk";
 import { Listr as Listr3 } from "listr2";
 import pLimit from "p-limit";
-import _28 from "lodash";
+import _29 from "lodash";
 var MAX_WORKER_COUNT = 10;
 async function execute(input2) {
   const effectiveConcurrency = Math.min(
@@ -6916,7 +6966,7 @@ async function execute(input2) {
           const workerTasks = [];
           for (let i = 0; i < workersCount; i++) {
             const assignedTasks = ctx.tasks.filter(
-              (_29, idx) => idx % workersCount === i
+              (_30, idx) => idx % workersCount === i
             );
             workerTasks.push(
               createWorkerTask({
@@ -7015,7 +7065,7 @@ function createWorkerTask(args) {
               targetData,
               checksums
             });
-            const processableData = _28.chain(sourceData).entries().filter(
+            const processableData = _29.chain(sourceData).entries().filter(
               ([key, value]) => delta.added.includes(key) || delta.updated.includes(key) || !!args.ctx.flags.force
             ).fromPairs().value();
             if (!Object.keys(processableData).length) {
@@ -7036,13 +7086,13 @@ function createWorkerTask(args) {
                 });
               }
             );
-            let finalTargetData = _28.merge(
+            let finalTargetData = _29.merge(
               {},
               sourceData,
               targetData,
               processedTargetData
             );
-            finalTargetData = _28.chain(finalTargetData).entries().map(([key, value]) => {
+            finalTargetData = _29.chain(finalTargetData).entries().map(([key, value]) => {
               const renaming = delta.renamed.find(
                 ([oldKey]) => oldKey === key
               );