lingo.dev 0.111.15 → 0.112.0

package/build/cli.mjs

@@ -569,26 +569,49 @@ async function pauseIfDebug(debug) {
 }
 async function renderSummary(results) {
   console.log(chalk.hex(colors.green)("[Done]"));
-  const skippedTasksCount = Array.from(results.values()).filter(
+  const skippedResults = Array.from(results.values()).filter(
     (r) => r.status === "skipped"
-  ).length;
-  console.log(`\u2022 ${chalk.hex(colors.yellow)(skippedTasksCount)} from cache`);
-  const succeededTasksCount = Array.from(results.values()).filter(
+  );
+  const succeededResults = Array.from(results.values()).filter(
     (r) => r.status === "success"
-  ).length;
-  console.log(`\u2022 ${chalk.hex(colors.yellow)(succeededTasksCount)} processed`);
-  const failedTasksCount = Array.from(results.values()).filter(
+  );
+  const failedResults = Array.from(results.values()).filter(
     (r) => r.status === "error"
-  ).length;
-  console.log(`\u2022 ${chalk.hex(colors.yellow)(failedTasksCount)} failed`);
-  if (failedTasksCount > 0) {
-    console.log(chalk.hex(colors.orange)("\n[Failed]"));
-    for (const result of Array.from(results.values()).filter(
-      (r) => r.status === "error"
-    )) {
+  );
+  console.log(
+    `\u2022 ${chalk.hex(colors.yellow)(skippedResults.length)} from cache`
+  );
+  console.log(
+    `\u2022 ${chalk.hex(colors.yellow)(succeededResults.length)} processed`
+  );
+  console.log(`\u2022 ${chalk.hex(colors.yellow)(failedResults.length)} failed`);
+  if (succeededResults.length > 0) {
+    console.log(chalk.hex(colors.green)("\n[Processed Files]"));
+    for (const result of succeededResults) {
+      const displayPath = result.pathPattern?.replace("[locale]", result.targetLocale) || "unknown";
       console.log(
-        `\u274C ${chalk.hex(colors.white)(String(result.error.message))}
-`
+        `  \u2713 ${chalk.dim(displayPath)} ${chalk.hex(colors.yellow)(`(${result.sourceLocale} \u2192 ${result.targetLocale})`)}`
+      );
+    }
+  }
+  if (skippedResults.length > 0) {
+    console.log(chalk.hex(colors.blue)("\n[Cached Files]"));
+    for (const result of skippedResults) {
+      const displayPath = result.pathPattern?.replace("[locale]", result.targetLocale) || "unknown";
+      console.log(
+        `  \u26A1 ${chalk.dim(displayPath)} ${chalk.hex(colors.yellow)(`(${result.sourceLocale} \u2192 ${result.targetLocale})`)}`
+      );
+    }
+  }
+  if (failedResults.length > 0) {
+    console.log(chalk.hex(colors.orange)("\n[Failed Files]"));
+    for (const result of failedResults) {
+      const displayPath = result.pathPattern?.replace("[locale]", result.targetLocale) || "unknown";
+      console.log(
+        `  \u274C ${chalk.dim(displayPath)} ${chalk.hex(colors.yellow)(`(${result.sourceLocale} \u2192 ${result.targetLocale})`)}`
+      );
+      console.log(
+        `     ${chalk.hex(colors.white)(String(result.error?.message || "Unknown error"))}`
       );
     }
   }
@@ -2690,6 +2713,133 @@ ${content}`;
   });
 }
 
+// src/cli/loaders/markdoc.ts
+import Markdoc from "@markdoc/markdoc";
+import YAML3 from "yaml";
+var FM_ATTR_PREFIX2 = "fm-attr-";
+function createMarkdocLoader() {
+  return createLoader({
+    async pull(locale, input2) {
+      const ast = Markdoc.parse(input2);
+      const result = {};
+      const counters = {};
+      traverseAndExtract(ast, "", result, counters);
+      if (ast.attributes?.frontmatter) {
+        const frontmatter = YAML3.parse(ast.attributes.frontmatter);
+        Object.entries(frontmatter).forEach(([key, value]) => {
+          if (typeof value === "string") {
+            result[`${FM_ATTR_PREFIX2}${key}`] = value;
+          }
+        });
+      }
+      return result;
+    },
+    async push(locale, data, originalInput) {
+      if (!originalInput) {
+        throw new Error("Original input is required for push");
+      }
+      const ast = Markdoc.parse(originalInput);
+      const counters = {};
+      const pathMap = {};
+      buildPathMap(ast, "", counters, pathMap);
+      const frontmatterEntries = Object.entries(data).filter(([key]) => key.startsWith(FM_ATTR_PREFIX2)).map(([key, value]) => [key.replace(FM_ATTR_PREFIX2, ""), value]);
+      if (frontmatterEntries.length > 0 && ast.attributes) {
+        const frontmatter = Object.fromEntries(frontmatterEntries);
+        ast.attributes.frontmatter = YAML3.stringify(frontmatter, {
+          defaultStringType: "PLAIN"
+        }).trim();
+      }
+      const contentData = Object.fromEntries(
+        Object.entries(data).filter(([key]) => !key.startsWith(FM_ATTR_PREFIX2))
+      );
+      applyTranslations(ast, "", contentData, pathMap);
+      return Markdoc.format(ast);
+    }
+  });
+}
+function getSemanticNodeType(node) {
+  if (node.type === "tag") return node.tag || "tag";
+  return node.type;
+}
+function traverseAndExtract(node, path19, result, counters, parentType) {
+  if (!node || typeof node !== "object") {
+    return;
+  }
+  let semanticType = parentType;
+  const nodeSemanticType = getSemanticNodeType(node);
+  if (nodeSemanticType && !["text", "strong", "em", "inline", "link"].includes(nodeSemanticType)) {
+    semanticType = nodeSemanticType;
+  }
+  if (node.type === "text" && node.attributes?.content) {
+    const content = node.attributes.content;
+    if (typeof content === "string" && content.trim()) {
+      if (semanticType) {
+        const index = counters[semanticType] || 0;
+        counters[semanticType] = index + 1;
+        const semanticKey = `${semanticType}-${index}`;
+        result[semanticKey] = content;
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    node.children.forEach((child, index) => {
+      const childPath = path19 ? `${path19}/children/${index}` : `children/${index}`;
+      traverseAndExtract(child, childPath, result, counters, semanticType);
+    });
+  }
+}
+function buildPathMap(node, path19, counters, pathMap, parentType) {
+  if (!node || typeof node !== "object") {
+    return;
+  }
+  let semanticType = parentType;
+  const nodeSemanticType = getSemanticNodeType(node);
+  if (nodeSemanticType && !["text", "strong", "em", "inline", "link"].includes(nodeSemanticType)) {
+    semanticType = nodeSemanticType;
+  }
+  if (node.type === "text" && node.attributes?.content) {
+    const content = node.attributes.content;
+    if (typeof content === "string" && content.trim()) {
+      if (semanticType) {
+        const index = counters[semanticType] || 0;
+        counters[semanticType] = index + 1;
+        const semanticKey = `${semanticType}-${index}`;
+        const contentPath = path19 ? `${path19}/attributes/content` : "attributes/content";
+        pathMap[semanticKey] = contentPath;
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    node.children.forEach((child, index) => {
+      const childPath = path19 ? `${path19}/children/${index}` : `children/${index}`;
+      buildPathMap(child, childPath, counters, pathMap, semanticType);
+    });
+  }
+}
+function applyTranslations(node, path19, data, pathMap) {
+  if (!node || typeof node !== "object") {
+    return;
+  }
+  if (node.type === "text" && node.attributes?.content) {
+    const content = node.attributes.content;
+    if (typeof content === "string") {
+      const contentPath = path19 ? `${path19}/attributes/content` : "attributes/content";
+      const semanticKey = Object.keys(pathMap).find(
+        (key) => pathMap[key] === contentPath
+      );
+      if (semanticKey && data[semanticKey] !== void 0) {
+        node.attributes.content = data[semanticKey];
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    node.children.forEach((child, index) => {
+      const childPath = path19 ? `${path19}/children/${index}` : `children/${index}`;
+      applyTranslations(child, childPath, data, pathMap);
+    });
+  }
+}
+
 // src/cli/loaders/properties.ts
 function createPropertiesLoader() {
   return createLoader({
@@ -6633,7 +6783,7 @@ function _isLockedKey(key, lockedKeys) {
 
 // src/cli/loaders/mdx2/frontmatter-split.ts
 import matter2 from "gray-matter";
-import YAML3 from "yaml";
+import YAML4 from "yaml";
 function createMdxFrontmatterSplitLoader() {
   const fmEngine = createFmEngine();
   return createLoader({
@@ -6654,8 +6804,8 @@ function createMdxFrontmatterSplitLoader() {
 }
 function createFmEngine() {
   const yamlEngine2 = {
-    parse: (str) => YAML3.parse(str),
-    stringify: (obj) => YAML3.stringify(obj, { defaultStringType: "PLAIN" })
+    parse: (str) => YAML4.parse(str),
+    stringify: (obj) => YAML4.stringify(obj, { defaultStringType: "PLAIN" })
   };
   return {
     parse: (input2) => matter2(input2, {
@@ -7263,6 +7413,15 @@ function createBucketLoader(bucketType, bucketPathPattern, options, lockedKeys,
         createSyncLoader(),
         createUnlocalizableLoader(options.returnUnlocalizedKeys)
       );
+    case "markdoc":
+      return composeLoaders(
+        createTextFileLoader(bucketPathPattern),
+        createMarkdocLoader(),
+        createFlatLoader(),
+        createEnsureKeyOrderLoader(),
+        createSyncLoader(),
+        createUnlocalizableLoader(options.returnUnlocalizedKeys)
+      );
     case "mdx":
       return composeLoaders(
         createTextFileLoader(bucketPathPattern),
@@ -7494,7 +7653,7 @@ function createLingoLocalizer(params) {
 // src/cli/processor/basic.ts
 import { generateText } from "ai";
 import _29 from "lodash";
-function createBasicTranslator(model, systemPrompt) {
+function createBasicTranslator(model, systemPrompt, settings = {}) {
   return async (input2, onProgress) => {
     const chunks = extractPayloadChunks(input2.processableData);
     const subResults = [];
@@ -7516,6 +7675,7 @@ function createBasicTranslator(model, systemPrompt) {
     }
     const response = await generateText({
       model,
+      ...settings,
       messages: [
         {
           role: "system",
@@ -7606,7 +7766,8 @@ function createProcessor(provider, params) {
     return result;
   } else {
     const model = getPureModelProvider(provider);
-    const result = createBasicTranslator(model, provider.prompt);
+    const settings = provider.settings || {};
+    const result = createBasicTranslator(model, provider.prompt, settings);
     return result;
   }
 }
@@ -7809,7 +7970,7 @@ function checkIfFileExists(filePath) {
 
 // src/cli/utils/delta.ts
 import * as path15 from "path";
-import YAML4 from "yaml";
+import YAML5 from "yaml";
 var LockSchema = z.object({
   version: z.literal(1).default(1),
   checksums: z.record(
@@ -7874,7 +8035,7 @@ function createDeltaProcessor(fileKey) {
     },
     async loadLock() {
       const lockfileContent = tryReadFile(lockfilePath, null);
-      const lockfileYaml = lockfileContent ? YAML4.parse(lockfileContent) : null;
+      const lockfileYaml = lockfileContent ? YAML5.parse(lockfileContent) : null;
       const lockfileData = lockfileYaml ? LockSchema.parse(lockfileYaml) : {
         version: 1,
         checksums: {}
@@ -7882,7 +8043,7 @@ function createDeltaProcessor(fileKey) {
       return lockfileData;
     },
     async saveLock(lockData) {
-      const lockfileYaml = YAML4.stringify(lockData);
+      const lockfileYaml = YAML5.stringify(lockData);
       writeFile(lockfilePath, lockfileYaml);
     },
     async loadChecksums() {
@@ -8564,7 +8725,7 @@ import Ora8 from "ora";
 import fs12 from "fs";
 import path16 from "path";
 import Z4 from "zod";
-import YAML5 from "yaml";
+import YAML6 from "yaml";
 import { MD5 as MD52 } from "object-hash";
 import _32 from "lodash";
 function createLockfileHelper() {
@@ -8612,12 +8773,12 @@ function createLockfileHelper() {
       return LockfileSchema.parse({});
     }
     const content = fs12.readFileSync(lockfilePath, "utf-8");
-    const result = LockfileSchema.parse(YAML5.parse(content));
+    const result = LockfileSchema.parse(YAML6.parse(content));
     return result;
   }
   function _saveLockfile(lockfile) {
     const lockfilePath = _getLockfilePath();
-    const content = YAML5.stringify(lockfile);
+    const content = YAML6.stringify(lockfile);
     fs12.writeFileSync(lockfilePath, content);
   }
   function _getLockfilePath() {
@@ -8958,8 +9119,9 @@ function createLingoDotDevLocalizer(explicitApiKey) {
           authenticated: !!response,
           username: response?.email
         };
-      } catch {
-        return { authenticated: false };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return { authenticated: false, error: errorMessage };
       }
     },
     localize: async (input2, onProgress) => {
@@ -8996,6 +9158,7 @@ import { generateText as generateText2 } from "ai";
 import { jsonrepair as jsonrepair3 } from "jsonrepair";
 import { createOllama as createOllama2 } from "ollama-ai-provider";
 function createExplicitLocalizer(provider) {
+  const settings = provider.settings || {};
   switch (provider.id) {
     default:
       throw new Error(
@@ -9019,7 +9182,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "OPENAI_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "anthropic":
       return createAiSdkLocalizer({
@@ -9027,7 +9191,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "ANTHROPIC_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "google":
       return createAiSdkLocalizer({
@@ -9035,7 +9200,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "GOOGLE_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "openrouter":
       return createAiSdkLocalizer({
@@ -9043,14 +9209,16 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "OPENROUTER_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "ollama":
       return createAiSdkLocalizer({
         factory: (_params) => createOllama2().languageModel(provider.model),
         id: provider.id,
         prompt: provider.prompt,
-        skipAuth: true
+        skipAuth: true,
+        settings
       });
     case "mistral":
       return createAiSdkLocalizer({
@@ -9058,7 +9226,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "MISTRAL_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
   }
 }
@@ -9092,9 +9261,13 @@ function createAiSdkLocalizer(params) {
   return {
     id: params.id,
     checkAuth: async () => {
+      return { authenticated: true, username: "anonymous" };
+    },
+    validateSettings: async () => {
       try {
         await generateText2({
           model,
+          ...params.settings,
           messages: [
             { role: "system", content: "You are an echo server" },
             { role: "user", content: "OK" },
@@ -9102,9 +9275,10 @@ function createAiSdkLocalizer(params) {
             { role: "user", content: "OK" }
           ]
         });
-        return { authenticated: true, username: "anonymous" };
+        return { valid: true };
       } catch (error) {
-        return { authenticated: false };
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return { valid: false, error: errorMessage };
       }
     },
     localize: async (input2) => {
@@ -9134,6 +9308,7 @@ function createAiSdkLocalizer(params) {
       };
       const response = await generateText2({
         model,
+        ...params.settings,
         messages: [
           { role: "system", content: systemPrompt },
           { role: "user", content: "OK" },
@@ -9147,6 +9322,9 @@ function createAiSdkLocalizer(params) {
         ]
       });
       const result = JSON.parse(response.text);
+      if (typeof result.data === "object" && result.data !== null) {
+        return result.data;
+      }
       const index = result.data.indexOf("{");
       const lastIndex = result.data.lastIndexOf("}");
       const trimmed = result.data.slice(index, lastIndex + 1);
@@ -9216,20 +9394,30 @@ async function setup(input2) {
       },
       {
         title: "Checking authentication",
+        enabled: (ctx) => ctx.localizer?.id === "Lingo.dev",
         task: async (ctx, task) => {
           const authStatus = await ctx.localizer.checkAuth();
           if (!authStatus.authenticated) {
-            throw new Error(
-              `Failed to authenticate with ${chalk10.hex(colors.yellow)(
-                ctx.localizer.id
-              )} provider. Please check your API key and try again.`
-            );
+            throw new Error(authStatus.error || "Authentication failed");
           }
           task.title = `Authenticated as ${chalk10.hex(colors.yellow)(
             authStatus.username
           )}`;
         }
       },
+      {
+        title: "Validating configuration",
+        enabled: (ctx) => ctx.localizer?.id !== "Lingo.dev",
+        task: async (ctx, task) => {
+          const validationStatus = await ctx.localizer.validateSettings();
+          if (!validationStatus.valid) {
+            throw new Error(
+              validationStatus.error || "Configuration validation failed"
+            );
+          }
+          task.title = `Configuration validated`;
+        }
+      },
       {
         title: "Initializing localization provider",
         async task(ctx, task) {
@@ -9407,12 +9595,21 @@ async function execute(input2) {
         title: `Processing localization tasks ${chalk12.dim(
           `(tasks: ${input2.tasks.length}, concurrency: ${effectiveConcurrency})`
         )}`,
-        task: (ctx, task) => {
+        task: async (ctx, task) => {
           if (input2.tasks.length < 1) {
             task.title = `Skipping, nothing to localize.`;
             task.skip();
             return;
           }
+          const initialChecksumsMap = /* @__PURE__ */ new Map();
+          const uniqueBucketPatterns = _34.uniq(
+            ctx.tasks.map((t2) => t2.bucketPathPattern)
+          );
+          for (const bucketPathPattern of uniqueBucketPatterns) {
+            const deltaProcessor = createDeltaProcessor(bucketPathPattern);
+            const checksums = await deltaProcessor.loadChecksums();
+            initialChecksumsMap.set(bucketPathPattern, checksums);
+          }
           const i18nLimiter = pLimit(effectiveConcurrency);
           const ioLimiter = pLimit(1);
           const workersCount = effectiveConcurrency;
@@ -9427,6 +9624,7 @@ async function execute(input2) {
                 assignedTasks,
                 ioLimiter,
                 i18nLimiter,
+                initialChecksumsMap,
                 onDone() {
                   task.title = createExecutionProgressMessage(ctx);
                 }
@@ -9506,6 +9704,7 @@ function createWorkerTask(args) {
         const deltaProcessor = createDeltaProcessor(
           assignedTask.bucketPathPattern
         );
+        const initialChecksums = args.initialChecksumsMap.get(assignedTask.bucketPathPattern) || {};
         const taskResult = await args.i18nLimiter(async () => {
           try {
             const sourceData = await bucketLoader.pull(
@@ -9515,11 +9714,10 @@ function createWorkerTask(args) {
             const targetData = await bucketLoader.pull(
               assignedTask.targetLocale
             );
-            const checksums = await deltaProcessor.loadChecksums();
             const delta = await deltaProcessor.calculateDelta({
               sourceData,
               targetData,
-              checksums
+              checksums: initialChecksums
             });
             const processableData = _34.chain(sourceData).entries().filter(
               ([key, value]) => delta.added.includes(key) || delta.updated.includes(key) || !!args.ctx.flags.force
@@ -9532,7 +9730,12 @@ function createWorkerTask(args) {
               await args.ioLimiter(async () => {
                 await bucketLoader.push(assignedTask.targetLocale, targetData);
               });
-              return { status: "skipped" };
+              return {
+                status: "skipped",
+                pathPattern: assignedTask.bucketPathPattern,
+                sourceLocale: assignedTask.sourceLocale,
+                targetLocale: assignedTask.targetLocale
+              };
             }
             const relevantHints = _34.pick(hints, Object.keys(processableData));
             const processedTargetData = await args.ctx.localizer.localize(
@@ -9586,16 +9789,24 @@ function createWorkerTask(args) {
                 assignedTask.targetLocale,
                 finalRenamedTargetData
               );
-              const checksums2 = await deltaProcessor.createChecksums(sourceData);
+              const checksums = await deltaProcessor.createChecksums(sourceData);
               if (!args.ctx.flags.targetLocale?.length) {
-                await deltaProcessor.saveChecksums(checksums2);
+                await deltaProcessor.saveChecksums(checksums);
               }
             });
-            return { status: "success" };
+            return {
+              status: "success",
+              pathPattern: assignedTask.bucketPathPattern,
+              sourceLocale: assignedTask.sourceLocale,
+              targetLocale: assignedTask.targetLocale
+            };
           } catch (error) {
             return {
               status: "error",
-              error
+              error,
+              pathPattern: assignedTask.bucketPathPattern,
+              sourceLocale: assignedTask.sourceLocale,
+              targetLocale: assignedTask.targetLocale
             };
           }
         });
@@ -11273,7 +11484,7 @@ async function renderHero2() {
 // package.json
 var package_default = {
   name: "lingo.dev",
-  version: "0.111.15",
+  version: "0.112.0",
   description: "Lingo.dev CLI",
   private: false,
   publishConfig: {
@@ -11414,6 +11625,7 @@ var package_default = {
     "@lingo.dev/_react": "workspace:*",
     "@lingo.dev/_sdk": "workspace:*",
     "@lingo.dev/_spec": "workspace:*",
+    "@markdoc/markdoc": "^0.5.4",
     "@modelcontextprotocol/sdk": "^1.5.0",
     "@openrouter/ai-sdk-provider": "^0.7.1",
     "@paralleldrive/cuid2": "^2.2.2",