lingo.dev 0.111.16 → 0.112.0

package/build/cli.mjs

@@ -2713,6 +2713,133 @@ ${content}`;
   });
 }
 
+// src/cli/loaders/markdoc.ts
+import Markdoc from "@markdoc/markdoc";
+import YAML3 from "yaml";
+var FM_ATTR_PREFIX2 = "fm-attr-";
+function createMarkdocLoader() {
+  return createLoader({
+    async pull(locale, input2) {
+      const ast = Markdoc.parse(input2);
+      const result = {};
+      const counters = {};
+      traverseAndExtract(ast, "", result, counters);
+      if (ast.attributes?.frontmatter) {
+        const frontmatter = YAML3.parse(ast.attributes.frontmatter);
+        Object.entries(frontmatter).forEach(([key, value]) => {
+          if (typeof value === "string") {
+            result[`${FM_ATTR_PREFIX2}${key}`] = value;
+          }
+        });
+      }
+      return result;
+    },
+    async push(locale, data, originalInput) {
+      if (!originalInput) {
+        throw new Error("Original input is required for push");
+      }
+      const ast = Markdoc.parse(originalInput);
+      const counters = {};
+      const pathMap = {};
+      buildPathMap(ast, "", counters, pathMap);
+      const frontmatterEntries = Object.entries(data).filter(([key]) => key.startsWith(FM_ATTR_PREFIX2)).map(([key, value]) => [key.replace(FM_ATTR_PREFIX2, ""), value]);
+      if (frontmatterEntries.length > 0 && ast.attributes) {
+        const frontmatter = Object.fromEntries(frontmatterEntries);
+        ast.attributes.frontmatter = YAML3.stringify(frontmatter, {
+          defaultStringType: "PLAIN"
+        }).trim();
+      }
+      const contentData = Object.fromEntries(
+        Object.entries(data).filter(([key]) => !key.startsWith(FM_ATTR_PREFIX2))
+      );
+      applyTranslations(ast, "", contentData, pathMap);
+      return Markdoc.format(ast);
+    }
+  });
+}
+function getSemanticNodeType(node) {
+  if (node.type === "tag") return node.tag || "tag";
+  return node.type;
+}
+function traverseAndExtract(node, path19, result, counters, parentType) {
+  if (!node || typeof node !== "object") {
+    return;
+  }
+  let semanticType = parentType;
+  const nodeSemanticType = getSemanticNodeType(node);
+  if (nodeSemanticType && !["text", "strong", "em", "inline", "link"].includes(nodeSemanticType)) {
+    semanticType = nodeSemanticType;
+  }
+  if (node.type === "text" && node.attributes?.content) {
+    const content = node.attributes.content;
+    if (typeof content === "string" && content.trim()) {
+      if (semanticType) {
+        const index = counters[semanticType] || 0;
+        counters[semanticType] = index + 1;
+        const semanticKey = `${semanticType}-${index}`;
+        result[semanticKey] = content;
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    node.children.forEach((child, index) => {
+      const childPath = path19 ? `${path19}/children/${index}` : `children/${index}`;
+      traverseAndExtract(child, childPath, result, counters, semanticType);
+    });
+  }
+}
+function buildPathMap(node, path19, counters, pathMap, parentType) {
+  if (!node || typeof node !== "object") {
+    return;
+  }
+  let semanticType = parentType;
+  const nodeSemanticType = getSemanticNodeType(node);
+  if (nodeSemanticType && !["text", "strong", "em", "inline", "link"].includes(nodeSemanticType)) {
+    semanticType = nodeSemanticType;
+  }
+  if (node.type === "text" && node.attributes?.content) {
+    const content = node.attributes.content;
+    if (typeof content === "string" && content.trim()) {
+      if (semanticType) {
+        const index = counters[semanticType] || 0;
+        counters[semanticType] = index + 1;
+        const semanticKey = `${semanticType}-${index}`;
+        const contentPath = path19 ? `${path19}/attributes/content` : "attributes/content";
+        pathMap[semanticKey] = contentPath;
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    node.children.forEach((child, index) => {
+      const childPath = path19 ? `${path19}/children/${index}` : `children/${index}`;
+      buildPathMap(child, childPath, counters, pathMap, semanticType);
+    });
+  }
+}
+function applyTranslations(node, path19, data, pathMap) {
+  if (!node || typeof node !== "object") {
+    return;
+  }
+  if (node.type === "text" && node.attributes?.content) {
+    const content = node.attributes.content;
+    if (typeof content === "string") {
+      const contentPath = path19 ? `${path19}/attributes/content` : "attributes/content";
+      const semanticKey = Object.keys(pathMap).find(
+        (key) => pathMap[key] === contentPath
+      );
+      if (semanticKey && data[semanticKey] !== void 0) {
+        node.attributes.content = data[semanticKey];
+      }
+    }
+  }
+  if (Array.isArray(node.children)) {
+    node.children.forEach((child, index) => {
+      const childPath = path19 ? `${path19}/children/${index}` : `children/${index}`;
+      applyTranslations(child, childPath, data, pathMap);
+    });
+  }
+}
+
 // src/cli/loaders/properties.ts
 function createPropertiesLoader() {
   return createLoader({
@@ -6656,7 +6783,7 @@ function _isLockedKey(key, lockedKeys) {
 
 // src/cli/loaders/mdx2/frontmatter-split.ts
 import matter2 from "gray-matter";
-import YAML3 from "yaml";
+import YAML4 from "yaml";
 function createMdxFrontmatterSplitLoader() {
   const fmEngine = createFmEngine();
   return createLoader({
@@ -6677,8 +6804,8 @@ function createMdxFrontmatterSplitLoader() {
 }
 function createFmEngine() {
   const yamlEngine2 = {
-    parse: (str) => YAML3.parse(str),
-    stringify: (obj) => YAML3.stringify(obj, { defaultStringType: "PLAIN" })
+    parse: (str) => YAML4.parse(str),
+    stringify: (obj) => YAML4.stringify(obj, { defaultStringType: "PLAIN" })
   };
   return {
     parse: (input2) => matter2(input2, {
@@ -7286,6 +7413,15 @@ function createBucketLoader(bucketType, bucketPathPattern, options, lockedKeys,
         createSyncLoader(),
         createUnlocalizableLoader(options.returnUnlocalizedKeys)
       );
+    case "markdoc":
+      return composeLoaders(
+        createTextFileLoader(bucketPathPattern),
+        createMarkdocLoader(),
+        createFlatLoader(),
+        createEnsureKeyOrderLoader(),
+        createSyncLoader(),
+        createUnlocalizableLoader(options.returnUnlocalizedKeys)
+      );
     case "mdx":
       return composeLoaders(
         createTextFileLoader(bucketPathPattern),
@@ -7517,7 +7653,7 @@ function createLingoLocalizer(params) {
 // src/cli/processor/basic.ts
 import { generateText } from "ai";
 import _29 from "lodash";
-function createBasicTranslator(model, systemPrompt) {
+function createBasicTranslator(model, systemPrompt, settings = {}) {
   return async (input2, onProgress) => {
     const chunks = extractPayloadChunks(input2.processableData);
     const subResults = [];
@@ -7539,6 +7675,7 @@ function createBasicTranslator(model, systemPrompt) {
     }
     const response = await generateText({
       model,
+      ...settings,
       messages: [
         {
           role: "system",
@@ -7629,7 +7766,8 @@ function createProcessor(provider, params) {
     return result;
   } else {
     const model = getPureModelProvider(provider);
-    const result = createBasicTranslator(model, provider.prompt);
+    const settings = provider.settings || {};
+    const result = createBasicTranslator(model, provider.prompt, settings);
     return result;
   }
 }
@@ -7832,7 +7970,7 @@ function checkIfFileExists(filePath) {
 
 // src/cli/utils/delta.ts
 import * as path15 from "path";
-import YAML4 from "yaml";
+import YAML5 from "yaml";
 var LockSchema = z.object({
   version: z.literal(1).default(1),
   checksums: z.record(
@@ -7897,7 +8035,7 @@ function createDeltaProcessor(fileKey) {
     },
     async loadLock() {
       const lockfileContent = tryReadFile(lockfilePath, null);
-      const lockfileYaml = lockfileContent ? YAML4.parse(lockfileContent) : null;
+      const lockfileYaml = lockfileContent ? YAML5.parse(lockfileContent) : null;
       const lockfileData = lockfileYaml ? LockSchema.parse(lockfileYaml) : {
         version: 1,
         checksums: {}
@@ -7905,7 +8043,7 @@ function createDeltaProcessor(fileKey) {
       return lockfileData;
     },
     async saveLock(lockData) {
-      const lockfileYaml = YAML4.stringify(lockData);
+      const lockfileYaml = YAML5.stringify(lockData);
       writeFile(lockfilePath, lockfileYaml);
     },
     async loadChecksums() {
@@ -8587,7 +8725,7 @@ import Ora8 from "ora";
 import fs12 from "fs";
 import path16 from "path";
 import Z4 from "zod";
-import YAML5 from "yaml";
+import YAML6 from "yaml";
 import { MD5 as MD52 } from "object-hash";
 import _32 from "lodash";
 function createLockfileHelper() {
@@ -8635,12 +8773,12 @@ function createLockfileHelper() {
       return LockfileSchema.parse({});
     }
     const content = fs12.readFileSync(lockfilePath, "utf-8");
-    const result = LockfileSchema.parse(YAML5.parse(content));
+    const result = LockfileSchema.parse(YAML6.parse(content));
     return result;
   }
   function _saveLockfile(lockfile) {
     const lockfilePath = _getLockfilePath();
-    const content = YAML5.stringify(lockfile);
+    const content = YAML6.stringify(lockfile);
     fs12.writeFileSync(lockfilePath, content);
   }
   function _getLockfilePath() {
@@ -8981,8 +9119,9 @@ function createLingoDotDevLocalizer(explicitApiKey) {
           authenticated: !!response,
           username: response?.email
         };
-      } catch {
-        return { authenticated: false };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return { authenticated: false, error: errorMessage };
       }
     },
     localize: async (input2, onProgress) => {
@@ -9019,6 +9158,7 @@ import { generateText as generateText2 } from "ai";
 import { jsonrepair as jsonrepair3 } from "jsonrepair";
 import { createOllama as createOllama2 } from "ollama-ai-provider";
 function createExplicitLocalizer(provider) {
+  const settings = provider.settings || {};
   switch (provider.id) {
     default:
       throw new Error(
@@ -9042,7 +9182,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "OPENAI_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "anthropic":
       return createAiSdkLocalizer({
@@ -9050,7 +9191,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "ANTHROPIC_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "google":
       return createAiSdkLocalizer({
@@ -9058,7 +9200,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "GOOGLE_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "openrouter":
       return createAiSdkLocalizer({
@@ -9066,14 +9209,16 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "OPENROUTER_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
     case "ollama":
       return createAiSdkLocalizer({
         factory: (_params) => createOllama2().languageModel(provider.model),
         id: provider.id,
         prompt: provider.prompt,
-        skipAuth: true
+        skipAuth: true,
+        settings
       });
     case "mistral":
       return createAiSdkLocalizer({
@@ -9081,7 +9226,8 @@ function createExplicitLocalizer(provider) {
         id: provider.id,
         prompt: provider.prompt,
         apiKeyName: "MISTRAL_API_KEY",
-        baseUrl: provider.baseUrl
+        baseUrl: provider.baseUrl,
+        settings
       });
   }
 }
@@ -9115,9 +9261,13 @@ function createAiSdkLocalizer(params) {
   return {
     id: params.id,
     checkAuth: async () => {
+      return { authenticated: true, username: "anonymous" };
+    },
+    validateSettings: async () => {
       try {
         await generateText2({
           model,
+          ...params.settings,
           messages: [
             { role: "system", content: "You are an echo server" },
             { role: "user", content: "OK" },
@@ -9125,9 +9275,10 @@ function createAiSdkLocalizer(params) {
             { role: "user", content: "OK" }
           ]
         });
-        return { authenticated: true, username: "anonymous" };
+        return { valid: true };
       } catch (error) {
-        return { authenticated: false };
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        return { valid: false, error: errorMessage };
       }
     },
     localize: async (input2) => {
@@ -9157,6 +9308,7 @@ function createAiSdkLocalizer(params) {
       };
       const response = await generateText2({
         model,
+        ...params.settings,
         messages: [
           { role: "system", content: systemPrompt },
           { role: "user", content: "OK" },
@@ -9170,6 +9322,9 @@ function createAiSdkLocalizer(params) {
         ]
       });
       const result = JSON.parse(response.text);
+      if (typeof result.data === "object" && result.data !== null) {
+        return result.data;
+      }
       const index = result.data.indexOf("{");
       const lastIndex = result.data.lastIndexOf("}");
       const trimmed = result.data.slice(index, lastIndex + 1);
@@ -9239,20 +9394,30 @@ async function setup(input2) {
       },
       {
         title: "Checking authentication",
+        enabled: (ctx) => ctx.localizer?.id === "Lingo.dev",
         task: async (ctx, task) => {
           const authStatus = await ctx.localizer.checkAuth();
           if (!authStatus.authenticated) {
-            throw new Error(
-              `Failed to authenticate with ${chalk10.hex(colors.yellow)(
-                ctx.localizer.id
-              )} provider. Please check your API key and try again.`
-            );
+            throw new Error(authStatus.error || "Authentication failed");
           }
           task.title = `Authenticated as ${chalk10.hex(colors.yellow)(
             authStatus.username
           )}`;
         }
       },
+      {
+        title: "Validating configuration",
+        enabled: (ctx) => ctx.localizer?.id !== "Lingo.dev",
+        task: async (ctx, task) => {
+          const validationStatus = await ctx.localizer.validateSettings();
+          if (!validationStatus.valid) {
+            throw new Error(
+              validationStatus.error || "Configuration validation failed"
+            );
+          }
+          task.title = `Configuration validated`;
+        }
+      },
       {
         title: "Initializing localization provider",
         async task(ctx, task) {
@@ -11319,7 +11484,7 @@ async function renderHero2() {
 // package.json
 var package_default = {
   name: "lingo.dev",
-  version: "0.111.16",
+  version: "0.112.0",
   description: "Lingo.dev CLI",
   private: false,
   publishConfig: {
@@ -11460,6 +11625,7 @@ var package_default = {
     "@lingo.dev/_react": "workspace:*",
     "@lingo.dev/_sdk": "workspace:*",
     "@lingo.dev/_spec": "workspace:*",
+    "@markdoc/markdoc": "^0.5.4",
     "@modelcontextprotocol/sdk": "^1.5.0",
     "@openrouter/ai-sdk-provider": "^0.7.1",
     "@paralleldrive/cuid2": "^2.2.2",