lingo.dev 0.79.4 → 0.80.0

package/build/cli.mjs

@@ -158,7 +158,8 @@ function createAuthenticator(params) {
             return null;
           }
           return {
-            email: payload.email
+            email: payload.email,
+            id: payload.id
           };
         }
         return null;
@@ -988,7 +989,6 @@ var show_default = new Command5().command("show").description("Prints out the cu
 
 // src/cli/cmd/i18n.ts
 import { bucketTypeSchema, localeCodeSchema, resolveOverriddenLocale as resolveOverriddenLocale3 } from "@lingo.dev/_spec";
-import { LingoDotDevEngine } from "@lingo.dev/_sdk";
 import { Command as Command6 } from "interactive-commander";
 import Z4 from "zod";
 import _20 from "lodash";
@@ -2742,7 +2742,8 @@ function variableExtractLoader(params) {
   return createLoader({
     pull: async (locale, input2) => {
       const result = {};
-      for (const [key, value] of Object.entries(input2)) {
+      const inputValues = _16.omitBy(input2, _16.isEmpty);
+      for (const [key, value] of Object.entries(inputValues)) {
         const matches = value.match(specifierPattern) || [];
         result[key] = result[key] || {
           value,
@@ -2993,12 +2994,13 @@ function createInjectLocaleLoader(injectLocaleKeys) {
       if (!injectLocaleKeys) {
         return data;
       }
+      const mergedData = _18.merge({}, originalInput, data);
       injectLocaleKeys.forEach((key) => {
-        if (_18.get(originalInput, key) === originalLocale) {
-          _18.set(data, key, locale);
+        if (_18.get(mergedData, key) === originalLocale) {
+          _18.set(mergedData, key, locale);
         }
       });
-      return data;
+      return mergedData;
     }
   });
 }
@@ -3056,8 +3058,8 @@ function createBucketLoader(bucketType, bucketPathPattern, options) {
         createPoLoader(),
         createFlatLoader(),
         createSyncLoader(),
-        createUnlocalizableLoader(options.isCacheRestore, options.returnUnlocalizedKeys),
-        createVariableLoader({ type: "python" })
+        createVariableLoader({ type: "python" }),
+        createUnlocalizableLoader(options.isCacheRestore, options.returnUnlocalizedKeys)
       );
     case "properties":
       return composeLoaders(
@@ -3089,8 +3091,8 @@ function createBucketLoader(bucketType, bucketPathPattern, options) {
         createXcodeXcstringsLoader(options.defaultLocale),
         createFlatLoader(),
         createSyncLoader(),
-        createUnlocalizableLoader(options.isCacheRestore, options.returnUnlocalizedKeys),
-        createVariableLoader({ type: "ieee" })
+        createVariableLoader({ type: "ieee" }),
+        createUnlocalizableLoader(options.isCacheRestore, options.returnUnlocalizedKeys)
       );
     case "yaml":
       return composeLoaders(
@@ -3319,6 +3321,152 @@ function _tryParseJSON(line) {
   }
 }
 
+// src/cli/processor/lingo.ts
+import { LingoDotDevEngine } from "@lingo.dev/_sdk";
+function createLingoLocalizer(params) {
+  return async (input2, onProgress) => {
+    const lingo = new LingoDotDevEngine({
+      apiKey: params.apiKey,
+      apiUrl: params.apiUrl
+    });
+    const result = await lingo.localizeObject(
+      input2.processableData,
+      {
+        sourceLocale: input2.sourceLocale,
+        targetLocale: input2.targetLocale,
+        reference: {
+          [input2.sourceLocale]: input2.sourceData,
+          [input2.targetLocale]: input2.targetData
+        }
+      },
+      onProgress
+    );
+    return result;
+  };
+}
+
+// src/cli/processor/openai.ts
+import { generateText } from "ai";
+function createBasicTranslator(model, systemPrompt) {
+  return async (input2, onProgress) => {
+    if (!process.env.OPENAI_API_KEY) {
+      throw new Error("OPENAI_API_KEY is not set");
+    }
+    const response = await generateText({
+      model,
+      messages: [
+        {
+          role: "system",
+          content: JSON.stringify({
+            role: "system",
+            content: systemPrompt.replaceAll("{source}", input2.sourceLocale).replaceAll("{target}", input2.targetLocale)
+          })
+        },
+        {
+          role: "user",
+          content: JSON.stringify({
+            sourceLocale: "en",
+            targetLocale: "es",
+            data: {
+              message: "Hello, world!"
+            }
+          })
+        },
+        {
+          role: "assistant",
+          content: JSON.stringify({
+            sourceLocale: "en",
+            targetLocale: "es",
+            data: {
+              message: "Hola, mundo!"
+            }
+          })
+        },
+        {
+          role: "user",
+          content: JSON.stringify({
+            sourceLocale: "en",
+            targetLocale: "es",
+            data: input2.processableData
+          })
+        }
+      ]
+    });
+    const result = JSON.parse(response.text);
+    return result;
+  };
+}
+
+// src/cli/processor/index.ts
+import { createOpenAI } from "@ai-sdk/openai";
+import { createAnthropic } from "@ai-sdk/anthropic";
+function createProcessor(provider, params) {
+  if (!provider || provider.id === "lingo") {
+    const result = createLingoLocalizer(params);
+    return result;
+  } else {
+    const model = getPureModelProvider(provider);
+    const result = createBasicTranslator(model, provider.prompt);
+    return result;
+  }
+}
+function getPureModelProvider(provider) {
+  switch (provider?.id) {
+    case "openai":
+      if (!process.env.OPENAI_API_KEY) {
+        throw new Error("OPENAI_API_KEY is not set.");
+      }
+      return createOpenAI({
+        apiKey: process.env.OPENAI_API_KEY,
+        baseURL: provider.baseUrl
+      })(provider.model);
+    case "anthropic":
+      if (!process.env.ANTHROPIC_API_KEY) {
+        throw new Error("ANTHROPIC_API_KEY is not set.");
+      }
+      return createAnthropic({
+        apiKey: process.env.ANTHROPIC_API_KEY
+      })(provider.model);
+    default:
+      throw new Error(`Unsupported provider: ${provider?.id}`);
+  }
+}
+
+// src/cli/utils/exp-backoff.ts
+function withExponentialBackoff(fn, maxAttempts = 3, baseDelay = 1e3) {
+  return async (...args) => {
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      try {
+        return await fn(...args);
+      } catch (error) {
+        if (attempt === maxAttempts - 1) throw error;
+        const delay = baseDelay * Math.pow(2, attempt);
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+    throw new Error("Unreachable code");
+  };
+}
+
+// src/cli/utils/observability.ts
+import { PostHog } from "posthog-node";
+async function trackEvent(distinctId, event, properties) {
+  if (process.env.DO_NOT_TRACK) {
+    return;
+  }
+  const posthog = new PostHog("phc_eR0iSoQufBxNY36k0f0T15UvHJdTfHlh8rJcxsfhfXk", {
+    host: "https://eu.i.posthog.com",
+    flushAt: 1,
+    flushInterval: 0
+  });
+  await posthog.capture({
+    distinctId,
+    event,
+    properties
+  });
+  await posthog.shutdown();
+}
+
 // src/cli/cmd/i18n.ts
 var i18n_default = new Command6().command("i18n").description("Run Localization engine").helpOption("-h, --help", "Show help").option("--locale <locale>", "Locale to process", (val, prev) => prev ? [...prev, val] : [val]).option("--bucket <bucket>", "Bucket to process", (val, prev) => prev ? [...prev, val] : [val]).option(
   "--key <key>",
@@ -3343,6 +3491,7 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
     ]);
   }
   let hasErrors = false;
+  let authId = null;
   try {
     ora.start("Loading configuration...");
     const i18nConfig = getConfig();
@@ -3353,7 +3502,12 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
     ora.succeed("Localization configuration is valid");
     ora.start("Connecting to Lingo.dev Localization Engine...");
     const auth = await validateAuth(settings);
+    authId = auth.id;
     ora.succeed(`Authenticated as ${auth.email}`);
+    trackEvent(authId, "cmd.i18n.start", {
+      i18nConfig,
+      flags
+    });
     let buckets = getBuckets(i18nConfig);
     if (flags.bucket?.length) {
       buckets = buckets.filter((bucket) => flags.bucket.includes(bucket.type));
@@ -3537,11 +3691,12 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
               bucketOra.start(
                 `[${sourceLocale} -> ${targetLocale}] [${Object.keys(processableData).length} entries] (0%) AI localization in progress...`
               );
-              const localizationEngine = createLocalizationEngineConnection({
+              let processPayload = createProcessor(i18nConfig.provider, {
                 apiKey: settings.auth.apiKey,
                 apiUrl: settings.auth.apiUrl
               });
-              const processedTargetData = await localizationEngine.process(
+              processPayload = withExponentialBackoff(processPayload, 3, 1e3);
+              const processedTargetData = await processPayload(
                 {
                   sourceLocale,
                   sourceData,
@@ -3615,11 +3770,19 @@ var i18n_default = new Command6().command("i18n").description("Run Localization
       if (flags.verbose) {
         ora.info("Cache file deleted.");
       }
+      trackEvent(auth.id, "cmd.i18n.success", {
+        i18nConfig,
+        flags
+      });
     } else {
       ora.warn("Localization completed with errors.");
     }
   } catch (error) {
     ora.fail(error.message);
+    trackEvent(authId || "unknown", "cmd.i18n.error", {
+      flags,
+      error
+    });
     process.exit(1);
   }
 });
@@ -3629,43 +3792,6 @@ function calculateDataDelta(args) {
   const result = _20.chain(args.sourceData).pickBy((value, key) => newKeys.includes(key) || updatedKeys.includes(key)).value();
   return result;
 }
-async function retryWithExponentialBackoff(operation, maxAttempts, baseDelay = 1e3) {
-  for (let attempt = 0; attempt < maxAttempts; attempt++) {
-    try {
-      return await operation();
-    } catch (error) {
-      if (attempt === maxAttempts - 1) throw error;
-      const delay = baseDelay * Math.pow(2, attempt);
-      await new Promise((resolve) => setTimeout(resolve, delay));
-    }
-  }
-  throw new Error("Unreachable code");
-}
-function createLocalizationEngineConnection(params) {
-  const engine = new LingoDotDevEngine({
-    apiKey: params.apiKey,
-    apiUrl: params.apiUrl
-  });
-  return {
-    process: async (args, onProgress) => {
-      return retryWithExponentialBackoff(
-        () => engine.localizeObject(
-          args.processableData,
-          {
-            sourceLocale: args.sourceLocale,
-            targetLocale: args.targetLocale,
-            reference: {
-              [args.sourceLocale]: args.sourceData,
-              [args.targetLocale]: args.targetData
-            }
-          },
-          onProgress
-        ),
-        params.maxRetries ?? 3
-      );
-    }
-  };
-}
 function parseFlags(options) {
   return Z4.object({
     apiKey: Z4.string().optional(),
@@ -4619,7 +4745,7 @@ var ci_default = new Command10().command("ci").description("Run Lingo.dev CI/CD
 // package.json
 var package_default = {
   name: "lingo.dev",
-  version: "0.79.4",
+  version: "0.80.0",
   description: "Lingo.dev CLI",
   private: false,
   publishConfig: {
@@ -4676,6 +4802,8 @@ var package_default = {
   author: "",
   license: "Apache-2.0",
   dependencies: {
+    "@ai-sdk/anthropic": "^1.2.6",
+    "@ai-sdk/openai": "^1.3.7",
     "@datocms/cma-client-node": "^3.4.0",
     "@gitbeaker/rest": "^39.34.3",
     "@inquirer/prompts": "^7.2.3",
@@ -4683,6 +4811,7 @@ var package_default = {
     "@lingo.dev/_spec": "workspace:*",
     "@modelcontextprotocol/sdk": "^1.5.0",
     "@paralleldrive/cuid2": "^2.2.2",
+    ai: "^4.3.2",
     bitbucket: "^2.12.0",
     chalk: "^5.4.1",
     cors: "^2.8.5",
@@ -4719,6 +4848,7 @@ var package_default = {
     "p-limit": "^6.2.0",
     "php-array-reader": "^2.1.2",
     plist: "^3.1.0",
+    "posthog-node": "^4.11.2",
     prettier: "^3.4.2",
     "properties-parser": "^0.6.0",
     slugify: "^1.6.6",