lingo.dev 0.108.0 → 0.109.0

package/build/cli.mjs

@@ -1525,6 +1525,16 @@ function composeLoaders(...loaders) {
         result = await loaders[i].push(locale, result);
       }
       return result;
+    },
+    pullHints: async (originalInput) => {
+      let result = originalInput;
+      for (let i = 0; i < loaders.length; i++) {
+        const subResult = await loaders[i].pullHints?.(result);
+        if (subResult) {
+          result = subResult;
+        }
+      }
+      return result;
     }
   };
 }
@@ -1551,6 +1561,9 @@ function createLoader(lDefinition) {
       state.defaultLocale = locale;
       return this;
     },
+    async pullHints() {
+      return lDefinition.pullHints?.(state.originalInput);
+    },
     async pull(locale, input2) {
       if (!state.defaultLocale) {
         throw new Error("Default locale not set");
@@ -1613,12 +1626,220 @@ function createJsonLoader() {
   });
 }
 
+// src/cli/loaders/json5.ts
+import JSON5 from "json5";
+function createJson5Loader() {
+  return createLoader({
+    pull: async (locale, input2) => {
+      const json5String = input2 || "{}";
+      return JSON5.parse(json5String);
+    },
+    push: async (locale, data) => {
+      const serializedData = JSON5.stringify(data, null, 2);
+      return serializedData;
+    }
+  });
+}
+
+// src/cli/loaders/jsonc.ts
+import { parse } from "jsonc-parser";
+function extractCommentsFromJsonc(jsoncString) {
+  const lines = jsoncString.split("\n");
+  const comments = {};
+  const errors = [];
+  const result = parse(jsoncString, errors, {
+    allowTrailingComma: true,
+    disallowComments: false,
+    allowEmptyContent: true
+  });
+  if (errors.length > 0) {
+    return {};
+  }
+  const contextStack = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const trimmedLine = line.trim();
+    if (!trimmedLine) continue;
+    const commentData = extractCommentFromLine(line, lines, i);
+    if (commentData.hint) {
+      let keyInfo;
+      if (commentData.isInline) {
+        const keyMatch = line.match(/^\s*["']?([^"':,\s]+)["']?\s*:/);
+        if (keyMatch) {
+          const key = keyMatch[1];
+          const path17 = contextStack.map((ctx) => ctx.key).filter(Boolean);
+          keyInfo = { key, path: path17 };
+        }
+      } else {
+        keyInfo = findAssociatedKey(lines, commentData.lineIndex, contextStack);
+      }
+      if (keyInfo && keyInfo.key) {
+        setCommentAtPath(comments, keyInfo.path, keyInfo.key, commentData.hint);
+      }
+      i = commentData.endIndex;
+      continue;
+    }
+    updateContext(contextStack, line, result);
+  }
+  return comments;
+}
+function extractCommentFromLine(line, lines, lineIndex) {
+  const trimmed = line.trim();
+  if (trimmed.startsWith("//")) {
+    const hint = trimmed.replace(/^\/\/\s*/, "").trim();
+    return { hint, lineIndex, endIndex: lineIndex, isInline: false };
+  }
+  if (trimmed.startsWith("/*")) {
+    const blockResult = extractBlockComment(lines, lineIndex);
+    return { ...blockResult, isInline: false };
+  }
+  const singleInlineMatch = line.match(/^(.+?)\s*\/\/\s*(.+)$/);
+  if (singleInlineMatch && singleInlineMatch[1].includes(":")) {
+    const hint = singleInlineMatch[2].trim();
+    return { hint, lineIndex, endIndex: lineIndex, isInline: true };
+  }
+  const blockInlineMatch = line.match(/^(.+?)\s*\/\*\s*(.*?)\s*\*\/.*$/);
+  if (blockInlineMatch && blockInlineMatch[1].includes(":")) {
+    const hint = blockInlineMatch[2].trim();
+    return { hint, lineIndex, endIndex: lineIndex, isInline: true };
+  }
+  return { hint: null, lineIndex, endIndex: lineIndex, isInline: false };
+}
+function extractBlockComment(lines, startIndex) {
+  const startLine = lines[startIndex];
+  const singleMatch = startLine.match(/\/\*\s*(.*?)\s*\*\//);
+  if (singleMatch) {
+    return {
+      hint: singleMatch[1].trim(),
+      lineIndex: startIndex,
+      endIndex: startIndex
+    };
+  }
+  const commentParts = [];
+  let endIndex = startIndex;
+  const firstContent = startLine.replace(/.*?\/\*\s*/, "").trim();
+  if (firstContent && !firstContent.includes("*/")) {
+    commentParts.push(firstContent);
+  }
+  for (let i = startIndex + 1; i < lines.length; i++) {
+    const line = lines[i];
+    endIndex = i;
+    if (line.includes("*/")) {
+      const lastContent = line.replace(/\*\/.*$/, "").replace(/^\s*\*?\s*/, "").trim();
+      if (lastContent) {
+        commentParts.push(lastContent);
+      }
+      break;
+    } else {
+      const content = line.replace(/^\s*\*?\s*/, "").trim();
+      if (content) {
+        commentParts.push(content);
+      }
+    }
+  }
+  return {
+    hint: commentParts.join(" ").trim() || null,
+    lineIndex: startIndex,
+    endIndex
+  };
+}
+function findAssociatedKey(lines, commentLineIndex, contextStack) {
+  for (let i = commentLineIndex + 1; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line || line.startsWith("//") || line.startsWith("/*") || line === "{" || line === "}") {
+      continue;
+    }
+    const keyMatch = line.match(/^\s*["']?([^"':,\s]+)["']?\s*:/);
+    if (keyMatch) {
+      const key = keyMatch[1];
+      const path17 = contextStack.map((ctx) => ctx.key).filter(Boolean);
+      return { key, path: path17 };
+    }
+  }
+  return { key: null, path: [] };
+}
+function updateContext(contextStack, line, parsedJson) {
+  const openBraces = (line.match(/\{/g) || []).length;
+  const closeBraces = (line.match(/\}/g) || []).length;
+  if (openBraces > closeBraces) {
+    const keyMatch = line.match(/^\s*["']?([^"':,\s]+)["']?\s*:\s*\{/);
+    if (keyMatch) {
+      contextStack.push({ key: keyMatch[1], isArray: false });
+    }
+  } else if (closeBraces > openBraces) {
+    for (let i = 0; i < closeBraces - openBraces; i++) {
+      contextStack.pop();
+    }
+  }
+}
+function setCommentAtPath(comments, path17, key, hint) {
+  let current = comments;
+  for (const pathKey of path17) {
+    if (!current[pathKey]) {
+      current[pathKey] = {};
+    }
+    current = current[pathKey];
+  }
+  if (!current[key]) {
+    current[key] = {};
+  }
+  if (typeof current[key] === "object" && current[key] !== null) {
+    current[key].hint = hint;
+  } else {
+    current[key] = { hint };
+  }
+}
+function createJsoncLoader() {
+  return createLoader({
+    pull: async (locale, input2) => {
+      const jsoncString = input2 || "{}";
+      const errors = [];
+      const result = parse(jsoncString, errors, {
+        allowTrailingComma: true,
+        disallowComments: false,
+        allowEmptyContent: true
+      });
+      if (errors.length > 0) {
+        throw new Error(`Failed to parse JSONC: ${errors[0].error}`);
+      }
+      return result || {};
+    },
+    push: async (locale, data) => {
+      const serializedData = JSON.stringify(data, null, 2);
+      return serializedData;
+    },
+    pullHints: async (input2) => {
+      if (!input2 || typeof input2 !== "string") {
+        return {};
+      }
+      try {
+        return extractCommentsFromJsonc(input2);
+      } catch (error) {
+        console.warn("Failed to extract comments from JSONC:", error);
+        return {};
+      }
+    }
+  });
+}
+
 // src/cli/loaders/flat.ts
 import { flatten, unflatten } from "flat";
 import _9 from "lodash";
 var OBJECT_NUMERIC_KEY_PREFIX = "__lingodotdev__obj__";
 function createFlatLoader() {
-  return composeLoaders(createDenormalizeLoader(), createNormalizeLoader());
+  const composedLoader = composeLoaders(
+    createDenormalizeLoader(),
+    createNormalizeLoader()
+  );
+  return {
+    ...composedLoader,
+    pullHints: async (input2) => {
+      if (!input2 || typeof input2 !== "object") {
+        return {};
+      }
+      return flattenHints(input2);
+    }
+  };
 }
 function createDenormalizeLoader() {
   return createLoader({
@@ -1709,6 +1930,33 @@ function normalizeObjectKeys(obj) {
     return obj;
   }
 }
+function flattenHints(obj, parentHints = [], parentPath = "") {
+  const result = {};
+  for (const [key, _value] of Object.entries(obj)) {
+    if (_9.isObject(_value) && !_9.isArray(_value)) {
+      const value = _value;
+      const currentHints = [...parentHints];
+      const currentPath = parentPath ? `${parentPath}/${key}` : key;
+      if (value.hint && typeof value.hint === "string") {
+        currentHints.push(value.hint);
+      }
+      const nestedObj = _9.omit(value, "hint");
+      if (Object.keys(nestedObj).length === 0) {
+        if (currentHints.length > 0) {
+          result[currentPath] = currentHints;
+        }
+      } else {
+        const nestedComments = flattenHints(
+          nestedObj,
+          currentHints,
+          currentPath
+        );
+        Object.assign(result, nestedComments);
+      }
+    }
+  }
+  return result;
+}
 
 // src/cli/loaders/text-file.ts
 import fs8 from "fs/promises";
@@ -2020,11 +2268,11 @@ function createAndroidLoader() {
 }
 
 // src/cli/loaders/csv.ts
-import { parse } from "csv-parse/sync";
+import { parse as parse2 } from "csv-parse/sync";
 import { stringify } from "csv-stringify/sync";
 import _11 from "lodash";
 function detectKeyColumnName(csvString) {
-  const row = parse(csvString)[0];
+  const row = parse2(csvString)[0];
   const firstColumn = row?.[0]?.trim();
   return firstColumn || "KEY";
 }
@@ -2037,7 +2285,7 @@ function _createCsvLoader() {
       const keyColumnName = detectKeyColumnName(
         input2.split("\n").find((l) => l.length)
       );
-      const inputParsed = parse(input2, {
+      const inputParsed = parse2(input2, {
         columns: true,
         skip_empty_lines: true,
         relax_column_count_less: true
@@ -2474,6 +2722,36 @@ function createXcodeXcstringsLoader(defaultLocale) {
       const originalInputWithoutLocale = originalInput ? _removeLocale(originalInput, locale) : {};
       const result = _12.merge({}, originalInputWithoutLocale, langDataToMerge);
       return result;
+    },
+    async pullHints(originalInput) {
+      if (!originalInput || !originalInput.strings) {
+        return {};
+      }
+      const hints = {};
+      for (const [translationKey, translationEntity] of Object.entries(
+        originalInput.strings
+      )) {
+        const entity = translationEntity;
+        if (entity.comment && typeof entity.comment === "string") {
+          hints[translationKey] = { hint: entity.comment };
+        }
+        if (entity.localizations) {
+          for (const [locale, localization] of Object.entries(
+            entity.localizations
+          )) {
+            if (localization.variations?.plural) {
+              const pluralForms = localization.variations.plural;
+              for (const form in pluralForms) {
+                const pluralKey = `${translationKey}/${form}`;
+                if (entity.comment && typeof entity.comment === "string") {
+                  hints[pluralKey] = { hint: entity.comment };
+                }
+              }
+            }
+          }
+        }
+      }
+      return hints;
     }
   });
 }
@@ -3238,7 +3516,7 @@ function createSrtLoader() {
 
 // src/cli/loaders/dato/index.ts
 import fs9 from "fs";
-import JSON5 from "json5";
+import JSON52 from "json5";
 
 // src/cli/loaders/dato/_base.ts
 import Z2 from "zod";
@@ -3925,13 +4203,13 @@ function _isVideo(rawDatoValue) {
 function createDatoLoader(configFilePath) {
   try {
     const configContent = fs9.readFileSync(configFilePath, "utf-8");
-    const datoConfig = datoConfigSchema.parse(JSON5.parse(configContent));
+    const datoConfig = datoConfigSchema.parse(JSON52.parse(configContent));
     return composeLoaders(
       createDatoApiLoader(
         datoConfig,
         (updatedConfig) => fs9.writeFileSync(
           configFilePath,
-          JSON5.stringify(updatedConfig, null, 2)
+          JSON52.stringify(updatedConfig, null, 2)
         )
       ),
       createDatoFilterLoader(),
@@ -4256,7 +4534,7 @@ function parseVueFile(input2) {
 }
 
 // src/cli/loaders/typescript/index.ts
-import { parse as parse2 } from "@babel/parser";
+import { parse as parse3 } from "@babel/parser";
 import _21 from "lodash";
 import babelTraverseModule from "@babel/traverse";
 import * as t from "@babel/types";
@@ -4305,7 +4583,7 @@ function createTypescriptLoader() {
   });
 }
 function parseTypeScript(input2) {
-  return parse2(input2, {
+  return parse3(input2, {
     sourceType: "module",
     plugins: ["typescript"]
   });
@@ -6702,6 +6980,28 @@ function createBucketLoader(bucketType, bucketPathPattern, options, lockedKeys,
         createSyncLoader(),
         createUnlocalizableLoader(options.returnUnlocalizedKeys)
       );
+    case "json5":
+      return composeLoaders(
+        createTextFileLoader(bucketPathPattern),
+        createJson5Loader(),
+        createEnsureKeyOrderLoader(),
+        createFlatLoader(),
+        createInjectLocaleLoader(options.injectLocale),
+        createLockedKeysLoader(lockedKeys || []),
+        createSyncLoader(),
+        createUnlocalizableLoader(options.returnUnlocalizedKeys)
+      );
+    case "jsonc":
+      return composeLoaders(
+        createTextFileLoader(bucketPathPattern),
+        createJsoncLoader(),
+        createEnsureKeyOrderLoader(),
+        createFlatLoader(),
+        createInjectLocaleLoader(options.injectLocale),
+        createLockedKeysLoader(lockedKeys || []),
+        createSyncLoader(),
+        createUnlocalizableLoader(options.returnUnlocalizedKeys)
+      );
     case "markdown":
       return composeLoaders(
         createTextFileLoader(bucketPathPattern),
@@ -8292,7 +8592,8 @@ function createLingoDotDevLocalizer(explicitApiKey) {
           reference: {
             [input2.sourceLocale]: input2.sourceData,
             [input2.targetLocale]: input2.targetData
-          }
+          },
+          hints: input2.hints
         },
         onProgress
       );
@@ -8826,6 +9127,7 @@ function createWorkerTask(args) {
             const sourceData = await bucketLoader.pull(
               assignedTask.sourceLocale
             );
+            const hints = await bucketLoader.pullHints();
             const targetData = await bucketLoader.pull(
               assignedTask.targetLocale
             );
@@ -8848,13 +9150,15 @@ function createWorkerTask(args) {
               });
               return { status: "skipped" };
             }
+            const relevantHints = _34.pick(hints, Object.keys(processableData));
             const processedTargetData = await args.ctx.localizer.localize(
               {
                 sourceLocale: assignedTask.sourceLocale,
                 targetLocale: assignedTask.targetLocale,
                 sourceData,
                 targetData,
-                processableData
+                processableData,
+                hints: relevantHints
               },
               async (progress, _sourceChunk, processedChunk) => {
                 await args.ioLimiter(async () => {
@@ -10496,7 +10800,7 @@ async function renderHero2() {
 // package.json
 var package_default = {
   name: "lingo.dev",
-  version: "0.108.0",
+  version: "0.109.0",
   description: "Lingo.dev CLI",
   private: false,
   publishConfig: {
@@ -10660,6 +10964,7 @@ var package_default = {
     "is-url": "^1.2.4",
     jsdom: "^25.0.1",
     json5: "^2.2.3",
+    "jsonc-parser": "^3.3.1",
     jsonrepair: "^3.11.2",
     listr2: "^8.3.2",
     lodash: "^4.17.21",