@verbatra/sdk 0.4.3 → 0.4.4

package/README.md

@@ -86,7 +86,7 @@ Discovers and validates the configuration. With no arguments it searches upward
 
 ### `translate(input): Promise<RunSummary>`
 
-Runs the one-shot read, diff, translate, write flow over every target locale. `input` is `{ config, cwd?, dryRun?, prune?, generatePlurals? }`. With `dryRun: true` it reads, diffs, and reports without calling the provider or writing anything. `prune` and `generatePlurals` each override the matching config option for this run. Resolves to a `RunSummary` (`dryRun`, `locales`, `succeeded`, and `failed`); each locale summary lists `translated`, `generated`, `pruned`, `integrityMismatches`, and `notices`.
+Runs the one-shot read, diff, translate, write flow over every target locale. `input` is `{ config, cwd?, dryRun?, prune?, generatePlurals? }`. With `dryRun: true` it reads, diffs, and reports without calling the provider or writing anything. `prune` and `generatePlurals` each override the matching config option for this run. Resolves to a `RunSummary` (`dryRun`, `locales`, `succeeded`, and `failed`); each locale summary lists `translated`, `generated`, `pruned`, `integrityMismatches`, `providerFailures`, and `notices`. `integrityMismatches` is a translation that came back and failed the placeholder-integrity check; `providerFailures` is a key withheld because the provider call itself failed (nothing was translated), with the secret-free failure code and message reported in `notices`.
 
 ```ts
 const preview = await translate({ config, dryRun: true });

package/dist/index.cjs

@@ -1182,19 +1182,53 @@ function childMessages(element) {
   }
   return [];
 }
-function collect(elements, add, state) {
+function addCount(target, token, count) {
+  target.set(token, (target.get(token) ?? 0) + count);
+}
+function combineBranches(branchMultisets) {
+  const combined = /* @__PURE__ */ new Map();
+  const tokens = /* @__PURE__ */ new Set();
+  for (const branch of branchMultisets) {
+    for (const token of branch.keys()) {
+      tokens.add(token);
+    }
+  }
+  for (const token of tokens) {
+    const minCount = Math.min(...branchMultisets.map((branch) => branch.get(token) ?? 0));
+    if (minCount > 0) {
+      combined.set(token, minCount);
+    }
+  }
+  return combined;
+}
+function collect(elements, state) {
+  const result = /* @__PURE__ */ new Map();
   for (const element of elements) {
     const token = tokenOf(element);
     if (token !== void 0) {
-      add(token);
+      addCount(result, token, 1);
     }
     if (element.type === icuMessageformatParser.TYPE.plural) {
       state.isPlural = true;
     }
-    for (const child of childMessages(element)) {
-      collect(child, add, state);
+    const branches = childMessages(element);
+    if (branches.length > 0) {
+      const branchMultisets = branches.map((branch) => collect(branch, state));
+      for (const [branchToken, branchCount] of combineBranches(branchMultisets)) {
+        addCount(result, branchToken, branchCount);
+      }
+    }
+  }
+  return result;
+}
+function flatten(multiset) {
+  const flattened = [];
+  for (const [token, count] of multiset) {
+    for (let i = 0; i < count; i += 1) {
+      flattened.push(token);
     }
   }
+  return flattened;
 }
 function analyzeIcuValue(value) {
   if (!value.includes("{") && !value.includes("<")) {
@@ -1202,16 +1236,9 @@ function analyzeIcuValue(value) {
   }
   try {
     const ast = icuMessageformatParser.parse(value);
-    const placeholders = [];
     const state = { isPlural: false };
-    collect(
-      ast,
-      (token) => {
-        placeholders.push(token);
-      },
-      state
-    );
-    return { placeholders, isPlural: state.isPlural, valid: true };
+    const multiset = collect(ast, state);
+    return { placeholders: flatten(multiset), isPlural: state.isPlural, valid: true };
   } catch {
     return INVALID;
   }
@@ -1473,21 +1500,28 @@ function addEntries(ctx, prefix, node) {
     }
   }
 }
-function addPathEntries(node, prefix, namespace, derive, out) {
+function addPathEntries(node, prefix, namespace, derive, out, claimedPaths) {
   for (const [key, value] of Object.entries(node)) {
     const path = prefix === "" ? key : `${prefix}.${key}`;
+    if (claimedPaths.has(path)) {
+      throw new AdapterError(
+        "INVALID_STRUCTURE",
+        "A dotted key and a nested key path resolve to the same path."
+      );
+    }
+    claimedPaths.add(path);
     if (typeof value === "string") {
       const { placeholders, isPlural } = derive(key, value);
       out.set(path, { key: path, namespace, value, placeholders, isPlural });
     } else {
-      addPathEntries(value, path, namespace, derive, out);
+      addPathEntries(value, path, namespace, derive, out, claimedPaths);
     }
   }
 }
 function flattenTree(tree, namespace, derive, keyMode = "literal-leaf") {
   const out = /* @__PURE__ */ new Map();
   if (keyMode === "path-notation") {
-    addPathEntries(tree, "", namespace, derive, out);
+    addPathEntries(tree, "", namespace, derive, out, /* @__PURE__ */ new Set());
     return out;
   }
   addEntries({ namespace, derive, out, claimed: /* @__PURE__ */ new Map() }, [], tree);
@@ -1609,21 +1643,20 @@ function messagesFromEntries(entries) {
   }
   return out;
 }
+function isEnoent(error) {
+  return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
+}
 async function readDestinationPairs(filePath) {
-  let parsed;
+  let content;
   try {
-    const outcome = await readBounded2(filePath);
-    if (outcome.kind !== "ok") {
+    content = await readFileContent(filePath);
+  } catch (error) {
+    if (isEnoent(error)) {
       return null;
     }
-    parsed = JSON.parse(outcome.content);
-  } catch {
-    return null;
-  }
-  if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
-    return null;
+    throw error;
   }
-  return Object.entries(parsed);
+  return Object.entries(parseArbObject(content));
 }
 async function buildArbWriteTree(entries, filePath) {
   const messages = messagesFromEntries(entries);
@@ -1724,6 +1757,20 @@ function createNextIntlJsonAdapter() {
     validateMessage: icuIsValid
   });
 }
+function assertNoDottedNestedKey(tree) {
+  for (const [key, value] of Object.entries(tree)) {
+    if (typeof value !== "object") {
+      continue;
+    }
+    if (key.includes(".")) {
+      throw new AdapterError(
+        "MIXED_STRUCTURE",
+        "A nested object key contains a literal dot, which is ambiguous with a dotted path."
+      );
+    }
+    assertNoDottedNestedKey(value);
+  }
+}
 function assertNotMixed(tree) {
   let hasNested = false;
   let hasFlatDottedKey = false;
@@ -1740,6 +1787,7 @@ function assertNotMixed(tree) {
       "The file mixes flat dotted keys with nested objects."
     );
   }
+  assertNoDottedNestedKey(tree);
 }
 async function detectStyle(filePath) {
   let parsed;
@@ -2237,6 +2285,7 @@ function failureSummary(locale, error) {
     pruned: [],
     invalidIcuSource: [],
     integrityMismatches: [],
+    providerFailures: [],
     generated: [],
     notices: [],
     error: describeError(error)
@@ -2248,6 +2297,29 @@ function partition(locales) {
   return { succeeded, failed };
 }
 
+// src/flow/batching.ts
+function chunk(items, size) {
+  const chunks = [];
+  for (let index = 0; index < items.length; index += size) {
+    chunks.push(items.slice(index, index + size));
+  }
+  return chunks;
+}
+var GENERIC_PROVIDER_FAILURE_MESSAGE = "The provider call failed.";
+function classifyProviderFailure(error) {
+  if (error instanceof ProviderError) {
+    return { code: error.code, message: error.message };
+  }
+  return { code: "PROVIDER_CALL_FAILED", message: GENERIC_PROVIDER_FAILURE_MESSAGE };
+}
+function subBatchFailedNotice(count, error) {
+  const { code, message } = classifyProviderFailure(error);
+  return {
+    code: "SUB_BATCH_FAILED",
+    message: `A sub-batch of ${count} entries failed (${code}: ${message}) and was withheld; it will be retried next run.`
+  };
+}
+
 // src/flow/notices.ts
 function isNotice(value) {
   return typeof value === "object" && value !== null && typeof value.code === "string" && typeof value.message === "string";
@@ -2430,13 +2502,36 @@ async function generatePluralForms(context) {
   const plan = planPluralGeneration(context.source, context.targetLocale, context.format);
   const stale = staleItems(plan.items, context.baseline);
   if (stale.length === 0) {
-    return { accepted: [], withheld: [] };
+    return { accepted: [], withheld: [], providerFailures: [], notices: [] };
   }
-  const entries = stale.map(syntheticEntry);
-  const result = await context.provider.translateBatch(buildRequest2(context, entries));
   const accepted = [];
   const withheld = [];
-  for (const item of stale) {
+  const providerFailures = [];
+  const notices = [];
+  for (const batch of chunk(stale, context.maxBatchSize)) {
+    const batchNotices = await runGenerationSubBatch(
+      context,
+      batch,
+      accepted,
+      withheld,
+      providerFailures
+    );
+    notices.push(...batchNotices);
+  }
+  return { accepted, withheld, providerFailures, notices };
+}
+async function runGenerationSubBatch(context, batch, accepted, withheld, providerFailures) {
+  let result;
+  try {
+    const entries = batch.map(syntheticEntry);
+    result = await context.provider.translateBatch(buildRequest2(context, entries));
+  } catch (error) {
+    for (const item of batch) {
+      providerFailures.push(item.targetKey);
+    }
+    return [subBatchFailedNotice(batch.length, error)];
+  }
+  for (const item of batch) {
     const value = result.values.get(item.targetKey);
     const integrity = result.integrity.get(item.targetKey);
     if (value !== void 0 && integrity?.matches === true) {
@@ -2449,7 +2544,7 @@ async function generatePluralForms(context) {
       withheld.push(item.targetKey);
     }
   }
-  return { accepted, withheld };
+  return readNotices(result);
 }
 
 // src/flow/locale-run.ts
@@ -2499,6 +2594,7 @@ async function runLocale(params) {
         translated: toTranslate,
         generated: [],
         integrityMismatches: [],
+        providerFailures: [],
         pruned,
         notices: sdkNotices
       }),
@@ -2508,12 +2604,14 @@ async function runLocale(params) {
   const entries = toTranslate.map((key) => params.source.entries.get(key)).filter((entry) => entry !== void 0);
   const accepted = /* @__PURE__ */ new Map();
   const integrityMismatches = [];
+  const providerFailures = [];
   const subBatchNotices = await translateAndCheck(
     provider,
     params,
     entries,
     accepted,
-    integrityMismatches
+    integrityMismatches,
+    providerFailures
   );
   const merged = new Map(target.entries);
   for (const key of pruned) {
@@ -2537,8 +2635,18 @@ async function runLocale(params) {
     path
   );
   const pluralNotices = params.generatePlurals ? pluralNoticeFor(params, merged) : sdkNotices;
-  const notices = [...pluralNotices, ...subBatchNotices];
-  const withheld = /* @__PURE__ */ new Set([...integrityMismatches, ...invalidIcuSource, ...generation.withheld]);
+  const notices = [
+    ...pluralNotices,
+    ...subBatchNotices,
+    ...generation.notices
+  ];
+  const withheld = /* @__PURE__ */ new Set([
+    ...integrityMismatches,
+    ...providerFailures,
+    ...invalidIcuSource,
+    ...generation.withheld,
+    ...generation.providerFailures
+  ]);
   return {
     summary: baseSummary({
       locale: params.targetLocale,
@@ -2549,6 +2657,8 @@ async function runLocale(params) {
       generated: generation.accepted.map((form) => form.targetKey).sort(),
       // Withheld generated forms surface alongside withheld translations: both failed integrity.
       integrityMismatches: [...integrityMismatches, ...generation.withheld].sort(),
+      // A generation sub-batch whose provider call itself threw is a provider failure, never integrity.
+      providerFailures: [...providerFailures, ...generation.providerFailures].sort(),
       pruned,
       notices
     }),
@@ -2557,7 +2667,7 @@ async function runLocale(params) {
 }
 async function runGeneration(params, provider) {
   if (!params.generatePlurals || provider.kind !== "llm") {
-    return { accepted: [], withheld: [] };
+    return { accepted: [], withheld: [], providerFailures: [], notices: [] };
   }
   return generatePluralForms({
     source: params.source,
@@ -2568,7 +2678,8 @@ async function runGeneration(params, provider) {
     provider,
     glossary: params.glossary,
     tone: params.tone,
-    baseline: params.baseline
+    baseline: params.baseline,
+    maxBatchSize: params.maxBatchSize
   });
 }
 function pluralNoticeFor(params, merged) {
@@ -2590,27 +2701,35 @@ function baseSummary(parts) {
     pruned: parts.pruned,
     invalidIcuSource: parts.invalidIcuSource,
     integrityMismatches: parts.integrityMismatches,
+    providerFailures: parts.providerFailures,
     generated: parts.generated,
     notices: parts.notices
   };
 }
-async function translateAndCheck(provider, params, entries, accepted, integrityMismatches) {
+async function translateAndCheck(provider, params, entries, accepted, integrityMismatches, providerFailures) {
   const notices = [];
   for (const batch of chunk(entries, params.maxBatchSize)) {
-    const subNotices = await runSubBatch(provider, params, batch, accepted, integrityMismatches);
+    const subNotices = await runSubBatch(
+      provider,
+      params,
+      batch,
+      accepted,
+      integrityMismatches,
+      providerFailures
+    );
     notices.push(...subNotices);
   }
   return notices;
 }
-async function runSubBatch(provider, params, batch, accepted, integrityMismatches) {
+async function runSubBatch(provider, params, batch, accepted, integrityMismatches, providerFailures) {
   let result;
   try {
     result = await provider.translateBatch(buildRequest3(params, batch));
-  } catch {
+  } catch (error) {
     for (const entry of batch) {
-      integrityMismatches.push(entry.key);
+      providerFailures.push(entry.key);
     }
-    return [subBatchFailedNotice(batch.length)];
+    return [subBatchFailedNotice(batch.length, error)];
   }
   for (const entry of batch) {
     const value = result.values.get(entry.key);
@@ -2623,19 +2742,6 @@ async function runSubBatch(provider, params, batch, accepted, integrityMismatche
   }
   return readNotices(result);
 }
-function subBatchFailedNotice(count) {
-  return {
-    code: "SUB_BATCH_FAILED",
-    message: `A sub-batch of ${count} entries failed and was withheld; it will be retried next run.`
-  };
-}
-function chunk(items, size) {
-  const chunks = [];
-  for (let index = 0; index < items.length; index += size) {
-    chunks.push(items.slice(index, index + size));
-  }
-  return chunks;
-}
 function computeLockEntries(params, merged, withheld, generated) {
   const lockEntries = {};
   const sourceBaseKeys = sourcePluralBaseKeys(params.source);
@@ -2783,6 +2889,7 @@ var COLUMN_WIDTHS = {
   [COLUMN.status]: 12,
   [COLUMN.translation]: 50
 };
+var TEXT_NUMBER_FORMAT = "@";
 function styleHeader(sheet) {
   const header = sheet.getRow(HEADER_ROW);
   HEADERS.forEach((label, index) => {
@@ -2798,6 +2905,7 @@ function applyColumnGeometry(sheet) {
     sheet.getColumn(Number(column)).width = width;
   }
   sheet.getColumn(COLUMN.sourceHash).hidden = true;
+  sheet.getColumn(COLUMN.translation).numFmt = TEXT_NUMBER_FORMAT;
   sheet.views = [{ state: "frozen", ySplit: HEADER_ROW }];
 }
 function writeRow(sheet, sheetRow) {
@@ -2806,7 +2914,9 @@ function writeRow(sheet, sheetRow) {
   row.getCell(COLUMN.source).value = sheetRow.source;
   row.getCell(COLUMN.current).value = sheetRow.currentTarget;
   row.getCell(COLUMN.status).value = sheetRow.status;
-  row.getCell(COLUMN.translation).value = sheetRow.translation === "" ? null : sheetRow.translation;
+  const translationCell = row.getCell(COLUMN.translation);
+  translationCell.numFmt = TEXT_NUMBER_FORMAT;
+  translationCell.value = sheetRow.translation === "" ? null : sheetRow.translation;
   row.getCell(COLUMN.sourceHash).value = sheetRow.sourceHash;
   for (let column = COLUMN.key; column <= COLUMN.sourceHash; column += 1) {
     const cell = row.getCell(column);
@@ -3158,9 +3268,20 @@ function judge(row, sourceEntry, adapter) {
   }
   return void 0;
 }
+function trackBlankDrift(row, params, buckets) {
+  const sourceEntry = params.source.entries.get(row.key);
+  if (sourceEntry === void 0) {
+    return;
+  }
+  const priorHash = params.baseline.get(row.key);
+  if (priorHash !== void 0 && priorHash !== contentHash(sourceEntry)) {
+    buckets.blankDrifted.add(row.key);
+  }
+}
 function classifyRows(params, buckets) {
   for (const row of params.sheet.rows) {
     if (row.translation === "") {
+      trackBlankDrift(row, params, buckets);
       continue;
     }
     if (isUnknownKey(row, params.source, params.target)) {
@@ -3179,9 +3300,20 @@ function classifyRows(params, buckets) {
     }
   }
 }
+function blankRowBaselineNotice(count) {
+  return {
+    code: "BLANK_ROW_BASELINE_RETAINED",
+    message: `${count} row(s) were left blank for a key whose source changed since the row's baseline was recorded; the prior baseline was kept so the drift keeps being reported.`
+  };
+}
 function importLocale(params) {
   const diff2 = diffResources(params.source, params.target, { baseline: params.baseline });
-  const buckets = { accepted: /* @__PURE__ */ new Map(), mismatches: [], withheld: /* @__PURE__ */ new Set() };
+  const buckets = {
+    accepted: /* @__PURE__ */ new Map(),
+    mismatches: [],
+    withheld: /* @__PURE__ */ new Set(),
+    blankDrifted: /* @__PURE__ */ new Set()
+  };
   classifyRows(params, buckets);
   const rowKeys = new Set(params.sheet.rows.map((row) => row.key));
   const invalidIcuSource = [...new Set(params.sourceInvalidIcuKeys)].filter((key) => rowKeys.has(key)).sort();
@@ -3195,9 +3327,11 @@ function importLocale(params) {
     pruned: [],
     invalidIcuSource,
     integrityMismatches: [...buckets.mismatches].sort(),
+    // A workbook import never calls a provider, so a provider-call failure cannot occur here.
+    providerFailures: [],
     // Plural generation is a translate-flow concern; the manual workbook import never generates forms.
     generated: [],
-    notices: []
+    notices: buckets.blankDrifted.size > 0 ? [blankRowBaselineNotice(buckets.blankDrifted.size)] : []
   };
   return { summary, accepted: buckets.accepted, withheld: buckets.withheld };
 }
@@ -3231,21 +3365,19 @@ function mergeAccepted(target, accepted) {
   }
   return merged;
 }
-function computeLockEntries2(source, merged, baseline, withheld) {
+function computeLockEntries2(source, merged, baseline, accepted) {
   const entries = {};
   for (const key of merged.keys()) {
     const sourceEntry = source.entries.get(key);
     if (sourceEntry === void 0) {
       continue;
     }
-    if (withheld.has(key)) {
-      const prior = baseline.get(key);
-      if (prior !== void 0) {
-        entries[key] = prior;
-      }
+    if (accepted.has(key)) {
+      entries[key] = contentHash(sourceEntry);
       continue;
     }
-    entries[key] = contentHash(sourceEntry);
+    const prior = baseline.get(key);
+    entries[key] = prior !== void 0 ? prior : contentHash(sourceEntry);
   }
   return entries;
 }
@@ -3258,7 +3390,7 @@ async function runSheet(ctx, sheet, lock) {
   }
   const target = await readTarget4(ctx.cwd, ctx.config, ctx.adapter, ctx.fs, sheet.locale);
   const baseline = baselineFor(lock, sheet.locale);
-  const { summary, accepted, withheld } = importLocale({
+  const { summary, accepted } = importLocale({
     sheet,
     source: ctx.source,
     target,
@@ -3282,7 +3414,7 @@ async function runSheet(ctx, sheet, lock) {
       path
     );
   }
-  return { summary, lockEntries: computeLockEntries2(ctx.source, merged, baseline, withheld) };
+  return { summary, lockEntries: computeLockEntries2(ctx.source, merged, baseline, accepted) };
 }
 async function importWorkbook(input, deps = {}) {
   const config = input.config;