@verbatra/sdk 0.5.0-next.1 → 0.5.0-next.2

package/README.md

@@ -71,7 +71,7 @@ export default defineConfig({
 });
 ```
 
-`files.pattern` must contain the `{locale}` token, and `targetLocales` must not include `sourceLocale`; both are enforced when the config is validated. The supported `format` values are `i18next-json`, `vue-i18n-json`, `next-intl-json`, `ngx-translate-json`, `xliff`, `yaml`, and `arb`. The optional `glossary` (a term map) and `tone` (`"formal"`, `"informal"`, or `"neutral"`) refine the output. The optional `prune` boolean (off by default) opts in to removing orphaned keys (present in a target file but absent from the source) from the written target files and the lock; the `translate --prune` flag overrides it per run. The optional `generatePlurals` boolean (off by default) opts in to synthesizing the CLDR plural forms a richer target language requires but the source lacks (i18next-JSON projects translated by an LLM provider only; DeepL, non-i18next formats, and unknown languages fall back to the per-locale plural warning and never fail); a per-run `generatePlurals` override on `translate` takes precedence, and generated keys are reported separately from translated keys on the summary. OpenAI and Gemini take `{ model, maxOutputTokens }`; DeepL takes `{}` (with an optional `glossaryId`). API keys are never part of the config. Each provider reads its own environment variable (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DEEPL_API_KEY`).
+`files.pattern` must contain the `{locale}` token, and `targetLocales` must not include `sourceLocale`; both are enforced when the config is validated. The supported `format` values are `i18next-json`, `vue-i18n-json`, `next-intl-json`, `ngx-translate-json`, `xliff`, `yaml`, and `arb`. The optional `glossary` (a term map) and `tone` (`"formal"`, `"informal"`, or `"neutral"`) refine the output. The optional `prune` boolean (off by default) opts in to removing orphaned keys (present in a target file but absent from the source) from the written target files and the lock; the `translate --prune` flag overrides it per run. The optional `generatePlurals` boolean (off by default) opts in to synthesizing the CLDR plural forms a richer target language requires but the source lacks (i18next-JSON projects translated by an LLM provider only; DeepL, non-i18next formats, and unknown languages fall back to the per-locale plural warning and never fail); a per-run `generatePlurals` override on `translate` takes precedence, and generated keys are reported separately from translated keys on the summary. OpenAI and Gemini take `{ model, maxOutputTokens }`; `openai-compatible` additionally takes a `baseUrl` (for a local or self-hosted server such as LM Studio, Ollama, or vLLM) and an optional `apiKeyEnvVar`; DeepL takes `{}` (with an optional `glossaryId`). API keys are never part of the config. Each provider reads its own environment variable (`ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DEEPL_API_KEY`; `openai-compatible` resolves its key from `apiKeyEnvVar`, then `OPENAI_COMPATIBLE_API_KEY`, then falls back to a keyless placeholder).
 
 ## API reference
 

package/dist/index.cjs

@@ -503,16 +503,14 @@ function validateRequest(request) {
   return parsed.data;
 }
 function toIntegrityInputs(entries, values) {
-  return entries.map((entry) => {
+  const inputs = [];
+  for (const entry of entries) {
     const translatedValue = values.get(entry.key);
-    if (translatedValue === void 0) {
-      throw new ProviderError(
-        "INVALID_RESPONSE",
-        "The provider response is missing one or more keys."
-      );
+    if (translatedValue !== void 0) {
+      inputs.push({ key: entry.key, sourceValue: entry.value, translatedValue });
     }
-    return { key: entry.key, sourceValue: entry.value, translatedValue };
-  });
+  }
+  return inputs;
 }
 function toItem(entry) {
   return {
@@ -544,25 +542,29 @@ function deriveJsonSchema(schema) {
   }
   return result;
 }
-function reconcile(translations, requestedKeys) {
-  const requested = new Set(requestedKeys);
-  const values = /* @__PURE__ */ new Map();
+function partitionTranslations(translations, requested) {
+  const firstSeen = /* @__PURE__ */ new Map();
+  const duplicated = /* @__PURE__ */ new Set();
   for (const { key, value } of translations) {
-    if (!requested.has(key) || values.has(key)) {
-      throw new ProviderError(
-        "INVALID_RESPONSE",
-        "The provider returned an unexpected or duplicate key."
-      );
+    if (!requested.has(key)) {
+      throw new ProviderError("INVALID_RESPONSE", "The provider returned an unexpected key.");
     }
-    values.set(key, value);
+    if (firstSeen.has(key)) {
+      duplicated.add(key);
+      continue;
+    }
+    firstSeen.set(key, value);
   }
-  if (values.size !== requested.size) {
-    throw new ProviderError(
-      "INVALID_RESPONSE",
-      "The provider response is missing one or more keys."
-    );
+  return { firstSeen, duplicated };
+}
+function reconcile(translations, requestedKeys) {
+  const requested = new Set(requestedKeys);
+  const { firstSeen, duplicated } = partitionTranslations(translations, requested);
+  const missingKeys = requestedKeys.filter((key) => duplicated.has(key) || !firstSeen.has(key));
+  for (const key of duplicated) {
+    firstSeen.delete(key);
   }
-  return values;
+  return { accepted: firstSeen, missingKeys };
 }
 function reconcileResult(raw, requestedKeys) {
   const parsed = translationsResultSchema.safeParse(raw);
@@ -574,22 +576,54 @@ function reconcileResult(raw, requestedKeys) {
   }
   return reconcile(parsed.data.translations, requestedKeys);
 }
+var MAX_REPAIR_ROUNDS = 1;
 async function runLlmTranslation(request, mechanism) {
   const data = validateRequest(request);
+  const signal = request.signal;
+  const first = await requestTranslations(mechanism, data, signal);
+  const values = first.outcome.accepted;
+  let usage = first.completion.usage;
+  let toRepair = entriesFor(data.entries, first.outcome.missingKeys);
+  for (let round = 0; round < MAX_REPAIR_ROUNDS && toRepair.length > 0; round += 1) {
+    const repair = await requestTranslations(mechanism, { ...data, entries: toRepair }, signal);
+    for (const [key, value] of repair.outcome.accepted) {
+      values.set(key, value);
+    }
+    usage = mergeUsage(usage, repair.completion.usage);
+    toRepair = entriesFor(data.entries, repair.outcome.missingKeys);
+  }
+  const integrity = checkBatchIntegrity(
+    toIntegrityInputs(data.entries, values),
+    request.extractPlaceholders,
+    request.comparePlaceholders
+  );
+  return usage === void 0 ? { values, integrity, notices: [] } : { values, integrity, usage, notices: [] };
+}
+async function requestTranslations(mechanism, data, signal) {
   const payloadJson = JSON.stringify(buildDataPayload(data));
   const requestedKeys = data.entries.map((entry) => entry.key);
   const completion = await mechanism.translate({
     payloadJson,
     requestedKeys,
-    ...request.signal !== void 0 ? { signal: request.signal } : {}
+    ...signal !== void 0 ? { signal } : {}
   });
-  const values = reconcileResult(completion.raw, requestedKeys);
-  const integrity = checkBatchIntegrity(
-    toIntegrityInputs(data.entries, values),
-    request.extractPlaceholders,
-    request.comparePlaceholders
-  );
-  return completion.usage === void 0 ? { values, integrity } : { values, integrity, usage: completion.usage };
+  return { completion, outcome: reconcileResult(completion.raw, requestedKeys) };
+}
+function entriesFor(entries, keys) {
+  const wanted = new Set(keys);
+  return entries.filter((entry) => wanted.has(entry.key));
+}
+function mergeUsage(first, second) {
+  if (first === void 0) {
+    return second;
+  }
+  if (second === void 0) {
+    return first;
+  }
+  return {
+    inputTokens: first.inputTokens + second.inputTokens,
+    outputTokens: first.outputTokens + second.outputTokens
+  };
 }
 var OUTPUT_TRUNCATED_MESSAGE = "The provider stopped because the output-token limit was reached. Reduce the batch size or raise the configured max output tokens.";
 function assertNotTruncated(truncated) {
@@ -837,7 +871,10 @@ function createDeepLProvider(config, deps = {}) {
   return {
     id: PROVIDER_ID2,
     kind: "machine-translation",
-    supportsGlossary: true,
+    // Honest capability signal: DeepL only ever applies a pre-created native glossary id, never a
+    // generic term map (see buildTranslateOptions). Without one configured, a supplied term map is
+    // silently ignored and surfaces only as a GLOSSARY_IGNORED notice.
+    supportsGlossary: validConfig.glossaryId !== void 0,
     translateBatch: (request) => translate(bundle, validConfig, request)
   };
 }
@@ -1141,16 +1178,19 @@ function buildResponseFormat(mode) {
     }
   };
 }
-function buildOpenAiRequest(config, payloadJson, mode = "strict-schema") {
-  return {
-    model: config.model,
-    max_completion_tokens: config.maxOutputTokens,
+function buildOpenAiRequest(config, payloadJson, mode = "strict-schema", tokenLimitField = "max_completion_tokens") {
+  const base = {
     messages: [
       { role: "system", content: OPENAI_SYSTEM_RULES },
       { role: "user", content: payloadJson }
     ],
+    model: config.model,
     response_format: buildResponseFormat(mode)
   };
+  if (tokenLimitField === "max_tokens") {
+    return { ...base, max_tokens: config.maxOutputTokens };
+  }
+  return { ...base, max_completion_tokens: config.maxOutputTokens };
 }
 function advanceStringScan(char, escaped) {
   if (escaped) {
@@ -1321,7 +1361,7 @@ function createOpenAiCompatibleProvider(config, deps = {}) {
 function createMechanism4(client, config) {
   return {
     translate: async ({ payloadJson, signal }) => {
-      const body = buildOpenAiRequest(config, payloadJson, "strict-schema");
+      const body = buildOpenAiRequest(config, payloadJson, "strict-schema", "max_tokens");
       const completion = await callClient5(client, body, signal);
       return extractOpenAiResult(completion, true);
     }
@@ -1922,6 +1962,9 @@ async function atomicWriteFile(path, data, ops = nodeOps) {
     throw error;
   }
 }
+function stripLeadingBom(content) {
+  return content.charCodeAt(0) === 65279 ? content.slice(1) : content;
+}
 async function readBoundedUtf82(handle, size) {
   const buffer = Buffer.allocUnsafe(size);
   let offset = 0;
@@ -1932,7 +1975,7 @@ async function readBoundedUtf82(handle, size) {
     }
     offset += bytesRead;
   }
-  return buffer.toString("utf8", 0, offset);
+  return stripLeadingBom(buffer.toString("utf8", 0, offset));
 }
 async function readBounded2(filePath) {
   const handle = await promises.open(filePath, "r");
@@ -2975,15 +3018,8 @@ function subBatchFailedNotice(count, error) {
 }
 
 // src/flow/notices.ts
-function isNotice(value) {
-  return typeof value === "object" && value !== null && typeof value.code === "string" && typeof value.message === "string";
-}
 function readNotices(result) {
-  const candidate = result.notices;
-  if (!Array.isArray(candidate)) {
-    return [];
-  }
-  return candidate.filter(isNotice);
+  return result.notices ?? [];
 }
 
 // src/flow/plural-categories.ts
@@ -3187,20 +3223,26 @@ async function runGenerationSubBatch(context, batch, accepted, withheld, provide
     return [subBatchFailedNotice(batch.length, error)];
   }
   for (const item of batch) {
-    const value = result.values.get(item.targetKey);
-    const integrity = result.integrity.get(item.targetKey);
-    if (value !== void 0 && integrity?.matches === true) {
-      accepted.push({
-        targetKey: item.targetKey,
-        entry: { ...syntheticEntry(item), value },
-        lockHash: generatedLockHash(item.governingEntries, item.category)
-      });
-    } else {
-      withheld.push(item.targetKey);
-    }
+    foldGenerationItem(item, result, accepted, withheld, providerFailures);
   }
   return readNotices(result);
 }
+function foldGenerationItem(item, result, accepted, withheld, providerFailures) {
+  const value = result.values.get(item.targetKey);
+  if (value === void 0) {
+    providerFailures.push(item.targetKey);
+    return;
+  }
+  if (result.integrity.get(item.targetKey)?.matches === true) {
+    accepted.push({
+      targetKey: item.targetKey,
+      entry: { ...syntheticEntry(item), value },
+      lockHash: generatedLockHash(item.governingEntries, item.category)
+    });
+  } else {
+    withheld.push(item.targetKey);
+  }
+}
 
 // src/flow/locale-run.ts
 function emptyResource(locale, format) {
@@ -3388,16 +3430,22 @@ async function runSubBatch(provider, params, batch, accepted, integrityMismatche
     return [subBatchFailedNotice(batch.length, error)];
   }
   for (const entry of batch) {
-    const value = result.values.get(entry.key);
-    const integrity = result.integrity.get(entry.key);
-    if (value !== void 0 && integrity?.matches === true) {
-      accepted.set(entry.key, { value, source: entry });
-    } else {
-      integrityMismatches.push(entry.key);
-    }
+    foldEntryResult(entry, result, accepted, integrityMismatches, providerFailures);
   }
   return readNotices(result);
 }
+function foldEntryResult(entry, result, accepted, integrityMismatches, providerFailures) {
+  const value = result.values.get(entry.key);
+  if (value === void 0) {
+    providerFailures.push(entry.key);
+    return;
+  }
+  if (result.integrity.get(entry.key)?.matches === true) {
+    accepted.set(entry.key, { value, source: entry });
+  } else {
+    integrityMismatches.push(entry.key);
+  }
+}
 function computeLockEntries(params, merged, withheld, generated) {
   const lockEntries = {};
   const sourceBaseKeys = sourcePluralBaseKeys(params.source);