@verbatra/sdk 0.4.1 → 0.4.3

package/README.md

@@ -10,6 +10,8 @@
 
 <p align="center">
   <a href="https://www.npmjs.com/package/@verbatra/sdk"><img src="https://img.shields.io/npm/v/@verbatra/sdk?label=%40verbatra%2Fsdk" alt="@verbatra/sdk npm version" /></a>
+  <a href="https://github.com/mariokreitz/verbatra/actions/workflows/ci.yml"><img src="https://github.com/mariokreitz/verbatra/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI" /></a>
+  <a href="https://github.com/mariokreitz/verbatra/actions/workflows/e2e.yml"><img src="https://github.com/mariokreitz/verbatra/actions/workflows/e2e.yml/badge.svg?branch=main" alt="E2E" /></a>
   <a href="https://codecov.io/gh/mariokreitz/verbatra"><img src="https://codecov.io/gh/mariokreitz/verbatra/graph/badge.svg" alt="Coverage" /></a>
   <a href="https://github.com/mariokreitz/verbatra/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="License: MIT" /></a>
 </p>

package/dist/index.cjs

@@ -17,6 +17,7 @@ var icuMessageformatParser = require('@formatjs/icu-messageformat-parser');
 var xmldom = require('@xmldom/xmldom');
 var yaml = require('yaml');
 var ExcelJS = require('exceljs');
+var stream = require('stream');
 var JSZip = require('jszip');
 var chokidar = require('chokidar');
 
@@ -181,7 +182,7 @@ function checkPlaceholders(source, translated) {
   const extra = multisetExcess(translatedCounts, sourceCounts);
   const reordered = missing.length === 0 && extra.length === 0 && !sameOrder(source, translated);
   return {
-    matches: missing.length === 0 && extra.length === 0 && !reordered,
+    matches: missing.length === 0 && extra.length === 0,
     missing,
     extra,
     reordered
@@ -502,6 +503,19 @@ function createDefaultClient2() {
   };
   return { client, freeAccount };
 }
+var PLACEHOLDER_UNSUPPORTED_MESSAGE = "Some entries contain placeholders or ICU syntax that DeepL cannot preserve; they were left untranslated. Use an LLM provider to translate placeholder-bearing strings.";
+function partitionByPlaceholders(entries) {
+  const protectable = [];
+  const unprotectable = [];
+  for (const entry of entries) {
+    if (entry.placeholders.length > 0) {
+      unprotectable.push(entry);
+    } else {
+      protectable.push(entry);
+    }
+  }
+  return { protectable, unprotectable };
+}
 var FORMALITY_DOWNGRADED_MESSAGE = "Formality was not applied: the configured DeepL key is a free-tier key, which does not support formality.";
 var GLOSSARY_IGNORED_MESSAGE = "The supplied glossary term map was not applied: DeepL uses configured glossary IDs, not term maps.";
 function buildTranslateOptions(input) {
@@ -565,6 +579,7 @@ function resolveClient(deps) {
 }
 async function translate(bundle, config, request) {
   const data = validateRequest(request);
+  const { protectable, unprotectable } = partitionByPlaceholders(data.entries);
   const genericGlossarySupplied = request.glossary !== void 0 && Object.keys(request.glossary).length > 0;
   const { options, notices } = buildTranslateOptions({
     freeAccount: bundle.freeAccount,
@@ -572,18 +587,28 @@ async function translate(bundle, config, request) {
     ...data.tone !== void 0 ? { tone: data.tone } : {},
     ...config.glossaryId !== void 0 ? { glossaryId: config.glossaryId } : {}
   });
-  const texts = data.entries.map((entry) => entry.value);
-  const results = await callClient2(
+  const { values, integrity } = await translateProtectable(
     bundle.client,
-    texts,
-    data.sourceLocale,
-    data.targetLocale,
-    options
+    data,
+    protectable,
+    options,
+    request.extractPlaceholders
   );
-  const { values, integrityInputs } = zipResults(data.entries, results);
-  const integrity = checkBatchIntegrity(integrityInputs, request.extractPlaceholders);
+  if (unprotectable.length > 0) {
+    notices.push({ code: "PLACEHOLDER_UNSUPPORTED", message: PLACEHOLDER_UNSUPPORTED_MESSAGE });
+  }
   return { values, integrity, notices };
 }
+async function translateProtectable(client, data, protectable, options, extract) {
+  if (protectable.length === 0) {
+    return { values: /* @__PURE__ */ new Map(), integrity: /* @__PURE__ */ new Map() };
+  }
+  const texts = protectable.map((entry) => entry.value);
+  const results = await callClient2(client, texts, data.sourceLocale, data.targetLocale, options);
+  const { values, integrityInputs } = zipResults(protectable, results);
+  const integrity = checkBatchIntegrity(integrityInputs, extract);
+  return { values, integrity };
+}
 function callClient2(client, texts, sourceLang, targetLang, options) {
   return guardProviderCall(() => client.translateText(texts, sourceLang, targetLang, options));
 }
@@ -2087,6 +2112,24 @@ function selectAdapter(format, registry = createDefaultRegistry()) {
   );
 }
 
+// src/flow/select-locales.ts
+function selectLocales(config, requested) {
+  if (requested === void 0) {
+    return config.targetLocales;
+  }
+  const configured = new Set(config.targetLocales);
+  const unknown = requested.filter((locale) => !configured.has(locale));
+  if (unknown.length > 0) {
+    const label = unknown.length === 1 ? "locale" : "locales";
+    throw new SdkError(
+      "UNKNOWN_LOCALE",
+      `Requested ${label} not in the configured target locales: ${unknown.join(", ")}. Configured targets: ${config.targetLocales.join(", ")}.`
+    );
+  }
+  const wanted = new Set(requested);
+  return config.targetLocales.filter((locale) => wanted.has(locale));
+}
+
 // src/flow/source.ts
 async function readSource(config, cwd, fs, adapter) {
   const sourcePath = localeFilePath(cwd, config.files.pattern, config.sourceLocale);
@@ -2115,13 +2158,6 @@ async function readTarget(cwd, config, adapter, fs, locale) {
   }
   return (await adapter.read(path, locale)).resource;
 }
-function selectedLocales(config, requested) {
-  if (requested === void 0) {
-    return config.targetLocales;
-  }
-  const wanted = new Set(requested);
-  return config.targetLocales.filter((locale) => wanted.has(locale));
-}
 async function diffLocales(input, deps = {}) {
   const config = input.config;
   const cwd = input.cwd ?? process.cwd();
@@ -2130,7 +2166,7 @@ async function diffLocales(input, deps = {}) {
   const source = await readSource(config, cwd, fs, adapter);
   const lock = await readLockFile(lockFilePath(cwd), fs);
   return Promise.all(
-    selectedLocales(config, input.locales).map(async (locale) => {
+    selectLocales(config, input.locales).map(async (locale) => {
       const target = await readTarget(cwd, config, adapter, fs, locale);
       const diff2 = diffResources(source.resource, target, { baseline: baselineFor(lock, locale) });
       return { locale, diff: diff2 };
@@ -2856,6 +2892,34 @@ function declaredSize(file) {
   const size = data?.uncompressedSize;
   return typeof size === "number" && Number.isFinite(size) ? size : void 0;
 }
+async function streamEntryBounded(file, remaining) {
+  const chunks = [];
+  let raw = 0;
+  let oversize = false;
+  const source = new stream.Readable({ objectMode: true }).wrap(
+    file.nodeStream("nodebuffer")
+  );
+  try {
+    for await (const chunk2 of source) {
+      const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
+      raw += buf.length;
+      if (raw > remaining) {
+        oversize = true;
+        break;
+      }
+      chunks.push(buf);
+    }
+  } catch {
+    throw new ExchangeError("WORKBOOK_INVALID", "A workbook entry could not be decompressed.");
+  }
+  if (oversize) {
+    throw new ExchangeError(
+      "WORKBOOK_INVALID",
+      "The workbook decompresses to more than the permitted maximum bytes."
+    );
+  }
+  return Buffer.concat(chunks).toString("utf8");
+}
 async function guardWorkbookBytes(bytes, limits) {
   let zip;
   try {
@@ -2885,12 +2949,8 @@ async function guardWorkbookBytes(bytes, limits) {
   }
   let actualTotal = 0;
   for (const file of files) {
-    let content;
-    try {
-      content = await file.async("string");
-    } catch {
-      throw new ExchangeError("WORKBOOK_INVALID", "A workbook entry could not be decompressed.");
-    }
+    const remaining = limits.maxDecompressedBytes - actualTotal;
+    const content = await streamEntryBounded(file, remaining);
     actualTotal += Buffer.byteLength(content, "utf8");
     if (actualTotal > limits.maxDecompressedBytes) {
       throw new ExchangeError(
@@ -3040,13 +3100,6 @@ function buildRows(source, target, baseline, includeUnchanged) {
   }
   return [...rows].sort((a, b) => a.key < b.key ? -1 : 1);
 }
-function selectedLocales2(config, requested) {
-  if (requested === void 0) {
-    return config.targetLocales;
-  }
-  const wanted = new Set(requested);
-  return config.targetLocales.filter((locale) => wanted.has(locale));
-}
 async function exportWorkbook(input, deps = {}) {
   const config = input.config;
   const cwd = input.cwd ?? process.cwd();
@@ -3054,7 +3107,7 @@ async function exportWorkbook(input, deps = {}) {
   const adapter = selectAdapter(config.format, deps.adapterRegistry);
   const source = await readSource(config, cwd, fs, adapter);
   const lock = await readLockFile(lockFilePath(cwd), fs);
-  const locales = selectedLocales2(config, input.locales);
+  const locales = selectLocales(config, input.locales);
   const sheets = await Promise.all(
     locales.map(async (locale) => {
       const target = await readTarget3(cwd, config, adapter, fs, locale);