@verbatra/sdk 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.cjs +283 -98
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -8
- package/dist/index.d.ts +28 -8
- package/dist/index.js +283 -98
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -15,6 +15,7 @@ import { parse as parse$1, TYPE } from '@formatjs/icu-messageformat-parser';
|
|
|
15
15
|
import { XMLSerializer, DOMParser } from '@xmldom/xmldom';
|
|
16
16
|
import { stringify, parse } from 'yaml';
|
|
17
17
|
import ExcelJS from 'exceljs';
|
|
18
|
+
import { Readable } from 'stream';
|
|
18
19
|
import JSZip from 'jszip';
|
|
19
20
|
import { watch as watch$1 } from 'chokidar';
|
|
20
21
|
|
|
@@ -151,7 +152,7 @@ function checkPlaceholders(source, translated) {
|
|
|
151
152
|
const extra = multisetExcess(translatedCounts, sourceCounts);
|
|
152
153
|
const reordered = missing.length === 0 && extra.length === 0 && !sameOrder(source, translated);
|
|
153
154
|
return {
|
|
154
|
-
matches: missing.length === 0 && extra.length === 0
|
|
155
|
+
matches: missing.length === 0 && extra.length === 0,
|
|
155
156
|
missing,
|
|
156
157
|
extra,
|
|
157
158
|
reordered
|
|
@@ -472,6 +473,19 @@ function createDefaultClient2() {
|
|
|
472
473
|
};
|
|
473
474
|
return { client, freeAccount };
|
|
474
475
|
}
|
|
476
|
+
var PLACEHOLDER_UNSUPPORTED_MESSAGE = "Some entries contain placeholders or ICU syntax that DeepL cannot preserve; they were left untranslated. Use an LLM provider to translate placeholder-bearing strings.";
|
|
477
|
+
function partitionByPlaceholders(entries) {
|
|
478
|
+
const protectable = [];
|
|
479
|
+
const unprotectable = [];
|
|
480
|
+
for (const entry of entries) {
|
|
481
|
+
if (entry.placeholders.length > 0) {
|
|
482
|
+
unprotectable.push(entry);
|
|
483
|
+
} else {
|
|
484
|
+
protectable.push(entry);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
return { protectable, unprotectable };
|
|
488
|
+
}
|
|
475
489
|
var FORMALITY_DOWNGRADED_MESSAGE = "Formality was not applied: the configured DeepL key is a free-tier key, which does not support formality.";
|
|
476
490
|
var GLOSSARY_IGNORED_MESSAGE = "The supplied glossary term map was not applied: DeepL uses configured glossary IDs, not term maps.";
|
|
477
491
|
function buildTranslateOptions(input) {
|
|
@@ -535,6 +549,7 @@ function resolveClient(deps) {
|
|
|
535
549
|
}
|
|
536
550
|
async function translate(bundle, config, request) {
|
|
537
551
|
const data = validateRequest(request);
|
|
552
|
+
const { protectable, unprotectable } = partitionByPlaceholders(data.entries);
|
|
538
553
|
const genericGlossarySupplied = request.glossary !== void 0 && Object.keys(request.glossary).length > 0;
|
|
539
554
|
const { options, notices } = buildTranslateOptions({
|
|
540
555
|
freeAccount: bundle.freeAccount,
|
|
@@ -542,18 +557,28 @@ async function translate(bundle, config, request) {
|
|
|
542
557
|
...data.tone !== void 0 ? { tone: data.tone } : {},
|
|
543
558
|
...config.glossaryId !== void 0 ? { glossaryId: config.glossaryId } : {}
|
|
544
559
|
});
|
|
545
|
-
const
|
|
546
|
-
const results = await callClient2(
|
|
560
|
+
const { values, integrity } = await translateProtectable(
|
|
547
561
|
bundle.client,
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
562
|
+
data,
|
|
563
|
+
protectable,
|
|
564
|
+
options,
|
|
565
|
+
request.extractPlaceholders
|
|
552
566
|
);
|
|
553
|
-
|
|
554
|
-
|
|
567
|
+
if (unprotectable.length > 0) {
|
|
568
|
+
notices.push({ code: "PLACEHOLDER_UNSUPPORTED", message: PLACEHOLDER_UNSUPPORTED_MESSAGE });
|
|
569
|
+
}
|
|
555
570
|
return { values, integrity, notices };
|
|
556
571
|
}
|
|
572
|
+
async function translateProtectable(client, data, protectable, options, extract) {
|
|
573
|
+
if (protectable.length === 0) {
|
|
574
|
+
return { values: /* @__PURE__ */ new Map(), integrity: /* @__PURE__ */ new Map() };
|
|
575
|
+
}
|
|
576
|
+
const texts = protectable.map((entry) => entry.value);
|
|
577
|
+
const results = await callClient2(client, texts, data.sourceLocale, data.targetLocale, options);
|
|
578
|
+
const { values, integrityInputs } = zipResults(protectable, results);
|
|
579
|
+
const integrity = checkBatchIntegrity(integrityInputs, extract);
|
|
580
|
+
return { values, integrity };
|
|
581
|
+
}
|
|
557
582
|
function callClient2(client, texts, sourceLang, targetLang, options) {
|
|
558
583
|
return guardProviderCall(() => client.translateText(texts, sourceLang, targetLang, options));
|
|
559
584
|
}
|
|
@@ -1127,19 +1152,53 @@ function childMessages(element) {
|
|
|
1127
1152
|
}
|
|
1128
1153
|
return [];
|
|
1129
1154
|
}
|
|
1130
|
-
function
|
|
1155
|
+
function addCount(target, token, count) {
|
|
1156
|
+
target.set(token, (target.get(token) ?? 0) + count);
|
|
1157
|
+
}
|
|
1158
|
+
function combineBranches(branchMultisets) {
|
|
1159
|
+
const combined = /* @__PURE__ */ new Map();
|
|
1160
|
+
const tokens = /* @__PURE__ */ new Set();
|
|
1161
|
+
for (const branch of branchMultisets) {
|
|
1162
|
+
for (const token of branch.keys()) {
|
|
1163
|
+
tokens.add(token);
|
|
1164
|
+
}
|
|
1165
|
+
}
|
|
1166
|
+
for (const token of tokens) {
|
|
1167
|
+
const minCount = Math.min(...branchMultisets.map((branch) => branch.get(token) ?? 0));
|
|
1168
|
+
if (minCount > 0) {
|
|
1169
|
+
combined.set(token, minCount);
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
return combined;
|
|
1173
|
+
}
|
|
1174
|
+
function collect(elements, state) {
|
|
1175
|
+
const result = /* @__PURE__ */ new Map();
|
|
1131
1176
|
for (const element of elements) {
|
|
1132
1177
|
const token = tokenOf(element);
|
|
1133
1178
|
if (token !== void 0) {
|
|
1134
|
-
|
|
1179
|
+
addCount(result, token, 1);
|
|
1135
1180
|
}
|
|
1136
1181
|
if (element.type === TYPE.plural) {
|
|
1137
1182
|
state.isPlural = true;
|
|
1138
1183
|
}
|
|
1139
|
-
|
|
1140
|
-
|
|
1184
|
+
const branches = childMessages(element);
|
|
1185
|
+
if (branches.length > 0) {
|
|
1186
|
+
const branchMultisets = branches.map((branch) => collect(branch, state));
|
|
1187
|
+
for (const [branchToken, branchCount] of combineBranches(branchMultisets)) {
|
|
1188
|
+
addCount(result, branchToken, branchCount);
|
|
1189
|
+
}
|
|
1141
1190
|
}
|
|
1142
1191
|
}
|
|
1192
|
+
return result;
|
|
1193
|
+
}
|
|
1194
|
+
function flatten(multiset) {
|
|
1195
|
+
const flattened = [];
|
|
1196
|
+
for (const [token, count] of multiset) {
|
|
1197
|
+
for (let i = 0; i < count; i += 1) {
|
|
1198
|
+
flattened.push(token);
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
return flattened;
|
|
1143
1202
|
}
|
|
1144
1203
|
function analyzeIcuValue(value) {
|
|
1145
1204
|
if (!value.includes("{") && !value.includes("<")) {
|
|
@@ -1147,16 +1206,9 @@ function analyzeIcuValue(value) {
|
|
|
1147
1206
|
}
|
|
1148
1207
|
try {
|
|
1149
1208
|
const ast = parse$1(value);
|
|
1150
|
-
const placeholders = [];
|
|
1151
1209
|
const state = { isPlural: false };
|
|
1152
|
-
collect(
|
|
1153
|
-
|
|
1154
|
-
(token) => {
|
|
1155
|
-
placeholders.push(token);
|
|
1156
|
-
},
|
|
1157
|
-
state
|
|
1158
|
-
);
|
|
1159
|
-
return { placeholders, isPlural: state.isPlural, valid: true };
|
|
1210
|
+
const multiset = collect(ast, state);
|
|
1211
|
+
return { placeholders: flatten(multiset), isPlural: state.isPlural, valid: true };
|
|
1160
1212
|
} catch {
|
|
1161
1213
|
return INVALID;
|
|
1162
1214
|
}
|
|
@@ -1418,21 +1470,28 @@ function addEntries(ctx, prefix, node) {
|
|
|
1418
1470
|
}
|
|
1419
1471
|
}
|
|
1420
1472
|
}
|
|
1421
|
-
function addPathEntries(node, prefix, namespace, derive, out) {
|
|
1473
|
+
function addPathEntries(node, prefix, namespace, derive, out, claimedPaths) {
|
|
1422
1474
|
for (const [key, value] of Object.entries(node)) {
|
|
1423
1475
|
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
1476
|
+
if (claimedPaths.has(path)) {
|
|
1477
|
+
throw new AdapterError(
|
|
1478
|
+
"INVALID_STRUCTURE",
|
|
1479
|
+
"A dotted key and a nested key path resolve to the same path."
|
|
1480
|
+
);
|
|
1481
|
+
}
|
|
1482
|
+
claimedPaths.add(path);
|
|
1424
1483
|
if (typeof value === "string") {
|
|
1425
1484
|
const { placeholders, isPlural } = derive(key, value);
|
|
1426
1485
|
out.set(path, { key: path, namespace, value, placeholders, isPlural });
|
|
1427
1486
|
} else {
|
|
1428
|
-
addPathEntries(value, path, namespace, derive, out);
|
|
1487
|
+
addPathEntries(value, path, namespace, derive, out, claimedPaths);
|
|
1429
1488
|
}
|
|
1430
1489
|
}
|
|
1431
1490
|
}
|
|
1432
1491
|
function flattenTree(tree, namespace, derive, keyMode = "literal-leaf") {
|
|
1433
1492
|
const out = /* @__PURE__ */ new Map();
|
|
1434
1493
|
if (keyMode === "path-notation") {
|
|
1435
|
-
addPathEntries(tree, "", namespace, derive, out);
|
|
1494
|
+
addPathEntries(tree, "", namespace, derive, out, /* @__PURE__ */ new Set());
|
|
1436
1495
|
return out;
|
|
1437
1496
|
}
|
|
1438
1497
|
addEntries({ namespace, derive, out, claimed: /* @__PURE__ */ new Map() }, [], tree);
|
|
@@ -1554,21 +1613,20 @@ function messagesFromEntries(entries) {
|
|
|
1554
1613
|
}
|
|
1555
1614
|
return out;
|
|
1556
1615
|
}
|
|
1616
|
+
function isEnoent(error) {
|
|
1617
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
1618
|
+
}
|
|
1557
1619
|
async function readDestinationPairs(filePath) {
|
|
1558
|
-
let
|
|
1620
|
+
let content;
|
|
1559
1621
|
try {
|
|
1560
|
-
|
|
1561
|
-
|
|
1622
|
+
content = await readFileContent(filePath);
|
|
1623
|
+
} catch (error) {
|
|
1624
|
+
if (isEnoent(error)) {
|
|
1562
1625
|
return null;
|
|
1563
1626
|
}
|
|
1564
|
-
|
|
1565
|
-
} catch {
|
|
1566
|
-
return null;
|
|
1567
|
-
}
|
|
1568
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1569
|
-
return null;
|
|
1627
|
+
throw error;
|
|
1570
1628
|
}
|
|
1571
|
-
return Object.entries(
|
|
1629
|
+
return Object.entries(parseArbObject(content));
|
|
1572
1630
|
}
|
|
1573
1631
|
async function buildArbWriteTree(entries, filePath) {
|
|
1574
1632
|
const messages = messagesFromEntries(entries);
|
|
@@ -1669,6 +1727,20 @@ function createNextIntlJsonAdapter() {
|
|
|
1669
1727
|
validateMessage: icuIsValid
|
|
1670
1728
|
});
|
|
1671
1729
|
}
|
|
1730
|
+
function assertNoDottedNestedKey(tree) {
|
|
1731
|
+
for (const [key, value] of Object.entries(tree)) {
|
|
1732
|
+
if (typeof value !== "object") {
|
|
1733
|
+
continue;
|
|
1734
|
+
}
|
|
1735
|
+
if (key.includes(".")) {
|
|
1736
|
+
throw new AdapterError(
|
|
1737
|
+
"MIXED_STRUCTURE",
|
|
1738
|
+
"A nested object key contains a literal dot, which is ambiguous with a dotted path."
|
|
1739
|
+
);
|
|
1740
|
+
}
|
|
1741
|
+
assertNoDottedNestedKey(value);
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1672
1744
|
function assertNotMixed(tree) {
|
|
1673
1745
|
let hasNested = false;
|
|
1674
1746
|
let hasFlatDottedKey = false;
|
|
@@ -1685,6 +1757,7 @@ function assertNotMixed(tree) {
|
|
|
1685
1757
|
"The file mixes flat dotted keys with nested objects."
|
|
1686
1758
|
);
|
|
1687
1759
|
}
|
|
1760
|
+
assertNoDottedNestedKey(tree);
|
|
1688
1761
|
}
|
|
1689
1762
|
async function detectStyle(filePath) {
|
|
1690
1763
|
let parsed;
|
|
@@ -2057,6 +2130,24 @@ function selectAdapter(format, registry = createDefaultRegistry()) {
|
|
|
2057
2130
|
);
|
|
2058
2131
|
}
|
|
2059
2132
|
|
|
2133
|
+
// src/flow/select-locales.ts
|
|
2134
|
+
function selectLocales(config, requested) {
|
|
2135
|
+
if (requested === void 0) {
|
|
2136
|
+
return config.targetLocales;
|
|
2137
|
+
}
|
|
2138
|
+
const configured = new Set(config.targetLocales);
|
|
2139
|
+
const unknown = requested.filter((locale) => !configured.has(locale));
|
|
2140
|
+
if (unknown.length > 0) {
|
|
2141
|
+
const label = unknown.length === 1 ? "locale" : "locales";
|
|
2142
|
+
throw new SdkError(
|
|
2143
|
+
"UNKNOWN_LOCALE",
|
|
2144
|
+
`Requested ${label} not in the configured target locales: ${unknown.join(", ")}. Configured targets: ${config.targetLocales.join(", ")}.`
|
|
2145
|
+
);
|
|
2146
|
+
}
|
|
2147
|
+
const wanted = new Set(requested);
|
|
2148
|
+
return config.targetLocales.filter((locale) => wanted.has(locale));
|
|
2149
|
+
}
|
|
2150
|
+
|
|
2060
2151
|
// src/flow/source.ts
|
|
2061
2152
|
async function readSource(config, cwd, fs, adapter) {
|
|
2062
2153
|
const sourcePath = localeFilePath(cwd, config.files.pattern, config.sourceLocale);
|
|
@@ -2085,13 +2176,6 @@ async function readTarget(cwd, config, adapter, fs, locale) {
|
|
|
2085
2176
|
}
|
|
2086
2177
|
return (await adapter.read(path, locale)).resource;
|
|
2087
2178
|
}
|
|
2088
|
-
function selectedLocales(config, requested) {
|
|
2089
|
-
if (requested === void 0) {
|
|
2090
|
-
return config.targetLocales;
|
|
2091
|
-
}
|
|
2092
|
-
const wanted = new Set(requested);
|
|
2093
|
-
return config.targetLocales.filter((locale) => wanted.has(locale));
|
|
2094
|
-
}
|
|
2095
2179
|
async function diffLocales(input, deps = {}) {
|
|
2096
2180
|
const config = input.config;
|
|
2097
2181
|
const cwd = input.cwd ?? process.cwd();
|
|
@@ -2100,7 +2184,7 @@ async function diffLocales(input, deps = {}) {
|
|
|
2100
2184
|
const source = await readSource(config, cwd, fs, adapter);
|
|
2101
2185
|
const lock = await readLockFile(lockFilePath(cwd), fs);
|
|
2102
2186
|
return Promise.all(
|
|
2103
|
-
|
|
2187
|
+
selectLocales(config, input.locales).map(async (locale) => {
|
|
2104
2188
|
const target = await readTarget(cwd, config, adapter, fs, locale);
|
|
2105
2189
|
const diff2 = diffResources(source.resource, target, { baseline: baselineFor(lock, locale) });
|
|
2106
2190
|
return { locale, diff: diff2 };
|
|
@@ -2171,6 +2255,7 @@ function failureSummary(locale, error) {
|
|
|
2171
2255
|
pruned: [],
|
|
2172
2256
|
invalidIcuSource: [],
|
|
2173
2257
|
integrityMismatches: [],
|
|
2258
|
+
providerFailures: [],
|
|
2174
2259
|
generated: [],
|
|
2175
2260
|
notices: [],
|
|
2176
2261
|
error: describeError(error)
|
|
@@ -2182,6 +2267,29 @@ function partition(locales) {
|
|
|
2182
2267
|
return { succeeded, failed };
|
|
2183
2268
|
}
|
|
2184
2269
|
|
|
2270
|
+
// src/flow/batching.ts
|
|
2271
|
+
function chunk(items, size) {
|
|
2272
|
+
const chunks = [];
|
|
2273
|
+
for (let index = 0; index < items.length; index += size) {
|
|
2274
|
+
chunks.push(items.slice(index, index + size));
|
|
2275
|
+
}
|
|
2276
|
+
return chunks;
|
|
2277
|
+
}
|
|
2278
|
+
var GENERIC_PROVIDER_FAILURE_MESSAGE = "The provider call failed.";
|
|
2279
|
+
function classifyProviderFailure(error) {
|
|
2280
|
+
if (error instanceof ProviderError) {
|
|
2281
|
+
return { code: error.code, message: error.message };
|
|
2282
|
+
}
|
|
2283
|
+
return { code: "PROVIDER_CALL_FAILED", message: GENERIC_PROVIDER_FAILURE_MESSAGE };
|
|
2284
|
+
}
|
|
2285
|
+
function subBatchFailedNotice(count, error) {
|
|
2286
|
+
const { code, message } = classifyProviderFailure(error);
|
|
2287
|
+
return {
|
|
2288
|
+
code: "SUB_BATCH_FAILED",
|
|
2289
|
+
message: `A sub-batch of ${count} entries failed (${code}: ${message}) and was withheld; it will be retried next run.`
|
|
2290
|
+
};
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2185
2293
|
// src/flow/notices.ts
|
|
2186
2294
|
function isNotice(value) {
|
|
2187
2295
|
return typeof value === "object" && value !== null && typeof value.code === "string" && typeof value.message === "string";
|
|
@@ -2364,13 +2472,36 @@ async function generatePluralForms(context) {
|
|
|
2364
2472
|
const plan = planPluralGeneration(context.source, context.targetLocale, context.format);
|
|
2365
2473
|
const stale = staleItems(plan.items, context.baseline);
|
|
2366
2474
|
if (stale.length === 0) {
|
|
2367
|
-
return { accepted: [], withheld: [] };
|
|
2475
|
+
return { accepted: [], withheld: [], providerFailures: [], notices: [] };
|
|
2368
2476
|
}
|
|
2369
|
-
const entries = stale.map(syntheticEntry);
|
|
2370
|
-
const result = await context.provider.translateBatch(buildRequest2(context, entries));
|
|
2371
2477
|
const accepted = [];
|
|
2372
2478
|
const withheld = [];
|
|
2373
|
-
|
|
2479
|
+
const providerFailures = [];
|
|
2480
|
+
const notices = [];
|
|
2481
|
+
for (const batch of chunk(stale, context.maxBatchSize)) {
|
|
2482
|
+
const batchNotices = await runGenerationSubBatch(
|
|
2483
|
+
context,
|
|
2484
|
+
batch,
|
|
2485
|
+
accepted,
|
|
2486
|
+
withheld,
|
|
2487
|
+
providerFailures
|
|
2488
|
+
);
|
|
2489
|
+
notices.push(...batchNotices);
|
|
2490
|
+
}
|
|
2491
|
+
return { accepted, withheld, providerFailures, notices };
|
|
2492
|
+
}
|
|
2493
|
+
async function runGenerationSubBatch(context, batch, accepted, withheld, providerFailures) {
|
|
2494
|
+
let result;
|
|
2495
|
+
try {
|
|
2496
|
+
const entries = batch.map(syntheticEntry);
|
|
2497
|
+
result = await context.provider.translateBatch(buildRequest2(context, entries));
|
|
2498
|
+
} catch (error) {
|
|
2499
|
+
for (const item of batch) {
|
|
2500
|
+
providerFailures.push(item.targetKey);
|
|
2501
|
+
}
|
|
2502
|
+
return [subBatchFailedNotice(batch.length, error)];
|
|
2503
|
+
}
|
|
2504
|
+
for (const item of batch) {
|
|
2374
2505
|
const value = result.values.get(item.targetKey);
|
|
2375
2506
|
const integrity = result.integrity.get(item.targetKey);
|
|
2376
2507
|
if (value !== void 0 && integrity?.matches === true) {
|
|
@@ -2383,7 +2514,7 @@ async function generatePluralForms(context) {
|
|
|
2383
2514
|
withheld.push(item.targetKey);
|
|
2384
2515
|
}
|
|
2385
2516
|
}
|
|
2386
|
-
return
|
|
2517
|
+
return readNotices(result);
|
|
2387
2518
|
}
|
|
2388
2519
|
|
|
2389
2520
|
// src/flow/locale-run.ts
|
|
@@ -2433,6 +2564,7 @@ async function runLocale(params) {
|
|
|
2433
2564
|
translated: toTranslate,
|
|
2434
2565
|
generated: [],
|
|
2435
2566
|
integrityMismatches: [],
|
|
2567
|
+
providerFailures: [],
|
|
2436
2568
|
pruned,
|
|
2437
2569
|
notices: sdkNotices
|
|
2438
2570
|
}),
|
|
@@ -2442,12 +2574,14 @@ async function runLocale(params) {
|
|
|
2442
2574
|
const entries = toTranslate.map((key) => params.source.entries.get(key)).filter((entry) => entry !== void 0);
|
|
2443
2575
|
const accepted = /* @__PURE__ */ new Map();
|
|
2444
2576
|
const integrityMismatches = [];
|
|
2577
|
+
const providerFailures = [];
|
|
2445
2578
|
const subBatchNotices = await translateAndCheck(
|
|
2446
2579
|
provider,
|
|
2447
2580
|
params,
|
|
2448
2581
|
entries,
|
|
2449
2582
|
accepted,
|
|
2450
|
-
integrityMismatches
|
|
2583
|
+
integrityMismatches,
|
|
2584
|
+
providerFailures
|
|
2451
2585
|
);
|
|
2452
2586
|
const merged = new Map(target.entries);
|
|
2453
2587
|
for (const key of pruned) {
|
|
@@ -2471,8 +2605,18 @@ async function runLocale(params) {
|
|
|
2471
2605
|
path
|
|
2472
2606
|
);
|
|
2473
2607
|
const pluralNotices = params.generatePlurals ? pluralNoticeFor(params, merged) : sdkNotices;
|
|
2474
|
-
const notices = [
|
|
2475
|
-
|
|
2608
|
+
const notices = [
|
|
2609
|
+
...pluralNotices,
|
|
2610
|
+
...subBatchNotices,
|
|
2611
|
+
...generation.notices
|
|
2612
|
+
];
|
|
2613
|
+
const withheld = /* @__PURE__ */ new Set([
|
|
2614
|
+
...integrityMismatches,
|
|
2615
|
+
...providerFailures,
|
|
2616
|
+
...invalidIcuSource,
|
|
2617
|
+
...generation.withheld,
|
|
2618
|
+
...generation.providerFailures
|
|
2619
|
+
]);
|
|
2476
2620
|
return {
|
|
2477
2621
|
summary: baseSummary({
|
|
2478
2622
|
locale: params.targetLocale,
|
|
@@ -2483,6 +2627,8 @@ async function runLocale(params) {
|
|
|
2483
2627
|
generated: generation.accepted.map((form) => form.targetKey).sort(),
|
|
2484
2628
|
// Withheld generated forms surface alongside withheld translations: both failed integrity.
|
|
2485
2629
|
integrityMismatches: [...integrityMismatches, ...generation.withheld].sort(),
|
|
2630
|
+
// A generation sub-batch whose provider call itself threw is a provider failure, never integrity.
|
|
2631
|
+
providerFailures: [...providerFailures, ...generation.providerFailures].sort(),
|
|
2486
2632
|
pruned,
|
|
2487
2633
|
notices
|
|
2488
2634
|
}),
|
|
@@ -2491,7 +2637,7 @@ async function runLocale(params) {
|
|
|
2491
2637
|
}
|
|
2492
2638
|
async function runGeneration(params, provider) {
|
|
2493
2639
|
if (!params.generatePlurals || provider.kind !== "llm") {
|
|
2494
|
-
return { accepted: [], withheld: [] };
|
|
2640
|
+
return { accepted: [], withheld: [], providerFailures: [], notices: [] };
|
|
2495
2641
|
}
|
|
2496
2642
|
return generatePluralForms({
|
|
2497
2643
|
source: params.source,
|
|
@@ -2502,7 +2648,8 @@ async function runGeneration(params, provider) {
|
|
|
2502
2648
|
provider,
|
|
2503
2649
|
glossary: params.glossary,
|
|
2504
2650
|
tone: params.tone,
|
|
2505
|
-
baseline: params.baseline
|
|
2651
|
+
baseline: params.baseline,
|
|
2652
|
+
maxBatchSize: params.maxBatchSize
|
|
2506
2653
|
});
|
|
2507
2654
|
}
|
|
2508
2655
|
function pluralNoticeFor(params, merged) {
|
|
@@ -2524,27 +2671,35 @@ function baseSummary(parts) {
|
|
|
2524
2671
|
pruned: parts.pruned,
|
|
2525
2672
|
invalidIcuSource: parts.invalidIcuSource,
|
|
2526
2673
|
integrityMismatches: parts.integrityMismatches,
|
|
2674
|
+
providerFailures: parts.providerFailures,
|
|
2527
2675
|
generated: parts.generated,
|
|
2528
2676
|
notices: parts.notices
|
|
2529
2677
|
};
|
|
2530
2678
|
}
|
|
2531
|
-
async function translateAndCheck(provider, params, entries, accepted, integrityMismatches) {
|
|
2679
|
+
async function translateAndCheck(provider, params, entries, accepted, integrityMismatches, providerFailures) {
|
|
2532
2680
|
const notices = [];
|
|
2533
2681
|
for (const batch of chunk(entries, params.maxBatchSize)) {
|
|
2534
|
-
const subNotices = await runSubBatch(
|
|
2682
|
+
const subNotices = await runSubBatch(
|
|
2683
|
+
provider,
|
|
2684
|
+
params,
|
|
2685
|
+
batch,
|
|
2686
|
+
accepted,
|
|
2687
|
+
integrityMismatches,
|
|
2688
|
+
providerFailures
|
|
2689
|
+
);
|
|
2535
2690
|
notices.push(...subNotices);
|
|
2536
2691
|
}
|
|
2537
2692
|
return notices;
|
|
2538
2693
|
}
|
|
2539
|
-
async function runSubBatch(provider, params, batch, accepted, integrityMismatches) {
|
|
2694
|
+
async function runSubBatch(provider, params, batch, accepted, integrityMismatches, providerFailures) {
|
|
2540
2695
|
let result;
|
|
2541
2696
|
try {
|
|
2542
2697
|
result = await provider.translateBatch(buildRequest3(params, batch));
|
|
2543
|
-
} catch {
|
|
2698
|
+
} catch (error) {
|
|
2544
2699
|
for (const entry of batch) {
|
|
2545
|
-
|
|
2700
|
+
providerFailures.push(entry.key);
|
|
2546
2701
|
}
|
|
2547
|
-
return [subBatchFailedNotice(batch.length)];
|
|
2702
|
+
return [subBatchFailedNotice(batch.length, error)];
|
|
2548
2703
|
}
|
|
2549
2704
|
for (const entry of batch) {
|
|
2550
2705
|
const value = result.values.get(entry.key);
|
|
@@ -2557,19 +2712,6 @@ async function runSubBatch(provider, params, batch, accepted, integrityMismatche
|
|
|
2557
2712
|
}
|
|
2558
2713
|
return readNotices(result);
|
|
2559
2714
|
}
|
|
2560
|
-
function subBatchFailedNotice(count) {
|
|
2561
|
-
return {
|
|
2562
|
-
code: "SUB_BATCH_FAILED",
|
|
2563
|
-
message: `A sub-batch of ${count} entries failed and was withheld; it will be retried next run.`
|
|
2564
|
-
};
|
|
2565
|
-
}
|
|
2566
|
-
function chunk(items, size) {
|
|
2567
|
-
const chunks = [];
|
|
2568
|
-
for (let index = 0; index < items.length; index += size) {
|
|
2569
|
-
chunks.push(items.slice(index, index + size));
|
|
2570
|
-
}
|
|
2571
|
-
return chunks;
|
|
2572
|
-
}
|
|
2573
2715
|
function computeLockEntries(params, merged, withheld, generated) {
|
|
2574
2716
|
const lockEntries = {};
|
|
2575
2717
|
const sourceBaseKeys = sourcePluralBaseKeys(params.source);
|
|
@@ -2717,6 +2859,7 @@ var COLUMN_WIDTHS = {
|
|
|
2717
2859
|
[COLUMN.status]: 12,
|
|
2718
2860
|
[COLUMN.translation]: 50
|
|
2719
2861
|
};
|
|
2862
|
+
var TEXT_NUMBER_FORMAT = "@";
|
|
2720
2863
|
function styleHeader(sheet) {
|
|
2721
2864
|
const header = sheet.getRow(HEADER_ROW);
|
|
2722
2865
|
HEADERS.forEach((label, index) => {
|
|
@@ -2732,6 +2875,7 @@ function applyColumnGeometry(sheet) {
|
|
|
2732
2875
|
sheet.getColumn(Number(column)).width = width;
|
|
2733
2876
|
}
|
|
2734
2877
|
sheet.getColumn(COLUMN.sourceHash).hidden = true;
|
|
2878
|
+
sheet.getColumn(COLUMN.translation).numFmt = TEXT_NUMBER_FORMAT;
|
|
2735
2879
|
sheet.views = [{ state: "frozen", ySplit: HEADER_ROW }];
|
|
2736
2880
|
}
|
|
2737
2881
|
function writeRow(sheet, sheetRow) {
|
|
@@ -2740,7 +2884,9 @@ function writeRow(sheet, sheetRow) {
|
|
|
2740
2884
|
row.getCell(COLUMN.source).value = sheetRow.source;
|
|
2741
2885
|
row.getCell(COLUMN.current).value = sheetRow.currentTarget;
|
|
2742
2886
|
row.getCell(COLUMN.status).value = sheetRow.status;
|
|
2743
|
-
row.getCell(COLUMN.translation)
|
|
2887
|
+
const translationCell = row.getCell(COLUMN.translation);
|
|
2888
|
+
translationCell.numFmt = TEXT_NUMBER_FORMAT;
|
|
2889
|
+
translationCell.value = sheetRow.translation === "" ? null : sheetRow.translation;
|
|
2744
2890
|
row.getCell(COLUMN.sourceHash).value = sheetRow.sourceHash;
|
|
2745
2891
|
for (let column = COLUMN.key; column <= COLUMN.sourceHash; column += 1) {
|
|
2746
2892
|
const cell = row.getCell(column);
|
|
@@ -2826,6 +2972,34 @@ function declaredSize(file) {
|
|
|
2826
2972
|
const size = data?.uncompressedSize;
|
|
2827
2973
|
return typeof size === "number" && Number.isFinite(size) ? size : void 0;
|
|
2828
2974
|
}
|
|
2975
|
+
async function streamEntryBounded(file, remaining) {
|
|
2976
|
+
const chunks = [];
|
|
2977
|
+
let raw = 0;
|
|
2978
|
+
let oversize = false;
|
|
2979
|
+
const source = new Readable({ objectMode: true }).wrap(
|
|
2980
|
+
file.nodeStream("nodebuffer")
|
|
2981
|
+
);
|
|
2982
|
+
try {
|
|
2983
|
+
for await (const chunk2 of source) {
|
|
2984
|
+
const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
|
|
2985
|
+
raw += buf.length;
|
|
2986
|
+
if (raw > remaining) {
|
|
2987
|
+
oversize = true;
|
|
2988
|
+
break;
|
|
2989
|
+
}
|
|
2990
|
+
chunks.push(buf);
|
|
2991
|
+
}
|
|
2992
|
+
} catch {
|
|
2993
|
+
throw new ExchangeError("WORKBOOK_INVALID", "A workbook entry could not be decompressed.");
|
|
2994
|
+
}
|
|
2995
|
+
if (oversize) {
|
|
2996
|
+
throw new ExchangeError(
|
|
2997
|
+
"WORKBOOK_INVALID",
|
|
2998
|
+
"The workbook decompresses to more than the permitted maximum bytes."
|
|
2999
|
+
);
|
|
3000
|
+
}
|
|
3001
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
3002
|
+
}
|
|
2829
3003
|
async function guardWorkbookBytes(bytes, limits) {
|
|
2830
3004
|
let zip;
|
|
2831
3005
|
try {
|
|
@@ -2855,12 +3029,8 @@ async function guardWorkbookBytes(bytes, limits) {
|
|
|
2855
3029
|
}
|
|
2856
3030
|
let actualTotal = 0;
|
|
2857
3031
|
for (const file of files) {
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
content = await file.async("string");
|
|
2861
|
-
} catch {
|
|
2862
|
-
throw new ExchangeError("WORKBOOK_INVALID", "A workbook entry could not be decompressed.");
|
|
2863
|
-
}
|
|
3032
|
+
const remaining = limits.maxDecompressedBytes - actualTotal;
|
|
3033
|
+
const content = await streamEntryBounded(file, remaining);
|
|
2864
3034
|
actualTotal += Buffer.byteLength(content, "utf8");
|
|
2865
3035
|
if (actualTotal > limits.maxDecompressedBytes) {
|
|
2866
3036
|
throw new ExchangeError(
|
|
@@ -3010,13 +3180,6 @@ function buildRows(source, target, baseline, includeUnchanged) {
|
|
|
3010
3180
|
}
|
|
3011
3181
|
return [...rows].sort((a, b) => a.key < b.key ? -1 : 1);
|
|
3012
3182
|
}
|
|
3013
|
-
function selectedLocales2(config, requested) {
|
|
3014
|
-
if (requested === void 0) {
|
|
3015
|
-
return config.targetLocales;
|
|
3016
|
-
}
|
|
3017
|
-
const wanted = new Set(requested);
|
|
3018
|
-
return config.targetLocales.filter((locale) => wanted.has(locale));
|
|
3019
|
-
}
|
|
3020
3183
|
async function exportWorkbook(input, deps = {}) {
|
|
3021
3184
|
const config = input.config;
|
|
3022
3185
|
const cwd = input.cwd ?? process.cwd();
|
|
@@ -3024,7 +3187,7 @@ async function exportWorkbook(input, deps = {}) {
|
|
|
3024
3187
|
const adapter = selectAdapter(config.format, deps.adapterRegistry);
|
|
3025
3188
|
const source = await readSource(config, cwd, fs, adapter);
|
|
3026
3189
|
const lock = await readLockFile(lockFilePath(cwd), fs);
|
|
3027
|
-
const locales =
|
|
3190
|
+
const locales = selectLocales(config, input.locales);
|
|
3028
3191
|
const sheets = await Promise.all(
|
|
3029
3192
|
locales.map(async (locale) => {
|
|
3030
3193
|
const target = await readTarget3(cwd, config, adapter, fs, locale);
|
|
@@ -3075,9 +3238,20 @@ function judge(row, sourceEntry, adapter) {
|
|
|
3075
3238
|
}
|
|
3076
3239
|
return void 0;
|
|
3077
3240
|
}
|
|
3241
|
+
function trackBlankDrift(row, params, buckets) {
|
|
3242
|
+
const sourceEntry = params.source.entries.get(row.key);
|
|
3243
|
+
if (sourceEntry === void 0) {
|
|
3244
|
+
return;
|
|
3245
|
+
}
|
|
3246
|
+
const priorHash = params.baseline.get(row.key);
|
|
3247
|
+
if (priorHash !== void 0 && priorHash !== contentHash(sourceEntry)) {
|
|
3248
|
+
buckets.blankDrifted.add(row.key);
|
|
3249
|
+
}
|
|
3250
|
+
}
|
|
3078
3251
|
function classifyRows(params, buckets) {
|
|
3079
3252
|
for (const row of params.sheet.rows) {
|
|
3080
3253
|
if (row.translation === "") {
|
|
3254
|
+
trackBlankDrift(row, params, buckets);
|
|
3081
3255
|
continue;
|
|
3082
3256
|
}
|
|
3083
3257
|
if (isUnknownKey(row, params.source, params.target)) {
|
|
@@ -3096,9 +3270,20 @@ function classifyRows(params, buckets) {
|
|
|
3096
3270
|
}
|
|
3097
3271
|
}
|
|
3098
3272
|
}
|
|
3273
|
+
function blankRowBaselineNotice(count) {
|
|
3274
|
+
return {
|
|
3275
|
+
code: "BLANK_ROW_BASELINE_RETAINED",
|
|
3276
|
+
message: `${count} row(s) were left blank for a key whose source changed since the row's baseline was recorded; the prior baseline was kept so the drift keeps being reported.`
|
|
3277
|
+
};
|
|
3278
|
+
}
|
|
3099
3279
|
function importLocale(params) {
|
|
3100
3280
|
const diff2 = diffResources(params.source, params.target, { baseline: params.baseline });
|
|
3101
|
-
const buckets = {
|
|
3281
|
+
const buckets = {
|
|
3282
|
+
accepted: /* @__PURE__ */ new Map(),
|
|
3283
|
+
mismatches: [],
|
|
3284
|
+
withheld: /* @__PURE__ */ new Set(),
|
|
3285
|
+
blankDrifted: /* @__PURE__ */ new Set()
|
|
3286
|
+
};
|
|
3102
3287
|
classifyRows(params, buckets);
|
|
3103
3288
|
const rowKeys = new Set(params.sheet.rows.map((row) => row.key));
|
|
3104
3289
|
const invalidIcuSource = [...new Set(params.sourceInvalidIcuKeys)].filter((key) => rowKeys.has(key)).sort();
|
|
@@ -3112,9 +3297,11 @@ function importLocale(params) {
|
|
|
3112
3297
|
pruned: [],
|
|
3113
3298
|
invalidIcuSource,
|
|
3114
3299
|
integrityMismatches: [...buckets.mismatches].sort(),
|
|
3300
|
+
// A workbook import never calls a provider, so a provider-call failure cannot occur here.
|
|
3301
|
+
providerFailures: [],
|
|
3115
3302
|
// Plural generation is a translate-flow concern; the manual workbook import never generates forms.
|
|
3116
3303
|
generated: [],
|
|
3117
|
-
notices: []
|
|
3304
|
+
notices: buckets.blankDrifted.size > 0 ? [blankRowBaselineNotice(buckets.blankDrifted.size)] : []
|
|
3118
3305
|
};
|
|
3119
3306
|
return { summary, accepted: buckets.accepted, withheld: buckets.withheld };
|
|
3120
3307
|
}
|
|
@@ -3148,21 +3335,19 @@ function mergeAccepted(target, accepted) {
|
|
|
3148
3335
|
}
|
|
3149
3336
|
return merged;
|
|
3150
3337
|
}
|
|
3151
|
-
function computeLockEntries2(source, merged, baseline,
|
|
3338
|
+
function computeLockEntries2(source, merged, baseline, accepted) {
|
|
3152
3339
|
const entries = {};
|
|
3153
3340
|
for (const key of merged.keys()) {
|
|
3154
3341
|
const sourceEntry = source.entries.get(key);
|
|
3155
3342
|
if (sourceEntry === void 0) {
|
|
3156
3343
|
continue;
|
|
3157
3344
|
}
|
|
3158
|
-
if (
|
|
3159
|
-
|
|
3160
|
-
if (prior !== void 0) {
|
|
3161
|
-
entries[key] = prior;
|
|
3162
|
-
}
|
|
3345
|
+
if (accepted.has(key)) {
|
|
3346
|
+
entries[key] = contentHash(sourceEntry);
|
|
3163
3347
|
continue;
|
|
3164
3348
|
}
|
|
3165
|
-
|
|
3349
|
+
const prior = baseline.get(key);
|
|
3350
|
+
entries[key] = prior !== void 0 ? prior : contentHash(sourceEntry);
|
|
3166
3351
|
}
|
|
3167
3352
|
return entries;
|
|
3168
3353
|
}
|
|
@@ -3175,7 +3360,7 @@ async function runSheet(ctx, sheet, lock) {
|
|
|
3175
3360
|
}
|
|
3176
3361
|
const target = await readTarget4(ctx.cwd, ctx.config, ctx.adapter, ctx.fs, sheet.locale);
|
|
3177
3362
|
const baseline = baselineFor(lock, sheet.locale);
|
|
3178
|
-
const { summary, accepted
|
|
3363
|
+
const { summary, accepted } = importLocale({
|
|
3179
3364
|
sheet,
|
|
3180
3365
|
source: ctx.source,
|
|
3181
3366
|
target,
|
|
@@ -3199,7 +3384,7 @@ async function runSheet(ctx, sheet, lock) {
|
|
|
3199
3384
|
path
|
|
3200
3385
|
);
|
|
3201
3386
|
}
|
|
3202
|
-
return { summary, lockEntries: computeLockEntries2(ctx.source, merged, baseline,
|
|
3387
|
+
return { summary, lockEntries: computeLockEntries2(ctx.source, merged, baseline, accepted) };
|
|
3203
3388
|
}
|
|
3204
3389
|
async function importWorkbook(input, deps = {}) {
|
|
3205
3390
|
const config = input.config;
|