@verbatra/sdk 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.cjs +283 -98
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +28 -8
- package/dist/index.d.ts +28 -8
- package/dist/index.js +283 -98
- package/dist/index.js.map +1 -1
- package/package.json +4 -4
package/dist/index.cjs
CHANGED
|
@@ -17,6 +17,7 @@ var icuMessageformatParser = require('@formatjs/icu-messageformat-parser');
|
|
|
17
17
|
var xmldom = require('@xmldom/xmldom');
|
|
18
18
|
var yaml = require('yaml');
|
|
19
19
|
var ExcelJS = require('exceljs');
|
|
20
|
+
var stream = require('stream');
|
|
20
21
|
var JSZip = require('jszip');
|
|
21
22
|
var chokidar = require('chokidar');
|
|
22
23
|
|
|
@@ -181,7 +182,7 @@ function checkPlaceholders(source, translated) {
|
|
|
181
182
|
const extra = multisetExcess(translatedCounts, sourceCounts);
|
|
182
183
|
const reordered = missing.length === 0 && extra.length === 0 && !sameOrder(source, translated);
|
|
183
184
|
return {
|
|
184
|
-
matches: missing.length === 0 && extra.length === 0
|
|
185
|
+
matches: missing.length === 0 && extra.length === 0,
|
|
185
186
|
missing,
|
|
186
187
|
extra,
|
|
187
188
|
reordered
|
|
@@ -502,6 +503,19 @@ function createDefaultClient2() {
|
|
|
502
503
|
};
|
|
503
504
|
return { client, freeAccount };
|
|
504
505
|
}
|
|
506
|
+
var PLACEHOLDER_UNSUPPORTED_MESSAGE = "Some entries contain placeholders or ICU syntax that DeepL cannot preserve; they were left untranslated. Use an LLM provider to translate placeholder-bearing strings.";
|
|
507
|
+
function partitionByPlaceholders(entries) {
|
|
508
|
+
const protectable = [];
|
|
509
|
+
const unprotectable = [];
|
|
510
|
+
for (const entry of entries) {
|
|
511
|
+
if (entry.placeholders.length > 0) {
|
|
512
|
+
unprotectable.push(entry);
|
|
513
|
+
} else {
|
|
514
|
+
protectable.push(entry);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
return { protectable, unprotectable };
|
|
518
|
+
}
|
|
505
519
|
var FORMALITY_DOWNGRADED_MESSAGE = "Formality was not applied: the configured DeepL key is a free-tier key, which does not support formality.";
|
|
506
520
|
var GLOSSARY_IGNORED_MESSAGE = "The supplied glossary term map was not applied: DeepL uses configured glossary IDs, not term maps.";
|
|
507
521
|
function buildTranslateOptions(input) {
|
|
@@ -565,6 +579,7 @@ function resolveClient(deps) {
|
|
|
565
579
|
}
|
|
566
580
|
async function translate(bundle, config, request) {
|
|
567
581
|
const data = validateRequest(request);
|
|
582
|
+
const { protectable, unprotectable } = partitionByPlaceholders(data.entries);
|
|
568
583
|
const genericGlossarySupplied = request.glossary !== void 0 && Object.keys(request.glossary).length > 0;
|
|
569
584
|
const { options, notices } = buildTranslateOptions({
|
|
570
585
|
freeAccount: bundle.freeAccount,
|
|
@@ -572,18 +587,28 @@ async function translate(bundle, config, request) {
|
|
|
572
587
|
...data.tone !== void 0 ? { tone: data.tone } : {},
|
|
573
588
|
...config.glossaryId !== void 0 ? { glossaryId: config.glossaryId } : {}
|
|
574
589
|
});
|
|
575
|
-
const
|
|
576
|
-
const results = await callClient2(
|
|
590
|
+
const { values, integrity } = await translateProtectable(
|
|
577
591
|
bundle.client,
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
592
|
+
data,
|
|
593
|
+
protectable,
|
|
594
|
+
options,
|
|
595
|
+
request.extractPlaceholders
|
|
582
596
|
);
|
|
583
|
-
|
|
584
|
-
|
|
597
|
+
if (unprotectable.length > 0) {
|
|
598
|
+
notices.push({ code: "PLACEHOLDER_UNSUPPORTED", message: PLACEHOLDER_UNSUPPORTED_MESSAGE });
|
|
599
|
+
}
|
|
585
600
|
return { values, integrity, notices };
|
|
586
601
|
}
|
|
602
|
+
async function translateProtectable(client, data, protectable, options, extract) {
|
|
603
|
+
if (protectable.length === 0) {
|
|
604
|
+
return { values: /* @__PURE__ */ new Map(), integrity: /* @__PURE__ */ new Map() };
|
|
605
|
+
}
|
|
606
|
+
const texts = protectable.map((entry) => entry.value);
|
|
607
|
+
const results = await callClient2(client, texts, data.sourceLocale, data.targetLocale, options);
|
|
608
|
+
const { values, integrityInputs } = zipResults(protectable, results);
|
|
609
|
+
const integrity = checkBatchIntegrity(integrityInputs, extract);
|
|
610
|
+
return { values, integrity };
|
|
611
|
+
}
|
|
587
612
|
function callClient2(client, texts, sourceLang, targetLang, options) {
|
|
588
613
|
return guardProviderCall(() => client.translateText(texts, sourceLang, targetLang, options));
|
|
589
614
|
}
|
|
@@ -1157,19 +1182,53 @@ function childMessages(element) {
|
|
|
1157
1182
|
}
|
|
1158
1183
|
return [];
|
|
1159
1184
|
}
|
|
1160
|
-
function
|
|
1185
|
+
function addCount(target, token, count) {
|
|
1186
|
+
target.set(token, (target.get(token) ?? 0) + count);
|
|
1187
|
+
}
|
|
1188
|
+
function combineBranches(branchMultisets) {
|
|
1189
|
+
const combined = /* @__PURE__ */ new Map();
|
|
1190
|
+
const tokens = /* @__PURE__ */ new Set();
|
|
1191
|
+
for (const branch of branchMultisets) {
|
|
1192
|
+
for (const token of branch.keys()) {
|
|
1193
|
+
tokens.add(token);
|
|
1194
|
+
}
|
|
1195
|
+
}
|
|
1196
|
+
for (const token of tokens) {
|
|
1197
|
+
const minCount = Math.min(...branchMultisets.map((branch) => branch.get(token) ?? 0));
|
|
1198
|
+
if (minCount > 0) {
|
|
1199
|
+
combined.set(token, minCount);
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
return combined;
|
|
1203
|
+
}
|
|
1204
|
+
function collect(elements, state) {
|
|
1205
|
+
const result = /* @__PURE__ */ new Map();
|
|
1161
1206
|
for (const element of elements) {
|
|
1162
1207
|
const token = tokenOf(element);
|
|
1163
1208
|
if (token !== void 0) {
|
|
1164
|
-
|
|
1209
|
+
addCount(result, token, 1);
|
|
1165
1210
|
}
|
|
1166
1211
|
if (element.type === icuMessageformatParser.TYPE.plural) {
|
|
1167
1212
|
state.isPlural = true;
|
|
1168
1213
|
}
|
|
1169
|
-
|
|
1170
|
-
|
|
1214
|
+
const branches = childMessages(element);
|
|
1215
|
+
if (branches.length > 0) {
|
|
1216
|
+
const branchMultisets = branches.map((branch) => collect(branch, state));
|
|
1217
|
+
for (const [branchToken, branchCount] of combineBranches(branchMultisets)) {
|
|
1218
|
+
addCount(result, branchToken, branchCount);
|
|
1219
|
+
}
|
|
1171
1220
|
}
|
|
1172
1221
|
}
|
|
1222
|
+
return result;
|
|
1223
|
+
}
|
|
1224
|
+
function flatten(multiset) {
|
|
1225
|
+
const flattened = [];
|
|
1226
|
+
for (const [token, count] of multiset) {
|
|
1227
|
+
for (let i = 0; i < count; i += 1) {
|
|
1228
|
+
flattened.push(token);
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
return flattened;
|
|
1173
1232
|
}
|
|
1174
1233
|
function analyzeIcuValue(value) {
|
|
1175
1234
|
if (!value.includes("{") && !value.includes("<")) {
|
|
@@ -1177,16 +1236,9 @@ function analyzeIcuValue(value) {
|
|
|
1177
1236
|
}
|
|
1178
1237
|
try {
|
|
1179
1238
|
const ast = icuMessageformatParser.parse(value);
|
|
1180
|
-
const placeholders = [];
|
|
1181
1239
|
const state = { isPlural: false };
|
|
1182
|
-
collect(
|
|
1183
|
-
|
|
1184
|
-
(token) => {
|
|
1185
|
-
placeholders.push(token);
|
|
1186
|
-
},
|
|
1187
|
-
state
|
|
1188
|
-
);
|
|
1189
|
-
return { placeholders, isPlural: state.isPlural, valid: true };
|
|
1240
|
+
const multiset = collect(ast, state);
|
|
1241
|
+
return { placeholders: flatten(multiset), isPlural: state.isPlural, valid: true };
|
|
1190
1242
|
} catch {
|
|
1191
1243
|
return INVALID;
|
|
1192
1244
|
}
|
|
@@ -1448,21 +1500,28 @@ function addEntries(ctx, prefix, node) {
|
|
|
1448
1500
|
}
|
|
1449
1501
|
}
|
|
1450
1502
|
}
|
|
1451
|
-
function addPathEntries(node, prefix, namespace, derive, out) {
|
|
1503
|
+
function addPathEntries(node, prefix, namespace, derive, out, claimedPaths) {
|
|
1452
1504
|
for (const [key, value] of Object.entries(node)) {
|
|
1453
1505
|
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
1506
|
+
if (claimedPaths.has(path)) {
|
|
1507
|
+
throw new AdapterError(
|
|
1508
|
+
"INVALID_STRUCTURE",
|
|
1509
|
+
"A dotted key and a nested key path resolve to the same path."
|
|
1510
|
+
);
|
|
1511
|
+
}
|
|
1512
|
+
claimedPaths.add(path);
|
|
1454
1513
|
if (typeof value === "string") {
|
|
1455
1514
|
const { placeholders, isPlural } = derive(key, value);
|
|
1456
1515
|
out.set(path, { key: path, namespace, value, placeholders, isPlural });
|
|
1457
1516
|
} else {
|
|
1458
|
-
addPathEntries(value, path, namespace, derive, out);
|
|
1517
|
+
addPathEntries(value, path, namespace, derive, out, claimedPaths);
|
|
1459
1518
|
}
|
|
1460
1519
|
}
|
|
1461
1520
|
}
|
|
1462
1521
|
function flattenTree(tree, namespace, derive, keyMode = "literal-leaf") {
|
|
1463
1522
|
const out = /* @__PURE__ */ new Map();
|
|
1464
1523
|
if (keyMode === "path-notation") {
|
|
1465
|
-
addPathEntries(tree, "", namespace, derive, out);
|
|
1524
|
+
addPathEntries(tree, "", namespace, derive, out, /* @__PURE__ */ new Set());
|
|
1466
1525
|
return out;
|
|
1467
1526
|
}
|
|
1468
1527
|
addEntries({ namespace, derive, out, claimed: /* @__PURE__ */ new Map() }, [], tree);
|
|
@@ -1584,21 +1643,20 @@ function messagesFromEntries(entries) {
|
|
|
1584
1643
|
}
|
|
1585
1644
|
return out;
|
|
1586
1645
|
}
|
|
1646
|
+
function isEnoent(error) {
|
|
1647
|
+
return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
|
|
1648
|
+
}
|
|
1587
1649
|
async function readDestinationPairs(filePath) {
|
|
1588
|
-
let
|
|
1650
|
+
let content;
|
|
1589
1651
|
try {
|
|
1590
|
-
|
|
1591
|
-
|
|
1652
|
+
content = await readFileContent(filePath);
|
|
1653
|
+
} catch (error) {
|
|
1654
|
+
if (isEnoent(error)) {
|
|
1592
1655
|
return null;
|
|
1593
1656
|
}
|
|
1594
|
-
|
|
1595
|
-
} catch {
|
|
1596
|
-
return null;
|
|
1597
|
-
}
|
|
1598
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1599
|
-
return null;
|
|
1657
|
+
throw error;
|
|
1600
1658
|
}
|
|
1601
|
-
return Object.entries(
|
|
1659
|
+
return Object.entries(parseArbObject(content));
|
|
1602
1660
|
}
|
|
1603
1661
|
async function buildArbWriteTree(entries, filePath) {
|
|
1604
1662
|
const messages = messagesFromEntries(entries);
|
|
@@ -1699,6 +1757,20 @@ function createNextIntlJsonAdapter() {
|
|
|
1699
1757
|
validateMessage: icuIsValid
|
|
1700
1758
|
});
|
|
1701
1759
|
}
|
|
1760
|
+
function assertNoDottedNestedKey(tree) {
|
|
1761
|
+
for (const [key, value] of Object.entries(tree)) {
|
|
1762
|
+
if (typeof value !== "object") {
|
|
1763
|
+
continue;
|
|
1764
|
+
}
|
|
1765
|
+
if (key.includes(".")) {
|
|
1766
|
+
throw new AdapterError(
|
|
1767
|
+
"MIXED_STRUCTURE",
|
|
1768
|
+
"A nested object key contains a literal dot, which is ambiguous with a dotted path."
|
|
1769
|
+
);
|
|
1770
|
+
}
|
|
1771
|
+
assertNoDottedNestedKey(value);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1702
1774
|
function assertNotMixed(tree) {
|
|
1703
1775
|
let hasNested = false;
|
|
1704
1776
|
let hasFlatDottedKey = false;
|
|
@@ -1715,6 +1787,7 @@ function assertNotMixed(tree) {
|
|
|
1715
1787
|
"The file mixes flat dotted keys with nested objects."
|
|
1716
1788
|
);
|
|
1717
1789
|
}
|
|
1790
|
+
assertNoDottedNestedKey(tree);
|
|
1718
1791
|
}
|
|
1719
1792
|
async function detectStyle(filePath) {
|
|
1720
1793
|
let parsed;
|
|
@@ -2087,6 +2160,24 @@ function selectAdapter(format, registry = createDefaultRegistry()) {
|
|
|
2087
2160
|
);
|
|
2088
2161
|
}
|
|
2089
2162
|
|
|
2163
|
+
// src/flow/select-locales.ts
|
|
2164
|
+
function selectLocales(config, requested) {
|
|
2165
|
+
if (requested === void 0) {
|
|
2166
|
+
return config.targetLocales;
|
|
2167
|
+
}
|
|
2168
|
+
const configured = new Set(config.targetLocales);
|
|
2169
|
+
const unknown = requested.filter((locale) => !configured.has(locale));
|
|
2170
|
+
if (unknown.length > 0) {
|
|
2171
|
+
const label = unknown.length === 1 ? "locale" : "locales";
|
|
2172
|
+
throw new SdkError(
|
|
2173
|
+
"UNKNOWN_LOCALE",
|
|
2174
|
+
`Requested ${label} not in the configured target locales: ${unknown.join(", ")}. Configured targets: ${config.targetLocales.join(", ")}.`
|
|
2175
|
+
);
|
|
2176
|
+
}
|
|
2177
|
+
const wanted = new Set(requested);
|
|
2178
|
+
return config.targetLocales.filter((locale) => wanted.has(locale));
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2090
2181
|
// src/flow/source.ts
|
|
2091
2182
|
async function readSource(config, cwd, fs, adapter) {
|
|
2092
2183
|
const sourcePath = localeFilePath(cwd, config.files.pattern, config.sourceLocale);
|
|
@@ -2115,13 +2206,6 @@ async function readTarget(cwd, config, adapter, fs, locale) {
|
|
|
2115
2206
|
}
|
|
2116
2207
|
return (await adapter.read(path, locale)).resource;
|
|
2117
2208
|
}
|
|
2118
|
-
function selectedLocales(config, requested) {
|
|
2119
|
-
if (requested === void 0) {
|
|
2120
|
-
return config.targetLocales;
|
|
2121
|
-
}
|
|
2122
|
-
const wanted = new Set(requested);
|
|
2123
|
-
return config.targetLocales.filter((locale) => wanted.has(locale));
|
|
2124
|
-
}
|
|
2125
2209
|
async function diffLocales(input, deps = {}) {
|
|
2126
2210
|
const config = input.config;
|
|
2127
2211
|
const cwd = input.cwd ?? process.cwd();
|
|
@@ -2130,7 +2214,7 @@ async function diffLocales(input, deps = {}) {
|
|
|
2130
2214
|
const source = await readSource(config, cwd, fs, adapter);
|
|
2131
2215
|
const lock = await readLockFile(lockFilePath(cwd), fs);
|
|
2132
2216
|
return Promise.all(
|
|
2133
|
-
|
|
2217
|
+
selectLocales(config, input.locales).map(async (locale) => {
|
|
2134
2218
|
const target = await readTarget(cwd, config, adapter, fs, locale);
|
|
2135
2219
|
const diff2 = diffResources(source.resource, target, { baseline: baselineFor(lock, locale) });
|
|
2136
2220
|
return { locale, diff: diff2 };
|
|
@@ -2201,6 +2285,7 @@ function failureSummary(locale, error) {
|
|
|
2201
2285
|
pruned: [],
|
|
2202
2286
|
invalidIcuSource: [],
|
|
2203
2287
|
integrityMismatches: [],
|
|
2288
|
+
providerFailures: [],
|
|
2204
2289
|
generated: [],
|
|
2205
2290
|
notices: [],
|
|
2206
2291
|
error: describeError(error)
|
|
@@ -2212,6 +2297,29 @@ function partition(locales) {
|
|
|
2212
2297
|
return { succeeded, failed };
|
|
2213
2298
|
}
|
|
2214
2299
|
|
|
2300
|
+
// src/flow/batching.ts
|
|
2301
|
+
function chunk(items, size) {
|
|
2302
|
+
const chunks = [];
|
|
2303
|
+
for (let index = 0; index < items.length; index += size) {
|
|
2304
|
+
chunks.push(items.slice(index, index + size));
|
|
2305
|
+
}
|
|
2306
|
+
return chunks;
|
|
2307
|
+
}
|
|
2308
|
+
var GENERIC_PROVIDER_FAILURE_MESSAGE = "The provider call failed.";
|
|
2309
|
+
function classifyProviderFailure(error) {
|
|
2310
|
+
if (error instanceof ProviderError) {
|
|
2311
|
+
return { code: error.code, message: error.message };
|
|
2312
|
+
}
|
|
2313
|
+
return { code: "PROVIDER_CALL_FAILED", message: GENERIC_PROVIDER_FAILURE_MESSAGE };
|
|
2314
|
+
}
|
|
2315
|
+
function subBatchFailedNotice(count, error) {
|
|
2316
|
+
const { code, message } = classifyProviderFailure(error);
|
|
2317
|
+
return {
|
|
2318
|
+
code: "SUB_BATCH_FAILED",
|
|
2319
|
+
message: `A sub-batch of ${count} entries failed (${code}: ${message}) and was withheld; it will be retried next run.`
|
|
2320
|
+
};
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2215
2323
|
// src/flow/notices.ts
|
|
2216
2324
|
function isNotice(value) {
|
|
2217
2325
|
return typeof value === "object" && value !== null && typeof value.code === "string" && typeof value.message === "string";
|
|
@@ -2394,13 +2502,36 @@ async function generatePluralForms(context) {
|
|
|
2394
2502
|
const plan = planPluralGeneration(context.source, context.targetLocale, context.format);
|
|
2395
2503
|
const stale = staleItems(plan.items, context.baseline);
|
|
2396
2504
|
if (stale.length === 0) {
|
|
2397
|
-
return { accepted: [], withheld: [] };
|
|
2505
|
+
return { accepted: [], withheld: [], providerFailures: [], notices: [] };
|
|
2398
2506
|
}
|
|
2399
|
-
const entries = stale.map(syntheticEntry);
|
|
2400
|
-
const result = await context.provider.translateBatch(buildRequest2(context, entries));
|
|
2401
2507
|
const accepted = [];
|
|
2402
2508
|
const withheld = [];
|
|
2403
|
-
|
|
2509
|
+
const providerFailures = [];
|
|
2510
|
+
const notices = [];
|
|
2511
|
+
for (const batch of chunk(stale, context.maxBatchSize)) {
|
|
2512
|
+
const batchNotices = await runGenerationSubBatch(
|
|
2513
|
+
context,
|
|
2514
|
+
batch,
|
|
2515
|
+
accepted,
|
|
2516
|
+
withheld,
|
|
2517
|
+
providerFailures
|
|
2518
|
+
);
|
|
2519
|
+
notices.push(...batchNotices);
|
|
2520
|
+
}
|
|
2521
|
+
return { accepted, withheld, providerFailures, notices };
|
|
2522
|
+
}
|
|
2523
|
+
async function runGenerationSubBatch(context, batch, accepted, withheld, providerFailures) {
|
|
2524
|
+
let result;
|
|
2525
|
+
try {
|
|
2526
|
+
const entries = batch.map(syntheticEntry);
|
|
2527
|
+
result = await context.provider.translateBatch(buildRequest2(context, entries));
|
|
2528
|
+
} catch (error) {
|
|
2529
|
+
for (const item of batch) {
|
|
2530
|
+
providerFailures.push(item.targetKey);
|
|
2531
|
+
}
|
|
2532
|
+
return [subBatchFailedNotice(batch.length, error)];
|
|
2533
|
+
}
|
|
2534
|
+
for (const item of batch) {
|
|
2404
2535
|
const value = result.values.get(item.targetKey);
|
|
2405
2536
|
const integrity = result.integrity.get(item.targetKey);
|
|
2406
2537
|
if (value !== void 0 && integrity?.matches === true) {
|
|
@@ -2413,7 +2544,7 @@ async function generatePluralForms(context) {
|
|
|
2413
2544
|
withheld.push(item.targetKey);
|
|
2414
2545
|
}
|
|
2415
2546
|
}
|
|
2416
|
-
return
|
|
2547
|
+
return readNotices(result);
|
|
2417
2548
|
}
|
|
2418
2549
|
|
|
2419
2550
|
// src/flow/locale-run.ts
|
|
@@ -2463,6 +2594,7 @@ async function runLocale(params) {
|
|
|
2463
2594
|
translated: toTranslate,
|
|
2464
2595
|
generated: [],
|
|
2465
2596
|
integrityMismatches: [],
|
|
2597
|
+
providerFailures: [],
|
|
2466
2598
|
pruned,
|
|
2467
2599
|
notices: sdkNotices
|
|
2468
2600
|
}),
|
|
@@ -2472,12 +2604,14 @@ async function runLocale(params) {
|
|
|
2472
2604
|
const entries = toTranslate.map((key) => params.source.entries.get(key)).filter((entry) => entry !== void 0);
|
|
2473
2605
|
const accepted = /* @__PURE__ */ new Map();
|
|
2474
2606
|
const integrityMismatches = [];
|
|
2607
|
+
const providerFailures = [];
|
|
2475
2608
|
const subBatchNotices = await translateAndCheck(
|
|
2476
2609
|
provider,
|
|
2477
2610
|
params,
|
|
2478
2611
|
entries,
|
|
2479
2612
|
accepted,
|
|
2480
|
-
integrityMismatches
|
|
2613
|
+
integrityMismatches,
|
|
2614
|
+
providerFailures
|
|
2481
2615
|
);
|
|
2482
2616
|
const merged = new Map(target.entries);
|
|
2483
2617
|
for (const key of pruned) {
|
|
@@ -2501,8 +2635,18 @@ async function runLocale(params) {
|
|
|
2501
2635
|
path
|
|
2502
2636
|
);
|
|
2503
2637
|
const pluralNotices = params.generatePlurals ? pluralNoticeFor(params, merged) : sdkNotices;
|
|
2504
|
-
const notices = [
|
|
2505
|
-
|
|
2638
|
+
const notices = [
|
|
2639
|
+
...pluralNotices,
|
|
2640
|
+
...subBatchNotices,
|
|
2641
|
+
...generation.notices
|
|
2642
|
+
];
|
|
2643
|
+
const withheld = /* @__PURE__ */ new Set([
|
|
2644
|
+
...integrityMismatches,
|
|
2645
|
+
...providerFailures,
|
|
2646
|
+
...invalidIcuSource,
|
|
2647
|
+
...generation.withheld,
|
|
2648
|
+
...generation.providerFailures
|
|
2649
|
+
]);
|
|
2506
2650
|
return {
|
|
2507
2651
|
summary: baseSummary({
|
|
2508
2652
|
locale: params.targetLocale,
|
|
@@ -2513,6 +2657,8 @@ async function runLocale(params) {
|
|
|
2513
2657
|
generated: generation.accepted.map((form) => form.targetKey).sort(),
|
|
2514
2658
|
// Withheld generated forms surface alongside withheld translations: both failed integrity.
|
|
2515
2659
|
integrityMismatches: [...integrityMismatches, ...generation.withheld].sort(),
|
|
2660
|
+
// A generation sub-batch whose provider call itself threw is a provider failure, never integrity.
|
|
2661
|
+
providerFailures: [...providerFailures, ...generation.providerFailures].sort(),
|
|
2516
2662
|
pruned,
|
|
2517
2663
|
notices
|
|
2518
2664
|
}),
|
|
@@ -2521,7 +2667,7 @@ async function runLocale(params) {
|
|
|
2521
2667
|
}
|
|
2522
2668
|
async function runGeneration(params, provider) {
|
|
2523
2669
|
if (!params.generatePlurals || provider.kind !== "llm") {
|
|
2524
|
-
return { accepted: [], withheld: [] };
|
|
2670
|
+
return { accepted: [], withheld: [], providerFailures: [], notices: [] };
|
|
2525
2671
|
}
|
|
2526
2672
|
return generatePluralForms({
|
|
2527
2673
|
source: params.source,
|
|
@@ -2532,7 +2678,8 @@ async function runGeneration(params, provider) {
|
|
|
2532
2678
|
provider,
|
|
2533
2679
|
glossary: params.glossary,
|
|
2534
2680
|
tone: params.tone,
|
|
2535
|
-
baseline: params.baseline
|
|
2681
|
+
baseline: params.baseline,
|
|
2682
|
+
maxBatchSize: params.maxBatchSize
|
|
2536
2683
|
});
|
|
2537
2684
|
}
|
|
2538
2685
|
function pluralNoticeFor(params, merged) {
|
|
@@ -2554,27 +2701,35 @@ function baseSummary(parts) {
|
|
|
2554
2701
|
pruned: parts.pruned,
|
|
2555
2702
|
invalidIcuSource: parts.invalidIcuSource,
|
|
2556
2703
|
integrityMismatches: parts.integrityMismatches,
|
|
2704
|
+
providerFailures: parts.providerFailures,
|
|
2557
2705
|
generated: parts.generated,
|
|
2558
2706
|
notices: parts.notices
|
|
2559
2707
|
};
|
|
2560
2708
|
}
|
|
2561
|
-
async function translateAndCheck(provider, params, entries, accepted, integrityMismatches) {
|
|
2709
|
+
async function translateAndCheck(provider, params, entries, accepted, integrityMismatches, providerFailures) {
|
|
2562
2710
|
const notices = [];
|
|
2563
2711
|
for (const batch of chunk(entries, params.maxBatchSize)) {
|
|
2564
|
-
const subNotices = await runSubBatch(
|
|
2712
|
+
const subNotices = await runSubBatch(
|
|
2713
|
+
provider,
|
|
2714
|
+
params,
|
|
2715
|
+
batch,
|
|
2716
|
+
accepted,
|
|
2717
|
+
integrityMismatches,
|
|
2718
|
+
providerFailures
|
|
2719
|
+
);
|
|
2565
2720
|
notices.push(...subNotices);
|
|
2566
2721
|
}
|
|
2567
2722
|
return notices;
|
|
2568
2723
|
}
|
|
2569
|
-
async function runSubBatch(provider, params, batch, accepted, integrityMismatches) {
|
|
2724
|
+
async function runSubBatch(provider, params, batch, accepted, integrityMismatches, providerFailures) {
|
|
2570
2725
|
let result;
|
|
2571
2726
|
try {
|
|
2572
2727
|
result = await provider.translateBatch(buildRequest3(params, batch));
|
|
2573
|
-
} catch {
|
|
2728
|
+
} catch (error) {
|
|
2574
2729
|
for (const entry of batch) {
|
|
2575
|
-
|
|
2730
|
+
providerFailures.push(entry.key);
|
|
2576
2731
|
}
|
|
2577
|
-
return [subBatchFailedNotice(batch.length)];
|
|
2732
|
+
return [subBatchFailedNotice(batch.length, error)];
|
|
2578
2733
|
}
|
|
2579
2734
|
for (const entry of batch) {
|
|
2580
2735
|
const value = result.values.get(entry.key);
|
|
@@ -2587,19 +2742,6 @@ async function runSubBatch(provider, params, batch, accepted, integrityMismatche
|
|
|
2587
2742
|
}
|
|
2588
2743
|
return readNotices(result);
|
|
2589
2744
|
}
|
|
2590
|
-
function subBatchFailedNotice(count) {
|
|
2591
|
-
return {
|
|
2592
|
-
code: "SUB_BATCH_FAILED",
|
|
2593
|
-
message: `A sub-batch of ${count} entries failed and was withheld; it will be retried next run.`
|
|
2594
|
-
};
|
|
2595
|
-
}
|
|
2596
|
-
function chunk(items, size) {
|
|
2597
|
-
const chunks = [];
|
|
2598
|
-
for (let index = 0; index < items.length; index += size) {
|
|
2599
|
-
chunks.push(items.slice(index, index + size));
|
|
2600
|
-
}
|
|
2601
|
-
return chunks;
|
|
2602
|
-
}
|
|
2603
2745
|
function computeLockEntries(params, merged, withheld, generated) {
|
|
2604
2746
|
const lockEntries = {};
|
|
2605
2747
|
const sourceBaseKeys = sourcePluralBaseKeys(params.source);
|
|
@@ -2747,6 +2889,7 @@ var COLUMN_WIDTHS = {
|
|
|
2747
2889
|
[COLUMN.status]: 12,
|
|
2748
2890
|
[COLUMN.translation]: 50
|
|
2749
2891
|
};
|
|
2892
|
+
var TEXT_NUMBER_FORMAT = "@";
|
|
2750
2893
|
function styleHeader(sheet) {
|
|
2751
2894
|
const header = sheet.getRow(HEADER_ROW);
|
|
2752
2895
|
HEADERS.forEach((label, index) => {
|
|
@@ -2762,6 +2905,7 @@ function applyColumnGeometry(sheet) {
|
|
|
2762
2905
|
sheet.getColumn(Number(column)).width = width;
|
|
2763
2906
|
}
|
|
2764
2907
|
sheet.getColumn(COLUMN.sourceHash).hidden = true;
|
|
2908
|
+
sheet.getColumn(COLUMN.translation).numFmt = TEXT_NUMBER_FORMAT;
|
|
2765
2909
|
sheet.views = [{ state: "frozen", ySplit: HEADER_ROW }];
|
|
2766
2910
|
}
|
|
2767
2911
|
function writeRow(sheet, sheetRow) {
|
|
@@ -2770,7 +2914,9 @@ function writeRow(sheet, sheetRow) {
|
|
|
2770
2914
|
row.getCell(COLUMN.source).value = sheetRow.source;
|
|
2771
2915
|
row.getCell(COLUMN.current).value = sheetRow.currentTarget;
|
|
2772
2916
|
row.getCell(COLUMN.status).value = sheetRow.status;
|
|
2773
|
-
row.getCell(COLUMN.translation)
|
|
2917
|
+
const translationCell = row.getCell(COLUMN.translation);
|
|
2918
|
+
translationCell.numFmt = TEXT_NUMBER_FORMAT;
|
|
2919
|
+
translationCell.value = sheetRow.translation === "" ? null : sheetRow.translation;
|
|
2774
2920
|
row.getCell(COLUMN.sourceHash).value = sheetRow.sourceHash;
|
|
2775
2921
|
for (let column = COLUMN.key; column <= COLUMN.sourceHash; column += 1) {
|
|
2776
2922
|
const cell = row.getCell(column);
|
|
@@ -2856,6 +3002,34 @@ function declaredSize(file) {
|
|
|
2856
3002
|
const size = data?.uncompressedSize;
|
|
2857
3003
|
return typeof size === "number" && Number.isFinite(size) ? size : void 0;
|
|
2858
3004
|
}
|
|
3005
|
+
async function streamEntryBounded(file, remaining) {
|
|
3006
|
+
const chunks = [];
|
|
3007
|
+
let raw = 0;
|
|
3008
|
+
let oversize = false;
|
|
3009
|
+
const source = new stream.Readable({ objectMode: true }).wrap(
|
|
3010
|
+
file.nodeStream("nodebuffer")
|
|
3011
|
+
);
|
|
3012
|
+
try {
|
|
3013
|
+
for await (const chunk2 of source) {
|
|
3014
|
+
const buf = Buffer.isBuffer(chunk2) ? chunk2 : Buffer.from(chunk2);
|
|
3015
|
+
raw += buf.length;
|
|
3016
|
+
if (raw > remaining) {
|
|
3017
|
+
oversize = true;
|
|
3018
|
+
break;
|
|
3019
|
+
}
|
|
3020
|
+
chunks.push(buf);
|
|
3021
|
+
}
|
|
3022
|
+
} catch {
|
|
3023
|
+
throw new ExchangeError("WORKBOOK_INVALID", "A workbook entry could not be decompressed.");
|
|
3024
|
+
}
|
|
3025
|
+
if (oversize) {
|
|
3026
|
+
throw new ExchangeError(
|
|
3027
|
+
"WORKBOOK_INVALID",
|
|
3028
|
+
"The workbook decompresses to more than the permitted maximum bytes."
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
3032
|
+
}
|
|
2859
3033
|
async function guardWorkbookBytes(bytes, limits) {
|
|
2860
3034
|
let zip;
|
|
2861
3035
|
try {
|
|
@@ -2885,12 +3059,8 @@ async function guardWorkbookBytes(bytes, limits) {
|
|
|
2885
3059
|
}
|
|
2886
3060
|
let actualTotal = 0;
|
|
2887
3061
|
for (const file of files) {
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
content = await file.async("string");
|
|
2891
|
-
} catch {
|
|
2892
|
-
throw new ExchangeError("WORKBOOK_INVALID", "A workbook entry could not be decompressed.");
|
|
2893
|
-
}
|
|
3062
|
+
const remaining = limits.maxDecompressedBytes - actualTotal;
|
|
3063
|
+
const content = await streamEntryBounded(file, remaining);
|
|
2894
3064
|
actualTotal += Buffer.byteLength(content, "utf8");
|
|
2895
3065
|
if (actualTotal > limits.maxDecompressedBytes) {
|
|
2896
3066
|
throw new ExchangeError(
|
|
@@ -3040,13 +3210,6 @@ function buildRows(source, target, baseline, includeUnchanged) {
|
|
|
3040
3210
|
}
|
|
3041
3211
|
return [...rows].sort((a, b) => a.key < b.key ? -1 : 1);
|
|
3042
3212
|
}
|
|
3043
|
-
function selectedLocales2(config, requested) {
|
|
3044
|
-
if (requested === void 0) {
|
|
3045
|
-
return config.targetLocales;
|
|
3046
|
-
}
|
|
3047
|
-
const wanted = new Set(requested);
|
|
3048
|
-
return config.targetLocales.filter((locale) => wanted.has(locale));
|
|
3049
|
-
}
|
|
3050
3213
|
async function exportWorkbook(input, deps = {}) {
|
|
3051
3214
|
const config = input.config;
|
|
3052
3215
|
const cwd = input.cwd ?? process.cwd();
|
|
@@ -3054,7 +3217,7 @@ async function exportWorkbook(input, deps = {}) {
|
|
|
3054
3217
|
const adapter = selectAdapter(config.format, deps.adapterRegistry);
|
|
3055
3218
|
const source = await readSource(config, cwd, fs, adapter);
|
|
3056
3219
|
const lock = await readLockFile(lockFilePath(cwd), fs);
|
|
3057
|
-
const locales =
|
|
3220
|
+
const locales = selectLocales(config, input.locales);
|
|
3058
3221
|
const sheets = await Promise.all(
|
|
3059
3222
|
locales.map(async (locale) => {
|
|
3060
3223
|
const target = await readTarget3(cwd, config, adapter, fs, locale);
|
|
@@ -3105,9 +3268,20 @@ function judge(row, sourceEntry, adapter) {
|
|
|
3105
3268
|
}
|
|
3106
3269
|
return void 0;
|
|
3107
3270
|
}
|
|
3271
|
+
function trackBlankDrift(row, params, buckets) {
|
|
3272
|
+
const sourceEntry = params.source.entries.get(row.key);
|
|
3273
|
+
if (sourceEntry === void 0) {
|
|
3274
|
+
return;
|
|
3275
|
+
}
|
|
3276
|
+
const priorHash = params.baseline.get(row.key);
|
|
3277
|
+
if (priorHash !== void 0 && priorHash !== contentHash(sourceEntry)) {
|
|
3278
|
+
buckets.blankDrifted.add(row.key);
|
|
3279
|
+
}
|
|
3280
|
+
}
|
|
3108
3281
|
function classifyRows(params, buckets) {
|
|
3109
3282
|
for (const row of params.sheet.rows) {
|
|
3110
3283
|
if (row.translation === "") {
|
|
3284
|
+
trackBlankDrift(row, params, buckets);
|
|
3111
3285
|
continue;
|
|
3112
3286
|
}
|
|
3113
3287
|
if (isUnknownKey(row, params.source, params.target)) {
|
|
@@ -3126,9 +3300,20 @@ function classifyRows(params, buckets) {
|
|
|
3126
3300
|
}
|
|
3127
3301
|
}
|
|
3128
3302
|
}
|
|
3303
|
+
function blankRowBaselineNotice(count) {
|
|
3304
|
+
return {
|
|
3305
|
+
code: "BLANK_ROW_BASELINE_RETAINED",
|
|
3306
|
+
message: `${count} row(s) were left blank for a key whose source changed since the row's baseline was recorded; the prior baseline was kept so the drift keeps being reported.`
|
|
3307
|
+
};
|
|
3308
|
+
}
|
|
3129
3309
|
function importLocale(params) {
|
|
3130
3310
|
const diff2 = diffResources(params.source, params.target, { baseline: params.baseline });
|
|
3131
|
-
const buckets = {
|
|
3311
|
+
const buckets = {
|
|
3312
|
+
accepted: /* @__PURE__ */ new Map(),
|
|
3313
|
+
mismatches: [],
|
|
3314
|
+
withheld: /* @__PURE__ */ new Set(),
|
|
3315
|
+
blankDrifted: /* @__PURE__ */ new Set()
|
|
3316
|
+
};
|
|
3132
3317
|
classifyRows(params, buckets);
|
|
3133
3318
|
const rowKeys = new Set(params.sheet.rows.map((row) => row.key));
|
|
3134
3319
|
const invalidIcuSource = [...new Set(params.sourceInvalidIcuKeys)].filter((key) => rowKeys.has(key)).sort();
|
|
@@ -3142,9 +3327,11 @@ function importLocale(params) {
|
|
|
3142
3327
|
pruned: [],
|
|
3143
3328
|
invalidIcuSource,
|
|
3144
3329
|
integrityMismatches: [...buckets.mismatches].sort(),
|
|
3330
|
+
// A workbook import never calls a provider, so a provider-call failure cannot occur here.
|
|
3331
|
+
providerFailures: [],
|
|
3145
3332
|
// Plural generation is a translate-flow concern; the manual workbook import never generates forms.
|
|
3146
3333
|
generated: [],
|
|
3147
|
-
notices: []
|
|
3334
|
+
notices: buckets.blankDrifted.size > 0 ? [blankRowBaselineNotice(buckets.blankDrifted.size)] : []
|
|
3148
3335
|
};
|
|
3149
3336
|
return { summary, accepted: buckets.accepted, withheld: buckets.withheld };
|
|
3150
3337
|
}
|
|
@@ -3178,21 +3365,19 @@ function mergeAccepted(target, accepted) {
|
|
|
3178
3365
|
}
|
|
3179
3366
|
return merged;
|
|
3180
3367
|
}
|
|
3181
|
-
function computeLockEntries2(source, merged, baseline,
|
|
3368
|
+
function computeLockEntries2(source, merged, baseline, accepted) {
|
|
3182
3369
|
const entries = {};
|
|
3183
3370
|
for (const key of merged.keys()) {
|
|
3184
3371
|
const sourceEntry = source.entries.get(key);
|
|
3185
3372
|
if (sourceEntry === void 0) {
|
|
3186
3373
|
continue;
|
|
3187
3374
|
}
|
|
3188
|
-
if (
|
|
3189
|
-
|
|
3190
|
-
if (prior !== void 0) {
|
|
3191
|
-
entries[key] = prior;
|
|
3192
|
-
}
|
|
3375
|
+
if (accepted.has(key)) {
|
|
3376
|
+
entries[key] = contentHash(sourceEntry);
|
|
3193
3377
|
continue;
|
|
3194
3378
|
}
|
|
3195
|
-
|
|
3379
|
+
const prior = baseline.get(key);
|
|
3380
|
+
entries[key] = prior !== void 0 ? prior : contentHash(sourceEntry);
|
|
3196
3381
|
}
|
|
3197
3382
|
return entries;
|
|
3198
3383
|
}
|
|
@@ -3205,7 +3390,7 @@ async function runSheet(ctx, sheet, lock) {
|
|
|
3205
3390
|
}
|
|
3206
3391
|
const target = await readTarget4(ctx.cwd, ctx.config, ctx.adapter, ctx.fs, sheet.locale);
|
|
3207
3392
|
const baseline = baselineFor(lock, sheet.locale);
|
|
3208
|
-
const { summary, accepted
|
|
3393
|
+
const { summary, accepted } = importLocale({
|
|
3209
3394
|
sheet,
|
|
3210
3395
|
source: ctx.source,
|
|
3211
3396
|
target,
|
|
@@ -3229,7 +3414,7 @@ async function runSheet(ctx, sheet, lock) {
|
|
|
3229
3414
|
path
|
|
3230
3415
|
);
|
|
3231
3416
|
}
|
|
3232
|
-
return { summary, lockEntries: computeLockEntries2(ctx.source, merged, baseline,
|
|
3417
|
+
return { summary, lockEntries: computeLockEntries2(ctx.source, merged, baseline, accepted) };
|
|
3233
3418
|
}
|
|
3234
3419
|
async function importWorkbook(input, deps = {}) {
|
|
3235
3420
|
const config = input.config;
|