champollion 0.3.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -26
- package/bin/cli.js +53 -5
- package/index.js +63 -2
- package/lib/api-key.js +17 -4
- package/lib/autofix.js +82 -37
- package/lib/bridge/method_bridge.py +15 -3
- package/lib/cards/reader.js +34 -0
- package/lib/cards/remote.js +15 -0
- package/lib/cards/search-names.js +178 -0
- package/lib/command-help.js +286 -85
- package/lib/commands/audit.js +10 -3
- package/lib/commands/card.js +583 -226
- package/lib/commands/doctor.js +54 -18
- package/lib/commands/help.js +37 -32
- package/lib/commands/init.js +1709 -101
- package/lib/commands/integrity.js +127 -40
- package/lib/commands/leaderboard.js +187 -67
- package/lib/commands/models.js +9 -2
- package/lib/commands/provenance.js +7 -2
- package/lib/commands/recommend.js +43 -14
- package/lib/commands/register-corpus.js +632 -125
- package/lib/commands/seal-corpus.js +1 -1
- package/lib/commands/status.js +564 -27
- package/lib/commands/submit.js +17 -12
- package/lib/commands/sync.js +31 -7
- package/lib/commands/tm.js +15 -9
- package/lib/commands/verify.js +27 -3
- package/lib/commands/wrap.js +63 -5
- package/lib/commands/xliff.js +135 -64
- package/lib/commercial-eligibility.js +1 -1
- package/lib/config.js +230 -17
- package/lib/content-estimate.js +96 -0
- package/lib/content-refusals.js +284 -0
- package/lib/content-review.js +372 -0
- package/lib/content-sync.js +1136 -345
- package/lib/content.js +94 -7
- package/lib/corpus-registration.mjs +194 -35
- package/lib/cost-label.js +29 -0
- package/lib/cost-report.js +727 -79
- package/lib/diff.js +38 -4
- package/lib/docusaurus-sync.js +975 -262
- package/lib/edit-distance.js +31 -0
- package/lib/fallback.js +1102 -0
- package/lib/file-scope.js +106 -0
- package/lib/flatten.js +80 -3
- package/lib/flutter-locales.js +124 -0
- package/lib/format.js +266 -12
- package/lib/hash.js +146 -21
- package/lib/icu-structure.js +929 -0
- package/lib/integrity.js +223 -75
- package/lib/language-pair.js +157 -0
- package/lib/lint.js +78 -16
- package/lib/local-only-marks.js +106 -0
- package/lib/locale-layout.js +1103 -0
- package/lib/locale-state.js +575 -0
- package/lib/methods/anthropic.js +5 -0
- package/lib/methods/apertium.js +6 -3
- package/lib/methods/api.js +138 -25
- package/lib/methods/base.js +17 -0
- package/lib/methods/coaching-data.js +153 -0
- package/lib/methods/content-separator.js +43 -0
- package/lib/methods/deepl.js +1 -1
- package/lib/methods/direct-llm.js +257 -103
- package/lib/methods/external.js +146 -63
- package/lib/methods/gemini.js +1 -0
- package/lib/methods/google-translate.js +1 -0
- package/lib/methods/http-utils.js +41 -0
- package/lib/methods/libretranslate.js +7 -2
- package/lib/methods/llm-coached.js +71 -129
- package/lib/methods/llm.js +87 -33
- package/lib/methods/local.js +93 -10
- package/lib/methods/microsoft-translator.js +1 -2
- package/lib/methods/openai.js +4 -2
- package/lib/methods/openrouter-client.js +20 -19
- package/lib/methods/openrouter-pricing.js +150 -13
- package/lib/methods/prompt-methods.js +20 -0
- package/lib/methods/provider-pricing.js +42 -1
- package/lib/methods/request-capture.js +104 -0
- package/lib/methods/tilde.js +1 -1
- package/lib/methods/translated.js +1 -2
- package/lib/missing-key.js +93 -0
- package/lib/models.js +102 -55
- package/lib/name-rules.js +32 -0
- package/lib/named-keys.js +172 -0
- package/lib/no-translate.js +4 -3
- package/lib/output.js +160 -19
- package/lib/pairs.js +586 -30
- package/lib/placeholders.js +394 -0
- package/lib/plugins.js +8 -0
- package/lib/plural-gap-redo.js +109 -0
- package/lib/plurals.js +323 -0
- package/lib/po.js +1187 -0
- package/lib/public-catalogue.js +74 -0
- package/lib/recommend.js +527 -32
- package/lib/redo.js +95 -0
- package/lib/refusal-category.js +44 -0
- package/lib/registers.js +255 -11
- package/lib/repair-script.js +20 -13
- package/lib/scripts.js +6 -1
- package/lib/seal.mjs +4 -3
- package/lib/sealed-qualifier.mjs +1 -1
- package/lib/segment.js +24 -8
- package/lib/seo.js +19 -9
- package/lib/serve.js +43 -6
- package/lib/shared-output-seed.js +164 -0
- package/lib/source-contexts.js +39 -0
- package/lib/submit.mjs +57 -5
- package/lib/sync.js +2923 -474
- package/lib/terminology.js +13 -4
- package/lib/tm-evict.js +179 -0
- package/lib/tm-seed.js +5 -2
- package/lib/tm.js +818 -36
- package/lib/translate-pair.js +639 -34
- package/lib/translate.js +78 -5
- package/lib/types.js +22 -3
- package/lib/validate.js +996 -21
- package/lib/verify.js +1332 -104
- package/lib/watch.js +32 -13
- package/lib/xliff.js +44 -3
- package/package.json +1 -1
- package/shared/CORPORA-CARDS.md +2 -0
- package/shared/cards-fallback.json +1 -1
- package/shared/curated-orthography-conventions.json +26 -8
- package/shared/gettext-plural-forms.json +45 -0
- package/shared/licenses.json +99 -3
- package/shared/method-registry.json +2 -0
- package/shared/metric-registry.json +96 -18
- package/shared/retired-model-aliases.json +9 -0
- package/shared/schemas/champollion-plugin.schema.json +4 -0
- package/shared/schemas/corpora-card.schema.json +8 -2
- package/shared/schemas/human-services.schema.json +1 -1
- package/shared/schemas/method-index-record.schema.json +67 -0
- package/shared/schemas/method-registry.schema.json +4 -0
- package/shared/schemas/metric-registry.schema.json +55 -1
- package/shared/schemas/retired-model-aliases.schema.json +32 -0
- package/shared/docent/corpus.json +0 -11739
- package/shared/model-aliases.json +0 -7
- package/shared/schemas/model-aliases.schema.json +0 -27
package/lib/validate.js
CHANGED
|
@@ -21,8 +21,26 @@
|
|
|
21
21
|
* source-relative caps)
|
|
22
22
|
* 2. Length ratio check (source vs translated length)
|
|
23
23
|
* 3. Script compliance (non-Latin locales must produce non-ASCII)
|
|
24
|
-
* 4. Source echo check (translated value ≠ source value
|
|
24
|
+
* 4. Source echo check (translated value ≠ source value — and, for a
|
|
25
|
+
* source of 3+ words, not the source with only case, diacritics,
|
|
26
|
+
* spacing or invisible characters changed: see isDisguisedEcho)
|
|
25
27
|
* 5. Content preservation (the output is not the source, hollowed out)
|
|
28
|
+
* 6. ICU / placeholder structure (lib/icu-structure.js): argument names,
|
|
29
|
+
* plural/select keywords, selectors, # and printf conversions are
|
|
30
|
+
* code — only the text inside the branches may change; markup tags
|
|
31
|
+
* are code too (checkMarkup: same tags opened, closed, nested); and
|
|
32
|
+
* i18next {{…}} / single-brace / tag tokens by the rule verify reports
|
|
33
|
+
* them by (lib/placeholders.js placeholderChanges); and a sentence
|
|
34
|
+
* break the translation inserts right beside a placeholder where the
|
|
35
|
+
* source has none (placeholderSentenceBreaks)
|
|
36
|
+
*
|
|
37
|
+
* 7. Plural forms (lib/icu-structure.js pluralGaps): a plural message
|
|
38
|
+
* that passed everything else but has no branch for a category the
|
|
39
|
+
* target language uses for ordinary counts (Russian few/many) is a
|
|
40
|
+
* SOFT failure (`pluralGap`): the caller asks once more, naming the
|
|
41
|
+
* missing forms; a second answer without them is accepted
|
|
42
|
+
* (acceptPluralGaps) and reported — never a retry loop, and never a
|
|
43
|
+
* form made up by the tool.
|
|
26
44
|
*
|
|
27
45
|
* Keys that fail any check are removed and logged as [GATE] failures.
|
|
28
46
|
* The caller receives only validated translations.
|
|
@@ -33,6 +51,9 @@
|
|
|
33
51
|
*/
|
|
34
52
|
|
|
35
53
|
import { getAllLanguageCodes, getLanguageCard } from './registers.js';
|
|
54
|
+
import { output } from './output.js';
|
|
55
|
+
import { checkICUStructure, checkMarkup, pluralGaps, describeCategories, pluralBranchPairs, pluralBranchTexts } from './icu-structure.js';
|
|
56
|
+
import { placeholderChanges, placeholderSentenceBreaks, PLACEHOLDER_SYNTAX_NAMES } from './placeholders.js';
|
|
36
57
|
|
|
37
58
|
/**
|
|
38
59
|
* Locales whose scripts are predominantly non-Latin.
|
|
@@ -242,11 +263,313 @@ function checkContentPreservation(source, translated, minRetention = DEFAULT_THR
|
|
|
242
263
|
};
|
|
243
264
|
}
|
|
244
265
|
|
|
266
|
+
// ---------------------------------------------------------------------------
|
|
267
|
+
// Disguised echoes — the source handed back with accents sprinkled on
|
|
268
|
+
// ---------------------------------------------------------------------------
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Fewest words (whitespace-delimited, each carrying a letter) a source needs
|
|
272
|
+
* before a FOLDED match counts as an echo. Below it, only exact equality does.
|
|
273
|
+
*
|
|
274
|
+
* WHY A FLOOR: in a Latin-script target a short legitimate translation can
|
|
275
|
+
* differ from English only by accents — 'cafe' → 'café', 'Resume' → 'Résumé',
|
|
276
|
+
* 'Cafe Menu' → 'Café Menu' — and refusing those would cost users a retry and
|
|
277
|
+
* then the key. Three independent words that all survive translation
|
|
278
|
+
* unchanged apart from accents and case is what a copy looks like.
|
|
279
|
+
*/
|
|
280
|
+
const MIN_FOLDED_ECHO_WORDS = 3;
|
|
281
|
+
|
|
282
|
+
/** Combining marks (Mn) and invisible format characters (Cf). */
|
|
283
|
+
const FOLD_STRIP = /[\p{Mn}\p{Cf}]/gu;
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* The form two strings are compared in to decide "is this output the source?".
|
|
287
|
+
*
|
|
288
|
+
* Twin of the fold in arena/mt_eval_harness/text_compare.py (its steps 1–4,
|
|
289
|
+
* `_fold` / token_compare_key, plus its whitespace collapse), so the CLI gate
|
|
290
|
+
* sees through the same disguises the harness does:
|
|
291
|
+
* 1. NFKD — precomposed letters split into base + combining mark, and
|
|
292
|
+
* compatibility forms (fullwidth letters, ligatures) fold to plain ones;
|
|
293
|
+
* 2. case folding (JavaScript has no casefold(); upper-then-lower makes the
|
|
294
|
+
* same equality decisions for this purpose: 'ß' and 'ss' meet, and the
|
|
295
|
+
* Greek sigma forms meet — JS lands on the contextual 'ς' where Python
|
|
296
|
+
* lands on 'σ', but both sides of a comparison land together), then
|
|
297
|
+
* NFKD again because a case mapping can produce a decomposable
|
|
298
|
+
* character;
|
|
299
|
+
* 3. drop combining marks (Mn: 'á' → 'a'; 'ŋ' is a letter and stays) and
|
|
300
|
+
* format characters (Cf: zero-width space/joiners, bidi marks, soft
|
|
301
|
+
* hyphen) — they change the bytes, not the text a reader sees;
|
|
302
|
+
* 4. collapse whitespace runs and trim.
|
|
303
|
+
* Punctuation is compared AS WRITTEN (the harness's per-token rule). Its
|
|
304
|
+
* whole-string form also turns punctuation into spaces; this gate does not —
|
|
305
|
+
* a gate that REFUSES output (and spends a retry) keeps the narrower fold, so
|
|
306
|
+
* a copy that also edits its punctuation is not caught here, as before.
|
|
307
|
+
*
|
|
308
|
+
* @param {string} text
|
|
309
|
+
* @returns {string}
|
|
310
|
+
*/
|
|
311
|
+
function foldForEchoCompare(text) {
|
|
312
|
+
return String(text)
|
|
313
|
+
.normalize('NFKD')
|
|
314
|
+
.toUpperCase()
|
|
315
|
+
.toLowerCase()
|
|
316
|
+
.normalize('NFKD')
|
|
317
|
+
.replace(FOLD_STRIP, '')
|
|
318
|
+
.replace(/\s+/gu, ' ')
|
|
319
|
+
.trim();
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* How many words carrying a letter `text` has — after setting aside what is
|
|
324
|
+
* code rather than words: ICU/brace placeholders ("{count}"), printf
|
|
325
|
+
* conversions ("%s", "%1$d") and markup tags ("<b>", "</a>").
|
|
326
|
+
*
|
|
327
|
+
* @param {string} text
|
|
328
|
+
* @returns {number}
|
|
329
|
+
*/
|
|
330
|
+
function letterWordCount(text) {
|
|
331
|
+
return String(text)
|
|
332
|
+
.replace(/\{[^}]*\}/g, ' ')
|
|
333
|
+
.replace(/%(\d+\$)?[-+ 0#]*\d*(\.\d+)?[a-zA-Z@]/g, ' ')
|
|
334
|
+
.replace(/<[^>]*>/g, ' ')
|
|
335
|
+
.split(/\s+/u)
|
|
336
|
+
.filter((w) => /\p{L}/u.test(w))
|
|
337
|
+
.length;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Is `translated` the source in disguise — identical once case, diacritics,
|
|
342
|
+
* spacing and invisible format characters are folded away, without being
|
|
343
|
+
* byte-identical? ('Thank you very much!' → 'Thánk yóú véry múch!')
|
|
344
|
+
*
|
|
345
|
+
* Only for a source of MIN_FOLDED_ECHO_WORDS+ words with letters; a shorter
|
|
346
|
+
* source keeps exact equality as its only echo test, exactly as before.
|
|
347
|
+
* An exact copy is NOT a disguised echo — it keeps its own rule (and the
|
|
348
|
+
* short-name lane), so this predicate changes nothing for it.
|
|
349
|
+
*
|
|
350
|
+
* Why the short-name lane does not apply here: that lane exists because a
|
|
351
|
+
* short value is often a NAME correctly kept as written ("GitHub", "Curtis
|
|
352
|
+
* Forbes"). A disguised copy is by construction not kept as written — it was
|
|
353
|
+
* altered without being translated — so it is refused like any long echo.
|
|
354
|
+
*
|
|
355
|
+
* Exported so an on-disk auditor that adopts it applies the same floor.
|
|
356
|
+
*
|
|
357
|
+
* @param {string} source
|
|
358
|
+
* @param {string} translated
|
|
359
|
+
* @returns {boolean}
|
|
360
|
+
*/
|
|
361
|
+
function isDisguisedEcho(source, translated) {
|
|
362
|
+
if (typeof source !== 'string' || typeof translated !== 'string') return false;
|
|
363
|
+
if (translated === source) return false;
|
|
364
|
+
if (letterWordCount(source) < MIN_FOLDED_ECHO_WORDS) return false;
|
|
365
|
+
return foldForEchoCompare(translated) === foldForEchoCompare(source);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* The first echo among a plural message's branches (validateTranslations
|
|
370
|
+
* check 2c), as a failure record's fields, or null.
|
|
371
|
+
*
|
|
372
|
+
* @returns {{ reason: string, disguisedEcho?: true, nameOrLabel?: true }|null}
|
|
373
|
+
*/
|
|
374
|
+
function pluralBranchEcho(source, translated, { isNonLatin = false, requireNonLatin = true, acceptLatinNames = false, protectedTerms = [] } = {}) {
|
|
375
|
+
const pairs = pluralBranchPairs(source, translated);
|
|
376
|
+
if (pairs.length === 0) return null;
|
|
377
|
+
const exact = [];
|
|
378
|
+
const disguised = [];
|
|
379
|
+
const nameLike = [];
|
|
380
|
+
for (const p of pairs) {
|
|
381
|
+
const src = p.source.trim();
|
|
382
|
+
const tgt = p.translated.trim();
|
|
383
|
+
if (!/\p{L}/u.test(src) || isProtectedTermValue(tgt, protectedTerms)) continue;
|
|
384
|
+
if (tgt === src) {
|
|
385
|
+
const asciiRatio = src.replace(/[^\x20-\x7E]/g, '').length / Math.max(src.length, 1);
|
|
386
|
+
const isShortAscii = src.length <= 30 && asciiRatio > 0.8;
|
|
387
|
+
if (!isShortAscii) exact.push(p.selector);
|
|
388
|
+
else if (isNonLatin && requireNonLatin && !acceptLatinNames) nameLike.push(p.selector);
|
|
389
|
+
} else if (isDisguisedEcho(src, tgt)) {
|
|
390
|
+
disguised.push(p.selector);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
const forms = (sels) => [...new Set(sels)].map(x => `"${x}"`).join(', ');
|
|
394
|
+
if (disguised.length > 0) {
|
|
395
|
+
return { reason: `${DISGUISED_ECHO} — in the plural form(s) ${forms(disguised)}`, disguisedEcho: true };
|
|
396
|
+
}
|
|
397
|
+
if (exact.length > 0) {
|
|
398
|
+
return { reason: `source echo (identical to English) in the plural form(s) ${forms(exact)}` };
|
|
399
|
+
}
|
|
400
|
+
if (nameLike.length > 0) {
|
|
401
|
+
return { reason: `${LATIN_NAME_OR_LABEL} (plural form(s) ${forms(nameLike)})`, nameOrLabel: true };
|
|
402
|
+
}
|
|
403
|
+
return null;
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** Gate reason for a disguised echo. */
|
|
407
|
+
const DISGUISED_ECHO =
|
|
408
|
+
'source echo (the source handed back with only case, accents, spacing or invisible characters changed — not a translation)';
|
|
409
|
+
|
|
245
410
|
// A deliberately repetitive source ("Every language, into every language.")
|
|
246
411
|
// licenses an equally repetitive translation: the effective caps are raised
|
|
247
412
|
// to the source's own measured repetition plus this margin.
|
|
248
413
|
const REPETITION_SOURCE_MARGIN = 0.10;
|
|
249
414
|
|
|
415
|
+
/** Gate reason for a short Latin-script value in a non-Latin target. */
|
|
416
|
+
const LATIN_NAME_OR_LABEL = 'kept in Latin script — a name, or a label left untranslated?';
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Is this value made ONLY of names the project declared (config.protectedTerms)?
|
|
420
|
+
*
|
|
421
|
+
* Strips every declared term, then ICU placeholders, digits, punctuation,
|
|
422
|
+
* symbols and whitespace; nothing left means the value is a name (or names)
|
|
423
|
+
* kept as written — correct in every script, never an "untranslated" error.
|
|
424
|
+
* "Curtis Forbes", "Game Day Suits", "Curtis Forbes · Game Day Suits" all
|
|
425
|
+
* qualify; "Curtis Forbes's portfolio" does not (the rest needs translating).
|
|
426
|
+
*
|
|
427
|
+
* @param {string} value
|
|
428
|
+
* @param {string[]} [protectedTerms]
|
|
429
|
+
* @returns {boolean}
|
|
430
|
+
*/
|
|
431
|
+
function isProtectedTermValue(value, protectedTerms = []) {
|
|
432
|
+
if (typeof value !== 'string' || !protectedTerms || protectedTerms.length === 0) return false;
|
|
433
|
+
let rest = value;
|
|
434
|
+
let matched = false;
|
|
435
|
+
// Longest first, so "Game Day Suits Ltd" is removed before "Game Day Suits".
|
|
436
|
+
for (const term of [...protectedTerms].sort((a, b) => b.length - a.length)) {
|
|
437
|
+
if (typeof term === 'string' && term && rest.includes(term)) {
|
|
438
|
+
rest = rest.split(term).join(' ');
|
|
439
|
+
matched = true;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
if (!matched) return false;
|
|
443
|
+
return rest.replace(/\{[^}]*\}/g, '').replace(/[\d\s\p{P}\p{S}]/gu, '').length === 0;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* The gate's version. A refusal is remembered with the version that made it
|
|
448
|
+
* (lib/content-refusals.js, lib/locale-state.js), and a hold made by an
|
|
449
|
+
* earlier gate lifts by itself: what an over-strict gate refused is asked
|
|
450
|
+
* again once the gate is fixed, instead of staying in the source language
|
|
451
|
+
* until someone names it for a redo. Bump it whenever a check is loosened.
|
|
452
|
+
* 2 — 2026-10-05: names with citations/anchors, reference entries, tables,
|
|
453
|
+
* short titles and source-shown fullwidth letters stopped being refused.
|
|
454
|
+
*/
|
|
455
|
+
const GATE_VERSION = 2;
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Refusals that may be the model keeping text that is correct as written —
|
|
459
|
+
* a name, title, citation, identifier or code: the source handed back, or
|
|
460
|
+
* Latin script kept in a non-Latin target. Told why and asked again, a model
|
|
461
|
+
* that gives the same answer twice is taken at its word (lib/fallback.js
|
|
462
|
+
* retryRefusedBlocks), as the key-value lane takes a name it is asked about
|
|
463
|
+
* twice (LATIN_NAME_OR_LABEL). Every other refusal (repetition, length,
|
|
464
|
+
* hollowing, damaged markup, fullwidth disguise) is never accepted this way.
|
|
465
|
+
*
|
|
466
|
+
* @param {string|null} reason
|
|
467
|
+
* @returns {boolean}
|
|
468
|
+
*/
|
|
469
|
+
function isKeepAsWrittenFault(reason) {
|
|
470
|
+
if (typeof reason !== 'string') return false;
|
|
471
|
+
return reason.startsWith('source echo')
|
|
472
|
+
|| reason === LATIN_NAME_OR_LABEL
|
|
473
|
+
|| reason.startsWith('wrong script (ASCII-only');
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
/**
|
|
477
|
+
* How much longer than its source a value may be before the length ratio
|
|
478
|
+
* counts. The ratio is meaningless on a tiny source: "FAQ" → "Preguntas
|
|
479
|
+
* frecuentes" is 6.7x and correct (dogfood 2026-10-05, a docs page title),
|
|
480
|
+
* while "Feast" → a whole sentence still grows by far more than this.
|
|
481
|
+
*/
|
|
482
|
+
const SHORT_SOURCE_LENGTH_SLACK = 20;
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* The part of a string that would need translating, for the "short name kept
|
|
486
|
+
* as written" rule: inline code, quoted strings, parentheticals and Markdown
|
|
487
|
+
* block markers removed. "### METEOR (Banerjee & Lavie, 2005)" is the name
|
|
488
|
+
* METEOR, and "## Hugo (TOML / YAML / Markdown)" the name Hugo — both were
|
|
489
|
+
* refused as source echo on the whole string's length (dogfood 2026-10-05).
|
|
490
|
+
*
|
|
491
|
+
* @param {string} text
|
|
492
|
+
* @returns {string}
|
|
493
|
+
*/
|
|
494
|
+
function nameCore(text) {
|
|
495
|
+
return String(text)
|
|
496
|
+
.replace(/^[ \t]*(?:#{1,6}[ \t]+|[-*+][ \t]+|\d{1,9}[.)][ \t]+)/, '')
|
|
497
|
+
.replace(/\{#[^}\s]+\}\s*$/, ' ')
|
|
498
|
+
.replace(/`[^`]*`/g, ' ')
|
|
499
|
+
.replace(/"[^"]*"|“[^”]*”|「[^」]*」/g, ' ')
|
|
500
|
+
.replace(/\([^()]*\)|([^()]*)/g, ' ')
|
|
501
|
+
.replace(/\s+/g, ' ')
|
|
502
|
+
.trim();
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* A reference-list entry — numbered, with a year or a link, and a quoted or
|
|
507
|
+
* italic title: "4. Snover, M., … (2006). \"A Study of …\" *Proceedings …*".
|
|
508
|
+
* Cited works are kept as published, so the same text back is correct, not
|
|
509
|
+
* an echo (dogfood 2026-10-05: twelve references of the scoring spec were
|
|
510
|
+
* refused and published with the fallback prefix).
|
|
511
|
+
*
|
|
512
|
+
* @param {string} text
|
|
513
|
+
* @returns {boolean}
|
|
514
|
+
*/
|
|
515
|
+
function isBibliographicEntry(text) {
|
|
516
|
+
const t = String(text).trim();
|
|
517
|
+
// A reference list in one block: every entry must be one.
|
|
518
|
+
if (t.includes('\n')) {
|
|
519
|
+
const lines = t.split('\n').map(l => l.trim()).filter(Boolean);
|
|
520
|
+
return lines.length > 1 && lines.every(l => !l.includes('\n') && isBibliographicEntry(l));
|
|
521
|
+
}
|
|
522
|
+
return /^(?:\[?[A-Z]?\d{1,3}\]?[.)]?|[-*+])\s+\S/.test(t)
|
|
523
|
+
&& (/\(\d{4}[a-z]?\)/.test(t) || /\]\(https?:\/\//.test(t))
|
|
524
|
+
&& (/"[^"]{8,}"|“[^”]{8,}”/.test(t) || /\*[^*]{8,}\*/.test(t));
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* A Markdown table measured as its cells' text: delimiter rows dropped and
|
|
529
|
+
* pipes turned to spaces. The delimiter row ("|---|---|") is repetition by
|
|
530
|
+
* construction, so a table whose model output padded it differently was
|
|
531
|
+
* refused as a repetition hallucination (dogfood 2026-10-05, four tables).
|
|
532
|
+
* Anything that is not a table comes back unchanged.
|
|
533
|
+
*
|
|
534
|
+
* @param {string} text
|
|
535
|
+
* @returns {string}
|
|
536
|
+
*/
|
|
537
|
+
function tableProse(text) {
|
|
538
|
+
const lines = String(text).split('\n');
|
|
539
|
+
if (!lines.some((l) => /^\s*\|/.test(l))) return text;
|
|
540
|
+
return lines
|
|
541
|
+
.filter((l) => !/^\s*\|?(?:\s*:?-{3,}:?\s*\|)+\s*:?-{0,}:?\s*$/.test(l))
|
|
542
|
+
.map((l) => (/^\s*\|/.test(l) ? l.replace(/\|/g, ' ').replace(/[ \t]+/g, ' ').trim() : l))
|
|
543
|
+
.join('\n');
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
/**
|
|
547
|
+
* Why the quality gate refuses one Markdown block or front-matter field, or
|
|
548
|
+
* null when it passes — the key-value gate's own checks (empty, source echo,
|
|
549
|
+
* disguised echo, repetition, length inflation and truncation, hollowing,
|
|
550
|
+
* script), so a short heading turned into a sentence is refused in a
|
|
551
|
+
* newsletter exactly as it is for an app key (Round 7, school persona: "##
|
|
552
|
+
* Feast" became a full sentence and was accepted, while the same output for
|
|
553
|
+
* the key Nav.home was refused at 9.5x). Structure (code, links, markup) is
|
|
554
|
+
* checked by the lanes through their protected placeholders; a short
|
|
555
|
+
* Latin-script value kept as written is a name, as on the key-value lane's
|
|
556
|
+
* second ask (there is no per-block retry to ask once first).
|
|
557
|
+
*
|
|
558
|
+
* @param {string} source - The block or field's source text (placeholders restored)
|
|
559
|
+
* @param {string} value - The translation
|
|
560
|
+
* @param {object} [pairConfig] - The pair (target locale, thresholds, protectedTerms)
|
|
561
|
+
* @returns {string|null}
|
|
562
|
+
*/
|
|
563
|
+
function contentGateFault(source, value, pairConfig = {}) {
|
|
564
|
+
if (typeof source !== 'string' || typeof value !== 'string') return null;
|
|
565
|
+
if (value === source && isBibliographicEntry(source)) return null;
|
|
566
|
+
source = tableProse(source);
|
|
567
|
+
value = tableProse(value);
|
|
568
|
+
const { failures } = validateTranslations({ block: value }, { block: source }, pairConfig || {},
|
|
569
|
+
{ prose: true, acceptLatinNames: true, acceptPluralGaps: true });
|
|
570
|
+
return failures.length > 0 ? failures[0].reason : null;
|
|
571
|
+
}
|
|
572
|
+
|
|
250
573
|
/**
|
|
251
574
|
* Validate a batch of translations and return only passing keys.
|
|
252
575
|
*
|
|
@@ -272,6 +595,17 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
272
595
|
|
|
273
596
|
const validated = {};
|
|
274
597
|
const failures = [];
|
|
598
|
+
// config.protectedTerms rides the pair config (lib/pairs.js).
|
|
599
|
+
const protectedTerms = options.protectedTerms ?? pairConfig.protectedTerms ?? [];
|
|
600
|
+
// Second pass of the name-or-label retry: a short Latin-script value the
|
|
601
|
+
// model returned twice is accepted as a name.
|
|
602
|
+
const acceptLatinNames = options.acceptLatinNames === true;
|
|
603
|
+
// Second pass of the plural-forms retry: accepted, and reported by the caller.
|
|
604
|
+
const acceptPluralGaps = options.acceptPluralGaps === true;
|
|
605
|
+
// A Markdown block or front-matter field (contentGateFault): its structure
|
|
606
|
+
// (code, links, markup) rides protected placeholders, checked by the lane;
|
|
607
|
+
// the ICU/markup/plural checks are for key-value messages.
|
|
608
|
+
const prose = options.prose === true;
|
|
275
609
|
|
|
276
610
|
for (const [key, translated] of Object.entries(translations)) {
|
|
277
611
|
const source = sourceFlat[key] || '';
|
|
@@ -298,20 +632,120 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
298
632
|
continue;
|
|
299
633
|
}
|
|
300
634
|
|
|
635
|
+
// Names the project declared (config.protectedTerms) are correct as
|
|
636
|
+
// written in every language — no echo or script check applies.
|
|
637
|
+
if (isProtectedTermValue(translated, protectedTerms)) {
|
|
638
|
+
validated[key] = translated;
|
|
639
|
+
continue;
|
|
640
|
+
}
|
|
641
|
+
|
|
301
642
|
// Check 2: Source echo — translated value is identical to source.
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
//
|
|
305
|
-
//
|
|
306
|
-
//
|
|
643
|
+
// Short mostly-ASCII strings (≤30 chars) are often names ("GitHub",
|
|
644
|
+
// "Curtis Forbes") that legitimately stay as written — but just as often
|
|
645
|
+
// descriptive labels the model failed to translate ("Translation CLI").
|
|
646
|
+
// - Latin-script targets: accepted (an English label in French reads
|
|
647
|
+
// as a missed translation, not a broken page; verify warns).
|
|
648
|
+
// - Non-Latin targets: rejected ONCE with a name-or-label hint (the
|
|
649
|
+
// caller's feedback retry). If the model returns the same text again
|
|
650
|
+
// it is taken at its word as a name (acceptLatinNames) and cached, so
|
|
651
|
+
// it is never re-billed. One bounded retry — never a retry loop.
|
|
307
652
|
if (translated === source) {
|
|
308
|
-
const
|
|
309
|
-
const
|
|
653
|
+
const core = nameCore(source);
|
|
654
|
+
const asciiRatio = core.replace(/[^\x20-\x7E]/g, '').length / Math.max(core.length, 1);
|
|
655
|
+
// Nothing left once code, quotes and parentheticals go: nothing to translate.
|
|
656
|
+
const isShortAscii = core.length === 0 || (core.length <= 30 && asciiRatio > 0.8);
|
|
310
657
|
if (!isShortAscii) {
|
|
311
658
|
failures.push({ key, reason: 'source echo (identical to English)', value: translated });
|
|
312
659
|
continue;
|
|
313
660
|
}
|
|
314
|
-
|
|
661
|
+
if (isNonLatin && thresholds.requireNonLatin && !acceptLatinNames) {
|
|
662
|
+
failures.push({ key, reason: LATIN_NAME_OR_LABEL, value: translated, nameOrLabel: true });
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
} else if (isDisguisedEcho(source, translated)) {
|
|
666
|
+
// Check 2a: the same echo, disguised — 'Thank you very much!' →
|
|
667
|
+
// 'Thánk yóú véry múch!'. Byte comparison calls it different, and the
|
|
668
|
+
// script check cannot fire (Latin in, Latin out; and in a non-Latin
|
|
669
|
+
// target the accents make it not-ASCII). Same suppressions as the
|
|
670
|
+
// exact check: declared names were accepted above, no-translate keys
|
|
671
|
+
// never reach the gate, letter-free and short (< 3-word) sources keep
|
|
672
|
+
// exact equality. Refused like a long echo — the caller's feedback
|
|
673
|
+
// retry, then the fallback method — never accepted as a name, because
|
|
674
|
+
// a name kept as written is byte-identical (isDisguisedEcho).
|
|
675
|
+
failures.push({ key, reason: DISGUISED_ECHO, value: translated, disguisedEcho: true });
|
|
676
|
+
continue;
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// Check 2b: ICU MessageFormat / placeholder structure. Argument names,
|
|
680
|
+
// plural/select keywords, selectors, # and printf conversions are code:
|
|
681
|
+
// "{cóúnt, plúrál, óné {…} óthér {…}}" breaks the app (next-intl throws,
|
|
682
|
+
// Flutter gen-l10n refuses to build) while reading as a fine French
|
|
683
|
+
// string. Only the text inside the branches may change; a plural may
|
|
684
|
+
// ADD the CLDR categories the target language uses. Runs before the
|
|
685
|
+
// statistical checks so the retry hears the precise reason.
|
|
686
|
+
if (translated !== source && !prose) {
|
|
687
|
+
const icu = checkICUStructure(source, translated, targetLocale);
|
|
688
|
+
if (icu) {
|
|
689
|
+
failures.push({ key, reason: icu.reason, value: translated, icu: true });
|
|
690
|
+
continue;
|
|
691
|
+
}
|
|
692
|
+
// Markup is code as well: every tag opened, closed and nested as in the
|
|
693
|
+
// source ("<strong>Book</strong>" → "<strong>Réserver" breaks the page).
|
|
694
|
+
const markup = checkMarkup(source, translated);
|
|
695
|
+
if (markup) {
|
|
696
|
+
failures.push({ key, reason: markup.reason, value: translated, markup: true });
|
|
697
|
+
continue;
|
|
698
|
+
}
|
|
699
|
+
// The other placeholders verify reports — i18next {{…}} (renamed, lost,
|
|
700
|
+
// or written as a single-brace {name}, which i18next prints as is), a
|
|
701
|
+
// single-brace {name} outside an ICU message, a tag token — by the one
|
|
702
|
+
// rule verify reports them by (lib/placeholders.js). The gate let
|
|
703
|
+
// {{name}} → {{nom}} through, verify flagged it after the sync, and the
|
|
704
|
+
// run exited 2; refused here, the pair's fallback gets the text instead
|
|
705
|
+
// (Round 12, i18next persona).
|
|
706
|
+
const tokens = placeholderChanges(source, translated);
|
|
707
|
+
if (tokens.length > 0) {
|
|
708
|
+
failures.push({
|
|
709
|
+
key,
|
|
710
|
+
reason: `placeholder structure damaged: ${tokens.map(t => `${PLACEHOLDER_SYNTAX_NAMES[t.syntax]} ${t.issue}`).join('; ')}`,
|
|
711
|
+
value: translated,
|
|
712
|
+
placeholder: true,
|
|
713
|
+
});
|
|
714
|
+
continue;
|
|
715
|
+
}
|
|
716
|
+
// A sentence break the translation put right beside a placeholder
|
|
717
|
+
// where the source has none: "Take this medicine at {time}." →
|
|
718
|
+
// "… sina. {time}." passed every check (Round 14, hospital persona).
|
|
719
|
+
// The same rule verify flags by (lib/placeholders.js); refused here,
|
|
720
|
+
// the retry hears why and the pair's fallback runs.
|
|
721
|
+
const breaks = placeholderSentenceBreaks(source, translated);
|
|
722
|
+
if (breaks.length > 0) {
|
|
723
|
+
failures.push({
|
|
724
|
+
key,
|
|
725
|
+
reason: `sentence break beside a placeholder: ${breaks.map(b => b.issue).join('; ')}`,
|
|
726
|
+
value: translated,
|
|
727
|
+
placeholder: true,
|
|
728
|
+
});
|
|
729
|
+
continue;
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
// Check 2c: the echo checks, per plural/select branch. A plural message
|
|
734
|
+
// differs from its source as a WHOLE as soon as the target adds a form
|
|
735
|
+
// (or one branch is translated), so the two checks above never saw a
|
|
736
|
+
// branch handed back as the English — accented, or as it was (Round 4,
|
|
737
|
+
// Django persona: the plural entry passed while the identical singular
|
|
738
|
+
// change was refused). Each branch is held to the singular rules: an
|
|
739
|
+
// exact copy of a short Latin-script branch is accepted in a Latin-script
|
|
740
|
+
// target (a name, "emails") and asked about once in a non-Latin one; any
|
|
741
|
+
// longer copy is refused; a disguised copy (3+ words) is refused. A
|
|
742
|
+
// branch made only of declared names (protectedTerms) is never an echo.
|
|
743
|
+
if (translated !== source && !prose) {
|
|
744
|
+
const branchEcho = pluralBranchEcho(source, translated, { isNonLatin, requireNonLatin: thresholds.requireNonLatin, acceptLatinNames, protectedTerms });
|
|
745
|
+
if (branchEcho) {
|
|
746
|
+
failures.push({ key, value: translated, ...branchEcho });
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
315
749
|
}
|
|
316
750
|
|
|
317
751
|
// Check 3: Repetition detection — catches hallucination loops.
|
|
@@ -353,7 +787,7 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
353
787
|
// Check 4: Length ratio — catches padding and truncation
|
|
354
788
|
if (source.length > 0) {
|
|
355
789
|
const ratio = translated.length / source.length;
|
|
356
|
-
if (ratio > thresholds.maxLengthRatio) {
|
|
790
|
+
if (ratio > thresholds.maxLengthRatio && translated.length - source.length > SHORT_SOURCE_LENGTH_SLACK) {
|
|
357
791
|
failures.push({
|
|
358
792
|
key,
|
|
359
793
|
reason: `length inflation (${ratio.toFixed(1)}x source, max ${thresholds.maxLengthRatio}x)`,
|
|
@@ -387,19 +821,62 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
387
821
|
// or version strings like "3.2.0" have nothing to write in another script.
|
|
388
822
|
// - Short ASCII strings (≤30 chars, >80% ASCII) are likely proper nouns
|
|
389
823
|
// or brand names (e.g. "GitHub", "npm") that stay in English everywhere.
|
|
824
|
+
// Fullwidth Latin letters ("Book an appointment") outside CJK typography:
|
|
825
|
+
// English in disguise, whatever the target's script. Never a name — a
|
|
826
|
+
// name kept as written is in plain letters.
|
|
827
|
+
if (hasForeignFullwidthLatin(translated, targetLocale) && !hasForeignFullwidthLatin(source, targetLocale)) {
|
|
828
|
+
failures.push({
|
|
829
|
+
key,
|
|
830
|
+
reason: `wrong script (fullwidth Latin letters in a ${targetLocale} value — English in disguise, not a translation)`,
|
|
831
|
+
value: translated.slice(0, 80),
|
|
832
|
+
});
|
|
833
|
+
continue;
|
|
834
|
+
}
|
|
390
835
|
if (isNonLatin && thresholds.requireNonLatin) {
|
|
391
836
|
// Strip ICU placeholders, digits, punctuation, whitespace → what's left?
|
|
392
837
|
const translatableText = translated
|
|
393
838
|
.replace(/\{[^}]*\}/g, '') // ICU placeholders
|
|
394
839
|
.replace(/[\d\s\p{P}\p{S}]/gu, '') // digits, whitespace, punctuation, symbols
|
|
395
840
|
.trim();
|
|
396
|
-
const
|
|
397
|
-
const
|
|
398
|
-
|
|
841
|
+
const core = nameCore(source);
|
|
842
|
+
const asciiRatio = core.replace(/[^\x20-\x7E]/g, '').length / Math.max(core.length, 1);
|
|
843
|
+
const isShortAscii = core.length === 0 || (core.length <= 30 && asciiRatio > 0.8);
|
|
844
|
+
// Letters classified by Unicode script: accented Latin is Latin too
|
|
845
|
+
// (an ASCII test passed "Thánk yóú" in a Russian catalog).
|
|
846
|
+
if (translatableText.length > 0 && isLatinOnly(translated, targetLocale)) {
|
|
847
|
+
if (isShortAscii && acceptLatinNames) {
|
|
848
|
+
// Second answer still Latin-script: the model says it is a name.
|
|
849
|
+
} else if (isShortAscii) {
|
|
850
|
+
failures.push({ key, reason: LATIN_NAME_OR_LABEL, value: translated.slice(0, 80), nameOrLabel: true });
|
|
851
|
+
continue;
|
|
852
|
+
} else {
|
|
853
|
+
failures.push({
|
|
854
|
+
key,
|
|
855
|
+
reason: `wrong script (ASCII-only for ${targetLocale}, expected non-Latin characters)`,
|
|
856
|
+
value: translated.slice(0, 80),
|
|
857
|
+
});
|
|
858
|
+
continue;
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// Check 7 (last, so a soft failure means everything else passed): a
|
|
864
|
+
// plural message without a form the target language uses for ordinary
|
|
865
|
+
// counts. Soft — the caller asks once more; see the module header.
|
|
866
|
+
if (!acceptPluralGaps && !prose && translated !== source) {
|
|
867
|
+
// A gettext catalog's slots (pairConfig.pluralSlots, set by sync for a
|
|
868
|
+
// .po file): a form it has no msgstr[] for is never asked again for.
|
|
869
|
+
const gaps = pluralGaps(source, translated, targetLocale, options.pluralSlots ?? pairConfig.pluralSlots ?? null)
|
|
870
|
+
.filter(g => g.everyday.length > 0);
|
|
871
|
+
if (gaps.length > 0) {
|
|
872
|
+
const missing = [...new Set(gaps.flatMap(g => g.everyday))];
|
|
873
|
+
const type = gaps[0].type;
|
|
399
874
|
failures.push({
|
|
400
875
|
key,
|
|
401
|
-
reason: `
|
|
402
|
-
|
|
876
|
+
reason: `plural form(s) ${missing.map(c => `"${c}"`).join(', ')} missing — ${targetLocale} uses `
|
|
877
|
+
+ `${describeCategories(targetLocale, missing, type)}`,
|
|
878
|
+
value: translated,
|
|
879
|
+
pluralGap: { missing, type },
|
|
403
880
|
});
|
|
404
881
|
continue;
|
|
405
882
|
}
|
|
@@ -412,6 +889,336 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
412
889
|
return { validated, failures };
|
|
413
890
|
}
|
|
414
891
|
|
|
892
|
+
// ---------------------------------------------------------------------------
|
|
893
|
+
// Different inputs, same output
|
|
894
|
+
// ---------------------------------------------------------------------------
|
|
895
|
+
|
|
896
|
+
/** Distinct source strings one output may answer before it is suspect. */
|
|
897
|
+
const SHARED_OUTPUT_MIN_SOURCES = 3;
|
|
898
|
+
/** Words (with letters) the shared output needs, on its own, to be suspect. */
|
|
899
|
+
const SHARED_OUTPUT_MIN_WORDS = 4;
|
|
900
|
+
|
|
901
|
+
/**
|
|
902
|
+
* Simple placeholders and markup tags in an output — not words of the
|
|
903
|
+
* translation (SharedOutputIndex.outputForm): {name} {0} {{count}} %(name)s
|
|
904
|
+
* %s %1$d <b> </a>. A plural/select argument ({n, plural, …}) has a comma and
|
|
905
|
+
* is not matched.
|
|
906
|
+
*/
|
|
907
|
+
const PLACEHOLDER_IN_OUTPUT = /\{\{\s*[\w.$-]+\s*\}\}|\{\s*[\w.$-]+\s*\}|%\([\w.]+\)[sdifr]|%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?[sdifuxXeEgGcp@]|<\/?[A-Za-z][^<>]*>/gu;
|
|
908
|
+
/** The same without single-brace arguments: for a whole plural/select message. */
|
|
909
|
+
const PLACEHOLDER_NOT_BRACED = /\{\{\s*[\w.$-]+\s*\}\}|%\([\w.]+\)[sdifr]|%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?[sdifuxXeEgGcp@]|<\/?[A-Za-z][^<>]*>/gu;
|
|
910
|
+
/** A plural/select/selectordinal argument ("{n, plural, …"). */
|
|
911
|
+
const ICU_COMPLEX_ARGUMENT = /\{\s*[\w.$-]+\s*,\s*(?:plural|select|selectordinal)\s*,/u;
|
|
912
|
+
|
|
913
|
+
/**
|
|
914
|
+
* The sentences of a text, for the repeat check: split after . ! ? (and the
|
|
915
|
+
* CJK 。!?) followed by space, each kept only when it has a letter once
|
|
916
|
+
* its placeholders are gone ("S. {name}!" is one sentence).
|
|
917
|
+
*
|
|
918
|
+
* @param {string} text
|
|
919
|
+
* @returns {string[]}
|
|
920
|
+
*/
|
|
921
|
+
function sentencesOf(text) {
|
|
922
|
+
return String(text)
|
|
923
|
+
.split(/(?<=[.!?\u3002\uFF01\uFF1F])\s+/u)
|
|
924
|
+
.map(t => t.trim())
|
|
925
|
+
.filter(t => /\p{L}/u.test(t.replace(PLACEHOLDER_IN_OUTPUT, ' ')));
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
/** A source in the form two sources are compared in ("Save", "save!" meet). */
|
|
929
|
+
function sourceIdentity(text) {
|
|
930
|
+
return foldForEchoCompare(String(text).replace(/[\p{P}\p{S}]+/gu, ' '));
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/** The words of a folded string, as a set. */
|
|
934
|
+
function wordSet(text) {
|
|
935
|
+
return new Set(sourceIdentity(text).split(' ').filter(w => /\p{L}/u.test(w)));
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/** True when every pair of sources shares under half its words (Jaccard). */
|
|
939
|
+
function clearlyDifferent(sources) {
|
|
940
|
+
const sets = sources.map(wordSet);
|
|
941
|
+
for (let i = 0; i < sets.length; i++) {
|
|
942
|
+
for (let j = i + 1; j < sets.length; j++) {
|
|
943
|
+
const a = sets[i];
|
|
944
|
+
const b = sets[j];
|
|
945
|
+
const inter = [...a].filter(w => b.has(w)).length;
|
|
946
|
+
const union = new Set([...a, ...b]).size;
|
|
947
|
+
if (union === 0 || inter / union >= 0.5) return false;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
return true;
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
/**
|
|
954
|
+
* Different inputs, same output: ONE translation text returned for several
|
|
955
|
+
* DIFFERENT source strings is what a model that memorized a training sentence
|
|
956
|
+
* does (Round 4 school persona: a trained eng→crk model answered the app
|
|
957
|
+
* title, "Contact the school", the newsletter title and its heading with the
|
|
958
|
+
* same sentence — and every per-key check passed it).
|
|
959
|
+
*
|
|
960
|
+
* Suspect when one output answers SHARED_OUTPUT_MIN_SOURCES+ distinct sources
|
|
961
|
+
* (compared folded, punctuation aside — "Save" and "Save!" are one source)
|
|
962
|
+
* AND either the output has SHARED_OUTPUT_MIN_WORDS+ words, or every source
|
|
963
|
+
* has two or more words and no two of them share half their words. So
|
|
964
|
+
* synonyms collapsing to one short translation pass ('OK'/'Okay'/'Sure' →
|
|
965
|
+
* "D'accord"; 'Close'/'Dismiss'/'Cancel' → "Fermer"), and so does the same
|
|
966
|
+
* source text under several keys.
|
|
967
|
+
*
|
|
968
|
+
* TWO sources are already enough when the evidence is strong (Round 6,
|
|
969
|
+
* school persona: 'Thank you, {name}!' and 'Please bring the forms.' got one
|
|
970
|
+
* memorized sentence and passed, because only a third source counted): the
|
|
971
|
+
* output has SHARED_OUTPUT_MIN_WORDS+ words, both sources have two or more
|
|
972
|
+
* words, and they share under half their words (clearly different). Short
|
|
973
|
+
* outputs and synonym-like sources (sharing half their words or more) still
|
|
974
|
+
* need a third source. The cost of a false alarm is one more ask (an LLM is
|
|
975
|
+
* told to translate this string on its own; a method that takes no
|
|
976
|
+
* instructions is asked once more), then the pair's fallback.
|
|
977
|
+
*
|
|
978
|
+
* MEMORIZED outputs (markMemorized): an output an earlier sync found
|
|
979
|
+
* answering different source strings is suspect from its first source on —
|
|
980
|
+
* a repair that asks again and gets the same sentence back must not write it
|
|
981
|
+
* (Round 6, school persona: `--force-content` re-served it in silence).
|
|
982
|
+
*
|
|
983
|
+
* Same suppressions as the echo checks: a value made only of declared names
|
|
984
|
+
* (protectedTerms) is never suspect, nor a value equal to its own source
|
|
985
|
+
* (an echo — its own rule), nor one without letters. No-translate keys never
|
|
986
|
+
* reach a gate.
|
|
987
|
+
*
|
|
988
|
+
* The index spans a run's locale: the caller adds what it accepted, so a
|
|
989
|
+
* later batch (or a content file's block) that repeats an earlier answer is
|
|
990
|
+
* caught; only the NEW items are returned as suspect — what was written
|
|
991
|
+
* earlier is reported by `verify`. An output found suspect earlier in the
|
|
992
|
+
* run is suspect from its first new source on, like a MEMORIZED one: its
|
|
993
|
+
* refused members were never added, and the run remembers it only when it
|
|
994
|
+
* ends.
|
|
995
|
+
*/
|
|
996
|
+
class SharedOutputIndex {
|
|
997
|
+
constructor({ protectedTerms = [] } = {}) {
|
|
998
|
+
this.protectedTerms = protectedTerms;
|
|
999
|
+
// output form → Map(source identity → { key, source })
|
|
1000
|
+
this.byOutput = new Map();
|
|
1001
|
+
// output form → the first text seen in that form (what reports show)
|
|
1002
|
+
this.display = new Map();
|
|
1003
|
+
// output text → { value, sources, keys } — every group found suspect,
|
|
1004
|
+
// with ALL its members (earlier, accepted ones too), for the run report.
|
|
1005
|
+
this.flagged = new Map();
|
|
1006
|
+
// output form → the text as first seen: outputs an earlier sync found
|
|
1007
|
+
// answering different source strings (markMemorized).
|
|
1008
|
+
this.memorized = new Map();
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
/**
|
|
1012
|
+
* Outputs an earlier run found answering several different source strings:
|
|
1013
|
+
* suspect from their first source on (see the class comment).
|
|
1014
|
+
*
|
|
1015
|
+
* @param {Iterable<string>} values
|
|
1016
|
+
*/
|
|
1017
|
+
markMemorized(values) {
|
|
1018
|
+
for (const value of values || []) {
|
|
1019
|
+
if (typeof value !== 'string') continue;
|
|
1020
|
+
const form = SharedOutputIndex.outputForm(value);
|
|
1021
|
+
if (form && /\p{L}/u.test(form)) this.memorized.set(form, String(value).replace(/\s+/gu, ' ').trim());
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
/**
|
|
1026
|
+
* The form outputs are grouped by: case, punctuation and symbols aside, and
|
|
1027
|
+
* without a Markdown block marker — so "S?", "S." and a heading "# S" are
|
|
1028
|
+
* one output (Round 5: an NMT model copies the source's end punctuation,
|
|
1029
|
+
* and a newsletter H1 kept its "# "). Letters and their accents are kept:
|
|
1030
|
+
* two words that differ by a diacritic are two words.
|
|
1031
|
+
*
|
|
1032
|
+
* Placeholders and markup tags are not words of the output: "S. {name}!"
|
|
1033
|
+
* and "S." are one output (Round 9, school persona: the memorized sentence
|
|
1034
|
+
* came back as "S. {name}!" for 'Thank you, {name}!' and as "S." inside a
|
|
1035
|
+
* newsletter paragraph, and the "name" left over from "{name}" kept the two
|
|
1036
|
+
* apart). Simple arguments only ({name}, {0}, {{count}}, %(name)s, %s,
|
|
1037
|
+
* %1$d, <b>…</b>); an unsplit plural/select message keeps its branches.
|
|
1038
|
+
*/
|
|
1039
|
+
static outputForm(value) {
|
|
1040
|
+
const text = String(value);
|
|
1041
|
+
// Inside a whole plural/select message, `{un}` is a branch, not an argument.
|
|
1042
|
+
const placeholders = ICU_COMPLEX_ARGUMENT.test(text) ? PLACEHOLDER_NOT_BRACED : PLACEHOLDER_IN_OUTPUT;
|
|
1043
|
+
return text
|
|
1044
|
+
.normalize('NFC')
|
|
1045
|
+
.replace(/^\s{0,3}(?:#{1,6}\s+|[-*+]\s+|\d+[.)]\s+|>\s*)/u, '')
|
|
1046
|
+
.replace(placeholders, ' ')
|
|
1047
|
+
.toLowerCase()
|
|
1048
|
+
.replace(/[\p{P}\p{S}]+/gu, ' ')
|
|
1049
|
+
.replace(/\s+/gu, ' ')
|
|
1050
|
+
.trim();
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
/**
|
|
1054
|
+
* The items, plus — for a value of several sentences whose source has as
|
|
1055
|
+
* many — each sentence on its own, paired with its source sentence (the
|
|
1056
|
+
* parent kept in `whole`). One memorized sentence inside a paragraph is
|
|
1057
|
+
* then the same output as that sentence answering a UI string (Round 9,
|
|
1058
|
+
* school persona: 'Please bring the forms.' in the newsletter and 'Thank
|
|
1059
|
+
* you, {name}!' got one sentence, and the whole-paragraph comparison never
|
|
1060
|
+
* met it). Pairing by position keeps a sentence two paragraphs genuinely
|
|
1061
|
+
* share ("Please bring the forms." in both) as ONE source — not suspect;
|
|
1062
|
+
* a value whose sentence count differs from its source's is compared whole
|
|
1063
|
+
* only.
|
|
1064
|
+
*
|
|
1065
|
+
* @param {Array<{ key: string, source: string, value: string }>} items
|
|
1066
|
+
* @returns {Array<{ key: string, source: string, value: string, whole?: { source: string, value: string } }>}
|
|
1067
|
+
*/
|
|
1068
|
+
static withSentences(items) {
|
|
1069
|
+
const out = [];
|
|
1070
|
+
for (const it of items) {
|
|
1071
|
+
out.push(it);
|
|
1072
|
+
if (typeof it.source !== 'string' || typeof it.value !== 'string' || it.whole) continue;
|
|
1073
|
+
const vs = sentencesOf(it.value);
|
|
1074
|
+
if (vs.length < 2) continue;
|
|
1075
|
+
const ss = sentencesOf(it.source);
|
|
1076
|
+
if (ss.length !== vs.length) continue;
|
|
1077
|
+
vs.forEach((v, i) => out.push({ key: it.key, source: ss[i], value: v, whole: { source: it.source, value: it.value } }));
|
|
1078
|
+
}
|
|
1079
|
+
return out;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
eligible(source, value) {
|
|
1083
|
+
if (typeof source !== 'string' || typeof value !== 'string') return false;
|
|
1084
|
+
if (value === source || !/\p{L}/u.test(value) || !SharedOutputIndex.outputForm(value)) return false;
|
|
1085
|
+
if (isProtectedTermValue(value, this.protectedTerms)) return false;
|
|
1086
|
+
return true;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
/**
|
|
1090
|
+
* Items that would make (or join) a suspect group.
|
|
1091
|
+
*
|
|
1092
|
+
* @param {Array<{ key: string, source: string, value: string }>} items
|
|
1093
|
+
* @returns {Map<string, { value: string, sources: string[] }>} key → its group
|
|
1094
|
+
*/
|
|
1095
|
+
suspects(items) {
|
|
1096
|
+
const groups = new Map();
|
|
1097
|
+
for (const it of SharedOutputIndex.withSentences(items)) {
|
|
1098
|
+
if (!this.eligible(it.source, it.value)) continue;
|
|
1099
|
+
const form = SharedOutputIndex.outputForm(it.value);
|
|
1100
|
+
if (!this.display.has(form)) this.display.set(form, String(it.value).replace(/\s+/gu, ' ').trim());
|
|
1101
|
+
if (!groups.has(form)) {
|
|
1102
|
+
groups.set(form, { sources: new Map(this.byOutput.get(form) || []), fresh: [] });
|
|
1103
|
+
}
|
|
1104
|
+
const g = groups.get(form);
|
|
1105
|
+
const id = sourceIdentity(it.source);
|
|
1106
|
+
if (!g.sources.has(id)) g.sources.set(id, { key: it.key, source: it.source });
|
|
1107
|
+
g.fresh.push(it);
|
|
1108
|
+
}
|
|
1109
|
+
const out = new Map();
|
|
1110
|
+
for (const [form, g] of groups) {
|
|
1111
|
+
let sources = [...g.sources.values()].map(s => s.source);
|
|
1112
|
+
const byRule = SharedOutputIndex.suspectGroup(form, sources);
|
|
1113
|
+
// `memorized` marks a group suspect ONLY because an earlier sync
|
|
1114
|
+
// remembered its output (reports word it that way); a group the rule
|
|
1115
|
+
// catches on its own is reported as before.
|
|
1116
|
+
const memorized = !byRule && this.memorized.has(form);
|
|
1117
|
+
// An output this index already found answering different source
|
|
1118
|
+
// strings EARLIER IN THIS RUN is suspect from its first new source on,
|
|
1119
|
+
// as a remembered one is from an earlier sync. The members refused
|
|
1120
|
+
// then were never added (only accepted items are), so without this the
|
|
1121
|
+
// same sentence came back for the newsletter's title after the app's
|
|
1122
|
+
// three keys were refused for it, and was written — the run remembers
|
|
1123
|
+
// it only when it ends, and `verify` flagged it (Round 10, school
|
|
1124
|
+
// persona). A gate retry that returns the refused sentence for one key
|
|
1125
|
+
// is refused the same way.
|
|
1126
|
+
const earlier = !byRule && !memorized ? this.flagged.get(form) : null;
|
|
1127
|
+
if (earlier) sources = [...new Set([...earlier.sources, ...sources])];
|
|
1128
|
+
if (!byRule && !memorized && !earlier) continue;
|
|
1129
|
+
const shown = this.memorized.get(form) || this.display.get(form) || form;
|
|
1130
|
+
for (const it of g.fresh) out.set(it.key, { value: shown, sources, ...(memorized && { memorized: true }) });
|
|
1131
|
+
const prior = this.flagged.get(form) || { value: shown, sources: [], keys: [] };
|
|
1132
|
+
this.flagged.set(form, {
|
|
1133
|
+
value: shown,
|
|
1134
|
+
sources: [...new Set([...prior.sources, ...sources])],
|
|
1135
|
+
keys: [...new Set([...prior.keys, ...[...g.sources.values()].map(v => v.key), ...g.fresh.map(it => it.key)])],
|
|
1136
|
+
...(memorized && { memorized: true }),
|
|
1137
|
+
});
|
|
1138
|
+
}
|
|
1139
|
+
return out;
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
/**
|
|
1143
|
+
* Is one output form answering these distinct sources the pattern? Three or
|
|
1144
|
+
* more: a long output, or clearly different multi-word sources. Two: only
|
|
1145
|
+
* the strong case (see the class comment).
|
|
1146
|
+
*
|
|
1147
|
+
* @param {string} form - outputForm() of the shared output
|
|
1148
|
+
* @param {string[]} sources - Distinct sources it answers
|
|
1149
|
+
* @returns {boolean}
|
|
1150
|
+
*/
|
|
1151
|
+
static suspectGroup(form, sources) {
|
|
1152
|
+
const outWords = letterWordCount(form);
|
|
1153
|
+
if (sources.length >= SHARED_OUTPUT_MIN_SOURCES) {
|
|
1154
|
+
return outWords >= SHARED_OUTPUT_MIN_WORDS
|
|
1155
|
+
|| (sources.every(src => letterWordCount(src) >= 2) && clearlyDifferent(sources));
|
|
1156
|
+
}
|
|
1157
|
+
if (sources.length !== 2) return false;
|
|
1158
|
+
return outWords >= SHARED_OUTPUT_MIN_WORDS
|
|
1159
|
+
&& sources.every(src => letterWordCount(src) >= 2)
|
|
1160
|
+
&& clearlyDifferent(sources);
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
/**
|
|
1164
|
+
* Record accepted items (what was written) for later batches. A sentence
|
|
1165
|
+
* of a longer value is recorded with its parent (`whole`): what was written,
|
|
1166
|
+
* and cached, is the whole value.
|
|
1167
|
+
*/
|
|
1168
|
+
add(items) {
|
|
1169
|
+
for (const it of SharedOutputIndex.withSentences(items)) {
|
|
1170
|
+
if (!this.eligible(it.source, it.value)) continue;
|
|
1171
|
+
const form = SharedOutputIndex.outputForm(it.value);
|
|
1172
|
+
if (!this.display.has(form)) this.display.set(form, String(it.value).replace(/\s+/gu, ' ').trim());
|
|
1173
|
+
if (!this.byOutput.has(form)) this.byOutput.set(form, new Map());
|
|
1174
|
+
const id = sourceIdentity(it.source);
|
|
1175
|
+
if (!this.byOutput.get(form).has(id)) {
|
|
1176
|
+
this.byOutput.get(form).set(id, { key: it.key, source: it.source, value: it.value, ...(it.whole && { whole: it.whole }) });
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
/**
|
|
1183
|
+
* What one translated value puts in the shared-output index: the value
|
|
1184
|
+
* itself, or — for an ICU plural/select message — each leaf branch on its
|
|
1185
|
+
* own (Round 5, school persona: one memorized sentence filled BOTH branches
|
|
1186
|
+
* of a plural message and two other keys, and nothing counted the branches).
|
|
1187
|
+
*
|
|
1188
|
+
* Every branch of one `plural`/`selectordinal` argument counts as ONE source
|
|
1189
|
+
* (the message's): a language without number inflection writes the same
|
|
1190
|
+
* text in every branch by design, so repeating it across branches is never
|
|
1191
|
+
* evidence on its own. `select` branches count as their own source branches —
|
|
1192
|
+
* different only when those differ.
|
|
1193
|
+
*
|
|
1194
|
+
* @param {string} key
|
|
1195
|
+
* @param {string} source
|
|
1196
|
+
* @param {string} value
|
|
1197
|
+
* @returns {Array<{ key: string, source: string, value: string }>}
|
|
1198
|
+
*/
|
|
1199
|
+
function sharedOutputItems(key, source, value) {
|
|
1200
|
+
if (typeof source !== 'string' || typeof value !== 'string') return [];
|
|
1201
|
+
const pairs = value.includes('{') && source.includes('{') ? pluralBranchPairs(source, value) : [];
|
|
1202
|
+
if (pairs.length === 0) return [{ key, source, value }];
|
|
1203
|
+
return pairs.map(p => ({
|
|
1204
|
+
key,
|
|
1205
|
+
source: p.keyword === 'select' ? p.source : source,
|
|
1206
|
+
value: p.translated,
|
|
1207
|
+
}));
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
/** Gate reason for an output shared by several different sources. */
|
|
1211
|
+
function sharedOutputReason(group) {
|
|
1212
|
+
if (group.memorized) {
|
|
1213
|
+
const text = group.value.length > 60 ? `${group.value.slice(0, 57)}...` : group.value;
|
|
1214
|
+
return `the sentence the model gave for other, different source strings on an earlier sync (${JSON.stringify(text)}) `
|
|
1215
|
+
+ '— a memorized sentence, not a translation of this string; asking the same model again returns it again';
|
|
1216
|
+
}
|
|
1217
|
+
const shown = group.sources.slice(0, 4).map(s => JSON.stringify(s.length > 40 ? `${s.slice(0, 37)}...` : s)).join(', ');
|
|
1218
|
+
return `same output for ${group.sources.length} different source strings (${shown}${group.sources.length > 4 ? ', …' : ''}) `
|
|
1219
|
+
+ '— a model repeating one memorized sentence, not a translation of this string';
|
|
1220
|
+
}
|
|
1221
|
+
|
|
415
1222
|
/**
|
|
416
1223
|
* Split a value into independently-measurable segments: pipe-delimited
|
|
417
1224
|
* plural variants ("one doc|{count} docs") legitimately share most of
|
|
@@ -421,6 +1228,10 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
|
|
|
421
1228
|
* @returns {string[]} Trimmed segments (always at least one)
|
|
422
1229
|
*/
|
|
423
1230
|
function splitPluralSegments(text) {
|
|
1231
|
+
// ICU plural/select branches too: a Russian plural repeats one sentence in
|
|
1232
|
+
// four forms by design, and measured as one string it read as a loop.
|
|
1233
|
+
const branches = text.includes('{') ? pluralBranchTexts(text) : null;
|
|
1234
|
+
if (branches) return branches.map(seg => seg.trim());
|
|
424
1235
|
return (text.includes('|') ? text.split('|') : [text]).map(seg => seg.trim());
|
|
425
1236
|
}
|
|
426
1237
|
|
|
@@ -477,6 +1288,138 @@ function isAsciiOnly(text) {
|
|
|
477
1288
|
return /^[\x00-\x7F]*$/.test(text);
|
|
478
1289
|
}
|
|
479
1290
|
|
|
1291
|
+
/** Scripts whose text legitimately uses fullwidth Latin letters (CJK typography). */
|
|
1292
|
+
const FULLWIDTH_SCRIPTS = new Set(['Hani', 'Hans', 'Hant', 'Jpan', 'Kore', 'Hira', 'Kana', 'Hang', 'Bopo']);
|
|
1293
|
+
|
|
1294
|
+
/** Fullwidth Latin letters: A–Z, a–z. */
|
|
1295
|
+
const FULLWIDTH_LATIN = /[A-Za-z]/u;
|
|
1296
|
+
|
|
1297
|
+
/**
|
|
1298
|
+
* The text of a value with its code set aside: ICU/brace placeholders,
|
|
1299
|
+
* printf conversions, markup tags. (A URL stays: in a non-Latin locale a
|
|
1300
|
+
* URL value is a wrong-script answer unless the key is declared
|
|
1301
|
+
* no-translate — which never reaches this check.)
|
|
1302
|
+
*/
|
|
1303
|
+
function proseOf(text) {
|
|
1304
|
+
// A plural/select message: its prose is the text of its branches (the
|
|
1305
|
+
// keywords and selectors — "plural", "one", "other" — are code).
|
|
1306
|
+
const str = String(text);
|
|
1307
|
+
const branches = str.includes('{') ? pluralBranchTexts(str) : null;
|
|
1308
|
+
return (branches ? branches.join(' ') : str)
|
|
1309
|
+
// Simple placeholders only: {name}, {count, number}. Branch text such as
|
|
1310
|
+
// "{Один файл}" is prose, never stripped.
|
|
1311
|
+
.replace(/\{\s*[\w.$-]+\s*(?:,[^{}]*)?\}/g, ' ')
|
|
1312
|
+
.replace(/\{\{\s*[\w.$-]+\s*\}\}/g, ' ')
|
|
1313
|
+
.replace(/%(\([^)]*\))?[-#0 +]*\d*(\.\d+)?[a-zA-Z@]/g, ' ')
|
|
1314
|
+
.replace(/<\/?[A-Za-z0-9][^<>]*>/g, ' ');
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
/**
|
|
1318
|
+
* Are ALL the letters of a value's prose Latin script — by Unicode script,
|
|
1319
|
+
* not by byte? Fullwidth Latin ("Hello") and accented Latin ("Thánk")
|
|
1320
|
+
* are Latin: an ASCII test let both through as "not English" in a Russian
|
|
1321
|
+
* catalog (Round 4, Django persona). False when the prose has no letters.
|
|
1322
|
+
*
|
|
1323
|
+
* @param {string} text
|
|
1324
|
+
* @returns {boolean}
|
|
1325
|
+
*/
|
|
1326
|
+
function isLatinOnly(text, locale = null) {
|
|
1327
|
+
const prose = proseOf(text);
|
|
1328
|
+
const letters = prose.match(/\p{L}/gu);
|
|
1329
|
+
if (!letters) return false;
|
|
1330
|
+
// CJK typography sets Latin in fullwidth forms ("OK" in Japanese): there,
|
|
1331
|
+
// fullwidth letters are the target's own usage, as they always were here.
|
|
1332
|
+
if (locale && FULLWIDTH_LATIN.test(prose) && usesFullwidthLatin(locale)) return false;
|
|
1333
|
+
return letters.every(ch => /\p{Script=Latin}/u.test(ch));
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
/** Does this locale's script set Latin in fullwidth forms (CJK)? */
|
|
1337
|
+
function usesFullwidthLatin(locale) {
|
|
1338
|
+
const card = getLanguageCard(String(locale)) || getLanguageCard(String(locale).split(/[-_]/)[0]);
|
|
1339
|
+
return FULLWIDTH_SCRIPTS.has(card?.script);
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
/**
|
|
1343
|
+
* Fullwidth Latin letters in a value whose language does not set Latin in
|
|
1344
|
+
* fullwidth forms (anything but CJK typography): a disguised copy of English,
|
|
1345
|
+
* never a translation.
|
|
1346
|
+
*
|
|
1347
|
+
* @param {string} text
|
|
1348
|
+
* @param {string} locale - Target locale
|
|
1349
|
+
* @returns {boolean}
|
|
1350
|
+
*/
|
|
1351
|
+
function hasForeignFullwidthLatin(text, locale) {
|
|
1352
|
+
if (!FULLWIDTH_LATIN.test(proseOf(text))) return false;
|
|
1353
|
+
return !usesFullwidthLatin(locale);
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
// ---------------------------------------------------------------------------
|
|
1357
|
+
// A question or exclamation that lost its mark
|
|
1358
|
+
// ---------------------------------------------------------------------------
|
|
1359
|
+
|
|
1360
|
+
// Marks that end a question / an exclamation in some writing system: Latin
|
|
1361
|
+
// and most scripts (?), CJK fullwidth (?), Arabic script (؟), Greek (U+037E
|
|
1362
|
+
// and the ASCII semicolon Greek text is typed with), Armenian (՞), Ethiopic
|
|
1363
|
+
// (፧), and the interrobang family. Spanish closes with "?" / "!" as well.
|
|
1364
|
+
const QUESTION_ENDS = new Set(['?', '?', '؟', '\u037e', ';', '՞', '፧', '⸮', '⁇', '⁈', '‽', '⁉']);
|
|
1365
|
+
const EXCLAMATION_ENDS = new Set(['!', '!', '‼', '⁉', '⁈', '‽', '՜', '¡']);
|
|
1366
|
+
// Closing quotes, brackets, ICU braces and direction marks after the mark.
|
|
1367
|
+
const TRAILING_CLOSERS = /[\s"'»”’)\]}›〉》」』】〕\u200e\u200f]+$/u;
|
|
1368
|
+
|
|
1369
|
+
/** The last visible character of a text, closers aside. */
|
|
1370
|
+
function terminalChar(text) {
|
|
1371
|
+
const chars = Array.from(String(text).replace(TRAILING_CLOSERS, ''));
|
|
1372
|
+
return chars.length > 0 ? chars[chars.length - 1] : '';
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* A source that ends with "?" or "!" whose translation ends with neither it
|
|
1377
|
+
* nor an equivalent the target's script uses (Round 6, hospital persona:
|
|
1378
|
+
* "Where does it hurt?" passed as a statement). A WARNING, never a refusal:
|
|
1379
|
+
* some languages mark a question with a word or particle instead of a mark,
|
|
1380
|
+
* and a translation that does so is right.
|
|
1381
|
+
*
|
|
1382
|
+
* @param {string} source
|
|
1383
|
+
* @param {string} translated
|
|
1384
|
+
* @returns {{ mark: '?'|'!' } | null}
|
|
1385
|
+
*/
|
|
1386
|
+
function droppedTerminalMark(source, translated) {
|
|
1387
|
+
if (typeof source !== 'string' || typeof translated !== 'string' || translated === source) return null;
|
|
1388
|
+
if (!/\p{L}/u.test(translated)) return null;
|
|
1389
|
+
const end = terminalChar(source);
|
|
1390
|
+
if (end !== '?' && end !== '!') return null;
|
|
1391
|
+
const got = terminalChar(translated);
|
|
1392
|
+
if (end === '?' ? QUESTION_ENDS.has(got) : EXCLAMATION_ENDS.has(got)) return null;
|
|
1393
|
+
return { mark: end };
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
/**
|
|
1397
|
+
* Keys whose translation dropped the source's closing "?" / "!" — one
|
|
1398
|
+
* finding for sync and verify alike. Declared names and no-translate keys
|
|
1399
|
+
* are skipped by the callers' own rules.
|
|
1400
|
+
*
|
|
1401
|
+
* @param {Array<[string, string, string]>} entries - [key, source, translated]
|
|
1402
|
+
* @returns {Array<{ key: string, mark: string, value: string }>}
|
|
1403
|
+
*/
|
|
1404
|
+
function droppedTerminalMarks(entries) {
|
|
1405
|
+
const out = [];
|
|
1406
|
+
for (const [key, source, value] of entries) {
|
|
1407
|
+
const d = droppedTerminalMark(source, value);
|
|
1408
|
+
if (d) out.push({ key, mark: d.mark, value });
|
|
1409
|
+
}
|
|
1410
|
+
return out;
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
/** The warning text for droppedTerminalMarks findings (sync and verify say the same). */
|
|
1414
|
+
function describeDroppedMarks(found, fix) {
|
|
1415
|
+
const marks = [...new Set(found.map(f => `"${f.mark}"`))].join(' or ');
|
|
1416
|
+
const shown = found.slice(0, 3).map(f => `${f.key} (${JSON.stringify(f.value.length > 40 ? `${f.value.slice(0, 37)}...` : f.value)})`).join(', ');
|
|
1417
|
+
return `${found.length} translation(s) dropped the source's closing ${marks}: ${shown}${found.length > 3 ? ', …' : ''} — `
|
|
1418
|
+
+ 'a question or exclamation may now read as a statement. Some languages mark a question with a word or '
|
|
1419
|
+
+ 'particle instead of a mark, so this is a warning, not a refusal: check them'
|
|
1420
|
+
+ (fix ? `, or ask again (--fresh: the cache holds this answer): \`${fix}\`` : '') + '.';
|
|
1421
|
+
}
|
|
1422
|
+
|
|
480
1423
|
/**
|
|
481
1424
|
* Log quality gate failures in a structured, actionable format.
|
|
482
1425
|
*
|
|
@@ -485,21 +1428,31 @@ function isAsciiOnly(text) {
|
|
|
485
1428
|
*/
|
|
486
1429
|
function logGateFailures(failures, pairKey) {
|
|
487
1430
|
if (failures.length === 0) return;
|
|
1431
|
+
if (output.getMode() === 'json') {
|
|
1432
|
+
// One structured record instead of indented prose on stderr.
|
|
1433
|
+
output.warn(`${pairKey}: ${failures.length} key(s) failed quality validation`, {
|
|
1434
|
+
pairKey, gateFailures: failures.map(({ key, reason, value }) => ({ key, reason, value })),
|
|
1435
|
+
});
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
488
1438
|
|
|
489
|
-
|
|
1439
|
+
const lines = ['', ` [GATE] ${pairKey}: ${failures.length} key(s) failed quality validation:`];
|
|
490
1440
|
for (const { key, reason, value } of failures) {
|
|
491
|
-
|
|
492
|
-
if (value) {
|
|
493
|
-
console.error(` → "${value}"`);
|
|
494
|
-
}
|
|
1441
|
+
lines.push(` ✗ "${key}": ${reason}`);
|
|
1442
|
+
if (value) lines.push(` → "${value}"`);
|
|
495
1443
|
}
|
|
496
|
-
|
|
1444
|
+
lines.push('');
|
|
1445
|
+
output.block(lines);
|
|
497
1446
|
}
|
|
498
1447
|
|
|
499
1448
|
export {
|
|
1449
|
+
checkICUStructure,
|
|
500
1450
|
validateTranslations,
|
|
1451
|
+
contentGateFault,
|
|
501
1452
|
measureRepetition,
|
|
502
1453
|
isAsciiOnly,
|
|
1454
|
+
isLatinOnly,
|
|
1455
|
+
hasForeignFullwidthLatin,
|
|
503
1456
|
logGateFailures,
|
|
504
1457
|
checkContentPreservation,
|
|
505
1458
|
contentCharacters,
|
|
@@ -507,4 +1460,26 @@ export {
|
|
|
507
1460
|
NON_LATIN_LOCALES,
|
|
508
1461
|
DEFAULT_THRESHOLDS,
|
|
509
1462
|
MIN_MEASURABLE_CONTENT,
|
|
1463
|
+
isProtectedTermValue,
|
|
1464
|
+
LATIN_NAME_OR_LABEL,
|
|
1465
|
+
foldForEchoCompare,
|
|
1466
|
+
letterWordCount,
|
|
1467
|
+
nameCore,
|
|
1468
|
+
GATE_VERSION,
|
|
1469
|
+
isKeepAsWrittenFault,
|
|
1470
|
+
isBibliographicEntry,
|
|
1471
|
+
tableProse,
|
|
1472
|
+
SHORT_SOURCE_LENGTH_SLACK,
|
|
1473
|
+
isDisguisedEcho,
|
|
1474
|
+
pluralBranchEcho,
|
|
1475
|
+
MIN_FOLDED_ECHO_WORDS,
|
|
1476
|
+
DISGUISED_ECHO,
|
|
1477
|
+
SharedOutputIndex,
|
|
1478
|
+
sharedOutputItems,
|
|
1479
|
+
sharedOutputReason,
|
|
1480
|
+
SHARED_OUTPUT_MIN_SOURCES,
|
|
1481
|
+
droppedTerminalMark,
|
|
1482
|
+
droppedTerminalMarks,
|
|
1483
|
+
describeDroppedMarks,
|
|
1484
|
+
SHARED_OUTPUT_MIN_WORDS,
|
|
510
1485
|
};
|