champollion 0.3.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +41 -26
  2. package/bin/cli.js +53 -5
  3. package/index.js +63 -2
  4. package/lib/api-key.js +17 -4
  5. package/lib/autofix.js +82 -37
  6. package/lib/bridge/method_bridge.py +15 -3
  7. package/lib/cards/reader.js +34 -0
  8. package/lib/cards/remote.js +15 -0
  9. package/lib/cards/search-names.js +178 -0
  10. package/lib/command-help.js +286 -85
  11. package/lib/commands/audit.js +10 -3
  12. package/lib/commands/card.js +583 -226
  13. package/lib/commands/doctor.js +54 -18
  14. package/lib/commands/help.js +37 -32
  15. package/lib/commands/init.js +1709 -101
  16. package/lib/commands/integrity.js +127 -40
  17. package/lib/commands/leaderboard.js +187 -67
  18. package/lib/commands/models.js +9 -2
  19. package/lib/commands/provenance.js +7 -2
  20. package/lib/commands/recommend.js +43 -14
  21. package/lib/commands/register-corpus.js +632 -125
  22. package/lib/commands/seal-corpus.js +1 -1
  23. package/lib/commands/status.js +564 -27
  24. package/lib/commands/submit.js +17 -12
  25. package/lib/commands/sync.js +31 -7
  26. package/lib/commands/tm.js +15 -9
  27. package/lib/commands/verify.js +27 -3
  28. package/lib/commands/wrap.js +63 -5
  29. package/lib/commands/xliff.js +135 -64
  30. package/lib/commercial-eligibility.js +1 -1
  31. package/lib/config.js +230 -17
  32. package/lib/content-estimate.js +96 -0
  33. package/lib/content-refusals.js +284 -0
  34. package/lib/content-review.js +372 -0
  35. package/lib/content-sync.js +1136 -345
  36. package/lib/content.js +94 -7
  37. package/lib/corpus-registration.mjs +194 -35
  38. package/lib/cost-label.js +29 -0
  39. package/lib/cost-report.js +727 -79
  40. package/lib/diff.js +38 -4
  41. package/lib/docusaurus-sync.js +975 -262
  42. package/lib/edit-distance.js +31 -0
  43. package/lib/fallback.js +1102 -0
  44. package/lib/file-scope.js +106 -0
  45. package/lib/flatten.js +80 -3
  46. package/lib/flutter-locales.js +124 -0
  47. package/lib/format.js +266 -12
  48. package/lib/hash.js +146 -21
  49. package/lib/icu-structure.js +929 -0
  50. package/lib/integrity.js +223 -75
  51. package/lib/language-pair.js +157 -0
  52. package/lib/lint.js +78 -16
  53. package/lib/local-only-marks.js +106 -0
  54. package/lib/locale-layout.js +1103 -0
  55. package/lib/locale-state.js +575 -0
  56. package/lib/methods/anthropic.js +5 -0
  57. package/lib/methods/apertium.js +6 -3
  58. package/lib/methods/api.js +138 -25
  59. package/lib/methods/base.js +17 -0
  60. package/lib/methods/coaching-data.js +153 -0
  61. package/lib/methods/content-separator.js +43 -0
  62. package/lib/methods/deepl.js +1 -1
  63. package/lib/methods/direct-llm.js +257 -103
  64. package/lib/methods/external.js +146 -63
  65. package/lib/methods/gemini.js +1 -0
  66. package/lib/methods/google-translate.js +1 -0
  67. package/lib/methods/http-utils.js +41 -0
  68. package/lib/methods/libretranslate.js +7 -2
  69. package/lib/methods/llm-coached.js +71 -129
  70. package/lib/methods/llm.js +87 -33
  71. package/lib/methods/local.js +93 -10
  72. package/lib/methods/microsoft-translator.js +1 -2
  73. package/lib/methods/openai.js +4 -2
  74. package/lib/methods/openrouter-client.js +20 -19
  75. package/lib/methods/openrouter-pricing.js +150 -13
  76. package/lib/methods/prompt-methods.js +20 -0
  77. package/lib/methods/provider-pricing.js +42 -1
  78. package/lib/methods/request-capture.js +104 -0
  79. package/lib/methods/tilde.js +1 -1
  80. package/lib/methods/translated.js +1 -2
  81. package/lib/missing-key.js +93 -0
  82. package/lib/models.js +102 -55
  83. package/lib/name-rules.js +32 -0
  84. package/lib/named-keys.js +172 -0
  85. package/lib/no-translate.js +4 -3
  86. package/lib/output.js +160 -19
  87. package/lib/pairs.js +586 -30
  88. package/lib/placeholders.js +394 -0
  89. package/lib/plugins.js +8 -0
  90. package/lib/plural-gap-redo.js +109 -0
  91. package/lib/plurals.js +323 -0
  92. package/lib/po.js +1187 -0
  93. package/lib/public-catalogue.js +74 -0
  94. package/lib/recommend.js +527 -32
  95. package/lib/redo.js +95 -0
  96. package/lib/refusal-category.js +44 -0
  97. package/lib/registers.js +255 -11
  98. package/lib/repair-script.js +20 -13
  99. package/lib/scripts.js +6 -1
  100. package/lib/seal.mjs +4 -3
  101. package/lib/sealed-qualifier.mjs +1 -1
  102. package/lib/segment.js +24 -8
  103. package/lib/seo.js +19 -9
  104. package/lib/serve.js +43 -6
  105. package/lib/shared-output-seed.js +164 -0
  106. package/lib/source-contexts.js +39 -0
  107. package/lib/submit.mjs +57 -5
  108. package/lib/sync.js +2923 -474
  109. package/lib/terminology.js +13 -4
  110. package/lib/tm-evict.js +179 -0
  111. package/lib/tm-seed.js +5 -2
  112. package/lib/tm.js +818 -36
  113. package/lib/translate-pair.js +639 -34
  114. package/lib/translate.js +78 -5
  115. package/lib/types.js +22 -3
  116. package/lib/validate.js +996 -21
  117. package/lib/verify.js +1332 -104
  118. package/lib/watch.js +32 -13
  119. package/lib/xliff.js +44 -3
  120. package/package.json +1 -1
  121. package/shared/CORPORA-CARDS.md +2 -0
  122. package/shared/cards-fallback.json +1 -1
  123. package/shared/curated-orthography-conventions.json +26 -8
  124. package/shared/gettext-plural-forms.json +45 -0
  125. package/shared/licenses.json +99 -3
  126. package/shared/method-registry.json +2 -0
  127. package/shared/metric-registry.json +96 -18
  128. package/shared/retired-model-aliases.json +9 -0
  129. package/shared/schemas/champollion-plugin.schema.json +4 -0
  130. package/shared/schemas/corpora-card.schema.json +8 -2
  131. package/shared/schemas/human-services.schema.json +1 -1
  132. package/shared/schemas/method-index-record.schema.json +67 -0
  133. package/shared/schemas/method-registry.schema.json +4 -0
  134. package/shared/schemas/metric-registry.schema.json +55 -1
  135. package/shared/schemas/retired-model-aliases.schema.json +32 -0
  136. package/shared/docent/corpus.json +0 -11739
  137. package/shared/model-aliases.json +0 -7
  138. package/shared/schemas/model-aliases.schema.json +0 -27
package/lib/validate.js CHANGED
@@ -21,8 +21,26 @@
21
21
  * source-relative caps)
22
22
  * 2. Length ratio check (source vs translated length)
23
23
  * 3. Script compliance (non-Latin locales must produce non-ASCII)
24
- * 4. Source echo check (translated value ≠ source value)
24
+ * 4. Source echo check (translated value ≠ source value — and, for a
25
+ * source of 3+ words, not the source with only case, diacritics,
26
+ * spacing or invisible characters changed: see isDisguisedEcho)
25
27
  * 5. Content preservation (the output is not the source, hollowed out)
28
+ * 6. ICU / placeholder structure (lib/icu-structure.js): argument names,
29
+ * plural/select keywords, selectors, # and printf conversions are
30
+ * code — only the text inside the branches may change; markup tags
31
+ * are code too (checkMarkup: same tags opened, closed, nested); and
32
+ * i18next {{…}} / single-brace / tag tokens by the rule verify reports
33
+ * them by (lib/placeholders.js placeholderChanges); and a sentence
34
+ * break the translation inserts right beside a placeholder where the
35
+ * source has none (placeholderSentenceBreaks)
36
+ *
37
+ * 7. Plural forms (lib/icu-structure.js pluralGaps): a plural message
38
+ * that passed everything else but has no branch for a category the
39
+ * target language uses for ordinary counts (Russian few/many) is a
40
+ * SOFT failure (`pluralGap`): the caller asks once more, naming the
41
+ * missing forms; a second answer without them is accepted
42
+ * (acceptPluralGaps) and reported — never a retry loop, and never a
43
+ * form made up by the tool.
26
44
  *
27
45
  * Keys that fail any check are removed and logged as [GATE] failures.
28
46
  * The caller receives only validated translations.
@@ -33,6 +51,9 @@
33
51
  */
34
52
 
35
53
  import { getAllLanguageCodes, getLanguageCard } from './registers.js';
54
+ import { output } from './output.js';
55
+ import { checkICUStructure, checkMarkup, pluralGaps, describeCategories, pluralBranchPairs, pluralBranchTexts } from './icu-structure.js';
56
+ import { placeholderChanges, placeholderSentenceBreaks, PLACEHOLDER_SYNTAX_NAMES } from './placeholders.js';
36
57
 
37
58
  /**
38
59
  * Locales whose scripts are predominantly non-Latin.
@@ -242,11 +263,313 @@ function checkContentPreservation(source, translated, minRetention = DEFAULT_THR
242
263
  };
243
264
  }
244
265
 
266
+ // ---------------------------------------------------------------------------
267
+ // Disguised echoes — the source handed back with accents sprinkled on
268
+ // ---------------------------------------------------------------------------
269
+
270
+ /**
271
+ * Fewest words (whitespace-delimited, each carrying a letter) a source needs
272
+ * before a FOLDED match counts as an echo. Below it, only exact equality does.
273
+ *
274
+ * WHY A FLOOR: in a Latin-script target a short legitimate translation can
275
+ * differ from English only by accents — 'cafe' → 'café', 'Resume' → 'Résumé',
276
+ * 'Cafe Menu' → 'Café Menu' — and refusing those would cost users a retry and
277
+ * then the key. Three independent words that all survive translation
278
+ * unchanged apart from accents and case is what a copy looks like.
279
+ */
280
+ const MIN_FOLDED_ECHO_WORDS = 3;
281
+
282
+ /** Combining marks (Mn) and invisible format characters (Cf). */
283
+ const FOLD_STRIP = /[\p{Mn}\p{Cf}]/gu;
284
+
285
+ /**
286
+ * The form two strings are compared in to decide "is this output the source?".
287
+ *
288
+ * Twin of the fold in arena/mt_eval_harness/text_compare.py (its steps 1–4,
289
+ * `_fold` / token_compare_key, plus its whitespace collapse), so the CLI gate
290
+ * sees through the same disguises the harness does:
291
+ * 1. NFKD — precomposed letters split into base + combining mark, and
292
+ * compatibility forms (fullwidth letters, ligatures) fold to plain ones;
293
+ * 2. case folding (JavaScript has no casefold(); upper-then-lower makes the
294
+ * same equality decisions for this purpose: 'ß' and 'ss' meet, and the
295
+ * Greek sigma forms meet — JS lands on the contextual 'ς' where Python
296
+ * lands on 'σ', but both sides of a comparison land together), then
297
+ * NFKD again because a case mapping can produce a decomposable
298
+ * character;
299
+ * 3. drop combining marks (Mn: 'á' → 'a'; 'ŋ' is a letter and stays) and
300
+ * format characters (Cf: zero-width space/joiners, bidi marks, soft
301
+ * hyphen) — they change the bytes, not the text a reader sees;
302
+ * 4. collapse whitespace runs and trim.
303
+ * Punctuation is compared AS WRITTEN (the harness's per-token rule). Its
304
+ * whole-string form also turns punctuation into spaces; this gate does not —
305
+ * a gate that REFUSES output (and spends a retry) keeps the narrower fold, so
306
+ * a copy that also edits its punctuation is not caught here, as before.
307
+ *
308
+ * @param {string} text
309
+ * @returns {string}
310
+ */
311
+ function foldForEchoCompare(text) {
312
+ return String(text)
313
+ .normalize('NFKD')
314
+ .toUpperCase()
315
+ .toLowerCase()
316
+ .normalize('NFKD')
317
+ .replace(FOLD_STRIP, '')
318
+ .replace(/\s+/gu, ' ')
319
+ .trim();
320
+ }
321
+
322
+ /**
323
+ * How many words carrying a letter `text` has — after setting aside what is
324
+ * code rather than words: ICU/brace placeholders ("{count}"), printf
325
+ * conversions ("%s", "%1$d") and markup tags ("<b>", "</a>").
326
+ *
327
+ * @param {string} text
328
+ * @returns {number}
329
+ */
330
+ function letterWordCount(text) {
331
+ return String(text)
332
+ .replace(/\{[^}]*\}/g, ' ')
333
+ .replace(/%(\d+\$)?[-+ 0#]*\d*(\.\d+)?[a-zA-Z@]/g, ' ')
334
+ .replace(/<[^>]*>/g, ' ')
335
+ .split(/\s+/u)
336
+ .filter((w) => /\p{L}/u.test(w))
337
+ .length;
338
+ }
339
+
340
+ /**
341
+ * Is `translated` the source in disguise — identical once case, diacritics,
342
+ * spacing and invisible format characters are folded away, without being
343
+ * byte-identical? ('Thank you very much!' → 'Thánk yóú véry múch!')
344
+ *
345
+ * Only for a source of MIN_FOLDED_ECHO_WORDS+ words with letters; a shorter
346
+ * source keeps exact equality as its only echo test, exactly as before.
347
+ * An exact copy is NOT a disguised echo — it keeps its own rule (and the
348
+ * short-name lane), so this predicate changes nothing for it.
349
+ *
350
+ * Why the short-name lane does not apply here: that lane exists because a
351
+ * short value is often a NAME correctly kept as written ("GitHub", "Curtis
352
+ * Forbes"). A disguised copy is by construction not kept as written — it was
353
+ * altered without being translated — so it is refused like any long echo.
354
+ *
355
+ * Exported so an on-disk auditor that adopts it applies the same floor.
356
+ *
357
+ * @param {string} source
358
+ * @param {string} translated
359
+ * @returns {boolean}
360
+ */
361
+ function isDisguisedEcho(source, translated) {
362
+ if (typeof source !== 'string' || typeof translated !== 'string') return false;
363
+ if (translated === source) return false;
364
+ if (letterWordCount(source) < MIN_FOLDED_ECHO_WORDS) return false;
365
+ return foldForEchoCompare(translated) === foldForEchoCompare(source);
366
+ }
367
+
368
+ /**
369
+ * The first echo among a plural message's branches (validateTranslations
370
+ * check 2c), as a failure record's fields, or null.
371
+ *
372
+ * @returns {{ reason: string, disguisedEcho?: true, nameOrLabel?: true }|null}
373
+ */
374
+ function pluralBranchEcho(source, translated, { isNonLatin = false, requireNonLatin = true, acceptLatinNames = false, protectedTerms = [] } = {}) {
375
+ const pairs = pluralBranchPairs(source, translated);
376
+ if (pairs.length === 0) return null;
377
+ const exact = [];
378
+ const disguised = [];
379
+ const nameLike = [];
380
+ for (const p of pairs) {
381
+ const src = p.source.trim();
382
+ const tgt = p.translated.trim();
383
+ if (!/\p{L}/u.test(src) || isProtectedTermValue(tgt, protectedTerms)) continue;
384
+ if (tgt === src) {
385
+ const asciiRatio = src.replace(/[^\x20-\x7E]/g, '').length / Math.max(src.length, 1);
386
+ const isShortAscii = src.length <= 30 && asciiRatio > 0.8;
387
+ if (!isShortAscii) exact.push(p.selector);
388
+ else if (isNonLatin && requireNonLatin && !acceptLatinNames) nameLike.push(p.selector);
389
+ } else if (isDisguisedEcho(src, tgt)) {
390
+ disguised.push(p.selector);
391
+ }
392
+ }
393
+ const forms = (sels) => [...new Set(sels)].map(x => `"${x}"`).join(', ');
394
+ if (disguised.length > 0) {
395
+ return { reason: `${DISGUISED_ECHO} — in the plural form(s) ${forms(disguised)}`, disguisedEcho: true };
396
+ }
397
+ if (exact.length > 0) {
398
+ return { reason: `source echo (identical to English) in the plural form(s) ${forms(exact)}` };
399
+ }
400
+ if (nameLike.length > 0) {
401
+ return { reason: `${LATIN_NAME_OR_LABEL} (plural form(s) ${forms(nameLike)})`, nameOrLabel: true };
402
+ }
403
+ return null;
404
+ }
405
+
406
+ /** Gate reason for a disguised echo. */
407
+ const DISGUISED_ECHO =
408
+ 'source echo (the source handed back with only case, accents, spacing or invisible characters changed — not a translation)';
409
+
245
410
  // A deliberately repetitive source ("Every language, into every language.")
246
411
  // licenses an equally repetitive translation: the effective caps are raised
247
412
  // to the source's own measured repetition plus this margin.
248
413
  const REPETITION_SOURCE_MARGIN = 0.10;
249
414
 
415
+ /** Gate reason for a short Latin-script value in a non-Latin target. */
416
+ const LATIN_NAME_OR_LABEL = 'kept in Latin script — a name, or a label left untranslated?';
417
+
418
+ /**
419
+ * Is this value made ONLY of names the project declared (config.protectedTerms)?
420
+ *
421
+ * Strips every declared term, then ICU placeholders, digits, punctuation,
422
+ * symbols and whitespace; nothing left means the value is a name (or names)
423
+ * kept as written — correct in every script, never an "untranslated" error.
424
+ * "Curtis Forbes", "Game Day Suits", "Curtis Forbes · Game Day Suits" all
425
+ * qualify; "Curtis Forbes's portfolio" does not (the rest needs translating).
426
+ *
427
+ * @param {string} value
428
+ * @param {string[]} [protectedTerms]
429
+ * @returns {boolean}
430
+ */
431
+ function isProtectedTermValue(value, protectedTerms = []) {
432
+ if (typeof value !== 'string' || !protectedTerms || protectedTerms.length === 0) return false;
433
+ let rest = value;
434
+ let matched = false;
435
+ // Longest first, so "Game Day Suits Ltd" is removed before "Game Day Suits".
436
+ for (const term of [...protectedTerms].sort((a, b) => b.length - a.length)) {
437
+ if (typeof term === 'string' && term && rest.includes(term)) {
438
+ rest = rest.split(term).join(' ');
439
+ matched = true;
440
+ }
441
+ }
442
+ if (!matched) return false;
443
+ return rest.replace(/\{[^}]*\}/g, '').replace(/[\d\s\p{P}\p{S}]/gu, '').length === 0;
444
+ }
445
+
446
+ /**
447
+ * The gate's version. A refusal is remembered with the version that made it
448
+ * (lib/content-refusals.js, lib/locale-state.js), and a hold made by an
449
+ * earlier gate lifts by itself: what an over-strict gate refused is asked
450
+ * again once the gate is fixed, instead of staying in the source language
451
+ * until someone names it for a redo. Bump it whenever a check is loosened.
452
+ * 2 — 2026-10-05: names with citations/anchors, reference entries, tables,
453
+ * short titles and source-shown fullwidth letters stopped being refused.
454
+ */
455
+ const GATE_VERSION = 2;
456
+
457
+ /**
458
+ * Refusals that may be the model keeping text that is correct as written —
459
+ * a name, title, citation, identifier or code: the source handed back, or
460
+ * Latin script kept in a non-Latin target. Told why and asked again, a model
461
+ * that gives the same answer twice is taken at its word (lib/fallback.js
462
+ * retryRefusedBlocks), as the key-value lane takes a name it is asked about
463
+ * twice (LATIN_NAME_OR_LABEL). Every other refusal (repetition, length,
464
+ * hollowing, damaged markup, fullwidth disguise) is never accepted this way.
465
+ *
466
+ * @param {string|null} reason
467
+ * @returns {boolean}
468
+ */
469
+ function isKeepAsWrittenFault(reason) {
470
+ if (typeof reason !== 'string') return false;
471
+ return reason.startsWith('source echo')
472
+ || reason === LATIN_NAME_OR_LABEL
473
+ || reason.startsWith('wrong script (ASCII-only');
474
+ }
475
+
476
+ /**
477
+ * How much longer than its source a value may be before the length ratio
478
+ * counts. The ratio is meaningless on a tiny source: "FAQ" → "Preguntas
479
+ * frecuentes" is 6.7x and correct (dogfood 2026-10-05, a docs page title),
480
+ * while "Feast" → a whole sentence still grows by far more than this.
481
+ */
482
+ const SHORT_SOURCE_LENGTH_SLACK = 20;
483
+
484
+ /**
485
+ * The part of a string that would need translating, for the "short name kept
486
+ * as written" rule: inline code, quoted strings, parentheticals and Markdown
487
+ * block markers removed. "### METEOR (Banerjee & Lavie, 2005)" is the name
488
+ * METEOR, and "## Hugo (TOML / YAML / Markdown)" the name Hugo — both were
489
+ * refused as source echo on the whole string's length (dogfood 2026-10-05).
490
+ *
491
+ * @param {string} text
492
+ * @returns {string}
493
+ */
494
+ function nameCore(text) {
495
+ return String(text)
496
+ .replace(/^[ \t]*(?:#{1,6}[ \t]+|[-*+][ \t]+|\d{1,9}[.)][ \t]+)/, '')
497
+ .replace(/\{#[^}\s]+\}\s*$/, ' ')
498
+ .replace(/`[^`]*`/g, ' ')
499
+ .replace(/"[^"]*"|“[^”]*”|「[^」]*」/g, ' ')
500
+ .replace(/\([^()]*\)|([^()]*)/g, ' ')
501
+ .replace(/\s+/g, ' ')
502
+ .trim();
503
+ }
504
+
505
+ /**
506
+ * A reference-list entry — numbered, with a year or a link, and a quoted or
507
+ * italic title: "4. Snover, M., … (2006). \"A Study of …\" *Proceedings …*".
508
+ * Cited works are kept as published, so the same text back is correct, not
509
+ * an echo (dogfood 2026-10-05: twelve references of the scoring spec were
510
+ * refused and published with the fallback prefix).
511
+ *
512
+ * @param {string} text
513
+ * @returns {boolean}
514
+ */
515
+ function isBibliographicEntry(text) {
516
+ const t = String(text).trim();
517
+ // A reference list in one block: every entry must be one.
518
+ if (t.includes('\n')) {
519
+ const lines = t.split('\n').map(l => l.trim()).filter(Boolean);
520
+ return lines.length > 1 && lines.every(l => !l.includes('\n') && isBibliographicEntry(l));
521
+ }
522
+ return /^(?:\[?[A-Z]?\d{1,3}\]?[.)]?|[-*+])\s+\S/.test(t)
523
+ && (/\(\d{4}[a-z]?\)/.test(t) || /\]\(https?:\/\//.test(t))
524
+ && (/"[^"]{8,}"|“[^”]{8,}”/.test(t) || /\*[^*]{8,}\*/.test(t));
525
+ }
526
+
527
+ /**
528
+ * A Markdown table measured as its cells' text: delimiter rows dropped and
529
+ * pipes turned to spaces. The delimiter row ("|---|---|") is repetition by
530
+ * construction, so a table whose model output padded it differently was
531
+ * refused as a repetition hallucination (dogfood 2026-10-05, four tables).
532
+ * Anything that is not a table comes back unchanged.
533
+ *
534
+ * @param {string} text
535
+ * @returns {string}
536
+ */
537
+ function tableProse(text) {
538
+ const lines = String(text).split('\n');
539
+ if (!lines.some((l) => /^\s*\|/.test(l))) return text;
540
+ return lines
541
+ .filter((l) => !/^\s*\|?(?:\s*:?-{3,}:?\s*\|)+\s*:?-{0,}:?\s*$/.test(l))
542
+ .map((l) => (/^\s*\|/.test(l) ? l.replace(/\|/g, ' ').replace(/[ \t]+/g, ' ').trim() : l))
543
+ .join('\n');
544
+ }
545
+
546
+ /**
547
+ * Why the quality gate refuses one Markdown block or front-matter field, or
548
+ * null when it passes — the key-value gate's own checks (empty, source echo,
549
+ * disguised echo, repetition, length inflation and truncation, hollowing,
550
+ * script), so a short heading turned into a sentence is refused in a
551
+ * newsletter exactly as it is for an app key (Round 7, school persona: "##
552
+ * Feast" became a full sentence and was accepted, while the same output for
553
+ * the key Nav.home was refused at 9.5x). Structure (code, links, markup) is
554
+ * checked by the lanes through their protected placeholders; a short
555
+ * Latin-script value kept as written is a name, as on the key-value lane's
556
+ * second ask (there is no per-block retry to ask once first).
557
+ *
558
+ * @param {string} source - The block or field's source text (placeholders restored)
559
+ * @param {string} value - The translation
560
+ * @param {object} [pairConfig] - The pair (target locale, thresholds, protectedTerms)
561
+ * @returns {string|null}
562
+ */
563
+ function contentGateFault(source, value, pairConfig = {}) {
564
+ if (typeof source !== 'string' || typeof value !== 'string') return null;
565
+ if (value === source && isBibliographicEntry(source)) return null;
566
+ source = tableProse(source);
567
+ value = tableProse(value);
568
+ const { failures } = validateTranslations({ block: value }, { block: source }, pairConfig || {},
569
+ { prose: true, acceptLatinNames: true, acceptPluralGaps: true });
570
+ return failures.length > 0 ? failures[0].reason : null;
571
+ }
572
+
250
573
  /**
251
574
  * Validate a batch of translations and return only passing keys.
252
575
  *
@@ -272,6 +595,17 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
272
595
 
273
596
  const validated = {};
274
597
  const failures = [];
598
+ // config.protectedTerms rides the pair config (lib/pairs.js).
599
+ const protectedTerms = options.protectedTerms ?? pairConfig.protectedTerms ?? [];
600
+ // Second pass of the name-or-label retry: a short Latin-script value the
601
+ // model returned twice is accepted as a name.
602
+ const acceptLatinNames = options.acceptLatinNames === true;
603
+ // Second pass of the plural-forms retry: accepted, and reported by the caller.
604
+ const acceptPluralGaps = options.acceptPluralGaps === true;
605
+ // A Markdown block or front-matter field (contentGateFault): its structure
606
+ // (code, links, markup) rides protected placeholders, checked by the lane;
607
+ // the ICU/markup/plural checks are for key-value messages.
608
+ const prose = options.prose === true;
275
609
 
276
610
  for (const [key, translated] of Object.entries(translations)) {
277
611
  const source = sourceFlat[key] || '';
@@ -298,20 +632,120 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
298
632
  continue;
299
633
  }
300
634
 
635
+ // Names the project declared (config.protectedTerms) are correct as
636
+ // written in every language — no echo or script check applies.
637
+ if (isProtectedTermValue(translated, protectedTerms)) {
638
+ validated[key] = translated;
639
+ continue;
640
+ }
641
+
301
642
  // Check 2: Source echo — translated value is identical to source.
302
- // EXEMPTION: Short strings (≤30 chars) that are mostly ASCII are likely
303
- // proper nouns, brand names, or technical terms (e.g. "Blog", "GitHub",
304
- // "npm", "CLI Reference") that legitimately stay in English across all
305
- // languages. Rejecting these creates an infinite retry loop where the
306
- // correct translation is rejected every time, burning API calls forever.
643
+ // Short mostly-ASCII strings (≤30 chars) are often names ("GitHub",
644
+ // "Curtis Forbes") that legitimately stay as written — but just as often
645
+ // descriptive labels the model failed to translate ("Translation CLI").
646
+ // - Latin-script targets: accepted (an English label in French reads
647
+ // as a missed translation, not a broken page; verify warns).
648
+ // - Non-Latin targets: rejected ONCE with a name-or-label hint (the
649
+ // caller's feedback retry). If the model returns the same text again
650
+ // it is taken at its word as a name (acceptLatinNames) and cached, so
651
+ // it is never re-billed. One bounded retry — never a retry loop.
307
652
  if (translated === source) {
308
- const asciiRatio = source.replace(/[^\x20-\x7E]/g, '').length / Math.max(source.length, 1);
309
- const isShortAscii = source.length <= 30 && asciiRatio > 0.8;
653
+ const core = nameCore(source);
654
+ const asciiRatio = core.replace(/[^\x20-\x7E]/g, '').length / Math.max(core.length, 1);
655
+ // Nothing left once code, quotes and parentheticals go: nothing to translate.
656
+ const isShortAscii = core.length === 0 || (core.length <= 30 && asciiRatio > 0.8);
310
657
  if (!isShortAscii) {
311
658
  failures.push({ key, reason: 'source echo (identical to English)', value: translated });
312
659
  continue;
313
660
  }
314
- // Short ASCII string echoed back — accept it as a valid translation
661
+ if (isNonLatin && thresholds.requireNonLatin && !acceptLatinNames) {
662
+ failures.push({ key, reason: LATIN_NAME_OR_LABEL, value: translated, nameOrLabel: true });
663
+ continue;
664
+ }
665
+ } else if (isDisguisedEcho(source, translated)) {
666
+ // Check 2a: the same echo, disguised — 'Thank you very much!' →
667
+ // 'Thánk yóú véry múch!'. Byte comparison calls it different, and the
668
+ // script check cannot fire (Latin in, Latin out; and in a non-Latin
669
+ // target the accents make it not-ASCII). Same suppressions as the
670
+ // exact check: declared names were accepted above, no-translate keys
671
+ // never reach the gate, letter-free and short (< 3-word) sources keep
672
+ // exact equality. Refused like a long echo — the caller's feedback
673
+ // retry, then the fallback method — never accepted as a name, because
674
+ // a name kept as written is byte-identical (isDisguisedEcho).
675
+ failures.push({ key, reason: DISGUISED_ECHO, value: translated, disguisedEcho: true });
676
+ continue;
677
+ }
678
+
679
+ // Check 2b: ICU MessageFormat / placeholder structure. Argument names,
680
+ // plural/select keywords, selectors, # and printf conversions are code:
681
+ // "{cóúnt, plúrál, óné {…} óthér {…}}" breaks the app (next-intl throws,
682
+ // Flutter gen-l10n refuses to build) while reading as a fine French
683
+ // string. Only the text inside the branches may change; a plural may
684
+ // ADD the CLDR categories the target language uses. Runs before the
685
+ // statistical checks so the retry hears the precise reason.
686
+ if (translated !== source && !prose) {
687
+ const icu = checkICUStructure(source, translated, targetLocale);
688
+ if (icu) {
689
+ failures.push({ key, reason: icu.reason, value: translated, icu: true });
690
+ continue;
691
+ }
692
+ // Markup is code as well: every tag opened, closed and nested as in the
693
+ // source ("<strong>Book</strong>" → "<strong>Réserver" breaks the page).
694
+ const markup = checkMarkup(source, translated);
695
+ if (markup) {
696
+ failures.push({ key, reason: markup.reason, value: translated, markup: true });
697
+ continue;
698
+ }
699
+ // The other placeholders verify reports — i18next {{…}} (renamed, lost,
700
+ // or written as a single-brace {name}, which i18next prints as is), a
701
+ // single-brace {name} outside an ICU message, a tag token — by the one
702
+ // rule verify reports them by (lib/placeholders.js). The gate let
703
+ // {{name}} → {{nom}} through, verify flagged it after the sync, and the
704
+ // run exited 2; refused here, the pair's fallback gets the text instead
705
+ // (Round 12, i18next persona).
706
+ const tokens = placeholderChanges(source, translated);
707
+ if (tokens.length > 0) {
708
+ failures.push({
709
+ key,
710
+ reason: `placeholder structure damaged: ${tokens.map(t => `${PLACEHOLDER_SYNTAX_NAMES[t.syntax]} ${t.issue}`).join('; ')}`,
711
+ value: translated,
712
+ placeholder: true,
713
+ });
714
+ continue;
715
+ }
716
+ // A sentence break the translation put right beside a placeholder
717
+ // where the source has none: "Take this medicine at {time}." →
718
+ // "… sina. {time}." passed every check (Round 14, hospital persona).
719
+ // The same rule verify flags by (lib/placeholders.js); refused here,
720
+ // the retry hears why and the pair's fallback runs.
721
+ const breaks = placeholderSentenceBreaks(source, translated);
722
+ if (breaks.length > 0) {
723
+ failures.push({
724
+ key,
725
+ reason: `sentence break beside a placeholder: ${breaks.map(b => b.issue).join('; ')}`,
726
+ value: translated,
727
+ placeholder: true,
728
+ });
729
+ continue;
730
+ }
731
+ }
732
+
733
+ // Check 2c: the echo checks, per plural/select branch. A plural message
734
+ // differs from its source as a WHOLE as soon as the target adds a form
735
+ // (or one branch is translated), so the two checks above never saw a
736
+ // branch handed back as the English — accented, or as it was (Round 4,
737
+ // Django persona: the plural entry passed while the identical singular
738
+ // change was refused). Each branch is held to the singular rules: an
739
+ // exact copy of a short Latin-script branch is accepted in a Latin-script
740
+ // target (a name, "emails") and asked about once in a non-Latin one; any
741
+ // longer copy is refused; a disguised copy (3+ words) is refused. A
742
+ // branch made only of declared names (protectedTerms) is never an echo.
743
+ if (translated !== source && !prose) {
744
+ const branchEcho = pluralBranchEcho(source, translated, { isNonLatin, requireNonLatin: thresholds.requireNonLatin, acceptLatinNames, protectedTerms });
745
+ if (branchEcho) {
746
+ failures.push({ key, value: translated, ...branchEcho });
747
+ continue;
748
+ }
315
749
  }
316
750
 
317
751
  // Check 3: Repetition detection — catches hallucination loops.
@@ -353,7 +787,7 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
353
787
  // Check 4: Length ratio — catches padding and truncation
354
788
  if (source.length > 0) {
355
789
  const ratio = translated.length / source.length;
356
- if (ratio > thresholds.maxLengthRatio) {
790
+ if (ratio > thresholds.maxLengthRatio && translated.length - source.length > SHORT_SOURCE_LENGTH_SLACK) {
357
791
  failures.push({
358
792
  key,
359
793
  reason: `length inflation (${ratio.toFixed(1)}x source, max ${thresholds.maxLengthRatio}x)`,
@@ -387,19 +821,62 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
387
821
  // or version strings like "3.2.0" have nothing to write in another script.
388
822
  // - Short ASCII strings (≤30 chars, >80% ASCII) are likely proper nouns
389
823
  // or brand names (e.g. "GitHub", "npm") that stay in English everywhere.
824
+ // Fullwidth Latin letters ("Book an appointment") outside CJK typography:
825
+ // English in disguise, whatever the target's script. Never a name — a
826
+ // name kept as written is in plain letters.
827
+ if (hasForeignFullwidthLatin(translated, targetLocale) && !hasForeignFullwidthLatin(source, targetLocale)) {
828
+ failures.push({
829
+ key,
830
+ reason: `wrong script (fullwidth Latin letters in a ${targetLocale} value — English in disguise, not a translation)`,
831
+ value: translated.slice(0, 80),
832
+ });
833
+ continue;
834
+ }
390
835
  if (isNonLatin && thresholds.requireNonLatin) {
391
836
  // Strip ICU placeholders, digits, punctuation, whitespace → what's left?
392
837
  const translatableText = translated
393
838
  .replace(/\{[^}]*\}/g, '') // ICU placeholders
394
839
  .replace(/[\d\s\p{P}\p{S}]/gu, '') // digits, whitespace, punctuation, symbols
395
840
  .trim();
396
- const asciiRatio = source.replace(/[^\x20-\x7E]/g, '').length / Math.max(source.length, 1);
397
- const isShortAsciiPropNoun = source.length <= 30 && asciiRatio > 0.8;
398
- if (translatableText.length > 0 && isAsciiOnly(translated) && !isShortAsciiPropNoun) {
841
+ const core = nameCore(source);
842
+ const asciiRatio = core.replace(/[^\x20-\x7E]/g, '').length / Math.max(core.length, 1);
843
+ const isShortAscii = core.length === 0 || (core.length <= 30 && asciiRatio > 0.8);
844
+ // Letters classified by Unicode script: accented Latin is Latin too
845
+ // (an ASCII test passed "Thánk yóú" in a Russian catalog).
846
+ if (translatableText.length > 0 && isLatinOnly(translated, targetLocale)) {
847
+ if (isShortAscii && acceptLatinNames) {
848
+ // Second answer still Latin-script: the model says it is a name.
849
+ } else if (isShortAscii) {
850
+ failures.push({ key, reason: LATIN_NAME_OR_LABEL, value: translated.slice(0, 80), nameOrLabel: true });
851
+ continue;
852
+ } else {
853
+ failures.push({
854
+ key,
855
+ reason: `wrong script (ASCII-only for ${targetLocale}, expected non-Latin characters)`,
856
+ value: translated.slice(0, 80),
857
+ });
858
+ continue;
859
+ }
860
+ }
861
+ }
862
+
863
+ // Check 7 (last, so a soft failure means everything else passed): a
864
+ // plural message without a form the target language uses for ordinary
865
+ // counts. Soft — the caller asks once more; see the module header.
866
+ if (!acceptPluralGaps && !prose && translated !== source) {
867
+ // A gettext catalog's slots (pairConfig.pluralSlots, set by sync for a
868
+ // .po file): a form it has no msgstr[] for is never asked again for.
869
+ const gaps = pluralGaps(source, translated, targetLocale, options.pluralSlots ?? pairConfig.pluralSlots ?? null)
870
+ .filter(g => g.everyday.length > 0);
871
+ if (gaps.length > 0) {
872
+ const missing = [...new Set(gaps.flatMap(g => g.everyday))];
873
+ const type = gaps[0].type;
399
874
  failures.push({
400
875
  key,
401
- reason: `wrong script (ASCII-only for ${targetLocale}, expected non-Latin characters)`,
402
- value: translated.slice(0, 80),
876
+ reason: `plural form(s) ${missing.map(c => `"${c}"`).join(', ')} missing — ${targetLocale} uses `
877
+ + `${describeCategories(targetLocale, missing, type)}`,
878
+ value: translated,
879
+ pluralGap: { missing, type },
403
880
  });
404
881
  continue;
405
882
  }
@@ -412,6 +889,336 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
412
889
  return { validated, failures };
413
890
  }
414
891
 
892
+ // ---------------------------------------------------------------------------
893
+ // Different inputs, same output
894
+ // ---------------------------------------------------------------------------
895
+
896
+ /** Distinct source strings one output may answer before it is suspect. */
897
+ const SHARED_OUTPUT_MIN_SOURCES = 3;
898
+ /** Words (with letters) the shared output needs, on its own, to be suspect. */
899
+ const SHARED_OUTPUT_MIN_WORDS = 4;
900
+
901
+ /**
902
+ * Simple placeholders and markup tags in an output — not words of the
903
+ * translation (SharedOutputIndex.outputForm): {name} {0} {{count}} %(name)s
904
+ * %s %1$d <b> </a>. A plural/select argument ({n, plural, …}) has a comma and
905
+ * is not matched.
906
+ */
907
+ const PLACEHOLDER_IN_OUTPUT = /\{\{\s*[\w.$-]+\s*\}\}|\{\s*[\w.$-]+\s*\}|%\([\w.]+\)[sdifr]|%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?[sdifuxXeEgGcp@]|<\/?[A-Za-z][^<>]*>/gu;
908
+ /** The same without single-brace arguments: for a whole plural/select message. */
909
+ const PLACEHOLDER_NOT_BRACED = /\{\{\s*[\w.$-]+\s*\}\}|%\([\w.]+\)[sdifr]|%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?[sdifuxXeEgGcp@]|<\/?[A-Za-z][^<>]*>/gu;
910
+ /** A plural/select/selectordinal argument ("{n, plural, …"). */
911
+ const ICU_COMPLEX_ARGUMENT = /\{\s*[\w.$-]+\s*,\s*(?:plural|select|selectordinal)\s*,/u;
912
+
913
+ /**
914
+ * The sentences of a text, for the repeat check: split after . ! ? (and the
915
+ * CJK 。!?) followed by space, each kept only when it has a letter once
916
+ * its placeholders are gone ("S. {name}!" is one sentence).
917
+ *
918
+ * @param {string} text
919
+ * @returns {string[]}
920
+ */
921
+ function sentencesOf(text) {
922
+ return String(text)
923
+ .split(/(?<=[.!?\u3002\uFF01\uFF1F])\s+/u)
924
+ .map(t => t.trim())
925
+ .filter(t => /\p{L}/u.test(t.replace(PLACEHOLDER_IN_OUTPUT, ' ')));
926
+ }
927
+
928
+ /** A source in the form two sources are compared in ("Save", "save!" meet). */
929
+ function sourceIdentity(text) {
930
+ return foldForEchoCompare(String(text).replace(/[\p{P}\p{S}]+/gu, ' '));
931
+ }
932
+
933
+ /** The words of a folded string, as a set. */
934
+ function wordSet(text) {
935
+ return new Set(sourceIdentity(text).split(' ').filter(w => /\p{L}/u.test(w)));
936
+ }
937
+
938
+ /** True when every pair of sources shares under half its words (Jaccard). */
939
+ function clearlyDifferent(sources) {
940
+ const sets = sources.map(wordSet);
941
+ for (let i = 0; i < sets.length; i++) {
942
+ for (let j = i + 1; j < sets.length; j++) {
943
+ const a = sets[i];
944
+ const b = sets[j];
945
+ const inter = [...a].filter(w => b.has(w)).length;
946
+ const union = new Set([...a, ...b]).size;
947
+ if (union === 0 || inter / union >= 0.5) return false;
948
+ }
949
+ }
950
+ return true;
951
+ }
952
+
953
+ /**
954
+ * Different inputs, same output: ONE translation text returned for several
955
+ * DIFFERENT source strings is what a model that memorized a training sentence
956
+ * does (Round 4 school persona: a trained eng→crk model answered the app
957
+ * title, "Contact the school", the newsletter title and its heading with the
958
+ * same sentence — and every per-key check passed it).
959
+ *
960
+ * Suspect when one output answers SHARED_OUTPUT_MIN_SOURCES+ distinct sources
961
+ * (compared folded, punctuation aside — "Save" and "Save!" are one source)
962
+ * AND either the output has SHARED_OUTPUT_MIN_WORDS+ words, or every source
963
+ * has two or more words and no two of them share half their words. So
964
+ * synonyms collapsing to one short translation pass ('OK'/'Okay'/'Sure' →
965
+ * "D'accord"; 'Close'/'Dismiss'/'Cancel' → "Fermer"), and so does the same
966
+ * source text under several keys.
967
+ *
968
+ * TWO sources are already enough when the evidence is strong (Round 6,
969
+ * school persona: 'Thank you, {name}!' and 'Please bring the forms.' got one
970
+ * memorized sentence and passed, because only a third source counted): the
971
+ * output has SHARED_OUTPUT_MIN_WORDS+ words, both sources have two or more
972
+ * words, and they share under half their words (clearly different). Short
973
+ * outputs and synonym-like sources (sharing half their words or more) still
974
+ * need a third source. The cost of a false alarm is one more ask (an LLM is
975
+ * told to translate this string on its own; a method that takes no
976
+ * instructions is asked once more), then the pair's fallback.
977
+ *
978
+ * MEMORIZED outputs (markMemorized): an output an earlier sync found
979
+ * answering different source strings is suspect from its first source on —
980
+ * a repair that asks again and gets the same sentence back must not write it
981
+ * (Round 6, school persona: `--force-content` re-served it in silence).
982
+ *
983
+ * Same suppressions as the echo checks: a value made only of declared names
984
+ * (protectedTerms) is never suspect, nor a value equal to its own source
985
+ * (an echo — its own rule), nor one without letters. No-translate keys never
986
+ * reach a gate.
987
+ *
988
+ * The index spans a run's locale: the caller adds what it accepted, so a
989
+ * later batch (or a content file's block) that repeats an earlier answer is
990
+ * caught; only the NEW items are returned as suspect — what was written
991
+ * earlier is reported by `verify`. An output found suspect earlier in the
992
+ * run is suspect from its first new source on, like a MEMORIZED one: its
993
+ * refused members were never added, and the run remembers it only when it
994
+ * ends.
995
+ */
996
+ class SharedOutputIndex {
997
+ constructor({ protectedTerms = [] } = {}) {
998
+ this.protectedTerms = protectedTerms;
999
+ // output form → Map(source identity → { key, source })
1000
+ this.byOutput = new Map();
1001
+ // output form → the first text seen in that form (what reports show)
1002
+ this.display = new Map();
1003
+ // output text → { value, sources, keys } — every group found suspect,
1004
+ // with ALL its members (earlier, accepted ones too), for the run report.
1005
+ this.flagged = new Map();
1006
+ // output form → the text as first seen: outputs an earlier sync found
1007
+ // answering different source strings (markMemorized).
1008
+ this.memorized = new Map();
1009
+ }
1010
+
1011
+ /**
1012
+ * Outputs an earlier run found answering several different source strings:
1013
+ * suspect from their first source on (see the class comment).
1014
+ *
1015
+ * @param {Iterable<string>} values
1016
+ */
1017
+ markMemorized(values) {
1018
+ for (const value of values || []) {
1019
+ if (typeof value !== 'string') continue;
1020
+ const form = SharedOutputIndex.outputForm(value);
1021
+ if (form && /\p{L}/u.test(form)) this.memorized.set(form, String(value).replace(/\s+/gu, ' ').trim());
1022
+ }
1023
+ }
1024
+
1025
+ /**
1026
+ * The form outputs are grouped by: case, punctuation and symbols aside, and
1027
+ * without a Markdown block marker — so "S?", "S." and a heading "# S" are
1028
+ * one output (Round 5: an NMT model copies the source's end punctuation,
1029
+ * and a newsletter H1 kept its "# "). Letters and their accents are kept:
1030
+ * two words that differ by a diacritic are two words.
1031
+ *
1032
+ * Placeholders and markup tags are not words of the output: "S. {name}!"
1033
+ * and "S." are one output (Round 9, school persona: the memorized sentence
1034
+ * came back as "S. {name}!" for 'Thank you, {name}!' and as "S." inside a
1035
+ * newsletter paragraph, and the "name" left over from "{name}" kept the two
1036
+ * apart). Simple arguments only ({name}, {0}, {{count}}, %(name)s, %s,
1037
+ * %1$d, <b>…</b>); an unsplit plural/select message keeps its branches.
1038
+ */
1039
+ static outputForm(value) {
1040
+ const text = String(value);
1041
+ // Inside a whole plural/select message, `{un}` is a branch, not an argument.
1042
+ const placeholders = ICU_COMPLEX_ARGUMENT.test(text) ? PLACEHOLDER_NOT_BRACED : PLACEHOLDER_IN_OUTPUT;
1043
+ return text
1044
+ .normalize('NFC')
1045
+ .replace(/^\s{0,3}(?:#{1,6}\s+|[-*+]\s+|\d+[.)]\s+|>\s*)/u, '')
1046
+ .replace(placeholders, ' ')
1047
+ .toLowerCase()
1048
+ .replace(/[\p{P}\p{S}]+/gu, ' ')
1049
+ .replace(/\s+/gu, ' ')
1050
+ .trim();
1051
+ }
1052
+
1053
+ /**
1054
+ * The items, plus — for a value of several sentences whose source has as
1055
+ * many — each sentence on its own, paired with its source sentence (the
1056
+ * parent kept in `whole`). One memorized sentence inside a paragraph is
1057
+ * then the same output as that sentence answering a UI string (Round 9,
1058
+ * school persona: 'Please bring the forms.' in the newsletter and 'Thank
1059
+ * you, {name}!' got one sentence, and the whole-paragraph comparison never
1060
+ * met it). Pairing by position keeps a sentence two paragraphs genuinely
1061
+ * share ("Please bring the forms." in both) as ONE source — not suspect;
1062
+ * a value whose sentence count differs from its source's is compared whole
1063
+ * only.
1064
+ *
1065
+ * @param {Array<{ key: string, source: string, value: string }>} items
1066
+ * @returns {Array<{ key: string, source: string, value: string, whole?: { source: string, value: string } }>}
1067
+ */
1068
+ static withSentences(items) {
1069
+ const out = [];
1070
+ for (const it of items) {
1071
+ out.push(it);
1072
+ if (typeof it.source !== 'string' || typeof it.value !== 'string' || it.whole) continue;
1073
+ const vs = sentencesOf(it.value);
1074
+ if (vs.length < 2) continue;
1075
+ const ss = sentencesOf(it.source);
1076
+ if (ss.length !== vs.length) continue;
1077
+ vs.forEach((v, i) => out.push({ key: it.key, source: ss[i], value: v, whole: { source: it.source, value: it.value } }));
1078
+ }
1079
+ return out;
1080
+ }
1081
+
1082
+ eligible(source, value) {
1083
+ if (typeof source !== 'string' || typeof value !== 'string') return false;
1084
+ if (value === source || !/\p{L}/u.test(value) || !SharedOutputIndex.outputForm(value)) return false;
1085
+ if (isProtectedTermValue(value, this.protectedTerms)) return false;
1086
+ return true;
1087
+ }
1088
+
1089
+ /**
1090
+ * Items that would make (or join) a suspect group.
1091
+ *
1092
+ * @param {Array<{ key: string, source: string, value: string }>} items
1093
+ * @returns {Map<string, { value: string, sources: string[] }>} key → its group
1094
+ */
1095
+ suspects(items) {
1096
+ const groups = new Map();
1097
+ for (const it of SharedOutputIndex.withSentences(items)) {
1098
+ if (!this.eligible(it.source, it.value)) continue;
1099
+ const form = SharedOutputIndex.outputForm(it.value);
1100
+ if (!this.display.has(form)) this.display.set(form, String(it.value).replace(/\s+/gu, ' ').trim());
1101
+ if (!groups.has(form)) {
1102
+ groups.set(form, { sources: new Map(this.byOutput.get(form) || []), fresh: [] });
1103
+ }
1104
+ const g = groups.get(form);
1105
+ const id = sourceIdentity(it.source);
1106
+ if (!g.sources.has(id)) g.sources.set(id, { key: it.key, source: it.source });
1107
+ g.fresh.push(it);
1108
+ }
1109
+ const out = new Map();
1110
+ for (const [form, g] of groups) {
1111
+ let sources = [...g.sources.values()].map(s => s.source);
1112
+ const byRule = SharedOutputIndex.suspectGroup(form, sources);
1113
+ // `memorized` marks a group suspect ONLY because an earlier sync
1114
+ // remembered its output (reports word it that way); a group the rule
1115
+ // catches on its own is reported as before.
1116
+ const memorized = !byRule && this.memorized.has(form);
1117
+ // An output this index already found answering different source
1118
+ // strings EARLIER IN THIS RUN is suspect from its first new source on,
1119
+ // as a remembered one is from an earlier sync. The members refused
1120
+ // then were never added (only accepted items are), so without this the
1121
+ // same sentence came back for the newsletter's title after the app's
1122
+ // three keys were refused for it, and was written — the run remembers
1123
+ // it only when it ends, and `verify` flagged it (Round 10, school
1124
+ // persona). A gate retry that returns the refused sentence for one key
1125
+ // is refused the same way.
1126
+ const earlier = !byRule && !memorized ? this.flagged.get(form) : null;
1127
+ if (earlier) sources = [...new Set([...earlier.sources, ...sources])];
1128
+ if (!byRule && !memorized && !earlier) continue;
1129
+ const shown = this.memorized.get(form) || this.display.get(form) || form;
1130
+ for (const it of g.fresh) out.set(it.key, { value: shown, sources, ...(memorized && { memorized: true }) });
1131
+ const prior = this.flagged.get(form) || { value: shown, sources: [], keys: [] };
1132
+ this.flagged.set(form, {
1133
+ value: shown,
1134
+ sources: [...new Set([...prior.sources, ...sources])],
1135
+ keys: [...new Set([...prior.keys, ...[...g.sources.values()].map(v => v.key), ...g.fresh.map(it => it.key)])],
1136
+ ...(memorized && { memorized: true }),
1137
+ });
1138
+ }
1139
+ return out;
1140
+ }
1141
+
1142
+ /**
1143
+ * Is one output form answering these distinct sources the pattern? Three or
1144
+ * more: a long output, or clearly different multi-word sources. Two: only
1145
+ * the strong case (see the class comment).
1146
+ *
1147
+ * @param {string} form - outputForm() of the shared output
1148
+ * @param {string[]} sources - Distinct sources it answers
1149
+ * @returns {boolean}
1150
+ */
1151
+ static suspectGroup(form, sources) {
1152
+ const outWords = letterWordCount(form);
1153
+ if (sources.length >= SHARED_OUTPUT_MIN_SOURCES) {
1154
+ return outWords >= SHARED_OUTPUT_MIN_WORDS
1155
+ || (sources.every(src => letterWordCount(src) >= 2) && clearlyDifferent(sources));
1156
+ }
1157
+ if (sources.length !== 2) return false;
1158
+ return outWords >= SHARED_OUTPUT_MIN_WORDS
1159
+ && sources.every(src => letterWordCount(src) >= 2)
1160
+ && clearlyDifferent(sources);
1161
+ }
1162
+
1163
+ /**
1164
+ * Record accepted items (what was written) for later batches. A sentence
1165
+ * of a longer value is recorded with its parent (`whole`): what was written,
1166
+ * and cached, is the whole value.
1167
+ */
1168
+ add(items) {
1169
+ for (const it of SharedOutputIndex.withSentences(items)) {
1170
+ if (!this.eligible(it.source, it.value)) continue;
1171
+ const form = SharedOutputIndex.outputForm(it.value);
1172
+ if (!this.display.has(form)) this.display.set(form, String(it.value).replace(/\s+/gu, ' ').trim());
1173
+ if (!this.byOutput.has(form)) this.byOutput.set(form, new Map());
1174
+ const id = sourceIdentity(it.source);
1175
+ if (!this.byOutput.get(form).has(id)) {
1176
+ this.byOutput.get(form).set(id, { key: it.key, source: it.source, value: it.value, ...(it.whole && { whole: it.whole }) });
1177
+ }
1178
+ }
1179
+ }
1180
+ }
1181
+
1182
+ /**
1183
+ * What one translated value puts in the shared-output index: the value
1184
+ * itself, or — for an ICU plural/select message — each leaf branch on its
1185
+ * own (Round 5, school persona: one memorized sentence filled BOTH branches
1186
+ * of a plural message and two other keys, and nothing counted the branches).
1187
+ *
1188
+ * Every branch of one `plural`/`selectordinal` argument counts as ONE source
1189
+ * (the message's): a language without number inflection writes the same
1190
+ * text in every branch by design, so repeating it across branches is never
1191
+ * evidence on its own. `select` branches count as their own source branches —
1192
+ * different only when those differ.
1193
+ *
1194
+ * @param {string} key
1195
+ * @param {string} source
1196
+ * @param {string} value
1197
+ * @returns {Array<{ key: string, source: string, value: string }>}
1198
+ */
1199
+ function sharedOutputItems(key, source, value) {
1200
+ if (typeof source !== 'string' || typeof value !== 'string') return [];
1201
+ const pairs = value.includes('{') && source.includes('{') ? pluralBranchPairs(source, value) : [];
1202
+ if (pairs.length === 0) return [{ key, source, value }];
1203
+ return pairs.map(p => ({
1204
+ key,
1205
+ source: p.keyword === 'select' ? p.source : source,
1206
+ value: p.translated,
1207
+ }));
1208
+ }
1209
+
1210
+ /** Gate reason for an output shared by several different sources. */
1211
+ function sharedOutputReason(group) {
1212
+ if (group.memorized) {
1213
+ const text = group.value.length > 60 ? `${group.value.slice(0, 57)}...` : group.value;
1214
+ return `the sentence the model gave for other, different source strings on an earlier sync (${JSON.stringify(text)}) `
1215
+ + '— a memorized sentence, not a translation of this string; asking the same model again returns it again';
1216
+ }
1217
+ const shown = group.sources.slice(0, 4).map(s => JSON.stringify(s.length > 40 ? `${s.slice(0, 37)}...` : s)).join(', ');
1218
+ return `same output for ${group.sources.length} different source strings (${shown}${group.sources.length > 4 ? ', …' : ''}) `
1219
+ + '— a model repeating one memorized sentence, not a translation of this string';
1220
+ }
1221
+
415
1222
  /**
416
1223
  * Split a value into independently-measurable segments: pipe-delimited
417
1224
  * plural variants ("one doc|{count} docs") legitimately share most of
@@ -421,6 +1228,10 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
421
1228
  * @returns {string[]} Trimmed segments (always at least one)
422
1229
  */
423
1230
  function splitPluralSegments(text) {
1231
+ // ICU plural/select branches too: a Russian plural repeats one sentence in
1232
+ // four forms by design, and measured as one string it read as a loop.
1233
+ const branches = text.includes('{') ? pluralBranchTexts(text) : null;
1234
+ if (branches) return branches.map(seg => seg.trim());
424
1235
  return (text.includes('|') ? text.split('|') : [text]).map(seg => seg.trim());
425
1236
  }
426
1237
 
@@ -477,6 +1288,138 @@ function isAsciiOnly(text) {
477
1288
  return /^[\x00-\x7F]*$/.test(text);
478
1289
  }
479
1290
 
1291
+ /** Scripts whose text legitimately uses fullwidth Latin letters (CJK typography). */
1292
+ const FULLWIDTH_SCRIPTS = new Set(['Hani', 'Hans', 'Hant', 'Jpan', 'Kore', 'Hira', 'Kana', 'Hang', 'Bopo']);
1293
+
1294
+ /** Fullwidth Latin letters: A–Z, a–z. */
1295
+ const FULLWIDTH_LATIN = /[A-Za-z]/u;
1296
+
1297
+ /**
1298
+ * The text of a value with its code set aside: ICU/brace placeholders,
1299
+ * printf conversions, markup tags. (A URL stays: in a non-Latin locale a
1300
+ * URL value is a wrong-script answer unless the key is declared
1301
+ * no-translate — which never reaches this check.)
1302
+ */
1303
+ function proseOf(text) {
1304
+ // A plural/select message: its prose is the text of its branches (the
1305
+ // keywords and selectors — "plural", "one", "other" — are code).
1306
+ const str = String(text);
1307
+ const branches = str.includes('{') ? pluralBranchTexts(str) : null;
1308
+ return (branches ? branches.join(' ') : str)
1309
+ // Simple placeholders only: {name}, {count, number}. Branch text such as
1310
+ // "{Один файл}" is prose, never stripped.
1311
+ .replace(/\{\s*[\w.$-]+\s*(?:,[^{}]*)?\}/g, ' ')
1312
+ .replace(/\{\{\s*[\w.$-]+\s*\}\}/g, ' ')
1313
+ .replace(/%(\([^)]*\))?[-#0 +]*\d*(\.\d+)?[a-zA-Z@]/g, ' ')
1314
+ .replace(/<\/?[A-Za-z0-9][^<>]*>/g, ' ');
1315
+ }
1316
+
1317
+ /**
1318
+ * Are ALL the letters of a value's prose Latin script — by Unicode script,
1319
+ * not by byte? Fullwidth Latin ("Hello") and accented Latin ("Thánk")
1320
+ * are Latin: an ASCII test let both through as "not English" in a Russian
1321
+ * catalog (Round 4, Django persona). False when the prose has no letters.
1322
+ *
1323
+ * @param {string} text
1324
+ * @returns {boolean}
1325
+ */
1326
+ function isLatinOnly(text, locale = null) {
1327
+ const prose = proseOf(text);
1328
+ const letters = prose.match(/\p{L}/gu);
1329
+ if (!letters) return false;
1330
+ // CJK typography sets Latin in fullwidth forms ("OK" in Japanese): there,
1331
+ // fullwidth letters are the target's own usage, as they always were here.
1332
+ if (locale && FULLWIDTH_LATIN.test(prose) && usesFullwidthLatin(locale)) return false;
1333
+ return letters.every(ch => /\p{Script=Latin}/u.test(ch));
1334
+ }
1335
+
1336
+ /** Does this locale's script set Latin in fullwidth forms (CJK)? */
1337
+ function usesFullwidthLatin(locale) {
1338
+ const card = getLanguageCard(String(locale)) || getLanguageCard(String(locale).split(/[-_]/)[0]);
1339
+ return FULLWIDTH_SCRIPTS.has(card?.script);
1340
+ }
1341
+
1342
+ /**
1343
+ * Fullwidth Latin letters in a value whose language does not set Latin in
1344
+ * fullwidth forms (anything but CJK typography): a disguised copy of English,
1345
+ * never a translation.
1346
+ *
1347
+ * @param {string} text
1348
+ * @param {string} locale - Target locale
1349
+ * @returns {boolean}
1350
+ */
1351
+ function hasForeignFullwidthLatin(text, locale) {
1352
+ if (!FULLWIDTH_LATIN.test(proseOf(text))) return false;
1353
+ return !usesFullwidthLatin(locale);
1354
+ }
1355
+
1356
+ // ---------------------------------------------------------------------------
1357
+ // A question or exclamation that lost its mark
1358
+ // ---------------------------------------------------------------------------
1359
+
1360
+ // Marks that end a question / an exclamation in some writing system: Latin
1361
+ // and most scripts (?), CJK fullwidth (?), Arabic script (؟), Greek (U+037E
1362
+ // and the ASCII semicolon Greek text is typed with), Armenian (՞), Ethiopic
1363
+ // (፧), and the interrobang family. Spanish closes with "?" / "!" as well.
1364
+ const QUESTION_ENDS = new Set(['?', '?', '؟', '\u037e', ';', '՞', '፧', '⸮', '⁇', '⁈', '‽', '⁉']);
1365
+ const EXCLAMATION_ENDS = new Set(['!', '!', '‼', '⁉', '⁈', '‽', '՜', '¡']);
1366
+ // Closing quotes, brackets, ICU braces and direction marks after the mark.
1367
+ const TRAILING_CLOSERS = /[\s"'»”’)\]}›〉》」』】〕\u200e\u200f]+$/u;
1368
+
1369
+ /** The last visible character of a text, closers aside. */
1370
+ function terminalChar(text) {
1371
+ const chars = Array.from(String(text).replace(TRAILING_CLOSERS, ''));
1372
+ return chars.length > 0 ? chars[chars.length - 1] : '';
1373
+ }
1374
+
1375
+ /**
1376
+ * A source that ends with "?" or "!" whose translation ends with neither it
1377
+ * nor an equivalent the target's script uses (Round 6, hospital persona:
1378
+ * "Where does it hurt?" passed as a statement). A WARNING, never a refusal:
1379
+ * some languages mark a question with a word or particle instead of a mark,
1380
+ * and a translation that does so is right.
1381
+ *
1382
+ * @param {string} source
1383
+ * @param {string} translated
1384
+ * @returns {{ mark: '?'|'!' } | null}
1385
+ */
1386
+ function droppedTerminalMark(source, translated) {
1387
+ if (typeof source !== 'string' || typeof translated !== 'string' || translated === source) return null;
1388
+ if (!/\p{L}/u.test(translated)) return null;
1389
+ const end = terminalChar(source);
1390
+ if (end !== '?' && end !== '!') return null;
1391
+ const got = terminalChar(translated);
1392
+ if (end === '?' ? QUESTION_ENDS.has(got) : EXCLAMATION_ENDS.has(got)) return null;
1393
+ return { mark: end };
1394
+ }
1395
+
1396
+ /**
1397
+ * Keys whose translation dropped the source's closing "?" / "!" — one
1398
+ * finding for sync and verify alike. Declared names and no-translate keys
1399
+ * are skipped by the callers' own rules.
1400
+ *
1401
+ * @param {Array<[string, string, string]>} entries - [key, source, translated]
1402
+ * @returns {Array<{ key: string, mark: string, value: string }>}
1403
+ */
1404
+ function droppedTerminalMarks(entries) {
1405
+ const out = [];
1406
+ for (const [key, source, value] of entries) {
1407
+ const d = droppedTerminalMark(source, value);
1408
+ if (d) out.push({ key, mark: d.mark, value });
1409
+ }
1410
+ return out;
1411
+ }
1412
+
1413
+ /** The warning text for droppedTerminalMarks findings (sync and verify say the same). */
1414
+ function describeDroppedMarks(found, fix) {
1415
+ const marks = [...new Set(found.map(f => `"${f.mark}"`))].join(' or ');
1416
+ const shown = found.slice(0, 3).map(f => `${f.key} (${JSON.stringify(f.value.length > 40 ? `${f.value.slice(0, 37)}...` : f.value)})`).join(', ');
1417
+ return `${found.length} translation(s) dropped the source's closing ${marks}: ${shown}${found.length > 3 ? ', …' : ''} — `
1418
+ + 'a question or exclamation may now read as a statement. Some languages mark a question with a word or '
1419
+ + 'particle instead of a mark, so this is a warning, not a refusal: check them'
1420
+ + (fix ? `, or ask again (--fresh: the cache holds this answer): \`${fix}\`` : '') + '.';
1421
+ }
1422
+
480
1423
  /**
481
1424
  * Log quality gate failures in a structured, actionable format.
482
1425
  *
@@ -485,21 +1428,31 @@ function isAsciiOnly(text) {
485
1428
  */
486
1429
  function logGateFailures(failures, pairKey) {
487
1430
  if (failures.length === 0) return;
1431
+ if (output.getMode() === 'json') {
1432
+ // One structured record instead of indented prose on stderr.
1433
+ output.warn(`${pairKey}: ${failures.length} key(s) failed quality validation`, {
1434
+ pairKey, gateFailures: failures.map(({ key, reason, value }) => ({ key, reason, value })),
1435
+ });
1436
+ return;
1437
+ }
488
1438
 
489
- console.error(`\n [GATE] ${pairKey}: ${failures.length} key(s) failed quality validation:`);
1439
+ const lines = ['', ` [GATE] ${pairKey}: ${failures.length} key(s) failed quality validation:`];
490
1440
  for (const { key, reason, value } of failures) {
491
- console.error(` ✗ "${key}": ${reason}`);
492
- if (value) {
493
- console.error(` → "${value}"`);
494
- }
1441
+ lines.push(` ✗ "${key}": ${reason}`);
1442
+ if (value) lines.push(` → "${value}"`);
495
1443
  }
496
- console.error('');
1444
+ lines.push('');
1445
+ output.block(lines);
497
1446
  }
498
1447
 
499
1448
  export {
1449
+ checkICUStructure,
500
1450
  validateTranslations,
1451
+ contentGateFault,
501
1452
  measureRepetition,
502
1453
  isAsciiOnly,
1454
+ isLatinOnly,
1455
+ hasForeignFullwidthLatin,
503
1456
  logGateFailures,
504
1457
  checkContentPreservation,
505
1458
  contentCharacters,
@@ -507,4 +1460,26 @@ export {
507
1460
  NON_LATIN_LOCALES,
508
1461
  DEFAULT_THRESHOLDS,
509
1462
  MIN_MEASURABLE_CONTENT,
1463
+ isProtectedTermValue,
1464
+ LATIN_NAME_OR_LABEL,
1465
+ foldForEchoCompare,
1466
+ letterWordCount,
1467
+ nameCore,
1468
+ GATE_VERSION,
1469
+ isKeepAsWrittenFault,
1470
+ isBibliographicEntry,
1471
+ tableProse,
1472
+ SHORT_SOURCE_LENGTH_SLACK,
1473
+ isDisguisedEcho,
1474
+ pluralBranchEcho,
1475
+ MIN_FOLDED_ECHO_WORDS,
1476
+ DISGUISED_ECHO,
1477
+ SharedOutputIndex,
1478
+ sharedOutputItems,
1479
+ sharedOutputReason,
1480
+ SHARED_OUTPUT_MIN_SOURCES,
1481
+ droppedTerminalMark,
1482
+ droppedTerminalMarks,
1483
+ describeDroppedMarks,
1484
+ SHARED_OUTPUT_MIN_WORDS,
510
1485
  };