champollion 0.3.3 → 0.4.0

package/lib/validate.js

@@ -21,8 +21,26 @@
  *        source-relative caps)
  *     2. Length ratio check (source vs translated length)
  *     3. Script compliance (non-Latin locales must produce non-ASCII)
- *     4. Source echo check (translated value ≠ source value)
+ *     4. Source echo check (translated value ≠ source value — and, for a
+ *        source of 3+ words, not the source with only case, diacritics,
+ *        spacing or invisible characters changed: see isDisguisedEcho)
  *     5. Content preservation (the output is not the source, hollowed out)
+ *     6. ICU / placeholder structure (lib/icu-structure.js): argument names,
+ *        plural/select keywords, selectors, # and printf conversions are
+ *        code — only the text inside the branches may change; markup tags
+ *        are code too (checkMarkup: same tags opened, closed, nested); and
+ *        i18next {{…}} / single-brace / tag tokens by the rule verify reports
+ *        them by (lib/placeholders.js placeholderChanges); and a sentence
+ *        break the translation inserts right beside a placeholder where the
+ *        source has none (placeholderSentenceBreaks)
+ *
+ *     7. Plural forms (lib/icu-structure.js pluralGaps): a plural message
+ *        that passed everything else but has no branch for a category the
+ *        target language uses for ordinary counts (Russian few/many) is a
+ *        SOFT failure (`pluralGap`): the caller asks once more, naming the
+ *        missing forms; a second answer without them is accepted
+ *        (acceptPluralGaps) and reported — never a retry loop, and never a
+ *        form made up by the tool.
  *
  *   Keys that fail any check are removed and logged as [GATE] failures.
  *   The caller receives only validated translations.
@@ -33,6 +51,9 @@
  */
 
 import { getAllLanguageCodes, getLanguageCard } from './registers.js';
+import { output } from './output.js';
+import { checkICUStructure, checkMarkup, pluralGaps, describeCategories, pluralBranchPairs, pluralBranchTexts } from './icu-structure.js';
+import { placeholderChanges, placeholderSentenceBreaks, PLACEHOLDER_SYNTAX_NAMES } from './placeholders.js';
 
 /**
  * Locales whose scripts are predominantly non-Latin.
@@ -242,11 +263,210 @@ function checkContentPreservation(source, translated, minRetention = DEFAULT_THR
   };
 }
 
+// ---------------------------------------------------------------------------
+// Disguised echoes — the source handed back with accents sprinkled on
+// ---------------------------------------------------------------------------
+
+/**
+ * Fewest words (whitespace-delimited, each carrying a letter) a source needs
+ * before a FOLDED match counts as an echo. Below it, only exact equality does.
+ *
+ * WHY A FLOOR: in a Latin-script target a short legitimate translation can
+ * differ from English only by accents — 'cafe' → 'café', 'Resume' → 'Résumé',
+ * 'Cafe Menu' → 'Café Menu' — and refusing those would cost users a retry and
+ * then the key. Three independent words that all survive translation
+ * unchanged apart from accents and case is what a copy looks like.
+ */
+const MIN_FOLDED_ECHO_WORDS = 3;
+
+/** Combining marks (Mn) and invisible format characters (Cf). */
+const FOLD_STRIP = /[\p{Mn}\p{Cf}]/gu;
+
+/**
+ * The form two strings are compared in to decide "is this output the source?".
+ *
+ * Twin of the fold in arena/mt_eval_harness/text_compare.py (its steps 1–4,
+ * `_fold` / token_compare_key, plus its whitespace collapse), so the CLI gate
+ * sees through the same disguises the harness does:
+ *   1. NFKD — precomposed letters split into base + combining mark, and
+ *      compatibility forms (fullwidth letters, ligatures) fold to plain ones;
+ *   2. case folding (JavaScript has no casefold(); upper-then-lower makes the
+ *      same equality decisions for this purpose: 'ß' and 'ss' meet, and the
+ *      Greek sigma forms meet — JS lands on the contextual 'ς' where Python
+ *      lands on 'σ', but both sides of a comparison land together), then
+ *      NFKD again because a case mapping can produce a decomposable
+ *      character;
+ *   3. drop combining marks (Mn: 'á' → 'a'; 'ŋ' is a letter and stays) and
+ *      format characters (Cf: zero-width space/joiners, bidi marks, soft
+ *      hyphen) — they change the bytes, not the text a reader sees;
+ *   4. collapse whitespace runs and trim.
+ * Punctuation is compared AS WRITTEN (the harness's per-token rule). Its
+ * whole-string form also turns punctuation into spaces; this gate does not —
+ * a gate that REFUSES output (and spends a retry) keeps the narrower fold, so
+ * a copy that also edits its punctuation is not caught here, as before.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function foldForEchoCompare(text) {
+  return String(text)
+    .normalize('NFKD')
+    .toUpperCase()
+    .toLowerCase()
+    .normalize('NFKD')
+    .replace(FOLD_STRIP, '')
+    .replace(/\s+/gu, ' ')
+    .trim();
+}
+
+/**
+ * How many words carrying a letter `text` has — after setting aside what is
+ * code rather than words: ICU/brace placeholders ("{count}"), printf
+ * conversions ("%s", "%1$d") and markup tags ("<b>", "</a>").
+ *
+ * @param {string} text
+ * @returns {number}
+ */
+function letterWordCount(text) {
+  return String(text)
+    .replace(/\{[^}]*\}/g, ' ')
+    .replace(/%(\d+\$)?[-+ 0#]*\d*(\.\d+)?[a-zA-Z@]/g, ' ')
+    .replace(/<[^>]*>/g, ' ')
+    .split(/\s+/u)
+    .filter((w) => /\p{L}/u.test(w))
+    .length;
+}
+
+/**
+ * Is `translated` the source in disguise — identical once case, diacritics,
+ * spacing and invisible format characters are folded away, without being
+ * byte-identical? ('Thank you very much!' → 'Thánk yóú véry múch!')
+ *
+ * Only for a source of MIN_FOLDED_ECHO_WORDS+ words with letters; a shorter
+ * source keeps exact equality as its only echo test, exactly as before.
+ * An exact copy is NOT a disguised echo — it keeps its own rule (and the
+ * short-name lane), so this predicate changes nothing for it.
+ *
+ * Why the short-name lane does not apply here: that lane exists because a
+ * short value is often a NAME correctly kept as written ("GitHub", "Curtis
+ * Forbes"). A disguised copy is by construction not kept as written — it was
+ * altered without being translated — so it is refused like any long echo.
+ *
+ * Exported so an on-disk auditor that adopts it applies the same floor.
+ *
+ * @param {string} source
+ * @param {string} translated
+ * @returns {boolean}
+ */
+function isDisguisedEcho(source, translated) {
+  if (typeof source !== 'string' || typeof translated !== 'string') return false;
+  if (translated === source) return false;
+  if (letterWordCount(source) < MIN_FOLDED_ECHO_WORDS) return false;
+  return foldForEchoCompare(translated) === foldForEchoCompare(source);
+}
+
+/**
+ * The first echo among a plural message's branches (validateTranslations
+ * check 2c), as a failure record's fields, or null.
+ *
+ * @returns {{ reason: string, disguisedEcho?: true, nameOrLabel?: true }|null}
+ */
+function pluralBranchEcho(source, translated, { isNonLatin = false, requireNonLatin = true, acceptLatinNames = false, protectedTerms = [] } = {}) {
+  const pairs = pluralBranchPairs(source, translated);
+  if (pairs.length === 0) return null;
+  const exact = [];
+  const disguised = [];
+  const nameLike = [];
+  for (const p of pairs) {
+    const src = p.source.trim();
+    const tgt = p.translated.trim();
+    if (!/\p{L}/u.test(src) || isProtectedTermValue(tgt, protectedTerms)) continue;
+    if (tgt === src) {
+      const asciiRatio = src.replace(/[^\x20-\x7E]/g, '').length / Math.max(src.length, 1);
+      const isShortAscii = src.length <= 30 && asciiRatio > 0.8;
+      if (!isShortAscii) exact.push(p.selector);
+      else if (isNonLatin && requireNonLatin && !acceptLatinNames) nameLike.push(p.selector);
+    } else if (isDisguisedEcho(src, tgt)) {
+      disguised.push(p.selector);
+    }
+  }
+  const forms = (sels) => [...new Set(sels)].map(x => `"${x}"`).join(', ');
+  if (disguised.length > 0) {
+    return { reason: `${DISGUISED_ECHO} — in the plural form(s) ${forms(disguised)}`, disguisedEcho: true };
+  }
+  if (exact.length > 0) {
+    return { reason: `source echo (identical to English) in the plural form(s) ${forms(exact)}` };
+  }
+  if (nameLike.length > 0) {
+    return { reason: `${LATIN_NAME_OR_LABEL} (plural form(s) ${forms(nameLike)})`, nameOrLabel: true };
+  }
+  return null;
+}
+
+/** Gate reason for a disguised echo. */
+const DISGUISED_ECHO =
+  'source echo (the source handed back with only case, accents, spacing or invisible characters changed — not a translation)';
+
 // A deliberately repetitive source ("Every language, into every language.")
 // licenses an equally repetitive translation: the effective caps are raised
 // to the source's own measured repetition plus this margin.
 const REPETITION_SOURCE_MARGIN = 0.10;
 
+/** Gate reason for a short Latin-script value in a non-Latin target. */
+const LATIN_NAME_OR_LABEL = 'kept in Latin script — a name, or a label left untranslated?';
+
+/**
+ * Is this value made ONLY of names the project declared (config.protectedTerms)?
+ *
+ * Strips every declared term, then ICU placeholders, digits, punctuation,
+ * symbols and whitespace; nothing left means the value is a name (or names)
+ * kept as written — correct in every script, never an "untranslated" error.
+ * "Curtis Forbes", "Game Day Suits", "Curtis Forbes · Game Day Suits" all
+ * qualify; "Curtis Forbes's portfolio" does not (the rest needs translating).
+ *
+ * @param {string} value
+ * @param {string[]} [protectedTerms]
+ * @returns {boolean}
+ */
+function isProtectedTermValue(value, protectedTerms = []) {
+  if (typeof value !== 'string' || !protectedTerms || protectedTerms.length === 0) return false;
+  let rest = value;
+  let matched = false;
+  // Longest first, so "Game Day Suits Ltd" is removed before "Game Day Suits".
+  for (const term of [...protectedTerms].sort((a, b) => b.length - a.length)) {
+    if (typeof term === 'string' && term && rest.includes(term)) {
+      rest = rest.split(term).join(' ');
+      matched = true;
+    }
+  }
+  if (!matched) return false;
+  return rest.replace(/\{[^}]*\}/g, '').replace(/[\d\s\p{P}\p{S}]/gu, '').length === 0;
+}
+
+/**
+ * Why the quality gate refuses one Markdown block or front-matter field, or
+ * null when it passes — the key-value gate's own checks (empty, source echo,
+ * disguised echo, repetition, length inflation and truncation, hollowing,
+ * script), so a short heading turned into a sentence is refused in a
+ * newsletter exactly as it is for an app key (Round 7, school persona: "##
+ * Feast" became a full sentence and was accepted, while the same output for
+ * the key Nav.home was refused at 9.5x). Structure (code, links, markup) is
+ * checked by the lanes through their protected placeholders; a short
+ * Latin-script value kept as written is a name, as on the key-value lane's
+ * second ask (there is no per-block retry to ask once first).
+ *
+ * @param {string} source - The block or field's source text (placeholders restored)
+ * @param {string} value - The translation
+ * @param {object} [pairConfig] - The pair (target locale, thresholds, protectedTerms)
+ * @returns {string|null}
+ */
+function contentGateFault(source, value, pairConfig = {}) {
+  if (typeof source !== 'string' || typeof value !== 'string') return null;
+  const { failures } = validateTranslations({ block: value }, { block: source }, pairConfig || {},
+    { prose: true, acceptLatinNames: true, acceptPluralGaps: true });
+  return failures.length > 0 ? failures[0].reason : null;
+}
+
 /**
  * Validate a batch of translations and return only passing keys.
  *
@@ -272,6 +492,17 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
 
   const validated = {};
   const failures = [];
+  // config.protectedTerms rides the pair config (lib/pairs.js).
+  const protectedTerms = options.protectedTerms ?? pairConfig.protectedTerms ?? [];
+  // Second pass of the name-or-label retry: a short Latin-script value the
+  // model returned twice is accepted as a name.
+  const acceptLatinNames = options.acceptLatinNames === true;
+  // Second pass of the plural-forms retry: accepted, and reported by the caller.
+  const acceptPluralGaps = options.acceptPluralGaps === true;
+  // A Markdown block or front-matter field (contentGateFault): its structure
+  // (code, links, markup) rides protected placeholders, checked by the lane;
+  // the ICU/markup/plural checks are for key-value messages.
+  const prose = options.prose === true;
 
   for (const [key, translated] of Object.entries(translations)) {
     const source = sourceFlat[key] || '';
@@ -298,12 +529,23 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
       continue;
     }
 
+    // Names the project declared (config.protectedTerms) are correct as
+    // written in every language — no echo or script check applies.
+    if (isProtectedTermValue(translated, protectedTerms)) {
+      validated[key] = translated;
+      continue;
+    }
+
     // Check 2: Source echo — translated value is identical to source.
-    // EXEMPTION: Short strings (≤30 chars) that are mostly ASCII are likely
-    // proper nouns, brand names, or technical terms (e.g. "Blog", "GitHub",
-    // "npm", "CLI Reference") that legitimately stay in English across all
-    // languages. Rejecting these creates an infinite retry loop where the
-    // correct translation is rejected every time, burning API calls forever.
+    // Short mostly-ASCII strings (≤30 chars) are often names ("GitHub",
+    // "Curtis Forbes") that legitimately stay as written — but just as often
+    // descriptive labels the model failed to translate ("Translation CLI").
+    //   - Latin-script targets: accepted (an English label in French reads
+    //     as a missed translation, not a broken page; verify warns).
+    //   - Non-Latin targets: rejected ONCE with a name-or-label hint (the
+    //     caller's feedback retry). If the model returns the same text again
+    //     it is taken at its word as a name (acceptLatinNames) and cached, so
+    //     it is never re-billed. One bounded retry — never a retry loop.
     if (translated === source) {
       const asciiRatio = source.replace(/[^\x20-\x7E]/g, '').length / Math.max(source.length, 1);
       const isShortAscii = source.length <= 30 && asciiRatio > 0.8;
@@ -311,7 +553,94 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
         failures.push({ key, reason: 'source echo (identical to English)', value: translated });
         continue;
       }
-      // Short ASCII string echoed back — accept it as a valid translation
+      if (isNonLatin && thresholds.requireNonLatin && !acceptLatinNames) {
+        failures.push({ key, reason: LATIN_NAME_OR_LABEL, value: translated, nameOrLabel: true });
+        continue;
+      }
+    } else if (isDisguisedEcho(source, translated)) {
+      // Check 2a: the same echo, disguised — 'Thank you very much!' →
+      // 'Thánk yóú véry múch!'. Byte comparison calls it different, and the
+      // script check cannot fire (Latin in, Latin out; and in a non-Latin
+      // target the accents make it not-ASCII). Same suppressions as the
+      // exact check: declared names were accepted above, no-translate keys
+      // never reach the gate, letter-free and short (< 3-word) sources keep
+      // exact equality. Refused like a long echo — the caller's feedback
+      // retry, then the fallback method — never accepted as a name, because
+      // a name kept as written is byte-identical (isDisguisedEcho).
+      failures.push({ key, reason: DISGUISED_ECHO, value: translated, disguisedEcho: true });
+      continue;
+    }
+
+    // Check 2b: ICU MessageFormat / placeholder structure. Argument names,
+    // plural/select keywords, selectors, # and printf conversions are code:
+    // "{cóúnt, plúrál, óné {…} óthér {…}}" breaks the app (next-intl throws,
+    // Flutter gen-l10n refuses to build) while reading as a fine French
+    // string. Only the text inside the branches may change; a plural may
+    // ADD the CLDR categories the target language uses. Runs before the
+    // statistical checks so the retry hears the precise reason.
+    if (translated !== source && !prose) {
+      const icu = checkICUStructure(source, translated, targetLocale);
+      if (icu) {
+        failures.push({ key, reason: icu.reason, value: translated, icu: true });
+        continue;
+      }
+      // Markup is code as well: every tag opened, closed and nested as in the
+      // source ("<strong>Book</strong>" → "<strong>Réserver" breaks the page).
+      const markup = checkMarkup(source, translated);
+      if (markup) {
+        failures.push({ key, reason: markup.reason, value: translated, markup: true });
+        continue;
+      }
+      // The other placeholders verify reports — i18next {{…}} (renamed, lost,
+      // or written as a single-brace {name}, which i18next prints as is), a
+      // single-brace {name} outside an ICU message, a tag token — by the one
+      // rule verify reports them by (lib/placeholders.js). The gate let
+      // {{name}} → {{nom}} through, verify flagged it after the sync, and the
+      // run exited 2; refused here, the pair's fallback gets the text instead
+      // (Round 12, i18next persona).
+      const tokens = placeholderChanges(source, translated);
+      if (tokens.length > 0) {
+        failures.push({
+          key,
+          reason: `placeholder structure damaged: ${tokens.map(t => `${PLACEHOLDER_SYNTAX_NAMES[t.syntax]} ${t.issue}`).join('; ')}`,
+          value: translated,
+          placeholder: true,
+        });
+        continue;
+      }
+      // A sentence break the translation put right beside a placeholder
+      // where the source has none: "Take this medicine at {time}." →
+      // "… sina. {time}." passed every check (Round 14, hospital persona).
+      // The same rule verify flags by (lib/placeholders.js); refused here,
+      // the retry hears why and the pair's fallback runs.
+      const breaks = placeholderSentenceBreaks(source, translated);
+      if (breaks.length > 0) {
+        failures.push({
+          key,
+          reason: `sentence break beside a placeholder: ${breaks.map(b => b.issue).join('; ')}`,
+          value: translated,
+          placeholder: true,
+        });
+        continue;
+      }
+    }
+
+    // Check 2c: the echo checks, per plural/select branch. A plural message
+    // differs from its source as a WHOLE as soon as the target adds a form
+    // (or one branch is translated), so the two checks above never saw a
+    // branch handed back as the English — accented, or as it was (Round 4,
+    // Django persona: the plural entry passed while the identical singular
+    // change was refused). Each branch is held to the singular rules: an
+    // exact copy of a short Latin-script branch is accepted in a Latin-script
+    // target (a name, "emails") and asked about once in a non-Latin one; any
+    // longer copy is refused; a disguised copy (3+ words) is refused. A
+    // branch made only of declared names (protectedTerms) is never an echo.
+    if (translated !== source && !prose) {
+      const branchEcho = pluralBranchEcho(source, translated, { isNonLatin, requireNonLatin: thresholds.requireNonLatin, acceptLatinNames, protectedTerms });
+      if (branchEcho) {
+        failures.push({ key, value: translated, ...branchEcho });
+        continue;
+      }
     }
 
     // Check 3: Repetition detection — catches hallucination loops.
@@ -387,6 +716,17 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
     //     or version strings like "3.2.0" have nothing to write in another script.
     //   - Short ASCII strings (≤30 chars, >80% ASCII) are likely proper nouns
     //     or brand names (e.g. "GitHub", "npm") that stay in English everywhere.
+    // Fullwidth Latin letters ("Ｂｏｏｋ ａｎ ａｐｐｏｉｎｔｍｅｎｔ") outside CJK typography:
+    // English in disguise, whatever the target's script. Never a name — a
+    // name kept as written is in plain letters.
+    if (hasForeignFullwidthLatin(translated, targetLocale)) {
+      failures.push({
+        key,
+        reason: `wrong script (fullwidth Latin letters in a ${targetLocale} value — English in disguise, not a translation)`,
+        value: translated.slice(0, 80),
+      });
+      continue;
+    }
     if (isNonLatin && thresholds.requireNonLatin) {
       // Strip ICU placeholders, digits, punctuation, whitespace → what's left?
       const translatableText = translated
@@ -394,12 +734,43 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
         .replace(/[\d\s\p{P}\p{S}]/gu, '')  // digits, whitespace, punctuation, symbols
         .trim();
       const asciiRatio = source.replace(/[^\x20-\x7E]/g, '').length / Math.max(source.length, 1);
-      const isShortAsciiPropNoun = source.length <= 30 && asciiRatio > 0.8;
-      if (translatableText.length > 0 && isAsciiOnly(translated) && !isShortAsciiPropNoun) {
+      const isShortAscii = source.length <= 30 && asciiRatio > 0.8;
+      // Letters classified by Unicode script: accented Latin is Latin too
+      // (an ASCII test passed "Thánk yóú" in a Russian catalog).
+      if (translatableText.length > 0 && isLatinOnly(translated, targetLocale)) {
+        if (isShortAscii && acceptLatinNames) {
+          // Second answer still Latin-script: the model says it is a name.
+        } else if (isShortAscii) {
+          failures.push({ key, reason: LATIN_NAME_OR_LABEL, value: translated.slice(0, 80), nameOrLabel: true });
+          continue;
+        } else {
+          failures.push({
+            key,
+            reason: `wrong script (ASCII-only for ${targetLocale}, expected non-Latin characters)`,
+            value: translated.slice(0, 80),
+          });
+          continue;
+        }
+      }
+    }
+
+    // Check 7 (last, so a soft failure means everything else passed): a
+    // plural message without a form the target language uses for ordinary
+    // counts. Soft — the caller asks once more; see the module header.
+    if (!acceptPluralGaps && !prose && translated !== source) {
+      // A gettext catalog's slots (pairConfig.pluralSlots, set by sync for a
+      // .po file): a form it has no msgstr[] for is never asked again for.
+      const gaps = pluralGaps(source, translated, targetLocale, options.pluralSlots ?? pairConfig.pluralSlots ?? null)
+        .filter(g => g.everyday.length > 0);
+      if (gaps.length > 0) {
+        const missing = [...new Set(gaps.flatMap(g => g.everyday))];
+        const type = gaps[0].type;
         failures.push({
           key,
-          reason: `wrong script (ASCII-only for ${targetLocale}, expected non-Latin characters)`,
-          value: translated.slice(0, 80),
+          reason: `plural form(s) ${missing.map(c => `"${c}"`).join(', ')} missing — ${targetLocale} uses `
+            + `${describeCategories(targetLocale, missing, type)}`,
+          value: translated,
+          pluralGap: { missing, type },
         });
         continue;
       }
@@ -412,6 +783,336 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
   return { validated, failures };
 }
 
+// ---------------------------------------------------------------------------
+// Different inputs, same output
+// ---------------------------------------------------------------------------
+
+/** Distinct source strings one output may answer before it is suspect. */
+const SHARED_OUTPUT_MIN_SOURCES = 3;
+/** Words (with letters) the shared output needs, on its own, to be suspect. */
+const SHARED_OUTPUT_MIN_WORDS = 4;
+
+/**
+ * Simple placeholders and markup tags in an output — not words of the
+ * translation (SharedOutputIndex.outputForm): {name} {0} {{count}} %(name)s
+ * %s %1$d <b> </a>. A plural/select argument ({n, plural, …}) has a comma and
+ * is not matched.
+ */
+const PLACEHOLDER_IN_OUTPUT = /\{\{\s*[\w.$-]+\s*\}\}|\{\s*[\w.$-]+\s*\}|%\([\w.]+\)[sdifr]|%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?[sdifuxXeEgGcp@]|<\/?[A-Za-z][^<>]*>/gu;
+/** The same without single-brace arguments: for a whole plural/select message. */
+const PLACEHOLDER_NOT_BRACED = /\{\{\s*[\w.$-]+\s*\}\}|%\([\w.]+\)[sdifr]|%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?[sdifuxXeEgGcp@]|<\/?[A-Za-z][^<>]*>/gu;
+/** A plural/select/selectordinal argument ("{n, plural, …"). */
+const ICU_COMPLEX_ARGUMENT = /\{\s*[\w.$-]+\s*,\s*(?:plural|select|selectordinal)\s*,/u;
+
+/**
+ * The sentences of a text, for the repeat check: split after . ! ? (and the
+ * CJK 。！？) followed by space, each kept only when it has a letter once
+ * its placeholders are gone ("S. {name}!" is one sentence).
+ *
+ * @param {string} text
+ * @returns {string[]}
+ */
+function sentencesOf(text) {
+  return String(text)
+    .split(/(?<=[.!?\u3002\uFF01\uFF1F])\s+/u)
+    .map(t => t.trim())
+    .filter(t => /\p{L}/u.test(t.replace(PLACEHOLDER_IN_OUTPUT, ' ')));
+}
+
+/** A source in the form two sources are compared in ("Save", "save!" meet). */
+function sourceIdentity(text) {
+  return foldForEchoCompare(String(text).replace(/[\p{P}\p{S}]+/gu, ' '));
+}
+
+/** The words of a folded string, as a set. */
+function wordSet(text) {
+  return new Set(sourceIdentity(text).split(' ').filter(w => /\p{L}/u.test(w)));
+}
+
+/** True when every pair of sources shares under half its words (Jaccard). */
+function clearlyDifferent(sources) {
+  const sets = sources.map(wordSet);
+  for (let i = 0; i < sets.length; i++) {
+    for (let j = i + 1; j < sets.length; j++) {
+      const a = sets[i];
+      const b = sets[j];
+      const inter = [...a].filter(w => b.has(w)).length;
+      const union = new Set([...a, ...b]).size;
+      if (union === 0 || inter / union >= 0.5) return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Different inputs, same output: ONE translation text returned for several
+ * DIFFERENT source strings is what a model that memorized a training sentence
+ * does (Round 4 school persona: a trained eng→crk model answered the app
+ * title, "Contact the school", the newsletter title and its heading with the
+ * same sentence — and every per-key check passed it).
+ *
+ * Suspect when one output answers SHARED_OUTPUT_MIN_SOURCES+ distinct sources
+ * (compared folded, punctuation aside — "Save" and "Save!" are one source)
+ * AND either the output has SHARED_OUTPUT_MIN_WORDS+ words, or every source
+ * has two or more words and no two of them share half their words. So
+ * synonyms collapsing to one short translation pass ('OK'/'Okay'/'Sure' →
+ * "D'accord"; 'Close'/'Dismiss'/'Cancel' → "Fermer"), and so does the same
+ * source text under several keys.
+ *
+ * TWO sources are already enough when the evidence is strong (Round 6,
+ * school persona: 'Thank you, {name}!' and 'Please bring the forms.' got one
+ * memorized sentence and passed, because only a third source counted): the
+ * output has SHARED_OUTPUT_MIN_WORDS+ words, both sources have two or more
+ * words, and they share under half their words (clearly different). Short
+ * outputs and synonym-like sources (sharing half their words or more) still
+ * need a third source. The cost of a false alarm is one more ask (an LLM is
+ * told to translate this string on its own; a method that takes no
+ * instructions is asked once more), then the pair's fallback.
+ *
+ * MEMORIZED outputs (markMemorized): an output an earlier sync found
+ * answering different source strings is suspect from its first source on —
+ * a repair that asks again and gets the same sentence back must not write it
+ * (Round 6, school persona: `--force-content` re-served it in silence).
+ *
+ * Same suppressions as the echo checks: a value made only of declared names
+ * (protectedTerms) is never suspect, nor a value equal to its own source
+ * (an echo — its own rule), nor one without letters. No-translate keys never
+ * reach a gate.
+ *
+ * The index spans a run's locale: the caller adds what it accepted, so a
+ * later batch (or a content file's block) that repeats an earlier answer is
+ * caught; only the NEW items are returned as suspect — what was written
+ * earlier is reported by `verify`. An output found suspect earlier in the
+ * run is suspect from its first new source on, like a MEMORIZED one: its
+ * refused members were never added, and the run remembers it only when it
+ * ends.
+ */
+class SharedOutputIndex {
+  constructor({ protectedTerms = [] } = {}) {
+    this.protectedTerms = protectedTerms;
+    // output form → Map(source identity → { key, source })
+    this.byOutput = new Map();
+    // output form → the first text seen in that form (what reports show)
+    this.display = new Map();
+    // output text → { value, sources, keys } — every group found suspect,
+    // with ALL its members (earlier, accepted ones too), for the run report.
+    this.flagged = new Map();
+    // output form → the text as first seen: outputs an earlier sync found
+    // answering different source strings (markMemorized).
+    this.memorized = new Map();
+  }
+
+  /**
+   * Outputs an earlier run found answering several different source strings:
+   * suspect from their first source on (see the class comment).
+   *
+   * @param {Iterable<string>} values
+   */
+  markMemorized(values) {
+    for (const value of values || []) {
+      if (typeof value !== 'string') continue;
+      const form = SharedOutputIndex.outputForm(value);
+      if (form && /\p{L}/u.test(form)) this.memorized.set(form, String(value).replace(/\s+/gu, ' ').trim());
+    }
+  }
+
+  /**
+   * The form outputs are grouped by: case, punctuation and symbols aside, and
+   * without a Markdown block marker — so "S?", "S." and a heading "# S" are
+   * one output (Round 5: an NMT model copies the source's end punctuation,
+   * and a newsletter H1 kept its "# "). Letters and their accents are kept:
+   * two words that differ by a diacritic are two words.
+   *
+   * Placeholders and markup tags are not words of the output: "S. {name}!"
+   * and "S." are one output (Round 9, school persona: the memorized sentence
+   * came back as "S. {name}!" for 'Thank you, {name}!' and as "S." inside a
+   * newsletter paragraph, and the "name" left over from "{name}" kept the two
+   * apart). Simple arguments only ({name}, {0}, {{count}}, %(name)s, %s,
+   * %1$d, <b>…</b>); an unsplit plural/select message keeps its branches.
+   */
+  static outputForm(value) {
+    const text = String(value);
+    // Inside a whole plural/select message, `{un}` is a branch, not an argument.
+    const placeholders = ICU_COMPLEX_ARGUMENT.test(text) ? PLACEHOLDER_NOT_BRACED : PLACEHOLDER_IN_OUTPUT;
+    return text
+      .normalize('NFC')
+      .replace(/^\s{0,3}(?:#{1,6}\s+|[-*+]\s+|\d+[.)]\s+|>\s*)/u, '')
+      .replace(placeholders, ' ')
+      .toLowerCase()
+      .replace(/[\p{P}\p{S}]+/gu, ' ')
+      .replace(/\s+/gu, ' ')
+      .trim();
+  }
+
+  /**
+   * The items, plus — for a value of several sentences whose source has as
+   * many — each sentence on its own, paired with its source sentence (the
+   * parent kept in `whole`). One memorized sentence inside a paragraph is
+   * then the same output as that sentence answering a UI string (Round 9,
+   * school persona: 'Please bring the forms.' in the newsletter and 'Thank
+   * you, {name}!' got one sentence, and the whole-paragraph comparison never
+   * met it). Pairing by position keeps a sentence two paragraphs genuinely
+   * share ("Please bring the forms." in both) as ONE source — not suspect;
+   * a value whose sentence count differs from its source's is compared whole
+   * only.
+   *
+   * @param {Array<{ key: string, source: string, value: string }>} items
+   * @returns {Array<{ key: string, source: string, value: string, whole?: { source: string, value: string } }>}
+   */
+  static withSentences(items) {
+    const out = [];
+    for (const it of items) {
+      out.push(it);
+      if (typeof it.source !== 'string' || typeof it.value !== 'string' || it.whole) continue;
+      const vs = sentencesOf(it.value);
+      if (vs.length < 2) continue;
+      const ss = sentencesOf(it.source);
+      if (ss.length !== vs.length) continue;
+      vs.forEach((v, i) => out.push({ key: it.key, source: ss[i], value: v, whole: { source: it.source, value: it.value } }));
+    }
+    return out;
+  }
+
+  eligible(source, value) {
+    if (typeof source !== 'string' || typeof value !== 'string') return false;
+    if (value === source || !/\p{L}/u.test(value) || !SharedOutputIndex.outputForm(value)) return false;
+    if (isProtectedTermValue(value, this.protectedTerms)) return false;
+    return true;
+  }
+
+  /**
+   * Items that would make (or join) a suspect group.
+   *
+   * @param {Array<{ key: string, source: string, value: string }>} items
+   * @returns {Map<string, { value: string, sources: string[] }>} key → its group
+   */
+  suspects(items) {
+    const groups = new Map();
+    for (const it of SharedOutputIndex.withSentences(items)) {
+      if (!this.eligible(it.source, it.value)) continue;
+      const form = SharedOutputIndex.outputForm(it.value);
+      if (!this.display.has(form)) this.display.set(form, String(it.value).replace(/\s+/gu, ' ').trim());
+      if (!groups.has(form)) {
+        groups.set(form, { sources: new Map(this.byOutput.get(form) || []), fresh: [] });
+      }
+      const g = groups.get(form);
+      const id = sourceIdentity(it.source);
+      if (!g.sources.has(id)) g.sources.set(id, { key: it.key, source: it.source });
+      g.fresh.push(it);
+    }
+    const out = new Map();
+    for (const [form, g] of groups) {
+      let sources = [...g.sources.values()].map(s => s.source);
+      const byRule = SharedOutputIndex.suspectGroup(form, sources);
+      // `memorized` marks a group suspect ONLY because an earlier sync
+      // remembered its output (reports word it that way); a group the rule
+      // catches on its own is reported as before.
+      const memorized = !byRule && this.memorized.has(form);
+      // An output this index already found answering different source
+      // strings EARLIER IN THIS RUN is suspect from its first new source on,
+      // as a remembered one is from an earlier sync. The members refused
+      // then were never added (only accepted items are), so without this the
+      // same sentence came back for the newsletter's title after the app's
+      // three keys were refused for it, and was written — the run remembers
+      // it only when it ends, and `verify` flagged it (Round 10, school
+      // persona). A gate retry that returns the refused sentence for one key
+      // is refused the same way.
+      const earlier = !byRule && !memorized ? this.flagged.get(form) : null;
+      if (earlier) sources = [...new Set([...earlier.sources, ...sources])];
+      if (!byRule && !memorized && !earlier) continue;
+      const shown = this.memorized.get(form) || this.display.get(form) || form;
+      for (const it of g.fresh) out.set(it.key, { value: shown, sources, ...(memorized && { memorized: true }) });
+      const prior = this.flagged.get(form) || { value: shown, sources: [], keys: [] };
+      this.flagged.set(form, {
+        value: shown,
+        sources: [...new Set([...prior.sources, ...sources])],
+        keys: [...new Set([...prior.keys, ...[...g.sources.values()].map(v => v.key), ...g.fresh.map(it => it.key)])],
+        ...(memorized && { memorized: true }),
+      });
+    }
+    return out;
+  }
+
+  /**
+   * Is one output form answering these distinct sources the pattern? Three or
+   * more: a long output, or clearly different multi-word sources. Two: only
+   * the strong case (see the class comment).
+   *
+   * @param {string} form - outputForm() of the shared output
+   * @param {string[]} sources - Distinct sources it answers
+   * @returns {boolean}
+   */
+  static suspectGroup(form, sources) {
+    const outWords = letterWordCount(form);
+    if (sources.length >= SHARED_OUTPUT_MIN_SOURCES) {
+      return outWords >= SHARED_OUTPUT_MIN_WORDS
+        || (sources.every(src => letterWordCount(src) >= 2) && clearlyDifferent(sources));
+    }
+    if (sources.length !== 2) return false;
+    return outWords >= SHARED_OUTPUT_MIN_WORDS
+      && sources.every(src => letterWordCount(src) >= 2)
+      && clearlyDifferent(sources);
+  }
+
+  /**
+   * Record accepted items (what was written) for later batches. A sentence
+   * of a longer value is recorded with its parent (`whole`): what was written,
+   * and cached, is the whole value.
+   */
+  add(items) {
+    for (const it of SharedOutputIndex.withSentences(items)) {
+      if (!this.eligible(it.source, it.value)) continue;
+      const form = SharedOutputIndex.outputForm(it.value);
+      if (!this.display.has(form)) this.display.set(form, String(it.value).replace(/\s+/gu, ' ').trim());
+      if (!this.byOutput.has(form)) this.byOutput.set(form, new Map());
+      const id = sourceIdentity(it.source);
+      if (!this.byOutput.get(form).has(id)) {
+        this.byOutput.get(form).set(id, { key: it.key, source: it.source, value: it.value, ...(it.whole && { whole: it.whole }) });
+      }
+    }
+  }
+}
+
+/**
+ * What one translated value puts in the shared-output index: the value
+ * itself, or — for an ICU plural/select message — each leaf branch on its
+ * own (Round 5, school persona: one memorized sentence filled BOTH branches
+ * of a plural message and two other keys, and nothing counted the branches).
+ *
+ * Every branch of one `plural`/`selectordinal` argument counts as ONE source
+ * (the message's): a language without number inflection writes the same
+ * text in every branch by design, so repeating it across branches is never
+ * evidence on its own. `select` branches count as their own source branches —
+ * different only when those differ.
+ *
+ * @param {string} key
+ * @param {string} source
+ * @param {string} value
+ * @returns {Array<{ key: string, source: string, value: string }>}
+ */
+function sharedOutputItems(key, source, value) {
+  if (typeof source !== 'string' || typeof value !== 'string') return [];
+  const pairs = value.includes('{') && source.includes('{') ? pluralBranchPairs(source, value) : [];
+  if (pairs.length === 0) return [{ key, source, value }];
+  return pairs.map(p => ({
+    key,
+    source: p.keyword === 'select' ? p.source : source,
+    value: p.translated,
+  }));
+}
+
+/** Gate reason for an output shared by several different sources. */
+function sharedOutputReason(group) {
+  if (group.memorized) {
+    const text = group.value.length > 60 ? `${group.value.slice(0, 57)}...` : group.value;
+    return `the sentence the model gave for other, different source strings on an earlier sync (${JSON.stringify(text)}) `
+      + '— a memorized sentence, not a translation of this string; asking the same model again returns it again';
+  }
+  const shown = group.sources.slice(0, 4).map(s => JSON.stringify(s.length > 40 ? `${s.slice(0, 37)}...` : s)).join(', ');
+  return `same output for ${group.sources.length} different source strings (${shown}${group.sources.length > 4 ? ', …' : ''}) `
+    + '— a model repeating one memorized sentence, not a translation of this string';
+}
+
 /**
  * Split a value into independently-measurable segments: pipe-delimited
  * plural variants ("one doc|{count} docs") legitimately share most of
@@ -421,6 +1122,10 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
  * @returns {string[]} Trimmed segments (always at least one)
  */
 function splitPluralSegments(text) {
+  // ICU plural/select branches too: a Russian plural repeats one sentence in
+  // four forms by design, and measured as one string it read as a loop.
+  const branches = text.includes('{') ? pluralBranchTexts(text) : null;
+  if (branches) return branches.map(seg => seg.trim());
   return (text.includes('|') ? text.split('|') : [text]).map(seg => seg.trim());
 }
 
@@ -477,6 +1182,138 @@ function isAsciiOnly(text) {
   return /^[\x00-\x7F]*$/.test(text);
 }
 
+/** Scripts whose text legitimately uses fullwidth Latin letters (CJK typography). */
+const FULLWIDTH_SCRIPTS = new Set(['Hani', 'Hans', 'Hant', 'Jpan', 'Kore', 'Hira', 'Kana', 'Hang', 'Bopo']);
+
+/** Fullwidth Latin letters: Ａ–Ｚ, ａ–ｚ. */
+const FULLWIDTH_LATIN = /[Ａ-Ｚａ-ｚ]/u;
+
+/**
+ * The text of a value with its code set aside: ICU/brace placeholders,
+ * printf conversions, markup tags. (A URL stays: in a non-Latin locale a
+ * URL value is a wrong-script answer unless the key is declared
+ * no-translate — which never reaches this check.)
+ */
+function proseOf(text) {
+  // A plural/select message: its prose is the text of its branches (the
+  // keywords and selectors — "plural", "one", "other" — are code).
+  const str = String(text);
+  const branches = str.includes('{') ? pluralBranchTexts(str) : null;
+  return (branches ? branches.join(' ') : str)
+    // Simple placeholders only: {name}, {count, number}. Branch text such as
+    // "{Один файл}" is prose, never stripped.
+    .replace(/\{\s*[\w.$-]+\s*(?:,[^{}]*)?\}/g, ' ')
+    .replace(/\{\{\s*[\w.$-]+\s*\}\}/g, ' ')
+    .replace(/%(\([^)]*\))?[-#0 +]*\d*(\.\d+)?[a-zA-Z@]/g, ' ')
+    .replace(/<\/?[A-Za-z0-9][^<>]*>/g, ' ');
+}
+
+/**
+ * Are ALL the letters of a value's prose Latin script — by Unicode script,
+ * not by byte? Fullwidth Latin ("Ｈｅｌｌｏ") and accented Latin ("Thánk")
+ * are Latin: an ASCII test let both through as "not English" in a Russian
+ * catalog (Round 4, Django persona). False when the prose has no letters.
+ *
+ * @param {string} text
+ * @returns {boolean}
+ */
+function isLatinOnly(text, locale = null) {
+  const prose = proseOf(text);
+  const letters = prose.match(/\p{L}/gu);
+  if (!letters) return false;
+  // CJK typography sets Latin in fullwidth forms ("ＯＫ" in Japanese): there,
+  // fullwidth letters are the target's own usage, as they always were here.
+  if (locale && FULLWIDTH_LATIN.test(prose) && usesFullwidthLatin(locale)) return false;
+  return letters.every(ch => /\p{Script=Latin}/u.test(ch));
+}
+
+/** Does this locale's script set Latin in fullwidth forms (CJK)? */
+function usesFullwidthLatin(locale) {
+  const card = getLanguageCard(String(locale)) || getLanguageCard(String(locale).split(/[-_]/)[0]);
+  return FULLWIDTH_SCRIPTS.has(card?.script);
+}
+
+/**
+ * Fullwidth Latin letters in a value whose language does not set Latin in
+ * fullwidth forms (anything but CJK typography): a disguised copy of English,
+ * never a translation.
+ *
+ * @param {string} text
+ * @param {string} locale - Target locale
+ * @returns {boolean}
+ */
+function hasForeignFullwidthLatin(text, locale) {
+  if (!FULLWIDTH_LATIN.test(proseOf(text))) return false;
+  return !usesFullwidthLatin(locale);
+}
+
+// ---------------------------------------------------------------------------
+// A question or exclamation that lost its mark
+// ---------------------------------------------------------------------------
+
+// Marks that end a question / an exclamation in some writing system: Latin
+// and most scripts (?), CJK fullwidth (？), Arabic script (؟), Greek (U+037E
+// and the ASCII semicolon Greek text is typed with), Armenian (՞), Ethiopic
+// (፧), and the interrobang family. Spanish closes with "?" / "!" as well.
+const QUESTION_ENDS = new Set(['?', '？', '؟', '\u037e', ';', '՞', '፧', '⸮', '⁇', '⁈', '‽', '⁉']);
+const EXCLAMATION_ENDS = new Set(['!', '！', '‼', '⁉', '⁈', '‽', '՜', '¡']);
+// Closing quotes, brackets, ICU braces and direction marks after the mark.
+const TRAILING_CLOSERS = /[\s"'»”’)\]}›〉》」』】〕\u200e\u200f]+$/u;
+
+/** The last visible character of a text, closers aside. */
+function terminalChar(text) {
+  const chars = Array.from(String(text).replace(TRAILING_CLOSERS, ''));
+  return chars.length > 0 ? chars[chars.length - 1] : '';
+}
+
+/**
+ * A source that ends with "?" or "!" whose translation ends with neither it
+ * nor an equivalent the target's script uses (Round 6, hospital persona:
+ * "Where does it hurt?" passed as a statement). A WARNING, never a refusal:
+ * some languages mark a question with a word or particle instead of a mark,
+ * and a translation that does so is right.
+ *
+ * @param {string} source
+ * @param {string} translated
+ * @returns {{ mark: '?'|'!' } | null}
+ */
+function droppedTerminalMark(source, translated) {
+  if (typeof source !== 'string' || typeof translated !== 'string' || translated === source) return null;
+  if (!/\p{L}/u.test(translated)) return null;
+  const end = terminalChar(source);
+  if (end !== '?' && end !== '!') return null;
+  const got = terminalChar(translated);
+  if (end === '?' ? QUESTION_ENDS.has(got) : EXCLAMATION_ENDS.has(got)) return null;
+  return { mark: end };
+}
+
+/**
+ * Keys whose translation dropped the source's closing "?" / "!" — one
+ * finding for sync and verify alike. Declared names and no-translate keys
+ * are skipped by the callers' own rules.
+ *
+ * @param {Array<[string, string, string]>} entries - [key, source, translated]
+ * @returns {Array<{ key: string, mark: string, value: string }>}
+ */
+function droppedTerminalMarks(entries) {
+  const out = [];
+  for (const [key, source, value] of entries) {
+    const d = droppedTerminalMark(source, value);
+    if (d) out.push({ key, mark: d.mark, value });
+  }
+  return out;
+}
+
+/** The warning text for droppedTerminalMarks findings (sync and verify say the same). */
+function describeDroppedMarks(found, fix) {
+  const marks = [...new Set(found.map(f => `"${f.mark}"`))].join(' or ');
+  const shown = found.slice(0, 3).map(f => `${f.key} (${JSON.stringify(f.value.length > 40 ? `${f.value.slice(0, 37)}...` : f.value)})`).join(', ');
+  return `${found.length} translation(s) dropped the source's closing ${marks}: ${shown}${found.length > 3 ? ', …' : ''} — `
+    + 'a question or exclamation may now read as a statement. Some languages mark a question with a word or '
+    + 'particle instead of a mark, so this is a warning, not a refusal: check them'
+    + (fix ? `, or ask again (--fresh: the cache holds this answer): \`${fix}\`` : '') + '.';
+}
+
 /**
  * Log quality gate failures in a structured, actionable format.
  *
@@ -485,21 +1322,31 @@ function isAsciiOnly(text) {
  */
 function logGateFailures(failures, pairKey) {
   if (failures.length === 0) return;
+  if (output.getMode() === 'json') {
+    // One structured record instead of indented prose on stderr.
+    output.warn(`${pairKey}: ${failures.length} key(s) failed quality validation`, {
+      pairKey, gateFailures: failures.map(({ key, reason, value }) => ({ key, reason, value })),
+    });
+    return;
+  }
 
-  console.error(`\n     [GATE] ${pairKey}: ${failures.length} key(s) failed quality validation:`);
+  const lines = ['', `     [GATE] ${pairKey}: ${failures.length} key(s) failed quality validation:`];
   for (const { key, reason, value } of failures) {
-    console.error(`            ✗ "${key}": ${reason}`);
-    if (value) {
-      console.error(`              → "${value}"`);
-    }
+    lines.push(`            ✗ "${key}": ${reason}`);
+    if (value) lines.push(`              → "${value}"`);
   }
-  console.error('');
+  lines.push('');
+  output.block(lines);
 }
 
 export {
+  checkICUStructure,
   validateTranslations,
+  contentGateFault,
   measureRepetition,
   isAsciiOnly,
+  isLatinOnly,
+  hasForeignFullwidthLatin,
   logGateFailures,
   checkContentPreservation,
   contentCharacters,
@@ -507,4 +1354,20 @@ export {
   NON_LATIN_LOCALES,
   DEFAULT_THRESHOLDS,
   MIN_MEASURABLE_CONTENT,
+  isProtectedTermValue,
+  LATIN_NAME_OR_LABEL,
+  foldForEchoCompare,
+  letterWordCount,
+  isDisguisedEcho,
+  pluralBranchEcho,
+  MIN_FOLDED_ECHO_WORDS,
+  DISGUISED_ECHO,
+  SharedOutputIndex,
+  sharedOutputItems,
+  sharedOutputReason,
+  SHARED_OUTPUT_MIN_SOURCES,
+  droppedTerminalMark,
+  droppedTerminalMarks,
+  describeDroppedMarks,
+  SHARED_OUTPUT_MIN_WORDS,
 };