champollion 0.5.1 → 0.5.3

package/lib/validate.js

@@ -451,8 +451,11 @@ function isProtectedTermValue(value, protectedTerms = []) {
  * until someone names it for a redo. Bump it whenever a check is loosened.
  *   2 — 2026-10-05: names with citations/anchors, reference entries, tables,
  *       short titles and source-shown fullwidth letters stopped being refused.
+ *   3 — 2026-10-06: inline code is not prose (an ICU sample in backticks no
+ *       longer makes a translated paragraph "ASCII-only"); a phrase said
+ *       twice is not a loop; "keep it" twice is accepted give or take a stop.
  */
-const GATE_VERSION = 2;
+const GATE_VERSION = 3;
 
 /**
  * Refusals that may be the model keeping text that is correct as written —
@@ -519,7 +522,7 @@ function isBibliographicEntry(text) {
     const lines = t.split('\n').map(l => l.trim()).filter(Boolean);
     return lines.length > 1 && lines.every(l => !l.includes('\n') && isBibliographicEntry(l));
   }
-  return /^(?:\[?[A-Z]?\d{1,3}\]?[.)]?|[-*+])\s+\S/.test(t)
+  return /^(?:\[?[A-Z]?\d{1,3}[a-z]?\]?[.)]?|[-*+])\s+\S/.test(t)
     && (/\(\d{4}[a-z]?\)/.test(t) || /\]\(https?:\/\//.test(t))
     && (/"[^"]{8,}"|“[^”]{8,}”/.test(t) || /\*[^*]{8,}\*/.test(t));
 }
@@ -570,6 +573,43 @@ function contentGateFault(source, value, pairConfig = {}) {
   return failures.length > 0 ? failures[0].reason : null;
 }
 
+/**
+ * Does `translated` show the shape of a degeneration loop, beyond a high
+ * repeated-n-gram rate? A phrase a language says twice where English
+ * elides it is not a loop: "From most to least trustworthy:" is correctly
+ * "Từ đáng tin cậy nhất đến ít đáng tin cậy nhất:" in Vietnamese, and was
+ * refused as a repetition hallucination (dogfood 2026-10-06). A loop does
+ * one of three things: repeats some span three times or more, repeats a whole
+ * line, or hands the source back verbatim beside a translation of it (the
+ * doubled Arabic headings, "### METEOR (…)\n### METEOR (…)").
+ *
+ * @param {string} translated
+ * @param {string} source
+ * @returns {boolean}
+ */
+function hasLoopEvidence(translated, source) {
+  const t = String(translated);
+  const grams = new Map();
+  for (let i = 0; i + REPETITION_LONG_N <= t.length; i++) {
+    const g = t.slice(i, i + REPETITION_LONG_N);
+    const n = (grams.get(g) || 0) + 1;
+    if (n >= 3 && g.trim().length >= 4) return true;
+    grams.set(g, n);
+  }
+  // A word said three times or more ("Qo' Qo' Qo' Qo'" — too short for an
+  // 8-character span to recur three times).
+  const words = new Map();
+  for (const w of t.toLowerCase().split(/\s+/u).filter(x => /\p{L}/u.test(x))) {
+    const n = (words.get(w) || 0) + 1;
+    if (n >= 3) return true;
+    words.set(w, n);
+  }
+  const lines = t.split('\n').map(l => l.trim()).filter(l => l.length >= 8);
+  if (new Set(lines).size < lines.length) return true;
+  const src = String(source).trim();
+  return src.length >= 8 && t.trim() !== src && t.includes(src);
+}
+
 /**
  * Validate a batch of translations and return only passing keys.
  *
@@ -775,7 +815,7 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
         longGramRate: measureRepetition(seg, REPETITION_LONG_N),
       }))
       .find(m => m.trigramRate > trigramCap && m.longGramRate > longGramCap);
-    if (degenerateSegment) {
+    if (degenerateSegment && hasLoopEvidence(translated, source)) {
       failures.push({
         key,
         reason: `repetition hallucination (${(degenerateSegment.trigramRate * 100).toFixed(0)}% repeated trigrams, ${(degenerateSegment.longGramRate * 100).toFixed(0)}% repeated ${REPETITION_LONG_N}-grams)`,
@@ -1300,12 +1340,30 @@ const FULLWIDTH_LATIN = /[Ａ-Ｚａ-ｚ]/u;
  * URL value is a wrong-script answer unless the key is declared
  * no-translate — which never reaches this check.)
  */
+/** `text` with every balanced {…} group removed (nested groups included). */
+function withoutBraceGroups(text) {
+  let out = '';
+  let depth = 0;
+  for (const ch of text) {
+    if (ch === '{') { depth++; continue; }
+    if (ch === '}' && depth > 0) { depth--; continue; }
+    if (depth === 0) out += ch;
+  }
+  return out;
+}
+
 function proseOf(text) {
+  // Inline code is never prose: a Markdown paragraph quoting an ICU message
+  // in backticks (`{n, plural, one {One file} other {…}}`) was read as that
+  // message, its English branches were taken for the whole value, and 470
+  // characters of correct Japanese were refused as "ASCII-only" — twice, in
+  // five languages (dogfood 2026-10-06, docs/getting-started/configuration.md).
+  const str = String(text).replace(/(`+)[^`]*?\1/g, ' ');
   // A plural/select message: its prose is the text of its branches (the
-  // keywords and selectors — "plural", "one", "other" — are code).
-  const str = String(text);
+  // keywords and selectors — "plural", "one", "other" — are code) PLUS the
+  // text around it ("You have {count, plural, …}" — "You have" is prose).
   const branches = str.includes('{') ? pluralBranchTexts(str) : null;
-  return (branches ? branches.join(' ') : str)
+  return (branches ? `${branches.join(' ')} ${withoutBraceGroups(str)}` : str)
     // Simple placeholders only: {name}, {count, number}. Branch text such as
     // "{Один файл}" is prose, never stripped.
     .replace(/\{\s*[\w.$-]+\s*(?:,[^{}]*)?\}/g, ' ')
@@ -1467,6 +1525,7 @@ export {
   nameCore,
   GATE_VERSION,
   isKeepAsWrittenFault,
+  hasLoopEvidence,
   isBibliographicEntry,
   tableProse,
   SHORT_SOURCE_LENGTH_SLACK,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "champollion",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "Research-grade translation engine for i18n projects. Pluggable methods, per-pair quality tiers, and deterministic script converters. Supports JSON (next-intl, i18next), TOML, and YAML (Hugo).",
   "type": "module",
   "bin": {