champollion 0.5.1 → 0.5.2

package/lib/content-refusals.js

@@ -51,6 +51,9 @@
  *   paragraph written by hand — drops its record.
  */
 
+import fs from 'node:fs';
+import path from 'node:path';
+import { output } from './output.js';
 import { tmMethodKey, lookupTM } from './tm.js';
 import { GATE_VERSION } from './validate.js';
 import { holdState, shortSourceHash } from './locale-state.js';
@@ -282,3 +285,54 @@ export function previewHeld({ tm, code, pairConfig, fields, body, segMode, holds
   }
   return { names, pageHeld: false };
 }
+
+// ── What a run left in the source language ───────────────────────────
+
+/** Where refused answers are logged (the per-machine cache folder, never committed). */
+export const REFUSAL_LOG = path.join('.champollion', 'refused.jsonl');
+
+/**
+ * Append one refused answer to `.champollion/refused.jsonl`: the source, what
+ * the model answered, and why the gate refused it. WHY: a refused answer was
+ * thrown away, so "why did it refuse that twice?" could only be answered by
+ * paying the model again (2026-10-06 — and the answer then showed the gate,
+ * not the model, was wrong). Never fails the run.
+ *
+ * @param {string} cwd
+ * @param {object} rec - { pair, method, file, locale, paragraph, reason, source, answer }
+ */
+export function logRefusal(cwd, rec) {
+  try {
+    const file = path.join(cwd, REFUSAL_LOG);
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    fs.appendFileSync(file, `${JSON.stringify({ at: new Date().toISOString(), ...rec })}\n`, 'utf-8');
+  } catch { /* a diagnostic: never fails the run */ }
+}
+
+/**
+ * The run's closing report when anything was left in the source language —
+ * last, as an error, never under an [OK] (2026-10-06: a 1,082-page sync ended
+ * on "[OK] Created 1082 content file(s)" with seven untranslated blocks named
+ * only in warnings mid-stream).
+ *
+ * @param {Array<{ file: string, locale: string, pair: string, where: string, reason: string }>} items
+ */
+export function reportLeftInSource(items) {
+  if (!items || items.length === 0) return;
+  const pages = new Map();
+  for (const it of items) {
+    const k = `${it.file}\u0000${it.pair}`;
+    if (!pages.has(k)) pages.set(k, { ...it, wheres: [] });
+    pages.get(k).wheres.push(`${it.where}: ${it.reason}`);
+  }
+  output.raw('');
+  output.error(`Not finished: ${items.length} part(s) of ${pages.size} page translation(s) are still in the source language — `
+    + 'the quality gate refused the model\'s answer, also when asked again with the reason:');
+  for (const p of pages.values()) {
+    output.raw(`    ${p.file} → ${p.locale}: ${p.wheres.slice(0, 3).join('; ')}${p.wheres.length > 3 ? `; +${p.wheres.length - 3} more` : ''}`);
+  }
+  const first = [...pages.values()][0];
+  output.raw(`  The rest of each page is written. What the model answered is in ${REFUSAL_LOG}. `
+    + `Ask again: \`${contentRedoCommand(first.file, { pair: first.pair })}\`${pages.size > 1 ? ' (one per page)' : ''}, `
+    + 'or add a "fallback" method to the pair.');
+}

package/lib/content-sync.js

@@ -89,7 +89,7 @@ import {
 } from './fallback.js';
 import {
   readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
-  describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock,
+  describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock, logRefusal, reportLeftInSource,
 } from './content-refusals.js';
 import { contentRedoCommand } from './verify.js';
 
@@ -115,6 +115,8 @@ import { contentRedoCommand } from './verify.js';
  *   (null = no cap).
  */
 async function runContentSync(options) {
+  // Parts this run left in the source language — reported at the end, as an error.
+  const leftInSource = [];
   const {
     contentDir,
     sourceLocale,
@@ -700,6 +702,11 @@ async function runContentSync(options) {
               // field keeps its source text; the page is still written.
               fieldsLeftInSource = true;
               for (const h of fm.hollowed) {
+                leftInSource.push({ file: relPath, locale: code, pair: pairKey, where: `front matter "${h.field}"`, reason: h.reason });
+                logRefusal(cwd, {
+                  pair: pairKey, method: tmMethodKey(pairConfig), file: relPath, locale: code,
+                  field: h.field, reason: h.reason, source: fieldsToTranslate[h.field], answer: h.value ?? null,
+                });
                 const advice = h.sharedOutput
                   ? ` ${pairConfig.method} answers it only with a sentence it gave for other strings — `
                     + `${pairConfig.fallback ? 'and the fallback did not translate it either' : 'add a "fallback" method to the pair'}, or write it in ${targetRel} by hand (kept).`
@@ -990,6 +997,13 @@ async function runContentSync(options) {
                 // Blocks the quality gate refused — the key-value gate's checks
                 // (length inflation, echo, truncation, script), per block.
                 const refusedByGate = batchOutcome.refused || [];
+                for (const r of refusedByGate) {
+                  leftInSource.push({ file: relPath, locale: code, pair: pairKey, where: `paragraph ${r.block}`, reason: r.reason });
+                  logRefusal(cwd, {
+                    pair: pairKey, method: tmMethodKey(pairConfig), file: relPath, locale: code,
+                    paragraph: r.block, reason: r.reason, source: missed[r.i].source, answer: r.answer,
+                  });
+                }
                 if (refusedByGate.length > 0) {
                   output.warn(
                     `Content sync body for ${code}: ${refusedByGate.length} block(s) of ${relPath} refused by the quality gate, also when asked again with the reason — `
@@ -1210,9 +1224,11 @@ async function runContentSync(options) {
       const heldBackNote = heldBackSeen > 0 ? `; ${heldBackSeen} block(s)/field(s) refused before would be held back (not sent)` : '';
       output.info(`Would have created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}${heldBackNote}.`);
     } else {
-      output.ok(`Created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}.`);
+      const unfinished = leftInSource.length > 0 ? ` — ${leftInSource.length} part(s) left in the source language (below)` : '';
+      (unfinished ? output.warn : output.ok).call(output, `Created ${totalCreated} content file(s)${retranslateNote}${keptNote}, ${skipped} unchanged${heldNote}${unfinished}.`);
     }
   }
+  reportLeftInSource(leftInSource);
   if (heldBack > 0) {
     // Not sent, not billed — but not translated either (exit 2, as for a
     // held-back key). Each page was named above with its own repair.

package/lib/docusaurus-sync.js

@@ -74,7 +74,7 @@ import {
 import { walkFiles } from './locale-layout.js';
 import {
   readRefusals, contentHolds, nextRefusals, storeRefusals, blockUnit, fieldUnit, describeHeld, describeNewHold,
-  describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock,
+  describeFallenBack, previewHeld, PAGE_UNIT, PAGE_NAME, pendingLockValue, isPendingLock, logRefusal, reportLeftInSource,
 } from './content-refusals.js';
 import { applyNamedKeyRule, reportUnmatchedKeys, unmatchedKeysSummary, reportNamedFromCache } from './named-keys.js';
 
@@ -1209,6 +1209,8 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
   let contentHeldBack = 0;
   const contentHeldBackItems = [];
   let contentRefused = 0;
+  // Parts this run left in the source language — said last, as an error.
+  const leftInSource = [];
 
   if (contentSources.length === 0) {
     output.info('No docs/ or blog/ directories found — skipping content sync.');
@@ -1443,6 +1445,11 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
                 // field keeps its source text; the page is still written.
                 fieldsLeftInSource = true;
                 for (const h of fm.hollowed) {
+                  leftInSource.push({ file: label, locale: code, pair: pairKey, where: `front matter "${h.field}"`, reason: h.reason });
+                  logRefusal(cwd, {
+                    pair: pairKey, method: tmMethodKey(pairConfig), file: label, locale: code,
+                    field: h.field, reason: h.reason, source: fieldsToTranslate[h.field], answer: h.value ?? null,
+                  });
                   output.warn(
                     `${label} → ${code}: front matter "${h.field}" kept in the source language — ${h.reason}`
                     + `${h.fallbackReason ? `; the fallback (${pairConfig.fallback.method}): ${h.fallbackReason}` : ''}.`
@@ -1666,6 +1673,13 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
                       'the model gave for other, different source strings (a memorized sentence, not a translation) — refused.'
                     );
                   }
+                  for (const r of outcome.refused || []) {
+                    leftInSource.push({ file: `${dirName}/${relPath}`, locale: code, pair: pairKey, where: `paragraph ${r.block}`, reason: r.reason });
+                    logRefusal(cwd, {
+                      pair: pairKey, method: tmMethodKey(pairConfig), file: `${dirName}/${relPath}`, locale: code,
+                      paragraph: r.block, reason: r.reason, source: missed[r.i].source, answer: r.answer,
+                    });
+                  }
                   if ((outcome.refused || []).length > 0) {
                     output.warn(
                       `Docusaurus body for ${code}: ${outcome.refused.length} block(s) of ${dirName}/${relPath} refused by the quality gate, also when asked again with the reason — `
@@ -1845,7 +1859,8 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
     if (totalContent > 0 || totalContentSkipped > 0 || totalContentRetranslated > 0) {
       const action = dryRun ? 'Would create' : 'Created';
       const retranslateNote = totalContentRetranslated > 0 ? ` (${totalContentRetranslated} re-translated)` : '';
-      output.ok(`${action} ${totalContent} content file(s)${retranslateNote}, ${totalContentSkipped} unchanged`);
+      const unfinished = leftInSource.length > 0 ? ` — ${leftInSource.length} part(s) left in the source language (below)` : '';
+      (unfinished ? output.warn : output.ok).call(output, `${action} ${totalContent} content file(s)${retranslateNote}, ${totalContentSkipped} unchanged${unfinished}`);
     }
 
     contentTranslated = totalContent;
@@ -1891,6 +1906,10 @@ async function runDocusaurusSync(options, config, cwd, resolveRuntime) {
     saveTM(cwd, tm);
     output.info(`[TM] Saved ${describeTMChanges(tm)} this sync`);
   }
+  {
+    // Last, so it is the run's closing word: what is still untranslated.
+    reportLeftInSource(leftInSource);
+  }
 
   // Failures are reported, not thrown: a partly failed run still did real
   // work, and the exit code says so (2 = partial, 1 = nothing succeeded).

package/lib/fallback.js

@@ -572,7 +572,8 @@ export async function translateFieldsWithFallback({
         if (typeof value !== 'string') continue;
         const f = failed.get(field);
         const fault = contentGateFault(fields[field], value, pairConfig);
-        const insisted = isKeepAsWrittenFault(fault) && isKeepAsWrittenFault(f.reason) && value === f.value;
+        const insisted = isKeepAsWrittenFault(fault) && isKeepAsWrittenFault(f.reason)
+          && [f.value, fields[field]].some(v => sameAnswer(v, value));
         if (!fault || insisted) againPassing.push([field, value]);
         else f.reason = `${f.reason}; asked again with that reason: ${fault}`;
       }
@@ -698,6 +699,17 @@ export function blockFault(protectedSource, protectedOut, restoredOut, restoredS
   return contentGateFault(restoredSource, restoredOut, pairConfig);
 }
 
+/**
+ * The same answer, give or take spacing and closing punctuation: a citation
+ * kept as written once with its final "." and once without was refused as a
+ * changed answer (dogfood 2026-10-06, a Thai reference) — the model had said
+ * "keep it" twice.
+ */
+function sameAnswer(a, b) {
+  const norm = (v) => String(v ?? '').replace(/\s+/g, ' ').trim().replace(/[.,;:!?。、]+$/u, '');
+  return norm(a) === norm(b);
+}
+
 /**
  * Ask the pair's method once more for the blocks the gate refused, telling it
  * why. WHY: a refusal used to be final for the run, and the block was left in
@@ -724,7 +736,7 @@ export function blockFault(protectedSource, protectedOut, restoredOut, restoredS
  * @param {object|null} [p.budget]
  * @returns {Promise<Map<number,string>>} index → accepted (restored) answer
  */
-async function retryRefusedBlocks({ idx, texts, missed, blocks, pairConfig, runBatch, firstOut, reasons, budget = null }) {
+async function retryRefusedBlocks({ idx, texts, missed, blocks, pairConfig, runBatch, firstOut, reasons, answers = null, budget = null }) {
   const accepted = new Map();
   if (idx.length === 0) return accepted;
   // An endpoint that declares it follows no instructions (a trained NMT
@@ -751,9 +763,13 @@ async function retryRefusedBlocks({ idx, texts, missed, blocks, pairConfig, runB
     if (!res || fell.has(j) || typeof res.blocks[j] !== 'string') return;
     const restored = restoreBlocks(res.blocks[j], blocks);
     const fault = blockFault(texts[i], res.blocks[j], restored, missed[i].source, pairConfig);
-    const insisted = isKeepAsWrittenFault(fault) && isKeepAsWrittenFault(reasons.get(i)) && restored === firstOut.get(i);
+    const insisted = isKeepAsWrittenFault(fault) && isKeepAsWrittenFault(reasons.get(i))
+      && [firstOut.get(i), missed[i].source].some(v => sameAnswer(v, restored));
     if (!fault || insisted) accepted.set(i, restored);
-    else reasons.set(i, `${reasons.get(i)}; asked again with that reason: ${fault}`);
+    else {
+      reasons.set(i, `${reasons.get(i)}; asked again with that reason: ${fault}`);
+      if (answers) answers.set(i, restored);
+    }
   });
   return accepted;
 }
@@ -828,9 +844,13 @@ export async function translateBlocksWithFallback({
   };
   // Why the gate refused a block (index → reason), for the lane's warning.
   const refusedReason = new Map();
+  // The last answer the gate refused, per block — kept for the refusal log
+  // (.champollion/refused.jsonl), so "why was this refused?" can be read off
+  // what the model actually said instead of asked again.
+  const refusedAnswer = new Map();
   /** Blocks left as their source text because the gate refused them. */
   const refusedList = (fellBack) => fellBack.filter(i => refusedReason.has(i))
-    .map(i => ({ i, block: (Number.isInteger(missed[i].pos) ? missed[i].pos : i) + 1, reason: refusedReason.get(i) }));
+    .map(i => ({ i, block: (Number.isInteger(missed[i].pos) ? missed[i].pos : i) + 1, reason: refusedReason.get(i), answer: refusedAnswer.get(i) ?? null }));
 
   if (!fb) {
     const { blocks: translatedBlocks, fellBack } = await runBatch(texts, pairConfig);
@@ -841,14 +861,14 @@ export async function translateBlocksWithFallback({
     for (let i = 0; i < outs.length; i++) {
       if (fellSet.has(i)) continue;
       const fault = contentGateFault(missed[i].source, outs[i], pairConfig);
-      if (fault) refusedReason.set(i, fault);
+      if (fault) { refusedReason.set(i, fault); refusedAnswer.set(i, outs[i]); }
     }
     const shared = sharedFault(outs.map((out, i) => ({ i, out })).filter(({ i }) => !fellSet.has(i) && !refusedReason.has(i)));
     for (const i of shared) refusedReason.set(i, 'the same text the model gave for other, different source strings');
     // Refused: asked once more, with the reason.
     const again = await retryRefusedBlocks({
       idx: [...refusedReason.keys()], texts, missed, blocks, pairConfig, runBatch,
-      firstOut: new Map([...refusedReason.keys()].map(i => [i, outs[i]])), reasons: refusedReason, budget,
+      firstOut: new Map([...refusedReason.keys()].map(i => [i, outs[i]])), reasons: refusedReason, answers: refusedAnswer, budget,
     });
     const againShared = sharedFault([...again].map(([i, out]) => ({ i, out })));
     for (const [i, out] of again) {
@@ -914,6 +934,7 @@ export async function translateBlocksWithFallback({
       const fault = blockFault(texts[i], primary.blocks[i], restored, missed[i].source, pairConfig);
       if (fault) {
         refusedReason.set(i, fault);
+        refusedAnswer.set(i, restored);
         primaryRefused.add(i);
         failed.push(i);
         continue;
@@ -934,7 +955,7 @@ export async function translateBlocksWithFallback({
       if (refusedReason.has(i)) firstOut.set(i, restoreBlocks(primary.blocks[i], blocks));
     }
     const again = await retryRefusedBlocks({
-      idx: [...firstOut.keys()], texts, missed, blocks, pairConfig, runBatch, firstOut, reasons: refusedReason, budget,
+      idx: [...firstOut.keys()], texts, missed, blocks, pairConfig, runBatch, firstOut, reasons: refusedReason, answers: refusedAnswer, budget,
     });
     const againShared = sharedFault([...again].map(([i, out]) => ({ i, out })));
     const recovered = new Set();
@@ -1015,6 +1036,7 @@ export async function translateBlocksWithFallback({
       if (!fbFault) fbGood.push({ i, out: restored });
       else {
         refusedReason.set(i, refusedReason.has(i) ? `${refusedReason.get(i)}; the fallback's answer: ${fbFault}` : `the fallback's answer: ${fbFault}`);
+        refusedAnswer.set(i, restored);
         fallbackRefused.add(i);
       }
     }

package/lib/validate.js

@@ -451,8 +451,11 @@ function isProtectedTermValue(value, protectedTerms = []) {
  * until someone names it for a redo. Bump it whenever a check is loosened.
  *   2 — 2026-10-05: names with citations/anchors, reference entries, tables,
  *       short titles and source-shown fullwidth letters stopped being refused.
+ *   3 — 2026-10-06: inline code is not prose (an ICU sample in backticks no
+ *       longer makes a translated paragraph "ASCII-only"); a phrase said
+ *       twice is not a loop; "keep it" twice is accepted give or take a stop.
  */
-const GATE_VERSION = 2;
+const GATE_VERSION = 3;
 
 /**
  * Refusals that may be the model keeping text that is correct as written —
@@ -519,7 +522,7 @@ function isBibliographicEntry(text) {
     const lines = t.split('\n').map(l => l.trim()).filter(Boolean);
     return lines.length > 1 && lines.every(l => !l.includes('\n') && isBibliographicEntry(l));
   }
-  return /^(?:\[?[A-Z]?\d{1,3}\]?[.)]?|[-*+])\s+\S/.test(t)
+  return /^(?:\[?[A-Z]?\d{1,3}[a-z]?\]?[.)]?|[-*+])\s+\S/.test(t)
     && (/\(\d{4}[a-z]?\)/.test(t) || /\]\(https?:\/\//.test(t))
     && (/"[^"]{8,}"|“[^”]{8,}”/.test(t) || /\*[^*]{8,}\*/.test(t));
 }
@@ -570,6 +573,43 @@ function contentGateFault(source, value, pairConfig = {}) {
   return failures.length > 0 ? failures[0].reason : null;
 }
 
+/**
+ * Does `translated` show the shape of a degeneration loop, beyond a high
+ * repeated-n-gram rate? A phrase a language says twice where English
+ * elides it is not a loop: "From most to least trustworthy:" is correctly
+ * "Từ đáng tin cậy nhất đến ít đáng tin cậy nhất:" in Vietnamese, and was
+ * refused as a repetition hallucination (dogfood 2026-10-06). A loop does
+ * one of three things: repeats some span three times or more, repeats a whole
+ * line, or hands the source back verbatim beside a translation of it (the
+ * doubled Arabic headings, "### METEOR (…)\n### METEOR (…)").
+ *
+ * @param {string} translated
+ * @param {string} source
+ * @returns {boolean}
+ */
+function hasLoopEvidence(translated, source) {
+  const t = String(translated);
+  const grams = new Map();
+  for (let i = 0; i + REPETITION_LONG_N <= t.length; i++) {
+    const g = t.slice(i, i + REPETITION_LONG_N);
+    const n = (grams.get(g) || 0) + 1;
+    if (n >= 3 && g.trim().length >= 4) return true;
+    grams.set(g, n);
+  }
+  // A word said three times or more ("Qo' Qo' Qo' Qo'" — too short for an
+  // 8-character span to recur three times).
+  const words = new Map();
+  for (const w of t.toLowerCase().split(/\s+/u).filter(x => /\p{L}/u.test(x))) {
+    const n = (words.get(w) || 0) + 1;
+    if (n >= 3) return true;
+    words.set(w, n);
+  }
+  const lines = t.split('\n').map(l => l.trim()).filter(l => l.length >= 8);
+  if (new Set(lines).size < lines.length) return true;
+  const src = String(source).trim();
+  return src.length >= 8 && t.trim() !== src && t.includes(src);
+}
+
 /**
  * Validate a batch of translations and return only passing keys.
  *
@@ -775,7 +815,7 @@ function validateTranslations(translations, sourceFlat, pairConfig, options = {}
         longGramRate: measureRepetition(seg, REPETITION_LONG_N),
       }))
       .find(m => m.trigramRate > trigramCap && m.longGramRate > longGramCap);
-    if (degenerateSegment) {
+    if (degenerateSegment && hasLoopEvidence(translated, source)) {
       failures.push({
         key,
         reason: `repetition hallucination (${(degenerateSegment.trigramRate * 100).toFixed(0)}% repeated trigrams, ${(degenerateSegment.longGramRate * 100).toFixed(0)}% repeated ${REPETITION_LONG_N}-grams)`,
@@ -1300,12 +1340,30 @@ const FULLWIDTH_LATIN = /[Ａ-Ｚａ-ｚ]/u;
  * URL value is a wrong-script answer unless the key is declared
  * no-translate — which never reaches this check.)
  */
+/** `text` with every balanced {…} group removed (nested groups included). */
+function withoutBraceGroups(text) {
+  let out = '';
+  let depth = 0;
+  for (const ch of text) {
+    if (ch === '{') { depth++; continue; }
+    if (ch === '}' && depth > 0) { depth--; continue; }
+    if (depth === 0) out += ch;
+  }
+  return out;
+}
+
 function proseOf(text) {
+  // Inline code is never prose: a Markdown paragraph quoting an ICU message
+  // in backticks (`{n, plural, one {One file} other {…}}`) was read as that
+  // message, its English branches were taken for the whole value, and 470
+  // characters of correct Japanese were refused as "ASCII-only" — twice, in
+  // five languages (dogfood 2026-10-06, docs/getting-started/configuration.md).
+  const str = String(text).replace(/(`+)[^`]*?\1/g, ' ');
   // A plural/select message: its prose is the text of its branches (the
-  // keywords and selectors — "plural", "one", "other" — are code).
-  const str = String(text);
+  // keywords and selectors — "plural", "one", "other" — are code) PLUS the
+  // text around it ("You have {count, plural, …}" — "You have" is prose).
   const branches = str.includes('{') ? pluralBranchTexts(str) : null;
-  return (branches ? branches.join(' ') : str)
+  return (branches ? `${branches.join(' ')} ${withoutBraceGroups(str)}` : str)
     // Simple placeholders only: {name}, {count, number}. Branch text such as
     // "{Один файл}" is prose, never stripped.
     .replace(/\{\s*[\w.$-]+\s*(?:,[^{}]*)?\}/g, ' ')
@@ -1467,6 +1525,7 @@ export {
   nameCore,
   GATE_VERSION,
   isKeepAsWrittenFault,
+  hasLoopEvidence,
   isBibliographicEntry,
   tableProse,
   SHORT_SOURCE_LENGTH_SLACK,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "champollion",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "Research-grade translation engine for i18n projects. Pluggable methods, per-pair quality tiers, and deterministic script converters. Supports JSON (next-intl, i18next), TOML, and YAML (Hugo).",
   "type": "module",
   "bin": {