champollion 0.3.3 → 0.4.0

package/lib/content-estimate.js

@@ -0,0 +1,96 @@
+/**
+ * content-estimate.js — what a content file will actually BILL.
+ *
+ * The cost gate (--max-cost) must price what the run will pay for, not the
+ * whole file: front-matter fields and body blocks the Translation Memory
+ * already holds are served at $0 by the sync. Pricing whole files made a
+ * $100 cap refuse runs whose real cost was ~$30, and operators raised the
+ * cap past the honest number to get work done (dogfood 2026-08-28, finding
+ * 1) — the guardrail stopped guarding.
+ *
+ * This walks the SAME ladder both content lanes run at sync time:
+ *   1. each front-matter field, cached on its own source text;
+ *   2. the whole body (a revert or lock-loss re-run is free);
+ *   3. in 'block' segmentation, each translatable block.
+ * One implementation for the Hugo and Docusaurus estimators, so the two can
+ * no longer disagree about what is billable.
+ *
+ * Pure lookups (lookupTM, never the evicting lookupTMValidated): an
+ * estimate must not change the cache it is estimating against.
+ */
+
+import { lookupTM } from './tm.js';
+import { splitBlocks } from './segment.js';
+import { protectBlocks, restoreBlocks } from './content.js';
+
+/**
+ * Restored source texts of a body's translatable blocks — the exact unit
+ * block-mode sync caches and bills.
+ *
+ * @param {string} body - Markdown body (front matter stripped)
+ * @returns {string[]}
+ */
+export function translatableBlockSources(body) {
+  const { protectedBody, blocks } = protectBlocks(body);
+  return splitBlocks(protectedBody)
+    .filter(seg => seg.type === 'translatable')
+    .map(seg => restoreBlocks(seg.text, blocks));
+}
+
+/**
+ * Billable vs total source characters for one (file × locale) work item.
+ *
+ * @param {object} args
+ * @param {object} args.tm - Loaded TM (carry-over settings apply)
+ * @param {string} args.code - Target locale
+ * @param {string} args.tmKey - tmMethodKey(pairConfig)
+ * @param {string|null} [args.fallbackTmKey] - tmMethodKey(pairConfig.fallback)
+ *   when the pair has a fallback: the sync serves the fallback's cached text
+ *   before asking the primary (lib/fallback.js), so it is free here too
+ * @param {Record<string, string>} args.fields - Front-matter fields to translate
+ * @param {string} args.body - Markdown body
+ * @param {'block'|'page'} args.segMode - Content segmentation mode
+ * @param {() => string[]} args.blockSources - Lazily computed block sources
+ *   (callers cache per source file: segmentation is locale-independent)
+ * @param {ReturnType<import('./content-refusals.js').contentHolds>|null} [args.holds] -
+ *   What the quality gate refused before (lib/content-refusals.js): a block
+ *   or field the pair's method refused is not sent to it (held, or the
+ *   fallback's — which this estimate does not price); a held front-matter
+ *   field — or a held page body in 'page' segmentation — holds its whole
+ *   page, so nothing of it is billed
+ * @returns {{ billedChars: number, totalChars: number }}
+ */
+export function billableContentChars({ tm, code, tmKey, fallbackTmKey = null, fields, body, segMode, blockSources, holds = null }) {
+  let billedChars = 0;
+  let totalChars = 0;
+  let pageHeld = false;
+  const cached = (text) => lookupTM(tm, text, code, tmKey) !== null
+    || (fallbackTmKey !== null && lookupTM(tm, text, code, fallbackTmKey) !== null);
+
+  for (const [field, text] of Object.entries(fields)) {
+    totalChars += text.length;
+    if (cached(text)) continue;
+    const hold = holds ? holds.field(field, text) : 'send';
+    if (hold === 'held') pageHeld = true;
+    else if (hold === 'send') billedChars += text.length;
+  }
+
+  if (body.trim()) {
+    totalChars += body.length;
+    if (!cached(body)) {
+      if (segMode === 'page') {
+        // A page translated whole and refused before: held (its page is
+        // not processed — nothing of it is billed), or the fallback's.
+        const hold = holds ? holds.page(body) : 'send';
+        if (hold === 'held') pageHeld = true;
+        else if (hold === 'send') billedChars += body.length;
+      } else {
+        for (const source of blockSources()) {
+          if (!cached(source) && (!holds || holds.block(source) === 'send')) billedChars += source.length;
+        }
+      }
+    }
+  }
+
+  return { billedChars: pageHeld ? 0 : billedChars, totalChars };
+}

package/lib/content-refusals.js

@@ -0,0 +1,270 @@
+/**
+ * content-refusals.js — Markdown blocks and front-matter fields the quality
+ * gate refused, remembered so a plain sync does not send them to the same
+ * model again. The key-value rule (lib/locale-state.js, Round 4) for the two
+ * content lanes (lib/content-sync.js, lib/docusaurus-sync.js).
+ *
+ * WHY: a block the gate refused (lib/validate.js contentGateFault) was
+ * written as the '[EN] ' last resort, never cached, and its file's lock was
+ * not advanced — so EVERY sync sent it to the same paid model again, which
+ * answered the same way and was refused again. A refused front-matter field
+ * failed its page, and every sync re-sent the page's fields. With a paid
+ * model and no fallback that repeated the charge on each run, unlike keys,
+ * which are held back.
+ *
+ * THE RECORD — in .champollion-content.lock (committed, beside the lane's
+ * own entries), one entry per (page × locale) that has refusals:
+ *
+ *   "refused:<manifestKey>": {
+ *     "block:<hash12>": { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" },
+ *     "field:title":    { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" },
+ *     "page":           { "source": "<hash12>", "methods": ["<method key>"], "on": "2026-10-04" }
+ *   }
+ *
+ * <manifestKey> is the lane's own key for the translation ("posts/a.md:fr",
+ * "docusaurus:docs/intro.md:fr"). A block is named by the hash of its source
+ * text, so an edited paragraph is a new block (the hold lifts); `source` is
+ * the 12-hex SHA-256 the key-value record uses, `methods` the TM method keys
+ * (method|model|register|coaching) whose answers the gate refused.
+ *
+ * "page" is the whole body of a page translated in one prompt
+ * (contentSegmentation: "page"): refused when the answer damages a protected
+ * placeholder or hollows the page (the whole-body checks), its `source` the
+ * hash of the body text. Editing the body — or switching to block
+ * segmentation — lifts it. A held page is not written, as a held field's
+ * page is not: there is no '[EN] ' text for a page translated whole.
+ *
+ * PRECEDENCE — the key-value rule, unit by unit:
+ *   1. Named for a redo — `--redo files:<page>`, `--redo content`
+ *      (= --force-content), `--retranslate`, or anything under `--fresh`:
+ *      always sent.
+ *   2. Refused before by the pair's method, for the unit's current source
+ *      text: held back — not sent to that method again. A fallback method
+ *      that has not refused it still gets it. A held block keeps its
+ *      '[EN] ' text until it is filled; a page whose front-matter field is
+ *      held is not written (the field gate fails a page whole, as it always
+ *      did) — and nothing of it is sent.
+ *   3. A model or method change lifts the hold (the refusal was that method
+ *      key's), as does a new source text.
+ *   The cache is always consulted first (free): holding back only stops a
+ *   paid call. A unit filled any other way — a fallback, the cache, a
+ *   paragraph written by hand — drops its record.
+ */
+
+import { tmMethodKey, lookupTM } from './tm.js';
+import { holdState, shortSourceHash } from './locale-state.js';
+import { contentRedoCommand } from './verify.js';
+import { translatableBlockSources } from './content-estimate.js';
+
+/** The content-lock key of one page × locale's refusals. */
+export function refusalRecordKey(manifestKey) {
+  return `refused:${manifestKey}`;
+}
+
+/** A block's unit id: the hash of its (restored) source text. */
+export const blockUnit = (source) => `block:${shortSourceHash(source)}`;
+
+/** A front-matter field's unit id. */
+export const fieldUnit = (field) => `field:${field}`;
+
+/** The unit id of a page body translated whole (contentSegmentation: "page"). */
+export const PAGE_UNIT = 'page';
+
+/** How a held page body is named in messages and summaries. */
+export const PAGE_NAME = 'the whole page body';
+
+/**
+ * One page × locale's refusal record (entries that do not parse are dropped).
+ *
+ * @param {object} manifest - The content lock
+ * @param {string} manifestKey
+ * @returns {Record<string, { source: string, methods: string[], on?: string }>}
+ */
+export function readRefusals(manifest, manifestKey) {
+  const rec = manifest?.[refusalRecordKey(manifestKey)];
+  if (!rec || typeof rec !== 'object' || Array.isArray(rec)) return {};
+  const out = {};
+  for (const [id, r] of Object.entries(rec)) {
+    if (r && typeof r === 'object' && typeof r.source === 'string' && Array.isArray(r.methods)) out[id] = r;
+  }
+  return out;
+}
+
+/**
+ * What a run may do with each unit of one page: 'send' (ask the pair's
+ * method), 'fallback-only' (the pair's method refused it; its fallback has
+ * not), 'held' (not sent at all).
+ *
+ * @param {object} units - readRefusals()
+ * @param {object} pairConfig
+ * @param {{ redo?: boolean }} [opts] - redo: the page was named for a redo (always sent)
+ * @returns {{ block: (source: string) => 'send'|'fallback-only'|'held',
+ *   field: (field: string, source: string) => 'send'|'fallback-only'|'held',
+ *   page: (body: string) => 'send'|'fallback-only'|'held' }}
+ */
+export function contentHolds(units, pairConfig, { redo = false } = {}) {
+  // The key-value rule (lib/locale-state.js holdState), unit by unit.
+  const stateOf = (id, source) => (redo ? 'send' : holdState(units[id], source, pairConfig));
+  return {
+    block: (source) => stateOf(blockUnit(source), source),
+    field: (field, source) => stateOf(fieldUnit(field), source),
+    page: (body) => stateOf(PAGE_UNIT, body),
+  };
+}
+
+/**
+ * One page's record after a run processed it: earlier refusals that still
+ * apply (same source text, not filled this run), plus this run's.
+ *
+ * @param {object} prior - readRefusals()
+ * @param {object} p
+ * @param {string[]} p.blockSources - The page's current translatable block sources ([] in page mode)
+ * @param {string|null} [p.pageSource] - Its body, in page mode (null in block mode)
+ * @param {Record<string, string>} p.fields - Its current translatable front-matter fields
+ * @param {Array<{ unit: string, source: string, methods: string[] }>} p.refused - Refused this run, left unfilled
+ * @param {Set<string>} p.filled - Units filled this run (any method, the cache, a person)
+ * @returns {object|null} The record, or null when nothing is left to hold
+ */
+export function nextRefusals(prior, { blockSources = [], pageSource = null, fields = {}, refused = [], filled = new Set() }) {
+  const live = new Map();
+  for (const s of blockSources) live.set(blockUnit(s), shortSourceHash(s));
+  if (typeof pageSource === 'string' && pageSource.trim()) live.set(PAGE_UNIT, shortSourceHash(pageSource));
+  for (const [f, v] of Object.entries(fields)) if (typeof v === 'string') live.set(fieldUnit(f), shortSourceHash(v));
+  const out = {};
+  for (const [id, r] of Object.entries(prior)) {
+    if (filled.has(id) || live.get(id) !== r.source) continue;
+    out[id] = r;
+  }
+  const on = new Date().toISOString().slice(0, 10);
+  for (const { unit, source, methods } of refused) {
+    if (!methods || methods.length === 0 || filled.has(unit)) continue;
+    const src = shortSourceHash(source);
+    const known = out[unit] && out[unit].source === src ? out[unit].methods : [];
+    out[unit] = { source: src, methods: [...new Set([...known, ...methods])], on };
+  }
+  return Object.keys(out).length > 0 ? out : null;
+}
+
+/** Write (or remove) one page's record in the content lock object. */
+export function storeRefusals(manifest, manifestKey, record) {
+  const key = refusalRecordKey(manifestKey);
+  if (record) manifest[key] = record;
+  else delete manifest[key];
+}
+
+/**
+ * The warning for one page's held units — both lanes say it the same way.
+ *
+ * @param {object} p
+ * @param {string} p.file - The page as sync names it (what `--redo files:` matches)
+ * @param {string} p.code
+ * @param {string} p.pairKey
+ * @param {object} p.pairConfig
+ * @param {string[]} p.names - "paragraph 2", 'front matter "title"'
+ * @param {boolean} [p.pageHeld] - A front-matter field (or the page body, in
+ *   page mode) is held: the page is not written
+ * @param {boolean} [p.handWritten] - The lane keeps a paragraph written by hand (contentDir)
+ * @param {string} [p.fallbackPrefix]
+ * @returns {string}
+ */
+export function describeHeld({ file, code, pairKey, pairConfig, names, pageHeld = false, handWritten = false, fallbackPrefix = '[EN] ' }) {
+  const shown = names.slice(0, 5).join(', ') + (names.length > 5 ? `, +${names.length - 5} more` : '');
+  const fb = pairConfig.fallback;
+  const who = fb ? `${pairConfig.method}'s and its fallback's (${fb.method})` : `${pairConfig.method}'s`;
+  const left = !pageHeld
+    ? `The '${fallbackPrefix.trim()}' text stays until they are filled.`
+    : names.includes(PAGE_NAME)
+      ? 'A page translated whole has no \'[EN]\' text to keep, so it is not written (and nothing of it is sent) until it is filled.'
+      : 'A front-matter field the gate refuses fails its page, so the page is not written (and nothing of it is sent) until it is filled.';
+  const other = fb
+    ? 'another "fallback" method on the pair'
+    : `a "fallback" method on the pair (it is asked for what ${pairConfig.method} refused)`;
+  const hand = handWritten && !pageHeld ? ', or write those paragraphs by hand (a hand-written paragraph is kept)' : '';
+  return `${file} → ${code}: ${names.length} block(s)/field(s) held back (${shown}) — the quality gate refused ${who} translation of their `
+    + `current text before; not sent, not billed. ${left} Ask again: \`${contentRedoCommand(file, { pair: pairKey })}\`; `
+    + `or fill them another way — ${other}${hand}.`;
+}
+
+/**
+ * Said with a refusal this run recorded: what the next sync does with it.
+ *
+ * @param {object} p
+ * @param {string} p.file
+ * @param {string} p.pairKey
+ * @param {object} p.pairConfig
+ * @param {number} p.count - Units refused (and left unfilled) this run
+ * @param {boolean} [p.fallbackAsked] - Some of them still go to the fallback
+ *   (it has not refused them — skipped by --max-cost, or no answer)
+ * @returns {string}
+ */
+export function describeNewHold({ file, pairKey, pairConfig, count, fallbackAsked = false, lead = 'Remembered:' }) {
+  const them = count === 1 ? 'it' : `these ${count}`;
+  const fb = pairConfig.fallback;
+  const where = fb && fallbackAsked
+    ? `${pairConfig.method} again (its fallback, ${fb.method}, is asked for what it has not refused)`
+    : `${pairConfig.method}${fb ? ` or its fallback (${fb.method})` : ''} again`;
+  return `${lead} the next sync does not send ${them} to ${where} — not billed again; `
+    + `\`${contentRedoCommand(file, { pair: pairKey })}\` asks again.`;
+}
+
+/**
+ * The end of a lane's "N block(s) … written as '[EN] '-prefixed source" line:
+ * what the next sync does with them — asks again for a block that got no
+ * usable answer, holds back one the gate refused.
+ *
+ * @param {object} p
+ * @param {string} p.file
+ * @param {string} p.pairKey
+ * @param {object} p.pairConfig
+ * @param {number} p.fellBack - Blocks written as '[EN] ' this run
+ * @param {number[]} p.newlyHeld - Of those (indices into the batch), the ones the gate refused
+ * @param {string[][]} [p.refusedBy] - translateBlocksWithFallback().refusedBy
+ * @returns {string}
+ */
+export function describeFallenBack({ file, pairKey, pairConfig, fellBack, newlyHeld, refusedBy = [] }) {
+  if (newlyHeld.length === 0) return 'Not cached, lock not advanced: the next sync retries just those block(s).';
+  const retry = fellBack - newlyHeld.length;
+  const fbKey = pairConfig.fallback ? tmMethodKey(pairConfig.fallback) : null;
+  const fallbackAsked = !!fbKey && newlyHeld.some(i => !(refusedBy[i] || []).includes(fbKey));
+  const n = newlyHeld.length;
+  return 'Not cached, lock not advanced. '
+    + (retry > 0 ? `The next sync asks again for the ${retry} that got no usable answer. ` : '')
+    + describeNewHold({
+      file, pairKey, pairConfig, count: n, fallbackAsked,
+      lead: `The ${n === 1 ? 'one' : n} the quality gate refused ${n === 1 ? 'is' : 'are'} remembered:`,
+    });
+}
+
+/**
+ * What a real run would hold back for one page — a dry run says so instead
+ * of "would create". The cache is consulted as the sync consults it (pure
+ * lookups: a preview changes nothing): only what it does not hold is held.
+ *
+ * @param {object} p
+ * @param {object} p.tm
+ * @param {string} p.code
+ * @param {object} p.pairConfig
+ * @param {Record<string, string>} p.fields - Translatable front-matter fields
+ * @param {string} p.body
+ * @param {'block'|'page'} p.segMode
+ * @param {ReturnType<typeof contentHolds>} p.holds
+ * @returns {{ names: string[], pageHeld: boolean }}
+ */
+export function previewHeld({ tm, code, pairConfig, fields, body, segMode, holds }) {
+  const keys = [tmMethodKey(pairConfig), ...(pairConfig.fallback ? [tmMethodKey(pairConfig.fallback)] : [])];
+  const cached = (text) => keys.some(k => lookupTM(tm, text, code, k) !== null);
+  const names = [];
+  for (const [field, text] of Object.entries(fields || {})) {
+    if (typeof text === 'string' && !cached(text) && holds.field(field, text) === 'held') names.push(`front matter "${field}"`);
+  }
+  if (names.length > 0) return { names, pageHeld: true };
+  if (segMode === 'page' && typeof body === 'string' && body.trim() && !cached(body) && holds.page(body) === 'held') {
+    return { names: [PAGE_NAME], pageHeld: true };
+  }
+  if (segMode === 'block' && typeof body === 'string' && body.trim() && !cached(body)) {
+    translatableBlockSources(body).forEach((source, i) => {
+      if (!cached(source) && holds.block(source) === 'held') names.push(`paragraph ${i + 1}`);
+    });
+  }
+  return { names, pageHeld: false };
+}