champollion 0.3.4 → 0.4.0

package/lib/serve.js

@@ -20,8 +20,10 @@
  *
  * NO FORKED TRANSLATION LOGIC:
  *   Requests run through the SAME pipeline `sync` uses — resolveRuntime for
- *   config/plugin/preflight resolution and translateAndValidate for the
- *   TM partition → API call → deterministic quality gate → TM store sequence.
+ *   config/plugin/preflight resolution and translateWithFallback for the
+ *   TM partition → API call → deterministic quality gate → TM store sequence
+ *   (plus the pair's fallback method, when one is configured, for what the
+ *   pair's own method could not translate safely).
  *   TM hits are served from cache at $0; gate failures come back as
  *   structured per-key errors, never as silently degraded output.
  *
@@ -49,7 +51,8 @@ import { createRequire } from 'node:module';
 import { resolveConfig } from './config.js';
 import { resolveRuntime } from './sync.js';
 import { loadTM, saveTM, isTMDirty, partitionByTM, tmMethodKey } from './tm.js';
-import { translateAndValidate } from './translate-pair.js';
+import { translateWithFallback } from './translate-pair.js';
+import { createFallbackBudget, fallbackSummary } from './fallback.js';
 import { estimateCost, QUALITY_TIERS } from './pairs.js';
 import { getMethod } from './translate.js';
 import { isUnsafeKey } from './security.js';
@@ -502,7 +505,7 @@ async function handleTranslate(ctx, body, res) {
   const { misses } = partitionByTM(runtime.tm, payload, keyNames, targetLocale, tmKey);
   let estimate = null;
   if (misses.length > 0) {
-    estimate = await estimateCost(misses.length, pairConfig);
+    estimate = await estimateCost(misses.length, pairConfig, { cwd: runtime.cwd });
     const estimated = estimate?.estimatedCost ?? null;
 
     if (maxCostPerRequest !== null || maxSessionCost !== null) {
@@ -525,26 +528,47 @@ async function handleTranslate(ctx, body, res) {
     }
   }
 
+  // ── The pair's fallback method, under the same caps ────────────────
+  // What the pair's own method cannot translate safely goes to its
+  // fallback (if configured). Under a cap, that batch is priced before it
+  // runs and must fit in what the primary's estimate left of the tighter of
+  // the per-request cap and the remaining session ceiling (lib/fallback.js).
+  const capsLeft = [
+    maxCostPerRequest,
+    maxSessionCost !== null ? maxSessionCost - session.spendUsd : null,
+  ].filter(v => v !== null);
+  const primaryEstimate = misses.length > 0 ? (estimate?.estimatedCost ?? 0) : 0;
+  const fallbackBudget = createFallbackBudget({
+    maxCost: capsLeft.length > 0 ? Math.min(...capsLeft) : null,
+    committed: primaryEstimate,
+    cwd: runtime.cwd,
+  });
+
   // ── The real pipeline: TM partition → API → quality gate → TM store ──
   let result;
   try {
-    result = await translateAndValidate(keyNames, payload, pairConfig, pairKey, {
+    result = await translateWithFallback(keyNames, payload, pairConfig, pairKey, {
       apiKey: runtime.apiKey,
+      cwd: runtime.cwd,
       tm: runtime.tm,
       targetCode: targetLocale,
       onProgress: null,
+      budget: fallbackBudget,
     });
   } catch (err) {
     output.error(`[serve] ${pairKey}: pipeline error — ${err.message}`);
     return sendError(res, 500, 'internal_error', 'Translation pipeline failed. See server logs.');
   }
-  if (result.apiCalled) output.progress(' done\n');
+  if (result.apiCalled) output.progressDone(`[serve] ${pairKey}`, 'done');
 
   // Session ledger: count the estimate for any request that actually hit
   // the upstream API (conservative — estimates deliberately err high).
   if (result.apiCalled && estimate?.estimatedCost != null) {
     session.spendUsd += estimate.estimatedCost;
   }
+  // Fallback batches approved under a cap carry their estimate too.
+  const fallbackSpend = fallbackBudget.committed - primaryEstimate;
+  if (fallbackSpend > 0) session.spendUsd += fallbackSpend;
 
   // Persist TM mutations (stores AND evictions) after every request so a
   // crash never loses paid, gate-validated translations. Atomic write.
@@ -616,6 +640,17 @@ async function handleTranslate(ctx, body, res) {
     costUsd = estimate.estimatedCost;
     costBasis = 'estimate';
   }
+  // A fallback that ran adds its own spend: priced when a cap made us price
+  // it, otherwise unknown — never silently left out of a known figure.
+  if (result.fallback?.attempted > 0) {
+    if (fallbackBudget.maxCost !== null && costUsd !== null) {
+      costUsd += fallbackSpend;
+      costBasis = 'estimate';
+    } else {
+      costUsd = null;
+      costBasis = 'unknown';
+    }
+  }
 
   const meta = {
     served_by: `champollion-serve/${CLI_VERSION}`,
@@ -629,6 +664,8 @@ async function handleTranslate(ctx, body, res) {
     translated: translatedCount,
     failed: errorCount,
     tm_hits: result.tmHitCount,
+    // The pair's fallback method and what it did (pairs with a fallback).
+    ...(result.fallback && { fallback: fallbackSummary(result.fallback) }),
     // Script conversion status for this pair: which script was written and
     // how many values stayed in the working script (unmapped letters).
     script_conversion: {

package/lib/shared-output-seed.js

@@ -0,0 +1,164 @@
+/**
+ * shared-output-seed.js — what a locale's shared-output index starts a sync
+ * with: the translations already on disk that this run leaves alone.
+ *
+ * WHY: `verify` checks one locale's whole state for a model repeating one
+ * memorized sentence (lib/verify.js sharedOutputErrors): every key value of
+ * every file AND every Markdown page, counted together. Sync's gate
+ * (lib/validate.js SharedOutputIndex) saw less — only what this run accepted,
+ * plus, per key-value file, what that file kept, added when the file was
+ * reached. So a new answer repeating a sentence that sat in an untouched
+ * newsletter page, in a key-value file with nothing to translate (it returns
+ * before it seeds), or in a namespace file processed later, was written —
+ * and `verify` then flagged it (Round 10, school persona). Seeding the index
+ * with what the run leaves alone, before anything is checked, gives the gate
+ * the same scope `verify` reads.
+ *
+ * "Left alone" is what the run will not re-translate:
+ *   - a key value Champollion recorded writing for the CURRENT source text
+ *     (the lock's written record matches the source and the value), or one
+ *     with no record whose source did not change — never an '[EN] '
+ *     placeholder, a pending key, a key named for a redo, or any key under a
+ *     bulk `--redo all`;
+ *   - a Markdown page whose content lock says it was made from the current
+ *     source, when no --redo files:/content, --force-content or --retranslate
+ *     scope reaches it.
+ * Values the run replaces are left out, so their old text never counts
+ * against the new one. Read-only.
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { expectedForTarget, readLocaleFlat, lockKey, discoverLocaleLayout, loadSourceUnits } from './locale-layout.js';
+import { originKey } from './plurals.js';
+import { decodeWritten, shortSourceHash, valueHash, LockState } from './locale-state.js';
+import { SharedOutputIndex, sharedOutputItems } from './validate.js';
+import { readLock } from './hash.js';
+import { discoverContentFiles, getTargetContentPath } from './content.js';
+import { contentItemsFor } from './verify.js';
+import { readContentManifest } from './content-sync.js';
+
+/**
+ * Items (lib/validate.js sharedOutputItems shape) for the translations of
+ * `code` on disk that this run leaves alone.
+ *
+ * @param {object} p
+ * @param {object} p.config - Resolved config (inputLocale, contentDir, format, fallbackPrefix, …)
+ * @param {object} p.layout - Locale layout (lib/locale-layout.js)
+ * @param {Array<object>} p.units - Source units (loadSourceUnits)
+ * @param {string} p.code - Target locale
+ * @param {{ written: object, pending: object }} p.localeState - LockState.peek(code)
+ * @param {string[]} [p.namedKeys] - Keys named for a redo (source key space, namespaced)
+ * @param {boolean} [p.bulk] - `--redo all`: every key is re-translated
+ * @param {object|null} [p.fileScope] - lib/file-scope.js (--redo files: / --files / --retranslate)
+ * @param {boolean} [p.forceContent] - --force-content / --redo content
+ * @param {string} p.cwd - Project directory
+ * @returns {Array<{ key: string, source: string, value: string }>}
+ */
+export function itemsLeftAlone({
+  config, layout, units, code, localeState, namedKeys = [], bulk = false, fileScope = null, forceContent = false, cwd,
+}) {
+  const items = [];
+  const fallbackPrefix = config.fallbackPrefix || '[EN] ';
+  const named = new Set(namedKeys);
+  const written = localeState?.written || {};
+  const pending = localeState?.pending || {};
+
+  if (!bulk) {
+    for (const unit of units || []) {
+      const file = layout.fileFor(code, unit.ns);
+      if (!fs.existsSync(file.path)) continue;
+      let target;
+      try { target = readLocaleFlat(file) || {}; } catch { continue; }
+      const { flat: expected, expansion } = expectedForTarget(unit, config.inputLocale, code);
+      const changed = new Set(unit.changedKeys || []);
+      for (const [k, v] of Object.entries(target)) {
+        const src = expected[k];
+        if (typeof v !== 'string' || typeof src !== 'string' || v.startsWith(fallbackPrefix)) continue;
+        const lk = lockKey(layout, unit.ns, k);
+        const origin = originKey(k, expansion);
+        if (pending[lk] !== undefined || named.has(lk) || named.has(lockKey(layout, unit.ns, origin))) continue;
+        const record = decodeWritten(written[lk]);
+        if (record) {
+          if (record.source !== shortSourceHash(src) || record.value !== valueHash(v)) continue;
+        } else if (changed.has(origin)) {
+          continue;
+        }
+        items.push(...sharedOutputItems(lk, src, v));
+      }
+    }
+  }
+
+  if (config.contentDir && config.format !== 'docusaurus') {
+    const contentDir = path.resolve(cwd, config.contentDir);
+    const manifest = readContentManifest(cwd);
+    let sources = [];
+    try { sources = fs.existsSync(contentDir) ? discoverContentFiles(contentDir, config.inputLocale) : []; } catch { sources = []; }
+    const leftAlone = new Set();
+    for (const sourcePath of sources) {
+      const relPath = path.relative(contentDir, sourcePath);
+      if (!fs.existsSync(getTargetContentPath(sourcePath, code, config.inputLocale))) continue;
+      if (fileScope && !fileScope.includes(relPath)) { leftAlone.add(sourcePath); continue; }
+      if (forceContent || (fileScope && fileScope.retranslates(relPath))) continue;
+      let hash;
+      try { hash = crypto.createHash('sha256').update(fs.readFileSync(sourcePath, 'utf-8'), 'utf-8').digest('hex'); } catch { continue; }
+      if (manifest[`${relPath}:${code}`] === hash) leftAlone.add(sourcePath);
+    }
+    if (leftAlone.size > 0) items.push(...contentItemsFor(config, cwd, code, { only: (p) => leftAlone.has(p) }));
+  }
+  return items;
+}
+
+/**
+ * The shared-output index a sync of `code` starts with — ONE construction for
+ * sync's gate and for anything else that must refuse what sync refuses (the
+ * MCP translate tool with a project_dir): the sentences an earlier sync caught
+ * a model repeating (the cache's `_meta.memorized`, refused from their first
+ * source on), plus every translation of the locale on disk that the run
+ * leaves alone (itemsLeftAlone). Round 10, school persona: the translate tool
+ * returned the sentence the project already knew to be memorized, because
+ * its index held only that call's answers.
+ *
+ * @param {object} p
+ * @param {object} p.config - Resolved config
+ * @param {string} p.cwd - Project directory
+ * @param {string} p.code - Target locale
+ * @param {object|null} [p.tm] - The project's cache (lib/tm.js loadTM)
+ * @param {object} [p.layout] - Locale layout (discovered when absent)
+ * @param {Array<object>} [p.units] - Source units (loaded when absent)
+ * @param {object} [p.localeState] - LockState.peek(code) (read from the lock when absent)
+ * @param {string[]} [p.namedKeys] - Keys named for a redo this run
+ * @param {boolean} [p.bulk] - `--redo all`
+ * @param {object|null} [p.fileScope] - --redo files: / --files / --retranslate
+ * @param {boolean} [p.forceContent] - --force-content / --redo content
+ * @returns {SharedOutputIndex}
+ */
+export function projectSharedOutputIndex({
+  config, cwd, code, tm = null, layout = null, units = null, localeState = null,
+  namedKeys = [], bulk = false, fileScope = null, forceContent = false,
+}) {
+  const index = new SharedOutputIndex({ protectedTerms: config.protectedTerms || [] });
+  index.markMemorized(tm?._meta?.memorized?.[code] || []);
+  let lay = layout;
+  let us = units;
+  let state = localeState;
+  // A project whose key-value files cannot be read (a Docusaurus site, a
+  // missing locales folder) still has its pages and its memorized sentences.
+  if (!lay || !us) {
+    try {
+      lay = discoverLocaleLayout(config, { cwd });
+      us = loadSourceUnits(lay);
+    } catch {
+      lay = null;
+      us = [];
+    }
+  }
+  if (!state) {
+    try { state = new LockState(readLock(cwd).locales).peek(code); } catch { state = { written: {}, pending: {} }; }
+  }
+  index.add(itemsLeftAlone({
+    config, layout: lay, units: lay ? us : [], code, localeState: state, namedKeys, bulk, fileScope, forceContent, cwd,
+  }));
+  return index;
+}

package/lib/source-contexts.js

@@ -0,0 +1,39 @@
+/**
+ * source-contexts.js — which gettext contexts (msgctxt) a project's source
+ * gives each text.
+ *
+ * Sync keys a gettext entry with a context as `msgctxt\u0004msgid`, and caches
+ * its translation under the context folded into the text (lib/tm-evict.js
+ * tmSourceText): "Cancel" the button and "Cancel" the verb are two entries.
+ * A caller that translates a bare text in a project (the MCP translate tool
+ * with project_dir) needs to know when that text exists there ONLY with a
+ * context: a context-free cache entry for it is never read by sync, and only
+ * shadows the context-keyed ones (Round 13, Django persona).
+ */
+
+import { discoverLocaleLayout, loadSourceUnits } from './locale-layout.js';
+import { CONTEXT_SEPARATOR } from './tm-evict.js';
+
+/**
+ * Every source text of the project, with whether it has an entry without a
+ * context and the contexts it has.
+ *
+ * @param {object} config - A resolved config (lib/config.js resolveConfig)
+ * @param {{ cwd: string }} options
+ * @returns {Map<string, { plain: boolean, contexts: string[] }>} source text → its entries
+ */
+export function sourceTextContexts(config, { cwd }) {
+  const layout = discoverLocaleLayout(config, { cwd });
+  const out = new Map();
+  for (const unit of loadSourceUnits(layout)) {
+    for (const [key, value] of Object.entries(unit.flat)) {
+      if (typeof value !== 'string') continue;
+      if (!out.has(value)) out.set(value, { plain: false, contexts: [] });
+      const entry = out.get(value);
+      const at = key.indexOf(CONTEXT_SEPARATOR);
+      if (at < 0) entry.plain = true;
+      else if (!entry.contexts.includes(key.slice(0, at))) entry.contexts.push(key.slice(0, at));
+    }
+  }
+  return out;
+}

package/lib/submit.mjs

@@ -1,5 +1,5 @@
 /**
- * submit — declarative model behind `champollion submit`.
+ * submit — declarative model behind `champollion network submit`.
  *
  * The CLI front door to Champollion's REVIEW-GATED community submission path.
  * It gathers the fields a submission needs and hands the contributor a
@@ -22,6 +22,8 @@
  * @see .github/ISSUE_TEMPLATE/
  */
 
+import { parseLanguagePair, formatLanguagePair } from './language-pair.js';
+
 /**
  * The compliance attestation every submission must carry. A single source of
  * truth so the CLI, the issue forms, and the docs all say the SAME thing.
@@ -30,6 +32,14 @@ export const ATTESTATION_TEXT =
   'I confirm this is publicly listable, contains NO corpus content or personal data, ' +
   "and respects the source's license and any community/sovereignty restrictions.";
 
+/**
+ * The label of every "language pair(s)" field. Three types asked for pairs in
+ * two spellings (eng-crk in one, source→target in two); the reader below
+ * takes any spelling lib/language-pair.js accepts and the submission carries
+ * each pair as source>target, the form the leaderboard writes.
+ */
+export const PAIRS_LABEL = 'Language pair(s), ISO 639-3, one per line, source>target (e.g. eng>crk; eng-crk works too)';
+
 /** Repo the issue forms live in. Overridable, but this is the real default. */
 export const DEFAULT_REPO_BASE = 'https://github.com/gamedaysuits/Champollion';
 
@@ -37,6 +47,7 @@ export const DEFAULT_REPO_BASE = 'https://github.com/gamedaysuits/Champollion';
 //   input  — single-line free text
 //   text   — multi-line free text (textarea)
 //   choice — one-of a fixed list (`choices`)
+//   (a text field with `pairs: true` holds language pairs, one per line — readPairLines)
 //   bool   — a required confirmation (attestation / consent); never sent in the
 //            pre-fill URL (GitHub can't pre-check a checkbox), only recorded
 //            locally and enforced by validation.
@@ -57,7 +68,7 @@ export const SUBMISSION_TYPES = [
     blurb: 'An evaluation corpus or benchmark — metadata + a fetch-from-source pointer only, never the corpus content.',
     fields: [
       { id: 'dataset-name', label: 'Dataset name', kind: 'input', required: true, primary: true },
-      { id: 'pairs', label: 'Language pair(s) (ISO 639-3, e.g. eng-crk; one per line)', kind: 'text', required: true },
+      { id: 'pairs', label: PAIRS_LABEL, kind: 'text', pairs: true, required: true },
       { id: 'license', label: 'License', kind: 'choice', required: true,
         choices: ['CC-BY-4.0', 'CC-BY-SA-4.0', 'CC0-1.0', 'CC-BY-NC-4.0', 'Proprietary / all rights reserved', 'Other / unsure'] },
       { id: 'license-url', label: 'License URL', kind: 'input' },
@@ -135,7 +146,7 @@ export const SUBMISSION_TYPES = [
       { id: 'display-name', label: 'Provider display name (consented public name)', kind: 'input', required: true, primary: true },
       { id: 'provider-type', label: 'Provider type', kind: 'choice', required: true,
         choices: ['community_org', 'agency', 'individual'] },
-      { id: 'pairs', label: 'Language pair(s) (ISO 639-3 source→target, one per line)', kind: 'text', required: true },
+      { id: 'pairs', label: PAIRS_LABEL, kind: 'text', pairs: true, required: true },
       { id: 'variety', label: 'Variety / script (e.g. SRO, Latn)', kind: 'input' },
       { id: 'dispatch-channel', label: 'Dispatch channel (the address itself is PII — omit it)', kind: 'choice',
         choices: ['email', 'webhook', 'api'] },
@@ -158,7 +169,7 @@ export const SUBMISSION_TYPES = [
     blurb: 'A published result from another system/paper — cited, never re-hosted or re-ranked as our own measurement.',
     fields: [
       { id: 'system-name', label: 'System / method name', kind: 'input', required: true, primary: true },
-      { id: 'pairs', label: 'Language pair(s) (ISO 639-3 source→target, one per line)', kind: 'text', required: true },
+      { id: 'pairs', label: PAIRS_LABEL, kind: 'text', pairs: true, required: true },
       { id: 'dataset', label: 'Dataset / benchmark', kind: 'input', required: true },
       { id: 'metric', label: 'Metric', kind: 'choice', required: true,
         choices: ['chrF++', 'chrF', 'BLEU', 'COMET', 'TER', 'Other'] },
@@ -252,7 +263,7 @@ export function repoBaseFromRepository(repoUrl) {
  */
 export function validateSubmission(type, values = {}) {
   const errors = [];
-  if (!type) return { ok: false, errors: ['unknown submission type — run `champollion submit --list`'] };
+  if (!type) return { ok: false, errors: ['unknown submission type — run `champollion network submit --list`'] };
   for (const f of type.fields) {
     const v = values[f.id];
     if (f.kind === 'bool') {
@@ -266,10 +277,51 @@ export function validateSubmission(type, values = {}) {
       const empty = v == null || String(v).trim() === '';
       if (empty) errors.push(`missing required field: ${f.id} (${f.label})`);
     }
+    if (f.pairs && v != null && String(v).trim() !== '') {
+      for (const e of readPairLines(v).errors) errors.push(`${f.id}: ${e}`);
+    }
   }
   return { ok: errors.length === 0, errors };
 }
 
+/**
+ * Read a "language pair(s)" field: one pair per line, each in any spelling
+ * lib/language-pair.js accepts. Blank lines are skipped.
+ *
+ * @param {*} value
+ * @returns {{pairs: string[], errors: string[]}} pairs written source>target
+ */
+export function readPairLines(value) {
+  const pairs = [];
+  const errors = [];
+  String(value).split(/\r?\n/).forEach((line, i) => {
+    if (!line.trim()) return;
+    const p = parseLanguagePair(line, { label: `line ${i + 1}:` });
+    if (p.ok) pairs.push(formatLanguagePair(p));
+    else errors.push(p.error);
+  });
+  return { pairs, errors };
+}
+
+/**
+ * The values with every pair field written the one way (source>target, one
+ * per line), for the issue URL and the local record. A field that does not
+ * read cleanly is left as typed — validateSubmission reports it.
+ *
+ * @param {object} type one of SUBMISSION_TYPES
+ * @param {Record<string,*>} values
+ * @returns {Record<string,*>} a new object; `values` is not changed
+ */
+export function normalizeSubmissionValues(type, values = {}) {
+  const out = { ...values };
+  for (const f of type.fields) {
+    if (!f.pairs || out[f.id] == null || String(out[f.id]).trim() === '') continue;
+    const r = readPairLines(out[f.id]);
+    if (r.errors.length === 0) out[f.id] = r.pairs.join('\n');
+  }
+  return out;
+}
+
 /** The field whose value seeds the issue title (the `primary` one). */
 function primaryField(type) {
   return type.fields.find((f) => f.primary) || null;