champollion 0.3.3 → 0.4.0

package/lib/autofix.js

@@ -12,6 +12,14 @@
  *   5. One-undo — `wrap --undo` restores from backup
  *   6. Diff output — prints git-style diff of every change
  *
+ * NOT FOR FLUTTER OR GETTEXT. wrap writes dotted keys for t('…') calls.
+ * An ARB key must be a Dart identifier, read as
+ * AppLocalizations.of(context)!.homeTitle; a gettext key IS the source
+ * text, marked with _("…") and collected by the project's own extractor.
+ * Writing "general.welcome" into either produces a catalog the app cannot
+ * use, so wrap refuses those formats before any component is rewritten
+ * (wrapUnsupportedReason, checked by the wrap command up front).
+ *
  * KEY GENERATION:
  *   "Welcome to my portfolio" → t("general.welcome_to_my_portfolio")
  *   "Get in Touch" → t("general.get_in_touch")
@@ -24,6 +32,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { execSync } from 'node:child_process';
 import { isNonTranslatableString } from './string-classify.js';
+import { readLocaleFile, writeLocaleFile, detectYAMLStyle } from './format.js';
 
 // -----------------------------------------------------------------
 // Key generation
@@ -343,57 +352,94 @@ function isAmbiguous(text, lineContext) {
 }
 
 /**
- * Add generated keys to locale files.
+ * Why `wrap` cannot store extracted keys in this locale file, or null when
+ * it can. Checked by the wrap command BEFORE any component is rewritten
+ * (a rewritten t('…') call with nowhere to store its key shows the raw key
+ * name in the app), and again by addKeysToLocales.
  *
- * @param {object[]} fixes - Array of { key, text } from processFile
- * @param {string} localesDir - Path to locale files directory
- * @param {string} sourceLocale - Source locale code (e.g., 'en')
- * @param {string[]} targetLocales - Target locale codes
+ * @param {{ format: string, rel?: string, path?: string }} file
+ * @returns {string|null}
  */
-function addKeysToLocales(fixes, localesDir, sourceLocale, targetLocales) {
-  if (fixes.length === 0) return;
-
-  // Build key-value map from fixes
-  const newKeys = {};
-  for (const fix of fixes) {
-    // Expand dot-notation key into nested object
-    const parts = fix.key.split('.');
-    let current = newKeys;
-    for (let i = 0; i < parts.length - 1; i++) {
-      if (!current[parts[i]]) current[parts[i]] = {};
-      current = current[parts[i]];
-    }
-    current[parts[parts.length - 1]] = fix.text;
+function wrapUnsupportedReason(file) {
+  const name = file.rel || (file.path ? path.basename(file.path) : 'the locale file');
+  if (file.format === 'arb') {
+    return `wrap does not apply to Flutter ARB files (${name}): it writes dotted keys for t('…') calls, `
+      + 'but an ARB key must be a Dart identifier, read as AppLocalizations.of(context)!.myKey. '
+      + `Add messages to ${name} yourself — champollion sync translates them.`;
   }
+  if (file.format === 'po') {
+    return `wrap does not apply to gettext catalogs (${name}): in gettext the key IS the source text. `
+      + 'Mark strings with _("…") / gettext("…") (Django: {% translate %}), collect them with your '
+      + 'extractor (makemessages, pybabel extract, xgettext), and champollion sync translates the catalog.';
+  }
+  return null;
+}
 
-  // Update source locale file
-  const sourcePath = path.join(localesDir, `${sourceLocale}.json`);
-  if (fs.existsSync(sourcePath)) {
-    const existing = JSON.parse(fs.readFileSync(sourcePath, 'utf-8'));
-    const merged = deepMerge(existing, newKeys);
-    fs.writeFileSync(sourcePath, JSON.stringify(merged, null, 2) + '\n', 'utf-8');
+/**
+ * Add generated keys to locale files.
+ *
+ * The files come from the project's locale layout (lib/locale-layout.js),
+ * resolved by the caller BEFORE any source file is rewritten: one file for
+ * a flat project, the chosen namespace's file for a folder-per-locale one.
+ * The format is the file's own — a Hugo project's en.toml gains TOML keys
+ * (this used to look only for `<locale>.json`, so TOML/YAML projects lost
+ * every extracted key without a word).
+ *
+ * Source gets the extracted text; each EXISTING target file gets the same
+ * keys with an "[EN] " placeholder, which the next sync translates. Keys
+ * already present are never overwritten.
+ *
+ * @param {object[]} fixes - Array of { key, text } from processFile
+ * @param {{ path: string, format: string }} sourceFile - Source locale file
+ * @param {Array<{ path: string, format: string }>} targetFiles - Target locale files
+ * @returns {{ source: boolean, targets: number }} What was written
+ */
+function addKeysToLocales(fixes, sourceFile, targetFiles = []) {
+  const written = { source: false, targets: 0 };
+  if (fixes.length === 0) return written;
+  // Defense in depth: the wrap command refuses these formats up front.
+  for (const file of [sourceFile, ...targetFiles]) {
+    const reason = wrapUnsupportedReason(file);
+    if (reason) throw new Error(reason);
   }
 
-  // Add placeholder entries to target locale files
-  for (const locale of targetLocales) {
-    const targetPath = path.join(localesDir, `${locale}.json`);
-    if (fs.existsSync(targetPath)) {
-      const existing = JSON.parse(fs.readFileSync(targetPath, 'utf-8'));
-      // Use [EN] prefix for target locales as untranslated markers
-      const placeholders = {};
+  const addTo = (file, prefix) => {
+    if (!fs.existsSync(file.path)) return false;
+    if (file.format === 'json') {
+      // Build the nested object of new keys from dot-notation keys.
+      const newKeys = {};
       for (const fix of fixes) {
         const parts = fix.key.split('.');
-        let current = placeholders;
+        let current = newKeys;
         for (let i = 0; i < parts.length - 1; i++) {
           if (!current[parts[i]]) current[parts[i]] = {};
           current = current[parts[i]];
         }
-        current[parts[parts.length - 1]] = `[EN] ${fix.text}`;
+        current[parts[parts.length - 1]] = `${prefix}${fix.text}`;
       }
-      const merged = deepMerge(existing, placeholders);
-      fs.writeFileSync(targetPath, JSON.stringify(merged, null, 2) + '\n', 'utf-8');
+      const existing = JSON.parse(fs.readFileSync(file.path, 'utf-8'));
+      const merged = deepMerge(existing, newKeys);
+      fs.writeFileSync(file.path, JSON.stringify(merged, null, 2) + '\n', 'utf-8');
+      return true;
+    }
+    // TOML / YAML: flat maps through the same reader/writer sync uses.
+    const flat = readLocaleFile(file.path, file.format);
+    for (const fix of fixes) {
+      if (!(fix.key in flat)) flat[fix.key] = `${prefix}${fix.text}`;
     }
+    const yamlStyle = file.format === 'yaml'
+      ? detectYAMLStyle(fs.readFileSync(file.path, 'utf-8'))
+      : null;
+    writeLocaleFile(file.path, flat, file.format, flat, yamlStyle);
+    return true;
+  };
+
+  written.source = addTo(sourceFile, '');
+  // Use [EN] prefix for target locales as untranslated markers
+  for (const target of targetFiles) {
+    if (addTo(target, '[EN] ')) written.targets++;
   }
+  return written;
 }
 
 /**
@@ -428,5 +474,6 @@ export {
   shouldFixText,
   isAmbiguous,
   addKeysToLocales,
+  wrapUnsupportedReason,
   deepMerge,
 };

package/lib/bridge/method_bridge.py

@@ -27,7 +27,7 @@ the method modules themselves, which bring their own dependencies).
 Why self-contained:
     The bridge inlines the method_loader logic (~60 lines) so the user
     does NOT need to pip install the Arena. Only the method module
-    (e.g., pip install crk-translate) is required.
+    (e.g., python3 -m pip install crk-translate) is required.
 
 Maintained alongside the Arena's method_loader.py — if the manifest
 format changes there, it must be updated here. Both live in the same
@@ -58,6 +58,13 @@ logger = logging.getLogger("champollion.bridge")
 
 _REQUIRED_MANIFEST_FIELDS = {"name", "entry_point"}
 
+# Temperature a plugin receives when the CLI config sets none. It MUST equal
+# the harness RunConfig default (arena/mt_eval_harness/config.py
+# DEFAULT_TEMPERATURE) — a plugin validated in the Arena must see the same
+# config here. cli/test/external-method.test.js checks the two against each
+# other. (This was 0.3: a CLI-only default the plugin never saw in the Arena.)
+HARNESS_DEFAULT_TEMPERATURE = 0.0
+
 
 class MethodLoadError(Exception):
     """Raised when a method plugin cannot be loaded."""
@@ -145,7 +152,7 @@ def _load_method(method_dir: Path) -> tuple:
         raise MethodLoadError(
             f"Failed to load {module_file}: {type(e).__name__}: {e}\n"
             f"  Check that the method module's dependencies are installed.\n"
-            f"  Try: pip install -e {method_dir.parent}"
+            f"  Try: {sys.executable} -m pip install -e '{method_dir.parent}'"
         ) from e
 
     if not hasattr(module, class_name):
@@ -280,7 +287,12 @@ async def _handle_translate(method, request: dict) -> dict:
     cfg.model_id = config_overrides.get("model")  # CrkPipelineMethod reads this
     cfg.source_lang = request.get("source_locale", "en")
     cfg.target_lang = request.get("target_locale", "")
-    cfg.temperature = config_overrides.get("temperature", 0.3)
+    # The CLI sends null (or omits the key) when the config sets no
+    # temperature; either way the plugin gets the harness default.
+    temperature = config_overrides.get("temperature")
+    cfg.temperature = (
+        HARNESS_DEFAULT_TEMPERATURE if temperature is None else float(temperature)
+    )
 
     # Call the method's translate — same interface the Arena uses
     results = await method.translate(entries, cfg)

package/lib/cards/reader.js

@@ -122,6 +122,40 @@ export function isDisputed(value) {
       || value.agreement === AGREEMENT.INCOMMENSURABLE);
 }
 
+/**
+ * Every attribution envelope on a card, with the dotted path it sits at.
+ *
+ * A display layer that lists the envelopes it knows by name goes stale the
+ * day the atlas starts attributing another field — and the new field's
+ * disagreement then never reaches a reader. Walking the card finds them all,
+ * so a surface can show every disputed field, including ones it was not
+ * written for. `_`-prefixed keys are bookkeeping (provenance, build stamps),
+ * not facts, and are skipped; an envelope's own values are not walked into.
+ *
+ * @param {object} card  a raw or normalized card
+ * @returns {{path: string, value: object}[]}
+ */
+export function attributedFields(card) {
+  const found = [];
+  const walk = (v, p) => {
+    if (!v || typeof v !== 'object') return;
+    if (isAttributed(v)) {
+      found.push({ path: p, value: v });
+      return;
+    }
+    if (Array.isArray(v)) {
+      v.forEach((x, i) => walk(x, `${p}[${i}]`));
+      return;
+    }
+    for (const [k, x] of Object.entries(v)) {
+      if (k.startsWith('_')) continue;
+      walk(x, p ? `${p}.${k}` : k);
+    }
+  };
+  walk(card, '');
+  return found;
+}
+
 /**
  * Read one card. Returns null when the language has no card — which is a real
  * answer: a language with no asserted value gets no card, rather than an empty
@@ -386,10 +420,24 @@ export function normalizeCard(card) {
     if (scales) {
       const claims = attributions(out.endangerment);
       for (const source of scales.authorityOrder) {
-        const hit = claims.find((c) => String(c.source ?? '').startsWith(source));
+        // One source can hold several assessments (ELCat keeps one per
+        // record, each with its own certainty). The projector stores them
+        // alphabetically, which says nothing about confidence — reading the
+        // first one showed Plains Cree as "endangered" from a 0.2-certainty,
+        // BC-only record over ELCat's 0.8-certainty "threatened". Take the
+        // assessment the source is most certain of; ties keep stored order.
+        let hit = null;
+        let tier = null;
+        let best = -Infinity;
+        for (const c of claims) {
+          if (!String(c.source ?? '').startsWith(source)) continue;
+          const t = scales.scales[source]?.map?.[String(c.value).trim()];
+          if (!t) continue;
+          const m = /Certainty:\s*([0-9.]+)/.exec(String(c.note ?? ''));
+          const certainty = m ? Number(m[1]) : -1;
+          if (certainty > best) { best = certainty; hit = c; tier = t; }
+        }
         if (!hit) continue;
-        const tier = scales.scales[source]?.map?.[String(hit.value).trim()];
-        if (!tier) continue;
         out.vitality = {
           unescoStatus: tier,
           assessedBy: hit.source,

package/lib/cards/remote.js

@@ -262,6 +262,15 @@ export function deriveRegistersFromFormality(formality) {
  * be worse than omitting them.
  */
 const DETAIL_PASSTHROUGH = [
+  // Attributed envelope, same shape as the card's ({agreement, values:
+  // [{value, source, note}]}); the reader derives vitality from it by source.
+  // It was published but not passed through, so every long-tail card lost
+  // its endangerment (MCP get_language on abp, 2026-10-03).
+  'endangerment',
+  // Dictionaries / grammars / wordlists — "what exists for this language".
+  // Published by build-trading-card-data.mjs from 2026-10-03 on; absent on
+  // rows uploaded before that, in which case the field simply stays unset.
+  'lexicalResources',
   'classification',
   'vitality',
   'speakerEstimates',
@@ -290,6 +299,10 @@ const DETAIL_PASSTHROUGH = [
   'macrolanguage',
   'members',
   'taxonomyNotes',
+  // Per-field citations (build-trading-card-data.mjs from 2026-10-04 on).
+  // Absent on rows uploaded before that: readers then report the source as
+  // not carried rather than inventing one.
+  '_fieldSources',
 ];
 
 /**
@@ -352,7 +365,9 @@ export function buildCardFromRemote(indexRow, detailRow, { aliases } = {}) {
     if (derived.defaultKey && !card.formality.default) {
       card.formality = { ...card.formality, default: derived.defaultKey };
     }
+    // Merged, never replaced: the detail blob's own citations came first.
     card._fieldSources = {
+      ...(card._fieldSources || {}),
       registers: `derived-from-formality (${card.formality.source || 'unknown'})`,
     };
   }

package/lib/cards/search-names.js

@@ -0,0 +1,178 @@
+/**
+ * search-names.js — the names a language is FOUND by, for the bundled manifest.
+ *
+ * An npm install ships the full card for ~1,150 core languages only; every
+ * other language is a manifest entry in shared/cards-fallback.json. That entry
+ * used to carry the displayed name and code aliases and nothing else, so an
+ * install could not find Innu (moe) by "Montagnais" — the alternate name
+ * ISO 639-3 records — nor 1,543 languages by their endonyms nor 411 by the
+ * name another registry gives them (Round 11). The repo corpus found them all.
+ *
+ * This module turns a card's recorded names into the manifest's compact
+ * search-name list, and back:
+ *
+ *   - which names: every value of the `name` envelope (each registry's name),
+ *     every value of the `endonym` envelope, and `alternateNames` (ISO 639-3's
+ *     list) — read through the card adapter's `attributions()`, each with the
+ *     source that records it;
+ *   - deduplicated the way search compares names (`foldForSearch`, the same
+ *     fold as the MCP server's normalizeForMatch): a name that folds to the
+ *     displayed name or a code alias is dropped (search already finds it), and
+ *     names that fold alike are kept once — the first spelling, cited to the
+ *     sources that record exactly that spelling (never to a source that wrote
+ *     it differently);
+ *   - each name's sources kept compactly: `[text, ref, ref, …]`, where a ref
+ *     indexes the bundle's `nameRefs` table of `[field, source]` pairs
+ *     (three sources × three fields today — a source id is written once, not
+ *     once per name).
+ *
+ * A search name is for FINDING a language. Anything a result shows from one
+ * carries its field and source (decodeSearchNames returns both), so the
+ * result can say how it matched and cite it.
+ */
+
+import { attributions, isAttributed } from './reader.js';
+
+/** Card fields whose values are names a language is known by, in card order. */
+export const SEARCH_NAME_FIELDS = Object.freeze(['name', 'endonym', 'alternateNames']);
+
+/**
+ * Fold a string the way language search compares names: Unicode-decompose,
+ * drop combining marks, lowercase, collapse every run of non-letters/digits
+ * to one space. "Èdè Yorùbá" → "ede yoruba"; "Ta'Izzi-Adeni" → "ta izzi adeni".
+ * Must stay identical to normalizeForMatch in mcp-server/src/tools/languages.js
+ * (a test there holds the two together).
+ *
+ * @param {unknown} s
+ * @returns {string}
+ */
+export function foldForSearch(s) {
+  return String(s ?? '')
+    .normalize('NFD')
+    .replace(/\p{M}+/gu, '')
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, ' ')
+    .trim();
+}
+
+/** A bare lowercase 2–3 letter string is a code, not a name ("Ata" is a name). */
+const CODE_LIKE = /^[a-z]{2,3}$/;
+
+/** The sources a card stamps on a flat field (`_fieldSources`). */
+function stampedSources(card, field) {
+  const v = card?._fieldSources?.[field];
+  return (Array.isArray(v) ? v : [v]).filter((s) => typeof s === 'string' && s);
+}
+
+/**
+ * Every recorded name on a RAW card (before normalizeCard flattens `name`),
+ * each with the field and source that record it.
+ *
+ * @param {object} raw  a card as stored (attribution envelopes intact)
+ * @returns {Array<{text: string, field: string, source: string|null}>}
+ */
+export function searchNameClaims(raw) {
+  const out = [];
+  for (const field of SEARCH_NAME_FIELDS) {
+    const value = raw?.[field];
+    if (value === null || value === undefined) continue;
+    // An envelope names each value's source; a flat value (alternateNames is
+    // a plain list) carries the sources the card stamps on the field.
+    const stamped = isAttributed(value) ? null : stampedSources(raw, field);
+    for (const claim of attributions(value)) {
+      const values = Array.isArray(claim?.value) ? claim.value : [claim?.value];
+      const sources = stamped ?? [claim?.source];
+      for (const v of values) {
+        if (typeof v !== 'string' || !v.trim()) continue;
+        const text = v.trim();
+        if (field === 'alternateNames' && CODE_LIKE.test(text)) continue;
+        for (const s of sources.length ? sources : [null]) {
+          out.push({ text, field, source: typeof s === 'string' && s ? s : null });
+        }
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * The `nameRefs` table for a set of claims: every `[field, source]` pair that
+ * occurs, sorted (so the bundle is deterministic), and the lookup that turns
+ * a pair into its index.
+ *
+ * @param {Iterable<{field: string, source: string|null}>} claims
+ * @returns {{ table: Array<[string, string|null]>, ref: (field: string, source: string|null) => number }}
+ */
+export function buildRefTable(claims) {
+  const keys = new Set();
+  for (const { field, source } of claims) keys.add(JSON.stringify([field, source ?? null]));
+  const sorted = [...keys].sort();
+  const index = new Map(sorted.map((k, i) => [k, i]));
+  return {
+    table: sorted.map((k) => JSON.parse(k)),
+    ref: (field, source) => {
+      const i = index.get(JSON.stringify([field, source ?? null]));
+      if (i === undefined) throw new Error(`no name ref for [${field}, ${source}] — build the table from the same claims`);
+      return i;
+    },
+  };
+}
+
+/**
+ * The compact search-name list for one manifest entry.
+ *
+ * @param {Array<{text: string, field: string, source: string|null}>} claims
+ *   from searchNameClaims()
+ * @param {{ exclude?: string[], ref: (field: string, source: string|null) => number }} opts
+ *   exclude: names search already reaches (the displayed name, code aliases)
+ * @returns {Array<Array<string|number>>}  [[text, ref, …], …] — empty when
+ *   the card records no name search would not already find
+ */
+export function encodeSearchNames(claims, { exclude = [], ref }) {
+  const reached = new Set(exclude.map(foldForSearch).filter(Boolean));
+  const byFold = new Map();
+  for (const { text, field, source } of claims) {
+    const key = foldForSearch(text);
+    if (!key || reached.has(key)) continue;
+    let entry = byFold.get(key);
+    if (!entry) {
+      entry = { text, refs: [] };
+      byFold.set(key, entry);
+    }
+    // Cited only to the sources that record THIS spelling: a source that wrote
+    // "kweyol" is not quoted as writing "Kwéyòl". The other spelling folds the
+    // same, so search loses nothing by keeping one.
+    if (text !== entry.text) continue;
+    const r = ref(field, source);
+    if (!entry.refs.includes(r)) entry.refs.push(r);
+  }
+  return [...byFold.values()].map((e) => [e.text, ...e.refs]);
+}
+
+/**
+ * Read a manifest entry's search names back, each with its field and sources.
+ *
+ * @param {object} entry  a manifest entry ({ n, a, d, s })
+ * @param {Array<[string, string|null]>} refs  the bundle's `nameRefs`
+ * @returns {Array<{text: string, fields: Array<{field: string, sources: string[]}>}>}
+ *   fields in first-cited order; an unknown ref is skipped, never guessed
+ */
+export function decodeSearchNames(entry, refs) {
+  const out = [];
+  for (const tuple of Array.isArray(entry?.s) ? entry.s : []) {
+    if (!Array.isArray(tuple) || typeof tuple[0] !== 'string') continue;
+    const fields = [];
+    for (const r of tuple.slice(1)) {
+      const pair = Array.isArray(refs) ? refs[r] : undefined;
+      if (!Array.isArray(pair) || typeof pair[0] !== 'string') continue;
+      let f = fields.find((x) => x.field === pair[0]);
+      if (!f) {
+        f = { field: pair[0], sources: [] };
+        fields.push(f);
+      }
+      if (typeof pair[1] === 'string' && pair[1] && !f.sources.includes(pair[1])) f.sources.push(pair[1]);
+    }
+    if (fields.length) out.push({ text: tuple[0], fields });
+  }
+  return out;
+}