champollion 0.3.4 → 0.4.0

package/lib/commercial-eligibility.js

@@ -189,7 +189,7 @@ export class CommercialRouteBlockedError extends Error {
       + `${verdict.reason}`
       + (verdict.license ? ` (license: ${verdict.license})` : '')
       + '. Route it in the non-commercial lane, or use a method whose license '
-      + 'permits commercial use (`champollion recommend <src> <tgt> '
+      + 'permits commercial use (`champollion network recommend <src> <tgt> '
       + '--use commercial` lists them).'
     );
     this.name = 'CommercialRouteBlockedError';

package/lib/config.js

@@ -16,6 +16,10 @@ import path from 'node:path';
 import { DEFAULT_REGISTERS, getLanguageCard, getRegister, resolveCode } from './registers.js';
 import { resolveModel } from './models.js';
 import { validateNoTranslateConfig } from './no-translate.js';
+import { output } from './output.js';
+import { splitKeyList } from './redo.js';
+import { LOCALE_FILE_FORMATS } from './format.js';
+import { discoverLocaleLayout, compileLocalesPattern, LAYOUT_KINDS, FORMAT_BY_EXT } from './locale-layout.js';
 
 const CONFIG_FILENAMES = ['champollion.config.json'];
 
@@ -67,13 +71,32 @@ const DEFAULTS = {
   inputLocale: 'en',
   baseUrl: '',
   localesDir: './locales',
-  contentDir: null,  // Hugo content directory (e.g. './content'). null = disabled.
+  // Where every locale's key-value files live when the default shapes do
+  // not fit (lib/locale-layout.js): a path with {lang} and optional {ns},
+  // e.g. "public/locales/{lang}/{ns}.json". Relative to the project root.
+  // When set, it fixes localesDir (the part before the first placeholder).
+  localesPattern: null,
+  // Force the layout auto-detection's answer: 'flat' (<dir>/<lang>.json)
+  // or 'dir' (<dir>/<lang>/<ns>.json). null = detect from the source files.
+  localesLayout: null,
+  // Namespace `wrap` adds extracted keys to in a folder-per-locale project
+  // with several namespace files (e.g. "common"). null = the only file.
+  defaultNamespace: null,
+  contentDir: null,  // Folder of Markdown/MDX to translate (e.g. './content', './newsletters'). null = disabled.
   // Markdown body translation granularity: 'block' (default — segment the
   // body, TM-cache per block, one batched API call for the misses) or
   // 'page' (single whole-body prompt, still TM-threaded). Overridable per
   // pair. Validated in docusaurus-sync.js — anything else fails loud.
   contentSegmentation: 'block',
   promptContext: null, // Global context injected into all translation prompts (e.g. "This is a developer tool README")
+  // Gender guidance in LLM prompts: null = each language's guidance from the
+  // catalogue (French: écriture inclusive, "Connecté·e"); false = none; a
+  // string = your own instruction. Overridable per language and per pair.
+  genderGuidance: null,
+  // Names kept exactly as written in every language (people, companies,
+  // products). Told to the model, and exempt from the wrong-script /
+  // untranslated checks. Value-level, unlike noTranslate (which skips KEYS).
+  protectedTerms: [],
   translatableFields: null,  // Override DEFAULT_TRANSLATABLE_FIELDS from content.js
   languages: [],
   // Keys whose correct translation is the source value, verbatim: dot-paths
@@ -89,6 +112,11 @@ const DEFAULTS = {
   pairs: null,       // Advanced per-pair overrides (see pairs.js)
   model: DEFAULT_OPENROUTER_MODEL,
   defaultMethod: 'llm',     // Global default: llm, llm-coached, google-translate, api, deepl, microsoft-translator, libretranslate, openai, anthropic, gemini
+  // Transport for the LLM lane (llm / llm-coached pairs): openrouter (the
+  // default when null), openai, anthropic, gemini, local. Harness
+  // `export-config` emits it for runs validated on a direct provider.
+  // Overridable per language and per pair; validated in pairs.js.
+  provider: null,
   batchSize: DEFAULT_BATCH_SIZE,
   temperature: null,  // null = use method default (0.3 standard, 0.2 coached)
   coachingFile: null,  // Path to free-text coaching prompt file (relative to cwd)
@@ -190,11 +218,12 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
         batch: 'batchSize',
         key: 'apiKeyEnvVar',
         apiKey: 'apiKeyEnvVar',
-        provider: 'defaultMethod',
         concurrency: 'concurrency',  // valid — not in DEFAULTS but consumed by sync
         skipKeys: 'noTranslate',     // canonicalized above; alias kept for the hint
         noTranslateURLs: 'noTranslateUrls',
         neverTranslate: 'noTranslate',
+        doNotTranslate: 'protectedTerms',
+        keepVerbatim: 'protectedTerms',
       };
 
       // Warn on unknown config fields — prevents silent acceptance of
@@ -208,20 +237,36 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
         if (!knownFields.has(key)) {
           const suggestion = FIELD_ALIASES[key];
           if (suggestion) {
-            console.warn(`[WARN] Unknown config field "${key}" — did you mean "${suggestion}"?`);
+            output.warn(`Unknown config field "${key}" — did you mean "${suggestion}"?`);
           } else {
-            console.warn(`[WARN] Unknown config field "${key}" in ${path.basename(configPath)} — this field has no effect. Check spelling or see docs for supported fields.`);
+            output.warn(`Unknown config field "${key}" in ${path.basename(configPath)} — this field has no effect. Check spelling or see docs for supported fields.`);
           }
         }
       }
 
       Object.assign(config, fileConfig);
+      // localesPattern fixes localesDir; a separately WRITTEN localesDir
+      // that disagrees with it is checked below, once paths are resolved.
+      config._localesDirExplicit = fileConfig.localesDir != null;
+      // Whether the top-level model was WRITTEN (vs. the built-in OpenRouter
+      // default). pairs.js needs the difference: a top-level `model` is sent
+      // to a top-level direct `provider` only when someone named it — the
+      // OpenRouter default slug must never reach OpenAI or Ollama.
+      config._modelExplicit = fileConfig.model != null;
 
       // Validate the no-translate fields against the file that set them, so
       // the error names the right place. A bad value must never fall through
       // to "translate everything" — that is the corruption path the feature
       // exists to close.
       validateNoTranslateConfig(config.noTranslate, config.noTranslateUrls);
+      if (!Array.isArray(config.protectedTerms)
+          || config.protectedTerms.some(t => typeof t !== 'string' || t.trim() === '')) {
+        const e = new Error(
+          '"protectedTerms" must be an array of non-empty strings — the names to keep as written, '
+          + 'e.g. "protectedTerms": ["Curtis Forbes", "Game Day Suits"].');
+        e.code = 'CHAMPOLLION_CONFIG_INVALID';
+        throw e;
+      }
     } catch (err) {
       // A field-level validation error is already specific and actionable —
       // wrapping it in "Could not parse" would misreport valid JSON as a
@@ -253,9 +298,49 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
 
   // Layer 3: CLI overrides
   if (cliArgs.source) config.inputLocale = cliArgs.source;
-  if (cliArgs.dir) config.localesDir = cliArgs.dir;
-  if (cliArgs.model) config.model = cliArgs.model;
-  if (cliArgs.method) config.defaultMethod = cliArgs.method;
+  if (cliArgs.dir) {
+    config.localesDir = cliArgs.dir;
+    config._localesDirExplicit = true;
+  }
+  // A pair's `fallback` is NOT touched by --model / --method (they override
+  // the pair's own method only), so pairs.js resolves fallbacks against the
+  // model and default method the FILE set. Recorded before the flags apply.
+  config._fileModel = config.model;
+  config._fileModelExplicit = !!config._modelExplicit;
+  config._fileDefaultMethod = config.defaultMethod;
+  if (cliArgs.model) {
+    config.model = cliArgs.model;
+    config._modelExplicit = true;
+    // Like --method: the flag beats a pair's own model for this run (how you
+    // try or price a switch before committing to it), and says so.
+    config._modelOverride = cliArgs.model;
+    const pinned = Object.entries(config.pairs || {})
+      .filter(([, p]) => p && p.model && p.model !== cliArgs.model)
+      .map(([key, p]) => `${key} (${p.model})`);
+    if (pinned.length > 0) {
+      output.info(`--model ${cliArgs.model} overrides the configured model for this run: ${pinned.join(', ')}. Use --pair to limit it.`);
+    }
+  }
+  if (cliArgs.method) {
+    config.defaultMethod = cliArgs.method;
+    // A flag beats the file for this run — including a pair's own `method`.
+    // It used to set only the default, so `sync --method local` (what the
+    // forge export's DEPLOY.md says to run) silently kept a pair configured
+    // for llm. Say which pairs it overrides; --pair scopes it.
+    config._methodOverride = cliArgs.method;
+    const overridden = [];
+    for (const [key, p] of Object.entries(config.pairs || {})) {
+      if (p && p.method && p.method !== cliArgs.method) overridden.push(`${key} (${p.method})`);
+    }
+    if (config.languages && !Array.isArray(config.languages)) {
+      for (const [code, l] of Object.entries(config.languages)) {
+        if (l && l.method && l.method !== cliArgs.method) overridden.push(`${code} (${l.method})`);
+      }
+    }
+    if (overridden.length > 0) {
+      output.info(`--method ${cliArgs.method} overrides the configured method for this run: ${overridden.join(', ')}. Use --pair to limit it.`);
+    }
+  }
   // --batch-size: keys per translation API call. Accept both the CLI flag
   // spelling ('batch-size' from bin/cli.js parseArgs) and the programmatic
   // camelCase form (watch mode / tests pass cliArgs objects directly).
@@ -290,10 +375,9 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
     config.contentConcurrency = parseConcurrency(cliArgs['content-concurrency'], '--content-concurrency');
   }
 
-  // Parse --force-keys: comma-separated dot-notation keys to force re-translate
-  config.forceKeys = cliArgs['force-keys']
-    ? cliArgs['force-keys'].split(',').map(k => k.trim()).filter(Boolean)
-    : [];
+  // Parse --force-keys: comma-separated keys to force re-translate (`\,` is a
+  // comma inside a key — gettext msgids are sentences)
+  config.forceKeys = cliArgs['force-keys'] ? splitKeyList(cliArgs['force-keys']) : [];
 
   // Docusaurus auto-detection: if format is still 'auto' and docusaurus.config.js
   // exists in the project root, switch to 'docusaurus' mode and use the standard
@@ -308,12 +392,60 @@ function resolveConfig(cliArgs = {}, cwd = process.cwd()) {
     }
   }
 
+  // `format` must name something a reader exists for. An unknown value used
+  // to fall through every `format === ...` branch and parse the files as
+  // JSON — "jsn" or "yml" produced confusing parse errors (or, for TOML,
+  // garbage) instead of a clear answer.
+  const validFormats = ['auto', 'docusaurus', ...LOCALE_FILE_FORMATS];
+  if (!validFormats.includes(config.format)) {
+    const hint = config.format === 'yml' ? ' (use "yaml" — .yml files are read as YAML)' : '';
+    const e = new Error(
+      `Unknown "format": ${JSON.stringify(config.format)}${hint}. `
+      + `Supported: ${validFormats.join(', ')}.`);
+    e.code = 'CHAMPOLLION_CONFIG_INVALID';
+    throw e;
+  }
+
+  if (config.localesLayout != null && !LAYOUT_KINDS.includes(config.localesLayout)) {
+    const e = new Error(
+      `"localesLayout" must be one of ${LAYOUT_KINDS.join(', ')} (got ${JSON.stringify(config.localesLayout)}). `
+      + 'For any other shape, set "localesPattern", e.g. "public/locales/{lang}/{ns}.json".');
+    e.code = 'CHAMPOLLION_CONFIG_INVALID';
+    throw e;
+  }
+
   // Resolve localesDir and contentDir to absolute paths
   config.localesDir = path.resolve(cwd, config.localesDir);
   if (config.contentDir) {
     config.contentDir = path.resolve(cwd, config.contentDir);
   }
 
+  // localesPattern: absolute, validated now (a malformed pattern must stop
+  // every command, not surface as "0 files" mid-sync), and the source of
+  // truth for localesDir — every path-containment check uses localesDir.
+  if (config.localesPattern != null) {
+    if (config.format === 'docusaurus') {
+      const e = new Error('"localesPattern" does not apply to Docusaurus projects — their locale files live in i18n/<locale>/.');
+      e.code = 'CHAMPOLLION_CONFIG_INVALID';
+      throw e;
+    }
+    if (config.localesLayout != null) {
+      const e = new Error('Set either "localesPattern" or "localesLayout", not both — a pattern already fixes the layout.');
+      e.code = 'CHAMPOLLION_CONFIG_INVALID';
+      throw e;
+    }
+    const compiled = compileLocalesPattern(path.resolve(cwd, String(config.localesPattern)));
+    if (config._localesDirExplicit && path.resolve(config.localesDir) !== compiled.base) {
+      const e = new Error(
+        `"localesDir" (${config.localesDir}) disagrees with "localesPattern" (files under ${compiled.base}). `
+        + 'The pattern already says where locale files live — remove "localesDir".');
+      e.code = 'CHAMPOLLION_CONFIG_INVALID';
+      throw e;
+    }
+    config.localesPattern = path.resolve(cwd, String(config.localesPattern));
+    config.localesDir = compiled.base;
+  }
+
   // Resolve model alias (e.g., "gemini-flash" → "google/gemini-3.5-flash")
   config.model = resolveModel(config.model);
 
@@ -407,10 +539,30 @@ function resolveLanguages(config) {
           formalitySystem: card?.formality?.system || null,
           ...(value.method && { method: value.method }),
           ...(value.model && { model: value.model }),
+          ...(value.provider != null && { provider: value.provider }),
           ...(value.batchSize && { batchSize: value.batchSize }),
           ...(value.maxRetries != null && { maxRetries: value.maxRetries }),
           ...(value.script && { script: value.script }),
           ...(value.scriptFallback && { scriptFallback: value.scriptFallback }),
+          // The remaining fields pairs.js reads from a language. They used
+          // to be dropped here, so `{ "method": "api", "endpoint": … }` in
+          // the object form ran with no endpoint, and the documented
+          // temperature / coachingFile / promptContext had no effect.
+          ...(value.endpoint && { endpoint: value.endpoint }),
+          ...(value.apiKey && { apiKey: value.apiKey }),
+          // An `api` endpoint's declared instruction capability (lib/methods/api.js).
+          ...(typeof value.acceptsInstructions === 'boolean' && { acceptsInstructions: value.acceptsInstructions }),
+          ...(value.temperature != null && { temperature: value.temperature }),
+          ...(value.coachingFile != null && { coachingFile: value.coachingFile }),
+          ...(value.coachingPrompt != null && { coachingPrompt: value.coachingPrompt }),
+          ...(value.promptContext != null && { promptContext: value.promptContext }),
+          ...(value.contentSegmentation != null && { contentSegmentation: value.contentSegmentation }),
+          // Gender guidance in the prompt: false = none, a string = your own
+          // (absent: the catalogue's for the language — lib/pairs.js).
+          ...((value.genderGuidance === false || typeof value.genderGuidance === 'string') && { genderGuidance: value.genderGuidance }),
+          // A second method for what this language's own method cannot
+          // translate safely — resolved and validated in pairs.js.
+          ...(value.fallback !== undefined && { fallback: value.fallback }),
         };
       }
     }
@@ -422,7 +574,8 @@ function resolveLanguages(config) {
 
 /**
  * Auto-detect target languages by scanning the locales directory
- * for locale files (JSON, TOML, or YAML) that aren't the source file.
+ * for locale files (every extension the layout module reads — JSON, TOML,
+ * YAML, gettext .po, Flutter .arb) that aren't the source file.
  *
  * @param {import('./types.js').ChampollionConfig} config - Resolved config
  * @returns {Object<string, import('./types.js').LanguageConfig & { filename: string }>} Map of locale code → language config with filename
@@ -431,10 +584,39 @@ function autoDetectLanguages(config) {
   const detected = {};
   const inputLocale = config.inputLocale || 'en';
 
+  // Folder-per-locale and pattern layouts: the locales are what the layout
+  // finds on disk — sub-folders of localesDir (stray files at its root are
+  // not locales) or every {lang} the pattern matches. Exact code match, so
+  // en-GB is a target when the source is en.
+  const sourceDirExists = (() => {
+    try { return fs.statSync(path.join(config.localesDir, inputLocale)).isDirectory(); } catch { return false; }
+  })();
+  if (config.format !== 'docusaurus'
+      && (config.localesPattern || config.localesLayout === 'dir' || sourceDirExists)) {
+    const layout = discoverLocaleLayout(config);
+    if (layout.kind !== 'flat') {
+      for (const code of layout.listLocales()) {
+        const canonical = resolveCode(code);
+        const card = getLanguageCard(canonical);
+        detected[code] = {
+          name: card?.name || code,
+          register: getRegister(canonical),
+          registerPreset: card?.formality?.default || null,
+          dir: card?.dir || 'ltr',
+          formalitySystem: card?.formality?.system || null,
+          filename: layout.filesFor(code)[0]?.rel ?? code,
+        };
+      }
+      return detected;
+    }
+  }
+
   if (!fs.existsSync(config.localesDir)) return detected;
 
-  // Supported locale file extensions
-  const LOCALE_EXTS = ['.json', '.toml', '.yaml', '.yml'];
+  // Locale file extensions — the layout module's own list (lib/locale-layout.js
+  // FORMAT_BY_EXT), so a format the reader learns is a format detection
+  // finds. A gettext template (.pot) is not a locale and is not listed there.
+  const LOCALE_EXTS = Object.keys(FORMAT_BY_EXT);
 
   const files = fs.readdirSync(config.localesDir)
     .filter(f => {

package/lib/content-estimate.js

@@ -0,0 +1,96 @@
+/**
+ * content-estimate.js — what a content file will actually BILL.
+ *
+ * The cost gate (--max-cost) must price what the run will pay for, not the
+ * whole file: front-matter fields and body blocks the Translation Memory
+ * already holds are served at $0 by the sync. Pricing whole files made a
+ * $100 cap refuse runs whose real cost was ~$30, and operators raised the
+ * cap past the honest number to get work done (dogfood 2026-08-28, finding
+ * 1) — the guardrail stopped guarding.
+ *
+ * This walks the SAME ladder both content lanes run at sync time:
+ *   1. each front-matter field, cached on its own source text;
+ *   2. the whole body (a revert or lock-loss re-run is free);
+ *   3. in 'block' segmentation, each translatable block.
+ * One implementation for the Hugo and Docusaurus estimators, so the two can
+ * no longer disagree about what is billable.
+ *
+ * Pure lookups (lookupTM, never the evicting lookupTMValidated): an
+ * estimate must not change the cache it is estimating against.
+ */
+
+import { lookupTM } from './tm.js';
+import { splitBlocks } from './segment.js';
+import { protectBlocks, restoreBlocks } from './content.js';
+
+/**
+ * Restored source texts of a body's translatable blocks — the exact unit
+ * block-mode sync caches and bills.
+ *
+ * @param {string} body - Markdown body (front matter stripped)
+ * @returns {string[]}
+ */
+export function translatableBlockSources(body) {
+  const { protectedBody, blocks } = protectBlocks(body);
+  return splitBlocks(protectedBody)
+    .filter(seg => seg.type === 'translatable')
+    .map(seg => restoreBlocks(seg.text, blocks));
+}
+
+/**
+ * Billable vs total source characters for one (file × locale) work item.
+ *
+ * @param {object} args
+ * @param {object} args.tm - Loaded TM (carry-over settings apply)
+ * @param {string} args.code - Target locale
+ * @param {string} args.tmKey - tmMethodKey(pairConfig)
+ * @param {string|null} [args.fallbackTmKey] - tmMethodKey(pairConfig.fallback)
+ *   when the pair has a fallback: the sync serves the fallback's cached text
+ *   before asking the primary (lib/fallback.js), so it is free here too
+ * @param {Record<string, string>} args.fields - Front-matter fields to translate
+ * @param {string} args.body - Markdown body
+ * @param {'block'|'page'} args.segMode - Content segmentation mode
+ * @param {() => string[]} args.blockSources - Lazily computed block sources
+ *   (callers cache per source file: segmentation is locale-independent)
+ * @param {ReturnType<import('./content-refusals.js').contentHolds>|null} [args.holds] -
+ *   What the quality gate refused before (lib/content-refusals.js): a block
+ *   or field the pair's method refused is not sent to it (held, or the
+ *   fallback's — which this estimate does not price); a held front-matter
+ *   field — or a held page body in 'page' segmentation — holds its whole
+ *   page, so nothing of it is billed
+ * @returns {{ billedChars: number, totalChars: number }}
+ */
+export function billableContentChars({ tm, code, tmKey, fallbackTmKey = null, fields, body, segMode, blockSources, holds = null }) {
+  let billedChars = 0;
+  let totalChars = 0;
+  let pageHeld = false;
+  const cached = (text) => lookupTM(tm, text, code, tmKey) !== null
+    || (fallbackTmKey !== null && lookupTM(tm, text, code, fallbackTmKey) !== null);
+
+  for (const [field, text] of Object.entries(fields)) {
+    totalChars += text.length;
+    if (cached(text)) continue;
+    const hold = holds ? holds.field(field, text) : 'send';
+    if (hold === 'held') pageHeld = true;
+    else if (hold === 'send') billedChars += text.length;
+  }
+
+  if (body.trim()) {
+    totalChars += body.length;
+    if (!cached(body)) {
+      if (segMode === 'page') {
+        // A page translated whole and refused before: held (its page is
+        // not processed — nothing of it is billed), or the fallback's.
+        const hold = holds ? holds.page(body) : 'send';
+        if (hold === 'held') pageHeld = true;
+        else if (hold === 'send') billedChars += body.length;
+      } else {
+        for (const source of blockSources()) {
+          if (!cached(source) && (!holds || holds.block(source) === 'send')) billedChars += source.length;
+        }
+      }
+    }
+  }
+
+  return { billedChars: pageHeld ? 0 : billedChars, totalChars };
+}