champollion 0.3.4 → 0.4.0

package/lib/methods/api.js

@@ -23,7 +23,9 @@
  *     source_locale: "en",
  *     target_locale: "crk",
  *     method: "crk-coached-v1",
- *     keys: { "hero.title": "Welcome", ... }
+ *     keys: { "hero.title": "Welcome", ... },
+ *     instructions: { "hero.title": "…" }   // only when the endpoint declares
+ *                                          // "acceptsInstructions": true
  *   }
  *
  * RESPONSE FORMAT (what the API returns):
@@ -32,6 +34,12 @@
  *     meta: { model, cost_usd, quality_tier, ... }
  *   }
  *
+ * CONTENT (Markdown bodies) uses the SAME contract: the body pieces go out as
+ * keys ("segment.<N>" per block in block mode, "body" in page mode) with the
+ * optional field  text_format: "markdown"  so a server can tell document text
+ * from app strings (servers that do not know the field ignore it). Only the
+ * Markdown leaves — never the LLM instruction prompt the CLI wraps it in.
+ *
  * COST PROFILE: Varies by method — determined server-side
  * QUALITY TIER: Varies by method — read from plugin manifest
  */
@@ -44,10 +52,19 @@ import {
 import { pMap } from '../concurrent.js';
 import { DEFAULT_METHOD_CONCURRENCY } from '../config.js';
 import { output } from '../output.js';
+import { SEGMENT_MARKER_PREFIX, SEGMENT_MARKER_SUFFIX } from '../segment.js';
+import { parseContentPrompt } from './content-separator.js';
+import { getEnvOrFileVar } from '../api-key.js';
+import { isLoopbackEndpoint, localMachineCost } from './http-utils.js';
+import { captureRequest, isCapturing, PREVIEW_KEY } from './request-capture.js';
 
 // Maximum keys per API request (server-side limit)
 const MAX_KEYS_PER_REQUEST = 100;
 
+// The block/page parser lives in content-separator.js (shared with the
+// raw-text engines); api sends the blocks as keys.
+const contentRequest = parseContentPrompt;
+
 class APIMethod extends TranslationMethod {
   constructor(options = {}) {
     super('api', options);
@@ -58,6 +75,14 @@ class APIMethod extends TranslationMethod {
     this.methodVersion = options.methodVersion || null;
     this.qualityTier = options.qualityTier || 'standard';
     this.pluginProvenance = options.provenance || null;
+    // Per-key instructions (plural forms, a quality-gate retry's feedback)
+    // reach the endpoint only when it declares it follows them
+    // ("acceptsInstructions": true on the pair or in the plugin manifest) —
+    // they travel as an "instructions" object beside "keys". false (a trained
+    // NMT model, e.g. nmt-forge serve) and unknown send the text alone.
+    this.declaredInstructions = typeof options.acceptsInstructions === 'boolean' ? options.acceptsInstructions : null;
+    this.acceptsKeyInstructions = this.declaredInstructions === true;
+    this.supportsRequestPreview = true; // its transport reports to request-capture.js
   }
 
   /**
@@ -70,12 +95,13 @@ class APIMethod extends TranslationMethod {
    * @returns {object|null} Map of key → translated value, or null
    */
   async translate(keys, sourceFlat, pairConfig, options) {
-    const apiKey = options.apiKey
-      || process.env.CHAMPOLLION_API_KEY;
+    const endpointForKey = this.endpoint || pairConfig.endpoint || options.endpoint;
+    // A request preview needs no key: it is shown, never sent.
+    const apiKey = resolveApiMethodKey(pairConfig, endpointForKey, options.cwd) || (isCapturing() ? PREVIEW_KEY : null);
 
     if (!apiKey) {
-      output.error('API method: No API key found.');
-      output.error('Set CHAMPOLLION_API_KEY in your environment.');
+      output.error(`API method: no key for ${endpointForKey || 'this endpoint'}.`);
+      output.error('Set CHAMPOLLION_API_KEY, or "apiKey": "${YOUR_VAR}" on the pair.');
       return null;
     }
 
@@ -93,6 +119,9 @@ class APIMethod extends TranslationMethod {
     const sourceLocale = pairConfig.source || 'en';
     const targetLocale = pairConfig.target;
     const method = this.methodName || pairConfig.methodPlugin || 'default';
+    // 'markdown' for content-body pieces (translateContent below); unset for
+    // app strings, so key-value requests are unchanged.
+    const textFormat = options.textFormat || null;
 
     const allTranslated = {};
 
@@ -113,6 +142,17 @@ class APIMethod extends TranslationMethod {
 
       if (Object.keys(keysPayload).length === 0) return;
 
+      // Declared instruction-following endpoints get the per-key notes.
+      let instructions = null;
+      if (this.acceptsKeyInstructions && options.descriptions) {
+        for (const key of Object.keys(keysPayload)) {
+          if (typeof options.descriptions[key] === 'string' && options.descriptions[key]) {
+            instructions = instructions || {};
+            instructions[key] = options.descriptions[key];
+          }
+        }
+      }
+
       const result = await this._translateBatchWithRetry(
         keysPayload,
         sourceLocale,
@@ -121,6 +161,8 @@ class APIMethod extends TranslationMethod {
         endpoint,
         apiKey,
         idx + 1,
+        textFormat,
+        instructions,
       );
 
       if (result) {
@@ -132,14 +174,39 @@ class APIMethod extends TranslationMethod {
   }
 
   /**
-   * Freeform content translation via the API.
+   * Content (Markdown body) translation via the API.
    *
-   * The API method could support this in the future, but for now
-   * we only support key-value translation. Return null so the
-   * orchestrator falls back to the local LLM method.
+   * The content lane passes the prompt it built for an LLM (see
+   * contentRequest above). The Markdown inside it is sent over the same
+   * key → string contract with text_format "markdown", and the answer is
+   * rebuilt in the shape the content lane parses: one ⟦SEG_N⟧ marker per
+   * block (a block the server did not return is LEFT OUT, so the lane's
+   * self-repair ladder retries it and then marks it '[EN]' — visible, never
+   * silent), or the translated body in page mode.
+   *
+   * (This used to return null unconditionally "so the orchestrator falls
+   * back to the local LLM method" — no such fallback exists: every content
+   * file failed with "block-batch translation returned no results" and no
+   * reason. Synthetic users, 2026-10.)
    */
-  async translateContent(_prompt, _pairConfig, _options) {
-    return null;
+  async translateContent(prompt, pairConfig, options = {}) {
+    const request = contentRequest(prompt);
+    if (!request) {
+      output.error('API method: this content prompt carries neither ⟦SEG_N⟧ blocks nor a "---" body separator — nothing to send.');
+      return null;
+    }
+    const names = Object.keys(request.keys).filter((k) => request.keys[k].trim());
+    if (names.length === 0) return null;
+    const result = await this.translate(names, request.keys, pairConfig,
+      { ...options, textFormat: 'markdown' });
+    if (!result) return null;
+    if (request.mode === 'page') {
+      return typeof result.body === 'string' ? result.body : null;
+    }
+    const parts = request.ids
+      .filter((id) => typeof result[`segment.${id}`] === 'string')
+      .map((id) => `${SEGMENT_MARKER_PREFIX}${id}${SEGMENT_MARKER_SUFFIX}\n${result[`segment.${id}`]}`);
+    return parts.length > 0 ? parts.join('\n\n') : null;
   }
 
   /**
@@ -152,10 +219,26 @@ class APIMethod extends TranslationMethod {
    * @param {string} endpoint - API endpoint URL
    * @param {string} apiKey - Remote API key
    * @param {number} batchNum - Batch number for logging
+   * @param {string|null} [textFormat] - 'markdown' for content-body pieces
    * @returns {object|null} Map of key → translated value
    */
-  async _translateBatchWithRetry(keysPayload, sourceLocale, targetLocale, method, endpoint, apiKey, batchNum) {
+  async _translateBatchWithRetry(keysPayload, sourceLocale, targetLocale, method, endpoint, apiKey, batchNum, textFormat = null, instructions = null) {
     const keyCount = Object.keys(keysPayload).length;
+    const headers = {
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+      'User-Agent': 'champollion',
+    };
+    const body = {
+      source_locale: sourceLocale,
+      target_locale: targetLocale,
+      method,
+      keys: keysPayload,
+      ...(textFormat ? { text_format: textFormat } : {}),
+      ...(instructions ? { instructions } : {}),
+    };
+    // `sync --dry --show-prompt`: hand over the exact request, send nothing.
+    if (captureRequest({ url: endpoint, headers, body })) return null;
 
     for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
       try {
@@ -164,17 +247,8 @@ class APIMethod extends TranslationMethod {
 
         const response = await fetch(endpoint, {
           method: 'POST',
-          headers: {
-            'Authorization': `Bearer ${apiKey}`,
-            'Content-Type': 'application/json',
-            'User-Agent': 'champollion',
-          },
-          body: JSON.stringify({
-            source_locale: sourceLocale,
-            target_locale: targetLocale,
-            method,
-            keys: keysPayload,
-          }),
+          headers,
+          body: JSON.stringify(body),
           signal: controller.signal,
         });
 
@@ -286,7 +360,12 @@ class APIMethod extends TranslationMethod {
    * Cost estimation — API method pricing is determined by the remote server.
    * We cannot estimate cost without querying the endpoint.
    */
-  estimateCost(keyCount) {
+  estimateCost(keyCount, pairConfig = {}) {
+    // An endpoint on this machine (nmt-forge serve, champollion serve on
+    // 127.0.0.1) has no API bill — say $0, and why. Anything else is priced
+    // by its server: unknown here, never $0.
+    const endpoint = pairConfig?.endpoint || this.endpoint;
+    if (isLoopbackEndpoint(endpoint)) return localMachineCost(endpoint);
     return {
       estimatedCost: null,
       currency: 'USD',
@@ -313,4 +392,38 @@ class APIMethod extends TranslationMethod {
   }
 }
 
-export { APIMethod };
+
+const LOOPBACK = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);
+
+/**
+ * The bearer token for an `api` endpoint — and ONLY a token meant for it.
+ *
+ *   1. the pair's own "apiKey": "${VAR}" (read from the environment or
+ *      .env.local/.env), or a literal value;
+ *   2. CHAMPOLLION_API_KEY;
+ *   3. for a loopback endpoint (nmt-forge serve, champollion serve) with no
+ *      token configured, a placeholder: those servers need none.
+ *
+ * It used to fall back to `options.apiKey` — the generic provider key sync
+ * resolves for the llm method — so a user's OPENROUTER_API_KEY was sent as
+ * the Bearer token to whatever endpoint a pair named (found 2026-10-03). The
+ * documented per-pair "apiKey" was never read at all.
+ *
+ * @returns {string|null}
+ */
+function resolveApiMethodKey(pairConfig, endpoint, cwd) {
+  const own = pairConfig && typeof pairConfig.apiKey === 'string' ? pairConfig.apiKey.trim() : '';
+  if (own) {
+    const ref = /^\$\{([A-Za-z_][A-Za-z0-9_]*)\}$/.exec(own);
+    if (!ref) return own;
+    const v = getEnvOrFileVar(ref[1], cwd);
+    if (v) return v;
+  }
+  const shared = getEnvOrFileVar('CHAMPOLLION_API_KEY', cwd);
+  if (shared) return shared;
+  let host = '';
+  try { host = new URL(endpoint).hostname; } catch { /* no endpoint */ }
+  return LOOPBACK.has(host) ? 'local-no-token' : null;
+}
+
+export { APIMethod, resolveApiMethodKey };

package/lib/methods/base.js

@@ -33,6 +33,23 @@ class TranslationMethod {
   constructor(name, options = {}) {
     this.name = name;
     this.options = options;
+    // True for machine-translation engines that translate raw text and do
+    // not understand an LLM instruction prompt (Google, DeepL, Microsoft,
+    // LibreTranslate, Apertium, Tilde, Translated). translateRawContent sends
+    // them one Markdown block at a time instead of the block-batch prompt.
+    this.translatesRawText = false;
+    // True for methods that read per-key instructions (the "UI context"
+    // lines of the prompt): which plural form a generated i18next key needs,
+    // which CLDR categories an ICU plural must cover, a gettext msgctxt, a
+    // quality-gate retry's feedback. Only LLM methods do; a machine
+    // translation engine translates the text and nothing else, so sync must
+    // not claim it was asked for, e.g., French's "many" form.
+    this.acceptsKeyInstructions = false;
+    // True for methods whose transport hands its finished request to
+    // lib/methods/request-capture.js instead of sending it while a capture
+    // is on (`sync --dry --show-prompt`). Only those are previewed: running
+    // any other method's translate() under a preview could reach its API.
+    this.supportsRequestPreview = false;
   }
 
   /**

package/lib/methods/coaching-data.js

@@ -0,0 +1,153 @@
+/**
+ * The project's coaching file (.champollion/coaching/<locale>.json) and its
+ * glossary — read in ONE place for every method that uses them.
+ *
+ * THE FILE:
+ *   {
+ *     "grammar_rules": ["French adjectives agree in gender and number…"],
+ *     "dictionary":    { "dashboard": "tableau de bord" },
+ *     "style_notes":   "Prefer active voice."
+ *   }
+ *
+ * WHO READS WHAT:
+ *   - "dictionary" is the project's GLOSSARY. Every LLM method is told the
+ *     glossary terms a batch contains (lib/methods/llm.js buildUserMessage),
+ *     DeepL sends it as a glossary, and sync checks every method's output
+ *     against it.
+ *   - "grammar_rules" and "style_notes" are COACHING, read by the
+ *     llm-coached method only (on any provider). The plain LLM methods
+ *     (llm, openai, anthropic, gemini, local) build the same prompt as each
+ *     other, so a pair gets the same instructions whichever of them runs it.
+ *
+ * WHY A MODULE OF ITS OWN: lib/methods/llm.js needs the glossary helpers and
+ * lib/methods/llm-coached.js imports llm.js — keeping these here avoids an
+ * import cycle. llm-coached.js re-exports them (the public names are unchanged).
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { output } from '../output.js';
+
+/**
+ * Default coaching data directory, relative to project root.
+ */
+const DEFAULT_COACHING_DIR = '.champollion/coaching';
+
+/**
+ * Load coaching data for a locale from a JSON file, with caching.
+ *
+ * @param {string} coachingDir - Path to coaching data directory
+ * @param {string} locale - Target locale code (e.g., 'fr', 'crk')
+ * @param {Map} cache - Cache map to store loaded data (avoids re-reading files)
+ * @returns {object|null} Coaching data { grammar_rules, dictionary, style_notes }, or null
+ */
+function loadCoachingData(coachingDir, locale, cache) {
+  if (!locale) return null;
+
+  const cacheKey = `${coachingDir}:${locale}`;
+  if (cache.has(cacheKey)) {
+    return cache.get(cacheKey);
+  }
+
+  const filePath = path.join(coachingDir, `${locale}.json`);
+
+  if (!fs.existsSync(filePath)) {
+    cache.set(cacheKey, null);
+    return null;
+  }
+
+  try {
+    const raw = fs.readFileSync(filePath, 'utf-8');
+    const data = JSON.parse(raw);
+
+    // Validate required structure — normalize missing fields to safe defaults
+    const coaching = {
+      grammar_rules: Array.isArray(data.grammar_rules) ? data.grammar_rules : [],
+      dictionary: (data.dictionary && typeof data.dictionary === 'object') ? data.dictionary : {},
+      style_notes: typeof data.style_notes === 'string' ? data.style_notes : '',
+    };
+
+    cache.set(cacheKey, coaching);
+    return coaching;
+  } catch (err) {
+    output.warn(`Failed to load coaching data: ${filePath}`);
+    output.warn(err.message);
+    cache.set(cacheKey, null);
+    return null;
+  }
+}
+
+/**
+ * The project glossary for a pair: the term → translation map the pair is
+ * told about and checked against. sync loads it once for every pair
+ * (pairConfig.glossary, null when there is none); a caller that did not (a
+ * library user, `serve`) gets it from the coaching file in `cwd`.
+ *
+ * @param {object} pairConfig
+ * @param {{ cwd?: string }} [options]
+ * @param {Map} [cache] - loadCoachingData cache
+ * @returns {Object<string, string>|null}
+ */
+function projectGlossary(pairConfig, options = {}, cache = new Map()) {
+  // Set by sync (null = the project has none for this locale).
+  if (pairConfig && Object.prototype.hasOwnProperty.call(pairConfig, 'glossary')) {
+    const g = pairConfig.glossary;
+    return g && typeof g === 'object' && Object.keys(g).length > 0 ? g : null;
+  }
+  const target = pairConfig && (pairConfig.target || pairConfig.locale);
+  if (!target) return null;
+  const data = loadCoachingData(path.join(options.cwd || process.cwd(), DEFAULT_COACHING_DIR), target, cache);
+  return data && Object.keys(data.dictionary).length > 0 ? data.dictionary : null;
+}
+
+/**
+ * Scan source values for dictionary term matches.
+ *
+ * Case-insensitive substring matching of each term against the batch's
+ * source values (the dictionary is usually small).
+ *
+ * @param {object} toTranslate - Key-value map to scan
+ * @param {object} dictionary - Term → translation map
+ * @returns {Array<{ term: string, translation: string }>} Matched hints
+ */
+function findDictionaryMatches(toTranslate, dictionary) {
+  if (!dictionary || Object.keys(dictionary).length === 0) return [];
+
+  const matches = [];
+  const seen = new Set();
+  const values = Object.values(toTranslate).join(' ').toLowerCase();
+
+  for (const [term, translation] of Object.entries(dictionary)) {
+    if (seen.has(term)) continue;
+    if (values.includes(term.toLowerCase())) {
+      matches.push({ term, translation });
+      seen.add(term);
+    }
+  }
+
+  return matches;
+}
+
+/**
+ * The REQUIRED TERMINOLOGY block for a batch: the glossary terms its source
+ * values contain ('' when none).
+ *
+ * @param {object} toTranslate - Key-value map of the batch
+ * @param {Object<string, string>|null} glossary
+ * @returns {string}
+ */
+function terminologyBlock(toTranslate, glossary) {
+  const hints = findDictionaryMatches(toTranslate, glossary);
+  if (hints.length === 0) return '';
+  return 'REQUIRED TERMINOLOGY (use these exact translations):\n'
+    + hints.map(h => `  • "${h.term}" → "${h.translation}"`).join('\n')
+    + '\n\n';
+}
+
+export {
+  DEFAULT_COACHING_DIR,
+  loadCoachingData,
+  projectGlossary,
+  findDictionaryMatches,
+  terminologyBlock,
+};

package/lib/methods/content-separator.js

@@ -20,6 +20,49 @@
  */
 export const CONTENT_SEPARATOR = '\n---\n';
 
+import { SEGMENT_MARKER_PREFIX, SEGMENT_MARKER_SUFFIX } from '../segment.js';
+
+const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+// One block-batch segment marker on its own line (segment.js
+// buildBlockBatchPrompt writes "⟦SEG_N⟧\n<block>" separated by blank lines).
+const SEGMENT_LINE = new RegExp(
+  `^${escapeRegExp(SEGMENT_MARKER_PREFIX)}(\\d+)${escapeRegExp(SEGMENT_MARKER_SUFFIX)}[ \\t]*$`,
+  'gm',
+);
+
+/**
+ * The Markdown a content prompt carries, without the LLM instructions.
+ *
+ * The content lane builds prompts for an LLM: the block-batch prompt
+ * (instructions, then one ⟦SEG_N⟧ marker line per block) or the page prompt
+ * (instructions, "---", the body). A method that is not an LLM must send
+ * only the Markdown — never the instructions.
+ *
+ * @param {string} prompt
+ * @returns {{ mode: 'segments', keys: object, ids: string[] } | { mode: 'page', keys: object } | null}
+ */
+export function parseContentPrompt(prompt) {
+  const text = String(prompt);
+  const marks = [...text.matchAll(SEGMENT_LINE)];
+  if (marks.length > 0) {
+    const keys = {};
+    marks.forEach((m, i) => {
+      const end = i + 1 < marks.length ? marks[i + 1].index : text.length;
+      keys[`segment.${m[1]}`] = text
+        .slice(m.index + m[0].length, end)
+        .replace(/^\r?\n/, '')
+        .replace(/[\r\n]+\s*$/, '');
+    });
+    return { mode: 'segments', keys, ids: marks.map((m) => m[1]) };
+  }
+  const sep = text.indexOf(CONTENT_SEPARATOR);
+  if (sep !== -1) {
+    return { mode: 'page', keys: { body: text.slice(sep + CONTENT_SEPARATOR.length) } };
+  }
+  return null;
+}
+
 /**
  * Extract the Markdown body from a content prompt.
  *

package/lib/methods/deepl.js

@@ -19,6 +19,7 @@ const DEEPL_REQUEST_TIMEOUT_MS = 15000;
 class DeepLMethod extends TranslationMethod {
   constructor(options = {}) {
     super('deepl', options);
+    this.translatesRawText = true; // see base.js
     this._coachingCache = new Map();
   }
 
@@ -31,7 +32,6 @@ class DeepLMethod extends TranslationMethod {
    */
   _resolveApiKey(options) {
     return options.deeplApiKey
-      || getEnvOrFileVar('DEEPL_API_KEY')
       || getEnvOrFileVar('DEEPL_API_KEY', options.cwd);
   }