champollion 0.3.3

package/lib/methods/translation-error.js

@@ -0,0 +1,80 @@
+/**
+ * translation-error.js — shared recorder for the most recent translation
+ * HTTP failure, used to give accurate setup help.
+ *
+ * WHY: When a sync fails, getSetupHelp() previously always said "check your
+ * dashboard for quota/billing" — wrong and misleading when the real problem
+ * was a bad/expired key (HTTP 401/403). The user burns time checking billing
+ * when they should be re-checking the key value.
+ *
+ * The HTTP clients (openrouter-client, direct-llm, fetch-with-retry) record
+ * the status of a failing request here as they give up on it. getSetupHelp()
+ * then reads the last-seen failure and tailors its guidance:
+ *   - auth   (401/403): the key itself is the problem, not billing
+ *   - quota  (429):     rate-limited / over quota → check billing/quota
+ *   - server (5xx):     upstream problem → wait and retry
+ *
+ * The method instance that ran the translation is discarded before
+ * getSetupHelp() is called (sync.js builds a fresh instance), so per-instance
+ * state can't carry the status across. A module-level "last error" is the
+ * pragmatic carrier: failures are almost always systemic (one bad key fails
+ * every pair), so the last-recorded status is representative of the failure
+ * the user is about to read help for.
+ *
+ * Node is single-threaded, so the assignment itself never races. Callers that
+ * want a clean slate (e.g. the start of a sync run) call reset first.
+ */
+
+let _last = null; // { status: number|null, kind: string }
+
+/**
+ * Classify an HTTP status into a coarse failure kind.
+ *
+ * @param {number} status - HTTP status code
+ * @returns {'auth'|'quota'|'server'|'unknown'}
+ */
+function classifyTranslationError(status) {
+  if (typeof status !== 'number') return 'unknown';
+  if (status === 401 || status === 403) return 'auth';
+  if (status === 429) return 'quota';
+  if (status >= 500 && status <= 599) return 'server';
+  return 'unknown';
+}
+
+/**
+ * Record a failing HTTP status. No-ops for non-numbers and for 2xx
+ * (a successful response is not a failure to report).
+ *
+ * @param {number} status - HTTP status code of the failed request
+ * @returns {{ status: number|null, kind: string }|null} the recorded entry
+ */
+function recordTranslationError(status) {
+  if (typeof status !== 'number' || (status >= 200 && status < 300)) return _last;
+  _last = { status, kind: classifyTranslationError(status) };
+  return _last;
+}
+
+/**
+ * Get the most recently recorded translation failure, or null if none.
+ *
+ * @returns {{ status: number|null, kind: string }|null}
+ */
+function getLastTranslationError() {
+  return _last;
+}
+
+/**
+ * Clear the recorded failure. Call at the start of a fresh sync run so a
+ * stale error from a prior in-process sync (e.g. watch mode) can't leak
+ * into the next run's help text.
+ */
+function resetTranslationError() {
+  _last = null;
+}
+
+export {
+  classifyTranslationError,
+  recordTranslationError,
+  getLastTranslationError,
+  resetTranslationError,
+};

package/lib/models.js

@@ -0,0 +1,258 @@
+/**
+ * Model listing service — fetches available models from provider APIs.
+ *
+ * WHY THIS EXISTS:
+ *   The init wizard, `champollion models` command, and method-level validation
+ *   all need the same thing: "what models does this provider offer for my
+ *   API key?" Previously, each method class had its own _fetchModels()
+ *   with duplicated fetch logic. This module centralizes it.
+ *
+ * DESIGN:
+ *   - Each provider entry defines how to call its model list API and
+ *     how to filter the results to chat-capable models.
+ *   - Results are cached per-process to avoid redundant API calls.
+ *   - Returns null on failure (network, invalid key) — callers decide
+ *     how to handle (fallback prompt, skip, etc.).
+ *
+ * PROVIDER API ENDPOINTS:
+ *   - Gemini:    GET https://generativelanguage.googleapis.com/v1beta/models?key=...
+ *   - OpenAI:    GET https://api.openai.com/v1/models  (Bearer token)
+ *   - Anthropic: GET https://api.anthropic.com/v1/models (x-api-key header)
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { getEnvOrFileVar } from './api-key.js';
+
+// Per-process cache: provider name → model ID array (or null if fetch failed)
+const _modelCache = new Map();
+
+// Lazy-loaded alias map (loaded once from shared/model-aliases.json)
+let _aliasCache = null;
+
+/**
+ * Load the model alias map from shared/model-aliases.json.
+ *
+ * Resolves the path relative to the monorepo root (two levels up from cli/lib/).
+ * Returns an empty object if the file doesn't exist or fails to parse,
+ * so callers can always safely check `aliases[name]`.
+ *
+ * @returns {Object<string, string>} Short name → full OpenRouter slug
+ */
+function _loadAliases() {
+  if (_aliasCache) return _aliasCache;
+
+  try {
+    const __dirname = path.dirname(fileURLToPath(import.meta.url));
+    // Prefer the package-bundled copy (cli/shared/, shipped via the prepack
+    // build-cards-fallback.mjs), then fall back to the monorepo-root SSOT for
+    // in-repo dev. Without the package copy, an installed CLI found no alias map
+    // and `--model <alias>` silently failed to resolve.
+    const aliasPaths = [
+      path.resolve(__dirname, '..', 'shared', 'model-aliases.json'),
+      path.resolve(__dirname, '..', '..', 'shared', 'model-aliases.json'),
+    ];
+    let raw = null;
+    for (const p of aliasPaths) {
+      try { raw = fs.readFileSync(p, 'utf-8'); break; } catch { /* try next */ }
+    }
+    const parsed = raw ? JSON.parse(raw) : {};
+    // Strip metadata keys (e.g., _comment) — only keep actual aliases
+    _aliasCache = {};
+    for (const [key, value] of Object.entries(parsed)) {
+      if (!key.startsWith('_') && typeof value === 'string') {
+        _aliasCache[key] = value;
+      }
+    }
+  } catch {
+    _aliasCache = {};
+  }
+
+  return _aliasCache;
+}
+
+/**
+ * Resolve a model name or alias to a full OpenRouter slug.
+ *
+ * Resolution order:
+ *   1. If contains '/' → already a full slug, pass through
+ *   2. Check shared/model-aliases.json for a matching alias
+ *   3. Pass through as-is (user may know what they're doing)
+ *
+ * @param {string} nameOrSlug - User-provided model identifier
+ * @returns {string} Full OpenRouter model slug
+ */
+function resolveModel(nameOrSlug) {
+  if (!nameOrSlug || typeof nameOrSlug !== 'string') return nameOrSlug;
+
+  // Already a full slug (contains provider prefix)
+  if (nameOrSlug.includes('/')) return nameOrSlug;
+
+  // Check aliases
+  const aliases = _loadAliases();
+  if (aliases[nameOrSlug]) return aliases[nameOrSlug];
+
+  // Pass through — could be a provider-specific model name (e.g., gemini-2.0-flash)
+  return nameOrSlug;
+}
+
+/**
+ * Provider configurations — how to fetch and filter models for each provider.
+ *
+ * Adding a new provider:
+ *   1. Add an entry here with fetch + filter functions
+ *   2. The method class's _fetchModels() delegates to fetchAvailableModels()
+ *   3. The init wizard and `models` command automatically pick it up
+ */
+const PROVIDERS = {
+  gemini: {
+    envVar: 'GEMINI_API_KEY',
+    label: 'Google Gemini',
+    async fetch(apiKey) {
+      const response = await fetch(
+        `https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`
+      );
+      if (!response.ok) return null;
+      const data = await response.json();
+      // Only include models that support generateContent (not embeddings-only).
+      // Strip the "models/" prefix that Gemini returns.
+      return (data.models || [])
+        .filter(m => m.supportedGenerationMethods?.includes('generateContent'))
+        .map(m => m.name.replace('models/', ''));
+    },
+  },
+
+  openai: {
+    envVar: 'OPENAI_API_KEY',
+    label: 'OpenAI',
+    async fetch(apiKey) {
+      const response = await fetch('https://api.openai.com/v1/models', {
+        headers: { 'Authorization': `Bearer ${apiKey}` },
+      });
+      if (!response.ok) return null;
+      const data = await response.json();
+      // Filter to chat-capable models — skip embeddings, whisper, dall-e, tts, etc.
+      return (data.data || [])
+        .map(m => m.id)
+        .filter(id =>
+          id.startsWith('gpt-') ||
+          id.startsWith('o1') ||
+          id.startsWith('o3') ||
+          id.startsWith('o4') ||
+          id.startsWith('chatgpt-')
+        );
+    },
+  },
+
+  anthropic: {
+    envVar: 'ANTHROPIC_API_KEY',
+    label: 'Anthropic',
+    async fetch(apiKey) {
+      const response = await fetch('https://api.anthropic.com/v1/models', {
+        headers: {
+          'x-api-key': apiKey,
+          'anthropic-version': '2023-06-01',
+        },
+      });
+      if (!response.ok) return null;
+      const data = await response.json();
+      return (data.data || []).map(m => m.id);
+    },
+  },
+};
+
+/**
+ * Fetch available chat-capable models for a provider.
+ *
+ * Uses the provider's real API to get the actual model list the user's
+ * API key has access to. Results are cached per-process.
+ *
+ * @param {string} provider - Provider name: 'gemini', 'openai', 'anthropic'
+ * @param {string} apiKey - The provider's API key
+ * @returns {Promise<string[]|null>} Array of model IDs, or null on failure
+ */
+async function fetchAvailableModels(provider, apiKey) {
+  if (!apiKey) return null;
+
+  const config = PROVIDERS[provider];
+  if (!config) return null;
+
+  // Check cache — undefined = never tried, null = tried and failed
+  const cached = _modelCache.get(provider);
+  if (cached !== undefined) return cached;
+
+  try {
+    const models = await config.fetch(apiKey);
+    if (models && models.length > 0) {
+      _modelCache.set(provider, models);
+      return models;
+    }
+    _modelCache.set(provider, null);
+    return null;
+  } catch {
+    _modelCache.set(provider, null);
+    return null;
+  }
+}
+
+/**
+ * Resolve an API key for a provider from environment or .env files.
+ *
+ * @param {string} provider - Provider name
+ * @param {string} [cwd] - Working directory for .env file lookup
+ * @returns {string|null} API key or null
+ */
+function resolveProviderApiKey(provider, cwd) {
+  const config = PROVIDERS[provider];
+  if (!config) return null;
+  return getEnvOrFileVar(config.envVar) || getEnvOrFileVar(config.envVar, cwd);
+}
+
+/**
+ * Get the display label for a provider.
+ *
+ * @param {string} provider - Provider name
+ * @returns {string} Human-readable label
+ */
+function getProviderLabel(provider) {
+  return PROVIDERS[provider]?.label || provider;
+}
+
+/**
+ * Check if a provider has model listing support.
+ *
+ * @param {string} provider - Provider name
+ * @returns {boolean}
+ */
+function isListableProvider(provider) {
+  return provider in PROVIDERS;
+}
+
+/**
+ * Get all provider names that support model listing.
+ *
+ * @returns {string[]}
+ */
+function getListableProviders() {
+  return Object.keys(PROVIDERS);
+}
+
+/**
+ * Clear the per-process model cache.
+ * Primarily for testing — allows re-fetching in a long-running process.
+ */
+function clearModelCache() {
+  _modelCache.clear();
+}
+
+export {
+  fetchAvailableModels,
+  resolveModel,
+  resolveProviderApiKey,
+  getProviderLabel,
+  isListableProvider,
+  getListableProviders,
+  clearModelCache,
+  PROVIDERS,
+};

package/lib/no-translate.js

@@ -0,0 +1,233 @@
+/**
+ * no-translate.js — keys whose correct translation is the source, verbatim.
+ *
+ * THE PROBLEM THIS SOLVES:
+ *   Some values have exactly one correct rendering in every locale: a URL,
+ *   a repo path, a package name. The post-translation quality gate rejects
+ *   source-echo (lib/validate.js check 2), so for these keys the CORRECT
+ *   answer always FAILS. That has two observed failure modes, both real,
+ *   both seen in production:
+ *
+ *     1. Weak models learn to defeat the gate by bending the value just
+ *        enough to stop being an echo. Observed on a live site: 48 corrupted
+ *        URLs across 13 locales — fabricated fragments (".../view/1954#fr"),
+ *        stray trailing "#" and "/", a U+200E LEFT-TO-RIGHT MARK prepended
+ *        in Arabic, a U+200B ZERO WIDTH SPACE appended in Hindi. The
+ *        invisible-character ones break the link outright.
+ *     2. Strong models return the value unchanged, correctly, and fail the
+ *        gate — so `champollion sync` exits non-zero forever. A pre-commit
+ *        hook wired to sync can then never be satisfied, and the only way
+ *        past is to disable the whole gate.
+ *
+ *   There is no threshold that fixes this, because the gate is asking the
+ *   wrong question. The fix is to declare the key out of scope: never send
+ *   it to a backend, never gate it, never bill it — copy it verbatim.
+ *
+ * TWO WAYS A KEY BECOMES NO-TRANSLATE:
+ *   1. `noTranslate` config patterns — dot-path keys and/or globs:
+ *        "noTranslate": ["**.url", "pages.software.*.repo", "meta.appId"]
+ *   2. Auto-detected bare URLs — a source value that is nothing but a
+ *      `scheme://…` URL. On by default (`noTranslateUrls`), because the
+ *      current behaviour has no correct outcome. Opt out with
+ *      `"noTranslateUrls": false`.
+ *
+ * WHERE IT PLUGS IN: `diffLocale` (lib/diff.js) takes the resulting matcher
+ * and routes matching keys into a `noTranslate` bucket instead of
+ * `toProcess`. Because the cost estimator diffs with the same matcher, the
+ * keys are excluded from the bill by construction rather than by a second
+ * rule that could drift.
+ */
+
+/**
+ * A value is "a bare URL" when the whole thing, trimmed, is one absolute
+ * URL and nothing else.
+ *
+ * Deliberately NOT a substring match: "Read the paper at https://…" is
+ * prose with a URL in it and must still be translated. Only a value that
+ * IS the URL has no translatable content.
+ *
+ * Scheme grammar follows RFC 3986 (ALPHA *( ALPHA / DIGIT / "+" / "-" / "."))
+ * and requires the `://` authority form, so `https://`, `ftp://` and
+ * `ipfs://` match while `mailto:` and a bare `example.com` do not.
+ */
+const BARE_URL = /^[a-z][a-z0-9+.-]*:\/\/\S+$/i;
+
+/**
+ * Is this source value a bare URL?
+ *
+ * @param {unknown} value - Source value to test
+ * @returns {boolean} True when the trimmed value is exactly one absolute URL
+ */
+function isBareUrl(value) {
+  return typeof value === 'string' && BARE_URL.test(value.trim());
+}
+
+/**
+ * Compile one dot-path pattern into a segment matcher.
+ *
+ * Pattern grammar (segments split on `.`):
+ *   - a literal segment matches that segment exactly
+ *   - `*`  matches any characters WITHIN one segment (`page*` → `pageTitle`)
+ *   - `**` matches zero or more whole segments (`**.url` matches `a.b.url`
+ *     and a top-level `url`)
+ *   - a pattern with no wildcard is an exact dot-path
+ *
+ * @param {string} pattern - e.g. '**.url', 'pages.software.*.repo'
+ * @returns {(keySegments: string[]) => boolean} Segment-array matcher
+ */
+function compilePattern(pattern) {
+  const patternSegments = pattern.split('.');
+
+  // Per-segment regexes, built once. `**` is handled structurally below and
+  // never reaches this map.
+  const segmentTests = patternSegments.map(seg => {
+    if (seg === '**') return null;
+    if (!seg.includes('*')) return (s) => s === seg;
+    // Escape everything regex-significant, then turn `*` into "any run of
+    // characters that is not a segment separator".
+    const source = '^' + seg
+      .split('*')
+      .map(part => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
+      .join('[^.]*') + '$';
+    const re = new RegExp(source);
+    return (s) => re.test(s);
+  });
+
+  /**
+   * Classic backtracking glob match over segment arrays. Locale key depth is
+   * small (single digits), so recursion is cheap and the code stays readable.
+   */
+  function match(pi, ki, keySegments) {
+    if (pi === patternSegments.length) return ki === keySegments.length;
+
+    if (patternSegments[pi] === '**') {
+      // `**` consumes zero or more segments — try every split point.
+      for (let skip = ki; skip <= keySegments.length; skip++) {
+        if (match(pi + 1, skip, keySegments)) return true;
+      }
+      return false;
+    }
+
+    if (ki >= keySegments.length) return false;
+    if (!segmentTests[pi](keySegments[ki])) return false;
+    return match(pi + 1, ki + 1, keySegments);
+  }
+
+  return (keySegments) => match(0, 0, keySegments);
+}
+
+/**
+ * Validate the no-translate config fields, failing loud on anything the
+ * matcher could not honour.
+ *
+ * A misspelled or wrong-typed `noTranslate` must never degrade to "translate
+ * everything": the user wrote it precisely so certain keys would be left
+ * alone, and silently ignoring it re-opens the corruption path this module
+ * exists to close.
+ *
+ * @param {unknown} patterns - Raw config.noTranslate
+ * @param {unknown} urls - Raw config.noTranslateUrls
+ * @throws {Error} With code CHAMPOLLION_CONFIG_INVALID
+ */
+function validateNoTranslateConfig(patterns, urls) {
+  const fail = (message) => {
+    const e = new Error(message);
+    e.code = 'CHAMPOLLION_CONFIG_INVALID';
+    throw e;
+  };
+
+  if (patterns != null) {
+    if (!Array.isArray(patterns)) {
+      fail(
+        '"noTranslate" must be an array of dot-path keys or glob patterns, '
+        + `got ${typeof patterns}. Example: "noTranslate": ["**.url", "pages.software.*.repo"]`,
+      );
+    }
+    for (const p of patterns) {
+      if (typeof p !== 'string' || p.trim() === '') {
+        fail(`"noTranslate" entries must be non-empty strings — found ${JSON.stringify(p)}.`);
+      }
+      if (p.includes('..')) {
+        fail(`"noTranslate" pattern "${p}" has an empty path segment. Use "**" to match any depth.`);
+      }
+    }
+  }
+
+  if (urls != null && typeof urls !== 'boolean') {
+    fail(`"noTranslateUrls" must be a boolean, got ${typeof urls}. Set it to false to translate URL-valued keys.`);
+  }
+}
+
+/**
+ * Build the no-translate matcher for a resolved config.
+ *
+ * @param {object} config - Resolved config (reads `noTranslate`, `noTranslateUrls`)
+ * @returns {NoTranslateMatcher}
+ *
+ * @typedef {object} NoTranslateMatcher
+ * @property {boolean} active - False when nothing is configured and URL
+ *   auto-detection is off. Callers can skip the whole lane.
+ * @property {(key: string, value: unknown) => boolean} matches - Is this key
+ *   exempt from translation?
+ * @property {(key: string, value: unknown) => string|null} reason - Why it is
+ *   exempt ('pattern "**.url"' / 'auto-detected URL'), or null.
+ * @property {string[]} patterns - The configured patterns, for reporting.
+ * @property {boolean} urls - Whether URL auto-detection is on.
+ */
+function compileNoTranslate(config = {}) {
+  validateNoTranslateConfig(config.noTranslate, config.noTranslateUrls);
+
+  const patterns = Array.isArray(config.noTranslate) ? [...config.noTranslate] : [];
+  const urls = config.noTranslateUrls !== false;
+  const compiled = patterns.map(p => ({ pattern: p, test: compilePattern(p) }));
+
+  // Segment splits are pure and repeated across every locale in a run.
+  const segmentCache = new Map();
+  const segmentsOf = (key) => {
+    let segs = segmentCache.get(key);
+    if (!segs) {
+      segs = key.split('.');
+      segmentCache.set(key, segs);
+    }
+    return segs;
+  };
+
+  const reason = (key, value) => {
+    const segs = segmentsOf(key);
+    for (const { pattern, test } of compiled) {
+      if (test(segs)) return `pattern "${pattern}"`;
+    }
+    if (urls && isBareUrl(value)) return 'auto-detected URL';
+    return null;
+  };
+
+  return {
+    active: compiled.length > 0 || urls,
+    matches: (key, value) => reason(key, value) !== null,
+    reason,
+    patterns,
+    urls,
+  };
+}
+
+/**
+ * Matcher that exempts nothing — for callers with no config in hand.
+ *
+ * @type {NoTranslateMatcher}
+ */
+const NO_TRANSLATE_NONE = {
+  active: false,
+  matches: () => false,
+  reason: () => null,
+  patterns: [],
+  urls: false,
+};
+
+export {
+  compileNoTranslate,
+  compilePattern,
+  isBareUrl,
+  validateNoTranslateConfig,
+  NO_TRANSLATE_NONE,
+  BARE_URL,
+};