@aidriftline/driftline 0.0.0-stage → 0.1.10

package/src/prompts.js

@@ -0,0 +1,98 @@
+import { readFileSync, existsSync } from "node:fs";
+import { normalizeId } from "./registry.js";
+
+/**
+ * Load the prompt file. JSON is parsed as-is. YAML supports the documented subset:
+ *
+ * prompts:
+ *   - id: invoice-json
+ *     model: gpt-4o-2024-05-13       # optional
+ *     compare_to: gpt-5.6-sol        # optional; defaults to the registry replacement
+ *     expect: json                   # optional: json | text
+ *     system: You extract invoice fields.
+ *     prompt: |
+ *       Extract the fields from this invoice as JSON:
+ *       ...
+ *
+ * Block values can be literal (|) or folded (>), with optional chomping (+ or -).
+ */
+export function loadPrompts(path) {
+  if (!existsSync(path)) return null;
+  const text = readFileSync(path, "utf8");
+  const data = path.toLowerCase().endsWith(".json") ? JSON.parse(text) : parseYamlSubset(text);
+  const list = Array.isArray(data) ? data : data?.prompts;
+  if (!Array.isArray(list)) throw new Error(`${path}: expected a "prompts:" list`);
+  return list.map((p, i) => {
+    if (!p.prompt) throw new Error(`${path}: prompt #${i + 1} has no "prompt" field`);
+    return {
+      ...p,
+      id: String(p.id || `prompt-${i + 1}`).replace(/[\r\n]+/g, " "),
+      expect: String(p.expect || "text").toLowerCase(),
+      ...(p.model ? { model: normalizeId(String(p.model).replace(/\s+/g, "")) } : {}),
+      ...(p.compare_to ? { compare_to: normalizeId(String(p.compare_to).replace(/\s+/g, "")) } : {}),
+    };
+  });
+}
+
+export function parseYamlSubset(text) {
+  const lines = text.replace(/\t/g, "  ").split(/\r?\n/);
+  const out = { prompts: [] };
+  let item = null;
+  let block = null; // { key, indent, lines }
+
+  const flushBlock = () => {
+    if (block && item) {
+      const lines = block.lines.join("\n").replace(/\n+$/, "").split("\n");
+      // Folded (>): lines join with spaces; a blank line starts a new paragraph.
+      item[block.key] = block.folded
+        ? lines.join("\n").split(/\n{2,}/).map((para) => para.split("\n").join(" ")).join("\n")
+        : lines.join("\n");
+    }
+    block = null;
+  };
+  const scalar = (v) => {
+    v = v.trim();
+    let q = v.match(/^"((?:[^"\\]|\\.)*)"\s*(?:#.*)?$/);
+    if (q) return JSON.parse(`"${q[1]}"`);
+    q = v.match(/^'((?:[^']|'')*)'\s*(?:#.*)?$/);
+    if (q) return q[1].replace(/''/g, "'");
+    return v.replace(/\s+#.*$/, "").trim();
+  };
+
+  for (const raw of lines) {
+    const indent = raw.match(/^ */)[0].length;
+    const line = raw.trim();
+    if (block) {
+      if (block.indent === null && line !== "") {
+        if (indent > block.keyCol) block.indent = indent;
+      }
+      if (line === "" || (block.indent !== null && indent >= block.indent)) {
+        block.lines.push(line === "" ? "" : raw.slice(block.indent));
+        continue;
+      }
+      flushBlock();
+    }
+    if (line === "" || line.startsWith("#")) continue;
+    if (indent === 0 && /^(---|\.\.\.)(\s.*)?$/.test(line)) continue; // YAML document markers
+    if (/^prompts:\s*(\[\s*\])?\s*(#.*)?$/.test(line) && indent === 0) continue;
+
+    let m = line.match(/^-\s+([\w-]+):\s*(.*)$/);
+    if (m) {
+      item = {};
+      out.prompts.push(item);
+    } else {
+      m = line.match(/^([\w-]+):\s*(.*)$/);
+      if (!m || !item) throw new Error(`Cannot read prompt file line: "${raw}"`);
+    }
+    const [, key, value] = m;
+    const blockStart = value.replace(/\s+#.*$/, "").match(/^([|>])[+-]?$/);
+    if (blockStart) {
+      const keyCol = indent + (raw.trimStart().startsWith("-") ? 2 : 0);
+      block = { key, keyCol, indent: null, lines: [], folded: blockStart[1] === ">" };
+    } else {
+      item[key] = scalar(value);
+    }
+  }
+  flushBlock();
+  return out;
+}

package/src/providers.js

@@ -0,0 +1,100 @@
+// Calls each provider with the customer's own API key. Keys never leave their CI runner.
+
+export function providerOf(model) {
+  if (/^claude-/.test(model)) return "anthropic";
+  if (/^(gemini-|text-embedding-0)/.test(model)) return "google"; // OpenAI's are text-embedding-3-* / -ada-*
+  return "openai";
+}
+
+const KEY_ENV = { openai: "OPENAI_API_KEY", anthropic: "ANTHROPIC_API_KEY", google: "GEMINI_API_KEY" };
+
+export function keyFor(provider, env = process.env) {
+  return env[KEY_ENV[provider]] || (provider === "google" ? env.GOOGLE_API_KEY : undefined);
+}
+
+export function missingKeyMessage(provider) {
+  return `No API key for ${provider}. Add ${KEY_ENV[provider]} as a repository secret and pass it in "env".`;
+}
+
+const TIMEOUT_MS = 120_000;
+const RETRY_STATUS = new Set([429, 500, 502, 503, 504, 529]); // busy or overloaded: worth one more try
+const wait = (ms) => new Promise((r) => setTimeout(r, ms));
+export const retry = { delayMs: 5000 }; // tests set this to 0
+
+async function postJson(fetchImpl, url, headers, body) {
+  try {
+    return await postOnce(fetchImpl, url, headers, body);
+  } catch (e) {
+    if (!RETRY_STATUS.has(e.status)) throw e;
+    await wait(Math.min(e.retryAfterMs ?? retry.delayMs, retry.delayMs ? 30_000 : 0));
+    return postOnce(fetchImpl, url, headers, body);
+  }
+}
+
+// An empty answer is not a "changed" answer: say why the model gave nothing.
+function nothing(reason) {
+  throw new Error(`Empty answer${reason ? ` (${reason})` : ""}`);
+}
+
+async function postOnce(fetchImpl, url, headers, body) {
+  let res;
+  try {
+    res = await fetchImpl(url, {
+      method: "POST",
+      headers: { "content-type": "application/json", ...headers },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(TIMEOUT_MS), // a hung request must not hold the job for hours
+    });
+  } catch (e) {
+    if (e.name === "TimeoutError") throw new Error(`No answer after ${TIMEOUT_MS / 1000} seconds`);
+    throw e;
+  }
+  const text = await res.text();
+  let json;
+  try { json = JSON.parse(text); } catch { json = null; }
+  if (!res.ok) {
+    const msg = json?.error?.message || text.slice(0, 300);
+    const err = new Error(`HTTP ${res.status}: ${String(msg).replace(/\s+/g, " ").slice(0, 300)}`);
+    err.status = res.status;
+    const ra = Number(res.headers?.get?.("retry-after"));
+    if (ra > 0) err.retryAfterMs = ra * 1000;
+    throw err;
+  }
+  return json;
+}
+
+export async function complete({ model, prompt, system, env = process.env, fetchImpl = fetch }) {
+  const provider = providerOf(model);
+  const key = keyFor(provider, env);
+  if (!key) throw new Error(missingKeyMessage(provider));
+
+  if (provider === "openai") {
+    const data = await postJson(fetchImpl, "https://api.openai.com/v1/responses",
+      { authorization: `Bearer ${key}` },
+      { model, input: prompt, ...(system ? { instructions: system } : {}) });
+    const text = typeof data.output_text === "string" ? data.output_text : (data.output || [])
+      .flatMap((o) => o.content || [])
+      .filter((c) => c.type === "output_text" || typeof c.text === "string")
+      .map((c) => c.text)
+      .join("");
+    const refusal = (data.output || []).flatMap((o) => o.content || []).find((c) => c.type === "refusal");
+    return text.trim() ? text : nothing(refusal ? "refused" : data.incomplete_details?.reason || data.status);
+  }
+
+  if (provider === "anthropic") {
+    const data = await postJson(fetchImpl, "https://api.anthropic.com/v1/messages",
+      { "x-api-key": key, "anthropic-version": "2023-06-01" },
+      { model, max_tokens: 4096, ...(system ? { system } : {}), messages: [{ role: "user", content: prompt }] });
+    const text = (data.content || []).filter((c) => c.type === "text").map((c) => c.text).join("");
+    if (data.stop_reason === "max_tokens") throw new Error("Answer cut off at 4,096 tokens; shorten the prompt or ask for less");
+    return text.trim() ? text : nothing(data.stop_reason);
+  }
+
+  const data = await postJson(fetchImpl,
+    `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(model)}:generateContent`,
+    { "x-goog-api-key": key },
+    { contents: [{ role: "user", parts: [{ text: prompt }] }], ...(system ? { systemInstruction: { parts: [{ text: system }] } } : {}) });
+  const c = data.candidates?.[0];
+  const text = (c?.content?.parts || []).map((p) => p.text || "").join("");
+  return text.trim() ? text : nothing(c?.finishReason || data.promptFeedback?.blockReason);
+}

package/src/registry.js

@@ -0,0 +1,90 @@
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const DAY = 24 * 60 * 60 * 1000;
+
+export const REGISTRY_PATH = join(here, "..", "registry", "models.json");
+
+export function loadRegistry(path = REGISTRY_PATH) {
+  return buildRegistry(JSON.parse(readFileSync(path, "utf8")));
+}
+
+// Same as loadRegistry, from already-parsed JSON (e.g. the live copy at aidriftline.com/models.json).
+export function buildRegistry(data) {
+  const byId = new Map(data.models.map((m) => [m.id, { ...m }]));
+  // Providers sometimes name a replacement that is itself being retired (o1-mini -> o4-mini -> ...).
+  // Follow the chain so customers are pointed at a model that stays.
+  for (const m of byId.values()) {
+    if (!m.replacement) continue;
+    const seen = new Set([m.id]);
+    let next = m.replacement;
+    seen.add(next);
+    while (byId.get(next)?.replacement && byId.get(next).qualifier !== "not-before" && !seen.has(byId.get(next).replacement)) {
+      next = byId.get(next).replacement;
+      seen.add(next);
+    }
+    if (next !== m.replacement && next !== m.id) { m.via = m.replacement; m.replacement = next; }
+  }
+  // Anthropic short names point at a dated snapshot: "claude-sonnet-4-5" and "-latest" -> claude-sonnet-4-5-20250929,
+  // "claude-sonnet-4-0" -> claude-sonnet-4-20250514. The deprecation page lists only snapshots, so a short name
+  // is added only when every snapshot behind it shares one date. Aliases go in the lookup, not in `models`.
+  const groups = new Map();
+  for (const m of byId.values()) {
+    const base = m.provider === "anthropic" && m.id.match(/^(claude-.+)-\d{8}$/)?.[1];
+    if (base) groups.set(base, [...(groups.get(base) || []), m]);
+  }
+  const lookup = new Map(byId);
+  for (const [base, snaps] of groups) {
+    if (snaps.some((s) => s.retires !== snaps[0].retires || s.qualifier !== snaps[0].qualifier)) continue;
+    const latest = snaps.reduce((a, b) => (a.id > b.id ? a : b));
+    const names = [base, `${base}-latest`, ...(/-[a-z]+-\d$/.test(base) ? [`${base}-0`] : [])];
+    for (const n of names) if (!lookup.has(n)) lookup.set(n, { ...latest, aliasOf: latest.id });
+  }
+  return { ...data, models: [...byId.values()], byId: lookup };
+}
+
+// Strip prefixes people put in front of model IDs, e.g. "models/gemini-2.0-flash" or "openai/gpt-4",
+// and map cloud-platform spellings to the provider's ID:
+//   Amazon Bedrock  "us.anthropic.claude-3-5-sonnet-20240620-v1:0" -> "claude-3-5-sonnet-20240620"
+//   Google Vertex   "claude-3-5-sonnet@20240620"                     -> "claude-3-5-sonnet-20240620"
+//   Bedrock ARN     "arn:aws:bedrock:...:foundation-model/anthropic.claude-..." -> "claude-..."
+//   Router prefixes "openrouter/openai/gpt-4", "vertex_ai/gemini-2.0-flash" -> plain ID (not azure/: Azure retires on its own dates)
+export function normalizeId(raw) {
+  let id = raw.trim().toLowerCase();
+  if (id.startsWith("arn:aws:bedrock")) id = id.slice(id.lastIndexOf("/") + 1); // foundation-model/… or inference-profile/…
+  let prev;
+  do { prev = id; id = id.replace(/^(models|openai|anthropic|google|gemini|vertex_ai|bedrock|openrouter)\//, ""); } while (id !== prev);
+  const bedrock = /^(?:(?:us|eu|apac|global)\.)?anthropic\./;
+  if (bedrock.test(id)) id = id.replace(bedrock, "").replace(/-v\d+(?::\d+)?$/, "");
+  id = id.replace(/(?:-v\d+)?@(\d{8})$/, "-$1");     // Vertex: claude-3-5-sonnet-v2@20241022
+  return id.startsWith("claude-") ? id.replace(/(\d)\.(\d)/g, "$1-$2") : id; // OpenRouter: claude-sonnet-4.5
+}
+
+// Is this string shaped like a model ID from one of the three providers?
+// IDs listed in the registry (e.g. "whisper-1", "dall-e-3", "imagen-4.0-generate-001") always count.
+export function looksLikeModelId(id, known) {
+  if (known && known.has(id)) return true;
+  return /^(gpt-\d|gpt-[a-z]|chatgpt-|o[1-9](-|$)|claude-|gemini-|text-embedding-|davinci-|babbage-)/.test(id);
+}
+
+// Only text models can be drift-tested; audio, image, video, embedding and the rest are scan-only.
+export const isTextModel = (entry) => !entry?.kind || entry.kind === "text";
+
+/**
+ * Classify one model ID against the registry on a given date.
+ * status: retired | at-risk | deprecated | active | unknown
+ */
+export function classify(id, registry, { today = new Date(), warnDays = 90 } = {}) {
+  const entry = registry.byId.get(id);
+  if (!entry) return { id, status: "unknown" };
+  const retires = new Date(entry.retires + "T00:00:00Z");
+  const days = Math.ceil((retires - today) / DAY);
+  let status;
+  if (entry.qualifier === "not-before") status = "active";
+  else if (days <= 0) status = entry.qualifier === "earliest" ? "at-risk" : "retired"; // Google only gives earliest dates
+  else if (days <= warnDays) status = "at-risk";
+  else status = "deprecated";
+  return { id, status, days, entry };
+}

package/src/report.js

@@ -0,0 +1,102 @@
+const STATUS_LABEL = {
+  retired: "🔴 Retired",
+  "at-risk": "🟠 Retires soon",
+  deprecated: "🟡 Deprecated",
+  active: "🟢 Active",
+  unknown: "⚪ No retirement listed",
+};
+const RESULT_LABEL = { same: "✅ Same", changed: "🟡 Changed", broken: "🔴 Broken", error: "⚠️ Error", skipped: "⏭️ Skipped" };
+
+// Google gives only earliest dates; once one has passed the model may be gone already, so "retires soon" would be wrong.
+// Routed models keep answering after the date, but from the replacement model ("retired" would be wrong too).
+const label = (c) => c.status === "at-risk" && c.entry?.qualifier === "earliest" && c.days <= 0
+  ? "🟠 May already be off"
+  : c.status === "retired" && c.entry?.routes ? "🔴 Now answered by the replacement"
+  : c.status === "at-risk" && c.entry?.routes ? "🟠 Switches model soon"
+  : STATUS_LABEL[c.status];
+
+function when(c) {
+  if (!c.entry) return "";
+  const q = c.entry.routes ? " (routing starts)" : c.entry.qualifier === "earliest" ? " (earliest)" : c.entry.qualifier === "not-before" ? " (not before)" : "";
+  const rel = c.days <= 0
+    ? (c.entry.qualifier === "earliest" ? `earliest shutdown passed ${-c.days} days ago` : `${-c.days} days ago`)
+    : `in ${c.days} days`;
+  return `${c.entry.retires}${q} · ${rel}`;
+}
+
+// Table cells: no pipes, no line breaks, no raw HTML (provider error pages, odd file names).
+const esc = (s) => String(s ?? "").replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
+  .replace(/\|/g, "\\|").replace(/[\r\n]+/g, " ");
+// Inside a code span HTML is shown as text, so only pipes, line breaks and backticks need handling.
+const code = (s) => `\`${String(s ?? "").replace(/[\r\n]+/g, " ").replace(/`/g, "'").replace(/\|/g, "\\|")}\``;
+
+// A code fence longer than any run of backticks inside the text, so a model's own ```json fence can't close it.
+function fenced(text) {
+  const longest = Math.max(0, ...(String(text).match(/`+/g) || []).map((r) => r.length));
+  const fence = "`".repeat(Math.max(3, longest + 1));
+  return [fence, text, fence];
+}
+
+// GitHub issues hold at most 65,536 characters; keep the report well under that.
+const MAX_REPORT = 60000;
+const MAX_ANSWER = 1500;
+const MAX_ROWS = 150;
+const clip = (s) => { s = String(s || ""); return s.length > MAX_ANSWER ? s.slice(0, MAX_ANSWER) + "\n[…cut]" : s; };
+
+export function renderReport({ findings, drift, registry, notices = [], answers = true }) {
+  const lines = ["## Driftline report", ""];
+  const flagged = findings.filter((f) => ["retired", "at-risk", "deprecated"].includes(f.status));
+  const byModel = new Map();
+  for (const f of findings) {
+    if (!byModel.has(f.id)) byModel.set(f.id, { ...f, places: [], docs: 0 });
+    byModel.get(f.id).places.push(code(`${f.file}:${f.line}`));
+    if (f.doc) byModel.get(f.id).docs++;
+  }
+
+  if (!findings.length) {
+    lines.push("No AI model IDs found in this repository.");
+  } else {
+    const n = flagged.length;
+    lines.push(n
+      ? `**${n} reference${n === 1 ? "" : "s"} to retired or retiring models** across ${byModel.size} model ID${byModel.size === 1 ? "" : "s"} found.`
+      : `Found ${byModel.size} model ID${byModel.size === 1 ? "" : "s"}. None are scheduled for retirement.`);
+    lines.push("", "| Model | Status | Retirement | Replacement | Where |", "|---|---|---|---|---|");
+    const order = ["retired", "at-risk", "deprecated", "active", "unknown"];
+    const all = [...byModel.values()].sort((a, b) => order.indexOf(a.status) - order.indexOf(b.status));
+    const rows = all.slice(0, MAX_ROWS); // flagged models sort first, so only unknown/active rows get cut
+    for (const r of rows) {
+      const places = r.places.slice(0, 3).join(", ") + (r.places.length > 3 ? ` +${r.places.length - 3} more` : "");
+      const docsOnly = r.places.length && byModel.get(r.id).docs === r.places.length ? " · docs only" : "";
+      lines.push(`| ${code(r.id)}${r.entry?.aliasOf ? " · alias of " + code(r.entry.aliasOf) : ""}${r.entry?.kind && r.entry.kind !== "text" ? " · " + r.entry.kind : ""} | ${label(r)}${docsOnly} | ${esc(when(r))} | ${r.entry?.replacement ? code(r.entry.replacement) : ""} | ${places} |`);
+    }
+    if (all.length > rows.length) lines.push("", `_${all.length - rows.length} more model ID(s) with no retirement listed are not shown._`);
+  }
+
+  if (drift?.length) {
+    lines.push("", "### Drift test", "", "| Prompt | Current → replacement | Similarity | Result | Note |", "|---|---|---|---|---|");
+    for (const d of drift) {
+      const score = typeof d.score === "number" ? d.score.toFixed(2) : "";
+      lines.push(`| ${esc(d.id)} | ${code(d.from)} → ${code(d.to)} | ${score} | ${RESULT_LABEL[d.result] || d.result} | ${esc(d.note)} |`);
+    }
+    const details = drift.filter((d) => d.before !== undefined);
+    if (details.length && !answers) {
+      lines.push("", "_Answers are not included here. They are in the job summary of the workflow run._");
+    } else if (details.length) {
+      lines.push("", "<details><summary>Answers</summary>", "");
+      let shown = 0;
+      for (const d of details) {
+        const block = [`#### ${esc(d.id)}`, "", code(d.from), "", ...fenced(clip(d.before)), "", code(d.to), "", ...fenced(clip(d.after)), ""];
+        if (lines.join("\n").length + block.join("\n").length > MAX_REPORT - 2000) break;
+        lines.push(...block);
+        shown++;
+      }
+      if (shown < details.length) lines.push(`_${details.length - shown} more answer(s) left out to stay within GitHub's size limit._`, "");
+      lines.push("</details>");
+    }
+  }
+
+  if (notices.length) lines.push("", ...notices.map((n) => `> ${n}`));
+  lines.push("", `<sub>Retirement dates checked against official provider pages on ${registry.checked}. ` +
+    `Sources: [OpenAI](${registry.sources.openai}) · [Anthropic](${registry.sources.anthropic}) · [Google](${registry.sources.google}).</sub>`);
+  return lines.join("\n");
+}

package/src/run.js

@@ -0,0 +1,133 @@
+import { existsSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import { loadRegistry, classify, isTextModel } from "./registry.js";
+import { scanDirectory } from "./scan.js";
+import { loadPrompts } from "./prompts.js";
+import { complete, providerOf, keyFor, missingKeyMessage } from "./providers.js";
+import { compare, extractJson } from "./compare.js";
+import { checkLicense } from "./license.js";
+import { renderReport } from "./report.js";
+
+const MAX_PROMPTS = 20;
+const oneLine = (s) => String(s).replace(/[\r\n]+/g, " ");
+const FLAGGED = new Set(["retired", "at-risk", "deprecated"]);
+
+export async function run({
+  root = ".",
+  repoRoot = root,      // the repository root: prompt file and report paths are relative to it
+  warnDays = 90,
+  promptsFile = ".driftline/prompts.yml",
+  licenseKey = "",
+  account = "",         // GitHub owner the workflow runs in; the license must be activated for it
+  env = process.env,
+  fetchImpl = fetch,
+  today = new Date(),
+  registry = loadRegistry(),
+  log = () => {},
+  answers = true,       // include model answers in the job summary
+} = {}) {
+  const notices = [];
+  const prefix = relative(repoRoot, root).split(sep).join("/");
+  const raw = scanDirectory(root, { ignore: [".driftline/"], known: registry.byId });
+  const findings = raw.map((h) => ({
+    ...h,
+    file: prefix && !prefix.startsWith("..") ? `${prefix}/${h.file.split(sep).join("/")}` : h.file.split(sep).join("/"),
+    ...classify(h.id, registry, { today, warnDays }),
+  }));
+  log(`Scanned ${root}: ${findings.length} model reference(s).`);
+
+  let drift = [];
+  let prompts = null;
+  try {
+    // Repo root first; a monorepo may keep the file next to the scanned folder instead.
+    const file = [join(repoRoot, promptsFile), join(root, promptsFile)].find((f) => existsSync(f));
+    prompts = file ? loadPrompts(file) : null;
+  } catch (e) {
+    notices.push(`Could not read ${promptsFile}: ${e.message}. The scan still ran; drift tests were skipped.`);
+  }
+  if (prompts && !prompts.length) {
+    notices.push(`${promptsFile} has no prompts in it, so drift tests were skipped.`);
+  } else if (prompts?.length) {
+    try {
+      const lic = await checkLicense(licenseKey, { fetchImpl, account });
+      if (lic.reason) notices.push(lic.reason);
+      if (lic.ok) drift = await runDrift({ prompts, findings, registry, env, fetchImpl, today, warnDays, notices, log });
+    } catch (e) {
+      // Drift tests are extra; the free scan must still produce its report.
+      notices.push(`Drift tests stopped: ${e.message}. The scan results above are complete.`);
+    }
+  } else if (findings.some((f) => FLAGGED.has(f.status))) {
+    notices.push(`Add ${promptsFile} to test your real prompts on the replacement model before you switch.`);
+  }
+
+  const atRisk = findings.filter((f) => !f.doc && (f.status === "retired" || f.status === "at-risk")).length;
+  const markdown = renderReport({ findings, drift, registry, notices, answers });
+  // The issue never carries model answers: in a public repository anyone can read it.
+  const issueMarkdown = renderReport({ findings, drift, registry, notices, answers: false });
+  return { findings, drift, notices, atRisk, markdown, issueMarkdown };
+}
+
+async function runDrift({ prompts, findings, registry, env, fetchImpl, today, warnDays, notices, log }) {
+  if (prompts.length > MAX_PROMPTS) {
+    notices.push(`Only the first ${MAX_PROMPTS} prompts were tested.`);
+    prompts = prompts.slice(0, MAX_PROMPTS);
+  }
+  // Models in code (not docs) that are going away and have a known replacement, soonest first.
+  const flaggedModels = [...new Set(findings
+    .filter((f) => !f.doc && FLAGGED.has(f.status) && f.entry?.replacement && isTextModel(f.entry))
+    .sort((a, b) => a.days - b.days)
+    .map((f) => f.id))];
+  // A prompt without "model" is tested once, on the model that retires first: two API calls per prompt.
+  if (flaggedModels.length > 1 && prompts.some((p) => !p.model)) {
+    notices.push(`Prompts without "model" were tested on ${flaggedModels[0]}, the model that retires first. Set "model" on a prompt to test it on another one.`);
+  }
+  const out = [];
+
+  for (const p of prompts) {
+    const froms = p.model ? [p.model] : flaggedModels.slice(0, 1);
+    if (!froms.length) {
+      out.push({ id: p.id, from: "—", to: "—", result: "skipped", note: "No retiring model found and no \"model\" set for this prompt" });
+      continue;
+    }
+    for (const from of froms) {
+      const status = classify(from, registry, { today, warnDays });
+      const to = p.compare_to || status.entry?.replacement;
+      if (!to) {
+        out.push({ id: p.id, from, to: "—", result: "skipped", note: "No known replacement; set compare_to" });
+        continue;
+      }
+      const missing = [from, to].map(providerOf).find((pr) => !keyFor(pr, env));
+      if (missing) {
+        out.push({ id: p.id, from, to, result: "error", note: missingKeyMessage(missing) });
+        continue;
+      }
+      log(oneLine(`Drift test ${p.id}: ${from} → ${to}`));
+      try {
+        const after = await complete({ model: to, prompt: p.prompt, system: p.system, env, fetchImpl });
+        if (status.status === "retired") {
+          if (!after.trim()) {
+            out.push({ id: p.id, from, to, after, before: "(current model already retired)", result: "broken", note: "Replacement returned an empty answer" });
+            continue;
+          }
+          const jsonOk = p.expect !== "json" || extractJson(after).ok;
+          out.push({ id: p.id, from, to, after, before: "(current model already retired)", result: jsonOk ? "changed" : "broken",
+            note: jsonOk ? "Current model is retired; review the replacement's answer" : "Replacement did not return valid JSON" });
+          continue;
+        }
+        let before;
+        try {
+          before = await complete({ model: from, prompt: p.prompt, system: p.system, env, fetchImpl });
+        } catch (e) {
+          if (!/^(Empty answer|Answer cut off)/.test(e.message)) throw e;
+          // The current model gave nothing usable; the replacement's answer is still worth showing.
+          out.push({ id: p.id, from, to, after, before: `(${e.message})`, result: "changed", note: `Current model: ${e.message}. Review the replacement's answer` });
+          continue;
+        }
+        out.push({ id: p.id, from, to, before, after, ...compare(before, after, p.expect) });
+      } catch (e) {
+        out.push({ id: p.id, from, to, result: "error", note: e.message.slice(0, 200) });
+      }
+    }
+  }
+  return out;
+}

package/src/scan.js

@@ -0,0 +1,82 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join, relative, extname, sep } from "node:path";
+import { normalizeId, looksLikeModelId } from "./registry.js";
+
+const SKIP_DIRS = new Set([
+  ".git", "node_modules", "vendor", "dist", "build", "out", ".next", ".nuxt",
+  "coverage", "__pycache__", ".venv", "venv", "target", ".turbo", ".cache",
+]);
+const EXTENSIONS = new Set([
+  ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".py", ".rb", ".go", ".rs", ".java",
+  ".kt", ".php", ".cs", ".swift", ".json", ".yaml", ".yml", ".toml", ".env", ".ini",
+  ".cfg", ".md", ".txt", ".sh", ".tf", ".vue", ".svelte", ".astro", ".mts", ".cts", ".html",
+  ".jsonc", ".ipynb",
+]);
+const MAX_BYTES = 1024 * 1024;
+// Mentions in docs are reported but never fail the build (changelogs name old models on purpose).
+const DOC_EXTENSIONS = new Set([".md", ".txt"]);
+export const isDocFile = (file) => DOC_EXTENSIONS.has(extname(file).toLowerCase());
+
+// Quoted strings: "gpt-4o", 'claude-3-haiku-20240307', `gemini-2.0-flash`
+const QUOTED = /(["'`])([A-Za-z0-9][\w.\-:/@]{1,160})\1/g;
+// Unquoted config values: MODEL=gpt-4o  or  model: claude-sonnet-4-5-20250929
+// Bounded repeats keep this linear on long lines (minified files, generated data).
+const ASSIGNED = /(?:^|[\s,{])[\w.\-]{0,40}model[\w.\-]{0,40}\s*[=:]\s*([A-Za-z0-9][\w.\-/]{1,80})\s*(?:#.*)?$/i;
+const MAX_ASSIGNED_LINE = 2000;
+// "o1", "o3": too short to trust without some sign the line is about a model.
+const BARE_SHORT = /^o\d$/;
+const MODEL_CONTEXT = /model|engine|llm|openai|reasoning/i;
+
+export function* walk(root, dir = root) {
+  let entries;
+  try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+  for (const e of entries) {
+    const full = join(dir, e.name);
+    if (e.isDirectory()) {
+      if (!SKIP_DIRS.has(e.name)) yield* walk(root, full);
+    } else if (e.isFile()) {
+      const ext = extname(e.name).toLowerCase();
+      const isEnv = e.name.startsWith(".env");
+      const isDocker = /^dockerfile/i.test(e.name) || ext === ".dockerfile";
+      if (!EXTENSIONS.has(ext) && !isEnv && !isDocker) continue;
+      if (e.name === "package-lock.json" || e.name.endsWith(".lock")) continue;
+      try { if (statSync(full).size > MAX_BYTES) continue; } catch { continue; }
+      yield { full, rel: relative(root, full) };
+    }
+  }
+}
+
+export function scanText(text, file, known) {
+  const hits = [];
+  const doc = isDocFile(file);
+  const lines = text.split(/\r?\n/);
+  lines.forEach((raw, i) => {
+    // Escaped quotes (\"gpt-4\") appear in notebooks and JSON-in-JSON; read them as plain quotes.
+    const line = raw.replace(/\\(["'`])/g, "$1");
+    const seen = new Set();
+    for (const m of line.matchAll(QUOTED)) seen.add(m[2]);
+    if (line.length <= MAX_ASSIGNED_LINE && /model/i.test(line)) {
+      const a = line.match(ASSIGNED);
+      if (a) seen.add(a[1]);
+    }
+    const ids = [...seen].map(normalizeId).filter((id) => looksLikeModelId(id, known));
+    const context = MODEL_CONTEXT.test(line) || ids.some((id) => !BARE_SHORT.test(id)); // e.g. ["o1", "gpt-4o"]
+    for (const id of ids) {
+      if (BARE_SHORT.test(id) && !context) continue;
+      hits.push({ id, file, line: i + 1, doc, text: line.trim().slice(0, 160) });
+    }
+  });
+  return hits;
+}
+
+export function scanDirectory(root, { ignore = [], known } = {}) {
+  const hits = [];
+  for (const { full, rel } of walk(root)) {
+    if (ignore.some((p) => rel.split(sep).join("/").startsWith(p))) continue;
+    let text;
+    try { text = readFileSync(full, "utf8"); } catch { continue; }
+    if (text.includes("\u0000")) continue; // binary
+    hits.push(...scanText(text, rel, known));
+  }
+  return hits;
+}