mixdog 0.9.4 → 0.9.5

package/scripts/bench/round-r36-orchestrated-capped.json

@@ -0,0 +1,128 @@
+{
+  "round": "36",
+  "runner": "lead",
+  "opts": {
+    "provider": "openai-oauth",
+    "model": "gpt-5.5",
+    "effort": null,
+    "fast": false,
+    "workflow": "default",
+    "env": {
+      "MIXDOG_DATA_DIR": "C:\\Users\\tempe\\.mixdog\\bench-data"
+    }
+  },
+  "tasks": 1,
+  "completed": 1,
+  "completion_rate": 100,
+  "sessions": [
+    "sess_54872_1783078178435_1783078178685_d27922b6978571ab4599bcc38f64fa0f"
+  ],
+  "results": [
+    {
+      "id": "r5-orchestrated-audit",
+      "agent": "lead",
+      "ok": true,
+      "ms": 255448,
+      "sessionId": "sess_54872_1783078178435_1783078178685_d27922b6978571ab4599bcc38f64fa0f",
+      "output": "sessionId: sess_54872_1783078178435_1783078178685_d27922b6978571ab4599bcc38f64fa0f\n완료했습니다. 분석만 수행했고 파일은 수정하지 않았습니다.\n\n**Scope A — classifier bucket reachability**\n\n- `classifyError()`는 `auth|permanent|transient|unknown`만 반환합니다(`retry-classifier.mjs:100-135`). `openai-oauth-ws.mjs`는 이 함수를 import하지 않으므로 WS 경로에서는 직접 도달하지 않지만(`openai-oauth-ws.mjs:33-40`), 전역적으로는 `withRetry()`가 사용합니다(`retry-classifier.mjs:515-570`).\n\n- `classifyHandshakeError()`는 `timeout|reset|dns|refused|network|acquire_timeout|http_${status}|null`을 반환합니다(`retry-classifier.mjs:471-494`). `openai-oauth-ws`는 `_classifyHandshakeError()`로 래핑해 acquire 실패마다 소비합니다(`openai-oauth-ws.mjs:112-114`, `openai-oauth-ws.mjs:403-430`).\n\n- WS midstream classifier는 `first_byte_timeout|ws_send_failed|agent_stall|stream_stalled|ws_1006|ws_1011|ws_1012|ws_4000|ws_1000|http_${status}|response_failed_disconnected|response_failed_network|response_failed_auth_expired|null`을 반환합니다(`retry-classifier.mjs:286-343`). `openai-oauth-ws`는 항상 `mode:'ws'`로만 호출합니다(`openai-oauth-ws.mjs:87-91`, `openai-oauth-ws.mjs:157-159`).\n\n- SSE midstream bucket `agent_stall|stream_stalled|reset|timeout|dns|sse_idle_timeout|stream_terminated|fetch_failed`는 `retry-classifier.mjs` 안에는 존재하지만(`retry-classifier.mjs:348-381`), `openai-oauth-ws` 경로에서는 `mode:'sse'`를 넘기지 않아 도달 불가입니다.\n\n- “생성 불가”로 볼 수 있는 것은 현재 WS classifier가 반환하지 않는 레거시/문서성 `http_5xx` 문자열입니다. 실제 반환은 `http_500` 같은 `http_${status}`이고(`retry-classifier.mjs:309`, `retry-classifier.mjs:480-482`), fallback 쪽만 `http_5xx`와 regex 둘 다 허용합니다(`retry-classifier.mjs:386-410`).\n\n- `response_failed_auth_expired`는 repo 내부가 문자열을 생성하진 않지만, 서버의 `response.failed`/failed `response.done` payload를 그대로 저장해 classifier가 sniff하므로 런타임상 완전한 dead bucket은 아닙니다(`openai-ws-stream.mjs:887-896`, `openai-ws-stream.mjs:953-963`, `retry-classifier.mjs:332-339`).\n\n**Scope B — retry loop budgets/backoff**\n\n- Handshake retry는 `HANDSHAKE_MAX_ATTEMPTS = PROVIDER_RETRY_MAX_ATTEMPTS = 5`라서 최초 1회 + 최대 4회 재시도입니다(`openai-oauth-ws.mjs:104`, `stall-policy.mjs:200-201`). backoff는 `500ms * 2^(attempt-1)`, cap `5000ms`, `jitterDelayMs()` 적용입니다(`openai-oauth-ws.mjs:175-178`, `stall-policy.mjs:202-206`).\n\n- Midstream retry는 outer loop가 `attemptIndex <= 4`까지 돌 수 있지만(`openai-oauth-ws.mjs:514`, `openai-oauth-ws.mjs:557`), 실제 허용 횟수는 classifier별입니다. `ws_1006|ws_1011`은 4회, 나머지 WS bucket은 2회입니다(`retry-classifier.mjs:245-268`, `openai-oauth-ws.mjs:164-168`, `openai-oauth-ws.mjs:843-860`).\n\n- Midstream backoff는 retry 번호별 `[250,1000,2000,4000]`를 쓰고 jitter를 적용합니다(`retry-classifier.mjs:245-247`, `openai-oauth-ws.mjs:170-173`). 단, 로컬 주석은 “two retries / one retry”라고 되어 있어 실제 4/2 budget과 불일치합니다(`openai-oauth-ws.mjs:503-508`).\n\n- 같은 fault가 두 계층에서 이중 재시도되지는 않습니다. acquire 실패는 midstream classifier로 넘어가지 않고 바로 throw되며(`openai-oauth-ws.mjs:578-601`), midstream 실패만 `_classifyMidstreamError()` 후 재시도되고 다음 시도에서 fresh acquire를 새로 수행합니다(`openai-oauth-ws.mjs:511-513`, `openai-oauth-ws.mjs:564-577`, `openai-oauth-ws.mjs:843-860`).\n\n- 다만 한 사용자 요청 안에서 “midstream fault 재시도 → 새 acquire 중 handshake fault 재시도”는 가능하므로, 두 budget은 같은 error class에 중첩되는 것이 아니라 서로 다른 단계의 fault에 순차적으로 소비됩니다.",
+      "card": {
+        "session": "sess_54872_1783078178435_1783078178685_d27922b6978571ab4599bcc38f64fa0f",
+        "provider": "openai-oauth",
+        "model": "gpt-5.5",
+        "wall_ms": 244855,
+        "turns": 44,
+        "tool_calls": 42,
+        "tools_per_turn": 1,
+        "total_tool_ms": 2277,
+        "llm_stream_ms": 305080,
+        "cache_ratio": 0.7561787493012467,
+        "cached_tokens": 597906,
+        "prompt_tokens": 790694,
+        "uncached_tokens": 35297,
+        "output_tokens": 14439,
+        "cache_weighted_input_10": 95087.6,
+        "cache_weighted_input_25": 184773.5,
+        "prompt_growth": 33692,
+        "antipatterns": 4,
+        "issues": 22,
+        "issue_types": [
+          "cache",
+          "cache_break",
+          "cache_miss",
+          "readonly_stall",
+          "broad_tool_result",
+          "service_tier",
+          "read_fragmentation",
+          "missed_parallelism",
+          "tool_churn_cluster"
+        ],
+        "read_fragmentation_paths": [
+          "src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs"
+        ],
+        "grep_sweep_count": 0
+      }
+    }
+  ],
+  "task_complete": true,
+  "task_errors": [],
+  "score_complete": true,
+  "score_errors": [],
+  "group": {
+    "n": 1,
+    "wall_ms": 244855,
+    "turns": 44,
+    "tool_calls": 42,
+    "tools_per_turn": 1,
+    "total_tool_ms": 2277,
+    "llm_stream_ms": 305080,
+    "cache_ratio": 0.8,
+    "cached_tokens": 597906,
+    "prompt_tokens": 790694,
+    "uncached_tokens": 35297,
+    "output_tokens": 14439,
+    "cache_weighted_input_10": 95087.6,
+    "cache_weighted_input_25": 184773.5,
+    "prompt_growth": 33692,
+    "antipatterns": 4,
+    "issues": 22,
+    "grep_sweep_count": 0
+  },
+  "cards": [
+    {
+      "session": "sess_54872_1783078178435_1783078178685_d27922b6978571ab4599bcc38f64fa0f",
+      "provider": "openai-oauth",
+      "model": "gpt-5.5",
+      "wall_ms": 244855,
+      "turns": 44,
+      "tool_calls": 42,
+      "tools_per_turn": 1,
+      "total_tool_ms": 2277,
+      "llm_stream_ms": 305080,
+      "cache_ratio": 0.7561787493012467,
+      "cached_tokens": 597906,
+      "prompt_tokens": 790694,
+      "uncached_tokens": 35297,
+      "output_tokens": 14439,
+      "cache_weighted_input_10": 95087.6,
+      "cache_weighted_input_25": 184773.5,
+      "prompt_growth": 33692,
+      "antipatterns": 4,
+      "issues": 22,
+      "issue_types": [
+        "cache",
+        "cache_break",
+        "cache_miss",
+        "readonly_stall",
+        "broad_tool_result",
+        "service_tier",
+        "read_fragmentation",
+        "missed_parallelism",
+        "tool_churn_cluster"
+      ],
+      "read_fragmentation_paths": [
+        "src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs"
+      ],
+      "grep_sweep_count": 0
+    }
+  ]
+}

package/scripts/bench-run.mjs

@@ -24,6 +24,7 @@ import { dirname, resolve } from 'node:path';
 
 const __dir = dirname(fileURLToPath(import.meta.url));
 const HEADLESS = pathToFileURL(resolve(__dir, '../src/headless-role.mjs')).href;
+const WORKFLOW_MOD = pathToFileURL(resolve(__dir, '../src/session-runtime/workflow.mjs')).href;
 const TASK_BENCH = resolve(__dir, 'task-bench.mjs');
 
 function argValue(name, fallback = null) {
@@ -65,11 +66,15 @@ const RUNNERS = {
     const RUNTIME_URL = pathToFileURL(resolve(__dir, '../src/mixdog-session-runtime.mjs')).href;
     const driver = [
       `import { createMixdogSessionRuntime } from ${JSON.stringify(RUNTIME_URL)};`,
+      `import { normalizeWorkflowId } from ${JSON.stringify(WORKFLOW_MOD)};`,
       `process.chdir(${JSON.stringify(task.cwd ? resolve(task.cwd) : process.cwd())});`,
       `const rt = await createMixdogSessionRuntime({`,
       `  provider: ${JSON.stringify(opts.provider || null)} || undefined,`,
       `  model: ${JSON.stringify(opts.model || null)} || undefined,`,
       `});`,
+      `if (${JSON.stringify(opts.workflow || null)}) {`,
+      `  await rt.setWorkflow(normalizeWorkflowId(${JSON.stringify(opts.workflow || null)}));`,
+      `}`,
       `let text = '';`,
       `// Mirror the TUI auto-resume EXACTLY (engine/agent-job-feed.mjs):`,
       `// runtime notifications enqueue the model-visible completion into the`,
@@ -145,7 +150,6 @@ const RUNNERS = {
         maxBuffer: 64 * 1024 * 1024,
         env: {
           ...process.env,
-          ...(opts.effort ? { MIXDOG_AGENT_EFFORT: opts.effort } : {}),
           ...(opts.env || {}),
         },
       }, (err, stdout, stderrOut) => {
@@ -164,6 +168,8 @@ const RUNNERS = {
       `  message: ${JSON.stringify(task.prompt || '')},`,
       `  provider: ${JSON.stringify(opts.provider || null)},`,
       `  model: ${JSON.stringify(opts.model || null)},`,
+      `  effort: ${JSON.stringify(opts.effort || null)},`,
+      `  fast: ${JSON.stringify(opts.fast === true)},`,
       `  cwd: ${JSON.stringify(task.cwd ? resolve(task.cwd) : process.cwd())},`,
       `  write: (t) => out.push(t),`,
       `  writeErr: (t) => process.stderr.write(t),`,
@@ -178,8 +184,6 @@ const RUNNERS = {
         maxBuffer: 64 * 1024 * 1024,
         env: {
           ...process.env,
-          ...(opts.effort ? { MIXDOG_AGENT_EFFORT: opts.effort } : {}),
-          ...(opts.fast ? { MIXDOG_AGENT_FAST: '1' } : {}),
           ...(opts.env || {}),
         },
       }, (err, stdout, stderrOut) => {
@@ -198,6 +202,7 @@ const RUNNERS = {
       '-C', task.cwd ? resolve(task.cwd) : process.cwd()];
     if (opts.model) args.push('-m', opts.model);
     if (opts.effort) args.push('-c', `model_reasoning_effort="${opts.effort}"`);
+    if (opts.fast) args.push('-c', 'service_tier="fast"');
     args.push('-');
     const started = Date.now();
     const { raw, ok } = await new Promise((resolveRun) => {
@@ -241,6 +246,7 @@ const RUNNERS = {
       out_tokens: usage.out_tokens,
       cache_ratio: totalIn > 0 ? Math.round((usage.cached_in / totalIn) * 1000) / 1000 : 0,
       completed: ok ? 1 : 0,
+      usd_cost: usdCost(opts.model, usage.uncached_in, usage.cached_in, usage.out_tokens),
     };
     return { sessionId: card.session, ok, ms, raw, card, finalMessage };
   },
@@ -290,6 +296,23 @@ function num(v) {
   return Number.isFinite(n) ? n : 0;
 }
 
+// USD list pricing per 1M tokens (input / cached input / output), 2026-07.
+// Cross-harness cost comparison needs a real currency: OAuth lanes bill
+// quota, not dollars, but token counts priced at API list rates are the
+// only harness-neutral metric. Unknown models fall back to gpt-5.5 rates.
+const USD_PER_1M = {
+  'gpt-5.5': { in: 5.00, cachedIn: 0.50, out: 30.00 },
+  'claude-sonnet-5': { in: 2.00, cachedIn: 0.20, out: 10.00 },
+  'claude-fable-5': { in: 5.00, cachedIn: 0.50, out: 25.00 },
+};
+function usdCost(model, uncachedIn, cachedIn, outTokens) {
+  const key = String(model || '').toLowerCase();
+  const p = USD_PER_1M[key]
+    || USD_PER_1M[Object.keys(USD_PER_1M).find((k) => key.includes(k))]
+    || USD_PER_1M['gpt-5.5'];
+  return Math.round(((num(uncachedIn) * p.in + num(cachedIn) * p.cachedIn + num(outTokens) * p.out) / 1e6) * 10000) / 10000;
+}
+
 function averageCards(cards) {
   if (!cards.length) return null;
   const keys = Object.keys(cards[0]).filter((k) => cards.some((c) => typeof c[k] === 'number' && Number.isFinite(c[k])));
@@ -334,9 +357,10 @@ const opts = {
   model: _mo.model,
   effort: argValue('--effort', null),
   fast: hasFlag('--fast'),
+  workflow: argValue('--workflow', null),
   env: _env,
 };
-process.stderr.write(`[bench-run] model=${opts.model || '(agent default)'} provider=${opts.provider || '(agent default)'} effort=${opts.effort || '-'} fast=${opts.fast} env=${JSON.stringify(_env)}\n`);
+process.stderr.write(`[bench-run] model=${opts.model || '(agent default)'} provider=${opts.provider || '(agent default)'} effort=${opts.effort || '-'} fast=${opts.fast} workflow=${opts.workflow || '-'} env=${JSON.stringify(_env)}\n`);
 
 // --parallel N: run up to N tasks concurrently (default: all). Scoring stays
 // sequential after runs so task-bench reads a settled trace.
@@ -344,7 +368,11 @@ const parallel = Math.max(1, Number.parseInt(argValue('--parallel', String(tasks
 // --repeat N: run the whole task set N times (repeat r appended to task id as
 // "id#r") so group averages settle over more samples.
 const repeat = Math.max(1, Number.parseInt(argValue('--repeat', '1'), 10) || 1);
-process.stderr.write(`[bench-run] parallel=${parallel} repeat=${repeat}\n`);
+// --stagger MS: delay each parallel lane's start by laneIndex*MS so session
+// first-calls don't race the server prompt-cache write window (A/B for
+// parallel fan-out cache misses).
+const staggerMs = Math.max(0, Number.parseInt(argValue('--stagger', '0'), 10) || 0);
+process.stderr.write(`[bench-run] parallel=${parallel} repeat=${repeat} stagger=${staggerMs}ms\n`);
 const runList = [];
 for (let r = 1; r <= repeat; r += 1) {
   for (const task of tasks) {
@@ -373,14 +401,15 @@ async function runTask(task) {
 const results = new Array(runList.length);
 {
   let next = 0;
-  async function drain() {
+  async function drain(lane) {
+    if (staggerMs > 0 && lane > 0) await new Promise((res) => setTimeout(res, lane * staggerMs));
     for (;;) {
       const i = next++;
       if (i >= runList.length) return;
       results[i] = await runTask(runList[i]);
     }
   }
-  await Promise.all(Array.from({ length: Math.min(parallel, runList.length) }, drain));
+  await Promise.all(Array.from({ length: Math.min(parallel, runList.length) }, (_, lane) => drain(lane)));
 }
 
 for (const result of results) {
@@ -389,6 +418,10 @@ for (const result of results) {
   const scored = scoreSession(result.sessionId, { env: opts.env });
   if (scored.ok) {
     result.card = scored.card;
+    // task-bench cards expose uncached/cached/output token fields; price at
+    // the session's own model when present, else the round's --model.
+    const c = scored.card;
+    c.usd_cost = usdCost(c.model || opts.model, c.uncached_tokens, c.cached_tokens, c.output_tokens);
     process.stderr.write(`[bench-run]   -> scored ${result.sessionId.slice(0, 22)} turns=${scored.card.turns} tools=${scored.card.tool_calls}\n`);
   } else {
     result.scoreError = scored.error;

package/scripts/internal-comms-bench.mjs

@@ -212,6 +212,8 @@ function runHeadlessWorker({ pluginRoot, dataDir, taskCwd, prompt, provider, mod
     `  message: ${JSON.stringify(prompt)},`,
     `  provider: ${JSON.stringify(provider || null)},`,
     `  model: ${JSON.stringify(model || null)},`,
+    `  effort: ${JSON.stringify(effort || null)},`,
+    `  fast: ${JSON.stringify(fast === true)},`,
     `  cwd: ${JSON.stringify(taskCwd)},`,
     `  write: (t) => out.push(t),`,
     `  writeErr: (t) => process.stderr.write(t),`,
@@ -230,8 +232,6 @@ function runHeadlessWorker({ pluginRoot, dataDir, taskCwd, prompt, provider, mod
         ...process.env,
         MIXDOG_ROOT: pluginRoot,
         MIXDOG_DATA_DIR: dataDir,
-        ...(effort ? { MIXDOG_AGENT_EFFORT: effort } : {}),
-        ...(fast ? { MIXDOG_AGENT_FAST: '1' } : {}),
       },
     });
     ok = true;
@@ -478,8 +478,6 @@ function runLiveLeadDelegation({ pluginRoot, dataDir, taskCwd, prompt, provider,
         ...process.env,
         MIXDOG_ROOT: pluginRoot,
         MIXDOG_DATA_DIR: dataDir,
-        ...(effort ? { MIXDOG_AGENT_EFFORT: effort } : {}),
-        ...(fast ? { MIXDOG_AGENT_FAST: '1' } : {}),
       },
     });
     const j = raw.lastIndexOf('{');

package/scripts/model-catalog-audit.mjs

@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+import { loadConfig } from '../src/runtime/agent/orchestrator/config.mjs';
+import { initProviders, getAllProviders } from '../src/runtime/agent/orchestrator/providers/registry.mjs';
+import {
+  loadModelsDevCatalog,
+  warmModelMetadataCatalogs,
+} from '../src/runtime/agent/orchestrator/providers/model-catalog.mjs';
+import { QUICK_SEARCH_MODELS } from '../src/session-runtime/quick-search-models.mjs';
+
+const args = new Set(process.argv.slice(2));
+const json = args.has('--json');
+const allRows = args.has('--all');
+const refresh = !args.has('--no-refresh');
+
+const PROVIDER_ALIAS = Object.freeze({
+  'openai-oauth': 'openai',
+  openai: 'openai',
+  'grok-oauth': 'xai',
+  xai: 'xai',
+  gemini: 'google',
+  google: 'google',
+  'anthropic-oauth': 'anthropic',
+  anthropic: 'anthropic',
+});
+
+const LIT_PREFIXES = Object.freeze({
+  openai: ['openai/'],
+  xai: ['xai/'],
+  google: ['gemini/', 'google/'],
+  anthropic: ['anthropic/'],
+  deepseek: ['deepseek/'],
+});
+
+function n(value) {
+  const num = Number(value);
+  return Number.isFinite(num) && num > 0 ? Math.floor(num) : null;
+}
+
+function contextLabel(value) {
+  const v = n(value);
+  if (!v) return '-';
+  if (v >= 1_000_000) {
+    const m = v / 1_000_000;
+    const label = Number.isInteger(m)
+      ? m.toFixed(0)
+      : m.toFixed(3).replace(/0+$/, '').replace(/\.$/, '');
+    return `${label}M`;
+  }
+  return `${Math.round(v / 1000)}k`;
+}
+
+function isInteresting(id) {
+  const s = String(id || '').toLowerCase();
+  return /^(gpt-5|gpt-4\.1|claude-|gemini-|grok-|deepseek-v4|glm-|qwen|minimax|kimi)/.test(s);
+}
+
+function litellmRow(litellm, provider, id) {
+  const p = PROVIDER_ALIAS[provider] || provider;
+  const keys = [id, ...(LIT_PREFIXES[p] || []).map(prefix => prefix + id)];
+  if (p === 'anthropic') keys.push(`anthropic.${id}-v1:0`, `bedrock/anthropic.${id}-v1:0`);
+  for (const key of keys) {
+    if (litellm?.[key]) return { key, row: litellm[key] };
+  }
+  return null;
+}
+
+function externalMeta(modelsDev, litellm, provider, id) {
+  const p = PROVIDER_ALIAS[provider] || provider;
+  const md = modelsDev?.[p]?.models?.[id] || null;
+  const lit = litellmRow(litellm, provider, id);
+  return {
+    modelsDevContext: n(md?.limit?.context),
+    modelsDevOutput: n(md?.limit?.output),
+    litellmContext: n(lit?.row?.max_input_tokens ?? lit?.row?.max_tokens),
+    litellmOutput: n(lit?.row?.max_output_tokens),
+    externalName: md?.name || null,
+    litellmKey: lit?.key || null,
+  };
+}
+
+function classifyOneMillion(value) {
+  const v = n(value);
+  if (!v) return 'unknown';
+  if (v === 1_000_000) return 'exact-1M';
+  if (v > 900_000 && v < 1_100_000) return 'near-1M-not-exact';
+  return 'not-1M';
+}
+
+async function listProviderModels(name, provider) {
+  let models = null;
+  let refreshed = false;
+  let error = null;
+  try {
+    if (refresh && typeof provider?._refreshModelCache === 'function') {
+      models = await provider._refreshModelCache();
+      refreshed = true;
+    }
+    if (!Array.isArray(models) && typeof provider?.listModels === 'function') {
+      models = await provider.listModels();
+    }
+  } catch (err) {
+    error = err?.message || String(err);
+  }
+  return { provider: name, refreshed, error, models: Array.isArray(models) ? models : [] };
+}
+
+const cfg = loadConfig();
+try {
+  await initProviders(cfg.providers || {});
+} catch (err) {
+  if (!json) console.error(`[model-audit] init failed: ${err?.message || err}`);
+}
+
+const [modelsDev, litellm] = await Promise.all([
+  loadModelsDevCatalog().catch(() => ({})),
+  warmModelMetadataCatalogs().catch(() => ({})),
+]);
+
+const providerResults = [];
+for (const [name, provider] of getAllProviders()) {
+  if (typeof provider?.listModels !== 'function') continue;
+  const listed = await listProviderModels(name, provider);
+  const rows = listed.models
+    .filter(row => row?.id && (allRows || isInteresting(row.id)))
+    .map(row => {
+      const liveContext = n(row.contextWindow ?? row.maxContextWindow);
+      const liveOutput = n(row.outputTokens);
+      const external = externalMeta(modelsDev, litellm, name, row.id);
+      const externalContext = external.modelsDevContext ?? external.litellmContext;
+      const externalOutput = external.modelsDevOutput ?? external.litellmOutput;
+      return {
+        provider: name,
+        id: row.id,
+        display: row.display || row.name || null,
+        liveContext,
+        liveContextLabel: contextLabel(liveContext),
+        liveOutput,
+        externalContext,
+        externalContextLabel: contextLabel(externalContext),
+        externalOutput,
+        contextVerdict: classifyOneMillion(liveContext),
+        contextMismatch: liveContext != null && externalContext != null && liveContext !== externalContext,
+        outputMismatch: liveOutput != null && externalOutput != null && liveOutput !== externalOutput,
+        source: listed.refreshed ? 'live-refresh' : 'live-list',
+        ...external,
+      };
+    });
+  providerResults.push({
+    provider: name,
+    refreshed: listed.refreshed,
+    error: listed.error,
+    count: listed.models.length,
+    rows,
+  });
+}
+
+const quickRows = [];
+for (const [provider, models] of Object.entries(QUICK_SEARCH_MODELS)) {
+  for (const model of models) {
+    if (model?.contextWindow == null) continue;
+    const external = externalMeta(modelsDev, litellm, provider, model.id);
+    const externalContext = external.modelsDevContext ?? external.litellmContext;
+    quickRows.push({
+      provider,
+      id: model.id,
+      staticContext: n(model.contextWindow),
+      externalContext,
+      mismatch: n(model.contextWindow) != null && externalContext != null && n(model.contextWindow) !== externalContext,
+    });
+  }
+}
+
+const mismatches = providerResults.flatMap(p => p.rows.filter(r => r.contextMismatch || r.outputMismatch));
+const exactOneM = providerResults.flatMap(p => p.rows.filter(r => r.liveContext === 1_000_000));
+const nearOneM = providerResults.flatMap(p => p.rows.filter(r => r.contextVerdict === 'near-1M-not-exact'));
+
+const report = {
+  generatedAt: new Date().toISOString(),
+  enabledProviders: Object.entries(cfg.providers || {}).filter(([, v]) => v?.enabled).map(([k]) => k),
+  providerResults,
+  mismatches,
+  exactOneM,
+  nearOneM,
+  quickStaticContextRows: quickRows,
+};
+
+if (json) {
+  console.log(JSON.stringify(report, null, 2));
+} else {
+  console.log(`model catalog audit @ ${report.generatedAt}`);
+  console.log(`enabled providers: ${report.enabledProviders.join(', ') || '-'}`);
+  for (const p of providerResults) {
+    console.log(`\n[${p.provider}] count=${p.count} refreshed=${p.refreshed}${p.error ? ` error=${p.error}` : ''}`);
+    for (const r of p.rows) {
+      const flags = [
+        r.contextMismatch ? `context ${r.liveContextLabel} != external ${r.externalContextLabel}` : '',
+        r.outputMismatch ? `output ${r.liveOutput} != external ${r.externalOutput}` : '',
+      ].filter(Boolean).join('; ');
+      console.log(`  ${r.id}: ${r.liveContextLabel} ctx, out=${r.liveOutput ?? '-'}${flags ? `  [${flags}]` : ''}`);
+    }
+  }
+  console.log(`\nsummary: mismatches=${mismatches.length}, exact1M=${exactOneM.length}, near1MNotExact=${nearOneM.length}, staticContextRows=${quickRows.length}`);
+  if (mismatches.length) {
+    console.log('mismatches:');
+    for (const r of mismatches.slice(0, 50)) {
+      console.log(`  ${r.provider}/${r.id}: live=${r.liveContextLabel}/${r.liveOutput ?? '-'} external=${r.externalContextLabel}/${r.externalOutput ?? '-'}`);
+    }
+  }
+}

package/scripts/model-list-sanitize-test.mjs

@@ -0,0 +1,37 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { sanitizeModelList } from '../src/runtime/agent/orchestrator/providers/model-list-sanitize.mjs';
+
+test('Anthropic staleness keeps Claude subfamilies separate', () => {
+  const prev = process.env.MIXDOG_MODEL_STALE_MONTHS;
+  process.env.MIXDOG_MODEL_STALE_MONTHS = '0';
+  try {
+    const models = [
+      { id: 'claude-opus-4-8', mode: 'chat' },
+      { id: 'claude-sonnet-4-6', mode: 'chat' },
+      { id: 'claude-haiku-4-5-20251001', mode: 'chat' },
+    ];
+    const catalogRow = (release_date) => ({
+      family: 'claude',
+      release_date,
+      tool_call: true,
+      modalities: { output: ['text'] },
+    });
+    const out = sanitizeModelList(models, {
+      provider: 'anthropic',
+      _testCatalog: {
+        anthropic: {
+          models: {
+            'claude-opus-4-8': catalogRow('2025-12-01'),
+            'claude-sonnet-4-6': catalogRow('2025-11-01'),
+            'claude-haiku-4-5-20251001': catalogRow('2025-10-15'),
+          },
+        },
+      },
+    });
+    assert.deepEqual(out.map((m) => m.id), models.map((m) => m.id));
+  } finally {
+    if (prev == null) delete process.env.MIXDOG_MODEL_STALE_MONTHS;
+    else process.env.MIXDOG_MODEL_STALE_MONTHS = prev;
+  }
+});