@wrongstack/cli 0.295.1 → 0.296.2

package/dist/modeldiag-HENYJ4N7.js

@@ -0,0 +1,1290 @@
+import {
+  discoverAndMergeProviders
+} from "./chunk-TGDHN4LM.js";
+import {
+  mutateConfigProviders,
+  normalizeKeys,
+  writeKeysBack
+} from "./chunk-PJHET45A.js";
+import {
+  activeProfileConfigPath
+} from "./chunk-DPJZQCVP.js";
+import "./chunk-7OCVIDC7.js";
+
+// src/subcommands/handlers/modeldiag.ts
+import * as fs from "node:fs/promises";
+import { color as color2 } from "@wrongstack/core/utils";
+import { toErrorMessage as toErrorMessage2 } from "@wrongstack/core/utils";
+import { makeProviderFromConfig, setOAuthTokenPersister } from "@wrongstack/providers";
+
+// src/subcommands/handlers/model-smoke-test.ts
+import { ProviderError } from "@wrongstack/core/types";
+import { toErrorMessage } from "@wrongstack/core/utils";
+var DEFAULT_TIMEOUT_MS = 45e3;
+var DEFAULT_MAX_TOKENS = 32;
+function csv(value) {
+  if (typeof value !== "string") return void 0;
+  const values = value.split(",").map((part) => part.trim()).filter(Boolean);
+  return values.length > 0 ? values : void 0;
+}
+function positiveInteger(value, fallback, label) {
+  if (value === void 0) return fallback;
+  const parsed = typeof value === "string" ? Number(value) : Number.NaN;
+  if (!Number.isSafeInteger(parsed) || parsed <= 0) {
+    throw new Error(`${label} must be a positive integer.`);
+  }
+  return parsed;
+}
+function parseModelSmokeOptions(args, flags = {}) {
+  const local = /* @__PURE__ */ new Map();
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    if (!arg?.startsWith("--")) continue;
+    const eq = arg.indexOf("=");
+    if (eq >= 0) {
+      local.set(arg.slice(2, eq), arg.slice(eq + 1));
+      continue;
+    }
+    const name = arg.slice(2);
+    const next = args[i + 1];
+    if (next && !next.startsWith("--")) {
+      local.set(name, next);
+      i++;
+    } else {
+      local.set(name, true);
+    }
+  }
+  const get = (name) => local.get(name) ?? flags[name];
+  const providerFilter = csv(get("providers") ?? get("provider"));
+  const modelFilter = csv(get("models") ?? get("model"));
+  return {
+    allModels: get("all-models") === true || get("all-models") === "true",
+    planOnly: get("plan") === true || get("plan") === "true",
+    json: get("json") === true || get("json") === "true",
+    timeoutMs: positiveInteger(get("timeout"), DEFAULT_TIMEOUT_MS, "--timeout"),
+    maxTokens: positiveInteger(get("max-tokens"), DEFAULT_MAX_TOKENS, "--max-tokens"),
+    ...providerFilter ? { providerFilter } : {},
+    ...modelFilter ? { modelFilter } : {}
+  };
+}
+function pushUnique(target, values) {
+  const seen = new Set(target);
+  for (const value of values) {
+    const normalized = value?.trim();
+    if (!normalized || seen.has(normalized)) continue;
+    seen.add(normalized);
+    target.push(normalized);
+  }
+}
+async function catalogModelsFor(providerId, providerConfig, registry) {
+  const direct = await registry.getProvider(providerId).catch(() => void 0);
+  const inherited = direct ?? (providerConfig?.type && providerConfig.type !== providerId ? await registry.getProvider(providerConfig.type).catch(() => void 0) : void 0);
+  return inherited?.models ?? [];
+}
+function supportsTextCompletion(model) {
+  const outputs = model?.modalities?.output;
+  return !outputs?.length || outputs.includes("text");
+}
+async function buildModelSmokeTargets(config, registry, options) {
+  const providerIds = Object.keys(config.providers ?? {});
+  if (config.provider && !providerIds.includes(config.provider)) providerIds.push(config.provider);
+  const providerFilter = options.providerFilter ? new Set(options.providerFilter.map((id) => id.toLowerCase())) : void 0;
+  const modelFilter = options.modelFilter ? new Set(options.modelFilter) : void 0;
+  const targets = [];
+  for (const providerId of providerIds) {
+    if (providerFilter && !providerFilter.has(providerId.toLowerCase())) continue;
+    const saved = config.providers?.[providerId];
+    const catalogModels = await catalogModelsFor(providerId, saved, registry);
+    const catalogById = new Map(catalogModels.map((model) => [model.id, model]));
+    const textCatalogModels = catalogModels.filter(supportsTextCompletion);
+    const candidates = [];
+    if (options.allModels) {
+      pushUnique(candidates, saved?.models ?? []);
+      pushUnique(candidates, Object.keys(saved?.customModels ?? {}));
+      pushUnique(
+        candidates,
+        textCatalogModels.map((model) => model.id)
+      );
+      if (providerId === config.provider) pushUnique(candidates, [config.model]);
+      pushUnique(candidates, [saved?.model]);
+    } else {
+      const representative = (providerId === config.provider ? config.model : void 0) ?? saved?.model ?? saved?.models?.find(Boolean) ?? Object.keys(saved?.customModels ?? {})[0] ?? await registry.suggestModel(providerId).catch(() => void 0) ?? textCatalogModels[0]?.id;
+      pushUnique(candidates, [representative]);
+    }
+    for (const modelId of candidates) {
+      if (modelFilter && !modelFilter.has(modelId)) continue;
+      if (!supportsTextCompletion(catalogById.get(modelId))) continue;
+      targets.push({ providerId, modelId });
+    }
+  }
+  return targets;
+}
+function classifyFailure(err) {
+  if (err instanceof ProviderError || ProviderError.isProviderError(err)) {
+    const providerError = err;
+    return {
+      error: providerError.describe(),
+      errorKind: providerError.kind,
+      httpStatus: providerError.status
+    };
+  }
+  const isTimeout = err instanceof Error && (err.name === "TimeoutError" || err.name === "AbortError") || /timed?\s*out/i.test(toErrorMessage(err));
+  return {
+    error: toErrorMessage(err),
+    errorKind: isTimeout ? "timeout" : "unknown"
+  };
+}
+async function runModelSmokeTests(params) {
+  const results = [];
+  let providerId;
+  let provider;
+  let providerError;
+  for (let index = 0; index < params.targets.length; index++) {
+    const target = params.targets[index];
+    params.onTargetStart?.(target, index, params.targets.length);
+    const started = Date.now();
+    if (target.providerId !== providerId) {
+      providerId = target.providerId;
+      provider = void 0;
+      providerError = void 0;
+      try {
+        provider = await params.createProvider(target.providerId);
+      } catch (err) {
+        providerError = err;
+      }
+    }
+    if (!provider) {
+      const failure = classifyFailure(providerError ?? new Error("Provider could not be created."));
+      const result = {
+        providerId: target.providerId,
+        modelId: target.modelId,
+        status: "failed",
+        latencyMs: Date.now() - started,
+        ...failure
+      };
+      results.push(result);
+      params.onTargetComplete?.(result, index, params.targets.length);
+      continue;
+    }
+    try {
+      const response = await provider.complete(
+        {
+          model: target.modelId,
+          messages: [
+            {
+              role: "user",
+              content: [{ type: "text", text: "Reply with exactly: OK" }]
+            }
+          ],
+          maxTokens: params.options.maxTokens
+        },
+        { signal: AbortSignal.timeout(params.options.timeoutMs) }
+      );
+      const result = {
+        providerId: target.providerId,
+        modelId: target.modelId,
+        status: "passed",
+        latencyMs: Date.now() - started,
+        stopReason: response.stopReason,
+        usage: response.usage
+      };
+      results.push(result);
+      params.onTargetComplete?.(result, index, params.targets.length);
+    } catch (err) {
+      const result = {
+        providerId: target.providerId,
+        modelId: target.modelId,
+        status: "failed",
+        latencyMs: Date.now() - started,
+        ...classifyFailure(err)
+      };
+      results.push(result);
+      params.onTargetComplete?.(result, index, params.targets.length);
+    }
+  }
+  return results;
+}
+
+// src/subcommands/handlers/modeldiag-profiles.ts
+import { color } from "@wrongstack/core/utils";
+var MODEL_PROFILES = [
+  {
+    provider: "anthropic",
+    pattern: /claude-opus/i,
+    family: "Claude Opus",
+    strengths: ["reasoning", "planning"],
+    bestFor: ["planning", "security", "debugging"],
+    costTier: "premium",
+    speedTier: "slow"
+  },
+  {
+    provider: "anthropic",
+    pattern: /claude-haiku/i,
+    family: "Claude Haiku",
+    strengths: ["speed"],
+    bestFor: ["lightweight", "docs"],
+    avoidFor: ["planning"],
+    costTier: "budget",
+    speedTier: "fast"
+  },
+  {
+    provider: "openai",
+    pattern: /gpt-5|o3|o4/i,
+    family: "GPT-5/o3/o4",
+    strengths: ["reasoning", "coding"],
+    bestFor: ["planning", "coding", "debugging"],
+    costTier: "premium",
+    speedTier: "normal"
+  },
+  {
+    provider: "openai",
+    pattern: /gpt-4/i,
+    family: "GPT-4",
+    strengths: ["coding"],
+    bestFor: ["coding", "docs"],
+    costTier: "standard",
+    speedTier: "fast"
+  },
+  {
+    provider: "openai",
+    pattern: /gpt-4o-mini/i,
+    family: "GPT-4o Mini",
+    strengths: ["speed"],
+    bestFor: ["lightweight", "docs"],
+    avoidFor: ["planning"],
+    costTier: "budget",
+    speedTier: "fast"
+  },
+  {
+    provider: "google",
+    pattern: /gemini-(?:2\.5|3)/i,
+    family: "Gemini 2.5/3",
+    strengths: ["context", "coding"],
+    bestFor: ["coding", "data"],
+    costTier: "standard",
+    speedTier: "normal"
+  },
+  {
+    provider: "google",
+    pattern: /gemini.*flash/i,
+    family: "Gemini Flash",
+    strengths: ["speed"],
+    bestFor: ["lightweight", "docs"],
+    avoidFor: ["planning"],
+    costTier: "budget",
+    speedTier: "fast"
+  },
+  {
+    provider: "deepseek",
+    pattern: /deepseek/i,
+    family: "DeepSeek",
+    strengths: ["coding", "cost-effective"],
+    bestFor: ["coding", "general"],
+    costTier: "standard",
+    speedTier: "normal"
+  }
+];
+function fmtTokens(n) {
+  if (n >= 1e6) return `${(n / 1e6).toFixed(1)}M`;
+  if (n >= 1e3) return `${(n / 1e3).toFixed(1)}k`;
+  return String(n);
+}
+function fmtMs(ms) {
+  if (ms >= 1e3) return `${(ms / 1e3).toFixed(1)}s`;
+  return `${ms}ms`;
+}
+function fmtPrice(usdPer1M) {
+  if (usdPer1M === void 0) return color.dim("?");
+  if (usdPer1M >= 10) return `$${usdPer1M.toFixed(1)}`;
+  return `$${usdPer1M.toFixed(2)}`;
+}
+function checkMark(ok) {
+  return ok ? color.green("\u2713") : color.red("\u2717");
+}
+function costLabel(tier) {
+  switch (tier) {
+    case "premium":
+      return color.red("$$$");
+    case "standard":
+      return color.amber("$$");
+    case "budget":
+      return color.green("$");
+    default:
+      return color.dim("?");
+  }
+}
+function speedLabel(tier) {
+  switch (tier) {
+    case "fast":
+      return color.green("\u26A1");
+    case "normal":
+      return color.amber("\u2192");
+    case "slow":
+      return color.red("\u{1F422}");
+    default:
+      return color.dim("?");
+  }
+}
+function scoreBar(score, max) {
+  const pct = Math.min(1, Math.max(0, score / max));
+  const filled = Math.round(pct * 10);
+  const bar = color.green("\u2588".repeat(filled)) + color.dim("\u2591".repeat(10 - filled));
+  return `${bar} ${score}/${max}`;
+}
+var ROLE_CATEGORY = {
+  "security-scanner": "security",
+  "security-reviewer": "security",
+  "bug-hunter": "debugging",
+  debugger: "debugging",
+  tracer: "debugging",
+  planner: "planning",
+  architect: "planning",
+  "refactor-planner": "planning",
+  verifier: "testing",
+  test: "testing",
+  e2e: "testing",
+  document: "docs",
+  simplifier: "docs",
+  "code-reviewer": "review",
+  reviewer: "review",
+  critic: "review",
+  executor: "coding",
+  refactor: "refactoring",
+  migration: "coding",
+  frontend: "frontend",
+  backend: "backend",
+  api: "backend",
+  auth: "backend",
+  designer: "frontend",
+  analyst: "data",
+  data: "data",
+  database: "data",
+  explore: "planning",
+  search: "planning",
+  researcher: "planning"
+};
+function findProfile(pid, mid) {
+  for (const p of MODEL_PROFILES) {
+    if (p.provider === pid && p.pattern.test(mid)) return p;
+  }
+  return void 0;
+}
+function scoreModel(pid, mid, category, ctxWindow) {
+  const profile = findProfile(pid, mid);
+  let score = 50;
+  if (profile) {
+    if (profile.bestFor.includes(category)) score += 35;
+    if (profile.avoidFor?.includes(category)) score -= 50;
+    if (category === "planning" && profile.costTier === "premium") score += 15;
+    if (profile.speedTier === "slow" && category === "planning") score += 10;
+    if (profile.costTier === "budget" && category !== "planning" && category !== "security")
+      score += 10;
+  }
+  if (ctxWindow > 2e5) score += 10;
+  else if (ctxWindow > 1e5) score += 5;
+  else if (ctxWindow > 32e3) score += 2;
+  return { score, profile };
+}
+function rankModels(providers, hasKey, category, limit) {
+  const candidates = [];
+  for (const prov of providers) {
+    if (!hasKey(prov.id)) continue;
+    for (const m of prov.models ?? []) {
+      const ctxWindow = m.capabilities?.contextWindow ?? 0;
+      const { score, profile } = scoreModel(prov.id, m.id, category, ctxWindow);
+      if (score > 0) {
+        candidates.push({
+          provider: prov.id,
+          model: m.id,
+          profile,
+          score,
+          ctxWindow,
+          maxOutput: m.capabilities?.maxOutputTokens ?? 0,
+          inputPrice: m.pricing?.input,
+          outputPrice: m.pricing?.output
+        });
+      }
+    }
+  }
+  candidates.sort((a, b) => b.score - a.score);
+  return candidates.slice(0, limit);
+}
+var EVAL_TASKS = {
+  coding: {
+    label: "Code Generation",
+    prompt: "Write a TypeScript function parseCSV(input: string): { headers: string[]; rows: string[][] } that handles quoted fields, escaped quotes, and empty lines. Return an error string on malformed input. Keep under 40 lines."
+  },
+  planning: {
+    label: "Architecture Planning",
+    prompt: "Design the folder structure and key interfaces for a monorepo CLI tool with slash commands, model routing, subagent spawning, and config persistence. List packages, their responsibilities, and the 5 most important TypeScript interfaces."
+  },
+  security: {
+    label: "Vulnerability Detection",
+    prompt: 'Review this code for security issues:\n```ts\napp.get("/api/user", (req, res) => {\n  const id = req.query.id;\n  const user = db.query("SELECT * FROM users WHERE id = " + id);\n  res.json(user);\n});\n\napp.post("/api/run", (req, res) => {\n  const { cmd } = req.body;\n  exec("echo " + cmd, (err, stdout) => res.send(stdout));\n});\n```\nList every vulnerability, its severity (critical/high/medium), and the exact fix.'
+  },
+  debugging: {
+    label: "Bug Diagnosis",
+    prompt: 'This async function has 2 bugs. Find and fix both:\n```ts\nasync function processBatch(items: string[]) {\n  const results = [];\n  for (const item of items) {\n    const result = await fetch("https://api.example.com/" + item);\n    results.push(result);\n  }\n  return results.map(r => r.json());\n}\n```\nExplain what each bug is, why it fails, and write the corrected version.'
+  },
+  testing: {
+    label: "Test Authoring",
+    prompt: 'Write vitest test cases for this deepMerge function:\n```ts\nfunction deepMerge(base: Record<string, unknown>, overrides: Record<string, unknown>): Record<string, unknown> {\n  const merged = { ...base };\n  for (const [key, val] of Object.entries(overrides)) {\n    if (val === null) { delete merged[key]; continue; }\n    if (typeof val === "object" && !Array.isArray(val) && typeof merged[key] === "object" && !Array.isArray(merged[key])) {\n      merged[key] = deepMerge(merged[key] as Record<string, unknown>, val as Record<string, unknown>);\n    } else { merged[key] = val; }\n  }\n  return merged;\n}\n```\nCover: happy path, edge cases, and error conditions.'
+  },
+  docs: {
+    label: "Documentation",
+    prompt: "Write TSDoc comments for this RateLimiter interface. Include @param, @returns, @throws, and @example for each method:\n```ts\ninterface RateLimiter {\n  tryAcquire(key: string, maxPerWindow: number, windowMs: number): Promise<boolean>;\n  getRemaining(key: string): Promise<number>;\n  reset(key: string): Promise<void>;\n}\n```"
+  },
+  review: {
+    label: "Code Review",
+    prompt: 'Review this PR change:\n```diff\n async function loadConfig(path: string) {\n-  const raw = await fs.readFile(path, "utf8");\n-  return JSON.parse(raw);\n+  const raw = await fs.readFile(path);\n+  const config = JSON.parse(raw);\n+  process.env.API_KEY = config.apiKey;\n+  return config;\n }\n```\nList issues by severity (blocking / should-fix / nit) and explain your reasoning.'
+  },
+  refactoring: {
+    label: "Refactoring",
+    prompt: 'Refactor this nested condition into a cleaner pattern:\n```ts\nfunction getDiscount(user: { type: string; years: number; coupon?: string }): number {\n  if (user.type === "premium") {\n    if (user.years > 5) {\n      if (user.coupon === "BLACKFRIDAY") return 0.5;\n      return 0.3;\n    }\n    return 0.2;\n  }\n  if (user.type === "standard") {\n    if (user.years > 3) return 0.15;\n    return 0.1;\n  }\n  return 0;\n}\n```\nShow your refactored code and explain why your approach is cleaner.'
+  }
+};
+var EVAL_CATEGORIES = Object.keys(EVAL_TASKS);
+function roleCat(role) {
+  return ROLE_CATEGORY[role] ?? "general";
+}
+
+// src/subcommands/handlers/modeldiag.ts
+function createProviderForId(providerId, cfg) {
+  const savedCfg = cfg.providers?.[providerId];
+  const cfgWithType = Object.assign(
+    { type: providerId },
+    savedCfg ?? { apiKey: cfg.apiKey, baseUrl: cfg.baseUrl }
+  );
+  try {
+    return makeProviderFromConfig(providerId, cfgWithType);
+  } catch {
+    return void 0;
+  }
+}
+async function rankResponses(provider, leaderModel, taskPrompt, responses) {
+  const labelToIdx = /* @__PURE__ */ new Map();
+  const responseBlock = responses.map((r, i) => {
+    const label = String.fromCharCode(65 + i);
+    labelToIdx.set(label, i);
+    return `=== Response ${label} ===
+${r.text.slice(0, 800)}`;
+  }).join("\n\n");
+  const rankingPrompt = `Rank these responses from BEST (1) to WORST.
+
+TASK:
+${taskPrompt.slice(0, 600)}
+
+RESPONSES:
+${responseBlock}
+
+Output ONLY a ranked list, one per line:
+1. Response X \u2014 brief reason
+2. Response Y \u2014 brief reason`;
+  try {
+    const resp = await provider.complete(
+      {
+        model: leaderModel,
+        system: [
+          {
+            type: "text",
+            text: "You are an expert evaluator. Rank responses concisely. Output ONLY the ranked list."
+          }
+        ],
+        messages: [{ role: "user", content: [{ type: "text", text: rankingPrompt }] }],
+        maxTokens: 400
+      },
+      { signal: AbortSignal.timeout(3e4) }
+    );
+    const text = resp.content[0] && "text" in resp.content[0] ? resp.content[0].text : "";
+    const rankings = [];
+    for (const line of text.split("\n")) {
+      const m = line.match(/^\s*(\d+)[.)]\s*Response\s+([A-Z])/i);
+      if (m) {
+        const label = m[2]?.toUpperCase();
+        const idx = label !== void 0 ? labelToIdx.get(label) : void 0;
+        if (idx !== void 0 && !rankings.includes(idx)) {
+          rankings.push(idx);
+        }
+      }
+    }
+    return rankings.length > 0 ? rankings : responses.map((_, i) => i);
+  } catch {
+    return responses.map((_, i) => i);
+  }
+}
+async function readProviders(cachePath) {
+  if (!cachePath) {
+    return `${color2.red("Models cache not available")}.`;
+  }
+  try {
+    const raw = await fs.readFile(cachePath, "utf8");
+    const parsed = JSON.parse(raw);
+    const payload = parsed.payload ?? parsed;
+    return Object.entries(payload).map(([id, p]) => ({
+      id: p.id ?? id,
+      name: p.name ?? id,
+      family: p.npm ?? id,
+      models: Object.values(p.models ?? {}).map(
+        (m) => ({
+          id: m.id,
+          name: m.name,
+          capabilities: {
+            contextWindow: m.limit?.context,
+            maxOutputTokens: m.limit?.output
+          },
+          pricing: m.cost
+        })
+      )
+    }));
+  } catch {
+    return `${color2.amber("Models cache not available")}. Run wstack sync-models.`;
+  }
+}
+function checkHasKey(pid, config) {
+  if (pid === config.provider && config.provider) return true;
+  const pc = config.providers?.[pid];
+  if (!pc) return false;
+  if (typeof pc.apiKey === "string" && pc.apiKey.length > 0) return true;
+  if (Array.isArray(pc.apiKeys) && pc.apiKeys.some((k) => k?.apiKey)) return true;
+  return false;
+}
+var modeldiagCmd = async (args, deps) => {
+  const sub = args[0]?.toLowerCase() || "full";
+  const cacheResult = await readProviders(deps.paths.modelsCache);
+  if (typeof cacheResult === "string") {
+    deps.renderer.write(`${cacheResult}
+`);
+    return cacheResult.includes(color2.red("")) ? 1 : 0;
+  }
+  const providers = cacheResult;
+  const config = deps.config;
+  const modelMatrix = config.modelMatrix ?? {};
+  function hasKey(pid) {
+    return checkHasKey(pid, config);
+  }
+  function writeLine(line = "") {
+    deps.renderer.write(`${line}
+`);
+  }
+  if (sub === "keys") {
+    writeLine(`${color2.bold("API Key Status")}`);
+    writeLine();
+    for (const prov of providers) {
+      const k = hasKey(prov.id);
+      writeLine(`  ${checkMark(k)} ${color2.bold(prov.id.padEnd(18))} ${color2.dim(prov.name)}`);
+    }
+    writeLine();
+    writeLine(`${color2.dim(`Leader: ${config.provider}/${config.model}`)}`);
+    return 0;
+  }
+  if (sub === "caps") {
+    writeLine(`${color2.bold("Model Capabilities")} ${color2.dim("\u2014 matched to known profiles")}`);
+    writeLine();
+    for (const prov of providers) {
+      if (!hasKey(prov.id)) continue;
+      writeLine(`  ${color2.bold(prov.id)} ${color2.dim(`(${prov.name})`)}`);
+      const tiers = {
+        premium: [],
+        standard: [],
+        budget: [],
+        unknown: []
+      };
+      for (const m of prov.models ?? []) {
+        const profile = findProfile(prov.id, m.id);
+        tiers[profile?.costTier ?? "unknown"]?.push(m);
+      }
+      for (const tier of ["premium", "standard", "budget", "unknown"]) {
+        const tierModels = tiers[tier];
+        if (tierModels.length === 0) continue;
+        const label = tier === "unknown" ? color2.dim("unmatched") : `${costLabel(tier)} ${tier}`;
+        writeLine(`    ${label}`);
+        for (const m of tierModels) {
+          const cap = m.capabilities;
+          const ctx = cap?.contextWindow ?? 0;
+          const maxOut = cap?.maxOutputTokens ?? 0;
+          const profile = findProfile(prov.id, m.id);
+          const family = profile ? `${speedLabel(profile.speedTier)} ${color2.green(profile.family)}` : color2.dim("no profile match");
+          const pricing = m.pricing ? `${color2.dim("in")}${fmtPrice(m.pricing.input)} ${color2.dim("out")}${fmtPrice(m.pricing.output)}` : color2.dim("pricing ?");
+          writeLine(
+            `      ${color2.cyan(m.id.padEnd(34))}${ctx > 0 ? `ctx ${fmtTokens(ctx).padEnd(6)}` : color2.dim("ctx ?  ")}${maxOut > 0 ? `out ${fmtTokens(maxOut).padEnd(6)}` : "        "}${family}   ${pricing}`
+          );
+        }
+      }
+      writeLine();
+    }
+    writeLine(
+      color2.dim(
+        "Prices in USD per 1M tokens (input/output). ctx = context window, out = max output."
+      )
+    );
+    return 0;
+  }
+  async function renderSuggest() {
+    writeLine();
+    writeLine(
+      `${color2.bold("Agent \u2192 Model Suggestions")} ${color2.amber("(heuristic \u2014 untested)")}`
+    );
+    writeLine(
+      color2.dim(
+        'These are profile-based best guesses. Test them with wstack modeldiag bench <role> "<prompt>".'
+      )
+    );
+    writeLine();
+    const keyedProviders = providers.filter((p) => hasKey(p.id));
+    if (keyedProviders.length === 0) {
+      writeLine(
+        `  ${color2.amber("No providers have API keys configured. Add keys with wstack auth.")}`
+      );
+    } else {
+      const roles = [
+        "security-scanner",
+        "bug-hunter",
+        "planner",
+        "architect",
+        "refactor-planner",
+        "verifier",
+        "test",
+        "document",
+        "reviewer",
+        "code-reviewer",
+        "executor",
+        "debugger"
+      ];
+      for (const role of roles) {
+        if (modelMatrix[role]) {
+          const entry = modelMatrix[role];
+          const p = entry.provider ?? config.provider;
+          writeLine(
+            `  ${color2.dim(role.padEnd(20))} \u2192 ${color2.cyan(`${p}/${entry.model}`)}  ${color2.dim("(user-configured)")}`
+          );
+          continue;
+        }
+        const cat = roleCat(role);
+        const ranked = rankModels(providers, hasKey, cat, 3);
+        if (ranked.length === 0) {
+          writeLine(`  ${color2.dim(role.padEnd(20))} \u2192 ${color2.dim("no candidates")}`);
+          continue;
+        }
+        const best = ranked[0];
+        const family = best.profile ? ` ${color2.dim(`(${best.profile.family})`)}` : "";
+        const bar = scoreBar(best.score, 110);
+        writeLine(
+          `  ${color2.amber(role.padEnd(20))} \u2192 ${color2.cyan(`${best.provider}/${best.model}`)}${family}`
+        );
+        writeLine(`  ${" ".repeat(22)}  ${bar}  ${color2.dim(cat)}`);
+        if (ranked.length > 1 && (ranked[1]?.score ?? Number.NEGATIVE_INFINITY) >= best.score - 15) {
+          for (const alt of ranked.slice(1)) {
+            const af = alt.profile ? ` (${alt.profile.family})` : "";
+            writeLine(
+              `  ${" ".repeat(22)}  ${color2.dim(`${alt.provider}/${alt.model}${af}  score ${alt.score}`)}`
+            );
+          }
+        }
+      }
+      writeLine();
+      writeLine(
+        `  ${color2.bold("leader".padEnd(20))} \u2192 ${color2.cyan(`${config.provider}/${config.model}`)}`
+      );
+    }
+  }
+  if (sub === "suggest") {
+    await renderSuggest();
+    writeLine();
+    writeLine(color2.dim("Pin a suggestion:  wstack setmodel set <role> <provider>/<model>"));
+    writeLine(color2.dim('Test candidates:   wstack modeldiag bench <role> "<test prompt>"'));
+    return 0;
+  }
+  if (sub === "test") {
+    if (args.slice(1).includes("--help") || deps.flags?.["help"] === true) {
+      writeLine(`${color2.bold("Usage:")} wstack modeldiag test [options]`);
+      writeLine();
+      writeLine("  (no flags)              Probe one representative model per configured provider");
+      writeLine("  --all-models             Probe every catalog-visible model sequentially");
+      writeLine("  --plan                   Show targets without making model calls");
+      writeLine("  --provider=<id,...>      Restrict provider ids");
+      writeLine("  --model=<id,...>         Restrict model ids");
+      writeLine("  --timeout=<ms>           Per-model timeout (default 45000)");
+      writeLine("  --max-tokens=<n>         Output cap for the tiny probe (default 32)");
+      writeLine("  --json                   Emit machine-readable output");
+      writeLine("  --yes                    Confirm a run larger than 50 requests");
+      return 0;
+    }
+    let smokeOptions;
+    try {
+      smokeOptions = parseModelSmokeOptions(args.slice(1), deps.flags);
+    } catch (err) {
+      writeLine(`${color2.red("Invalid smoke-test options:")} ${toErrorMessage2(err)}`);
+      return 2;
+    }
+    if ("mergeOverlay" in deps.modelsRegistry) {
+      await discoverAndMergeProviders({
+        config,
+        registry: deps.modelsRegistry,
+        cacheDir: deps.paths.cacheDir
+      });
+    }
+    const targets = await buildModelSmokeTargets(config, deps.modelsRegistry, smokeOptions);
+    const byProvider = /* @__PURE__ */ new Map();
+    for (const target of targets) {
+      byProvider.set(target.providerId, (byProvider.get(target.providerId) ?? 0) + 1);
+    }
+    if (smokeOptions.planOnly) {
+      const plan = {
+        mode: smokeOptions.allModels ? "all-models" : "representative",
+        targetCount: targets.length,
+        providers: Object.fromEntries(byProvider),
+        targets
+      };
+      if (smokeOptions.json) {
+        writeLine(JSON.stringify(plan, null, 2));
+      } else {
+        writeLine(`${color2.bold("Provider/Model Smoke Test Plan")}`);
+        writeLine();
+        for (const [providerId, count] of byProvider) {
+          writeLine(`  ${color2.cyan(providerId.padEnd(24))} ${count} model(s)`);
+        }
+        writeLine();
+        writeLine(
+          color2.dim(
+            `${targets.length} request(s) would run sequentially. Remove --plan to execute.`
+          )
+        );
+      }
+      return targets.length > 0 ? 0 : 2;
+    }
+    if (targets.length === 0) {
+      const message = "No provider/model targets matched. Check the active profile or remove filters.";
+      if (smokeOptions.json) {
+        writeLine(JSON.stringify({ results: [], summary: { passed: 0, failed: 0 }, error: message }));
+      } else {
+        writeLine(color2.amber(message));
+      }
+      return 2;
+    }
+    const confirmed = deps.flags?.["yes"] === true || deps.flags?.["yes"] === "true" || args.slice(1).includes("--yes");
+    if (targets.length > 50 && !confirmed) {
+      const message = `${targets.length} live requests are planned. Review with --plan, then add --yes to run them.`;
+      if (smokeOptions.json) {
+        writeLine(JSON.stringify({ error: message, targetCount: targets.length }));
+      } else {
+        writeLine(color2.amber(message));
+      }
+      return 2;
+    }
+    if (!smokeOptions.json) {
+      writeLine(`${color2.bold("Live Provider/Model Smoke Test")}`);
+      writeLine(
+        color2.dim(
+          `${targets.length} request(s), sequential, ${smokeOptions.timeoutMs}ms timeout, max ${smokeOptions.maxTokens} output tokens.`
+        )
+      );
+      writeLine();
+    }
+    const resultLines = (result, index, total) => {
+      if (smokeOptions.json) return;
+      const label = `[${String(index + 1).padStart(String(total).length)}/${total}]`;
+      const target = `${result.providerId}/${result.modelId}`;
+      if (result.status === "passed") {
+        writeLine(
+          `  ${color2.green("\u2713")} ${color2.dim(label)} ${color2.cyan(target)} ${color2.dim(
+            `${fmtMs(result.latencyMs)} \xB7 in${result.usage.input}/out${result.usage.output} \xB7 ${result.stopReason}`
+          )}`
+        );
+      } else {
+        const status = result.httpStatus ? `HTTP ${result.httpStatus} \xB7 ` : "";
+        writeLine(
+          `  ${color2.red("\u2717")} ${color2.dim(label)} ${color2.red(target)} ${color2.dim(
+            `${fmtMs(result.latencyMs)} \xB7 ${status}${result.errorKind}`
+          )}`
+        );
+        writeLine(`      ${color2.red(result.error.slice(0, 240))}`);
+      }
+    };
+    let oauthWriteChain = Promise.resolve();
+    let oauthWriteScheduled = false;
+    const profilePath = activeProfileConfigPath(deps.paths, config);
+    setOAuthTokenPersister((providerId, credentials) => {
+      oauthWriteScheduled = true;
+      oauthWriteChain = oauthWriteChain.then(
+        () => mutateConfigProviders(profilePath, deps.vault, (all) => {
+          const providerConfig = all[providerId];
+          if (!providerConfig) return;
+          const keys = normalizeKeys(providerConfig);
+          const active = providerConfig.activeKey ? keys.find((key) => key.label === providerConfig.activeKey) : keys[0];
+          if (!active) return;
+          active.apiKey = credentials.accessToken;
+          active.refreshToken = credentials.refreshToken;
+          active.expiresAt = new Date(credentials.expiresAt).toISOString();
+          if (credentials.accountId) active.accountId = credentials.accountId;
+          writeKeysBack(providerConfig, keys);
+        })
+      );
+    });
+    let results;
+    try {
+      results = await runModelSmokeTests({
+        targets,
+        options: smokeOptions,
+        createProvider: async (providerId) => {
+          const saved = config.providers?.[providerId];
+          const resolved = await deps.modelsRegistry.getProvider(providerId).catch(() => void 0) ?? (saved?.type && saved.type !== providerId ? await deps.modelsRegistry.getProvider(saved.type).catch(() => void 0) : void 0);
+          const providerConfig = {
+            ...providerId === config.provider ? {
+              ...config.apiKey ? { apiKey: config.apiKey } : {},
+              ...config.baseUrl ? { baseUrl: config.baseUrl } : {}
+            } : {},
+            ...saved,
+            type: providerId,
+            ...saved?.family ?? resolved?.family ? { family: saved?.family ?? resolved?.family } : {},
+            ...saved?.baseUrl ?? resolved?.apiBase ? { baseUrl: saved?.baseUrl ?? resolved?.apiBase } : {},
+            ...saved?.envVars ?? resolved?.envVars ? { envVars: saved?.envVars ?? resolved?.envVars } : {}
+          };
+          return makeProviderFromConfig(providerId, providerConfig);
+        },
+        onTargetComplete: resultLines
+      });
+    } finally {
+      setOAuthTokenPersister(void 0);
+    }
+    let oauthPersistenceError;
+    if (oauthWriteScheduled) {
+      try {
+        await oauthWriteChain;
+      } catch (err) {
+        oauthPersistenceError = toErrorMessage2(err);
+      }
+    }
+    const passed = results.filter((result) => result.status === "passed").length;
+    const failed = results.length - passed;
+    const summary = { passed, failed, total: results.length };
+    if (smokeOptions.json) {
+      writeLine(
+        JSON.stringify(
+          {
+            results,
+            summary,
+            ...oauthPersistenceError ? { warning: `OAuth token refresh worked in-memory but could not be saved: ${oauthPersistenceError}` } : {}
+          },
+          null,
+          2
+        )
+      );
+    } else {
+      writeLine();
+      writeLine(
+        `${color2.bold("Summary:")} ${color2.green(`${passed} passed`)} \xB7 ${failed > 0 ? color2.red(`${failed} failed`) : color2.dim("0 failed")}`
+      );
+      if (!smokeOptions.allModels) {
+        writeLine(
+          color2.dim("Use --all-models to test every catalog-visible model sequentially.")
+        );
+      }
+      if (oauthPersistenceError) {
+        writeLine(
+          color2.amber(
+            `OAuth token refresh worked in-memory but could not be saved: ${oauthPersistenceError}`
+          )
+        );
+      }
+    }
+    return failed === 0 ? 0 : 1;
+  }
+  if (sub === "bench") {
+    const benchArgs = args.slice(1);
+    if (benchArgs.length < 2) {
+      writeLine(
+        `${color2.amber("Usage:")} wstack modeldiag bench <role> "<test prompt>" [--providers=p1,p2]`
+      );
+      writeLine();
+      writeLine(
+        color2.dim(
+          'Example: wstack modeldiag bench verify "Write a function that checks if a string is a palindrome"'
+        )
+      );
+      writeLine(
+        color2.dim(
+          "Tests the top 5 candidate models for the role with your prompt and reports results."
+        )
+      );
+      writeLine(color2.dim("Add --providers=anthropic,google to test across multiple providers."));
+      return 0;
+    }
+    const providersEqIdx = benchArgs.findIndex((a) => a.startsWith("--providers="));
+    let providerFilter;
+    if (providersEqIdx >= 0) {
+      const rawFilter = benchArgs[providersEqIdx]?.replace("--providers=", "").split(",");
+      if (rawFilter) {
+        providerFilter = rawFilter.map((s) => s.trim()).filter(Boolean);
+        benchArgs.splice(providersEqIdx, 1);
+      }
+    } else {
+      const rawFilter = deps.flags?.["providers"] ?? deps.flags?.["provider"];
+      if (typeof rawFilter === "string") {
+        providerFilter = rawFilter.split(",").map((s) => s.trim()).filter(Boolean);
+      }
+    }
+    const benchRole = benchArgs[0];
+    if (!benchRole) {
+      writeLine(
+        `${color2.amber("No benchmark role specified")}. Usage: wstack diag bench <role> [prompt]`
+      );
+      return 1;
+    }
+    const benchPrompt = benchArgs.slice(1).join(" ");
+    const cat = roleCat(benchRole);
+    const candidates = rankModels(providers, hasKey, cat, 5);
+    if (candidates.length === 0) {
+      writeLine(
+        `${color2.amber("No candidate models found")} for role "${benchRole}" (category: ${cat}).`
+      );
+      return 0;
+    }
+    let targetCandidates;
+    if (providerFilter && providerFilter.length > 0) {
+      targetCandidates = candidates.filter((c) => providerFilter.includes(c.provider));
+      if (targetCandidates.length === 0) {
+        writeLine(
+          `${color2.amber("No candidates match the specified providers")}: ${providerFilter.join(", ")}`
+        );
+        writeLine(
+          `Candidate providers: ${[...new Set(candidates.map((c) => c.provider))].join(", ")}`
+        );
+        return 0;
+      }
+    } else {
+      targetCandidates = candidates;
+    }
+    writeLine(
+      `${color2.bold("Model Benchmark")} \u2014 ${color2.amber(benchRole)} ${color2.dim(`(category: ${cat})`)}`
+    );
+    writeLine(
+      `${color2.dim("Prompt:")} "${benchPrompt.slice(0, 120)}${benchPrompt.length > 120 ? "\u2026" : ""}"`
+    );
+    writeLine();
+    writeLine(
+      `  ${color2.dim("#  model".padEnd(52))} ${color2.dim("score".padEnd(12))} ${color2.dim("latency".padEnd(10))} ${color2.dim("tokens".padEnd(14))} ${color2.dim("first line")}`
+    );
+    writeLine(`  ${color2.dim("\u2500".repeat(108))}`);
+    const providerInstances = /* @__PURE__ */ new Map();
+    for (const pid of [...new Set(targetCandidates.map((c) => c.provider))]) {
+      const prov = createProviderForId(pid, config);
+      if (prov) providerInstances.set(pid, prov);
+    }
+    let idx = 0;
+    for (const c of targetCandidates.slice(0, 20)) {
+      idx++;
+      const label = `${idx}`.padStart(2);
+      const modelKey = `${c.provider}/${c.model}`;
+      const prov = providerInstances.get(c.provider);
+      if (!prov) {
+        writeLine(
+          `  ${label} ${color2.red(modelKey.padEnd(50))} ${scoreBar(c.score, 110).slice(0, 11)}  ${color2.red("NO PROVIDER")}`
+        );
+        continue;
+      }
+      try {
+        const start = Date.now();
+        const resp = await prov.complete(
+          {
+            model: c.model,
+            messages: [{ role: "user", content: [{ type: "text", text: benchPrompt }] }],
+            maxTokens: 256
+          },
+          { signal: AbortSignal.timeout(3e4) }
+        );
+        const latency = Date.now() - start;
+        const firstText = resp.content[0] && "text" in resp.content[0] ? resp.content[0].text : "";
+        const firstLineClean = firstText.replace(/\n/g, " ").slice(0, 80) || color2.dim("(empty)");
+        const provColor = c.provider === config.provider ? color2.green : color2.cyan;
+        const usage = resp.usage;
+        writeLine(
+          `  ${label} ${provColor(modelKey.padEnd(50))} ${scoreBar(c.score, 110).slice(0, 11)}  ${color2.amber(fmtMs(latency).padEnd(8))} ${color2.dim(`in${usage?.input ?? "?"}/out${usage?.output ?? "?"}`.padEnd(12))} ${firstLineClean}`
+        );
+      } catch (err) {
+        const errMsg = toErrorMessage2(err);
+        writeLine(
+          `  ${label} ${color2.red(modelKey.padEnd(50))} ${scoreBar(c.score, 110).slice(0, 11)}  ${color2.red("FAILED")}    ${color2.dim(errMsg.slice(0, 40))}`
+        );
+      }
+    }
+    const testedProviders = [...new Set(targetCandidates.map((c) => c.provider))];
+    writeLine();
+    writeLine(
+      color2.dim(
+        `Tested ${idx} model(s) across ${testedProviders.length} provider(s): ${testedProviders.join(", ")}.`
+      )
+    );
+    writeLine(color2.dim("Pin the best: wstack setmodel set <role> <provider>/<model>"));
+    return 0;
+  }
+  if (sub === "eval" || sub === "evall") {
+    const evalArgs = args.slice(1);
+    const providersEq = evalArgs.find((a) => a.startsWith("--providers="));
+    const providersValue = providersEq?.replace("--providers=", "") ?? (typeof deps.flags?.["providers"] === "string" ? deps.flags["providers"] : typeof deps.flags?.["provider"] === "string" ? deps.flags["provider"] : void 0);
+    const providerFilter = providersValue ? providersValue.split(",").map((s) => s.trim()).filter(Boolean) : void 0;
+    const maxEq = evalArgs.find((a) => a.startsWith("--max="));
+    const maxValue = maxEq?.replace("--max=", "") ?? (typeof deps.flags?.["max"] === "string" ? deps.flags["max"] : void 0);
+    const maxModels = maxValue ? Math.max(1, parseInt(maxValue, 10) || 2) : 2;
+    const quick = evalArgs.includes("--quick") || deps.flags?.["quick"] === true || deps.flags?.["quick"] === "true";
+    const modelsPerCat = quick ? 1 : maxModels;
+    const roleFilter = evalArgs.find((a) => !a.startsWith("--"));
+    const targetCategories = roleFilter ? EVAL_CATEGORIES.includes(roleCat(roleFilter)) ? [roleCat(roleFilter)] : [] : EVAL_CATEGORIES;
+    if (targetCategories.length === 0 && roleFilter) {
+      writeLine(
+        `${color2.amber("Unknown role/category")}: "${roleFilter}". Try: ${EVAL_CATEGORIES.join(", ")}`
+      );
+      return 1;
+    }
+    const keyedProviderIds = providers.filter((p) => hasKey(p.id)).map((p) => p.id);
+    let targetProviderIds;
+    if (providerFilter && providerFilter.length > 0) {
+      const unknown = providerFilter.filter((pid) => !keyedProviderIds.includes(pid));
+      targetProviderIds = providerFilter.filter((pid) => keyedProviderIds.includes(pid));
+      if (targetProviderIds.length === 0) {
+        const noKeyMsg = unknown.length > 0 ? `None of the specified providers (${unknown.join(", ")}) have API keys. Add keys with wstack auth.` : "None of the specified providers have API keys configured.";
+        writeLine(`${color2.amber(noKeyMsg)}`);
+        return 0;
+      }
+    } else if (keyedProviderIds.length === 0) {
+      writeLine(`${color2.amber("No providers have API keys. Add keys with wstack auth.")}`);
+      return 0;
+    } else if (keyedProviderIds.length === 1) {
+      targetProviderIds = keyedProviderIds;
+    } else {
+      const providerList = keyedProviderIds.map((pid, i) => {
+        const info = providers.find((p) => p.id === pid);
+        return `  ${color2.cyan(String(i + 1))}) ${color2.bold(pid.padEnd(16))} ${color2.dim(info?.name ?? "")}`;
+      }).join("\n");
+      deps.renderer.write(
+        `
+${color2.bold("Select providers to evaluate")}
+
+${providerList}
+
+${color2.dim('Enter numbers or provider IDs (comma-separated, or "all"):')}
+`
+      );
+      const input = await deps.reader.readLine("  > ");
+      const selected = input.trim().toLowerCase();
+      if (selected === "" || selected === "all") {
+        targetProviderIds = keyedProviderIds;
+      } else {
+        targetProviderIds = [];
+        for (const part of selected.split(",").map((s) => s.trim())) {
+          const idx = parseInt(part, 10);
+          if (idx >= 1 && idx <= keyedProviderIds.length) {
+            const pid = keyedProviderIds[idx - 1];
+            if (!targetProviderIds.includes(pid)) targetProviderIds.push(pid);
+          } else if (keyedProviderIds.includes(part)) {
+            if (!targetProviderIds.includes(part)) targetProviderIds.push(part);
+          }
+        }
+      }
+      if (targetProviderIds.length === 0) {
+        writeLine(color2.dim("No providers selected."));
+        return 0;
+      }
+    }
+    const leaderModel = config.model ?? "unknown";
+    const unknownProviders = providerFilter ? providerFilter.filter((pid) => !keyedProviderIds.includes(pid)) : [];
+    const warningLine = unknownProviders.length > 0 ? `  ${color2.amber("\u26A0 skipped (no key):")} ${unknownProviders.join(", ")}
+` : "";
+    writeLine(`${color2.bold("Model Competency Evaluation")}`);
+    writeLine(
+      color2.dim(
+        `Providers: ${targetProviderIds.join(", ")}  |  ${targetCategories.length} cats  |  ${modelsPerCat} model(s)/cat/provider`
+      )
+    );
+    writeLine(warningLine);
+    writeLine(color2.dim(`Leader (ranker): ${config.provider}/${leaderModel}`));
+    writeLine();
+    const collected = /* @__PURE__ */ new Map();
+    let total = 0;
+    let ok = 0;
+    for (const pid of targetProviderIds) {
+      const prov = createProviderForId(pid, config);
+      if (!prov) {
+        writeLine(color2.dim(`  \u2298 ${pid}: provider unavailable, skipping`));
+        continue;
+      }
+      for (const cat of targetCategories) {
+        const task = EVAL_TASKS[cat];
+        if (!task) continue;
+        const candidates = rankModels(
+          providers,
+          (providerId) => providerId === pid && hasKey(providerId),
+          cat,
+          modelsPerCat
+        );
+        if (candidates.length === 0) continue;
+        if (!collected.has(cat)) collected.set(cat, /* @__PURE__ */ new Map());
+        for (const c of candidates) {
+          total++;
+          const modelKey = `${pid}/${c.model}`;
+          try {
+            const start = Date.now();
+            const resp = await prov.complete(
+              {
+                model: c.model,
+                system: [{ type: "text", text: "Be thorough and correct." }],
+                messages: [
+                  { role: "user", content: [{ type: "text", text: task.prompt }] }
+                ],
+                maxTokens: 1024
+              },
+              { signal: AbortSignal.timeout(45e3) }
+            );
+            const respText = resp.content[0] && "text" in resp.content[0] ? resp.content[0].text : "";
+            const respUsage = resp.usage;
+            collected.get(cat)?.set(modelKey, {
+              model: modelKey,
+              latency: Date.now() - start,
+              tokens: (respUsage?.input ?? 0) + (respUsage?.output ?? 0),
+              text: respText
+            });
+            ok++;
+          } catch {
+            collected.get(cat)?.set(modelKey, {
+              model: modelKey,
+              latency: -1,
+              tokens: 0,
+              text: ""
+            });
+          }
+        }
+      }
+    }
+    writeLine(`${color2.dim(`Phase 1: ${ok}/${total} calls succeeded`)}`);
+    writeLine();
+    if (collected.size === 0) {
+      writeLine(color2.amber("No responses collected. Check provider configuration."));
+      return 0;
+    }
+    const leaderProvider = createProviderForId(config.provider, config);
+    if (leaderProvider) {
+      writeLine(`${color2.bold("Phase 2")} \u2014 ${color2.dim("leader ranks responses")}`);
+      writeLine();
+    }
+    const rankings = /* @__PURE__ */ new Map();
+    for (const [cat, responses] of collected) {
+      const valid = Array.from(responses.values()).filter((r) => r.latency >= 0);
+      if (valid.length < 2) {
+        if (valid.length === 1) {
+          const m = valid[0]?.model;
+          if (m !== void 0) {
+            if (!rankings.has(m)) rankings.set(m, /* @__PURE__ */ new Map());
+            rankings.get(m)?.set(cat, { rank: 1, total: 1 });
+          }
+        }
+        continue;
+      }
+      const task = EVAL_TASKS[cat];
+      if (leaderProvider) {
+        const ranked = await rankResponses(leaderProvider, leaderModel, task.prompt, valid);
+        for (let i = 0; i < valid.length; i++) {
+          const m = valid[ranked[i] ?? i]?.model;
+          if (m === void 0) continue;
+          if (!rankings.has(m)) rankings.set(m, /* @__PURE__ */ new Map());
+          rankings.get(m)?.set(cat, { rank: i + 1, total: valid.length });
+        }
+      } else {
+        for (const r of valid) {
+          if (!rankings.has(r.model)) rankings.set(r.model, /* @__PURE__ */ new Map());
+          rankings.get(r.model)?.set(cat, { rank: 1, total: valid.length });
+        }
+      }
+    }
+    writeLine(`${color2.bold("Competency Report")}`);
+    writeLine();
+    const allModels = [.../* @__PURE__ */ new Set([...rankings.keys()])].sort();
+    const catList = [...collected.keys()];
+    const modelColWidth = Math.max(24, ...allModels.map((m) => m.length)) + 2;
+    const cw = 12;
+    writeLine(
+      `  ${color2.dim("model".padEnd(modelColWidth))}` + catList.map((c) => color2.dim((EVAL_TASKS[c]?.label ?? c).slice(0, cw).padEnd(cw + 2))).join("")
+    );
+    writeLine(`  ${color2.dim("\u2500".repeat(modelColWidth + catList.length * (cw + 2)))}`);
+    for (const model of allModels) {
+      const mr = rankings.get(model);
+      const provFromModel = model.split("/")[0] ?? "";
+      const modelColor = provFromModel === config.provider ? color2.cyan : color2.green;
+      let row = `  ${modelColor(model.padEnd(modelColWidth))}`;
+      for (const cat of catList) {
+        const e = mr.get(cat);
+        if (e) {
+          const pct = Math.round((1 - (e.rank - 1) / Math.max(1, e.total - 1)) * 100);
+          const pc = pct >= 80 ? color2.green : pct >= 50 ? color2.amber : color2.red;
+          row += `${pc(`#${e.rank} ${pct}%`.padEnd(cw + 2))}`;
+        } else {
+          row += color2.dim("\u2014".padEnd(cw + 2));
+        }
+      }
+      writeLine(row);
+    }
+    writeLine();
+    writeLine(color2.dim("#1 100% = best in category. \u2014 = not tested."));
+    writeLine();
+    writeLine(color2.dim("Pin: wstack setmodel set <role> <provider>/<model>"));
+    writeLine(
+      color2.dim(
+        "Full: wstack modeldiag eval          Providers: wstack modeldiag eval --providers=id1,id2"
+      )
+    );
+    writeLine(
+      color2.dim("Max:  wstack modeldiag eval --max=3   Quick: wstack modeldiag eval --quick")
+    );
+    return 0;
+  }
+  writeLine(`${color2.bold("API Key Status")}`);
+  writeLine();
+  for (const prov of providers) {
+    const k = hasKey(prov.id);
+    writeLine(`  ${checkMark(k)} ${color2.bold(prov.id.padEnd(18))} ${color2.dim(prov.name)}`);
+  }
+  writeLine();
+  writeLine(`${color2.dim(`Leader: ${config.provider}/${config.model}`)}`);
+  writeLine();
+  writeLine(`${color2.bold("Model Capabilities")} ${color2.dim("\u2014 matched to known profiles")}`);
+  writeLine();
+  for (const prov of providers) {
+    if (!hasKey(prov.id)) continue;
+    writeLine(`  ${color2.bold(prov.id)} ${color2.dim(`(${prov.name})`)}`);
+    const tiers = {
+      premium: [],
+      standard: [],
+      budget: [],
+      unknown: []
+    };
+    for (const m of prov.models ?? []) {
+      const profile = findProfile(prov.id, m.id);
+      tiers[profile?.costTier ?? "unknown"]?.push(m);
+    }
+    for (const tier of ["premium", "standard", "budget", "unknown"]) {
+      const tierModels = tiers[tier];
+      if (tierModels.length === 0) continue;
+      const label = tier === "unknown" ? color2.dim("unmatched") : `${costLabel(tier)} ${tier}`;
+      writeLine(`    ${label}`);
+      for (const m of tierModels) {
+        const cap = m.capabilities;
+        const ctx = cap?.contextWindow ?? 0;
+        const maxOut = cap?.maxOutputTokens ?? 0;
+        const profile = findProfile(prov.id, m.id);
+        const family = profile ? `${speedLabel(profile.speedTier)} ${color2.green(profile.family)}` : color2.dim("no profile match");
+        const pricing = m.pricing ? `${color2.dim("in")}${fmtPrice(m.pricing.input)} ${color2.dim("out")}${fmtPrice(m.pricing.output)}` : color2.dim("pricing ?");
+        writeLine(
+          `      ${color2.cyan(m.id.padEnd(34))}${ctx > 0 ? `ctx ${fmtTokens(ctx).padEnd(6)}` : color2.dim("ctx ?  ")}${maxOut > 0 ? `out ${fmtTokens(maxOut).padEnd(6)}` : "        "}${family}   ${pricing}`
+        );
+      }
+    }
+    writeLine();
+  }
+  await renderSuggest();
+  writeLine();
+  writeLine(color2.dim("Pin a suggestion:  wstack setmodel set <role> <provider>/<model>"));
+  writeLine(color2.dim('Test candidates:   wstack modeldiag bench <role> "<test prompt>"'));
+  return 0;
+};
+export {
+  modeldiagCmd
+};
+//# sourceMappingURL=modeldiag-HENYJ4N7.js.map