katman-mcp 0.1.0

package/dist/tools/visibility/run.js

@@ -0,0 +1,684 @@
+import { listFiles, loadConfig, loadResearch, logEvent, readJson, stamp, writeJson, writeText } from "../../state.js";
+import { mapLimit } from "../../net.js";
+import { truncate } from "../../util.js";
+import { analyzeAnswer, buildContext, canonicalCompetitor, extractLinks, mentions, normalizeHost, } from "./analyze.js";
+import { estimateRun, money, taskCost } from "./estimate.js";
+import { buildJudgeBody, buildJudgeMessages, parseJudge } from "./judge.js";
+import { engineOf, profileFor, resolveJudge, resolvePanel, vendorOf } from "./models.js";
+import { OpenRouterError, buildAnswerBody, costOf, fetchKeyInfo, getApiKey, isInCatalog, loadCatalog, parseCompletion, postChat, priceOf, redact, searchUnitPrice, tuningFor, } from "./openrouter.js";
+import { PROXY_NOTE, renderDryRun, renderMeasurements, renderRunReport, shortFamily } from "./render.js";
+import { addDays, diffRuns, summarize } from "./summary.js";
+import { composeOneSentence } from "../../util.js";
+export const DEFAULT_MAX_COST_USD = 3;
+export const DEFAULT_TEMPERATURE = 0.7;
+export const MAX_SAMPLES = 5;
+const ACTIVE = new Map();
+function activeMessage(a) {
+    return `A visibility run started at ${a.started.slice(11, 16)} UTC is still in progress (${a.done} of ${a.total} answers done). Its results will be saved to .katman/runs/${a.id}.json and .katman/measurements.md when it finishes; don't start another one (it would spend the credits twice).`;
+}
+// ---------------------------------------------------------------------------
+// Research checks
+const isFill = (s) => !s || !s.trim() || /\[FILL\]/i.test(s);
+export function researchProblems(r) {
+    if (!r)
+        return ["No .katman/research.json yet."];
+    const p = [];
+    if (isFill(r.brand?.name))
+        p.push("research.json has no brand name (brand.name).");
+    if (!normalizeHost(r.site?.domain || r.site?.url || ""))
+        p.push("research.json has no site domain (site.domain / site.url).");
+    const usable = (r.questions ?? []).filter((q) => q && !isFill(q.text));
+    if (!usable.length)
+        p.push("research.json has no question set (questions: 12 unbranded + 2 diagnostic).");
+    return p;
+}
+export function researchHelp(problems) {
+    return [
+        "# Visibility run: research needed first",
+        "",
+        ...problems.map((x) => `- ${x}`),
+        "",
+        "The run asks the assistants your question set: 12 questions that don't name the brand (situation, tool, best-of, alternative, language/market, problem) plus 2 diagnostics (\"What is [brand]?\", \"What is [brand] ([domain])?\"), book ch. 15.",
+        "Next: call `katman_research` with `generate_questions: true` (it needs the brand name, domain and your 10–12 customer situations), then run `katman_visibility_run` again with `dry_run: true`.",
+    ].join("\n");
+}
+export function identityOf(r) {
+    const o = r.brand.oneLiner;
+    const composed = composeOneSentence(o, 2) ?? "";
+    const sentence = !isFill(r.brand.sentence) ? r.brand.sentence : composed || undefined;
+    return { sentence, notX: isFill(r.brand.notX) ? undefined : r.brand.notX, category: isFill(r.brand.category) ? undefined : r.brand.category };
+}
+export function keyHelp(e) {
+    return [
+        "## Set up an OpenRouter key (runs use your own credits; Katman never stores or prints the key)",
+        `1. Create a key at https://openrouter.ai/keys and add credit${e ? `: a run like this costs about ${money(e.total.expected)}, at most ${money(e.total.high)}` : ""}. Set a credit limit on the key to cap what it can spend.`,
+        "2. Put it in this MCP server's `env` and restart the server:",
+        '   - Cursor (`.cursor/mcp.json` or `~/.cursor/mcp.json`): `"katman": {"command": "npx", "args": ["-y", "katman-mcp"], "env": {"OPENROUTER_API_KEY": "${env:OPENROUTER_API_KEY}"}}` with the key exported in your shell (or the key itself in place of `${env:…}`).',
+        "   - Claude Code: `claude mcp add katman --scope user -e OPENROUTER_API_KEY=sk-or-... -- npx -y katman-mcp`",
+        '   - VS Code (`.vscode/mcp.json`): `"env": {"OPENROUTER_API_KEY": "${input:openrouter_api_key}"}` with a password `inputs` entry.',
+        "   - Windsurf/Devin, Codex, Gemini CLI, Zed, Cline: add `OPENROUTER_API_KEY` under the server's `env` (Gemini CLI strips inherited *KEY* variables, so list it explicitly). `KATMAN_OPENROUTER_API_KEY` also works.",
+        "3. Never paste the key into the chat or commit it.",
+        "",
+        "No-cost path: `katman_manual_check` gives the questions to ask ChatGPT, Claude, Gemini, Perplexity, Copilot or Grok yourself (private window, logged out, web search on) and records the answers you paste back.",
+    ].join("\n");
+}
+const clamp = (n, lo, hi) => Math.min(hi, Math.max(lo, n));
+async function readRuns(root) {
+    const files = (await listFiles(root, "runs")).filter((f) => f.endsWith(".json"));
+    const out = [];
+    for (const f of files) {
+        const r = await readJson(root, `runs/${f}`);
+        if (r && Array.isArray(r.answers) && typeof r.ts === "string")
+            out.push({ file: f, run: r });
+    }
+    return out;
+}
+/** Latest API (not manual) run, optionally before a given run id. */
+export async function latestApiRun(root, beforeId) {
+    const runs = (await readRuns(root)).filter(({ file, run }) => !file.startsWith("manual-") && run.answers.some((a) => a.source === "api"));
+    runs.sort((a, b) => a.run.ts.localeCompare(b.run.ts));
+    for (let i = runs.length - 1; i >= 0; i--) {
+        if (beforeId && runs[i].run.id >= beforeId)
+            continue;
+        return runs[i].run;
+    }
+    return undefined;
+}
+function lastActualFor(prev, spec) {
+    if (!prev || !prev.models.some((m) => m.id === spec.id && m.search === spec.search))
+        return undefined;
+    const costs = prev.answers
+        .filter((a) => !a.error && a.source === "api" && a.family === spec.family && typeof a.costUsd === "number")
+        .map((a) => a.costUsd);
+    return costs.length >= 3 ? costs.reduce((s, x) => s + x, 0) / costs.length : undefined;
+}
+export function estimateFor(models, p) {
+    return estimateRun({
+        models: models.map((spec) => {
+            const price = priceOf(p.catalog, spec.id);
+            return { spec, price, profile: profileFor(spec).profile, searchUnit: searchUnitPrice(spec, price), lastActual: lastActualFor(p.previous, spec) };
+        }),
+        questions: p.questions.length,
+        scoredQuestions: p.questions.filter((q) => q.type !== "diagnostic" && !p.branded.has(q.id)).length,
+        samples: p.samples,
+        judge: p.judgeModel ? { model: p.judgeModel, price: priceOf(p.catalog, p.judgeModel) } : undefined,
+        followUps: p.followUps,
+    });
+}
+export async function buildPlan(root, args, deps = {}) {
+    const research = await loadResearch(root);
+    const problems = researchProblems(research);
+    if (!research || problems.length)
+        return { error: { text: researchHelp(problems), isError: true, structured: { status: "research_needed", problems } } };
+    const config = await loadConfig(root);
+    const notes = [];
+    const ctx = buildContext(research);
+    const domain = normalizeHost(research.site.domain || research.site.url);
+    let questions = (research.questions ?? []).filter((q) => q && typeof q.text === "string" && q.text.trim());
+    const fill = questions.filter((q) => isFill(q.text));
+    if (fill.length)
+        notes.push(`Skipped ${fill.length} question(s) still containing [FILL]: ${fill.map((q) => q.id).join(", ")}.`);
+    questions = questions.filter((q) => !isFill(q.text));
+    if (args.question_ids?.length) {
+        const want = new Set(args.question_ids.map((s) => s.trim()));
+        const unknown = [...want].filter((id) => !questions.some((q) => q.id === id));
+        if (unknown.length) {
+            return { error: { text: `Unknown question id(s): ${unknown.join(", ")}. Available: ${questions.map((q) => q.id).join(", ")}.`, isError: true } };
+        }
+        questions = questions.filter((q) => want.has(q.id));
+    }
+    const branded = new Set(questions.filter((q) => q.type !== "diagnostic" && mentions(ctx, q.text)).map((q) => q.id));
+    if (branded.size) {
+        notes.push(`${[...branded].join(", ")} name${branded.size === 1 ? "s" : ""} the brand, so ${branded.size === 1 ? "it is" : "they are"} scored like a diagnostic (not in the rates). Rewrite without the brand to measure discovery (book ch. 7).`);
+    }
+    const siteLangs = (research.site.languages ?? []).map((l) => l.toLowerCase().split(/[-_]/)[0]).filter(Boolean);
+    const langOf = new Map(questions.map((q) => [q.id, (q.lang || siteLangs[0] || "en").toLowerCase().split(/[-_]/)[0]]));
+    const measured = new Set(langOf.values());
+    const missing = [...new Set(siteLangs)].filter((l) => !measured.has(l));
+    if (missing.length) {
+        notes.push(`The site is published in ${[...new Set(siteLangs)].join(", ")} but no question is in ${missing.join(", ")}: a zero there would be unmeasured, not real. Add questions per language with katman_research (set each question's lang).`);
+    }
+    const multiLanguage = measured.size > 1;
+    const fetchImpl = deps.fetchImpl;
+    const now = deps.now?.() ?? new Date();
+    const catalog0 = await loadCatalog(root, { fetchImpl, now, offline: deps.offline });
+    const isAvailable = (slug) => isInCatalog(catalog0, slug);
+    const panel = resolvePanel({ input: args.models, config, googleGrounding: !!args.allow_google_grounding, isAvailable });
+    if (panel.errors.length)
+        return { error: { text: `# Visibility run: model selection problem\n\n${panel.errors.map((e) => `- ${e}`).join("\n")}`, isError: true } };
+    if (!panel.models.length)
+        return { error: { text: "# Visibility run: no model available\n\nNone of the requested assistant models is available on OpenRouter right now.\n\n" + panel.notes.map((n) => `- ${n}`).join("\n"), isError: true } };
+    const judgeModel = args.judge === false ? undefined : resolveJudge(config, isAvailable);
+    const withEndpoints = await loadCatalog(root, { fetchImpl, now, offline: deps.offline, endpointIds: panel.models.filter((m) => engineOf(m) === "native").map((m) => m.id) });
+    // The second load reads the cache the first one just wrote: keep the first load's provenance.
+    const catalog = { ...withEndpoints, source: catalog0.source === "live" ? "live" : withEndpoints.source, warning: catalog0.warning ?? withEndpoints.warning };
+    if (catalog.warning)
+        notes.push(catalog.warning);
+    notes.push(...panel.notes);
+    if (panel.models.some((m) => vendorOf(m.id) === "google" && m.search === "native")) {
+        notes.push("Google grounding is on (allow_google_grounding). Google's Gemini API grounding terms restrict storing and analysing grounded results; you are responsible for compliance. Katman stores excerpts and sources in .katman/runs/.");
+    }
+    if (panel.models.some((m) => m.family.includes("Exa search"))) {
+        notes.push("Gemini answers use Exa search (a proxy index), not Google grounding: Google's grounding terms forbid caching or analysing grounded results. Set allow_google_grounding: true to use Google's own grounding at your own responsibility.");
+    }
+    for (const m of panel.models) {
+        const ep = catalog.endpoints[m.id];
+        if (engineOf(m) === "native" && vendorOf(m.id) !== "perplexity" && ep && ep.tags.length && !ep.nativeSearchTags.length) {
+            notes.push(`${m.id}: no OpenRouter endpoint runs native search for it right now; OpenRouter would fall back to Exa.`);
+        }
+    }
+    const tunings = new Map(panel.models.map((m) => [m.id, tuningFor(m, catalog.models.get(m.id), catalog.endpoints[m.id])]));
+    const previous = await latestApiRun(root);
+    const samples = clamp(Math.floor(args.samples ?? 1), 1, MAX_SAMPLES);
+    const temperature = clamp(args.temperature ?? config.temperature ?? DEFAULT_TEMPERATURE, 0, 2);
+    const maxCost = Math.max(0.05, args.max_cost_usd ?? DEFAULT_MAX_COST_USD);
+    const concurrency = clamp(Math.floor(args.concurrency ?? 4), 1, 8);
+    const followUps = !!args.follow_up_sources;
+    const partial = { catalog, questions, branded, samples, followUps, judgeModel, previous };
+    const estimate = estimateFor(panel.models, partial);
+    return {
+        plan: {
+            root, research, config, ctx, domain, questions, branded, langOf, multiLanguage, panel, models: panel.models, catalog, tunings,
+            judgeModel, estimate, samples, temperature, maxCost, concurrency, followUps, notes, previous,
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Dry run
+function alternativesFor(plan, args) {
+    const out = [];
+    const isAvailable = (slug) => isInCatalog(plan.catalog, slug);
+    for (const preset of ["economy", "probe"]) {
+        if (plan.panel.preset === preset)
+            continue;
+        const p = resolvePanel({ input: preset, config: plan.config, googleGrounding: !!args.allow_google_grounding, isAvailable });
+        if (p.errors.length || !p.models.length)
+            continue;
+        const e = estimateFor(p.models, plan);
+        if (e.total.expected >= plan.estimate.total.expected)
+            continue;
+        const label = preset === "economy" ? `"economy" (same 5 assistants and indexes, smaller models)` : `"probe" (${p.models.map((m) => shortFamily(m.family)).join(" + ")} only)`;
+        out.push({ label, call: `models: "${preset}"`, expected: e.total.expected, high: e.total.high });
+    }
+    return out;
+}
+export async function dryRun(plan, args, deps = {}) {
+    const { key, source } = getApiKey(deps.env ?? process.env);
+    let info;
+    if (key)
+        info = await fetchKeyInfo(key, deps.fetchImpl);
+    const active = ACTIVE.get(plan.root);
+    const e = plan.estimate;
+    const text = renderDryRun({
+        brand: plan.research.brand.name,
+        domain: plan.domain,
+        questions: plan.questions,
+        scoredCount: plan.questions.filter((q) => q.type !== "diagnostic" && !plan.branded.has(q.id)).length,
+        diagnosticCount: plan.questions.filter((q) => q.type === "diagnostic").length,
+        brandedCount: plan.branded.size,
+        models: plan.models,
+        estimate: e,
+        maxCost: plan.maxCost,
+        samples: plan.samples,
+        temperature: plan.temperature,
+        concurrency: plan.concurrency,
+        followUps: plan.followUps,
+        key: { present: !!key, source, limitRemaining: info?.limitRemaining, usage: info?.usage, error: info && !info.ok ? info.error : undefined },
+        catalog: { source: plan.catalog.source, fetchedAt: plan.catalog.fetchedAt },
+        alternatives: e.total.expected > plan.maxCost || e.total.high > plan.maxCost ? alternativesFor(plan, args) : [],
+        notes: plan.notes,
+        keyHelp: key ? (info && !info.ok && info.status === 401 ? "The key was rejected by OpenRouter (401). Create a new key at https://openrouter.ai/keys and update the MCP server's env.\n\n" + keyHelp(e) : undefined) : keyHelp(e),
+        activeRun: active ? activeMessage(active) : undefined,
+        lastRun: plan.previous ? lastRunHeadline(plan.previous) : undefined,
+    });
+    return {
+        text,
+        structured: {
+            status: "dry_run",
+            answers: e.answers,
+            questions: plan.questions.length,
+            models: plan.models.map((m) => ({ id: m.id, family: m.family, search: m.search })),
+            estimateUsd: { low: round(e.total.low), expected: round(e.total.expected), high: round(e.total.high) },
+            maxCostUsd: plan.maxCost,
+            fitsCap: e.total.expected <= plan.maxCost,
+            keyPresent: !!key,
+            keyLimitRemainingUsd: info?.limitRemaining ?? null,
+        },
+    };
+}
+const round = (n, d = 4) => Math.round(n * 10 ** d) / 10 ** d;
+function lastRunHeadline(r) {
+    const b = Object.values(r.summary?.byFamily ?? {});
+    const answered = r.summary?.answered ?? 0;
+    const citedOf = b.reduce((t, x) => t + x.answered, 0);
+    return {
+        ts: r.ts,
+        answered,
+        mentioned: b.reduce((t, x) => t + x.mentioned, 0),
+        cited: b.reduce((t, x) => t + x.cited, 0),
+        citedOf,
+        cost: r.totalCostUsd,
+    };
+}
+/** Run id from a timestamp, suffixed (-2, -3…) when a run file with that name already exists. */
+export async function uniqueRunId(root, base) {
+    const taken = new Set(await listFiles(root, "runs"));
+    let id = base;
+    for (let n = 2; taken.has(`${id}.json`); n++)
+        id = `${base}-${n}`;
+    return id;
+}
+// ---------------------------------------------------------------------------
+// Real run
+const FOLLOW_UP = {
+    en: "Which articles or pages did you use for this answer? List the URLs.",
+    tr: "Bu cevabı yazarken hangi makaleleri veya sayfaları kullandın? URL'lerini listele.",
+    de: "Welche Artikel oder Seiten hast du für diese Antwort verwendet? Liste die URLs auf.",
+    fr: "Quels articles ou pages as-tu utilisés pour cette réponse ? Liste les URL.",
+    es: "¿Qué artículos o páginas usaste para esta respuesta? Enumera las URL.",
+};
+function blankRecord(t, samples) {
+    const a = {
+        questionId: t.q.id,
+        question: t.q.text,
+        questionType: t.q.type,
+        model: t.spec.id,
+        family: t.spec.family,
+        source: "api",
+        surface: `openrouter:${engineOf(t.spec)}`,
+        ts: new Date().toISOString(),
+        mentioned: false,
+        recommendedFirst: false,
+        rank: null,
+        cited: false,
+        namedProducts: [],
+        competitorsNamed: [],
+        citations: [],
+        answerExcerpt: "",
+        lang: t.lang,
+    };
+    if (samples > 1)
+        a.sample = t.sample;
+    return a;
+}
+function describeError(e) {
+    if (e instanceof OpenRouterError)
+        return truncate(`${e.kind}${e.status ? ` ${e.status}` : ""}: ${e.message}`, 220);
+    return truncate(redact(e instanceof Error ? e.message : String(e)), 220);
+}
+export async function realRun(plan, deps = {}) {
+    const { key } = getApiKey(deps.env ?? process.env);
+    if (!key)
+        return { text: "# Visibility run: no OpenRouter key\n\n" + keyHelp(plan.estimate), isError: true, structured: { status: "no_key" } };
+    const running = ACTIVE.get(plan.root);
+    if (running)
+        return { text: activeMessage(running), structured: { status: "in_progress", ...running } };
+    // Take the lock before the first await, so a retried tool call can't start a second, paid run.
+    const now = deps.now ?? (() => new Date());
+    const active = { id: "starting", started: now().toISOString(), done: 0, total: plan.samples * plan.questions.length * plan.models.length };
+    ACTIVE.set(plan.root, active);
+    try {
+        return await executeRun(plan, deps, key, active, now);
+    }
+    finally {
+        if (ACTIVE.get(plan.root) === active)
+            ACTIVE.delete(plan.root);
+    }
+}
+async function executeRun(plan, deps, key, active, now) {
+    const e = plan.estimate;
+    if (e.total.expected > plan.maxCost) {
+        return {
+            text: [
+                "# Visibility run: not started (over the cost cap)",
+                "",
+                `Expected cost ${money(e.total.expected)} (range ${money(e.total.low)}–${money(e.total.high)}) is above max_cost_usd ${money(plan.maxCost)}. Nothing was spent.`,
+                `Run the dry run to see the options, then ask the user before raising the cap (e.g. max_cost_usd: ${Math.ceil(e.total.high * 2) / 2}) or choosing models: "economy" / "probe".`,
+            ].join("\n"),
+            isError: true,
+            structured: { status: "refused_over_cap", expectedUsd: round(e.total.expected), maxCostUsd: plan.maxCost },
+        };
+    }
+    const fetchImpl = deps.fetchImpl;
+    const info = await fetchKeyInfo(key, fetchImpl);
+    if (!info.ok && info.status === 401) {
+        return { text: "# Visibility run: key rejected\n\nOpenRouter rejected the key (401). Nothing was spent.\n\n" + keyHelp(e), isError: true, structured: { status: "key_rejected" } };
+    }
+    if (info.ok && typeof info.limitRemaining === "number" && info.limitRemaining < e.total.low) {
+        return {
+            text: `# Visibility run: not started\n\nThis key has ${money(info.limitRemaining)} of its credit limit left, less than the lowest estimate (${money(e.total.low)}). Raise the key's limit at https://openrouter.ai/keys or add credit, then try again. Nothing was spent.`,
+            isError: true,
+            structured: { status: "key_limit_too_low", limitRemainingUsd: info.limitRemaining },
+        };
+    }
+    const started = now();
+    const runId = await uniqueRunId(plan.root, stamp(started));
+    const tasks = [];
+    for (let s = 1; s <= plan.samples; s++) {
+        for (const q of plan.questions)
+            for (const spec of plan.models)
+                tasks.push({ q, spec, sample: s, lang: plan.langOf.get(q.id) ?? "en" });
+    }
+    active.id = runId;
+    active.started = started.toISOString();
+    active.total = tasks.length;
+    const rows = new Map(e.rows.map((r) => [`${r.spec.id}|${r.spec.search}`, r]));
+    const prices = new Map(plan.models.map((m) => [m.id, priceOf(plan.catalog, m.id)]));
+    const judgePrice = plan.judgeModel ? priceOf(plan.catalog, plan.judgeModel) : undefined;
+    const chatOpts = { fetchImpl, sleepFn: deps.sleepFn };
+    const guard = { spent: 0, reserved: 0 };
+    const costs = { answers: 0, followUps: 0, judge: 0 };
+    const runtime = [];
+    const surfaceOf = new Map();
+    const triedPlugin = new Set();
+    const dead = new Map();
+    let stopReason;
+    let stopShort;
+    let userCancelled = false;
+    let judgeFailures = 0;
+    let truncated = 0;
+    let vetoed = 0;
+    const signal = deps.signal;
+    const onAbort = () => {
+        const reason = String(signal?.reason ?? "");
+        if (/time(d)?\s*out|timeout/i.test(reason))
+            runtime.push("The MCP client timed out while waiting; the run finished in the background.");
+        else
+            userCancelled = true;
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+    if (signal?.aborted)
+        onAbort();
+    const identity = identityOf(plan.research);
+    const ask = async (spec, messages, lang, maxTokens) => {
+        const tuning = plan.tunings.get(spec.id);
+        const price = prices.get(spec.id);
+        const surface = surfaceOf.get(spec.id) ?? "tool";
+        let json;
+        try {
+            json = await postChat(key, buildAnswerBody(spec, messages, tuning, { temperature: plan.temperature, lang, surface, maxTokens }), chatOpts);
+        }
+        catch (err) {
+            const searchOn = engineOf(spec) !== "none" && vendorOf(spec.id) !== "perplexity";
+            // 400s are not billed: any task whose server-tool request is rejected may retry once on the legacy web plugin.
+            if (err instanceof OpenRouterError && err.kind === "bad_request" && searchOn && surface === "tool") {
+                json = await postChat(key, buildAnswerBody(spec, messages, tuning, { temperature: plan.temperature, lang, surface: "plugin", maxTokens }), chatOpts);
+                surfaceOf.set(spec.id, "plugin");
+                if (!triedPlugin.has(spec.id)) {
+                    triedPlugin.add(spec.id);
+                    runtime.push(`${spec.id}: OpenRouter rejected the web-search server tool (${truncate(err.message, 120)}); used the legacy web plugin instead.`);
+                }
+            }
+            else
+                throw err;
+        }
+        const c = parseCompletion(json);
+        return { c, cost: costOf(c, price, searchUnitPrice(spec, price)) };
+    };
+    const judge = async (t, answer) => {
+        if (!plan.judgeModel || !judgePrice)
+            return { verdict: null, cost: 0 };
+        const messages = buildJudgeMessages({
+            brandName: plan.research.brand.name,
+            alternateNames: plan.research.brand.alternateNames,
+            domain: plan.domain,
+            competitors: (plan.research.competitors ?? []).map((c) => c.name).filter(Boolean),
+            question: t.q.text,
+            questionType: t.q.type,
+            answer,
+            identity: t.q.type === "diagnostic" ? identity : undefined,
+        });
+        const info = plan.catalog.models.get(plan.judgeModel);
+        let cost = 0;
+        for (const structured of [true, false]) {
+            try {
+                const json = await postChat(key, buildJudgeBody(plan.judgeModel, messages, info, structured), { ...chatOpts, timeoutMs: 60_000, attempts: 2 });
+                const c = parseCompletion(json);
+                cost += costOf(c, judgePrice, 0);
+                const v = parseJudge(c.text);
+                if (v)
+                    return { verdict: v, cost };
+            }
+            catch (err) {
+                if (err instanceof OpenRouterError && (err.kind === "credits" || err.kind === "auth"))
+                    throw Object.assign(err, { judgeCost: cost });
+                if (!(err instanceof OpenRouterError && err.kind === "bad_request"))
+                    break;
+            }
+        }
+        return { verdict: null, cost };
+    };
+    const stopFor = (err, spec) => {
+        if (err.kind === "credits" && !stopReason) {
+            stopShort = "out of credits (402)";
+            stopReason = `OpenRouter returned 402 (${truncate(err.message, 140)}): out of credits or the key's spending limit is reached. Add credit or raise the key limit at https://openrouter.ai/settings/credits, then re-run.`;
+        }
+        if (err.kind === "auth" && !stopReason) {
+            stopShort = "key rejected (401)";
+            stopReason = "OpenRouter rejected the key (401) during the run.";
+        }
+        if (err.kind === "not_found" && !dead.has(spec.id)) {
+            dead.set(spec.id, `not run: ${spec.id} is not available on OpenRouter (404)`);
+            runtime.push(`${spec.id} returned 404 (model not found); its remaining questions were not asked. The next run picks the next model in the family from the live catalog.`);
+        }
+    };
+    const runTask = async (t) => {
+        const rec = blankRecord(t, plan.samples);
+        if (stopReason)
+            return { ...rec, error: `not run: stopped, ${stopShort ?? "run stopped"}` };
+        if (userCancelled)
+            return { ...rec, error: "not run: cancelled by the user" };
+        const deadReason = dead.get(t.spec.id);
+        if (deadReason)
+            return { ...rec, error: deadReason };
+        const row = rows.get(`${t.spec.id}|${t.spec.search}`);
+        const scoredQ = t.q.type !== "diagnostic" && !plan.branded.has(t.q.id);
+        const cost = taskCost(row, e, plan.followUps && scoredQ);
+        if (guard.spent + guard.reserved + cost.high > plan.maxCost) {
+            return { ...rec, error: `skipped: cost cap ${money(plan.maxCost)} reached (spent ${money(guard.spent)})` };
+        }
+        guard.reserved += cost.expected;
+        let actual = 0;
+        try {
+            let answer;
+            try {
+                answer = await ask(t.spec, [{ role: "user", content: t.q.text }], t.lang);
+            }
+            catch (err) {
+                if (err instanceof OpenRouterError)
+                    stopFor(err, t.spec);
+                return { ...rec, error: describeError(err) };
+            }
+            actual += answer.cost;
+            costs.answers += answer.cost;
+            const c = answer.c;
+            const engine = engineOf(t.spec);
+            const surfaceTag = (surfaceOf.get(t.spec.id) === "plugin" ? "plugin:" : "") + engine;
+            const base = {
+                ...rec,
+                model: c.model || t.spec.id,
+                surface: `openrouter:${surfaceTag}${c.provider ? `@${c.provider}` : ""}`,
+                ts: new Date().toISOString(),
+                searched: engine !== "none" && (c.webSearches > 0 || c.sources.length > 0),
+                costUsd: round(answer.cost, 6),
+            };
+            if (plan.tunings.get(t.spec.id)?.temperature)
+                base.temperature = plan.temperature;
+            if (!c.text.trim())
+                return { ...base, error: "empty answer (charged but not counted)" };
+            if (c.finishReason === "length")
+                truncated++;
+            let followUpSources = [];
+            if (plan.followUps && scoredQ && !mentions(plan.ctx, c.text) && !stopReason) {
+                try {
+                    const fu = await ask(t.spec, [
+                        { role: "user", content: t.q.text },
+                        { role: "assistant", content: c.text },
+                        { role: "user", content: FOLLOW_UP[t.lang] ?? FOLLOW_UP.en },
+                    ], t.lang);
+                    actual += fu.cost;
+                    costs.followUps += fu.cost;
+                    base.costUsd = round((base.costUsd ?? 0) + fu.cost, 6);
+                    followUpSources = [...fu.c.sources, ...extractLinks(fu.c.text)];
+                }
+                catch (err) {
+                    if (err instanceof OpenRouterError)
+                        stopFor(err, t.spec);
+                }
+            }
+            let verdict = null;
+            if (plan.judgeModel && !stopReason) {
+                try {
+                    const j = await judge(t, c.text);
+                    verdict = j.verdict;
+                    actual += j.cost;
+                    costs.judge += j.cost;
+                    if (!verdict)
+                        judgeFailures++;
+                }
+                catch (err) {
+                    const jc = err.judgeCost ?? 0;
+                    actual += jc;
+                    costs.judge += jc;
+                    judgeFailures++;
+                    if (err instanceof OpenRouterError)
+                        stopFor(err, t.spec);
+                }
+            }
+            const r = analyzeAnswer(plan.ctx, { text: c.text, sources: c.sources, followUpSources, questionType: t.q.type, judge: verdict });
+            if (r.vetoed)
+                vetoed++;
+            const out = {
+                ...base,
+                mentioned: r.mentioned,
+                recommendedFirst: r.recommendedFirst,
+                rank: r.rank,
+                cited: r.cited,
+                namedProducts: r.namedProducts,
+                competitorsNamed: r.competitorsNamed,
+                citations: r.citations,
+                answerExcerpt: r.answerExcerpt,
+            };
+            if (r.identityCorrect !== undefined)
+                out.identityCorrect = r.identityCorrect;
+            return out;
+        }
+        finally {
+            guard.reserved -= cost.expected;
+            guard.spent += actual;
+        }
+    };
+    let answers;
+    try {
+        answers = await mapLimit(tasks, plan.concurrency, async (t) => {
+            const r = await runTask(t);
+            active.done++;
+            deps.onProgress?.(active.done, tasks.length, `${shortFamily(t.spec.family)} · ${t.q.id}${r.error ? ` (${r.error.split(":")[0]})` : r.mentioned ? " · named" : ""}`);
+            return r;
+        });
+    }
+    finally {
+        signal?.removeEventListener("abort", onAbort);
+    }
+    const { summary, extras } = summarize(answers, {
+        branded: plan.branded,
+        ownDomains: plan.ctx.ownDomains,
+        isBrand: (n) => mentions(plan.ctx, n),
+        canonicalCompetitor: (n) => canonicalCompetitor(plan.ctx, n),
+    });
+    const run = {
+        id: runId,
+        ts: started.toISOString(),
+        brand: plan.research.brand.name,
+        domain: plan.domain,
+        models: plan.models,
+        temperature: plan.temperature,
+        answers,
+        summary,
+        totalCostUsd: round(guard.spent, 6),
+    };
+    const runFile = `.katman/runs/${runId}.json`;
+    await writeJson(plan.root, `runs/${runId}.json`, run);
+    await regenerateMeasurements(plan.root, plan.research, plan.ctx, now());
+    const diff = plan.previous ? diffRuns(plan.previous, run, plan.branded) : undefined;
+    const warnings = [...plan.notes, ...runtime];
+    if (extras.citationBlind.length)
+        warnings.push(`${extras.citationBlind.join(", ")} returned no sources at all in this run, so ${extras.citationBlind.length === 1 ? "it is" : "they are"} left out of the cited rate.`);
+    if (extras.unsearched)
+        warnings.push(`${extras.unsearched} answer${extras.unsearched === 1 ? " was" : "s were"} written without a web search (the model answered from memory). Consumer apps decide the same way; generic questions are often answered from memory with big brands.`);
+    if (truncated)
+        warnings.push(`${truncated} answer${truncated === 1 ? "" : "s"} hit the token limit and ${truncated === 1 ? "was" : "were"} cut off; a brand named at the very end could be missed.`);
+    if (judgeFailures)
+        warnings.push(`The judge failed on ${judgeFailures} answer${judgeFailures === 1 ? "" : "s"}; the product order there comes from the heuristic (list order).`);
+    if (vetoed)
+        warnings.push(`${vetoed} name-only match${vetoed === 1 ? " was" : "es were"} judged to be the ordinary word, not your brand, and not counted as named.`);
+    const mismatched = [...new Set(answers.filter((a) => !a.error && a.source === "api").filter((a) => !plan.models.some((m) => a.model === m.id || a.model.startsWith(m.id))).map((a) => a.model))];
+    if (mismatched.length)
+        warnings.push(`OpenRouter answered with a different model than requested: ${mismatched.join(", ")}.`);
+    if (extras.errors)
+        warnings.push(`${extras.errors} call${extras.errors === 1 ? "" : "s"} failed after retries (marked "err"; see the run file).`);
+    if (extras.skipped) {
+        const reasons = new Map();
+        for (const a of answers) {
+            if (!a.error || !/^(skipped|not run)/i.test(a.error))
+                continue;
+            const r = a.error.replace(/\s*\(spent[^)]*\)$/, "");
+            reasons.set(r, (reasons.get(r) ?? 0) + 1);
+        }
+        warnings.push(`${extras.skipped} answer${extras.skipped === 1 ? " was" : "s were"} not run (marked "skip"): ${[...reasons.entries()].map(([r, n]) => `${n} × ${r}`).join("; ")}.`);
+    }
+    if (userCancelled && !stopReason)
+        stopReason = "the run was cancelled; calls already running were finished and saved.";
+    if (!stopReason && answers.some((a) => a.error?.startsWith("skipped: cost cap")))
+        stopReason = `the cost cap (${money(plan.maxCost)}) was reached before every question was asked.`;
+    if (!stopReason && dead.size)
+        stopReason = `${[...dead.keys()].join(", ")} returned 404 (model not found), so its questions were not asked.`;
+    const nextRun = addDays(run.ts, 14);
+    const text = renderRunReport({
+        run, extras, diff, questions: plan.questions, branded: plan.branded, costs, runFile, warnings,
+        stopped: stopReason, nextRun, multiLanguage: plan.multiLanguage,
+    });
+    const answered = answers.filter((a) => !a.error).length;
+    await logEvent(plan.root, `katman_visibility_run ${runId}: ${answered}/${answers.length} answers, named ${Math.round(summary.mentionRate * 100)}% of ${summary.answered} scored, cited ${Math.round(summary.citationRate * 100)}%, cost ${money(run.totalCostUsd ?? 0)}${stopReason ? " (incomplete)" : ""}`);
+    return {
+        text,
+        structured: {
+            status: stopReason ? "partial" : "complete",
+            runId,
+            runFile,
+            answers: answers.length,
+            answered,
+            scored: summary.answered,
+            mentionRate: summary.mentionRate,
+            citationRate: summary.citationRate,
+            recommendedFirstRate: summary.recommendedFirstRate,
+            byFamily: summary.byFamily,
+            byLanguage: extras.byLanguage,
+            topCompetitors: summary.topCompetitors,
+            topCitedDomains: summary.topCitedDomains,
+            totalCostUsd: run.totalCostUsd,
+            delta: diff ? { previousRun: diff.previousId, pairs: diff.pairs, before: diff.before, after: diff.after } : null,
+            nextRunDate: nextRun,
+            stoppedReason: stopReason ?? null,
+            note: PROXY_NOTE,
+        },
+    };
+}
+export async function regenerateMeasurements(root, research, ctx, now = new Date()) {
+    const runs = (await readRuns(root)).map((r) => r.run);
+    const md = renderMeasurements(runs, { brand: research.brand.name, domain: normalizeHost(research.site.domain || research.site.url), ownDomains: ctx.ownDomains, now });
+    return writeText(root, "measurements.md", md);
+}
+/** Entry point shared by the MCP tool and the CLI. */
+export async function visibilityRun(root, args, deps = {}) {
+    const { plan, error } = await buildPlan(root, args, deps);
+    if (error || !plan)
+        return error ?? { text: "Could not plan the run.", isError: true };
+    if (args.dry_run !== false)
+        return dryRun(plan, args, deps);
+    return realRun(plan, deps);
+}
+/** For tests: the in-memory run lock. */
+export function _activeRuns() {
+    return ACTIVE;
+}