ai-runtime-engine 1.1.0

package/dist/comparison/analysis.js

@@ -0,0 +1,177 @@
+/**
+ * Deterministic comparison analysis — no model calls, no randomness. Outputs are compared by token-set
+ * similarity (reusing the memory tokenizer), clustered into agreement groups, diffed pairwise, checked
+ * for unique ("missing from others") sentences, and — for structured (JSON) outputs — checked for direct
+ * field-level contradictions. Everything here is a pure function of the candidates, so it is fully unit
+ * testable and never fabricates a contradiction it cannot ground (free-text contradictions are left to a
+ * judge; only structured fields are asserted as contradictions).
+ */
+import { tokenize } from '../memory/bm25.js';
+/** Token-set Jaccard similarity of two token bags, 0..1. Two empty bags count as identical (1). */
+export function jaccard(a, b) {
+    const sa = new Set(a);
+    const sb = new Set(b);
+    if (sa.size === 0 && sb.size === 0)
+        return 1;
+    let inter = 0;
+    for (const t of sa)
+        if (sb.has(t))
+            inter += 1;
+    const union = sa.size + sb.size - inter;
+    return union === 0 ? 1 : inter / union;
+}
+/**
+ * Similarity of two output texts, 0..1 (Jaccard over their tokenized bags). When either side tokenizes
+ * to nothing (e.g. all stopwords/punctuation), tokens carry no signal — fall back to exact normalized
+ * text equality so two DIFFERENT all-stopword outputs are not reported as identical.
+ */
+export function similarity(a, b) {
+    const ta = tokenize(a);
+    const tb = tokenize(b);
+    if (ta.length === 0 || tb.length === 0)
+        return normalizeText(a) === normalizeText(b) ? 1 : 0;
+    return jaccard(ta, tb);
+}
+/** Lowercase, collapse runs of whitespace, and trim — the fallback comparison when there are no tokens. */
+function normalizeText(s) {
+    return s.toLowerCase().replace(/\s+/g, ' ').trim();
+}
+/** Deterministic key-sorted JSON stringification (so field order never affects comparison). */
+export function stableStringify(value) {
+    return JSON.stringify(sortValue(value));
+}
+function sortValue(value) {
+    if (Array.isArray(value))
+        return value.map(sortValue);
+    if (value && typeof value === 'object') {
+        const out = {};
+        for (const key of Object.keys(value).sort())
+            out[key] = sortValue(value[key]);
+        return out;
+    }
+    return value;
+}
+/** The text used to compare a candidate: its text, or a stable stringification of its JSON. */
+export function comparableText(c) {
+    if (c.text !== undefined && c.text !== '')
+        return c.text;
+    if (c.json !== undefined)
+        return stableStringify(c.json);
+    return '';
+}
+/** Candidates that both ran successfully and produced comparable output. */
+export function comparable(candidates) {
+    return candidates.filter((c) => c.ok && comparableText(c) !== '');
+}
+/**
+ * Greedy agreement clustering: each candidate joins the first cluster whose representative it is at
+ * least `threshold`-similar to, else starts its own. Deterministic in input order.
+ */
+export function clusterByAgreement(candidates, threshold) {
+    const clusters = [];
+    for (const c of candidates) {
+        const text = comparableText(c);
+        const hit = clusters.find((cl) => similarity(cl.repText, text) >= threshold);
+        if (hit)
+            hit.labels.push(c.label);
+        else
+            clusters.push({ rep: c.label, repText: text, labels: [c.label] });
+    }
+    return clusters.map((cl) => ({ labels: cl.labels, representative: cl.rep }));
+}
+/** Every pair of candidates whose outputs are below the agreement threshold, most-different first. */
+export function pairwiseDifferences(candidates, threshold) {
+    const out = [];
+    for (let i = 0; i < candidates.length; i += 1) {
+        for (let j = i + 1; j < candidates.length; j += 1) {
+            const sim = similarity(comparableText(candidates[i]), comparableText(candidates[j]));
+            if (sim < threshold)
+                out.push({ a: candidates[i].label, b: candidates[j].label, similarity: round(sim) });
+        }
+    }
+    return out.sort((x, y) => x.similarity - y.similarity);
+}
+/** Split text into trimmed sentence-ish units on sentence terminators and newlines. */
+export function splitSentences(text) {
+    return text
+        .split(/(?<=[.!?])\s+|\n+/)
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+}
+/**
+ * For each candidate, sentences that no OTHER candidate expresses (similarity < 0.6 to every sentence of
+ * every other candidate). This surfaces information one model included and others omitted. Skipped when
+ * fewer than two candidates. Capped to keep output readable.
+ */
+export function missingPoints(candidates, perCandidateCap = 5) {
+    if (candidates.length < 2)
+        return [];
+    const sentences = candidates.map((c) => splitSentences(comparableText(c)).map((s) => ({ text: s, tokens: tokenize(s) })));
+    const out = [];
+    for (let i = 0; i < candidates.length; i += 1) {
+        const others = sentences.filter((_, k) => k !== i).flat();
+        const unique = [];
+        for (const s of sentences[i]) {
+            if (s.tokens.length === 0)
+                continue;
+            const echoed = others.some((o) => jaccard(s.tokens, o.tokens) >= 0.6);
+            if (!echoed)
+                unique.push(s.text);
+            if (unique.length >= perCandidateCap)
+                break;
+        }
+        if (unique.length)
+            out.push({ label: candidates[i].label, points: unique });
+    }
+    return out;
+}
+/**
+ * Direct contradictions across STRUCTURED outputs only: for each top-level scalar field present in ≥2
+ * candidates' JSON, if the candidates disagree on its value it is a contradiction. Objects/arrays are not
+ * asserted as contradictions here (that needs semantic judgment, not a deterministic diff).
+ */
+export function structuredContradictions(candidates) {
+    const byField = new Map();
+    for (const c of candidates) {
+        if (!c.json || typeof c.json !== 'object' || Array.isArray(c.json))
+            continue;
+        for (const [key, raw] of Object.entries(c.json)) {
+            if (!isScalar(raw))
+                continue;
+            const list = byField.get(key) ?? [];
+            list.push({ label: c.label, value: String(raw) });
+            byField.set(key, list);
+        }
+    }
+    const out = [];
+    for (const [field, values] of byField) {
+        if (values.length < 2)
+            continue;
+        const distinct = new Set(values.map((v) => v.value));
+        if (distinct.size >= 2)
+            out.push({ field, values });
+    }
+    return out.sort((a, b) => a.field.localeCompare(b.field));
+}
+function isScalar(v) {
+    return typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean';
+}
+function round(n) {
+    return Math.round(n * 100) / 100;
+}
+/** Full deterministic analysis of a set of candidates at the given agreement threshold. */
+export function analyze(candidates, threshold) {
+    const usable = comparable(candidates);
+    const clusters = clusterByAgreement(usable, threshold);
+    const differences = pairwiseDifferences(usable, threshold);
+    return {
+        clusters,
+        // Unanimity means EVERY pair agrees. Deriving it from pairwise differences (not cluster count) keeps
+        // it self-consistent: greedy single-link clustering is non-transitive, so `clusters.length === 1`
+        // could otherwise be true while `differences` is non-empty.
+        unanimous: usable.length > 0 && differences.length === 0,
+        differences,
+        missing: missingPoints(usable),
+        contradictions: structuredContradictions(usable),
+    };
+}

package/dist/comparison/comparator.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Comparator — the fan-out engine. It sends the SAME task to each pin through the ONE router
+ * (`AI.run()` with a provider/model pin), collects the outputs, analyzes them deterministically, and
+ * ranks them EVIDENCE-first:
+ *   • evidence — a caller `validate()` scores each output objectively (e.g. run the tests); the passing
+ *     one wins. This is authoritative and beats any judge, even when a judge is also configured.
+ *   • judge    — a labeled fallback: a pinned judge model picks a winner. Recorded as OPINION.
+ *   • none     — neither available: report agreement/differences only, no winner claimed.
+ * Budgets are obeyed: `maxCalls` caps total model calls (fan-out + judge), `maxParallelSteps` bounds
+ * concurrency, and `maxCostUsd` is passed to each run so no single candidate overspends. The comparator
+ * mutates nothing itself; an evidence `validate()` that runs tools does so through the caller's
+ * permission-gated runtime.
+ */
+import type { AI } from '../plugin/ai.js';
+import type { RoutingPreferences, RunRequest } from '../types.js';
+import type { ComparisonCandidate, ComparisonPin, ComparisonResult, EvidenceResult } from './comparison.js';
+export interface CompareInput {
+    /** Human description of what is being compared (for rendering/telemetry). */
+    goal: string;
+    /** Router task id. Defaults to 'chat'. */
+    task?: string;
+    /** The input sent to every pin. */
+    input: RunRequest['input'];
+    system?: string;
+    /** Output format for every pin (e.g. `{ format: 'json' }` to compare structured outputs). */
+    output?: RunRequest['output'];
+    /** The candidates to compare — at least two. */
+    pins: ComparisonPin[];
+    ai: AI;
+    /** Budget/concurrency (read from ExecutionPolicy by the Runtime). */
+    policy?: {
+        maxCalls?: number;
+        maxCostUsd?: number;
+        maxParallelSteps?: number;
+    };
+    /** Evidence function: objective validation of a candidate's output. Beats any judge. */
+    validate?: (candidate: ComparisonCandidate) => Promise<EvidenceResult> | EvidenceResult;
+    /** Optional judge model (a labeled fallback used only when no evidence is available). */
+    judge?: ComparisonPin;
+    /** User exclude/prefer routing applied to every pin (and the judge) — a hard-excluded model won't run. */
+    routing?: RoutingPreferences;
+    /** Agreement/clustering similarity threshold, 0..1 (default 0.75). */
+    agreementThreshold?: number;
+    signal?: AbortSignal;
+}
+export declare function compare(input: CompareInput): Promise<ComparisonResult>;

package/dist/comparison/comparator.js

@@ -0,0 +1,270 @@
+/**
+ * Comparator — the fan-out engine. It sends the SAME task to each pin through the ONE router
+ * (`AI.run()` with a provider/model pin), collects the outputs, analyzes them deterministically, and
+ * ranks them EVIDENCE-first:
+ *   • evidence — a caller `validate()` scores each output objectively (e.g. run the tests); the passing
+ *     one wins. This is authoritative and beats any judge, even when a judge is also configured.
+ *   • judge    — a labeled fallback: a pinned judge model picks a winner. Recorded as OPINION.
+ *   • none     — neither available: report agreement/differences only, no winner claimed.
+ * Budgets are obeyed: `maxCalls` caps total model calls (fan-out + judge), `maxParallelSteps` bounds
+ * concurrency, and `maxCostUsd` is passed to each run so no single candidate overspends. The comparator
+ * mutates nothing itself; an evidence `validate()` that runs tools does so through the caller's
+ * permission-gated runtime.
+ */
+import { analyze } from './analysis.js';
+import { comparable, comparableText } from './analysis.js';
+const DEFAULT_THRESHOLD = 0.75;
+function pinLabel(pin, index) {
+    if (pin.label)
+        return pin.label;
+    if (pin.provider)
+        return pin.model ? `${pin.provider}/${pin.model}` : pin.provider;
+    return `candidate-${index + 1}`;
+}
+/**
+ * Assign genuinely-unique display labels, preserving order. A generated `base#n` is re-checked against
+ * every label already emitted, so it can never collide with a caller-supplied label like "a#2".
+ */
+function uniqueLabels(pins) {
+    const used = new Set();
+    return pins.map((pin, i) => {
+        const base = pinLabel(pin, i);
+        let label = base;
+        let n = 2;
+        while (used.has(label))
+            label = `${base}#${n++}`;
+        used.add(label);
+        return label;
+    });
+}
+/** Run `items` through `fn` in order, at most `limit` concurrently, aborting further batches on signal. */
+async function boundedMap(items, limit, fn, signal) {
+    const max = Math.max(1, limit);
+    const out = [];
+    for (let i = 0; i < items.length; i += max) {
+        if (signal?.aborted)
+            break;
+        const batch = items.slice(i, i + max);
+        const results = await Promise.all(batch.map((item, j) => fn(item, i + j)));
+        out.push(...results);
+    }
+    return out;
+}
+function latencyOf(run) {
+    return run.routing.attempts.reduce((sum, a) => sum + (a.latencyMs ?? 0), 0);
+}
+/** The provider/model of the router's last attempt — used to attribute a failed run that selected nothing. */
+function lastAttempt(run) {
+    const a = run.routing.attempts[run.routing.attempts.length - 1];
+    return a ? { providerId: a.providerId, model: a.model } : undefined;
+}
+export async function compare(input) {
+    const task = input.task ?? 'chat';
+    const threshold = input.agreementThreshold ?? DEFAULT_THRESHOLD;
+    if (input.pins.length < 2)
+        throw new Error('comparison requires at least two pins');
+    const labels = uniqueLabels(input.pins);
+    const maxCalls = input.policy?.maxCalls;
+    const parallel = input.policy?.maxParallelSteps ?? 2;
+    // Budget: maxCalls caps TOTAL model calls (fan-out pins + an optional judge). Reserve one call for the
+    // judge when one is configured and no evidence function will decide the ranking.
+    const judgeReserve = input.judge && !input.validate ? 1 : 0;
+    const pinBudget = maxCalls === undefined ? input.pins.length : Math.max(0, maxCalls - judgeReserve);
+    const toRun = Math.min(input.pins.length, pinBudget);
+    const budgetSkipped = input.pins.length - toRun;
+    const runPins = input.pins.slice(0, toRun);
+    const runLabels = labels.slice(0, toRun);
+    const candidates = await boundedMap(runPins, parallel, async (pin, index) => {
+        const req = {
+            task,
+            input: input.input,
+            ...(input.system ? { system: input.system } : {}),
+            ...(input.output ? { output: input.output } : {}),
+            ...(input.routing ? { routing: input.routing } : {}),
+            ...(pin.provider ? { provider: pin.provider } : {}),
+            ...(pin.model ? { model: pin.model } : {}),
+            ...(input.policy?.maxCostUsd !== undefined ? { budget: { maxCostUsd: input.policy.maxCostUsd } } : {}),
+        };
+        let run;
+        try {
+            run = await input.ai.run(req);
+        }
+        catch (err) {
+            // The router normalizes provider failures into ok:false results; an unexpected throw must not
+            // crash the whole fan-out — record it as a failed candidate instead.
+            return { label: runLabels[index], pin, ok: false, error: { category: 'UNKNOWN', message: err instanceof Error ? err.message : String(err), retryable: false } };
+        }
+        const attempted = run.routing.selected ?? lastAttempt(run);
+        return {
+            label: runLabels[index],
+            pin,
+            ok: run.ok,
+            ...(run.response?.text !== undefined ? { text: run.response.text } : {}),
+            ...(run.response?.json !== undefined ? { json: run.response.json } : {}),
+            ...(run.routing.selected ? { selected: run.routing.selected } : {}),
+            ...(attempted ? { attempted } : {}),
+            confidence: run.confidence,
+            latencyMs: latencyOf(run),
+            ...(run.error ? { error: run.error } : {}),
+        };
+    }, input.signal);
+    const usable = comparable(candidates);
+    // Evidence pass (authoritative). Runs the caller's objective check on each successful candidate.
+    if (input.validate) {
+        await boundedMap(usable, parallel, async (c) => {
+            c.evidence = await input.validate(c);
+            return c;
+        }, input.signal);
+    }
+    const analysis = analyze(candidates, threshold);
+    const ranking = await rank(usable, task, input);
+    const signals = buildSignals(candidates, task, ranking);
+    // Accounting: `ran` is what actually completed; `skipped` = requested − ran covers BOTH budget-skipped
+    // pins and any pins dropped by an abort (so requested === ran + skipped always holds).
+    const aborted = (input.signal?.aborted ?? false) || candidates.length < toRun;
+    const usage = {
+        requested: input.pins.length,
+        ran: candidates.length,
+        skipped: input.pins.length - candidates.length,
+        ...(aborted ? { aborted: true } : {}),
+        ...(aborted
+            ? { note: `aborted before all pins ran (${candidates.length}/${input.pins.length})` }
+            : budgetSkipped > 0
+                ? { note: `skipped ${budgetSkipped} pin(s): call budget (maxCalls=${maxCalls}) reached` }
+                : {}),
+    };
+    return { goal: input.goal, task, candidates, analysis, ranking, usage, signals };
+}
+async function rank(usable, task, input) {
+    // 1) Evidence — authoritative. Ordered by (passed, score, confidence). Winner only if one passed. Only
+    // taken when evidence was actually gathered for EVERY usable candidate — a run cut short by an abort
+    // must not be reported as an authoritative evidence verdict.
+    if (input.validate && usable.length > 0 && usable.every((c) => c.evidence !== undefined)) {
+        const ranked = [...usable].sort(byEvidence);
+        const winner = ranked.find((c) => c.evidence?.ok);
+        return {
+            method: 'evidence',
+            judgeOpinion: false,
+            ranked: ranked.map((c) => ({ label: c.label, score: c.evidence?.score ?? 0, why: evidenceWhy(c) })),
+            ...(winner ? { winner: winner.label } : {}),
+        };
+    }
+    // 2) Judge — labeled opinion. Requires at least two candidates to actually compare (a verdict over a
+    // lone survivor is vacuous). The judge runs as one extra model call reserved from the budget.
+    if (input.judge && usable.length >= 2) {
+        const verdict = await runJudge(usable, task, input);
+        if (verdict) {
+            const ordered = [verdict.winner, ...usable.map((c) => c.label).filter((l) => l !== verdict.winner)];
+            const ranked = ordered.map((label, i) => ({
+                label,
+                score: i === 0 ? 1 : 0,
+                why: i === 0 ? `judge picked this (opinion, not evidence): ${verdict.reason}` : 'not selected by judge',
+            }));
+            return { method: 'judge', judgeOpinion: true, ranked, winner: verdict.winner };
+        }
+    }
+    // 3) None — report only. Ordered by router confidence, but NO winner is claimed (not a verdict).
+    const ranked = [...usable].sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0));
+    return {
+        method: 'none',
+        judgeOpinion: false,
+        ranked: ranked.map((c) => ({ label: c.label, score: round(c.confidence ?? 0), why: 'no evidence or judge — ordered by router-reported confidence (not a quality verdict)' })),
+    };
+}
+function byEvidence(a, b) {
+    const ao = a.evidence?.ok ? 1 : 0;
+    const bo = b.evidence?.ok ? 1 : 0;
+    if (ao !== bo)
+        return bo - ao;
+    const as = a.evidence?.score ?? 0;
+    const bs = b.evidence?.score ?? 0;
+    if (as !== bs)
+        return bs - as;
+    return (b.confidence ?? 0) - (a.confidence ?? 0);
+}
+function evidenceWhy(c) {
+    const e = c.evidence;
+    if (!e)
+        return 'no evidence gathered';
+    const passed = e.checks.filter((k) => k.ok).length;
+    const failed = e.checks.find((k) => !k.ok);
+    return e.ok
+        ? `evidence passed${e.checks.length ? ` (${passed}/${e.checks.length} checks)` : ''}`
+        : `evidence failed${e.checks.length ? ` (${passed}/${e.checks.length} checks${failed?.why ? `; ${failed.why}` : ''})` : ''}`;
+}
+/** Ask a pinned judge model to pick the best candidate. Returns undefined if it errors or votes for an unknown label. */
+async function runJudge(usable, task, input) {
+    const judge = input.judge;
+    const listing = usable.map((c) => `[${c.label}]\n${truncate(comparableText(c), 1200)}`).join('\n\n');
+    const prompt = [
+        `Compare the following candidate answers to this goal and pick the single best one by correctness and completeness.`,
+        `Goal: ${input.goal}`,
+        ``,
+        `Candidates:`,
+        listing,
+        ``,
+        `Respond ONLY as JSON: {"winner": "<one of the candidate labels above>", "reason": "<one sentence>"}.`,
+    ].join('\n');
+    const req = {
+        task,
+        input: { text: prompt },
+        output: { format: 'json' },
+        ...(input.routing ? { routing: input.routing } : {}),
+        ...(judge.provider ? { provider: judge.provider } : {}),
+        ...(judge.model ? { model: judge.model } : {}),
+        ...(input.policy?.maxCostUsd !== undefined ? { budget: { maxCostUsd: input.policy.maxCostUsd } } : {}),
+    };
+    let run;
+    try {
+        run = await input.ai.run(req);
+    }
+    catch {
+        return undefined;
+    }
+    // The judge output is untrusted DATA: it may only NAME a candidate label. A non-object (null, array,
+    // primitive) or malformed JSON is discarded, never dereferenced.
+    const j = run.response?.json;
+    if (!run.ok || typeof j !== 'object' || j === null || Array.isArray(j))
+        return undefined;
+    const parsed = j;
+    const winnerRaw = typeof parsed.winner === 'string' ? parsed.winner.trim() : '';
+    // The judge must name a real candidate; match case-insensitively, else discard the verdict.
+    const match = usable.find((c) => c.label.toLowerCase() === winnerRaw.toLowerCase());
+    if (!match)
+        return undefined;
+    return { winner: match.label, reason: typeof parsed.reason === 'string' ? parsed.reason : 'no reason given' };
+}
+/**
+ * One learning signal per fanned-out candidate (Phase 9 consumes these). `preferred` is set ONLY when
+ * EVIDENCE established the winner — never a judge or confidence — so preference learning is evidence-gated.
+ */
+function buildSignals(candidates, task, ranking) {
+    const out = [];
+    for (const c of candidates) {
+        // Attribution prefers what actually ran (selected), then what the router attempted (so a FAILED
+        // model-less pin still yields a negative signal), then the pin itself.
+        const providerId = c.selected?.providerId ?? c.attempted?.providerId ?? c.pin.provider;
+        const model = c.selected?.model ?? c.attempted?.model ?? c.pin.model;
+        if (!providerId || !model)
+            continue; // cannot attribute a signal without a concrete provider+model
+        const preferred = ranking.method === 'evidence' && ranking.winner === c.label && c.evidence?.ok === true;
+        out.push({
+            providerId,
+            model,
+            task,
+            ran: true,
+            ok: c.ok,
+            ...(c.ok && c.confidence !== undefined ? { confidence: c.confidence } : {}),
+            ...(c.latencyMs !== undefined ? { latencyMs: c.latencyMs } : {}),
+            ...(c.evidence ? { evidenceOk: c.evidence.ok, evidenceScore: c.evidence.score } : {}),
+            preferred,
+        });
+    }
+    return out;
+}
+function truncate(s, n) {
+    return s.length > n ? s.slice(0, n) + '…' : s;
+}
+function round(n) {
+    return Math.round(n * 100) / 100;
+}

package/dist/comparison/comparison.d.ts

@@ -0,0 +1,140 @@
+/**
+ * Comparison types (Phase 8). A comparison fans out the SAME task across several **pins** (candidate
+ * provider/model pairs), collects the outputs, and analyzes them: agreement, differences,
+ * contradictions, missing information. Ranking is EVIDENCE-first — a caller-supplied validation of each
+ * output (e.g. "run the tests") beats any judge model's opinion; a judge is a labeled fallback, and with
+ * neither we report only (no verdict). Pure types — no router/AI import, so `runtime/types.ts` can widen
+ * its `comparison` placeholder to `ComparisonResult` without an import cycle.
+ */
+import type { RouterErrorInfo } from '../types.js';
+/** A candidate to compare: a pinned provider (and optionally model). Passes through all router filters. */
+export interface ComparisonPin {
+    /** Provider id to pin. Omit to let the router choose (rarely useful for a comparison). */
+    provider?: string;
+    /** Model id to pin (optional; the provider's default model when omitted). */
+    model?: string;
+    /** Display label; defaults to `provider/model` (or `provider`, or `candidate-N`). */
+    label?: string;
+}
+/** Objective, evidence-based validation of a single candidate's output. Beats judge opinion. */
+export interface EvidenceResult {
+    ok: boolean;
+    /** Fraction of checks that passed, 0..1 (1/0 when there are no discrete checks). */
+    score: number;
+    checks: Array<{
+        name: string;
+        ok: boolean;
+        why?: string;
+    }>;
+}
+/** One fanned-out result: what a single pin produced, plus any evidence gathered for it. */
+export interface ComparisonCandidate {
+    label: string;
+    pin: ComparisonPin;
+    /** Whether the underlying `AI.run()` for this pin succeeded. */
+    ok: boolean;
+    text?: string;
+    json?: unknown;
+    /** What the router actually selected under this pin (may differ when the model was left open). */
+    selected?: {
+        providerId: string;
+        model: string;
+    };
+    /** The provider/model the router last ATTEMPTED — for attribution when a run failed with nothing selected. */
+    attempted?: {
+        providerId: string;
+        model: string;
+    };
+    /** Router-reported confidence for this candidate's run (an estimate, never a quality verdict alone). */
+    confidence?: number;
+    /** Total latency across the run's attempts (primary + any fallbacks/retries). */
+    latencyMs?: number;
+    error?: RouterErrorInfo;
+    /** Present only when a `validate()` evidence function ran for this candidate. */
+    evidence?: EvidenceResult;
+}
+/** How the ranking was decided. `evidence` is authoritative; `judge` is opinion; `none` is report-only. */
+export type ComparisonMethod = 'evidence' | 'judge' | 'none';
+/** A set of candidates whose outputs agree (by normalized token similarity). */
+export interface AgreementCluster {
+    labels: string[];
+    representative: string;
+}
+export interface ComparisonDifference {
+    a: string;
+    b: string;
+    /** Token-set Jaccard similarity of the two outputs, 0..1. */
+    similarity: number;
+}
+/** Content one candidate surfaced that the others did not (missing-information). */
+export interface MissingPoint {
+    label: string;
+    points: string[];
+}
+/** A direct contradiction: the same structured field carrying different values across candidates. */
+export interface Contradiction {
+    field: string;
+    values: Array<{
+        label: string;
+        value: string;
+    }>;
+}
+export interface ComparisonAnalysis {
+    clusters: AgreementCluster[];
+    /** True when every successful candidate agreed (a single cluster). */
+    unanimous: boolean;
+    differences: ComparisonDifference[];
+    missing: MissingPoint[];
+    contradictions: Contradiction[];
+}
+export interface RankEntry {
+    label: string;
+    score: number;
+    why: string;
+}
+export interface ComparisonRanking {
+    method: ComparisonMethod;
+    /** True when the verdict is a judge model's OPINION, not objective evidence. */
+    judgeOpinion: boolean;
+    ranked: RankEntry[];
+    /** Set only for an evidence-grounded (or judge) winner; absent for report-only comparisons. */
+    winner?: string;
+}
+/**
+ * A learning signal emitted per fanned-out candidate for the Phase 9 learning stores to consume.
+ * `preferred` is set ONLY when EVIDENCE (never a judge or router confidence) established the winner —
+ * so preference learning can never be driven by opinion. Failed runs are emitted too (`ran:true`,
+ * `ok:false`) as negative signal.
+ */
+export interface ComparisonSignal {
+    providerId: string;
+    model: string;
+    task: string;
+    ran: boolean;
+    ok: boolean;
+    confidence?: number;
+    latencyMs?: number;
+    evidenceOk?: boolean;
+    evidenceScore?: number;
+    preferred: boolean;
+}
+export interface ComparisonUsage {
+    /** Pins the caller asked to compare. */
+    requested: number;
+    /** Pins actually run (bounded by the call budget). */
+    ran: number;
+    /** Pins that did not run — budget-skipped and/or dropped by an abort (= requested − ran). */
+    skipped: number;
+    /** True when the fan-out was aborted before every intended pin ran (the result is partial). */
+    aborted?: boolean;
+    note?: string;
+}
+export interface ComparisonResult {
+    goal: string;
+    task: string;
+    candidates: ComparisonCandidate[];
+    analysis: ComparisonAnalysis;
+    ranking: ComparisonRanking;
+    usage: ComparisonUsage;
+    signals: ComparisonSignal[];
+}

package/dist/comparison/comparison.js

@@ -0,0 +1,9 @@
+/**
+ * Comparison types (Phase 8). A comparison fans out the SAME task across several **pins** (candidate
+ * provider/model pairs), collects the outputs, and analyzes them: agreement, differences,
+ * contradictions, missing information. Ranking is EVIDENCE-first — a caller-supplied validation of each
+ * output (e.g. "run the tests") beats any judge model's opinion; a judge is a labeled fallback, and with
+ * neither we report only (no verdict). Pure types — no router/AI import, so `runtime/types.ts` can widen
+ * its `comparison` placeholder to `ComparisonResult` without an import cycle.
+ */
+export {};

package/dist/comparison/render.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Human-readable rendering of a ComparisonResult for the terminal (REPL + one-shot). Pure: turns the
+ * structured result into lines. It never claims a verdict the ranking did not make — a judge verdict is
+ * labeled as opinion, and a report-only comparison shows no winner.
+ */
+import type { ComparisonResult } from './comparison.js';
+export declare function renderComparison(result: ComparisonResult): string[];