@haystackeditor/cli 0.15.21 → 0.15.22

package/dist/commands/verify-mcp.js

@@ -99,6 +99,19 @@ export const VERIFY_TOOLS = [
             required: ['cell'],
         },
     },
+    {
+        name: 'verify_universe',
+        description: 'Get or open a live universe for one test: returns the running environment when one is still live, otherwise reopens the retained state onto fresh compute (reusing an existing attachment when present).',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                run: runInput,
+                cell: { type: 'string', description: 'Stable test selector: ordinal (1-based), riskCellId, or exact planner cellKey.' },
+                side: { type: 'string', enum: ['base', 'head'], description: 'Matched universe side (default head).' },
+            },
+            required: ['cell'],
+        },
+    },
     {
         name: 'verify_cleanup',
         description: 'Read retained-state cleanup for one test universe by its exact fork ID: shared references, provider deletion attempts, and the durable cleanup-SLO verdict.',
@@ -292,6 +305,52 @@ export async function runVerifyMcpServer(options) {
                     return ok(match.cell);
                 return ok(cellDetail(loaded.run, match));
             }
+            if (name === 'verify_universe') {
+                if (typeof input.cell !== 'string')
+                    return fail('cell required');
+                if (input.side !== undefined
+                    && input.side !== 'base'
+                    && input.side !== 'head') {
+                    return fail('side must be base or head');
+                }
+                const loaded = await loadRunAuto({ server, repoRoot, selector });
+                const match = findCellByStableSelector(loaded.run, input.cell);
+                const side = input.side === 'base' ? 'base' : 'head';
+                const universe = side === 'base' ? match.cell.base : match.cell.head;
+                const expired = universe.expiresAt
+                    ? Date.parse(universe.expiresAt) < Date.now()
+                    : false;
+                if (universe.browserUrl && universe.status === 'running' && !expired) {
+                    return ok({
+                        schema_version: '1.0.0',
+                        run_id: loaded.run.runId,
+                        ordinal: match.cell.ordinal,
+                        side,
+                        source: 'running',
+                        url: universe.browserUrl,
+                        expires_at: universe.expiresAt ?? null,
+                    });
+                }
+                if (!universe.forkId) {
+                    return fail(`Test ${match.cell.ordinal}'s ${side} universe is not running and `
+                        + 'has no durable fork ID to reopen.');
+                }
+                const { requestUniverseReopen } = await import('./verify-ops.js');
+                const reopened = await requestUniverseReopen(server, universe.forkId);
+                return ok({
+                    schema_version: '1.0.0',
+                    run_id: loaded.run.runId,
+                    ordinal: match.cell.ordinal,
+                    side,
+                    source: 'reopened',
+                    url: reopened.browserUrl,
+                    expires_at: reopened.expiresAt,
+                    fork_id: reopened.forkId,
+                    sandbox_id: reopened.sandboxId,
+                    state_expires_at: reopened.stateExpiresAt,
+                    reused: reopened.reused,
+                });
+            }
             if (name === 'verify_cleanup') {
                 if (typeof input.cell !== 'string')
                     return fail('cell required');

package/dist/commands/verify-ops.js

@@ -26,7 +26,7 @@ function renderStatus(summary) {
         : summary.status === 'complete' ? chalk.green(summary.status)
             : summary.status === 'failed' ? chalk.red(summary.status)
                 : chalk.cyan(summary.status);
-    console.log(`${chalk.bold(summary.run_id)} — ${state}${summary.age ? chalk.dim(` (updated ${summary.age} ago)`) : ''}`);
+    console.log(`${chalk.bold(summary.run_id)} — ${state}${summary.age ? chalk.dim(` (started ${summary.age} ago)`) : ''}`);
     for (const chapter of summary.chapters) {
         console.log(`${chalk.dim(chapter.repository)} ${chapter.pull_request} — ${chapter.label}`);
         if (chapter.execution_mode === 'bisect' && chapter.bisect) {

package/dist/commands/verify-reproducibility.d.ts

@@ -0,0 +1,85 @@
+import { type RiskCell, type VerificationRun } from "./verify-core.js";
+import { type VerifyOptions } from "./verify.js";
+export type ReproducibilityScope = "executor" | "planner" | "full";
+export interface CellOutcomeObservation {
+    run_id: string;
+    status: string;
+    verdict: string | null;
+}
+export interface CellPlanObservation {
+    run_id: string;
+    fingerprint: string;
+}
+export interface ReproducibilityCell {
+    cell_id: string;
+    chapter_id: string;
+    title: string;
+    outcomes: CellOutcomeObservation[];
+    plans: CellPlanObservation[];
+    missing_run_ids: string[];
+}
+export interface BisectOutcomeObservation {
+    run_id: string;
+    status: string;
+    culprit_sha: string | null;
+    parent_sha: string | null;
+    incident: {
+        commit: string;
+        status: string;
+    } | null;
+    probes: Array<{
+        commit: string;
+        status: string;
+    }>;
+}
+export interface ReproducibilityBisect {
+    chapter_id: string;
+    label: string;
+    outcomes: BisectOutcomeObservation[];
+    missing_run_ids: string[];
+}
+export interface ReproducibilityReport {
+    schema_version: "1.0.0";
+    scope: ReproducibilityScope;
+    requested_runs: number;
+    completed_runs: number;
+    run_ids: string[];
+    plan_source_run_id: string | null;
+    compared_cells: number;
+    stable_cells: number;
+    compared_bisects: number;
+    stable_bisects: number;
+    plan_drift_cells: ReproducibilityCell[];
+    outcome_drift_cells: ReproducibilityCell[];
+    unrunnable_cells: ReproducibilityCell[];
+    bisect_drift_chapters: ReproducibilityBisect[];
+    unrunnable_bisects: ReproducibilityBisect[];
+    planner_incomplete_run_ids: string[];
+    executor_incomplete_run_ids: string[];
+    incomplete_run_ids: string[];
+    timed_out_run_ids: string[];
+    planner_reproducible: boolean;
+    executor_reproducible: boolean;
+    reproducible: boolean;
+}
+export declare function cellPlanFingerprint(cell: RiskCell): string;
+/**
+ * Plan identity and runtime outcomes are independent axes:
+ *
+ * - missing IDs or changed frozen contracts are planner drift;
+ * - status/judgment changes under an identical contract are executor drift.
+ *
+ * Titles and other prose are fingerprinted to detect a changed judge input,
+ * but cells are correlated only by their exact stable IDs.
+ */
+export declare function analyzeReproducibility(runs: VerificationRun[], requestedRuns?: number, timedOutRunIds?: string[], scope?: ReproducibilityScope): ReproducibilityReport;
+export interface VerifyReproducibilityOptions extends VerifyOptions {
+    runs?: string;
+    interval?: string;
+    timeout?: string;
+    repoRoot?: string;
+    scope?: string;
+    planSource?: string;
+}
+export declare function runHasFrozenRiskPlan(run: VerificationRun): boolean;
+export declare function verifyReproducibilityCommand(specimen: string, options: VerifyReproducibilityOptions): Promise<void>;

package/dist/commands/verify-reproducibility.js

@@ -0,0 +1,494 @@
+import { createHash } from "node:crypto";
+import chalk from "chalk";
+import { discoverVerifyRoot, resolveServer, summarizeRun, waitForTerminal, } from "./verify-core.js";
+import { startVerificationRun, } from "./verify.js";
+function compareCodeUnits(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function requiredPlanString(value, label) {
+    if (typeof value !== "string" || value.trim() === "") {
+        throw new Error(`Frozen plan ${label} must be a non-empty string`);
+    }
+    return value;
+}
+function requiredPlanStrings(value, label) {
+    if (!Array.isArray(value)) {
+        throw new Error(`Frozen plan ${label} must be an array`);
+    }
+    return value.map((item, index) => requiredPlanString(item, `${label}[${index}]`));
+}
+function canonicalCellPlan(cell) {
+    if (!Array.isArray(cell.sourceEvidence)) {
+        throw new Error("Frozen plan sourceEvidence must be an array");
+    }
+    if (!Array.isArray(cell.oracles)) {
+        throw new Error("Frozen plan oracles must be an array");
+    }
+    if (typeof cell.confidence !== "number" || !Number.isFinite(cell.confidence)) {
+        throw new Error("Frozen plan confidence must be a finite number");
+    }
+    return {
+        risk_source_id: requiredPlanString(cell.riskSourceId, "riskSourceId"),
+        capability_id: requiredPlanString(cell.capabilityId, "capabilityId"),
+        title: requiredPlanString(cell.title, "title"),
+        rationale: requiredPlanString(cell.rationale, "rationale"),
+        observable_consequence: requiredPlanString(cell.observableConsequence, "observableConsequence"),
+        activation_conditions: requiredPlanStrings(cell.activationConditions, "activationConditions").sort(compareCodeUnits),
+        causal_path: requiredPlanStrings(cell.causalPath, "causalPath"),
+        severity: requiredPlanString(cell.severity, "severity"),
+        confidence: cell.confidence,
+        source_evidence: cell.sourceEvidence.map((item, index) => ({
+            evidence_id: requiredPlanString(item.evidenceId, `sourceEvidence[${index}].evidenceId`),
+            file: requiredPlanString(item.file, `sourceEvidence[${index}].file`),
+            line: item.line ?? null,
+            excerpt: requiredPlanString(item.excerpt, `sourceEvidence[${index}].excerpt`),
+            relationship: requiredPlanString(item.relationship, `sourceEvidence[${index}].relationship`),
+        })).sort((left, right) => compareCodeUnits(JSON.stringify(left), JSON.stringify(right))),
+        oracles: cell.oracles.map((oracle, index) => ({
+            oracle_id: requiredPlanString(oracle.oracleId, `oracles[${index}].oracleId`),
+            kind: requiredPlanString(oracle.kind, `oracles[${index}].kind`),
+            measurement: requiredPlanString(oracle.measurement, `oracles[${index}].measurement`),
+            threshold: oracle.threshold ?? null,
+            description: requiredPlanString(oracle.description, `oracles[${index}].description`),
+        })).sort((left, right) => compareCodeUnits(JSON.stringify(left), JSON.stringify(right))),
+    };
+}
+function bisectOutcomeSignature(observation) {
+    if (!observation)
+        return "missing";
+    return JSON.stringify({
+        status: observation.status,
+        culprit_sha: observation.culprit_sha,
+        parent_sha: observation.parent_sha,
+        incident: observation.incident,
+        probes: observation.probes,
+    });
+}
+function canonicalBisectObservation(runId, chapter) {
+    const bisect = chapter.bisect;
+    if (!bisect || !Array.isArray(bisect.probes))
+        return null;
+    return {
+        run_id: runId,
+        status: bisect.status,
+        culprit_sha: bisect.culpritSha ?? null,
+        parent_sha: bisect.parentSha ?? null,
+        incident: bisect.incidentProbe
+            ? {
+                commit: bisect.incidentProbe.commit,
+                status: bisect.incidentProbe.status,
+            }
+            : null,
+        probes: bisect.probes
+            .map(probe => ({
+            commit: probe.commit,
+            status: probe.status,
+        }))
+            .sort((left, right) => compareCodeUnits(left.commit, right.commit)),
+    };
+}
+function reportBisectOf(bisect, runIds) {
+    return {
+        chapter_id: bisect.chapter_id,
+        label: bisect.label,
+        outcomes: runIds.flatMap((runId) => {
+            const observation = bisect.byRun.get(runId);
+            return observation ? [observation] : [];
+        }),
+        missing_run_ids: runIds.filter((runId) => !bisect.byRun.has(runId)),
+    };
+}
+export function cellPlanFingerprint(cell) {
+    return createHash("sha256")
+        .update(JSON.stringify(canonicalCellPlan(cell)))
+        .digest("hex");
+}
+function outcomeSignature(observation) {
+    if (!observation)
+        return "missing";
+    return JSON.stringify({
+        status: observation.status,
+        verdict: observation.verdict,
+    });
+}
+function reportCellOf(cell, runIds) {
+    return {
+        cell_id: cell.cell_id,
+        chapter_id: cell.chapter_id,
+        title: cell.title,
+        outcomes: runIds.flatMap((runId) => {
+            const observation = cell.byRun.get(runId);
+            return observation ? [observation] : [];
+        }),
+        plans: runIds.flatMap((runId) => {
+            const observation = cell.planByRun.get(runId);
+            return observation ? [observation] : [];
+        }),
+        missing_run_ids: runIds.filter((runId) => !cell.byRun.has(runId)),
+    };
+}
+/**
+ * Plan identity and runtime outcomes are independent axes:
+ *
+ * - missing IDs or changed frozen contracts are planner drift;
+ * - status/judgment changes under an identical contract are executor drift.
+ *
+ * Titles and other prose are fingerprinted to detect a changed judge input,
+ * but cells are correlated only by their exact stable IDs.
+ */
+export function analyzeReproducibility(runs, requestedRuns = runs.length, timedOutRunIds = [], scope = "full") {
+    const runIds = runs.map((run) => run.runId);
+    const cells = new Map();
+    const bisects = new Map();
+    const plannerIncompleteRunIds = new Set(timedOutRunIds);
+    const executorIncompleteRunIds = new Set(timedOutRunIds);
+    for (const run of runs) {
+        if (!["complete", "failed"].includes(run.status)
+            || run.events.some(event => event.eventId.startsWith("evt-stale-"))) {
+            plannerIncompleteRunIds.add(run.runId);
+            executorIncompleteRunIds.add(run.runId);
+        }
+        const seenInRun = new Set();
+        const seenBisectsInRun = new Set();
+        for (const chapter of run.chapters) {
+            if (chapter.executionMode === "bisect") {
+                if (seenBisectsInRun.has(chapter.chapterId)) {
+                    throw new Error(`Run ${run.runId} contains duplicate stable bisect chapter id ${chapter.chapterId}`);
+                }
+                seenBisectsInRun.add(chapter.chapterId);
+                const observation = canonicalBisectObservation(run.runId, chapter);
+                if (!observation) {
+                    executorIncompleteRunIds.add(run.runId);
+                    continue;
+                }
+                const indexed = bisects.get(chapter.chapterId) ?? {
+                    chapter_id: chapter.chapterId,
+                    label: chapter.label,
+                    byRun: new Map(),
+                };
+                indexed.byRun.set(run.runId, observation);
+                bisects.set(chapter.chapterId, indexed);
+                if (observation.status !== "found") {
+                    executorIncompleteRunIds.add(run.runId);
+                }
+                continue;
+            }
+            if (!Array.isArray(chapter.riskCells)) {
+                plannerIncompleteRunIds.add(run.runId);
+                executorIncompleteRunIds.add(run.runId);
+                continue;
+            }
+            for (const cell of chapter.riskCells) {
+                const stableKey = `${chapter.chapterId}:${cell.riskCellId}`;
+                if (seenInRun.has(stableKey)) {
+                    throw new Error(`Run ${run.runId} contains duplicate stable cell id ${stableKey}`);
+                }
+                seenInRun.add(stableKey);
+                const indexed = cells.get(stableKey) ?? {
+                    cell_id: cell.riskCellId,
+                    chapter_id: chapter.chapterId,
+                    title: cell.title,
+                    byRun: new Map(),
+                    planByRun: new Map(),
+                };
+                indexed.byRun.set(run.runId, {
+                    run_id: run.runId,
+                    status: cell.status,
+                    verdict: cell.agentJudgment?.verdict ?? null,
+                });
+                indexed.planByRun.set(run.runId, {
+                    run_id: run.runId,
+                    fingerprint: cellPlanFingerprint(cell),
+                });
+                cells.set(stableKey, indexed);
+                if (cell.status === "infra_error"
+                    || cell.status === "inconclusive"
+                    || !["passed", "failed"].includes(cell.status)) {
+                    executorIncompleteRunIds.add(run.runId);
+                }
+            }
+        }
+    }
+    const planDriftCells = [];
+    const outcomeDriftCells = [];
+    const unrunnableCells = [];
+    const bisectDriftChapters = [];
+    const unrunnableBisects = [];
+    let stableCells = 0;
+    let stableBisects = 0;
+    for (const cell of [...cells.values()].sort((left, right) => compareCodeUnits(`${left.chapter_id}:${left.cell_id}`, `${right.chapter_id}:${right.cell_id}`))) {
+        const reportCell = reportCellOf(cell, runIds);
+        const planSignatures = new Set(runIds.map((runId) => cell.planByRun.get(runId)?.fingerprint ?? "missing"));
+        const planDrift = planSignatures.size > 1;
+        if (planDrift)
+            planDriftCells.push(reportCell);
+        const outcomeSignatures = new Set(runIds.map((runId) => outcomeSignature(cell.byRun.get(runId))));
+        const outcomeDrift = !planDrift && outcomeSignatures.size > 1;
+        if (outcomeDrift)
+            outcomeDriftCells.push(reportCell);
+        const outcomes = [...cell.byRun.values()];
+        const unrunnable = outcomes.some((outcome) => !["passed", "failed"].includes(outcome.status));
+        if (unrunnable)
+            unrunnableCells.push(reportCell);
+        if (!planDrift
+            && !outcomeDrift
+            && !unrunnable
+            && reportCell.missing_run_ids.length === 0) {
+            stableCells += 1;
+        }
+    }
+    for (const bisect of [...bisects.values()].sort((left, right) => compareCodeUnits(left.chapter_id, right.chapter_id))) {
+        const reportBisect = reportBisectOf(bisect, runIds);
+        const outcomeSignatures = new Set(runIds.map((runId) => bisectOutcomeSignature(bisect.byRun.get(runId))));
+        const outcomeDrift = outcomeSignatures.size > 1;
+        if (outcomeDrift)
+            bisectDriftChapters.push(reportBisect);
+        const unrunnable = [...bisect.byRun.values()].some(observation => observation.status !== "found");
+        if (unrunnable)
+            unrunnableBisects.push(reportBisect);
+        if (!outcomeDrift
+            && !unrunnable
+            && reportBisect.missing_run_ids.length === 0) {
+            stableBisects += 1;
+        }
+    }
+    const plannerHasComparableWork = cells.size > 0;
+    const executorHasComparableWork = cells.size > 0 || bisects.size > 0;
+    const plannerIncomplete = plannerIncompleteRunIds.size > 0
+        || runs.length < requestedRuns
+        || !plannerHasComparableWork;
+    const executorIncomplete = executorIncompleteRunIds.size > 0
+        || runs.length < requestedRuns
+        || !executorHasComparableWork;
+    const plannerReproducible = !plannerIncomplete
+        && planDriftCells.length === 0;
+    const executorReproducible = !executorIncomplete
+        && planDriftCells.length === 0
+        && outcomeDriftCells.length === 0
+        && unrunnableCells.length === 0
+        && bisectDriftChapters.length === 0
+        && unrunnableBisects.length === 0;
+    const reproducible = scope === "planner"
+        ? plannerReproducible
+        : scope === "executor"
+            ? executorReproducible
+            : plannerReproducible && executorReproducible;
+    const incompleteRunIds = scope === "planner"
+        ? plannerIncompleteRunIds
+        : executorIncompleteRunIds;
+    return {
+        schema_version: "1.0.0",
+        scope,
+        requested_runs: requestedRuns,
+        completed_runs: runs.filter((run) => ["complete", "failed"].includes(run.status)).length,
+        run_ids: runIds,
+        plan_source_run_id: scope === "executor" && cells.size > 0
+            ? runs[0]?.runId ?? null
+            : null,
+        compared_cells: cells.size,
+        stable_cells: stableCells,
+        compared_bisects: bisects.size,
+        stable_bisects: stableBisects,
+        plan_drift_cells: planDriftCells,
+        outcome_drift_cells: outcomeDriftCells,
+        unrunnable_cells: unrunnableCells,
+        bisect_drift_chapters: bisectDriftChapters,
+        unrunnable_bisects: unrunnableBisects,
+        planner_incomplete_run_ids: [...plannerIncompleteRunIds],
+        executor_incomplete_run_ids: [...executorIncompleteRunIds],
+        incomplete_run_ids: [...incompleteRunIds],
+        timed_out_run_ids: timedOutRunIds,
+        planner_reproducible: plannerReproducible,
+        executor_reproducible: executorReproducible,
+        reproducible,
+    };
+}
+function positiveNumber(raw, fallback, flag) {
+    const value = raw === undefined ? fallback : Number(raw);
+    if (!Number.isFinite(value) || value <= 0) {
+        throw new Error(`${flag} takes a positive number`);
+    }
+    return value;
+}
+function parseScope(raw) {
+    const scope = raw ?? "executor";
+    if (scope === "executor" || scope === "planner" || scope === "full") {
+        return scope;
+    }
+    throw new Error("--scope must be executor, planner, or full");
+}
+function renderCell(cell, kind) {
+    const glyph = kind === "unrunnable" ? chalk.yellow("!") : chalk.red("✗");
+    console.log(`  ${glyph} ${cell.chapter_id}/${cell.cell_id} ${chalk.dim(cell.title)}`);
+    if (kind === "plan") {
+        for (const plan of cell.plans) {
+            console.log(`      ${plan.run_id}: plan ${plan.fingerprint.slice(0, 12)}`);
+        }
+    }
+    else {
+        for (const outcome of cell.outcomes) {
+            console.log(`      ${outcome.run_id}: ${outcome.status}`
+                + `${outcome.verdict ? ` / ${outcome.verdict}` : ""}`);
+        }
+    }
+    for (const runId of cell.missing_run_ids) {
+        console.log(`      ${runId}: ${chalk.yellow("missing from frozen plan")}`);
+    }
+}
+function renderBisect(bisect, kind) {
+    const glyph = kind === "unrunnable" ? chalk.yellow("!") : chalk.red("✗");
+    console.log(`  ${glyph} ${bisect.chapter_id} ${chalk.dim(bisect.label)}`);
+    for (const outcome of bisect.outcomes) {
+        console.log(`      ${outcome.run_id}: ${outcome.status}`
+            + `${outcome.culprit_sha
+                ? ` / ${outcome.parent_sha ?? "?"}..${outcome.culprit_sha}`
+                : ""}`);
+    }
+    for (const runId of bisect.missing_run_ids) {
+        console.log(`      ${runId}: ${chalk.yellow("missing bisect result")}`);
+    }
+}
+export function runHasFrozenRiskPlan(run) {
+    return run.chapters.some(chapter => chapter.executionMode !== "bisect"
+        && (chapter.riskCells?.length ?? 0) > 0);
+}
+export async function verifyReproducibilityCommand(specimen, options) {
+    const requestedRuns = positiveNumber(options.runs, 3, "--runs");
+    if (!Number.isInteger(requestedRuns) || requestedRuns < 2 || requestedRuns > 20) {
+        throw new Error("--runs must be an integer from 2 to 20");
+    }
+    const scope = parseScope(options.scope);
+    if (options.planSource && scope !== "executor") {
+        throw new Error("--plan-source is only valid with --scope executor");
+    }
+    const intervalMs = positiveNumber(options.interval, 3, "--interval") * 1_000;
+    const timeoutMs = options.timeout
+        ? positiveNumber(options.timeout, 0, "--timeout") * 60_000
+        : undefined;
+    const server = resolveServer(options.server);
+    const repoRoot = discoverVerifyRoot(options.repoRoot);
+    const runs = [];
+    const timedOutRunIds = [];
+    let executorPlanSourceRunId = options.planSource;
+    for (let index = 0; index < requestedRuns; index += 1) {
+        const planSourceRunId = scope === "executor"
+            ? executorPlanSourceRunId
+            : undefined;
+        if (!options.json) {
+            console.error(chalk.dim(`Reproducibility run ${index + 1}/${requestedRuns}: `
+                + `${planSourceRunId
+                    ? `replaying plan from ${planSourceRunId}`
+                    : `planning ${specimen}`}…`));
+        }
+        const started = await startVerificationRun(specimen, {
+            ...options,
+            json: true,
+            open: false,
+            planSourceRunId,
+        });
+        const waited = await waitForTerminal({
+            server,
+            repoRoot,
+            selector: started.runId,
+            intervalMs,
+            timeoutMs,
+        });
+        runs.push(waited.loaded.run);
+        if (scope === "executor"
+            && !executorPlanSourceRunId
+            && runHasFrozenRiskPlan(waited.loaded.run)) {
+            executorPlanSourceRunId = waited.loaded.run.runId;
+        }
+        if (!options.json) {
+            const summary = summarizeRun(waited.loaded, server);
+            console.error(`  ${started.runId}: ${summary.counts.passed} passed, `
+                + `${summary.counts.failed} failed, `
+                + `${summary.counts.could_not_run} could not run`);
+        }
+        if (waited.timedOut) {
+            timedOutRunIds.push(started.runId);
+            break;
+        }
+        if (waited.endReason === "stale")
+            break;
+    }
+    const report = analyzeReproducibility(runs, requestedRuns, timedOutRunIds, scope);
+    if (scope === "executor") {
+        report.plan_source_run_id = executorPlanSourceRunId ?? null;
+    }
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+    }
+    else {
+        const scopeHasComparableWork = scope === "planner"
+            ? report.compared_cells > 0
+            : report.compared_cells > 0 || report.compared_bisects > 0;
+        console.log("");
+        console.log(chalk.bold(`Compared ${report.compared_cells} stable cell ID`
+            + `${report.compared_cells === 1 ? "" : "s"} and `
+            + `${report.compared_bisects} stable bisect chapter ID`
+            + `${report.compared_bisects === 1 ? "" : "s"} across `
+            + `${report.completed_runs}/${report.requested_runs} runs (${scope}).`));
+        if (report.plan_drift_cells.length > 0) {
+            console.log(chalk.red.bold(`${report.plan_drift_cells.length} planner-drift cell`
+                + `${report.plan_drift_cells.length === 1 ? "" : "s"}:`));
+            for (const cell of report.plan_drift_cells)
+                renderCell(cell, "plan");
+        }
+        else if (report.compared_cells > 0) {
+            console.log(chalk.green("✓ Frozen plan contracts were identical."));
+        }
+        if (scope !== "planner") {
+            if (report.outcome_drift_cells.length > 0) {
+                console.log(chalk.red.bold(`${report.outcome_drift_cells.length} runtime/judgment-drift cell`
+                    + `${report.outcome_drift_cells.length === 1 ? "" : "s"}:`));
+                for (const cell of report.outcome_drift_cells) {
+                    renderCell(cell, "outcome");
+                }
+            }
+            else if (report.executor_reproducible && report.compared_cells > 0) {
+                console.log(chalk.green(`✓ All ${report.stable_cells} cells produced the same status and judgment.`));
+            }
+            if (report.bisect_drift_chapters.length > 0) {
+                console.log(chalk.red.bold(`${report.bisect_drift_chapters.length} bisect chapter`
+                    + `${report.bisect_drift_chapters.length === 1 ? "" : "s"} changed outcomes:`));
+                for (const bisect of report.bisect_drift_chapters) {
+                    renderBisect(bisect, "drift");
+                }
+            }
+            else if (report.compared_bisects > 0) {
+                console.log(chalk.green(`✓ All ${report.stable_bisects} bisect chapters found the same exact boundary.`));
+            }
+        }
+        const driftIds = new Set([...report.plan_drift_cells, ...report.outcome_drift_cells].map(cell => `${cell.chapter_id}:${cell.cell_id}`));
+        const repeatablyUnrunnable = report.unrunnable_cells.filter(cell => !driftIds.has(`${cell.chapter_id}:${cell.cell_id}`));
+        if (scope !== "planner" && repeatablyUnrunnable.length > 0) {
+            console.log(chalk.yellow.bold(`${repeatablyUnrunnable.length} cell`
+                + `${repeatablyUnrunnable.length === 1 ? "" : "s"} never produced a verdict:`));
+            for (const cell of repeatablyUnrunnable) {
+                renderCell(cell, "unrunnable");
+            }
+        }
+        if (scope !== "planner" && report.unrunnable_bisects.length > 0) {
+            console.log(chalk.yellow.bold(`${report.unrunnable_bisects.length} bisect chapter`
+                + `${report.unrunnable_bisects.length === 1 ? "" : "s"} did not find a boundary:`));
+            for (const bisect of report.unrunnable_bisects) {
+                renderBisect(bisect, "unrunnable");
+            }
+        }
+        if (report.incomplete_run_ids.length > 0
+            || !scopeHasComparableWork) {
+            console.log(chalk.yellow("Reproducibility is inconclusive because at least one run or cell did not produce the required result."));
+        }
+    }
+    const scopeHasComparableWork = scope === "planner"
+        ? report.compared_cells > 0
+        : report.compared_cells > 0 || report.compared_bisects > 0;
+    process.exitCode = report.incomplete_run_ids.length > 0
+        || report.completed_runs < report.requested_runs
+        || !scopeHasComparableWork
+        ? 2
+        : report.reproducible
+            ? 0
+            : 1;
+}

package/dist/commands/verify.d.ts

@@ -1,3 +1,4 @@
+export declare const VERIFY_SPECIMENS: Set<string>;
 export interface VerifyOptions {
     server?: string;
     token?: string;
@@ -7,5 +8,15 @@ export interface VerifyOptions {
     intentTranscript?: string;
     json?: boolean;
     open?: boolean;
+    /** Internal reproducibility option: execute an already-frozen plan. */
+    planSourceRunId?: string;
 }
+export interface StartRunResponse {
+    runId: string;
+    status: string;
+    resultsUrl: string;
+}
+export declare function parseStartRunResponse(raw: string): StartRunResponse;
+export declare function verifierErrorReason(raw: string, contentType: string | null): string;
+export declare function startVerificationRun(specimenArg: string | undefined, options: VerifyOptions): Promise<StartRunResponse>;
 export declare function verifyCommand(specimenArg: string | undefined, options: VerifyOptions): Promise<void>;