@haystackeditor/cli 0.15.21 → 0.15.23

package/dist/commands/verify-reproducibility.js

@@ -0,0 +1,494 @@
+import { createHash } from "node:crypto";
+import chalk from "chalk";
+import { discoverVerifyRoot, resolveServer, summarizeRun, waitForTerminal, } from "./verify-core.js";
+import { startVerificationRun, } from "./verify.js";
+function compareCodeUnits(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function requiredPlanString(value, label) {
+    if (typeof value !== "string" || value.trim() === "") {
+        throw new Error(`Frozen plan ${label} must be a non-empty string`);
+    }
+    return value;
+}
+function requiredPlanStrings(value, label) {
+    if (!Array.isArray(value)) {
+        throw new Error(`Frozen plan ${label} must be an array`);
+    }
+    return value.map((item, index) => requiredPlanString(item, `${label}[${index}]`));
+}
+function canonicalCellPlan(cell) {
+    if (!Array.isArray(cell.sourceEvidence)) {
+        throw new Error("Frozen plan sourceEvidence must be an array");
+    }
+    if (!Array.isArray(cell.oracles)) {
+        throw new Error("Frozen plan oracles must be an array");
+    }
+    if (typeof cell.confidence !== "number" || !Number.isFinite(cell.confidence)) {
+        throw new Error("Frozen plan confidence must be a finite number");
+    }
+    return {
+        risk_source_id: requiredPlanString(cell.riskSourceId, "riskSourceId"),
+        capability_id: requiredPlanString(cell.capabilityId, "capabilityId"),
+        title: requiredPlanString(cell.title, "title"),
+        rationale: requiredPlanString(cell.rationale, "rationale"),
+        observable_consequence: requiredPlanString(cell.observableConsequence, "observableConsequence"),
+        activation_conditions: requiredPlanStrings(cell.activationConditions, "activationConditions").sort(compareCodeUnits),
+        causal_path: requiredPlanStrings(cell.causalPath, "causalPath"),
+        severity: requiredPlanString(cell.severity, "severity"),
+        confidence: cell.confidence,
+        source_evidence: cell.sourceEvidence.map((item, index) => ({
+            evidence_id: requiredPlanString(item.evidenceId, `sourceEvidence[${index}].evidenceId`),
+            file: requiredPlanString(item.file, `sourceEvidence[${index}].file`),
+            line: item.line ?? null,
+            excerpt: requiredPlanString(item.excerpt, `sourceEvidence[${index}].excerpt`),
+            relationship: requiredPlanString(item.relationship, `sourceEvidence[${index}].relationship`),
+        })).sort((left, right) => compareCodeUnits(JSON.stringify(left), JSON.stringify(right))),
+        oracles: cell.oracles.map((oracle, index) => ({
+            oracle_id: requiredPlanString(oracle.oracleId, `oracles[${index}].oracleId`),
+            kind: requiredPlanString(oracle.kind, `oracles[${index}].kind`),
+            measurement: requiredPlanString(oracle.measurement, `oracles[${index}].measurement`),
+            threshold: oracle.threshold ?? null,
+            description: requiredPlanString(oracle.description, `oracles[${index}].description`),
+        })).sort((left, right) => compareCodeUnits(JSON.stringify(left), JSON.stringify(right))),
+    };
+}
+function bisectOutcomeSignature(observation) {
+    if (!observation)
+        return "missing";
+    return JSON.stringify({
+        status: observation.status,
+        culprit_sha: observation.culprit_sha,
+        parent_sha: observation.parent_sha,
+        incident: observation.incident,
+        probes: observation.probes,
+    });
+}
+function canonicalBisectObservation(runId, chapter) {
+    const bisect = chapter.bisect;
+    if (!bisect || !Array.isArray(bisect.probes))
+        return null;
+    return {
+        run_id: runId,
+        status: bisect.status,
+        culprit_sha: bisect.culpritSha ?? null,
+        parent_sha: bisect.parentSha ?? null,
+        incident: bisect.incidentProbe
+            ? {
+                commit: bisect.incidentProbe.commit,
+                status: bisect.incidentProbe.status,
+            }
+            : null,
+        probes: bisect.probes
+            .map(probe => ({
+            commit: probe.commit,
+            status: probe.status,
+        }))
+            .sort((left, right) => compareCodeUnits(left.commit, right.commit)),
+    };
+}
+function reportBisectOf(bisect, runIds) {
+    return {
+        chapter_id: bisect.chapter_id,
+        label: bisect.label,
+        outcomes: runIds.flatMap((runId) => {
+            const observation = bisect.byRun.get(runId);
+            return observation ? [observation] : [];
+        }),
+        missing_run_ids: runIds.filter((runId) => !bisect.byRun.has(runId)),
+    };
+}
+export function cellPlanFingerprint(cell) {
+    return createHash("sha256")
+        .update(JSON.stringify(canonicalCellPlan(cell)))
+        .digest("hex");
+}
+function outcomeSignature(observation) {
+    if (!observation)
+        return "missing";
+    return JSON.stringify({
+        status: observation.status,
+        verdict: observation.verdict,
+    });
+}
+function reportCellOf(cell, runIds) {
+    return {
+        cell_id: cell.cell_id,
+        chapter_id: cell.chapter_id,
+        title: cell.title,
+        outcomes: runIds.flatMap((runId) => {
+            const observation = cell.byRun.get(runId);
+            return observation ? [observation] : [];
+        }),
+        plans: runIds.flatMap((runId) => {
+            const observation = cell.planByRun.get(runId);
+            return observation ? [observation] : [];
+        }),
+        missing_run_ids: runIds.filter((runId) => !cell.byRun.has(runId)),
+    };
+}
+/**
+ * Plan identity and runtime outcomes are independent axes:
+ *
+ * - missing IDs or changed frozen contracts are planner drift;
+ * - status/judgment changes under an identical contract are executor drift.
+ *
+ * Titles and other prose are fingerprinted to detect a changed judge input,
+ * but cells are correlated only by their exact stable IDs.
+ */
+export function analyzeReproducibility(runs, requestedRuns = runs.length, timedOutRunIds = [], scope = "full") {
+    const runIds = runs.map((run) => run.runId);
+    const cells = new Map();
+    const bisects = new Map();
+    const plannerIncompleteRunIds = new Set(timedOutRunIds);
+    const executorIncompleteRunIds = new Set(timedOutRunIds);
+    for (const run of runs) {
+        if (!["complete", "failed"].includes(run.status)
+            || run.events.some(event => event.eventId.startsWith("evt-stale-"))) {
+            plannerIncompleteRunIds.add(run.runId);
+            executorIncompleteRunIds.add(run.runId);
+        }
+        const seenInRun = new Set();
+        const seenBisectsInRun = new Set();
+        for (const chapter of run.chapters) {
+            if (chapter.executionMode === "bisect") {
+                if (seenBisectsInRun.has(chapter.chapterId)) {
+                    throw new Error(`Run ${run.runId} contains duplicate stable bisect chapter id ${chapter.chapterId}`);
+                }
+                seenBisectsInRun.add(chapter.chapterId);
+                const observation = canonicalBisectObservation(run.runId, chapter);
+                if (!observation) {
+                    executorIncompleteRunIds.add(run.runId);
+                    continue;
+                }
+                const indexed = bisects.get(chapter.chapterId) ?? {
+                    chapter_id: chapter.chapterId,
+                    label: chapter.label,
+                    byRun: new Map(),
+                };
+                indexed.byRun.set(run.runId, observation);
+                bisects.set(chapter.chapterId, indexed);
+                if (observation.status !== "found") {
+                    executorIncompleteRunIds.add(run.runId);
+                }
+                continue;
+            }
+            if (!Array.isArray(chapter.riskCells)) {
+                plannerIncompleteRunIds.add(run.runId);
+                executorIncompleteRunIds.add(run.runId);
+                continue;
+            }
+            for (const cell of chapter.riskCells) {
+                const stableKey = `${chapter.chapterId}:${cell.riskCellId}`;
+                if (seenInRun.has(stableKey)) {
+                    throw new Error(`Run ${run.runId} contains duplicate stable cell id ${stableKey}`);
+                }
+                seenInRun.add(stableKey);
+                const indexed = cells.get(stableKey) ?? {
+                    cell_id: cell.riskCellId,
+                    chapter_id: chapter.chapterId,
+                    title: cell.title,
+                    byRun: new Map(),
+                    planByRun: new Map(),
+                };
+                indexed.byRun.set(run.runId, {
+                    run_id: run.runId,
+                    status: cell.status,
+                    verdict: cell.agentJudgment?.verdict ?? null,
+                });
+                indexed.planByRun.set(run.runId, {
+                    run_id: run.runId,
+                    fingerprint: cellPlanFingerprint(cell),
+                });
+                cells.set(stableKey, indexed);
+                if (cell.status === "infra_error"
+                    || cell.status === "inconclusive"
+                    || !["passed", "failed"].includes(cell.status)) {
+                    executorIncompleteRunIds.add(run.runId);
+                }
+            }
+        }
+    }
+    const planDriftCells = [];
+    const outcomeDriftCells = [];
+    const unrunnableCells = [];
+    const bisectDriftChapters = [];
+    const unrunnableBisects = [];
+    let stableCells = 0;
+    let stableBisects = 0;
+    for (const cell of [...cells.values()].sort((left, right) => compareCodeUnits(`${left.chapter_id}:${left.cell_id}`, `${right.chapter_id}:${right.cell_id}`))) {
+        const reportCell = reportCellOf(cell, runIds);
+        const planSignatures = new Set(runIds.map((runId) => cell.planByRun.get(runId)?.fingerprint ?? "missing"));
+        const planDrift = planSignatures.size > 1;
+        if (planDrift)
+            planDriftCells.push(reportCell);
+        const outcomeSignatures = new Set(runIds.map((runId) => outcomeSignature(cell.byRun.get(runId))));
+        const outcomeDrift = !planDrift && outcomeSignatures.size > 1;
+        if (outcomeDrift)
+            outcomeDriftCells.push(reportCell);
+        const outcomes = [...cell.byRun.values()];
+        const unrunnable = outcomes.some((outcome) => !["passed", "failed"].includes(outcome.status));
+        if (unrunnable)
+            unrunnableCells.push(reportCell);
+        if (!planDrift
+            && !outcomeDrift
+            && !unrunnable
+            && reportCell.missing_run_ids.length === 0) {
+            stableCells += 1;
+        }
+    }
+    for (const bisect of [...bisects.values()].sort((left, right) => compareCodeUnits(left.chapter_id, right.chapter_id))) {
+        const reportBisect = reportBisectOf(bisect, runIds);
+        const outcomeSignatures = new Set(runIds.map((runId) => bisectOutcomeSignature(bisect.byRun.get(runId))));
+        const outcomeDrift = outcomeSignatures.size > 1;
+        if (outcomeDrift)
+            bisectDriftChapters.push(reportBisect);
+        const unrunnable = [...bisect.byRun.values()].some(observation => observation.status !== "found");
+        if (unrunnable)
+            unrunnableBisects.push(reportBisect);
+        if (!outcomeDrift
+            && !unrunnable
+            && reportBisect.missing_run_ids.length === 0) {
+            stableBisects += 1;
+        }
+    }
+    const plannerHasComparableWork = cells.size > 0;
+    const executorHasComparableWork = cells.size > 0 || bisects.size > 0;
+    const plannerIncomplete = plannerIncompleteRunIds.size > 0
+        || runs.length < requestedRuns
+        || !plannerHasComparableWork;
+    const executorIncomplete = executorIncompleteRunIds.size > 0
+        || runs.length < requestedRuns
+        || !executorHasComparableWork;
+    const plannerReproducible = !plannerIncomplete
+        && planDriftCells.length === 0;
+    const executorReproducible = !executorIncomplete
+        && planDriftCells.length === 0
+        && outcomeDriftCells.length === 0
+        && unrunnableCells.length === 0
+        && bisectDriftChapters.length === 0
+        && unrunnableBisects.length === 0;
+    const reproducible = scope === "planner"
+        ? plannerReproducible
+        : scope === "executor"
+            ? executorReproducible
+            : plannerReproducible && executorReproducible;
+    const incompleteRunIds = scope === "planner"
+        ? plannerIncompleteRunIds
+        : executorIncompleteRunIds;
+    return {
+        schema_version: "1.0.0",
+        scope,
+        requested_runs: requestedRuns,
+        completed_runs: runs.filter((run) => ["complete", "failed"].includes(run.status)).length,
+        run_ids: runIds,
+        plan_source_run_id: scope === "executor" && cells.size > 0
+            ? runs[0]?.runId ?? null
+            : null,
+        compared_cells: cells.size,
+        stable_cells: stableCells,
+        compared_bisects: bisects.size,
+        stable_bisects: stableBisects,
+        plan_drift_cells: planDriftCells,
+        outcome_drift_cells: outcomeDriftCells,
+        unrunnable_cells: unrunnableCells,
+        bisect_drift_chapters: bisectDriftChapters,
+        unrunnable_bisects: unrunnableBisects,
+        planner_incomplete_run_ids: [...plannerIncompleteRunIds],
+        executor_incomplete_run_ids: [...executorIncompleteRunIds],
+        incomplete_run_ids: [...incompleteRunIds],
+        timed_out_run_ids: timedOutRunIds,
+        planner_reproducible: plannerReproducible,
+        executor_reproducible: executorReproducible,
+        reproducible,
+    };
+}
+function positiveNumber(raw, fallback, flag) {
+    const value = raw === undefined ? fallback : Number(raw);
+    if (!Number.isFinite(value) || value <= 0) {
+        throw new Error(`${flag} takes a positive number`);
+    }
+    return value;
+}
+function parseScope(raw) {
+    const scope = raw ?? "executor";
+    if (scope === "executor" || scope === "planner" || scope === "full") {
+        return scope;
+    }
+    throw new Error("--scope must be executor, planner, or full");
+}
+function renderCell(cell, kind) {
+    const glyph = kind === "unrunnable" ? chalk.yellow("!") : chalk.red("✗");
+    console.log(`  ${glyph} ${cell.chapter_id}/${cell.cell_id} ${chalk.dim(cell.title)}`);
+    if (kind === "plan") {
+        for (const plan of cell.plans) {
+            console.log(`      ${plan.run_id}: plan ${plan.fingerprint.slice(0, 12)}`);
+        }
+    }
+    else {
+        for (const outcome of cell.outcomes) {
+            console.log(`      ${outcome.run_id}: ${outcome.status}`
+                + `${outcome.verdict ? ` / ${outcome.verdict}` : ""}`);
+        }
+    }
+    for (const runId of cell.missing_run_ids) {
+        console.log(`      ${runId}: ${chalk.yellow("missing from frozen plan")}`);
+    }
+}
+function renderBisect(bisect, kind) {
+    const glyph = kind === "unrunnable" ? chalk.yellow("!") : chalk.red("✗");
+    console.log(`  ${glyph} ${bisect.chapter_id} ${chalk.dim(bisect.label)}`);
+    for (const outcome of bisect.outcomes) {
+        console.log(`      ${outcome.run_id}: ${outcome.status}`
+            + `${outcome.culprit_sha
+                ? ` / ${outcome.parent_sha ?? "?"}..${outcome.culprit_sha}`
+                : ""}`);
+    }
+    for (const runId of bisect.missing_run_ids) {
+        console.log(`      ${runId}: ${chalk.yellow("missing bisect result")}`);
+    }
+}
+export function runHasFrozenRiskPlan(run) {
+    return run.chapters.some(chapter => chapter.executionMode !== "bisect"
+        && (chapter.riskCells?.length ?? 0) > 0);
+}
+export async function verifyReproducibilityCommand(specimen, options) {
+    const requestedRuns = positiveNumber(options.runs, 3, "--runs");
+    if (!Number.isInteger(requestedRuns) || requestedRuns < 2 || requestedRuns > 20) {
+        throw new Error("--runs must be an integer from 2 to 20");
+    }
+    const scope = parseScope(options.scope);
+    if (options.planSource && scope !== "executor") {
+        throw new Error("--plan-source is only valid with --scope executor");
+    }
+    const intervalMs = positiveNumber(options.interval, 3, "--interval") * 1_000;
+    const timeoutMs = options.timeout
+        ? positiveNumber(options.timeout, 0, "--timeout") * 60_000
+        : undefined;
+    const server = resolveServer(options.server);
+    const repoRoot = discoverVerifyRoot(options.repoRoot);
+    const runs = [];
+    const timedOutRunIds = [];
+    let executorPlanSourceRunId = options.planSource;
+    for (let index = 0; index < requestedRuns; index += 1) {
+        const planSourceRunId = scope === "executor"
+            ? executorPlanSourceRunId
+            : undefined;
+        if (!options.json) {
+            console.error(chalk.dim(`Reproducibility run ${index + 1}/${requestedRuns}: `
+                + `${planSourceRunId
+                    ? `replaying plan from ${planSourceRunId}`
+                    : `planning ${specimen}`}…`));
+        }
+        const started = await startVerificationRun(specimen, {
+            ...options,
+            json: true,
+            open: false,
+            planSourceRunId,
+        });
+        const waited = await waitForTerminal({
+            server,
+            repoRoot,
+            selector: started.runId,
+            intervalMs,
+            timeoutMs,
+        });
+        runs.push(waited.loaded.run);
+        if (scope === "executor"
+            && !executorPlanSourceRunId
+            && runHasFrozenRiskPlan(waited.loaded.run)) {
+            executorPlanSourceRunId = waited.loaded.run.runId;
+        }
+        if (!options.json) {
+            const summary = summarizeRun(waited.loaded, server);
+            console.error(`  ${started.runId}: ${summary.counts.passed} passed, `
+                + `${summary.counts.failed} failed, `
+                + `${summary.counts.could_not_run} could not run`);
+        }
+        if (waited.timedOut) {
+            timedOutRunIds.push(started.runId);
+            break;
+        }
+        if (waited.endReason === "stale")
+            break;
+    }
+    const report = analyzeReproducibility(runs, requestedRuns, timedOutRunIds, scope);
+    if (scope === "executor") {
+        report.plan_source_run_id = executorPlanSourceRunId ?? null;
+    }
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+    }
+    else {
+        const scopeHasComparableWork = scope === "planner"
+            ? report.compared_cells > 0
+            : report.compared_cells > 0 || report.compared_bisects > 0;
+        console.log("");
+        console.log(chalk.bold(`Compared ${report.compared_cells} stable cell ID`
+            + `${report.compared_cells === 1 ? "" : "s"} and `
+            + `${report.compared_bisects} stable bisect chapter ID`
+            + `${report.compared_bisects === 1 ? "" : "s"} across `
+            + `${report.completed_runs}/${report.requested_runs} runs (${scope}).`));
+        if (report.plan_drift_cells.length > 0) {
+            console.log(chalk.red.bold(`${report.plan_drift_cells.length} planner-drift cell`
+                + `${report.plan_drift_cells.length === 1 ? "" : "s"}:`));
+            for (const cell of report.plan_drift_cells)
+                renderCell(cell, "plan");
+        }
+        else if (report.compared_cells > 0) {
+            console.log(chalk.green("✓ Frozen plan contracts were identical."));
+        }
+        if (scope !== "planner") {
+            if (report.outcome_drift_cells.length > 0) {
+                console.log(chalk.red.bold(`${report.outcome_drift_cells.length} runtime/judgment-drift cell`
+                    + `${report.outcome_drift_cells.length === 1 ? "" : "s"}:`));
+                for (const cell of report.outcome_drift_cells) {
+                    renderCell(cell, "outcome");
+                }
+            }
+            else if (report.executor_reproducible && report.compared_cells > 0) {
+                console.log(chalk.green(`✓ All ${report.stable_cells} cells produced the same status and judgment.`));
+            }
+            if (report.bisect_drift_chapters.length > 0) {
+                console.log(chalk.red.bold(`${report.bisect_drift_chapters.length} bisect chapter`
+                    + `${report.bisect_drift_chapters.length === 1 ? "" : "s"} changed outcomes:`));
+                for (const bisect of report.bisect_drift_chapters) {
+                    renderBisect(bisect, "drift");
+                }
+            }
+            else if (report.compared_bisects > 0) {
+                console.log(chalk.green(`✓ All ${report.stable_bisects} bisect chapters found the same exact boundary.`));
+            }
+        }
+        const driftIds = new Set([...report.plan_drift_cells, ...report.outcome_drift_cells].map(cell => `${cell.chapter_id}:${cell.cell_id}`));
+        const repeatablyUnrunnable = report.unrunnable_cells.filter(cell => !driftIds.has(`${cell.chapter_id}:${cell.cell_id}`));
+        if (scope !== "planner" && repeatablyUnrunnable.length > 0) {
+            console.log(chalk.yellow.bold(`${repeatablyUnrunnable.length} cell`
+                + `${repeatablyUnrunnable.length === 1 ? "" : "s"} never produced a verdict:`));
+            for (const cell of repeatablyUnrunnable) {
+                renderCell(cell, "unrunnable");
+            }
+        }
+        if (scope !== "planner" && report.unrunnable_bisects.length > 0) {
+            console.log(chalk.yellow.bold(`${report.unrunnable_bisects.length} bisect chapter`
+                + `${report.unrunnable_bisects.length === 1 ? "" : "s"} did not find a boundary:`));
+            for (const bisect of report.unrunnable_bisects) {
+                renderBisect(bisect, "unrunnable");
+            }
+        }
+        if (report.incomplete_run_ids.length > 0
+            || !scopeHasComparableWork) {
+            console.log(chalk.yellow("Reproducibility is inconclusive because at least one run or cell did not produce the required result."));
+        }
+    }
+    const scopeHasComparableWork = scope === "planner"
+        ? report.compared_cells > 0
+        : report.compared_cells > 0 || report.compared_bisects > 0;
+    process.exitCode = report.incomplete_run_ids.length > 0
+        || report.completed_runs < report.requested_runs
+        || !scopeHasComparableWork
+        ? 2
+        : report.reproducible
+            ? 0
+            : 1;
+}

package/dist/commands/verify.d.ts

@@ -1,3 +1,4 @@
+export declare const VERIFY_SPECIMENS: Set<string>;
 export interface VerifyOptions {
     server?: string;
     token?: string;
@@ -7,5 +8,15 @@ export interface VerifyOptions {
     intentTranscript?: string;
     json?: boolean;
     open?: boolean;
+    /** Internal reproducibility option: execute an already-frozen plan. */
+    planSourceRunId?: string;
 }
+export interface StartRunResponse {
+    runId: string;
+    status: string;
+    resultsUrl: string;
+}
+export declare function parseStartRunResponse(raw: string): StartRunResponse;
+export declare function verifierErrorReason(raw: string, contentType: string | null): string;
+export declare function startVerificationRun(specimenArg: string | undefined, options: VerifyOptions): Promise<StartRunResponse>;
 export declare function verifyCommand(specimenArg: string | undefined, options: VerifyOptions): Promise<void>;

package/dist/commands/verify.js

@@ -4,7 +4,51 @@ import { resolve } from 'node:path';
 import chalk from 'chalk';
 import { setVerifyToken, verifyAuthHeaders } from './verify-core.js';
 const DEFAULT_CONTROL_PLANE = 'http://127.0.0.1:3000';
-const SPECIMENS = new Set(['all', 'posthog-js-478', 'analytics-js-255']);
+export const VERIFY_SPECIMENS = new Set(['all', 'posthog-js-478', 'analytics-js-255']);
+export function parseStartRunResponse(raw) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        throw new Error('Verifier returned an invalid JSON response.');
+    }
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        throw new Error('Verifier returned an invalid response object.');
+    }
+    const response = parsed;
+    const requiredString = (field) => {
+        const value = response[field];
+        if (typeof value !== 'string' || value.trim() === '') {
+            throw new Error(`Verifier response is missing a valid ${field}.`);
+        }
+        return value;
+    };
+    return {
+        runId: requiredString('runId'),
+        status: requiredString('status'),
+        resultsUrl: requiredString('resultsUrl'),
+    };
+}
+export function verifierErrorReason(raw, contentType) {
+    if (!contentType?.toLowerCase().includes('application/json'))
+        return raw;
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        return `malformed JSON error response (${error instanceof Error ? error.message : String(error)}); raw body: ${raw}`;
+    }
+    if (parsed
+        && typeof parsed === 'object'
+        && !Array.isArray(parsed)
+        && typeof parsed.error === 'string'
+        && parsed.error.trim() !== '') {
+        return parsed.error;
+    }
+    return `JSON error response omitted a string error; raw body: ${raw}`;
+}
 function openResults(url) {
     const platform = process.platform;
     const command = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open';
@@ -24,18 +68,15 @@ async function readIntentTranscript(path) {
         throw new Error(`Could not read intent transcript at ${absolutePath}: ${message}`);
     }
 }
-export async function verifyCommand(specimenArg, options) {
+export async function startVerificationRun(specimenArg, options) {
     const specimen = specimenArg || 'all';
-    if (!SPECIMENS.has(specimen)) {
+    if (!VERIFY_SPECIMENS.has(specimen)) {
         throw new Error(`Unknown or unreproduced specimen "${specimen}". Choose all, posthog-js-478, or analytics-js-255.`);
     }
     setVerifyToken(options.token);
     const server = (options.server || process.env.HAYSTACK_VERIFY_SERVER || DEFAULT_CONTROL_PLANE)
         .replace(/\/+$/, '');
     const intentTranscript = await readIntentTranscript(options.intentTranscript);
-    if (!options.json) {
-        console.error(chalk.dim('Preparing the cloud verification run…'));
-    }
     let response;
     try {
         response = await fetch(`${server}/api/cloud-verifier/runs`, {
@@ -47,6 +88,7 @@ export async function verifyCommand(specimenArg, options) {
                 baseRef: options.base,
                 headRef: options.head,
                 intentTranscript,
+                planSourceRunId: options.planSourceRunId,
             }),
         });
     }
@@ -59,25 +101,16 @@ export async function verifyCommand(specimenArg, options) {
         throw new Error(`Control plane at ${server} requires a token (HTTP 401). Pass --token or set HAYSTACK_VERIFY_TOKEN.`);
     }
     if (!response.ok) {
-        let reason = raw;
-        try {
-            reason = JSON.parse(raw).error || raw;
-        }
-        catch {
-            // Preserve the response body when it is not JSON.
-        }
+        const reason = verifierErrorReason(raw, response.headers.get('content-type'));
         throw new Error(`Verifier rejected the run (${response.status}): ${reason}`);
     }
-    let started;
-    try {
-        started = JSON.parse(raw);
-    }
-    catch {
-        throw new Error('Verifier returned an invalid response.');
-    }
-    if (!started.runId || !started.resultsUrl) {
-        throw new Error('Verifier response is missing runId or resultsUrl.');
+    return parseStartRunResponse(raw);
+}
+export async function verifyCommand(specimenArg, options) {
+    if (!options.json) {
+        console.error(chalk.dim('Preparing the cloud verification run…'));
     }
+    const started = await startVerificationRun(specimenArg, options);
     if (options.json) {
         process.stdout.write(`${JSON.stringify({
             schema_version: '1.0.0',