@haystackeditor/cli 0.15.23 → 0.15.25

package/README.md

@@ -105,6 +105,21 @@ haystack verify hosted status cv_<48-lowercase-hex-characters> \
   --json
 ```
 
+To execute a completed run's exact validated risk plan again, keep the same
+repository and commit pair and pass:
+
+```bash
+haystack verify hosted start owner/repo \
+  --base <40-character-commit-sha> \
+  --head <40-character-commit-sha> \
+  --replay-plan-from cv_<48-lowercase-hex-characters> \
+  --json
+```
+
+A successful replay emits a schema-9 terminal result whose
+`riskPlanReplayReceipt` binds the source run and canonical risk-plan hash. The
+replay does not mint or deliver a planning-model credential.
+
 Agents can use the same production lifecycle over stdio MCP:
 
 ```bash
@@ -116,6 +131,25 @@ The hosted server exposes only `verify_start`, `verify_status`, and
 credentials, repository source, provider metadata, and sandbox mutation are
 not exposed.
 
+Freeze one production plan and repeat its exact execution to detect executor or
+assessment drift:
+
+```bash
+haystack verify hosted reproducibility owner/repo \
+  --base <40-character-commit-sha> \
+  --head <40-character-commit-sha> \
+  --runs 3 \
+  --json
+```
+
+The first run creates the source plan; every later run names it through
+`replayPlanFromRunId`. Each run gets a distinct idempotency key derived from one
+printed series key. Pass `--series-key <key>` to resume the same series without
+creating duplicate runs. The report verifies schema-9 replay provenance and
+compares stable risk-cell, capability, and comparison IDs. Missing replay
+provenance, assessment drift, timeouts, unsupported receipts, and inconclusive
+cells all prevent a reproducible verdict and exit 2.
+
 **`haystack setup --json`** speaks NDJSON: events (`question`, `permission`,
 `progress`, `result`) on stdout, replies on stdin keyed by
 `requestID`/`permissionID`. `haystack schema setup` documents the full

package/dist/commands/verify-hosted-mcp.js

@@ -53,6 +53,10 @@ export const HOSTED_VERIFY_TOOLS = [
                     pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$',
                     description: 'Optional stable retry key. Omit to derive it from the exact repository and commits.',
                 },
+                replay_plan_from_run_id: {
+                    ...runIdInput,
+                    description: 'Optional completed owned run whose exact validated risk plan must be replayed.',
+                },
                 account: accountInput,
             },
             required: ['repository', 'base_commit', 'head_commit'],
@@ -120,6 +124,7 @@ function assertToolInputFields(name, input) {
             'base_commit',
             'head_commit',
             'idempotency_key',
+            'replay_plan_from_run_id',
             'repository',
         ]);
         return;
@@ -147,6 +152,7 @@ export async function executeHostedVerifyTool(name, input, context) {
             base: requireInputString(input, 'base_commit'),
             head: requireInputString(input, 'head_commit'),
             idempotencyKey: optionalInputString(input, 'idempotency_key'),
+            replayPlanFrom: optionalInputString(input, 'replay_plan_from_run_id'),
             wait: false,
         }, context.token);
         return buildHostedJson('start', started.state, {

package/dist/commands/verify-hosted-reproducibility.d.ts

@@ -0,0 +1,85 @@
+import { type HostedRunState } from './verify-hosted.js';
+export interface HostedReproducibilityOptions {
+    base: string;
+    head: string;
+    runs?: string;
+    seriesKey?: string;
+    account?: string;
+    interval?: string;
+    timeout?: string;
+    json?: boolean;
+}
+interface HostedReproducibilityRun {
+    state: HostedRunState;
+    timedOut: boolean;
+}
+interface HostedReproducibilityIdentity {
+    seriesKey: string;
+    repository: string;
+    baseCommit: string;
+    headCommit: string;
+    requestedRuns?: number;
+}
+interface ParsedAssessment {
+    schemaVersion: 8 | 9;
+    riskPlanSha256: string;
+    replaySourceRunId: string | null;
+    replaySourceResultSchemaVersion: 8 | 9 | null;
+    planIdentities: Array<{
+        cellId: string;
+        capabilityId: string | null;
+    }>;
+    cellIds: string[];
+    status: string;
+    outcomes: Array<{
+        cellId: string;
+        capabilityId: string | null;
+        comparisonId: string | null;
+        status: string;
+        reasonCode: string;
+    }>;
+}
+interface RunFailure {
+    run_id: string;
+    status: string;
+    reason: string;
+}
+interface Variant {
+    fingerprint: string;
+    run_ids: string[];
+    cell_ids: string[];
+}
+interface OutcomeVariant {
+    fingerprint: string;
+    run_ids: string[];
+    assessment_status: string;
+    outcomes: ParsedAssessment['outcomes'];
+}
+export interface HostedReproducibilityReport {
+    series_key: string;
+    repository: string;
+    base_commit: string;
+    head_commit: string;
+    requested_runs: number;
+    launched_runs: number;
+    unstarted_runs: number;
+    analyzable_runs: number;
+    run_ids: string[];
+    replay_source_run_id: string | null;
+    plan_replay_verified: boolean;
+    planner_reproducibility_tested: false;
+    planner_reproducible: boolean;
+    assessment_reproducible: boolean;
+    reproducible: boolean;
+    plan_variants: Variant[];
+    assessment_variants: OutcomeVariant[];
+    plan_drift_cell_ids: string[];
+    assessment_drift_cell_ids: string[];
+    inconclusive_cell_ids: string[];
+    unrunnable_runs: RunFailure[];
+}
+export declare function hostedReproducibilityRunKey(seriesKey: string, repository: string, baseCommit: string, headCommit: string, ordinal: number): string;
+export declare function analyzeHostedReproducibility(runs: HostedReproducibilityRun[], identity: HostedReproducibilityIdentity): HostedReproducibilityReport;
+export declare function isHostedReproducibilityReplaySource(run: HostedReproducibilityRun, identity: HostedReproducibilityIdentity): boolean;
+export declare function verifyHostedReproducibilityCommand(repositoryArg: string, options: HostedReproducibilityOptions): Promise<void>;
+export {};

package/dist/commands/verify-hosted-reproducibility.js

@@ -0,0 +1,531 @@
+import { createHash, randomBytes } from 'node:crypto';
+import chalk from 'chalk';
+import { withSchema } from '../schema.js';
+import { resolveAuthContext } from '../utils/auth.js';
+import { hostedPollingOptions, parseHostedRepository, startHostedVerification, waitForHostedVerification, } from './verify-hosted.js';
+const CELL_ID = /^cell_[0-9a-f]{32}$/;
+const CAPABILITY_ID = /^repository_driver_[0-9a-f]{64}$/;
+const COMPARISON_ID = /^driver_comparison_[0-9a-f]{64}$/;
+const RUN_ID = /^cv_[0-9a-f]{48}$/;
+const SHA256 = /^[0-9a-f]{64}$/;
+const SERIES_KEY = /^[A-Za-z0-9][A-Za-z0-9._:-]{7,63}$/;
+const TERMINAL_STATUSES = new Set([
+    'complete',
+    'errored',
+    'expired',
+    'terminated',
+]);
+const ASSESSMENT_STATUSES = new Set([
+    'improvement',
+    'inconclusive',
+    'regression',
+    'unchanged',
+]);
+const ASSESSMENT_REASON_CODES = new Set([
+    'case_universe_changed',
+    'coverage_increased',
+    'coverage_reduced',
+    'inconclusive',
+    'mixed_outcomes',
+    'new_failures',
+    'no_bound_driver',
+    'resolved_failures',
+    'unchanged',
+]);
+const OVERALL_ASSESSMENT_STATUSES = new Set([
+    'inconclusive',
+    'no_regressions_detected',
+    'regressions_detected',
+]);
+const DEFAULT_RUNS = 3;
+const MAX_RUNS = 10;
+function record(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error(`${label} is malformed`);
+    }
+    return value;
+}
+function array(value, label) {
+    if (!Array.isArray(value))
+        throw new Error(`${label} is malformed`);
+    return value;
+}
+function boundedInteger(value, fallback, label, minimum, maximum) {
+    if (value === undefined)
+        return fallback;
+    const parsed = Number(value);
+    if (!Number.isSafeInteger(parsed)
+        || parsed < minimum
+        || parsed > maximum) {
+        throw new Error(`${label} must be an integer from ${minimum} through ${maximum}.`);
+    }
+    return parsed;
+}
+function requireSeriesKey(value) {
+    const key = value
+        ?? `series:${new Date().toISOString().replace(/[^0-9]/g, '').slice(0, 14)}:${randomBytes(6).toString('hex')}`;
+    if (!SERIES_KEY.test(key)) {
+        throw new Error('--series-key must be 8-64 characters and use only letters, numbers, ".", "_", ":", or "-".');
+    }
+    return key;
+}
+export function hostedReproducibilityRunKey(seriesKey, repository, baseCommit, headCommit, ordinal) {
+    const digest = createHash('sha256')
+        .update(JSON.stringify([
+        seriesKey,
+        repository,
+        baseCommit,
+        headCommit,
+        ordinal,
+    ]))
+        .digest('hex');
+    return `repro:${digest}`;
+}
+function stable(value) {
+    if (Array.isArray(value))
+        return value.map(stable);
+    if (value && typeof value === 'object') {
+        const object = value;
+        return Object.fromEntries(Object.keys(object)
+            .sort()
+            .map(key => [key, stable(object[key])]));
+    }
+    return value;
+}
+function fingerprint(value) {
+    return createHash('sha256')
+        .update(JSON.stringify(stable(value)))
+        .digest('hex');
+}
+function expectedCellStatus(reasonCode) {
+    if (reasonCode === 'new_failures'
+        || reasonCode === 'coverage_reduced'
+        || reasonCode === 'mixed_outcomes') {
+        return 'regression';
+    }
+    if (reasonCode === 'resolved_failures'
+        || reasonCode === 'coverage_increased') {
+        return 'improvement';
+    }
+    if (reasonCode === 'unchanged')
+        return 'unchanged';
+    return 'inconclusive';
+}
+function expectedOverallStatus(outcomes) {
+    if (outcomes.some(outcome => outcome.status === 'regression')) {
+        return 'regressions_detected';
+    }
+    if (outcomes.length === 0
+        || outcomes.some(outcome => outcome.status === 'inconclusive')) {
+        return 'inconclusive';
+    }
+    return 'no_regressions_detected';
+}
+function requireCount(receipt, field, expected) {
+    if (receipt[field] !== expected) {
+        throw new Error(`driver assessment ${field} is inconsistent`);
+    }
+}
+function parseAssessment(state, identity) {
+    if (state.status !== 'complete' || !state.output) {
+        throw new Error(`run ended with status ${state.status}`);
+    }
+    if (state.repository !== identity.repository
+        || state.baseCommit !== identity.baseCommit
+        || state.headCommit !== identity.headCommit) {
+        throw new Error('run identity does not match the requested exact launch');
+    }
+    const output = record(state.output, 'terminal output');
+    if ((output.schemaVersion !== 8 && output.schemaVersion !== 9)
+        || output.status !== 'driver_assessment_recorded') {
+        throw new Error('run did not produce a schema-8 or schema-9 driver assessment');
+    }
+    const riskPlan = record(output.riskPlanReceipt, 'risk plan receipt');
+    const riskPlanSha256 = fingerprint(riskPlan);
+    let replaySourceRunId = null;
+    let replaySourceResultSchemaVersion = null;
+    if (output.schemaVersion === 9) {
+        if (output.planningCredentialDelivery !== 'not-required-plan-replay') {
+            throw new Error('schema-9 replay used an invalid planning credential mode');
+        }
+        const replay = record(output.riskPlanReplayReceipt, 'risk plan replay receipt');
+        const fields = Object.keys(replay);
+        if (fields.length !== 6
+            || fields.some(field => ![
+                'schemaVersion',
+                'sourceContentPersisted',
+                'sourceResultSchemaVersion',
+                'sourceRiskPlanReceiptSha256',
+                'sourceRunId',
+                'status',
+            ].includes(field))
+            || replay.schemaVersion !== 1
+            || replay.status !== 'risk_plan_replay_verified'
+            || typeof replay.sourceRunId !== 'string'
+            || !RUN_ID.test(replay.sourceRunId)
+            || (replay.sourceResultSchemaVersion !== 8
+                && replay.sourceResultSchemaVersion !== 9)
+            || replay.sourceRiskPlanReceiptSha256 !== riskPlanSha256
+            || typeof replay.sourceRiskPlanReceiptSha256 !== 'string'
+            || !SHA256.test(replay.sourceRiskPlanReceiptSha256)
+            || replay.sourceContentPersisted !== false) {
+            throw new Error('risk plan replay receipt is malformed');
+        }
+        replaySourceRunId = replay.sourceRunId;
+        replaySourceResultSchemaVersion = replay.sourceResultSchemaVersion;
+    }
+    else if (output.riskPlanReplayReceipt !== undefined) {
+        throw new Error('schema-8 result contains unversioned replay provenance');
+    }
+    else if (output.planningCredentialDelivery
+        !== 'per-command-env-gateway-placeholder') {
+        throw new Error('schema-8 result used an invalid planning credential mode');
+    }
+    const assessment = record(output.driverAssessmentReceipt, 'driver assessment receipt');
+    const planCells = array(riskPlan.cells, 'risk plan cells');
+    const assessmentCells = array(assessment.cells, 'assessment cells');
+    if (riskPlan.schemaVersion !== 2
+        || riskPlan.status !== 'risk_plan_ready'
+        || riskPlan.baseCommit !== identity.baseCommit
+        || riskPlan.headCommit !== identity.headCommit
+        || riskPlan.sourceContentPersisted !== false) {
+        throw new Error('risk plan receipt identity is malformed');
+    }
+    const planIdentities = planCells.map((raw) => {
+        const cell = record(raw, 'risk plan cell');
+        if (typeof cell.cellId !== 'string'
+            || !CELL_ID.test(cell.cellId)
+            || (cell.capabilityId !== null
+                && (typeof cell.capabilityId !== 'string'
+                    || !CAPABILITY_ID.test(cell.capabilityId)))) {
+            throw new Error('risk plan cell identity is malformed');
+        }
+        return {
+            cellId: cell.cellId,
+            capabilityId: cell.capabilityId,
+        };
+    }).sort((left, right) => left.cellId.localeCompare(right.cellId));
+    const cellIds = planIdentities.map(cell => cell.cellId);
+    if (cellIds.length === 0
+        || new Set(cellIds).size !== cellIds.length
+        || assessmentCells.length !== cellIds.length) {
+        throw new Error('risk plan and assessment cell sets are inconsistent');
+    }
+    if (assessment.schemaVersion !== 1
+        || typeof assessment.status !== 'string'
+        || !OVERALL_ASSESSMENT_STATUSES.has(assessment.status)
+        || assessment.baseCommit !== identity.baseCommit
+        || assessment.headCommit !== identity.headCommit
+        || typeof assessment.driverComparisonReceiptSha256 !== 'string'
+        || !SHA256.test(assessment.driverComparisonReceiptSha256)
+        || assessment.sourceContentPersisted !== false) {
+        throw new Error('driver assessment receipt identity is malformed');
+    }
+    const outcomes = assessmentCells.map((raw) => {
+        const cell = record(raw, 'assessment cell');
+        if (typeof cell.cellId !== 'string'
+            || !CELL_ID.test(cell.cellId)
+            || (cell.capabilityId !== null
+                && (typeof cell.capabilityId !== 'string'
+                    || !CAPABILITY_ID.test(cell.capabilityId)))
+            || (cell.comparisonId !== null
+                && (typeof cell.comparisonId !== 'string'
+                    || !COMPARISON_ID.test(cell.comparisonId)))
+            || typeof cell.status !== 'string'
+            || !ASSESSMENT_STATUSES.has(cell.status)
+            || typeof cell.reasonCode !== 'string'
+            || !ASSESSMENT_REASON_CODES.has(cell.reasonCode)
+            || cell.status !== expectedCellStatus(cell.reasonCode)
+            || (cell.capabilityId === null
+                ? (cell.comparisonId !== null
+                    || cell.reasonCode !== 'no_bound_driver')
+                : (cell.comparisonId === null
+                    || cell.reasonCode === 'no_bound_driver'))) {
+            throw new Error('assessment cell outcome is malformed');
+        }
+        return {
+            cellId: cell.cellId,
+            capabilityId: cell.capabilityId,
+            comparisonId: cell.comparisonId,
+            status: cell.status,
+            reasonCode: cell.reasonCode,
+        };
+    }).sort((left, right) => left.cellId.localeCompare(right.cellId));
+    if (new Set(outcomes.map(outcome => outcome.cellId)).size !== outcomes.length
+        || outcomes.some((outcome, index) => outcome.cellId !== cellIds[index]
+            || outcome.capabilityId !== planIdentities[index].capabilityId)) {
+        throw new Error('risk plan and assessment cell identities do not match');
+    }
+    const counts = {
+        comparedCellCount: outcomes.filter(outcome => outcome.status !== 'inconclusive').length,
+        regressionCellCount: outcomes.filter(outcome => outcome.status === 'regression').length,
+        improvementCellCount: outcomes.filter(outcome => outcome.status === 'improvement').length,
+        unchangedCellCount: outcomes.filter(outcome => outcome.status === 'unchanged').length,
+        inconclusiveCellCount: outcomes.filter(outcome => outcome.status === 'inconclusive').length,
+    };
+    requireCount(assessment, 'plannedCellCount', outcomes.length);
+    for (const [field, count] of Object.entries(counts)) {
+        requireCount(assessment, field, count);
+    }
+    if (assessment.status !== expectedOverallStatus(outcomes)) {
+        throw new Error('driver assessment overall status is inconsistent');
+    }
+    return {
+        schemaVersion: output.schemaVersion,
+        riskPlanSha256,
+        replaySourceRunId,
+        replaySourceResultSchemaVersion,
+        planIdentities,
+        cellIds,
+        status: assessment.status,
+        outcomes,
+    };
+}
+function groupedPlanVariants(parsed) {
+    const variants = new Map();
+    for (const item of parsed) {
+        const digest = fingerprint(item.assessment.planIdentities);
+        const existing = variants.get(digest);
+        if (existing) {
+            existing.run_ids.push(item.runId);
+        }
+        else {
+            variants.set(digest, {
+                fingerprint: digest,
+                run_ids: [item.runId],
+                cell_ids: item.assessment.cellIds,
+            });
+        }
+    }
+    return [...variants.values()].sort((left, right) => left.fingerprint.localeCompare(right.fingerprint));
+}
+function groupedOutcomeVariants(parsed) {
+    const variants = new Map();
+    for (const item of parsed) {
+        const digest = fingerprint({
+            status: item.assessment.status,
+            outcomes: item.assessment.outcomes,
+        });
+        const existing = variants.get(digest);
+        if (existing) {
+            existing.run_ids.push(item.runId);
+        }
+        else {
+            variants.set(digest, {
+                fingerprint: digest,
+                run_ids: [item.runId],
+                assessment_status: item.assessment.status,
+                outcomes: item.assessment.outcomes,
+            });
+        }
+    }
+    return [...variants.values()].sort((left, right) => left.fingerprint.localeCompare(right.fingerprint));
+}
+function driftIds(parsed, values) {
+    const byCell = new Map();
+    for (const [runIndex, item] of parsed.entries()) {
+        const present = values(item.assessment);
+        for (const [cellId, observed] of byCell) {
+            observed.add(present.has(cellId)
+                ? JSON.stringify(present.get(cellId))
+                : '__missing__');
+        }
+        for (const [cellId, value] of present) {
+            if (byCell.has(cellId))
+                continue;
+            const observed = new Set();
+            if (runIndex > 0)
+                observed.add('__missing__');
+            observed.add(JSON.stringify(value));
+            byCell.set(cellId, observed);
+        }
+    }
+    return [...byCell.entries()]
+        .filter(([, observed]) => observed.size > 1)
+        .map(([cellId]) => cellId)
+        .sort();
+}
+export function analyzeHostedReproducibility(runs, identity) {
+    const parsed = [];
+    const unrunnableRuns = [];
+    for (const run of runs) {
+        if (run.timedOut) {
+            unrunnableRuns.push({
+                run_id: run.state.runId,
+                status: run.state.status,
+                reason: 'wait_timed_out',
+            });
+            continue;
+        }
+        try {
+            parsed.push({
+                runId: run.state.runId,
+                assessment: parseAssessment(run.state, identity),
+            });
+        }
+        catch (error) {
+            unrunnableRuns.push({
+                run_id: run.state.runId,
+                status: run.state.status,
+                reason: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    const planVariants = groupedPlanVariants(parsed);
+    const assessmentVariants = groupedOutcomeVariants(parsed);
+    const replaySourceRunId = parsed[0]?.runId ?? null;
+    const planReplayVerified = parsed.length === runs.length
+        && parsed.length > 1
+        && parsed[0]?.assessment.schemaVersion === 8
+        && parsed[0]?.assessment.replaySourceRunId === null
+        && parsed.slice(1).every(item => item.assessment.schemaVersion === 9
+            && item.assessment.replaySourceRunId === replaySourceRunId
+            && item.assessment.replaySourceResultSchemaVersion
+                === parsed[0].assessment.schemaVersion
+            && item.assessment.riskPlanSha256
+                === parsed[0].assessment.riskPlanSha256);
+    const assessmentReproducible = planReplayVerified && assessmentVariants.length === 1;
+    const planDriftCellIds = driftIds(parsed, assessment => new Map(assessment.planIdentities.map(plan => [
+        plan.cellId,
+        { capabilityId: plan.capabilityId },
+    ])));
+    const assessmentDriftCellIds = driftIds(parsed, assessment => new Map(assessment.outcomes.map(outcome => [
+        outcome.cellId,
+        {
+            capabilityId: outcome.capabilityId,
+            comparisonId: outcome.comparisonId,
+            status: outcome.status,
+            reasonCode: outcome.reasonCode,
+        },
+    ])));
+    const inconclusiveCellIds = [...new Set(parsed.flatMap(item => item.assessment.outcomes
+            .filter(outcome => outcome.status === 'inconclusive')
+            .map(outcome => outcome.cellId)))].sort();
+    const requestedRuns = identity.requestedRuns ?? runs.length;
+    return {
+        series_key: identity.seriesKey,
+        repository: identity.repository,
+        base_commit: identity.baseCommit,
+        head_commit: identity.headCommit,
+        requested_runs: requestedRuns,
+        launched_runs: runs.length,
+        unstarted_runs: Math.max(0, requestedRuns - runs.length),
+        analyzable_runs: parsed.length,
+        run_ids: runs.map(run => run.state.runId),
+        replay_source_run_id: replaySourceRunId,
+        plan_replay_verified: planReplayVerified,
+        planner_reproducibility_tested: false,
+        planner_reproducible: false,
+        assessment_reproducible: assessmentReproducible,
+        reproducible: planReplayVerified
+            && assessmentReproducible
+            && inconclusiveCellIds.length === 0,
+        plan_variants: planVariants,
+        assessment_variants: assessmentVariants,
+        plan_drift_cell_ids: planDriftCellIds,
+        assessment_drift_cell_ids: assessmentDriftCellIds,
+        inconclusive_cell_ids: inconclusiveCellIds,
+        unrunnable_runs: unrunnableRuns,
+    };
+}
+export function isHostedReproducibilityReplaySource(run, identity) {
+    if (run.timedOut || run.state.status !== 'complete')
+        return false;
+    try {
+        const assessment = parseAssessment(run.state, identity);
+        return assessment.schemaVersion === 8
+            && assessment.replaySourceRunId === null;
+    }
+    catch {
+        return false;
+    }
+}
+function renderHostedReproducibility(report) {
+    const icon = report.reproducible ? chalk.green('✓') : chalk.yellow('!');
+    console.log(`${icon} Hosted reproducibility: ${report.reproducible ? 'reproducible' : 'not established'}`);
+    console.log(`  ${chalk.dim('Runs:')} ${report.analyzable_runs}/${report.requested_runs} analyzable; ${report.launched_runs} launched`);
+    console.log(`  ${chalk.dim('Plan variants:')} ${report.plan_variants.length}`);
+    console.log(`  ${chalk.dim('Exact plan replay:')} ${report.plan_replay_verified ? 'verified' : 'not verified'}`);
+    console.log(`  ${chalk.dim('Independent planner reproducibility:')} not tested`);
+    console.log(`  ${chalk.dim('Assessment variants:')} ${report.assessment_variants.length}`);
+    if (report.plan_drift_cell_ids.length > 0) {
+        console.log(`  ${chalk.dim('Plan drift:')} ${report.plan_drift_cell_ids.length} stable cell id(s)`);
+    }
+    if (report.assessment_drift_cell_ids.length > 0) {
+        console.log(`  ${chalk.dim('Assessment drift:')} ${report.assessment_drift_cell_ids.length} cell(s)`);
+    }
+    if (report.inconclusive_cell_ids.length > 0) {
+        console.log(`  ${chalk.dim('Inconclusive:')} ${report.inconclusive_cell_ids.length} cell(s)`);
+    }
+    if (report.unrunnable_runs.length > 0) {
+        console.log(`  ${chalk.dim('Unrunnable:')} ${report.unrunnable_runs.length} run(s)`);
+    }
+}
+export async function verifyHostedReproducibilityCommand(repositoryArg, options) {
+    const repository = parseHostedRepository(repositoryArg);
+    const requestedRuns = boundedInteger(options.runs, DEFAULT_RUNS, '--runs', 2, MAX_RUNS);
+    const seriesKey = requireSeriesKey(options.seriesKey);
+    const auth = await resolveAuthContext({
+        preferredLogin: options.account,
+        owner: repository.owner,
+        repo: repository.repository,
+    });
+    console.error(chalk.dim(`Hosted reproducibility series: ${seriesKey}`));
+    const runs = [];
+    for (let index = 0; index < requestedRuns; index += 1) {
+        if (!options.json) {
+            console.error(chalk.dim(`Starting hosted reproducibility run ${index + 1}/${requestedRuns}…`));
+        }
+        const started = await startHostedVerification(repositoryArg, {
+            base: options.base,
+            head: options.head,
+            idempotencyKey: hostedReproducibilityRunKey(seriesKey, repository.fullName, options.base, options.head, index + 1),
+            replayPlanFrom: index === 0
+                ? undefined
+                : runs[0]?.state.runId,
+            wait: false,
+        }, auth.token);
+        let state = started.state;
+        let timedOut = false;
+        if (!TERMINAL_STATUSES.has(state.status)) {
+            const waited = await waitForHostedVerification(state.runId, auth.token, hostedPollingOptions(options.interval, options.timeout, current => {
+                if (!options.json) {
+                    console.error(chalk.dim(`Run ${index + 1}/${requestedRuns}: ${current.status}…`));
+                }
+            }));
+            state = waited.state;
+            timedOut = waited.timedOut;
+        }
+        runs.push({ state, timedOut });
+        if (index === 0
+            && !isHostedReproducibilityReplaySource(runs[0], {
+                seriesKey,
+                repository: repository.fullName,
+                baseCommit: options.base,
+                headCommit: options.head,
+            })) {
+            if (!options.json) {
+                console.error(chalk.yellow('Source run is not replayable; later runs were not launched.'));
+            }
+            break;
+        }
+    }
+    const report = analyzeHostedReproducibility(runs, {
+        seriesKey,
+        repository: repository.fullName,
+        baseCommit: options.base,
+        headCommit: options.head,
+        requestedRuns,
+    });
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify(withSchema('cloud-verifier', {
+            operation: 'reproducibility',
+            ...report,
+        }), null, 2)}\n`);
+    }
+    else {
+        renderHostedReproducibility(report);
+    }
+    if (!report.reproducible)
+        process.exitCode = 2;
+}

package/dist/commands/verify-hosted.d.ts

@@ -2,6 +2,7 @@ export interface HostedStartOptions {
     base: string;
     head: string;
     idempotencyKey?: string;
+    replayPlanFrom?: string;
     account?: string;
     wait?: boolean;
     interval?: string;
@@ -15,7 +16,7 @@ export interface HostedStatusOptions {
     timeout?: string;
     json?: boolean;
 }
-interface HostedRunState {
+export interface HostedRunState {
     runId: string;
     status: string;
     repository?: string;
@@ -26,7 +27,7 @@ interface HostedRunState {
     output?: Record<string, unknown>;
     error?: string;
 }
-interface HostedPollOptions {
+export interface HostedPollOptions {
     intervalMs: number;
     timeoutMs: number;
     onStatus?: (state: HostedRunState) => void;
@@ -41,7 +42,8 @@ export declare function parseHostedRepository(value: string): {
     repository: string;
     fullName: string;
 };
-export declare function defaultHostedIdempotencyKey(repository: string, baseCommit: string, headCommit: string): string;
+export declare function defaultHostedIdempotencyKey(repository: string, baseCommit: string, headCommit: string, replayPlanFrom?: string): string;
+export declare function hostedPollingOptions(interval: string | undefined, timeout: string | undefined, onStatus?: (state: HostedRunState) => void): HostedPollOptions;
 export declare function startHostedVerification(repositoryArg: string, options: HostedStartOptions, token: string): Promise<{
     state: HostedRunState;
     idempotencyKey: string;

package/dist/commands/verify-hosted.js

@@ -121,9 +121,11 @@ function requireCommit(value, flag) {
     }
     return value;
 }
-export function defaultHostedIdempotencyKey(repository, baseCommit, headCommit) {
+export function defaultHostedIdempotencyKey(repository, baseCommit, headCommit, replayPlanFrom) {
     const digest = createHash('sha256')
-        .update(JSON.stringify([repository, baseCommit, headCommit]))
+        .update(JSON.stringify(replayPlanFrom
+        ? [repository, baseCommit, headCommit, replayPlanFrom]
+        : [repository, baseCommit, headCommit]))
         .digest('hex');
     return `cli:${digest}`;
 }
@@ -142,7 +144,7 @@ function parsePositiveNumber(value, fallback, flag, maximum) {
     }
     return parsed;
 }
-function pollingOptions(interval, timeout, onStatus) {
+export function hostedPollingOptions(interval, timeout, onStatus) {
     return {
         intervalMs: parsePositiveNumber(interval, DEFAULT_INTERVAL_SECONDS, '--interval', 300) * 1000,
         timeoutMs: parsePositiveNumber(timeout, DEFAULT_TIMEOUT_MINUTES, '--timeout', 1_440) * 60_000,
@@ -153,8 +155,12 @@ export async function startHostedVerification(repositoryArg, options, token) {
     const repository = parseHostedRepository(repositoryArg);
     const baseCommit = requireCommit(options.base, '--base');
     const headCommit = requireCommit(options.head, '--head');
+    const replayPlanFrom = options.replayPlanFrom;
+    if (replayPlanFrom !== undefined && !RUN_ID.test(replayPlanFrom)) {
+        throw new Error('--replay-plan-from must match cv_ followed by 48 lowercase hex characters.');
+    }
     const idempotencyKey = requireIdempotencyKey(options.idempotencyKey
-        ?? defaultHostedIdempotencyKey(repository.fullName, baseCommit, headCommit));
+        ?? defaultHostedIdempotencyKey(repository.fullName, baseCommit, headCommit, replayPlanFrom));
     const response = await haystackJson('/api/agent/cloud-verifier/runs', token, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
@@ -164,6 +170,9 @@ export async function startHostedVerification(repositoryArg, options, token) {
             baseCommit,
             headCommit,
             idempotencyKey,
+            ...(replayPlanFrom
+                ? { replayPlanFromRunId: replayPlanFrom }
+                : {}),
         }),
     });
     const state = parseHostedRunState(response);
@@ -200,7 +209,7 @@ function delay(milliseconds) {
 export async function waitForHostedVerification(runId, token, options) {
     const startedAt = Date.now();
     let lastStatus;
-    while (true) {
+    for (;;) {
         const state = await getHostedVerification(runId, token);
         if (state.status !== lastStatus) {
             options.onStatus?.(state);
@@ -295,7 +304,7 @@ export async function verifyHostedStartCommand(repositoryArg, options) {
     let state = started.state;
     let timedOut;
     if (options.wait !== false && !TERMINAL_STATUSES.has(state.status)) {
-        const waited = await waitForHostedVerification(state.runId, auth.token, pollingOptions(options.interval, options.timeout, current => {
+        const waited = await waitForHostedVerification(state.runId, auth.token, hostedPollingOptions(options.interval, options.timeout, current => {
             if (!options.json) {
                 console.error(chalk.dim(`Hosted verifier: ${current.status}…`));
             }
@@ -322,7 +331,7 @@ export async function verifyHostedStatusCommand(runId, options) {
     let state = await getHostedVerification(runId, auth.token);
     let timedOut;
     if (options.wait && !TERMINAL_STATUSES.has(state.status)) {
-        const waited = await waitForHostedVerification(runId, auth.token, pollingOptions(options.interval, options.timeout, current => {
+        const waited = await waitForHostedVerification(runId, auth.token, hostedPollingOptions(options.interval, options.timeout, current => {
             if (!options.json) {
                 console.error(chalk.dim(`Hosted verifier: ${current.status}…`));
             }

package/dist/index.js

@@ -219,6 +219,7 @@ hostedVerify
     .option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
     .option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
     .option('--idempotency-key <key>', 'Stable retry key (default: deterministic from repository and commits)')
+    .option('--replay-plan-from <run-id>', 'Reuse the exact validated risk plan from a completed owned run')
     .option('--account <login>', 'Use a specific saved Haystack account')
     .option('--no-wait', 'Return after the production workflow is queued')
     .option('--interval <seconds>', 'Polling interval while waiting (default 5)')
@@ -228,9 +229,12 @@ hostedVerify
 The default idempotency key makes an exact retry return the original run. Pass
 --idempotency-key with a new stable key only when you intentionally want a
 fresh production execution of the same commit pair.
+--replay-plan-from requires the same repository and exact commit pair and emits
+a schema-9 terminal receipt with replay provenance.
 
 Examples:
   haystack verify hosted start owner/repo --base <sha> --head <sha>
+  haystack verify hosted start owner/repo --base <sha> --head <sha> --replay-plan-from cv_<48 lowercase hex characters>
   haystack verify hosted start owner/repo --base <sha> --head <sha> --no-wait --json
 `)
     .action(async (repository, _options, cmd) => {
@@ -256,6 +260,32 @@ Example:
     const { verifyHostedStatusCommand } = await import('./commands/verify-hosted.js');
     return runPublicCommand(() => verifyHostedStatusCommand(runId, options), options.json);
 });
+hostedVerify
+    .command('reproducibility')
+    .description('Replay one exact hosted risk plan and report executor or assessment drift')
+    .argument('<repository>', 'Exact GitHub owner/repository name')
+    .option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
+    .option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
+    .option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
+    .option('--series-key <key>', 'Stable series key for resumable idempotent retries')
+    .option('--account <login>', 'Use a specific saved Haystack account')
+    .option('--interval <seconds>', 'Polling interval for each run (default 5)')
+    .option('--timeout <minutes>', 'Maximum wait per run (default 35)')
+    .option('--json', 'Versioned machine-readable reproducibility report')
+    .addHelpText('after', `
+The first run creates a fresh risk plan. Every later run replays that exact
+validated plan against the same repository and commit pair with a distinct,
+series-derived idempotency key. A reproducible verdict requires schema-9 replay
+provenance, stable assessment outcomes, and no inconclusive cells.
+
+Example:
+  haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3
+`)
+    .action(async (repository, _options, cmd) => {
+    const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
+    const { verifyHostedReproducibilityCommand } = await import('./commands/verify-hosted-reproducibility.js');
+    return runPublicCommand(() => verifyHostedReproducibilityCommand(repository, options), options.json);
+});
 hostedVerify
     .command('mcp')
     .description('Run a stdio MCP server for the production verifier lifecycle')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.15.23",
+  "version": "0.15.25",
   "description": "Set up Haystack for your project — automated PR review, triage, and merge queue",
   "type": "module",
   "bin": {