@haystackeditor/cli 0.15.22 → 0.15.24

package/README.md

@@ -105,6 +105,33 @@ haystack verify hosted status cv_<48-lowercase-hex-characters> \
   --json
 ```
 
+Agents can use the same production lifecycle over stdio MCP:
+
+```bash
+haystack verify hosted mcp --account <login>
+```
+
+The hosted server exposes only `verify_start`, `verify_status`, and
+`verify_wait`. Tool results use the same `cloud-verifier` schema as the CLI;
+credentials, repository source, provider metadata, and sandbox mutation are
+not exposed.
+
+Repeat the same exact production launch to detect planner or assessment drift:
+
+```bash
+haystack verify hosted reproducibility owner/repo \
+  --base <40-character-commit-sha> \
+  --head <40-character-commit-sha> \
+  --runs 3 \
+  --json
+```
+
+Each run gets a distinct idempotency key derived from one printed series key.
+Pass `--series-key <key>` to resume the same series without creating duplicate
+runs. The report compares stable risk-cell, capability, and comparison IDs.
+Plan drift, assessment drift, timeouts, unsupported receipts, and inconclusive
+cells all prevent a reproducible verdict and exit 2.
+
 **`haystack setup --json`** speaks NDJSON: events (`question`, `permission`,
 `progress`, `result`) on stdout, replies on stdin keyed by
 `requestID`/`permissionID`. `haystack schema setup` documents the full

package/dist/commands/verify-hosted-mcp.d.ts

@@ -0,0 +1,12 @@
+import { type Tool } from '@modelcontextprotocol/sdk/types.js';
+export declare const HOSTED_VERIFY_TOOLS: Tool[];
+export interface HostedVerifyMcpOptions {
+    account?: string;
+}
+interface HostedToolContext {
+    token: string;
+    account?: string;
+}
+export declare function executeHostedVerifyTool(name: string, input: Record<string, unknown>, context: HostedToolContext): Promise<Record<string, unknown>>;
+export declare function runHostedVerifyMcpServer(options: HostedVerifyMcpOptions): Promise<void>;
+export {};

package/dist/commands/verify-hosted-mcp.js

@@ -0,0 +1,234 @@
+// Production Cloud Verifier MCP server: `haystack verify hosted mcp`.
+//
+// This intentionally exposes only the durable hosted run lifecycle. The local
+// specimen MCP server has artifact and retained-sandbox tools whose data model
+// does not exist in the source-free production result contract.
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
+import { resolveAuthContext } from '../utils/auth.js';
+import { buildHostedJson, getHostedVerification, hostedResumeCommand, parseHostedRepository, startHostedVerification, waitForHostedVerification, } from './verify-hosted.js';
+const TERMINAL_STATUSES = new Set([
+    'complete',
+    'errored',
+    'expired',
+    'terminated',
+]);
+const DEFAULT_TIMEOUT_SECONDS = 30 * 60;
+const MAX_TIMEOUT_SECONDS = 60 * 60;
+const accountInput = {
+    type: 'string',
+    description: 'Saved Haystack account login. Omit to use the server default or active account.',
+};
+const runIdInput = {
+    type: 'string',
+    pattern: '^cv_[0-9a-f]{48}$',
+    description: 'Owned hosted run id.',
+};
+export const HOSTED_VERIFY_TOOLS = [
+    {
+        name: 'verify_start',
+        description: 'Start an authenticated production verification for an exact GitHub commit pair. Returns immediately; call verify_wait next.',
+        inputSchema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                repository: {
+                    type: 'string',
+                    pattern: '^[A-Za-z0-9_.-]{1,100}/[A-Za-z0-9_.-]{1,100}$',
+                    description: 'Exact GitHub owner/repository name.',
+                },
+                base_commit: {
+                    type: 'string',
+                    pattern: '^[0-9a-f]{40}$',
+                    description: 'Exact 40-character lowercase base commit SHA.',
+                },
+                head_commit: {
+                    type: 'string',
+                    pattern: '^[0-9a-f]{40}$',
+                    description: 'Exact 40-character lowercase head commit SHA.',
+                },
+                idempotency_key: {
+                    type: 'string',
+                    pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$',
+                    description: 'Optional stable retry key. Omit to derive it from the exact repository and commits.',
+                },
+                account: accountInput,
+            },
+            required: ['repository', 'base_commit', 'head_commit'],
+        },
+    },
+    {
+        name: 'verify_status',
+        description: 'Read the current source-free production receipt for an owned hosted run.',
+        inputSchema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                run_id: runIdInput,
+                account: accountInput,
+            },
+            required: ['run_id'],
+        },
+    },
+    {
+        name: 'verify_wait',
+        description: 'Wait for an owned production run to finish. A timeout returns the last observed status with timed_out=true; the run keeps going.',
+        inputSchema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                run_id: runIdInput,
+                timeout_seconds: {
+                    type: 'integer',
+                    minimum: 10,
+                    maximum: MAX_TIMEOUT_SECONDS,
+                    description: 'Bounded wait in seconds (default 1800, maximum 3600).',
+                },
+                account: accountInput,
+            },
+            required: ['run_id'],
+        },
+    },
+];
+function requireInputString(input, field) {
+    const value = input[field];
+    if (typeof value !== 'string' || value.length === 0) {
+        throw new Error(`${field} required`);
+    }
+    return value;
+}
+function optionalInputString(input, field) {
+    const value = input[field];
+    if (value === undefined)
+        return undefined;
+    if (typeof value !== 'string' || value.length === 0) {
+        throw new Error(`${field} must be a non-empty string`);
+    }
+    return value;
+}
+function assertExactInputFields(input, allowed) {
+    const allowedFields = new Set(allowed);
+    const unsupported = Object.keys(input).find(field => !allowedFields.has(field));
+    if (unsupported)
+        throw new Error(`unsupported input field: ${unsupported}`);
+}
+function assertToolInputFields(name, input) {
+    if (name === 'verify_start') {
+        assertExactInputFields(input, [
+            'account',
+            'base_commit',
+            'head_commit',
+            'idempotency_key',
+            'repository',
+        ]);
+        return;
+    }
+    if (name === 'verify_status') {
+        assertExactInputFields(input, ['account', 'run_id']);
+        return;
+    }
+    if (name === 'verify_wait') {
+        assertExactInputFields(input, ['account', 'run_id', 'timeout_seconds']);
+        return;
+    }
+    throw new Error(`unknown tool: ${name}`);
+}
+function nextCommand(state, account) {
+    return TERMINAL_STATUSES.has(state.status)
+        ? undefined
+        : hostedResumeCommand(state.runId, account);
+}
+export async function executeHostedVerifyTool(name, input, context) {
+    assertToolInputFields(name, input);
+    if (name === 'verify_start') {
+        const repository = requireInputString(input, 'repository');
+        const started = await startHostedVerification(repository, {
+            base: requireInputString(input, 'base_commit'),
+            head: requireInputString(input, 'head_commit'),
+            idempotencyKey: optionalInputString(input, 'idempotency_key'),
+            wait: false,
+        }, context.token);
+        return buildHostedJson('start', started.state, {
+            account: context.account,
+            idempotencyKey: started.idempotencyKey,
+            nextCommand: nextCommand(started.state, context.account),
+        });
+    }
+    if (name === 'verify_status') {
+        const state = await getHostedVerification(requireInputString(input, 'run_id'), context.token);
+        return buildHostedJson('status', state, {
+            account: context.account,
+            nextCommand: nextCommand(state, context.account),
+        });
+    }
+    if (name === 'verify_wait') {
+        const runId = requireInputString(input, 'run_id');
+        const rawTimeout = input.timeout_seconds;
+        const timeoutSeconds = rawTimeout === undefined
+            ? DEFAULT_TIMEOUT_SECONDS
+            : rawTimeout;
+        if (typeof timeoutSeconds !== 'number'
+            || !Number.isInteger(timeoutSeconds)
+            || timeoutSeconds < 10
+            || timeoutSeconds > MAX_TIMEOUT_SECONDS) {
+            throw new Error('timeout_seconds must be an integer from 10 to 3600');
+        }
+        const waited = await waitForHostedVerification(runId, context.token, {
+            intervalMs: 3_000,
+            timeoutMs: timeoutSeconds * 1_000,
+        });
+        return buildHostedJson('status', waited.state, {
+            account: context.account,
+            nextCommand: nextCommand(waited.state, context.account),
+            timedOut: waited.timedOut,
+        });
+    }
+    throw new Error(`unknown tool: ${name}`);
+}
+async function resolveToolContext(name, input, options) {
+    const requestedAccount = optionalInputString(input, 'account') ?? options.account;
+    const repository = name === 'verify_start'
+        ? parseHostedRepository(requireInputString(input, 'repository'))
+        : undefined;
+    const auth = await resolveAuthContext({
+        preferredLogin: requestedAccount,
+        owner: repository?.owner,
+        repo: repository?.repository,
+    });
+    return {
+        token: auth.token,
+        account: auth.token.startsWith('hsk_live_') ? undefined : auth.login,
+    };
+}
+export async function runHostedVerifyMcpServer(options) {
+    const mcp = new Server({ name: 'haystack-verify-hosted', version: '1.0.0' }, { capabilities: { tools: {} } });
+    mcp.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: HOSTED_VERIFY_TOOLS }));
+    mcp.setRequestHandler(CallToolRequestSchema, async (request) => {
+        const { name, arguments: args } = request.params;
+        const input = (args ?? {});
+        try {
+            // Reject undeclared fields before account resolution so callers cannot
+            // use an invalid request to trigger credential or repository lookup.
+            assertToolInputFields(name, input);
+            const context = await resolveToolContext(name, input, options);
+            const result = await executeHostedVerifyTool(name, input, context);
+            return {
+                content: [{
+                        type: 'text',
+                        text: JSON.stringify(result, null, 2),
+                    }],
+            };
+        }
+        catch (error) {
+            return {
+                isError: true,
+                content: [{
+                        type: 'text',
+                        text: error instanceof Error ? error.message : String(error),
+                    }],
+            };
+        }
+    });
+    await mcp.connect(new StdioServerTransport());
+}

package/dist/commands/verify-hosted-reproducibility.d.ts

@@ -0,0 +1,73 @@
+import { type HostedRunState } from './verify-hosted.js';
+export interface HostedReproducibilityOptions {
+    base: string;
+    head: string;
+    runs?: string;
+    seriesKey?: string;
+    account?: string;
+    interval?: string;
+    timeout?: string;
+    json?: boolean;
+}
+interface HostedReproducibilityRun {
+    state: HostedRunState;
+    timedOut: boolean;
+}
+interface ParsedAssessment {
+    planIdentities: Array<{
+        cellId: string;
+        capabilityId: string | null;
+    }>;
+    cellIds: string[];
+    status: string;
+    outcomes: Array<{
+        cellId: string;
+        capabilityId: string | null;
+        comparisonId: string | null;
+        status: string;
+        reasonCode: string;
+    }>;
+}
+interface RunFailure {
+    run_id: string;
+    status: string;
+    reason: string;
+}
+interface Variant {
+    fingerprint: string;
+    run_ids: string[];
+    cell_ids: string[];
+}
+interface OutcomeVariant {
+    fingerprint: string;
+    run_ids: string[];
+    assessment_status: string;
+    outcomes: ParsedAssessment['outcomes'];
+}
+export interface HostedReproducibilityReport {
+    series_key: string;
+    repository: string;
+    base_commit: string;
+    head_commit: string;
+    requested_runs: number;
+    analyzable_runs: number;
+    run_ids: string[];
+    planner_reproducible: boolean;
+    assessment_reproducible: boolean;
+    reproducible: boolean;
+    plan_variants: Variant[];
+    assessment_variants: OutcomeVariant[];
+    plan_drift_cell_ids: string[];
+    assessment_drift_cell_ids: string[];
+    inconclusive_cell_ids: string[];
+    unrunnable_runs: RunFailure[];
+}
+export declare function hostedReproducibilityRunKey(seriesKey: string, repository: string, baseCommit: string, headCommit: string, ordinal: number): string;
+export declare function analyzeHostedReproducibility(runs: HostedReproducibilityRun[], identity: {
+    seriesKey: string;
+    repository: string;
+    baseCommit: string;
+    headCommit: string;
+}): HostedReproducibilityReport;
+export declare function verifyHostedReproducibilityCommand(repositoryArg: string, options: HostedReproducibilityOptions): Promise<void>;
+export {};

package/dist/commands/verify-hosted-reproducibility.js

@@ -0,0 +1,427 @@
+import { createHash, randomBytes } from 'node:crypto';
+import chalk from 'chalk';
+import { withSchema } from '../schema.js';
+import { resolveAuthContext } from '../utils/auth.js';
+import { hostedPollingOptions, parseHostedRepository, startHostedVerification, waitForHostedVerification, } from './verify-hosted.js';
+const CELL_ID = /^cell_[0-9a-f]{32}$/;
+const CAPABILITY_ID = /^repository_driver_[0-9a-f]{64}$/;
+const COMPARISON_ID = /^driver_comparison_[0-9a-f]{64}$/;
+const SHA256 = /^[0-9a-f]{64}$/;
+const SERIES_KEY = /^[A-Za-z0-9][A-Za-z0-9._:-]{7,63}$/;
+const TERMINAL_STATUSES = new Set([
+    'complete',
+    'errored',
+    'expired',
+    'terminated',
+]);
+const ASSESSMENT_STATUSES = new Set([
+    'improvement',
+    'inconclusive',
+    'regression',
+    'unchanged',
+]);
+const ASSESSMENT_REASON_CODES = new Set([
+    'case_universe_changed',
+    'coverage_increased',
+    'coverage_reduced',
+    'inconclusive',
+    'mixed_outcomes',
+    'new_failures',
+    'no_bound_driver',
+    'resolved_failures',
+    'unchanged',
+]);
+const OVERALL_ASSESSMENT_STATUSES = new Set([
+    'inconclusive',
+    'no_regressions_detected',
+    'regressions_detected',
+]);
+const DEFAULT_RUNS = 3;
+const MAX_RUNS = 10;
+function record(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error(`${label} is malformed`);
+    }
+    return value;
+}
+function array(value, label) {
+    if (!Array.isArray(value))
+        throw new Error(`${label} is malformed`);
+    return value;
+}
+function boundedInteger(value, fallback, label, minimum, maximum) {
+    if (value === undefined)
+        return fallback;
+    const parsed = Number(value);
+    if (!Number.isSafeInteger(parsed)
+        || parsed < minimum
+        || parsed > maximum) {
+        throw new Error(`${label} must be an integer from ${minimum} through ${maximum}.`);
+    }
+    return parsed;
+}
+function requireSeriesKey(value) {
+    const key = value
+        ?? `series:${new Date().toISOString().replace(/[^0-9]/g, '').slice(0, 14)}:${randomBytes(6).toString('hex')}`;
+    if (!SERIES_KEY.test(key)) {
+        throw new Error('--series-key must be 8-64 characters and use only letters, numbers, ".", "_", ":", or "-".');
+    }
+    return key;
+}
+export function hostedReproducibilityRunKey(seriesKey, repository, baseCommit, headCommit, ordinal) {
+    const digest = createHash('sha256')
+        .update(JSON.stringify([
+        seriesKey,
+        repository,
+        baseCommit,
+        headCommit,
+        ordinal,
+    ]))
+        .digest('hex');
+    return `repro:${digest}`;
+}
+function fingerprint(value) {
+    return createHash('sha256')
+        .update(JSON.stringify(value))
+        .digest('hex');
+}
+function expectedCellStatus(reasonCode) {
+    if (reasonCode === 'new_failures'
+        || reasonCode === 'coverage_reduced'
+        || reasonCode === 'mixed_outcomes') {
+        return 'regression';
+    }
+    if (reasonCode === 'resolved_failures'
+        || reasonCode === 'coverage_increased') {
+        return 'improvement';
+    }
+    if (reasonCode === 'unchanged')
+        return 'unchanged';
+    return 'inconclusive';
+}
+function expectedOverallStatus(outcomes) {
+    if (outcomes.some(outcome => outcome.status === 'regression')) {
+        return 'regressions_detected';
+    }
+    if (outcomes.length === 0
+        || outcomes.some(outcome => outcome.status === 'inconclusive')) {
+        return 'inconclusive';
+    }
+    return 'no_regressions_detected';
+}
+function requireCount(receipt, field, expected) {
+    if (receipt[field] !== expected) {
+        throw new Error(`driver assessment ${field} is inconsistent`);
+    }
+}
+function parseAssessment(state, identity) {
+    if (state.status !== 'complete' || !state.output) {
+        throw new Error(`run ended with status ${state.status}`);
+    }
+    if (state.repository !== identity.repository
+        || state.baseCommit !== identity.baseCommit
+        || state.headCommit !== identity.headCommit) {
+        throw new Error('run identity does not match the requested exact launch');
+    }
+    const output = record(state.output, 'terminal output');
+    if (output.schemaVersion !== 8
+        || output.status !== 'driver_assessment_recorded') {
+        throw new Error('run did not produce a schema-8 driver assessment');
+    }
+    const riskPlan = record(output.riskPlanReceipt, 'risk plan receipt');
+    const assessment = record(output.driverAssessmentReceipt, 'driver assessment receipt');
+    const planCells = array(riskPlan.cells, 'risk plan cells');
+    const assessmentCells = array(assessment.cells, 'assessment cells');
+    if (riskPlan.schemaVersion !== 2
+        || riskPlan.status !== 'risk_plan_ready'
+        || riskPlan.baseCommit !== identity.baseCommit
+        || riskPlan.headCommit !== identity.headCommit
+        || riskPlan.sourceContentPersisted !== false) {
+        throw new Error('risk plan receipt identity is malformed');
+    }
+    const planIdentities = planCells.map((raw) => {
+        const cell = record(raw, 'risk plan cell');
+        if (typeof cell.cellId !== 'string'
+            || !CELL_ID.test(cell.cellId)
+            || (cell.capabilityId !== null
+                && (typeof cell.capabilityId !== 'string'
+                    || !CAPABILITY_ID.test(cell.capabilityId)))) {
+            throw new Error('risk plan cell identity is malformed');
+        }
+        return {
+            cellId: cell.cellId,
+            capabilityId: cell.capabilityId,
+        };
+    }).sort((left, right) => left.cellId.localeCompare(right.cellId));
+    const cellIds = planIdentities.map(cell => cell.cellId);
+    if (cellIds.length === 0
+        || new Set(cellIds).size !== cellIds.length
+        || assessmentCells.length !== cellIds.length) {
+        throw new Error('risk plan and assessment cell sets are inconsistent');
+    }
+    if (assessment.schemaVersion !== 1
+        || typeof assessment.status !== 'string'
+        || !OVERALL_ASSESSMENT_STATUSES.has(assessment.status)
+        || assessment.baseCommit !== identity.baseCommit
+        || assessment.headCommit !== identity.headCommit
+        || typeof assessment.driverComparisonReceiptSha256 !== 'string'
+        || !SHA256.test(assessment.driverComparisonReceiptSha256)
+        || assessment.sourceContentPersisted !== false) {
+        throw new Error('driver assessment receipt identity is malformed');
+    }
+    const outcomes = assessmentCells.map((raw) => {
+        const cell = record(raw, 'assessment cell');
+        if (typeof cell.cellId !== 'string'
+            || !CELL_ID.test(cell.cellId)
+            || (cell.capabilityId !== null
+                && (typeof cell.capabilityId !== 'string'
+                    || !CAPABILITY_ID.test(cell.capabilityId)))
+            || (cell.comparisonId !== null
+                && (typeof cell.comparisonId !== 'string'
+                    || !COMPARISON_ID.test(cell.comparisonId)))
+            || typeof cell.status !== 'string'
+            || !ASSESSMENT_STATUSES.has(cell.status)
+            || typeof cell.reasonCode !== 'string'
+            || !ASSESSMENT_REASON_CODES.has(cell.reasonCode)
+            || cell.status !== expectedCellStatus(cell.reasonCode)
+            || (cell.capabilityId === null
+                ? (cell.comparisonId !== null
+                    || cell.reasonCode !== 'no_bound_driver')
+                : (cell.comparisonId === null
+                    || cell.reasonCode === 'no_bound_driver'))) {
+            throw new Error('assessment cell outcome is malformed');
+        }
+        return {
+            cellId: cell.cellId,
+            capabilityId: cell.capabilityId,
+            comparisonId: cell.comparisonId,
+            status: cell.status,
+            reasonCode: cell.reasonCode,
+        };
+    }).sort((left, right) => left.cellId.localeCompare(right.cellId));
+    if (new Set(outcomes.map(outcome => outcome.cellId)).size !== outcomes.length
+        || outcomes.some((outcome, index) => outcome.cellId !== cellIds[index]
+            || outcome.capabilityId !== planIdentities[index].capabilityId)) {
+        throw new Error('risk plan and assessment cell identities do not match');
+    }
+    const counts = {
+        comparedCellCount: outcomes.filter(outcome => outcome.status !== 'inconclusive').length,
+        regressionCellCount: outcomes.filter(outcome => outcome.status === 'regression').length,
+        improvementCellCount: outcomes.filter(outcome => outcome.status === 'improvement').length,
+        unchangedCellCount: outcomes.filter(outcome => outcome.status === 'unchanged').length,
+        inconclusiveCellCount: outcomes.filter(outcome => outcome.status === 'inconclusive').length,
+    };
+    requireCount(assessment, 'plannedCellCount', outcomes.length);
+    for (const [field, count] of Object.entries(counts)) {
+        requireCount(assessment, field, count);
+    }
+    if (assessment.status !== expectedOverallStatus(outcomes)) {
+        throw new Error('driver assessment overall status is inconsistent');
+    }
+    return {
+        planIdentities,
+        cellIds,
+        status: assessment.status,
+        outcomes,
+    };
+}
+function groupedPlanVariants(parsed) {
+    const variants = new Map();
+    for (const item of parsed) {
+        const digest = fingerprint(item.assessment.planIdentities);
+        const existing = variants.get(digest);
+        if (existing) {
+            existing.run_ids.push(item.runId);
+        }
+        else {
+            variants.set(digest, {
+                fingerprint: digest,
+                run_ids: [item.runId],
+                cell_ids: item.assessment.cellIds,
+            });
+        }
+    }
+    return [...variants.values()].sort((left, right) => left.fingerprint.localeCompare(right.fingerprint));
+}
+function groupedOutcomeVariants(parsed) {
+    const variants = new Map();
+    for (const item of parsed) {
+        const digest = fingerprint({
+            status: item.assessment.status,
+            outcomes: item.assessment.outcomes,
+        });
+        const existing = variants.get(digest);
+        if (existing) {
+            existing.run_ids.push(item.runId);
+        }
+        else {
+            variants.set(digest, {
+                fingerprint: digest,
+                run_ids: [item.runId],
+                assessment_status: item.assessment.status,
+                outcomes: item.assessment.outcomes,
+            });
+        }
+    }
+    return [...variants.values()].sort((left, right) => left.fingerprint.localeCompare(right.fingerprint));
+}
+function driftIds(parsed, values) {
+    const byCell = new Map();
+    for (const [runIndex, item] of parsed.entries()) {
+        const present = values(item.assessment);
+        for (const [cellId, observed] of byCell) {
+            observed.add(present.has(cellId)
+                ? JSON.stringify(present.get(cellId))
+                : '__missing__');
+        }
+        for (const [cellId, value] of present) {
+            if (byCell.has(cellId))
+                continue;
+            const observed = new Set();
+            if (runIndex > 0)
+                observed.add('__missing__');
+            observed.add(JSON.stringify(value));
+            byCell.set(cellId, observed);
+        }
+    }
+    return [...byCell.entries()]
+        .filter(([, observed]) => observed.size > 1)
+        .map(([cellId]) => cellId)
+        .sort();
+}
+export function analyzeHostedReproducibility(runs, identity) {
+    const parsed = [];
+    const unrunnableRuns = [];
+    for (const run of runs) {
+        if (run.timedOut) {
+            unrunnableRuns.push({
+                run_id: run.state.runId,
+                status: run.state.status,
+                reason: 'wait_timed_out',
+            });
+            continue;
+        }
+        try {
+            parsed.push({
+                runId: run.state.runId,
+                assessment: parseAssessment(run.state, identity),
+            });
+        }
+        catch (error) {
+            unrunnableRuns.push({
+                run_id: run.state.runId,
+                status: run.state.status,
+                reason: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    const planVariants = groupedPlanVariants(parsed);
+    const assessmentVariants = groupedOutcomeVariants(parsed);
+    const plannerReproducible = parsed.length === runs.length && planVariants.length === 1;
+    const assessmentReproducible = plannerReproducible && assessmentVariants.length === 1;
+    const planDriftCellIds = driftIds(parsed, assessment => new Map(assessment.planIdentities.map(plan => [
+        plan.cellId,
+        { capabilityId: plan.capabilityId },
+    ])));
+    const assessmentDriftCellIds = driftIds(parsed, assessment => new Map(assessment.outcomes.map(outcome => [
+        outcome.cellId,
+        {
+            capabilityId: outcome.capabilityId,
+            comparisonId: outcome.comparisonId,
+            status: outcome.status,
+            reasonCode: outcome.reasonCode,
+        },
+    ])));
+    const inconclusiveCellIds = [...new Set(parsed.flatMap(item => item.assessment.outcomes
+            .filter(outcome => outcome.status === 'inconclusive')
+            .map(outcome => outcome.cellId)))].sort();
+    return {
+        series_key: identity.seriesKey,
+        repository: identity.repository,
+        base_commit: identity.baseCommit,
+        head_commit: identity.headCommit,
+        requested_runs: runs.length,
+        analyzable_runs: parsed.length,
+        run_ids: runs.map(run => run.state.runId),
+        planner_reproducible: plannerReproducible,
+        assessment_reproducible: assessmentReproducible,
+        reproducible: assessmentReproducible && inconclusiveCellIds.length === 0,
+        plan_variants: planVariants,
+        assessment_variants: assessmentVariants,
+        plan_drift_cell_ids: planDriftCellIds,
+        assessment_drift_cell_ids: assessmentDriftCellIds,
+        inconclusive_cell_ids: inconclusiveCellIds,
+        unrunnable_runs: unrunnableRuns,
+    };
+}
+function renderHostedReproducibility(report) {
+    const icon = report.reproducible ? chalk.green('✓') : chalk.yellow('!');
+    console.log(`${icon} Hosted reproducibility: ${report.reproducible ? 'reproducible' : 'not established'}`);
+    console.log(`  ${chalk.dim('Runs:')} ${report.analyzable_runs}/${report.requested_runs} analyzable`);
+    console.log(`  ${chalk.dim('Plan variants:')} ${report.plan_variants.length}`);
+    console.log(`  ${chalk.dim('Assessment variants:')} ${report.assessment_variants.length}`);
+    if (report.plan_drift_cell_ids.length > 0) {
+        console.log(`  ${chalk.dim('Plan drift:')} ${report.plan_drift_cell_ids.length} stable cell id(s)`);
+    }
+    if (report.assessment_drift_cell_ids.length > 0) {
+        console.log(`  ${chalk.dim('Assessment drift:')} ${report.assessment_drift_cell_ids.length} cell(s)`);
+    }
+    if (report.inconclusive_cell_ids.length > 0) {
+        console.log(`  ${chalk.dim('Inconclusive:')} ${report.inconclusive_cell_ids.length} cell(s)`);
+    }
+    if (report.unrunnable_runs.length > 0) {
+        console.log(`  ${chalk.dim('Unrunnable:')} ${report.unrunnable_runs.length} run(s)`);
+    }
+}
+export async function verifyHostedReproducibilityCommand(repositoryArg, options) {
+    const repository = parseHostedRepository(repositoryArg);
+    const requestedRuns = boundedInteger(options.runs, DEFAULT_RUNS, '--runs', 2, MAX_RUNS);
+    const seriesKey = requireSeriesKey(options.seriesKey);
+    const auth = await resolveAuthContext({
+        preferredLogin: options.account,
+        owner: repository.owner,
+        repo: repository.repository,
+    });
+    console.error(chalk.dim(`Hosted reproducibility series: ${seriesKey}`));
+    const runs = [];
+    for (let index = 0; index < requestedRuns; index += 1) {
+        if (!options.json) {
+            console.error(chalk.dim(`Starting hosted reproducibility run ${index + 1}/${requestedRuns}…`));
+        }
+        const started = await startHostedVerification(repositoryArg, {
+            base: options.base,
+            head: options.head,
+            idempotencyKey: hostedReproducibilityRunKey(seriesKey, repository.fullName, options.base, options.head, index + 1),
+            wait: false,
+        }, auth.token);
+        let state = started.state;
+        let timedOut = false;
+        if (!TERMINAL_STATUSES.has(state.status)) {
+            const waited = await waitForHostedVerification(state.runId, auth.token, hostedPollingOptions(options.interval, options.timeout, current => {
+                if (!options.json) {
+                    console.error(chalk.dim(`Run ${index + 1}/${requestedRuns}: ${current.status}…`));
+                }
+            }));
+            state = waited.state;
+            timedOut = waited.timedOut;
+        }
+        runs.push({ state, timedOut });
+    }
+    const report = analyzeHostedReproducibility(runs, {
+        seriesKey,
+        repository: repository.fullName,
+        baseCommit: options.base,
+        headCommit: options.head,
+    });
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify(withSchema('cloud-verifier', {
+            operation: 'reproducibility',
+            ...report,
+        }), null, 2)}\n`);
+    }
+    else {
+        renderHostedReproducibility(report);
+    }
+    if (!report.reproducible)
+        process.exitCode = 2;
+}

package/dist/commands/verify-hosted.d.ts

@@ -15,7 +15,7 @@ export interface HostedStatusOptions {
     timeout?: string;
     json?: boolean;
 }
-interface HostedRunState {
+export interface HostedRunState {
     runId: string;
     status: string;
     repository?: string;
@@ -26,7 +26,7 @@ interface HostedRunState {
     output?: Record<string, unknown>;
     error?: string;
 }
-interface HostedPollOptions {
+export interface HostedPollOptions {
     intervalMs: number;
     timeoutMs: number;
     onStatus?: (state: HostedRunState) => void;
@@ -42,6 +42,7 @@ export declare function parseHostedRepository(value: string): {
     fullName: string;
 };
 export declare function defaultHostedIdempotencyKey(repository: string, baseCommit: string, headCommit: string): string;
+export declare function hostedPollingOptions(interval: string | undefined, timeout: string | undefined, onStatus?: (state: HostedRunState) => void): HostedPollOptions;
 export declare function startHostedVerification(repositoryArg: string, options: HostedStartOptions, token: string): Promise<{
     state: HostedRunState;
     idempotencyKey: string;

package/dist/commands/verify-hosted.js

@@ -142,7 +142,7 @@ function parsePositiveNumber(value, fallback, flag, maximum) {
     }
     return parsed;
 }
-function pollingOptions(interval, timeout, onStatus) {
+export function hostedPollingOptions(interval, timeout, onStatus) {
     return {
         intervalMs: parsePositiveNumber(interval, DEFAULT_INTERVAL_SECONDS, '--interval', 300) * 1000,
         timeoutMs: parsePositiveNumber(timeout, DEFAULT_TIMEOUT_MINUTES, '--timeout', 1_440) * 60_000,
@@ -295,7 +295,7 @@ export async function verifyHostedStartCommand(repositoryArg, options) {
     let state = started.state;
     let timedOut;
     if (options.wait !== false && !TERMINAL_STATUSES.has(state.status)) {
-        const waited = await waitForHostedVerification(state.runId, auth.token, pollingOptions(options.interval, options.timeout, current => {
+        const waited = await waitForHostedVerification(state.runId, auth.token, hostedPollingOptions(options.interval, options.timeout, current => {
             if (!options.json) {
                 console.error(chalk.dim(`Hosted verifier: ${current.status}…`));
             }
@@ -322,7 +322,7 @@ export async function verifyHostedStatusCommand(runId, options) {
     let state = await getHostedVerification(runId, auth.token);
     let timedOut;
     if (options.wait && !TERMINAL_STATUSES.has(state.status)) {
-        const waited = await waitForHostedVerification(runId, auth.token, pollingOptions(options.interval, options.timeout, current => {
+        const waited = await waitForHostedVerification(runId, auth.token, hostedPollingOptions(options.interval, options.timeout, current => {
             if (!options.json) {
                 console.error(chalk.dim(`Hosted verifier: ${current.status}…`));
             }

package/dist/index.js

@@ -256,6 +256,52 @@ Example:
     const { verifyHostedStatusCommand } = await import('./commands/verify-hosted.js');
     return runPublicCommand(() => verifyHostedStatusCommand(runId, options), options.json);
 });
+hostedVerify
+    .command('reproducibility')
+    .description('Repeat an exact hosted verification and report plan or assessment drift')
+    .argument('<repository>', 'Exact GitHub owner/repository name')
+    .option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
+    .option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
+    .option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
+    .option('--series-key <key>', 'Stable series key for resumable idempotent retries')
+    .option('--account <login>', 'Use a specific saved Haystack account')
+    .option('--interval <seconds>', 'Polling interval for each run (default 5)')
+    .option('--timeout <minutes>', 'Maximum wait per run (default 35)')
+    .option('--json', 'Versioned machine-readable reproducibility report')
+    .addHelpText('after', `
+Every run uses the same repository and exact commit pair with a distinct,
+series-derived idempotency key. The report compares stable risk-cell identities
+and deterministic assessment outcomes; inconclusive cells never count as a
+reproducible verdict.
+
+Example:
+  haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3
+`)
+    .action(async (repository, _options, cmd) => {
+    const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
+    const { verifyHostedReproducibilityCommand } = await import('./commands/verify-hosted-reproducibility.js');
+    return runPublicCommand(() => verifyHostedReproducibilityCommand(repository, options), options.json);
+});
+hostedVerify
+    .command('mcp')
+    .description('Run a stdio MCP server for the production verifier lifecycle')
+    .option('--account <login>', 'Default saved Haystack account for status and wait tools')
+    .addHelpText('after', `
+Tools: verify_start, verify_status, verify_wait.
+No credential, repository source, provider metadata, or sandbox mutation tool
+is exposed through this production server.
+`)
+    .action(async (_options, cmd) => {
+    const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
+    try {
+        const { runHostedVerifyMcpServer } = await import('./commands/verify-hosted-mcp.js');
+        await runHostedVerifyMcpServer(options);
+    }
+    catch (err) {
+        console.error(chalk.red('hosted verify mcp server failed:'), err instanceof Error ? err.message : err);
+        process.exit(1);
+    }
+});
 verify
     .command('reproducibility')
     .description('Separate frozen-plan drift from executor/judgment nondeterminism')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.15.22",
+  "version": "0.15.24",
   "description": "Set up Haystack for your project — automated PR review, triage, and merge queue",
   "type": "module",
   "bin": {