@haystackeditor/cli 0.15.31 → 0.16.0

package/dist/commands/verify-hosted-reproducibility.js

@@ -2,569 +2,71 @@ import { createHash, randomBytes } from 'node:crypto';
 import chalk from 'chalk';
 import { withSchema } from '../schema.js';
 import { resolveAuthContext } from '../utils/auth.js';
-import { hostedPollingOptions, parseHostedRepository, readHostedIntentFile, startHostedVerification, waitForHostedVerification, } from './verify-hosted.js';
-const CELL_ID = /^cell_[0-9a-f]{32}$/;
-const CAPABILITY_ID = /^repository_driver_[0-9a-f]{64}$/;
-const COMPARISON_ID = /^driver_comparison_[0-9a-f]{64}$/;
-const RUN_ID = /^cv_[0-9a-f]{48}$/;
-const SHA256 = /^[0-9a-f]{64}$/;
-const SERIES_KEY = /^[A-Za-z0-9][A-Za-z0-9._:-]{7,63}$/;
-const TERMINAL_STATUSES = new Set([
-    'complete',
-    'errored',
-    'expired',
-    'terminated',
-]);
-const ASSESSMENT_STATUSES = new Set([
-    'improvement',
-    'inconclusive',
-    'regression',
-    'unchanged',
-]);
-const ASSESSMENT_REASON_CODES = new Set([
-    'case_universe_changed',
-    'coverage_increased',
-    'coverage_reduced',
-    'inconclusive',
-    'mixed_outcomes',
-    'new_failures',
-    'no_bound_driver',
-    'resolved_failures',
-    'unchanged',
-]);
-const OVERALL_ASSESSMENT_STATUSES = new Set([
-    'inconclusive',
-    'no_regressions_detected',
-    'regressions_detected',
-]);
-const DEFAULT_RUNS = 3;
-const MAX_RUNS = 10;
-function record(value, label) {
-    if (!value || typeof value !== 'object' || Array.isArray(value)) {
-        throw new Error(`${label} is malformed`);
+import { buildHostedJson, hostedPollingOptions, hostedResumeCommand, parseHostedRepository, readHostedIntentFile, startHostedVerification, waitForHostedVerification, } from './verify-hosted.js';
+const TERMINAL_STATUSES = new Set(['complete', 'errored', 'expired', 'terminated']);
+export async function verifyHostedReproducibilityCommand(repositoryArg, options) {
+    const repository = parseHostedRepository(repositoryArg);
+    if (!/^[0-9a-f]{40}$/.test(options.base ?? '') || !/^[0-9a-f]{40}$/.test(options.head ?? '')) {
+        throw new Error('--base and --head must be exact 40-character lowercase commit SHAs.');
     }
-    return value;
-}
-function array(value, label) {
-    if (!Array.isArray(value))
-        throw new Error(`${label} is malformed`);
-    return value;
-}
-function boundedInteger(value, fallback, label, minimum, maximum) {
-    if (value === undefined)
-        return fallback;
-    const parsed = Number(value);
-    if (!Number.isSafeInteger(parsed)
-        || parsed < minimum
-        || parsed > maximum) {
-        throw new Error(`${label} must be an integer from ${minimum} through ${maximum}.`);
+    const intent = options.intentFile === undefined ? undefined : readHostedIntentFile(options.intentFile);
+    const requestedRuns = options.runs === undefined ? 3 : Number(options.runs);
+    if (!Number.isInteger(requestedRuns) || requestedRuns < 2 || requestedRuns > 10) {
+        throw new Error('--runs must be an integer from 2 through 10.');
     }
-    return parsed;
-}
-function requireSeriesKey(value) {
-    const key = value
-        ?? `series:${new Date().toISOString().replace(/[^0-9]/g, '').slice(0, 14)}:${randomBytes(6).toString('hex')}`;
-    if (!SERIES_KEY.test(key)) {
+    const seriesKey = options.seriesKey ?? `series:${randomBytes(12).toString('hex')}`;
+    if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{7,63}$/.test(seriesKey)) {
         throw new Error('--series-key must be 8-64 characters and use only letters, numbers, ".", "_", ":", or "-".');
     }
-    return key;
-}
-export function hostedReproducibilityRunKey(seriesKey, repository, baseCommit, headCommit, ordinal, mode = 'exact_plan_replay') {
-    const digest = createHash('sha256')
-        .update(JSON.stringify(mode === 'exact_plan_replay'
-        ? [
-            seriesKey,
-            repository,
-            baseCommit,
-            headCommit,
-            ordinal,
-        ]
-        : [
-            mode,
-            seriesKey,
-            repository,
-            baseCommit,
-            headCommit,
-            ordinal,
-        ]))
-        .digest('hex');
-    return `repro:${digest}`;
-}
-function stable(value) {
-    if (Array.isArray(value))
-        return value.map(stable);
-    if (value && typeof value === 'object') {
-        const object = value;
-        return Object.fromEntries(Object.keys(object)
-            .sort()
-            .map(key => [key, stable(object[key])]));
-    }
-    return value;
-}
-function fingerprint(value) {
-    return createHash('sha256')
-        .update(JSON.stringify(stable(value)))
-        .digest('hex');
-}
-function expectedCellStatus(reasonCode) {
-    if (reasonCode === 'new_failures'
-        || reasonCode === 'coverage_reduced'
-        || reasonCode === 'mixed_outcomes') {
-        return 'regression';
-    }
-    if (reasonCode === 'resolved_failures'
-        || reasonCode === 'coverage_increased') {
-        return 'improvement';
-    }
-    if (reasonCode === 'unchanged')
-        return 'unchanged';
-    return 'inconclusive';
-}
-function expectedOverallStatus(outcomes) {
-    if (outcomes.some(outcome => outcome.status === 'regression')) {
-        return 'regressions_detected';
-    }
-    if (outcomes.length === 0
-        || outcomes.some(outcome => outcome.status === 'inconclusive')) {
-        return 'inconclusive';
-    }
-    return 'no_regressions_detected';
-}
-function requireCount(receipt, field, expected) {
-    if (receipt[field] !== expected) {
-        throw new Error(`driver assessment ${field} is inconsistent`);
-    }
-}
-function parseAssessment(state, identity) {
-    if (state.status !== 'complete' || !state.output) {
-        throw new Error(`run ended with status ${state.status}`);
-    }
-    if (state.repository !== identity.repository
-        || state.baseCommit !== identity.baseCommit
-        || state.headCommit !== identity.headCommit) {
-        throw new Error('run identity does not match the requested exact launch');
-    }
-    const output = record(state.output, 'terminal output');
-    if ((output.schemaVersion !== 8 && output.schemaVersion !== 9)
-        || output.status !== 'driver_assessment_recorded') {
-        throw new Error('run did not produce a schema-8 or schema-9 driver assessment');
-    }
-    const riskPlan = record(output.riskPlanReceipt, 'risk plan receipt');
-    const riskPlanSha256 = fingerprint(riskPlan);
-    let replaySourceRunId = null;
-    let replaySourceResultSchemaVersion = null;
-    if (output.schemaVersion === 9) {
-        if (output.planningCredentialDelivery !== 'not-required-plan-replay') {
-            throw new Error('schema-9 replay used an invalid planning credential mode');
-        }
-        const replay = record(output.riskPlanReplayReceipt, 'risk plan replay receipt');
-        const fields = Object.keys(replay);
-        if (fields.length !== 6
-            || fields.some(field => ![
-                'schemaVersion',
-                'sourceContentPersisted',
-                'sourceResultSchemaVersion',
-                'sourceRiskPlanReceiptSha256',
-                'sourceRunId',
-                'status',
-            ].includes(field))
-            || replay.schemaVersion !== 1
-            || replay.status !== 'risk_plan_replay_verified'
-            || typeof replay.sourceRunId !== 'string'
-            || !RUN_ID.test(replay.sourceRunId)
-            || (replay.sourceResultSchemaVersion !== 8
-                && replay.sourceResultSchemaVersion !== 9)
-            || replay.sourceRiskPlanReceiptSha256 !== riskPlanSha256
-            || typeof replay.sourceRiskPlanReceiptSha256 !== 'string'
-            || !SHA256.test(replay.sourceRiskPlanReceiptSha256)
-            || replay.sourceContentPersisted !== false) {
-            throw new Error('risk plan replay receipt is malformed');
-        }
-        replaySourceRunId = replay.sourceRunId;
-        replaySourceResultSchemaVersion = replay.sourceResultSchemaVersion;
-    }
-    else if (output.riskPlanReplayReceipt !== undefined) {
-        throw new Error('schema-8 result contains unversioned replay provenance');
-    }
-    else if (output.planningCredentialDelivery
-        !== 'per-command-env-gateway-placeholder') {
-        throw new Error('schema-8 result used an invalid planning credential mode');
-    }
-    const assessment = record(output.driverAssessmentReceipt, 'driver assessment receipt');
-    const planCells = array(riskPlan.cells, 'risk plan cells');
-    const assessmentCells = array(assessment.cells, 'assessment cells');
-    if (riskPlan.schemaVersion !== 2
-        || riskPlan.status !== 'risk_plan_ready'
-        || riskPlan.baseCommit !== identity.baseCommit
-        || riskPlan.headCommit !== identity.headCommit
-        || riskPlan.sourceContentPersisted !== false) {
-        throw new Error('risk plan receipt identity is malformed');
-    }
-    const planIdentities = planCells.map((raw) => {
-        const cell = record(raw, 'risk plan cell');
-        if (typeof cell.cellId !== 'string'
-            || !CELL_ID.test(cell.cellId)
-            || (cell.capabilityId !== null
-                && (typeof cell.capabilityId !== 'string'
-                    || !CAPABILITY_ID.test(cell.capabilityId)))) {
-            throw new Error('risk plan cell identity is malformed');
-        }
-        return {
-            cellId: cell.cellId,
-            capabilityId: cell.capabilityId,
-        };
-    }).sort((left, right) => left.cellId.localeCompare(right.cellId));
-    const cellIds = planIdentities.map(cell => cell.cellId);
-    if (cellIds.length === 0
-        || new Set(cellIds).size !== cellIds.length
-        || assessmentCells.length !== cellIds.length) {
-        throw new Error('risk plan and assessment cell sets are inconsistent');
-    }
-    if (assessment.schemaVersion !== 1
-        || typeof assessment.status !== 'string'
-        || !OVERALL_ASSESSMENT_STATUSES.has(assessment.status)
-        || assessment.baseCommit !== identity.baseCommit
-        || assessment.headCommit !== identity.headCommit
-        || typeof assessment.driverComparisonReceiptSha256 !== 'string'
-        || !SHA256.test(assessment.driverComparisonReceiptSha256)
-        || assessment.sourceContentPersisted !== false) {
-        throw new Error('driver assessment receipt identity is malformed');
-    }
-    const outcomes = assessmentCells.map((raw) => {
-        const cell = record(raw, 'assessment cell');
-        if (typeof cell.cellId !== 'string'
-            || !CELL_ID.test(cell.cellId)
-            || (cell.capabilityId !== null
-                && (typeof cell.capabilityId !== 'string'
-                    || !CAPABILITY_ID.test(cell.capabilityId)))
-            || (cell.comparisonId !== null
-                && (typeof cell.comparisonId !== 'string'
-                    || !COMPARISON_ID.test(cell.comparisonId)))
-            || typeof cell.status !== 'string'
-            || !ASSESSMENT_STATUSES.has(cell.status)
-            || typeof cell.reasonCode !== 'string'
-            || !ASSESSMENT_REASON_CODES.has(cell.reasonCode)
-            || cell.status !== expectedCellStatus(cell.reasonCode)
-            || (cell.capabilityId === null
-                ? (cell.comparisonId !== null
-                    || cell.reasonCode !== 'no_bound_driver')
-                : (cell.comparisonId === null
-                    || cell.reasonCode === 'no_bound_driver'))) {
-            throw new Error('assessment cell outcome is malformed');
-        }
-        return {
-            cellId: cell.cellId,
-            capabilityId: cell.capabilityId,
-            comparisonId: cell.comparisonId,
-            status: cell.status,
-            reasonCode: cell.reasonCode,
-        };
-    }).sort((left, right) => left.cellId.localeCompare(right.cellId));
-    if (new Set(outcomes.map(outcome => outcome.cellId)).size !== outcomes.length
-        || outcomes.some((outcome, index) => outcome.cellId !== cellIds[index]
-            || outcome.capabilityId !== planIdentities[index].capabilityId)) {
-        throw new Error('risk plan and assessment cell identities do not match');
-    }
-    const counts = {
-        comparedCellCount: outcomes.filter(outcome => outcome.status !== 'inconclusive').length,
-        regressionCellCount: outcomes.filter(outcome => outcome.status === 'regression').length,
-        improvementCellCount: outcomes.filter(outcome => outcome.status === 'improvement').length,
-        unchangedCellCount: outcomes.filter(outcome => outcome.status === 'unchanged').length,
-        inconclusiveCellCount: outcomes.filter(outcome => outcome.status === 'inconclusive').length,
-    };
-    requireCount(assessment, 'plannedCellCount', outcomes.length);
-    for (const [field, count] of Object.entries(counts)) {
-        requireCount(assessment, field, count);
-    }
-    if (assessment.status !== expectedOverallStatus(outcomes)) {
-        throw new Error('driver assessment overall status is inconsistent');
-    }
-    return {
-        schemaVersion: output.schemaVersion,
-        riskPlanSha256,
-        replaySourceRunId,
-        replaySourceResultSchemaVersion,
-        planIdentities,
-        cellIds,
-        status: assessment.status,
-        outcomes,
-    };
-}
-function groupedPlanVariants(parsed) {
-    const variants = new Map();
-    for (const item of parsed) {
-        const digest = fingerprint(item.assessment.planIdentities);
-        const existing = variants.get(digest);
-        if (existing) {
-            existing.run_ids.push(item.runId);
-        }
-        else {
-            variants.set(digest, {
-                fingerprint: digest,
-                run_ids: [item.runId],
-                cell_ids: item.assessment.cellIds,
-            });
-        }
-    }
-    return [...variants.values()].sort((left, right) => left.fingerprint.localeCompare(right.fingerprint));
-}
-function groupedOutcomeVariants(parsed) {
-    const variants = new Map();
-    for (const item of parsed) {
-        const digest = fingerprint({
-            status: item.assessment.status,
-            outcomes: item.assessment.outcomes,
-        });
-        const existing = variants.get(digest);
-        if (existing) {
-            existing.run_ids.push(item.runId);
-        }
-        else {
-            variants.set(digest, {
-                fingerprint: digest,
-                run_ids: [item.runId],
-                assessment_status: item.assessment.status,
-                outcomes: item.assessment.outcomes,
-            });
-        }
-    }
-    return [...variants.values()].sort((left, right) => left.fingerprint.localeCompare(right.fingerprint));
-}
-function driftIds(parsed, values) {
-    const byCell = new Map();
-    for (const [runIndex, item] of parsed.entries()) {
-        const present = values(item.assessment);
-        for (const [cellId, observed] of byCell) {
-            observed.add(present.has(cellId)
-                ? JSON.stringify(present.get(cellId))
-                : '__missing__');
-        }
-        for (const [cellId, value] of present) {
-            if (byCell.has(cellId))
-                continue;
-            const observed = new Set();
-            if (runIndex > 0)
-                observed.add('__missing__');
-            observed.add(JSON.stringify(value));
-            byCell.set(cellId, observed);
-        }
-    }
-    return [...byCell.entries()]
-        .filter(([, observed]) => observed.size > 1)
-        .map(([cellId]) => cellId)
-        .sort();
-}
-export function analyzeHostedReproducibility(runs, identity) {
-    const parsed = [];
-    const unrunnableRuns = [];
-    for (const run of runs) {
-        if (run.timedOut) {
-            unrunnableRuns.push({
-                run_id: run.state.runId,
-                status: run.state.status,
-                reason: 'wait_timed_out',
-            });
-            continue;
-        }
-        try {
-            parsed.push({
-                runId: run.state.runId,
-                assessment: parseAssessment(run.state, identity),
-            });
-        }
-        catch (error) {
-            unrunnableRuns.push({
-                run_id: run.state.runId,
-                status: run.state.status,
-                reason: error instanceof Error ? error.message : String(error),
-            });
-        }
-    }
-    const requestedRuns = identity.requestedRuns ?? runs.length;
-    const allRequestedRunsLaunched = runs.length === requestedRuns;
-    const planVariants = groupedPlanVariants(parsed);
-    const assessmentVariants = groupedOutcomeVariants(parsed);
-    const mode = identity.mode ?? 'exact_plan_replay';
-    const replaySourceRunId = mode === 'exact_plan_replay'
-        ? parsed[0]?.runId ?? null
-        : null;
-    const planReplayVerified = mode === 'exact_plan_replay'
-        && allRequestedRunsLaunched
-        && parsed.length === runs.length
-        && parsed.length > 1
-        && parsed[0]?.assessment.schemaVersion === 8
-        && parsed[0]?.assessment.replaySourceRunId === null
-        && parsed.slice(1).every(item => item.assessment.schemaVersion === 9
-            && item.assessment.replaySourceRunId === replaySourceRunId
-            && item.assessment.replaySourceResultSchemaVersion
-                === parsed[0].assessment.schemaVersion
-            && item.assessment.riskPlanSha256
-                === parsed[0].assessment.riskPlanSha256);
-    const plannerReproducibilityTested = mode === 'fresh_plans' && parsed.length > 1;
-    const plannerReproducible = plannerReproducibilityTested
-        && allRequestedRunsLaunched
-        && parsed.length === runs.length
-        && parsed.every(item => item.assessment.schemaVersion === 8
-            && item.assessment.replaySourceRunId === null)
-        && planVariants.length === 1;
-    const assessmentReproducible = (mode === 'exact_plan_replay'
-        ? planReplayVerified
-        : plannerReproducible)
-        && assessmentVariants.length === 1;
-    const planDriftCellIds = driftIds(parsed, assessment => new Map(assessment.planIdentities.map(plan => [
-        plan.cellId,
-        { capabilityId: plan.capabilityId },
-    ])));
-    const assessmentDriftCellIds = driftIds(parsed, assessment => new Map(assessment.outcomes.map(outcome => [
-        outcome.cellId,
-        {
-            capabilityId: outcome.capabilityId,
-            comparisonId: outcome.comparisonId,
-            status: outcome.status,
-            reasonCode: outcome.reasonCode,
-        },
-    ])));
-    const inconclusiveCellIds = [...new Set(parsed.flatMap(item => item.assessment.outcomes
-            .filter(outcome => outcome.status === 'inconclusive')
-            .map(outcome => outcome.cellId)))].sort();
-    return {
-        series_key: identity.seriesKey,
-        repository: identity.repository,
-        base_commit: identity.baseCommit,
-        head_commit: identity.headCommit,
-        reproducibility_mode: mode,
-        requested_runs: requestedRuns,
-        launched_runs: runs.length,
-        unstarted_runs: Math.max(0, requestedRuns - runs.length),
-        analyzable_runs: parsed.length,
-        run_ids: runs.map(run => run.state.runId),
-        replay_source_run_id: replaySourceRunId,
-        plan_replay_verified: planReplayVerified,
-        planner_reproducibility_tested: plannerReproducibilityTested,
-        planner_reproducible: plannerReproducible,
-        assessment_reproducible: assessmentReproducible,
-        reproducible: (mode === 'exact_plan_replay'
-            ? planReplayVerified
-            : plannerReproducible)
-            && assessmentReproducible
-            && inconclusiveCellIds.length === 0,
-        plan_variants: planVariants,
-        assessment_variants: assessmentVariants,
-        plan_drift_cell_ids: planDriftCellIds,
-        assessment_drift_cell_ids: assessmentDriftCellIds,
-        inconclusive_cell_ids: inconclusiveCellIds,
-        unrunnable_runs: unrunnableRuns,
-    };
-}
-export function isHostedReproducibilityReplaySource(run, identity) {
-    if (run.timedOut || run.state.status !== 'complete')
-        return false;
-    try {
-        const assessment = parseAssessment(run.state, identity);
-        return assessment.schemaVersion === 8
-            && assessment.replaySourceRunId === null;
-    }
-    catch {
-        return false;
-    }
-}
-function renderHostedReproducibility(report) {
-    const icon = report.reproducible ? chalk.green('✓') : chalk.yellow('!');
-    console.log(`${icon} Hosted reproducibility: ${report.reproducible ? 'reproducible' : 'not established'}`);
-    console.log(`  ${chalk.dim('Runs:')} ${report.analyzable_runs}/${report.requested_runs} analyzable; ${report.launched_runs} launched`);
-    console.log(`  ${chalk.dim('Mode:')} ${report.reproducibility_mode === 'fresh_plans'
-        ? 'independent fresh plans'
-        : 'exact plan replay'}`);
-    console.log(`  ${chalk.dim('Plan variants:')} ${report.plan_variants.length}`);
-    console.log(`  ${chalk.dim('Exact plan replay:')} ${report.reproducibility_mode === 'fresh_plans'
-        ? 'not requested'
-        : report.plan_replay_verified
-            ? 'verified'
-            : 'not verified'}`);
-    console.log(`  ${chalk.dim('Independent planner reproducibility:')} ${report.planner_reproducibility_tested
-        ? report.planner_reproducible
-            ? 'verified'
-            : 'not reproducible'
-        : 'not tested'}`);
-    console.log(`  ${chalk.dim('Assessment variants:')} ${report.assessment_variants.length}`);
-    if (report.plan_drift_cell_ids.length > 0) {
-        console.log(`  ${chalk.dim('Plan drift:')} ${report.plan_drift_cell_ids.length} stable cell id(s)`);
-    }
-    if (report.assessment_drift_cell_ids.length > 0) {
-        console.log(`  ${chalk.dim('Assessment drift:')} ${report.assessment_drift_cell_ids.length} cell(s)`);
-    }
-    if (report.inconclusive_cell_ids.length > 0) {
-        console.log(`  ${chalk.dim('Inconclusive:')} ${report.inconclusive_cell_ids.length} cell(s)`);
-    }
-    if (report.unrunnable_runs.length > 0) {
-        console.log(`  ${chalk.dim('Unrunnable:')} ${report.unrunnable_runs.length} run(s)`);
-    }
-}
-export async function verifyHostedReproducibilityCommand(repositoryArg, options) {
-    const repository = parseHostedRepository(repositoryArg);
-    // A reproducibility series reuses one explicit agent intent for every run.
-    readHostedIntentFile(options.intentFile);
-    const requestedRuns = boundedInteger(options.runs, DEFAULT_RUNS, '--runs', 2, MAX_RUNS);
-    const seriesKey = requireSeriesKey(options.seriesKey);
-    const mode = options.freshPlans
-        ? 'fresh_plans'
-        : 'exact_plan_replay';
+    const polling = hostedPollingOptions(options.interval, options.timeout);
     const auth = await resolveAuthContext({
         preferredLogin: options.account,
         owner: repository.owner,
         repo: repository.repository,
     });
+    const account = auth.token.startsWith('hsk_live_') ? undefined : auth.login;
     console.error(chalk.dim(`Hosted reproducibility series: ${seriesKey}`));
-    const runs = [];
+    const report = {
+        series_key: seriesKey,
+        repository: repository.fullName,
+        base_commit: options.base,
+        head_commit: options.head,
+        requested_runs: requestedRuns,
+        runs: [],
+        reproducible: false,
+        reason: 'Independent fleet plans do not share an execution identity, so reproducibility is not established.',
+    };
     for (let index = 0; index < requestedRuns; index += 1) {
-        if (!options.json) {
-            console.error(chalk.dim(`Starting hosted reproducibility run ${index + 1}/${requestedRuns}…`));
-        }
+        if (!options.json)
+            console.error(chalk.dim(`Starting fleet run ${index + 1}/${requestedRuns}…`));
+        const digest = createHash('sha256').update(JSON.stringify([
+            'haystack', seriesKey, repository.fullName, options.base, options.head, intent ?? null, index + 1,
+        ])).digest('hex');
         const started = await startHostedVerification(repositoryArg, {
+            provider: 'haystack',
             base: options.base,
             head: options.head,
             intentFile: options.intentFile,
-            idempotencyKey: hostedReproducibilityRunKey(seriesKey, repository.fullName, options.base, options.head, index + 1, mode),
-            replayPlanFrom: options.freshPlans || index === 0
-                ? undefined
-                : runs[0]?.state.runId,
+            idempotencyKey: `series:${digest}`,
             wait: false,
         }, auth.token);
-        let state = started.state;
-        let timedOut = false;
-        if (!TERMINAL_STATUSES.has(state.status)) {
-            const waited = await waitForHostedVerification(state.runId, auth.token, hostedPollingOptions(options.interval, options.timeout, current => {
-                if (!options.json) {
-                    console.error(chalk.dim(`Run ${index + 1}/${requestedRuns}: ${current.status}…`));
-                }
-            }));
-            state = waited.state;
-            timedOut = waited.timedOut;
-        }
-        runs.push({ state, timedOut });
-        if (!options.freshPlans
-            &&
-                index === 0
-            && !isHostedReproducibilityReplaySource(runs[0], {
-                seriesKey,
-                repository: repository.fullName,
-                baseCommit: options.base,
-                headCommit: options.head,
-            })) {
-            if (!options.json) {
-                console.error(chalk.yellow('Source run is not replayable; later runs were not launched.'));
-            }
+        const { state, timedOut } = TERMINAL_STATUSES.has(started.state.status)
+            ? { state: started.state, timedOut: false }
+            : await waitForHostedVerification(started.state.runId, auth.token, polling);
+        if (state.repository !== repository.fullName || state.baseCommit !== options.base || state.headCommit !== options.head) {
+            throw new Error('Hosted fleet run does not match the requested repository and exact commits.');
+        }
+        report.runs.push(buildHostedJson('status', state, {
+            account,
+            idempotencyKey: started.idempotencyKey,
+            timedOut,
+            nextCommand: TERMINAL_STATUSES.has(state.status) ? undefined : hostedResumeCommand(state.runId, account),
+        }));
+        if (!options.json)
+            console.log(`  ${state.runId}: ${state.status}${timedOut ? ' (wait timed out)' : ''}`);
+        if (timedOut)
             break;
-        }
     }
-    const report = analyzeHostedReproducibility(runs, {
-        seriesKey,
-        repository: repository.fullName,
-        baseCommit: options.base,
-        headCommit: options.head,
-        requestedRuns,
-        mode,
-    });
     if (options.json) {
         process.stdout.write(`${JSON.stringify(withSchema('cloud-verifier', {
             operation: 'reproducibility',
@@ -572,8 +74,8 @@ export async function verifyHostedReproducibilityCommand(repositoryArg, options)
         }), null, 2)}\n`);
     }
     else {
-        renderHostedReproducibility(report);
+        console.log(`Fleet runs recorded: ${report.runs.length}/${report.requested_runs}`);
+        console.log(report.reason);
     }
-    if (!report.reproducible)
-        process.exitCode = 2;
+    process.exitCode = 2;
 }