@haystackeditor/cli 0.15.21 → 0.15.23

package/dist/commands/verify-hosted.js

@@ -0,0 +1,342 @@
+import { createHash } from 'node:crypto';
+import chalk from 'chalk';
+import { withSchema } from '../schema.js';
+import { resolveAuthContext } from '../utils/auth.js';
+import { HaystackApiError, haystackJson } from '../utils/haystack-api.js';
+const REPOSITORY_COMPONENT = /^[A-Za-z0-9_.-]{1,100}$/;
+const COMMIT_SHA = /^[0-9a-f]{40}$/;
+const IDEMPOTENCY_KEY = /^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$/;
+const RUN_ID = /^cv_[0-9a-f]{48}$/;
+const STATUS = /^[a-z][a-z0-9_-]{1,31}$/;
+const TERMINAL_STATUSES = new Set(['complete', 'errored', 'expired', 'terminated']);
+const DEFAULT_INTERVAL_SECONDS = 5;
+const DEFAULT_TIMEOUT_MINUTES = 35;
+function requireRecord(value, context) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error(`${context} returned an invalid response object.`);
+    }
+    return value;
+}
+function requireString(value, field, pattern) {
+    const candidate = value[field];
+    if (typeof candidate !== 'string'
+        || candidate.length === 0
+        || (pattern && !pattern.test(candidate))) {
+        throw new Error(`Hosted verifier response is missing a valid ${field}.`);
+    }
+    return candidate;
+}
+function optionalString(value, field) {
+    const candidate = value[field];
+    if (candidate === undefined)
+        return undefined;
+    if (typeof candidate !== 'string' || candidate.length === 0) {
+        throw new Error(`Hosted verifier response has an invalid ${field}.`);
+    }
+    return candidate;
+}
+function optionalTimestamp(value, field) {
+    const candidate = optionalString(value, field);
+    if (candidate === undefined)
+        return undefined;
+    const parsed = new Date(candidate);
+    if (!Number.isFinite(parsed.getTime())
+        || parsed.toISOString() !== candidate) {
+        throw new Error(`Hosted verifier response has an invalid ${field}.`);
+    }
+    return candidate;
+}
+export function parseHostedRunState(value, expectedRunId) {
+    const response = requireRecord(value, 'Hosted verifier');
+    const runId = requireString(response, 'runId', RUN_ID);
+    if (expectedRunId && runId !== expectedRunId) {
+        throw new Error(`Hosted verifier returned run ${runId} while status for ${expectedRunId} was requested.`);
+    }
+    const status = requireString(response, 'status', STATUS);
+    const repository = optionalString(response, 'repository');
+    if (repository) {
+        const parts = repository.split('/');
+        if (parts.length !== 2
+            || !parts.every(part => REPOSITORY_COMPONENT.test(part))) {
+            throw new Error('Hosted verifier response has an invalid repository.');
+        }
+    }
+    const baseCommit = optionalString(response, 'baseCommit');
+    const headCommit = optionalString(response, 'headCommit');
+    if (baseCommit && !COMMIT_SHA.test(baseCommit)) {
+        throw new Error('Hosted verifier response has an invalid baseCommit.');
+    }
+    if (headCommit && !COMMIT_SHA.test(headCommit)) {
+        throw new Error('Hosted verifier response has an invalid headCommit.');
+    }
+    const outputValue = response.output;
+    const output = outputValue === undefined
+        ? undefined
+        : requireRecord(outputValue, 'Hosted verifier output');
+    const createdAt = optionalTimestamp(response, 'createdAt');
+    const completedAt = optionalTimestamp(response, 'completedAt');
+    if (status === 'complete') {
+        if (!repository
+            || !baseCommit
+            || !headCommit
+            || !createdAt
+            || !completedAt
+            || !output
+            || output.runId !== runId
+            || output.repository !== repository
+            || output.baseCommit !== baseCommit
+            || output.headCommit !== headCommit
+            || output.sourceOfflineReady !== true
+            || output.sandboxDeleted !== true) {
+            throw new Error('Hosted verifier returned a complete run without a matching terminal receipt.');
+        }
+    }
+    return {
+        runId,
+        status,
+        repository,
+        baseCommit,
+        headCommit,
+        createdAt,
+        completedAt,
+        output,
+        error: optionalString(response, 'error'),
+    };
+}
+export function parseHostedRepository(value) {
+    const parts = value.split('/');
+    if (parts.length !== 2
+        || !parts.every(part => REPOSITORY_COMPONENT.test(part))) {
+        throw new Error(`Invalid repository "${value}". Use an exact GitHub owner/repository name.`);
+    }
+    return {
+        owner: parts[0],
+        repository: parts[1],
+        fullName: value,
+    };
+}
+function requireCommit(value, flag) {
+    if (typeof value !== 'string' || !COMMIT_SHA.test(value)) {
+        throw new Error(`${flag} must be an exact 40-character lowercase commit SHA.`);
+    }
+    return value;
+}
+export function defaultHostedIdempotencyKey(repository, baseCommit, headCommit) {
+    const digest = createHash('sha256')
+        .update(JSON.stringify([repository, baseCommit, headCommit]))
+        .digest('hex');
+    return `cli:${digest}`;
+}
+function requireIdempotencyKey(value) {
+    if (!IDEMPOTENCY_KEY.test(value)) {
+        throw new Error('--idempotency-key must be 8-128 characters and use only letters, numbers, ".", "_", ":", or "-".');
+    }
+    return value;
+}
+function parsePositiveNumber(value, fallback, flag, maximum) {
+    if (value === undefined)
+        return fallback;
+    const parsed = Number(value);
+    if (!Number.isFinite(parsed) || parsed <= 0 || parsed > maximum) {
+        throw new Error(`${flag} must be a positive number no greater than ${maximum}.`);
+    }
+    return parsed;
+}
+function pollingOptions(interval, timeout, onStatus) {
+    return {
+        intervalMs: parsePositiveNumber(interval, DEFAULT_INTERVAL_SECONDS, '--interval', 300) * 1000,
+        timeoutMs: parsePositiveNumber(timeout, DEFAULT_TIMEOUT_MINUTES, '--timeout', 1_440) * 60_000,
+        onStatus,
+    };
+}
+export async function startHostedVerification(repositoryArg, options, token) {
+    const repository = parseHostedRepository(repositoryArg);
+    const baseCommit = requireCommit(options.base, '--base');
+    const headCommit = requireCommit(options.head, '--head');
+    const idempotencyKey = requireIdempotencyKey(options.idempotencyKey
+        ?? defaultHostedIdempotencyKey(repository.fullName, baseCommit, headCommit));
+    const response = await haystackJson('/api/agent/cloud-verifier/runs', token, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            owner: repository.owner,
+            repository: repository.repository,
+            baseCommit,
+            headCommit,
+            idempotencyKey,
+        }),
+    });
+    const state = parseHostedRunState(response);
+    return {
+        state: {
+            ...state,
+            repository: state.repository ?? repository.fullName,
+            baseCommit: state.baseCommit ?? baseCommit,
+            headCommit: state.headCommit ?? headCommit,
+        },
+        idempotencyKey,
+    };
+}
+export async function getHostedVerification(runId, token) {
+    if (!RUN_ID.test(runId)) {
+        throw new Error('Run id must match cv_ followed by 48 lowercase hex characters.');
+    }
+    let response;
+    try {
+        response = await haystackJson(`/api/agent/cloud-verifier/runs/${encodeURIComponent(runId)}`, token, {}, [410]);
+    }
+    catch (error) {
+        if (error instanceof HaystackApiError && error.status === 404) {
+            throw new Error(`Hosted verifier run ${runId} was not found for the selected account. `
+                + 'If the run was started with another saved account, retry with `--account <login>`.');
+        }
+        throw error;
+    }
+    return parseHostedRunState(response, runId);
+}
+function delay(milliseconds) {
+    return new Promise(resolve => setTimeout(resolve, milliseconds));
+}
+export async function waitForHostedVerification(runId, token, options) {
+    const startedAt = Date.now();
+    let lastStatus;
+    while (true) {
+        const state = await getHostedVerification(runId, token);
+        if (state.status !== lastStatus) {
+            options.onStatus?.(state);
+            lastStatus = state.status;
+        }
+        if (TERMINAL_STATUSES.has(state.status)) {
+            return { state, timedOut: false };
+        }
+        const elapsed = Date.now() - startedAt;
+        if (elapsed >= options.timeoutMs) {
+            return { state, timedOut: true };
+        }
+        await delay(Math.min(options.intervalMs, options.timeoutMs - elapsed));
+    }
+}
+export function buildHostedJson(operation, state, extras = {}) {
+    return withSchema('cloud-verifier', {
+        operation,
+        run_id: state.runId,
+        status: state.status,
+        ...(state.repository ? { repository: state.repository } : {}),
+        ...(state.baseCommit ? { base_commit: state.baseCommit } : {}),
+        ...(state.headCommit ? { head_commit: state.headCommit } : {}),
+        ...(state.createdAt ? { created_at: state.createdAt } : {}),
+        ...(state.completedAt ? { completed_at: state.completedAt } : {}),
+        ...(extras.account ? { account: extras.account } : {}),
+        ...(extras.idempotencyKey
+            ? { idempotency_key: extras.idempotencyKey }
+            : {}),
+        ...(extras.nextCommand ? { next_command: extras.nextCommand } : {}),
+        ...(extras.timedOut !== undefined ? { timed_out: extras.timedOut } : {}),
+        ...(state.output ? { output: state.output } : {}),
+        ...(state.error ? { error: state.error } : {}),
+    });
+}
+export function hostedResumeCommand(runId, account) {
+    return `haystack verify hosted status ${runId}${account ? ` --account ${account}` : ''} --wait`;
+}
+function printHostedState(operation, state, options) {
+    const nextCommand = TERMINAL_STATUSES.has(state.status)
+        ? undefined
+        : hostedResumeCommand(state.runId, options.account);
+    if (options.json) {
+        process.stdout.write(`${JSON.stringify(buildHostedJson(operation, state, {
+            account: options.account,
+            idempotencyKey: options.idempotencyKey,
+            nextCommand,
+            timedOut: options.timedOut,
+        }), null, 2)}\n`);
+        return;
+    }
+    const icon = state.status === 'complete'
+        ? chalk.green('✓')
+        : TERMINAL_STATUSES.has(state.status)
+            ? chalk.red('✗')
+            : chalk.cyan('•');
+    console.log(`${icon} Hosted verification ${chalk.bold(state.runId)}: ${state.status}`);
+    if (state.repository) {
+        console.log(`  ${chalk.dim('Repository:')} ${state.repository}`);
+    }
+    if (state.baseCommit && state.headCommit) {
+        console.log(`  ${chalk.dim('Commits:')} ${state.baseCommit.slice(0, 12)} → ${state.headCommit.slice(0, 12)}`);
+    }
+    if (state.error)
+        console.log(`  ${chalk.dim('Error:')} ${state.error}`);
+    if (options.timedOut) {
+        console.log(chalk.yellow('  Wait timed out; the production run is still active.'));
+    }
+    if (nextCommand)
+        console.log(`  ${chalk.dim('Resume:')} ${nextCommand}`);
+}
+function markNonSuccess(state, timedOut = false) {
+    if (timedOut
+        || (state.status !== 'complete' && TERMINAL_STATUSES.has(state.status))) {
+        process.exitCode = 2;
+    }
+}
+export async function verifyHostedStartCommand(repositoryArg, options) {
+    const repository = parseHostedRepository(repositoryArg);
+    const auth = await resolveAuthContext({
+        preferredLogin: options.account,
+        owner: repository.owner,
+        repo: repository.repository,
+    });
+    if (!options.json) {
+        console.error(chalk.dim('Submitting the exact commit pair to the hosted verifier…'));
+    }
+    const started = await startHostedVerification(repositoryArg, options, auth.token);
+    const selectedAccount = auth.token.startsWith('hsk_live_')
+        ? undefined
+        : auth.login;
+    let state = started.state;
+    let timedOut;
+    if (options.wait !== false && !TERMINAL_STATUSES.has(state.status)) {
+        const waited = await waitForHostedVerification(state.runId, auth.token, pollingOptions(options.interval, options.timeout, current => {
+            if (!options.json) {
+                console.error(chalk.dim(`Hosted verifier: ${current.status}…`));
+            }
+        }));
+        state = waited.state;
+        timedOut = waited.timedOut;
+    }
+    else if (options.wait !== false) {
+        timedOut = false;
+    }
+    printHostedState('start', state, {
+        json: options.json,
+        account: selectedAccount,
+        idempotencyKey: started.idempotencyKey,
+        timedOut,
+    });
+    markNonSuccess(state, timedOut);
+}
+export async function verifyHostedStatusCommand(runId, options) {
+    const auth = await resolveAuthContext({ preferredLogin: options.account });
+    const selectedAccount = auth.token.startsWith('hsk_live_')
+        ? undefined
+        : auth.login;
+    let state = await getHostedVerification(runId, auth.token);
+    let timedOut;
+    if (options.wait && !TERMINAL_STATUSES.has(state.status)) {
+        const waited = await waitForHostedVerification(runId, auth.token, pollingOptions(options.interval, options.timeout, current => {
+            if (!options.json) {
+                console.error(chalk.dim(`Hosted verifier: ${current.status}…`));
+            }
+        }));
+        state = waited.state;
+        timedOut = waited.timedOut;
+    }
+    else if (options.wait) {
+        timedOut = false;
+    }
+    printHostedState('status', state, {
+        json: options.json,
+        account: selectedAccount,
+        timedOut,
+    });
+    markNonSuccess(state, timedOut);
+}

package/dist/commands/verify-mcp.js

@@ -99,6 +99,19 @@ export const VERIFY_TOOLS = [
             required: ['cell'],
         },
     },
+    {
+        name: 'verify_universe',
+        description: 'Get or open a live universe for one test: returns the running environment when one is still live, otherwise reopens the retained state onto fresh compute (reusing an existing attachment when present).',
+        inputSchema: {
+            type: 'object',
+            properties: {
+                run: runInput,
+                cell: { type: 'string', description: 'Stable test selector: ordinal (1-based), riskCellId, or exact planner cellKey.' },
+                side: { type: 'string', enum: ['base', 'head'], description: 'Matched universe side (default head).' },
+            },
+            required: ['cell'],
+        },
+    },
     {
         name: 'verify_cleanup',
         description: 'Read retained-state cleanup for one test universe by its exact fork ID: shared references, provider deletion attempts, and the durable cleanup-SLO verdict.',
@@ -292,6 +305,52 @@ export async function runVerifyMcpServer(options) {
                     return ok(match.cell);
                 return ok(cellDetail(loaded.run, match));
             }
+            if (name === 'verify_universe') {
+                if (typeof input.cell !== 'string')
+                    return fail('cell required');
+                if (input.side !== undefined
+                    && input.side !== 'base'
+                    && input.side !== 'head') {
+                    return fail('side must be base or head');
+                }
+                const loaded = await loadRunAuto({ server, repoRoot, selector });
+                const match = findCellByStableSelector(loaded.run, input.cell);
+                const side = input.side === 'base' ? 'base' : 'head';
+                const universe = side === 'base' ? match.cell.base : match.cell.head;
+                const expired = universe.expiresAt
+                    ? Date.parse(universe.expiresAt) < Date.now()
+                    : false;
+                if (universe.browserUrl && universe.status === 'running' && !expired) {
+                    return ok({
+                        schema_version: '1.0.0',
+                        run_id: loaded.run.runId,
+                        ordinal: match.cell.ordinal,
+                        side,
+                        source: 'running',
+                        url: universe.browserUrl,
+                        expires_at: universe.expiresAt ?? null,
+                    });
+                }
+                if (!universe.forkId) {
+                    return fail(`Test ${match.cell.ordinal}'s ${side} universe is not running and `
+                        + 'has no durable fork ID to reopen.');
+                }
+                const { requestUniverseReopen } = await import('./verify-ops.js');
+                const reopened = await requestUniverseReopen(server, universe.forkId);
+                return ok({
+                    schema_version: '1.0.0',
+                    run_id: loaded.run.runId,
+                    ordinal: match.cell.ordinal,
+                    side,
+                    source: 'reopened',
+                    url: reopened.browserUrl,
+                    expires_at: reopened.expiresAt,
+                    fork_id: reopened.forkId,
+                    sandbox_id: reopened.sandboxId,
+                    state_expires_at: reopened.stateExpiresAt,
+                    reused: reopened.reused,
+                });
+            }
             if (name === 'verify_cleanup') {
                 if (typeof input.cell !== 'string')
                     return fail('cell required');

package/dist/commands/verify-ops.js

@@ -26,7 +26,7 @@ function renderStatus(summary) {
         : summary.status === 'complete' ? chalk.green(summary.status)
             : summary.status === 'failed' ? chalk.red(summary.status)
                 : chalk.cyan(summary.status);
-    console.log(`${chalk.bold(summary.run_id)} — ${state}${summary.age ? chalk.dim(` (updated ${summary.age} ago)`) : ''}`);
+    console.log(`${chalk.bold(summary.run_id)} — ${state}${summary.age ? chalk.dim(` (started ${summary.age} ago)`) : ''}`);
     for (const chapter of summary.chapters) {
         console.log(`${chalk.dim(chapter.repository)} ${chapter.pull_request} — ${chapter.label}`);
         if (chapter.execution_mode === 'bisect' && chapter.bisect) {

package/dist/commands/verify-reproducibility.d.ts

@@ -0,0 +1,85 @@
+import { type RiskCell, type VerificationRun } from "./verify-core.js";
+import { type VerifyOptions } from "./verify.js";
+export type ReproducibilityScope = "executor" | "planner" | "full";
+export interface CellOutcomeObservation {
+    run_id: string;
+    status: string;
+    verdict: string | null;
+}
+export interface CellPlanObservation {
+    run_id: string;
+    fingerprint: string;
+}
+export interface ReproducibilityCell {
+    cell_id: string;
+    chapter_id: string;
+    title: string;
+    outcomes: CellOutcomeObservation[];
+    plans: CellPlanObservation[];
+    missing_run_ids: string[];
+}
+export interface BisectOutcomeObservation {
+    run_id: string;
+    status: string;
+    culprit_sha: string | null;
+    parent_sha: string | null;
+    incident: {
+        commit: string;
+        status: string;
+    } | null;
+    probes: Array<{
+        commit: string;
+        status: string;
+    }>;
+}
+export interface ReproducibilityBisect {
+    chapter_id: string;
+    label: string;
+    outcomes: BisectOutcomeObservation[];
+    missing_run_ids: string[];
+}
+export interface ReproducibilityReport {
+    schema_version: "1.0.0";
+    scope: ReproducibilityScope;
+    requested_runs: number;
+    completed_runs: number;
+    run_ids: string[];
+    plan_source_run_id: string | null;
+    compared_cells: number;
+    stable_cells: number;
+    compared_bisects: number;
+    stable_bisects: number;
+    plan_drift_cells: ReproducibilityCell[];
+    outcome_drift_cells: ReproducibilityCell[];
+    unrunnable_cells: ReproducibilityCell[];
+    bisect_drift_chapters: ReproducibilityBisect[];
+    unrunnable_bisects: ReproducibilityBisect[];
+    planner_incomplete_run_ids: string[];
+    executor_incomplete_run_ids: string[];
+    incomplete_run_ids: string[];
+    timed_out_run_ids: string[];
+    planner_reproducible: boolean;
+    executor_reproducible: boolean;
+    reproducible: boolean;
+}
+export declare function cellPlanFingerprint(cell: RiskCell): string;
+/**
+ * Plan identity and runtime outcomes are independent axes:
+ *
+ * - missing IDs or changed frozen contracts are planner drift;
+ * - status/judgment changes under an identical contract are executor drift.
+ *
+ * Titles and other prose are fingerprinted to detect a changed judge input,
+ * but cells are correlated only by their exact stable IDs.
+ */
+export declare function analyzeReproducibility(runs: VerificationRun[], requestedRuns?: number, timedOutRunIds?: string[], scope?: ReproducibilityScope): ReproducibilityReport;
+export interface VerifyReproducibilityOptions extends VerifyOptions {
+    runs?: string;
+    interval?: string;
+    timeout?: string;
+    repoRoot?: string;
+    scope?: string;
+    planSource?: string;
+}
+export declare function runHasFrozenRiskPlan(run: VerificationRun): boolean;
+export declare function verifyReproducibilityCommand(specimen: string, options: VerifyReproducibilityOptions): Promise<void>;