@haystackeditor/cli 0.15.21 → 0.15.22

package/dist/commands/verify.js

@@ -4,7 +4,51 @@ import { resolve } from 'node:path';
 import chalk from 'chalk';
 import { setVerifyToken, verifyAuthHeaders } from './verify-core.js';
 const DEFAULT_CONTROL_PLANE = 'http://127.0.0.1:3000';
-const SPECIMENS = new Set(['all', 'posthog-js-478', 'analytics-js-255']);
+export const VERIFY_SPECIMENS = new Set(['all', 'posthog-js-478', 'analytics-js-255']);
+export function parseStartRunResponse(raw) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        throw new Error('Verifier returned an invalid JSON response.');
+    }
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        throw new Error('Verifier returned an invalid response object.');
+    }
+    const response = parsed;
+    const requiredString = (field) => {
+        const value = response[field];
+        if (typeof value !== 'string' || value.trim() === '') {
+            throw new Error(`Verifier response is missing a valid ${field}.`);
+        }
+        return value;
+    };
+    return {
+        runId: requiredString('runId'),
+        status: requiredString('status'),
+        resultsUrl: requiredString('resultsUrl'),
+    };
+}
+export function verifierErrorReason(raw, contentType) {
+    if (!contentType?.toLowerCase().includes('application/json'))
+        return raw;
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        return `malformed JSON error response (${error instanceof Error ? error.message : String(error)}); raw body: ${raw}`;
+    }
+    if (parsed
+        && typeof parsed === 'object'
+        && !Array.isArray(parsed)
+        && typeof parsed.error === 'string'
+        && parsed.error.trim() !== '') {
+        return parsed.error;
+    }
+    return `JSON error response omitted a string error; raw body: ${raw}`;
+}
 function openResults(url) {
     const platform = process.platform;
     const command = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open';
@@ -24,18 +68,15 @@ async function readIntentTranscript(path) {
         throw new Error(`Could not read intent transcript at ${absolutePath}: ${message}`);
     }
 }
-export async function verifyCommand(specimenArg, options) {
+export async function startVerificationRun(specimenArg, options) {
     const specimen = specimenArg || 'all';
-    if (!SPECIMENS.has(specimen)) {
+    if (!VERIFY_SPECIMENS.has(specimen)) {
         throw new Error(`Unknown or unreproduced specimen "${specimen}". Choose all, posthog-js-478, or analytics-js-255.`);
     }
     setVerifyToken(options.token);
     const server = (options.server || process.env.HAYSTACK_VERIFY_SERVER || DEFAULT_CONTROL_PLANE)
         .replace(/\/+$/, '');
     const intentTranscript = await readIntentTranscript(options.intentTranscript);
-    if (!options.json) {
-        console.error(chalk.dim('Preparing the cloud verification run…'));
-    }
     let response;
     try {
         response = await fetch(`${server}/api/cloud-verifier/runs`, {
@@ -47,6 +88,7 @@ export async function verifyCommand(specimenArg, options) {
                 baseRef: options.base,
                 headRef: options.head,
                 intentTranscript,
+                planSourceRunId: options.planSourceRunId,
             }),
         });
     }
@@ -59,25 +101,16 @@ export async function verifyCommand(specimenArg, options) {
         throw new Error(`Control plane at ${server} requires a token (HTTP 401). Pass --token or set HAYSTACK_VERIFY_TOKEN.`);
     }
     if (!response.ok) {
-        let reason = raw;
-        try {
-            reason = JSON.parse(raw).error || raw;
-        }
-        catch {
-            // Preserve the response body when it is not JSON.
-        }
+        const reason = verifierErrorReason(raw, response.headers.get('content-type'));
         throw new Error(`Verifier rejected the run (${response.status}): ${reason}`);
     }
-    let started;
-    try {
-        started = JSON.parse(raw);
-    }
-    catch {
-        throw new Error('Verifier returned an invalid response.');
-    }
-    if (!started.runId || !started.resultsUrl) {
-        throw new Error('Verifier response is missing runId or resultsUrl.');
+    return parseStartRunResponse(raw);
+}
+export async function verifyCommand(specimenArg, options) {
+    if (!options.json) {
+        console.error(chalk.dim('Preparing the cloud verification run…'));
     }
+    const started = await startVerificationRun(specimenArg, options);
     if (options.json) {
         process.stdout.write(`${JSON.stringify({
             schema_version: '1.0.0',

package/dist/index.js

@@ -187,7 +187,9 @@ The local control plane must be running. From the Haystack repository:
   pnpm dev
 
 Examples:
+  haystack verify hosted start owner/repo --base <sha> --head <sha>
   haystack verify posthog-js-478         # start a run, then:
+  haystack verify reproducibility posthog-js-478 --runs 3
   haystack verify watch                  # block until it finishes (exit 1 = tests failed)
   haystack verify status                 # one-screen verdicts + live environment URLs
   haystack verify findings               # every problem found, with the readings that moved
@@ -203,6 +205,75 @@ Examples:
 `)
     .action((specimen, options) => runPublicCommand(() => verifyCommand(specimen, options), options.json));
 const verifyRunArg = 'Run id, chapter id (newest run of that chapter), or "latest" (default)';
+const hostedVerify = verify
+    .command('hosted')
+    .description('Run the production Cloud Verifier against exact GitHub commits');
+hostedVerify
+    .command('start')
+    .description('Start an authenticated production verification and wait for its terminal result')
+    .argument('<repository>', 'Exact GitHub owner/repository name')
+    // `verify` already owns --base/--head for the local specimen launcher.
+    // Re-declaring these as Commander requiredOption values makes the parent
+    // consume them and the child then reject a valid invocation. Runtime
+    // validation below accepts the inherited values and still fails closed.
+    .option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
+    .option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
+    .option('--idempotency-key <key>', 'Stable retry key (default: deterministic from repository and commits)')
+    .option('--account <login>', 'Use a specific saved Haystack account')
+    .option('--no-wait', 'Return after the production workflow is queued')
+    .option('--interval <seconds>', 'Polling interval while waiting (default 5)')
+    .option('--timeout <minutes>', 'Maximum wait (default 35)')
+    .option('--json', 'Versioned machine-readable run status and terminal output')
+    .addHelpText('after', `
+The default idempotency key makes an exact retry return the original run. Pass
+--idempotency-key with a new stable key only when you intentionally want a
+fresh production execution of the same commit pair.
+
+Examples:
+  haystack verify hosted start owner/repo --base <sha> --head <sha>
+  haystack verify hosted start owner/repo --base <sha> --head <sha> --no-wait --json
+`)
+    .action(async (repository, _options, cmd) => {
+    const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
+    const { verifyHostedStartCommand } = await import('./commands/verify-hosted.js');
+    return runPublicCommand(() => verifyHostedStartCommand(repository, options), options.json);
+});
+hostedVerify
+    .command('status')
+    .description('Read an owned production verification, optionally waiting until terminal')
+    .argument('<run-id>', 'Hosted run id (cv_ followed by 48 lowercase hex characters)')
+    .option('--account <login>', 'Use a specific saved Haystack account')
+    .option('--wait', 'Keep polling until the run reaches a terminal status')
+    .option('--interval <seconds>', 'Polling interval while waiting (default 5)')
+    .option('--timeout <minutes>', 'Maximum wait (default 35)')
+    .option('--json', 'Versioned machine-readable run status and terminal output')
+    .addHelpText('after', `
+Example:
+  haystack verify hosted status cv_<48 lowercase hex characters> --wait --json
+`)
+    .action(async (runId, _options, cmd) => {
+    const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
+    const { verifyHostedStatusCommand } = await import('./commands/verify-hosted.js');
+    return runPublicCommand(() => verifyHostedStatusCommand(runId, options), options.json);
+});
+verify
+    .command('reproducibility')
+    .description('Separate frozen-plan drift from executor/judgment nondeterminism')
+    .argument('<specimen>', 'Specimen id, e.g. posthog-js-478')
+    .option('--runs <n>', 'Number of sequential runs to compare (default 3; min 2, max 20)')
+    .option('--scope <scope>', 'executor replays one frozen plan (default); planner/full independently replan each run', 'executor')
+    .option('--plan-source <runId>', 'executor scope: replay this existing structured frozen plan for every run')
+    .option('--interval <seconds>', 'Polling interval for each run (default 3)')
+    .option('--timeout <minutes>', 'Per-run timeout (default: wait forever)')
+    .option('--server <url>', 'Verifier control-plane URL (default: http://127.0.0.1:3000)')
+    .option('--token <token>', 'Bearer token for a control plane that requires auth (or HAYSTACK_VERIFY_TOKEN)')
+    .option('--repo-root <path>', 'Repository holding .haystack/verify (default: walk up from cwd)')
+    .option('--json', 'Machine-readable stable/unstable cell report')
+    .action(async (specimen, commandOptions, cmd) => {
+    const options = { ...cmd.optsWithGlobals(), ...commandOptions };
+    const { verifyReproducibilityCommand } = await import('./commands/verify-reproducibility.js');
+    return runPublicCommand(() => verifyReproducibilityCommand(specimen, options), options.json);
+});
 verify
     .command('status')
     .description('One screen for a run: per-test verdicts, counts, live environments and expiry')

package/dist/schema.d.ts

@@ -19,6 +19,7 @@ export declare const SCHEMA_VERSIONS: {
     readonly traces: "1.0.0";
     readonly submit: "1.0.0";
     readonly action: "1.0.0";
+    readonly 'cloud-verifier': "1.0.0";
     readonly error: "1.0.0";
 };
 export type SchemaName = keyof typeof SCHEMA_VERSIONS;

package/dist/schema.js

@@ -19,6 +19,7 @@ export const SCHEMA_VERSIONS = {
     traces: '1.0.0',
     submit: '1.0.0',
     action: '1.0.0',
+    'cloud-verifier': '1.0.0',
     error: '1.0.0',
 };
 /** Wrap a payload with the `schema_version` envelope. */

package/dist/utils/haystack-api.d.ts

@@ -27,7 +27,7 @@ export declare class HaystackApiError extends Error {
  * class it failed with, and the next command to run.
  */
 export declare function classifyHttpError(response: Response, context: string): Promise<HaystackApiError>;
-export declare function haystackJson<T>(path: string, token: string, init?: RequestInit): Promise<T>;
+export declare function haystackJson<T>(path: string, token: string, init?: RequestInit, acceptedStatuses?: readonly number[]): Promise<T>;
 export declare function fetchCliIdentity(token: string): Promise<CliIdentity>;
 export declare function fetchHaystackInstallations(token: string): Promise<UserInstallation[]>;
 export declare function fetchGitHubJson<T>(path: string, token: string, init?: RequestInit): Promise<T>;

package/dist/utils/haystack-api.js

@@ -69,12 +69,12 @@ export async function classifyHttpError(response, context) {
     }
     return new HaystackApiError(response.status, `${context}: unexpected response (HTTP ${response.status})${suffix}.`);
 }
-export async function haystackJson(path, token, init = {}) {
+export async function haystackJson(path, token, init = {}, acceptedStatuses = []) {
     const response = await fetch(`${HAYSTACK_API_BASE}${path}`, {
         ...init,
         headers: headers(token, init.headers),
     });
-    if (!response.ok) {
+    if (!response.ok && !acceptedStatuses.includes(response.status)) {
         throw await classifyHttpError(response, `Haystack API ${path}`);
     }
     return response.json();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@haystackeditor/cli",
-  "version": "0.15.21",
+  "version": "0.15.22",
   "description": "Set up Haystack for your project — automated PR review, triage, and merge queue",
   "type": "module",
   "bin": {

package/schemas/cloud-verifier.v1.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://docs.haystackeditor.com/cli/schemas/cloud-verifier.v1.json",
+  "title": "Haystack hosted Cloud Verifier result",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "operation",
+    "run_id",
+    "status"
+  ],
+  "properties": {
+    "schema_version": {
+      "const": "1.0.0"
+    },
+    "operation": {
+      "enum": [
+        "start",
+        "status"
+      ]
+    },
+    "run_id": {
+      "type": "string",
+      "pattern": "^cv_[0-9a-f]{48}$"
+    },
+    "status": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9_-]{1,31}$"
+    },
+    "repository": {
+      "type": "string",
+      "pattern": "^[A-Za-z0-9_.-]{1,100}/[A-Za-z0-9_.-]{1,100}$"
+    },
+    "base_commit": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{40}$"
+    },
+    "head_commit": {
+      "type": "string",
+      "pattern": "^[0-9a-f]{40}$"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "completed_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "account": {
+      "type": "string",
+      "pattern": "^[A-Za-z0-9](?:[A-Za-z0-9]|-(?=[A-Za-z0-9])){0,38}$"
+    },
+    "idempotency_key": {
+      "type": "string",
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$"
+    },
+    "next_command": {
+      "type": "string"
+    },
+    "timed_out": {
+      "type": "boolean"
+    },
+    "output": {
+      "type": "object"
+    },
+    "error": {
+      "type": "string"
+    }
+  },
+  "allOf": [
+    {
+      "if": {
+        "properties": {
+          "status": {
+            "const": "complete"
+          }
+        },
+        "required": [
+          "status"
+        ]
+      },
+      "then": {
+        "required": [
+          "repository",
+          "base_commit",
+          "head_commit",
+          "created_at",
+          "completed_at",
+          "output"
+        ]
+      }
+    }
+  ]
+}