@haystackeditor/cli 0.15.29 → 0.15.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/assets/telemetry/browser-runtime.js +148 -2
  2. package/dist/commands/design-verify.d.ts +2 -0
  3. package/dist/commands/design-verify.js +27 -1
  4. package/dist/commands/precompute-delivery-contract.d.ts +68 -0
  5. package/dist/commands/precompute-delivery-contract.js +1 -0
  6. package/dist/commands/precompute-delivery-worker.d.ts +1 -0
  7. package/dist/commands/precompute-delivery-worker.js +68 -0
  8. package/dist/commands/precompute-delivery.d.ts +20 -0
  9. package/dist/commands/precompute-delivery.js +319 -0
  10. package/dist/commands/verify-explore.d.ts +14 -0
  11. package/dist/commands/verify-explore.js +180 -0
  12. package/dist/commands/verify-history.d.ts +14 -0
  13. package/dist/commands/verify-history.js +152 -0
  14. package/dist/commands/verify-hosted.d.ts +12 -2
  15. package/dist/commands/verify-hosted.js +305 -50
  16. package/dist/commands/verify-precompute.d.ts +12 -1
  17. package/dist/commands/verify-precompute.js +135 -27
  18. package/dist/commands/verify.d.ts +6 -0
  19. package/dist/commands/verify.js +22 -0
  20. package/dist/index.js +71 -5
  21. package/dist/types/verify-history.d.ts +38 -0
  22. package/dist/types/verify-history.js +1 -0
  23. package/dist/utils/design-verifier-api.d.ts +8 -0
  24. package/dist/utils/design-verifier-api.js +23 -0
  25. package/dist/utils/design-verifier-history.d.ts +46 -0
  26. package/dist/utils/design-verifier-history.js +79 -0
  27. package/dist/utils/design-verifier-result.d.ts +17 -0
  28. package/dist/utils/design-verifier-result.js +80 -1
  29. package/dist/utils/git.js +5 -1
  30. package/dist/utils/telemetry.js +8 -2
  31. package/package.json +1 -1
  32. package/schemas/cloud-verifier.v1.json +108 -45
@@ -1,6 +1,6 @@
1
1
  import { spawn, execFileSync } from 'node:child_process';
2
- import { createHash } from 'node:crypto';
3
- import { cpSync, existsSync, mkdtempSync, rmSync } from 'node:fs';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
+ import { closeSync, constants, cpSync, existsSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, renameSync, rmSync, writeFileSync } from 'node:fs';
4
4
  import { createGzip } from 'node:zlib';
5
5
  import { isAbsolute, join, resolve } from 'node:path';
6
6
  import { tmpdir } from 'node:os';
@@ -9,11 +9,21 @@ import { NotLoggedInError, resolveAuthContext } from '../utils/auth.js';
9
9
  import { findGitRoot, parseRemoteUrl, resolveDiffBaseRef, } from '../utils/git.js';
10
10
  import { HaystackApiError, haystackJson } from '../utils/haystack-api.js';
11
11
  import { trackPrecomputeEvent } from '../utils/telemetry.js';
12
- const TOTAL_WALL_MS = 5_000;
12
+ import { startPrecomputeDelivery } from './precompute-delivery.js';
13
+ // --hook has five seconds to capture the exact source and publish local custody.
14
+ // Its detached sender owns the bounded network delivery and acknowledgement.
15
+ // An explicit invocation still waits synchronously for the server response.
16
+ const HOOK_WALL_MS = 5_000;
17
+ const EXPLICIT_WALL_MS = 60_000;
13
18
  const REMOTE_COMMAND_MS = 750;
14
- const COMMIT_SHA = /^[0-9a-f]{40}$/;
15
- const REPOSITORY_COMPONENT = /^[A-Za-z0-9_.-]{1,100}$/;
16
- const RUN_ID = /^cv_[0-9a-f]{48}$/;
19
+ // The published CLI cannot import verifier/contracts outside its package root.
20
+ const hasOnlyCharacters = (value, allowed) => value.length > 0
21
+ && [...value].every(character => allowed.includes(character));
22
+ const isCommitSha = (value) => value.length === 40 && hasOnlyCharacters(value, '0123456789abcdef');
23
+ const isRepositoryComponent = (value) => value.length <= 100
24
+ && hasOnlyCharacters(value, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-');
25
+ const isRunId = (value) => value.startsWith('cv_') && value.length === 51
26
+ && hasOnlyCharacters(value.slice(3), '0123456789abcdef');
17
27
  const FIXED_COMMIT_ENV = {
18
28
  GIT_AUTHOR_NAME: 'haystack-stop-hook',
19
29
  GIT_AUTHOR_EMAIL: 'stop-hook@haystack.invalid',
@@ -36,7 +46,7 @@ function remainingMs(deadline) {
36
46
  return remaining;
37
47
  }
38
48
  function gitOutput(args, gitRoot, deadline, options = {}) {
39
- const timeout = Math.max(1, Math.min(remainingMs(deadline), options.timeoutCapMs ?? TOTAL_WALL_MS));
49
+ const timeout = Math.max(1, Math.min(remainingMs(deadline), options.timeoutCapMs ?? HOOK_WALL_MS));
40
50
  try {
41
51
  return execFileSync('git', args, {
42
52
  cwd: gitRoot,
@@ -53,8 +63,8 @@ function gitOutput(args, gitRoot, deadline, options = {}) {
53
63
  function resolveOriginRepository(gitRoot) {
54
64
  try {
55
65
  const remote = parseRemoteUrl('origin');
56
- if (!REPOSITORY_COMPONENT.test(remote.owner)
57
- || !REPOSITORY_COMPONENT.test(remote.repo)) {
66
+ if (!isRepositoryComponent(remote.owner)
67
+ || !isRepositoryComponent(remote.repo)) {
58
68
  throw new Error('invalid repository identity');
59
69
  }
60
70
  return { owner: remote.owner, repository: remote.repo };
@@ -74,15 +84,19 @@ function resolveOriginDefaultBranch(gitRoot, deadline) {
74
84
  catch {
75
85
  // A clone does not have to carry origin/HEAD; ask the remote below.
76
86
  }
77
- const remoteSummary = gitOutput(['remote', 'show', 'origin'], gitRoot, deadline, {
87
+ const remoteRefs = gitOutput(['ls-remote', '--symref', 'origin', 'HEAD'], gitRoot, deadline, {
78
88
  env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
79
89
  timeoutCapMs: REMOTE_COMMAND_MS,
80
90
  });
81
- const match = remoteSummary.match(/^\s*HEAD branch:\s*(\S+)\s*$/m);
82
- if (!match) {
91
+ const prefix = 'ref: refs/heads/';
92
+ const heads = remoteRefs.split('\n').map(line => line.split('\t'))
93
+ .filter(fields => fields.length === 2 && fields[1] === 'HEAD' && fields[0].startsWith(prefix))
94
+ .map(fields => fields[0].slice(prefix.length));
95
+ if (heads.length !== 1 || heads[0].length === 0) {
83
96
  throw new PrecomputeFailure('Could not resolve the origin default branch.');
84
97
  }
85
- return match[1];
98
+ gitOutput(['check-ref-format', `refs/heads/${heads[0]}`], gitRoot, deadline);
99
+ return heads[0];
86
100
  }
87
101
  function deriveBaseSha(gitRoot, deadline) {
88
102
  const defaultBranch = resolveOriginDefaultBranch(gitRoot, deadline);
@@ -91,7 +105,7 @@ function deriveBaseSha(gitRoot, deadline) {
91
105
  timeoutMs: Math.max(1, Math.min(remainingMs(deadline), REMOTE_COMMAND_MS)),
92
106
  });
93
107
  const baseSha = gitOutput(['merge-base', 'HEAD', baseRef], gitRoot, deadline);
94
- if (!COMMIT_SHA.test(baseSha)) {
108
+ if (!isCommitSha(baseSha)) {
95
109
  throw new PrecomputeFailure('Could not derive the base commit.');
96
110
  }
97
111
  return baseSha;
@@ -115,7 +129,7 @@ function deriveWorktreeTreeSha(gitRoot, deadline) {
115
129
  }
116
130
  gitOutput(['add', '-A'], gitRoot, deadline, { env: indexEnv });
117
131
  const treeSha = gitOutput(['write-tree'], gitRoot, deadline, { env: indexEnv });
118
- if (!COMMIT_SHA.test(treeSha)) {
132
+ if (!isCommitSha(treeSha)) {
119
133
  throw new PrecomputeFailure('Could not derive the working-tree snapshot.');
120
134
  }
121
135
  return treeSha;
@@ -127,14 +141,14 @@ function deriveWorktreeTreeSha(gitRoot, deadline) {
127
141
  function deriveWorkCommit(gitRoot, baseSha, treeSha, deadline) {
128
142
  const headSha = gitOutput(['rev-parse', 'HEAD'], gitRoot, deadline);
129
143
  const headTreeSha = gitOutput(['rev-parse', 'HEAD^{tree}'], gitRoot, deadline);
130
- if (!COMMIT_SHA.test(headSha) || !COMMIT_SHA.test(headTreeSha)) {
144
+ if (!isCommitSha(headSha) || !isCommitSha(headTreeSha)) {
131
145
  throw new PrecomputeFailure('Could not resolve the current commit.');
132
146
  }
133
147
  // A clean checkout of the base already has the exact required identity.
134
148
  if (baseSha === headSha && treeSha === headTreeSha)
135
149
  return headSha;
136
150
  const workCommit = gitOutput(['commit-tree', treeSha, '-p', baseSha, '-m', 'front-half-snapshot'], gitRoot, deadline, { env: { ...process.env, ...FIXED_COMMIT_ENV } });
137
- if (!COMMIT_SHA.test(workCommit)) {
151
+ if (!isCommitSha(workCommit)) {
138
152
  throw new PrecomputeFailure('Could not derive the working commit.');
139
153
  }
140
154
  return workCommit;
@@ -202,7 +216,10 @@ async function buildCompressedPatch(gitRoot, baseSha, workCommit, deadline) {
202
216
  child.stdout.pipe(gzip);
203
217
  });
204
218
  }
205
- export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() + TOTAL_WALL_MS) {
219
+ export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() + HOOK_WALL_MS) {
220
+ // Capture this before git and compression work so delivery latency cannot
221
+ // make an older stop look newer than a later one.
222
+ const observedAt = Date.now();
206
223
  const baseSha = deriveBaseSha(gitRoot, deadline);
207
224
  const treeSha = deriveWorktreeTreeSha(gitRoot, deadline);
208
225
  const workCommit = deriveWorkCommit(gitRoot, baseSha, treeSha, deadline);
@@ -221,6 +238,7 @@ export async function derivePrecomputeRequest(gitRoot, owner, repository, deadli
221
238
  repository,
222
239
  baseCommit: baseSha,
223
240
  workCommit,
241
+ observedAt,
224
242
  cacheKey,
225
243
  patchGzBase64: patchGzip.toString('base64'),
226
244
  patchSha256: createHash('sha256').update(patchGzip).digest('hex'),
@@ -231,16 +249,75 @@ function precomputeResult(response) {
231
249
  if (!response || typeof response !== 'object' || Array.isArray(response)) {
232
250
  throw new PrecomputeFailure('The precompute service returned an invalid response.');
233
251
  }
234
- const { status, runId } = response;
235
- if (status === 'queued' && typeof runId === 'string' && RUN_ID.test(runId)) {
236
- return { outcome: 'enqueued', runId };
252
+ const { status, runId, walker: rawWalker } = response;
253
+ const walker = rawWalker === undefined ? undefined : readWalkerStatus(rawWalker);
254
+ if (status === 'queued' && typeof runId === 'string' && isRunId(runId)) {
255
+ return { outcome: 'enqueued', runId, ...(walker === undefined ? {} : { walker }) };
237
256
  }
238
257
  if (status === 'cached')
239
- return { outcome: 'cached' };
258
+ return { outcome: 'cached', ...(walker === undefined ? {} : { walker }) };
240
259
  if (status === 'existing')
241
- return { outcome: 'existing' };
260
+ return { outcome: 'existing', ...(walker === undefined ? {} : { walker }) };
242
261
  throw new PrecomputeFailure('The precompute service returned an invalid response.');
243
262
  }
263
+ /** One authenticated submission of the already captured request. No source reads or retries. */
264
+ export const deliverPrecomputeRequest = async (request, signal) => {
265
+ signal.throwIfAborted();
266
+ const auth = await resolveAuthContext({ owner: request.owner, repo: request.repository });
267
+ signal.throwIfAborted();
268
+ return precomputeResult(await haystackJson('/api/agent/cloud-verifier/front-half', auth.token, { method: 'POST', headers: { 'Content-Type': 'application/json' },
269
+ body: JSON.stringify(request), signal }));
270
+ };
271
+ function readWalkerStatus(value) {
272
+ if (value && typeof value === 'object' && !Array.isArray(value)) {
273
+ const record = value;
274
+ if (Object.keys(record).length === 2) {
275
+ if ((record.status === 'queued' || record.status === 'existing')
276
+ && typeof record.runId === 'string' && isRunId(record.runId))
277
+ return { status: record.status, runId: record.runId };
278
+ if (record.status === 'unavailable'
279
+ && (record.reason === 'runner-unavailable' || record.reason === 'enqueue-failed'))
280
+ return { status: record.status, reason: record.reason };
281
+ }
282
+ }
283
+ throw new PrecomputeFailure('The precompute service returned an invalid walker status.');
284
+ }
285
+ function walkerMetadataDirectory(gitRoot, deadline) {
286
+ const common = gitOutput(['rev-parse', '--git-common-dir'], gitRoot, deadline);
287
+ return join(isAbsolute(common) ? common : resolve(gitRoot, common), 'haystack');
288
+ }
289
+ function saveWalkerStatus(directory, request, walker) {
290
+ // Frozen local snapshot shape: {observedAt, repository, sourceCommit, walker}.
291
+ // Git's common metadata directory is shared by linked worktrees and excluded
292
+ // from the synthetic source tree. Rename publishes one complete JSON record.
293
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
294
+ if (!lstatSync(directory).isDirectory())
295
+ throw new PrecomputeFailure('Could not save the walker status.');
296
+ const temporary = join(directory, `.walker-status-${randomUUID()}.tmp`);
297
+ const descriptor = openSync(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
298
+ try {
299
+ try {
300
+ writeFileSync(descriptor, JSON.stringify({ observedAt: new Date(request.observedAt).toISOString(),
301
+ repository: `${request.owner}/${request.repository}`.toLowerCase(), sourceCommit: request.workCommit,
302
+ walker: readWalkerStatus(walker) }) + '\n');
303
+ fsyncSync(descriptor);
304
+ }
305
+ finally {
306
+ closeSync(descriptor);
307
+ }
308
+ renameSync(temporary, join(directory, 'walker-status.json'));
309
+ const parent = openSync(directory, constants.O_RDONLY);
310
+ try {
311
+ fsyncSync(parent);
312
+ }
313
+ finally {
314
+ closeSync(parent);
315
+ }
316
+ }
317
+ finally {
318
+ rmSync(temporary, { force: true });
319
+ }
320
+ }
244
321
  function normalModeFailure(error) {
245
322
  if (error instanceof PrecomputeFailure)
246
323
  return error;
@@ -256,9 +333,14 @@ function normalModeFailure(error) {
256
333
  }
257
334
  return new PrecomputeFailure(`The precompute request failed (HTTP ${error.status}).`);
258
335
  }
336
+ // The deadline fired while the POST was in flight. The server may still
337
+ // have accepted the job, so say what happened rather than "failed".
338
+ if (error instanceof Error && error.name === 'AbortError') {
339
+ return new PrecomputeFailure('Precompute timed out.');
340
+ }
259
341
  return new PrecomputeFailure('The precompute request failed.');
260
342
  }
261
- function printResult(outcome, runId) {
343
+ function printResult(outcome, runId, walker) {
262
344
  if (outcome === 'enqueued') {
263
345
  console.log(chalk.green(`Precompute request queued${runId ? ` (${runId})` : ''}.`));
264
346
  }
@@ -268,18 +350,36 @@ function printResult(outcome, runId) {
268
350
  else if (outcome === 'existing') {
269
351
  console.log(chalk.green('A matching precompute request already exists.'));
270
352
  }
353
+ if (walker?.status === 'queued')
354
+ console.log(`Walker queued (${walker.runId}).`);
355
+ else if (walker?.status === 'existing')
356
+ console.log(`A matching walker run already exists (${walker.runId}).`);
357
+ else if (walker?.status === 'unavailable')
358
+ console.log(walker.reason === 'runner-unavailable'
359
+ ? 'Walker unavailable: no runner is available for this repository.'
360
+ : 'Walker unavailable: the enqueue request failed.');
361
+ else
362
+ console.log('Walker status was not reported by the service.');
271
363
  }
272
364
  export async function verifyPrecomputeCommand(options) {
273
365
  const startedAt = Date.now();
274
- const deadline = startedAt + TOTAL_WALL_MS;
366
+ const totalWallMs = options.hook ? HOOK_WALL_MS : EXPLICIT_WALL_MS;
367
+ const deadline = startedAt + totalWallMs;
275
368
  let outcome = 'error';
276
369
  try {
277
370
  const gitRoot = findGitRoot();
278
371
  if (!gitRoot) {
279
372
  throw new PrecomputeFailure('Run this command from a git repository.');
280
373
  }
374
+ const metadataDirectory = walkerMetadataDirectory(gitRoot, deadline);
281
375
  const repository = resolveOriginRepository(gitRoot);
282
376
  const derivation = await derivePrecomputeRequest(gitRoot, repository.owner, repository.repository, deadline);
377
+ if (options.hook) {
378
+ remainingMs(deadline);
379
+ await startPrecomputeDelivery(metadataDirectory, derivation.request);
380
+ outcome = 'pending';
381
+ return;
382
+ }
283
383
  let token;
284
384
  try {
285
385
  const auth = await resolveAuthContext({
@@ -312,8 +412,16 @@ export async function verifyPrecomputeCommand(options) {
312
412
  }
313
413
  const result = precomputeResult(response);
314
414
  outcome = result.outcome;
415
+ if (result.walker) {
416
+ try {
417
+ saveWalkerStatus(metadataDirectory, derivation.request, result.walker);
418
+ }
419
+ catch {
420
+ throw new PrecomputeFailure('Precompute response received, but its walker status could not be saved.');
421
+ }
422
+ }
315
423
  if (!options.hook)
316
- printResult(outcome, result.runId);
424
+ printResult(outcome, result.runId, result.walker);
317
425
  }
318
426
  catch (error) {
319
427
  if (!options.hook)
@@ -321,7 +429,7 @@ export async function verifyPrecomputeCommand(options) {
321
429
  }
322
430
  finally {
323
431
  const wallMs = Date.now() - startedAt;
324
- trackPrecomputeEvent(outcome, wallMs, Math.max(1, TOTAL_WALL_MS - wallMs));
432
+ trackPrecomputeEvent(outcome, wallMs, options.hook ? 1 : Math.max(1, totalWallMs - wallMs));
325
433
  if (options.hook)
326
434
  process.exitCode = 0;
327
435
  }
@@ -6,6 +6,12 @@ export interface VerifyOptions {
6
6
  base?: string;
7
7
  head?: string;
8
8
  intentTranscript?: string;
9
+ intentFile?: string;
10
+ idempotencyKey?: string;
11
+ account?: string;
12
+ wait?: boolean;
13
+ interval?: string;
14
+ timeout?: string;
9
15
  json?: boolean;
10
16
  open?: boolean;
11
17
  /** Internal reproducibility option: execute an already-frozen plan. */
@@ -107,6 +107,28 @@ export async function startVerificationRun(specimenArg, options) {
107
107
  return parseStartRunResponse(raw);
108
108
  }
109
109
  export async function verifyCommand(specimenArg, options) {
110
+ if (specimenArg === undefined && (options.repo !== undefined || options.base !== undefined || options.head !== undefined)) {
111
+ if (!options.repo || !options.base || !options.head) {
112
+ throw new Error('Repository verification requires --repo owner/repo, --base <ref>, and --head <ref>.');
113
+ }
114
+ if (options.intentTranscript !== undefined) {
115
+ throw new Error('Repository verification accepts optional --intent-file JSON with problem, goal, and intended_outcomes.');
116
+ }
117
+ const { verifyHostedStartCommand } = await import('./verify-hosted.js');
118
+ await verifyHostedStartCommand(options.repo, {
119
+ provider: 'haystack',
120
+ base: options.base,
121
+ head: options.head,
122
+ intentFile: options.intentFile,
123
+ idempotencyKey: options.idempotencyKey,
124
+ account: options.account,
125
+ wait: options.wait,
126
+ interval: options.interval,
127
+ timeout: options.timeout,
128
+ json: options.json,
129
+ });
130
+ return;
131
+ }
110
132
  if (!options.json) {
111
133
  console.error(chalk.dim('Preparing the cloud verification run…'));
112
134
  }
package/dist/index.js CHANGED
@@ -174,24 +174,35 @@ const verify = program
174
174
  .command('verify')
175
175
  .description('Verify a change in matched cloud universes, or bisect a historical regression')
176
176
  .argument('[specimen]', 'Launch specimen to run (default: all)')
177
- .option('--repo <path>', 'Repository to analyze (defaults to the selected launch specimen)')
177
+ .option('--repo <owner/repo>', 'GitHub repository to verify through the authenticated hosted service')
178
178
  .option('--base <ref>', 'Base revision for a repository run')
179
179
  .option('--head <ref>', 'Head revision for a repository run')
180
+ .option('--intent-file <path>', 'Optional intent JSON with problem, goal, and intended_outcomes')
181
+ .option('--idempotency-key <key>', 'Stable retry key (default: derived from repository, refs, and intent)')
182
+ .option('--account <login>', 'Use a specific saved Haystack account')
183
+ .addOption(new Option('--no-wait', 'Return after the hosted run is queued').default(undefined))
184
+ .option('--interval <seconds>', 'Hosted polling interval (default 5)')
185
+ .option('--timeout <minutes>', 'Maximum hosted wait (default 35)')
180
186
  .option('--intent-transcript <file>', 'Coding-agent transcript used as intent evidence')
181
187
  .option('--server <url>', 'Verifier control-plane URL (default: http://127.0.0.1:3000)')
182
188
  .option('--token <token>', 'Bearer token for a control plane that requires auth (or HAYSTACK_VERIFY_TOKEN)')
183
189
  .option('--no-open', 'Do not open the live results page')
184
- .option('--json', 'Machine-readable run id, status, and results URL')
190
+ .option('--json', 'Machine-readable run status and results')
185
191
  .addHelpText('after', `
186
192
  Launch specimens:
187
193
  posthog-js-478 Replay PostHog/posthog-js PR #478 in matched browser environments (latent-issue mode)
188
194
  analytics-js-255 Identity routing regression with sandbox bisect
189
195
  all Run each reproduced specimen as an isolated chapter (default)
190
196
 
191
- The local control plane must be running. From the Haystack repository:
197
+ Repository verification uses the authenticated hosted fleet and resolves refs
198
+ once. Intent is optional. Hosted status also resumes these runs.
199
+
200
+ For launch specimens, run the local control plane from the Haystack repository:
192
201
  pnpm dev
193
202
 
194
203
  Examples:
204
+ haystack verify --repo owner/repo --base main --head feature --json
205
+ haystack verify history owner/repo --limit 20
195
206
  haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path>
196
207
  haystack verify posthog-js-478 # start a run, then:
197
208
  haystack verify reproducibility posthog-js-478 --runs 3
@@ -210,6 +221,26 @@ Examples:
210
221
  `)
211
222
  .action((specimen, options) => runPublicCommand(() => verifyCommand(specimen, options), options.json));
212
223
  const verifyRunArg = 'Run id, chapter id (newest run of that chapter), or "latest" (default)';
224
+ verify
225
+ .command('history')
226
+ .description('Read your recent finished hosted verification runs without starting a run')
227
+ .argument('<repository>', 'Exact GitHub owner/repository name')
228
+ .option('--limit <runs>', 'Number of recent finished runs to show, from 1 to 100 (default 20)')
229
+ .option('--account <login>', 'Use a specific saved Haystack account')
230
+ .option('--json', 'Versioned history with execution health and separate bug counts')
231
+ .addHelpText('after', `
232
+ History covers your hosted fleet runs for this repository. Reliability counts
233
+ only these returned runs; it does not measure whether the code is bug-free.
234
+
235
+ Examples:
236
+ haystack verify history owner/repo
237
+ haystack verify history owner/repo --limit 50 --account my-login --json
238
+ `)
239
+ .action(async (repository, _options, cmd) => {
240
+ const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
241
+ const { verifyHistoryCommand } = await import('./commands/verify-history.js');
242
+ return runPublicCommand(() => verifyHistoryCommand(repository, options), options.json);
243
+ });
213
244
  verify
214
245
  .command('precompute')
215
246
  .description('Precompute reusable verification inputs for the current working tree')
@@ -235,7 +266,7 @@ hostedVerify
235
266
  // validation below accepts the inherited values and still fails closed.
236
267
  .option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
237
268
  .option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
238
- .requiredOption('--intent-file <path>', 'Agent-supplied JSON with exactly problem, goal, and intended_outcomes')
269
+ .option('--intent-file <path>', 'Required for this alias: JSON with exactly problem, goal, and intended_outcomes')
239
270
  .option('--idempotency-key <key>', 'Stable retry key (default: deterministic from repository and commits)')
240
271
  .option('--replay-plan-from <run-id>', 'Reuse the exact validated risk plan from a completed owned run')
241
272
  .option('--account <login>', 'Use a specific saved Haystack account')
@@ -284,7 +315,7 @@ hostedVerify
284
315
  .argument('<repository>', 'Exact GitHub owner/repository name')
285
316
  .option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
286
317
  .option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
287
- .requiredOption('--intent-file <path>', 'Agent-supplied JSON reused for every hosted run in the series')
318
+ .option('--intent-file <path>', 'Required: agent-supplied JSON reused for every hosted run in the series')
288
319
  .option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
289
320
  .option('--series-key <key>', 'Stable series key for resumable idempotent retries')
290
321
  .option('--fresh-plans', 'Run the planner independently for every run instead of replaying the first plan')
@@ -365,6 +396,34 @@ verify
365
396
  const { verifyStatusCommand } = await import('./commands/verify-ops.js');
366
397
  return runPublicCommand(() => verifyStatusCommand(run, options), options.json);
367
398
  });
399
+ verify
400
+ .command('cases <run>')
401
+ .description('List cloud verification cases available to explore for 24 hours after completion')
402
+ .option('--repo <owner/repo>', 'Required exact GitHub repository; uses your Haystack login')
403
+ .option('--json', 'Machine-readable case IDs and expiry times')
404
+ .action(async (run, _options, cmd) => {
405
+ const options = cmd.optsWithGlobals();
406
+ const { verifyCasesCommand } = await import('./commands/verify-explore.js');
407
+ return runPublicCommand(() => verifyCasesCommand(run, options), options.json);
408
+ });
409
+ verify
410
+ .command('explore <run> <case-id>')
411
+ .description('Open or use your own copy of a saved fleet case; never changes the verification original')
412
+ .option('--repo <owner/repo>', 'Required exact GitHub repository; uses your Haystack login')
413
+ .option('--side <side>', 'base or head (default head)')
414
+ .option('--exec-json <argv>', 'Command arguments as a JSON array, executed inside your working sandbox')
415
+ .option('--cwd <path>', 'Absolute guest directory (default /home/user/pj)')
416
+ .option('--command-timeout <seconds>', 'Command timeout, 1–300 seconds (default 60)')
417
+ .option('--close', 'Discard your working copy; leave the saved case unchanged')
418
+ .option('--action <id>', 'Read/follow an existing action without submitting another command')
419
+ .option('--request-id <uuid>', 'Idempotency ID for a new request; reuse only with identical arguments')
420
+ .option('--no-wait', 'Return after submitting the action')
421
+ .option('--json', 'Machine-readable action and command output')
422
+ .action(async (run, caseId, _options, cmd) => {
423
+ const options = cmd.optsWithGlobals();
424
+ const { verifyExploreCommand } = await import('./commands/verify-explore.js');
425
+ return runPublicCommand(() => verifyExploreCommand(run, caseId, options), options.json);
426
+ });
368
427
  verify
369
428
  .command('watch')
370
429
  .description('Follow a run until it finishes; exit 0 all passed, 1 tests failed, 2 run/infra broke')
@@ -863,6 +922,8 @@ program
863
922
  // keeps retired auto-fix/verification wording out of the CLI surface).
864
923
  .description('Run a blind design-by-execution review of a PR')
865
924
  .option('--json', 'Machine-readable output')
925
+ .option('--history', 'Show recent execution reliability for this repository without starting a run')
926
+ .option('--limit <attempts>', 'Maximum recorded terminal attempts to show with --history (1–100, default 20)')
866
927
  .option('--no-wait', 'Return after the run is admitted instead of waiting for its verdict')
867
928
  .option('--run <run-id>', 'Resume and wait for an existing exact run id')
868
929
  .option('--poll-interval <seconds>', 'Status polling interval', '3')
@@ -883,6 +944,11 @@ outside the registered intent.
883
944
  The command waits for a behavioral verdict by default. Use --no-wait to
884
945
  return the run id immediately, or --run <run-id> to resume waiting.
885
946
 
947
+ Read recent execution reliability across this repository's PRs:
948
+ haystack design-verify owner/repo#123 --history --limit 20
949
+ History covers recorded terminal attempts, including partial or failed runs;
950
+ it does not measure bug-detection accuracy or whether changes are bug-free.
951
+
886
952
  After a completed run, inspect or replay one retained case universe without
887
953
  receiving provider credentials:
888
954
  haystack design-verify owner/repo#123 --run RUN --case case-00001 --universe before --replay
@@ -0,0 +1,38 @@
1
+ /** Read-only history from the active hosted verifier, scoped to the caller's
2
+ * stable GitHub user and repository IDs. Counts describe the returned window,
3
+ * never bug correctness. Missing results remain unknown. */
4
+ export type VerifyHistoryHealth = 'healthy' | 'degraded' | 'infrastructure-error' | 'not-executed' | 'unknown';
5
+ export interface VerifyHistoryRun {
6
+ runId: string;
7
+ baseCommit: string;
8
+ headCommit: string;
9
+ createdAt: string;
10
+ completedAt: string;
11
+ status: 'complete' | 'errored';
12
+ lifecycle: 'complete' | 'failed' | 'cancelled' | 'budget-stopped' | 'account-action' | null;
13
+ health: VerifyHistoryHealth;
14
+ regressionCount: number | null;
15
+ executedCaseCount: number | null;
16
+ degradationCodes: string[];
17
+ supersededBy: string | null;
18
+ }
19
+ export interface VerifyHistoryReport {
20
+ version: 'dv-verify-history-v1';
21
+ provider: 'haystack';
22
+ scope: 'user';
23
+ repository: {
24
+ owner: string;
25
+ name: string;
26
+ repositoryId: string;
27
+ };
28
+ limit: number;
29
+ hasMore: boolean;
30
+ runs: VerifyHistoryRun[];
31
+ summary: {
32
+ total: number;
33
+ counts: Record<VerifyHistoryHealth, number>;
34
+ /** Healthy/degraded runs with executedCaseCount > 0, divided by total;
35
+ * null for an empty window. */
36
+ usableRate: number | null;
37
+ };
38
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -7,6 +7,7 @@
7
7
  * the server-side shared secret. This client just POSTs and interprets the
8
8
  * response.
9
9
  */
10
+ import { type DesignVerifierHistoryReport } from './design-verifier-history.js';
10
11
  export type DesignVerifierTriggerResult = {
11
12
  status: 'started' | 'queued' | 'duplicate';
12
13
  runId: string;
@@ -184,6 +185,13 @@ export declare function waitForDesignVerifierUniverseAction(owner: string, repo:
184
185
  }>;
185
186
  export declare function triggerDesignVerifier(owner: string, repo: string, prNumber: number, token: string): Promise<DesignVerifierTriggerResult>;
186
187
  export declare function getDesignVerifierStatus(owner: string, repo: string, prNumber: number, runId: string, token: string): Promise<DesignVerifierStatusFetchResult>;
188
+ export declare function getDesignVerifierHistory(owner: string, repo: string, token: string, limit?: number): Promise<{
189
+ status: 'ok';
190
+ history: DesignVerifierHistoryReport;
191
+ } | {
192
+ status: 'denied' | 'unavailable' | 'error';
193
+ message: string;
194
+ }>;
187
195
  export declare function waitForDesignVerifier(owner: string, repo: string, prNumber: number, runId: string, token: string, options?: {
188
196
  intervalMs?: number;
189
197
  timeoutMs?: number;
@@ -8,6 +8,7 @@
8
8
  * response.
9
9
  */
10
10
  import { haystackApiUrl } from './haystack-api.js';
11
+ import { isDesignVerifierHistoryReport } from './design-verifier-history.js';
11
12
  function isDuration(value) {
12
13
  return value === null || (typeof value === 'number' && Number.isFinite(value) && value >= 0);
13
14
  }
@@ -239,6 +240,28 @@ export async function getDesignVerifierStatus(owner, repo, prNumber, runId, toke
239
240
  return { status: 'unavailable', message };
240
241
  return { status: 'error', message };
241
242
  }
243
+ export async function getDesignVerifierHistory(owner, repo, token, limit = 20) {
244
+ const url = haystackApiUrl(`/api/design-verifier/history/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}?limit=${limit}`);
245
+ try {
246
+ const response = await fetch(url, { method: 'GET', headers: apiHeaders(token) });
247
+ const data = await response.json();
248
+ if (response.ok) {
249
+ if (!isDesignVerifierHistoryReport(data, { owner, repo, limit })) {
250
+ return { status: 'error', message: 'History endpoint returned an invalid or mismatched result.' };
251
+ }
252
+ return { status: 'ok', history: data };
253
+ }
254
+ const message = data?.message || data?.error || `HTTP ${response.status}`;
255
+ if (response.status === 401 || response.status === 403)
256
+ return { status: 'denied', message };
257
+ if ([404, 502, 503].includes(response.status))
258
+ return { status: 'unavailable', message };
259
+ return { status: 'error', message };
260
+ }
261
+ catch (error) {
262
+ return { status: 'error', message: error.message };
263
+ }
264
+ }
242
265
  export async function waitForDesignVerifier(owner, repo, prNumber, runId, token, options = {}) {
243
266
  const intervalMs = options.intervalMs ?? 3000;
244
267
  const timeoutMs = options.timeoutMs ?? 2 * 60 * 60 * 1000;
@@ -0,0 +1,46 @@
1
+ /** FIXED CONTRACT: historical execution health, never a score for bug-detection accuracy. */
2
+ export type DesignVerifierHistoryOutcome = 'healthy' | 'degraded' | 'invalid' | 'failed' | 'unknown';
3
+ export interface DesignVerifierHistoryRun {
4
+ runId: string;
5
+ pr: number;
6
+ baseSha: string | null;
7
+ headSha: string | null;
8
+ createdAt: string;
9
+ finishedAt: string | null;
10
+ status: string;
11
+ outcome: DesignVerifierHistoryOutcome;
12
+ admitted: number | null;
13
+ losses: Array<{
14
+ code: string;
15
+ owner: string;
16
+ impact: 'degraded' | 'invalid';
17
+ count: number;
18
+ }>;
19
+ }
20
+ export interface DesignVerifierHistoryReport {
21
+ version: 'dv-cli-history-v1';
22
+ owner: string;
23
+ repo: string;
24
+ retentionDays: number;
25
+ limit: number;
26
+ runs: DesignVerifierHistoryRun[];
27
+ summary: {
28
+ total: number;
29
+ healthy: number;
30
+ degraded: number;
31
+ invalid: number;
32
+ failed: number;
33
+ unknown: number;
34
+ usableRate: number | null;
35
+ topBlockers: Array<{
36
+ code: string;
37
+ affectedRuns: number;
38
+ }>;
39
+ };
40
+ }
41
+ export declare function isDesignVerifierHistoryReport(value: unknown, expected: {
42
+ owner: string;
43
+ repo: string;
44
+ limit: number;
45
+ }): value is DesignVerifierHistoryReport;
46
+ export declare function formatDesignVerifierHistory(report: DesignVerifierHistoryReport): string;