@haystackeditor/cli 0.15.29 → 0.15.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/telemetry/browser-runtime.js +148 -2
- package/dist/commands/design-verify.d.ts +2 -0
- package/dist/commands/design-verify.js +27 -1
- package/dist/commands/precompute-delivery-contract.d.ts +68 -0
- package/dist/commands/precompute-delivery-contract.js +1 -0
- package/dist/commands/precompute-delivery-worker.d.ts +1 -0
- package/dist/commands/precompute-delivery-worker.js +68 -0
- package/dist/commands/precompute-delivery.d.ts +20 -0
- package/dist/commands/precompute-delivery.js +319 -0
- package/dist/commands/verify-explore.d.ts +14 -0
- package/dist/commands/verify-explore.js +180 -0
- package/dist/commands/verify-history.d.ts +14 -0
- package/dist/commands/verify-history.js +152 -0
- package/dist/commands/verify-hosted.d.ts +12 -2
- package/dist/commands/verify-hosted.js +305 -50
- package/dist/commands/verify-precompute.d.ts +12 -1
- package/dist/commands/verify-precompute.js +135 -27
- package/dist/commands/verify.d.ts +6 -0
- package/dist/commands/verify.js +22 -0
- package/dist/index.js +71 -5
- package/dist/types/verify-history.d.ts +38 -0
- package/dist/types/verify-history.js +1 -0
- package/dist/utils/design-verifier-api.d.ts +8 -0
- package/dist/utils/design-verifier-api.js +23 -0
- package/dist/utils/design-verifier-history.d.ts +46 -0
- package/dist/utils/design-verifier-history.js +79 -0
- package/dist/utils/design-verifier-result.d.ts +17 -0
- package/dist/utils/design-verifier-result.js +80 -1
- package/dist/utils/git.js +5 -1
- package/dist/utils/telemetry.js +8 -2
- package/package.json +1 -1
- package/schemas/cloud-verifier.v1.json +108 -45
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn, execFileSync } from 'node:child_process';
|
|
2
|
-
import { createHash } from 'node:crypto';
|
|
3
|
-
import { cpSync, existsSync, mkdtempSync, rmSync } from 'node:fs';
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { closeSync, constants, cpSync, existsSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { createGzip } from 'node:zlib';
|
|
5
5
|
import { isAbsolute, join, resolve } from 'node:path';
|
|
6
6
|
import { tmpdir } from 'node:os';
|
|
@@ -9,11 +9,21 @@ import { NotLoggedInError, resolveAuthContext } from '../utils/auth.js';
|
|
|
9
9
|
import { findGitRoot, parseRemoteUrl, resolveDiffBaseRef, } from '../utils/git.js';
|
|
10
10
|
import { HaystackApiError, haystackJson } from '../utils/haystack-api.js';
|
|
11
11
|
import { trackPrecomputeEvent } from '../utils/telemetry.js';
|
|
12
|
-
|
|
12
|
+
import { startPrecomputeDelivery } from './precompute-delivery.js';
|
|
13
|
+
// --hook has five seconds to capture the exact source and publish local custody.
|
|
14
|
+
// Its detached sender owns the bounded network delivery and acknowledgement.
|
|
15
|
+
// An explicit invocation still waits synchronously for the server response.
|
|
16
|
+
const HOOK_WALL_MS = 5_000;
|
|
17
|
+
const EXPLICIT_WALL_MS = 60_000;
|
|
13
18
|
const REMOTE_COMMAND_MS = 750;
|
|
14
|
-
|
|
15
|
-
const
|
|
16
|
-
|
|
19
|
+
// The published CLI cannot import verifier/contracts outside its package root.
|
|
20
|
+
const hasOnlyCharacters = (value, allowed) => value.length > 0
|
|
21
|
+
&& [...value].every(character => allowed.includes(character));
|
|
22
|
+
const isCommitSha = (value) => value.length === 40 && hasOnlyCharacters(value, '0123456789abcdef');
|
|
23
|
+
const isRepositoryComponent = (value) => value.length <= 100
|
|
24
|
+
&& hasOnlyCharacters(value, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-');
|
|
25
|
+
const isRunId = (value) => value.startsWith('cv_') && value.length === 51
|
|
26
|
+
&& hasOnlyCharacters(value.slice(3), '0123456789abcdef');
|
|
17
27
|
const FIXED_COMMIT_ENV = {
|
|
18
28
|
GIT_AUTHOR_NAME: 'haystack-stop-hook',
|
|
19
29
|
GIT_AUTHOR_EMAIL: 'stop-hook@haystack.invalid',
|
|
@@ -36,7 +46,7 @@ function remainingMs(deadline) {
|
|
|
36
46
|
return remaining;
|
|
37
47
|
}
|
|
38
48
|
function gitOutput(args, gitRoot, deadline, options = {}) {
|
|
39
|
-
const timeout = Math.max(1, Math.min(remainingMs(deadline), options.timeoutCapMs ??
|
|
49
|
+
const timeout = Math.max(1, Math.min(remainingMs(deadline), options.timeoutCapMs ?? HOOK_WALL_MS));
|
|
40
50
|
try {
|
|
41
51
|
return execFileSync('git', args, {
|
|
42
52
|
cwd: gitRoot,
|
|
@@ -53,8 +63,8 @@ function gitOutput(args, gitRoot, deadline, options = {}) {
|
|
|
53
63
|
function resolveOriginRepository(gitRoot) {
|
|
54
64
|
try {
|
|
55
65
|
const remote = parseRemoteUrl('origin');
|
|
56
|
-
if (!
|
|
57
|
-
|| !
|
|
66
|
+
if (!isRepositoryComponent(remote.owner)
|
|
67
|
+
|| !isRepositoryComponent(remote.repo)) {
|
|
58
68
|
throw new Error('invalid repository identity');
|
|
59
69
|
}
|
|
60
70
|
return { owner: remote.owner, repository: remote.repo };
|
|
@@ -74,15 +84,19 @@ function resolveOriginDefaultBranch(gitRoot, deadline) {
|
|
|
74
84
|
catch {
|
|
75
85
|
// A clone does not have to carry origin/HEAD; ask the remote below.
|
|
76
86
|
}
|
|
77
|
-
const
|
|
87
|
+
const remoteRefs = gitOutput(['ls-remote', '--symref', 'origin', 'HEAD'], gitRoot, deadline, {
|
|
78
88
|
env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
|
|
79
89
|
timeoutCapMs: REMOTE_COMMAND_MS,
|
|
80
90
|
});
|
|
81
|
-
const
|
|
82
|
-
|
|
91
|
+
const prefix = 'ref: refs/heads/';
|
|
92
|
+
const heads = remoteRefs.split('\n').map(line => line.split('\t'))
|
|
93
|
+
.filter(fields => fields.length === 2 && fields[1] === 'HEAD' && fields[0].startsWith(prefix))
|
|
94
|
+
.map(fields => fields[0].slice(prefix.length));
|
|
95
|
+
if (heads.length !== 1 || heads[0].length === 0) {
|
|
83
96
|
throw new PrecomputeFailure('Could not resolve the origin default branch.');
|
|
84
97
|
}
|
|
85
|
-
|
|
98
|
+
gitOutput(['check-ref-format', `refs/heads/${heads[0]}`], gitRoot, deadline);
|
|
99
|
+
return heads[0];
|
|
86
100
|
}
|
|
87
101
|
function deriveBaseSha(gitRoot, deadline) {
|
|
88
102
|
const defaultBranch = resolveOriginDefaultBranch(gitRoot, deadline);
|
|
@@ -91,7 +105,7 @@ function deriveBaseSha(gitRoot, deadline) {
|
|
|
91
105
|
timeoutMs: Math.max(1, Math.min(remainingMs(deadline), REMOTE_COMMAND_MS)),
|
|
92
106
|
});
|
|
93
107
|
const baseSha = gitOutput(['merge-base', 'HEAD', baseRef], gitRoot, deadline);
|
|
94
|
-
if (!
|
|
108
|
+
if (!isCommitSha(baseSha)) {
|
|
95
109
|
throw new PrecomputeFailure('Could not derive the base commit.');
|
|
96
110
|
}
|
|
97
111
|
return baseSha;
|
|
@@ -115,7 +129,7 @@ function deriveWorktreeTreeSha(gitRoot, deadline) {
|
|
|
115
129
|
}
|
|
116
130
|
gitOutput(['add', '-A'], gitRoot, deadline, { env: indexEnv });
|
|
117
131
|
const treeSha = gitOutput(['write-tree'], gitRoot, deadline, { env: indexEnv });
|
|
118
|
-
if (!
|
|
132
|
+
if (!isCommitSha(treeSha)) {
|
|
119
133
|
throw new PrecomputeFailure('Could not derive the working-tree snapshot.');
|
|
120
134
|
}
|
|
121
135
|
return treeSha;
|
|
@@ -127,14 +141,14 @@ function deriveWorktreeTreeSha(gitRoot, deadline) {
|
|
|
127
141
|
function deriveWorkCommit(gitRoot, baseSha, treeSha, deadline) {
|
|
128
142
|
const headSha = gitOutput(['rev-parse', 'HEAD'], gitRoot, deadline);
|
|
129
143
|
const headTreeSha = gitOutput(['rev-parse', 'HEAD^{tree}'], gitRoot, deadline);
|
|
130
|
-
if (!
|
|
144
|
+
if (!isCommitSha(headSha) || !isCommitSha(headTreeSha)) {
|
|
131
145
|
throw new PrecomputeFailure('Could not resolve the current commit.');
|
|
132
146
|
}
|
|
133
147
|
// A clean checkout of the base already has the exact required identity.
|
|
134
148
|
if (baseSha === headSha && treeSha === headTreeSha)
|
|
135
149
|
return headSha;
|
|
136
150
|
const workCommit = gitOutput(['commit-tree', treeSha, '-p', baseSha, '-m', 'front-half-snapshot'], gitRoot, deadline, { env: { ...process.env, ...FIXED_COMMIT_ENV } });
|
|
137
|
-
if (!
|
|
151
|
+
if (!isCommitSha(workCommit)) {
|
|
138
152
|
throw new PrecomputeFailure('Could not derive the working commit.');
|
|
139
153
|
}
|
|
140
154
|
return workCommit;
|
|
@@ -202,7 +216,10 @@ async function buildCompressedPatch(gitRoot, baseSha, workCommit, deadline) {
|
|
|
202
216
|
child.stdout.pipe(gzip);
|
|
203
217
|
});
|
|
204
218
|
}
|
|
205
|
-
export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() +
|
|
219
|
+
export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() + HOOK_WALL_MS) {
|
|
220
|
+
// Capture this before git and compression work so delivery latency cannot
|
|
221
|
+
// make an older stop look newer than a later one.
|
|
222
|
+
const observedAt = Date.now();
|
|
206
223
|
const baseSha = deriveBaseSha(gitRoot, deadline);
|
|
207
224
|
const treeSha = deriveWorktreeTreeSha(gitRoot, deadline);
|
|
208
225
|
const workCommit = deriveWorkCommit(gitRoot, baseSha, treeSha, deadline);
|
|
@@ -221,6 +238,7 @@ export async function derivePrecomputeRequest(gitRoot, owner, repository, deadli
|
|
|
221
238
|
repository,
|
|
222
239
|
baseCommit: baseSha,
|
|
223
240
|
workCommit,
|
|
241
|
+
observedAt,
|
|
224
242
|
cacheKey,
|
|
225
243
|
patchGzBase64: patchGzip.toString('base64'),
|
|
226
244
|
patchSha256: createHash('sha256').update(patchGzip).digest('hex'),
|
|
@@ -231,16 +249,75 @@ function precomputeResult(response) {
|
|
|
231
249
|
if (!response || typeof response !== 'object' || Array.isArray(response)) {
|
|
232
250
|
throw new PrecomputeFailure('The precompute service returned an invalid response.');
|
|
233
251
|
}
|
|
234
|
-
const { status, runId } = response;
|
|
235
|
-
|
|
236
|
-
|
|
252
|
+
const { status, runId, walker: rawWalker } = response;
|
|
253
|
+
const walker = rawWalker === undefined ? undefined : readWalkerStatus(rawWalker);
|
|
254
|
+
if (status === 'queued' && typeof runId === 'string' && isRunId(runId)) {
|
|
255
|
+
return { outcome: 'enqueued', runId, ...(walker === undefined ? {} : { walker }) };
|
|
237
256
|
}
|
|
238
257
|
if (status === 'cached')
|
|
239
|
-
return { outcome: 'cached' };
|
|
258
|
+
return { outcome: 'cached', ...(walker === undefined ? {} : { walker }) };
|
|
240
259
|
if (status === 'existing')
|
|
241
|
-
return { outcome: 'existing' };
|
|
260
|
+
return { outcome: 'existing', ...(walker === undefined ? {} : { walker }) };
|
|
242
261
|
throw new PrecomputeFailure('The precompute service returned an invalid response.');
|
|
243
262
|
}
|
|
263
|
+
/** One authenticated submission of the already captured request. No source reads or retries. */
|
|
264
|
+
export const deliverPrecomputeRequest = async (request, signal) => {
|
|
265
|
+
signal.throwIfAborted();
|
|
266
|
+
const auth = await resolveAuthContext({ owner: request.owner, repo: request.repository });
|
|
267
|
+
signal.throwIfAborted();
|
|
268
|
+
return precomputeResult(await haystackJson('/api/agent/cloud-verifier/front-half', auth.token, { method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
269
|
+
body: JSON.stringify(request), signal }));
|
|
270
|
+
};
|
|
271
|
+
function readWalkerStatus(value) {
|
|
272
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
273
|
+
const record = value;
|
|
274
|
+
if (Object.keys(record).length === 2) {
|
|
275
|
+
if ((record.status === 'queued' || record.status === 'existing')
|
|
276
|
+
&& typeof record.runId === 'string' && isRunId(record.runId))
|
|
277
|
+
return { status: record.status, runId: record.runId };
|
|
278
|
+
if (record.status === 'unavailable'
|
|
279
|
+
&& (record.reason === 'runner-unavailable' || record.reason === 'enqueue-failed'))
|
|
280
|
+
return { status: record.status, reason: record.reason };
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
throw new PrecomputeFailure('The precompute service returned an invalid walker status.');
|
|
284
|
+
}
|
|
285
|
+
function walkerMetadataDirectory(gitRoot, deadline) {
|
|
286
|
+
const common = gitOutput(['rev-parse', '--git-common-dir'], gitRoot, deadline);
|
|
287
|
+
return join(isAbsolute(common) ? common : resolve(gitRoot, common), 'haystack');
|
|
288
|
+
}
|
|
289
|
+
function saveWalkerStatus(directory, request, walker) {
|
|
290
|
+
// Frozen local snapshot shape: {observedAt, repository, sourceCommit, walker}.
|
|
291
|
+
// Git's common metadata directory is shared by linked worktrees and excluded
|
|
292
|
+
// from the synthetic source tree. Rename publishes one complete JSON record.
|
|
293
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
294
|
+
if (!lstatSync(directory).isDirectory())
|
|
295
|
+
throw new PrecomputeFailure('Could not save the walker status.');
|
|
296
|
+
const temporary = join(directory, `.walker-status-${randomUUID()}.tmp`);
|
|
297
|
+
const descriptor = openSync(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
|
|
298
|
+
try {
|
|
299
|
+
try {
|
|
300
|
+
writeFileSync(descriptor, JSON.stringify({ observedAt: new Date(request.observedAt).toISOString(),
|
|
301
|
+
repository: `${request.owner}/${request.repository}`.toLowerCase(), sourceCommit: request.workCommit,
|
|
302
|
+
walker: readWalkerStatus(walker) }) + '\n');
|
|
303
|
+
fsyncSync(descriptor);
|
|
304
|
+
}
|
|
305
|
+
finally {
|
|
306
|
+
closeSync(descriptor);
|
|
307
|
+
}
|
|
308
|
+
renameSync(temporary, join(directory, 'walker-status.json'));
|
|
309
|
+
const parent = openSync(directory, constants.O_RDONLY);
|
|
310
|
+
try {
|
|
311
|
+
fsyncSync(parent);
|
|
312
|
+
}
|
|
313
|
+
finally {
|
|
314
|
+
closeSync(parent);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
finally {
|
|
318
|
+
rmSync(temporary, { force: true });
|
|
319
|
+
}
|
|
320
|
+
}
|
|
244
321
|
function normalModeFailure(error) {
|
|
245
322
|
if (error instanceof PrecomputeFailure)
|
|
246
323
|
return error;
|
|
@@ -256,9 +333,14 @@ function normalModeFailure(error) {
|
|
|
256
333
|
}
|
|
257
334
|
return new PrecomputeFailure(`The precompute request failed (HTTP ${error.status}).`);
|
|
258
335
|
}
|
|
336
|
+
// The deadline fired while the POST was in flight. The server may still
|
|
337
|
+
// have accepted the job, so say what happened rather than "failed".
|
|
338
|
+
if (error instanceof Error && error.name === 'AbortError') {
|
|
339
|
+
return new PrecomputeFailure('Precompute timed out.');
|
|
340
|
+
}
|
|
259
341
|
return new PrecomputeFailure('The precompute request failed.');
|
|
260
342
|
}
|
|
261
|
-
function printResult(outcome, runId) {
|
|
343
|
+
function printResult(outcome, runId, walker) {
|
|
262
344
|
if (outcome === 'enqueued') {
|
|
263
345
|
console.log(chalk.green(`Precompute request queued${runId ? ` (${runId})` : ''}.`));
|
|
264
346
|
}
|
|
@@ -268,18 +350,36 @@ function printResult(outcome, runId) {
|
|
|
268
350
|
else if (outcome === 'existing') {
|
|
269
351
|
console.log(chalk.green('A matching precompute request already exists.'));
|
|
270
352
|
}
|
|
353
|
+
if (walker?.status === 'queued')
|
|
354
|
+
console.log(`Walker queued (${walker.runId}).`);
|
|
355
|
+
else if (walker?.status === 'existing')
|
|
356
|
+
console.log(`A matching walker run already exists (${walker.runId}).`);
|
|
357
|
+
else if (walker?.status === 'unavailable')
|
|
358
|
+
console.log(walker.reason === 'runner-unavailable'
|
|
359
|
+
? 'Walker unavailable: no runner is available for this repository.'
|
|
360
|
+
: 'Walker unavailable: the enqueue request failed.');
|
|
361
|
+
else
|
|
362
|
+
console.log('Walker status was not reported by the service.');
|
|
271
363
|
}
|
|
272
364
|
export async function verifyPrecomputeCommand(options) {
|
|
273
365
|
const startedAt = Date.now();
|
|
274
|
-
const
|
|
366
|
+
const totalWallMs = options.hook ? HOOK_WALL_MS : EXPLICIT_WALL_MS;
|
|
367
|
+
const deadline = startedAt + totalWallMs;
|
|
275
368
|
let outcome = 'error';
|
|
276
369
|
try {
|
|
277
370
|
const gitRoot = findGitRoot();
|
|
278
371
|
if (!gitRoot) {
|
|
279
372
|
throw new PrecomputeFailure('Run this command from a git repository.');
|
|
280
373
|
}
|
|
374
|
+
const metadataDirectory = walkerMetadataDirectory(gitRoot, deadline);
|
|
281
375
|
const repository = resolveOriginRepository(gitRoot);
|
|
282
376
|
const derivation = await derivePrecomputeRequest(gitRoot, repository.owner, repository.repository, deadline);
|
|
377
|
+
if (options.hook) {
|
|
378
|
+
remainingMs(deadline);
|
|
379
|
+
await startPrecomputeDelivery(metadataDirectory, derivation.request);
|
|
380
|
+
outcome = 'pending';
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
283
383
|
let token;
|
|
284
384
|
try {
|
|
285
385
|
const auth = await resolveAuthContext({
|
|
@@ -312,8 +412,16 @@ export async function verifyPrecomputeCommand(options) {
|
|
|
312
412
|
}
|
|
313
413
|
const result = precomputeResult(response);
|
|
314
414
|
outcome = result.outcome;
|
|
415
|
+
if (result.walker) {
|
|
416
|
+
try {
|
|
417
|
+
saveWalkerStatus(metadataDirectory, derivation.request, result.walker);
|
|
418
|
+
}
|
|
419
|
+
catch {
|
|
420
|
+
throw new PrecomputeFailure('Precompute response received, but its walker status could not be saved.');
|
|
421
|
+
}
|
|
422
|
+
}
|
|
315
423
|
if (!options.hook)
|
|
316
|
-
printResult(outcome, result.runId);
|
|
424
|
+
printResult(outcome, result.runId, result.walker);
|
|
317
425
|
}
|
|
318
426
|
catch (error) {
|
|
319
427
|
if (!options.hook)
|
|
@@ -321,7 +429,7 @@ export async function verifyPrecomputeCommand(options) {
|
|
|
321
429
|
}
|
|
322
430
|
finally {
|
|
323
431
|
const wallMs = Date.now() - startedAt;
|
|
324
|
-
trackPrecomputeEvent(outcome, wallMs, Math.max(1,
|
|
432
|
+
trackPrecomputeEvent(outcome, wallMs, options.hook ? 1 : Math.max(1, totalWallMs - wallMs));
|
|
325
433
|
if (options.hook)
|
|
326
434
|
process.exitCode = 0;
|
|
327
435
|
}
|
|
@@ -6,6 +6,12 @@ export interface VerifyOptions {
|
|
|
6
6
|
base?: string;
|
|
7
7
|
head?: string;
|
|
8
8
|
intentTranscript?: string;
|
|
9
|
+
intentFile?: string;
|
|
10
|
+
idempotencyKey?: string;
|
|
11
|
+
account?: string;
|
|
12
|
+
wait?: boolean;
|
|
13
|
+
interval?: string;
|
|
14
|
+
timeout?: string;
|
|
9
15
|
json?: boolean;
|
|
10
16
|
open?: boolean;
|
|
11
17
|
/** Internal reproducibility option: execute an already-frozen plan. */
|
package/dist/commands/verify.js
CHANGED
|
@@ -107,6 +107,28 @@ export async function startVerificationRun(specimenArg, options) {
|
|
|
107
107
|
return parseStartRunResponse(raw);
|
|
108
108
|
}
|
|
109
109
|
export async function verifyCommand(specimenArg, options) {
|
|
110
|
+
if (specimenArg === undefined && (options.repo !== undefined || options.base !== undefined || options.head !== undefined)) {
|
|
111
|
+
if (!options.repo || !options.base || !options.head) {
|
|
112
|
+
throw new Error('Repository verification requires --repo owner/repo, --base <ref>, and --head <ref>.');
|
|
113
|
+
}
|
|
114
|
+
if (options.intentTranscript !== undefined) {
|
|
115
|
+
throw new Error('Repository verification accepts optional --intent-file JSON with problem, goal, and intended_outcomes.');
|
|
116
|
+
}
|
|
117
|
+
const { verifyHostedStartCommand } = await import('./verify-hosted.js');
|
|
118
|
+
await verifyHostedStartCommand(options.repo, {
|
|
119
|
+
provider: 'haystack',
|
|
120
|
+
base: options.base,
|
|
121
|
+
head: options.head,
|
|
122
|
+
intentFile: options.intentFile,
|
|
123
|
+
idempotencyKey: options.idempotencyKey,
|
|
124
|
+
account: options.account,
|
|
125
|
+
wait: options.wait,
|
|
126
|
+
interval: options.interval,
|
|
127
|
+
timeout: options.timeout,
|
|
128
|
+
json: options.json,
|
|
129
|
+
});
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
110
132
|
if (!options.json) {
|
|
111
133
|
console.error(chalk.dim('Preparing the cloud verification run…'));
|
|
112
134
|
}
|
package/dist/index.js
CHANGED
|
@@ -174,24 +174,35 @@ const verify = program
|
|
|
174
174
|
.command('verify')
|
|
175
175
|
.description('Verify a change in matched cloud universes, or bisect a historical regression')
|
|
176
176
|
.argument('[specimen]', 'Launch specimen to run (default: all)')
|
|
177
|
-
.option('--repo <
|
|
177
|
+
.option('--repo <owner/repo>', 'GitHub repository to verify through the authenticated hosted service')
|
|
178
178
|
.option('--base <ref>', 'Base revision for a repository run')
|
|
179
179
|
.option('--head <ref>', 'Head revision for a repository run')
|
|
180
|
+
.option('--intent-file <path>', 'Optional intent JSON with problem, goal, and intended_outcomes')
|
|
181
|
+
.option('--idempotency-key <key>', 'Stable retry key (default: derived from repository, refs, and intent)')
|
|
182
|
+
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
183
|
+
.addOption(new Option('--no-wait', 'Return after the hosted run is queued').default(undefined))
|
|
184
|
+
.option('--interval <seconds>', 'Hosted polling interval (default 5)')
|
|
185
|
+
.option('--timeout <minutes>', 'Maximum hosted wait (default 35)')
|
|
180
186
|
.option('--intent-transcript <file>', 'Coding-agent transcript used as intent evidence')
|
|
181
187
|
.option('--server <url>', 'Verifier control-plane URL (default: http://127.0.0.1:3000)')
|
|
182
188
|
.option('--token <token>', 'Bearer token for a control plane that requires auth (or HAYSTACK_VERIFY_TOKEN)')
|
|
183
189
|
.option('--no-open', 'Do not open the live results page')
|
|
184
|
-
.option('--json', 'Machine-readable run
|
|
190
|
+
.option('--json', 'Machine-readable run status and results')
|
|
185
191
|
.addHelpText('after', `
|
|
186
192
|
Launch specimens:
|
|
187
193
|
posthog-js-478 Replay PostHog/posthog-js PR #478 in matched browser environments (latent-issue mode)
|
|
188
194
|
analytics-js-255 Identity routing regression with sandbox bisect
|
|
189
195
|
all Run each reproduced specimen as an isolated chapter (default)
|
|
190
196
|
|
|
191
|
-
|
|
197
|
+
Repository verification uses the authenticated hosted fleet and resolves refs
|
|
198
|
+
once. Intent is optional. Hosted status also resumes these runs.
|
|
199
|
+
|
|
200
|
+
For launch specimens, run the local control plane from the Haystack repository:
|
|
192
201
|
pnpm dev
|
|
193
202
|
|
|
194
203
|
Examples:
|
|
204
|
+
haystack verify --repo owner/repo --base main --head feature --json
|
|
205
|
+
haystack verify history owner/repo --limit 20
|
|
195
206
|
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path>
|
|
196
207
|
haystack verify posthog-js-478 # start a run, then:
|
|
197
208
|
haystack verify reproducibility posthog-js-478 --runs 3
|
|
@@ -210,6 +221,26 @@ Examples:
|
|
|
210
221
|
`)
|
|
211
222
|
.action((specimen, options) => runPublicCommand(() => verifyCommand(specimen, options), options.json));
|
|
212
223
|
const verifyRunArg = 'Run id, chapter id (newest run of that chapter), or "latest" (default)';
|
|
224
|
+
verify
|
|
225
|
+
.command('history')
|
|
226
|
+
.description('Read your recent finished hosted verification runs without starting a run')
|
|
227
|
+
.argument('<repository>', 'Exact GitHub owner/repository name')
|
|
228
|
+
.option('--limit <runs>', 'Number of recent finished runs to show, from 1 to 100 (default 20)')
|
|
229
|
+
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
230
|
+
.option('--json', 'Versioned history with execution health and separate bug counts')
|
|
231
|
+
.addHelpText('after', `
|
|
232
|
+
History covers your hosted fleet runs for this repository. Reliability counts
|
|
233
|
+
only these returned runs; it does not measure whether the code is bug-free.
|
|
234
|
+
|
|
235
|
+
Examples:
|
|
236
|
+
haystack verify history owner/repo
|
|
237
|
+
haystack verify history owner/repo --limit 50 --account my-login --json
|
|
238
|
+
`)
|
|
239
|
+
.action(async (repository, _options, cmd) => {
|
|
240
|
+
const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
|
|
241
|
+
const { verifyHistoryCommand } = await import('./commands/verify-history.js');
|
|
242
|
+
return runPublicCommand(() => verifyHistoryCommand(repository, options), options.json);
|
|
243
|
+
});
|
|
213
244
|
verify
|
|
214
245
|
.command('precompute')
|
|
215
246
|
.description('Precompute reusable verification inputs for the current working tree')
|
|
@@ -235,7 +266,7 @@ hostedVerify
|
|
|
235
266
|
// validation below accepts the inherited values and still fails closed.
|
|
236
267
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
237
268
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
238
|
-
.
|
|
269
|
+
.option('--intent-file <path>', 'Required for this alias: JSON with exactly problem, goal, and intended_outcomes')
|
|
239
270
|
.option('--idempotency-key <key>', 'Stable retry key (default: deterministic from repository and commits)')
|
|
240
271
|
.option('--replay-plan-from <run-id>', 'Reuse the exact validated risk plan from a completed owned run')
|
|
241
272
|
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
@@ -284,7 +315,7 @@ hostedVerify
|
|
|
284
315
|
.argument('<repository>', 'Exact GitHub owner/repository name')
|
|
285
316
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
286
317
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
287
|
-
.
|
|
318
|
+
.option('--intent-file <path>', 'Required: agent-supplied JSON reused for every hosted run in the series')
|
|
288
319
|
.option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
|
|
289
320
|
.option('--series-key <key>', 'Stable series key for resumable idempotent retries')
|
|
290
321
|
.option('--fresh-plans', 'Run the planner independently for every run instead of replaying the first plan')
|
|
@@ -365,6 +396,34 @@ verify
|
|
|
365
396
|
const { verifyStatusCommand } = await import('./commands/verify-ops.js');
|
|
366
397
|
return runPublicCommand(() => verifyStatusCommand(run, options), options.json);
|
|
367
398
|
});
|
|
399
|
+
verify
|
|
400
|
+
.command('cases <run>')
|
|
401
|
+
.description('List cloud verification cases available to explore for 24 hours after completion')
|
|
402
|
+
.option('--repo <owner/repo>', 'Required exact GitHub repository; uses your Haystack login')
|
|
403
|
+
.option('--json', 'Machine-readable case IDs and expiry times')
|
|
404
|
+
.action(async (run, _options, cmd) => {
|
|
405
|
+
const options = cmd.optsWithGlobals();
|
|
406
|
+
const { verifyCasesCommand } = await import('./commands/verify-explore.js');
|
|
407
|
+
return runPublicCommand(() => verifyCasesCommand(run, options), options.json);
|
|
408
|
+
});
|
|
409
|
+
verify
|
|
410
|
+
.command('explore <run> <case-id>')
|
|
411
|
+
.description('Open or use your own copy of a saved fleet case; never changes the verification original')
|
|
412
|
+
.option('--repo <owner/repo>', 'Required exact GitHub repository; uses your Haystack login')
|
|
413
|
+
.option('--side <side>', 'base or head (default head)')
|
|
414
|
+
.option('--exec-json <argv>', 'Command arguments as a JSON array, executed inside your working sandbox')
|
|
415
|
+
.option('--cwd <path>', 'Absolute guest directory (default /home/user/pj)')
|
|
416
|
+
.option('--command-timeout <seconds>', 'Command timeout, 1–300 seconds (default 60)')
|
|
417
|
+
.option('--close', 'Discard your working copy; leave the saved case unchanged')
|
|
418
|
+
.option('--action <id>', 'Read/follow an existing action without submitting another command')
|
|
419
|
+
.option('--request-id <uuid>', 'Idempotency ID for a new request; reuse only with identical arguments')
|
|
420
|
+
.option('--no-wait', 'Return after submitting the action')
|
|
421
|
+
.option('--json', 'Machine-readable action and command output')
|
|
422
|
+
.action(async (run, caseId, _options, cmd) => {
|
|
423
|
+
const options = cmd.optsWithGlobals();
|
|
424
|
+
const { verifyExploreCommand } = await import('./commands/verify-explore.js');
|
|
425
|
+
return runPublicCommand(() => verifyExploreCommand(run, caseId, options), options.json);
|
|
426
|
+
});
|
|
368
427
|
verify
|
|
369
428
|
.command('watch')
|
|
370
429
|
.description('Follow a run until it finishes; exit 0 all passed, 1 tests failed, 2 run/infra broke')
|
|
@@ -863,6 +922,8 @@ program
|
|
|
863
922
|
// keeps retired auto-fix/verification wording out of the CLI surface).
|
|
864
923
|
.description('Run a blind design-by-execution review of a PR')
|
|
865
924
|
.option('--json', 'Machine-readable output')
|
|
925
|
+
.option('--history', 'Show recent execution reliability for this repository without starting a run')
|
|
926
|
+
.option('--limit <attempts>', 'Maximum recorded terminal attempts to show with --history (1–100, default 20)')
|
|
866
927
|
.option('--no-wait', 'Return after the run is admitted instead of waiting for its verdict')
|
|
867
928
|
.option('--run <run-id>', 'Resume and wait for an existing exact run id')
|
|
868
929
|
.option('--poll-interval <seconds>', 'Status polling interval', '3')
|
|
@@ -883,6 +944,11 @@ outside the registered intent.
|
|
|
883
944
|
The command waits for a behavioral verdict by default. Use --no-wait to
|
|
884
945
|
return the run id immediately, or --run <run-id> to resume waiting.
|
|
885
946
|
|
|
947
|
+
Read recent execution reliability across this repository's PRs:
|
|
948
|
+
haystack design-verify owner/repo#123 --history --limit 20
|
|
949
|
+
History covers recorded terminal attempts, including partial or failed runs;
|
|
950
|
+
it does not measure bug-detection accuracy or whether changes are bug-free.
|
|
951
|
+
|
|
886
952
|
After a completed run, inspect or replay one retained case universe without
|
|
887
953
|
receiving provider credentials:
|
|
888
954
|
haystack design-verify owner/repo#123 --run RUN --case case-00001 --universe before --replay
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/** Read-only history from the active hosted verifier, scoped to the caller's
|
|
2
|
+
* stable GitHub user and repository IDs. Counts describe the returned window,
|
|
3
|
+
* never bug correctness. Missing results remain unknown. */
|
|
4
|
+
export type VerifyHistoryHealth = 'healthy' | 'degraded' | 'infrastructure-error' | 'not-executed' | 'unknown';
|
|
5
|
+
export interface VerifyHistoryRun {
|
|
6
|
+
runId: string;
|
|
7
|
+
baseCommit: string;
|
|
8
|
+
headCommit: string;
|
|
9
|
+
createdAt: string;
|
|
10
|
+
completedAt: string;
|
|
11
|
+
status: 'complete' | 'errored';
|
|
12
|
+
lifecycle: 'complete' | 'failed' | 'cancelled' | 'budget-stopped' | 'account-action' | null;
|
|
13
|
+
health: VerifyHistoryHealth;
|
|
14
|
+
regressionCount: number | null;
|
|
15
|
+
executedCaseCount: number | null;
|
|
16
|
+
degradationCodes: string[];
|
|
17
|
+
supersededBy: string | null;
|
|
18
|
+
}
|
|
19
|
+
export interface VerifyHistoryReport {
|
|
20
|
+
version: 'dv-verify-history-v1';
|
|
21
|
+
provider: 'haystack';
|
|
22
|
+
scope: 'user';
|
|
23
|
+
repository: {
|
|
24
|
+
owner: string;
|
|
25
|
+
name: string;
|
|
26
|
+
repositoryId: string;
|
|
27
|
+
};
|
|
28
|
+
limit: number;
|
|
29
|
+
hasMore: boolean;
|
|
30
|
+
runs: VerifyHistoryRun[];
|
|
31
|
+
summary: {
|
|
32
|
+
total: number;
|
|
33
|
+
counts: Record<VerifyHistoryHealth, number>;
|
|
34
|
+
/** Healthy/degraded runs with executedCaseCount > 0, divided by total;
|
|
35
|
+
* null for an empty window. */
|
|
36
|
+
usableRate: number | null;
|
|
37
|
+
};
|
|
38
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* the server-side shared secret. This client just POSTs and interprets the
|
|
8
8
|
* response.
|
|
9
9
|
*/
|
|
10
|
+
import { type DesignVerifierHistoryReport } from './design-verifier-history.js';
|
|
10
11
|
export type DesignVerifierTriggerResult = {
|
|
11
12
|
status: 'started' | 'queued' | 'duplicate';
|
|
12
13
|
runId: string;
|
|
@@ -184,6 +185,13 @@ export declare function waitForDesignVerifierUniverseAction(owner: string, repo:
|
|
|
184
185
|
}>;
|
|
185
186
|
export declare function triggerDesignVerifier(owner: string, repo: string, prNumber: number, token: string): Promise<DesignVerifierTriggerResult>;
|
|
186
187
|
export declare function getDesignVerifierStatus(owner: string, repo: string, prNumber: number, runId: string, token: string): Promise<DesignVerifierStatusFetchResult>;
|
|
188
|
+
export declare function getDesignVerifierHistory(owner: string, repo: string, token: string, limit?: number): Promise<{
|
|
189
|
+
status: 'ok';
|
|
190
|
+
history: DesignVerifierHistoryReport;
|
|
191
|
+
} | {
|
|
192
|
+
status: 'denied' | 'unavailable' | 'error';
|
|
193
|
+
message: string;
|
|
194
|
+
}>;
|
|
187
195
|
export declare function waitForDesignVerifier(owner: string, repo: string, prNumber: number, runId: string, token: string, options?: {
|
|
188
196
|
intervalMs?: number;
|
|
189
197
|
timeoutMs?: number;
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* response.
|
|
9
9
|
*/
|
|
10
10
|
import { haystackApiUrl } from './haystack-api.js';
|
|
11
|
+
import { isDesignVerifierHistoryReport } from './design-verifier-history.js';
|
|
11
12
|
function isDuration(value) {
|
|
12
13
|
return value === null || (typeof value === 'number' && Number.isFinite(value) && value >= 0);
|
|
13
14
|
}
|
|
@@ -239,6 +240,28 @@ export async function getDesignVerifierStatus(owner, repo, prNumber, runId, toke
|
|
|
239
240
|
return { status: 'unavailable', message };
|
|
240
241
|
return { status: 'error', message };
|
|
241
242
|
}
|
|
243
|
+
export async function getDesignVerifierHistory(owner, repo, token, limit = 20) {
|
|
244
|
+
const url = haystackApiUrl(`/api/design-verifier/history/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}?limit=${limit}`);
|
|
245
|
+
try {
|
|
246
|
+
const response = await fetch(url, { method: 'GET', headers: apiHeaders(token) });
|
|
247
|
+
const data = await response.json();
|
|
248
|
+
if (response.ok) {
|
|
249
|
+
if (!isDesignVerifierHistoryReport(data, { owner, repo, limit })) {
|
|
250
|
+
return { status: 'error', message: 'History endpoint returned an invalid or mismatched result.' };
|
|
251
|
+
}
|
|
252
|
+
return { status: 'ok', history: data };
|
|
253
|
+
}
|
|
254
|
+
const message = data?.message || data?.error || `HTTP ${response.status}`;
|
|
255
|
+
if (response.status === 401 || response.status === 403)
|
|
256
|
+
return { status: 'denied', message };
|
|
257
|
+
if ([404, 502, 503].includes(response.status))
|
|
258
|
+
return { status: 'unavailable', message };
|
|
259
|
+
return { status: 'error', message };
|
|
260
|
+
}
|
|
261
|
+
catch (error) {
|
|
262
|
+
return { status: 'error', message: error.message };
|
|
263
|
+
}
|
|
264
|
+
}
|
|
242
265
|
export async function waitForDesignVerifier(owner, repo, prNumber, runId, token, options = {}) {
|
|
243
266
|
const intervalMs = options.intervalMs ?? 3000;
|
|
244
267
|
const timeoutMs = options.timeoutMs ?? 2 * 60 * 60 * 1000;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/** FIXED CONTRACT: historical execution health, never a score for bug-detection accuracy. */
|
|
2
|
+
export type DesignVerifierHistoryOutcome = 'healthy' | 'degraded' | 'invalid' | 'failed' | 'unknown';
|
|
3
|
+
export interface DesignVerifierHistoryRun {
|
|
4
|
+
runId: string;
|
|
5
|
+
pr: number;
|
|
6
|
+
baseSha: string | null;
|
|
7
|
+
headSha: string | null;
|
|
8
|
+
createdAt: string;
|
|
9
|
+
finishedAt: string | null;
|
|
10
|
+
status: string;
|
|
11
|
+
outcome: DesignVerifierHistoryOutcome;
|
|
12
|
+
admitted: number | null;
|
|
13
|
+
losses: Array<{
|
|
14
|
+
code: string;
|
|
15
|
+
owner: string;
|
|
16
|
+
impact: 'degraded' | 'invalid';
|
|
17
|
+
count: number;
|
|
18
|
+
}>;
|
|
19
|
+
}
|
|
20
|
+
export interface DesignVerifierHistoryReport {
|
|
21
|
+
version: 'dv-cli-history-v1';
|
|
22
|
+
owner: string;
|
|
23
|
+
repo: string;
|
|
24
|
+
retentionDays: number;
|
|
25
|
+
limit: number;
|
|
26
|
+
runs: DesignVerifierHistoryRun[];
|
|
27
|
+
summary: {
|
|
28
|
+
total: number;
|
|
29
|
+
healthy: number;
|
|
30
|
+
degraded: number;
|
|
31
|
+
invalid: number;
|
|
32
|
+
failed: number;
|
|
33
|
+
unknown: number;
|
|
34
|
+
usableRate: number | null;
|
|
35
|
+
topBlockers: Array<{
|
|
36
|
+
code: string;
|
|
37
|
+
affectedRuns: number;
|
|
38
|
+
}>;
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
export declare function isDesignVerifierHistoryReport(value: unknown, expected: {
|
|
42
|
+
owner: string;
|
|
43
|
+
repo: string;
|
|
44
|
+
limit: number;
|
|
45
|
+
}): value is DesignVerifierHistoryReport;
|
|
46
|
+
export declare function formatDesignVerifierHistory(report: DesignVerifierHistoryReport): string;
|