@haystackeditor/cli 0.15.28 → 0.15.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/telemetry/browser-runtime.js +167 -4
- package/dist/commands/precompute-delivery-contract.d.ts +68 -0
- package/dist/commands/precompute-delivery-contract.js +1 -0
- package/dist/commands/precompute-delivery-worker.d.ts +1 -0
- package/dist/commands/precompute-delivery-worker.js +68 -0
- package/dist/commands/precompute-delivery.d.ts +20 -0
- package/dist/commands/precompute-delivery.js +319 -0
- package/dist/commands/verify-explore.d.ts +14 -0
- package/dist/commands/verify-explore.js +180 -0
- package/dist/commands/verify-hosted-mcp.js +12 -2
- package/dist/commands/verify-hosted-reproducibility.d.ts +1 -0
- package/dist/commands/verify-hosted-reproducibility.js +4 -1
- package/dist/commands/verify-hosted.d.ts +18 -1
- package/dist/commands/verify-hosted.js +247 -51
- package/dist/commands/verify-precompute.d.ts +12 -1
- package/dist/commands/verify-precompute.js +137 -29
- package/dist/commands/verify.d.ts +6 -0
- package/dist/commands/verify.js +22 -0
- package/dist/index.js +49 -9
- package/dist/utils/design-verifier-result.d.ts +17 -0
- package/dist/utils/design-verifier-result.js +80 -1
- package/dist/utils/git.js +5 -1
- package/dist/utils/telemetry.js +8 -2
- package/package.json +1 -1
- package/schemas/cloud-verifier.v1.json +108 -45
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn, execFileSync } from 'node:child_process';
|
|
2
|
-
import { createHash } from 'node:crypto';
|
|
3
|
-
import { cpSync, existsSync, mkdtempSync, rmSync } from 'node:fs';
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { closeSync, constants, cpSync, existsSync, fsyncSync, lstatSync, mkdirSync, mkdtempSync, openSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
4
|
import { createGzip } from 'node:zlib';
|
|
5
5
|
import { isAbsolute, join, resolve } from 'node:path';
|
|
6
6
|
import { tmpdir } from 'node:os';
|
|
@@ -9,11 +9,21 @@ import { NotLoggedInError, resolveAuthContext } from '../utils/auth.js';
|
|
|
9
9
|
import { findGitRoot, parseRemoteUrl, resolveDiffBaseRef, } from '../utils/git.js';
|
|
10
10
|
import { HaystackApiError, haystackJson } from '../utils/haystack-api.js';
|
|
11
11
|
import { trackPrecomputeEvent } from '../utils/telemetry.js';
|
|
12
|
-
|
|
12
|
+
import { startPrecomputeDelivery } from './precompute-delivery.js';
|
|
13
|
+
// --hook has five seconds to capture the exact source and publish local custody.
|
|
14
|
+
// Its detached sender owns the bounded network delivery and acknowledgement.
|
|
15
|
+
// An explicit invocation still waits synchronously for the server response.
|
|
16
|
+
const HOOK_WALL_MS = 5_000;
|
|
17
|
+
const EXPLICIT_WALL_MS = 60_000;
|
|
13
18
|
const REMOTE_COMMAND_MS = 750;
|
|
14
|
-
|
|
15
|
-
const
|
|
16
|
-
|
|
19
|
+
// The published CLI cannot import verifier/contracts outside its package root.
|
|
20
|
+
const hasOnlyCharacters = (value, allowed) => value.length > 0
|
|
21
|
+
&& [...value].every(character => allowed.includes(character));
|
|
22
|
+
const isCommitSha = (value) => value.length === 40 && hasOnlyCharacters(value, '0123456789abcdef');
|
|
23
|
+
const isRepositoryComponent = (value) => value.length <= 100
|
|
24
|
+
&& hasOnlyCharacters(value, 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-');
|
|
25
|
+
const isRunId = (value) => value.startsWith('cv_') && value.length === 51
|
|
26
|
+
&& hasOnlyCharacters(value.slice(3), '0123456789abcdef');
|
|
17
27
|
const FIXED_COMMIT_ENV = {
|
|
18
28
|
GIT_AUTHOR_NAME: 'haystack-stop-hook',
|
|
19
29
|
GIT_AUTHOR_EMAIL: 'stop-hook@haystack.invalid',
|
|
@@ -36,7 +46,7 @@ function remainingMs(deadline) {
|
|
|
36
46
|
return remaining;
|
|
37
47
|
}
|
|
38
48
|
function gitOutput(args, gitRoot, deadline, options = {}) {
|
|
39
|
-
const timeout = Math.max(1, Math.min(remainingMs(deadline), options.timeoutCapMs ??
|
|
49
|
+
const timeout = Math.max(1, Math.min(remainingMs(deadline), options.timeoutCapMs ?? HOOK_WALL_MS));
|
|
40
50
|
try {
|
|
41
51
|
return execFileSync('git', args, {
|
|
42
52
|
cwd: gitRoot,
|
|
@@ -53,8 +63,8 @@ function gitOutput(args, gitRoot, deadline, options = {}) {
|
|
|
53
63
|
function resolveOriginRepository(gitRoot) {
|
|
54
64
|
try {
|
|
55
65
|
const remote = parseRemoteUrl('origin');
|
|
56
|
-
if (!
|
|
57
|
-
|| !
|
|
66
|
+
if (!isRepositoryComponent(remote.owner)
|
|
67
|
+
|| !isRepositoryComponent(remote.repo)) {
|
|
58
68
|
throw new Error('invalid repository identity');
|
|
59
69
|
}
|
|
60
70
|
return { owner: remote.owner, repository: remote.repo };
|
|
@@ -74,15 +84,19 @@ function resolveOriginDefaultBranch(gitRoot, deadline) {
|
|
|
74
84
|
catch {
|
|
75
85
|
// A clone does not have to carry origin/HEAD; ask the remote below.
|
|
76
86
|
}
|
|
77
|
-
const
|
|
87
|
+
const remoteRefs = gitOutput(['ls-remote', '--symref', 'origin', 'HEAD'], gitRoot, deadline, {
|
|
78
88
|
env: { ...process.env, GIT_TERMINAL_PROMPT: '0' },
|
|
79
89
|
timeoutCapMs: REMOTE_COMMAND_MS,
|
|
80
90
|
});
|
|
81
|
-
const
|
|
82
|
-
|
|
91
|
+
const prefix = 'ref: refs/heads/';
|
|
92
|
+
const heads = remoteRefs.split('\n').map(line => line.split('\t'))
|
|
93
|
+
.filter(fields => fields.length === 2 && fields[1] === 'HEAD' && fields[0].startsWith(prefix))
|
|
94
|
+
.map(fields => fields[0].slice(prefix.length));
|
|
95
|
+
if (heads.length !== 1 || heads[0].length === 0) {
|
|
83
96
|
throw new PrecomputeFailure('Could not resolve the origin default branch.');
|
|
84
97
|
}
|
|
85
|
-
|
|
98
|
+
gitOutput(['check-ref-format', `refs/heads/${heads[0]}`], gitRoot, deadline);
|
|
99
|
+
return heads[0];
|
|
86
100
|
}
|
|
87
101
|
function deriveBaseSha(gitRoot, deadline) {
|
|
88
102
|
const defaultBranch = resolveOriginDefaultBranch(gitRoot, deadline);
|
|
@@ -91,7 +105,7 @@ function deriveBaseSha(gitRoot, deadline) {
|
|
|
91
105
|
timeoutMs: Math.max(1, Math.min(remainingMs(deadline), REMOTE_COMMAND_MS)),
|
|
92
106
|
});
|
|
93
107
|
const baseSha = gitOutput(['merge-base', 'HEAD', baseRef], gitRoot, deadline);
|
|
94
|
-
if (!
|
|
108
|
+
if (!isCommitSha(baseSha)) {
|
|
95
109
|
throw new PrecomputeFailure('Could not derive the base commit.');
|
|
96
110
|
}
|
|
97
111
|
return baseSha;
|
|
@@ -115,7 +129,7 @@ function deriveWorktreeTreeSha(gitRoot, deadline) {
|
|
|
115
129
|
}
|
|
116
130
|
gitOutput(['add', '-A'], gitRoot, deadline, { env: indexEnv });
|
|
117
131
|
const treeSha = gitOutput(['write-tree'], gitRoot, deadline, { env: indexEnv });
|
|
118
|
-
if (!
|
|
132
|
+
if (!isCommitSha(treeSha)) {
|
|
119
133
|
throw new PrecomputeFailure('Could not derive the working-tree snapshot.');
|
|
120
134
|
}
|
|
121
135
|
return treeSha;
|
|
@@ -127,22 +141,22 @@ function deriveWorktreeTreeSha(gitRoot, deadline) {
|
|
|
127
141
|
function deriveWorkCommit(gitRoot, baseSha, treeSha, deadline) {
|
|
128
142
|
const headSha = gitOutput(['rev-parse', 'HEAD'], gitRoot, deadline);
|
|
129
143
|
const headTreeSha = gitOutput(['rev-parse', 'HEAD^{tree}'], gitRoot, deadline);
|
|
130
|
-
if (!
|
|
144
|
+
if (!isCommitSha(headSha) || !isCommitSha(headTreeSha)) {
|
|
131
145
|
throw new PrecomputeFailure('Could not resolve the current commit.');
|
|
132
146
|
}
|
|
133
147
|
// A clean checkout of the base already has the exact required identity.
|
|
134
148
|
if (baseSha === headSha && treeSha === headTreeSha)
|
|
135
149
|
return headSha;
|
|
136
150
|
const workCommit = gitOutput(['commit-tree', treeSha, '-p', baseSha, '-m', 'front-half-snapshot'], gitRoot, deadline, { env: { ...process.env, ...FIXED_COMMIT_ENV } });
|
|
137
|
-
if (!
|
|
151
|
+
if (!isCommitSha(workCommit)) {
|
|
138
152
|
throw new PrecomputeFailure('Could not derive the working commit.');
|
|
139
153
|
}
|
|
140
154
|
return workCommit;
|
|
141
155
|
}
|
|
142
156
|
function slimInputHash() {
|
|
143
|
-
// Mirrors printf '%s\0%s\0%s\0%s\0' slim
|
|
157
|
+
// Mirrors printf '%s\0%s\0%s\0%s\0' slim agent-deferred none none.
|
|
144
158
|
return createHash('sha256')
|
|
145
|
-
.update(Buffer.from('slim\
|
|
159
|
+
.update(Buffer.from('slim\0agent-deferred\0none\0none\0'))
|
|
146
160
|
.digest('hex');
|
|
147
161
|
}
|
|
148
162
|
async function buildCompressedPatch(gitRoot, baseSha, workCommit, deadline) {
|
|
@@ -202,7 +216,10 @@ async function buildCompressedPatch(gitRoot, baseSha, workCommit, deadline) {
|
|
|
202
216
|
child.stdout.pipe(gzip);
|
|
203
217
|
});
|
|
204
218
|
}
|
|
205
|
-
export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() +
|
|
219
|
+
export async function derivePrecomputeRequest(gitRoot, owner, repository, deadline = Date.now() + HOOK_WALL_MS) {
|
|
220
|
+
// Capture this before git and compression work so delivery latency cannot
|
|
221
|
+
// make an older stop look newer than a later one.
|
|
222
|
+
const observedAt = Date.now();
|
|
206
223
|
const baseSha = deriveBaseSha(gitRoot, deadline);
|
|
207
224
|
const treeSha = deriveWorktreeTreeSha(gitRoot, deadline);
|
|
208
225
|
const workCommit = deriveWorkCommit(gitRoot, baseSha, treeSha, deadline);
|
|
@@ -221,6 +238,7 @@ export async function derivePrecomputeRequest(gitRoot, owner, repository, deadli
|
|
|
221
238
|
repository,
|
|
222
239
|
baseCommit: baseSha,
|
|
223
240
|
workCommit,
|
|
241
|
+
observedAt,
|
|
224
242
|
cacheKey,
|
|
225
243
|
patchGzBase64: patchGzip.toString('base64'),
|
|
226
244
|
patchSha256: createHash('sha256').update(patchGzip).digest('hex'),
|
|
@@ -231,16 +249,75 @@ function precomputeResult(response) {
|
|
|
231
249
|
if (!response || typeof response !== 'object' || Array.isArray(response)) {
|
|
232
250
|
throw new PrecomputeFailure('The precompute service returned an invalid response.');
|
|
233
251
|
}
|
|
234
|
-
const { status, runId } = response;
|
|
235
|
-
|
|
236
|
-
|
|
252
|
+
const { status, runId, walker: rawWalker } = response;
|
|
253
|
+
const walker = rawWalker === undefined ? undefined : readWalkerStatus(rawWalker);
|
|
254
|
+
if (status === 'queued' && typeof runId === 'string' && isRunId(runId)) {
|
|
255
|
+
return { outcome: 'enqueued', runId, ...(walker === undefined ? {} : { walker }) };
|
|
237
256
|
}
|
|
238
257
|
if (status === 'cached')
|
|
239
|
-
return { outcome: 'cached' };
|
|
258
|
+
return { outcome: 'cached', ...(walker === undefined ? {} : { walker }) };
|
|
240
259
|
if (status === 'existing')
|
|
241
|
-
return { outcome: 'existing' };
|
|
260
|
+
return { outcome: 'existing', ...(walker === undefined ? {} : { walker }) };
|
|
242
261
|
throw new PrecomputeFailure('The precompute service returned an invalid response.');
|
|
243
262
|
}
|
|
263
|
+
/** One authenticated submission of the already captured request. No source reads or retries. */
|
|
264
|
+
export const deliverPrecomputeRequest = async (request, signal) => {
|
|
265
|
+
signal.throwIfAborted();
|
|
266
|
+
const auth = await resolveAuthContext({ owner: request.owner, repo: request.repository });
|
|
267
|
+
signal.throwIfAborted();
|
|
268
|
+
return precomputeResult(await haystackJson('/api/agent/cloud-verifier/front-half', auth.token, { method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
269
|
+
body: JSON.stringify(request), signal }));
|
|
270
|
+
};
|
|
271
|
+
function readWalkerStatus(value) {
|
|
272
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
273
|
+
const record = value;
|
|
274
|
+
if (Object.keys(record).length === 2) {
|
|
275
|
+
if ((record.status === 'queued' || record.status === 'existing')
|
|
276
|
+
&& typeof record.runId === 'string' && isRunId(record.runId))
|
|
277
|
+
return { status: record.status, runId: record.runId };
|
|
278
|
+
if (record.status === 'unavailable'
|
|
279
|
+
&& (record.reason === 'runner-unavailable' || record.reason === 'enqueue-failed'))
|
|
280
|
+
return { status: record.status, reason: record.reason };
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
throw new PrecomputeFailure('The precompute service returned an invalid walker status.');
|
|
284
|
+
}
|
|
285
|
+
function walkerMetadataDirectory(gitRoot, deadline) {
|
|
286
|
+
const common = gitOutput(['rev-parse', '--git-common-dir'], gitRoot, deadline);
|
|
287
|
+
return join(isAbsolute(common) ? common : resolve(gitRoot, common), 'haystack');
|
|
288
|
+
}
|
|
289
|
+
function saveWalkerStatus(directory, request, walker) {
|
|
290
|
+
// Frozen local snapshot shape: {observedAt, repository, sourceCommit, walker}.
|
|
291
|
+
// Git's common metadata directory is shared by linked worktrees and excluded
|
|
292
|
+
// from the synthetic source tree. Rename publishes one complete JSON record.
|
|
293
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
294
|
+
if (!lstatSync(directory).isDirectory())
|
|
295
|
+
throw new PrecomputeFailure('Could not save the walker status.');
|
|
296
|
+
const temporary = join(directory, `.walker-status-${randomUUID()}.tmp`);
|
|
297
|
+
const descriptor = openSync(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600);
|
|
298
|
+
try {
|
|
299
|
+
try {
|
|
300
|
+
writeFileSync(descriptor, JSON.stringify({ observedAt: new Date(request.observedAt).toISOString(),
|
|
301
|
+
repository: `${request.owner}/${request.repository}`.toLowerCase(), sourceCommit: request.workCommit,
|
|
302
|
+
walker: readWalkerStatus(walker) }) + '\n');
|
|
303
|
+
fsyncSync(descriptor);
|
|
304
|
+
}
|
|
305
|
+
finally {
|
|
306
|
+
closeSync(descriptor);
|
|
307
|
+
}
|
|
308
|
+
renameSync(temporary, join(directory, 'walker-status.json'));
|
|
309
|
+
const parent = openSync(directory, constants.O_RDONLY);
|
|
310
|
+
try {
|
|
311
|
+
fsyncSync(parent);
|
|
312
|
+
}
|
|
313
|
+
finally {
|
|
314
|
+
closeSync(parent);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
finally {
|
|
318
|
+
rmSync(temporary, { force: true });
|
|
319
|
+
}
|
|
320
|
+
}
|
|
244
321
|
function normalModeFailure(error) {
|
|
245
322
|
if (error instanceof PrecomputeFailure)
|
|
246
323
|
return error;
|
|
@@ -256,9 +333,14 @@ function normalModeFailure(error) {
|
|
|
256
333
|
}
|
|
257
334
|
return new PrecomputeFailure(`The precompute request failed (HTTP ${error.status}).`);
|
|
258
335
|
}
|
|
336
|
+
// The deadline fired while the POST was in flight. The server may still
|
|
337
|
+
// have accepted the job, so say what happened rather than "failed".
|
|
338
|
+
if (error instanceof Error && error.name === 'AbortError') {
|
|
339
|
+
return new PrecomputeFailure('Precompute timed out.');
|
|
340
|
+
}
|
|
259
341
|
return new PrecomputeFailure('The precompute request failed.');
|
|
260
342
|
}
|
|
261
|
-
function printResult(outcome, runId) {
|
|
343
|
+
function printResult(outcome, runId, walker) {
|
|
262
344
|
if (outcome === 'enqueued') {
|
|
263
345
|
console.log(chalk.green(`Precompute request queued${runId ? ` (${runId})` : ''}.`));
|
|
264
346
|
}
|
|
@@ -268,18 +350,36 @@ function printResult(outcome, runId) {
|
|
|
268
350
|
else if (outcome === 'existing') {
|
|
269
351
|
console.log(chalk.green('A matching precompute request already exists.'));
|
|
270
352
|
}
|
|
353
|
+
if (walker?.status === 'queued')
|
|
354
|
+
console.log(`Walker queued (${walker.runId}).`);
|
|
355
|
+
else if (walker?.status === 'existing')
|
|
356
|
+
console.log(`A matching walker run already exists (${walker.runId}).`);
|
|
357
|
+
else if (walker?.status === 'unavailable')
|
|
358
|
+
console.log(walker.reason === 'runner-unavailable'
|
|
359
|
+
? 'Walker unavailable: no runner is available for this repository.'
|
|
360
|
+
: 'Walker unavailable: the enqueue request failed.');
|
|
361
|
+
else
|
|
362
|
+
console.log('Walker status was not reported by the service.');
|
|
271
363
|
}
|
|
272
364
|
export async function verifyPrecomputeCommand(options) {
|
|
273
365
|
const startedAt = Date.now();
|
|
274
|
-
const
|
|
366
|
+
const totalWallMs = options.hook ? HOOK_WALL_MS : EXPLICIT_WALL_MS;
|
|
367
|
+
const deadline = startedAt + totalWallMs;
|
|
275
368
|
let outcome = 'error';
|
|
276
369
|
try {
|
|
277
370
|
const gitRoot = findGitRoot();
|
|
278
371
|
if (!gitRoot) {
|
|
279
372
|
throw new PrecomputeFailure('Run this command from a git repository.');
|
|
280
373
|
}
|
|
374
|
+
const metadataDirectory = walkerMetadataDirectory(gitRoot, deadline);
|
|
281
375
|
const repository = resolveOriginRepository(gitRoot);
|
|
282
376
|
const derivation = await derivePrecomputeRequest(gitRoot, repository.owner, repository.repository, deadline);
|
|
377
|
+
if (options.hook) {
|
|
378
|
+
remainingMs(deadline);
|
|
379
|
+
await startPrecomputeDelivery(metadataDirectory, derivation.request);
|
|
380
|
+
outcome = 'pending';
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
283
383
|
let token;
|
|
284
384
|
try {
|
|
285
385
|
const auth = await resolveAuthContext({
|
|
@@ -312,8 +412,16 @@ export async function verifyPrecomputeCommand(options) {
|
|
|
312
412
|
}
|
|
313
413
|
const result = precomputeResult(response);
|
|
314
414
|
outcome = result.outcome;
|
|
415
|
+
if (result.walker) {
|
|
416
|
+
try {
|
|
417
|
+
saveWalkerStatus(metadataDirectory, derivation.request, result.walker);
|
|
418
|
+
}
|
|
419
|
+
catch {
|
|
420
|
+
throw new PrecomputeFailure('Precompute response received, but its walker status could not be saved.');
|
|
421
|
+
}
|
|
422
|
+
}
|
|
315
423
|
if (!options.hook)
|
|
316
|
-
printResult(outcome, result.runId);
|
|
424
|
+
printResult(outcome, result.runId, result.walker);
|
|
317
425
|
}
|
|
318
426
|
catch (error) {
|
|
319
427
|
if (!options.hook)
|
|
@@ -321,7 +429,7 @@ export async function verifyPrecomputeCommand(options) {
|
|
|
321
429
|
}
|
|
322
430
|
finally {
|
|
323
431
|
const wallMs = Date.now() - startedAt;
|
|
324
|
-
trackPrecomputeEvent(outcome, wallMs, Math.max(1,
|
|
432
|
+
trackPrecomputeEvent(outcome, wallMs, options.hook ? 1 : Math.max(1, totalWallMs - wallMs));
|
|
325
433
|
if (options.hook)
|
|
326
434
|
process.exitCode = 0;
|
|
327
435
|
}
|
|
@@ -6,6 +6,12 @@ export interface VerifyOptions {
|
|
|
6
6
|
base?: string;
|
|
7
7
|
head?: string;
|
|
8
8
|
intentTranscript?: string;
|
|
9
|
+
intentFile?: string;
|
|
10
|
+
idempotencyKey?: string;
|
|
11
|
+
account?: string;
|
|
12
|
+
wait?: boolean;
|
|
13
|
+
interval?: string;
|
|
14
|
+
timeout?: string;
|
|
9
15
|
json?: boolean;
|
|
10
16
|
open?: boolean;
|
|
11
17
|
/** Internal reproducibility option: execute an already-frozen plan. */
|
package/dist/commands/verify.js
CHANGED
|
@@ -107,6 +107,28 @@ export async function startVerificationRun(specimenArg, options) {
|
|
|
107
107
|
return parseStartRunResponse(raw);
|
|
108
108
|
}
|
|
109
109
|
export async function verifyCommand(specimenArg, options) {
|
|
110
|
+
if (specimenArg === undefined && (options.repo !== undefined || options.base !== undefined || options.head !== undefined)) {
|
|
111
|
+
if (!options.repo || !options.base || !options.head) {
|
|
112
|
+
throw new Error('Repository verification requires --repo owner/repo, --base <ref>, and --head <ref>.');
|
|
113
|
+
}
|
|
114
|
+
if (options.intentTranscript !== undefined) {
|
|
115
|
+
throw new Error('Repository verification accepts optional --intent-file JSON with problem, goal, and intended_outcomes.');
|
|
116
|
+
}
|
|
117
|
+
const { verifyHostedStartCommand } = await import('./verify-hosted.js');
|
|
118
|
+
await verifyHostedStartCommand(options.repo, {
|
|
119
|
+
provider: 'haystack',
|
|
120
|
+
base: options.base,
|
|
121
|
+
head: options.head,
|
|
122
|
+
intentFile: options.intentFile,
|
|
123
|
+
idempotencyKey: options.idempotencyKey,
|
|
124
|
+
account: options.account,
|
|
125
|
+
wait: options.wait,
|
|
126
|
+
interval: options.interval,
|
|
127
|
+
timeout: options.timeout,
|
|
128
|
+
json: options.json,
|
|
129
|
+
});
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
110
132
|
if (!options.json) {
|
|
111
133
|
console.error(chalk.dim('Preparing the cloud verification run…'));
|
|
112
134
|
}
|
package/dist/index.js
CHANGED
|
@@ -174,25 +174,35 @@ const verify = program
|
|
|
174
174
|
.command('verify')
|
|
175
175
|
.description('Verify a change in matched cloud universes, or bisect a historical regression')
|
|
176
176
|
.argument('[specimen]', 'Launch specimen to run (default: all)')
|
|
177
|
-
.option('--repo <
|
|
177
|
+
.option('--repo <owner/repo>', 'GitHub repository to verify through the authenticated hosted service')
|
|
178
178
|
.option('--base <ref>', 'Base revision for a repository run')
|
|
179
179
|
.option('--head <ref>', 'Head revision for a repository run')
|
|
180
|
+
.option('--intent-file <path>', 'Optional intent JSON with problem, goal, and intended_outcomes')
|
|
181
|
+
.option('--idempotency-key <key>', 'Stable retry key (default: derived from repository, refs, and intent)')
|
|
182
|
+
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
183
|
+
.addOption(new Option('--no-wait', 'Return after the hosted run is queued').default(undefined))
|
|
184
|
+
.option('--interval <seconds>', 'Hosted polling interval (default 5)')
|
|
185
|
+
.option('--timeout <minutes>', 'Maximum hosted wait (default 35)')
|
|
180
186
|
.option('--intent-transcript <file>', 'Coding-agent transcript used as intent evidence')
|
|
181
187
|
.option('--server <url>', 'Verifier control-plane URL (default: http://127.0.0.1:3000)')
|
|
182
188
|
.option('--token <token>', 'Bearer token for a control plane that requires auth (or HAYSTACK_VERIFY_TOKEN)')
|
|
183
189
|
.option('--no-open', 'Do not open the live results page')
|
|
184
|
-
.option('--json', 'Machine-readable run
|
|
190
|
+
.option('--json', 'Machine-readable run status and results')
|
|
185
191
|
.addHelpText('after', `
|
|
186
192
|
Launch specimens:
|
|
187
193
|
posthog-js-478 Replay PostHog/posthog-js PR #478 in matched browser environments (latent-issue mode)
|
|
188
194
|
analytics-js-255 Identity routing regression with sandbox bisect
|
|
189
195
|
all Run each reproduced specimen as an isolated chapter (default)
|
|
190
196
|
|
|
191
|
-
|
|
197
|
+
Repository verification uses the authenticated hosted fleet and resolves refs
|
|
198
|
+
once. Intent is optional. Hosted status also resumes these runs.
|
|
199
|
+
|
|
200
|
+
For launch specimens, run the local control plane from the Haystack repository:
|
|
192
201
|
pnpm dev
|
|
193
202
|
|
|
194
203
|
Examples:
|
|
195
|
-
haystack verify
|
|
204
|
+
haystack verify --repo owner/repo --base main --head feature --json
|
|
205
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path>
|
|
196
206
|
haystack verify posthog-js-478 # start a run, then:
|
|
197
207
|
haystack verify reproducibility posthog-js-478 --runs 3
|
|
198
208
|
haystack verify watch # block until it finishes (exit 1 = tests failed)
|
|
@@ -235,6 +245,7 @@ hostedVerify
|
|
|
235
245
|
// validation below accepts the inherited values and still fails closed.
|
|
236
246
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
237
247
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
248
|
+
.option('--intent-file <path>', 'Required for this alias: JSON with exactly problem, goal, and intended_outcomes')
|
|
238
249
|
.option('--idempotency-key <key>', 'Stable retry key (default: deterministic from repository and commits)')
|
|
239
250
|
.option('--replay-plan-from <run-id>', 'Reuse the exact validated risk plan from a completed owned run')
|
|
240
251
|
.option('--account <login>', 'Use a specific saved Haystack account')
|
|
@@ -250,9 +261,9 @@ fresh production execution of the same commit pair.
|
|
|
250
261
|
a schema-9 terminal receipt with replay provenance.
|
|
251
262
|
|
|
252
263
|
Examples:
|
|
253
|
-
haystack verify hosted start owner/repo --base <sha> --head <sha>
|
|
254
|
-
haystack verify hosted start owner/repo --base <sha> --head <sha> --replay-plan-from cv_<48 lowercase hex characters>
|
|
255
|
-
haystack verify hosted start owner/repo --base <sha> --head <sha> --no-wait --json
|
|
264
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path>
|
|
265
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path> --replay-plan-from cv_<48 lowercase hex characters>
|
|
266
|
+
haystack verify hosted start owner/repo --base <sha> --head <sha> --intent-file <path> --no-wait --json
|
|
256
267
|
`)
|
|
257
268
|
.action(async (repository, _options, cmd) => {
|
|
258
269
|
const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
|
|
@@ -283,6 +294,7 @@ hostedVerify
|
|
|
283
294
|
.argument('<repository>', 'Exact GitHub owner/repository name')
|
|
284
295
|
.option('--base <sha>', 'Required: exact 40-character lowercase base commit SHA')
|
|
285
296
|
.option('--head <sha>', 'Required: exact 40-character lowercase head commit SHA')
|
|
297
|
+
.option('--intent-file <path>', 'Required: agent-supplied JSON reused for every hosted run in the series')
|
|
286
298
|
.option('--runs <n>', 'Number of sequential production runs (default 3; min 2, max 10)')
|
|
287
299
|
.option('--series-key <key>', 'Stable series key for resumable idempotent retries')
|
|
288
300
|
.option('--fresh-plans', 'Run the planner independently for every run instead of replaying the first plan')
|
|
@@ -302,8 +314,8 @@ provenance, and no inconclusive cells. Use this mode to detect planner drift;
|
|
|
302
314
|
use the default mode to isolate executor or assessment drift.
|
|
303
315
|
|
|
304
316
|
Example:
|
|
305
|
-
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3
|
|
306
|
-
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --runs 3 --fresh-plans
|
|
317
|
+
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --intent-file <path> --runs 3
|
|
318
|
+
haystack verify hosted reproducibility owner/repo --base <sha> --head <sha> --intent-file <path> --runs 3 --fresh-plans
|
|
307
319
|
`)
|
|
308
320
|
.action(async (repository, _options, cmd) => {
|
|
309
321
|
const options = { ...cmd.optsWithGlobals(), ...cmd.opts() };
|
|
@@ -363,6 +375,34 @@ verify
|
|
|
363
375
|
const { verifyStatusCommand } = await import('./commands/verify-ops.js');
|
|
364
376
|
return runPublicCommand(() => verifyStatusCommand(run, options), options.json);
|
|
365
377
|
});
|
|
378
|
+
verify
|
|
379
|
+
.command('cases <run>')
|
|
380
|
+
.description('List cloud verification cases available to explore for 24 hours after completion')
|
|
381
|
+
.option('--repo <owner/repo>', 'Required exact GitHub repository; uses your Haystack login')
|
|
382
|
+
.option('--json', 'Machine-readable case IDs and expiry times')
|
|
383
|
+
.action(async (run, _options, cmd) => {
|
|
384
|
+
const options = cmd.optsWithGlobals();
|
|
385
|
+
const { verifyCasesCommand } = await import('./commands/verify-explore.js');
|
|
386
|
+
return runPublicCommand(() => verifyCasesCommand(run, options), options.json);
|
|
387
|
+
});
|
|
388
|
+
verify
|
|
389
|
+
.command('explore <run> <case-id>')
|
|
390
|
+
.description('Open or use your own copy of a saved fleet case; never changes the verification original')
|
|
391
|
+
.option('--repo <owner/repo>', 'Required exact GitHub repository; uses your Haystack login')
|
|
392
|
+
.option('--side <side>', 'base or head (default head)')
|
|
393
|
+
.option('--exec-json <argv>', 'Command arguments as a JSON array, executed inside your working sandbox')
|
|
394
|
+
.option('--cwd <path>', 'Absolute guest directory (default /home/user/pj)')
|
|
395
|
+
.option('--command-timeout <seconds>', 'Command timeout, 1–300 seconds (default 60)')
|
|
396
|
+
.option('--close', 'Discard your working copy; leave the saved case unchanged')
|
|
397
|
+
.option('--action <id>', 'Read/follow an existing action without submitting another command')
|
|
398
|
+
.option('--request-id <uuid>', 'Idempotency ID for a new request; reuse only with identical arguments')
|
|
399
|
+
.option('--no-wait', 'Return after submitting the action')
|
|
400
|
+
.option('--json', 'Machine-readable action and command output')
|
|
401
|
+
.action(async (run, caseId, _options, cmd) => {
|
|
402
|
+
const options = cmd.optsWithGlobals();
|
|
403
|
+
const { verifyExploreCommand } = await import('./commands/verify-explore.js');
|
|
404
|
+
return runPublicCommand(() => verifyExploreCommand(run, caseId, options), options.json);
|
|
405
|
+
});
|
|
366
406
|
verify
|
|
367
407
|
.command('watch')
|
|
368
408
|
.description('Follow a run until it finishes; exit 0 all passed, 1 tests failed, 2 run/infra broke')
|
|
@@ -30,6 +30,22 @@ export interface DesignVerifierAgentCase {
|
|
|
30
30
|
errors: string[];
|
|
31
31
|
} | null;
|
|
32
32
|
}
|
|
33
|
+
export interface DesignVerifierRunHealthLoss {
|
|
34
|
+
code: string;
|
|
35
|
+
owner: 'harness' | 'case';
|
|
36
|
+
impact: 'degraded' | 'invalid';
|
|
37
|
+
count: number;
|
|
38
|
+
caseIds: string[];
|
|
39
|
+
obligationIds: string[];
|
|
40
|
+
slotIds: string[];
|
|
41
|
+
sides: string[];
|
|
42
|
+
}
|
|
43
|
+
export interface DesignVerifierRunHealth {
|
|
44
|
+
version: 'dv-run-health-v1';
|
|
45
|
+
status: 'healthy' | 'degraded' | 'invalid';
|
|
46
|
+
admitted: number;
|
|
47
|
+
losses: DesignVerifierRunHealthLoss[];
|
|
48
|
+
}
|
|
33
49
|
export interface DesignVerifierAgentReport {
|
|
34
50
|
ok: boolean;
|
|
35
51
|
status: 'complete' | 'failed';
|
|
@@ -47,6 +63,7 @@ export interface DesignVerifierAgentReport {
|
|
|
47
63
|
timings: DesignVerifierTimingSummary | null;
|
|
48
64
|
changedFiles: string[];
|
|
49
65
|
cases: DesignVerifierAgentCase[];
|
|
66
|
+
runHealth: DesignVerifierRunHealth | null;
|
|
50
67
|
inspectCommand: string;
|
|
51
68
|
replay: Record<string, unknown> | null;
|
|
52
69
|
error: string | null;
|
|
@@ -30,6 +30,54 @@ function universeSummaries(value, caseId) {
|
|
|
30
30
|
['control-before', 'before', 'after'].includes(universe.role) &&
|
|
31
31
|
universe.lease && typeof universe.lease.available === 'boolean');
|
|
32
32
|
}
|
|
33
|
+
function runHealthOf(value) {
|
|
34
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
35
|
+
return null;
|
|
36
|
+
const health = value;
|
|
37
|
+
if (health.version !== 'dv-run-health-v1' ||
|
|
38
|
+
!['healthy', 'degraded', 'invalid'].includes(health.status) ||
|
|
39
|
+
!Number.isInteger(health.admitted) || health.admitted < 0 ||
|
|
40
|
+
!Array.isArray(health.losses))
|
|
41
|
+
return null;
|
|
42
|
+
const losses = health.losses.filter((loss) => loss && typeof loss === 'object' &&
|
|
43
|
+
typeof loss.code === 'string' && loss.code.length > 0 &&
|
|
44
|
+
['harness', 'case'].includes(loss.owner) &&
|
|
45
|
+
['degraded', 'invalid'].includes(loss.impact) &&
|
|
46
|
+
Number.isInteger(loss.count) && loss.count > 0 &&
|
|
47
|
+
[loss.caseIds, loss.obligationIds, loss.slotIds, loss.sides]
|
|
48
|
+
.every(items => Array.isArray(items) && items.every(item => typeof item === 'string')));
|
|
49
|
+
if (losses.length !== health.losses.length)
|
|
50
|
+
return null;
|
|
51
|
+
const invalidHarnessLoss = losses.some(loss => loss.owner === 'harness' && loss.impact === 'invalid');
|
|
52
|
+
const expectedStatus = health.admitted === 0 || invalidHarnessLoss
|
|
53
|
+
? 'invalid'
|
|
54
|
+
: losses.length > 0 ? 'degraded' : 'healthy';
|
|
55
|
+
if (health.status !== expectedStatus)
|
|
56
|
+
return null;
|
|
57
|
+
return {
|
|
58
|
+
version: health.version,
|
|
59
|
+
status: health.status,
|
|
60
|
+
admitted: health.admitted,
|
|
61
|
+
losses,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function missingRunHealth() {
|
|
65
|
+
return {
|
|
66
|
+
version: 'dv-run-health-v1',
|
|
67
|
+
status: 'invalid',
|
|
68
|
+
admitted: 0,
|
|
69
|
+
losses: [{
|
|
70
|
+
code: 'run-health-missing',
|
|
71
|
+
owner: 'harness',
|
|
72
|
+
impact: 'invalid',
|
|
73
|
+
count: 1,
|
|
74
|
+
caseIds: [],
|
|
75
|
+
obligationIds: [],
|
|
76
|
+
slotIds: [],
|
|
77
|
+
sides: [],
|
|
78
|
+
}],
|
|
79
|
+
};
|
|
80
|
+
}
|
|
33
81
|
export function buildDesignVerifierAgentReport(run) {
|
|
34
82
|
const repository = `${run.owner}/${run.repo}`;
|
|
35
83
|
if (run.status !== 'succeeded' || !run.result) {
|
|
@@ -47,6 +95,7 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
47
95
|
timings: null,
|
|
48
96
|
changedFiles: [],
|
|
49
97
|
cases: [],
|
|
98
|
+
runHealth: null,
|
|
50
99
|
inspectCommand: `haystack design-verify ${repository}#${run.pr} --run ${run.runId}`,
|
|
51
100
|
replay: null,
|
|
52
101
|
error: run.error,
|
|
@@ -55,6 +104,7 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
55
104
|
}
|
|
56
105
|
const verdict = run.result.verdict || {};
|
|
57
106
|
const summary = verdict.summary || {};
|
|
107
|
+
const runHealth = runHealthOf(verdict.runHealth) || missingRunHealth();
|
|
58
108
|
const findings = Array.isArray(verdict.findings) ? verdict.findings : [];
|
|
59
109
|
const confirmations = Array.isArray(verdict.confirmations) ? verdict.confirmations : [];
|
|
60
110
|
const findingsByCase = new Map();
|
|
@@ -133,7 +183,19 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
133
183
|
let headline;
|
|
134
184
|
let explanation;
|
|
135
185
|
let exitCode;
|
|
136
|
-
if (
|
|
186
|
+
if (runHealth.status === 'invalid') {
|
|
187
|
+
agentVerdict = 'inconclusive';
|
|
188
|
+
headline = 'VERIFICATION INCONCLUSIVE';
|
|
189
|
+
explanation = 'No admissible evidence — this is not a pass.';
|
|
190
|
+
exitCode = 2;
|
|
191
|
+
}
|
|
192
|
+
else if (runHealth.status === 'degraded') {
|
|
193
|
+
agentVerdict = 'inconclusive';
|
|
194
|
+
headline = 'VERIFICATION INCONCLUSIVE';
|
|
195
|
+
explanation = 'Evidence losses degraded this run; inspect the loss table before drawing a conclusion.';
|
|
196
|
+
exitCode = 2;
|
|
197
|
+
}
|
|
198
|
+
else if (findings.length > 0) {
|
|
137
199
|
agentVerdict = 'anomaly_candidate';
|
|
138
200
|
headline = 'ANOMALY CANDIDATE';
|
|
139
201
|
explanation = findingExplanation(String(findings[0]?.kind || ''));
|
|
@@ -171,6 +233,7 @@ export function buildDesignVerifierAgentReport(run) {
|
|
|
171
233
|
timings: run.result.timings || null,
|
|
172
234
|
changedFiles: run.result.changedFiles,
|
|
173
235
|
cases,
|
|
236
|
+
runHealth,
|
|
174
237
|
inspectCommand: `haystack design-verify ${repository}#${run.pr} --run ${run.runId}`,
|
|
175
238
|
replay: run.result.replay,
|
|
176
239
|
error: null,
|
|
@@ -243,6 +306,22 @@ export function formatDesignVerifierAgentReport(report) {
|
|
|
243
306
|
lines.push(` orchestration/finalization not recorded in a phase: ${preciseDurationLabel(report.timings.unattributedMs)}`);
|
|
244
307
|
}
|
|
245
308
|
}
|
|
309
|
+
if (report.runHealth && report.runHealth.status !== 'healthy') {
|
|
310
|
+
lines.push('', `Run health: ${report.runHealth.status}; admitted evidence: ${report.runHealth.admitted}`);
|
|
311
|
+
if (report.runHealth.losses.length > 0) {
|
|
312
|
+
lines.push('Evidence loss table:', ' owner impact count loss / stable identifiers');
|
|
313
|
+
for (const loss of report.runHealth.losses) {
|
|
314
|
+
const identifiers = [
|
|
315
|
+
...loss.caseIds,
|
|
316
|
+
...loss.obligationIds,
|
|
317
|
+
...loss.slotIds,
|
|
318
|
+
...loss.sides,
|
|
319
|
+
];
|
|
320
|
+
lines.push(` ${loss.owner.padEnd(7)} ${loss.impact.padEnd(9)} ${String(loss.count).padStart(5)} ` +
|
|
321
|
+
`${loss.code}${identifiers.length ? `: ${identifiers.join(', ')}` : ''}`);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
}
|
|
246
325
|
for (const testCase of report.cases) {
|
|
247
326
|
lines.push('', `Test: ${testCase.title}`);
|
|
248
327
|
if (testCase.story)
|
package/dist/utils/git.js
CHANGED
|
@@ -386,7 +386,11 @@ function gitCommandOptions(options) {
|
|
|
386
386
|
/** True if `ref` resolves to a commit in the current repo. */
|
|
387
387
|
function gitRefResolves(ref, options = {}) {
|
|
388
388
|
try {
|
|
389
|
-
|
|
389
|
+
// `--end-of-options`, not `--`: rev-parse treats everything after `--` as
|
|
390
|
+
// a pathspec, so `-- refs/remotes/origin/main^{commit}` exits 1 for every
|
|
391
|
+
// ref and the caller fell back to a bare local `main` (which a fresh clone
|
|
392
|
+
// does not have). Verified on git 2.39: `--` rc=1, `--end-of-options` rc=0.
|
|
393
|
+
execFileSync('git', ['rev-parse', '--verify', '--quiet', '--end-of-options', `${ref}^{commit}`], {
|
|
390
394
|
...gitCommandOptions(options),
|
|
391
395
|
});
|
|
392
396
|
return true;
|
package/dist/utils/telemetry.js
CHANGED
|
@@ -103,7 +103,10 @@ export function trackError(errorType, metadata) {
|
|
|
103
103
|
method: 'POST',
|
|
104
104
|
headers: { 'Content-Type': 'application/json' },
|
|
105
105
|
body: JSON.stringify(event),
|
|
106
|
-
}).catch(() => {
|
|
106
|
+
}).catch(() => {
|
|
107
|
+
// Documented fire-and-forget product telemetry: delivery cannot affect
|
|
108
|
+
// the CLI command being measured.
|
|
109
|
+
});
|
|
107
110
|
}
|
|
108
111
|
catch {
|
|
109
112
|
// Telemetry must never crash the CLI
|
|
@@ -135,7 +138,10 @@ export function trackSetupEvent(step, metadata) {
|
|
|
135
138
|
method: 'POST',
|
|
136
139
|
headers: { 'Content-Type': 'application/json' },
|
|
137
140
|
body: JSON.stringify(event),
|
|
138
|
-
}).catch(() => {
|
|
141
|
+
}).catch(() => {
|
|
142
|
+
// Documented fire-and-forget product telemetry: delivery cannot affect
|
|
143
|
+
// the CLI command being measured.
|
|
144
|
+
});
|
|
139
145
|
}
|
|
140
146
|
catch {
|
|
141
147
|
// Telemetry must never crash the CLI
|