@haystackeditor/cli 0.15.30 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -60
- package/dist/assets/hooks/scripts/commit-msg.sh +3 -0
- package/dist/assets/hooks/scripts/post-commit.sh +3 -0
- package/dist/assets/hooks/scripts/pre-commit.sh +11 -6
- package/dist/assets/hooks/scripts/pre-push.sh +3 -0
- package/dist/assets/hooks/scripts/prepare-commit-msg.sh +3 -0
- package/dist/commands/case-batch-contract.js +608 -0
- package/dist/commands/case-batch.js +713 -0
- package/dist/commands/cloud-verifier-identity-census.js +5 -2
- package/dist/commands/combination-search-hook.js +139 -0
- package/dist/commands/design-verify.js +27 -1
- package/dist/commands/dismiss.js +2 -1
- package/dist/commands/hooks.js +66 -7
- package/dist/commands/install-session-hooks.js +131 -44
- package/dist/commands/policy.js +32 -45
- package/dist/commands/precompute-delivery.js +6 -1
- package/dist/commands/scaffold-provisional-universe.js +8 -10
- package/dist/commands/setup.js +32 -10
- package/dist/commands/submit.js +7 -2
- package/dist/commands/telemetry.js +17 -2
- package/dist/commands/triage.js +2 -1
- package/dist/commands/verify-explore.js +1 -0
- package/dist/commands/verify-history.js +152 -0
- package/dist/commands/verify-hosted-mcp.js +3 -12
- package/dist/commands/verify-hosted-reproducibility.js +49 -547
- package/dist/commands/verify-hosted.js +180 -163
- package/dist/commands/verify-precompute.js +57 -10
- package/dist/commands/verify.js +51 -139
- package/dist/index.js +264 -348
- package/dist/lazy.js +8 -0
- package/dist/schema.js +1 -0
- package/dist/tools/detect.js +3 -24
- package/dist/triage/prompts.js +46 -45
- package/dist/triage/runner.js +119 -33
- package/dist/types/verify-history.js +1 -0
- package/dist/types.js +3 -3
- package/dist/utils/auth.js +14 -2
- package/dist/utils/design-verifier-api.js +23 -0
- package/dist/utils/design-verifier-history.js +79 -0
- package/dist/utils/git.js +60 -29
- package/dist/utils/github-api.js +14 -1
- package/dist/utils/haystack-api.js +26 -7
- package/dist/utils/hooks.js +43 -6
- package/dist/utils/prompter.js +25 -10
- package/dist/utils/safe-write.js +31 -0
- package/dist/utils/secret-paths.js +115 -0
- package/dist/utils/secrets.js +0 -1
- package/dist/utils/telemetry.js +34 -14
- package/dist/utils/update-check.js +151 -0
- package/package.json +18 -14
- package/schemas/case-batch.v1.json +184 -0
- package/schemas/cloud-verifier.v1.json +55 -100
- package/dist/commands/ask.d.ts +0 -14
- package/dist/commands/cloud-verifier-behaviors.d.ts +0 -27
- package/dist/commands/cloud-verifier-data-store-census.d.ts +0 -47
- package/dist/commands/cloud-verifier-data-store-drift.d.ts +0 -42
- package/dist/commands/cloud-verifier-identity-census.d.ts +0 -88
- package/dist/commands/cloud-verifier-materialization.d.ts +0 -16
- package/dist/commands/cloud-verifier-pascal-selector-census.d.ts +0 -29
- package/dist/commands/cloud-verifier-python-manifest-selector-census.d.ts +0 -27
- package/dist/commands/cloud-verifier-specialized-operational-census.d.ts +0 -51
- package/dist/commands/cloud-verifier-universe.d.ts +0 -31
- package/dist/commands/config.d.ts +0 -46
- package/dist/commands/design-verify.d.ts +0 -31
- package/dist/commands/dismiss.d.ts +0 -29
- package/dist/commands/hooks.d.ts +0 -13
- package/dist/commands/inbox.d.ts +0 -65
- package/dist/commands/init.d.ts +0 -10
- package/dist/commands/install-session-hooks.d.ts +0 -17
- package/dist/commands/login.d.ts +0 -8
- package/dist/commands/mcp.d.ts +0 -1
- package/dist/commands/policy.d.ts +0 -31
- package/dist/commands/pr-status.d.ts +0 -144
- package/dist/commands/pr.d.ts +0 -40
- package/dist/commands/precompute-delivery-contract.d.ts +0 -68
- package/dist/commands/precompute-delivery.d.ts +0 -20
- package/dist/commands/prepare-universe-review.d.ts +0 -115
- package/dist/commands/production-source-deny-policy.d.ts +0 -15
- package/dist/commands/request-review.d.ts +0 -26
- package/dist/commands/review.d.ts +0 -25
- package/dist/commands/rules.d.ts +0 -4
- package/dist/commands/scaffold-provisional-universe.d.ts +0 -468
- package/dist/commands/schema-cmd.d.ts +0 -2
- package/dist/commands/setup.d.ts +0 -28
- package/dist/commands/skills.d.ts +0 -8
- package/dist/commands/status.d.ts +0 -4
- package/dist/commands/submit.d.ts +0 -30
- package/dist/commands/system-map.d.ts +0 -42
- package/dist/commands/telemetry.d.ts +0 -53
- package/dist/commands/tokens.d.ts +0 -14
- package/dist/commands/triage.d.ts +0 -35
- package/dist/commands/verify-core.d.ts +0 -449
- package/dist/commands/verify-core.js +0 -789
- package/dist/commands/verify-explore.d.ts +0 -14
- package/dist/commands/verify-hosted-mcp.d.ts +0 -12
- package/dist/commands/verify-hosted-reproducibility.d.ts +0 -90
- package/dist/commands/verify-hosted.d.ts +0 -92
- package/dist/commands/verify-mcp.d.ts +0 -8
- package/dist/commands/verify-mcp.js +0 -517
- package/dist/commands/verify-ops.d.ts +0 -158
- package/dist/commands/verify-ops.js +0 -1148
- package/dist/commands/verify-precompute.d.ts +0 -30
- package/dist/commands/verify-reproducibility.d.ts +0 -85
- package/dist/commands/verify-reproducibility.js +0 -494
- package/dist/commands/verify-reseal.d.ts +0 -9
- package/dist/commands/verify-reseal.js +0 -148
- package/dist/commands/verify-sandboxes.d.ts +0 -95
- package/dist/commands/verify-sandboxes.js +0 -352
- package/dist/commands/verify.d.ts +0 -28
- package/dist/commands/webhooks.d.ts +0 -30
- package/dist/index.d.ts +0 -22
- package/dist/schema.d.ts +0 -28
- package/dist/states.d.ts +0 -29
- package/dist/tools/detect.d.ts +0 -50
- package/dist/triage/prompts.d.ts +0 -24
- package/dist/triage/runner.d.ts +0 -34
- package/dist/triage/types.d.ts +0 -42
- package/dist/types.d.ts +0 -1684
- package/dist/utils/action-output.d.ts +0 -24
- package/dist/utils/analysis-api.d.ts +0 -187
- package/dist/utils/auth.d.ts +0 -79
- package/dist/utils/config.d.ts +0 -24
- package/dist/utils/design-verifier-api.d.ts +0 -200
- package/dist/utils/design-verifier-result.d.ts +0 -74
- package/dist/utils/detect.d.ts +0 -43
- package/dist/utils/git.d.ts +0 -135
- package/dist/utils/github-api.d.ts +0 -104
- package/dist/utils/haystack-api.d.ts +0 -37
- package/dist/utils/hooks.d.ts +0 -12
- package/dist/utils/pending-state.d.ts +0 -40
- package/dist/utils/pr-ref.d.ts +0 -27
- package/dist/utils/prompter.d.ts +0 -85
- package/dist/utils/secrets.d.ts +0 -47
- package/dist/utils/telemetry.d.ts +0 -19
- /package/dist/commands/{precompute-delivery-worker.d.ts → combination-search-hook-contract.js} +0 -0
|
@@ -1,1148 +0,0 @@
|
|
|
1
|
-
// Operator commands over cloud-verifier runs: status, watch, list, show,
|
|
2
|
-
// artifacts. Data plumbing lives in verify-core.ts (shared with the MCP
|
|
3
|
-
// server); this file is terminal rendering and exit codes.
|
|
4
|
-
import { spawn } from 'node:child_process';
|
|
5
|
-
import { createHash } from 'node:crypto';
|
|
6
|
-
import * as fs from 'node:fs';
|
|
7
|
-
import * as os from 'node:os';
|
|
8
|
-
import * as path from 'node:path';
|
|
9
|
-
import chalk from 'chalk';
|
|
10
|
-
import { artifactFileOf, cellDetail, discoverVerifyRoot, exitCodeFor, findCell, findCellByStableSelector, loadRunAuto, listRunRows, resolveServer, runDir, setVerifyToken, summarizeRun, verifyAuthHeaders, waitForTerminal, } from './verify-core.js';
|
|
11
|
-
function statusGlyph(status) {
|
|
12
|
-
switch (status) {
|
|
13
|
-
case 'passed': return chalk.green('✓');
|
|
14
|
-
case 'failed': return chalk.red('✗');
|
|
15
|
-
case 'infra_error': return chalk.yellow('⚠');
|
|
16
|
-
case 'inconclusive': return chalk.yellow('?');
|
|
17
|
-
case 'queued': return chalk.dim('·');
|
|
18
|
-
default: return chalk.cyan('◷');
|
|
19
|
-
}
|
|
20
|
-
}
|
|
21
|
-
function emitJson(payload) {
|
|
22
|
-
process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
|
23
|
-
}
|
|
24
|
-
function renderStatus(summary) {
|
|
25
|
-
const state = summary.stale ? `${chalk.red('failed')} ${chalk.dim('(stale — orchestrator gone)')}`
|
|
26
|
-
: summary.status === 'complete' ? chalk.green(summary.status)
|
|
27
|
-
: summary.status === 'failed' ? chalk.red(summary.status)
|
|
28
|
-
: chalk.cyan(summary.status);
|
|
29
|
-
console.log(`${chalk.bold(summary.run_id)} — ${state}${summary.age ? chalk.dim(` (started ${summary.age} ago)`) : ''}`);
|
|
30
|
-
for (const chapter of summary.chapters) {
|
|
31
|
-
console.log(`${chalk.dim(chapter.repository)} ${chapter.pull_request} — ${chapter.label}`);
|
|
32
|
-
if (chapter.execution_mode === 'bisect' && chapter.bisect) {
|
|
33
|
-
const b = chapter.bisect;
|
|
34
|
-
const probeBits = Object.entries(b.probes_by_status).map(([k, v]) => `${v} ${k}`).join(', ');
|
|
35
|
-
console.log(` bisect ${b.status} — ${b.probes_total}/${b.total_candidates} candidates probed${probeBits ? ` (${probeBits})` : ''}`);
|
|
36
|
-
if (b.culprit_sha) {
|
|
37
|
-
console.log(` ${chalk.red('culprit')} ${b.culprit_sha.slice(0, 8)} ${b.culprit_subject ?? ''}${b.parent_sha ? chalk.dim(` (clean parent ${b.parent_sha.slice(0, 8)})`) : ''}`);
|
|
38
|
-
}
|
|
39
|
-
continue;
|
|
40
|
-
}
|
|
41
|
-
for (const test of chapter.tests) {
|
|
42
|
-
const verdict = test.verdict ? chalk.dim(` ${test.verdict}`) : '';
|
|
43
|
-
const duration = test.duration ? chalk.dim(` (${test.duration})`) : '';
|
|
44
|
-
const severity = test.severity ? chalk.dim(` [${test.severity}]`) : '';
|
|
45
|
-
console.log(` ${statusGlyph(test.status)} ${test.ordinal}. ${test.title}${severity}${verdict}${duration}`);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
const c = summary.counts;
|
|
49
|
-
const bisectOnly = c.total === 0 && summary.chapters.some(chapter => chapter.execution_mode === 'bisect');
|
|
50
|
-
if (!bisectOnly) {
|
|
51
|
-
console.log('');
|
|
52
|
-
console.log(`${c.total} tests: ${chalk.green(`${c.passed} passed`)}, ${c.failed > 0 ? chalk.red(`${c.failed} failed`) : `${c.failed} failed`}${c.could_not_run > 0 ? `, ${chalk.yellow(`${c.could_not_run} could not run`)}` : ''}`);
|
|
53
|
-
}
|
|
54
|
-
if (summary.live_environments.length > 0) {
|
|
55
|
-
console.log('');
|
|
56
|
-
console.log(chalk.bold('Live environments (failing tests):'));
|
|
57
|
-
for (const env of summary.live_environments) {
|
|
58
|
-
console.log(` ${env.ordinal}. ${chalk.cyan(env.url)} ${chalk.dim(`expires ${env.expires ?? env.expires_at ?? 'unknown'}`)}`);
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
if (summary.problems.length > 0) {
|
|
62
|
-
console.log('');
|
|
63
|
-
console.log(chalk.yellow.bold('Problems (ours, not findings):'));
|
|
64
|
-
for (const problem of summary.problems)
|
|
65
|
-
console.log(` ${chalk.yellow('!')} ${problem}`);
|
|
66
|
-
}
|
|
67
|
-
if (!isTerminalStatus(summary) && summary.last_event) {
|
|
68
|
-
console.log('');
|
|
69
|
-
console.log(chalk.dim(`Last event [${summary.last_event.phase}] ${summary.last_event.message}`));
|
|
70
|
-
}
|
|
71
|
-
console.log('');
|
|
72
|
-
console.log(chalk.dim(`Results: ${summary.results_url}`));
|
|
73
|
-
}
|
|
74
|
-
function isTerminalStatus(summary) {
|
|
75
|
-
return ['complete', 'failed'].includes(summary.status);
|
|
76
|
-
}
|
|
77
|
-
export async function verifyStatusCommand(runSelector, options) {
|
|
78
|
-
setVerifyToken(options.token);
|
|
79
|
-
const server = resolveServer(options.server);
|
|
80
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
81
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: runSelector || 'latest' });
|
|
82
|
-
const summary = summarizeRun(loaded, server);
|
|
83
|
-
if (options.json)
|
|
84
|
-
return emitJson(summary);
|
|
85
|
-
renderStatus(summary);
|
|
86
|
-
}
|
|
87
|
-
export async function verifyWatchCommand(runSelector, options) {
|
|
88
|
-
setVerifyToken(options.token);
|
|
89
|
-
const server = resolveServer(options.server);
|
|
90
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
91
|
-
const intervalMs = Math.max(1000, Number(options.interval ?? 3) * 1000);
|
|
92
|
-
const timeoutMs = options.timeout ? Number(options.timeout) * 60_000 : undefined;
|
|
93
|
-
if ((options.interval && Number.isNaN(Number(options.interval))) || (options.timeout && Number.isNaN(Number(options.timeout)))) {
|
|
94
|
-
throw new Error('--interval and --timeout take numbers (seconds and minutes respectively)');
|
|
95
|
-
}
|
|
96
|
-
const quiet = Boolean(options.json);
|
|
97
|
-
const result = await waitForTerminal({
|
|
98
|
-
server,
|
|
99
|
-
repoRoot,
|
|
100
|
-
selector: runSelector || 'latest',
|
|
101
|
-
intervalMs,
|
|
102
|
-
timeoutMs,
|
|
103
|
-
callbacks: quiet ? {} : {
|
|
104
|
-
onResolved: runId => console.error(chalk.dim(`Watching ${runId} (poll every ${intervalMs / 1000}s)…`)),
|
|
105
|
-
onEvent: event => console.error(`${chalk.dim(`[${event.phase}]`)} ${event.message}`),
|
|
106
|
-
onCellChange: (_chapterId, cell, previous) => console.error(` ${statusGlyph(cell.status)} test ${cell.ordinal} ${chalk.dim(cell.title.slice(0, 70))}: ${previous} → ${chalk.bold(cell.status)}`),
|
|
107
|
-
},
|
|
108
|
-
});
|
|
109
|
-
const summary = summarizeRun(result.loaded, server);
|
|
110
|
-
if (result.timedOut) {
|
|
111
|
-
if (options.json)
|
|
112
|
-
emitJson({ ...summary, timed_out: true });
|
|
113
|
-
else
|
|
114
|
-
console.error(chalk.yellow(`Timed out after ${options.timeout}m; run is still ${summary.status}.`));
|
|
115
|
-
process.exitCode = 2;
|
|
116
|
-
return;
|
|
117
|
-
}
|
|
118
|
-
if (options.json) {
|
|
119
|
-
emitJson({ ...summary, timed_out: false });
|
|
120
|
-
}
|
|
121
|
-
else {
|
|
122
|
-
console.error('');
|
|
123
|
-
renderStatus(summary);
|
|
124
|
-
}
|
|
125
|
-
process.exitCode = exitCodeFor(summary);
|
|
126
|
-
}
|
|
127
|
-
export async function verifyListCommand(options) {
|
|
128
|
-
setVerifyToken(options.token);
|
|
129
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
130
|
-
const limit = options.limit ? Number(options.limit) : 20;
|
|
131
|
-
if (Number.isNaN(limit) || limit < 1)
|
|
132
|
-
throw new Error('--limit takes a positive number');
|
|
133
|
-
const server = resolveServer(options.server);
|
|
134
|
-
const { rows, source, total } = await listRunRows({ server, repoRoot, limit });
|
|
135
|
-
if (options.json) {
|
|
136
|
-
return emitJson({ schema_version: '1.0.0', source, total, runs: rows });
|
|
137
|
-
}
|
|
138
|
-
if (rows.length === 0) {
|
|
139
|
-
console.log(chalk.dim('No runs found.'));
|
|
140
|
-
return;
|
|
141
|
-
}
|
|
142
|
-
const idWidth = Math.max(...rows.map(row => row.run_id.length));
|
|
143
|
-
for (const row of rows) {
|
|
144
|
-
const verdicts = row.tests > 0
|
|
145
|
-
? `${chalk.green(`${row.passed}✓`)} ${row.failed > 0 ? chalk.red(`${row.failed}✗`) : `${row.failed}✗`}${row.could_not_run ? chalk.yellow(` ${row.could_not_run}⚠`) : ''}`
|
|
146
|
-
: row.culprit_sha ? `culprit ${row.culprit_sha.slice(0, 8)}` : chalk.dim('—');
|
|
147
|
-
const status = row.status === 'complete' ? chalk.green(row.status)
|
|
148
|
-
: row.status === 'failed' ? chalk.red(row.status) : chalk.cyan(row.status);
|
|
149
|
-
console.log(`${row.run_id.padEnd(idWidth)} ${status.padEnd(10)} ${verdicts} ${chalk.dim(`${row.age} ago`)}`);
|
|
150
|
-
}
|
|
151
|
-
if (total > rows.length) {
|
|
152
|
-
console.log(chalk.dim(`… ${total - rows.length} older run(s); use --limit to see more.`));
|
|
153
|
-
}
|
|
154
|
-
console.log(chalk.dim(`\nDetails: haystack verify status <run> · UI: ${server}/dev/verify/<run>`));
|
|
155
|
-
}
|
|
156
|
-
function renderTimeline(label, universe) {
|
|
157
|
-
console.log(chalk.bold(` ${label} timeline (${universe.network.length} requests)`));
|
|
158
|
-
for (const entry of universe.network) {
|
|
159
|
-
const offset = entry.offset_ms !== null ? `+${(entry.offset_ms / 1000).toFixed(1)}s`.padStart(7) : ' ';
|
|
160
|
-
const eventName = entry.event_name ? chalk.dim(` (${entry.event_name})`) : '';
|
|
161
|
-
const bad = entry.status.startsWith('failed') || /^HTTP [45]/.test(entry.status);
|
|
162
|
-
console.log(` ${chalk.dim(offset)} ${entry.method ?? 'GET'} ${entry.path} ${bad ? chalk.red(entry.status) : chalk.dim(entry.status)}${eventName}`);
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
function renderSdkLog(label, universe) {
|
|
166
|
-
if (universe.sdk_log.length === 0)
|
|
167
|
-
return;
|
|
168
|
-
console.log(chalk.bold(` ${label} SDK log (${universe.sdk_log.length} lines)`));
|
|
169
|
-
for (const line of universe.sdk_log)
|
|
170
|
-
console.log(` ${chalk.dim(line)}`);
|
|
171
|
-
}
|
|
172
|
-
function renderShow(detail) {
|
|
173
|
-
const glyph = statusGlyph(detail.status);
|
|
174
|
-
console.log(`${glyph} ${chalk.bold(`Test ${detail.ordinal}: ${detail.title}`)}`);
|
|
175
|
-
console.log(chalk.dim(` ${detail.run_id} · ${detail.chapter_id} · ${detail.cell_id}${detail.severity ? ` · ${detail.severity}` : ''}${detail.duration ? ` · ${detail.duration}` : ''}`));
|
|
176
|
-
console.log('');
|
|
177
|
-
if (detail.status === 'infra_error') {
|
|
178
|
-
console.log(chalk.yellow(' This test could not run (infrastructure failure). That is our defect, not a verdict about the change.'));
|
|
179
|
-
if (detail.infrastructure_error) {
|
|
180
|
-
console.log(chalk.yellow(` ${detail.infrastructure_error.reason}: ${detail.infrastructure_error.message}`));
|
|
181
|
-
}
|
|
182
|
-
for (const failure of [...detail.base.driver_failures, ...detail.head.driver_failures]) {
|
|
183
|
-
console.log(chalk.yellow(` ${failure}`));
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
if (detail.verdict) {
|
|
187
|
-
console.log(` ${chalk.bold('Verdict:')} ${detail.verdict}`);
|
|
188
|
-
if (detail.explanation)
|
|
189
|
-
console.log(` ${detail.explanation.split('\n').join('\n ')}`);
|
|
190
|
-
if (detail.intent_evidence)
|
|
191
|
-
console.log(chalk.dim(` Intent evidence: ${detail.intent_evidence}`));
|
|
192
|
-
console.log('');
|
|
193
|
-
}
|
|
194
|
-
if (detail.oracle_readings.length > 0) {
|
|
195
|
-
console.log(chalk.bold(' Readings'));
|
|
196
|
-
for (const reading of detail.oracle_readings) {
|
|
197
|
-
const mark = reading.result === 'pass' ? chalk.green('pass') : reading.result === 'fail' ? chalk.red('FAIL') : chalk.yellow('unmeasurable');
|
|
198
|
-
const change = reading.changed ? chalk.yellow(`${JSON.stringify(reading.base)} → ${JSON.stringify(reading.head)}`) : chalk.dim(`${JSON.stringify(reading.head)} (unchanged)`);
|
|
199
|
-
console.log(` ${mark.padEnd(6)} ${reading.metric} ${chalk.dim(`[${reading.comparison}]`)} ${change}`);
|
|
200
|
-
}
|
|
201
|
-
console.log('');
|
|
202
|
-
}
|
|
203
|
-
for (const failure of detail.base.driver_failures)
|
|
204
|
-
console.log(chalk.yellow(` base driver failure: ${failure}`));
|
|
205
|
-
for (const failure of detail.head.driver_failures)
|
|
206
|
-
console.log(chalk.yellow(` head driver failure: ${failure}`));
|
|
207
|
-
renderTimeline('base', detail.base);
|
|
208
|
-
renderTimeline('head', detail.head);
|
|
209
|
-
console.log('');
|
|
210
|
-
renderSdkLog('base', detail.base);
|
|
211
|
-
renderSdkLog('head', detail.head);
|
|
212
|
-
const artifactCount = detail.base.artifacts.length + detail.head.artifacts.length;
|
|
213
|
-
if (artifactCount > 0) {
|
|
214
|
-
console.log('');
|
|
215
|
-
console.log(chalk.dim(` ${artifactCount} artifacts (screenshots/DOM/video) — fetch with: haystack verify artifacts ${detail.run_id} --cell ${detail.ordinal}`));
|
|
216
|
-
}
|
|
217
|
-
if (detail.live_environment) {
|
|
218
|
-
console.log('');
|
|
219
|
-
console.log(` ${chalk.bold('Live environment:')} ${chalk.cyan(detail.live_environment.url)} ${chalk.dim(`expires ${detail.live_environment.expires ?? ''}`)}`);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
export async function verifyShowCommand(runSelector, cellSelector, options) {
|
|
223
|
-
setVerifyToken(options.token);
|
|
224
|
-
const server = resolveServer(options.server);
|
|
225
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
226
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: runSelector });
|
|
227
|
-
const match = findCell(loaded.run, cellSelector);
|
|
228
|
-
const detail = cellDetail(loaded.run, match, { timeline: options.timeline !== false });
|
|
229
|
-
if (options.json)
|
|
230
|
-
return emitJson(detail);
|
|
231
|
-
renderShow(detail);
|
|
232
|
-
}
|
|
233
|
-
/** Collect artifact URLs (evidence-declared) for optional per-cell filtering. */
|
|
234
|
-
function artifactFilesForCell(run, cellSelector) {
|
|
235
|
-
const match = findCell(run, cellSelector);
|
|
236
|
-
const files = new Set();
|
|
237
|
-
for (const universe of [match.cell.base, match.cell.head]) {
|
|
238
|
-
for (const evidence of universe.evidence) {
|
|
239
|
-
if (!evidence.artifactUrl)
|
|
240
|
-
continue;
|
|
241
|
-
files.add(artifactFileOf(evidence.artifactUrl));
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
return files;
|
|
245
|
-
}
|
|
246
|
-
export async function verifyArtifactsCommand(runSelector, options) {
|
|
247
|
-
setVerifyToken(options.token);
|
|
248
|
-
const server = resolveServer(options.server);
|
|
249
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
250
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: runSelector });
|
|
251
|
-
const run = loaded.run;
|
|
252
|
-
const outRoot = path.resolve(options.out || `verify-artifacts/${run.runId}`);
|
|
253
|
-
const cellFilter = options.cell ? artifactFilesForCell(run, options.cell) : null;
|
|
254
|
-
const copied = [];
|
|
255
|
-
const skipped = [];
|
|
256
|
-
const localRunDir = repoRoot ? runDir(repoRoot, run.runId) : null;
|
|
257
|
-
if (localRunDir && fs.existsSync(path.join(localRunDir, 'chapters'))) {
|
|
258
|
-
// Same machine as the control plane: copy straight from the run directory.
|
|
259
|
-
for (const chapter of run.chapters) {
|
|
260
|
-
const artifactsDir = path.join(localRunDir, 'chapters', chapter.chapterId, 'artifacts');
|
|
261
|
-
if (!fs.existsSync(artifactsDir))
|
|
262
|
-
continue;
|
|
263
|
-
const destDir = path.join(outRoot, chapter.chapterId);
|
|
264
|
-
fs.mkdirSync(destDir, { recursive: true });
|
|
265
|
-
for (const file of fs.readdirSync(artifactsDir)) {
|
|
266
|
-
if (cellFilter && !cellFilter.has(file)) {
|
|
267
|
-
skipped.push(file);
|
|
268
|
-
continue;
|
|
269
|
-
}
|
|
270
|
-
const source = path.join(artifactsDir, file);
|
|
271
|
-
if (!fs.statSync(source).isFile())
|
|
272
|
-
continue;
|
|
273
|
-
fs.copyFileSync(source, path.join(destDir, file));
|
|
274
|
-
copied.push({ file: path.join(chapter.chapterId, file), bytes: fs.statSync(source).size });
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
else {
|
|
279
|
-
// No disk access to the run — pull every evidence-declared artifact over HTTP.
|
|
280
|
-
const urls = new Map();
|
|
281
|
-
for (const chapter of run.chapters) {
|
|
282
|
-
for (const cell of chapter.riskCells ?? []) {
|
|
283
|
-
for (const universe of [cell.base, cell.head]) {
|
|
284
|
-
for (const evidence of universe.evidence) {
|
|
285
|
-
if (!evidence.artifactUrl)
|
|
286
|
-
continue;
|
|
287
|
-
const file = artifactFileOf(evidence.artifactUrl);
|
|
288
|
-
if (cellFilter && !cellFilter.has(file))
|
|
289
|
-
continue;
|
|
290
|
-
urls.set(`${chapter.chapterId}/${file}`, evidence.artifactUrl);
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
}
|
|
295
|
-
for (const [rel, url] of urls) {
|
|
296
|
-
const response = await fetch(`${server}${url}`, { headers: verifyAuthHeaders() });
|
|
297
|
-
if (!response.ok) {
|
|
298
|
-
skipped.push(`${rel} (HTTP ${response.status})`);
|
|
299
|
-
continue;
|
|
300
|
-
}
|
|
301
|
-
const bytes = Buffer.from(await response.arrayBuffer());
|
|
302
|
-
const dest = path.join(outRoot, rel);
|
|
303
|
-
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
304
|
-
fs.writeFileSync(dest, bytes);
|
|
305
|
-
copied.push({ file: rel, bytes: bytes.length });
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
const totalBytes = copied.reduce((sum, entry) => sum + entry.bytes, 0);
|
|
309
|
-
if (options.json) {
|
|
310
|
-
return emitJson({
|
|
311
|
-
schema_version: '1.0.0',
|
|
312
|
-
run_id: run.runId,
|
|
313
|
-
out_dir: outRoot,
|
|
314
|
-
copied: copied.length,
|
|
315
|
-
total_bytes: totalBytes,
|
|
316
|
-
skipped_by_filter: options.cell ? skipped.length : 0,
|
|
317
|
-
files: copied.map(entry => entry.file),
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
if (copied.length === 0) {
|
|
321
|
-
console.log(chalk.yellow(`No artifacts ${options.cell ? `matching test "${options.cell}" ` : ''}found for ${run.runId}.`));
|
|
322
|
-
process.exitCode = 1;
|
|
323
|
-
return;
|
|
324
|
-
}
|
|
325
|
-
console.log(`${chalk.green('✓')} ${copied.length} artifacts (${(totalBytes / 1024 / 1024).toFixed(1)}MB) → ${chalk.bold(outRoot)}`);
|
|
326
|
-
const byKind = new Map();
|
|
327
|
-
for (const entry of copied) {
|
|
328
|
-
const ext = path.extname(entry.file) || 'other';
|
|
329
|
-
byKind.set(ext, (byKind.get(ext) ?? 0) + 1);
|
|
330
|
-
}
|
|
331
|
-
console.log(chalk.dim(` ${[...byKind.entries()].map(([ext, count]) => `${count} ${ext.replace('.', '')}`).join(', ')}`));
|
|
332
|
-
}
|
|
333
|
-
// ---------- findings ----------
|
|
334
|
-
export async function verifyFindingsCommand(runSelector, options) {
|
|
335
|
-
setVerifyToken(options.token);
|
|
336
|
-
const server = resolveServer(options.server);
|
|
337
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
338
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: runSelector || 'latest' });
|
|
339
|
-
const summary = summarizeRun(loaded, server);
|
|
340
|
-
if (options.json) {
|
|
341
|
-
return emitJson({
|
|
342
|
-
schema_version: '1.0.0',
|
|
343
|
-
run_id: summary.run_id,
|
|
344
|
-
status: summary.status,
|
|
345
|
-
counts: summary.counts,
|
|
346
|
-
findings: summary.findings,
|
|
347
|
-
problems: summary.problems,
|
|
348
|
-
});
|
|
349
|
-
}
|
|
350
|
-
if (summary.findings.length === 0 && summary.problems.length === 0) {
|
|
351
|
-
console.log(`${chalk.green('✓')} ${summary.run_id}: nothing broke — ${summary.counts.passed}/${summary.counts.total} tests passed.`);
|
|
352
|
-
return;
|
|
353
|
-
}
|
|
354
|
-
// The digest keeps only the explanation's first paragraph; --full goes back
|
|
355
|
-
// to the run for the whole thing.
|
|
356
|
-
const fullExplanations = new Map();
|
|
357
|
-
if (options.full) {
|
|
358
|
-
for (const chapter of loaded.run.chapters) {
|
|
359
|
-
for (const cell of chapter.riskCells ?? []) {
|
|
360
|
-
if (cell.agentJudgment) {
|
|
361
|
-
fullExplanations.set(`${chapter.chapterId}|${cell.ordinal}`, cell.agentJudgment.explanation);
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
}
|
|
365
|
-
}
|
|
366
|
-
console.log(`${chalk.bold(summary.run_id)} — ${summary.findings.length} finding(s)`);
|
|
367
|
-
for (const finding of summary.findings) {
|
|
368
|
-
console.log('');
|
|
369
|
-
const head = finding.verdict === 'culprit'
|
|
370
|
-
? `${chalk.red('◉')} ${chalk.bold(finding.title)}`
|
|
371
|
-
: `${chalk.red('✗')} ${finding.ordinal}. ${chalk.bold(finding.title)}${finding.severity ? chalk.dim(` [${finding.severity}]`) : ''} — ${finding.verdict ?? 'failed'}`;
|
|
372
|
-
console.log(head);
|
|
373
|
-
for (const reading of finding.changed_readings)
|
|
374
|
-
console.log(` ${chalk.yellow(reading)}`);
|
|
375
|
-
if (finding.summary) {
|
|
376
|
-
const full = fullExplanations.get(`${finding.chapter_id}|${finding.ordinal}`);
|
|
377
|
-
const text = options.full
|
|
378
|
-
? full ?? finding.summary
|
|
379
|
-
: finding.summary.length > 320 ? `${finding.summary.slice(0, 320)}… (--full for the rest)` : finding.summary;
|
|
380
|
-
console.log(` ${chalk.dim(text.split('\n').join('\n '))}`);
|
|
381
|
-
}
|
|
382
|
-
if (finding.live_url)
|
|
383
|
-
console.log(` live: ${chalk.cyan(finding.live_url)}`);
|
|
384
|
-
}
|
|
385
|
-
if (summary.problems.length > 0) {
|
|
386
|
-
console.log('');
|
|
387
|
-
console.log(chalk.yellow.bold('Problems (ours, not findings):'));
|
|
388
|
-
for (const problem of summary.problems)
|
|
389
|
-
console.log(` ${chalk.yellow('!')} ${problem}`);
|
|
390
|
-
}
|
|
391
|
-
console.log('');
|
|
392
|
-
console.log(chalk.dim(`Full detail per test: haystack verify show ${summary.run_id} <n>`));
|
|
393
|
-
}
|
|
394
|
-
// ---------- open / shell ----------
|
|
395
|
-
function launch(target) {
|
|
396
|
-
const platform = process.platform;
|
|
397
|
-
const command = platform === 'darwin' ? 'open' : platform === 'win32' ? 'cmd' : 'xdg-open';
|
|
398
|
-
const args = platform === 'win32' ? ['/c', 'start', '', target] : [target];
|
|
399
|
-
const child = spawn(command, args, { detached: true, stdio: 'ignore' });
|
|
400
|
-
child.unref();
|
|
401
|
-
}
|
|
402
|
-
/**
|
|
403
|
-
* Open one piece of a run: a live environment (`--live`), a specific artifact
|
|
404
|
-
* by file name, or a cell's video/screenshot/DOM snapshot picked by
|
|
405
|
-
* --side/--step/--kind.
|
|
406
|
-
*/
|
|
407
|
-
export async function verifyOpenCommand(runSelector, target, options) {
|
|
408
|
-
setVerifyToken(options.token);
|
|
409
|
-
const server = resolveServer(options.server);
|
|
410
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
411
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: runSelector });
|
|
412
|
-
const run = loaded.run;
|
|
413
|
-
// A file name (has an extension) opens that artifact directly, from any cell.
|
|
414
|
-
if (/\.[a-z0-9]+$/i.test(target) && !options.live) {
|
|
415
|
-
const fileName = path.posix.basename(target);
|
|
416
|
-
const localRunDir = repoRoot ? runDir(repoRoot, run.runId) : null;
|
|
417
|
-
if (localRunDir) {
|
|
418
|
-
for (const chapter of run.chapters) {
|
|
419
|
-
const candidate = path.join(localRunDir, 'chapters', chapter.chapterId, 'artifacts', fileName);
|
|
420
|
-
if (fs.existsSync(candidate)) {
|
|
421
|
-
launch(candidate);
|
|
422
|
-
console.log(`${chalk.green('✓')} opened ${candidate}`);
|
|
423
|
-
return;
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
// Fall back to the control plane's artifact endpoint.
|
|
428
|
-
for (const chapter of run.chapters) {
|
|
429
|
-
const url = `${server}/api/cloud-verifier/runs/${run.runId}/artifacts/${encodeURIComponent(`${chapter.chapterId}/artifacts/${fileName}`)}`;
|
|
430
|
-
const response = await fetch(url, { headers: verifyAuthHeaders() });
|
|
431
|
-
if (!response.ok)
|
|
432
|
-
continue;
|
|
433
|
-
const dest = path.join(os.tmpdir(), 'haystack-verify-artifacts', run.runId, fileName);
|
|
434
|
-
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
435
|
-
fs.writeFileSync(dest, Buffer.from(await response.arrayBuffer()));
|
|
436
|
-
launch(dest);
|
|
437
|
-
console.log(`${chalk.green('✓')} opened ${dest}`);
|
|
438
|
-
return;
|
|
439
|
-
}
|
|
440
|
-
throw new Error(`Artifact "${fileName}" not found in ${run.runId}. List artifacts with: haystack verify show ${run.runId} <test> or verify artifacts.`);
|
|
441
|
-
}
|
|
442
|
-
const match = findCell(run, target);
|
|
443
|
-
const detail = cellDetail(run, match, { timeline: false });
|
|
444
|
-
if (options.live) {
|
|
445
|
-
if (!detail.live_environment) {
|
|
446
|
-
throw new Error(`Test ${detail.ordinal} has no live environment${detail.status === 'failed' ? ' any more — it expired (failing tests keep one for 90 minutes). Re-run the chapter for a fresh one.' : ' — only failing tests keep one alive.'}`);
|
|
447
|
-
}
|
|
448
|
-
launch(detail.live_environment.url);
|
|
449
|
-
console.log(`${chalk.green('✓')} opened ${chalk.cyan(detail.live_environment.url)} ${chalk.dim(`(expires ${detail.live_environment.expires ?? ''})`)}`);
|
|
450
|
-
return;
|
|
451
|
-
}
|
|
452
|
-
const side = options.side === 'base' ? detail.base : detail.head;
|
|
453
|
-
const sideName = options.side === 'base' ? 'base' : 'head';
|
|
454
|
-
let pick;
|
|
455
|
-
if (options.step) {
|
|
456
|
-
const stepOrdinal = Number(options.step);
|
|
457
|
-
const wantKind = options.kind ?? 'screenshot';
|
|
458
|
-
pick = side.artifacts.find(artifact => artifact.kind === wantKind && artifact.label.startsWith(`step ${stepOrdinal}:`));
|
|
459
|
-
}
|
|
460
|
-
else if (options.kind) {
|
|
461
|
-
pick = side.artifacts.find(artifact => artifact.kind === options.kind);
|
|
462
|
-
}
|
|
463
|
-
else {
|
|
464
|
-
pick = side.artifacts.find(artifact => artifact.kind === 'video')
|
|
465
|
-
?? side.artifacts.filter(artifact => artifact.kind === 'screenshot').pop();
|
|
466
|
-
}
|
|
467
|
-
if (!pick) {
|
|
468
|
-
const available = side.artifacts.map(artifact => ` ${artifact.kind.padEnd(10)} ${artifact.label} — ${artifact.file}`).join('\n');
|
|
469
|
-
throw new Error(`No matching artifact on the ${sideName} side of test ${detail.ordinal}. Available:\n${available || ' (none)'}`);
|
|
470
|
-
}
|
|
471
|
-
await verifyOpenCommand(runSelector, pick.file, { ...options, live: false });
|
|
472
|
-
}
|
|
473
|
-
/** Copy an artifact to destDir (from the run directory, else the control plane). Returns the local path or null. */
|
|
474
|
-
async function materializeArtifact(run, repoRoot, server, fileName, destDir) {
|
|
475
|
-
const dest = path.join(destDir, fileName);
|
|
476
|
-
const localRunDir = repoRoot ? runDir(repoRoot, run.runId) : null;
|
|
477
|
-
if (localRunDir) {
|
|
478
|
-
for (const chapter of run.chapters) {
|
|
479
|
-
const candidate = path.join(localRunDir, 'chapters', chapter.chapterId, 'artifacts', fileName);
|
|
480
|
-
if (fs.existsSync(candidate)) {
|
|
481
|
-
fs.mkdirSync(destDir, { recursive: true });
|
|
482
|
-
fs.copyFileSync(candidate, dest);
|
|
483
|
-
return dest;
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
for (const chapter of run.chapters) {
|
|
488
|
-
const url = `${server}/api/cloud-verifier/runs/${run.runId}/artifacts/${encodeURIComponent(`${chapter.chapterId}/artifacts/${fileName}`)}`;
|
|
489
|
-
const response = await fetch(url, { headers: verifyAuthHeaders() });
|
|
490
|
-
if (!response.ok)
|
|
491
|
-
continue;
|
|
492
|
-
fs.mkdirSync(destDir, { recursive: true });
|
|
493
|
-
fs.writeFileSync(dest, Buffer.from(await response.arrayBuffer()));
|
|
494
|
-
return dest;
|
|
495
|
-
}
|
|
496
|
-
return null;
|
|
497
|
-
}
|
|
498
|
-
const escapeHtml = (text) => text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
499
|
-
/**
|
|
500
|
-
* The same viewer as the results page (CloudVerifierPage RenderCompare): one
|
|
501
|
-
* fixed frame per step whose content is swapped in place — 1 "what changed"
|
|
502
|
-
* (computed magenta diff), 2 "before", 3 "after"; arrow keys step. Comparing
|
|
503
|
-
* two near-identical images is a question about the same pixels, so the thing
|
|
504
|
-
* to change is what occupies them; making the eye travel between copies is
|
|
505
|
-
* precisely what hides a subtle difference. The magenta diff is computed
|
|
506
|
-
* client-side with the product's algorithm: channel delta > 12 → magenta
|
|
507
|
-
* (a colour no real interface ships), everything else lightened toward white.
|
|
508
|
-
*/
|
|
509
|
-
function renderDiffHtml(detail, steps, videos) {
|
|
510
|
-
const stepSections = steps.map(step => {
|
|
511
|
-
const badge = step.identical === null
|
|
512
|
-
? '<span class="badge missing">one side missing</span>'
|
|
513
|
-
: step.identical
|
|
514
|
-
? '<span class="badge same">pixel-identical</span>'
|
|
515
|
-
: '<span class="badge diff">differs</span>';
|
|
516
|
-
return `
|
|
517
|
-
<section class="step" data-base="${step.base ?? ''}" data-head="${step.head ?? ''}">
|
|
518
|
-
<h2>step ${step.ordinal}: ${escapeHtml(step.name)} ${badge}</h2>
|
|
519
|
-
<nav class="render-jump" aria-label="Which render to show"></nav>
|
|
520
|
-
<figure>
|
|
521
|
-
<figcaption></figcaption>
|
|
522
|
-
<div class="frame"></div>
|
|
523
|
-
</figure>
|
|
524
|
-
</section>`;
|
|
525
|
-
}).join('\n');
|
|
526
|
-
const videoSection = videos.base || videos.head ? `
|
|
527
|
-
<section>
|
|
528
|
-
<h2>full visit</h2>
|
|
529
|
-
<div class="videos">
|
|
530
|
-
<div class="pane"><h3>before</h3>${videos.base ? `<video controls muted src="${encodeURI(videos.base)}"></video>` : '<div class="missing-pane">no base video</div>'}</div>
|
|
531
|
-
<div class="pane"><h3>after</h3>${videos.head ? `<video controls muted src="${encodeURI(videos.head)}"></video>` : '<div class="missing-pane">no head video</div>'}</div>
|
|
532
|
-
</div>
|
|
533
|
-
</section>` : '';
|
|
534
|
-
return `<!doctype html>
|
|
535
|
-
<html><head><meta charset="utf-8"><title>${escapeHtml(detail.run_id)} — test ${detail.ordinal} screenshot diff</title>
|
|
536
|
-
<style>
|
|
537
|
-
body { font: 14px -apple-system, system-ui, sans-serif; margin: 24px; background: #111; color: #ddd; }
|
|
538
|
-
h1 { font-size: 18px; } h2 { font-size: 15px; margin: 28px 0 8px; } h3 { font-size: 12px; color: #999; margin: 4px 0; }
|
|
539
|
-
.badge { font-size: 11px; padding: 2px 8px; border-radius: 10px; margin-left: 8px; vertical-align: middle; }
|
|
540
|
-
.same { background: #14432a; color: #7bd88f; } .diff { background: #4a2a12; color: #f0a45d; } .missing { background: #333; color: #aaa; }
|
|
541
|
-
.render-jump { display: flex; gap: 6px; align-items: center; margin: 6px 0; }
|
|
542
|
-
.render-jump button { background: #222; color: #ddd; border: 1px solid #444; border-radius: 4px; padding: 4px 10px; cursor: pointer; display: inline-flex; gap: 6px; align-items: center; }
|
|
543
|
-
.render-jump button[aria-pressed="true"] { background: #2d4a7a; border-color: #4a7ab5; }
|
|
544
|
-
.render-jump kbd { background: #111; border: 1px solid #555; border-radius: 3px; padding: 0 5px; font-size: 11px; }
|
|
545
|
-
.render-jump .hint { color: #777; font-size: 12px; margin-left: 8px; }
|
|
546
|
-
figure { margin: 0; } figcaption { font-size: 11px; letter-spacing: 0.06em; color: #999; margin: 4px 0; }
|
|
547
|
-
.frame img { width: 100%; border: 1px solid #333; border-radius: 4px; display: block; }
|
|
548
|
-
.pending, .missing-pane { padding: 40px; border: 1px dashed #444; border-radius: 4px; color: #777; text-align: center; }
|
|
549
|
-
.videos { display: flex; gap: 12px; } .videos .pane { flex: 1; min-width: 0; } video { width: 100%; border: 1px solid #333; border-radius: 4px; display: block; }
|
|
550
|
-
.meta { color: #888; }
|
|
551
|
-
</style></head>
|
|
552
|
-
<body>
|
|
553
|
-
<h1>test ${detail.ordinal}: ${escapeHtml(detail.title)}</h1>
|
|
554
|
-
<p class="meta">${escapeHtml(detail.run_id)} · ${escapeHtml(detail.chapter_id)} · ${detail.status}${detail.verdict ? ` · ${escapeHtml(detail.verdict)}` : ''}</p>
|
|
555
|
-
${stepSections}
|
|
556
|
-
${videoSection}
|
|
557
|
-
<script>
|
|
558
|
-
const DELTA = 12;
|
|
559
|
-
const load = src => new Promise((resolve, reject) => {
|
|
560
|
-
const image = new Image();
|
|
561
|
-
image.onload = () => resolve(image);
|
|
562
|
-
image.onerror = () => reject(new Error('could not load image'));
|
|
563
|
-
image.src = src;
|
|
564
|
-
});
|
|
565
|
-
|
|
566
|
-
// The product's diff render: magenta where the pixels moved, everything
|
|
567
|
-
// else washed toward white so the magenta is unmissable.
|
|
568
|
-
async function diffRender(baseSrc, headSrc) {
|
|
569
|
-
const [before, after] = await Promise.all([load(baseSrc), load(headSrc)]);
|
|
570
|
-
// Renders of different sizes cannot be compared pixel for pixel; show
|
|
571
|
-
// nothing rather than paint a misleading picture.
|
|
572
|
-
if (before.naturalWidth !== after.naturalWidth || before.naturalHeight !== after.naturalHeight) return null;
|
|
573
|
-
const width = after.naturalWidth, height = after.naturalHeight;
|
|
574
|
-
const read = image => {
|
|
575
|
-
const canvas = document.createElement('canvas');
|
|
576
|
-
canvas.width = width; canvas.height = height;
|
|
577
|
-
const context = canvas.getContext('2d');
|
|
578
|
-
context.drawImage(image, 0, 0);
|
|
579
|
-
return context.getImageData(0, 0, width, height);
|
|
580
|
-
};
|
|
581
|
-
const previous = read(before).data;
|
|
582
|
-
const out = read(after);
|
|
583
|
-
const data = out.data;
|
|
584
|
-
for (let i = 0; i < data.length; i += 4) {
|
|
585
|
-
const delta = Math.max(
|
|
586
|
-
Math.abs(data[i] - previous[i]),
|
|
587
|
-
Math.abs(data[i + 1] - previous[i + 1]),
|
|
588
|
-
Math.abs(data[i + 2] - previous[i + 2]),
|
|
589
|
-
);
|
|
590
|
-
if (delta > DELTA) { data[i] = 255; data[i + 1] = 0; data[i + 2] = 200; }
|
|
591
|
-
else {
|
|
592
|
-
data[i] = data[i] + (255 - data[i]) * 0.55;
|
|
593
|
-
data[i + 1] = data[i + 1] + (255 - data[i + 1]) * 0.55;
|
|
594
|
-
data[i + 2] = data[i + 2] + (255 - data[i + 2]) * 0.55;
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
const canvas = document.createElement('canvas');
|
|
598
|
-
canvas.width = width; canvas.height = height;
|
|
599
|
-
canvas.getContext('2d').putImageData(out, 0, 0);
|
|
600
|
-
return canvas.toDataURL('image/png');
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
const stepStates = [];
|
|
604
|
-
let view = 0;
|
|
605
|
-
|
|
606
|
-
function paint() {
|
|
607
|
-
for (const state of stepStates) {
|
|
608
|
-
const current = state.renders[Math.min(view, state.renders.length - 1)];
|
|
609
|
-
state.nav.replaceChildren(...state.renders.map((render, index) => {
|
|
610
|
-
const button = document.createElement('button');
|
|
611
|
-
button.type = 'button';
|
|
612
|
-
button.setAttribute('aria-pressed', String(render === current));
|
|
613
|
-
const kbd = document.createElement('kbd');
|
|
614
|
-
kbd.textContent = String(index + 1);
|
|
615
|
-
button.append(kbd, render.label);
|
|
616
|
-
button.addEventListener('click', () => { view = index; paint(); });
|
|
617
|
-
return button;
|
|
618
|
-
}));
|
|
619
|
-
const hint = document.createElement('span');
|
|
620
|
-
hint.className = 'hint';
|
|
621
|
-
hint.textContent = 'press a key, or \u2190 \u2192 to step';
|
|
622
|
-
state.nav.append(hint);
|
|
623
|
-
state.caption.textContent = current.caption;
|
|
624
|
-
state.frame.replaceChildren();
|
|
625
|
-
if (current.url) {
|
|
626
|
-
const image = document.createElement('img');
|
|
627
|
-
image.alt = current.key;
|
|
628
|
-
image.src = current.url;
|
|
629
|
-
state.frame.append(image);
|
|
630
|
-
} else {
|
|
631
|
-
const pending = document.createElement('div');
|
|
632
|
-
pending.className = 'pending';
|
|
633
|
-
pending.textContent = current.key === 'differences' ? 'Comparing the two renders\u2026' : 'Screenshot missing';
|
|
634
|
-
state.frame.append(pending);
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
}
|
|
638
|
-
|
|
639
|
-
for (const section of document.querySelectorAll('section.step')) {
|
|
640
|
-
const base = section.dataset.base || null;
|
|
641
|
-
const head = section.dataset.head || null;
|
|
642
|
-
const state = {
|
|
643
|
-
nav: section.querySelector('.render-jump'),
|
|
644
|
-
caption: section.querySelector('figcaption'),
|
|
645
|
-
frame: section.querySelector('.frame'),
|
|
646
|
-
renders: [
|
|
647
|
-
...(base && head ? [{ key: 'differences', label: 'what changed', caption: 'WHAT CHANGED \u00b7 comparing\u2026', url: null }] : []),
|
|
648
|
-
{ key: 'before', label: 'before', caption: 'BEFORE \u00b7 without the change', url: base ?? undefined },
|
|
649
|
-
{ key: 'after', label: 'after', caption: 'AFTER \u00b7 with the change', url: head ?? undefined },
|
|
650
|
-
],
|
|
651
|
-
};
|
|
652
|
-
stepStates.push(state);
|
|
653
|
-
if (base && head) {
|
|
654
|
-
diffRender(base, head).then(url => {
|
|
655
|
-
if (url) {
|
|
656
|
-
state.renders[0].url = url;
|
|
657
|
-
state.renders[0].caption = 'WHAT CHANGED \u00b7 everything in magenta';
|
|
658
|
-
} else {
|
|
659
|
-
// Sizes differ: drop the computed render rather than mislead.
|
|
660
|
-
state.renders.shift();
|
|
661
|
-
}
|
|
662
|
-
paint();
|
|
663
|
-
}).catch(() => { state.renders.shift(); paint(); });
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
window.addEventListener('keydown', event => {
|
|
668
|
-
if (event.metaKey || event.ctrlKey || event.altKey) return;
|
|
669
|
-
const target = event.target;
|
|
670
|
-
if (target && /^(INPUT|TEXTAREA|SELECT)$/.test(target.tagName)) return;
|
|
671
|
-
const index = ['1', '2', '3'].indexOf(event.key);
|
|
672
|
-
if (index >= 0) view = index;
|
|
673
|
-
else if (event.key === 'ArrowLeft') view = Math.max(0, view - 1);
|
|
674
|
-
else if (event.key === 'ArrowRight') view = Math.min(2, view + 1);
|
|
675
|
-
else return;
|
|
676
|
-
event.preventDefault();
|
|
677
|
-
paint();
|
|
678
|
-
});
|
|
679
|
-
|
|
680
|
-
paint();
|
|
681
|
-
</script>
|
|
682
|
-
</body></html>`;
|
|
683
|
-
}
|
|
684
|
-
/**
|
|
685
|
-
* Build the screenshot-diff page for one test and report which steps differ
|
|
686
|
-
* byte-for-byte. Shared by `haystack verify diff` and the MCP verify_diff tool.
|
|
687
|
-
*/
|
|
688
|
-
export async function buildScreenshotDiff(options) {
|
|
689
|
-
const { server, repoRoot } = options;
|
|
690
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: options.runSelector });
|
|
691
|
-
const match = findCell(loaded.run, options.cellSelector);
|
|
692
|
-
const detail = cellDetail(loaded.run, match, { timeline: false });
|
|
693
|
-
const outDir = path.resolve(options.outDir
|
|
694
|
-
|| path.join(os.tmpdir(), 'haystack-verify-diff', `${detail.run_id}-cell-${detail.ordinal}`));
|
|
695
|
-
const assetsDir = path.join(outDir, 'assets');
|
|
696
|
-
const stepOf = (label) => {
|
|
697
|
-
const parsed = label.match(/^step (\d+): (.*)$/);
|
|
698
|
-
return parsed ? { ordinal: Number(parsed[1]), name: parsed[2] } : null;
|
|
699
|
-
};
|
|
700
|
-
const shotsBySide = (side) => {
|
|
701
|
-
const shots = new Map();
|
|
702
|
-
for (const artifact of side.artifacts) {
|
|
703
|
-
if (artifact.kind !== 'screenshot')
|
|
704
|
-
continue;
|
|
705
|
-
const step = stepOf(artifact.label);
|
|
706
|
-
if (step)
|
|
707
|
-
shots.set(step.ordinal, { name: step.name, file: artifact.file });
|
|
708
|
-
}
|
|
709
|
-
return shots;
|
|
710
|
-
};
|
|
711
|
-
const baseShots = shotsBySide(detail.base);
|
|
712
|
-
const headShots = shotsBySide(detail.head);
|
|
713
|
-
const wantStep = options.step ?? null;
|
|
714
|
-
const ordinals = [...new Set([...baseShots.keys(), ...headShots.keys()])]
|
|
715
|
-
.filter(ordinal => wantStep === null || ordinal === wantStep)
|
|
716
|
-
.sort((a, b) => a - b);
|
|
717
|
-
if (ordinals.length === 0) {
|
|
718
|
-
const stillRunning = !['passed', 'failed', 'infra_error', 'inconclusive'].includes(detail.status);
|
|
719
|
-
throw new Error(wantStep !== null
|
|
720
|
-
? `No screenshots at step ${wantStep}. Steps with screenshots: ${[...new Set([...baseShots.keys(), ...headShots.keys()])].sort((a, b) => a - b).join(', ') || '(none)'}`
|
|
721
|
-
: stillRunning
|
|
722
|
-
? `Test ${detail.ordinal} is still ${detail.status} — screenshots appear once its flow has run. Wait for the run (verify watch / verify_wait) and retry.`
|
|
723
|
-
: `Test ${detail.ordinal} has no step screenshots to compare.`);
|
|
724
|
-
}
|
|
725
|
-
// Screenshots are embedded as data: URIs — the page computes the magenta
|
|
726
|
-
// "what changed" render on a canvas, and file:// images would taint it.
|
|
727
|
-
const dataUri = (file) => `data:image/png;base64,${fs.readFileSync(file).toString('base64')}`;
|
|
728
|
-
const sha256 = (file) => createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
729
|
-
const steps = [];
|
|
730
|
-
const reported = [];
|
|
731
|
-
for (const ordinal of ordinals) {
|
|
732
|
-
const basePick = baseShots.get(ordinal) ?? null;
|
|
733
|
-
const headPick = headShots.get(ordinal) ?? null;
|
|
734
|
-
const baseLocal = basePick ? await materializeArtifact(loaded.run, repoRoot, server, basePick.file, assetsDir) : null;
|
|
735
|
-
const headLocal = headPick ? await materializeArtifact(loaded.run, repoRoot, server, headPick.file, assetsDir) : null;
|
|
736
|
-
const name = (headPick ?? basePick)?.name ?? `step ${ordinal}`;
|
|
737
|
-
const identical = baseLocal && headLocal ? sha256(baseLocal) === sha256(headLocal) : null;
|
|
738
|
-
steps.push({
|
|
739
|
-
ordinal,
|
|
740
|
-
name,
|
|
741
|
-
base: baseLocal ? dataUri(baseLocal) : null,
|
|
742
|
-
head: headLocal ? dataUri(headLocal) : null,
|
|
743
|
-
identical,
|
|
744
|
-
});
|
|
745
|
-
reported.push({ ordinal, name, identical });
|
|
746
|
-
}
|
|
747
|
-
const videoFile = (side) => side.artifacts.find(artifact => artifact.kind === 'video')?.file ?? null;
|
|
748
|
-
const baseVideoName = videoFile(detail.base);
|
|
749
|
-
const headVideoName = videoFile(detail.head);
|
|
750
|
-
const baseVideo = wantStep === null && baseVideoName ? await materializeArtifact(loaded.run, repoRoot, server, baseVideoName, assetsDir) : null;
|
|
751
|
-
const headVideo = wantStep === null && headVideoName ? await materializeArtifact(loaded.run, repoRoot, server, headVideoName, assetsDir) : null;
|
|
752
|
-
const htmlPath = path.join(outDir, 'index.html');
|
|
753
|
-
fs.mkdirSync(outDir, { recursive: true });
|
|
754
|
-
fs.writeFileSync(htmlPath, renderDiffHtml(detail, steps, {
|
|
755
|
-
base: baseVideo ? path.join('assets', path.basename(baseVideo)) : null,
|
|
756
|
-
head: headVideo ? path.join('assets', path.basename(headVideo)) : null,
|
|
757
|
-
}));
|
|
758
|
-
return {
|
|
759
|
-
html: htmlPath,
|
|
760
|
-
steps: reported,
|
|
761
|
-
differing: reported.filter(step => step.identical === false).length,
|
|
762
|
-
identical: reported.filter(step => step.identical === true).length,
|
|
763
|
-
detail,
|
|
764
|
-
};
|
|
765
|
-
}
|
|
766
|
-
/**
|
|
767
|
-
* Build and open the base-vs-head screenshot differ for one test — the same
|
|
768
|
-
* viewer as the results page: 1 what changed / 2 before / 3 after, swapped in
|
|
769
|
-
* place, with arrow keys stepping.
|
|
770
|
-
*/
|
|
771
|
-
export async function verifyDiffCommand(runSelector, cellSelector, options) {
|
|
772
|
-
if (options.step && Number.isNaN(Number(options.step)))
|
|
773
|
-
throw new Error('--step takes a number');
|
|
774
|
-
setVerifyToken(options.token);
|
|
775
|
-
const result = await buildScreenshotDiff({
|
|
776
|
-
server: resolveServer(options.server),
|
|
777
|
-
repoRoot: discoverVerifyRoot(options.repoRoot),
|
|
778
|
-
runSelector,
|
|
779
|
-
cellSelector,
|
|
780
|
-
step: options.step ? Number(options.step) : undefined,
|
|
781
|
-
outDir: options.out,
|
|
782
|
-
});
|
|
783
|
-
if (options.json) {
|
|
784
|
-
emitJson({
|
|
785
|
-
schema_version: '1.0.0',
|
|
786
|
-
run_id: result.detail.run_id,
|
|
787
|
-
ordinal: result.detail.ordinal,
|
|
788
|
-
html: result.html,
|
|
789
|
-
steps: result.steps,
|
|
790
|
-
differing: result.differing,
|
|
791
|
-
identical: result.identical,
|
|
792
|
-
});
|
|
793
|
-
}
|
|
794
|
-
else {
|
|
795
|
-
console.log(`${chalk.green('✓')} diff ready: ${result.steps.length} step(s) — ${result.differing > 0 ? `${chalk.yellow(`${result.differing} differ`)}${result.identical > 0 ? chalk.dim(`, ${result.identical} identical`) : ''}` : chalk.green('all pixel-identical')}`);
|
|
796
|
-
console.log(chalk.dim(` ${result.html} — 1 what changed · 2 before · 3 after`));
|
|
797
|
-
}
|
|
798
|
-
if (options.open !== false && !options.json)
|
|
799
|
-
launch(result.html);
|
|
800
|
-
}
|
|
801
|
-
export async function requestUniverseCleanupStatus(server, forkId) {
|
|
802
|
-
const response = await fetch(`${server}/api/cloud-verifier/universes/${encodeURIComponent(forkId)}/cleanup`, { headers: verifyAuthHeaders() });
|
|
803
|
-
if (!response.ok) {
|
|
804
|
-
let detail = '';
|
|
805
|
-
try {
|
|
806
|
-
const body = await response.json();
|
|
807
|
-
if (typeof body.error === 'string')
|
|
808
|
-
detail = `: ${body.error}`;
|
|
809
|
-
}
|
|
810
|
-
catch { /* status is sufficient */ }
|
|
811
|
-
throw new Error(`Could not read retained-state cleanup (HTTP ${response.status})${detail}`);
|
|
812
|
-
}
|
|
813
|
-
return await response.json();
|
|
814
|
-
}
|
|
815
|
-
export async function verifyCleanupCommand(runSelector, cellSelector, options) {
|
|
816
|
-
setVerifyToken(options.token);
|
|
817
|
-
if (options.side && options.side !== 'base' && options.side !== 'head') {
|
|
818
|
-
throw new Error('--side must be base or head');
|
|
819
|
-
}
|
|
820
|
-
const server = resolveServer(options.server);
|
|
821
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
822
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: runSelector });
|
|
823
|
-
const match = findCellByStableSelector(loaded.run, cellSelector);
|
|
824
|
-
const side = options.side === 'base' ? 'base' : 'head';
|
|
825
|
-
const universe = side === 'base' ? match.cell.base : match.cell.head;
|
|
826
|
-
if (!universe.forkId) {
|
|
827
|
-
throw new Error(`Test ${match.cell.ordinal}'s ${side} universe has no durable fork ID.`);
|
|
828
|
-
}
|
|
829
|
-
const cleanup = await requestUniverseCleanupStatus(server, universe.forkId);
|
|
830
|
-
const output = {
|
|
831
|
-
schema_version: '1.0.0',
|
|
832
|
-
run_id: loaded.run.runId,
|
|
833
|
-
ordinal: match.cell.ordinal,
|
|
834
|
-
side,
|
|
835
|
-
...cleanup,
|
|
836
|
-
};
|
|
837
|
-
if (options.json) {
|
|
838
|
-
emitJson(output);
|
|
839
|
-
}
|
|
840
|
-
else {
|
|
841
|
-
const glyph = cleanup.status === 'complete'
|
|
842
|
-
? chalk.green('✓')
|
|
843
|
-
: cleanup.status === 'missed'
|
|
844
|
-
? chalk.red('✗')
|
|
845
|
-
: cleanup.status === 'pending'
|
|
846
|
-
? chalk.yellow('◷')
|
|
847
|
-
: chalk.cyan('·');
|
|
848
|
-
console.log(`${glyph} ${chalk.bold(cleanup.status)} cleanup for test `
|
|
849
|
-
+ `${match.cell.ordinal}'s ${side} universe`);
|
|
850
|
-
console.log(chalk.dim(` fork ${cleanup.forkId} · catalog ${cleanup.catalogStatus} · checked ${cleanup.checkedAt}`));
|
|
851
|
-
for (const resource of cleanup.resources) {
|
|
852
|
-
const region = resource.resource.kind === 'archil-disk'
|
|
853
|
-
? ` · ${resource.resource.region}`
|
|
854
|
-
: '';
|
|
855
|
-
console.log(` ${resource.resource.kind} ${chalk.cyan(resource.resource.id)}`
|
|
856
|
-
+ `${chalk.dim(region)} — ${resource.status}, `
|
|
857
|
-
+ `${resource.referenceCount} reference(s), `
|
|
858
|
-
+ `${resource.deleteAttempts} delete attempt(s)`);
|
|
859
|
-
console.log(chalk.dim(` exact id ${resource.resourceIdentity}`));
|
|
860
|
-
if (resource.deletion) {
|
|
861
|
-
console.log(chalk.dim(` deletion ${resource.deletion.status}; `
|
|
862
|
-
+ `deadline ${resource.deletion.deadlineAt}`
|
|
863
|
-
+ (resource.deletion.deletedAt
|
|
864
|
-
? `; confirmed ${resource.deletion.deletedAt}`
|
|
865
|
-
: '')));
|
|
866
|
-
}
|
|
867
|
-
}
|
|
868
|
-
}
|
|
869
|
-
if (cleanup.status === 'missed')
|
|
870
|
-
process.exitCode = 2;
|
|
871
|
-
}
|
|
872
|
-
export async function requestUniverseReopen(server, forkId) {
|
|
873
|
-
const response = await fetch(`${server}/api/cloud-verifier/universes/${encodeURIComponent(forkId)}/reopen`, {
|
|
874
|
-
method: 'POST',
|
|
875
|
-
headers: { 'content-type': 'application/json', ...verifyAuthHeaders() },
|
|
876
|
-
body: '{}',
|
|
877
|
-
});
|
|
878
|
-
if (!response.ok) {
|
|
879
|
-
let detail = '';
|
|
880
|
-
try {
|
|
881
|
-
const body = await response.json();
|
|
882
|
-
if (typeof body.error === 'string')
|
|
883
|
-
detail = `: ${body.error}`;
|
|
884
|
-
}
|
|
885
|
-
catch { /* status is sufficient */ }
|
|
886
|
-
throw new Error(`Could not reopen retained universe (HTTP ${response.status})${detail}`);
|
|
887
|
-
}
|
|
888
|
-
return await response.json();
|
|
889
|
-
}
|
|
890
|
-
export async function requestUniverseRefork(server, forkId, sandboxId, childKey) {
|
|
891
|
-
const response = await fetch(`${server}/api/cloud-verifier/universes/${encodeURIComponent(forkId)}`
|
|
892
|
-
+ `/attachments/${encodeURIComponent(sandboxId)}/refork`, {
|
|
893
|
-
method: 'POST',
|
|
894
|
-
headers: { 'content-type': 'application/json', ...verifyAuthHeaders() },
|
|
895
|
-
body: JSON.stringify({ childKey }),
|
|
896
|
-
});
|
|
897
|
-
if (!response.ok) {
|
|
898
|
-
let detail = '';
|
|
899
|
-
try {
|
|
900
|
-
const body = await response.json();
|
|
901
|
-
if (typeof body.error === 'string')
|
|
902
|
-
detail = `: ${body.error}`;
|
|
903
|
-
}
|
|
904
|
-
catch { /* status is sufficient */ }
|
|
905
|
-
throw new Error(`Could not refork retained universe (HTTP ${response.status})${detail}`);
|
|
906
|
-
}
|
|
907
|
-
return await response.json();
|
|
908
|
-
}
|
|
909
|
-
function validateOperatorSide(side) {
|
|
910
|
-
if (side && side !== 'base' && side !== 'head') {
|
|
911
|
-
throw new Error('--side must be base or head');
|
|
912
|
-
}
|
|
913
|
-
return side === 'base' ? 'base' : 'head';
|
|
914
|
-
}
|
|
915
|
-
function isStructuredArchilForkId(value) {
|
|
916
|
-
return /^archil:[A-Za-z0-9_-]+:branch:[A-Za-z0-9._-]+$/.test(value);
|
|
917
|
-
}
|
|
918
|
-
export async function resolveRetainedUniverseTarget(options) {
|
|
919
|
-
if (isStructuredArchilForkId(options.source)) {
|
|
920
|
-
if (options.cellSelector) {
|
|
921
|
-
throw new Error('An exact Archil fork ID does not take a test selector.');
|
|
922
|
-
}
|
|
923
|
-
if (options.side) {
|
|
924
|
-
throw new Error('--side only applies when selecting state from a run.');
|
|
925
|
-
}
|
|
926
|
-
return {
|
|
927
|
-
forkId: options.source,
|
|
928
|
-
runId: null,
|
|
929
|
-
ordinal: null,
|
|
930
|
-
side: null,
|
|
931
|
-
};
|
|
932
|
-
}
|
|
933
|
-
if (options.source.startsWith('archil:')) {
|
|
934
|
-
throw new Error('Invalid structured Archil fork ID; expected '
|
|
935
|
-
+ 'archil:<disk-id>:branch:<branch-id>.');
|
|
936
|
-
}
|
|
937
|
-
if (!options.cellSelector) {
|
|
938
|
-
throw new Error('A test selector is required when the source is a run. '
|
|
939
|
-
+ 'Use an ordinal, riskCellId, or exact planner cellKey.');
|
|
940
|
-
}
|
|
941
|
-
const side = validateOperatorSide(options.side);
|
|
942
|
-
const loaded = await loadRunAuto({
|
|
943
|
-
server: options.server,
|
|
944
|
-
repoRoot: options.repoRoot,
|
|
945
|
-
selector: options.source,
|
|
946
|
-
});
|
|
947
|
-
const match = findCellByStableSelector(loaded.run, options.cellSelector);
|
|
948
|
-
const universe = side === 'base' ? match.cell.base : match.cell.head;
|
|
949
|
-
if (universe.stateProvider !== 'archil' || !universe.forkId) {
|
|
950
|
-
throw new Error(`Test ${match.cell.ordinal}'s ${side} universe has no retained Archil state.`);
|
|
951
|
-
}
|
|
952
|
-
if (universe.stateExpiresAt
|
|
953
|
-
&& Date.parse(universe.stateExpiresAt) <= Date.now()) {
|
|
954
|
-
throw new Error(`Test ${match.cell.ordinal}'s retained state expired at `
|
|
955
|
-
+ `${universe.stateExpiresAt}.`);
|
|
956
|
-
}
|
|
957
|
-
return {
|
|
958
|
-
forkId: universe.forkId,
|
|
959
|
-
runId: loaded.run.runId,
|
|
960
|
-
ordinal: match.cell.ordinal,
|
|
961
|
-
side,
|
|
962
|
-
};
|
|
963
|
-
}
|
|
964
|
-
export async function reforkRetainedUniverse(server, target, childKey) {
|
|
965
|
-
const reopened = await requestUniverseReopen(server, target.forkId);
|
|
966
|
-
try {
|
|
967
|
-
const reforked = await requestUniverseRefork(server, target.forkId, reopened.sandboxId, childKey);
|
|
968
|
-
if (reforked.parentForkId !== target.forkId) {
|
|
969
|
-
throw new Error(`Control plane reforked unexpected parent ${reforked.parentForkId}; `
|
|
970
|
-
+ `expected ${target.forkId}.`);
|
|
971
|
-
}
|
|
972
|
-
return { reopened, reforked };
|
|
973
|
-
}
|
|
974
|
-
catch (error) {
|
|
975
|
-
// A newly created attachment exists only to service this command. Reclaim
|
|
976
|
-
// it on a pre-checkpoint failure; never tear down a reused attachment that
|
|
977
|
-
// an operator may still be inspecting or mutating.
|
|
978
|
-
if (!reopened.reused) {
|
|
979
|
-
await fetch(`${server}/api/cloud-verifier/universes/`
|
|
980
|
-
+ `${encodeURIComponent(target.forkId)}/attachments/`
|
|
981
|
-
+ `${encodeURIComponent(reopened.sandboxId)}/release`, {
|
|
982
|
-
method: 'POST',
|
|
983
|
-
headers: {
|
|
984
|
-
'content-type': 'application/json',
|
|
985
|
-
...verifyAuthHeaders(),
|
|
986
|
-
},
|
|
987
|
-
body: '{}',
|
|
988
|
-
}).catch(() => undefined);
|
|
989
|
-
}
|
|
990
|
-
throw error;
|
|
991
|
-
}
|
|
992
|
-
}
|
|
993
|
-
export async function verifyReopenCommand(source, cellSelector, options) {
|
|
994
|
-
setVerifyToken(options.token);
|
|
995
|
-
const server = resolveServer(options.server);
|
|
996
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
997
|
-
const target = await resolveRetainedUniverseTarget({
|
|
998
|
-
server,
|
|
999
|
-
repoRoot,
|
|
1000
|
-
source,
|
|
1001
|
-
cellSelector,
|
|
1002
|
-
side: options.side,
|
|
1003
|
-
});
|
|
1004
|
-
const reopened = await requestUniverseReopen(server, target.forkId);
|
|
1005
|
-
const output = {
|
|
1006
|
-
schema_version: '1.0.0',
|
|
1007
|
-
run_id: target.runId,
|
|
1008
|
-
ordinal: target.ordinal,
|
|
1009
|
-
side: target.side,
|
|
1010
|
-
...reopened,
|
|
1011
|
-
};
|
|
1012
|
-
if (options.json) {
|
|
1013
|
-
emitJson(output);
|
|
1014
|
-
return;
|
|
1015
|
-
}
|
|
1016
|
-
console.log(`${chalk.green('✓')} ${reopened.reused ? 'using' : 'reopened'} retained universe `
|
|
1017
|
-
+ `${chalk.cyan(reopened.forkId)} on sandbox ${chalk.cyan(reopened.sandboxId)}`);
|
|
1018
|
-
console.log(chalk.dim(` compute expires ${reopened.expiresAt}; state expires ${reopened.stateExpiresAt}`));
|
|
1019
|
-
console.log(` e2b sandbox connect ${reopened.sandboxId}`);
|
|
1020
|
-
if (reopened.browserUrl) {
|
|
1021
|
-
console.log(` ${reopened.browserUrl}`);
|
|
1022
|
-
if (options.openBrowser)
|
|
1023
|
-
launch(reopened.browserUrl);
|
|
1024
|
-
}
|
|
1025
|
-
}
|
|
1026
|
-
export async function verifyReforkCommand(source, cellOrChildKey, requestedChildKey, options) {
|
|
1027
|
-
setVerifyToken(options.token);
|
|
1028
|
-
const directFork = source.startsWith('archil:');
|
|
1029
|
-
const cellSelector = directFork ? undefined : cellOrChildKey;
|
|
1030
|
-
const childKey = directFork ? cellOrChildKey : requestedChildKey;
|
|
1031
|
-
if (!childKey) {
|
|
1032
|
-
throw new Error('A structured child key is required (for example attempt:checkout-fix-2).');
|
|
1033
|
-
}
|
|
1034
|
-
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(childKey)) {
|
|
1035
|
-
throw new Error('Child key must start with a letter or number and contain at most 128 '
|
|
1036
|
-
+ 'letters, numbers, dots, underscores, colons, or hyphens.');
|
|
1037
|
-
}
|
|
1038
|
-
const server = resolveServer(options.server);
|
|
1039
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
1040
|
-
const target = await resolveRetainedUniverseTarget({
|
|
1041
|
-
server,
|
|
1042
|
-
repoRoot,
|
|
1043
|
-
source,
|
|
1044
|
-
cellSelector,
|
|
1045
|
-
side: options.side,
|
|
1046
|
-
});
|
|
1047
|
-
const { reopened, reforked } = await reforkRetainedUniverse(server, target, childKey);
|
|
1048
|
-
const output = {
|
|
1049
|
-
schema_version: '1.0.0',
|
|
1050
|
-
run_id: target.runId,
|
|
1051
|
-
ordinal: target.ordinal,
|
|
1052
|
-
side: target.side,
|
|
1053
|
-
child_key: childKey,
|
|
1054
|
-
attachment_sandbox_id: reopened.sandboxId,
|
|
1055
|
-
reused_attachment: reopened.reused,
|
|
1056
|
-
...reforked,
|
|
1057
|
-
};
|
|
1058
|
-
if (options.json) {
|
|
1059
|
-
emitJson(output);
|
|
1060
|
-
return;
|
|
1061
|
-
}
|
|
1062
|
-
console.log(`${chalk.green('✓')} reforked retained universe for `
|
|
1063
|
-
+ `${target.ordinal === null ? chalk.cyan(target.forkId) : `test ${target.ordinal}'s ${target.side} side`}`);
|
|
1064
|
-
console.log(chalk.dim(` parent ${reforked.parentForkId}`));
|
|
1065
|
-
console.log(` child ${chalk.cyan(reforked.forkId)}`);
|
|
1066
|
-
console.log(chalk.dim(` checkpoint ${reforked.checkpointName}; state expires `
|
|
1067
|
-
+ `${reforked.stateExpiresAt}`));
|
|
1068
|
-
console.log(` haystack verify reopen ${reforked.forkId}`);
|
|
1069
|
-
}
|
|
1070
|
-
/**
|
|
1071
|
-
* Drop a terminal into a test's live universe via the E2B CLI. Only failing
|
|
1072
|
-
* tests keep a universe alive. An expired Archil-backed compute attachment is
|
|
1073
|
-
* transparently replaced while its durable state is still retained.
|
|
1074
|
-
*/
|
|
1075
|
-
export async function verifyShellCommand(runSelector, cellSelector, options) {
|
|
1076
|
-
setVerifyToken(options.token);
|
|
1077
|
-
const server = resolveServer(options.server);
|
|
1078
|
-
const repoRoot = discoverVerifyRoot(options.repoRoot);
|
|
1079
|
-
const loaded = await loadRunAuto({ server, repoRoot, selector: runSelector });
|
|
1080
|
-
const match = findCellByStableSelector(loaded.run, cellSelector);
|
|
1081
|
-
const side = validateOperatorSide(options.side);
|
|
1082
|
-
const universe = side === 'base' ? match.cell.base : match.cell.head;
|
|
1083
|
-
if (!universe.sandboxId && (!universe.forkId || universe.stateProvider !== 'archil')) {
|
|
1084
|
-
throw new Error(`Test ${match.cell.ordinal} has no ${options.side ?? 'head'} sandbox recorded.`);
|
|
1085
|
-
}
|
|
1086
|
-
const goneError = new Error(`Test ${match.cell.ordinal}'s universe is no longer running (failing tests keep one for 90 minutes). Re-run the chapter, then shell in.`);
|
|
1087
|
-
// Ask E2B whether the sandbox is actually alive — run.json can say `running`
|
|
1088
|
-
// for a universe that has since expired or been reclaimed. If the API is not
|
|
1089
|
-
// reachable, fall back to what the run recorded.
|
|
1090
|
-
let sandboxId = universe.sandboxId ?? '';
|
|
1091
|
-
let confirmedRunning = null;
|
|
1092
|
-
try {
|
|
1093
|
-
const { listE2BSandboxes, resolveE2BApiKey } = await import('./verify-sandboxes.js');
|
|
1094
|
-
confirmedRunning = sandboxId
|
|
1095
|
-
? (await listE2BSandboxes(resolveE2BApiKey()))
|
|
1096
|
-
.some(sandbox => sandbox.sandboxId === sandboxId)
|
|
1097
|
-
: false;
|
|
1098
|
-
}
|
|
1099
|
-
catch {
|
|
1100
|
-
confirmedRunning = null;
|
|
1101
|
-
}
|
|
1102
|
-
let reopened = null;
|
|
1103
|
-
const recordedLive = universe.status === 'running'
|
|
1104
|
-
&& (!universe.expiresAt || Date.parse(universe.expiresAt) > Date.now());
|
|
1105
|
-
const shouldReopen = confirmedRunning === false
|
|
1106
|
-
|| (confirmedRunning === null && !recordedLive);
|
|
1107
|
-
if (shouldReopen) {
|
|
1108
|
-
if (universe.stateProvider !== 'archil'
|
|
1109
|
-
|| !universe.forkId
|
|
1110
|
-
|| (universe.stateExpiresAt
|
|
1111
|
-
&& Date.parse(universe.stateExpiresAt) <= Date.now())) {
|
|
1112
|
-
throw goneError;
|
|
1113
|
-
}
|
|
1114
|
-
reopened = await requestUniverseReopen(server, universe.forkId);
|
|
1115
|
-
sandboxId = reopened.sandboxId;
|
|
1116
|
-
confirmedRunning = true;
|
|
1117
|
-
}
|
|
1118
|
-
if (confirmedRunning === null) {
|
|
1119
|
-
if (!recordedLive)
|
|
1120
|
-
throw goneError;
|
|
1121
|
-
}
|
|
1122
|
-
const appUrl = reopened?.browserUrl
|
|
1123
|
-
?? universe.browserUrl
|
|
1124
|
-
?? match.cell.head.browserUrl
|
|
1125
|
-
?? null;
|
|
1126
|
-
console.error(chalk.dim(`Connecting to sandbox ${sandboxId} (universe of test ${match.cell.ordinal}, ${side} side${reopened ? '; reopened from retained Archil state' : ''}${appUrl ? `; app at ${appUrl}` : ''}). The app lives under /home/user/universe.`));
|
|
1127
|
-
if (!process.stdin.isTTY) {
|
|
1128
|
-
// Not interactive — hand the operator the command instead of hanging.
|
|
1129
|
-
console.log(`e2b sandbox connect ${sandboxId}`);
|
|
1130
|
-
return;
|
|
1131
|
-
}
|
|
1132
|
-
try {
|
|
1133
|
-
await new Promise((resolve, reject) => {
|
|
1134
|
-
const child = spawn('e2b', ['sandbox', 'connect', sandboxId], { stdio: 'inherit' });
|
|
1135
|
-
child.on('error', error => reject(new Error(`Could not run the e2b CLI (${error.message}). Install it (npm i -g @e2b/cli) or connect manually: e2b sandbox connect ${sandboxId}`)));
|
|
1136
|
-
child.on('close', () => resolve());
|
|
1137
|
-
});
|
|
1138
|
-
}
|
|
1139
|
-
finally {
|
|
1140
|
-
if (reopened && universe.forkId) {
|
|
1141
|
-
await fetch(`${server}/api/cloud-verifier/universes/${encodeURIComponent(universe.forkId)}/attachments/${encodeURIComponent(sandboxId)}/release`, {
|
|
1142
|
-
method: 'POST',
|
|
1143
|
-
headers: { 'content-type': 'application/json', ...verifyAuthHeaders() },
|
|
1144
|
-
body: '{}',
|
|
1145
|
-
}).catch(() => undefined);
|
|
1146
|
-
}
|
|
1147
|
-
}
|
|
1148
|
-
}
|