@haystackeditor/cli 0.15.13 → 0.15.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,115 @@
1
+ /**
2
+ * Haystack Verification Contract — `.haystack/verification.yml`
3
+ *
4
+ * The contract is the repo-local answer to five questions:
5
+ * 1. How do I set up and start the app?
6
+ * 2. How do I know the app is ready? (preflight)
7
+ * 3. How do I log in as useful test users? (personas)
8
+ * 4. Which verification flows can I run? (scenarios)
9
+ * 5. Where do logs, screenshots, and artifacts go?
10
+ *
11
+ * This is deliberately separate from `.haystack.json`: that file configures
12
+ * Haystack's own sandbox verification (services/flows/fixtures the sandbox
13
+ * drives). The contract instead describes commands the REPO owns — anything
14
+ * (Haystack, CI, a coding agent, a human) can run them and get the same
15
+ * evidence. See docs/VERIFICATION-CONTRACT.md.
16
+ */
17
+ import * as fs from 'node:fs/promises';
18
+ import * as path from 'node:path';
19
+ import { parse as parseYaml } from 'yaml';
20
+ import { z } from 'zod';
21
+ export const CONTRACT_RELATIVE_PATH = '.haystack/verification.yml';
22
+ export const DEFAULT_ARTIFACT_DIR = '.haystack/artifacts';
23
+ const commandString = z.string().min(1, 'command must be a non-empty string');
24
+ const personaSchema = z.object({
25
+ login: commandString.describe('Command that logs the given persona in (or prints credentials)'),
26
+ description: z.string().optional(),
27
+ });
28
+ const scenarioSchema = z.object({
29
+ command: commandString,
30
+ description: z.string().optional(),
31
+ /** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
32
+ artifacts: z.array(z.string()).optional(),
33
+ /** Wall-clock cap for the scenario command. Default applied by the runner. */
34
+ timeout_seconds: z.number().int().positive().max(7200).optional(),
35
+ /**
36
+ * Whether `run` gates this scenario on environment.preflight. Set false for
37
+ * self-contained scenarios that boot the app themselves (like the scaffolded
38
+ * smoke script) — preflight against a cold environment would always fail.
39
+ */
40
+ preflight: z.boolean().default(true),
41
+ });
42
+ const safetySchema = z.object({
43
+ /** Command proving dev-only helpers are disabled in production builds. */
44
+ production_disabled_check: commandString.optional(),
45
+ external_services: z.enum(['sandbox_only', 'mocked', 'none']).optional(),
46
+ allow_real_email: z.boolean().default(false),
47
+ allow_real_payments: z.boolean().default(false),
48
+ allow_customer_data: z.boolean().default(false),
49
+ });
50
+ export const verificationContractSchema = z.object({
51
+ // Accept 1 or "1" — YAML authors will write both.
52
+ version: z
53
+ .union([z.literal(1), z.literal('1')])
54
+ .transform(() => 1),
55
+ app: z.object({
56
+ name: commandString,
57
+ framework: z.string().optional(),
58
+ }),
59
+ environment: z.object({
60
+ setup: commandString.optional(),
61
+ start: commandString.optional(),
62
+ preflight: commandString,
63
+ reset: commandString.optional(),
64
+ artifact_dir: z.string().default(DEFAULT_ARTIFACT_DIR),
65
+ log_paths: z.array(z.string()).default([]),
66
+ }),
67
+ personas: z.record(personaSchema).default({}),
68
+ scenarios: z
69
+ .record(scenarioSchema)
70
+ .refine((s) => Object.keys(s).length > 0, {
71
+ message: 'at least one scenario is required',
72
+ }),
73
+ safety: safetySchema,
74
+ });
75
+ /**
76
+ * Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
77
+ * Never throws — parse/schema failures come back as `status: 'invalid'` so
78
+ * callers can map them onto readiness / manifest statuses.
79
+ */
80
+ export async function loadContract(rootDir = process.cwd()) {
81
+ const contractPath = path.join(rootDir, CONTRACT_RELATIVE_PATH);
82
+ let raw;
83
+ try {
84
+ raw = await fs.readFile(contractPath, 'utf-8');
85
+ }
86
+ catch {
87
+ return { status: 'missing', path: contractPath };
88
+ }
89
+ let parsed;
90
+ try {
91
+ parsed = parseYaml(raw);
92
+ }
93
+ catch (err) {
94
+ return {
95
+ status: 'invalid',
96
+ path: contractPath,
97
+ errors: [`YAML parse error: ${err instanceof Error ? err.message : String(err)}`],
98
+ };
99
+ }
100
+ const result = verificationContractSchema.safeParse(parsed);
101
+ if (!result.success) {
102
+ return {
103
+ status: 'invalid',
104
+ path: contractPath,
105
+ errors: result.error.issues.map((issue) => `${issue.path.length ? issue.path.join('.') : '(root)'}: ${issue.message}`),
106
+ };
107
+ }
108
+ return { status: 'ok', contract: result.data, path: contractPath };
109
+ }
110
+ /** Resolve the artifact directory to an absolute path, creating it if needed. */
111
+ export async function ensureArtifactDir(contract, rootDir = process.cwd()) {
112
+ const dir = path.resolve(rootDir, contract.environment.artifact_dir);
113
+ await fs.mkdir(path.join(dir, 'logs'), { recursive: true });
114
+ return dir;
115
+ }
@@ -0,0 +1,10 @@
1
+ export interface InitResult {
2
+ created: string[];
3
+ skipped: string[];
4
+ }
5
+ export interface InitOptions {
6
+ force?: boolean;
7
+ /** Only write the agent skill file (for handing the integration to a coding agent). */
8
+ skillOnly?: boolean;
9
+ }
10
+ export declare function initVerification(rootDir?: string, options?: InitOptions): Promise<InitResult>;
@@ -0,0 +1,243 @@
1
+ /**
2
+ * `haystack verification init` — scaffold the verification contract.
3
+ *
4
+ * Detection-driven where evidence exists (framework, package manager, dev
5
+ * command, port, auth-bypass env var), honest TODOs where it doesn't. The
6
+ * generated smoke scenario is real, not a placeholder: it boots the app,
7
+ * waits for readiness, captures the root page + server log, and shuts down.
8
+ */
9
+ import * as fs from 'node:fs/promises';
10
+ import * as path from 'node:path';
11
+ import { fileURLToPath } from 'node:url';
12
+ import { detectProject } from '../utils/detect.js';
13
+ import { CONTRACT_RELATIVE_PATH, DEFAULT_ARTIFACT_DIR } from './contract.js';
14
+ const SKILL_TARGET = '.haystack/skills/install-verification/SKILL.md';
15
+ const AGENTS_TARGET = '.haystack/AGENTS.md';
16
+ const PREFLIGHT_SCRIPT = 'scripts/haystack-preflight';
17
+ const SMOKE_SCRIPT = 'scripts/haystack-smoke';
18
+ export async function initVerification(rootDir = process.cwd(), options = {}) {
19
+ const result = { created: [], skipped: [] };
20
+ const writeFile = async (relPath, content, executable = false) => {
21
+ const target = path.join(rootDir, relPath);
22
+ if (!options.force && (await exists(target))) {
23
+ result.skipped.push(relPath);
24
+ return;
25
+ }
26
+ await fs.mkdir(path.dirname(target), { recursive: true });
27
+ await fs.writeFile(target, content, { encoding: 'utf-8', mode: executable ? 0o755 : 0o644 });
28
+ result.created.push(relPath);
29
+ };
30
+ await writeFile(SKILL_TARGET, await readSkillAsset());
31
+ if (options.skillOnly)
32
+ return result;
33
+ const detected = await detectProject(rootDir);
34
+ const appName = (await readPackageName(rootDir)) ?? path.basename(rootDir);
35
+ const setupCommand = setupCommandFor(detected.packageManager);
36
+ const port = detected.suggestedPort ?? 3000;
37
+ // Evidence-based auth bypass: only prefix the env var when detection found
38
+ // one in the repo's own env examples (never invent a flag the app ignores).
39
+ const startCommand = detected.suggestedAuthBypass
40
+ ? `${detected.suggestedAuthBypass} ${detected.suggestedDevCommand}`
41
+ : (detected.suggestedDevCommand ?? 'echo "TODO: set your dev-server command" && exit 1');
42
+ await writeFile(CONTRACT_RELATIVE_PATH, renderContract({ appName, framework: detected.framework, setupCommand, startCommand }));
43
+ await writeFile(PREFLIGHT_SCRIPT, renderPreflightScript(port), true);
44
+ await writeFile(SMOKE_SCRIPT, renderSmokeScript(port, startCommand), true);
45
+ await writeFile(AGENTS_TARGET, renderAgentsDoc());
46
+ return result;
47
+ }
48
+ async function exists(p) {
49
+ try {
50
+ await fs.access(p);
51
+ return true;
52
+ }
53
+ catch {
54
+ return false;
55
+ }
56
+ }
57
+ async function readPackageName(rootDir) {
58
+ try {
59
+ const pkg = JSON.parse(await fs.readFile(path.join(rootDir, 'package.json'), 'utf-8'));
60
+ return pkg.name?.replace(/^@[^/]+\//, '') ?? null;
61
+ }
62
+ catch {
63
+ return null;
64
+ }
65
+ }
66
+ async function readSkillAsset() {
67
+ // dist/verification/init.js → dist/assets/skills/install-verification.md
68
+ const assetPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'assets', 'skills', 'install-verification.md');
69
+ return fs.readFile(assetPath, 'utf-8');
70
+ }
71
+ function setupCommandFor(pm) {
72
+ switch (pm) {
73
+ case 'pnpm':
74
+ return 'pnpm install --frozen-lockfile';
75
+ case 'yarn':
76
+ return 'yarn install --frozen-lockfile';
77
+ case 'bun':
78
+ return 'bun install';
79
+ default:
80
+ return 'npm ci';
81
+ }
82
+ }
83
+ function renderContract(opts) {
84
+ return `version: 1
85
+
86
+ # Haystack Verification Contract
87
+ #
88
+ # Defines how Haystack (and any coding agent) verifies this app: how to set it
89
+ # up, how to know it's ready, how to log in, which flows to run, and where
90
+ # evidence goes. Check it with:
91
+ #
92
+ # haystack verification validate
93
+ # haystack verification check-safety
94
+ # haystack verification run smoke
95
+
96
+ app:
97
+ name: "${opts.appName}"
98
+ ${opts.framework ? ` framework: "${opts.framework}"\n` : ''}
99
+ environment:
100
+ setup: "${opts.setupCommand}"
101
+ start: "${opts.startCommand}"
102
+ preflight: "bash ${PREFLIGHT_SCRIPT}"
103
+ # reset: "pnpm db:reset" # optional: return the environment to a clean state
104
+ artifact_dir: "${DEFAULT_ARTIFACT_DIR}"
105
+ log_paths:
106
+ - "${DEFAULT_ARTIFACT_DIR}/logs/server.log"
107
+
108
+ # Personas are seeded FAKE users an agent can act as. Wire these up once you
109
+ # have a dev-only login helper — and add safety.production_disabled_check
110
+ # proving that helper is disabled in production builds.
111
+ # personas:
112
+ # admin:
113
+ # login: "pnpm haystack:login admin"
114
+ # description: "Workspace admin test user"
115
+ # member:
116
+ # login: "pnpm haystack:login member"
117
+ # description: "Regular workspace member"
118
+
119
+ scenarios:
120
+ smoke:
121
+ command: "bash ${SMOKE_SCRIPT}"
122
+ description: "Boot the app, wait until it answers, capture the root page and server log"
123
+ preflight: false # the smoke script boots the app itself
124
+ artifacts:
125
+ - "server_logs"
126
+ - "root_html"
127
+
128
+ safety:
129
+ # production_disabled_check: "pnpm test:dev-routes-disabled" # prove dev-only helpers are off in prod builds
130
+ external_services: "sandbox_only"
131
+ allow_real_email: false
132
+ allow_real_payments: false
133
+ allow_customer_data: false
134
+ `;
135
+ }
136
+ function renderPreflightScript(port) {
137
+ return `#!/usr/bin/env bash
138
+ # Haystack preflight — exits 0 only when the app is up and answering.
139
+ # Prefer pointing PATH_SUFFIX at a real health endpoint over the root page.
140
+ set -euo pipefail
141
+
142
+ PORT="\${PORT:-${port}}"
143
+ PATH_SUFFIX="\${PATH_SUFFIX:-/}"
144
+ URL="http://localhost:\${PORT}\${PATH_SUFFIX}"
145
+ TIMEOUT_S="\${TIMEOUT_S:-60}"
146
+
147
+ for _ in $(seq 1 "\${TIMEOUT_S}"); do
148
+ if curl -sf -o /dev/null "\${URL}"; then
149
+ echo "preflight ok: \${URL} is responding"
150
+ exit 0
151
+ fi
152
+ sleep 1
153
+ done
154
+
155
+ echo "preflight failed: \${URL} did not respond within \${TIMEOUT_S}s" >&2
156
+ exit 1
157
+ `;
158
+ }
159
+ function renderSmokeScript(port, startCommand) {
160
+ return `#!/usr/bin/env bash
161
+ # Haystack smoke scenario — boots the app, waits for readiness, captures
162
+ # evidence into the artifact directory, then shuts the app down.
163
+ #
164
+ # Evidence produced:
165
+ # $ARTIFACT_DIR/logs/server.log full server output
166
+ # $ARTIFACT_DIR/smoke-root.html the rendered root page
167
+ #
168
+ # Adapt this to your app: navigate real flows, take screenshots, and append
169
+ # anything you could NOT test to $ARTIFACT_DIR/gaps.txt (one line each).
170
+ set -euo pipefail
171
+
172
+ ARTIFACT_DIR="\${ARTIFACT_DIR:-${DEFAULT_ARTIFACT_DIR}}"
173
+ PORT="\${PORT:-${port}}"
174
+ mkdir -p "\${ARTIFACT_DIR}/logs"
175
+
176
+ # Evidence integrity: if something is already listening on the port, captured
177
+ # "evidence" would come from the wrong server. Refuse instead of lying.
178
+ if curl -sf -o /dev/null "http://localhost:\${PORT}/"; then
179
+ echo "smoke failed: something is already listening on port \${PORT} — stop it or set PORT" >&2
180
+ exit 1
181
+ fi
182
+
183
+ ${startCommand} > "\${ARTIFACT_DIR}/logs/server.log" 2>&1 &
184
+ SERVER_PID=$!
185
+ # Kill the server's children too — package-manager wrappers (pnpm dev) leave
186
+ # the real dev server as a grandchild.
187
+ cleanup() {
188
+ pkill -P "\${SERVER_PID}" 2>/dev/null || true
189
+ kill "\${SERVER_PID}" 2>/dev/null || true
190
+ }
191
+ trap cleanup EXIT
192
+
193
+ PORT="\${PORT}" bash ${PREFLIGHT_SCRIPT}
194
+
195
+ curl -sf "http://localhost:\${PORT}/" -o "\${ARTIFACT_DIR}/smoke-root.html"
196
+ test -s "\${ARTIFACT_DIR}/smoke-root.html"
197
+
198
+ echo "smoke ok: captured \${ARTIFACT_DIR}/smoke-root.html and server log"
199
+ `;
200
+ }
201
+ function renderAgentsDoc() {
202
+ return `# Haystack Verification — Agent Guide
203
+
204
+ This repo has a Haystack verification contract at \`${CONTRACT_RELATIVE_PATH}\`.
205
+ Use it whenever you need to prove a change works, not just that it compiles.
206
+
207
+ ## Commands
208
+
209
+ \`\`\`bash
210
+ haystack verification validate # is the contract trustworthy?
211
+ haystack verification preflight # is the app up and answering?
212
+ haystack verification run smoke # run a scenario, capture evidence
213
+ haystack verification run <scenario> # any scenario defined in the contract
214
+ haystack verification check-safety # prove the setup can't hurt anyone
215
+ haystack verification collect-artifacts # rebuild the manifest from what's on disk
216
+ \`\`\`
217
+
218
+ ## Conventions
219
+
220
+ - **Artifacts** go under the contract's \`artifact_dir\`. Screenshots, logs,
221
+ captured pages — anything a human reviewer should see. Each \`run\` writes a
222
+ \`manifest-<scenario>.json\` and \`review-packet-<scenario>.md\` there.
223
+ - **Gaps**: if a scenario could not test something (a path you skipped, a
224
+ service you mocked away), append one line describing it to
225
+ \`<artifact_dir>/gaps.txt\`. Gaps end up in the review packet — silence reads
226
+ as "covered", so declare what wasn't.
227
+ - **Personas** are seeded fake users. Never use real accounts or customer data.
228
+
229
+ ## Safety rules
230
+
231
+ - Verification must never send real email, charge real cards, call production
232
+ third-party APIs, or touch customer data.
233
+ - Dev-only helpers (login shortcuts, seed endpoints) must be gated so they
234
+ cannot run in production, with a test proving it
235
+ (\`safety.production_disabled_check\`).
236
+ - Don't write secrets into artifacts — \`check-safety\` scans for them.
237
+
238
+ ## Extending the contract
239
+
240
+ Follow \`.haystack/skills/install-verification/SKILL.md\` when adding
241
+ scenarios or personas, and keep changes dev-only and reviewable.
242
+ `;
243
+ }
@@ -0,0 +1,49 @@
1
+ import type { CommandRunRecord } from './runner.js';
2
+ export type VerificationStatus = 'verified' | 'failed' | 'inconclusive' | 'environment_not_ready' | 'unsafe_to_verify' | 'missing_contract';
3
+ export type ArtifactType = 'screenshot' | 'video' | 'log' | 'json' | 'html' | 'trace' | 'file';
4
+ export interface ArtifactEntry {
5
+ type: ArtifactType;
6
+ /** Repo-relative path. */
7
+ path: string;
8
+ description?: string;
9
+ size_bytes?: number;
10
+ }
11
+ export interface ManifestSafety {
12
+ real_email_sent: boolean;
13
+ real_payment_attempted: boolean;
14
+ customer_data_accessed: boolean;
15
+ }
16
+ export interface VerificationManifest {
17
+ contract_version: '1';
18
+ status: VerificationStatus;
19
+ scenario: string;
20
+ commit_sha: string | null;
21
+ started_at: string;
22
+ completed_at: string;
23
+ commands_run: CommandRunRecord[];
24
+ artifacts: ArtifactEntry[];
25
+ /** Known untested paths, read from <artifact_dir>/gaps.txt if the scenario wrote one. */
26
+ gaps: string[];
27
+ safety: ManifestSafety;
28
+ }
29
+ export declare function classifyArtifact(filePath: string): ArtifactType;
30
+ export declare function getHeadSha(cwd: string): string | null;
31
+ export interface CollectOptions {
32
+ rootDir: string;
33
+ artifactDir: string;
34
+ /** Extra files (repo-relative or absolute) to include when they exist, e.g. contract log_paths. */
35
+ extraPaths?: string[];
36
+ /** Only include files modified at/after this time (a run's start). Omit to include everything. */
37
+ since?: Date;
38
+ }
39
+ /**
40
+ * Walk the artifact directory (plus configured log paths) and build the
41
+ * manifest's artifact list. Manifest/packet outputs from previous runs are
42
+ * excluded so evidence doesn't nest.
43
+ */
44
+ export declare function collectArtifacts(opts: CollectOptions): Promise<ArtifactEntry[]>;
45
+ /** Read scenario-declared gaps from `<artifact_dir>/gaps.txt` (one per line), if present. */
46
+ export declare function readGaps(artifactDir: string): Promise<string[]>;
47
+ export declare function writeManifest(manifest: VerificationManifest, artifactDir: string): Promise<string>;
48
+ export declare function renderReviewPacket(manifest: VerificationManifest): string;
49
+ export declare function writeReviewPacket(manifest: VerificationManifest, artifactDir: string): Promise<string>;
@@ -0,0 +1,186 @@
1
+ /**
2
+ * Artifact manifest + review packet.
3
+ *
4
+ * The manifest is the standardized output of every verification run — it's
5
+ * what turns messy command output into reviewable evidence. The review packet
6
+ * is the same data rendered as markdown for humans (and for posting on PRs).
7
+ *
8
+ * Status vocabulary (distinct on purpose — "failed verification" is different
9
+ * from "could not verify"):
10
+ * verified scenario ran and passed
11
+ * failed scenario ran and failed
12
+ * inconclusive artifacts collected without a run (or mixed signal)
13
+ * environment_not_ready preflight/setup failed before the scenario ran
14
+ * unsafe_to_verify safety policy forbids running (real side effects)
15
+ * missing_contract no valid .haystack/verification.yml
16
+ */
17
+ import { spawnSync } from 'node:child_process';
18
+ import * as fs from 'node:fs/promises';
19
+ import * as path from 'node:path';
20
+ const EXTENSION_TYPES = {
21
+ '.png': 'screenshot',
22
+ '.jpg': 'screenshot',
23
+ '.jpeg': 'screenshot',
24
+ '.webp': 'screenshot',
25
+ '.gif': 'screenshot',
26
+ '.webm': 'video',
27
+ '.mp4': 'video',
28
+ '.mov': 'video',
29
+ '.log': 'log',
30
+ '.txt': 'log',
31
+ '.json': 'json',
32
+ '.html': 'html',
33
+ '.zip': 'trace',
34
+ };
35
+ export function classifyArtifact(filePath) {
36
+ return EXTENSION_TYPES[path.extname(filePath).toLowerCase()] ?? 'file';
37
+ }
38
+ export function getHeadSha(cwd) {
39
+ const res = spawnSync('git', ['rev-parse', 'HEAD'], { cwd, encoding: 'utf-8' });
40
+ if (res.status !== 0)
41
+ return null;
42
+ return res.stdout.trim() || null;
43
+ }
44
+ /**
45
+ * Walk the artifact directory (plus configured log paths) and build the
46
+ * manifest's artifact list. Manifest/packet outputs from previous runs are
47
+ * excluded so evidence doesn't nest.
48
+ */
49
+ export async function collectArtifacts(opts) {
50
+ const entries = [];
51
+ const seen = new Set();
52
+ const sinceMs = opts.since ? opts.since.getTime() - 1000 : null; // 1s slack for coarse mtimes
53
+ const addFile = async (absPath, description) => {
54
+ const relPath = path.relative(opts.rootDir, absPath);
55
+ if (seen.has(relPath))
56
+ return;
57
+ if (/(^|\/)(manifest[^/]*\.json|review-packet[^/]*\.md)$/.test(relPath))
58
+ return;
59
+ let stat;
60
+ try {
61
+ stat = await fs.stat(absPath);
62
+ }
63
+ catch {
64
+ return;
65
+ }
66
+ if (!stat.isFile())
67
+ return;
68
+ if (sinceMs !== null && stat.mtimeMs < sinceMs)
69
+ return;
70
+ seen.add(relPath);
71
+ entries.push({
72
+ type: classifyArtifact(absPath),
73
+ path: relPath,
74
+ ...(description ? { description } : {}),
75
+ size_bytes: stat.size,
76
+ });
77
+ };
78
+ const walk = async (dir) => {
79
+ let dirEntries;
80
+ try {
81
+ dirEntries = await fs.readdir(dir, { withFileTypes: true });
82
+ }
83
+ catch {
84
+ return;
85
+ }
86
+ for (const entry of dirEntries) {
87
+ const full = path.join(dir, entry.name);
88
+ if (entry.isDirectory()) {
89
+ await walk(full);
90
+ }
91
+ else if (entry.isFile()) {
92
+ await addFile(full);
93
+ }
94
+ }
95
+ };
96
+ await walk(opts.artifactDir);
97
+ for (const extra of opts.extraPaths ?? []) {
98
+ await addFile(path.resolve(opts.rootDir, extra), 'configured log path');
99
+ }
100
+ entries.sort((a, b) => a.path.localeCompare(b.path));
101
+ return entries;
102
+ }
103
+ /** Read scenario-declared gaps from `<artifact_dir>/gaps.txt` (one per line), if present. */
104
+ export async function readGaps(artifactDir) {
105
+ try {
106
+ const raw = await fs.readFile(path.join(artifactDir, 'gaps.txt'), 'utf-8');
107
+ return raw
108
+ .split('\n')
109
+ .map((line) => line.trim())
110
+ .filter((line) => line.length > 0 && !line.startsWith('#'));
111
+ }
112
+ catch {
113
+ return [];
114
+ }
115
+ }
116
+ export async function writeManifest(manifest, artifactDir) {
117
+ await fs.mkdir(artifactDir, { recursive: true });
118
+ const manifestPath = path.join(artifactDir, `manifest-${manifest.scenario}.json`);
119
+ await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
120
+ return manifestPath;
121
+ }
122
+ const STATUS_HEADLINES = {
123
+ verified: '✅ Verified — evidence looks strong.',
124
+ failed: '❌ Blocked — verification failed.',
125
+ inconclusive: '🟡 Needs human review — verification is incomplete.',
126
+ environment_not_ready: '⚠️ Not ready — the environment could not be prepared, so nothing was verified.',
127
+ unsafe_to_verify: '🛑 Unsafe — the safety policy allows real external side effects; verification was not run.',
128
+ missing_contract: '⚠️ Not ready — repo verification contract is missing or broken.',
129
+ };
130
+ export function renderReviewPacket(manifest) {
131
+ const lines = [];
132
+ lines.push(`# Haystack Verification — \`${manifest.scenario}\``);
133
+ lines.push('');
134
+ lines.push(STATUS_HEADLINES[manifest.status]);
135
+ lines.push('');
136
+ lines.push(`- Commit: \`${manifest.commit_sha ?? 'unknown'}\``);
137
+ lines.push(`- Started: ${manifest.started_at}`);
138
+ lines.push(`- Completed: ${manifest.completed_at}`);
139
+ lines.push('');
140
+ if (manifest.commands_run.length > 0) {
141
+ lines.push('## Commands run');
142
+ lines.push('');
143
+ lines.push('| Command | Status | Duration |');
144
+ lines.push('|---------|--------|----------|');
145
+ for (const cmd of manifest.commands_run) {
146
+ const icon = cmd.status === 'passed' ? '✅' : cmd.status === 'timeout' ? '⏱ timeout' : '❌ failed';
147
+ lines.push(`| \`${cmd.command.replace(/\|/g, '\\|')}\` | ${icon} | ${(cmd.duration_ms / 1000).toFixed(1)}s |`);
148
+ }
149
+ lines.push('');
150
+ }
151
+ lines.push('## Artifacts');
152
+ lines.push('');
153
+ if (manifest.artifacts.length === 0) {
154
+ lines.push('_No artifacts were captured._');
155
+ }
156
+ else {
157
+ for (const artifact of manifest.artifacts) {
158
+ const desc = artifact.description ? ` — ${artifact.description}` : '';
159
+ lines.push(`- \`${artifact.path}\` (${artifact.type})${desc}`);
160
+ }
161
+ }
162
+ lines.push('');
163
+ lines.push('## Gaps');
164
+ lines.push('');
165
+ if (manifest.gaps.length === 0) {
166
+ lines.push('_No gaps declared by this scenario._');
167
+ }
168
+ else {
169
+ for (const gap of manifest.gaps) {
170
+ lines.push(`- ${gap}`);
171
+ }
172
+ }
173
+ lines.push('');
174
+ lines.push('## Safety');
175
+ lines.push('');
176
+ lines.push(`- Real email sent: ${manifest.safety.real_email_sent ? '⚠️ yes' : 'no'}`);
177
+ lines.push(`- Real payment attempted: ${manifest.safety.real_payment_attempted ? '⚠️ yes' : 'no'}`);
178
+ lines.push(`- Customer data accessed: ${manifest.safety.customer_data_accessed ? '⚠️ yes' : 'no'}`);
179
+ lines.push('');
180
+ return lines.join('\n');
181
+ }
182
+ export async function writeReviewPacket(manifest, artifactDir) {
183
+ const packetPath = path.join(artifactDir, `review-packet-${manifest.scenario}.md`);
184
+ await fs.writeFile(packetPath, renderReviewPacket(manifest), 'utf-8');
185
+ return packetPath;
186
+ }
@@ -0,0 +1,20 @@
1
+ export type CommandStatus = 'passed' | 'failed' | 'timeout';
2
+ export interface CommandRunRecord {
3
+ command: string;
4
+ status: CommandStatus;
5
+ exit_code: number | null;
6
+ duration_ms: number;
7
+ /** Repo-relative path to the captured stdout+stderr log, when one was written. */
8
+ log_path?: string;
9
+ }
10
+ export interface RunCommandOptions {
11
+ cwd: string;
12
+ /** Wall-clock cap; the process group is killed when exceeded. */
13
+ timeoutMs: number;
14
+ /** Absolute path to tee combined stdout+stderr into (created if missing). */
15
+ logFile?: string;
16
+ env?: Record<string, string>;
17
+ /** Suppress streaming to the terminal (still captured to logFile). */
18
+ quiet?: boolean;
19
+ }
20
+ export declare function runCommand(command: string, opts: RunCommandOptions): Promise<CommandRunRecord>;