@haystackeditor/cli 0.15.13 → 0.15.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/skills/install-verification.md +67 -0
- package/dist/assets/skills/submit.md +14 -1
- package/dist/commands/init.js +71 -4
- package/dist/commands/setup.js +26 -26
- package/dist/commands/triage.js +98 -17
- package/dist/commands/verification.d.ts +21 -0
- package/dist/commands/verification.js +291 -0
- package/dist/index.js +130 -0
- package/dist/utils/detect.js +3 -1
- package/dist/verification/contract.d.ts +240 -0
- package/dist/verification/contract.js +115 -0
- package/dist/verification/init.d.ts +10 -0
- package/dist/verification/init.js +243 -0
- package/dist/verification/manifest.d.ts +49 -0
- package/dist/verification/manifest.js +186 -0
- package/dist/verification/runner.d.ts +20 -0
- package/dist/verification/runner.js +102 -0
- package/dist/verification/safety.d.ts +10 -0
- package/dist/verification/safety.js +252 -0
- package/dist/verification/validate.d.ts +12 -0
- package/dist/verification/validate.js +219 -0
- package/package.json +3 -1
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Haystack Verification Contract — `.haystack/verification.yml`
|
|
3
|
+
*
|
|
4
|
+
* The contract is the repo-local answer to five questions:
|
|
5
|
+
* 1. How do I set up and start the app?
|
|
6
|
+
* 2. How do I know the app is ready? (preflight)
|
|
7
|
+
* 3. How do I log in as useful test users? (personas)
|
|
8
|
+
* 4. Which verification flows can I run? (scenarios)
|
|
9
|
+
* 5. Where do logs, screenshots, and artifacts go?
|
|
10
|
+
*
|
|
11
|
+
* This is deliberately separate from `.haystack.json`: that file configures
|
|
12
|
+
* Haystack's own sandbox verification (services/flows/fixtures the sandbox
|
|
13
|
+
* drives). The contract instead describes commands the REPO owns — anything
|
|
14
|
+
* (Haystack, CI, a coding agent, a human) can run them and get the same
|
|
15
|
+
* evidence. See docs/VERIFICATION-CONTRACT.md.
|
|
16
|
+
*/
|
|
17
|
+
import * as fs from 'node:fs/promises';
|
|
18
|
+
import * as path from 'node:path';
|
|
19
|
+
import { parse as parseYaml } from 'yaml';
|
|
20
|
+
import { z } from 'zod';
|
|
21
|
+
export const CONTRACT_RELATIVE_PATH = '.haystack/verification.yml';
|
|
22
|
+
export const DEFAULT_ARTIFACT_DIR = '.haystack/artifacts';
|
|
23
|
+
const commandString = z.string().min(1, 'command must be a non-empty string');
|
|
24
|
+
const personaSchema = z.object({
|
|
25
|
+
login: commandString.describe('Command that logs the given persona in (or prints credentials)'),
|
|
26
|
+
description: z.string().optional(),
|
|
27
|
+
});
|
|
28
|
+
const scenarioSchema = z.object({
|
|
29
|
+
command: commandString,
|
|
30
|
+
description: z.string().optional(),
|
|
31
|
+
/** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
|
|
32
|
+
artifacts: z.array(z.string()).optional(),
|
|
33
|
+
/** Wall-clock cap for the scenario command. Default applied by the runner. */
|
|
34
|
+
timeout_seconds: z.number().int().positive().max(7200).optional(),
|
|
35
|
+
/**
|
|
36
|
+
* Whether `run` gates this scenario on environment.preflight. Set false for
|
|
37
|
+
* self-contained scenarios that boot the app themselves (like the scaffolded
|
|
38
|
+
* smoke script) — preflight against a cold environment would always fail.
|
|
39
|
+
*/
|
|
40
|
+
preflight: z.boolean().default(true),
|
|
41
|
+
});
|
|
42
|
+
const safetySchema = z.object({
|
|
43
|
+
/** Command proving dev-only helpers are disabled in production builds. */
|
|
44
|
+
production_disabled_check: commandString.optional(),
|
|
45
|
+
external_services: z.enum(['sandbox_only', 'mocked', 'none']).optional(),
|
|
46
|
+
allow_real_email: z.boolean().default(false),
|
|
47
|
+
allow_real_payments: z.boolean().default(false),
|
|
48
|
+
allow_customer_data: z.boolean().default(false),
|
|
49
|
+
});
|
|
50
|
+
export const verificationContractSchema = z.object({
|
|
51
|
+
// Accept 1 or "1" — YAML authors will write both.
|
|
52
|
+
version: z
|
|
53
|
+
.union([z.literal(1), z.literal('1')])
|
|
54
|
+
.transform(() => 1),
|
|
55
|
+
app: z.object({
|
|
56
|
+
name: commandString,
|
|
57
|
+
framework: z.string().optional(),
|
|
58
|
+
}),
|
|
59
|
+
environment: z.object({
|
|
60
|
+
setup: commandString.optional(),
|
|
61
|
+
start: commandString.optional(),
|
|
62
|
+
preflight: commandString,
|
|
63
|
+
reset: commandString.optional(),
|
|
64
|
+
artifact_dir: z.string().default(DEFAULT_ARTIFACT_DIR),
|
|
65
|
+
log_paths: z.array(z.string()).default([]),
|
|
66
|
+
}),
|
|
67
|
+
personas: z.record(personaSchema).default({}),
|
|
68
|
+
scenarios: z
|
|
69
|
+
.record(scenarioSchema)
|
|
70
|
+
.refine((s) => Object.keys(s).length > 0, {
|
|
71
|
+
message: 'at least one scenario is required',
|
|
72
|
+
}),
|
|
73
|
+
safety: safetySchema,
|
|
74
|
+
});
|
|
75
|
+
/**
|
|
76
|
+
* Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
|
|
77
|
+
* Never throws — parse/schema failures come back as `status: 'invalid'` so
|
|
78
|
+
* callers can map them onto readiness / manifest statuses.
|
|
79
|
+
*/
|
|
80
|
+
export async function loadContract(rootDir = process.cwd()) {
|
|
81
|
+
const contractPath = path.join(rootDir, CONTRACT_RELATIVE_PATH);
|
|
82
|
+
let raw;
|
|
83
|
+
try {
|
|
84
|
+
raw = await fs.readFile(contractPath, 'utf-8');
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return { status: 'missing', path: contractPath };
|
|
88
|
+
}
|
|
89
|
+
let parsed;
|
|
90
|
+
try {
|
|
91
|
+
parsed = parseYaml(raw);
|
|
92
|
+
}
|
|
93
|
+
catch (err) {
|
|
94
|
+
return {
|
|
95
|
+
status: 'invalid',
|
|
96
|
+
path: contractPath,
|
|
97
|
+
errors: [`YAML parse error: ${err instanceof Error ? err.message : String(err)}`],
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
const result = verificationContractSchema.safeParse(parsed);
|
|
101
|
+
if (!result.success) {
|
|
102
|
+
return {
|
|
103
|
+
status: 'invalid',
|
|
104
|
+
path: contractPath,
|
|
105
|
+
errors: result.error.issues.map((issue) => `${issue.path.length ? issue.path.join('.') : '(root)'}: ${issue.message}`),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
return { status: 'ok', contract: result.data, path: contractPath };
|
|
109
|
+
}
|
|
110
|
+
/** Resolve the artifact directory to an absolute path, creating it if needed. */
|
|
111
|
+
export async function ensureArtifactDir(contract, rootDir = process.cwd()) {
|
|
112
|
+
const dir = path.resolve(rootDir, contract.environment.artifact_dir);
|
|
113
|
+
await fs.mkdir(path.join(dir, 'logs'), { recursive: true });
|
|
114
|
+
return dir;
|
|
115
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface InitResult {
|
|
2
|
+
created: string[];
|
|
3
|
+
skipped: string[];
|
|
4
|
+
}
|
|
5
|
+
export interface InitOptions {
|
|
6
|
+
force?: boolean;
|
|
7
|
+
/** Only write the agent skill file (for handing the integration to a coding agent). */
|
|
8
|
+
skillOnly?: boolean;
|
|
9
|
+
}
|
|
10
|
+
export declare function initVerification(rootDir?: string, options?: InitOptions): Promise<InitResult>;
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `haystack verification init` — scaffold the verification contract.
|
|
3
|
+
*
|
|
4
|
+
* Detection-driven where evidence exists (framework, package manager, dev
|
|
5
|
+
* command, port, auth-bypass env var), honest TODOs where it doesn't. The
|
|
6
|
+
* generated smoke scenario is real, not a placeholder: it boots the app,
|
|
7
|
+
* waits for readiness, captures the root page + server log, and shuts down.
|
|
8
|
+
*/
|
|
9
|
+
import * as fs from 'node:fs/promises';
|
|
10
|
+
import * as path from 'node:path';
|
|
11
|
+
import { fileURLToPath } from 'node:url';
|
|
12
|
+
import { detectProject } from '../utils/detect.js';
|
|
13
|
+
import { CONTRACT_RELATIVE_PATH, DEFAULT_ARTIFACT_DIR } from './contract.js';
|
|
14
|
+
const SKILL_TARGET = '.haystack/skills/install-verification/SKILL.md';
|
|
15
|
+
const AGENTS_TARGET = '.haystack/AGENTS.md';
|
|
16
|
+
const PREFLIGHT_SCRIPT = 'scripts/haystack-preflight';
|
|
17
|
+
const SMOKE_SCRIPT = 'scripts/haystack-smoke';
|
|
18
|
+
export async function initVerification(rootDir = process.cwd(), options = {}) {
|
|
19
|
+
const result = { created: [], skipped: [] };
|
|
20
|
+
const writeFile = async (relPath, content, executable = false) => {
|
|
21
|
+
const target = path.join(rootDir, relPath);
|
|
22
|
+
if (!options.force && (await exists(target))) {
|
|
23
|
+
result.skipped.push(relPath);
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
27
|
+
await fs.writeFile(target, content, { encoding: 'utf-8', mode: executable ? 0o755 : 0o644 });
|
|
28
|
+
result.created.push(relPath);
|
|
29
|
+
};
|
|
30
|
+
await writeFile(SKILL_TARGET, await readSkillAsset());
|
|
31
|
+
if (options.skillOnly)
|
|
32
|
+
return result;
|
|
33
|
+
const detected = await detectProject(rootDir);
|
|
34
|
+
const appName = (await readPackageName(rootDir)) ?? path.basename(rootDir);
|
|
35
|
+
const setupCommand = setupCommandFor(detected.packageManager);
|
|
36
|
+
const port = detected.suggestedPort ?? 3000;
|
|
37
|
+
// Evidence-based auth bypass: only prefix the env var when detection found
|
|
38
|
+
// one in the repo's own env examples (never invent a flag the app ignores).
|
|
39
|
+
const startCommand = detected.suggestedAuthBypass
|
|
40
|
+
? `${detected.suggestedAuthBypass} ${detected.suggestedDevCommand}`
|
|
41
|
+
: (detected.suggestedDevCommand ?? 'echo "TODO: set your dev-server command" && exit 1');
|
|
42
|
+
await writeFile(CONTRACT_RELATIVE_PATH, renderContract({ appName, framework: detected.framework, setupCommand, startCommand }));
|
|
43
|
+
await writeFile(PREFLIGHT_SCRIPT, renderPreflightScript(port), true);
|
|
44
|
+
await writeFile(SMOKE_SCRIPT, renderSmokeScript(port, startCommand), true);
|
|
45
|
+
await writeFile(AGENTS_TARGET, renderAgentsDoc());
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
async function exists(p) {
|
|
49
|
+
try {
|
|
50
|
+
await fs.access(p);
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
async function readPackageName(rootDir) {
|
|
58
|
+
try {
|
|
59
|
+
const pkg = JSON.parse(await fs.readFile(path.join(rootDir, 'package.json'), 'utf-8'));
|
|
60
|
+
return pkg.name?.replace(/^@[^/]+\//, '') ?? null;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async function readSkillAsset() {
|
|
67
|
+
// dist/verification/init.js → dist/assets/skills/install-verification.md
|
|
68
|
+
const assetPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'assets', 'skills', 'install-verification.md');
|
|
69
|
+
return fs.readFile(assetPath, 'utf-8');
|
|
70
|
+
}
|
|
71
|
+
function setupCommandFor(pm) {
|
|
72
|
+
switch (pm) {
|
|
73
|
+
case 'pnpm':
|
|
74
|
+
return 'pnpm install --frozen-lockfile';
|
|
75
|
+
case 'yarn':
|
|
76
|
+
return 'yarn install --frozen-lockfile';
|
|
77
|
+
case 'bun':
|
|
78
|
+
return 'bun install';
|
|
79
|
+
default:
|
|
80
|
+
return 'npm ci';
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function renderContract(opts) {
|
|
84
|
+
return `version: 1
|
|
85
|
+
|
|
86
|
+
# Haystack Verification Contract
|
|
87
|
+
#
|
|
88
|
+
# Defines how Haystack (and any coding agent) verifies this app: how to set it
|
|
89
|
+
# up, how to know it's ready, how to log in, which flows to run, and where
|
|
90
|
+
# evidence goes. Check it with:
|
|
91
|
+
#
|
|
92
|
+
# haystack verification validate
|
|
93
|
+
# haystack verification check-safety
|
|
94
|
+
# haystack verification run smoke
|
|
95
|
+
|
|
96
|
+
app:
|
|
97
|
+
name: "${opts.appName}"
|
|
98
|
+
${opts.framework ? ` framework: "${opts.framework}"\n` : ''}
|
|
99
|
+
environment:
|
|
100
|
+
setup: "${opts.setupCommand}"
|
|
101
|
+
start: "${opts.startCommand}"
|
|
102
|
+
preflight: "bash ${PREFLIGHT_SCRIPT}"
|
|
103
|
+
# reset: "pnpm db:reset" # optional: return the environment to a clean state
|
|
104
|
+
artifact_dir: "${DEFAULT_ARTIFACT_DIR}"
|
|
105
|
+
log_paths:
|
|
106
|
+
- "${DEFAULT_ARTIFACT_DIR}/logs/server.log"
|
|
107
|
+
|
|
108
|
+
# Personas are seeded FAKE users an agent can act as. Wire these up once you
|
|
109
|
+
# have a dev-only login helper — and add safety.production_disabled_check
|
|
110
|
+
# proving that helper is disabled in production builds.
|
|
111
|
+
# personas:
|
|
112
|
+
# admin:
|
|
113
|
+
# login: "pnpm haystack:login admin"
|
|
114
|
+
# description: "Workspace admin test user"
|
|
115
|
+
# member:
|
|
116
|
+
# login: "pnpm haystack:login member"
|
|
117
|
+
# description: "Regular workspace member"
|
|
118
|
+
|
|
119
|
+
scenarios:
|
|
120
|
+
smoke:
|
|
121
|
+
command: "bash ${SMOKE_SCRIPT}"
|
|
122
|
+
description: "Boot the app, wait until it answers, capture the root page and server log"
|
|
123
|
+
preflight: false # the smoke script boots the app itself
|
|
124
|
+
artifacts:
|
|
125
|
+
- "server_logs"
|
|
126
|
+
- "root_html"
|
|
127
|
+
|
|
128
|
+
safety:
|
|
129
|
+
# production_disabled_check: "pnpm test:dev-routes-disabled" # prove dev-only helpers are off in prod builds
|
|
130
|
+
external_services: "sandbox_only"
|
|
131
|
+
allow_real_email: false
|
|
132
|
+
allow_real_payments: false
|
|
133
|
+
allow_customer_data: false
|
|
134
|
+
`;
|
|
135
|
+
}
|
|
136
|
+
function renderPreflightScript(port) {
|
|
137
|
+
return `#!/usr/bin/env bash
|
|
138
|
+
# Haystack preflight — exits 0 only when the app is up and answering.
|
|
139
|
+
# Prefer pointing PATH_SUFFIX at a real health endpoint over the root page.
|
|
140
|
+
set -euo pipefail
|
|
141
|
+
|
|
142
|
+
PORT="\${PORT:-${port}}"
|
|
143
|
+
PATH_SUFFIX="\${PATH_SUFFIX:-/}"
|
|
144
|
+
URL="http://localhost:\${PORT}\${PATH_SUFFIX}"
|
|
145
|
+
TIMEOUT_S="\${TIMEOUT_S:-60}"
|
|
146
|
+
|
|
147
|
+
for _ in $(seq 1 "\${TIMEOUT_S}"); do
|
|
148
|
+
if curl -sf -o /dev/null "\${URL}"; then
|
|
149
|
+
echo "preflight ok: \${URL} is responding"
|
|
150
|
+
exit 0
|
|
151
|
+
fi
|
|
152
|
+
sleep 1
|
|
153
|
+
done
|
|
154
|
+
|
|
155
|
+
echo "preflight failed: \${URL} did not respond within \${TIMEOUT_S}s" >&2
|
|
156
|
+
exit 1
|
|
157
|
+
`;
|
|
158
|
+
}
|
|
159
|
+
function renderSmokeScript(port, startCommand) {
|
|
160
|
+
return `#!/usr/bin/env bash
|
|
161
|
+
# Haystack smoke scenario — boots the app, waits for readiness, captures
|
|
162
|
+
# evidence into the artifact directory, then shuts the app down.
|
|
163
|
+
#
|
|
164
|
+
# Evidence produced:
|
|
165
|
+
# $ARTIFACT_DIR/logs/server.log full server output
|
|
166
|
+
# $ARTIFACT_DIR/smoke-root.html the rendered root page
|
|
167
|
+
#
|
|
168
|
+
# Adapt this to your app: navigate real flows, take screenshots, and append
|
|
169
|
+
# anything you could NOT test to $ARTIFACT_DIR/gaps.txt (one line each).
|
|
170
|
+
set -euo pipefail
|
|
171
|
+
|
|
172
|
+
ARTIFACT_DIR="\${ARTIFACT_DIR:-${DEFAULT_ARTIFACT_DIR}}"
|
|
173
|
+
PORT="\${PORT:-${port}}"
|
|
174
|
+
mkdir -p "\${ARTIFACT_DIR}/logs"
|
|
175
|
+
|
|
176
|
+
# Evidence integrity: if something is already listening on the port, captured
|
|
177
|
+
# "evidence" would come from the wrong server. Refuse instead of lying.
|
|
178
|
+
if curl -sf -o /dev/null "http://localhost:\${PORT}/"; then
|
|
179
|
+
echo "smoke failed: something is already listening on port \${PORT} — stop it or set PORT" >&2
|
|
180
|
+
exit 1
|
|
181
|
+
fi
|
|
182
|
+
|
|
183
|
+
${startCommand} > "\${ARTIFACT_DIR}/logs/server.log" 2>&1 &
|
|
184
|
+
SERVER_PID=$!
|
|
185
|
+
# Kill the server's children too — package-manager wrappers (pnpm dev) leave
|
|
186
|
+
# the real dev server as a grandchild.
|
|
187
|
+
cleanup() {
|
|
188
|
+
pkill -P "\${SERVER_PID}" 2>/dev/null || true
|
|
189
|
+
kill "\${SERVER_PID}" 2>/dev/null || true
|
|
190
|
+
}
|
|
191
|
+
trap cleanup EXIT
|
|
192
|
+
|
|
193
|
+
PORT="\${PORT}" bash ${PREFLIGHT_SCRIPT}
|
|
194
|
+
|
|
195
|
+
curl -sf "http://localhost:\${PORT}/" -o "\${ARTIFACT_DIR}/smoke-root.html"
|
|
196
|
+
test -s "\${ARTIFACT_DIR}/smoke-root.html"
|
|
197
|
+
|
|
198
|
+
echo "smoke ok: captured \${ARTIFACT_DIR}/smoke-root.html and server log"
|
|
199
|
+
`;
|
|
200
|
+
}
|
|
201
|
+
function renderAgentsDoc() {
|
|
202
|
+
return `# Haystack Verification — Agent Guide
|
|
203
|
+
|
|
204
|
+
This repo has a Haystack verification contract at \`${CONTRACT_RELATIVE_PATH}\`.
|
|
205
|
+
Use it whenever you need to prove a change works, not just that it compiles.
|
|
206
|
+
|
|
207
|
+
## Commands
|
|
208
|
+
|
|
209
|
+
\`\`\`bash
|
|
210
|
+
haystack verification validate # is the contract trustworthy?
|
|
211
|
+
haystack verification preflight # is the app up and answering?
|
|
212
|
+
haystack verification run smoke # run a scenario, capture evidence
|
|
213
|
+
haystack verification run <scenario> # any scenario defined in the contract
|
|
214
|
+
haystack verification check-safety # prove the setup can't hurt anyone
|
|
215
|
+
haystack verification collect-artifacts # rebuild the manifest from what's on disk
|
|
216
|
+
\`\`\`
|
|
217
|
+
|
|
218
|
+
## Conventions
|
|
219
|
+
|
|
220
|
+
- **Artifacts** go under the contract's \`artifact_dir\`. Screenshots, logs,
|
|
221
|
+
captured pages — anything a human reviewer should see. Each \`run\` writes a
|
|
222
|
+
\`manifest-<scenario>.json\` and \`review-packet-<scenario>.md\` there.
|
|
223
|
+
- **Gaps**: if a scenario could not test something (a path you skipped, a
|
|
224
|
+
service you mocked away), append one line describing it to
|
|
225
|
+
\`<artifact_dir>/gaps.txt\`. Gaps end up in the review packet — silence reads
|
|
226
|
+
as "covered", so declare what wasn't.
|
|
227
|
+
- **Personas** are seeded fake users. Never use real accounts or customer data.
|
|
228
|
+
|
|
229
|
+
## Safety rules
|
|
230
|
+
|
|
231
|
+
- Verification must never send real email, charge real cards, call production
|
|
232
|
+
third-party APIs, or touch customer data.
|
|
233
|
+
- Dev-only helpers (login shortcuts, seed endpoints) must be gated so they
|
|
234
|
+
cannot run in production, with a test proving it
|
|
235
|
+
(\`safety.production_disabled_check\`).
|
|
236
|
+
- Don't write secrets into artifacts — \`check-safety\` scans for them.
|
|
237
|
+
|
|
238
|
+
## Extending the contract
|
|
239
|
+
|
|
240
|
+
Follow \`.haystack/skills/install-verification/SKILL.md\` when adding
|
|
241
|
+
scenarios or personas, and keep changes dev-only and reviewable.
|
|
242
|
+
`;
|
|
243
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { CommandRunRecord } from './runner.js';
|
|
2
|
+
export type VerificationStatus = 'verified' | 'failed' | 'inconclusive' | 'environment_not_ready' | 'unsafe_to_verify' | 'missing_contract';
|
|
3
|
+
export type ArtifactType = 'screenshot' | 'video' | 'log' | 'json' | 'html' | 'trace' | 'file';
|
|
4
|
+
export interface ArtifactEntry {
|
|
5
|
+
type: ArtifactType;
|
|
6
|
+
/** Repo-relative path. */
|
|
7
|
+
path: string;
|
|
8
|
+
description?: string;
|
|
9
|
+
size_bytes?: number;
|
|
10
|
+
}
|
|
11
|
+
export interface ManifestSafety {
|
|
12
|
+
real_email_sent: boolean;
|
|
13
|
+
real_payment_attempted: boolean;
|
|
14
|
+
customer_data_accessed: boolean;
|
|
15
|
+
}
|
|
16
|
+
export interface VerificationManifest {
|
|
17
|
+
contract_version: '1';
|
|
18
|
+
status: VerificationStatus;
|
|
19
|
+
scenario: string;
|
|
20
|
+
commit_sha: string | null;
|
|
21
|
+
started_at: string;
|
|
22
|
+
completed_at: string;
|
|
23
|
+
commands_run: CommandRunRecord[];
|
|
24
|
+
artifacts: ArtifactEntry[];
|
|
25
|
+
/** Known untested paths, read from <artifact_dir>/gaps.txt if the scenario wrote one. */
|
|
26
|
+
gaps: string[];
|
|
27
|
+
safety: ManifestSafety;
|
|
28
|
+
}
|
|
29
|
+
export declare function classifyArtifact(filePath: string): ArtifactType;
|
|
30
|
+
export declare function getHeadSha(cwd: string): string | null;
|
|
31
|
+
export interface CollectOptions {
|
|
32
|
+
rootDir: string;
|
|
33
|
+
artifactDir: string;
|
|
34
|
+
/** Extra files (repo-relative or absolute) to include when they exist, e.g. contract log_paths. */
|
|
35
|
+
extraPaths?: string[];
|
|
36
|
+
/** Only include files modified at/after this time (a run's start). Omit to include everything. */
|
|
37
|
+
since?: Date;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Walk the artifact directory (plus configured log paths) and build the
|
|
41
|
+
* manifest's artifact list. Manifest/packet outputs from previous runs are
|
|
42
|
+
* excluded so evidence doesn't nest.
|
|
43
|
+
*/
|
|
44
|
+
export declare function collectArtifacts(opts: CollectOptions): Promise<ArtifactEntry[]>;
|
|
45
|
+
/** Read scenario-declared gaps from `<artifact_dir>/gaps.txt` (one per line), if present. */
|
|
46
|
+
export declare function readGaps(artifactDir: string): Promise<string[]>;
|
|
47
|
+
export declare function writeManifest(manifest: VerificationManifest, artifactDir: string): Promise<string>;
|
|
48
|
+
export declare function renderReviewPacket(manifest: VerificationManifest): string;
|
|
49
|
+
export declare function writeReviewPacket(manifest: VerificationManifest, artifactDir: string): Promise<string>;
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Artifact manifest + review packet.
|
|
3
|
+
*
|
|
4
|
+
* The manifest is the standardized output of every verification run — it's
|
|
5
|
+
* what turns messy command output into reviewable evidence. The review packet
|
|
6
|
+
* is the same data rendered as markdown for humans (and for posting on PRs).
|
|
7
|
+
*
|
|
8
|
+
* Status vocabulary (distinct on purpose — "failed verification" is different
|
|
9
|
+
* from "could not verify"):
|
|
10
|
+
* verified scenario ran and passed
|
|
11
|
+
* failed scenario ran and failed
|
|
12
|
+
* inconclusive artifacts collected without a run (or mixed signal)
|
|
13
|
+
* environment_not_ready preflight/setup failed before the scenario ran
|
|
14
|
+
* unsafe_to_verify safety policy forbids running (real side effects)
|
|
15
|
+
* missing_contract no valid .haystack/verification.yml
|
|
16
|
+
*/
|
|
17
|
+
import { spawnSync } from 'node:child_process';
|
|
18
|
+
import * as fs from 'node:fs/promises';
|
|
19
|
+
import * as path from 'node:path';
|
|
20
|
+
const EXTENSION_TYPES = {
|
|
21
|
+
'.png': 'screenshot',
|
|
22
|
+
'.jpg': 'screenshot',
|
|
23
|
+
'.jpeg': 'screenshot',
|
|
24
|
+
'.webp': 'screenshot',
|
|
25
|
+
'.gif': 'screenshot',
|
|
26
|
+
'.webm': 'video',
|
|
27
|
+
'.mp4': 'video',
|
|
28
|
+
'.mov': 'video',
|
|
29
|
+
'.log': 'log',
|
|
30
|
+
'.txt': 'log',
|
|
31
|
+
'.json': 'json',
|
|
32
|
+
'.html': 'html',
|
|
33
|
+
'.zip': 'trace',
|
|
34
|
+
};
|
|
35
|
+
export function classifyArtifact(filePath) {
|
|
36
|
+
return EXTENSION_TYPES[path.extname(filePath).toLowerCase()] ?? 'file';
|
|
37
|
+
}
|
|
38
|
+
export function getHeadSha(cwd) {
|
|
39
|
+
const res = spawnSync('git', ['rev-parse', 'HEAD'], { cwd, encoding: 'utf-8' });
|
|
40
|
+
if (res.status !== 0)
|
|
41
|
+
return null;
|
|
42
|
+
return res.stdout.trim() || null;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Walk the artifact directory (plus configured log paths) and build the
|
|
46
|
+
* manifest's artifact list. Manifest/packet outputs from previous runs are
|
|
47
|
+
* excluded so evidence doesn't nest.
|
|
48
|
+
*/
|
|
49
|
+
export async function collectArtifacts(opts) {
|
|
50
|
+
const entries = [];
|
|
51
|
+
const seen = new Set();
|
|
52
|
+
const sinceMs = opts.since ? opts.since.getTime() - 1000 : null; // 1s slack for coarse mtimes
|
|
53
|
+
const addFile = async (absPath, description) => {
|
|
54
|
+
const relPath = path.relative(opts.rootDir, absPath);
|
|
55
|
+
if (seen.has(relPath))
|
|
56
|
+
return;
|
|
57
|
+
if (/(^|\/)(manifest[^/]*\.json|review-packet[^/]*\.md)$/.test(relPath))
|
|
58
|
+
return;
|
|
59
|
+
let stat;
|
|
60
|
+
try {
|
|
61
|
+
stat = await fs.stat(absPath);
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
if (!stat.isFile())
|
|
67
|
+
return;
|
|
68
|
+
if (sinceMs !== null && stat.mtimeMs < sinceMs)
|
|
69
|
+
return;
|
|
70
|
+
seen.add(relPath);
|
|
71
|
+
entries.push({
|
|
72
|
+
type: classifyArtifact(absPath),
|
|
73
|
+
path: relPath,
|
|
74
|
+
...(description ? { description } : {}),
|
|
75
|
+
size_bytes: stat.size,
|
|
76
|
+
});
|
|
77
|
+
};
|
|
78
|
+
const walk = async (dir) => {
|
|
79
|
+
let dirEntries;
|
|
80
|
+
try {
|
|
81
|
+
dirEntries = await fs.readdir(dir, { withFileTypes: true });
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
for (const entry of dirEntries) {
|
|
87
|
+
const full = path.join(dir, entry.name);
|
|
88
|
+
if (entry.isDirectory()) {
|
|
89
|
+
await walk(full);
|
|
90
|
+
}
|
|
91
|
+
else if (entry.isFile()) {
|
|
92
|
+
await addFile(full);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
await walk(opts.artifactDir);
|
|
97
|
+
for (const extra of opts.extraPaths ?? []) {
|
|
98
|
+
await addFile(path.resolve(opts.rootDir, extra), 'configured log path');
|
|
99
|
+
}
|
|
100
|
+
entries.sort((a, b) => a.path.localeCompare(b.path));
|
|
101
|
+
return entries;
|
|
102
|
+
}
|
|
103
|
+
/** Read scenario-declared gaps from `<artifact_dir>/gaps.txt` (one per line), if present. */
|
|
104
|
+
export async function readGaps(artifactDir) {
|
|
105
|
+
try {
|
|
106
|
+
const raw = await fs.readFile(path.join(artifactDir, 'gaps.txt'), 'utf-8');
|
|
107
|
+
return raw
|
|
108
|
+
.split('\n')
|
|
109
|
+
.map((line) => line.trim())
|
|
110
|
+
.filter((line) => line.length > 0 && !line.startsWith('#'));
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return [];
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
export async function writeManifest(manifest, artifactDir) {
|
|
117
|
+
await fs.mkdir(artifactDir, { recursive: true });
|
|
118
|
+
const manifestPath = path.join(artifactDir, `manifest-${manifest.scenario}.json`);
|
|
119
|
+
await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
|
|
120
|
+
return manifestPath;
|
|
121
|
+
}
|
|
122
|
+
const STATUS_HEADLINES = {
|
|
123
|
+
verified: '✅ Verified — evidence looks strong.',
|
|
124
|
+
failed: '❌ Blocked — verification failed.',
|
|
125
|
+
inconclusive: '🟡 Needs human review — verification is incomplete.',
|
|
126
|
+
environment_not_ready: '⚠️ Not ready — the environment could not be prepared, so nothing was verified.',
|
|
127
|
+
unsafe_to_verify: '🛑 Unsafe — the safety policy allows real external side effects; verification was not run.',
|
|
128
|
+
missing_contract: '⚠️ Not ready — repo verification contract is missing or broken.',
|
|
129
|
+
};
|
|
130
|
+
export function renderReviewPacket(manifest) {
|
|
131
|
+
const lines = [];
|
|
132
|
+
lines.push(`# Haystack Verification — \`${manifest.scenario}\``);
|
|
133
|
+
lines.push('');
|
|
134
|
+
lines.push(STATUS_HEADLINES[manifest.status]);
|
|
135
|
+
lines.push('');
|
|
136
|
+
lines.push(`- Commit: \`${manifest.commit_sha ?? 'unknown'}\``);
|
|
137
|
+
lines.push(`- Started: ${manifest.started_at}`);
|
|
138
|
+
lines.push(`- Completed: ${manifest.completed_at}`);
|
|
139
|
+
lines.push('');
|
|
140
|
+
if (manifest.commands_run.length > 0) {
|
|
141
|
+
lines.push('## Commands run');
|
|
142
|
+
lines.push('');
|
|
143
|
+
lines.push('| Command | Status | Duration |');
|
|
144
|
+
lines.push('|---------|--------|----------|');
|
|
145
|
+
for (const cmd of manifest.commands_run) {
|
|
146
|
+
const icon = cmd.status === 'passed' ? '✅' : cmd.status === 'timeout' ? '⏱ timeout' : '❌ failed';
|
|
147
|
+
lines.push(`| \`${cmd.command.replace(/\|/g, '\\|')}\` | ${icon} | ${(cmd.duration_ms / 1000).toFixed(1)}s |`);
|
|
148
|
+
}
|
|
149
|
+
lines.push('');
|
|
150
|
+
}
|
|
151
|
+
lines.push('## Artifacts');
|
|
152
|
+
lines.push('');
|
|
153
|
+
if (manifest.artifacts.length === 0) {
|
|
154
|
+
lines.push('_No artifacts were captured._');
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
for (const artifact of manifest.artifacts) {
|
|
158
|
+
const desc = artifact.description ? ` — ${artifact.description}` : '';
|
|
159
|
+
lines.push(`- \`${artifact.path}\` (${artifact.type})${desc}`);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
lines.push('');
|
|
163
|
+
lines.push('## Gaps');
|
|
164
|
+
lines.push('');
|
|
165
|
+
if (manifest.gaps.length === 0) {
|
|
166
|
+
lines.push('_No gaps declared by this scenario._');
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
for (const gap of manifest.gaps) {
|
|
170
|
+
lines.push(`- ${gap}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
lines.push('');
|
|
174
|
+
lines.push('## Safety');
|
|
175
|
+
lines.push('');
|
|
176
|
+
lines.push(`- Real email sent: ${manifest.safety.real_email_sent ? '⚠️ yes' : 'no'}`);
|
|
177
|
+
lines.push(`- Real payment attempted: ${manifest.safety.real_payment_attempted ? '⚠️ yes' : 'no'}`);
|
|
178
|
+
lines.push(`- Customer data accessed: ${manifest.safety.customer_data_accessed ? '⚠️ yes' : 'no'}`);
|
|
179
|
+
lines.push('');
|
|
180
|
+
return lines.join('\n');
|
|
181
|
+
}
|
|
182
|
+
export async function writeReviewPacket(manifest, artifactDir) {
|
|
183
|
+
const packetPath = path.join(artifactDir, `review-packet-${manifest.scenario}.md`);
|
|
184
|
+
await fs.writeFile(packetPath, renderReviewPacket(manifest), 'utf-8');
|
|
185
|
+
return packetPath;
|
|
186
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export type CommandStatus = 'passed' | 'failed' | 'timeout';
|
|
2
|
+
export interface CommandRunRecord {
|
|
3
|
+
command: string;
|
|
4
|
+
status: CommandStatus;
|
|
5
|
+
exit_code: number | null;
|
|
6
|
+
duration_ms: number;
|
|
7
|
+
/** Repo-relative path to the captured stdout+stderr log, when one was written. */
|
|
8
|
+
log_path?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface RunCommandOptions {
|
|
11
|
+
cwd: string;
|
|
12
|
+
/** Wall-clock cap; the process group is killed when exceeded. */
|
|
13
|
+
timeoutMs: number;
|
|
14
|
+
/** Absolute path to tee combined stdout+stderr into (created if missing). */
|
|
15
|
+
logFile?: string;
|
|
16
|
+
env?: Record<string, string>;
|
|
17
|
+
/** Suppress streaming to the terminal (still captured to logFile). */
|
|
18
|
+
quiet?: boolean;
|
|
19
|
+
}
|
|
20
|
+
export declare function runCommand(command: string, opts: RunCommandOptions): Promise<CommandRunRecord>;
|