@haystackeditor/cli 0.15.17 → 0.15.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +45 -6
  2. package/dist/assets/skills/map-your-system.md +138 -0
  3. package/dist/commands/ask.d.ts +14 -0
  4. package/dist/commands/ask.js +20 -0
  5. package/dist/commands/inbox.d.ts +65 -0
  6. package/dist/commands/inbox.js +137 -0
  7. package/dist/commands/mcp.js +88 -134
  8. package/dist/commands/pr.d.ts +37 -1
  9. package/dist/commands/pr.js +95 -77
  10. package/dist/commands/review.d.ts +24 -0
  11. package/dist/commands/review.js +193 -0
  12. package/dist/commands/schema-cmd.js +1 -1
  13. package/dist/commands/skills.js +4 -0
  14. package/dist/commands/traces.d.ts +48 -0
  15. package/dist/commands/traces.js +92 -0
  16. package/dist/commands/triage.d.ts +12 -0
  17. package/dist/commands/triage.js +90 -58
  18. package/dist/commands/webhooks.js +3 -6
  19. package/dist/index.js +79 -148
  20. package/dist/schema.d.ts +5 -2
  21. package/dist/schema.js +5 -2
  22. package/dist/utils/analysis-api.d.ts +40 -0
  23. package/dist/utils/analysis-api.js +108 -3
  24. package/dist/utils/haystack-api.d.ts +20 -0
  25. package/dist/utils/haystack-api.js +66 -0
  26. package/package.json +4 -4
  27. package/schemas/ask.v1.json +40 -0
  28. package/schemas/inbox.v1.json +27 -0
  29. package/schemas/pr.v2.json +54 -0
  30. package/schemas/traces.v1.json +53 -0
  31. package/schemas/triage.v2.json +64 -0
  32. package/dist/assets/skills/install-verification.md +0 -67
  33. package/dist/commands/verification.d.ts +0 -21
  34. package/dist/commands/verification.js +0 -291
  35. package/dist/verification/contract.d.ts +0 -240
  36. package/dist/verification/contract.js +0 -115
  37. package/dist/verification/init.d.ts +0 -10
  38. package/dist/verification/init.js +0 -243
  39. package/dist/verification/manifest.d.ts +0 -49
  40. package/dist/verification/manifest.js +0 -186
  41. package/dist/verification/runner.d.ts +0 -20
  42. package/dist/verification/runner.js +0 -102
  43. package/dist/verification/safety.d.ts +0 -10
  44. package/dist/verification/safety.js +0 -252
  45. package/dist/verification/validate.d.ts +0 -12
  46. package/dist/verification/validate.js +0 -219
@@ -1,240 +0,0 @@
1
- import { z } from 'zod';
2
- export declare const CONTRACT_RELATIVE_PATH = ".haystack/verification.yml";
3
- export declare const DEFAULT_ARTIFACT_DIR = ".haystack/artifacts";
4
- declare const scenarioSchema: z.ZodObject<{
5
- command: z.ZodString;
6
- description: z.ZodOptional<z.ZodString>;
7
- /** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
8
- artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
9
- /** Wall-clock cap for the scenario command. Default applied by the runner. */
10
- timeout_seconds: z.ZodOptional<z.ZodNumber>;
11
- /**
12
- * Whether `run` gates this scenario on environment.preflight. Set false for
13
- * self-contained scenarios that boot the app themselves (like the scaffolded
14
- * smoke script) — preflight against a cold environment would always fail.
15
- */
16
- preflight: z.ZodDefault<z.ZodBoolean>;
17
- }, "strip", z.ZodTypeAny, {
18
- command: string;
19
- preflight: boolean;
20
- description?: string | undefined;
21
- artifacts?: string[] | undefined;
22
- timeout_seconds?: number | undefined;
23
- }, {
24
- command: string;
25
- description?: string | undefined;
26
- artifacts?: string[] | undefined;
27
- timeout_seconds?: number | undefined;
28
- preflight?: boolean | undefined;
29
- }>;
30
- declare const safetySchema: z.ZodObject<{
31
- /** Command proving dev-only helpers are disabled in production builds. */
32
- production_disabled_check: z.ZodOptional<z.ZodString>;
33
- external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
34
- allow_real_email: z.ZodDefault<z.ZodBoolean>;
35
- allow_real_payments: z.ZodDefault<z.ZodBoolean>;
36
- allow_customer_data: z.ZodDefault<z.ZodBoolean>;
37
- }, "strip", z.ZodTypeAny, {
38
- allow_real_email: boolean;
39
- allow_real_payments: boolean;
40
- allow_customer_data: boolean;
41
- production_disabled_check?: string | undefined;
42
- external_services?: "none" | "sandbox_only" | "mocked" | undefined;
43
- }, {
44
- production_disabled_check?: string | undefined;
45
- external_services?: "none" | "sandbox_only" | "mocked" | undefined;
46
- allow_real_email?: boolean | undefined;
47
- allow_real_payments?: boolean | undefined;
48
- allow_customer_data?: boolean | undefined;
49
- }>;
50
- export declare const verificationContractSchema: z.ZodObject<{
51
- version: z.ZodEffects<z.ZodUnion<[z.ZodLiteral<1>, z.ZodLiteral<"1">]>, 1, 1 | "1">;
52
- app: z.ZodObject<{
53
- name: z.ZodString;
54
- framework: z.ZodOptional<z.ZodString>;
55
- }, "strip", z.ZodTypeAny, {
56
- name: string;
57
- framework?: string | undefined;
58
- }, {
59
- name: string;
60
- framework?: string | undefined;
61
- }>;
62
- environment: z.ZodObject<{
63
- setup: z.ZodOptional<z.ZodString>;
64
- start: z.ZodOptional<z.ZodString>;
65
- preflight: z.ZodString;
66
- reset: z.ZodOptional<z.ZodString>;
67
- artifact_dir: z.ZodDefault<z.ZodString>;
68
- log_paths: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
69
- }, "strip", z.ZodTypeAny, {
70
- preflight: string;
71
- artifact_dir: string;
72
- log_paths: string[];
73
- start?: string | undefined;
74
- setup?: string | undefined;
75
- reset?: string | undefined;
76
- }, {
77
- preflight: string;
78
- start?: string | undefined;
79
- setup?: string | undefined;
80
- reset?: string | undefined;
81
- artifact_dir?: string | undefined;
82
- log_paths?: string[] | undefined;
83
- }>;
84
- personas: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
85
- login: z.ZodString;
86
- description: z.ZodOptional<z.ZodString>;
87
- }, "strip", z.ZodTypeAny, {
88
- login: string;
89
- description?: string | undefined;
90
- }, {
91
- login: string;
92
- description?: string | undefined;
93
- }>>>;
94
- scenarios: z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodObject<{
95
- command: z.ZodString;
96
- description: z.ZodOptional<z.ZodString>;
97
- /** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
98
- artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
99
- /** Wall-clock cap for the scenario command. Default applied by the runner. */
100
- timeout_seconds: z.ZodOptional<z.ZodNumber>;
101
- /**
102
- * Whether `run` gates this scenario on environment.preflight. Set false for
103
- * self-contained scenarios that boot the app themselves (like the scaffolded
104
- * smoke script) — preflight against a cold environment would always fail.
105
- */
106
- preflight: z.ZodDefault<z.ZodBoolean>;
107
- }, "strip", z.ZodTypeAny, {
108
- command: string;
109
- preflight: boolean;
110
- description?: string | undefined;
111
- artifacts?: string[] | undefined;
112
- timeout_seconds?: number | undefined;
113
- }, {
114
- command: string;
115
- description?: string | undefined;
116
- artifacts?: string[] | undefined;
117
- timeout_seconds?: number | undefined;
118
- preflight?: boolean | undefined;
119
- }>>, Record<string, {
120
- command: string;
121
- preflight: boolean;
122
- description?: string | undefined;
123
- artifacts?: string[] | undefined;
124
- timeout_seconds?: number | undefined;
125
- }>, Record<string, {
126
- command: string;
127
- description?: string | undefined;
128
- artifacts?: string[] | undefined;
129
- timeout_seconds?: number | undefined;
130
- preflight?: boolean | undefined;
131
- }>>;
132
- safety: z.ZodObject<{
133
- /** Command proving dev-only helpers are disabled in production builds. */
134
- production_disabled_check: z.ZodOptional<z.ZodString>;
135
- external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
136
- allow_real_email: z.ZodDefault<z.ZodBoolean>;
137
- allow_real_payments: z.ZodDefault<z.ZodBoolean>;
138
- allow_customer_data: z.ZodDefault<z.ZodBoolean>;
139
- }, "strip", z.ZodTypeAny, {
140
- allow_real_email: boolean;
141
- allow_real_payments: boolean;
142
- allow_customer_data: boolean;
143
- production_disabled_check?: string | undefined;
144
- external_services?: "none" | "sandbox_only" | "mocked" | undefined;
145
- }, {
146
- production_disabled_check?: string | undefined;
147
- external_services?: "none" | "sandbox_only" | "mocked" | undefined;
148
- allow_real_email?: boolean | undefined;
149
- allow_real_payments?: boolean | undefined;
150
- allow_customer_data?: boolean | undefined;
151
- }>;
152
- }, "strip", z.ZodTypeAny, {
153
- version: 1;
154
- app: {
155
- name: string;
156
- framework?: string | undefined;
157
- };
158
- environment: {
159
- preflight: string;
160
- artifact_dir: string;
161
- log_paths: string[];
162
- start?: string | undefined;
163
- setup?: string | undefined;
164
- reset?: string | undefined;
165
- };
166
- personas: Record<string, {
167
- login: string;
168
- description?: string | undefined;
169
- }>;
170
- scenarios: Record<string, {
171
- command: string;
172
- preflight: boolean;
173
- description?: string | undefined;
174
- artifacts?: string[] | undefined;
175
- timeout_seconds?: number | undefined;
176
- }>;
177
- safety: {
178
- allow_real_email: boolean;
179
- allow_real_payments: boolean;
180
- allow_customer_data: boolean;
181
- production_disabled_check?: string | undefined;
182
- external_services?: "none" | "sandbox_only" | "mocked" | undefined;
183
- };
184
- }, {
185
- version: 1 | "1";
186
- app: {
187
- name: string;
188
- framework?: string | undefined;
189
- };
190
- environment: {
191
- preflight: string;
192
- start?: string | undefined;
193
- setup?: string | undefined;
194
- reset?: string | undefined;
195
- artifact_dir?: string | undefined;
196
- log_paths?: string[] | undefined;
197
- };
198
- scenarios: Record<string, {
199
- command: string;
200
- description?: string | undefined;
201
- artifacts?: string[] | undefined;
202
- timeout_seconds?: number | undefined;
203
- preflight?: boolean | undefined;
204
- }>;
205
- safety: {
206
- production_disabled_check?: string | undefined;
207
- external_services?: "none" | "sandbox_only" | "mocked" | undefined;
208
- allow_real_email?: boolean | undefined;
209
- allow_real_payments?: boolean | undefined;
210
- allow_customer_data?: boolean | undefined;
211
- };
212
- personas?: Record<string, {
213
- login: string;
214
- description?: string | undefined;
215
- }> | undefined;
216
- }>;
217
- export type VerificationContract = z.infer<typeof verificationContractSchema>;
218
- export type VerificationScenario = z.infer<typeof scenarioSchema>;
219
- export type VerificationSafety = z.infer<typeof safetySchema>;
220
- export type ContractLoadResult = {
221
- status: 'ok';
222
- contract: VerificationContract;
223
- path: string;
224
- } | {
225
- status: 'missing';
226
- path: string;
227
- } | {
228
- status: 'invalid';
229
- path: string;
230
- errors: string[];
231
- };
232
- /**
233
- * Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
234
- * Never throws — parse/schema failures come back as `status: 'invalid'` so
235
- * callers can map them onto readiness / manifest statuses.
236
- */
237
- export declare function loadContract(rootDir?: string): Promise<ContractLoadResult>;
238
- /** Resolve the artifact directory to an absolute path, creating it if needed. */
239
- export declare function ensureArtifactDir(contract: VerificationContract, rootDir?: string): Promise<string>;
240
- export {};
@@ -1,115 +0,0 @@
1
- /**
2
- * Haystack Verification Contract — `.haystack/verification.yml`
3
- *
4
- * The contract is the repo-local answer to five questions:
5
- * 1. How do I set up and start the app?
6
- * 2. How do I know the app is ready? (preflight)
7
- * 3. How do I log in as useful test users? (personas)
8
- * 4. Which verification flows can I run? (scenarios)
9
- * 5. Where do logs, screenshots, and artifacts go?
10
- *
11
- * This is deliberately separate from `.haystack.json`: that file configures
12
- * Haystack's own sandbox verification (services/flows/fixtures the sandbox
13
- * drives). The contract instead describes commands the REPO owns — anything
14
- * (Haystack, CI, a coding agent, a human) can run them and get the same
15
- * evidence. See docs/VERIFICATION-CONTRACT.md.
16
- */
17
- import * as fs from 'node:fs/promises';
18
- import * as path from 'node:path';
19
- import { parse as parseYaml } from 'yaml';
20
- import { z } from 'zod';
21
- export const CONTRACT_RELATIVE_PATH = '.haystack/verification.yml';
22
- export const DEFAULT_ARTIFACT_DIR = '.haystack/artifacts';
23
- const commandString = z.string().min(1, 'command must be a non-empty string');
24
- const personaSchema = z.object({
25
- login: commandString.describe('Command that logs the given persona in (or prints credentials)'),
26
- description: z.string().optional(),
27
- });
28
- const scenarioSchema = z.object({
29
- command: commandString,
30
- description: z.string().optional(),
31
- /** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
32
- artifacts: z.array(z.string()).optional(),
33
- /** Wall-clock cap for the scenario command. Default applied by the runner. */
34
- timeout_seconds: z.number().int().positive().max(7200).optional(),
35
- /**
36
- * Whether `run` gates this scenario on environment.preflight. Set false for
37
- * self-contained scenarios that boot the app themselves (like the scaffolded
38
- * smoke script) — preflight against a cold environment would always fail.
39
- */
40
- preflight: z.boolean().default(true),
41
- });
42
- const safetySchema = z.object({
43
- /** Command proving dev-only helpers are disabled in production builds. */
44
- production_disabled_check: commandString.optional(),
45
- external_services: z.enum(['sandbox_only', 'mocked', 'none']).optional(),
46
- allow_real_email: z.boolean().default(false),
47
- allow_real_payments: z.boolean().default(false),
48
- allow_customer_data: z.boolean().default(false),
49
- });
50
- export const verificationContractSchema = z.object({
51
- // Accept 1 or "1" — YAML authors will write both.
52
- version: z
53
- .union([z.literal(1), z.literal('1')])
54
- .transform(() => 1),
55
- app: z.object({
56
- name: commandString,
57
- framework: z.string().optional(),
58
- }),
59
- environment: z.object({
60
- setup: commandString.optional(),
61
- start: commandString.optional(),
62
- preflight: commandString,
63
- reset: commandString.optional(),
64
- artifact_dir: z.string().default(DEFAULT_ARTIFACT_DIR),
65
- log_paths: z.array(z.string()).default([]),
66
- }),
67
- personas: z.record(personaSchema).default({}),
68
- scenarios: z
69
- .record(scenarioSchema)
70
- .refine((s) => Object.keys(s).length > 0, {
71
- message: 'at least one scenario is required',
72
- }),
73
- safety: safetySchema,
74
- });
75
- /**
76
- * Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
77
- * Never throws — parse/schema failures come back as `status: 'invalid'` so
78
- * callers can map them onto readiness / manifest statuses.
79
- */
80
- export async function loadContract(rootDir = process.cwd()) {
81
- const contractPath = path.join(rootDir, CONTRACT_RELATIVE_PATH);
82
- let raw;
83
- try {
84
- raw = await fs.readFile(contractPath, 'utf-8');
85
- }
86
- catch {
87
- return { status: 'missing', path: contractPath };
88
- }
89
- let parsed;
90
- try {
91
- parsed = parseYaml(raw);
92
- }
93
- catch (err) {
94
- return {
95
- status: 'invalid',
96
- path: contractPath,
97
- errors: [`YAML parse error: ${err instanceof Error ? err.message : String(err)}`],
98
- };
99
- }
100
- const result = verificationContractSchema.safeParse(parsed);
101
- if (!result.success) {
102
- return {
103
- status: 'invalid',
104
- path: contractPath,
105
- errors: result.error.issues.map((issue) => `${issue.path.length ? issue.path.join('.') : '(root)'}: ${issue.message}`),
106
- };
107
- }
108
- return { status: 'ok', contract: result.data, path: contractPath };
109
- }
110
- /** Resolve the artifact directory to an absolute path, creating it if needed. */
111
- export async function ensureArtifactDir(contract, rootDir = process.cwd()) {
112
- const dir = path.resolve(rootDir, contract.environment.artifact_dir);
113
- await fs.mkdir(path.join(dir, 'logs'), { recursive: true });
114
- return dir;
115
- }
@@ -1,10 +0,0 @@
1
- export interface InitResult {
2
- created: string[];
3
- skipped: string[];
4
- }
5
- export interface InitOptions {
6
- force?: boolean;
7
- /** Only write the agent skill file (for handing the integration to a coding agent). */
8
- skillOnly?: boolean;
9
- }
10
- export declare function initVerification(rootDir?: string, options?: InitOptions): Promise<InitResult>;
@@ -1,243 +0,0 @@
1
- /**
2
- * `haystack verification init` — scaffold the verification contract.
3
- *
4
- * Detection-driven where evidence exists (framework, package manager, dev
5
- * command, port, auth-bypass env var), honest TODOs where it doesn't. The
6
- * generated smoke scenario is real, not a placeholder: it boots the app,
7
- * waits for readiness, captures the root page + server log, and shuts down.
8
- */
9
- import * as fs from 'node:fs/promises';
10
- import * as path from 'node:path';
11
- import { fileURLToPath } from 'node:url';
12
- import { detectProject } from '../utils/detect.js';
13
- import { CONTRACT_RELATIVE_PATH, DEFAULT_ARTIFACT_DIR } from './contract.js';
14
- const SKILL_TARGET = '.haystack/skills/install-verification/SKILL.md';
15
- const AGENTS_TARGET = '.haystack/AGENTS.md';
16
- const PREFLIGHT_SCRIPT = 'scripts/haystack-preflight';
17
- const SMOKE_SCRIPT = 'scripts/haystack-smoke';
18
- export async function initVerification(rootDir = process.cwd(), options = {}) {
19
- const result = { created: [], skipped: [] };
20
- const writeFile = async (relPath, content, executable = false) => {
21
- const target = path.join(rootDir, relPath);
22
- if (!options.force && (await exists(target))) {
23
- result.skipped.push(relPath);
24
- return;
25
- }
26
- await fs.mkdir(path.dirname(target), { recursive: true });
27
- await fs.writeFile(target, content, { encoding: 'utf-8', mode: executable ? 0o755 : 0o644 });
28
- result.created.push(relPath);
29
- };
30
- await writeFile(SKILL_TARGET, await readSkillAsset());
31
- if (options.skillOnly)
32
- return result;
33
- const detected = await detectProject(rootDir);
34
- const appName = (await readPackageName(rootDir)) ?? path.basename(rootDir);
35
- const setupCommand = setupCommandFor(detected.packageManager);
36
- const port = detected.suggestedPort ?? 3000;
37
- // Evidence-based auth bypass: only prefix the env var when detection found
38
- // one in the repo's own env examples (never invent a flag the app ignores).
39
- const startCommand = detected.suggestedAuthBypass
40
- ? `${detected.suggestedAuthBypass} ${detected.suggestedDevCommand}`
41
- : (detected.suggestedDevCommand ?? 'echo "TODO: set your dev-server command" && exit 1');
42
- await writeFile(CONTRACT_RELATIVE_PATH, renderContract({ appName, framework: detected.framework, setupCommand, startCommand }));
43
- await writeFile(PREFLIGHT_SCRIPT, renderPreflightScript(port), true);
44
- await writeFile(SMOKE_SCRIPT, renderSmokeScript(port, startCommand), true);
45
- await writeFile(AGENTS_TARGET, renderAgentsDoc());
46
- return result;
47
- }
48
- async function exists(p) {
49
- try {
50
- await fs.access(p);
51
- return true;
52
- }
53
- catch {
54
- return false;
55
- }
56
- }
57
- async function readPackageName(rootDir) {
58
- try {
59
- const pkg = JSON.parse(await fs.readFile(path.join(rootDir, 'package.json'), 'utf-8'));
60
- return pkg.name?.replace(/^@[^/]+\//, '') ?? null;
61
- }
62
- catch {
63
- return null;
64
- }
65
- }
66
- async function readSkillAsset() {
67
- // dist/verification/init.js → dist/assets/skills/install-verification.md
68
- const assetPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'assets', 'skills', 'install-verification.md');
69
- return fs.readFile(assetPath, 'utf-8');
70
- }
71
- function setupCommandFor(pm) {
72
- switch (pm) {
73
- case 'pnpm':
74
- return 'pnpm install --frozen-lockfile';
75
- case 'yarn':
76
- return 'yarn install --frozen-lockfile';
77
- case 'bun':
78
- return 'bun install';
79
- default:
80
- return 'npm ci';
81
- }
82
- }
83
- function renderContract(opts) {
84
- return `version: 1
85
-
86
- # Haystack Verification Contract
87
- #
88
- # Defines how Haystack (and any coding agent) verifies this app: how to set it
89
- # up, how to know it's ready, how to log in, which flows to run, and where
90
- # evidence goes. Check it with:
91
- #
92
- # haystack verification validate
93
- # haystack verification check-safety
94
- # haystack verification run smoke
95
-
96
- app:
97
- name: "${opts.appName}"
98
- ${opts.framework ? ` framework: "${opts.framework}"\n` : ''}
99
- environment:
100
- setup: "${opts.setupCommand}"
101
- start: "${opts.startCommand}"
102
- preflight: "bash ${PREFLIGHT_SCRIPT}"
103
- # reset: "pnpm db:reset" # optional: return the environment to a clean state
104
- artifact_dir: "${DEFAULT_ARTIFACT_DIR}"
105
- log_paths:
106
- - "${DEFAULT_ARTIFACT_DIR}/logs/server.log"
107
-
108
- # Personas are seeded FAKE users an agent can act as. Wire these up once you
109
- # have a dev-only login helper — and add safety.production_disabled_check
110
- # proving that helper is disabled in production builds.
111
- # personas:
112
- # admin:
113
- # login: "pnpm haystack:login admin"
114
- # description: "Workspace admin test user"
115
- # member:
116
- # login: "pnpm haystack:login member"
117
- # description: "Regular workspace member"
118
-
119
- scenarios:
120
- smoke:
121
- command: "bash ${SMOKE_SCRIPT}"
122
- description: "Boot the app, wait until it answers, capture the root page and server log"
123
- preflight: false # the smoke script boots the app itself
124
- artifacts:
125
- - "server_logs"
126
- - "root_html"
127
-
128
- safety:
129
- # production_disabled_check: "pnpm test:dev-routes-disabled" # prove dev-only helpers are off in prod builds
130
- external_services: "sandbox_only"
131
- allow_real_email: false
132
- allow_real_payments: false
133
- allow_customer_data: false
134
- `;
135
- }
136
- function renderPreflightScript(port) {
137
- return `#!/usr/bin/env bash
138
- # Haystack preflight — exits 0 only when the app is up and answering.
139
- # Prefer pointing PATH_SUFFIX at a real health endpoint over the root page.
140
- set -euo pipefail
141
-
142
- PORT="\${PORT:-${port}}"
143
- PATH_SUFFIX="\${PATH_SUFFIX:-/}"
144
- URL="http://localhost:\${PORT}\${PATH_SUFFIX}"
145
- TIMEOUT_S="\${TIMEOUT_S:-60}"
146
-
147
- for _ in $(seq 1 "\${TIMEOUT_S}"); do
148
- if curl -sf -o /dev/null "\${URL}"; then
149
- echo "preflight ok: \${URL} is responding"
150
- exit 0
151
- fi
152
- sleep 1
153
- done
154
-
155
- echo "preflight failed: \${URL} did not respond within \${TIMEOUT_S}s" >&2
156
- exit 1
157
- `;
158
- }
159
- function renderSmokeScript(port, startCommand) {
160
- return `#!/usr/bin/env bash
161
- # Haystack smoke scenario — boots the app, waits for readiness, captures
162
- # evidence into the artifact directory, then shuts the app down.
163
- #
164
- # Evidence produced:
165
- # $ARTIFACT_DIR/logs/server.log full server output
166
- # $ARTIFACT_DIR/smoke-root.html the rendered root page
167
- #
168
- # Adapt this to your app: navigate real flows, take screenshots, and append
169
- # anything you could NOT test to $ARTIFACT_DIR/gaps.txt (one line each).
170
- set -euo pipefail
171
-
172
- ARTIFACT_DIR="\${ARTIFACT_DIR:-${DEFAULT_ARTIFACT_DIR}}"
173
- PORT="\${PORT:-${port}}"
174
- mkdir -p "\${ARTIFACT_DIR}/logs"
175
-
176
- # Evidence integrity: if something is already listening on the port, captured
177
- # "evidence" would come from the wrong server. Refuse instead of lying.
178
- if curl -sf -o /dev/null "http://localhost:\${PORT}/"; then
179
- echo "smoke failed: something is already listening on port \${PORT} — stop it or set PORT" >&2
180
- exit 1
181
- fi
182
-
183
- ${startCommand} > "\${ARTIFACT_DIR}/logs/server.log" 2>&1 &
184
- SERVER_PID=$!
185
- # Kill the server's children too — package-manager wrappers (pnpm dev) leave
186
- # the real dev server as a grandchild.
187
- cleanup() {
188
- pkill -P "\${SERVER_PID}" 2>/dev/null || true
189
- kill "\${SERVER_PID}" 2>/dev/null || true
190
- }
191
- trap cleanup EXIT
192
-
193
- PORT="\${PORT}" bash ${PREFLIGHT_SCRIPT}
194
-
195
- curl -sf "http://localhost:\${PORT}/" -o "\${ARTIFACT_DIR}/smoke-root.html"
196
- test -s "\${ARTIFACT_DIR}/smoke-root.html"
197
-
198
- echo "smoke ok: captured \${ARTIFACT_DIR}/smoke-root.html and server log"
199
- `;
200
- }
201
- function renderAgentsDoc() {
202
- return `# Haystack Verification — Agent Guide
203
-
204
- This repo has a Haystack verification contract at \`${CONTRACT_RELATIVE_PATH}\`.
205
- Use it whenever you need to prove a change works, not just that it compiles.
206
-
207
- ## Commands
208
-
209
- \`\`\`bash
210
- haystack verification validate # is the contract trustworthy?
211
- haystack verification preflight # is the app up and answering?
212
- haystack verification run smoke # run a scenario, capture evidence
213
- haystack verification run <scenario> # any scenario defined in the contract
214
- haystack verification check-safety # prove the setup can't hurt anyone
215
- haystack verification collect-artifacts # rebuild the manifest from what's on disk
216
- \`\`\`
217
-
218
- ## Conventions
219
-
220
- - **Artifacts** go under the contract's \`artifact_dir\`. Screenshots, logs,
221
- captured pages — anything a human reviewer should see. Each \`run\` writes a
222
- \`manifest-<scenario>.json\` and \`review-packet-<scenario>.md\` there.
223
- - **Gaps**: if a scenario could not test something (a path you skipped, a
224
- service you mocked away), append one line describing it to
225
- \`<artifact_dir>/gaps.txt\`. Gaps end up in the review packet — silence reads
226
- as "covered", so declare what wasn't.
227
- - **Personas** are seeded fake users. Never use real accounts or customer data.
228
-
229
- ## Safety rules
230
-
231
- - Verification must never send real email, charge real cards, call production
232
- third-party APIs, or touch customer data.
233
- - Dev-only helpers (login shortcuts, seed endpoints) must be gated so they
234
- cannot run in production, with a test proving it
235
- (\`safety.production_disabled_check\`).
236
- - Don't write secrets into artifacts — \`check-safety\` scans for them.
237
-
238
- ## Extending the contract
239
-
240
- Follow \`.haystack/skills/install-verification/SKILL.md\` when adding
241
- scenarios or personas, and keep changes dev-only and reviewable.
242
- `;
243
- }
@@ -1,49 +0,0 @@
1
- import type { CommandRunRecord } from './runner.js';
2
- export type VerificationStatus = 'verified' | 'failed' | 'inconclusive' | 'environment_not_ready' | 'unsafe_to_verify' | 'missing_contract';
3
- export type ArtifactType = 'screenshot' | 'video' | 'log' | 'json' | 'html' | 'trace' | 'file';
4
- export interface ArtifactEntry {
5
- type: ArtifactType;
6
- /** Repo-relative path. */
7
- path: string;
8
- description?: string;
9
- size_bytes?: number;
10
- }
11
- export interface ManifestSafety {
12
- real_email_sent: boolean;
13
- real_payment_attempted: boolean;
14
- customer_data_accessed: boolean;
15
- }
16
- export interface VerificationManifest {
17
- contract_version: '1';
18
- status: VerificationStatus;
19
- scenario: string;
20
- commit_sha: string | null;
21
- started_at: string;
22
- completed_at: string;
23
- commands_run: CommandRunRecord[];
24
- artifacts: ArtifactEntry[];
25
- /** Known untested paths, read from <artifact_dir>/gaps.txt if the scenario wrote one. */
26
- gaps: string[];
27
- safety: ManifestSafety;
28
- }
29
- export declare function classifyArtifact(filePath: string): ArtifactType;
30
- export declare function getHeadSha(cwd: string): string | null;
31
- export interface CollectOptions {
32
- rootDir: string;
33
- artifactDir: string;
34
- /** Extra files (repo-relative or absolute) to include when they exist, e.g. contract log_paths. */
35
- extraPaths?: string[];
36
- /** Only include files modified at/after this time (a run's start). Omit to include everything. */
37
- since?: Date;
38
- }
39
- /**
40
- * Walk the artifact directory (plus configured log paths) and build the
41
- * manifest's artifact list. Manifest/packet outputs from previous runs are
42
- * excluded so evidence doesn't nest.
43
- */
44
- export declare function collectArtifacts(opts: CollectOptions): Promise<ArtifactEntry[]>;
45
- /** Read scenario-declared gaps from `<artifact_dir>/gaps.txt` (one per line), if present. */
46
- export declare function readGaps(artifactDir: string): Promise<string[]>;
47
- export declare function writeManifest(manifest: VerificationManifest, artifactDir: string): Promise<string>;
48
- export declare function renderReviewPacket(manifest: VerificationManifest): string;
49
- export declare function writeReviewPacket(manifest: VerificationManifest, artifactDir: string): Promise<string>;