@haystackeditor/cli 0.15.17 → 0.15.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -6
- package/dist/assets/skills/map-your-system.md +138 -0
- package/dist/commands/ask.d.ts +14 -0
- package/dist/commands/ask.js +20 -0
- package/dist/commands/inbox.d.ts +65 -0
- package/dist/commands/inbox.js +137 -0
- package/dist/commands/mcp.js +88 -134
- package/dist/commands/pr.d.ts +37 -1
- package/dist/commands/pr.js +95 -77
- package/dist/commands/review.d.ts +24 -0
- package/dist/commands/review.js +193 -0
- package/dist/commands/schema-cmd.js +1 -1
- package/dist/commands/skills.js +4 -0
- package/dist/commands/traces.d.ts +48 -0
- package/dist/commands/traces.js +92 -0
- package/dist/commands/triage.d.ts +12 -0
- package/dist/commands/triage.js +90 -58
- package/dist/commands/webhooks.js +3 -6
- package/dist/index.js +79 -148
- package/dist/schema.d.ts +5 -2
- package/dist/schema.js +5 -2
- package/dist/utils/analysis-api.d.ts +40 -0
- package/dist/utils/analysis-api.js +108 -3
- package/dist/utils/haystack-api.d.ts +20 -0
- package/dist/utils/haystack-api.js +66 -0
- package/package.json +4 -4
- package/schemas/ask.v1.json +40 -0
- package/schemas/inbox.v1.json +27 -0
- package/schemas/pr.v2.json +54 -0
- package/schemas/traces.v1.json +53 -0
- package/schemas/triage.v2.json +64 -0
- package/dist/assets/skills/install-verification.md +0 -67
- package/dist/commands/verification.d.ts +0 -21
- package/dist/commands/verification.js +0 -291
- package/dist/verification/contract.d.ts +0 -240
- package/dist/verification/contract.js +0 -115
- package/dist/verification/init.d.ts +0 -10
- package/dist/verification/init.js +0 -243
- package/dist/verification/manifest.d.ts +0 -49
- package/dist/verification/manifest.js +0 -186
- package/dist/verification/runner.d.ts +0 -20
- package/dist/verification/runner.js +0 -102
- package/dist/verification/safety.d.ts +0 -10
- package/dist/verification/safety.js +0 -252
- package/dist/verification/validate.d.ts +0 -12
- package/dist/verification/validate.js +0 -219
|
@@ -1,240 +0,0 @@
|
|
|
1
|
-
import { z } from 'zod';
|
|
2
|
-
export declare const CONTRACT_RELATIVE_PATH = ".haystack/verification.yml";
|
|
3
|
-
export declare const DEFAULT_ARTIFACT_DIR = ".haystack/artifacts";
|
|
4
|
-
declare const scenarioSchema: z.ZodObject<{
|
|
5
|
-
command: z.ZodString;
|
|
6
|
-
description: z.ZodOptional<z.ZodString>;
|
|
7
|
-
/** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
|
|
8
|
-
artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
9
|
-
/** Wall-clock cap for the scenario command. Default applied by the runner. */
|
|
10
|
-
timeout_seconds: z.ZodOptional<z.ZodNumber>;
|
|
11
|
-
/**
|
|
12
|
-
* Whether `run` gates this scenario on environment.preflight. Set false for
|
|
13
|
-
* self-contained scenarios that boot the app themselves (like the scaffolded
|
|
14
|
-
* smoke script) — preflight against a cold environment would always fail.
|
|
15
|
-
*/
|
|
16
|
-
preflight: z.ZodDefault<z.ZodBoolean>;
|
|
17
|
-
}, "strip", z.ZodTypeAny, {
|
|
18
|
-
command: string;
|
|
19
|
-
preflight: boolean;
|
|
20
|
-
description?: string | undefined;
|
|
21
|
-
artifacts?: string[] | undefined;
|
|
22
|
-
timeout_seconds?: number | undefined;
|
|
23
|
-
}, {
|
|
24
|
-
command: string;
|
|
25
|
-
description?: string | undefined;
|
|
26
|
-
artifacts?: string[] | undefined;
|
|
27
|
-
timeout_seconds?: number | undefined;
|
|
28
|
-
preflight?: boolean | undefined;
|
|
29
|
-
}>;
|
|
30
|
-
declare const safetySchema: z.ZodObject<{
|
|
31
|
-
/** Command proving dev-only helpers are disabled in production builds. */
|
|
32
|
-
production_disabled_check: z.ZodOptional<z.ZodString>;
|
|
33
|
-
external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
|
|
34
|
-
allow_real_email: z.ZodDefault<z.ZodBoolean>;
|
|
35
|
-
allow_real_payments: z.ZodDefault<z.ZodBoolean>;
|
|
36
|
-
allow_customer_data: z.ZodDefault<z.ZodBoolean>;
|
|
37
|
-
}, "strip", z.ZodTypeAny, {
|
|
38
|
-
allow_real_email: boolean;
|
|
39
|
-
allow_real_payments: boolean;
|
|
40
|
-
allow_customer_data: boolean;
|
|
41
|
-
production_disabled_check?: string | undefined;
|
|
42
|
-
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
43
|
-
}, {
|
|
44
|
-
production_disabled_check?: string | undefined;
|
|
45
|
-
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
46
|
-
allow_real_email?: boolean | undefined;
|
|
47
|
-
allow_real_payments?: boolean | undefined;
|
|
48
|
-
allow_customer_data?: boolean | undefined;
|
|
49
|
-
}>;
|
|
50
|
-
export declare const verificationContractSchema: z.ZodObject<{
|
|
51
|
-
version: z.ZodEffects<z.ZodUnion<[z.ZodLiteral<1>, z.ZodLiteral<"1">]>, 1, 1 | "1">;
|
|
52
|
-
app: z.ZodObject<{
|
|
53
|
-
name: z.ZodString;
|
|
54
|
-
framework: z.ZodOptional<z.ZodString>;
|
|
55
|
-
}, "strip", z.ZodTypeAny, {
|
|
56
|
-
name: string;
|
|
57
|
-
framework?: string | undefined;
|
|
58
|
-
}, {
|
|
59
|
-
name: string;
|
|
60
|
-
framework?: string | undefined;
|
|
61
|
-
}>;
|
|
62
|
-
environment: z.ZodObject<{
|
|
63
|
-
setup: z.ZodOptional<z.ZodString>;
|
|
64
|
-
start: z.ZodOptional<z.ZodString>;
|
|
65
|
-
preflight: z.ZodString;
|
|
66
|
-
reset: z.ZodOptional<z.ZodString>;
|
|
67
|
-
artifact_dir: z.ZodDefault<z.ZodString>;
|
|
68
|
-
log_paths: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
69
|
-
}, "strip", z.ZodTypeAny, {
|
|
70
|
-
preflight: string;
|
|
71
|
-
artifact_dir: string;
|
|
72
|
-
log_paths: string[];
|
|
73
|
-
start?: string | undefined;
|
|
74
|
-
setup?: string | undefined;
|
|
75
|
-
reset?: string | undefined;
|
|
76
|
-
}, {
|
|
77
|
-
preflight: string;
|
|
78
|
-
start?: string | undefined;
|
|
79
|
-
setup?: string | undefined;
|
|
80
|
-
reset?: string | undefined;
|
|
81
|
-
artifact_dir?: string | undefined;
|
|
82
|
-
log_paths?: string[] | undefined;
|
|
83
|
-
}>;
|
|
84
|
-
personas: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
85
|
-
login: z.ZodString;
|
|
86
|
-
description: z.ZodOptional<z.ZodString>;
|
|
87
|
-
}, "strip", z.ZodTypeAny, {
|
|
88
|
-
login: string;
|
|
89
|
-
description?: string | undefined;
|
|
90
|
-
}, {
|
|
91
|
-
login: string;
|
|
92
|
-
description?: string | undefined;
|
|
93
|
-
}>>>;
|
|
94
|
-
scenarios: z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
95
|
-
command: z.ZodString;
|
|
96
|
-
description: z.ZodOptional<z.ZodString>;
|
|
97
|
-
/** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
|
|
98
|
-
artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
99
|
-
/** Wall-clock cap for the scenario command. Default applied by the runner. */
|
|
100
|
-
timeout_seconds: z.ZodOptional<z.ZodNumber>;
|
|
101
|
-
/**
|
|
102
|
-
* Whether `run` gates this scenario on environment.preflight. Set false for
|
|
103
|
-
* self-contained scenarios that boot the app themselves (like the scaffolded
|
|
104
|
-
* smoke script) — preflight against a cold environment would always fail.
|
|
105
|
-
*/
|
|
106
|
-
preflight: z.ZodDefault<z.ZodBoolean>;
|
|
107
|
-
}, "strip", z.ZodTypeAny, {
|
|
108
|
-
command: string;
|
|
109
|
-
preflight: boolean;
|
|
110
|
-
description?: string | undefined;
|
|
111
|
-
artifacts?: string[] | undefined;
|
|
112
|
-
timeout_seconds?: number | undefined;
|
|
113
|
-
}, {
|
|
114
|
-
command: string;
|
|
115
|
-
description?: string | undefined;
|
|
116
|
-
artifacts?: string[] | undefined;
|
|
117
|
-
timeout_seconds?: number | undefined;
|
|
118
|
-
preflight?: boolean | undefined;
|
|
119
|
-
}>>, Record<string, {
|
|
120
|
-
command: string;
|
|
121
|
-
preflight: boolean;
|
|
122
|
-
description?: string | undefined;
|
|
123
|
-
artifacts?: string[] | undefined;
|
|
124
|
-
timeout_seconds?: number | undefined;
|
|
125
|
-
}>, Record<string, {
|
|
126
|
-
command: string;
|
|
127
|
-
description?: string | undefined;
|
|
128
|
-
artifacts?: string[] | undefined;
|
|
129
|
-
timeout_seconds?: number | undefined;
|
|
130
|
-
preflight?: boolean | undefined;
|
|
131
|
-
}>>;
|
|
132
|
-
safety: z.ZodObject<{
|
|
133
|
-
/** Command proving dev-only helpers are disabled in production builds. */
|
|
134
|
-
production_disabled_check: z.ZodOptional<z.ZodString>;
|
|
135
|
-
external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
|
|
136
|
-
allow_real_email: z.ZodDefault<z.ZodBoolean>;
|
|
137
|
-
allow_real_payments: z.ZodDefault<z.ZodBoolean>;
|
|
138
|
-
allow_customer_data: z.ZodDefault<z.ZodBoolean>;
|
|
139
|
-
}, "strip", z.ZodTypeAny, {
|
|
140
|
-
allow_real_email: boolean;
|
|
141
|
-
allow_real_payments: boolean;
|
|
142
|
-
allow_customer_data: boolean;
|
|
143
|
-
production_disabled_check?: string | undefined;
|
|
144
|
-
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
145
|
-
}, {
|
|
146
|
-
production_disabled_check?: string | undefined;
|
|
147
|
-
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
148
|
-
allow_real_email?: boolean | undefined;
|
|
149
|
-
allow_real_payments?: boolean | undefined;
|
|
150
|
-
allow_customer_data?: boolean | undefined;
|
|
151
|
-
}>;
|
|
152
|
-
}, "strip", z.ZodTypeAny, {
|
|
153
|
-
version: 1;
|
|
154
|
-
app: {
|
|
155
|
-
name: string;
|
|
156
|
-
framework?: string | undefined;
|
|
157
|
-
};
|
|
158
|
-
environment: {
|
|
159
|
-
preflight: string;
|
|
160
|
-
artifact_dir: string;
|
|
161
|
-
log_paths: string[];
|
|
162
|
-
start?: string | undefined;
|
|
163
|
-
setup?: string | undefined;
|
|
164
|
-
reset?: string | undefined;
|
|
165
|
-
};
|
|
166
|
-
personas: Record<string, {
|
|
167
|
-
login: string;
|
|
168
|
-
description?: string | undefined;
|
|
169
|
-
}>;
|
|
170
|
-
scenarios: Record<string, {
|
|
171
|
-
command: string;
|
|
172
|
-
preflight: boolean;
|
|
173
|
-
description?: string | undefined;
|
|
174
|
-
artifacts?: string[] | undefined;
|
|
175
|
-
timeout_seconds?: number | undefined;
|
|
176
|
-
}>;
|
|
177
|
-
safety: {
|
|
178
|
-
allow_real_email: boolean;
|
|
179
|
-
allow_real_payments: boolean;
|
|
180
|
-
allow_customer_data: boolean;
|
|
181
|
-
production_disabled_check?: string | undefined;
|
|
182
|
-
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
183
|
-
};
|
|
184
|
-
}, {
|
|
185
|
-
version: 1 | "1";
|
|
186
|
-
app: {
|
|
187
|
-
name: string;
|
|
188
|
-
framework?: string | undefined;
|
|
189
|
-
};
|
|
190
|
-
environment: {
|
|
191
|
-
preflight: string;
|
|
192
|
-
start?: string | undefined;
|
|
193
|
-
setup?: string | undefined;
|
|
194
|
-
reset?: string | undefined;
|
|
195
|
-
artifact_dir?: string | undefined;
|
|
196
|
-
log_paths?: string[] | undefined;
|
|
197
|
-
};
|
|
198
|
-
scenarios: Record<string, {
|
|
199
|
-
command: string;
|
|
200
|
-
description?: string | undefined;
|
|
201
|
-
artifacts?: string[] | undefined;
|
|
202
|
-
timeout_seconds?: number | undefined;
|
|
203
|
-
preflight?: boolean | undefined;
|
|
204
|
-
}>;
|
|
205
|
-
safety: {
|
|
206
|
-
production_disabled_check?: string | undefined;
|
|
207
|
-
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
208
|
-
allow_real_email?: boolean | undefined;
|
|
209
|
-
allow_real_payments?: boolean | undefined;
|
|
210
|
-
allow_customer_data?: boolean | undefined;
|
|
211
|
-
};
|
|
212
|
-
personas?: Record<string, {
|
|
213
|
-
login: string;
|
|
214
|
-
description?: string | undefined;
|
|
215
|
-
}> | undefined;
|
|
216
|
-
}>;
|
|
217
|
-
export type VerificationContract = z.infer<typeof verificationContractSchema>;
|
|
218
|
-
export type VerificationScenario = z.infer<typeof scenarioSchema>;
|
|
219
|
-
export type VerificationSafety = z.infer<typeof safetySchema>;
|
|
220
|
-
export type ContractLoadResult = {
|
|
221
|
-
status: 'ok';
|
|
222
|
-
contract: VerificationContract;
|
|
223
|
-
path: string;
|
|
224
|
-
} | {
|
|
225
|
-
status: 'missing';
|
|
226
|
-
path: string;
|
|
227
|
-
} | {
|
|
228
|
-
status: 'invalid';
|
|
229
|
-
path: string;
|
|
230
|
-
errors: string[];
|
|
231
|
-
};
|
|
232
|
-
/**
|
|
233
|
-
* Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
|
|
234
|
-
* Never throws — parse/schema failures come back as `status: 'invalid'` so
|
|
235
|
-
* callers can map them onto readiness / manifest statuses.
|
|
236
|
-
*/
|
|
237
|
-
export declare function loadContract(rootDir?: string): Promise<ContractLoadResult>;
|
|
238
|
-
/** Resolve the artifact directory to an absolute path, creating it if needed. */
|
|
239
|
-
export declare function ensureArtifactDir(contract: VerificationContract, rootDir?: string): Promise<string>;
|
|
240
|
-
export {};
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Haystack Verification Contract — `.haystack/verification.yml`
|
|
3
|
-
*
|
|
4
|
-
* The contract is the repo-local answer to five questions:
|
|
5
|
-
* 1. How do I set up and start the app?
|
|
6
|
-
* 2. How do I know the app is ready? (preflight)
|
|
7
|
-
* 3. How do I log in as useful test users? (personas)
|
|
8
|
-
* 4. Which verification flows can I run? (scenarios)
|
|
9
|
-
* 5. Where do logs, screenshots, and artifacts go?
|
|
10
|
-
*
|
|
11
|
-
* This is deliberately separate from `.haystack.json`: that file configures
|
|
12
|
-
* Haystack's own sandbox verification (services/flows/fixtures the sandbox
|
|
13
|
-
* drives). The contract instead describes commands the REPO owns — anything
|
|
14
|
-
* (Haystack, CI, a coding agent, a human) can run them and get the same
|
|
15
|
-
* evidence. See docs/VERIFICATION-CONTRACT.md.
|
|
16
|
-
*/
|
|
17
|
-
import * as fs from 'node:fs/promises';
|
|
18
|
-
import * as path from 'node:path';
|
|
19
|
-
import { parse as parseYaml } from 'yaml';
|
|
20
|
-
import { z } from 'zod';
|
|
21
|
-
export const CONTRACT_RELATIVE_PATH = '.haystack/verification.yml';
|
|
22
|
-
export const DEFAULT_ARTIFACT_DIR = '.haystack/artifacts';
|
|
23
|
-
const commandString = z.string().min(1, 'command must be a non-empty string');
|
|
24
|
-
const personaSchema = z.object({
|
|
25
|
-
login: commandString.describe('Command that logs the given persona in (or prints credentials)'),
|
|
26
|
-
description: z.string().optional(),
|
|
27
|
-
});
|
|
28
|
-
const scenarioSchema = z.object({
|
|
29
|
-
command: commandString,
|
|
30
|
-
description: z.string().optional(),
|
|
31
|
-
/** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
|
|
32
|
-
artifacts: z.array(z.string()).optional(),
|
|
33
|
-
/** Wall-clock cap for the scenario command. Default applied by the runner. */
|
|
34
|
-
timeout_seconds: z.number().int().positive().max(7200).optional(),
|
|
35
|
-
/**
|
|
36
|
-
* Whether `run` gates this scenario on environment.preflight. Set false for
|
|
37
|
-
* self-contained scenarios that boot the app themselves (like the scaffolded
|
|
38
|
-
* smoke script) — preflight against a cold environment would always fail.
|
|
39
|
-
*/
|
|
40
|
-
preflight: z.boolean().default(true),
|
|
41
|
-
});
|
|
42
|
-
const safetySchema = z.object({
|
|
43
|
-
/** Command proving dev-only helpers are disabled in production builds. */
|
|
44
|
-
production_disabled_check: commandString.optional(),
|
|
45
|
-
external_services: z.enum(['sandbox_only', 'mocked', 'none']).optional(),
|
|
46
|
-
allow_real_email: z.boolean().default(false),
|
|
47
|
-
allow_real_payments: z.boolean().default(false),
|
|
48
|
-
allow_customer_data: z.boolean().default(false),
|
|
49
|
-
});
|
|
50
|
-
export const verificationContractSchema = z.object({
|
|
51
|
-
// Accept 1 or "1" — YAML authors will write both.
|
|
52
|
-
version: z
|
|
53
|
-
.union([z.literal(1), z.literal('1')])
|
|
54
|
-
.transform(() => 1),
|
|
55
|
-
app: z.object({
|
|
56
|
-
name: commandString,
|
|
57
|
-
framework: z.string().optional(),
|
|
58
|
-
}),
|
|
59
|
-
environment: z.object({
|
|
60
|
-
setup: commandString.optional(),
|
|
61
|
-
start: commandString.optional(),
|
|
62
|
-
preflight: commandString,
|
|
63
|
-
reset: commandString.optional(),
|
|
64
|
-
artifact_dir: z.string().default(DEFAULT_ARTIFACT_DIR),
|
|
65
|
-
log_paths: z.array(z.string()).default([]),
|
|
66
|
-
}),
|
|
67
|
-
personas: z.record(personaSchema).default({}),
|
|
68
|
-
scenarios: z
|
|
69
|
-
.record(scenarioSchema)
|
|
70
|
-
.refine((s) => Object.keys(s).length > 0, {
|
|
71
|
-
message: 'at least one scenario is required',
|
|
72
|
-
}),
|
|
73
|
-
safety: safetySchema,
|
|
74
|
-
});
|
|
75
|
-
/**
|
|
76
|
-
* Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
|
|
77
|
-
* Never throws — parse/schema failures come back as `status: 'invalid'` so
|
|
78
|
-
* callers can map them onto readiness / manifest statuses.
|
|
79
|
-
*/
|
|
80
|
-
export async function loadContract(rootDir = process.cwd()) {
|
|
81
|
-
const contractPath = path.join(rootDir, CONTRACT_RELATIVE_PATH);
|
|
82
|
-
let raw;
|
|
83
|
-
try {
|
|
84
|
-
raw = await fs.readFile(contractPath, 'utf-8');
|
|
85
|
-
}
|
|
86
|
-
catch {
|
|
87
|
-
return { status: 'missing', path: contractPath };
|
|
88
|
-
}
|
|
89
|
-
let parsed;
|
|
90
|
-
try {
|
|
91
|
-
parsed = parseYaml(raw);
|
|
92
|
-
}
|
|
93
|
-
catch (err) {
|
|
94
|
-
return {
|
|
95
|
-
status: 'invalid',
|
|
96
|
-
path: contractPath,
|
|
97
|
-
errors: [`YAML parse error: ${err instanceof Error ? err.message : String(err)}`],
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
const result = verificationContractSchema.safeParse(parsed);
|
|
101
|
-
if (!result.success) {
|
|
102
|
-
return {
|
|
103
|
-
status: 'invalid',
|
|
104
|
-
path: contractPath,
|
|
105
|
-
errors: result.error.issues.map((issue) => `${issue.path.length ? issue.path.join('.') : '(root)'}: ${issue.message}`),
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
return { status: 'ok', contract: result.data, path: contractPath };
|
|
109
|
-
}
|
|
110
|
-
/** Resolve the artifact directory to an absolute path, creating it if needed. */
|
|
111
|
-
export async function ensureArtifactDir(contract, rootDir = process.cwd()) {
|
|
112
|
-
const dir = path.resolve(rootDir, contract.environment.artifact_dir);
|
|
113
|
-
await fs.mkdir(path.join(dir, 'logs'), { recursive: true });
|
|
114
|
-
return dir;
|
|
115
|
-
}
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export interface InitResult {
|
|
2
|
-
created: string[];
|
|
3
|
-
skipped: string[];
|
|
4
|
-
}
|
|
5
|
-
export interface InitOptions {
|
|
6
|
-
force?: boolean;
|
|
7
|
-
/** Only write the agent skill file (for handing the integration to a coding agent). */
|
|
8
|
-
skillOnly?: boolean;
|
|
9
|
-
}
|
|
10
|
-
export declare function initVerification(rootDir?: string, options?: InitOptions): Promise<InitResult>;
|
|
@@ -1,243 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `haystack verification init` — scaffold the verification contract.
|
|
3
|
-
*
|
|
4
|
-
* Detection-driven where evidence exists (framework, package manager, dev
|
|
5
|
-
* command, port, auth-bypass env var), honest TODOs where it doesn't. The
|
|
6
|
-
* generated smoke scenario is real, not a placeholder: it boots the app,
|
|
7
|
-
* waits for readiness, captures the root page + server log, and shuts down.
|
|
8
|
-
*/
|
|
9
|
-
import * as fs from 'node:fs/promises';
|
|
10
|
-
import * as path from 'node:path';
|
|
11
|
-
import { fileURLToPath } from 'node:url';
|
|
12
|
-
import { detectProject } from '../utils/detect.js';
|
|
13
|
-
import { CONTRACT_RELATIVE_PATH, DEFAULT_ARTIFACT_DIR } from './contract.js';
|
|
14
|
-
const SKILL_TARGET = '.haystack/skills/install-verification/SKILL.md';
|
|
15
|
-
const AGENTS_TARGET = '.haystack/AGENTS.md';
|
|
16
|
-
const PREFLIGHT_SCRIPT = 'scripts/haystack-preflight';
|
|
17
|
-
const SMOKE_SCRIPT = 'scripts/haystack-smoke';
|
|
18
|
-
export async function initVerification(rootDir = process.cwd(), options = {}) {
|
|
19
|
-
const result = { created: [], skipped: [] };
|
|
20
|
-
const writeFile = async (relPath, content, executable = false) => {
|
|
21
|
-
const target = path.join(rootDir, relPath);
|
|
22
|
-
if (!options.force && (await exists(target))) {
|
|
23
|
-
result.skipped.push(relPath);
|
|
24
|
-
return;
|
|
25
|
-
}
|
|
26
|
-
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
27
|
-
await fs.writeFile(target, content, { encoding: 'utf-8', mode: executable ? 0o755 : 0o644 });
|
|
28
|
-
result.created.push(relPath);
|
|
29
|
-
};
|
|
30
|
-
await writeFile(SKILL_TARGET, await readSkillAsset());
|
|
31
|
-
if (options.skillOnly)
|
|
32
|
-
return result;
|
|
33
|
-
const detected = await detectProject(rootDir);
|
|
34
|
-
const appName = (await readPackageName(rootDir)) ?? path.basename(rootDir);
|
|
35
|
-
const setupCommand = setupCommandFor(detected.packageManager);
|
|
36
|
-
const port = detected.suggestedPort ?? 3000;
|
|
37
|
-
// Evidence-based auth bypass: only prefix the env var when detection found
|
|
38
|
-
// one in the repo's own env examples (never invent a flag the app ignores).
|
|
39
|
-
const startCommand = detected.suggestedAuthBypass
|
|
40
|
-
? `${detected.suggestedAuthBypass} ${detected.suggestedDevCommand}`
|
|
41
|
-
: (detected.suggestedDevCommand ?? 'echo "TODO: set your dev-server command" && exit 1');
|
|
42
|
-
await writeFile(CONTRACT_RELATIVE_PATH, renderContract({ appName, framework: detected.framework, setupCommand, startCommand }));
|
|
43
|
-
await writeFile(PREFLIGHT_SCRIPT, renderPreflightScript(port), true);
|
|
44
|
-
await writeFile(SMOKE_SCRIPT, renderSmokeScript(port, startCommand), true);
|
|
45
|
-
await writeFile(AGENTS_TARGET, renderAgentsDoc());
|
|
46
|
-
return result;
|
|
47
|
-
}
|
|
48
|
-
async function exists(p) {
|
|
49
|
-
try {
|
|
50
|
-
await fs.access(p);
|
|
51
|
-
return true;
|
|
52
|
-
}
|
|
53
|
-
catch {
|
|
54
|
-
return false;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
async function readPackageName(rootDir) {
|
|
58
|
-
try {
|
|
59
|
-
const pkg = JSON.parse(await fs.readFile(path.join(rootDir, 'package.json'), 'utf-8'));
|
|
60
|
-
return pkg.name?.replace(/^@[^/]+\//, '') ?? null;
|
|
61
|
-
}
|
|
62
|
-
catch {
|
|
63
|
-
return null;
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
async function readSkillAsset() {
|
|
67
|
-
// dist/verification/init.js → dist/assets/skills/install-verification.md
|
|
68
|
-
const assetPath = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'assets', 'skills', 'install-verification.md');
|
|
69
|
-
return fs.readFile(assetPath, 'utf-8');
|
|
70
|
-
}
|
|
71
|
-
function setupCommandFor(pm) {
|
|
72
|
-
switch (pm) {
|
|
73
|
-
case 'pnpm':
|
|
74
|
-
return 'pnpm install --frozen-lockfile';
|
|
75
|
-
case 'yarn':
|
|
76
|
-
return 'yarn install --frozen-lockfile';
|
|
77
|
-
case 'bun':
|
|
78
|
-
return 'bun install';
|
|
79
|
-
default:
|
|
80
|
-
return 'npm ci';
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
function renderContract(opts) {
|
|
84
|
-
return `version: 1
|
|
85
|
-
|
|
86
|
-
# Haystack Verification Contract
|
|
87
|
-
#
|
|
88
|
-
# Defines how Haystack (and any coding agent) verifies this app: how to set it
|
|
89
|
-
# up, how to know it's ready, how to log in, which flows to run, and where
|
|
90
|
-
# evidence goes. Check it with:
|
|
91
|
-
#
|
|
92
|
-
# haystack verification validate
|
|
93
|
-
# haystack verification check-safety
|
|
94
|
-
# haystack verification run smoke
|
|
95
|
-
|
|
96
|
-
app:
|
|
97
|
-
name: "${opts.appName}"
|
|
98
|
-
${opts.framework ? ` framework: "${opts.framework}"\n` : ''}
|
|
99
|
-
environment:
|
|
100
|
-
setup: "${opts.setupCommand}"
|
|
101
|
-
start: "${opts.startCommand}"
|
|
102
|
-
preflight: "bash ${PREFLIGHT_SCRIPT}"
|
|
103
|
-
# reset: "pnpm db:reset" # optional: return the environment to a clean state
|
|
104
|
-
artifact_dir: "${DEFAULT_ARTIFACT_DIR}"
|
|
105
|
-
log_paths:
|
|
106
|
-
- "${DEFAULT_ARTIFACT_DIR}/logs/server.log"
|
|
107
|
-
|
|
108
|
-
# Personas are seeded FAKE users an agent can act as. Wire these up once you
|
|
109
|
-
# have a dev-only login helper — and add safety.production_disabled_check
|
|
110
|
-
# proving that helper is disabled in production builds.
|
|
111
|
-
# personas:
|
|
112
|
-
# admin:
|
|
113
|
-
# login: "pnpm haystack:login admin"
|
|
114
|
-
# description: "Workspace admin test user"
|
|
115
|
-
# member:
|
|
116
|
-
# login: "pnpm haystack:login member"
|
|
117
|
-
# description: "Regular workspace member"
|
|
118
|
-
|
|
119
|
-
scenarios:
|
|
120
|
-
smoke:
|
|
121
|
-
command: "bash ${SMOKE_SCRIPT}"
|
|
122
|
-
description: "Boot the app, wait until it answers, capture the root page and server log"
|
|
123
|
-
preflight: false # the smoke script boots the app itself
|
|
124
|
-
artifacts:
|
|
125
|
-
- "server_logs"
|
|
126
|
-
- "root_html"
|
|
127
|
-
|
|
128
|
-
safety:
|
|
129
|
-
# production_disabled_check: "pnpm test:dev-routes-disabled" # prove dev-only helpers are off in prod builds
|
|
130
|
-
external_services: "sandbox_only"
|
|
131
|
-
allow_real_email: false
|
|
132
|
-
allow_real_payments: false
|
|
133
|
-
allow_customer_data: false
|
|
134
|
-
`;
|
|
135
|
-
}
|
|
136
|
-
function renderPreflightScript(port) {
|
|
137
|
-
return `#!/usr/bin/env bash
|
|
138
|
-
# Haystack preflight — exits 0 only when the app is up and answering.
|
|
139
|
-
# Prefer pointing PATH_SUFFIX at a real health endpoint over the root page.
|
|
140
|
-
set -euo pipefail
|
|
141
|
-
|
|
142
|
-
PORT="\${PORT:-${port}}"
|
|
143
|
-
PATH_SUFFIX="\${PATH_SUFFIX:-/}"
|
|
144
|
-
URL="http://localhost:\${PORT}\${PATH_SUFFIX}"
|
|
145
|
-
TIMEOUT_S="\${TIMEOUT_S:-60}"
|
|
146
|
-
|
|
147
|
-
for _ in $(seq 1 "\${TIMEOUT_S}"); do
|
|
148
|
-
if curl -sf -o /dev/null "\${URL}"; then
|
|
149
|
-
echo "preflight ok: \${URL} is responding"
|
|
150
|
-
exit 0
|
|
151
|
-
fi
|
|
152
|
-
sleep 1
|
|
153
|
-
done
|
|
154
|
-
|
|
155
|
-
echo "preflight failed: \${URL} did not respond within \${TIMEOUT_S}s" >&2
|
|
156
|
-
exit 1
|
|
157
|
-
`;
|
|
158
|
-
}
|
|
159
|
-
function renderSmokeScript(port, startCommand) {
|
|
160
|
-
return `#!/usr/bin/env bash
|
|
161
|
-
# Haystack smoke scenario — boots the app, waits for readiness, captures
|
|
162
|
-
# evidence into the artifact directory, then shuts the app down.
|
|
163
|
-
#
|
|
164
|
-
# Evidence produced:
|
|
165
|
-
# $ARTIFACT_DIR/logs/server.log full server output
|
|
166
|
-
# $ARTIFACT_DIR/smoke-root.html the rendered root page
|
|
167
|
-
#
|
|
168
|
-
# Adapt this to your app: navigate real flows, take screenshots, and append
|
|
169
|
-
# anything you could NOT test to $ARTIFACT_DIR/gaps.txt (one line each).
|
|
170
|
-
set -euo pipefail
|
|
171
|
-
|
|
172
|
-
ARTIFACT_DIR="\${ARTIFACT_DIR:-${DEFAULT_ARTIFACT_DIR}}"
|
|
173
|
-
PORT="\${PORT:-${port}}"
|
|
174
|
-
mkdir -p "\${ARTIFACT_DIR}/logs"
|
|
175
|
-
|
|
176
|
-
# Evidence integrity: if something is already listening on the port, captured
|
|
177
|
-
# "evidence" would come from the wrong server. Refuse instead of lying.
|
|
178
|
-
if curl -sf -o /dev/null "http://localhost:\${PORT}/"; then
|
|
179
|
-
echo "smoke failed: something is already listening on port \${PORT} — stop it or set PORT" >&2
|
|
180
|
-
exit 1
|
|
181
|
-
fi
|
|
182
|
-
|
|
183
|
-
${startCommand} > "\${ARTIFACT_DIR}/logs/server.log" 2>&1 &
|
|
184
|
-
SERVER_PID=$!
|
|
185
|
-
# Kill the server's children too — package-manager wrappers (pnpm dev) leave
|
|
186
|
-
# the real dev server as a grandchild.
|
|
187
|
-
cleanup() {
|
|
188
|
-
pkill -P "\${SERVER_PID}" 2>/dev/null || true
|
|
189
|
-
kill "\${SERVER_PID}" 2>/dev/null || true
|
|
190
|
-
}
|
|
191
|
-
trap cleanup EXIT
|
|
192
|
-
|
|
193
|
-
PORT="\${PORT}" bash ${PREFLIGHT_SCRIPT}
|
|
194
|
-
|
|
195
|
-
curl -sf "http://localhost:\${PORT}/" -o "\${ARTIFACT_DIR}/smoke-root.html"
|
|
196
|
-
test -s "\${ARTIFACT_DIR}/smoke-root.html"
|
|
197
|
-
|
|
198
|
-
echo "smoke ok: captured \${ARTIFACT_DIR}/smoke-root.html and server log"
|
|
199
|
-
`;
|
|
200
|
-
}
|
|
201
|
-
function renderAgentsDoc() {
|
|
202
|
-
return `# Haystack Verification — Agent Guide
|
|
203
|
-
|
|
204
|
-
This repo has a Haystack verification contract at \`${CONTRACT_RELATIVE_PATH}\`.
|
|
205
|
-
Use it whenever you need to prove a change works, not just that it compiles.
|
|
206
|
-
|
|
207
|
-
## Commands
|
|
208
|
-
|
|
209
|
-
\`\`\`bash
|
|
210
|
-
haystack verification validate # is the contract trustworthy?
|
|
211
|
-
haystack verification preflight # is the app up and answering?
|
|
212
|
-
haystack verification run smoke # run a scenario, capture evidence
|
|
213
|
-
haystack verification run <scenario> # any scenario defined in the contract
|
|
214
|
-
haystack verification check-safety # prove the setup can't hurt anyone
|
|
215
|
-
haystack verification collect-artifacts # rebuild the manifest from what's on disk
|
|
216
|
-
\`\`\`
|
|
217
|
-
|
|
218
|
-
## Conventions
|
|
219
|
-
|
|
220
|
-
- **Artifacts** go under the contract's \`artifact_dir\`. Screenshots, logs,
|
|
221
|
-
captured pages — anything a human reviewer should see. Each \`run\` writes a
|
|
222
|
-
\`manifest-<scenario>.json\` and \`review-packet-<scenario>.md\` there.
|
|
223
|
-
- **Gaps**: if a scenario could not test something (a path you skipped, a
|
|
224
|
-
service you mocked away), append one line describing it to
|
|
225
|
-
\`<artifact_dir>/gaps.txt\`. Gaps end up in the review packet — silence reads
|
|
226
|
-
as "covered", so declare what wasn't.
|
|
227
|
-
- **Personas** are seeded fake users. Never use real accounts or customer data.
|
|
228
|
-
|
|
229
|
-
## Safety rules
|
|
230
|
-
|
|
231
|
-
- Verification must never send real email, charge real cards, call production
|
|
232
|
-
third-party APIs, or touch customer data.
|
|
233
|
-
- Dev-only helpers (login shortcuts, seed endpoints) must be gated so they
|
|
234
|
-
cannot run in production, with a test proving it
|
|
235
|
-
(\`safety.production_disabled_check\`).
|
|
236
|
-
- Don't write secrets into artifacts — \`check-safety\` scans for them.
|
|
237
|
-
|
|
238
|
-
## Extending the contract
|
|
239
|
-
|
|
240
|
-
Follow \`.haystack/skills/install-verification/SKILL.md\` when adding
|
|
241
|
-
scenarios or personas, and keep changes dev-only and reviewable.
|
|
242
|
-
`;
|
|
243
|
-
}
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
import type { CommandRunRecord } from './runner.js';
|
|
2
|
-
export type VerificationStatus = 'verified' | 'failed' | 'inconclusive' | 'environment_not_ready' | 'unsafe_to_verify' | 'missing_contract';
|
|
3
|
-
export type ArtifactType = 'screenshot' | 'video' | 'log' | 'json' | 'html' | 'trace' | 'file';
|
|
4
|
-
export interface ArtifactEntry {
|
|
5
|
-
type: ArtifactType;
|
|
6
|
-
/** Repo-relative path. */
|
|
7
|
-
path: string;
|
|
8
|
-
description?: string;
|
|
9
|
-
size_bytes?: number;
|
|
10
|
-
}
|
|
11
|
-
export interface ManifestSafety {
|
|
12
|
-
real_email_sent: boolean;
|
|
13
|
-
real_payment_attempted: boolean;
|
|
14
|
-
customer_data_accessed: boolean;
|
|
15
|
-
}
|
|
16
|
-
export interface VerificationManifest {
|
|
17
|
-
contract_version: '1';
|
|
18
|
-
status: VerificationStatus;
|
|
19
|
-
scenario: string;
|
|
20
|
-
commit_sha: string | null;
|
|
21
|
-
started_at: string;
|
|
22
|
-
completed_at: string;
|
|
23
|
-
commands_run: CommandRunRecord[];
|
|
24
|
-
artifacts: ArtifactEntry[];
|
|
25
|
-
/** Known untested paths, read from <artifact_dir>/gaps.txt if the scenario wrote one. */
|
|
26
|
-
gaps: string[];
|
|
27
|
-
safety: ManifestSafety;
|
|
28
|
-
}
|
|
29
|
-
export declare function classifyArtifact(filePath: string): ArtifactType;
|
|
30
|
-
export declare function getHeadSha(cwd: string): string | null;
|
|
31
|
-
export interface CollectOptions {
|
|
32
|
-
rootDir: string;
|
|
33
|
-
artifactDir: string;
|
|
34
|
-
/** Extra files (repo-relative or absolute) to include when they exist, e.g. contract log_paths. */
|
|
35
|
-
extraPaths?: string[];
|
|
36
|
-
/** Only include files modified at/after this time (a run's start). Omit to include everything. */
|
|
37
|
-
since?: Date;
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Walk the artifact directory (plus configured log paths) and build the
|
|
41
|
-
* manifest's artifact list. Manifest/packet outputs from previous runs are
|
|
42
|
-
* excluded so evidence doesn't nest.
|
|
43
|
-
*/
|
|
44
|
-
export declare function collectArtifacts(opts: CollectOptions): Promise<ArtifactEntry[]>;
|
|
45
|
-
/** Read scenario-declared gaps from `<artifact_dir>/gaps.txt` (one per line), if present. */
|
|
46
|
-
export declare function readGaps(artifactDir: string): Promise<string[]>;
|
|
47
|
-
export declare function writeManifest(manifest: VerificationManifest, artifactDir: string): Promise<string>;
|
|
48
|
-
export declare function renderReviewPacket(manifest: VerificationManifest): string;
|
|
49
|
-
export declare function writeReviewPacket(manifest: VerificationManifest, artifactDir: string): Promise<string>;
|