@haystackeditor/cli 0.15.13 → 0.15.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/skills/install-verification.md +67 -0
- package/dist/assets/skills/submit.md +14 -1
- package/dist/commands/init.js +71 -4
- package/dist/commands/setup.js +26 -26
- package/dist/commands/triage.js +98 -17
- package/dist/commands/verification.d.ts +21 -0
- package/dist/commands/verification.js +291 -0
- package/dist/index.js +130 -0
- package/dist/utils/detect.js +3 -1
- package/dist/verification/contract.d.ts +240 -0
- package/dist/verification/contract.js +115 -0
- package/dist/verification/init.d.ts +10 -0
- package/dist/verification/init.js +243 -0
- package/dist/verification/manifest.d.ts +49 -0
- package/dist/verification/manifest.js +186 -0
- package/dist/verification/runner.d.ts +20 -0
- package/dist/verification/runner.js +102 -0
- package/dist/verification/safety.d.ts +10 -0
- package/dist/verification/safety.js +252 -0
- package/dist/verification/validate.d.ts +12 -0
- package/dist/verification/validate.js +219 -0
- package/package.json +3 -1
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `haystack verification ...` — repo-local verification contract commands.
|
|
3
|
+
*
|
|
4
|
+
* init scaffold .haystack/verification.yml + scripts + skill
|
|
5
|
+
* validate can Haystack trust this repo's verification setup?
|
|
6
|
+
* preflight is the app up and answering?
|
|
7
|
+
* run <scenario> run a scenario, capture artifacts, write manifest + review packet
|
|
8
|
+
* collect-artifacts rebuild a manifest from artifacts already on disk
|
|
9
|
+
* check-safety prove the verification setup can't hurt anyone
|
|
10
|
+
*
|
|
11
|
+
* See docs/VERIFICATION-CONTRACT.md for the product framing and schema.
|
|
12
|
+
*/
|
|
13
|
+
import * as path from 'node:path';
|
|
14
|
+
import chalk from 'chalk';
|
|
15
|
+
import { CONTRACT_RELATIVE_PATH, ensureArtifactDir, loadContract, } from '../verification/contract.js';
|
|
16
|
+
import { initVerification } from '../verification/init.js';
|
|
17
|
+
import { validateContract } from '../verification/validate.js';
|
|
18
|
+
import { checkSafety } from '../verification/safety.js';
|
|
19
|
+
import { runCommand } from '../verification/runner.js';
|
|
20
|
+
import { collectArtifacts, getHeadSha, readGaps, writeManifest, writeReviewPacket, } from '../verification/manifest.js';
|
|
21
|
+
const DEFAULT_SCENARIO_TIMEOUT_S = 900;
|
|
22
|
+
const PREFLIGHT_TIMEOUT_S = 300;
|
|
23
|
+
// ─── init ─────────────────────────────────────────────────────────────────────
|
|
24
|
+
export async function verificationInitCommand(options) {
|
|
25
|
+
const result = await initVerification(process.cwd(), {
|
|
26
|
+
force: options.force,
|
|
27
|
+
skillOnly: options.skill,
|
|
28
|
+
});
|
|
29
|
+
for (const file of result.created) {
|
|
30
|
+
console.log(chalk.green(' created ') + file);
|
|
31
|
+
}
|
|
32
|
+
for (const file of result.skipped) {
|
|
33
|
+
console.log(chalk.yellow(' skipped ') + file + chalk.dim(' (exists, use --force to overwrite)'));
|
|
34
|
+
}
|
|
35
|
+
if (result.created.length > 0) {
|
|
36
|
+
console.log();
|
|
37
|
+
console.log(chalk.white('Next steps:'));
|
|
38
|
+
console.log(chalk.cyan(' 1.') + ` Review ${CONTRACT_RELATIVE_PATH} — detection is a starting point, not truth`);
|
|
39
|
+
console.log(chalk.cyan(' 2.') + ' haystack verification validate');
|
|
40
|
+
console.log(chalk.cyan(' 3.') + ' haystack verification run smoke');
|
|
41
|
+
if (options.skill) {
|
|
42
|
+
console.log();
|
|
43
|
+
console.log(chalk.dim('Hand .haystack/skills/install-verification/SKILL.md to your coding agent to complete the integration.'));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// ─── validate ─────────────────────────────────────────────────────────────────
|
|
48
|
+
export async function verificationValidateCommand(options) {
|
|
49
|
+
const report = await validateContract();
|
|
50
|
+
if (options.json) {
|
|
51
|
+
console.log(JSON.stringify(report, null, 2));
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
printValidationReport(report);
|
|
55
|
+
}
|
|
56
|
+
if (report.readiness === 'unsafe')
|
|
57
|
+
process.exit(2);
|
|
58
|
+
if (report.readiness === 'not_ready')
|
|
59
|
+
process.exit(1);
|
|
60
|
+
}
|
|
61
|
+
function printValidationReport(report) {
|
|
62
|
+
const styles = {
|
|
63
|
+
ready: chalk.green,
|
|
64
|
+
partially_ready: chalk.yellow,
|
|
65
|
+
not_ready: chalk.red,
|
|
66
|
+
unsafe: chalk.red,
|
|
67
|
+
};
|
|
68
|
+
console.log(chalk.white('Haystack Verification Contract: ') + styles[report.readiness](report.readiness));
|
|
69
|
+
console.log();
|
|
70
|
+
const sections = [
|
|
71
|
+
{ title: 'Passed', level: 'pass', bullet: chalk.green('✓') },
|
|
72
|
+
{ title: 'Missing', level: 'warn', bullet: chalk.yellow('•') },
|
|
73
|
+
{ title: 'Broken', level: 'fail', bullet: chalk.red('✗') },
|
|
74
|
+
{ title: 'Unsafe', level: 'unsafe', bullet: chalk.red('⚠') },
|
|
75
|
+
];
|
|
76
|
+
for (const { title, level, bullet } of sections) {
|
|
77
|
+
const items = report.items.filter((i) => i.level === level);
|
|
78
|
+
if (items.length === 0)
|
|
79
|
+
continue;
|
|
80
|
+
console.log(chalk.white(`${title}:`));
|
|
81
|
+
for (const item of items) {
|
|
82
|
+
console.log(`${bullet} ${item.message}`);
|
|
83
|
+
}
|
|
84
|
+
console.log();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// ─── preflight ────────────────────────────────────────────────────────────────
|
|
88
|
+
export async function verificationPreflightCommand() {
|
|
89
|
+
const contract = await requireContract();
|
|
90
|
+
const artifactDir = await ensureArtifactDir(contract);
|
|
91
|
+
const record = await runCommand(contract.environment.preflight, {
|
|
92
|
+
cwd: process.cwd(),
|
|
93
|
+
timeoutMs: PREFLIGHT_TIMEOUT_S * 1000,
|
|
94
|
+
logFile: path.join(artifactDir, 'logs', 'preflight.log'),
|
|
95
|
+
env: { ARTIFACT_DIR: artifactDir },
|
|
96
|
+
});
|
|
97
|
+
if (record.status === 'passed') {
|
|
98
|
+
console.log(chalk.green('✓ preflight passed') + chalk.dim(` (${(record.duration_ms / 1000).toFixed(1)}s)`));
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
console.error(chalk.red(`✗ preflight ${record.status}`) + chalk.dim(` — see ${record.log_path}`));
|
|
102
|
+
process.exit(1);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
// ─── run <scenario> ───────────────────────────────────────────────────────────
|
|
106
|
+
export async function verificationRunCommand(scenarioName, options) {
|
|
107
|
+
const rootDir = process.cwd();
|
|
108
|
+
const contract = await requireContract();
|
|
109
|
+
const scenario = contract.scenarios[scenarioName];
|
|
110
|
+
if (!scenario) {
|
|
111
|
+
console.error(chalk.red(`Unknown scenario "${scenarioName}".`) +
|
|
112
|
+
` Available: ${Object.keys(contract.scenarios).join(', ')}`);
|
|
113
|
+
process.exit(1);
|
|
114
|
+
}
|
|
115
|
+
const artifactDir = await ensureArtifactDir(contract);
|
|
116
|
+
const startedAt = new Date();
|
|
117
|
+
const commandsRun = [];
|
|
118
|
+
const env = { ARTIFACT_DIR: artifactDir };
|
|
119
|
+
// In --json mode stdout must carry ONLY the manifest — send progress to
|
|
120
|
+
// stderr and keep command streaming off (output still lands in log files).
|
|
121
|
+
const quiet = Boolean(options.json);
|
|
122
|
+
const progress = (msg) => (quiet ? console.error(msg) : console.log(msg));
|
|
123
|
+
const finish = async (status) => {
|
|
124
|
+
const manifest = {
|
|
125
|
+
contract_version: '1',
|
|
126
|
+
status,
|
|
127
|
+
scenario: scenarioName,
|
|
128
|
+
commit_sha: getHeadSha(rootDir),
|
|
129
|
+
started_at: startedAt.toISOString(),
|
|
130
|
+
completed_at: new Date().toISOString(),
|
|
131
|
+
commands_run: commandsRun,
|
|
132
|
+
artifacts: await collectArtifacts({
|
|
133
|
+
rootDir,
|
|
134
|
+
artifactDir,
|
|
135
|
+
extraPaths: contract.environment.log_paths,
|
|
136
|
+
since: startedAt,
|
|
137
|
+
}),
|
|
138
|
+
gaps: await readGaps(artifactDir),
|
|
139
|
+
safety: {
|
|
140
|
+
real_email_sent: contract.safety.allow_real_email,
|
|
141
|
+
real_payment_attempted: contract.safety.allow_real_payments,
|
|
142
|
+
customer_data_accessed: contract.safety.allow_customer_data,
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
const manifestPath = await writeManifest(manifest, artifactDir);
|
|
146
|
+
const packetPath = await writeReviewPacket(manifest, artifactDir);
|
|
147
|
+
if (options.json) {
|
|
148
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
149
|
+
}
|
|
150
|
+
else {
|
|
151
|
+
printRunSummary(manifest, manifestPath, packetPath, rootDir);
|
|
152
|
+
}
|
|
153
|
+
process.exit(status === 'verified' ? 0 : status === 'unsafe_to_verify' ? 2 : 1);
|
|
154
|
+
};
|
|
155
|
+
// Safety gate before anything runs.
|
|
156
|
+
const { safety } = contract;
|
|
157
|
+
if (safety.allow_real_email || safety.allow_real_payments || safety.allow_customer_data) {
|
|
158
|
+
console.error(chalk.red('Refusing to run: the safety policy permits real external side effects.') +
|
|
159
|
+
' Fix the safety section of the contract first.');
|
|
160
|
+
await finish('unsafe_to_verify');
|
|
161
|
+
}
|
|
162
|
+
if (options.setup && contract.environment.setup) {
|
|
163
|
+
progress(chalk.dim(`$ ${contract.environment.setup}`));
|
|
164
|
+
const setup = await runCommand(contract.environment.setup, {
|
|
165
|
+
cwd: rootDir,
|
|
166
|
+
timeoutMs: 15 * 60 * 1000,
|
|
167
|
+
logFile: path.join(artifactDir, 'logs', 'setup.log'),
|
|
168
|
+
env,
|
|
169
|
+
quiet,
|
|
170
|
+
});
|
|
171
|
+
commandsRun.push(setup);
|
|
172
|
+
if (setup.status !== 'passed')
|
|
173
|
+
await finish('environment_not_ready');
|
|
174
|
+
}
|
|
175
|
+
if (!options.skipPreflight && scenario.preflight) {
|
|
176
|
+
progress(chalk.dim(`$ ${contract.environment.preflight}`));
|
|
177
|
+
const preflight = await runCommand(contract.environment.preflight, {
|
|
178
|
+
cwd: rootDir,
|
|
179
|
+
timeoutMs: PREFLIGHT_TIMEOUT_S * 1000,
|
|
180
|
+
logFile: path.join(artifactDir, 'logs', 'preflight.log'),
|
|
181
|
+
env,
|
|
182
|
+
quiet,
|
|
183
|
+
});
|
|
184
|
+
commandsRun.push(preflight);
|
|
185
|
+
if (preflight.status !== 'passed')
|
|
186
|
+
await finish('environment_not_ready');
|
|
187
|
+
}
|
|
188
|
+
const timeoutS = options.timeout ?? scenario.timeout_seconds ?? DEFAULT_SCENARIO_TIMEOUT_S;
|
|
189
|
+
progress(chalk.dim(`$ ${scenario.command}`));
|
|
190
|
+
const run = await runCommand(scenario.command, {
|
|
191
|
+
cwd: rootDir,
|
|
192
|
+
timeoutMs: timeoutS * 1000,
|
|
193
|
+
logFile: path.join(artifactDir, 'logs', `scenario-${scenarioName}.log`),
|
|
194
|
+
env,
|
|
195
|
+
quiet,
|
|
196
|
+
});
|
|
197
|
+
commandsRun.push(run);
|
|
198
|
+
await finish(run.status === 'passed' ? 'verified' : 'failed');
|
|
199
|
+
}
|
|
200
|
+
function printRunSummary(manifest, manifestPath, packetPath, rootDir) {
|
|
201
|
+
const statusStyle = {
|
|
202
|
+
verified: chalk.green,
|
|
203
|
+
failed: chalk.red,
|
|
204
|
+
inconclusive: chalk.yellow,
|
|
205
|
+
environment_not_ready: chalk.yellow,
|
|
206
|
+
unsafe_to_verify: chalk.red,
|
|
207
|
+
missing_contract: chalk.red,
|
|
208
|
+
};
|
|
209
|
+
console.log();
|
|
210
|
+
console.log(chalk.white('Scenario: ') + manifest.scenario);
|
|
211
|
+
console.log(chalk.white('Status: ') + statusStyle[manifest.status](manifest.status));
|
|
212
|
+
console.log(chalk.white('Evidence: ') + `${manifest.artifacts.length} artifact(s)`);
|
|
213
|
+
if (manifest.gaps.length > 0) {
|
|
214
|
+
console.log(chalk.white('Gaps: ') + chalk.yellow(`${manifest.gaps.length} declared`));
|
|
215
|
+
}
|
|
216
|
+
console.log();
|
|
217
|
+
console.log(chalk.dim(`Manifest: ${path.relative(rootDir, manifestPath)}`));
|
|
218
|
+
console.log(chalk.dim(`Review packet: ${path.relative(rootDir, packetPath)}`));
|
|
219
|
+
}
|
|
220
|
+
// ─── collect-artifacts ────────────────────────────────────────────────────────
|
|
221
|
+
export async function verificationCollectCommand(options) {
|
|
222
|
+
const rootDir = process.cwd();
|
|
223
|
+
const contract = await requireContract();
|
|
224
|
+
const artifactDir = await ensureArtifactDir(contract);
|
|
225
|
+
const now = new Date().toISOString();
|
|
226
|
+
const manifest = {
|
|
227
|
+
contract_version: '1',
|
|
228
|
+
// Artifacts without a recorded run prove presence, not success.
|
|
229
|
+
status: 'inconclusive',
|
|
230
|
+
scenario: options.scenario ?? 'collected',
|
|
231
|
+
commit_sha: getHeadSha(rootDir),
|
|
232
|
+
started_at: now,
|
|
233
|
+
completed_at: now,
|
|
234
|
+
commands_run: [],
|
|
235
|
+
artifacts: await collectArtifacts({
|
|
236
|
+
rootDir,
|
|
237
|
+
artifactDir,
|
|
238
|
+
extraPaths: contract.environment.log_paths,
|
|
239
|
+
}),
|
|
240
|
+
gaps: await readGaps(artifactDir),
|
|
241
|
+
safety: {
|
|
242
|
+
real_email_sent: contract.safety.allow_real_email,
|
|
243
|
+
real_payment_attempted: contract.safety.allow_real_payments,
|
|
244
|
+
customer_data_accessed: contract.safety.allow_customer_data,
|
|
245
|
+
},
|
|
246
|
+
};
|
|
247
|
+
const manifestPath = await writeManifest(manifest, artifactDir);
|
|
248
|
+
const packetPath = await writeReviewPacket(manifest, artifactDir);
|
|
249
|
+
if (options.json) {
|
|
250
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
251
|
+
}
|
|
252
|
+
else {
|
|
253
|
+
printRunSummary(manifest, manifestPath, packetPath, rootDir);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
// ─── check-safety ─────────────────────────────────────────────────────────────
|
|
257
|
+
export async function verificationCheckSafetyCommand(options) {
|
|
258
|
+
const report = await checkSafety();
|
|
259
|
+
if (options.json) {
|
|
260
|
+
console.log(JSON.stringify(report, null, 2));
|
|
261
|
+
}
|
|
262
|
+
else {
|
|
263
|
+
for (const check of report.checks) {
|
|
264
|
+
const bullet = check.status === 'pass' ? chalk.green('✓') : check.status === 'warn' ? chalk.yellow('•') : chalk.red('✗');
|
|
265
|
+
console.log(`${bullet} ${chalk.white(check.name)}: ${check.detail}`);
|
|
266
|
+
}
|
|
267
|
+
console.log();
|
|
268
|
+
console.log(report.ok
|
|
269
|
+
? chalk.green('Safety checks passed.')
|
|
270
|
+
: chalk.red('Safety checks failed — fix the items above before running verification.'));
|
|
271
|
+
}
|
|
272
|
+
if (!report.ok)
|
|
273
|
+
process.exit(1);
|
|
274
|
+
}
|
|
275
|
+
// ─── shared ───────────────────────────────────────────────────────────────────
|
|
276
|
+
async function requireContract() {
|
|
277
|
+
const loaded = await loadContract();
|
|
278
|
+
if (loaded.status === 'missing') {
|
|
279
|
+
console.error(chalk.red(`No verification contract found at ${CONTRACT_RELATIVE_PATH}.`) +
|
|
280
|
+
' Run `haystack verification init` to scaffold one.');
|
|
281
|
+
process.exit(1);
|
|
282
|
+
}
|
|
283
|
+
if (loaded.status === 'invalid') {
|
|
284
|
+
console.error(chalk.red(`Invalid ${CONTRACT_RELATIVE_PATH}:`));
|
|
285
|
+
for (const error of loaded.errors) {
|
|
286
|
+
console.error(chalk.red(` ✗ ${error}`));
|
|
287
|
+
}
|
|
288
|
+
process.exit(1);
|
|
289
|
+
}
|
|
290
|
+
return loaded.contract;
|
|
291
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -41,6 +41,7 @@ import { setupCommand } from './commands/setup.js';
|
|
|
41
41
|
import { schemaCommand, listSchemas } from './commands/schema-cmd.js';
|
|
42
42
|
import { registerWebhook, listWebhooks, rotateWebhookSecret, setWebhookEnabled, listDeliveries, replayDelivery, } from './commands/webhooks.js';
|
|
43
43
|
import { editRulesCommand, validateRulesCommand } from './commands/rules.js';
|
|
44
|
+
import { verificationInitCommand, verificationValidateCommand, verificationPreflightCommand, verificationRunCommand, verificationCollectCommand, verificationCheckSafetyCommand, } from './commands/verification.js';
|
|
44
45
|
import { headlessLoginCommand, headlessLogoutCommand, listTokensCommand, revokeTokenCommand, } from './commands/tokens.js';
|
|
45
46
|
// `mcp` is imported lazily inside its command action (below), NOT at the top level.
|
|
46
47
|
// It is the only module that pulls in `@modelcontextprotocol/sdk`; keeping it out of
|
|
@@ -817,6 +818,135 @@ rules
|
|
|
817
818
|
process.exit(1);
|
|
818
819
|
}
|
|
819
820
|
});
|
|
821
|
+
// ─── verification ────────────────────────────────────────────────────────────
|
|
822
|
+
//
|
|
823
|
+
// Repo-local verification contract (.haystack/verification.yml): how to boot
|
|
824
|
+
// the app, prove it's ready, log in as test personas, run scenarios, and
|
|
825
|
+
// collect evidence into a standard artifact manifest + review packet.
|
|
826
|
+
// See docs/VERIFICATION-CONTRACT.md.
|
|
827
|
+
const verification = program
|
|
828
|
+
.command('verification')
|
|
829
|
+
.description('Repo verification contract — validate, run scenarios, collect evidence');
|
|
830
|
+
verification
|
|
831
|
+
.command('init')
|
|
832
|
+
.description('Scaffold .haystack/verification.yml, preflight/smoke scripts, and the agent skill')
|
|
833
|
+
.option('-f, --force', 'Overwrite existing files')
|
|
834
|
+
.option('--skill', 'Only write the install-verification agent skill (hand the integration to a coding agent)')
|
|
835
|
+
.addHelpText('after', `
|
|
836
|
+
Creates:
|
|
837
|
+
.haystack/verification.yml the verification contract
|
|
838
|
+
scripts/haystack-preflight app-is-ready check
|
|
839
|
+
scripts/haystack-smoke boot + capture-evidence smoke scenario
|
|
840
|
+
.haystack/AGENTS.md how agents should use the contract
|
|
841
|
+
.haystack/skills/install-verification/SKILL.md skill for completing the integration
|
|
842
|
+
|
|
843
|
+
Detection fills in framework, package manager, dev command, port, and any
|
|
844
|
+
auth-bypass env var found in the repo's own env examples. Review the output —
|
|
845
|
+
detection is a starting point, not truth.
|
|
846
|
+
`)
|
|
847
|
+
.action(async (opts) => {
|
|
848
|
+
try {
|
|
849
|
+
await verificationInitCommand(opts);
|
|
850
|
+
}
|
|
851
|
+
catch (err) {
|
|
852
|
+
console.error(chalk.red('init failed:'), err instanceof Error ? err.message : err);
|
|
853
|
+
process.exit(1);
|
|
854
|
+
}
|
|
855
|
+
});
|
|
856
|
+
verification
|
|
857
|
+
.command('validate')
|
|
858
|
+
.description('Check whether the repo satisfies the verification contract')
|
|
859
|
+
.option('--json', 'Machine-readable report')
|
|
860
|
+
.addHelpText('after', `
|
|
861
|
+
Readiness verdicts:
|
|
862
|
+
ready all checks pass (exit 0)
|
|
863
|
+
partially_ready usable, with gaps (exit 0)
|
|
864
|
+
not_ready contract missing/invalid or commands broken (exit 1)
|
|
865
|
+
unsafe contract permits real side effects, or login helpers
|
|
866
|
+
lack a production-disabled proof (exit 2)
|
|
867
|
+
`)
|
|
868
|
+
.action(async (opts) => {
|
|
869
|
+
try {
|
|
870
|
+
await verificationValidateCommand(opts);
|
|
871
|
+
}
|
|
872
|
+
catch (err) {
|
|
873
|
+
console.error(chalk.red('validate failed:'), err instanceof Error ? err.message : err);
|
|
874
|
+
process.exit(1);
|
|
875
|
+
}
|
|
876
|
+
});
|
|
877
|
+
verification
|
|
878
|
+
.command('preflight')
|
|
879
|
+
.description('Run the contract preflight — is the app up and answering?')
|
|
880
|
+
.action(async () => {
|
|
881
|
+
try {
|
|
882
|
+
await verificationPreflightCommand();
|
|
883
|
+
}
|
|
884
|
+
catch (err) {
|
|
885
|
+
console.error(chalk.red('preflight failed:'), err instanceof Error ? err.message : err);
|
|
886
|
+
process.exit(1);
|
|
887
|
+
}
|
|
888
|
+
});
|
|
889
|
+
verification
|
|
890
|
+
.command('run <scenario>')
|
|
891
|
+
.description('Run a contract scenario and write the artifact manifest + review packet')
|
|
892
|
+
.option('--json', 'Print the manifest as JSON')
|
|
893
|
+
.option('--setup', 'Run environment.setup first (clean-clone mode)')
|
|
894
|
+
.option('--skip-preflight', 'Skip the preflight check')
|
|
895
|
+
.option('--timeout <seconds>', 'Override the scenario timeout', (v) => parseInt(v, 10))
|
|
896
|
+
.addHelpText('after', `
|
|
897
|
+
Runs preflight, then the scenario command, capturing stdout/stderr to the
|
|
898
|
+
artifact directory. Ends with a status:
|
|
899
|
+
verified scenario passed (exit 0)
|
|
900
|
+
failed scenario failed (exit 1)
|
|
901
|
+
environment_not_ready setup/preflight failed before the scenario ran (exit 1)
|
|
902
|
+
unsafe_to_verify safety policy permits real side effects (exit 2)
|
|
903
|
+
|
|
904
|
+
Evidence lands in the contract's artifact_dir:
|
|
905
|
+
manifest-<scenario>.json machine-readable evidence manifest
|
|
906
|
+
review-packet-<scenario>.md human-readable review packet
|
|
907
|
+
logs/ captured command output
|
|
908
|
+
|
|
909
|
+
Examples:
|
|
910
|
+
haystack verification run smoke
|
|
911
|
+
haystack verification run billing_upgrade --json
|
|
912
|
+
haystack verification run smoke --setup # clean-clone mode (CI)
|
|
913
|
+
`)
|
|
914
|
+
.action(async (scenario, opts) => {
|
|
915
|
+
try {
|
|
916
|
+
await verificationRunCommand(scenario, opts);
|
|
917
|
+
}
|
|
918
|
+
catch (err) {
|
|
919
|
+
console.error(chalk.red('run failed:'), err instanceof Error ? err.message : err);
|
|
920
|
+
process.exit(1);
|
|
921
|
+
}
|
|
922
|
+
});
|
|
923
|
+
verification
|
|
924
|
+
.command('collect-artifacts')
|
|
925
|
+
.description('Rebuild an evidence manifest from artifacts already on disk (status: inconclusive)')
|
|
926
|
+
.option('--scenario <name>', 'Scenario name to record in the manifest')
|
|
927
|
+
.option('--json', 'Print the manifest as JSON')
|
|
928
|
+
.action(async (opts) => {
|
|
929
|
+
try {
|
|
930
|
+
await verificationCollectCommand(opts);
|
|
931
|
+
}
|
|
932
|
+
catch (err) {
|
|
933
|
+
console.error(chalk.red('collect-artifacts failed:'), err instanceof Error ? err.message : err);
|
|
934
|
+
process.exit(1);
|
|
935
|
+
}
|
|
936
|
+
});
|
|
937
|
+
verification
|
|
938
|
+
.command('check-safety')
|
|
939
|
+
.description('Verify the verification setup is safe (policy, prod-disabled proof, dev-route gating, secret scan)')
|
|
940
|
+
.option('--json', 'Machine-readable report')
|
|
941
|
+
.action(async (opts) => {
|
|
942
|
+
try {
|
|
943
|
+
await verificationCheckSafetyCommand(opts);
|
|
944
|
+
}
|
|
945
|
+
catch (err) {
|
|
946
|
+
console.error(chalk.red('check-safety failed:'), err instanceof Error ? err.message : err);
|
|
947
|
+
process.exit(1);
|
|
948
|
+
}
|
|
949
|
+
});
|
|
820
950
|
// ─── mcp ─────────────────────────────────────────────────────────────────────
|
|
821
951
|
program
|
|
822
952
|
.command('mcp')
|
package/dist/utils/detect.js
CHANGED
|
@@ -283,7 +283,9 @@ async function detectAuthBypass(rootDir) {
|
|
|
283
283
|
// File not found
|
|
284
284
|
}
|
|
285
285
|
}
|
|
286
|
-
|
|
286
|
+
// No evidence of an auth-bypass variable — suggest nothing rather than
|
|
287
|
+
// fabricating a SKIP_AUTH the app doesn't read.
|
|
288
|
+
return undefined;
|
|
287
289
|
}
|
|
288
290
|
/**
|
|
289
291
|
* Set suggestions based on detected framework
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const CONTRACT_RELATIVE_PATH = ".haystack/verification.yml";
|
|
3
|
+
export declare const DEFAULT_ARTIFACT_DIR = ".haystack/artifacts";
|
|
4
|
+
declare const scenarioSchema: z.ZodObject<{
|
|
5
|
+
command: z.ZodString;
|
|
6
|
+
description: z.ZodOptional<z.ZodString>;
|
|
7
|
+
/** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
|
|
8
|
+
artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
9
|
+
/** Wall-clock cap for the scenario command. Default applied by the runner. */
|
|
10
|
+
timeout_seconds: z.ZodOptional<z.ZodNumber>;
|
|
11
|
+
/**
|
|
12
|
+
* Whether `run` gates this scenario on environment.preflight. Set false for
|
|
13
|
+
* self-contained scenarios that boot the app themselves (like the scaffolded
|
|
14
|
+
* smoke script) — preflight against a cold environment would always fail.
|
|
15
|
+
*/
|
|
16
|
+
preflight: z.ZodDefault<z.ZodBoolean>;
|
|
17
|
+
}, "strip", z.ZodTypeAny, {
|
|
18
|
+
command: string;
|
|
19
|
+
preflight: boolean;
|
|
20
|
+
description?: string | undefined;
|
|
21
|
+
artifacts?: string[] | undefined;
|
|
22
|
+
timeout_seconds?: number | undefined;
|
|
23
|
+
}, {
|
|
24
|
+
command: string;
|
|
25
|
+
description?: string | undefined;
|
|
26
|
+
artifacts?: string[] | undefined;
|
|
27
|
+
timeout_seconds?: number | undefined;
|
|
28
|
+
preflight?: boolean | undefined;
|
|
29
|
+
}>;
|
|
30
|
+
declare const safetySchema: z.ZodObject<{
|
|
31
|
+
/** Command proving dev-only helpers are disabled in production builds. */
|
|
32
|
+
production_disabled_check: z.ZodOptional<z.ZodString>;
|
|
33
|
+
external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
|
|
34
|
+
allow_real_email: z.ZodDefault<z.ZodBoolean>;
|
|
35
|
+
allow_real_payments: z.ZodDefault<z.ZodBoolean>;
|
|
36
|
+
allow_customer_data: z.ZodDefault<z.ZodBoolean>;
|
|
37
|
+
}, "strip", z.ZodTypeAny, {
|
|
38
|
+
allow_real_email: boolean;
|
|
39
|
+
allow_real_payments: boolean;
|
|
40
|
+
allow_customer_data: boolean;
|
|
41
|
+
production_disabled_check?: string | undefined;
|
|
42
|
+
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
43
|
+
}, {
|
|
44
|
+
production_disabled_check?: string | undefined;
|
|
45
|
+
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
46
|
+
allow_real_email?: boolean | undefined;
|
|
47
|
+
allow_real_payments?: boolean | undefined;
|
|
48
|
+
allow_customer_data?: boolean | undefined;
|
|
49
|
+
}>;
|
|
50
|
+
export declare const verificationContractSchema: z.ZodObject<{
|
|
51
|
+
version: z.ZodEffects<z.ZodUnion<[z.ZodLiteral<1>, z.ZodLiteral<"1">]>, 1, 1 | "1">;
|
|
52
|
+
app: z.ZodObject<{
|
|
53
|
+
name: z.ZodString;
|
|
54
|
+
framework: z.ZodOptional<z.ZodString>;
|
|
55
|
+
}, "strip", z.ZodTypeAny, {
|
|
56
|
+
name: string;
|
|
57
|
+
framework?: string | undefined;
|
|
58
|
+
}, {
|
|
59
|
+
name: string;
|
|
60
|
+
framework?: string | undefined;
|
|
61
|
+
}>;
|
|
62
|
+
environment: z.ZodObject<{
|
|
63
|
+
setup: z.ZodOptional<z.ZodString>;
|
|
64
|
+
start: z.ZodOptional<z.ZodString>;
|
|
65
|
+
preflight: z.ZodString;
|
|
66
|
+
reset: z.ZodOptional<z.ZodString>;
|
|
67
|
+
artifact_dir: z.ZodDefault<z.ZodString>;
|
|
68
|
+
log_paths: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
69
|
+
}, "strip", z.ZodTypeAny, {
|
|
70
|
+
preflight: string;
|
|
71
|
+
artifact_dir: string;
|
|
72
|
+
log_paths: string[];
|
|
73
|
+
start?: string | undefined;
|
|
74
|
+
setup?: string | undefined;
|
|
75
|
+
reset?: string | undefined;
|
|
76
|
+
}, {
|
|
77
|
+
preflight: string;
|
|
78
|
+
start?: string | undefined;
|
|
79
|
+
setup?: string | undefined;
|
|
80
|
+
reset?: string | undefined;
|
|
81
|
+
artifact_dir?: string | undefined;
|
|
82
|
+
log_paths?: string[] | undefined;
|
|
83
|
+
}>;
|
|
84
|
+
personas: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
85
|
+
login: z.ZodString;
|
|
86
|
+
description: z.ZodOptional<z.ZodString>;
|
|
87
|
+
}, "strip", z.ZodTypeAny, {
|
|
88
|
+
login: string;
|
|
89
|
+
description?: string | undefined;
|
|
90
|
+
}, {
|
|
91
|
+
login: string;
|
|
92
|
+
description?: string | undefined;
|
|
93
|
+
}>>>;
|
|
94
|
+
scenarios: z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
95
|
+
command: z.ZodString;
|
|
96
|
+
description: z.ZodOptional<z.ZodString>;
|
|
97
|
+
/** Free-form labels of what the scenario captures (screenshots, server_logs, ...). Informational. */
|
|
98
|
+
artifacts: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
99
|
+
/** Wall-clock cap for the scenario command. Default applied by the runner. */
|
|
100
|
+
timeout_seconds: z.ZodOptional<z.ZodNumber>;
|
|
101
|
+
/**
|
|
102
|
+
* Whether `run` gates this scenario on environment.preflight. Set false for
|
|
103
|
+
* self-contained scenarios that boot the app themselves (like the scaffolded
|
|
104
|
+
* smoke script) — preflight against a cold environment would always fail.
|
|
105
|
+
*/
|
|
106
|
+
preflight: z.ZodDefault<z.ZodBoolean>;
|
|
107
|
+
}, "strip", z.ZodTypeAny, {
|
|
108
|
+
command: string;
|
|
109
|
+
preflight: boolean;
|
|
110
|
+
description?: string | undefined;
|
|
111
|
+
artifacts?: string[] | undefined;
|
|
112
|
+
timeout_seconds?: number | undefined;
|
|
113
|
+
}, {
|
|
114
|
+
command: string;
|
|
115
|
+
description?: string | undefined;
|
|
116
|
+
artifacts?: string[] | undefined;
|
|
117
|
+
timeout_seconds?: number | undefined;
|
|
118
|
+
preflight?: boolean | undefined;
|
|
119
|
+
}>>, Record<string, {
|
|
120
|
+
command: string;
|
|
121
|
+
preflight: boolean;
|
|
122
|
+
description?: string | undefined;
|
|
123
|
+
artifacts?: string[] | undefined;
|
|
124
|
+
timeout_seconds?: number | undefined;
|
|
125
|
+
}>, Record<string, {
|
|
126
|
+
command: string;
|
|
127
|
+
description?: string | undefined;
|
|
128
|
+
artifacts?: string[] | undefined;
|
|
129
|
+
timeout_seconds?: number | undefined;
|
|
130
|
+
preflight?: boolean | undefined;
|
|
131
|
+
}>>;
|
|
132
|
+
safety: z.ZodObject<{
|
|
133
|
+
/** Command proving dev-only helpers are disabled in production builds. */
|
|
134
|
+
production_disabled_check: z.ZodOptional<z.ZodString>;
|
|
135
|
+
external_services: z.ZodOptional<z.ZodEnum<["sandbox_only", "mocked", "none"]>>;
|
|
136
|
+
allow_real_email: z.ZodDefault<z.ZodBoolean>;
|
|
137
|
+
allow_real_payments: z.ZodDefault<z.ZodBoolean>;
|
|
138
|
+
allow_customer_data: z.ZodDefault<z.ZodBoolean>;
|
|
139
|
+
}, "strip", z.ZodTypeAny, {
|
|
140
|
+
allow_real_email: boolean;
|
|
141
|
+
allow_real_payments: boolean;
|
|
142
|
+
allow_customer_data: boolean;
|
|
143
|
+
production_disabled_check?: string | undefined;
|
|
144
|
+
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
145
|
+
}, {
|
|
146
|
+
production_disabled_check?: string | undefined;
|
|
147
|
+
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
148
|
+
allow_real_email?: boolean | undefined;
|
|
149
|
+
allow_real_payments?: boolean | undefined;
|
|
150
|
+
allow_customer_data?: boolean | undefined;
|
|
151
|
+
}>;
|
|
152
|
+
}, "strip", z.ZodTypeAny, {
|
|
153
|
+
version: 1;
|
|
154
|
+
app: {
|
|
155
|
+
name: string;
|
|
156
|
+
framework?: string | undefined;
|
|
157
|
+
};
|
|
158
|
+
environment: {
|
|
159
|
+
preflight: string;
|
|
160
|
+
artifact_dir: string;
|
|
161
|
+
log_paths: string[];
|
|
162
|
+
start?: string | undefined;
|
|
163
|
+
setup?: string | undefined;
|
|
164
|
+
reset?: string | undefined;
|
|
165
|
+
};
|
|
166
|
+
personas: Record<string, {
|
|
167
|
+
login: string;
|
|
168
|
+
description?: string | undefined;
|
|
169
|
+
}>;
|
|
170
|
+
scenarios: Record<string, {
|
|
171
|
+
command: string;
|
|
172
|
+
preflight: boolean;
|
|
173
|
+
description?: string | undefined;
|
|
174
|
+
artifacts?: string[] | undefined;
|
|
175
|
+
timeout_seconds?: number | undefined;
|
|
176
|
+
}>;
|
|
177
|
+
safety: {
|
|
178
|
+
allow_real_email: boolean;
|
|
179
|
+
allow_real_payments: boolean;
|
|
180
|
+
allow_customer_data: boolean;
|
|
181
|
+
production_disabled_check?: string | undefined;
|
|
182
|
+
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
183
|
+
};
|
|
184
|
+
}, {
|
|
185
|
+
version: 1 | "1";
|
|
186
|
+
app: {
|
|
187
|
+
name: string;
|
|
188
|
+
framework?: string | undefined;
|
|
189
|
+
};
|
|
190
|
+
environment: {
|
|
191
|
+
preflight: string;
|
|
192
|
+
start?: string | undefined;
|
|
193
|
+
setup?: string | undefined;
|
|
194
|
+
reset?: string | undefined;
|
|
195
|
+
artifact_dir?: string | undefined;
|
|
196
|
+
log_paths?: string[] | undefined;
|
|
197
|
+
};
|
|
198
|
+
scenarios: Record<string, {
|
|
199
|
+
command: string;
|
|
200
|
+
description?: string | undefined;
|
|
201
|
+
artifacts?: string[] | undefined;
|
|
202
|
+
timeout_seconds?: number | undefined;
|
|
203
|
+
preflight?: boolean | undefined;
|
|
204
|
+
}>;
|
|
205
|
+
safety: {
|
|
206
|
+
production_disabled_check?: string | undefined;
|
|
207
|
+
external_services?: "none" | "sandbox_only" | "mocked" | undefined;
|
|
208
|
+
allow_real_email?: boolean | undefined;
|
|
209
|
+
allow_real_payments?: boolean | undefined;
|
|
210
|
+
allow_customer_data?: boolean | undefined;
|
|
211
|
+
};
|
|
212
|
+
personas?: Record<string, {
|
|
213
|
+
login: string;
|
|
214
|
+
description?: string | undefined;
|
|
215
|
+
}> | undefined;
|
|
216
|
+
}>;
|
|
217
|
+
export type VerificationContract = z.infer<typeof verificationContractSchema>;
|
|
218
|
+
export type VerificationScenario = z.infer<typeof scenarioSchema>;
|
|
219
|
+
export type VerificationSafety = z.infer<typeof safetySchema>;
|
|
220
|
+
export type ContractLoadResult = {
|
|
221
|
+
status: 'ok';
|
|
222
|
+
contract: VerificationContract;
|
|
223
|
+
path: string;
|
|
224
|
+
} | {
|
|
225
|
+
status: 'missing';
|
|
226
|
+
path: string;
|
|
227
|
+
} | {
|
|
228
|
+
status: 'invalid';
|
|
229
|
+
path: string;
|
|
230
|
+
errors: string[];
|
|
231
|
+
};
|
|
232
|
+
/**
|
|
233
|
+
* Load and validate the contract from `<rootDir>/.haystack/verification.yml`.
|
|
234
|
+
* Never throws — parse/schema failures come back as `status: 'invalid'` so
|
|
235
|
+
* callers can map them onto readiness / manifest statuses.
|
|
236
|
+
*/
|
|
237
|
+
export declare function loadContract(rootDir?: string): Promise<ContractLoadResult>;
|
|
238
|
+
/** Resolve the artifact directory to an absolute path, creating it if needed. */
|
|
239
|
+
export declare function ensureArtifactDir(contract: VerificationContract, rootDir?: string): Promise<string>;
|
|
240
|
+
export {};
|