@izkac/forgekit 0.1.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/forge.mjs +7 -1
- package/package.json +1 -1
- package/scripts/run-tests.mjs +7 -0
- package/src/brief-cli.mjs +62 -0
- package/src/brief.mjs +104 -0
- package/src/brief.test.mjs +186 -0
- package/src/cleanup-sessions.mjs +2 -0
- package/src/e2e.mjs +172 -0
- package/src/fleet.mjs +233 -0
- package/src/fleet.test.mjs +137 -0
- package/src/integrity-check.mjs +5 -3
- package/src/integrity.mjs +310 -20
- package/src/integrity.test.mjs +169 -17
- package/src/lib/fleet.mjs +212 -0
- package/src/lib.mjs +5 -0
- package/src/session-reminder.mjs +12 -1
- package/src/set-phase.mjs +28 -0
- package/src/set-phase.test.mjs +9 -1
- package/vendor/skills/forge/SKILL.md +1 -1
- package/vendor/skills/forge/docs/forge.md +53 -31
- package/vendor/skills/forge/phases/finish.md +1 -1
- package/vendor/skills/forge/phases/implement.md +1 -0
- package/vendor/skills/forge/phases/plan-openspec.md +23 -4
- package/vendor/skills/forge/phases/plan-specs.md +23 -4
- package/vendor/skills/forge/phases/verify.md +7 -3
- package/vendor/skills/forge/references/operator-brief.md +74 -0
- package/vendor/skills/forge/references/runtime-integrity.md +28 -10
- package/vendor/skills/forge/subagents/final-reviewer-prompt.md +9 -6
- package/vendor/skills/forge/subagents/task-reviewer-prompt.md +1 -0
package/src/integrity.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Forge runtime-integrity mechanics: spine matrix, deferral registry,
|
|
3
|
-
* and the integrity checks that gate
|
|
3
|
+
* executable E2E acceptance, and the integrity checks that gate
|
|
4
|
+
* `forge phase done|finish`.
|
|
4
5
|
*
|
|
5
6
|
* Spine matrix — `spine.json` in the change dir (or session dir when the
|
|
6
7
|
* session has no tracked change). One row per capability/REQ cluster:
|
|
@@ -8,13 +9,21 @@
|
|
|
8
9
|
* Library-only rows (missing runtime owner / writes / evidence) fail
|
|
9
10
|
* validation, so "wire later" cannot be checkboxed past `forge phase done`.
|
|
10
11
|
*
|
|
12
|
+
* E2E acceptance — `e2e.json` next to the spine: the closed product loop as
|
|
13
|
+
* an executable step list. `forge e2e run` executes it and records
|
|
14
|
+
* `e2e-results.json` (session dir) with a hash of the steps, so results go
|
|
15
|
+
* stale when steps change. When the spine has real rows, the gate requires a
|
|
16
|
+
* green, current run — prose in verify-evidence.md no longer satisfies it.
|
|
17
|
+
*
|
|
11
18
|
* Deferral registry — `deferrals.json` in the session dir. Reviewers may only
|
|
12
19
|
* accept "wiring deferred" when a registered deferral names the open task;
|
|
13
20
|
* unresolved deferrals block done/finish.
|
|
14
21
|
*/
|
|
15
22
|
|
|
23
|
+
import crypto from 'node:crypto';
|
|
16
24
|
import fs from 'node:fs';
|
|
17
25
|
import path from 'node:path';
|
|
26
|
+
import { spawnSync } from 'node:child_process';
|
|
18
27
|
import { readJson, writeJson } from './lib.mjs';
|
|
19
28
|
import { DEFAULT_SPECS_DIR, resolveProjectPlanEngine } from './plan-engine.mjs';
|
|
20
29
|
|
|
@@ -169,6 +178,292 @@ export function validateSpine(doc) {
|
|
|
169
178
|
return { ok: problems.length === 0, problems };
|
|
170
179
|
}
|
|
171
180
|
|
|
181
|
+
/* ------------------------------------------------------------------ */
|
|
182
|
+
/* E2E acceptance — executable product loop */
|
|
183
|
+
/* ------------------------------------------------------------------ */
|
|
184
|
+
|
|
185
|
+
const E2E_FILE = 'e2e.json';
|
|
186
|
+
const E2E_RESULTS_FILE = 'e2e-results.json';
|
|
187
|
+
export const E2E_DEFAULT_TIMEOUT_MS = 300_000;
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Path to e2e.json: change dir when available, else session dir.
|
|
191
|
+
*
|
|
192
|
+
* @param {{ cwd?: string, session?: Record<string, unknown> | null, sessionDir?: string }} opts
|
|
193
|
+
*/
|
|
194
|
+
export function e2ePath(opts = {}) {
|
|
195
|
+
const changeDir = resolveChangeDir(opts);
|
|
196
|
+
if (changeDir) return path.join(changeDir, E2E_FILE);
|
|
197
|
+
if (opts.sessionDir) return path.join(opts.sessionDir, E2E_FILE);
|
|
198
|
+
throw new Error('Cannot resolve e2e.json location: no change and no session dir');
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* @param {{ change?: string | null }} [opts]
|
|
203
|
+
*/
|
|
204
|
+
export function e2eTemplate(opts = {}) {
|
|
205
|
+
return {
|
|
206
|
+
change: opts.change ?? null,
|
|
207
|
+
notApplicable: null,
|
|
208
|
+
steps: [
|
|
209
|
+
{
|
|
210
|
+
name: '<boot>',
|
|
211
|
+
cmd: '<command that starts the system, e.g. docker compose up -d api worker>',
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
name: '<produce>',
|
|
215
|
+
cmd: '<command that drives the real production entry point, e.g. node scripts/e2e/enqueue-analyze.mjs>',
|
|
216
|
+
expect: '<regex the combined output must match — delete this field if exit code 0 is enough>',
|
|
217
|
+
},
|
|
218
|
+
{
|
|
219
|
+
name: '<consume-assert>',
|
|
220
|
+
cmd: '<command that proves the domain side effects exist, e.g. node scripts/e2e/assert-ratified.mjs>',
|
|
221
|
+
},
|
|
222
|
+
],
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Scaffold e2e.json (refuses to overwrite unless force).
|
|
228
|
+
*
|
|
229
|
+
* @param {{ file: string, change?: string | null, force?: boolean }} opts
|
|
230
|
+
*/
|
|
231
|
+
export function initE2e(opts) {
|
|
232
|
+
if (fs.existsSync(opts.file) && !opts.force) {
|
|
233
|
+
throw new Error(`e2e.json already exists: ${opts.file} (use --force to overwrite)`);
|
|
234
|
+
}
|
|
235
|
+
writeJson(opts.file, e2eTemplate({ change: opts.change }));
|
|
236
|
+
return opts.file;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Validate an e2e document.
|
|
241
|
+
*
|
|
242
|
+
* Valid when either:
|
|
243
|
+
* - `notApplicable` is a non-empty string (loop cannot be driven by any
|
|
244
|
+
* command — reviewers police the reason), or
|
|
245
|
+
* - `steps` is a non-empty array where every step has a filled `name` and
|
|
246
|
+
* `cmd` (no scaffold placeholders), `expect` (optional) is a valid regex,
|
|
247
|
+
* and `timeoutMs` (optional) is a positive number.
|
|
248
|
+
*
|
|
249
|
+
* @param {unknown} doc
|
|
250
|
+
* @returns {{ ok: boolean, problems: string[] }}
|
|
251
|
+
*/
|
|
252
|
+
export function validateE2e(doc) {
|
|
253
|
+
/** @type {string[]} */
|
|
254
|
+
const problems = [];
|
|
255
|
+
if (!doc || typeof doc !== 'object' || Array.isArray(doc)) {
|
|
256
|
+
return { ok: false, problems: ['e2e.json is not an object'] };
|
|
257
|
+
}
|
|
258
|
+
const e2e = /** @type {Record<string, unknown>} */ (doc);
|
|
259
|
+
|
|
260
|
+
if (isNonEmptyString(e2e.notApplicable)) {
|
|
261
|
+
return { ok: true, problems: [] };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const steps = e2e.steps;
|
|
265
|
+
if (!Array.isArray(steps) || steps.length === 0) {
|
|
266
|
+
return {
|
|
267
|
+
ok: false,
|
|
268
|
+
problems: [
|
|
269
|
+
'e2e.steps is empty — add executable product-loop steps, or set notApplicable with a reason',
|
|
270
|
+
],
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
steps.forEach((step, i) => {
|
|
275
|
+
if (!step || typeof step !== 'object' || Array.isArray(step)) {
|
|
276
|
+
problems.push(`step ${i + 1}: not an object`);
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
const s = /** @type {Record<string, unknown>} */ (step);
|
|
280
|
+
for (const field of ['name', 'cmd']) {
|
|
281
|
+
const value = s[field];
|
|
282
|
+
if (!isNonEmptyString(value)) {
|
|
283
|
+
problems.push(`step ${i + 1} (${s.name ?? '?'}): missing ${field}`);
|
|
284
|
+
} else if (/^<.*>$/.test(value.trim())) {
|
|
285
|
+
problems.push(`step ${i + 1} (${s.name ?? '?'}): ${field} still has scaffold placeholder`);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
if (s.expect !== undefined && s.expect !== null) {
|
|
289
|
+
if (!isNonEmptyString(s.expect)) {
|
|
290
|
+
problems.push(`step ${i + 1} (${s.name ?? '?'}): expect must be a non-empty regex string`);
|
|
291
|
+
} else if (/^<.*>$/.test(s.expect.trim())) {
|
|
292
|
+
problems.push(`step ${i + 1} (${s.name ?? '?'}): expect still has scaffold placeholder`);
|
|
293
|
+
} else {
|
|
294
|
+
try {
|
|
295
|
+
new RegExp(s.expect);
|
|
296
|
+
} catch {
|
|
297
|
+
problems.push(`step ${i + 1} (${s.name ?? '?'}): expect is not a valid regex`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
if (s.timeoutMs !== undefined && (typeof s.timeoutMs !== 'number' || s.timeoutMs <= 0)) {
|
|
302
|
+
problems.push(`step ${i + 1} (${s.name ?? '?'}): timeoutMs must be a positive number`);
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
return { ok: problems.length === 0, problems };
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Hash of the step list — recorded in results so editing e2e.json after a
|
|
311
|
+
* green run invalidates the results.
|
|
312
|
+
*
|
|
313
|
+
* @param {unknown[]} steps
|
|
314
|
+
*/
|
|
315
|
+
export function e2eStepsHash(steps) {
|
|
316
|
+
return crypto.createHash('sha256').update(JSON.stringify(steps ?? [])).digest('hex');
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* @param {string} text
|
|
321
|
+
*/
|
|
322
|
+
function outputTail(text, lines = 30) {
|
|
323
|
+
if (!text) return '';
|
|
324
|
+
return text.split(/\r?\n/).slice(-lines).join('\n').trim();
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Execute e2e steps sequentially (shell). Stops at the first failure —
|
|
329
|
+
* later steps depend on earlier ones. Exit code must be 0 and `expect`
|
|
330
|
+
* (when present) must match combined stdout+stderr.
|
|
331
|
+
*
|
|
332
|
+
* @param {{ steps?: unknown[] }} doc — a validated e2e document with steps
|
|
333
|
+
* @param {{ cwd?: string }} [opts]
|
|
334
|
+
*/
|
|
335
|
+
export function runE2eSteps(doc, opts = {}) {
|
|
336
|
+
const cwd = opts.cwd ?? process.cwd();
|
|
337
|
+
const steps = Array.isArray(doc?.steps) ? doc.steps : [];
|
|
338
|
+
/** @type {Array<Record<string, unknown>>} */
|
|
339
|
+
const results = [];
|
|
340
|
+
let ok = true;
|
|
341
|
+
|
|
342
|
+
for (const step of steps) {
|
|
343
|
+
const s = /** @type {Record<string, any>} */ (step);
|
|
344
|
+
if (!ok) {
|
|
345
|
+
results.push({ name: s.name, cmd: s.cmd, skipped: true });
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
const started = Date.now();
|
|
349
|
+
const r = spawnSync(s.cmd, {
|
|
350
|
+
shell: true,
|
|
351
|
+
cwd,
|
|
352
|
+
encoding: 'utf8',
|
|
353
|
+
timeout: typeof s.timeoutMs === 'number' ? s.timeoutMs : E2E_DEFAULT_TIMEOUT_MS,
|
|
354
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
355
|
+
});
|
|
356
|
+
const output = `${r.stdout ?? ''}${r.stderr ?? ''}`;
|
|
357
|
+
const exitCode = typeof r.status === 'number' ? r.status : null;
|
|
358
|
+
let expectMatched = null;
|
|
359
|
+
let stepOk = exitCode === 0;
|
|
360
|
+
if (stepOk && isNonEmptyString(s.expect)) {
|
|
361
|
+
expectMatched = new RegExp(s.expect).test(output);
|
|
362
|
+
stepOk = expectMatched;
|
|
363
|
+
}
|
|
364
|
+
results.push({
|
|
365
|
+
name: s.name,
|
|
366
|
+
cmd: s.cmd,
|
|
367
|
+
exitCode,
|
|
368
|
+
expectMatched,
|
|
369
|
+
ok: stepOk,
|
|
370
|
+
durationMs: Date.now() - started,
|
|
371
|
+
outputTail: outputTail(output),
|
|
372
|
+
error: r.error ? String(r.error.message ?? r.error) : null,
|
|
373
|
+
});
|
|
374
|
+
if (!stepOk) ok = false;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
return {
|
|
378
|
+
ok,
|
|
379
|
+
ranAt: new Date().toISOString(),
|
|
380
|
+
stepsHash: e2eStepsHash(steps),
|
|
381
|
+
steps: results,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* @param {string} sessionDir
|
|
387
|
+
*/
|
|
388
|
+
export function e2eResultsPath(sessionDir) {
|
|
389
|
+
return path.join(sessionDir, E2E_RESULTS_FILE);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* @param {string} sessionDir
|
|
394
|
+
* @param {ReturnType<typeof runE2eSteps>} results
|
|
395
|
+
*/
|
|
396
|
+
export function writeE2eResults(sessionDir, results) {
|
|
397
|
+
writeJson(e2eResultsPath(sessionDir), results);
|
|
398
|
+
return e2eResultsPath(sessionDir);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* @param {string} sessionDir
|
|
403
|
+
* @returns {Record<string, any> | null}
|
|
404
|
+
*/
|
|
405
|
+
export function loadE2eResults(sessionDir) {
|
|
406
|
+
const file = e2eResultsPath(sessionDir);
|
|
407
|
+
if (!fs.existsSync(file)) return null;
|
|
408
|
+
try {
|
|
409
|
+
return readJson(file);
|
|
410
|
+
} catch {
|
|
411
|
+
return null;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Gate check for the executable E2E acceptance. Returns the problems that
|
|
417
|
+
* block `forge phase done` — empty when the loop was executed green (and the
|
|
418
|
+
* results are current), or when e2e.json honestly opts out via notApplicable.
|
|
419
|
+
*
|
|
420
|
+
* @param {{ e2eFile: string, sessionDir: string }} opts
|
|
421
|
+
* @returns {{ problems: string[], notApplicable: boolean }}
|
|
422
|
+
*/
|
|
423
|
+
export function checkE2eGate(opts) {
|
|
424
|
+
/** @type {string[]} */
|
|
425
|
+
const problems = [];
|
|
426
|
+
|
|
427
|
+
if (!fs.existsSync(opts.e2eFile)) {
|
|
428
|
+
problems.push(
|
|
429
|
+
`e2e.json required at ${opts.e2eFile} — run forge e2e init, author the product-loop steps, then forge e2e run. Spine rows mean an async loop exists; it must be executed, not described.`,
|
|
430
|
+
);
|
|
431
|
+
return { problems, notApplicable: false };
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
let doc;
|
|
435
|
+
try {
|
|
436
|
+
doc = readJson(opts.e2eFile);
|
|
437
|
+
} catch (err) {
|
|
438
|
+
problems.push(`e2e.json unreadable: ${err instanceof Error ? err.message : err}`);
|
|
439
|
+
return { problems, notApplicable: false };
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
const valid = validateE2e(doc);
|
|
443
|
+
if (!valid.ok) {
|
|
444
|
+
problems.push(...valid.problems.map((p) => `e2e: ${p}`));
|
|
445
|
+
return { problems, notApplicable: false };
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
if (isNonEmptyString(doc.notApplicable)) {
|
|
449
|
+
return { problems: [], notApplicable: true };
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
const results = loadE2eResults(opts.sessionDir);
|
|
453
|
+
if (!results) {
|
|
454
|
+
problems.push('e2e-results.json missing — run forge e2e run (a green run is required before done)');
|
|
455
|
+
} else if (results.stepsHash !== e2eStepsHash(doc.steps)) {
|
|
456
|
+
problems.push('e2e-results.json is stale — e2e.json changed since the last run; re-run forge e2e run');
|
|
457
|
+
} else if (!results.ok) {
|
|
458
|
+
const failed = Array.isArray(results.steps) ? results.steps.find((s) => s?.ok === false) : null;
|
|
459
|
+
problems.push(
|
|
460
|
+
`e2e run failed${failed ? ` at step "${failed.name}"` : ''} — fix and re-run forge e2e run`,
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
return { problems, notApplicable: false };
|
|
465
|
+
}
|
|
466
|
+
|
|
172
467
|
/**
|
|
173
468
|
* @param {string} sessionDir
|
|
174
469
|
*/
|
|
@@ -245,13 +540,15 @@ export function sessionJobsSignalText(session) {
|
|
|
245
540
|
* 2. spine.json — **always required** (filled rows, or `notApplicable` with a
|
|
246
541
|
* reason). Keyword sniffing is not enough to decide; missing spine is how
|
|
247
542
|
* library-only platforms checkbox past gaps.
|
|
248
|
-
* 3.
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
*
|
|
543
|
+
* 3. E2E acceptance — when a spine has real rows (not notApplicable):
|
|
544
|
+
* e2e.json must exist with filled steps (or its own notApplicable reason)
|
|
545
|
+
* and e2e-results.json must record a green, current run (steps hash must
|
|
546
|
+
* match). Prose in verify-evidence.md does not satisfy this; an explicit
|
|
547
|
+
* BLOCKED marker there still means the change cannot be done. Sync-only
|
|
548
|
+
* work should prefer spine `notApplicable` over inventing a fake loop.
|
|
252
549
|
*
|
|
253
550
|
* @param {{ cwd?: string, sessionDir: string, session: Record<string, unknown> }} opts
|
|
254
|
-
* @returns {{ ok: boolean, problems: string[], spineFile: string, spineExists: boolean }}
|
|
551
|
+
* @returns {{ ok: boolean, problems: string[], spineFile: string, spineExists: boolean, e2eFile: string | null }}
|
|
255
552
|
*/
|
|
256
553
|
export function runIntegrityChecks(opts) {
|
|
257
554
|
/** @type {string[]} */
|
|
@@ -295,23 +592,16 @@ export function runIntegrityChecks(opts) {
|
|
|
295
592
|
}
|
|
296
593
|
}
|
|
297
594
|
|
|
595
|
+
let e2eFile = null;
|
|
298
596
|
if (spineExists && spineHasRows) {
|
|
597
|
+
e2eFile = e2ePath({ cwd, session, sessionDir });
|
|
598
|
+
problems.push(...checkE2eGate({ e2eFile, sessionDir }).problems);
|
|
599
|
+
|
|
299
600
|
const evidenceFile = path.join(sessionDir, 'verify-evidence.md');
|
|
300
|
-
if (
|
|
301
|
-
problems.push(
|
|
302
|
-
'verify-evidence.md missing — spine rows require product-loop evidence (or use notApplicable for sync-only work)',
|
|
303
|
-
);
|
|
304
|
-
} else {
|
|
305
|
-
const body = fs.readFileSync(evidenceFile, 'utf8');
|
|
306
|
-
if (/\bBLOCKED\b/.test(body)) {
|
|
307
|
-
problems.push('verify-evidence.md contains BLOCKED — change cannot be marked done while E2E is blocked');
|
|
308
|
-
} else if (!/product[- ]loop/i.test(body)) {
|
|
309
|
-
problems.push(
|
|
310
|
-
'verify-evidence.md has no "Product loop" section — record the closed producer→consumer loop (or BLOCKED). Sync-only changes should use spine notApplicable instead.',
|
|
311
|
-
);
|
|
312
|
-
}
|
|
601
|
+
if (fs.existsSync(evidenceFile) && /\bBLOCKED\b/.test(fs.readFileSync(evidenceFile, 'utf8'))) {
|
|
602
|
+
problems.push('verify-evidence.md contains BLOCKED — change cannot be marked done while E2E is blocked');
|
|
313
603
|
}
|
|
314
604
|
}
|
|
315
605
|
|
|
316
|
-
return { ok: problems.length === 0, problems, spineFile, spineExists };
|
|
606
|
+
return { ok: problems.length === 0, problems, spineFile, spineExists, e2eFile };
|
|
317
607
|
}
|
package/src/integrity.test.mjs
CHANGED
|
@@ -6,16 +6,24 @@ import { tmpdir } from 'node:os';
|
|
|
6
6
|
import {
|
|
7
7
|
JOBS_SIGNAL_RE,
|
|
8
8
|
addDeferral,
|
|
9
|
+
checkE2eGate,
|
|
10
|
+
e2ePath,
|
|
11
|
+
e2eStepsHash,
|
|
12
|
+
e2eTemplate,
|
|
13
|
+
initE2e,
|
|
9
14
|
initSpine,
|
|
10
15
|
loadDeferrals,
|
|
11
16
|
openDeferrals,
|
|
12
17
|
resolveChangeDir,
|
|
13
18
|
resolveDeferral,
|
|
19
|
+
runE2eSteps,
|
|
14
20
|
runIntegrityChecks,
|
|
15
21
|
sessionJobsSignalText,
|
|
16
22
|
spinePath,
|
|
17
23
|
spineTemplate,
|
|
24
|
+
validateE2e,
|
|
18
25
|
validateSpine,
|
|
26
|
+
writeE2eResults,
|
|
19
27
|
} from './integrity.mjs';
|
|
20
28
|
|
|
21
29
|
function tmp(prefix) {
|
|
@@ -215,40 +223,184 @@ test('runIntegrityChecks: empty slug without spine still fails', () => {
|
|
|
215
223
|
}
|
|
216
224
|
});
|
|
217
225
|
|
|
218
|
-
|
|
226
|
+
function greenStep(overrides = {}) {
|
|
227
|
+
return {
|
|
228
|
+
name: 'produce',
|
|
229
|
+
cmd: 'node -e "console.log(\'proposals: 3\')"',
|
|
230
|
+
...overrides,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function writeSpineWithRows(sessionDir) {
|
|
235
|
+
fs.writeFileSync(
|
|
236
|
+
path.join(sessionDir, 'spine.json'),
|
|
237
|
+
`${JSON.stringify({ change: null, notApplicable: null, rows: [validRow()] }, null, 2)}\n`,
|
|
238
|
+
'utf8',
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function writeE2eDoc(sessionDir, doc) {
|
|
243
|
+
fs.writeFileSync(path.join(sessionDir, 'e2e.json'), `${JSON.stringify(doc, null, 2)}\n`, 'utf8');
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
test('validateE2e: filled steps pass; template placeholders rejected', () => {
|
|
247
|
+
assert.equal(validateE2e({ notApplicable: null, steps: [greenStep()] }).ok, true);
|
|
248
|
+
const scaffold = validateE2e(e2eTemplate({ change: 'x' }));
|
|
249
|
+
assert.equal(scaffold.ok, false);
|
|
250
|
+
assert.match(scaffold.problems.join('\n'), /scaffold placeholder/);
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
test('validateE2e: missing cmd, bad regex, bad timeout, empty steps', () => {
|
|
254
|
+
assert.match(
|
|
255
|
+
validateE2e({ steps: [{ name: 'x', cmd: '' }] }).problems.join('\n'),
|
|
256
|
+
/missing cmd/,
|
|
257
|
+
);
|
|
258
|
+
assert.match(
|
|
259
|
+
validateE2e({ steps: [greenStep({ expect: '(' })] }).problems.join('\n'),
|
|
260
|
+
/not a valid regex/,
|
|
261
|
+
);
|
|
262
|
+
assert.match(
|
|
263
|
+
validateE2e({ steps: [greenStep({ timeoutMs: -5 })] }).problems.join('\n'),
|
|
264
|
+
/timeoutMs/,
|
|
265
|
+
);
|
|
266
|
+
assert.match(validateE2e({ steps: [] }).problems.join('\n'), /steps is empty/);
|
|
267
|
+
assert.equal(validateE2e({ steps: [], notApplicable: 'no headless env — manual device loop' }).ok, true);
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test('e2e init writes template and refuses overwrite without force', () => {
|
|
271
|
+
const dir = tmp('forge-e2e-init-');
|
|
272
|
+
try {
|
|
273
|
+
const file = path.join(dir, 'e2e.json');
|
|
274
|
+
initE2e({ file, change: 'my-change' });
|
|
275
|
+
assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).change, 'my-change');
|
|
276
|
+
assert.throws(() => initE2e({ file, change: 'my-change' }), /already exists/);
|
|
277
|
+
initE2e({ file, change: 'other', force: true });
|
|
278
|
+
assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).change, 'other');
|
|
279
|
+
assert.equal(
|
|
280
|
+
e2ePath({ cwd: dir, session: { openspecChange: null }, sessionDir: dir }),
|
|
281
|
+
file,
|
|
282
|
+
);
|
|
283
|
+
} finally {
|
|
284
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
285
|
+
}
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
test('runE2eSteps: green run with expect match', () => {
|
|
289
|
+
const results = runE2eSteps({ steps: [greenStep({ expect: 'proposals: \\d+' })] });
|
|
290
|
+
assert.equal(results.ok, true);
|
|
291
|
+
assert.equal(results.steps[0].exitCode, 0);
|
|
292
|
+
assert.equal(results.steps[0].expectMatched, true);
|
|
293
|
+
assert.equal(results.stepsHash, e2eStepsHash([greenStep({ expect: 'proposals: \\d+' })]));
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
test('runE2eSteps: non-zero exit fails and skips later steps', () => {
|
|
297
|
+
const results = runE2eSteps({
|
|
298
|
+
steps: [
|
|
299
|
+
{ name: 'boom', cmd: 'node -e "process.exit(3)"' },
|
|
300
|
+
greenStep({ name: 'never' }),
|
|
301
|
+
],
|
|
302
|
+
});
|
|
303
|
+
assert.equal(results.ok, false);
|
|
304
|
+
assert.equal(results.steps[0].exitCode, 3);
|
|
305
|
+
assert.equal(results.steps[1].skipped, true);
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
test('runE2eSteps: exit 0 but expect mismatch fails', () => {
|
|
309
|
+
const results = runE2eSteps({ steps: [greenStep({ expect: 'ratified: \\d+' })] });
|
|
310
|
+
assert.equal(results.ok, false);
|
|
311
|
+
assert.equal(results.steps[0].expectMatched, false);
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
test('checkE2eGate: missing file, missing results, stale hash, failed run, green, notApplicable', () => {
|
|
315
|
+
const dir = tmp('forge-e2e-gate-');
|
|
316
|
+
try {
|
|
317
|
+
const e2eFile = path.join(dir, 'e2e.json');
|
|
318
|
+
|
|
319
|
+
let gate = checkE2eGate({ e2eFile, sessionDir: dir });
|
|
320
|
+
assert.match(gate.problems.join('\n'), /e2e\.json required/);
|
|
321
|
+
|
|
322
|
+
writeE2eDoc(dir, { notApplicable: null, steps: [greenStep()] });
|
|
323
|
+
gate = checkE2eGate({ e2eFile, sessionDir: dir });
|
|
324
|
+
assert.match(gate.problems.join('\n'), /e2e-results\.json missing/);
|
|
325
|
+
|
|
326
|
+
const results = runE2eSteps({ steps: [greenStep()] });
|
|
327
|
+
writeE2eResults(dir, results);
|
|
328
|
+
gate = checkE2eGate({ e2eFile, sessionDir: dir });
|
|
329
|
+
assert.deepEqual(gate.problems, []);
|
|
330
|
+
|
|
331
|
+
writeE2eDoc(dir, { notApplicable: null, steps: [greenStep({ name: 'edited' })] });
|
|
332
|
+
gate = checkE2eGate({ e2eFile, sessionDir: dir });
|
|
333
|
+
assert.match(gate.problems.join('\n'), /stale/);
|
|
334
|
+
|
|
335
|
+
writeE2eDoc(dir, { notApplicable: null, steps: [{ name: 'boom', cmd: 'node -e "process.exit(1)"' }] });
|
|
336
|
+
writeE2eResults(dir, runE2eSteps({ steps: [{ name: 'boom', cmd: 'node -e "process.exit(1)"' }] }));
|
|
337
|
+
gate = checkE2eGate({ e2eFile, sessionDir: dir });
|
|
338
|
+
assert.match(gate.problems.join('\n'), /failed at step "boom"/);
|
|
339
|
+
|
|
340
|
+
writeE2eDoc(dir, { notApplicable: 'loop needs a physical device', steps: [] });
|
|
341
|
+
gate = checkE2eGate({ e2eFile, sessionDir: dir });
|
|
342
|
+
assert.deepEqual(gate.problems, []);
|
|
343
|
+
assert.equal(gate.notApplicable, true);
|
|
344
|
+
} finally {
|
|
345
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
346
|
+
}
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
test('runIntegrityChecks: spine rows demand an executed green e2e run', () => {
|
|
219
350
|
const cwd = tmp('forge-int-loop-');
|
|
220
351
|
try {
|
|
221
352
|
const sessionDir = makeSessionDir(cwd);
|
|
222
|
-
|
|
223
|
-
fs.writeFileSync(
|
|
224
|
-
spineFile,
|
|
225
|
-
`${JSON.stringify({ change: null, notApplicable: null, rows: [validRow()] }, null, 2)}\n`,
|
|
226
|
-
'utf8',
|
|
227
|
-
);
|
|
353
|
+
writeSpineWithRows(sessionDir);
|
|
228
354
|
const session = { slug: 'wire-worker-jobs', openspecChange: null };
|
|
229
355
|
|
|
230
356
|
let result = runIntegrityChecks({ cwd, sessionDir, session });
|
|
231
357
|
assert.equal(result.ok, false);
|
|
232
|
-
assert.match(result.problems.join('\n'), /
|
|
358
|
+
assert.match(result.problems.join('\n'), /e2e\.json required/);
|
|
233
359
|
|
|
234
|
-
|
|
235
|
-
fs.writeFileSync(evidenceFile, '# Verify\n\ntier 3 green\n', 'utf8');
|
|
360
|
+
writeE2eDoc(sessionDir, { notApplicable: null, steps: [greenStep()] });
|
|
236
361
|
result = runIntegrityChecks({ cwd, sessionDir, session });
|
|
237
362
|
assert.equal(result.ok, false);
|
|
238
|
-
assert.match(result.problems.join('\n'), /
|
|
363
|
+
assert.match(result.problems.join('\n'), /e2e-results\.json missing/);
|
|
239
364
|
|
|
240
|
-
|
|
365
|
+
writeE2eResults(sessionDir, runE2eSteps({ steps: [greenStep()] }));
|
|
241
366
|
result = runIntegrityChecks({ cwd, sessionDir, session });
|
|
242
|
-
assert.equal(result.ok,
|
|
243
|
-
assert.
|
|
367
|
+
assert.equal(result.ok, true);
|
|
368
|
+
assert.equal(result.e2eFile, path.join(sessionDir, 'e2e.json'));
|
|
244
369
|
|
|
370
|
+
// prose "## Product loop" alone no longer satisfies the gate
|
|
371
|
+
fs.rmSync(path.join(sessionDir, 'e2e-results.json'));
|
|
245
372
|
fs.writeFileSync(
|
|
246
|
-
|
|
247
|
-
'# Verify\n\n## Product loop\n\ningest
|
|
373
|
+
path.join(sessionDir, 'verify-evidence.md'),
|
|
374
|
+
'# Verify\n\n## Product loop\n\ningest -> analyze -> ratify: output differs\n',
|
|
248
375
|
'utf8',
|
|
249
376
|
);
|
|
250
377
|
result = runIntegrityChecks({ cwd, sessionDir, session });
|
|
251
|
-
assert.equal(result.ok,
|
|
378
|
+
assert.equal(result.ok, false);
|
|
379
|
+
assert.match(result.problems.join('\n'), /e2e-results\.json missing/);
|
|
380
|
+
} finally {
|
|
381
|
+
fs.rmSync(cwd, { recursive: true, force: true });
|
|
382
|
+
}
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
test('runIntegrityChecks: BLOCKED in verify-evidence blocks even with green e2e', () => {
|
|
386
|
+
const cwd = tmp('forge-int-blocked-');
|
|
387
|
+
try {
|
|
388
|
+
const sessionDir = makeSessionDir(cwd);
|
|
389
|
+
writeSpineWithRows(sessionDir);
|
|
390
|
+
writeE2eDoc(sessionDir, { notApplicable: null, steps: [greenStep()] });
|
|
391
|
+
writeE2eResults(sessionDir, runE2eSteps({ steps: [greenStep()] }));
|
|
392
|
+
fs.writeFileSync(
|
|
393
|
+
path.join(sessionDir, 'verify-evidence.md'),
|
|
394
|
+
'# Verify\n\nBLOCKED: ratify UI unreachable in CI\n',
|
|
395
|
+
'utf8',
|
|
396
|
+
);
|
|
397
|
+
const result = runIntegrityChecks({
|
|
398
|
+
cwd,
|
|
399
|
+
sessionDir,
|
|
400
|
+
session: { slug: 'wire-worker-jobs', openspecChange: null },
|
|
401
|
+
});
|
|
402
|
+
assert.equal(result.ok, false);
|
|
403
|
+
assert.match(result.problems.join('\n'), /BLOCKED/);
|
|
252
404
|
} finally {
|
|
253
405
|
fs.rmSync(cwd, { recursive: true, force: true });
|
|
254
406
|
}
|