@haposoft/cafekit 0.18.2 → 0.18.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/README.md +1 -1
  2. package/bin/__tests__/adv-skill.test.js +370 -0
  3. package/bin/__tests__/agent-authority.test.js +38 -0
  4. package/bin/__tests__/agent-instructions.test.js +54 -0
  5. package/bin/__tests__/claude-plugin-build.test.js +151 -0
  6. package/bin/__tests__/codex-hook-runtime-fixture.cjs +26 -0
  7. package/bin/__tests__/codex-hooks-ownership.test.js +8 -11
  8. package/bin/__tests__/codex-hooks.test.js +87 -85
  9. package/bin/__tests__/codex-implicit-policy.test.js +160 -0
  10. package/bin/__tests__/codex-native-hook-boundaries.test.js +2 -4
  11. package/bin/__tests__/codex-native.test.js +136 -2298
  12. package/bin/__tests__/codex-plugin-build.test.js +230 -0
  13. package/bin/__tests__/codex-plugin-context.test.js +135 -0
  14. package/bin/__tests__/codex-plugin-hooks.test.js +220 -0
  15. package/bin/__tests__/codex-quiet-hooks.test.js +2 -8
  16. package/bin/__tests__/codex-runtime-layout.test.js +81 -0
  17. package/bin/__tests__/codex-setup-agents.test.js +216 -0
  18. package/bin/__tests__/codex-skill-projections.test.js +101 -0
  19. package/bin/__tests__/develop-contract.test.js +100 -874
  20. package/bin/__tests__/legacy-hooks-retired.test.js +63 -13
  21. package/bin/__tests__/migrate-apply.test.js +373 -0
  22. package/bin/__tests__/migrate-cli.test.js +152 -0
  23. package/bin/__tests__/migrate-merged.test.js +269 -0
  24. package/bin/__tests__/migrate-plan.test.js +271 -0
  25. package/bin/__tests__/omp-bridge.test.js +81 -5
  26. package/bin/__tests__/omp-hooks.test.js +24 -0
  27. package/bin/__tests__/orca-session.test.js +2 -16
  28. package/bin/__tests__/package-inventory.test.js +228 -2044
  29. package/bin/__tests__/plugin-safety-keys.test.js +121 -0
  30. package/bin/__tests__/privacy-paths.test.js +184 -0
  31. package/bin/__tests__/secret-scanner.test.js +73 -0
  32. package/bin/__tests__/skill-routing-source.test.js +6 -0
  33. package/bin/__tests__/toaa-skill.test.js +195 -0
  34. package/bin/__tests__/usage-hook-retired.test.js +24 -6
  35. package/bin/__tests__/validator-grounding.test.js +0 -198
  36. package/bin/__tests__/vibe-run.test.js +215 -0
  37. package/bin/__tests__/vibe-skill.test.js +548 -0
  38. package/bin/__tests__/website-docs.test.js +58 -0
  39. package/bin/install.js +99 -1
  40. package/bin/lib/backup.js +43 -24
  41. package/bin/lib/codex-install.js +12 -3
  42. package/bin/lib/context.js +5 -0
  43. package/bin/lib/lock.js +98 -18
  44. package/bin/lib/managed-writer.js +16 -10
  45. package/bin/lib/migrate-apply.js +202 -0
  46. package/bin/lib/migrate-merged.js +342 -0
  47. package/bin/lib/migrate-plan.js +419 -0
  48. package/bin/lib/plugin-inventory.js +33 -0
  49. package/bin/phases/claude-runtime.js +5 -3
  50. package/bin/phases/codex-hooks.js +38 -0
  51. package/bin/phases/codex-runtime.js +106 -74
  52. package/bin/phases/copy-payload.js +49 -1
  53. package/bin/phases/omp-runtime.js +4 -26
  54. package/bin/phases/root-config.js +5 -2
  55. package/package.json +1 -1
  56. package/src/claude/CLAUDE.md +1 -1
  57. package/src/claude/agents/advisor.md +55 -0
  58. package/src/claude/agents/debugger.md +2 -3
  59. package/src/claude/agents/deployer.md +2 -1
  60. package/src/claude/agents/docs-keeper.md +12 -0
  61. package/src/claude/agents/inspector.md +1 -1
  62. package/src/claude/agents/project-manager.md +2 -2
  63. package/src/claude/agents/researcher.md +0 -1
  64. package/src/claude/agents/spec-maker.md +4 -1
  65. package/src/claude/hooks/__tests__/plugin-context.test.js +96 -0
  66. package/src/claude/hooks/__tests__/plugin-options.test.js +141 -0
  67. package/src/claude/hooks/__tests__/plugin-state-isolation.test.js +125 -0
  68. package/src/claude/hooks/__tests__/precompact.test.js +7 -1
  69. package/src/claude/hooks/__tests__/privacy-block.test.js +33 -35
  70. package/src/claude/hooks/__tests__/setup-statusline.test.js +179 -0
  71. package/src/claude/hooks/__tests__/spec-gate.test.js +86 -48
  72. package/src/claude/hooks/__tests__/state.test.js +21 -0
  73. package/src/claude/hooks/agent.cjs +5 -2
  74. package/src/claude/hooks/inspect-block.cjs +3 -1
  75. package/src/claude/hooks/lib/hook-state-dir.cjs +19 -1
  76. package/src/claude/hooks/lib/plugin-options.cjs +47 -0
  77. package/src/claude/hooks/lib/privacy-command-analysis.cjs +130 -11
  78. package/src/claude/hooks/lib/privacy-paths.cjs +370 -10
  79. package/src/claude/hooks/lib/spec-touch.cjs +14 -1
  80. package/src/claude/hooks/plugin-context.cjs +80 -0
  81. package/src/claude/hooks/privacy-block.cjs +2 -1
  82. package/src/claude/hooks/rules.cjs +5 -2
  83. package/src/claude/hooks/session.cjs +4 -1
  84. package/src/claude/hooks/spec-state.cjs +5 -4
  85. package/src/claude/hooks/state.cjs +27 -1
  86. package/src/claude/migration-manifest.json +7 -1
  87. package/src/claude/plugin/commands/setup-statusline.md +12 -0
  88. package/src/claude/plugin/core.md +71 -0
  89. package/src/claude/rules/manage-docs.md +1 -2
  90. package/src/claude/rules/review-audit-self-decision.md +1 -1
  91. package/src/claude/rules/skill-workflow-routing.md +1 -1
  92. package/src/claude/rules/state-sync.md +14 -0
  93. package/src/claude/rules/workflow.md +1 -1
  94. package/src/claude/scripts/generate-skill-catalog.cjs +5 -0
  95. package/src/claude/scripts/provenance.cjs +40 -16
  96. package/src/claude/scripts/scan-staged-secrets.cjs +49 -3
  97. package/src/claude/scripts/setup-statusline.cjs +88 -0
  98. package/src/claude/skills/adv/SKILL.md +139 -0
  99. package/src/claude/skills/adv/references/optional-modes.md +41 -0
  100. package/src/claude/skills/adv/references/ultra-mode.md +29 -0
  101. package/src/claude/skills/brainstorm/SKILL.md +1 -1
  102. package/src/claude/skills/code-review/SKILL.md +4 -1
  103. package/src/claude/skills/code-review/references/pr-fix-reply.md +42 -0
  104. package/src/claude/skills/develop/SKILL.md +11 -3
  105. package/src/claude/skills/develop/references/quality-gate.md +1 -1
  106. package/src/claude/skills/fix/SKILL.md +2 -2
  107. package/src/claude/skills/git/SKILL.md +4 -2
  108. package/src/claude/skills/git/references/merge-pr.md +125 -0
  109. package/src/claude/skills/specs/SKILL.md +2 -2
  110. package/src/claude/skills/specs/references/review.md +1 -1
  111. package/src/claude/skills/specs/references/templates.md +2 -0
  112. package/src/claude/skills/sync/SKILL.md +4 -0
  113. package/src/claude/skills/toaa/SKILL.md +82 -0
  114. package/src/claude/skills/toaa/references/autonomous-orchestration.md +54 -0
  115. package/src/claude/skills/toaa/references/prerequisites-checklist.md +37 -0
  116. package/src/claude/skills/vibe/SKILL.md +69 -0
  117. package/src/claude/skills/vibe/references/pipeline-skill-integration.md +34 -0
  118. package/src/claude/skills/vibe/references/report-templates.md +84 -0
  119. package/src/claude/skills/vibe/references/subagent-orchestration.md +23 -0
  120. package/src/codex/AGENTS.md +1 -1
  121. package/src/codex/hooks/inspect-block.cjs +7 -4
  122. package/src/codex/hooks/lib/hook-context.cjs +119 -4
  123. package/src/codex/hooks/lib/plugin-options.cjs +2 -0
  124. package/src/codex/hooks/lib/privacy-state.cjs +18 -1
  125. package/src/codex/hooks/lib/state-store.cjs +31 -6
  126. package/src/codex/hooks/precompact.cjs +9 -8
  127. package/src/codex/hooks/privacy-approval.cjs +2 -2
  128. package/src/codex/hooks/privacy-block.cjs +5 -4
  129. package/src/codex/hooks/rules.cjs +12 -6
  130. package/src/codex/hooks/session.cjs +22 -10
  131. package/src/codex/plugin/core.md +76 -0
  132. package/src/codex/plugin/hooks/plugin-context.cjs +63 -0
  133. package/src/codex/plugin/skills/setup-agents/SKILL.md +17 -0
  134. package/src/codex/scripts/setup-agents.cjs +124 -0
  135. package/src/common/AGENTS.md +1 -1
  136. package/src/omp/extensions/{cafekit-bridge.mjs → cafekit-bridge.js} +73 -5
  137. package/src/omp/hooks/privacy-block.cjs +2 -1
  138. package/bin/__tests__/codex-research-projection.test.js +0 -70
  139. package/bin/__tests__/codex-specs-projection.test.js +0 -76
  140. package/bin/__tests__/specs-v2-execution-closeout.test.js +0 -58
  141. package/src/claude/skills/chrome-devtools/scripts/__tests__/error-handling.test.js +0 -102
@@ -36,13 +36,12 @@ const PROCESS_FIRST_AGENTS = [
36
36
  'deployer.md',
37
37
  ].map((name) => path.join(PACKAGE_ROOT, 'src/claude/agents', name));
38
38
  const PROVENANCE = path.join(PACKAGE_ROOT, '../../docs/provenance.md');
39
- const SPECS_USAGE_GUIDE = path.join(PACKAGE_ROOT, '../../docs/specs-usage-guide.md');
40
39
  const RUNTIME_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-policy-runtime-'));
41
- const RUNTIME_SPEC = path.join(RUNTIME_ROOT, 'specs', 'demo', 'spec.json');
40
+ const RUNTIME_SPEC = path.join(RUNTIME_ROOT, 'specs', 'demo', 'plan.md');
42
41
  fs.mkdirSync(path.dirname(RUNTIME_SPEC), { recursive: true });
43
42
  fs.mkdirSync(path.join(RUNTIME_ROOT, 'src'), { recursive: true });
44
43
  fs.writeFileSync(path.join(RUNTIME_ROOT, 'src', 'app.js'), 'runtime fixture\n');
45
- fs.writeFileSync(RUNTIME_SPEC, JSON.stringify({ status: 'in_progress', feature_name: 'demo', task_registry: {} }));
44
+ fs.writeFileSync(RUNTIME_SPEC, '# Demo\nSpecs-Contract: process-first-ready-v1\n');
46
45
  for (const args of [['init', '-q'], ['config', 'user.email', 'cafekit@example.invalid'], ['config', 'user.name', 'CafeKit Test'], ['add', '-A'], ['commit', '-qm', 'fixture']]) {
47
46
  const result = spawnSync('git', ['-C', RUNTIME_ROOT, ...args], { encoding: 'utf8' });
48
47
  assert.equal(result.status, 0, result.stderr);
@@ -94,37 +93,7 @@ function installClaude(root) {
94
93
  });
95
94
  }
96
95
 
97
- function markdownLegacyRegions(content) {
98
- const headings = [];
99
- const primaryLines = [];
100
- const legacyLines = [];
101
- const hierarchy = [];
102
-
103
- for (const [index, line] of content.split('\n').entries()) {
104
- const heading = line.match(/^(#{1,6})\s+(.+?)\s*$/);
105
- if (heading) {
106
- const level = heading[1].length;
107
- while (hierarchy.at(-1)?.level >= level) hierarchy.pop();
108
- const entry = { level, text: heading[2], line: index + 1 };
109
- hierarchy.push(entry);
110
- headings.push(entry);
111
- }
112
-
113
- const target = hierarchy.some(({ text }) => /legacy/i.test(text))
114
- ? legacyLines
115
- : primaryLines;
116
- target.push({ line: index + 1, text: line });
117
- }
118
-
119
- return {
120
- headings,
121
- primary: primaryLines.map(({ text }) => text).join('\n'),
122
- legacy: legacyLines.map(({ text }) => text).join('\n'),
123
- primaryLines,
124
- };
125
- }
126
-
127
- function assertVocabularyIsLegacyOnly(filePath, regions) {
96
+ function assertNoLegacyVocabulary(filePath, content) {
128
97
  const vocabulary = [
129
98
  ['v2.1', /\bv2\.1\b/i],
130
99
  ['spec.json', /\bspec\.json\b/i],
@@ -140,16 +109,12 @@ function assertVocabularyIsLegacyOnly(filePath, regions) {
140
109
  ['lane', /\blane\b/i],
141
110
  ];
142
111
  const violations = [];
143
- for (const { line, text } of regions.primaryLines) {
112
+ for (const [index, text] of content.split('\n').entries()) {
144
113
  for (const [term, pattern] of vocabulary) {
145
- if (pattern.test(text)) violations.push(`${line}:${term}`);
114
+ if (pattern.test(text)) violations.push(`${index + 1}:${term}`);
146
115
  }
147
116
  }
148
- assert.deepEqual(
149
- violations,
150
- [],
151
- `${path.relative(PACKAGE_ROOT, filePath)} has v2.1 vocabulary outside a Legacy heading`,
152
- );
117
+ assert.deepEqual(violations, [], `${path.relative(PACKAGE_ROOT, filePath)} teaches a retired packet vocabulary`);
153
118
  }
154
119
 
155
120
  function canonicalReceipt(command = 'pnpm test', extra = '') {
@@ -160,16 +125,8 @@ function read(filePath) {
160
125
  return fs.readFileSync(filePath, 'utf8');
161
126
  }
162
127
 
163
- function manifestAgents() {
164
- const manifest = JSON.parse(read(path.join(PACKAGE_ROOT, 'src/claude/migration-manifest.json')));
165
- return manifest.agents.required.map((fileName) => path.basename(fileName, path.extname(fileName)));
166
- }
167
-
168
128
  test('CLI rejects flash and parallel before any mutation', () => {
169
129
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-policy-cli-'));
170
- const statePath = path.join(root, 'state.json');
171
- fs.writeFileSync(statePath, '{"status":"in_progress"}\n');
172
- const before = fs.readFileSync(statePath, 'utf8');
173
130
  try {
174
131
  const result = spawnSync(process.execPath, [POLICY_PATH, '--flash', '--parallel', '--json'], {
175
132
  cwd: root,
@@ -180,7 +137,6 @@ test('CLI rejects flash and parallel before any mutation', () => {
180
137
  assert.equal(payload.ok, false);
181
138
  assert.equal(payload.failFast, true);
182
139
  assert.match(payload.message, /incompatible/);
183
- assert.equal(fs.readFileSync(statePath, 'utf8'), before);
184
140
  } finally {
185
141
  fs.rmSync(root, { recursive: true, force: true });
186
142
  }
@@ -190,12 +146,9 @@ test('develop flags retain their process-v3 contracts', () => {
190
146
  const develop = read(DEVELOP);
191
147
  assert.match(develop, /node \.claude\/scripts\/workflow-policy\.cjs --flash --parallel --json/);
192
148
  assert.match(develop, /exits `2` before a task edit, receipt, worktree, subagent, or commit/i);
193
- assert.match(develop, /--notes` is opt-in/i);
194
- assert.match(develop, /implementation-notes-template\.html/);
195
- assert.match(develop, /Never create it by default/i);
196
- assert.match(develop, /--parallel \[N\][\s\S]*parallel-waves\.md[\s\S]*isolated worktrees[\s\S]*one controller writer/i);
197
- assert.match(develop, /--flash[\s\S]*Status: in_progress[\s\S]*FLASH_UNVERIFIED[\s\S]*awaiting \/cf:test <feature>/i);
198
- assert.match(develop, /FLASH_UNVERIFIED[\s\S]*do not unblock dependents[\s\S]*sync-finalize path/i);
149
+ assert.match(develop, /--parallel \[N\][\s\S]*parallel-waves\.md[\s\S]*one controller writer/i);
150
+ assert.match(develop, /--flash[\s\S]*Status: in_progress[\s\S]*FLASH_UNVERIFIED/);
151
+ assert.match(develop, /FLASH_UNVERIFIED[\s\S]*do not unblock dependents/i);
199
152
  });
200
153
 
201
154
  test('review and sync skills keep the verdict vocabulary and the sync-finalize path', () => {
@@ -210,32 +163,22 @@ test('review and sync skills keep the verdict vocabulary and the sync-finalize p
210
163
 
211
164
  test('Develop names the process-first packet as its only authority', () => {
212
165
  const develop = read(DEVELOP);
213
- const regions = markdownLegacyRegions(develop);
214
- assert.doesNotMatch(develop, /DO NOT write implementation code until an approved spec exists/i);
215
- assert.doesNotMatch(regions.primary, /planning_depth|assurance_level|execution_tier|\blane\b/i);
216
- assert.doesNotMatch(develop, /## Legacy workflow compatibility|Also supports existing legacy Specs packets/);
217
- assert.match(regions.primary, /specs\/<feature>\/plan\.md[\s\S]*task-NN-\*\.md/i);
218
- assert.match(regions.primary, /one unblocked task at a time/i);
219
- assert.match(regions.primary, /Verification Plan/);
220
- assert.match(regions.primary, /inline Receipt/i);
221
- assert.doesNotMatch(develop, /D2\[Step 3/);
222
- assert.match(develop, /--notes.*opt-in/i);
223
- assert.doesNotMatch(develop, /--no-notes/);
224
- assert.doesNotMatch(develop, /planner|implementer|test-runner|code-auditor|docs-keeper/i);
166
+ assertNoLegacyVocabulary(DEVELOP, develop);
167
+ assert.doesNotMatch(develop, /Legacy workflow compatibility|existing legacy Specs packets/i);
168
+ assert.match(develop, /specs\/<feature>\/plan\.md[\s\S]*task-NN-\*\.md/i);
169
+ assert.match(develop, /one unblocked task at a time/i);
170
+ assert.match(develop, /Verification Plan/);
171
+ assert.match(develop, /inline Receipt/i);
225
172
  });
226
173
 
227
- test('Specs primary output is a flat process-first packet with isolated legacy compatibility', () => {
174
+ test('Specs primary output is a flat process-first packet with no legacy packet vocabulary', () => {
228
175
  const specs = read(SPECS);
229
- const regions = markdownLegacyRegions(specs);
230
- const legacyHeadings = regions.headings.filter(({ text }) => /legacy/i.test(text));
231
- assert.equal(legacyHeadings.length, 0);
232
- assert.match(regions.primary, /specs\/<feature>\/[\s\S]*plan\.md[\s\S]*task-01-<slug>\.md[\s\S]*task-02-<slug>\.md/i);
233
- assert.match(regions.primary, /Task files are flat beside `plan\.md`/i);
234
- assert.match(regions.primary, /one task at a time/i);
235
- assert.match(regions.primary, /inline `## Receipt`/i);
236
- assert.match(regions.primary, /GATE-SCOPE[\s\S]*GATE-REVIEW[\s\S]*GATE-DONE/i);
237
- assert.doesNotMatch(specs, /installed legacy adapters?/i);
238
- assertVocabularyIsLegacyOnly(SPECS, regions);
176
+ assertNoLegacyVocabulary(SPECS, specs);
177
+ assert.doesNotMatch(specs, /^#{1,6}\s.*legacy/im);
178
+ assert.match(specs, /specs\/<feature>\/[\s\S]*plan\.md[\s\S]*task-01-<slug>\.md[\s\S]*task-02-<slug>\.md/i);
179
+ assert.match(specs, /Task files are flat beside `plan\.md`/i);
180
+ assert.match(specs, /inline `## Receipt`/i);
181
+ assert.match(specs, /GATE-SCOPE[\s\S]*GATE-REVIEW[\s\S]*GATE-DONE/i);
239
182
  });
240
183
 
241
184
  test('core execution agents default to process-first state and carry no legacy adapter', () => {
@@ -248,75 +191,66 @@ test('core execution agents default to process-first state and carry no legacy a
248
191
  }
249
192
 
250
193
  const implementer = read(PROCESS_FIRST_AGENTS[0]);
251
- assert.match(implementer, /Under every dispatch mode[\s\S]{0,160}do NOT edit[\s\S]{0,80}`plan\.md`[\s\S]{0,80}`Status:`[\s\S]{0,80}`## Receipt`/i);
194
+ assert.match(implementer, /do NOT edit[\s\S]{0,80}`plan\.md`[\s\S]{0,80}`Status:`[\s\S]{0,80}`## Receipt`/i);
252
195
  assert.match(implementer, /controller/i);
253
- assert.match(implementer, /process-first work[\s\S]{0,100}Ownership\s+boundary\./i);
254
-
255
- const auditor = read(PROCESS_FIRST_AGENTS[2]);
256
- assert.match(auditor, /accepted GATE-SCOPE\/GATE-REVIEW decisions/i);
257
- assert.doesNotMatch(auditor, /CAFEKIT_SEMANTIC_REVIEW_ATTESTATION/);
258
-
259
- const docsKeeper = read(PROCESS_FIRST_AGENTS[3]);
260
- assert.match(docsKeeper, /never[\s\S]{0,80}invent or write Status, Receipt, approval, or execution proof/i);
261
-
196
+ assert.match(read(PROCESS_FIRST_AGENTS[2]), /GATE-SCOPE\/GATE-REVIEW/);
197
+ assert.match(read(PROCESS_FIRST_AGENTS[3]), /never[\s\S]{0,80}Status, Receipt/i);
262
198
  const deployer = read(PROCESS_FIRST_AGENTS[4]);
263
- assert.match(deployer, /current final inline Receipt/i);
264
- assert.match(deployer, /GATE-DONE\/release authorization/i);
199
+ assert.match(deployer, /GATE-DONE/);
265
200
  assert.match(deployer, /does not write process-first Status\/Receipt/i);
266
201
  });
267
202
 
268
- test('R7 Develop and Sync surfaces teach process-v3 and isolate hierarchical Legacy sections', () => {
269
- const surfaces = [
270
- ['develop', DEVELOP, 0],
271
- ['parallel waves', PARALLEL_WAVES, 0],
272
- ['sync', SYNC_SKILL, 0],
273
- ['sync protocols', SYNC_PROTOCOLS, 0],
274
- ];
275
- const regionsByName = new Map();
276
-
277
- for (const [name, filePath, expectedLegacy] of surfaces) {
278
- const regions = markdownLegacyRegions(read(filePath));
279
- const legacyHeadings = regions.headings.filter(({ text }) => /legacy/i.test(text));
280
- assert.equal(legacyHeadings.length, expectedLegacy, `${name} must have exactly ${expectedLegacy} Legacy heading(s)`);
281
- assert.match(regions.primary, /inline (?:`## )?Receipts?/i, `${name} must teach inline Receipts`);
282
- if (expectedLegacy === 1) {
283
- assert.match(regions.legacy, /spec\.json/i, `${name} must retain the spec.json adapter`);
284
- assert.match(regions.legacy, /task_registry/i, `${name} must retain task_registry compatibility`);
285
- }
286
- assertVocabularyIsLegacyOnly(filePath, regions);
287
- regionsByName.set(name, regions);
203
+ test('Develop and Sync surfaces teach the flat process-first packet and no legacy vocabulary', () => {
204
+ for (const [name, filePath] of [
205
+ ['develop', DEVELOP], ['parallel waves', PARALLEL_WAVES], ['sync', SYNC_SKILL], ['sync protocols', SYNC_PROTOCOLS],
206
+ ]) {
207
+ const content = read(filePath);
208
+ assert.doesNotMatch(content, /^#{1,6}\s.*legacy/im, `${name} must have no Legacy section`);
209
+ assert.match(content, /inline (?:`## )?Receipts?/i, `${name} must teach inline Receipts`);
210
+ assertNoLegacyVocabulary(filePath, content);
288
211
  }
212
+ assert.match(read(DEVELOP), /`specs\/<feature>\/plan\.md` plus flat `task-NN-\*\.md`/i);
213
+ assert.match(read(SYNC_SKILL), /<task-NN-slug\.md>/i);
214
+ assert.match(read(SYNC_SKILL), /sync-finalize/i);
215
+ assert.match(read(PARALLEL_WAVES), /one file has one writer per wave/i);
216
+ assert.match(read(PARALLEL_WAVES), /writes inline Receipts and Status one task at a time/i);
217
+ assert.match(read(SYNC_PROTOCOLS), /`plan\.md` and flat `task-\*\.md` files/i);
218
+ });
289
219
 
290
- const develop = regionsByName.get('develop').primary;
291
- assert.match(develop, /`specs\/<feature>\/plan\.md` plus flat `task-NN-\*\.md`/i);
292
- assert.match(develop, /one unblocked task at a time/i);
293
- assert.match(develop, /Outcome and Acceptance/i);
294
- assert.match(develop, /Verification Plan/);
295
- assert.match(develop, /final inline `## Receipt`/i);
296
-
297
- const sync = regionsByName.get('sync').primary;
298
- assert.match(sync, /<task-NN-slug\.md>/i);
299
- assert.match(sync, /specs\/<feature>\/task-\*\.md` beside `plan\.md`/i);
300
- assert.match(sync, /Acceptance[\s\S]*Verification Plan/i);
301
- assert.match(sync, /current inline Receipt/i);
302
-
303
- const waves = regionsByName.get('parallel waves').primary;
304
- assert.match(waves, /dependencies from the plan table and each task/i);
305
- assert.match(waves, /acceptance, proof command/i);
306
- assert.match(waves, /one file has one writer per wave/i);
307
- assert.match(waves, /writes inline Receipts and Status one task at a time/i);
308
-
309
- const protocols = regionsByName.get('sync protocols').primary;
310
- assert.match(protocols, /`plan\.md` and flat `task-\*\.md` files/i);
311
- assert.match(protocols, /acceptance IDs/i);
312
- assert.match(protocols, /current inline Receipt/i);
313
- assert.match(regionsByName.get('sync').primary, /sync-finalize/i);
220
+ test('develop and sync carry symptom-first gotchas', () => {
221
+ // The section runs from its heading to the next level-two heading or the end of the file.
222
+ const gotchas = (filePath) => {
223
+ const content = read(filePath);
224
+ const name = path.basename(path.dirname(filePath));
225
+ assert.equal((content.match(/^## Gotchas$/gm) || []).length, 1, `${name} must have exactly one Gotchas section`);
226
+ const body = content.slice(content.indexOf('\n## Gotchas\n') + 1);
227
+ const next = body.indexOf('\n## ', 1);
228
+ return next === -1 ? body : body.slice(0, next);
229
+ };
230
+
231
+ const develop = gotchas(DEVELOP);
232
+ assert.match(develop, /command_identity/);
233
+ assert.match(develop, /last top-level `- Command:`/);
234
+ assert.match(develop, /\bprovenance\b/);
235
+ assert.match(develop, /outside the specs root/);
236
+ assert.match(develop, /specs\/_shared\/active-feature\.json/);
237
+ assert.match(develop, /explicit_not_found/);
238
+
239
+ const sync = gotchas(SYNC_SKILL);
240
+ assert.match(sync, /committed copy/);
241
+ assert.match(sync, /\/cf:sync rebind/);
242
+
243
+ // A bulleted Receipt is accepted by the parser, so that trap must not come back.
244
+ for (const [name, section] of [['develop', develop], ['sync', sync]]) {
245
+ assert.doesNotMatch(section, /list marker|bullet/i, `${name} gotchas must not list the fixed bulleted-Receipt trap`);
246
+ assert.doesNotMatch(section, /\b(MUST|NEVER)\b/, `${name} gotchas must not shout`);
247
+ }
314
248
  });
315
249
 
316
250
  test('Develop process-first source contract preserves selection, recovery, final-Head, parallel, and Flash boundaries', () => {
317
- const develop = markdownLegacyRegions(read(DEVELOP)).primary;
251
+ const develop = read(DEVELOP);
318
252
  const quality = read(GATE);
319
- const waves = markdownLegacyRegions(read(PARALLEL_WAVES)).primary;
253
+ const waves = read(PARALLEL_WAVES);
320
254
  const dispatch = read(path.join(path.dirname(PARALLEL_WAVES), 'subagent-patterns.md'));
321
255
 
322
256
  for (const clause of [
@@ -325,13 +259,10 @@ test('Develop process-first source contract preserves selection, recovery, final
325
259
  /More than one `in_progress`/i,
326
260
  /For an interrupted `in_progress` task/i,
327
261
  /only the controller writes Status or Receipt/i,
328
- /do not research, replan, or add a routine user gate/i,
329
262
  ]) assert.match(develop, clause);
330
263
  assert.match(quality, /consecutive Head captures are identical/i);
331
264
  assert.match(quality, /skip, skipped, todo, cancel, canceled, cancelled/i);
332
- assert.match(quality, /exact command, current Head, required proof level, oracle/i);
333
265
  assert.match(waves, /every commit[\s\S]*complete changed tree/i);
334
- assert.match(waves, /handoff metadata cannot substitute/i);
335
266
  assert.match(develop, /later explicit non-Flash invocation may recover/i);
336
267
  assert.match(dispatch, /controller alone writes Status and inline Receipt/i);
337
268
 
@@ -398,34 +329,11 @@ test('Develop process-first source contract preserves selection, recovery, final
398
329
  }
399
330
  });
400
331
 
401
- test('Claude installed Develop preserves plan-native execution and references', () => {
402
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-claude-develop-'));
403
- try {
404
- const result = installClaude(root);
405
- assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
406
- const sourceRoot = path.dirname(DEVELOP);
407
- const installedRoot = path.join(root, '.claude/skills/develop');
408
- for (const relative of ['SKILL.md', ...DEVELOP_REFERENCES.map((name) => `references/${name}`)]) {
409
- assert.equal(
410
- fs.readFileSync(path.join(installedRoot, relative), 'utf8'),
411
- fs.readFileSync(path.join(sourceRoot, relative), 'utf8'),
412
- `Claude Develop projection drifted: ${relative}`,
413
- );
414
- }
415
- assert.equal(fs.existsSync(path.join(root, '.agents/skills/develop')), false);
416
- } finally {
417
- fs.rmSync(root, { recursive: true, force: true });
418
- }
419
- });
420
-
421
332
  test('Test process-first handoff preserves proof ownership and side-effect boundaries', () => {
422
333
  const skill = read(TEST_SKILL);
423
334
  const strategy = read(path.join(TEST_SOURCE_ROOT, 'references/execution-strategy.md'));
424
335
  const runner = read(TEST_RUNNER);
425
336
  const review = read(CODE_REVIEW_SKILL);
426
- const developConsumers = [read(DEVELOP), read(GATE), read(path.join(
427
- path.dirname(PARALLEL_WAVES), 'subagent-patterns.md',
428
- ))].join('\n');
429
337
 
430
338
  const exactKeys = [
431
339
  'schema_version', 'target', 'verdict', 'command', 'exit', 'counts',
@@ -433,24 +341,9 @@ test('Test process-first handoff preserves proof ownership and side-effect bound
433
341
  'artifacts', 'branches', 'raw_output', 'redactions', 'payload_sha256',
434
342
  ];
435
343
  for (const key of exactKeys) assert.match(strategy, new RegExp(`\\b${key}\\b`));
436
- const developHandoffFields = new Map([
437
- ['command', /Verification handoff separately contains the fresh command/i],
438
- ['exit', /fresh command, exit/i],
439
- ['branches', /named probes/i],
440
- ['counts', /counts, output/i],
441
- ['raw_output', /counts, output/i],
442
- ['provenance', /observed Head/i],
443
- ['reachability', /runtime reachability/i],
444
- ['proof_level', /required proof level/i],
445
- ]);
446
- for (const [field, sourceClause] of developHandoffFields) {
447
- assert.match(developConsumers, sourceClause, `Develop must currently consume ${field}`);
448
- assert.ok(exactKeys.includes(field), `test-proof-v1 must preserve Develop field ${field}`);
449
- }
450
344
  assert.match(runner, /Emit only `schema_version: "test-proof-v1"`/);
451
345
  assert.match(runner, /sole writer of[\s\S]*Status and inline Receipt/i);
452
346
  assert.match(review, /consume only the controller-validated[\s\S]*`test-proof-v1`[\s\S]*handoff/i);
453
- assert.match(review, /Never create or search for a separate[\s\S]*process-first receipt/i);
454
347
  assert.doesNotMatch([skill, strategy, runner].join('\n'), /npm\s+install|inject-auth\.js|--cookies\s|Bearer eyJ/);
455
348
 
456
349
  const stable = (value) => {
@@ -534,18 +427,24 @@ test('Test process-first handoff preserves proof ownership and side-effect bound
534
427
  assert.equal(fs.existsSync(externalTemp), false, 'Test-owned external temp must be cleaned');
535
428
  });
536
429
 
537
- test('Claude installed Test preserves plan-native proof and references', () => {
538
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-claude-test-proof-'));
430
+ test('Claude installed Develop and Test preserve plan-native execution, proof and references', () => {
431
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-claude-develop-test-'));
539
432
  try {
540
433
  const result = installClaude(root);
541
434
  assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
542
- const installedRoot = path.join(root, '.claude/skills/test');
543
- for (const relative of ['SKILL.md', ...TEST_REFERENCES.map((name) => `references/${name}`)]) {
544
- assert.equal(
545
- fs.readFileSync(path.join(installedRoot, relative), 'utf8'),
546
- fs.readFileSync(path.join(TEST_SOURCE_ROOT, relative), 'utf8'),
547
- `Claude Test projection drifted: ${relative}`,
548
- );
435
+ const projections = [
436
+ ['develop', path.dirname(DEVELOP), DEVELOP_REFERENCES],
437
+ ['test', TEST_SOURCE_ROOT, TEST_REFERENCES],
438
+ ];
439
+ for (const [name, sourceRoot, references] of projections) {
440
+ for (const relative of ['SKILL.md', ...references.map((ref) => `references/${ref}`)]) {
441
+ assert.equal(
442
+ fs.readFileSync(path.join(root, '.claude/skills', name, relative), 'utf8'),
443
+ fs.readFileSync(path.join(sourceRoot, relative), 'utf8'),
444
+ `Claude ${name} projection drifted: ${relative}`,
445
+ );
446
+ }
447
+ assert.equal(fs.existsSync(path.join(root, '.agents/skills', name)), false);
549
448
  }
550
449
  const installedRunner = fs.readFileSync(path.join(root, '.claude/agents/test-runner.md'), 'utf8');
551
450
  assert.equal(installedRunner, fs.readFileSync(TEST_RUNNER, 'utf8'));
@@ -553,27 +452,11 @@ test('Claude installed Test preserves plan-native proof and references', () => {
553
452
  const installedReview = fs.readFileSync(path.join(root, '.claude/skills/code-review/SKILL.md'), 'utf8');
554
453
  assert.equal(installedReview, fs.readFileSync(CODE_REVIEW_SKILL, 'utf8'));
555
454
  assert.match(installedReview, /controller-validated `test-proof-v1`/);
556
- assert.match(installedReview, /For proof consumption, this skill's `## Execution-proof boundary` is\s+authoritative/i);
557
- assert.equal(fs.existsSync(path.join(root, '.agents/skills/test')), false);
558
455
  } finally {
559
456
  fs.rmSync(root, { recursive: true, force: true });
560
457
  }
561
458
  });
562
459
 
563
- test('specs-usage-guide documents Test proof handoff without timing claims', () => {
564
- const guide = read(SPECS_USAGE_GUIDE);
565
- const section = guide.split('## Test proof handoff')[1]?.split('### Receipt canonical')[0] || '';
566
- assert.match(section, /`test-proof-v1`/);
567
- assert.match(section, /Test không ghi `Status:` hay inline[\s\S]*Develop controller là writer duy nhất/i);
568
- assert.match(section, /tracked\/untracked\/ignored drift/i);
569
- assert.match(section, /HTTPS\/localhost origin[\s\S]*fresh consent/i);
570
- assert.match(section, /chỉ nằm trong `## Receipt` inline/i);
571
- assert.match(section, /không phải benchmark thời gian/i);
572
- assert.match(section, /không chứng minh live adherence/i);
573
- assert.doesNotMatch(section, /\b\d+(?:\.\d+)?\s*(?:ms|milliseconds?|seconds?|minutes?)\b/i);
574
- assert.doesNotMatch(section, /\[(?:LIVE_)?VERIFIED\]/i);
575
- });
576
-
577
460
  test('first-run cache bypass is blocked - canonical receipt required even without cache', () => {
578
461
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-first-run-'));
579
462
  const claude = path.join(root, '.claude');
@@ -696,13 +579,13 @@ test('runtime provenance derives exact Git evidence, CLI context, and stale/forg
696
579
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-provenance-adversarial-'));
697
580
  const specsRoot = path.join(root, 'specs');
698
581
  const featureRoot = path.join(specsRoot, 'demo');
699
- const specFile = path.join(featureRoot, 'spec.json');
582
+ const specFile = path.join(featureRoot, 'plan.md');
700
583
  const sourceFile = path.join(root, 'src', 'app.js');
701
584
  const renameSource = path.join(root, 'src', 'rename-old.js');
702
585
  const renameTarget = path.join(root, 'src', 'rename-new.js');
703
586
  fs.mkdirSync(path.dirname(sourceFile), { recursive: true });
704
587
  fs.mkdirSync(featureRoot, { recursive: true });
705
- fs.writeFileSync(specFile, JSON.stringify({ status: 'in_progress', feature_name: 'demo', task_registry: {} }));
588
+ fs.writeFileSync(specFile, '# Demo\n');
706
589
  fs.writeFileSync(sourceFile, 'original source\n');
707
590
  fs.writeFileSync(renameSource, 'rename source\n');
708
591
  fs.writeFileSync(path.join(root, '.gitignore'), 'ignored-secret.env\nnode_modules/\n.cache/\n');
@@ -734,13 +617,8 @@ test('runtime provenance derives exact Git evidence, CLI context, and stale/forg
734
617
  'valid-shaped arbitrary SHA values must not bind',
735
618
  );
736
619
 
737
- fs.writeFileSync(specFile, JSON.stringify({
738
- status: 'in_progress',
739
- feature_name: 'demo',
740
- task_registry: {},
741
- receipt_refresh: true,
742
- }));
743
- const specCommit = spawnSync('git', ['-C', root, 'add', 'specs/demo/spec.json'], { encoding: 'utf8' });
620
+ fs.writeFileSync(specFile, '# Demo\n\nrefreshed\n');
621
+ const specCommit = spawnSync('git', ['-C', root, 'add', 'specs/demo/plan.md'], { encoding: 'utf8' });
744
622
  assert.equal(specCommit.status, 0, specCommit.stderr);
745
623
  const commitReceipt = spawnSync('git', ['-C', root, 'commit', '-qm', 'refresh spec receipt'], { encoding: 'utf8' });
746
624
  assert.equal(commitReceipt.status, 0, commitReceipt.stderr);
@@ -826,9 +704,9 @@ test('runtime provenance derives exact Git evidence, CLI context, and stale/forg
826
704
  }
827
705
 
828
706
  const nonGit = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-provenance-non-git-'));
829
- const nonGitSpec = path.join(nonGit, 'specs', 'demo', 'spec.json');
707
+ const nonGitSpec = path.join(nonGit, 'specs', 'demo', 'plan.md');
830
708
  fs.mkdirSync(path.dirname(nonGitSpec), { recursive: true });
831
- fs.writeFileSync(nonGitSpec, JSON.stringify({ status: 'in_progress', feature_name: 'demo' }));
709
+ fs.writeFileSync(nonGitSpec, '# Demo\n');
832
710
  try {
833
711
  const blocked = spawnSync(process.execPath, [
834
712
  path.join(PACKAGE_ROOT, 'src/claude/scripts/provenance.cjs'), '--json',
@@ -846,15 +724,15 @@ test('runtime provenance chooses a deterministic root when all history is Specs-
846
724
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-provenance-multi-root-'));
847
725
  const specsRoot = path.join(root, 'specs');
848
726
  const featureRoot = path.join(specsRoot, 'demo');
849
- const specFile = path.join(featureRoot, 'spec.json');
727
+ const specFile = path.join(featureRoot, 'plan.md');
850
728
  try {
851
729
  fs.mkdirSync(featureRoot, { recursive: true });
852
- fs.writeFileSync(specFile, JSON.stringify({ status: 'in_progress', feature_name: 'demo', task_registry: {} }));
730
+ fs.writeFileSync(specFile, '# Demo\n');
853
731
  for (const args of [
854
732
  ['init', '-q', '-b', 'first-root'],
855
733
  ['config', 'user.email', 'cafekit@example.invalid'],
856
734
  ['config', 'user.name', 'CafeKit Test'],
857
- ['add', 'specs/demo/spec.json'],
735
+ ['add', 'specs/demo/plan.md'],
858
736
  ['commit', '-qm', 'first specs root'],
859
737
  ['checkout', '--orphan', 'unrelated-specs'],
860
738
  ['rm', '-qrf', '.'],
@@ -862,10 +740,10 @@ test('runtime provenance chooses a deterministic root when all history is Specs-
862
740
  const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' });
863
741
  assert.equal(result.status, 0, result.stderr);
864
742
  }
865
- const unrelatedSpec = path.join(specsRoot, 'unrelated', 'spec.json');
743
+ const unrelatedSpec = path.join(specsRoot, 'unrelated', 'plan.md');
866
744
  fs.mkdirSync(path.dirname(unrelatedSpec), { recursive: true });
867
- fs.writeFileSync(unrelatedSpec, JSON.stringify({ status: 'in_progress', feature_name: 'unrelated', task_registry: {} }));
868
- let result = spawnSync('git', ['-C', root, 'add', 'specs/unrelated/spec.json'], { encoding: 'utf8' });
745
+ fs.writeFileSync(unrelatedSpec, '# Unrelated\n');
746
+ let result = spawnSync('git', ['-C', root, 'add', 'specs/unrelated/plan.md'], { encoding: 'utf8' });
869
747
  assert.equal(result.status, 0, result.stderr);
870
748
  result = spawnSync('git', ['-C', root, 'commit', '-qm', 'second specs root'], { encoding: 'utf8' });
871
749
  assert.equal(result.status, 0, result.stderr);
@@ -889,56 +767,26 @@ test('runtime provenance chooses a deterministic root when all history is Specs-
889
767
  });
890
768
 
891
769
  test('parallel waves require immutable provenance receipts and safe recovery', () => {
892
- const waves = markdownLegacyRegions(read(PARALLEL_WAVES)).primary;
893
- assert.match(waves, /only for explicit parallel execution/i);
894
- assert.match(waves, /isolated worktree support[\s\S]*fall back to the sequential loop/i);
770
+ const waves = read(PARALLEL_WAVES);
895
771
  assert.match(waves, /Clamp `--parallel N` to 1\.\.5; default to 3/i);
896
772
  assert.match(waves, /clean destination[\s\S]*base_sha[\s\S]*incompatible bases/i);
897
- assert.match(waves, /consent for worker commits and controller cherry-picks/i);
898
773
  for (const field of ['base_sha', 'head_sha', 'branch', 'worktree_path', 'commit_range']) {
899
774
  assert.match(waves, new RegExp(`\\b${field}\\b`));
900
775
  }
901
- assert.match(waves, /git rev-list --reverse <base_sha>\.\.<head_sha>/i);
902
- assert.match(waves, /git diff --name-status <base_sha>\.\.<head_sha>/i);
903
- assert.match(waves, /code\/spec repair[\s\S]*new commit/i);
904
- assert.match(waves, /dependencies from the plan table and each task/i);
905
- for (const sharedSurface of ['registry', 'lockfile', 'manifest', 'generated output', 'migration', 'export barrel']) {
906
- assert.match(waves, new RegExp(sharedSurface, 'i'));
907
- }
908
776
  assert.match(waves, /One file has one writer per wave/i);
909
777
  assert.match(waves, /only the controller writes state\/proof/i);
910
778
  assert.match(waves, /commit only with user authorization/i);
911
- assert.match(waves, /blocked or failed worker keeps[\s\S]*worktree and branch/i);
912
779
  assert.match(waves, /Never[\s\S]*force-delete[\s\S]*merge success or explicit discard authorization/i);
913
- assert.match(waves, /git cherry-pick <next-worker-commit>/i);
914
780
  assert.match(waves, /On conflict, abort the pick[\s\S]*retain its branch\/worktree/i);
915
781
  assert.match(waves, /cleanup_authorization: merged-release/i);
916
- assert.match(waves, /affected integration|final.*integration/i);
917
- for (const category of ['baseline', 'environment', 'spec', 'code']) {
918
- assert.match(waves, new RegExp(`\\b${category}\\b`));
919
- }
920
782
  assert.match(waves, /Do not blind-retry/i);
921
783
  });
922
784
 
923
- test('provenance ledger defines reuse contract and source anchors', () => {
785
+ test('provenance ledger forbids copied source text and the shipped runtime carries none', () => {
924
786
  assert.equal(fs.existsSync(PROVENANCE), true);
925
787
  const provenance = read(PROVENANCE);
926
- assert.match(provenance, /\bidea\b/);
927
788
  assert.match(provenance, /\bclean-room\b/);
928
- assert.match(provenance, /copied-text.*never valid|never valid.*copied-text/i);
929
789
  assert.match(provenance, /never copy source text verbatim/i);
930
- for (const column of ['Pattern', 'Source anchor', 'Reuse type', 'CafeKit destination', 'Evidence/status']) {
931
- assert.match(provenance, new RegExp(`\\b${column.replace('/', '\\/')}\\b`, 'i'));
932
- }
933
- assert.match(provenance, /AgentKit|cafekit-ref/);
934
- assert.match(provenance, /before implementation/i);
935
- // H1 remediation: distinguish survey vs verified implementation, fail unsupported claims
936
- assert.match(provenance, /external survey|not committed/i);
937
- assert.match(provenance, /survey only|not used|No borrowed text recorded/i);
938
- assert.match(provenance, /Source anchor.*plans\//);
939
- assert.match(provenance, /Evidence\/status/i);
940
- assert.match(provenance, /grep -r.*cafekit-ref/i);
941
- assert.doesNotMatch(provenance, /AgentKit T1.*implemented as direct source/i);
942
790
  // shipped runtime must not contain cafekit-ref/AgentKit verbatim (only ledger/plans may)
943
791
  const shippedHits = spawnSync('grep', ['-rn', 'cafekit-ref', 'packages/spec/src', '--include=*.md', '--include=*.cjs', '--include=*.ts'], { encoding: 'utf8' });
944
792
  const agentKitHits = spawnSync('grep', ['-rn', 'AgentKit', 'packages/spec/src', '--include=*.md', '--include=*.cjs', '--include=*.ts'], { encoding: 'utf8' });
@@ -946,628 +794,6 @@ test('provenance ledger defines reuse contract and source anchors', () => {
946
794
  assert.equal(agentKitHits.stdout.trim(), '', 'shipped runtime must not contain AgentKit verbatim');
947
795
  });
948
796
 
949
- const BENCHMARK_SCRIPT = path.join(PACKAGE_ROOT, 'scripts/benchmark-workflow.mjs');
950
- const BENCHMARK_CORPUS_SCHEMA = path.join(PACKAGE_ROOT, 'benchmarks/corpus.schema.json');
951
- const BENCHMARK_CONFIG_EXAMPLE = path.join(PACKAGE_ROOT, 'benchmarks/benchmark-config.example.json');
952
- const BENCHMARK_RUBRIC = path.join(PACKAGE_ROOT, 'benchmarks/rubric.md');
953
- const BENCHMARK_DOCS = path.join(PACKAGE_ROOT, '../../docs/benchmark-workflow.md');
954
-
955
- function canonicalBenchmark(value) {
956
- if (Array.isArray(value)) return `[${value.map(canonicalBenchmark).join(',')}]`;
957
- if (value && typeof value === 'object') return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalBenchmark(value[key])}`).join(',')}}`;
958
- return JSON.stringify(value);
959
- }
960
-
961
- function benchmarkHash(value) {
962
- const crypto = require('node:crypto');
963
- return `sha256:${crypto.createHash('sha256').update(canonicalBenchmark(value)).digest('hex')}`;
964
- }
965
-
966
- function benchmarkRawHash(value) {
967
- return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`;
968
- }
969
-
970
- function writeBenchmarkConfig(file, config) {
971
- const frozen = { ...config };
972
- delete frozen.config_sha256;
973
- frozen.config_sha256 = benchmarkHash(frozen);
974
- fs.writeFileSync(file, JSON.stringify(frozen));
975
- }
976
-
977
- function makeBenchmarkFixture(root) {
978
- const corpus = {
979
- schema_version: 'b1.v1', status: 'frozen', corpus_id: 'fixture-only-corpus',
980
- tasks: [
981
- { task_id: 'fixture-direct', lane: 'Direct', prompt: 'fixture-only reversible task', repo_sample: 'fixture-repo', acceptance: { criteria: ['fixture criterion'] }, risk: { level: 'low' } },
982
- { task_id: 'fixture-critical', lane: 'Critical', prompt: 'fixture-only negative control', repo_sample: 'fixture-repo', acceptance: { criteria: ['fixture contract'] }, risk: { level: 'high', reasons: ['fixture negative control'] } },
983
- ],
984
- };
985
- const corpusSha = benchmarkHash(corpus);
986
- const common = {
987
- schema_version: 'b1.v1', status: 'frozen', experiment_id: 'fixture-only-experiment',
988
- model: { name: 'fixture-model', version: 'fixture-version' }, reasoning_effort: 'fixture',
989
- repo: { identifier: 'fixture-repo', commit: 'abc1234567890', clean_initial_tree_sha: benchmarkHash('fixture-clean-tree') },
990
- permissions_fingerprint: benchmarkHash('fixture-permissions'), tool_availability_fingerprint: benchmarkHash('fixture-tools'),
991
- corpus_sha256: corpusSha, repeat_policy: { repeats_per_task: 2, context_isolated: true },
992
- input_usd_per_1k: 1, output_usd_per_1k: 2,
993
- };
994
- const configs = ['baseline', 'treatment'].map((arm) => {
995
- const config = { ...common, arm };
996
- config.config_sha256 = benchmarkHash(config);
997
- return config;
998
- });
999
- const artifactRef = 'fixture-artifact.bin';
1000
- const artifactBytes = Buffer.from('fixture-only benchmark artifact\n');
1001
- fs.writeFileSync(path.join(root, artifactRef), artifactBytes);
1002
- const artifactSha = benchmarkRawHash(artifactBytes);
1003
- const receipts = [];
1004
- for (const config of configs) for (const task of corpus.tasks) for (const repeat of [1, 2]) {
1005
- const treatment = config.arm === 'treatment';
1006
- const critical = task.lane === 'Critical';
1007
- const wall = critical ? 300 : (treatment ? 50 : 100) * repeat;
1008
- const input = (treatment ? 100 : 200) * repeat;
1009
- const output = (treatment ? 50 : 100) * repeat;
1010
- receipts.push({
1011
- task_id: task.task_id, lane: task.lane, arm: config.arm, repeat,
1012
- model: config.model, reasoning_effort: config.reasoning_effort,
1013
- repo_commit: config.repo.commit, clean_initial_tree_sha: config.repo.clean_initial_tree_sha,
1014
- permissions_fingerprint: config.permissions_fingerprint, tool_availability_fingerprint: config.tool_availability_fingerprint,
1015
- corpus_sha256: config.corpus_sha256, config_sha256: config.config_sha256,
1016
- wall_ms: wall, input_tokens: input, output_tokens: output, context_loaded_tokens: 500,
1017
- tool_calls: 2, subagent_calls: critical ? 1 : 0, correctness: true, regression: false,
1018
- unsupported_completion_claim: false, user_corrections: 0, useful_reviewer_findings: 1,
1019
- false_positive_reviewer_findings: 0,
1020
- evidence: { artifact_ref: artifactRef, artifact_sha256: artifactSha, command: 'fixture-only command' },
1021
- });
1022
- }
1023
- const paths = { corpus: path.join(root, 'corpus.json'), receipts: path.join(root, 'receipts.json') };
1024
- fs.writeFileSync(paths.corpus, JSON.stringify(corpus));
1025
- fs.writeFileSync(paths.receipts, JSON.stringify({ receipts }));
1026
- paths.configs = configs.map((config, index) => {
1027
- const file = path.join(root, `${config.arm}-${index}.json`);
1028
- fs.writeFileSync(file, JSON.stringify(config));
1029
- return file;
1030
- });
1031
- return paths;
1032
- }
1033
-
1034
- test('B1 benchmark artifacts expose bounded CLI contract', () => {
1035
- for (const file of [BENCHMARK_SCRIPT, BENCHMARK_CORPUS_SCHEMA, BENCHMARK_CONFIG_EXAMPLE, BENCHMARK_RUBRIC, BENCHMARK_DOCS]) assert.equal(fs.existsSync(file), true, file);
1036
- assert.match(read(BENCHMARK_DOCS), /live baseline.*treatment.*pending/i);
1037
- assert.match(read(BENCHMARK_DOCS), /`npm test`.*workflow correctness/i);
1038
- assert.match(read(BENCHMARK_CONFIG_EXAMPLE), /TEMPLATE ONLY|example.template/i);
1039
- assert.match(read(BENCHMARK_RUBRIC), /Direct/);
1040
- assert.match(read(BENCHMARK_RUBRIC), /Standard/);
1041
- assert.match(read(BENCHMARK_RUBRIC), /Critical/);
1042
- });
1043
-
1044
- test('B1 validator rejects missing freeze fields and accepts fixture-only corpus', () => {
1045
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-validation-'));
1046
- const fixture = makeBenchmarkFixture(root);
1047
- try {
1048
- assert.equal(JSON.parse(read(fixture.corpus)).status, 'frozen');
1049
- const valid = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', fixture.receipts], { encoding: 'utf8' });
1050
- assert.equal(valid.status, 0, `${valid.stdout}\n${valid.stderr}`);
1051
- assert.equal(JSON.parse(valid.stdout).status, 'valid');
1052
- const invalidConfig = JSON.parse(read(fixture.configs[0]));
1053
- delete invalidConfig.clean_initial_tree_sha;
1054
- delete invalidConfig.config_sha256;
1055
- const invalidPath = path.join(root, 'missing-freeze.json');
1056
- fs.writeFileSync(invalidPath, JSON.stringify(invalidConfig));
1057
- const invalid = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', invalidPath], { encoding: 'utf8' });
1058
- assert.equal(invalid.status, 2);
1059
- assert.match(invalid.stderr, /missing or placeholder freeze field/);
1060
- } finally {
1061
- fs.rmSync(root, { recursive: true, force: true });
1062
- }
1063
- });
1064
-
1065
- test('B1 validator fails closed for templates, prompts, parity, and artifacts', () => {
1066
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-integrity-'));
1067
- const fixture = makeBenchmarkFixture(root);
1068
- try {
1069
- const templateCorpus = JSON.parse(read(fixture.corpus));
1070
- templateCorpus.status = 'example_template';
1071
- const templatePath = path.join(root, 'template-corpus.json');
1072
- fs.writeFileSync(templatePath, JSON.stringify(templateCorpus));
1073
- const template = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', templatePath, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', fixture.receipts], { encoding: 'utf8' });
1074
- assert.equal(template.status, 2);
1075
- assert.match(template.stderr, /example_template.*receipt validation.*live summary/);
1076
-
1077
- const placeholderCorpus = JSON.parse(read(fixture.corpus));
1078
- placeholderCorpus.tasks[0].prompt = '{{replace_me}}';
1079
- const placeholderPath = path.join(root, 'placeholder-corpus.json');
1080
- fs.writeFileSync(placeholderPath, JSON.stringify(placeholderCorpus));
1081
- const placeholder = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', placeholderPath, '--config', fixture.configs[0]], { encoding: 'utf8' });
1082
- assert.equal(placeholder.status, 2);
1083
- assert.match(placeholder.stderr, /tasks\[0\]\.prompt.*missing or placeholder/);
1084
-
1085
- const emptyPromptWithHashCorpus = JSON.parse(read(fixture.corpus));
1086
- emptyPromptWithHashCorpus.tasks[0].prompt = '';
1087
- emptyPromptWithHashCorpus.tasks[0].prompt_sha256 = `sha256:${'1'.repeat(64)}`;
1088
- const emptyPromptWithHashPath = path.join(root, 'empty-prompt-with-hash-corpus.json');
1089
- fs.writeFileSync(emptyPromptWithHashPath, JSON.stringify(emptyPromptWithHashCorpus));
1090
- const emptyPromptWithHash = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', emptyPromptWithHashPath, '--config', fixture.configs[0]], { encoding: 'utf8' });
1091
- assert.equal(emptyPromptWithHash.status, 2);
1092
- assert.match(emptyPromptWithHash.stderr, /tasks\[0\]: exactly one of prompt or prompt_sha256 required/);
1093
-
1094
- const treatment = JSON.parse(read(fixture.configs[1]));
1095
- treatment.experiment_id = 'different-experiment';
1096
- const parityPath = path.join(root, 'parity-treatment.json');
1097
- writeBenchmarkConfig(parityPath, treatment);
1098
- const parity = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', parityPath], { encoding: 'utf8' });
1099
- assert.equal(parity.status, 2);
1100
- assert.match(parity.stderr, /differ only by arm/);
1101
-
1102
- const receipts = JSON.parse(read(fixture.receipts));
1103
- const missingArtifactHash = JSON.parse(read(fixture.receipts));
1104
- delete missingArtifactHash.receipts[0].evidence.artifact_sha256;
1105
- const missingArtifactHashPath = path.join(root, 'missing-artifact-hash.json');
1106
- fs.writeFileSync(missingArtifactHashPath, JSON.stringify(missingArtifactHash));
1107
- const missingHash = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', missingArtifactHashPath], { encoding: 'utf8' });
1108
- assert.equal(missingHash.status, 2);
1109
- assert.match(missingHash.stderr, /evidence\.artifact_sha256.*missing or placeholder/);
1110
-
1111
- receipts.receipts[0].evidence.artifact_sha256 = benchmarkRawHash(Buffer.from('wrong artifact'));
1112
- const wrongHashPath = path.join(root, 'wrong-artifact-hash.json');
1113
- fs.writeFileSync(wrongHashPath, JSON.stringify(receipts));
1114
- const wrongHash = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', wrongHashPath], { encoding: 'utf8' });
1115
- assert.equal(wrongHash.status, 2);
1116
- assert.match(wrongHash.stderr, /artifact_sha256.*raw-byte hash mismatch/);
1117
-
1118
- receipts.receipts[0].evidence.artifact_ref = 'https://example.invalid/artifact.json';
1119
- const uriPath = path.join(root, 'uri-artifact.json');
1120
- fs.writeFileSync(uriPath, JSON.stringify(receipts));
1121
- const uri = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', uriPath], { encoding: 'utf8' });
1122
- assert.equal(uri.status, 2);
1123
- assert.match(uri.stderr, /artifact_ref.*relative local file.*URI/);
1124
-
1125
- const outsideArtifact = path.join(path.dirname(root), `${path.basename(root)}-outside.bin`);
1126
- fs.writeFileSync(outsideArtifact, Buffer.from('outside receipt bundle'));
1127
- const traversalReceipts = JSON.parse(read(fixture.receipts));
1128
- traversalReceipts.receipts[0].evidence.artifact_ref = path.relative(root, outsideArtifact);
1129
- traversalReceipts.receipts[0].evidence.artifact_sha256 = benchmarkRawHash(fs.readFileSync(outsideArtifact));
1130
- const traversalPath = path.join(root, 'traversal-artifact.json');
1131
- fs.writeFileSync(traversalPath, JSON.stringify(traversalReceipts));
1132
- const traversal = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', traversalPath], { encoding: 'utf8' });
1133
- assert.equal(traversal.status, 2);
1134
- assert.match(traversal.stderr, /artifact_ref.*path escapes receipt bundle directory/);
1135
- fs.rmSync(outsideArtifact, { force: true });
1136
- } finally {
1137
- fs.rmSync(root, { recursive: true, force: true });
1138
- }
1139
- });
1140
-
1141
- test('B1 summary groups fixture-only receipts by arm/lane with deterministic quantiles', () => {
1142
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-summary-'));
1143
- const fixture = makeBenchmarkFixture(root);
1144
- try {
1145
- const result = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', fixture.receipts], { encoding: 'utf8' });
1146
- assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
1147
- const summary = JSON.parse(result.stdout);
1148
- assert.equal(summary.status, 'ready');
1149
- assert.equal(summary.groups.baseline.Direct.metrics.wall_ms.median, 150);
1150
- assert.equal(summary.groups.baseline.Direct.metrics.wall_ms.p25, 125);
1151
- assert.equal(summary.groups.baseline.Direct.metrics.wall_ms.p75, 175);
1152
- assert.equal(summary.groups.treatment.Direct.quality_rates.correctness, 1);
1153
- assert.equal(summary.rollout_recommendation.gates.Critical.pass, true);
1154
-
1155
- const countAggregation = JSON.parse(read(fixture.receipts));
1156
- const countedReceipt = countAggregation.receipts.find((item) => item.arm === 'baseline' && item.lane === 'Direct' && item.repeat === 1);
1157
- countedReceipt.user_corrections = 2;
1158
- countedReceipt.useful_reviewer_findings = 3;
1159
- countedReceipt.false_positive_reviewer_findings = 4;
1160
- const countAggregationPath = path.join(root, 'count-aggregation.json');
1161
- fs.writeFileSync(countAggregationPath, JSON.stringify(countAggregation));
1162
- const countAggregationResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', countAggregationPath], { encoding: 'utf8' });
1163
- assert.equal(countAggregationResult.status, 0, `${countAggregationResult.stdout}\n${countAggregationResult.stderr}`);
1164
- const countAggregationSummary = JSON.parse(countAggregationResult.stdout);
1165
- assert.equal(countAggregationSummary.groups.baseline.Direct.quality_rates.user_correction_rate, 1);
1166
- assert.equal(countAggregationSummary.groups.baseline.Direct.quality_rates.useful_reviewer_finding_rate, 2);
1167
- assert.equal(countAggregationSummary.groups.baseline.Direct.quality_rates.false_positive_reviewer_finding_rate, 2);
1168
-
1169
- const degraded = JSON.parse(read(fixture.receipts));
1170
- for (const receipt of degraded.receipts.filter((item) => item.arm === 'treatment')) {
1171
- receipt.useful_reviewer_findings = 0;
1172
- receipt.false_positive_reviewer_findings = 1;
1173
- }
1174
- const degradedPath = path.join(root, 'degraded-review-quality.json');
1175
- fs.writeFileSync(degradedPath, JSON.stringify(degraded));
1176
- const degradedResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', degradedPath], { encoding: 'utf8' });
1177
- assert.equal(degradedResult.status, 0, `${degradedResult.stdout}\n${degradedResult.stderr}`);
1178
- const degradedSummary = JSON.parse(degradedResult.stdout);
1179
- assert.equal(degradedSummary.status, 'not-ready');
1180
- assert.equal(degradedSummary.rollout_recommendation.gates.Critical.quality_pass, false);
1181
- assert.equal(degradedSummary.rollout_recommendation.gates.Critical.pass, false);
1182
-
1183
- const lowRiskQualityFailure = JSON.parse(read(fixture.receipts));
1184
- for (const receipt of lowRiskQualityFailure.receipts.filter((item) => item.lane === 'Direct')) receipt.correctness = false;
1185
- const lowRiskQualityFailurePath = path.join(root, 'low-risk-quality-failure.json');
1186
- fs.writeFileSync(lowRiskQualityFailurePath, JSON.stringify(lowRiskQualityFailure));
1187
- const lowRiskQualityFailureResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', lowRiskQualityFailurePath], { encoding: 'utf8' });
1188
- assert.equal(lowRiskQualityFailureResult.status, 0, `${lowRiskQualityFailureResult.stdout}\n${lowRiskQualityFailureResult.stderr}`);
1189
- const lowRiskQualityFailureSummary = JSON.parse(lowRiskQualityFailureResult.stdout);
1190
- assert.equal(lowRiskQualityFailureSummary.rollout_recommendation.gates.Direct.quality_pass, false);
1191
- assert.equal(lowRiskQualityFailureSummary.rollout_recommendation.gates.Direct.pass, false);
1192
- assert.equal(lowRiskQualityFailureSummary.rollout_recommendation.gates.Critical.pass, true);
1193
- } finally {
1194
- fs.rmSync(root, { recursive: true, force: true });
1195
- }
1196
- });
1197
-
1198
- test('B1 rollout rejects incomplete fixture-only lane/repeat matrices', () => {
1199
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-completeness-'));
1200
- const fixture = makeBenchmarkFixture(root);
1201
- try {
1202
- const payload = JSON.parse(read(fixture.receipts));
1203
- const completeReceipts = payload.receipts;
1204
- const partialPath = path.join(root, 'partial.json');
1205
- fs.writeFileSync(partialPath, JSON.stringify({ receipts: completeReceipts.filter((receipt) => !(receipt.arm === 'treatment' && receipt.lane === 'Direct' && receipt.repeat === 1)) }));
1206
- const partial = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', partialPath], { encoding: 'utf8' });
1207
- assert.equal(partial.status, 0, `${partial.stdout}\n${partial.stderr}`);
1208
- const partialSummary = JSON.parse(partial.stdout);
1209
- assert.equal(partialSummary.status, 'not-ready');
1210
- assert.equal(partialSummary.rollout_recommendation.gates.Direct.treatment_matrix.complete, false);
1211
- const incompleteValidation = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', partialPath], { encoding: 'utf8' });
1212
- assert.equal(incompleteValidation.status, 2);
1213
- assert.match(incompleteValidation.stderr, /incomplete receipt matrix.*treatment\/Direct\/fixture-direct\/1/);
1214
-
1215
- const missingLanePath = path.join(root, 'missing-lane.json');
1216
- fs.writeFileSync(missingLanePath, JSON.stringify({ receipts: completeReceipts.filter((receipt) => !(receipt.arm === 'treatment' && receipt.lane === 'Critical')) }));
1217
- const missingLane = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', missingLanePath], { encoding: 'utf8' });
1218
- assert.equal(missingLane.status, 0, `${missingLane.stdout}\n${missingLane.stderr}`);
1219
- const missingLaneSummary = JSON.parse(missingLane.stdout);
1220
- assert.equal(missingLaneSummary.status, 'not-ready');
1221
- assert.equal(missingLaneSummary.rollout_recommendation.gates.Critical.treatment_matrix.complete, false);
1222
-
1223
- const outOfRange = JSON.parse(JSON.stringify(completeReceipts));
1224
- outOfRange[0].repeat = 3;
1225
- const outOfRangePath = path.join(root, 'out-of-range.json');
1226
- fs.writeFileSync(outOfRangePath, JSON.stringify({ receipts: outOfRange }));
1227
- const invalidRepeat = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', outOfRangePath], { encoding: 'utf8' });
1228
- assert.equal(invalidRepeat.status, 2);
1229
- assert.match(invalidRepeat.stderr, /exceeds repeat_policy/);
1230
-
1231
- const duplicateArm = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[0]], { encoding: 'utf8' });
1232
- assert.equal(duplicateArm.status, 2);
1233
- assert.match(duplicateArm.stderr, /duplicate arm/);
1234
-
1235
- const invalidCorpus = JSON.parse(read(fixture.corpus));
1236
- delete invalidCorpus.corpus_id;
1237
- const invalidCorpusPath = path.join(root, 'missing-corpus-id.json');
1238
- fs.writeFileSync(invalidCorpusPath, JSON.stringify(invalidCorpus));
1239
- const missingCorpusId = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', invalidCorpusPath, '--config', fixture.configs[0]], { encoding: 'utf8' });
1240
- assert.equal(missingCorpusId.status, 2);
1241
- assert.match(missingCorpusId.stderr, /corpus_id.*missing or placeholder/);
1242
-
1243
- const invalidCost = JSON.parse(read(fixture.configs[0]));
1244
- invalidCost.input_usd_per_1k = 'not-a-rate';
1245
- const invalidCostPath = path.join(root, 'invalid-cost.json');
1246
- fs.writeFileSync(invalidCostPath, JSON.stringify(invalidCost));
1247
- const invalidCostResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', invalidCostPath], { encoding: 'utf8' });
1248
- assert.equal(invalidCostResult.status, 2);
1249
- assert.match(invalidCostResult.stderr, /config\.input_usd_per_1k.*non-negative number/);
1250
- } finally {
1251
- fs.rmSync(root, { recursive: true, force: true });
1252
- }
1253
- });
1254
-
1255
- test('B1 summary marks no receipts exploratory instead of claiming a run', () => {
1256
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-empty-'));
1257
- const fixture = makeBenchmarkFixture(root);
1258
- try {
1259
- const result = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0]], { encoding: 'utf8' });
1260
- assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
1261
- const summary = JSON.parse(result.stdout);
1262
- assert.equal(summary.status, 'exploratory/no-live-runs');
1263
- assert.equal(summary.live_runs, false);
1264
- } finally {
1265
- fs.rmSync(root, { recursive: true, force: true });
1266
- }
1267
- });
1268
-
1269
- test('B1 run requires explicit runner contract and rejects placeholders/shell strings', () => {
1270
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-run-runner-'));
1271
- const fixture = makeBenchmarkFixture(root);
1272
- try {
1273
- // Missing runner
1274
- const missing = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--out', path.join(root, 'out-missing')], { encoding: 'utf8' });
1275
- assert.equal(missing.status, 2);
1276
- assert.match(missing.stderr, /runner contract is required/);
1277
- assert.match(missing.stderr, /explicit command array/);
1278
-
1279
- // Placeholder runner (command contains placeholder)
1280
- const placeholderRunner = path.join(root, 'placeholder-runner.json');
1281
- fs.writeFileSync(placeholderRunner, JSON.stringify({ schema_version: 'b1.v1', command: ['node', '{{replace_me}}'] }));
1282
- const placeholder = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', placeholderRunner, '--out', path.join(root, 'out-placeholder')], { encoding: 'utf8' });
1283
- assert.equal(placeholder.status, 2);
1284
- assert.match(placeholder.stderr, /missing or placeholder freeze field/);
1285
-
1286
- // Shell string forbidden (command as string)
1287
- const shellRunner = path.join(root, 'shell-runner.json');
1288
- fs.writeFileSync(shellRunner, JSON.stringify({ schema_version: 'b1.v1', command: 'node runner.mjs' }));
1289
- const shell = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', shellRunner, '--out', path.join(root, 'out-shell')], { encoding: 'utf8' });
1290
- assert.equal(shell.status, 2);
1291
- assert.match(shell.stderr, /explicit argv|shell string is forbidden/);
1292
-
1293
- // Missing out/receipts
1294
- const minimalRunner = path.join(root, 'minimal-runner.json');
1295
- fs.writeFileSync(minimalRunner, JSON.stringify({ schema_version: 'b1.v1', command: ['node', '-e', 'process.exit(0)'] }));
1296
- const noOut = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', minimalRunner], { encoding: 'utf8' });
1297
- assert.equal(noOut.status, 2);
1298
- assert.match(noOut.stderr, /output is required/);
1299
- } finally {
1300
- fs.rmSync(root, { recursive: true, force: true });
1301
- }
1302
- });
1303
-
1304
- test('B1 run executes via explicit argv, captures wall_ms/artifact, and remains honest no-live-runs without execution', () => {
1305
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-run-execute-'));
1306
- const fixture = makeBenchmarkFixture(root);
1307
- try {
1308
- // Honest no-live-runs: summarize without receipts must stay exploratory
1309
- const noLive = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0]], { encoding: 'utf8' });
1310
- assert.equal(noLive.status, 0);
1311
- const noLiveSummary = JSON.parse(noLive.stdout);
1312
- assert.equal(noLiveSummary.status, 'exploratory/no-live-runs');
1313
- assert.equal(noLiveSummary.live_runs, false);
1314
- assert.equal(noLiveSummary.rollout_recommendation.status, 'exploratory');
1315
- // Also validate without receipts is valid_no_receipts, not fabricated success
1316
- const validNoReceipts = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0]], { encoding: 'utf8' });
1317
- assert.equal(validNoReceipts.status, 0);
1318
- assert.equal(JSON.parse(validNoReceipts.stdout).status, 'valid_no_receipts');
1319
-
1320
- // Create minimal deterministic runner that writes artifact and emits metrics JSON
1321
- const runnerScript = path.join(root, 'runner.mjs');
1322
- fs.writeFileSync(runnerScript, `
1323
- import fs from 'node:fs';
1324
- import path from 'node:path';
1325
- let input='';
1326
- process.stdin.on('data', c=>input+=c);
1327
- process.stdin.on('end', ()=>{
1328
- const payload=JSON.parse(input);
1329
- fs.mkdirSync(path.dirname(payload.artifact_path), {recursive:true});
1330
- fs.writeFileSync(payload.artifact_path, 'artifact:'+payload.task_id+'/'+payload.repeat+'\\n');
1331
- const out={input_tokens:100, output_tokens:50, context_loaded_tokens:500, tool_calls:2, subagent_calls:0, correctness:true, regression:false, unsupported_completion_claim:false, user_corrections:0, useful_reviewer_findings:1, false_positive_reviewer_findings:0};
1332
- process.stdout.write(JSON.stringify(out));
1333
- });
1334
- `);
1335
- const runnerJson = path.join(root, 'runner.json');
1336
- fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', runnerScript] }));
1337
- const outDir = path.join(root, 'out-live');
1338
- const run = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', runnerJson, '--out', outDir], { encoding: 'utf8' });
1339
- assert.equal(run.status, 0, `${run.stdout}\n${run.stderr}`);
1340
- const runResult = JSON.parse(run.stdout);
1341
- assert.equal(runResult.status, 'executed');
1342
- assert.equal(runResult.live_runs, true);
1343
- assert.ok(fs.existsSync(path.join(outDir, 'receipts.json')), 'receipts.json must be written');
1344
- // Receipts must be consumable by existing validate/summarize path
1345
- const receipts = JSON.parse(fs.readFileSync(path.join(outDir, 'receipts.json'), 'utf8'));
1346
- assert.ok(Array.isArray(receipts.receipts) || Array.isArray(receipts));
1347
- const validate = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--receipts', path.join(outDir, 'receipts.json')], { encoding: 'utf8' });
1348
- assert.equal(validate.status, 0, `${validate.stdout}\n${validate.stderr}`);
1349
- assert.equal(JSON.parse(validate.stdout).status, 'valid');
1350
- // Ensure receipts capture wall_ms, command, artifact hash and are frozen
1351
- const firstReceipt = (receipts.receipts || receipts)[0];
1352
- assert.ok(typeof firstReceipt.wall_ms === 'number' && firstReceipt.wall_ms >= 0, 'wall_ms captured');
1353
- assert.ok(typeof firstReceipt.evidence.command === 'string' && firstReceipt.evidence.command.includes('node'), 'command captured');
1354
- assert.match(firstReceipt.evidence.artifact_sha256, /^sha256:[0-9a-f]{64}$/);
1355
- assert.equal(firstReceipt.corpus_sha256, JSON.parse(fs.readFileSync(fixture.corpus, 'utf8')).corpus_sha256 || runResult.corpus_sha256 || firstReceipt.corpus_sha256);
1356
- // Artifact file must exist and hash must match raw bytes
1357
- const artifactPath = path.join(outDir, firstReceipt.evidence.artifact_ref);
1358
- assert.ok(fs.existsSync(artifactPath), 'artifact file must exist inside bundle');
1359
- const bytes = fs.readFileSync(artifactPath);
1360
- const expectedHash = `sha256:${crypto.createHash('sha256').update(bytes).digest('hex')}`;
1361
- assert.equal(firstReceipt.evidence.artifact_sha256, expectedHash);
1362
- // Ensure no secrets leakage: receipts must not contain env secrets (check that known secret pattern not in file)
1363
- const receiptsText = fs.readFileSync(path.join(outDir, 'receipts.json'), 'utf8');
1364
- assert.doesNotMatch(receiptsText, /OPENAI_API_KEY|AWS_SECRET|GITHUB_TOKEN/);
1365
- // Ensure artifact_ref is relative and does not escape
1366
- assert.doesNotMatch(firstReceipt.evidence.artifact_ref, /^\//);
1367
- assert.doesNotMatch(firstReceipt.evidence.artifact_ref, /^\.\./);
1368
- assert.doesNotMatch(firstReceipt.evidence.artifact_ref, /^https?:/);
1369
- } finally {
1370
- fs.rmSync(root, { recursive: true, force: true });
1371
- }
1372
- });
1373
-
1374
- test('B1 run rejects mismatched corpus hash, artifact escape, and partial matrices fail-closed', () => {
1375
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-run-safety-'));
1376
- const fixture = makeBenchmarkFixture(root);
1377
- try {
1378
- const runnerScript = path.join(root, 'safe-runner.mjs');
1379
- fs.writeFileSync(runnerScript, `
1380
- import fs from 'node:fs';
1381
- import path from 'node:path';
1382
- let input='';
1383
- process.stdin.on('data', c=>input+=c);
1384
- process.stdin.on('end', ()=>{
1385
- const payload=JSON.parse(input);
1386
- fs.mkdirSync(path.dirname(payload.artifact_path), {recursive:true});
1387
- fs.writeFileSync(payload.artifact_path, 'ok');
1388
- const out={input_tokens:100, output_tokens:50, context_loaded_tokens:500, tool_calls:2, subagent_calls:0, correctness:true, regression:false, unsupported_completion_claim:false, user_corrections:0, useful_reviewer_findings:1, false_positive_reviewer_findings:0};
1389
- process.stdout.write(JSON.stringify(out));
1390
- });
1391
- `);
1392
- const runnerJson = path.join(root, 'runner.json');
1393
- fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', runnerScript] }));
1394
-
1395
- // Mismatched corpus_sha256 in config must be rejected before execution
1396
- const badCorpus = JSON.parse(fs.readFileSync(fixture.corpus, 'utf8'));
1397
- badCorpus.corpus_id = 'tampered-id';
1398
- const badCorpusPath = path.join(root, 'bad-corpus.json');
1399
- fs.writeFileSync(badCorpusPath, JSON.stringify(badCorpus));
1400
- const mismatch = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', badCorpusPath, '--config', fixture.configs[0], '--runner', runnerJson, '--out', path.join(root, 'out-mismatch')], { encoding: 'utf8' });
1401
- assert.equal(mismatch.status, 2);
1402
- assert.match(mismatch.stderr, /corpus_sha256.*does not match corpus|freeze hash mismatch/);
1403
-
1404
- // Artifact escape via validator: craft receipt with traversal artifact_ref and expect reject
1405
- const traversalReceipts = JSON.parse(fs.readFileSync(fixture.receipts, 'utf8'));
1406
- const outsideArtifact = path.join(path.dirname(root), 'outside.bin');
1407
- fs.writeFileSync(outsideArtifact, Buffer.from('outside'));
1408
- traversalReceipts.receipts[0].evidence.artifact_ref = path.relative(root, outsideArtifact);
1409
- traversalReceipts.receipts[0].evidence.artifact_sha256 = `sha256:${crypto.createHash('sha256').update(fs.readFileSync(outsideArtifact)).digest('hex')}`;
1410
- const traversalPath = path.join(root, 'traversal.json');
1411
- fs.writeFileSync(traversalPath, JSON.stringify(traversalReceipts));
1412
- const traversal = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', traversalPath], { encoding: 'utf8' });
1413
- assert.equal(traversal.status, 2);
1414
- assert.match(traversal.stderr, /path escapes receipt bundle directory/);
1415
- fs.rmSync(outsideArtifact, { force: true });
1416
-
1417
- // Partial matrix: run with runner that fails on one task/repeat must not be reported as valid success
1418
- // Simulate by creating a runner that exits non-zero for one specific task
1419
- const flakyRunner = path.join(root, 'flaky-runner.mjs');
1420
- fs.writeFileSync(flakyRunner, `
1421
- import fs from 'node:fs';
1422
- import path from 'node:path';
1423
- let input='';
1424
- process.stdin.on('data', c=>input+=c);
1425
- process.stdin.on('end', ()=>{
1426
- const payload=JSON.parse(input);
1427
- if (payload.task_id==='fixture-direct' && payload.repeat===1 && payload.arm==='baseline') process.exit(7);
1428
- fs.mkdirSync(path.dirname(payload.artifact_path), {recursive:true});
1429
- fs.writeFileSync(payload.artifact_path, 'ok');
1430
- const out={input_tokens:100, output_tokens:50, context_loaded_tokens:500, tool_calls:2, subagent_calls:0, correctness:true, regression:false, unsupported_completion_claim:false, user_corrections:0, useful_reviewer_findings:1, false_positive_reviewer_findings:0};
1431
- process.stdout.write(JSON.stringify(out));
1432
- });
1433
- `);
1434
- const flakyJson = path.join(root, 'flaky.json');
1435
- fs.writeFileSync(flakyJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', flakyRunner] }));
1436
- const flakyOut = path.join(root, 'out-flaky');
1437
- const flakyRun = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', flakyJson, '--out', flakyOut], { encoding: 'utf8' });
1438
- assert.equal(flakyRun.status, 2);
1439
- assert.match(flakyRun.stderr, /runner exited non-zero|no receipt fabricated/);
1440
- // No valid live success should be claimed from partial run
1441
- if (fs.existsSync(path.join(flakyOut, 'receipts.json'))) {
1442
- const maybeReceipts = JSON.parse(fs.readFileSync(path.join(flakyOut, 'receipts.json'), 'utf8'));
1443
- if (maybeReceipts.receipts) {
1444
- const incompleteValidate = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--receipts', path.join(flakyOut, 'receipts.json')], { encoding: 'utf8' });
1445
- assert.equal(incompleteValidate.status, 2);
1446
- assert.match(incompleteValidate.stderr, /incomplete receipt matrix|runner exited/);
1447
- }
1448
- }
1449
- } finally {
1450
- fs.rmSync(root, { recursive: true, force: true });
1451
- }
1452
- });
1453
-
1454
- test('B1 runner secret safety: command argv with secret-like assignment/flag is fail-closed and stderr is redacted', () => {
1455
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-secret-'));
1456
- const fixture = makeBenchmarkFixture(root);
1457
- try {
1458
- // Helper runner that would succeed if not blocked
1459
- const okRunnerScript = path.join(root, 'ok.mjs');
1460
- fs.writeFileSync(okRunnerScript, `
1461
- import fs from 'node:fs';
1462
- import path from 'node:path';
1463
- let input='';
1464
- process.stdin.on('data', c=>input+=c);
1465
- process.stdin.on('end', ()=>{
1466
- const p=JSON.parse(input);
1467
- fs.mkdirSync(path.dirname(p.artifact_path), {recursive:true});
1468
- fs.writeFileSync(p.artifact_path,'ok');
1469
- process.stdout.write(JSON.stringify({input_tokens:1,output_tokens:1,context_loaded_tokens:1,tool_calls:0,subagent_calls:0,correctness:true,regression:false,unsupported_completion_claim:false,user_corrections:0,useful_reviewer_findings:0,false_positive_reviewer_findings:0}));
1470
- });
1471
- `);
1472
- // Secret-like argv must be rejected fail-closed, value not shown, keep shell:false
1473
- const cases = [
1474
- { desc: 'env assignment', command: ['node', okRunnerScript, 'OPENAI_API_KEY=sk-1234567890abcdefghij123456'] },
1475
- { desc: 'flag equals', command: ['node', okRunnerScript, '--api-key=sk-1234567890abcdefghij123456'] },
1476
- { desc: 'flag spaced', command: ['node', okRunnerScript, '--api-key', 'sk-1234567890abcdefghij123456'] },
1477
- { desc: 'password', command: ['node', okRunnerScript, '--password=supersecret1234'] },
1478
- { desc: 'jwt', command: ['node', okRunnerScript, '--jwt-secret=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9abcdefghij'] },
1479
- { desc: 'github token', command: ['node', okRunnerScript, 'GITHUB_TOKEN=ghp_1234567890abcdefghijklmnopqrstuv'] },
1480
- ];
1481
- for (const c of cases) {
1482
- const runnerJson = path.join(root, `runner-${c.desc.replace(/\s+/g,'-')}.json`);
1483
- fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command: c.command }));
1484
- const out = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', runnerJson, '--out', path.join(root, `out-${c.desc}`)], { encoding: 'utf8' });
1485
- assert.equal(out.status, 2, `case ${c.desc} should be blocked`);
1486
- assert.match(out.stderr, /secret-like assignment\/flag is forbidden/);
1487
- assert.match(out.stderr, /Value not shown/);
1488
- // Must not leak secret value in error or evidence
1489
- assert.doesNotMatch(out.stderr, /sk-1234567890/);
1490
- assert.doesNotMatch(out.stderr, /supersecret/);
1491
- assert.doesNotMatch(out.stderr, /ghp_/);
1492
- assert.doesNotMatch(out.stderr, /eyJhbGci/);
1493
- // Also ensure stderr snippet does not contain raw secret if leaked via stderr
1494
- // (the runner never executed, so no stderr from runner, just validation error)
1495
- }
1496
-
1497
- // Safe names must NOT be flagged (no false positive)
1498
- const safeCases = [
1499
- ['node', okRunnerScript, '--tokenizer', 'bert-base'],
1500
- ['node', okRunnerScript, '--api-key-file', '/tmp/keyfile'],
1501
- ['node', okRunnerScript, '--token-path', '/tmp/token'],
1502
- ['node', okRunnerScript, '--password-hint', 'my hint'],
1503
- ['node', okRunnerScript, '--password-label', 'label'],
1504
- ];
1505
- for (const [idx, command] of safeCases.entries()) {
1506
- const runnerJson = path.join(root, `safe-${idx}.json`);
1507
- fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command }));
1508
- const out = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', runnerJson, '--out', path.join(root, `out-safe-${idx}`)], { encoding: 'utf8' });
1509
- // Should not be blocked for secret-like; it should either succeed or fail for other reasons but not secret-like
1510
- assert.doesNotMatch(out.stderr, /secret-like assignment\/flag is forbidden/, `safe case ${command.join(' ')} must not be flagged`);
1511
- assert.equal(out.status, 0, `safe case ${command.join(' ')} should execute (got ${out.stderr})`);
1512
- // Clean out for next
1513
- fs.rmSync(path.join(root, `out-safe-${idx}`), { recursive: true, force: true });
1514
- }
1515
-
1516
- // Stderr redaction: runner that exits non-zero and leaks secret in stderr must have [REDACTED] and not raw secret
1517
- const leakyRunner = path.join(root, 'leaky.mjs');
1518
- fs.writeFileSync(leakyRunner, `
1519
- import fs from 'node:fs';
1520
- let input='';
1521
- process.stdin.on('data', c=>input+=c);
1522
- process.stdin.on('end', ()=>{
1523
- console.error('failed to connect with OPENAI_API_KEY=sk-1234567890abcdefghijklmnopqrstuv and token ghp_1234567890abcdefghijklmnopqrstuv');
1524
- console.error('also password supersecret1234 and jwt eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9abcdefghij');
1525
- process.exit(2);
1526
- });
1527
- `);
1528
- const leakyJson = path.join(root, 'leaky.json');
1529
- fs.writeFileSync(leakyJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', leakyRunner] }));
1530
- const leakyOut = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', leakyJson, '--out', path.join(root, 'out-leaky')], { encoding: 'utf8' });
1531
- assert.equal(leakyOut.status, 2);
1532
- assert.match(leakyOut.stderr, /runner exited non-zero/);
1533
- assert.match(leakyOut.stderr, /\[REDACTED\]/);
1534
- assert.doesNotMatch(leakyOut.stderr, /sk-1234567890/);
1535
- assert.doesNotMatch(leakyOut.stderr, /ghp_1234567890/);
1536
- assert.doesNotMatch(leakyOut.stderr, /supersecret/);
1537
- assert.doesNotMatch(leakyOut.stderr, /eyJhbGci/);
1538
- // Ensure no receipt was fabricated
1539
- assert.equal(fs.existsSync(path.join(root, 'out-leaky', 'receipts.json')), false, 'no receipt should be fabricated on stderr secret leak');
1540
-
1541
- // Also test that evidence.command never contains secret when runner would have been allowed (it is blocked, so no evidence)
1542
- // For a runner that does not contain secret, evidence should contain the safe command
1543
- const safeRunnerForEvidence = path.join(root, 'evidence.mjs');
1544
- fs.writeFileSync(safeRunnerForEvidence, `
1545
- import fs from 'node:fs';
1546
- import path from 'node:path';
1547
- let input='';
1548
- process.stdin.on('data', c=>input+=c);
1549
- process.stdin.on('end', ()=>{
1550
- const p=JSON.parse(input);
1551
- fs.mkdirSync(path.dirname(p.artifact_path),{recursive:true});
1552
- fs.writeFileSync(p.artifact_path,'evidence');
1553
- process.stdout.write(JSON.stringify({input_tokens:1,output_tokens:1,context_loaded_tokens:1,tool_calls:0,subagent_calls:0,correctness:true,regression:false,unsupported_completion_claim:false,user_corrections:0,useful_reviewer_findings:0,false_positive_reviewer_findings:0}));
1554
- });
1555
- `);
1556
- const evidenceRunnerJson = path.join(root, 'evidence.json');
1557
- fs.writeFileSync(evidenceRunnerJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', safeRunnerForEvidence, '--verbose'] }));
1558
- const evidenceOut = path.join(root, 'out-evidence');
1559
- const evidenceRun = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', evidenceRunnerJson, '--out', evidenceOut], { encoding: 'utf8' });
1560
- assert.equal(evidenceRun.status, 0);
1561
- const evidenceReceipts = JSON.parse(fs.readFileSync(path.join(evidenceOut, 'receipts.json'), 'utf8'));
1562
- const ev = (evidenceReceipts.receipts || evidenceReceipts)[0].evidence;
1563
- assert.match(ev.command, /node/);
1564
- assert.doesNotMatch(ev.command, /sk-|ghp_|eyJ/);
1565
- assert.doesNotMatch(JSON.stringify(evidenceReceipts), /OPENAI_API_KEY/);
1566
- } finally {
1567
- fs.rmSync(root, { recursive: true, force: true });
1568
- }
1569
- });
1570
-
1571
797
  // ── P0 regressions ──────────────────────────────────────────────────────
1572
798
  test('P0 receipt fail-closed: Exit 1/-1/abc/conflict/empty command rejected', () => {
1573
799
  const base = 'Verification: PASS\nCommand: pnpm test\nBase: 0123456789abcdef0123456789abcdef01234567\nHead: 89abcdef0123456789abcdef0123456789abcdef\n';