@haposoft/cafekit 0.18.2 → 0.18.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/__tests__/adv-skill.test.js +370 -0
- package/bin/__tests__/agent-authority.test.js +38 -0
- package/bin/__tests__/agent-instructions.test.js +54 -0
- package/bin/__tests__/claude-plugin-build.test.js +151 -0
- package/bin/__tests__/codex-hook-runtime-fixture.cjs +26 -0
- package/bin/__tests__/codex-hooks-ownership.test.js +8 -11
- package/bin/__tests__/codex-hooks.test.js +87 -85
- package/bin/__tests__/codex-implicit-policy.test.js +160 -0
- package/bin/__tests__/codex-native-hook-boundaries.test.js +2 -4
- package/bin/__tests__/codex-native.test.js +136 -2298
- package/bin/__tests__/codex-plugin-build.test.js +230 -0
- package/bin/__tests__/codex-plugin-context.test.js +135 -0
- package/bin/__tests__/codex-plugin-hooks.test.js +220 -0
- package/bin/__tests__/codex-quiet-hooks.test.js +2 -8
- package/bin/__tests__/codex-runtime-layout.test.js +81 -0
- package/bin/__tests__/codex-setup-agents.test.js +216 -0
- package/bin/__tests__/codex-skill-projections.test.js +101 -0
- package/bin/__tests__/develop-contract.test.js +100 -874
- package/bin/__tests__/legacy-hooks-retired.test.js +63 -13
- package/bin/__tests__/migrate-apply.test.js +373 -0
- package/bin/__tests__/migrate-cli.test.js +152 -0
- package/bin/__tests__/migrate-merged.test.js +269 -0
- package/bin/__tests__/migrate-plan.test.js +271 -0
- package/bin/__tests__/omp-bridge.test.js +81 -5
- package/bin/__tests__/omp-hooks.test.js +24 -0
- package/bin/__tests__/orca-session.test.js +2 -16
- package/bin/__tests__/package-inventory.test.js +228 -2044
- package/bin/__tests__/plugin-safety-keys.test.js +121 -0
- package/bin/__tests__/privacy-paths.test.js +184 -0
- package/bin/__tests__/secret-scanner.test.js +73 -0
- package/bin/__tests__/skill-routing-source.test.js +6 -0
- package/bin/__tests__/toaa-skill.test.js +195 -0
- package/bin/__tests__/usage-hook-retired.test.js +24 -6
- package/bin/__tests__/validator-grounding.test.js +0 -198
- package/bin/__tests__/vibe-run.test.js +215 -0
- package/bin/__tests__/vibe-skill.test.js +548 -0
- package/bin/__tests__/website-docs.test.js +58 -0
- package/bin/install.js +99 -1
- package/bin/lib/backup.js +43 -24
- package/bin/lib/codex-install.js +12 -3
- package/bin/lib/context.js +5 -0
- package/bin/lib/lock.js +98 -18
- package/bin/lib/managed-writer.js +16 -10
- package/bin/lib/migrate-apply.js +202 -0
- package/bin/lib/migrate-merged.js +342 -0
- package/bin/lib/migrate-plan.js +419 -0
- package/bin/lib/plugin-inventory.js +33 -0
- package/bin/phases/claude-runtime.js +5 -3
- package/bin/phases/codex-hooks.js +38 -0
- package/bin/phases/codex-runtime.js +106 -74
- package/bin/phases/copy-payload.js +49 -1
- package/bin/phases/omp-runtime.js +4 -26
- package/bin/phases/root-config.js +5 -2
- package/package.json +1 -1
- package/src/claude/CLAUDE.md +1 -1
- package/src/claude/agents/advisor.md +55 -0
- package/src/claude/agents/debugger.md +2 -3
- package/src/claude/agents/deployer.md +2 -1
- package/src/claude/agents/docs-keeper.md +12 -0
- package/src/claude/agents/inspector.md +1 -1
- package/src/claude/agents/project-manager.md +2 -2
- package/src/claude/agents/researcher.md +0 -1
- package/src/claude/agents/spec-maker.md +4 -1
- package/src/claude/hooks/__tests__/plugin-context.test.js +96 -0
- package/src/claude/hooks/__tests__/plugin-options.test.js +141 -0
- package/src/claude/hooks/__tests__/plugin-state-isolation.test.js +125 -0
- package/src/claude/hooks/__tests__/precompact.test.js +7 -1
- package/src/claude/hooks/__tests__/privacy-block.test.js +33 -35
- package/src/claude/hooks/__tests__/setup-statusline.test.js +179 -0
- package/src/claude/hooks/__tests__/spec-gate.test.js +86 -48
- package/src/claude/hooks/__tests__/state.test.js +21 -0
- package/src/claude/hooks/agent.cjs +5 -2
- package/src/claude/hooks/inspect-block.cjs +3 -1
- package/src/claude/hooks/lib/hook-state-dir.cjs +19 -1
- package/src/claude/hooks/lib/plugin-options.cjs +47 -0
- package/src/claude/hooks/lib/privacy-command-analysis.cjs +130 -11
- package/src/claude/hooks/lib/privacy-paths.cjs +370 -10
- package/src/claude/hooks/lib/spec-touch.cjs +14 -1
- package/src/claude/hooks/plugin-context.cjs +80 -0
- package/src/claude/hooks/privacy-block.cjs +2 -1
- package/src/claude/hooks/rules.cjs +5 -2
- package/src/claude/hooks/session.cjs +4 -1
- package/src/claude/hooks/spec-state.cjs +5 -4
- package/src/claude/hooks/state.cjs +27 -1
- package/src/claude/migration-manifest.json +7 -1
- package/src/claude/plugin/commands/setup-statusline.md +12 -0
- package/src/claude/plugin/core.md +71 -0
- package/src/claude/rules/manage-docs.md +1 -2
- package/src/claude/rules/review-audit-self-decision.md +1 -1
- package/src/claude/rules/skill-workflow-routing.md +1 -1
- package/src/claude/rules/state-sync.md +14 -0
- package/src/claude/rules/workflow.md +1 -1
- package/src/claude/scripts/generate-skill-catalog.cjs +5 -0
- package/src/claude/scripts/provenance.cjs +40 -16
- package/src/claude/scripts/scan-staged-secrets.cjs +49 -3
- package/src/claude/scripts/setup-statusline.cjs +88 -0
- package/src/claude/skills/adv/SKILL.md +139 -0
- package/src/claude/skills/adv/references/optional-modes.md +41 -0
- package/src/claude/skills/adv/references/ultra-mode.md +29 -0
- package/src/claude/skills/brainstorm/SKILL.md +1 -1
- package/src/claude/skills/code-review/SKILL.md +4 -1
- package/src/claude/skills/code-review/references/pr-fix-reply.md +42 -0
- package/src/claude/skills/develop/SKILL.md +11 -3
- package/src/claude/skills/develop/references/quality-gate.md +1 -1
- package/src/claude/skills/fix/SKILL.md +2 -2
- package/src/claude/skills/git/SKILL.md +4 -2
- package/src/claude/skills/git/references/merge-pr.md +125 -0
- package/src/claude/skills/specs/SKILL.md +2 -2
- package/src/claude/skills/specs/references/review.md +1 -1
- package/src/claude/skills/specs/references/templates.md +2 -0
- package/src/claude/skills/sync/SKILL.md +4 -0
- package/src/claude/skills/toaa/SKILL.md +82 -0
- package/src/claude/skills/toaa/references/autonomous-orchestration.md +54 -0
- package/src/claude/skills/toaa/references/prerequisites-checklist.md +37 -0
- package/src/claude/skills/vibe/SKILL.md +69 -0
- package/src/claude/skills/vibe/references/pipeline-skill-integration.md +34 -0
- package/src/claude/skills/vibe/references/report-templates.md +84 -0
- package/src/claude/skills/vibe/references/subagent-orchestration.md +23 -0
- package/src/codex/AGENTS.md +1 -1
- package/src/codex/hooks/inspect-block.cjs +7 -4
- package/src/codex/hooks/lib/hook-context.cjs +119 -4
- package/src/codex/hooks/lib/plugin-options.cjs +2 -0
- package/src/codex/hooks/lib/privacy-state.cjs +18 -1
- package/src/codex/hooks/lib/state-store.cjs +31 -6
- package/src/codex/hooks/precompact.cjs +9 -8
- package/src/codex/hooks/privacy-approval.cjs +2 -2
- package/src/codex/hooks/privacy-block.cjs +5 -4
- package/src/codex/hooks/rules.cjs +12 -6
- package/src/codex/hooks/session.cjs +22 -10
- package/src/codex/plugin/core.md +76 -0
- package/src/codex/plugin/hooks/plugin-context.cjs +63 -0
- package/src/codex/plugin/skills/setup-agents/SKILL.md +17 -0
- package/src/codex/scripts/setup-agents.cjs +124 -0
- package/src/common/AGENTS.md +1 -1
- package/src/omp/extensions/{cafekit-bridge.mjs → cafekit-bridge.js} +73 -5
- package/src/omp/hooks/privacy-block.cjs +2 -1
- package/bin/__tests__/codex-research-projection.test.js +0 -70
- package/bin/__tests__/codex-specs-projection.test.js +0 -76
- package/bin/__tests__/specs-v2-execution-closeout.test.js +0 -58
- package/src/claude/skills/chrome-devtools/scripts/__tests__/error-handling.test.js +0 -102
|
@@ -36,13 +36,12 @@ const PROCESS_FIRST_AGENTS = [
|
|
|
36
36
|
'deployer.md',
|
|
37
37
|
].map((name) => path.join(PACKAGE_ROOT, 'src/claude/agents', name));
|
|
38
38
|
const PROVENANCE = path.join(PACKAGE_ROOT, '../../docs/provenance.md');
|
|
39
|
-
const SPECS_USAGE_GUIDE = path.join(PACKAGE_ROOT, '../../docs/specs-usage-guide.md');
|
|
40
39
|
const RUNTIME_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-policy-runtime-'));
|
|
41
|
-
const RUNTIME_SPEC = path.join(RUNTIME_ROOT, 'specs', 'demo', '
|
|
40
|
+
const RUNTIME_SPEC = path.join(RUNTIME_ROOT, 'specs', 'demo', 'plan.md');
|
|
42
41
|
fs.mkdirSync(path.dirname(RUNTIME_SPEC), { recursive: true });
|
|
43
42
|
fs.mkdirSync(path.join(RUNTIME_ROOT, 'src'), { recursive: true });
|
|
44
43
|
fs.writeFileSync(path.join(RUNTIME_ROOT, 'src', 'app.js'), 'runtime fixture\n');
|
|
45
|
-
fs.writeFileSync(RUNTIME_SPEC,
|
|
44
|
+
fs.writeFileSync(RUNTIME_SPEC, '# Demo\nSpecs-Contract: process-first-ready-v1\n');
|
|
46
45
|
for (const args of [['init', '-q'], ['config', 'user.email', 'cafekit@example.invalid'], ['config', 'user.name', 'CafeKit Test'], ['add', '-A'], ['commit', '-qm', 'fixture']]) {
|
|
47
46
|
const result = spawnSync('git', ['-C', RUNTIME_ROOT, ...args], { encoding: 'utf8' });
|
|
48
47
|
assert.equal(result.status, 0, result.stderr);
|
|
@@ -94,37 +93,7 @@ function installClaude(root) {
|
|
|
94
93
|
});
|
|
95
94
|
}
|
|
96
95
|
|
|
97
|
-
function
|
|
98
|
-
const headings = [];
|
|
99
|
-
const primaryLines = [];
|
|
100
|
-
const legacyLines = [];
|
|
101
|
-
const hierarchy = [];
|
|
102
|
-
|
|
103
|
-
for (const [index, line] of content.split('\n').entries()) {
|
|
104
|
-
const heading = line.match(/^(#{1,6})\s+(.+?)\s*$/);
|
|
105
|
-
if (heading) {
|
|
106
|
-
const level = heading[1].length;
|
|
107
|
-
while (hierarchy.at(-1)?.level >= level) hierarchy.pop();
|
|
108
|
-
const entry = { level, text: heading[2], line: index + 1 };
|
|
109
|
-
hierarchy.push(entry);
|
|
110
|
-
headings.push(entry);
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
const target = hierarchy.some(({ text }) => /legacy/i.test(text))
|
|
114
|
-
? legacyLines
|
|
115
|
-
: primaryLines;
|
|
116
|
-
target.push({ line: index + 1, text: line });
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
return {
|
|
120
|
-
headings,
|
|
121
|
-
primary: primaryLines.map(({ text }) => text).join('\n'),
|
|
122
|
-
legacy: legacyLines.map(({ text }) => text).join('\n'),
|
|
123
|
-
primaryLines,
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
function assertVocabularyIsLegacyOnly(filePath, regions) {
|
|
96
|
+
function assertNoLegacyVocabulary(filePath, content) {
|
|
128
97
|
const vocabulary = [
|
|
129
98
|
['v2.1', /\bv2\.1\b/i],
|
|
130
99
|
['spec.json', /\bspec\.json\b/i],
|
|
@@ -140,16 +109,12 @@ function assertVocabularyIsLegacyOnly(filePath, regions) {
|
|
|
140
109
|
['lane', /\blane\b/i],
|
|
141
110
|
];
|
|
142
111
|
const violations = [];
|
|
143
|
-
for (const
|
|
112
|
+
for (const [index, text] of content.split('\n').entries()) {
|
|
144
113
|
for (const [term, pattern] of vocabulary) {
|
|
145
|
-
if (pattern.test(text)) violations.push(`${
|
|
114
|
+
if (pattern.test(text)) violations.push(`${index + 1}:${term}`);
|
|
146
115
|
}
|
|
147
116
|
}
|
|
148
|
-
assert.deepEqual(
|
|
149
|
-
violations,
|
|
150
|
-
[],
|
|
151
|
-
`${path.relative(PACKAGE_ROOT, filePath)} has v2.1 vocabulary outside a Legacy heading`,
|
|
152
|
-
);
|
|
117
|
+
assert.deepEqual(violations, [], `${path.relative(PACKAGE_ROOT, filePath)} teaches a retired packet vocabulary`);
|
|
153
118
|
}
|
|
154
119
|
|
|
155
120
|
function canonicalReceipt(command = 'pnpm test', extra = '') {
|
|
@@ -160,16 +125,8 @@ function read(filePath) {
|
|
|
160
125
|
return fs.readFileSync(filePath, 'utf8');
|
|
161
126
|
}
|
|
162
127
|
|
|
163
|
-
function manifestAgents() {
|
|
164
|
-
const manifest = JSON.parse(read(path.join(PACKAGE_ROOT, 'src/claude/migration-manifest.json')));
|
|
165
|
-
return manifest.agents.required.map((fileName) => path.basename(fileName, path.extname(fileName)));
|
|
166
|
-
}
|
|
167
|
-
|
|
168
128
|
test('CLI rejects flash and parallel before any mutation', () => {
|
|
169
129
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-policy-cli-'));
|
|
170
|
-
const statePath = path.join(root, 'state.json');
|
|
171
|
-
fs.writeFileSync(statePath, '{"status":"in_progress"}\n');
|
|
172
|
-
const before = fs.readFileSync(statePath, 'utf8');
|
|
173
130
|
try {
|
|
174
131
|
const result = spawnSync(process.execPath, [POLICY_PATH, '--flash', '--parallel', '--json'], {
|
|
175
132
|
cwd: root,
|
|
@@ -180,7 +137,6 @@ test('CLI rejects flash and parallel before any mutation', () => {
|
|
|
180
137
|
assert.equal(payload.ok, false);
|
|
181
138
|
assert.equal(payload.failFast, true);
|
|
182
139
|
assert.match(payload.message, /incompatible/);
|
|
183
|
-
assert.equal(fs.readFileSync(statePath, 'utf8'), before);
|
|
184
140
|
} finally {
|
|
185
141
|
fs.rmSync(root, { recursive: true, force: true });
|
|
186
142
|
}
|
|
@@ -190,12 +146,9 @@ test('develop flags retain their process-v3 contracts', () => {
|
|
|
190
146
|
const develop = read(DEVELOP);
|
|
191
147
|
assert.match(develop, /node \.claude\/scripts\/workflow-policy\.cjs --flash --parallel --json/);
|
|
192
148
|
assert.match(develop, /exits `2` before a task edit, receipt, worktree, subagent, or commit/i);
|
|
193
|
-
assert.match(develop, /--
|
|
194
|
-
assert.match(develop, /
|
|
195
|
-
assert.match(develop, /
|
|
196
|
-
assert.match(develop, /--parallel \[N\][\s\S]*parallel-waves\.md[\s\S]*isolated worktrees[\s\S]*one controller writer/i);
|
|
197
|
-
assert.match(develop, /--flash[\s\S]*Status: in_progress[\s\S]*FLASH_UNVERIFIED[\s\S]*awaiting \/cf:test <feature>/i);
|
|
198
|
-
assert.match(develop, /FLASH_UNVERIFIED[\s\S]*do not unblock dependents[\s\S]*sync-finalize path/i);
|
|
149
|
+
assert.match(develop, /--parallel \[N\][\s\S]*parallel-waves\.md[\s\S]*one controller writer/i);
|
|
150
|
+
assert.match(develop, /--flash[\s\S]*Status: in_progress[\s\S]*FLASH_UNVERIFIED/);
|
|
151
|
+
assert.match(develop, /FLASH_UNVERIFIED[\s\S]*do not unblock dependents/i);
|
|
199
152
|
});
|
|
200
153
|
|
|
201
154
|
test('review and sync skills keep the verdict vocabulary and the sync-finalize path', () => {
|
|
@@ -210,32 +163,22 @@ test('review and sync skills keep the verdict vocabulary and the sync-finalize p
|
|
|
210
163
|
|
|
211
164
|
test('Develop names the process-first packet as its only authority', () => {
|
|
212
165
|
const develop = read(DEVELOP);
|
|
213
|
-
|
|
214
|
-
assert.doesNotMatch(develop, /
|
|
215
|
-
assert.
|
|
216
|
-
assert.
|
|
217
|
-
assert.match(
|
|
218
|
-
assert.match(
|
|
219
|
-
assert.match(regions.primary, /Verification Plan/);
|
|
220
|
-
assert.match(regions.primary, /inline Receipt/i);
|
|
221
|
-
assert.doesNotMatch(develop, /D2\[Step 3/);
|
|
222
|
-
assert.match(develop, /--notes.*opt-in/i);
|
|
223
|
-
assert.doesNotMatch(develop, /--no-notes/);
|
|
224
|
-
assert.doesNotMatch(develop, /planner|implementer|test-runner|code-auditor|docs-keeper/i);
|
|
166
|
+
assertNoLegacyVocabulary(DEVELOP, develop);
|
|
167
|
+
assert.doesNotMatch(develop, /Legacy workflow compatibility|existing legacy Specs packets/i);
|
|
168
|
+
assert.match(develop, /specs\/<feature>\/plan\.md[\s\S]*task-NN-\*\.md/i);
|
|
169
|
+
assert.match(develop, /one unblocked task at a time/i);
|
|
170
|
+
assert.match(develop, /Verification Plan/);
|
|
171
|
+
assert.match(develop, /inline Receipt/i);
|
|
225
172
|
});
|
|
226
173
|
|
|
227
|
-
test('Specs primary output is a flat process-first packet with
|
|
174
|
+
test('Specs primary output is a flat process-first packet with no legacy packet vocabulary', () => {
|
|
228
175
|
const specs = read(SPECS);
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
assert.
|
|
232
|
-
assert.match(
|
|
233
|
-
assert.match(
|
|
234
|
-
assert.match(
|
|
235
|
-
assert.match(regions.primary, /inline `## Receipt`/i);
|
|
236
|
-
assert.match(regions.primary, /GATE-SCOPE[\s\S]*GATE-REVIEW[\s\S]*GATE-DONE/i);
|
|
237
|
-
assert.doesNotMatch(specs, /installed legacy adapters?/i);
|
|
238
|
-
assertVocabularyIsLegacyOnly(SPECS, regions);
|
|
176
|
+
assertNoLegacyVocabulary(SPECS, specs);
|
|
177
|
+
assert.doesNotMatch(specs, /^#{1,6}\s.*legacy/im);
|
|
178
|
+
assert.match(specs, /specs\/<feature>\/[\s\S]*plan\.md[\s\S]*task-01-<slug>\.md[\s\S]*task-02-<slug>\.md/i);
|
|
179
|
+
assert.match(specs, /Task files are flat beside `plan\.md`/i);
|
|
180
|
+
assert.match(specs, /inline `## Receipt`/i);
|
|
181
|
+
assert.match(specs, /GATE-SCOPE[\s\S]*GATE-REVIEW[\s\S]*GATE-DONE/i);
|
|
239
182
|
});
|
|
240
183
|
|
|
241
184
|
test('core execution agents default to process-first state and carry no legacy adapter', () => {
|
|
@@ -248,75 +191,66 @@ test('core execution agents default to process-first state and carry no legacy a
|
|
|
248
191
|
}
|
|
249
192
|
|
|
250
193
|
const implementer = read(PROCESS_FIRST_AGENTS[0]);
|
|
251
|
-
assert.match(implementer, /
|
|
194
|
+
assert.match(implementer, /do NOT edit[\s\S]{0,80}`plan\.md`[\s\S]{0,80}`Status:`[\s\S]{0,80}`## Receipt`/i);
|
|
252
195
|
assert.match(implementer, /controller/i);
|
|
253
|
-
assert.match(
|
|
254
|
-
|
|
255
|
-
const auditor = read(PROCESS_FIRST_AGENTS[2]);
|
|
256
|
-
assert.match(auditor, /accepted GATE-SCOPE\/GATE-REVIEW decisions/i);
|
|
257
|
-
assert.doesNotMatch(auditor, /CAFEKIT_SEMANTIC_REVIEW_ATTESTATION/);
|
|
258
|
-
|
|
259
|
-
const docsKeeper = read(PROCESS_FIRST_AGENTS[3]);
|
|
260
|
-
assert.match(docsKeeper, /never[\s\S]{0,80}invent or write Status, Receipt, approval, or execution proof/i);
|
|
261
|
-
|
|
196
|
+
assert.match(read(PROCESS_FIRST_AGENTS[2]), /GATE-SCOPE\/GATE-REVIEW/);
|
|
197
|
+
assert.match(read(PROCESS_FIRST_AGENTS[3]), /never[\s\S]{0,80}Status, Receipt/i);
|
|
262
198
|
const deployer = read(PROCESS_FIRST_AGENTS[4]);
|
|
263
|
-
assert.match(deployer, /
|
|
264
|
-
assert.match(deployer, /GATE-DONE\/release authorization/i);
|
|
199
|
+
assert.match(deployer, /GATE-DONE/);
|
|
265
200
|
assert.match(deployer, /does not write process-first Status\/Receipt/i);
|
|
266
201
|
});
|
|
267
202
|
|
|
268
|
-
test('
|
|
269
|
-
const
|
|
270
|
-
['develop', DEVELOP,
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
for (const [name, filePath, expectedLegacy] of surfaces) {
|
|
278
|
-
const regions = markdownLegacyRegions(read(filePath));
|
|
279
|
-
const legacyHeadings = regions.headings.filter(({ text }) => /legacy/i.test(text));
|
|
280
|
-
assert.equal(legacyHeadings.length, expectedLegacy, `${name} must have exactly ${expectedLegacy} Legacy heading(s)`);
|
|
281
|
-
assert.match(regions.primary, /inline (?:`## )?Receipts?/i, `${name} must teach inline Receipts`);
|
|
282
|
-
if (expectedLegacy === 1) {
|
|
283
|
-
assert.match(regions.legacy, /spec\.json/i, `${name} must retain the spec.json adapter`);
|
|
284
|
-
assert.match(regions.legacy, /task_registry/i, `${name} must retain task_registry compatibility`);
|
|
285
|
-
}
|
|
286
|
-
assertVocabularyIsLegacyOnly(filePath, regions);
|
|
287
|
-
regionsByName.set(name, regions);
|
|
203
|
+
test('Develop and Sync surfaces teach the flat process-first packet and no legacy vocabulary', () => {
|
|
204
|
+
for (const [name, filePath] of [
|
|
205
|
+
['develop', DEVELOP], ['parallel waves', PARALLEL_WAVES], ['sync', SYNC_SKILL], ['sync protocols', SYNC_PROTOCOLS],
|
|
206
|
+
]) {
|
|
207
|
+
const content = read(filePath);
|
|
208
|
+
assert.doesNotMatch(content, /^#{1,6}\s.*legacy/im, `${name} must have no Legacy section`);
|
|
209
|
+
assert.match(content, /inline (?:`## )?Receipts?/i, `${name} must teach inline Receipts`);
|
|
210
|
+
assertNoLegacyVocabulary(filePath, content);
|
|
288
211
|
}
|
|
212
|
+
assert.match(read(DEVELOP), /`specs\/<feature>\/plan\.md` plus flat `task-NN-\*\.md`/i);
|
|
213
|
+
assert.match(read(SYNC_SKILL), /<task-NN-slug\.md>/i);
|
|
214
|
+
assert.match(read(SYNC_SKILL), /sync-finalize/i);
|
|
215
|
+
assert.match(read(PARALLEL_WAVES), /one file has one writer per wave/i);
|
|
216
|
+
assert.match(read(PARALLEL_WAVES), /writes inline Receipts and Status one task at a time/i);
|
|
217
|
+
assert.match(read(SYNC_PROTOCOLS), /`plan\.md` and flat `task-\*\.md` files/i);
|
|
218
|
+
});
|
|
289
219
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
assert.match(
|
|
305
|
-
assert.match(
|
|
306
|
-
assert.match(
|
|
307
|
-
assert.match(
|
|
308
|
-
|
|
309
|
-
const
|
|
310
|
-
assert.match(
|
|
311
|
-
assert.match(
|
|
312
|
-
|
|
313
|
-
|
|
220
|
+
test('develop and sync carry symptom-first gotchas', () => {
|
|
221
|
+
// The section runs from its heading to the next level-two heading or the end of the file.
|
|
222
|
+
const gotchas = (filePath) => {
|
|
223
|
+
const content = read(filePath);
|
|
224
|
+
const name = path.basename(path.dirname(filePath));
|
|
225
|
+
assert.equal((content.match(/^## Gotchas$/gm) || []).length, 1, `${name} must have exactly one Gotchas section`);
|
|
226
|
+
const body = content.slice(content.indexOf('\n## Gotchas\n') + 1);
|
|
227
|
+
const next = body.indexOf('\n## ', 1);
|
|
228
|
+
return next === -1 ? body : body.slice(0, next);
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
const develop = gotchas(DEVELOP);
|
|
232
|
+
assert.match(develop, /command_identity/);
|
|
233
|
+
assert.match(develop, /last top-level `- Command:`/);
|
|
234
|
+
assert.match(develop, /\bprovenance\b/);
|
|
235
|
+
assert.match(develop, /outside the specs root/);
|
|
236
|
+
assert.match(develop, /specs\/_shared\/active-feature\.json/);
|
|
237
|
+
assert.match(develop, /explicit_not_found/);
|
|
238
|
+
|
|
239
|
+
const sync = gotchas(SYNC_SKILL);
|
|
240
|
+
assert.match(sync, /committed copy/);
|
|
241
|
+
assert.match(sync, /\/cf:sync rebind/);
|
|
242
|
+
|
|
243
|
+
// A bulleted Receipt is accepted by the parser, so that trap must not come back.
|
|
244
|
+
for (const [name, section] of [['develop', develop], ['sync', sync]]) {
|
|
245
|
+
assert.doesNotMatch(section, /list marker|bullet/i, `${name} gotchas must not list the fixed bulleted-Receipt trap`);
|
|
246
|
+
assert.doesNotMatch(section, /\b(MUST|NEVER)\b/, `${name} gotchas must not shout`);
|
|
247
|
+
}
|
|
314
248
|
});
|
|
315
249
|
|
|
316
250
|
test('Develop process-first source contract preserves selection, recovery, final-Head, parallel, and Flash boundaries', () => {
|
|
317
|
-
const develop =
|
|
251
|
+
const develop = read(DEVELOP);
|
|
318
252
|
const quality = read(GATE);
|
|
319
|
-
const waves =
|
|
253
|
+
const waves = read(PARALLEL_WAVES);
|
|
320
254
|
const dispatch = read(path.join(path.dirname(PARALLEL_WAVES), 'subagent-patterns.md'));
|
|
321
255
|
|
|
322
256
|
for (const clause of [
|
|
@@ -325,13 +259,10 @@ test('Develop process-first source contract preserves selection, recovery, final
|
|
|
325
259
|
/More than one `in_progress`/i,
|
|
326
260
|
/For an interrupted `in_progress` task/i,
|
|
327
261
|
/only the controller writes Status or Receipt/i,
|
|
328
|
-
/do not research, replan, or add a routine user gate/i,
|
|
329
262
|
]) assert.match(develop, clause);
|
|
330
263
|
assert.match(quality, /consecutive Head captures are identical/i);
|
|
331
264
|
assert.match(quality, /skip, skipped, todo, cancel, canceled, cancelled/i);
|
|
332
|
-
assert.match(quality, /exact command, current Head, required proof level, oracle/i);
|
|
333
265
|
assert.match(waves, /every commit[\s\S]*complete changed tree/i);
|
|
334
|
-
assert.match(waves, /handoff metadata cannot substitute/i);
|
|
335
266
|
assert.match(develop, /later explicit non-Flash invocation may recover/i);
|
|
336
267
|
assert.match(dispatch, /controller alone writes Status and inline Receipt/i);
|
|
337
268
|
|
|
@@ -398,34 +329,11 @@ test('Develop process-first source contract preserves selection, recovery, final
|
|
|
398
329
|
}
|
|
399
330
|
});
|
|
400
331
|
|
|
401
|
-
test('Claude installed Develop preserves plan-native execution and references', () => {
|
|
402
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-claude-develop-'));
|
|
403
|
-
try {
|
|
404
|
-
const result = installClaude(root);
|
|
405
|
-
assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
|
|
406
|
-
const sourceRoot = path.dirname(DEVELOP);
|
|
407
|
-
const installedRoot = path.join(root, '.claude/skills/develop');
|
|
408
|
-
for (const relative of ['SKILL.md', ...DEVELOP_REFERENCES.map((name) => `references/${name}`)]) {
|
|
409
|
-
assert.equal(
|
|
410
|
-
fs.readFileSync(path.join(installedRoot, relative), 'utf8'),
|
|
411
|
-
fs.readFileSync(path.join(sourceRoot, relative), 'utf8'),
|
|
412
|
-
`Claude Develop projection drifted: ${relative}`,
|
|
413
|
-
);
|
|
414
|
-
}
|
|
415
|
-
assert.equal(fs.existsSync(path.join(root, '.agents/skills/develop')), false);
|
|
416
|
-
} finally {
|
|
417
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
418
|
-
}
|
|
419
|
-
});
|
|
420
|
-
|
|
421
332
|
test('Test process-first handoff preserves proof ownership and side-effect boundaries', () => {
|
|
422
333
|
const skill = read(TEST_SKILL);
|
|
423
334
|
const strategy = read(path.join(TEST_SOURCE_ROOT, 'references/execution-strategy.md'));
|
|
424
335
|
const runner = read(TEST_RUNNER);
|
|
425
336
|
const review = read(CODE_REVIEW_SKILL);
|
|
426
|
-
const developConsumers = [read(DEVELOP), read(GATE), read(path.join(
|
|
427
|
-
path.dirname(PARALLEL_WAVES), 'subagent-patterns.md',
|
|
428
|
-
))].join('\n');
|
|
429
337
|
|
|
430
338
|
const exactKeys = [
|
|
431
339
|
'schema_version', 'target', 'verdict', 'command', 'exit', 'counts',
|
|
@@ -433,24 +341,9 @@ test('Test process-first handoff preserves proof ownership and side-effect bound
|
|
|
433
341
|
'artifacts', 'branches', 'raw_output', 'redactions', 'payload_sha256',
|
|
434
342
|
];
|
|
435
343
|
for (const key of exactKeys) assert.match(strategy, new RegExp(`\\b${key}\\b`));
|
|
436
|
-
const developHandoffFields = new Map([
|
|
437
|
-
['command', /Verification handoff separately contains the fresh command/i],
|
|
438
|
-
['exit', /fresh command, exit/i],
|
|
439
|
-
['branches', /named probes/i],
|
|
440
|
-
['counts', /counts, output/i],
|
|
441
|
-
['raw_output', /counts, output/i],
|
|
442
|
-
['provenance', /observed Head/i],
|
|
443
|
-
['reachability', /runtime reachability/i],
|
|
444
|
-
['proof_level', /required proof level/i],
|
|
445
|
-
]);
|
|
446
|
-
for (const [field, sourceClause] of developHandoffFields) {
|
|
447
|
-
assert.match(developConsumers, sourceClause, `Develop must currently consume ${field}`);
|
|
448
|
-
assert.ok(exactKeys.includes(field), `test-proof-v1 must preserve Develop field ${field}`);
|
|
449
|
-
}
|
|
450
344
|
assert.match(runner, /Emit only `schema_version: "test-proof-v1"`/);
|
|
451
345
|
assert.match(runner, /sole writer of[\s\S]*Status and inline Receipt/i);
|
|
452
346
|
assert.match(review, /consume only the controller-validated[\s\S]*`test-proof-v1`[\s\S]*handoff/i);
|
|
453
|
-
assert.match(review, /Never create or search for a separate[\s\S]*process-first receipt/i);
|
|
454
347
|
assert.doesNotMatch([skill, strategy, runner].join('\n'), /npm\s+install|inject-auth\.js|--cookies\s|Bearer eyJ/);
|
|
455
348
|
|
|
456
349
|
const stable = (value) => {
|
|
@@ -534,18 +427,24 @@ test('Test process-first handoff preserves proof ownership and side-effect bound
|
|
|
534
427
|
assert.equal(fs.existsSync(externalTemp), false, 'Test-owned external temp must be cleaned');
|
|
535
428
|
});
|
|
536
429
|
|
|
537
|
-
test('Claude installed Test
|
|
538
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-claude-test-
|
|
430
|
+
test('Claude installed Develop and Test preserve plan-native execution, proof and references', () => {
|
|
431
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-claude-develop-test-'));
|
|
539
432
|
try {
|
|
540
433
|
const result = installClaude(root);
|
|
541
434
|
assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
|
|
542
|
-
const
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
435
|
+
const projections = [
|
|
436
|
+
['develop', path.dirname(DEVELOP), DEVELOP_REFERENCES],
|
|
437
|
+
['test', TEST_SOURCE_ROOT, TEST_REFERENCES],
|
|
438
|
+
];
|
|
439
|
+
for (const [name, sourceRoot, references] of projections) {
|
|
440
|
+
for (const relative of ['SKILL.md', ...references.map((ref) => `references/${ref}`)]) {
|
|
441
|
+
assert.equal(
|
|
442
|
+
fs.readFileSync(path.join(root, '.claude/skills', name, relative), 'utf8'),
|
|
443
|
+
fs.readFileSync(path.join(sourceRoot, relative), 'utf8'),
|
|
444
|
+
`Claude ${name} projection drifted: ${relative}`,
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
assert.equal(fs.existsSync(path.join(root, '.agents/skills', name)), false);
|
|
549
448
|
}
|
|
550
449
|
const installedRunner = fs.readFileSync(path.join(root, '.claude/agents/test-runner.md'), 'utf8');
|
|
551
450
|
assert.equal(installedRunner, fs.readFileSync(TEST_RUNNER, 'utf8'));
|
|
@@ -553,27 +452,11 @@ test('Claude installed Test preserves plan-native proof and references', () => {
|
|
|
553
452
|
const installedReview = fs.readFileSync(path.join(root, '.claude/skills/code-review/SKILL.md'), 'utf8');
|
|
554
453
|
assert.equal(installedReview, fs.readFileSync(CODE_REVIEW_SKILL, 'utf8'));
|
|
555
454
|
assert.match(installedReview, /controller-validated `test-proof-v1`/);
|
|
556
|
-
assert.match(installedReview, /For proof consumption, this skill's `## Execution-proof boundary` is\s+authoritative/i);
|
|
557
|
-
assert.equal(fs.existsSync(path.join(root, '.agents/skills/test')), false);
|
|
558
455
|
} finally {
|
|
559
456
|
fs.rmSync(root, { recursive: true, force: true });
|
|
560
457
|
}
|
|
561
458
|
});
|
|
562
459
|
|
|
563
|
-
test('specs-usage-guide documents Test proof handoff without timing claims', () => {
|
|
564
|
-
const guide = read(SPECS_USAGE_GUIDE);
|
|
565
|
-
const section = guide.split('## Test proof handoff')[1]?.split('### Receipt canonical')[0] || '';
|
|
566
|
-
assert.match(section, /`test-proof-v1`/);
|
|
567
|
-
assert.match(section, /Test không ghi `Status:` hay inline[\s\S]*Develop controller là writer duy nhất/i);
|
|
568
|
-
assert.match(section, /tracked\/untracked\/ignored drift/i);
|
|
569
|
-
assert.match(section, /HTTPS\/localhost origin[\s\S]*fresh consent/i);
|
|
570
|
-
assert.match(section, /chỉ nằm trong `## Receipt` inline/i);
|
|
571
|
-
assert.match(section, /không phải benchmark thời gian/i);
|
|
572
|
-
assert.match(section, /không chứng minh live adherence/i);
|
|
573
|
-
assert.doesNotMatch(section, /\b\d+(?:\.\d+)?\s*(?:ms|milliseconds?|seconds?|minutes?)\b/i);
|
|
574
|
-
assert.doesNotMatch(section, /\[(?:LIVE_)?VERIFIED\]/i);
|
|
575
|
-
});
|
|
576
|
-
|
|
577
460
|
test('first-run cache bypass is blocked - canonical receipt required even without cache', () => {
|
|
578
461
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-first-run-'));
|
|
579
462
|
const claude = path.join(root, '.claude');
|
|
@@ -696,13 +579,13 @@ test('runtime provenance derives exact Git evidence, CLI context, and stale/forg
|
|
|
696
579
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-provenance-adversarial-'));
|
|
697
580
|
const specsRoot = path.join(root, 'specs');
|
|
698
581
|
const featureRoot = path.join(specsRoot, 'demo');
|
|
699
|
-
const specFile = path.join(featureRoot, '
|
|
582
|
+
const specFile = path.join(featureRoot, 'plan.md');
|
|
700
583
|
const sourceFile = path.join(root, 'src', 'app.js');
|
|
701
584
|
const renameSource = path.join(root, 'src', 'rename-old.js');
|
|
702
585
|
const renameTarget = path.join(root, 'src', 'rename-new.js');
|
|
703
586
|
fs.mkdirSync(path.dirname(sourceFile), { recursive: true });
|
|
704
587
|
fs.mkdirSync(featureRoot, { recursive: true });
|
|
705
|
-
fs.writeFileSync(specFile,
|
|
588
|
+
fs.writeFileSync(specFile, '# Demo\n');
|
|
706
589
|
fs.writeFileSync(sourceFile, 'original source\n');
|
|
707
590
|
fs.writeFileSync(renameSource, 'rename source\n');
|
|
708
591
|
fs.writeFileSync(path.join(root, '.gitignore'), 'ignored-secret.env\nnode_modules/\n.cache/\n');
|
|
@@ -734,13 +617,8 @@ test('runtime provenance derives exact Git evidence, CLI context, and stale/forg
|
|
|
734
617
|
'valid-shaped arbitrary SHA values must not bind',
|
|
735
618
|
);
|
|
736
619
|
|
|
737
|
-
fs.writeFileSync(specFile,
|
|
738
|
-
|
|
739
|
-
feature_name: 'demo',
|
|
740
|
-
task_registry: {},
|
|
741
|
-
receipt_refresh: true,
|
|
742
|
-
}));
|
|
743
|
-
const specCommit = spawnSync('git', ['-C', root, 'add', 'specs/demo/spec.json'], { encoding: 'utf8' });
|
|
620
|
+
fs.writeFileSync(specFile, '# Demo\n\nrefreshed\n');
|
|
621
|
+
const specCommit = spawnSync('git', ['-C', root, 'add', 'specs/demo/plan.md'], { encoding: 'utf8' });
|
|
744
622
|
assert.equal(specCommit.status, 0, specCommit.stderr);
|
|
745
623
|
const commitReceipt = spawnSync('git', ['-C', root, 'commit', '-qm', 'refresh spec receipt'], { encoding: 'utf8' });
|
|
746
624
|
assert.equal(commitReceipt.status, 0, commitReceipt.stderr);
|
|
@@ -826,9 +704,9 @@ test('runtime provenance derives exact Git evidence, CLI context, and stale/forg
|
|
|
826
704
|
}
|
|
827
705
|
|
|
828
706
|
const nonGit = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-provenance-non-git-'));
|
|
829
|
-
const nonGitSpec = path.join(nonGit, 'specs', 'demo', '
|
|
707
|
+
const nonGitSpec = path.join(nonGit, 'specs', 'demo', 'plan.md');
|
|
830
708
|
fs.mkdirSync(path.dirname(nonGitSpec), { recursive: true });
|
|
831
|
-
fs.writeFileSync(nonGitSpec,
|
|
709
|
+
fs.writeFileSync(nonGitSpec, '# Demo\n');
|
|
832
710
|
try {
|
|
833
711
|
const blocked = spawnSync(process.execPath, [
|
|
834
712
|
path.join(PACKAGE_ROOT, 'src/claude/scripts/provenance.cjs'), '--json',
|
|
@@ -846,15 +724,15 @@ test('runtime provenance chooses a deterministic root when all history is Specs-
|
|
|
846
724
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-provenance-multi-root-'));
|
|
847
725
|
const specsRoot = path.join(root, 'specs');
|
|
848
726
|
const featureRoot = path.join(specsRoot, 'demo');
|
|
849
|
-
const specFile = path.join(featureRoot, '
|
|
727
|
+
const specFile = path.join(featureRoot, 'plan.md');
|
|
850
728
|
try {
|
|
851
729
|
fs.mkdirSync(featureRoot, { recursive: true });
|
|
852
|
-
fs.writeFileSync(specFile,
|
|
730
|
+
fs.writeFileSync(specFile, '# Demo\n');
|
|
853
731
|
for (const args of [
|
|
854
732
|
['init', '-q', '-b', 'first-root'],
|
|
855
733
|
['config', 'user.email', 'cafekit@example.invalid'],
|
|
856
734
|
['config', 'user.name', 'CafeKit Test'],
|
|
857
|
-
['add', 'specs/demo/
|
|
735
|
+
['add', 'specs/demo/plan.md'],
|
|
858
736
|
['commit', '-qm', 'first specs root'],
|
|
859
737
|
['checkout', '--orphan', 'unrelated-specs'],
|
|
860
738
|
['rm', '-qrf', '.'],
|
|
@@ -862,10 +740,10 @@ test('runtime provenance chooses a deterministic root when all history is Specs-
|
|
|
862
740
|
const result = spawnSync('git', ['-C', root, ...args], { encoding: 'utf8' });
|
|
863
741
|
assert.equal(result.status, 0, result.stderr);
|
|
864
742
|
}
|
|
865
|
-
const unrelatedSpec = path.join(specsRoot, 'unrelated', '
|
|
743
|
+
const unrelatedSpec = path.join(specsRoot, 'unrelated', 'plan.md');
|
|
866
744
|
fs.mkdirSync(path.dirname(unrelatedSpec), { recursive: true });
|
|
867
|
-
fs.writeFileSync(unrelatedSpec,
|
|
868
|
-
let result = spawnSync('git', ['-C', root, 'add', 'specs/unrelated/
|
|
745
|
+
fs.writeFileSync(unrelatedSpec, '# Unrelated\n');
|
|
746
|
+
let result = spawnSync('git', ['-C', root, 'add', 'specs/unrelated/plan.md'], { encoding: 'utf8' });
|
|
869
747
|
assert.equal(result.status, 0, result.stderr);
|
|
870
748
|
result = spawnSync('git', ['-C', root, 'commit', '-qm', 'second specs root'], { encoding: 'utf8' });
|
|
871
749
|
assert.equal(result.status, 0, result.stderr);
|
|
@@ -889,56 +767,26 @@ test('runtime provenance chooses a deterministic root when all history is Specs-
|
|
|
889
767
|
});
|
|
890
768
|
|
|
891
769
|
test('parallel waves require immutable provenance receipts and safe recovery', () => {
|
|
892
|
-
const waves =
|
|
893
|
-
assert.match(waves, /only for explicit parallel execution/i);
|
|
894
|
-
assert.match(waves, /isolated worktree support[\s\S]*fall back to the sequential loop/i);
|
|
770
|
+
const waves = read(PARALLEL_WAVES);
|
|
895
771
|
assert.match(waves, /Clamp `--parallel N` to 1\.\.5; default to 3/i);
|
|
896
772
|
assert.match(waves, /clean destination[\s\S]*base_sha[\s\S]*incompatible bases/i);
|
|
897
|
-
assert.match(waves, /consent for worker commits and controller cherry-picks/i);
|
|
898
773
|
for (const field of ['base_sha', 'head_sha', 'branch', 'worktree_path', 'commit_range']) {
|
|
899
774
|
assert.match(waves, new RegExp(`\\b${field}\\b`));
|
|
900
775
|
}
|
|
901
|
-
assert.match(waves, /git rev-list --reverse <base_sha>\.\.<head_sha>/i);
|
|
902
|
-
assert.match(waves, /git diff --name-status <base_sha>\.\.<head_sha>/i);
|
|
903
|
-
assert.match(waves, /code\/spec repair[\s\S]*new commit/i);
|
|
904
|
-
assert.match(waves, /dependencies from the plan table and each task/i);
|
|
905
|
-
for (const sharedSurface of ['registry', 'lockfile', 'manifest', 'generated output', 'migration', 'export barrel']) {
|
|
906
|
-
assert.match(waves, new RegExp(sharedSurface, 'i'));
|
|
907
|
-
}
|
|
908
776
|
assert.match(waves, /One file has one writer per wave/i);
|
|
909
777
|
assert.match(waves, /only the controller writes state\/proof/i);
|
|
910
778
|
assert.match(waves, /commit only with user authorization/i);
|
|
911
|
-
assert.match(waves, /blocked or failed worker keeps[\s\S]*worktree and branch/i);
|
|
912
779
|
assert.match(waves, /Never[\s\S]*force-delete[\s\S]*merge success or explicit discard authorization/i);
|
|
913
|
-
assert.match(waves, /git cherry-pick <next-worker-commit>/i);
|
|
914
780
|
assert.match(waves, /On conflict, abort the pick[\s\S]*retain its branch\/worktree/i);
|
|
915
781
|
assert.match(waves, /cleanup_authorization: merged-release/i);
|
|
916
|
-
assert.match(waves, /affected integration|final.*integration/i);
|
|
917
|
-
for (const category of ['baseline', 'environment', 'spec', 'code']) {
|
|
918
|
-
assert.match(waves, new RegExp(`\\b${category}\\b`));
|
|
919
|
-
}
|
|
920
782
|
assert.match(waves, /Do not blind-retry/i);
|
|
921
783
|
});
|
|
922
784
|
|
|
923
|
-
test('provenance ledger
|
|
785
|
+
test('provenance ledger forbids copied source text and the shipped runtime carries none', () => {
|
|
924
786
|
assert.equal(fs.existsSync(PROVENANCE), true);
|
|
925
787
|
const provenance = read(PROVENANCE);
|
|
926
|
-
assert.match(provenance, /\bidea\b/);
|
|
927
788
|
assert.match(provenance, /\bclean-room\b/);
|
|
928
|
-
assert.match(provenance, /copied-text.*never valid|never valid.*copied-text/i);
|
|
929
789
|
assert.match(provenance, /never copy source text verbatim/i);
|
|
930
|
-
for (const column of ['Pattern', 'Source anchor', 'Reuse type', 'CafeKit destination', 'Evidence/status']) {
|
|
931
|
-
assert.match(provenance, new RegExp(`\\b${column.replace('/', '\\/')}\\b`, 'i'));
|
|
932
|
-
}
|
|
933
|
-
assert.match(provenance, /AgentKit|cafekit-ref/);
|
|
934
|
-
assert.match(provenance, /before implementation/i);
|
|
935
|
-
// H1 remediation: distinguish survey vs verified implementation, fail unsupported claims
|
|
936
|
-
assert.match(provenance, /external survey|not committed/i);
|
|
937
|
-
assert.match(provenance, /survey only|not used|No borrowed text recorded/i);
|
|
938
|
-
assert.match(provenance, /Source anchor.*plans\//);
|
|
939
|
-
assert.match(provenance, /Evidence\/status/i);
|
|
940
|
-
assert.match(provenance, /grep -r.*cafekit-ref/i);
|
|
941
|
-
assert.doesNotMatch(provenance, /AgentKit T1.*implemented as direct source/i);
|
|
942
790
|
// shipped runtime must not contain cafekit-ref/AgentKit verbatim (only ledger/plans may)
|
|
943
791
|
const shippedHits = spawnSync('grep', ['-rn', 'cafekit-ref', 'packages/spec/src', '--include=*.md', '--include=*.cjs', '--include=*.ts'], { encoding: 'utf8' });
|
|
944
792
|
const agentKitHits = spawnSync('grep', ['-rn', 'AgentKit', 'packages/spec/src', '--include=*.md', '--include=*.cjs', '--include=*.ts'], { encoding: 'utf8' });
|
|
@@ -946,628 +794,6 @@ test('provenance ledger defines reuse contract and source anchors', () => {
|
|
|
946
794
|
assert.equal(agentKitHits.stdout.trim(), '', 'shipped runtime must not contain AgentKit verbatim');
|
|
947
795
|
});
|
|
948
796
|
|
|
949
|
-
const BENCHMARK_SCRIPT = path.join(PACKAGE_ROOT, 'scripts/benchmark-workflow.mjs');
|
|
950
|
-
const BENCHMARK_CORPUS_SCHEMA = path.join(PACKAGE_ROOT, 'benchmarks/corpus.schema.json');
|
|
951
|
-
const BENCHMARK_CONFIG_EXAMPLE = path.join(PACKAGE_ROOT, 'benchmarks/benchmark-config.example.json');
|
|
952
|
-
const BENCHMARK_RUBRIC = path.join(PACKAGE_ROOT, 'benchmarks/rubric.md');
|
|
953
|
-
const BENCHMARK_DOCS = path.join(PACKAGE_ROOT, '../../docs/benchmark-workflow.md');
|
|
954
|
-
|
|
955
|
-
function canonicalBenchmark(value) {
|
|
956
|
-
if (Array.isArray(value)) return `[${value.map(canonicalBenchmark).join(',')}]`;
|
|
957
|
-
if (value && typeof value === 'object') return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalBenchmark(value[key])}`).join(',')}}`;
|
|
958
|
-
return JSON.stringify(value);
|
|
959
|
-
}
|
|
960
|
-
|
|
961
|
-
function benchmarkHash(value) {
|
|
962
|
-
const crypto = require('node:crypto');
|
|
963
|
-
return `sha256:${crypto.createHash('sha256').update(canonicalBenchmark(value)).digest('hex')}`;
|
|
964
|
-
}
|
|
965
|
-
|
|
966
|
-
function benchmarkRawHash(value) {
|
|
967
|
-
return `sha256:${crypto.createHash('sha256').update(value).digest('hex')}`;
|
|
968
|
-
}
|
|
969
|
-
|
|
970
|
-
function writeBenchmarkConfig(file, config) {
|
|
971
|
-
const frozen = { ...config };
|
|
972
|
-
delete frozen.config_sha256;
|
|
973
|
-
frozen.config_sha256 = benchmarkHash(frozen);
|
|
974
|
-
fs.writeFileSync(file, JSON.stringify(frozen));
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
function makeBenchmarkFixture(root) {
|
|
978
|
-
const corpus = {
|
|
979
|
-
schema_version: 'b1.v1', status: 'frozen', corpus_id: 'fixture-only-corpus',
|
|
980
|
-
tasks: [
|
|
981
|
-
{ task_id: 'fixture-direct', lane: 'Direct', prompt: 'fixture-only reversible task', repo_sample: 'fixture-repo', acceptance: { criteria: ['fixture criterion'] }, risk: { level: 'low' } },
|
|
982
|
-
{ task_id: 'fixture-critical', lane: 'Critical', prompt: 'fixture-only negative control', repo_sample: 'fixture-repo', acceptance: { criteria: ['fixture contract'] }, risk: { level: 'high', reasons: ['fixture negative control'] } },
|
|
983
|
-
],
|
|
984
|
-
};
|
|
985
|
-
const corpusSha = benchmarkHash(corpus);
|
|
986
|
-
const common = {
|
|
987
|
-
schema_version: 'b1.v1', status: 'frozen', experiment_id: 'fixture-only-experiment',
|
|
988
|
-
model: { name: 'fixture-model', version: 'fixture-version' }, reasoning_effort: 'fixture',
|
|
989
|
-
repo: { identifier: 'fixture-repo', commit: 'abc1234567890', clean_initial_tree_sha: benchmarkHash('fixture-clean-tree') },
|
|
990
|
-
permissions_fingerprint: benchmarkHash('fixture-permissions'), tool_availability_fingerprint: benchmarkHash('fixture-tools'),
|
|
991
|
-
corpus_sha256: corpusSha, repeat_policy: { repeats_per_task: 2, context_isolated: true },
|
|
992
|
-
input_usd_per_1k: 1, output_usd_per_1k: 2,
|
|
993
|
-
};
|
|
994
|
-
const configs = ['baseline', 'treatment'].map((arm) => {
|
|
995
|
-
const config = { ...common, arm };
|
|
996
|
-
config.config_sha256 = benchmarkHash(config);
|
|
997
|
-
return config;
|
|
998
|
-
});
|
|
999
|
-
const artifactRef = 'fixture-artifact.bin';
|
|
1000
|
-
const artifactBytes = Buffer.from('fixture-only benchmark artifact\n');
|
|
1001
|
-
fs.writeFileSync(path.join(root, artifactRef), artifactBytes);
|
|
1002
|
-
const artifactSha = benchmarkRawHash(artifactBytes);
|
|
1003
|
-
const receipts = [];
|
|
1004
|
-
for (const config of configs) for (const task of corpus.tasks) for (const repeat of [1, 2]) {
|
|
1005
|
-
const treatment = config.arm === 'treatment';
|
|
1006
|
-
const critical = task.lane === 'Critical';
|
|
1007
|
-
const wall = critical ? 300 : (treatment ? 50 : 100) * repeat;
|
|
1008
|
-
const input = (treatment ? 100 : 200) * repeat;
|
|
1009
|
-
const output = (treatment ? 50 : 100) * repeat;
|
|
1010
|
-
receipts.push({
|
|
1011
|
-
task_id: task.task_id, lane: task.lane, arm: config.arm, repeat,
|
|
1012
|
-
model: config.model, reasoning_effort: config.reasoning_effort,
|
|
1013
|
-
repo_commit: config.repo.commit, clean_initial_tree_sha: config.repo.clean_initial_tree_sha,
|
|
1014
|
-
permissions_fingerprint: config.permissions_fingerprint, tool_availability_fingerprint: config.tool_availability_fingerprint,
|
|
1015
|
-
corpus_sha256: config.corpus_sha256, config_sha256: config.config_sha256,
|
|
1016
|
-
wall_ms: wall, input_tokens: input, output_tokens: output, context_loaded_tokens: 500,
|
|
1017
|
-
tool_calls: 2, subagent_calls: critical ? 1 : 0, correctness: true, regression: false,
|
|
1018
|
-
unsupported_completion_claim: false, user_corrections: 0, useful_reviewer_findings: 1,
|
|
1019
|
-
false_positive_reviewer_findings: 0,
|
|
1020
|
-
evidence: { artifact_ref: artifactRef, artifact_sha256: artifactSha, command: 'fixture-only command' },
|
|
1021
|
-
});
|
|
1022
|
-
}
|
|
1023
|
-
const paths = { corpus: path.join(root, 'corpus.json'), receipts: path.join(root, 'receipts.json') };
|
|
1024
|
-
fs.writeFileSync(paths.corpus, JSON.stringify(corpus));
|
|
1025
|
-
fs.writeFileSync(paths.receipts, JSON.stringify({ receipts }));
|
|
1026
|
-
paths.configs = configs.map((config, index) => {
|
|
1027
|
-
const file = path.join(root, `${config.arm}-${index}.json`);
|
|
1028
|
-
fs.writeFileSync(file, JSON.stringify(config));
|
|
1029
|
-
return file;
|
|
1030
|
-
});
|
|
1031
|
-
return paths;
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
test('B1 benchmark artifacts expose bounded CLI contract', () => {
|
|
1035
|
-
for (const file of [BENCHMARK_SCRIPT, BENCHMARK_CORPUS_SCHEMA, BENCHMARK_CONFIG_EXAMPLE, BENCHMARK_RUBRIC, BENCHMARK_DOCS]) assert.equal(fs.existsSync(file), true, file);
|
|
1036
|
-
assert.match(read(BENCHMARK_DOCS), /live baseline.*treatment.*pending/i);
|
|
1037
|
-
assert.match(read(BENCHMARK_DOCS), /`npm test`.*workflow correctness/i);
|
|
1038
|
-
assert.match(read(BENCHMARK_CONFIG_EXAMPLE), /TEMPLATE ONLY|example.template/i);
|
|
1039
|
-
assert.match(read(BENCHMARK_RUBRIC), /Direct/);
|
|
1040
|
-
assert.match(read(BENCHMARK_RUBRIC), /Standard/);
|
|
1041
|
-
assert.match(read(BENCHMARK_RUBRIC), /Critical/);
|
|
1042
|
-
});
|
|
1043
|
-
|
|
1044
|
-
test('B1 validator rejects missing freeze fields and accepts fixture-only corpus', () => {
|
|
1045
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-validation-'));
|
|
1046
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1047
|
-
try {
|
|
1048
|
-
assert.equal(JSON.parse(read(fixture.corpus)).status, 'frozen');
|
|
1049
|
-
const valid = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', fixture.receipts], { encoding: 'utf8' });
|
|
1050
|
-
assert.equal(valid.status, 0, `${valid.stdout}\n${valid.stderr}`);
|
|
1051
|
-
assert.equal(JSON.parse(valid.stdout).status, 'valid');
|
|
1052
|
-
const invalidConfig = JSON.parse(read(fixture.configs[0]));
|
|
1053
|
-
delete invalidConfig.clean_initial_tree_sha;
|
|
1054
|
-
delete invalidConfig.config_sha256;
|
|
1055
|
-
const invalidPath = path.join(root, 'missing-freeze.json');
|
|
1056
|
-
fs.writeFileSync(invalidPath, JSON.stringify(invalidConfig));
|
|
1057
|
-
const invalid = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', invalidPath], { encoding: 'utf8' });
|
|
1058
|
-
assert.equal(invalid.status, 2);
|
|
1059
|
-
assert.match(invalid.stderr, /missing or placeholder freeze field/);
|
|
1060
|
-
} finally {
|
|
1061
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1062
|
-
}
|
|
1063
|
-
});
|
|
1064
|
-
|
|
1065
|
-
test('B1 validator fails closed for templates, prompts, parity, and artifacts', () => {
|
|
1066
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-integrity-'));
|
|
1067
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1068
|
-
try {
|
|
1069
|
-
const templateCorpus = JSON.parse(read(fixture.corpus));
|
|
1070
|
-
templateCorpus.status = 'example_template';
|
|
1071
|
-
const templatePath = path.join(root, 'template-corpus.json');
|
|
1072
|
-
fs.writeFileSync(templatePath, JSON.stringify(templateCorpus));
|
|
1073
|
-
const template = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', templatePath, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', fixture.receipts], { encoding: 'utf8' });
|
|
1074
|
-
assert.equal(template.status, 2);
|
|
1075
|
-
assert.match(template.stderr, /example_template.*receipt validation.*live summary/);
|
|
1076
|
-
|
|
1077
|
-
const placeholderCorpus = JSON.parse(read(fixture.corpus));
|
|
1078
|
-
placeholderCorpus.tasks[0].prompt = '{{replace_me}}';
|
|
1079
|
-
const placeholderPath = path.join(root, 'placeholder-corpus.json');
|
|
1080
|
-
fs.writeFileSync(placeholderPath, JSON.stringify(placeholderCorpus));
|
|
1081
|
-
const placeholder = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', placeholderPath, '--config', fixture.configs[0]], { encoding: 'utf8' });
|
|
1082
|
-
assert.equal(placeholder.status, 2);
|
|
1083
|
-
assert.match(placeholder.stderr, /tasks\[0\]\.prompt.*missing or placeholder/);
|
|
1084
|
-
|
|
1085
|
-
const emptyPromptWithHashCorpus = JSON.parse(read(fixture.corpus));
|
|
1086
|
-
emptyPromptWithHashCorpus.tasks[0].prompt = '';
|
|
1087
|
-
emptyPromptWithHashCorpus.tasks[0].prompt_sha256 = `sha256:${'1'.repeat(64)}`;
|
|
1088
|
-
const emptyPromptWithHashPath = path.join(root, 'empty-prompt-with-hash-corpus.json');
|
|
1089
|
-
fs.writeFileSync(emptyPromptWithHashPath, JSON.stringify(emptyPromptWithHashCorpus));
|
|
1090
|
-
const emptyPromptWithHash = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', emptyPromptWithHashPath, '--config', fixture.configs[0]], { encoding: 'utf8' });
|
|
1091
|
-
assert.equal(emptyPromptWithHash.status, 2);
|
|
1092
|
-
assert.match(emptyPromptWithHash.stderr, /tasks\[0\]: exactly one of prompt or prompt_sha256 required/);
|
|
1093
|
-
|
|
1094
|
-
const treatment = JSON.parse(read(fixture.configs[1]));
|
|
1095
|
-
treatment.experiment_id = 'different-experiment';
|
|
1096
|
-
const parityPath = path.join(root, 'parity-treatment.json');
|
|
1097
|
-
writeBenchmarkConfig(parityPath, treatment);
|
|
1098
|
-
const parity = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', parityPath], { encoding: 'utf8' });
|
|
1099
|
-
assert.equal(parity.status, 2);
|
|
1100
|
-
assert.match(parity.stderr, /differ only by arm/);
|
|
1101
|
-
|
|
1102
|
-
const receipts = JSON.parse(read(fixture.receipts));
|
|
1103
|
-
const missingArtifactHash = JSON.parse(read(fixture.receipts));
|
|
1104
|
-
delete missingArtifactHash.receipts[0].evidence.artifact_sha256;
|
|
1105
|
-
const missingArtifactHashPath = path.join(root, 'missing-artifact-hash.json');
|
|
1106
|
-
fs.writeFileSync(missingArtifactHashPath, JSON.stringify(missingArtifactHash));
|
|
1107
|
-
const missingHash = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', missingArtifactHashPath], { encoding: 'utf8' });
|
|
1108
|
-
assert.equal(missingHash.status, 2);
|
|
1109
|
-
assert.match(missingHash.stderr, /evidence\.artifact_sha256.*missing or placeholder/);
|
|
1110
|
-
|
|
1111
|
-
receipts.receipts[0].evidence.artifact_sha256 = benchmarkRawHash(Buffer.from('wrong artifact'));
|
|
1112
|
-
const wrongHashPath = path.join(root, 'wrong-artifact-hash.json');
|
|
1113
|
-
fs.writeFileSync(wrongHashPath, JSON.stringify(receipts));
|
|
1114
|
-
const wrongHash = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', wrongHashPath], { encoding: 'utf8' });
|
|
1115
|
-
assert.equal(wrongHash.status, 2);
|
|
1116
|
-
assert.match(wrongHash.stderr, /artifact_sha256.*raw-byte hash mismatch/);
|
|
1117
|
-
|
|
1118
|
-
receipts.receipts[0].evidence.artifact_ref = 'https://example.invalid/artifact.json';
|
|
1119
|
-
const uriPath = path.join(root, 'uri-artifact.json');
|
|
1120
|
-
fs.writeFileSync(uriPath, JSON.stringify(receipts));
|
|
1121
|
-
const uri = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', uriPath], { encoding: 'utf8' });
|
|
1122
|
-
assert.equal(uri.status, 2);
|
|
1123
|
-
assert.match(uri.stderr, /artifact_ref.*relative local file.*URI/);
|
|
1124
|
-
|
|
1125
|
-
const outsideArtifact = path.join(path.dirname(root), `${path.basename(root)}-outside.bin`);
|
|
1126
|
-
fs.writeFileSync(outsideArtifact, Buffer.from('outside receipt bundle'));
|
|
1127
|
-
const traversalReceipts = JSON.parse(read(fixture.receipts));
|
|
1128
|
-
traversalReceipts.receipts[0].evidence.artifact_ref = path.relative(root, outsideArtifact);
|
|
1129
|
-
traversalReceipts.receipts[0].evidence.artifact_sha256 = benchmarkRawHash(fs.readFileSync(outsideArtifact));
|
|
1130
|
-
const traversalPath = path.join(root, 'traversal-artifact.json');
|
|
1131
|
-
fs.writeFileSync(traversalPath, JSON.stringify(traversalReceipts));
|
|
1132
|
-
const traversal = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', traversalPath], { encoding: 'utf8' });
|
|
1133
|
-
assert.equal(traversal.status, 2);
|
|
1134
|
-
assert.match(traversal.stderr, /artifact_ref.*path escapes receipt bundle directory/);
|
|
1135
|
-
fs.rmSync(outsideArtifact, { force: true });
|
|
1136
|
-
} finally {
|
|
1137
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1138
|
-
}
|
|
1139
|
-
});
|
|
1140
|
-
|
|
1141
|
-
test('B1 summary groups fixture-only receipts by arm/lane with deterministic quantiles', () => {
|
|
1142
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-summary-'));
|
|
1143
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1144
|
-
try {
|
|
1145
|
-
const result = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', fixture.receipts], { encoding: 'utf8' });
|
|
1146
|
-
assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
|
|
1147
|
-
const summary = JSON.parse(result.stdout);
|
|
1148
|
-
assert.equal(summary.status, 'ready');
|
|
1149
|
-
assert.equal(summary.groups.baseline.Direct.metrics.wall_ms.median, 150);
|
|
1150
|
-
assert.equal(summary.groups.baseline.Direct.metrics.wall_ms.p25, 125);
|
|
1151
|
-
assert.equal(summary.groups.baseline.Direct.metrics.wall_ms.p75, 175);
|
|
1152
|
-
assert.equal(summary.groups.treatment.Direct.quality_rates.correctness, 1);
|
|
1153
|
-
assert.equal(summary.rollout_recommendation.gates.Critical.pass, true);
|
|
1154
|
-
|
|
1155
|
-
const countAggregation = JSON.parse(read(fixture.receipts));
|
|
1156
|
-
const countedReceipt = countAggregation.receipts.find((item) => item.arm === 'baseline' && item.lane === 'Direct' && item.repeat === 1);
|
|
1157
|
-
countedReceipt.user_corrections = 2;
|
|
1158
|
-
countedReceipt.useful_reviewer_findings = 3;
|
|
1159
|
-
countedReceipt.false_positive_reviewer_findings = 4;
|
|
1160
|
-
const countAggregationPath = path.join(root, 'count-aggregation.json');
|
|
1161
|
-
fs.writeFileSync(countAggregationPath, JSON.stringify(countAggregation));
|
|
1162
|
-
const countAggregationResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', countAggregationPath], { encoding: 'utf8' });
|
|
1163
|
-
assert.equal(countAggregationResult.status, 0, `${countAggregationResult.stdout}\n${countAggregationResult.stderr}`);
|
|
1164
|
-
const countAggregationSummary = JSON.parse(countAggregationResult.stdout);
|
|
1165
|
-
assert.equal(countAggregationSummary.groups.baseline.Direct.quality_rates.user_correction_rate, 1);
|
|
1166
|
-
assert.equal(countAggregationSummary.groups.baseline.Direct.quality_rates.useful_reviewer_finding_rate, 2);
|
|
1167
|
-
assert.equal(countAggregationSummary.groups.baseline.Direct.quality_rates.false_positive_reviewer_finding_rate, 2);
|
|
1168
|
-
|
|
1169
|
-
const degraded = JSON.parse(read(fixture.receipts));
|
|
1170
|
-
for (const receipt of degraded.receipts.filter((item) => item.arm === 'treatment')) {
|
|
1171
|
-
receipt.useful_reviewer_findings = 0;
|
|
1172
|
-
receipt.false_positive_reviewer_findings = 1;
|
|
1173
|
-
}
|
|
1174
|
-
const degradedPath = path.join(root, 'degraded-review-quality.json');
|
|
1175
|
-
fs.writeFileSync(degradedPath, JSON.stringify(degraded));
|
|
1176
|
-
const degradedResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', degradedPath], { encoding: 'utf8' });
|
|
1177
|
-
assert.equal(degradedResult.status, 0, `${degradedResult.stdout}\n${degradedResult.stderr}`);
|
|
1178
|
-
const degradedSummary = JSON.parse(degradedResult.stdout);
|
|
1179
|
-
assert.equal(degradedSummary.status, 'not-ready');
|
|
1180
|
-
assert.equal(degradedSummary.rollout_recommendation.gates.Critical.quality_pass, false);
|
|
1181
|
-
assert.equal(degradedSummary.rollout_recommendation.gates.Critical.pass, false);
|
|
1182
|
-
|
|
1183
|
-
const lowRiskQualityFailure = JSON.parse(read(fixture.receipts));
|
|
1184
|
-
for (const receipt of lowRiskQualityFailure.receipts.filter((item) => item.lane === 'Direct')) receipt.correctness = false;
|
|
1185
|
-
const lowRiskQualityFailurePath = path.join(root, 'low-risk-quality-failure.json');
|
|
1186
|
-
fs.writeFileSync(lowRiskQualityFailurePath, JSON.stringify(lowRiskQualityFailure));
|
|
1187
|
-
const lowRiskQualityFailureResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', lowRiskQualityFailurePath], { encoding: 'utf8' });
|
|
1188
|
-
assert.equal(lowRiskQualityFailureResult.status, 0, `${lowRiskQualityFailureResult.stdout}\n${lowRiskQualityFailureResult.stderr}`);
|
|
1189
|
-
const lowRiskQualityFailureSummary = JSON.parse(lowRiskQualityFailureResult.stdout);
|
|
1190
|
-
assert.equal(lowRiskQualityFailureSummary.rollout_recommendation.gates.Direct.quality_pass, false);
|
|
1191
|
-
assert.equal(lowRiskQualityFailureSummary.rollout_recommendation.gates.Direct.pass, false);
|
|
1192
|
-
assert.equal(lowRiskQualityFailureSummary.rollout_recommendation.gates.Critical.pass, true);
|
|
1193
|
-
} finally {
|
|
1194
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1195
|
-
}
|
|
1196
|
-
});
|
|
1197
|
-
|
|
1198
|
-
test('B1 rollout rejects incomplete fixture-only lane/repeat matrices', () => {
|
|
1199
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-completeness-'));
|
|
1200
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1201
|
-
try {
|
|
1202
|
-
const payload = JSON.parse(read(fixture.receipts));
|
|
1203
|
-
const completeReceipts = payload.receipts;
|
|
1204
|
-
const partialPath = path.join(root, 'partial.json');
|
|
1205
|
-
fs.writeFileSync(partialPath, JSON.stringify({ receipts: completeReceipts.filter((receipt) => !(receipt.arm === 'treatment' && receipt.lane === 'Direct' && receipt.repeat === 1)) }));
|
|
1206
|
-
const partial = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', partialPath], { encoding: 'utf8' });
|
|
1207
|
-
assert.equal(partial.status, 0, `${partial.stdout}\n${partial.stderr}`);
|
|
1208
|
-
const partialSummary = JSON.parse(partial.stdout);
|
|
1209
|
-
assert.equal(partialSummary.status, 'not-ready');
|
|
1210
|
-
assert.equal(partialSummary.rollout_recommendation.gates.Direct.treatment_matrix.complete, false);
|
|
1211
|
-
const incompleteValidation = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', partialPath], { encoding: 'utf8' });
|
|
1212
|
-
assert.equal(incompleteValidation.status, 2);
|
|
1213
|
-
assert.match(incompleteValidation.stderr, /incomplete receipt matrix.*treatment\/Direct\/fixture-direct\/1/);
|
|
1214
|
-
|
|
1215
|
-
const missingLanePath = path.join(root, 'missing-lane.json');
|
|
1216
|
-
fs.writeFileSync(missingLanePath, JSON.stringify({ receipts: completeReceipts.filter((receipt) => !(receipt.arm === 'treatment' && receipt.lane === 'Critical')) }));
|
|
1217
|
-
const missingLane = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', missingLanePath], { encoding: 'utf8' });
|
|
1218
|
-
assert.equal(missingLane.status, 0, `${missingLane.stdout}\n${missingLane.stderr}`);
|
|
1219
|
-
const missingLaneSummary = JSON.parse(missingLane.stdout);
|
|
1220
|
-
assert.equal(missingLaneSummary.status, 'not-ready');
|
|
1221
|
-
assert.equal(missingLaneSummary.rollout_recommendation.gates.Critical.treatment_matrix.complete, false);
|
|
1222
|
-
|
|
1223
|
-
const outOfRange = JSON.parse(JSON.stringify(completeReceipts));
|
|
1224
|
-
outOfRange[0].repeat = 3;
|
|
1225
|
-
const outOfRangePath = path.join(root, 'out-of-range.json');
|
|
1226
|
-
fs.writeFileSync(outOfRangePath, JSON.stringify({ receipts: outOfRange }));
|
|
1227
|
-
const invalidRepeat = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', outOfRangePath], { encoding: 'utf8' });
|
|
1228
|
-
assert.equal(invalidRepeat.status, 2);
|
|
1229
|
-
assert.match(invalidRepeat.stderr, /exceeds repeat_policy/);
|
|
1230
|
-
|
|
1231
|
-
const duplicateArm = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[0]], { encoding: 'utf8' });
|
|
1232
|
-
assert.equal(duplicateArm.status, 2);
|
|
1233
|
-
assert.match(duplicateArm.stderr, /duplicate arm/);
|
|
1234
|
-
|
|
1235
|
-
const invalidCorpus = JSON.parse(read(fixture.corpus));
|
|
1236
|
-
delete invalidCorpus.corpus_id;
|
|
1237
|
-
const invalidCorpusPath = path.join(root, 'missing-corpus-id.json');
|
|
1238
|
-
fs.writeFileSync(invalidCorpusPath, JSON.stringify(invalidCorpus));
|
|
1239
|
-
const missingCorpusId = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', invalidCorpusPath, '--config', fixture.configs[0]], { encoding: 'utf8' });
|
|
1240
|
-
assert.equal(missingCorpusId.status, 2);
|
|
1241
|
-
assert.match(missingCorpusId.stderr, /corpus_id.*missing or placeholder/);
|
|
1242
|
-
|
|
1243
|
-
const invalidCost = JSON.parse(read(fixture.configs[0]));
|
|
1244
|
-
invalidCost.input_usd_per_1k = 'not-a-rate';
|
|
1245
|
-
const invalidCostPath = path.join(root, 'invalid-cost.json');
|
|
1246
|
-
fs.writeFileSync(invalidCostPath, JSON.stringify(invalidCost));
|
|
1247
|
-
const invalidCostResult = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', invalidCostPath], { encoding: 'utf8' });
|
|
1248
|
-
assert.equal(invalidCostResult.status, 2);
|
|
1249
|
-
assert.match(invalidCostResult.stderr, /config\.input_usd_per_1k.*non-negative number/);
|
|
1250
|
-
} finally {
|
|
1251
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1252
|
-
}
|
|
1253
|
-
});
|
|
1254
|
-
|
|
1255
|
-
test('B1 summary marks no receipts exploratory instead of claiming a run', () => {
|
|
1256
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-empty-'));
|
|
1257
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1258
|
-
try {
|
|
1259
|
-
const result = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0]], { encoding: 'utf8' });
|
|
1260
|
-
assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
|
|
1261
|
-
const summary = JSON.parse(result.stdout);
|
|
1262
|
-
assert.equal(summary.status, 'exploratory/no-live-runs');
|
|
1263
|
-
assert.equal(summary.live_runs, false);
|
|
1264
|
-
} finally {
|
|
1265
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1266
|
-
}
|
|
1267
|
-
});
|
|
1268
|
-
|
|
1269
|
-
test('B1 run requires explicit runner contract and rejects placeholders/shell strings', () => {
|
|
1270
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-run-runner-'));
|
|
1271
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1272
|
-
try {
|
|
1273
|
-
// Missing runner
|
|
1274
|
-
const missing = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--out', path.join(root, 'out-missing')], { encoding: 'utf8' });
|
|
1275
|
-
assert.equal(missing.status, 2);
|
|
1276
|
-
assert.match(missing.stderr, /runner contract is required/);
|
|
1277
|
-
assert.match(missing.stderr, /explicit command array/);
|
|
1278
|
-
|
|
1279
|
-
// Placeholder runner (command contains placeholder)
|
|
1280
|
-
const placeholderRunner = path.join(root, 'placeholder-runner.json');
|
|
1281
|
-
fs.writeFileSync(placeholderRunner, JSON.stringify({ schema_version: 'b1.v1', command: ['node', '{{replace_me}}'] }));
|
|
1282
|
-
const placeholder = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', placeholderRunner, '--out', path.join(root, 'out-placeholder')], { encoding: 'utf8' });
|
|
1283
|
-
assert.equal(placeholder.status, 2);
|
|
1284
|
-
assert.match(placeholder.stderr, /missing or placeholder freeze field/);
|
|
1285
|
-
|
|
1286
|
-
// Shell string forbidden (command as string)
|
|
1287
|
-
const shellRunner = path.join(root, 'shell-runner.json');
|
|
1288
|
-
fs.writeFileSync(shellRunner, JSON.stringify({ schema_version: 'b1.v1', command: 'node runner.mjs' }));
|
|
1289
|
-
const shell = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', shellRunner, '--out', path.join(root, 'out-shell')], { encoding: 'utf8' });
|
|
1290
|
-
assert.equal(shell.status, 2);
|
|
1291
|
-
assert.match(shell.stderr, /explicit argv|shell string is forbidden/);
|
|
1292
|
-
|
|
1293
|
-
// Missing out/receipts
|
|
1294
|
-
const minimalRunner = path.join(root, 'minimal-runner.json');
|
|
1295
|
-
fs.writeFileSync(minimalRunner, JSON.stringify({ schema_version: 'b1.v1', command: ['node', '-e', 'process.exit(0)'] }));
|
|
1296
|
-
const noOut = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', minimalRunner], { encoding: 'utf8' });
|
|
1297
|
-
assert.equal(noOut.status, 2);
|
|
1298
|
-
assert.match(noOut.stderr, /output is required/);
|
|
1299
|
-
} finally {
|
|
1300
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1301
|
-
}
|
|
1302
|
-
});
|
|
1303
|
-
|
|
1304
|
-
test('B1 run executes via explicit argv, captures wall_ms/artifact, and remains honest no-live-runs without execution', () => {
|
|
1305
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-run-execute-'));
|
|
1306
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1307
|
-
try {
|
|
1308
|
-
// Honest no-live-runs: summarize without receipts must stay exploratory
|
|
1309
|
-
const noLive = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'summarize', '--corpus', fixture.corpus, '--config', fixture.configs[0]], { encoding: 'utf8' });
|
|
1310
|
-
assert.equal(noLive.status, 0);
|
|
1311
|
-
const noLiveSummary = JSON.parse(noLive.stdout);
|
|
1312
|
-
assert.equal(noLiveSummary.status, 'exploratory/no-live-runs');
|
|
1313
|
-
assert.equal(noLiveSummary.live_runs, false);
|
|
1314
|
-
assert.equal(noLiveSummary.rollout_recommendation.status, 'exploratory');
|
|
1315
|
-
// Also validate without receipts is valid_no_receipts, not fabricated success
|
|
1316
|
-
const validNoReceipts = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0]], { encoding: 'utf8' });
|
|
1317
|
-
assert.equal(validNoReceipts.status, 0);
|
|
1318
|
-
assert.equal(JSON.parse(validNoReceipts.stdout).status, 'valid_no_receipts');
|
|
1319
|
-
|
|
1320
|
-
// Create minimal deterministic runner that writes artifact and emits metrics JSON
|
|
1321
|
-
const runnerScript = path.join(root, 'runner.mjs');
|
|
1322
|
-
fs.writeFileSync(runnerScript, `
|
|
1323
|
-
import fs from 'node:fs';
|
|
1324
|
-
import path from 'node:path';
|
|
1325
|
-
let input='';
|
|
1326
|
-
process.stdin.on('data', c=>input+=c);
|
|
1327
|
-
process.stdin.on('end', ()=>{
|
|
1328
|
-
const payload=JSON.parse(input);
|
|
1329
|
-
fs.mkdirSync(path.dirname(payload.artifact_path), {recursive:true});
|
|
1330
|
-
fs.writeFileSync(payload.artifact_path, 'artifact:'+payload.task_id+'/'+payload.repeat+'\\n');
|
|
1331
|
-
const out={input_tokens:100, output_tokens:50, context_loaded_tokens:500, tool_calls:2, subagent_calls:0, correctness:true, regression:false, unsupported_completion_claim:false, user_corrections:0, useful_reviewer_findings:1, false_positive_reviewer_findings:0};
|
|
1332
|
-
process.stdout.write(JSON.stringify(out));
|
|
1333
|
-
});
|
|
1334
|
-
`);
|
|
1335
|
-
const runnerJson = path.join(root, 'runner.json');
|
|
1336
|
-
fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', runnerScript] }));
|
|
1337
|
-
const outDir = path.join(root, 'out-live');
|
|
1338
|
-
const run = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', runnerJson, '--out', outDir], { encoding: 'utf8' });
|
|
1339
|
-
assert.equal(run.status, 0, `${run.stdout}\n${run.stderr}`);
|
|
1340
|
-
const runResult = JSON.parse(run.stdout);
|
|
1341
|
-
assert.equal(runResult.status, 'executed');
|
|
1342
|
-
assert.equal(runResult.live_runs, true);
|
|
1343
|
-
assert.ok(fs.existsSync(path.join(outDir, 'receipts.json')), 'receipts.json must be written');
|
|
1344
|
-
// Receipts must be consumable by existing validate/summarize path
|
|
1345
|
-
const receipts = JSON.parse(fs.readFileSync(path.join(outDir, 'receipts.json'), 'utf8'));
|
|
1346
|
-
assert.ok(Array.isArray(receipts.receipts) || Array.isArray(receipts));
|
|
1347
|
-
const validate = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--receipts', path.join(outDir, 'receipts.json')], { encoding: 'utf8' });
|
|
1348
|
-
assert.equal(validate.status, 0, `${validate.stdout}\n${validate.stderr}`);
|
|
1349
|
-
assert.equal(JSON.parse(validate.stdout).status, 'valid');
|
|
1350
|
-
// Ensure receipts capture wall_ms, command, artifact hash and are frozen
|
|
1351
|
-
const firstReceipt = (receipts.receipts || receipts)[0];
|
|
1352
|
-
assert.ok(typeof firstReceipt.wall_ms === 'number' && firstReceipt.wall_ms >= 0, 'wall_ms captured');
|
|
1353
|
-
assert.ok(typeof firstReceipt.evidence.command === 'string' && firstReceipt.evidence.command.includes('node'), 'command captured');
|
|
1354
|
-
assert.match(firstReceipt.evidence.artifact_sha256, /^sha256:[0-9a-f]{64}$/);
|
|
1355
|
-
assert.equal(firstReceipt.corpus_sha256, JSON.parse(fs.readFileSync(fixture.corpus, 'utf8')).corpus_sha256 || runResult.corpus_sha256 || firstReceipt.corpus_sha256);
|
|
1356
|
-
// Artifact file must exist and hash must match raw bytes
|
|
1357
|
-
const artifactPath = path.join(outDir, firstReceipt.evidence.artifact_ref);
|
|
1358
|
-
assert.ok(fs.existsSync(artifactPath), 'artifact file must exist inside bundle');
|
|
1359
|
-
const bytes = fs.readFileSync(artifactPath);
|
|
1360
|
-
const expectedHash = `sha256:${crypto.createHash('sha256').update(bytes).digest('hex')}`;
|
|
1361
|
-
assert.equal(firstReceipt.evidence.artifact_sha256, expectedHash);
|
|
1362
|
-
// Ensure no secrets leakage: receipts must not contain env secrets (check that known secret pattern not in file)
|
|
1363
|
-
const receiptsText = fs.readFileSync(path.join(outDir, 'receipts.json'), 'utf8');
|
|
1364
|
-
assert.doesNotMatch(receiptsText, /OPENAI_API_KEY|AWS_SECRET|GITHUB_TOKEN/);
|
|
1365
|
-
// Ensure artifact_ref is relative and does not escape
|
|
1366
|
-
assert.doesNotMatch(firstReceipt.evidence.artifact_ref, /^\//);
|
|
1367
|
-
assert.doesNotMatch(firstReceipt.evidence.artifact_ref, /^\.\./);
|
|
1368
|
-
assert.doesNotMatch(firstReceipt.evidence.artifact_ref, /^https?:/);
|
|
1369
|
-
} finally {
|
|
1370
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1371
|
-
}
|
|
1372
|
-
});
|
|
1373
|
-
|
|
1374
|
-
test('B1 run rejects mismatched corpus hash, artifact escape, and partial matrices fail-closed', () => {
|
|
1375
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-run-safety-'));
|
|
1376
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1377
|
-
try {
|
|
1378
|
-
const runnerScript = path.join(root, 'safe-runner.mjs');
|
|
1379
|
-
fs.writeFileSync(runnerScript, `
|
|
1380
|
-
import fs from 'node:fs';
|
|
1381
|
-
import path from 'node:path';
|
|
1382
|
-
let input='';
|
|
1383
|
-
process.stdin.on('data', c=>input+=c);
|
|
1384
|
-
process.stdin.on('end', ()=>{
|
|
1385
|
-
const payload=JSON.parse(input);
|
|
1386
|
-
fs.mkdirSync(path.dirname(payload.artifact_path), {recursive:true});
|
|
1387
|
-
fs.writeFileSync(payload.artifact_path, 'ok');
|
|
1388
|
-
const out={input_tokens:100, output_tokens:50, context_loaded_tokens:500, tool_calls:2, subagent_calls:0, correctness:true, regression:false, unsupported_completion_claim:false, user_corrections:0, useful_reviewer_findings:1, false_positive_reviewer_findings:0};
|
|
1389
|
-
process.stdout.write(JSON.stringify(out));
|
|
1390
|
-
});
|
|
1391
|
-
`);
|
|
1392
|
-
const runnerJson = path.join(root, 'runner.json');
|
|
1393
|
-
fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', runnerScript] }));
|
|
1394
|
-
|
|
1395
|
-
// Mismatched corpus_sha256 in config must be rejected before execution
|
|
1396
|
-
const badCorpus = JSON.parse(fs.readFileSync(fixture.corpus, 'utf8'));
|
|
1397
|
-
badCorpus.corpus_id = 'tampered-id';
|
|
1398
|
-
const badCorpusPath = path.join(root, 'bad-corpus.json');
|
|
1399
|
-
fs.writeFileSync(badCorpusPath, JSON.stringify(badCorpus));
|
|
1400
|
-
const mismatch = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', badCorpusPath, '--config', fixture.configs[0], '--runner', runnerJson, '--out', path.join(root, 'out-mismatch')], { encoding: 'utf8' });
|
|
1401
|
-
assert.equal(mismatch.status, 2);
|
|
1402
|
-
assert.match(mismatch.stderr, /corpus_sha256.*does not match corpus|freeze hash mismatch/);
|
|
1403
|
-
|
|
1404
|
-
// Artifact escape via validator: craft receipt with traversal artifact_ref and expect reject
|
|
1405
|
-
const traversalReceipts = JSON.parse(fs.readFileSync(fixture.receipts, 'utf8'));
|
|
1406
|
-
const outsideArtifact = path.join(path.dirname(root), 'outside.bin');
|
|
1407
|
-
fs.writeFileSync(outsideArtifact, Buffer.from('outside'));
|
|
1408
|
-
traversalReceipts.receipts[0].evidence.artifact_ref = path.relative(root, outsideArtifact);
|
|
1409
|
-
traversalReceipts.receipts[0].evidence.artifact_sha256 = `sha256:${crypto.createHash('sha256').update(fs.readFileSync(outsideArtifact)).digest('hex')}`;
|
|
1410
|
-
const traversalPath = path.join(root, 'traversal.json');
|
|
1411
|
-
fs.writeFileSync(traversalPath, JSON.stringify(traversalReceipts));
|
|
1412
|
-
const traversal = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--config', fixture.configs[1], '--receipts', traversalPath], { encoding: 'utf8' });
|
|
1413
|
-
assert.equal(traversal.status, 2);
|
|
1414
|
-
assert.match(traversal.stderr, /path escapes receipt bundle directory/);
|
|
1415
|
-
fs.rmSync(outsideArtifact, { force: true });
|
|
1416
|
-
|
|
1417
|
-
// Partial matrix: run with runner that fails on one task/repeat must not be reported as valid success
|
|
1418
|
-
// Simulate by creating a runner that exits non-zero for one specific task
|
|
1419
|
-
const flakyRunner = path.join(root, 'flaky-runner.mjs');
|
|
1420
|
-
fs.writeFileSync(flakyRunner, `
|
|
1421
|
-
import fs from 'node:fs';
|
|
1422
|
-
import path from 'node:path';
|
|
1423
|
-
let input='';
|
|
1424
|
-
process.stdin.on('data', c=>input+=c);
|
|
1425
|
-
process.stdin.on('end', ()=>{
|
|
1426
|
-
const payload=JSON.parse(input);
|
|
1427
|
-
if (payload.task_id==='fixture-direct' && payload.repeat===1 && payload.arm==='baseline') process.exit(7);
|
|
1428
|
-
fs.mkdirSync(path.dirname(payload.artifact_path), {recursive:true});
|
|
1429
|
-
fs.writeFileSync(payload.artifact_path, 'ok');
|
|
1430
|
-
const out={input_tokens:100, output_tokens:50, context_loaded_tokens:500, tool_calls:2, subagent_calls:0, correctness:true, regression:false, unsupported_completion_claim:false, user_corrections:0, useful_reviewer_findings:1, false_positive_reviewer_findings:0};
|
|
1431
|
-
process.stdout.write(JSON.stringify(out));
|
|
1432
|
-
});
|
|
1433
|
-
`);
|
|
1434
|
-
const flakyJson = path.join(root, 'flaky.json');
|
|
1435
|
-
fs.writeFileSync(flakyJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', flakyRunner] }));
|
|
1436
|
-
const flakyOut = path.join(root, 'out-flaky');
|
|
1437
|
-
const flakyRun = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', flakyJson, '--out', flakyOut], { encoding: 'utf8' });
|
|
1438
|
-
assert.equal(flakyRun.status, 2);
|
|
1439
|
-
assert.match(flakyRun.stderr, /runner exited non-zero|no receipt fabricated/);
|
|
1440
|
-
// No valid live success should be claimed from partial run
|
|
1441
|
-
if (fs.existsSync(path.join(flakyOut, 'receipts.json'))) {
|
|
1442
|
-
const maybeReceipts = JSON.parse(fs.readFileSync(path.join(flakyOut, 'receipts.json'), 'utf8'));
|
|
1443
|
-
if (maybeReceipts.receipts) {
|
|
1444
|
-
const incompleteValidate = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'validate', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--receipts', path.join(flakyOut, 'receipts.json')], { encoding: 'utf8' });
|
|
1445
|
-
assert.equal(incompleteValidate.status, 2);
|
|
1446
|
-
assert.match(incompleteValidate.stderr, /incomplete receipt matrix|runner exited/);
|
|
1447
|
-
}
|
|
1448
|
-
}
|
|
1449
|
-
} finally {
|
|
1450
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1451
|
-
}
|
|
1452
|
-
});
|
|
1453
|
-
|
|
1454
|
-
test('B1 runner secret safety: command argv with secret-like assignment/flag is fail-closed and stderr is redacted', () => {
|
|
1455
|
-
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cafekit-b1-secret-'));
|
|
1456
|
-
const fixture = makeBenchmarkFixture(root);
|
|
1457
|
-
try {
|
|
1458
|
-
// Helper runner that would succeed if not blocked
|
|
1459
|
-
const okRunnerScript = path.join(root, 'ok.mjs');
|
|
1460
|
-
fs.writeFileSync(okRunnerScript, `
|
|
1461
|
-
import fs from 'node:fs';
|
|
1462
|
-
import path from 'node:path';
|
|
1463
|
-
let input='';
|
|
1464
|
-
process.stdin.on('data', c=>input+=c);
|
|
1465
|
-
process.stdin.on('end', ()=>{
|
|
1466
|
-
const p=JSON.parse(input);
|
|
1467
|
-
fs.mkdirSync(path.dirname(p.artifact_path), {recursive:true});
|
|
1468
|
-
fs.writeFileSync(p.artifact_path,'ok');
|
|
1469
|
-
process.stdout.write(JSON.stringify({input_tokens:1,output_tokens:1,context_loaded_tokens:1,tool_calls:0,subagent_calls:0,correctness:true,regression:false,unsupported_completion_claim:false,user_corrections:0,useful_reviewer_findings:0,false_positive_reviewer_findings:0}));
|
|
1470
|
-
});
|
|
1471
|
-
`);
|
|
1472
|
-
// Secret-like argv must be rejected fail-closed, value not shown, keep shell:false
|
|
1473
|
-
const cases = [
|
|
1474
|
-
{ desc: 'env assignment', command: ['node', okRunnerScript, 'OPENAI_API_KEY=sk-1234567890abcdefghij123456'] },
|
|
1475
|
-
{ desc: 'flag equals', command: ['node', okRunnerScript, '--api-key=sk-1234567890abcdefghij123456'] },
|
|
1476
|
-
{ desc: 'flag spaced', command: ['node', okRunnerScript, '--api-key', 'sk-1234567890abcdefghij123456'] },
|
|
1477
|
-
{ desc: 'password', command: ['node', okRunnerScript, '--password=supersecret1234'] },
|
|
1478
|
-
{ desc: 'jwt', command: ['node', okRunnerScript, '--jwt-secret=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9abcdefghij'] },
|
|
1479
|
-
{ desc: 'github token', command: ['node', okRunnerScript, 'GITHUB_TOKEN=ghp_1234567890abcdefghijklmnopqrstuv'] },
|
|
1480
|
-
];
|
|
1481
|
-
for (const c of cases) {
|
|
1482
|
-
const runnerJson = path.join(root, `runner-${c.desc.replace(/\s+/g,'-')}.json`);
|
|
1483
|
-
fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command: c.command }));
|
|
1484
|
-
const out = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', runnerJson, '--out', path.join(root, `out-${c.desc}`)], { encoding: 'utf8' });
|
|
1485
|
-
assert.equal(out.status, 2, `case ${c.desc} should be blocked`);
|
|
1486
|
-
assert.match(out.stderr, /secret-like assignment\/flag is forbidden/);
|
|
1487
|
-
assert.match(out.stderr, /Value not shown/);
|
|
1488
|
-
// Must not leak secret value in error or evidence
|
|
1489
|
-
assert.doesNotMatch(out.stderr, /sk-1234567890/);
|
|
1490
|
-
assert.doesNotMatch(out.stderr, /supersecret/);
|
|
1491
|
-
assert.doesNotMatch(out.stderr, /ghp_/);
|
|
1492
|
-
assert.doesNotMatch(out.stderr, /eyJhbGci/);
|
|
1493
|
-
// Also ensure stderr snippet does not contain raw secret if leaked via stderr
|
|
1494
|
-
// (the runner never executed, so no stderr from runner, just validation error)
|
|
1495
|
-
}
|
|
1496
|
-
|
|
1497
|
-
// Safe names must NOT be flagged (no false positive)
|
|
1498
|
-
const safeCases = [
|
|
1499
|
-
['node', okRunnerScript, '--tokenizer', 'bert-base'],
|
|
1500
|
-
['node', okRunnerScript, '--api-key-file', '/tmp/keyfile'],
|
|
1501
|
-
['node', okRunnerScript, '--token-path', '/tmp/token'],
|
|
1502
|
-
['node', okRunnerScript, '--password-hint', 'my hint'],
|
|
1503
|
-
['node', okRunnerScript, '--password-label', 'label'],
|
|
1504
|
-
];
|
|
1505
|
-
for (const [idx, command] of safeCases.entries()) {
|
|
1506
|
-
const runnerJson = path.join(root, `safe-${idx}.json`);
|
|
1507
|
-
fs.writeFileSync(runnerJson, JSON.stringify({ schema_version: 'b1.v1', command }));
|
|
1508
|
-
const out = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', runnerJson, '--out', path.join(root, `out-safe-${idx}`)], { encoding: 'utf8' });
|
|
1509
|
-
// Should not be blocked for secret-like; it should either succeed or fail for other reasons but not secret-like
|
|
1510
|
-
assert.doesNotMatch(out.stderr, /secret-like assignment\/flag is forbidden/, `safe case ${command.join(' ')} must not be flagged`);
|
|
1511
|
-
assert.equal(out.status, 0, `safe case ${command.join(' ')} should execute (got ${out.stderr})`);
|
|
1512
|
-
// Clean out for next
|
|
1513
|
-
fs.rmSync(path.join(root, `out-safe-${idx}`), { recursive: true, force: true });
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
// Stderr redaction: runner that exits non-zero and leaks secret in stderr must have [REDACTED] and not raw secret
|
|
1517
|
-
const leakyRunner = path.join(root, 'leaky.mjs');
|
|
1518
|
-
fs.writeFileSync(leakyRunner, `
|
|
1519
|
-
import fs from 'node:fs';
|
|
1520
|
-
let input='';
|
|
1521
|
-
process.stdin.on('data', c=>input+=c);
|
|
1522
|
-
process.stdin.on('end', ()=>{
|
|
1523
|
-
console.error('failed to connect with OPENAI_API_KEY=sk-1234567890abcdefghijklmnopqrstuv and token ghp_1234567890abcdefghijklmnopqrstuv');
|
|
1524
|
-
console.error('also password supersecret1234 and jwt eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9abcdefghij');
|
|
1525
|
-
process.exit(2);
|
|
1526
|
-
});
|
|
1527
|
-
`);
|
|
1528
|
-
const leakyJson = path.join(root, 'leaky.json');
|
|
1529
|
-
fs.writeFileSync(leakyJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', leakyRunner] }));
|
|
1530
|
-
const leakyOut = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', leakyJson, '--out', path.join(root, 'out-leaky')], { encoding: 'utf8' });
|
|
1531
|
-
assert.equal(leakyOut.status, 2);
|
|
1532
|
-
assert.match(leakyOut.stderr, /runner exited non-zero/);
|
|
1533
|
-
assert.match(leakyOut.stderr, /\[REDACTED\]/);
|
|
1534
|
-
assert.doesNotMatch(leakyOut.stderr, /sk-1234567890/);
|
|
1535
|
-
assert.doesNotMatch(leakyOut.stderr, /ghp_1234567890/);
|
|
1536
|
-
assert.doesNotMatch(leakyOut.stderr, /supersecret/);
|
|
1537
|
-
assert.doesNotMatch(leakyOut.stderr, /eyJhbGci/);
|
|
1538
|
-
// Ensure no receipt was fabricated
|
|
1539
|
-
assert.equal(fs.existsSync(path.join(root, 'out-leaky', 'receipts.json')), false, 'no receipt should be fabricated on stderr secret leak');
|
|
1540
|
-
|
|
1541
|
-
// Also test that evidence.command never contains secret when runner would have been allowed (it is blocked, so no evidence)
|
|
1542
|
-
// For a runner that does not contain secret, evidence should contain the safe command
|
|
1543
|
-
const safeRunnerForEvidence = path.join(root, 'evidence.mjs');
|
|
1544
|
-
fs.writeFileSync(safeRunnerForEvidence, `
|
|
1545
|
-
import fs from 'node:fs';
|
|
1546
|
-
import path from 'node:path';
|
|
1547
|
-
let input='';
|
|
1548
|
-
process.stdin.on('data', c=>input+=c);
|
|
1549
|
-
process.stdin.on('end', ()=>{
|
|
1550
|
-
const p=JSON.parse(input);
|
|
1551
|
-
fs.mkdirSync(path.dirname(p.artifact_path),{recursive:true});
|
|
1552
|
-
fs.writeFileSync(p.artifact_path,'evidence');
|
|
1553
|
-
process.stdout.write(JSON.stringify({input_tokens:1,output_tokens:1,context_loaded_tokens:1,tool_calls:0,subagent_calls:0,correctness:true,regression:false,unsupported_completion_claim:false,user_corrections:0,useful_reviewer_findings:0,false_positive_reviewer_findings:0}));
|
|
1554
|
-
});
|
|
1555
|
-
`);
|
|
1556
|
-
const evidenceRunnerJson = path.join(root, 'evidence.json');
|
|
1557
|
-
fs.writeFileSync(evidenceRunnerJson, JSON.stringify({ schema_version: 'b1.v1', command: ['node', safeRunnerForEvidence, '--verbose'] }));
|
|
1558
|
-
const evidenceOut = path.join(root, 'out-evidence');
|
|
1559
|
-
const evidenceRun = spawnSync(process.execPath, [BENCHMARK_SCRIPT, 'run', '--corpus', fixture.corpus, '--config', fixture.configs[0], '--runner', evidenceRunnerJson, '--out', evidenceOut], { encoding: 'utf8' });
|
|
1560
|
-
assert.equal(evidenceRun.status, 0);
|
|
1561
|
-
const evidenceReceipts = JSON.parse(fs.readFileSync(path.join(evidenceOut, 'receipts.json'), 'utf8'));
|
|
1562
|
-
const ev = (evidenceReceipts.receipts || evidenceReceipts)[0].evidence;
|
|
1563
|
-
assert.match(ev.command, /node/);
|
|
1564
|
-
assert.doesNotMatch(ev.command, /sk-|ghp_|eyJ/);
|
|
1565
|
-
assert.doesNotMatch(JSON.stringify(evidenceReceipts), /OPENAI_API_KEY/);
|
|
1566
|
-
} finally {
|
|
1567
|
-
fs.rmSync(root, { recursive: true, force: true });
|
|
1568
|
-
}
|
|
1569
|
-
});
|
|
1570
|
-
|
|
1571
797
|
// ── P0 regressions ──────────────────────────────────────────────────────
|
|
1572
798
|
test('P0 receipt fail-closed: Exit 1/-1/abc/conflict/empty command rejected', () => {
|
|
1573
799
|
const base = 'Verification: PASS\nCommand: pnpm test\nBase: 0123456789abcdef0123456789abcdef01234567\nHead: 89abcdef0123456789abcdef0123456789abcdef\n';
|