@nforma.ai/nforma 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/quorum-preflight.cjs +89 -0
- package/commands/nf/quorum.md +3 -53
- package/package.json +4 -2
- package/hooks/dist/nf-circuit-breaker.test.js +0 -1002
- package/hooks/dist/nf-precompact.test.js +0 -227
- package/hooks/dist/nf-prompt.test.js +0 -698
- package/hooks/dist/nf-session-start.test.js +0 -354
- package/hooks/dist/nf-slot-correlator.test.js +0 -85
- package/hooks/dist/nf-spec-regen.test.js +0 -73
- package/hooks/dist/nf-statusline.test.js +0 -157
- package/hooks/dist/nf-stop.test.js +0 -1388
- package/hooks/dist/nf-token-collector.test.js +0 -262
|
@@ -1,1388 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// Test suite for hooks/nf-stop.js
|
|
3
|
-
// Uses Node.js built-in test runner: node --test hooks/nf-stop.test.js
|
|
4
|
-
//
|
|
5
|
-
// Each test spawns the hook as a child process with mock stdin and a synthetic
|
|
6
|
-
// JSONL transcript written to a temp file. Captures stdout + exit code.
|
|
7
|
-
|
|
8
|
-
const { test } = require('node:test');
|
|
9
|
-
const assert = require('node:assert/strict');
|
|
10
|
-
const { spawnSync } = require('child_process');
|
|
11
|
-
const fs = require('fs');
|
|
12
|
-
const os = require('os');
|
|
13
|
-
const path = require('path');
|
|
14
|
-
|
|
15
|
-
const HOOK_PATH = path.join(__dirname, 'nf-stop.js');
|
|
16
|
-
|
|
17
|
-
// Helper: write a temp JSONL file and return its path
|
|
18
|
-
function writeTempTranscript(lines) {
|
|
19
|
-
const tmpFile = path.join(os.tmpdir(), `nf-stop-test-${Date.now()}-${Math.random().toString(36).slice(2)}.jsonl`);
|
|
20
|
-
fs.writeFileSync(tmpFile, lines.join('\n') + '\n', 'utf8');
|
|
21
|
-
return tmpFile;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
// Helper: run the hook with a given stdin JSON payload, return { stdout, exitCode }
|
|
25
|
-
function runHook(stdinPayload) {
|
|
26
|
-
const result = spawnSync('node', [HOOK_PATH], {
|
|
27
|
-
input: JSON.stringify(stdinPayload),
|
|
28
|
-
encoding: 'utf8',
|
|
29
|
-
timeout: 5000,
|
|
30
|
-
});
|
|
31
|
-
return {
|
|
32
|
-
stdout: result.stdout || '',
|
|
33
|
-
stderr: result.stderr || '',
|
|
34
|
-
exitCode: result.status,
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
// Helper: run the hook with a given stdin JSON payload and additional env vars
|
|
39
|
-
// Used for TC11-TC13 to inject NF_CLAUDE_JSON for deterministic MCP availability testing
|
|
40
|
-
function runHookWithEnv(stdinPayload, extraEnv) {
|
|
41
|
-
const result = spawnSync('node', [HOOK_PATH], {
|
|
42
|
-
input: JSON.stringify(stdinPayload),
|
|
43
|
-
encoding: 'utf8',
|
|
44
|
-
timeout: 5000,
|
|
45
|
-
env: { ...process.env, ...extraEnv },
|
|
46
|
-
});
|
|
47
|
-
return {
|
|
48
|
-
stdout: result.stdout || '',
|
|
49
|
-
stderr: result.stderr || '',
|
|
50
|
-
exitCode: result.status,
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
// JSONL builder helpers
|
|
55
|
-
function userLine(content, uuid = 'user-1') {
|
|
56
|
-
return JSON.stringify({
|
|
57
|
-
type: 'user',
|
|
58
|
-
message: { role: 'user', content },
|
|
59
|
-
timestamp: '2026-02-20T00:00:00Z',
|
|
60
|
-
uuid,
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
function assistantLine(contentBlocks, uuid = 'assistant-1') {
|
|
65
|
-
return JSON.stringify({
|
|
66
|
-
type: 'assistant',
|
|
67
|
-
message: {
|
|
68
|
-
role: 'assistant',
|
|
69
|
-
content: contentBlocks,
|
|
70
|
-
stop_reason: contentBlocks.some(b => b.type === 'tool_use') ? 'tool_use' : 'end_turn',
|
|
71
|
-
},
|
|
72
|
-
timestamp: '2026-02-20T00:01:00Z',
|
|
73
|
-
uuid,
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
function toolUseBlock(name) {
|
|
78
|
-
return { type: 'tool_use', id: `toolu_${name}`, name, input: { content: 'test plan' } };
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
function bashCommitBlock(commitCmd) {
|
|
82
|
-
return { type: 'tool_use', id: 'toolu_bash', name: 'Bash', input: { command: commitCmd } };
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
// --- Test Cases ---
|
|
86
|
-
|
|
87
|
-
// Test 1: stop_hook_active: true → exit 0, no stdout (infinite loop guard)
|
|
88
|
-
test('TC1: stop_hook_active true exits immediately with no output', () => {
|
|
89
|
-
const tmpFile = writeTempTranscript([userLine('/gsd:plan-phase 1')]);
|
|
90
|
-
try {
|
|
91
|
-
const { stdout, exitCode } = runHook({
|
|
92
|
-
stop_hook_active: true,
|
|
93
|
-
hook_event_name: 'Stop',
|
|
94
|
-
transcript_path: tmpFile,
|
|
95
|
-
last_assistant_message: 'Here is the plan',
|
|
96
|
-
});
|
|
97
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
98
|
-
assert.strictEqual(stdout, '', 'stdout must be empty (no block decision)');
|
|
99
|
-
} finally {
|
|
100
|
-
fs.unlinkSync(tmpFile);
|
|
101
|
-
}
|
|
102
|
-
});
|
|
103
|
-
|
|
104
|
-
// Test 2: hook_event_name SubagentStop → exit 0, no stdout (subagent exclusion)
|
|
105
|
-
test('TC2: SubagentStop exits immediately with no output', () => {
|
|
106
|
-
const tmpFile = writeTempTranscript([userLine('/gsd:plan-phase 1')]);
|
|
107
|
-
try {
|
|
108
|
-
const { stdout, exitCode } = runHook({
|
|
109
|
-
stop_hook_active: false,
|
|
110
|
-
hook_event_name: 'SubagentStop',
|
|
111
|
-
transcript_path: tmpFile,
|
|
112
|
-
last_assistant_message: 'SubagentStop response',
|
|
113
|
-
});
|
|
114
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
115
|
-
assert.strictEqual(stdout, '', 'stdout must be empty');
|
|
116
|
-
} finally {
|
|
117
|
-
fs.unlinkSync(tmpFile);
|
|
118
|
-
}
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
// Test 3: transcript_path nonexistent → exit 0, no stdout (fail-open)
|
|
122
|
-
test('TC3: nonexistent transcript_path exits 0 with no output (fail-open)', () => {
|
|
123
|
-
const { stdout, exitCode } = runHook({
|
|
124
|
-
stop_hook_active: false,
|
|
125
|
-
hook_event_name: 'Stop',
|
|
126
|
-
transcript_path: '/nonexistent/path/that/does/not/exist.jsonl',
|
|
127
|
-
last_assistant_message: '',
|
|
128
|
-
});
|
|
129
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
130
|
-
assert.strictEqual(stdout, '', 'stdout must be empty');
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
// Test 4: transcript with no planning command in current turn → exit 0, no stdout
|
|
134
|
-
test('TC4: no planning command in current turn passes (no block)', () => {
|
|
135
|
-
const tmpFile = writeTempTranscript([
|
|
136
|
-
userLine('What time is it?'),
|
|
137
|
-
assistantLine([{ type: 'text', text: 'It is noon.' }]),
|
|
138
|
-
]);
|
|
139
|
-
try {
|
|
140
|
-
const { stdout, exitCode } = runHook({
|
|
141
|
-
stop_hook_active: false,
|
|
142
|
-
hook_event_name: 'Stop',
|
|
143
|
-
transcript_path: tmpFile,
|
|
144
|
-
last_assistant_message: 'It is noon.',
|
|
145
|
-
});
|
|
146
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
147
|
-
assert.strictEqual(stdout, '', 'stdout must be empty (no block needed)');
|
|
148
|
-
} finally {
|
|
149
|
-
fs.unlinkSync(tmpFile);
|
|
150
|
-
}
|
|
151
|
-
});
|
|
152
|
-
|
|
153
|
-
// Test 5: /gsd:plan-phase in current turn + all 3 quorum tool_use blocks → exit 0, no stdout
|
|
154
|
-
test('TC5: planning command with all three quorum tool calls passes', () => {
|
|
155
|
-
const tmpFile = writeTempTranscript([
|
|
156
|
-
userLine('/gsd:plan-phase 1'),
|
|
157
|
-
assistantLine([
|
|
158
|
-
toolUseBlock('mcp__codex-cli__review'),
|
|
159
|
-
toolUseBlock('mcp__gemini-cli__gemini'),
|
|
160
|
-
toolUseBlock('mcp__opencode__opencode'),
|
|
161
|
-
]),
|
|
162
|
-
assistantLine([{ type: 'text', text: 'Here is the plan.' }], 'assistant-2'),
|
|
163
|
-
]);
|
|
164
|
-
try {
|
|
165
|
-
const { stdout, exitCode } = runHook({
|
|
166
|
-
stop_hook_active: false,
|
|
167
|
-
hook_event_name: 'Stop',
|
|
168
|
-
transcript_path: tmpFile,
|
|
169
|
-
last_assistant_message: 'Here is the plan.',
|
|
170
|
-
});
|
|
171
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
172
|
-
assert.strictEqual(stdout, '', 'stdout must be empty (quorum complete)');
|
|
173
|
-
} finally {
|
|
174
|
-
fs.unlinkSync(tmpFile);
|
|
175
|
-
}
|
|
176
|
-
});
|
|
177
|
-
|
|
178
|
-
// Test 5b: /qgsd:plan-phase — quorum present → pass (mirrors TC5 with /qgsd: prefix)
|
|
179
|
-
test('TC5b: /qgsd:plan-phase — quorum present → pass', () => {
|
|
180
|
-
const tmpFile = writeTempTranscript([
|
|
181
|
-
userLine('/qgsd:plan-phase 1'),
|
|
182
|
-
assistantLine([
|
|
183
|
-
toolUseBlock('mcp__codex-cli__review'),
|
|
184
|
-
toolUseBlock('mcp__gemini-cli__gemini'),
|
|
185
|
-
toolUseBlock('mcp__opencode__opencode'),
|
|
186
|
-
]),
|
|
187
|
-
assistantLine([{ type: 'text', text: 'Here is the plan.' }], 'assistant-2'),
|
|
188
|
-
]);
|
|
189
|
-
try {
|
|
190
|
-
const { stdout, exitCode } = runHook({
|
|
191
|
-
stop_hook_active: false,
|
|
192
|
-
hook_event_name: 'Stop',
|
|
193
|
-
transcript_path: tmpFile,
|
|
194
|
-
last_assistant_message: 'Here is the plan.',
|
|
195
|
-
});
|
|
196
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
197
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — /qgsd: prefix recognized and quorum complete');
|
|
198
|
-
} finally {
|
|
199
|
-
fs.unlinkSync(tmpFile);
|
|
200
|
-
}
|
|
201
|
-
});
|
|
202
|
-
|
|
203
|
-
// Test 6: /gsd:plan-phase in current turn + only codex tool_use → block with decision:block
|
|
204
|
-
// TC6 updated (step 1a): includes a PLAN.md artifact commit so GUARD 5 classifies this as a
|
|
205
|
-
// decision turn, preserving the invariant: quorum-command + decision-turn + partial quorum = block.
|
|
206
|
-
// TC6 uses runHookWithEnv to isolate from ~/.claude/nf.json (which may override DEFAULT_CONFIG
|
|
207
|
-
// prefixes with -1 suffixes). HOME points to an empty temp dir so loadConfig() uses DEFAULT_CONFIG.
|
|
208
|
-
// NF_CLAUDE_JSON points to a temp file listing gemini-cli and opencode as available MCP servers
|
|
209
|
-
// so they appear in the missing list and satisfy the string assertions.
|
|
210
|
-
test('TC6: planning command with only codex tool call triggers block', () => {
|
|
211
|
-
// Create a temp HOME dir with no nf.json — forces DEFAULT_CONFIG usage
|
|
212
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-tc6-${Date.now()}`);
|
|
213
|
-
fs.mkdirSync(homeDir, { recursive: true });
|
|
214
|
-
|
|
215
|
-
// Create a temp ~/.claude.json listing gemini-cli and opencode as available MCP servers
|
|
216
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-tc6-${Date.now()}.json`);
|
|
217
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({
|
|
218
|
-
mcpServers: {
|
|
219
|
-
'codex-cli': {},
|
|
220
|
-
'gemini-cli': {},
|
|
221
|
-
'opencode': {},
|
|
222
|
-
},
|
|
223
|
-
}), 'utf8');
|
|
224
|
-
|
|
225
|
-
const tmpFile = writeTempTranscript([
|
|
226
|
-
userLine('/gsd:plan-phase 1'),
|
|
227
|
-
assistantLine([
|
|
228
|
-
toolUseBlock('mcp__codex-cli__review'),
|
|
229
|
-
]),
|
|
230
|
-
assistantLine([
|
|
231
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "feat: plan" --files 04-01-PLAN.md'),
|
|
232
|
-
], 'assistant-commit'),
|
|
233
|
-
assistantLine([{ type: 'text', text: 'Here is the plan.' }], 'assistant-2'),
|
|
234
|
-
]);
|
|
235
|
-
try {
|
|
236
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
237
|
-
{
|
|
238
|
-
stop_hook_active: false,
|
|
239
|
-
hook_event_name: 'Stop',
|
|
240
|
-
transcript_path: tmpFile,
|
|
241
|
-
last_assistant_message: 'Here is the plan.',
|
|
242
|
-
},
|
|
243
|
-
{
|
|
244
|
-
HOME: homeDir, // No ~/.claude/nf.json → loadConfig() uses DEFAULT_CONFIG
|
|
245
|
-
NF_CLAUDE_JSON: claudeJsonTmp, // Deterministic MCP server list
|
|
246
|
-
}
|
|
247
|
-
);
|
|
248
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
249
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision JSON');
|
|
250
|
-
const parsed = JSON.parse(stdout);
|
|
251
|
-
assert.strictEqual(parsed.decision, 'block', 'decision must be "block"');
|
|
252
|
-
assert.ok(parsed.reason.startsWith('QUORUM REQUIRED:'), 'reason must start with QUORUM REQUIRED:');
|
|
253
|
-
// Should name the missing tools (DEFAULT_CONFIG prefixes)
|
|
254
|
-
assert.ok(parsed.reason.includes('mcp__gemini-cli__'), 'reason must name missing gemini tool');
|
|
255
|
-
assert.ok(parsed.reason.includes('mcp__opencode__'), 'reason must name missing opencode tool');
|
|
256
|
-
} finally {
|
|
257
|
-
fs.unlinkSync(tmpFile);
|
|
258
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
259
|
-
}
|
|
260
|
-
});
|
|
261
|
-
|
|
262
|
-
// Test 7: /gsd:plan-phase in OLD turn (before current turn boundary) → exit 0, no stdout (scope filter)
|
|
263
|
-
test('TC7: planning command only in old turn (before boundary) is not in scope', () => {
|
|
264
|
-
const tmpFile = writeTempTranscript([
|
|
265
|
-
// Old turn: planning command + assistant response (but no quorum)
|
|
266
|
-
userLine('/gsd:plan-phase 1', 'old-user'),
|
|
267
|
-
assistantLine([{ type: 'text', text: 'Old response, no quorum done.' }], 'old-assistant'),
|
|
268
|
-
// New turn: unrelated user message (this becomes the current turn boundary)
|
|
269
|
-
userLine('Thanks, looks good.', 'new-user'),
|
|
270
|
-
assistantLine([{ type: 'text', text: 'Great!' }], 'new-assistant'),
|
|
271
|
-
]);
|
|
272
|
-
try {
|
|
273
|
-
const { stdout, exitCode } = runHook({
|
|
274
|
-
stop_hook_active: false,
|
|
275
|
-
hook_event_name: 'Stop',
|
|
276
|
-
transcript_path: tmpFile,
|
|
277
|
-
last_assistant_message: 'Great!',
|
|
278
|
-
});
|
|
279
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
280
|
-
assert.strictEqual(stdout, '', 'stdout must be empty (old turn planning not in scope)');
|
|
281
|
-
} finally {
|
|
282
|
-
fs.unlinkSync(tmpFile);
|
|
283
|
-
}
|
|
284
|
-
});
|
|
285
|
-
|
|
286
|
-
// Test 8: malformed JSONL lines → skip gracefully, hook still works
|
|
287
|
-
test('TC8: malformed JSONL lines are skipped gracefully', () => {
|
|
288
|
-
const tmpFile = writeTempTranscript([
|
|
289
|
-
'this is not valid json',
|
|
290
|
-
userLine('/gsd:plan-phase 1'),
|
|
291
|
-
'{broken json: [',
|
|
292
|
-
assistantLine([
|
|
293
|
-
toolUseBlock('mcp__codex-cli__review'),
|
|
294
|
-
toolUseBlock('mcp__gemini-cli__gemini'),
|
|
295
|
-
toolUseBlock('mcp__opencode__opencode'),
|
|
296
|
-
]),
|
|
297
|
-
'another bad line',
|
|
298
|
-
]);
|
|
299
|
-
try {
|
|
300
|
-
const { stdout, exitCode } = runHook({
|
|
301
|
-
stop_hook_active: false,
|
|
302
|
-
hook_event_name: 'Stop',
|
|
303
|
-
transcript_path: tmpFile,
|
|
304
|
-
last_assistant_message: 'Here is the plan.',
|
|
305
|
-
});
|
|
306
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
307
|
-
assert.strictEqual(stdout, '', 'stdout must be empty (quorum found despite malformed lines)');
|
|
308
|
-
} finally {
|
|
309
|
-
fs.unlinkSync(tmpFile);
|
|
310
|
-
}
|
|
311
|
-
});
|
|
312
|
-
|
|
313
|
-
// Test 9: config file missing → DEFAULT_CONFIG used, hook still works
|
|
314
|
-
// TC9 updated (step 1a): includes a RESEARCH.md artifact commit so GUARD 5 classifies this as a
|
|
315
|
-
// decision turn, preserving the invariant: quorum-command + decision-turn + no quorum = block.
|
|
316
|
-
// TC9 uses runHookWithEnv to isolate from ~/.claude/nf.json (which may override DEFAULT_CONFIG
|
|
317
|
-
// prefixes with -1 suffixes). HOME points to an empty temp dir so loadConfig() uses DEFAULT_CONFIG.
|
|
318
|
-
// NF_CLAUDE_JSON points to a non-existent path so getAvailableMcpPrefixes() returns null,
|
|
319
|
-
// meaning all models are treated as available (conservative enforcement) and DEFAULT_CONFIG names appear.
|
|
320
|
-
test('TC9: missing config file falls back to DEFAULT_CONFIG', () => {
|
|
321
|
-
// Create a temp HOME dir with no nf.json — forces DEFAULT_CONFIG usage
|
|
322
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-tc9-${Date.now()}`);
|
|
323
|
-
fs.mkdirSync(homeDir, { recursive: true });
|
|
324
|
-
|
|
325
|
-
// Point NF_CLAUDE_JSON to a non-existent file → getAvailableMcpPrefixes() returns null
|
|
326
|
-
// → all models treated as available (conservative) → DEFAULT_CONFIG names appear in block reason
|
|
327
|
-
const nonExistentClaudeJson = path.join(os.tmpdir(), `nf-claude-tc9-nonexistent-${Date.now()}.json`);
|
|
328
|
-
|
|
329
|
-
const tmpFile = writeTempTranscript([
|
|
330
|
-
userLine('/gsd:research-phase 1'),
|
|
331
|
-
assistantLine([
|
|
332
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "docs: research" --files 04-RESEARCH.md'),
|
|
333
|
-
], 'assistant-commit'),
|
|
334
|
-
assistantLine([{ type: 'text', text: 'No quorum calls here.' }]),
|
|
335
|
-
]);
|
|
336
|
-
try {
|
|
337
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
338
|
-
{
|
|
339
|
-
stop_hook_active: false,
|
|
340
|
-
hook_event_name: 'Stop',
|
|
341
|
-
transcript_path: tmpFile,
|
|
342
|
-
last_assistant_message: 'No quorum calls here.',
|
|
343
|
-
},
|
|
344
|
-
{
|
|
345
|
-
HOME: homeDir, // No ~/.claude/nf.json → DEFAULT_CONFIG
|
|
346
|
-
NF_CLAUDE_JSON: nonExistentClaudeJson, // Missing → null prefixes → conservative enforcement
|
|
347
|
-
}
|
|
348
|
-
);
|
|
349
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
350
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision');
|
|
351
|
-
const parsed = JSON.parse(stdout);
|
|
352
|
-
assert.strictEqual(parsed.decision, 'block');
|
|
353
|
-
assert.ok(parsed.reason.includes('QUORUM REQUIRED:'), 'reason must include QUORUM REQUIRED');
|
|
354
|
-
// Default config model tool prefixes should be in the reason
|
|
355
|
-
assert.ok(
|
|
356
|
-
parsed.reason.includes('mcp__codex-cli__') ||
|
|
357
|
-
parsed.reason.includes('mcp__gemini-cli__') ||
|
|
358
|
-
parsed.reason.includes('mcp__opencode__'),
|
|
359
|
-
'reason must name at least one default model tool'
|
|
360
|
-
);
|
|
361
|
-
} finally {
|
|
362
|
-
fs.unlinkSync(tmpFile);
|
|
363
|
-
}
|
|
364
|
-
});
|
|
365
|
-
|
|
366
|
-
// Test 10: Regression — quorum calls interleaved with tool_result user messages
|
|
367
|
-
// This reproduces the live false-positive: getCurrentTurnLines() must skip
|
|
368
|
-
// tool_result user messages and use the human text message as the boundary.
|
|
369
|
-
function toolResultLine(toolUseId, resultContent, uuid) {
|
|
370
|
-
return JSON.stringify({
|
|
371
|
-
type: 'user',
|
|
372
|
-
message: {
|
|
373
|
-
role: 'user',
|
|
374
|
-
content: [{ type: 'tool_result', tool_use_id: toolUseId, content: resultContent }],
|
|
375
|
-
},
|
|
376
|
-
timestamp: '2026-02-20T00:01:00Z',
|
|
377
|
-
uuid: uuid || `tr-${toolUseId}`,
|
|
378
|
-
});
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
test('TC10: quorum calls interleaved with tool_result user messages are in scope', () => {
|
|
382
|
-
// Simulates a multi-tool turn: human message → tool calls → tool_results → more calls
|
|
383
|
-
// The quorum calls appear between intermediate tool_result user messages.
|
|
384
|
-
// getCurrentTurnLines() must find the human message as the boundary, not a tool_result.
|
|
385
|
-
const tmpFile = writeTempTranscript([
|
|
386
|
-
// Human turn starts here
|
|
387
|
-
userLine('/gsd:plan-phase 1', 'human-msg'),
|
|
388
|
-
// First batch of tool calls (non-quorum — e.g., Task/Bash)
|
|
389
|
-
assistantLine([toolUseBlock('Bash')], 'assistant-1'),
|
|
390
|
-
toolResultLine('toolu_Bash', 'bash output', 'tr-1'),
|
|
391
|
-
// Second batch — quorum calls
|
|
392
|
-
assistantLine([
|
|
393
|
-
toolUseBlock('mcp__codex-cli__review'),
|
|
394
|
-
], 'assistant-2'),
|
|
395
|
-
toolResultLine('toolu_codex', 'codex review result', 'tr-2'),
|
|
396
|
-
assistantLine([
|
|
397
|
-
toolUseBlock('mcp__gemini-cli__gemini'),
|
|
398
|
-
], 'assistant-3'),
|
|
399
|
-
toolResultLine('toolu_gemini', 'gemini result', 'tr-3'),
|
|
400
|
-
assistantLine([
|
|
401
|
-
toolUseBlock('mcp__opencode__opencode'),
|
|
402
|
-
], 'assistant-4'),
|
|
403
|
-
// Final tool_result before the final assistant text
|
|
404
|
-
toolResultLine('toolu_opencode', 'opencode result', 'tr-4'),
|
|
405
|
-
assistantLine([{ type: 'text', text: 'Here is the plan with quorum complete.' }], 'assistant-5'),
|
|
406
|
-
]);
|
|
407
|
-
try {
|
|
408
|
-
const { stdout, exitCode } = runHook({
|
|
409
|
-
stop_hook_active: false,
|
|
410
|
-
hook_event_name: 'Stop',
|
|
411
|
-
transcript_path: tmpFile,
|
|
412
|
-
last_assistant_message: 'Here is the plan with quorum complete.',
|
|
413
|
-
});
|
|
414
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
415
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — quorum calls are in scope despite tool_result boundaries');
|
|
416
|
-
} finally {
|
|
417
|
-
fs.unlinkSync(tmpFile);
|
|
418
|
-
}
|
|
419
|
-
});
|
|
420
|
-
|
|
421
|
-
// ── TC11-TC13: Fail-open unavailability detection ──────────────────────────────
|
|
422
|
-
//
|
|
423
|
-
// These tests use NF_CLAUDE_JSON env var to inject a deterministic ~/.claude.json
|
|
424
|
-
// substitute. The hook must read this env var via:
|
|
425
|
-
// const claudeJsonPath = process.env.NF_CLAUDE_JSON || path.join(os.homedir(), '.claude.json');
|
|
426
|
-
// This env var is for testing only — production always uses ~/.claude.json.
|
|
427
|
-
//
|
|
428
|
-
// TC11: Model prefix not in mcpServers → unavailable → fail-open (pass)
|
|
429
|
-
// TC12: Partial availability — one model unavailable (pass), one available+missing (block)
|
|
430
|
-
// TC13: MCP-06 regression — renamed prefix matched correctly (pass)
|
|
431
|
-
|
|
432
|
-
// TC11: opencode prefix not in mcpServers (empty servers) → unavailable → pass (fail-open)
|
|
433
|
-
test('TC11: model prefix not in mcpServers → unavailable → fail-open pass', () => {
|
|
434
|
-
// Create temp ~/.claude.json substitute with empty mcpServers
|
|
435
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-tc11-${Date.now()}.json`);
|
|
436
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers: {} }), 'utf8');
|
|
437
|
-
|
|
438
|
-
// Transcript: quorum command issued, but no quorum tool_use calls at all
|
|
439
|
-
const tmpFile = writeTempTranscript([
|
|
440
|
-
userLine('/gsd:plan-phase 1', 'human-msg'),
|
|
441
|
-
assistantLine([{ type: 'text', text: 'Here is the plan.' }], 'assistant-1'),
|
|
442
|
-
]);
|
|
443
|
-
|
|
444
|
-
try {
|
|
445
|
-
// Config requires codex-cli prefix; empty mcpServers → codex-cli unavailable → pass
|
|
446
|
-
const configPayload = JSON.stringify({
|
|
447
|
-
quorum_commands: ['plan-phase'],
|
|
448
|
-
fail_mode: 'open',
|
|
449
|
-
required_models: {
|
|
450
|
-
codex: { tool_prefix: 'mcp__codex-cli__', required: true },
|
|
451
|
-
},
|
|
452
|
-
});
|
|
453
|
-
const configTmp = path.join(os.tmpdir(), `nf-cfg-tc11-${Date.now()}.json`);
|
|
454
|
-
const nfConfigDir = path.join(os.tmpdir(), `nf-home-tc11-${Date.now()}`);
|
|
455
|
-
fs.mkdirSync(nfConfigDir, { recursive: true });
|
|
456
|
-
fs.writeFileSync(path.join(nfConfigDir, 'nf.json'), configPayload, 'utf8');
|
|
457
|
-
|
|
458
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
459
|
-
{
|
|
460
|
-
stop_hook_active: false,
|
|
461
|
-
hook_event_name: 'Stop',
|
|
462
|
-
transcript_path: tmpFile,
|
|
463
|
-
last_assistant_message: 'Here is the plan.',
|
|
464
|
-
},
|
|
465
|
-
{
|
|
466
|
-
NF_CLAUDE_JSON: claudeJsonTmp,
|
|
467
|
-
HOME: nfConfigDir, // Makes loadConfig() read from our temp ~/.claude/nf.json
|
|
468
|
-
}
|
|
469
|
-
);
|
|
470
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 — unavailable model → fail-open pass');
|
|
471
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — no block for unavailable model');
|
|
472
|
-
} finally {
|
|
473
|
-
fs.unlinkSync(tmpFile);
|
|
474
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
475
|
-
}
|
|
476
|
-
});
|
|
477
|
-
|
|
478
|
-
// TC12: Partial availability — gemini not in mcpServers (unavailable → skip), codex IS in mcpServers but not called → block
|
|
479
|
-
// TC12 updated (step 1a): includes a PLAN.md artifact commit so GUARD 5 classifies this as a
|
|
480
|
-
// decision turn, preserving the invariant: quorum-command + decision-turn + available-but-missing = block.
|
|
481
|
-
test('TC12: partial availability — unavailable model skipped, available-but-missing model blocks', () => {
|
|
482
|
-
// Create temp ~/.claude.json with only codex-cli in mcpServers (gemini absent)
|
|
483
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-tc12-${Date.now()}.json`);
|
|
484
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers: { 'codex-cli': {} } }), 'utf8');
|
|
485
|
-
|
|
486
|
-
// Transcript: quorum command issued, PLAN.md artifact commit, no quorum calls
|
|
487
|
-
const tmpFile = writeTempTranscript([
|
|
488
|
-
userLine('/gsd:plan-phase 1', 'human-msg'),
|
|
489
|
-
assistantLine([
|
|
490
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "feat: plan" --files 04-01-PLAN.md'),
|
|
491
|
-
], 'assistant-commit'),
|
|
492
|
-
assistantLine([{ type: 'text', text: 'Here is the plan.' }], 'assistant-1'),
|
|
493
|
-
]);
|
|
494
|
-
|
|
495
|
-
try {
|
|
496
|
-
const configPayload = JSON.stringify({
|
|
497
|
-
quorum_commands: ['plan-phase'],
|
|
498
|
-
fail_mode: 'open',
|
|
499
|
-
required_models: {
|
|
500
|
-
codex: { tool_prefix: 'mcp__codex-cli__', required: true },
|
|
501
|
-
gemini: { tool_prefix: 'mcp__gemini-cli__', required: true },
|
|
502
|
-
},
|
|
503
|
-
});
|
|
504
|
-
const nfConfigDir = path.join(os.tmpdir(), `nf-home-tc12-${Date.now()}`);
|
|
505
|
-
fs.mkdirSync(nfConfigDir, { recursive: true });
|
|
506
|
-
fs.writeFileSync(path.join(nfConfigDir, 'nf.json'), configPayload, 'utf8');
|
|
507
|
-
|
|
508
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
509
|
-
{
|
|
510
|
-
stop_hook_active: false,
|
|
511
|
-
hook_event_name: 'Stop',
|
|
512
|
-
transcript_path: tmpFile,
|
|
513
|
-
last_assistant_message: 'Here is the plan.',
|
|
514
|
-
},
|
|
515
|
-
{
|
|
516
|
-
NF_CLAUDE_JSON: claudeJsonTmp,
|
|
517
|
-
HOME: nfConfigDir,
|
|
518
|
-
}
|
|
519
|
-
);
|
|
520
|
-
// codex IS in mcpServers but was not called → block
|
|
521
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 — hook communicates via stdout JSON, not exit code');
|
|
522
|
-
const parsed = JSON.parse(stdout);
|
|
523
|
-
assert.strictEqual(parsed.decision, 'block', 'should block — codex available+missing');
|
|
524
|
-
assert.ok(parsed.reason.includes('codex') || parsed.reason.includes('mcp__codex-cli__'), 'block reason should name codex');
|
|
525
|
-
} finally {
|
|
526
|
-
fs.unlinkSync(tmpFile);
|
|
527
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
528
|
-
}
|
|
529
|
-
});
|
|
530
|
-
|
|
531
|
-
// TC13: MCP-06 regression — renamed prefix matched correctly (pass when called)
|
|
532
|
-
test('TC13: MCP-06 regression — renamed prefix detected and matched correctly', () => {
|
|
533
|
-
// Config has a custom prefix (renamed MCP server); mcpServers has that server; transcript has a call
|
|
534
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-tc13-${Date.now()}.json`);
|
|
535
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers: { 'my-custom-codex': {} } }), 'utf8');
|
|
536
|
-
|
|
537
|
-
const tmpFile = writeTempTranscript([
|
|
538
|
-
userLine('/gsd:plan-phase 1', 'human-msg'),
|
|
539
|
-
assistantLine([toolUseBlock('mcp__my-custom-codex__review')], 'assistant-1'),
|
|
540
|
-
assistantLine([{ type: 'text', text: 'Plan with custom codex.' }], 'assistant-2'),
|
|
541
|
-
]);
|
|
542
|
-
|
|
543
|
-
try {
|
|
544
|
-
const configPayload = JSON.stringify({
|
|
545
|
-
quorum_commands: ['plan-phase'],
|
|
546
|
-
fail_mode: 'open',
|
|
547
|
-
required_models: {
|
|
548
|
-
custom: { tool_prefix: 'mcp__my-custom-codex__', required: true },
|
|
549
|
-
},
|
|
550
|
-
});
|
|
551
|
-
const nfConfigDir = path.join(os.tmpdir(), `nf-home-tc13-${Date.now()}`);
|
|
552
|
-
fs.mkdirSync(nfConfigDir, { recursive: true });
|
|
553
|
-
fs.writeFileSync(path.join(nfConfigDir, 'nf.json'), configPayload, 'utf8');
|
|
554
|
-
|
|
555
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
556
|
-
{
|
|
557
|
-
stop_hook_active: false,
|
|
558
|
-
hook_event_name: 'Stop',
|
|
559
|
-
transcript_path: tmpFile,
|
|
560
|
-
last_assistant_message: 'Plan with custom codex.',
|
|
561
|
-
},
|
|
562
|
-
{
|
|
563
|
-
NF_CLAUDE_JSON: claudeJsonTmp,
|
|
564
|
-
HOME: nfConfigDir,
|
|
565
|
-
}
|
|
566
|
-
);
|
|
567
|
-
// custom prefix IS in mcpServers AND was called → pass
|
|
568
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
569
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — renamed prefix found evidence → pass');
|
|
570
|
-
} finally {
|
|
571
|
-
fs.unlinkSync(tmpFile);
|
|
572
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
573
|
-
}
|
|
574
|
-
});
|
|
575
|
-
|
|
576
|
-
// ── TC-COPILOT: deriveMissingToolName returns 'ask' for modelKey 'copilot' ──────────────────────
|
|
577
|
-
//
|
|
578
|
-
// Verifies that when copilot is a required model and is not called on a decision turn,
|
|
579
|
-
// the block reason correctly names mcp__copilot-cli__ask (not mcp__copilot-cli__copilot).
|
|
580
|
-
//
|
|
581
|
-
// Uses NF_CLAUDE_JSON with copilot-cli in mcpServers to force it to be treated as available,
|
|
582
|
-
// so it triggers a block (not unavailable skip).
|
|
583
|
-
|
|
584
|
-
test('TC-COPILOT: deriveMissingToolName returns "ask" for copilot — block reason names mcp__copilot-cli__ask', () => {
|
|
585
|
-
// claude.json with copilot-cli in mcpServers → available
|
|
586
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-tc-copilot-${Date.now()}.json`);
|
|
587
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers: { 'copilot-cli': {} } }), 'utf8');
|
|
588
|
-
|
|
589
|
-
// Config: only copilot required
|
|
590
|
-
const configPayload = JSON.stringify({
|
|
591
|
-
quorum_commands: ['plan-phase'],
|
|
592
|
-
fail_mode: 'open',
|
|
593
|
-
required_models: {
|
|
594
|
-
copilot: { tool_prefix: 'mcp__copilot-cli__', required: true },
|
|
595
|
-
},
|
|
596
|
-
});
|
|
597
|
-
const nfConfigDir = path.join(os.tmpdir(), `nf-home-tc-copilot-${Date.now()}`);
|
|
598
|
-
fs.mkdirSync(nfConfigDir, { recursive: true });
|
|
599
|
-
fs.writeFileSync(path.join(nfConfigDir, 'nf.json'), configPayload, 'utf8');
|
|
600
|
-
|
|
601
|
-
// Transcript: plan-phase command + PLAN.md artifact commit + no copilot tool call
|
|
602
|
-
const tmpFile = writeTempTranscript([
|
|
603
|
-
userLine('/gsd:plan-phase 1', 'human-msg'),
|
|
604
|
-
assistantLine([
|
|
605
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "feat: plan" --files 04-01-PLAN.md'),
|
|
606
|
-
], 'assistant-commit'),
|
|
607
|
-
assistantLine([{ type: 'text', text: 'Here is the plan.' }], 'assistant-1'),
|
|
608
|
-
]);
|
|
609
|
-
|
|
610
|
-
try {
|
|
611
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
612
|
-
{
|
|
613
|
-
stop_hook_active: false,
|
|
614
|
-
hook_event_name: 'Stop',
|
|
615
|
-
transcript_path: tmpFile,
|
|
616
|
-
last_assistant_message: 'Here is the plan.',
|
|
617
|
-
},
|
|
618
|
-
{
|
|
619
|
-
NF_CLAUDE_JSON: claudeJsonTmp,
|
|
620
|
-
HOME: nfConfigDir,
|
|
621
|
-
}
|
|
622
|
-
);
|
|
623
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
624
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision');
|
|
625
|
-
const parsed = JSON.parse(stdout);
|
|
626
|
-
assert.strictEqual(parsed.decision, 'block', 'should block — copilot available+missing');
|
|
627
|
-
assert.ok(
|
|
628
|
-
parsed.reason.includes('mcp__copilot-cli__ask'),
|
|
629
|
-
'block reason must name mcp__copilot-cli__ask (not mcp__copilot-cli__copilot)'
|
|
630
|
-
);
|
|
631
|
-
} finally {
|
|
632
|
-
fs.unlinkSync(tmpFile);
|
|
633
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
634
|
-
}
|
|
635
|
-
});
|
|
636
|
-
|
|
637
|
-
// ── TC14-TC19: GUARD 5 — Decision turn detection (SCOPE-01/02/03/05/06/07) ─────────────────────
|
|
638
|
-
//
|
|
639
|
-
// TC14: intermediate plan-phase turn (no artifact commit, no marker) → PASS (not a decision turn)
|
|
640
|
-
// TC15: final plan-phase turn with PLAN.md artifact committed → QUORUM REQUIRED (decision turn)
|
|
641
|
-
// TC16: map-codebase turn (codebase/*.md commit, no artifact pattern match) → PASS
|
|
642
|
-
// TC17: new-project routing turn (no artifact commit, no marker) → PASS
|
|
643
|
-
// TC18: discuss-phase final turn with CONTEXT.md artifact committed → QUORUM REQUIRED
|
|
644
|
-
// TC19: verify-work turn with <!-- GSD_DECISION --> marker in last text block → QUORUM REQUIRED
|
|
645
|
-
|
|
646
|
-
// TC14: intermediate plan-phase turn — assistant spawns an agent, no artifact commit, no marker
|
|
647
|
-
test('TC14: intermediate plan-phase turn (no artifact commit, no marker) passes without quorum block', () => {
|
|
648
|
-
const tmpFile = writeTempTranscript([
|
|
649
|
-
userLine('/gsd:plan-phase 1'),
|
|
650
|
-
assistantLine([{ type: 'text', text: 'Spawning researcher agent...' }]),
|
|
651
|
-
]);
|
|
652
|
-
try {
|
|
653
|
-
const { stdout, exitCode } = runHook({
|
|
654
|
-
stop_hook_active: false,
|
|
655
|
-
hook_event_name: 'Stop',
|
|
656
|
-
transcript_path: tmpFile,
|
|
657
|
-
last_assistant_message: 'Spawning researcher agent...',
|
|
658
|
-
});
|
|
659
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
660
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — intermediate turn is not a decision turn');
|
|
661
|
-
} finally {
|
|
662
|
-
fs.unlinkSync(tmpFile);
|
|
663
|
-
}
|
|
664
|
-
});
|
|
665
|
-
|
|
666
|
-
// TC15: final plan-phase turn — PLAN.md artifact committed + no quorum calls → QUORUM REQUIRED
|
|
667
|
-
test('TC15: final plan-phase turn with PLAN.md artifact committed blocks when quorum missing', () => {
|
|
668
|
-
const tmpFile = writeTempTranscript([
|
|
669
|
-
userLine('/gsd:plan-phase 1'),
|
|
670
|
-
assistantLine([
|
|
671
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "feat: plan" --files 04-01-PLAN.md'),
|
|
672
|
-
], 'assistant-commit'),
|
|
673
|
-
assistantLine([{ type: 'text', text: 'Here is the plan.' }], 'assistant-2'),
|
|
674
|
-
]);
|
|
675
|
-
try {
|
|
676
|
-
const { stdout, exitCode } = runHook({
|
|
677
|
-
stop_hook_active: false,
|
|
678
|
-
hook_event_name: 'Stop',
|
|
679
|
-
transcript_path: tmpFile,
|
|
680
|
-
last_assistant_message: 'Here is the plan.',
|
|
681
|
-
});
|
|
682
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
683
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision JSON');
|
|
684
|
-
const parsed = JSON.parse(stdout);
|
|
685
|
-
assert.strictEqual(parsed.decision, 'block', 'decision must be "block"');
|
|
686
|
-
assert.ok(parsed.reason.startsWith('QUORUM REQUIRED:'), 'reason must start with QUORUM REQUIRED:');
|
|
687
|
-
} finally {
|
|
688
|
-
fs.unlinkSync(tmpFile);
|
|
689
|
-
}
|
|
690
|
-
});
|
|
691
|
-
|
|
692
|
-
// TC16: map-codebase turn — commits codebase/STACK.md (no artifact pattern match) → PASS
|
|
693
|
-
// Guards against Pitfall 2 from RESEARCH.md: bare STACK.md must NOT trigger artifact detection.
|
|
694
|
-
test('TC16: map-codebase turn with codebase/*.md commit passes without quorum block', () => {
|
|
695
|
-
const tmpFile = writeTempTranscript([
|
|
696
|
-
userLine('/gsd:plan-phase 1'),
|
|
697
|
-
assistantLine([
|
|
698
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "docs: codebase" --files .planning/codebase/STACK.md'),
|
|
699
|
-
], 'assistant-commit'),
|
|
700
|
-
assistantLine([{ type: 'text', text: 'Codebase mapped.' }], 'assistant-2'),
|
|
701
|
-
]);
|
|
702
|
-
try {
|
|
703
|
-
const { stdout, exitCode } = runHook({
|
|
704
|
-
stop_hook_active: false,
|
|
705
|
-
hook_event_name: 'Stop',
|
|
706
|
-
transcript_path: tmpFile,
|
|
707
|
-
last_assistant_message: 'Codebase mapped.',
|
|
708
|
-
});
|
|
709
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
710
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — codebase/*.md is not a planning artifact');
|
|
711
|
-
} finally {
|
|
712
|
-
fs.unlinkSync(tmpFile);
|
|
713
|
-
}
|
|
714
|
-
});
|
|
715
|
-
|
|
716
|
-
// TC17: new-project routing turn — assistant asks a question, no artifact commit, no marker → PASS
|
|
717
|
-
test('TC17: new-project routing turn (questioning step) passes without quorum block', () => {
|
|
718
|
-
const tmpFile = writeTempTranscript([
|
|
719
|
-
userLine('/gsd:new-project'),
|
|
720
|
-
assistantLine([{ type: 'text', text: 'What do you want to build?' }]),
|
|
721
|
-
]);
|
|
722
|
-
try {
|
|
723
|
-
const { stdout, exitCode } = runHook({
|
|
724
|
-
stop_hook_active: false,
|
|
725
|
-
hook_event_name: 'Stop',
|
|
726
|
-
transcript_path: tmpFile,
|
|
727
|
-
last_assistant_message: 'What do you want to build?',
|
|
728
|
-
});
|
|
729
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
730
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — routing/questioning turn is not a decision turn');
|
|
731
|
-
} finally {
|
|
732
|
-
fs.unlinkSync(tmpFile);
|
|
733
|
-
}
|
|
734
|
-
});
|
|
735
|
-
|
|
736
|
-
// TC18: discuss-phase final turn — CONTEXT.md artifact committed + no quorum calls → QUORUM REQUIRED
|
|
737
|
-
test('TC18: discuss-phase final turn with CONTEXT.md artifact committed blocks when quorum missing', () => {
|
|
738
|
-
const tmpFile = writeTempTranscript([
|
|
739
|
-
userLine('/gsd:discuss-phase 4'),
|
|
740
|
-
assistantLine([
|
|
741
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "docs: context" --files 04-CONTEXT.md'),
|
|
742
|
-
], 'assistant-commit'),
|
|
743
|
-
assistantLine([{ type: 'text', text: 'Here are the filtered questions.' }], 'assistant-2'),
|
|
744
|
-
]);
|
|
745
|
-
try {
|
|
746
|
-
const { stdout, exitCode } = runHook({
|
|
747
|
-
stop_hook_active: false,
|
|
748
|
-
hook_event_name: 'Stop',
|
|
749
|
-
transcript_path: tmpFile,
|
|
750
|
-
last_assistant_message: 'Here are the filtered questions.',
|
|
751
|
-
});
|
|
752
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
753
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision JSON');
|
|
754
|
-
const parsed = JSON.parse(stdout);
|
|
755
|
-
assert.strictEqual(parsed.decision, 'block', 'decision must be "block"');
|
|
756
|
-
assert.ok(parsed.reason.startsWith('QUORUM REQUIRED:'), 'reason must start with QUORUM REQUIRED:');
|
|
757
|
-
} finally {
|
|
758
|
-
fs.unlinkSync(tmpFile);
|
|
759
|
-
}
|
|
760
|
-
});
|
|
761
|
-
|
|
762
|
-
// TC19: verify-work turn with <!-- GSD_DECISION --> in last assistant text block → QUORUM REQUIRED
|
|
763
|
-
test('TC19: verify-work turn with decision marker in last assistant text block blocks when quorum missing', () => {
|
|
764
|
-
const tmpFile = writeTempTranscript([
|
|
765
|
-
userLine('/gsd:verify-work'),
|
|
766
|
-
assistantLine([{ type: 'text', text: 'Verification complete.\n\n<!-- GSD_DECISION -->' }]),
|
|
767
|
-
]);
|
|
768
|
-
try {
|
|
769
|
-
const { stdout, exitCode } = runHook({
|
|
770
|
-
stop_hook_active: false,
|
|
771
|
-
hook_event_name: 'Stop',
|
|
772
|
-
transcript_path: tmpFile,
|
|
773
|
-
last_assistant_message: 'Verification complete.\n\n<!-- GSD_DECISION -->',
|
|
774
|
-
});
|
|
775
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
776
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision JSON');
|
|
777
|
-
const parsed = JSON.parse(stdout);
|
|
778
|
-
assert.strictEqual(parsed.decision, 'block', 'decision must be "block"');
|
|
779
|
-
assert.ok(parsed.reason.startsWith('QUORUM REQUIRED:'), 'reason must start with QUORUM REQUIRED:');
|
|
780
|
-
} finally {
|
|
781
|
-
fs.unlinkSync(tmpFile);
|
|
782
|
-
}
|
|
783
|
-
});
|
|
784
|
-
|
|
785
|
-
// ── TC20/TC20b/TC20c: @file-expansion false-positive regression ───────────────────────────────
|
|
786
|
-
//
|
|
787
|
-
// When Claude Code expands a workflow file (e.g. quick.md) via @-reference, the expanded content
|
|
788
|
-
// is appended to the user message body. Workflow files often mention other /qgsd: commands by name
|
|
789
|
-
// (e.g. "If you meant /qgsd:new-project, run that instead."). The old JSON.stringify full-body
|
|
790
|
-
// scan would match these mentions and trigger a false-positive GUARD 4 hit.
|
|
791
|
-
//
|
|
792
|
-
// Fix: hasQuorumCommand reads the <command-name> XML tag first. This tag is injected by Claude
|
|
793
|
-
// Code only for real slash command invocations — never in @file-expanded content. When the tag
|
|
794
|
-
// is present, only the tag value is tested; the body is never scanned.
|
|
795
|
-
//
|
|
796
|
-
// TC20 — false-positive regression: /qgsd:quick tag, body mentions /qgsd:new-project → pass
|
|
797
|
-
// TC20b — positive control: /qgsd:new-project real tag, questioning turn → pass (GUARD 5)
|
|
798
|
-
// TC20c — end-to-end: /qgsd:new-project real tag + decision turn + no quorum → block
|
|
799
|
-
|
|
800
|
-
// Helper: build a user JSONL line whose message.content begins with the <command-name> XML tag
|
|
801
|
-
// (simulating Claude Code's injection for real slash command invocations) followed by the body.
|
|
802
|
-
function userLineWithTag(commandTag, bodyText, uuid) {
|
|
803
|
-
const content = '<command-name>' + commandTag + '</command-name>\n\n' + bodyText;
|
|
804
|
-
return JSON.stringify({
|
|
805
|
-
type: 'user',
|
|
806
|
-
message: { role: 'user', content },
|
|
807
|
-
timestamp: '2026-02-20T00:00:00Z',
|
|
808
|
-
uuid: uuid || 'user-tagged',
|
|
809
|
-
});
|
|
810
|
-
}
|
|
811
|
-
|
|
812
|
-
// TC20 — The false-positive regression:
|
|
813
|
-
// User invokes /qgsd:quick (tag = "/qgsd:quick"); body contains "new-project" text from
|
|
814
|
-
// expanded quick.md workflow. With the fix, the tag is read first ("/qgsd:quick" is not in
|
|
815
|
-
// quorum_commands) → GUARD 4 returns false → exit 0. Body is never scanned.
|
|
816
|
-
test('TC20: @file-expanded body containing /qgsd:new-project text does not false-positive when real command is /qgsd:quick', () => {
|
|
817
|
-
const expandedBody =
|
|
818
|
-
'Execute the quick task.\n\n' +
|
|
819
|
-
'If you meant /qgsd:new-project, run that instead. ' +
|
|
820
|
-
'See /qgsd:new-project documentation for details.';
|
|
821
|
-
const tmpFile = writeTempTranscript([
|
|
822
|
-
userLineWithTag('/qgsd:quick', expandedBody, 'user-quick'),
|
|
823
|
-
assistantLine([{ type: 'text', text: 'Running quick task.' }], 'assistant-1'),
|
|
824
|
-
]);
|
|
825
|
-
try {
|
|
826
|
-
const { stdout, exitCode } = runHook({
|
|
827
|
-
stop_hook_active: false,
|
|
828
|
-
hook_event_name: 'Stop',
|
|
829
|
-
transcript_path: tmpFile,
|
|
830
|
-
last_assistant_message: 'Running quick task.',
|
|
831
|
-
});
|
|
832
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 — /qgsd:quick is not a quorum command');
|
|
833
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — new-project in body must not trigger GUARD 4');
|
|
834
|
-
} finally {
|
|
835
|
-
fs.unlinkSync(tmpFile);
|
|
836
|
-
}
|
|
837
|
-
});
|
|
838
|
-
|
|
839
|
-
// TC20b — Positive control: new-project IS the real command (tag present), but no artifact commit
|
|
840
|
-
// and no decision marker — GUARD 5 passes (routing/questioning turn).
|
|
841
|
-
// Verifies the XML tag strategy correctly identifies real /qgsd:new-project invocations.
|
|
842
|
-
test('TC20b: real /qgsd:new-project tag on a questioning turn passes (GUARD 5 — not a decision turn)', () => {
|
|
843
|
-
const tmpFile = writeTempTranscript([
|
|
844
|
-
userLineWithTag('/qgsd:new-project', 'I want to start a new project.', 'user-np'),
|
|
845
|
-
assistantLine([{ type: 'text', text: 'What do you want to build?' }], 'assistant-1'),
|
|
846
|
-
]);
|
|
847
|
-
try {
|
|
848
|
-
const { stdout, exitCode } = runHook({
|
|
849
|
-
stop_hook_active: false,
|
|
850
|
-
hook_event_name: 'Stop',
|
|
851
|
-
transcript_path: tmpFile,
|
|
852
|
-
last_assistant_message: 'What do you want to build?',
|
|
853
|
-
});
|
|
854
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 — questioning turn is not a decision turn');
|
|
855
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — GUARD 5 passes, no artifact or marker');
|
|
856
|
-
} finally {
|
|
857
|
-
fs.unlinkSync(tmpFile);
|
|
858
|
-
}
|
|
859
|
-
});
|
|
860
|
-
|
|
861
|
-
// TC20c — End-to-end: new-project IS real (tag) + ROADMAP.md artifact commit (decision turn) + no quorum → block
|
|
862
|
-
// Verifies real /qgsd:new-project invocations still trigger quorum enforcement on decision turns.
|
|
863
|
-
test('TC20c: real /qgsd:new-project tag on a decision turn blocks when quorum missing', () => {
|
|
864
|
-
const tmpFile = writeTempTranscript([
|
|
865
|
-
userLineWithTag('/qgsd:new-project', 'Build a task management app.', 'user-np2'),
|
|
866
|
-
assistantLine([
|
|
867
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "docs: roadmap" --files ROADMAP.md'),
|
|
868
|
-
], 'assistant-commit'),
|
|
869
|
-
assistantLine([{ type: 'text', text: 'Here is the roadmap.' }], 'assistant-2'),
|
|
870
|
-
]);
|
|
871
|
-
try {
|
|
872
|
-
const { stdout, exitCode } = runHook({
|
|
873
|
-
stop_hook_active: false,
|
|
874
|
-
hook_event_name: 'Stop',
|
|
875
|
-
transcript_path: tmpFile,
|
|
876
|
-
last_assistant_message: 'Here is the roadmap.',
|
|
877
|
-
});
|
|
878
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
879
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision JSON');
|
|
880
|
-
const parsed = JSON.parse(stdout);
|
|
881
|
-
assert.strictEqual(parsed.decision, 'block', 'decision must be "block"');
|
|
882
|
-
assert.ok(parsed.reason.startsWith('QUORUM REQUIRED:'), 'reason must start with QUORUM REQUIRED:');
|
|
883
|
-
} finally {
|
|
884
|
-
fs.unlinkSync(tmpFile);
|
|
885
|
-
}
|
|
886
|
-
});
|
|
887
|
-
|
|
888
|
-
// ── TC-CEIL-1/2/3: Hard ceiling and error-response exclusion ──────────────────────────────────
|
|
889
|
-
//
|
|
890
|
-
// TC-CEIL-1: ceiling passes with exactly 5 successful calls out of an 11-agent pool
|
|
891
|
-
// TC-CEIL-2: ceiling blocks when only 4 of 5 required agents have been called
|
|
892
|
-
// TC-CEIL-3: error response does not count — 5 calls made but one is error → still blocks
|
|
893
|
-
//
|
|
894
|
-
// These tests use quorum_active (not required_models) and minSize config to exercise
|
|
895
|
-
// the success-counter loop in nf-stop.js main().
|
|
896
|
-
|
|
897
|
-
// Helper: build a user JSONL line with a tool_result that has is_error:true
|
|
898
|
-
// Simulates an agent returning a quota/error response.
|
|
899
|
-
function toolResultErrorLine(toolUseId, errorText, uuid) {
|
|
900
|
-
return JSON.stringify({
|
|
901
|
-
type: 'user',
|
|
902
|
-
message: {
|
|
903
|
-
role: 'user',
|
|
904
|
-
content: [{
|
|
905
|
-
type: 'tool_result',
|
|
906
|
-
tool_use_id: toolUseId,
|
|
907
|
-
is_error: true,
|
|
908
|
-
content: [{ type: 'text', text: errorText }],
|
|
909
|
-
}],
|
|
910
|
-
},
|
|
911
|
-
timestamp: '2026-02-20T00:02:00Z',
|
|
912
|
-
uuid: uuid || `tr-err-${toolUseId}`,
|
|
913
|
-
});
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
// Helper: build a user JSONL line with a successful (non-error) tool_result
|
|
917
|
-
function toolResultSuccessLine(toolUseId, resultText, uuid) {
|
|
918
|
-
return JSON.stringify({
|
|
919
|
-
type: 'user',
|
|
920
|
-
message: {
|
|
921
|
-
role: 'user',
|
|
922
|
-
content: [{
|
|
923
|
-
type: 'tool_result',
|
|
924
|
-
tool_use_id: toolUseId,
|
|
925
|
-
content: [{ type: 'text', text: resultText }],
|
|
926
|
-
}],
|
|
927
|
-
},
|
|
928
|
-
timestamp: '2026-02-20T00:02:00Z',
|
|
929
|
-
uuid: uuid || `tr-ok-${toolUseId}`,
|
|
930
|
-
});
|
|
931
|
-
}
|
|
932
|
-
|
|
933
|
-
// TC-CEIL-1: 11-agent pool, minSize=5, preferSub=true
|
|
934
|
-
// First 5 sorted agents (4 sub + 1 api) all have successful responses → ceiling satisfied → pass
|
|
935
|
-
test('TC-CEIL-1: ceiling passes with exactly 5 successful calls out of 11-agent pool', () => {
|
|
936
|
-
// Build config: 11 slots (4 sub + 7 api), minSize = 5, preferSub = true
|
|
937
|
-
const slots = [
|
|
938
|
-
'slot-sub-1', 'slot-sub-2', 'slot-sub-3', 'slot-sub-4', // sub agents
|
|
939
|
-
'slot-api-1', 'slot-api-2', 'slot-api-3', 'slot-api-4',
|
|
940
|
-
'slot-api-5', 'slot-api-6', 'slot-api-7', // api agents
|
|
941
|
-
];
|
|
942
|
-
const agentConfig = {};
|
|
943
|
-
for (const s of slots) {
|
|
944
|
-
agentConfig[s] = { auth_type: s.startsWith('slot-sub') ? 'sub' : 'api' };
|
|
945
|
-
}
|
|
946
|
-
const configPayload = JSON.stringify({
|
|
947
|
-
quorum_commands: ['plan-phase'],
|
|
948
|
-
quorum_active: slots,
|
|
949
|
-
agent_config: agentConfig,
|
|
950
|
-
quorum: { minSize: 5, preferSub: true },
|
|
951
|
-
});
|
|
952
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-ceil1-${Date.now()}`);
|
|
953
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
954
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
955
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload, 'utf8');
|
|
956
|
-
|
|
957
|
-
// ~/.claude.json lists all 11 slots as available MCP servers
|
|
958
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-ceil1-${Date.now()}.json`);
|
|
959
|
-
const mcpServers = {};
|
|
960
|
-
for (const s of slots) mcpServers[s] = {};
|
|
961
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }), 'utf8');
|
|
962
|
-
|
|
963
|
-
// Transcript: plan-phase command + PLAN.md commit + first 5 agents called with successful results
|
|
964
|
-
// Sorted order (sub-first): slot-sub-1, slot-sub-2, slot-sub-3, slot-sub-4, slot-api-1
|
|
965
|
-
const first5 = ['slot-sub-1', 'slot-sub-2', 'slot-sub-3', 'slot-sub-4', 'slot-api-1'];
|
|
966
|
-
const transcriptLines = [
|
|
967
|
-
userLine('/qgsd:plan-phase 1', 'human-ceil1'),
|
|
968
|
-
assistantLine([
|
|
969
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files 04-01-PLAN.md'),
|
|
970
|
-
], 'assistant-commit'),
|
|
971
|
-
];
|
|
972
|
-
for (const slot of first5) {
|
|
973
|
-
const toolName = `mcp__${slot}__review`;
|
|
974
|
-
transcriptLines.push(
|
|
975
|
-
assistantLine([toolUseBlock(toolName)], `assistant-${slot}`),
|
|
976
|
-
toolResultSuccessLine(`toolu_${toolName}`, `${slot} review OK`, `tr-${slot}`)
|
|
977
|
-
);
|
|
978
|
-
}
|
|
979
|
-
transcriptLines.push(
|
|
980
|
-
assistantLine([{ type: 'text', text: 'Plan complete with ceiling satisfied.' }], 'assistant-final')
|
|
981
|
-
);
|
|
982
|
-
|
|
983
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
984
|
-
|
|
985
|
-
try {
|
|
986
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
987
|
-
{
|
|
988
|
-
stop_hook_active: false,
|
|
989
|
-
hook_event_name: 'Stop',
|
|
990
|
-
transcript_path: tmpFile,
|
|
991
|
-
last_assistant_message: 'Plan complete with ceiling satisfied.',
|
|
992
|
-
},
|
|
993
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
994
|
-
);
|
|
995
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0');
|
|
996
|
-
assert.strictEqual(stdout, '', 'stdout must be empty — 5 successful responses satisfies ceiling');
|
|
997
|
-
} finally {
|
|
998
|
-
fs.unlinkSync(tmpFile);
|
|
999
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1000
|
-
}
|
|
1001
|
-
});
|
|
1002
|
-
|
|
1003
|
-
// TC-CEIL-2: Same 11-agent pool, minSize=5, but only 4 of the first 5 agents called → block
|
|
1004
|
-
test('TC-CEIL-2: ceiling blocks when only 4 of 5 required agents have been called', () => {
|
|
1005
|
-
const slots = [
|
|
1006
|
-
'slot-sub-1', 'slot-sub-2', 'slot-sub-3', 'slot-sub-4',
|
|
1007
|
-
'slot-api-1', 'slot-api-2', 'slot-api-3', 'slot-api-4',
|
|
1008
|
-
'slot-api-5', 'slot-api-6', 'slot-api-7',
|
|
1009
|
-
];
|
|
1010
|
-
const agentConfig = {};
|
|
1011
|
-
for (const s of slots) {
|
|
1012
|
-
agentConfig[s] = { auth_type: s.startsWith('slot-sub') ? 'sub' : 'api' };
|
|
1013
|
-
}
|
|
1014
|
-
const configPayload = JSON.stringify({
|
|
1015
|
-
quorum_commands: ['plan-phase'],
|
|
1016
|
-
quorum_active: slots,
|
|
1017
|
-
agent_config: agentConfig,
|
|
1018
|
-
quorum: { maxSize: 5, preferSub: true },
|
|
1019
|
-
});
|
|
1020
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-ceil2-${Date.now()}`);
|
|
1021
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
1022
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
1023
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload, 'utf8');
|
|
1024
|
-
|
|
1025
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-ceil2-${Date.now()}.json`);
|
|
1026
|
-
const mcpServers = {};
|
|
1027
|
-
for (const s of slots) mcpServers[s] = {};
|
|
1028
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }), 'utf8');
|
|
1029
|
-
|
|
1030
|
-
// Only 4 of the first 5 sorted agents called (skip slot-api-1 — the 5th)
|
|
1031
|
-
const only4 = ['slot-sub-1', 'slot-sub-2', 'slot-sub-3', 'slot-sub-4'];
|
|
1032
|
-
const transcriptLines = [
|
|
1033
|
-
userLine('/qgsd:plan-phase 1', 'human-ceil2'),
|
|
1034
|
-
assistantLine([
|
|
1035
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files 04-01-PLAN.md'),
|
|
1036
|
-
], 'assistant-commit'),
|
|
1037
|
-
];
|
|
1038
|
-
for (const slot of only4) {
|
|
1039
|
-
const toolName = `mcp__${slot}__review`;
|
|
1040
|
-
transcriptLines.push(
|
|
1041
|
-
assistantLine([toolUseBlock(toolName)], `assistant-${slot}`),
|
|
1042
|
-
toolResultSuccessLine(`toolu_${toolName}`, `${slot} review OK`, `tr-${slot}`)
|
|
1043
|
-
);
|
|
1044
|
-
}
|
|
1045
|
-
transcriptLines.push(
|
|
1046
|
-
assistantLine([{ type: 'text', text: 'Plan with only 4 agents.' }], 'assistant-final')
|
|
1047
|
-
);
|
|
1048
|
-
|
|
1049
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
1050
|
-
|
|
1051
|
-
try {
|
|
1052
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
1053
|
-
{
|
|
1054
|
-
stop_hook_active: false,
|
|
1055
|
-
hook_event_name: 'Stop',
|
|
1056
|
-
transcript_path: tmpFile,
|
|
1057
|
-
last_assistant_message: 'Plan with only 4 agents.',
|
|
1058
|
-
},
|
|
1059
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
1060
|
-
);
|
|
1061
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
1062
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision JSON');
|
|
1063
|
-
const parsed = JSON.parse(stdout);
|
|
1064
|
-
assert.strictEqual(parsed.decision, 'block', 'decision must be block — only 4 of 5 required');
|
|
1065
|
-
assert.ok(parsed.reason.startsWith('QUORUM REQUIRED:'), 'reason must start with QUORUM REQUIRED:');
|
|
1066
|
-
// The 5th agent (slot-api-1) should be named as missing
|
|
1067
|
-
assert.ok(parsed.reason.includes('slot-api-1'), 'block reason must name slot-api-1 as missing');
|
|
1068
|
-
} finally {
|
|
1069
|
-
fs.unlinkSync(tmpFile);
|
|
1070
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1071
|
-
}
|
|
1072
|
-
});
|
|
1073
|
-
|
|
1074
|
-
// TC-CEIL-3: error response does not count toward ceiling
|
|
1075
|
-
// 6 slots active, minSize=5 (so failover scenario: 5 calls made but slot-3 returns error → only 4 successful → block)
|
|
1076
|
-
test('TC-CEIL-3: error response does not count toward ceiling — still blocks when one call errored', () => {
|
|
1077
|
-
// 6 slots configured (>= minSize) to test the failover-beyond-ceiling scenario
|
|
1078
|
-
// minSize = 5, but one of the 5 calls returns an error → successCount = 4 → block
|
|
1079
|
-
const slots = [
|
|
1080
|
-
'slot-api-1', 'slot-api-2', 'slot-api-3',
|
|
1081
|
-
'slot-api-4', 'slot-api-5', 'slot-api-6',
|
|
1082
|
-
];
|
|
1083
|
-
const agentConfig = {};
|
|
1084
|
-
for (const s of slots) {
|
|
1085
|
-
agentConfig[s] = { auth_type: 'api' };
|
|
1086
|
-
}
|
|
1087
|
-
const configPayload = JSON.stringify({
|
|
1088
|
-
quorum_commands: ['plan-phase'],
|
|
1089
|
-
quorum_active: slots,
|
|
1090
|
-
agent_config: agentConfig,
|
|
1091
|
-
quorum: { maxSize: 5, preferSub: false },
|
|
1092
|
-
});
|
|
1093
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-ceil3-${Date.now()}`);
|
|
1094
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
1095
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
1096
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload, 'utf8');
|
|
1097
|
-
|
|
1098
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-ceil3-${Date.now()}.json`);
|
|
1099
|
-
const mcpServers = {};
|
|
1100
|
-
for (const s of slots) mcpServers[s] = {};
|
|
1101
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }), 'utf8');
|
|
1102
|
-
|
|
1103
|
-
// Transcript: all 5 first agents called, but slot-api-3's tool_result is an error
|
|
1104
|
-
// toolUseBlock(name) uses id: `toolu_${name}` pattern
|
|
1105
|
-
const firstFive = ['slot-api-1', 'slot-api-2', 'slot-api-3', 'slot-api-4', 'slot-api-5'];
|
|
1106
|
-
const transcriptLines = [
|
|
1107
|
-
userLine('/qgsd:plan-phase 1', 'human-ceil3'),
|
|
1108
|
-
assistantLine([
|
|
1109
|
-
bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files 04-01-PLAN.md'),
|
|
1110
|
-
], 'assistant-commit'),
|
|
1111
|
-
];
|
|
1112
|
-
for (const slot of firstFive) {
|
|
1113
|
-
const toolName = `mcp__${slot}__review`;
|
|
1114
|
-
transcriptLines.push(
|
|
1115
|
-
assistantLine([toolUseBlock(toolName)], `assistant-${slot}`)
|
|
1116
|
-
);
|
|
1117
|
-
if (slot === 'slot-api-3') {
|
|
1118
|
-
// Slot 3 returns quota error — does NOT count toward ceiling
|
|
1119
|
-
transcriptLines.push(
|
|
1120
|
-
toolResultErrorLine(`toolu_${toolName}`, 'quota exceeded', `tr-${slot}`)
|
|
1121
|
-
);
|
|
1122
|
-
} else {
|
|
1123
|
-
transcriptLines.push(
|
|
1124
|
-
toolResultSuccessLine(`toolu_${toolName}`, `${slot} review OK`, `tr-${slot}`)
|
|
1125
|
-
);
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
transcriptLines.push(
|
|
1129
|
-
assistantLine([{ type: 'text', text: 'Plan with one errored agent.' }], 'assistant-final')
|
|
1130
|
-
);
|
|
1131
|
-
|
|
1132
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
1133
|
-
|
|
1134
|
-
try {
|
|
1135
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
1136
|
-
{
|
|
1137
|
-
stop_hook_active: false,
|
|
1138
|
-
hook_event_name: 'Stop',
|
|
1139
|
-
transcript_path: tmpFile,
|
|
1140
|
-
last_assistant_message: 'Plan with one errored agent.',
|
|
1141
|
-
},
|
|
1142
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
1143
|
-
);
|
|
1144
|
-
// Only 4 successful responses (slot-api-3 errored) — ceiling requires 5 → block
|
|
1145
|
-
assert.strictEqual(exitCode, 0, 'exit code must be 0 even when blocking');
|
|
1146
|
-
assert.ok(stdout.length > 0, 'stdout must contain block decision JSON');
|
|
1147
|
-
const parsed = JSON.parse(stdout);
|
|
1148
|
-
assert.strictEqual(parsed.decision, 'block', 'decision must be block — error response does not count');
|
|
1149
|
-
assert.ok(parsed.reason.startsWith('QUORUM REQUIRED:'), 'reason must start with QUORUM REQUIRED:');
|
|
1150
|
-
} finally {
|
|
1151
|
-
fs.unlinkSync(tmpFile);
|
|
1152
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1153
|
-
}
|
|
1154
|
-
});
|
|
1155
|
-
|
|
1156
|
-
// ── TC-DEFAULT-CEIL: Default ceiling = 2 (no quorum.maxSize in config) ────────
|
|
1157
|
-
//
|
|
1158
|
-
// TC-DEFAULT-CEIL-PASS: 3-slot pool, no maxSize config → default = 2.
|
|
1159
|
-
// First 2 slots (sub-first) called successfully → ceiling satisfied → pass.
|
|
1160
|
-
// TC-DEFAULT-CEIL-BLOCK: Same pool, only 1 successful call → block.
|
|
1161
|
-
|
|
1162
|
-
test('TC-DEFAULT-CEIL-PASS: default ceiling=2 passes with 2 successful calls', () => {
|
|
1163
|
-
const slots = ['slot-sub-1', 'slot-sub-2', 'slot-api-1'];
|
|
1164
|
-
const agentConfig = {};
|
|
1165
|
-
for (const s of slots) agentConfig[s] = { auth_type: s.startsWith('slot-sub') ? 'sub' : 'api' };
|
|
1166
|
-
const configPayload = JSON.stringify({
|
|
1167
|
-
quorum_commands: ['quick'],
|
|
1168
|
-
quorum_active: slots,
|
|
1169
|
-
agent_config: agentConfig,
|
|
1170
|
-
// No quorum.maxSize — default kicks in (= 2)
|
|
1171
|
-
});
|
|
1172
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-dceil-pass-${Date.now()}`);
|
|
1173
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
1174
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
1175
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload);
|
|
1176
|
-
|
|
1177
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-dceil-pass-${Date.now()}.json`);
|
|
1178
|
-
const mcpServers = {};
|
|
1179
|
-
for (const s of slots) mcpServers[s] = {};
|
|
1180
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }));
|
|
1181
|
-
|
|
1182
|
-
// 2 successful calls (sub-1 and sub-2, sorted sub-first by default preferSub=true)
|
|
1183
|
-
const transcriptLines = [
|
|
1184
|
-
userLine('/qgsd:quick add something', 'human-dceil-pass'),
|
|
1185
|
-
assistantLine([bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files quick-115-PLAN.md')], 'assistant-commit'),
|
|
1186
|
-
assistantLine([toolUseBlock('mcp__slot-sub-1__review')], 'assistant-sub1'),
|
|
1187
|
-
toolResultSuccessLine('toolu_mcp__slot-sub-1__review', 'sub-1 OK'),
|
|
1188
|
-
assistantLine([toolUseBlock('mcp__slot-sub-2__review')], 'assistant-sub2'),
|
|
1189
|
-
toolResultSuccessLine('toolu_mcp__slot-sub-2__review', 'sub-2 OK'),
|
|
1190
|
-
assistantLine([{ type: 'text', text: 'Done.' }], 'assistant-final'),
|
|
1191
|
-
];
|
|
1192
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
1193
|
-
try {
|
|
1194
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
1195
|
-
{ stop_hook_active: false, hook_event_name: 'Stop', transcript_path: tmpFile, last_assistant_message: 'Done.' },
|
|
1196
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
1197
|
-
);
|
|
1198
|
-
assert.strictEqual(exitCode, 0);
|
|
1199
|
-
assert.strictEqual(stdout, '', 'default ceiling=2 satisfied by 2 calls — must not block');
|
|
1200
|
-
} finally {
|
|
1201
|
-
fs.unlinkSync(tmpFile);
|
|
1202
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1203
|
-
}
|
|
1204
|
-
});
|
|
1205
|
-
|
|
1206
|
-
test('TC-DEFAULT-CEIL-BLOCK: default ceiling=2 blocks with only 1 successful call', () => {
|
|
1207
|
-
const slots = ['slot-sub-1', 'slot-sub-2', 'slot-api-1'];
|
|
1208
|
-
const agentConfig = {};
|
|
1209
|
-
for (const s of slots) agentConfig[s] = { auth_type: s.startsWith('slot-sub') ? 'sub' : 'api' };
|
|
1210
|
-
const configPayload = JSON.stringify({
|
|
1211
|
-
quorum_commands: ['quick'],
|
|
1212
|
-
quorum_active: slots,
|
|
1213
|
-
agent_config: agentConfig,
|
|
1214
|
-
// No quorum.maxSize — default = 2
|
|
1215
|
-
});
|
|
1216
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-dceil-block-${Date.now()}`);
|
|
1217
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
1218
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
1219
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload);
|
|
1220
|
-
|
|
1221
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-dceil-block-${Date.now()}.json`);
|
|
1222
|
-
const mcpServers = {};
|
|
1223
|
-
for (const s of slots) mcpServers[s] = {};
|
|
1224
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }));
|
|
1225
|
-
|
|
1226
|
-
// Only 1 successful call (sub-1) — needs 2 → block
|
|
1227
|
-
const transcriptLines = [
|
|
1228
|
-
userLine('/qgsd:quick add something', 'human-dceil-block'),
|
|
1229
|
-
assistantLine([bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files quick-115-PLAN.md')], 'assistant-commit'),
|
|
1230
|
-
assistantLine([toolUseBlock('mcp__slot-sub-1__review')], 'assistant-sub1'),
|
|
1231
|
-
toolResultSuccessLine('toolu_mcp__slot-sub-1__review', 'sub-1 OK'),
|
|
1232
|
-
assistantLine([{ type: 'text', text: 'Done.' }], 'assistant-final'),
|
|
1233
|
-
];
|
|
1234
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
1235
|
-
try {
|
|
1236
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
1237
|
-
{ stop_hook_active: false, hook_event_name: 'Stop', transcript_path: tmpFile, last_assistant_message: 'Done.' },
|
|
1238
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
1239
|
-
);
|
|
1240
|
-
assert.strictEqual(exitCode, 0);
|
|
1241
|
-
assert.ok(stdout.length > 0, 'must block — only 1 of 2 required calls made');
|
|
1242
|
-
const parsed = JSON.parse(stdout);
|
|
1243
|
-
assert.strictEqual(parsed.decision, 'block');
|
|
1244
|
-
} finally {
|
|
1245
|
-
fs.unlinkSync(tmpFile);
|
|
1246
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1247
|
-
}
|
|
1248
|
-
});
|
|
1249
|
-
|
|
1250
|
-
// ── TC-SOLO-STOP: --n 1 triggers GUARD 6 solo bypass ─────────────────────────
|
|
1251
|
-
//
|
|
1252
|
-
// User prompt contains --n 1. Stop hook detects solo mode and exits 0
|
|
1253
|
-
// even with zero external slot calls and a decision turn present.
|
|
1254
|
-
|
|
1255
|
-
test('TC-SOLO-STOP: --n 1 solo mode bypasses quorum enforcement (GUARD 6)', () => {
|
|
1256
|
-
const slots = ['slot-sub-1', 'slot-sub-2'];
|
|
1257
|
-
const agentConfig = {};
|
|
1258
|
-
for (const s of slots) agentConfig[s] = { auth_type: 'sub' };
|
|
1259
|
-
const configPayload = JSON.stringify({
|
|
1260
|
-
quorum_commands: ['quick'],
|
|
1261
|
-
quorum_active: slots,
|
|
1262
|
-
agent_config: agentConfig,
|
|
1263
|
-
quorum: { maxSize: 2 },
|
|
1264
|
-
});
|
|
1265
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-solo-${Date.now()}`);
|
|
1266
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
1267
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
1268
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload);
|
|
1269
|
-
|
|
1270
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-solo-${Date.now()}.json`);
|
|
1271
|
-
const mcpServers = {};
|
|
1272
|
-
for (const s of slots) mcpServers[s] = {};
|
|
1273
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }));
|
|
1274
|
-
|
|
1275
|
-
// No external slot calls — but prompt has --n 1 → solo bypass
|
|
1276
|
-
const transcriptLines = [
|
|
1277
|
-
userLine('/qgsd:quick add something --n 1', 'human-solo'),
|
|
1278
|
-
assistantLine([bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files quick-115-PLAN.md')], 'assistant-commit'),
|
|
1279
|
-
assistantLine([{ type: 'text', text: 'Done solo.' }], 'assistant-final'),
|
|
1280
|
-
];
|
|
1281
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
1282
|
-
try {
|
|
1283
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
1284
|
-
{ stop_hook_active: false, hook_event_name: 'Stop', transcript_path: tmpFile, last_assistant_message: 'Done solo.' },
|
|
1285
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
1286
|
-
);
|
|
1287
|
-
assert.strictEqual(exitCode, 0);
|
|
1288
|
-
assert.strictEqual(stdout, '', '--n 1 solo mode must not block even with zero external calls');
|
|
1289
|
-
} finally {
|
|
1290
|
-
fs.unlinkSync(tmpFile);
|
|
1291
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1292
|
-
}
|
|
1293
|
-
});
|
|
1294
|
-
|
|
1295
|
-
// ── TC-N-OVERRIDE: --n N overrides config maxSize ────────────────────────────
|
|
1296
|
-
//
|
|
1297
|
-
// TC-N-OVERRIDE-PASS: config maxSize=5, --n 3 overrides to N-1=2 external required.
|
|
1298
|
-
// 2 successful calls → ceiling satisfied at 2 → pass.
|
|
1299
|
-
// TC-N-OVERRIDE-BLOCK: config maxSize=5, --n 3 overrides to 2 required.
|
|
1300
|
-
// 1 successful call → block (1 < 2).
|
|
1301
|
-
|
|
1302
|
-
test('TC-N-OVERRIDE-PASS: --n 3 overrides maxSize=5 config, 2 calls satisfy N-1=2', () => {
|
|
1303
|
-
const slots = ['slot-sub-1', 'slot-sub-2', 'slot-api-1', 'slot-api-2', 'slot-api-3'];
|
|
1304
|
-
const agentConfig = {};
|
|
1305
|
-
for (const s of slots) agentConfig[s] = { auth_type: s.startsWith('slot-sub') ? 'sub' : 'api' };
|
|
1306
|
-
const configPayload = JSON.stringify({
|
|
1307
|
-
quorum_commands: ['quick'],
|
|
1308
|
-
quorum_active: slots,
|
|
1309
|
-
agent_config: agentConfig,
|
|
1310
|
-
quorum: { maxSize: 5 },
|
|
1311
|
-
});
|
|
1312
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-nov-pass-${Date.now()}`);
|
|
1313
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
1314
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
1315
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload);
|
|
1316
|
-
|
|
1317
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-nov-pass-${Date.now()}.json`);
|
|
1318
|
-
const mcpServers = {};
|
|
1319
|
-
for (const s of slots) mcpServers[s] = {};
|
|
1320
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }));
|
|
1321
|
-
|
|
1322
|
-
// --n 3 → need N-1=2 external calls. Only 2 sub slots called (satisfies override).
|
|
1323
|
-
const transcriptLines = [
|
|
1324
|
-
userLine('/qgsd:quick add something --n 3', 'human-nov-pass'),
|
|
1325
|
-
assistantLine([bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files quick-115-PLAN.md')], 'assistant-commit'),
|
|
1326
|
-
assistantLine([toolUseBlock('mcp__slot-sub-1__review')], 'assistant-sub1'),
|
|
1327
|
-
toolResultSuccessLine('toolu_mcp__slot-sub-1__review', 'sub-1 OK'),
|
|
1328
|
-
assistantLine([toolUseBlock('mcp__slot-sub-2__review')], 'assistant-sub2'),
|
|
1329
|
-
toolResultSuccessLine('toolu_mcp__slot-sub-2__review', 'sub-2 OK'),
|
|
1330
|
-
assistantLine([{ type: 'text', text: 'Done n3.' }], 'assistant-final'),
|
|
1331
|
-
];
|
|
1332
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
1333
|
-
try {
|
|
1334
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
1335
|
-
{ stop_hook_active: false, hook_event_name: 'Stop', transcript_path: tmpFile, last_assistant_message: 'Done n3.' },
|
|
1336
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
1337
|
-
);
|
|
1338
|
-
assert.strictEqual(exitCode, 0);
|
|
1339
|
-
assert.strictEqual(stdout, '', '--n 3 override: 2 calls satisfy N-1=2 ceiling — must not block');
|
|
1340
|
-
} finally {
|
|
1341
|
-
fs.unlinkSync(tmpFile);
|
|
1342
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1343
|
-
}
|
|
1344
|
-
});
|
|
1345
|
-
|
|
1346
|
-
test('TC-N-OVERRIDE-BLOCK: --n 3 requires 2 calls; 1 call blocks despite config maxSize=5', () => {
|
|
1347
|
-
const slots = ['slot-sub-1', 'slot-sub-2', 'slot-api-1', 'slot-api-2', 'slot-api-3'];
|
|
1348
|
-
const agentConfig = {};
|
|
1349
|
-
for (const s of slots) agentConfig[s] = { auth_type: s.startsWith('slot-sub') ? 'sub' : 'api' };
|
|
1350
|
-
const configPayload = JSON.stringify({
|
|
1351
|
-
quorum_commands: ['quick'],
|
|
1352
|
-
quorum_active: slots,
|
|
1353
|
-
agent_config: agentConfig,
|
|
1354
|
-
quorum: { maxSize: 5 },
|
|
1355
|
-
});
|
|
1356
|
-
const homeDir = path.join(os.tmpdir(), `nf-home-nov-block-${Date.now()}`);
|
|
1357
|
-
const claudeDir = path.join(homeDir, '.claude');
|
|
1358
|
-
fs.mkdirSync(claudeDir, { recursive: true });
|
|
1359
|
-
fs.writeFileSync(path.join(claudeDir, 'nf.json'), configPayload);
|
|
1360
|
-
|
|
1361
|
-
const claudeJsonTmp = path.join(os.tmpdir(), `nf-claude-nov-block-${Date.now()}.json`);
|
|
1362
|
-
const mcpServers = {};
|
|
1363
|
-
for (const s of slots) mcpServers[s] = {};
|
|
1364
|
-
fs.writeFileSync(claudeJsonTmp, JSON.stringify({ mcpServers }));
|
|
1365
|
-
|
|
1366
|
-
// --n 3 → need N-1=2 external calls. Only 1 call made → block.
|
|
1367
|
-
const transcriptLines = [
|
|
1368
|
-
userLine('/qgsd:quick add something --n 3', 'human-nov-block'),
|
|
1369
|
-
assistantLine([bashCommitBlock('node /path/gsd-tools.cjs commit "docs: plan" --files quick-115-PLAN.md')], 'assistant-commit'),
|
|
1370
|
-
assistantLine([toolUseBlock('mcp__slot-sub-1__review')], 'assistant-sub1'),
|
|
1371
|
-
toolResultSuccessLine('toolu_mcp__slot-sub-1__review', 'sub-1 OK'),
|
|
1372
|
-
assistantLine([{ type: 'text', text: 'Done n3 one call.' }], 'assistant-final'),
|
|
1373
|
-
];
|
|
1374
|
-
const tmpFile = writeTempTranscript(transcriptLines);
|
|
1375
|
-
try {
|
|
1376
|
-
const { stdout, exitCode } = runHookWithEnv(
|
|
1377
|
-
{ stop_hook_active: false, hook_event_name: 'Stop', transcript_path: tmpFile, last_assistant_message: 'Done n3 one call.' },
|
|
1378
|
-
{ HOME: homeDir, NF_CLAUDE_JSON: claudeJsonTmp }
|
|
1379
|
-
);
|
|
1380
|
-
assert.strictEqual(exitCode, 0);
|
|
1381
|
-
assert.ok(stdout.length > 0, 'must block — only 1 of 2 required calls made (--n 3 override)');
|
|
1382
|
-
const parsed = JSON.parse(stdout);
|
|
1383
|
-
assert.strictEqual(parsed.decision, 'block');
|
|
1384
|
-
} finally {
|
|
1385
|
-
fs.unlinkSync(tmpFile);
|
|
1386
|
-
fs.unlinkSync(claudeJsonTmp);
|
|
1387
|
-
}
|
|
1388
|
-
});
|