cohorte 1.3.3 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +133 -0
- package/README.md +4 -7
- package/bin/cli.js +49 -4
- package/core/agents/implementer.template.md +10 -5
- package/core/agents/profile-reader.md +22 -0
- package/core/commands/doctor.md +8 -4
- package/core/hooks/gate.py +21 -6
- package/core/templates/agent-handoff.md +7 -2
- package/core/templates/review-feedback.md +7 -4
- package/core/templates/spec.template.md +5 -2
- package/core/templates/steps/init-pipeline/04-write-render.md +2 -1
- package/core/workflows/audit.js +88 -7
- package/core/workflows/refactor.js +85 -9
- package/core/workflows/review.js +132 -11
- package/dashboard/README.md +22 -5
- package/dashboard/dist/assets/index-AFQnlfjO.css +1 -0
- package/dashboard/dist/assets/{index-BxgA_mz1.js → index-DLBzciIC.js} +12 -11
- package/dashboard/dist/index.html +2 -2
- package/dashboard/server/doctor.js +68 -20
- package/dashboard/server/fleet.js +19 -5
- package/dashboard/server/index.js +79 -7
- package/dashboard/server/metrics.js +16 -4
- package/dashboard/server/versions.js +28 -6
- package/dashboard/server/yaml.js +4 -1
- package/install.ps1 +4 -0
- package/install.sh +19 -1
- package/package.json +5 -2
- package/profile/SCHEMA.md +23 -26
- package/scripts/kanban-move.sh +34 -20
- package/scripts/metrics/collect.mjs +495 -0
- package/scripts/metrics/prices.json +39 -0
- package/scripts/new-feature.sh.template +3 -1
- package/scripts/preflight.sh +16 -3
- package/scripts/remove-feature.sh.template +2 -1
- package/scripts/telemetry-send.sh +15 -1
- package/scripts/test-dashboard.mjs +362 -0
- package/scripts/test-gate.mjs +273 -0
- package/scripts/test-metrics.mjs +135 -0
- package/scripts/test-workflows.mjs +321 -0
- package/scripts/validate-core.mjs +51 -1
- package/core/commands/cycle.md +0 -54
- package/core/workflows/cycle.js +0 -407
- package/dashboard/dist/assets/index-Cj0SpgEY.css +0 -1
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// test-metrics.mjs — end-to-end checks for scripts/metrics/collect.mjs.
|
|
3
|
+
//
|
|
4
|
+
// Builds a throwaway repo plus a synthetic ~/.claude/projects transcript, runs the real
|
|
5
|
+
// collector against it via --json, and asserts the numbers. The cases are the ones that
|
|
6
|
+
// silently produce plausible-but-wrong output rather than crashing:
|
|
7
|
+
//
|
|
8
|
+
// 1. one API response written as several transcript lines, each repeating `usage`
|
|
9
|
+
// 2. a <task-notification> arriving mid-command (must not split the run)
|
|
10
|
+
// 3. subagent spend, which lives in a separate file tree
|
|
11
|
+
// 4. <synthetic> harness messages, which carry usage but cost nothing
|
|
12
|
+
// 5. the cache-tier pricing arithmetic itself
|
|
13
|
+
//
|
|
14
|
+
// Run: node scripts/test-metrics.mjs
|
|
15
|
+
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import os from 'node:os';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import { execFileSync, spawnSync } from 'node:child_process';
|
|
20
|
+
import { fileURLToPath } from 'node:url';
|
|
21
|
+
|
|
22
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
23
|
+
const COLLECT = path.join(HERE, 'metrics', 'collect.mjs');
|
|
24
|
+
|
|
25
|
+
let failures = 0;
|
|
26
|
+
const ok = (label) => console.log(` ✓ ${label}`);
|
|
27
|
+
function check(label, actual, expected) {
|
|
28
|
+
const a = JSON.stringify(actual), e = JSON.stringify(expected);
|
|
29
|
+
if (a === e) return ok(label);
|
|
30
|
+
failures += 1;
|
|
31
|
+
console.log(` ✗ ${label}\n expected ${e}\n actual ${a}`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'cohorte-metrics-'));
|
|
35
|
+
const repo = path.join(tmp, 'repo');
|
|
36
|
+
const cfg = path.join(tmp, 'claude');
|
|
37
|
+
const SESSION = 'sess-test-0001';
|
|
38
|
+
const projectDir = path.join(cfg, 'projects', 'test-slug');
|
|
39
|
+
fs.mkdirSync(path.join(projectDir, SESSION, 'subagents'), { recursive: true });
|
|
40
|
+
fs.mkdirSync(repo, { recursive: true });
|
|
41
|
+
execFileSync('git', ['init', '-q'], { cwd: repo, stdio: 'ignore' });
|
|
42
|
+
|
|
43
|
+
const T0 = Date.parse('2026-07-30T10:00:00.000Z');
|
|
44
|
+
const at = (s) => new Date(T0 + s * 1000).toISOString();
|
|
45
|
+
|
|
46
|
+
const assistant = (id, tsS, model, usage, content = [{ type: 'text', text: 'x' }]) => ({
|
|
47
|
+
type: 'assistant', timestamp: at(tsS), cwd: repo, sessionId: SESSION,
|
|
48
|
+
message: { id, model, content, usage },
|
|
49
|
+
});
|
|
50
|
+
const user = (tsS, text) => ({
|
|
51
|
+
type: 'user', timestamp: at(tsS), cwd: repo, sessionId: SESSION,
|
|
52
|
+
message: { role: 'user', content: text },
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
const usageOpus = {
|
|
56
|
+
input_tokens: 100, output_tokens: 1000,
|
|
57
|
+
cache_creation_input_tokens: 1000, cache_read_input_tokens: 10000,
|
|
58
|
+
cache_creation: { ephemeral_5m_input_tokens: 1000, ephemeral_1h_input_tokens: 0 },
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const lines = [
|
|
62
|
+
user(0, '<command-message>build</command-message>\n<command-name>/build</command-name>'),
|
|
63
|
+
// Case 1: one response, three lines, identical usage on each. Only one should be billed.
|
|
64
|
+
assistant('m1', 5, 'claude-opus-5', usageOpus, [{ type: 'thinking', thinking: '...' }]),
|
|
65
|
+
assistant('m1', 5, 'claude-opus-5', usageOpus, [{ type: 'text', text: 'hello' }]),
|
|
66
|
+
assistant('m1', 6, 'claude-opus-5', usageOpus, [{ type: 'tool_use', id: 'toolu_A', name: 'Task', input: {} }]),
|
|
67
|
+
// Case 2: a background agent finished mid-run. This is not the human starting anything.
|
|
68
|
+
user(10, '<task-notification>\n<task-id>a1</task-id>\n</task-notification>'),
|
|
69
|
+
// Case 4: harness-authored message, has usage, costs nothing.
|
|
70
|
+
assistant('m2', 12, '<synthetic>', { input_tokens: 0, output_tokens: 999999 }),
|
|
71
|
+
assistant('m3', 20, 'claude-opus-5', { input_tokens: 0, output_tokens: 500 }),
|
|
72
|
+
// A second, genuinely separate run — long enough not to read as a steering turn.
|
|
73
|
+
user(600, 'unrelated question about the repository layout and its conventions'),
|
|
74
|
+
assistant('m4', 605, 'claude-opus-5', { input_tokens: 0, output_tokens: 40 }),
|
|
75
|
+
// Case 6: a command named inside ordinary prose. The harness emits no <command-name>
|
|
76
|
+
// for this, but it is the way commands actually get invoked in practice.
|
|
77
|
+
user(1200, 'move on branding-ramp and /review'),
|
|
78
|
+
assistant('m5', 1205, 'claude-opus-5', { input_tokens: 0, output_tokens: 60 }),
|
|
79
|
+
// Case 7: a short steer continues the /review rather than opening an anonymous run.
|
|
80
|
+
user(1260, 'continue'),
|
|
81
|
+
assistant('m6', 1265, 'claude-opus-5', { input_tokens: 0, output_tokens: 70 }),
|
|
82
|
+
// Case 8: a slash token that is not a command must not invent one.
|
|
83
|
+
user(1800, 'look at the /usr/local/share directory and report what you find there'),
|
|
84
|
+
assistant('m7', 1805, 'claude-opus-5', { input_tokens: 0, output_tokens: 10 }),
|
|
85
|
+
];
|
|
86
|
+
fs.writeFileSync(path.join(projectDir, `${SESSION}.jsonl`),
|
|
87
|
+
lines.map((l) => JSON.stringify(l)).join('\n') + '\n');
|
|
88
|
+
|
|
89
|
+
// Case 3: subagent spend, linked back to /build by the Task tool_use id.
|
|
90
|
+
const agentDir = path.join(projectDir, SESSION, 'subagents');
|
|
91
|
+
fs.writeFileSync(path.join(agentDir, 'agent-a1.meta.json'),
|
|
92
|
+
JSON.stringify({ agentType: 'core', description: 'Build core surface', toolUseId: 'toolu_A', spawnDepth: 1 }));
|
|
93
|
+
fs.writeFileSync(path.join(agentDir, 'agent-a1.jsonl'),
|
|
94
|
+
JSON.stringify(assistant('s1', 8, 'claude-sonnet-5', { input_tokens: 0, output_tokens: 2000 })) + '\n');
|
|
95
|
+
|
|
96
|
+
const run = spawnSync(process.execPath, [COLLECT, repo, '--json', '--runs'], {
|
|
97
|
+
encoding: 'utf8',
|
|
98
|
+
env: { ...process.env, CLAUDE_CONFIG_DIR: cfg },
|
|
99
|
+
});
|
|
100
|
+
if (run.status !== 0) {
|
|
101
|
+
console.error('collector failed:\n' + (run.stderr || run.stdout));
|
|
102
|
+
process.exit(1);
|
|
103
|
+
}
|
|
104
|
+
const out = JSON.parse(run.stdout);
|
|
105
|
+
const build = out.commands.find((c) => c.command === '/build');
|
|
106
|
+
const chat = out.commands.find((c) => c.command === '(chat)');
|
|
107
|
+
const review = out.commands.find((c) => c.command === '/review');
|
|
108
|
+
|
|
109
|
+
console.log('test-metrics');
|
|
110
|
+
check('the mid-command task-notification did not split the run', out.totals.runs, 4);
|
|
111
|
+
check('/build is one run, not three', build.runs, 1);
|
|
112
|
+
check('duplicate lines of one response are billed once', build.tokens.output, 1000 + 500 + 2000);
|
|
113
|
+
check('the <synthetic> message contributed no tokens', build.tokens.output < 999999, true);
|
|
114
|
+
check('cache-write tokens are kept on their own tier', build.tokens.cacheWrite5m, 1000);
|
|
115
|
+
check('cache-read tokens are kept on their own tier', build.tokens.cacheRead, 10000);
|
|
116
|
+
check('the subagent was attributed to the command that spawned it', build.agents.total, 1);
|
|
117
|
+
check('the second prompt is a separate (chat) run', chat.runs, 2);
|
|
118
|
+
check('a command named inside prose is attributed to that command', review && review.runs, 1);
|
|
119
|
+
check('a short steer continues the run instead of opening a new one', review.continuations, 1);
|
|
120
|
+
check('the continued turn counts toward the command it continued', review.tokens.output, 60 + 70);
|
|
121
|
+
check('a non-command slash token does not invent a command', chat.tokens.output, 40 + 10);
|
|
122
|
+
|
|
123
|
+
// opus-5 $5 in / $25 out per MTok; 5m cache write 1.25x input, cache read 0.1x input.
|
|
124
|
+
// m1 100*5 + 1000*25 + 1000*6.25 + 10000*0.5 = 36750
|
|
125
|
+
// m3 500*25 = 12500
|
|
126
|
+
// s1 sonnet-5 2000*15 = 30000 (subagent)
|
|
127
|
+
check('cost sums the cache tiers at their own rates', Number(build.cost.total.toFixed(6)), 0.07925);
|
|
128
|
+
check('the unpriced list stays empty for known models', build.unpriced, []);
|
|
129
|
+
|
|
130
|
+
const detail = out.runs.find((r) => r.command === '/build');
|
|
131
|
+
check('per-run detail carries the subagent', detail.agents.map((a) => a.type), ['core']);
|
|
132
|
+
|
|
133
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
134
|
+
console.log(failures ? `\ntest-metrics: ${failures} FAILED` : '\ntest-metrics: OK');
|
|
135
|
+
process.exit(failures ? 1 : 0);
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Behavioural tests for core/workflows/*.js.
|
|
3
|
+
//
|
|
4
|
+
// The workflow runtime hands a script an async function body with agent() /
|
|
5
|
+
// parallel() / pipeline() / phase() / log() / args / budget injected. Nothing in
|
|
6
|
+
// a script touches the filesystem, so the whole orchestration is testable by
|
|
7
|
+
// injecting stub agents and asserting the returned verdict object.
|
|
8
|
+
//
|
|
9
|
+
// This exists because of one specific failure mode: agent() resolves to `null`
|
|
10
|
+
// when a subagent dies, and a dead reviewer produces zero findings — which is
|
|
11
|
+
// byte-identical to a clean surface. review.js scored that as SHIP over code no
|
|
12
|
+
// reviewer had read. A unit test is the only thing that catches it: the
|
|
13
|
+
// structural checks in validate-core.mjs cannot see verdict logic.
|
|
14
|
+
//
|
|
15
|
+
// node scripts/test-workflows.mjs
|
|
16
|
+
|
|
17
|
+
import { readFileSync } from "node:fs";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { fileURLToPath } from "node:url";
|
|
20
|
+
|
|
21
|
+
const root = fileURLToPath(new URL("..", import.meta.url));
|
|
22
|
+
const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
|
|
23
|
+
|
|
24
|
+
let failures = 0;
|
|
25
|
+
const check = (name, cond, detail = "") => {
|
|
26
|
+
if (cond) console.log(` ✓ ${name}`);
|
|
27
|
+
else { failures++; console.error(` ✗ ${name}${detail ? ` — ${detail}` : ""}`); }
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const PROFILE = {
|
|
31
|
+
name: "testproj",
|
|
32
|
+
vcs: { default_branch: "main" },
|
|
33
|
+
contract: { enabled: false, path: "packages/shared/src", ext: "ts", mechanism: "none" },
|
|
34
|
+
commands: { typecheck: "tsc --noEmit", lint_quiet: "lint -q", test_quiet: "test --dot" },
|
|
35
|
+
surfaces: [
|
|
36
|
+
{ key: "backend", path: "apps/api", agent: "backend", uses_design: false },
|
|
37
|
+
{ key: "frontend", path: "apps/web", agent: "frontend", uses_design: true },
|
|
38
|
+
],
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
const TOUCHED = [
|
|
42
|
+
{ key: "backend", diff: "specs/reports/f.backend.diff", files: ["apps/api/a.ts"] },
|
|
43
|
+
{ key: "frontend", diff: "specs/reports/f.frontend.diff", files: ["apps/web/b.tsx"] },
|
|
44
|
+
];
|
|
45
|
+
|
|
46
|
+
const finding = (over = {}) => ({
|
|
47
|
+
severity: "HIGH", file: "apps/api/a.ts", line: 3, kind: "quality",
|
|
48
|
+
problem: "p", fix: "f", ...over,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
// Run one workflow script with a `reply(prompt, opts) => value` stub in place of
|
|
52
|
+
// every agent call. Returns { result, calls }.
|
|
53
|
+
async function run(script, reply, args = { feature: "feat-x" }) {
|
|
54
|
+
const text = readFileSync(join(root, "core/workflows", script), "utf8")
|
|
55
|
+
.replace(/^export const meta/m, "const meta");
|
|
56
|
+
const calls = [];
|
|
57
|
+
const agent = async (prompt, opts = {}) => {
|
|
58
|
+
calls.push(opts.label || "(unlabelled)");
|
|
59
|
+
return reply(prompt, opts, calls);
|
|
60
|
+
};
|
|
61
|
+
// Mirrors the runtime's contract: a thunk that throws resolves to null, the
|
|
62
|
+
// call itself never rejects.
|
|
63
|
+
const parallel = thunks =>
|
|
64
|
+
Promise.all(thunks.map(t => Promise.resolve().then(t).catch(() => null)));
|
|
65
|
+
// Each item runs through every stage independently; a throwing stage drops
|
|
66
|
+
// that item to null and skips its remaining stages.
|
|
67
|
+
const pipeline = (items, ...stages) =>
|
|
68
|
+
Promise.all(items.map(async (item, i) => {
|
|
69
|
+
let v = item;
|
|
70
|
+
for (const s of stages) {
|
|
71
|
+
try { v = await s(v, item, i); } catch { return null; }
|
|
72
|
+
}
|
|
73
|
+
return v;
|
|
74
|
+
}));
|
|
75
|
+
const fn = new AsyncFunction(
|
|
76
|
+
"agent", "parallel", "pipeline", "phase", "log", "args", "budget", "workflow", text);
|
|
77
|
+
const result = await fn(
|
|
78
|
+
agent, parallel, pipeline, () => {}, () => {}, args,
|
|
79
|
+
{ total: null, spent: () => 0, remaining: () => Infinity }, async () => {});
|
|
80
|
+
return { result, calls };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// A reply table keyed by label prefix; the first matching prefix wins.
|
|
84
|
+
const replier = table => (prompt, opts) => {
|
|
85
|
+
const label = opts.label || "";
|
|
86
|
+
for (const [prefix, value] of table) {
|
|
87
|
+
if (label === prefix || label.startsWith(prefix)) {
|
|
88
|
+
return typeof value === "function" ? value(label) : value;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return "ok";
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
const BASE_REVIEW = [
|
|
95
|
+
["profile", PROFILE],
|
|
96
|
+
["preflight", { pass: true }],
|
|
97
|
+
["stage-diff", { surfaces: TOUCHED }],
|
|
98
|
+
["stage-report", "done"],
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
console.log("review.js");
|
|
102
|
+
{
|
|
103
|
+
const { result } = await run("review.js", replier([
|
|
104
|
+
["review:", { verdict: "SHIP", findings: [] }], ...BASE_REVIEW,
|
|
105
|
+
]));
|
|
106
|
+
check("clean run ⇒ SHIP", result.verdict === "SHIP", `got ${result.verdict}`);
|
|
107
|
+
check("clean run ⇒ no unreviewed surfaces", (result.unreviewedSurfaces || []).length === 0);
|
|
108
|
+
check("clean run ⇒ next is /ship", String(result.next).startsWith("/ship"), result.next);
|
|
109
|
+
}
|
|
110
|
+
{
|
|
111
|
+
// THE regression: every reviewer dies ⇒ zero findings ⇒ must NOT read as SHIP.
|
|
112
|
+
const { result } = await run("review.js", replier([
|
|
113
|
+
["review:", null], ...BASE_REVIEW,
|
|
114
|
+
]));
|
|
115
|
+
check("all reviewers dead ⇒ not SHIP", result.verdict !== "SHIP", `got ${result.verdict}`);
|
|
116
|
+
check("all reviewers dead ⇒ both surfaces reported unreviewed",
|
|
117
|
+
(result.unreviewedSurfaces || []).join(",") === "backend,frontend",
|
|
118
|
+
JSON.stringify(result.unreviewedSurfaces));
|
|
119
|
+
check("all reviewers dead ⇒ next says re-run",
|
|
120
|
+
/re-run the review/.test(result.next), result.next);
|
|
121
|
+
}
|
|
122
|
+
{
|
|
123
|
+
// One dead reviewer must not be masked by the other surface coming back clean.
|
|
124
|
+
const { result } = await run("review.js", replier([
|
|
125
|
+
["review:backend", null],
|
|
126
|
+
["review:", { verdict: "SHIP", findings: [] }],
|
|
127
|
+
...BASE_REVIEW,
|
|
128
|
+
]));
|
|
129
|
+
check("one reviewer dead ⇒ not SHIP", result.verdict !== "SHIP", `got ${result.verdict}`);
|
|
130
|
+
check("one reviewer dead ⇒ names only that surface",
|
|
131
|
+
(result.unreviewedSurfaces || []).join(",") === "backend", JSON.stringify(result.unreviewedSurfaces));
|
|
132
|
+
}
|
|
133
|
+
{
|
|
134
|
+
// A SHIP carrying HIGH findings is a real verdict, but it is not "go ship it":
|
|
135
|
+
// the conversational /review routes any surviving HIGH to /fix.
|
|
136
|
+
const { result } = await run("review.js", replier([
|
|
137
|
+
["review:", { verdict: "SHIP", findings: [finding()] }], ...BASE_REVIEW,
|
|
138
|
+
]));
|
|
139
|
+
check("SHIP + HIGH findings ⇒ verdict still SHIP", result.verdict === "SHIP");
|
|
140
|
+
check("SHIP + HIGH findings ⇒ next routes to /fix, not /ship",
|
|
141
|
+
String(result.next).startsWith("/fix"), result.next);
|
|
142
|
+
}
|
|
143
|
+
{
|
|
144
|
+
const { result } = await run("review.js", replier([
|
|
145
|
+
["review:", { verdict: "SHIP", findings: [finding({ severity: "LOW" })] }], ...BASE_REVIEW,
|
|
146
|
+
]));
|
|
147
|
+
check("SHIP + only LOW ⇒ next is /ship", String(result.next).startsWith("/ship"), result.next);
|
|
148
|
+
}
|
|
149
|
+
{
|
|
150
|
+
const { result } = await run("review.js", replier([
|
|
151
|
+
["preflight", { pass: false, tail: "boom" }], ...BASE_REVIEW,
|
|
152
|
+
]));
|
|
153
|
+
check("red preflight ⇒ ABORTED", result.verdict === "ABORTED", `got ${result.verdict}`);
|
|
154
|
+
}
|
|
155
|
+
{
|
|
156
|
+
const { calls } = await run("review.js", replier([
|
|
157
|
+
["preflight", { pass: false, tail: "boom" }], ...BASE_REVIEW,
|
|
158
|
+
]));
|
|
159
|
+
check("red preflight ⇒ zero reviewers spawned",
|
|
160
|
+
!calls.some(c => c.startsWith("review:")), calls.join(","));
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// ── args normalisation ───────────────────────────────────────────────────────
|
|
164
|
+
// The runtime passes `args` through verbatim, so a caller that JSON-encodes it
|
|
165
|
+
// hands the script a string. That string used to become the feature id itself —
|
|
166
|
+
// which is how a report was written to `specs/reports/{"feature": "x"}.md`.
|
|
167
|
+
console.log("args");
|
|
168
|
+
{
|
|
169
|
+
const { result } = await run("review.js", replier([
|
|
170
|
+
["review:", { verdict: "SHIP", findings: [] }], ...BASE_REVIEW,
|
|
171
|
+
]), JSON.stringify({ feature: "feat-x" }));
|
|
172
|
+
check("review: a JSON-encoded args string is parsed, not used as the id",
|
|
173
|
+
result.verdict === "SHIP", `got ${result.verdict}`);
|
|
174
|
+
}
|
|
175
|
+
{
|
|
176
|
+
let threw = "";
|
|
177
|
+
try {
|
|
178
|
+
await run("review.js", replier([...BASE_REVIEW]), { feature: '{"feature": "feat-x"}' });
|
|
179
|
+
} catch (e) { threw = e.message; }
|
|
180
|
+
check("review: a non-slug feature id throws before anything is written",
|
|
181
|
+
/not a slug/.test(threw), threw || "(did not throw)");
|
|
182
|
+
}
|
|
183
|
+
{
|
|
184
|
+
let threw = "";
|
|
185
|
+
try {
|
|
186
|
+
await run("review.js", replier([...BASE_REVIEW]), { feature: "../../etc/passwd" });
|
|
187
|
+
} catch (e) { threw = e.message; }
|
|
188
|
+
check("review: a path-shaped feature id is rejected",
|
|
189
|
+
/not a slug/.test(threw), threw || "(did not throw)");
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// ── Phase 0 profile handling ─────────────────────────────────────────────────
|
|
193
|
+
// A haiku profile-reader intermittently returns the profile as a JSON *string*
|
|
194
|
+
// under a wrapper field instead of at the top level. The old schema accepted that
|
|
195
|
+
// wrapper, so `surfaces` read as undefined ⇒ [] ⇒ parallel([]) ⇒ zero agents
|
|
196
|
+
// dispatched — and because every later guard compares against `surfaces`, an
|
|
197
|
+
// empty list made them all vacuously pass: a run reported a verdict having done
|
|
198
|
+
// nothing, indistinguishable from a clean run with an empty diff. Two properties
|
|
199
|
+
// are pinned per workflow: a wrapped return is recovered, an empty one aborts.
|
|
200
|
+
console.log("profile phase");
|
|
201
|
+
const WRAPPED = { output: JSON.stringify(PROFILE) };
|
|
202
|
+
const EMPTY_PROFILE = { ...PROFILE, surfaces: [] };
|
|
203
|
+
{
|
|
204
|
+
const { result } = await run("review.js", replier([
|
|
205
|
+
["profile", WRAPPED],
|
|
206
|
+
["review:", { verdict: "SHIP", findings: [] }], ...BASE_REVIEW,
|
|
207
|
+
]));
|
|
208
|
+
check("review: a string-wrapped profile is unwrapped, not silently empty",
|
|
209
|
+
result.verdict === "SHIP", `got ${result.verdict}`);
|
|
210
|
+
}
|
|
211
|
+
{
|
|
212
|
+
const { result, calls } = await run("review.js", replier([["profile", EMPTY_PROFILE], ...BASE_REVIEW]));
|
|
213
|
+
check("review: no surfaces ⇒ ABORTED, not a verdict",
|
|
214
|
+
result.verdict === "ABORTED", `got ${result.verdict}`);
|
|
215
|
+
check("review: no surfaces ⇒ zero reviewers spawned",
|
|
216
|
+
!calls.some(c => c.startsWith("review:")), calls.join(","));
|
|
217
|
+
}
|
|
218
|
+
{
|
|
219
|
+
const { result } = await run("audit.js", replier([
|
|
220
|
+
["profile", EMPTY_PROFILE], ["gates", { failures: [] }], ["write-backlog", "done"],
|
|
221
|
+
]), {});
|
|
222
|
+
check("audit: no surfaces ⇒ error, not an empty backlog",
|
|
223
|
+
/no surfaces/.test(result.error || ""), JSON.stringify(result));
|
|
224
|
+
}
|
|
225
|
+
{
|
|
226
|
+
const { result } = await run("refactor.js", replier([
|
|
227
|
+
["profile", EMPTY_PROFILE], ["read-backlog", { domains: [] }],
|
|
228
|
+
]), { domains: "all" });
|
|
229
|
+
check("refactor: no surfaces ⇒ error, not a no-op success",
|
|
230
|
+
/no surfaces/.test(result.error || ""), JSON.stringify(result));
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// ── the dead-agent family, swept across every terminal/staging agent ─────────
|
|
234
|
+
// `agent()` returns null when a subagent dies. Any call whose result is turned
|
|
235
|
+
// into a CLAIM (a verdict, a path, "it is on disk") must distinguish "died" from
|
|
236
|
+
// "succeeded with nothing to say". This block is the sweep.
|
|
237
|
+
console.log("dead-agent sweep");
|
|
238
|
+
{
|
|
239
|
+
const { result } = await run("review.js", replier([
|
|
240
|
+
["stage-diff", null], ...BASE_REVIEW,
|
|
241
|
+
]));
|
|
242
|
+
check("review: dead diff-stager ⇒ ABORTED, not 'SHIP — nothing to review'",
|
|
243
|
+
result.verdict === "ABORTED", `got ${result.verdict}: ${result.reason}`);
|
|
244
|
+
}
|
|
245
|
+
{
|
|
246
|
+
const { result } = await run("review.js", replier([
|
|
247
|
+
["stage-report", null],
|
|
248
|
+
["review:", { verdict: "SHIP", findings: [] }], ...BASE_REVIEW,
|
|
249
|
+
]));
|
|
250
|
+
check("review: dead report-stager ⇒ reportStaged false", result.reportStaged === false);
|
|
251
|
+
check("review: dead report-stager ⇒ report path not claimed",
|
|
252
|
+
!/^specs\//.test(String(result.report)), result.report);
|
|
253
|
+
check("review: dead report-stager ⇒ next says nothing was written",
|
|
254
|
+
/NEVER written/.test(result.next), result.next);
|
|
255
|
+
}
|
|
256
|
+
{
|
|
257
|
+
const { result } = await run("audit.js", replier([
|
|
258
|
+
["profile", PROFILE], ["gates", { failures: [] }],
|
|
259
|
+
["audit:backend", null],
|
|
260
|
+
["audit:", { items: [] }], ["write-backlog", "done"],
|
|
261
|
+
]), {});
|
|
262
|
+
check("audit: dead auditor ⇒ the domain is listed as NOT audited",
|
|
263
|
+
(result.notAudited || []).join(",") === "backend", JSON.stringify(result.notAudited));
|
|
264
|
+
check("audit: dead auditor ⇒ next tells you to re-audit it",
|
|
265
|
+
/re-audit backend/.test(result.next), result.next);
|
|
266
|
+
}
|
|
267
|
+
{
|
|
268
|
+
const { result } = await run("audit.js", replier([
|
|
269
|
+
["profile", PROFILE], ["gates", { failures: [] }],
|
|
270
|
+
["audit:", { items: [] }], ["write-backlog", null],
|
|
271
|
+
]), {});
|
|
272
|
+
check("audit: dead backlog writer ⇒ path not claimed",
|
|
273
|
+
!/^specs\//.test(String(result.backlog)), result.backlog);
|
|
274
|
+
}
|
|
275
|
+
{
|
|
276
|
+
const { result } = await run("refactor.js", replier([
|
|
277
|
+
["profile", PROFILE], ["read-backlog", null],
|
|
278
|
+
]), { domains: "all" });
|
|
279
|
+
check("refactor: dead backlog reader ⇒ says it died, not 'no open items'",
|
|
280
|
+
/agent died/.test(String(result.error)), result.error);
|
|
281
|
+
}
|
|
282
|
+
{
|
|
283
|
+
const items = ["- [ ] a", "- [ ] b", "- [ ] c", "- [ ] d", "- [ ] e"];
|
|
284
|
+
const { result } = await run("refactor.js", replier([
|
|
285
|
+
["profile", PROFILE],
|
|
286
|
+
["read-backlog", { domains: [{ key: "backend", items }] }],
|
|
287
|
+
["verify:", { cleared: items, remaining: [], gatesGreen: true }],
|
|
288
|
+
["reverify:", { cleared: items, remaining: [], gatesGreen: true }],
|
|
289
|
+
["tick-backlog", null],
|
|
290
|
+
["refactor:", "handoff"],
|
|
291
|
+
]), { domains: "all" });
|
|
292
|
+
check("refactor: dead ticker ⇒ backlogTicked false", result.backlogTicked === false);
|
|
293
|
+
check("refactor: dead ticker ⇒ next warns the backlog still shows them open",
|
|
294
|
+
/NOT ticked/.test(result.next), result.next);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// ── audit.js / refactor.js — smoke-level: they must return, not throw ────────
|
|
298
|
+
console.log("audit.js / refactor.js");
|
|
299
|
+
{
|
|
300
|
+
const { result } = await run("audit.js", replier([
|
|
301
|
+
["profile", PROFILE],
|
|
302
|
+
["gates", { failures: [] }],
|
|
303
|
+
["audit:", { items: [{ severity: "HIGH", file: "apps/api/a.ts", line: 1, kind: "tdd", fix: "add a test" }] }],
|
|
304
|
+
["write-backlog", "done"],
|
|
305
|
+
]), {});
|
|
306
|
+
check("audit returns a backlog path", result.backlog === "specs/refactor-backlog.md", JSON.stringify(result));
|
|
307
|
+
check("audit counts every domain (surfaces + shared)",
|
|
308
|
+
Object.keys(result.domains || {}).join(",") === "backend,frontend,shared", JSON.stringify(result.domains));
|
|
309
|
+
}
|
|
310
|
+
{
|
|
311
|
+
const { result } = await run("refactor.js", replier([
|
|
312
|
+
["profile", PROFILE],
|
|
313
|
+
["read-backlog", { domains: [{ key: "backend", items: ["- [ ] a", "- [ ] b"] }] }],
|
|
314
|
+
]), { domains: "all" });
|
|
315
|
+
check("refactor skips a domain below the item threshold",
|
|
316
|
+
result.skipped && result.skipped.backend === 2, JSON.stringify(result));
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
console.log("");
|
|
320
|
+
if (failures) { console.error(`test-workflows: ${failures} failure(s)`); process.exit(1); }
|
|
321
|
+
console.log("test-workflows: OK");
|
|
@@ -23,7 +23,7 @@ const frontmatter = (text) => {
|
|
|
23
23
|
// orchestration turn silently bills at the session model — Opus/Fable).
|
|
24
24
|
// Interactive commands must stay unpinned (they inherit on purpose).
|
|
25
25
|
const PINNED = ["build", "review", "fix", "smoke", "ship", "audit",
|
|
26
|
-
"refactor", "doctor", "align-ds", "update-pipeline"
|
|
26
|
+
"refactor", "doctor", "align-ds", "update-pipeline"];
|
|
27
27
|
const UNPINNED = ["brainstorm", "spec", "init-pipeline"];
|
|
28
28
|
|
|
29
29
|
for (const f of readdirSync(join(root, "core/commands"))) {
|
|
@@ -168,6 +168,11 @@ else for (const f of readdirSync(workflowsDir)) {
|
|
|
168
168
|
fail(path, "phase 0 must read the profile via the profile-reader agent");
|
|
169
169
|
if (/\bDate\.now\(\)|\bMath\.random\(\)|new Date\(\)/.test(text))
|
|
170
170
|
fail(path, "Date.now()/Math.random()/new Date() are unavailable in workflow scripts");
|
|
171
|
+
// Prompts hand agents literal `<core>/…` paths; an agent can only resolve that
|
|
172
|
+
// token if the same script also spells out what <core> means. A bare token +
|
|
173
|
+
// `|| true` = the command fails silently and the ping/metrics never happen.
|
|
174
|
+
if (text.includes("<core>/") && !/<core> = /.test(text))
|
|
175
|
+
fail(path, "uses <core>/ paths in prompts without defining `<core> = …` anywhere");
|
|
171
176
|
try {
|
|
172
177
|
new AsyncFunction("agent", "parallel", "pipeline", "phase", "log", "args",
|
|
173
178
|
"budget", "workflow", text.replace(/^export const meta/m, "const meta"));
|
|
@@ -182,6 +187,51 @@ if (!installSh.includes("core/workflows"))
|
|
|
182
187
|
if (!installPs1.includes("core\\workflows"))
|
|
183
188
|
fail("install.ps1", "does not copy core\\workflows (Copy-Core)");
|
|
184
189
|
|
|
190
|
+
// A new workflow script must also be KNOWN to the things that check for it, or it
|
|
191
|
+
// ships and nothing notices when an installer stops copying it. This check exists
|
|
192
|
+
// because a workflow once shipped while three call sites still named only the
|
|
193
|
+
// three that preceded it.
|
|
194
|
+
const workflowNames = existsSync(workflowsDir)
|
|
195
|
+
? readdirSync(workflowsDir).filter((f) => f.endsWith(".js"))
|
|
196
|
+
: [];
|
|
197
|
+
const ci = existsSync(join(root, ".github/workflows/ci.yml")) ? read(".github/workflows/ci.yml") : "";
|
|
198
|
+
const dashDoctor = read("dashboard/server/doctor.js");
|
|
199
|
+
for (const f of workflowNames) {
|
|
200
|
+
if (!ci.includes(`workflows/${f}`))
|
|
201
|
+
fail(".github/workflows/ci.yml", `install dry-run never asserts .claude/workflows/${f}`);
|
|
202
|
+
if (!dashDoctor.includes(`'${f}'`))
|
|
203
|
+
fail("dashboard/server/doctor.js", `checkWorkflows() does not list ${f}`);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// ── dashboard: the metrics phase list is duplicated server/client ────────────
|
|
207
|
+
// A phase present in one and not the other parses fine and renders in no column —
|
|
208
|
+
// silently invisible data, which is how a phase batch once went unnoticed.
|
|
209
|
+
const phaseList = (text, file) => {
|
|
210
|
+
const m = text.match(/const PHASES = \[([^\]]*)\]/);
|
|
211
|
+
if (!m) { fail(file, "no `const PHASES = [...]` found"); return null; }
|
|
212
|
+
return m[1].split(",").map((s) => s.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
|
|
213
|
+
};
|
|
214
|
+
const serverPhases = phaseList(read("dashboard/server/metrics.js"), "dashboard/server/metrics.js");
|
|
215
|
+
const clientPhases = phaseList(read("dashboard/app/src/components/MetricsPanel.jsx"),
|
|
216
|
+
"dashboard/app/src/components/MetricsPanel.jsx");
|
|
217
|
+
if (serverPhases && clientPhases && serverPhases.join("|") !== clientPhases.join("|"))
|
|
218
|
+
fail("dashboard/app/src/components/MetricsPanel.jsx",
|
|
219
|
+
`PHASES drifted from dashboard/server/metrics.js ([${clientPhases}] vs [${serverPhases}])`);
|
|
220
|
+
|
|
221
|
+
// ── packaging: no build artifacts in the published tarball ──────────────────
|
|
222
|
+
// `.npmignore` is INERT under an explicit package.json `files` allowlist, so its
|
|
223
|
+
// `__pycache__/` rule never fired — a maintainer who had compiled gate.py shipped
|
|
224
|
+
// their machine's bytecode cache. The negations in `files` are what actually work.
|
|
225
|
+
const pkg = JSON.parse(read("package.json"));
|
|
226
|
+
for (const negation of ["!core/hooks/__pycache__", "!**/*.pyc"])
|
|
227
|
+
if (!(pkg.files || []).includes(negation))
|
|
228
|
+
fail("package.json", `\`files\` must carry the ${negation} negation (.npmignore cannot do this)`);
|
|
229
|
+
for (const [name, src] of Object.entries(installers))
|
|
230
|
+
if (!src.includes("__pycache__"))
|
|
231
|
+
fail(name, "never scrubs hooks/__pycache__ — cp -R would carry it into the user's .claude");
|
|
232
|
+
if (!read("bin/cli.js").includes("__pycache__"))
|
|
233
|
+
fail("bin/cli.js", "copyCore() never excludes __pycache__ from the hooks copy");
|
|
234
|
+
|
|
185
235
|
// ── report ──────────────────────────────────────────────────────────────────
|
|
186
236
|
if (errors.length) {
|
|
187
237
|
console.error(`validate-core: ${errors.length} error(s)\n`);
|
package/core/commands/cycle.md
DELETED
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
model: sonnet
|
|
3
|
-
description: Launch the full dev-cycle workflow (contract → build → smoke ∥ review → fix, until zero findings) for a frozen spec; relay its verdict + deferred questions.
|
|
4
|
-
argument-hint: <feature_id> [max_rounds]
|
|
5
|
-
---
|
|
6
|
-
|
|
7
|
-
You are the **lead**. Launch the full dev-cycle **workflow** for feature **$ARGUMENTS** — the
|
|
8
|
-
deterministic script does the orchestration (SCHEMA.md §Workflows, `cycle.js`); your job is only to
|
|
9
|
-
start it and relay its result. Do NOT run the phases yourself here — that's the conversational path
|
|
10
|
-
(`/build` → `/smoke` → `/review` → `/fix`), which remains the fallback below.
|
|
11
|
-
|
|
12
|
-
> **Kanban** (SCHEMA.md §Kanban): move card `#<feature_id>` → **Building** at launch. No-op silently
|
|
13
|
-
> if no board.
|
|
14
|
-
|
|
15
|
-
## 1. Resolve & check (fail fast, before spending anything)
|
|
16
|
-
|
|
17
|
-
- Parse `$ARGUMENTS`: the first token is `<feature_id>`, an optional second numeric token is
|
|
18
|
-
`<max_rounds>` (the workflow defaults to 5).
|
|
19
|
-
- Resolve the script: `.claude/workflows/cycle.js` if it exists, else `~/.claude/workflows/cycle.js`
|
|
20
|
-
(`test -f`). **Missing both** ⇒ the core predates 1.3.0 or is half-copied: tell the human to run
|
|
21
|
-
`/update-pipeline`, and stop.
|
|
22
|
-
- **Workflow runtime available?** If the `Workflow` tool is not in your toolset (Claude Code
|
|
23
|
-
< 2.1.154 or workflows disabled), say so and hand over the conversational path instead:
|
|
24
|
-
`/build <feature_id>` → `/smoke` → `/review` → `/fix` — same phases, interactive. Stop.
|
|
25
|
-
- Quick spec sanity (the workflow re-checks properly — this just saves a doomed launch):
|
|
26
|
-
`grep '^status:' specs/<feature_id>.md` must say `frozen` or `in-review`; otherwise tell the human
|
|
27
|
-
to run `/spec` first, and stop.
|
|
28
|
-
|
|
29
|
-
## 2. Launch
|
|
30
|
-
|
|
31
|
-
Call the `Workflow` tool: `scriptPath: <resolved cycle.js path>`,
|
|
32
|
-
`args: {"feature": "<feature_id>", "maxRounds": <max_rounds, omit if not given>}`.
|
|
33
|
-
It runs in the background — tell the human it's off and what it will do (build, then smoke ∥ review
|
|
34
|
-
→ fix rounds until zero findings + PASS; no questions mid-run), and that `/workflows` shows live
|
|
35
|
-
progress. Then END YOUR TURN — never poll, never sleep; the completion notification re-wakes you.
|
|
36
|
-
|
|
37
|
-
## 3. Relay the result (when the task notification arrives)
|
|
38
|
-
|
|
39
|
-
The workflow returns only a verdict object — the bulk is already on disk
|
|
40
|
-
(`specs/reports/<feature_id>.md`, spec `## Remediation`). Print, without re-reading any of it into
|
|
41
|
-
context:
|
|
42
|
-
|
|
43
|
-
- `outcome` · rounds used · review verdict · smoke result.
|
|
44
|
-
- `contractChanges` if any — flag them explicitly: the loop re-authored the frozen contract
|
|
45
|
-
lead-style; the human should eyeball those hunks in the diff.
|
|
46
|
-
- **The `questions` array, verbatim** — this is the human's inbox from the run (empty when the spec
|
|
47
|
-
pre-answered everything). Each one is a decision to make, usually by sharpening the spec.
|
|
48
|
-
- The `next` line: **SHIP-READY** ⇒ `/ship <feature_id>` (DoD ticked + freshness stamped — ship is a
|
|
49
|
-
straight shot, its human confirmation stays). **STOPPED** ⇒ answer the questions, then rerun
|
|
50
|
-
`/cycle <feature_id>` (it picks up from the spec's Remediation) or finish conversationally with
|
|
51
|
-
`/fix <feature_id>` + `/review <feature_id>`.
|
|
52
|
-
- **Kanban:** outcome SHIP-READY ⇒ move card → **Review** (the cycle's last verdict is a review);
|
|
53
|
-
otherwise → **Fix**. No-op silently if no board.
|
|
54
|
-
- **Recommend a `/clear`** — everything the next command needs is on disk.
|