pan-wizard 3.35.0 → 3.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -5
- package/agents/pan-designer.md +1 -1
- package/agents/pan-executor.md +25 -0
- package/agents/pan-hardener.md +2 -0
- package/agents/pan-phase-researcher.md +14 -0
- package/agents/pan-plan-checker.md +28 -1
- package/agents/pan-planner.md +11 -1
- package/agents/pan-previewer.md +2 -0
- package/agents/pan-project-researcher.md +8 -0
- package/agents/pan-reviewer.md +5 -0
- package/agents/pan-verifier.md +2 -2
- package/commands/pan/cost.md +21 -1
- package/commands/pan/preview.md +1 -1
- package/package.json +1 -1
- package/pan-wizard-core/bin/lib/config.cjs +10 -3
- package/pan-wizard-core/bin/lib/core.cjs +9 -4
- package/pan-wizard-core/bin/lib/cost.cjs +146 -0
- package/pan-wizard-core/bin/lib/doc-lint.cjs +5 -3
- package/pan-wizard-core/bin/lib/findings.cjs +115 -6
- package/pan-wizard-core/bin/lib/git.cjs +1 -1
- package/pan-wizard-core/bin/lib/mutate.cjs +257 -0
- package/pan-wizard-core/bin/lib/phase-gate.cjs +15 -9
- package/pan-wizard-core/bin/lib/phase-remove.cjs +91 -11
- package/pan-wizard-core/bin/lib/phase.cjs +37 -9
- package/pan-wizard-core/bin/lib/plan-lint.cjs +161 -0
- package/pan-wizard-core/bin/lib/preview.cjs +3 -0
- package/pan-wizard-core/bin/lib/research-lint.cjs +121 -0
- package/pan-wizard-core/bin/lib/roadmap.cjs +1 -0
- package/pan-wizard-core/bin/lib/test-gate.cjs +433 -0
- package/pan-wizard-core/bin/lib/usage.cjs +104 -0
- package/pan-wizard-core/bin/lib/verify-plan-set.cjs +1 -1
- package/pan-wizard-core/bin/lib/verify-red.cjs +138 -0
- package/pan-wizard-core/bin/lib/verify-scope.cjs +143 -2
- package/pan-wizard-core/bin/lib/verify.cjs +27 -27
- package/pan-wizard-core/bin/pan-tools.cjs +94 -6
- package/pan-wizard-core/mcp/orchestrator.cjs +4 -1
- package/pan-wizard-core/references/design-methodology.md +1 -1
- package/pan-wizard-core/references/handoff-decisions.md +1 -1
- package/pan-wizard-core/templates/config.json +5 -1
- package/pan-wizard-core/templates/design.md +5 -4
- package/pan-wizard-core/templates/phase-prompt.md +7 -0
- package/pan-wizard-core/templates/research-project/architecture.md +6 -0
- package/pan-wizard-core/templates/research-project/features.md +6 -0
- package/pan-wizard-core/templates/research-project/pitfalls.md +6 -0
- package/pan-wizard-core/templates/research-project/stack.md +6 -0
- package/pan-wizard-core/templates/research.md +10 -0
- package/pan-wizard-core/templates/summary-complex.md +5 -1
- package/pan-wizard-core/templates/summary-minimal.md +4 -0
- package/pan-wizard-core/templates/summary-standard.md +5 -1
- package/pan-wizard-core/templates/summary.md +8 -1
- package/pan-wizard-core/workflows/exec-phase.md +43 -7
- package/pan-wizard-core/workflows/execute-plan.md +3 -1
- package/pan-wizard-core/workflows/help.md +103 -44
- package/pan-wizard-core/workflows/plan-phase.md +20 -2
- package/pan-wizard-core/workflows/remove-phase.md +11 -1
- package/pan-wizard-core/workflows/settings.md +17 -0
- package/pan-wizard-core/workflows/verify-phase.md +39 -46
- package/scripts/build-help.cjs +106 -0
- package/scripts/mutation-probe.cjs +7 -80
- package/scripts/test-surface.cjs +168 -6
|
@@ -85,65 +85,58 @@ This step provides awareness; the hard gate is enforced by exec-phase.
|
|
|
85
85
|
</step>
|
|
86
86
|
|
|
87
87
|
<step name="run_test_suite">
|
|
88
|
-
**Run the project's test suite as a verification gate.**
|
|
88
|
+
**Run the project's test suite as a verification gate, through the engine.**
|
|
89
89
|
|
|
90
|
-
This step catches test regressions that goal-backward analysis cannot detect.
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
90
|
+
This step catches test regressions that goal-backward analysis cannot detect. The engine
|
|
91
|
+
runs the suite, not you: it finds the project's test command in any language
|
|
92
|
+
(`workflow.test_command` in config.json, else package.json's test script, pytest, `go test
|
|
93
|
+
./...`, cargo, Maven, Gradle or `make test`), reads the runner's own output, compares it with
|
|
94
|
+
the baseline exec-phase recorded before the first wave, and writes the result to the phase
|
|
95
|
+
directory as `{NN}-test-run.json`.
|
|
96
96
|
|
|
97
97
|
```bash
|
|
98
|
-
|
|
98
|
+
TEST_RUN=$(node ~/.claude/pan-wizard-core/bin/pan-tools.cjs test run --phase "${phase_number}")
|
|
99
99
|
```
|
|
100
100
|
|
|
101
|
-
|
|
102
|
-
|
|
101
|
+
Read from the JSON: `status`, `reason`, `command`, `total`, `passed`, `failed`,
|
|
102
|
+
`regressions`, `preexisting_failures` and `record`.
|
|
103
103
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
TEST_COUNT=$(echo "$TEST_OUTPUT" | grep -E "^(ℹ|#) tests" | awk '{print $NF}')
|
|
109
|
-
TEST_PASS=$(echo "$TEST_OUTPUT" | grep -E "^(ℹ|#) pass" | awk '{print $NF}')
|
|
110
|
-
TEST_FAIL=$(echo "$TEST_OUTPUT" | grep -E "^(ℹ|#) fail" | awk '{print $NF}')
|
|
111
|
-
```
|
|
104
|
+
**Copy the status; do not judge it.** Write `test_gate: <status>` into the verification.md
|
|
105
|
+
frontmatter exactly as the record says. Reconcile compares the two, so a verification whose
|
|
106
|
+
`test_gate` disagrees with the record is refused at the phase gate, and a failed record
|
|
107
|
+
lowers a `passed` verification to `gaps_found` whatever the frontmatter says.
|
|
112
108
|
|
|
113
|
-
|
|
109
|
+
| `status` | What it means | What you do |
|
|
110
|
+
|----------|---------------|-------------|
|
|
111
|
+
| `passed` | The suite ran and nothing that passed at the baseline fails now | Record the counts. If `preexisting_failures` is not empty, list them as known failures the phase inherited, not as gaps |
|
|
112
|
+
| `failed` | A test that passed at the baseline fails now (`regressions`), or the suite did not run at all (a crash, a build error, a timeout: `reason` says `suite did not run`) | Set the overall status to `gaps_found` and list each regression, or the reason, as a gap |
|
|
113
|
+
| `skipped` | There was nothing to run: no test command, or one that ran no tests | Add a `not_checked` entry: `check: "tests"`, `reason:` the record's `reason` |
|
|
114
114
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
115
|
+
A suite that exists and did not run is `failed`, never `skipped`: the engine decides that
|
|
116
|
+
from the exit code and the runner's output, and you copy it.
|
|
117
|
+
|
|
118
|
+
**If test gate FAILED:** Continue verification (gather full picture) but set overall status to `gaps_found` regardless of goal-backward results. Include test failures as gaps in the report.
|
|
119
|
+
</step>
|
|
120
120
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
| `TEST_EXIT` = 0 **and** `TEST_FAIL` = 0 (a real number) | Record counts, continue to must-haves |
|
|
124
|
-
| `TEST_EXIT` = 0 **and** no `ℹ`/`#` count lines (a runner other than Node's built-in one) | Set `test_gate_status: passed`; record the runner's own summary line as the counts |
|
|
125
|
-
| `TEST_EXIT` ≠ 0 **and** failures were reported | Set `test_gate_status: failed`, include failure details |
|
|
126
|
-
| `TEST_EXIT` ≠ 0 **and** `TEST_FAIL` is empty/absent — the suite CRASHED or could not run (syntax error, missing module, bad import, no runner) | Set `test_gate_status: failed`, and record the reason as `suite did not run`. **Never `skipped`, never `passed`.** A suite that cannot execute is stronger evidence of a broken phase than one that runs and fails |
|
|
127
|
-
| `HAS_TEST` = `no` (handled in step 1) | Record as `test_gate_status: skipped`, continue. This is the ONLY legitimate route to `skipped` |
|
|
121
|
+
<step name="mutation_report">
|
|
122
|
+
**Measure whether the tests pin the code the phase wrote** (a report, never a gate).
|
|
128
123
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
project. They must never share a verdict; the first is `skipped`, the second is
|
|
132
|
-
`failed`.
|
|
124
|
+
Only when the test gate passed: a mutant cannot be read against a suite that already fails.
|
|
125
|
+
With `workflow.mutation_gate: off` the command runs nothing and says so.
|
|
133
126
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
TEST_GATE_STATUS: passed | failed | skipped
|
|
137
|
-
TEST_TOTAL: ${TEST_COUNT}
|
|
138
|
-
TEST_PASSED: ${TEST_PASS}
|
|
139
|
-
TEST_FAILED: ${TEST_FAIL}
|
|
127
|
+
```bash
|
|
128
|
+
MUTATION=$(node ~/.claude/pan-wizard-core/bin/pan-tools.cjs verify mutate --phase "${phase_number}" --max 5)
|
|
140
129
|
```
|
|
141
|
-
Write `test_gate: ${TEST_GATE_STATUS}` into the verification.md frontmatter. When it is
|
|
142
|
-
`skipped`, also add a `not_checked` entry (`check: "tests"`, `reason: "no test script in
|
|
143
|
-
package.json"`). The recorder reads both: a pass whose tests never ran is recorded as a
|
|
144
|
-
pass with tests not checked, never as a plain pass.
|
|
145
130
|
|
|
146
|
-
|
|
131
|
+
The engine flips one comparison, boolean or `+ 1` at a time on the lines the phase's plan
|
|
132
|
+
commits changed, in a throwaway worktree at HEAD, and runs the suite against each mutant. Read
|
|
133
|
+
`checked`, `sampled`, `killed`, `survivors` (file, line, the swap) and `aborted`.
|
|
134
|
+
|
|
135
|
+
Write a **Test strength** section in verification.md: `killed/sampled`, and each survivor as
|
|
136
|
+
`file:line — before → after`, framed as a test the phase could add. A survivor is a lead, not a
|
|
137
|
+
gap: it never changes the status. `aborted: flaky` (the unmutated suite passed once and failed
|
|
138
|
+
once) is worth reporting as a finding of its own; `checked: false` with its `reason` goes in
|
|
139
|
+
`not_checked` as `check: "mutation"`.
|
|
147
140
|
</step>
|
|
148
141
|
|
|
149
142
|
<step name="establish_must_haves">
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
/**
|
|
4
|
+
* Generate /pan:help's command index from command frontmatter (program L5).
|
|
5
|
+
*
|
|
6
|
+
* node scripts/build-help.cjs # print the block
|
|
7
|
+
* node scripts/build-help.cjs --write # rewrite the block in pan-wizard-core/workflows/help.md
|
|
8
|
+
* node scripts/build-help.cjs --check # exit 1 when the committed block is stale
|
|
9
|
+
*
|
|
10
|
+
* The block between GENERATED_START and GENERATED_END in help.md lists every
|
|
11
|
+
* commands/pan/*.md by its frontmatter `group` and `description`, so help cannot leave
|
|
12
|
+
* a command out. tests/help-generated.test.cjs fails when it is stale.
|
|
13
|
+
*/
|
|
14
|
+
const fs = require('fs');
|
|
15
|
+
const path = require('path');
|
|
16
|
+
const { extractFrontmatter } = require('../pan-wizard-core/bin/lib/frontmatter.cjs');
|
|
17
|
+
|
|
18
|
+
const ROOT = path.join(__dirname, '..');
|
|
19
|
+
const HELP_REL = path.join('pan-wizard-core', 'workflows', 'help.md');
|
|
20
|
+
const GENERATED_START = '<!-- generated:command-index:start (scripts/build-help.cjs --write) -->';
|
|
21
|
+
const GENERATED_END = '<!-- generated:command-index:end -->';
|
|
22
|
+
|
|
23
|
+
// The order a new user reads the groups in; groups not listed follow, alphabetically.
|
|
24
|
+
const GROUP_ORDER = [
|
|
25
|
+
'Getting Started', 'Phase Lifecycle', 'Phase Management', 'Session & Progress',
|
|
26
|
+
'Milestone', 'Milestone Lifecycle', 'Focus', 'Foresight', 'Review', 'Validation',
|
|
27
|
+
'Observability', 'Knowledge', 'Self-Improvement', 'Git Workflow', 'Army',
|
|
28
|
+
'External tools', 'System', 'Community',
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
/** Every shipped command as {name, group, description}, sorted by name. */
|
|
32
|
+
function readCommands(root = ROOT) {
|
|
33
|
+
const dir = path.join(root, 'commands', 'pan');
|
|
34
|
+
return fs.readdirSync(dir).filter((f) => f.endsWith('.md')).sort().map((f) => {
|
|
35
|
+
const fm = extractFrontmatter(fs.readFileSync(path.join(dir, f), 'utf-8')) || {};
|
|
36
|
+
return {
|
|
37
|
+
// The file name is the command (`/pan:<file>`); some frontmatter names omit the prefix.
|
|
38
|
+
name: `pan:${f.replace(/\.md$/, '')}`,
|
|
39
|
+
group: String(fm.group || '').trim(),
|
|
40
|
+
description: String(fm.description || '').replace(/\s+/g, ' ').trim(),
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Backslashes first, so one in a description cannot turn the escaped pipe after it back into a cell break.
|
|
46
|
+
const cell = (s) => s.replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
|
|
47
|
+
|
|
48
|
+
/** The markdown command index, grouped. Throws on a command with no group or description. */
|
|
49
|
+
function renderCommandIndex(commands) {
|
|
50
|
+
const groups = new Map();
|
|
51
|
+
for (const c of commands) {
|
|
52
|
+
if (!c.group) throw new Error(`${c.name} has no group in its frontmatter`);
|
|
53
|
+
if (!c.description) throw new Error(`${c.name} has no description in its frontmatter`);
|
|
54
|
+
if (!groups.has(c.group)) groups.set(c.group, []);
|
|
55
|
+
groups.get(c.group).push(c);
|
|
56
|
+
}
|
|
57
|
+
const rank = (g) => { const i = GROUP_ORDER.indexOf(g); return i === -1 ? GROUP_ORDER.length : i; };
|
|
58
|
+
const order = [...groups.keys()].sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
|
|
59
|
+
const out = [];
|
|
60
|
+
for (const g of order) {
|
|
61
|
+
out.push(`### ${g}`, '| Command | Description |', '|---------|-------------|');
|
|
62
|
+
for (const c of groups.get(g).sort((a, b) => a.name.localeCompare(b.name))) out.push(`| \`/${c.name}\` | ${cell(c.description)} |`);
|
|
63
|
+
out.push('');
|
|
64
|
+
}
|
|
65
|
+
return out.join('\n').trimEnd();
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The generated block's content in a help.md text, or null when the markers are missing. */
|
|
69
|
+
function extractBlock(text) {
|
|
70
|
+
const t = String(text).replace(/\r\n/g, '\n');
|
|
71
|
+
const a = t.indexOf(GENERATED_START);
|
|
72
|
+
const b = t.indexOf(GENERATED_END);
|
|
73
|
+
if (a === -1 || b === -1 || b < a) return null;
|
|
74
|
+
return t.slice(a + GENERATED_START.length, b).trim();
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** help.md with its block replaced by `block`. Throws when the markers are missing. */
|
|
78
|
+
function replaceBlock(text, block) {
|
|
79
|
+
const t = String(text).replace(/\r\n/g, '\n');
|
|
80
|
+
const a = t.indexOf(GENERATED_START);
|
|
81
|
+
const b = t.indexOf(GENERATED_END);
|
|
82
|
+
if (a === -1 || b === -1 || b < a) throw new Error(`${HELP_REL} has no generated command-index markers`);
|
|
83
|
+
return t.slice(0, a + GENERATED_START.length) + '\n' + block + '\n' + t.slice(b);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function main(argv) {
|
|
87
|
+
const file = path.join(ROOT, HELP_REL);
|
|
88
|
+
const block = renderCommandIndex(readCommands(ROOT));
|
|
89
|
+
if (argv.includes('--write')) {
|
|
90
|
+
fs.writeFileSync(file, replaceBlock(fs.readFileSync(file, 'utf-8'), block));
|
|
91
|
+
process.stdout.write(`wrote the command index into ${HELP_REL}\n`);
|
|
92
|
+
} else if (argv.includes('--check')) {
|
|
93
|
+
if (extractBlock(fs.readFileSync(file, 'utf-8')) !== block) {
|
|
94
|
+
process.stderr.write(`${HELP_REL}'s command index is stale: run node scripts/build-help.cjs --write\n`);
|
|
95
|
+
process.exitCode = 1;
|
|
96
|
+
} else {
|
|
97
|
+
process.stdout.write(`${HELP_REL}'s command index is up to date\n`);
|
|
98
|
+
}
|
|
99
|
+
} else {
|
|
100
|
+
process.stdout.write(block + '\n');
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
if (require.main === module) main(process.argv.slice(2));
|
|
105
|
+
|
|
106
|
+
module.exports = { GROUP_ORDER, GENERATED_START, GENERATED_END, readCommands, renderCommandIndex, extractBlock, replaceBlock };
|
|
@@ -51,86 +51,13 @@ const TARGETS = [
|
|
|
51
51
|
{ file: 'pan-wizard-core/bin/pan-tools.cjs', tests: ['tests/dispatcher.test.cjs', 'tests/dispatcher-arms.test.cjs'] },
|
|
52
52
|
];
|
|
53
53
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const
|
|
61
|
-
{ id: 'gte→gt', find: ' >= ', replace: ' > ' },
|
|
62
|
-
{ id: 'gt→gte', find: ' > ', replace: ' >= ' },
|
|
63
|
-
{ id: 'lte→lt', find: ' <= ', replace: ' < ' },
|
|
64
|
-
{ id: 'lt→lte', find: ' < ', replace: ' <= ' },
|
|
65
|
-
{ id: 'eq→neq', find: ' === ', replace: ' !== ' },
|
|
66
|
-
{ id: 'neq→eq', find: ' !== ', replace: ' === ' },
|
|
67
|
-
{ id: 'and→or', find: ' && ', replace: ' || ' },
|
|
68
|
-
{ id: 'or→and', find: ' || ', replace: ' && ' },
|
|
69
|
-
{ id: 'true→false', find: 'return true', replace: 'return false' },
|
|
70
|
-
{ id: 'false→true', find: 'return false', replace: 'return true' },
|
|
71
|
-
{ id: 'plus→minus', find: ' + 1', replace: ' - 1' },
|
|
72
|
-
];
|
|
73
|
-
|
|
74
|
-
/**
|
|
75
|
-
* Is this line worth mutating? Skips comments, and skips a line whose only occurrence of
|
|
76
|
-
* the token is inside a string literal — mutating a message changes nothing a test should
|
|
77
|
-
* be pinning, and a survivor there would be noise.
|
|
78
|
-
*
|
|
79
|
-
* Crude but conservative: a line containing a quote is only mutated when the token also
|
|
80
|
-
* occurs before the first quote.
|
|
81
|
-
*/
|
|
82
|
-
function mutableLine(line, token) {
|
|
83
|
-
const trimmed = line.trim();
|
|
84
|
-
if (!trimmed || trimmed.startsWith('//') || trimmed.startsWith('*') || trimmed.startsWith('/*')) return false;
|
|
85
|
-
const at = line.indexOf(token);
|
|
86
|
-
if (at === -1) return false;
|
|
87
|
-
const firstQuote = Math.min(
|
|
88
|
-
...["'", '"', '`'].map((q) => { const i = line.indexOf(q); return i === -1 ? Infinity : i; }),
|
|
89
|
-
);
|
|
90
|
-
return at < firstQuote;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* Every mutation available in a source file: one per (line, operator) pair, first
|
|
95
|
-
* occurrence on the line.
|
|
96
|
-
*
|
|
97
|
-
* @returns {Array<{line: number, op: string, before: string, after: string}>}
|
|
98
|
-
*/
|
|
99
|
-
function mutationsFor(src) {
|
|
100
|
-
const lines = src.split(/\r?\n/);
|
|
101
|
-
const out = [];
|
|
102
|
-
lines.forEach((line, i) => {
|
|
103
|
-
for (const op of OPERATORS) {
|
|
104
|
-
if (!mutableLine(line, op.find)) continue;
|
|
105
|
-
out.push({ line: i + 1, op: op.id, before: line, after: line.replace(op.find, op.replace) });
|
|
106
|
-
}
|
|
107
|
-
});
|
|
108
|
-
return out;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/** Apply one mutation to a source string, by line number. Throws if the line moved. */
|
|
112
|
-
function applyMutation(src, mutation) {
|
|
113
|
-
const eol = src.includes('\r\n') ? '\r\n' : '\n';
|
|
114
|
-
const lines = src.split(/\r?\n/);
|
|
115
|
-
const idx = mutation.line - 1;
|
|
116
|
-
if (lines[idx] !== mutation.before) {
|
|
117
|
-
throw new Error(`line ${mutation.line} is not what the mutation was built from`);
|
|
118
|
-
}
|
|
119
|
-
lines[idx] = mutation.after;
|
|
120
|
-
return lines.join(eol);
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/** Deterministic shuffle, so a run is reproducible from its seed. */
|
|
124
|
-
function sample(items, n, seed) {
|
|
125
|
-
let s = seed >>> 0;
|
|
126
|
-
const rand = () => { s = (s * 1664525 + 1013904223) >>> 0; return s / 0x100000000; };
|
|
127
|
-
const copy = items.slice();
|
|
128
|
-
for (let i = copy.length - 1; i > 0; i--) {
|
|
129
|
-
const j = Math.floor(rand() * (i + 1));
|
|
130
|
-
[copy[i], copy[j]] = [copy[j], copy[i]];
|
|
131
|
-
}
|
|
132
|
-
return copy.slice(0, n);
|
|
133
|
-
}
|
|
54
|
+
// The operators, the line filter, mutation and the seeded sample live in the shipped
|
|
55
|
+
// mutation report (pan-wizard-core/bin/lib/mutate.cjs, program V5), which a user's phase
|
|
56
|
+
// diff is measured with; this probe applies the JavaScript table to PAN itself.
|
|
57
|
+
const M = require('../pan-wizard-core/bin/lib/mutate.cjs');
|
|
58
|
+
const OPERATORS = M.OPERATORS.js;
|
|
59
|
+
const { mutableLine, applyMutation, sample } = M;
|
|
60
|
+
const mutationsFor = (src) => M.mutationsFor(src, { family: 'js' });
|
|
134
61
|
|
|
135
62
|
function parseArgs(argv) {
|
|
136
63
|
const a = { max: 20, seed: 1, json: false, targets: [], timeoutMs: 600000, help: false };
|
package/scripts/test-surface.cjs
CHANGED
|
@@ -19,7 +19,9 @@
|
|
|
19
19
|
* Modes:
|
|
20
20
|
* node scripts/test-surface.cjs summary
|
|
21
21
|
* node scripts/test-surface.cjs --write write tests/fixtures/surface.json (the committed registry)
|
|
22
|
-
* node scripts/test-surface.cjs --check exit 1 when the committed registry differs from the code
|
|
22
|
+
* node scripts/test-surface.cjs --check exit 1 when the committed registry differs from the code, or
|
|
23
|
+
* a command, agent, verb or config key has no user (liveness, L1)
|
|
24
|
+
* node scripts/test-surface.cjs --liveness who uses each command, agent, verb, subcommand and config key
|
|
23
25
|
* node scripts/test-surface.cjs --map which test files reference each surface row; lists the misses
|
|
24
26
|
* node scripts/test-surface.cjs --scaffold <dir> one todo stub per unreferenced row (for a suite rebuilt from scratch)
|
|
25
27
|
*
|
|
@@ -297,6 +299,153 @@ function scaffold(missingRows, outDir) {
|
|
|
297
299
|
|
|
298
300
|
function readJson(p) { return JSON.parse(fs.readFileSync(p, 'utf8')); }
|
|
299
301
|
|
|
302
|
+
// ─── Liveness (program L1) ──────────────────────────────────────────────────
|
|
303
|
+
//
|
|
304
|
+
// The map above asks "does a test name it?". Liveness asks "does anything USE it?":
|
|
305
|
+
// - every shipped command and agent is named by a harness scenario, so some behavioural
|
|
306
|
+
// run exercises it as a user would;
|
|
307
|
+
// - every pan-tools verb and subcommand has a caller in what PAN ships: a prompt
|
|
308
|
+
// (commands, agents, workflows, references, templates), a native workflow's prompt
|
|
309
|
+
// (bin/install-lib.cjs), or an MCP tool or resource (the bridge registry);
|
|
310
|
+
// - every config default has a reader: code under the core, hooks, installer or MCP
|
|
311
|
+
// bridge that reads the key, or a shipped prompt that runs `config-get <key>`.
|
|
312
|
+
// An orphan either gains a user or sits in tests/fixtures/liveness-allowlist.json with a
|
|
313
|
+
// reason; `--check` fails on an orphan without one, and on an entry no longer needed.
|
|
314
|
+
|
|
315
|
+
const LIVENESS_ALLOWLIST_REL = path.join('tests', 'fixtures', 'liveness-allowlist.json');
|
|
316
|
+
const PROMPT_DIRS = Object.freeze(['commands/pan', 'agents', 'pan-wizard-core/workflows', 'pan-wizard-core/references', 'pan-wizard-core/templates']);
|
|
317
|
+
const NATIVE_WORKFLOW_SOURCES = Object.freeze(['bin/install-lib.cjs']);
|
|
318
|
+
const READER_DIRS = Object.freeze(['pan-wizard-core/bin', 'pan-wizard-core/mcp', 'pan-wizard-core/opencode', 'hooks', 'bin']);
|
|
319
|
+
const SCENARIO_DIR = 'harness/scenarios';
|
|
320
|
+
|
|
321
|
+
/** Every file under rel (recursively) whose name matches re, as posix paths; skips dist/ and node_modules/. */
|
|
322
|
+
function walkFiles(root, rel, re) {
|
|
323
|
+
const out = [];
|
|
324
|
+
const walk = (r) => {
|
|
325
|
+
let ents;
|
|
326
|
+
try { ents = fs.readdirSync(path.join(root, r), { withFileTypes: true }); } catch { return; }
|
|
327
|
+
for (const e of ents) {
|
|
328
|
+
const p = path.posix.join(r, e.name);
|
|
329
|
+
if (e.isDirectory()) { if (e.name !== 'dist' && e.name !== 'node_modules') walk(p); } else if (re.test(e.name)) out.push(p);
|
|
330
|
+
}
|
|
331
|
+
};
|
|
332
|
+
walk(rel);
|
|
333
|
+
return out.sort();
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* The verbs and `verb sub` pairs a text invokes: the script path (`pan-tools.cjs <verb>`),
|
|
338
|
+
* a backticked `pan-tools <verb>`, or `pan-tools <verb>` as a shell command (opening a
|
|
339
|
+
* line, or after `|`, `;`, `&` or `(`), then the verb and an optional subcommand. Prose
|
|
340
|
+
* that mentions pan-tools is not a call.
|
|
341
|
+
*/
|
|
342
|
+
function pantoolsCalls(text) {
|
|
343
|
+
const out = new Set();
|
|
344
|
+
const re = /(?:pan-tools\.cjs["']?|`pan-tools|(?:^|[|;&(])[ \t]*pan-tools)[ \t]+([a-z][a-z0-9-]*)(?:[ \t]+([a-z][a-z0-9-]*))?/gm;
|
|
345
|
+
let m;
|
|
346
|
+
while ((m = re.exec(text))) { out.add(m[1]); if (m[2]) out.add(`${m[1]} ${m[2]}`); }
|
|
347
|
+
return out;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/** The text a liveness check reads, from the tree. Tests pass a hand-built object instead. */
|
|
351
|
+
function loadLivenessInputs(root = ROOT) {
|
|
352
|
+
const read = (rel) => fs.readFileSync(path.join(root, rel), 'utf8');
|
|
353
|
+
const prompts = PROMPT_DIRS.flatMap((d) => walkFiles(root, d, /\.(md|js)$/)).map((file) => ({ file, src: read(file) }));
|
|
354
|
+
const native = NATIVE_WORKFLOW_SOURCES.filter((f) => fs.existsSync(path.join(root, f))).map((file) => ({ file, src: read(file) }));
|
|
355
|
+
const registry = require(path.join(root, SOURCES.mcpRegistry));
|
|
356
|
+
const mcp = [...(registry.TOOLS || []), ...(registry.RESOURCES || [])].map((t) => ({ name: t.name || t.uri, verb: t.verb, sub: (t.args && t.args[0]) || null }));
|
|
357
|
+
const scenarios = walkFiles(root, SCENARIO_DIR, /\.json$/).map((file) => ({ file, src: read(file) }));
|
|
358
|
+
const configSrc = path.posix.normalize(SOURCES.config);
|
|
359
|
+
const readers = READER_DIRS.flatMap((d) => walkFiles(root, d, /\.(c?js|mjs)$/)).map((file) => ({ file, src: read(file), isConfig: file === configSrc }));
|
|
360
|
+
return { prompts, native, mcp, scenarios, readers };
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** The body of buildConfigDefaults in config.cjs: where defaults are written, which is not a read. */
|
|
364
|
+
function stripDefaultsBuilder(src) {
|
|
365
|
+
const start = src.indexOf('function buildConfigDefaults');
|
|
366
|
+
if (start === -1) return src;
|
|
367
|
+
const open = src.indexOf('{', start);
|
|
368
|
+
let depth = 0;
|
|
369
|
+
for (let i = open; i < src.length; i++) {
|
|
370
|
+
if (src[i] === '{') depth++;
|
|
371
|
+
else if (src[i] === '}' && --depth === 0) return src.slice(0, start) + src.slice(i + 1);
|
|
372
|
+
}
|
|
373
|
+
return src;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Who uses each piece of the surface. Pure over `surface` (extractSurface) and `inputs`
|
|
378
|
+
* (loadLivenessInputs). Returns one row per command, agent, verb, subcommand and config
|
|
379
|
+
* key: `{id, kind, users: string[]}`; a row with no users is an orphan.
|
|
380
|
+
*/
|
|
381
|
+
function liveness(surface, inputs) {
|
|
382
|
+
const rows = [];
|
|
383
|
+
const scen = inputs.scenarios;
|
|
384
|
+
const named = (re) => scen.filter((s) => re.test(s.src)).map((s) => s.file);
|
|
385
|
+
for (const f of surface.content['commands/pan'] || []) {
|
|
386
|
+
const name = f.replace(/\.md$/, '');
|
|
387
|
+
// /pan:<name> (Claude, Gemini), /pan-<name> or pan-<name> as a skill (Codex, Copilot, OpenCode), or the file itself.
|
|
388
|
+
rows.push({ id: `command:${name}`, kind: 'command', users: named(new RegExp(`(?:/pan[:-]|commands/pan/|\\bpan-)${esc(name)}(?![a-z0-9-])`)) });
|
|
389
|
+
}
|
|
390
|
+
for (const f of surface.content.agents || []) {
|
|
391
|
+
const name = f.replace(/\.md$/, '');
|
|
392
|
+
rows.push({ id: `agent:${name}`, kind: 'agent', users: named(new RegExp(`(?<![a-z0-9-])${esc(name)}(?![a-z0-9-])`)) });
|
|
393
|
+
}
|
|
394
|
+
const callers = new Map();
|
|
395
|
+
const add = (key, who) => { if (!callers.has(key)) callers.set(key, new Set()); callers.get(key).add(who); };
|
|
396
|
+
for (const t of [...inputs.prompts, ...inputs.native]) for (const c of pantoolsCalls(t.src)) add(c, t.file);
|
|
397
|
+
// A registry entry with no verb (one served by the bridge itself) calls nothing.
|
|
398
|
+
for (const t of inputs.mcp.filter((x) => x.verb)) { add(t.verb, `mcp:${t.name}`); if (t.sub) add(`${t.verb} ${t.sub}`, `mcp:${t.name}`); }
|
|
399
|
+
for (const v of surface.verbs) {
|
|
400
|
+
const direct = callers.get(v) || new Set();
|
|
401
|
+
// A verb is alive when it is called, bare or through any of its subcommands.
|
|
402
|
+
const viaSub = [...callers.entries()].filter(([k]) => k.startsWith(`${v} `)).flatMap(([, s]) => [...s]);
|
|
403
|
+
rows.push({ id: `caller:verb:${v}`, kind: 'verb', users: [...new Set([...direct, ...viaSub])].sort() });
|
|
404
|
+
}
|
|
405
|
+
for (const s of surface.subcommands) rows.push({ id: `caller:sub:${s}`, kind: 'sub', users: [...(callers.get(s) || [])].sort() });
|
|
406
|
+
const readerText = inputs.readers.map((r) => ({ file: r.file, src: r.isConfig ? stripDefaultsBuilder(r.src) : r.src }));
|
|
407
|
+
const promptText = [...inputs.prompts, ...inputs.native];
|
|
408
|
+
for (const key of surface.config_keys) {
|
|
409
|
+
const last = esc(key.split('.').pop());
|
|
410
|
+
const codeRe = new RegExp(`['"\`]${esc(key)}['"\`]|\\.${last}\\b|\\[['"]${last}['"]\\]|['"]${last}['"]`);
|
|
411
|
+
// A prompt reads a key with `config-get <key>`, or names the full dotted key in backticks
|
|
412
|
+
// for the agent to check in the config or the init payload (`workflow.x: true`).
|
|
413
|
+
const promptRe = new RegExp(`config-get[ \\t]+["']?${esc(key)}\\b|\`${esc(key)}[\`:= ]`);
|
|
414
|
+
const users = [...readerText.filter((r) => codeRe.test(r.src)).map((r) => r.file), ...promptText.filter((p) => promptRe.test(p.src)).map((p) => p.file)];
|
|
415
|
+
rows.push({ id: `reader:config:${key}`, kind: 'config', users: [...new Set(users)].sort() });
|
|
416
|
+
}
|
|
417
|
+
return rows;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* Orphans not allowlisted, and allowlist entries that are wrong (no reason, no such row, or
|
|
422
|
+
* a row that has users now). `--check` fails when either list is non-empty.
|
|
423
|
+
*/
|
|
424
|
+
function livenessProblems(rows, allowlist) {
|
|
425
|
+
const allow = new Map(allowlist.map((a) => [a.id, a]));
|
|
426
|
+
const problems = [];
|
|
427
|
+
for (const a of allowlist) {
|
|
428
|
+
if (!a.reason || !String(a.reason).trim()) problems.push(`liveness allowlist entry ${a.id} has no reason`);
|
|
429
|
+
const row = rows.find((r) => r.id === a.id);
|
|
430
|
+
if (!row) problems.push(`liveness allowlist entry ${a.id} names nothing that exists — remove it`);
|
|
431
|
+
else if (row.users.length) problems.push(`liveness allowlist entry ${a.id} has a user now (${row.users[0]}) — remove the entry`);
|
|
432
|
+
}
|
|
433
|
+
const hint = { command: 'name it in a harness scenario', agent: 'name it in a harness scenario', verb: 'call it from a shipped prompt', sub: 'call it from a shipped prompt', config: 'read it in code, or drop the default' };
|
|
434
|
+
for (const r of rows) if (!r.users.length && !allow.has(r.id)) problems.push(`orphan ${r.id} — ${hint[r.kind]}, or allowlist it in ${LIVENESS_ALLOWLIST_REL} with a reason`);
|
|
435
|
+
return problems;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/** What `--check` decides: registry drift plus liveness problems. Pure. */
|
|
439
|
+
function checkReport(committed, surface, inputs, allowlist) {
|
|
440
|
+
const drift = diffSurface(committed, surface);
|
|
441
|
+
const live = livenessProblems(liveness(surface, inputs), allowlist);
|
|
442
|
+
return { drift, live, ok: !drift.added.length && !drift.removed.length && !live.length };
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
function readLivenessAllowlist(root = ROOT) {
|
|
446
|
+
try { return readJson(path.join(root, LIVENESS_ALLOWLIST_REL)); } catch { return []; }
|
|
447
|
+
}
|
|
448
|
+
|
|
300
449
|
// ─── CLI ────────────────────────────────────────────────────────────────────
|
|
301
450
|
|
|
302
451
|
function main(argv) {
|
|
@@ -310,13 +459,25 @@ function main(argv) {
|
|
|
310
459
|
if (argv.includes('--check')) {
|
|
311
460
|
let committed;
|
|
312
461
|
try { committed = readJson(registryPath); } catch { console.error(`no committed registry at ${REGISTRY_REL} — run --write`); return 1; }
|
|
313
|
-
const d =
|
|
314
|
-
if (
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
462
|
+
const { drift: d, live, ok } = checkReport(committed, surface, loadLivenessInputs(ROOT), readLivenessAllowlist(ROOT));
|
|
463
|
+
if (ok) { console.log('surface registry matches the code; every command, agent, verb and config key is alive'); return 0; }
|
|
464
|
+
if (d.added.length || d.removed.length) {
|
|
465
|
+
console.error(`surface registry is stale — run \`node scripts/test-surface.cjs --write\` and commit it`);
|
|
466
|
+
for (const x of d.added) console.error(` + ${x}`);
|
|
467
|
+
for (const x of d.removed) console.error(` - ${x}`);
|
|
468
|
+
}
|
|
469
|
+
if (live.length) {
|
|
470
|
+
console.error('liveness: orphans without a user or an allowlist reason');
|
|
471
|
+
for (const x of live) console.error(` ${x}`);
|
|
472
|
+
}
|
|
318
473
|
return 1;
|
|
319
474
|
}
|
|
475
|
+
if (argv.includes('--liveness')) {
|
|
476
|
+
const rows = liveness(surface, loadLivenessInputs(ROOT));
|
|
477
|
+
for (const r of rows) console.log(`${r.users.length ? 'ok ' : 'ORPHAN'} ${r.id.padEnd(48)} ${r.users.slice(0, 2).join(', ')}${r.users.length > 2 ? ` +${r.users.length - 2}` : ''}`);
|
|
478
|
+
console.log(`liveness rows: ${rows.length} · orphans: ${rows.filter((r) => !r.users.length).length}`);
|
|
479
|
+
return 0;
|
|
480
|
+
}
|
|
320
481
|
const rows = mapSurface(surfaceRows(surface), loadTestSources(ROOT));
|
|
321
482
|
const missing = rows.filter((r) => !r.hits.length);
|
|
322
483
|
if (argv.includes('--scaffold')) {
|
|
@@ -339,4 +500,5 @@ module.exports = {
|
|
|
339
500
|
ROOT, REGISTRY_REL, ALLOWLIST_REL, SOURCES, EXTRA_HOOK_ROWS,
|
|
340
501
|
parseTopLevelCommands, parseGroupSubcommands, parseCaseArms, parseInstallerFlags, flattenKeys, hookMatrix,
|
|
341
502
|
extractSurface, surfaceRows, referencePattern, mapSurface, listTestFiles, loadTestSources, diffSurface, scaffold,
|
|
503
|
+
LIVENESS_ALLOWLIST_REL, pantoolsCalls, loadLivenessInputs, stripDefaultsBuilder, liveness, livenessProblems, checkReport, readLivenessAllowlist,
|
|
342
504
|
};
|