pan-wizard 3.28.0 → 3.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -39
- package/bin/install-lib.cjs +65 -58
- package/bin/install.js +199 -188
- package/commands/pan/army.md +2 -2
- package/commands/pan/audit-deployment.md +2 -2
- package/commands/pan/cost.md +19 -7
- package/commands/pan/exec-phase.md +2 -0
- package/commands/pan/focus-auto.md +5 -5
- package/hooks/dist/pan-check-update.js +4 -0
- package/hooks/dist/pan-cost-logger.js +322 -43
- package/hooks/dist/pan-stop-guard.js +81 -2
- package/hooks/dist/pan-trace-logger.js +275 -32
- package/package.json +4 -1
- package/pan-wizard-core/bin/lib/agents-md.cjs +3 -2
- package/pan-wizard-core/bin/lib/commands.cjs +3 -1
- package/pan-wizard-core/bin/lib/constants.cjs +17 -0
- package/pan-wizard-core/bin/lib/context-budget.cjs +10 -0
- package/pan-wizard-core/bin/lib/core.cjs +17 -4
- package/pan-wizard-core/bin/lib/cost-rebuild.cjs +511 -0
- package/pan-wizard-core/bin/lib/cost.cjs +165 -55
- package/pan-wizard-core/bin/lib/git.cjs +5 -1
- package/pan-wizard-core/bin/lib/hud.cjs +5 -3
- package/pan-wizard-core/bin/lib/hygiene.cjs +22 -25
- package/pan-wizard-core/bin/lib/memory-rebuild.cjs +3 -3
- package/pan-wizard-core/bin/lib/memory.cjs +14 -8
- package/pan-wizard-core/bin/lib/optimize.cjs +78 -2
- package/pan-wizard-core/bin/lib/utils.cjs +22 -0
- package/pan-wizard-core/bin/lib/verify-deploy.cjs +1 -1
- package/pan-wizard-core/bin/lib/verify.cjs +24 -10
- package/pan-wizard-core/bin/pan-tools.cjs +8 -1
- package/pan-wizard-core/references/model-profiles.md +4 -4
- package/pan-wizard-core/references/planning-config.md +19 -23
- package/pan-wizard-core/workflows/health.md +1 -0
- package/pan-wizard-core/workflows/settings.md +2 -4
- package/scripts/coverage-gate.cjs +257 -0
- package/scripts/install-git-hooks.js +5 -0
- package/scripts/mutation-probe.cjs +272 -0
- package/scripts/release-check.js +33 -12
- package/scripts/test-quality-lint.cjs +240 -0
- package/scripts/test-surface.cjs +336 -0
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
'use strict';
|
|
3
|
+
/**
|
|
4
|
+
* Mutation probe — does the suite NOTICE when the code is wrong?
|
|
5
|
+
*
|
|
6
|
+
* Coverage says a line executed. It does not say an assertion depended on it. This probe
|
|
7
|
+
* answers the harder question by breaking the code on purpose: apply one small mutation,
|
|
8
|
+
* run the tests that claim to cover that file, and see whether anything turns red. A
|
|
9
|
+
* mutation the suite does not catch is a **survivor** — a line that runs during the tests
|
|
10
|
+
* and that no assertion actually constrains.
|
|
11
|
+
*
|
|
12
|
+
* Phase 6 of docs/specs/testing-system-redesign-2026-09.md. That phase proposed Stryker as
|
|
13
|
+
* a devDependency; this is the same idea without one, for a specific reason: nearly every
|
|
14
|
+
* test here asserts through a SPAWNED `pan-tools` process, so a mutation-testing framework
|
|
15
|
+
* built around in-process instrumentation would have to re-run whole subprocess suites per
|
|
16
|
+
* mutant anyway. What it would add over this file is a mutant catalogue and an HTML report,
|
|
17
|
+
* at the cost of a large dependency tree in a repo whose headline claim is zero
|
|
18
|
+
* dependencies. So: a sampled probe, dependency-free, and REPORT-ONLY.
|
|
19
|
+
*
|
|
20
|
+
* **This is not a gate and must never become one.** A survivor is a question ("should an
|
|
21
|
+
* assertion pin this?"), and some survivors are correct — equivalent mutants, defensive
|
|
22
|
+
* branches, log strings. Release-check does not run it and CI does not run it.
|
|
23
|
+
*
|
|
24
|
+
* Safety: mutations are applied inside a throwaway `git worktree`, never in your checkout.
|
|
25
|
+
* The worktree is removed at the end, including after a crash.
|
|
26
|
+
*
|
|
27
|
+
* node scripts/mutation-probe.cjs # default targets, 20 mutants
|
|
28
|
+
* node scripts/mutation-probe.cjs --max 50
|
|
29
|
+
* node scripts/mutation-probe.cjs --target hooks/pan-cost-logger.js
|
|
30
|
+
* node scripts/mutation-probe.cjs --seed 7 --json
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
const fs = require('fs');
|
|
34
|
+
const path = require('path');
|
|
35
|
+
const os = require('os');
|
|
36
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
37
|
+
|
|
38
|
+
const ROOT = path.resolve(__dirname, '..');
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* What to probe, and which tests are supposed to catch a break in it. The spec names the
|
|
42
|
+
* hooks and the dispatcher first: the hooks because they are the code that writes the
|
|
43
|
+
* ledger every field number comes from, the dispatcher because its arms are the surface a
|
|
44
|
+
* user and an orchestrator hit.
|
|
45
|
+
*/
|
|
46
|
+
const TARGETS = [
|
|
47
|
+
{ file: 'hooks/pan-cost-logger.js', tests: ['tests/cost-logger-hook.test.cjs', 'tests/cost-logger.test.cjs'] },
|
|
48
|
+
{ file: 'hooks/pan-trace-logger.js', tests: ['tests/trace-logger.test.cjs'] },
|
|
49
|
+
{ file: 'pan-wizard-core/bin/lib/cost.cjs', tests: ['tests/cost.test.cjs'] },
|
|
50
|
+
{ file: 'pan-wizard-core/bin/lib/cost-rebuild.cjs', tests: ['tests/cost-rebuild.test.cjs'] },
|
|
51
|
+
{ file: 'pan-wizard-core/bin/pan-tools.cjs', tests: ['tests/dispatcher.test.cjs', 'tests/dispatcher-arms.test.cjs'] },
|
|
52
|
+
];
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The mutation operators. Each is a plain textual swap with a token that must appear
|
|
56
|
+
* OUTSIDE a string or comment to be worth mutating (see mutableLine). They are deliberately
|
|
57
|
+
* few and blunt: the point is to find unconstrained logic, not to enumerate every possible
|
|
58
|
+
* defect.
|
|
59
|
+
*/
|
|
60
|
+
const OPERATORS = [
|
|
61
|
+
{ id: 'gte→gt', find: ' >= ', replace: ' > ' },
|
|
62
|
+
{ id: 'gt→gte', find: ' > ', replace: ' >= ' },
|
|
63
|
+
{ id: 'lte→lt', find: ' <= ', replace: ' < ' },
|
|
64
|
+
{ id: 'lt→lte', find: ' < ', replace: ' <= ' },
|
|
65
|
+
{ id: 'eq→neq', find: ' === ', replace: ' !== ' },
|
|
66
|
+
{ id: 'neq→eq', find: ' !== ', replace: ' === ' },
|
|
67
|
+
{ id: 'and→or', find: ' && ', replace: ' || ' },
|
|
68
|
+
{ id: 'or→and', find: ' || ', replace: ' && ' },
|
|
69
|
+
{ id: 'true→false', find: 'return true', replace: 'return false' },
|
|
70
|
+
{ id: 'false→true', find: 'return false', replace: 'return true' },
|
|
71
|
+
{ id: 'plus→minus', find: ' + 1', replace: ' - 1' },
|
|
72
|
+
];
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Is this line worth mutating? Skips comments, and skips a line whose only occurrence of
|
|
76
|
+
* the token is inside a string literal — mutating a message changes nothing a test should
|
|
77
|
+
* be pinning, and a survivor there would be noise.
|
|
78
|
+
*
|
|
79
|
+
* Crude but conservative: a line containing a quote is only mutated when the token also
|
|
80
|
+
* occurs before the first quote.
|
|
81
|
+
*/
|
|
82
|
+
function mutableLine(line, token) {
|
|
83
|
+
const trimmed = line.trim();
|
|
84
|
+
if (!trimmed || trimmed.startsWith('//') || trimmed.startsWith('*') || trimmed.startsWith('/*')) return false;
|
|
85
|
+
const at = line.indexOf(token);
|
|
86
|
+
if (at === -1) return false;
|
|
87
|
+
const firstQuote = Math.min(
|
|
88
|
+
...["'", '"', '`'].map((q) => { const i = line.indexOf(q); return i === -1 ? Infinity : i; }),
|
|
89
|
+
);
|
|
90
|
+
return at < firstQuote;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Every mutation available in a source file: one per (line, operator) pair, first
|
|
95
|
+
* occurrence on the line.
|
|
96
|
+
*
|
|
97
|
+
* @returns {Array<{line: number, op: string, before: string, after: string}>}
|
|
98
|
+
*/
|
|
99
|
+
function mutationsFor(src) {
|
|
100
|
+
const lines = src.split(/\r?\n/);
|
|
101
|
+
const out = [];
|
|
102
|
+
lines.forEach((line, i) => {
|
|
103
|
+
for (const op of OPERATORS) {
|
|
104
|
+
if (!mutableLine(line, op.find)) continue;
|
|
105
|
+
out.push({ line: i + 1, op: op.id, before: line, after: line.replace(op.find, op.replace) });
|
|
106
|
+
}
|
|
107
|
+
});
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Apply one mutation to a source string, by line number. Throws if the line moved. */
|
|
112
|
+
function applyMutation(src, mutation) {
|
|
113
|
+
const eol = src.includes('\r\n') ? '\r\n' : '\n';
|
|
114
|
+
const lines = src.split(/\r?\n/);
|
|
115
|
+
const idx = mutation.line - 1;
|
|
116
|
+
if (lines[idx] !== mutation.before) {
|
|
117
|
+
throw new Error(`line ${mutation.line} is not what the mutation was built from`);
|
|
118
|
+
}
|
|
119
|
+
lines[idx] = mutation.after;
|
|
120
|
+
return lines.join(eol);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Deterministic shuffle, so a run is reproducible from its seed. */
|
|
124
|
+
function sample(items, n, seed) {
|
|
125
|
+
let s = seed >>> 0;
|
|
126
|
+
const rand = () => { s = (s * 1664525 + 1013904223) >>> 0; return s / 0x100000000; };
|
|
127
|
+
const copy = items.slice();
|
|
128
|
+
for (let i = copy.length - 1; i > 0; i--) {
|
|
129
|
+
const j = Math.floor(rand() * (i + 1));
|
|
130
|
+
[copy[i], copy[j]] = [copy[j], copy[i]];
|
|
131
|
+
}
|
|
132
|
+
return copy.slice(0, n);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function parseArgs(argv) {
|
|
136
|
+
const a = { max: 20, seed: 1, json: false, targets: [], timeoutMs: 600000, help: false };
|
|
137
|
+
for (let i = 0; i < argv.length; i++) {
|
|
138
|
+
const k = argv[i]; const v = argv[i + 1];
|
|
139
|
+
if (k === '--max') { a.max = Math.max(1, Number(v) || 1); i++; }
|
|
140
|
+
else if (k === '--seed') { a.seed = Number(v) || 1; i++; }
|
|
141
|
+
else if (k === '--target') { a.targets.push(v); i++; }
|
|
142
|
+
else if (k === '--timeout') { a.timeoutMs = Math.max(30000, Number(v) || 30000); i++; }
|
|
143
|
+
else if (k === '--json') a.json = true;
|
|
144
|
+
else if (k === '--help' || k === '-h') a.help = true;
|
|
145
|
+
else throw new Error(`unknown argument: ${k}`);
|
|
146
|
+
}
|
|
147
|
+
return a;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Run one target's tests inside `cwd`. A mutation is CAUGHT when they fail.
|
|
152
|
+
* A test run that cannot start at all counts as caught too — the code is broken enough
|
|
153
|
+
* that nothing ran, which is not a survivor.
|
|
154
|
+
*/
|
|
155
|
+
function testsFail(cwd, tests, timeoutMs) {
|
|
156
|
+
const r = spawnSync(process.execPath, ['--test', ...tests], {
|
|
157
|
+
cwd, encoding: 'utf-8', timeout: timeoutMs, stdio: ['ignore', 'pipe', 'pipe'],
|
|
158
|
+
});
|
|
159
|
+
if (r.error && r.error.code === 'ETIMEDOUT') return true; // an infinite loop is a detection
|
|
160
|
+
return r.status !== 0;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function main() {
|
|
164
|
+
const args = parseArgs(process.argv.slice(2));
|
|
165
|
+
if (args.help) {
|
|
166
|
+
process.stdout.write('node scripts/mutation-probe.cjs [--max n] [--seed n] [--target <file>]... [--timeout ms] [--json]\n');
|
|
167
|
+
return 0;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const targets = args.targets.length
|
|
171
|
+
? TARGETS.filter((t) => args.targets.some((want) => t.file.endsWith(want.replace(/\\/g, '/'))))
|
|
172
|
+
: TARGETS;
|
|
173
|
+
if (!targets.length) {
|
|
174
|
+
process.stderr.write(`no known target matched ${args.targets.join(', ')}\nknown: ${TARGETS.map((t) => t.file).join(', ')}\n`);
|
|
175
|
+
return 1;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Build the candidate list from the real sources, then sample across all targets so one
|
|
179
|
+
// large file cannot crowd the others out.
|
|
180
|
+
const candidates = [];
|
|
181
|
+
for (const t of targets) {
|
|
182
|
+
const src = fs.readFileSync(path.join(ROOT, t.file), 'utf-8');
|
|
183
|
+
for (const m of mutationsFor(src)) candidates.push({ ...m, ...t });
|
|
184
|
+
}
|
|
185
|
+
const chosen = sample(candidates, args.max, args.seed);
|
|
186
|
+
|
|
187
|
+
// Everything happens in a throwaway worktree: the probe never edits your checkout.
|
|
188
|
+
const wt = fs.mkdtempSync(path.join(os.tmpdir(), 'pan-mutation-'));
|
|
189
|
+
let created = false;
|
|
190
|
+
const survivors = [];
|
|
191
|
+
let caught = 0;
|
|
192
|
+
try {
|
|
193
|
+
execFileSync('git', ['worktree', 'add', '--detach', wt, 'HEAD'], { cwd: ROOT, stdio: 'ignore' });
|
|
194
|
+
created = true;
|
|
195
|
+
// The hooks are copied to hooks/dist by the build; tests that spawn the built copy
|
|
196
|
+
// need it present in the worktree.
|
|
197
|
+
spawnSync(process.execPath, ['scripts/build-hooks.js'], { cwd: wt, stdio: 'ignore' });
|
|
198
|
+
|
|
199
|
+
let i = 0;
|
|
200
|
+
for (const m of chosen) {
|
|
201
|
+
i += 1;
|
|
202
|
+
const file = path.join(wt, m.file);
|
|
203
|
+
const original = fs.readFileSync(file, 'utf-8');
|
|
204
|
+
let mutated;
|
|
205
|
+
try {
|
|
206
|
+
mutated = applyMutation(original, m);
|
|
207
|
+
} catch {
|
|
208
|
+
continue; // the worktree's copy differs from the working tree — skip, do not guess
|
|
209
|
+
}
|
|
210
|
+
if (mutated === original) continue;
|
|
211
|
+
try {
|
|
212
|
+
fs.writeFileSync(file, mutated);
|
|
213
|
+
// A hook mutation must reach the built copy the tests spawn.
|
|
214
|
+
if (m.file.startsWith('hooks/')) spawnSync(process.execPath, ['scripts/build-hooks.js'], { cwd: wt, stdio: 'ignore' });
|
|
215
|
+
const detected = testsFail(wt, m.tests, args.timeoutMs);
|
|
216
|
+
if (detected) caught += 1;
|
|
217
|
+
else survivors.push(m);
|
|
218
|
+
if (!args.json) {
|
|
219
|
+
process.stdout.write(` [${String(i).padStart(2)}/${chosen.length}] ${detected ? 'caught ' : 'SURVIVED'} ${m.file}:${m.line} ${m.op}\n`);
|
|
220
|
+
}
|
|
221
|
+
} finally {
|
|
222
|
+
fs.writeFileSync(file, original);
|
|
223
|
+
if (m.file.startsWith('hooks/')) spawnSync(process.execPath, ['scripts/build-hooks.js'], { cwd: wt, stdio: 'ignore' });
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
} finally {
|
|
227
|
+
if (created) {
|
|
228
|
+
try { execFileSync('git', ['worktree', 'remove', '--force', wt], { cwd: ROOT, stdio: 'ignore' }); } catch { /* fall through */ }
|
|
229
|
+
}
|
|
230
|
+
fs.rmSync(wt, { recursive: true, force: true });
|
|
231
|
+
try { execFileSync('git', ['worktree', 'prune'], { cwd: ROOT, stdio: 'ignore' }); } catch { /* best effort */ }
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const ran = caught + survivors.length;
|
|
235
|
+
const report = {
|
|
236
|
+
candidates: candidates.length,
|
|
237
|
+
sampled: chosen.length,
|
|
238
|
+
ran,
|
|
239
|
+
caught,
|
|
240
|
+
survived: survivors.length,
|
|
241
|
+
score: ran ? Number(((caught / ran) * 100).toFixed(1)) : null,
|
|
242
|
+
seed: args.seed,
|
|
243
|
+
survivors: survivors.map((s) => ({ file: s.file, line: s.line, op: s.op, code: s.before.trim().slice(0, 120) })),
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
if (args.json) {
|
|
247
|
+
process.stdout.write(JSON.stringify(report, null, 2) + '\n');
|
|
248
|
+
return 0;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
process.stdout.write(`\nmutation probe — REPORT ONLY, never a gate\n`);
|
|
252
|
+
process.stdout.write(` ${report.caught} caught / ${report.ran} run (${report.score === null ? 'n/a' : report.score + '%'}), from ${report.candidates} candidates, seed ${report.seed}\n`);
|
|
253
|
+
if (survivors.length) {
|
|
254
|
+
process.stdout.write(`\n survivors — the suite ran this line and no assertion constrained it:\n`);
|
|
255
|
+
for (const s of report.survivors) process.stdout.write(` ${s.file}:${s.line} ${s.op}\n ${s.code}\n`);
|
|
256
|
+
process.stdout.write('\n Each is a question, not a verdict: some are equivalent mutants or defensive\n branches where no assertion is owed. Pin the ones that describe real behaviour.\n');
|
|
257
|
+
} else {
|
|
258
|
+
process.stdout.write('\n no survivors in this sample\n');
|
|
259
|
+
}
|
|
260
|
+
return 0;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
module.exports = { mutationsFor, applyMutation, mutableLine, sample, OPERATORS, TARGETS };
|
|
264
|
+
|
|
265
|
+
if (require.main === module) {
|
|
266
|
+
try {
|
|
267
|
+
process.exit(main());
|
|
268
|
+
} catch (e) {
|
|
269
|
+
process.stderr.write(`mutation probe failed: ${e.message}\n`);
|
|
270
|
+
process.exit(1);
|
|
271
|
+
}
|
|
272
|
+
}
|
package/scripts/release-check.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* release-check.js — Pre-publish validation gate.
|
|
4
4
|
*
|
|
5
5
|
* Wired into `prepublishOnly` so `npm publish` fails BEFORE upload if any
|
|
6
|
-
* gate is red. Runs
|
|
6
|
+
* gate is red. Runs nine checks in order; first failure aborts.
|
|
7
7
|
*
|
|
8
8
|
* 1. build:hooks — hook scripts copy/build cleanly
|
|
9
9
|
* 2. test:all — full test suite (unit + scenario) passes
|
|
@@ -11,9 +11,13 @@
|
|
|
11
11
|
* (we have zero runtime deps, but the dev-deps are checked)
|
|
12
12
|
* 4. doc-lint counts — no drift-prone count violations in user-facing docs
|
|
13
13
|
* 5. links validate — doc↔code link graph resolves (no broken references)
|
|
14
|
-
* 6. npm pack dry-run — package builds; size is sane
|
|
14
|
+
* 6. npm pack dry-run — package builds; size is sane; zero runtime dependencies
|
|
15
15
|
* 7. smoke install — npm pack + install into temp dir + run pan-tools list
|
|
16
16
|
* catches "ships but doesn't actually work" failures
|
|
17
|
+
* 8. bundles — both plugin builders build; dist/pan-agent-plugin is fresh
|
|
18
|
+
* 9. coverage gate — the suite under Node's coverage: every dispatcher arm
|
|
19
|
+
* executed, line/function floors per module group
|
|
20
|
+
* (scripts/coverage-gate.cjs; skipped on Node < 22)
|
|
17
21
|
*
|
|
18
22
|
* Usage:
|
|
19
23
|
* node scripts/release-check.js # all gates
|
|
@@ -57,7 +61,7 @@ function run(cmd, args, opts = {}) {
|
|
|
57
61
|
}
|
|
58
62
|
|
|
59
63
|
// Gate 1: build:hooks
|
|
60
|
-
process.stderr.write('\n[release-check] Gate 1/
|
|
64
|
+
process.stderr.write('\n[release-check] Gate 1/9: build:hooks\n');
|
|
61
65
|
{
|
|
62
66
|
const r = run('npm', ['run', 'build:hooks']);
|
|
63
67
|
logGate('build:hooks', r.status === 0, r.status !== 0 ? `exit ${r.status}` : '');
|
|
@@ -65,7 +69,7 @@ process.stderr.write('\n[release-check] Gate 1/8: build:hooks\n');
|
|
|
65
69
|
}
|
|
66
70
|
|
|
67
71
|
// Gate 2: test:all
|
|
68
|
-
process.stderr.write('\n[release-check] Gate 2/
|
|
72
|
+
process.stderr.write('\n[release-check] Gate 2/9: test:all\n');
|
|
69
73
|
{
|
|
70
74
|
const r = run('npm', ['run', 'test:all']);
|
|
71
75
|
logGate('test:all', r.status === 0, r.status !== 0 ? `exit ${r.status}` : '');
|
|
@@ -74,9 +78,9 @@ process.stderr.write('\n[release-check] Gate 2/8: test:all\n');
|
|
|
74
78
|
|
|
75
79
|
// Gate 3: npm audit (production deps only)
|
|
76
80
|
if (SKIP_AUDIT) {
|
|
77
|
-
process.stderr.write('\n[release-check] Gate 3/
|
|
81
|
+
process.stderr.write('\n[release-check] Gate 3/9: npm audit (SKIPPED)\n');
|
|
78
82
|
} else {
|
|
79
|
-
process.stderr.write('\n[release-check] Gate 3/
|
|
83
|
+
process.stderr.write('\n[release-check] Gate 3/9: npm audit --omit=dev\n');
|
|
80
84
|
const r = run('npm', ['audit', '--omit=dev', '--audit-level=high'], { capture: true });
|
|
81
85
|
// npm audit exits non-zero on findings. We tolerate moderate; fail on high+.
|
|
82
86
|
const ok = r.status === 0;
|
|
@@ -88,7 +92,7 @@ if (SKIP_AUDIT) {
|
|
|
88
92
|
}
|
|
89
93
|
|
|
90
94
|
// Gate 4: doc-lint counts on user-facing docs (count-SSoT enforcement)
|
|
91
|
-
process.stderr.write('\n[release-check] Gate 4/
|
|
95
|
+
process.stderr.write('\n[release-check] Gate 4/9: doc-lint counts docs/\n');
|
|
92
96
|
{
|
|
93
97
|
const tools = path.join(REPO_ROOT, 'pan-wizard-core', 'bin', 'pan-tools.cjs');
|
|
94
98
|
const docsDir = path.join(REPO_ROOT, 'docs');
|
|
@@ -103,7 +107,7 @@ process.stderr.write('\n[release-check] Gate 4/8: doc-lint counts docs/\n');
|
|
|
103
107
|
|
|
104
108
|
// Gate 5: doc↔code link graph resolves (anti-fake — a doc cannot reference a
|
|
105
109
|
// code anchor that doesn't exist; deterministic, self-enforcing exit 1).
|
|
106
|
-
process.stderr.write('\n[release-check] Gate 5/
|
|
110
|
+
process.stderr.write('\n[release-check] Gate 5/9: links validate\n');
|
|
107
111
|
{
|
|
108
112
|
const tools = path.join(REPO_ROOT, 'pan-wizard-core', 'bin', 'pan-tools.cjs');
|
|
109
113
|
const r = run('node', [tools, 'links', 'validate', '--raw'], { capture: true });
|
|
@@ -125,7 +129,7 @@ process.stderr.write('\n[release-check] Gate 5/8: links validate\n');
|
|
|
125
129
|
|
|
126
130
|
// Gate 6: npm pack — produces a non-empty, sanely-sized tarball (read the file,
|
|
127
131
|
// never parse stdout)
|
|
128
|
-
process.stderr.write('\n[release-check] Gate 6/
|
|
132
|
+
process.stderr.write('\n[release-check] Gate 6/9: npm pack (size sanity)\n');
|
|
129
133
|
{
|
|
130
134
|
const tmp6 = fs.mkdtempSync(path.join(os.tmpdir(), 'pan-release-pack-'));
|
|
131
135
|
const r = run('npm', ['pack', '--pack-destination', tmp6], { capture: true });
|
|
@@ -149,9 +153,9 @@ process.stderr.write('\n[release-check] Gate 6/8: npm pack (size sanity)\n');
|
|
|
149
153
|
|
|
150
154
|
// Gate 7: smoke install — pack and install into temp dir, run pan-tools
|
|
151
155
|
if (SKIP_SMOKE) {
|
|
152
|
-
process.stderr.write('\n[release-check] Gate 7/
|
|
156
|
+
process.stderr.write('\n[release-check] Gate 7/9: smoke install (SKIPPED)\n');
|
|
153
157
|
} else {
|
|
154
|
-
process.stderr.write('\n[release-check] Gate 7/
|
|
158
|
+
process.stderr.write('\n[release-check] Gate 7/9: smoke install (npm pack + install + sanity)\n');
|
|
155
159
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pan-release-smoke-'));
|
|
156
160
|
try {
|
|
157
161
|
// Pack — read the .tgz npm writes to tmpDir; never parse its stdout (see note).
|
|
@@ -201,7 +205,7 @@ if (SKIP_SMOKE) {
|
|
|
201
205
|
// temp dirs (never dist/) so the gate cannot race a concurrently running test and
|
|
202
206
|
// leaves the checkout untouched. A bundle that fails to build is a release that
|
|
203
207
|
// ships a broken marketplace entry.
|
|
204
|
-
process.stderr.write('\n[release-check] Gate 8/
|
|
208
|
+
process.stderr.write('\n[release-check] Gate 8/9: distribution bundles (build:plugin + build:agent-plugin)\n');
|
|
205
209
|
{
|
|
206
210
|
const tmp8 = fs.mkdtempSync(path.join(os.tmpdir(), 'pan-release-bundles-'));
|
|
207
211
|
try {
|
|
@@ -238,6 +242,23 @@ process.stderr.write('\n[release-check] Gate 8/8: distribution bundles (build:pl
|
|
|
238
242
|
}
|
|
239
243
|
}
|
|
240
244
|
|
|
245
|
+
// Gate 9: coverage gate — the whole suite once more, under Node's own coverage
|
|
246
|
+
// instrumentation, then: every dispatcher case arm executed (or allowlisted with a
|
|
247
|
+
// reason in tests/fixtures/coverage-policy.json) and line/function floors per module
|
|
248
|
+
// group. Gate 2 says the tests pass; this gate says the shipped code ran. On Node
|
|
249
|
+
// below 22 the script reports "skipped" and exits 0 — the CI Node-22 job carries it.
|
|
250
|
+
process.stderr.write('\n[release-check] Gate 9/9: coverage gate (dispatcher arms + coverage floors)\n');
|
|
251
|
+
{
|
|
252
|
+
const r = run('node', [path.join(REPO_ROOT, 'scripts', 'coverage-gate.cjs')], { capture: true });
|
|
253
|
+
const text = ((r.stdout || '') + (r.stderr || '')).trim();
|
|
254
|
+
const first = text.split('\n')[0] || '';
|
|
255
|
+
logGate('coverage gate', r.status === 0, first.replace(/^coverage gate — /, ''));
|
|
256
|
+
if (r.status !== 0) {
|
|
257
|
+
process.stderr.write(text + '\n');
|
|
258
|
+
process.exit(1);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
241
262
|
// Summary
|
|
242
263
|
process.stderr.write('\n[release-check] Summary:\n');
|
|
243
264
|
for (const c of checks) {
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* test-quality-lint.cjs — the assertion shapes that proved vacuous, as lint rules.
|
|
4
|
+
*
|
|
5
|
+
* Each rule came from a test that passed while the feature it named was broken
|
|
6
|
+
* (audit of 2026-09-17, spec docs/specs/testing-system-redesign-2026-09.md §3.1):
|
|
7
|
+
* Q1 an OR-shaped liveness assert — `assert.ok(a.length > 0 || b.length > 0)` is
|
|
8
|
+
* satisfied by a crash with a stack trace;
|
|
9
|
+
* Q2 an in-process call to a lib module's `cmd*` function — they end in output()/
|
|
10
|
+
* error(), which exit the process, so the test child dies and node --test
|
|
11
|
+
* reports the file as one passing test;
|
|
12
|
+
* Q3 `assert.ok(true)` / `assert(true)`;
|
|
13
|
+
* Q4 a runPanTools result asserted only by `.output.length`;
|
|
14
|
+
* Q5 a platform conditional that bare-`return`s (counted as a pass) instead of
|
|
15
|
+
* `t.skip(reason)`;
|
|
16
|
+
* Q6 a wall-clock upper bound under two seconds on a spawned process;
|
|
17
|
+
* Q7 a read of the developer's real HOME / USERPROFILE / os.homedir();
|
|
18
|
+
* Q8 a committed `test.todo` (scaffold stubs must be filled before commit);
|
|
19
|
+
* Q9 an OR of bare property reads inside assert.ok — `assert.ok(a.x || a.y)` asks only
|
|
20
|
+
* whether one of them exists, which an empty object, the wrong field, or a payload
|
|
21
|
+
* that means failure all satisfy. Q1 covers the result-status fields; this covers
|
|
22
|
+
* the same vacuity for arbitrary ones.
|
|
23
|
+
*
|
|
24
|
+
* `lintTestSource(src, file)` is pure; tests/test-quality.test.cjs applies it to the
|
|
25
|
+
* suite with tests/fixtures/test-quality-allowlist.json (entries { file, rule,
|
|
26
|
+
* count, reason } — allowed occurrences per file and rule; an entry that allows more
|
|
27
|
+
* than the file has is stale and fails too).
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
function stripComments(line) {
|
|
31
|
+
// Good enough for test sources: drop a trailing // comment that is not inside
|
|
32
|
+
// quotes, and ignore the body lines of a /** … */ block comment.
|
|
33
|
+
if (/^\s*(\*|\/\*)/.test(line)) return '';
|
|
34
|
+
let inS = null;
|
|
35
|
+
for (let i = 0; i < line.length; i++) {
|
|
36
|
+
const c = line[i];
|
|
37
|
+
if (inS) { if (c === '\\') i++; else if (c === inS) inS = null; continue; }
|
|
38
|
+
if (c === '\'' || c === '"' || c === '`') inS = c;
|
|
39
|
+
else if (c === '/' && line[i + 1] === '/') return line.slice(0, i);
|
|
40
|
+
}
|
|
41
|
+
return line;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Join an assert statement that spans lines until its parentheses balance (max 8 lines). */
|
|
45
|
+
function statementAt(lines, i) {
|
|
46
|
+
let text = lines[i];
|
|
47
|
+
let depth = 0;
|
|
48
|
+
for (let j = i; j < Math.min(lines.length, i + 8); j++) {
|
|
49
|
+
const t = stripComments(lines[j]);
|
|
50
|
+
if (j > i) text += ' ' + t.trim();
|
|
51
|
+
for (const c of t) { if (c === '(') depth++; else if (c === ')') depth--; }
|
|
52
|
+
if (depth <= 0 && j > i) break;
|
|
53
|
+
if (depth <= 0 && j === i && /\)\s*;?\s*$/.test(t)) break;
|
|
54
|
+
}
|
|
55
|
+
return text;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const TIME_WORDS = /\b(ms|elapsed|duration|took|Date\.now|hrtime|performance\.now|timing)\b/;
|
|
59
|
+
|
|
60
|
+
// The result-status fields whose bare truthiness a liveness assert ORs together:
|
|
61
|
+
// `result.output || result.error`, `!r.success || r.error`, `parsed.error || parsed.state`.
|
|
62
|
+
// A crash satisfies every one of them — stderr is non-empty. An OR between two
|
|
63
|
+
// content checks (`x.includes('a') || x.includes('b')`) is a legitimate either-format
|
|
64
|
+
// assert and is not this rule's business.
|
|
65
|
+
const STATUS_FIELD = /^!?\(?\s*[\w$.]*\b(success|error|output|stderr|stdout|reason|state|status)\b\s*\)?$/;
|
|
66
|
+
const LENGTH_LIVENESS = /^!?\(?\s*[\w$.]*\.(?:output|error|stderr|stdout)\.length\s*>\s*0\s*\)?$/;
|
|
67
|
+
|
|
68
|
+
function isLivenessOr(inner) {
|
|
69
|
+
const operands = inner.split(/\|\|/).map((s) => s.trim());
|
|
70
|
+
if (operands.length < 2) return false;
|
|
71
|
+
const bare = operands.filter((o) => STATUS_FIELD.test(o) || LENGTH_LIVENESS.test(o));
|
|
72
|
+
return bare.length >= 2;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const RULES = [
|
|
76
|
+
{
|
|
77
|
+
id: 'Q1',
|
|
78
|
+
title: 'OR-shaped liveness assert',
|
|
79
|
+
fix: 'assert the exit code and parse the JSON payload (or assert each branch on its own)',
|
|
80
|
+
detect(lines) {
|
|
81
|
+
const out = [];
|
|
82
|
+
lines.forEach((raw, i) => {
|
|
83
|
+
const line = stripComments(raw);
|
|
84
|
+
if (!/\bassert(?:\.ok)?\s*\(/.test(line)) return;
|
|
85
|
+
const stmt = statementAt(lines, i);
|
|
86
|
+
// The asserted expression: everything before the message argument.
|
|
87
|
+
let inner = stmt.slice(stmt.indexOf('(') + 1).replace(/\)\s*;?\s*$/, '');
|
|
88
|
+
const msg = inner.search(/,\s*(['"`])/);
|
|
89
|
+
if (msg >= 0) inner = inner.slice(0, msg);
|
|
90
|
+
if (isLivenessOr(inner)) out.push({ line: i + 1, text: raw.trim() });
|
|
91
|
+
});
|
|
92
|
+
return out;
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
id: 'Q2',
|
|
97
|
+
title: 'in-process call to a lib cmd* function',
|
|
98
|
+
fix: 'go through runPanTools (or the module’s pure functions) — cmd* functions end in output()/error(), which exit the process',
|
|
99
|
+
detect(lines) {
|
|
100
|
+
const out = [];
|
|
101
|
+
lines.forEach((raw, i) => {
|
|
102
|
+
const line = stripComments(raw);
|
|
103
|
+
if (/\bcmd[A-Z][A-Za-z0-9]*\s*\(/.test(line) && !/^\s*(const|let|var)\s+\{/.test(line)) out.push({ line: i + 1, text: raw.trim() });
|
|
104
|
+
});
|
|
105
|
+
return out;
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
id: 'Q3',
|
|
110
|
+
title: 'assert(true)',
|
|
111
|
+
fix: 'assert a specific value',
|
|
112
|
+
detect(lines) {
|
|
113
|
+
const out = [];
|
|
114
|
+
lines.forEach((raw, i) => { if (/\bassert(?:\.ok)?\(\s*true\s*[,)]/.test(stripComments(raw))) out.push({ line: i + 1, text: raw.trim() }); });
|
|
115
|
+
return out;
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
id: 'Q4',
|
|
120
|
+
title: 'CLI output asserted only by its length',
|
|
121
|
+
fix: 'parse the JSON payload and assert its fields; assert success/exit code',
|
|
122
|
+
detect(lines) {
|
|
123
|
+
const out = [];
|
|
124
|
+
lines.forEach((raw, i) => { if (/assert\.ok\(\s*[A-Za-z_$][\w$.]*\.output\.length\s*[><!=]/.test(stripComments(raw))) out.push({ line: i + 1, text: raw.trim() }); });
|
|
125
|
+
return out;
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
{
|
|
129
|
+
id: 'Q5',
|
|
130
|
+
title: 'platform conditional that bare-returns',
|
|
131
|
+
fix: 'use t.skip(reason) so the skip is reported, never counted as a pass',
|
|
132
|
+
detect(lines) {
|
|
133
|
+
const out = [];
|
|
134
|
+
lines.forEach((raw, i) => {
|
|
135
|
+
const line = stripComments(raw);
|
|
136
|
+
if (!/process\.platform/.test(line) || !/\bif\s*\(/.test(line)) return;
|
|
137
|
+
const window = [line, lines[i + 1] || '', lines[i + 2] || ''].map(stripComments).join(' ');
|
|
138
|
+
if (/\bskip\s*\(/.test(window) || /\btodo\s*\(/.test(window)) return;
|
|
139
|
+
if (/\)\s*\{?\s*return\b/.test(line) || /^\s*return\s*;?\s*\}?\s*$/.test(stripComments(lines[i + 1] || ''))) out.push({ line: i + 1, text: raw.trim() });
|
|
140
|
+
});
|
|
141
|
+
return out;
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
id: 'Q6',
|
|
146
|
+
title: 'wall-clock upper bound under two seconds',
|
|
147
|
+
fix: 'assert the behaviour, not the speed; if timing is the contract, bound it generously and allowlist with the reason',
|
|
148
|
+
detect(lines) {
|
|
149
|
+
const out = [];
|
|
150
|
+
lines.forEach((raw, i) => {
|
|
151
|
+
const line = stripComments(raw);
|
|
152
|
+
if (!/\bassert(?:\.ok)?\s*\(/.test(line) || !TIME_WORDS.test(line)) return;
|
|
153
|
+
const m = /<\s*=?\s*(\d{2,4})\b/.exec(line);
|
|
154
|
+
if (m && Number(m[1]) < 2000) out.push({ line: i + 1, text: raw.trim() });
|
|
155
|
+
});
|
|
156
|
+
return out;
|
|
157
|
+
},
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
id: 'Q7',
|
|
161
|
+
title: 'read of the real HOME',
|
|
162
|
+
fix: 'sandbox with withFakeHome() from tests/helpers.cjs, or pass an explicit config dir',
|
|
163
|
+
detect(lines) {
|
|
164
|
+
const out = [];
|
|
165
|
+
lines.forEach((raw, i) => {
|
|
166
|
+
const line = stripComments(raw);
|
|
167
|
+
if (/os\.homedir\(\)|process\.env\.(?:HOME|USERPROFILE)\b(?!\s*=[^=])/.test(line)) out.push({ line: i + 1, text: raw.trim() });
|
|
168
|
+
});
|
|
169
|
+
return out;
|
|
170
|
+
},
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
id: 'Q8',
|
|
174
|
+
title: 'committed todo',
|
|
175
|
+
fix: 'fill the stub or delete it; todo stubs are scaffold output, not tests',
|
|
176
|
+
detect(lines) {
|
|
177
|
+
const out = [];
|
|
178
|
+
lines.forEach((raw, i) => { if (/\b(?:test|it|describe)\.todo\s*\(|\{\s*todo\s*:/.test(stripComments(raw))) out.push({ line: i + 1, text: raw.trim() }); });
|
|
179
|
+
return out;
|
|
180
|
+
},
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
id: 'Q9',
|
|
184
|
+
title: 'OR of bare property reads in assert.ok',
|
|
185
|
+
fix: 'assert the specific field and its expected value, not that one of several exists',
|
|
186
|
+
detect(lines) {
|
|
187
|
+
const out = [];
|
|
188
|
+
lines.forEach((raw, i) => {
|
|
189
|
+
const line = stripComments(raw);
|
|
190
|
+
const m = line.match(/assert\.ok\(\s*([^;]*?)\s*(?:,\s*['"`][^;]*)?\)\s*;/);
|
|
191
|
+
if (!m || !m[1].includes('||')) return;
|
|
192
|
+
const expr = m[1];
|
|
193
|
+
// A comparison, a call, a negation or a length check is making a claim about a
|
|
194
|
+
// value; only a bare existence test is vacuous in this way.
|
|
195
|
+
if (/[=<>]|\(|\)|!|\.length|typeof|Array\.isArray/.test(expr)) return;
|
|
196
|
+
const operands = expr.split('||').map((s) => s.trim());
|
|
197
|
+
if (operands.length < 2) return;
|
|
198
|
+
if (!operands.every((o) => /^[A-Za-z_$][\w$]*(\.[A-Za-z_$][\w$]*)+$/.test(o))) return;
|
|
199
|
+
out.push({ line: i + 1, text: raw.trim() });
|
|
200
|
+
});
|
|
201
|
+
return out;
|
|
202
|
+
},
|
|
203
|
+
},
|
|
204
|
+
];
|
|
205
|
+
|
|
206
|
+
/** Findings for one test source: [{ rule, title, fix, line, text }]. */
|
|
207
|
+
function lintTestSource(src, file = '') {
|
|
208
|
+
const lines = src.split(/\r?\n/);
|
|
209
|
+
const findings = [];
|
|
210
|
+
for (const rule of RULES) {
|
|
211
|
+
for (const f of rule.detect(lines)) findings.push({ rule: rule.id, title: rule.title, fix: rule.fix, file, line: f.line, text: f.text });
|
|
212
|
+
}
|
|
213
|
+
return findings;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Apply an allowlist ([{ file, rule, count, reason }]) to findings.
|
|
218
|
+
* Returns { remaining, stale } — stale entries allow more than the file has, or
|
|
219
|
+
* carry no reason. Both fail the suite: an allowlist is a debt register, not a bin.
|
|
220
|
+
*/
|
|
221
|
+
function applyAllowlist(findings, allowlist) {
|
|
222
|
+
const remaining = [];
|
|
223
|
+
const stale = [];
|
|
224
|
+
const byKey = new Map();
|
|
225
|
+
for (const f of findings) { const k = `${f.file}|${f.rule}`; byKey.set(k, (byKey.get(k) || []).concat(f)); }
|
|
226
|
+
const seen = new Set();
|
|
227
|
+
for (const entry of allowlist || []) {
|
|
228
|
+
const k = `${entry.file}|${entry.rule}`;
|
|
229
|
+
seen.add(k);
|
|
230
|
+
const have = (byKey.get(k) || []).length;
|
|
231
|
+
if (!entry.reason || !String(entry.reason).trim()) stale.push({ ...entry, why: 'no reason given' });
|
|
232
|
+
else if (have === 0) stale.push({ ...entry, why: 'file has no such finding any more — remove the entry' });
|
|
233
|
+
else if (have < (entry.count || 0)) stale.push({ ...entry, why: `allows ${entry.count} but the file has ${have} — lower the count` });
|
|
234
|
+
else if (have > (entry.count || 0)) remaining.push(...(byKey.get(k) || []).slice(entry.count || 0));
|
|
235
|
+
}
|
|
236
|
+
for (const [k, list] of byKey) if (!seen.has(k)) remaining.push(...list);
|
|
237
|
+
return { remaining, stale };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
module.exports = { RULES, lintTestSource, applyAllowlist, stripComments };
|