pan-wizard 3.35.0 → 3.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -5
- package/agents/pan-designer.md +1 -1
- package/agents/pan-executor.md +25 -0
- package/agents/pan-hardener.md +2 -0
- package/agents/pan-phase-researcher.md +14 -0
- package/agents/pan-plan-checker.md +28 -1
- package/agents/pan-planner.md +11 -1
- package/agents/pan-previewer.md +2 -0
- package/agents/pan-project-researcher.md +8 -0
- package/agents/pan-reviewer.md +5 -0
- package/agents/pan-verifier.md +2 -2
- package/commands/pan/cost.md +21 -1
- package/commands/pan/preview.md +1 -1
- package/package.json +1 -1
- package/pan-wizard-core/bin/lib/config.cjs +10 -3
- package/pan-wizard-core/bin/lib/core.cjs +9 -4
- package/pan-wizard-core/bin/lib/cost.cjs +146 -0
- package/pan-wizard-core/bin/lib/doc-lint.cjs +5 -3
- package/pan-wizard-core/bin/lib/findings.cjs +115 -6
- package/pan-wizard-core/bin/lib/git.cjs +1 -1
- package/pan-wizard-core/bin/lib/mutate.cjs +257 -0
- package/pan-wizard-core/bin/lib/phase-gate.cjs +15 -9
- package/pan-wizard-core/bin/lib/phase-remove.cjs +91 -11
- package/pan-wizard-core/bin/lib/phase.cjs +37 -9
- package/pan-wizard-core/bin/lib/plan-lint.cjs +161 -0
- package/pan-wizard-core/bin/lib/preview.cjs +3 -0
- package/pan-wizard-core/bin/lib/research-lint.cjs +121 -0
- package/pan-wizard-core/bin/lib/roadmap.cjs +1 -0
- package/pan-wizard-core/bin/lib/test-gate.cjs +433 -0
- package/pan-wizard-core/bin/lib/usage.cjs +104 -0
- package/pan-wizard-core/bin/lib/verify-plan-set.cjs +1 -1
- package/pan-wizard-core/bin/lib/verify-red.cjs +138 -0
- package/pan-wizard-core/bin/lib/verify-scope.cjs +143 -2
- package/pan-wizard-core/bin/lib/verify.cjs +27 -27
- package/pan-wizard-core/bin/pan-tools.cjs +94 -6
- package/pan-wizard-core/mcp/orchestrator.cjs +4 -1
- package/pan-wizard-core/references/design-methodology.md +1 -1
- package/pan-wizard-core/references/handoff-decisions.md +1 -1
- package/pan-wizard-core/templates/config.json +5 -1
- package/pan-wizard-core/templates/design.md +5 -4
- package/pan-wizard-core/templates/phase-prompt.md +7 -0
- package/pan-wizard-core/templates/research-project/architecture.md +6 -0
- package/pan-wizard-core/templates/research-project/features.md +6 -0
- package/pan-wizard-core/templates/research-project/pitfalls.md +6 -0
- package/pan-wizard-core/templates/research-project/stack.md +6 -0
- package/pan-wizard-core/templates/research.md +10 -0
- package/pan-wizard-core/templates/summary-complex.md +5 -1
- package/pan-wizard-core/templates/summary-minimal.md +4 -0
- package/pan-wizard-core/templates/summary-standard.md +5 -1
- package/pan-wizard-core/templates/summary.md +8 -1
- package/pan-wizard-core/workflows/exec-phase.md +43 -7
- package/pan-wizard-core/workflows/execute-plan.md +3 -1
- package/pan-wizard-core/workflows/help.md +103 -44
- package/pan-wizard-core/workflows/plan-phase.md +20 -2
- package/pan-wizard-core/workflows/remove-phase.md +11 -1
- package/pan-wizard-core/workflows/settings.md +17 -0
- package/pan-wizard-core/workflows/verify-phase.md +39 -46
- package/scripts/build-help.cjs +106 -0
- package/scripts/mutation-probe.cjs +7 -80
- package/scripts/test-surface.cjs +168 -6
|
@@ -27,6 +27,12 @@
|
|
|
27
27
|
* - IDEMPOTENCY: recording the same artifact twice is a no-op that returns the
|
|
28
28
|
* first verdict (record_sig). That is why exec-phase and verify-phase may both
|
|
29
29
|
* record one verification.md.
|
|
30
|
+
* - LEARNED SUPPRESSIONS (R3): a dismissed finding is noise for its file as it was
|
|
31
|
+
* when dismissed. The same judge raising it again (same signature, sameFinding)
|
|
32
|
+
* is filtered, in the dismissal's phase (candidate) or, once dismissed in two
|
|
33
|
+
* phases, in any phase (promoted). When the file changes the suppression retires
|
|
34
|
+
* and the finding comes back open. Security and logic findings, and findings
|
|
35
|
+
* with no file, are never suppressed: a re-report reopens them.
|
|
30
36
|
*/
|
|
31
37
|
const fs = require('fs');
|
|
32
38
|
const path = require('path');
|
|
@@ -40,6 +46,11 @@ const LEDGER_V = 1;
|
|
|
40
46
|
const REASON_MAX = 300;
|
|
41
47
|
const FINDING_ID_RE = /^f_[0-9a-f]{10}$/;
|
|
42
48
|
const STATUSES = Object.freeze(['open', ...V.DISPOSITIONS]);
|
|
49
|
+
// Logic findings are never suppressed (R3): what the code does wrong, or not at all.
|
|
50
|
+
const LOGIC_CLASSES = Object.freeze(['defect', 'contradicts', 'missing', 'partial']);
|
|
51
|
+
// Security findings are never suppressed either, whatever class a judge gave them.
|
|
52
|
+
const SECURITY_RE = /secur|inject|xss|csrf|ssrf|secret|credential|password|authenticat|authoriz|\bauth\b|token|crypt|traversal|privilege|sanitiz|escap|cve-|owasp|\brce\b|deserializ/i;
|
|
53
|
+
const SECURITY_AGENTS = Object.freeze(['pan-hardener']);
|
|
43
54
|
|
|
44
55
|
function ledgerPath(cwd) {
|
|
45
56
|
return planningPath(cwd, FINDINGS_FILE);
|
|
@@ -80,7 +91,7 @@ function foldFindings(rows) {
|
|
|
80
91
|
id: r.id, phase: r.phase, milestone: r.milestone || null, agent: r.agent, class: r.class,
|
|
81
92
|
severity: r.severity, where: r.where || null, summary: r.summary, first_seen: r.ts,
|
|
82
93
|
first_verdict_id: r.verdict_id || null, reports: 0, last_reported: null, _reportedSeq: -1,
|
|
83
|
-
disposition: null, reason: null, disposed_at: null, auto: false, _disposedSeq: -1,
|
|
94
|
+
disposition: null, reason: null, disposed_at: null, auto: false, file_sha: null, _disposedSeq: -1, _returnedSeq: -1,
|
|
84
95
|
});
|
|
85
96
|
} else if (r.kind === 'verdict') {
|
|
86
97
|
verdicts.push({ ...r, _seq: seq });
|
|
@@ -91,6 +102,11 @@ function foldFindings(rows) {
|
|
|
91
102
|
f.last_reported = r.ts;
|
|
92
103
|
f._reportedSeq = seq;
|
|
93
104
|
}
|
|
105
|
+
// A dismissed finding raised again once its suppression no longer holds (R3).
|
|
106
|
+
for (const id of Array.isArray(r.returned_ids) ? r.returned_ids : []) {
|
|
107
|
+
const f = findings.get(id);
|
|
108
|
+
if (f) f._returnedSeq = seq;
|
|
109
|
+
}
|
|
94
110
|
} else if (r.kind === 'disposition' && typeof r.id === 'string') {
|
|
95
111
|
const f = findings.get(r.id);
|
|
96
112
|
if (!f || !V.DISPOSITIONS.includes(r.disposition)) return;
|
|
@@ -98,12 +114,14 @@ function foldFindings(rows) {
|
|
|
98
114
|
f.reason = r.reason || null;
|
|
99
115
|
f.disposed_at = r.ts;
|
|
100
116
|
f.auto = r.auto === true;
|
|
117
|
+
f.file_sha = r.file_sha || null;
|
|
101
118
|
f._disposedSeq = seq;
|
|
102
119
|
}
|
|
103
120
|
});
|
|
104
121
|
for (const f of findings.values()) {
|
|
105
122
|
if (!f.disposition) f.status = 'open';
|
|
106
123
|
else if (f.disposition === 'fixed' && f._reportedSeq > f._disposedSeq) f.status = 'open';
|
|
124
|
+
else if (f._returnedSeq > f._disposedSeq) f.status = 'open';
|
|
107
125
|
else f.status = f.disposition;
|
|
108
126
|
}
|
|
109
127
|
return { verdicts, findings };
|
|
@@ -111,7 +129,7 @@ function foldFindings(rows) {
|
|
|
111
129
|
|
|
112
130
|
/** A finding as the CLI shows it (internal sequence fields removed). */
|
|
113
131
|
function publicFinding(f) {
|
|
114
|
-
const { _reportedSeq, _disposedSeq, ...rest } = f;
|
|
132
|
+
const { _reportedSeq, _disposedSeq, _returnedSeq, ...rest } = f;
|
|
115
133
|
return rest;
|
|
116
134
|
}
|
|
117
135
|
|
|
@@ -168,6 +186,52 @@ function truncateReason(s) {
|
|
|
168
186
|
return String(s).replace(/\s+/g, ' ').trim().slice(0, REASON_MAX);
|
|
169
187
|
}
|
|
170
188
|
|
|
189
|
+
// ─── Learned suppressions (R3) ──────────────────────────────────────────────
|
|
190
|
+
|
|
191
|
+
/** Security and logic findings are always reported, dismissed or not. */
|
|
192
|
+
function neverSuppressed(f, agent) {
|
|
193
|
+
return LOGIC_CLASSES.includes(f.class) || SECURITY_AGENTS.includes(agent) || SECURITY_RE.test(String(f.summary || ''));
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** The content hash of the project file a `where` names, or null (no file, outside the project, unreadable). */
|
|
197
|
+
function fileSha(cwd, where) {
|
|
198
|
+
const file = fileOf(where);
|
|
199
|
+
if (!file || file.startsWith('code-fact:')) return null;
|
|
200
|
+
const abs = path.resolve(cwd, file);
|
|
201
|
+
const rel = path.relative(path.resolve(cwd), abs);
|
|
202
|
+
if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return null;
|
|
203
|
+
try { return crypto.createHash('sha256').update(fs.readFileSync(abs)).digest('hex'); } catch { return null; }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Every learned suppression: a dismissed finding scoped to a file, of a kind that may
|
|
208
|
+
* be suppressed. `active` while the file is as it was when dismissed; `promoted` once
|
|
209
|
+
* the same judge dismissed the same signature in two phases (active ones only).
|
|
210
|
+
*/
|
|
211
|
+
function learnedSuppressions(cwd, fold) {
|
|
212
|
+
const shas = new Map();
|
|
213
|
+
const current = (where) => {
|
|
214
|
+
const k = fileOf(where);
|
|
215
|
+
if (!shas.has(k)) shas.set(k, fileSha(cwd, where));
|
|
216
|
+
return shas.get(k);
|
|
217
|
+
};
|
|
218
|
+
const all = [...fold.findings.values()]
|
|
219
|
+
.filter((f) => f.status === 'dismissed' && f.file_sha && !neverSuppressed(f, f.agent))
|
|
220
|
+
.map((f) => ({ f, active: current(f.where) === f.file_sha }));
|
|
221
|
+
return all.map((s) => {
|
|
222
|
+
const phases = new Set(all.filter((o) => o.active && o.f.agent === s.f.agent && sameFinding(o.f, s.f)).map((o) => V.comparablePhase(o.f.phase)));
|
|
223
|
+
return { ...s, scope: phases.size >= 2 ? 'promoted' : 'candidate' };
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** The suppression that filters this judge's finding in this phase, or null. */
|
|
228
|
+
function suppressionFor(supps, agent, phase, f) {
|
|
229
|
+
if (neverSuppressed(f, agent)) return null;
|
|
230
|
+
const hits = supps.filter((s) => s.active && s.f.agent === agent && sameFinding(s.f, f)
|
|
231
|
+
&& (s.scope === 'promoted' || samePhase(s.f.phase, phase)));
|
|
232
|
+
return hits.find((s) => samePhase(s.f.phase, phase)) || hits[0] || null;
|
|
233
|
+
}
|
|
234
|
+
|
|
171
235
|
/**
|
|
172
236
|
* How stable a judge's findings are across its reports on one phase (V6): between
|
|
173
237
|
* consecutive verdicts, how many ids appeared and how many disappeared. A judge whose
|
|
@@ -218,8 +282,14 @@ function recordVerdict(cwd, input) {
|
|
|
218
282
|
const text = typeof input.text === 'string' ? input.text : '';
|
|
219
283
|
if (!text.trim()) return { error: 'empty_input', reason: 'nothing to record' };
|
|
220
284
|
|
|
221
|
-
|
|
285
|
+
let parsed = V.parseVerdictText(text, { agent: input.agent, phase });
|
|
222
286
|
if (!parsed.ok) return { error: parsed.error, reason: parsed.reason };
|
|
287
|
+
// A verification is read against the engine's test run (V2): a failed run is a code
|
|
288
|
+
// fact that lowers the verifier's pass, as a failed test_gate already does (V9).
|
|
289
|
+
if (parsed.verdict.agent === 'pan-verifier') {
|
|
290
|
+
const facts = require('./test-gate.cjs').testRunFacts(cwd, phase);
|
|
291
|
+
if (facts.length) parsed = V.parseVerdictText(text, { agent: input.agent, phase, facts });
|
|
292
|
+
}
|
|
223
293
|
const verdict = parsed.verdict;
|
|
224
294
|
const agent = verdict.agent;
|
|
225
295
|
const recordSig = V.recordSig(text);
|
|
@@ -237,7 +307,7 @@ function recordVerdict(cwd, input) {
|
|
|
237
307
|
|
|
238
308
|
const dup = fold.verdicts.find((r) => r.record_sig === recordSig && r.agent === agent && samePhase(r.phase, phase));
|
|
239
309
|
if (dup) {
|
|
240
|
-
return { ...base, recorded: false, duplicate: true, verdict_id: dup.verdict_id, attempt: dup.attempt, phase, findings: (dup.finding_ids || []).length, new: 0, rematched: 0, resolved: 0, unaddressed: [], reopened: 0 };
|
|
310
|
+
return { ...base, recorded: false, duplicate: true, verdict_id: dup.verdict_id, attempt: dup.attempt, phase, findings: (dup.finding_ids || []).length, new: 0, rematched: 0, resolved: 0, unaddressed: [], reopened: 0, suppressed: dup.suppressed || [], returned: (dup.returned_ids || []).length };
|
|
241
311
|
}
|
|
242
312
|
|
|
243
313
|
const prior = fold.verdicts.filter((r) => r.agent === agent && samePhase(r.phase, phase));
|
|
@@ -251,6 +321,9 @@ function recordVerdict(cwd, input) {
|
|
|
251
321
|
const newRows = [];
|
|
252
322
|
const findingRows = [];
|
|
253
323
|
const mine = [...fold.findings.values()].filter((e) => e.agent === agent && samePhase(e.phase, phase));
|
|
324
|
+
const supps = learnedSuppressions(cwd, fold);
|
|
325
|
+
const suppressed = [];
|
|
326
|
+
const returned = [];
|
|
254
327
|
let rematched = 0;
|
|
255
328
|
for (const f of verdict.findings) {
|
|
256
329
|
let id = V.findingId(phase, agent, f);
|
|
@@ -261,6 +334,17 @@ function recordVerdict(cwd, input) {
|
|
|
261
334
|
if (earlier) { id = earlier.id; rematched++; }
|
|
262
335
|
}
|
|
263
336
|
if (ids.includes(id)) continue; // the same finding twice in one report is one finding
|
|
337
|
+
// A learned suppression (R3) filters it; a dismissed finding no suppression
|
|
338
|
+
// covers any more comes back open.
|
|
339
|
+
const existing = fold.findings.get(id);
|
|
340
|
+
if (!existing || existing.status === 'dismissed') {
|
|
341
|
+
const s = suppressionFor(supps, agent, phase, f);
|
|
342
|
+
if (s) {
|
|
343
|
+
suppressed.push({ id: s.f.id, scope: s.scope, summary: f.summary });
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
if (existing) returned.push(id);
|
|
347
|
+
}
|
|
264
348
|
ids.push(id);
|
|
265
349
|
if (!fold.findings.has(id)) {
|
|
266
350
|
findingRows.push({ v: LEDGER_V, kind: 'finding', ts, id, verdict_id: verdictId, phase, milestone, agent, class: f.class, severity: f.severity, where: f.where, summary: f.summary });
|
|
@@ -307,6 +391,8 @@ function recordVerdict(cwd, input) {
|
|
|
307
391
|
newRows.push({
|
|
308
392
|
v: LEDGER_V, kind: 'verdict', ts, verdict_id: verdictId, record_sig: recordSig, phase, milestone, agent,
|
|
309
393
|
verdict: verdict.verdict, outcome: verdict.outcome, ...lowered, attempt, source, finding_ids: ids,
|
|
394
|
+
...(returned.length ? { returned_ids: returned } : {}),
|
|
395
|
+
...(suppressed.length ? { suppressed } : {}),
|
|
310
396
|
by_severity: bySeverity, by_class: byClass, warnings: parsed.warnings,
|
|
311
397
|
...(notChecked.length ? { not_checked: notChecked } : {}),
|
|
312
398
|
// A plan check is bound to the plans it checked (V7): exec-phase compares this
|
|
@@ -338,7 +424,7 @@ function recordVerdict(cwd, input) {
|
|
|
338
424
|
});
|
|
339
425
|
}
|
|
340
426
|
|
|
341
|
-
return { ...base, recorded: true, duplicate: false, verdict_id: verdictId, attempt, phase, findings: ids.length, new: findingRows.length, rematched, resolved: resolveRows.length, unaddressed, reopened, ledger: planningRel(FINDINGS_FILE) };
|
|
427
|
+
return { ...base, recorded: true, duplicate: false, verdict_id: verdictId, attempt, phase, findings: ids.length, new: findingRows.length, rematched, resolved: resolveRows.length, unaddressed, reopened, suppressed, returned: returned.length, ledger: planningRel(FINDINGS_FILE) };
|
|
342
428
|
}
|
|
343
429
|
|
|
344
430
|
/**
|
|
@@ -373,7 +459,10 @@ function disposeFindings(cwd, input) {
|
|
|
373
459
|
const f = fold.findings.get(id);
|
|
374
460
|
if (!f) { skipped.push({ id, why: 'unknown id' }); continue; }
|
|
375
461
|
if (f.status === as) { skipped.push({ id, why: `already ${as}` }); continue; }
|
|
376
|
-
|
|
462
|
+
// A dismissal remembers its file as it was (R3): it suppresses the finding only
|
|
463
|
+
// until that file changes.
|
|
464
|
+
const fsha = as === 'dismissed' ? fileSha(cwd, f.where) : null;
|
|
465
|
+
out.push({ v: LEDGER_V, kind: 'disposition', ts, id, disposition: as, reason: reason || null, auto: false, ...(fsha ? { file_sha: fsha } : {}) });
|
|
377
466
|
disposed.push(id);
|
|
378
467
|
}
|
|
379
468
|
appendRows(cwd, out);
|
|
@@ -408,6 +497,19 @@ function listFindings(cwd, filters = {}) {
|
|
|
408
497
|
};
|
|
409
498
|
}
|
|
410
499
|
|
|
500
|
+
/** Learned suppressions (R3), each with its scope and whether its file is unchanged. */
|
|
501
|
+
function listSuppressions(cwd, filters = {}) {
|
|
502
|
+
const fold = foldFindings(readLedger(cwd).rows);
|
|
503
|
+
let list = learnedSuppressions(cwd, fold);
|
|
504
|
+
if (filters.agent) list = list.filter((s) => s.f.agent === filters.agent);
|
|
505
|
+
if (filters.phase) list = list.filter((s) => s.scope === 'promoted' || samePhase(s.f.phase, filters.phase));
|
|
506
|
+
const suppressions = list.map((s) => ({
|
|
507
|
+
id: s.f.id, agent: s.f.agent, phase: s.f.phase, class: s.f.class, file: fileOf(s.f.where), summary: s.f.summary,
|
|
508
|
+
reason: s.f.reason, scope: s.scope, active: s.active,
|
|
509
|
+
}));
|
|
510
|
+
return { contract: V.VERDICT_CONTRACT, suppressions, active: suppressions.filter((s) => s.active).length };
|
|
511
|
+
}
|
|
512
|
+
|
|
411
513
|
/**
|
|
412
514
|
* Tech debt for a milestone audit: deferred findings (with the reason recorded at
|
|
413
515
|
* the time) and findings nobody disposed, grouped by phase.
|
|
@@ -470,6 +572,11 @@ function cmdFindingsList(cwd, opts, raw) {
|
|
|
470
572
|
output(r, raw, String(r.counts.total));
|
|
471
573
|
}
|
|
472
574
|
|
|
575
|
+
function cmdFindingsSuppressions(cwd, opts, raw) {
|
|
576
|
+
const r = listSuppressions(cwd, opts);
|
|
577
|
+
output(r, raw, String(r.active));
|
|
578
|
+
}
|
|
579
|
+
|
|
473
580
|
function cmdFindingsDebt(cwd, opts, raw) {
|
|
474
581
|
const r = findingsDebt(cwd, opts);
|
|
475
582
|
output(r, raw, `${r.deferred_count} deferred, ${r.open_count} open`);
|
|
@@ -488,6 +595,7 @@ module.exports = {
|
|
|
488
595
|
recordVerdict,
|
|
489
596
|
disposeFindings,
|
|
490
597
|
listFindings,
|
|
598
|
+
listSuppressions,
|
|
491
599
|
findingsDebt,
|
|
492
600
|
findingsChurn,
|
|
493
601
|
sameFinding,
|
|
@@ -496,4 +604,5 @@ module.exports = {
|
|
|
496
604
|
cmdFindingsDispose,
|
|
497
605
|
cmdFindingsList,
|
|
498
606
|
cmdFindingsDebt,
|
|
607
|
+
cmdFindingsSuppressions,
|
|
499
608
|
};
|
|
@@ -175,7 +175,7 @@ function cmdGitPush(cwd, opts, raw) {
|
|
|
175
175
|
function cmdGitStatus(cwd, opts, raw) {
|
|
176
176
|
if (!isGitRepo(cwd)) { _notGitRepo(raw); return; }
|
|
177
177
|
const branch = getCurrentBranch(cwd);
|
|
178
|
-
const r = execGit(cwd, ['status', '--porcelain']);
|
|
178
|
+
const r = execGit(cwd, ['status', '--porcelain'], { keepLeading: true });
|
|
179
179
|
if (r.exitCode !== 0) {
|
|
180
180
|
output({ error: 'status_failed', detail: r.stderr }, raw, 'status failed');
|
|
181
181
|
return;
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* Mutation report on a phase's diff (program V5): would a test fail if the code the phase
|
|
4
|
+
* wrote were wrong? Coverage says a line ran; a surviving mutant says no test pins what
|
|
5
|
+
* the line does.
|
|
6
|
+
*
|
|
7
|
+
* lines the lines the phase's plan commits changed (git diff from the phase's
|
|
8
|
+
* base, the same base `verify red` uses), in source files that are not tests
|
|
9
|
+
* mutants blunt textual swaps per language (a comparison flipped, && for ||, a
|
|
10
|
+
* boolean return inverted, + 1 for - 1), on a changed line only, outside a
|
|
11
|
+
* comment or string; a seeded sample of them (`--max`, `--seed`)
|
|
12
|
+
* where a throwaway worktree at HEAD, never the user's tree
|
|
13
|
+
* flake the unmutated suite runs twice first; if it fails, or the two runs
|
|
14
|
+
* disagree, nothing a mutant does could be read, so the report aborts
|
|
15
|
+
* result each mutant is killed (the suite fails or times out) or survives;
|
|
16
|
+
* survivors are listed with file, line and the swap, and written to
|
|
17
|
+
* `{NN}-mutation.json`
|
|
18
|
+
*
|
|
19
|
+
* `workflow.mutation_gate`: `off` · `report` (the default: a report, never a refusal) ·
|
|
20
|
+
* `gate` (reserved for V5G, which turns survivors into gaps). No coverage is collected:
|
|
21
|
+
* the full suite runs per mutant, so the sample is kept small.
|
|
22
|
+
*
|
|
23
|
+
* The operator table started as scripts/mutation-probe.cjs's, which PAN runs on itself;
|
|
24
|
+
* that probe now imports it from here.
|
|
25
|
+
*/
|
|
26
|
+
const fs = require('fs');
|
|
27
|
+
const path = require('path');
|
|
28
|
+
const { execGit, findPhaseInternal, output, error, toPosix } = require('./core.cjs');
|
|
29
|
+
|
|
30
|
+
const MODES = Object.freeze(['off', 'report', 'gate']);
|
|
31
|
+
const DEFAULT_MAX = 8;
|
|
32
|
+
const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
|
|
33
|
+
|
|
34
|
+
const C_FAMILY_COMPARE = [
|
|
35
|
+
{ id: 'gte→gt', find: ' >= ', replace: ' > ' },
|
|
36
|
+
{ id: 'gt→gte', find: ' > ', replace: ' >= ' },
|
|
37
|
+
{ id: 'lte→lt', find: ' <= ', replace: ' < ' },
|
|
38
|
+
{ id: 'lt→lte', find: ' < ', replace: ' <= ' },
|
|
39
|
+
];
|
|
40
|
+
|
|
41
|
+
/** The operators, per language family. Each is a plain textual swap, first occurrence on a line. */
|
|
42
|
+
const OPERATORS = Object.freeze({
|
|
43
|
+
js: [
|
|
44
|
+
...C_FAMILY_COMPARE,
|
|
45
|
+
{ id: 'eq→neq', find: ' === ', replace: ' !== ' },
|
|
46
|
+
{ id: 'neq→eq', find: ' !== ', replace: ' === ' },
|
|
47
|
+
{ id: 'and→or', find: ' && ', replace: ' || ' },
|
|
48
|
+
{ id: 'or→and', find: ' || ', replace: ' && ' },
|
|
49
|
+
{ id: 'true→false', find: 'return true', replace: 'return false' },
|
|
50
|
+
{ id: 'false→true', find: 'return false', replace: 'return true' },
|
|
51
|
+
{ id: 'plus→minus', find: ' + 1', replace: ' - 1' },
|
|
52
|
+
],
|
|
53
|
+
c: [
|
|
54
|
+
...C_FAMILY_COMPARE,
|
|
55
|
+
{ id: 'eq→neq', find: ' == ', replace: ' != ' },
|
|
56
|
+
{ id: 'neq→eq', find: ' != ', replace: ' == ' },
|
|
57
|
+
{ id: 'and→or', find: ' && ', replace: ' || ' },
|
|
58
|
+
{ id: 'or→and', find: ' || ', replace: ' && ' },
|
|
59
|
+
{ id: 'true→false', find: 'return true', replace: 'return false' },
|
|
60
|
+
{ id: 'false→true', find: 'return false', replace: 'return true' },
|
|
61
|
+
{ id: 'plus→minus', find: ' + 1', replace: ' - 1' },
|
|
62
|
+
],
|
|
63
|
+
py: [
|
|
64
|
+
...C_FAMILY_COMPARE,
|
|
65
|
+
{ id: 'eq→neq', find: ' == ', replace: ' != ' },
|
|
66
|
+
{ id: 'neq→eq', find: ' != ', replace: ' == ' },
|
|
67
|
+
{ id: 'and→or', find: ' and ', replace: ' or ' },
|
|
68
|
+
{ id: 'or→and', find: ' or ', replace: ' and ' },
|
|
69
|
+
{ id: 'true→false', find: 'return True', replace: 'return False' },
|
|
70
|
+
{ id: 'false→true', find: 'return False', replace: 'return True' },
|
|
71
|
+
{ id: 'plus→minus', find: ' + 1', replace: ' - 1' },
|
|
72
|
+
],
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
const LANGUAGES = [
|
|
76
|
+
{ family: 'js', re: /\.(c|m)?(j|t)sx?$/i, comments: ['//', '/*', '*'] },
|
|
77
|
+
{ family: 'py', re: /\.py$/i, comments: ['#'] },
|
|
78
|
+
{ family: 'c', re: /\.(go|rs|java|kt|kts|cs|c|cc|cpp|h|hpp|swift|scala)$/i, comments: ['//', '/*', '*'] },
|
|
79
|
+
];
|
|
80
|
+
|
|
81
|
+
/** The language family of a file, or null when PAN has no operators for it. */
|
|
82
|
+
function languageOf(file) {
|
|
83
|
+
return LANGUAGES.find((l) => l.re.test(file)) || null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Is this line worth mutating? Skips comments, and a line whose only occurrence of the
|
|
88
|
+
* token is inside a string: a mutated message changes nothing a test should pin. Crude
|
|
89
|
+
* but conservative: on a line with a quote, the token must come before the first quote.
|
|
90
|
+
*/
|
|
91
|
+
function mutableLine(line, token, comments = ['//', '/*', '*']) {
|
|
92
|
+
const trimmed = line.trim();
|
|
93
|
+
if (!trimmed || comments.some((c) => trimmed.startsWith(c))) return false;
|
|
94
|
+
const at = line.indexOf(token);
|
|
95
|
+
if (at === -1) return false;
|
|
96
|
+
const firstQuote = Math.min(...["'", '"', '`'].map((q) => { const i = line.indexOf(q); return i === -1 ? Infinity : i; }));
|
|
97
|
+
return at < firstQuote;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Every mutation in a source text: one per (line, operator), first occurrence on the line.
|
|
102
|
+
* @param {string} src
|
|
103
|
+
* @param {{family?: string, comments?: string[], lines?: Set<number>}} [opts] - `lines` limits it to those line numbers
|
|
104
|
+
*/
|
|
105
|
+
function mutationsFor(src, opts = {}) {
|
|
106
|
+
const ops = OPERATORS[opts.family || 'js'];
|
|
107
|
+
const out = [];
|
|
108
|
+
src.split(/\r?\n/).forEach((line, i) => {
|
|
109
|
+
if (opts.lines && !opts.lines.has(i + 1)) return;
|
|
110
|
+
for (const op of ops) {
|
|
111
|
+
if (!mutableLine(line, op.find, opts.comments)) continue;
|
|
112
|
+
out.push({ line: i + 1, op: op.id, before: line, after: line.replace(op.find, op.replace) });
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
return out;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Apply one mutation to a source string, by line number. Throws if the line moved. */
|
|
119
|
+
function applyMutation(src, mutation) {
|
|
120
|
+
const eol = src.includes('\r\n') ? '\r\n' : '\n';
|
|
121
|
+
const lines = src.split(/\r?\n/);
|
|
122
|
+
if (lines[mutation.line - 1] !== mutation.before) throw new Error(`line ${mutation.line} is not what the mutation was built from`);
|
|
123
|
+
lines[mutation.line - 1] = mutation.after;
|
|
124
|
+
return lines.join(eol);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Deterministic shuffle-and-take, so a report is reproducible from its seed. */
|
|
128
|
+
function sample(items, n, seed) {
|
|
129
|
+
let s = seed >>> 0;
|
|
130
|
+
const rand = () => { s = (s * 1664525 + 1013904223) >>> 0; return s / 0x100000000; };
|
|
131
|
+
const copy = items.slice();
|
|
132
|
+
for (let i = copy.length - 1; i > 0; i--) {
|
|
133
|
+
const j = Math.floor(rand() * (i + 1));
|
|
134
|
+
[copy[i], copy[j]] = [copy[j], copy[i]];
|
|
135
|
+
}
|
|
136
|
+
return copy.slice(0, n);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The line numbers each file gained or changed between two revisions (`git diff -U0`).
|
|
141
|
+
* @returns {Map<string, Set<number>>}
|
|
142
|
+
*/
|
|
143
|
+
function changedLines(cwd, base, head = 'HEAD') {
|
|
144
|
+
const r = execGit(cwd, ['-c', 'core.quotepath=off', 'diff', '-U0', '--no-color', '--relative', base, head]);
|
|
145
|
+
const out = new Map();
|
|
146
|
+
if (r.exitCode !== 0) return out;
|
|
147
|
+
let file = null;
|
|
148
|
+
for (const line of r.stdout.split('\n')) {
|
|
149
|
+
const f = line.match(/^\+\+\+ (?:b\/)?(.+)$/);
|
|
150
|
+
if (f) { file = f[1] === '/dev/null' ? null : f[1].trim(); continue; }
|
|
151
|
+
const h = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
|
|
152
|
+
if (h && file) {
|
|
153
|
+
const start = Number(h[1]);
|
|
154
|
+
const count = h[2] === undefined ? 1 : Number(h[2]);
|
|
155
|
+
if (!out.has(file)) out.set(file, new Set());
|
|
156
|
+
for (let n = start; n < start + count; n++) out.get(file).add(n);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
return out;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** `workflow.mutation_gate` from config.json; `report` unless set. */
|
|
163
|
+
function mutationMode(cwd) {
|
|
164
|
+
const v = (require('./test-gate.cjs').rawConfig(cwd).workflow || {}).mutation_gate;
|
|
165
|
+
return MODES.includes(v) ? v : 'report';
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* The mutation report for one phase.
|
|
170
|
+
* @param {string} cwd
|
|
171
|
+
* @param {string} phase
|
|
172
|
+
* @param {{max?: number, seed?: number, timeoutMs?: number}} [opts]
|
|
173
|
+
*/
|
|
174
|
+
function mutatePhase(cwd, phase, opts = {}) {
|
|
175
|
+
const TG = require('./test-gate.cjs');
|
|
176
|
+
const { phaseBase, withWorktree } = require('./verify-red.cjs');
|
|
177
|
+
const { isTestFile } = require('./verify-integrity.cjs');
|
|
178
|
+
const info = findPhaseInternal(cwd, phase);
|
|
179
|
+
if (!info) return { error: 'phase_not_found', reason: `no phase directory for phase ${phase}` };
|
|
180
|
+
const phaseNumber = info.phase_number || String(phase);
|
|
181
|
+
const mode = mutationMode(cwd);
|
|
182
|
+
const seed = Number.isFinite(opts.seed) ? opts.seed : 1;
|
|
183
|
+
const max = opts.max || DEFAULT_MAX;
|
|
184
|
+
const base = { phase: phaseNumber, mode, seed, checked: false };
|
|
185
|
+
if (mode === 'off' && !opts.force) return { ...base, reason: 'workflow.mutation_gate is off' };
|
|
186
|
+
const b = phaseBase(cwd, phaseNumber);
|
|
187
|
+
if (!b.base) return { ...base, reason: b.reason };
|
|
188
|
+
const suite = TG.detectTestCommand(cwd);
|
|
189
|
+
if (!suite.command) return { ...base, base: b.base, reason: suite.reason };
|
|
190
|
+
|
|
191
|
+
// Candidates: changed lines of the phase's own source files, per language.
|
|
192
|
+
const changed = changedLines(cwd, b.base);
|
|
193
|
+
const candidates = [];
|
|
194
|
+
for (const [file, lines] of changed) {
|
|
195
|
+
const lang = languageOf(file);
|
|
196
|
+
if (!lang || isTestFile(file) || !b.files.has(file) || /^\.planning\//.test(file)) continue;
|
|
197
|
+
let src;
|
|
198
|
+
try { src = fs.readFileSync(path.join(cwd, file), 'utf8'); } catch { continue; }
|
|
199
|
+
for (const m of mutationsFor(src, { family: lang.family, comments: lang.comments, lines })) candidates.push({ file, ...m });
|
|
200
|
+
}
|
|
201
|
+
if (!candidates.length) return { ...base, base: b.base, command: suite.command, candidates: 0, reason: 'no mutable line in the source the phase changed' };
|
|
202
|
+
const picked = sample(candidates, max, seed);
|
|
203
|
+
const timeoutMs = opts.timeoutMs || DEFAULT_TIMEOUT_MS;
|
|
204
|
+
const killedBy = (r) => r.timed_out || r.exit !== 0;
|
|
205
|
+
|
|
206
|
+
const wt = withWorktree(cwd, 'HEAD', (dir) => {
|
|
207
|
+
// Flake pre-check: the unmutated suite must pass, twice, or no mutant can be read.
|
|
208
|
+
const first = TG.runCommand(dir, suite.command, timeoutMs);
|
|
209
|
+
const second = TG.runCommand(dir, suite.command, timeoutMs);
|
|
210
|
+
if (killedBy(first) || killedBy(second)) {
|
|
211
|
+
return { aborted: killedBy(first) !== killedBy(second) ? 'flaky' : 'suite_fails', reason: killedBy(first) !== killedBy(second) ? 'the unmutated suite passed once and failed once: a flaky suite would make every mutant unreadable' : 'the unmutated suite fails at HEAD: fix it before measuring its tests' };
|
|
212
|
+
}
|
|
213
|
+
const results = [];
|
|
214
|
+
for (const m of picked) {
|
|
215
|
+
const file = path.join(dir, m.file);
|
|
216
|
+
const original = fs.readFileSync(file, 'utf8');
|
|
217
|
+
let mutated;
|
|
218
|
+
try { mutated = applyMutation(original, m); } catch { continue; }
|
|
219
|
+
fs.writeFileSync(file, mutated);
|
|
220
|
+
try {
|
|
221
|
+
results.push({ ...m, killed: killedBy(TG.runCommand(dir, suite.command, timeoutMs)) });
|
|
222
|
+
} finally {
|
|
223
|
+
fs.writeFileSync(file, original);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
return { results };
|
|
227
|
+
});
|
|
228
|
+
if (wt.error) return { ...base, base: b.base, reason: wt.error };
|
|
229
|
+
const head = (execGit(cwd, ['rev-parse', 'HEAD']).stdout || '').trim() || null;
|
|
230
|
+
if (wt.value.aborted) return { ...base, base: b.base, head, command: suite.command, candidates: candidates.length, aborted: wt.value.aborted, reason: wt.value.reason };
|
|
231
|
+
const results = wt.value.results;
|
|
232
|
+
const killed = results.filter((r) => r.killed).length;
|
|
233
|
+
const survivors = results.filter((r) => !r.killed).map(({ file, line, op, before, after }) => ({ file: toPosix(file), line, op, before: before.trim(), after: after.trim() }));
|
|
234
|
+
return {
|
|
235
|
+
...base, checked: results.length > 0, base: b.base, head, command: suite.command,
|
|
236
|
+
candidates: candidates.length, sampled: results.length, killed, survived: survivors.length,
|
|
237
|
+
score: results.length ? Number((killed / results.length).toFixed(2)) : null, survivors,
|
|
238
|
+
};
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* `pan-tools verify mutate --phase N [--max n] [--seed n] [--timeout ms]`: writes
|
|
243
|
+
* `{NN}-mutation.json` and exits 0 (report mode; V5G adds the gate).
|
|
244
|
+
*/
|
|
245
|
+
function cmdVerifyMutate(cwd, opts, raw) {
|
|
246
|
+
if (!opts.phase) error('Usage: verify mutate --phase <N> [--max <n>] [--seed <n>] [--timeout <ms>]');
|
|
247
|
+
const r = mutatePhase(cwd, opts.phase, { max: opts.max ? Number(opts.max) : undefined, seed: opts.seed !== undefined && opts.seed !== null ? Number(opts.seed) : undefined, timeoutMs: opts.timeout ? Number(opts.timeout) : undefined });
|
|
248
|
+
if (r.error) error(r.reason);
|
|
249
|
+
const info = findPhaseInternal(cwd, opts.phase);
|
|
250
|
+
const prefix = String(info.phase_number).split('.').map((p, i) => (i === 0 ? p.padStart(2, '0') : p)).join('.');
|
|
251
|
+
const file = path.join(cwd, info.directory, `${prefix}-mutation.json`);
|
|
252
|
+
fs.writeFileSync(file, JSON.stringify(r, null, 2) + '\n');
|
|
253
|
+
r.record = toPosix(path.relative(cwd, file));
|
|
254
|
+
output(r, raw, r.aborted ? `aborted:${r.aborted}` : (r.checked ? `${r.killed}/${r.sampled} killed` : 'not_checked'));
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
module.exports = { OPERATORS, MODES, languageOf, mutableLine, mutationsFor, applyMutation, sample, changedLines, mutationMode, mutatePhase, cmdVerifyMutate };
|
|
@@ -17,8 +17,8 @@
|
|
|
17
17
|
*
|
|
18
18
|
* What the gate could not check is listed in `not_checked`, never refused on:
|
|
19
19
|
* a skipped test gate, an unknown freshness, no git, no plan commits, the verifier
|
|
20
|
-
* switched off. `workflow.phase_gate` is `
|
|
21
|
-
*
|
|
20
|
+
* switched off. `workflow.phase_gate` is `enforce` (the default since V1E: refuse
|
|
21
|
+
* unless `--force --reason`, which records a decision) or `warn` (report and proceed).
|
|
22
22
|
*/
|
|
23
23
|
const fs = require('fs');
|
|
24
24
|
const path = require('path');
|
|
@@ -29,12 +29,12 @@ const { planningPath } = require('./utils.cjs');
|
|
|
29
29
|
const MODES = Object.freeze(['warn', 'enforce']);
|
|
30
30
|
const SERIOUS = new Set(['critical', 'high']);
|
|
31
31
|
|
|
32
|
-
/** `workflow.phase_gate` from config.json: `
|
|
32
|
+
/** `workflow.phase_gate` from config.json: `warn`, or (anything else, absent included) `enforce`. */
|
|
33
33
|
function gateMode(cwd) {
|
|
34
34
|
let cfg = null;
|
|
35
35
|
try { cfg = JSON.parse(fs.readFileSync(planningPath(cwd, 'config.json'), 'utf-8')); } catch { /* defaults */ }
|
|
36
36
|
const v = cfg && cfg.workflow && cfg.workflow.phase_gate;
|
|
37
|
-
return MODES.includes(v) ? v : '
|
|
37
|
+
return MODES.includes(v) ? v : 'enforce';
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
/**
|
|
@@ -51,6 +51,7 @@ function phaseGate(cwd, phase, opts = {}) {
|
|
|
51
51
|
const { reconcilePhase, verificationFreshness, scanStubs } = require('./verify.cjs');
|
|
52
52
|
const { committedFiles } = require('./verify-scope.cjs');
|
|
53
53
|
const { checkTestIntegrity } = require('./verify-integrity.cjs');
|
|
54
|
+
const TG = require('./test-gate.cjs');
|
|
54
55
|
|
|
55
56
|
const mode = MODES.includes(opts.mode) ? opts.mode : gateMode(cwd);
|
|
56
57
|
const reasons = [];
|
|
@@ -77,18 +78,23 @@ function phaseGate(cwd, phase, opts = {}) {
|
|
|
77
78
|
if (verifierOn) refuse('no_verification', 'the phase has no verification; run /pan:verify-phase');
|
|
78
79
|
else skip('verification', 'workflow.verifier is off');
|
|
79
80
|
} else {
|
|
80
|
-
|
|
81
|
+
// The engine's test run (V2) is a code fact: a failed run lowers a written pass.
|
|
82
|
+
const run = TG.readTestRun(cwd, phaseNumber);
|
|
83
|
+
const parsed = V.parseVerdictText(safeReadFile(path.join(dir, verFile)) || '', { agent: 'pan-verifier', phase: phaseNumber, facts: TG.testRunFacts(cwd, phaseNumber) });
|
|
81
84
|
if (!parsed.ok) {
|
|
82
85
|
refuse('verification_unreadable', `${verFile}: ${parsed.reason}`);
|
|
83
86
|
} else {
|
|
84
87
|
const v = parsed.verdict;
|
|
85
88
|
const fm = V.verdictFromVerificationFrontmatter(safeReadFile(path.join(dir, verFile)) || '') || {};
|
|
86
89
|
facts.verification = v.verdict;
|
|
87
|
-
|
|
90
|
+
// The verification's own word, else the engine's record of the run.
|
|
91
|
+
const gate = fm.test_gate || (run && run.status) || null;
|
|
92
|
+
facts.test_gate = gate;
|
|
93
|
+
if (run) facts.test_run = run.status;
|
|
88
94
|
if (parsed.lowered_by && parsed.lowered_by.length) facts.lowered_by = parsed.lowered_by;
|
|
89
|
-
if (
|
|
90
|
-
else if (
|
|
91
|
-
else if (!
|
|
95
|
+
if (gate === 'failed') refuse('test_failed', fm.test_gate === 'failed' ? 'the verification records test_gate: failed' : `the engine's test run failed: ${run.reason || 'see the test-run record'}`);
|
|
96
|
+
else if (gate === 'skipped') skip('tests', fm.test_gate ? 'the test gate was skipped' : (run.reason || 'the test gate was skipped'));
|
|
97
|
+
else if (!gate) skip('tests', 'the verification does not record its test gate');
|
|
92
98
|
if (v.outcome === 'fail') {
|
|
93
99
|
refuse('verification_failed', `the verification is ${v.verdict}${v.agent_verdict ? ` (the verifier wrote ${v.agent_verdict})` : ''}`);
|
|
94
100
|
} else if (v.outcome === 'needs_human') {
|