pan-wizard 3.34.0 → 3.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/agents/pan-conductor.md +2 -2
- package/agents/pan-debugger.md +3 -1
- package/agents/pan-design-checker.md +1 -0
- package/agents/pan-executor.md +18 -1
- package/agents/pan-experiment-runner.md +1 -1
- package/agents/pan-hardener.md +1 -1
- package/agents/pan-meta-reviewer.md +1 -1
- package/agents/pan-phase-researcher.md +2 -2
- package/agents/pan-plan-checker.md +9 -8
- package/agents/pan-planner.md +3 -1
- package/agents/pan-project-researcher.md +1 -1
- package/agents/pan-research-synthesizer.md +1 -1
- package/agents/pan-reviewer.md +3 -2
- package/agents/pan-verifier.md +1 -1
- package/bin/install-lib.cjs +119 -16
- package/bin/install.js +34 -62
- package/commands/pan/cost.md +0 -4
- package/commands/pan/dashboard.md +0 -3
- package/commands/pan/design-phase.md +2 -2
- package/commands/pan/hud.md +1 -5
- package/commands/pan/knowledge.md +0 -1
- package/commands/pan/mcp-bridge.md +1 -3
- package/commands/pan/optimize.md +2 -0
- package/commands/pan/phase-tests.md +1 -1
- package/commands/pan/preview.md +1 -2
- package/commands/pan/report.md +1 -5
- package/commands/pan/research-phase.md +4 -4
- package/commands/pan/review-deep.md +0 -2
- package/commands/pan/what-if.md +1 -2
- package/hooks/dist/pan-context-monitor.js +12 -3
- package/hooks/dist/pan-guard.js +244 -0
- package/hooks/dist/pan-statusline.js +5 -2
- package/package.json +1 -1
- package/pan-wizard-core/bin/lib/commands.cjs +24 -2
- package/pan-wizard-core/bin/lib/config.cjs +5 -0
- package/pan-wizard-core/bin/lib/findings.cjs +126 -19
- package/pan-wizard-core/bin/lib/hygiene.cjs +11 -5
- package/pan-wizard-core/bin/lib/init.cjs +6 -0
- package/pan-wizard-core/bin/lib/optimize.cjs +13 -1
- package/pan-wizard-core/bin/lib/phase-gate.cjs +174 -0
- package/pan-wizard-core/bin/lib/phase.cjs +24 -0
- package/pan-wizard-core/bin/lib/preview.cjs +32 -31
- package/pan-wizard-core/bin/lib/verdict.cjs +140 -10
- package/pan-wizard-core/bin/lib/verify-integrity.cjs +181 -0
- package/pan-wizard-core/bin/lib/verify-plan-set.cjs +230 -0
- package/pan-wizard-core/bin/lib/verify-scope.cjs +6 -5
- package/pan-wizard-core/bin/lib/whatif.cjs +48 -6
- package/pan-wizard-core/bin/pan-tools.cjs +13 -4
- package/pan-wizard-core/mcp/merge-gate.cjs +14 -3
- package/pan-wizard-core/mcp/native-tools.cjs +3 -3
- package/pan-wizard-core/opencode/pan-wizard.js +36 -8
- package/pan-wizard-core/references/model-profiles.md +1 -1
- package/pan-wizard-core/references/planning-config.md +3 -3
- package/pan-wizard-core/references/verdict-contract.md +6 -1
- package/pan-wizard-core/templates/config.json +2 -6
- package/pan-wizard-core/templates/debug-subagent-prompt.md +2 -1
- package/pan-wizard-core/templates/phase-prompt.md +2 -0
- package/pan-wizard-core/templates/planner-subagent-prompt.md +2 -0
- package/pan-wizard-core/templates/preview-report.md +1 -1
- package/pan-wizard-core/templates/state.md +4 -4
- package/pan-wizard-core/workflows/diagnose-issues.md +4 -4
- package/pan-wizard-core/workflows/discuss-phase.md +2 -1
- package/pan-wizard-core/workflows/exec-phase.md +51 -40
- package/pan-wizard-core/workflows/milestone-gaps.md +2 -2
- package/pan-wizard-core/workflows/new-project.md +8 -16
- package/pan-wizard-core/workflows/plan-phase.md +22 -8
- package/pan-wizard-core/workflows/progress.md +1 -1
- package/pan-wizard-core/workflows/quick.md +2 -2
- package/pan-wizard-core/workflows/todo-check.md +3 -2
- package/pan-wizard-core/workflows/transition.md +4 -0
- package/pan-wizard-core/workflows/verify-phase.md +6 -6
- package/scripts/build-hooks.js +1 -0
- package/scripts/generate-skills-docs.py +2 -2
- package/scripts/prompt-lint.cjs +167 -0
- package/scripts/test-surface.cjs +1 -0
|
@@ -161,7 +161,8 @@ function buildPhasePreview(cwd, phaseNum) {
|
|
|
161
161
|
* Produce a dependency graph + mermaid source + parallel-batch recommendation.
|
|
162
162
|
*
|
|
163
163
|
* Dependency detection:
|
|
164
|
-
* -
|
|
164
|
+
* - the roadmap's `**Depends on:** Phase N` line (explicit)
|
|
165
|
+
* - plan frontmatter `depends_on: [phase:NN, phase:MM]` (explicit; bare plan ids are not phases)
|
|
165
166
|
* - mentions of prior phases in plan text (heuristic, flagged as "hidden")
|
|
166
167
|
*
|
|
167
168
|
* @param {string} cwd - Project root
|
|
@@ -195,6 +196,13 @@ function buildPhaseDependencyGraph(cwd) {
|
|
|
195
196
|
}
|
|
196
197
|
}
|
|
197
198
|
|
|
199
|
+
// Graph keys are the roadmap checklist's numbers; a dependency written "01" or
|
|
200
|
+
// "Phase 1" resolves to the key whose unpadded form matches.
|
|
201
|
+
const unpad = (n) => String(n).trim().replace(/^0+(?=\d)/, '');
|
|
202
|
+
const keyByUnpadded = new Map(phaseList.map(p => [unpad(p.num), p.num]));
|
|
203
|
+
const keyOf = (n) => keyByUnpadded.get(unpad(n)) || unpad(n);
|
|
204
|
+
const { phaseDependencies } = require('./roadmap.cjs');
|
|
205
|
+
|
|
198
206
|
for (const p of phaseList) {
|
|
199
207
|
graph[p.num] = {
|
|
200
208
|
num: p.num,
|
|
@@ -204,6 +212,15 @@ function buildPhaseDependencyGraph(cwd) {
|
|
|
204
212
|
hidden_deps: [],
|
|
205
213
|
};
|
|
206
214
|
|
|
215
|
+
const addExplicit = (d) => {
|
|
216
|
+
const key = keyOf(d);
|
|
217
|
+
if (key !== keyOf(p.num) && !graph[p.num].explicit_deps.includes(key)) graph[p.num].explicit_deps.push(key);
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
// Explicit via the roadmap: the phase's `**Depends on:** Phase N` line.
|
|
221
|
+
const roadmapPhase = getRoadmapPhaseInternal(cwd, p.num);
|
|
222
|
+
for (const d of phaseDependencies(roadmapPhase && roadmapPhase.section)) addExplicit(d);
|
|
223
|
+
|
|
207
224
|
// Read plan files for depends_on frontmatter + prior-phase mentions.
|
|
208
225
|
const dir = phaseDirs[p.num];
|
|
209
226
|
if (!dir) continue;
|
|
@@ -212,35 +229,18 @@ function buildPhaseDependencyGraph(cwd) {
|
|
|
212
229
|
try { files = fs.readdirSync(fullDir).filter(isPlanFile); } catch { continue; }
|
|
213
230
|
|
|
214
231
|
// LF, so the frontmatter match below also finds a CRLF plan's depends_on.
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
//
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
// Also support block-list form:
|
|
231
|
-
// depends_on:
|
|
232
|
-
// - phase:1
|
|
233
|
-
// - phase:2
|
|
234
|
-
const blockMatch = fmMatch[1].match(/^depends_on:\s*\n((?:\s*-\s*.+\n?)*)/m);
|
|
235
|
-
if (blockMatch && !depsLine) {
|
|
236
|
-
const items = blockMatch[1].match(/^\s*-\s*(.+)$/gm) || [];
|
|
237
|
-
for (const item of items) {
|
|
238
|
-
const d = item.match(/\d+(?:\.\d+)?/);
|
|
239
|
-
if (d && d[0] !== p.num && !graph[p.num].explicit_deps.includes(d[0])) {
|
|
240
|
-
graph[p.num].explicit_deps.push(d[0]);
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
}
|
|
232
|
+
const planTexts = files.map(f => toLf(safeReadFile(path.join(fullDir, f)) || ''));
|
|
233
|
+
const phaseText = planTexts.join('\n');
|
|
234
|
+
|
|
235
|
+
// Explicit via each plan's frontmatter. A plan's `depends_on` names plan ids
|
|
236
|
+
// inside its own phase (`["01"]`), so only `phase:N` entries are phase
|
|
237
|
+
// dependencies, inline (`[phase:1, phase:2]`) or as a block list.
|
|
238
|
+
for (const text of planTexts) {
|
|
239
|
+
const fmMatch = text.match(/^---\n([\s\S]*?)\n---/);
|
|
240
|
+
if (!fmMatch) continue;
|
|
241
|
+
const depsBlock = fmMatch[1].match(/^depends_on:[^\n]*(?:\n[ \t]*-[^\n]*)*/m);
|
|
242
|
+
if (!depsBlock) continue;
|
|
243
|
+
for (const m of depsBlock[0].matchAll(/\bphase[:\s]*(\d+(?:\.\d+)?)/gi)) addExplicit(m[1]);
|
|
244
244
|
}
|
|
245
245
|
|
|
246
246
|
// Hidden via mentions: "phase N", "from phase N", "as in phase N".
|
|
@@ -248,7 +248,8 @@ function buildPhaseDependencyGraph(cwd) {
|
|
|
248
248
|
let mm;
|
|
249
249
|
const mentions = new Set();
|
|
250
250
|
while ((mm = mentionRe.exec(phaseText)) !== null) {
|
|
251
|
-
|
|
251
|
+
const k = keyOf(mm[1]);
|
|
252
|
+
if (k !== keyOf(p.num)) mentions.add(k);
|
|
252
253
|
}
|
|
253
254
|
graph[p.num].hidden_deps = [...mentions].filter(d => !graph[p.num].explicit_deps.includes(d));
|
|
254
255
|
}
|
|
@@ -28,10 +28,6 @@ const FINDING_CLASSES = Object.freeze(['missing', 'partial', 'contradicts', 'unr
|
|
|
28
28
|
const SEVERITIES = Object.freeze(['critical', 'high', 'medium', 'low', 'info']);
|
|
29
29
|
const DISPOSITIONS = Object.freeze(['fixed', 'deferred', 'dismissed', 'decision']);
|
|
30
30
|
const REASON_REQUIRED = Object.freeze(['deferred', 'dismissed', 'decision']);
|
|
31
|
-
// A re-run does not re-check these, so their absence from a later verdict is not
|
|
32
|
-
// evidence that they were resolved (D6).
|
|
33
|
-
const AUTO_FIX_EXEMPT = Object.freeze(['human', 'unrequested']);
|
|
34
|
-
|
|
35
31
|
/** Each judge's own verdict word → the normalised outcome it implies. */
|
|
36
32
|
const KNOWN_VERDICTS = Object.freeze({
|
|
37
33
|
'pan-verifier': Object.freeze({ passed: 'pass', gaps_found: 'fail', human_needed: 'needs_human' }),
|
|
@@ -43,6 +39,8 @@ const KNOWN_VERDICTS = Object.freeze({
|
|
|
43
39
|
const LIMITS = Object.freeze({ findings: 100, where: 200, summary: 300, verdict: 40, score: 40, not_checked: 20, check: 80 });
|
|
44
40
|
// What the verifier's test gate can report (verify-phase.md, run_test_suite).
|
|
45
41
|
const TEST_GATE_VALUES = Object.freeze(['passed', 'failed', 'skipped']);
|
|
42
|
+
const RESOLUTION_STATUSES = Object.freeze(['resolved', 'still_present', 'cannot_tell']);
|
|
43
|
+
const FINDING_ID_RE = /^f_[0-9a-f]{10}$/;
|
|
46
44
|
const AGENT_RE = /^[a-z][a-z0-9-]{0,63}$/;
|
|
47
45
|
const OPEN_FENCE_RE = /^[ \t]*(`{3,}|~{3,})[ \t]*pan-verdict[ \t]*$/;
|
|
48
46
|
const CLOSE_FENCE_RE = /^[ \t]*(`{3,}|~{3,})[ \t]*$/;
|
|
@@ -198,7 +196,42 @@ function validateVerdict(obj, opts = {}) {
|
|
|
198
196
|
});
|
|
199
197
|
|
|
200
198
|
const notChecked = normaliseNotChecked(obj.not_checked, warnings);
|
|
201
|
-
|
|
199
|
+
const resolutions = normaliseResolutions(obj.resolutions, warnings);
|
|
200
|
+
return { ok: true, verdict: { contract, agent, outcome, verdict, phase, score, findings, not_checked: notChecked, resolutions }, warnings };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* A judge's answers about its earlier open findings (V6): each `{id|summary, status,
|
|
205
|
+
* evidence}` with status resolved | still_present | cannot_tell. Silence is not an
|
|
206
|
+
* answer: a finding the judge leaves out stays open. `resolved` without evidence is
|
|
207
|
+
* recorded as cannot_tell, so a bare "fixed" closes nothing.
|
|
208
|
+
*/
|
|
209
|
+
function normaliseResolutions(value, warnings) {
|
|
210
|
+
if (value == null) return [];
|
|
211
|
+
if (!Array.isArray(value)) {
|
|
212
|
+
warnings.push('resolutions_not_an_array');
|
|
213
|
+
return [];
|
|
214
|
+
}
|
|
215
|
+
const out = [];
|
|
216
|
+
value.slice(0, LIMITS.findings).forEach((r, i) => {
|
|
217
|
+
if (!r || typeof r !== 'object' || Array.isArray(r)) { warnings.push(`resolution_dropped: #${i} is not an object`); return; }
|
|
218
|
+
const id = typeof r.id === 'string' && FINDING_ID_RE.test(r.id.trim()) ? r.id.trim() : null;
|
|
219
|
+
const named = r.summary ?? r.truth;
|
|
220
|
+
const summary = named == null ? null : (truncate(cleanText(String(named)), LIMITS.summary) || null);
|
|
221
|
+
if (!id && !summary) { warnings.push(`resolution_dropped: #${i} names no finding id or summary`); return; }
|
|
222
|
+
let status = typeof r.status === 'string' ? r.status.trim().toLowerCase() : '';
|
|
223
|
+
if (!RESOLUTION_STATUSES.includes(status)) {
|
|
224
|
+
warnings.push(`resolution_unknown_status: #${i} ${JSON.stringify(r.status === undefined ? null : r.status)} recorded as cannot_tell`);
|
|
225
|
+
status = 'cannot_tell';
|
|
226
|
+
}
|
|
227
|
+
const evidence = r.evidence == null ? '' : truncate(cleanText(String(r.evidence)), LIMITS.summary);
|
|
228
|
+
if (status === 'resolved' && !evidence) {
|
|
229
|
+
warnings.push(`resolution_without_evidence: #${i} recorded as cannot_tell`);
|
|
230
|
+
status = 'cannot_tell';
|
|
231
|
+
}
|
|
232
|
+
out.push({ id, summary, status, evidence: evidence || null });
|
|
233
|
+
});
|
|
234
|
+
return out;
|
|
202
235
|
}
|
|
203
236
|
|
|
204
237
|
/**
|
|
@@ -336,6 +369,49 @@ function listOfMappings(yaml, key) {
|
|
|
336
369
|
return items;
|
|
337
370
|
}
|
|
338
371
|
|
|
372
|
+
/**
|
|
373
|
+
* A list nested one level under a top-level key, as the verifier writes
|
|
374
|
+
* `re_verification.gaps_closed`. Items are bare strings (read as `{truth}`) or
|
|
375
|
+
* mappings (`- truth: "…"` with `evidence: "…"` below it).
|
|
376
|
+
*/
|
|
377
|
+
function nestedList(yaml, parent, key) {
|
|
378
|
+
const lines = yaml.split('\n');
|
|
379
|
+
const start = lines.findIndex((l) => {
|
|
380
|
+
const m = KEY_VALUE_RE.exec(l);
|
|
381
|
+
return m && m[1] === parent && indentOf(l) === 0;
|
|
382
|
+
});
|
|
383
|
+
if (start === -1) return [];
|
|
384
|
+
const items = [];
|
|
385
|
+
let keyIndent = -1;
|
|
386
|
+
let cur = null;
|
|
387
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
388
|
+
const line = lines[i];
|
|
389
|
+
if (!line.trim()) continue;
|
|
390
|
+
const ind = indentOf(line);
|
|
391
|
+
if (ind === 0) break;
|
|
392
|
+
const body = line.trim();
|
|
393
|
+
if (keyIndent === -1) {
|
|
394
|
+
const kv = KEY_VALUE_RE.exec(body);
|
|
395
|
+
if (kv && kv[1] === key) {
|
|
396
|
+
if (stripComment(kv[2]) !== '') return []; // `gaps_closed: []`
|
|
397
|
+
keyIndent = ind;
|
|
398
|
+
}
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
if (ind <= keyIndent) break;
|
|
402
|
+
const it = ITEM_RE.exec(line);
|
|
403
|
+
if (it) {
|
|
404
|
+
const kv = KEY_VALUE_RE.exec(it[2]);
|
|
405
|
+
cur = kv ? { [kv[1]]: unquote(kv[2]) } : { truth: unquote(it[2]) };
|
|
406
|
+
items.push(cur);
|
|
407
|
+
} else if (cur) {
|
|
408
|
+
const kv = KEY_VALUE_RE.exec(body);
|
|
409
|
+
if (kv) cur[kv[1]] = unquote(kv[2]);
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
return items.filter((x) => x.truth || x.summary || x.id);
|
|
413
|
+
}
|
|
414
|
+
|
|
339
415
|
/** A top-level scalar from the frontmatter (e.g. `status`), or null. */
|
|
340
416
|
function frontmatterScalar(yaml, key) {
|
|
341
417
|
for (const l of yaml.split('\n')) {
|
|
@@ -397,12 +473,57 @@ function verdictFromVerificationFrontmatter(text, agent = 'pan-verifier') {
|
|
|
397
473
|
notChecked = notChecked.filter((n) => !isTests(n));
|
|
398
474
|
contradicted = true;
|
|
399
475
|
}
|
|
400
|
-
|
|
476
|
+
// Re-verification answers (V6): a gap the verifier re-checked and closed is
|
|
477
|
+
// resolved; a regression or a remaining gap is still present.
|
|
478
|
+
const resolutions = [
|
|
479
|
+
...nestedList(yaml, 're_verification', 'gaps_closed').map((g) => ({
|
|
480
|
+
id: g.id, summary: g.truth || g.summary, status: 'resolved', evidence: g.evidence || 'closed by the verifier\'s re-verification',
|
|
481
|
+
})),
|
|
482
|
+
...nestedList(yaml, 're_verification', 'regressions').map((g) => ({ id: g.id, summary: g.truth || g.summary, status: 'still_present', evidence: g.evidence || null })),
|
|
483
|
+
...nestedList(yaml, 're_verification', 'gaps_remaining').map((g) => ({ id: g.id, summary: g.truth || g.summary, status: 'still_present', evidence: g.evidence || null })),
|
|
484
|
+
];
|
|
485
|
+
return { contract: VERDICT_CONTRACT, agent, verdict: status, outcome, findings, not_checked: notChecked, resolutions, test_gate: testGate, not_checked_contradicted: contradicted };
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/** Names of the code facts that passed: a passing fact closes its own code-fact finding (V6). */
|
|
489
|
+
function passingFacts(facts) {
|
|
490
|
+
return (Array.isArray(facts) ? facts : []).filter((f) => f && f.ok === true && f.fact).map((f) => f.fact);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** The verdict word an agent uses for a fail, or null for an agent outside KNOWN_VERDICTS. */
|
|
494
|
+
function failWordFor(agent) {
|
|
495
|
+
const table = KNOWN_VERDICTS[agent];
|
|
496
|
+
return table ? Object.keys(table).find((k) => table[k] === 'fail') || null : null;
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/**
|
|
500
|
+
* Two-key verdict (V9): a code fact can lower a judge's verdict, never raise it.
|
|
501
|
+
* Any failing fact turns a pass or needs_human into the agent's fail word, keeps
|
|
502
|
+
* the agent's own word as `agent_verdict`, and adds one high finding per fact
|
|
503
|
+
* (`where: code-fact:<fact>`) so the gap round has something to close.
|
|
504
|
+
*
|
|
505
|
+
* @param {Object} verdict - a validated verdict
|
|
506
|
+
* @param {Array<{fact: string, ok: boolean, summary: string}>} facts
|
|
507
|
+
* @returns {{verdict: Object, lowered_by: string[]}} the same verdict object when nothing lowers it
|
|
508
|
+
*/
|
|
509
|
+
function lowerByCodeFacts(verdict, facts) {
|
|
510
|
+
const failing = (Array.isArray(facts) ? facts : []).filter((f) => f && f.ok === false && f.fact);
|
|
511
|
+
if (!failing.length || verdict.outcome === 'fail') return { verdict, lowered_by: [] };
|
|
512
|
+
const findings = [...verdict.findings];
|
|
513
|
+
for (const f of failing) {
|
|
514
|
+
findings.push({ class: 'defect', severity: 'high', where: `code-fact:${f.fact}`, summary: truncate(cleanText(f.summary || `${f.fact} failed`), LIMITS.summary) });
|
|
515
|
+
}
|
|
516
|
+
return {
|
|
517
|
+
verdict: { ...verdict, outcome: 'fail', verdict: failWordFor(verdict.agent) || 'fail', agent_verdict: verdict.verdict, findings },
|
|
518
|
+
lowered_by: failing.map((f) => f.fact),
|
|
519
|
+
};
|
|
401
520
|
}
|
|
402
521
|
|
|
403
522
|
/**
|
|
404
523
|
* Read a judge's verdict from an artifact: the LAST `pan-verdict` block wins; with
|
|
405
524
|
* no block, a verification.md frontmatter `status` is read through the adapter.
|
|
525
|
+
* Code facts (`opts.facts`, plus a verification's own `test_gate`) are applied
|
|
526
|
+
* after parsing through lowerByCodeFacts.
|
|
406
527
|
* @param {string} text
|
|
407
528
|
* @param {{agent?: string, phase?: string|number}} [opts]
|
|
408
529
|
* @returns {{ok: true, verdict: Object, warnings: string[], source_kind: 'block'|'frontmatter'}
|
|
@@ -418,7 +539,9 @@ function parseVerdictText(text, opts = {}) {
|
|
|
418
539
|
return { ok: false, error: 'invalid_json', reason: `the last pan-verdict block is not valid JSON: ${e.message}` };
|
|
419
540
|
}
|
|
420
541
|
const r = validateVerdict(obj, opts);
|
|
421
|
-
|
|
542
|
+
if (!r.ok) return r;
|
|
543
|
+
const low = lowerByCodeFacts(r.verdict, opts.facts);
|
|
544
|
+
return { ...r, verdict: low.verdict, lowered_by: low.lowered_by, facts_ok: passingFacts(opts.facts), source_kind: 'block' };
|
|
422
545
|
}
|
|
423
546
|
const adapted = verdictFromVerificationFrontmatter(text, opts.agent || 'pan-verifier');
|
|
424
547
|
if (adapted) {
|
|
@@ -431,9 +554,13 @@ function parseVerdictText(text, opts = {}) {
|
|
|
431
554
|
// gate under a pass: both make "passed" claim more than the verifier checked.
|
|
432
555
|
if (!adapted.test_gate) r.warnings.push('test_gate_unrecorded: the verification does not say whether the test suite ran');
|
|
433
556
|
else if (!TEST_GATE_VALUES.includes(adapted.test_gate)) r.warnings.push(`test_gate_unknown: "${adapted.test_gate}" is not passed, failed or skipped`);
|
|
434
|
-
else if (adapted.test_gate === 'failed' && adapted.outcome === 'pass') r.warnings.push('test_gate_failed_under_pass: the test gate failed but the status is passed');
|
|
557
|
+
else if (adapted.test_gate === 'failed' && adapted.outcome === 'pass') r.warnings.push('test_gate_failed_under_pass: the test gate failed but the status is passed, so it records as gaps_found');
|
|
435
558
|
if (adapted.not_checked_contradicted) r.warnings.push(`not_checked_contradicted: test_gate is ${adapted.test_gate}, so the not_checked tests entry was dropped`);
|
|
436
|
-
|
|
559
|
+
const facts = [...(Array.isArray(opts.facts) ? opts.facts : [])];
|
|
560
|
+
if (adapted.test_gate === 'failed') facts.push({ fact: 'test_gate', ok: false, summary: 'The test gate failed: the suite reported failures or did not run' });
|
|
561
|
+
if (adapted.test_gate === 'passed') facts.push({ fact: 'test_gate', ok: true, summary: 'the test gate passed' });
|
|
562
|
+
const low = lowerByCodeFacts(r.verdict, facts);
|
|
563
|
+
return { ...r, verdict: low.verdict, lowered_by: low.lowered_by, facts_ok: passingFacts(facts), source_kind: 'frontmatter' };
|
|
437
564
|
}
|
|
438
565
|
return { ok: false, error: 'no_verdict', reason: 'no pan-verdict block and no verification frontmatter status' };
|
|
439
566
|
}
|
|
@@ -466,13 +593,16 @@ module.exports = {
|
|
|
466
593
|
SEVERITIES,
|
|
467
594
|
DISPOSITIONS,
|
|
468
595
|
REASON_REQUIRED,
|
|
469
|
-
AUTO_FIX_EXEMPT,
|
|
470
596
|
KNOWN_VERDICTS,
|
|
471
597
|
LIMITS,
|
|
598
|
+
RESOLUTION_STATUSES,
|
|
599
|
+
FINDING_ID_RE,
|
|
472
600
|
findVerdictBlocks,
|
|
473
601
|
validateVerdict,
|
|
474
602
|
verdictFromVerificationFrontmatter,
|
|
475
603
|
parseVerdictText,
|
|
604
|
+
lowerByCodeFacts,
|
|
605
|
+
failWordFor,
|
|
476
606
|
recordSig,
|
|
477
607
|
findingId,
|
|
478
608
|
outcomeForVerdict,
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/**
|
|
3
|
+
* Test integrity (program V3): did this phase weaken the tests that judge it?
|
|
4
|
+
*
|
|
5
|
+
* An agent that cannot make a test pass can make the test stop failing instead:
|
|
6
|
+
* delete it, skip it, drop its assertions, rewrite its expectation or snapshot, or
|
|
7
|
+
* add an assertion that cannot fail. ImpossibleBench measured agents doing this on a
|
|
8
|
+
* large share of impossible tasks, and a stated way out (stop and report the
|
|
9
|
+
* conflict) cut it sharply. This is the check in code; the executor's way out is
|
|
10
|
+
* the `BLOCKED_SPEC_CONFLICT` return.
|
|
11
|
+
*
|
|
12
|
+
* It walks the phase's own plan commits (the `{type}({phase}-{plan}):` subjects that
|
|
13
|
+
* verify-scope reads) one at a time and reads each test file's diff:
|
|
14
|
+
*
|
|
15
|
+
* errors test_deleted · skip_marker_added · assertions_dropped ·
|
|
16
|
+
* undeclared_test_edit (an existing assertion changed in a test no plan
|
|
17
|
+
* declared) · snapshot_rewritten (undeclared) · always_true_assert
|
|
18
|
+
* warnings test_renamed
|
|
19
|
+
*
|
|
20
|
+
* A file a plan declares in `files_modified` may be edited freely, except that an
|
|
21
|
+
* added skip marker or an always-true assertion is an error anywhere.
|
|
22
|
+
*/
|
|
23
|
+
const fs = require('fs');
|
|
24
|
+
const path = require('path');
|
|
25
|
+
const { execGit, findPhaseInternal, safeReadFile, output, error, toPosix } = require('./core.cjs');
|
|
26
|
+
const { extractFrontmatter } = require('./frontmatter.cjs');
|
|
27
|
+
const { isPlanFile } = require('./constants.cjs');
|
|
28
|
+
const { committedFiles, TEST_PATH_RE, TEST_FILE_RE } = require('./verify-scope.cjs');
|
|
29
|
+
|
|
30
|
+
const TEST_NAME_RE = /(^|\/)(test_[^/]+\.py|[^/]+_test\.(py|go)|[^/]+Tests?\.(java|kt|cs))$/;
|
|
31
|
+
// A snapshot directory anywhere in the path, or a `.snap` file: two patterns, so neither
|
|
32
|
+
// alternative's anchoring reads as the other's.
|
|
33
|
+
const SNAPSHOT_DIR_RE = /(?:^|\/)(?:__snapshots__|snapshots)\//;
|
|
34
|
+
const SNAPSHOT_EXT_RE = /\.snap$/;
|
|
35
|
+
|
|
36
|
+
/** Markers that stop a test from running, per ecosystem. */
|
|
37
|
+
const SKIP_RE = [
|
|
38
|
+
/\b(?:it|test|describe|suite|context)\.(?:skip|only|todo)\s*\(/, // jest / mocha / vitest / node:test
|
|
39
|
+
/\bx(?:it|test|describe)\s*\(/, // jasmine / mocha
|
|
40
|
+
/\b(?:skip|only|todo)\s*:\s*(?:true|['"`])/, // node:test options
|
|
41
|
+
/@pytest\.mark\.(?:skip|skipif|xfail)\b/,
|
|
42
|
+
/\bpytest\.skip\s*\(/,
|
|
43
|
+
/@unittest\.skip\w*\b/,
|
|
44
|
+
/\bt\.Skip(?:Now|f)?\s*\(/, // go
|
|
45
|
+
/#\[ignore\]/, // rust
|
|
46
|
+
/@(?:Disabled|Ignore)\b/, // junit
|
|
47
|
+
];
|
|
48
|
+
|
|
49
|
+
/** Lines that assert, per ecosystem. */
|
|
50
|
+
const ASSERT_RE = [
|
|
51
|
+
/\bassert(?:\.\w+)?\s*\(/,
|
|
52
|
+
/\bassert(?:_eq|_ne)?!\s*\(/,
|
|
53
|
+
/\bexpect\s*\(/,
|
|
54
|
+
/\.should\b/,
|
|
55
|
+
/\bself\.assert\w+\s*\(/,
|
|
56
|
+
/^\s*assert\s+\S/,
|
|
57
|
+
/\bt\.(?:Error|Errorf|Fatal|Fatalf|Fail|FailNow)\s*\(/,
|
|
58
|
+
/\b(?:require|assert)\.\w+\s*\(\s*t\b/, // testify
|
|
59
|
+
/\bAssert\.\w+\s*\(/,
|
|
60
|
+
];
|
|
61
|
+
|
|
62
|
+
/** Assertions that cannot fail. */
|
|
63
|
+
const ALWAYS_TRUE_RE = [
|
|
64
|
+
/\bassert(?:\.ok)?\s*\(\s*true\s*[,)]/,
|
|
65
|
+
/\bexpect\s*\(\s*true\s*\)\s*\.\s*(?:toBe|toEqual|toBeTruthy)\s*\(\s*(?:true)?\s*\)/,
|
|
66
|
+
/^\s*assert\s+True\s*(?:#.*)?$/,
|
|
67
|
+
/^\s*assert\s+(\d+)\s*==\s*\1\s*(?:#.*)?$/,
|
|
68
|
+
/\bself\.assertTrue\s*\(\s*True\s*\)/,
|
|
69
|
+
/\bassert!\s*\(\s*true\s*\)/,
|
|
70
|
+
/\bassertTrue\s*\(\s*true\s*\)/,
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
const isTestFile = (f) => TEST_PATH_RE.test(f) || TEST_FILE_RE.test(f) || TEST_NAME_RE.test(f);
|
|
74
|
+
const isSnapshot = (f) => SNAPSHOT_DIR_RE.test(f) || SNAPSHOT_EXT_RE.test(f);
|
|
75
|
+
const any = (res, line) => res.some((re) => re.test(line));
|
|
76
|
+
const norm = (p) => toPosix(String(p || '').trim()).replace(/^\.\/+/, '');
|
|
77
|
+
|
|
78
|
+
/** A Rust source file that holds its own unit tests (`#[test]`) at this commit. */
|
|
79
|
+
function hasInlineTests(cwd, hash, file) {
|
|
80
|
+
if (!file.endsWith('.rs')) return false;
|
|
81
|
+
const r = execGit(cwd, ['show', `${hash}:${file}`]);
|
|
82
|
+
return r.exitCode === 0 && /#\[(?:test|cfg\(test\))\]/.test(r.stdout);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** The union of the phase plans' `files_modified`. */
|
|
86
|
+
function declaredFiles(cwd, info) {
|
|
87
|
+
const dir = path.join(cwd, info.directory);
|
|
88
|
+
const out = new Set();
|
|
89
|
+
for (const f of info.plans.filter(isPlanFile)) {
|
|
90
|
+
const fm = extractFrontmatter(safeReadFile(path.join(dir, f)) || '');
|
|
91
|
+
const raw = fm.files_modified || fm['files-modified'];
|
|
92
|
+
for (const p of Array.isArray(raw) ? raw : (raw ? [raw] : [])) if (norm(p)) out.add(norm(p));
|
|
93
|
+
}
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function isDeclared(file, declared) {
|
|
98
|
+
for (const d of declared) if (d === file || (d.endsWith('/') && file.startsWith(d))) return true;
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** One commit's file changes: `[{status, file, from?}]`, renames detected. */
|
|
103
|
+
function commitChanges(cwd, hash) {
|
|
104
|
+
const r = execGit(cwd, ['-c', 'core.quotepath=off', 'show', '--relative', '-M', '--name-status', '--format=', hash]);
|
|
105
|
+
if (r.exitCode !== 0) return [];
|
|
106
|
+
return r.stdout.split(/\r?\n/).filter(Boolean).map((line) => {
|
|
107
|
+
const parts = line.split('\t');
|
|
108
|
+
const status = parts[0][0];
|
|
109
|
+
return status === 'R' || status === 'C'
|
|
110
|
+
? { status, from: norm(parts[1]), file: norm(parts[2]) }
|
|
111
|
+
: { status, file: norm(parts[1]) };
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** One file's added and removed lines in one commit. */
|
|
116
|
+
function fileDiff(cwd, hash, file) {
|
|
117
|
+
const r = execGit(cwd, ['-c', 'core.quotepath=off', 'show', '--relative', '--format=', '-U0', '--no-color', hash, '--', file]);
|
|
118
|
+
const added = [];
|
|
119
|
+
const removed = [];
|
|
120
|
+
if (r.exitCode !== 0) return { added, removed };
|
|
121
|
+
for (const line of r.stdout.split(/\r?\n/)) {
|
|
122
|
+
if (line.startsWith('+++') || line.startsWith('---')) continue;
|
|
123
|
+
if (line.startsWith('+')) added.push(line.slice(1));
|
|
124
|
+
else if (line.startsWith('-')) removed.push(line.slice(1));
|
|
125
|
+
}
|
|
126
|
+
return { added, removed };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The integrity check for one phase.
|
|
131
|
+
* @returns {{phase, checked: boolean, ok: boolean, reason?: string, commits: number, violations: Array, warnings: Array}|{error, reason}}
|
|
132
|
+
*/
|
|
133
|
+
function checkTestIntegrity(cwd, phase) {
|
|
134
|
+
const info = findPhaseInternal(cwd, phase);
|
|
135
|
+
if (!info || !info.found) return { error: 'phase_not_found', reason: `no phase directory for phase ${phase}` };
|
|
136
|
+
const git = committedFiles(cwd, info.phase_number);
|
|
137
|
+
const base = { phase: info.phase_number, commits: git.commits, violations: [], warnings: [] };
|
|
138
|
+
if (!git.git) return { ...base, checked: false, ok: true, reason: 'git is unavailable or this is not a repository' };
|
|
139
|
+
if (!git.commits) return { ...base, checked: false, ok: true, reason: 'no plan commits for this phase (subjects like "feat(03-01): …")' };
|
|
140
|
+
|
|
141
|
+
const declared = declaredFiles(cwd, info);
|
|
142
|
+
const violations = [];
|
|
143
|
+
const warnings = [];
|
|
144
|
+
const add = (list, code, file, commit, detail) => list.push({ code, file, commit: commit.slice(0, 12), detail });
|
|
145
|
+
|
|
146
|
+
for (const hash of [...git.hashes].reverse()) { // oldest first
|
|
147
|
+
for (const ch of commitChanges(cwd, hash)) {
|
|
148
|
+
const { file } = ch;
|
|
149
|
+
if (ch.status === 'D' && isTestFile(file)) { add(violations, 'test_deleted', file, hash, `${file} was deleted`); continue; }
|
|
150
|
+
if (ch.status === 'R' && isTestFile(ch.from)) add(warnings, 'test_renamed', file, hash, `${ch.from} was renamed to ${file}`);
|
|
151
|
+
if (isSnapshot(file) && (ch.status === 'M' || ch.status === 'D') && !isDeclared(file, declared)) {
|
|
152
|
+
add(violations, 'snapshot_rewritten', file, hash, `snapshot ${file} was ${ch.status === 'D' ? 'deleted' : 'rewritten'} and no plan declares it`);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
if (ch.status === 'D' || !(isTestFile(file) || hasInlineTests(cwd, hash, file))) continue;
|
|
156
|
+
const { added, removed } = fileDiff(cwd, hash, file);
|
|
157
|
+
const skipsAdded = added.filter((l) => any(SKIP_RE, l)).length - removed.filter((l) => any(SKIP_RE, l)).length;
|
|
158
|
+
if (skipsAdded > 0) add(violations, 'skip_marker_added', file, hash, `${skipsAdded} skip/only marker(s) added`);
|
|
159
|
+
const trivial = added.filter((l) => any(ALWAYS_TRUE_RE, l));
|
|
160
|
+
if (trivial.length) add(violations, 'always_true_assert', file, hash, `an assertion that cannot fail: ${trivial[0].trim().slice(0, 120)}`);
|
|
161
|
+
const assertsAdded = added.filter((l) => any(ASSERT_RE, l) && !any(ALWAYS_TRUE_RE, l)).length;
|
|
162
|
+
const assertsRemoved = removed.filter((l) => any(ASSERT_RE, l)).length;
|
|
163
|
+
if (assertsRemoved > assertsAdded) {
|
|
164
|
+
add(violations, 'assertions_dropped', file, hash, `${assertsRemoved} assertion line(s) removed, ${assertsAdded} added`);
|
|
165
|
+
} else if (assertsRemoved > 0 && ch.status === 'M' && !isDeclared(file, declared)) {
|
|
166
|
+
add(violations, 'undeclared_test_edit', file, hash, `${assertsRemoved} existing assertion(s) changed in a test no plan declares`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return { ...base, checked: true, ok: violations.length === 0, violations, warnings };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function cmdVerifyTestIntegrity(cwd, phase, raw) {
|
|
174
|
+
if (!phase) error('Usage: verify test-integrity <phase>');
|
|
175
|
+
const r = checkTestIntegrity(cwd, phase);
|
|
176
|
+
if (r.error) { output(r, raw); return; }
|
|
177
|
+
const word = !r.checked ? 'not_checked' : (r.ok ? 'clean' : 'violated');
|
|
178
|
+
output(r, raw, word, r.ok ? 0 : 1);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
module.exports = { checkTestIntegrity, cmdVerifyTestIntegrity, isTestFile, SKIP_RE, ASSERT_RE, ALWAYS_TRUE_RE };
|