@olegkoval/agent-skills 1.62.0 → 1.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/claude/olko-github-pr/skills/lekker-review/SKILL.md +10 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/evals/manifest.json +16 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +93 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/selftest.mjs +25 -4
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/SKILL.md +10 -0
- package/plugins/olko-github-pr/skills/lekker-review/evals/manifest.json +16 -0
- package/plugins/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +93 -0
- package/plugins/olko-github-pr/skills/lekker-review/scripts/selftest.mjs +25 -4
- package/plugins/olko-github-pr/skills/lekker-review/workflow.js +14 -1
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
- package/scripts/build-adapters.sh +1 -1
|
@@ -1325,6 +1325,16 @@ is not failure, the review may find things the bots never would. `0 bot
|
|
|
1325
1325
|
comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
|
|
1326
1326
|
turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
|
|
1327
1327
|
|
|
1328
|
+
**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
|
|
1329
|
+
"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
|
|
1330
|
+
each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
|
|
1331
|
+
`node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
|
|
1332
|
+
prints recall on buggy PRs and the false-positive count (critical or important
|
|
1333
|
+
findings) on clean ones. The shipped manifest holds two placeholders; choosing
|
|
1334
|
+
real PRs is a human job. A hit is location-only (same file, within 10 lines),
|
|
1335
|
+
so a nearby finding about a different problem still counts; read recall as an
|
|
1336
|
+
upper bound.
|
|
1337
|
+
|
|
1328
1338
|
If you maintain a persistent notes/memory system across reviews, launch ONE
|
|
1329
1339
|
background Agent with `model: 'haiku'`, passing the review file path +
|
|
1330
1340
|
repo-short-name + PR author login, instructed to follow
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
|
|
3
|
+
"prs": [
|
|
4
|
+
{
|
|
5
|
+
"repo": "example-org/example-repo",
|
|
6
|
+
"pr": 1,
|
|
7
|
+
"expect": "buggy",
|
|
8
|
+
"knownBug": "src/orders/sync.ts:42 off-by-one drops the last line item"
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"repo": "example-org/example-repo",
|
|
12
|
+
"pr": 2,
|
|
13
|
+
"expect": "clean"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Score lekker findings against a manifest of PRs with known outcomes.
|
|
3
|
+
//
|
|
4
|
+
// Usage:
|
|
5
|
+
// node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
|
|
6
|
+
//
|
|
7
|
+
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
|
|
8
|
+
// results.json: { "<repo>#<pr>": [ <finding>, ... ] } the findings of one lekker run per PR
|
|
9
|
+
// (the `findings` array the workflow returns). A PR missing from results is
|
|
10
|
+
// reported as unscored, never as zero findings.
|
|
11
|
+
//
|
|
12
|
+
// A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
|
|
13
|
+
// of the knownBug line. Known limitation: the match is by location only, so a
|
|
14
|
+
// same-file finding near the line counts even if it describes a different problem;
|
|
15
|
+
// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits / scored buggy PRs. On a clean PR every
|
|
16
|
+
// verdict-affecting finding (critical or important) is a false positive.
|
|
17
|
+
|
|
18
|
+
import { readFileSync } from 'node:fs';
|
|
19
|
+
import { fileURLToPath } from 'node:url';
|
|
20
|
+
|
|
21
|
+
export const LINE_WINDOW = 10;
|
|
22
|
+
const BLOCKING = new Set(['critical', 'important']);
|
|
23
|
+
|
|
24
|
+
export function parseKnownBug(text) {
|
|
25
|
+
// The file is everything before the last ":<digits>" that is followed by whitespace
|
|
26
|
+
// or the end, so paths containing spaces parse.
|
|
27
|
+
const m = /^(.+):(\d+)(?=\s|$)/.exec(String(text || '').trim());
|
|
28
|
+
if (!m) return null;
|
|
29
|
+
const line = Number(m[2]);
|
|
30
|
+
return line >= 1 ? { file: m[1], line } : null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function isHit(findings, knownBug) {
|
|
34
|
+
const spot = parseKnownBug(knownBug);
|
|
35
|
+
if (!spot) return false;
|
|
36
|
+
return findings.some((f) => f.file === spot.file
|
|
37
|
+
&& Number.isInteger(f.line) && f.line > 0
|
|
38
|
+
&& Math.abs(f.line - spot.line) <= LINE_WINDOW);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function validateManifest(manifest) {
|
|
42
|
+
const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
|
|
43
|
+
if (!prs) throw new Error('manifest needs a "prs" array');
|
|
44
|
+
const seen = new Set();
|
|
45
|
+
prs.forEach((e, i) => {
|
|
46
|
+
if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
|
|
47
|
+
if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
|
|
48
|
+
const key = `${e.repo}#${e.pr}`;
|
|
49
|
+
if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
|
|
50
|
+
seen.add(key);
|
|
51
|
+
if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
|
|
52
|
+
throw new Error(`prs[${i}]: a buggy entry needs knownBug starting with "file:line" (line >= 1)`);
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
return prs;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function score(manifest, results) {
|
|
59
|
+
const rows = validateManifest(manifest).map((e) => {
|
|
60
|
+
const key = `${e.repo}#${e.pr}`;
|
|
61
|
+
const findings = results[key];
|
|
62
|
+
if (!Array.isArray(findings)) return { key, expect: e.expect, scored: false };
|
|
63
|
+
if (e.expect === 'buggy') return { key, expect: 'buggy', scored: true, hit: isHit(findings, e.knownBug) };
|
|
64
|
+
return {
|
|
65
|
+
key, expect: 'clean', scored: true,
|
|
66
|
+
falsePositives: findings.filter((f) => BLOCKING.has(f.severity)).length,
|
|
67
|
+
};
|
|
68
|
+
});
|
|
69
|
+
const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
|
|
70
|
+
const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
|
|
71
|
+
return {
|
|
72
|
+
recall: buggy.length === 0 ? null : buggy.filter((r) => r.hit).length / buggy.length,
|
|
73
|
+
buggyScored: buggy.length,
|
|
74
|
+
falsePositives: clean.reduce((n, r) => n + r.falsePositives, 0),
|
|
75
|
+
cleanScored: clean.length,
|
|
76
|
+
unscored: rows.filter((r) => !r.scored).map((r) => r.key),
|
|
77
|
+
rows,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function main(argv) {
|
|
82
|
+
const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
|
|
83
|
+
if (!flag('manifest') || !flag('results')) {
|
|
84
|
+
console.error('usage: eval-runner.mjs --manifest <manifest.json> --results <results.json> [--json]');
|
|
85
|
+
process.exit(2);
|
|
86
|
+
}
|
|
87
|
+
const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
|
|
88
|
+
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
89
|
+
const pct = out.recall === null ? 'n/a' : `${Math.round(out.recall * 100)}%`;
|
|
90
|
+
console.log(`recall ${pct} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -53,13 +53,19 @@ const preamble = [
|
|
|
53
53
|
// hardRuleCorroborated is likewise gone from the current file and kept in this
|
|
54
54
|
// list for the same reason: an older workflow.js calls it from isHardRule.
|
|
55
55
|
...['titleTokens', 'sameIssue', 'nearbyLines', 'spanWithinWindow', 'hardRuleCorroborated',
|
|
56
|
-
'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify'].map(n => {
|
|
57
|
-
|
|
56
|
+
'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify', 'isProveAttempt', 'countProveAttempt'].map(n => {
|
|
57
|
+
// countProveAttempt is called directly by the checks, so its stub returns NaN and the
|
|
58
|
+
// checks FAIL with a diff instead of the run crashing on an older workflow.js.
|
|
59
|
+
try { return lift(n) } catch {
|
|
60
|
+
return n === 'countProveAttempt'
|
|
61
|
+
? 'function countProveAttempt() { return NaN }'
|
|
62
|
+
: `function ${n}() { throw new Error('${n} absent from this workflow.js') }`
|
|
63
|
+
}
|
|
58
64
|
}),
|
|
59
65
|
].join('\n')
|
|
60
66
|
|
|
61
|
-
const { dedup, isHardRule, shouldVerify, sameIssue } =
|
|
62
|
-
new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue }')()
|
|
67
|
+
const { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt } =
|
|
68
|
+
new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt }')()
|
|
63
69
|
|
|
64
70
|
let failed = 0
|
|
65
71
|
function check(name, actual, expected) {
|
|
@@ -676,5 +682,20 @@ console.log('\nprove: skipped without an installed test runner')
|
|
|
676
682
|
rmSync(tmp, { recursive: true, force: true })
|
|
677
683
|
}
|
|
678
684
|
|
|
685
|
+
console.log('\nprove ledger: a declined prover is a skip, not an attempt')
|
|
686
|
+
// Observed: 6 critical findings, every prover answered attempted:false (no node_modules),
|
|
687
|
+
// and the ledger recorded proven 0/6 as if six tests had run and not failed.
|
|
688
|
+
check('not_attempted is not an attempt', isProveAttempt({ attempted: false, proven: false, outcome: 'not_attempted' }), false)
|
|
689
|
+
check('inconclusive run is an attempt', isProveAttempt({ attempted: true, proven: false, outcome: 'inconclusive' }), true)
|
|
690
|
+
check('proven run is an attempt', isProveAttempt({ attempted: true, proven: true, outcome: 'proven' }), true)
|
|
691
|
+
// The counter path itself, not just the predicate: a declined prover must leave the count unchanged.
|
|
692
|
+
check('ledger: null proof (prover failed or rejected) adds nothing', countProveAttempt(0, null), 0)
|
|
693
|
+
check('ledger: declined prover adds nothing', countProveAttempt(2, { attempted: false, proven: false, outcome: 'not_attempted' }), 2)
|
|
694
|
+
check('ledger: inconclusive prover adds one', countProveAttempt(2, { attempted: true, proven: false, outcome: 'inconclusive' }), 3)
|
|
695
|
+
check('ledger: proven prover adds one', countProveAttempt(2, { attempted: true, proven: true, outcome: 'proven' }), 3)
|
|
696
|
+
// Source guard: a bare increment would bypass the predicate, and no unit check above would notice.
|
|
697
|
+
const bareWrites = src.split('\n').filter(l => /proveAttemptCount\s*(\+\+|\+=|=[^=])/.test(l) && !/countProveAttempt\(|let proveAttemptCount/.test(l))
|
|
698
|
+
check('the attempt count is written only through countProveAttempt', bareWrites, [])
|
|
699
|
+
|
|
679
700
|
console.log(failed === 0 ? '\nall checks passed\n' : `\n${failed} check(s) FAILED\n`)
|
|
680
701
|
process.exit(failed === 0 ? 0 : 1)
|
package/package.json
CHANGED
|
@@ -1324,6 +1324,16 @@ is not failure, the review may find things the bots never would. `0 bot
|
|
|
1324
1324
|
comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
|
|
1325
1325
|
turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
|
|
1326
1326
|
|
|
1327
|
+
**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
|
|
1328
|
+
"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
|
|
1329
|
+
each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
|
|
1330
|
+
`node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
|
|
1331
|
+
prints recall on buggy PRs and the false-positive count (critical or important
|
|
1332
|
+
findings) on clean ones. The shipped manifest holds two placeholders; choosing
|
|
1333
|
+
real PRs is a human job. A hit is location-only (same file, within 10 lines),
|
|
1334
|
+
so a nearby finding about a different problem still counts; read recall as an
|
|
1335
|
+
upper bound.
|
|
1336
|
+
|
|
1327
1337
|
If you maintain a persistent notes/memory system across reviews, launch ONE
|
|
1328
1338
|
background Agent with `model: 'haiku'`, passing the review file path +
|
|
1329
1339
|
repo-short-name + PR author login, instructed to follow
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
|
|
3
|
+
"prs": [
|
|
4
|
+
{
|
|
5
|
+
"repo": "example-org/example-repo",
|
|
6
|
+
"pr": 1,
|
|
7
|
+
"expect": "buggy",
|
|
8
|
+
"knownBug": "src/orders/sync.ts:42 off-by-one drops the last line item"
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"repo": "example-org/example-repo",
|
|
12
|
+
"pr": 2,
|
|
13
|
+
"expect": "clean"
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Score lekker findings against a manifest of PRs with known outcomes.
|
|
3
|
+
//
|
|
4
|
+
// Usage:
|
|
5
|
+
// node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
|
|
6
|
+
//
|
|
7
|
+
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
|
|
8
|
+
// results.json: { "<repo>#<pr>": [ <finding>, ... ] } the findings of one lekker run per PR
|
|
9
|
+
// (the `findings` array the workflow returns). A PR missing from results is
|
|
10
|
+
// reported as unscored, never as zero findings.
|
|
11
|
+
//
|
|
12
|
+
// A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
|
|
13
|
+
// of the knownBug line. Known limitation: the match is by location only, so a
|
|
14
|
+
// same-file finding near the line counts even if it describes a different problem;
|
|
15
|
+
// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits / scored buggy PRs. On a clean PR every
|
|
16
|
+
// verdict-affecting finding (critical or important) is a false positive.
|
|
17
|
+
|
|
18
|
+
import { readFileSync } from 'node:fs';
|
|
19
|
+
import { fileURLToPath } from 'node:url';
|
|
20
|
+
|
|
21
|
+
export const LINE_WINDOW = 10;
|
|
22
|
+
const BLOCKING = new Set(['critical', 'important']);
|
|
23
|
+
|
|
24
|
+
export function parseKnownBug(text) {
|
|
25
|
+
// The file is everything before the last ":<digits>" that is followed by whitespace
|
|
26
|
+
// or the end, so paths containing spaces parse.
|
|
27
|
+
const m = /^(.+):(\d+)(?=\s|$)/.exec(String(text || '').trim());
|
|
28
|
+
if (!m) return null;
|
|
29
|
+
const line = Number(m[2]);
|
|
30
|
+
return line >= 1 ? { file: m[1], line } : null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function isHit(findings, knownBug) {
|
|
34
|
+
const spot = parseKnownBug(knownBug);
|
|
35
|
+
if (!spot) return false;
|
|
36
|
+
return findings.some((f) => f.file === spot.file
|
|
37
|
+
&& Number.isInteger(f.line) && f.line > 0
|
|
38
|
+
&& Math.abs(f.line - spot.line) <= LINE_WINDOW);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function validateManifest(manifest) {
|
|
42
|
+
const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
|
|
43
|
+
if (!prs) throw new Error('manifest needs a "prs" array');
|
|
44
|
+
const seen = new Set();
|
|
45
|
+
prs.forEach((e, i) => {
|
|
46
|
+
if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
|
|
47
|
+
if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
|
|
48
|
+
const key = `${e.repo}#${e.pr}`;
|
|
49
|
+
if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
|
|
50
|
+
seen.add(key);
|
|
51
|
+
if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
|
|
52
|
+
throw new Error(`prs[${i}]: a buggy entry needs knownBug starting with "file:line" (line >= 1)`);
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
return prs;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function score(manifest, results) {
|
|
59
|
+
const rows = validateManifest(manifest).map((e) => {
|
|
60
|
+
const key = `${e.repo}#${e.pr}`;
|
|
61
|
+
const findings = results[key];
|
|
62
|
+
if (!Array.isArray(findings)) return { key, expect: e.expect, scored: false };
|
|
63
|
+
if (e.expect === 'buggy') return { key, expect: 'buggy', scored: true, hit: isHit(findings, e.knownBug) };
|
|
64
|
+
return {
|
|
65
|
+
key, expect: 'clean', scored: true,
|
|
66
|
+
falsePositives: findings.filter((f) => BLOCKING.has(f.severity)).length,
|
|
67
|
+
};
|
|
68
|
+
});
|
|
69
|
+
const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
|
|
70
|
+
const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
|
|
71
|
+
return {
|
|
72
|
+
recall: buggy.length === 0 ? null : buggy.filter((r) => r.hit).length / buggy.length,
|
|
73
|
+
buggyScored: buggy.length,
|
|
74
|
+
falsePositives: clean.reduce((n, r) => n + r.falsePositives, 0),
|
|
75
|
+
cleanScored: clean.length,
|
|
76
|
+
unscored: rows.filter((r) => !r.scored).map((r) => r.key),
|
|
77
|
+
rows,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function main(argv) {
|
|
82
|
+
const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
|
|
83
|
+
if (!flag('manifest') || !flag('results')) {
|
|
84
|
+
console.error('usage: eval-runner.mjs --manifest <manifest.json> --results <results.json> [--json]');
|
|
85
|
+
process.exit(2);
|
|
86
|
+
}
|
|
87
|
+
const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
|
|
88
|
+
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
89
|
+
const pct = out.recall === null ? 'n/a' : `${Math.round(out.recall * 100)}%`;
|
|
90
|
+
console.log(`recall ${pct} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -53,13 +53,19 @@ const preamble = [
|
|
|
53
53
|
// hardRuleCorroborated is likewise gone from the current file and kept in this
|
|
54
54
|
// list for the same reason: an older workflow.js calls it from isHardRule.
|
|
55
55
|
...['titleTokens', 'sameIssue', 'nearbyLines', 'spanWithinWindow', 'hardRuleCorroborated',
|
|
56
|
-
'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify'].map(n => {
|
|
57
|
-
|
|
56
|
+
'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify', 'isProveAttempt', 'countProveAttempt'].map(n => {
|
|
57
|
+
// countProveAttempt is called directly by the checks, so its stub returns NaN and the
|
|
58
|
+
// checks FAIL with a diff instead of the run crashing on an older workflow.js.
|
|
59
|
+
try { return lift(n) } catch {
|
|
60
|
+
return n === 'countProveAttempt'
|
|
61
|
+
? 'function countProveAttempt() { return NaN }'
|
|
62
|
+
: `function ${n}() { throw new Error('${n} absent from this workflow.js') }`
|
|
63
|
+
}
|
|
58
64
|
}),
|
|
59
65
|
].join('\n')
|
|
60
66
|
|
|
61
|
-
const { dedup, isHardRule, shouldVerify, sameIssue } =
|
|
62
|
-
new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue }')()
|
|
67
|
+
const { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt } =
|
|
68
|
+
new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt }')()
|
|
63
69
|
|
|
64
70
|
let failed = 0
|
|
65
71
|
function check(name, actual, expected) {
|
|
@@ -676,5 +682,20 @@ console.log('\nprove: skipped without an installed test runner')
|
|
|
676
682
|
rmSync(tmp, { recursive: true, force: true })
|
|
677
683
|
}
|
|
678
684
|
|
|
685
|
+
console.log('\nprove ledger: a declined prover is a skip, not an attempt')
|
|
686
|
+
// Observed: 6 critical findings, every prover answered attempted:false (no node_modules),
|
|
687
|
+
// and the ledger recorded proven 0/6 as if six tests had run and not failed.
|
|
688
|
+
check('not_attempted is not an attempt', isProveAttempt({ attempted: false, proven: false, outcome: 'not_attempted' }), false)
|
|
689
|
+
check('inconclusive run is an attempt', isProveAttempt({ attempted: true, proven: false, outcome: 'inconclusive' }), true)
|
|
690
|
+
check('proven run is an attempt', isProveAttempt({ attempted: true, proven: true, outcome: 'proven' }), true)
|
|
691
|
+
// The counter path itself, not just the predicate: a declined prover must leave the count unchanged.
|
|
692
|
+
check('ledger: null proof (prover failed or rejected) adds nothing', countProveAttempt(0, null), 0)
|
|
693
|
+
check('ledger: declined prover adds nothing', countProveAttempt(2, { attempted: false, proven: false, outcome: 'not_attempted' }), 2)
|
|
694
|
+
check('ledger: inconclusive prover adds one', countProveAttempt(2, { attempted: true, proven: false, outcome: 'inconclusive' }), 3)
|
|
695
|
+
check('ledger: proven prover adds one', countProveAttempt(2, { attempted: true, proven: true, outcome: 'proven' }), 3)
|
|
696
|
+
// Source guard: a bare increment would bypass the predicate, and no unit check above would notice.
|
|
697
|
+
const bareWrites = src.split('\n').filter(l => /proveAttemptCount\s*(\+\+|\+=|=[^=])/.test(l) && !/countProveAttempt\(|let proveAttemptCount/.test(l))
|
|
698
|
+
check('the attempt count is written only through countProveAttempt', bareWrites, [])
|
|
699
|
+
|
|
679
700
|
console.log(failed === 0 ? '\nall checks passed\n' : `\n${failed} check(s) FAILED\n`)
|
|
680
701
|
process.exit(failed === 0 ? 0 : 1)
|
|
@@ -393,6 +393,19 @@ function normalizeProof(result) {
|
|
|
393
393
|
}
|
|
394
394
|
}
|
|
395
395
|
|
|
396
|
+
// A prover that declines at its testability gate (no runner, not importable)
|
|
397
|
+
// returns outcome 'not_attempted'. That is a skip, not a failed attempt, so it
|
|
398
|
+
// must not inflate the proven/attempts ratio in the run ledger.
|
|
399
|
+
function isProveAttempt(proof) {
|
|
400
|
+
return proof.attempted === true
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// The only writer of the run ledger's attempt count. A null proof means the
|
|
404
|
+
// prover agent failed or was rejected, so there is no evidence a test ran.
|
|
405
|
+
function countProveAttempt(count, proof) {
|
|
406
|
+
return Boolean(proof) && isProveAttempt(proof) ? count + 1 : count
|
|
407
|
+
}
|
|
408
|
+
|
|
396
409
|
// Run thunks in sequential batches, giving the per-finding stages a ceiling.
|
|
397
410
|
//
|
|
398
411
|
// The review stage is bounded by its dimension count, but verify, critic and
|
|
@@ -951,7 +964,6 @@ const budgetAtStart = budget.spent()
|
|
|
951
964
|
await batched(capped.map(function(finding) {
|
|
952
965
|
return async function() {
|
|
953
966
|
agentCount++
|
|
954
|
-
proveAttemptCount++
|
|
955
967
|
const result = await agent(proverPrompt(finding), {
|
|
956
968
|
label: 'prove:' + finding.file + ':' + finding.line,
|
|
957
969
|
phase: 'Prove',
|
|
@@ -968,6 +980,7 @@ const budgetAtStart = budget.spent()
|
|
|
968
980
|
|
|
969
981
|
const proof = normalizeProof(result)
|
|
970
982
|
finding.proof = proof
|
|
983
|
+
proveAttemptCount = countProveAttempt(proveAttemptCount, proof)
|
|
971
984
|
if (proof.proven === true) {
|
|
972
985
|
provenCount++
|
|
973
986
|
finding.verificationStatus = 'proven'
|
|
@@ -28,7 +28,7 @@ const writeJson = (p, value) => {
|
|
|
28
28
|
writeFileSync(p, JSON.stringify(value, null, 2) + '\n')
|
|
29
29
|
}
|
|
30
30
|
const copyResources = (skill, destDir) => {
|
|
31
|
-
for (const d of ['scripts', 'references', 'templates']) {
|
|
31
|
+
for (const d of ['scripts', 'references', 'templates', 'evals']) {
|
|
32
32
|
const src = join(root, skill.path, d)
|
|
33
33
|
if (!existsSync(src)) continue
|
|
34
34
|
const dest = join(destDir, d)
|