@olegkoval/agent-skills 1.62.0 → 1.63.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/adapters/claude/olko-github-pr/skills/lekker-review/SKILL.md +10 -0
  2. package/adapters/claude/olko-github-pr/skills/lekker-review/evals/manifest.json +16 -0
  3. package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +93 -0
  4. package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/selftest.mjs +25 -4
  5. package/package.json +1 -1
  6. package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
  7. package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
  8. package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
  9. package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
  10. package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
  11. package/plugins/olko-github-pr/skills/lekker-review/SKILL.md +10 -0
  12. package/plugins/olko-github-pr/skills/lekker-review/evals/manifest.json +16 -0
  13. package/plugins/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +93 -0
  14. package/plugins/olko-github-pr/skills/lekker-review/scripts/selftest.mjs +25 -4
  15. package/plugins/olko-github-pr/skills/lekker-review/workflow.js +14 -1
  16. package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
  17. package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
  18. package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
  19. package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
  20. package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
  21. package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
  22. package/scripts/build-adapters.sh +1 -1
@@ -1325,6 +1325,16 @@ is not failure, the review may find things the bots never would. `0 bot
1325
1325
  comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
1326
1326
  turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
1327
1327
 
1328
+ **Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
1329
+ "buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
1330
+ each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
1331
+ `node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
1332
+ prints recall on buggy PRs and the false-positive count (critical or important
1333
+ findings) on clean ones. The shipped manifest holds two placeholders; choosing
1334
+ real PRs is a human job. A hit is location-only (same file, within 10 lines),
1335
+ so a nearby finding about a different problem still counts; read recall as an
1336
+ upper bound.
1337
+
1328
1338
  If you maintain a persistent notes/memory system across reviews, launch ONE
1329
1339
  background Agent with `model: 'haiku'`, passing the review file path +
1330
1340
  repo-short-name + PR author login, instructed to follow
@@ -0,0 +1,16 @@
1
+ {
2
+ "note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
3
+ "prs": [
4
+ {
5
+ "repo": "example-org/example-repo",
6
+ "pr": 1,
7
+ "expect": "buggy",
8
+ "knownBug": "src/orders/sync.ts:42 off-by-one drops the last line item"
9
+ },
10
+ {
11
+ "repo": "example-org/example-repo",
12
+ "pr": 2,
13
+ "expect": "clean"
14
+ }
15
+ ]
16
+ }
@@ -0,0 +1,93 @@
1
+ #!/usr/bin/env node
2
+ // Score lekker findings against a manifest of PRs with known outcomes.
3
+ //
4
+ // Usage:
5
+ // node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
6
+ //
7
+ // manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
8
+ // results.json: { "<repo>#<pr>": [ <finding>, ... ] } the findings of one lekker run per PR
9
+ // (the `findings` array the workflow returns). A PR missing from results is
10
+ // reported as unscored, never as zero findings.
11
+ //
12
+ // A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
13
+ // of the knownBug line. Known limitation: the match is by location only, so a
14
+ // same-file finding near the line counts even if it describes a different problem;
15
+ // judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits / scored buggy PRs. On a clean PR every
16
+ // verdict-affecting finding (critical or important) is a false positive.
17
+
18
+ import { readFileSync } from 'node:fs';
19
+ import { fileURLToPath } from 'node:url';
20
+
21
+ export const LINE_WINDOW = 10;
22
+ const BLOCKING = new Set(['critical', 'important']);
23
+
24
+ export function parseKnownBug(text) {
25
+ // The file is everything before the last ":<digits>" that is followed by whitespace
26
+ // or the end, so paths containing spaces parse.
27
+ const m = /^(.+):(\d+)(?=\s|$)/.exec(String(text || '').trim());
28
+ if (!m) return null;
29
+ const line = Number(m[2]);
30
+ return line >= 1 ? { file: m[1], line } : null;
31
+ }
32
+
33
+ export function isHit(findings, knownBug) {
34
+ const spot = parseKnownBug(knownBug);
35
+ if (!spot) return false;
36
+ return findings.some((f) => f.file === spot.file
37
+ && Number.isInteger(f.line) && f.line > 0
38
+ && Math.abs(f.line - spot.line) <= LINE_WINDOW);
39
+ }
40
+
41
+ export function validateManifest(manifest) {
42
+ const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
43
+ if (!prs) throw new Error('manifest needs a "prs" array');
44
+ const seen = new Set();
45
+ prs.forEach((e, i) => {
46
+ if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
47
+ if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
48
+ const key = `${e.repo}#${e.pr}`;
49
+ if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
50
+ seen.add(key);
51
+ if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
52
+ throw new Error(`prs[${i}]: a buggy entry needs knownBug starting with "file:line" (line >= 1)`);
53
+ }
54
+ });
55
+ return prs;
56
+ }
57
+
58
+ export function score(manifest, results) {
59
+ const rows = validateManifest(manifest).map((e) => {
60
+ const key = `${e.repo}#${e.pr}`;
61
+ const findings = results[key];
62
+ if (!Array.isArray(findings)) return { key, expect: e.expect, scored: false };
63
+ if (e.expect === 'buggy') return { key, expect: 'buggy', scored: true, hit: isHit(findings, e.knownBug) };
64
+ return {
65
+ key, expect: 'clean', scored: true,
66
+ falsePositives: findings.filter((f) => BLOCKING.has(f.severity)).length,
67
+ };
68
+ });
69
+ const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
70
+ const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
71
+ return {
72
+ recall: buggy.length === 0 ? null : buggy.filter((r) => r.hit).length / buggy.length,
73
+ buggyScored: buggy.length,
74
+ falsePositives: clean.reduce((n, r) => n + r.falsePositives, 0),
75
+ cleanScored: clean.length,
76
+ unscored: rows.filter((r) => !r.scored).map((r) => r.key),
77
+ rows,
78
+ };
79
+ }
80
+
81
+ function main(argv) {
82
+ const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
83
+ if (!flag('manifest') || !flag('results')) {
84
+ console.error('usage: eval-runner.mjs --manifest <manifest.json> --results <results.json> [--json]');
85
+ process.exit(2);
86
+ }
87
+ const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
88
+ if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
89
+ const pct = out.recall === null ? 'n/a' : `${Math.round(out.recall * 100)}%`;
90
+ console.log(`recall ${pct} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
91
+ }
92
+
93
+ if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
@@ -53,13 +53,19 @@ const preamble = [
53
53
  // hardRuleCorroborated is likewise gone from the current file and kept in this
54
54
  // list for the same reason: an older workflow.js calls it from isHardRule.
55
55
  ...['titleTokens', 'sameIssue', 'nearbyLines', 'spanWithinWindow', 'hardRuleCorroborated',
56
- 'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify'].map(n => {
57
- try { return lift(n) } catch { return `function ${n}() { throw new Error('${n} absent from this workflow.js') }` }
56
+ 'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify', 'isProveAttempt', 'countProveAttempt'].map(n => {
57
+ // countProveAttempt is called directly by the checks, so its stub returns NaN and the
58
+ // checks FAIL with a diff instead of the run crashing on an older workflow.js.
59
+ try { return lift(n) } catch {
60
+ return n === 'countProveAttempt'
61
+ ? 'function countProveAttempt() { return NaN }'
62
+ : `function ${n}() { throw new Error('${n} absent from this workflow.js') }`
63
+ }
58
64
  }),
59
65
  ].join('\n')
60
66
 
61
- const { dedup, isHardRule, shouldVerify, sameIssue } =
62
- new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue }')()
67
+ const { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt } =
68
+ new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt }')()
63
69
 
64
70
  let failed = 0
65
71
  function check(name, actual, expected) {
@@ -676,5 +682,20 @@ console.log('\nprove: skipped without an installed test runner')
676
682
  rmSync(tmp, { recursive: true, force: true })
677
683
  }
678
684
 
685
+ console.log('\nprove ledger: a declined prover is a skip, not an attempt')
686
+ // Observed: 6 critical findings, every prover answered attempted:false (no node_modules),
687
+ // and the ledger recorded proven 0/6 as if six tests had run and not failed.
688
+ check('not_attempted is not an attempt', isProveAttempt({ attempted: false, proven: false, outcome: 'not_attempted' }), false)
689
+ check('inconclusive run is an attempt', isProveAttempt({ attempted: true, proven: false, outcome: 'inconclusive' }), true)
690
+ check('proven run is an attempt', isProveAttempt({ attempted: true, proven: true, outcome: 'proven' }), true)
691
+ // The counter path itself, not just the predicate: a declined prover must leave the count unchanged.
692
+ check('ledger: null proof (prover failed or rejected) adds nothing', countProveAttempt(0, null), 0)
693
+ check('ledger: declined prover adds nothing', countProveAttempt(2, { attempted: false, proven: false, outcome: 'not_attempted' }), 2)
694
+ check('ledger: inconclusive prover adds one', countProveAttempt(2, { attempted: true, proven: false, outcome: 'inconclusive' }), 3)
695
+ check('ledger: proven prover adds one', countProveAttempt(2, { attempted: true, proven: true, outcome: 'proven' }), 3)
696
+ // Source guard: a bare increment would bypass the predicate, and no unit check above would notice.
697
+ const bareWrites = src.split('\n').filter(l => /proveAttemptCount\s*(\+\+|\+=|=[^=])/.test(l) && !/countProveAttempt\(|let proveAttemptCount/.test(l))
698
+ check('the attempt count is written only through countProveAttempt', bareWrites, [])
699
+
679
700
  console.log(failed === 0 ? '\nall checks passed\n' : `\n${failed} check(s) FAILED\n`)
680
701
  process.exit(failed === 0 ? 0 : 1)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@olegkoval/agent-skills",
3
- "version": "1.62.0",
3
+ "version": "1.63.0",
4
4
  "private": false,
5
5
  "publishConfig": {
6
6
  "access": "public"
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-apple-kit",
3
3
  "description": "Build and ship Apple platform apps: macOS menubar apps, App Store submissions.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-creative",
3
3
  "description": "Creative and personal projects: photo galleries, music players, listings, wiki editing.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-garmin-kit",
3
3
  "description": "Build, test and publish Garmin Connect IQ watch faces.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-git-tools",
3
3
  "description": "Everyday git and GitHub CLI operations: conventional commits, branch hygiene.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-github-pr",
3
3
  "description": "Drive GitHub pull requests to merge-ready: review-bot loops, CI fixes, descriptions, dependency triage.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1324,6 +1324,16 @@ is not failure, the review may find things the bots never would. `0 bot
1324
1324
  comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
1325
1325
  turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
1326
1326
 
1327
+ **Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
1328
+ "buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
1329
+ each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
1330
+ `node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
1331
+ prints recall on buggy PRs and the false-positive count (critical or important
1332
+ findings) on clean ones. The shipped manifest holds two placeholders; choosing
1333
+ real PRs is a human job. A hit is location-only (same file, within 10 lines),
1334
+ so a nearby finding about a different problem still counts; read recall as an
1335
+ upper bound.
1336
+
1327
1337
  If you maintain a persistent notes/memory system across reviews, launch ONE
1328
1338
  background Agent with `model: 'haiku'`, passing the review file path +
1329
1339
  repo-short-name + PR author login, instructed to follow
@@ -0,0 +1,16 @@
1
+ {
2
+ "note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
3
+ "prs": [
4
+ {
5
+ "repo": "example-org/example-repo",
6
+ "pr": 1,
7
+ "expect": "buggy",
8
+ "knownBug": "src/orders/sync.ts:42 off-by-one drops the last line item"
9
+ },
10
+ {
11
+ "repo": "example-org/example-repo",
12
+ "pr": 2,
13
+ "expect": "clean"
14
+ }
15
+ ]
16
+ }
@@ -0,0 +1,93 @@
1
+ #!/usr/bin/env node
2
+ // Score lekker findings against a manifest of PRs with known outcomes.
3
+ //
4
+ // Usage:
5
+ // node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
6
+ //
7
+ // manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
8
+ // results.json: { "<repo>#<pr>": [ <finding>, ... ] } the findings of one lekker run per PR
9
+ // (the `findings` array the workflow returns). A PR missing from results is
10
+ // reported as unscored, never as zero findings.
11
+ //
12
+ // A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
13
+ // of the knownBug line. Known limitation: the match is by location only, so a
14
+ // same-file finding near the line counts even if it describes a different problem;
15
+ // judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits / scored buggy PRs. On a clean PR every
16
+ // verdict-affecting finding (critical or important) is a false positive.
17
+
18
+ import { readFileSync } from 'node:fs';
19
+ import { fileURLToPath } from 'node:url';
20
+
21
+ export const LINE_WINDOW = 10;
22
+ const BLOCKING = new Set(['critical', 'important']);
23
+
24
+ export function parseKnownBug(text) {
25
+ // The file is everything before the last ":<digits>" that is followed by whitespace
26
+ // or the end, so paths containing spaces parse.
27
+ const m = /^(.+):(\d+)(?=\s|$)/.exec(String(text || '').trim());
28
+ if (!m) return null;
29
+ const line = Number(m[2]);
30
+ return line >= 1 ? { file: m[1], line } : null;
31
+ }
32
+
33
+ export function isHit(findings, knownBug) {
34
+ const spot = parseKnownBug(knownBug);
35
+ if (!spot) return false;
36
+ return findings.some((f) => f.file === spot.file
37
+ && Number.isInteger(f.line) && f.line > 0
38
+ && Math.abs(f.line - spot.line) <= LINE_WINDOW);
39
+ }
40
+
41
+ export function validateManifest(manifest) {
42
+ const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
43
+ if (!prs) throw new Error('manifest needs a "prs" array');
44
+ const seen = new Set();
45
+ prs.forEach((e, i) => {
46
+ if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
47
+ if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
48
+ const key = `${e.repo}#${e.pr}`;
49
+ if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
50
+ seen.add(key);
51
+ if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
52
+ throw new Error(`prs[${i}]: a buggy entry needs knownBug starting with "file:line" (line >= 1)`);
53
+ }
54
+ });
55
+ return prs;
56
+ }
57
+
58
+ export function score(manifest, results) {
59
+ const rows = validateManifest(manifest).map((e) => {
60
+ const key = `${e.repo}#${e.pr}`;
61
+ const findings = results[key];
62
+ if (!Array.isArray(findings)) return { key, expect: e.expect, scored: false };
63
+ if (e.expect === 'buggy') return { key, expect: 'buggy', scored: true, hit: isHit(findings, e.knownBug) };
64
+ return {
65
+ key, expect: 'clean', scored: true,
66
+ falsePositives: findings.filter((f) => BLOCKING.has(f.severity)).length,
67
+ };
68
+ });
69
+ const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
70
+ const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
71
+ return {
72
+ recall: buggy.length === 0 ? null : buggy.filter((r) => r.hit).length / buggy.length,
73
+ buggyScored: buggy.length,
74
+ falsePositives: clean.reduce((n, r) => n + r.falsePositives, 0),
75
+ cleanScored: clean.length,
76
+ unscored: rows.filter((r) => !r.scored).map((r) => r.key),
77
+ rows,
78
+ };
79
+ }
80
+
81
+ function main(argv) {
82
+ const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
83
+ if (!flag('manifest') || !flag('results')) {
84
+ console.error('usage: eval-runner.mjs --manifest <manifest.json> --results <results.json> [--json]');
85
+ process.exit(2);
86
+ }
87
+ const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
88
+ if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
89
+ const pct = out.recall === null ? 'n/a' : `${Math.round(out.recall * 100)}%`;
90
+ console.log(`recall ${pct} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
91
+ }
92
+
93
+ if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
@@ -53,13 +53,19 @@ const preamble = [
53
53
  // hardRuleCorroborated is likewise gone from the current file and kept in this
54
54
  // list for the same reason: an older workflow.js calls it from isHardRule.
55
55
  ...['titleTokens', 'sameIssue', 'nearbyLines', 'spanWithinWindow', 'hardRuleCorroborated',
56
- 'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify'].map(n => {
57
- try { return lift(n) } catch { return `function ${n}() { throw new Error('${n} absent from this workflow.js') }` }
56
+ 'isHardRule', 'longest', 'mergeFindings', 'dedup', 'shouldVerify', 'isProveAttempt', 'countProveAttempt'].map(n => {
57
+ // countProveAttempt is called directly by the checks, so its stub returns NaN and the
58
+ // checks FAIL with a diff instead of the run crashing on an older workflow.js.
59
+ try { return lift(n) } catch {
60
+ return n === 'countProveAttempt'
61
+ ? 'function countProveAttempt() { return NaN }'
62
+ : `function ${n}() { throw new Error('${n} absent from this workflow.js') }`
63
+ }
58
64
  }),
59
65
  ].join('\n')
60
66
 
61
- const { dedup, isHardRule, shouldVerify, sameIssue } =
62
- new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue }')()
67
+ const { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt } =
68
+ new Function(preamble + '\nreturn { dedup, isHardRule, shouldVerify, sameIssue, isProveAttempt, countProveAttempt }')()
63
69
 
64
70
  let failed = 0
65
71
  function check(name, actual, expected) {
@@ -676,5 +682,20 @@ console.log('\nprove: skipped without an installed test runner')
676
682
  rmSync(tmp, { recursive: true, force: true })
677
683
  }
678
684
 
685
+ console.log('\nprove ledger: a declined prover is a skip, not an attempt')
686
+ // Observed: 6 critical findings, every prover answered attempted:false (no node_modules),
687
+ // and the ledger recorded proven 0/6 as if six tests had run and not failed.
688
+ check('not_attempted is not an attempt', isProveAttempt({ attempted: false, proven: false, outcome: 'not_attempted' }), false)
689
+ check('inconclusive run is an attempt', isProveAttempt({ attempted: true, proven: false, outcome: 'inconclusive' }), true)
690
+ check('proven run is an attempt', isProveAttempt({ attempted: true, proven: true, outcome: 'proven' }), true)
691
+ // The counter path itself, not just the predicate: a declined prover must leave the count unchanged.
692
+ check('ledger: null proof (prover failed or rejected) adds nothing', countProveAttempt(0, null), 0)
693
+ check('ledger: declined prover adds nothing', countProveAttempt(2, { attempted: false, proven: false, outcome: 'not_attempted' }), 2)
694
+ check('ledger: inconclusive prover adds one', countProveAttempt(2, { attempted: true, proven: false, outcome: 'inconclusive' }), 3)
695
+ check('ledger: proven prover adds one', countProveAttempt(2, { attempted: true, proven: true, outcome: 'proven' }), 3)
696
+ // Source guard: a bare increment would bypass the predicate, and no unit check above would notice.
697
+ const bareWrites = src.split('\n').filter(l => /proveAttemptCount\s*(\+\+|\+=|=[^=])/.test(l) && !/countProveAttempt\(|let proveAttemptCount/.test(l))
698
+ check('the attempt count is written only through countProveAttempt', bareWrites, [])
699
+
679
700
  console.log(failed === 0 ? '\nall checks passed\n' : `\n${failed} check(s) FAILED\n`)
680
701
  process.exit(failed === 0 ? 0 : 1)
@@ -393,6 +393,19 @@ function normalizeProof(result) {
393
393
  }
394
394
  }
395
395
 
396
+ // A prover that declines at its testability gate (no runner, not importable)
397
+ // returns outcome 'not_attempted'. That is a skip, not a failed attempt, so it
398
+ // must not inflate the proven/attempts ratio in the run ledger.
399
+ function isProveAttempt(proof) {
400
+ return proof.attempted === true
401
+ }
402
+
403
+ // The only writer of the run ledger's attempt count. A null proof means the
404
+ // prover agent failed or was rejected, so there is no evidence a test ran.
405
+ function countProveAttempt(count, proof) {
406
+ return Boolean(proof) && isProveAttempt(proof) ? count + 1 : count
407
+ }
408
+
396
409
  // Run thunks in sequential batches, giving the per-finding stages a ceiling.
397
410
  //
398
411
  // The review stage is bounded by its dimension count, but verify, critic and
@@ -951,7 +964,6 @@ const budgetAtStart = budget.spent()
951
964
  await batched(capped.map(function(finding) {
952
965
  return async function() {
953
966
  agentCount++
954
- proveAttemptCount++
955
967
  const result = await agent(proverPrompt(finding), {
956
968
  label: 'prove:' + finding.file + ':' + finding.line,
957
969
  phase: 'Prove',
@@ -968,6 +980,7 @@ const budgetAtStart = budget.spent()
968
980
 
969
981
  const proof = normalizeProof(result)
970
982
  finding.proof = proof
983
+ proveAttemptCount = countProveAttempt(proveAttemptCount, proof)
971
984
  if (proof.proven === true) {
972
985
  provenCount++
973
986
  finding.verificationStatus = 'proven'
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-obsidian",
3
3
  "description": "Keep an Obsidian vault in sync with work: PR sync, task rollover, morning routine.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-product",
3
3
  "description": "Take a product idea to a shippable build: MVP passes, full-stack scaffolds, UX/UI quality loops, launch plans.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-reflection",
3
3
  "description": "Look back and improve: self-critique, retrospectives, performance review, rapid learning.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-release",
3
3
  "description": "Ship a release: semantic-release setup, changelogs, store listing copy, release-day routine.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-skill-meta",
3
3
  "description": "Author and maintain agent skills and the AI toolchain itself.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "olko-web-ops",
3
3
  "description": "Operate a website: WAF rules, search console audits, analytics bootstrap, docs indexes.",
4
- "version": "1.62.0",
4
+ "version": "1.63.0",
5
5
  "author": {
6
6
  "name": "Oleg Koval"
7
7
  },
@@ -28,7 +28,7 @@ const writeJson = (p, value) => {
28
28
  writeFileSync(p, JSON.stringify(value, null, 2) + '\n')
29
29
  }
30
30
  const copyResources = (skill, destDir) => {
31
- for (const d of ['scripts', 'references', 'templates']) {
31
+ for (const d of ['scripts', 'references', 'templates', 'evals']) {
32
32
  const src = join(root, skill.path, d)
33
33
  if (!existsSync(src)) continue
34
34
  const dest = join(destDir, d)