@olegkoval/agent-skills 1.62.1 → 1.63.0

package/adapters/claude/olko-github-pr/skills/lekker-review/SKILL.md

@@ -1325,6 +1325,16 @@ is not failure, the review may find things the bots never would. `0 bot
 comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
 turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
 
+**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
+"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
+each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
+`node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
+prints recall on buggy PRs and the false-positive count (critical or important
+findings) on clean ones. The shipped manifest holds two placeholders; choosing
+real PRs is a human job. A hit is location-only (same file, within 10 lines),
+so a nearby finding about a different problem still counts; read recall as an
+upper bound.
+
 If you maintain a persistent notes/memory system across reviews, launch ONE
 background Agent with `model: 'haiku'`, passing the review file path +
 repo-short-name + PR author login, instructed to follow

package/adapters/claude/olko-github-pr/skills/lekker-review/evals/manifest.json

@@ -0,0 +1,16 @@
+{
+  "note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
+  "prs": [
+    {
+      "repo": "example-org/example-repo",
+      "pr": 1,
+      "expect": "buggy",
+      "knownBug": "src/orders/sync.ts:42 off-by-one drops the last line item"
+    },
+    {
+      "repo": "example-org/example-repo",
+      "pr": 2,
+      "expect": "clean"
+    }
+  ]
+}

package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+// Score lekker findings against a manifest of PRs with known outcomes.
+//
+// Usage:
+//   node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
+//
+// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
+// results.json:  { "<repo>#<pr>": [ <finding>, ... ] }  the findings of one lekker run per PR
+//                (the `findings` array the workflow returns). A PR missing from results is
+//                reported as unscored, never as zero findings.
+//
+// A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
+// of the knownBug line. Known limitation: the match is by location only, so a
+// same-file finding near the line counts even if it describes a different problem;
+// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits / scored buggy PRs. On a clean PR every
+// verdict-affecting finding (critical or important) is a false positive.
+
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+export const LINE_WINDOW = 10;
+const BLOCKING = new Set(['critical', 'important']);
+
+export function parseKnownBug(text) {
+  // The file is everything before the last ":<digits>" that is followed by whitespace
+  // or the end, so paths containing spaces parse.
+  const m = /^(.+):(\d+)(?=\s|$)/.exec(String(text || '').trim());
+  if (!m) return null;
+  const line = Number(m[2]);
+  return line >= 1 ? { file: m[1], line } : null;
+}
+
+export function isHit(findings, knownBug) {
+  const spot = parseKnownBug(knownBug);
+  if (!spot) return false;
+  return findings.some((f) => f.file === spot.file
+    && Number.isInteger(f.line) && f.line > 0
+    && Math.abs(f.line - spot.line) <= LINE_WINDOW);
+}
+
+export function validateManifest(manifest) {
+  const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
+  if (!prs) throw new Error('manifest needs a "prs" array');
+  const seen = new Set();
+  prs.forEach((e, i) => {
+    if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
+    if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
+    const key = `${e.repo}#${e.pr}`;
+    if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
+    seen.add(key);
+    if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
+      throw new Error(`prs[${i}]: a buggy entry needs knownBug starting with "file:line" (line >= 1)`);
+    }
+  });
+  return prs;
+}
+
+export function score(manifest, results) {
+  const rows = validateManifest(manifest).map((e) => {
+    const key = `${e.repo}#${e.pr}`;
+    const findings = results[key];
+    if (!Array.isArray(findings)) return { key, expect: e.expect, scored: false };
+    if (e.expect === 'buggy') return { key, expect: 'buggy', scored: true, hit: isHit(findings, e.knownBug) };
+    return {
+      key, expect: 'clean', scored: true,
+      falsePositives: findings.filter((f) => BLOCKING.has(f.severity)).length,
+    };
+  });
+  const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
+  const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
+  return {
+    recall: buggy.length === 0 ? null : buggy.filter((r) => r.hit).length / buggy.length,
+    buggyScored: buggy.length,
+    falsePositives: clean.reduce((n, r) => n + r.falsePositives, 0),
+    cleanScored: clean.length,
+    unscored: rows.filter((r) => !r.scored).map((r) => r.key),
+    rows,
+  };
+}
+
+function main(argv) {
+  const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
+  if (!flag('manifest') || !flag('results')) {
+    console.error('usage: eval-runner.mjs --manifest <manifest.json> --results <results.json> [--json]');
+    process.exit(2);
+  }
+  const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
+  if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
+  const pct = out.recall === null ? 'n/a' : `${Math.round(out.recall * 100)}%`;
+  console.log(`recall ${pct} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@olegkoval/agent-skills",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "private": false,
   "publishConfig": {
     "access": "public"

package/plugins/olko-apple-kit/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-apple-kit",
   "description": "Build and ship Apple platform apps: macOS menubar apps, App Store submissions.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-creative/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-creative",
   "description": "Creative and personal projects: photo galleries, music players, listings, wiki editing.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-garmin-kit/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-garmin-kit",
   "description": "Build, test and publish Garmin Connect IQ watch faces.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-git-tools/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-git-tools",
   "description": "Everyday git and GitHub CLI operations: conventional commits, branch hygiene.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-github-pr/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-github-pr",
   "description": "Drive GitHub pull requests to merge-ready: review-bot loops, CI fixes, descriptions, dependency triage.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-github-pr/skills/lekker-review/SKILL.md

@@ -1324,6 +1324,16 @@ is not failure, the review may find things the bots never would. `0 bot
 comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
 turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
 
+**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
+"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
+each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
+`node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
+prints recall on buggy PRs and the false-positive count (critical or important
+findings) on clean ones. The shipped manifest holds two placeholders; choosing
+real PRs is a human job. A hit is location-only (same file, within 10 lines),
+so a nearby finding about a different problem still counts; read recall as an
+upper bound.
+
 If you maintain a persistent notes/memory system across reviews, launch ONE
 background Agent with `model: 'haiku'`, passing the review file path +
 repo-short-name + PR author login, instructed to follow

package/plugins/olko-github-pr/skills/lekker-review/evals/manifest.json

@@ -0,0 +1,16 @@
+{
+  "note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
+  "prs": [
+    {
+      "repo": "example-org/example-repo",
+      "pr": 1,
+      "expect": "buggy",
+      "knownBug": "src/orders/sync.ts:42 off-by-one drops the last line item"
+    },
+    {
+      "repo": "example-org/example-repo",
+      "pr": 2,
+      "expect": "clean"
+    }
+  ]
+}

package/plugins/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+// Score lekker findings against a manifest of PRs with known outcomes.
+//
+// Usage:
+//   node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
+//
+// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
+// results.json:  { "<repo>#<pr>": [ <finding>, ... ] }  the findings of one lekker run per PR
+//                (the `findings` array the workflow returns). A PR missing from results is
+//                reported as unscored, never as zero findings.
+//
+// A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
+// of the knownBug line. Known limitation: the match is by location only, so a
+// same-file finding near the line counts even if it describes a different problem;
+// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits / scored buggy PRs. On a clean PR every
+// verdict-affecting finding (critical or important) is a false positive.
+
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+
+export const LINE_WINDOW = 10;
+const BLOCKING = new Set(['critical', 'important']);
+
+export function parseKnownBug(text) {
+  // The file is everything before the last ":<digits>" that is followed by whitespace
+  // or the end, so paths containing spaces parse.
+  const m = /^(.+):(\d+)(?=\s|$)/.exec(String(text || '').trim());
+  if (!m) return null;
+  const line = Number(m[2]);
+  return line >= 1 ? { file: m[1], line } : null;
+}
+
+export function isHit(findings, knownBug) {
+  const spot = parseKnownBug(knownBug);
+  if (!spot) return false;
+  return findings.some((f) => f.file === spot.file
+    && Number.isInteger(f.line) && f.line > 0
+    && Math.abs(f.line - spot.line) <= LINE_WINDOW);
+}
+
+export function validateManifest(manifest) {
+  const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
+  if (!prs) throw new Error('manifest needs a "prs" array');
+  const seen = new Set();
+  prs.forEach((e, i) => {
+    if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
+    if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
+    const key = `${e.repo}#${e.pr}`;
+    if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
+    seen.add(key);
+    if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
+      throw new Error(`prs[${i}]: a buggy entry needs knownBug starting with "file:line" (line >= 1)`);
+    }
+  });
+  return prs;
+}
+
+export function score(manifest, results) {
+  const rows = validateManifest(manifest).map((e) => {
+    const key = `${e.repo}#${e.pr}`;
+    const findings = results[key];
+    if (!Array.isArray(findings)) return { key, expect: e.expect, scored: false };
+    if (e.expect === 'buggy') return { key, expect: 'buggy', scored: true, hit: isHit(findings, e.knownBug) };
+    return {
+      key, expect: 'clean', scored: true,
+      falsePositives: findings.filter((f) => BLOCKING.has(f.severity)).length,
+    };
+  });
+  const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
+  const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
+  return {
+    recall: buggy.length === 0 ? null : buggy.filter((r) => r.hit).length / buggy.length,
+    buggyScored: buggy.length,
+    falsePositives: clean.reduce((n, r) => n + r.falsePositives, 0),
+    cleanScored: clean.length,
+    unscored: rows.filter((r) => !r.scored).map((r) => r.key),
+    rows,
+  };
+}
+
+function main(argv) {
+  const flag = (n) => { const i = argv.indexOf('--' + n); return i === -1 ? null : argv[i + 1]; };
+  if (!flag('manifest') || !flag('results')) {
+    console.error('usage: eval-runner.mjs --manifest <manifest.json> --results <results.json> [--json]');
+    process.exit(2);
+  }
+  const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
+  if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
+  const pct = out.recall === null ? 'n/a' : `${Math.round(out.recall * 100)}%`;
+  console.log(`recall ${pct} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));

package/plugins/olko-obsidian/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-obsidian",
   "description": "Keep an Obsidian vault in sync with work: PR sync, task rollover, morning routine.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-product/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-product",
   "description": "Take a product idea to a shippable build: MVP passes, full-stack scaffolds, UX/UI quality loops, launch plans.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-reflection/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-reflection",
   "description": "Look back and improve: self-critique, retrospectives, performance review, rapid learning.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-release/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-release",
   "description": "Ship a release: semantic-release setup, changelogs, store listing copy, release-day routine.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-skill-meta/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-skill-meta",
   "description": "Author and maintain agent skills and the AI toolchain itself.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/plugins/olko-web-ops/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "olko-web-ops",
   "description": "Operate a website: WAF rules, search console audits, analytics bootstrap, docs indexes.",
-  "version": "1.62.1",
+  "version": "1.63.0",
   "author": {
     "name": "Oleg Koval"
   },

package/scripts/build-adapters.sh

@@ -28,7 +28,7 @@ const writeJson = (p, value) => {
   writeFileSync(p, JSON.stringify(value, null, 2) + '\n')
 }
 const copyResources = (skill, destDir) => {
-  for (const d of ['scripts', 'references', 'templates']) {
+  for (const d of ['scripts', 'references', 'templates', 'evals']) {
     const src = join(root, skill.path, d)
     if (!existsSync(src)) continue
     const dest = join(destDir, d)