@olegkoval/agent-skills 1.54.4 → 1.55.0

package/plugins/olko-github-pr/skills/lekker-review/scripts/bot-precision.mjs

@@ -0,0 +1,285 @@
+#!/usr/bin/env node
+// Close the loop on bot-comment prediction: record what the review predicted,
+// then score it against what the bots actually said.
+//
+// Without this, "we pre-empted the bot comments" is a claim nobody checks. The
+// only honest way to know whether pre-emption works is to write the prediction
+// down BEFORE the bots run and compare afterwards - so that is what this does.
+//
+// Usage:
+//   node bot-precision.mjs record --repo <slug> --pr <n> --predicted <findings.json> [--at <iso>]
+//   node bot-precision.mjs score  --repo <slug> --pr <n>
+//   node bot-precision.mjs summary
+//
+// Store: $LEKKER_DATA_DIR/bot-precision.jsonl, default ~/.cache/lekker-review
+// (one JSON object per line, append-only). Override with --jsonl <path>.
+//
+// `record` takes either a lint-findings.mjs output ({findings:[...]}) or a bare
+// array of findings; each needs at least `file`, and `line`/`title` when known.
+
+import { execFileSync } from 'node:child_process';
+import { readFileSync, appendFileSync, existsSync, mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join } from 'node:path';
+
+const argv = process.argv.slice(2);
+const cmd = argv[0];
+
+function flag(name, fallback = null) {
+  const i = argv.indexOf('--' + name);
+  return i === -1 ? fallback : argv[i + 1];
+}
+
+const JSONL = flag(
+  'jsonl',
+  join(process.env.LEKKER_DATA_DIR || join(homedir(), '.cache', 'lekker-review'), 'bot-precision.jsonl'),
+);
+
+function die(msg, code = 2) {
+  console.error(msg);
+  process.exit(code);
+}
+
+// Private: the store quotes review comments, which can come from private repos.
+function ensureStore() {
+  mkdirSync(dirname(JSONL), { recursive: true, mode: 0o700 });
+}
+
+function readRecords() {
+  if (!existsSync(JSONL)) return [];
+  return readFileSync(JSONL, 'utf8')
+    .split('\n')
+    .filter(l => l.trim())
+    .map(l => {
+      try {
+        return JSON.parse(l);
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean);
+}
+
+function append(record) {
+  ensureStore();
+  appendFileSync(JSONL, JSON.stringify(record) + '\n', { mode: 0o600 });
+}
+
+// --- matching -------------------------------------------------------------
+//
+// A prediction counts as matched when a bot commented on the SAME FILE within
+// LINE_WINDOW lines of it. Deliberately not semantic: an LLM judging its own
+// prediction against the comment it was trying to predict would grade itself,
+// and file+line is checkable by hand from the jsonl.
+const LINE_WINDOW = 20;
+
+// Both sides are repo-relative paths, so only a leading "./" or "/" differs.
+function normFile(p) {
+  return String(p || '').replace(/^(\.\/|\/)+/, '');
+}
+
+// null/undefined is "no line", not line 0 (Number(null) is 0).
+function toLine(v) {
+  return v == null ? NaN : Number(v);
+}
+
+function matches(prediction, comment) {
+  if (normFile(prediction.file) !== normFile(comment.path)) return false;
+  const a = toLine(prediction.line);
+  const b = toLine(comment.line ?? comment.original_line);
+  if (!Number.isFinite(a) || !Number.isFinite(b)) return true; // same file, no usable line
+  return Math.abs(a - b) <= LINE_WINDOW;
+}
+
+const BOT_RE = /gemini-code-assist|greptile|coderabbit/i;
+
+const pct = v => (v === null ? 'n/a' : `${Math.round(v * 100)}%`);
+
+function fetchBotComments(repo, pr) {
+  const raw = execFileSync(
+    'gh',
+    ['api', `repos/${repo}/pulls/${pr}/comments?per_page=100`, '--paginate', '--slurp'],
+    { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 },
+  );
+  const pages = JSON.parse(raw);
+  const flat = [];
+  for (const page of pages) {
+    if (Array.isArray(page)) flat.push(...page);
+    else if (page) flat.push(page);
+  }
+  // Replies in a bot's own thread ("Thanks, updated") are not new findings.
+  return flat
+    .filter(c => c?.user?.type === 'Bot' && BOT_RE.test(c?.user?.login || '') && !c.in_reply_to_id)
+    .map(c => ({
+      path: c.path,
+      line: c.line ?? c.original_line,
+      createdAt: c.created_at || null,
+      author: c.user.login,
+      url: c.html_url,
+      claim: String(c.body || '')
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, 200),
+    }));
+}
+
+// --- commands -------------------------------------------------------------
+
+if (cmd === 'record') {
+  const repo = flag('repo');
+  const pr = flag('pr');
+  const predictedPath = flag('predicted');
+  if (!repo || !pr || !predictedPath) {
+    die('usage: bot-precision.mjs record --repo <slug> --pr <n> --predicted <findings.json>');
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(readFileSync(predictedPath, 'utf8'));
+  } catch (err) {
+    die(`could not read predictions: ${err.message}`);
+  }
+  const findings = Array.isArray(parsed) ? parsed : parsed.findings || [];
+  // A grouped lint finding stands for every site it lists; a bot comment on
+  // any of them was predicted.
+  const predicted = findings.flatMap(f =>
+    (Array.isArray(f.lintSites) && f.lintSites.length ? f.lintSites : [f]).map(s => ({
+      file: s.file,
+      line: s.line,
+      title: f.title,
+      rule: f.rule || null,
+      severity: f.severity || null,
+      source: f.botPredicted ? 'lint-preflight' : 'review-agent',
+    })),
+  );
+  // --at backdates the prediction to when it was made: branch mode records it
+  // after creating the PR, and a bot can comment in between.
+  const at = flag('at') || new Date().toISOString();
+  if (Number.isNaN(Date.parse(at))) die(`--at is not a timestamp: ${at}`);
+  append({
+    kind: 'prediction',
+    repo,
+    pr: Number(pr),
+    at,
+    predictedCount: predicted.length,
+    predicted,
+  });
+  console.log(`bot-precision: recorded ${predicted.length} prediction(s) for ${repo}#${pr}`);
+  process.exit(0);
+}
+
+if (cmd === 'score') {
+  const repo = flag('repo');
+  const pr = flag('pr');
+  if (!repo || !pr) die('usage: bot-precision.mjs score --repo <slug> --pr <n>');
+
+  // The FIRST prediction is the one made before the bots ran; a re-review
+  // records another after seeing their comments.
+  const prediction = readRecords().find(
+    r => r.kind === 'prediction' && r.repo === repo && r.pr === Number(pr),
+  );
+  if (!prediction) {
+    // No prediction is "unknown", not "0% precision" - scoring an unrecorded
+    // review would manufacture a number out of a missing input.
+    console.log(`bot-precision: no recorded prediction for ${repo}#${pr} - nothing to score`);
+    process.exit(0);
+  }
+
+  let fetched;
+  try {
+    fetched = fetchBotComments(repo, pr);
+  } catch (err) {
+    die(`gh api failed: ${err.message}`, 1);
+  }
+  // A comment posted before the prediction was visible when it was made, so
+  // it cannot count as predicted or missed.
+  // Compared as instants at whole-second precision: GitHub timestamps have no
+  // fraction, and a string compare against "...:00.500Z" orders them wrongly.
+  const cutoff = Math.floor(Date.parse(prediction.at) / 1000) * 1000;
+  const comments = fetched.filter(c => !c.createdAt || Date.parse(c.createdAt) >= cutoff);
+  const preexisting = fetched.length - comments.length;
+
+  if (comments.length === 0) {
+    // Distinguish "the bots left nothing" from "the bots have not run yet":
+    // the caller knows which, this script does not, so say both.
+    console.log(
+      `bot-precision: ${repo}#${pr} has 0 bot comment(s). ` +
+        `Either the bots found nothing (${prediction.predictedCount} prediction(s) unrefuted) ` +
+        `or they have not run yet - re-run score once they have.`,
+    );
+    process.exit(0);
+  }
+
+  const hitPredictions = prediction.predicted.filter(p => comments.some(c => matches(p, c)));
+  const predictedComments = comments.filter(c => prediction.predicted.some(p => matches(p, c)));
+  const missed = comments.filter(c => !prediction.predicted.some(p => matches(p, c)));
+
+  const precision = prediction.predicted.length
+    ? hitPredictions.length / prediction.predicted.length
+    : null;
+  const recall = comments.length ? predictedComments.length / comments.length : null;
+
+  const record = {
+    kind: 'score',
+    repo,
+    pr: Number(pr),
+    at: new Date().toISOString(),
+    predictedCount: prediction.predicted.length,
+    botCommentCount: comments.length,
+    preexistingExcluded: preexisting,
+    predictedAndRaised: hitPredictions.length,
+    botCommentsPredicted: predictedComments.length,
+    precision,
+    recall,
+    matchRule: `same file within ${LINE_WINDOW} lines`,
+    missedBotComments: missed.slice(0, 20),
+  };
+  append(record);
+
+  console.log(
+    `bot-precision ${repo}#${pr}: ` +
+      `predicted ${record.predictedCount}, bots left ${record.botCommentCount}, ` +
+      `precision ${pct(precision)} (${hitPredictions.length}/${record.predictedCount}), ` +
+      `recall ${pct(recall)} (${predictedComments.length}/${record.botCommentCount}) ` +
+      `[match: ${record.matchRule}]`,
+  );
+  if (missed.length) {
+    console.log(`bot-precision: ${missed.length} bot comment(s) not predicted:`);
+    for (const m of missed.slice(0, 5)) {
+      console.log(`  - ${m.path}:${m.line ?? '?'} ${m.claim.slice(0, 110)}`);
+    }
+  }
+  process.exit(0);
+}
+
+if (cmd === 'summary') {
+  // A PR is scored again once more bot comments land; only its latest score
+  // counts, or a re-scored PR would weigh double in the means.
+  const latest = new Map();
+  for (const r of readRecords()) {
+    if (r.kind === 'score') latest.set(`${r.repo}#${r.pr}`, r);
+  }
+  const scores = [...latest.values()];
+  if (!scores.length) {
+    console.log('bot-precision: no scored reviews yet');
+    process.exit(0);
+  }
+  const avg = key => {
+    const vals = scores.map(s => s[key]).filter(v => typeof v === 'number');
+    return vals.length ? vals.reduce((a, b) => a + b, 0) / vals.length : null;
+  };
+  console.log(
+    `bot-precision summary: ${scores.length} scored review(s), ` +
+      `mean precision ${pct(avg('precision'))}, mean recall ${pct(avg('recall'))}`,
+  );
+  for (const s of scores.slice(-10)) {
+    console.log(
+      `  ${s.repo}#${s.pr} ${s.at.slice(0, 10)}: ` +
+        `precision ${pct(s.precision)}, recall ${pct(s.recall)} ` +
+        `(${s.predictedCount} predicted / ${s.botCommentCount} bot comments)`,
+    );
+  }
+  process.exit(0);
+}
+
+die('usage: bot-precision.mjs <record|score|summary> [flags]');

package/plugins/olko-github-pr/skills/lekker-review/scripts/lint-findings.mjs

@@ -0,0 +1,239 @@
+#!/usr/bin/env node
+// eslint JSON report -> lekker-review findings, tagged botPredicted.
+//
+// Why this exists: the bot reviewers (Greptile, Gemini) spend most of their
+// budget restating what the repo's own eslint config already says, and they are
+// wrong about it more often than eslint is. On one observed PR three of four
+// predicted bot comments were eslint rules, and eslint OVERRULED the model's
+// guess (it had suggested `.forEach`, which that config bans). So the linter is
+// the authority for this class of finding, not an LLM.
+//
+// Usage:
+//   node lint-findings.mjs <eslint-json-file> <out-findings-json> [--max N] [--base <sha>]
+//
+// --base keeps only messages on lines the change added (git diff -U0 <sha> HEAD
+// in LEKKER_WORKTREE_PATH). eslint lints whole files, but bots comment on diff
+// lines, so standing debt elsewhere in a touched file is not a prediction and
+// must not reach fix mode. The dropped count is reported as summary.outOfDiff.
+//
+// Reads an `eslint -f json` report. Writes {"findings":[...], "summary": {...}}
+// where each finding matches the FINDINGS_SCHEMA shape used by workflow.js,
+// plus `botPredicted: true` and `rule: "LINT:<ruleId>"`.
+//
+// Findings are grouped by ruleId and file so a rule that fires 40 times in a
+// file produces one finding listing the other sites, not 40 findings that drown
+// the review. Per file, because fix mode routes each finding to its one file.
+
+import { execFileSync } from 'node:child_process';
+import { readFileSync, realpathSync, writeFileSync } from 'node:fs';
+import { isAbsolute, join, relative } from 'node:path';
+
+const [, , inPath, outPath, ...rest] = process.argv;
+
+if (!inPath || !outPath) {
+  console.error('usage: lint-findings.mjs <eslint-json-file> <out-json> [--max N] [--base <sha>]');
+  process.exit(2);
+}
+
+const maxIdx = rest.indexOf('--max');
+const MAX_FINDINGS = maxIdx === -1 ? 25 : Number(rest[maxIdx + 1]) || 25;
+const baseIdx = rest.indexOf('--base');
+const BASE = baseIdx === -1 ? null : rest[baseIdx + 1];
+
+// eslint severity 2 = error, 1 = warning.
+//
+// An error is something CI will reject and a bot will certainly comment on, so
+// it is `important`. A warning is style the team has chosen not to enforce, so
+// it lands in `idiomatic` where it cannot affect the verdict.
+function severityFor(eslintSeverity) {
+  return eslintSeverity === 2 ? 'important' : 'idiomatic';
+}
+
+// Write a zero-finding result whose status says why, then stop. A missing,
+// empty or malformed report is "unknown", never "clean", so the review header
+// cannot claim a green lint that never ran.
+function bail(status, why) {
+  writeFileSync(
+    outPath,
+    JSON.stringify({ findings: [], summary: { status, ruleCount: 0, messageCount: 0 } }, null, 2),
+  );
+  console.log(`lint-findings: ${why}, 0 findings`);
+  process.exit(0);
+}
+
+let report;
+try {
+  const raw = readFileSync(inPath, 'utf8').trim();
+  if (!raw) bail('empty-report', 'empty eslint report');
+  report = JSON.parse(raw);
+} catch (err) {
+  bail('unreadable: ' + err.message, `could not read eslint report (${err.message})`);
+}
+
+if (!Array.isArray(report)) bail('not-an-eslint-json-report', 'report is not an eslint JSON array');
+// eslint emits one entry per linted file, so [] means nothing was linted.
+if (report.length === 0) bail('empty-report', 'eslint report lists no files');
+
+// Resolve symlinks on both sides: on macOS /tmp is /private/tmp, so eslint's
+// filePath and the worktree path given here can spell the same dir differently.
+function real(p) {
+  try {
+    return realpathSync(p);
+  } catch {
+    return p;
+  }
+}
+
+const repoRoot = process.env.LEKKER_WORKTREE_PATH ? real(process.env.LEKKER_WORKTREE_PATH) : '';
+
+function relativize(p) {
+  if (!repoRoot || !isAbsolute(p)) return p;
+  const rel = relative(repoRoot, real(p));
+  return rel.startsWith('..') ? p : rel;
+}
+
+// repo-relative path -> Set of line numbers the change added, or null when no
+// --base was given or git could not answer (then nothing is filtered).
+function addedLines() {
+  if (!BASE || !repoRoot) return null;
+  let out;
+  try {
+    // Explicit prefixes: diff.noprefix / diff.mnemonicPrefix would otherwise
+    // change the "+++ b/" header the parser below slices.
+    const args = ['-C', repoRoot, 'diff', '-U0', '--no-color', '--no-ext-diff', '--src-prefix=a/', '--dst-prefix=b/', BASE, 'HEAD'];
+    out = execFileSync('git', args, {
+      encoding: 'utf8',
+      maxBuffer: 64 * 1024 * 1024,
+    });
+  } catch {
+    return null;
+  }
+  const added = new Map();
+  let file = null;
+  for (const line of out.split('\n')) {
+    if (line.startsWith('+++ ')) {
+      file = line === '+++ /dev/null' ? null : line.slice(6);
+      if (file && !added.has(file)) added.set(file, new Set());
+      continue;
+    }
+    const hunk = file && /^@@ -\S+ \+(\d+)(?:,(\d+))? @@/.exec(line);
+    if (!hunk) continue;
+    const start = Number(hunk[1]);
+    const count = hunk[2] === undefined ? 1 : Number(hunk[2]);
+    for (let i = 0; i < count; i++) added.get(file).add(start + i);
+  }
+  return added;
+}
+
+const added = addedLines();
+let outOfDiff = 0;
+
+// Group every message by ruleId. Parse errors arrive with ruleId null; they are
+// real and worth surfacing, so they get their own bucket.
+const byRule = new Map();
+let messageCount = 0;
+
+for (const file of report) {
+  const filePath = String(file?.filePath ?? '');
+  for (const m of file?.messages ?? []) {
+    // A message with no ruleId is a parse error only when eslint marks it
+    // fatal. The rest are notices such as "File ignored because of a matching
+    // ignore pattern", which flag nothing wrong with the code.
+    if (!m.ruleId && !m.fatal) continue;
+    const ruleId = m.ruleId ?? 'parse-error';
+    // A parse error breaks the whole file, so it counts wherever it lands.
+    if (added && ruleId !== 'parse-error' && !added.get(relativize(filePath))?.has(m.line)) {
+      outOfDiff++;
+      continue;
+    }
+    messageCount++;
+    const key = `${ruleId}\0${filePath}`;
+    if (!byRule.has(key)) {
+      byRule.set(key, { ruleId, severity: m.severity ?? 2, fixable: false, sites: [] });
+    }
+    const bucket = byRule.get(key);
+    // A rule reported at both levels keeps the louder one.
+    bucket.severity = Math.max(bucket.severity, m.severity ?? 2);
+    if (m.fix) {
+      bucket.fixable = true;
+    }
+    bucket.sites.push({
+      file: filePath,
+      line: m.line ?? 1,
+      column: m.column ?? 1,
+      message: String(m.message ?? '').trim(),
+    });
+  }
+}
+
+// Errors before warnings, then the rules that fire most - a rule hit 12 times
+// is a pattern the author is repeating, and worth more than a one-off.
+const ruleCount = new Set([...byRule.values()].map(b => b.ruleId)).size;
+const buckets = [...byRule.values()].sort((a, b) => {
+  if (b.severity !== a.severity) return b.severity - a.severity;
+  return b.sites.length - a.sites.length;
+});
+
+// The offending source line, so badCode is code and not a location string.
+function sourceLine(file, line) {
+  try {
+    const abs = isAbsolute(file) ? file : join(repoRoot, file);
+    const text = readFileSync(abs, 'utf8').split('\n')[line - 1];
+    if (text?.trim()) return text.trim();
+  } catch {
+    // unreadable file: fall back to the location
+  }
+  return null;
+}
+
+const findings = buckets.slice(0, MAX_FINDINGS).map(bucket => {
+  const first = bucket.sites[0];
+  const others = bucket.sites.slice(1);
+  const extra = others.length
+    ? ' Also at: ' + others.slice(0, 8).map(s => `${relativize(s.file)}:${s.line}`).join(', ') +
+      (others.length > 8 ? ` (+${others.length - 8} more)` : '') + '.'
+    : '';
+
+  return {
+    file: relativize(first.file),
+    line: first.line,
+    severity: severityFor(bucket.severity),
+    title: `${bucket.ruleId} (${bucket.sites.length} site${bucket.sites.length === 1 ? '' : 's'})`,
+    description:
+      `The repo's own eslint config reports \`${bucket.ruleId}\` on the changed files: ` +
+      `"${first.message}".${extra} ` +
+      `This is the repo's configured rule, not a preference - it is the single most likely ` +
+      `bot-reviewer comment on this diff, and CI enforces it.` +
+      (bucket.fixable ? ' eslint can fix this automatically (`eslint --fix`).' : ''),
+    badCode:
+      sourceLine(first.file, first.line) ?? `${relativize(first.file)}:${first.line}:${first.column}`,
+    fix: bucket.fixable
+      ? `Apply eslint's autofix for \`${bucket.ruleId}\` to the cited lines only, then re-read the hunk. ` +
+        `A file-wide \`eslint --fix\` also rewrites lines this change did not touch.`
+      : `Resolve \`${bucket.ruleId}\` as the config intends. Check what the rule permits before ` +
+        `substituting another form: a config that bans one idiom often bans its obvious replacement too.`,
+    rule: `LINT:${bucket.ruleId}`,
+    botPredicted: true,
+    lintFixable: bucket.fixable,
+    lintSiteCount: bucket.sites.length,
+    // Every site, so bot-precision can match a bot comment on any of them.
+    lintSites: bucket.sites.map(s => ({ file: relativize(s.file), line: s.line })),
+  };
+});
+
+const result = {
+  findings,
+  summary: {
+    status: messageCount === 0 ? 'clean' : 'issues',
+    ruleCount: ruleCount,
+    messageCount,
+    truncated: Math.max(0, buckets.length - findings.length),
+    outOfDiff,
+  },
+};
+
+writeFileSync(outPath, JSON.stringify(result, null, 2));
+console.log(
+  `lint-findings: ${messageCount} message(s) across ${ruleCount} rule(s) -> ` +
+    `${findings.length} finding(s)${result.summary.truncated ? ` (${result.summary.truncated} rule(s) truncated)` : ''}`,
+);