@iceinvein/agent-skills 0.22.0 → 0.23.0

package/skills/magpie/evals/quality/__tests__/recall.test.ts

@@ -0,0 +1,234 @@
+import { afterEach, beforeEach, expect, test } from 'bun:test'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { ReviewFinding } from '../../../scripts/types.ts'
+import { listCorpusRuns } from '../corpus.ts'
+import {
+  loadMisses,
+  type Miss,
+  parseRecallVerdicts,
+  type RecallVerdict,
+  recallJudgeCommand,
+  recallPrompt,
+  scoreRecall,
+} from '../recall.ts'
+
+function finding(over: Partial<ReviewFinding> & { id: string }): ReviewFinding {
+  return {
+    file: 'src/orders.ts',
+    line: 10,
+    severity: 'medium',
+    risk: { impact: 'medium', likelihood: 'possible', confidence: 'medium', action: 'should-fix' },
+    title: 'Something is off',
+    description: 'Details.',
+    domain: 'bugs',
+    ...over,
+  }
+}
+
+function miss(over: Partial<Miss> & { id: string }): Miss {
+  return {
+    file: 'src/orders.ts',
+    line: 42,
+    title: 'Refund ignores partial captures',
+    summary: 'A partially captured order refunds the full amount.',
+    ownerDomain: 'bugs',
+    bugClass: 'contract',
+    outsideDiff: false,
+    preExisting: false,
+    source: 'human review comment',
+    ...over,
+  }
+}
+
+const findings = [finding({ id: 'bugs-1' }), finding({ id: 'security-2' })]
+const misses = [miss({ id: 'm1' }), miss({ id: 'm2' })]
+
+test('a bare verdict array covering every miss is accepted', () => {
+  const text = JSON.stringify([
+    { missId: 'm1', findingId: 'bugs-1', reason: 'Same refund defect.' },
+    { missId: 'm2', findingId: null, reason: 'Nothing describes it.' },
+  ])
+  expect(parseRecallVerdicts(text, misses, findings)).toEqual([
+    { missId: 'm1', findingId: 'bugs-1', reason: 'Same refund defect.' },
+    { missId: 'm2', findingId: null, reason: 'Nothing describes it.' },
+  ])
+})
+
+test('a verdict array inside a fenced json block is accepted', () => {
+  const text =
+    'Here are the verdicts.\n```json\n' +
+    '[{"missId":"m1","findingId":null,"reason":"No match."},' +
+    '{"missId":"m2","findingId":"security-2","reason":"Same defect."}]\n```\n'
+  expect(parseRecallVerdicts(text, misses, findings)).toEqual([
+    { missId: 'm1', findingId: null, reason: 'No match.' },
+    { missId: 'm2', findingId: 'security-2', reason: 'Same defect.' },
+  ])
+})
+
+test('a verdict array inside an untagged fence is accepted', () => {
+  const text =
+    '```\n[{"missId":"m1","findingId":"bugs-1","reason":"Same defect."},' +
+    '{"missId":"m2","findingId":null,"reason":"No match."}]\n```'
+  expect(parseRecallVerdicts(text, misses, findings)).toEqual([
+    { missId: 'm1', findingId: 'bugs-1', reason: 'Same defect.' },
+    { missId: 'm2', findingId: null, reason: 'No match.' },
+  ])
+})
+
+test('a verdict array wrapped in prose without a fence is accepted', () => {
+  const text =
+    'Verdicts follow. [{"missId":"m1","findingId":null,"reason":"No match [none]."},' +
+    '{"missId":"m2","findingId":"security-2","reason":"Same defect."}] That is all.'
+  expect(parseRecallVerdicts(text, misses, findings)).toEqual([
+    { missId: 'm1', findingId: null, reason: 'No match [none].' },
+    { missId: 'm2', findingId: 'security-2', reason: 'Same defect.' },
+  ])
+})
+
+test('a verdict with no findingId is rejected', () => {
+  const text = JSON.stringify([
+    { missId: 'm1', reason: 'No match.' },
+    { missId: 'm2', findingId: null, reason: 'No match.' },
+  ])
+  expect(() => parseRecallVerdicts(text, misses, findings)).toThrow('[0]')
+})
+
+test('a verdict whose reason is not a string is rejected', () => {
+  const text = JSON.stringify([
+    { missId: 'm1', findingId: null, reason: 'No match.' },
+    { missId: 'm2', findingId: null, reason: 7 },
+  ])
+  expect(() => parseRecallVerdicts(text, misses, findings)).toThrow('[1]')
+})
+
+test('the recall judge runs with no MCP servers and no tools', () => {
+  expect(recallJudgeCommand('claude-sonnet-5-5')).toEqual([
+    'claude',
+    '-p',
+    '--model',
+    'claude-sonnet-5-5',
+    '--output-format',
+    'json',
+    '--strict-mcp-config',
+    '--tools',
+    '',
+  ])
+})
+
+test('verdicts that leave a miss unjudged are rejected', () => {
+  const text = JSON.stringify([{ missId: 'm1', findingId: null, reason: 'No match.' }])
+  expect(() => parseRecallVerdicts(text, misses, findings)).toThrow('m2')
+})
+
+test('verdicts that judge a miss twice are rejected', () => {
+  const text = JSON.stringify([
+    { missId: 'm1', findingId: null, reason: 'No match.' },
+    { missId: 'm1', findingId: 'bugs-1', reason: 'Same defect.' },
+    { missId: 'm2', findingId: null, reason: 'No match.' },
+  ])
+  expect(() => parseRecallVerdicts(text, misses, findings)).toThrow('m1')
+})
+
+test('a verdict for a miss not in the set is rejected', () => {
+  const text = JSON.stringify([
+    { missId: 'm1', findingId: null, reason: 'No match.' },
+    { missId: 'm2', findingId: null, reason: 'No match.' },
+    { missId: 'm9', findingId: null, reason: 'No match.' },
+  ])
+  expect(() => parseRecallVerdicts(text, misses, findings)).toThrow('m9')
+})
+
+test('a verdict naming a finding that was not kept is rejected', () => {
+  const text = JSON.stringify([
+    { missId: 'm1', findingId: 'bugs-7', reason: 'Same defect.' },
+    { missId: 'm2', findingId: null, reason: 'No match.' },
+  ])
+  expect(() => parseRecallVerdicts(text, misses, findings)).toThrow('bugs-7')
+})
+
+test('a judge reply that is not JSON is rejected', () => {
+  expect(() => parseRecallVerdicts('I could not decide.', misses, findings)).toThrow()
+})
+
+test('a judge reply that is a JSON object rather than an array is rejected', () => {
+  const text = '{"missId":"m1","findingId":null,"reason":"No match."}'
+  expect(() => parseRecallVerdicts(text, misses, findings)).toThrow('array')
+})
+
+test('recall counts caught misses overall, by domain, by class and without pre-existing ones', () => {
+  const set = [
+    miss({ id: 'a', ownerDomain: 'bugs', bugClass: 'contract' }),
+    miss({ id: 'b', ownerDomain: 'bugs', bugClass: 'null-handling', preExisting: true }),
+    miss({ id: 'c', ownerDomain: 'security', bugClass: 'contract' }),
+  ]
+  const verdicts: RecallVerdict[] = [
+    { missId: 'a', findingId: 'bugs-1', reason: 'Same.' },
+    { missId: 'b', findingId: 'bugs-3', reason: 'Same.' },
+    { missId: 'c', findingId: null, reason: 'None.' },
+  ]
+  expect(scoreRecall(set, verdicts)).toEqual({
+    misses: 3,
+    caught: 2,
+    recall: 2 / 3,
+    byDomain: { bugs: { misses: 2, caught: 2 }, security: { misses: 1, caught: 0 } },
+    byClass: { contract: { misses: 2, caught: 1 }, 'null-handling': { misses: 1, caught: 1 } },
+    excludingPreExisting: { misses: 2, caught: 1, recall: 1 / 2 },
+  })
+})
+
+test('recall over an empty miss set is null rather than zero', () => {
+  expect(scoreRecall([], [])).toEqual({
+    misses: 0,
+    caught: 0,
+    recall: null,
+    byDomain: {},
+    byClass: {},
+    excludingPreExisting: { misses: 0, caught: 0, recall: null },
+  })
+})
+
+test('the recall prompt names every miss and every kept finding', () => {
+  const prompt = recallPrompt(
+    [miss({ id: 'miss-alpha' }), miss({ id: 'miss-beta' })],
+    [finding({ id: 'bugs-11' }), finding({ id: 'contracts-4' })],
+  )
+  for (const id of ['miss-alpha', 'miss-beta', 'bugs-11', 'contracts-4']) {
+    expect(prompt).toContain(id)
+  }
+})
+
+let root: string
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), 'magpie-recall-'))
+})
+
+afterEach(async () => {
+  await rm(root, { recursive: true, force: true })
+})
+
+test('a missing recall set loads as null', async () => {
+  expect(await loadMisses(join(root, 'misses', 'pr-1-1.json'))).toBeNull()
+})
+
+test('a recall set file loads its misses', async () => {
+  const path = join(root, 'pr-1-1.json')
+  await writeFile(path, JSON.stringify([miss({ id: 'm1' })]))
+  expect(await loadMisses(path)).toEqual([miss({ id: 'm1' })])
+})
+
+test('a malformed miss is rejected naming the file and its index', async () => {
+  const path = join(root, 'pr-1-1.json')
+  const { bugClass: _, ...noClass } = miss({ id: 'm2' })
+  await writeFile(path, JSON.stringify([miss({ id: 'm1' }), noClass]))
+  await expect(loadMisses(path)).rejects.toThrow(`${path}[1]`)
+})
+
+test('the corpus run list leaves out the results and misses directories', async () => {
+  await mkdir(join(root, 'results'))
+  await mkdir(join(root, 'misses'))
+  await mkdir(join(root, 'pr-46-1787748571'))
+  expect(await listCorpusRuns(root)).toEqual(['pr-46-1787748571'])
+})

package/skills/magpie/evals/quality/__tests__/replay-full.test.ts

@@ -0,0 +1,164 @@
+import { expect, test } from 'bun:test'
+import { mkdir, mkdtemp, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { replayDiagnostics, replaySummary, shardCoverageCount } from '../replay-full.ts'
+
+const base = {
+  runId: 'pr-50-1789099641',
+  kept: 12,
+  unlabelled: 4,
+  precision: 0.5,
+  recall: 0.75,
+  costUsd: 3.456,
+  outPath: '/corpus/results/x-full-pr-50-1789099641.json',
+  coverage: { reviewed: 3, total: 7 },
+}
+
+test('a replay without a recall set prints no missed-recall figure', () => {
+  expect(replaySummary({ ...base, missedRecall: null })).toBe(
+    'pr-50-1789099641: kept 12 (4 unlabelled), precision 0.50, recall 0.75, cost $3.46, shards 3/7\n' +
+      'wrote /corpus/results/x-full-pr-50-1789099641.json\n',
+  )
+})
+
+test('a replay judged against a recall set prints caught over misses', () => {
+  expect(replaySummary({ ...base, missedRecall: { caught: 3, misses: 14 } })).toBe(
+    'pr-50-1789099641: kept 12 (4 unlabelled), precision 0.50, recall 0.75, cost $3.46, ' +
+      'missed-recall 3/14, shards 3/7\nwrote /corpus/results/x-full-pr-50-1789099641.json\n',
+  )
+})
+
+test('a replay whose cost Claude did not report says so', () => {
+  expect(
+    replaySummary({ ...base, precision: null, recall: null, costUsd: null, missedRecall: null }),
+  ).toBe(
+    'pr-50-1789099641: kept 12 (4 unlabelled), precision -, recall -, cost unreported, shards 3/7\n' +
+      'wrote /corpus/results/x-full-pr-50-1789099641.json\n',
+  )
+})
+
+function manifestOf(shardCount: number) {
+  return {
+    budget: 6000,
+    maxFiles: 80,
+    totalFiles: shardCount,
+    totalLines: 10 * shardCount,
+    shards: Array.from({ length: shardCount }, (_, i) => ({
+      id: i + 1,
+      path: shardCount === 1 ? 'diff.patch' : `shards/shard-${i + 1}.patch`,
+      files: [`src/f${i + 1}.ts`],
+      lines: 10,
+    })),
+  }
+}
+
+test('a sharded replay counts only the shards some specialist wrote findings for', () => {
+  const files = [
+    'bugs.shard-4.json',
+    'bugs.shard-5.json',
+    'security.shard-5.json',
+    'architecture.shard-6.json',
+    'code-smells.shard-7.json',
+    'tests.json',
+  ]
+  expect(shardCoverageCount(manifestOf(7), files)).toEqual({ reviewed: 4, total: 7 })
+})
+
+test('a single-shard replay counts as reviewed when a specialist wrote its unsharded file', () => {
+  expect(shardCoverageCount(manifestOf(1), ['bugs.json', 'tests.json'])).toEqual({
+    reviewed: 1,
+    total: 1,
+  })
+})
+
+test('a single-shard replay with only the tests finding counts as unreviewed', () => {
+  expect(shardCoverageCount(manifestOf(1), ['tests.json'])).toEqual({ reviewed: 0, total: 1 })
+})
+
+test('a replay with no shard manifest reports no shards', () => {
+  expect(shardCoverageCount(null, ['bugs.json'])).toEqual({ reviewed: 0, total: 0 })
+})
+
+test('a manifest without a shards array is rejected', () => {
+  expect(() => shardCoverageCount({ budget: 6000 }, [])).toThrow('shards')
+})
+
+async function scratchRunDir(): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), 'magpie-replay-diag-'))
+  await mkdir(join(dir, 'findings'))
+  return dir
+}
+
+test('diagnostics carry the log, manifest, findings files, deduped count and critic drops', async () => {
+  const dir = await scratchRunDir()
+  await writeFile(
+    join(dir, 'log.jsonl'),
+    '{"stage":"setup","status":"done"}\n\n{"stage":"critic","status":"done","kept":8}\n',
+  )
+  await mkdir(join(dir, 'shards'))
+  await writeFile(join(dir, 'shards', 'manifest.json'), JSON.stringify(manifestOf(2)))
+  await writeFile(join(dir, 'findings', 'security.shard-2.json'), '[]')
+  await writeFile(join(dir, 'findings', 'bugs.shard-1.json'), '[]')
+  await writeFile(join(dir, 'findings.deduped.json'), '[{"id":"a"},{"id":"b"},{"id":"c"}]')
+  await writeFile(join(dir, 'critic-dropped.json'), '[{"id":"b","reason":"speculative"}]')
+
+  expect(await replayDiagnostics(dir)).toEqual({
+    log: [
+      { stage: 'setup', status: 'done' },
+      { stage: 'critic', status: 'done', kept: 8 },
+    ],
+    shardManifest: manifestOf(2),
+    findingsFiles: ['bugs.shard-1.json', 'security.shard-2.json'],
+    dedupedCount: 3,
+    criticDropped: [{ id: 'b', reason: 'speculative' }],
+    evidenceDropped: null,
+    thresholdDropped: null,
+  })
+})
+
+test('diagnostics report absent manifest, deduped and drop files as null', async () => {
+  const dir = await scratchRunDir()
+  await writeFile(join(dir, 'log.jsonl'), '{"stage":"setup","status":"done"}\n')
+
+  expect(await replayDiagnostics(dir)).toEqual({
+    log: [{ stage: 'setup', status: 'done' }],
+    shardManifest: null,
+    findingsFiles: [],
+    dedupedCount: null,
+    criticDropped: null,
+    evidenceDropped: null,
+    thresholdDropped: null,
+  })
+})
+
+test('a log line that is not a JSON object is kept raw in its place with its line number', async () => {
+  const dir = await scratchRunDir()
+  await writeFile(
+    join(dir, 'log.jsonl'),
+    '{"stage":"setup","status":"done"}\n{"stage":"shard"\n[1,2]\n{"stage":"critic"}\n',
+  )
+  expect((await replayDiagnostics(dir)).log).toEqual([
+    { stage: 'setup', status: 'done' },
+    { unparsed: '{"stage":"shard"', line: 2 },
+    { unparsed: '[1,2]', line: 3 },
+    { stage: 'critic' },
+  ])
+})
+
+test('diagnostics carry the evidence and threshold drops when the run wrote them', async () => {
+  const dir = await scratchRunDir()
+  await writeFile(join(dir, 'log.jsonl'), '{"stage":"setup","status":"done"}\n')
+  await writeFile(join(dir, 'evidence-dropped.json'), '[{"id":"e1","reason":"quote not in diff"}]')
+  await writeFile(join(dir, 'threshold-dropped.json'), '[{"id":"t1","confidence":0.2}]')
+  const diagnostics = await replayDiagnostics(dir)
+  expect(diagnostics.evidenceDropped).toEqual([{ id: 'e1', reason: 'quote not in diff' }])
+  expect(diagnostics.thresholdDropped).toEqual([{ id: 't1', confidence: 0.2 }])
+})
+
+test('a deduped findings file that is not an array is rejected', async () => {
+  const dir = await scratchRunDir()
+  await writeFile(join(dir, 'log.jsonl'), '{"stage":"setup","status":"done"}\n')
+  await writeFile(join(dir, 'findings.deduped.json'), '{"id":"a"}')
+  await expect(replayDiagnostics(dir)).rejects.toThrow('findings.deduped.json')
+})

package/skills/magpie/evals/quality/corpus.ts

@@ -154,8 +154,11 @@ export async function loadCorpusRun(dir: string): Promise<CorpusRun> {
   }
 }
 
+// Directories beside the runs that hold eval output and recall sets, not runs.
+const NON_RUN_DIRS = ['results', 'misses']
+
 export async function listCorpusRuns(corpusRoot: string): Promise<string[]> {
-  const names = (await readdir(corpusRoot)).filter((n) => n !== 'results').sort()
+  const names = (await readdir(corpusRoot)).filter((n) => !NON_RUN_DIRS.includes(n)).sort()
   const runs: string[] = []
   for (const name of names) {
     if ((await stat(join(corpusRoot, name))).isDirectory()) runs.push(name)

package/skills/magpie/evals/quality/recall.ts

@@ -0,0 +1,249 @@
+// Recall against known misses: defects a human found that magpie's kept
+// findings did not raise. A model judges which kept finding, if any, describes
+// each miss, because a miss is often anchored somewhere other than where a
+// finding about the same defect lands.
+
+import { existsSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import type { ReviewFinding } from '../../scripts/types.ts'
+import { claudeFailureMessage } from './replay-critic.ts'
+import { parseClaudeResult } from './score.ts'
+
+export const DEFAULT_JUDGE_MODEL = 'claude-sonnet-5-5'
+
+export type Miss = {
+  id: string
+  file: string
+  line: number | null
+  title: string
+  summary: string
+  ownerDomain: string
+  bugClass: string
+  outsideDiff: boolean
+  preExisting: boolean
+  source: string
+}
+
+export type RecallVerdict = { missId: string; findingId: string | null; reason: string }
+
+type Tally = { misses: number; caught: number }
+
+export type RecallScore = {
+  misses: number
+  caught: number
+  recall: number | null
+  byDomain: Record<string, Tally>
+  byClass: Record<string, Tally>
+  excludingPreExisting: { misses: number; caught: number; recall: number | null }
+}
+
+const MISS_STRING_FIELDS = ['id', 'file', 'title', 'summary', 'ownerDomain', 'bugClass', 'source']
+const MISS_BOOLEAN_FIELDS = ['outsideDiff', 'preExisting']
+
+function parseMiss(raw: unknown): Miss {
+  if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
+    throw new Error('is not an object')
+  }
+  const entry = raw as Record<string, unknown>
+  for (const field of MISS_STRING_FIELDS) {
+    if (typeof entry[field] !== 'string' || entry[field] === '') {
+      throw new Error(`${field} is not a non-empty string`)
+    }
+  }
+  for (const field of MISS_BOOLEAN_FIELDS) {
+    if (typeof entry[field] !== 'boolean') throw new Error(`${field} is not a boolean`)
+  }
+  if (entry.line !== null && !Number.isInteger(entry.line)) {
+    throw new Error('line is not an integer or null')
+  }
+  return entry as unknown as Miss
+}
+
+/** The recall set for one run, or null when no one has written one. */
+export async function loadMisses(path: string): Promise<Miss[] | null> {
+  if (!existsSync(path)) return null
+  let raw: unknown
+  try {
+    raw = JSON.parse(await readFile(path, 'utf8'))
+  } catch (err) {
+    throw new Error(`${path} is not valid JSON: ${err instanceof Error ? err.message : err}`)
+  }
+  if (!Array.isArray(raw)) throw new Error(`${path} is not a JSON array`)
+  return raw.map((m, i) => {
+    try {
+      return parseMiss(m)
+    } catch (err) {
+      throw new Error(`${path}[${i}]: ${err instanceof Error ? err.message : err}`)
+    }
+  })
+}
+
+function anchor(file: string, line: number | null): string {
+  return line === null ? file : `${file}:${line}`
+}
+
+export function recallPrompt(misses: Miss[], findings: ReviewFinding[]): string {
+  const missBlock = misses
+    .map(
+      (m) =>
+        `- id: ${m.id}\n  at: ${anchor(m.file, m.line)}\n  title: ${m.title}\n  summary: ${m.summary}`,
+    )
+    .join('\n')
+  const findingBlock = findings
+    .map(
+      (f) =>
+        `- id: ${f.id}\n  at: ${anchor(f.file, f.line)}\n  title: ${f.title}\n  description: ${f.description}`,
+    )
+    .join('\n')
+  return [
+    'You judge whether a code review caught known defects.',
+    '',
+    'Below are MISSES, defects a human reviewer found in a pull request, and FINDINGS, the',
+    'comments an automated review kept for the same pull request. For each miss, name the one',
+    'finding that describes the same underlying defect, wherever that finding is anchored, or',
+    'null when none does. Partial overlap that would not lead the author to the fix is null.',
+    'Give a reason under 25 words for each verdict.',
+    '',
+    'Reply with a JSON array only, one entry per miss, each miss exactly once:',
+    '[{"missId": "<miss id>", "findingId": "<finding id or null>", "reason": "<why>"}]',
+    '',
+    'MISSES',
+    missBlock,
+    '',
+    'FINDINGS',
+    findings.length === 0 ? '(none)' : findingBlock,
+    '',
+  ].join('\n')
+}
+
+const FENCED = /```(?:json)?\s*\n([\s\S]*?)```/
+
+/** The verdict array from a fence if there is one, else the outermost [...] in the reply. */
+function verdictText(text: string): string {
+  const fenced = FENCED.exec(text)?.[1]
+  if (fenced !== undefined) return fenced.trim()
+  const start = text.indexOf('[')
+  const end = text.lastIndexOf(']')
+  return start === -1 || end < start ? text.trim() : text.slice(start, end + 1)
+}
+
+export function parseRecallVerdicts(
+  text: string,
+  misses: Miss[],
+  findings: ReviewFinding[],
+): RecallVerdict[] {
+  const body = verdictText(text)
+  let raw: unknown
+  try {
+    raw = JSON.parse(body)
+  } catch {
+    throw new Error(`recall judge printed no JSON: ${text.slice(0, 200)}`)
+  }
+  if (!Array.isArray(raw)) throw new Error('recall judge reply is not a JSON array')
+
+  const missIds = new Set(misses.map((m) => m.id))
+  const findingIds = new Set(findings.map((f) => f.id))
+  const judged = new Set<string>()
+  const verdicts = raw.map((v, i): RecallVerdict => {
+    const entry = v as Record<string, unknown> | null
+    if (
+      typeof entry?.missId !== 'string' ||
+      !(typeof entry.findingId === 'string' || entry.findingId === null) ||
+      typeof entry.reason !== 'string'
+    ) {
+      throw new Error(`recall verdict [${i}] is not a {missId, findingId, reason} object`)
+    }
+    if (!missIds.has(entry.missId)) throw new Error(`recall verdict for unknown miss ${entry.missId}`)
+    if (judged.has(entry.missId)) throw new Error(`miss ${entry.missId} is judged twice`)
+    judged.add(entry.missId)
+    if (entry.findingId !== null && !findingIds.has(entry.findingId)) {
+      throw new Error(`miss ${entry.missId} names ${entry.findingId}, which is not a kept finding`)
+    }
+    return { missId: entry.missId, findingId: entry.findingId, reason: entry.reason }
+  })
+  const unjudged = misses.filter((m) => !judged.has(m.id)).map((m) => m.id)
+  if (unjudged.length > 0) throw new Error(`recall judge left unjudged: ${unjudged.join(', ')}`)
+  return verdicts
+}
+
+// `--strict-mcp-config` with no config loads no MCP servers, whose tools
+// `--tools ""` would not remove.
+export function recallJudgeCommand(model: string): string[] {
+  return [
+    'claude',
+    '-p',
+    '--model',
+    model,
+    '--output-format',
+    'json',
+    '--strict-mcp-config',
+    '--tools',
+    '',
+  ]
+}
+
+/** One `claude -p` with no tools or MCP servers: the judge reads only what the prompt gives it. */
+export async function judgeRecall(input: {
+  misses: Miss[]
+  findings: ReviewFinding[]
+  model: string
+  cwd: string
+}): Promise<{ verdicts: RecallVerdict[]; costUsd: number | null }> {
+  const label = 'recall judge'
+  const proc = Bun.spawn(recallJudgeCommand(input.model), {
+    cwd: input.cwd,
+    stdin: new Blob([recallPrompt(input.misses, input.findings)]),
+    stdout: 'pipe',
+    stderr: 'pipe',
+  })
+  const [stdout, stderr, exit] = await Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+    proc.exited,
+  ])
+  if (exit !== 0) throw new Error(claudeFailureMessage(label, exit, stdout, stderr))
+  const { costUsd } = parseClaudeResult(stdout, label)
+  const result = (JSON.parse(stdout) as Record<string, unknown>).result
+  if (typeof result !== 'string') throw new Error(`${label}: claude result is not a string`)
+  return { verdicts: parseRecallVerdicts(result, input.misses, input.findings), costUsd }
+}
+
+function ratio(caught: number, misses: number): number | null {
+  return misses === 0 ? null : caught / misses
+}
+
+export function scoreRecall(misses: Miss[], verdicts: RecallVerdict[]): RecallScore {
+  const caughtIds = new Set(verdicts.filter((v) => v.findingId !== null).map((v) => v.missId))
+  const byDomain: Record<string, Tally> = {}
+  const byClass: Record<string, Tally> = {}
+  let caught = 0
+  let newMisses = 0
+  let newCaught = 0
+  for (const m of misses) {
+    const isCaught = caughtIds.has(m.id)
+    const domain = byDomain[m.ownerDomain] ?? { misses: 0, caught: 0 }
+    byDomain[m.ownerDomain] = domain
+    const bugClass = byClass[m.bugClass] ?? { misses: 0, caught: 0 }
+    byClass[m.bugClass] = bugClass
+    domain.misses++
+    bugClass.misses++
+    if (!m.preExisting) newMisses++
+    if (!isCaught) continue
+    caught++
+    domain.caught++
+    bugClass.caught++
+    if (!m.preExisting) newCaught++
+  }
+  return {
+    misses: misses.length,
+    caught,
+    recall: ratio(caught, misses.length),
+    byDomain,
+    byClass,
+    excludingPreExisting: {
+      misses: newMisses,
+      caught: newCaught,
+      recall: ratio(newCaught, newMisses),
+    },
+  }
+}

package/skills/magpie/evals/quality/replay-critic.ts

@@ -7,6 +7,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import type { Subprocess } from 'bun'
+import { writeTarget } from '../../scripts/target.ts'
 import { originalSelection } from './baseline.ts'
 import {
   copyCorpusFiles,
@@ -127,6 +128,8 @@ async function main(argv: string[]): Promise<number> {
       'merge-candidates.json',
       'brief.json',
     ])
+    // magpie refuses a dir with neither target.json nor log.jsonl as not a run.
+    await writeTarget(scratch, { mode: 'pr', root: worktree })
 
     const batches = (await exec(['bun', MAGPIE_BIN, 'critic-prompt', scratch]))
       .split('\n')