@iceinvein/agent-skills 0.23.8 → 0.23.12

package/skills/magpie/evals/quality/README.md

@@ -138,7 +138,8 @@ Re-runs stages 3 to 6 (context, specialists, dedupe, critic) on the run's `diff.
    `results/<ts>-full-<runId>.json` with `recall`, `recallVerdicts`, `judgeModel` and
    `judgeCostUsd` all null, plus `log`, `shardManifest`, `findingsFiles`, `dedupedCount`,
    `criticDropped`, `evidenceDropped` and `thresholdDropped` read from the scratch run dir
-   before it is deleted. `shardCoverage` is derived from the manifest and the findings file
+   before it is deleted, and `impactBytes`, the size of `impact.md`. It is null unless
+   `magpie impact` was run on the replay's run dir, which no review stage does now. `shardCoverage` is derived from the manifest and the findings file
    names: a shard counts as reviewed when any specialist wrote a file for it, so it overstates
    a shard only some focuses reviewed. The printed line ends with `, shards <reviewed>/<total>`.
 4. When the run has a recall set (see [Recall](#recall)), judges the new kept findings
@@ -153,6 +154,78 @@ spend cap; `claude -p` has no `--max-cost-usd` flag. Hitting the cap stops the s
 and the replay then fails without scoring, on Claude's error result or on the missing
 `findings.kept.json`.
 
+## Specialist probe
+
+```
+bun evals/quality/probe-specialist.ts --corpus <runId> --repo <path-to-local-clone> \
+  --focus <focus> --file <diff path> --misses <id,id> --arm base|impact --repeats <n> \
+  [--max-cost-usd N] [--judge-model M] [--dry-run]
+```
+
+Runs one specialist on one shard, with (`impact`) or without (`base`) the impact map in
+its prompt, and judges its findings against the named misses from the run's recall set.
+It asks a narrower question than a full replay, for about a tenth of the cost.
+
+The `impact` arm needs a `references/specialists.md` that carries a `magpie-impact-part`
+block, as it did at commit 731a686 (since reverted). Without that block the probe exits
+before any paid session, naming the missing block.
+
+Repeats run one after another, each in its own scratch run dir and worktree:
+
+1. Scaffolds the run dir as the full replay does (worktree at the PR head, `pr.json`,
+   `diff.patch` and `brief.json` copied, a `setup` done line, then `magpie shard`), and
+   picks the shard whose files include `--file`; no such shard exits 1, naming the file.
+2. Impact arm only: `magpie impact`. A skipped impact stage fails the probe, since the arm
+   would then be a base arm. The prompt then carries `Outside the diff: <path>` under
+   `Diff:` and the `magpie-impact-part` block, with `<top entries>` filled from the
+   shard's `impact.shard-<n>.brief.md` and `<the path above>` with the path
+   (`impact.md` and `impact.brief.md` on an unsharded run). A shard with no impact file
+   exits 1, naming the shard, before any session is paid for.
+3. Builds the prompt from this checkout's `references/specialists.md` the way stage 4
+   does: focus block, run header, Output Contract, the excluded-files block when the
+   corpus run has `excluded-files.json`, and the brief rendered from `brief.json`.
+4. Runs `claude -p --output-format stream-json --verbose --allowedTools
+   Bash,Read,Grep,Glob,Write --max-budget-usd <N>` from the worktree, prompt on stdin.
+5. Writes `results/<ts>-probe-<runId>-<focus>-s<shard>-<arm>-r<k>.json` as soon as the
+   session ends, with `costUsd` and `sessionError` (a non-zero exit, an `is_error` result
+   such as a spent budget, or no result event; the probe then exits 1 after finishing).
+   `transcriptBadLines` counts transcript lines that are not JSON (a killed session cuts
+   its last line off); they are skipped, not fatal. The results dir is checked writable
+   before the session starts. Each later step fills its fields and rewrites the file; a
+   step that throws records `error`, rewrites it, prints one stderr line naming the
+   repeat and moves on to the next repeat, and the command exits 1 at the end. A failure
+   before the session (setup, worktree, a missing impact file) stops every repeat. In
+   order:
+   - `findings`, from the focus's findings file for the shard, numbering any finding
+     with no id as `<focus>-s<n>-<i>`. A missing file records `findings: 0` and
+     `missingFindings` with the reason rather than failing. `readImpact`: the session
+     opened the impact file with the Read tool (a Bash `cat` is not counted); null on
+     the base arm.
+   - `citesOutside`: findings whose description cites a `file:line` the shard's impact
+     Markdown lists outside the shard. The base arm runs `magpie impact` after its
+     session ends, so both arms count against the same map. It is a lower bound: only an
+     exact `path:line` match counts, so a citation by line range or by a nearby line is
+     missed.
+   - `judged`, `caught`, `verdicts` and `judgeCostUsd` from `judgeRecall`. With no
+     findings the judge is not called, `judged` stays false and `notJudged` says why.
+6. Removes the worktree and scratch dir whatever happened. A probe killed before its
+   cleanup ran leaves a stale worktree registered in the clone: run
+   `git -C <repo> worktree prune` there.
+
+`--dry-run` stops after step 3, prints the chosen shard and the assembled prompt, cleans
+up and exits 0. It calls no model and spends nothing.
+
+**Reading results:** a phase A tally counts only runs with `judged: true` or with zero
+findings (`findings: 0`); a run with an `error`, or with findings that were never judged,
+is left out and re-run. Findings are judged straight from the specialist, before dedupe's
+evidence filter, on both arms alike, so a finding dedupe would drop can still count as a
+catch.
+
+**Cost:** one specialist session per repeat, about $1.1 to $1.3 on the corpus PRs, capped
+by `--max-cost-usd` (default 3, passed as `--max-budget-usd`), plus one judge call when
+there are findings. A session that hits the cap is recorded with its cost and
+`sessionError`, and its findings are still read and judged.
+
 ## Recall
 
 The labels above only score findings magpie raised. The recall set scores the other side:

package/skills/magpie/evals/quality/__tests__/probe-specialist.test.ts

@@ -0,0 +1,498 @@
+import { expect, test } from 'bun:test'
+import { mkdtemp, readFile, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { ReviewFinding } from '../../../scripts/types.ts'
+import {
+  assembleSpecialistPrompt,
+  countOutsideCitations,
+  EXCLUDED_FILES_BLOCK,
+  numberProbeFindings,
+  readImpactPart,
+  renderBriefPart,
+  RUN_HEADER_SHARDED,
+  RUN_HEADER_UNSHARDED,
+  BRIEF_CLOSING,
+  PostSessionError,
+  runRepeats,
+  sessionOutcome,
+  transcriptBadLines,
+  transcriptReadPath,
+} from '../probe-specialist.ts'
+
+const FENCE = '```'
+
+const SPECIALISTS_MD = `# Specialist prompts
+
+Prose for the orchestrator.
+
+## Output Contract
+
+Write findings to <RUN_DIR>/findings/<focus>.json.
+
+${FENCE}
+{ "id": string }
+${FENCE}
+
+If you have no findings, write [].
+
+## Focus blocks
+
+### security
+
+${FENCE}magpie-specialist-security
+You are a security reviewer.
+
+## What to look for
+
+Injection.
+${FENCE}
+
+### bugs
+
+${FENCE}magpie-specialist-bugs
+You are a bug hunter.
+${FENCE}
+`
+
+const IMPACT_PART = `
+6. The impact map:
+
+${FENCE}magpie-impact-part
+## Outside the diff
+
+<top entries>
+
+<the path above> has the full list.
+${FENCE}
+`
+
+const base = {
+  specialistsMd: SPECIALISTS_MD,
+  focus: 'security',
+  runDir: '/runs/r1',
+  worktree: '/runs/r1/worktree',
+  target: 'PR #70',
+  shard: null,
+  brief: null,
+  excludedFiles: false,
+  impact: null,
+}
+
+const SECURITY_BLOCK = 'You are a security reviewer.\n\n## What to look for\n\nInjection.'
+const CONTRACT =
+  '## Output Contract\n\nWrite findings to /runs/r1/findings/security.json.\n\n' +
+  '```\n{ "id": string }\n```\n\nIf you have no findings, write [].'
+
+test('an unsharded prompt is the focus block, the run header and the output contract', () => {
+  expect(assembleSpecialistPrompt(base)).toBe(
+    `${SECURITY_BLOCK}\n\n` +
+      'You are reviewing PR #70.\nWorking directory: /runs/r1/worktree\nDiff: /runs/r1/diff.patch\n\n' +
+      CONTRACT,
+  )
+})
+
+test('the output contract names the focus id, not a placeholder the session fills in', () => {
+  const prompt = assembleSpecialistPrompt(base)
+  expect(prompt).toContain('Write findings to /runs/r1/findings/security.json.')
+  expect(prompt).not.toContain('<focus>')
+})
+
+test('a sharded prompt names the shard, its patch and the scope rule', () => {
+  expect(assembleSpecialistPrompt({ ...base, shard: { n: 2, total: 5 } })).toBe(
+    `${SECURITY_BLOCK}\n\n` +
+      'You are reviewing PR #70.\nWorking directory: /runs/r1/worktree\nShard: 2 of 5\n' +
+      'Diff: /runs/r1/shards/shard-2.patch\n\n' +
+      'This shard is your review scope. The other shards belong to other agents working\n' +
+      'in parallel: do not review files outside your shard, and do not report findings\n' +
+      'anchored to them. Reading any file in the worktree for context is expected and\n' +
+      'encouraged.\n\n' +
+      CONTRACT,
+  )
+})
+
+test('the excluded-files block and the brief follow the contract in that order', () => {
+  const prompt = assembleSpecialistPrompt({
+    ...base,
+    excludedFiles: true,
+    brief: '## What this PR is for\n\nLocks the namespace.',
+  })
+  expect(prompt).toBe(
+    `${SECURITY_BLOCK}\n\n` +
+      'You are reviewing PR #70.\nWorking directory: /runs/r1/worktree\nDiff: /runs/r1/diff.patch\n\n' +
+      `${CONTRACT}\n\n` +
+      'Files excluded by the path filter are absent from your diff but present in the\n' +
+      'worktree, with their full patches in /runs/r1/diff.full.patch and the exclusion\n' +
+      'list in /runs/r1/excluded-files.json. When a change in your scope implies a change\n' +
+      'in an excluded file (a migration implies a model snapshot, a schema change implies\n' +
+      'generated types), open it and cross-check rather than treating it as out of scope.\n\n' +
+      '## What this PR is for\n\nLocks the namespace.',
+  )
+})
+
+test('an impact prompt puts the outside-diff line under Diff and the impact part last', () => {
+  const prompt = assembleSpecialistPrompt({
+    ...base,
+    specialistsMd: SPECIALISTS_MD + IMPACT_PART,
+    shard: { n: 2, total: 5 },
+    brief: '## What this PR is for\n\nLocks the namespace.',
+    impact: {
+      path: '/runs/r1/impact.shard-2.md',
+      inline: '### Sibling routes\n- `/a` (diff: a.ts:3)',
+    },
+  })
+  expect(prompt).toBe(
+    `${SECURITY_BLOCK}\n\n` +
+      'You are reviewing PR #70.\nWorking directory: /runs/r1/worktree\nShard: 2 of 5\n' +
+      'Diff: /runs/r1/shards/shard-2.patch\nOutside the diff: /runs/r1/impact.shard-2.md\n\n' +
+      'This shard is your review scope. The other shards belong to other agents working\n' +
+      'in parallel: do not review files outside your shard, and do not report findings\n' +
+      'anchored to them. Reading any file in the worktree for context is expected and\n' +
+      'encouraged.\n\n' +
+      `${CONTRACT}\n\n` +
+      '## What this PR is for\n\nLocks the namespace.\n\n' +
+      '## Outside the diff\n\n### Sibling routes\n- `/a` (diff: a.ts:3)\n\n' +
+      '/runs/r1/impact.shard-2.md has the full list.',
+  )
+})
+
+test('dollar patterns in the inline entries are pasted as written', () => {
+  const prompt = assembleSpecialistPrompt({
+    ...base,
+    specialistsMd: SPECIALISTS_MD + IMPACT_PART,
+    impact: { path: '/runs/r1/impact.md', inline: "  - a.ts:3 `s.replace(x, '$&$1')` `$'` `$$`" },
+  })
+  expect(prompt).toEndWith(
+    "## Outside the diff\n\n  - a.ts:3 `s.replace(x, '$&$1')` `$'` `$$`\n\n" +
+      '/runs/r1/impact.md has the full list.',
+  )
+})
+
+test('dollar patterns in the impact path are pasted as written', () => {
+  const prompt = assembleSpecialistPrompt({
+    ...base,
+    specialistsMd: SPECIALISTS_MD + IMPACT_PART,
+    impact: { path: "/runs/$&$'/impact.md", inline: 'top' },
+  })
+  expect(prompt).toEndWith("top\n\n/runs/$&$'/impact.md has the full list.")
+  expect(prompt).toContain("Diff: /runs/r1/diff.patch\nOutside the diff: /runs/$&$'/impact.md\n")
+})
+
+test('an unsharded impact prompt adds the outside-diff line as the header’s last line', () => {
+  const prompt = assembleSpecialistPrompt({
+    ...base,
+    specialistsMd: SPECIALISTS_MD + IMPACT_PART,
+    impact: { path: '/runs/r1/impact.md', inline: 'top' },
+  })
+  expect(prompt).toContain(
+    'Diff: /runs/r1/diff.patch\nOutside the diff: /runs/r1/impact.md\n\n## Output Contract',
+  )
+})
+
+test('an impact prompt without the impact part in specialists.md is refused, naming the block', () => {
+  expect(() =>
+    assembleSpecialistPrompt({ ...base, impact: { path: '/runs/r1/impact.md', inline: 'top' } }),
+  ).toThrow('magpie-impact-part')
+})
+
+test('a focus with no block in specialists.md is refused, naming the focus', () => {
+  expect(() => assembleSpecialistPrompt({ ...base, focus: 'performance' })).toThrow(
+    'magpie-specialist-performance',
+  )
+})
+
+test('the live specialists.md carries the run header, excluded-files and brief text verbatim', async () => {
+  const live = await readFile(join(import.meta.dir, '../../../references/specialists.md'), 'utf8')
+  for (const template of [
+    RUN_HEADER_UNSHARDED,
+    RUN_HEADER_SHARDED,
+    EXCLUDED_FILES_BLOCK,
+    BRIEF_CLOSING,
+  ]) {
+    expect(live).toContain(template)
+  }
+})
+
+test('a full brief renders every heading in the part-5 order with the closing paragraph', () => {
+  expect(
+    renderBriefPart({
+      purpose: 'Lock the UUID namespace.',
+      changes: ['Adds a namespace constant', 'Verifies reports'],
+      watchItems: ['Old reports'],
+      unclear: ['Is the namespace public?'],
+      reviewRules: [{ rule: 'No raw SQL', source: 'CONTRIBUTING.md' }],
+    }),
+  ).toBe(
+    '## What this PR is for\n\nLock the UUID namespace.\n\n' +
+      'What it does:\n- Adds a namespace constant\n- Verifies reports\n\n' +
+      'Watch items:\n- Old reports\n\n' +
+      'Open questions the scout could not resolve:\n- Is the namespace public?\n\n' +
+      'Repository review rules:\n- No raw SQL (CONTRIBUTING.md)\n\n' +
+      "This brief is the author's claim as understood by a reader who has not reviewed the\n" +
+      'code. It is not ground truth. Where the diff contradicts it, that is a finding in\n' +
+      'your domain, not a correction to the brief. A watch item is a pointer, not a verdict:\n' +
+      'escalate it into a finding with your own risk fields, or leave it alone. A repository\n' +
+      'review rule is a convention the maintainers wrote down: a change that breaks one is a\n' +
+      "finding in your domain when the rule bears on it, citing the rule's source.",
+  )
+})
+
+test('a brief with no watch items, open questions or rules omits those headings', () => {
+  expect(
+    renderBriefPart({
+      purpose: 'Lock it.',
+      changes: ['Adds a constant'],
+      watchItems: [],
+      unclear: [],
+      reviewRules: [],
+    }),
+  ).toBe(
+    '## What this PR is for\n\nLock it.\n\nWhat it does:\n- Adds a constant\n\n' +
+      "This brief is the author's claim as understood by a reader who has not reviewed the\n" +
+      'code. It is not ground truth. Where the diff contradicts it, that is a finding in\n' +
+      'your domain, not a correction to the brief. A watch item is a pointer, not a verdict:\n' +
+      'escalate it into a finding with your own risk fields, or leave it alone. A repository\n' +
+      'review rule is a convention the maintainers wrote down: a change that breaks one is a\n' +
+      "finding in your domain when the rule bears on it, citing the rule's source.",
+  )
+})
+
+function assistantToolUse(name: string, input: Record<string, unknown>): string {
+  return JSON.stringify({
+    type: 'assistant',
+    message: {
+      content: [
+        { type: 'text', text: 'looking' },
+        { type: 'tool_use', name, input },
+      ],
+    },
+  })
+}
+
+const STREAM_HEAD = '{"type":"system","subtype":"init"}\n'
+
+test('a transcript that Reads the impact file counts as reading it', () => {
+  const stream = `${STREAM_HEAD}${assistantToolUse('Read', { file_path: '/runs/r1/impact.md' })}\n`
+  expect(transcriptReadPath(stream, '/runs/r1/impact.md')).toBe(true)
+})
+
+test('a transcript that only cats the impact file through Bash does not count', () => {
+  const stream = `${STREAM_HEAD}${assistantToolUse('Bash', { command: 'cat /runs/r1/impact.md' })}\n`
+  expect(transcriptReadPath(stream, '/runs/r1/impact.md')).toBe(false)
+})
+
+test('a transcript that Reads other files does not count as reading the impact file', () => {
+  const stream =
+    `${STREAM_HEAD}${assistantToolUse('Read', { file_path: '/runs/r1/impact.shard-2.md' })}\n` +
+    `${assistantToolUse('Read', { file_path: '/runs/r1/diff.patch' })}\n`
+  expect(transcriptReadPath(stream, '/runs/r1/impact.md')).toBe(false)
+})
+
+const READ_IMPACT = assistantToolUse('Read', { file_path: '/runs/r1/impact.md' })
+
+test('a transcript cut off mid-line still shows the Read made before the cut', () => {
+  const stream = `${STREAM_HEAD}${READ_IMPACT}\n{"type":"assistant","message":{"con`
+  expect(transcriptReadPath(stream, '/runs/r1/impact.md')).toBe(true)
+})
+
+test('a transcript cut off mid-line counts the cut line as bad', () => {
+  const stream = `${STREAM_HEAD}${READ_IMPACT}\n{"type":"assistant","message":{"con`
+  expect(transcriptBadLines(stream)).toBe(1)
+})
+
+test('a whole transcript has no bad lines', () => {
+  expect(transcriptBadLines(`${STREAM_HEAD}${READ_IMPACT}\n`)).toBe(0)
+})
+
+test('a successful session reports its cost and no error', () => {
+  const stream = `${STREAM_HEAD}{"type":"result","subtype":"success","is_error":false,"total_cost_usd":1.25}\n`
+  expect(sessionOutcome(stream, 0, '')).toEqual({ costUsd: 1.25, error: null })
+})
+
+test('a result event with no cost reports the cost as null', () => {
+  expect(
+    sessionOutcome(`${STREAM_HEAD}{"type":"result","subtype":"success","is_error":false}\n`, 0, ''),
+  ).toEqual({ costUsd: null, error: null })
+})
+
+test('a session that ended in error keeps its cost and records the error', () => {
+  const stream =
+    `${STREAM_HEAD}{"type":"result","subtype":"error_max_budget_usd","is_error":true,` +
+    '"total_cost_usd":3.02,"result":"budget spent"}\n'
+  expect(sessionOutcome(stream, 1, 'boom')).toEqual({
+    costUsd: 3.02,
+    error: 'session ended in error (error_max_budget_usd): budget spent; exit 1: boom',
+  })
+})
+
+test('a session with no result event records that instead of throwing', () => {
+  expect(sessionOutcome(STREAM_HEAD, 0, '')).toEqual({
+    costUsd: null,
+    error: 'stream-json has no result event',
+  })
+})
+
+test('a killed session whose transcript is cut off records no result event and the exit', () => {
+  expect(sessionOutcome(`${STREAM_HEAD}{"type":"res`, 137, 'killed')).toEqual({
+    costUsd: null,
+    error: 'stream-json has no result event; exit 137: killed',
+  })
+})
+
+function finding(id: string, file: string, description: string): ReviewFinding {
+  return {
+    id,
+    file,
+    line: 1,
+    severity: 'medium',
+    risk: { impact: 'medium', likelihood: 'possible', confidence: 'medium', action: 'should-fix' },
+    title: 't',
+    description,
+    domain: 'security',
+  }
+}
+
+const IMPACT_MD = [
+  '### Other writers of the same data',
+  '- `reports` (diff: src/api/lock.ts:10)',
+  '  - src/db/writer.ts:44 `insert into reports`',
+  '  - src/api/lock.ts:80 `update reports`',
+].join('\n')
+
+test('a finding citing a map location outside the shard counts as an outside citation', () => {
+  const findings = [
+    finding('a', 'src/api/lock.ts', 'Observation: src/db/writer.ts:44 also writes reports.'),
+  ]
+  expect(countOutsideCitations(findings, IMPACT_MD, ['src/api/lock.ts'])).toBe(1)
+})
+
+test('a map location inside the shard, or a location not in the map, is not an outside citation', () => {
+  const findings = [
+    finding('a', 'src/api/lock.ts', 'Observation: src/api/lock.ts:80 updates reports.'),
+    finding('b', 'src/api/lock.ts', 'Observation: src/db/other.ts:12 reads reports.'),
+    finding('c', 'src/api/lock.ts', 'Observation: src/db/writer.ts:45 is near the writer.'),
+  ]
+  expect(countOutsideCitations(findings, IMPACT_MD, ['src/api/lock.ts'])).toBe(0)
+})
+
+test('a finding citing several outside locations counts once', () => {
+  const impact = `${IMPACT_MD}\n  - src/db/audit.ts:7 \`insert into reports\``
+  const findings = [
+    finding('a', 'src/api/lock.ts', 'See src/db/writer.ts:44 and src/db/audit.ts:7.'),
+  ]
+  expect(countOutsideCitations(findings, impact, ['src/api/lock.ts'])).toBe(1)
+})
+
+const RISK = { impact: 'high', likelihood: 'likely', confidence: 'high', action: 'must-fix' }
+
+test('findings without an id are numbered by focus and shard from 1', () => {
+  const findings = numberProbeFindings(
+    [
+      { file: 'a.ts', line: 1, risk: RISK, title: 'x', description: 'd' },
+      { id: 'security-7', file: 'b.ts', line: 2, risk: RISK, title: 'y', description: 'd' },
+      { file: 'c.ts', line: 3, risk: RISK, title: 'z', description: 'd' },
+    ],
+    'security',
+    3,
+    'findings/security.shard-3.json',
+  )
+  expect(findings.map((f) => f.id)).toEqual(['security-s3-1', 'security-7', 'security-s3-3'])
+})
+
+test('a findings file that is not an array is rejected, naming it', () => {
+  expect(() => numberProbeFindings({}, 'security', 1, 'findings/security.json')).toThrow(
+    'findings/security.json',
+  )
+})
+
+async function runDirWith(files: Record<string, string>): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), 'magpie-probe-impact-'))
+  for (const [name, text] of Object.entries(files)) await writeFile(join(dir, name), text)
+  return dir
+}
+
+test('a shard with no impact file is refused, naming the shard', async () => {
+  const dir = await runDirWith({ 'impact.md': 'whole run\n', 'impact.brief.md': 'top\n' })
+  await expect(readImpactPart(dir, { n: 2, total: 5 })).rejects.toThrow('shard 2')
+})
+
+test('a sharded run reads its shard’s impact file and brief', async () => {
+  const dir = await runDirWith({
+    'impact.shard-2.md': 'full\n',
+    'impact.shard-2.brief.md': 'top entries\n',
+  })
+  expect(await readImpactPart(dir, { n: 2, total: 5 })).toEqual({
+    path: join(dir, 'impact.shard-2.md'),
+    inline: 'top entries',
+  })
+})
+
+test('an unsharded run with no impact.md is refused', async () => {
+  const dir = await runDirWith({})
+  await expect(readImpactPart(dir, null)).rejects.toThrow('impact.md')
+})
+
+const LIVE_MD = join(import.meta.dir, '../../../references/specialists.md')
+
+test('a base prompt from the live specialists.md carries the focus block and the whole contract', async () => {
+  const prompt = assembleSpecialistPrompt({
+    ...base,
+    specialistsMd: await readFile(LIVE_MD, 'utf8'),
+    focus: 'bugs',
+  })
+  expect(prompt).toStartWith(
+    'You are a senior software engineer specialized in finding bugs through code review.\n',
+  )
+  expect(prompt).toContain(
+    'If you have no findings, write []. Return as your final tool result a single line: ' +
+      '`bugs: <N> findings (<critical>/<high>/<medium>/<low> by risk.impact)`. ' +
+      'Do not include other prose.',
+  )
+})
+
+test('the live specialists.md names the part-5 headings renderBriefPart writes', async () => {
+  const live = await readFile(LIVE_MD, 'utf8')
+  for (const heading of [
+    '## What this PR is for',
+    'What it does:',
+    'Watch items:',
+    'Open questions the scout could not resolve:',
+    'Repository review rules:',
+  ]) {
+    expect(live).toContain(`\n${heading}\n`)
+  }
+})
+
+test('a repeat that fails after its session is reported and the next repeat still runs', async () => {
+  const ran: number[] = []
+  const warnings: string[] = []
+  const failed = await runRepeats(
+    3,
+    async (repeat) => {
+      ran.push(repeat)
+      if (repeat === 2) throw new PostSessionError('judge failed (probe result written to /r.json)')
+    },
+    (line) => warnings.push(line),
+  )
+  expect({ ran, failed, warnings }).toEqual({
+    ran: [1, 2, 3],
+    failed: 1,
+    warnings: [
+      'repeat 2 failed after its session: judge failed (probe result written to /r.json)\n',
+    ],
+  })
+})
+
+test('a repeat that fails before its session stops the remaining repeats', async () => {
+  const ran: number[] = []
+  const run = runRepeats(
+    3,
+    async (repeat) => {
+      ran.push(repeat)
+      if (repeat === 1) throw new Error('worktree add failed')
+    },
+    () => {},
+  )
+  await expect(run).rejects.toThrow('worktree add failed')
+  expect(ran).toEqual([1])
+})

package/skills/magpie/evals/quality/__tests__/replay-full.test.ts

@@ -114,6 +114,7 @@ test('diagnostics carry the log, manifest, findings files, deduped count and cri
     criticDropped: [{ id: 'b', reason: 'speculative' }],
     evidenceDropped: null,
     thresholdDropped: null,
+    impactBytes: null,
   })
 })
 
@@ -129,6 +130,7 @@ test('diagnostics report absent manifest, deduped and drop files as null', async
     criticDropped: null,
     evidenceDropped: null,
     thresholdDropped: null,
+    impactBytes: null,
   })
 })
 
@@ -162,3 +164,10 @@ test('a deduped findings file that is not an array is rejected', async () => {
   await writeFile(join(dir, 'findings.deduped.json'), '{"id":"a"}')
   await expect(replayDiagnostics(dir)).rejects.toThrow('findings.deduped.json')
 })
+
+test('diagnostics report the impact map size when the run wrote impact.md', async () => {
+  const dir = await scratchRunDir()
+  await writeFile(join(dir, 'log.jsonl'), '{"stage":"setup","status":"done"}\n')
+  await writeFile(join(dir, 'impact.md'), '### Sibling routes\n- `/é`\n')
+  expect((await replayDiagnostics(dir)).impactBytes).toBe(27)
+})