@iceinvein/agent-skills 0.21.0 → 0.21.2

package/skills/magpie/evals/quality/README.md

@@ -0,0 +1,141 @@
+# Quality eval
+
+Measures how well magpie picks findings, using what reviewers actually did with past runs
+as labels: a final finding is `posted`, `dismissed` (with a reason) or `ignored`. The
+other evals in `evals/` check that the skill follows its walkthrough; this one checks
+whether the findings it keeps are the ones a reviewer wanted.
+
+Run every command from `skills/magpie`.
+
+## The corpus
+
+```
+bun evals/quality/corpus.ts
+```
+
+Copies every run under `~/.magpie` (or `$MAGPIE_HOME`; active and `.archived-` alike,
+`preview-*` skipped)
+that has both `post-status.json` and `findings.final.json` into
+`~/.magpie/corpus/<runId>/`: `pr.json`, `diff.patch`, `findings.deduped.json`,
+`findings.kept.json`, `findings.final.json`, `merge-candidates.json` and `brief.json`,
+each when present. It then writes `labels.json` by folding `post-status.json`,
+`state/events` and `log.jsonl`, and prints one line of counts per run. Rebuilding replaces
+each run's corpus directory.
+
+The corpus holds client code, so it never lives in the repo: the script exits 1 when the
+corpus dir resolves inside the git repository that holds it. `--out <dir>` points it
+elsewhere, for tests and dry runs; every script below takes the same flag.
+
+Older runs predate merge candidates, briefs and the dismiss UI. A missing optional file is
+read as absent; a file that is present but does not parse fails the build, naming it.
+
+## Metrics
+
+`score.ts` exports `scoreSelection`, which scores a set of kept ids against the labels:
+
+- `kept`: findings in the selection.
+- `posted`: posted labels on findings in the candidate pool the selection was made from
+  (`findings.deduped.json` for the critic). Peer-review additions reach
+  `findings.final.json` without passing the critic, so they are not held against it.
+- `postedKept`: kept findings that were posted.
+- `precision`: `postedKept` over kept findings that have a label. Every id in
+  `findings.final.json` has one; a kept id that peer review removed before the report is
+  labelled `ignored`, since the reviewer never saw it. A kept id with no label at all (a
+  replay keeping a candidate the original critic dropped) is left out of the denominator.
+  Null when nothing labelled was kept.
+- `recall`: `postedKept` over `posted`. Null when nothing was posted.
+- `dismissedKept`: kept findings dismissed, by reason (`wrong`, `not-worth-it`,
+  `duplicate`, `style`).
+- `byDomain`: `kept` and `postedKept` per specialist domain (`unassigned` when a finding
+  has none).
+- `byVia`: posted kept findings by post route (`recommended`, `selected`, `one`, `cli`;
+  `unrecorded` for runs that logged no route).
+
+`matchFindings` pairs a replay's freshly generated findings with labelled ones: same file,
+line within 5, title Dice of at least 0.4 on `tokenize(title)`. Pairs are taken best Dice
+first, each labelled finding matched at most once.
+
+## Confounds
+
+Read every number here with these in mind.
+
+- **Post Recommended.** The report's "Post recommended" button posts the top findings in
+  one click. A finding posted that way was not individually judged, so `posted` partly
+  measures rank, not worth. `byVia` separates those posts out where the run logged a
+  route.
+- **Ignored is not rejected.** Most runs predate the dismiss UI, so a finding the reviewer
+  disliked usually reads as `ignored`, not `dismissed`. Precision treats `ignored` as a
+  miss, which is harsh on any finding the reviewer simply did not get to.
+- **Repeated PRs.** Several runs review the same PR at different heads (three runs of
+  PR 50 today). Their findings overlap heavily, and a reviewer who posted a finding on one
+  run tends not to post it again on the next, which drags the later run's precision down.
+  Pooled totals weight those PRs once per run.
+- **Baseline recall is 1 by construction.** The reviewer only ever saw kept findings, so
+  every posted finding in the pool was kept. Recall only means something for a replay,
+  where a different critic can drop a finding the reviewer posted.
+
+## Baseline
+
+```
+bun evals/quality/baseline.ts
+```
+
+Scores each run's original `findings.kept.json` against its labels, prints a table with a
+pooled `TOTAL` row, and writes `~/.magpie/corpus/results/<ts>-baseline.json`. No model
+calls; free to run.
+
+## Critic replay
+
+```
+bun evals/quality/replay-critic.ts --corpus <runId> --repo <path-to-local-clone> [--design-cap <n>]
+```
+
+Re-runs stage 6 alone on one corpus run:
+
+1. `git -C <repo> worktree add --detach <scratch>/worktree <headRefOid>`. The PR head must
+   already be in the clone (`git fetch origin pull/<n>/head`); a missing sha exits 1.
+2. Copies the corpus files into a scratch run dir under the system temp dir.
+3. `magpie critic-prompt`, then one `claude -p --allowedTools Read,Grep,Glob,Write
+   --output-format json --add-dir <scratch>` per batch, prompt on stdin, from the
+   worktree. Batches run in parallel. A non-zero exit, an `is_error` result or a missing
+   output file fails the replay.
+4. `magpie critic-apply` (with `--design-cap <n>` when given, so a cap can be compared
+   against the default of 3), then `scoreSelection` over the new `findings.kept.json`,
+   written with the run's baseline score to `results/<ts>-critic[-cap<n>]-<runId>.json`.
+   The result also keeps every critic verdict, `critic-dropped.json`, and
+   `postedDropped`: each posted candidate the replay did not keep, with the critic's
+   reason, `design-cap`, or its merge target. Those lines are printed too, so a recall
+   loss can be pinned on the critic or the cap. `unlabelledKept` counts kept findings the
+   original critic had dropped: nobody ever judged them, so precision leaves them out
+   and the count says how much of the selection precision does not cover.
+5. Removes the worktree and scratch dir whatever happened.
+
+**Cost:** one Claude session per batch of up to 30 candidates; the corpus runs have 17 to
+84 deduped candidates, so 1 to 3 sessions each. Each session reads the candidates' code in
+the worktree, so cost grows with candidate count. The results file records the
+`total_cost_usd` Claude reports, summed over batches. There is no spend cap on this tier.
+
+## Full replay
+
+```
+bun evals/quality/replay-full.ts --corpus <runId> --repo <path-to-local-clone> [--max-cost-usd N]
+```
+
+Re-runs stages 3 to 6 (context, specialists, dedupe, critic) on the run's `diff.patch`:
+
+1. Scaffolds a scratch run dir: worktree at the PR head as above, `pr.json` and
+   `diff.patch` copied, the deterministic tests finding and a `setup` done line written in
+   place of `magpie setup` (which needs the live PR), then `magpie shard`.
+2. Runs one `claude -p --allowedTools Bash,Read,Grep,Glob,Write,Edit,Agent
+   --max-budget-usd <N> --output-format json` from the worktree, telling it to follow this
+   checkout's `SKILL.md`, resume from `magpie status`, and stop once `findings.kept.json`
+   is written. Serve, peer review, report, post and cleanup are skipped.
+3. Matches the new kept findings to `findings.final.json` with `matchFindings` and scores
+   them; unmatched findings count as `unlabelled` and stay out of precision. Writes
+   `results/<ts>-full-<runId>.json`.
+
+**Cost:** a whole review: a scout, five specialists per shard, and the critic batches.
+`--max-cost-usd` (default 10) is passed to Claude as `--max-budget-usd`, the CLI's own
+spend cap; `claude -p` has no `--max-cost-usd` flag. Hitting the cap stops the session,
+and the replay then fails without scoring, on Claude's error result or on the missing
+`findings.kept.json`.

package/skills/magpie/evals/quality/__tests__/replay-critic.test.ts

@@ -0,0 +1,31 @@
+import { expect, test } from 'bun:test'
+import { claudeFailureMessage, worktreeRemoveCommand } from '../replay-critic.ts'
+
+test('a failed claude run names its label and the result subtype from stdout', () => {
+  const stdout = JSON.stringify({ type: 'result', is_error: true, subtype: 'error_max_turns' })
+  expect(claudeFailureMessage('critic batch 2', 1, stdout, 'boom\n')).toBe(
+    'critic batch 2: claude -p exit 1; is_error true, subtype error_max_turns; stderr: boom',
+  )
+})
+
+test('a failed claude run with non-JSON stdout carries the raw stdout', () => {
+  expect(claudeFailureMessage('full replay', 137, 'Killed\n', '')).toBe(
+    'full replay: claude -p exit 137; stdout: Killed; stderr: ',
+  )
+})
+
+test('a worktree that was never added has no removal command', () => {
+  expect(worktreeRemoveCommand('/repo', '/scratch/worktree', false)).toBeNull()
+})
+
+test('an added worktree is removed with a forced git worktree remove', () => {
+  expect(worktreeRemoveCommand('/repo', '/scratch/worktree', true)).toEqual([
+    'git',
+    '-C',
+    '/repo',
+    'worktree',
+    'remove',
+    '--force',
+    '/scratch/worktree',
+  ])
+})

package/skills/magpie/evals/quality/__tests__/score.test.ts

@@ -0,0 +1,325 @@
+import { afterEach, beforeEach, expect, test } from 'bun:test'
+import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import type { FindingLabel } from '../../../scripts/labels.ts'
+import type { ReviewFinding } from '../../../scripts/types.ts'
+import { assertOutsideRepo, buildCorpusRun, listSourceRuns, loadCorpusRun } from '../corpus.ts'
+import {
+  actionableIds,
+  explainPostedDrops,
+  matchFindings,
+  parseClaudeResult,
+  scoreReplay,
+  scoreSelection,
+} from '../score.ts'
+
+function finding(over: Partial<ReviewFinding> & { id: string }): ReviewFinding {
+  return {
+    file: 'src/orders.ts',
+    line: 10,
+    severity: 'medium',
+    risk: { impact: 'medium', likelihood: 'possible', confidence: 'medium', action: 'should-fix' },
+    title: 'Something is off',
+    description: 'Details.',
+    domain: 'bugs',
+    ...over,
+  }
+}
+
+// 2 posted kept, 1 posted dropped, 1 dismissed `wrong` kept, 2 ignored kept.
+const fixtureFindings: ReviewFinding[] = [
+  finding({ id: 'p1', domain: 'bugs' }),
+  finding({ id: 'p2', domain: 'security' }),
+  finding({ id: 'p3', domain: 'bugs' }),
+  finding({ id: 'd1', domain: 'bugs' }),
+  finding({ id: 'i1', domain: 'code-smells' }),
+  finding({ id: 'i2', domain: 'code-smells' }),
+]
+const fixtureLabels: FindingLabel[] = [
+  { id: 'p1', label: 'posted', via: 'recommended' },
+  { id: 'p2', label: 'posted', via: 'selected' },
+  { id: 'p3', label: 'posted', via: 'selected' },
+  { id: 'd1', label: 'dismissed', reason: 'wrong' },
+  { id: 'i1', label: 'ignored' },
+  { id: 'i2', label: 'ignored' },
+]
+const fixtureKept = ['p1', 'p2', 'd1', 'i1', 'i2']
+
+function scoreFixture() {
+  return scoreSelection({ labels: fixtureLabels, keptIds: fixtureKept, findings: fixtureFindings })
+}
+
+test('precision is posted kept over labelled kept', () => {
+  expect(scoreFixture().precision).toBe(2 / 5)
+})
+
+test('recall is posted kept over all posted', () => {
+  expect(scoreFixture().recall).toBe(2 / 3)
+})
+
+test('kept dismissals are counted by reason', () => {
+  expect(scoreFixture().dismissedKept).toEqual({ wrong: 1 })
+})
+
+test('kept and posted-kept counts are split by domain', () => {
+  expect(scoreFixture().byDomain).toEqual({
+    bugs: { kept: 2, postedKept: 1 },
+    security: { kept: 1, postedKept: 1 },
+    'code-smells': { kept: 2, postedKept: 0 },
+  })
+})
+
+test('posted kept findings are counted by post route', () => {
+  expect(scoreFixture().byVia).toEqual({ recommended: 1, selected: 1 })
+})
+
+test('totals count kept, posted and posted kept', () => {
+  const s = scoreFixture()
+  expect([s.kept, s.posted, s.postedKept]).toEqual([5, 3, 2])
+})
+
+test('zero kept findings gives a null precision', () => {
+  const s = scoreSelection({ labels: fixtureLabels, keptIds: [], findings: fixtureFindings })
+  expect(s.precision).toBeNull()
+})
+
+test('a kept id with no label stays out of the precision denominator', () => {
+  const findings = [...fixtureFindings, finding({ id: 'new-1' })]
+  const s = scoreSelection({
+    labels: fixtureLabels,
+    keptIds: [...fixtureKept, 'new-1'],
+    findings,
+  })
+  expect([s.kept, s.precision]).toEqual([6, 2 / 5])
+})
+
+test('kept ids with no label are counted as unlabelled', () => {
+  const findings = [...fixtureFindings, finding({ id: 'new-1' }), finding({ id: 'new-2' })]
+  const s = scoreSelection({
+    labels: fixtureLabels,
+    keptIds: [...fixtureKept, 'new-1', 'new-2'],
+    findings,
+  })
+  expect(s.unlabelledKept).toBe(2)
+})
+
+test('a posted finding the critic dropped is explained by its drop reason', () => {
+  const drops = explainPostedDrops({
+    labels: fixtureLabels,
+    candidateIds: fixtureFindings.map((f) => f.id),
+    keptIds: fixtureKept,
+    dropped: [{ id: 'p3', reason: 'design-cap' }],
+    verdicts: [],
+  })
+  expect(drops).toEqual([{ id: 'p3', reason: 'design-cap' }])
+})
+
+test('a posted finding merged into another is explained by its merge target', () => {
+  const drops = explainPostedDrops({
+    labels: fixtureLabels,
+    candidateIds: fixtureFindings.map((f) => f.id),
+    keptIds: fixtureKept,
+    dropped: [],
+    verdicts: [{ id: 'p3', verdict: 'merge', mergeInto: 'p1' }],
+  })
+  expect(drops).toEqual([{ id: 'p3', reason: 'merged into p1' }])
+})
+
+test('a posted finding missing with no drop or merge record is an error naming it', () => {
+  expect(() =>
+    explainPostedDrops({
+      labels: fixtureLabels,
+      candidateIds: fixtureFindings.map((f) => f.id),
+      keptIds: fixtureKept,
+      dropped: [],
+      verdicts: [],
+    }),
+  ).toThrow('p3')
+})
+
+test('a posted finding outside the candidate pool is not reported as dropped', () => {
+  const drops = explainPostedDrops({
+    labels: [...fixtureLabels, { id: 'peer-1', label: 'posted' }],
+    candidateIds: fixtureFindings.map((f) => f.id),
+    keptIds: fixtureKept,
+    dropped: [{ id: 'p3', reason: 'speculative' }],
+    verdicts: [],
+  })
+  expect(drops).toEqual([{ id: 'p3', reason: 'speculative' }])
+})
+
+test('actionable ids keep must-fix and should-fix findings and leave out suggestions', () => {
+  const risk = (action: ReviewFinding['risk']['action']) => ({
+    impact: 'medium' as const,
+    likelihood: 'possible' as const,
+    confidence: 'high' as const,
+    action,
+  })
+  const kept = [
+    finding({ id: 'm', risk: risk('must-fix') }),
+    finding({ id: 's', risk: risk('should-fix') }),
+    finding({ id: 'c', risk: risk('consider') }),
+    finding({ id: 'o', risk: risk('optional') }),
+  ]
+  expect(actionableIds(kept)).toEqual(['m', 's'])
+})
+
+test('a kept id missing from the findings is an error naming it', () => {
+  expect(() =>
+    scoreSelection({ labels: fixtureLabels, keptIds: ['ghost'], findings: fixtureFindings }),
+  ).toThrow('ghost')
+})
+
+const labelled = finding({
+  id: 'L',
+  file: 'src/orders.ts',
+  line: 10,
+  title: 'Unbounded retry loop in fetchOrders handler',
+})
+
+test('a 4-line offset with a similar title matches', () => {
+  const moved = finding({
+    id: 'N',
+    line: 14,
+    title: 'Unbounded retry loop when fetchOrders fails',
+  })
+  expect(matchFindings([moved], [labelled])).toEqual(new Map([['N', 'L']]))
+})
+
+test('a 6-line offset does not match', () => {
+  const moved = finding({ id: 'N', line: 16, title: labelled.title })
+  expect(matchFindings([moved], [labelled]).size).toBe(0)
+})
+
+test('a different file does not match', () => {
+  const elsewhere = finding({ id: 'N', file: 'src/billing.ts', title: labelled.title })
+  expect(matchFindings([elsewhere], [labelled]).size).toBe(0)
+})
+
+test('two candidates for one labelled finding resolve to the higher dice', () => {
+  const weaker = finding({ id: 'B', line: 11, title: 'Retry loop missing backoff' })
+  const stronger = finding({ id: 'A', line: 12, title: labelled.title })
+  expect(matchFindings([weaker, stronger], [labelled])).toEqual(new Map([['A', 'L']]))
+})
+
+test('a replayed finding that matches a posted one scores as posted kept', () => {
+  const moved = finding({ id: 'bugs-9', line: 12, title: labelled.title })
+  const { score } = scoreReplay({
+    labels: [{ id: 'L', label: 'posted' }],
+    labelled: [labelled],
+    newKept: [moved],
+  })
+  expect([score.postedKept, score.precision]).toEqual([1, 1])
+})
+
+test('an unmatched replayed finding is unlabelled even when its id repeats a labelled id', () => {
+  const unrelated = finding({ id: 'L', file: 'src/billing.ts', title: 'Totally different' })
+  const { score, unlabelled } = scoreReplay({
+    labels: [{ id: 'L', label: 'posted' }],
+    labelled: [labelled],
+    newKept: [unrelated],
+  })
+  expect([unlabelled, score.postedKept, score.precision]).toEqual([1, 0, null])
+})
+
+let root: string
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), 'magpie-quality-'))
+})
+
+afterEach(async () => {
+  await rm(root, { recursive: true, force: true })
+})
+
+async function writeRun(dir: string, files: Record<string, string>): Promise<void> {
+  await mkdir(dir, { recursive: true })
+  for (const [name, content] of Object.entries(files)) {
+    await mkdir(join(dir, name, '..'), { recursive: true })
+    await writeFile(join(dir, name), content)
+  }
+}
+
+const postedRun = {
+  'post-status.json': '{"a":"posted"}',
+  'findings.final.json': JSON.stringify([finding({ id: 'a' })]),
+}
+
+test('an archived run with post status and final findings is a source run', async () => {
+  await writeRun(join(root, 'pr-50-1.archived-2'), postedRun)
+  expect(await listSourceRuns(root)).toEqual(['pr-50-1.archived-2'])
+})
+
+test('a preview run is not a source run', async () => {
+  await writeRun(join(root, 'preview-123'), postedRun)
+  expect(await listSourceRuns(root)).toEqual([])
+})
+
+test('a run that never posted is not a source run', async () => {
+  await writeRun(join(root, 'pr-7-1'), {
+    'findings.final.json': JSON.stringify([finding({ id: 'a' })]),
+  })
+  expect(await listSourceRuns(root)).toEqual([])
+})
+
+test('a corpus dir inside the repo is refused', () => {
+  expect(() => assertOutsideRepo('/work/repo/corpus', '/work/repo')).toThrow('/work/repo')
+})
+
+test('a corpus dir beside the repo with a shared name prefix is allowed', () => {
+  expect(() => assertOutsideRepo('/work/repo-corpus', '/work/repo')).not.toThrow()
+})
+
+test('a kept id absent from final findings is labelled ignored in the corpus', async () => {
+  const src = join(root, 'pr-9-1')
+  await writeRun(src, {
+    ...postedRun,
+    'findings.kept.json': JSON.stringify([finding({ id: 'a' }), finding({ id: 'b' })]),
+  })
+  await buildCorpusRun(src, join(root, 'corpus', 'pr-9-1'))
+  const labels = JSON.parse(await readFile(join(root, 'corpus', 'pr-9-1', 'labels.json'), 'utf8'))
+  expect(labels).toEqual([
+    { id: 'a', label: 'posted' },
+    { id: 'b', label: 'ignored' },
+  ])
+})
+
+test('a corpus run without merge candidates loads them as absent', async () => {
+  const dir = join(root, 'pr-9-1')
+  await writeRun(dir, {
+    'findings.final.json': JSON.stringify([finding({ id: 'a' })]),
+    'labels.json': JSON.stringify([{ id: 'a', label: 'ignored' }]),
+  })
+  expect((await loadCorpusRun(dir)).mergeCandidates).toBeNull()
+})
+
+test('a corpus file that fails to parse is an error naming the file', async () => {
+  const dir = join(root, 'pr-9-1')
+  await writeRun(dir, {
+    'findings.final.json': JSON.stringify([finding({ id: 'a' })]),
+    'labels.json': JSON.stringify([{ id: 'a', label: 'ignored' }]),
+    'merge-candidates.json': '{not json',
+  })
+  await expect(loadCorpusRun(dir)).rejects.toThrow(join(dir, 'merge-candidates.json'))
+})
+
+test('a claude result marked as an error is an error naming the step', () => {
+  const stdout = JSON.stringify({
+    type: 'result',
+    subtype: 'error_max_budget_usd',
+    is_error: true,
+    result: 'budget exceeded',
+  })
+  expect(() => parseClaudeResult(stdout, 'critic batch 2')).toThrow('critic batch 2')
+})
+
+test('a successful claude result reports its cost', () => {
+  const stdout = JSON.stringify({
+    type: 'result',
+    subtype: 'success',
+    is_error: false,
+    total_cost_usd: 0.42,
+  })
+  expect(parseClaudeResult(stdout, 'critic batch 1')).toEqual({ costUsd: 0.42 })
+})

package/skills/magpie/evals/quality/baseline.ts

@@ -0,0 +1,80 @@
+#!/usr/bin/env bun
+// Scores each corpus run's original critic selection (findings.kept.json)
+// against its labels: the number a replayed critic has to beat.
+
+import { mkdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { type CorpusRun, listCorpusRuns, loadCorpusRun, resolveCorpusRoot } from './corpus.ts'
+import {
+  formatRatio,
+  parseFlags,
+  poolRuns,
+  type QualityScore,
+  resultTimestamp,
+  scoreSelection,
+} from './score.ts'
+
+/** The critic chose from findings.deduped.json, so that is the pool recall counts against. */
+export function originalSelection(run: CorpusRun) {
+  if (!run.kept) throw new Error(`${run.dir} has no findings.kept.json`)
+  if (!run.deduped) throw new Error(`${run.dir} has no findings.deduped.json`)
+  return {
+    runId: run.runId,
+    labels: run.labels,
+    keptIds: run.kept.map((f) => f.id),
+    findings: run.deduped,
+  }
+}
+
+const DISMISS_COLUMNS = ['wrong', 'not-worth-it', 'duplicate', 'style']
+
+function row(name: string, s: QualityScore): string[] {
+  return [
+    name,
+    String(s.kept),
+    String(s.posted),
+    String(s.postedKept),
+    formatRatio(s.precision),
+    formatRatio(s.recall),
+    ...DISMISS_COLUMNS.map((r) => String(s.dismissedKept[r] ?? 0)),
+  ]
+}
+
+function renderTable(rows: string[][]): string {
+  const header = ['run', 'kept', 'posted', 'postedKept', 'precision', 'recall', ...DISMISS_COLUMNS]
+  const all = [header, ...rows]
+  const widths = header.map((_, i) => Math.max(...all.map((r) => (r[i] ?? '').length)))
+  return all.map((r) => r.map((cell, i) => cell.padEnd(widths[i] ?? 0)).join('  ')).join('\n')
+}
+
+async function main(argv: string[]): Promise<number> {
+  const corpusRoot = resolveCorpusRoot(parseFlags(argv, ['out']))
+  const selections = []
+  for (const runId of await listCorpusRuns(corpusRoot)) {
+    selections.push(originalSelection(await loadCorpusRun(join(corpusRoot, runId))))
+  }
+  if (selections.length === 0) throw new Error(`no corpus runs in ${corpusRoot}; run corpus.ts`)
+
+  const runs = selections.map((s) => ({ runId: s.runId, score: scoreSelection(s) }))
+  const total = scoreSelection(poolRuns(selections))
+  process.stdout.write(
+    `${renderTable([...runs.map((r) => row(r.runId, r.score)), row('TOTAL', total)])}\n`,
+  )
+
+  const resultsDir = join(corpusRoot, 'results')
+  await mkdir(resultsDir, { recursive: true })
+  const outPath = join(resultsDir, `${resultTimestamp(new Date())}-baseline.json`)
+  await writeFile(outPath, `${JSON.stringify({ tier: 'baseline', runs, total }, null, 2)}\n`)
+  process.stdout.write(`wrote ${outPath}\n`)
+  return 0
+}
+
+if (import.meta.main) {
+  main(process.argv.slice(2)).then(
+    (code) => process.exit(code),
+    (err) => {
+      process.stderr.write(`baseline: ${err instanceof Error ? err.message : String(err)}\n`)
+      process.exit(1)
+    },
+  )
+}