@iceinvein/agent-skills 0.23.9 → 0.23.13

package/skills/magpie/scripts/__tests__/proof.test.ts

@@ -0,0 +1,226 @@
+import { expect, test } from 'bun:test'
+import {
+  applyProofResults,
+  buildProofPrompts,
+  type ProofResult,
+  parseProofResult,
+  selectForProof,
+} from '../proof.ts'
+import type { ReviewFinding, Risk } from '../types.ts'
+
+function finding(id: string, risk: Risk): ReviewFinding {
+  return {
+    id,
+    file: 'src/a.ts',
+    line: 10,
+    severity: 'high',
+    risk,
+    title: `title ${id}`,
+    description: `description ${id}`,
+    domain: 'bugs',
+    evidence: 'return cache[key]',
+  }
+}
+
+const HIGH_SHOULD: Risk = {
+  impact: 'high',
+  likelihood: 'possible',
+  confidence: 'medium',
+  action: 'should-fix',
+}
+const CRITICAL_MUST: Risk = {
+  impact: 'critical',
+  likelihood: 'likely',
+  confidence: 'high',
+  action: 'must-fix',
+}
+const MEDIUM_SHOULD: Risk = { ...HIGH_SHOULD, impact: 'medium' }
+const HIGH_CONSIDER: Risk = { ...HIGH_SHOULD, action: 'consider' }
+
+const REPRO = {
+  path: 'src/__tests__/magpie-proof-a.test.ts',
+  command: 'bun test src/__tests__/magpie-proof-a.test.ts',
+  source: "test('x', () => expect(get('k')).toBe(1))",
+  output: 'expected 1, received undefined',
+}
+
+test('findings marked must-fix or should-fix are selected whatever their impact', () => {
+  const selected = selectForProof(
+    [
+      finding('high-should', HIGH_SHOULD),
+      finding('medium-should', MEDIUM_SHOULD),
+      finding('low-should', { ...HIGH_SHOULD, impact: 'low' }),
+      finding('high-consider', HIGH_CONSIDER),
+      finding('high-optional', { ...HIGH_SHOULD, action: 'optional' }),
+      finding('critical-must', CRITICAL_MUST),
+    ],
+    8,
+  )
+  expect(selected.map((f) => f.id)).toEqual([
+    'critical-must',
+    'high-should',
+    'medium-should',
+    'low-should',
+  ])
+})
+
+test('selection keeps the highest-scoring findings when there are more than the cap', () => {
+  const selected = selectForProof(
+    [
+      finding('low-score', { ...HIGH_SHOULD, likelihood: 'edge-case' }),
+      finding('top', CRITICAL_MUST),
+      finding('middle', HIGH_SHOULD),
+    ],
+    2,
+  )
+  expect(selected.map((f) => f.id)).toEqual(['top', 'middle'])
+})
+
+const TEMPLATE = 'F=<<FINDING>> W=<<WORKTREE>> D=<<DIFF_PATH>> O=<<OUTPUT_PATH>>'
+
+test('one prompt per finding, each naming its own output file', () => {
+  const prompts = buildProofPrompts({
+    template: TEMPLATE,
+    findings: [finding('a', HIGH_SHOULD)],
+    worktree: '/wt',
+    diffPath: '/run/diff.patch',
+    runDir: '/run',
+  })
+  expect(prompts).toHaveLength(1)
+  expect(prompts[0]?.id).toBe('a')
+  expect(prompts[0]?.outputPath).toBe('/run/proof-a.json')
+  expect(prompts[0]?.prompt).toBe(
+    `F=${JSON.stringify(
+      {
+        id: 'a',
+        file: 'src/a.ts',
+        line: 10,
+        risk: HIGH_SHOULD,
+        domain: 'bugs',
+        title: 'title a',
+        description: 'description a',
+        evidence: 'return cache[key]',
+      },
+      null,
+      2,
+    )} W=/wt D=/run/diff.patch O=/run/proof-a.json`,
+  )
+})
+
+test('a template missing a placeholder is rejected, naming it', () => {
+  expect(() =>
+    buildProofPrompts({
+      template: 'F=<<FINDING>> W=<<WORKTREE>> D=<<DIFF_PATH>>',
+      findings: [],
+      worktree: '/wt',
+      diffPath: '/d',
+      runDir: '/run',
+    }),
+  ).toThrow('<<OUTPUT_PATH>>')
+})
+
+test('an id that is not safe in a file name is rejected', () => {
+  expect(() =>
+    buildProofPrompts({
+      template: TEMPLATE,
+      findings: [finding('../x', HIGH_SHOULD)],
+      worktree: '/wt',
+      diffPath: '/d',
+      runDir: '/run',
+    }),
+  ).toThrow('../x')
+})
+
+test('a reproduced result carries its repro', () => {
+  expect(
+    parseProofResult(
+      { id: 'a', outcome: 'reproduced', reason: 'fails as described', repro: REPRO },
+      'proof-a.json',
+    ),
+  ).toEqual({ id: 'a', outcome: 'reproduced', reason: 'fails as described', repro: REPRO })
+})
+
+test('a not-proven result needs only a reason', () => {
+  expect(
+    parseProofResult({ id: 'a', outcome: 'not-proven', reason: 'no test runner' }, 'proof-a.json'),
+  ).toEqual({ id: 'a', outcome: 'not-proven', reason: 'no test runner' })
+})
+
+test('a reproduced or disproved result without a repro is rejected', () => {
+  expect(() =>
+    parseProofResult({ id: 'a', outcome: 'reproduced', reason: 'r' }, 'proof-a.json'),
+  ).toThrow('proof-a.json: reproduced needs a repro with path, command, source and output')
+  expect(() =>
+    parseProofResult(
+      { id: 'a', outcome: 'disproved', reason: 'r', repro: { ...REPRO, output: '' } },
+      'proof-a.json',
+    ),
+  ).toThrow('proof-a.json: disproved needs a repro with path, command, source and output')
+})
+
+test('an unknown outcome is rejected', () => {
+  expect(() =>
+    parseProofResult({ id: 'a', outcome: 'maybe', reason: 'r' }, 'proof-a.json'),
+  ).toThrow('proof-a.json: outcome must be reproduced, disproved or not-proven')
+})
+
+test('applying results marks reproduced findings, raises their confidence and keeps the repro', () => {
+  // high impact (0.4 x 7) + possible (0.25 x 6) + high confidence (0.2 x 10) + must-fix (0.15 x 10)
+  const risk: Risk = { ...HIGH_SHOULD, action: 'must-fix' }
+  const a = finding('a', risk)
+  const result = applyProofResults({
+    findings: [a],
+    results: [{ id: 'a', outcome: 'reproduced', reason: 'fails as described', repro: REPRO }],
+  })
+  expect(result.findings).toEqual([
+    {
+      ...a,
+      risk: { ...risk, confidence: 'high' },
+      score: 7.8,
+      proof: { status: 'reproduced', reason: 'fails as described', repro: REPRO },
+    },
+  ])
+  expect(result.dropped).toEqual([])
+})
+
+test('a not-proven finding keeps its risk and records why', () => {
+  const a = finding('a', HIGH_SHOULD)
+  const result = applyProofResults({
+    findings: [a],
+    results: [{ id: 'a', outcome: 'not-proven', reason: 'needs a live database' }],
+  })
+  expect(result.findings).toEqual([
+    { ...a, proof: { status: 'not-proven', reason: 'needs a live database' } },
+  ])
+})
+
+test('a disproved finding is removed and recorded with its repro', () => {
+  const result = applyProofResults({
+    findings: [finding('a', HIGH_SHOULD), finding('b', MEDIUM_SHOULD)],
+    results: [{ id: 'a', outcome: 'disproved', reason: 'the guard at a.ts:4 holds', repro: REPRO }],
+  })
+  expect(result.findings.map((f) => f.id)).toEqual(['b'])
+  expect(result.dropped).toEqual([
+    {
+      id: 'a',
+      file: 'src/a.ts',
+      line: 10,
+      title: 'title a',
+      reason: 'the guard at a.ts:4 holds',
+      repro: REPRO,
+    },
+  ])
+})
+
+test('findings with no result pass through unchanged', () => {
+  const b = finding('b', MEDIUM_SHOULD)
+  const result = applyProofResults({ findings: [b], results: [] })
+  expect(result.findings).toEqual([b])
+})
+
+test('a result for an id not in the findings is rejected', () => {
+  const results: ProofResult[] = [{ id: 'ghost', outcome: 'not-proven', reason: 'r' }]
+  expect(() => applyProofResults({ findings: [], results })).toThrow(
+    'proof result for unknown id: ghost',
+  )
+})

package/skills/magpie/scripts/__tests__/refresh.test.ts

@@ -151,3 +151,74 @@ test('refreshFindings rejects on a truncated events line and leaves the previous
   const kept = await readFile(join(runDir, 'screen', 'findings.html'), 'utf8')
   expect(kept).toBe('<html>OLD</html>')
 })
+
+test('refreshFindings keeps the disproved-by-test panel on re-render', async () => {
+  await writeFile(join(runDir, 'findings.final.json'), JSON.stringify([sampleFinding]))
+  await writeFile(
+    join(runDir, 'proof-dropped.json'),
+    JSON.stringify([
+      {
+        id: 'b',
+        file: 'src/y.ts',
+        line: 4,
+        title: 'Cache never evicts',
+        reason: 'evict runs on every put',
+        repro: { path: 'p', command: 'bun test p', source: 's', output: 'o' },
+      },
+    ]),
+  )
+  await refreshFindings(runDir)
+  const fresh = await readFile(join(runDir, 'screen', 'findings.html'), 'utf8')
+  expect(fresh).toContain('Disproved by test (1)')
+  expect(fresh).toContain('Cache never evicts: evict runs on every put')
+})
+
+test('refreshFindings keeps the skipped-shards note on re-render', async () => {
+  await writeFile(join(runDir, 'findings.final.json'), JSON.stringify([sampleFinding]))
+  await mkdir(join(runDir, 'shards'))
+  await writeFile(
+    join(runDir, 'shards', 'manifest.json'),
+    JSON.stringify({
+      budget: 6000,
+      maxFiles: 80,
+      totalFiles: 2,
+      totalLines: 20,
+      shards: [
+        { id: 1, path: 'shards/shard-1.patch', files: ['src/x.ts'], lines: 10 },
+        { id: 2, path: 'shards/shard-2.patch', files: ['src/z.ts'], lines: 10 },
+      ],
+    }),
+  )
+  await writeFile(
+    join(runDir, 'log.jsonl'),
+    `${JSON.stringify({ stage: 'shard-coverage', status: 'partial', reviewed: [1], skipped: [2] })}\n`,
+  )
+  await refreshFindings(runDir)
+  const fresh = await readFile(join(runDir, 'screen', 'findings.html'), 'utf8')
+  expect(fresh).toContain('Not reviewed: shard 2 (1 file)')
+})
+
+test('refreshFindings keeps the previous-run line on a re-review', async () => {
+  await writeFile(join(runDir, 'findings.final.json'), JSON.stringify([sampleFinding]))
+  await writeFile(
+    join(runDir, 'previous-findings.json'),
+    JSON.stringify({
+      previousRunId: 'pr-7-100',
+      findings: [
+        {
+          id: 'bugs-1',
+          fromRunId: 'pr-7-100',
+          file: 'src/x.ts',
+          line: 1,
+          title: 't',
+          domain: 'bugs',
+          label: 'posted',
+          anchor: 'changed',
+        },
+      ],
+    }),
+  )
+  await refreshFindings(runDir)
+  const fresh = await readFile(join(runDir, 'screen', 'findings.html'), 'utf8')
+  expect(fresh).toContain('Last review: 1 posted · 1 on changed code · 0 on unchanged code')
+})

package/skills/magpie/scripts/__tests__/render-annotation.test.ts

@@ -221,4 +221,38 @@ describe('renderAnnotation', () => {
     expect(reasons).toEqual(['wrong', 'not-worth-it', 'duplicate', 'style'])
     expect(html).not.toContain('data-dismissed=')
   })
+  test('a reproduced finding shows a reproduced chip and its repro', () => {
+    const html = renderAnnotation(
+      {
+        ...finding,
+        proof: {
+          status: 'reproduced',
+          reason: 'fails as described',
+          repro: {
+            path: 'cache.test.ts',
+            command: 'bun test cache.test.ts',
+            source: 'expect(a).toBe(b)',
+            output: 'expected <b>',
+          },
+        },
+      },
+      { checked: false, posted: false, asCard: true, highlighter: hl },
+    )
+    expect(html).toContain(
+      '<span class="proof-chip reproduced" title="fails as described">REPRODUCED</span>',
+    )
+    expect(html).toContain('<code class="inline-code">bun test cache.test.ts</code>')
+    expect(html).toContain('<pre class="repro">expect(a).toBe(b)</pre>')
+    expect(html).toContain('<pre class="repro">expected &lt;b&gt;</pre>')
+  })
+
+  test('a not-proven finding shows a not-proven chip with the reason', () => {
+    const html = renderAnnotation(
+      { ...finding, proof: { status: 'not-proven', reason: 'needs a live database' } },
+      { checked: false, posted: false, asCard: true, highlighter: hl },
+    )
+    expect(html).toContain(
+      '<span class="proof-chip not-proven" title="needs a live database">NOT PROVEN</span>',
+    )
+  })
 })

package/skills/magpie/scripts/__tests__/render-findings.test.ts

@@ -381,6 +381,81 @@ test('findings header notes a locally-built diff', () => {
   expect(html).toContain('aaaaaaaaaaaa')
 })
 
+test('findings header carries the previous-run line on a re-review', () => {
+  const html = renderFindingsHtml({
+    findings: [],
+    postStatus: {},
+    pr: { number: 7, branch: 'feature-x', headSha: 'abc123abc123' },
+    previousRun: 'Last review: 3 posted · 2 on changed code · 1 on unchanged code',
+    dismissed: new Map(),
+    topN: DEFAULT_TOP_N,
+    highlighter: hl,
+  })
+  expect(html).toContain('Last review: 3 posted · 2 on changed code · 1 on unchanged code')
+})
+
+test('the page lists findings a passing test disproved, folded, with reason and repro', () => {
+  const html = renderFindingsHtml({
+    findings: [],
+    postStatus: {},
+    pr: { number: 7, branch: 'feature-x', headSha: 'abc123abc123' },
+    disproved: [
+      {
+        id: 'bugs-2',
+        file: 'src/cache.ts',
+        line: 40,
+        title: 'Cache <never> evicts',
+        reason: 'evict() runs on every put at cache.ts:12',
+        repro: {
+          path: 'cache.magpie-proof-bugs-2.test.ts',
+          command: 'bun test cache.magpie-proof-bugs-2.test.ts',
+          source: 'expect(size()).toBe(1)',
+          output: '1 pass',
+        },
+      },
+    ],
+    dismissed: new Map(),
+    topN: DEFAULT_TOP_N,
+    highlighter: hl,
+  })
+  expect(html).toContain(
+    '<details class="pr-brief disproved"><summary class="brief-summary">Disproved by test (1)</summary>',
+  )
+  expect(html).toContain(
+    '<li><code>src/cache.ts:40</code> Cache &lt;never&gt; evicts: evict() runs on every put at cache.ts:12',
+  )
+  expect(html).toContain('<code>bun test cache.magpie-proof-bugs-2.test.ts</code>')
+  expect(html).toContain('<pre class="repro">expect(size()).toBe(1)</pre>')
+  expect(html).toContain('<pre class="repro">1 pass</pre>')
+})
+
+test('the page has no disproved panel when proof dropped nothing', () => {
+  const html = renderFindingsHtml({
+    findings: [],
+    postStatus: {},
+    pr: { number: 7, branch: 'feature-x', headSha: 'abc123abc123' },
+    dismissed: new Map(),
+    topN: DEFAULT_TOP_N,
+    highlighter: hl,
+  })
+  expect(html).not.toContain('Disproved by test')
+})
+
+test('findings header names the shards nobody reviewed, with their files on hover', () => {
+  const html = renderFindingsHtml({
+    findings: [],
+    postStatus: {},
+    pr: { number: 7, branch: 'feature-x', headSha: 'abc123abc123' },
+    unreviewed: { skipped: [{ id: 3, files: ['docs/d.md', 'src/<x>.ts'] }] },
+    dismissed: new Map(),
+    topN: DEFAULT_TOP_N,
+    highlighter: hl,
+  })
+  expect(html).toContain(
+    '<span class="diff-source" title="docs/d.md, src/&lt;x&gt;.ts">Not reviewed: shard 3 (2 files)</span>',
+  )
+})
+
 test('findings header stays quiet when the diff came from gh', () => {
   const html = renderFindingsHtml({
     findings: [],

package/skills/magpie/scripts/__tests__/render-progress.test.ts

@@ -13,6 +13,7 @@ test('renderProgressHtml shows all stages with status classes and PR header', ()
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -44,6 +45,7 @@ test('renderProgressHtml renders empty specialists-line before any focus reports
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -64,6 +66,7 @@ test('pipeline carries a --done-count style for the filling connector line', ()
       dedupe: 'running',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -85,6 +88,7 @@ test('each pipeline stage exposes a short sublabel so first-timers can learn wha
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -107,6 +111,7 @@ test('renderProgressHtml includes the archived banner element', () => {
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -127,6 +132,7 @@ test('emits the shared pr-header chrome', () => {
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -149,6 +155,7 @@ test('progress-pane block replaces diff-pane on the progress page', () => {
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -170,6 +177,7 @@ test('progress names the shard count while specialists run', () => {
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },
@@ -191,6 +199,7 @@ test('progress keeps the single-shard wording when there is one shard', () => {
       dedupe: 'pending',
       critic: 'pending',
       'peer-review': 'pending',
+      proof: 'pending',
       report: 'pending',
       post: 'pending',
     },