@iceinvein/agent-skills 0.23.9 → 0.23.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/index.json +2 -2
- package/skills/magpie/SKILL.md +21 -20
- package/skills/magpie/bin/magpie.ts +63 -2
- package/skills/magpie/evals/quality/README.md +74 -1
- package/skills/magpie/evals/quality/__tests__/probe-specialist.test.ts +498 -0
- package/skills/magpie/evals/quality/__tests__/replay-full.test.ts +9 -0
- package/skills/magpie/evals/quality/probe-specialist.ts +672 -0
- package/skills/magpie/evals/quality/replay-full.ts +6 -2
- package/skills/magpie/fixtures/fake-gh-with-lockfile.sh +37 -2
- package/skills/magpie/fixtures/fake-gh.sh +49 -2
- package/skills/magpie/references/critic.md +20 -4
- package/skills/magpie/references/proof.md +102 -0
- package/skills/magpie/references/scout.md +6 -1
- package/skills/magpie/references/specialists.md +15 -9
- package/skills/magpie/scripts/__tests__/cleanup-cmd.test.ts +103 -7
- package/skills/magpie/scripts/__tests__/cli.test.ts +69 -0
- package/skills/magpie/scripts/__tests__/critic-cmd.test.ts +46 -0
- package/skills/magpie/scripts/__tests__/critic.test.ts +11 -1
- package/skills/magpie/scripts/__tests__/dedupe-cmd.test.ts +20 -1
- package/skills/magpie/scripts/__tests__/git-diff.test.ts +23 -4
- package/skills/magpie/scripts/__tests__/handoff-cmd.test.ts +68 -0
- package/skills/magpie/scripts/__tests__/history.test.ts +232 -0
- package/skills/magpie/scripts/__tests__/impact-cmd.test.ts +477 -0
- package/skills/magpie/scripts/__tests__/impact-render.test.ts +365 -0
- package/skills/magpie/scripts/__tests__/impact.test.ts +915 -0
- package/skills/magpie/scripts/__tests__/incremental.test.ts +59 -8
- package/skills/magpie/scripts/__tests__/pipeline-e2e.test.ts +1 -1
- package/skills/magpie/scripts/__tests__/post-cmd.test.ts +58 -0
- package/skills/magpie/scripts/__tests__/pr-activity.test.ts +281 -0
- package/skills/magpie/scripts/__tests__/preview-cmd.test.ts +4 -4
- package/skills/magpie/scripts/__tests__/previous-findings.test.ts +267 -0
- package/skills/magpie/scripts/__tests__/proof-cmd.test.ts +260 -0
- package/skills/magpie/scripts/__tests__/proof.test.ts +226 -0
- package/skills/magpie/scripts/__tests__/refresh.test.ts +71 -0
- package/skills/magpie/scripts/__tests__/render-annotation.test.ts +34 -0
- package/skills/magpie/scripts/__tests__/render-findings.test.ts +75 -0
- package/skills/magpie/scripts/__tests__/render-progress.test.ts +9 -0
- package/skills/magpie/scripts/__tests__/setup-cmd.test.ts +376 -1
- package/skills/magpie/scripts/__tests__/shard-coverage.test.ts +79 -0
- package/skills/magpie/scripts/__tests__/skill-lint.test.ts +48 -2
- package/skills/magpie/scripts/__tests__/status-cmd.test.ts +20 -8
- package/skills/magpie/scripts/__tests__/types.test.ts +46 -0
- package/skills/magpie/scripts/cleanup-cmd.ts +88 -0
- package/skills/magpie/scripts/critic-cmd.ts +2 -0
- package/skills/magpie/scripts/critic.ts +4 -0
- package/skills/magpie/scripts/dedupe-cmd.ts +4 -0
- package/skills/magpie/scripts/evidence-filter.ts +10 -0
- package/skills/magpie/scripts/git-diff.ts +50 -19
- package/skills/magpie/scripts/handoff-cmd.ts +42 -3
- package/skills/magpie/scripts/history.ts +143 -0
- package/skills/magpie/scripts/impact-cmd.ts +202 -0
- package/skills/magpie/scripts/impact-render.ts +158 -0
- package/skills/magpie/scripts/impact.ts +622 -0
- package/skills/magpie/scripts/incremental.ts +44 -5
- package/skills/magpie/scripts/labels-cmd.ts +7 -3
- package/skills/magpie/scripts/post-cmd.ts +46 -0
- package/skills/magpie/scripts/pr-activity.ts +243 -0
- package/skills/magpie/scripts/preview-cmd.ts +3 -0
- package/skills/magpie/scripts/previous-findings.ts +181 -0
- package/skills/magpie/scripts/proof-cmd.ts +226 -0
- package/skills/magpie/scripts/proof.ts +150 -0
- package/skills/magpie/scripts/refresh.ts +11 -0
- package/skills/magpie/scripts/render-annotation.ts +17 -0
- package/skills/magpie/scripts/render-cmd.ts +12 -0
- package/skills/magpie/scripts/render-findings.ts +37 -1
- package/skills/magpie/scripts/render-progress.ts +3 -0
- package/skills/magpie/scripts/setup-cmd.ts +142 -11
- package/skills/magpie/scripts/shard-coverage.ts +53 -0
- package/skills/magpie/scripts/status-cmd.ts +2 -0
- package/skills/magpie/scripts/types.ts +39 -0
- package/skills/magpie/skill.json +1 -1
- package/skills/magpie/templates/styles.css +30 -0
- package/skills/sluice/scripts/status.sh +32 -13
- package/skills/sluice/skill.json +1 -1
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import { appendFile, copyFile, readFile, writeFile } from 'node:fs/promises'
|
|
2
|
+
import { join } from 'node:path'
|
|
3
|
+
import {
|
|
4
|
+
applyProofResults,
|
|
5
|
+
buildProofPrompts,
|
|
6
|
+
DEFAULT_PROOF_CAP,
|
|
7
|
+
type ProofDropped,
|
|
8
|
+
type ProofResult,
|
|
9
|
+
parseProofResult,
|
|
10
|
+
selectForProof,
|
|
11
|
+
} from './proof.ts'
|
|
12
|
+
import { reviewRoot } from './target.ts'
|
|
13
|
+
import { isRepro, parseFinding, type ReviewFinding } from './types.ts'
|
|
14
|
+
|
|
15
|
+
const DEFAULT_TEMPLATE_PATH = new URL('../references/proof.md', import.meta.url).pathname
|
|
16
|
+
|
|
17
|
+
async function logLine(runDir: string, entry: Record<string, unknown>): Promise<void> {
|
|
18
|
+
const line = `${JSON.stringify({ ...entry, ts: Date.now() })}\n`
|
|
19
|
+
await appendFile(join(runDir, 'log.jsonl'), line)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function isMissingFile(err: unknown): boolean {
|
|
23
|
+
return (err as NodeJS.ErrnoException)?.code === 'ENOENT'
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** The prompt is the body of the fenced block tagged `magpie-proof` (3 or 4 backticks). */
|
|
27
|
+
function extractTemplate(doc: string, path: string): string {
|
|
28
|
+
const match = doc.match(/^(`{3,4})magpie-proof\n([\s\S]*?)\n\1[ \t]*$/m)
|
|
29
|
+
if (!match?.[2]) throw new Error(`${path}: no magpie-proof fenced block`)
|
|
30
|
+
return match[2]
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function readFindings(path: string): Promise<ReviewFinding[]> {
|
|
34
|
+
const raw: unknown = JSON.parse(await readFile(path, 'utf8'))
|
|
35
|
+
if (!Array.isArray(raw)) throw new Error(`${path}: expected a JSON array`)
|
|
36
|
+
return raw.map(parseFinding)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* proof-apply rewrites findings.final.json, so the findings as the peer review
|
|
41
|
+
* left them are kept aside on the first proof-prompt. Selecting from that copy
|
|
42
|
+
* makes a re-run of either command land on the same findings.
|
|
43
|
+
*/
|
|
44
|
+
async function readUnproven(runDir: string): Promise<ReviewFinding[]> {
|
|
45
|
+
const unproven = join(runDir, 'findings.unproven.json')
|
|
46
|
+
try {
|
|
47
|
+
return await readFindings(unproven)
|
|
48
|
+
} catch (err) {
|
|
49
|
+
if (!isMissingFile(err)) throw err
|
|
50
|
+
}
|
|
51
|
+
await copyFile(join(runDir, 'findings.final.json'), unproven)
|
|
52
|
+
return readFindings(unproven)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export type RunProofPromptOptions = {
|
|
56
|
+
/** Markdown file holding the `magpie-proof` block. Defaults to `references/proof.md`. */
|
|
57
|
+
templatePath?: string
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export async function runProofPrompt(
|
|
61
|
+
runDir: string,
|
|
62
|
+
options: RunProofPromptOptions = {},
|
|
63
|
+
): Promise<number> {
|
|
64
|
+
const templatePath = options.templatePath ?? DEFAULT_TEMPLATE_PATH
|
|
65
|
+
try {
|
|
66
|
+
const template = extractTemplate(await readFile(templatePath, 'utf8'), templatePath)
|
|
67
|
+
const prompts = buildProofPrompts({
|
|
68
|
+
template,
|
|
69
|
+
findings: selectForProof(await readUnproven(runDir), DEFAULT_PROOF_CAP),
|
|
70
|
+
worktree: await reviewRoot(runDir),
|
|
71
|
+
diffPath: join(runDir, 'diff.patch'),
|
|
72
|
+
runDir,
|
|
73
|
+
})
|
|
74
|
+
const lines: string[] = []
|
|
75
|
+
for (const p of prompts) {
|
|
76
|
+
const promptPath = join(runDir, `proof-prompt-${p.id}.md`)
|
|
77
|
+
await writeFile(promptPath, p.prompt)
|
|
78
|
+
lines.push(`${promptPath}\t${p.outputPath}\n`)
|
|
79
|
+
}
|
|
80
|
+
process.stdout.write(lines.join(''))
|
|
81
|
+
return 0
|
|
82
|
+
} catch (err) {
|
|
83
|
+
process.stderr.write(`proof-prompt: ${err instanceof Error ? err.message : String(err)}\n`)
|
|
84
|
+
return 1
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Null when the subagent wrote no file, so the caller can name every missing one. */
|
|
89
|
+
async function readResult(path: string, id: string): Promise<ProofResult | null> {
|
|
90
|
+
let text: string
|
|
91
|
+
try {
|
|
92
|
+
text = await readFile(path, 'utf8')
|
|
93
|
+
} catch (err) {
|
|
94
|
+
if (isMissingFile(err)) return null
|
|
95
|
+
throw err
|
|
96
|
+
}
|
|
97
|
+
let raw: unknown
|
|
98
|
+
try {
|
|
99
|
+
raw = JSON.parse(text)
|
|
100
|
+
} catch (err) {
|
|
101
|
+
throw new Error(`${path}: ${err instanceof Error ? err.message : String(err)}`)
|
|
102
|
+
}
|
|
103
|
+
const result = parseProofResult(raw, path)
|
|
104
|
+
if (result.id !== id) throw new Error(`${path}: id is ${result.id}, expected ${id}`)
|
|
105
|
+
return result
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Provers name their file `magpie-proof-<id>`, or `MagpieProof<id>` where a name must be an identifier.
|
|
109
|
+
const REPRO_NAME = /magpie[-_]?proof/i
|
|
110
|
+
|
|
111
|
+
/** A prover that crashed or skipped its cleanup leaves its test in the tree, which in local mode is the author's. */
|
|
112
|
+
async function leftoverRepros(root: string, gitBin: string): Promise<string[]> {
|
|
113
|
+
const proc = Bun.spawn([gitBin, 'ls-files', '--others', '--exclude-standard', '-z'], {
|
|
114
|
+
cwd: root,
|
|
115
|
+
stdout: 'pipe',
|
|
116
|
+
stderr: 'pipe',
|
|
117
|
+
})
|
|
118
|
+
const [exit, out, err] = await Promise.all([
|
|
119
|
+
proc.exited,
|
|
120
|
+
new Response(proc.stdout).text(),
|
|
121
|
+
new Response(proc.stderr).text(),
|
|
122
|
+
])
|
|
123
|
+
if (exit !== 0) throw new Error(`git ls-files in ${root} exit ${exit}: ${err.trim()}`)
|
|
124
|
+
return out.split('\0').filter((path) => REPRO_NAME.test(path.split('/').at(-1) ?? ''))
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Runner output carries the worktree's absolute path, which names the reviewer's home directory on GitHub. */
|
|
128
|
+
function relativise(result: ProofResult, root: string): ProofResult {
|
|
129
|
+
if (result.outcome === 'not-proven') return result
|
|
130
|
+
const strip = (text: string) => text.replaceAll(root, '.')
|
|
131
|
+
const { path, command, source, output } = result.repro
|
|
132
|
+
return {
|
|
133
|
+
...result,
|
|
134
|
+
repro: { path, command: strip(command), source: strip(source), output: strip(output) },
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export async function runProofApply(
|
|
139
|
+
runDir: string,
|
|
140
|
+
opts: { gitBin?: string } = {},
|
|
141
|
+
): Promise<number> {
|
|
142
|
+
try {
|
|
143
|
+
const root = await reviewRoot(runDir)
|
|
144
|
+
const unprovenPath = join(runDir, 'findings.unproven.json')
|
|
145
|
+
let findings: ReviewFinding[]
|
|
146
|
+
try {
|
|
147
|
+
findings = await readFindings(unprovenPath)
|
|
148
|
+
} catch (err) {
|
|
149
|
+
if (isMissingFile(err)) throw new Error(`${unprovenPath} is missing; run proof-prompt first`)
|
|
150
|
+
throw err
|
|
151
|
+
}
|
|
152
|
+
const missing: string[] = []
|
|
153
|
+
const results: ProofResult[] = []
|
|
154
|
+
for (const f of selectForProof(findings, DEFAULT_PROOF_CAP)) {
|
|
155
|
+
const path = join(runDir, `proof-${f.id}.json`)
|
|
156
|
+
const result = await readResult(path, f.id)
|
|
157
|
+
if (result === null) missing.push(path)
|
|
158
|
+
else results.push(relativise(result, root))
|
|
159
|
+
}
|
|
160
|
+
if (missing.length > 0) throw new Error(`no proof result: ${missing.join(', ')}`)
|
|
161
|
+
const leftovers = await leftoverRepros(root, opts.gitBin ?? 'git')
|
|
162
|
+
if (leftovers.length > 0) {
|
|
163
|
+
throw new Error(`provers left files in ${root}; delete them: ${leftovers.join(', ')}`)
|
|
164
|
+
}
|
|
165
|
+
const applied = applyProofResults({ findings, results })
|
|
166
|
+
await writeFile(
|
|
167
|
+
join(runDir, 'findings.final.json'),
|
|
168
|
+
`${JSON.stringify(applied.findings, null, 2)}\n`,
|
|
169
|
+
)
|
|
170
|
+
await writeFile(
|
|
171
|
+
join(runDir, 'proof-dropped.json'),
|
|
172
|
+
`${JSON.stringify(applied.dropped, null, 2)}\n`,
|
|
173
|
+
)
|
|
174
|
+
await logLine(runDir, {
|
|
175
|
+
stage: 'proof',
|
|
176
|
+
status: 'done',
|
|
177
|
+
reproduced: results.filter((r) => r.outcome === 'reproduced').length,
|
|
178
|
+
notProven: results.filter((r) => r.outcome === 'not-proven').length,
|
|
179
|
+
disproved: applied.dropped.length,
|
|
180
|
+
})
|
|
181
|
+
return 0
|
|
182
|
+
} catch (err) {
|
|
183
|
+
process.stderr.write(`proof-apply: ${err instanceof Error ? err.message : String(err)}\n`)
|
|
184
|
+
return 1
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** What proof-apply removed, for the report and the handoff; a run before proof has none. */
|
|
189
|
+
export async function readProofDropped(runDir: string): Promise<ProofDropped[]> {
|
|
190
|
+
const path = join(runDir, 'proof-dropped.json')
|
|
191
|
+
let text: string
|
|
192
|
+
try {
|
|
193
|
+
text = await readFile(path, 'utf8')
|
|
194
|
+
} catch (err) {
|
|
195
|
+
if (isMissingFile(err)) return []
|
|
196
|
+
throw err
|
|
197
|
+
}
|
|
198
|
+
let raw: unknown
|
|
199
|
+
try {
|
|
200
|
+
raw = JSON.parse(text)
|
|
201
|
+
} catch (err) {
|
|
202
|
+
throw new Error(`${path}: ${err instanceof Error ? err.message : String(err)}`)
|
|
203
|
+
}
|
|
204
|
+
if (!Array.isArray(raw)) throw new Error(`${path}: expected a JSON array`)
|
|
205
|
+
return raw.map((entry, i) => {
|
|
206
|
+
const e = (entry ?? {}) as Record<string, unknown>
|
|
207
|
+
if (
|
|
208
|
+
typeof e.id !== 'string' ||
|
|
209
|
+
typeof e.file !== 'string' ||
|
|
210
|
+
(e.line !== null && typeof e.line !== 'number') ||
|
|
211
|
+
typeof e.title !== 'string' ||
|
|
212
|
+
typeof e.reason !== 'string' ||
|
|
213
|
+
!isRepro(e.repro)
|
|
214
|
+
) {
|
|
215
|
+
throw new Error(`${path}: entry ${i} needs id, file, line, title, reason and repro`)
|
|
216
|
+
}
|
|
217
|
+
return {
|
|
218
|
+
id: e.id,
|
|
219
|
+
file: e.file,
|
|
220
|
+
line: e.line as number | null,
|
|
221
|
+
title: e.title,
|
|
222
|
+
reason: e.reason,
|
|
223
|
+
repro: e.repro,
|
|
224
|
+
}
|
|
225
|
+
})
|
|
226
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { join } from 'node:path'
|
|
2
|
+
import { scoreRisk } from './score.ts'
|
|
3
|
+
import { isRepro, type Repro, type ReviewFinding } from './types.ts'
|
|
4
|
+
|
|
5
|
+
/** Each proof is one subagent writing and running a test, so the count is the cost. */
|
|
6
|
+
export const DEFAULT_PROOF_CAP = 8
|
|
7
|
+
|
|
8
|
+
export const PROOF_PLACEHOLDERS = [
|
|
9
|
+
'<<FINDING>>',
|
|
10
|
+
'<<WORKTREE>>',
|
|
11
|
+
'<<DIFF_PATH>>',
|
|
12
|
+
'<<OUTPUT_PATH>>',
|
|
13
|
+
] as const
|
|
14
|
+
|
|
15
|
+
const PLACEHOLDER_PATTERN = new RegExp(PROOF_PLACEHOLDERS.join('|'), 'g')
|
|
16
|
+
|
|
17
|
+
// The id names the prompt and output files.
|
|
18
|
+
const SAFE_ID = /^[A-Za-z0-9_-]+$/
|
|
19
|
+
|
|
20
|
+
const OUTCOMES = ['reproduced', 'disproved', 'not-proven'] as const
|
|
21
|
+
|
|
22
|
+
export type ProofResult =
|
|
23
|
+
| { id: string; outcome: 'reproduced' | 'disproved'; reason: string; repro: Repro }
|
|
24
|
+
| { id: string; outcome: 'not-proven'; reason: string }
|
|
25
|
+
|
|
26
|
+
export type ProofDropped = {
|
|
27
|
+
id: string
|
|
28
|
+
file: string
|
|
29
|
+
line: number | null
|
|
30
|
+
title: string
|
|
31
|
+
reason: string
|
|
32
|
+
repro: Repro
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function selectForProof(findings: ReviewFinding[], cap: number): ReviewFinding[] {
|
|
36
|
+
// Action, not impact: the critic keeps a confirmed defect at should-fix even when
|
|
37
|
+
// it rates the impact low, and a real PR left nothing high to prove.
|
|
38
|
+
const eligible = findings.filter(
|
|
39
|
+
(f) => f.risk.action === 'must-fix' || f.risk.action === 'should-fix',
|
|
40
|
+
)
|
|
41
|
+
// Scored from risk here, not read from `score`: the peer reviewer may have
|
|
42
|
+
// changed risk or added findings after dedupe set it.
|
|
43
|
+
return [...eligible].sort((a, b) => scoreRisk(b.risk) - scoreRisk(a.risk)).slice(0, cap)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function buildProofPrompts(input: {
|
|
47
|
+
template: string
|
|
48
|
+
findings: ReviewFinding[]
|
|
49
|
+
worktree: string
|
|
50
|
+
diffPath: string
|
|
51
|
+
runDir: string
|
|
52
|
+
}): Array<{ id: string; prompt: string; outputPath: string }> {
|
|
53
|
+
const { template, findings, worktree, diffPath, runDir } = input
|
|
54
|
+
const missing = PROOF_PLACEHOLDERS.filter((p) => !template.includes(p))
|
|
55
|
+
if (missing.length > 0) {
|
|
56
|
+
throw new Error(`proof template is missing placeholders: ${missing.join(', ')}`)
|
|
57
|
+
}
|
|
58
|
+
return findings.map((f) => {
|
|
59
|
+
if (!SAFE_ID.test(f.id)) throw new Error(`finding id ${f.id} is not safe in a file name`)
|
|
60
|
+
const outputPath = join(runDir, `proof-${f.id}.json`)
|
|
61
|
+
const finding = JSON.stringify(
|
|
62
|
+
{
|
|
63
|
+
id: f.id,
|
|
64
|
+
file: f.file,
|
|
65
|
+
line: f.line,
|
|
66
|
+
risk: f.risk,
|
|
67
|
+
domain: f.domain,
|
|
68
|
+
title: f.title,
|
|
69
|
+
description: f.description,
|
|
70
|
+
evidence: f.evidence,
|
|
71
|
+
},
|
|
72
|
+
null,
|
|
73
|
+
2,
|
|
74
|
+
)
|
|
75
|
+
const values: Record<(typeof PROOF_PLACEHOLDERS)[number], string> = {
|
|
76
|
+
'<<FINDING>>': finding,
|
|
77
|
+
'<<WORKTREE>>': worktree,
|
|
78
|
+
'<<DIFF_PATH>>': diffPath,
|
|
79
|
+
'<<OUTPUT_PATH>>': outputPath,
|
|
80
|
+
}
|
|
81
|
+
// A function replacer, so `$` in a finding's text is not read as a pattern.
|
|
82
|
+
const prompt = template.replace(PLACEHOLDER_PATTERN, (m) => values[m as keyof typeof values])
|
|
83
|
+
return { id: f.id, prompt, outputPath }
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function parseProofResult(raw: unknown, source: string): ProofResult {
|
|
88
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
|
89
|
+
throw new Error(`${source}: expected a JSON object`)
|
|
90
|
+
}
|
|
91
|
+
const r = raw as Record<string, unknown>
|
|
92
|
+
if (typeof r.id !== 'string') throw new Error(`${source}: id must be a string`)
|
|
93
|
+
if (typeof r.reason !== 'string' || r.reason === '') {
|
|
94
|
+
throw new Error(`${source}: reason must be a non-empty string`)
|
|
95
|
+
}
|
|
96
|
+
if (!(OUTCOMES as readonly unknown[]).includes(r.outcome)) {
|
|
97
|
+
throw new Error(`${source}: outcome must be reproduced, disproved or not-proven`)
|
|
98
|
+
}
|
|
99
|
+
if (r.outcome === 'not-proven') return { id: r.id, outcome: 'not-proven', reason: r.reason }
|
|
100
|
+
const outcome = r.outcome as 'reproduced' | 'disproved'
|
|
101
|
+
if (!isRepro(r.repro)) {
|
|
102
|
+
throw new Error(`${source}: ${outcome} needs a repro with path, command, source and output`)
|
|
103
|
+
}
|
|
104
|
+
const { path, command, source: code, output } = r.repro
|
|
105
|
+
return {
|
|
106
|
+
id: r.id,
|
|
107
|
+
outcome,
|
|
108
|
+
reason: r.reason,
|
|
109
|
+
repro: { path, command, source: code, output },
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function applyProofResults(input: { findings: ReviewFinding[]; results: ProofResult[] }): {
|
|
114
|
+
findings: ReviewFinding[]
|
|
115
|
+
dropped: ProofDropped[]
|
|
116
|
+
} {
|
|
117
|
+
const ids = new Set(input.findings.map((f) => f.id))
|
|
118
|
+
const unknown = input.results.filter((r) => !ids.has(r.id)).map((r) => r.id)
|
|
119
|
+
if (unknown.length > 0) throw new Error(`proof result for unknown id: ${unknown.join(', ')}`)
|
|
120
|
+
const byId = new Map(input.results.map((r) => [r.id, r]))
|
|
121
|
+
const findings: ReviewFinding[] = []
|
|
122
|
+
const dropped: ProofDropped[] = []
|
|
123
|
+
for (const f of input.findings) {
|
|
124
|
+
const r = byId.get(f.id)
|
|
125
|
+
if (r === undefined) {
|
|
126
|
+
findings.push(f)
|
|
127
|
+
} else if (r.outcome === 'disproved') {
|
|
128
|
+
dropped.push({
|
|
129
|
+
id: f.id,
|
|
130
|
+
file: f.file,
|
|
131
|
+
line: f.line,
|
|
132
|
+
title: f.title,
|
|
133
|
+
reason: r.reason,
|
|
134
|
+
repro: r.repro,
|
|
135
|
+
})
|
|
136
|
+
} else if (r.outcome === 'not-proven') {
|
|
137
|
+
findings.push({ ...f, proof: { status: 'not-proven', reason: r.reason } })
|
|
138
|
+
} else {
|
|
139
|
+
// A failing test that shows the defect leaves nothing to be unsure of.
|
|
140
|
+
const risk = { ...f.risk, confidence: 'high' as const }
|
|
141
|
+
findings.push({
|
|
142
|
+
...f,
|
|
143
|
+
risk,
|
|
144
|
+
score: scoreRisk(risk),
|
|
145
|
+
proof: { status: 'reproduced', reason: r.reason, repro: r.repro },
|
|
146
|
+
})
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return { findings, dropped }
|
|
150
|
+
}
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { readdir, readFile, rename, unlink } from 'node:fs/promises'
|
|
2
2
|
import { basename, join } from 'node:path'
|
|
3
|
+
import { readHistory } from './history.ts'
|
|
3
4
|
import { readDismissed } from './labels.ts'
|
|
5
|
+
import { describePreviousPosted } from './previous-findings.ts'
|
|
6
|
+
import { readProofDropped } from './proof-cmd.ts'
|
|
4
7
|
import { type PostStatusMap, parseClosingIssues, renderFindingsToDisk } from './render-findings.ts'
|
|
5
8
|
import { DEFAULT_TOP_N } from './render-issues-list.ts'
|
|
9
|
+
import { readShardCoverage } from './shard-coverage.ts'
|
|
6
10
|
import { type PrFileEntry, parseBrief, parseFinding } from './types.ts'
|
|
7
11
|
|
|
8
12
|
export type RefreshResult = {
|
|
@@ -96,6 +100,10 @@ export async function refreshFindings(runDir: string): Promise<RefreshResult> {
|
|
|
96
100
|
// to a temp file first, so a failure leaves the previous findings.html
|
|
97
101
|
// in place instead of an empty screen dir.
|
|
98
102
|
const dismissed = await readDismissed(runDir)
|
|
103
|
+
const { previous } = await readHistory(runDir)
|
|
104
|
+
const previousRun = previous ? (describePreviousPosted(previous) ?? undefined) : undefined
|
|
105
|
+
const unreviewed = await readShardCoverage(runDir)
|
|
106
|
+
const disproved = await readProofDropped(runDir)
|
|
99
107
|
const screenDir = join(runDir, 'screen')
|
|
100
108
|
const tempPath = join(screenDir, '.findings.html.tmp')
|
|
101
109
|
await renderFindingsToDisk(
|
|
@@ -109,6 +117,9 @@ export async function refreshFindings(runDir: string): Promise<RefreshResult> {
|
|
|
109
117
|
brief: brief ?? undefined,
|
|
110
118
|
issues,
|
|
111
119
|
diffSource,
|
|
120
|
+
...(previousRun ? { previousRun } : {}),
|
|
121
|
+
...(unreviewed ? { unreviewed } : {}),
|
|
122
|
+
disproved,
|
|
112
123
|
dismissed,
|
|
113
124
|
topN: DEFAULT_TOP_N,
|
|
114
125
|
},
|
|
@@ -108,6 +108,21 @@ function renderSuggestion(f: ReviewFinding, highlighter: Highlighter): string {
|
|
|
108
108
|
return `<section class="section"><div class="section-label">Suggested change</div>${withClass}</section>`
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
function renderProofChip(f: ReviewFinding): string {
|
|
112
|
+
if (!f.proof) return ''
|
|
113
|
+
const label = f.proof.status === 'reproduced' ? 'REPRODUCED' : 'NOT PROVEN'
|
|
114
|
+
return `<span class="proof-chip ${f.proof.status}" title="${esc(f.proof.reason)}">${label}</span>`
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function renderProof(f: ReviewFinding): string {
|
|
118
|
+
if (f.proof?.status === 'not-proven') {
|
|
119
|
+
return `<section class="section"><div class="section-label">Not proven</div><div class="section-body">${renderInline(f.proof.reason)}</div></section>`
|
|
120
|
+
}
|
|
121
|
+
if (f.proof?.status !== 'reproduced') return ''
|
|
122
|
+
const { command, source, output } = f.proof.repro
|
|
123
|
+
return `<section class="section"><div class="section-label">Reproduced with <code class="inline-code">${esc(command)}</code></div><pre class="repro">${esc(source)}</pre><pre class="repro">${esc(output)}</pre></section>`
|
|
124
|
+
}
|
|
125
|
+
|
|
111
126
|
function renderRisk(f: ReviewFinding): string {
|
|
112
127
|
return `<div class="risk">Impact: <span class="tag">${esc(f.risk.impact)}</span> · Likelihood: <span class="tag">${esc(f.risk.likelihood)}</span> · Confidence: <span class="tag">${esc(f.risk.confidence)}</span> · Action: <span class="tag">${esc(f.risk.action)}</span></div>`
|
|
113
128
|
}
|
|
@@ -140,10 +155,12 @@ export function renderAnnotation(f: ReviewFinding, opts: RenderAnnotationOptions
|
|
|
140
155
|
<span class="sev-label sev-${f.severity}">${sevLabel}</span>
|
|
141
156
|
<span class="annot-title">${esc(f.title)}</span>
|
|
142
157
|
<span class="domain-chip">${esc(domainLabel)}</span>
|
|
158
|
+
${renderProofChip(f)}
|
|
143
159
|
${statusChip}
|
|
144
160
|
</div>
|
|
145
161
|
${renderSections(f.description)}
|
|
146
162
|
${renderSuggestion(f, opts.highlighter)}
|
|
163
|
+
${renderProof(f)}
|
|
147
164
|
${renderRisk(f)}
|
|
148
165
|
${opts.posted ? '' : renderDismiss(f.id, dismissed)}
|
|
149
166
|
</div>
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { readdir, readFile } from 'node:fs/promises'
|
|
2
2
|
import { basename, join } from 'node:path'
|
|
3
|
+
import { readHistory } from './history.ts'
|
|
3
4
|
import { readDismissed } from './labels.ts'
|
|
5
|
+
import { describePreviousPosted } from './previous-findings.ts'
|
|
6
|
+
import { readProofDropped } from './proof-cmd.ts'
|
|
4
7
|
import { type PostStatusMap, parseClosingIssues, renderFindingsToDisk } from './render-findings.ts'
|
|
5
8
|
import { DEFAULT_TOP_N } from './render-issues-list.ts'
|
|
6
9
|
import { renderProgressToDisk } from './render-progress.ts'
|
|
10
|
+
import { readShardCoverage } from './shard-coverage.ts'
|
|
7
11
|
import { parseBrief, parseFinding } from './types.ts'
|
|
8
12
|
|
|
9
13
|
async function nextVersionedPath(screenDir: string, base: string): Promise<string> {
|
|
@@ -47,6 +51,7 @@ function summarizeStages(log: Array<Record<string, unknown>>): {
|
|
|
47
51
|
dedupe: 'pending',
|
|
48
52
|
critic: 'pending',
|
|
49
53
|
'peer-review': 'pending',
|
|
54
|
+
proof: 'pending',
|
|
50
55
|
report: 'pending',
|
|
51
56
|
post: 'pending',
|
|
52
57
|
}
|
|
@@ -150,6 +155,10 @@ export async function runRender(
|
|
|
150
155
|
null,
|
|
151
156
|
)) ?? undefined
|
|
152
157
|
const dismissed = await readDismissed(runDir)
|
|
158
|
+
const { previous } = await readHistory(runDir)
|
|
159
|
+
const previousRun = previous ? (describePreviousPosted(previous) ?? undefined) : undefined
|
|
160
|
+
const unreviewed = await readShardCoverage(runDir)
|
|
161
|
+
const disproved = await readProofDropped(runDir)
|
|
153
162
|
const outPath = await nextVersionedPath(screenDir, 'findings')
|
|
154
163
|
await renderFindingsToDisk(
|
|
155
164
|
{
|
|
@@ -162,6 +171,9 @@ export async function runRender(
|
|
|
162
171
|
brief,
|
|
163
172
|
issues,
|
|
164
173
|
diffSource,
|
|
174
|
+
...(previousRun ? { previousRun } : {}),
|
|
175
|
+
...(unreviewed ? { unreviewed } : {}),
|
|
176
|
+
disproved,
|
|
165
177
|
dismissed,
|
|
166
178
|
topN: opts.topN ?? DEFAULT_TOP_N,
|
|
167
179
|
},
|
|
@@ -3,10 +3,12 @@ import { basename } from 'node:path'
|
|
|
3
3
|
import type { Highlighter } from 'shiki'
|
|
4
4
|
import { parseUnifiedDiffToHunks, splitDiffByFile } from './diff-utils.ts'
|
|
5
5
|
import { getHighlighter } from './highlight.ts'
|
|
6
|
+
import type { ProofDropped } from './proof.ts'
|
|
6
7
|
import { renderActionBar } from './render-action-bar.ts'
|
|
7
8
|
import { renderSplitDiff, renderUnifiedDiff } from './render-diff.ts'
|
|
8
9
|
import { renderFileTree } from './render-file-tree.ts'
|
|
9
10
|
import { renderIssuesList } from './render-issues-list.ts'
|
|
11
|
+
import { describeUnreviewed, type ShardCoverage } from './shard-coverage.ts'
|
|
10
12
|
import type { PostStatusMap, PrBrief, PrFileEntry, ReviewFinding } from './types.ts'
|
|
11
13
|
|
|
12
14
|
export type { PostStatusEntry, PostStatusMap } from './types.ts'
|
|
@@ -65,6 +67,12 @@ export type RenderFindingsInput = {
|
|
|
65
67
|
topN: number
|
|
66
68
|
/** Where diff.patch came from. Absent means gh, which needs no note. */
|
|
67
69
|
diffSource?: { source: 'gh' | 'git'; mergeBase: string | null }
|
|
70
|
+
/** `describePreviousPosted` on a re-review; absent on a first run. */
|
|
71
|
+
previousRun?: string
|
|
72
|
+
/** Shards skipped at the stage 4 gate; absent when every shard was reviewed. */
|
|
73
|
+
unreviewed?: ShardCoverage
|
|
74
|
+
/** Findings the proof stage removed because a passing test showed the code is right. */
|
|
75
|
+
disproved?: ProofDropped[]
|
|
68
76
|
}
|
|
69
77
|
|
|
70
78
|
function esc(s: string): string {
|
|
@@ -90,6 +98,24 @@ function brandBlock(): string {
|
|
|
90
98
|
return `<div class="brand">${BRAND_MARK_SVG}<span class="brand-text">magpie</span></div>`
|
|
91
99
|
}
|
|
92
100
|
|
|
101
|
+
// Closed by default: these are not findings any more, but a reader should be able
|
|
102
|
+
// to check the one test that removed each of them.
|
|
103
|
+
function disprovedBlock(disproved: ProofDropped[] | undefined): string {
|
|
104
|
+
if (!disproved || disproved.length === 0) return ''
|
|
105
|
+
const items = disproved
|
|
106
|
+
.map((d) => {
|
|
107
|
+
const anchor = d.line === null ? d.file : `${d.file}:${d.line}`
|
|
108
|
+
return `<li><code>${esc(anchor)}</code> ${esc(d.title)}: ${esc(d.reason)}
|
|
109
|
+
<div>Test: <code>${esc(d.repro.command)}</code></div>
|
|
110
|
+
<pre class="repro">${esc(d.repro.source)}</pre>
|
|
111
|
+
<pre class="repro">${esc(d.repro.output)}</pre></li>`
|
|
112
|
+
})
|
|
113
|
+
.join('\n')
|
|
114
|
+
return `<details class="pr-brief disproved"><summary class="brief-summary">Disproved by test (${disproved.length})</summary>
|
|
115
|
+
<ul class="brief-changes">${items}</ul>
|
|
116
|
+
</details>`
|
|
117
|
+
}
|
|
118
|
+
|
|
93
119
|
function prHeader(input: RenderFindingsInput): string {
|
|
94
120
|
const pr = input.pr
|
|
95
121
|
const src = input.diffSource
|
|
@@ -99,14 +125,22 @@ function prHeader(input: RenderFindingsInput): string {
|
|
|
99
125
|
src.mergeBase ? ` at <code>${esc(src.mergeBase.slice(0, 12))}</code>` : ''
|
|
100
126
|
}</span>`
|
|
101
127
|
: ''
|
|
128
|
+
const previousNote = input.previousRun
|
|
129
|
+
? `<span class="diff-source" title="Changed code is not the same as fixed: the defect can survive an edit">${esc(input.previousRun)}</span>`
|
|
130
|
+
: ''
|
|
131
|
+
const unreviewedNote = input.unreviewed
|
|
132
|
+
? `<span class="diff-source" title="${esc(input.unreviewed.skipped.flatMap((s) => s.files).join(', '))}">${esc(describeUnreviewed(input.unreviewed))}</span>`
|
|
133
|
+
: ''
|
|
102
134
|
const prMeta = pr
|
|
103
135
|
? `<div class="pr-meta">
|
|
104
136
|
<span class="pr-number">PR #${pr.number}</span>
|
|
105
137
|
<span class="pr-branch">${esc(pr.branch)}</span>
|
|
106
138
|
<code>${esc(pr.headSha.slice(0, 12))}</code>
|
|
107
139
|
${diffNote}
|
|
140
|
+
${previousNote}
|
|
141
|
+
${unreviewedNote}
|
|
108
142
|
</div>`
|
|
109
|
-
: `<div class="pr-meta">${diffNote}</div>`
|
|
143
|
+
: `<div class="pr-meta">${diffNote}${previousNote}${unreviewedNote}</div>`
|
|
110
144
|
const total = input.findings.length
|
|
111
145
|
return `<header class="pr-header">
|
|
112
146
|
${brandBlock()}
|
|
@@ -252,6 +286,7 @@ export function renderFindingsHtml(input: RenderFindingsInput): string {
|
|
|
252
286
|
<body data-run-id="${esc(runId)}" data-page="findings" data-view="files" data-diff-mode="unified" data-show-suggestions="false">
|
|
253
287
|
${prHeader(input)}
|
|
254
288
|
${briefHtml}
|
|
289
|
+
${disprovedBlock(input.disproved)}
|
|
255
290
|
<main class="page-main">
|
|
256
291
|
<div class="empty-state"><h1>No findings</h1><p>All specialists returned cleanly.</p></div>
|
|
257
292
|
</main>
|
|
@@ -310,6 +345,7 @@ ${briefHtml}
|
|
|
310
345
|
<body data-run-id="${esc(runId)}" data-page="findings" data-view="files" data-diff-mode="unified" data-show-suggestions="false">
|
|
311
346
|
${prHeader(input)}
|
|
312
347
|
${briefHtml}
|
|
348
|
+
${disprovedBlock(input.disproved)}
|
|
313
349
|
<main class="page-main">
|
|
314
350
|
<div class="view files-view">
|
|
315
351
|
${tree}
|
|
@@ -7,6 +7,7 @@ const STAGES = [
|
|
|
7
7
|
'dedupe',
|
|
8
8
|
'critic',
|
|
9
9
|
'peer-review',
|
|
10
|
+
'proof',
|
|
10
11
|
'report',
|
|
11
12
|
'post',
|
|
12
13
|
] as const
|
|
@@ -20,6 +21,7 @@ const STAGE_HINT: Record<StageId, string> = {
|
|
|
20
21
|
dedupe: 'merge overlaps',
|
|
21
22
|
critic: 'keep the high-signal ones',
|
|
22
23
|
'peer-review': 'independent second opinion',
|
|
24
|
+
proof: 'reproduce the serious ones',
|
|
23
25
|
report: 'render this page',
|
|
24
26
|
post: 'comment on the PR',
|
|
25
27
|
}
|
|
@@ -31,6 +33,7 @@ const STAGE_NOW_DOING: Record<StageId, string> = {
|
|
|
31
33
|
dedupe: 'Merging overlapping findings',
|
|
32
34
|
critic: 'Keeping only the high-signal ones',
|
|
33
35
|
'peer-review': 'Getting an independent second opinion',
|
|
36
|
+
proof: 'Writing failing tests for the serious findings',
|
|
34
37
|
report: 'Composing the report page',
|
|
35
38
|
post: 'Ready to post; switch tabs to pick findings',
|
|
36
39
|
}
|