@iceinvein/agent-skills 0.21.4 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +4 -1
- package/skills/bug-hunter/SKILL.md +82 -0
- package/skills/bug-hunter/evals/README.md +282 -0
- package/skills/bug-hunter/evals/baselines/claude-opus-5-5.json +93 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/case.yaml +4 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/fixture.sh +71 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/bug-cites-the-spec.md +36 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/intent-written.md +8 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/prompt.md +11 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/case.yaml +4 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/fixture.sh +67 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/source-unchanged.md +10 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/tests-unchanged.md +10 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/tree-unchanged.md +15 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/prompt.md +11 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/case.yaml +4 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/fixture.sh +82 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/bugs-empty.md +18 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/written-with-write-tool.md +13 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/prompt.md +11 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/case.yaml +4 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/fixture.sh +68 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/graders/bug-for-sixty-seconds.md +37 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/prompt.md +11 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/case.yaml +4 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/fixture.sh +79 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/bug-cites-the-limit.md +31 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/findings-written.md +10 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/prompt.md +11 -0
- package/skills/bug-hunter/evals/rules.json +177 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/case.yaml +4 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/fixture.sh +97 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/bugs-empty.md +18 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/cap-is-an-ambiguity.md +12 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/written-with-write-tool.md +13 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/prompt.md +11 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/case.yaml +4 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/fixture.sh +82 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/graders/bug-cites-the-spec.md +40 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/prompt.md +11 -0
- package/skills/bug-hunter/references/findings-format.md +82 -0
- package/skills/bug-hunter/references/scenario-catalog.md +92 -0
- package/skills/bug-hunter/skill.json +24 -0
- package/skills/index.json +17 -5
- package/skills/magpie/evals/README.md +13 -0
- package/skills/magpie/evals/baselines/claude-opus-5-5.json +69 -0
- package/skills/magpie/evals/rules.json +192 -0
- package/skills/magpie/skill.json +1 -1
- package/skills/migrate/evals/README.md +13 -0
- package/skills/migrate/evals/baselines/claude-opus-5-5.json +129 -0
- package/skills/migrate/evals/rules.json +160 -0
- package/skills/migrate/skill.json +10 -4
- package/skills/scope-estimate/README.md +173 -0
- package/skills/scope-estimate/SKILL.md +168 -0
- package/skills/scope-estimate/references/capability-map.md +122 -0
- package/skills/scope-estimate/references/deliverables.md +60 -0
- package/skills/scope-estimate/references/requirements.md +104 -0
- package/skills/scope-estimate/references/schedule.md +138 -0
- package/skills/scope-estimate/scripts/__tests__/schedule.test.ts +366 -0
- package/skills/scope-estimate/scripts/schedule.ts +553 -0
- package/skills/scope-estimate/skill.json +19 -0
- package/skills/scope-estimate/templates/plan.example.json +33 -0
- package/skills/sluice/SKILL.md +35 -12
- package/skills/sluice/evals/README.md +111 -6
- package/skills/sluice/evals/announcement-reaches-the-ledger/fixture.sh +32 -19
- package/skills/sluice/evals/announcement-reaches-the-ledger/graders/routes-main-before-source.md +14 -0
- package/skills/sluice/evals/announcement-reaches-the-ledger/prompt.md +2 -2
- package/skills/sluice/evals/baselines/claude-opus-5-5.json +237 -0
- package/skills/sluice/evals/deep-run-blocks-on-a-real-decision/fixture.sh +19 -12
- package/skills/sluice/evals/deep-run-fans-out/graders/task-1-merged-back.md +8 -0
- package/skills/sluice/evals/deep-run-fans-out/graders/task-2-merged-back.md +8 -0
- package/skills/sluice/evals/deep-run-fans-out/graders/task-3-merged-back.md +6 -0
- package/skills/sluice/evals/deep-run-fans-out/graders/worktree-per-implementer.md +12 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/case.yaml +5 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/fixture.sh +258 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/bug-hunter-fired.md +5 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/four-attempts-allowed.md +34 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/fourth-attempt-tested.md +20 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/hunt-before-the-final-review.md +21 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/record-has-a-hunt-section.md +11 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/prompt.md +12 -0
- package/skills/sluice/evals/deep-run-survives-a-milestone/fixture.sh +46 -24
- package/skills/sluice/evals/fast-verifies-before-handback/case.yaml +4 -0
- package/skills/sluice/evals/fast-verifies-before-handback/fixture.sh +73 -0
- package/skills/sluice/evals/fast-verifies-before-handback/graders/quiet-flag-implemented.md +5 -0
- package/skills/sluice/evals/fast-verifies-before-handback/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/fast-verifies-before-handback/graders/verified-after-the-last-edit.md +18 -0
- package/skills/sluice/evals/fast-verifies-before-handback/prompt.md +11 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/case.yaml +5 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/fixture.sh +64 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/announces-main-channel.md +12 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/bug-hunter-fired.md +5 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/hunt-after-the-source-edits.md +12 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/last-page-holds-the-remainder.md +27 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/remainder-page-tested.md +21 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/prompt.md +12 -0
- package/skills/sluice/evals/main-new-interface/graders/shape-agreed-before-building.md +8 -4
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/case.yaml +5 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/fixture.sh +73 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/announces-main-channel.md +12 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/no-hunt.md +12 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/skip-said-in-one-line.md +21 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/prompt.md +12 -0
- package/skills/sluice/evals/rules.json +364 -0
- package/skills/sluice/references/deep-channel.md +64 -18
- package/skills/sluice/references/finish.md +8 -2
- package/skills/sluice/references/intent.md +4 -2
- package/skills/sluice/references/review.md +32 -0
- package/skills/sluice/references/status.md +17 -4
- package/skills/sluice/references/verify.md +38 -0
- package/skills/sluice/scripts/content-digest.sh +65 -0
- package/skills/sluice/scripts/run-stats.sh +9 -8
- package/skills/sluice/scripts/status.sh +78 -13
- package/skills/sluice/scripts/stop-guard.sh +97 -24
- package/skills/sluice/skill.json +2 -2
- package/skills/terse/evals/README.md +16 -0
- package/skills/terse/evals/baselines/claude-opus-5-5.json +105 -0
- package/skills/terse/evals/rules.json +161 -0
- package/skills/terse/skill.json +1 -1
- package/skills/sluice/evals/announcement-reaches-the-ledger/graders/escalates-fast-to-main.md +0 -12
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "bug-hunter",
|
|
3
|
+
"version": "0.4.0",
|
|
4
|
+
"description": "Hunts logic and correctness bugs in a change or area and proves each one with a failing reproduction checked against stated intent. One agent collects intent from the request, specs, docs and unchanged code, runs QA-style scenarios from a catalog as repros in the repo's own test runner inside a scratch copy, and re-runs each failure clean before reporting it. Counts a contradicted claim, or a result wrong under every reading, as a bug; keeps ambiguities for questions with more than one acceptable answer; reports latent defects without filing them. No repro, no finding. Writes report.md and findings.json (bugs, ambiguities, covered) to .bug-hunter/<date>-<slug>/ and never modifies the caller's tree. Language-agnostic; ships no per-language scripts.",
|
|
5
|
+
"author": "iceinvein",
|
|
6
|
+
"type": "prompt",
|
|
7
|
+
"tools": [
|
|
8
|
+
"claude"
|
|
9
|
+
],
|
|
10
|
+
"files": {
|
|
11
|
+
"prompt": "SKILL.md"
|
|
12
|
+
},
|
|
13
|
+
"bundle": {
|
|
14
|
+
"include": [
|
|
15
|
+
"references"
|
|
16
|
+
]
|
|
17
|
+
},
|
|
18
|
+
"install": {
|
|
19
|
+
"claude": {
|
|
20
|
+
"prompt": ".claude/skills/bug-hunter/SKILL.md",
|
|
21
|
+
"bundleRoot": ".claude/skills/bug-hunter"
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
package/skills/index.json
CHANGED
|
@@ -9,6 +9,12 @@
|
|
|
9
9
|
],
|
|
10
10
|
"quick": false
|
|
11
11
|
},
|
|
12
|
+
{
|
|
13
|
+
"name": "bug-hunter",
|
|
14
|
+
"description": "Hunts logic and correctness bugs in a change or area and proves each one with a failing reproduction checked against stated intent. One agent collects intent from the request, specs, docs and unchanged code, runs QA-style scenarios from a catalog as repros in the repo's own test runner inside a scratch copy, and re-runs each failure clean before reporting it. Counts a contradicted claim, or a result wrong under every reading, as a bug; keeps ambiguities for questions with more than one acceptable answer; reports latent defects without filing them. No repro, no finding. Writes report.md and findings.json (bugs, ambiguities, covered) to .bug-hunter/<date>-<slug>/ and never modifies the caller's tree. Language-agnostic; ships no per-language scripts.",
|
|
15
|
+
"type": "prompt",
|
|
16
|
+
"version": "0.4.0"
|
|
17
|
+
},
|
|
12
18
|
{
|
|
13
19
|
"name": "codebase-architecture",
|
|
14
20
|
"description": "Architecture review for existing codebases or structured design for new projects, with patterns reference",
|
|
@@ -221,13 +227,13 @@
|
|
|
221
227
|
"name": "magpie",
|
|
222
228
|
"description": "Interactive PR review pipeline. Runs five parallel specialist subagents (security, bugs, performance, code-smells, architecture), drops findings whose quoted evidence is not in the code, dedupes, runs critic subagents that re-check each finding against the worktree and set its risk label, peer-reviews via codex exec (falling back to a Claude second opinion when codex is unavailable), and serves an interactive HTML report for selecting or dismissing findings to post via gh, recording the outcome per finding as labels. Bundles a Bun CLI installed onto PATH via the skill's postinstall step. Use when the user asks to review a GitHub pull request. Splits oversized diffs into budgeted shards and rebuilds the diff from the local clone when gh pr diff refuses it.",
|
|
223
229
|
"type": "prompt",
|
|
224
|
-
"version": "0.13.
|
|
230
|
+
"version": "0.13.1"
|
|
225
231
|
},
|
|
226
232
|
{
|
|
227
233
|
"name": "migrate",
|
|
228
234
|
"description": "Source-agnostic legacy migration mapping. Walks a legacy codebase through probe, enumerate, seam, extract, parity, queue, adjudicate and handoff, building an auditable requirements ledger with mandatory citations and a `migrate check` gate in place of self-reported completeness. Use when the user asks to migrate, re-specify, replatform, or map a legacy system onto a new stack, to hand mapped requirements to a delivery team, or to resume, check, report, or forecast a mapping run already under way.",
|
|
229
235
|
"type": "prompt",
|
|
230
|
-
"version": "0.4.
|
|
236
|
+
"version": "0.4.1"
|
|
231
237
|
},
|
|
232
238
|
{
|
|
233
239
|
"name": "module-secret-auditor",
|
|
@@ -259,6 +265,12 @@
|
|
|
259
265
|
],
|
|
260
266
|
"quick": true
|
|
261
267
|
},
|
|
268
|
+
{
|
|
269
|
+
"name": "scope-estimate",
|
|
270
|
+
"description": "Scopes and estimates a rebuild or respec of an existing system for a proposal: read-only discovery agents write cited inventories, a capability register gives every capability a status and a proposed disposition, area agents turn it into sized functional requirements, and a bundled Bun scheduler groups them into slices and contracts-first work packages and schedules them across engineers x worktrees at fast, planning and floor rates. The scheduler refuses a plan that misses or double-counts a requirement or has a dependency cycle, so every figure in the docs comes from one reproducible run. Also covers current and target architecture pages, proposal review, and a two-week calibration before committing to a date. Lighter than migrate: no store and no phase gates.",
|
|
271
|
+
"type": "prompt",
|
|
272
|
+
"version": "0.1.0"
|
|
273
|
+
},
|
|
262
274
|
{
|
|
263
275
|
"name": "seam-finder",
|
|
264
276
|
"description": "Feathers-inspired legacy code modification: find seams, make minimal incisions, preserve existing behavior",
|
|
@@ -281,9 +293,9 @@
|
|
|
281
293
|
},
|
|
282
294
|
{
|
|
283
295
|
"name": "sluice",
|
|
284
|
-
"description": "Routes work by change shape into four channels (bypass, fast, main, deep) and applies only the rules each channel needs, so a one-line fix does not pay the cost of a multi-subsystem build. Carries
|
|
296
|
+
"description": "Routes work by change shape into four channels (bypass, fast, main, deep) and applies only the rules each channel needs, so a one-line fix does not pay the cost of a multi-subsystem build. Carries nine rules as one-liners in the router and the full treatment in references read only on friction. Checks the finished plan with plan.sh validate rather than trusting it to memory, seeds the run state from it, keeps a deep run's task breakdown in .sluice/run.json so a statusline segment, one status command and a SessionStart hook can answer where the run is (the hook prints a live run at every session start, compaction included), and closes each run with a ledger read out of the session transcript: elapsed, tools, tokens, and what each dispatched agent cost where the transcript recorded it. Claude Code only; stands down where the superpowers pipeline governs the repo.",
|
|
285
297
|
"type": "prompt",
|
|
286
|
-
"version": "0.
|
|
298
|
+
"version": "0.24.1"
|
|
287
299
|
},
|
|
288
300
|
{
|
|
289
301
|
"name": "temporal-coupling-detector",
|
|
@@ -299,7 +311,7 @@
|
|
|
299
311
|
"name": "terse",
|
|
300
312
|
"description": "Professional output compression. Cuts ~20-30% of output tokens with proper grammar, a plain-language register, and semantic accuracy. Three levels: clean, tight, sharp.",
|
|
301
313
|
"type": "prompt",
|
|
302
|
-
"version": "1.3.
|
|
314
|
+
"version": "1.3.4"
|
|
303
315
|
},
|
|
304
316
|
{
|
|
305
317
|
"name": "type-driven-designer",
|
|
@@ -48,6 +48,19 @@ claude plugin eval skills/magpie --scaffold --allow-tools Bash Write Edit \
|
|
|
48
48
|
--trust-plugin --threshold 0.8
|
|
49
49
|
```
|
|
50
50
|
|
|
51
|
+
To check the skill against its last accepted baseline, use `eval:fresh`:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
bun run eval:fresh run skills/magpie --model claude-opus-5-5
|
|
55
|
+
bun run eval:fresh diff skills/magpie <result.json> [--against <model>]
|
|
56
|
+
bun run eval:fresh accept skills/magpie <result.json>
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
`rules.json` maps each rule the skill states to the cases that pin it, or says
|
|
60
|
+
why none does; `diff` reads it to name rules at risk or ready to expire. What
|
|
61
|
+
each verdict means is in `skills/sluice/evals/README.md`. No baseline has been
|
|
62
|
+
accepted yet.
|
|
63
|
+
|
|
51
64
|
## How the fixtures fake the pipeline
|
|
52
65
|
|
|
53
66
|
The eval child runs in a sandbox that refuses to execute anything outside it, so
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
{
|
|
2
|
+
"model": "claude-opus-5-5",
|
|
3
|
+
"judgeModel": "claude-opus-5-5",
|
|
4
|
+
"claudeVersion": "2.1.300",
|
|
5
|
+
"skillVersion": "0.13.0",
|
|
6
|
+
"date": "2026-10-15T14:11:31.564Z",
|
|
7
|
+
"cases": {
|
|
8
|
+
"codex-missing-falls-back": {
|
|
9
|
+
"with": [
|
|
10
|
+
1,
|
|
11
|
+
1,
|
|
12
|
+
1
|
|
13
|
+
],
|
|
14
|
+
"without": [
|
|
15
|
+
0.5,
|
|
16
|
+
0.5,
|
|
17
|
+
0.5
|
|
18
|
+
]
|
|
19
|
+
},
|
|
20
|
+
"post-folds-selection-events": {
|
|
21
|
+
"with": [
|
|
22
|
+
1,
|
|
23
|
+
1,
|
|
24
|
+
1
|
|
25
|
+
],
|
|
26
|
+
"without": [
|
|
27
|
+
0.7777777777777778,
|
|
28
|
+
0.7777777777777778,
|
|
29
|
+
0.7777777777777778
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
"report-ends-the-turn": {
|
|
33
|
+
"with": [
|
|
34
|
+
1,
|
|
35
|
+
1,
|
|
36
|
+
1
|
|
37
|
+
],
|
|
38
|
+
"without": [
|
|
39
|
+
0.4,
|
|
40
|
+
0.8,
|
|
41
|
+
0.4
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
"resume-finds-active-run": {
|
|
45
|
+
"with": [
|
|
46
|
+
1,
|
|
47
|
+
1,
|
|
48
|
+
1
|
|
49
|
+
],
|
|
50
|
+
"without": [
|
|
51
|
+
0.5,
|
|
52
|
+
0.5,
|
|
53
|
+
0.5
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"shard-gate-stops-and-asks": {
|
|
57
|
+
"with": [
|
|
58
|
+
1,
|
|
59
|
+
1,
|
|
60
|
+
1
|
|
61
|
+
],
|
|
62
|
+
"without": [
|
|
63
|
+
0.625,
|
|
64
|
+
0.625,
|
|
65
|
+
0.625
|
|
66
|
+
]
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
{
|
|
2
|
+
"rules": [
|
|
3
|
+
{
|
|
4
|
+
"id": "check-list-runs-before-minting-a-run",
|
|
5
|
+
"states": "SKILL.md: 0. Identify the PR",
|
|
6
|
+
"origin": "commit:63d0fb1",
|
|
7
|
+
"cases": [
|
|
8
|
+
"resume-finds-active-run"
|
|
9
|
+
]
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"id": "setup-failure-surfaces-stderr-and-stops",
|
|
13
|
+
"states": "SKILL.md: 1. Setup",
|
|
14
|
+
"origin": "commit:5040cc1",
|
|
15
|
+
"cases": [],
|
|
16
|
+
"unpinned": "Every case starts past setup, and the one that could reach it is graded on never calling setup at all."
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"id": "missing-brief-logs-context-skipped",
|
|
20
|
+
"states": "SKILL.md: 3. Context",
|
|
21
|
+
"origin": "commit:dbe295e",
|
|
22
|
+
"cases": [],
|
|
23
|
+
"unpinned": "No case starts at the context stage, so no scout runs and none returns without a brief."
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"id": "specialist-prompts-from-the-reference-verbatim",
|
|
27
|
+
"states": "SKILL.md: 4. Specialists",
|
|
28
|
+
"origin": "commit:e49534b",
|
|
29
|
+
"cases": [],
|
|
30
|
+
"unpinned": "The only case at stage 4 stops before dispatch, so no specialist prompt is ever written for a grader to read."
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "shard-gate-asks-above-four-shards",
|
|
34
|
+
"states": "SKILL.md: 4. Specialists",
|
|
35
|
+
"origin": "commit:a744cc2",
|
|
36
|
+
"cases": [
|
|
37
|
+
"shard-gate-stops-and-asks"
|
|
38
|
+
]
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "sharded-dispatch-one-shard-per-wave",
|
|
42
|
+
"states": "SKILL.md: 4. Specialists",
|
|
43
|
+
"origin": "commit:a744cc2",
|
|
44
|
+
"cases": [],
|
|
45
|
+
"unpinned": "The sharded case stops at the gate, and no case proceeds past it to dispatch waves."
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"id": "every-focus-shard-pair-confirmed-before-dedupe",
|
|
49
|
+
"states": "SKILL.md: 4. Specialists",
|
|
50
|
+
"origin": "commit:e49534b",
|
|
51
|
+
"cases": [],
|
|
52
|
+
"unpinned": "No case reaches the end of stage 4 with a findings file missing for a re-dispatch to fill."
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "critic-subagents-run-the-prompt-files-verbatim",
|
|
56
|
+
"states": "SKILL.md: 6. Critic",
|
|
57
|
+
"origin": "commit:773467d",
|
|
58
|
+
"cases": [],
|
|
59
|
+
"unpinned": "Every case past stage 5 starts after the critic, and none runs critic-prompt."
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"id": "critic-apply-failure-redispatches-named-batches-once",
|
|
63
|
+
"states": "SKILL.md: 6. Critic",
|
|
64
|
+
"origin": "commit:ec1ca71",
|
|
65
|
+
"cases": [],
|
|
66
|
+
"unpinned": "No case runs critic-apply, so no non-zero exit is ever seen."
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
"id": "peer-review-always-runs-on-a-fallback",
|
|
70
|
+
"states": "SKILL.md: 7. Peer review",
|
|
71
|
+
"origin": "commit:f55276f",
|
|
72
|
+
"cases": [
|
|
73
|
+
"codex-missing-falls-back"
|
|
74
|
+
]
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"id": "claude-fallback-carries-the-independence-preamble",
|
|
78
|
+
"states": "SKILL.md: 7. Peer review",
|
|
79
|
+
"origin": "commit:f55276f",
|
|
80
|
+
"cases": [
|
|
81
|
+
"codex-missing-falls-back"
|
|
82
|
+
]
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "codex-failure-never-logged-as-error",
|
|
86
|
+
"states": "SKILL.md: 7. Peer review",
|
|
87
|
+
"origin": "commit:1762265",
|
|
88
|
+
"cases": [],
|
|
89
|
+
"unpinned": "`codex-missing-falls-back` removes codex, so codex never exits non-zero; its no-error grader reads the missing-binary path, not this one."
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"id": "peer-batches-keep-one-id-counter-and-merge-once",
|
|
93
|
+
"states": "SKILL.md: 7. Peer review",
|
|
94
|
+
"origin": "commit:a744cc2",
|
|
95
|
+
"cases": [],
|
|
96
|
+
"unpinned": "The fallback case carries three findings, one batch, so neither the id counter across batches nor the single peer.json write is exercised."
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
"id": "report-stage-logged-done",
|
|
100
|
+
"states": "SKILL.md: 8. Report",
|
|
101
|
+
"origin": "commit:1762265",
|
|
102
|
+
"cases": [
|
|
103
|
+
"report-ends-the-turn"
|
|
104
|
+
]
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"id": "report-ends-the-turn-without-posting",
|
|
108
|
+
"states": "SKILL.md: 8. Report",
|
|
109
|
+
"origin": "commit:5040cc1",
|
|
110
|
+
"cases": [
|
|
111
|
+
"report-ends-the-turn"
|
|
112
|
+
]
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"id": "post-folds-events-last-event-wins",
|
|
116
|
+
"states": "SKILL.md: 9. Post",
|
|
117
|
+
"origin": "commit:1762265",
|
|
118
|
+
"cases": [
|
|
119
|
+
"post-folds-selection-events"
|
|
120
|
+
]
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"id": "empty-selection-asks-instead-of-posting",
|
|
124
|
+
"states": "SKILL.md: 9. Post",
|
|
125
|
+
"origin": "commit:63d0fb1",
|
|
126
|
+
"cases": [],
|
|
127
|
+
"unpinned": "The post case's events always leave two findings selected."
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"id": "post-stage-logged-done",
|
|
131
|
+
"states": "SKILL.md: 9. Post",
|
|
132
|
+
"origin": "commit:1762265",
|
|
133
|
+
"cases": [
|
|
134
|
+
"post-folds-selection-events"
|
|
135
|
+
]
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
"id": "report-re-rendered-after-posting",
|
|
139
|
+
"states": "SKILL.md: 9. Post",
|
|
140
|
+
"origin": "commit:5040cc1",
|
|
141
|
+
"cases": [
|
|
142
|
+
"post-folds-selection-events"
|
|
143
|
+
]
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"id": "cleanup-surfaces-the-view-later-line",
|
|
147
|
+
"states": "SKILL.md: 10. Cleanup",
|
|
148
|
+
"origin": "commit:e49534b",
|
|
149
|
+
"cases": [],
|
|
150
|
+
"unpinned": "No case runs cleanup, and the fakes have no cleanup subcommand."
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"id": "resumable-means-log-not-server-info",
|
|
154
|
+
"states": "SKILL.md: Resuming a crashed run",
|
|
155
|
+
"origin": "commit:63d0fb1",
|
|
156
|
+
"cases": [
|
|
157
|
+
"resume-finds-active-run"
|
|
158
|
+
]
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
"id": "resume-confirms-before-rerunning-a-failed-stage",
|
|
162
|
+
"states": "SKILL.md: Resuming a crashed run",
|
|
163
|
+
"origin": "commit:63d0fb1",
|
|
164
|
+
"cases": [],
|
|
165
|
+
"unpinned": "No fixture's log carries an error entry."
|
|
166
|
+
},
|
|
167
|
+
{
|
|
168
|
+
"id": "resume-redispatches-only-missing-pairs",
|
|
169
|
+
"states": "SKILL.md: Resuming a crashed run",
|
|
170
|
+
"origin": "commit:a744cc2",
|
|
171
|
+
"cases": [],
|
|
172
|
+
"unpinned": "The stage-4 resume has no findings files at all, so there is no partial set to re-dispatch around."
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
"id": "resume-restarts-the-server",
|
|
176
|
+
"states": "SKILL.md: Resuming a crashed run",
|
|
177
|
+
"origin": "commit:e49534b",
|
|
178
|
+
"cases": [],
|
|
179
|
+
"unpinned": "No grader checks for a serve call, and the shard-gate rubric is told to ignore one."
|
|
180
|
+
},
|
|
181
|
+
{
|
|
182
|
+
"id": "abort-runs-cleanup",
|
|
183
|
+
"states": "SKILL.md: Aborting",
|
|
184
|
+
"origin": "commit:5040cc1",
|
|
185
|
+
"cases": [],
|
|
186
|
+
"unpinned": "No case has the user type abort."
|
|
187
|
+
}
|
|
188
|
+
],
|
|
189
|
+
"sandboxSensitive": [],
|
|
190
|
+
"vacuousWithout": [],
|
|
191
|
+
"unclaimed": []
|
|
192
|
+
}
|
package/skills/magpie/skill.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "magpie",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.1",
|
|
4
4
|
"description": "Interactive PR review pipeline. Runs five parallel specialist subagents (security, bugs, performance, code-smells, architecture), drops findings whose quoted evidence is not in the code, dedupes, runs critic subagents that re-check each finding against the worktree and set its risk label, peer-reviews via codex exec (falling back to a Claude second opinion when codex is unavailable), and serves an interactive HTML report for selecting or dismissing findings to post via gh, recording the outcome per finding as labels. Bundles a Bun CLI installed onto PATH via the skill's postinstall step. Use when the user asks to review a GitHub pull request. Splits oversized diffs into budgeted shards and rebuilds the diff from the local clone when gh pr diff refuses it.",
|
|
5
5
|
"author": "iceinvein",
|
|
6
6
|
"type": "prompt",
|
|
@@ -41,6 +41,19 @@ From a session isolated in a worktree, the worktree guard refuses
|
|
|
41
41
|
`claude plugin eval` because it reads the word as a shell `eval`. Run the suite
|
|
42
42
|
from a session that is not isolated.
|
|
43
43
|
|
|
44
|
+
To check the skill against its last accepted baseline, use `eval:fresh`:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
bun run eval:fresh run skills/migrate --model claude-opus-5-5
|
|
48
|
+
bun run eval:fresh diff skills/migrate <result.json> [--against <model>]
|
|
49
|
+
bun run eval:fresh accept skills/migrate <result.json>
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
`rules.json` maps each rule the skill states to the cases that pin it, or says
|
|
53
|
+
why none does; `diff` reads it to name rules at risk or ready to expire. What
|
|
54
|
+
each verdict means is in `skills/sluice/evals/README.md`. No baseline has been
|
|
55
|
+
accepted yet.
|
|
56
|
+
|
|
44
57
|
## How the fixtures build a store
|
|
45
58
|
|
|
46
59
|
The cases run the real CLI, not a fake. magpie's evals fake their binaries
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
{
|
|
2
|
+
"model": "claude-opus-5-5",
|
|
3
|
+
"judgeModel": "claude-opus-5-5",
|
|
4
|
+
"claudeVersion": "2.1.300",
|
|
5
|
+
"skillVersion": "0.4.0",
|
|
6
|
+
"date": "2026-10-15T15:17:22.077Z",
|
|
7
|
+
"cases": {
|
|
8
|
+
"adjudicate-drafts-not-decides": {
|
|
9
|
+
"with": [
|
|
10
|
+
1,
|
|
11
|
+
1,
|
|
12
|
+
1
|
|
13
|
+
],
|
|
14
|
+
"without": [
|
|
15
|
+
0.5833333333333334,
|
|
16
|
+
0.5833333333333334,
|
|
17
|
+
0.5833333333333334
|
|
18
|
+
]
|
|
19
|
+
},
|
|
20
|
+
"done-means-plain-check": {
|
|
21
|
+
"with": [
|
|
22
|
+
1,
|
|
23
|
+
1,
|
|
24
|
+
1
|
|
25
|
+
],
|
|
26
|
+
"without": [
|
|
27
|
+
1,
|
|
28
|
+
1,
|
|
29
|
+
0.5714285714285714
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
"extract-files-queue-in-same-pass": {
|
|
33
|
+
"with": [
|
|
34
|
+
1,
|
|
35
|
+
1,
|
|
36
|
+
0.8
|
|
37
|
+
],
|
|
38
|
+
"without": [
|
|
39
|
+
1,
|
|
40
|
+
1,
|
|
41
|
+
1
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
"forecast-refuses-without-attestation": {
|
|
45
|
+
"with": [
|
|
46
|
+
1,
|
|
47
|
+
0.5714285714285714,
|
|
48
|
+
1
|
|
49
|
+
],
|
|
50
|
+
"without": [
|
|
51
|
+
1,
|
|
52
|
+
1,
|
|
53
|
+
0.5714285714285714
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"handoff-blocked-reports-blockers": {
|
|
57
|
+
"with": [
|
|
58
|
+
1,
|
|
59
|
+
1,
|
|
60
|
+
1
|
|
61
|
+
],
|
|
62
|
+
"without": [
|
|
63
|
+
0.8181818181818182,
|
|
64
|
+
1,
|
|
65
|
+
1
|
|
66
|
+
]
|
|
67
|
+
},
|
|
68
|
+
"probe-asks-for-the-target": {
|
|
69
|
+
"with": [
|
|
70
|
+
1,
|
|
71
|
+
1,
|
|
72
|
+
1
|
|
73
|
+
],
|
|
74
|
+
"without": [
|
|
75
|
+
0.375,
|
|
76
|
+
0.375,
|
|
77
|
+
0.375
|
|
78
|
+
]
|
|
79
|
+
},
|
|
80
|
+
"resume-reads-status-first": {
|
|
81
|
+
"with": [
|
|
82
|
+
1,
|
|
83
|
+
1,
|
|
84
|
+
1
|
|
85
|
+
],
|
|
86
|
+
"without": [
|
|
87
|
+
1,
|
|
88
|
+
1,
|
|
89
|
+
1
|
|
90
|
+
]
|
|
91
|
+
},
|
|
92
|
+
"seam-low-q-escalates": {
|
|
93
|
+
"with": [
|
|
94
|
+
1,
|
|
95
|
+
1,
|
|
96
|
+
1
|
|
97
|
+
],
|
|
98
|
+
"without": [
|
|
99
|
+
0.5384615384615384,
|
|
100
|
+
0.5384615384615384,
|
|
101
|
+
0.5384615384615384
|
|
102
|
+
]
|
|
103
|
+
},
|
|
104
|
+
"seam-shared-element-queued": {
|
|
105
|
+
"with": [
|
|
106
|
+
1,
|
|
107
|
+
1,
|
|
108
|
+
1
|
|
109
|
+
],
|
|
110
|
+
"without": [
|
|
111
|
+
0.5714285714285714,
|
|
112
|
+
0.5714285714285714,
|
|
113
|
+
0.5714285714285714
|
|
114
|
+
]
|
|
115
|
+
},
|
|
116
|
+
"stale-lock-not-forced": {
|
|
117
|
+
"with": [
|
|
118
|
+
1,
|
|
119
|
+
1,
|
|
120
|
+
1
|
|
121
|
+
],
|
|
122
|
+
"without": [
|
|
123
|
+
1,
|
|
124
|
+
1,
|
|
125
|
+
1
|
|
126
|
+
]
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|