@iceinvein/agent-skills 0.21.4 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +4 -1
- package/skills/bug-hunter/SKILL.md +82 -0
- package/skills/bug-hunter/evals/README.md +282 -0
- package/skills/bug-hunter/evals/baselines/claude-opus-5-5.json +93 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/case.yaml +4 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/fixture.sh +71 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/bug-cites-the-spec.md +36 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/intent-written.md +8 -0
- package/skills/bug-hunter/evals/bug-cites-intent-from-spec/prompt.md +11 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/case.yaml +4 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/fixture.sh +67 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/source-unchanged.md +10 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/tests-unchanged.md +10 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/graders/tree-unchanged.md +15 -0
- package/skills/bug-hunter/evals/callers-tree-unchanged/prompt.md +11 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/case.yaml +4 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/fixture.sh +82 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/bugs-empty.md +18 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/written-with-write-tool.md +13 -0
- package/skills/bug-hunter/evals/correct-code-has-no-bugs/prompt.md +11 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/case.yaml +4 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/fixture.sh +68 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/graders/bug-for-sixty-seconds.md +37 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/impossible-value-is-a-bug/prompt.md +11 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/case.yaml +4 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/fixture.sh +79 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/bug-cites-the-limit.md +31 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/findings-written.md +10 -0
- package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/prompt.md +11 -0
- package/skills/bug-hunter/evals/rules.json +177 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/case.yaml +4 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/fixture.sh +97 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/bugs-empty.md +18 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/cap-is-an-ambiguity.md +12 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/written-with-write-tool.md +13 -0
- package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/prompt.md +11 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/case.yaml +4 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/fixture.sh +82 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/graders/bug-cites-the-spec.md +40 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/graders/bug-hunter-fired.md +5 -0
- package/skills/bug-hunter/evals/spec-outranks-target-docstring/prompt.md +11 -0
- package/skills/bug-hunter/references/findings-format.md +82 -0
- package/skills/bug-hunter/references/scenario-catalog.md +92 -0
- package/skills/bug-hunter/skill.json +24 -0
- package/skills/index.json +17 -5
- package/skills/magpie/evals/README.md +13 -0
- package/skills/magpie/evals/baselines/claude-opus-5-5.json +69 -0
- package/skills/magpie/evals/rules.json +192 -0
- package/skills/magpie/skill.json +1 -1
- package/skills/migrate/evals/README.md +13 -0
- package/skills/migrate/evals/baselines/claude-opus-5-5.json +129 -0
- package/skills/migrate/evals/rules.json +160 -0
- package/skills/migrate/skill.json +10 -4
- package/skills/scope-estimate/README.md +173 -0
- package/skills/scope-estimate/SKILL.md +168 -0
- package/skills/scope-estimate/references/capability-map.md +122 -0
- package/skills/scope-estimate/references/deliverables.md +60 -0
- package/skills/scope-estimate/references/requirements.md +104 -0
- package/skills/scope-estimate/references/schedule.md +138 -0
- package/skills/scope-estimate/scripts/__tests__/schedule.test.ts +366 -0
- package/skills/scope-estimate/scripts/schedule.ts +553 -0
- package/skills/scope-estimate/skill.json +19 -0
- package/skills/scope-estimate/templates/plan.example.json +33 -0
- package/skills/sluice/SKILL.md +35 -12
- package/skills/sluice/evals/README.md +111 -6
- package/skills/sluice/evals/announcement-reaches-the-ledger/fixture.sh +32 -19
- package/skills/sluice/evals/announcement-reaches-the-ledger/graders/routes-main-before-source.md +14 -0
- package/skills/sluice/evals/announcement-reaches-the-ledger/prompt.md +2 -2
- package/skills/sluice/evals/baselines/claude-opus-5-5.json +237 -0
- package/skills/sluice/evals/deep-run-blocks-on-a-real-decision/fixture.sh +19 -12
- package/skills/sluice/evals/deep-run-fans-out/graders/task-1-merged-back.md +8 -0
- package/skills/sluice/evals/deep-run-fans-out/graders/task-2-merged-back.md +8 -0
- package/skills/sluice/evals/deep-run-fans-out/graders/task-3-merged-back.md +6 -0
- package/skills/sluice/evals/deep-run-fans-out/graders/worktree-per-implementer.md +12 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/case.yaml +5 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/fixture.sh +258 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/bug-hunter-fired.md +5 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/four-attempts-allowed.md +34 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/fourth-attempt-tested.md +20 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/hunt-before-the-final-review.md +21 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/record-has-a-hunt-section.md +11 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/deep-run-hunts-after-the-flip/prompt.md +12 -0
- package/skills/sluice/evals/deep-run-survives-a-milestone/fixture.sh +46 -24
- package/skills/sluice/evals/fast-verifies-before-handback/case.yaml +4 -0
- package/skills/sluice/evals/fast-verifies-before-handback/fixture.sh +73 -0
- package/skills/sluice/evals/fast-verifies-before-handback/graders/quiet-flag-implemented.md +5 -0
- package/skills/sluice/evals/fast-verifies-before-handback/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/fast-verifies-before-handback/graders/verified-after-the-last-edit.md +18 -0
- package/skills/sluice/evals/fast-verifies-before-handback/prompt.md +11 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/case.yaml +5 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/fixture.sh +64 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/announces-main-channel.md +12 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/bug-hunter-fired.md +5 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/hunt-after-the-source-edits.md +12 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/last-page-holds-the-remainder.md +27 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/remainder-page-tested.md +21 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/main-hunts-a-checkable-claim/prompt.md +12 -0
- package/skills/sluice/evals/main-new-interface/graders/shape-agreed-before-building.md +8 -4
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/case.yaml +5 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/fixture.sh +73 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/announces-main-channel.md +12 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/no-hunt.md +12 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/skip-said-in-one-line.md +21 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/sluice-fired.md +5 -0
- package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/prompt.md +12 -0
- package/skills/sluice/evals/rules.json +364 -0
- package/skills/sluice/references/deep-channel.md +64 -18
- package/skills/sluice/references/finish.md +8 -2
- package/skills/sluice/references/intent.md +4 -2
- package/skills/sluice/references/review.md +32 -0
- package/skills/sluice/references/status.md +17 -4
- package/skills/sluice/references/verify.md +38 -0
- package/skills/sluice/scripts/content-digest.sh +65 -0
- package/skills/sluice/scripts/run-stats.sh +9 -8
- package/skills/sluice/scripts/status.sh +78 -13
- package/skills/sluice/scripts/stop-guard.sh +97 -24
- package/skills/sluice/skill.json +2 -2
- package/skills/terse/evals/README.md +16 -0
- package/skills/terse/evals/baselines/claude-opus-5-5.json +105 -0
- package/skills/terse/evals/rules.json +161 -0
- package/skills/terse/skill.json +1 -1
- package/skills/sluice/evals/announcement-reaches-the-ledger/graders/escalates-fast-to-main.md +0 -12
package/skills/sluice/SKILL.md
CHANGED
|
@@ -21,16 +21,21 @@ has to be taken back.
|
|
|
21
21
|
|---------|--------|-------|-----|
|
|
22
22
|
| `bypass` | No code change: a question, an explanation, a read | none | nothing, just answer |
|
|
23
23
|
| `fast` | Existing interfaces, one subsystem (a new flag on an existing command still counts as existing) | test-first, verify | "Fast channel, existing interfaces. Test first, then implement." |
|
|
24
|
-
| `main` | Adds an interface, or crosses subsystems | + agree intent, review before merge | "Main channel, new interface. Agreeing the shape first." |
|
|
25
|
-
| `deep` | Several subsystems, or a plan was asked for | + written design and plan | "Deep channel, several subsystems. Design before code." |
|
|
24
|
+
| `main` | Adds an interface, or crosses subsystems | + agree intent, hunt a checkable claim, review before merge | "Main channel, new interface. Agreeing the shape first." |
|
|
25
|
+
| `deep` | Several subsystems, or a plan was asked for | + written design and plan, hunt after the flip | "Deep channel, several subsystems. Design before code." |
|
|
26
26
|
|
|
27
27
|
Two subsystems is `main`; the third is what makes it `deep`.
|
|
28
28
|
|
|
29
29
|
`bypass`, `fast`, and `main` proceed without stopping for approval; only
|
|
30
30
|
`deep` stops: once for design sign-off before code, once for the plan and
|
|
31
|
-
pre-flight together before Task 1. A stop ends your turn.
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
pre-flight together before Task 1. A stop ends your turn.
|
|
32
|
+
|
|
33
|
+
A choice that is your partner's to make goes to them through `AskUserQuestion`
|
|
34
|
+
wherever the harness has it, at a stop or between them: options to pick from,
|
|
35
|
+
your recommendation first, not a paragraph or a lettered list to reply to.
|
|
36
|
+
Asking is not itself a stop, however many options it carried, because the
|
|
37
|
+
answer comes back to you and the run never left your hands. So at a stop, ask
|
|
38
|
+
with the tool, then end the turn once the answers are in.
|
|
34
39
|
|
|
35
40
|
Name the channel and the signal that actually routed you there. The strings above
|
|
36
41
|
are examples, not fixed copy, and a channel with a two-part signal should say
|
|
@@ -66,9 +71,9 @@ announces nothing to measure from.
|
|
|
66
71
|
One line each. Read the reference only on friction: the moment you notice
|
|
67
72
|
yourself wanting to skip the rule, or arguing that this one is the exception.
|
|
68
73
|
|
|
69
|
-
- **Agree intent** before building. One question at a time
|
|
70
|
-
with a recommendation, not a survey.
|
|
71
|
-
it down. `references/intent.md`
|
|
74
|
+
- **Agree intent** before building. One question at a time, through
|
|
75
|
+
`AskUserQuestion`. Propose approaches with a recommendation, not a survey.
|
|
76
|
+
`main` agrees in a message, `deep` writes it down. `references/intent.md`
|
|
72
77
|
- **Test first.** The test comes before the code; run it while it should
|
|
73
78
|
still be failing, then write the least code that turns it green. Skip that
|
|
74
79
|
watching step and a green result is only an unchecked guess. A test no
|
|
@@ -78,7 +83,21 @@ yourself wanting to skip the rule, or arguing that this one is the exception.
|
|
|
78
83
|
`references/root-cause.md`
|
|
79
84
|
- **Verify** before claiming. Run the command in this turn and read its
|
|
80
85
|
output; the claim comes after that, never before. How sure you feel is not
|
|
81
|
-
something anyone else can check.
|
|
86
|
+
something anyone else can check, so record the check: `status.sh verify --
|
|
87
|
+
<cmd>` notes the tree it passed on, or `--none "<reason>"` where no check
|
|
88
|
+
can cover the change. In `fast` and `main` a Stop hook refuses, once per
|
|
89
|
+
turn, a handback whose tree has moved since. `references/verify.md`
|
|
90
|
+
- **Hunt** in `main`, once green and before review: when the intent agreed in
|
|
91
|
+
the message states a checkable claim (a limit, a boundary, an invariant, a
|
|
92
|
+
stated result), run bug-hunter on the diff; otherwise say in one line there
|
|
93
|
+
was none and skip it. Fix each proven bug test-first; ambiguities go to your
|
|
94
|
+
partner at the handback. Record the check last, `status.sh verify --` after
|
|
95
|
+
the hunt and any fix, since the hunt's report lands in the tree. No
|
|
96
|
+
bug-hunter? Say so in the announcement and again at the merge.
|
|
97
|
+
`references/review.md`
|
|
98
|
+
- **Hunt** in `deep`, after the flip, once every task is done and before the
|
|
99
|
+
final review: bug-hunter on `<base>..HEAD` against the spec, each proven bug
|
|
100
|
+
one fix, test-first. `references/deep-channel.md`
|
|
82
101
|
- **Review** before merge. Dispatch a reviewer with fresh context and put the
|
|
83
102
|
diff on disk for it to read, so those bytes fill their context instead of
|
|
84
103
|
yours. A session that dispatches only when asked has not refused you one, so
|
|
@@ -104,7 +123,10 @@ collapses to `fast`.
|
|
|
104
123
|
|
|
105
124
|
Design to `docs/specs/YYYY-MM-DD-<topic>.md`, plan to
|
|
106
125
|
`docs/plans/YYYY-MM-DD-<topic>.md`, unless the repo has a convention or
|
|
107
|
-
your partner states a preference. Get the design signed off before code.
|
|
126
|
+
your partner states a preference. Get the design signed off before code. When
|
|
127
|
+
the design's shape turns on a fork only your partner can settle, ask that one
|
|
128
|
+
question through `AskUserQuestion`, with your recommendation, before the design
|
|
129
|
+
is written. `references/deep-channel.md` has the test for what counts.
|
|
108
130
|
|
|
109
131
|
Take the design stop through the harness's plan mode where there is one. Its
|
|
110
132
|
gate is enforced rather than requested and it holds edits shut while it is open,
|
|
@@ -162,8 +184,9 @@ instead of to the branch at large.
|
|
|
162
184
|
|
|
163
185
|
Pre-flight rides in that same stop: which flagged tasks get a reviewer, which
|
|
164
186
|
mechanical ones the plan marked for low effort, and whether the work runs
|
|
165
|
-
in a worktree. Ask them
|
|
166
|
-
|
|
187
|
+
in a worktree. Ask them through `AskUserQuestion`, one question each with the
|
|
188
|
+
counts in the options, never as a paragraph or a lettered list, then end the
|
|
189
|
+
turn on the answers. Task 1 opens on their next
|
|
167
190
|
instruction, and an answer that already carries one is that instruction. A
|
|
168
191
|
session that dispatches only when asked has not ruled dispatch out, it has made
|
|
169
192
|
this question the place to ask; genuine unavailability is the tool not being
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# sluice evals
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Nineteen cases for `claude plugin eval`. Twelve pin the routing decision: which
|
|
4
4
|
channel the run names, and whether the behaviour that channel owes actually
|
|
5
|
-
happened.
|
|
5
|
+
happened. Seven pin what a deep run does after pre-flight, where the question is
|
|
6
6
|
no longer which channel but whether the run keeps going.
|
|
7
7
|
|
|
8
8
|
| Case | Signal under test | What it pins |
|
|
@@ -14,8 +14,11 @@ no longer which channel but whether the run keeps going.
|
|
|
14
14
|
| `deep-plan-asks-the-fork` | The same plan with nothing saying where the counters can live | One question with a recommendation before any design; no spec yet |
|
|
15
15
|
| `explicit-instruction-collapses-to-fast` | Main-shaped work plus "just do it" | Collapses to fast; no design, no proposal |
|
|
16
16
|
| `superpowers-conflict-stands-down` | Repo mandates the superpowers sequence | Stands down once, names no channel |
|
|
17
|
-
| `announcement-reaches-the-ledger` | A flag that a
|
|
17
|
+
| `announcement-reaches-the-ledger` | A flag that a surface test turns into a new export | `status.sh route main` before any write under `src/` and before any fast or deep route |
|
|
18
18
|
| `fast-reads-the-unmentioned-convention` | Flags live in a table the prompt never names | The flag lands in the registry table and the generated `--help` |
|
|
19
|
+
| `fast-verifies-before-handback` | A fast flag, then the handback | A passing `status.sh verify` after the last edit, before handing back |
|
|
20
|
+
| `main-hunts-a-checkable-claim` | Main work whose intent states a checkable claim | bug-hunter on the diff once green; the `lastPage` remainder bug it proves, fixed test-first |
|
|
21
|
+
| `main-skips-the-hunt-without-a-claim` | A new interface with nothing to check | No bug-hunter; one line saying the intent stated no checkable claim |
|
|
19
22
|
|
|
20
23
|
Execution, where the run is already past both stops:
|
|
21
24
|
|
|
@@ -26,11 +29,12 @@ Execution, where the run is already past both stops:
|
|
|
26
29
|
| `deep-run-survives-a-milestone` | Six tasks, the first three named a milestone | All six done; the Stop hook never had to refuse; the final message is a handback |
|
|
27
30
|
| `deep-run-decides-a-non-blocking-choice` | A task leaves a choice open and says either is fine | Decided, written into the record, no task blocked, no decision list at the end |
|
|
28
31
|
| `deep-run-waits-for-a-running-agent` | Every task goes to an implementer agent | A `T2:` labelled dispatch; the handback comes after the last result |
|
|
29
|
-
| `deep-run-fans-out` | Three disjoint tasks under a worktree-per-implementer answer | Three dispatches in one message, then the flip; also the baseline for a time-budget A/B |
|
|
32
|
+
| `deep-run-fans-out` | Three disjoint tasks under a worktree-per-implementer answer | Three dispatches in one message, then the flip; each task's files merged back into the run's own tree; also the baseline for a time-budget A/B |
|
|
33
|
+
| `deep-run-hunts-after-the-flip` | Every task done, the flip included | bug-hunter on the plan diff against the spec before the final review; the proven off-by-one fixed test-first; a Hunt section in the record |
|
|
30
34
|
|
|
31
35
|
## Running
|
|
32
36
|
|
|
33
|
-
|
|
37
|
+
Eighteen cases scaffold a small Node repo and then change it, so they need the
|
|
34
38
|
scaffold flag and a tool grant. Pin the model, because the eval child does not
|
|
35
39
|
inherit it, and pin the judge, because `--model` does not reach it and the
|
|
36
40
|
`llm` graders otherwise run on Haiku. From the repo root:
|
|
@@ -52,6 +56,14 @@ and halves the cost, and `--runs 1` drops the repeats. Traces are deleted when a
|
|
|
52
56
|
ends, so a failing `llm` grader cannot be read back afterwards; add
|
|
53
57
|
`--keep-temp` to any run whose failures you will need to diagnose.
|
|
54
58
|
|
|
59
|
+
The verification gate is not in the case-local plugin copies: they register no
|
|
60
|
+
hook, and the Stop hook a run meets is the globally installed one
|
|
61
|
+
(`~/.claude/skills/sluice/scripts/stop-guard.sh`, wired by your settings). So
|
|
62
|
+
the gate is measured only once this sluice version is installed. Once it is, a
|
|
63
|
+
`fast` or `main` case that never calls `status.sh verify` takes one refusal
|
|
64
|
+
turn before it stops, and a judge reading the last message reads the one
|
|
65
|
+
written after that refusal.
|
|
66
|
+
|
|
55
67
|
To gate CI, pick a floor and let a miss fail the job:
|
|
56
68
|
|
|
57
69
|
```bash
|
|
@@ -59,6 +71,78 @@ claude plugin eval skills/sluice --scaffold --allow-tools Bash Write Edit \
|
|
|
59
71
|
--trust-plugin --threshold 0.8
|
|
60
72
|
```
|
|
61
73
|
|
|
74
|
+
To check whether the rules in `rules.json` still earn their place on a model,
|
|
75
|
+
run both arms through `eval:fresh`. It runs on your Claude plan, at
|
|
76
|
+
`--runs 3 -j 2` by default, then diffs the result against
|
|
77
|
+
`evals/baselines/<model>.json`:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
bun run eval:fresh run skills/sluice --model claude-opus-5-5
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Re-reading a result costs nothing, and `--against` diffs it against another
|
|
84
|
+
model's baseline:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
bun run eval:fresh diff skills/sluice <result.json> [--against <model>]
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Writing the baseline is always a deliberate step of its own:
|
|
91
|
+
|
|
92
|
+
```bash
|
|
93
|
+
bun run eval:fresh accept skills/sluice <result.json>
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
`run` takes `--case <glob>`, `--runs <n>`, `-j <n>` and `--tolerance <x>`;
|
|
97
|
+
`diff` takes `--against <model>` and `--tolerance <x>`. Both exit 0 when no
|
|
98
|
+
case regressed, 1 when one did, and 2 on bad input (a missing flag value, an
|
|
99
|
+
unreadable result or a malformed manifest). `run` also exits 2 when the eval
|
|
100
|
+
writes no result. `accept` exits 2 on a result filtered with `--case`, since a
|
|
101
|
+
baseline needs a full run, and on a result with any errored run. `diff` lists
|
|
102
|
+
errored runs after the report without changing its exit code, and leaves them
|
|
103
|
+
out of every mean, since an errored run's score says nothing about the skill; a
|
|
104
|
+
case with no finished with-arm run is left out of the report. `diff` exits 2
|
|
105
|
+
when `--against` names a model with no baseline, where a missing baseline for
|
|
106
|
+
the result's own model only reads as "against no baseline". `run` passes
|
|
107
|
+
`--no-publish`, so the HTML report stays local.
|
|
108
|
+
|
|
109
|
+
A run cut short by a timeout, a quota limit or a closed session needs only
|
|
110
|
+
the cases it lacks. Rerun those with `run --case`, then fold the rerun into
|
|
111
|
+
the full result:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
bun run eval:fresh merge skills/sluice <full.json> <rerun.json>
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
It writes `<full>-merged.json` beside the full result, which `accept` takes.
|
|
118
|
+
It exits 2 when the two results differ in model, judge, CLI version, ablation,
|
|
119
|
+
skill or runs per case, or when the full result was itself filtered with
|
|
120
|
+
`--case`. It also exits 2 when the rerun misses a case the full result lacks,
|
|
121
|
+
has an errored or short case of its own, names a case the skill does not have,
|
|
122
|
+
or reruns a case the full result already finished. That last refusal keeps a
|
|
123
|
+
finished case's score from being rerolled. It cannot stop a case that needed
|
|
124
|
+
filling from being rerun several times with only the best rerun merged, so
|
|
125
|
+
merge the first clean rerun.
|
|
126
|
+
|
|
127
|
+
The diff gives each case its verdicts. `regressed` means the with-arm mean
|
|
128
|
+
dropped below the baseline by more than `--tolerance` (default 0.1).
|
|
129
|
+
`redundant` means the bare model matches the skill at 1.0. `skill-worse` means
|
|
130
|
+
the without-arm mean beats the with-arm mean by more than the tolerance, so the
|
|
131
|
+
skill is costing score. `flaky` means the runs disagree. `sandbox` is a drop past the tolerance on a case listed in
|
|
132
|
+
`sandboxSensitive`, and it replaces `regressed` there, because the drop is read
|
|
133
|
+
as the harness (see "Git inside the eval sandbox" below). `new` is a case the
|
|
134
|
+
baseline does not have, and `steady` is a case none of the others apply to. A
|
|
135
|
+
rule is "at risk" if any of its cases regressed, and an "expiry candidate" if
|
|
136
|
+
all of its cases are redundant.
|
|
137
|
+
|
|
138
|
+
Since a sandbox-sensitive case never reads `regressed`, a rule pinned only by
|
|
139
|
+
such cases never reads "at risk", on this machine or any other.
|
|
140
|
+
|
|
141
|
+
`vacuousWithout` lists cases whose graders, leaving out the `tool_used: Skill`
|
|
142
|
+
grader the no-plugin arm drops, can pass when the model does nothing the rule
|
|
143
|
+
asks for; those cases are never called redundant or skill-worse. `unclaimed` lists cases that
|
|
144
|
+
pin no rule SKILL.md states, each with its reason.
|
|
145
|
+
|
|
62
146
|
## Scoring notes
|
|
63
147
|
|
|
64
148
|
`tool_used: Skill` graders are excluded from the score in a two-arm run and
|
|
@@ -68,7 +152,7 @@ the model's own habits.
|
|
|
68
152
|
|
|
69
153
|
The two stops a deep run is allowed are both before Task 1: design sign-off and
|
|
70
154
|
plan-plus-pre-flight. `deep-plan-across-subsystems` pins the first. Everything
|
|
71
|
-
after pre-flight is covered by the
|
|
155
|
+
after pre-flight is covered by the execution cases, which is where a run
|
|
72
156
|
that checks in per task would show up. Their `.sluice/run.json` fixtures were
|
|
73
157
|
produced by `status.sh` and `plan.sh import` rather than typed by hand, so the
|
|
74
158
|
tiers and the graph columns match what the plan actually says; the plans pass
|
|
@@ -88,6 +172,27 @@ only from the case's own directory.
|
|
|
88
172
|
|
|
89
173
|
## Verification status
|
|
90
174
|
|
|
175
|
+
The accepted baseline is `baselines/claude-opus-5-5.json`; what follows is the
|
|
176
|
+
history that led to it.
|
|
177
|
+
|
|
178
|
+
### Opus 5.5, 2026-10-14
|
|
179
|
+
|
|
180
|
+
CLI 2.1.296 and then 2.1.299, `eval:fresh` with both arms and three runs per
|
|
181
|
+
case, judged by Opus 5.5. The first baseline (skill 0.23.2, CLI 2.1.296) left
|
|
182
|
+
three cases reading the harness or the fixture rather than the skill, each
|
|
183
|
+
fixed and then measured in the 2.1.299 baseline (skill 0.23.4):
|
|
184
|
+
|
|
185
|
+
| Case | With / without before | With / without after | Fix |
|
|
186
|
+
|---|---|---|---|
|
|
187
|
+
| `deep-run-survives-a-milestone` | below the without arm | 1.00 / 1.00 | The fixture's Task 5 timed steps it never ran, so the reviewer rightly blocked on 0 ms timings; it now runs them through `runStep` |
|
|
188
|
+
| `deep-run-fans-out` | 0.62 / 0.90 | 1.00 / 0.90 | The harness refuses Agent worktree isolation; `references/deep-channel.md` now cuts each implementer's worktree by hand under `.claude/worktrees/` |
|
|
189
|
+
| `announcement-reaches-the-ledger` | 0.40 / 0.40 | 1.00 / 0.40 | Opus 5.5 reads a fixture this small whole before routing, so the staged fast-then-main escalation never happened; the case now pins routing main first, and `escalate-out-loud` is unpinned |
|
|
190
|
+
|
|
191
|
+
Git in the sandbox failed differently this time: `/opt/homebrew/bin/git` is a
|
|
192
|
+
link into the Cellar, which the sandbox cannot read, so the shell fell back to
|
|
193
|
+
the xcrun shim. A one-line wrapper at `/usr/local/bin/git` that execs Homebrew's
|
|
194
|
+
git fixed it on this machine; another machine needs the same.
|
|
195
|
+
|
|
91
196
|
### Opus 5.5, 2026-09-23
|
|
92
197
|
|
|
93
198
|
CLI 2.1.280, `--model claude-opus-5-5`, one run per case, no no-plugin arm, so
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
|
-
# The fast-flag repo plus a house rule the prompt does not mention:
|
|
3
|
-
#
|
|
4
|
-
# API.md
|
|
5
|
-
# reads
|
|
2
|
+
# The fast-flag repo plus a house rule the prompt does not mention: a surface
|
|
3
|
+
# test fails once the CLI accepts a flag beyond --dry-run, saying flags are
|
|
4
|
+
# public API and need a flags() export listed in API.md. That export is a new
|
|
5
|
+
# interface, so a run that reads the repo before routing has its reason to go
|
|
6
|
+
# main before it touches source.
|
|
6
7
|
set -euo pipefail
|
|
7
8
|
|
|
8
9
|
mkdir -p src/cli tests
|
|
@@ -69,6 +70,31 @@ test("a full deploy reports every step", () => {
|
|
|
69
70
|
});
|
|
70
71
|
JS
|
|
71
72
|
|
|
73
|
+
cat > tests/surface.test.js <<'JS'
|
|
74
|
+
import assert from "node:assert/strict";
|
|
75
|
+
import { readFileSync } from "node:fs";
|
|
76
|
+
import { test } from "node:test";
|
|
77
|
+
|
|
78
|
+
// Integrators wrap shipit and read its flags from the package rather than
|
|
79
|
+
// parsing --help, so the flag surface is public API. --dry-run predates the
|
|
80
|
+
// rule; the first flag after it brings the flags() export with it.
|
|
81
|
+
test("every flag the CLI accepts is published by flags()", async () => {
|
|
82
|
+
const source = readFileSync(new URL("../src/cli/deploy.js", import.meta.url), "utf8");
|
|
83
|
+
const accepted = [...new Set([...source.matchAll(/"(--[a-z][a-z-]*)"/g)].map((m) => m[1]))];
|
|
84
|
+
if (accepted.every((flag) => flag === "--dry-run")) return;
|
|
85
|
+
|
|
86
|
+
const api = await import("../src/index.js");
|
|
87
|
+
assert.equal(
|
|
88
|
+
typeof api.flags,
|
|
89
|
+
"function",
|
|
90
|
+
"flags are public API: src/index.js must export flags(), returning { name, description } for every accepted flag including --dry-run, and API.md must document it",
|
|
91
|
+
);
|
|
92
|
+
const published = api.flags().map((flag) => flag.name).sort();
|
|
93
|
+
assert.deepEqual(published, [...accepted].sort());
|
|
94
|
+
for (const flag of api.flags()) assert.ok(flag.description, `${flag.name} has no description`);
|
|
95
|
+
});
|
|
96
|
+
JS
|
|
97
|
+
|
|
72
98
|
cat > API.md <<'MD'
|
|
73
99
|
# shipit public API
|
|
74
100
|
|
|
@@ -84,21 +110,8 @@ MD
|
|
|
84
110
|
cat > CONTRIBUTING.md <<'MD'
|
|
85
111
|
# Contributing to shipit
|
|
86
112
|
|
|
87
|
-
Run `npm test` before you push. Every behaviour change comes with a test
|
|
88
|
-
|
|
89
|
-
## Flags are public API
|
|
90
|
-
|
|
91
|
-
Integrators build their own wrappers around shipit and need to know which flags
|
|
92
|
-
the installed version accepts without parsing `--help`. So, starting with the
|
|
93
|
-
next flag we add:
|
|
94
|
-
|
|
95
|
-
- every flag the CLI accepts is returned by a `flags()` function exported from
|
|
96
|
-
`src/index.js`, as `{ name, description }` objects;
|
|
97
|
-
- `flags()` is documented in `API.md`, like every other export;
|
|
98
|
-
- `--dry-run` predates this rule and goes into `flags()` alongside the first new
|
|
99
|
-
flag.
|
|
100
|
-
|
|
101
|
-
A PR that adds a flag without both is not merged.
|
|
113
|
+
Run `npm test` before you push. Every behaviour change comes with a test, and
|
|
114
|
+
a PR with a red suite is not merged.
|
|
102
115
|
MD
|
|
103
116
|
|
|
104
117
|
cat > README.md <<'MD'
|
package/skills/sluice/evals/announcement-reaches-the-ledger/graders/routes-main-before-source.md
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
---
|
|
2
|
+
type: regex
|
|
3
|
+
# The surface test turns the first new flag into a new public flags() export,
|
|
4
|
+
# so the route that fits is main, and it has to come before source changes.
|
|
5
|
+
# Matches from the start of the trace to a Bash call invoking
|
|
6
|
+
# status.sh route main, refusing any Write or Edit under src/ or any fast or
|
|
7
|
+
# deep route on the way. Anchored as the announces graders are, so a quoted
|
|
8
|
+
# mention such as grep 'status.sh route main' does not count.
|
|
9
|
+
# file_path is found wherever it sits in the input: an Edit's opens with
|
|
10
|
+
# replace_all, so anchoring on file_path first let an early Edit through.
|
|
11
|
+
pattern: '^(?:(?!"name":"(?:Write|Edit|MultiEdit)","input":\{(?:"(?:[^"\\]|\\.)*"|[^"{}])*?"file_path":"[^"]*/src/|"command":"(?:(?:[^"\\]|\\.)*?(?:[\s/]|\\[nt]))?status\.sh(?:\\"|'')?(?:\s|\\[nt])+route(?:\s|\\[nt])+(?:fast|deep)(?:\s|\\[nt]|[;&|]|"))[\s\S])*?"command":"(?:(?:[^"\\]|\\.)*?(?:[\s/]|\\[nt]))?status\.sh(?:\\"|'')?(?:\s|\\[nt])+route(?:\s|\\[nt])+main(?:\s|\\[nt]|[;&|]|")'
|
|
12
|
+
target: trace
|
|
13
|
+
weight: 3
|
|
14
|
+
---
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: announcement-reaches-the-ledger
|
|
3
|
-
description: A new flag
|
|
3
|
+
description: A new flag the repo's surface test turns into a new public export. Pins that the run reads far enough to route main through status.sh before touching source.
|
|
4
4
|
tags: [routing, fast, main, escalation, scaffold, write]
|
|
5
5
|
max_turns: 60
|
|
6
6
|
timeout_seconds: 1800
|
|
7
7
|
allowed_tools: [Read, Glob, Grep, Skill, Write, Edit, Bash]
|
|
8
|
-
expected_outcome:
|
|
8
|
+
expected_outcome: Reads the repo, sees from the surface test that the first new flag needs a public flags() export listed in API.md, routes main through status.sh before editing src/, then ships --verbose and flags() with the suite green.
|
|
9
9
|
---
|
|
10
10
|
|
|
11
11
|
Add a `--verbose` flag to the deploy command. After each step's progress line it prints one more line saying the step finished, for example ` build done`. Without the flag the output stays exactly as it is.
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
{
|
|
2
|
+
"model": "claude-opus-5-5",
|
|
3
|
+
"judgeModel": "claude-opus-5-5",
|
|
4
|
+
"claudeVersion": "2.1.300",
|
|
5
|
+
"skillVersion": "0.24.1",
|
|
6
|
+
"date": "2026-10-15T10:16:06.877Z",
|
|
7
|
+
"cases": {
|
|
8
|
+
"announcement-reaches-the-ledger": {
|
|
9
|
+
"with": [
|
|
10
|
+
1,
|
|
11
|
+
1,
|
|
12
|
+
1
|
|
13
|
+
],
|
|
14
|
+
"without": [
|
|
15
|
+
0.4,
|
|
16
|
+
0.4,
|
|
17
|
+
0.4
|
|
18
|
+
]
|
|
19
|
+
},
|
|
20
|
+
"bypass-question-stays-silent": {
|
|
21
|
+
"with": [
|
|
22
|
+
1,
|
|
23
|
+
1,
|
|
24
|
+
1
|
|
25
|
+
],
|
|
26
|
+
"without": [
|
|
27
|
+
1,
|
|
28
|
+
1,
|
|
29
|
+
1
|
|
30
|
+
]
|
|
31
|
+
},
|
|
32
|
+
"deep-plan-across-subsystems": {
|
|
33
|
+
"with": [
|
|
34
|
+
0.7142857142857143,
|
|
35
|
+
1,
|
|
36
|
+
1
|
|
37
|
+
],
|
|
38
|
+
"without": [
|
|
39
|
+
0.42857142857142855,
|
|
40
|
+
0.42857142857142855,
|
|
41
|
+
0.42857142857142855
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
"deep-plan-asks-the-fork": {
|
|
45
|
+
"with": [
|
|
46
|
+
0.6666666666666666,
|
|
47
|
+
0.6666666666666666,
|
|
48
|
+
1
|
|
49
|
+
],
|
|
50
|
+
"without": [
|
|
51
|
+
0.4444444444444444,
|
|
52
|
+
0.4444444444444444,
|
|
53
|
+
0.4444444444444444
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"deep-run-blocks-on-a-real-decision": {
|
|
57
|
+
"with": [
|
|
58
|
+
1,
|
|
59
|
+
1,
|
|
60
|
+
1
|
|
61
|
+
],
|
|
62
|
+
"without": [
|
|
63
|
+
0.7272727272727273,
|
|
64
|
+
1,
|
|
65
|
+
1
|
|
66
|
+
]
|
|
67
|
+
},
|
|
68
|
+
"deep-run-decides-a-non-blocking-choice": {
|
|
69
|
+
"with": [
|
|
70
|
+
1,
|
|
71
|
+
1,
|
|
72
|
+
1
|
|
73
|
+
],
|
|
74
|
+
"without": [
|
|
75
|
+
1,
|
|
76
|
+
1,
|
|
77
|
+
1
|
|
78
|
+
]
|
|
79
|
+
},
|
|
80
|
+
"deep-run-fans-out": {
|
|
81
|
+
"with": [
|
|
82
|
+
1,
|
|
83
|
+
1,
|
|
84
|
+
1
|
|
85
|
+
],
|
|
86
|
+
"without": [
|
|
87
|
+
0.5,
|
|
88
|
+
1,
|
|
89
|
+
1
|
|
90
|
+
]
|
|
91
|
+
},
|
|
92
|
+
"deep-run-finishes-every-task": {
|
|
93
|
+
"with": [
|
|
94
|
+
1,
|
|
95
|
+
1,
|
|
96
|
+
1
|
|
97
|
+
],
|
|
98
|
+
"without": [
|
|
99
|
+
1,
|
|
100
|
+
1,
|
|
101
|
+
1
|
|
102
|
+
]
|
|
103
|
+
},
|
|
104
|
+
"deep-run-hunts-after-the-flip": {
|
|
105
|
+
"with": [
|
|
106
|
+
1,
|
|
107
|
+
1,
|
|
108
|
+
1
|
|
109
|
+
],
|
|
110
|
+
"without": [
|
|
111
|
+
0.4444444444444444,
|
|
112
|
+
0.6666666666666666,
|
|
113
|
+
0.4444444444444444
|
|
114
|
+
]
|
|
115
|
+
},
|
|
116
|
+
"deep-run-survives-a-milestone": {
|
|
117
|
+
"with": [
|
|
118
|
+
1,
|
|
119
|
+
1,
|
|
120
|
+
1
|
|
121
|
+
],
|
|
122
|
+
"without": [
|
|
123
|
+
1,
|
|
124
|
+
1,
|
|
125
|
+
1
|
|
126
|
+
]
|
|
127
|
+
},
|
|
128
|
+
"deep-run-waits-for-a-running-agent": {
|
|
129
|
+
"with": [
|
|
130
|
+
1,
|
|
131
|
+
1,
|
|
132
|
+
1
|
|
133
|
+
],
|
|
134
|
+
"without": [
|
|
135
|
+
0.75,
|
|
136
|
+
0.75,
|
|
137
|
+
0.75
|
|
138
|
+
]
|
|
139
|
+
},
|
|
140
|
+
"explicit-instruction-collapses-to-fast": {
|
|
141
|
+
"with": [
|
|
142
|
+
1,
|
|
143
|
+
1,
|
|
144
|
+
1
|
|
145
|
+
],
|
|
146
|
+
"without": [
|
|
147
|
+
0.7142857142857143,
|
|
148
|
+
0.42857142857142855,
|
|
149
|
+
0.42857142857142855
|
|
150
|
+
]
|
|
151
|
+
},
|
|
152
|
+
"fast-flag-on-existing-command": {
|
|
153
|
+
"with": [
|
|
154
|
+
1,
|
|
155
|
+
1,
|
|
156
|
+
0.7142857142857143
|
|
157
|
+
],
|
|
158
|
+
"without": [
|
|
159
|
+
0.2857142857142857,
|
|
160
|
+
0.2857142857142857,
|
|
161
|
+
0.2857142857142857
|
|
162
|
+
]
|
|
163
|
+
},
|
|
164
|
+
"fast-reads-the-unmentioned-convention": {
|
|
165
|
+
"with": [
|
|
166
|
+
1,
|
|
167
|
+
1,
|
|
168
|
+
1
|
|
169
|
+
],
|
|
170
|
+
"without": [
|
|
171
|
+
1,
|
|
172
|
+
1,
|
|
173
|
+
1
|
|
174
|
+
]
|
|
175
|
+
},
|
|
176
|
+
"fast-verifies-before-handback": {
|
|
177
|
+
"with": [
|
|
178
|
+
1,
|
|
179
|
+
1,
|
|
180
|
+
1
|
|
181
|
+
],
|
|
182
|
+
"without": [
|
|
183
|
+
0.25,
|
|
184
|
+
0.25,
|
|
185
|
+
0.25
|
|
186
|
+
]
|
|
187
|
+
},
|
|
188
|
+
"main-hunts-a-checkable-claim": {
|
|
189
|
+
"with": [
|
|
190
|
+
1,
|
|
191
|
+
1,
|
|
192
|
+
1
|
|
193
|
+
],
|
|
194
|
+
"without": [
|
|
195
|
+
0.42857142857142855,
|
|
196
|
+
0.42857142857142855,
|
|
197
|
+
0.42857142857142855
|
|
198
|
+
]
|
|
199
|
+
},
|
|
200
|
+
"main-new-interface": {
|
|
201
|
+
"with": [
|
|
202
|
+
1,
|
|
203
|
+
0.6666666666666666,
|
|
204
|
+
1
|
|
205
|
+
],
|
|
206
|
+
"without": [
|
|
207
|
+
0.3333333333333333,
|
|
208
|
+
0.3333333333333333,
|
|
209
|
+
0.3333333333333333
|
|
210
|
+
]
|
|
211
|
+
},
|
|
212
|
+
"main-skips-the-hunt-without-a-claim": {
|
|
213
|
+
"with": [
|
|
214
|
+
1,
|
|
215
|
+
1,
|
|
216
|
+
0.5
|
|
217
|
+
],
|
|
218
|
+
"without": [
|
|
219
|
+
0,
|
|
220
|
+
0,
|
|
221
|
+
0
|
|
222
|
+
]
|
|
223
|
+
},
|
|
224
|
+
"superpowers-conflict-stands-down": {
|
|
225
|
+
"with": [
|
|
226
|
+
1,
|
|
227
|
+
1,
|
|
228
|
+
1
|
|
229
|
+
],
|
|
230
|
+
"without": [
|
|
231
|
+
0.5,
|
|
232
|
+
0.5,
|
|
233
|
+
0.5
|
|
234
|
+
]
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|