@iceinvein/agent-skills 0.21.4 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +9 -0
  2. package/package.json +4 -1
  3. package/skills/bug-hunter/SKILL.md +82 -0
  4. package/skills/bug-hunter/evals/README.md +282 -0
  5. package/skills/bug-hunter/evals/baselines/claude-opus-5-5.json +93 -0
  6. package/skills/bug-hunter/evals/bug-cites-intent-from-spec/case.yaml +4 -0
  7. package/skills/bug-hunter/evals/bug-cites-intent-from-spec/fixture.sh +71 -0
  8. package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/bug-cites-the-spec.md +36 -0
  9. package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/bug-hunter-fired.md +5 -0
  10. package/skills/bug-hunter/evals/bug-cites-intent-from-spec/graders/intent-written.md +8 -0
  11. package/skills/bug-hunter/evals/bug-cites-intent-from-spec/prompt.md +11 -0
  12. package/skills/bug-hunter/evals/callers-tree-unchanged/case.yaml +4 -0
  13. package/skills/bug-hunter/evals/callers-tree-unchanged/fixture.sh +67 -0
  14. package/skills/bug-hunter/evals/callers-tree-unchanged/graders/bug-hunter-fired.md +5 -0
  15. package/skills/bug-hunter/evals/callers-tree-unchanged/graders/source-unchanged.md +10 -0
  16. package/skills/bug-hunter/evals/callers-tree-unchanged/graders/tests-unchanged.md +10 -0
  17. package/skills/bug-hunter/evals/callers-tree-unchanged/graders/tree-unchanged.md +15 -0
  18. package/skills/bug-hunter/evals/callers-tree-unchanged/prompt.md +11 -0
  19. package/skills/bug-hunter/evals/correct-code-has-no-bugs/case.yaml +4 -0
  20. package/skills/bug-hunter/evals/correct-code-has-no-bugs/fixture.sh +82 -0
  21. package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/bug-hunter-fired.md +5 -0
  22. package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/bugs-empty.md +18 -0
  23. package/skills/bug-hunter/evals/correct-code-has-no-bugs/graders/written-with-write-tool.md +13 -0
  24. package/skills/bug-hunter/evals/correct-code-has-no-bugs/prompt.md +11 -0
  25. package/skills/bug-hunter/evals/impossible-value-is-a-bug/case.yaml +4 -0
  26. package/skills/bug-hunter/evals/impossible-value-is-a-bug/fixture.sh +68 -0
  27. package/skills/bug-hunter/evals/impossible-value-is-a-bug/graders/bug-for-sixty-seconds.md +37 -0
  28. package/skills/bug-hunter/evals/impossible-value-is-a-bug/graders/bug-hunter-fired.md +5 -0
  29. package/skills/bug-hunter/evals/impossible-value-is-a-bug/prompt.md +11 -0
  30. package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/case.yaml +4 -0
  31. package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/fixture.sh +79 -0
  32. package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/bug-cites-the-limit.md +31 -0
  33. package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/bug-hunter-fired.md +5 -0
  34. package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/graders/findings-written.md +10 -0
  35. package/skills/bug-hunter/evals/proves-a-spec-boundary-bug/prompt.md +11 -0
  36. package/skills/bug-hunter/evals/rules.json +177 -0
  37. package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/case.yaml +4 -0
  38. package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/fixture.sh +97 -0
  39. package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/bug-hunter-fired.md +5 -0
  40. package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/bugs-empty.md +18 -0
  41. package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/cap-is-an-ambiguity.md +12 -0
  42. package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/graders/written-with-write-tool.md +13 -0
  43. package/skills/bug-hunter/evals/silent-spec-is-an-ambiguity/prompt.md +11 -0
  44. package/skills/bug-hunter/evals/spec-outranks-target-docstring/case.yaml +4 -0
  45. package/skills/bug-hunter/evals/spec-outranks-target-docstring/fixture.sh +82 -0
  46. package/skills/bug-hunter/evals/spec-outranks-target-docstring/graders/bug-cites-the-spec.md +40 -0
  47. package/skills/bug-hunter/evals/spec-outranks-target-docstring/graders/bug-hunter-fired.md +5 -0
  48. package/skills/bug-hunter/evals/spec-outranks-target-docstring/prompt.md +11 -0
  49. package/skills/bug-hunter/references/findings-format.md +82 -0
  50. package/skills/bug-hunter/references/scenario-catalog.md +92 -0
  51. package/skills/bug-hunter/skill.json +24 -0
  52. package/skills/index.json +17 -5
  53. package/skills/magpie/evals/README.md +13 -0
  54. package/skills/magpie/evals/baselines/claude-opus-5-5.json +69 -0
  55. package/skills/magpie/evals/rules.json +192 -0
  56. package/skills/magpie/skill.json +1 -1
  57. package/skills/migrate/evals/README.md +13 -0
  58. package/skills/migrate/evals/baselines/claude-opus-5-5.json +129 -0
  59. package/skills/migrate/evals/rules.json +160 -0
  60. package/skills/migrate/skill.json +10 -4
  61. package/skills/scope-estimate/README.md +173 -0
  62. package/skills/scope-estimate/SKILL.md +168 -0
  63. package/skills/scope-estimate/references/capability-map.md +122 -0
  64. package/skills/scope-estimate/references/deliverables.md +60 -0
  65. package/skills/scope-estimate/references/requirements.md +104 -0
  66. package/skills/scope-estimate/references/schedule.md +138 -0
  67. package/skills/scope-estimate/scripts/__tests__/schedule.test.ts +366 -0
  68. package/skills/scope-estimate/scripts/schedule.ts +553 -0
  69. package/skills/scope-estimate/skill.json +19 -0
  70. package/skills/scope-estimate/templates/plan.example.json +33 -0
  71. package/skills/sluice/SKILL.md +35 -12
  72. package/skills/sluice/evals/README.md +111 -6
  73. package/skills/sluice/evals/announcement-reaches-the-ledger/fixture.sh +32 -19
  74. package/skills/sluice/evals/announcement-reaches-the-ledger/graders/routes-main-before-source.md +14 -0
  75. package/skills/sluice/evals/announcement-reaches-the-ledger/prompt.md +2 -2
  76. package/skills/sluice/evals/baselines/claude-opus-5-5.json +237 -0
  77. package/skills/sluice/evals/deep-run-blocks-on-a-real-decision/fixture.sh +19 -12
  78. package/skills/sluice/evals/deep-run-fans-out/graders/task-1-merged-back.md +8 -0
  79. package/skills/sluice/evals/deep-run-fans-out/graders/task-2-merged-back.md +8 -0
  80. package/skills/sluice/evals/deep-run-fans-out/graders/task-3-merged-back.md +6 -0
  81. package/skills/sluice/evals/deep-run-fans-out/graders/worktree-per-implementer.md +12 -0
  82. package/skills/sluice/evals/deep-run-hunts-after-the-flip/case.yaml +5 -0
  83. package/skills/sluice/evals/deep-run-hunts-after-the-flip/fixture.sh +258 -0
  84. package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/bug-hunter-fired.md +5 -0
  85. package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/four-attempts-allowed.md +34 -0
  86. package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/fourth-attempt-tested.md +20 -0
  87. package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/hunt-before-the-final-review.md +21 -0
  88. package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/record-has-a-hunt-section.md +11 -0
  89. package/skills/sluice/evals/deep-run-hunts-after-the-flip/graders/sluice-fired.md +5 -0
  90. package/skills/sluice/evals/deep-run-hunts-after-the-flip/prompt.md +12 -0
  91. package/skills/sluice/evals/deep-run-survives-a-milestone/fixture.sh +46 -24
  92. package/skills/sluice/evals/fast-verifies-before-handback/case.yaml +4 -0
  93. package/skills/sluice/evals/fast-verifies-before-handback/fixture.sh +73 -0
  94. package/skills/sluice/evals/fast-verifies-before-handback/graders/quiet-flag-implemented.md +5 -0
  95. package/skills/sluice/evals/fast-verifies-before-handback/graders/sluice-fired.md +5 -0
  96. package/skills/sluice/evals/fast-verifies-before-handback/graders/verified-after-the-last-edit.md +18 -0
  97. package/skills/sluice/evals/fast-verifies-before-handback/prompt.md +11 -0
  98. package/skills/sluice/evals/main-hunts-a-checkable-claim/case.yaml +5 -0
  99. package/skills/sluice/evals/main-hunts-a-checkable-claim/fixture.sh +64 -0
  100. package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/announces-main-channel.md +12 -0
  101. package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/bug-hunter-fired.md +5 -0
  102. package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/hunt-after-the-source-edits.md +12 -0
  103. package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/last-page-holds-the-remainder.md +27 -0
  104. package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/remainder-page-tested.md +21 -0
  105. package/skills/sluice/evals/main-hunts-a-checkable-claim/graders/sluice-fired.md +5 -0
  106. package/skills/sluice/evals/main-hunts-a-checkable-claim/prompt.md +12 -0
  107. package/skills/sluice/evals/main-new-interface/graders/shape-agreed-before-building.md +8 -4
  108. package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/case.yaml +5 -0
  109. package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/fixture.sh +73 -0
  110. package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/announces-main-channel.md +12 -0
  111. package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/no-hunt.md +12 -0
  112. package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/skip-said-in-one-line.md +21 -0
  113. package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/graders/sluice-fired.md +5 -0
  114. package/skills/sluice/evals/main-skips-the-hunt-without-a-claim/prompt.md +12 -0
  115. package/skills/sluice/evals/rules.json +364 -0
  116. package/skills/sluice/references/deep-channel.md +64 -18
  117. package/skills/sluice/references/finish.md +8 -2
  118. package/skills/sluice/references/intent.md +4 -2
  119. package/skills/sluice/references/review.md +32 -0
  120. package/skills/sluice/references/status.md +17 -4
  121. package/skills/sluice/references/verify.md +38 -0
  122. package/skills/sluice/scripts/content-digest.sh +65 -0
  123. package/skills/sluice/scripts/run-stats.sh +9 -8
  124. package/skills/sluice/scripts/status.sh +78 -13
  125. package/skills/sluice/scripts/stop-guard.sh +97 -24
  126. package/skills/sluice/skill.json +2 -2
  127. package/skills/terse/evals/README.md +16 -0
  128. package/skills/terse/evals/baselines/claude-opus-5-5.json +105 -0
  129. package/skills/terse/evals/rules.json +161 -0
  130. package/skills/terse/skill.json +1 -1
  131. package/skills/sluice/evals/announcement-reaches-the-ledger/graders/escalates-fast-to-main.md +0 -12
@@ -21,16 +21,21 @@ has to be taken back.
21
21
  |---------|--------|-------|-----|
22
22
  | `bypass` | No code change: a question, an explanation, a read | none | nothing, just answer |
23
23
  | `fast` | Existing interfaces, one subsystem (a new flag on an existing command still counts as existing) | test-first, verify | "Fast channel, existing interfaces. Test first, then implement." |
24
- | `main` | Adds an interface, or crosses subsystems | + agree intent, review before merge | "Main channel, new interface. Agreeing the shape first." |
25
- | `deep` | Several subsystems, or a plan was asked for | + written design and plan | "Deep channel, several subsystems. Design before code." |
24
+ | `main` | Adds an interface, or crosses subsystems | + agree intent, hunt a checkable claim, review before merge | "Main channel, new interface. Agreeing the shape first." |
25
+ | `deep` | Several subsystems, or a plan was asked for | + written design and plan, hunt after the flip | "Deep channel, several subsystems. Design before code." |
26
26
 
27
27
  Two subsystems is `main`; the third is what makes it `deep`.
28
28
 
29
29
  `bypass`, `fast`, and `main` proceed without stopping for approval; only
30
30
  `deep` stops: once for design sign-off before code, once for the plan and
31
- pre-flight together before Task 1. A stop ends your turn. Asking a question
32
- with a tool is not one, however many options it carried: the answer comes
33
- back to you and the run never left your hands.
31
+ pre-flight together before Task 1. A stop ends your turn.
32
+
33
+ A choice that is your partner's to make goes to them through `AskUserQuestion`
34
+ wherever the harness has it, at a stop or between them: options to pick from,
35
+ your recommendation first, not a paragraph or a lettered list to reply to.
36
+ Asking is not itself a stop, however many options it carried, because the
37
+ answer comes back to you and the run never left your hands. So at a stop, ask
38
+ with the tool, then end the turn once the answers are in.
34
39
 
35
40
  Name the channel and the signal that actually routed you there. The strings above
36
41
  are examples, not fixed copy, and a channel with a two-part signal should say
@@ -66,9 +71,9 @@ announces nothing to measure from.
66
71
  One line each. Read the reference only on friction: the moment you notice
67
72
  yourself wanting to skip the rule, or arguing that this one is the exception.
68
73
 
69
- - **Agree intent** before building. One question at a time. Propose approaches
70
- with a recommendation, not a survey. `main` agrees in a message, `deep` writes
71
- it down. `references/intent.md`
74
+ - **Agree intent** before building. One question at a time, through
75
+ `AskUserQuestion`. Propose approaches with a recommendation, not a survey.
76
+ `main` agrees in a message, `deep` writes it down. `references/intent.md`
72
77
  - **Test first.** The test comes before the code; run it while it should
73
78
  still be failing, then write the least code that turns it green. Skip that
74
79
  watching step and a green result is only an unchecked guess. A test no
@@ -78,7 +83,21 @@ yourself wanting to skip the rule, or arguing that this one is the exception.
78
83
  `references/root-cause.md`
79
84
  - **Verify** before claiming. Run the command in this turn and read its
80
85
  output; the claim comes after that, never before. How sure you feel is not
81
- something anyone else can check. `references/verify.md`
86
+ something anyone else can check, so record the check: `status.sh verify --
87
+ <cmd>` notes the tree it passed on, or `--none "<reason>"` where no check
88
+ can cover the change. In `fast` and `main` a Stop hook refuses, once per
89
+ turn, a handback whose tree has moved since. `references/verify.md`
90
+ - **Hunt** in `main`, once green and before review: when the intent agreed in
91
+ the message states a checkable claim (a limit, a boundary, an invariant, a
92
+ stated result), run bug-hunter on the diff; otherwise say in one line there
93
+ was none and skip it. Fix each proven bug test-first; ambiguities go to your
94
+ partner at the handback. Record the check last, `status.sh verify --` after
95
+ the hunt and any fix, since the hunt's report lands in the tree. No
96
+ bug-hunter? Say so in the announcement and again at the merge.
97
+ `references/review.md`
98
+ - **Hunt** in `deep`, after the flip, once every task is done and before the
99
+ final review: bug-hunter on `<base>..HEAD` against the spec, each proven bug
100
+ one fix, test-first. `references/deep-channel.md`
82
101
  - **Review** before merge. Dispatch a reviewer with fresh context and put the
83
102
  diff on disk for it to read, so those bytes fill their context instead of
84
103
  yours. A session that dispatches only when asked has not refused you one, so
@@ -104,7 +123,10 @@ collapses to `fast`.
104
123
 
105
124
  Design to `docs/specs/YYYY-MM-DD-<topic>.md`, plan to
106
125
  `docs/plans/YYYY-MM-DD-<topic>.md`, unless the repo has a convention or
107
- your partner states a preference. Get the design signed off before code.
126
+ your partner states a preference. Get the design signed off before code. When
127
+ the design's shape turns on a fork only your partner can settle, ask that one
128
+ question through `AskUserQuestion`, with your recommendation, before the design
129
+ is written. `references/deep-channel.md` has the test for what counts.
108
130
 
109
131
  Take the design stop through the harness's plan mode where there is one. Its
110
132
  gate is enforced rather than requested and it holds edits shut while it is open,
@@ -162,8 +184,9 @@ instead of to the branch at large.
162
184
 
163
185
  Pre-flight rides in that same stop: which flagged tasks get a reviewer, which
164
186
  mechanical ones the plan marked for low effort, and whether the work runs
165
- in a worktree. Ask them as choices with the counts in them, never as a
166
- paragraph, then end the turn. Task 1 opens on their next
187
+ in a worktree. Ask them through `AskUserQuestion`, one question each with the
188
+ counts in the options, never as a paragraph or a lettered list, then end the
189
+ turn on the answers. Task 1 opens on their next
167
190
  instruction, and an answer that already carries one is that instruction. A
168
191
  session that dispatches only when asked has not ruled dispatch out, it has made
169
192
  this question the place to ask; genuine unavailability is the tool not being
@@ -1,8 +1,8 @@
1
1
  # sluice evals
2
2
 
3
- Fifteen cases for `claude plugin eval`. Nine pin the routing decision: which
3
+ Nineteen cases for `claude plugin eval`. Twelve pin the routing decision: which
4
4
  channel the run names, and whether the behaviour that channel owes actually
5
- happened. Six pin what a deep run does after pre-flight, where the question is
5
+ happened. Seven pin what a deep run does after pre-flight, where the question is
6
6
  no longer which channel but whether the run keeps going.
7
7
 
8
8
  | Case | Signal under test | What it pins |
@@ -14,8 +14,11 @@ no longer which channel but whether the run keeps going.
14
14
  | `deep-plan-asks-the-fork` | The same plan with nothing saying where the counters can live | One question with a recommendation before any design; no spec yet |
15
15
  | `explicit-instruction-collapses-to-fast` | Main-shaped work plus "just do it" | Collapses to fast; no design, no proposal |
16
16
  | `superpowers-conflict-stands-down` | Repo mandates the superpowers sequence | Stands down once, names no channel |
17
- | `announcement-reaches-the-ledger` | A flag that a repo rule turns into a new export | `status.sh route fast`, then `status.sh route main` once the rule is read |
17
+ | `announcement-reaches-the-ledger` | A flag that a surface test turns into a new export | `status.sh route main` before any write under `src/` and before any fast or deep route |
18
18
  | `fast-reads-the-unmentioned-convention` | Flags live in a table the prompt never names | The flag lands in the registry table and the generated `--help` |
19
+ | `fast-verifies-before-handback` | A fast flag, then the handback | A passing `status.sh verify` after the last edit, before handing back |
20
+ | `main-hunts-a-checkable-claim` | Main work whose intent states a checkable claim | bug-hunter on the diff once green; the `lastPage` remainder bug it proves, fixed test-first |
21
+ | `main-skips-the-hunt-without-a-claim` | A new interface with nothing to check | No bug-hunter; one line saying the intent stated no checkable claim |
19
22
 
20
23
  Execution, where the run is already past both stops:
21
24
 
@@ -26,11 +29,12 @@ Execution, where the run is already past both stops:
26
29
  | `deep-run-survives-a-milestone` | Six tasks, the first three named a milestone | All six done; the Stop hook never had to refuse; the final message is a handback |
27
30
  | `deep-run-decides-a-non-blocking-choice` | A task leaves a choice open and says either is fine | Decided, written into the record, no task blocked, no decision list at the end |
28
31
  | `deep-run-waits-for-a-running-agent` | Every task goes to an implementer agent | A `T2:` labelled dispatch; the handback comes after the last result |
29
- | `deep-run-fans-out` | Three disjoint tasks under a worktree-per-implementer answer | Three dispatches in one message, then the flip; also the baseline for a time-budget A/B |
32
+ | `deep-run-fans-out` | Three disjoint tasks under a worktree-per-implementer answer | Three dispatches in one message, then the flip; each task's files merged back into the run's own tree; also the baseline for a time-budget A/B |
33
+ | `deep-run-hunts-after-the-flip` | Every task done, the flip included | bug-hunter on the plan diff against the spec before the final review; the proven off-by-one fixed test-first; a Hunt section in the record |
30
34
 
31
35
  ## Running
32
36
 
33
- Fourteen cases scaffold a small Node repo and then change it, so they need the
37
+ Eighteen cases scaffold a small Node repo and then change it, so they need the
34
38
  scaffold flag and a tool grant. Pin the model, because the eval child does not
35
39
  inherit it, and pin the judge, because `--model` does not reach it and the
36
40
  `llm` graders otherwise run on Haiku. From the repo root:
@@ -52,6 +56,14 @@ and halves the cost, and `--runs 1` drops the repeats. Traces are deleted when a
52
56
  ends, so a failing `llm` grader cannot be read back afterwards; add
53
57
  `--keep-temp` to any run whose failures you will need to diagnose.
54
58
 
59
+ The verification gate is not in the case-local plugin copies: they register no
60
+ hook, and the Stop hook a run meets is the globally installed one
61
+ (`~/.claude/skills/sluice/scripts/stop-guard.sh`, wired by your settings). So
62
+ the gate is measured only once this sluice version is installed. Once it is, a
63
+ `fast` or `main` case that never calls `status.sh verify` takes one refusal
64
+ turn before it stops, and a judge reading the last message reads the one
65
+ written after that refusal.
66
+
55
67
  To gate CI, pick a floor and let a miss fail the job:
56
68
 
57
69
  ```bash
@@ -59,6 +71,78 @@ claude plugin eval skills/sluice --scaffold --allow-tools Bash Write Edit \
59
71
  --trust-plugin --threshold 0.8
60
72
  ```
61
73
 
74
+ To check whether the rules in `rules.json` still earn their place on a model,
75
+ run both arms through `eval:fresh`. It runs on your Claude plan, at
76
+ `--runs 3 -j 2` by default, then diffs the result against
77
+ `evals/baselines/<model>.json`:
78
+
79
+ ```bash
80
+ bun run eval:fresh run skills/sluice --model claude-opus-5-5
81
+ ```
82
+
83
+ Re-reading a result costs nothing, and `--against` diffs it against another
84
+ model's baseline:
85
+
86
+ ```bash
87
+ bun run eval:fresh diff skills/sluice <result.json> [--against <model>]
88
+ ```
89
+
90
+ Writing the baseline is always a deliberate step of its own:
91
+
92
+ ```bash
93
+ bun run eval:fresh accept skills/sluice <result.json>
94
+ ```
95
+
96
+ `run` takes `--case <glob>`, `--runs <n>`, `-j <n>` and `--tolerance <x>`;
97
+ `diff` takes `--against <model>` and `--tolerance <x>`. Both exit 0 when no
98
+ case regressed, 1 when one did, and 2 on bad input (a missing flag value, an
99
+ unreadable result or a malformed manifest). `run` also exits 2 when the eval
100
+ writes no result. `accept` exits 2 on a result filtered with `--case`, since a
101
+ baseline needs a full run, and on a result with any errored run. `diff` lists
102
+ errored runs after the report without changing its exit code, and leaves them
103
+ out of every mean, since an errored run's score says nothing about the skill; a
104
+ case with no finished with-arm run is left out of the report. `diff` exits 2
105
+ when `--against` names a model with no baseline, where a missing baseline for
106
+ the result's own model only reads as "against no baseline". `run` passes
107
+ `--no-publish`, so the HTML report stays local.
108
+
109
+ A run cut short by a timeout, a quota limit or a closed session needs only
110
+ the cases it lacks. Rerun those with `run --case`, then fold the rerun into
111
+ the full result:
112
+
113
+ ```bash
114
+ bun run eval:fresh merge skills/sluice <full.json> <rerun.json>
115
+ ```
116
+
117
+ It writes `<full>-merged.json` beside the full result, which `accept` takes.
118
+ It exits 2 when the two results differ in model, judge, CLI version, ablation,
119
+ skill or runs per case, or when the full result was itself filtered with
120
+ `--case`. It also exits 2 when the rerun misses a case the full result lacks,
121
+ has an errored or short case of its own, names a case the skill does not have,
122
+ or reruns a case the full result already finished. That last refusal keeps a
123
+ finished case's score from being rerolled. It cannot stop a case that needed
124
+ filling from being rerun several times with only the best rerun merged, so
125
+ merge the first clean rerun.
126
+
127
+ The diff gives each case its verdicts. `regressed` means the with-arm mean
128
+ dropped below the baseline by more than `--tolerance` (default 0.1).
129
+ `redundant` means the bare model matches the skill at 1.0. `skill-worse` means
130
+ the without-arm mean beats the with-arm mean by more than the tolerance, so the
131
+ skill is costing score. `flaky` means the runs disagree. `sandbox` is a drop past the tolerance on a case listed in
132
+ `sandboxSensitive`, and it replaces `regressed` there, because the drop is read
133
+ as the harness (see "Git inside the eval sandbox" below). `new` is a case the
134
+ baseline does not have, and `steady` is a case none of the others apply to. A
135
+ rule is "at risk" if any of its cases regressed, and an "expiry candidate" if
136
+ all of its cases are redundant.
137
+
138
+ Since a sandbox-sensitive case never reads `regressed`, a rule pinned only by
139
+ such cases never reads "at risk", on this machine or any other.
140
+
141
+ `vacuousWithout` lists cases whose graders, leaving out the `tool_used: Skill`
142
+ grader the no-plugin arm drops, can pass when the model does nothing the rule
143
+ asks for; those cases are never called redundant or skill-worse. `unclaimed` lists cases that
144
+ pin no rule SKILL.md states, each with its reason.
145
+
62
146
  ## Scoring notes
63
147
 
64
148
  `tool_used: Skill` graders are excluded from the score in a two-arm run and
@@ -68,7 +152,7 @@ the model's own habits.
68
152
 
69
153
  The two stops a deep run is allowed are both before Task 1: design sign-off and
70
154
  plan-plus-pre-flight. `deep-plan-across-subsystems` pins the first. Everything
71
- after pre-flight is covered by the two execution cases, which is where a run
155
+ after pre-flight is covered by the execution cases, which is where a run
72
156
  that checks in per task would show up. Their `.sluice/run.json` fixtures were
73
157
  produced by `status.sh` and `plan.sh import` rather than typed by hand, so the
74
158
  tiers and the graph columns match what the plan actually says; the plans pass
@@ -88,6 +172,27 @@ only from the case's own directory.
88
172
 
89
173
  ## Verification status
90
174
 
175
+ The accepted baseline is `baselines/claude-opus-5-5.json`; what follows is the
176
+ history that led to it.
177
+
178
+ ### Opus 5.5, 2026-10-14
179
+
180
+ CLI 2.1.296 and then 2.1.299, `eval:fresh` with both arms and three runs per
181
+ case, judged by Opus 5.5. The first baseline (skill 0.23.2, CLI 2.1.296) left
182
+ three cases reading the harness or the fixture rather than the skill, each
183
+ fixed and then measured in the 2.1.299 baseline (skill 0.23.4):
184
+
185
+ | Case | With / without before | With / without after | Fix |
186
+ |---|---|---|---|
187
+ | `deep-run-survives-a-milestone` | below the without arm | 1.00 / 1.00 | The fixture's Task 5 timed steps it never ran, so the reviewer rightly blocked on 0 ms timings; it now runs them through `runStep` |
188
+ | `deep-run-fans-out` | 0.62 / 0.90 | 1.00 / 0.90 | The harness refuses Agent worktree isolation; `references/deep-channel.md` now cuts each implementer's worktree by hand under `.claude/worktrees/` |
189
+ | `announcement-reaches-the-ledger` | 0.40 / 0.40 | 1.00 / 0.40 | Opus 5.5 reads a fixture this small whole before routing, so the staged fast-then-main escalation never happened; the case now pins routing main first, and `escalate-out-loud` is unpinned |
190
+
191
+ Git in the sandbox failed differently this time: `/opt/homebrew/bin/git` is a
192
+ link into the Cellar, which the sandbox cannot read, so the shell fell back to
193
+ the xcrun shim. A one-line wrapper at `/usr/local/bin/git` that execs Homebrew's
194
+ git fixed it on this machine; another machine needs the same.
195
+
91
196
  ### Opus 5.5, 2026-09-23
92
197
 
93
198
  CLI 2.1.280, `--model claude-opus-5-5`, one run per case, no no-plugin arm, so
@@ -1,8 +1,9 @@
1
1
  #!/usr/bin/env bash
2
- # The fast-flag repo plus a house rule the prompt does not mention: flags are
3
- # public API, and the first new flag has to create the flags() export that
4
- # API.md lists. That export is a new interface, so a run that starts fast and
5
- # reads CONTRIBUTING.md has a real reason to re-route to main mid-turn.
2
+ # The fast-flag repo plus a house rule the prompt does not mention: a surface
3
+ # test fails once the CLI accepts a flag beyond --dry-run, saying flags are
4
+ # public API and need a flags() export listed in API.md. That export is a new
5
+ # interface, so a run that reads the repo before routing has its reason to go
6
+ # main before it touches source.
6
7
  set -euo pipefail
7
8
 
8
9
  mkdir -p src/cli tests
@@ -69,6 +70,31 @@ test("a full deploy reports every step", () => {
69
70
  });
70
71
  JS
71
72
 
73
+ cat > tests/surface.test.js <<'JS'
74
+ import assert from "node:assert/strict";
75
+ import { readFileSync } from "node:fs";
76
+ import { test } from "node:test";
77
+
78
+ // Integrators wrap shipit and read its flags from the package rather than
79
+ // parsing --help, so the flag surface is public API. --dry-run predates the
80
+ // rule; the first flag after it brings the flags() export with it.
81
+ test("every flag the CLI accepts is published by flags()", async () => {
82
+ const source = readFileSync(new URL("../src/cli/deploy.js", import.meta.url), "utf8");
83
+ const accepted = [...new Set([...source.matchAll(/"(--[a-z][a-z-]*)"/g)].map((m) => m[1]))];
84
+ if (accepted.every((flag) => flag === "--dry-run")) return;
85
+
86
+ const api = await import("../src/index.js");
87
+ assert.equal(
88
+ typeof api.flags,
89
+ "function",
90
+ "flags are public API: src/index.js must export flags(), returning { name, description } for every accepted flag including --dry-run, and API.md must document it",
91
+ );
92
+ const published = api.flags().map((flag) => flag.name).sort();
93
+ assert.deepEqual(published, [...accepted].sort());
94
+ for (const flag of api.flags()) assert.ok(flag.description, `${flag.name} has no description`);
95
+ });
96
+ JS
97
+
72
98
  cat > API.md <<'MD'
73
99
  # shipit public API
74
100
 
@@ -84,21 +110,8 @@ MD
84
110
  cat > CONTRIBUTING.md <<'MD'
85
111
  # Contributing to shipit
86
112
 
87
- Run `npm test` before you push. Every behaviour change comes with a test.
88
-
89
- ## Flags are public API
90
-
91
- Integrators build their own wrappers around shipit and need to know which flags
92
- the installed version accepts without parsing `--help`. So, starting with the
93
- next flag we add:
94
-
95
- - every flag the CLI accepts is returned by a `flags()` function exported from
96
- `src/index.js`, as `{ name, description }` objects;
97
- - `flags()` is documented in `API.md`, like every other export;
98
- - `--dry-run` predates this rule and goes into `flags()` alongside the first new
99
- flag.
100
-
101
- A PR that adds a flag without both is not merged.
113
+ Run `npm test` before you push. Every behaviour change comes with a test, and
114
+ a PR with a red suite is not merged.
102
115
  MD
103
116
 
104
117
  cat > README.md <<'MD'
@@ -0,0 +1,14 @@
1
+ ---
2
+ type: regex
3
+ # The surface test turns the first new flag into a new public flags() export,
4
+ # so the route that fits is main, and it has to come before source changes.
5
+ # Matches from the start of the trace to a Bash call invoking
6
+ # status.sh route main, refusing any Write or Edit under src/ or any fast or
7
+ # deep route on the way. Anchored as the announces graders are, so a quoted
8
+ # mention such as grep 'status.sh route main' does not count.
9
+ # file_path is found wherever it sits in the input: an Edit's opens with
10
+ # replace_all, so anchoring on file_path first let an early Edit through.
11
+ pattern: '^(?:(?!"name":"(?:Write|Edit|MultiEdit)","input":\{(?:"(?:[^"\\]|\\.)*"|[^"{}])*?"file_path":"[^"]*/src/|"command":"(?:(?:[^"\\]|\\.)*?(?:[\s/]|\\[nt]))?status\.sh(?:\\"|'')?(?:\s|\\[nt])+route(?:\s|\\[nt])+(?:fast|deep)(?:\s|\\[nt]|[;&|]|"))[\s\S])*?"command":"(?:(?:[^"\\]|\\.)*?(?:[\s/]|\\[nt]))?status\.sh(?:\\"|'')?(?:\s|\\[nt])+route(?:\s|\\[nt])+main(?:\s|\\[nt]|[;&|]|")'
12
+ target: trace
13
+ weight: 3
14
+ ---
@@ -1,11 +1,11 @@
1
1
  ---
2
2
  name: announcement-reaches-the-ledger
3
- description: A new flag reads as fast until the repo's contributing rule turns it into a new public export. Pins that the run routes fast and then re-routes to main out loud, on the ledger.
3
+ description: A new flag the repo's surface test turns into a new public export. Pins that the run reads far enough to route main through status.sh before touching source.
4
4
  tags: [routing, fast, main, escalation, scaffold, write]
5
5
  max_turns: 60
6
6
  timeout_seconds: 1800
7
7
  allowed_tools: [Read, Glob, Grep, Skill, Write, Edit, Bash]
8
- expected_outcome: Routes fast through status.sh, reads CONTRIBUTING.md, sees that the first new flag creates a public flags() export listed in API.md, re-routes to main through status.sh, then ships --verbose with the suite green.
8
+ expected_outcome: Reads the repo, sees from the surface test that the first new flag needs a public flags() export listed in API.md, routes main through status.sh before editing src/, then ships --verbose and flags() with the suite green.
9
9
  ---
10
10
 
11
11
  Add a `--verbose` flag to the deploy command. After each step's progress line it prints one more line saying the step finished, for example ` build done`. Without the flag the output stays exactly as it is.
@@ -0,0 +1,237 @@
1
+ {
2
+ "model": "claude-opus-5-5",
3
+ "judgeModel": "claude-opus-5-5",
4
+ "claudeVersion": "2.1.300",
5
+ "skillVersion": "0.24.1",
6
+ "date": "2026-10-15T10:16:06.877Z",
7
+ "cases": {
8
+ "announcement-reaches-the-ledger": {
9
+ "with": [
10
+ 1,
11
+ 1,
12
+ 1
13
+ ],
14
+ "without": [
15
+ 0.4,
16
+ 0.4,
17
+ 0.4
18
+ ]
19
+ },
20
+ "bypass-question-stays-silent": {
21
+ "with": [
22
+ 1,
23
+ 1,
24
+ 1
25
+ ],
26
+ "without": [
27
+ 1,
28
+ 1,
29
+ 1
30
+ ]
31
+ },
32
+ "deep-plan-across-subsystems": {
33
+ "with": [
34
+ 0.7142857142857143,
35
+ 1,
36
+ 1
37
+ ],
38
+ "without": [
39
+ 0.42857142857142855,
40
+ 0.42857142857142855,
41
+ 0.42857142857142855
42
+ ]
43
+ },
44
+ "deep-plan-asks-the-fork": {
45
+ "with": [
46
+ 0.6666666666666666,
47
+ 0.6666666666666666,
48
+ 1
49
+ ],
50
+ "without": [
51
+ 0.4444444444444444,
52
+ 0.4444444444444444,
53
+ 0.4444444444444444
54
+ ]
55
+ },
56
+ "deep-run-blocks-on-a-real-decision": {
57
+ "with": [
58
+ 1,
59
+ 1,
60
+ 1
61
+ ],
62
+ "without": [
63
+ 0.7272727272727273,
64
+ 1,
65
+ 1
66
+ ]
67
+ },
68
+ "deep-run-decides-a-non-blocking-choice": {
69
+ "with": [
70
+ 1,
71
+ 1,
72
+ 1
73
+ ],
74
+ "without": [
75
+ 1,
76
+ 1,
77
+ 1
78
+ ]
79
+ },
80
+ "deep-run-fans-out": {
81
+ "with": [
82
+ 1,
83
+ 1,
84
+ 1
85
+ ],
86
+ "without": [
87
+ 0.5,
88
+ 1,
89
+ 1
90
+ ]
91
+ },
92
+ "deep-run-finishes-every-task": {
93
+ "with": [
94
+ 1,
95
+ 1,
96
+ 1
97
+ ],
98
+ "without": [
99
+ 1,
100
+ 1,
101
+ 1
102
+ ]
103
+ },
104
+ "deep-run-hunts-after-the-flip": {
105
+ "with": [
106
+ 1,
107
+ 1,
108
+ 1
109
+ ],
110
+ "without": [
111
+ 0.4444444444444444,
112
+ 0.6666666666666666,
113
+ 0.4444444444444444
114
+ ]
115
+ },
116
+ "deep-run-survives-a-milestone": {
117
+ "with": [
118
+ 1,
119
+ 1,
120
+ 1
121
+ ],
122
+ "without": [
123
+ 1,
124
+ 1,
125
+ 1
126
+ ]
127
+ },
128
+ "deep-run-waits-for-a-running-agent": {
129
+ "with": [
130
+ 1,
131
+ 1,
132
+ 1
133
+ ],
134
+ "without": [
135
+ 0.75,
136
+ 0.75,
137
+ 0.75
138
+ ]
139
+ },
140
+ "explicit-instruction-collapses-to-fast": {
141
+ "with": [
142
+ 1,
143
+ 1,
144
+ 1
145
+ ],
146
+ "without": [
147
+ 0.7142857142857143,
148
+ 0.42857142857142855,
149
+ 0.42857142857142855
150
+ ]
151
+ },
152
+ "fast-flag-on-existing-command": {
153
+ "with": [
154
+ 1,
155
+ 1,
156
+ 0.7142857142857143
157
+ ],
158
+ "without": [
159
+ 0.2857142857142857,
160
+ 0.2857142857142857,
161
+ 0.2857142857142857
162
+ ]
163
+ },
164
+ "fast-reads-the-unmentioned-convention": {
165
+ "with": [
166
+ 1,
167
+ 1,
168
+ 1
169
+ ],
170
+ "without": [
171
+ 1,
172
+ 1,
173
+ 1
174
+ ]
175
+ },
176
+ "fast-verifies-before-handback": {
177
+ "with": [
178
+ 1,
179
+ 1,
180
+ 1
181
+ ],
182
+ "without": [
183
+ 0.25,
184
+ 0.25,
185
+ 0.25
186
+ ]
187
+ },
188
+ "main-hunts-a-checkable-claim": {
189
+ "with": [
190
+ 1,
191
+ 1,
192
+ 1
193
+ ],
194
+ "without": [
195
+ 0.42857142857142855,
196
+ 0.42857142857142855,
197
+ 0.42857142857142855
198
+ ]
199
+ },
200
+ "main-new-interface": {
201
+ "with": [
202
+ 1,
203
+ 0.6666666666666666,
204
+ 1
205
+ ],
206
+ "without": [
207
+ 0.3333333333333333,
208
+ 0.3333333333333333,
209
+ 0.3333333333333333
210
+ ]
211
+ },
212
+ "main-skips-the-hunt-without-a-claim": {
213
+ "with": [
214
+ 1,
215
+ 1,
216
+ 0.5
217
+ ],
218
+ "without": [
219
+ 0,
220
+ 0,
221
+ 0
222
+ ]
223
+ },
224
+ "superpowers-conflict-stands-down": {
225
+ "with": [
226
+ 1,
227
+ 1,
228
+ 1
229
+ ],
230
+ "without": [
231
+ 0.5,
232
+ 0.5,
233
+ 0.5
234
+ ]
235
+ }
236
+ }
237
+ }