@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +17 -3
  7. package/docs/codex-host.md +155 -49
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +141 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +307 -18
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +63 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +206 -0
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +13 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +133 -115
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/evaluation/types.ts +207 -0
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +15 -76
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/fixtures/workspace-state.ts +212 -0
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +57 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +91 -14
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +46 -16
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +122 -73
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -54
  158. package/src/hosts/codex-profile.ts +50 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +141 -19
  161. package/src/hosts/codex.ts +221 -179
  162. package/src/hosts/isolation-common.ts +15 -11
  163. package/src/hosts/isolation-roots.ts +13 -20
  164. package/src/hosts/linux-sandbox.ts +27 -3
  165. package/src/hosts/mac-sandbox.ts +111 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +221 -45
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +34 -15
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/run-owner.ts +4 -4
  186. package/src/{runtime-config.ts → runtime/config.ts} +182 -44
  187. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  188. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  189. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  190. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  191. package/src/{runtime-hooks.ts → runtime/hooks.ts} +33 -22
  192. package/src/{runtime-paths.ts → runtime/paths.ts} +6 -18
  193. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  194. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  195. package/src/runtime/shared-cache.ts +130 -0
  196. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  197. package/src/schema-types.ts +73 -161
  198. package/src/shell.ts +4 -0
  199. package/src/transcript-retention.ts +146 -0
  200. package/src/value-guards.ts +22 -0
  201. package/docs/assets/sevro-logo.png +0 -0
  202. package/docs/assets/sevro-symbol.svg +0 -1
  203. package/docs/assets/sevro-wordmark.svg +0 -1
  204. package/docs/cache-ownership-validation.md +0 -49
  205. package/docs/claude-continuation-validation.md +0 -47
  206. package/docs/concurrency-validation.md +0 -86
  207. package/docs/configuration-root-validation.md +0 -70
  208. package/docs/engine-slice.md +0 -114
  209. package/docs/evidence/runtime-startup-30/README.md +0 -185
  210. package/docs/evidence/runtime-startup-30/after.json +0 -319
  211. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  212. package/docs/evidence/runtime-startup-30/before.json +0 -403
  213. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  214. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  216. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  217. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  218. package/docs/evidence.md +0 -17
  219. package/docs/fixture-cleanup-validation.md +0 -70
  220. package/docs/guide-verification.json +0 -239
  221. package/src/evaluation-context.ts +0 -257
  222. package/src/evaluation-fixture.ts +0 -313
  223. package/src/evaluation-run.ts +0 -198
  224. package/src/evaluation-trial.ts +0 -946
  225. package/src/evaluation-types.ts +0 -149
  226. package/src/hosts/claude-nested-skills.ts +0 -272
  227. package/src/hosts/codex-command-output.ts +0 -510
  228. package/src/hosts/codex-executor-command.ts +0 -291
  229. package/src/hosts/codex-skill-reads.ts +0 -609
  230. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
@@ -0,0 +1,489 @@
1
+ # Run complete evaluation recipes
2
+
3
+ Copy a recipe, run its positive and negative controls, then select a native
4
+ model. The complete projects live in [examples/recipes](../examples/recipes).
5
+ They include cases, fixtures, evaluator checks, runtime files, adapters, and a
6
+ small extension.
7
+
8
+ ## Prepare a recipe project
9
+
10
+ Use Bun 1.3.13, Git, Node 24.13.0, and `rg`. Shell checks need macOS
11
+ `sandbox-exec` or Ubuntu `bubblewrap`; native Claude also needs `socat` on
12
+ Ubuntu. Python/UV and mise add the prerequisites stated below.
13
+ [Install Sevro](installing.md), then set the recipe path from the directory
14
+ containing its local installation:
15
+
16
+ ```sh
17
+ export SEVRO_RECIPES="$PWD/node_modules/@bjoernrochel/sevro/examples/recipes"
18
+ ```
19
+
20
+ Run commands below from an otherwise empty working directory to keep generated
21
+ outputs together. The launcher uses
22
+ the CLI beside its original package; it does not require a global installation.
23
+
24
+ To adapt a separate copy, preserve the small project's whole directory:
25
+
26
+ ```sh
27
+ export SEVRO_RECIPE_CLI="$(cd "$SEVRO_RECIPES/../../src" && pwd)/cli.ts"
28
+ cp -R "$SEVRO_RECIPES" ./my-recipes
29
+ export SEVRO_RECIPES="$PWD/my-recipes"
30
+ ```
31
+
32
+ Each command saves `result.json`, the materialized case or extension command,
33
+ and retained evidence under `.sevro-recipes/<recipe>/<route>/<mode>/` in your
34
+ current directory. The project snapshot is separate from recipe inputs and
35
+ results. Stderr names the exact result path and effective settings; stdout is
36
+ one `sevro.cli-result.v1` JSON document. Follow its absolute `evidencePath`.
37
+ The launcher supplies explicit source, project, config, runtime, isolation,
38
+ condition, trial and threshold options; inspect [run.ts](../examples/recipes/run.ts)
39
+ and the saved inputs when adapting a plain CLI invocation. Existing results
40
+ remain until you remove their directory.
41
+
42
+ For every deterministic positive control, expect exit `0`, execution
43
+ `completed`, grading `completed`, task `passed`. Each separate `--fail` command
44
+ deliberately exits `1` with execution/grading `completed` and task `failed`.
45
+ Run negative controls separately when using a shell with `set -e`. A missing
46
+ tool is a prerequisite error, not a passing or intentionally failing control.
47
+
48
+ `candidate.ts` is an explicitly trusted deterministic fixture editor. It changes
49
+ the arithmetic implementation for the coding recipe and supplies a fixed answer
50
+ for the other controls. It does not simulate native tool permissions: the real
51
+ isolated shell graders execute the stated tools and assertions. Native routes
52
+ run the actual task with the same fixture and runtime policy.
53
+
54
+ For any recipe that offers a deterministic control, replace `deterministic`
55
+ with `codex` or `claude` to use a real participant. First install/authenticate
56
+ the selected [native CLI](native-hosts.md). For Codex, set the model to an
57
+ available identifier and supply your private authentication file path:
58
+
59
+ ```sh
60
+ export SEVRO_RECIPE_MODEL='replace-with-an-available-Codex-model'
61
+ export SEVRO_RECIPE_CODEX_AUTH="${CODEX_HOME:-$HOME/.codex}/auth.json"
62
+ bun "$SEVRO_RECIPES/run.ts" coding-change codex --dry
63
+ bun "$SEVRO_RECIPES/run.ts" coding-change codex
64
+ ```
65
+
66
+ For Claude, use its supported credential route and a Claude model identifier:
67
+
68
+ ```sh
69
+ export SEVRO_RECIPE_MODEL='replace-with-an-available-Claude-model'
70
+ bun "$SEVRO_RECIPES/run.ts" coding-change claude --dry
71
+ bun "$SEVRO_RECIPES/run.ts" coding-change claude
72
+ ```
73
+
74
+ Substitute the other recipe name to run its native task. `--dry` prepares and
75
+ retains inputs without calling a model; remove it for execution. Add `--fail`
76
+ to a live command for its deliberate negative control. Native model outcomes
77
+ must be observed; a model can ignore a negative-control instruction. The
78
+ semantic recipe additionally requires its judge route described below.
79
+
80
+ ## Fix a coding bug with hidden evaluator checks
81
+
82
+ Files: [coding-change case](../examples/recipes/cases/coding-change.json),
83
+ [typed evaluator test](../examples/recipes/coding-change/check.ts),
84
+ [tool configuration](../examples/recipes/runtime/path.json), and
85
+ [candidate](../examples/recipes/candidate.ts). Prerequisite: Node on `PATH`.
86
+ The fixture contains a broken `math.cjs` and a task README. The task asks the
87
+ participant to implement addition. The evaluator checks positive, negative and
88
+ zero arguments. Its source stays outside the candidate fixture: the launcher
89
+ transpiles the checked TypeScript test and embeds its JavaScript in the
90
+ evaluator-owned shell command before execution.
91
+
92
+ <!-- sevro-example:recipe-coding-change-pass -->
93
+
94
+ ```sh
95
+ bun "$SEVRO_RECIPES/run.ts" coding-change deterministic
96
+ ```
97
+
98
+ The isolated check first asserts that the original evaluator test is unreadable,
99
+ then runs Node against the changed fixture. The passing shell observation
100
+ retains the result and stdout digest. The negative control leaves subtraction
101
+ in place, so the arithmetic assertion fails:
102
+
103
+ <!-- sevro-example:recipe-coding-change-fail -->
104
+
105
+ ```sh
106
+ bun "$SEVRO_RECIPES/run.ts" coding-change deterministic --fail
107
+ ```
108
+
109
+ Use a native route below to evaluate an actual coding agent. Adding a participant
110
+ test to the fixture would make it visible; it would not replace this hidden
111
+ evaluator-owned check. The launcher selects `--shell-isolation` and protects
112
+ the recipe source; declare additional private worktrees through the
113
+ [CLI's protected roots](development-cli.md).
114
+
115
+ ## Add host-tool access, one capability at a time
116
+
117
+ The smallest useful runtime file is
118
+ [runtime/empty.json](../examples/recipes/runtime/empty.json):
119
+
120
+ ```json
121
+ { "format": "sevro.runtime.v1" }
122
+ ```
123
+
124
+ It selects the existing isolation defaults without importing tools or hooks.
125
+ Add only a declared non-secret `PATH` to reuse installed tools:
126
+
127
+ ```json
128
+ {
129
+ "format": "sevro.runtime.v1",
130
+ "environment": { "inherit": ["PATH"] }
131
+ }
132
+ ```
133
+
134
+ Sevro filters credential-related PATH entries, discovers bounded installation
135
+ support trees, grants read-only tool access, and keeps private writable home
136
+ and temporary directories. It does not grant the full operator home. Runtime
137
+ values enter retained evidence; do not put secrets in this file. See
138
+ [runtime details](runtime-v1.md#environment-and-host-tools).
139
+
140
+ ### Minimal tool: ripgrep
141
+
142
+ Files: [host-tools case](../examples/recipes/cases/host-tools.json) and
143
+ [PATH runtime](../examples/recipes/runtime/path.json). Prerequisite: standalone `rg` on
144
+ `PATH`, installed outside Codex/Claude credential directories (for example,
145
+ `brew install ripgrep` on macOS or `apt-get install ripgrep` on Ubuntu).
146
+ Sevro deliberately filters a Codex session's injected `~/.codex/.../rg` path;
147
+ that embedded executable alone does not satisfy this recipe. The fixture's `app.env` contains `port=3000`; an actual isolated `rg`
148
+ process must find exactly that line.
149
+
150
+ <!-- sevro-example:recipe-host-tools-pass -->
151
+
152
+ ```sh
153
+ bun "$SEVRO_RECIPES/run.ts" host-tools deterministic
154
+ ```
155
+
156
+ The negative fixture contains port `9000`, so that same search fails:
157
+
158
+ <!-- sevro-example:recipe-host-tools-fail -->
159
+
160
+ ```sh
161
+ bun "$SEVRO_RECIPES/run.ts" host-tools deterministic --fail
162
+ ```
163
+
164
+ ### Node
165
+
166
+ Files: [Node case](../examples/recipes/cases/node.json) and the same PATH runtime.
167
+ Prerequisite: Node 24 on `PATH`. The fixture contains `app.json`. A real isolated
168
+ Node process parses it, asserts the port, and prints `3000`.
169
+
170
+ <!-- sevro-example:recipe-node-pass -->
171
+
172
+ ```sh
173
+ bun "$SEVRO_RECIPES/run.ts" node deterministic
174
+ ```
175
+
176
+ The negative fixture sets port `9000`, causing the Node assertion to fail:
177
+
178
+ <!-- sevro-example:recipe-node-fail -->
179
+
180
+ ```sh
181
+ bun "$SEVRO_RECIPES/run.ts" node deterministic --fail
182
+ ```
183
+
184
+ ### Python with UV
185
+
186
+ Files: [Python/UV case](../examples/recipes/cases/python-uv.json) and
187
+ [UV runtime](../examples/recipes/runtime/python-uv.json). Install `uv` and an
188
+ existing Python 3 first; CI uses UV `0.11.21`. Both executables must be on
189
+ `PATH`. The launcher selects `python3`; optionally export `RECIPE_PYTHON` to
190
+ the absolute existing interpreter you want. Put that interpreter's installation
191
+ bin on `PATH` too, so its bounded support installation can be discovered.
192
+
193
+ This adds a private writable `UV_CACHE_DIR` and disables Python downloads.
194
+ `uv run --offline --no-project` runs the selected existing interpreter, with no
195
+ dependency resolution or network. Python parses the fixture JSON, asserts its
196
+ port and prints `3000`:
197
+
198
+ <!-- sevro-example:recipe-python-uv-pass -->
199
+
200
+ ```sh
201
+ bun "$SEVRO_RECIPES/run.ts" python-uv deterministic
202
+ ```
203
+
204
+ The negative fixture changes the port to `9000`; Python raises an assertion:
205
+
206
+ <!-- sevro-example:recipe-python-uv-fail -->
207
+
208
+ ```sh
209
+ bun "$SEVRO_RECIPES/run.ts" python-uv deterministic --fail
210
+ ```
211
+
212
+ For real dependencies, prepare a public cache once outside the evaluated project
213
+ and declare a [shared cache](runtime-v1.md#reuse-a-shared-dependency-cache), then
214
+ use your locked offline install command. Select only the roles using UV; add
215
+ `checks` only when the grader needs it. The same directory can serve candidates,
216
+ hooks, and UV-based graders across trials and runs without copying the cache.
217
+ Use [private seeds](runtime-v1.md#private-writable-state) when isolated dependency
218
+ bytes matter more than disk reuse. This dependency-free recipe needs no cache seed.
219
+
220
+ ### An existing mise installation
221
+
222
+ Files: [mise case](../examples/recipes/cases/mise.json),
223
+ [runtime](../examples/recipes/runtime/mise.json), and the selection in
224
+ [run-options.ts](../examples/recipes/run-options.ts). Install mise first; CI
225
+ uses `2026.3.13`. Prepare its exact tool before evaluation:
226
+
227
+ ```sh
228
+ mise install node@24.13.0
229
+ mise where node@24.13.0
230
+ ```
231
+
232
+ The launcher asks `mise where node@24.13.0`, prepends that installation's `bin`
233
+ to the declared PATH, then runs its Node inside isolation. The evaluator checks
234
+ the exact Node version and fixture port and prints `3000`. This route executes
235
+ the resolved versioned tool; it does not run a mutable mise shim inside the
236
+ sandbox or grant mise's whole data directory.
237
+
238
+ <!-- sevro-example:recipe-mise-pass -->
239
+
240
+ ```sh
241
+ bun "$SEVRO_RECIPES/run.ts" mise deterministic
242
+ ```
243
+
244
+ The negative fixture changes the port to `9000`, failing the same Node check:
245
+
246
+ <!-- sevro-example:recipe-mise-fail -->
247
+
248
+ ```sh
249
+ bun "$SEVRO_RECIPES/run.ts" mise deterministic --fail
250
+ ```
251
+
252
+ An attempted isolated `mise exec` with read-only operator data returned
253
+ `Operation not permitted` on macOS. If your shim needs extra discovery or
254
+ writable state, resolve the existing installation before launch as above.
255
+ Do not solve that failure by granting the operator's whole home.
256
+
257
+ ## Evaluate a repository skill with Codex or Claude
258
+
259
+ Files: the complete [sample repository](../examples/recipes/repository-skill/README.md),
260
+ its [Codex skill](../examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md),
261
+ its [Claude skill](../examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md),
262
+ and the [extension](../examples/recipes/extension.ts). The extension builds a
263
+ fresh committed Git fixture and mounts the selected skill as a Git-excluded
264
+ artifact. The README is participant-visible; checks and extension source stay
265
+ outside. Copy these actual skill bodies and their supporting files when
266
+ substituting your own skill.
267
+
268
+ Install the [native host prerequisites](native-hosts.md). Dry preparation needs
269
+ the selected CLI on PATH, but no model login or quota. It negotiates repository
270
+ invocation, mounts and hashes the actual skill, validates the case and runtime,
271
+ and retains the artifacts. Expect exit `0`, execution `not_run`, grading
272
+ `not_requested`, task `not_assessed`:
273
+
274
+ <!-- sevro-example:recipe-repository-skill-codex-dry -->
275
+
276
+ ```sh
277
+ bun "$SEVRO_RECIPES/run.ts" repository-skill codex --dry
278
+ ```
279
+
280
+ <!-- sevro-example:recipe-repository-skill-claude-dry -->
281
+
282
+ ```sh
283
+ bun "$SEVRO_RECIPES/run.ts" repository-skill claude --dry
284
+ ```
285
+
286
+ For a real Codex trial, export `SEVRO_RECIPE_MODEL` to an available Codex model
287
+ and `SEVRO_RECIPE_CODEX_AUTH` to your private absolute `auth.json` path, then
288
+ remove `--dry`. For Claude, export `SEVRO_RECIPE_MODEL` to an available Claude
289
+ model and authenticate through one of its documented credential routes, then
290
+ remove `--dry`. Credentials stay outside recipe/project inputs and argv.
291
+ These calls consume account quota.
292
+
293
+ ```sh
294
+ bun "$SEVRO_RECIPES/run.ts" repository-skill codex
295
+ bun "$SEVRO_RECIPES/run.ts" repository-skill codex --fail
296
+ ```
297
+
298
+ With `SEVRO_RECIPE_MODEL` set for Claude instead:
299
+
300
+ ```sh
301
+ bun "$SEVRO_RECIPES/run.ts" repository-skill claude
302
+ bun "$SEVRO_RECIPES/run.ts" repository-skill claude --fail
303
+ ```
304
+
305
+ The neutral invocation placeholder becomes `$startup-guide` for Codex and
306
+ `/startup-guide` at the start of Claude's prompt. Claude uses the required
307
+ `--claude-project-settings`; its runtime leaves goals/hooks disabled. Passing
308
+ requires answer content (command and port), the prepared-file assertion, and a
309
+ complete explicit-invocation receipt. That receipt proves the bounded dispatch,
310
+ not general grounding or quality across other questions.
311
+
312
+ The deliberate failure is the same command without `--dry`, with `--fail`
313
+ added. It explicitly requests the wrong response; expected task verdict is
314
+ `failed`, exit `1`. An actual model can disregard a negative-control prompt;
315
+ inspect its observed answer and receipt if it unexpectedly passes. Dry
316
+ `--fail` only validates that negative-control preparation; it does not observe
317
+ model failure.
318
+
319
+ ## Add a separate semantic judge
320
+
321
+ Files: [semantic case](../examples/recipes/cases/semantic.json), candidate
322
+ adapter and [protocol judge](../examples/recipes/semantic-judge.ts). The case
323
+ asks for the startup explanation and declares a proposition requiring the
324
+ command and port. These credential-free controls exercise routing, source
325
+ construction, verdict parsing, and retained semantic evidence:
326
+
327
+ <!-- sevro-example:recipe-semantic-pass -->
328
+
329
+ ```sh
330
+ bun "$SEVRO_RECIPES/run.ts" semantic deterministic
331
+ ```
332
+
333
+ <!-- sevro-example:recipe-semantic-fail -->
334
+
335
+ ```sh
336
+ bun "$SEVRO_RECIPES/run.ts" semantic deterministic --fail
337
+ ```
338
+
339
+ The protocol judge derives pass/fail from the actual candidate-response field
340
+ using an exact fixture oracle. It is not a model and does not establish
341
+ qualitative judgment quality. The negative response gives the wrong command
342
+ and port; semantic grading completes with a failing proposition.
343
+
344
+ For an actual model candidate and judge, set the native variables above and
345
+ export `SEVRO_RECIPE_JUDGE_MODEL` to an available Codex judge model:
346
+
347
+ ```sh
348
+ bun "$SEVRO_RECIPES/run.ts" semantic codex --dry
349
+ bun "$SEVRO_RECIPES/run.ts" semantic codex
350
+ ```
351
+
352
+ `semantic claude` selects a Claude candidate with a separate Codex judge;
353
+ it needs both CLIs and the Codex auth path. A correct response is expected to
354
+ pass; add `--fail` for the deliberately wrong-response task and an expected
355
+ failed verdict. Live verdicts remain observations, not guarantees.
356
+
357
+ This response-only recipe makes one candidate model call and one additional
358
+ judge call per completed trial. The judge receives all its propositions in an
359
+ empty temporary workspace. Separate document sources can add further calls;
360
+ missing candidate output can avoid a judge call. Judge usage and raw structured
361
+ verdicts are retained separately. A malformed or incomplete judge response is
362
+ a grading error, not a failed proposition or a passing task. Read
363
+ [semantic grading](builtin-graders-v1.md) before adapting its oracle.
364
+
365
+ ## Run the complete extension lifecycle
366
+
367
+ Files: [extension.ts](../examples/recipes/extension.ts),
368
+ [methods](../examples/recipes/extension-methods.ts), and
369
+ [artifact preparation](../examples/recipes/extension-artifacts.ts). Prerequisites:
370
+ Bun, Git and the platform's shell isolation tool. No model is required:
371
+
372
+ <!-- sevro-example:recipe-lifecycle-pass -->
373
+
374
+ ```sh
375
+ bun "$SEVRO_RECIPES/run.ts" lifecycle deterministic
376
+ ```
377
+
378
+ `describe` advertises version/capabilities/grader; `resolve` returns the task
379
+ and committed fixture; `prepare` supplies hashed Git-excluded `startup.txt`;
380
+ the candidate runs; an isolated built-in check reads the mounted bytes; then
381
+ `evaluate` checks the complete final-message evidence for port `3000` and cites
382
+ that observation. The default task policy combines these checks. Evidence is
383
+ saved before cleanup. Every extension method runs in a fresh process, so prior
384
+ process globals are not state. Stdout contains only its matching protocol
385
+ response; diagnostics belong on stderr.
386
+
387
+ <!-- sevro-example:recipe-lifecycle-fail -->
388
+
389
+ ```sh
390
+ bun "$SEVRO_RECIPES/run.ts" lifecycle deterministic --fail
391
+ ```
392
+
393
+ The wrong final answer fails the built-in output assertion and extension check,
394
+ while preparation's shell check still passes. The extension cannot turn missing
395
+ evidence or a failed exchange into success. Inspect the saved argv command and
396
+ source closure, including its skill resources, to adapt it into your own
397
+ [trusted extension](extension-protocol-v1.md).
398
+
399
+ ## Pair plugins, goals, and hooks with preparation
400
+
401
+ Files: the same extension and artifact builder, plus
402
+ [goals runtime](../examples/recipes/runtime/goals.json) and
403
+ [selected-plugin runtime](../examples/recipes/runtime/plugins.json).
404
+ Prerequisites: a native CLI with plugin and native-goal support; use the
405
+ [checked host versions](typescript-quality.md#environment-and-ci). The plugin
406
+ manifests, skill and SessionStart command hook are all prepared artifacts, not
407
+ an undeclared global installation. The Codex package uses a local marketplace;
408
+ Claude uses declared plugin directories. The extension explicitly invokes the
409
+ prepared `startup:startup-guide` skill.
410
+
411
+ First add only native goals to the empty runtime and validate preparation:
412
+
413
+ <!-- sevro-example:recipe-goals-codex-dry -->
414
+
415
+ ```sh
416
+ bun "$SEVRO_RECIPES/run.ts" plugins codex --dry --goals-only
417
+ ```
418
+
419
+ <!-- sevro-example:recipe-goals-claude-dry -->
420
+
421
+ ```sh
422
+ bun "$SEVRO_RECIPES/run.ts" plugins claude --dry --goals-only
423
+ ```
424
+
425
+ Then add `hooks.plugins: ["startup"]` so only this prepared plugin's command
426
+ handlers are authorized. Codex selects app-server for its native goal route;
427
+ Claude uses plugin-only settings, so do not add `--claude-project-settings`:
428
+
429
+ <!-- sevro-example:recipe-plugins-codex-dry -->
430
+
431
+ ```sh
432
+ bun "$SEVRO_RECIPES/run.ts" plugins codex --dry
433
+ ```
434
+
435
+ <!-- sevro-example:recipe-plugins-claude-dry -->
436
+
437
+ ```sh
438
+ bun "$SEVRO_RECIPES/run.ts" plugins claude --dry
439
+ ```
440
+
441
+ All four dry commands expect `not_run`/`not_requested`/`not_assessed`, exit `0`.
442
+ They validate preparation and retain its actual hashed files. They do not
443
+ install a plugin into a live native host, execute its hook, or observe a goal.
444
+ `--goals-only` is deliberately a preparation step; the final recipe's hook
445
+ assertion requires the subsequent full policy.
446
+
447
+ With the native model/credentials configured, remove `--dry` from the full
448
+ plugin command. Expect `passed`, exit `0`: the selected SessionStart hook
449
+ writes `hook-ran.txt`, an isolated shell assertion reads `hook ran`, and the
450
+ skill answer supplies the startup command/port. The hook runs credential-free
451
+ with network denied and only the prepared plugin subtree readable.
452
+ `hooks.nativeGoal` enables the native goal facility; inspect retained
453
+ `sevro.host.native-goal` observations for actual creation/status. Enabling the
454
+ facility or passing this startup task alone does not prove goal completion.
455
+
456
+ ```sh
457
+ bun "$SEVRO_RECIPES/run.ts" plugins codex
458
+ bun "$SEVRO_RECIPES/run.ts" plugins codex --fail
459
+ ```
460
+
461
+ Or, with the Claude model/credential route configured:
462
+
463
+ ```sh
464
+ bun "$SEVRO_RECIPES/run.ts" plugins claude
465
+ bun "$SEVRO_RECIPES/run.ts" plugins claude --fail
466
+ ```
467
+
468
+ For a deliberate failure, run that live command with `--fail`; the requested
469
+ wrong response should fail the answer checks while the hook assertion passes.
470
+ Removing `startup` from the runtime's allowed plugins is a separate diagnostic:
471
+ the command hook is filtered and the hook-file assertion fails. Native/model
472
+ behavior must be observed before claiming either live outcome.
473
+
474
+ ## What tutorial verification proves
475
+
476
+ `bun run test:docs-examples` packs this candidate, copies the packaged recipe
477
+ project to a fresh consumer directory, and executes every marked deterministic
478
+ positive/negative command. It runs real isolated `rg`, Node, UV/Python and
479
+ mise-selected Node checks, the hidden arithmetic test, semantic protocol
480
+ control, and the entire extension lifecycle. Missing UV, Python, mise or its
481
+ declared Node installation fails with an actionable prerequisite error; no
482
+ recipe is silently skipped. CI provisions these tools explicitly.
483
+
484
+ The verifier also dry-prepares every recipe with both native hosts, including
485
+ both skill/plugin variants, negative-control inputs and the goals-only step,
486
+ and verifies retained artifact digests. These are not authenticated model,
487
+ goal, plugin-installation or hook-execution trials. Run the live commands to
488
+ measure those routes. The separate published-consumer check executes the
489
+ installed-package tutorial using the exact current version and license metadata.
package/docs/releases.md CHANGED
@@ -12,6 +12,26 @@ Published `rc.1` and `rc.2` retain their original BSL grants. Always select a ne
12
12
  version for publication; a local development tarball does not replace an
13
13
  already published artifact.
14
14
 
15
+ ## Unreleased: smaller npm package
16
+
17
+ The npm package omits logo images, historical validation and usability pages,
18
+ and raw evidence, including the Darrow migration patch and old guide verification
19
+ record. All remain in the repository; packaged pages link to their online copies.
20
+ The current `LICENSE`, contribution terms, branding permissions and retained
21
+ earlier-release license text still ship. No licensing or branding grants change.
22
+
23
+ For this cleanup, `npm pack --dry-run --ignore-scripts --json` measured the
24
+ development tarball at about 0.39 MB compressed, down from 3,545,849 bytes, and
25
+ about 1.86 MB unpacked, down from 6,632,891 bytes (about 89% and 72% smaller).
26
+ These are candidate measurements, not a newly published release.
27
+
28
+ `runnerBuildDigest` hashes packaged paths and bytes under `docs/` as well as the
29
+ runtime and other package inputs. This packaging change therefore causes a
30
+ one-time digest change even without runner behavior changes. Existing evidence
31
+ keeps its original identity; exact-repeat comparisons across this boundary
32
+ will differ on `runnerBuildDigest`. The identity format and hashing rules stay
33
+ unchanged.
34
+
15
35
  ## Candidate contract
16
36
 
17
37
  Prepare one candidate from the intended release tag and package metadata. The
@@ -51,9 +71,11 @@ with their own CI before starting the release.
51
71
  2. Merge the release PR after its static checks and reused CI evidence pass.
52
72
  Release CI compares the candidate against a successful ancestor run on
53
73
  `main`, permitting only version/distribution-tag changes and the exact
54
- release documentation and workflow files inventoried in
55
- [the CI selector](../.github/scripts/release-ci.py). All other tracked inputs
56
- must match. The original run must have completed the full quality gate;
74
+ release documentation files inventoried in
75
+ [the CI selector](../.github/scripts/release-ci.py). Changes to the CI workflow
76
+ or selector require the full quality gate. All other tracked inputs must
77
+ match. The original run must have completed all four quality phases
78
+ and their gate steps, or the historical complete quality gate;
57
79
  another release's reused status is insufficient. CI retains the source SHA
58
80
  and run URL in `release-ci.json`. Missing evidence fails the release check;
59
81
  it does not trigger another test run. Ordinary code changes and explicit
@@ -104,25 +126,22 @@ retained checksum immediately before publication:
104
126
  ```sh
105
127
  cd /absolute/reviewed
106
128
  shasum -a 256 -c SHA256SUMS
107
- npm publish ./bjoernrochel-sevro-0.1.0-rc.1.tgz --ignore-scripts --access public --tag latest
129
+ npm publish "./bjoernrochel-sevro-${SEVRO_RELEASE_VERSION}.tgz" --ignore-scripts --access public --tag latest
108
130
  ```
109
131
 
110
- Replace the example version and distribution tag with the reviewed candidate's
111
- values. The package repository URL must match this public repository. Preparing
112
- a tarball or opening a pull request does not authorize publication.
132
+ Replace the distribution tag with the reviewed candidate's value. The package
133
+ repository URL must match this public repository. Preparing a tarball or
134
+ opening a pull request does not authorize publication.
113
135
 
114
136
  An existing published candidate can become the default without republishing:
115
137
 
116
138
  ```sh
117
- npm dist-tag add @bjoernrochel/sevro@0.1.0-rc.2 latest
139
+ npm dist-tag add "@bjoernrochel/sevro@${SEVRO_RELEASE_VERSION}" latest
118
140
  npm view @bjoernrochel/sevro dist-tags --json
119
141
  ```
120
142
 
121
143
  Promotion changes the registry tag; it does not update the published tarball.
122
144
 
123
- The first published candidate was `0.1.0-rc.1`, using the owner-selected
124
- `BUSL-1.1` license. Its `LICENSE` preserves Darrow's terms and parameters,
125
- changing only the Licensed Work name and description to Sevro. Local checks do
126
- not claim a remote CI run or publication. Darrow's exact
145
+ Local checks do not claim a remote CI run or publication. Darrow's exact
127
146
  registry pin, default caller switch, remaining workflow migrations, focused live validation, and
128
147
  generic runner removal remain separate extraction gates.
package/docs/report-v1.md CHANGED
@@ -3,10 +3,14 @@
3
3
  `sevro report` reads one or more retained `sevro.cli-result.v1` files and their
4
4
  referenced `sevro.run-evidence.v1` files. It validates both public schemas and
5
5
  checks run identity, exit category, case count, and trial references before
6
- reporting. Paths must be absolute and distinct.
6
+ reporting. Programmatic `createReport` paths must be absolute and distinct.
7
+ CLI `--result-file` paths may be relative to the invoking directory; they are
8
+ canonicalized before duplicate detection and loading.
7
9
 
8
10
  ```sh
9
11
  sevro report --result-file /absolute/path/to/result.json
12
+ sevro report --latest
13
+ sevro report --latest --results-root ./other-results
10
14
  sevro report --json --result-file /absolute/path/to/first.json \
11
15
  --result-file /absolute/path/to/second.json
12
16
  ```
@@ -17,6 +21,30 @@ inputs return exit 64 with a diagnostic on stderr. Reporting does not change
17
21
  the exit category of any input run; each row retains its original code.
18
22
  Pre-run failures with no case rows remain visible in the input-run section.
19
23
 
24
+ `--latest` selects
25
+ `<cwd>/.sevro/results/latest.json`; `--results-root` changes that root and requires
26
+ `--latest`. It is exclusive with `--result-file`. Each retained CLI evaluation
27
+ saves a `result.json` beside its run evidence and atomically updates latest,
28
+ including dry and failed runs. Pre-run failures do not replace latest. Missing
29
+ latest or referenced evidence is an error. See
30
+ [the evaluation workflow](evaluation-workflow.md).
31
+
32
+ [`sevro prune`](reading-results.md#prune-retained-results) maintains this index
33
+ when removing retained evidence. After pruning, latest selects the newest
34
+ remaining completed result; if none remain, the index is removed and `--latest`
35
+ reports a missing-input diagnostic. Explicit reports still require their selected
36
+ results and evidence to exist.
37
+
38
+ After [`sevro prune --native-transcripts`](reading-results.md#remove-native-transcript-bundles),
39
+ reports accept absent native bundles only with the matching
40
+ [pruning receipt](results-v1.md#native-transcript-bundles). Present bundles must
41
+ still match their recorded hashes; unexplained missing bytes remain an error.
42
+ New report rows include optional `nativeTranscripts` with nonnegative integer
43
+ `retained` and `pruned` bundle counts across the case's trials and host roles.
44
+ Markdown shows the same counts beside retained evidence. Older report documents
45
+ without this field remain valid. Pruning does not change verdicts, routes,
46
+ identity, usage, or duration.
47
+
20
48
  Each case row keeps task verdict, execution, and grading separate. Task pass
21
49
  rate is measured only when every trial completed execution and grading with an
22
50
  assessed task verdict. Candidate duration sums the measured host call time