@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +9 -2
  7. package/docs/codex-host.md +155 -57
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +122 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +255 -22
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +43 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +42 -11
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +9 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +50 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +46 -9
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +28 -15
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +121 -81
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -183
  158. package/src/hosts/codex-profile.ts +36 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +134 -18
  161. package/src/hosts/codex.ts +210 -183
  162. package/src/hosts/isolation-common.ts +8 -11
  163. package/src/hosts/isolation-roots.ts +2 -2
  164. package/src/hosts/linux-sandbox.ts +15 -2
  165. package/src/hosts/mac-sandbox.ts +87 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +216 -48
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +26 -11
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/{runtime-config.ts → runtime/config.ts} +160 -46
  186. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  187. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  188. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  189. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  190. package/src/{runtime-hooks.ts → runtime/hooks.ts} +32 -22
  191. package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
  192. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  193. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  194. package/src/runtime/shared-cache.ts +130 -0
  195. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  196. package/src/schema-types.ts +73 -161
  197. package/src/shell.ts +4 -0
  198. package/src/transcript-retention.ts +146 -0
  199. package/src/value-guards.ts +22 -0
  200. package/docs/assets/sevro-logo.png +0 -0
  201. package/docs/assets/sevro-symbol.svg +0 -1
  202. package/docs/assets/sevro-wordmark.svg +0 -1
  203. package/docs/cache-ownership-validation.md +0 -49
  204. package/docs/claude-continuation-validation.md +0 -47
  205. package/docs/concurrency-validation.md +0 -86
  206. package/docs/configuration-root-validation.md +0 -70
  207. package/docs/engine-slice.md +0 -114
  208. package/docs/evidence/runtime-startup-30/README.md +0 -185
  209. package/docs/evidence/runtime-startup-30/after.json +0 -319
  210. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  211. package/docs/evidence/runtime-startup-30/before.json +0 -403
  212. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  213. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  214. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  216. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  217. package/docs/evidence.md +0 -17
  218. package/docs/fixture-cleanup-validation.md +0 -73
  219. package/docs/guide-verification.json +0 -239
  220. package/src/evaluation-context.ts +0 -259
  221. package/src/evaluation-fixture.ts +0 -302
  222. package/src/evaluation-run.ts +0 -200
  223. package/src/evaluation-trial.ts +0 -1016
  224. package/src/hosts/claude-nested-skills.ts +0 -272
  225. package/src/hosts/codex-command-output.ts +0 -510
  226. package/src/hosts/codex-executor-command.ts +0 -291
  227. package/src/hosts/codex-skill-reads.ts +0 -783
  228. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
@@ -1,8 +1,7 @@
1
1
  # Runtime configuration v1
2
2
 
3
- Configure access to existing host tools, private writable caches, native goals,
4
- and selected plugin hooks in `<project-root>/sevro.json`. This interface is
5
- available in `0.1.0-rc.3` and later.
3
+ Configure access to existing host tools, private state, reusable caches, native goals,
4
+ and selected plugin hooks in `<project-root>/sevro.json`.
6
5
 
7
6
  `--runtime-config-file /absolute/path/runtime.json` selects a replacement file.
8
7
  Sevro reads exactly one file before execution, including dry runs. It does not
@@ -14,6 +13,17 @@ non-symlink files of at most 64 KiB.
14
13
 
15
14
  The [JSON schema](../schemas/runtime-v1.schema.json) has format
16
15
  `sevro.runtime.v1`. All properties except `format` are optional.
16
+ Start with the smallest file:
17
+
18
+ ```json
19
+ { "format": "sevro.runtime.v1" }
20
+ ```
21
+
22
+ Then add `environment.inherit: ["PATH"]` for installed tools. Follow the
23
+ [runnable runtime recipes](recipes.md#add-host-tool-access-one-capability-at-a-time)
24
+ to add Node, Python/UV, mise, goals and selected plugin hooks separately.
25
+ The following composed example is for a workflow that already needs all these
26
+ capabilities:
17
27
 
18
28
  ```json
19
29
  {
@@ -26,7 +36,13 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
26
36
  }
27
37
  },
28
38
  "runtime": {
29
- "seedDirectories": [{ "source": "./public-uv-cache", "target": "uv-cache" }]
39
+ "sharedCacheDirectories": [
40
+ {
41
+ "source": "../public-uv-cache",
42
+ "target": "uv-cache",
43
+ "roles": ["candidate", "hooks"]
44
+ }
45
+ ]
30
46
  },
31
47
  "hooks": {
32
48
  "nativeGoal": true,
@@ -35,7 +51,7 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
35
51
  }
36
52
  ```
37
53
 
38
- Create the example's `public-uv-cache` directory beside the configuration, or
54
+ Prepare the example's `public-uv-cache` directory outside the project, or
39
55
  omit `runtime` when no prepared cache is needed. Selected plugin names must be
40
56
  present in the case's prepared mounts; remove names that the case does not mount.
41
57
  `DARROW_CACHE_DIR` is an ordinary explicit variable in this example, not a
@@ -144,8 +160,80 @@ minimal system read tree such as `/usr` or `/etc`, Sevro retains its explicit de
144
160
  the native preflight may refuse that overlapping setup rather than expose home
145
161
  contents.
146
162
 
163
+ ## Local HTTP test servers
164
+
165
+ Sevro supports local HTTP servers in isolated Codex candidate
166
+ commands and `sevro.shell` graders through an explicit operator selection:
167
+
168
+ ```json
169
+ {
170
+ "format": "sevro.runtime.v1",
171
+ "network": { "localHttp": true }
172
+ }
173
+ ```
174
+
175
+ Add this property to the selected `sevro.json`, or select a replacement with
176
+ `--runtime-config-file`. Declare the existing Node/Python installations through
177
+ the environment and filesystem options above. The selection is snapshotted and
178
+ enters evaluation identity. Absent or `false` preserves existing permissions.
179
+ Hooks and semantic/advisory hosts keep their existing network permissions; this
180
+ option applies to Codex candidates and shell grading.
181
+
182
+ On macOS, shell isolation permits IP socket binding, loopback inbound traffic,
183
+ and outbound connections to loopback addresses. Wildcard listeners such as
184
+ Node's `server.listen(0)` work with loopback clients. Other outbound IP traffic
185
+ remains denied. On Linux, shell checks retain their private network namespace
186
+ with usable loopback and no external route.
187
+
188
+ Codex uses its managed command network proxy with `allow_local_binding = true`,
189
+ upstream proxy chaining disabled, and only `localhost`, `127.0.0.1`, and `::1`
190
+ allowlisted. Its native sandbox permits local binding and loopback traffic while
191
+ refusing direct outbound TCP. The native proxy may permit DNS queries; public
192
+ HTTP destinations are not allowlisted. The candidate can reach other services
193
+ on the available loopback interface, so choose this option for evaluations
194
+ that need that access. It does not provide a separate per-trial loopback service
195
+ namespace on macOS.
196
+
197
+ The supported host boundary is the native Codex sandbox and its experimental
198
+ `network_proxy` feature, documented in the [OpenAI permission reference](https://learn.chatgpt.com/docs/permissions).
199
+ Setting `allow_local_binding` while leaving command networking disabled still
200
+ denies `listen()` on macOS. Integration tests execute the installed native
201
+ Codex sandbox, a wildcard HTTP listener and loopback client, and an outbound TCP
202
+ denial probe. Host versions and actual upstream evaluation outcomes belong in
203
+ [validation evidence](https://github.com/BjRo/sevro/blob/main/docs/evidence.md), rather than being inferred from a profile's
204
+ text. An unsupported host must not be interpreted as a passing behavior test.
205
+
206
+ ## macOS ancestor metadata
207
+
208
+ Sevro's macOS outer sandbox for isolated shell checks and authorized command
209
+ hooks permits metadata reads on the exact ancestor paths of existing validated
210
+ grants. This includes declared read-only support trees, private writable
211
+ runtimes, and bounded transcript, candidate artifact, and managed workspace
212
+ access. The permissions are derived automatically from those grants; no
213
+ additional configuration is required. They let libc `realpath()` canonicalize
214
+ paths inside a permitted subtree.
215
+
216
+ Ancestor directory metadata, such as existence, type, ownership, permissions,
217
+ and timestamps, is visible. Directory listings, ancestor file contents, and
218
+ ungranted sibling metadata and contents remain denied under protected roots.
219
+ Writes remain limited by the existing writable grants; read-only grants stay
220
+ read-only. This metadata exception does not grant access to an ancestor subtree
221
+ or change network isolation.
222
+
147
223
  ## Private writable state
148
224
 
225
+ Codex candidate commands can write the isolated fixture's `.git` directory,
226
+ including its index, refs, objects, and task-owned admission locks. Sevro grants
227
+ that exact path explicitly because Codex's workspace permission otherwise keeps
228
+ Git metadata read-only. Real commits and publication to a fixture-local remote
229
+ need no additional runtime option. This grant does not authorize writes to
230
+ source repositories, grading inputs, host authentication or configuration, or
231
+ peer trials. Command network access remains disabled unless the operator selects
232
+ the [local HTTP option](#local-http-test-servers).
233
+ An existing fixture `.git` must be a physical directory. Symlinks and Git
234
+ directory pointer files are refused before launch because their metadata may
235
+ belong to a protected source repository rather than the isolated trial.
236
+
149
237
  `runtime.seedDirectories` copies prepared public dependencies into a private
150
238
  slot for each execution role. `source` uses the filesystem path rules above;
151
239
  `target` is a single lowercase slot name, up to 64 characters, with letters,
@@ -153,10 +241,12 @@ digits, and hyphens. `home` and `tmp` are reserved. Targets must be unique.
153
241
  An optional nonempty `roles` array selects any of `candidate`, `checks`, `hooks`,
154
242
  `semantic`, and `advisory`. For example, `"roles": ["candidate", "checks", "hooks"]`
155
243
  keeps dependency slots out of semantic and advisory runtimes. Without `roles`,
156
- execution and artifact-grading roles retain the existing selection. Response-only
244
+ candidate, hook, artifact semantic, and advisory roles receive private copies.
245
+ Shell checks require explicit `"checks"` selection; a non-UV grader receives no
246
+ implicit UV/Python cache. Existing configurations whose shell checks need seeded
247
+ dependencies must add `"checks"` to `roles`. Response-only
157
248
  semantic grading omits implicit seeds; include `"semantic"` explicitly in `roles`
158
- when that route needs dependencies. Artifact semantic grading and executable
159
- shell checks keep their declared tools and default seed selection.
249
+ when that route needs dependencies. Declared host tools remain available to shell checks.
160
250
 
161
251
  Each source tree is bounded to 16,384 entries and 512 MiB. Escaping symlinks,
162
252
  directory cycles, dangling links, and special files are refused. File content
@@ -172,19 +262,64 @@ retaining large policies can retain up to the configured per-seed byte bounds.
172
262
  Source caches are never a writable runtime. Use prepared public caches rather
173
263
  than credential-bearing homes or evaluator directories.
174
264
 
265
+ ### Reuse a shared dependency cache
266
+
267
+ `runtime.sharedCacheDirectories` opts into persistent writable caches without
268
+ copying or hashing their contents for each trial. Each entry requires `source`,
269
+ `target`, and a nonempty `roles` array drawn from the same five roles above.
270
+ The source must already be an owned, readable, writable directory. Relative paths
271
+ resolve beside the runtime file. Sevro resolves declaration symlinks to their
272
+ canonical source and revalidates that directory before execution.
273
+ Targets use the seed slot rules and must be unique across seeds and shared caches.
274
+ Sources cannot overlap each other, private seed sources, declared read-only
275
+ inputs, evaluated project/source trees, results, credentials, or native private state.
276
+
277
+ Select only roles that actually use UV, for example `"roles": ["candidate", "hooks"]`.
278
+ If a grader also uses UV, add `"checks"` to that same entry; both roles use the
279
+ same cache. Each selected private slot aliases the original directory, with a
280
+ narrow sandbox write grant. Other roles receive no slot or source grant.
281
+ Shared content survives successful, failed, and cancelled trials and later runs.
282
+ Sevro removes only disposable role state, never the declared shared source.
283
+
284
+ Prepare UV packages once outside evaluation, point `UV_CACHE_DIR` at
285
+ `{{sevro.runtime}}/uv-cache`, and reuse the existing Python installation through
286
+ declared read-only tool roots. Set `UV_PYTHON_DOWNLOADS=never` to require that
287
+ installation. Sevro keeps `UV_OFFLINE=1`; a missing dependency fails rather than
288
+ downloading during a trial. Project virtual environments and temporary files
289
+ remain disposable; UV's install/link mode can still allocate files in a virtual
290
+ environment. The shared option removes the full cache copy per trial.
291
+
292
+ Shared caches are mutable across trials, roles, and runs. Use them only for public
293
+ dependencies and accept that a participant can change them for later participants.
294
+ Use isolated seeds when dependency independence is required. Configuration identity
295
+ binds the source, slot, and roles, but excludes mutable cache contents; equal
296
+ identities therefore do not prove identical shared-cache bytes. Sevro does not
297
+ age-delete shared caches. Manage their size between evaluations with the cache
298
+ tool, such as `uv cache prune --cache-dir /absolute/public-uv-cache`.
299
+
175
300
  `{{sevro.runtime}}` in an environment value resolves to that role's writable
176
- root. Sevro supplies private `HOME` and temporary storage and sets `UV_OFFLINE=1`
177
- and `PYTHONDONTWRITEBYTECODE=1`. Candidate, hook, checker, semantic, and advisory
301
+ root. Codex commands receive a `TMPPREFIX` inside their private `TMPDIR`,
302
+ so zsh here-documents use their existing private temporary storage while shared
303
+ host temporary directories remain denied. `TMPPREFIX` is runner-owned and cannot
304
+ be inherited or set through runtime configuration.
305
+ Sevro supplies private `HOME` and temporary storage and sets `UV_OFFLINE=1`
306
+ and `PYTHONDONTWRITEBYTECODE=1`. Fixture setup uses the candidate commands'
307
+ private `HOME`, so `$HOME` paths initialize the same state the candidate will
308
+ use. Its temporary directory stays disposable.
309
+ Candidate, hook, checker, semantic, and advisory
178
310
  state are separate. With a runtime policy, native candidate, semantic, and
179
311
  advisory roots live in a unique runner-owned host directory outside the Git
180
- fixture. The root contains `home`, `tmp`, and declared seed slots; plugins can
312
+ fixture. The root contains `home`, `tmp`, private seed slots and selected shared-cache links; plugins can
181
313
  use paths such as `~/.darrow/reviews` without creating repository-local review
182
314
  state. Command runtime directories are separate from host authentication,
183
- configuration, and live transcript sources. Checker and hook caches also live
184
- outside candidate-controlled workspace state. Private roots survive native
315
+ configuration, and live transcript sources. Checker and hook private state also lives
316
+ outside candidate-controlled workspace state. Shared sources remain at their declared locations. Private roots survive native
185
317
  continuation turns and children, then are removed with their owning trial or
186
- host state after owned hooks terminate. Opted-in candidate artifact runtimes
187
- remain alive through grading as described below. Runs without a selected runtime policy
318
+ host state after owned hooks terminate. Checker home, temporary storage, and copied
319
+ seeds are removed after grading even when the fixture is retained after failure
320
+ or cancellation. Results and evidence are preserved. Opted-in candidate artifact runtimes
321
+ remain alive through grading as described below. Runs with neither fixture setup
322
+ nor a selected runtime policy
188
323
  retain their previous writable-state locations.
189
324
 
190
325
  Claude refuses managed workspace or runtime namespaces overlapping its
@@ -200,8 +335,7 @@ data and remain trusted code responsible for their own execution boundary.
200
335
  ## Candidate artifact access
201
336
 
202
337
  Use `runtime.artifactDirectories` to let graders read selected artifacts in the
203
- candidate's private home after execution. This option is available in the current
204
- source checkout; published `0.1.0-rc.7` does not include it.
338
+ candidate's private home after execution.
205
339
 
206
340
  ```json
207
341
  {
@@ -212,7 +346,8 @@ source checkout; published `0.1.0-rc.7` does not include it.
212
346
  }
213
347
  ```
214
348
 
215
- The example permits only the candidate-created `~/.darrow/reviews` subtree.
349
+ The example permits only the `~/.darrow/reviews` subtree, which may be initialized
350
+ by trusted fixture setup or created by the candidate.
216
351
  Its original absolute path reaches isolated shell checks and native
217
352
  semantic/advisory commands as `SEVRO_CANDIDATE_ARTIFACT_REVIEWS`. Graders keep
218
353
  their own private `HOME`; use the supplied variable to read candidate evidence.
@@ -228,12 +363,31 @@ to 16,384 entries, including its root. Missing directories, symlinked ancestors,
228
363
  descendant symlinks, hardlinks and special files make grading an error; missing
229
364
  evidence cannot produce a pass.
230
365
 
366
+ Initialize an empty directory or seed prior Review state in the extension's
367
+ trusted `fixtureSetup` command before the candidate starts. For example:
368
+
369
+ ```sh
370
+ mkdir -p "$HOME/.darrow/reviews"
371
+ printf '%s\n' '{"verdict":"passed"}' > "$HOME/.darrow/reviews/prior.json"
372
+ ```
373
+
374
+ Setup and candidate commands share this private trial home, including when no
375
+ artifact grants or runtime file are selected. Graders keep their own homes and
376
+ receive only declared read-only artifact grants. To test missing reports or
377
+ absence of new artifacts, initialize the directory without writing a report;
378
+ an empty safe directory reaches ordinary task checks. Missing required reports
379
+ can then fail those checks, while negative cases can assert the directory is
380
+ empty. Sevro does not automatically create declared artifact directories: if
381
+ setup and candidate both leave one absent, or replace it with an unsafe path,
382
+ grading still errors. Setup state is temporary and is removed with its trial,
383
+ including setup failure, cancellation, and dry runs.
384
+
231
385
  The grant permits reads and denies writes, creation, deletion and renaming.
232
386
  Credentials, other home contents and peer role/trial state stay protected.
233
387
  Ordinary `filesystem.readOnlyRoots` cannot select this private-state exception.
234
388
  Artifact grants and native transcript access are independent opt-ins.
235
389
 
236
- The trial allocates the retained runtime before launching its candidate and
390
+ The trial allocates the retained runtime before fixture setup and
237
391
  removes it after grading, advisory review and evidence persistence, including
238
392
  failure and cancellation paths. Review state remains outside the Git fixture.
239
393
  These artifacts are temporary; declaring access does not copy their bytes into
@@ -248,6 +402,72 @@ hosts validate it before use. Grading adapters must preserve the read-only
248
402
  boundary. The directory declarations enter runtime configuration identity;
249
403
  per-trial absolute paths do not.
250
404
 
405
+ ## Candidate Codex configuration and user skills
406
+
407
+ Set top-level `"codexState": true` to expose the bundled Codex candidate's actual
408
+ isolated configuration and user-skill state. This opt-in is independent of
409
+ `nativeTranscripts` and ordinary
410
+ candidate-home artifact declarations:
411
+
412
+ ```json
413
+ {
414
+ "format": "sevro.runtime.v1",
415
+ "codexState": true
416
+ }
417
+ ```
418
+
419
+ Candidate commands receive the real isolated `CODEX_HOME`. They may read its
420
+ `config.toml` and write non-system skills below `CODEX_HOME/skills`. The native
421
+ process's separate user-skill directory is available as
422
+ `SEVRO_CODEX_AGENT_SKILLS`; it names that process's `HOME/.agents/skills`.
423
+ Bundled `CODEX_HOME/skills/.system` skills remain read-only. Authentication,
424
+ other private siblings, evaluator inputs and peer requests remain denied.
425
+ Configuration remains read-only, and the preflight still checks credential denial.
426
+
427
+ A physical fixture-local `.codex` directory is accepted only with this opt-in.
428
+ Sevro marks the isolated project untrusted in its generated Codex configuration,
429
+ so project-local configuration, hooks and rules remain represented inputs rather
430
+ than effective host settings. The fixture is preserved. Diagnose the absolute
431
+ `CODEX_HOME/config.toml`, not the checkout's `.codex/config.toml`. The runner's
432
+ selected concurrency limit is written into the private config; an omitted limit
433
+ still means the native host default, not a fabricated configured capacity.
434
+ CLI model/effort and vetted-hook overrides keep their existing precedence; the
435
+ file alone does not establish every effective native default or child route.
436
+
437
+ After execution, isolated shell and native semantic/advisory graders receive
438
+ read-only bindings to these original candidate paths:
439
+
440
+ | Environment variable | Original candidate path |
441
+ | ----------------------------------------------- | ------------------------------------------ |
442
+ | `SEVRO_CANDIDATE_ARTIFACT_CODEX_CONFIG` | Native `CODEX_HOME/config.toml` (one file) |
443
+ | `SEVRO_CANDIDATE_ARTIFACT_CODEX_SKILLS` | Native `CODEX_HOME/skills` |
444
+ | `SEVRO_CANDIDATE_ARTIFACT_AGENT_SKILLS` | Native process `HOME/.agents/skills` |
445
+ | `SEVRO_CANDIDATE_ARTIFACT_COMMAND_AGENT_SKILLS` | Command `HOME/.agents/skills` |
446
+ | `SEVRO_CANDIDATE_ARTIFACT_COMMAND_CODEX_SKILLS` | Command `HOME/.codex/skills` |
447
+
448
+ Native and command homes are separate. The runner initializes these actual
449
+ skill locations before the candidate starts and retains their resulting state;
450
+ it does not create replacement empty directories for grading. Graders keep their
451
+ own homes and must use these bindings. To assert absence, require every selected
452
+ directory to exist, then search for `SKILL.md`, excluding only the native Codex
453
+ root's read-only `.system` subtree. Missing evidence is an error. Do not search
454
+ the grader's home or retired fixture state.
455
+
456
+ The five artifact names are reserved. Each binding belongs to one active trial
457
+ and one native request; reuse by another host request is refused. The same
458
+ canonical ancestor, entry-limit, symlink, hardlink and special-file checks as
459
+ ordinary artifacts apply. Configuration cannot be replaced by a directory grant.
460
+ The native home is retained through grading and removed with its trial, including
461
+ failure and cancellation paths. Standalone host calls clean up immediately.
462
+ These grants do not persist native files in run evidence; private absolute paths
463
+ do not enter identity, while the `codexState` selection does. Other candidate
464
+ hosts cannot supply this Codex evidence and fail grading when it is requested.
465
+
466
+ For the Darrow doctor and undefined-skill cases, select this configuration with
467
+ `--runtime-config-file`, preserve the doctor fixture's checkout value `99`, and
468
+ replace the retired-state absence check with the four skill-directory bindings
469
+ above. Keep the original repository, no-commit, activation and semantic checks.
470
+
251
471
  ## Goals and plugin hooks
252
472
 
253
473
  `hooks.nativeGoal` enables Claude's native goal Stop-hook path while preserving
@@ -272,6 +492,17 @@ instead of being skipped. The wrapper denies network access and protected
272
492
  roots and owns asynchronous hook cleanup. Native timeout/cancellation stops
273
493
  owned hook descendants before private state is removed.
274
494
 
495
+ Prepared native plugin projections live outside the fixture in the current
496
+ host request's private native state. Curated hook read grants validate the
497
+ canonical projection layout, ownership, private state permissions, and binding
498
+ to that request. Only the selected plugin subtree is readable; it stays
499
+ read-only, and credentials, private siblings, and peer requests remain denied.
500
+ Ordinary filesystem declarations cannot select this exception. Codex validates
501
+ and filters installed plugins before enabling its runner-owned skill collector,
502
+ including requests without a runtime policy. Plugin command handlers still
503
+ require explicit hook authority. Projections are removed with their owning host
504
+ request.
505
+
275
506
  Auxiliary `.mcp.json` and `.lsp.json` files and plugin mods are unsupported under
276
507
  hook authority. Manifest MCP/LSP declarations are removed from curated mounts.
277
508
 
@@ -339,7 +570,7 @@ run evidence. Extensions and external benchmarks consume the
339
570
  ## Evidence and compatibility
340
571
 
341
572
  `configuration.redacted.runtimePolicy` retains the selected environment,
342
- canonical read roots, seed digests, and hook declaration. The engine snapshots
573
+ canonical read roots, seed digests, shared-cache declarations, and hook declaration. The engine snapshots
343
574
  this input once; it participates in `configurationDigest` and evaluation
344
575
  identity. Symbolic runtime placeholders remain symbolic in identity, while
345
576
  execution uses per-role paths. This binds declared setup and seed contents;
@@ -353,7 +584,8 @@ file contents do not. The same discovery and skip evidence reaches
353
584
  `sevro.host.runtime` observations.
354
585
 
355
586
  Bundled hosts supply `sevro.host.runtime` with the applied policy digest, role,
356
- environment names, read roots, and seed slots. `sevro.host.hooks` records source
587
+ environment names, read roots, seed slots, and selected shared-cache declarations.
588
+ Shared entries contain no content digest. `sevro.host.hooks` records source
357
589
  and effective plugin digests plus observed executions and their exit codes.
358
590
  An empty execution list does not prove hook activation. Partial lifecycle
359
591
  evidence remains partial. Cases can require these observations; missing required
@@ -363,7 +595,8 @@ task verdict.
363
595
  `--toolchain-bin-dir` and `--claude-uv-cache-dir` are deprecated. They keep their
364
596
  previous behavior with a warning when no runtime file is selected. Combining
365
597
  them with a runtime file fails before execution. Migrate to `environment`,
366
- `filesystem`, and private seed slots; removal is no earlier than `0.2.0`.
598
+ `filesystem`, and either private seed slots or shared-cache declarations; removal
599
+ is no earlier than `0.2.0`. The deprecated UV option continues to copy privately.
367
600
  See the [CLI reference](development-cli.md), [native hosts](native-hosts.md),
368
601
  and [identity contract](identity-v1.md).
369
602
 
@@ -3,18 +3,18 @@
3
3
  Start with execution, grading, task states, and stderr. A documented possible
4
4
  cause is not a diagnosis of your machine.
5
5
 
6
- | Symptom | Check and next step |
7
- | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
8
- | Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md). |
9
- | Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot. |
10
- | Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md). |
11
- | Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md). |
12
- | Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md). |
13
- | Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md). |
14
- | Native or shell isolation fails | Verify macOS and `sandbox-exec`. Other platforms are unverified. |
15
- | Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md). |
16
- | Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout. |
17
- | External links fail | Follow [network failure triage](documentation-quality.md#external-links). |
6
+ | Symptom | Check and next step |
7
+ | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
8
+ | Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md). |
9
+ | Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot. |
10
+ | Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md). |
11
+ | Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md). |
12
+ | Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md). |
13
+ | Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md). |
14
+ | Native or shell isolation fails | On macOS, verify `sandbox-exec`; on Ubuntu 24.04 x64, verify `bubblewrap` and Claude's `socat`. See [validated scope](native-hosts.md#validated-platform-scope). Windows is unverified. |
15
+ | Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md). |
16
+ | Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout. |
17
+ | External links fail | Follow [network failure triage](documentation-quality.md#external-links). |
18
18
 
19
19
  ## The repository guide is missing
20
20
 
@@ -1,6 +1,7 @@
1
1
  # TypeScript quality
2
2
 
3
- `bun run check:typescript` is the local and CI acceptance command. It performs a
3
+ `bun run check:typescript` is the complete local acceptance command. CI runs the
4
+ same checks in parallel phases. The complete command performs a
4
5
  frozen dependency installation, checks the complete source inventory, verifies
5
6
  generated schemas, documentation contracts, formatting, type-aware lint and typing,
6
7
  checks the Codex guide evaluation in dry mode and runs documentation examples, exercises coverage
@@ -14,13 +15,44 @@ schemas, documentation contracts, formatting, lint and types without the full in
14
15
  or package installation. Install dependencies with `bun install --frozen-lockfile`
15
16
  first. Quality dependencies are development dependencies pinned in `bun.lock`.
16
17
 
18
+ After frozen installation, `bun run check:typescript --phase <name>` runs one
19
+ phase and checks the complete source inventory. Each phase reports its elapsed
20
+ time, including when it fails. The complete command runs these same phases in
21
+ order; `--fast` runs only `static` without installing dependencies.
22
+
23
+ | Phase | Checks |
24
+ | --------- | --------------------------------------------------------------------------------------------- |
25
+ | `static` | Schema freshness, documentation contracts, formatting, type-aware lint and typing |
26
+ | `docs` | All guide dry runs, tutorial examples and recipe checks |
27
+ | `tests` | Coverage integrity controls, the complete instrumented Bun suite and both coverage thresholds |
28
+ | `package` | Packed-package installation and installed CLI workflows |
29
+
30
+ Use the same prerequisites as the corresponding complete-gate checks. A single
31
+ phase is partial evidence; full acceptance requires all four on the same candidate.
32
+ CI assigns every test file to exactly one of four duration-balanced shards, then
33
+ enforces both coverage thresholds on their combined counters over the complete
34
+ production inventory. Local `--phase tests` and the complete command still run
35
+ the suite in one process. See the [shard commands](../scripts/coverage/README.md#test-shards)
36
+ for reproducing the CI split.
37
+
17
38
  ## Source dispositions
18
39
 
19
- [`typescript-sources.json`](../typescript-sources.json) explicitly lists every
20
- TypeScript and JavaScript source. The guard refuses new, missing or duplicated
21
- files and declarations outside the configured TypeScript project. Changing a
22
- source disposition requires review; adding a glob is not a substitute for the
23
- inventory. The coverage baseline includes the platform-reachable subset of
40
+ The [inventory guard](../scripts/typescript-inventory.ts) derives every TypeScript
41
+ and JavaScript source disposition from Git paths, following the
42
+ [maintainer decision in #143](https://github.com/BjRo/sevro/issues/143#issuecomment-6382662399).
43
+ It classifies `git ls-files --cached --others --exclude-standard -z` output using
44
+ the checked path rules, so supported additions and staged renames or deletions
45
+ need no manual manifest edit. Non-ignored untracked sources are included in local
46
+ candidate checks; committed CI checkouts use their tracked paths. Declarations
47
+ take precedence over directory roles, and only the validators in the shared
48
+ [schema catalog](../scripts/schema-catalog.ts) receive the generated disposition.
49
+ Unknown or ambiguous paths, missing tracked files, an incomplete generated set,
50
+ and sources outside lint or typing are refused. An independent filesystem audit
51
+ also refuses ignored authored sources rather than silently dropping them.
52
+ Changing a disposition rule still requires review. Isolated coverage snapshots
53
+ index their copied candidate independently, preserving additions and deletions
54
+ without changing the source checkout's index.
55
+ The coverage baseline includes the platform-reachable subset of
24
56
  `inventory.production`; `scripts/coverage/platform-scope.ts` names each excluded
25
57
  host-specific file. Every production file remains inventoried, typed and linted,
26
58
  and the opposite platform measures it. Current instrumentation supports authored
@@ -29,14 +61,14 @@ validators in `inventory.generated` are excluded from instrumentation and the
29
61
  coverage denominator. New `.tsx`, `.mts`, `.cts` or JavaScript
30
62
  production requires an explicit instrumentation and quality-policy change first.
31
63
 
32
- | Inventory | Lint and typing | Test and coverage disposition |
33
- | -------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
34
- | `production` | Type-aware strict lint and strict compiler | All platform-reachable authored executable `src` statements and branch outcomes, including unimported files in that scope |
35
- | `generated` | CommonJS syntax parsing, exact `schemas:check` freshness, and checked `.d.cts` boundary declarations | Real Bun schema tests; the five declared generated validators are outside the coverage denominator |
36
- | `declarations` | Type-aware lint and compiler | Declaration-only files have no executable counters |
37
- | `tooling` | Same type-aware lint and strict compiler; `.mjs`/`.cjs` use `checkJs` and explicit JSDoc contracts | Durable public gate probes, schema freshness, release and installed-package tests; outside the runner-production denominator |
38
- | `tests` | Same type-aware lint and strict compiler, including fixtures | Real Bun filesystem/process/protocol tests; outside the production denominator |
39
- | `examples` | Same type-aware lint and strict compiler | Packaged demonstration host adapters are synthetic consumer fixtures, exercised by installed-package validation; they implement no runner behavior and are outside the runner-production denominator |
64
+ | Inventory | Lint and typing | Test and coverage disposition |
65
+ | -------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
66
+ | `production` | Type-aware strict lint and strict compiler | All platform-reachable authored executable `src` statements and branch outcomes, including unimported files in that scope |
67
+ | `generated` | CommonJS syntax parsing, exact `schemas:check` freshness, and checked `.d.cts` boundary declarations | Real Bun schema tests; the five declared generated validators are outside the coverage denominator |
68
+ | `declarations` | Type-aware lint and compiler | Declaration-only files have no executable counters |
69
+ | `tooling` | Same type-aware lint and strict compiler; `.mjs`/`.cjs` use `checkJs` and explicit JSDoc contracts | Durable public gate probes, schema freshness, release and installed-package tests; outside the runner-production denominator |
70
+ | `tests` | Same type-aware lint and strict compiler, including fixtures | Real Bun filesystem/process/protocol tests; outside the production denominator |
71
+ | `examples` | Same type-aware lint and strict compiler | Packaged recipe adapters, extensions and evaluator checks are consumer examples, exercised by tutorial and installed-package validation; they implement no runner behavior and are outside the runner-production denominator |
40
72
 
41
73
  Generated AJV CommonJS is deliberately not rewritten to conform to authored-code
42
74
  rules. An earlier audit of the original four generated files using ESLint's
@@ -45,8 +77,8 @@ recommended JavaScript rules found 879 redeclarations, 281 useless assignments,
45
77
  generation freshness, typed public boundaries and runtime behavior remain checked.
46
78
  Generated compiler output is excluded from coverage because the coverage target
47
79
  measures authored runner behavior. This applies only to
48
- `src/generated/cli-result.cjs`, `extension.cjs`, `report.cjs`, `run-evidence.cjs` and `runtime.cjs`
49
- as explicitly declared in the inventory. This disposition does not apply to authored CommonJS, including
80
+ the exact validator paths derived from the [schema catalog](../scripts/schema-catalog.ts),
81
+ which also drives schema generation. This disposition does not apply to authored CommonJS, including
50
82
  the coverage capture hook.
51
83
 
52
84
  ## Types, functions and review
@@ -120,7 +152,7 @@ macOS `sandbox-exec`, Keychain and Apple metadata modules are excluded; on macOS
120
152
  `bubblewrap` module is excluded. Shared isolation and credential code is counted
121
153
  on both.
122
154
 
123
- Apple developer-tool discovery lives in `src/runtime-discovery-apple.ts`, which
155
+ Apple developer-tool discovery lives in `src/runtime/discovery-apple.ts`, which
124
156
  the public loader imports only under its explicit Darwin guard. Linux excludes
125
157
  that exact native module; macOS measures it, and both platforms still typecheck
126
158
  and lint it. The provider lookup, bounded query and read-root safeguards remain
@@ -146,12 +178,24 @@ summary, process ledger, Bun test log, unmodified source snapshot, LCOV and HTML
146
178
  production files, branch exclusions, and exact projected counts.
147
179
  `coverage-raw.json` and the raw summary preserve all validated counters before
148
180
  platform projection.
181
+ CI also retains `.quality/shards/<index>/` with each shard's source snapshot,
182
+ process ledger, test log, raw and portable counters, and manifest. Process IDs
183
+ remain scoped to their original shard. Aggregation verifies candidate contents,
184
+ workflow run, platform, architecture, Bun version, instrumentation
185
+ layout, and exact test assignments before combining counters with normalized
186
+ source paths. Missing, repeated, incomplete, or mismatched shards fail the gate.
187
+ Failed-job reruns can reuse successful shards from earlier attempts of the same
188
+ workflow run; manifests retain each attempt, and rerun shards replace only their
189
+ own artifacts. Another candidate or workflow run cannot supply missing evidence.
149
190
  `--coverage [Bun test arguments]` is a diagnostic measurement; it still enforces
150
191
  both thresholds and writes `.quality/coverage-targeted`, keeping it distinct from
151
192
  the full-suite evidence. Sparse runs are expected to fail the full denominator.
152
193
 
153
194
  The durable public probes are `--probe snapshot`, `--probe scope`, `--probe branch`,
154
195
  `--probe missing-reports`, `--probe integrity`, and `--probe source-map`.
196
+ `tests/scripts/typescript-quality.test.ts` invokes each probe once during the suite;
197
+ the phase entrypoint and shard merge do not rerun them. The commands remain
198
+ available for targeted diagnostics.
155
199
  The scope probe records a zero-seeded baseline and merged real process report
156
200
  under `.quality/coverage-scope`; it verifies that ordinary generated CommonJS
157
201
  validation still runs. Branch and missing-report controls
@@ -167,21 +211,40 @@ Codex 0.160.1 and Claude Code 2.1.284 for deterministic native hook tests. Hoste
167
211
  CI targets `ubuntu-24.04` x64 with Bun 1.3.13 and Node 24.13.0. Its first
168
212
  successful hosted run is required before describing that environment as validated.
169
213
  GitHub documents the [Ubuntu runner labels](https://docs.github.com/en/actions/reference/runners/github-hosted-runners).
170
- Ubuntu installs `bubblewrap` and `socat` and runs the same native isolation cases;
171
- local macOS runs retain `sandbox-exec`. Windows remains unverified.
214
+ Ubuntu installs `bubblewrap`, `ripgrep`, and `socat` and runs the same native isolation cases;
215
+ local macOS runs retain `sandbox-exec`. The macOS quality and coverage gate is
216
+ local-only; the hosted workflow does not run a macOS gate. Windows remains unverified.
217
+
218
+ Ordinary changes and explicit workflow dispatches run the four canonical phases
219
+ concurrently. The tests phase consists of four shard jobs followed by a combined
220
+ coverage job. Each job installs locked dependencies; native host tools are
221
+ installed for `docs` and the test shards, and UV and mise for `docs`. Each shard
222
+ retains its evidence, and the combined `tests` job uploads `.quality` even when
223
+ coverage fails. Shards do not cancel their siblings on failure. A new commit to
224
+ the same pull request cancels its superseded workflow run; main pushes and manual
225
+ runs have independent concurrency groups. The stable aggregate `TypeScript quality`
226
+ status requires every phase and all four shards to succeed. A missing, cancelled,
227
+ skipped or failed phase or shard cannot turn that status green.
172
228
 
173
- Ordinary changes and explicit workflow dispatches invoke the canonical gate,
174
- upload `.quality` even when coverage fails, and publish the stable aggregate
175
- `TypeScript quality` status. A cancelled or failed gate cannot turn that status
176
- green.
229
+ CI caches Bun's package downloads by Bun version, OS, architecture and `bun.lock`;
230
+ every job still performs frozen installation. The docs job and test shards also
231
+ cache the installed Codex and Claude packages by OS, architecture and the setup
232
+ script's content hash, which includes both pinned versions and their installation
233
+ layout. There are no fallback keys for native tools. On a hit, setup copies the
234
+ cached packages into a fresh temporary tool root and verifies both native versions
235
+ before publishing their paths. The cache never contains credentials or replaces
236
+ the isolation prerequisites or required checks.
177
237
 
178
238
  Release metadata changes follow [the release workflow](releases.md) and reuse a
179
239
  successful full gate from an ancestor on `main`, with identical code,
180
240
  dependencies, schemas, tests, and other quality inputs. The selector permits
181
241
  only version/distribution-tag changes and its exact release-file inventory.
182
- It verifies the original full-gate step, so a reused release run cannot serve
183
- as fresh test evidence. Release CI runs `check:typescript --fast`, records the
184
- tested source and CI run URL, and skips the test/coverage job. The aggregate
242
+ It verifies each original phase job and its gate step, including every shard and
243
+ the combined tests job for sharded runs, or a historical complete gate job and
244
+ step. A partial matrix or reused release run cannot serve as fresh test evidence.
245
+ Duplicate phase or shard jobs are ambiguous and refused. Release CI
246
+ runs `check:typescript --fast`, records the tested source and CI run URL, and
247
+ skips all four full-gate phase jobs. The aggregate
185
248
  status requires both verified prior evidence and successful static checks;
186
249
  missing, cancelled, failed, or ambiguous evidence cannot establish a pass.
187
250
  Reused results remain attributed to their original run rather than being
@@ -0,0 +1,26 @@
1
+ import { readFile, writeFile } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+
4
+ /** Trusted, deterministic participant for exercising the recipe's real graders. */
5
+ const host = {
6
+ id: "recipes.deterministic",
7
+ model: "fixture-editor",
8
+ effort: "none",
9
+ async run({ workspace }: { workspace: string }) {
10
+ const task = await readFile(join(workspace, "README.md"), "utf8");
11
+ const failing = process.env.SEVRO_RECIPE_FAIL === "1";
12
+ if (task.includes("Fix add")) {
13
+ await writeFile(
14
+ join(workspace, "math.cjs"),
15
+ `exports.add = (a, b) => a ${failing ? "-" : "+"} b;\n`,
16
+ );
17
+ }
18
+ return {
19
+ finalMessage: failing
20
+ ? "Start with npm start on port 9000."
21
+ : "Run bun run dev on port 3000.",
22
+ complete: true,
23
+ };
24
+ },
25
+ };
26
+ export default host;