@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +17 -3
  7. package/docs/codex-host.md +155 -49
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +141 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +307 -18
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +63 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +206 -0
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +13 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +133 -115
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/evaluation/types.ts +207 -0
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +15 -76
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/fixtures/workspace-state.ts +212 -0
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +57 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +91 -14
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +46 -16
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +122 -73
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -54
  158. package/src/hosts/codex-profile.ts +50 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +141 -19
  161. package/src/hosts/codex.ts +221 -179
  162. package/src/hosts/isolation-common.ts +15 -11
  163. package/src/hosts/isolation-roots.ts +13 -20
  164. package/src/hosts/linux-sandbox.ts +27 -3
  165. package/src/hosts/mac-sandbox.ts +111 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +221 -45
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +34 -15
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/run-owner.ts +4 -4
  186. package/src/{runtime-config.ts → runtime/config.ts} +182 -44
  187. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  188. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  189. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  190. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  191. package/src/{runtime-hooks.ts → runtime/hooks.ts} +33 -22
  192. package/src/{runtime-paths.ts → runtime/paths.ts} +6 -18
  193. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  194. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  195. package/src/runtime/shared-cache.ts +130 -0
  196. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  197. package/src/schema-types.ts +73 -161
  198. package/src/shell.ts +4 -0
  199. package/src/transcript-retention.ts +146 -0
  200. package/src/value-guards.ts +22 -0
  201. package/docs/assets/sevro-logo.png +0 -0
  202. package/docs/assets/sevro-symbol.svg +0 -1
  203. package/docs/assets/sevro-wordmark.svg +0 -1
  204. package/docs/cache-ownership-validation.md +0 -49
  205. package/docs/claude-continuation-validation.md +0 -47
  206. package/docs/concurrency-validation.md +0 -86
  207. package/docs/configuration-root-validation.md +0 -70
  208. package/docs/engine-slice.md +0 -114
  209. package/docs/evidence/runtime-startup-30/README.md +0 -185
  210. package/docs/evidence/runtime-startup-30/after.json +0 -319
  211. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  212. package/docs/evidence/runtime-startup-30/before.json +0 -403
  213. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  214. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  216. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  217. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  218. package/docs/evidence.md +0 -17
  219. package/docs/fixture-cleanup-validation.md +0 -70
  220. package/docs/guide-verification.json +0 -239
  221. package/src/evaluation-context.ts +0 -257
  222. package/src/evaluation-fixture.ts +0 -313
  223. package/src/evaluation-run.ts +0 -198
  224. package/src/evaluation-trial.ts +0 -946
  225. package/src/evaluation-types.ts +0 -149
  226. package/src/hosts/claude-nested-skills.ts +0 -272
  227. package/src/hosts/codex-command-output.ts +0 -510
  228. package/src/hosts/codex-executor-command.ts +0 -291
  229. package/src/hosts/codex-skill-reads.ts +0 -609
  230. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
@@ -1,8 +1,7 @@
1
1
  # Runtime configuration v1
2
2
 
3
- Configure access to existing host tools, private writable caches, native goals,
4
- and selected plugin hooks in `<project-root>/sevro.json`. This interface is
5
- available in `0.1.0-rc.3` and later.
3
+ Configure access to existing host tools, private state, reusable caches, native goals,
4
+ and selected plugin hooks in `<project-root>/sevro.json`.
6
5
 
7
6
  `--runtime-config-file /absolute/path/runtime.json` selects a replacement file.
8
7
  Sevro reads exactly one file before execution, including dry runs. It does not
@@ -14,6 +13,17 @@ non-symlink files of at most 64 KiB.
14
13
 
15
14
  The [JSON schema](../schemas/runtime-v1.schema.json) has format
16
15
  `sevro.runtime.v1`. All properties except `format` are optional.
16
+ Start with the smallest file:
17
+
18
+ ```json
19
+ { "format": "sevro.runtime.v1" }
20
+ ```
21
+
22
+ Then add `environment.inherit: ["PATH"]` for installed tools. Follow the
23
+ [runnable runtime recipes](recipes.md#add-host-tool-access-one-capability-at-a-time)
24
+ to add Node, Python/UV, mise, goals and selected plugin hooks separately.
25
+ The following composed example is for a workflow that already needs all these
26
+ capabilities:
17
27
 
18
28
  ```json
19
29
  {
@@ -26,7 +36,13 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
26
36
  }
27
37
  },
28
38
  "runtime": {
29
- "seedDirectories": [{ "source": "./public-uv-cache", "target": "uv-cache" }]
39
+ "sharedCacheDirectories": [
40
+ {
41
+ "source": "../public-uv-cache",
42
+ "target": "uv-cache",
43
+ "roles": ["candidate", "hooks"]
44
+ }
45
+ ]
30
46
  },
31
47
  "hooks": {
32
48
  "nativeGoal": true,
@@ -35,7 +51,7 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
35
51
  }
36
52
  ```
37
53
 
38
- Create the example's `public-uv-cache` directory beside the configuration, or
54
+ Prepare the example's `public-uv-cache` directory outside the project, or
39
55
  omit `runtime` when no prepared cache is needed. Selected plugin names must be
40
56
  present in the case's prepared mounts; remove names that the case does not mount.
41
57
  `DARROW_CACHE_DIR` is an ordinary explicit variable in this example, not a
@@ -102,6 +118,10 @@ Existing directory paths and symlink targets are canonicalized. Reads under
102
118
  the real home are allowed only through bounded runtime grants. Source and
103
119
  configuration worktrees, evaluator inputs, credentials, results, run ownership
104
120
  state, and peer fixtures remain protected. A conflicting or broad grant fails.
121
+ Managed workspaces share a private parent that protects sibling workspaces,
122
+ including those created after launch. Independently prepared workspaces require
123
+ explicit protection for other private inputs. Claude hosts require Claude Code
124
+ 2.1.257 or later for managed-workspace isolation; see [checked versions](typescript-quality.md#environment-and-ci).
105
125
 
106
126
  Use `filesystem.optionalReadOnlyRoots` for supporting directories that may not
107
127
  be installed on every host, with the same path rules as `readOnlyRoots`:
@@ -140,8 +160,80 @@ minimal system read tree such as `/usr` or `/etc`, Sevro retains its explicit de
140
160
  the native preflight may refuse that overlapping setup rather than expose home
141
161
  contents.
142
162
 
163
+ ## Local HTTP test servers
164
+
165
+ Sevro supports local HTTP servers in isolated Codex candidate
166
+ commands and `sevro.shell` graders through an explicit operator selection:
167
+
168
+ ```json
169
+ {
170
+ "format": "sevro.runtime.v1",
171
+ "network": { "localHttp": true }
172
+ }
173
+ ```
174
+
175
+ Add this property to the selected `sevro.json`, or select a replacement with
176
+ `--runtime-config-file`. Declare the existing Node/Python installations through
177
+ the environment and filesystem options above. The selection is snapshotted and
178
+ enters evaluation identity. Absent or `false` preserves existing permissions.
179
+ Hooks and semantic/advisory hosts keep their existing network permissions; this
180
+ option applies to Codex candidates and shell grading.
181
+
182
+ On macOS, shell isolation permits IP socket binding, loopback inbound traffic,
183
+ and outbound connections to loopback addresses. Wildcard listeners such as
184
+ Node's `server.listen(0)` work with loopback clients. Other outbound IP traffic
185
+ remains denied. On Linux, shell checks retain their private network namespace
186
+ with usable loopback and no external route.
187
+
188
+ Codex uses its managed command network proxy with `allow_local_binding = true`,
189
+ upstream proxy chaining disabled, and only `localhost`, `127.0.0.1`, and `::1`
190
+ allowlisted. Its native sandbox permits local binding and loopback traffic while
191
+ refusing direct outbound TCP. The native proxy may permit DNS queries; public
192
+ HTTP destinations are not allowlisted. The candidate can reach other services
193
+ on the available loopback interface, so choose this option for evaluations
194
+ that need that access. It does not provide a separate per-trial loopback service
195
+ namespace on macOS.
196
+
197
+ The supported host boundary is the native Codex sandbox and its experimental
198
+ `network_proxy` feature, documented in the [OpenAI permission reference](https://learn.chatgpt.com/docs/permissions).
199
+ Setting `allow_local_binding` while leaving command networking disabled still
200
+ denies `listen()` on macOS. Integration tests execute the installed native
201
+ Codex sandbox, a wildcard HTTP listener and loopback client, and an outbound TCP
202
+ denial probe. Host versions and actual upstream evaluation outcomes belong in
203
+ [validation evidence](https://github.com/BjRo/sevro/blob/main/docs/evidence.md), rather than being inferred from a profile's
204
+ text. An unsupported host must not be interpreted as a passing behavior test.
205
+
206
+ ## macOS ancestor metadata
207
+
208
+ Sevro's macOS outer sandbox for isolated shell checks and authorized command
209
+ hooks permits metadata reads on the exact ancestor paths of existing validated
210
+ grants. This includes declared read-only support trees, private writable
211
+ runtimes, and bounded transcript, candidate artifact, and managed workspace
212
+ access. The permissions are derived automatically from those grants; no
213
+ additional configuration is required. They let libc `realpath()` canonicalize
214
+ paths inside a permitted subtree.
215
+
216
+ Ancestor directory metadata, such as existence, type, ownership, permissions,
217
+ and timestamps, is visible. Directory listings, ancestor file contents, and
218
+ ungranted sibling metadata and contents remain denied under protected roots.
219
+ Writes remain limited by the existing writable grants; read-only grants stay
220
+ read-only. This metadata exception does not grant access to an ancestor subtree
221
+ or change network isolation.
222
+
143
223
  ## Private writable state
144
224
 
225
+ Codex candidate commands can write the isolated fixture's `.git` directory,
226
+ including its index, refs, objects, and task-owned admission locks. Sevro grants
227
+ that exact path explicitly because Codex's workspace permission otherwise keeps
228
+ Git metadata read-only. Real commits and publication to a fixture-local remote
229
+ need no additional runtime option. This grant does not authorize writes to
230
+ source repositories, grading inputs, host authentication or configuration, or
231
+ peer trials. Command network access remains disabled unless the operator selects
232
+ the [local HTTP option](#local-http-test-servers).
233
+ An existing fixture `.git` must be a physical directory. Symlinks and Git
234
+ directory pointer files are refused before launch because their metadata may
235
+ belong to a protected source repository rather than the isolated trial.
236
+
145
237
  `runtime.seedDirectories` copies prepared public dependencies into a private
146
238
  slot for each execution role. `source` uses the filesystem path rules above;
147
239
  `target` is a single lowercase slot name, up to 64 characters, with letters,
@@ -149,10 +241,12 @@ digits, and hyphens. `home` and `tmp` are reserved. Targets must be unique.
149
241
  An optional nonempty `roles` array selects any of `candidate`, `checks`, `hooks`,
150
242
  `semantic`, and `advisory`. For example, `"roles": ["candidate", "checks", "hooks"]`
151
243
  keeps dependency slots out of semantic and advisory runtimes. Without `roles`,
152
- execution and artifact-grading roles retain the existing selection. Response-only
244
+ candidate, hook, artifact semantic, and advisory roles receive private copies.
245
+ Shell checks require explicit `"checks"` selection; a non-UV grader receives no
246
+ implicit UV/Python cache. Existing configurations whose shell checks need seeded
247
+ dependencies must add `"checks"` to `roles`. Response-only
153
248
  semantic grading omits implicit seeds; include `"semantic"` explicitly in `roles`
154
- when that route needs dependencies. Artifact semantic grading and executable
155
- shell checks keep their declared tools and default seed selection.
249
+ when that route needs dependencies. Declared host tools remain available to shell checks.
156
250
 
157
251
  Each source tree is bounded to 16,384 entries and 512 MiB. Escaping symlinks,
158
252
  directory cycles, dangling links, and special files are refused. File content
@@ -168,21 +262,67 @@ retaining large policies can retain up to the configured per-seed byte bounds.
168
262
  Source caches are never a writable runtime. Use prepared public caches rather
169
263
  than credential-bearing homes or evaluator directories.
170
264
 
265
+ ### Reuse a shared dependency cache
266
+
267
+ `runtime.sharedCacheDirectories` opts into persistent writable caches without
268
+ copying or hashing their contents for each trial. Each entry requires `source`,
269
+ `target`, and a nonempty `roles` array drawn from the same five roles above.
270
+ The source must already be an owned, readable, writable directory. Relative paths
271
+ resolve beside the runtime file. Sevro resolves declaration symlinks to their
272
+ canonical source and revalidates that directory before execution.
273
+ Targets use the seed slot rules and must be unique across seeds and shared caches.
274
+ Sources cannot overlap each other, private seed sources, declared read-only
275
+ inputs, evaluated project/source trees, results, credentials, or native private state.
276
+
277
+ Select only roles that actually use UV, for example `"roles": ["candidate", "hooks"]`.
278
+ If a grader also uses UV, add `"checks"` to that same entry; both roles use the
279
+ same cache. Each selected private slot aliases the original directory, with a
280
+ narrow sandbox write grant. Other roles receive no slot or source grant.
281
+ Shared content survives successful, failed, and cancelled trials and later runs.
282
+ Sevro removes only disposable role state, never the declared shared source.
283
+
284
+ Prepare UV packages once outside evaluation, point `UV_CACHE_DIR` at
285
+ `{{sevro.runtime}}/uv-cache`, and reuse the existing Python installation through
286
+ declared read-only tool roots. Set `UV_PYTHON_DOWNLOADS=never` to require that
287
+ installation. Sevro keeps `UV_OFFLINE=1`; a missing dependency fails rather than
288
+ downloading during a trial. Project virtual environments and temporary files
289
+ remain disposable; UV's install/link mode can still allocate files in a virtual
290
+ environment. The shared option removes the full cache copy per trial.
291
+
292
+ Shared caches are mutable across trials, roles, and runs. Use them only for public
293
+ dependencies and accept that a participant can change them for later participants.
294
+ Use isolated seeds when dependency independence is required. Configuration identity
295
+ binds the source, slot, and roles, but excludes mutable cache contents; equal
296
+ identities therefore do not prove identical shared-cache bytes. Sevro does not
297
+ age-delete shared caches. Manage their size between evaluations with the cache
298
+ tool, such as `uv cache prune --cache-dir /absolute/public-uv-cache`.
299
+
171
300
  `{{sevro.runtime}}` in an environment value resolves to that role's writable
172
- root. Sevro supplies private `HOME` and temporary storage and sets `UV_OFFLINE=1`
173
- and `PYTHONDONTWRITEBYTECODE=1`. Candidate, hook, checker, semantic, and advisory
301
+ root. Codex commands receive a `TMPPREFIX` inside their private `TMPDIR`,
302
+ so zsh here-documents use their existing private temporary storage while shared
303
+ host temporary directories remain denied. `TMPPREFIX` is runner-owned and cannot
304
+ be inherited or set through runtime configuration.
305
+ Sevro supplies private `HOME` and temporary storage and sets `UV_OFFLINE=1`
306
+ and `PYTHONDONTWRITEBYTECODE=1`. Fixture setup uses the candidate commands'
307
+ private `HOME`, so `$HOME` paths initialize the same state the candidate will
308
+ use. Its temporary directory stays disposable.
309
+ Candidate, hook, checker, semantic, and advisory
174
310
  state are separate. With a runtime policy, native candidate, semantic, and
175
311
  advisory roots live in a unique runner-owned host directory outside the Git
176
- fixture. The root contains `home`, `tmp`, and declared seed slots; plugins can
312
+ fixture. The root contains `home`, `tmp`, private seed slots and selected shared-cache links; plugins can
177
313
  use paths such as `~/.darrow/reviews` without creating repository-local review
178
314
  state. Command runtime directories are separate from host authentication,
179
- configuration, and live transcript sources. Checker and hook caches also live
180
- outside candidate-controlled workspace state. Private roots survive native
315
+ configuration, and live transcript sources. Checker and hook private state also lives
316
+ outside candidate-controlled workspace state. Shared sources remain at their declared locations. Private roots survive native
181
317
  continuation turns and children, then are removed with their owning trial or
182
- host state after owned hooks terminate. Runs without a selected runtime policy
318
+ host state after owned hooks terminate. Checker home, temporary storage, and copied
319
+ seeds are removed after grading even when the fixture is retained after failure
320
+ or cancellation. Results and evidence are preserved. Opted-in candidate artifact runtimes
321
+ remain alive through grading as described below. Runs with neither fixture setup
322
+ nor a selected runtime policy
183
323
  retain their previous writable-state locations.
184
324
 
185
- Claude runtime policies refuse a private namespace overlapping the native CLI's
325
+ Claude refuses managed workspace or runtime namespaces overlapping its
186
326
  implicit `/tmp/claude` or `/private/tmp/claude` write roots, including canonical
187
327
  symlink targets. A conflicting `TMPDIR` placement fails before native launch;
188
328
  choose a temporary base outside those cache trees.
@@ -192,6 +332,142 @@ checks, and native semantic/advisory hosts. Candidate plugin-hook authority is
192
332
  not forwarded to grading hosts. Injected adapters receive the policy as request
193
333
  data and remain trusted code responsible for their own execution boundary.
194
334
 
335
+ ## Candidate artifact access
336
+
337
+ Use `runtime.artifactDirectories` to let graders read selected artifacts in the
338
+ candidate's private home after execution.
339
+
340
+ ```json
341
+ {
342
+ "format": "sevro.runtime.v1",
343
+ "runtime": {
344
+ "artifactDirectories": [{ "name": "reviews", "path": ".darrow/reviews" }]
345
+ }
346
+ }
347
+ ```
348
+
349
+ The example permits only the `~/.darrow/reviews` subtree, which may be initialized
350
+ by trusted fixture setup or created by the candidate.
351
+ Its original absolute path reaches isolated shell checks and native
352
+ semantic/advisory commands as `SEVRO_CANDIDATE_ARTIFACT_REVIEWS`. Graders keep
353
+ their own private `HOME`; use the supplied variable to read candidate evidence.
354
+ For each declared name, the variable is `SEVRO_CANDIDATE_ARTIFACT_` followed by
355
+ the name in uppercase. Artifact content is untrusted candidate evidence.
356
+
357
+ At most 16 directories may be declared. Names are unique, contain lowercase
358
+ letters, digits and underscores, start with a letter, and are at most 64
359
+ characters. Paths are relative to candidate home, are at most 4,096 characters,
360
+ and cannot contain empty, `.` or `..` components, backslashes or control
361
+ characters. Whole-home and absolute grants are refused. Each subtree is bounded
362
+ to 16,384 entries, including its root. Missing directories, symlinked ancestors,
363
+ descendant symlinks, hardlinks and special files make grading an error; missing
364
+ evidence cannot produce a pass.
365
+
366
+ Initialize an empty directory or seed prior Review state in the extension's
367
+ trusted `fixtureSetup` command before the candidate starts. For example:
368
+
369
+ ```sh
370
+ mkdir -p "$HOME/.darrow/reviews"
371
+ printf '%s\n' '{"verdict":"passed"}' > "$HOME/.darrow/reviews/prior.json"
372
+ ```
373
+
374
+ Setup and candidate commands share this private trial home, including when no
375
+ artifact grants or runtime file are selected. Graders keep their own homes and
376
+ receive only declared read-only artifact grants. To test missing reports or
377
+ absence of new artifacts, initialize the directory without writing a report;
378
+ an empty safe directory reaches ordinary task checks. Missing required reports
379
+ can then fail those checks, while negative cases can assert the directory is
380
+ empty. Sevro does not automatically create declared artifact directories: if
381
+ setup and candidate both leave one absent, or replace it with an unsafe path,
382
+ grading still errors. Setup state is temporary and is removed with its trial,
383
+ including setup failure, cancellation, and dry runs.
384
+
385
+ The grant permits reads and denies writes, creation, deletion and renaming.
386
+ Credentials, other home contents and peer role/trial state stay protected.
387
+ Ordinary `filesystem.readOnlyRoots` cannot select this private-state exception.
388
+ Artifact grants and native transcript access are independent opt-ins.
389
+
390
+ The trial allocates the retained runtime before fixture setup and
391
+ removes it after grading, advisory review and evidence persistence, including
392
+ failure and cancellation paths. Review state remains outside the Git fixture.
393
+ These artifacts are temporary; declaring access does not copy their bytes into
394
+ retained run evidence. Failed-fixture retention does not retain this runtime.
395
+ Standalone bundled-host calls retain their immediate cleanup behavior.
396
+
397
+ Trusted injected adapters receive a runner-owned `candidateRuntimeRoot` on the
398
+ candidate request and a named `candidateArtifactRoots` path map on grading
399
+ requests. Candidate adapters opting into this feature must put their private
400
+ runtime at the supplied root. The lifetime binding is process-local; bundled
401
+ hosts validate it before use. Grading adapters must preserve the read-only
402
+ boundary. The directory declarations enter runtime configuration identity;
403
+ per-trial absolute paths do not.
404
+
405
+ ## Candidate Codex configuration and user skills
406
+
407
+ Set top-level `"codexState": true` to expose the bundled Codex candidate's actual
408
+ isolated configuration and user-skill state. This opt-in is independent of
409
+ `nativeTranscripts` and ordinary
410
+ candidate-home artifact declarations:
411
+
412
+ ```json
413
+ {
414
+ "format": "sevro.runtime.v1",
415
+ "codexState": true
416
+ }
417
+ ```
418
+
419
+ Candidate commands receive the real isolated `CODEX_HOME`. They may read its
420
+ `config.toml` and write non-system skills below `CODEX_HOME/skills`. The native
421
+ process's separate user-skill directory is available as
422
+ `SEVRO_CODEX_AGENT_SKILLS`; it names that process's `HOME/.agents/skills`.
423
+ Bundled `CODEX_HOME/skills/.system` skills remain read-only. Authentication,
424
+ other private siblings, evaluator inputs and peer requests remain denied.
425
+ Configuration remains read-only, and the preflight still checks credential denial.
426
+
427
+ A physical fixture-local `.codex` directory is accepted only with this opt-in.
428
+ Sevro marks the isolated project untrusted in its generated Codex configuration,
429
+ so project-local configuration, hooks and rules remain represented inputs rather
430
+ than effective host settings. The fixture is preserved. Diagnose the absolute
431
+ `CODEX_HOME/config.toml`, not the checkout's `.codex/config.toml`. The runner's
432
+ selected concurrency limit is written into the private config; an omitted limit
433
+ still means the native host default, not a fabricated configured capacity.
434
+ CLI model/effort and vetted-hook overrides keep their existing precedence; the
435
+ file alone does not establish every effective native default or child route.
436
+
437
+ After execution, isolated shell and native semantic/advisory graders receive
438
+ read-only bindings to these original candidate paths:
439
+
440
+ | Environment variable | Original candidate path |
441
+ | ----------------------------------------------- | ------------------------------------------ |
442
+ | `SEVRO_CANDIDATE_ARTIFACT_CODEX_CONFIG` | Native `CODEX_HOME/config.toml` (one file) |
443
+ | `SEVRO_CANDIDATE_ARTIFACT_CODEX_SKILLS` | Native `CODEX_HOME/skills` |
444
+ | `SEVRO_CANDIDATE_ARTIFACT_AGENT_SKILLS` | Native process `HOME/.agents/skills` |
445
+ | `SEVRO_CANDIDATE_ARTIFACT_COMMAND_AGENT_SKILLS` | Command `HOME/.agents/skills` |
446
+ | `SEVRO_CANDIDATE_ARTIFACT_COMMAND_CODEX_SKILLS` | Command `HOME/.codex/skills` |
447
+
448
+ Native and command homes are separate. The runner initializes these actual
449
+ skill locations before the candidate starts and retains their resulting state;
450
+ it does not create replacement empty directories for grading. Graders keep their
451
+ own homes and must use these bindings. To assert absence, require every selected
452
+ directory to exist, then search for `SKILL.md`, excluding only the native Codex
453
+ root's read-only `.system` subtree. Missing evidence is an error. Do not search
454
+ the grader's home or retired fixture state.
455
+
456
+ The five artifact names are reserved. Each binding belongs to one active trial
457
+ and one native request; reuse by another host request is refused. The same
458
+ canonical ancestor, entry-limit, symlink, hardlink and special-file checks as
459
+ ordinary artifacts apply. Configuration cannot be replaced by a directory grant.
460
+ The native home is retained through grading and removed with its trial, including
461
+ failure and cancellation paths. Standalone host calls clean up immediately.
462
+ These grants do not persist native files in run evidence; private absolute paths
463
+ do not enter identity, while the `codexState` selection does. Other candidate
464
+ hosts cannot supply this Codex evidence and fail grading when it is requested.
465
+
466
+ For the Darrow doctor and undefined-skill cases, select this configuration with
467
+ `--runtime-config-file`, preserve the doctor fixture's checkout value `99`, and
468
+ replace the retired-state absence check with the four skill-directory bindings
469
+ above. Keep the original repository, no-commit, activation and semantic checks.
470
+
195
471
  ## Goals and plugin hooks
196
472
 
197
473
  `hooks.nativeGoal` enables Claude's native goal Stop-hook path while preserving
@@ -216,6 +492,17 @@ instead of being skipped. The wrapper denies network access and protected
216
492
  roots and owns asynchronous hook cleanup. Native timeout/cancellation stops
217
493
  owned hook descendants before private state is removed.
218
494
 
495
+ Prepared native plugin projections live outside the fixture in the current
496
+ host request's private native state. Curated hook read grants validate the
497
+ canonical projection layout, ownership, private state permissions, and binding
498
+ to that request. Only the selected plugin subtree is readable; it stays
499
+ read-only, and credentials, private siblings, and peer requests remain denied.
500
+ Ordinary filesystem declarations cannot select this exception. Codex validates
501
+ and filters installed plugins before enabling its runner-owned skill collector,
502
+ including requests without a runtime policy. Plugin command handlers still
503
+ require explicit hook authority. Projections are removed with their owning host
504
+ request.
505
+
219
506
  Auxiliary `.mcp.json` and `.lsp.json` files and plugin mods are unsupported under
220
507
  hook authority. Manifest MCP/LSP declarations are removed from curated mounts.
221
508
 
@@ -283,7 +570,7 @@ run evidence. Extensions and external benchmarks consume the
283
570
  ## Evidence and compatibility
284
571
 
285
572
  `configuration.redacted.runtimePolicy` retains the selected environment,
286
- canonical read roots, seed digests, and hook declaration. The engine snapshots
573
+ canonical read roots, seed digests, shared-cache declarations, and hook declaration. The engine snapshots
287
574
  this input once; it participates in `configurationDigest` and evaluation
288
575
  identity. Symbolic runtime placeholders remain symbolic in identity, while
289
576
  execution uses per-role paths. This binds declared setup and seed contents;
@@ -297,7 +584,8 @@ file contents do not. The same discovery and skip evidence reaches
297
584
  `sevro.host.runtime` observations.
298
585
 
299
586
  Bundled hosts supply `sevro.host.runtime` with the applied policy digest, role,
300
- environment names, read roots, and seed slots. `sevro.host.hooks` records source
587
+ environment names, read roots, seed slots, and selected shared-cache declarations.
588
+ Shared entries contain no content digest. `sevro.host.hooks` records source
301
589
  and effective plugin digests plus observed executions and their exit codes.
302
590
  An empty execution list does not prove hook activation. Partial lifecycle
303
591
  evidence remains partial. Cases can require these observations; missing required
@@ -307,7 +595,8 @@ task verdict.
307
595
  `--toolchain-bin-dir` and `--claude-uv-cache-dir` are deprecated. They keep their
308
596
  previous behavior with a warning when no runtime file is selected. Combining
309
597
  them with a runtime file fails before execution. Migrate to `environment`,
310
- `filesystem`, and private seed slots; removal is no earlier than `0.2.0`.
598
+ `filesystem`, and either private seed slots or shared-cache declarations; removal
599
+ is no earlier than `0.2.0`. The deprecated UV option continues to copy privately.
311
600
  See the [CLI reference](development-cli.md), [native hosts](native-hosts.md),
312
601
  and [identity contract](identity-v1.md).
313
602
 
@@ -3,18 +3,18 @@
3
3
  Start with execution, grading, task states, and stderr. A documented possible
4
4
  cause is not a diagnosis of your machine.
5
5
 
6
- | Symptom | Check and next step |
7
- | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
8
- | Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md). |
9
- | Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot. |
10
- | Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md). |
11
- | Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md). |
12
- | Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md). |
13
- | Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md). |
14
- | Native or shell isolation fails | Verify macOS and `sandbox-exec`. Other platforms are unverified. |
15
- | Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md). |
16
- | Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout. |
17
- | External links fail | Follow [network failure triage](documentation-quality.md#external-links). |
6
+ | Symptom | Check and next step |
7
+ | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
8
+ | Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md). |
9
+ | Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot. |
10
+ | Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md). |
11
+ | Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md). |
12
+ | Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md). |
13
+ | Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md). |
14
+ | Native or shell isolation fails | On macOS, verify `sandbox-exec`; on Ubuntu 24.04 x64, verify `bubblewrap` and Claude's `socat`. See [validated scope](native-hosts.md#validated-platform-scope). Windows is unverified. |
15
+ | Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md). |
16
+ | Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout. |
17
+ | External links fail | Follow [network failure triage](documentation-quality.md#external-links). |
18
18
 
19
19
  ## The repository guide is missing
20
20