@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +22 -6
- package/README.md +5 -1
- package/docs/README.md +19 -15
- package/docs/architecture.md +138 -17
- package/docs/assets/README.md +5 -4
- package/docs/builtin-graders-v1.md +17 -3
- package/docs/codex-host.md +155 -49
- package/docs/contracts.md +2 -1
- package/docs/creating-cases.md +9 -2
- package/docs/development-cli.md +187 -19
- package/docs/documentation-quality.md +12 -2
- package/docs/evaluation-workflow.md +110 -0
- package/docs/extension-protocol-v1.md +26 -3
- package/docs/getting-started.md +23 -2
- package/docs/guide-evaluation.md +33 -1
- package/docs/identity-v1.md +25 -2
- package/docs/installing.md +11 -11
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +196 -7
- package/docs/reading-results.md +102 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +31 -12
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +141 -4
- package/docs/run-settings-v1.md +147 -0
- package/docs/runtime-v1.md +307 -18
- package/docs/troubleshooting.md +12 -12
- package/docs/typescript-quality.md +89 -26
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +11 -5
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +63 -7
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +206 -0
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +73 -50
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +87 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +60 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +198 -0
- package/src/cli/hosts.ts +55 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +154 -0
- package/src/cli/presentation.ts +93 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +128 -0
- package/src/cli/run-options.ts +403 -0
- package/src/cli/run-overrides.ts +69 -0
- package/src/cli/settings.ts +154 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -591
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +13 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +235 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +133 -115
- package/src/evaluation/trial.ts +800 -0
- package/src/evaluation/types.ts +207 -0
- package/src/evidence/host-results.ts +85 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +34 -96
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +15 -76
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +183 -0
- package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
- package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
- package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/fixtures/workspace-state.ts +212 -0
- package/src/generated/cli-result.d.cts +2 -1
- package/src/generated/extension.d.cts +2 -1
- package/src/generated/report.cjs +1 -1
- package/src/generated/report.d.cts +2 -1
- package/src/generated/run-evidence.cjs +1 -1
- package/src/generated/run-evidence.d.cts +2 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/generated/runtime.d.cts +2 -1
- package/src/git.ts +93 -0
- package/src/graders/git-head.ts +29 -84
- package/src/graders/shell.ts +57 -68
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +331 -0
- package/src/hosts/claude-continuation.ts +57 -40
- package/src/hosts/claude-credential.ts +160 -63
- package/src/hosts/claude-events.ts +34 -18
- package/src/hosts/claude-keychain.ts +78 -21
- package/src/hosts/claude-login-recovery.ts +85 -0
- package/src/hosts/claude-login-refresh.ts +178 -0
- package/src/hosts/claude-login-store.ts +162 -0
- package/src/hosts/claude-login.ts +107 -0
- package/src/hosts/claude-native-goal.ts +15 -12
- package/src/hosts/claude-process.ts +10 -50
- package/src/hosts/claude-refresh-lock.ts +81 -0
- package/src/hosts/claude-repository-invocation.ts +9 -34
- package/src/hosts/claude-settings.ts +91 -14
- package/src/hosts/claude-skill-activations.ts +169 -0
- package/src/hosts/claude-state.ts +46 -16
- package/src/hosts/claude-tool-calls.ts +13 -10
- package/src/hosts/claude.ts +122 -73
- package/src/hosts/codex-app-server.ts +78 -52
- package/src/hosts/codex-configuration.ts +5 -10
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-marketplace.ts +12 -10
- package/src/hosts/codex-native-calls.ts +15 -122
- package/src/hosts/codex-process.ts +10 -54
- package/src/hosts/codex-profile.ts +50 -21
- package/src/hosts/codex-session-files.ts +1 -9
- package/src/hosts/codex-state.ts +141 -19
- package/src/hosts/codex.ts +221 -179
- package/src/hosts/isolation-common.ts +15 -11
- package/src/hosts/isolation-roots.ts +13 -20
- package/src/hosts/linux-sandbox.ts +27 -3
- package/src/hosts/mac-sandbox.ts +111 -18
- package/src/hosts/native-controls.ts +72 -22
- package/src/hosts/native-request.ts +55 -0
- package/src/hosts/process-lifecycle.ts +221 -45
- package/src/hosts/protected-worktrees.ts +1 -1
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +577 -0
- package/src/instrumentation.ts +7 -8
- package/src/native-transcript-state.ts +22 -3
- package/src/native-transcript-view.ts +52 -13
- package/src/native-transcripts.ts +34 -15
- package/src/paths.ts +22 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +27 -7
- package/src/prune.ts +300 -0
- package/src/report.ts +63 -36
- package/src/resolved-case.ts +1 -1
- package/src/results-lock.ts +42 -0
- package/src/results.ts +14 -18
- package/src/run-owner.ts +4 -4
- package/src/{runtime-config.ts → runtime/config.ts} +182 -44
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +33 -22
- package/src/{runtime-paths.ts → runtime/paths.ts} +6 -18
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/runtime/shared-cache.ts +130 -0
- package/src/{runtime-state.ts → runtime/state.ts} +28 -11
- package/src/schema-types.ts +73 -161
- package/src/shell.ts +4 -0
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +22 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +0 -1
- package/docs/assets/sevro-wordmark.svg +0 -1
- package/docs/cache-ownership-validation.md +0 -49
- package/docs/claude-continuation-validation.md +0 -47
- package/docs/concurrency-validation.md +0 -86
- package/docs/configuration-root-validation.md +0 -70
- package/docs/engine-slice.md +0 -114
- package/docs/evidence/runtime-startup-30/README.md +0 -185
- package/docs/evidence/runtime-startup-30/after.json +0 -319
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
- package/docs/evidence/runtime-startup-30/before.json +0 -403
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
- package/docs/evidence.md +0 -17
- package/docs/fixture-cleanup-validation.md +0 -70
- package/docs/guide-verification.json +0 -239
- package/src/evaluation-context.ts +0 -257
- package/src/evaluation-fixture.ts +0 -313
- package/src/evaluation-run.ts +0 -198
- package/src/evaluation-trial.ts +0 -946
- package/src/evaluation-types.ts +0 -149
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -510
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -609
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
package/docs/runtime-v1.md
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
# Runtime configuration v1
|
|
2
2
|
|
|
3
|
-
Configure access to existing host tools, private
|
|
4
|
-
and selected plugin hooks in `<project-root>/sevro.json`.
|
|
5
|
-
available in `0.1.0-rc.3` and later.
|
|
3
|
+
Configure access to existing host tools, private state, reusable caches, native goals,
|
|
4
|
+
and selected plugin hooks in `<project-root>/sevro.json`.
|
|
6
5
|
|
|
7
6
|
`--runtime-config-file /absolute/path/runtime.json` selects a replacement file.
|
|
8
7
|
Sevro reads exactly one file before execution, including dry runs. It does not
|
|
@@ -14,6 +13,17 @@ non-symlink files of at most 64 KiB.
|
|
|
14
13
|
|
|
15
14
|
The [JSON schema](../schemas/runtime-v1.schema.json) has format
|
|
16
15
|
`sevro.runtime.v1`. All properties except `format` are optional.
|
|
16
|
+
Start with the smallest file:
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{ "format": "sevro.runtime.v1" }
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Then add `environment.inherit: ["PATH"]` for installed tools. Follow the
|
|
23
|
+
[runnable runtime recipes](recipes.md#add-host-tool-access-one-capability-at-a-time)
|
|
24
|
+
to add Node, Python/UV, mise, goals and selected plugin hooks separately.
|
|
25
|
+
The following composed example is for a workflow that already needs all these
|
|
26
|
+
capabilities:
|
|
17
27
|
|
|
18
28
|
```json
|
|
19
29
|
{
|
|
@@ -26,7 +36,13 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
|
|
|
26
36
|
}
|
|
27
37
|
},
|
|
28
38
|
"runtime": {
|
|
29
|
-
"
|
|
39
|
+
"sharedCacheDirectories": [
|
|
40
|
+
{
|
|
41
|
+
"source": "../public-uv-cache",
|
|
42
|
+
"target": "uv-cache",
|
|
43
|
+
"roles": ["candidate", "hooks"]
|
|
44
|
+
}
|
|
45
|
+
]
|
|
30
46
|
},
|
|
31
47
|
"hooks": {
|
|
32
48
|
"nativeGoal": true,
|
|
@@ -35,7 +51,7 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
|
|
|
35
51
|
}
|
|
36
52
|
```
|
|
37
53
|
|
|
38
|
-
|
|
54
|
+
Prepare the example's `public-uv-cache` directory outside the project, or
|
|
39
55
|
omit `runtime` when no prepared cache is needed. Selected plugin names must be
|
|
40
56
|
present in the case's prepared mounts; remove names that the case does not mount.
|
|
41
57
|
`DARROW_CACHE_DIR` is an ordinary explicit variable in this example, not a
|
|
@@ -102,6 +118,10 @@ Existing directory paths and symlink targets are canonicalized. Reads under
|
|
|
102
118
|
the real home are allowed only through bounded runtime grants. Source and
|
|
103
119
|
configuration worktrees, evaluator inputs, credentials, results, run ownership
|
|
104
120
|
state, and peer fixtures remain protected. A conflicting or broad grant fails.
|
|
121
|
+
Managed workspaces share a private parent that protects sibling workspaces,
|
|
122
|
+
including those created after launch. Independently prepared workspaces require
|
|
123
|
+
explicit protection for other private inputs. Claude hosts require Claude Code
|
|
124
|
+
2.1.257 or later for managed-workspace isolation; see [checked versions](typescript-quality.md#environment-and-ci).
|
|
105
125
|
|
|
106
126
|
Use `filesystem.optionalReadOnlyRoots` for supporting directories that may not
|
|
107
127
|
be installed on every host, with the same path rules as `readOnlyRoots`:
|
|
@@ -140,8 +160,80 @@ minimal system read tree such as `/usr` or `/etc`, Sevro retains its explicit de
|
|
|
140
160
|
the native preflight may refuse that overlapping setup rather than expose home
|
|
141
161
|
contents.
|
|
142
162
|
|
|
163
|
+
## Local HTTP test servers
|
|
164
|
+
|
|
165
|
+
Sevro supports local HTTP servers in isolated Codex candidate
|
|
166
|
+
commands and `sevro.shell` graders through an explicit operator selection:
|
|
167
|
+
|
|
168
|
+
```json
|
|
169
|
+
{
|
|
170
|
+
"format": "sevro.runtime.v1",
|
|
171
|
+
"network": { "localHttp": true }
|
|
172
|
+
}
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Add this property to the selected `sevro.json`, or select a replacement with
|
|
176
|
+
`--runtime-config-file`. Declare the existing Node/Python installations through
|
|
177
|
+
the environment and filesystem options above. The selection is snapshotted and
|
|
178
|
+
enters evaluation identity. Absent or `false` preserves existing permissions.
|
|
179
|
+
Hooks and semantic/advisory hosts keep their existing network permissions; this
|
|
180
|
+
option applies to Codex candidates and shell grading.
|
|
181
|
+
|
|
182
|
+
On macOS, shell isolation permits IP socket binding, loopback inbound traffic,
|
|
183
|
+
and outbound connections to loopback addresses. Wildcard listeners such as
|
|
184
|
+
Node's `server.listen(0)` work with loopback clients. Other outbound IP traffic
|
|
185
|
+
remains denied. On Linux, shell checks retain their private network namespace
|
|
186
|
+
with usable loopback and no external route.
|
|
187
|
+
|
|
188
|
+
Codex uses its managed command network proxy with `allow_local_binding = true`,
|
|
189
|
+
upstream proxy chaining disabled, and only `localhost`, `127.0.0.1`, and `::1`
|
|
190
|
+
allowlisted. Its native sandbox permits local binding and loopback traffic while
|
|
191
|
+
refusing direct outbound TCP. The native proxy may permit DNS queries; public
|
|
192
|
+
HTTP destinations are not allowlisted. The candidate can reach other services
|
|
193
|
+
on the available loopback interface, so choose this option for evaluations
|
|
194
|
+
that need that access. It does not provide a separate per-trial loopback service
|
|
195
|
+
namespace on macOS.
|
|
196
|
+
|
|
197
|
+
The supported host boundary is the native Codex sandbox and its experimental
|
|
198
|
+
`network_proxy` feature, documented in the [OpenAI permission reference](https://learn.chatgpt.com/docs/permissions).
|
|
199
|
+
Setting `allow_local_binding` while leaving command networking disabled still
|
|
200
|
+
denies `listen()` on macOS. Integration tests execute the installed native
|
|
201
|
+
Codex sandbox, a wildcard HTTP listener and loopback client, and an outbound TCP
|
|
202
|
+
denial probe. Host versions and actual upstream evaluation outcomes belong in
|
|
203
|
+
[validation evidence](https://github.com/BjRo/sevro/blob/main/docs/evidence.md), rather than being inferred from a profile's
|
|
204
|
+
text. An unsupported host must not be interpreted as a passing behavior test.
|
|
205
|
+
|
|
206
|
+
## macOS ancestor metadata
|
|
207
|
+
|
|
208
|
+
Sevro's macOS outer sandbox for isolated shell checks and authorized command
|
|
209
|
+
hooks permits metadata reads on the exact ancestor paths of existing validated
|
|
210
|
+
grants. This includes declared read-only support trees, private writable
|
|
211
|
+
runtimes, and bounded transcript, candidate artifact, and managed workspace
|
|
212
|
+
access. The permissions are derived automatically from those grants; no
|
|
213
|
+
additional configuration is required. They let libc `realpath()` canonicalize
|
|
214
|
+
paths inside a permitted subtree.
|
|
215
|
+
|
|
216
|
+
Ancestor directory metadata, such as existence, type, ownership, permissions,
|
|
217
|
+
and timestamps, is visible. Directory listings, ancestor file contents, and
|
|
218
|
+
ungranted sibling metadata and contents remain denied under protected roots.
|
|
219
|
+
Writes remain limited by the existing writable grants; read-only grants stay
|
|
220
|
+
read-only. This metadata exception does not grant access to an ancestor subtree
|
|
221
|
+
or change network isolation.
|
|
222
|
+
|
|
143
223
|
## Private writable state
|
|
144
224
|
|
|
225
|
+
Codex candidate commands can write the isolated fixture's `.git` directory,
|
|
226
|
+
including its index, refs, objects, and task-owned admission locks. Sevro grants
|
|
227
|
+
that exact path explicitly because Codex's workspace permission otherwise keeps
|
|
228
|
+
Git metadata read-only. Real commits and publication to a fixture-local remote
|
|
229
|
+
need no additional runtime option. This grant does not authorize writes to
|
|
230
|
+
source repositories, grading inputs, host authentication or configuration, or
|
|
231
|
+
peer trials. Command network access remains disabled unless the operator selects
|
|
232
|
+
the [local HTTP option](#local-http-test-servers).
|
|
233
|
+
An existing fixture `.git` must be a physical directory. Symlinks and Git
|
|
234
|
+
directory pointer files are refused before launch because their metadata may
|
|
235
|
+
belong to a protected source repository rather than the isolated trial.
|
|
236
|
+
|
|
145
237
|
`runtime.seedDirectories` copies prepared public dependencies into a private
|
|
146
238
|
slot for each execution role. `source` uses the filesystem path rules above;
|
|
147
239
|
`target` is a single lowercase slot name, up to 64 characters, with letters,
|
|
@@ -149,10 +241,12 @@ digits, and hyphens. `home` and `tmp` are reserved. Targets must be unique.
|
|
|
149
241
|
An optional nonempty `roles` array selects any of `candidate`, `checks`, `hooks`,
|
|
150
242
|
`semantic`, and `advisory`. For example, `"roles": ["candidate", "checks", "hooks"]`
|
|
151
243
|
keeps dependency slots out of semantic and advisory runtimes. Without `roles`,
|
|
152
|
-
|
|
244
|
+
candidate, hook, artifact semantic, and advisory roles receive private copies.
|
|
245
|
+
Shell checks require explicit `"checks"` selection; a non-UV grader receives no
|
|
246
|
+
implicit UV/Python cache. Existing configurations whose shell checks need seeded
|
|
247
|
+
dependencies must add `"checks"` to `roles`. Response-only
|
|
153
248
|
semantic grading omits implicit seeds; include `"semantic"` explicitly in `roles`
|
|
154
|
-
when that route needs dependencies.
|
|
155
|
-
shell checks keep their declared tools and default seed selection.
|
|
249
|
+
when that route needs dependencies. Declared host tools remain available to shell checks.
|
|
156
250
|
|
|
157
251
|
Each source tree is bounded to 16,384 entries and 512 MiB. Escaping symlinks,
|
|
158
252
|
directory cycles, dangling links, and special files are refused. File content
|
|
@@ -168,21 +262,67 @@ retaining large policies can retain up to the configured per-seed byte bounds.
|
|
|
168
262
|
Source caches are never a writable runtime. Use prepared public caches rather
|
|
169
263
|
than credential-bearing homes or evaluator directories.
|
|
170
264
|
|
|
265
|
+
### Reuse a shared dependency cache
|
|
266
|
+
|
|
267
|
+
`runtime.sharedCacheDirectories` opts into persistent writable caches without
|
|
268
|
+
copying or hashing their contents for each trial. Each entry requires `source`,
|
|
269
|
+
`target`, and a nonempty `roles` array drawn from the same five roles above.
|
|
270
|
+
The source must already be an owned, readable, writable directory. Relative paths
|
|
271
|
+
resolve beside the runtime file. Sevro resolves declaration symlinks to their
|
|
272
|
+
canonical source and revalidates that directory before execution.
|
|
273
|
+
Targets use the seed slot rules and must be unique across seeds and shared caches.
|
|
274
|
+
Sources cannot overlap each other, private seed sources, declared read-only
|
|
275
|
+
inputs, evaluated project/source trees, results, credentials, or native private state.
|
|
276
|
+
|
|
277
|
+
Select only roles that actually use UV, for example `"roles": ["candidate", "hooks"]`.
|
|
278
|
+
If a grader also uses UV, add `"checks"` to that same entry; both roles use the
|
|
279
|
+
same cache. Each selected private slot aliases the original directory, with a
|
|
280
|
+
narrow sandbox write grant. Other roles receive no slot or source grant.
|
|
281
|
+
Shared content survives successful, failed, and cancelled trials and later runs.
|
|
282
|
+
Sevro removes only disposable role state, never the declared shared source.
|
|
283
|
+
|
|
284
|
+
Prepare UV packages once outside evaluation, point `UV_CACHE_DIR` at
|
|
285
|
+
`{{sevro.runtime}}/uv-cache`, and reuse the existing Python installation through
|
|
286
|
+
declared read-only tool roots. Set `UV_PYTHON_DOWNLOADS=never` to require that
|
|
287
|
+
installation. Sevro keeps `UV_OFFLINE=1`; a missing dependency fails rather than
|
|
288
|
+
downloading during a trial. Project virtual environments and temporary files
|
|
289
|
+
remain disposable; UV's install/link mode can still allocate files in a virtual
|
|
290
|
+
environment. The shared option removes the full cache copy per trial.
|
|
291
|
+
|
|
292
|
+
Shared caches are mutable across trials, roles, and runs. Use them only for public
|
|
293
|
+
dependencies and accept that a participant can change them for later participants.
|
|
294
|
+
Use isolated seeds when dependency independence is required. Configuration identity
|
|
295
|
+
binds the source, slot, and roles, but excludes mutable cache contents; equal
|
|
296
|
+
identities therefore do not prove identical shared-cache bytes. Sevro does not
|
|
297
|
+
age-delete shared caches. Manage their size between evaluations with the cache
|
|
298
|
+
tool, such as `uv cache prune --cache-dir /absolute/public-uv-cache`.
|
|
299
|
+
|
|
171
300
|
`{{sevro.runtime}}` in an environment value resolves to that role's writable
|
|
172
|
-
root.
|
|
173
|
-
|
|
301
|
+
root. Codex commands receive a `TMPPREFIX` inside their private `TMPDIR`,
|
|
302
|
+
so zsh here-documents use their existing private temporary storage while shared
|
|
303
|
+
host temporary directories remain denied. `TMPPREFIX` is runner-owned and cannot
|
|
304
|
+
be inherited or set through runtime configuration.
|
|
305
|
+
Sevro supplies private `HOME` and temporary storage and sets `UV_OFFLINE=1`
|
|
306
|
+
and `PYTHONDONTWRITEBYTECODE=1`. Fixture setup uses the candidate commands'
|
|
307
|
+
private `HOME`, so `$HOME` paths initialize the same state the candidate will
|
|
308
|
+
use. Its temporary directory stays disposable.
|
|
309
|
+
Candidate, hook, checker, semantic, and advisory
|
|
174
310
|
state are separate. With a runtime policy, native candidate, semantic, and
|
|
175
311
|
advisory roots live in a unique runner-owned host directory outside the Git
|
|
176
|
-
fixture. The root contains `home`, `tmp`, and
|
|
312
|
+
fixture. The root contains `home`, `tmp`, private seed slots and selected shared-cache links; plugins can
|
|
177
313
|
use paths such as `~/.darrow/reviews` without creating repository-local review
|
|
178
314
|
state. Command runtime directories are separate from host authentication,
|
|
179
|
-
configuration, and live transcript sources. Checker and hook
|
|
180
|
-
outside candidate-controlled workspace state. Private roots survive native
|
|
315
|
+
configuration, and live transcript sources. Checker and hook private state also lives
|
|
316
|
+
outside candidate-controlled workspace state. Shared sources remain at their declared locations. Private roots survive native
|
|
181
317
|
continuation turns and children, then are removed with their owning trial or
|
|
182
|
-
host state after owned hooks terminate.
|
|
318
|
+
host state after owned hooks terminate. Checker home, temporary storage, and copied
|
|
319
|
+
seeds are removed after grading even when the fixture is retained after failure
|
|
320
|
+
or cancellation. Results and evidence are preserved. Opted-in candidate artifact runtimes
|
|
321
|
+
remain alive through grading as described below. Runs with neither fixture setup
|
|
322
|
+
nor a selected runtime policy
|
|
183
323
|
retain their previous writable-state locations.
|
|
184
324
|
|
|
185
|
-
Claude
|
|
325
|
+
Claude refuses managed workspace or runtime namespaces overlapping its
|
|
186
326
|
implicit `/tmp/claude` or `/private/tmp/claude` write roots, including canonical
|
|
187
327
|
symlink targets. A conflicting `TMPDIR` placement fails before native launch;
|
|
188
328
|
choose a temporary base outside those cache trees.
|
|
@@ -192,6 +332,142 @@ checks, and native semantic/advisory hosts. Candidate plugin-hook authority is
|
|
|
192
332
|
not forwarded to grading hosts. Injected adapters receive the policy as request
|
|
193
333
|
data and remain trusted code responsible for their own execution boundary.
|
|
194
334
|
|
|
335
|
+
## Candidate artifact access
|
|
336
|
+
|
|
337
|
+
Use `runtime.artifactDirectories` to let graders read selected artifacts in the
|
|
338
|
+
candidate's private home after execution.
|
|
339
|
+
|
|
340
|
+
```json
|
|
341
|
+
{
|
|
342
|
+
"format": "sevro.runtime.v1",
|
|
343
|
+
"runtime": {
|
|
344
|
+
"artifactDirectories": [{ "name": "reviews", "path": ".darrow/reviews" }]
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
The example permits only the `~/.darrow/reviews` subtree, which may be initialized
|
|
350
|
+
by trusted fixture setup or created by the candidate.
|
|
351
|
+
Its original absolute path reaches isolated shell checks and native
|
|
352
|
+
semantic/advisory commands as `SEVRO_CANDIDATE_ARTIFACT_REVIEWS`. Graders keep
|
|
353
|
+
their own private `HOME`; use the supplied variable to read candidate evidence.
|
|
354
|
+
For each declared name, the variable is `SEVRO_CANDIDATE_ARTIFACT_` followed by
|
|
355
|
+
the name in uppercase. Artifact content is untrusted candidate evidence.
|
|
356
|
+
|
|
357
|
+
At most 16 directories may be declared. Names are unique, contain lowercase
|
|
358
|
+
letters, digits and underscores, start with a letter, and are at most 64
|
|
359
|
+
characters. Paths are relative to candidate home, are at most 4,096 characters,
|
|
360
|
+
and cannot contain empty, `.` or `..` components, backslashes or control
|
|
361
|
+
characters. Whole-home and absolute grants are refused. Each subtree is bounded
|
|
362
|
+
to 16,384 entries, including its root. Missing directories, symlinked ancestors,
|
|
363
|
+
descendant symlinks, hardlinks and special files make grading an error; missing
|
|
364
|
+
evidence cannot produce a pass.
|
|
365
|
+
|
|
366
|
+
Initialize an empty directory or seed prior Review state in the extension's
|
|
367
|
+
trusted `fixtureSetup` command before the candidate starts. For example:
|
|
368
|
+
|
|
369
|
+
```sh
|
|
370
|
+
mkdir -p "$HOME/.darrow/reviews"
|
|
371
|
+
printf '%s\n' '{"verdict":"passed"}' > "$HOME/.darrow/reviews/prior.json"
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
Setup and candidate commands share this private trial home, including when no
|
|
375
|
+
artifact grants or runtime file are selected. Graders keep their own homes and
|
|
376
|
+
receive only declared read-only artifact grants. To test missing reports or
|
|
377
|
+
absence of new artifacts, initialize the directory without writing a report;
|
|
378
|
+
an empty safe directory reaches ordinary task checks. Missing required reports
|
|
379
|
+
can then fail those checks, while negative cases can assert the directory is
|
|
380
|
+
empty. Sevro does not automatically create declared artifact directories: if
|
|
381
|
+
setup and candidate both leave one absent, or replace it with an unsafe path,
|
|
382
|
+
grading still errors. Setup state is temporary and is removed with its trial,
|
|
383
|
+
including setup failure, cancellation, and dry runs.
|
|
384
|
+
|
|
385
|
+
The grant permits reads and denies writes, creation, deletion and renaming.
|
|
386
|
+
Credentials, other home contents and peer role/trial state stay protected.
|
|
387
|
+
Ordinary `filesystem.readOnlyRoots` cannot select this private-state exception.
|
|
388
|
+
Artifact grants and native transcript access are independent opt-ins.
|
|
389
|
+
|
|
390
|
+
The trial allocates the retained runtime before fixture setup and
|
|
391
|
+
removes it after grading, advisory review and evidence persistence, including
|
|
392
|
+
failure and cancellation paths. Review state remains outside the Git fixture.
|
|
393
|
+
These artifacts are temporary; declaring access does not copy their bytes into
|
|
394
|
+
retained run evidence. Failed-fixture retention does not retain this runtime.
|
|
395
|
+
Standalone bundled-host calls retain their immediate cleanup behavior.
|
|
396
|
+
|
|
397
|
+
Trusted injected adapters receive a runner-owned `candidateRuntimeRoot` on the
|
|
398
|
+
candidate request and a named `candidateArtifactRoots` path map on grading
|
|
399
|
+
requests. Candidate adapters opting into this feature must put their private
|
|
400
|
+
runtime at the supplied root. The lifetime binding is process-local; bundled
|
|
401
|
+
hosts validate it before use. Grading adapters must preserve the read-only
|
|
402
|
+
boundary. The directory declarations enter runtime configuration identity;
|
|
403
|
+
per-trial absolute paths do not.
|
|
404
|
+
|
|
405
|
+
## Candidate Codex configuration and user skills
|
|
406
|
+
|
|
407
|
+
Set top-level `"codexState": true` to expose the bundled Codex candidate's actual
|
|
408
|
+
isolated configuration and user-skill state. This opt-in is independent of
|
|
409
|
+
`nativeTranscripts` and ordinary
|
|
410
|
+
candidate-home artifact declarations:
|
|
411
|
+
|
|
412
|
+
```json
|
|
413
|
+
{
|
|
414
|
+
"format": "sevro.runtime.v1",
|
|
415
|
+
"codexState": true
|
|
416
|
+
}
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
Candidate commands receive the real isolated `CODEX_HOME`. They may read its
|
|
420
|
+
`config.toml` and write non-system skills below `CODEX_HOME/skills`. The native
|
|
421
|
+
process's separate user-skill directory is available as
|
|
422
|
+
`SEVRO_CODEX_AGENT_SKILLS`; it names that process's `HOME/.agents/skills`.
|
|
423
|
+
Bundled `CODEX_HOME/skills/.system` skills remain read-only. Authentication,
|
|
424
|
+
other private siblings, evaluator inputs and peer requests remain denied.
|
|
425
|
+
Configuration remains read-only, and the preflight still checks credential denial.
|
|
426
|
+
|
|
427
|
+
A physical fixture-local `.codex` directory is accepted only with this opt-in.
|
|
428
|
+
Sevro marks the isolated project untrusted in its generated Codex configuration,
|
|
429
|
+
so project-local configuration, hooks and rules remain represented inputs rather
|
|
430
|
+
than effective host settings. The fixture is preserved. Diagnose the absolute
|
|
431
|
+
`CODEX_HOME/config.toml`, not the checkout's `.codex/config.toml`. The runner's
|
|
432
|
+
selected concurrency limit is written into the private config; an omitted limit
|
|
433
|
+
still means the native host default, not a fabricated configured capacity.
|
|
434
|
+
CLI model/effort and vetted-hook overrides keep their existing precedence; the
|
|
435
|
+
file alone does not establish every effective native default or child route.
|
|
436
|
+
|
|
437
|
+
After execution, isolated shell and native semantic/advisory graders receive
|
|
438
|
+
read-only bindings to these original candidate paths:
|
|
439
|
+
|
|
440
|
+
| Environment variable | Original candidate path |
|
|
441
|
+
| ----------------------------------------------- | ------------------------------------------ |
|
|
442
|
+
| `SEVRO_CANDIDATE_ARTIFACT_CODEX_CONFIG` | Native `CODEX_HOME/config.toml` (one file) |
|
|
443
|
+
| `SEVRO_CANDIDATE_ARTIFACT_CODEX_SKILLS` | Native `CODEX_HOME/skills` |
|
|
444
|
+
| `SEVRO_CANDIDATE_ARTIFACT_AGENT_SKILLS` | Native process `HOME/.agents/skills` |
|
|
445
|
+
| `SEVRO_CANDIDATE_ARTIFACT_COMMAND_AGENT_SKILLS` | Command `HOME/.agents/skills` |
|
|
446
|
+
| `SEVRO_CANDIDATE_ARTIFACT_COMMAND_CODEX_SKILLS` | Command `HOME/.codex/skills` |
|
|
447
|
+
|
|
448
|
+
Native and command homes are separate. The runner initializes these actual
|
|
449
|
+
skill locations before the candidate starts and retains their resulting state;
|
|
450
|
+
it does not create replacement empty directories for grading. Graders keep their
|
|
451
|
+
own homes and must use these bindings. To assert absence, require every selected
|
|
452
|
+
directory to exist, then search for `SKILL.md`, excluding only the native Codex
|
|
453
|
+
root's read-only `.system` subtree. Missing evidence is an error. Do not search
|
|
454
|
+
the grader's home or retired fixture state.
|
|
455
|
+
|
|
456
|
+
The five artifact names are reserved. Each binding belongs to one active trial
|
|
457
|
+
and one native request; reuse by another host request is refused. The same
|
|
458
|
+
canonical ancestor, entry-limit, symlink, hardlink and special-file checks as
|
|
459
|
+
ordinary artifacts apply. Configuration cannot be replaced by a directory grant.
|
|
460
|
+
The native home is retained through grading and removed with its trial, including
|
|
461
|
+
failure and cancellation paths. Standalone host calls clean up immediately.
|
|
462
|
+
These grants do not persist native files in run evidence; private absolute paths
|
|
463
|
+
do not enter identity, while the `codexState` selection does. Other candidate
|
|
464
|
+
hosts cannot supply this Codex evidence and fail grading when it is requested.
|
|
465
|
+
|
|
466
|
+
For the Darrow doctor and undefined-skill cases, select this configuration with
|
|
467
|
+
`--runtime-config-file`, preserve the doctor fixture's checkout value `99`, and
|
|
468
|
+
replace the retired-state absence check with the four skill-directory bindings
|
|
469
|
+
above. Keep the original repository, no-commit, activation and semantic checks.
|
|
470
|
+
|
|
195
471
|
## Goals and plugin hooks
|
|
196
472
|
|
|
197
473
|
`hooks.nativeGoal` enables Claude's native goal Stop-hook path while preserving
|
|
@@ -216,6 +492,17 @@ instead of being skipped. The wrapper denies network access and protected
|
|
|
216
492
|
roots and owns asynchronous hook cleanup. Native timeout/cancellation stops
|
|
217
493
|
owned hook descendants before private state is removed.
|
|
218
494
|
|
|
495
|
+
Prepared native plugin projections live outside the fixture in the current
|
|
496
|
+
host request's private native state. Curated hook read grants validate the
|
|
497
|
+
canonical projection layout, ownership, private state permissions, and binding
|
|
498
|
+
to that request. Only the selected plugin subtree is readable; it stays
|
|
499
|
+
read-only, and credentials, private siblings, and peer requests remain denied.
|
|
500
|
+
Ordinary filesystem declarations cannot select this exception. Codex validates
|
|
501
|
+
and filters installed plugins before enabling its runner-owned skill collector,
|
|
502
|
+
including requests without a runtime policy. Plugin command handlers still
|
|
503
|
+
require explicit hook authority. Projections are removed with their owning host
|
|
504
|
+
request.
|
|
505
|
+
|
|
219
506
|
Auxiliary `.mcp.json` and `.lsp.json` files and plugin mods are unsupported under
|
|
220
507
|
hook authority. Manifest MCP/LSP declarations are removed from curated mounts.
|
|
221
508
|
|
|
@@ -283,7 +570,7 @@ run evidence. Extensions and external benchmarks consume the
|
|
|
283
570
|
## Evidence and compatibility
|
|
284
571
|
|
|
285
572
|
`configuration.redacted.runtimePolicy` retains the selected environment,
|
|
286
|
-
canonical read roots, seed digests, and hook declaration. The engine snapshots
|
|
573
|
+
canonical read roots, seed digests, shared-cache declarations, and hook declaration. The engine snapshots
|
|
287
574
|
this input once; it participates in `configurationDigest` and evaluation
|
|
288
575
|
identity. Symbolic runtime placeholders remain symbolic in identity, while
|
|
289
576
|
execution uses per-role paths. This binds declared setup and seed contents;
|
|
@@ -297,7 +584,8 @@ file contents do not. The same discovery and skip evidence reaches
|
|
|
297
584
|
`sevro.host.runtime` observations.
|
|
298
585
|
|
|
299
586
|
Bundled hosts supply `sevro.host.runtime` with the applied policy digest, role,
|
|
300
|
-
environment names, read roots,
|
|
587
|
+
environment names, read roots, seed slots, and selected shared-cache declarations.
|
|
588
|
+
Shared entries contain no content digest. `sevro.host.hooks` records source
|
|
301
589
|
and effective plugin digests plus observed executions and their exit codes.
|
|
302
590
|
An empty execution list does not prove hook activation. Partial lifecycle
|
|
303
591
|
evidence remains partial. Cases can require these observations; missing required
|
|
@@ -307,7 +595,8 @@ task verdict.
|
|
|
307
595
|
`--toolchain-bin-dir` and `--claude-uv-cache-dir` are deprecated. They keep their
|
|
308
596
|
previous behavior with a warning when no runtime file is selected. Combining
|
|
309
597
|
them with a runtime file fails before execution. Migrate to `environment`,
|
|
310
|
-
`filesystem`, and private seed slots
|
|
598
|
+
`filesystem`, and either private seed slots or shared-cache declarations; removal
|
|
599
|
+
is no earlier than `0.2.0`. The deprecated UV option continues to copy privately.
|
|
311
600
|
See the [CLI reference](development-cli.md), [native hosts](native-hosts.md),
|
|
312
601
|
and [identity contract](identity-v1.md).
|
|
313
602
|
|
package/docs/troubleshooting.md
CHANGED
|
@@ -3,18 +3,18 @@
|
|
|
3
3
|
Start with execution, grading, task states, and stderr. A documented possible
|
|
4
4
|
cause is not a diagnosis of your machine.
|
|
5
5
|
|
|
6
|
-
| Symptom | Check and next step
|
|
7
|
-
| ----------------------------------------- |
|
|
8
|
-
| Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md).
|
|
9
|
-
| Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot.
|
|
10
|
-
| Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md).
|
|
11
|
-
| Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md).
|
|
12
|
-
| Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md).
|
|
13
|
-
| Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md).
|
|
14
|
-
| Native or shell isolation fails |
|
|
15
|
-
| Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md).
|
|
16
|
-
| Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout.
|
|
17
|
-
| External links fail | Follow [network failure triage](documentation-quality.md#external-links).
|
|
6
|
+
| Symptom | Check and next step |
|
|
7
|
+
| ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
8
|
+
| Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md). |
|
|
9
|
+
| Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot. |
|
|
10
|
+
| Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md). |
|
|
11
|
+
| Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md). |
|
|
12
|
+
| Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md). |
|
|
13
|
+
| Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md). |
|
|
14
|
+
| Native or shell isolation fails | On macOS, verify `sandbox-exec`; on Ubuntu 24.04 x64, verify `bubblewrap` and Claude's `socat`. See [validated scope](native-hosts.md#validated-platform-scope). Windows is unverified. |
|
|
15
|
+
| Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md). |
|
|
16
|
+
| Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout. |
|
|
17
|
+
| External links fail | Follow [network failure triage](documentation-quality.md#external-links). |
|
|
18
18
|
|
|
19
19
|
## The repository guide is missing
|
|
20
20
|
|