@bjoernrochel/sevro 0.1.0-rc.17 → 0.1.0-rc.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/development-cli.md +7 -2
- package/docs/evaluation-workflow.md +4 -1
- package/docs/identity-v1.md +4 -1
- package/docs/installing.md +2 -2
- package/docs/native-hosts.md +5 -2
- package/docs/reading-results.md +69 -0
- package/docs/recipes.md +7 -3
- package/docs/report-v1.md +16 -0
- package/docs/results-v1.md +51 -4
- package/docs/runtime-v1.md +60 -13
- package/package.json +1 -1
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/runtime-v1.schema.json +32 -0
- package/src/cli/completion.ts +7 -4
- package/src/cli/help.ts +35 -0
- package/src/cli.ts +3 -0
- package/src/evaluation/trial.ts +12 -1
- package/src/fixtures/cleanup.ts +13 -7
- package/src/generated/report.cjs +1 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/graders/shell.ts +14 -4
- package/src/hosts/claude-settings.ts +42 -5
- package/src/hosts/claude-state.ts +14 -2
- package/src/hosts/codex-state.ts +24 -3
- package/src/hosts/isolation-common.ts +2 -0
- package/src/hosts/linux-sandbox.ts +13 -0
- package/src/hosts/mac-sandbox.ts +23 -3
- package/src/hosts/skill-hook-handler.ts +52 -10
- package/src/native-transcript-view.ts +51 -12
- package/src/native-transcripts.ts +4 -3
- package/src/prune.ts +215 -0
- package/src/report.ts +19 -2
- package/src/results-lock.ts +29 -0
- package/src/runtime/config.ts +58 -10
- package/src/runtime/hooks.ts +19 -6
- package/src/runtime/shared-cache.ts +143 -0
- package/src/runtime/state.ts +25 -3
- package/src/schema-types.ts +1 -0
- package/src/transcript-retention.ts +146 -0
package/docs/development-cli.md
CHANGED
|
@@ -8,10 +8,15 @@ grading, evidence, and extension lifecycle.
|
|
|
8
8
|
[versioned report contract](report-v1.md). It emits Markdown by default or
|
|
9
9
|
`sevro.report.v1` with `--json`.
|
|
10
10
|
|
|
11
|
+
`sevro prune` [removes older retained results](reading-results.md#prune-retained-results)
|
|
12
|
+
and maintains the latest report index. Bare pruning keeps the newest ten results;
|
|
13
|
+
`--keep`, `--older-than`, and `--latest` select explicit retention policies.
|
|
14
|
+
|
|
11
15
|
Examples assume [Sevro and Bun are on `PATH`](installing.md#make-the-command-available).
|
|
12
16
|
For a contributor checkout, use `bun src/cli.ts` in place of `sevro`.
|
|
13
17
|
|
|
14
|
-
Use `sevro --help`, `sevro run --help`,
|
|
18
|
+
Use `sevro --help`, `sevro run --help`, `sevro report --help`, or
|
|
19
|
+
`sevro prune --help` for examples,
|
|
15
20
|
required options, and defaults without preparing a run. `sevro --version` prints
|
|
16
21
|
the installed package version. Invocation and configuration diagnostics go to
|
|
17
22
|
stderr in both output modes; `--json` keeps one `sevro.cli-result.v1` document
|
|
@@ -280,7 +285,7 @@ It runs candidate tools under Claude Code's native sandbox with private,
|
|
|
280
285
|
source, and result paths denied. The route disables hooks by default and does not expose
|
|
281
286
|
the Write tool. It currently supports passive candidate turns only. Use
|
|
282
287
|
`--protected-root` to exclude additional private paths from the candidate's
|
|
283
|
-
filesystem access. Configure shared host tools, private caches, goals, and selected
|
|
288
|
+
filesystem access. Configure shared host tools, private or reusable caches, goals, and selected
|
|
284
289
|
plugin hooks with [`sevro.json` or `--runtime-config-file`](runtime-v1.md).
|
|
285
290
|
The same policy reaches isolated checks and native grading hosts.
|
|
286
291
|
|
|
@@ -81,7 +81,10 @@ an earlier failed assessment into success.
|
|
|
81
81
|
`report --latest --results-root ./other-results` for another output root or working
|
|
82
82
|
directory. `--result-file ./path/to/result.json` selects older runs and accepts
|
|
83
83
|
relative paths. Reports validate referenced evidence; missing/moved evidence
|
|
84
|
-
produces a diagnostic.
|
|
84
|
+
produces a diagnostic. To reclaim native bundle space while keeping reportable
|
|
85
|
+
results and hashes, use [`sevro prune --native-transcripts`](reading-results.md#remove-native-transcript-bundles).
|
|
86
|
+
Reports label bundles intentionally removed by that command as pruned.
|
|
87
|
+
[Report v1](report-v1.md) defines fields and limits.
|
|
85
88
|
|
|
86
89
|
## Choose another starter
|
|
87
90
|
|
package/docs/identity-v1.md
CHANGED
|
@@ -87,7 +87,10 @@ contents and temporary native state paths do not enter identity.
|
|
|
87
87
|
Selected [runtime configuration](runtime-v1.md) is retained under
|
|
88
88
|
`configuration.redacted.runtimePolicy` and enters `configurationDigest`.
|
|
89
89
|
It includes declared non-secret environment values, canonical read roots,
|
|
90
|
-
private seed digests and optional role selections,
|
|
90
|
+
private seed digests and optional role selections, shared-cache sources/slots/roles,
|
|
91
|
+
and hook authority. Mutable shared-cache contents are excluded; equal identities
|
|
92
|
+
do not establish equal cache contents or trial independence in shared mode.
|
|
93
|
+
Per-role output paths are resolved
|
|
91
94
|
only during execution; runtime placeholders remain symbolic in identity.
|
|
92
95
|
Runtime roots also retain lexical aliases, metadata discovery sources,
|
|
93
96
|
and skipped optional directories. Moving or
|
package/docs/installing.md
CHANGED
|
@@ -23,10 +23,10 @@ Local installation records the selected version in the project lockfile.
|
|
|
23
23
|
|
|
24
24
|
## Current release
|
|
25
25
|
|
|
26
|
-
The current release is `0.1.0-rc.
|
|
26
|
+
The current release is `0.1.0-rc.19`. Install this exact version:
|
|
27
27
|
|
|
28
28
|
```sh
|
|
29
|
-
bun add --exact @bjoernrochel/sevro@0.1.0-rc.
|
|
29
|
+
bun add --exact @bjoernrochel/sevro@0.1.0-rc.19
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
<!-- sevro-current-release:end -->
|
package/docs/native-hosts.md
CHANGED
|
@@ -44,7 +44,7 @@ Repository skill mounts under `.claude/skills` require explicit
|
|
|
44
44
|
`--claude-project-settings`; otherwise repository dispatch is refused before
|
|
45
45
|
launch. The bundled route disables hooks by default and omits candidate Write
|
|
46
46
|
access. [Runtime configuration](runtime-v1.md) enables native goals and selected
|
|
47
|
-
plugin hooks and reuses declared host tools with private caches. These changes
|
|
47
|
+
plugin hooks and reuses declared host tools with private or explicitly shared caches. These changes
|
|
48
48
|
require `rc.3` or later. The CLI reference covers credential scrubbing
|
|
49
49
|
and follow-ups.
|
|
50
50
|
|
|
@@ -80,7 +80,10 @@ Page matching uses complete nonblank lines as anchors and exact fragments of
|
|
|
80
80
|
at least 32 string code units. Fragments with multiple possible positions in a
|
|
81
81
|
body cannot establish coverage. Recognizable unfinished fragments appear in
|
|
82
82
|
`partialSkills` and keep the observation partial; a later complete match resolves
|
|
83
|
-
them.
|
|
83
|
+
them. Within each tool result, text covered by unambiguous complete-body matches
|
|
84
|
+
does not also imply partial reads of skills sharing that text. Fragments returned
|
|
85
|
+
outside those complete-body spans or in other tool results remain independent
|
|
86
|
+
partial-read evidence. Comparison and anchor budgets also fail closed. Private receipts retain
|
|
84
87
|
body offsets rather than tool text; the public observation omits those offsets.
|
|
85
88
|
Shorter, transformed or unanchored fragments remain outside this matching scope.
|
|
86
89
|
Newly created or changed skill bodies are outside the prepared snapshot. A
|
package/docs/reading-results.md
CHANGED
|
@@ -43,6 +43,75 @@ For a contributor checkout, use `bun src/cli.ts` in place of `sevro`.
|
|
|
43
43
|
duration is unknown, not zero. Advisory recommendations remain independent
|
|
44
44
|
of the task verdict.
|
|
45
45
|
|
|
46
|
+
## Prune retained results
|
|
47
|
+
|
|
48
|
+
Run `sevro prune` to permanently delete older retained evidence. With no options,
|
|
49
|
+
it keeps the newest **10 results** under `<cwd>/.sevro/results`. Each completed
|
|
50
|
+
attempt counts separately, including repeated evaluations, dry runs, failures,
|
|
51
|
+
and interrupted runs that retained a CLI result.
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
sevro prune
|
|
55
|
+
sevro prune --results-root ./other-results --keep 5
|
|
56
|
+
sevro prune --older-than 30d
|
|
57
|
+
sevro prune --latest
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Choose at most one explicit policy. `--keep` accepts a positive safe integer;
|
|
61
|
+
`--latest` is equivalent to `--keep 1`. `--older-than` accepts a positive integer
|
|
62
|
+
followed by `ms`, `s`, `m`, `h`, `d`, or `w` (milliseconds through weeks). It
|
|
63
|
+
deletes results strictly older than that duration at invocation. Age and newest
|
|
64
|
+
ordering use each retained `result.json` file's modification time; copying or
|
|
65
|
+
editing that file can change its retention age. Equal timestamps sort by
|
|
66
|
+
absolute result-directory path, ascending. Relative roots resolve from the
|
|
67
|
+
invoking directory.
|
|
68
|
+
|
|
69
|
+
Pruning finds retained result directories inside the selected root, including
|
|
70
|
+
nested `attempts` trees, validates results and their colocated run evidence, and
|
|
71
|
+
removes each selected result's entire directory. Directories without a retained
|
|
72
|
+
`result.json` are not themselves deleted. Active and unrecognized ownership states
|
|
73
|
+
are excluded from the retention count. Descendant directory links are not
|
|
74
|
+
followed; linked result, evidence, ownership and lock files are refused. The
|
|
75
|
+
selected root itself may be a directory alias. Invalid policies or unreadable,
|
|
76
|
+
malformed, inconsistent or unsafe retained inputs return exit `64` with a
|
|
77
|
+
diagnostic. Validation completes before any result deletion or latest update.
|
|
78
|
+
|
|
79
|
+
Pruning and CLI result publication serialize access to the root. Before deleting
|
|
80
|
+
evidence, pruning atomically points `latest.json` at the newest remaining
|
|
81
|
+
completed result, or removes the index if none remain. `sevro report --latest`
|
|
82
|
+
and explicit reports for surviving results keep working; reports naming deleted
|
|
83
|
+
results fail. An empty or missing results root is a successful no-op. Ownership
|
|
84
|
+
coordination metadata, partial attempts without CLI results, temporary fixtures,
|
|
85
|
+
and a separate `--run-state-root` remain outside evidence pruning.
|
|
86
|
+
|
|
87
|
+
### Remove native transcript bundles
|
|
88
|
+
|
|
89
|
+
Use `--native-transcripts` to permanently remove captured native bundles while
|
|
90
|
+
keeping every result, recorded hash, observation, and other artifact:
|
|
91
|
+
|
|
92
|
+
```sh
|
|
93
|
+
sevro prune --native-transcripts
|
|
94
|
+
sevro prune --native-transcripts --older-than 30d
|
|
95
|
+
sevro prune --native-transcripts --keep 5
|
|
96
|
+
sevro report --latest
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Without a policy, this mode removes bundles from all completed results. With
|
|
100
|
+
`--keep 5`, it preserves the newest five results' bundles and removes bundles
|
|
101
|
+
from older results. `--latest` preserves the newest result's bundles;
|
|
102
|
+
`--older-than` removes bundles only from results older than the given duration.
|
|
103
|
+
Age, ordering, root selection, active-run protection, and input validation follow
|
|
104
|
+
the rules above. The mode includes candidate, semantic, and advisory bundles.
|
|
105
|
+
|
|
106
|
+
`result.json`, `run.json`, trial evidence, and `latest.json` stay unchanged.
|
|
107
|
+
Before removing bytes, pruning atomically saves a private
|
|
108
|
+
`native-transcripts-pruned.json` receipt beside `run.json`. Keep that receipt
|
|
109
|
+
with the evidence: reports require it to distinguish intentional pruning from
|
|
110
|
+
missing evidence. Reports show retained and pruned bundle counts while preserving
|
|
111
|
+
the original verdicts and measurements. Pruned native bytes cannot be decoded
|
|
112
|
+
or recovered from hashes. Repeating the command safely resumes interrupted
|
|
113
|
+
deletions; ordinary whole-result pruning still works afterward.
|
|
114
|
+
|
|
46
115
|
## Compare carefully
|
|
47
116
|
|
|
48
117
|
[Identity v1](identity-v1.md) binds inputs, evaluator content, host/model/effort,
|
package/docs/recipes.md
CHANGED
|
@@ -209,9 +209,13 @@ The negative fixture changes the port to `9000`; Python raises an assertion:
|
|
|
209
209
|
bun "$SEVRO_RECIPES/run.ts" python-uv deterministic --fail
|
|
210
210
|
```
|
|
211
211
|
|
|
212
|
-
For real dependencies, prepare a public cache
|
|
213
|
-
|
|
214
|
-
offline install command.
|
|
212
|
+
For real dependencies, prepare a public cache once outside the evaluated project
|
|
213
|
+
and declare a [shared cache](runtime-v1.md#reuse-a-shared-dependency-cache), then
|
|
214
|
+
use your locked offline install command. Select only the roles using UV; add
|
|
215
|
+
`checks` only when the grader needs it. The same directory can serve candidates,
|
|
216
|
+
hooks, and UV-based graders across trials and runs without copying the cache.
|
|
217
|
+
Use [private seeds](runtime-v1.md#private-writable-state) when isolated dependency
|
|
218
|
+
bytes matter more than disk reuse. This dependency-free recipe needs no cache seed.
|
|
215
219
|
|
|
216
220
|
### An existing mise installation
|
|
217
221
|
|
package/docs/report-v1.md
CHANGED
|
@@ -29,6 +29,22 @@ including dry and failed runs. Pre-run failures do not replace latest. Missing
|
|
|
29
29
|
latest or referenced evidence is an error. See
|
|
30
30
|
[the evaluation workflow](evaluation-workflow.md).
|
|
31
31
|
|
|
32
|
+
[`sevro prune`](reading-results.md#prune-retained-results) maintains this index
|
|
33
|
+
when removing retained evidence. After pruning, latest selects the newest
|
|
34
|
+
remaining completed result; if none remain, the index is removed and `--latest`
|
|
35
|
+
reports a missing-input diagnostic. Explicit reports still require their selected
|
|
36
|
+
results and evidence to exist.
|
|
37
|
+
|
|
38
|
+
After [`sevro prune --native-transcripts`](reading-results.md#remove-native-transcript-bundles),
|
|
39
|
+
reports accept absent native bundles only with the matching
|
|
40
|
+
[pruning receipt](results-v1.md#native-transcript-bundles). Present bundles must
|
|
41
|
+
still match their recorded hashes; unexplained missing bytes remain an error.
|
|
42
|
+
New report rows include optional `nativeTranscripts` with nonnegative integer
|
|
43
|
+
`retained` and `pruned` bundle counts across the case's trials and host roles.
|
|
44
|
+
Markdown shows the same counts beside retained evidence. Older report documents
|
|
45
|
+
without this field remain valid. Pruning does not change verdicts, routes,
|
|
46
|
+
identity, usage, or duration.
|
|
47
|
+
|
|
32
48
|
Each case row keeps task verdict, execution, and grading separate. Task pass
|
|
33
49
|
rate is measured only when every trial completed execution and grading with an
|
|
34
50
|
assessed task verdict. Candidate duration sums the measured host call time
|
package/docs/results-v1.md
CHANGED
|
@@ -142,11 +142,23 @@ results and evidence remain intact. Failed cleanup is retried on later evaluatio
|
|
|
142
142
|
`sevro.observation.workspaces` records candidate and retained auxiliary workspace
|
|
143
143
|
paths for inspection. These paths are temporary and do not affect evaluation identity.
|
|
144
144
|
|
|
145
|
+
Retained results have no automatic expiration. The explicit
|
|
146
|
+
[`sevro prune` command](reading-results.md#prune-retained-results) applies an
|
|
147
|
+
operator-selected age or count policy, defaulting to the newest ten results.
|
|
148
|
+
It deletes complete retained result directories and maintains `latest.json`
|
|
149
|
+
for the remaining results. Active or unrecognized attempts remain protected;
|
|
150
|
+
fixture cleanup and run-state coordination metadata have separate lifecycles.
|
|
151
|
+
|
|
145
152
|
Opted-in [candidate artifact runtimes](runtime-v1.md#candidate-artifact-access)
|
|
146
153
|
remain available through grading and are removed afterward, including failed
|
|
147
154
|
and cancelled trials. Failed-fixture retention does not preserve them.
|
|
148
155
|
Read access does not automatically retain artifact bytes in run evidence.
|
|
149
156
|
|
|
157
|
+
Checker private home, temporary storage, and copied dependency slots are removed
|
|
158
|
+
after grading on success, failure, and cancellation. Retained failed fixtures and
|
|
159
|
+
saved evidence do not retain checker runtime copies. Explicitly declared shared
|
|
160
|
+
caches survive this cleanup and remain under operator control.
|
|
161
|
+
|
|
150
162
|
A host adapter can return `executionFailed: true` with bounded artifacts from
|
|
151
163
|
a failed process. The engine retains those artifacts, skips all grading, and
|
|
152
164
|
reports `failed` / `not_requested` / `not_assessed` with exit code `2`. A final
|
|
@@ -212,14 +224,26 @@ failure; cleanup warnings contain no raw error/output bodies.
|
|
|
212
224
|
With [runtime transcript opt-in](runtime-v1.md#native-transcript-access), each
|
|
213
225
|
bundled host captures full native files before removing its private state.
|
|
214
226
|
Candidate artifact `sevro.native-transcripts.bundle` has format
|
|
215
|
-
`sevro.native-transcripts.
|
|
227
|
+
`sevro.native-transcripts.v2`. It contains `host` (`codex` or `claude`), `role`,
|
|
216
228
|
native `rootSessionId` (or `null`), `completeness`, `issues`, and `files`.
|
|
217
|
-
Each file records its native relative `path`, SHA-256 `sha256
|
|
218
|
-
|
|
229
|
+
Each file records its native relative `path`, SHA-256 `sha256` of the original
|
|
230
|
+
bytes, and `gzipBase64`: the exact file bytes compressed with gzip, then encoded
|
|
231
|
+
as base64. Decode base64, decompress gzip, and verify the original-byte hash
|
|
232
|
+
before using a file. Bound decompression to 5 MiB across all files in the bundle.
|
|
233
|
+
Decode bytes without rewriting native records. Native
|
|
219
234
|
session paths, root/child records and sidecar metadata preserve associations;
|
|
220
235
|
the bundle does not normalize model, effort, timing, token, or cost statistics.
|
|
221
236
|
Existing unknown metrics remain unknown.
|
|
222
237
|
|
|
238
|
+
Sevro reads both v2 bundles and existing `sevro.native-transcripts.v1` bundles.
|
|
239
|
+
V1 files use uncompressed `bytesBase64`; decode that field directly without
|
|
240
|
+
gzip decompression and verify the same original-byte hash. Readers must select
|
|
241
|
+
the payload field and decoding by `format` and reject unsupported formats,
|
|
242
|
+
corrupt compressed payloads, hash mismatches, or a decoded total above 5 MiB.
|
|
243
|
+
New captures emit v2; reading retained v1 evidence does not rewrite it.
|
|
244
|
+
The retained view exposes the same exact native files for either format, with
|
|
245
|
+
the original bundle in `index.json`.
|
|
246
|
+
|
|
223
247
|
`sevro.host.native-transcripts` records the same capture state and file hashes
|
|
224
248
|
without raw bytes. Semantic and advisory bundles retain their role in the
|
|
225
249
|
bundle and use the existing `sevro.semantic[.<source digest>].` and
|
|
@@ -230,7 +254,8 @@ benchmarks can use the same references and decode the native files.
|
|
|
230
254
|
|
|
231
255
|
Capture includes native transcript-tree sidecars and refuses descendant links,
|
|
232
256
|
special files and escaping paths. It is bounded to 4,096 directory entries,
|
|
233
|
-
5 MiB of native file bytes and 7 MiB of encoded file
|
|
257
|
+
5 MiB of native file bytes before compression and 7 MiB of encoded file
|
|
258
|
+
metadata/gzip content. The
|
|
234
259
|
bundle remains inside the existing 8 MiB item and 32 MiB host artifact budgets;
|
|
235
260
|
it groups sessions instead of spending an artifact per child. Limits produce
|
|
236
261
|
`byte_limit`, `entry_limit`, or `bundle_limit` issues. Unreadable, unsafe or
|
|
@@ -249,3 +274,25 @@ structure, not native semantic correctness.
|
|
|
249
274
|
An empty/missing native JSONL tree is `unavailable`. Failure, cancellation and
|
|
250
275
|
timeout retain available native bytes with incomplete capture state. Raw
|
|
251
276
|
bundles and views are private evidence and must stay out of Git.
|
|
277
|
+
|
|
278
|
+
The explicit [`sevro prune --native-transcripts` mode](reading-results.md#remove-native-transcript-bundles)
|
|
279
|
+
can remove these bundles without rewriting CLI results, run/trial evidence,
|
|
280
|
+
capture observations, native file hashes, or whole-bundle hashes. It retains
|
|
281
|
+
other artifacts and does not change evaluation identity or assessment states.
|
|
282
|
+
Reports validate present native bundles against their recorded SHA-256 and
|
|
283
|
+
require an exact pruning receipt for absent bundles.
|
|
284
|
+
|
|
285
|
+
Before deletion, Sevro atomically writes private
|
|
286
|
+
`native-transcripts-pruned.json` beside `run.json`, with format
|
|
287
|
+
`sevro.pruned-native-transcripts.v1`, the original `runId`, and `artifacts`.
|
|
288
|
+
Each entry contains the original artifact reference (`id`, `path`, `sha256`,
|
|
289
|
+
and any optional reference flags), plus its `caseId` and `trial`. Entries follow
|
|
290
|
+
run trial/reference order and cover every candidate, semantic (including
|
|
291
|
+
source-specific), and advisory native bundle in that run. A receipt must match
|
|
292
|
+
these original references exactly; malformed, inconsistent, or linked receipts
|
|
293
|
+
are errors. Bundle references must name regular files directly inside their
|
|
294
|
+
result directory. Missing bytes without a receipt, changed hashes, and escaping
|
|
295
|
+
references remain errors. An interrupted deletion may leave some marked bundles
|
|
296
|
+
present; reports count actual availability, and a repeated prune completes removal.
|
|
297
|
+
Transcript references that alias result/trial metadata, raw responses, or other
|
|
298
|
+
retained artifacts are refused before pruning.
|
package/docs/runtime-v1.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Runtime configuration v1
|
|
2
2
|
|
|
3
|
-
Configure access to existing host tools, private
|
|
3
|
+
Configure access to existing host tools, private state, reusable caches, native goals,
|
|
4
4
|
and selected plugin hooks in `<project-root>/sevro.json`.
|
|
5
5
|
|
|
6
6
|
`--runtime-config-file /absolute/path/runtime.json` selects a replacement file.
|
|
@@ -36,7 +36,13 @@ capabilities:
|
|
|
36
36
|
}
|
|
37
37
|
},
|
|
38
38
|
"runtime": {
|
|
39
|
-
"
|
|
39
|
+
"sharedCacheDirectories": [
|
|
40
|
+
{
|
|
41
|
+
"source": "../public-uv-cache",
|
|
42
|
+
"target": "uv-cache",
|
|
43
|
+
"roles": ["candidate", "hooks"]
|
|
44
|
+
}
|
|
45
|
+
]
|
|
40
46
|
},
|
|
41
47
|
"hooks": {
|
|
42
48
|
"nativeGoal": true,
|
|
@@ -45,7 +51,7 @@ capabilities:
|
|
|
45
51
|
}
|
|
46
52
|
```
|
|
47
53
|
|
|
48
|
-
|
|
54
|
+
Prepare the example's `public-uv-cache` directory outside the project, or
|
|
49
55
|
omit `runtime` when no prepared cache is needed. Selected plugin names must be
|
|
50
56
|
present in the case's prepared mounts; remove names that the case does not mount.
|
|
51
57
|
`DARROW_CACHE_DIR` is an ordinary explicit variable in this example, not a
|
|
@@ -235,10 +241,12 @@ digits, and hyphens. `home` and `tmp` are reserved. Targets must be unique.
|
|
|
235
241
|
An optional nonempty `roles` array selects any of `candidate`, `checks`, `hooks`,
|
|
236
242
|
`semantic`, and `advisory`. For example, `"roles": ["candidate", "checks", "hooks"]`
|
|
237
243
|
keeps dependency slots out of semantic and advisory runtimes. Without `roles`,
|
|
238
|
-
|
|
244
|
+
candidate, hook, artifact semantic, and advisory roles receive private copies.
|
|
245
|
+
Shell checks require explicit `"checks"` selection; a non-UV grader receives no
|
|
246
|
+
implicit UV/Python cache. Existing configurations whose shell checks need seeded
|
|
247
|
+
dependencies must add `"checks"` to `roles`. Response-only
|
|
239
248
|
semantic grading omits implicit seeds; include `"semantic"` explicitly in `roles`
|
|
240
|
-
when that route needs dependencies.
|
|
241
|
-
shell checks keep their declared tools and default seed selection.
|
|
249
|
+
when that route needs dependencies. Declared host tools remain available to shell checks.
|
|
242
250
|
|
|
243
251
|
Each source tree is bounded to 16,384 entries and 512 MiB. Escaping symlinks,
|
|
244
252
|
directory cycles, dangling links, and special files are refused. File content
|
|
@@ -254,6 +262,41 @@ retaining large policies can retain up to the configured per-seed byte bounds.
|
|
|
254
262
|
Source caches are never a writable runtime. Use prepared public caches rather
|
|
255
263
|
than credential-bearing homes or evaluator directories.
|
|
256
264
|
|
|
265
|
+
### Reuse a shared dependency cache
|
|
266
|
+
|
|
267
|
+
`runtime.sharedCacheDirectories` opts into persistent writable caches without
|
|
268
|
+
copying or hashing their contents for each trial. Each entry requires `source`,
|
|
269
|
+
`target`, and a nonempty `roles` array drawn from the same five roles above.
|
|
270
|
+
The source must already be an owned, readable, writable directory. Relative paths
|
|
271
|
+
resolve beside the runtime file. Sevro resolves declaration symlinks to their
|
|
272
|
+
canonical source and revalidates that directory before execution.
|
|
273
|
+
Targets use the seed slot rules and must be unique across seeds and shared caches.
|
|
274
|
+
Sources cannot overlap each other, private seed sources, declared read-only
|
|
275
|
+
inputs, evaluated project/source trees, results, credentials, or native private state.
|
|
276
|
+
|
|
277
|
+
Select only roles that actually use UV, for example `"roles": ["candidate", "hooks"]`.
|
|
278
|
+
If a grader also uses UV, add `"checks"` to that same entry; both roles use the
|
|
279
|
+
same cache. Each selected private slot aliases the original directory, with a
|
|
280
|
+
narrow sandbox write grant. Other roles receive no slot or source grant.
|
|
281
|
+
Shared content survives successful, failed, and cancelled trials and later runs.
|
|
282
|
+
Sevro removes only disposable role state, never the declared shared source.
|
|
283
|
+
|
|
284
|
+
Prepare UV packages once outside evaluation, point `UV_CACHE_DIR` at
|
|
285
|
+
`{{sevro.runtime}}/uv-cache`, and reuse the existing Python installation through
|
|
286
|
+
declared read-only tool roots. Set `UV_PYTHON_DOWNLOADS=never` to require that
|
|
287
|
+
installation. Sevro keeps `UV_OFFLINE=1`; a missing dependency fails rather than
|
|
288
|
+
downloading during a trial. Project virtual environments and temporary files
|
|
289
|
+
remain disposable; UV's install/link mode can still allocate files in a virtual
|
|
290
|
+
environment. The shared option removes the full cache copy per trial.
|
|
291
|
+
|
|
292
|
+
Shared caches are mutable across trials, roles, and runs. Use them only for public
|
|
293
|
+
dependencies and accept that a participant can change them for later participants.
|
|
294
|
+
Use isolated seeds when dependency independence is required. Configuration identity
|
|
295
|
+
binds the source, slot, and roles, but excludes mutable cache contents; equal
|
|
296
|
+
identities therefore do not prove identical shared-cache bytes. Sevro does not
|
|
297
|
+
age-delete shared caches. Manage their size between evaluations with the cache
|
|
298
|
+
tool, such as `uv cache prune --cache-dir /absolute/public-uv-cache`.
|
|
299
|
+
|
|
257
300
|
`{{sevro.runtime}}` in an environment value resolves to that role's writable
|
|
258
301
|
root. Codex commands receive a `TMPPREFIX` inside their private `TMPDIR`,
|
|
259
302
|
so zsh here-documents use their existing private temporary storage while shared
|
|
@@ -266,13 +309,15 @@ use. Its temporary directory stays disposable.
|
|
|
266
309
|
Candidate, hook, checker, semantic, and advisory
|
|
267
310
|
state are separate. With a runtime policy, native candidate, semantic, and
|
|
268
311
|
advisory roots live in a unique runner-owned host directory outside the Git
|
|
269
|
-
fixture. The root contains `home`, `tmp`, and
|
|
312
|
+
fixture. The root contains `home`, `tmp`, private seed slots and selected shared-cache links; plugins can
|
|
270
313
|
use paths such as `~/.darrow/reviews` without creating repository-local review
|
|
271
314
|
state. Command runtime directories are separate from host authentication,
|
|
272
|
-
configuration, and live transcript sources. Checker and hook
|
|
273
|
-
outside candidate-controlled workspace state. Private roots survive native
|
|
315
|
+
configuration, and live transcript sources. Checker and hook private state also lives
|
|
316
|
+
outside candidate-controlled workspace state. Shared sources remain at their declared locations. Private roots survive native
|
|
274
317
|
continuation turns and children, then are removed with their owning trial or
|
|
275
|
-
host state after owned hooks terminate.
|
|
318
|
+
host state after owned hooks terminate. Checker home, temporary storage, and copied
|
|
319
|
+
seeds are removed after grading even when the fixture is retained after failure
|
|
320
|
+
or cancellation. Results and evidence are preserved. Opted-in candidate artifact runtimes
|
|
276
321
|
remain alive through grading as described below. Runs with neither fixture setup
|
|
277
322
|
nor a selected runtime policy
|
|
278
323
|
retain their previous writable-state locations.
|
|
@@ -525,7 +570,7 @@ run evidence. Extensions and external benchmarks consume the
|
|
|
525
570
|
## Evidence and compatibility
|
|
526
571
|
|
|
527
572
|
`configuration.redacted.runtimePolicy` retains the selected environment,
|
|
528
|
-
canonical read roots, seed digests, and hook declaration. The engine snapshots
|
|
573
|
+
canonical read roots, seed digests, shared-cache declarations, and hook declaration. The engine snapshots
|
|
529
574
|
this input once; it participates in `configurationDigest` and evaluation
|
|
530
575
|
identity. Symbolic runtime placeholders remain symbolic in identity, while
|
|
531
576
|
execution uses per-role paths. This binds declared setup and seed contents;
|
|
@@ -539,7 +584,8 @@ file contents do not. The same discovery and skip evidence reaches
|
|
|
539
584
|
`sevro.host.runtime` observations.
|
|
540
585
|
|
|
541
586
|
Bundled hosts supply `sevro.host.runtime` with the applied policy digest, role,
|
|
542
|
-
environment names, read roots,
|
|
587
|
+
environment names, read roots, seed slots, and selected shared-cache declarations.
|
|
588
|
+
Shared entries contain no content digest. `sevro.host.hooks` records source
|
|
543
589
|
and effective plugin digests plus observed executions and their exit codes.
|
|
544
590
|
An empty execution list does not prove hook activation. Partial lifecycle
|
|
545
591
|
evidence remains partial. Cases can require these observations; missing required
|
|
@@ -549,7 +595,8 @@ task verdict.
|
|
|
549
595
|
`--toolchain-bin-dir` and `--claude-uv-cache-dir` are deprecated. They keep their
|
|
550
596
|
previous behavior with a warning when no runtime file is selected. Combining
|
|
551
597
|
them with a runtime file fails before execution. Migrate to `environment`,
|
|
552
|
-
`filesystem`, and private seed slots
|
|
598
|
+
`filesystem`, and either private seed slots or shared-cache declarations; removal
|
|
599
|
+
is no earlier than `0.2.0`. The deprecated UV option continues to copy privately.
|
|
553
600
|
See the [CLI reference](development-cli.md), [native hosts](native-hosts.md),
|
|
554
601
|
and [identity contract](identity-v1.md).
|
|
555
602
|
|
package/package.json
CHANGED
|
@@ -110,6 +110,15 @@
|
|
|
110
110
|
"domainOutcomes"
|
|
111
111
|
],
|
|
112
112
|
"properties": {
|
|
113
|
+
"nativeTranscripts": {
|
|
114
|
+
"type": "object",
|
|
115
|
+
"required": ["retained", "pruned"],
|
|
116
|
+
"properties": {
|
|
117
|
+
"retained": { "type": "integer", "minimum": 0 },
|
|
118
|
+
"pruned": { "type": "integer", "minimum": 0 }
|
|
119
|
+
},
|
|
120
|
+
"additionalProperties": false
|
|
121
|
+
},
|
|
113
122
|
"resultFile": { "$ref": "#/$defs/path" },
|
|
114
123
|
"evidencePath": { "type": ["string", "null"] },
|
|
115
124
|
"runId": { "type": ["string", "null"] },
|
|
@@ -62,6 +62,38 @@
|
|
|
62
62
|
}
|
|
63
63
|
}
|
|
64
64
|
},
|
|
65
|
+
"sharedCacheDirectories": {
|
|
66
|
+
"type": "array",
|
|
67
|
+
"maxItems": 16,
|
|
68
|
+
"items": {
|
|
69
|
+
"type": "object",
|
|
70
|
+
"additionalProperties": false,
|
|
71
|
+
"required": ["source", "target", "roles"],
|
|
72
|
+
"properties": {
|
|
73
|
+
"source": { "type": "string", "minLength": 1, "maxLength": 4096 },
|
|
74
|
+
"target": {
|
|
75
|
+
"type": "string",
|
|
76
|
+
"pattern": "^[a-z][a-z0-9-]{0,63}$",
|
|
77
|
+
"not": { "enum": ["home", "tmp"] }
|
|
78
|
+
},
|
|
79
|
+
"roles": {
|
|
80
|
+
"type": "array",
|
|
81
|
+
"minItems": 1,
|
|
82
|
+
"maxItems": 5,
|
|
83
|
+
"uniqueItems": true,
|
|
84
|
+
"items": {
|
|
85
|
+
"enum": [
|
|
86
|
+
"candidate",
|
|
87
|
+
"checks",
|
|
88
|
+
"hooks",
|
|
89
|
+
"semantic",
|
|
90
|
+
"advisory"
|
|
91
|
+
]
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
},
|
|
65
97
|
"seedDirectories": {
|
|
66
98
|
"type": "array",
|
|
67
99
|
"maxItems": 16,
|
package/src/cli/completion.ts
CHANGED
|
@@ -2,11 +2,14 @@ import { dirname, join } from "node:path";
|
|
|
2
2
|
import { atomicWriteJson } from "../storage";
|
|
3
3
|
import type { CliResult } from "../engine";
|
|
4
4
|
import { createReport } from "../report";
|
|
5
|
+
import { withResultsLock } from "../results-lock";
|
|
5
6
|
|
|
6
7
|
/** Latest means the last successfully retained CLI result, including dry/failing runs. */
|
|
7
8
|
export async function retainCliResult(result: CliResult, resultsRoot: string) {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
return withResultsLock(resultsRoot, async () => {
|
|
10
|
+
const resultFile = join(dirname(result.evidencePath), "result.json");
|
|
11
|
+
await atomicWriteJson(resultFile, result);
|
|
12
|
+
await atomicWriteJson(join(resultsRoot, "latest.json"), result);
|
|
13
|
+
return { resultFile, report: await createReport([resultFile]) };
|
|
14
|
+
});
|
|
12
15
|
}
|
package/src/cli/help.ts
CHANGED
|
@@ -10,12 +10,14 @@ Examples:
|
|
|
10
10
|
--condition passive --trials 1 --threshold 1
|
|
11
11
|
sevro run --help
|
|
12
12
|
sevro report --result-file /absolute/results/run/result.json
|
|
13
|
+
sevro prune
|
|
13
14
|
|
|
14
15
|
Commands:
|
|
15
16
|
demo Run the packaged credential-free assessed example; use demo --help.
|
|
16
17
|
init Create a new evaluation project; use init --help.
|
|
17
18
|
run Evaluate one resolved case or extension case; use run --help.
|
|
18
19
|
report Summarize retained results; use report --help.
|
|
20
|
+
prune Remove retained results using a retention policy; use prune --help.
|
|
19
21
|
|
|
20
22
|
Options:
|
|
21
23
|
-h, --help Show help without preparing a run.
|
|
@@ -131,6 +133,36 @@ Options and defaults:
|
|
|
131
133
|
-h, --help Show this help without reading result files.
|
|
132
134
|
`;
|
|
133
135
|
|
|
136
|
+
const PRUNE_HELP = `Usage: sevro prune [options]
|
|
137
|
+
|
|
138
|
+
Examples:
|
|
139
|
+
sevro prune
|
|
140
|
+
sevro prune --results-root ./other-results --keep 5
|
|
141
|
+
sevro prune --older-than 30d
|
|
142
|
+
sevro prune --latest
|
|
143
|
+
sevro prune --native-transcripts --older-than 30d
|
|
144
|
+
|
|
145
|
+
Defaults: keep the newest 10 completed results under <cwd>/.sevro/results.
|
|
146
|
+
Select at most one explicit retention policy:
|
|
147
|
+
--keep <positive-integer> Keep this many newest results.
|
|
148
|
+
--latest Keep only the newest result (same as --keep 1).
|
|
149
|
+
--older-than <duration> Remove results older than a positive integer plus ms, s, m, h, d or w.
|
|
150
|
+
--results-root <path> Relative or absolute results root, resolved from cwd.
|
|
151
|
+
--native-transcripts Remove only native transcript bundles; keep results and hashes.
|
|
152
|
+
-h, --help Show help without reading or deleting results.
|
|
153
|
+
|
|
154
|
+
Age and ordering use result.json modification times; equal times sort by path.
|
|
155
|
+
Each retained attempt counts separately, including dry, failed and interrupted runs.
|
|
156
|
+
Active/unrecognized attempts and directories without result.json are preserved.
|
|
157
|
+
Evidence is permanently deleted. Remaining results stay usable by sevro report;
|
|
158
|
+
latest.json is updated, or removed when no completed results remain.
|
|
159
|
+
With --native-transcripts, no policy removes bundles from all completed results;
|
|
160
|
+
--keep/--latest preserve bundles for the newest results, and --older-than selects older results.
|
|
161
|
+
Result/evidence files and latest.json stay unchanged. Reports label removed bundles as pruned.
|
|
162
|
+
Missing or empty results roots are a successful no-op. Invalid or unreadable inputs exit 64.
|
|
163
|
+
Reference: docs/reading-results.md in the installed package.
|
|
164
|
+
`;
|
|
165
|
+
|
|
134
166
|
const INIT_HELP = `Usage: sevro init [options]
|
|
135
167
|
|
|
136
168
|
Examples:
|
|
@@ -169,6 +201,9 @@ const HELP_REQUESTS: Record<string, string> = {
|
|
|
169
201
|
"report --help": REPORT_HELP,
|
|
170
202
|
"report -h": REPORT_HELP,
|
|
171
203
|
"help report": REPORT_HELP,
|
|
204
|
+
"prune --help": PRUNE_HELP,
|
|
205
|
+
"prune -h": PRUNE_HELP,
|
|
206
|
+
"help prune": PRUNE_HELP,
|
|
172
207
|
};
|
|
173
208
|
|
|
174
209
|
/** Handle informational commands before loading inputs or creating run state. */
|
package/src/cli.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { reportCommand } from "./report";
|
|
|
4
4
|
import { informationCommand } from "./cli/help";
|
|
5
5
|
import { demoCommand } from "./cli/demo";
|
|
6
6
|
import { initCommand } from "./cli/init";
|
|
7
|
+
import { pruneCommand } from "./prune";
|
|
7
8
|
|
|
8
9
|
if (informationCommand(process.argv.slice(2))) process.exitCode = 0;
|
|
9
10
|
else if (process.argv[2] === "demo") await demoCommand(process.argv.slice(3));
|
|
@@ -11,4 +12,6 @@ else if (process.argv[2] === "init")
|
|
|
11
12
|
process.exitCode = await initCommand(process.argv.slice(3));
|
|
12
13
|
else if (process.argv[2] === "report")
|
|
13
14
|
process.exitCode = await reportCommand(process.argv.slice(2));
|
|
15
|
+
else if (process.argv[2] === "prune")
|
|
16
|
+
process.exitCode = await pruneCommand(process.argv.slice(3));
|
|
14
17
|
else await main(process.argv.slice(2));
|
package/src/evaluation/trial.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { rm } from "node:fs/promises";
|
|
2
|
+
import { hashJson } from "../identity";
|
|
2
3
|
import {
|
|
3
4
|
allocateCandidateRuntime,
|
|
4
5
|
candidateArtifactRoots,
|
|
@@ -38,7 +39,10 @@ import type {
|
|
|
38
39
|
ObservationCompleteness,
|
|
39
40
|
TaskVerdict,
|
|
40
41
|
} from "../contract-types";
|
|
41
|
-
import {
|
|
42
|
+
import {
|
|
43
|
+
clearFixtureContents,
|
|
44
|
+
removeDisposableDirectory,
|
|
45
|
+
} from "../fixtures/cleanup";
|
|
42
46
|
import { retainWorkspace } from "../fixtures/workspace-state";
|
|
43
47
|
import { gitHeadRevision } from "../graders/git-head";
|
|
44
48
|
import {
|
|
@@ -789,6 +793,13 @@ class EvaluationTrial {
|
|
|
789
793
|
evaluationProgress(this.options, stage, status, this.trial);
|
|
790
794
|
}
|
|
791
795
|
private async cleanup(): Promise<void> {
|
|
796
|
+
await removeDisposableDirectory(
|
|
797
|
+
join(
|
|
798
|
+
this.context.resources.stateDir,
|
|
799
|
+
"shell-sandbox/runtime",
|
|
800
|
+
hashJson(this.fixture.workspace),
|
|
801
|
+
),
|
|
802
|
+
);
|
|
792
803
|
if (this.state.candidateTranscriptRoot)
|
|
793
804
|
await rm(this.state.candidateTranscriptRoot, {
|
|
794
805
|
recursive: true,
|