@bjoernrochel/sevro 0.1.0-rc.17 → 0.1.0-rc.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,10 +8,15 @@ grading, evidence, and extension lifecycle.
8
8
  [versioned report contract](report-v1.md). It emits Markdown by default or
9
9
  `sevro.report.v1` with `--json`.
10
10
 
11
+ `sevro prune` [removes older retained results](reading-results.md#prune-retained-results)
12
+ and maintains the latest report index. Bare pruning keeps the newest ten results;
13
+ `--keep`, `--older-than`, and `--latest` select explicit retention policies.
14
+
11
15
  Examples assume [Sevro and Bun are on `PATH`](installing.md#make-the-command-available).
12
16
  For a contributor checkout, use `bun src/cli.ts` in place of `sevro`.
13
17
 
14
- Use `sevro --help`, `sevro run --help`, or `sevro report --help` for examples,
18
+ Use `sevro --help`, `sevro run --help`, `sevro report --help`, or
19
+ `sevro prune --help` for examples,
15
20
  required options, and defaults without preparing a run. `sevro --version` prints
16
21
  the installed package version. Invocation and configuration diagnostics go to
17
22
  stderr in both output modes; `--json` keeps one `sevro.cli-result.v1` document
@@ -280,7 +285,7 @@ It runs candidate tools under Claude Code's native sandbox with private,
280
285
  source, and result paths denied. The route disables hooks by default and does not expose
281
286
  the Write tool. It currently supports passive candidate turns only. Use
282
287
  `--protected-root` to exclude additional private paths from the candidate's
283
- filesystem access. Configure shared host tools, private caches, goals, and selected
288
+ filesystem access. Configure shared host tools, private or reusable caches, goals, and selected
284
289
  plugin hooks with [`sevro.json` or `--runtime-config-file`](runtime-v1.md).
285
290
  The same policy reaches isolated checks and native grading hosts.
286
291
 
@@ -81,7 +81,10 @@ an earlier failed assessment into success.
81
81
  `report --latest --results-root ./other-results` for another output root or working
82
82
  directory. `--result-file ./path/to/result.json` selects older runs and accepts
83
83
  relative paths. Reports validate referenced evidence; missing/moved evidence
84
- produces a diagnostic. [Report v1](report-v1.md) defines fields and limits.
84
+ produces a diagnostic. To reclaim native bundle space while keeping reportable
85
+ results and hashes, use [`sevro prune --native-transcripts`](reading-results.md#remove-native-transcript-bundles).
86
+ Reports label bundles intentionally removed by that command as pruned.
87
+ [Report v1](report-v1.md) defines fields and limits.
85
88
 
86
89
  ## Choose another starter
87
90
 
@@ -87,7 +87,10 @@ contents and temporary native state paths do not enter identity.
87
87
  Selected [runtime configuration](runtime-v1.md) is retained under
88
88
  `configuration.redacted.runtimePolicy` and enters `configurationDigest`.
89
89
  It includes declared non-secret environment values, canonical read roots,
90
- private seed digests and optional role selections, and hook authority. Per-role output paths are resolved
90
+ private seed digests and optional role selections, shared-cache sources/slots/roles,
91
+ and hook authority. Mutable shared-cache contents are excluded; equal identities
92
+ do not establish equal cache contents or trial independence in shared mode.
93
+ Per-role output paths are resolved
91
94
  only during execution; runtime placeholders remain symbolic in identity.
92
95
  Runtime roots also retain lexical aliases, metadata discovery sources,
93
96
  and skipped optional directories. Moving or
@@ -23,10 +23,10 @@ Local installation records the selected version in the project lockfile.
23
23
 
24
24
  ## Current release
25
25
 
26
- The current release is `0.1.0-rc.17`. Install this exact version:
26
+ The current release is `0.1.0-rc.19`. Install this exact version:
27
27
 
28
28
  ```sh
29
- bun add --exact @bjoernrochel/sevro@0.1.0-rc.17
29
+ bun add --exact @bjoernrochel/sevro@0.1.0-rc.19
30
30
  ```
31
31
 
32
32
  <!-- sevro-current-release:end -->
@@ -44,7 +44,7 @@ Repository skill mounts under `.claude/skills` require explicit
44
44
  `--claude-project-settings`; otherwise repository dispatch is refused before
45
45
  launch. The bundled route disables hooks by default and omits candidate Write
46
46
  access. [Runtime configuration](runtime-v1.md) enables native goals and selected
47
- plugin hooks and reuses declared host tools with private caches. These changes
47
+ plugin hooks and reuses declared host tools with private or explicitly shared caches. These changes
48
48
  require `rc.3` or later. The CLI reference covers credential scrubbing
49
49
  and follow-ups.
50
50
 
@@ -80,7 +80,10 @@ Page matching uses complete nonblank lines as anchors and exact fragments of
80
80
  at least 32 string code units. Fragments with multiple possible positions in a
81
81
  body cannot establish coverage. Recognizable unfinished fragments appear in
82
82
  `partialSkills` and keep the observation partial; a later complete match resolves
83
- them. Comparison and anchor budgets also fail closed. Private receipts retain
83
+ them. Within each tool result, text covered by unambiguous complete-body matches
84
+ does not also imply partial reads of skills sharing that text. Fragments returned
85
+ outside those complete-body spans or in other tool results remain independent
86
+ partial-read evidence. Comparison and anchor budgets also fail closed. Private receipts retain
84
87
  body offsets rather than tool text; the public observation omits those offsets.
85
88
  Shorter, transformed or unanchored fragments remain outside this matching scope.
86
89
  Newly created or changed skill bodies are outside the prepared snapshot. A
@@ -43,6 +43,75 @@ For a contributor checkout, use `bun src/cli.ts` in place of `sevro`.
43
43
  duration is unknown, not zero. Advisory recommendations remain independent
44
44
  of the task verdict.
45
45
 
46
+ ## Prune retained results
47
+
48
+ Run `sevro prune` to permanently delete older retained evidence. With no options,
49
+ it keeps the newest **10 results** under `<cwd>/.sevro/results`. Each completed
50
+ attempt counts separately, including repeated evaluations, dry runs, failures,
51
+ and interrupted runs that retained a CLI result.
52
+
53
+ ```sh
54
+ sevro prune
55
+ sevro prune --results-root ./other-results --keep 5
56
+ sevro prune --older-than 30d
57
+ sevro prune --latest
58
+ ```
59
+
60
+ Choose at most one explicit policy. `--keep` accepts a positive safe integer;
61
+ `--latest` is equivalent to `--keep 1`. `--older-than` accepts a positive integer
62
+ followed by `ms`, `s`, `m`, `h`, `d`, or `w` (milliseconds through weeks). It
63
+ deletes results strictly older than that duration at invocation. Age and newest
64
+ ordering use each retained `result.json` file's modification time; copying or
65
+ editing that file can change its retention age. Equal timestamps sort by
66
+ absolute result-directory path, ascending. Relative roots resolve from the
67
+ invoking directory.
68
+
69
+ Pruning finds retained result directories inside the selected root, including
70
+ nested `attempts` trees, validates results and their colocated run evidence, and
71
+ removes each selected result's entire directory. Directories without a retained
72
+ `result.json` are not themselves deleted. Active and unrecognized ownership states
73
+ are excluded from the retention count. Descendant directory links are not
74
+ followed; linked result, evidence, ownership and lock files are refused. The
75
+ selected root itself may be a directory alias. Invalid policies or unreadable,
76
+ malformed, inconsistent or unsafe retained inputs return exit `64` with a
77
+ diagnostic. Validation completes before any result deletion or latest update.
78
+
79
+ Pruning and CLI result publication serialize access to the root. Before deleting
80
+ evidence, pruning atomically points `latest.json` at the newest remaining
81
+ completed result, or removes the index if none remain. `sevro report --latest`
82
+ and explicit reports for surviving results keep working; reports naming deleted
83
+ results fail. An empty or missing results root is a successful no-op. Ownership
84
+ coordination metadata, partial attempts without CLI results, temporary fixtures,
85
+ and a separate `--run-state-root` remain outside evidence pruning.
86
+
87
+ ### Remove native transcript bundles
88
+
89
+ Use `--native-transcripts` to permanently remove captured native bundles while
90
+ keeping every result, recorded hash, observation, and other artifact:
91
+
92
+ ```sh
93
+ sevro prune --native-transcripts
94
+ sevro prune --native-transcripts --older-than 30d
95
+ sevro prune --native-transcripts --keep 5
96
+ sevro report --latest
97
+ ```
98
+
99
+ Without a policy, this mode removes bundles from all completed results. With
100
+ `--keep 5`, it preserves the newest five results' bundles and removes bundles
101
+ from older results. `--latest` preserves the newest result's bundles;
102
+ `--older-than` removes bundles only from results older than the given duration.
103
+ Age, ordering, root selection, active-run protection, and input validation follow
104
+ the rules above. The mode includes candidate, semantic, and advisory bundles.
105
+
106
+ `result.json`, `run.json`, trial evidence, and `latest.json` stay unchanged.
107
+ Before removing bytes, pruning atomically saves a private
108
+ `native-transcripts-pruned.json` receipt beside `run.json`. Keep that receipt
109
+ with the evidence: reports require it to distinguish intentional pruning from
110
+ missing evidence. Reports show retained and pruned bundle counts while preserving
111
+ the original verdicts and measurements. Pruned native bytes cannot be decoded
112
+ or recovered from hashes. Repeating the command safely resumes interrupted
113
+ deletions; ordinary whole-result pruning still works afterward.
114
+
46
115
  ## Compare carefully
47
116
 
48
117
  [Identity v1](identity-v1.md) binds inputs, evaluator content, host/model/effort,
package/docs/recipes.md CHANGED
@@ -209,9 +209,13 @@ The negative fixture changes the port to `9000`; Python raises an assertion:
209
209
  bun "$SEVRO_RECIPES/run.ts" python-uv deterministic --fail
210
210
  ```
211
211
 
212
- For real dependencies, prepare a public cache before evaluation and add a
213
- declared [seed directory](runtime-v1.md#private-writable-state), then use your locked
214
- offline install command. This dependency-free recipe needs no cache seed.
212
+ For real dependencies, prepare a public cache once outside the evaluated project
213
+ and declare a [shared cache](runtime-v1.md#reuse-a-shared-dependency-cache), then
214
+ use your locked offline install command. Select only the roles using UV; add
215
+ `checks` only when the grader needs it. The same directory can serve candidates,
216
+ hooks, and UV-based graders across trials and runs without copying the cache.
217
+ Use [private seeds](runtime-v1.md#private-writable-state) when isolated dependency
218
+ bytes matter more than disk reuse. This dependency-free recipe needs no cache seed.
215
219
 
216
220
  ### An existing mise installation
217
221
 
package/docs/report-v1.md CHANGED
@@ -29,6 +29,22 @@ including dry and failed runs. Pre-run failures do not replace latest. Missing
29
29
  latest or referenced evidence is an error. See
30
30
  [the evaluation workflow](evaluation-workflow.md).
31
31
 
32
+ [`sevro prune`](reading-results.md#prune-retained-results) maintains this index
33
+ when removing retained evidence. After pruning, latest selects the newest
34
+ remaining completed result; if none remain, the index is removed and `--latest`
35
+ reports a missing-input diagnostic. Explicit reports still require their selected
36
+ results and evidence to exist.
37
+
38
+ After [`sevro prune --native-transcripts`](reading-results.md#remove-native-transcript-bundles),
39
+ reports accept absent native bundles only with the matching
40
+ [pruning receipt](results-v1.md#native-transcript-bundles). Present bundles must
41
+ still match their recorded hashes; unexplained missing bytes remain an error.
42
+ New report rows include optional `nativeTranscripts` with nonnegative integer
43
+ `retained` and `pruned` bundle counts across the case's trials and host roles.
44
+ Markdown shows the same counts beside retained evidence. Older report documents
45
+ without this field remain valid. Pruning does not change verdicts, routes,
46
+ identity, usage, or duration.
47
+
32
48
  Each case row keeps task verdict, execution, and grading separate. Task pass
33
49
  rate is measured only when every trial completed execution and grading with an
34
50
  assessed task verdict. Candidate duration sums the measured host call time
@@ -142,11 +142,23 @@ results and evidence remain intact. Failed cleanup is retried on later evaluatio
142
142
  `sevro.observation.workspaces` records candidate and retained auxiliary workspace
143
143
  paths for inspection. These paths are temporary and do not affect evaluation identity.
144
144
 
145
+ Retained results have no automatic expiration. The explicit
146
+ [`sevro prune` command](reading-results.md#prune-retained-results) applies an
147
+ operator-selected age or count policy, defaulting to the newest ten results.
148
+ It deletes complete retained result directories and maintains `latest.json`
149
+ for the remaining results. Active or unrecognized attempts remain protected;
150
+ fixture cleanup and run-state coordination metadata have separate lifecycles.
151
+
145
152
  Opted-in [candidate artifact runtimes](runtime-v1.md#candidate-artifact-access)
146
153
  remain available through grading and are removed afterward, including failed
147
154
  and cancelled trials. Failed-fixture retention does not preserve them.
148
155
  Read access does not automatically retain artifact bytes in run evidence.
149
156
 
157
+ Checker private home, temporary storage, and copied dependency slots are removed
158
+ after grading on success, failure, and cancellation. Retained failed fixtures and
159
+ saved evidence do not retain checker runtime copies. Explicitly declared shared
160
+ caches survive this cleanup and remain under operator control.
161
+
150
162
  A host adapter can return `executionFailed: true` with bounded artifacts from
151
163
  a failed process. The engine retains those artifacts, skips all grading, and
152
164
  reports `failed` / `not_requested` / `not_assessed` with exit code `2`. A final
@@ -212,14 +224,26 @@ failure; cleanup warnings contain no raw error/output bodies.
212
224
  With [runtime transcript opt-in](runtime-v1.md#native-transcript-access), each
213
225
  bundled host captures full native files before removing its private state.
214
226
  Candidate artifact `sevro.native-transcripts.bundle` has format
215
- `sevro.native-transcripts.v1`. It contains `host` (`codex` or `claude`), `role`,
227
+ `sevro.native-transcripts.v2`. It contains `host` (`codex` or `claude`), `role`,
216
228
  native `rootSessionId` (or `null`), `completeness`, `issues`, and `files`.
217
- Each file records its native relative `path`, SHA-256 `sha256`, and exact bytes
218
- as `bytesBase64`. Decode the bytes without rewriting native records. Native
229
+ Each file records its native relative `path`, SHA-256 `sha256` of the original
230
+ bytes, and `gzipBase64`: the exact file bytes compressed with gzip, then encoded
231
+ as base64. Decode base64, decompress gzip, and verify the original-byte hash
232
+ before using a file. Bound decompression to 5 MiB across all files in the bundle.
233
+ Decode bytes without rewriting native records. Native
219
234
  session paths, root/child records and sidecar metadata preserve associations;
220
235
  the bundle does not normalize model, effort, timing, token, or cost statistics.
221
236
  Existing unknown metrics remain unknown.
222
237
 
238
+ Sevro reads both v2 bundles and existing `sevro.native-transcripts.v1` bundles.
239
+ V1 files use uncompressed `bytesBase64`; decode that field directly without
240
+ gzip decompression and verify the same original-byte hash. Readers must select
241
+ the payload field and decoding by `format` and reject unsupported formats,
242
+ corrupt compressed payloads, hash mismatches, or a decoded total above 5 MiB.
243
+ New captures emit v2; reading retained v1 evidence does not rewrite it.
244
+ The retained view exposes the same exact native files for either format, with
245
+ the original bundle in `index.json`.
246
+
223
247
  `sevro.host.native-transcripts` records the same capture state and file hashes
224
248
  without raw bytes. Semantic and advisory bundles retain their role in the
225
249
  bundle and use the existing `sevro.semantic[.<source digest>].` and
@@ -230,7 +254,8 @@ benchmarks can use the same references and decode the native files.
230
254
 
231
255
  Capture includes native transcript-tree sidecars and refuses descendant links,
232
256
  special files and escaping paths. It is bounded to 4,096 directory entries,
233
- 5 MiB of native file bytes and 7 MiB of encoded file metadata/content. The
257
+ 5 MiB of native file bytes before compression and 7 MiB of encoded file
258
+ metadata/gzip content. The
234
259
  bundle remains inside the existing 8 MiB item and 32 MiB host artifact budgets;
235
260
  it groups sessions instead of spending an artifact per child. Limits produce
236
261
  `byte_limit`, `entry_limit`, or `bundle_limit` issues. Unreadable, unsafe or
@@ -249,3 +274,25 @@ structure, not native semantic correctness.
249
274
  An empty/missing native JSONL tree is `unavailable`. Failure, cancellation and
250
275
  timeout retain available native bytes with incomplete capture state. Raw
251
276
  bundles and views are private evidence and must stay out of Git.
277
+
278
+ The explicit [`sevro prune --native-transcripts` mode](reading-results.md#remove-native-transcript-bundles)
279
+ can remove these bundles without rewriting CLI results, run/trial evidence,
280
+ capture observations, native file hashes, or whole-bundle hashes. It retains
281
+ other artifacts and does not change evaluation identity or assessment states.
282
+ Reports validate present native bundles against their recorded SHA-256 and
283
+ require an exact pruning receipt for absent bundles.
284
+
285
+ Before deletion, Sevro atomically writes private
286
+ `native-transcripts-pruned.json` beside `run.json`, with format
287
+ `sevro.pruned-native-transcripts.v1`, the original `runId`, and `artifacts`.
288
+ Each entry contains the original artifact reference (`id`, `path`, `sha256`,
289
+ and any optional reference flags), plus its `caseId` and `trial`. Entries follow
290
+ run trial/reference order and cover every candidate, semantic (including
291
+ source-specific), and advisory native bundle in that run. A receipt must match
292
+ these original references exactly; malformed, inconsistent, or linked receipts
293
+ are errors. Bundle references must name regular files directly inside their
294
+ result directory. Missing bytes without a receipt, changed hashes, and escaping
295
+ references remain errors. An interrupted deletion may leave some marked bundles
296
+ present; reports count actual availability, and a repeated prune completes removal.
297
+ Transcript references that alias result/trial metadata, raw responses, or other
298
+ retained artifacts are refused before pruning.
@@ -1,6 +1,6 @@
1
1
  # Runtime configuration v1
2
2
 
3
- Configure access to existing host tools, private writable caches, native goals,
3
+ Configure access to existing host tools, private state, reusable caches, native goals,
4
4
  and selected plugin hooks in `<project-root>/sevro.json`.
5
5
 
6
6
  `--runtime-config-file /absolute/path/runtime.json` selects a replacement file.
@@ -36,7 +36,13 @@ capabilities:
36
36
  }
37
37
  },
38
38
  "runtime": {
39
- "seedDirectories": [{ "source": "./public-uv-cache", "target": "uv-cache" }]
39
+ "sharedCacheDirectories": [
40
+ {
41
+ "source": "../public-uv-cache",
42
+ "target": "uv-cache",
43
+ "roles": ["candidate", "hooks"]
44
+ }
45
+ ]
40
46
  },
41
47
  "hooks": {
42
48
  "nativeGoal": true,
@@ -45,7 +51,7 @@ capabilities:
45
51
  }
46
52
  ```
47
53
 
48
- Create the example's `public-uv-cache` directory beside the configuration, or
54
+ Prepare the example's `public-uv-cache` directory outside the project, or
49
55
  omit `runtime` when no prepared cache is needed. Selected plugin names must be
50
56
  present in the case's prepared mounts; remove names that the case does not mount.
51
57
  `DARROW_CACHE_DIR` is an ordinary explicit variable in this example, not a
@@ -235,10 +241,12 @@ digits, and hyphens. `home` and `tmp` are reserved. Targets must be unique.
235
241
  An optional nonempty `roles` array selects any of `candidate`, `checks`, `hooks`,
236
242
  `semantic`, and `advisory`. For example, `"roles": ["candidate", "checks", "hooks"]`
237
243
  keeps dependency slots out of semantic and advisory runtimes. Without `roles`,
238
- execution and artifact-grading roles retain the existing selection. Response-only
244
+ candidate, hook, artifact semantic, and advisory roles receive private copies.
245
+ Shell checks require explicit `"checks"` selection; a non-UV grader receives no
246
+ implicit UV/Python cache. Existing configurations whose shell checks need seeded
247
+ dependencies must add `"checks"` to `roles`. Response-only
239
248
  semantic grading omits implicit seeds; include `"semantic"` explicitly in `roles`
240
- when that route needs dependencies. Artifact semantic grading and executable
241
- shell checks keep their declared tools and default seed selection.
249
+ when that route needs dependencies. Declared host tools remain available to shell checks.
242
250
 
243
251
  Each source tree is bounded to 16,384 entries and 512 MiB. Escaping symlinks,
244
252
  directory cycles, dangling links, and special files are refused. File content
@@ -254,6 +262,41 @@ retaining large policies can retain up to the configured per-seed byte bounds.
254
262
  Source caches are never a writable runtime. Use prepared public caches rather
255
263
  than credential-bearing homes or evaluator directories.
256
264
 
265
+ ### Reuse a shared dependency cache
266
+
267
+ `runtime.sharedCacheDirectories` opts into persistent writable caches without
268
+ copying or hashing their contents for each trial. Each entry requires `source`,
269
+ `target`, and a nonempty `roles` array drawn from the same five roles above.
270
+ The source must already be an owned, readable, writable directory. Relative paths
271
+ resolve beside the runtime file. Sevro resolves declaration symlinks to their
272
+ canonical source and revalidates that directory before execution.
273
+ Targets use the seed slot rules and must be unique across seeds and shared caches.
274
+ Sources cannot overlap each other, private seed sources, declared read-only
275
+ inputs, evaluated project/source trees, results, credentials, or native private state.
276
+
277
+ Select only roles that actually use UV, for example `"roles": ["candidate", "hooks"]`.
278
+ If a grader also uses UV, add `"checks"` to that same entry; both roles use the
279
+ same cache. Each selected private slot aliases the original directory, with a
280
+ narrow sandbox write grant. Other roles receive no slot or source grant.
281
+ Shared content survives successful, failed, and cancelled trials and later runs.
282
+ Sevro removes only disposable role state, never the declared shared source.
283
+
284
+ Prepare UV packages once outside evaluation, point `UV_CACHE_DIR` at
285
+ `{{sevro.runtime}}/uv-cache`, and reuse the existing Python installation through
286
+ declared read-only tool roots. Set `UV_PYTHON_DOWNLOADS=never` to require that
287
+ installation. Sevro keeps `UV_OFFLINE=1`; a missing dependency fails rather than
288
+ downloading during a trial. Project virtual environments and temporary files
289
+ remain disposable; UV's install/link mode can still allocate files in a virtual
290
+ environment. The shared option removes the full cache copy per trial.
291
+
292
+ Shared caches are mutable across trials, roles, and runs. Use them only for public
293
+ dependencies and accept that a participant can change them for later participants.
294
+ Use isolated seeds when dependency independence is required. Configuration identity
295
+ binds the source, slot, and roles, but excludes mutable cache contents; equal
296
+ identities therefore do not prove identical shared-cache bytes. Sevro does not
297
+ age-delete shared caches. Manage their size between evaluations with the cache
298
+ tool, such as `uv cache prune --cache-dir /absolute/public-uv-cache`.
299
+
257
300
  `{{sevro.runtime}}` in an environment value resolves to that role's writable
258
301
  root. Codex commands receive a `TMPPREFIX` inside their private `TMPDIR`,
259
302
  so zsh here-documents use their existing private temporary storage while shared
@@ -266,13 +309,15 @@ use. Its temporary directory stays disposable.
266
309
  Candidate, hook, checker, semantic, and advisory
267
310
  state are separate. With a runtime policy, native candidate, semantic, and
268
311
  advisory roots live in a unique runner-owned host directory outside the Git
269
- fixture. The root contains `home`, `tmp`, and declared seed slots; plugins can
312
+ fixture. The root contains `home`, `tmp`, private seed slots and selected shared-cache links; plugins can
270
313
  use paths such as `~/.darrow/reviews` without creating repository-local review
271
314
  state. Command runtime directories are separate from host authentication,
272
- configuration, and live transcript sources. Checker and hook caches also live
273
- outside candidate-controlled workspace state. Private roots survive native
315
+ configuration, and live transcript sources. Checker and hook private state also lives
316
+ outside candidate-controlled workspace state. Shared sources remain at their declared locations. Private roots survive native
274
317
  continuation turns and children, then are removed with their owning trial or
275
- host state after owned hooks terminate. Opted-in candidate artifact runtimes
318
+ host state after owned hooks terminate. Checker home, temporary storage, and copied
319
+ seeds are removed after grading even when the fixture is retained after failure
320
+ or cancellation. Results and evidence are preserved. Opted-in candidate artifact runtimes
276
321
  remain alive through grading as described below. Runs with neither fixture setup
277
322
  nor a selected runtime policy
278
323
  retain their previous writable-state locations.
@@ -525,7 +570,7 @@ run evidence. Extensions and external benchmarks consume the
525
570
  ## Evidence and compatibility
526
571
 
527
572
  `configuration.redacted.runtimePolicy` retains the selected environment,
528
- canonical read roots, seed digests, and hook declaration. The engine snapshots
573
+ canonical read roots, seed digests, shared-cache declarations, and hook declaration. The engine snapshots
529
574
  this input once; it participates in `configurationDigest` and evaluation
530
575
  identity. Symbolic runtime placeholders remain symbolic in identity, while
531
576
  execution uses per-role paths. This binds declared setup and seed contents;
@@ -539,7 +584,8 @@ file contents do not. The same discovery and skip evidence reaches
539
584
  `sevro.host.runtime` observations.
540
585
 
541
586
  Bundled hosts supply `sevro.host.runtime` with the applied policy digest, role,
542
- environment names, read roots, and seed slots. `sevro.host.hooks` records source
587
+ environment names, read roots, seed slots, and selected shared-cache declarations.
588
+ Shared entries contain no content digest. `sevro.host.hooks` records source
543
589
  and effective plugin digests plus observed executions and their exit codes.
544
590
  An empty execution list does not prove hook activation. Partial lifecycle
545
591
  evidence remains partial. Cases can require these observations; missing required
@@ -549,7 +595,8 @@ task verdict.
549
595
  `--toolchain-bin-dir` and `--claude-uv-cache-dir` are deprecated. They keep their
550
596
  previous behavior with a warning when no runtime file is selected. Combining
551
597
  them with a runtime file fails before execution. Migrate to `environment`,
552
- `filesystem`, and private seed slots; removal is no earlier than `0.2.0`.
598
+ `filesystem`, and either private seed slots or shared-cache declarations; removal
599
+ is no earlier than `0.2.0`. The deprecated UV option continues to copy privately.
553
600
  See the [CLI reference](development-cli.md), [native hosts](native-hosts.md),
554
601
  and [identity contract](identity-v1.md).
555
602
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bjoernrochel/sevro",
3
- "version": "0.1.0-rc.17",
3
+ "version": "0.1.0-rc.19",
4
4
  "license": "SEE LICENSE IN LICENSE",
5
5
  "description": "A standalone evaluation runner for agentic coding capabilities",
6
6
  "type": "module",
@@ -110,6 +110,15 @@
110
110
  "domainOutcomes"
111
111
  ],
112
112
  "properties": {
113
+ "nativeTranscripts": {
114
+ "type": "object",
115
+ "required": ["retained", "pruned"],
116
+ "properties": {
117
+ "retained": { "type": "integer", "minimum": 0 },
118
+ "pruned": { "type": "integer", "minimum": 0 }
119
+ },
120
+ "additionalProperties": false
121
+ },
113
122
  "resultFile": { "$ref": "#/$defs/path" },
114
123
  "evidencePath": { "type": ["string", "null"] },
115
124
  "runId": { "type": ["string", "null"] },
@@ -62,6 +62,38 @@
62
62
  }
63
63
  }
64
64
  },
65
+ "sharedCacheDirectories": {
66
+ "type": "array",
67
+ "maxItems": 16,
68
+ "items": {
69
+ "type": "object",
70
+ "additionalProperties": false,
71
+ "required": ["source", "target", "roles"],
72
+ "properties": {
73
+ "source": { "type": "string", "minLength": 1, "maxLength": 4096 },
74
+ "target": {
75
+ "type": "string",
76
+ "pattern": "^[a-z][a-z0-9-]{0,63}$",
77
+ "not": { "enum": ["home", "tmp"] }
78
+ },
79
+ "roles": {
80
+ "type": "array",
81
+ "minItems": 1,
82
+ "maxItems": 5,
83
+ "uniqueItems": true,
84
+ "items": {
85
+ "enum": [
86
+ "candidate",
87
+ "checks",
88
+ "hooks",
89
+ "semantic",
90
+ "advisory"
91
+ ]
92
+ }
93
+ }
94
+ }
95
+ }
96
+ },
65
97
  "seedDirectories": {
66
98
  "type": "array",
67
99
  "maxItems": 16,
@@ -2,11 +2,14 @@ import { dirname, join } from "node:path";
2
2
  import { atomicWriteJson } from "../storage";
3
3
  import type { CliResult } from "../engine";
4
4
  import { createReport } from "../report";
5
+ import { withResultsLock } from "../results-lock";
5
6
 
6
7
  /** Latest means the last successfully retained CLI result, including dry/failing runs. */
7
8
  export async function retainCliResult(result: CliResult, resultsRoot: string) {
8
- const resultFile = join(dirname(result.evidencePath), "result.json");
9
- await atomicWriteJson(resultFile, result);
10
- await atomicWriteJson(join(resultsRoot, "latest.json"), result);
11
- return { resultFile, report: await createReport([resultFile]) };
9
+ return withResultsLock(resultsRoot, async () => {
10
+ const resultFile = join(dirname(result.evidencePath), "result.json");
11
+ await atomicWriteJson(resultFile, result);
12
+ await atomicWriteJson(join(resultsRoot, "latest.json"), result);
13
+ return { resultFile, report: await createReport([resultFile]) };
14
+ });
12
15
  }
package/src/cli/help.ts CHANGED
@@ -10,12 +10,14 @@ Examples:
10
10
  --condition passive --trials 1 --threshold 1
11
11
  sevro run --help
12
12
  sevro report --result-file /absolute/results/run/result.json
13
+ sevro prune
13
14
 
14
15
  Commands:
15
16
  demo Run the packaged credential-free assessed example; use demo --help.
16
17
  init Create a new evaluation project; use init --help.
17
18
  run Evaluate one resolved case or extension case; use run --help.
18
19
  report Summarize retained results; use report --help.
20
+ prune Remove retained results using a retention policy; use prune --help.
19
21
 
20
22
  Options:
21
23
  -h, --help Show help without preparing a run.
@@ -131,6 +133,36 @@ Options and defaults:
131
133
  -h, --help Show this help without reading result files.
132
134
  `;
133
135
 
136
+ const PRUNE_HELP = `Usage: sevro prune [options]
137
+
138
+ Examples:
139
+ sevro prune
140
+ sevro prune --results-root ./other-results --keep 5
141
+ sevro prune --older-than 30d
142
+ sevro prune --latest
143
+ sevro prune --native-transcripts --older-than 30d
144
+
145
+ Defaults: keep the newest 10 completed results under <cwd>/.sevro/results.
146
+ Select at most one explicit retention policy:
147
+ --keep <positive-integer> Keep this many newest results.
148
+ --latest Keep only the newest result (same as --keep 1).
149
+ --older-than <duration> Remove results older than a positive integer plus ms, s, m, h, d or w.
150
+ --results-root <path> Relative or absolute results root, resolved from cwd.
151
+ --native-transcripts Remove only native transcript bundles; keep results and hashes.
152
+ -h, --help Show help without reading or deleting results.
153
+
154
+ Age and ordering use result.json modification times; equal times sort by path.
155
+ Each retained attempt counts separately, including dry, failed and interrupted runs.
156
+ Active/unrecognized attempts and directories without result.json are preserved.
157
+ Evidence is permanently deleted. Remaining results stay usable by sevro report;
158
+ latest.json is updated, or removed when no completed results remain.
159
+ With --native-transcripts, no policy removes bundles from all completed results;
160
+ --keep/--latest preserve bundles for the newest results, and --older-than selects older results.
161
+ Result/evidence files and latest.json stay unchanged. Reports label removed bundles as pruned.
162
+ Missing or empty results roots are a successful no-op. Invalid or unreadable inputs exit 64.
163
+ Reference: docs/reading-results.md in the installed package.
164
+ `;
165
+
134
166
  const INIT_HELP = `Usage: sevro init [options]
135
167
 
136
168
  Examples:
@@ -169,6 +201,9 @@ const HELP_REQUESTS: Record<string, string> = {
169
201
  "report --help": REPORT_HELP,
170
202
  "report -h": REPORT_HELP,
171
203
  "help report": REPORT_HELP,
204
+ "prune --help": PRUNE_HELP,
205
+ "prune -h": PRUNE_HELP,
206
+ "help prune": PRUNE_HELP,
172
207
  };
173
208
 
174
209
  /** Handle informational commands before loading inputs or creating run state. */
package/src/cli.ts CHANGED
@@ -4,6 +4,7 @@ import { reportCommand } from "./report";
4
4
  import { informationCommand } from "./cli/help";
5
5
  import { demoCommand } from "./cli/demo";
6
6
  import { initCommand } from "./cli/init";
7
+ import { pruneCommand } from "./prune";
7
8
 
8
9
  if (informationCommand(process.argv.slice(2))) process.exitCode = 0;
9
10
  else if (process.argv[2] === "demo") await demoCommand(process.argv.slice(3));
@@ -11,4 +12,6 @@ else if (process.argv[2] === "init")
11
12
  process.exitCode = await initCommand(process.argv.slice(3));
12
13
  else if (process.argv[2] === "report")
13
14
  process.exitCode = await reportCommand(process.argv.slice(2));
15
+ else if (process.argv[2] === "prune")
16
+ process.exitCode = await pruneCommand(process.argv.slice(3));
14
17
  else await main(process.argv.slice(2));
@@ -1,4 +1,5 @@
1
1
  import { rm } from "node:fs/promises";
2
+ import { hashJson } from "../identity";
2
3
  import {
3
4
  allocateCandidateRuntime,
4
5
  candidateArtifactRoots,
@@ -38,7 +39,10 @@ import type {
38
39
  ObservationCompleteness,
39
40
  TaskVerdict,
40
41
  } from "../contract-types";
41
- import { clearFixtureContents } from "../fixtures/cleanup";
42
+ import {
43
+ clearFixtureContents,
44
+ removeDisposableDirectory,
45
+ } from "../fixtures/cleanup";
42
46
  import { retainWorkspace } from "../fixtures/workspace-state";
43
47
  import { gitHeadRevision } from "../graders/git-head";
44
48
  import {
@@ -789,6 +793,13 @@ class EvaluationTrial {
789
793
  evaluationProgress(this.options, stage, status, this.trial);
790
794
  }
791
795
  private async cleanup(): Promise<void> {
796
+ await removeDisposableDirectory(
797
+ join(
798
+ this.context.resources.stateDir,
799
+ "shell-sandbox/runtime",
800
+ hashJson(this.fixture.workspace),
801
+ ),
802
+ );
792
803
  if (this.state.candidateTranscriptRoot)
793
804
  await rm(this.state.candidateTranscriptRoot, {
794
805
  recursive: true,