@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (230) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +17 -3
  7. package/docs/codex-host.md +155 -49
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +141 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +307 -18
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +63 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +206 -0
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +13 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +133 -115
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/evaluation/types.ts +207 -0
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +15 -76
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/fixtures/workspace-state.ts +212 -0
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +57 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +91 -14
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +46 -16
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +122 -73
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -54
  158. package/src/hosts/codex-profile.ts +50 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +141 -19
  161. package/src/hosts/codex.ts +221 -179
  162. package/src/hosts/isolation-common.ts +15 -11
  163. package/src/hosts/isolation-roots.ts +13 -20
  164. package/src/hosts/linux-sandbox.ts +27 -3
  165. package/src/hosts/mac-sandbox.ts +111 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +221 -45
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +34 -15
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/run-owner.ts +4 -4
  186. package/src/{runtime-config.ts → runtime/config.ts} +182 -44
  187. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  188. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  189. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  190. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  191. package/src/{runtime-hooks.ts → runtime/hooks.ts} +33 -22
  192. package/src/{runtime-paths.ts → runtime/paths.ts} +6 -18
  193. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  194. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  195. package/src/runtime/shared-cache.ts +130 -0
  196. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  197. package/src/schema-types.ts +73 -161
  198. package/src/shell.ts +4 -0
  199. package/src/transcript-retention.ts +146 -0
  200. package/src/value-guards.ts +22 -0
  201. package/docs/assets/sevro-logo.png +0 -0
  202. package/docs/assets/sevro-symbol.svg +0 -1
  203. package/docs/assets/sevro-wordmark.svg +0 -1
  204. package/docs/cache-ownership-validation.md +0 -49
  205. package/docs/claude-continuation-validation.md +0 -47
  206. package/docs/concurrency-validation.md +0 -86
  207. package/docs/configuration-root-validation.md +0 -70
  208. package/docs/engine-slice.md +0 -114
  209. package/docs/evidence/runtime-startup-30/README.md +0 -185
  210. package/docs/evidence/runtime-startup-30/after.json +0 -319
  211. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  212. package/docs/evidence/runtime-startup-30/before.json +0 -403
  213. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  214. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  216. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  217. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  218. package/docs/evidence.md +0 -17
  219. package/docs/fixture-cleanup-validation.md +0 -70
  220. package/docs/guide-verification.json +0 -239
  221. package/src/evaluation-context.ts +0 -257
  222. package/src/evaluation-fixture.ts +0 -313
  223. package/src/evaluation-run.ts +0 -198
  224. package/src/evaluation-trial.ts +0 -946
  225. package/src/evaluation-types.ts +0 -149
  226. package/src/hosts/claude-nested-skills.ts +0 -272
  227. package/src/hosts/codex-command-output.ts +0 -510
  228. package/src/hosts/codex-executor-command.ts +0 -291
  229. package/src/hosts/codex-skill-reads.ts +0 -609
  230. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
package/CONTRIBUTING.md CHANGED
@@ -26,8 +26,10 @@ by applicable law.
26
26
 
27
27
  ## Set up development
28
28
 
29
- Use Bun 1.3.13, Git, Node 24, and npm. Full suite isolation uses `sandbox-exec`
30
- on macOS or `bubblewrap` and `socat` on Ubuntu. Windows acceptance is unverified.
29
+ Use Bun 1.3.13, Git, Node 24, npm, and ripgrep (`rg`). Full suite isolation uses `sandbox-exec`
30
+ on macOS or `bubblewrap` and `socat` on Ubuntu. The macOS synthetic libc regression
31
+ fixtures compile with `/usr/bin/cc`, supplied by Apple's Command Line Tools.
32
+ Windows acceptance is unverified.
31
33
  From a fresh checkout:
32
34
 
33
35
  ```sh
@@ -36,6 +38,13 @@ bun run test:docs-examples
36
38
  ```
37
39
 
38
40
  The second command checks the tutorial in fresh consumer/contributor directories.
41
+ Its complete recipe checks also require UV, Python 3, mise, and an existing
42
+ `mise install node@24.13.0` installation; see [recipe prerequisites](docs/recipes.md).
43
+ CI explicitly provisions these tools. Missing prerequisites fail with a named
44
+ setup instruction rather than skipping a recipe.
45
+ To run recipe commands from this source checkout, set
46
+ `export SEVRO_RECIPES="$PWD/examples/recipes"` from the repository root, then
47
+ follow the [recipe commands](docs/recipes.md#prepare-a-recipe-project).
39
48
  Native model trials need their own [credentials and prerequisites](docs/native-hosts.md).
40
49
 
41
50
  ## Quality contract and checks
@@ -54,8 +63,9 @@ documentation and guide dry-run checks, tutorial examples, the real Bun suite,
54
63
  coverage integrity and thresholds, and installed-package validation. Statements
55
64
  and branches must each reach at least 95% using raw exact counts over the
56
65
  platform-reachable authored production inventory, including unimported files.
57
- The macOS and Ubuntu gates each measure their reachable host modules; the exact
58
- module and branch-outcome exclusions are recorded in coverage artifacts. The five explicitly
66
+ The local-only macOS gate and Ubuntu CI gate each measure their reachable host
67
+ modules; the exact module and branch-outcome exclusions are recorded in coverage
68
+ artifacts. The five explicitly
59
69
  inventoried generated AJV validators are excluded from coverage because this
60
70
  target measures authored runner behavior; their syntax, schema freshness, typed
61
71
  boundaries and runtime tests remain checked. Tests and fixtures remain linted and
@@ -66,6 +76,12 @@ Before committing, use `bun run check:typescript --fast` after installing the
66
76
  locked dependencies. This runs inventory, schema, formatting, lint, typing and
67
77
  documentation checks; it does not replace full acceptance. CI retains machine-readable coverage
68
78
  reports under `.quality/` and exposes the stable `TypeScript quality` status.
79
+ The complete command's `static`, `docs`, `tests` and `package` phases run as
80
+ concurrent CI jobs. CI divides the tests among four duration-balanced jobs and
81
+ requires one combined coverage gate; every test and both 95% thresholds remain
82
+ required. Superseded runs for the same pull request are cancelled. After frozen installation, use
83
+ `bun run check:typescript --phase <name>` to run one phase locally; it does not
84
+ replace acceptance across all four. See the [phase inventory](docs/typescript-quality.md).
69
85
 
70
86
  Run `bun run check:docs:external` separately and [triage network failures](docs/documentation-quality.md#external-links).
71
87
  Guide changes also need [native evals](docs/guide-evaluation.md).
@@ -82,8 +98,8 @@ repairing in-scope failures; it does not authorize merging or unrelated changes.
82
98
  non-root user, and isolation prerequisites. Reproducing that environment locally
83
99
  with Docker is optional for diagnosis and is not a publication prerequisite.
84
100
  - After the local pre-commit checks, publish the authorized candidate to its PR
85
- and require `bun run check:typescript` to succeed in GitHub CI before reporting
86
- delivery as complete. A local run, fast check, targeted suite, or historical
101
+ and require all four canonical `check:typescript` phases to succeed in GitHub CI
102
+ before reporting delivery as complete. A local run, fast check, targeted suite, or historical
87
103
  result does not establish that the current Ubuntu gate passes.
88
104
  - Keep the quality bar intact. Repair behavior or add meaningful coverage for
89
105
  accepted behavior; do not lower thresholds, exempt authored sources, skip tests,
package/README.md CHANGED
@@ -1,7 +1,7 @@
1
1
  # Sevro
2
2
 
3
3
  <p align="center">
4
- <img src="docs/assets/sevro-logo.png" alt="Sevro logo: a black S-shaped track with red segments above the sevro wordmark" width="360">
4
+ <img src="https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-logo.png" alt="Sevro logo: a black S-shaped track with red segments above the sevro wordmark" width="360">
5
5
  </p>
6
6
 
7
7
  Sevro runs repeatable evaluations of coding agents. Give it a case, a host
@@ -40,6 +40,10 @@ Continue with [your first evaluation](docs/getting-started.md): the exact comman
40
40
  expected `passed` result, and retained evidence. Read [installation](docs/installing.md)
41
41
  for updates, removal, and source checkouts.
42
42
 
43
+ Then follow the [demo-to-iteration workflow](docs/evaluation-workflow.md):
44
+ initialize real assertions, prepare, execute, read progress and inspect the
45
+ latest result.
46
+
43
47
  ## Find your next task
44
48
 
45
49
  The [documentation hub](docs/README.md) routes to running evaluations, authoring
package/docs/README.md CHANGED
@@ -1,22 +1,25 @@
1
1
  # Sevro documentation
2
2
 
3
- <img src="assets/sevro-wordmark.svg" alt="Sevro wordmark" width="220">
3
+ <img src="https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-wordmark.svg" alt="Sevro wordmark" width="220">
4
4
 
5
5
  Start with the task you want to complete. These pages work without an agent.
6
6
 
7
- | I want to… | Read |
8
- | --------------------------------------------- | -------------------------------------------------------- |
9
- | Get a result without a model account | [Your first evaluation](getting-started.md) |
10
- | Install, update, remove, or use a checkout | [Installation](installing.md) |
11
- | Run evaluations and select a host | [Running evaluations](running-evaluations.md) |
12
- | Create cases and reusable policy | [Cases and extensions](creating-cases.md) |
13
- | Interpret verdicts, evidence, and comparisons | [Reading results](reading-results.md) |
14
- | Run Codex or Claude with isolation | [Native hosts](native-hosts.md) |
15
- | Reuse host tools or enable goals and hooks | [Runtime configuration](runtime-v1.md) |
16
- | Resolve a symptom | [Troubleshooting](troubleshooting.md) |
17
- | Understand runner responsibilities | [Architecture](architecture.md) |
18
- | Change Sevro or its documentation | [Contributing](../CONTRIBUTING.md) |
19
- | Understand licensing and branding | [Licensing](licensing.md) and [assets](assets/README.md) |
7
+ | I want to… | Read |
8
+ | ---------------------------------------------- | -------------------------------------------------------- |
9
+ | Get a result without a model account | [Your first evaluation](getting-started.md) |
10
+ | Initialize, prepare, run and iterate | [Evaluation workflow](evaluation-workflow.md) |
11
+ | Install, update, remove, or use a checkout | [Installation](installing.md) |
12
+ | Run evaluations and select a host | [Running evaluations](running-evaluations.md) |
13
+ | Create cases and reusable policy | [Cases and extensions](creating-cases.md) |
14
+ | Copy a complete coding, skill or runtime setup | [Runnable recipes](recipes.md) |
15
+ | Interpret verdicts, evidence, and comparisons | [Reading results](reading-results.md) |
16
+ | Run Codex or Claude with isolation | [Native hosts](native-hosts.md) |
17
+ | Reuse host tools or enable goals and hooks | [Runtime configuration](runtime-v1.md) |
18
+ | Save recurring evaluation selections | [Run settings](run-settings-v1.md) |
19
+ | Resolve a symptom | [Troubleshooting](troubleshooting.md) |
20
+ | Trace runner lifecycle, ownership, and changes | [Architecture](architecture.md) |
21
+ | Change Sevro or its documentation | [Contributing](../CONTRIBUTING.md) |
22
+ | Understand licensing and branding | [Licensing](licensing.md) and [assets](assets/README.md) |
20
23
 
21
24
  ## Ask the guide
22
25
 
@@ -30,11 +33,12 @@ It belongs to this repository, not an npm installation.
30
33
 
31
34
  [Current contracts](contracts.md) define interfaces. The
32
35
  [guide specification](specs/repository-guide.md) defines the repository assistant.
33
- [Historical validation](evidence.md) records observations, not current guarantees.
36
+ [Historical validation](https://github.com/BjRo/sevro/blob/main/docs/evidence.md) records observations, not current guarantees.
34
37
  There is no accepted Rust migration decision here;
35
38
  [issue #2](https://github.com/BjRo/sevro/issues/2) tracks research.
36
39
 
37
40
  Read [documentation quality](documentation-quality.md),
41
+ [the usability study protocol](https://github.com/BjRo/sevro/blob/main/docs/usability-study.md),
38
42
  [guide evaluation](guide-evaluation.md), [releases](releases.md), and
39
43
  [acknowledgements](acknowledgements.md) for maintenance.
40
44
  Return to the [Sevro introduction](../README.md).
@@ -4,22 +4,143 @@ Sevro owns generic execution/evidence contracts. Projects own cases and policy.
4
4
  This lets Darrow and other integrations use the same runner without importing
5
5
  their benchmark policy into it.
6
6
 
7
- | Responsibility | Sources |
8
- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ |
9
- | Parse commands/select explicit routes | [CLI](../src/cli.ts), [CLI reference](development-cli.md) |
10
- | Prepare, execute, grade, retain, clean up | [Engine](../src/engine.ts), [engine slice](engine-slice.md) |
11
- | Negotiate trusted extension calls | [Client](../src/extension-client.ts), [session](../src/extension-session.ts), [protocol](extension-protocol-v1.md) |
12
- | Run native hosts and retain bounded observations | [Codex](codex-host.md), [native controls](native-controls.md), [host sources](../src/hosts/) |
13
- | Reduce execution/grading/task states | [Result source](../src/results.ts), [contract](results-v1.md) |
14
- | Bind comparison identities and release provenance | [Identity](identity-v1.md), [releases](releases.md) |
15
- | Summarize retained results | [Report](report-v1.md) |
16
-
17
- Completed evidence is retained before cleanup. Missing observations remain
18
- unavailable; unknown measurements do not become zero. Extensions cannot
19
- silently replace isolation/persistence or built-in policy. The
7
+ This page explains this checkout's implementation and points to acceptance
8
+ tests for changes. [Current contracts](contracts.md) define public behavior;
9
+ [historical validation](https://github.com/BjRo/sevro/blob/main/docs/evidence.md) records bounded past observations. The
10
+ [engine slice](https://github.com/BjRo/sevro/blob/main/docs/engine-slice.md) retains its earlier scope and detailed lifecycle
11
+ explanation. Implementation paths and test scenarios here are navigation, not
12
+ evidence that a particular candidate passed validation.
13
+
14
+ [`engine.ts`](../src/engine.ts) remains the programmatic facade, and
15
+ [`cli.ts`](../src/cli.ts) remains the installed command entrypoint. The CLI
16
+ loads and assembles evaluation options; evaluation modules coordinate the run
17
+ and its trials.
18
+
19
+ ## Evaluation lifecycle
20
+
21
+ [`runEvaluation`](../src/evaluation/run.ts) prepares inputs before allocating
22
+ run resources, retains preparation artifacts, and reserves candidate and
23
+ semantic workspace roots as needed before calling the trial scheduler. The map
24
+ follows the path of a retained trial; individual stages can be skipped or fail.
25
+
26
+ ```mermaid
27
+ flowchart TD
28
+ P["Validate and prepare evaluation inputs"] --> R["Allocate run storage and owner; reserve workspace roots"]
29
+ R --> W["Admit trial and materialize its reserved fixture"]
30
+ subgraph Trial["Each admitted trial"]
31
+ W --> C["Initialize grading bases; execute candidate"]
32
+ C --> E["Retain candidate artifacts; observe output; prepare grading views"]
33
+ E --> G["Output, shell, Git, semantic, and extension grading"]
34
+ G --> A["Assess execution, grading, and task separately"]
35
+ A --> V["Optional independent advisory review"]
36
+ V --> T["Retain raw candidate response; atomically write trial evidence"]
37
+ T --> K["Collect completed trial and checkpoint its reference"]
38
+ K --> D["Release temporary views; clear or retain fixture contents"]
39
+ end
40
+ D --> J["All admitted trials drain"]
41
+ J --> F["Write final run evidence; finalize owner checkpoint"]
42
+ F --> X["Finalize retained workspaces; release other reservations"]
43
+ ```
44
+
45
+ [`scheduleTrials`](../src/evaluation/scheduler.ts) bounds active work by the
46
+ prepared job limit. It admits a trial before
47
+ [`prepareTrialWorkspace`](../src/evaluation/resources.ts) materializes that
48
+ trial's fixture. Each [`EvaluationTrial`](../src/evaluation/trial.ts) owns its
49
+ mutable state and writer. Its `run()` method keeps candidate execution,
50
+ artifact retention, grading, assessment, advisory review, persistence, and
51
+ cleanup in one visible order. Grading helpers return checks, observations,
52
+ and failures; the coordinator incorporates those results and stops new
53
+ admission when an execution or required grading error is detected, before
54
+ advisory work, persistence, or cleanup. The scheduler also stops admission on
55
+ cancellation and waits for active workers. A failed task check alone does not
56
+ close admission. Already admitted peers drain and retain their evidence, including
57
+ when they finish out of order.
58
+
59
+ [`runAdvisoryReview`](../src/evaluation/advisory.ts) handles its own host call,
60
+ assessment, retained response, and auxiliary workspace cleanup. It depends on
61
+ candidate completion, independently of the task assessment. The
20
62
  [advisory fixture](advisory-fixture.md) gives review a separate Git view with
21
- evaluator inputs withheld.
63
+ evaluator inputs withheld. Its recommendation or host failure does not change
64
+ task grading; a failure to retain its evidence remains a persistence error.
65
+
66
+ Successful [semantic response parsing](../src/evaluation/semantic.ts) clears
67
+ auxiliary fixture contents; successful advisory response parsing removes its
68
+ auxiliary fixture. Both retain response evidence first, including valid failed
69
+ check verdicts or advisory recommendations.
70
+
71
+ The [trial writer](../src/evidence/trial-writer.ts) writes the trial artifact
72
+ before the coordinator marks it persisted and hands it to the
73
+ [result collector](../src/evaluation/collection.ts). The collector registers
74
+ the completed record, orders records by trial number, and checkpoints retained
75
+ artifact paths. After active work drains, the run coordinator validates and
76
+ writes `run.json` before finalizing the owner as complete or interrupted.
77
+ Persistence failures cannot produce a successful result; the run's failure
78
+ path attempts a diagnostic checkpoint while preserving the original failure.
79
+
80
+ Trial cleanup releases candidate artifact runtimes and transcript views even
81
+ when the trial fails. Persisted successful fixture contents are cleared while
82
+ their empty reserved roots remain until active work drains. Failed, cancelled,
83
+ grading-error, unavailable-evidence, and failed-task fixtures are retained;
84
+ [workspace ownership](../src/fixtures/workspace-state.ts) tracks their inactive
85
+ retention and startup expiry after 24 hours. Saved evidence stays outside that
86
+ cleanup. See the [results retention contract](results-v1.md#identity-and-provenance)
87
+ for the public boundary, including ownership that cannot be verified.
88
+
89
+ Dry preparation still validates and materializes fixtures, including declared
90
+ fixture setup, and retains trial records. It skips candidate execution and
91
+ grading, recording `not_run` / `not_requested` / `not_assessed`, as exercised by
92
+ the [engine trial suite](../tests/src/engine-trials.test.ts). Missing observations
93
+ remain unavailable; missing required evidence cannot become a pass. Optional
94
+ unknown measurements remain unknown rather than becoming zero. Extensions
95
+ cannot silently replace isolation, persistence,
96
+ or built-in policy. The [results](results-v1.md) and
97
+ [extension protocol](extension-protocol-v1.md) contracts define those semantics.
98
+
99
+ ## State ownership
100
+
101
+ [`EvaluationContext`](../src/evaluation/context.ts) carries four explicit
102
+ collaborators: prepared inputs, run resources, the result collector, and retained
103
+ preparation-artifact references. It does not combine mutable trial fields with
104
+ the preparation plan.
105
+
106
+ | Data or state | Owner | Boundary |
107
+ | ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
108
+ | Prepared inputs | [`PreparedEvaluation` in `plan.ts`](../src/evaluation/plan.ts) | Named fixture, check, and invocation groups plus routes, provenance, redacted configuration, and comparison identity. Preparation completes before run allocation. |
109
+ | Run resources and reservations | [`RunResources` and `RunWorkspaceReservations` in `resources.ts`](../src/evaluation/resources.ts) | Run ID, evidence/state locations, process owner, and candidate/semantic reservations. Reservations outlive completed fixture contents; retained workspaces are tracked separately from completed roots. |
110
+ | Mutable trial state | [`TrialRuntimeState` and `EvaluationTrial` in `trial.ts`](../src/evaluation/trial.ts) | Candidate result, observations, checks, assessment, temporary views, advisory result, and persisted milestone belong to one admitted trial. Grading receives narrow inputs and explicit effects rather than this state object. |
111
+ | Collected run results | [`ResultCollector` in `collection.ts`](../src/evaluation/collection.ts) | Private completed summaries, evidence, and diagnostics; trial-number ordering, snapshots for aggregation, and checkpoint requests. [`run-owner.ts`](../src/run-owner.ts) verifies process ownership and writes owner/checkpoint state. |
112
+ | Evidence writes | [`TrialEvidenceWriter`](../src/evidence/trial-writer.ts), [preparation retention](../src/evaluation/resources.ts), [run finalization](../src/evaluation/run.ts) | The trial writer owns filenames, private writes, hashes, artifact references, and typed trial-persistence failures. Resource helpers retain preparation bytes; run finalization writes aggregate evidence. [`storage.ts`](../src/storage.ts) supplies atomic private JSON writes. |
113
+
114
+ Fixture modules own materialization and workspace preparation. Runtime modules
115
+ own declared host runtime policy and private state. Evidence modules validate
116
+ host observations and artifacts, verify retained bytes, and normalize optional
117
+ usage without constructing fixtures. Native host implementations and graders
118
+ remain in their own families.
119
+
120
+ ## Where to change behavior
121
+
122
+ Start with the responsibility, then inspect its observable acceptance scenarios.
123
+ These are useful starting suites, not an exhaustive validation list. The
124
+ [contribution checks](../CONTRIBUTING.md#quality-contract-and-checks) still govern
125
+ candidate acceptance.
126
+
127
+ | Responsibility | Implementation and reference | Relevant acceptance tests |
128
+ | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
129
+ | Parse commands/select explicit routes | [CLI entrypoint](../src/cli.ts), [invocation parsing](../src/cli/invocation.ts), [CLI reference](development-cli.md) | [Invalid invocations](../tests/src/cli/invocation-validation.test.ts) |
130
+ | Load and prepare a CLI evaluation | [Input loading](../src/cli/loading.ts), [preparation](../src/cli/preparation.ts), [host construction](../src/cli/hosts.ts), [extension resolution](../src/cli/extension.ts), [evaluation assembly](../src/cli/evaluation.ts) | [Case/configuration validation](../tests/src/cli/case-validation.test.ts) |
131
+ | Present CLI results and own cancellation | [Presentation](../src/cli/presentation.ts), [cancellation](../src/cli/cancellation.ts), [command lifecycle](../src/cli/command.ts) | [CLI states and exits](../tests/src/cli/results.test.ts), [signal retention](../tests/src/cli/trial-retention.test.ts) |
132
+ | Prepare evaluation inputs and grade trials | [Engine facade](../src/engine.ts), [plan](../src/evaluation/plan.ts), [grading](../src/evaluation/grading.ts), [semantic grading](../src/evaluation/semantic.ts), [evaluation modules](../src/evaluation/), [engine slice](https://github.com/BjRo/sevro/blob/main/docs/engine-slice.md) | [Preparation refusals](../tests/src/evaluation/preparation-validation.test.ts), [semantic grading failures](../tests/src/evaluation/semantic-grading-failures.test.ts), [engine lifecycle](../tests/src/engine-lifecycle.test.ts) |
133
+ | Schedule trials, retain evidence, and finalize ownership | [Scheduler](../src/evaluation/scheduler.ts), [trial coordinator](../src/evaluation/trial.ts), [reservations](../src/evaluation/resources.ts), [collection](../src/evaluation/collection.ts), [run owner](../src/run-owner.ts) | [Concurrent admission and ordering](../tests/src/cli/concurrency.test.ts), [trial failures and retention](../tests/src/evaluation/trial-retention.test.ts), [checkpoint refusals](../tests/src/run-owner-checkpoints.test.ts) |
134
+ | Prepare, construct, and clean up fixtures | [Fixture modules](../src/fixtures/), [construction](../src/fixtures/create.ts), [preparation artifacts](../src/fixtures/preparation.ts), [content cleanup](../src/fixtures/cleanup.ts), [workspace ownership](../src/fixtures/workspace-state.ts) | [Artifact preparation](../tests/src/fixtures/artifact-preparation.test.ts), [workspace lifecycle](../tests/src/fixtures/workspace-lifecycle.test.ts) |
135
+ | Resolve runtime policy, discovery, and private state | [Runtime modules](../src/runtime/), [configuration](../src/runtime/config.ts) | [Runtime environment](../tests/src/runtime/environment.test.ts), [candidate artifact lifetime](../tests/src/candidate-artifacts.test.ts) |
136
+ | Validate host results and retain bounded evidence | [Host results](../src/evidence/host-results.ts), [trial writer](../src/evidence/trial-writer.ts), [retained artifacts](../src/evidence/retained-artifacts.ts), [usage](../src/evidence/usage.ts) | [Host evidence refusals](../tests/src/evaluation/host-evidence.test.ts), [failed-host evidence retention](../tests/src/evaluation/trial-retention.test.ts) |
137
+ | Negotiate trusted extension calls | [Client](../src/extension-client.ts), [session](../src/extension-session.ts), [extension grading](../src/evaluation/grading.ts), [protocol](extension-protocol-v1.md) | [Negotiated engine behavior](../tests/src/evaluation/extension-engine.test.ts), [extension grading outcomes](../tests/src/evaluation/extension-grading.test.ts) |
138
+ | Run independent advisory review | [Advisory execution](../src/evaluation/advisory.ts), [blind fixture](../src/fixtures/advisory.ts), [advisory fixture explanation](advisory-fixture.md) | [Advisory failures](../tests/src/evaluation/advisory-failures.test.ts), [blind change view](../tests/src/fixtures/advisory.test.ts) |
139
+ | Run native hosts and retain bounded observations | [Codex](codex-host.md), [native controls](native-controls.md), [host sources](../src/hosts/), [shared process mechanics](../src/process-lifecycle.ts), [native waiting](../src/hosts/process-lifecycle.ts) | [Native process admission and byte bounds](../tests/src/hosts/native-process-boundaries.test.ts) |
140
+ | Reduce execution/grading/task states and maintain contract shapes | [Result reduction](../src/results.ts), [shared semantic types](../src/contract-types.ts), [wire types](../src/schema-types.ts), [runtime schema boundary](../src/schema.ts), [results contract](results-v1.md) | [State and exit precedence](../tests/src/results.test.ts), [evidence schema boundaries](../tests/src/schema/evidence-boundaries.test.ts) |
141
+ | Bind comparison identities and release provenance | [Preparation identity inputs](../src/evaluation/plan.ts), [identity hashing](../src/identity.ts), [provenance](../src/provenance.ts), [identity contract](identity-v1.md), [releases](releases.md) | [CLI identity](../tests/src/cli/identity.test.ts), [package release provenance](../tests/scripts/release.test.ts) |
142
+ | Summarize retained results | [Report implementation](../src/report.ts), [report contract](report-v1.md) | [Report states and CLI](../tests/src/report.test.ts) |
22
143
 
23
- These paths describe this checkout. [Historical validation](evidence.md) records
24
- bounded past observations. [Rust research](https://github.com/BjRo/sevro/issues/2)
25
- does not establish a migration; public interfaces remain useful independently.
144
+ [Rust research](https://github.com/BjRo/sevro/issues/2) does not establish a
145
+ migration; public interfaces remain useful independently. Return to the
146
+ [documentation hub](README.md).
@@ -1,11 +1,12 @@
1
1
  # Sevro brand assets
2
2
 
3
3
  Björn Rochel created the supplied logo in 2026. Its original artwork and white
4
- background are preserved in `sevro-logo.png`.
4
+ background are preserved in `sevro-logo.png` in the repository. The npm package
5
+ retains these permissions; download the artwork from the links below.
5
6
 
6
- - [Full logo](sevro-logo.png): supplied 1254 × 1254 PNG including the wordmark.
7
- - [Compact symbol](sevro-symbol.svg): the same artwork framed around the S track.
8
- - [Wordmark](sevro-wordmark.svg): the same artwork framed around the name.
7
+ - [Full logo](https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-logo.png): supplied 1254 × 1254 PNG including the wordmark.
8
+ - [Compact symbol](https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-symbol.svg): the same artwork framed around the S track.
9
+ - [Wordmark](https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-wordmark.svg): the same artwork framed around the name.
9
10
 
10
11
  SVG variants embed the supplied PNG; they are not traced vector redrawings.
11
12
  Keep a white backing on dark pages; no transparent variant is claimed. Use the
@@ -38,8 +38,10 @@ The shell-check process runner accepts a bounded evaluator-owned `run` string,
38
38
  optional `expectedExitCode` and `timeoutMs`, plus `expectExact`, `expectRegex`,
39
39
  `notRegex`, and regex `flags` for stdout. Exact matching removes one final
40
40
  newline; regex matching always uses multiline mode. The runner executes with
41
- `sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied,
42
- a credential-free environment, and caller-declared protected roots. Without a
41
+ `sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied
42
+ by default, a credential-free environment, and caller-declared protected roots.
43
+ The runtime [local HTTP option](runtime-v1.md#local-http-test-servers) permits
44
+ socket-using upstream tests while retaining outbound isolation. Without a
43
45
  stdout assertion, process output is discarded. With one, the runner captures
44
46
  at most 1 MiB in memory and retains only its digest and byte length. Oversized
45
47
  or invalid UTF-8 output is a grading error.
@@ -49,11 +51,16 @@ one bounded observation per check. A mismatched exit code or stdout assertion
49
51
  fails the task; an isolation, timeout, or process error makes grading
50
52
  an error. Callers must supply `shellIsolation.protectedRoots` for any shell
51
53
  check. The engine also protects its package source, project, results, user
52
- home, configured host homes, and active peer fixtures. The caller's list must
54
+ home, configured host homes, and other managed workspaces. The caller's list must
53
55
  include any other source worktrees or private roots. A peer path already
54
56
  canonicalized by the engine remains in the deny profile if that peer is
55
57
  removed before the sandbox starts; unverified missing roots still fail.
56
58
 
59
+ On macOS, validated runtime grants also permit limited
60
+ [ancestor metadata reads](runtime-v1.md#macos-ancestor-metadata) so libc can
61
+ canonicalize granted paths. Protected ancestor listings and ungranted sibling
62
+ data remain inaccessible.
63
+
57
64
  `sevro.git-head` accepts `kind: "changed"`, `"unchanged"`, or
58
65
  `"base-ancestor"` for a generated or repository Git fixture. The engine
59
66
  captures the fixture's base HEAD before the candidate turn, then compares the
@@ -98,3 +105,10 @@ shell checks and semantic hosts can read retained candidate native files through
98
105
  live native tree. Native transcripts are untrusted candidate evidence; a grader
99
106
  must not follow instructions inside them. Missing or partial capture remains
100
107
  explicit in the index and does not establish required complete evidence.
108
+
109
+ With [candidate artifact access](runtime-v1.md#candidate-artifact-access),
110
+ shell checks and semantic/advisory hosts can also read explicitly declared
111
+ subtrees in the candidate's original private home. Named paths reach commands
112
+ through `SEVRO_CANDIDATE_ARTIFACT_<UPPERCASE_NAME>`, independently of native
113
+ transcript access. These grants are read-only; other candidate home contents
114
+ remain protected. Missing or unsafe declared directories make grading an error.