@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +22 -6
- package/README.md +5 -1
- package/docs/README.md +19 -15
- package/docs/architecture.md +138 -17
- package/docs/assets/README.md +5 -4
- package/docs/builtin-graders-v1.md +17 -3
- package/docs/codex-host.md +155 -49
- package/docs/contracts.md +2 -1
- package/docs/creating-cases.md +9 -2
- package/docs/development-cli.md +187 -19
- package/docs/documentation-quality.md +12 -2
- package/docs/evaluation-workflow.md +110 -0
- package/docs/extension-protocol-v1.md +26 -3
- package/docs/getting-started.md +23 -2
- package/docs/guide-evaluation.md +33 -1
- package/docs/identity-v1.md +25 -2
- package/docs/installing.md +11 -11
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +196 -7
- package/docs/reading-results.md +102 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +31 -12
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +141 -4
- package/docs/run-settings-v1.md +147 -0
- package/docs/runtime-v1.md +307 -18
- package/docs/troubleshooting.md +12 -12
- package/docs/typescript-quality.md +89 -26
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +11 -5
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +63 -7
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +206 -0
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +73 -50
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +87 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +60 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +198 -0
- package/src/cli/hosts.ts +55 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +154 -0
- package/src/cli/presentation.ts +93 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +128 -0
- package/src/cli/run-options.ts +403 -0
- package/src/cli/run-overrides.ts +69 -0
- package/src/cli/settings.ts +154 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -591
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +13 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +235 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +133 -115
- package/src/evaluation/trial.ts +800 -0
- package/src/evaluation/types.ts +207 -0
- package/src/evidence/host-results.ts +85 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +34 -96
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +15 -76
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +183 -0
- package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
- package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
- package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/fixtures/workspace-state.ts +212 -0
- package/src/generated/cli-result.d.cts +2 -1
- package/src/generated/extension.d.cts +2 -1
- package/src/generated/report.cjs +1 -1
- package/src/generated/report.d.cts +2 -1
- package/src/generated/run-evidence.cjs +1 -1
- package/src/generated/run-evidence.d.cts +2 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/generated/runtime.d.cts +2 -1
- package/src/git.ts +93 -0
- package/src/graders/git-head.ts +29 -84
- package/src/graders/shell.ts +57 -68
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +331 -0
- package/src/hosts/claude-continuation.ts +57 -40
- package/src/hosts/claude-credential.ts +160 -63
- package/src/hosts/claude-events.ts +34 -18
- package/src/hosts/claude-keychain.ts +78 -21
- package/src/hosts/claude-login-recovery.ts +85 -0
- package/src/hosts/claude-login-refresh.ts +178 -0
- package/src/hosts/claude-login-store.ts +162 -0
- package/src/hosts/claude-login.ts +107 -0
- package/src/hosts/claude-native-goal.ts +15 -12
- package/src/hosts/claude-process.ts +10 -50
- package/src/hosts/claude-refresh-lock.ts +81 -0
- package/src/hosts/claude-repository-invocation.ts +9 -34
- package/src/hosts/claude-settings.ts +91 -14
- package/src/hosts/claude-skill-activations.ts +169 -0
- package/src/hosts/claude-state.ts +46 -16
- package/src/hosts/claude-tool-calls.ts +13 -10
- package/src/hosts/claude.ts +122 -73
- package/src/hosts/codex-app-server.ts +78 -52
- package/src/hosts/codex-configuration.ts +5 -10
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-marketplace.ts +12 -10
- package/src/hosts/codex-native-calls.ts +15 -122
- package/src/hosts/codex-process.ts +10 -54
- package/src/hosts/codex-profile.ts +50 -21
- package/src/hosts/codex-session-files.ts +1 -9
- package/src/hosts/codex-state.ts +141 -19
- package/src/hosts/codex.ts +221 -179
- package/src/hosts/isolation-common.ts +15 -11
- package/src/hosts/isolation-roots.ts +13 -20
- package/src/hosts/linux-sandbox.ts +27 -3
- package/src/hosts/mac-sandbox.ts +111 -18
- package/src/hosts/native-controls.ts +72 -22
- package/src/hosts/native-request.ts +55 -0
- package/src/hosts/process-lifecycle.ts +221 -45
- package/src/hosts/protected-worktrees.ts +1 -1
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +577 -0
- package/src/instrumentation.ts +7 -8
- package/src/native-transcript-state.ts +22 -3
- package/src/native-transcript-view.ts +52 -13
- package/src/native-transcripts.ts +34 -15
- package/src/paths.ts +22 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +27 -7
- package/src/prune.ts +300 -0
- package/src/report.ts +63 -36
- package/src/resolved-case.ts +1 -1
- package/src/results-lock.ts +42 -0
- package/src/results.ts +14 -18
- package/src/run-owner.ts +4 -4
- package/src/{runtime-config.ts → runtime/config.ts} +182 -44
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +33 -22
- package/src/{runtime-paths.ts → runtime/paths.ts} +6 -18
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/runtime/shared-cache.ts +130 -0
- package/src/{runtime-state.ts → runtime/state.ts} +28 -11
- package/src/schema-types.ts +73 -161
- package/src/shell.ts +4 -0
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +22 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +0 -1
- package/docs/assets/sevro-wordmark.svg +0 -1
- package/docs/cache-ownership-validation.md +0 -49
- package/docs/claude-continuation-validation.md +0 -47
- package/docs/concurrency-validation.md +0 -86
- package/docs/configuration-root-validation.md +0 -70
- package/docs/engine-slice.md +0 -114
- package/docs/evidence/runtime-startup-30/README.md +0 -185
- package/docs/evidence/runtime-startup-30/after.json +0 -319
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
- package/docs/evidence/runtime-startup-30/before.json +0 -403
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
- package/docs/evidence.md +0 -17
- package/docs/fixture-cleanup-validation.md +0 -70
- package/docs/guide-verification.json +0 -239
- package/src/evaluation-context.ts +0 -257
- package/src/evaluation-fixture.ts +0 -313
- package/src/evaluation-run.ts +0 -198
- package/src/evaluation-trial.ts +0 -946
- package/src/evaluation-types.ts +0 -149
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -510
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -609
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
package/CONTRIBUTING.md
CHANGED
|
@@ -26,8 +26,10 @@ by applicable law.
|
|
|
26
26
|
|
|
27
27
|
## Set up development
|
|
28
28
|
|
|
29
|
-
Use Bun 1.3.13, Git, Node 24, and
|
|
30
|
-
on macOS or `bubblewrap` and `socat` on Ubuntu.
|
|
29
|
+
Use Bun 1.3.13, Git, Node 24, npm, and ripgrep (`rg`). Full suite isolation uses `sandbox-exec`
|
|
30
|
+
on macOS or `bubblewrap` and `socat` on Ubuntu. The macOS synthetic libc regression
|
|
31
|
+
fixtures compile with `/usr/bin/cc`, supplied by Apple's Command Line Tools.
|
|
32
|
+
Windows acceptance is unverified.
|
|
31
33
|
From a fresh checkout:
|
|
32
34
|
|
|
33
35
|
```sh
|
|
@@ -36,6 +38,13 @@ bun run test:docs-examples
|
|
|
36
38
|
```
|
|
37
39
|
|
|
38
40
|
The second command checks the tutorial in fresh consumer/contributor directories.
|
|
41
|
+
Its complete recipe checks also require UV, Python 3, mise, and an existing
|
|
42
|
+
`mise install node@24.13.0` installation; see [recipe prerequisites](docs/recipes.md).
|
|
43
|
+
CI explicitly provisions these tools. Missing prerequisites fail with a named
|
|
44
|
+
setup instruction rather than skipping a recipe.
|
|
45
|
+
To run recipe commands from this source checkout, set
|
|
46
|
+
`export SEVRO_RECIPES="$PWD/examples/recipes"` from the repository root, then
|
|
47
|
+
follow the [recipe commands](docs/recipes.md#prepare-a-recipe-project).
|
|
39
48
|
Native model trials need their own [credentials and prerequisites](docs/native-hosts.md).
|
|
40
49
|
|
|
41
50
|
## Quality contract and checks
|
|
@@ -54,8 +63,9 @@ documentation and guide dry-run checks, tutorial examples, the real Bun suite,
|
|
|
54
63
|
coverage integrity and thresholds, and installed-package validation. Statements
|
|
55
64
|
and branches must each reach at least 95% using raw exact counts over the
|
|
56
65
|
platform-reachable authored production inventory, including unimported files.
|
|
57
|
-
The macOS and Ubuntu
|
|
58
|
-
module and branch-outcome exclusions are recorded in coverage
|
|
66
|
+
The local-only macOS gate and Ubuntu CI gate each measure their reachable host
|
|
67
|
+
modules; the exact module and branch-outcome exclusions are recorded in coverage
|
|
68
|
+
artifacts. The five explicitly
|
|
59
69
|
inventoried generated AJV validators are excluded from coverage because this
|
|
60
70
|
target measures authored runner behavior; their syntax, schema freshness, typed
|
|
61
71
|
boundaries and runtime tests remain checked. Tests and fixtures remain linted and
|
|
@@ -66,6 +76,12 @@ Before committing, use `bun run check:typescript --fast` after installing the
|
|
|
66
76
|
locked dependencies. This runs inventory, schema, formatting, lint, typing and
|
|
67
77
|
documentation checks; it does not replace full acceptance. CI retains machine-readable coverage
|
|
68
78
|
reports under `.quality/` and exposes the stable `TypeScript quality` status.
|
|
79
|
+
The complete command's `static`, `docs`, `tests` and `package` phases run as
|
|
80
|
+
concurrent CI jobs. CI divides the tests among four duration-balanced jobs and
|
|
81
|
+
requires one combined coverage gate; every test and both 95% thresholds remain
|
|
82
|
+
required. Superseded runs for the same pull request are cancelled. After frozen installation, use
|
|
83
|
+
`bun run check:typescript --phase <name>` to run one phase locally; it does not
|
|
84
|
+
replace acceptance across all four. See the [phase inventory](docs/typescript-quality.md).
|
|
69
85
|
|
|
70
86
|
Run `bun run check:docs:external` separately and [triage network failures](docs/documentation-quality.md#external-links).
|
|
71
87
|
Guide changes also need [native evals](docs/guide-evaluation.md).
|
|
@@ -82,8 +98,8 @@ repairing in-scope failures; it does not authorize merging or unrelated changes.
|
|
|
82
98
|
non-root user, and isolation prerequisites. Reproducing that environment locally
|
|
83
99
|
with Docker is optional for diagnosis and is not a publication prerequisite.
|
|
84
100
|
- After the local pre-commit checks, publish the authorized candidate to its PR
|
|
85
|
-
and require
|
|
86
|
-
delivery as complete. A local run, fast check, targeted suite, or historical
|
|
101
|
+
and require all four canonical `check:typescript` phases to succeed in GitHub CI
|
|
102
|
+
before reporting delivery as complete. A local run, fast check, targeted suite, or historical
|
|
87
103
|
result does not establish that the current Ubuntu gate passes.
|
|
88
104
|
- Keep the quality bar intact. Repair behavior or add meaningful coverage for
|
|
89
105
|
accepted behavior; do not lower thresholds, exempt authored sources, skip tests,
|
package/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Sevro
|
|
2
2
|
|
|
3
3
|
<p align="center">
|
|
4
|
-
<img src="docs/assets/sevro-logo.png" alt="Sevro logo: a black S-shaped track with red segments above the sevro wordmark" width="360">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-logo.png" alt="Sevro logo: a black S-shaped track with red segments above the sevro wordmark" width="360">
|
|
5
5
|
</p>
|
|
6
6
|
|
|
7
7
|
Sevro runs repeatable evaluations of coding agents. Give it a case, a host
|
|
@@ -40,6 +40,10 @@ Continue with [your first evaluation](docs/getting-started.md): the exact comman
|
|
|
40
40
|
expected `passed` result, and retained evidence. Read [installation](docs/installing.md)
|
|
41
41
|
for updates, removal, and source checkouts.
|
|
42
42
|
|
|
43
|
+
Then follow the [demo-to-iteration workflow](docs/evaluation-workflow.md):
|
|
44
|
+
initialize real assertions, prepare, execute, read progress and inspect the
|
|
45
|
+
latest result.
|
|
46
|
+
|
|
43
47
|
## Find your next task
|
|
44
48
|
|
|
45
49
|
The [documentation hub](docs/README.md) routes to running evaluations, authoring
|
package/docs/README.md
CHANGED
|
@@ -1,22 +1,25 @@
|
|
|
1
1
|
# Sevro documentation
|
|
2
2
|
|
|
3
|
-
<img src="assets/sevro-wordmark.svg" alt="Sevro wordmark" width="220">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-wordmark.svg" alt="Sevro wordmark" width="220">
|
|
4
4
|
|
|
5
5
|
Start with the task you want to complete. These pages work without an agent.
|
|
6
6
|
|
|
7
|
-
| I want to…
|
|
8
|
-
|
|
|
9
|
-
| Get a result without a model account
|
|
10
|
-
|
|
|
11
|
-
|
|
|
12
|
-
|
|
|
13
|
-
|
|
|
14
|
-
|
|
|
15
|
-
|
|
|
16
|
-
|
|
|
17
|
-
|
|
|
18
|
-
|
|
|
19
|
-
|
|
|
7
|
+
| I want to… | Read |
|
|
8
|
+
| ---------------------------------------------- | -------------------------------------------------------- |
|
|
9
|
+
| Get a result without a model account | [Your first evaluation](getting-started.md) |
|
|
10
|
+
| Initialize, prepare, run and iterate | [Evaluation workflow](evaluation-workflow.md) |
|
|
11
|
+
| Install, update, remove, or use a checkout | [Installation](installing.md) |
|
|
12
|
+
| Run evaluations and select a host | [Running evaluations](running-evaluations.md) |
|
|
13
|
+
| Create cases and reusable policy | [Cases and extensions](creating-cases.md) |
|
|
14
|
+
| Copy a complete coding, skill or runtime setup | [Runnable recipes](recipes.md) |
|
|
15
|
+
| Interpret verdicts, evidence, and comparisons | [Reading results](reading-results.md) |
|
|
16
|
+
| Run Codex or Claude with isolation | [Native hosts](native-hosts.md) |
|
|
17
|
+
| Reuse host tools or enable goals and hooks | [Runtime configuration](runtime-v1.md) |
|
|
18
|
+
| Save recurring evaluation selections | [Run settings](run-settings-v1.md) |
|
|
19
|
+
| Resolve a symptom | [Troubleshooting](troubleshooting.md) |
|
|
20
|
+
| Trace runner lifecycle, ownership, and changes | [Architecture](architecture.md) |
|
|
21
|
+
| Change Sevro or its documentation | [Contributing](../CONTRIBUTING.md) |
|
|
22
|
+
| Understand licensing and branding | [Licensing](licensing.md) and [assets](assets/README.md) |
|
|
20
23
|
|
|
21
24
|
## Ask the guide
|
|
22
25
|
|
|
@@ -30,11 +33,12 @@ It belongs to this repository, not an npm installation.
|
|
|
30
33
|
|
|
31
34
|
[Current contracts](contracts.md) define interfaces. The
|
|
32
35
|
[guide specification](specs/repository-guide.md) defines the repository assistant.
|
|
33
|
-
[Historical validation](evidence.md) records observations, not current guarantees.
|
|
36
|
+
[Historical validation](https://github.com/BjRo/sevro/blob/main/docs/evidence.md) records observations, not current guarantees.
|
|
34
37
|
There is no accepted Rust migration decision here;
|
|
35
38
|
[issue #2](https://github.com/BjRo/sevro/issues/2) tracks research.
|
|
36
39
|
|
|
37
40
|
Read [documentation quality](documentation-quality.md),
|
|
41
|
+
[the usability study protocol](https://github.com/BjRo/sevro/blob/main/docs/usability-study.md),
|
|
38
42
|
[guide evaluation](guide-evaluation.md), [releases](releases.md), and
|
|
39
43
|
[acknowledgements](acknowledgements.md) for maintenance.
|
|
40
44
|
Return to the [Sevro introduction](../README.md).
|
package/docs/architecture.md
CHANGED
|
@@ -4,22 +4,143 @@ Sevro owns generic execution/evidence contracts. Projects own cases and policy.
|
|
|
4
4
|
This lets Darrow and other integrations use the same runner without importing
|
|
5
5
|
their benchmark policy into it.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
7
|
+
This page explains this checkout's implementation and points to acceptance
|
|
8
|
+
tests for changes. [Current contracts](contracts.md) define public behavior;
|
|
9
|
+
[historical validation](https://github.com/BjRo/sevro/blob/main/docs/evidence.md) records bounded past observations. The
|
|
10
|
+
[engine slice](https://github.com/BjRo/sevro/blob/main/docs/engine-slice.md) retains its earlier scope and detailed lifecycle
|
|
11
|
+
explanation. Implementation paths and test scenarios here are navigation, not
|
|
12
|
+
evidence that a particular candidate passed validation.
|
|
13
|
+
|
|
14
|
+
[`engine.ts`](../src/engine.ts) remains the programmatic facade, and
|
|
15
|
+
[`cli.ts`](../src/cli.ts) remains the installed command entrypoint. The CLI
|
|
16
|
+
loads and assembles evaluation options; evaluation modules coordinate the run
|
|
17
|
+
and its trials.
|
|
18
|
+
|
|
19
|
+
## Evaluation lifecycle
|
|
20
|
+
|
|
21
|
+
[`runEvaluation`](../src/evaluation/run.ts) prepares inputs before allocating
|
|
22
|
+
run resources, retains preparation artifacts, and reserves candidate and
|
|
23
|
+
semantic workspace roots as needed before calling the trial scheduler. The map
|
|
24
|
+
follows the path of a retained trial; individual stages can be skipped or fail.
|
|
25
|
+
|
|
26
|
+
```mermaid
|
|
27
|
+
flowchart TD
|
|
28
|
+
P["Validate and prepare evaluation inputs"] --> R["Allocate run storage and owner; reserve workspace roots"]
|
|
29
|
+
R --> W["Admit trial and materialize its reserved fixture"]
|
|
30
|
+
subgraph Trial["Each admitted trial"]
|
|
31
|
+
W --> C["Initialize grading bases; execute candidate"]
|
|
32
|
+
C --> E["Retain candidate artifacts; observe output; prepare grading views"]
|
|
33
|
+
E --> G["Output, shell, Git, semantic, and extension grading"]
|
|
34
|
+
G --> A["Assess execution, grading, and task separately"]
|
|
35
|
+
A --> V["Optional independent advisory review"]
|
|
36
|
+
V --> T["Retain raw candidate response; atomically write trial evidence"]
|
|
37
|
+
T --> K["Collect completed trial and checkpoint its reference"]
|
|
38
|
+
K --> D["Release temporary views; clear or retain fixture contents"]
|
|
39
|
+
end
|
|
40
|
+
D --> J["All admitted trials drain"]
|
|
41
|
+
J --> F["Write final run evidence; finalize owner checkpoint"]
|
|
42
|
+
F --> X["Finalize retained workspaces; release other reservations"]
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
[`scheduleTrials`](../src/evaluation/scheduler.ts) bounds active work by the
|
|
46
|
+
prepared job limit. It admits a trial before
|
|
47
|
+
[`prepareTrialWorkspace`](../src/evaluation/resources.ts) materializes that
|
|
48
|
+
trial's fixture. Each [`EvaluationTrial`](../src/evaluation/trial.ts) owns its
|
|
49
|
+
mutable state and writer. Its `run()` method keeps candidate execution,
|
|
50
|
+
artifact retention, grading, assessment, advisory review, persistence, and
|
|
51
|
+
cleanup in one visible order. Grading helpers return checks, observations,
|
|
52
|
+
and failures; the coordinator incorporates those results and stops new
|
|
53
|
+
admission when an execution or required grading error is detected, before
|
|
54
|
+
advisory work, persistence, or cleanup. The scheduler also stops admission on
|
|
55
|
+
cancellation and waits for active workers. A failed task check alone does not
|
|
56
|
+
close admission. Already admitted peers drain and retain their evidence, including
|
|
57
|
+
when they finish out of order.
|
|
58
|
+
|
|
59
|
+
[`runAdvisoryReview`](../src/evaluation/advisory.ts) handles its own host call,
|
|
60
|
+
assessment, retained response, and auxiliary workspace cleanup. It depends on
|
|
61
|
+
candidate completion, independently of the task assessment. The
|
|
20
62
|
[advisory fixture](advisory-fixture.md) gives review a separate Git view with
|
|
21
|
-
evaluator inputs withheld.
|
|
63
|
+
evaluator inputs withheld. Its recommendation or host failure does not change
|
|
64
|
+
task grading; a failure to retain its evidence remains a persistence error.
|
|
65
|
+
|
|
66
|
+
Successful [semantic response parsing](../src/evaluation/semantic.ts) clears
|
|
67
|
+
auxiliary fixture contents; successful advisory response parsing removes its
|
|
68
|
+
auxiliary fixture. Both retain response evidence first, including valid failed
|
|
69
|
+
check verdicts or advisory recommendations.
|
|
70
|
+
|
|
71
|
+
The [trial writer](../src/evidence/trial-writer.ts) writes the trial artifact
|
|
72
|
+
before the coordinator marks it persisted and hands it to the
|
|
73
|
+
[result collector](../src/evaluation/collection.ts). The collector registers
|
|
74
|
+
the completed record, orders records by trial number, and checkpoints retained
|
|
75
|
+
artifact paths. After active work drains, the run coordinator validates and
|
|
76
|
+
writes `run.json` before finalizing the owner as complete or interrupted.
|
|
77
|
+
Persistence failures cannot produce a successful result; the run's failure
|
|
78
|
+
path attempts a diagnostic checkpoint while preserving the original failure.
|
|
79
|
+
|
|
80
|
+
Trial cleanup releases candidate artifact runtimes and transcript views even
|
|
81
|
+
when the trial fails. Persisted successful fixture contents are cleared while
|
|
82
|
+
their empty reserved roots remain until active work drains. Failed, cancelled,
|
|
83
|
+
grading-error, unavailable-evidence, and failed-task fixtures are retained;
|
|
84
|
+
[workspace ownership](../src/fixtures/workspace-state.ts) tracks their inactive
|
|
85
|
+
retention and startup expiry after 24 hours. Saved evidence stays outside that
|
|
86
|
+
cleanup. See the [results retention contract](results-v1.md#identity-and-provenance)
|
|
87
|
+
for the public boundary, including ownership that cannot be verified.
|
|
88
|
+
|
|
89
|
+
Dry preparation still validates and materializes fixtures, including declared
|
|
90
|
+
fixture setup, and retains trial records. It skips candidate execution and
|
|
91
|
+
grading, recording `not_run` / `not_requested` / `not_assessed`, as exercised by
|
|
92
|
+
the [engine trial suite](../tests/src/engine-trials.test.ts). Missing observations
|
|
93
|
+
remain unavailable; missing required evidence cannot become a pass. Optional
|
|
94
|
+
unknown measurements remain unknown rather than becoming zero. Extensions
|
|
95
|
+
cannot silently replace isolation, persistence,
|
|
96
|
+
or built-in policy. The [results](results-v1.md) and
|
|
97
|
+
[extension protocol](extension-protocol-v1.md) contracts define those semantics.
|
|
98
|
+
|
|
99
|
+
## State ownership
|
|
100
|
+
|
|
101
|
+
[`EvaluationContext`](../src/evaluation/context.ts) carries four explicit
|
|
102
|
+
collaborators: prepared inputs, run resources, the result collector, and retained
|
|
103
|
+
preparation-artifact references. It does not combine mutable trial fields with
|
|
104
|
+
the preparation plan.
|
|
105
|
+
|
|
106
|
+
| Data or state | Owner | Boundary |
|
|
107
|
+
| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
108
|
+
| Prepared inputs | [`PreparedEvaluation` in `plan.ts`](../src/evaluation/plan.ts) | Named fixture, check, and invocation groups plus routes, provenance, redacted configuration, and comparison identity. Preparation completes before run allocation. |
|
|
109
|
+
| Run resources and reservations | [`RunResources` and `RunWorkspaceReservations` in `resources.ts`](../src/evaluation/resources.ts) | Run ID, evidence/state locations, process owner, and candidate/semantic reservations. Reservations outlive completed fixture contents; retained workspaces are tracked separately from completed roots. |
|
|
110
|
+
| Mutable trial state | [`TrialRuntimeState` and `EvaluationTrial` in `trial.ts`](../src/evaluation/trial.ts) | Candidate result, observations, checks, assessment, temporary views, advisory result, and persisted milestone belong to one admitted trial. Grading receives narrow inputs and explicit effects rather than this state object. |
|
|
111
|
+
| Collected run results | [`ResultCollector` in `collection.ts`](../src/evaluation/collection.ts) | Private completed summaries, evidence, and diagnostics; trial-number ordering, snapshots for aggregation, and checkpoint requests. [`run-owner.ts`](../src/run-owner.ts) verifies process ownership and writes owner/checkpoint state. |
|
|
112
|
+
| Evidence writes | [`TrialEvidenceWriter`](../src/evidence/trial-writer.ts), [preparation retention](../src/evaluation/resources.ts), [run finalization](../src/evaluation/run.ts) | The trial writer owns filenames, private writes, hashes, artifact references, and typed trial-persistence failures. Resource helpers retain preparation bytes; run finalization writes aggregate evidence. [`storage.ts`](../src/storage.ts) supplies atomic private JSON writes. |
|
|
113
|
+
|
|
114
|
+
Fixture modules own materialization and workspace preparation. Runtime modules
|
|
115
|
+
own declared host runtime policy and private state. Evidence modules validate
|
|
116
|
+
host observations and artifacts, verify retained bytes, and normalize optional
|
|
117
|
+
usage without constructing fixtures. Native host implementations and graders
|
|
118
|
+
remain in their own families.
|
|
119
|
+
|
|
120
|
+
## Where to change behavior
|
|
121
|
+
|
|
122
|
+
Start with the responsibility, then inspect its observable acceptance scenarios.
|
|
123
|
+
These are useful starting suites, not an exhaustive validation list. The
|
|
124
|
+
[contribution checks](../CONTRIBUTING.md#quality-contract-and-checks) still govern
|
|
125
|
+
candidate acceptance.
|
|
126
|
+
|
|
127
|
+
| Responsibility | Implementation and reference | Relevant acceptance tests |
|
|
128
|
+
| ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
129
|
+
| Parse commands/select explicit routes | [CLI entrypoint](../src/cli.ts), [invocation parsing](../src/cli/invocation.ts), [CLI reference](development-cli.md) | [Invalid invocations](../tests/src/cli/invocation-validation.test.ts) |
|
|
130
|
+
| Load and prepare a CLI evaluation | [Input loading](../src/cli/loading.ts), [preparation](../src/cli/preparation.ts), [host construction](../src/cli/hosts.ts), [extension resolution](../src/cli/extension.ts), [evaluation assembly](../src/cli/evaluation.ts) | [Case/configuration validation](../tests/src/cli/case-validation.test.ts) |
|
|
131
|
+
| Present CLI results and own cancellation | [Presentation](../src/cli/presentation.ts), [cancellation](../src/cli/cancellation.ts), [command lifecycle](../src/cli/command.ts) | [CLI states and exits](../tests/src/cli/results.test.ts), [signal retention](../tests/src/cli/trial-retention.test.ts) |
|
|
132
|
+
| Prepare evaluation inputs and grade trials | [Engine facade](../src/engine.ts), [plan](../src/evaluation/plan.ts), [grading](../src/evaluation/grading.ts), [semantic grading](../src/evaluation/semantic.ts), [evaluation modules](../src/evaluation/), [engine slice](https://github.com/BjRo/sevro/blob/main/docs/engine-slice.md) | [Preparation refusals](../tests/src/evaluation/preparation-validation.test.ts), [semantic grading failures](../tests/src/evaluation/semantic-grading-failures.test.ts), [engine lifecycle](../tests/src/engine-lifecycle.test.ts) |
|
|
133
|
+
| Schedule trials, retain evidence, and finalize ownership | [Scheduler](../src/evaluation/scheduler.ts), [trial coordinator](../src/evaluation/trial.ts), [reservations](../src/evaluation/resources.ts), [collection](../src/evaluation/collection.ts), [run owner](../src/run-owner.ts) | [Concurrent admission and ordering](../tests/src/cli/concurrency.test.ts), [trial failures and retention](../tests/src/evaluation/trial-retention.test.ts), [checkpoint refusals](../tests/src/run-owner-checkpoints.test.ts) |
|
|
134
|
+
| Prepare, construct, and clean up fixtures | [Fixture modules](../src/fixtures/), [construction](../src/fixtures/create.ts), [preparation artifacts](../src/fixtures/preparation.ts), [content cleanup](../src/fixtures/cleanup.ts), [workspace ownership](../src/fixtures/workspace-state.ts) | [Artifact preparation](../tests/src/fixtures/artifact-preparation.test.ts), [workspace lifecycle](../tests/src/fixtures/workspace-lifecycle.test.ts) |
|
|
135
|
+
| Resolve runtime policy, discovery, and private state | [Runtime modules](../src/runtime/), [configuration](../src/runtime/config.ts) | [Runtime environment](../tests/src/runtime/environment.test.ts), [candidate artifact lifetime](../tests/src/candidate-artifacts.test.ts) |
|
|
136
|
+
| Validate host results and retain bounded evidence | [Host results](../src/evidence/host-results.ts), [trial writer](../src/evidence/trial-writer.ts), [retained artifacts](../src/evidence/retained-artifacts.ts), [usage](../src/evidence/usage.ts) | [Host evidence refusals](../tests/src/evaluation/host-evidence.test.ts), [failed-host evidence retention](../tests/src/evaluation/trial-retention.test.ts) |
|
|
137
|
+
| Negotiate trusted extension calls | [Client](../src/extension-client.ts), [session](../src/extension-session.ts), [extension grading](../src/evaluation/grading.ts), [protocol](extension-protocol-v1.md) | [Negotiated engine behavior](../tests/src/evaluation/extension-engine.test.ts), [extension grading outcomes](../tests/src/evaluation/extension-grading.test.ts) |
|
|
138
|
+
| Run independent advisory review | [Advisory execution](../src/evaluation/advisory.ts), [blind fixture](../src/fixtures/advisory.ts), [advisory fixture explanation](advisory-fixture.md) | [Advisory failures](../tests/src/evaluation/advisory-failures.test.ts), [blind change view](../tests/src/fixtures/advisory.test.ts) |
|
|
139
|
+
| Run native hosts and retain bounded observations | [Codex](codex-host.md), [native controls](native-controls.md), [host sources](../src/hosts/), [shared process mechanics](../src/process-lifecycle.ts), [native waiting](../src/hosts/process-lifecycle.ts) | [Native process admission and byte bounds](../tests/src/hosts/native-process-boundaries.test.ts) |
|
|
140
|
+
| Reduce execution/grading/task states and maintain contract shapes | [Result reduction](../src/results.ts), [shared semantic types](../src/contract-types.ts), [wire types](../src/schema-types.ts), [runtime schema boundary](../src/schema.ts), [results contract](results-v1.md) | [State and exit precedence](../tests/src/results.test.ts), [evidence schema boundaries](../tests/src/schema/evidence-boundaries.test.ts) |
|
|
141
|
+
| Bind comparison identities and release provenance | [Preparation identity inputs](../src/evaluation/plan.ts), [identity hashing](../src/identity.ts), [provenance](../src/provenance.ts), [identity contract](identity-v1.md), [releases](releases.md) | [CLI identity](../tests/src/cli/identity.test.ts), [package release provenance](../tests/scripts/release.test.ts) |
|
|
142
|
+
| Summarize retained results | [Report implementation](../src/report.ts), [report contract](report-v1.md) | [Report states and CLI](../tests/src/report.test.ts) |
|
|
22
143
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
144
|
+
[Rust research](https://github.com/BjRo/sevro/issues/2) does not establish a
|
|
145
|
+
migration; public interfaces remain useful independently. Return to the
|
|
146
|
+
[documentation hub](README.md).
|
package/docs/assets/README.md
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
# Sevro brand assets
|
|
2
2
|
|
|
3
3
|
Björn Rochel created the supplied logo in 2026. Its original artwork and white
|
|
4
|
-
background are preserved in `sevro-logo.png
|
|
4
|
+
background are preserved in `sevro-logo.png` in the repository. The npm package
|
|
5
|
+
retains these permissions; download the artwork from the links below.
|
|
5
6
|
|
|
6
|
-
- [Full logo](sevro-logo.png): supplied 1254 × 1254 PNG including the wordmark.
|
|
7
|
-
- [Compact symbol](sevro-symbol.svg): the same artwork framed around the S track.
|
|
8
|
-
- [Wordmark](sevro-wordmark.svg): the same artwork framed around the name.
|
|
7
|
+
- [Full logo](https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-logo.png): supplied 1254 × 1254 PNG including the wordmark.
|
|
8
|
+
- [Compact symbol](https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-symbol.svg): the same artwork framed around the S track.
|
|
9
|
+
- [Wordmark](https://raw.githubusercontent.com/BjRo/sevro/main/docs/assets/sevro-wordmark.svg): the same artwork framed around the name.
|
|
9
10
|
|
|
10
11
|
SVG variants embed the supplied PNG; they are not traced vector redrawings.
|
|
11
12
|
Keep a white backing on dark pages; no transparent variant is claimed. Use the
|
|
@@ -38,8 +38,10 @@ The shell-check process runner accepts a bounded evaluator-owned `run` string,
|
|
|
38
38
|
optional `expectedExitCode` and `timeoutMs`, plus `expectExact`, `expectRegex`,
|
|
39
39
|
`notRegex`, and regex `flags` for stdout. Exact matching removes one final
|
|
40
40
|
newline; regex matching always uses multiline mode. The runner executes with
|
|
41
|
-
`sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied
|
|
42
|
-
a credential-free environment, and caller-declared protected roots.
|
|
41
|
+
`sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied
|
|
42
|
+
by default, a credential-free environment, and caller-declared protected roots.
|
|
43
|
+
The runtime [local HTTP option](runtime-v1.md#local-http-test-servers) permits
|
|
44
|
+
socket-using upstream tests while retaining outbound isolation. Without a
|
|
43
45
|
stdout assertion, process output is discarded. With one, the runner captures
|
|
44
46
|
at most 1 MiB in memory and retains only its digest and byte length. Oversized
|
|
45
47
|
or invalid UTF-8 output is a grading error.
|
|
@@ -49,11 +51,16 @@ one bounded observation per check. A mismatched exit code or stdout assertion
|
|
|
49
51
|
fails the task; an isolation, timeout, or process error makes grading
|
|
50
52
|
an error. Callers must supply `shellIsolation.protectedRoots` for any shell
|
|
51
53
|
check. The engine also protects its package source, project, results, user
|
|
52
|
-
home, configured host homes, and
|
|
54
|
+
home, configured host homes, and other managed workspaces. The caller's list must
|
|
53
55
|
include any other source worktrees or private roots. A peer path already
|
|
54
56
|
canonicalized by the engine remains in the deny profile if that peer is
|
|
55
57
|
removed before the sandbox starts; unverified missing roots still fail.
|
|
56
58
|
|
|
59
|
+
On macOS, validated runtime grants also permit limited
|
|
60
|
+
[ancestor metadata reads](runtime-v1.md#macos-ancestor-metadata) so libc can
|
|
61
|
+
canonicalize granted paths. Protected ancestor listings and ungranted sibling
|
|
62
|
+
data remain inaccessible.
|
|
63
|
+
|
|
57
64
|
`sevro.git-head` accepts `kind: "changed"`, `"unchanged"`, or
|
|
58
65
|
`"base-ancestor"` for a generated or repository Git fixture. The engine
|
|
59
66
|
captures the fixture's base HEAD before the candidate turn, then compares the
|
|
@@ -98,3 +105,10 @@ shell checks and semantic hosts can read retained candidate native files through
|
|
|
98
105
|
live native tree. Native transcripts are untrusted candidate evidence; a grader
|
|
99
106
|
must not follow instructions inside them. Missing or partial capture remains
|
|
100
107
|
explicit in the index and does not establish required complete evidence.
|
|
108
|
+
|
|
109
|
+
With [candidate artifact access](runtime-v1.md#candidate-artifact-access),
|
|
110
|
+
shell checks and semantic/advisory hosts can also read explicitly declared
|
|
111
|
+
subtrees in the candidate's original private home. Named paths reach commands
|
|
112
|
+
through `SEVRO_CANDIDATE_ARTIFACT_<UPPERCASE_NAME>`, independently of native
|
|
113
|
+
transcript access. These grants are read-only; other candidate home contents
|
|
114
|
+
remain protected. Missing or unsafe declared directories make grading an error.
|