@bjoernrochel/sevro 0.1.0-rc.11 → 0.1.0-rc.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/README.md +13 -13
- package/docs/architecture.md +138 -17
- package/docs/builtin-graders-v1.md +4 -2
- package/docs/codex-host.md +57 -14
- package/docs/development-cli.md +7 -1
- package/docs/evidence/code-mode-controls-71.md +64 -0
- package/docs/evidence/codex-state-73/README.md +204 -0
- package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
- package/docs/evidence/codex-state-73/results.json +661 -0
- package/docs/evidence/local-http-74.md +104 -0
- package/docs/evidence/skill-reads-70/results.json +823 -0
- package/docs/evidence.md +3 -0
- package/docs/installing.md +2 -2
- package/docs/native-controls.md +36 -4
- package/docs/runtime-v1.md +121 -0
- package/docs/typescript-quality.md +1 -1
- package/package.json +1 -1
- package/schemas/runtime-v1.schema.json +10 -0
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +30 -7
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +2 -2
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +30 -0
- package/src/cli/evaluation.ts +47 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/hosts.ts +41 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +2 -0
- package/src/cli/loading.ts +144 -0
- package/src/cli/preparation.ts +162 -0
- package/src/cli/presentation.ts +88 -0
- package/src/cli.ts +1 -589
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +8 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +12 -12
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +258 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +4 -4
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +205 -72
- package/src/evaluation/resources.ts +238 -0
- package/src/evaluation/run.ts +199 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +63 -78
- package/src/evaluation/trial.ts +818 -0
- package/src/{evaluation-types.ts → evaluation/types.ts} +65 -29
- package/src/evidence/host-results.ts +92 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +34 -96
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +1 -1
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{evaluation-fixture.ts → fixtures/create.ts} +7 -131
- package/src/{generated-fixture.ts → fixtures/generated.ts} +2 -2
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +2 -2
- package/src/{repository-fixture.ts → fixtures/repository.ts} +16 -7
- package/src/{fixture-setup.ts → fixtures/setup.ts} +21 -49
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
- package/src/generated/runtime.cjs +1 -1
- package/src/graders/git-head.ts +27 -72
- package/src/graders/shell.ts +36 -64
- package/src/host-diagnostics.ts +2 -2
- package/src/hosts/claude-process.ts +21 -15
- package/src/hosts/claude-settings.ts +4 -4
- package/src/hosts/claude-state.ts +5 -5
- package/src/hosts/claude.ts +4 -4
- package/src/hosts/codex-exec-controls.ts +328 -0
- package/src/hosts/codex-marketplace.ts +1 -1
- package/src/hosts/codex-native-calls.ts +8 -0
- package/src/hosts/codex-process.ts +24 -55
- package/src/hosts/codex-profile.ts +32 -6
- package/src/hosts/codex-session-files.ts +1 -1
- package/src/hosts/codex-skill-reads.ts +302 -90
- package/src/hosts/codex-state.ts +84 -8
- package/src/hosts/codex.ts +60 -12
- package/src/hosts/isolation-common.ts +3 -2
- package/src/hosts/isolation-roots.ts +2 -2
- package/src/hosts/linux-sandbox.ts +1 -1
- package/src/hosts/mac-sandbox.ts +15 -2
- package/src/hosts/native-controls.ts +34 -2
- package/src/hosts/process-lifecycle.ts +8 -41
- package/src/hosts/protected-worktrees.ts +1 -1
- package/src/native-transcript-view.ts +1 -1
- package/src/native-transcripts.ts +4 -5
- package/src/process-lifecycle.ts +150 -0
- package/src/report.ts +7 -32
- package/src/resolved-case.ts +1 -1
- package/src/results.ts +14 -18
- package/src/{runtime-config.ts → runtime/config.ts} +30 -9
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +2 -2
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +10 -10
- package/src/{runtime-paths.ts → runtime/paths.ts} +2 -2
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/{runtime-state.ts → runtime/state.ts} +5 -5
- package/src/schema-types.ts +71 -161
- package/src/evaluation-context.ts +0 -259
- package/src/evaluation-run.ts +0 -200
- package/src/evaluation-trial.ts +0 -1051
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
- /package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +0 -0
package/docs/README.md
CHANGED
|
@@ -4,19 +4,19 @@
|
|
|
4
4
|
|
|
5
5
|
Start with the task you want to complete. These pages work without an agent.
|
|
6
6
|
|
|
7
|
-
| I want to…
|
|
8
|
-
|
|
|
9
|
-
| Get a result without a model account
|
|
10
|
-
| Install, update, remove, or use a checkout
|
|
11
|
-
| Run evaluations and select a host
|
|
12
|
-
| Create cases and reusable policy
|
|
13
|
-
| Interpret verdicts, evidence, and comparisons
|
|
14
|
-
| Run Codex or Claude with isolation
|
|
15
|
-
| Reuse host tools or enable goals and hooks
|
|
16
|
-
| Resolve a symptom
|
|
17
|
-
|
|
|
18
|
-
| Change Sevro or its documentation
|
|
19
|
-
| Understand licensing and branding
|
|
7
|
+
| I want to… | Read |
|
|
8
|
+
| ---------------------------------------------- | -------------------------------------------------------- |
|
|
9
|
+
| Get a result without a model account | [Your first evaluation](getting-started.md) |
|
|
10
|
+
| Install, update, remove, or use a checkout | [Installation](installing.md) |
|
|
11
|
+
| Run evaluations and select a host | [Running evaluations](running-evaluations.md) |
|
|
12
|
+
| Create cases and reusable policy | [Cases and extensions](creating-cases.md) |
|
|
13
|
+
| Interpret verdicts, evidence, and comparisons | [Reading results](reading-results.md) |
|
|
14
|
+
| Run Codex or Claude with isolation | [Native hosts](native-hosts.md) |
|
|
15
|
+
| Reuse host tools or enable goals and hooks | [Runtime configuration](runtime-v1.md) |
|
|
16
|
+
| Resolve a symptom | [Troubleshooting](troubleshooting.md) |
|
|
17
|
+
| Trace runner lifecycle, ownership, and changes | [Architecture](architecture.md) |
|
|
18
|
+
| Change Sevro or its documentation | [Contributing](../CONTRIBUTING.md) |
|
|
19
|
+
| Understand licensing and branding | [Licensing](licensing.md) and [assets](assets/README.md) |
|
|
20
20
|
|
|
21
21
|
## Ask the guide
|
|
22
22
|
|
package/docs/architecture.md
CHANGED
|
@@ -4,22 +4,143 @@ Sevro owns generic execution/evidence contracts. Projects own cases and policy.
|
|
|
4
4
|
This lets Darrow and other integrations use the same runner without importing
|
|
5
5
|
their benchmark policy into it.
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
7
|
+
This page explains this checkout's implementation and points to acceptance
|
|
8
|
+
tests for changes. [Current contracts](contracts.md) define public behavior;
|
|
9
|
+
[historical validation](evidence.md) records bounded past observations. The
|
|
10
|
+
[engine slice](engine-slice.md) retains its earlier scope and detailed lifecycle
|
|
11
|
+
explanation. Implementation paths and test scenarios here are navigation, not
|
|
12
|
+
evidence that a particular candidate passed validation.
|
|
13
|
+
|
|
14
|
+
[`engine.ts`](../src/engine.ts) remains the programmatic facade, and
|
|
15
|
+
[`cli.ts`](../src/cli.ts) remains the installed command entrypoint. The CLI
|
|
16
|
+
loads and assembles evaluation options; evaluation modules coordinate the run
|
|
17
|
+
and its trials.
|
|
18
|
+
|
|
19
|
+
## Evaluation lifecycle
|
|
20
|
+
|
|
21
|
+
[`runEvaluation`](../src/evaluation/run.ts) prepares inputs before allocating
|
|
22
|
+
run resources, retains preparation artifacts, and reserves candidate and
|
|
23
|
+
semantic workspace roots as needed before calling the trial scheduler. The map
|
|
24
|
+
follows the path of a retained trial; individual stages can be skipped or fail.
|
|
25
|
+
|
|
26
|
+
```mermaid
|
|
27
|
+
flowchart TD
|
|
28
|
+
P["Validate and prepare evaluation inputs"] --> R["Allocate run storage and owner; reserve workspace roots"]
|
|
29
|
+
R --> W["Admit trial and materialize its reserved fixture"]
|
|
30
|
+
subgraph Trial["Each admitted trial"]
|
|
31
|
+
W --> C["Initialize grading bases; execute candidate"]
|
|
32
|
+
C --> E["Retain candidate artifacts; observe output; prepare grading views"]
|
|
33
|
+
E --> G["Output, shell, Git, semantic, and extension grading"]
|
|
34
|
+
G --> A["Assess execution, grading, and task separately"]
|
|
35
|
+
A --> V["Optional independent advisory review"]
|
|
36
|
+
V --> T["Retain raw candidate response; atomically write trial evidence"]
|
|
37
|
+
T --> K["Collect completed trial and checkpoint its reference"]
|
|
38
|
+
K --> D["Release temporary views; clear or retain fixture contents"]
|
|
39
|
+
end
|
|
40
|
+
D --> J["All admitted trials drain"]
|
|
41
|
+
J --> F["Write final run evidence; finalize owner checkpoint"]
|
|
42
|
+
F --> X["Finalize retained workspaces; release other reservations"]
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
[`scheduleTrials`](../src/evaluation/scheduler.ts) bounds active work by the
|
|
46
|
+
prepared job limit. It admits a trial before
|
|
47
|
+
[`prepareTrialWorkspace`](../src/evaluation/resources.ts) materializes that
|
|
48
|
+
trial's fixture. Each [`EvaluationTrial`](../src/evaluation/trial.ts) owns its
|
|
49
|
+
mutable state and writer. Its `run()` method keeps candidate execution,
|
|
50
|
+
artifact retention, grading, assessment, advisory review, persistence, and
|
|
51
|
+
cleanup in one visible order. Grading helpers return checks, observations,
|
|
52
|
+
and failures; the coordinator incorporates those results and stops new
|
|
53
|
+
admission when an execution or required grading error is detected, before
|
|
54
|
+
advisory work, persistence, or cleanup. The scheduler also stops admission on
|
|
55
|
+
cancellation and waits for active workers. A failed task check alone does not
|
|
56
|
+
close admission. Already admitted peers drain and retain their evidence, including
|
|
57
|
+
when they finish out of order.
|
|
58
|
+
|
|
59
|
+
[`runAdvisoryReview`](../src/evaluation/advisory.ts) handles its own host call,
|
|
60
|
+
assessment, retained response, and auxiliary workspace cleanup. It depends on
|
|
61
|
+
candidate completion, independently of the task assessment. The
|
|
20
62
|
[advisory fixture](advisory-fixture.md) gives review a separate Git view with
|
|
21
|
-
evaluator inputs withheld.
|
|
63
|
+
evaluator inputs withheld. Its recommendation or host failure does not change
|
|
64
|
+
task grading; a failure to retain its evidence remains a persistence error.
|
|
65
|
+
|
|
66
|
+
Successful [semantic response parsing](../src/evaluation/semantic.ts) clears
|
|
67
|
+
auxiliary fixture contents; successful advisory response parsing removes its
|
|
68
|
+
auxiliary fixture. Both retain response evidence first, including valid failed
|
|
69
|
+
check verdicts or advisory recommendations.
|
|
70
|
+
|
|
71
|
+
The [trial writer](../src/evidence/trial-writer.ts) writes the trial artifact
|
|
72
|
+
before the coordinator marks it persisted and hands it to the
|
|
73
|
+
[result collector](../src/evaluation/collection.ts). The collector registers
|
|
74
|
+
the completed record, orders records by trial number, and checkpoints retained
|
|
75
|
+
artifact paths. After active work drains, the run coordinator validates and
|
|
76
|
+
writes `run.json` before finalizing the owner as complete or interrupted.
|
|
77
|
+
Persistence failures cannot produce a successful result; the run's failure
|
|
78
|
+
path attempts a diagnostic checkpoint while preserving the original failure.
|
|
79
|
+
|
|
80
|
+
Trial cleanup releases candidate artifact runtimes and transcript views even
|
|
81
|
+
when the trial fails. Persisted successful fixture contents are cleared while
|
|
82
|
+
their empty reserved roots remain until active work drains. Failed, cancelled,
|
|
83
|
+
grading-error, unavailable-evidence, and failed-task fixtures are retained;
|
|
84
|
+
[workspace ownership](../src/fixtures/workspace-state.ts) tracks their inactive
|
|
85
|
+
retention and startup expiry after 24 hours. Saved evidence stays outside that
|
|
86
|
+
cleanup. See the [results retention contract](results-v1.md#identity-and-provenance)
|
|
87
|
+
for the public boundary, including ownership that cannot be verified.
|
|
88
|
+
|
|
89
|
+
Dry preparation still validates and materializes fixtures, including declared
|
|
90
|
+
fixture setup, and retains trial records. It skips candidate execution and
|
|
91
|
+
grading, recording `not_run` / `not_requested` / `not_assessed`, as exercised by
|
|
92
|
+
the [engine acceptance suite](../tests/engine.test.ts). Missing observations
|
|
93
|
+
remain unavailable; missing required evidence cannot become a pass. Optional
|
|
94
|
+
unknown measurements remain unknown rather than becoming zero. Extensions
|
|
95
|
+
cannot silently replace isolation, persistence,
|
|
96
|
+
or built-in policy. The [results](results-v1.md) and
|
|
97
|
+
[extension protocol](extension-protocol-v1.md) contracts define those semantics.
|
|
98
|
+
|
|
99
|
+
## State ownership
|
|
100
|
+
|
|
101
|
+
[`EvaluationContext`](../src/evaluation/context.ts) carries four explicit
|
|
102
|
+
collaborators: prepared inputs, run resources, the result collector, and retained
|
|
103
|
+
preparation-artifact references. It does not combine mutable trial fields with
|
|
104
|
+
the preparation plan.
|
|
105
|
+
|
|
106
|
+
| Data or state | Owner | Boundary |
|
|
107
|
+
| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
108
|
+
| Prepared inputs | [`PreparedEvaluation` in `plan.ts`](../src/evaluation/plan.ts) | Named fixture, check, and invocation groups plus routes, provenance, redacted configuration, and comparison identity. Preparation completes before run allocation. |
|
|
109
|
+
| Run resources and reservations | [`RunResources` and `RunWorkspaceReservations` in `resources.ts`](../src/evaluation/resources.ts) | Run ID, evidence/state locations, process owner, and candidate/semantic reservations. Reservations outlive completed fixture contents; retained workspaces are tracked separately from completed roots. |
|
|
110
|
+
| Mutable trial state | [`TrialRuntimeState` and `EvaluationTrial` in `trial.ts`](../src/evaluation/trial.ts) | Candidate result, observations, checks, assessment, temporary views, advisory result, and persisted milestone belong to one admitted trial. Grading receives narrow inputs and explicit effects rather than this state object. |
|
|
111
|
+
| Collected run results | [`ResultCollector` in `collection.ts`](../src/evaluation/collection.ts) | Private completed summaries, evidence, and diagnostics; trial-number ordering, snapshots for aggregation, and checkpoint requests. [`run-owner.ts`](../src/run-owner.ts) verifies process ownership and writes owner/checkpoint state. |
|
|
112
|
+
| Evidence writes | [`TrialEvidenceWriter`](../src/evidence/trial-writer.ts), [preparation retention](../src/evaluation/resources.ts), [run finalization](../src/evaluation/run.ts) | The trial writer owns filenames, private writes, hashes, artifact references, and typed trial-persistence failures. Resource helpers retain preparation bytes; run finalization writes aggregate evidence. [`storage.ts`](../src/storage.ts) supplies atomic private JSON writes. |
|
|
113
|
+
|
|
114
|
+
Fixture modules own materialization and workspace preparation. Runtime modules
|
|
115
|
+
own declared host runtime policy and private state. Evidence modules validate
|
|
116
|
+
host observations and artifacts, verify retained bytes, and normalize optional
|
|
117
|
+
usage without constructing fixtures. Native host implementations and graders
|
|
118
|
+
remain in their own families.
|
|
119
|
+
|
|
120
|
+
## Where to change behavior
|
|
121
|
+
|
|
122
|
+
Start with the responsibility, then inspect its observable acceptance scenarios.
|
|
123
|
+
These are useful starting suites, not an exhaustive validation list. The
|
|
124
|
+
[contribution checks](../CONTRIBUTING.md#quality-contract-and-checks) still govern
|
|
125
|
+
candidate acceptance.
|
|
126
|
+
|
|
127
|
+
| Responsibility | Implementation and reference | Relevant acceptance tests |
|
|
128
|
+
| ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
129
|
+
| Parse commands/select explicit routes | [CLI entrypoint](../src/cli.ts), [invocation parsing](../src/cli/invocation.ts), [CLI reference](development-cli.md) | [Invalid invocations](../tests/cli/invocation-validation.test.ts) |
|
|
130
|
+
| Load and prepare a CLI evaluation | [Input loading](../src/cli/loading.ts), [preparation](../src/cli/preparation.ts), [host construction](../src/cli/hosts.ts), [extension resolution](../src/cli/extension.ts), [evaluation assembly](../src/cli/evaluation.ts) | [Case/configuration validation](../tests/cli/case-validation.test.ts) |
|
|
131
|
+
| Present CLI results and own cancellation | [Presentation](../src/cli/presentation.ts), [cancellation](../src/cli/cancellation.ts), [command lifecycle](../src/cli/command.ts) | [CLI states and exits](../tests/cli/results.test.ts), [signal retention](../tests/cli/trial-retention.test.ts) |
|
|
132
|
+
| Prepare evaluation inputs and grade trials | [Engine facade](../src/engine.ts), [plan](../src/evaluation/plan.ts), [grading](../src/evaluation/grading.ts), [semantic grading](../src/evaluation/semantic.ts), [evaluation modules](../src/evaluation/), [engine slice](engine-slice.md) | [Preparation refusals](../tests/evaluation/preparation-validation.test.ts), [semantic grading failures](../tests/evaluation/semantic-grading-failures.test.ts), [engine lifecycle](../tests/engine.test.ts) |
|
|
133
|
+
| Schedule trials, retain evidence, and finalize ownership | [Scheduler](../src/evaluation/scheduler.ts), [trial coordinator](../src/evaluation/trial.ts), [reservations](../src/evaluation/resources.ts), [collection](../src/evaluation/collection.ts), [run owner](../src/run-owner.ts) | [Concurrent admission and ordering](../tests/cli-concurrency.test.ts), [trial failures and retention](../tests/evaluation/trial-retention.test.ts), [checkpoint refusals](../tests/run-owner-checkpoints.test.ts) |
|
|
134
|
+
| Prepare, construct, and clean up fixtures | [Fixture modules](../src/fixtures/), [construction](../src/fixtures/create.ts), [preparation artifacts](../src/fixtures/preparation.ts), [content cleanup](../src/fixtures/cleanup.ts), [workspace ownership](../src/fixtures/workspace-state.ts) | [Artifact preparation](../tests/fixtures/artifact-preparation.test.ts), [workspace lifecycle](../tests/fixtures/workspace-lifecycle.test.ts) |
|
|
135
|
+
| Resolve runtime policy, discovery, and private state | [Runtime modules](../src/runtime/), [configuration](../src/runtime/config.ts) | [Runtime environment](../tests/runtime/environment.test.ts), [candidate artifact lifetime](../tests/runtime/candidate-artifacts.test.ts) |
|
|
136
|
+
| Validate host results and retain bounded evidence | [Host results](../src/evidence/host-results.ts), [trial writer](../src/evidence/trial-writer.ts), [retained artifacts](../src/evidence/retained-artifacts.ts), [usage](../src/evidence/usage.ts) | [Host evidence refusals](../tests/evaluation/host-evidence.test.ts), [failed-host evidence retention](../tests/evaluation/trial-retention.test.ts) |
|
|
137
|
+
| Negotiate trusted extension calls | [Client](../src/extension-client.ts), [session](../src/extension-session.ts), [extension grading](../src/evaluation/grading.ts), [protocol](extension-protocol-v1.md) | [Negotiated engine behavior](../tests/extension-engine.test.ts), [extension grading outcomes](../tests/evaluation/extension-grading.test.ts) |
|
|
138
|
+
| Run independent advisory review | [Advisory execution](../src/evaluation/advisory.ts), [blind fixture](../src/fixtures/advisory.ts), [advisory fixture explanation](advisory-fixture.md) | [Advisory failures](../tests/evaluation/advisory-failures.test.ts), [blind change view](../tests/advisory-fixture.test.ts) |
|
|
139
|
+
| Run native hosts and retain bounded observations | [Codex](codex-host.md), [native controls](native-controls.md), [host sources](../src/hosts/), [shared process mechanics](../src/process-lifecycle.ts), [native waiting](../src/hosts/process-lifecycle.ts) | [Native process admission and byte bounds](../tests/hosts/native-process-boundaries.test.ts) |
|
|
140
|
+
| Reduce execution/grading/task states and maintain contract shapes | [Result reduction](../src/results.ts), [shared semantic types](../src/contract-types.ts), [wire types](../src/schema-types.ts), [runtime schema boundary](../src/schema.ts), [results contract](results-v1.md) | [State and exit precedence](../tests/results.test.ts), [evidence schema boundaries](../tests/schema/evidence-boundaries.test.ts) |
|
|
141
|
+
| Bind comparison identities and release provenance | [Preparation identity inputs](../src/evaluation/plan.ts), [identity hashing](../src/identity.ts), [provenance](../src/provenance.ts), [identity contract](identity-v1.md), [releases](releases.md) | [CLI identity](../tests/cli/identity.test.ts), [package release provenance](../tests/release.test.ts) |
|
|
142
|
+
| Summarize retained results | [Report implementation](../src/report.ts), [report contract](report-v1.md) | [Report states and CLI](../tests/report.test.ts) |
|
|
22
143
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
144
|
+
[Rust research](https://github.com/BjRo/sevro/issues/2) does not establish a
|
|
145
|
+
migration; public interfaces remain useful independently. Return to the
|
|
146
|
+
[documentation hub](README.md).
|
|
@@ -38,8 +38,10 @@ The shell-check process runner accepts a bounded evaluator-owned `run` string,
|
|
|
38
38
|
optional `expectedExitCode` and `timeoutMs`, plus `expectExact`, `expectRegex`,
|
|
39
39
|
`notRegex`, and regex `flags` for stdout. Exact matching removes one final
|
|
40
40
|
newline; regex matching always uses multiline mode. The runner executes with
|
|
41
|
-
`sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied
|
|
42
|
-
a credential-free environment, and caller-declared protected roots.
|
|
41
|
+
`sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied
|
|
42
|
+
by default, a credential-free environment, and caller-declared protected roots.
|
|
43
|
+
The runtime [local HTTP option](runtime-v1.md#local-http-test-servers) permits
|
|
44
|
+
socket-using upstream tests while retaining outbound isolation. Without a
|
|
43
45
|
stdout assertion, process output is discarded. With one, the runner captures
|
|
44
46
|
at most 1 MiB in memory and retains only its digest and byte length. Oversized
|
|
45
47
|
or invalid UTF-8 output is a grading error.
|
package/docs/codex-host.md
CHANGED
|
@@ -31,8 +31,8 @@ because its parent directory belongs to a mounted skill; an existing mounted
|
|
|
31
31
|
target that fails or produces incomplete bytes remains partial. A completed direct
|
|
32
32
|
`cat` of one mounted `SKILL.md`, alone or alongside literal documentation
|
|
33
33
|
paths, counts only when the command output contains that file's exact body.
|
|
34
|
-
Batched reads naming
|
|
35
|
-
|
|
34
|
+
Batched reads naming more than 64 paths remain partial, as do failed,
|
|
35
|
+
malformed, or incomplete recognized reads. Direct
|
|
36
36
|
`sed -n` pages count after their verified line ranges cover the whole body. An
|
|
37
37
|
exact `cat` wrapped by `lean-ctx -c` is accepted only when its reported output
|
|
38
38
|
still contains the full body; a compressed summary cannot prove the read.
|
|
@@ -47,18 +47,36 @@ cat .agents/skills/darrow-guide/SKILL.md; cat docs/specs/repository-guide.md; rg
|
|
|
47
47
|
```
|
|
48
48
|
|
|
49
49
|
This sequence uses the shared stream and native read check. Quoted separators
|
|
50
|
-
remain part of their literal operand. The
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
remain part of their literal operand. The ordered file bodies must match the
|
|
51
|
+
retained output, so documentation or search output cannot replace an altered
|
|
52
|
+
or incomplete skill read. Literal `cat` commands may read multiple mounted
|
|
53
|
+
skills and documentation, before or after skills, with or without a terminal
|
|
54
|
+
search. Multiple literal terminal `rg` commands are accepted within the command
|
|
55
|
+
bound. Each verified skill receives its own ordered receipt. Documentation
|
|
56
|
+
must resolve within the workspace or a receipt-bound installed plugin root;
|
|
57
|
+
each file is bounded to 1 MiB. Aliases of mounted skills without a literal
|
|
58
|
+
`SKILL.md` operand remain unsupported.
|
|
54
59
|
The 64-operand bound applies across all reads in the sequence. Shell expansions,
|
|
55
60
|
including unquoted glob and tilde operands, are unsupported in this form.
|
|
56
61
|
|
|
62
|
+
The bounded composed-read grammar also accepts `unset` with literal variable
|
|
63
|
+
names, `pwd` or `pwd -P`, `git rev-parse --show-toplevel` (optionally with
|
|
64
|
+
`-C "$PWD"`), `printenv` with literal variable names, and `rg --files` with
|
|
65
|
+
literal arguments before the reads. Prefix `printenv` requires the reads to
|
|
66
|
+
end the command, so environment output cannot replace missing read bytes.
|
|
67
|
+
Literal `printenv` commands may follow reads without prefix `printenv`.
|
|
68
|
+
The fixed pair `repo=$(git -C "$PWD" rev-parse --show-toplevel)` and
|
|
69
|
+
`cd "$repo" && pwd -P` is supported only with absolute read operands. Other
|
|
70
|
+
assignments, substitutions, directory changes, or control flow remain
|
|
71
|
+
unsupported. Prefix and environment output cannot supply skill frontmatter;
|
|
72
|
+
ambiguous repeated output cannot establish an attributable read.
|
|
73
|
+
|
|
57
74
|
The check also accepts a chain of literal
|
|
58
75
|
`cat` fallbacks with `2>/dev/null` and `||`, ending in `true` followed by a newline
|
|
59
76
|
and one literal `rg` discovery command. A fallback chain must resolve to one
|
|
60
77
|
canonical file across all its read operands; absent alternate targets do not
|
|
61
|
-
invalidate the read.
|
|
78
|
+
invalidate the read. Each fallback `cat` names one literal file. Multiple
|
|
79
|
+
existing targets, including unmounted copies,
|
|
62
80
|
cannot establish an attributable read. The total read operands remain bounded
|
|
63
81
|
at 64. The complete command must succeed and expose the mounted file's entire
|
|
64
82
|
exact body. `cat` accepts literal file operands and the optional `--` delimiter;
|
|
@@ -99,6 +117,10 @@ The host also retains separate `sevro.codex.initial-skill-reads` and
|
|
|
99
117
|
`sevro.codex.follow-up-skill-reads` observations. Each applies the same exact
|
|
100
118
|
mounted-skill read check to one completed turn, so an extension can grade read
|
|
101
119
|
order or a missing repeat read without inferring it from the combined list.
|
|
120
|
+
For `app-server`, the verified same-thread feedback boundary separates the
|
|
121
|
+
initial phase from the follow-up phase, including subsequent native goal turns.
|
|
122
|
+
Absent, duplicated, foreign-thread, or failed boundaries do not yield phase
|
|
123
|
+
receipts.
|
|
102
124
|
|
|
103
125
|
It also checks native parent command records for exact mounted skill bodies,
|
|
104
126
|
including sessions with no spawn request. The bounded diagnostic distinguishes
|
|
@@ -137,12 +159,20 @@ discarded. Extensions interpret task names and decide whether a nested reader
|
|
|
137
159
|
satisfies their policy.
|
|
138
160
|
The record proves an invocation attempt, not its success. A missing, ambiguous,
|
|
139
161
|
unreadable, oversized, or malformed session is unavailable or partial, so its
|
|
140
|
-
absence cannot prove that a control was unused.
|
|
141
|
-
|
|
162
|
+
absence cannot prove that a control was unused. Code Mode submissions now have
|
|
163
|
+
separate bounded `execCalls` inspection receipts, described in the
|
|
164
|
+
[native control contract](native-controls.md). Direct native call ordinals and
|
|
165
|
+
acceptance receipts keep their original meaning; submitted requests do not
|
|
166
|
+
acquire direct native acceptance receipts.
|
|
142
167
|
For an explicitly declared `$plugin:skill`, the host verifies that the installed
|
|
143
168
|
skill exists and that the exact token occurs once in the delivered prompt. A
|
|
144
|
-
completed turn yields `sevro.codex.explicit-invocation` with the owner
|
|
145
|
-
verified supporting reads after it.
|
|
169
|
+
completed turn yields a complete `sevro.codex.explicit-invocation` with the owner
|
|
170
|
+
first and verified supporting reads after it. This receipt proves verified
|
|
171
|
+
owner dispatch independently of file reads; it does not prove that all
|
|
172
|
+
supporting skills were read. Consumers requiring complete supporting-read
|
|
173
|
+
evidence must also require `sevro.codex.skill-reads` (or the applicable phase
|
|
174
|
+
receipt) to be complete. Failed execution cannot yield a complete dispatch
|
|
175
|
+
receipt.
|
|
146
176
|
The `sevro.codex.repository-invocation` capability supports an explicitly
|
|
147
177
|
declared `$skill` from `.agents/skills/<skill>/SKILL.md`. The host checks that
|
|
148
178
|
the repository mount is a bounded regular file with no symlink components and
|
|
@@ -158,7 +188,11 @@ symlinks, and installs it with `codex plugin marketplace add` followed by
|
|
|
158
188
|
`codex plugin add`. It checks each installation receipt against the private
|
|
159
189
|
plugin cache. Candidate commands can read that cache, while the rest of the
|
|
160
190
|
private Codex home, including authentication and configuration, remains denied.
|
|
161
|
-
|
|
191
|
+
With the [Codex state opt-in](runtime-v1.md#candidate-codex-configuration-and-user-skills),
|
|
192
|
+
commands may read generated configuration and write actual non-system user skills;
|
|
193
|
+
graders retain read-only access to that request's original paths. Fixture-local
|
|
194
|
+
configuration is accepted as inert represented input on this route. Otherwise,
|
|
195
|
+
the host refuses fixture-local `.codex` configuration. It also refuses unsupported enforced
|
|
162
196
|
conditions, unreadable or oversized auth files, malformed event streams, and
|
|
163
197
|
incomplete turns. The host bounds output and runtime, kills its process group
|
|
164
198
|
on timeout or cancellation, and removes private state after the turn. It is
|
|
@@ -180,14 +214,18 @@ explicit workspace rule, denies all other filesystem roots except minimal
|
|
|
180
214
|
runtime files and the selected executable's install directories, and
|
|
181
215
|
specifically denies protected roots and the private Codex home. An executable
|
|
182
216
|
inside a protected root is rejected. The profile also disables command
|
|
183
|
-
network access and clears inherited command environment variables.
|
|
217
|
+
network access by default and clears inherited command environment variables.
|
|
218
|
+
The runtime [local HTTP option](runtime-v1.md#local-http-test-servers) enables
|
|
219
|
+
Codex's managed network proxy and local binding for socket-using tests.
|
|
220
|
+
The Codex
|
|
184
221
|
parent can read authentication before it starts sandboxed commands. Tests
|
|
185
222
|
invoke the installed `codex sandbox` CLI to prove fixture access, denied
|
|
186
223
|
source/auth reads, and executable access. The process adapter verifies these
|
|
187
224
|
properties in one sandbox invocation before each fresh host request, with a
|
|
188
225
|
60-second bounded startup deadline. The combined probe still requires readable
|
|
189
226
|
private command storage, denied project listing, denied authentication and
|
|
190
|
-
configuration reads
|
|
227
|
+
configuration reads (unless configuration access was explicitly enabled), and a
|
|
228
|
+
successful native executable `--version`. Continuation
|
|
191
229
|
turns reuse the same isolated state. The longer deadline accommodates observed
|
|
192
230
|
native policy startup; it is not counted as a performance saving.
|
|
193
231
|
|
|
@@ -207,3 +245,8 @@ yielded chunk count, and separate `completedCall` and `literalCommandCall`
|
|
|
207
245
|
flags. These facts distinguish recovery sources without retaining commands,
|
|
208
246
|
submitted code, or skill bodies. The diagnostic's completeness and truncation
|
|
209
247
|
fields still determine whether its read evidence can support grading.
|
|
248
|
+
|
|
249
|
+
[Issue #70 representative retests](evidence/skill-reads-70/results.json) retain
|
|
250
|
+
separate task, activation, and phase-read outcomes from the private candidate,
|
|
251
|
+
plus its source fingerprints and test-first regression results. Private native
|
|
252
|
+
transcripts remain outside the repository.
|
package/docs/development-cli.md
CHANGED
|
@@ -323,7 +323,13 @@ isolated shell checks receive `.git/fixture-bin` on `PATH`. Host adapters
|
|
|
323
323
|
receive `fixtureBinDir` and must add it to their native tool environment; the
|
|
324
324
|
bundled Codex adapter also preserves it across login shells. A negotiated
|
|
325
325
|
fixture setup command may then prepare the clone before the host runs.
|
|
326
|
-
Repository
|
|
326
|
+
Repository fixtures preserve committed `.gitmodules` files and submodule gitlink
|
|
327
|
+
revisions. Trial clones leave submodules uninitialized and do not fetch their
|
|
328
|
+
contents, even when the source has initialized submodules. A fixture that needs
|
|
329
|
+
those contents must initialize them during its negotiated setup. Modified or
|
|
330
|
+
untracked files and revision changes in initialized source submodules are
|
|
331
|
+
refused, including when Git's submodule ignore settings hide them. Preparation
|
|
332
|
+
artifacts targeting `.git/` are refused.
|
|
327
333
|
|
|
328
334
|
A direct case or extension can declare a generated Git fixture:
|
|
329
335
|
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Code Mode control evidence for issue 71
|
|
2
|
+
|
|
3
|
+
On 2026-10-14, the implementation for [issue 71](https://github.com/BjRo/sevro/issues/71)
|
|
4
|
+
replayed the four retained complete Codex native transcript bundles named in
|
|
5
|
+
that ticket through the changed Sevro observers. This is artifact rechecking,
|
|
6
|
+
not a fresh authenticated model trial. The original task checks and activation
|
|
7
|
+
outcomes were preserved separately from the new control-absence assessment.
|
|
8
|
+
Private native bundles, commands and output remain outside Git.
|
|
9
|
+
|
|
10
|
+
| Case | Inspected exec submissions | Submitted tool calls | Rechecked control absence | Original task checks | Original activation |
|
|
11
|
+
| -------------------------------------- | -------------------------- | -------------------- | ------------------------- | ----------------------- | ------------------- |
|
|
12
|
+
| guide-mutation | 2 | 4 | passed | 7 passed, 1 unavailable | passed |
|
|
13
|
+
| guide-orchestration | 3 | 6 | passed | 8 passed, 1 unavailable | passed |
|
|
14
|
+
| code-review-empty-diff | 6 | 8 | passed | 6 passed, 1 unavailable | passed |
|
|
15
|
+
| code-review-reviewer-route-unavailable | 11 | 12 | passed | 6 passed, 1 unavailable | unavailable |
|
|
16
|
+
|
|
17
|
+
The recheck found no submitted agent launch or prohibited goal mutation in any
|
|
18
|
+
case. All submission receipts had complete inspection and matching native
|
|
19
|
+
ordinals. No accepted-agent count was substituted for invocation evidence.
|
|
20
|
+
The older Darrow consumer still requires zero submitted exec calls; it needs
|
|
21
|
+
to adopt the [receipt validation contract](../native-controls.md) to grade
|
|
22
|
+
these new observations. Its original unavailable outcomes were not rewritten.
|
|
23
|
+
|
|
24
|
+
The reviewer-route case's first read used a literal Git environment reset and
|
|
25
|
+
repository-root query before its final mounted skill read. Its retained output
|
|
26
|
+
ended with the exact 20,627-character skill body still present in the Darrow
|
|
27
|
+
validation checkout. Rehydrating that independently matching mount and rebasing
|
|
28
|
+
its expired private path let the changed skill-read observer establish a
|
|
29
|
+
complete `code-review` receipt. This rechecked skill-file receipt is separate
|
|
30
|
+
from the original unavailable activation outcome; no historical trial was
|
|
31
|
+
changed to passed.
|
|
32
|
+
|
|
33
|
+
Durable Bun regressions exercise the host's public observations for completed
|
|
34
|
+
and rejected submissions, opaque code, and the repository-prefixed skill read.
|
|
35
|
+
Additional real session/file regressions cover malformed/truncated completion,
|
|
36
|
+
missing and duplicate receipts, forged textual labels, ambiguous code, namespace
|
|
37
|
+
and size limits, and contradictory receipt counts/ordinals. Existing integration
|
|
38
|
+
tests remain in place.
|
|
39
|
+
|
|
40
|
+
## Observed test-first slices
|
|
41
|
+
|
|
42
|
+
Each command below was run with its new regression present and its corresponding
|
|
43
|
+
product behavior unchanged, then rerun after the product fix. Commands invoke
|
|
44
|
+
the existing Bun harness; the shell routed their output through lean-ctx.
|
|
45
|
+
|
|
46
|
+
1. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode exposes completed literal"`: missing submitted-call inventory.
|
|
47
|
+
Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode exposes completed literal"`: 1 passed.
|
|
48
|
+
2. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode portable"`: rejected request receipt missing and opaque submission reported complete.
|
|
49
|
+
Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode portable"`: 2 passed.
|
|
50
|
+
3. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode inspects the retained"`: both retained parallel-read shapes reported partial.
|
|
51
|
+
Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode inspects the retained"`: 2 passed.
|
|
52
|
+
4. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode refuses malformed"`: malformed and truncated completion envelopes reported complete.
|
|
53
|
+
Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode refuses malformed"`: 2 passed.
|
|
54
|
+
5. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode accepts native array"`: the native array envelope reported partial.
|
|
55
|
+
Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode accepts native array"`: 1 passed.
|
|
56
|
+
6. Red — `bun test tests/codex-host.test.ts -t "reviewer-route repository-prefixed"`: complete mounted body did not establish a skill read.
|
|
57
|
+
Green — `bun test tests/codex-host.test.ts -t "reviewer-route repository-prefixed"`: 1 passed.
|
|
58
|
+
7. Red — `bun test tests/native-controls.test.ts -t "contradictory Code Mode"`: mismatched submission ordinal reported complete.
|
|
59
|
+
Green — `bun test tests/native-controls.test.ts -t "contradictory Code Mode"`: 1 passed.
|
|
60
|
+
8. Red — `bun test tests/codex-native-calls.test.ts -t "attributable control evidence"`: foreign namespace and oversized tool labels reported complete.
|
|
61
|
+
Green — `bun test tests/codex-native-calls.test.ts -t "attributable control evidence"`: 2 passed.
|
|
62
|
+
|
|
63
|
+
These focused slices do not substitute for the full GitHub CI acceptance gate
|
|
64
|
+
on the PR's exact published commit.
|