@bjoernrochel/sevro 0.1.0-rc.11 → 0.1.0-rc.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/docs/README.md +13 -13
  2. package/docs/architecture.md +138 -17
  3. package/docs/builtin-graders-v1.md +4 -2
  4. package/docs/codex-host.md +57 -14
  5. package/docs/development-cli.md +7 -1
  6. package/docs/evidence/code-mode-controls-71.md +64 -0
  7. package/docs/evidence/codex-state-73/README.md +204 -0
  8. package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
  9. package/docs/evidence/codex-state-73/results.json +661 -0
  10. package/docs/evidence/local-http-74.md +104 -0
  11. package/docs/evidence/skill-reads-70/results.json +823 -0
  12. package/docs/evidence.md +3 -0
  13. package/docs/installing.md +2 -2
  14. package/docs/native-controls.md +36 -4
  15. package/docs/runtime-v1.md +121 -0
  16. package/docs/typescript-quality.md +1 -1
  17. package/package.json +1 -1
  18. package/schemas/runtime-v1.schema.json +10 -0
  19. package/src/advisory.ts +2 -2
  20. package/src/candidate-artifacts.ts +30 -7
  21. package/src/candidate-codex-state.ts +185 -0
  22. package/src/case-validation.ts +2 -2
  23. package/src/cli/cancellation.ts +20 -0
  24. package/src/cli/command.ts +30 -0
  25. package/src/cli/evaluation.ts +47 -0
  26. package/src/cli/extension.ts +79 -0
  27. package/src/cli/hosts.ts +41 -0
  28. package/src/{cli-invocation.ts → cli/invocation.ts} +2 -0
  29. package/src/cli/loading.ts +144 -0
  30. package/src/cli/preparation.ts +162 -0
  31. package/src/cli/presentation.ts +88 -0
  32. package/src/cli.ts +1 -589
  33. package/src/contract-types.ts +94 -0
  34. package/src/engine.ts +8 -5
  35. package/src/evaluation/advisory.ts +150 -0
  36. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +12 -12
  37. package/src/evaluation/collection.ts +88 -0
  38. package/src/evaluation/context.ts +10 -0
  39. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  40. package/src/evaluation/grading.ts +258 -0
  41. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +4 -4
  42. package/src/{evaluation-plan.ts → evaluation/plan.ts} +205 -72
  43. package/src/evaluation/resources.ts +238 -0
  44. package/src/evaluation/run.ts +199 -0
  45. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +63 -78
  46. package/src/evaluation/trial.ts +818 -0
  47. package/src/{evaluation-types.ts → evaluation/types.ts} +65 -29
  48. package/src/evidence/host-results.ts +92 -0
  49. package/src/evidence/retained-artifacts.ts +18 -0
  50. package/src/evidence/trial-writer.ts +203 -0
  51. package/src/evidence/usage.ts +17 -0
  52. package/src/extension-client.ts +34 -96
  53. package/src/extension-session.ts +24 -48
  54. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +1 -1
  55. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  56. package/src/{evaluation-fixture.ts → fixtures/create.ts} +7 -131
  57. package/src/{generated-fixture.ts → fixtures/generated.ts} +2 -2
  58. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  59. package/src/{preparation.ts → fixtures/preparation.ts} +2 -2
  60. package/src/{repository-fixture.ts → fixtures/repository.ts} +16 -7
  61. package/src/{fixture-setup.ts → fixtures/setup.ts} +21 -49
  62. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  63. package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
  64. package/src/generated/runtime.cjs +1 -1
  65. package/src/graders/git-head.ts +27 -72
  66. package/src/graders/shell.ts +36 -64
  67. package/src/host-diagnostics.ts +2 -2
  68. package/src/hosts/claude-process.ts +21 -15
  69. package/src/hosts/claude-settings.ts +4 -4
  70. package/src/hosts/claude-state.ts +5 -5
  71. package/src/hosts/claude.ts +4 -4
  72. package/src/hosts/codex-exec-controls.ts +328 -0
  73. package/src/hosts/codex-marketplace.ts +1 -1
  74. package/src/hosts/codex-native-calls.ts +8 -0
  75. package/src/hosts/codex-process.ts +24 -55
  76. package/src/hosts/codex-profile.ts +32 -6
  77. package/src/hosts/codex-session-files.ts +1 -1
  78. package/src/hosts/codex-skill-reads.ts +302 -90
  79. package/src/hosts/codex-state.ts +84 -8
  80. package/src/hosts/codex.ts +60 -12
  81. package/src/hosts/isolation-common.ts +3 -2
  82. package/src/hosts/isolation-roots.ts +2 -2
  83. package/src/hosts/linux-sandbox.ts +1 -1
  84. package/src/hosts/mac-sandbox.ts +15 -2
  85. package/src/hosts/native-controls.ts +34 -2
  86. package/src/hosts/process-lifecycle.ts +8 -41
  87. package/src/hosts/protected-worktrees.ts +1 -1
  88. package/src/native-transcript-view.ts +1 -1
  89. package/src/native-transcripts.ts +4 -5
  90. package/src/process-lifecycle.ts +150 -0
  91. package/src/report.ts +7 -32
  92. package/src/resolved-case.ts +1 -1
  93. package/src/results.ts +14 -18
  94. package/src/{runtime-config.ts → runtime/config.ts} +30 -9
  95. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  96. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +2 -2
  97. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  98. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  99. package/src/{runtime-hooks.ts → runtime/hooks.ts} +10 -10
  100. package/src/{runtime-paths.ts → runtime/paths.ts} +2 -2
  101. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  102. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  103. package/src/{runtime-state.ts → runtime/state.ts} +5 -5
  104. package/src/schema-types.ts +71 -161
  105. package/src/evaluation-context.ts +0 -259
  106. package/src/evaluation-run.ts +0 -200
  107. package/src/evaluation-trial.ts +0 -1051
  108. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
  109. /package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +0 -0
package/docs/README.md CHANGED
@@ -4,19 +4,19 @@
4
4
 
5
5
  Start with the task you want to complete. These pages work without an agent.
6
6
 
7
- | I want to… | Read |
8
- | --------------------------------------------- | -------------------------------------------------------- |
9
- | Get a result without a model account | [Your first evaluation](getting-started.md) |
10
- | Install, update, remove, or use a checkout | [Installation](installing.md) |
11
- | Run evaluations and select a host | [Running evaluations](running-evaluations.md) |
12
- | Create cases and reusable policy | [Cases and extensions](creating-cases.md) |
13
- | Interpret verdicts, evidence, and comparisons | [Reading results](reading-results.md) |
14
- | Run Codex or Claude with isolation | [Native hosts](native-hosts.md) |
15
- | Reuse host tools or enable goals and hooks | [Runtime configuration](runtime-v1.md) |
16
- | Resolve a symptom | [Troubleshooting](troubleshooting.md) |
17
- | Understand runner responsibilities | [Architecture](architecture.md) |
18
- | Change Sevro or its documentation | [Contributing](../CONTRIBUTING.md) |
19
- | Understand licensing and branding | [Licensing](licensing.md) and [assets](assets/README.md) |
7
+ | I want to… | Read |
8
+ | ---------------------------------------------- | -------------------------------------------------------- |
9
+ | Get a result without a model account | [Your first evaluation](getting-started.md) |
10
+ | Install, update, remove, or use a checkout | [Installation](installing.md) |
11
+ | Run evaluations and select a host | [Running evaluations](running-evaluations.md) |
12
+ | Create cases and reusable policy | [Cases and extensions](creating-cases.md) |
13
+ | Interpret verdicts, evidence, and comparisons | [Reading results](reading-results.md) |
14
+ | Run Codex or Claude with isolation | [Native hosts](native-hosts.md) |
15
+ | Reuse host tools or enable goals and hooks | [Runtime configuration](runtime-v1.md) |
16
+ | Resolve a symptom | [Troubleshooting](troubleshooting.md) |
17
+ | Trace runner lifecycle, ownership, and changes | [Architecture](architecture.md) |
18
+ | Change Sevro or its documentation | [Contributing](../CONTRIBUTING.md) |
19
+ | Understand licensing and branding | [Licensing](licensing.md) and [assets](assets/README.md) |
20
20
 
21
21
  ## Ask the guide
22
22
 
@@ -4,22 +4,143 @@ Sevro owns generic execution/evidence contracts. Projects own cases and policy.
4
4
  This lets Darrow and other integrations use the same runner without importing
5
5
  their benchmark policy into it.
6
6
 
7
- | Responsibility | Sources |
8
- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ |
9
- | Parse commands/select explicit routes | [CLI](../src/cli.ts), [CLI reference](development-cli.md) |
10
- | Prepare, execute, grade, retain, clean up | [Engine](../src/engine.ts), [engine slice](engine-slice.md) |
11
- | Negotiate trusted extension calls | [Client](../src/extension-client.ts), [session](../src/extension-session.ts), [protocol](extension-protocol-v1.md) |
12
- | Run native hosts and retain bounded observations | [Codex](codex-host.md), [native controls](native-controls.md), [host sources](../src/hosts/) |
13
- | Reduce execution/grading/task states | [Result source](../src/results.ts), [contract](results-v1.md) |
14
- | Bind comparison identities and release provenance | [Identity](identity-v1.md), [releases](releases.md) |
15
- | Summarize retained results | [Report](report-v1.md) |
16
-
17
- Completed evidence is retained before cleanup. Missing observations remain
18
- unavailable; unknown measurements do not become zero. Extensions cannot
19
- silently replace isolation/persistence or built-in policy. The
7
+ This page explains this checkout's implementation and points to acceptance
8
+ tests for changes. [Current contracts](contracts.md) define public behavior;
9
+ [historical validation](evidence.md) records bounded past observations. The
10
+ [engine slice](engine-slice.md) retains its earlier scope and detailed lifecycle
11
+ explanation. Implementation paths and test scenarios here are navigation, not
12
+ evidence that a particular candidate passed validation.
13
+
14
+ [`engine.ts`](../src/engine.ts) remains the programmatic facade, and
15
+ [`cli.ts`](../src/cli.ts) remains the installed command entrypoint. The CLI
16
+ loads and assembles evaluation options; evaluation modules coordinate the run
17
+ and its trials.
18
+
19
+ ## Evaluation lifecycle
20
+
21
+ [`runEvaluation`](../src/evaluation/run.ts) prepares inputs before allocating
22
+ run resources, retains preparation artifacts, and reserves candidate and
23
+ semantic workspace roots as needed before calling the trial scheduler. The map
24
+ follows the path of a retained trial; individual stages can be skipped or fail.
25
+
26
+ ```mermaid
27
+ flowchart TD
28
+ P["Validate and prepare evaluation inputs"] --> R["Allocate run storage and owner; reserve workspace roots"]
29
+ R --> W["Admit trial and materialize its reserved fixture"]
30
+ subgraph Trial["Each admitted trial"]
31
+ W --> C["Initialize grading bases; execute candidate"]
32
+ C --> E["Retain candidate artifacts; observe output; prepare grading views"]
33
+ E --> G["Output, shell, Git, semantic, and extension grading"]
34
+ G --> A["Assess execution, grading, and task separately"]
35
+ A --> V["Optional independent advisory review"]
36
+ V --> T["Retain raw candidate response; atomically write trial evidence"]
37
+ T --> K["Collect completed trial and checkpoint its reference"]
38
+ K --> D["Release temporary views; clear or retain fixture contents"]
39
+ end
40
+ D --> J["All admitted trials drain"]
41
+ J --> F["Write final run evidence; finalize owner checkpoint"]
42
+ F --> X["Finalize retained workspaces; release other reservations"]
43
+ ```
44
+
45
+ [`scheduleTrials`](../src/evaluation/scheduler.ts) bounds active work by the
46
+ prepared job limit. It admits a trial before
47
+ [`prepareTrialWorkspace`](../src/evaluation/resources.ts) materializes that
48
+ trial's fixture. Each [`EvaluationTrial`](../src/evaluation/trial.ts) owns its
49
+ mutable state and writer. Its `run()` method keeps candidate execution,
50
+ artifact retention, grading, assessment, advisory review, persistence, and
51
+ cleanup in one visible order. Grading helpers return checks, observations,
52
+ and failures; the coordinator incorporates those results and stops new
53
+ admission when an execution or required grading error is detected, before
54
+ advisory work, persistence, or cleanup. The scheduler also stops admission on
55
+ cancellation and waits for active workers. A failed task check alone does not
56
+ close admission. Already admitted peers drain and retain their evidence, including
57
+ when they finish out of order.
58
+
59
+ [`runAdvisoryReview`](../src/evaluation/advisory.ts) handles its own host call,
60
+ assessment, retained response, and auxiliary workspace cleanup. It depends on
61
+ candidate completion, independently of the task assessment. The
20
62
  [advisory fixture](advisory-fixture.md) gives review a separate Git view with
21
- evaluator inputs withheld.
63
+ evaluator inputs withheld. Its recommendation or host failure does not change
64
+ task grading; a failure to retain its evidence remains a persistence error.
65
+
66
+ Successful [semantic response parsing](../src/evaluation/semantic.ts) clears
67
+ auxiliary fixture contents; successful advisory response parsing removes its
68
+ auxiliary fixture. Both retain response evidence first, including valid failed
69
+ check verdicts or advisory recommendations.
70
+
71
+ The [trial writer](../src/evidence/trial-writer.ts) writes the trial artifact
72
+ before the coordinator marks it persisted and hands it to the
73
+ [result collector](../src/evaluation/collection.ts). The collector registers
74
+ the completed record, orders records by trial number, and checkpoints retained
75
+ artifact paths. After active work drains, the run coordinator validates and
76
+ writes `run.json` before finalizing the owner as complete or interrupted.
77
+ Persistence failures cannot produce a successful result; the run's failure
78
+ path attempts a diagnostic checkpoint while preserving the original failure.
79
+
80
+ Trial cleanup releases candidate artifact runtimes and transcript views even
81
+ when the trial fails. Persisted successful fixture contents are cleared while
82
+ their empty reserved roots remain until active work drains. Failed, cancelled,
83
+ grading-error, unavailable-evidence, and failed-task fixtures are retained;
84
+ [workspace ownership](../src/fixtures/workspace-state.ts) tracks their inactive
85
+ retention and startup expiry after 24 hours. Saved evidence stays outside that
86
+ cleanup. See the [results retention contract](results-v1.md#identity-and-provenance)
87
+ for the public boundary, including ownership that cannot be verified.
88
+
89
+ Dry preparation still validates and materializes fixtures, including declared
90
+ fixture setup, and retains trial records. It skips candidate execution and
91
+ grading, recording `not_run` / `not_requested` / `not_assessed`, as exercised by
92
+ the [engine acceptance suite](../tests/engine.test.ts). Missing observations
93
+ remain unavailable; missing required evidence cannot become a pass. Optional
94
+ unknown measurements remain unknown rather than becoming zero. Extensions
95
+ cannot silently replace isolation, persistence,
96
+ or built-in policy. The [results](results-v1.md) and
97
+ [extension protocol](extension-protocol-v1.md) contracts define those semantics.
98
+
99
+ ## State ownership
100
+
101
+ [`EvaluationContext`](../src/evaluation/context.ts) carries four explicit
102
+ collaborators: prepared inputs, run resources, the result collector, and retained
103
+ preparation-artifact references. It does not combine mutable trial fields with
104
+ the preparation plan.
105
+
106
+ | Data or state | Owner | Boundary |
107
+ | ------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
108
+ | Prepared inputs | [`PreparedEvaluation` in `plan.ts`](../src/evaluation/plan.ts) | Named fixture, check, and invocation groups plus routes, provenance, redacted configuration, and comparison identity. Preparation completes before run allocation. |
109
+ | Run resources and reservations | [`RunResources` and `RunWorkspaceReservations` in `resources.ts`](../src/evaluation/resources.ts) | Run ID, evidence/state locations, process owner, and candidate/semantic reservations. Reservations outlive completed fixture contents; retained workspaces are tracked separately from completed roots. |
110
+ | Mutable trial state | [`TrialRuntimeState` and `EvaluationTrial` in `trial.ts`](../src/evaluation/trial.ts) | Candidate result, observations, checks, assessment, temporary views, advisory result, and persisted milestone belong to one admitted trial. Grading receives narrow inputs and explicit effects rather than this state object. |
111
+ | Collected run results | [`ResultCollector` in `collection.ts`](../src/evaluation/collection.ts) | Private completed summaries, evidence, and diagnostics; trial-number ordering, snapshots for aggregation, and checkpoint requests. [`run-owner.ts`](../src/run-owner.ts) verifies process ownership and writes owner/checkpoint state. |
112
+ | Evidence writes | [`TrialEvidenceWriter`](../src/evidence/trial-writer.ts), [preparation retention](../src/evaluation/resources.ts), [run finalization](../src/evaluation/run.ts) | The trial writer owns filenames, private writes, hashes, artifact references, and typed trial-persistence failures. Resource helpers retain preparation bytes; run finalization writes aggregate evidence. [`storage.ts`](../src/storage.ts) supplies atomic private JSON writes. |
113
+
114
+ Fixture modules own materialization and workspace preparation. Runtime modules
115
+ own declared host runtime policy and private state. Evidence modules validate
116
+ host observations and artifacts, verify retained bytes, and normalize optional
117
+ usage without constructing fixtures. Native host implementations and graders
118
+ remain in their own families.
119
+
120
+ ## Where to change behavior
121
+
122
+ Start with the responsibility, then inspect its observable acceptance scenarios.
123
+ These are useful starting suites, not an exhaustive validation list. The
124
+ [contribution checks](../CONTRIBUTING.md#quality-contract-and-checks) still govern
125
+ candidate acceptance.
126
+
127
+ | Responsibility | Implementation and reference | Relevant acceptance tests |
128
+ | ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
129
+ | Parse commands/select explicit routes | [CLI entrypoint](../src/cli.ts), [invocation parsing](../src/cli/invocation.ts), [CLI reference](development-cli.md) | [Invalid invocations](../tests/cli/invocation-validation.test.ts) |
130
+ | Load and prepare a CLI evaluation | [Input loading](../src/cli/loading.ts), [preparation](../src/cli/preparation.ts), [host construction](../src/cli/hosts.ts), [extension resolution](../src/cli/extension.ts), [evaluation assembly](../src/cli/evaluation.ts) | [Case/configuration validation](../tests/cli/case-validation.test.ts) |
131
+ | Present CLI results and own cancellation | [Presentation](../src/cli/presentation.ts), [cancellation](../src/cli/cancellation.ts), [command lifecycle](../src/cli/command.ts) | [CLI states and exits](../tests/cli/results.test.ts), [signal retention](../tests/cli/trial-retention.test.ts) |
132
+ | Prepare evaluation inputs and grade trials | [Engine facade](../src/engine.ts), [plan](../src/evaluation/plan.ts), [grading](../src/evaluation/grading.ts), [semantic grading](../src/evaluation/semantic.ts), [evaluation modules](../src/evaluation/), [engine slice](engine-slice.md) | [Preparation refusals](../tests/evaluation/preparation-validation.test.ts), [semantic grading failures](../tests/evaluation/semantic-grading-failures.test.ts), [engine lifecycle](../tests/engine.test.ts) |
133
+ | Schedule trials, retain evidence, and finalize ownership | [Scheduler](../src/evaluation/scheduler.ts), [trial coordinator](../src/evaluation/trial.ts), [reservations](../src/evaluation/resources.ts), [collection](../src/evaluation/collection.ts), [run owner](../src/run-owner.ts) | [Concurrent admission and ordering](../tests/cli-concurrency.test.ts), [trial failures and retention](../tests/evaluation/trial-retention.test.ts), [checkpoint refusals](../tests/run-owner-checkpoints.test.ts) |
134
+ | Prepare, construct, and clean up fixtures | [Fixture modules](../src/fixtures/), [construction](../src/fixtures/create.ts), [preparation artifacts](../src/fixtures/preparation.ts), [content cleanup](../src/fixtures/cleanup.ts), [workspace ownership](../src/fixtures/workspace-state.ts) | [Artifact preparation](../tests/fixtures/artifact-preparation.test.ts), [workspace lifecycle](../tests/fixtures/workspace-lifecycle.test.ts) |
135
+ | Resolve runtime policy, discovery, and private state | [Runtime modules](../src/runtime/), [configuration](../src/runtime/config.ts) | [Runtime environment](../tests/runtime/environment.test.ts), [candidate artifact lifetime](../tests/runtime/candidate-artifacts.test.ts) |
136
+ | Validate host results and retain bounded evidence | [Host results](../src/evidence/host-results.ts), [trial writer](../src/evidence/trial-writer.ts), [retained artifacts](../src/evidence/retained-artifacts.ts), [usage](../src/evidence/usage.ts) | [Host evidence refusals](../tests/evaluation/host-evidence.test.ts), [failed-host evidence retention](../tests/evaluation/trial-retention.test.ts) |
137
+ | Negotiate trusted extension calls | [Client](../src/extension-client.ts), [session](../src/extension-session.ts), [extension grading](../src/evaluation/grading.ts), [protocol](extension-protocol-v1.md) | [Negotiated engine behavior](../tests/extension-engine.test.ts), [extension grading outcomes](../tests/evaluation/extension-grading.test.ts) |
138
+ | Run independent advisory review | [Advisory execution](../src/evaluation/advisory.ts), [blind fixture](../src/fixtures/advisory.ts), [advisory fixture explanation](advisory-fixture.md) | [Advisory failures](../tests/evaluation/advisory-failures.test.ts), [blind change view](../tests/advisory-fixture.test.ts) |
139
+ | Run native hosts and retain bounded observations | [Codex](codex-host.md), [native controls](native-controls.md), [host sources](../src/hosts/), [shared process mechanics](../src/process-lifecycle.ts), [native waiting](../src/hosts/process-lifecycle.ts) | [Native process admission and byte bounds](../tests/hosts/native-process-boundaries.test.ts) |
140
+ | Reduce execution/grading/task states and maintain contract shapes | [Result reduction](../src/results.ts), [shared semantic types](../src/contract-types.ts), [wire types](../src/schema-types.ts), [runtime schema boundary](../src/schema.ts), [results contract](results-v1.md) | [State and exit precedence](../tests/results.test.ts), [evidence schema boundaries](../tests/schema/evidence-boundaries.test.ts) |
141
+ | Bind comparison identities and release provenance | [Preparation identity inputs](../src/evaluation/plan.ts), [identity hashing](../src/identity.ts), [provenance](../src/provenance.ts), [identity contract](identity-v1.md), [releases](releases.md) | [CLI identity](../tests/cli/identity.test.ts), [package release provenance](../tests/release.test.ts) |
142
+ | Summarize retained results | [Report implementation](../src/report.ts), [report contract](report-v1.md) | [Report states and CLI](../tests/report.test.ts) |
22
143
 
23
- These paths describe this checkout. [Historical validation](evidence.md) records
24
- bounded past observations. [Rust research](https://github.com/BjRo/sevro/issues/2)
25
- does not establish a migration; public interfaces remain useful independently.
144
+ [Rust research](https://github.com/BjRo/sevro/issues/2) does not establish a
145
+ migration; public interfaces remain useful independently. Return to the
146
+ [documentation hub](README.md).
@@ -38,8 +38,10 @@ The shell-check process runner accepts a bounded evaluator-owned `run` string,
38
38
  optional `expectedExitCode` and `timeoutMs`, plus `expectExact`, `expectRegex`,
39
39
  `notRegex`, and regex `flags` for stdout. Exact matching removes one final
40
40
  newline; regex matching always uses multiline mode. The runner executes with
41
- `sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied,
42
- a credential-free environment, and caller-declared protected roots. Without a
41
+ `sh -e`. On macOS and Linux it runs through the outer sandbox with network access denied
42
+ by default, a credential-free environment, and caller-declared protected roots.
43
+ The runtime [local HTTP option](runtime-v1.md#local-http-test-servers) permits
44
+ socket-using upstream tests while retaining outbound isolation. Without a
43
45
  stdout assertion, process output is discarded. With one, the runner captures
44
46
  at most 1 MiB in memory and retains only its digest and byte length. Oversized
45
47
  or invalid UTF-8 output is a grading error.
@@ -31,8 +31,8 @@ because its parent directory belongs to a mounted skill; an existing mounted
31
31
  target that fails or produces incomplete bytes remains partial. A completed direct
32
32
  `cat` of one mounted `SKILL.md`, alone or alongside literal documentation
33
33
  paths, counts only when the command output contains that file's exact body.
34
- Batched reads naming multiple `SKILL.md` files or more than 64 paths remain
35
- partial, as do failed, malformed, or incomplete recognized reads. Direct
34
+ Batched reads naming more than 64 paths remain partial, as do failed,
35
+ malformed, or incomplete recognized reads. Direct
36
36
  `sed -n` pages count after their verified line ranges cover the whole body. An
37
37
  exact `cat` wrapped by `lean-ctx -c` is accepted only when its reported output
38
38
  still contains the full body; a compressed summary cannot prove the read.
@@ -47,18 +47,36 @@ cat .agents/skills/darrow-guide/SKILL.md; cat docs/specs/repository-guide.md; rg
47
47
  ```
48
48
 
49
49
  This sequence uses the shared stream and native read check. Quoted separators
50
- remain part of their literal operand. The initial skill body must match the
51
- beginning of the retained output, so later documentation or search output cannot
52
- replace an altered or incomplete initial read. Subsequent reads must not name
53
- another `SKILL.md` or resolve to a mounted or receipt-bound skill through an alias.
50
+ remain part of their literal operand. The ordered file bodies must match the
51
+ retained output, so documentation or search output cannot replace an altered
52
+ or incomplete skill read. Literal `cat` commands may read multiple mounted
53
+ skills and documentation, before or after skills, with or without a terminal
54
+ search. Multiple literal terminal `rg` commands are accepted within the command
55
+ bound. Each verified skill receives its own ordered receipt. Documentation
56
+ must resolve within the workspace or a receipt-bound installed plugin root;
57
+ each file is bounded to 1 MiB. Aliases of mounted skills without a literal
58
+ `SKILL.md` operand remain unsupported.
54
59
  The 64-operand bound applies across all reads in the sequence. Shell expansions,
55
60
  including unquoted glob and tilde operands, are unsupported in this form.
56
61
 
62
+ The bounded composed-read grammar also accepts `unset` with literal variable
63
+ names, `pwd` or `pwd -P`, `git rev-parse --show-toplevel` (optionally with
64
+ `-C "$PWD"`), `printenv` with literal variable names, and `rg --files` with
65
+ literal arguments before the reads. Prefix `printenv` requires the reads to
66
+ end the command, so environment output cannot replace missing read bytes.
67
+ Literal `printenv` commands may follow reads without prefix `printenv`.
68
+ The fixed pair `repo=$(git -C "$PWD" rev-parse --show-toplevel)` and
69
+ `cd "$repo" && pwd -P` is supported only with absolute read operands. Other
70
+ assignments, substitutions, directory changes, or control flow remain
71
+ unsupported. Prefix and environment output cannot supply skill frontmatter;
72
+ ambiguous repeated output cannot establish an attributable read.
73
+
57
74
  The check also accepts a chain of literal
58
75
  `cat` fallbacks with `2>/dev/null` and `||`, ending in `true` followed by a newline
59
76
  and one literal `rg` discovery command. A fallback chain must resolve to one
60
77
  canonical file across all its read operands; absent alternate targets do not
61
- invalidate the read. Multiple existing targets, including unmounted copies,
78
+ invalidate the read. Each fallback `cat` names one literal file. Multiple
79
+ existing targets, including unmounted copies,
62
80
  cannot establish an attributable read. The total read operands remain bounded
63
81
  at 64. The complete command must succeed and expose the mounted file's entire
64
82
  exact body. `cat` accepts literal file operands and the optional `--` delimiter;
@@ -99,6 +117,10 @@ The host also retains separate `sevro.codex.initial-skill-reads` and
99
117
  `sevro.codex.follow-up-skill-reads` observations. Each applies the same exact
100
118
  mounted-skill read check to one completed turn, so an extension can grade read
101
119
  order or a missing repeat read without inferring it from the combined list.
120
+ For `app-server`, the verified same-thread feedback boundary separates the
121
+ initial phase from the follow-up phase, including subsequent native goal turns.
122
+ Absent, duplicated, foreign-thread, or failed boundaries do not yield phase
123
+ receipts.
102
124
 
103
125
  It also checks native parent command records for exact mounted skill bodies,
104
126
  including sessions with no spawn request. The bounded diagnostic distinguishes
@@ -137,12 +159,20 @@ discarded. Extensions interpret task names and decide whether a nested reader
137
159
  satisfies their policy.
138
160
  The record proves an invocation attempt, not its success. A missing, ambiguous,
139
161
  unreadable, oversized, or malformed session is unavailable or partial, so its
140
- absence cannot prove that a control was unused. Calls made through submitted
141
- code are not classified by this first observation.
162
+ absence cannot prove that a control was unused. Code Mode submissions now have
163
+ separate bounded `execCalls` inspection receipts, described in the
164
+ [native control contract](native-controls.md). Direct native call ordinals and
165
+ acceptance receipts keep their original meaning; submitted requests do not
166
+ acquire direct native acceptance receipts.
142
167
  For an explicitly declared `$plugin:skill`, the host verifies that the installed
143
168
  skill exists and that the exact token occurs once in the delivered prompt. A
144
- completed turn yields `sevro.codex.explicit-invocation` with the owner first and
145
- verified supporting reads after it. Incomplete reads make that receipt partial.
169
+ completed turn yields a complete `sevro.codex.explicit-invocation` with the owner
170
+ first and verified supporting reads after it. This receipt proves verified
171
+ owner dispatch independently of file reads; it does not prove that all
172
+ supporting skills were read. Consumers requiring complete supporting-read
173
+ evidence must also require `sevro.codex.skill-reads` (or the applicable phase
174
+ receipt) to be complete. Failed execution cannot yield a complete dispatch
175
+ receipt.
146
176
  The `sevro.codex.repository-invocation` capability supports an explicitly
147
177
  declared `$skill` from `.agents/skills/<skill>/SKILL.md`. The host checks that
148
178
  the repository mount is a bounded regular file with no symlink components and
@@ -158,7 +188,11 @@ symlinks, and installs it with `codex plugin marketplace add` followed by
158
188
  `codex plugin add`. It checks each installation receipt against the private
159
189
  plugin cache. Candidate commands can read that cache, while the rest of the
160
190
  private Codex home, including authentication and configuration, remains denied.
161
- It refuses fixture-local `.codex` configuration, unsupported enforced
191
+ With the [Codex state opt-in](runtime-v1.md#candidate-codex-configuration-and-user-skills),
192
+ commands may read generated configuration and write actual non-system user skills;
193
+ graders retain read-only access to that request's original paths. Fixture-local
194
+ configuration is accepted as inert represented input on this route. Otherwise,
195
+ the host refuses fixture-local `.codex` configuration. It also refuses unsupported enforced
162
196
  conditions, unreadable or oversized auth files, malformed event streams, and
163
197
  incomplete turns. The host bounds output and runtime, kills its process group
164
198
  on timeout or cancellation, and removes private state after the turn. It is
@@ -180,14 +214,18 @@ explicit workspace rule, denies all other filesystem roots except minimal
180
214
  runtime files and the selected executable's install directories, and
181
215
  specifically denies protected roots and the private Codex home. An executable
182
216
  inside a protected root is rejected. The profile also disables command
183
- network access and clears inherited command environment variables. The Codex
217
+ network access by default and clears inherited command environment variables.
218
+ The runtime [local HTTP option](runtime-v1.md#local-http-test-servers) enables
219
+ Codex's managed network proxy and local binding for socket-using tests.
220
+ The Codex
184
221
  parent can read authentication before it starts sandboxed commands. Tests
185
222
  invoke the installed `codex sandbox` CLI to prove fixture access, denied
186
223
  source/auth reads, and executable access. The process adapter verifies these
187
224
  properties in one sandbox invocation before each fresh host request, with a
188
225
  60-second bounded startup deadline. The combined probe still requires readable
189
226
  private command storage, denied project listing, denied authentication and
190
- configuration reads, and a successful native executable `--version`. Continuation
227
+ configuration reads (unless configuration access was explicitly enabled), and a
228
+ successful native executable `--version`. Continuation
191
229
  turns reuse the same isolated state. The longer deadline accommodates observed
192
230
  native policy startup; it is not counted as a performance saving.
193
231
 
@@ -207,3 +245,8 @@ yielded chunk count, and separate `completedCall` and `literalCommandCall`
207
245
  flags. These facts distinguish recovery sources without retaining commands,
208
246
  submitted code, or skill bodies. The diagnostic's completeness and truncation
209
247
  fields still determine whether its read evidence can support grading.
248
+
249
+ [Issue #70 representative retests](evidence/skill-reads-70/results.json) retain
250
+ separate task, activation, and phase-read outcomes from the private candidate,
251
+ plus its source fingerprints and test-first regression results. Private native
252
+ transcripts remain outside the repository.
@@ -323,7 +323,13 @@ isolated shell checks receive `.git/fixture-bin` on `PATH`. Host adapters
323
323
  receive `fixtureBinDir` and must add it to their native tool environment; the
324
324
  bundled Codex adapter also preserves it across login shells. A negotiated
325
325
  fixture setup command may then prepare the clone before the host runs.
326
- Repository submodules and preparation artifacts targeting `.git/` are refused.
326
+ Repository fixtures preserve committed `.gitmodules` files and submodule gitlink
327
+ revisions. Trial clones leave submodules uninitialized and do not fetch their
328
+ contents, even when the source has initialized submodules. A fixture that needs
329
+ those contents must initialize them during its negotiated setup. Modified or
330
+ untracked files and revision changes in initialized source submodules are
331
+ refused, including when Git's submodule ignore settings hide them. Preparation
332
+ artifacts targeting `.git/` are refused.
327
333
 
328
334
  A direct case or extension can declare a generated Git fixture:
329
335
 
@@ -0,0 +1,64 @@
1
+ # Code Mode control evidence for issue 71
2
+
3
+ On 2026-10-14, the implementation for [issue 71](https://github.com/BjRo/sevro/issues/71)
4
+ replayed the four retained complete Codex native transcript bundles named in
5
+ that ticket through the changed Sevro observers. This is artifact rechecking,
6
+ not a fresh authenticated model trial. The original task checks and activation
7
+ outcomes were preserved separately from the new control-absence assessment.
8
+ Private native bundles, commands and output remain outside Git.
9
+
10
+ | Case | Inspected exec submissions | Submitted tool calls | Rechecked control absence | Original task checks | Original activation |
11
+ | -------------------------------------- | -------------------------- | -------------------- | ------------------------- | ----------------------- | ------------------- |
12
+ | guide-mutation | 2 | 4 | passed | 7 passed, 1 unavailable | passed |
13
+ | guide-orchestration | 3 | 6 | passed | 8 passed, 1 unavailable | passed |
14
+ | code-review-empty-diff | 6 | 8 | passed | 6 passed, 1 unavailable | passed |
15
+ | code-review-reviewer-route-unavailable | 11 | 12 | passed | 6 passed, 1 unavailable | unavailable |
16
+
17
+ The recheck found no submitted agent launch or prohibited goal mutation in any
18
+ case. All submission receipts had complete inspection and matching native
19
+ ordinals. No accepted-agent count was substituted for invocation evidence.
20
+ The older Darrow consumer still requires zero submitted exec calls; it needs
21
+ to adopt the [receipt validation contract](../native-controls.md) to grade
22
+ these new observations. Its original unavailable outcomes were not rewritten.
23
+
24
+ The reviewer-route case's first read used a literal Git environment reset and
25
+ repository-root query before its final mounted skill read. Its retained output
26
+ ended with the exact 20,627-character skill body still present in the Darrow
27
+ validation checkout. Rehydrating that independently matching mount and rebasing
28
+ its expired private path let the changed skill-read observer establish a
29
+ complete `code-review` receipt. This rechecked skill-file receipt is separate
30
+ from the original unavailable activation outcome; no historical trial was
31
+ changed to passed.
32
+
33
+ Durable Bun regressions exercise the host's public observations for completed
34
+ and rejected submissions, opaque code, and the repository-prefixed skill read.
35
+ Additional real session/file regressions cover malformed/truncated completion,
36
+ missing and duplicate receipts, forged textual labels, ambiguous code, namespace
37
+ and size limits, and contradictory receipt counts/ordinals. Existing integration
38
+ tests remain in place.
39
+
40
+ ## Observed test-first slices
41
+
42
+ Each command below was run with its new regression present and its corresponding
43
+ product behavior unchanged, then rerun after the product fix. Commands invoke
44
+ the existing Bun harness; the shell routed their output through lean-ctx.
45
+
46
+ 1. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode exposes completed literal"`: missing submitted-call inventory.
47
+ Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode exposes completed literal"`: 1 passed.
48
+ 2. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode portable"`: rejected request receipt missing and opaque submission reported complete.
49
+ Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode portable"`: 2 passed.
50
+ 3. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode inspects the retained"`: both retained parallel-read shapes reported partial.
51
+ Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode inspects the retained"`: 2 passed.
52
+ 4. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode refuses malformed"`: malformed and truncated completion envelopes reported complete.
53
+ Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode refuses malformed"`: 2 passed.
54
+ 5. Red — `bun test tests/codex-native-calls.test.ts -t "Code Mode accepts native array"`: the native array envelope reported partial.
55
+ Green — `bun test tests/codex-native-calls.test.ts -t "Code Mode accepts native array"`: 1 passed.
56
+ 6. Red — `bun test tests/codex-host.test.ts -t "reviewer-route repository-prefixed"`: complete mounted body did not establish a skill read.
57
+ Green — `bun test tests/codex-host.test.ts -t "reviewer-route repository-prefixed"`: 1 passed.
58
+ 7. Red — `bun test tests/native-controls.test.ts -t "contradictory Code Mode"`: mismatched submission ordinal reported complete.
59
+ Green — `bun test tests/native-controls.test.ts -t "contradictory Code Mode"`: 1 passed.
60
+ 8. Red — `bun test tests/codex-native-calls.test.ts -t "attributable control evidence"`: foreign namespace and oversized tool labels reported complete.
61
+ Green — `bun test tests/codex-native-calls.test.ts -t "attributable control evidence"`: 2 passed.
62
+
63
+ These focused slices do not substitute for the full GitHub CI acceptance gate
64
+ on the PR's exact published commit.