@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +22 -6
- package/README.md +5 -1
- package/docs/README.md +19 -15
- package/docs/architecture.md +138 -17
- package/docs/assets/README.md +5 -4
- package/docs/builtin-graders-v1.md +9 -2
- package/docs/codex-host.md +155 -57
- package/docs/contracts.md +2 -1
- package/docs/creating-cases.md +9 -2
- package/docs/development-cli.md +187 -19
- package/docs/documentation-quality.md +12 -2
- package/docs/evaluation-workflow.md +110 -0
- package/docs/extension-protocol-v1.md +26 -3
- package/docs/getting-started.md +23 -2
- package/docs/guide-evaluation.md +33 -1
- package/docs/identity-v1.md +25 -2
- package/docs/installing.md +11 -11
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +196 -7
- package/docs/reading-results.md +102 -0
- package/docs/recipes.md +489 -0
- package/docs/releases.md +31 -12
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +122 -4
- package/docs/run-settings-v1.md +147 -0
- package/docs/runtime-v1.md +255 -22
- package/docs/troubleshooting.md +12 -12
- package/docs/typescript-quality.md +89 -26
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +37 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +188 -0
- package/examples/recipes/run-options.ts +131 -0
- package/examples/recipes/run.ts +63 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +31 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +11 -5
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +43 -7
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +42 -11
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +73 -50
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +87 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +60 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +198 -0
- package/src/cli/hosts.ts +55 -0
- package/src/cli/init-settings.ts +30 -0
- package/src/cli/init.ts +111 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +154 -0
- package/src/cli/presentation.ts +93 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +128 -0
- package/src/cli/run-options.ts +403 -0
- package/src/cli/run-overrides.ts +69 -0
- package/src/cli/settings.ts +154 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +13 -591
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +9 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +235 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
- package/src/evaluation/trial.ts +800 -0
- package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
- package/src/evidence/host-results.ts +85 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/extension-client.ts +34 -96
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +183 -0
- package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
- package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
- package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
- package/src/generated/cli-result.d.cts +2 -1
- package/src/generated/extension.d.cts +2 -1
- package/src/generated/report.cjs +1 -1
- package/src/generated/report.d.cts +2 -1
- package/src/generated/run-evidence.cjs +1 -1
- package/src/generated/run-evidence.d.cts +2 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/generated/runtime.d.cts +2 -1
- package/src/git.ts +93 -0
- package/src/graders/git-head.ts +29 -84
- package/src/graders/shell.ts +50 -68
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +331 -0
- package/src/hosts/claude-continuation.ts +57 -40
- package/src/hosts/claude-credential.ts +160 -63
- package/src/hosts/claude-events.ts +34 -18
- package/src/hosts/claude-keychain.ts +78 -21
- package/src/hosts/claude-login-recovery.ts +85 -0
- package/src/hosts/claude-login-refresh.ts +178 -0
- package/src/hosts/claude-login-store.ts +162 -0
- package/src/hosts/claude-login.ts +107 -0
- package/src/hosts/claude-native-goal.ts +15 -12
- package/src/hosts/claude-process.ts +10 -50
- package/src/hosts/claude-refresh-lock.ts +81 -0
- package/src/hosts/claude-repository-invocation.ts +9 -34
- package/src/hosts/claude-settings.ts +46 -9
- package/src/hosts/claude-skill-activations.ts +169 -0
- package/src/hosts/claude-state.ts +28 -15
- package/src/hosts/claude-tool-calls.ts +13 -10
- package/src/hosts/claude.ts +121 -81
- package/src/hosts/codex-app-server.ts +78 -52
- package/src/hosts/codex-configuration.ts +5 -10
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-marketplace.ts +12 -10
- package/src/hosts/codex-native-calls.ts +15 -122
- package/src/hosts/codex-process.ts +10 -183
- package/src/hosts/codex-profile.ts +36 -21
- package/src/hosts/codex-session-files.ts +1 -9
- package/src/hosts/codex-state.ts +134 -18
- package/src/hosts/codex.ts +210 -183
- package/src/hosts/isolation-common.ts +8 -11
- package/src/hosts/isolation-roots.ts +2 -2
- package/src/hosts/linux-sandbox.ts +15 -2
- package/src/hosts/mac-sandbox.ts +87 -18
- package/src/hosts/native-controls.ts +72 -22
- package/src/hosts/native-request.ts +55 -0
- package/src/hosts/process-lifecycle.ts +216 -48
- package/src/hosts/protected-worktrees.ts +1 -1
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +577 -0
- package/src/instrumentation.ts +7 -8
- package/src/native-transcript-state.ts +22 -3
- package/src/native-transcript-view.ts +52 -13
- package/src/native-transcripts.ts +26 -11
- package/src/paths.ts +22 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +27 -7
- package/src/prune.ts +300 -0
- package/src/report.ts +63 -36
- package/src/resolved-case.ts +1 -1
- package/src/results-lock.ts +42 -0
- package/src/results.ts +14 -18
- package/src/{runtime-config.ts → runtime/config.ts} +160 -46
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +32 -22
- package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/runtime/shared-cache.ts +130 -0
- package/src/{runtime-state.ts → runtime/state.ts} +28 -11
- package/src/schema-types.ts +73 -161
- package/src/shell.ts +4 -0
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +22 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +0 -1
- package/docs/assets/sevro-wordmark.svg +0 -1
- package/docs/cache-ownership-validation.md +0 -49
- package/docs/claude-continuation-validation.md +0 -47
- package/docs/concurrency-validation.md +0 -86
- package/docs/configuration-root-validation.md +0 -70
- package/docs/engine-slice.md +0 -114
- package/docs/evidence/runtime-startup-30/README.md +0 -185
- package/docs/evidence/runtime-startup-30/after.json +0 -319
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
- package/docs/evidence/runtime-startup-30/before.json +0 -403
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
- package/docs/evidence.md +0 -17
- package/docs/fixture-cleanup-validation.md +0 -73
- package/docs/guide-verification.json +0 -239
- package/src/evaluation-context.ts +0 -259
- package/src/evaluation-fixture.ts +0 -302
- package/src/evaluation-run.ts +0 -200
- package/src/evaluation-trial.ts +0 -1016
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -510
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -783
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
package/docs/runtime-v1.md
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
# Runtime configuration v1
|
|
2
2
|
|
|
3
|
-
Configure access to existing host tools, private
|
|
4
|
-
and selected plugin hooks in `<project-root>/sevro.json`.
|
|
5
|
-
available in `0.1.0-rc.3` and later.
|
|
3
|
+
Configure access to existing host tools, private state, reusable caches, native goals,
|
|
4
|
+
and selected plugin hooks in `<project-root>/sevro.json`.
|
|
6
5
|
|
|
7
6
|
`--runtime-config-file /absolute/path/runtime.json` selects a replacement file.
|
|
8
7
|
Sevro reads exactly one file before execution, including dry runs. It does not
|
|
@@ -14,6 +13,17 @@ non-symlink files of at most 64 KiB.
|
|
|
14
13
|
|
|
15
14
|
The [JSON schema](../schemas/runtime-v1.schema.json) has format
|
|
16
15
|
`sevro.runtime.v1`. All properties except `format` are optional.
|
|
16
|
+
Start with the smallest file:
|
|
17
|
+
|
|
18
|
+
```json
|
|
19
|
+
{ "format": "sevro.runtime.v1" }
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Then add `environment.inherit: ["PATH"]` for installed tools. Follow the
|
|
23
|
+
[runnable runtime recipes](recipes.md#add-host-tool-access-one-capability-at-a-time)
|
|
24
|
+
to add Node, Python/UV, mise, goals and selected plugin hooks separately.
|
|
25
|
+
The following composed example is for a workflow that already needs all these
|
|
26
|
+
capabilities:
|
|
17
27
|
|
|
18
28
|
```json
|
|
19
29
|
{
|
|
@@ -26,7 +36,13 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
|
|
|
26
36
|
}
|
|
27
37
|
},
|
|
28
38
|
"runtime": {
|
|
29
|
-
"
|
|
39
|
+
"sharedCacheDirectories": [
|
|
40
|
+
{
|
|
41
|
+
"source": "../public-uv-cache",
|
|
42
|
+
"target": "uv-cache",
|
|
43
|
+
"roles": ["candidate", "hooks"]
|
|
44
|
+
}
|
|
45
|
+
]
|
|
30
46
|
},
|
|
31
47
|
"hooks": {
|
|
32
48
|
"nativeGoal": true,
|
|
@@ -35,7 +51,7 @@ The [JSON schema](../schemas/runtime-v1.schema.json) has format
|
|
|
35
51
|
}
|
|
36
52
|
```
|
|
37
53
|
|
|
38
|
-
|
|
54
|
+
Prepare the example's `public-uv-cache` directory outside the project, or
|
|
39
55
|
omit `runtime` when no prepared cache is needed. Selected plugin names must be
|
|
40
56
|
present in the case's prepared mounts; remove names that the case does not mount.
|
|
41
57
|
`DARROW_CACHE_DIR` is an ordinary explicit variable in this example, not a
|
|
@@ -144,8 +160,80 @@ minimal system read tree such as `/usr` or `/etc`, Sevro retains its explicit de
|
|
|
144
160
|
the native preflight may refuse that overlapping setup rather than expose home
|
|
145
161
|
contents.
|
|
146
162
|
|
|
163
|
+
## Local HTTP test servers
|
|
164
|
+
|
|
165
|
+
Sevro supports local HTTP servers in isolated Codex candidate
|
|
166
|
+
commands and `sevro.shell` graders through an explicit operator selection:
|
|
167
|
+
|
|
168
|
+
```json
|
|
169
|
+
{
|
|
170
|
+
"format": "sevro.runtime.v1",
|
|
171
|
+
"network": { "localHttp": true }
|
|
172
|
+
}
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Add this property to the selected `sevro.json`, or select a replacement with
|
|
176
|
+
`--runtime-config-file`. Declare the existing Node/Python installations through
|
|
177
|
+
the environment and filesystem options above. The selection is snapshotted and
|
|
178
|
+
enters evaluation identity. Absent or `false` preserves existing permissions.
|
|
179
|
+
Hooks and semantic/advisory hosts keep their existing network permissions; this
|
|
180
|
+
option applies to Codex candidates and shell grading.
|
|
181
|
+
|
|
182
|
+
On macOS, shell isolation permits IP socket binding, loopback inbound traffic,
|
|
183
|
+
and outbound connections to loopback addresses. Wildcard listeners such as
|
|
184
|
+
Node's `server.listen(0)` work with loopback clients. Other outbound IP traffic
|
|
185
|
+
remains denied. On Linux, shell checks retain their private network namespace
|
|
186
|
+
with usable loopback and no external route.
|
|
187
|
+
|
|
188
|
+
Codex uses its managed command network proxy with `allow_local_binding = true`,
|
|
189
|
+
upstream proxy chaining disabled, and only `localhost`, `127.0.0.1`, and `::1`
|
|
190
|
+
allowlisted. Its native sandbox permits local binding and loopback traffic while
|
|
191
|
+
refusing direct outbound TCP. The native proxy may permit DNS queries; public
|
|
192
|
+
HTTP destinations are not allowlisted. The candidate can reach other services
|
|
193
|
+
on the available loopback interface, so choose this option for evaluations
|
|
194
|
+
that need that access. It does not provide a separate per-trial loopback service
|
|
195
|
+
namespace on macOS.
|
|
196
|
+
|
|
197
|
+
The supported host boundary is the native Codex sandbox and its experimental
|
|
198
|
+
`network_proxy` feature, documented in the [OpenAI permission reference](https://learn.chatgpt.com/docs/permissions).
|
|
199
|
+
Setting `allow_local_binding` while leaving command networking disabled still
|
|
200
|
+
denies `listen()` on macOS. Integration tests execute the installed native
|
|
201
|
+
Codex sandbox, a wildcard HTTP listener and loopback client, and an outbound TCP
|
|
202
|
+
denial probe. Host versions and actual upstream evaluation outcomes belong in
|
|
203
|
+
[validation evidence](https://github.com/BjRo/sevro/blob/main/docs/evidence.md), rather than being inferred from a profile's
|
|
204
|
+
text. An unsupported host must not be interpreted as a passing behavior test.
|
|
205
|
+
|
|
206
|
+
## macOS ancestor metadata
|
|
207
|
+
|
|
208
|
+
Sevro's macOS outer sandbox for isolated shell checks and authorized command
|
|
209
|
+
hooks permits metadata reads on the exact ancestor paths of existing validated
|
|
210
|
+
grants. This includes declared read-only support trees, private writable
|
|
211
|
+
runtimes, and bounded transcript, candidate artifact, and managed workspace
|
|
212
|
+
access. The permissions are derived automatically from those grants; no
|
|
213
|
+
additional configuration is required. They let libc `realpath()` canonicalize
|
|
214
|
+
paths inside a permitted subtree.
|
|
215
|
+
|
|
216
|
+
Ancestor directory metadata, such as existence, type, ownership, permissions,
|
|
217
|
+
and timestamps, is visible. Directory listings, ancestor file contents, and
|
|
218
|
+
ungranted sibling metadata and contents remain denied under protected roots.
|
|
219
|
+
Writes remain limited by the existing writable grants; read-only grants stay
|
|
220
|
+
read-only. This metadata exception does not grant access to an ancestor subtree
|
|
221
|
+
or change network isolation.
|
|
222
|
+
|
|
147
223
|
## Private writable state
|
|
148
224
|
|
|
225
|
+
Codex candidate commands can write the isolated fixture's `.git` directory,
|
|
226
|
+
including its index, refs, objects, and task-owned admission locks. Sevro grants
|
|
227
|
+
that exact path explicitly because Codex's workspace permission otherwise keeps
|
|
228
|
+
Git metadata read-only. Real commits and publication to a fixture-local remote
|
|
229
|
+
need no additional runtime option. This grant does not authorize writes to
|
|
230
|
+
source repositories, grading inputs, host authentication or configuration, or
|
|
231
|
+
peer trials. Command network access remains disabled unless the operator selects
|
|
232
|
+
the [local HTTP option](#local-http-test-servers).
|
|
233
|
+
An existing fixture `.git` must be a physical directory. Symlinks and Git
|
|
234
|
+
directory pointer files are refused before launch because their metadata may
|
|
235
|
+
belong to a protected source repository rather than the isolated trial.
|
|
236
|
+
|
|
149
237
|
`runtime.seedDirectories` copies prepared public dependencies into a private
|
|
150
238
|
slot for each execution role. `source` uses the filesystem path rules above;
|
|
151
239
|
`target` is a single lowercase slot name, up to 64 characters, with letters,
|
|
@@ -153,10 +241,12 @@ digits, and hyphens. `home` and `tmp` are reserved. Targets must be unique.
|
|
|
153
241
|
An optional nonempty `roles` array selects any of `candidate`, `checks`, `hooks`,
|
|
154
242
|
`semantic`, and `advisory`. For example, `"roles": ["candidate", "checks", "hooks"]`
|
|
155
243
|
keeps dependency slots out of semantic and advisory runtimes. Without `roles`,
|
|
156
|
-
|
|
244
|
+
candidate, hook, artifact semantic, and advisory roles receive private copies.
|
|
245
|
+
Shell checks require explicit `"checks"` selection; a non-UV grader receives no
|
|
246
|
+
implicit UV/Python cache. Existing configurations whose shell checks need seeded
|
|
247
|
+
dependencies must add `"checks"` to `roles`. Response-only
|
|
157
248
|
semantic grading omits implicit seeds; include `"semantic"` explicitly in `roles`
|
|
158
|
-
when that route needs dependencies.
|
|
159
|
-
shell checks keep their declared tools and default seed selection.
|
|
249
|
+
when that route needs dependencies. Declared host tools remain available to shell checks.
|
|
160
250
|
|
|
161
251
|
Each source tree is bounded to 16,384 entries and 512 MiB. Escaping symlinks,
|
|
162
252
|
directory cycles, dangling links, and special files are refused. File content
|
|
@@ -172,19 +262,64 @@ retaining large policies can retain up to the configured per-seed byte bounds.
|
|
|
172
262
|
Source caches are never a writable runtime. Use prepared public caches rather
|
|
173
263
|
than credential-bearing homes or evaluator directories.
|
|
174
264
|
|
|
265
|
+
### Reuse a shared dependency cache
|
|
266
|
+
|
|
267
|
+
`runtime.sharedCacheDirectories` opts into persistent writable caches without
|
|
268
|
+
copying or hashing their contents for each trial. Each entry requires `source`,
|
|
269
|
+
`target`, and a nonempty `roles` array drawn from the same five roles above.
|
|
270
|
+
The source must already be an owned, readable, writable directory. Relative paths
|
|
271
|
+
resolve beside the runtime file. Sevro resolves declaration symlinks to their
|
|
272
|
+
canonical source and revalidates that directory before execution.
|
|
273
|
+
Targets use the seed slot rules and must be unique across seeds and shared caches.
|
|
274
|
+
Sources cannot overlap each other, private seed sources, declared read-only
|
|
275
|
+
inputs, evaluated project/source trees, results, credentials, or native private state.
|
|
276
|
+
|
|
277
|
+
Select only roles that actually use UV, for example `"roles": ["candidate", "hooks"]`.
|
|
278
|
+
If a grader also uses UV, add `"checks"` to that same entry; both roles use the
|
|
279
|
+
same cache. Each selected private slot aliases the original directory, with a
|
|
280
|
+
narrow sandbox write grant. Other roles receive no slot or source grant.
|
|
281
|
+
Shared content survives successful, failed, and cancelled trials and later runs.
|
|
282
|
+
Sevro removes only disposable role state, never the declared shared source.
|
|
283
|
+
|
|
284
|
+
Prepare UV packages once outside evaluation, point `UV_CACHE_DIR` at
|
|
285
|
+
`{{sevro.runtime}}/uv-cache`, and reuse the existing Python installation through
|
|
286
|
+
declared read-only tool roots. Set `UV_PYTHON_DOWNLOADS=never` to require that
|
|
287
|
+
installation. Sevro keeps `UV_OFFLINE=1`; a missing dependency fails rather than
|
|
288
|
+
downloading during a trial. Project virtual environments and temporary files
|
|
289
|
+
remain disposable; UV's install/link mode can still allocate files in a virtual
|
|
290
|
+
environment. The shared option removes the full cache copy per trial.
|
|
291
|
+
|
|
292
|
+
Shared caches are mutable across trials, roles, and runs. Use them only for public
|
|
293
|
+
dependencies and accept that a participant can change them for later participants.
|
|
294
|
+
Use isolated seeds when dependency independence is required. Configuration identity
|
|
295
|
+
binds the source, slot, and roles, but excludes mutable cache contents; equal
|
|
296
|
+
identities therefore do not prove identical shared-cache bytes. Sevro does not
|
|
297
|
+
age-delete shared caches. Manage their size between evaluations with the cache
|
|
298
|
+
tool, such as `uv cache prune --cache-dir /absolute/public-uv-cache`.
|
|
299
|
+
|
|
175
300
|
`{{sevro.runtime}}` in an environment value resolves to that role's writable
|
|
176
|
-
root.
|
|
177
|
-
|
|
301
|
+
root. Codex commands receive a `TMPPREFIX` inside their private `TMPDIR`,
|
|
302
|
+
so zsh here-documents use their existing private temporary storage while shared
|
|
303
|
+
host temporary directories remain denied. `TMPPREFIX` is runner-owned and cannot
|
|
304
|
+
be inherited or set through runtime configuration.
|
|
305
|
+
Sevro supplies private `HOME` and temporary storage and sets `UV_OFFLINE=1`
|
|
306
|
+
and `PYTHONDONTWRITEBYTECODE=1`. Fixture setup uses the candidate commands'
|
|
307
|
+
private `HOME`, so `$HOME` paths initialize the same state the candidate will
|
|
308
|
+
use. Its temporary directory stays disposable.
|
|
309
|
+
Candidate, hook, checker, semantic, and advisory
|
|
178
310
|
state are separate. With a runtime policy, native candidate, semantic, and
|
|
179
311
|
advisory roots live in a unique runner-owned host directory outside the Git
|
|
180
|
-
fixture. The root contains `home`, `tmp`, and
|
|
312
|
+
fixture. The root contains `home`, `tmp`, private seed slots and selected shared-cache links; plugins can
|
|
181
313
|
use paths such as `~/.darrow/reviews` without creating repository-local review
|
|
182
314
|
state. Command runtime directories are separate from host authentication,
|
|
183
|
-
configuration, and live transcript sources. Checker and hook
|
|
184
|
-
outside candidate-controlled workspace state. Private roots survive native
|
|
315
|
+
configuration, and live transcript sources. Checker and hook private state also lives
|
|
316
|
+
outside candidate-controlled workspace state. Shared sources remain at their declared locations. Private roots survive native
|
|
185
317
|
continuation turns and children, then are removed with their owning trial or
|
|
186
|
-
host state after owned hooks terminate.
|
|
187
|
-
|
|
318
|
+
host state after owned hooks terminate. Checker home, temporary storage, and copied
|
|
319
|
+
seeds are removed after grading even when the fixture is retained after failure
|
|
320
|
+
or cancellation. Results and evidence are preserved. Opted-in candidate artifact runtimes
|
|
321
|
+
remain alive through grading as described below. Runs with neither fixture setup
|
|
322
|
+
nor a selected runtime policy
|
|
188
323
|
retain their previous writable-state locations.
|
|
189
324
|
|
|
190
325
|
Claude refuses managed workspace or runtime namespaces overlapping its
|
|
@@ -200,8 +335,7 @@ data and remain trusted code responsible for their own execution boundary.
|
|
|
200
335
|
## Candidate artifact access
|
|
201
336
|
|
|
202
337
|
Use `runtime.artifactDirectories` to let graders read selected artifacts in the
|
|
203
|
-
candidate's private home after execution.
|
|
204
|
-
source checkout; published `0.1.0-rc.7` does not include it.
|
|
338
|
+
candidate's private home after execution.
|
|
205
339
|
|
|
206
340
|
```json
|
|
207
341
|
{
|
|
@@ -212,7 +346,8 @@ source checkout; published `0.1.0-rc.7` does not include it.
|
|
|
212
346
|
}
|
|
213
347
|
```
|
|
214
348
|
|
|
215
|
-
The example permits only the
|
|
349
|
+
The example permits only the `~/.darrow/reviews` subtree, which may be initialized
|
|
350
|
+
by trusted fixture setup or created by the candidate.
|
|
216
351
|
Its original absolute path reaches isolated shell checks and native
|
|
217
352
|
semantic/advisory commands as `SEVRO_CANDIDATE_ARTIFACT_REVIEWS`. Graders keep
|
|
218
353
|
their own private `HOME`; use the supplied variable to read candidate evidence.
|
|
@@ -228,12 +363,31 @@ to 16,384 entries, including its root. Missing directories, symlinked ancestors,
|
|
|
228
363
|
descendant symlinks, hardlinks and special files make grading an error; missing
|
|
229
364
|
evidence cannot produce a pass.
|
|
230
365
|
|
|
366
|
+
Initialize an empty directory or seed prior Review state in the extension's
|
|
367
|
+
trusted `fixtureSetup` command before the candidate starts. For example:
|
|
368
|
+
|
|
369
|
+
```sh
|
|
370
|
+
mkdir -p "$HOME/.darrow/reviews"
|
|
371
|
+
printf '%s\n' '{"verdict":"passed"}' > "$HOME/.darrow/reviews/prior.json"
|
|
372
|
+
```
|
|
373
|
+
|
|
374
|
+
Setup and candidate commands share this private trial home, including when no
|
|
375
|
+
artifact grants or runtime file are selected. Graders keep their own homes and
|
|
376
|
+
receive only declared read-only artifact grants. To test missing reports or
|
|
377
|
+
absence of new artifacts, initialize the directory without writing a report;
|
|
378
|
+
an empty safe directory reaches ordinary task checks. Missing required reports
|
|
379
|
+
can then fail those checks, while negative cases can assert the directory is
|
|
380
|
+
empty. Sevro does not automatically create declared artifact directories: if
|
|
381
|
+
setup and candidate both leave one absent, or replace it with an unsafe path,
|
|
382
|
+
grading still errors. Setup state is temporary and is removed with its trial,
|
|
383
|
+
including setup failure, cancellation, and dry runs.
|
|
384
|
+
|
|
231
385
|
The grant permits reads and denies writes, creation, deletion and renaming.
|
|
232
386
|
Credentials, other home contents and peer role/trial state stay protected.
|
|
233
387
|
Ordinary `filesystem.readOnlyRoots` cannot select this private-state exception.
|
|
234
388
|
Artifact grants and native transcript access are independent opt-ins.
|
|
235
389
|
|
|
236
|
-
The trial allocates the retained runtime before
|
|
390
|
+
The trial allocates the retained runtime before fixture setup and
|
|
237
391
|
removes it after grading, advisory review and evidence persistence, including
|
|
238
392
|
failure and cancellation paths. Review state remains outside the Git fixture.
|
|
239
393
|
These artifacts are temporary; declaring access does not copy their bytes into
|
|
@@ -248,6 +402,72 @@ hosts validate it before use. Grading adapters must preserve the read-only
|
|
|
248
402
|
boundary. The directory declarations enter runtime configuration identity;
|
|
249
403
|
per-trial absolute paths do not.
|
|
250
404
|
|
|
405
|
+
## Candidate Codex configuration and user skills
|
|
406
|
+
|
|
407
|
+
Set top-level `"codexState": true` to expose the bundled Codex candidate's actual
|
|
408
|
+
isolated configuration and user-skill state. This opt-in is independent of
|
|
409
|
+
`nativeTranscripts` and ordinary
|
|
410
|
+
candidate-home artifact declarations:
|
|
411
|
+
|
|
412
|
+
```json
|
|
413
|
+
{
|
|
414
|
+
"format": "sevro.runtime.v1",
|
|
415
|
+
"codexState": true
|
|
416
|
+
}
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
Candidate commands receive the real isolated `CODEX_HOME`. They may read its
|
|
420
|
+
`config.toml` and write non-system skills below `CODEX_HOME/skills`. The native
|
|
421
|
+
process's separate user-skill directory is available as
|
|
422
|
+
`SEVRO_CODEX_AGENT_SKILLS`; it names that process's `HOME/.agents/skills`.
|
|
423
|
+
Bundled `CODEX_HOME/skills/.system` skills remain read-only. Authentication,
|
|
424
|
+
other private siblings, evaluator inputs and peer requests remain denied.
|
|
425
|
+
Configuration remains read-only, and the preflight still checks credential denial.
|
|
426
|
+
|
|
427
|
+
A physical fixture-local `.codex` directory is accepted only with this opt-in.
|
|
428
|
+
Sevro marks the isolated project untrusted in its generated Codex configuration,
|
|
429
|
+
so project-local configuration, hooks and rules remain represented inputs rather
|
|
430
|
+
than effective host settings. The fixture is preserved. Diagnose the absolute
|
|
431
|
+
`CODEX_HOME/config.toml`, not the checkout's `.codex/config.toml`. The runner's
|
|
432
|
+
selected concurrency limit is written into the private config; an omitted limit
|
|
433
|
+
still means the native host default, not a fabricated configured capacity.
|
|
434
|
+
CLI model/effort and vetted-hook overrides keep their existing precedence; the
|
|
435
|
+
file alone does not establish every effective native default or child route.
|
|
436
|
+
|
|
437
|
+
After execution, isolated shell and native semantic/advisory graders receive
|
|
438
|
+
read-only bindings to these original candidate paths:
|
|
439
|
+
|
|
440
|
+
| Environment variable | Original candidate path |
|
|
441
|
+
| ----------------------------------------------- | ------------------------------------------ |
|
|
442
|
+
| `SEVRO_CANDIDATE_ARTIFACT_CODEX_CONFIG` | Native `CODEX_HOME/config.toml` (one file) |
|
|
443
|
+
| `SEVRO_CANDIDATE_ARTIFACT_CODEX_SKILLS` | Native `CODEX_HOME/skills` |
|
|
444
|
+
| `SEVRO_CANDIDATE_ARTIFACT_AGENT_SKILLS` | Native process `HOME/.agents/skills` |
|
|
445
|
+
| `SEVRO_CANDIDATE_ARTIFACT_COMMAND_AGENT_SKILLS` | Command `HOME/.agents/skills` |
|
|
446
|
+
| `SEVRO_CANDIDATE_ARTIFACT_COMMAND_CODEX_SKILLS` | Command `HOME/.codex/skills` |
|
|
447
|
+
|
|
448
|
+
Native and command homes are separate. The runner initializes these actual
|
|
449
|
+
skill locations before the candidate starts and retains their resulting state;
|
|
450
|
+
it does not create replacement empty directories for grading. Graders keep their
|
|
451
|
+
own homes and must use these bindings. To assert absence, require every selected
|
|
452
|
+
directory to exist, then search for `SKILL.md`, excluding only the native Codex
|
|
453
|
+
root's read-only `.system` subtree. Missing evidence is an error. Do not search
|
|
454
|
+
the grader's home or retired fixture state.
|
|
455
|
+
|
|
456
|
+
The five artifact names are reserved. Each binding belongs to one active trial
|
|
457
|
+
and one native request; reuse by another host request is refused. The same
|
|
458
|
+
canonical ancestor, entry-limit, symlink, hardlink and special-file checks as
|
|
459
|
+
ordinary artifacts apply. Configuration cannot be replaced by a directory grant.
|
|
460
|
+
The native home is retained through grading and removed with its trial, including
|
|
461
|
+
failure and cancellation paths. Standalone host calls clean up immediately.
|
|
462
|
+
These grants do not persist native files in run evidence; private absolute paths
|
|
463
|
+
do not enter identity, while the `codexState` selection does. Other candidate
|
|
464
|
+
hosts cannot supply this Codex evidence and fail grading when it is requested.
|
|
465
|
+
|
|
466
|
+
For the Darrow doctor and undefined-skill cases, select this configuration with
|
|
467
|
+
`--runtime-config-file`, preserve the doctor fixture's checkout value `99`, and
|
|
468
|
+
replace the retired-state absence check with the four skill-directory bindings
|
|
469
|
+
above. Keep the original repository, no-commit, activation and semantic checks.
|
|
470
|
+
|
|
251
471
|
## Goals and plugin hooks
|
|
252
472
|
|
|
253
473
|
`hooks.nativeGoal` enables Claude's native goal Stop-hook path while preserving
|
|
@@ -272,6 +492,17 @@ instead of being skipped. The wrapper denies network access and protected
|
|
|
272
492
|
roots and owns asynchronous hook cleanup. Native timeout/cancellation stops
|
|
273
493
|
owned hook descendants before private state is removed.
|
|
274
494
|
|
|
495
|
+
Prepared native plugin projections live outside the fixture in the current
|
|
496
|
+
host request's private native state. Curated hook read grants validate the
|
|
497
|
+
canonical projection layout, ownership, private state permissions, and binding
|
|
498
|
+
to that request. Only the selected plugin subtree is readable; it stays
|
|
499
|
+
read-only, and credentials, private siblings, and peer requests remain denied.
|
|
500
|
+
Ordinary filesystem declarations cannot select this exception. Codex validates
|
|
501
|
+
and filters installed plugins before enabling its runner-owned skill collector,
|
|
502
|
+
including requests without a runtime policy. Plugin command handlers still
|
|
503
|
+
require explicit hook authority. Projections are removed with their owning host
|
|
504
|
+
request.
|
|
505
|
+
|
|
275
506
|
Auxiliary `.mcp.json` and `.lsp.json` files and plugin mods are unsupported under
|
|
276
507
|
hook authority. Manifest MCP/LSP declarations are removed from curated mounts.
|
|
277
508
|
|
|
@@ -339,7 +570,7 @@ run evidence. Extensions and external benchmarks consume the
|
|
|
339
570
|
## Evidence and compatibility
|
|
340
571
|
|
|
341
572
|
`configuration.redacted.runtimePolicy` retains the selected environment,
|
|
342
|
-
canonical read roots, seed digests, and hook declaration. The engine snapshots
|
|
573
|
+
canonical read roots, seed digests, shared-cache declarations, and hook declaration. The engine snapshots
|
|
343
574
|
this input once; it participates in `configurationDigest` and evaluation
|
|
344
575
|
identity. Symbolic runtime placeholders remain symbolic in identity, while
|
|
345
576
|
execution uses per-role paths. This binds declared setup and seed contents;
|
|
@@ -353,7 +584,8 @@ file contents do not. The same discovery and skip evidence reaches
|
|
|
353
584
|
`sevro.host.runtime` observations.
|
|
354
585
|
|
|
355
586
|
Bundled hosts supply `sevro.host.runtime` with the applied policy digest, role,
|
|
356
|
-
environment names, read roots,
|
|
587
|
+
environment names, read roots, seed slots, and selected shared-cache declarations.
|
|
588
|
+
Shared entries contain no content digest. `sevro.host.hooks` records source
|
|
357
589
|
and effective plugin digests plus observed executions and their exit codes.
|
|
358
590
|
An empty execution list does not prove hook activation. Partial lifecycle
|
|
359
591
|
evidence remains partial. Cases can require these observations; missing required
|
|
@@ -363,7 +595,8 @@ task verdict.
|
|
|
363
595
|
`--toolchain-bin-dir` and `--claude-uv-cache-dir` are deprecated. They keep their
|
|
364
596
|
previous behavior with a warning when no runtime file is selected. Combining
|
|
365
597
|
them with a runtime file fails before execution. Migrate to `environment`,
|
|
366
|
-
`filesystem`, and private seed slots
|
|
598
|
+
`filesystem`, and either private seed slots or shared-cache declarations; removal
|
|
599
|
+
is no earlier than `0.2.0`. The deprecated UV option continues to copy privately.
|
|
367
600
|
See the [CLI reference](development-cli.md), [native hosts](native-hosts.md),
|
|
368
601
|
and [identity contract](identity-v1.md).
|
|
369
602
|
|
package/docs/troubleshooting.md
CHANGED
|
@@ -3,18 +3,18 @@
|
|
|
3
3
|
Start with execution, grading, task states, and stderr. A documented possible
|
|
4
4
|
cause is not a diagnosis of your machine.
|
|
5
5
|
|
|
6
|
-
| Symptom | Check and next step
|
|
7
|
-
| ----------------------------------------- |
|
|
8
|
-
| Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md).
|
|
9
|
-
| Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot.
|
|
10
|
-
| Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md).
|
|
11
|
-
| Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md).
|
|
12
|
-
| Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md).
|
|
13
|
-
| Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md).
|
|
14
|
-
| Native or shell isolation fails |
|
|
15
|
-
| Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md).
|
|
16
|
-
| Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout.
|
|
17
|
-
| External links fail | Follow [network failure triage](documentation-quality.md#external-links).
|
|
6
|
+
| Symptom | Check and next step |
|
|
7
|
+
| ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
8
|
+
| Command cannot run | Use Bun 1.3.13 and verify the local package/version. Read [installation](installing.md). |
|
|
9
|
+
| Project snapshot contains a symbolic link | Use a separate evaluated project directory as in [the tutorial](getting-started.md); do not include installed dependencies in a plain-directory snapshot. |
|
|
10
|
+
| Exit 0 but no passing task | Prompt-only/dry runs are `not_assessed`. Read [results](reading-results.md). |
|
|
11
|
+
| Exit 64 before execution | Check absolute paths, exclusive route options, configuration, and capabilities in the [CLI reference](development-cli.md). |
|
|
12
|
+
| Grading unavailable | Inspect required evidence/completeness in [results v1](results-v1.md). |
|
|
13
|
+
| Native authentication fails | Check the exact release and private credential route. Read [native hosts](native-hosts.md). |
|
|
14
|
+
| Native or shell isolation fails | On macOS, verify `sandbox-exec`; on Ubuntu 24.04 x64, verify `bubblewrap` and Claude's `socat`. See [validated scope](native-hosts.md#validated-platform-scope). Windows is unverified. |
|
|
15
|
+
| Extension protocol fails | Return one matching stdout response; diagnostics use stderr. Read [protocol v1](extension-protocol-v1.md). |
|
|
16
|
+
| Schema freshness fails | Run `bun run schemas:generate`, then `bun run schemas:check`, from a checkout. |
|
|
17
|
+
| External links fail | Follow [network failure triage](documentation-quality.md#external-links). |
|
|
18
18
|
|
|
19
19
|
## The repository guide is missing
|
|
20
20
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# TypeScript quality
|
|
2
2
|
|
|
3
|
-
`bun run check:typescript` is the local
|
|
3
|
+
`bun run check:typescript` is the complete local acceptance command. CI runs the
|
|
4
|
+
same checks in parallel phases. The complete command performs a
|
|
4
5
|
frozen dependency installation, checks the complete source inventory, verifies
|
|
5
6
|
generated schemas, documentation contracts, formatting, type-aware lint and typing,
|
|
6
7
|
checks the Codex guide evaluation in dry mode and runs documentation examples, exercises coverage
|
|
@@ -14,13 +15,44 @@ schemas, documentation contracts, formatting, lint and types without the full in
|
|
|
14
15
|
or package installation. Install dependencies with `bun install --frozen-lockfile`
|
|
15
16
|
first. Quality dependencies are development dependencies pinned in `bun.lock`.
|
|
16
17
|
|
|
18
|
+
After frozen installation, `bun run check:typescript --phase <name>` runs one
|
|
19
|
+
phase and checks the complete source inventory. Each phase reports its elapsed
|
|
20
|
+
time, including when it fails. The complete command runs these same phases in
|
|
21
|
+
order; `--fast` runs only `static` without installing dependencies.
|
|
22
|
+
|
|
23
|
+
| Phase | Checks |
|
|
24
|
+
| --------- | --------------------------------------------------------------------------------------------- |
|
|
25
|
+
| `static` | Schema freshness, documentation contracts, formatting, type-aware lint and typing |
|
|
26
|
+
| `docs` | All guide dry runs, tutorial examples and recipe checks |
|
|
27
|
+
| `tests` | Coverage integrity controls, the complete instrumented Bun suite and both coverage thresholds |
|
|
28
|
+
| `package` | Packed-package installation and installed CLI workflows |
|
|
29
|
+
|
|
30
|
+
Use the same prerequisites as the corresponding complete-gate checks. A single
|
|
31
|
+
phase is partial evidence; full acceptance requires all four on the same candidate.
|
|
32
|
+
CI assigns every test file to exactly one of four duration-balanced shards, then
|
|
33
|
+
enforces both coverage thresholds on their combined counters over the complete
|
|
34
|
+
production inventory. Local `--phase tests` and the complete command still run
|
|
35
|
+
the suite in one process. See the [shard commands](../scripts/coverage/README.md#test-shards)
|
|
36
|
+
for reproducing the CI split.
|
|
37
|
+
|
|
17
38
|
## Source dispositions
|
|
18
39
|
|
|
19
|
-
[
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
40
|
+
The [inventory guard](../scripts/typescript-inventory.ts) derives every TypeScript
|
|
41
|
+
and JavaScript source disposition from Git paths, following the
|
|
42
|
+
[maintainer decision in #143](https://github.com/BjRo/sevro/issues/143#issuecomment-6382662399).
|
|
43
|
+
It classifies `git ls-files --cached --others --exclude-standard -z` output using
|
|
44
|
+
the checked path rules, so supported additions and staged renames or deletions
|
|
45
|
+
need no manual manifest edit. Non-ignored untracked sources are included in local
|
|
46
|
+
candidate checks; committed CI checkouts use their tracked paths. Declarations
|
|
47
|
+
take precedence over directory roles, and only the validators in the shared
|
|
48
|
+
[schema catalog](../scripts/schema-catalog.ts) receive the generated disposition.
|
|
49
|
+
Unknown or ambiguous paths, missing tracked files, an incomplete generated set,
|
|
50
|
+
and sources outside lint or typing are refused. An independent filesystem audit
|
|
51
|
+
also refuses ignored authored sources rather than silently dropping them.
|
|
52
|
+
Changing a disposition rule still requires review. Isolated coverage snapshots
|
|
53
|
+
index their copied candidate independently, preserving additions and deletions
|
|
54
|
+
without changing the source checkout's index.
|
|
55
|
+
The coverage baseline includes the platform-reachable subset of
|
|
24
56
|
`inventory.production`; `scripts/coverage/platform-scope.ts` names each excluded
|
|
25
57
|
host-specific file. Every production file remains inventoried, typed and linted,
|
|
26
58
|
and the opposite platform measures it. Current instrumentation supports authored
|
|
@@ -29,14 +61,14 @@ validators in `inventory.generated` are excluded from instrumentation and the
|
|
|
29
61
|
coverage denominator. New `.tsx`, `.mts`, `.cts` or JavaScript
|
|
30
62
|
production requires an explicit instrumentation and quality-policy change first.
|
|
31
63
|
|
|
32
|
-
| Inventory | Lint and typing | Test and coverage disposition
|
|
33
|
-
| -------------- | ---------------------------------------------------------------------------------------------------- |
|
|
34
|
-
| `production` | Type-aware strict lint and strict compiler | All platform-reachable authored executable `src` statements and branch outcomes, including unimported files in that scope
|
|
35
|
-
| `generated` | CommonJS syntax parsing, exact `schemas:check` freshness, and checked `.d.cts` boundary declarations | Real Bun schema tests; the five declared generated validators are outside the coverage denominator
|
|
36
|
-
| `declarations` | Type-aware lint and compiler | Declaration-only files have no executable counters
|
|
37
|
-
| `tooling` | Same type-aware lint and strict compiler; `.mjs`/`.cjs` use `checkJs` and explicit JSDoc contracts | Durable public gate probes, schema freshness, release and installed-package tests; outside the runner-production denominator
|
|
38
|
-
| `tests` | Same type-aware lint and strict compiler, including fixtures | Real Bun filesystem/process/protocol tests; outside the production denominator
|
|
39
|
-
| `examples` | Same type-aware lint and strict compiler | Packaged
|
|
64
|
+
| Inventory | Lint and typing | Test and coverage disposition |
|
|
65
|
+
| -------------- | ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
66
|
+
| `production` | Type-aware strict lint and strict compiler | All platform-reachable authored executable `src` statements and branch outcomes, including unimported files in that scope |
|
|
67
|
+
| `generated` | CommonJS syntax parsing, exact `schemas:check` freshness, and checked `.d.cts` boundary declarations | Real Bun schema tests; the five declared generated validators are outside the coverage denominator |
|
|
68
|
+
| `declarations` | Type-aware lint and compiler | Declaration-only files have no executable counters |
|
|
69
|
+
| `tooling` | Same type-aware lint and strict compiler; `.mjs`/`.cjs` use `checkJs` and explicit JSDoc contracts | Durable public gate probes, schema freshness, release and installed-package tests; outside the runner-production denominator |
|
|
70
|
+
| `tests` | Same type-aware lint and strict compiler, including fixtures | Real Bun filesystem/process/protocol tests; outside the production denominator |
|
|
71
|
+
| `examples` | Same type-aware lint and strict compiler | Packaged recipe adapters, extensions and evaluator checks are consumer examples, exercised by tutorial and installed-package validation; they implement no runner behavior and are outside the runner-production denominator |
|
|
40
72
|
|
|
41
73
|
Generated AJV CommonJS is deliberately not rewritten to conform to authored-code
|
|
42
74
|
rules. An earlier audit of the original four generated files using ESLint's
|
|
@@ -45,8 +77,8 @@ recommended JavaScript rules found 879 redeclarations, 281 useless assignments,
|
|
|
45
77
|
generation freshness, typed public boundaries and runtime behavior remain checked.
|
|
46
78
|
Generated compiler output is excluded from coverage because the coverage target
|
|
47
79
|
measures authored runner behavior. This applies only to
|
|
48
|
-
|
|
49
|
-
|
|
80
|
+
the exact validator paths derived from the [schema catalog](../scripts/schema-catalog.ts),
|
|
81
|
+
which also drives schema generation. This disposition does not apply to authored CommonJS, including
|
|
50
82
|
the coverage capture hook.
|
|
51
83
|
|
|
52
84
|
## Types, functions and review
|
|
@@ -120,7 +152,7 @@ macOS `sandbox-exec`, Keychain and Apple metadata modules are excluded; on macOS
|
|
|
120
152
|
`bubblewrap` module is excluded. Shared isolation and credential code is counted
|
|
121
153
|
on both.
|
|
122
154
|
|
|
123
|
-
Apple developer-tool discovery lives in `src/runtime
|
|
155
|
+
Apple developer-tool discovery lives in `src/runtime/discovery-apple.ts`, which
|
|
124
156
|
the public loader imports only under its explicit Darwin guard. Linux excludes
|
|
125
157
|
that exact native module; macOS measures it, and both platforms still typecheck
|
|
126
158
|
and lint it. The provider lookup, bounded query and read-root safeguards remain
|
|
@@ -146,12 +178,24 @@ summary, process ledger, Bun test log, unmodified source snapshot, LCOV and HTML
|
|
|
146
178
|
production files, branch exclusions, and exact projected counts.
|
|
147
179
|
`coverage-raw.json` and the raw summary preserve all validated counters before
|
|
148
180
|
platform projection.
|
|
181
|
+
CI also retains `.quality/shards/<index>/` with each shard's source snapshot,
|
|
182
|
+
process ledger, test log, raw and portable counters, and manifest. Process IDs
|
|
183
|
+
remain scoped to their original shard. Aggregation verifies candidate contents,
|
|
184
|
+
workflow run, platform, architecture, Bun version, instrumentation
|
|
185
|
+
layout, and exact test assignments before combining counters with normalized
|
|
186
|
+
source paths. Missing, repeated, incomplete, or mismatched shards fail the gate.
|
|
187
|
+
Failed-job reruns can reuse successful shards from earlier attempts of the same
|
|
188
|
+
workflow run; manifests retain each attempt, and rerun shards replace only their
|
|
189
|
+
own artifacts. Another candidate or workflow run cannot supply missing evidence.
|
|
149
190
|
`--coverage [Bun test arguments]` is a diagnostic measurement; it still enforces
|
|
150
191
|
both thresholds and writes `.quality/coverage-targeted`, keeping it distinct from
|
|
151
192
|
the full-suite evidence. Sparse runs are expected to fail the full denominator.
|
|
152
193
|
|
|
153
194
|
The durable public probes are `--probe snapshot`, `--probe scope`, `--probe branch`,
|
|
154
195
|
`--probe missing-reports`, `--probe integrity`, and `--probe source-map`.
|
|
196
|
+
`tests/scripts/typescript-quality.test.ts` invokes each probe once during the suite;
|
|
197
|
+
the phase entrypoint and shard merge do not rerun them. The commands remain
|
|
198
|
+
available for targeted diagnostics.
|
|
155
199
|
The scope probe records a zero-seeded baseline and merged real process report
|
|
156
200
|
under `.quality/coverage-scope`; it verifies that ordinary generated CommonJS
|
|
157
201
|
validation still runs. Branch and missing-report controls
|
|
@@ -167,21 +211,40 @@ Codex 0.160.1 and Claude Code 2.1.284 for deterministic native hook tests. Hoste
|
|
|
167
211
|
CI targets `ubuntu-24.04` x64 with Bun 1.3.13 and Node 24.13.0. Its first
|
|
168
212
|
successful hosted run is required before describing that environment as validated.
|
|
169
213
|
GitHub documents the [Ubuntu runner labels](https://docs.github.com/en/actions/reference/runners/github-hosted-runners).
|
|
170
|
-
Ubuntu installs `bubblewrap` and `socat` and runs the same native isolation cases;
|
|
171
|
-
local macOS runs retain `sandbox-exec`.
|
|
214
|
+
Ubuntu installs `bubblewrap`, `ripgrep`, and `socat` and runs the same native isolation cases;
|
|
215
|
+
local macOS runs retain `sandbox-exec`. The macOS quality and coverage gate is
|
|
216
|
+
local-only; the hosted workflow does not run a macOS gate. Windows remains unverified.
|
|
217
|
+
|
|
218
|
+
Ordinary changes and explicit workflow dispatches run the four canonical phases
|
|
219
|
+
concurrently. The tests phase consists of four shard jobs followed by a combined
|
|
220
|
+
coverage job. Each job installs locked dependencies; native host tools are
|
|
221
|
+
installed for `docs` and the test shards, and UV and mise for `docs`. Each shard
|
|
222
|
+
retains its evidence, and the combined `tests` job uploads `.quality` even when
|
|
223
|
+
coverage fails. Shards do not cancel their siblings on failure. A new commit to
|
|
224
|
+
the same pull request cancels its superseded workflow run; main pushes and manual
|
|
225
|
+
runs have independent concurrency groups. The stable aggregate `TypeScript quality`
|
|
226
|
+
status requires every phase and all four shards to succeed. A missing, cancelled,
|
|
227
|
+
skipped or failed phase or shard cannot turn that status green.
|
|
172
228
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
229
|
+
CI caches Bun's package downloads by Bun version, OS, architecture and `bun.lock`;
|
|
230
|
+
every job still performs frozen installation. The docs job and test shards also
|
|
231
|
+
cache the installed Codex and Claude packages by OS, architecture and the setup
|
|
232
|
+
script's content hash, which includes both pinned versions and their installation
|
|
233
|
+
layout. There are no fallback keys for native tools. On a hit, setup copies the
|
|
234
|
+
cached packages into a fresh temporary tool root and verifies both native versions
|
|
235
|
+
before publishing their paths. The cache never contains credentials or replaces
|
|
236
|
+
the isolation prerequisites or required checks.
|
|
177
237
|
|
|
178
238
|
Release metadata changes follow [the release workflow](releases.md) and reuse a
|
|
179
239
|
successful full gate from an ancestor on `main`, with identical code,
|
|
180
240
|
dependencies, schemas, tests, and other quality inputs. The selector permits
|
|
181
241
|
only version/distribution-tag changes and its exact release-file inventory.
|
|
182
|
-
It verifies
|
|
183
|
-
|
|
184
|
-
|
|
242
|
+
It verifies each original phase job and its gate step, including every shard and
|
|
243
|
+
the combined tests job for sharded runs, or a historical complete gate job and
|
|
244
|
+
step. A partial matrix or reused release run cannot serve as fresh test evidence.
|
|
245
|
+
Duplicate phase or shard jobs are ambiguous and refused. Release CI
|
|
246
|
+
runs `check:typescript --fast`, records the tested source and CI run URL, and
|
|
247
|
+
skips all four full-gate phase jobs. The aggregate
|
|
185
248
|
status requires both verified prior evidence and successful static checks;
|
|
186
249
|
missing, cancelled, failed, or ambiguous evidence cannot establish a pass.
|
|
187
250
|
Reused results remain attributed to their original run rather than being
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
|
|
4
|
+
/** Trusted, deterministic participant for exercising the recipe's real graders. */
|
|
5
|
+
const host = {
|
|
6
|
+
id: "recipes.deterministic",
|
|
7
|
+
model: "fixture-editor",
|
|
8
|
+
effort: "none",
|
|
9
|
+
async run({ workspace }: { workspace: string }) {
|
|
10
|
+
const task = await readFile(join(workspace, "README.md"), "utf8");
|
|
11
|
+
const failing = process.env.SEVRO_RECIPE_FAIL === "1";
|
|
12
|
+
if (task.includes("Fix add")) {
|
|
13
|
+
await writeFile(
|
|
14
|
+
join(workspace, "math.cjs"),
|
|
15
|
+
`exports.add = (a, b) => a ${failing ? "-" : "+"} b;\n`,
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
return {
|
|
19
|
+
finalMessage: failing
|
|
20
|
+
? "Start with npm start on port 9000."
|
|
21
|
+
: "Run bun run dev on port 3000.",
|
|
22
|
+
complete: true,
|
|
23
|
+
};
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
export default host;
|