@bjoernrochel/sevro 0.1.0-rc.4 → 0.1.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +43 -2
- package/docs/builtin-graders-v1.md +4 -1
- package/docs/codex-host.md +16 -1
- package/docs/evidence/runtime-startup-30/README.md +185 -0
- package/docs/evidence/runtime-startup-30/after.json +319 -0
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
- package/docs/evidence/runtime-startup-30/before.json +403 -0
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
- package/docs/identity-v1.md +1 -1
- package/docs/installing.md +2 -2
- package/docs/releases.md +47 -23
- package/docs/runtime-v1.md +58 -8
- package/docs/typescript-quality.md +16 -3
- package/package.json +1 -1
- package/schemas/runtime-v1.schema.json +49 -9
- package/src/case-validation.ts +82 -0
- package/src/cli.ts +16 -93
- package/src/evaluation-check-plan.ts +510 -0
- package/src/evaluation-context.ts +257 -0
- package/src/evaluation-fixture.ts +3 -5
- package/src/evaluation-grading-host.ts +39 -0
- package/src/evaluation-invocation-plan.ts +548 -0
- package/src/evaluation-plan.ts +15 -1034
- package/src/evaluation-run.ts +19 -1404
- package/src/evaluation-semantic.ts +290 -0
- package/src/evaluation-trial.ts +946 -0
- package/src/evaluation-types.ts +1 -0
- package/src/extension-session.ts +8 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/graders/shell.ts +2 -1
- package/src/hosts/claude-events.ts +6 -9
- package/src/hosts/claude-nested-skills.ts +7 -10
- package/src/hosts/claude-process.ts +24 -59
- package/src/hosts/claude-repository-invocation.ts +4 -7
- package/src/hosts/claude-settings.ts +86 -16
- package/src/hosts/claude-state.ts +373 -0
- package/src/hosts/claude-tool-calls.ts +10 -15
- package/src/hosts/claude.ts +28 -345
- package/src/hosts/codex-command-output.ts +7 -11
- package/src/hosts/codex-events.ts +5 -7
- package/src/hosts/codex-native-calls.ts +14 -16
- package/src/hosts/codex-process.ts +26 -66
- package/src/hosts/codex-profile.ts +52 -15
- package/src/hosts/codex-state.ts +387 -0
- package/src/hosts/codex.ts +41 -404
- package/src/hosts/native-controls.ts +5 -9
- package/src/hosts/process-lifecycle.ts +60 -0
- package/src/identity.ts +6 -3
- package/src/native-transcript-state.ts +20 -2
- package/src/native-transcript-view.ts +1 -1
- package/src/native-transcripts.ts +1 -1
- package/src/preparation.ts +2 -2
- package/src/provenance.ts +1 -6
- package/src/runtime-config.ts +61 -8
- package/src/runtime-hooks.ts +57 -25
- package/src/runtime-paths.ts +40 -11
- package/src/runtime-seeds.ts +76 -10
- package/src/runtime-state.ts +12 -21
- package/src/value-guards.ts +6 -0
package/CONTRIBUTING.md
CHANGED
|
@@ -71,6 +71,41 @@ Run `bun run check:docs:external` separately and [triage network failures](docs/
|
|
|
71
71
|
Guide changes also need [native evals](docs/guide-evaluation.md).
|
|
72
72
|
Documentation checks supplement #1 rather than replacing its contract.
|
|
73
73
|
|
|
74
|
+
## Delivery and CI ownership
|
|
75
|
+
|
|
76
|
+
An implementation PR is complete only when required CI checks pass for its exact
|
|
77
|
+
published commit. A request to open or update a PR includes monitoring CI and
|
|
78
|
+
repairing in-scope failures; it does not authorize merging or unrelated changes.
|
|
79
|
+
|
|
80
|
+
- Use GitHub CI in `.github/workflows/verify.yml` for full acceptance validation.
|
|
81
|
+
It provides the configured OS, architecture, runtime and native-tool versions,
|
|
82
|
+
non-root user, and isolation prerequisites. Reproducing that environment locally
|
|
83
|
+
with Docker is optional for diagnosis and is not a publication prerequisite.
|
|
84
|
+
- After the local pre-commit checks, publish the authorized candidate to its PR
|
|
85
|
+
and require `bun run check:typescript` to succeed in GitHub CI before reporting
|
|
86
|
+
delivery as complete. A local run, fast check, targeted suite, or historical
|
|
87
|
+
result does not establish that the current Ubuntu gate passes.
|
|
88
|
+
- Keep the quality bar intact. Repair behavior or add meaningful coverage for
|
|
89
|
+
accepted behavior; do not lower thresholds, exempt authored sources, skip tests,
|
|
90
|
+
bypass checks, or add artificial tests just to move a counter. Remove impossible
|
|
91
|
+
branches by clarifying validated invariants rather than inventing impossible inputs.
|
|
92
|
+
- Bind validation to the candidate's exact source and commit. Changes to source,
|
|
93
|
+
tests, tooling, dependencies, schemas, workflows, or guidance invalidate prior
|
|
94
|
+
candidate evidence; require the applicable CI gates on the updated published head.
|
|
95
|
+
- After authorized publication, observe required checks for the exact PR head
|
|
96
|
+
until they finish. Queued or running checks, a successful older commit, and a
|
|
97
|
+
passing subset are not completion.
|
|
98
|
+
- On failure, inspect the failed job and retained artifacts, reproduce the cause
|
|
99
|
+
locally when useful, repair within the authorized scope, and publish a non-force
|
|
100
|
+
update to the same PR. Continue monitoring without waiting for the user to notice
|
|
101
|
+
the failure or request correction; do not open a duplicate PR or merge it.
|
|
102
|
+
- If an external blocker prevents progress, report its concrete evidence and the
|
|
103
|
+
unfinished checks. Keep the PR's status and description honest; do not report
|
|
104
|
+
delivery as complete while required checks remain failing or unobserved.
|
|
105
|
+
|
|
106
|
+
Release metadata follows `docs/releases.md` and its existing verified-source CI
|
|
107
|
+
reuse policy; these implementation gates do not require release test reruns.
|
|
108
|
+
|
|
74
109
|
## Tests, fixtures, and schemas
|
|
75
110
|
|
|
76
111
|
Preserve real Bun filesystem/process integration tests and native isolation.
|
|
@@ -105,10 +140,16 @@ native trials or remote CI results.
|
|
|
105
140
|
|
|
106
141
|
## Maintainer releases
|
|
107
142
|
|
|
108
|
-
Follow [releases](docs/releases.md):
|
|
109
|
-
the
|
|
143
|
+
Follow [releases](docs/releases.md): reuse successful source CI, choose a new
|
|
144
|
+
version, and submit the release commit through a PR. Release PRs reuse the
|
|
145
|
+
existing full quality gate and run static metadata checks; releases do not
|
|
146
|
+
rerun local or CI tests. After the PR merges, prepare and inspect the exact
|
|
147
|
+
tarball and verify its license, provenance, and checksums before publication.
|
|
110
148
|
After changing `package.json.version`, run `bun run docs:sync` to refresh the
|
|
111
149
|
[current release](docs/installing.md#current-release). `check:docs`, release
|
|
112
150
|
preparation, and package-install validation reject stale release documentation.
|
|
151
|
+
Package-install, native, and Darrow compatibility tests belong to development
|
|
152
|
+
or an explicitly requested diagnostic, not the release process. Darrow's
|
|
153
|
+
dependency update is a separate PR.
|
|
113
154
|
Publication, tags, registry changes, and alternative licensing are separate
|
|
114
155
|
maintainer actions; preparing a contribution does not authorize them.
|
|
@@ -67,7 +67,10 @@ ancestry result; checks reference that observation.
|
|
|
67
67
|
per check. The operator supplies a separate semantic host route. After a
|
|
68
68
|
complete candidate response, the engine sends the bounded final message and
|
|
69
69
|
all declared propositions to that route in an empty, temporary workspace. It
|
|
70
|
-
|
|
70
|
+
omits dependency seeds unless their runtime `roles` explicitly include `semantic`.
|
|
71
|
+
Declared tool environment and read-only grants remain available. Artifact checks
|
|
72
|
+
retain the default dependency selection; executable shell checks retain their
|
|
73
|
+
separate `checks` runtime. The semantic route requires exactly one `pass` or `fail` verdict with a reason for every declared
|
|
71
74
|
check ID. A failed proposition fails the task. A missing or incomplete candidate
|
|
72
75
|
message makes semantic checks unavailable without calling the semantic host; a
|
|
73
76
|
failed, incomplete, or malformed semantic response makes grading an error.
|
package/docs/codex-host.md
CHANGED
|
@@ -148,7 +148,22 @@ network access and clears inherited command environment variables. The Codex
|
|
|
148
148
|
parent can read authentication before it starts sandboxed commands. Tests
|
|
149
149
|
invoke the installed `codex sandbox` CLI to prove fixture access, denied
|
|
150
150
|
source/auth reads, and executable access. The process adapter verifies these
|
|
151
|
-
properties before
|
|
151
|
+
properties in one sandbox invocation before each fresh host request, with a
|
|
152
|
+
60-second bounded startup deadline. The combined probe still requires readable
|
|
153
|
+
private command storage, denied project listing, denied authentication and
|
|
154
|
+
configuration reads, and a successful native executable `--version`. Continuation
|
|
155
|
+
turns reuse the same isolated state. The longer deadline accommodates observed
|
|
156
|
+
native policy startup; it is not counted as a performance saving.
|
|
157
|
+
|
|
158
|
+
Native plugin installation remains fresh for every host request. Reusing an
|
|
159
|
+
installed home would carry configuration, marketplace registrations, and
|
|
160
|
+
absolute installation paths into a different private namespace. Selected hook
|
|
161
|
+
wrappers additionally embed that request's runtime, sandbox profile, and receipt
|
|
162
|
+
paths. Replaying an old installation receipt cannot establish a new installed
|
|
163
|
+
path. Sevro therefore retains native marketplace/plugin installation and receipt
|
|
164
|
+
verification, fresh credentials/configuration, hook rewriting, preflight, and
|
|
165
|
+
invocation evidence for every request. See the [startup measurements](evidence/runtime-startup-30/README.md)
|
|
166
|
+
for measured boundaries, copy-on-write assessment, and limitations.
|
|
152
167
|
|
|
153
168
|
Native read diagnostics retain at most 64 `recoverySources` entries. Each
|
|
154
169
|
records its native ordinal, whether the native completion had output, the
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# Runtime startup measurements for issue 30
|
|
2
|
+
|
|
3
|
+
The matched synthetic native-host benchmark reduced candidate setup plus the
|
|
4
|
+
controlled participant from 14.37–14.92 seconds to 6.20–6.66 seconds. The response-only
|
|
5
|
+
grading host call, including its separate native startup, fell from 13.49–15.03
|
|
6
|
+
seconds to 5.45–5.65 seconds. These are runner-boundary measurements with synthetic
|
|
7
|
+
model responses, not measurements of API latency or live model quality.
|
|
8
|
+
|
|
9
|
+
## Reproduction and conditions
|
|
10
|
+
|
|
11
|
+
Run the same maintained [benchmark harness](../../../scripts/benchmark-runtime-startup.ts)
|
|
12
|
+
against an unchanged checkout of `c205482b25d4d52a395c70434f2b392f6acfb279` and this
|
|
13
|
+
candidate, sequentially on one machine:
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
bun scripts/benchmark-runtime-startup.ts /absolute/baseline /tmp/before.json
|
|
17
|
+
bun scripts/benchmark-runtime-startup.ts /absolute/candidate /tmp/after.json
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The retained [before](before.json) and [after](after.json) observations used
|
|
21
|
+
macOS arm64, Bun 1.3.13, Codex CLI 0.162.1, the same synthetic plugin/response case,
|
|
22
|
+
`synthetic-candidate` and `synthetic-semantic` model labels, and `low` effort.
|
|
23
|
+
There were three candidate/semantic repetitions in each process, with a 32 MiB,
|
|
24
|
+
2,049-entry public dependency tree. Every role received a fresh private home and
|
|
25
|
+
native plugin installation. The first candidate used a cold verified-snapshot
|
|
26
|
+
cache; repetitions two and three reused the policy. OS filesystem caches were
|
|
27
|
+
uncontrolled and are not described as cold. Baseline ran before candidate;
|
|
28
|
+
order effects and three repetitions limit generalization.
|
|
29
|
+
|
|
30
|
+
The harness wraps the same production functions and process boundaries without
|
|
31
|
+
changing their arguments or timeout. Only external model output is replaced by a
|
|
32
|
+
fixed event stream. The native plugin installer and `codex sandbox` remain real.
|
|
33
|
+
This fixture does not claim live skill invocation; separate native integration
|
|
34
|
+
coverage and consumer smoke own that evidence. Timings overlap: runtime and
|
|
35
|
+
sandbox phases are inside candidate/grading-host totals, and plugin commands are
|
|
36
|
+
inside plugin installation. Do not sum both levels. The `grading` measurement surrounds the semantic host
|
|
37
|
+
call with its response-only policy; it excludes engine fixture preparation,
|
|
38
|
+
verdict parsing, and extension bookkeeping.
|
|
39
|
+
|
|
40
|
+
## Observed timings
|
|
41
|
+
|
|
42
|
+
All values below are milliseconds, in repetition order.
|
|
43
|
+
|
|
44
|
+
| Boundary | Before | After |
|
|
45
|
+
| -------------------------------------------- | ------------------------ | --------------------- |
|
|
46
|
+
| Configuration fingerprint (once) | 262 | 299 |
|
|
47
|
+
| Candidate runtime preparation | 1,969 / 1,357 / 1,273 | 721 / 431 / 443 |
|
|
48
|
+
| Candidate hook dependency preparation | 1,226 / 1,465 / 1,235 | No invocation |
|
|
49
|
+
| Fresh native plugin installation | 385 / 141 / 143 | 188 / 141 / 143 |
|
|
50
|
+
| Candidate sandbox probes, summed per role | 10,938 / 11,032 / 11,622 | 5,490 / 5,404 / 5,448 |
|
|
51
|
+
| Controlled participant process | 27 / 29 / 39 | 27 / 27 / 24 |
|
|
52
|
+
| Candidate total | 14,925 / 14,369 / 14,666 | 6,659 / 6,199 / 6,257 |
|
|
53
|
+
| Semantic runtime preparation | 1,174 / 1,236 / 2,262 | Under 1 each |
|
|
54
|
+
| Semantic hook dependency preparation | 1,105 / 1,087 / 1,363 | No invocation |
|
|
55
|
+
| Semantic sandbox probes, summed per role | 10,855 / 10,803 / 11,032 | 5,548 / 5,360 / 5,365 |
|
|
56
|
+
| Controlled semantic response process | 28 / 44 / 27 | 29 / 28 / 28 |
|
|
57
|
+
| Grading host total, including native startup | 13,495 / 13,534 / 15,030 | 5,647 / 5,454 / 5,460 |
|
|
58
|
+
|
|
59
|
+
The probe count fell from two to one per fresh role. Candidate dependency
|
|
60
|
+
materialization reuses immutable verified bytes after its first capture, and
|
|
61
|
+
response-only grading no longer materializes implicit candidate dependencies.
|
|
62
|
+
Configuration hashing remains necessary for identity. Fresh-install warm timings
|
|
63
|
+
were essentially unchanged; no plugin-install speedup is claimed. Participant
|
|
64
|
+
and semantic outputs are synthetic controls, so no API/model saving is claimed.
|
|
65
|
+
|
|
66
|
+
## Startup deadline diagnosis
|
|
67
|
+
|
|
68
|
+
A separate authenticated Darrow baseline using Codex 0.162.1 and `gpt-6-luna`
|
|
69
|
+
with medium effort failed before participant execution: the first sandbox probe
|
|
70
|
+
hit its 10-second deadline at 10,004.8 ms. A seedless control retained the same
|
|
71
|
+
tool declarations and also timed out at 10,004.2 ms. With only the diagnostic
|
|
72
|
+
probe deadline increased to 60 seconds, both original full-policy probes passed
|
|
73
|
+
at 10,649.1 and 10,690.4 ms. Participant execution was deliberately withheld in
|
|
74
|
+
that diagnostic. A minimal-policy control without tool grants or seeds passed
|
|
75
|
+
at 5,372.8 and 5,681.1 ms.
|
|
76
|
+
|
|
77
|
+
Raw diagnostic records are retained for the [original timeout](baseline-timeout.jsonl),
|
|
78
|
+
[seedless timeout](seedless-timeout.jsonl), [full policy with diagnostic deadline](full-policy-diagnostic.jsonl),
|
|
79
|
+
and [minimal policy](minimal-policy-diagnostic.jsonl). The maintained
|
|
80
|
+
[diagnostic preload](../../../scripts/diagnose-codex-startup.ts) records bounded
|
|
81
|
+
phase timing/errors and withholds app-server participant execution. To reproduce
|
|
82
|
+
against an explicit checkout, set `SEVRO_DIAGNOSTIC_SOURCE_ROOT` and
|
|
83
|
+
`SEVRO_DIAGNOSTIC_PHASE_FILE`, then use a CLI wrapper running:
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
bun --preload /absolute/candidate/scripts/diagnose-codex-startup.ts /absolute/baseline/src/cli.ts "$@" --runner-checkout-root /absolute/baseline
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Point Darrow's `SEVRO_PACKAGE_BIN` at that wrapper and run:
|
|
90
|
+
|
|
91
|
+
```sh
|
|
92
|
+
bun evals/runner/run.ts --harness codex --owner-evaluation passive --skill create-commit --case create-commit-conventional-format --model gpt-6-luna --effort medium --trials 1 --jobs 1 --threshold 1 --results-root /absolute/outside-project-results --output /absolute/selection.json
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
For the deadline control, additionally set `SEVRO_DIAGNOSTIC_TIMEOUT_MS=60000`.
|
|
96
|
+
For runtime controls, append `-- --runtime-config-file /absolute/runtime.json`:
|
|
97
|
+
seedless keeps the original policy except `runtime.seedDirectories=[]`; minimal
|
|
98
|
+
keeps only `format`, `nativeTranscripts: true`, and `hooks.nativeGoal: true`.
|
|
99
|
+
The original diagnostic used the same wrappers with fixed paths; the retained
|
|
100
|
+
preload makes those paths explicit through environment variables.
|
|
101
|
+
|
|
102
|
+
These one-off controls locate the failure in native sandbox startup and show
|
|
103
|
+
that dependency copying is not necessary for the timeout. Declared tool grants
|
|
104
|
+
increase observed startup on this machine. They do not establish the exact
|
|
105
|
+
native CLI mechanism or a general policy-size scaling law. The product now uses
|
|
106
|
+
a bounded 60-second preflight; the measured saving comes from eliminating the
|
|
107
|
+
second invocation and repeated preparation, not from increasing that deadline.
|
|
108
|
+
The unavailable baseline participant, activation, and grading evidence remain
|
|
109
|
+
unavailable rather than being assigned successful outcomes.
|
|
110
|
+
|
|
111
|
+
## Snapshot and copy-on-write assessment
|
|
112
|
+
|
|
113
|
+
The production snapshot owns verified bytes only in runner memory, keyed by the
|
|
114
|
+
policy's seed descriptor. Candidate, peer, checker, and hook writable files never
|
|
115
|
+
alias it. First capture must match the configured digest; later source changes
|
|
116
|
+
do not change that snapshot. Regression coverage changes candidate files and the
|
|
117
|
+
operator source between trials and verifies that the next trial still receives
|
|
118
|
+
the original bytes. Executable modes, internal-link dereferencing, empty
|
|
119
|
+
directories, source protection, offline environment, and role separation remain
|
|
120
|
+
checked.
|
|
121
|
+
|
|
122
|
+
The benchmark also exercised `COPYFILE_FICLONE_FORCE` on a 32 MiB file. It
|
|
123
|
+
succeeded in about one millisecond in both runs, and changing the destination
|
|
124
|
+
left the source length unchanged. This establishes local copy-on-write support,
|
|
125
|
+
not portable availability or an immutable directory snapshot. The current
|
|
126
|
+
in-memory snapshot deliberately avoids a persistent cache path and its cleanup,
|
|
127
|
+
source-mutation, and cross-process trust requirements. Native clone reuse is not
|
|
128
|
+
implemented; ordinary materialization works on filesystems without cloning.
|
|
129
|
+
Retained policies retain their verified bytes until garbage collection, up to
|
|
130
|
+
the existing per-seed bounds. This memory tradeoff should be measured separately
|
|
131
|
+
for substantially larger dependency sets.
|
|
132
|
+
|
|
133
|
+
## Native installation reuse assessment
|
|
134
|
+
|
|
135
|
+
Every actual native installation in both benchmark runs produced configuration
|
|
136
|
+
containing its private home and fixture workspace paths, as retained in the
|
|
137
|
+
`installation-assets` observations. Native install receipts bind an
|
|
138
|
+
`installedPath` inside that particular home's plugin cache; production verifies
|
|
139
|
+
it before accepting explicit invocation. Authorized hook wrappers also bind the
|
|
140
|
+
private runtime, sandbox profile, and execution receipt directory.
|
|
141
|
+
|
|
142
|
+
A separate [native cache-only reuse experiment](plugin-template-assessment.json)
|
|
143
|
+
installed a local synthetic plugin into a fresh credential-free home. The native
|
|
144
|
+
registration wrote `source_type = "local"` and the original absolute marketplace
|
|
145
|
+
path; the receipt contained the original private `installedPath`. After copying
|
|
146
|
+
only `plugins/cache` into a second fresh home, `codex plugin list --json` returned
|
|
147
|
+
empty `installed` and `available` arrays. To reproduce, use fresh `HOME` and
|
|
148
|
+
`CODEX_HOME`, run `codex plugin marketplace add /absolute/market --json` and
|
|
149
|
+
`codex plugin add probe@sevro-assessment --json`, copy `plugins/cache` to another
|
|
150
|
+
fresh home, and list there. Paths in the raw record use `<assessment>` for the
|
|
151
|
+
throwaway root; no credential file was present.
|
|
152
|
+
|
|
153
|
+
An entire installed-home template therefore cannot be replayed into a new trial
|
|
154
|
+
without rewriting and reverifying those identities, isolating credentials and
|
|
155
|
+
configuration, and obtaining evidence for the new installed paths. Reusing
|
|
156
|
+
just package files would still require native registration and receipt checks.
|
|
157
|
+
For the measured fixture, warm fresh installation costs about 140 ms versus
|
|
158
|
+
roughly 5.4 seconds for one sandbox startup. No template reuse is implemented.
|
|
159
|
+
Fresh authentication/configuration, native registration and plugin installation,
|
|
160
|
+
receipt validation, hook projection, the combined preflight, and native
|
|
161
|
+
invocation evidence remain per-request work. Fresh-install integration coverage
|
|
162
|
+
is retained.
|
|
163
|
+
|
|
164
|
+
## Authenticated consumer smoke
|
|
165
|
+
|
|
166
|
+
The [bounded consumer summary](consumer-smoke.json) records Darrow revision
|
|
167
|
+
`34606ad9e1952b7f4e9c56befa446c18e1a2eb5d`, with no tracked consumer edits. The
|
|
168
|
+
candidate's explicit `tdd-explicit-invocation` case passed execution, shell and
|
|
169
|
+
semantic grading, and complete native explicit-invocation activation. Candidate
|
|
170
|
+
and semantic routes used Codex 0.162.1, `gpt-6-luna`, medium effort, one trial,
|
|
171
|
+
one job, and threshold one:
|
|
172
|
+
|
|
173
|
+
```sh
|
|
174
|
+
SEVRO_CHECKOUT=/absolute/candidate bun evals/runner/run.ts --harness codex --owner-evaluation passive --skill tdd --case tdd-explicit-invocation --model gpt-6-luna --effort medium --semantic-check-model gpt-6-luna --semantic-check-effort medium --trials 1 --jobs 1 --threshold 1 --results-root /absolute/outside-project-results --output /absolute/selection.json
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
The earlier implicit `create-commit-conventional-format` smoke completed
|
|
178
|
+
execution and grading on the candidate, including its three shell checks, but
|
|
179
|
+
failed the expected HEAD-change assertion. Its fixture mounted the backend under
|
|
180
|
+
`.agents` without plugin manifests required by the launcher; the participant
|
|
181
|
+
refused the commit. Activation stayed unavailable. The unchanged baseline had
|
|
182
|
+
failed that case before participant execution at sandbox preflight. These
|
|
183
|
+
outcomes are retained honestly; neither the fixture failure nor the later
|
|
184
|
+
single-trial success establishes a matched live latency comparison or repeated
|
|
185
|
+
model reliability. Private native transcripts and credentials remain outside Git.
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
{
|
|
2
|
+
"format": "sevro.runtime-startup-benchmark.v1",
|
|
3
|
+
"sourceRoot": "/Users/bjro/Sources/sevro/.worktrees/perf/30-runtime-startup-overhead",
|
|
4
|
+
"platform": "darwin",
|
|
5
|
+
"arch": "arm64",
|
|
6
|
+
"bun": "1.3.13",
|
|
7
|
+
"codex": "codex-cli 0.162.1",
|
|
8
|
+
"case": "synthetic-plugin-response",
|
|
9
|
+
"model": "synthetic-candidate / synthetic-semantic",
|
|
10
|
+
"effort": "low",
|
|
11
|
+
"seed": {
|
|
12
|
+
"entries": 2049,
|
|
13
|
+
"bytes": 33554432
|
|
14
|
+
},
|
|
15
|
+
"repetitions": 3,
|
|
16
|
+
"conditions": "Fresh role homes each invocation; first verified snapshot cold, later repetitions warm; OS filesystem caches uncontrolled; no live API/model calls",
|
|
17
|
+
"rows": [
|
|
18
|
+
{
|
|
19
|
+
"repetition": 0,
|
|
20
|
+
"role": "candidate",
|
|
21
|
+
"phase": "configuration",
|
|
22
|
+
"milliseconds": 298.645833,
|
|
23
|
+
"status": "completed"
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"repetition": 1,
|
|
27
|
+
"role": "candidate",
|
|
28
|
+
"phase": "runtime.candidate",
|
|
29
|
+
"milliseconds": 721.155541,
|
|
30
|
+
"status": "completed"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"repetition": 1,
|
|
34
|
+
"role": "candidate",
|
|
35
|
+
"phase": "plugin-command",
|
|
36
|
+
"milliseconds": 113.721,
|
|
37
|
+
"status": "completed"
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"repetition": 1,
|
|
41
|
+
"role": "candidate",
|
|
42
|
+
"phase": "plugin-command",
|
|
43
|
+
"milliseconds": 70.76491699999997,
|
|
44
|
+
"status": "completed"
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"repetition": 1,
|
|
48
|
+
"role": "candidate",
|
|
49
|
+
"phase": "installation-assets",
|
|
50
|
+
"files": [
|
|
51
|
+
"cache"
|
|
52
|
+
],
|
|
53
|
+
"configurationContainsPrivateHome": true,
|
|
54
|
+
"configurationContainsWorkspace": true
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"repetition": 1,
|
|
58
|
+
"role": "candidate",
|
|
59
|
+
"phase": "plugin-install",
|
|
60
|
+
"milliseconds": 187.57354199999986,
|
|
61
|
+
"status": "completed"
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"repetition": 1,
|
|
65
|
+
"role": "candidate",
|
|
66
|
+
"phase": "sandbox-probe",
|
|
67
|
+
"milliseconds": 5489.6383749999995,
|
|
68
|
+
"status": "completed"
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"repetition": 1,
|
|
72
|
+
"role": "candidate",
|
|
73
|
+
"phase": "participant-execution",
|
|
74
|
+
"milliseconds": 27.323707999999897,
|
|
75
|
+
"status": "completed"
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
"repetition": 1,
|
|
79
|
+
"role": "candidate",
|
|
80
|
+
"phase": "candidate-total",
|
|
81
|
+
"milliseconds": 6659.420667,
|
|
82
|
+
"status": "completed"
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"repetition": 1,
|
|
86
|
+
"role": "semantic",
|
|
87
|
+
"phase": "runtime.semantic",
|
|
88
|
+
"milliseconds": 0.3450830000001588,
|
|
89
|
+
"status": "completed"
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"repetition": 1,
|
|
93
|
+
"role": "semantic",
|
|
94
|
+
"phase": "plugin-install",
|
|
95
|
+
"milliseconds": 0.005458999999973457,
|
|
96
|
+
"status": "completed"
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
"repetition": 1,
|
|
100
|
+
"role": "semantic",
|
|
101
|
+
"phase": "sandbox-probe",
|
|
102
|
+
"milliseconds": 5548.128959000001,
|
|
103
|
+
"status": "completed"
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"repetition": 1,
|
|
107
|
+
"role": "semantic",
|
|
108
|
+
"phase": "participant-execution",
|
|
109
|
+
"milliseconds": 28.713165999999546,
|
|
110
|
+
"status": "completed"
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"repetition": 1,
|
|
114
|
+
"role": "semantic",
|
|
115
|
+
"phase": "grading",
|
|
116
|
+
"milliseconds": 5646.800708000001,
|
|
117
|
+
"status": "completed"
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
"repetition": 2,
|
|
121
|
+
"role": "candidate",
|
|
122
|
+
"phase": "runtime.candidate",
|
|
123
|
+
"milliseconds": 430.6864170000008,
|
|
124
|
+
"status": "completed"
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
"repetition": 2,
|
|
128
|
+
"role": "candidate",
|
|
129
|
+
"phase": "plugin-command",
|
|
130
|
+
"milliseconds": 73.67158399999971,
|
|
131
|
+
"status": "completed"
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
"repetition": 2,
|
|
135
|
+
"role": "candidate",
|
|
136
|
+
"phase": "plugin-command",
|
|
137
|
+
"milliseconds": 65.96320800000103,
|
|
138
|
+
"status": "completed"
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
"repetition": 2,
|
|
142
|
+
"role": "candidate",
|
|
143
|
+
"phase": "installation-assets",
|
|
144
|
+
"files": [
|
|
145
|
+
"cache"
|
|
146
|
+
],
|
|
147
|
+
"configurationContainsPrivateHome": true,
|
|
148
|
+
"configurationContainsWorkspace": true
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
"repetition": 2,
|
|
152
|
+
"role": "candidate",
|
|
153
|
+
"phase": "plugin-install",
|
|
154
|
+
"milliseconds": 141.00866599999972,
|
|
155
|
+
"status": "completed"
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
"repetition": 2,
|
|
159
|
+
"role": "candidate",
|
|
160
|
+
"phase": "sandbox-probe",
|
|
161
|
+
"milliseconds": 5404.434915999998,
|
|
162
|
+
"status": "completed"
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
"repetition": 2,
|
|
166
|
+
"role": "candidate",
|
|
167
|
+
"phase": "participant-execution",
|
|
168
|
+
"milliseconds": 26.502958000000945,
|
|
169
|
+
"status": "completed"
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
"repetition": 2,
|
|
173
|
+
"role": "candidate",
|
|
174
|
+
"phase": "candidate-total",
|
|
175
|
+
"milliseconds": 6199.1395840000005,
|
|
176
|
+
"status": "completed"
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
"repetition": 2,
|
|
180
|
+
"role": "semantic",
|
|
181
|
+
"phase": "runtime.semantic",
|
|
182
|
+
"milliseconds": 0.314249999999447,
|
|
183
|
+
"status": "completed"
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
"repetition": 2,
|
|
187
|
+
"role": "semantic",
|
|
188
|
+
"phase": "plugin-install",
|
|
189
|
+
"milliseconds": 0.005916999998589745,
|
|
190
|
+
"status": "completed"
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
"repetition": 2,
|
|
194
|
+
"role": "semantic",
|
|
195
|
+
"phase": "sandbox-probe",
|
|
196
|
+
"milliseconds": 5359.928749999999,
|
|
197
|
+
"status": "completed"
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
"repetition": 2,
|
|
201
|
+
"role": "semantic",
|
|
202
|
+
"phase": "participant-execution",
|
|
203
|
+
"milliseconds": 27.60699999999997,
|
|
204
|
+
"status": "completed"
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
"repetition": 2,
|
|
208
|
+
"role": "semantic",
|
|
209
|
+
"phase": "grading",
|
|
210
|
+
"milliseconds": 5454.279749999998,
|
|
211
|
+
"status": "completed"
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
"repetition": 3,
|
|
215
|
+
"role": "candidate",
|
|
216
|
+
"phase": "runtime.candidate",
|
|
217
|
+
"milliseconds": 442.62854199999856,
|
|
218
|
+
"status": "completed"
|
|
219
|
+
},
|
|
220
|
+
{
|
|
221
|
+
"repetition": 3,
|
|
222
|
+
"role": "candidate",
|
|
223
|
+
"phase": "plugin-command",
|
|
224
|
+
"milliseconds": 72.3325420000001,
|
|
225
|
+
"status": "completed"
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
"repetition": 3,
|
|
229
|
+
"role": "candidate",
|
|
230
|
+
"phase": "plugin-command",
|
|
231
|
+
"milliseconds": 68.99258300000292,
|
|
232
|
+
"status": "completed"
|
|
233
|
+
},
|
|
234
|
+
{
|
|
235
|
+
"repetition": 3,
|
|
236
|
+
"role": "candidate",
|
|
237
|
+
"phase": "installation-assets",
|
|
238
|
+
"files": [
|
|
239
|
+
"cache"
|
|
240
|
+
],
|
|
241
|
+
"configurationContainsPrivateHome": true,
|
|
242
|
+
"configurationContainsWorkspace": true
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
"repetition": 3,
|
|
246
|
+
"role": "candidate",
|
|
247
|
+
"phase": "plugin-install",
|
|
248
|
+
"milliseconds": 142.90399999999863,
|
|
249
|
+
"status": "completed"
|
|
250
|
+
},
|
|
251
|
+
{
|
|
252
|
+
"repetition": 3,
|
|
253
|
+
"role": "candidate",
|
|
254
|
+
"phase": "sandbox-probe",
|
|
255
|
+
"milliseconds": 5447.905040999998,
|
|
256
|
+
"status": "completed"
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
"repetition": 3,
|
|
260
|
+
"role": "candidate",
|
|
261
|
+
"phase": "participant-execution",
|
|
262
|
+
"milliseconds": 23.8626669999976,
|
|
263
|
+
"status": "completed"
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
"repetition": 3,
|
|
267
|
+
"role": "candidate",
|
|
268
|
+
"phase": "candidate-total",
|
|
269
|
+
"milliseconds": 6256.5165830000005,
|
|
270
|
+
"status": "completed"
|
|
271
|
+
},
|
|
272
|
+
{
|
|
273
|
+
"repetition": 3,
|
|
274
|
+
"role": "semantic",
|
|
275
|
+
"phase": "runtime.semantic",
|
|
276
|
+
"milliseconds": 0.3507499999977881,
|
|
277
|
+
"status": "completed"
|
|
278
|
+
},
|
|
279
|
+
{
|
|
280
|
+
"repetition": 3,
|
|
281
|
+
"role": "semantic",
|
|
282
|
+
"phase": "plugin-install",
|
|
283
|
+
"milliseconds": 0.01341600000159815,
|
|
284
|
+
"status": "completed"
|
|
285
|
+
},
|
|
286
|
+
{
|
|
287
|
+
"repetition": 3,
|
|
288
|
+
"role": "semantic",
|
|
289
|
+
"phase": "sandbox-probe",
|
|
290
|
+
"milliseconds": 5365.139791000001,
|
|
291
|
+
"status": "completed"
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
"repetition": 3,
|
|
295
|
+
"role": "semantic",
|
|
296
|
+
"phase": "participant-execution",
|
|
297
|
+
"milliseconds": 28.228125000001455,
|
|
298
|
+
"status": "completed"
|
|
299
|
+
},
|
|
300
|
+
{
|
|
301
|
+
"repetition": 3,
|
|
302
|
+
"role": "semantic",
|
|
303
|
+
"phase": "grading",
|
|
304
|
+
"milliseconds": 5460.212874999997,
|
|
305
|
+
"status": "completed"
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
"repetition": 4,
|
|
309
|
+
"role": "semantic",
|
|
310
|
+
"phase": "cow-copy",
|
|
311
|
+
"milliseconds": 1.0158329999976559,
|
|
312
|
+
"status": "completed"
|
|
313
|
+
},
|
|
314
|
+
{
|
|
315
|
+
"phase": "cow-isolation",
|
|
316
|
+
"sourceUnchanged": true
|
|
317
|
+
}
|
|
318
|
+
]
|
|
319
|
+
}
|