@bjoernrochel/sevro 0.1.0-rc.13 → 0.1.0-rc.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +3 -1
- package/docs/codex-host.md +136 -109
- package/docs/development-cli.md +8 -3
- package/docs/extension-protocol-v1.md +7 -1
- package/docs/guide-evaluation.md +4 -0
- package/docs/installing.md +2 -2
- package/docs/native-hosts.md +55 -0
- package/docs/runtime-v1.md +5 -3
- package/docs/typescript-quality.md +27 -10
- package/package.json +2 -2
- package/src/hosts/claude-repository-invocation.ts +0 -23
- package/src/hosts/claude-skill-activations.ts +166 -0
- package/src/hosts/claude.ts +3 -8
- package/src/hosts/codex-app-server.ts +14 -11
- package/src/hosts/codex-native-calls.ts +7 -151
- package/src/hosts/codex-session-files.ts +0 -8
- package/src/hosts/codex-state.ts +6 -5
- package/src/hosts/codex.ts +86 -133
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +332 -0
- package/src/hosts/skill-hooks.ts +372 -0
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -523
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -1392
package/CONTRIBUTING.md
CHANGED
|
@@ -76,7 +76,9 @@ locked dependencies. This runs inventory, schema, formatting, lint, typing and
|
|
|
76
76
|
documentation checks; it does not replace full acceptance. CI retains machine-readable coverage
|
|
77
77
|
reports under `.quality/` and exposes the stable `TypeScript quality` status.
|
|
78
78
|
The complete command's `static`, `docs`, `tests` and `package` phases run as
|
|
79
|
-
concurrent CI jobs.
|
|
79
|
+
concurrent CI jobs. CI divides the tests among four duration-balanced jobs and
|
|
80
|
+
requires one combined coverage gate; every test and both 95% thresholds remain
|
|
81
|
+
required. Superseded runs for the same pull request are cancelled. After frozen installation, use
|
|
80
82
|
`bun run check:typescript --phase <name>` to run one phase locally; it does not
|
|
81
83
|
replace acceptance across all four. See the [phase inventory](docs/typescript-quality.md).
|
|
82
84
|
|
package/docs/codex-host.md
CHANGED
|
@@ -20,75 +20,13 @@ The bounded JSONL stream is retained per trial as the private
|
|
|
20
20
|
`sevro.codex.events` host artifact. Its file URL and digest appear in trial
|
|
21
21
|
evidence and are available to extension grading; the stream bytes stay out of
|
|
22
22
|
the run JSON.
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
observation partial. A missing `SKILL.md` target does not count as a read just
|
|
30
|
-
because its parent directory belongs to a mounted skill; an existing mounted
|
|
31
|
-
target that fails or produces incomplete bytes remains partial. A completed direct
|
|
32
|
-
`cat` of one mounted `SKILL.md`, alone or alongside literal documentation
|
|
33
|
-
paths, counts only when the command output contains that file's exact body.
|
|
34
|
-
Batched reads naming more than 64 paths remain partial, as do failed,
|
|
35
|
-
malformed, or incomplete recognized reads. Direct
|
|
36
|
-
`sed -n` pages count after their verified line ranges cover the whole body. An
|
|
37
|
-
exact `cat` wrapped by `lean-ctx -c` is accepted only when its reported output
|
|
38
|
-
still contains the full body; a compressed summary cannot prove the read.
|
|
39
|
-
The same exact-body check accepts a literal read followed by one literal `rg`
|
|
40
|
-
command, separated by a semicolon or newline. It also accepts a single-file
|
|
41
|
-
literal `cat` of the skill followed by one or more literal `cat` reads of
|
|
42
|
-
non-skill files, then one literal `rg` command, with the same separators. For
|
|
43
|
-
example:
|
|
23
|
+
Skill activation uses the common `sevro.host.skill-activations` observation
|
|
24
|
+
from passive hooks in the request's private Codex home. The collector matches
|
|
25
|
+
prepared skill bodies in returned tool content without interpreting shell
|
|
26
|
+
syntax or recovering command output from native sessions. See
|
|
27
|
+
[skill activation evidence](native-hosts.md#skill-activation-evidence) for the
|
|
28
|
+
scope, limits, privacy boundaries, and incomplete collection states.
|
|
44
29
|
|
|
45
|
-
```sh
|
|
46
|
-
cat .agents/skills/darrow-guide/SKILL.md; cat docs/specs/repository-guide.md; rg -n -i 'offline|\bsla\b|service.level|guarantee|deployment' --glob '*.md' --glob '!docs/research/**' --glob '!**/SKILL.md' .
|
|
47
|
-
```
|
|
48
|
-
|
|
49
|
-
This sequence uses the shared stream and native read check. Quoted separators
|
|
50
|
-
remain part of their literal operand. The ordered file bodies must match the
|
|
51
|
-
retained output, so documentation or search output cannot replace an altered
|
|
52
|
-
or incomplete skill read. Literal `cat` commands may read multiple mounted
|
|
53
|
-
skills and documentation, before or after skills, with or without a terminal
|
|
54
|
-
search. Multiple literal terminal `rg` commands are accepted within the command
|
|
55
|
-
bound. Each verified skill receives its own ordered receipt. Documentation
|
|
56
|
-
must resolve within the workspace or a receipt-bound installed plugin root;
|
|
57
|
-
each file is bounded to 1 MiB. Aliases of mounted skills without a literal
|
|
58
|
-
`SKILL.md` operand remain unsupported.
|
|
59
|
-
The 64-operand bound applies across all reads in the sequence. Shell expansions,
|
|
60
|
-
including unquoted glob and tilde operands, are unsupported in this form.
|
|
61
|
-
|
|
62
|
-
The bounded composed-read grammar also accepts `unset` with literal variable
|
|
63
|
-
names, `pwd` or `pwd -P`, `git rev-parse --show-toplevel` (optionally with
|
|
64
|
-
`-C "$PWD"`), `printenv` with literal variable names, and `rg --files` with
|
|
65
|
-
literal arguments before the reads. Prefix `printenv` requires the reads to
|
|
66
|
-
end the command, so environment output cannot replace missing read bytes.
|
|
67
|
-
Literal `printenv` commands may follow reads without prefix `printenv`.
|
|
68
|
-
The fixed pair `repo=$(git -C "$PWD" rev-parse --show-toplevel)` and
|
|
69
|
-
`cd "$repo" && pwd -P` is supported only with absolute read operands. Other
|
|
70
|
-
assignments, substitutions, directory changes, or control flow remain
|
|
71
|
-
unsupported. Prefix and environment output cannot supply skill frontmatter;
|
|
72
|
-
ambiguous repeated output cannot establish an attributable read.
|
|
73
|
-
|
|
74
|
-
The check also accepts a chain of literal
|
|
75
|
-
`cat` fallbacks with `2>/dev/null` and `||`, ending in `true` followed by a newline
|
|
76
|
-
and one literal `rg` discovery command. A fallback chain must resolve to one
|
|
77
|
-
canonical file across all its read operands; absent alternate targets do not
|
|
78
|
-
invalidate the read. Each fallback `cat` names one literal file. Multiple
|
|
79
|
-
existing targets, including unmounted copies,
|
|
80
|
-
cannot establish an attributable read. The total read operands remain bounded
|
|
81
|
-
at 64. The complete command must succeed and expose the mounted file's entire
|
|
82
|
-
exact body. `cat` accepts literal file operands and the optional `--` delimiter;
|
|
83
|
-
other options cannot prove an unchanged read. Substitutions, pipelines, other shell control flow, and dynamic
|
|
84
|
-
search arguments cannot establish a compound read through this check.
|
|
85
|
-
Unrecognized commands remain partial when their output contains the identifying
|
|
86
|
-
`name: <skill>` line from a mounted or receipt-bound installed-plugin skill.
|
|
87
|
-
This is a host observation; an extension decides what selection means for its
|
|
88
|
-
case.
|
|
89
|
-
The same check accepts skill files below the exact installed plugin roots from
|
|
90
|
-
Codex's installation receipts. It does not infer activation from arbitrary
|
|
91
|
-
files in the private plugin cache.
|
|
92
30
|
The `sevro.codex.native-calls` capability retains allowlisted direct goal and
|
|
93
31
|
agent control calls from the native session bound to the completed thread. Its
|
|
94
32
|
observation contains only call names, namespaces, and source ordinals, plus the
|
|
@@ -113,37 +51,10 @@ only when plaintext is readable. It retains neither message nor prompt bytes.
|
|
|
113
51
|
For a second prompt, `sevro.codex.continuation` retains the last validated
|
|
114
52
|
native-session ordinal before resume. Extensions can compare later call
|
|
115
53
|
ordinals with that boundary without reading private arguments.
|
|
116
|
-
|
|
117
|
-
`sevro.codex.follow-up-skill-reads` observations. Each applies the same exact
|
|
118
|
-
mounted-skill read check to one completed turn, so an extension can grade read
|
|
119
|
-
order or a missing repeat read without inferring it from the combined list.
|
|
120
|
-
For `app-server`, the verified same-thread feedback boundary separates the
|
|
121
|
-
initial phase from the follow-up phase, including subsequent native goal turns.
|
|
122
|
-
Absent, duplicated, foreign-thread, or failed boundaries do not yield phase
|
|
123
|
-
receipts.
|
|
124
|
-
|
|
125
|
-
It also checks native parent command records for exact mounted skill bodies,
|
|
126
|
-
including sessions with no spawn request. The bounded diagnostic distinguishes
|
|
127
|
-
no read from an incomplete read without claiming skill activation.
|
|
128
|
-
It retains the skill name and native ordinal when verified reads complete a
|
|
129
|
-
whole body. Repeated complete reads produce separate receipts; paged reads
|
|
130
|
-
produce one receipt when their ranges first cover the body. An incomplete
|
|
131
|
-
later reread makes the diagnostic partial. Extensions can compare these
|
|
132
|
-
ordinals with an accepted owner boundary without retaining commands or output.
|
|
133
|
-
Earlier yielded output can complete a read only when its process and unique
|
|
134
|
-
call identity bind it to that actor's native command completion. Completed
|
|
135
|
-
literal command calls use a bounded TypeScript parser; conditional or ambiguous
|
|
136
|
-
code cannot establish that binding. Recovery verifies the exact mounted bytes
|
|
137
|
-
and recomputes completeness for the matching CLI command. Private commands,
|
|
138
|
-
submitted code and recovered skill bodies stay out of public observations.
|
|
139
|
-
For accepted spawns, it also checks up to eight child thread rollouts and
|
|
54
|
+
For accepted spawns, the host checks up to eight child thread rollouts and
|
|
140
55
|
reports whether each is available, unavailable, ambiguous, or partial, with an
|
|
141
|
-
explicit truncation flag.
|
|
142
|
-
|
|
143
|
-
the exact mounted `SKILL.md` body was read, and whether the check was complete
|
|
144
|
-
or truncated. A rollout with no read differs from a missing or malformed
|
|
145
|
-
rollout. These diagnostics do not establish skill activation or task completion
|
|
146
|
-
by themselves and retain no child prompt, command, or output.
|
|
56
|
+
explicit truncation flag. Child rollout metadata describes delegation and
|
|
57
|
+
completion; skill activation remains in the common hook observation.
|
|
147
58
|
An available child rollout also reports whether one nonempty final assistant
|
|
148
59
|
message matches a later native completion event for the same turn. This proves
|
|
149
60
|
a returned turn, not the accuracy of the child's work, and retains no message
|
|
@@ -166,13 +77,11 @@ acceptance receipts keep their original meaning; submitted requests do not
|
|
|
166
77
|
acquire direct native acceptance receipts.
|
|
167
78
|
For an explicitly declared `$plugin:skill`, the host verifies that the installed
|
|
168
79
|
skill exists and that the exact token occurs once in the delivered prompt. A
|
|
169
|
-
completed turn yields a complete `sevro.codex.explicit-invocation`
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
receipt) to be complete. Failed execution cannot yield a complete dispatch
|
|
175
|
-
receipt.
|
|
80
|
+
completed turn yields a complete `sevro.codex.explicit-invocation` naming the
|
|
81
|
+
dispatched skill. This receipt proves verified dispatch and contains no
|
|
82
|
+
`observedSkills` list. Activation checks require the common hook observation;
|
|
83
|
+
missing hook evidence cannot be repaired by a dispatch receipt or native
|
|
84
|
+
transcript. Failed execution cannot yield a complete dispatch receipt.
|
|
176
85
|
The `sevro.codex.repository-invocation` capability supports an explicitly
|
|
177
86
|
declared `$skill` from `.agents/skills/<skill>/SKILL.md`. The host checks that
|
|
178
87
|
the repository mount is a bounded regular file with no symlink components and
|
|
@@ -239,6 +148,127 @@ verification, fresh credentials/configuration, hook rewriting, preflight, and
|
|
|
239
148
|
invocation evidence for every request. See the [startup measurements](evidence/runtime-startup-30/README.md)
|
|
240
149
|
for measured boundaries, copy-on-write assessment, and limitations.
|
|
241
150
|
|
|
151
|
+
## Retired skill-read evidence
|
|
152
|
+
|
|
153
|
+
The current host no longer emits `sevro.codex.skill-reads`,
|
|
154
|
+
`sevro.codex.initial-skill-reads`, `sevro.codex.follow-up-skill-reads`, or native
|
|
155
|
+
parent/child read diagnostics. Migrate activation checks to
|
|
156
|
+
`sevro.host.skill-activations`; old required evidence remains unavailable rather
|
|
157
|
+
than silently receiving a different meaning. Existing retained run evidence
|
|
158
|
+
keeps its original interpretation.
|
|
159
|
+
|
|
160
|
+
[Issue #70 representative retests](evidence/skill-reads-70/results.json) retain
|
|
161
|
+
separate task, activation, and phase-read outcomes from the private candidate,
|
|
162
|
+
plus its source fingerprints and test-first regression results. Private native
|
|
163
|
+
transcripts remain outside the repository.
|
|
164
|
+
|
|
165
|
+
<details>
|
|
166
|
+
<summary>Historical command-pattern and read-recovery contract</summary>
|
|
167
|
+
|
|
168
|
+
The following describes the retired collector before the hook migration.
|
|
169
|
+
These rules are preserved for interpreting older evidence; they do not describe
|
|
170
|
+
the current host.
|
|
171
|
+
|
|
172
|
+
The host also retains a bounded `sevro.codex.skill-reads` observation. It
|
|
173
|
+
records the first verified mounted skill and ordered skill names, without
|
|
174
|
+
commands or skill bodies. A `cat` or `sed -n` attempt counts only when its path
|
|
175
|
+
resolves to a mounted skill or a skill below an installed plugin root verified
|
|
176
|
+
by Codex's installation receipt. Missing targets outside a mounted skill
|
|
177
|
+
directory, listings, and other harmless mentions of `SKILL.md` do not make the
|
|
178
|
+
observation partial. A missing `SKILL.md` target does not count as a read just
|
|
179
|
+
because its parent directory belongs to a mounted skill; an existing mounted
|
|
180
|
+
target that fails or produces incomplete bytes remains partial. A completed direct
|
|
181
|
+
`cat` of one mounted `SKILL.md`, alone or alongside literal documentation
|
|
182
|
+
paths, counts only when the command output contains that file's exact body.
|
|
183
|
+
Batched reads naming more than 64 paths remain partial, as do failed,
|
|
184
|
+
malformed, or incomplete recognized reads. Direct
|
|
185
|
+
`sed -n` pages count after their verified line ranges cover the whole body. An
|
|
186
|
+
exact `cat` wrapped by `lean-ctx -c` is accepted only when its reported output
|
|
187
|
+
still contains the full body; a compressed summary cannot prove the read.
|
|
188
|
+
The same exact-body check accepts a literal read followed by one literal `rg`
|
|
189
|
+
command, separated by a semicolon or newline. It also accepts a single-file
|
|
190
|
+
literal `cat` of the skill followed by one or more literal `cat` reads of
|
|
191
|
+
non-skill files, then one literal `rg` command, with the same separators. For
|
|
192
|
+
example:
|
|
193
|
+
|
|
194
|
+
```sh
|
|
195
|
+
cat .agents/skills/darrow-guide/SKILL.md; cat docs/specs/repository-guide.md; rg -n -i 'offline|\bsla\b|service.level|guarantee|deployment' --glob '*.md' --glob '!docs/research/**' --glob '!**/SKILL.md' .
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
This sequence uses the shared stream and native read check. Quoted separators
|
|
199
|
+
remain part of their literal operand. The ordered file bodies must match the
|
|
200
|
+
retained output, so documentation or search output cannot replace an altered
|
|
201
|
+
or incomplete skill read. Literal `cat` commands may read multiple mounted
|
|
202
|
+
skills and documentation, before or after skills, with or without a terminal
|
|
203
|
+
search. Multiple literal terminal `rg` commands are accepted within the command
|
|
204
|
+
bound. Each verified skill receives its own ordered receipt. Documentation
|
|
205
|
+
must resolve within the workspace or a receipt-bound installed plugin root;
|
|
206
|
+
each file is bounded to 1 MiB. Aliases of mounted skills without a literal
|
|
207
|
+
`SKILL.md` operand remain unsupported.
|
|
208
|
+
The 64-operand bound applies across all reads in the sequence. Shell expansions,
|
|
209
|
+
including unquoted glob and tilde operands, are unsupported in this form.
|
|
210
|
+
|
|
211
|
+
The bounded composed-read grammar also accepts `unset` with literal variable
|
|
212
|
+
names, `pwd` or `pwd -P`, `git rev-parse --show-toplevel` (optionally with
|
|
213
|
+
`-C "$PWD"`), `printenv` with literal variable names, and `rg --files` with
|
|
214
|
+
literal arguments before the reads. Prefix `printenv` requires the reads to
|
|
215
|
+
end the command, so environment output cannot replace missing read bytes.
|
|
216
|
+
Literal `printenv` commands may follow reads without prefix `printenv`.
|
|
217
|
+
The fixed pair `repo=$(git -C "$PWD" rev-parse --show-toplevel)` and
|
|
218
|
+
`cd "$repo" && pwd -P` is supported only with absolute read operands. Other
|
|
219
|
+
assignments, substitutions, directory changes, or control flow remain
|
|
220
|
+
unsupported. Prefix and environment output cannot supply skill frontmatter;
|
|
221
|
+
ambiguous repeated output cannot establish an attributable read.
|
|
222
|
+
|
|
223
|
+
The check also accepts a chain of literal
|
|
224
|
+
`cat` fallbacks with `2>/dev/null` and `||`, ending in `true` followed by a newline
|
|
225
|
+
and one literal `rg` discovery command. A fallback chain must resolve to one
|
|
226
|
+
canonical file across all its read operands; absent alternate targets do not
|
|
227
|
+
invalidate the read. Each fallback `cat` names one literal file. Multiple
|
|
228
|
+
existing targets, including unmounted copies,
|
|
229
|
+
cannot establish an attributable read. The total read operands remain bounded
|
|
230
|
+
at 64. The complete command must succeed and expose the mounted file's entire
|
|
231
|
+
exact body. `cat` accepts literal file operands and the optional `--` delimiter;
|
|
232
|
+
other options cannot prove an unchanged read. Substitutions, pipelines, other shell control flow, and dynamic
|
|
233
|
+
search arguments cannot establish a compound read through this check.
|
|
234
|
+
Unrecognized commands remain partial when their output contains the identifying
|
|
235
|
+
`name: <skill>` line from a mounted or receipt-bound installed-plugin skill.
|
|
236
|
+
This is a host observation; an extension decides what selection means for its
|
|
237
|
+
case.
|
|
238
|
+
The same check accepts skill files below the exact installed plugin roots from
|
|
239
|
+
Codex's installation receipts. It does not infer activation from arbitrary
|
|
240
|
+
files in the private plugin cache.
|
|
241
|
+
The host also retains separate `sevro.codex.initial-skill-reads` and
|
|
242
|
+
`sevro.codex.follow-up-skill-reads` observations. Each applies the same exact
|
|
243
|
+
mounted-skill read check to one completed turn, so an extension can grade read
|
|
244
|
+
order or a missing repeat read without inferring it from the combined list.
|
|
245
|
+
For `app-server`, the verified same-thread feedback boundary separates the
|
|
246
|
+
initial phase from the follow-up phase, including subsequent native goal turns.
|
|
247
|
+
Absent, duplicated, foreign-thread, or failed boundaries do not yield phase
|
|
248
|
+
receipts.
|
|
249
|
+
|
|
250
|
+
It also checks native parent command records for exact mounted skill bodies,
|
|
251
|
+
including sessions with no spawn request. The bounded diagnostic distinguishes
|
|
252
|
+
no read from an incomplete read without claiming skill activation.
|
|
253
|
+
It retains the skill name and native ordinal when verified reads complete a
|
|
254
|
+
whole body. Repeated complete reads produce separate receipts; paged reads
|
|
255
|
+
produce one receipt when their ranges first cover the body. An incomplete
|
|
256
|
+
later reread makes the diagnostic partial. Extensions can compare these
|
|
257
|
+
ordinals with an accepted owner boundary without retaining commands or output.
|
|
258
|
+
Earlier yielded output can complete a read only when its process and unique
|
|
259
|
+
call identity bind it to that actor's native command completion. Completed
|
|
260
|
+
literal command calls use a bounded TypeScript parser; conditional or ambiguous
|
|
261
|
+
code cannot establish that binding. Recovery verifies the exact mounted bytes
|
|
262
|
+
and recomputes completeness for the matching CLI command. Private commands,
|
|
263
|
+
submitted code and recovered skill bodies stay out of public observations.
|
|
264
|
+
For accepted spawns, it also checks up to eight child thread rollouts and
|
|
265
|
+
reports whether each is available, unavailable, ambiguous, or partial, with an
|
|
266
|
+
explicit truncation flag. An available rollout includes bounded diagnostics for
|
|
267
|
+
native child commands: the number of commands and skill read attempts, whether
|
|
268
|
+
the exact mounted `SKILL.md` body was read, and whether the check was complete
|
|
269
|
+
or truncated. A rollout with no read differs from a missing or malformed
|
|
270
|
+
rollout. These diagnostics do not establish skill activation or task completion
|
|
271
|
+
by themselves and retain no child prompt, command, or output.
|
|
242
272
|
Native read diagnostics retain at most 64 `recoverySources` entries. Each
|
|
243
273
|
records its native ordinal, whether the native completion had output, the
|
|
244
274
|
yielded chunk count, and separate `completedCall` and `literalCommandCall`
|
|
@@ -246,7 +276,4 @@ flags. These facts distinguish recovery sources without retaining commands,
|
|
|
246
276
|
submitted code, or skill bodies. The diagnostic's completeness and truncation
|
|
247
277
|
fields still determine whether its read evidence can support grading.
|
|
248
278
|
|
|
249
|
-
|
|
250
|
-
separate task, activation, and phase-read outcomes from the private candidate,
|
|
251
|
-
plus its source fingerprints and test-first regression results. Private native
|
|
252
|
-
transcripts remain outside the repository.
|
|
279
|
+
</details>
|
package/docs/development-cli.md
CHANGED
|
@@ -148,6 +148,10 @@ Codex route, including semantic and advisory roles, and to both `exec` and
|
|
|
148
148
|
`app-server` entrypoints. For a 20-minute app-server evaluation, add
|
|
149
149
|
`--codex-entrypoint app-server --codex-timeout-ms 1200000` to the Codex command
|
|
150
150
|
above. The option requires a Codex host route; invalid values fail before a run.
|
|
151
|
+
For `app-server`, this is the only deadline for the whole session, including
|
|
152
|
+
startup requests such as `thread/start`. If it expires while a request is
|
|
153
|
+
unanswered, the failure names that request, for example
|
|
154
|
+
`App-server evaluation time bound reached awaiting thread/start`.
|
|
151
155
|
|
|
152
156
|
### Retain native host diagnostics
|
|
153
157
|
|
|
@@ -305,9 +309,10 @@ Without that option, the host does not advertise
|
|
|
305
309
|
`sevro.claude.repository-invocation` and refuses explicit repository dispatch
|
|
306
310
|
before launching Claude. This avoids recording an unknown command as a usable
|
|
307
311
|
repository-skill route.
|
|
308
|
-
The host
|
|
309
|
-
|
|
310
|
-
|
|
312
|
+
The host collects successful `Skill` calls through the existing structured
|
|
313
|
+
stream as [skill activation evidence](native-hosts.md#skill-activation-evidence).
|
|
314
|
+
It uses the same collector for parent and nested actors. Missing or unmatched
|
|
315
|
+
results remain partial; separate transcript-based skill detectors are retired.
|
|
311
316
|
|
|
312
317
|
A case's `followUpPrompt` starts a UUID-bound session and resumes that same
|
|
313
318
|
session after a complete successful initial result. Both results must name
|
|
@@ -171,7 +171,7 @@ other invocation declarations and enters retained configuration and identity.
|
|
|
171
171
|
The host validates the bounded mount without following symbolic links. Its
|
|
172
172
|
`sevro.claude.repository-invocation` observation binds one native command, exact
|
|
173
173
|
arguments, and the complete mounted body before the first assistant entry in
|
|
174
|
-
the completed session. It retains only acceptance and
|
|
174
|
+
the completed session. It retains only acceptance and the dispatched skill name.
|
|
175
175
|
Missing or ambiguous session evidence yields a partial observation; later Skill
|
|
176
176
|
calls cannot repair an unverified explicit repository dispatch.
|
|
177
177
|
|
|
@@ -224,6 +224,12 @@ grader error, or unavailable required evidence.
|
|
|
224
224
|
Host adapters can supply additional namespaced observations. The engine bounds
|
|
225
225
|
and retains those records, then passes them to `evaluate`. Required observation
|
|
226
226
|
IDs must be present with `complete` evidence before a task can pass.
|
|
227
|
+
The bundled adapters expose [skill activation evidence](native-hosts.md#skill-activation-evidence)
|
|
228
|
+
as `sevro.host.skill-activations`, with separate methods for Codex tool-result
|
|
229
|
+
body exposure and successful Claude `Skill` calls.
|
|
230
|
+
This replaces the retired Codex skill-read and Claude nested-skill observations.
|
|
231
|
+
Explicit invocation receipts remain dispatch evidence and do not contribute to
|
|
232
|
+
`observedSkills`; extensions must migrate activation checks to the common ID.
|
|
227
233
|
Host adapters can also supply bounded evidence artifacts. The engine persists
|
|
228
234
|
their bytes outside the candidate workspace before `evaluate`, then passes
|
|
229
235
|
file URLs and digests through the existing artifact references. A required host
|
package/docs/guide-evaluation.md
CHANGED
|
@@ -73,6 +73,10 @@ Automatic checks remain signals, not a complete truth judgment. Acceptance
|
|
|
73
73
|
requires inspecting material claims against the sources actually read.
|
|
74
74
|
File listings do not count as source inspection; follow-ups require fresh
|
|
75
75
|
content. Explicit invocation requires Sevro's complete host dispatch receipt.
|
|
76
|
+
Automatic selection requires the complete `sevro.host.skill-activations`
|
|
77
|
+
observation on both hosts. Codex uses passive tool-result hooks; Claude uses
|
|
78
|
+
successful native `Skill` calls. The evaluator does not fall back to command
|
|
79
|
+
patterns, direct Claude file reads, or retired skill-read observations.
|
|
76
80
|
See the [guide contract](specs/repository-guide.md).
|
|
77
81
|
|
|
78
82
|
The [earlier verification record](guide-verification.json) describes historical
|
package/docs/installing.md
CHANGED
|
@@ -23,10 +23,10 @@ Local installation records the selected version in the project lockfile.
|
|
|
23
23
|
|
|
24
24
|
## Current release
|
|
25
25
|
|
|
26
|
-
The current release is `0.1.0-rc.
|
|
26
|
+
The current release is `0.1.0-rc.15`. Install this exact version:
|
|
27
27
|
|
|
28
28
|
```sh
|
|
29
|
-
bun add --exact @bjoernrochel/sevro@0.1.0-rc.
|
|
29
|
+
bun add --exact @bjoernrochel/sevro@0.1.0-rc.15
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
<!-- sevro-current-release:end -->
|
package/docs/native-hosts.md
CHANGED
|
@@ -48,6 +48,61 @@ plugin hooks and reuses declared host tools with private caches. These changes
|
|
|
48
48
|
require `rc.3` or later. The CLI reference covers credential scrubbing
|
|
49
49
|
and follow-ups.
|
|
50
50
|
|
|
51
|
+
## Skill activation evidence
|
|
52
|
+
|
|
53
|
+
Both bundled adapters advertise `sevro.host.skill-activations`. Its observation
|
|
54
|
+
retains qualified `observedSkills`, bounded call identities, and explicit
|
|
55
|
+
`complete` or `partial` collection status. This is the sole activation collector
|
|
56
|
+
for each host. It replaces Codex's command-pattern and native output-recovery
|
|
57
|
+
detectors and Claude's separate nested-skill collector. Consumers must require
|
|
58
|
+
this observation; the retired observation IDs are no longer emitted.
|
|
59
|
+
|
|
60
|
+
Codex installs passive runner-owned hooks in its private home for each request.
|
|
61
|
+
The `tool_result_body_match` method compares returned tool content against a
|
|
62
|
+
snapshot of prepared repository, plugin and private user skill bodies. It does
|
|
63
|
+
not parse shell commands, so variables, pipelines, Python and nested Code Mode
|
|
64
|
+
calls use the same collector. Pre/post call pairs and session start/stop markers
|
|
65
|
+
establish collection health. Unselected plugin handlers are removed before
|
|
66
|
+
hooks are enabled, and fixture-local Codex configuration remains untrusted.
|
|
67
|
+
No global hook installation or root privileges are required.
|
|
68
|
+
|
|
69
|
+
This Codex receipt proves exact body exposure to the tool caller. Code Mode can
|
|
70
|
+
discard that output before returning to the model. An identical ordinary
|
|
71
|
+
document also matches; the receipt does not prove its source file was opened.
|
|
72
|
+
Identical bodies registered under different names are reported as
|
|
73
|
+
`ambiguousSkills`, with partial evidence. Exact pages can establish a body match
|
|
74
|
+
across tool results from the same session and turn. Coverage is tied to the
|
|
75
|
+
prepared body digest; overlapping or repeated pages cannot fill a missing range.
|
|
76
|
+
The call that completes coverage receives the activation receipt. Fragments
|
|
77
|
+
from different sessions or turns never combine.
|
|
78
|
+
|
|
79
|
+
Page matching uses complete nonblank lines as anchors and exact fragments of
|
|
80
|
+
at least 32 string code units. Fragments with multiple possible positions in a
|
|
81
|
+
body cannot establish coverage. Recognizable unfinished fragments appear in
|
|
82
|
+
`partialSkills` and keep the observation partial; a later complete match resolves
|
|
83
|
+
them. Comparison and anchor budgets also fail closed. Private receipts retain
|
|
84
|
+
body offsets rather than tool text; the public observation omits those offsets.
|
|
85
|
+
Shorter, transformed or unanchored fragments remain outside this matching scope.
|
|
86
|
+
Newly created or changed skill bodies are outside the prepared snapshot. A
|
|
87
|
+
complete empty observation is limited to this scope and the supported local
|
|
88
|
+
tool hook paths.
|
|
89
|
+
|
|
90
|
+
Claude's `skill_tool_result` method observes its existing structured `Skill`
|
|
91
|
+
stream without enabling additional command hooks. It pairs each call with a
|
|
92
|
+
successful result under the same session, parent actor and tool-use ID.
|
|
93
|
+
Failed calls remain failed; unmatched attempts and malformed or conflicting
|
|
94
|
+
records remain partial. Native explicit command expansion can load a skill
|
|
95
|
+
without a `Skill` call. Explicit-dispatch receipts describe dispatch only;
|
|
96
|
+
they do not add entries to the activation observation. A direct Claude `Read`
|
|
97
|
+
also falls outside the `Skill` seam and is not inferred from its file path.
|
|
98
|
+
|
|
99
|
+
The collectors retain no tool arguments, outputs or skill bodies in the new
|
|
100
|
+
observation. Codex catalogs at most 128 skills, each at most 1 MiB, with an
|
|
101
|
+
8 MiB total body budget. Call receipts are bounded; missing collection,
|
|
102
|
+
unfinished calls, catalog or receipt limits, and failed execution remain
|
|
103
|
+
partial. Required evidence must not interpret these states as proof that a
|
|
104
|
+
skill was unused.
|
|
105
|
+
|
|
51
106
|
## Keep sources and credentials separate
|
|
52
107
|
|
|
53
108
|
Declare source/private roots as documented. Never put credentials in prompts,
|
package/docs/runtime-v1.md
CHANGED
|
@@ -428,9 +428,11 @@ host request's private native state. Curated hook read grants validate the
|
|
|
428
428
|
canonical projection layout, ownership, private state permissions, and binding
|
|
429
429
|
to that request. Only the selected plugin subtree is readable; it stays
|
|
430
430
|
read-only, and credentials, private siblings, and peer requests remain denied.
|
|
431
|
-
Ordinary filesystem declarations cannot select this exception. Codex
|
|
432
|
-
|
|
433
|
-
|
|
431
|
+
Ordinary filesystem declarations cannot select this exception. Codex validates
|
|
432
|
+
and filters installed plugins before enabling its runner-owned skill collector,
|
|
433
|
+
including requests without a runtime policy. Plugin command handlers still
|
|
434
|
+
require explicit hook authority. Projections are removed with their owning host
|
|
435
|
+
request.
|
|
434
436
|
|
|
435
437
|
Auxiliary `.mcp.json` and `.lsp.json` files and plugin mods are unsupported under
|
|
436
438
|
hook authority. Manifest MCP/LSP declarations are removed from curated mounts.
|
|
@@ -29,7 +29,11 @@ order; `--fast` runs only `static` without installing dependencies.
|
|
|
29
29
|
|
|
30
30
|
Use the same prerequisites as the corresponding complete-gate checks. A single
|
|
31
31
|
phase is partial evidence; full acceptance requires all four on the same candidate.
|
|
32
|
-
|
|
32
|
+
CI assigns every test file to exactly one of four duration-balanced shards, then
|
|
33
|
+
enforces both coverage thresholds on their combined counters over the complete
|
|
34
|
+
production inventory. Local `--phase tests` and the complete command still run
|
|
35
|
+
the suite in one process. See the [shard commands](../scripts/coverage/README.md#test-shards)
|
|
36
|
+
for reproducing the CI split.
|
|
33
37
|
|
|
34
38
|
## Source dispositions
|
|
35
39
|
|
|
@@ -163,6 +167,15 @@ summary, process ledger, Bun test log, unmodified source snapshot, LCOV and HTML
|
|
|
163
167
|
production files, branch exclusions, and exact projected counts.
|
|
164
168
|
`coverage-raw.json` and the raw summary preserve all validated counters before
|
|
165
169
|
platform projection.
|
|
170
|
+
CI also retains `.quality/shards/<index>/` with each shard's source snapshot,
|
|
171
|
+
process ledger, test log, raw and portable counters, and manifest. Process IDs
|
|
172
|
+
remain scoped to their original shard. Aggregation verifies candidate contents,
|
|
173
|
+
workflow run, platform, architecture, Bun version, instrumentation
|
|
174
|
+
layout, and exact test assignments before combining counters with normalized
|
|
175
|
+
source paths. Missing, repeated, incomplete, or mismatched shards fail the gate.
|
|
176
|
+
Failed-job reruns can reuse successful shards from earlier attempts of the same
|
|
177
|
+
workflow run; manifests retain each attempt, and rerun shards replace only their
|
|
178
|
+
own artifacts. Another candidate or workflow run cannot supply missing evidence.
|
|
166
179
|
`--coverage [Bun test arguments]` is a diagnostic measurement; it still enforces
|
|
167
180
|
both thresholds and writes `.quality/coverage-targeted`, keeping it distinct from
|
|
168
181
|
the full-suite evidence. Sparse runs are expected to fail the full denominator.
|
|
@@ -188,20 +201,24 @@ Ubuntu installs `bubblewrap`, `ripgrep`, and `socat` and runs the same native is
|
|
|
188
201
|
local macOS runs retain `sandbox-exec`. Windows remains unverified.
|
|
189
202
|
|
|
190
203
|
Ordinary changes and explicit workflow dispatches run the four canonical phases
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
`
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
204
|
+
concurrently. The tests phase consists of four shard jobs followed by a combined
|
|
205
|
+
coverage job. Each job installs locked dependencies; native host tools are
|
|
206
|
+
installed for `docs` and the test shards, and UV and mise for `docs`. Each shard
|
|
207
|
+
retains its evidence, and the combined `tests` job uploads `.quality` even when
|
|
208
|
+
coverage fails. Shards do not cancel their siblings on failure. A new commit to
|
|
209
|
+
the same pull request cancels its superseded workflow run; main pushes and manual
|
|
210
|
+
runs have independent concurrency groups. The stable aggregate `TypeScript quality`
|
|
211
|
+
status requires every phase and all four shards to succeed. A missing, cancelled,
|
|
212
|
+
skipped or failed phase or shard cannot turn that status green.
|
|
197
213
|
|
|
198
214
|
Release metadata changes follow [the release workflow](releases.md) and reuse a
|
|
199
215
|
successful full gate from an ancestor on `main`, with identical code,
|
|
200
216
|
dependencies, schemas, tests, and other quality inputs. The selector permits
|
|
201
217
|
only version/distribution-tag changes and its exact release-file inventory.
|
|
202
|
-
It verifies each original phase job and its gate step,
|
|
203
|
-
|
|
204
|
-
|
|
218
|
+
It verifies each original phase job and its gate step, including every shard and
|
|
219
|
+
the combined tests job for sharded runs, or a historical complete gate job and
|
|
220
|
+
step. A partial matrix or reused release run cannot serve as fresh test evidence.
|
|
221
|
+
Duplicate phase or shard jobs are ambiguous and refused. Release CI
|
|
205
222
|
runs `check:typescript --fast`, records the tested source and CI run URL, and
|
|
206
223
|
skips all four full-gate phase jobs. The aggregate
|
|
207
224
|
status requires both verified prior evidence and successful static checks;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bjoernrochel/sevro",
|
|
3
|
-
"version": "0.1.0-rc.
|
|
3
|
+
"version": "0.1.0-rc.15",
|
|
4
4
|
"license": "SEE LICENSE IN LICENSE",
|
|
5
5
|
"description": "A standalone evaluation runner for agentic coding capabilities",
|
|
6
6
|
"type": "module",
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"eval:guide": "bun scripts/eval-guide.ts",
|
|
45
45
|
"release:prepare": "bun scripts/prepare-release.ts",
|
|
46
46
|
"typecheck": "tsc --noEmit",
|
|
47
|
-
"format:check": "prettier --check README.md CONTRIBUTING.md AGENTS.md CLAUDE.md 'docs/**/*.md' '.agents/**/*.md' '.agents/**/*.json' '.claude/**/*.md' 'scripts/**/*.md' examples/**/*.json examples/**/*.ts schemas/*.json src/**/*.ts scripts/**/*.ts scripts/**/*.cjs tests/**/*.ts eslint.config.mjs package.json tsconfig.json typescript-sources.json .github/workflows/*.yml"
|
|
47
|
+
"format:check": "prettier --check README.md CONTRIBUTING.md AGENTS.md CLAUDE.md 'docs/**/*.md' '.agents/**/*.md' '.agents/**/*.json' '.claude/**/*.md' 'scripts/**/*.md' 'scripts/**/*.json' examples/**/*.json examples/**/*.ts schemas/*.json src/**/*.ts scripts/**/*.ts scripts/**/*.cjs tests/**/*.ts eslint.config.mjs package.json tsconfig.json typescript-sources.json .github/workflows/*.yml"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"ajv": "8.20.0",
|
|
@@ -264,14 +264,6 @@ export async function claudeRepositoryInvocationObservation(input: {
|
|
|
264
264
|
} catch {
|
|
265
265
|
/* Unavailable or malformed native evidence stays partial. */
|
|
266
266
|
}
|
|
267
|
-
const observed = input.tools.data.calls
|
|
268
|
-
.filter((call) => call.name === "Skill")
|
|
269
|
-
.map((call) => call.skill);
|
|
270
|
-
const observedSkills = repositoryObservedSkills(
|
|
271
|
-
receipt,
|
|
272
|
-
input.invocation.skillName,
|
|
273
|
-
observed,
|
|
274
|
-
);
|
|
275
267
|
return {
|
|
276
268
|
id: "sevro.claude.repository-invocation" as const,
|
|
277
269
|
completeness:
|
|
@@ -282,25 +274,10 @@ export async function claudeRepositoryInvocationObservation(input: {
|
|
|
282
274
|
method: "native_repository_command",
|
|
283
275
|
...receipt,
|
|
284
276
|
skill: input.invocation.skillName,
|
|
285
|
-
primarySkill: observedSkills[0] ?? null,
|
|
286
|
-
observedSkills,
|
|
287
277
|
},
|
|
288
278
|
};
|
|
289
279
|
}
|
|
290
280
|
|
|
291
|
-
function repositoryObservedSkills(
|
|
292
|
-
receipt: Receipt,
|
|
293
|
-
skillName: string,
|
|
294
|
-
observed: string[],
|
|
295
|
-
): string[] {
|
|
296
|
-
return [
|
|
297
|
-
...new Set([
|
|
298
|
-
...(receipt.accepted === true ? [skillName] : []),
|
|
299
|
-
...observed,
|
|
300
|
-
]),
|
|
301
|
-
];
|
|
302
|
-
}
|
|
303
|
-
|
|
304
281
|
function repositorySessionId(stream: string): string | null {
|
|
305
282
|
const sessions = [
|
|
306
283
|
...new Set(
|