@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/CONTRIBUTING.md +22 -6
  2. package/README.md +5 -1
  3. package/docs/README.md +19 -15
  4. package/docs/architecture.md +138 -17
  5. package/docs/assets/README.md +5 -4
  6. package/docs/builtin-graders-v1.md +9 -2
  7. package/docs/codex-host.md +155 -57
  8. package/docs/contracts.md +2 -1
  9. package/docs/creating-cases.md +9 -2
  10. package/docs/development-cli.md +187 -19
  11. package/docs/documentation-quality.md +12 -2
  12. package/docs/evaluation-workflow.md +110 -0
  13. package/docs/extension-protocol-v1.md +26 -3
  14. package/docs/getting-started.md +23 -2
  15. package/docs/guide-evaluation.md +33 -1
  16. package/docs/identity-v1.md +25 -2
  17. package/docs/installing.md +11 -11
  18. package/docs/native-controls.md +76 -5
  19. package/docs/native-hosts.md +196 -7
  20. package/docs/reading-results.md +102 -0
  21. package/docs/recipes.md +489 -0
  22. package/docs/releases.md +31 -12
  23. package/docs/report-v1.md +29 -1
  24. package/docs/results-v1.md +122 -4
  25. package/docs/run-settings-v1.md +147 -0
  26. package/docs/runtime-v1.md +255 -22
  27. package/docs/troubleshooting.md +12 -12
  28. package/docs/typescript-quality.md +89 -26
  29. package/examples/recipes/candidate.ts +26 -0
  30. package/examples/recipes/cases/coding-change.json +21 -0
  31. package/examples/recipes/cases/host-tools.json +21 -0
  32. package/examples/recipes/cases/mise.json +21 -0
  33. package/examples/recipes/cases/node.json +21 -0
  34. package/examples/recipes/cases/python-uv.json +21 -0
  35. package/examples/recipes/cases/semantic.json +17 -0
  36. package/examples/recipes/coding-change/check.ts +23 -0
  37. package/examples/recipes/extension-artifacts.ts +65 -0
  38. package/examples/recipes/extension-methods.ts +174 -0
  39. package/examples/recipes/extension.ts +37 -0
  40. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  41. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  42. package/examples/recipes/repository-skill/README.md +3 -0
  43. package/examples/recipes/run-inputs.ts +188 -0
  44. package/examples/recipes/run-options.ts +131 -0
  45. package/examples/recipes/run.ts +63 -0
  46. package/examples/recipes/runtime/empty.json +1 -0
  47. package/examples/recipes/runtime/goals.json +1 -0
  48. package/examples/recipes/runtime/mise.json +6 -0
  49. package/examples/recipes/runtime/path.json +1 -0
  50. package/examples/recipes/runtime/plugins.json +4 -0
  51. package/examples/recipes/runtime/python-uv.json +10 -0
  52. package/examples/recipes/semantic-judge.ts +29 -0
  53. package/examples/templates/coding/adapter.ts +17 -0
  54. package/examples/templates/coding/case.json +15 -0
  55. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  56. package/examples/templates/repository-skill/extension-data.ts +64 -0
  57. package/examples/templates/repository-skill/extension.ts +31 -0
  58. package/examples/templates/response/adapter.ts +12 -0
  59. package/examples/templates/response/case.json +13 -0
  60. package/package.json +11 -5
  61. package/schemas/report-v1.schema.json +9 -0
  62. package/schemas/run-evidence-v1.schema.json +2 -1
  63. package/schemas/runtime-v1.schema.json +43 -7
  64. package/src/advisory.ts +2 -2
  65. package/src/candidate-artifacts.ts +42 -11
  66. package/src/candidate-codex-state.ts +185 -0
  67. package/src/case-validation.ts +73 -50
  68. package/src/cli/cancellation.ts +20 -0
  69. package/src/cli/command.ts +87 -0
  70. package/src/cli/completion.ts +15 -0
  71. package/src/cli/demo.ts +60 -0
  72. package/src/cli/evaluation.ts +55 -0
  73. package/src/cli/extension.ts +79 -0
  74. package/src/cli/help.ts +198 -0
  75. package/src/cli/hosts.ts +55 -0
  76. package/src/cli/init-settings.ts +30 -0
  77. package/src/cli/init.ts +111 -0
  78. package/src/{cli-invocation.ts → cli/invocation.ts} +214 -218
  79. package/src/cli/loading.ts +151 -0
  80. package/src/cli/preparation.ts +154 -0
  81. package/src/cli/presentation.ts +93 -0
  82. package/src/cli/progress.ts +6 -0
  83. package/src/cli/run-manifest.ts +128 -0
  84. package/src/cli/run-options.ts +403 -0
  85. package/src/cli/run-overrides.ts +69 -0
  86. package/src/cli/settings.ts +154 -0
  87. package/src/cli/summary.ts +43 -0
  88. package/src/cli.ts +13 -591
  89. package/src/contract-types.ts +94 -0
  90. package/src/engine.ts +9 -5
  91. package/src/evaluation/advisory.ts +150 -0
  92. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  93. package/src/evaluation/collection.ts +88 -0
  94. package/src/evaluation/context.ts +10 -0
  95. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  96. package/src/evaluation/grading.ts +235 -0
  97. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  98. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  99. package/src/evaluation/progress.ts +19 -0
  100. package/src/evaluation/resources.ts +241 -0
  101. package/src/evaluation/run.ts +206 -0
  102. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
  103. package/src/evaluation/trial.ts +800 -0
  104. package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
  105. package/src/evidence/host-results.ts +85 -0
  106. package/src/evidence/retained-artifacts.ts +18 -0
  107. package/src/evidence/trial-writer.ts +203 -0
  108. package/src/evidence/usage.ts +17 -0
  109. package/src/extension-client.ts +34 -96
  110. package/src/extension-session.ts +24 -48
  111. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
  112. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  113. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  114. package/src/fixtures/create.ts +183 -0
  115. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  116. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  117. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  118. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  119. package/src/{fixture-setup.ts → fixtures/setup.ts} +33 -51
  120. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  121. package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
  122. package/src/generated/cli-result.d.cts +2 -1
  123. package/src/generated/extension.d.cts +2 -1
  124. package/src/generated/report.cjs +1 -1
  125. package/src/generated/report.d.cts +2 -1
  126. package/src/generated/run-evidence.cjs +1 -1
  127. package/src/generated/run-evidence.d.cts +2 -1
  128. package/src/generated/runtime.cjs +1 -1
  129. package/src/generated/runtime.d.cts +2 -1
  130. package/src/git.ts +93 -0
  131. package/src/graders/git-head.ts +29 -84
  132. package/src/graders/shell.ts +50 -68
  133. package/src/host-diagnostic-redaction.ts +52 -0
  134. package/src/host-diagnostics.ts +331 -0
  135. package/src/hosts/claude-continuation.ts +57 -40
  136. package/src/hosts/claude-credential.ts +160 -63
  137. package/src/hosts/claude-events.ts +34 -18
  138. package/src/hosts/claude-keychain.ts +78 -21
  139. package/src/hosts/claude-login-recovery.ts +85 -0
  140. package/src/hosts/claude-login-refresh.ts +178 -0
  141. package/src/hosts/claude-login-store.ts +162 -0
  142. package/src/hosts/claude-login.ts +107 -0
  143. package/src/hosts/claude-native-goal.ts +15 -12
  144. package/src/hosts/claude-process.ts +10 -50
  145. package/src/hosts/claude-refresh-lock.ts +81 -0
  146. package/src/hosts/claude-repository-invocation.ts +9 -34
  147. package/src/hosts/claude-settings.ts +46 -9
  148. package/src/hosts/claude-skill-activations.ts +169 -0
  149. package/src/hosts/claude-state.ts +28 -15
  150. package/src/hosts/claude-tool-calls.ts +13 -10
  151. package/src/hosts/claude.ts +121 -81
  152. package/src/hosts/codex-app-server.ts +78 -52
  153. package/src/hosts/codex-configuration.ts +5 -10
  154. package/src/hosts/codex-exec-controls.ts +516 -0
  155. package/src/hosts/codex-marketplace.ts +12 -10
  156. package/src/hosts/codex-native-calls.ts +15 -122
  157. package/src/hosts/codex-process.ts +10 -183
  158. package/src/hosts/codex-profile.ts +36 -21
  159. package/src/hosts/codex-session-files.ts +1 -9
  160. package/src/hosts/codex-state.ts +134 -18
  161. package/src/hosts/codex.ts +210 -183
  162. package/src/hosts/isolation-common.ts +8 -11
  163. package/src/hosts/isolation-roots.ts +2 -2
  164. package/src/hosts/linux-sandbox.ts +15 -2
  165. package/src/hosts/mac-sandbox.ts +87 -18
  166. package/src/hosts/native-controls.ts +72 -22
  167. package/src/hosts/native-request.ts +55 -0
  168. package/src/hosts/process-lifecycle.ts +216 -48
  169. package/src/hosts/protected-worktrees.ts +1 -1
  170. package/src/hosts/skill-hook-catalog.ts +131 -0
  171. package/src/hosts/skill-hook-handler.ts +374 -0
  172. package/src/hosts/skill-hooks.ts +577 -0
  173. package/src/instrumentation.ts +7 -8
  174. package/src/native-transcript-state.ts +22 -3
  175. package/src/native-transcript-view.ts +52 -13
  176. package/src/native-transcripts.ts +26 -11
  177. package/src/paths.ts +22 -0
  178. package/src/process-lifecycle.ts +150 -0
  179. package/src/provenance.ts +27 -7
  180. package/src/prune.ts +300 -0
  181. package/src/report.ts +63 -36
  182. package/src/resolved-case.ts +1 -1
  183. package/src/results-lock.ts +42 -0
  184. package/src/results.ts +14 -18
  185. package/src/{runtime-config.ts → runtime/config.ts} +160 -46
  186. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  187. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  188. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  189. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +1 -1
  190. package/src/{runtime-hooks.ts → runtime/hooks.ts} +32 -22
  191. package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
  192. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  193. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  194. package/src/runtime/shared-cache.ts +130 -0
  195. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  196. package/src/schema-types.ts +73 -161
  197. package/src/shell.ts +4 -0
  198. package/src/transcript-retention.ts +146 -0
  199. package/src/value-guards.ts +22 -0
  200. package/docs/assets/sevro-logo.png +0 -0
  201. package/docs/assets/sevro-symbol.svg +0 -1
  202. package/docs/assets/sevro-wordmark.svg +0 -1
  203. package/docs/cache-ownership-validation.md +0 -49
  204. package/docs/claude-continuation-validation.md +0 -47
  205. package/docs/concurrency-validation.md +0 -86
  206. package/docs/configuration-root-validation.md +0 -70
  207. package/docs/engine-slice.md +0 -114
  208. package/docs/evidence/runtime-startup-30/README.md +0 -185
  209. package/docs/evidence/runtime-startup-30/after.json +0 -319
  210. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  211. package/docs/evidence/runtime-startup-30/before.json +0 -403
  212. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  213. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  214. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  215. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  216. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  217. package/docs/evidence.md +0 -17
  218. package/docs/fixture-cleanup-validation.md +0 -73
  219. package/docs/guide-verification.json +0 -239
  220. package/src/evaluation-context.ts +0 -259
  221. package/src/evaluation-fixture.ts +0 -302
  222. package/src/evaluation-run.ts +0 -200
  223. package/src/evaluation-trial.ts +0 -1016
  224. package/src/hosts/claude-nested-skills.ts +0 -272
  225. package/src/hosts/codex-command-output.ts +0 -510
  226. package/src/hosts/codex-executor-command.ts +0 -291
  227. package/src/hosts/codex-skill-reads.ts +0 -783
  228. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
package/docs/evidence/runtime-startup-30/after.json DELETED
@@ -1,319 +0,0 @@
1
- {
2
- "format": "sevro.runtime-startup-benchmark.v1",
3
- "sourceRoot": "/Users/bjro/Sources/sevro/.worktrees/perf/30-runtime-startup-overhead",
4
- "platform": "darwin",
5
- "arch": "arm64",
6
- "bun": "1.3.13",
7
- "codex": "codex-cli 0.162.1",
8
- "case": "synthetic-plugin-response",
9
- "model": "synthetic-candidate / synthetic-semantic",
10
- "effort": "low",
11
- "seed": {
12
- "entries": 2049,
13
- "bytes": 33554432
14
- },
15
- "repetitions": 3,
16
- "conditions": "Fresh role homes each invocation; first verified snapshot cold, later repetitions warm; OS filesystem caches uncontrolled; no live API/model calls",
17
- "rows": [
18
- {
19
- "repetition": 0,
20
- "role": "candidate",
21
- "phase": "configuration",
22
- "milliseconds": 298.645833,
23
- "status": "completed"
24
- },
25
- {
26
- "repetition": 1,
27
- "role": "candidate",
28
- "phase": "runtime.candidate",
29
- "milliseconds": 721.155541,
30
- "status": "completed"
31
- },
32
- {
33
- "repetition": 1,
34
- "role": "candidate",
35
- "phase": "plugin-command",
36
- "milliseconds": 113.721,
37
- "status": "completed"
38
- },
39
- {
40
- "repetition": 1,
41
- "role": "candidate",
42
- "phase": "plugin-command",
43
- "milliseconds": 70.76491699999997,
44
- "status": "completed"
45
- },
46
- {
47
- "repetition": 1,
48
- "role": "candidate",
49
- "phase": "installation-assets",
50
- "files": [
51
- "cache"
52
- ],
53
- "configurationContainsPrivateHome": true,
54
- "configurationContainsWorkspace": true
55
- },
56
- {
57
- "repetition": 1,
58
- "role": "candidate",
59
- "phase": "plugin-install",
60
- "milliseconds": 187.57354199999986,
61
- "status": "completed"
62
- },
63
- {
64
- "repetition": 1,
65
- "role": "candidate",
66
- "phase": "sandbox-probe",
67
- "milliseconds": 5489.6383749999995,
68
- "status": "completed"
69
- },
70
- {
71
- "repetition": 1,
72
- "role": "candidate",
73
- "phase": "participant-execution",
74
- "milliseconds": 27.323707999999897,
75
- "status": "completed"
76
- },
77
- {
78
- "repetition": 1,
79
- "role": "candidate",
80
- "phase": "candidate-total",
81
- "milliseconds": 6659.420667,
82
- "status": "completed"
83
- },
84
- {
85
- "repetition": 1,
86
- "role": "semantic",
87
- "phase": "runtime.semantic",
88
- "milliseconds": 0.3450830000001588,
89
- "status": "completed"
90
- },
91
- {
92
- "repetition": 1,
93
- "role": "semantic",
94
- "phase": "plugin-install",
95
- "milliseconds": 0.005458999999973457,
96
- "status": "completed"
97
- },
98
- {
99
- "repetition": 1,
100
- "role": "semantic",
101
- "phase": "sandbox-probe",
102
- "milliseconds": 5548.128959000001,
103
- "status": "completed"
104
- },
105
- {
106
- "repetition": 1,
107
- "role": "semantic",
108
- "phase": "participant-execution",
109
- "milliseconds": 28.713165999999546,
110
- "status": "completed"
111
- },
112
- {
113
- "repetition": 1,
114
- "role": "semantic",
115
- "phase": "grading",
116
- "milliseconds": 5646.800708000001,
117
- "status": "completed"
118
- },
119
- {
120
- "repetition": 2,
121
- "role": "candidate",
122
- "phase": "runtime.candidate",
123
- "milliseconds": 430.6864170000008,
124
- "status": "completed"
125
- },
126
- {
127
- "repetition": 2,
128
- "role": "candidate",
129
- "phase": "plugin-command",
130
- "milliseconds": 73.67158399999971,
131
- "status": "completed"
132
- },
133
- {
134
- "repetition": 2,
135
- "role": "candidate",
136
- "phase": "plugin-command",
137
- "milliseconds": 65.96320800000103,
138
- "status": "completed"
139
- },
140
- {
141
- "repetition": 2,
142
- "role": "candidate",
143
- "phase": "installation-assets",
144
- "files": [
145
- "cache"
146
- ],
147
- "configurationContainsPrivateHome": true,
148
- "configurationContainsWorkspace": true
149
- },
150
- {
151
- "repetition": 2,
152
- "role": "candidate",
153
- "phase": "plugin-install",
154
- "milliseconds": 141.00866599999972,
155
- "status": "completed"
156
- },
157
- {
158
- "repetition": 2,
159
- "role": "candidate",
160
- "phase": "sandbox-probe",
161
- "milliseconds": 5404.434915999998,
162
- "status": "completed"
163
- },
164
- {
165
- "repetition": 2,
166
- "role": "candidate",
167
- "phase": "participant-execution",
168
- "milliseconds": 26.502958000000945,
169
- "status": "completed"
170
- },
171
- {
172
- "repetition": 2,
173
- "role": "candidate",
174
- "phase": "candidate-total",
175
- "milliseconds": 6199.1395840000005,
176
- "status": "completed"
177
- },
178
- {
179
- "repetition": 2,
180
- "role": "semantic",
181
- "phase": "runtime.semantic",
182
- "milliseconds": 0.314249999999447,
183
- "status": "completed"
184
- },
185
- {
186
- "repetition": 2,
187
- "role": "semantic",
188
- "phase": "plugin-install",
189
- "milliseconds": 0.005916999998589745,
190
- "status": "completed"
191
- },
192
- {
193
- "repetition": 2,
194
- "role": "semantic",
195
- "phase": "sandbox-probe",
196
- "milliseconds": 5359.928749999999,
197
- "status": "completed"
198
- },
199
- {
200
- "repetition": 2,
201
- "role": "semantic",
202
- "phase": "participant-execution",
203
- "milliseconds": 27.60699999999997,
204
- "status": "completed"
205
- },
206
- {
207
- "repetition": 2,
208
- "role": "semantic",
209
- "phase": "grading",
210
- "milliseconds": 5454.279749999998,
211
- "status": "completed"
212
- },
213
- {
214
- "repetition": 3,
215
- "role": "candidate",
216
- "phase": "runtime.candidate",
217
- "milliseconds": 442.62854199999856,
218
- "status": "completed"
219
- },
220
- {
221
- "repetition": 3,
222
- "role": "candidate",
223
- "phase": "plugin-command",
224
- "milliseconds": 72.3325420000001,
225
- "status": "completed"
226
- },
227
- {
228
- "repetition": 3,
229
- "role": "candidate",
230
- "phase": "plugin-command",
231
- "milliseconds": 68.99258300000292,
232
- "status": "completed"
233
- },
234
- {
235
- "repetition": 3,
236
- "role": "candidate",
237
- "phase": "installation-assets",
238
- "files": [
239
- "cache"
240
- ],
241
- "configurationContainsPrivateHome": true,
242
- "configurationContainsWorkspace": true
243
- },
244
- {
245
- "repetition": 3,
246
- "role": "candidate",
247
- "phase": "plugin-install",
248
- "milliseconds": 142.90399999999863,
249
- "status": "completed"
250
- },
251
- {
252
- "repetition": 3,
253
- "role": "candidate",
254
- "phase": "sandbox-probe",
255
- "milliseconds": 5447.905040999998,
256
- "status": "completed"
257
- },
258
- {
259
- "repetition": 3,
260
- "role": "candidate",
261
- "phase": "participant-execution",
262
- "milliseconds": 23.8626669999976,
263
- "status": "completed"
264
- },
265
- {
266
- "repetition": 3,
267
- "role": "candidate",
268
- "phase": "candidate-total",
269
- "milliseconds": 6256.5165830000005,
270
- "status": "completed"
271
- },
272
- {
273
- "repetition": 3,
274
- "role": "semantic",
275
- "phase": "runtime.semantic",
276
- "milliseconds": 0.3507499999977881,
277
- "status": "completed"
278
- },
279
- {
280
- "repetition": 3,
281
- "role": "semantic",
282
- "phase": "plugin-install",
283
- "milliseconds": 0.01341600000159815,
284
- "status": "completed"
285
- },
286
- {
287
- "repetition": 3,
288
- "role": "semantic",
289
- "phase": "sandbox-probe",
290
- "milliseconds": 5365.139791000001,
291
- "status": "completed"
292
- },
293
- {
294
- "repetition": 3,
295
- "role": "semantic",
296
- "phase": "participant-execution",
297
- "milliseconds": 28.228125000001455,
298
- "status": "completed"
299
- },
300
- {
301
- "repetition": 3,
302
- "role": "semantic",
303
- "phase": "grading",
304
- "milliseconds": 5460.212874999997,
305
- "status": "completed"
306
- },
307
- {
308
- "repetition": 4,
309
- "role": "semantic",
310
- "phase": "cow-copy",
311
- "milliseconds": 1.0158329999976559,
312
- "status": "completed"
313
- },
314
- {
315
- "phase": "cow-isolation",
316
- "sourceUnchanged": true
317
- }
318
- ]
319
- }
package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl DELETED
@@ -1,2 +0,0 @@
1
- {"phase":"sandbox-probe","durationMs":10004.7745,"error":"Codex run timed out"}
2
- {"phase":"host-failure","error":"Codex run timed out"}