@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/CONTRIBUTING.md +170 -0
  2. package/LICENSE +63 -136
  3. package/README.md +83 -66
  4. package/docs/README.md +44 -0
  5. package/docs/acknowledgements.md +14 -0
  6. package/docs/architecture.md +146 -0
  7. package/docs/assets/README.md +27 -0
  8. package/docs/assets/sevro-logo.png +0 -0
  9. package/docs/assets/sevro-symbol.svg +1 -0
  10. package/docs/assets/sevro-wordmark.svg +1 -0
  11. package/docs/builtin-graders-v1.md +41 -13
  12. package/docs/codex-host.md +184 -52
  13. package/docs/contracts.md +19 -0
  14. package/docs/creating-cases.md +139 -0
  15. package/docs/development-cli.md +190 -30
  16. package/docs/documentation-quality.md +61 -0
  17. package/docs/evaluation-workflow.md +110 -0
  18. package/docs/evidence/code-mode-controls-71.md +64 -0
  19. package/docs/evidence/code-mode-programs-85.md +64 -0
  20. package/docs/evidence/codex-state-73/README.md +204 -0
  21. package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
  22. package/docs/evidence/codex-state-73/results.json +661 -0
  23. package/docs/evidence/local-http-74.md +104 -0
  24. package/docs/evidence/runtime-startup-30/README.md +185 -0
  25. package/docs/evidence/runtime-startup-30/after.json +319 -0
  26. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
  27. package/docs/evidence/runtime-startup-30/before.json +403 -0
  28. package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
  29. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
  30. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
  31. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
  32. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
  33. package/docs/evidence/skill-reads-70/results.json +823 -0
  34. package/docs/evidence.md +21 -0
  35. package/docs/extension-protocol-v1.md +50 -16
  36. package/docs/fixture-cleanup-validation.md +3 -0
  37. package/docs/getting-started.md +102 -0
  38. package/docs/guide-evaluation.md +89 -0
  39. package/docs/guide-verification.json +239 -0
  40. package/docs/identity-v1.md +35 -1
  41. package/docs/installing.md +88 -0
  42. package/docs/licenses/BUSL-1.1-0.1.0-rc.2.txt +159 -0
  43. package/docs/licensing.md +42 -0
  44. package/docs/native-controls.md +76 -5
  45. package/docs/native-hosts.md +228 -0
  46. package/docs/reading-results.md +141 -0
  47. package/docs/recipes.md +489 -0
  48. package/docs/releases.md +81 -40
  49. package/docs/report-v1.md +29 -1
  50. package/docs/results-v1.md +180 -0
  51. package/docs/run-settings-v1.md +147 -0
  52. package/docs/running-evaluations.md +31 -0
  53. package/docs/runtime-v1.md +614 -0
  54. package/docs/specs/repository-guide.md +58 -0
  55. package/docs/troubleshooting.md +30 -0
  56. package/docs/typescript-quality.md +234 -0
  57. package/docs/usability-study.md +114 -0
  58. package/examples/basic/host.ts +2 -2
  59. package/examples/recipes/candidate.ts +26 -0
  60. package/examples/recipes/cases/coding-change.json +21 -0
  61. package/examples/recipes/cases/host-tools.json +21 -0
  62. package/examples/recipes/cases/mise.json +21 -0
  63. package/examples/recipes/cases/node.json +21 -0
  64. package/examples/recipes/cases/python-uv.json +21 -0
  65. package/examples/recipes/cases/semantic.json +17 -0
  66. package/examples/recipes/coding-change/check.ts +23 -0
  67. package/examples/recipes/extension-artifacts.ts +65 -0
  68. package/examples/recipes/extension-methods.ts +174 -0
  69. package/examples/recipes/extension.ts +37 -0
  70. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  71. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  72. package/examples/recipes/repository-skill/README.md +3 -0
  73. package/examples/recipes/run-inputs.ts +188 -0
  74. package/examples/recipes/run-options.ts +131 -0
  75. package/examples/recipes/run.ts +63 -0
  76. package/examples/recipes/runtime/empty.json +1 -0
  77. package/examples/recipes/runtime/goals.json +1 -0
  78. package/examples/recipes/runtime/mise.json +6 -0
  79. package/examples/recipes/runtime/path.json +1 -0
  80. package/examples/recipes/runtime/plugins.json +4 -0
  81. package/examples/recipes/runtime/python-uv.json +10 -0
  82. package/examples/recipes/semantic-judge.ts +29 -0
  83. package/examples/templates/coding/adapter.ts +17 -0
  84. package/examples/templates/coding/case.json +15 -0
  85. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  86. package/examples/templates/repository-skill/extension-data.ts +64 -0
  87. package/examples/templates/repository-skill/extension.ts +31 -0
  88. package/examples/templates/response/adapter.ts +12 -0
  89. package/examples/templates/response/case.json +13 -0
  90. package/package.json +39 -6
  91. package/schemas/report-v1.schema.json +9 -0
  92. package/schemas/run-evidence-v1.schema.json +2 -1
  93. package/schemas/runtime-v1.schema.json +193 -0
  94. package/src/advisory.ts +10 -5
  95. package/src/candidate-artifacts.ts +208 -0
  96. package/src/candidate-codex-state.ts +185 -0
  97. package/src/case-validation.ts +105 -0
  98. package/src/cli/cancellation.ts +20 -0
  99. package/src/cli/command.ts +81 -0
  100. package/src/cli/completion.ts +15 -0
  101. package/src/cli/demo.ts +59 -0
  102. package/src/cli/evaluation.ts +55 -0
  103. package/src/cli/extension.ts +79 -0
  104. package/src/cli/help.ts +222 -0
  105. package/src/cli/hosts.ts +59 -0
  106. package/src/cli/init-settings.ts +30 -0
  107. package/src/cli/init.ts +111 -0
  108. package/src/cli/invocation.ts +627 -0
  109. package/src/cli/loading.ts +151 -0
  110. package/src/cli/preparation.ts +163 -0
  111. package/src/cli/presentation.ts +91 -0
  112. package/src/cli/progress.ts +6 -0
  113. package/src/cli/run-manifest.ts +133 -0
  114. package/src/cli/run-options.ts +101 -0
  115. package/src/cli/run-overrides.ts +83 -0
  116. package/src/cli/settings.ts +160 -0
  117. package/src/cli/summary.ts +43 -0
  118. package/src/cli.ts +13 -843
  119. package/src/contract-types.ts +94 -0
  120. package/src/engine.ts +19 -2373
  121. package/src/evaluation/advisory.ts +150 -0
  122. package/src/evaluation/check-plan.ts +510 -0
  123. package/src/evaluation/collection.ts +88 -0
  124. package/src/evaluation/context.ts +10 -0
  125. package/src/evaluation/grading-host.ts +39 -0
  126. package/src/evaluation/grading.ts +258 -0
  127. package/src/evaluation/invocation-plan.ts +548 -0
  128. package/src/evaluation/plan.ts +736 -0
  129. package/src/evaluation/progress.ts +19 -0
  130. package/src/evaluation/resources.ts +241 -0
  131. package/src/evaluation/run.ts +206 -0
  132. package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +3 -1
  133. package/src/evaluation/semantic.ts +312 -0
  134. package/src/evaluation/trial.ts +866 -0
  135. package/src/evaluation/types.ts +207 -0
  136. package/src/evidence/host-results.ts +92 -0
  137. package/src/evidence/retained-artifacts.ts +18 -0
  138. package/src/evidence/trial-writer.ts +203 -0
  139. package/src/evidence/usage.ts +17 -0
  140. package/src/extension-client.ts +228 -148
  141. package/src/extension-session.ts +326 -195
  142. package/src/fixtures/advisory.ts +338 -0
  143. package/src/fixtures/bin.ts +57 -0
  144. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  145. package/src/fixtures/create.ts +187 -0
  146. package/src/fixtures/generated.ts +300 -0
  147. package/src/fixtures/git-hooks.ts +24 -0
  148. package/src/fixtures/preparation.ts +270 -0
  149. package/src/{repository-fixture.ts → fixtures/repository.ts} +207 -85
  150. package/src/fixtures/setup.ts +224 -0
  151. package/src/fixtures/tools.ts +88 -0
  152. package/src/fixtures/workspace-state.ts +212 -0
  153. package/src/generated/cli-result.cjs +2 -0
  154. package/src/generated/cli-result.d.cts +3 -0
  155. package/src/generated/extension.cjs +2 -0
  156. package/src/generated/extension.d.cts +3 -0
  157. package/src/generated/report.cjs +2 -0
  158. package/src/generated/report.d.cts +3 -0
  159. package/src/generated/run-evidence.cjs +2 -0
  160. package/src/generated/run-evidence.d.cts +3 -0
  161. package/src/generated/runtime.cjs +2 -0
  162. package/src/generated/runtime.d.cts +3 -0
  163. package/src/graders/artifact.ts +56 -23
  164. package/src/graders/git-head.ts +79 -75
  165. package/src/graders/output.ts +222 -140
  166. package/src/graders/semantic.ts +102 -43
  167. package/src/graders/shell.ts +379 -207
  168. package/src/host-diagnostic-redaction.ts +52 -0
  169. package/src/host-diagnostics.ts +319 -0
  170. package/src/hosts/claude-continuation.ts +63 -18
  171. package/src/hosts/claude-credential.ts +43 -62
  172. package/src/hosts/claude-events.ts +86 -51
  173. package/src/hosts/claude-keychain.ts +59 -0
  174. package/src/hosts/claude-native-goal.ts +57 -33
  175. package/src/hosts/claude-process.ts +111 -0
  176. package/src/hosts/claude-repository-invocation.ts +208 -117
  177. package/src/hosts/claude-settings.ts +240 -21
  178. package/src/hosts/claude-skill-activations.ts +166 -0
  179. package/src/hosts/claude-state.ts +403 -0
  180. package/src/hosts/claude-tool-calls.ts +171 -106
  181. package/src/hosts/claude.ts +561 -451
  182. package/src/hosts/codex-app-server.ts +326 -178
  183. package/src/hosts/codex-configuration.ts +44 -18
  184. package/src/hosts/codex-events.ts +97 -60
  185. package/src/hosts/codex-exec-controls.ts +516 -0
  186. package/src/hosts/codex-invocation.ts +70 -0
  187. package/src/hosts/codex-marketplace.ts +283 -0
  188. package/src/hosts/codex-native-calls.ts +546 -500
  189. package/src/hosts/codex-process.ts +202 -0
  190. package/src/hosts/codex-profile.ts +257 -28
  191. package/src/hosts/codex-session-files.ts +106 -0
  192. package/src/hosts/codex-state.ts +492 -0
  193. package/src/hosts/codex.ts +783 -832
  194. package/src/hosts/isolation-common.ts +94 -0
  195. package/src/hosts/isolation-roots.ts +54 -27
  196. package/src/hosts/isolation.ts +10 -0
  197. package/src/hosts/linux-sandbox.ts +159 -0
  198. package/src/hosts/mac-sandbox.ts +187 -61
  199. package/src/hosts/native-controls.ts +152 -62
  200. package/src/hosts/process-lifecycle.ts +37 -0
  201. package/src/hosts/protected-worktrees.ts +51 -19
  202. package/src/hosts/skill-hook-catalog.ts +131 -0
  203. package/src/hosts/skill-hook-handler.ts +374 -0
  204. package/src/hosts/skill-hooks.ts +580 -0
  205. package/src/hosts/workspace-fingerprint.ts +55 -35
  206. package/src/identity.ts +102 -48
  207. package/src/instrumentation.ts +109 -48
  208. package/src/native-transcript-state.ts +131 -0
  209. package/src/native-transcript-view.ts +90 -0
  210. package/src/native-transcripts.ts +286 -0
  211. package/src/process-lifecycle.ts +150 -0
  212. package/src/provenance.ts +161 -81
  213. package/src/prune.ts +215 -0
  214. package/src/report.ts +172 -140
  215. package/src/resolved-case.ts +33 -0
  216. package/src/results-lock.ts +29 -0
  217. package/src/results.ts +133 -86
  218. package/src/run-owner.ts +154 -63
  219. package/src/runtime/config.ts +456 -0
  220. package/src/runtime/discovery-apple.ts +90 -0
  221. package/src/runtime/discovery-providers.ts +63 -0
  222. package/src/runtime/discovery.ts +70 -0
  223. package/src/runtime/hook-processes.ts +97 -0
  224. package/src/runtime/hooks.ts +476 -0
  225. package/src/runtime/paths.ts +313 -0
  226. package/src/runtime/query.ts +85 -0
  227. package/src/runtime/seeds.ts +134 -0
  228. package/src/runtime/shared-cache.ts +143 -0
  229. package/src/runtime/state.ts +143 -0
  230. package/src/schema-types.ts +198 -0
  231. package/src/schema.ts +15 -19
  232. package/src/transcript-retention.ts +146 -0
  233. package/src/value-guards.ts +20 -0
  234. package/src/advisory-fixture.ts +0 -282
  235. package/src/fixture-bin.ts +0 -87
  236. package/src/fixture-setup.ts +0 -163
  237. package/src/generated-fixture.ts +0 -220
  238. package/src/git-hooks.ts +0 -63
  239. package/src/hosts/claude-nested-skills.ts +0 -218
  240. package/src/hosts/codex-command-output.ts +0 -399
  241. package/src/hosts/codex-executor-command.ts +0 -195
  242. package/src/hosts/codex-skill-reads.ts +0 -391
  243. package/src/preparation.ts +0 -211
package/docs/evidence/codex-state-73/results.json ADDED
@@ -0,0 +1,661 @@
1
+ {
2
+ "format": "sevro.codex-state-validation.v1",
3
+ "date": "2026-10-14",
4
+ "platform": "macOS arm64",
5
+ "codexVersion": "0.162.1",
6
+ "entrypoint": "app-server",
7
+ "candidate": {
8
+ "model": "gpt-6.1-sol",
9
+ "effort": "medium"
10
+ },
11
+ "semanticGrader": {
12
+ "model": "gpt-6-luna",
13
+ "effort": "medium"
14
+ },
15
+ "ownerEvaluation": "passive",
16
+ "trialsPerInvocation": 1,
17
+ "jobs": 1,
18
+ "threshold": 1,
19
+ "timeoutMs": 1200000,
20
+ "selectedCapacity": 8,
21
+ "fixtureCapacity": 99,
22
+ "sourceFiles": {
23
+ "src/candidate-artifacts.ts": "72bc3fb458e6614116abba927d51dd50aa96721a95cc653dc765024d249a7817",
24
+ "src/candidate-codex-state.ts": "f588d2aa30f0ee69fa434ff840db439ebbeac7c5cf75530ebcb897e2dfe71bfb",
25
+ "src/hosts/codex-profile.ts": "47dd50cf8b5ae4ee42ced2d1f180f1fa6d6e10708af748661e608ea8a883ef18",
26
+ "src/hosts/codex-state.ts": "b94455b9b1e6b232b25b5598d933734143c1c98ef7b13add394a1f34a98f8e2d",
27
+ "src/hosts/codex.ts": "0c3fe80de64553b0c5ce19f16f14eeae6477b8a2c8e063ae70599a94f39fc3c9",
28
+ "src/runtime-config.ts": "bdc03ca5ee0c34c68c8a26a9855190adb82bc72b73fb51e8fe6f86245bd4e784",
29
+ "src/generated/runtime.cjs": "d3d6d6ab1c22f37b7e8e6eeaa2bbbff068c548b31447f67aad9c445d96fcb791",
30
+ "schemas/runtime-v1.schema.json": "48cc9b45789920987805eb7a2d19805e173bd02497549b73d5e6e622fe733470",
31
+ "tests/runtime-host.test.ts": "f2f032abec53ffd5b7ad889914fa0c096a79257e32c3fd0517c57189a926616b",
32
+ "typescript-sources.json": "e33af2db9e697280e007322ef286a21680a87f1a6ce954ba921d57376943ae66"
33
+ },
34
+ "runs": [
35
+ {
36
+ "invocation": "doctor-01",
37
+ "caseId": "doctor-adaptive-goal-direct-codex",
38
+ "runId": "bf8cd173-2856-4d20-a904-1b5cf646d46e",
39
+ "evaluationIdentityDigest": "5bc9bb40d310f589dfc53f3585c4bd013d107b383955b4d78856162b31f7690d",
40
+ "configurationDigest": "b6f5aaa7574e98ae1a534e6e790ee4c372706b445dfc9a5ccaa8b23e0c327d71",
41
+ "execution": "completed",
42
+ "grading": "completed",
43
+ "task": "failed",
44
+ "exitCode": 1,
45
+ "activation": "passed",
46
+ "checks": [
47
+ {
48
+ "id": "darrow.shell.1",
49
+ "status": "passed"
50
+ },
51
+ {
52
+ "id": "darrow.semantic.1",
53
+ "status": "failed"
54
+ },
55
+ {
56
+ "id": "darrow.semantic.2",
57
+ "status": "failed"
58
+ },
59
+ {
60
+ "id": "darrow.evals.transcript.1",
61
+ "status": "passed"
62
+ }
63
+ ]
64
+ },
65
+ {
66
+ "invocation": "doctor-02",
67
+ "caseId": "doctor-adaptive-goal-direct-codex",
68
+ "runId": "e8df2318-70b3-4976-8750-40e89420e1a8",
69
+ "evaluationIdentityDigest": "88979915ac28d8097cf5c9421340308259d951879b761e021d1c2ea48f7b9156",
70
+ "configurationDigest": "a12a09ab260f227a0748a93015816eb32afa7259169255281a9749b7529a8528",
71
+ "execution": "completed",
72
+ "grading": "completed",
73
+ "task": "failed",
74
+ "exitCode": 1,
75
+ "activation": "passed",
76
+ "checks": [
77
+ {
78
+ "id": "darrow.shell.1",
79
+ "status": "passed"
80
+ },
81
+ {
82
+ "id": "darrow.semantic.1",
83
+ "status": "failed"
84
+ },
85
+ {
86
+ "id": "darrow.semantic.2",
87
+ "status": "passed"
88
+ },
89
+ {
90
+ "id": "darrow.evals.transcript.1",
91
+ "status": "passed"
92
+ }
93
+ ]
94
+ },
95
+ {
96
+ "invocation": "doctor-03",
97
+ "caseId": "doctor-adaptive-goal-direct-codex",
98
+ "runId": "c4ddd30d-db25-4a26-b491-cb26a0192d41",
99
+ "evaluationIdentityDigest": "ecd302a97147f786f8bb24aa12c462ec50047ae95b0b1d40940d7f156e816b51",
100
+ "configurationDigest": "a12a09ab260f227a0748a93015816eb32afa7259169255281a9749b7529a8528",
101
+ "execution": "completed",
102
+ "grading": "completed",
103
+ "task": "passed",
104
+ "exitCode": 0,
105
+ "activation": "passed",
106
+ "checks": [
107
+ {
108
+ "id": "darrow.shell.1",
109
+ "status": "passed"
110
+ },
111
+ {
112
+ "id": "darrow.semantic.1",
113
+ "status": "passed"
114
+ },
115
+ {
116
+ "id": "darrow.semantic.2",
117
+ "status": "passed"
118
+ },
119
+ {
120
+ "id": "darrow.evals.transcript.1",
121
+ "status": "passed"
122
+ }
123
+ ]
124
+ },
125
+ {
126
+ "invocation": "authoring-01",
127
+ "caseId": "create-agent-skill-refuse-undefined-skill",
128
+ "runId": "9f98a5cf-dc52-44c2-9e88-0d212c6ac502",
129
+ "evaluationIdentityDigest": "8b58abb7769297c8321d4198c8deeb0eb50944195c80f3d41d4b18968263e3c1",
130
+ "configurationDigest": "d99821da4925d5d61eb692125741bb68944bd9c1dd63d39fadbfc04b7637f8ee",
131
+ "execution": "completed",
132
+ "grading": "completed",
133
+ "task": "passed",
134
+ "exitCode": 0,
135
+ "activation": "passed",
136
+ "checks": [
137
+ {
138
+ "id": "darrow.shell.1",
139
+ "status": "passed"
140
+ },
141
+ {
142
+ "id": "darrow.shell.2",
143
+ "status": "passed"
144
+ },
145
+ {
146
+ "id": "darrow.shell.3",
147
+ "status": "passed"
148
+ },
149
+ {
150
+ "id": "darrow.semantic.1",
151
+ "status": "passed"
152
+ }
153
+ ]
154
+ },
155
+ {
156
+ "invocation": "doctor-04",
157
+ "sourceSnapshot": "repairSourceFiles",
158
+ "caseId": "doctor-adaptive-goal-direct-codex",
159
+ "runId": "3d5768e5-3df3-4a59-becc-b65c471c1cd9",
160
+ "evaluationIdentityDigest": "46a7761eaeeddc62ebdd3cc979229ef6f7c15b5e02e5afee6da6c7dd63651182",
161
+ "configurationDigest": "a12a09ab260f227a0748a93015816eb32afa7259169255281a9749b7529a8528",
162
+ "execution": "completed",
163
+ "grading": "completed",
164
+ "task": "failed",
165
+ "exitCode": 1,
166
+ "activation": "passed",
167
+ "checks": [
168
+ {
169
+ "id": "darrow.shell.1",
170
+ "status": "passed"
171
+ },
172
+ {
173
+ "id": "darrow.semantic.1",
174
+ "status": "failed"
175
+ },
176
+ {
177
+ "id": "darrow.semantic.2",
178
+ "status": "failed"
179
+ },
180
+ {
181
+ "id": "darrow.evals.transcript.1",
182
+ "status": "passed"
183
+ }
184
+ ]
185
+ },
186
+ {
187
+ "invocation": "authoring-02",
188
+ "sourceSnapshot": "repairSourceFiles",
189
+ "caseId": "create-agent-skill-refuse-undefined-skill",
190
+ "runId": "e37dda8b-18d5-4d55-a9c5-2b9b6122aa99",
191
+ "evaluationIdentityDigest": "11480309c9d1e7ef4015e4eeb5e275bb80bc9dc4d86412b1352fc2542323c564",
192
+ "configurationDigest": "d99821da4925d5d61eb692125741bb68944bd9c1dd63d39fadbfc04b7637f8ee",
193
+ "execution": "completed",
194
+ "grading": "completed",
195
+ "task": "passed",
196
+ "exitCode": 0,
197
+ "activation": "passed",
198
+ "checks": [
199
+ {
200
+ "id": "darrow.shell.1",
201
+ "status": "passed"
202
+ },
203
+ {
204
+ "id": "darrow.shell.2",
205
+ "status": "passed"
206
+ },
207
+ {
208
+ "id": "darrow.shell.3",
209
+ "status": "passed"
210
+ },
211
+ {
212
+ "id": "darrow.semantic.1",
213
+ "status": "passed"
214
+ }
215
+ ]
216
+ },
217
+ {
218
+ "invocation": "doctor-05",
219
+ "sourceSnapshot": "repairSourceFiles",
220
+ "caseId": "doctor-adaptive-goal-direct-codex",
221
+ "runId": "34e22ee5-7a6f-444a-b526-9569ba0b0a07",
222
+ "evaluationIdentityDigest": "74cd1daef63c21e4f40c91fb626e4eb2279dab8c879c2c8d48cd8f8b8724a7d3",
223
+ "configurationDigest": "1f550988ed24dfe49ca565b2c19bedd134090760f2346d97235f5f150917b2f3",
224
+ "execution": "completed",
225
+ "grading": "unavailable",
226
+ "task": "failed",
227
+ "exitCode": 4,
228
+ "activation": "passed",
229
+ "checks": [
230
+ {
231
+ "id": "darrow.shell.1",
232
+ "status": "passed"
233
+ },
234
+ {
235
+ "id": "darrow.semantic.1",
236
+ "status": "failed"
237
+ },
238
+ {
239
+ "id": "darrow.semantic.2",
240
+ "status": "passed"
241
+ },
242
+ {
243
+ "id": "darrow.evals.transcript.1",
244
+ "status": "unavailable"
245
+ }
246
+ ]
247
+ },
248
+ {
249
+ "invocation": "doctor-06",
250
+ "sourceSnapshot": "repairSourceFiles",
251
+ "caseId": "doctor-adaptive-goal-direct-codex",
252
+ "runId": "fb7cdbac-a617-43b8-9491-90667ba06d27",
253
+ "evaluationIdentityDigest": "42e84cceee343f8a9de6ba3a412972258ceef5e0836ec0706ae6d963bbf183ce",
254
+ "configurationDigest": "1f550988ed24dfe49ca565b2c19bedd134090760f2346d97235f5f150917b2f3",
255
+ "execution": "completed",
256
+ "grading": "completed",
257
+ "task": "failed",
258
+ "exitCode": 1,
259
+ "activation": "passed",
260
+ "checks": [
261
+ {
262
+ "id": "darrow.shell.1",
263
+ "status": "passed"
264
+ },
265
+ {
266
+ "id": "darrow.semantic.1",
267
+ "status": "failed"
268
+ },
269
+ {
270
+ "id": "darrow.semantic.2",
271
+ "status": "passed"
272
+ },
273
+ {
274
+ "id": "darrow.evals.transcript.1",
275
+ "status": "passed"
276
+ }
277
+ ]
278
+ },
279
+ {
280
+ "invocation": "doctor-07",
281
+ "sourceSnapshot": "repairSourceFiles",
282
+ "caseId": "doctor-adaptive-goal-direct-codex",
283
+ "runId": "6b7f58b3-cfde-4527-8a13-80908f50c496",
284
+ "evaluationIdentityDigest": "ea8a2da192ab891bd58b583c0effab4cd659cc1495af6271a5cb0548d0a4463a",
285
+ "configurationDigest": "1f550988ed24dfe49ca565b2c19bedd134090760f2346d97235f5f150917b2f3",
286
+ "execution": "completed",
287
+ "grading": "completed",
288
+ "task": "passed",
289
+ "exitCode": 0,
290
+ "activation": "passed",
291
+ "checks": [
292
+ {
293
+ "id": "darrow.shell.1",
294
+ "status": "passed"
295
+ },
296
+ {
297
+ "id": "darrow.semantic.1",
298
+ "status": "passed"
299
+ },
300
+ {
301
+ "id": "darrow.semantic.2",
302
+ "status": "passed"
303
+ },
304
+ {
305
+ "id": "darrow.evals.transcript.1",
306
+ "status": "passed"
307
+ }
308
+ ],
309
+ "semanticGrader": {
310
+ "model": "gpt-6.1-sol",
311
+ "effort": "medium"
312
+ }
313
+ },
314
+ {
315
+ "invocation": "authoring-03",
316
+ "sourceSnapshot": "repairSourceFiles",
317
+ "caseId": "create-agent-skill-refuse-undefined-skill",
318
+ "runId": "4bc791ad-3990-4318-b79f-4a053a0b240c",
319
+ "evaluationIdentityDigest": "ee367c23f643d2582d6cd50358161a38a29e57557b0ad671c637aa3ebac76423",
320
+ "configurationDigest": "b97881513ed2824c07ac254980f01734e7a021628c77a4104af83a308c8a06e8",
321
+ "execution": "completed",
322
+ "grading": "completed",
323
+ "task": "passed",
324
+ "exitCode": 0,
325
+ "activation": "passed",
326
+ "checks": [
327
+ {
328
+ "id": "darrow.shell.1",
329
+ "status": "passed"
330
+ },
331
+ {
332
+ "id": "darrow.shell.2",
333
+ "status": "passed"
334
+ },
335
+ {
336
+ "id": "darrow.shell.3",
337
+ "status": "passed"
338
+ },
339
+ {
340
+ "id": "darrow.semantic.1",
341
+ "status": "passed"
342
+ }
343
+ ],
344
+ "semanticGrader": {
345
+ "model": "gpt-6.1-sol",
346
+ "effort": "medium"
347
+ }
348
+ },
349
+ {
350
+ "invocation": "doctor-08",
351
+ "sourceSnapshot": "mergedSourceFiles",
352
+ "semanticGrader": {
353
+ "model": "gpt-6.1-sol",
354
+ "effort": "medium"
355
+ },
356
+ "caseId": "doctor-adaptive-goal-direct-codex",
357
+ "runId": "f85d5ed0-36c4-461e-aa97-d009561e0d41",
358
+ "evaluationIdentityDigest": "e0d638f6620323798a46b8e21319149e2c0f2249cf96cb42dc5348a9ea626859",
359
+ "configurationDigest": "1f550988ed24dfe49ca565b2c19bedd134090760f2346d97235f5f150917b2f3",
360
+ "execution": "completed",
361
+ "grading": "completed",
362
+ "task": "failed",
363
+ "exitCode": 1,
364
+ "activation": "passed",
365
+ "checks": [
366
+ {
367
+ "id": "darrow.shell.1",
368
+ "status": "passed"
369
+ },
370
+ {
371
+ "id": "darrow.semantic.1",
372
+ "status": "failed"
373
+ },
374
+ {
375
+ "id": "darrow.semantic.2",
376
+ "status": "failed"
377
+ },
378
+ {
379
+ "id": "darrow.evals.transcript.1",
380
+ "status": "passed"
381
+ }
382
+ ]
383
+ },
384
+ {
385
+ "invocation": "authoring-04",
386
+ "sourceSnapshot": "mergedSourceFiles",
387
+ "semanticGrader": {
388
+ "model": "gpt-6.1-sol",
389
+ "effort": "medium"
390
+ },
391
+ "caseId": "create-agent-skill-refuse-undefined-skill",
392
+ "runId": "df6f8653-c25d-4ad2-8ea8-76cb7009663a",
393
+ "evaluationIdentityDigest": "939c67c92cdb7b48d6b2a0e842af801faf5e9a04e1dbfe62a6497e28ecfdb6e4",
394
+ "configurationDigest": "b97881513ed2824c07ac254980f01734e7a021628c77a4104af83a308c8a06e8",
395
+ "execution": "completed",
396
+ "grading": "completed",
397
+ "task": "passed",
398
+ "exitCode": 0,
399
+ "activation": "passed",
400
+ "checks": [
401
+ {
402
+ "id": "darrow.shell.1",
403
+ "status": "passed"
404
+ },
405
+ {
406
+ "id": "darrow.shell.2",
407
+ "status": "passed"
408
+ },
409
+ {
410
+ "id": "darrow.shell.3",
411
+ "status": "passed"
412
+ },
413
+ {
414
+ "id": "darrow.semantic.1",
415
+ "status": "passed"
416
+ }
417
+ ]
418
+ },
419
+ {
420
+ "invocation": "doctor-09",
421
+ "sourceSnapshot": "finalSourceFiles",
422
+ "semanticGrader": {
423
+ "model": "gpt-6.1-sol",
424
+ "effort": "medium"
425
+ },
426
+ "caseId": "doctor-adaptive-goal-direct-codex",
427
+ "runId": "8927baae-fe09-4634-a620-9c792020a2eb",
428
+ "evaluationIdentityDigest": "d88fae17ec37200324dc924a82790e4e10d1c569991dfd4f4bbfedec0fa2a127",
429
+ "configurationDigest": "1f550988ed24dfe49ca565b2c19bedd134090760f2346d97235f5f150917b2f3",
430
+ "execution": "completed",
431
+ "grading": "unavailable",
432
+ "task": "failed",
433
+ "exitCode": 4,
434
+ "activation": "passed",
435
+ "checks": [
436
+ {
437
+ "id": "darrow.shell.1",
438
+ "status": "passed"
439
+ },
440
+ {
441
+ "id": "darrow.semantic.1",
442
+ "status": "failed"
443
+ },
444
+ {
445
+ "id": "darrow.semantic.2",
446
+ "status": "failed"
447
+ },
448
+ {
449
+ "id": "darrow.evals.transcript.1",
450
+ "status": "unavailable"
451
+ }
452
+ ]
453
+ },
454
+ {
455
+ "invocation": "authoring-05",
456
+ "sourceSnapshot": "finalSourceFiles",
457
+ "semanticGrader": {
458
+ "model": "gpt-6.1-sol",
459
+ "effort": "medium"
460
+ },
461
+ "caseId": "create-agent-skill-refuse-undefined-skill",
462
+ "runId": "df04f6fe-f2a1-4a7f-b68b-f8bf4997a6eb",
463
+ "evaluationIdentityDigest": "26633362f4e738a00c39353bd1a5ee225778f960b7761f4df1e7ae859b98a18d",
464
+ "configurationDigest": "b97881513ed2824c07ac254980f01734e7a021628c77a4104af83a308c8a06e8",
465
+ "execution": "completed",
466
+ "grading": "completed",
467
+ "task": "passed",
468
+ "exitCode": 0,
469
+ "activation": "passed",
470
+ "checks": [
471
+ {
472
+ "id": "darrow.shell.1",
473
+ "status": "passed"
474
+ },
475
+ {
476
+ "id": "darrow.shell.2",
477
+ "status": "passed"
478
+ },
479
+ {
480
+ "id": "darrow.shell.3",
481
+ "status": "passed"
482
+ },
483
+ {
484
+ "id": "darrow.semantic.1",
485
+ "status": "passed"
486
+ }
487
+ ]
488
+ },
489
+ {
490
+ "invocation": "doctor-10",
491
+ "sourceSnapshot": "finalSourceFiles",
492
+ "semanticGrader": {
493
+ "model": "gpt-6.1-sol",
494
+ "effort": "medium"
495
+ },
496
+ "caseId": "doctor-adaptive-goal-direct-codex",
497
+ "runId": "92551a4e-f226-4849-a261-c6b6165eef9e",
498
+ "evaluationIdentityDigest": "48a63d6a0020b6b5136a9c452111d50339d088f024a2e77bcc13a6ab923614e5",
499
+ "configurationDigest": "1f550988ed24dfe49ca565b2c19bedd134090760f2346d97235f5f150917b2f3",
500
+ "execution": "completed",
501
+ "grading": "completed",
502
+ "task": "failed",
503
+ "exitCode": 1,
504
+ "activation": "passed",
505
+ "checks": [
506
+ {
507
+ "id": "darrow.shell.1",
508
+ "status": "passed"
509
+ },
510
+ {
511
+ "id": "darrow.semantic.1",
512
+ "status": "failed"
513
+ },
514
+ {
515
+ "id": "darrow.semantic.2",
516
+ "status": "failed"
517
+ },
518
+ {
519
+ "id": "darrow.evals.transcript.1",
520
+ "status": "passed"
521
+ }
522
+ ],
523
+ "candidate": {
524
+ "model": "gpt-6.1-sol",
525
+ "effort": "high"
526
+ }
527
+ },
528
+ {
529
+ "invocation": "authoring-06",
530
+ "sourceSnapshot": "finalSourceFiles",
531
+ "semanticGrader": {
532
+ "model": "gpt-6.1-sol",
533
+ "effort": "medium"
534
+ },
535
+ "caseId": "create-agent-skill-refuse-undefined-skill",
536
+ "runId": "9e3c8344-e46a-4e0a-af9d-6257ff669dba",
537
+ "evaluationIdentityDigest": "d7737646321373e04ef3aa46cbda8348015ff35ba87a68333fc07711ce034ff8",
538
+ "configurationDigest": "b97881513ed2824c07ac254980f01734e7a021628c77a4104af83a308c8a06e8",
539
+ "execution": "completed",
540
+ "grading": "completed",
541
+ "task": "passed",
542
+ "exitCode": 0,
543
+ "activation": "passed",
544
+ "checks": [
545
+ {
546
+ "id": "darrow.shell.1",
547
+ "status": "passed"
548
+ },
549
+ {
550
+ "id": "darrow.shell.2",
551
+ "status": "passed"
552
+ },
553
+ {
554
+ "id": "darrow.shell.3",
555
+ "status": "passed"
556
+ },
557
+ {
558
+ "id": "darrow.semantic.1",
559
+ "status": "passed"
560
+ }
561
+ ],
562
+ "candidate": {
563
+ "model": "gpt-6.1-sol",
564
+ "effort": "high"
565
+ }
566
+ },
567
+ {
568
+ "invocation": "doctor-11",
569
+ "sourceSnapshot": "finalSourceFiles",
570
+ "semanticGrader": {
571
+ "model": "gpt-6.1-sol",
572
+ "effort": "medium"
573
+ },
574
+ "caseId": "doctor-adaptive-goal-direct-codex",
575
+ "runId": null,
576
+ "evaluationIdentityDigest": null,
577
+ "configurationDigest": null,
578
+ "execution": "not_run",
579
+ "grading": "not_requested",
580
+ "task": "not_assessed",
581
+ "exitCode": 64,
582
+ "activation": "unavailable",
583
+ "checks": [],
584
+ "candidate": {
585
+ "model": "gpt-6.1-sol",
586
+ "effort": "high"
587
+ },
588
+ "entrypoint": "exec",
589
+ "diagnostic": {
590
+ "code": "sevro.invocation.invalid",
591
+ "message": "native-goal runtime policy requires Codex app-server"
592
+ }
593
+ },
594
+ {
595
+ "invocation": "authoring-07",
596
+ "sourceSnapshot": "finalSourceFiles",
597
+ "semanticGrader": {
598
+ "model": "gpt-6.1-sol",
599
+ "effort": "medium"
600
+ },
601
+ "caseId": "create-agent-skill-refuse-undefined-skill",
602
+ "runId": null,
603
+ "evaluationIdentityDigest": null,
604
+ "configurationDigest": null,
605
+ "execution": "not_run",
606
+ "grading": "not_requested",
607
+ "task": "not_assessed",
608
+ "exitCode": 64,
609
+ "activation": "unavailable",
610
+ "checks": [],
611
+ "candidate": {
612
+ "model": "gpt-6.1-sol",
613
+ "effort": "high"
614
+ },
615
+ "entrypoint": "exec",
616
+ "diagnostic": {
617
+ "code": "sevro.invocation.invalid",
618
+ "message": "native-goal runtime policy requires Codex app-server"
619
+ }
620
+ }
621
+ ],
622
+ "repairSourceFiles": {
623
+ "src/candidate-artifacts.ts": "72bc3fb458e6614116abba927d51dd50aa96721a95cc653dc765024d249a7817",
624
+ "src/candidate-codex-state.ts": "f588d2aa30f0ee69fa434ff840db439ebbeac7c5cf75530ebcb897e2dfe71bfb",
625
+ "src/hosts/codex-profile.ts": "47dd50cf8b5ae4ee42ced2d1f180f1fa6d6e10708af748661e608ea8a883ef18",
626
+ "src/hosts/codex-state.ts": "054c0fd9ddbea0cf57b30dbf940106133fb1a85c44ee079aefdd7b7c3abb16ba",
627
+ "src/hosts/codex.ts": "0c3fe80de64553b0c5ce19f16f14eeae6477b8a2c8e063ae70599a94f39fc3c9",
628
+ "src/runtime-config.ts": "bdc03ca5ee0c34c68c8a26a9855190adb82bc72b73fb51e8fe6f86245bd4e784",
629
+ "src/generated/runtime.cjs": "d3d6d6ab1c22f37b7e8e6eeaa2bbbff068c548b31447f67aad9c445d96fcb791",
630
+ "schemas/runtime-v1.schema.json": "48cc9b45789920987805eb7a2d19805e173bd02497549b73d5e6e622fe733470",
631
+ "typescript-sources.json": "e33af2db9e697280e007322ef286a21680a87f1a6ce954ba921d57376943ae66"
632
+ },
633
+ "mergedSourceFiles": {
634
+ "src/candidate-artifacts.ts": "72bc3fb458e6614116abba927d51dd50aa96721a95cc653dc765024d249a7817",
635
+ "src/candidate-codex-state.ts": "f588d2aa30f0ee69fa434ff840db439ebbeac7c5cf75530ebcb897e2dfe71bfb",
636
+ "src/hosts/codex-profile.ts": "9a143e49d839cfac0145c9d22d53cd8fa6500983096bc677ddb187ea2a13fc11",
637
+ "src/hosts/codex-state.ts": "e51b25e49bd3c14d7b832f3896e8b187088db46023c4a388fd0afe06d3e58dac",
638
+ "src/hosts/codex.ts": "0c3fe80de64553b0c5ce19f16f14eeae6477b8a2c8e063ae70599a94f39fc3c9",
639
+ "src/runtime-config.ts": "7a544e22be5cf4c161b789b8b7d8d1ce941929e799500a36d07dc9d5ac36d3c7",
640
+ "src/generated/runtime.cjs": "59e40ce9d747608c997a31e701251f3287a820f7a3fb5d2e30907342ce4d09e6",
641
+ "schemas/runtime-v1.schema.json": "7d10b7f72e27e4c5d9a2893ab2a4e47734a78ec292e02bd1a054971f21b56dd5",
642
+ "typescript-sources.json": "43eda95b3414f66b4fcec7ea5c4055d8166f72a9d695e860b07c1e78ad2d9fcc",
643
+ "src/graders/shell.ts": "e2286a6d4c24e7c231768509b10a0bf147bb8c7687f64eb2d7903cb0171061da",
644
+ "src/hosts/mac-sandbox.ts": "35fbca0603582eccc76712272af5099f9199890c8013aaa3a1a0d9f893dbdaab",
645
+ "src/hosts/isolation-common.ts": "5a6384afadf4fbf5e8e1f6d6a06a613c6f4fd6aa2ca3226e3ee700aeb9576406"
646
+ },
647
+ "finalSourceFiles": {
648
+ "src/candidate-artifacts.ts": "72bc3fb458e6614116abba927d51dd50aa96721a95cc653dc765024d249a7817",
649
+ "src/candidate-codex-state.ts": "f588d2aa30f0ee69fa434ff840db439ebbeac7c5cf75530ebcb897e2dfe71bfb",
650
+ "src/hosts/codex-profile.ts": "9d2955dc398b934b2e76ca0e25059ee7d58f55d8936dc92f969fe4430b5b7557",
651
+ "src/hosts/codex-state.ts": "e51b25e49bd3c14d7b832f3896e8b187088db46023c4a388fd0afe06d3e58dac",
652
+ "src/hosts/codex.ts": "0c3fe80de64553b0c5ce19f16f14eeae6477b8a2c8e063ae70599a94f39fc3c9",
653
+ "src/runtime-config.ts": "c13222fecc447c88f42b1b5ace5c7fcd730ef50d0030619f85ee8ada80bd0e6f",
654
+ "src/generated/runtime.cjs": "59e40ce9d747608c997a31e701251f3287a820f7a3fb5d2e30907342ce4d09e6",
655
+ "schemas/runtime-v1.schema.json": "7d10b7f72e27e4c5d9a2893ab2a4e47734a78ec292e02bd1a054971f21b56dd5",
656
+ "typescript-sources.json": "43eda95b3414f66b4fcec7ea5c4055d8166f72a9d695e860b07c1e78ad2d9fcc",
657
+ "src/graders/shell.ts": "e2286a6d4c24e7c231768509b10a0bf147bb8c7687f64eb2d7903cb0171061da",
658
+ "src/hosts/mac-sandbox.ts": "35fbca0603582eccc76712272af5099f9199890c8013aaa3a1a0d9f893dbdaab",
659
+ "src/hosts/isolation-common.ts": "5a6384afadf4fbf5e8e1f6d6a06a613c6f4fd6aa2ca3226e3ee700aeb9576406"
660
+ }
661
+ }