@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (582) hide show
  1. package/CONTRIBUTING.md +52 -6
  2. package/README.md +12 -7
  3. package/dist/examples/basic/graded.json +13 -0
  4. package/dist/examples/basic/host.js +9 -0
  5. package/dist/examples/basic/host.js.map +1 -0
  6. package/dist/examples/basic/prompt-only.json +7 -0
  7. package/dist/examples/recipes/candidate.js +23 -0
  8. package/dist/examples/recipes/candidate.js.map +1 -0
  9. package/dist/examples/recipes/cases/coding-change.json +21 -0
  10. package/dist/examples/recipes/cases/host-tools.json +21 -0
  11. package/dist/examples/recipes/cases/mise.json +21 -0
  12. package/dist/examples/recipes/cases/node.json +21 -0
  13. package/dist/examples/recipes/cases/python-uv.json +21 -0
  14. package/dist/examples/recipes/cases/semantic.json +17 -0
  15. package/dist/examples/recipes/coding-change/check.js +16 -0
  16. package/dist/examples/recipes/coding-change/check.js.map +1 -0
  17. package/dist/examples/recipes/executable.js +20 -0
  18. package/dist/examples/recipes/executable.js.map +1 -0
  19. package/dist/examples/recipes/extension-artifacts.js +56 -0
  20. package/dist/examples/recipes/extension-artifacts.js.map +1 -0
  21. package/dist/examples/recipes/extension-methods.js +167 -0
  22. package/dist/examples/recipes/extension-methods.js.map +1 -0
  23. package/dist/examples/recipes/extension.js +28 -0
  24. package/dist/examples/recipes/extension.js.map +1 -0
  25. package/dist/examples/recipes/modules.js +9 -0
  26. package/dist/examples/recipes/modules.js.map +1 -0
  27. package/dist/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  28. package/dist/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  29. package/dist/examples/recipes/repository-skill/README.md +3 -0
  30. package/dist/examples/recipes/run-inputs.js +144 -0
  31. package/dist/examples/recipes/run-inputs.js.map +1 -0
  32. package/dist/examples/recipes/run-options.js +103 -0
  33. package/dist/examples/recipes/run-options.js.map +1 -0
  34. package/dist/examples/recipes/run.js +62 -0
  35. package/dist/examples/recipes/run.js.map +1 -0
  36. package/dist/examples/recipes/runtime/empty.json +1 -0
  37. package/dist/examples/recipes/runtime/goals.json +1 -0
  38. package/dist/examples/recipes/runtime/mise.json +6 -0
  39. package/dist/examples/recipes/runtime/path.json +1 -0
  40. package/dist/examples/recipes/runtime/plugins.json +4 -0
  41. package/dist/examples/recipes/runtime/python-uv.json +10 -0
  42. package/dist/examples/recipes/semantic-judge.js +27 -0
  43. package/dist/examples/recipes/semantic-judge.js.map +1 -0
  44. package/dist/examples/templates/coding/adapter.js +14 -0
  45. package/dist/examples/templates/coding/adapter.js.map +1 -0
  46. package/dist/examples/templates/coding/case.json +15 -0
  47. package/dist/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  48. package/dist/examples/templates/repository-skill/extension-data.js +61 -0
  49. package/dist/examples/templates/repository-skill/extension-data.js.map +1 -0
  50. package/dist/examples/templates/repository-skill/extension.js +29 -0
  51. package/dist/examples/templates/repository-skill/extension.js.map +1 -0
  52. package/dist/examples/templates/response/adapter.js +13 -0
  53. package/dist/examples/templates/response/adapter.js.map +1 -0
  54. package/dist/examples/templates/response/case.json +13 -0
  55. package/dist/src/advisory.js +86 -0
  56. package/dist/src/advisory.js.map +1 -0
  57. package/dist/src/candidate-artifacts.js +139 -0
  58. package/dist/src/candidate-artifacts.js.map +1 -0
  59. package/dist/src/candidate-codex-state.js +110 -0
  60. package/dist/src/candidate-codex-state.js.map +1 -0
  61. package/dist/src/case-validation.js +92 -0
  62. package/dist/src/case-validation.js.map +1 -0
  63. package/dist/src/cli/cancellation.js +21 -0
  64. package/dist/src/cli/cancellation.js.map +1 -0
  65. package/dist/src/cli/command.js +60 -0
  66. package/dist/src/cli/command.js.map +1 -0
  67. package/dist/src/cli/completion.js +14 -0
  68. package/dist/src/cli/completion.js.map +1 -0
  69. package/dist/src/cli/demo.js +45 -0
  70. package/dist/src/cli/demo.js.map +1 -0
  71. package/dist/src/cli/evaluation.js +40 -0
  72. package/dist/src/cli/evaluation.js.map +1 -0
  73. package/dist/src/cli/extension.js +54 -0
  74. package/dist/src/cli/extension.js.map +1 -0
  75. package/dist/src/cli/help.js +190 -0
  76. package/dist/src/cli/help.js.map +1 -0
  77. package/dist/src/cli/hosts.js +33 -0
  78. package/dist/src/cli/hosts.js.map +1 -0
  79. package/dist/src/cli/init-settings.js +29 -0
  80. package/dist/src/cli/init-settings.js.map +1 -0
  81. package/dist/src/cli/init.js +82 -0
  82. package/dist/src/cli/init.js.map +1 -0
  83. package/dist/src/cli/invocation.js +431 -0
  84. package/dist/src/cli/invocation.js.map +1 -0
  85. package/dist/src/cli/loading.js +110 -0
  86. package/dist/src/cli/loading.js.map +1 -0
  87. package/dist/src/cli/preparation.js +88 -0
  88. package/dist/src/cli/preparation.js.map +1 -0
  89. package/dist/src/cli/presentation.js +61 -0
  90. package/dist/src/cli/presentation.js.map +1 -0
  91. package/dist/src/cli/progress.js +5 -0
  92. package/dist/src/cli/progress.js.map +1 -0
  93. package/dist/src/cli/run-manifest.js +108 -0
  94. package/dist/src/cli/run-manifest.js.map +1 -0
  95. package/dist/src/cli/run-options.js +367 -0
  96. package/dist/src/cli/run-options.js.map +1 -0
  97. package/dist/src/cli/run-overrides.js +51 -0
  98. package/dist/src/cli/run-overrides.js.map +1 -0
  99. package/dist/src/cli/settings.js +127 -0
  100. package/dist/src/cli/settings.js.map +1 -0
  101. package/dist/src/cli/summary.js +28 -0
  102. package/dist/src/cli/summary.js.map +1 -0
  103. package/dist/src/cli.js +20 -0
  104. package/dist/src/cli.js.map +1 -0
  105. package/dist/src/contract-types.js +2 -0
  106. package/dist/src/contract-types.js.map +1 -0
  107. package/dist/src/engine.js +4 -0
  108. package/dist/src/engine.js.map +1 -0
  109. package/dist/src/evaluation/advisory.js +85 -0
  110. package/dist/src/evaluation/advisory.js.map +1 -0
  111. package/dist/src/evaluation/check-plan.js +292 -0
  112. package/dist/src/evaluation/check-plan.js.map +1 -0
  113. package/dist/src/evaluation/collection.js +61 -0
  114. package/dist/src/evaluation/collection.js.map +1 -0
  115. package/dist/src/evaluation/context.js +2 -0
  116. package/dist/src/evaluation/context.js.map +1 -0
  117. package/dist/src/evaluation/grading-host.js +28 -0
  118. package/dist/src/evaluation/grading-host.js.map +1 -0
  119. package/dist/src/evaluation/grading.js +154 -0
  120. package/dist/src/evaluation/grading.js.map +1 -0
  121. package/dist/src/evaluation/invocation-plan.js +312 -0
  122. package/dist/src/evaluation/invocation-plan.js.map +1 -0
  123. package/dist/src/evaluation/plan.js +376 -0
  124. package/dist/src/evaluation/plan.js.map +1 -0
  125. package/dist/src/evaluation/progress.js +14 -0
  126. package/dist/src/evaluation/progress.js.map +1 -0
  127. package/dist/src/evaluation/resources.js +159 -0
  128. package/dist/src/evaluation/resources.js.map +1 -0
  129. package/dist/src/evaluation/run.js +148 -0
  130. package/dist/src/evaluation/run.js.map +1 -0
  131. package/dist/src/evaluation/scheduler.js +27 -0
  132. package/dist/src/evaluation/scheduler.js.map +1 -0
  133. package/dist/src/evaluation/semantic.js +183 -0
  134. package/dist/src/evaluation/semantic.js.map +1 -0
  135. package/dist/src/evaluation/trial.js +593 -0
  136. package/dist/src/evaluation/trial.js.map +1 -0
  137. package/dist/src/evaluation/types.js +20 -0
  138. package/dist/src/evaluation/types.js.map +1 -0
  139. package/dist/src/evidence/host-results.js +67 -0
  140. package/dist/src/evidence/host-results.js.map +1 -0
  141. package/dist/src/evidence/retained-artifacts.js +17 -0
  142. package/dist/src/evidence/retained-artifacts.js.map +1 -0
  143. package/dist/src/evidence/trial-writer.js +131 -0
  144. package/dist/src/evidence/trial-writer.js.map +1 -0
  145. package/dist/src/evidence/usage.js +14 -0
  146. package/dist/src/evidence/usage.js.map +1 -0
  147. package/dist/src/executable.js +20 -0
  148. package/dist/src/executable.js.map +1 -0
  149. package/dist/src/extension-client.js +212 -0
  150. package/dist/src/extension-client.js.map +1 -0
  151. package/dist/src/extension-session.js +236 -0
  152. package/dist/src/extension-session.js.map +1 -0
  153. package/dist/src/fixtures/advisory.js +206 -0
  154. package/dist/src/fixtures/advisory.js.map +1 -0
  155. package/dist/src/fixtures/bin.js +41 -0
  156. package/dist/src/fixtures/bin.js.map +1 -0
  157. package/dist/src/fixtures/cleanup.js +43 -0
  158. package/dist/src/fixtures/cleanup.js.map +1 -0
  159. package/dist/src/fixtures/create.js +94 -0
  160. package/dist/src/fixtures/create.js.map +1 -0
  161. package/dist/src/fixtures/generated.js +181 -0
  162. package/dist/src/fixtures/generated.js.map +1 -0
  163. package/dist/src/fixtures/git-hooks.js +16 -0
  164. package/dist/src/fixtures/git-hooks.js.map +1 -0
  165. package/dist/src/fixtures/preparation.js +201 -0
  166. package/dist/src/fixtures/preparation.js.map +1 -0
  167. package/dist/src/fixtures/repository.js +255 -0
  168. package/dist/src/fixtures/repository.js.map +1 -0
  169. package/dist/src/fixtures/setup.js +175 -0
  170. package/dist/src/fixtures/setup.js.map +1 -0
  171. package/dist/src/fixtures/tools.js +57 -0
  172. package/dist/src/fixtures/tools.js.map +1 -0
  173. package/dist/src/fixtures/workspace-state.js +155 -0
  174. package/dist/src/fixtures/workspace-state.js.map +1 -0
  175. package/dist/src/generated/cli-result.cjs +2 -0
  176. package/dist/src/generated/extension.cjs +2 -0
  177. package/dist/src/generated/report.cjs +2 -0
  178. package/dist/src/generated/run-evidence.cjs +2 -0
  179. package/dist/src/generated/runtime.cjs +2 -0
  180. package/dist/src/git.js +72 -0
  181. package/dist/src/git.js.map +1 -0
  182. package/dist/src/graders/artifact.js +68 -0
  183. package/dist/src/graders/artifact.js.map +1 -0
  184. package/dist/src/graders/git-head.js +112 -0
  185. package/dist/src/graders/git-head.js.map +1 -0
  186. package/dist/src/graders/output.js +345 -0
  187. package/dist/src/graders/output.js.map +1 -0
  188. package/dist/src/graders/semantic.js +132 -0
  189. package/dist/src/graders/semantic.js.map +1 -0
  190. package/dist/src/graders/shell.js +321 -0
  191. package/dist/src/graders/shell.js.map +1 -0
  192. package/dist/src/host-diagnostic-redaction.js +52 -0
  193. package/dist/src/host-diagnostic-redaction.js.map +1 -0
  194. package/dist/src/host-diagnostics.js +248 -0
  195. package/dist/src/host-diagnostics.js.map +1 -0
  196. package/dist/src/hosts/claude-continuation.js +89 -0
  197. package/dist/src/hosts/claude-continuation.js.map +1 -0
  198. package/dist/src/hosts/claude-credential.js +121 -0
  199. package/dist/src/hosts/claude-credential.js.map +1 -0
  200. package/dist/src/hosts/claude-events.js +136 -0
  201. package/dist/src/hosts/claude-events.js.map +1 -0
  202. package/dist/src/hosts/claude-keychain.js +100 -0
  203. package/dist/src/hosts/claude-keychain.js.map +1 -0
  204. package/dist/src/hosts/claude-login-recovery.js +67 -0
  205. package/dist/src/hosts/claude-login-recovery.js.map +1 -0
  206. package/dist/src/hosts/claude-login-refresh.js +146 -0
  207. package/dist/src/hosts/claude-login-refresh.js.map +1 -0
  208. package/dist/src/hosts/claude-login-store.js +124 -0
  209. package/dist/src/hosts/claude-login-store.js.map +1 -0
  210. package/dist/src/hosts/claude-login.js +74 -0
  211. package/dist/src/hosts/claude-login.js.map +1 -0
  212. package/dist/src/hosts/claude-native-goal.js +138 -0
  213. package/dist/src/hosts/claude-native-goal.js.map +1 -0
  214. package/dist/src/hosts/claude-process.js +10 -0
  215. package/dist/src/hosts/claude-process.js.map +1 -0
  216. package/dist/src/hosts/claude-refresh-lock.js +76 -0
  217. package/dist/src/hosts/claude-refresh-lock.js.map +1 -0
  218. package/dist/src/hosts/claude-repository-invocation.js +229 -0
  219. package/dist/src/hosts/claude-repository-invocation.js.map +1 -0
  220. package/dist/src/hosts/claude-settings.js +156 -0
  221. package/dist/src/hosts/claude-settings.js.map +1 -0
  222. package/dist/src/hosts/claude-skill-activations.js +131 -0
  223. package/dist/src/hosts/claude-skill-activations.js.map +1 -0
  224. package/dist/src/hosts/claude-state.js +237 -0
  225. package/dist/src/hosts/claude-state.js.map +1 -0
  226. package/dist/src/hosts/claude-tool-calls.js +150 -0
  227. package/dist/src/hosts/claude-tool-calls.js.map +1 -0
  228. package/dist/src/hosts/claude.js +373 -0
  229. package/dist/src/hosts/claude.js.map +1 -0
  230. package/dist/src/hosts/codex-app-server.js +764 -0
  231. package/dist/src/hosts/codex-app-server.js.map +1 -0
  232. package/dist/src/hosts/codex-configuration.js +91 -0
  233. package/dist/src/hosts/codex-configuration.js.map +1 -0
  234. package/dist/src/hosts/codex-events.js +104 -0
  235. package/dist/src/hosts/codex-events.js.map +1 -0
  236. package/dist/src/hosts/codex-exec-controls.js +434 -0
  237. package/dist/src/hosts/codex-exec-controls.js.map +1 -0
  238. package/dist/src/hosts/codex-invocation.js +54 -0
  239. package/dist/src/hosts/codex-invocation.js.map +1 -0
  240. package/dist/src/hosts/codex-marketplace.js +185 -0
  241. package/dist/src/hosts/codex-marketplace.js.map +1 -0
  242. package/dist/src/hosts/codex-native-calls.js +516 -0
  243. package/dist/src/hosts/codex-native-calls.js.map +1 -0
  244. package/dist/src/hosts/codex-process.js +10 -0
  245. package/dist/src/hosts/codex-process.js.map +1 -0
  246. package/dist/src/hosts/codex-profile.js +206 -0
  247. package/dist/src/hosts/codex-profile.js.map +1 -0
  248. package/dist/src/hosts/codex-session-files.js +89 -0
  249. package/dist/src/hosts/codex-session-files.js.map +1 -0
  250. package/dist/src/hosts/codex-state.js +310 -0
  251. package/dist/src/hosts/codex-state.js.map +1 -0
  252. package/dist/src/hosts/codex.js +527 -0
  253. package/dist/src/hosts/codex.js.map +1 -0
  254. package/dist/src/hosts/isolation-common.js +43 -0
  255. package/dist/src/hosts/isolation-common.js.map +1 -0
  256. package/dist/src/hosts/isolation-roots.js +64 -0
  257. package/dist/src/hosts/isolation-roots.js.map +1 -0
  258. package/dist/src/hosts/isolation.js +12 -0
  259. package/dist/src/hosts/isolation.js.map +1 -0
  260. package/dist/src/hosts/linux-sandbox.js +127 -0
  261. package/dist/src/hosts/linux-sandbox.js.map +1 -0
  262. package/dist/src/hosts/mac-sandbox.js +137 -0
  263. package/dist/src/hosts/mac-sandbox.js.map +1 -0
  264. package/dist/src/hosts/native-controls.js +151 -0
  265. package/dist/src/hosts/native-controls.js.map +1 -0
  266. package/dist/src/hosts/native-request.js +33 -0
  267. package/dist/src/hosts/native-request.js.map +1 -0
  268. package/dist/src/hosts/process-lifecycle.js +167 -0
  269. package/dist/src/hosts/process-lifecycle.js.map +1 -0
  270. package/dist/src/hosts/protected-worktrees.js +81 -0
  271. package/dist/src/hosts/protected-worktrees.js.map +1 -0
  272. package/dist/src/hosts/skill-hook-catalog.js +100 -0
  273. package/dist/src/hosts/skill-hook-catalog.js.map +1 -0
  274. package/dist/src/hosts/skill-hook-handler.js +228 -0
  275. package/dist/src/hosts/skill-hook-handler.js.map +1 -0
  276. package/dist/src/hosts/skill-hooks.js +423 -0
  277. package/dist/src/hosts/skill-hooks.js.map +1 -0
  278. package/dist/src/hosts/workspace-fingerprint.js +72 -0
  279. package/dist/src/hosts/workspace-fingerprint.js.map +1 -0
  280. package/dist/src/identity.js +167 -0
  281. package/dist/src/identity.js.map +1 -0
  282. package/dist/src/instrumentation.js +83 -0
  283. package/dist/src/instrumentation.js.map +1 -0
  284. package/dist/src/native-transcript-state.js +85 -0
  285. package/dist/src/native-transcript-state.js.map +1 -0
  286. package/dist/src/native-transcript-view.js +65 -0
  287. package/dist/src/native-transcript-view.js.map +1 -0
  288. package/dist/src/native-transcripts.js +208 -0
  289. package/dist/src/native-transcripts.js.map +1 -0
  290. package/dist/src/paths.js +13 -0
  291. package/dist/src/paths.js.map +1 -0
  292. package/dist/src/process-lifecycle.js +101 -0
  293. package/dist/src/process-lifecycle.js.map +1 -0
  294. package/dist/src/provenance.js +263 -0
  295. package/dist/src/provenance.js.map +1 -0
  296. package/dist/src/prune.js +228 -0
  297. package/dist/src/prune.js.map +1 -0
  298. package/dist/src/report.js +276 -0
  299. package/dist/src/report.js.map +1 -0
  300. package/dist/src/resolved-case.js +29 -0
  301. package/dist/src/resolved-case.js.map +1 -0
  302. package/dist/src/results-lock.js +40 -0
  303. package/dist/src/results-lock.js.map +1 -0
  304. package/dist/src/results.js +127 -0
  305. package/dist/src/results.js.map +1 -0
  306. package/dist/src/run-owner.js +283 -0
  307. package/dist/src/run-owner.js.map +1 -0
  308. package/dist/src/runner-paths.js +11 -0
  309. package/dist/src/runner-paths.js.map +1 -0
  310. package/dist/src/runtime/config.js +323 -0
  311. package/dist/src/runtime/config.js.map +1 -0
  312. package/dist/src/runtime/discovery-apple.js +65 -0
  313. package/dist/src/runtime/discovery-apple.js.map +1 -0
  314. package/dist/src/runtime/discovery-providers.js +37 -0
  315. package/dist/src/runtime/discovery-providers.js.map +1 -0
  316. package/dist/src/runtime/discovery.js +40 -0
  317. package/dist/src/runtime/discovery.js.map +1 -0
  318. package/dist/src/runtime/hook-processes.js +81 -0
  319. package/dist/src/runtime/hook-processes.js.map +1 -0
  320. package/dist/src/runtime/hooks.js +304 -0
  321. package/dist/src/runtime/hooks.js.map +1 -0
  322. package/dist/src/runtime/paths.js +244 -0
  323. package/dist/src/runtime/paths.js.map +1 -0
  324. package/dist/src/runtime/query.js +76 -0
  325. package/dist/src/runtime/query.js.map +1 -0
  326. package/dist/src/runtime/seeds.js +98 -0
  327. package/dist/src/runtime/seeds.js.map +1 -0
  328. package/dist/src/runtime/shared-cache.js +81 -0
  329. package/dist/src/runtime/shared-cache.js.map +1 -0
  330. package/dist/src/runtime/state.js +106 -0
  331. package/dist/src/runtime/state.js.map +1 -0
  332. package/dist/src/runtime-process.js +148 -0
  333. package/dist/src/runtime-process.js.map +1 -0
  334. package/dist/src/schema-types.js +2 -0
  335. package/dist/src/schema-types.js.map +1 -0
  336. package/dist/src/schema.js +16 -0
  337. package/dist/src/schema.js.map +1 -0
  338. package/dist/src/shell.js +5 -0
  339. package/dist/src/shell.js.map +1 -0
  340. package/dist/src/sqlite-lock.js +18 -0
  341. package/dist/src/sqlite-lock.js.map +1 -0
  342. package/dist/src/storage.js +21 -0
  343. package/dist/src/storage.js.map +1 -0
  344. package/dist/src/transcript-retention.js +109 -0
  345. package/dist/src/transcript-retention.js.map +1 -0
  346. package/dist/src/value-guards.js +26 -0
  347. package/dist/src/value-guards.js.map +1 -0
  348. package/docs/README.md +19 -15
  349. package/docs/architecture.md +141 -17
  350. package/docs/assets/README.md +5 -4
  351. package/docs/builtin-graders-v1.md +9 -2
  352. package/docs/codex-host.md +155 -57
  353. package/docs/contracts.md +2 -1
  354. package/docs/creating-cases.md +10 -4
  355. package/docs/development-cli.md +197 -24
  356. package/docs/documentation-quality.md +16 -5
  357. package/docs/evaluation-workflow.md +110 -0
  358. package/docs/extension-protocol-v1.md +26 -3
  359. package/docs/getting-started.md +29 -7
  360. package/docs/guide-evaluation.md +34 -2
  361. package/docs/identity-v1.md +29 -3
  362. package/docs/installing.md +35 -17
  363. package/docs/licensing.md +2 -2
  364. package/docs/native-controls.md +76 -5
  365. package/docs/native-hosts.md +197 -8
  366. package/docs/reading-results.md +102 -0
  367. package/docs/recipes.md +490 -0
  368. package/docs/releases.md +48 -13
  369. package/docs/report-v1.md +29 -1
  370. package/docs/results-v1.md +122 -4
  371. package/docs/run-settings-v1.md +147 -0
  372. package/docs/runtime-v1.md +255 -22
  373. package/docs/specs/repository-guide.md +1 -1
  374. package/docs/troubleshooting.md +12 -12
  375. package/docs/typescript-quality.md +106 -26
  376. package/examples/recipes/candidate.ts +26 -0
  377. package/examples/recipes/cases/coding-change.json +21 -0
  378. package/examples/recipes/cases/host-tools.json +21 -0
  379. package/examples/recipes/cases/mise.json +21 -0
  380. package/examples/recipes/cases/node.json +21 -0
  381. package/examples/recipes/cases/python-uv.json +21 -0
  382. package/examples/recipes/cases/semantic.json +17 -0
  383. package/examples/recipes/coding-change/check.ts +23 -0
  384. package/examples/recipes/executable.ts +19 -0
  385. package/examples/recipes/extension-artifacts.ts +65 -0
  386. package/examples/recipes/extension-methods.ts +174 -0
  387. package/examples/recipes/extension.ts +38 -0
  388. package/examples/recipes/modules.ts +9 -0
  389. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  390. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  391. package/examples/recipes/repository-skill/README.md +3 -0
  392. package/examples/recipes/run-inputs.ts +183 -0
  393. package/examples/recipes/run-options.ts +132 -0
  394. package/examples/recipes/run.ts +77 -0
  395. package/examples/recipes/runtime/empty.json +1 -0
  396. package/examples/recipes/runtime/goals.json +1 -0
  397. package/examples/recipes/runtime/mise.json +6 -0
  398. package/examples/recipes/runtime/path.json +1 -0
  399. package/examples/recipes/runtime/plugins.json +4 -0
  400. package/examples/recipes/runtime/python-uv.json +10 -0
  401. package/examples/recipes/semantic-judge.ts +29 -0
  402. package/examples/templates/coding/adapter.ts +17 -0
  403. package/examples/templates/coding/case.json +15 -0
  404. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  405. package/examples/templates/repository-skill/extension-data.ts +64 -0
  406. package/examples/templates/repository-skill/extension.ts +32 -0
  407. package/examples/templates/response/adapter.ts +12 -0
  408. package/examples/templates/response/case.json +13 -0
  409. package/package.json +23 -9
  410. package/schemas/report-v1.schema.json +9 -0
  411. package/schemas/run-evidence-v1.schema.json +2 -1
  412. package/schemas/runtime-v1.schema.json +43 -7
  413. package/src/advisory.ts +2 -2
  414. package/src/candidate-artifacts.ts +42 -11
  415. package/src/candidate-codex-state.ts +185 -0
  416. package/src/case-validation.ts +73 -50
  417. package/src/cli/cancellation.ts +20 -0
  418. package/src/cli/command.ts +87 -0
  419. package/src/cli/completion.ts +15 -0
  420. package/src/cli/demo.ts +61 -0
  421. package/src/cli/evaluation.ts +55 -0
  422. package/src/cli/extension.ts +79 -0
  423. package/src/cli/help.ts +199 -0
  424. package/src/cli/hosts.ts +55 -0
  425. package/src/cli/init-settings.ts +31 -0
  426. package/src/cli/init.ts +112 -0
  427. package/src/{cli-invocation.ts → cli/invocation.ts} +215 -218
  428. package/src/cli/loading.ts +151 -0
  429. package/src/cli/preparation.ts +154 -0
  430. package/src/cli/presentation.ts +93 -0
  431. package/src/cli/progress.ts +6 -0
  432. package/src/cli/run-manifest.ts +128 -0
  433. package/src/cli/run-options.ts +403 -0
  434. package/src/cli/run-overrides.ts +69 -0
  435. package/src/cli/settings.ts +154 -0
  436. package/src/cli/summary.ts +43 -0
  437. package/src/cli.ts +14 -592
  438. package/src/contract-types.ts +94 -0
  439. package/src/engine.ts +9 -5
  440. package/src/evaluation/advisory.ts +150 -0
  441. package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
  442. package/src/evaluation/collection.ts +88 -0
  443. package/src/evaluation/context.ts +10 -0
  444. package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
  445. package/src/evaluation/grading.ts +235 -0
  446. package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
  447. package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
  448. package/src/evaluation/progress.ts +19 -0
  449. package/src/evaluation/resources.ts +241 -0
  450. package/src/evaluation/run.ts +206 -0
  451. package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
  452. package/src/evaluation/trial.ts +800 -0
  453. package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
  454. package/src/evidence/host-results.ts +85 -0
  455. package/src/evidence/retained-artifacts.ts +18 -0
  456. package/src/evidence/trial-writer.ts +203 -0
  457. package/src/evidence/usage.ts +17 -0
  458. package/src/executable.ts +19 -0
  459. package/src/extension-client.ts +40 -108
  460. package/src/extension-session.ts +24 -48
  461. package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
  462. package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
  463. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  464. package/src/fixtures/create.ts +183 -0
  465. package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
  466. package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
  467. package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
  468. package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
  469. package/src/{fixture-setup.ts → fixtures/setup.ts} +36 -53
  470. package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
  471. package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
  472. package/src/generated/cli-result.d.cts +2 -1
  473. package/src/generated/extension.d.cts +2 -1
  474. package/src/generated/report.cjs +1 -1
  475. package/src/generated/report.d.cts +2 -1
  476. package/src/generated/run-evidence.cjs +1 -1
  477. package/src/generated/run-evidence.d.cts +2 -1
  478. package/src/generated/runtime.cjs +1 -1
  479. package/src/generated/runtime.d.cts +2 -1
  480. package/src/git.ts +94 -0
  481. package/src/graders/git-head.ts +31 -85
  482. package/src/graders/shell.ts +54 -71
  483. package/src/host-diagnostic-redaction.ts +52 -0
  484. package/src/host-diagnostics.ts +331 -0
  485. package/src/hosts/claude-continuation.ts +57 -40
  486. package/src/hosts/claude-credential.ts +160 -63
  487. package/src/hosts/claude-events.ts +34 -18
  488. package/src/hosts/claude-keychain.ts +79 -21
  489. package/src/hosts/claude-login-recovery.ts +85 -0
  490. package/src/hosts/claude-login-refresh.ts +179 -0
  491. package/src/hosts/claude-login-store.ts +162 -0
  492. package/src/hosts/claude-login.ts +107 -0
  493. package/src/hosts/claude-native-goal.ts +15 -12
  494. package/src/hosts/claude-process.ts +10 -50
  495. package/src/hosts/claude-refresh-lock.ts +82 -0
  496. package/src/hosts/claude-repository-invocation.ts +9 -34
  497. package/src/hosts/claude-settings.ts +46 -9
  498. package/src/hosts/claude-skill-activations.ts +169 -0
  499. package/src/hosts/claude-state.ts +28 -15
  500. package/src/hosts/claude-tool-calls.ts +13 -10
  501. package/src/hosts/claude.ts +121 -81
  502. package/src/hosts/codex-app-server.ts +78 -52
  503. package/src/hosts/codex-configuration.ts +5 -10
  504. package/src/hosts/codex-exec-controls.ts +516 -0
  505. package/src/hosts/codex-marketplace.ts +12 -10
  506. package/src/hosts/codex-native-calls.ts +15 -122
  507. package/src/hosts/codex-process.ts +10 -183
  508. package/src/hosts/codex-profile.ts +36 -21
  509. package/src/hosts/codex-session-files.ts +1 -9
  510. package/src/hosts/codex-state.ts +134 -18
  511. package/src/hosts/codex.ts +210 -183
  512. package/src/hosts/isolation-common.ts +8 -11
  513. package/src/hosts/isolation-roots.ts +5 -4
  514. package/src/hosts/linux-sandbox.ts +15 -2
  515. package/src/hosts/mac-sandbox.ts +87 -18
  516. package/src/hosts/native-controls.ts +72 -22
  517. package/src/hosts/native-request.ts +55 -0
  518. package/src/hosts/process-lifecycle.ts +215 -49
  519. package/src/hosts/protected-worktrees.ts +3 -2
  520. package/src/hosts/skill-hook-catalog.ts +131 -0
  521. package/src/hosts/skill-hook-handler.ts +374 -0
  522. package/src/hosts/skill-hooks.ts +585 -0
  523. package/src/instrumentation.ts +7 -8
  524. package/src/native-transcript-state.ts +22 -3
  525. package/src/native-transcript-view.ts +52 -13
  526. package/src/native-transcripts.ts +26 -11
  527. package/src/paths.ts +22 -0
  528. package/src/process-lifecycle.ts +150 -0
  529. package/src/provenance.ts +34 -13
  530. package/src/prune.ts +300 -0
  531. package/src/report.ts +63 -36
  532. package/src/resolved-case.ts +1 -1
  533. package/src/results-lock.ts +42 -0
  534. package/src/results.ts +14 -18
  535. package/src/run-owner.ts +15 -9
  536. package/src/runner-paths.ts +21 -0
  537. package/src/{runtime-config.ts → runtime/config.ts} +160 -46
  538. package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
  539. package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
  540. package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
  541. package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +3 -2
  542. package/src/{runtime-hooks.ts → runtime/hooks.ts} +40 -28
  543. package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
  544. package/src/{runtime-query.ts → runtime/query.ts} +5 -5
  545. package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
  546. package/src/runtime/shared-cache.ts +130 -0
  547. package/src/{runtime-state.ts → runtime/state.ts} +28 -11
  548. package/src/runtime-process.ts +194 -0
  549. package/src/schema-types.ts +73 -161
  550. package/src/shell.ts +4 -0
  551. package/src/sqlite-lock.ts +25 -0
  552. package/src/transcript-retention.ts +146 -0
  553. package/src/value-guards.ts +22 -0
  554. package/docs/assets/sevro-logo.png +0 -0
  555. package/docs/assets/sevro-symbol.svg +0 -1
  556. package/docs/assets/sevro-wordmark.svg +0 -1
  557. package/docs/cache-ownership-validation.md +0 -49
  558. package/docs/claude-continuation-validation.md +0 -47
  559. package/docs/concurrency-validation.md +0 -86
  560. package/docs/configuration-root-validation.md +0 -70
  561. package/docs/engine-slice.md +0 -114
  562. package/docs/evidence/runtime-startup-30/README.md +0 -185
  563. package/docs/evidence/runtime-startup-30/after.json +0 -319
  564. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
  565. package/docs/evidence/runtime-startup-30/before.json +0 -403
  566. package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
  567. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
  568. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
  569. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
  570. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
  571. package/docs/evidence.md +0 -17
  572. package/docs/fixture-cleanup-validation.md +0 -73
  573. package/docs/guide-verification.json +0 -239
  574. package/src/evaluation-context.ts +0 -259
  575. package/src/evaluation-fixture.ts +0 -302
  576. package/src/evaluation-run.ts +0 -200
  577. package/src/evaluation-trial.ts +0 -1016
  578. package/src/hosts/claude-nested-skills.ts +0 -272
  579. package/src/hosts/codex-command-output.ts +0 -510
  580. package/src/hosts/codex-executor-command.ts +0 -291
  581. package/src/hosts/codex-skill-reads.ts +0 -783
  582. /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
@@ -1,41 +1,46 @@
1
1
  import { existsSync } from "node:fs";
2
- import {
3
- requireCandidateRuntime,
4
- requireCandidateArtifactRoots,
5
- } from "../candidate-artifacts";
6
2
  import { readFile, realpath, rm, stat } from "node:fs/promises";
7
3
  import { dirname, isAbsolute, join } from "node:path";
8
4
  import type { HostAdapter, HostResult } from "../engine";
9
- import {
10
- allocateNativeState,
11
- requireNativeTranscriptView,
12
- } from "../native-transcript-state";
5
+ import { allocateNativeState } from "../native-transcript-state";
13
6
  import {
14
7
  failedNativeTranscripts,
15
8
  retainEnabledNativeTranscripts,
16
9
  } from "../native-transcripts";
17
- import { fixtureParts } from "../preparation";
10
+ import { fixtureParts } from "../fixtures/preparation";
18
11
  import {
19
12
  prepareHookMounts,
20
13
  releaseRuntimeHooks,
21
14
  runtimeHookObservation,
22
15
  runtimeHooksEnabled,
23
16
  type HookMounts,
24
- } from "../runtime-hooks";
25
- import { insideRuntimeRoot as inside } from "../runtime-paths";
26
- import { runtimeObservations } from "../runtime-state";
27
- import { isOptionalNonblankString, isRecord } from "../value-guards";
17
+ } from "../runtime/hooks";
18
+ import { insideRuntimeRoot as inside } from "../runtime/paths";
19
+ import { runtimeObservations } from "../runtime/state";
20
+ import { isRecord } from "../value-guards";
28
21
  import { runClaudeTurns, type ClaudeSession } from "./claude-continuation";
22
+ import { validateNativeHostRequest, validHostTimeout } from "./native-request";
29
23
  import { summarizeClaudeEvents } from "./claude-events";
30
24
  import { observeClaudeNativeGoal } from "./claude-native-goal";
31
- import { claudeNestedSkillsObservation } from "./claude-nested-skills";
32
25
  import { runClaudeProcess } from "./claude-process";
33
26
  import {
34
27
  claudeRepositoryInvocationObservation,
35
28
  verifyClaudeRepositoryInvocation,
36
29
  } from "./claude-repository-invocation";
37
30
  import { prepareClaudeState, type ClaudeHostOptions } from "./claude-state";
31
+ import {
32
+ cleanupNativeState,
33
+ createHostDiagnostics,
34
+ diagnoseHostRequest,
35
+ diagnosticConfiguration,
36
+ type HostDiagnostics,
37
+ } from "../host-diagnostics";
38
+ import {
39
+ CLAUDE_AUTH_VARIABLES,
40
+ CLAUDE_LOGIN_MARGIN_MS,
41
+ } from "./claude-credential";
38
42
  import { claudeToolCallsObservation } from "./claude-tool-calls";
43
+ import { claudeSkillActivationsObservation } from "./claude-skill-activations";
39
44
  import { claudeNativeControls } from "./native-controls";
40
45
  export type { ClaudeHostOptions } from "./claude-state";
41
46
 
@@ -221,12 +226,7 @@ function optionalClaudePaths(options: ClaudeHostOptions): boolean {
221
226
  ].every((path) => path === undefined || isAbsolute(path));
222
227
  }
223
228
  function validClaudeEffortTimeout(effort: string, timeoutMs: number): boolean {
224
- return (
225
- !!effort &&
226
- Number.isSafeInteger(timeoutMs) &&
227
- timeoutMs >= 1 &&
228
- timeoutMs <= 30 * 60_000
229
- );
229
+ return !!effort && validHostTimeout(timeoutMs);
230
230
  }
231
231
 
232
232
  /** Run Claude with explicit plugins and its native tool sandbox. */
@@ -238,11 +238,17 @@ export function createClaudeHost(options: ClaudeHostOptions): HostAdapter {
238
238
  id: "sevro.host.claude",
239
239
  model: options.model,
240
240
  effort: options.effort,
241
+ ...(options.retainHostDiagnostics
242
+ ? {
243
+ configuration: diagnosticConfiguration(options.retainHostDiagnostics),
244
+ }
245
+ : {}),
241
246
  hostCapabilities: [
242
247
  "sevro.host.native-goal",
243
248
  "sevro.host.runtime",
244
249
  "sevro.host.native-transcripts",
245
250
  "sevro.host.hooks",
251
+ "sevro.host.skill-activations",
246
252
  "sevro.claude.plugin-dirs",
247
253
  "sevro.claude.explicit-invocation",
248
254
  ...(options.projectSettings
@@ -250,7 +256,6 @@ export function createClaudeHost(options: ClaudeHostOptions): HostAdapter {
250
256
  : []),
251
257
  "sevro.claude.tool-calls",
252
258
  "sevro.host.native-controls",
253
- "sevro.claude.nested-skills",
254
259
  "sevro.host.continuation",
255
260
  "sevro.claude.continuation",
256
261
  ],
@@ -260,24 +265,6 @@ export function createClaudeHost(options: ClaudeHostOptions): HostAdapter {
260
265
  };
261
266
  }
262
267
 
263
- function requireClaudeFollowUp(request: Request): void {
264
- if (!isOptionalNonblankString(request.followUpPrompt))
265
- throw new Error("Claude follow-up prompt must be nonempty");
266
- }
267
- function requirePassiveClaudeRequest(request: Request): void {
268
- if (request.instrumentation?.length || request.condition !== "passive")
269
- throw new Error("Claude enforcement instrumentation is unavailable");
270
- requireClaudeFixtureBin(request);
271
- }
272
-
273
- function requireClaudeFixtureBin(request: Request): void {
274
- if (
275
- request.fixtureBinDir !== undefined &&
276
- request.fixtureBinDir !== join(request.workspace, ".git", "fixture-bin")
277
- )
278
- throw new Error("fixture binary path is outside the workspace");
279
- }
280
-
281
268
  async function repositoryMount(
282
269
  request: Request,
283
270
  options: ClaudeHostOptions,
@@ -295,47 +282,58 @@ async function runClaudeRequest(
295
282
  timeoutMs: number,
296
283
  request: Request,
297
284
  ): Promise<HostResult> {
298
- requireClaudeFollowUp(request);
299
- await requireNativeTranscriptView(request.candidateTranscriptRoot);
300
- await requireCandidateRuntime(
301
- request.candidateRuntimeRoot,
302
- request.runtimeRole,
285
+ const diagnostics = createHostDiagnostics(
286
+ "claude",
287
+ options.retainHostDiagnostics,
288
+ request,
303
289
  );
304
- await requireCandidateArtifactRoots(request.candidateArtifactRoots);
305
- requirePassiveClaudeRequest(request);
306
- const pluginDirs = request.claudePluginDirs
307
- ? await pluginDirectories(request.workspace, request.claudePluginDirs)
308
- : [];
309
- await verifyInvocation(request, pluginDirs);
310
- requireIsolatedHookSources(request, options);
311
- const repositoryInvocation = await repositoryMount(request, options);
290
+ return diagnoseHostRequest(diagnostics, async () => {
291
+ diagnostics.stage = "request-validation";
292
+ await diagnostics.readCredentials(options.credentialFile);
293
+ return executeClaudeRequest(options, timeoutMs, request, diagnostics);
294
+ });
295
+ }
296
+
297
+ /** Every turn shares one access token, so it must outlive all of them. */
298
+ function claudeLoginLifetime(timeoutMs: number, request: Request): number {
299
+ const turns = request.followUpPrompt === undefined ? 1 : 2;
300
+ return timeoutMs * turns + CLAUDE_LOGIN_MARGIN_MS;
301
+ }
302
+
303
+ async function executeClaudeRequest(
304
+ options: ClaudeHostOptions,
305
+ timeoutMs: number,
306
+ request: Request,
307
+ diagnostics: HostDiagnostics,
308
+ ): Promise<HostResult> {
309
+ const { pluginDirs, repositoryInvocation } = await validateClaudeRequest(
310
+ options,
311
+ request,
312
+ );
313
+ diagnostics.stage = "state-allocation";
312
314
  const stateRoot = await allocateNativeState(
313
315
  "claude-",
314
316
  request.runtimePolicy ? request.workspace : undefined,
315
317
  );
316
318
  let hooks: HookMounts | undefined;
317
319
  try {
320
+ diagnostics.stage = "state-preparation";
318
321
  const state = await prepareClaudeState(
319
322
  stateRoot,
320
323
  options,
321
324
  request,
322
325
  pluginDirs,
326
+ claudeLoginLifetime(timeoutMs, request),
323
327
  );
328
+ diagnostics.registerCredentials(state.env, CLAUDE_AUTH_VARIABLES);
324
329
  hooks = state.hooks;
325
- const run = (prompt: string, session?: ClaudeSession) =>
326
- runClaudeProcess({
327
- argv: claudeTurnArgs(
328
- options,
329
- prompt,
330
- session,
331
- state.settingsPath,
332
- state.pluginDirs,
333
- ),
334
- cwd: request.workspace,
335
- env: state.env,
336
- timeoutMs,
337
- signal: request.signal,
338
- });
330
+ const run = claudeProcessRunner(
331
+ options,
332
+ timeoutMs,
333
+ request,
334
+ state,
335
+ diagnostics,
336
+ );
339
337
  const execution = await runClaudeTurns({
340
338
  prompt: request.prompt,
341
339
  followUpPrompt: request.followUpPrompt,
@@ -348,6 +346,10 @@ async function runClaudeRequest(
348
346
  request,
349
347
  repositoryInvocation,
350
348
  );
349
+ await diagnostics.retainFailure(
350
+ result,
351
+ "Claude host execution did not complete",
352
+ );
351
353
  await addHookObservation(result, state.hooks);
352
354
  await retainEnabledNativeTranscripts(
353
355
  result,
@@ -364,13 +366,56 @@ async function runClaudeRequest(
364
366
  join(stateRoot, "native-home", "projects"),
365
367
  "claude",
366
368
  request.runtimeRole,
369
+ diagnostics,
367
370
  );
368
371
  } finally {
369
- await releaseRuntimeHooks(hooks);
370
- await rm(stateRoot, { recursive: true, force: true });
372
+ await cleanupNativeState(diagnostics, async () => {
373
+ await releaseRuntimeHooks(hooks);
374
+ await rm(stateRoot, { recursive: true, force: true });
375
+ });
371
376
  }
372
377
  }
373
378
 
379
+ function claudeProcessRunner(
380
+ options: ClaudeHostOptions,
381
+ timeoutMs: number,
382
+ request: Request,
383
+ state: Awaited<ReturnType<typeof prepareClaudeState>>,
384
+ diagnostics: HostDiagnostics,
385
+ ) {
386
+ return (prompt: string, session?: ClaudeSession) => {
387
+ diagnostics.stage = "claude-execution";
388
+ return runClaudeProcess({
389
+ diagnostics,
390
+ argv: claudeTurnArgs(
391
+ options,
392
+ prompt,
393
+ session,
394
+ state.settingsPath,
395
+ state.pluginDirs,
396
+ ),
397
+ cwd: request.workspace,
398
+ env: state.env,
399
+ timeoutMs,
400
+ signal: request.signal,
401
+ });
402
+ };
403
+ }
404
+
405
+ async function validateClaudeRequest(
406
+ options: ClaudeHostOptions,
407
+ request: Request,
408
+ ) {
409
+ await validateNativeHostRequest(request, "Claude");
410
+ const pluginDirs = request.claudePluginDirs
411
+ ? await pluginDirectories(request.workspace, request.claudePluginDirs)
412
+ : [];
413
+ await verifyInvocation(request, pluginDirs);
414
+ requireIsolatedHookSources(request, options);
415
+ const repositoryInvocation = await repositoryMount(request, options);
416
+ return { pluginDirs, repositoryInvocation };
417
+ }
418
+
374
419
  function requireIsolatedHookSources(
375
420
  request: Request,
376
421
  options: ClaudeHostOptions,
@@ -442,11 +487,11 @@ async function repositoryObservation(
442
487
  if (!invocation) return null;
443
488
  return claudeRepositoryInvocationObservation({
444
489
  invocation,
445
- stream: execution.initialOut ?? execution.out,
490
+ stream: execution.initialEvents ?? execution.events,
446
491
  configRoot: configRoot,
447
492
  workspace: request.workspace,
448
- tools: execution.initialOut
449
- ? claudeToolCallsObservation(execution.initialOut, 0)
493
+ tools: execution.initialEvents
494
+ ? claudeToolCallsObservation(execution.initialEvents, 0)
450
495
  : tools,
451
496
  });
452
497
  }
@@ -457,12 +502,7 @@ async function claudeObservations(
457
502
  request: Request,
458
503
  invocation: RepositoryInvocation | null,
459
504
  ) {
460
- const nested = await claudeNestedSkillsObservation(
461
- execution.out,
462
- configRoot,
463
- request.workspace,
464
- );
465
- const tools = claudeToolCallsObservation(execution.out, execution.code);
505
+ const tools = claudeToolCallsObservation(execution.events, execution.code);
466
506
  const repository = await repositoryObservation(
467
507
  execution,
468
508
  configRoot,
@@ -475,12 +515,12 @@ async function claudeObservations(
475
515
  ...runtimeObservations(request.runtimePolicy, request.runtimeRole),
476
516
  await observeClaudeNativeGoal(
477
517
  configRoot,
478
- execution.followUpOut ?? execution.out,
518
+ execution.followUpEvents ?? execution.events,
479
519
  ),
480
- nested,
481
- claudeNativeControls(execution.out, execution.code),
520
+ claudeNativeControls(execution.events, execution.code),
482
521
  ...(repository ? [repository] : []),
483
522
  ...(execution.continuation ? [execution.continuation] : []),
523
+ claudeSkillActivationsObservation(execution.events, execution.code),
484
524
  ];
485
525
  }
486
526
 
@@ -535,7 +575,7 @@ async function claudeResult(
535
575
  request: Request,
536
576
  invocation: RepositoryInvocation | null,
537
577
  ): Promise<HostResult> {
538
- const summary = summarizeClaudeEvents(execution.out, execution.code);
578
+ const summary = summarizeClaudeEvents(execution.events, execution.code);
539
579
  const observations = await claudeObservations(
540
580
  execution,
541
581
  configRoot,
@@ -1,8 +1,11 @@
1
- import { isRecord } from "../value-guards";
2
- import { writeFile } from "node:fs/promises";
3
- import { join } from "node:path";
1
+ import { redactDiagnosticText } from "../host-diagnostic-redaction";
2
+ import { isRecord, requireRecord } from "../value-guards";
4
3
  import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
5
4
  import { createInterface } from "node:readline";
5
+ import {
6
+ HostProcessDiagnostics,
7
+ type HostDiagnostics,
8
+ } from "../host-diagnostics";
6
9
  interface HarnessRunRequest {
7
10
  repoDir: string;
8
11
  prompt: string;
@@ -14,9 +17,7 @@ interface HarnessRunRequest {
14
17
 
15
18
  type RecordValue = Record<string, unknown>;
16
19
  function record(value: unknown): RecordValue {
17
- if (!value || typeof value !== "object" || Array.isArray(value))
18
- throw new Error("Malformed app-server object");
19
- return value as RecordValue;
20
+ return requireRecord(value, "Malformed app-server object");
20
21
  }
21
22
  function identifier(value: unknown): string {
22
23
  if (typeof value !== "string" || !value)
@@ -70,15 +71,7 @@ function publicErrorMessage(value: unknown): {
70
71
  messageTruncated?: boolean;
71
72
  } {
72
73
  if (typeof value !== "string") return { message: null };
73
- const redacted = value
74
- .slice(0, 20000)
75
- .replace(/(authorization\s*:\s*)[^\r\n]+/gi, "$1[redacted]")
76
- .replace(/\bBearer\s+\S+/gi, "Bearer [redacted]")
77
- .replace(/\bsk-[A-Za-z0-9_-]+/g, "[redacted]")
78
- .replace(
79
- /((?:api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[=:]\s*)\S+/gi,
80
- "$1[redacted]",
81
- );
74
+ const redacted = redactDiagnosticText(value.slice(0, 20000), []);
82
75
  return {
83
76
  message: redacted.slice(0, 2000),
84
77
  ...(value.length > 20000 || redacted.length > 2000
@@ -145,7 +138,7 @@ class AppServerRpc {
145
138
  {
146
139
  resolve: (value: RecordValue) => void;
147
140
  reject: (error: Error) => void;
148
- timer: ReturnType<typeof setTimeout>;
141
+ method: string;
149
142
  }
150
143
  >();
151
144
  private wake?: () => void;
@@ -155,8 +148,15 @@ class AppServerRpc {
155
148
  private reads: Promise<void>;
156
149
  private errors: Promise<void>;
157
150
  stderr = "";
151
+ readonly capture: HostProcessDiagnostics;
158
152
 
159
- constructor(argv: string[], cwd: string, env: Record<string, string>) {
153
+ constructor(
154
+ argv: string[],
155
+ cwd: string,
156
+ env: Record<string, string>,
157
+ diagnostics: HostDiagnostics,
158
+ ) {
159
+ this.capture = new HostProcessDiagnostics(diagnostics);
160
160
  this.process = spawn(identifier(argv[0]), argv.slice(1), {
161
161
  detached: process.platform !== "win32",
162
162
  cwd,
@@ -166,10 +166,14 @@ class AppServerRpc {
166
166
  this.exited = new Promise((resolve) => {
167
167
  this.process.once("exit", resolve);
168
168
  this.process.once("error", (error) => {
169
+ if (this.process.pid === undefined) this.capture.recordUnstarted();
169
170
  this.fail(error);
170
171
  resolve(null);
171
172
  });
172
173
  });
174
+ this.process.stdout.on("data", (chunk: Buffer) => {
175
+ this.capture.stdout.add(chunk);
176
+ });
173
177
  this.reads = this.readLines().catch((error: unknown) => {
174
178
  this.fail(error);
175
179
  });
@@ -177,7 +181,8 @@ class AppServerRpc {
177
181
  this.fail(error);
178
182
  });
179
183
  this.errors = new Promise((resolve) => {
180
- this.process.stderr.on("data", (chunk) => {
184
+ this.process.stderr.on("data", (chunk: Buffer) => {
185
+ this.capture.stderr.add(chunk);
181
186
  this.stderr = (this.stderr + String(chunk)).slice(-16000);
182
187
  });
183
188
  this.process.stderr.on("end", resolve);
@@ -195,10 +200,7 @@ class AppServerRpc {
195
200
 
196
201
  fail(error: unknown): void {
197
202
  this.failure ??= error instanceof Error ? error : new Error(String(error));
198
- for (const pending of this.pending.values()) {
199
- clearTimeout(pending.timer);
200
- pending.reject(this.failure);
201
- }
203
+ for (const pending of this.pending.values()) pending.reject(this.failure);
202
204
  this.pending.clear();
203
205
  this.wake?.();
204
206
  }
@@ -263,7 +265,6 @@ class AppServerRpc {
263
265
  // Validate before removing the waiter so malformed responses reject it.
264
266
  const value = record(message.error ?? message.result);
265
267
  this.pending.delete(id);
266
- clearTimeout(pending.timer);
267
268
  if (message.error)
268
269
  pending.reject(
269
270
  new Error(`App-server RPC failed (${String(value.code)})`),
@@ -293,14 +294,12 @@ class AppServerRpc {
293
294
  this.wake?.();
294
295
  }
295
296
 
297
+ /** The session's configured time bound is the only deadline, including startup. */
296
298
  async call(method: string, params: RecordValue): Promise<RecordValue> {
297
299
  if (this.failure) throw this.failure;
298
300
  const id = this.nextId++;
299
301
  return new Promise((resolve, reject) => {
300
- const timer = setTimeout(() => {
301
- this.fail(new Error(`App-server RPC timeout: ${method}`));
302
- }, 45000);
303
- this.pending.set(id, { resolve, reject, timer });
302
+ this.pending.set(id, { resolve, reject, method });
304
303
  try {
305
304
  this.send({ id, method, params });
306
305
  } catch (error) {
@@ -309,6 +308,10 @@ class AppServerRpc {
309
308
  });
310
309
  }
311
310
 
311
+ pendingMethods(): string[] {
312
+ return Array.from(this.pending.values(), (pending) => pending.method);
313
+ }
314
+
312
315
  async initialize(): Promise<void> {
313
316
  await this.call("initialize", {
314
317
  clientInfo: { name: "sevro_eval", version: "0.1.0" },
@@ -332,11 +335,13 @@ class AppServerRpc {
332
335
  async close(): Promise<void> {
333
336
  this.closing = true;
334
337
  this.process.stdin.end();
338
+ let shutdownError: NodeJS.ErrnoException | undefined;
335
339
  const terminate = (signal: NodeJS.Signals) => {
336
340
  try {
337
341
  if (this.process.pid) process.kill(-this.process.pid, signal);
338
342
  } catch (error) {
339
- if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
343
+ if ((error as NodeJS.ErrnoException).code !== "ESRCH")
344
+ shutdownError ??= error as NodeJS.ErrnoException;
340
345
  }
341
346
  };
342
347
  const term = setTimeout(() => {
@@ -353,6 +358,7 @@ class AppServerRpc {
353
358
  clearTimeout(term);
354
359
  clearTimeout(kill);
355
360
  }
361
+ if (shutdownError !== undefined) throw shutdownError;
356
362
  }
357
363
  }
358
364
 
@@ -471,6 +477,7 @@ function requireGoalObjective(objective: unknown): void {
471
477
  }
472
478
 
473
479
  export interface AppServerRunOptions {
480
+ diagnostics: HostDiagnostics;
474
481
  vettedHooks?: boolean;
475
482
  request: HarnessRunRequest;
476
483
  argv: string[];
@@ -492,9 +499,9 @@ class AppServerSession {
492
499
  goals: [],
493
500
  };
494
501
  readonly out: string[] = [];
502
+ finalMessage: string | null = null;
495
503
  private threadId = "";
496
504
  private goalSeen = false;
497
- private usage: unknown;
498
505
  private emittedItems = new Set<string>();
499
506
  private followUp?: string;
500
507
 
@@ -621,14 +628,6 @@ class AppServerSession {
621
628
  if (params.willRetry !== true)
622
629
  throw new Error("App-server reported a fatal turn error");
623
630
  break;
624
- case "thread/tokenUsage/updated": {
625
- const total = record(record(params.tokenUsage).total);
626
- this.usage = {
627
- input_tokens: total.inputTokens,
628
- output_tokens: total.outputTokens,
629
- };
630
- break;
631
- }
632
631
  case "item/completed":
633
632
  this.emitItem(params.item);
634
633
  break;
@@ -651,13 +650,10 @@ class AppServerSession {
651
650
  const turnId = this.observeTurn(turn);
652
651
  this.requireCompletedTurn(turn, turnId);
653
652
  const status = await this.completedGoal(turnId);
654
- this.emit({
655
- type: "turn.completed",
656
- ...(this.usage ? { usage: this.usage } : {}),
657
- });
653
+ this.emit({ type: "turn.completed" });
658
654
  if (this.followUp !== undefined)
659
655
  return this.deliverFeedback(turnId, status, this.followUp);
660
- return status !== "active" && (await this.saveFinal(turnId));
656
+ return status !== "active" && (await this.captureFinal(turnId));
661
657
  }
662
658
 
663
659
  private requireCompletedTurn(turn: RecordValue, turnId: string): void {
@@ -686,7 +682,7 @@ class AppServerSession {
686
682
  ): Promise<boolean> {
687
683
  // User input does not require a terminal goal. Preserve the completed
688
684
  // response and actual boundary before delivering the declared follow-up.
689
- if (!(await this.saveFinal(turnId, true))) return false;
685
+ if (!(await this.captureFinal(turnId, true))) return false;
690
686
  const boundary = protocolMessage(
691
687
  await this.options.followUpBoundary(this.threadId, {
692
688
  nativeGoalObserved: this.goalSeen,
@@ -704,7 +700,7 @@ class AppServerSession {
704
700
  return false;
705
701
  }
706
702
 
707
- private async saveFinal(
703
+ private async captureFinal(
708
704
  turnId: string,
709
705
  forFeedback = false,
710
706
  ): Promise<boolean> {
@@ -718,10 +714,7 @@ class AppServerSession {
718
714
  if (!latestTurnReady(latest, turnId)) return false;
719
715
  const final = appServerFinal(thread, turnId);
720
716
  for (const value of turns) this.backfillItems(record(value));
721
- await writeFile(
722
- join(this.options.request.repoDir, ".git", "last-message.md"),
723
- final,
724
- );
717
+ this.finalMessage = final;
725
718
  this.evidence.finalTurnId = turnId;
726
719
  this.emit({
727
720
  type: "item.completed",
@@ -861,10 +854,16 @@ function requireAppServerRun(request: HarnessRunRequest, bound: number): void {
861
854
  throw new Error("Invalid app-server time bound");
862
855
  }
863
856
  function sessionFailure(session: AppServerSession, error: unknown): void {
857
+ session.options.diagnostics.failure = error;
864
858
  session.evidence.failure =
865
859
  error instanceof Error ? error.message : String(error);
866
860
  session.emit({ type: "turn.failed" });
867
861
  }
862
+ /** Name an unanswered request so a stalled startup stays distinguishable. */
863
+ function timeBoundFailure(pending: string[]): string {
864
+ const reached = "App-server evaluation time bound reached";
865
+ return pending.length ? `${reached} awaiting ${pending.join(", ")}` : reached;
866
+ }
868
867
  async function completeSession(
869
868
  session: AppServerSession,
870
869
  rpc: AppServerRpc,
@@ -876,19 +875,32 @@ async function completeSession(
876
875
  };
877
876
  request.signal?.addEventListener("abort", cancel, { once: true });
878
877
  const deadline = setTimeout(() => {
879
- rpc.fail(new Error("App-server evaluation time bound reached"));
878
+ rpc.fail(new Error(timeBoundFailure(rpc.pendingMethods())));
880
879
  }, bound);
880
+ let code = 0;
881
881
  try {
882
882
  await session.run();
883
883
  session.assertSettled();
884
- return 0;
885
884
  } catch (error) {
886
885
  sessionFailure(session, error);
887
- return 1;
886
+ code = 1;
888
887
  } finally {
889
888
  clearTimeout(deadline);
890
889
  request.signal?.removeEventListener("abort", cancel);
890
+ }
891
+ return closeSession(session, rpc, code);
892
+ }
893
+ async function closeSession(
894
+ session: AppServerSession,
895
+ rpc: AppServerRpc,
896
+ code: number,
897
+ ): Promise<number> {
898
+ try {
891
899
  await rpc.close();
900
+ return code;
901
+ } catch (error) {
902
+ if (code === 0) sessionFailure(session, error);
903
+ return 1;
892
904
  }
893
905
  }
894
906
  function settlementCode(session: AppServerSession, code: number): number {
@@ -906,20 +918,34 @@ export async function runCodexAppServer(options: AppServerRunOptions): Promise<{
906
918
  out: string;
907
919
  err: string;
908
920
  code: number;
921
+ finalMessage: string | null;
909
922
  evidence: AppServerEvidence;
910
923
  }> {
911
924
  const { request, argv, env } = options;
912
925
  const bound = request.control?.appServerTimeoutMs ?? 3600000;
913
926
  requireAppServerRun(request, bound);
914
- const rpc = new AppServerRpc(argv, request.repoDir, env);
927
+ const rpc = new AppServerRpc(argv, request.repoDir, env, options.diagnostics);
915
928
  const session = new AppServerSession(rpc, options);
916
929
  const code = await completeSession(session, rpc, request, bound);
917
930
  // Flush stdout before accepting success, including trailing data and terminal errors.
918
931
  const finalCode = settlementCode(session, code);
932
+ if (rpc.process.pid !== undefined)
933
+ await rpc.capture.record(
934
+ { exited: rpc.exited, signalCode: rpc.process.signalCode },
935
+ finalCode === 0 ? "exit" : "error",
936
+ );
919
937
  return {
920
938
  out: session.out.join("\n"),
921
939
  err: rpc.stderr,
922
940
  code: finalCode,
941
+ finalMessage: completedMessage(session, finalCode),
923
942
  evidence: session.evidence,
924
943
  };
925
944
  }
945
+
946
+ function completedMessage(
947
+ session: AppServerSession,
948
+ code: number,
949
+ ): string | null {
950
+ return code === 0 ? session.finalMessage : null;
951
+ }
@@ -1,7 +1,8 @@
1
1
  import type { Stats } from "node:fs";
2
2
  import { isRecord } from "../value-guards";
3
3
  import { lstat, readFile, realpath, stat } from "node:fs/promises";
4
- import { dirname, isAbsolute, join, relative, sep } from "node:path";
4
+ import { dirname, isAbsolute, join } from "node:path";
5
+ import { insideRoot } from "../paths";
5
6
  import { parse } from "smol-toml";
6
7
 
7
8
  export async function configurationRoot(root: string): Promise<string> {
@@ -52,7 +53,7 @@ async function readConfiguration(
52
53
  const info = await lstat(configPath);
53
54
  declared = true;
54
55
  const canonical = await realpath(configPath);
55
- validateConfigurationFile(info, relative(root, canonical));
56
+ validateConfigurationFile(info, insideRoot(root, canonical));
56
57
  source = await readFile(configPath, "utf8");
57
58
  } catch (error) {
58
59
  if (!declared && (error as NodeJS.ErrnoException).code === "ENOENT")
@@ -97,15 +98,9 @@ function concurrencyLimit(limit: unknown, configPath: string): number | null {
97
98
  return Number(limit);
98
99
  }
99
100
 
100
- function validateConfigurationFile(info: Stats, location: string): void {
101
- if (!info.isFile() || info.size > 64 * 1024 || escapedRoot(location))
101
+ function validateConfigurationFile(info: Stats, contained: boolean): void {
102
+ if (!info.isFile() || info.size > 64 * 1024 || !contained)
102
103
  throw new Error(
103
104
  "configuration must be a contained regular file up to 64 KiB",
104
105
  );
105
106
  }
106
-
107
- function escapedRoot(location: string): boolean {
108
- return (
109
- location === ".." || location.startsWith(`..${sep}`) || isAbsolute(location)
110
- );
111
- }