@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/CONTRIBUTING.md +170 -0
  2. package/LICENSE +63 -136
  3. package/README.md +83 -66
  4. package/docs/README.md +44 -0
  5. package/docs/acknowledgements.md +14 -0
  6. package/docs/architecture.md +146 -0
  7. package/docs/assets/README.md +27 -0
  8. package/docs/assets/sevro-logo.png +0 -0
  9. package/docs/assets/sevro-symbol.svg +1 -0
  10. package/docs/assets/sevro-wordmark.svg +1 -0
  11. package/docs/builtin-graders-v1.md +41 -13
  12. package/docs/codex-host.md +184 -52
  13. package/docs/contracts.md +19 -0
  14. package/docs/creating-cases.md +139 -0
  15. package/docs/development-cli.md +190 -30
  16. package/docs/documentation-quality.md +61 -0
  17. package/docs/evaluation-workflow.md +110 -0
  18. package/docs/evidence/code-mode-controls-71.md +64 -0
  19. package/docs/evidence/code-mode-programs-85.md +64 -0
  20. package/docs/evidence/codex-state-73/README.md +204 -0
  21. package/docs/evidence/codex-state-73/darrow-migration.patch +182 -0
  22. package/docs/evidence/codex-state-73/results.json +661 -0
  23. package/docs/evidence/local-http-74.md +104 -0
  24. package/docs/evidence/runtime-startup-30/README.md +185 -0
  25. package/docs/evidence/runtime-startup-30/after.json +319 -0
  26. package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +2 -0
  27. package/docs/evidence/runtime-startup-30/before.json +403 -0
  28. package/docs/evidence/runtime-startup-30/consumer-smoke.json +63 -0
  29. package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +3 -0
  30. package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +3 -0
  31. package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +20 -0
  32. package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +2 -0
  33. package/docs/evidence/skill-reads-70/results.json +823 -0
  34. package/docs/evidence.md +21 -0
  35. package/docs/extension-protocol-v1.md +50 -16
  36. package/docs/fixture-cleanup-validation.md +3 -0
  37. package/docs/getting-started.md +102 -0
  38. package/docs/guide-evaluation.md +89 -0
  39. package/docs/guide-verification.json +239 -0
  40. package/docs/identity-v1.md +35 -1
  41. package/docs/installing.md +88 -0
  42. package/docs/licenses/BUSL-1.1-0.1.0-rc.2.txt +159 -0
  43. package/docs/licensing.md +42 -0
  44. package/docs/native-controls.md +76 -5
  45. package/docs/native-hosts.md +228 -0
  46. package/docs/reading-results.md +141 -0
  47. package/docs/recipes.md +489 -0
  48. package/docs/releases.md +81 -40
  49. package/docs/report-v1.md +29 -1
  50. package/docs/results-v1.md +180 -0
  51. package/docs/run-settings-v1.md +147 -0
  52. package/docs/running-evaluations.md +31 -0
  53. package/docs/runtime-v1.md +614 -0
  54. package/docs/specs/repository-guide.md +58 -0
  55. package/docs/troubleshooting.md +30 -0
  56. package/docs/typescript-quality.md +234 -0
  57. package/docs/usability-study.md +114 -0
  58. package/examples/basic/host.ts +2 -2
  59. package/examples/recipes/candidate.ts +26 -0
  60. package/examples/recipes/cases/coding-change.json +21 -0
  61. package/examples/recipes/cases/host-tools.json +21 -0
  62. package/examples/recipes/cases/mise.json +21 -0
  63. package/examples/recipes/cases/node.json +21 -0
  64. package/examples/recipes/cases/python-uv.json +21 -0
  65. package/examples/recipes/cases/semantic.json +17 -0
  66. package/examples/recipes/coding-change/check.ts +23 -0
  67. package/examples/recipes/extension-artifacts.ts +65 -0
  68. package/examples/recipes/extension-methods.ts +174 -0
  69. package/examples/recipes/extension.ts +37 -0
  70. package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
  71. package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
  72. package/examples/recipes/repository-skill/README.md +3 -0
  73. package/examples/recipes/run-inputs.ts +188 -0
  74. package/examples/recipes/run-options.ts +131 -0
  75. package/examples/recipes/run.ts +63 -0
  76. package/examples/recipes/runtime/empty.json +1 -0
  77. package/examples/recipes/runtime/goals.json +1 -0
  78. package/examples/recipes/runtime/mise.json +6 -0
  79. package/examples/recipes/runtime/path.json +1 -0
  80. package/examples/recipes/runtime/plugins.json +4 -0
  81. package/examples/recipes/runtime/python-uv.json +10 -0
  82. package/examples/recipes/semantic-judge.ts +29 -0
  83. package/examples/templates/coding/adapter.ts +17 -0
  84. package/examples/templates/coding/case.json +15 -0
  85. package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
  86. package/examples/templates/repository-skill/extension-data.ts +64 -0
  87. package/examples/templates/repository-skill/extension.ts +31 -0
  88. package/examples/templates/response/adapter.ts +12 -0
  89. package/examples/templates/response/case.json +13 -0
  90. package/package.json +39 -6
  91. package/schemas/report-v1.schema.json +9 -0
  92. package/schemas/run-evidence-v1.schema.json +2 -1
  93. package/schemas/runtime-v1.schema.json +193 -0
  94. package/src/advisory.ts +10 -5
  95. package/src/candidate-artifacts.ts +208 -0
  96. package/src/candidate-codex-state.ts +185 -0
  97. package/src/case-validation.ts +105 -0
  98. package/src/cli/cancellation.ts +20 -0
  99. package/src/cli/command.ts +81 -0
  100. package/src/cli/completion.ts +15 -0
  101. package/src/cli/demo.ts +59 -0
  102. package/src/cli/evaluation.ts +55 -0
  103. package/src/cli/extension.ts +79 -0
  104. package/src/cli/help.ts +222 -0
  105. package/src/cli/hosts.ts +59 -0
  106. package/src/cli/init-settings.ts +30 -0
  107. package/src/cli/init.ts +111 -0
  108. package/src/cli/invocation.ts +627 -0
  109. package/src/cli/loading.ts +151 -0
  110. package/src/cli/preparation.ts +163 -0
  111. package/src/cli/presentation.ts +91 -0
  112. package/src/cli/progress.ts +6 -0
  113. package/src/cli/run-manifest.ts +133 -0
  114. package/src/cli/run-options.ts +101 -0
  115. package/src/cli/run-overrides.ts +83 -0
  116. package/src/cli/settings.ts +160 -0
  117. package/src/cli/summary.ts +43 -0
  118. package/src/cli.ts +13 -843
  119. package/src/contract-types.ts +94 -0
  120. package/src/engine.ts +19 -2373
  121. package/src/evaluation/advisory.ts +150 -0
  122. package/src/evaluation/check-plan.ts +510 -0
  123. package/src/evaluation/collection.ts +88 -0
  124. package/src/evaluation/context.ts +10 -0
  125. package/src/evaluation/grading-host.ts +39 -0
  126. package/src/evaluation/grading.ts +258 -0
  127. package/src/evaluation/invocation-plan.ts +548 -0
  128. package/src/evaluation/plan.ts +736 -0
  129. package/src/evaluation/progress.ts +19 -0
  130. package/src/evaluation/resources.ts +241 -0
  131. package/src/evaluation/run.ts +206 -0
  132. package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +3 -1
  133. package/src/evaluation/semantic.ts +312 -0
  134. package/src/evaluation/trial.ts +866 -0
  135. package/src/evaluation/types.ts +207 -0
  136. package/src/evidence/host-results.ts +92 -0
  137. package/src/evidence/retained-artifacts.ts +18 -0
  138. package/src/evidence/trial-writer.ts +203 -0
  139. package/src/evidence/usage.ts +17 -0
  140. package/src/extension-client.ts +228 -148
  141. package/src/extension-session.ts +326 -195
  142. package/src/fixtures/advisory.ts +338 -0
  143. package/src/fixtures/bin.ts +57 -0
  144. package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
  145. package/src/fixtures/create.ts +187 -0
  146. package/src/fixtures/generated.ts +300 -0
  147. package/src/fixtures/git-hooks.ts +24 -0
  148. package/src/fixtures/preparation.ts +270 -0
  149. package/src/{repository-fixture.ts → fixtures/repository.ts} +207 -85
  150. package/src/fixtures/setup.ts +224 -0
  151. package/src/fixtures/tools.ts +88 -0
  152. package/src/fixtures/workspace-state.ts +212 -0
  153. package/src/generated/cli-result.cjs +2 -0
  154. package/src/generated/cli-result.d.cts +3 -0
  155. package/src/generated/extension.cjs +2 -0
  156. package/src/generated/extension.d.cts +3 -0
  157. package/src/generated/report.cjs +2 -0
  158. package/src/generated/report.d.cts +3 -0
  159. package/src/generated/run-evidence.cjs +2 -0
  160. package/src/generated/run-evidence.d.cts +3 -0
  161. package/src/generated/runtime.cjs +2 -0
  162. package/src/generated/runtime.d.cts +3 -0
  163. package/src/graders/artifact.ts +56 -23
  164. package/src/graders/git-head.ts +79 -75
  165. package/src/graders/output.ts +222 -140
  166. package/src/graders/semantic.ts +102 -43
  167. package/src/graders/shell.ts +379 -207
  168. package/src/host-diagnostic-redaction.ts +52 -0
  169. package/src/host-diagnostics.ts +319 -0
  170. package/src/hosts/claude-continuation.ts +63 -18
  171. package/src/hosts/claude-credential.ts +43 -62
  172. package/src/hosts/claude-events.ts +86 -51
  173. package/src/hosts/claude-keychain.ts +59 -0
  174. package/src/hosts/claude-native-goal.ts +57 -33
  175. package/src/hosts/claude-process.ts +111 -0
  176. package/src/hosts/claude-repository-invocation.ts +208 -117
  177. package/src/hosts/claude-settings.ts +240 -21
  178. package/src/hosts/claude-skill-activations.ts +166 -0
  179. package/src/hosts/claude-state.ts +403 -0
  180. package/src/hosts/claude-tool-calls.ts +171 -106
  181. package/src/hosts/claude.ts +561 -451
  182. package/src/hosts/codex-app-server.ts +326 -178
  183. package/src/hosts/codex-configuration.ts +44 -18
  184. package/src/hosts/codex-events.ts +97 -60
  185. package/src/hosts/codex-exec-controls.ts +516 -0
  186. package/src/hosts/codex-invocation.ts +70 -0
  187. package/src/hosts/codex-marketplace.ts +283 -0
  188. package/src/hosts/codex-native-calls.ts +546 -500
  189. package/src/hosts/codex-process.ts +202 -0
  190. package/src/hosts/codex-profile.ts +257 -28
  191. package/src/hosts/codex-session-files.ts +106 -0
  192. package/src/hosts/codex-state.ts +492 -0
  193. package/src/hosts/codex.ts +783 -832
  194. package/src/hosts/isolation-common.ts +94 -0
  195. package/src/hosts/isolation-roots.ts +54 -27
  196. package/src/hosts/isolation.ts +10 -0
  197. package/src/hosts/linux-sandbox.ts +159 -0
  198. package/src/hosts/mac-sandbox.ts +187 -61
  199. package/src/hosts/native-controls.ts +152 -62
  200. package/src/hosts/process-lifecycle.ts +37 -0
  201. package/src/hosts/protected-worktrees.ts +51 -19
  202. package/src/hosts/skill-hook-catalog.ts +131 -0
  203. package/src/hosts/skill-hook-handler.ts +374 -0
  204. package/src/hosts/skill-hooks.ts +580 -0
  205. package/src/hosts/workspace-fingerprint.ts +55 -35
  206. package/src/identity.ts +102 -48
  207. package/src/instrumentation.ts +109 -48
  208. package/src/native-transcript-state.ts +131 -0
  209. package/src/native-transcript-view.ts +90 -0
  210. package/src/native-transcripts.ts +286 -0
  211. package/src/process-lifecycle.ts +150 -0
  212. package/src/provenance.ts +161 -81
  213. package/src/prune.ts +215 -0
  214. package/src/report.ts +172 -140
  215. package/src/resolved-case.ts +33 -0
  216. package/src/results-lock.ts +29 -0
  217. package/src/results.ts +133 -86
  218. package/src/run-owner.ts +154 -63
  219. package/src/runtime/config.ts +456 -0
  220. package/src/runtime/discovery-apple.ts +90 -0
  221. package/src/runtime/discovery-providers.ts +63 -0
  222. package/src/runtime/discovery.ts +70 -0
  223. package/src/runtime/hook-processes.ts +97 -0
  224. package/src/runtime/hooks.ts +476 -0
  225. package/src/runtime/paths.ts +313 -0
  226. package/src/runtime/query.ts +85 -0
  227. package/src/runtime/seeds.ts +134 -0
  228. package/src/runtime/shared-cache.ts +143 -0
  229. package/src/runtime/state.ts +143 -0
  230. package/src/schema-types.ts +198 -0
  231. package/src/schema.ts +15 -19
  232. package/src/transcript-retention.ts +146 -0
  233. package/src/value-guards.ts +20 -0
  234. package/src/advisory-fixture.ts +0 -282
  235. package/src/fixture-bin.ts +0 -87
  236. package/src/fixture-setup.ts +0 -163
  237. package/src/generated-fixture.ts +0 -220
  238. package/src/git-hooks.ts +0 -63
  239. package/src/hosts/claude-nested-skills.ts +0 -218
  240. package/src/hosts/codex-command-output.ts +0 -399
  241. package/src/hosts/codex-executor-command.ts +0 -195
  242. package/src/hosts/codex-skill-reads.ts +0 -391
  243. package/src/preparation.ts +0 -211
@@ -1,7 +1,12 @@
1
+ import { isRecord } from "../value-guards";
1
2
  import { writeFile } from "node:fs/promises";
2
3
  import { join } from "node:path";
3
4
  import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
4
5
  import { createInterface } from "node:readline";
6
+ import {
7
+ HostProcessDiagnostics,
8
+ type HostDiagnostics,
9
+ } from "../host-diagnostics";
5
10
  interface HarnessRunRequest {
6
11
  repoDir: string;
7
12
  prompt: string;
@@ -93,26 +98,38 @@ function errorClassification(value: unknown): {
93
98
  const tag = (s: unknown): s is string =>
94
99
  typeof s === "string" && /^[A-Za-z][A-Za-z0-9]{0,79}$/.test(s);
95
100
  if (tag(value)) return { code: value };
96
- if (!value || typeof value !== "object" || Array.isArray(value))
97
- return { code: null };
101
+ if (!isRecord(value)) return { code: null };
102
+ return classifiedErrorEntry(value, tag);
103
+ }
104
+
105
+ function classifiedErrorEntry(
106
+ value: RecordValue,
107
+ tag: (value: unknown) => value is string,
108
+ ) {
98
109
  const entries = Object.entries(value);
99
- if (entries.length !== 1 || !tag(entries[0]?.[0])) return { code: null };
100
- const [code, details] = entries[0]!;
110
+ const [entry] = entries;
111
+ if (entries.length !== 1 || entry === undefined || !tag(entry[0]))
112
+ return { code: null };
113
+ const [code, details] = entry;
101
114
  return { code, ...errorHttpStatus(details) };
102
115
  }
103
116
 
117
+ function validHttpStatus(value: unknown): value is number {
118
+ return (
119
+ typeof value === "number" &&
120
+ Number.isInteger(value) &&
121
+ value >= 0 &&
122
+ value <= 65535
123
+ );
124
+ }
125
+
104
126
  function errorHttpStatus(details: unknown): { httpStatusCode?: number } {
105
127
  const status =
106
128
  details && typeof details === "object"
107
129
  ? (details as RecordValue).httpStatusCode
108
130
  : undefined;
109
131
  return {
110
- ...(typeof status === "number" &&
111
- Number.isInteger(status) &&
112
- status >= 0 &&
113
- status <= 65535
114
- ? { httpStatusCode: status }
115
- : {}),
132
+ ...(validHttpStatus(status) ? { httpStatusCode: status } : {}),
116
133
  };
117
134
  }
118
135
 
@@ -132,7 +149,7 @@ class AppServerRpc {
132
149
  {
133
150
  resolve: (value: RecordValue) => void;
134
151
  reject: (error: Error) => void;
135
- timer: ReturnType<typeof setTimeout>;
152
+ method: string;
136
153
  }
137
154
  >();
138
155
  private wake?: () => void;
@@ -142,9 +159,16 @@ class AppServerRpc {
142
159
  private reads: Promise<void>;
143
160
  private errors: Promise<void>;
144
161
  stderr = "";
162
+ readonly capture: HostProcessDiagnostics;
145
163
 
146
- constructor(argv: string[], cwd: string, env: Record<string, string>) {
147
- this.process = spawn(argv[0]!, argv.slice(1), {
164
+ constructor(
165
+ argv: string[],
166
+ cwd: string,
167
+ env: Record<string, string>,
168
+ diagnostics?: HostDiagnostics,
169
+ ) {
170
+ this.capture = new HostProcessDiagnostics(diagnostics);
171
+ this.process = spawn(identifier(argv[0]), argv.slice(1), {
148
172
  detached: process.platform !== "win32",
149
173
  cwd,
150
174
  env,
@@ -157,13 +181,22 @@ class AppServerRpc {
157
181
  resolve(null);
158
182
  });
159
183
  });
160
- this.reads = this.readLines().catch((error) => this.fail(error));
161
- this.process.stdin.on("error", (error) => this.fail(error));
184
+ this.process.stdout.on("data", (chunk: Buffer) => {
185
+ this.capture.stdout.add(chunk);
186
+ });
187
+ this.reads = this.readLines().catch((error: unknown) => {
188
+ this.fail(error);
189
+ });
190
+ this.process.stdin.on("error", (error) => {
191
+ this.fail(error);
192
+ });
162
193
  this.errors = new Promise((resolve) => {
163
- this.process.stderr.on("data", (chunk) => {
194
+ this.process.stderr.on("data", (chunk: Buffer) => {
195
+ this.capture.stderr.add(chunk);
164
196
  this.stderr = (this.stderr + String(chunk)).slice(-16000);
165
197
  });
166
198
  this.process.stderr.on("end", resolve);
199
+ this.process.stderr.on("close", resolve);
167
200
  this.process.stderr.on("error", (error) => {
168
201
  this.fail(error);
169
202
  resolve();
@@ -177,10 +210,7 @@ class AppServerRpc {
177
210
 
178
211
  fail(error: unknown): void {
179
212
  this.failure ??= error instanceof Error ? error : new Error(String(error));
180
- for (const pending of this.pending.values()) {
181
- clearTimeout(pending.timer);
182
- pending.reject(this.failure);
183
- }
213
+ for (const pending of this.pending.values()) pending.reject(this.failure);
184
214
  this.pending.clear();
185
215
  this.wake?.();
186
216
  }
@@ -196,6 +226,9 @@ class AppServerRpc {
196
226
  private readLines(): Promise<void> {
197
227
  return new Promise((resolve) => {
198
228
  const lines = createInterface({ input: this.process.stdout });
229
+ this.process.stdout.on("close", () => {
230
+ lines.close();
231
+ });
199
232
  lines.on("line", (line) => {
200
233
  try {
201
234
  this.receivedBytes += Buffer.byteLength(line);
@@ -216,31 +249,39 @@ class AppServerRpc {
216
249
  }
217
250
 
218
251
  private receive(message: RecordValue): void {
219
- if (typeof message.method === "string" && message.id !== undefined) {
220
- this.send({
221
- id: message.id,
222
- error: {
223
- code: -32601,
224
- message: "Unattended eval cannot answer this server request",
225
- },
226
- });
227
- throw new Error(`Unsupported app-server request: ${message.method}`);
228
- }
229
- if (typeof message.id === "number") {
230
- const pending = this.pending.get(message.id);
231
- if (!pending) throw new Error("Unmatched app-server response");
232
- // Validate before removing the waiter so malformed responses reject it.
233
- const value = record(message.error ?? message.result);
234
- this.pending.delete(message.id);
235
- clearTimeout(pending.timer);
236
- if (message.error)
237
- pending.reject(new Error(`App-server RPC failed (${value.code})`));
238
- else pending.resolve(value);
239
- } else if (typeof message.method === "string")
252
+ this.refuseServerRequest(message);
253
+ if (typeof message.id === "number")
254
+ this.receiveResponse(message, message.id);
255
+ else if (typeof message.method === "string")
240
256
  this.receiveNotification(message.method, message.params);
241
257
  else throw new Error("Malformed app-server message");
242
258
  }
243
259
 
260
+ private refuseServerRequest(message: RecordValue): void {
261
+ if (typeof message.method !== "string" || message.id === undefined) return;
262
+ this.send({
263
+ id: message.id,
264
+ error: {
265
+ code: -32601,
266
+ message: "Unattended eval cannot answer this server request",
267
+ },
268
+ });
269
+ throw new Error(`Unsupported app-server request: ${message.method}`);
270
+ }
271
+
272
+ private receiveResponse(message: RecordValue, id: number): void {
273
+ const pending = this.pending.get(id);
274
+ if (!pending) throw new Error("Unmatched app-server response");
275
+ // Validate before removing the waiter so malformed responses reject it.
276
+ const value = record(message.error ?? message.result);
277
+ this.pending.delete(id);
278
+ if (message.error)
279
+ pending.reject(
280
+ new Error(`App-server RPC failed (${String(value.code)})`),
281
+ );
282
+ else pending.resolve(value);
283
+ }
284
+
244
285
  private receiveNotification(method: string, value: unknown): void {
245
286
  // Drop reasoning, account metadata, deltas and private raw events at ingress.
246
287
  if (
@@ -256,28 +297,19 @@ class AppServerRpc {
256
297
  )
257
298
  return;
258
299
  const params = record(value);
259
- if (
260
- method === "item/completed" &&
261
- !["commandExecution", "collabAgentToolCall"].includes(
262
- String(record(params.item).type),
263
- )
264
- )
265
- return;
300
+ if (!allowedNotificationItem(method, params)) return;
266
301
  if (this.queue.length >= 1024)
267
302
  throw new Error("App-server notification queue exceeds limit");
268
303
  this.queue.push({ method, params });
269
304
  this.wake?.();
270
305
  }
271
306
 
307
+ /** The session's configured time bound is the only deadline, including startup. */
272
308
  async call(method: string, params: RecordValue): Promise<RecordValue> {
273
309
  if (this.failure) throw this.failure;
274
310
  const id = this.nextId++;
275
311
  return new Promise((resolve, reject) => {
276
- const timer = setTimeout(
277
- () => this.fail(new Error(`App-server RPC timeout: ${method}`)),
278
- 45000,
279
- );
280
- this.pending.set(id, { resolve, reject, timer });
312
+ this.pending.set(id, { resolve, reject, method });
281
313
  try {
282
314
  this.send({ id, method, params });
283
315
  } catch (error) {
@@ -286,6 +318,10 @@ class AppServerRpc {
286
318
  });
287
319
  }
288
320
 
321
+ pendingMethods(): string[] {
322
+ return Array.from(this.pending.values(), (pending) => pending.method);
323
+ }
324
+
289
325
  async initialize(): Promise<void> {
290
326
  await this.call("initialize", {
291
327
  clientInfo: { name: "sevro_eval", version: "0.1.0" },
@@ -301,7 +337,9 @@ class AppServerRpc {
301
337
  });
302
338
  this.wake = undefined;
303
339
  if (this.failure) throw this.failure;
304
- return this.queue.shift()!;
340
+ const notification = this.queue.shift();
341
+ if (!notification) throw new Error("Missing app-server notification");
342
+ return notification;
305
343
  }
306
344
 
307
345
  async close(): Promise<void> {
@@ -314,8 +352,12 @@ class AppServerRpc {
314
352
  if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
315
353
  }
316
354
  };
317
- const term = setTimeout(() => terminate("SIGTERM"), 5000);
318
- const kill = setTimeout(() => terminate("SIGKILL"), 6000);
355
+ const term = setTimeout(() => {
356
+ terminate("SIGTERM");
357
+ }, 5000);
358
+ const kill = setTimeout(() => {
359
+ terminate("SIGKILL");
360
+ }, 6000);
319
361
  try {
320
362
  await this.exited;
321
363
  await this.reads;
@@ -327,8 +369,17 @@ class AppServerRpc {
327
369
  }
328
370
  }
329
371
 
372
+ function allowedNotificationItem(method: string, params: RecordValue): boolean {
373
+ return (
374
+ method !== "item/completed" ||
375
+ ["commandExecution", "collabAgentToolCall"].includes(
376
+ String(record(params.item).type),
377
+ )
378
+ );
379
+ }
380
+
330
381
  /** Convert only public tool items used by the existing bounded evidence readers. */
331
- function toolEvent(value: unknown): unknown | undefined {
382
+ function toolEvent(value: unknown): unknown {
332
383
  const item = record(value);
333
384
  if (item.type === "commandExecution")
334
385
  return {
@@ -363,17 +414,12 @@ function toolEvent(value: unknown): unknown | undefined {
363
414
  reasoning_effort: item.reasoningEffort,
364
415
  },
365
416
  };
417
+ return undefined;
366
418
  }
367
419
 
368
420
  /** Require the exact terminal turn, not a previous checkpoint or child response. */
369
421
  export function appServerFinal(thread: unknown, turnId: string): string {
370
- const turns = record(thread).turns;
371
- if (!Array.isArray(turns))
372
- throw new Error("App-server thread has no retained turns");
373
- const matches = turns.map(record).filter((turn) => turn.id === turnId);
374
- if (matches.length !== 1 || matches[0]!.status !== "completed")
375
- throw new Error("Missing completed app-server turn");
376
- const items = matches[0]!.items;
422
+ const items = completedAppServerTurn(thread, turnId).items;
377
423
  if (!Array.isArray(items))
378
424
  throw new Error("Completed app-server turn has no items");
379
425
  const finals = items
@@ -389,6 +435,17 @@ export function appServerFinal(thread: unknown, turnId: string): string {
389
435
  return finals.map((item) => item.text).join("\n\n");
390
436
  }
391
437
 
438
+ function completedAppServerTurn(thread: unknown, turnId: string): RecordValue {
439
+ const turns: unknown = record(thread).turns;
440
+ if (!Array.isArray(turns))
441
+ throw new Error("App-server thread has no retained turns");
442
+ const matches = turns.map(record).filter((turn) => turn.id === turnId);
443
+ const [turn] = matches;
444
+ if (matches.length !== 1 || turn === undefined || turn.status !== "completed")
445
+ throw new Error("Missing completed app-server turn");
446
+ return turn;
447
+ }
448
+
392
449
  const GOAL_STATUSES = [
393
450
  "active",
394
451
  "paused",
@@ -404,22 +461,31 @@ export function appServerGoalStatus(
404
461
  ): string | null {
405
462
  if (value === null) return null;
406
463
  const goal = record(value);
464
+ requireGoalIdentity(goal, threadId);
465
+ requireGoalObjective(goal.objective);
466
+ return String(goal.status);
467
+ }
468
+
469
+ function requireGoalIdentity(goal: RecordValue, threadId: string): void {
407
470
  if (
408
471
  goal.threadId !== threadId ||
409
- !GOAL_STATUSES.includes(String(goal.status)) ||
410
- typeof goal.objective !== "string" ||
411
- !goal.objective.length ||
412
- [...goal.objective].length > 4000
472
+ typeof goal.status !== "string" ||
473
+ !GOAL_STATUSES.includes(goal.status)
413
474
  )
414
475
  throw new Error("Invalid native goal readback");
415
- return String(goal.status);
416
476
  }
417
-
418
- export function codexAppServerArgv(): string[] {
419
- return ["codex", "app-server", "--stdio"];
477
+ function requireGoalObjective(objective: unknown): void {
478
+ if (
479
+ typeof objective !== "string" ||
480
+ !objective.length ||
481
+ Array.from(objective).length > 4000
482
+ )
483
+ throw new Error("Invalid native goal readback");
420
484
  }
421
485
 
422
486
  export interface AppServerRunOptions {
487
+ diagnostics?: HostDiagnostics;
488
+ vettedHooks?: boolean;
423
489
  request: HarnessRunRequest;
424
490
  argv: string[];
425
491
  env: Record<string, string>;
@@ -466,6 +532,9 @@ class AppServerSession {
466
532
  config: {
467
533
  model_reasoning_effort: request.effort,
468
534
  default_permissions: this.options.permissionProfile,
535
+ ...(this.options.vettedHooks
536
+ ? { bypass_hook_trust: true, features: { hooks: true } }
537
+ : {}),
469
538
  },
470
539
  approvalPolicy: "never",
471
540
  allowProviderModelFallback: false,
@@ -511,21 +580,17 @@ class AppServerSession {
511
580
  params: RecordValue,
512
581
  source: "notification" | "readback" = "notification",
513
582
  ): void {
514
- const status = appServerGoalStatus(params.goal, this.threadId)!;
583
+ const status = requiredGoalStatus(params.goal, this.threadId);
515
584
  this.goalSeen = true;
516
585
  const observation = {
517
586
  status,
518
- characters: [...String(record(params.goal).objective)].length,
587
+ characters: Array.from(String(record(params.goal).objective)).length,
519
588
  turnId: typeof params.turnId === "string" ? params.turnId : null,
520
589
  at: new Date().toISOString(),
521
590
  source,
522
591
  };
523
592
  const previous = this.evidence.goals.at(-1);
524
- if (
525
- previous?.status !== status ||
526
- previous.turnId !== observation.turnId ||
527
- previous.source !== source
528
- )
593
+ if (!sameGoalObservation(previous, observation))
529
594
  this.evidence.goals.push(observation);
530
595
  }
531
596
 
@@ -546,23 +611,12 @@ class AppServerSession {
546
611
  willRetry: unknown,
547
612
  value: unknown,
548
613
  ): void {
549
- const error =
550
- value && typeof value === "object" && !Array.isArray(value)
551
- ? (value as RecordValue)
552
- : {};
553
614
  const errors = (this.evidence.errors ??= []);
554
615
  if (errors.length === 32) {
555
616
  errors.shift();
556
617
  this.evidence.errorsDropped = (this.evidence.errorsDropped ?? 0) + 1;
557
618
  }
558
- errors.push({
559
- source,
560
- threadId: this.threadId,
561
- turnId: typeof turnId === "string" ? turnId.slice(0, 200) : null,
562
- willRetry: typeof willRetry === "boolean" ? willRetry : null,
563
- ...publicErrorMessage(error.message),
564
- ...errorClassification(error.codexErrorInfo),
565
- });
619
+ errors.push(errorReceipt(source, this.threadId, turnId, willRetry, value));
566
620
  }
567
621
 
568
622
  private async onNotification({
@@ -609,10 +663,24 @@ class AppServerSession {
609
663
 
610
664
  private async onCompletedTurn(turn: RecordValue): Promise<boolean> {
611
665
  const turnId = this.observeTurn(turn);
612
- if (turn.status !== "completed") {
613
- this.observeError("turn/completed", turnId, null, turn.error);
614
- throw new Error(`App-server turn ${String(turn.status)}`);
615
- }
666
+ this.requireCompletedTurn(turn, turnId);
667
+ const status = await this.completedGoal(turnId);
668
+ this.emit({
669
+ type: "turn.completed",
670
+ ...(this.usage ? { usage: this.usage } : {}),
671
+ });
672
+ if (this.followUp !== undefined)
673
+ return this.deliverFeedback(turnId, status, this.followUp);
674
+ return status !== "active" && (await this.saveFinal(turnId));
675
+ }
676
+
677
+ private requireCompletedTurn(turn: RecordValue, turnId: string): void {
678
+ if (turn.status === "completed") return;
679
+ this.observeError("turn/completed", turnId, null, turn.error);
680
+ throw new Error(`App-server turn ${String(turn.status)}`);
681
+ }
682
+
683
+ private async completedGoal(turnId: string): Promise<string | null> {
616
684
  const goal = await this.rpc.call("thread/goal/get", {
617
685
  threadId: this.threadId,
618
686
  });
@@ -622,32 +690,32 @@ class AppServerSession {
622
690
  throw new Error("Observed native goal disappeared");
623
691
  if (status !== null)
624
692
  this.observeGoal({ goal: goal.goal, turnId }, "readback");
693
+ return status;
694
+ }
695
+
696
+ private async deliverFeedback(
697
+ turnId: string,
698
+ status: string | null,
699
+ prompt: string,
700
+ ): Promise<boolean> {
701
+ // User input does not require a terminal goal. Preserve the completed
702
+ // response and actual boundary before delivering the declared follow-up.
703
+ if (!(await this.saveFinal(turnId, true))) return false;
704
+ const boundary = protocolMessage(
705
+ await this.options.followUpBoundary(this.threadId, {
706
+ nativeGoalObserved: this.goalSeen,
707
+ nativeGoalStatus: status,
708
+ }),
709
+ );
710
+ if (!this.responseBoundaryReady(turnId)) return false;
625
711
  this.emit({
626
- type: "turn.completed",
627
- ...(this.usage ? { usage: this.usage } : {}),
712
+ ...boundary,
713
+ native_goal_observed: this.goalSeen,
714
+ native_goal_status: status,
628
715
  });
629
- if (this.followUp !== undefined) {
630
- // User input does not require a terminal goal. Preserve the completed
631
- // response and actual boundary before delivering the declared follow-up.
632
- if (!(await this.saveFinal(turnId, true))) return false;
633
- const boundary = protocolMessage(
634
- await this.options.followUpBoundary(this.threadId, {
635
- nativeGoalObserved: this.goalSeen,
636
- nativeGoalStatus: status,
637
- }),
638
- );
639
- if (!this.responseBoundaryReady(turnId)) return false;
640
- this.emit({
641
- ...boundary,
642
- native_goal_observed: this.goalSeen,
643
- native_goal_status: status,
644
- });
645
- const prompt = this.followUp;
646
- this.followUp = undefined;
647
- await this.startUserTurn(prompt);
648
- return false;
649
- }
650
- return status !== "active" && (await this.saveFinal(turnId));
716
+ this.followUp = undefined;
717
+ await this.startUserTurn(prompt);
718
+ return false;
651
719
  }
652
720
 
653
721
  private async saveFinal(
@@ -658,18 +726,10 @@ class AppServerSession {
658
726
  threadId: this.threadId,
659
727
  includeTurns: true,
660
728
  });
661
- const thread = record(snapshot.thread);
662
- if (thread.id !== this.threadId)
663
- throw new Error("App-server returned another thread");
664
- const turns = thread.turns;
665
- if (!Array.isArray(turns)) throw new Error("Missing app-server history");
729
+ const { thread, turns } = retainedThread(snapshot, this.threadId);
666
730
  const latest = record(turns.at(-1));
667
731
  // A later native turn can start while readback is pending. Wait for it.
668
- if (latest.id !== turnId) {
669
- if (latest.status === "inProgress" || latest.status === "completed")
670
- return false;
671
- throw new Error("Later app-server turn did not complete");
672
- }
732
+ if (!latestTurnReady(latest, turnId)) return false;
673
733
  const final = appServerFinal(thread, turnId);
674
734
  for (const value of turns) this.backfillItems(record(value));
675
735
  await writeFile(
@@ -691,21 +751,21 @@ class AppServerSession {
691
751
  this.rpc.assertHealthy();
692
752
  let ready = true;
693
753
  for (const event of this.rpc.queue) {
694
- const { method, params } = event;
754
+ const { params } = event;
695
755
  if (params.threadId !== this.threadId) continue;
696
- if (method === "error" || method === "thread/goal/cleared")
697
- this.checkPendingSettlement(event);
698
- if (method === "thread/goal/updated")
699
- appServerGoalStatus(params.goal, this.threadId);
700
- if (
701
- ["turn/started", "turn/completed"].includes(method) &&
702
- record(params.turn).id !== turnId
703
- )
704
- ready = false;
756
+ this.validateResponseEvent(event);
757
+ if (laterRootTurn(event, turnId)) ready = false;
705
758
  }
706
759
  return ready;
707
760
  }
708
761
 
762
+ private validateResponseEvent(event: Notification): void {
763
+ if (event.method === "error" || event.method === "thread/goal/cleared")
764
+ this.checkPendingSettlement(event);
765
+ if (event.method === "thread/goal/updated")
766
+ appServerGoalStatus(event.params.goal, this.threadId);
767
+ }
768
+
709
769
  /** Readback and file I/O can race with a terminal protocol event. */
710
770
  assertSettled(): void {
711
771
  this.rpc.assertHealthy();
@@ -716,22 +776,29 @@ class AppServerSession {
716
776
  }
717
777
 
718
778
  private checkPendingSettlement({ method, params }: Notification): void {
779
+ this.requireNonfatalPendingError(method, params);
780
+ if (method === "thread/goal/cleared")
781
+ throw new Error("Native goal cleared during settlement");
782
+ this.requireSettledGoal(method, params);
783
+ if (laterRootTurn({ method, params }, this.evidence.finalTurnId))
784
+ throw new Error("Another root turn appeared during settlement");
785
+ }
786
+
787
+ private requireNonfatalPendingError(
788
+ method: string,
789
+ params: RecordValue,
790
+ ): void {
719
791
  if (method === "error" && params.willRetry !== true) {
720
792
  this.observeError("error", params.turnId, params.willRetry, params.error);
721
793
  throw new Error("App-server reported a fatal error during settlement");
722
794
  }
723
- if (method === "thread/goal/cleared")
724
- throw new Error("Native goal cleared during settlement");
795
+ }
796
+ private requireSettledGoal(method: string, params: RecordValue): void {
725
797
  if (
726
798
  method === "thread/goal/updated" &&
727
799
  appServerGoalStatus(params.goal, this.threadId) === "active"
728
800
  )
729
801
  throw new Error("Native goal became active during settlement");
730
- if (
731
- ["turn/started", "turn/completed"].includes(method) &&
732
- record(params.turn).id !== this.evidence.finalTurnId
733
- )
734
- throw new Error("Another root turn appeared during settlement");
735
802
  }
736
803
 
737
804
  private backfillItems(turn: RecordValue): void {
@@ -747,55 +814,136 @@ class AppServerSession {
747
814
  }
748
815
  }
749
816
 
750
- export async function runCodexAppServer(options: AppServerRunOptions): Promise<{
751
- out: string;
752
- err: string;
753
- code: number;
754
- evidence: AppServerEvidence;
755
- }> {
756
- const { request, argv, env } = options;
757
- const bound = request.control?.appServerTimeoutMs ?? 3600000;
817
+ function requiredGoalStatus(value: unknown, threadId: string): string {
818
+ const status = appServerGoalStatus(value, threadId);
819
+ if (status === null) throw new Error("Malformed app-server object");
820
+ return status;
821
+ }
822
+ function sameGoalObservation(
823
+ previous: AppServerEvidence["goals"][number] | undefined,
824
+ observation: AppServerEvidence["goals"][number],
825
+ ): boolean {
826
+ return (
827
+ previous?.status === observation.status &&
828
+ previous.turnId === observation.turnId &&
829
+ previous.source === observation.source
830
+ );
831
+ }
832
+ function errorReceipt(
833
+ source: AppServerErrorEvidence["source"],
834
+ threadId: string,
835
+ turnId: unknown,
836
+ willRetry: unknown,
837
+ value: unknown,
838
+ ): AppServerErrorEvidence {
839
+ const error = isRecord(value) ? value : {};
840
+ return {
841
+ source,
842
+ threadId,
843
+ turnId: typeof turnId === "string" ? turnId.slice(0, 200) : null,
844
+ willRetry: typeof willRetry === "boolean" ? willRetry : null,
845
+ ...publicErrorMessage(error.message),
846
+ ...errorClassification(error.codexErrorInfo),
847
+ };
848
+ }
849
+ function retainedThread(snapshot: RecordValue, threadId: string) {
850
+ const thread = record(snapshot.thread);
851
+ if (thread.id !== threadId)
852
+ throw new Error("App-server returned another thread");
853
+ const turns: unknown = thread.turns;
854
+ if (!Array.isArray(turns)) throw new Error("Missing app-server history");
855
+ return { thread, turns: turns as unknown[] };
856
+ }
857
+ function latestTurnReady(latest: RecordValue, turnId: string): boolean {
858
+ if (latest.id === turnId) return true;
859
+ if (latest.status === "inProgress" || latest.status === "completed")
860
+ return false;
861
+ throw new Error("Later app-server turn did not complete");
862
+ }
863
+ function laterRootTurn(
864
+ event: Notification,
865
+ turnId: string | undefined,
866
+ ): boolean {
867
+ return (
868
+ ["turn/started", "turn/completed"].includes(event.method) &&
869
+ record(event.params.turn).id !== turnId
870
+ );
871
+ }
872
+ function requireAppServerRun(request: HarnessRunRequest, bound: number): void {
758
873
  if (request.signal?.aborted) throw new Error("Codex run cancelled");
759
874
  if (!Number.isFinite(bound) || bound <= 0)
760
875
  throw new Error("Invalid app-server time bound");
761
- const rpc = new AppServerRpc(argv, request.repoDir, env);
762
- const session = new AppServerSession(rpc, options);
763
- const cancel = () => rpc.fail(new Error("Codex run cancelled"));
876
+ }
877
+ function sessionFailure(session: AppServerSession, error: unknown): void {
878
+ if (session.options.diagnostics) session.options.diagnostics.failure = error;
879
+ session.evidence.failure =
880
+ error instanceof Error ? error.message : String(error);
881
+ session.emit({ type: "turn.failed" });
882
+ }
883
+ /** Name an unanswered request so a stalled startup stays distinguishable. */
884
+ function timeBoundFailure(pending: string[]): string {
885
+ const reached = "App-server evaluation time bound reached";
886
+ return pending.length ? `${reached} awaiting ${pending.join(", ")}` : reached;
887
+ }
888
+ async function completeSession(
889
+ session: AppServerSession,
890
+ rpc: AppServerRpc,
891
+ request: HarnessRunRequest,
892
+ bound: number,
893
+ ): Promise<number> {
894
+ const cancel = () => {
895
+ rpc.fail(new Error("Codex run cancelled"));
896
+ };
764
897
  request.signal?.addEventListener("abort", cancel, { once: true });
765
- const deadline = setTimeout(
766
- () => rpc.fail(new Error("App-server evaluation time bound reached")),
767
- bound,
768
- );
769
- let code = 1;
898
+ const deadline = setTimeout(() => {
899
+ rpc.fail(new Error(timeBoundFailure(rpc.pendingMethods())));
900
+ }, bound);
770
901
  try {
771
902
  await session.run();
772
903
  session.assertSettled();
773
- code = 0;
904
+ return 0;
774
905
  } catch (error) {
775
- session.evidence.failure =
776
- error instanceof Error ? error.message : String(error);
777
- session.emit({ type: "turn.failed" });
906
+ sessionFailure(session, error);
907
+ return 1;
778
908
  } finally {
779
909
  clearTimeout(deadline);
780
910
  request.signal?.removeEventListener("abort", cancel);
781
911
  await rpc.close();
782
912
  }
783
- // Flush stdout before accepting success: malformed trailing data and queued
784
- // terminal errors must not be hidden by a successful readback response.
785
- if (code === 0) {
786
- try {
787
- session.assertSettled();
788
- } catch (error) {
789
- code = 1;
790
- session.evidence.failure =
791
- error instanceof Error ? error.message : String(error);
792
- session.emit({ type: "turn.failed" });
793
- }
913
+ }
914
+ function settlementCode(session: AppServerSession, code: number): number {
915
+ if (code !== 0) return code;
916
+ try {
917
+ session.assertSettled();
918
+ return 0;
919
+ } catch (error) {
920
+ sessionFailure(session, error);
921
+ return 1;
794
922
  }
923
+ }
924
+
925
+ export async function runCodexAppServer(options: AppServerRunOptions): Promise<{
926
+ out: string;
927
+ err: string;
928
+ code: number;
929
+ evidence: AppServerEvidence;
930
+ }> {
931
+ const { request, argv, env } = options;
932
+ const bound = request.control?.appServerTimeoutMs ?? 3600000;
933
+ requireAppServerRun(request, bound);
934
+ const rpc = new AppServerRpc(argv, request.repoDir, env, options.diagnostics);
935
+ const session = new AppServerSession(rpc, options);
936
+ const code = await completeSession(session, rpc, request, bound);
937
+ // Flush stdout before accepting success, including trailing data and terminal errors.
938
+ const finalCode = settlementCode(session, code);
939
+ await rpc.capture.record(
940
+ { exited: rpc.exited, signalCode: rpc.process.signalCode },
941
+ finalCode === 0 ? "exit" : "error",
942
+ );
795
943
  return {
796
944
  out: session.out.join("\n"),
797
945
  err: rpc.stderr,
798
- code,
946
+ code: finalCode,
799
947
  evidence: session.evidence,
800
948
  };
801
949
  }