@niwa-labs/harness-codex 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1239 @@
1
+ /**
2
+ * The process discipline of the Codex control side, against a fake `codex app-server`: the
3
+ * requests of the server are answered, a dead child fails every pending call, a turn that did
4
+ * not complete is not a success, and no process of the child's group outlives the control side.
5
+ */
6
+
7
+ import { afterEach, beforeEach, expect, test } from "bun:test";
8
+ import { chmod, readFile, rm, writeFile } from "node:fs/promises";
9
+ import { join } from "node:path";
10
+ import { fileURLToPath } from "node:url";
11
+
12
+ import { z } from "zod";
13
+
14
+ import { HarnessIdSchema, InputNotSent, InputPossiblySent, modelName, promptFromText, sessionId, type LifeEvent, Unavailable, Unsupported } from "@niwa-labs/harness";
15
+ import type { SessionId } from "@niwa-labs/harness";
16
+ import { codexControl } from "../src/control.ts";
17
+ import type { CodexControlOptions, CodexSessionControl } from "../src/control.ts";
18
+ import { codex, codexKind } from "../src/index.ts";
19
+ import { onBoxManifest } from "@niwa-labs/sandbox";
20
+ import { eventually, tempDir, unconfined } from "@niwa-labs/testing";
21
+
22
+ const THREAD = "11111111-2222-3333-4444-555555555555";
23
+
24
+ /** The frames the fake sends, built as the real server builds them; the fake script imports them by path. */
25
+ const FRAMES = fileURLToPath(new URL("./fixtures/codex-frames.ts", import.meta.url));
26
+
27
+ /**
28
+ * A fake server. FAKE_MODE: `ok`, `approval`, `die` (exits on turn/start), `fail` (turn fails),
29
+ * `hold` (every turn waits for a release), `novel` (says something the schemas do not know, and goes
30
+ * on), `novel-request` (asks something the schemas do not know, and goes on once it is answered),
31
+ * `drift` (the turn ends with a status the schemas do not know), `drift-message` (the agent's final
32
+ * message carries a field the schemas do not know), `ask-user` (the turn asks the questions in
33
+ * FAKE_QUESTIONS and ends once they are answered), `refuse-first` (the first turn is refused after a
34
+ * short wait, the later ones run), `announce-silent` (a turn is announced and never answered nor ended),
35
+ * `silent` (a turn is neither answered nor announced).
36
+ *
37
+ * The requests that come in are written to `requests.log`, the notifications that go out to
38
+ * `events.log`, so a test can see what was asked and in which order it happened.
39
+ */
40
+ const FAKE = `#!/usr/bin/env bun
41
+ import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
42
+ import { spawn } from "node:child_process";
43
+ import { agentMessageCompleted, commandApprovalParams, serverRequest, turnCompleted, turnStarted } from ${JSON.stringify(FRAMES)};
44
+ const THREAD = "${THREAD}";
45
+ const dir = process.env.FAKE_DIR;
46
+ const mode = process.env.FAKE_MODE;
47
+ appendFileSync(dir + "/spawns.log", JSON.stringify({ argv: process.argv.slice(2), home: process.env.CODEX_HOME ?? null }) + "\\n");
48
+ const grandchild = spawn("sleep", ["300"], { stdio: "ignore" });
49
+ writeFileSync(dir + "/grandchild.pid", String(grandchild.pid));
50
+ const send = (frame) => process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...frame }) + "\\n");
51
+ const notify = (method, params) => {
52
+ appendFileSync(dir + "/events.log", JSON.stringify({ method, params }) + "\\n");
53
+ send({ method, params });
54
+ };
55
+ const emit = (frame) => notify(frame.method, frame.params);
56
+ const finish = (status, id) => {
57
+ emit(agentMessageCompleted(THREAD, id, process.env.FAKE_TEXT ?? "ok"));
58
+ emit(turnCompleted(THREAD, id, status, status === "failed" ? "boom" : undefined));
59
+ };
60
+ // The turns that wait for a release, and the turns that were released.
61
+ let pending = [];
62
+ let released = 0;
63
+ let turns = 0;
64
+ let refused = false;
65
+ let staleSent = false;
66
+ setInterval(() => {
67
+ if (mode !== "hold") return;
68
+ // A late end of a turn that is not the running one, when the test asks for it.
69
+ if (!staleSent) {
70
+ try {
71
+ readFileSync(dir + "/stale", "utf8");
72
+ staleSent = true;
73
+ emit(turnCompleted(THREAD, "old-turn", "completed"));
74
+ } catch {}
75
+ }
76
+ let wanted = 0;
77
+ try {
78
+ wanted = Number(readFileSync(dir + "/release", "utf8"));
79
+ } catch {
80
+ return;
81
+ }
82
+ if (wanted > released && pending.length > 0) {
83
+ released += 1;
84
+ finish("completed", pending.shift());
85
+ }
86
+ }, 20);
87
+ let buffer = "";
88
+ process.stdin.on("data", (chunk) => {
89
+ buffer += chunk;
90
+ for (let nl = buffer.indexOf("\\n"); nl !== -1; nl = buffer.indexOf("\\n")) {
91
+ const frame = JSON.parse(buffer.slice(0, nl));
92
+ buffer = buffer.slice(nl + 1);
93
+ if (frame.method !== undefined) {
94
+ appendFileSync(dir + "/requests.log", JSON.stringify({ method: frame.method, params: frame.params }) + "\\n");
95
+ }
96
+ if (frame.method === "initialize") send({ id: frame.id, result: {} });
97
+ else if (frame.method === "thread/start") send({ id: frame.id, result: { thread: { id: "${THREAD}" }, model: process.env.FAKE_MODEL } });
98
+ else if (frame.method === "thread/resume") send({ id: frame.id, result: { thread: { id: frame.params.threadId }, model: process.env.FAKE_MODEL } });
99
+ else if (frame.method === "turn/start") {
100
+ if (mode === "die") process.exit(1);
101
+ if (mode === "refuse-first" && !refused) {
102
+ refused = true;
103
+ setTimeout(() => send({ id: frame.id, error: { code: -32600, message: "the turn was refused" } }), 300);
104
+ continue;
105
+ }
106
+ if (mode === "silent") continue;
107
+ if (mode === "announce-silent") {
108
+ turns += 1;
109
+ emit(turnStarted(THREAD, "t" + turns));
110
+ continue;
111
+ }
112
+ if (mode === "refuse-turn") {
113
+ send({ id: frame.id, error: { code: -32600, message: "the turn was refused" } });
114
+ continue;
115
+ }
116
+ turns += 1;
117
+ const turnId = "t" + turns;
118
+ send({ id: frame.id, result: { turn: { id: turnId } } });
119
+ emit(turnStarted(THREAD, turnId));
120
+ if (process.env.FAKE_REROUTE) notify("model/rerouted", { threadId: "${THREAD}", turnId, fromModel: process.env.FAKE_MODEL, toModel: process.env.FAKE_REROUTE, reason: "highRiskCyberActivity" });
121
+ if (mode === "approval") send(serverRequest("srv1", "item/commandExecution/requestApproval", THREAD, turnId, commandApprovalParams({ command: "ls" })));
122
+ else if (mode === "ask-user") send(serverRequest("srv1", "item/tool/requestUserInput", THREAD, turnId, { isBlocking: true, itemId: "call-2", questions: JSON.parse(process.env.FAKE_QUESTIONS) }));
123
+ else if (mode === "novel-request") send({ id: "srv2", method: "novel/ask", params: { why: "because" } });
124
+ else if (mode === "hold") pending.push(turnId);
125
+ else if (mode === "drift-message") {
126
+ const message = agentMessageCompleted(THREAD, turnId, process.env.FAKE_TEXT ?? "ok");
127
+ message.params.item.moderation = { from: "the future" };
128
+ emit(message);
129
+ emit(turnCompleted(THREAD, turnId, "completed"));
130
+ } else if (mode === "drift") {
131
+ emit(agentMessageCompleted(THREAD, turnId, "ok"));
132
+ const ended = turnCompleted(THREAD, turnId, "completed");
133
+ ended.params.turn.status = "paused";
134
+ emit(ended);
135
+ } else {
136
+ if (mode === "novel") notify("novel/thing", { n: 1 });
137
+ finish(mode === "fail" ? "failed" : "completed", turnId);
138
+ }
139
+ } else if (frame.method === "turn/steer") send({ id: frame.id, result: {} });
140
+ else if (frame.id === "srv1" || frame.id === "srv2") {
141
+ appendFileSync(dir + "/reply.json", JSON.stringify(frame));
142
+ finish("completed", "t1");
143
+ }
144
+ }
145
+ });
146
+ `;
147
+
148
+ let dir: string;
149
+
150
+ beforeEach(async () => {
151
+ dir = await tempDir("codex-fake");
152
+ const binary = join(dir, "codex");
153
+ await writeFile(binary, FAKE);
154
+ await chmod(binary, 0o755);
155
+ process.env.FAKE_DIR = dir;
156
+ });
157
+
158
+ afterEach(async () => {
159
+ // Every control side a test made is disposed, so no fake server outlives its test.
160
+ for (const made of controls.splice(0)) await made.dispose().catch(() => undefined);
161
+ delete process.env.FAKE_DIR;
162
+ delete process.env.FAKE_MODE;
163
+ delete process.env.FAKE_TEXT;
164
+ delete process.env.FAKE_MODEL;
165
+ delete process.env.FAKE_REROUTE;
166
+ delete process.env.FAKE_QUESTIONS;
167
+ await rm(dir, { recursive: true, force: true });
168
+ });
169
+
170
+ /** The control sides made by the test that runs; they are disposed after it. */
171
+ const controls: Array<ReturnType<typeof codexControl>> = [];
172
+
173
+ function control(extra: Partial<CodexControlOptions> = {}) {
174
+ const made = codexControl({ harness: "codex-fake", binary: join(dir, "codex"), turnTimeoutMs: 10_000, ...extra });
175
+ controls.push(made);
176
+ return made;
177
+ }
178
+
179
+ async function grandchildGone(): Promise<boolean> {
180
+ const pid = Number(await readFile(join(dir, "grandchild.pid"), "utf8"));
181
+ for (let attempt = 0; attempt < 50; attempt += 1) {
182
+ try {
183
+ process.kill(pid, 0);
184
+ } catch {
185
+ return true;
186
+ }
187
+ await Bun.sleep(50);
188
+ }
189
+ return false;
190
+ }
191
+
192
+ /** The requests the fake server was asked for, in the order they came. */
193
+ async function requests(): Promise<{ method: string; params: Record<string, unknown> }[]> {
194
+ return (await lines("requests.log")).map(
195
+ (line) => JSON.parse(line) as { method: string; params: Record<string, unknown> },
196
+ );
197
+ }
198
+
199
+ /** The processes the fake server was started as, in the order they came: their arguments and the home they were given. */
200
+ async function spawns(): Promise<{ argv: string[]; home: string | null }[]> {
201
+ return (await lines("spawns.log")).map((line) => JSON.parse(line) as { argv: string[]; home: string | null });
202
+ }
203
+
204
+ /** The notifications the fake server sent, in the order they went out. */
205
+ async function events(): Promise<{ method: string; params: Record<string, unknown> }[]> {
206
+ return (await lines("events.log")).map(
207
+ (line) => JSON.parse(line) as { method: string; params: Record<string, unknown> },
208
+ );
209
+ }
210
+
211
+ /** The lines of a log of the fake server; a log that is not there yet has none. */
212
+ async function lines(name: string): Promise<string[]> {
213
+ const text = await readFile(join(dir, name), "utf8").catch(() => "");
214
+ return text.split("\n").filter((line) => line.trim() !== "");
215
+ }
216
+
217
+ /** The text of every turn that was started, in the order the turns came. */
218
+ async function startedTurns(): Promise<string[]> {
219
+ const asked = await requests();
220
+ return asked
221
+ .filter((frame) => frame.method === "turn/start")
222
+ .map((frame) => {
223
+ const input = frame.params.input as { type: string; text: string }[];
224
+ return input.map((item) => item.text).join("");
225
+ });
226
+ }
227
+
228
+ /** Let the turn that waits be released, so the server completes it. */
229
+ async function release(count: number): Promise<void> {
230
+ await writeFile(join(dir, "release"), String(count));
231
+ }
232
+
233
+ /** Wait until the condition holds, so a test does not depend on a fixed delay. */
234
+ async function until(check: () => Promise<boolean>, what: string): Promise<void> {
235
+ for (let attempt = 0; attempt < 100; attempt += 1) {
236
+ if (await check()) return;
237
+ await Bun.sleep(50);
238
+ }
239
+ throw new Error(`the condition did not come: ${what}`);
240
+ }
241
+
242
+ test("a request of the server is answered, and the group dies with dispose", async () => {
243
+ process.env.FAKE_MODE = "approval";
244
+ const codex = control();
245
+ const { session: id } = await codex.start({ prompt: promptFromText("go") });
246
+ expect(String(id.native)).toBe(THREAD);
247
+ // The request comes in the first turn, which runs on after the start.
248
+ await until(async () => (await codex.state(id)) === "idle", "the first turn ended");
249
+ expect(JSON.parse(await readFile(join(dir, "reply.json"), "utf8")).result.decision).toBe("decline");
250
+ expect(await codex.state(id)).toBe("idle");
251
+
252
+ await codex.dispose();
253
+ expect(await codex.state(id)).toBe("unknown");
254
+ expect(await grandchildGone()).toBe(true);
255
+ });
256
+
257
+ /** The signs of life the first stream of life sees, up to the first turn that ends. */
258
+ async function signsOfFirstTurn(side: { life(id: SessionId): AsyncIterable<LifeEvent> }, id: SessionId): Promise<LifeEvent[]> {
259
+ const signs: LifeEvent[] = [];
260
+ for await (const event of side.life(id)) {
261
+ signs.push(event);
262
+ if (event.kind === "turn-ended") break;
263
+ }
264
+ return signs;
265
+ }
266
+
267
+ /** Run a first turn that asks the questions, with a person who answers every one with "blue"; what the person was asked, and what Codex was told. */
268
+ async function askedPerson(questions: readonly object[]): Promise<{ asked: string[]; reply: unknown }> {
269
+ process.env.FAKE_MODE = "ask-user";
270
+ process.env.FAKE_QUESTIONS = JSON.stringify(questions);
271
+ const asked: string[] = [];
272
+ const codex = control();
273
+ const { session: id } = await codex.start({
274
+ prompt: promptFromText("go"),
275
+ reverse: async (_session, { question }) => {
276
+ asked.push(question.title);
277
+ return { text: "blue" };
278
+ },
279
+ });
280
+ await until(async () => (await codex.state(id)) === "idle", "the first turn ended");
281
+ return { asked, reply: JSON.parse(await readFile(join(dir, "reply.json"), "utf8")).result };
282
+ }
283
+
284
+ test("a question of Codex is put to the person, and the answer goes back under its id", async () => {
285
+ const { asked, reply } = await askedPerson([{ id: "color", header: "", question: "Which color?", isOther: false, isSecret: false }]);
286
+ expect(asked).toEqual(["Which color?"]);
287
+ expect(reply).toEqual({ answers: { color: { answers: ["blue"] } } });
288
+ });
289
+
290
+ test("a request with a secret question is refused: the person is not asked, and no answer is kept", async () => {
291
+ const { asked, reply } = await askedPerson([
292
+ { id: "color", header: "", question: "Which color?", isOther: false, isSecret: false },
293
+ { id: "token", header: "", question: "Your token?", isOther: false, isSecret: true },
294
+ ]);
295
+ expect(asked).toEqual([]);
296
+ expect(reply).toEqual({ answers: {} });
297
+ });
298
+
299
+ test("a notification that is not in the snapshot is a typed sign of life that carries the message, and the turn goes on", async () => {
300
+ process.env.FAKE_MODE = "novel";
301
+ const codex = control();
302
+ const { session: id } = await codex.start({ prompt: promptFromText("go") });
303
+ const signs = await signsOfFirstTurn(codex, id);
304
+ const unknown = signs.filter((sign) => sign.kind === "unknown-variant");
305
+ expect(unknown).toHaveLength(1);
306
+ expect(unknown[0]).toMatchObject({
307
+ kind: "unknown-variant",
308
+ protocol: "codex app-server",
309
+ variant: "novel/thing",
310
+ raw: { method: "novel/thing", params: { n: 1 } },
311
+ });
312
+ expect(signs.map((sign) => sign.kind).filter((kind) => kind !== "activity")).toEqual(["turn-started", "unknown-variant", "turn-ended"]);
313
+ expect(await codex.state(id)).toBe("idle");
314
+ });
315
+
316
+ test("a request that is not in the snapshot is answered with an error, so the turn does not wait for it", async () => {
317
+ process.env.FAKE_MODE = "novel-request";
318
+ const codex = control();
319
+ const { session: id } = await codex.start({ prompt: promptFromText("go") });
320
+ const signs = await signsOfFirstTurn(codex, id);
321
+ expect(signs.find((sign) => sign.kind === "unknown-variant")).toMatchObject({ variant: "novel/ask", raw: { id: "srv2", method: "novel/ask" } });
322
+ const reply = JSON.parse(await readFile(join(dir, "reply.json"), "utf8"));
323
+ expect(reply).toMatchObject({ id: "srv2", error: { code: -32601 } });
324
+ });
325
+
326
+ test("a turn that ends with a status that is not in the snapshot is not a success, and does not wait for its timeout", async () => {
327
+ process.env.FAKE_MODE = "drift";
328
+ const codex = control();
329
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
330
+ const failure = await codex.ask(id, { schema: z.string(), prompt: promptFromText("question") }).catch((error: unknown) => error);
331
+ expect(failure).toBeInstanceOf(Unavailable);
332
+ expect(String(failure)).toContain("the turn paused");
333
+ expect(await codex.state(id)).toBe("idle");
334
+ });
335
+
336
+ test("the answer of a turn is still read when its message carries a field the snapshot does not know", async () => {
337
+ process.env.FAKE_MODE = "drift-message";
338
+ process.env.FAKE_TEXT = '{"v":7}';
339
+ const codex = control();
340
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
341
+ const signs: LifeEvent[] = [];
342
+ void (async () => {
343
+ for await (const event of codex.life(id)) signs.push(event);
344
+ })();
345
+ const answer = await codex.ask(id, { schema: z.object({ v: z.number() }), prompt: promptFromText("question") });
346
+ expect(answer.value).toEqual({ v: 7 });
347
+ await eventually(() => signs.some((sign) => sign.kind === "unknown-variant"), 10_000, "the unknown-variant sign");
348
+ expect(signs.find((sign) => sign.kind === "unknown-variant")).toMatchObject({ variant: "item/completed" });
349
+ });
350
+
351
+ test("a child that dies with the first prompt unanswered leaves a named session without a receipt and no orphan", async () => {
352
+ process.env.FAKE_MODE = "die";
353
+ const codex = control();
354
+ // turn/start was written and never answered: Codex may have begun the turn, so the thread it
355
+ // named is answered with, and the prompt is not a failure the caller would send again.
356
+ const started = await codex.start({ prompt: promptFromText("go") });
357
+ expect(String(started.session.native)).toBe(THREAD);
358
+ expect(started.first).toBeUndefined();
359
+ expect(await codex.state(started.session)).toBe("unknown");
360
+ expect(await grandchildGone()).toBe(true);
361
+ await codex.dispose();
362
+ });
363
+
364
+ test("a message Codex refuses, or one to a session it does not hold, was not sent", async () => {
365
+ process.env.FAKE_MODE = "refuse-turn";
366
+ const codex = control();
367
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
368
+ await expect(codex.send(id, "go")).rejects.toBeInstanceOf(InputNotSent);
369
+ await expect(codex.send({ ...id, native: "elsewhere" as never }, "go")).rejects.toBeInstanceOf(InputNotSent);
370
+ await codex.dispose();
371
+ });
372
+
373
+ test("a message queued behind a turn Codex refuses to start takes its turn, in its order", async () => {
374
+ process.env.FAKE_MODE = "refuse-first";
375
+ process.env.FAKE_TEXT = '{"v":1}';
376
+ const codex = control();
377
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
378
+
379
+ const refusedFirst = codex.send(id, "one").catch((error: unknown) => error);
380
+ // The first turn waits for Codex's answer, so these come in while the session is busy.
381
+ expect(await codex.send(id, "two")).toMatchObject({ status: "queued" });
382
+ const asked = codex.ask(id, { schema: z.object({ v: z.number() }), prompt: promptFromText("question") });
383
+
384
+ expect(await refusedFirst).toBeInstanceOf(InputNotSent);
385
+ expect((await asked).value).toEqual({ v: 1 });
386
+ expect(await startedTurns()).toEqual(["one", "two", "question"]);
387
+ await until(async () => (await codex.state(id)) === "idle", "the queue was served");
388
+ await codex.dispose();
389
+ });
390
+
391
+ test("a turn Codex announced but never answered keeps the session busy until its own end", async () => {
392
+ process.env.FAKE_MODE = "announce-silent";
393
+ const codex = control({ callTimeoutMs: 300 });
394
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
395
+
396
+ const failure = await codex.send(id, "one").catch((error: unknown) => error);
397
+ expect(failure).toBeInstanceOf(InputPossiblySent);
398
+ // The turn runs: another message waits behind it instead of starting a second turn.
399
+ expect(await codex.state(id)).toBe("running");
400
+ expect(await codex.send(id, "two")).toMatchObject({ status: "queued" });
401
+ expect(await startedTurns()).toEqual(["one"]);
402
+ await codex.dispose();
403
+ });
404
+
405
+ test("a turn Codex neither answered nor announced closes the session, and what waited behind it is told", async () => {
406
+ process.env.FAKE_MODE = "silent";
407
+ const codex = control({ callTimeoutMs: 300 });
408
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
409
+
410
+ const first = codex.send(id, "one").catch((error: unknown) => error);
411
+ const asked = codex.ask(id, { schema: z.string(), prompt: promptFromText("question") }).catch((error: unknown) => error);
412
+
413
+ expect(await first).toBeInstanceOf(InputPossiblySent);
414
+ expect(await asked).toBeInstanceOf(Error);
415
+ expect(await codex.state(id)).toBe("unknown");
416
+ expect(await grandchildGone()).toBe(true);
417
+ await codex.dispose();
418
+ });
419
+
420
+ test("a start whose first turn may have been taken answers with the session, so a retry cannot make a second thread", async () => {
421
+ process.env.FAKE_MODE = "silent";
422
+ const codex = control({ callTimeoutMs: 300 });
423
+ const started = await codex.start({ prompt: promptFromText("go") });
424
+ expect(String(started.session.native)).toBe(THREAD);
425
+ expect(started.first).toBeUndefined();
426
+ // The prompt reached Codex once, and the session is told apart from one that never was.
427
+ expect(await startedTurns()).toEqual(["go"]);
428
+ expect(await codex.state(started.session)).toBe("unknown");
429
+ await codex.dispose();
430
+ });
431
+
432
+ test("the thread is named to the caller before its first turn is sent", async () => {
433
+ process.env.FAKE_MODE = "silent";
434
+ const codex = control({ callTimeoutMs: 300 });
435
+ const seen: Array<{ native: string; turns: number }> = [];
436
+ await codex.start({
437
+ prompt: promptFromText("go"),
438
+ named: async (id) => {
439
+ seen.push({ native: String(id.native), turns: (await startedTurns()).length });
440
+ },
441
+ });
442
+ expect(seen).toEqual([{ native: THREAD, turns: 0 }]);
443
+ await codex.dispose();
444
+ });
445
+
446
+ test("a closed session's process is gone, and the side holds it no more", async () => {
447
+ process.env.FAKE_MODE = "ok";
448
+ const codex = control();
449
+ const started = await codex.start({ prompt: promptFromText("") });
450
+ await codex.close?.(started.session);
451
+ expect(await codex.state(started.session)).toBe("unknown");
452
+ expect(await grandchildGone()).toBe(true);
453
+ await codex.dispose();
454
+ });
455
+
456
+ test("a start whose first turn Codex refused leaves no session behind", async () => {
457
+ process.env.FAKE_MODE = "refuse-turn";
458
+ const codex = control();
459
+ expect(await codex.start({ prompt: promptFromText("go") }).catch((error: unknown) => error)).toBeInstanceOf(InputNotSent);
460
+ expect(await grandchildGone()).toBe(true);
461
+ await codex.dispose();
462
+ });
463
+
464
+ test("the home of a kind is the home of the processes it starts and resumes", async () => {
465
+ const kind = codexKind.bind({ id: HarnessIdSchema.parse("codex-fake"), home: join(dir, "other-home"), control: { binary: join(dir, "codex") } });
466
+ const codex = kind.control as CodexSessionControl;
467
+ controls.push(codex);
468
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
469
+ await codex.stop(id);
470
+ await codex.resume(id, {});
471
+ expect((await spawns()).map((spawned) => spawned.home)).toEqual([join(dir, "other-home"), join(dir, "other-home")]);
472
+ await codex.dispose();
473
+ });
474
+
475
+ test("a kind without a home takes the one Codex itself would use, for reading and for its processes", async () => {
476
+ const named = join(dir, "env-home");
477
+ const before = process.env.CODEX_HOME;
478
+ process.env.CODEX_HOME = named;
479
+ try {
480
+ const kind = codex({ id: HarnessIdSchema.parse("codex-fake"), control: { binary: join(dir, "codex") } });
481
+ const live = kind.control as CodexSessionControl;
482
+ controls.push(live);
483
+ await live.start({ prompt: promptFromText("") });
484
+ expect((await spawns()).map((spawned) => spawned.home)).toEqual([named]);
485
+ await live.dispose();
486
+ } finally {
487
+ if (before === undefined) delete process.env.CODEX_HOME;
488
+ else process.env.CODEX_HOME = before;
489
+ }
490
+ });
491
+
492
+ test("a first turn that fails is a failed turn of a session that was started, not a failed start", async () => {
493
+ process.env.FAKE_MODE = "fail";
494
+ const codex = control();
495
+ const { session: id, first } = await codex.start({ prompt: promptFromText("go") });
496
+ expect(String(id.native)).toBe(THREAD);
497
+ expect(first).toMatchObject({ status: "started", turn: 1 });
498
+ // The turn's end is a sign of life; the session stays, and takes the next message.
499
+ const signs = await turnSigns(codex, id);
500
+ expect(signs).toEqual(["turn-started:1", "turn-ended:1"]);
501
+ expect(await codex.state(id)).toBe("idle");
502
+ await codex.dispose();
503
+ });
504
+
505
+ /** What the first turn of a session that runs `mode` says of how it ended. */
506
+ async function endingOf(mode: string): Promise<string | undefined> {
507
+ process.env.FAKE_MODE = mode;
508
+ const codex = control();
509
+ const { session: id } = await codex.start({ prompt: promptFromText("go") });
510
+ for await (const event of codex.life(id)) {
511
+ if (event.kind === "turn-ended") return event.outcome;
512
+ }
513
+ return undefined;
514
+ }
515
+
516
+ test("the end of a turn says how Codex ended it", async () => {
517
+ expect(await endingOf("ok")).toBe("completed");
518
+ expect(await endingOf("fail")).toBe("failed");
519
+ // A status this build does not know proves nothing about how the turn went.
520
+ expect(await endingOf("drift")).toBe("unknown");
521
+ });
522
+
523
+ test("the start returns before the first turn ends: a first turn that never ends does not block it", async () => {
524
+ process.env.FAKE_MODE = "hold";
525
+ const codex = control();
526
+ const { session: id, first } = await codex.start({ prompt: promptFromText("go") });
527
+ expect(String(id.native)).toBe(THREAD);
528
+ expect(first).toMatchObject({ status: "started", turn: 1 });
529
+ expect(await codex.state(id)).toBe("running");
530
+ // The session takes messages at once: they wait behind the first turn.
531
+ expect(await codex.send(id, "two")).toMatchObject({ status: "queued", turn: 2 });
532
+ await codex.dispose();
533
+ });
534
+
535
+ /** Asks the fake server to end a turn that is not the running one. */
536
+ async function endAnotherTurn(): Promise<void> {
537
+ await writeFile(join(dir, "stale"), "1");
538
+ await until(async () => (await events()).some((event) => event.method === "turn/completed" && (event.params.turn as { id: string }).id === "old-turn"), "the stale end was sent");
539
+ // The control side has read it by now: nothing else is in flight.
540
+ await Bun.sleep(150);
541
+ }
542
+
543
+ test("the end of a turn that is not the running one does not end the running turn", async () => {
544
+ process.env.FAKE_MODE = "hold";
545
+ const codex = control();
546
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
547
+ const signs: LifeEvent[] = [];
548
+ void (async () => {
549
+ for await (const event of codex.life(id)) signs.push(event);
550
+ })();
551
+ await codex.send(id, "one");
552
+ await until(async () => (await codex.state(id)) === "running", "the turn started");
553
+
554
+ await endAnotherTurn();
555
+ expect(await codex.state(id)).toBe("running");
556
+ expect(signs.filter((sign) => sign.kind === "turn-ended")).toEqual([]);
557
+
558
+ // The running turn's own end is the end of the turn.
559
+ await release(1);
560
+ await eventually(() => signs.some((sign) => sign.kind === "turn-ended"), 10_000, "the turn ended");
561
+ expect(signs.filter((sign) => sign.kind === "turn-ended")).toMatchObject([{ turn: 1 }]);
562
+ expect(await codex.state(id)).toBe("idle");
563
+ });
564
+
565
+ test("an ask is answered by the end of its own turn, not by the end of another", async () => {
566
+ process.env.FAKE_MODE = "hold";
567
+ process.env.FAKE_TEXT = '{"v":7}';
568
+ const codex = control();
569
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
570
+ let settled = false;
571
+ const asked = codex.ask(id, { schema: z.object({ v: z.number() }), prompt: promptFromText("question") }).finally(() => {
572
+ settled = true;
573
+ });
574
+ await until(async () => (await startedTurns()).length === 1, "the ask began its turn");
575
+
576
+ await endAnotherTurn();
577
+ expect(settled).toBe(false);
578
+
579
+ await release(1);
580
+ expect((await asked).value).toEqual({ v: 7 });
581
+ });
582
+
583
+ /** A session this control side does not hold, named by its thread. */
584
+ const cold = sessionId(HarnessIdSchema.parse("codex-fake"), "99999999-2222-3333-4444-555555555555");
585
+
586
+ test("resumes of one thread that overlap share one process, so the thread has one writer", async () => {
587
+ process.env.FAKE_MODE = "hold";
588
+ const codex = control();
589
+ await Promise.all([codex.resume(cold), codex.resume(cold), codex.resume(cold)]);
590
+
591
+ expect(await spawns()).toHaveLength(1);
592
+ expect((await requests()).filter((frame) => frame.method === "thread/resume")).toHaveLength(1);
593
+ expect(await codex.state(cold)).toBe("idle");
594
+ await codex.stop(cold);
595
+ expect(await codex.state(cold)).toBe("unknown");
596
+ });
597
+
598
+ test("a resume that fails does not hold up the resume that follows it", async () => {
599
+ process.env.FAKE_MODE = "hold";
600
+ const codex = control({ binary: join(dir, "missing-codex") });
601
+ const failed = codex.resume(cold).catch((error: unknown) => error);
602
+ const second = codex.resume(cold).catch((error: unknown) => error);
603
+ expect(await failed).toBeInstanceOf(Error);
604
+ expect(await second).toBeInstanceOf(Error);
605
+ });
606
+
607
+ test("a control side disposed while a resume is starting its process does not keep that session", async () => {
608
+ process.env.FAKE_MODE = "hold";
609
+ const codex = control();
610
+ const resuming = codex.resume(cold).catch((error: unknown) => error);
611
+ // The launch has asked for its process, and a process is not there before the next turn of the event loop.
612
+ for (let tick = 0; tick < 10; tick += 1) await Promise.resolve();
613
+ await codex.dispose();
614
+
615
+ expect(await resuming).toBeInstanceOf(Unavailable);
616
+ expect(await codex.state(cold)).toBe("unknown");
617
+ expect((await requests()).filter((frame) => frame.method === "thread/resume")).toEqual([]);
618
+ });
619
+
620
+ test("a message to a busy session is queued, and the turns run in the order the messages came", async () => {
621
+ process.env.FAKE_MODE = "hold";
622
+ const codex = control();
623
+ // An empty prompt: the session is named without a turn of its own.
624
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
625
+ expect(await codex.state(id)).toBe("idle");
626
+
627
+ // The first message starts a turn, and that turn waits for a release.
628
+ const begun = await codex.send(id, "one");
629
+ expect(begun.status).toBe("started");
630
+ await until(async () => (await codex.state(id)) === "running", "the turn started");
631
+ expect(await codex.state(id)).toBe("running");
632
+
633
+ const second = await codex.send(id, "two");
634
+ expect(second.status).toBe("queued");
635
+ // The queued message waits for the turn after the running one.
636
+ expect(second.turn).toBe(2);
637
+ const third = await codex.send(id, "three");
638
+ expect(third.status).toBe("queued");
639
+ expect(third.turn).toBe(3);
640
+
641
+ // Nothing was steered, and no turn was started for the queued messages.
642
+ expect((await requests()).filter((frame) => frame.method === "turn/steer")).toEqual([]);
643
+ expect(await startedTurns()).toEqual(["one"]);
644
+
645
+ // The running turn ends: the first queued message becomes a turn of its own.
646
+ await release(1);
647
+ await until(async () => (await startedTurns()).length === 2, "the first queued turn started");
648
+ expect(await startedTurns()).toEqual(["one", "two"]);
649
+
650
+ // That turn ends: the second queued message follows it.
651
+ await release(2);
652
+ await until(async () => (await startedTurns()).length === 3, "the second queued turn started");
653
+ expect(await startedTurns()).toEqual(["one", "two", "three"]);
654
+
655
+ await release(3);
656
+ await until(async () => (await codex.state(id)) === "idle", "the session became idle");
657
+ expect(await codex.state(id)).toBe("idle");
658
+
659
+ // The turns were reported one after the other, never two at once.
660
+ expect((await events()).filter((event) => event.method === "turn/started")).toHaveLength(3);
661
+
662
+ await codex.dispose();
663
+ });
664
+
665
+ test("the turn that runs inside start is reported to the first watcher", async () => {
666
+ process.env.FAKE_MODE = "ok";
667
+ const codex = control();
668
+ const { session: id } = await codex.start({ prompt: promptFromText("go") });
669
+ // The turn is over before anyone watches: both signs can only come from what the side kept.
670
+ await until(async () => (await codex.state(id)) === "idle", "the first turn ended");
671
+ expect(await turnSigns(codex, id)).toEqual(["turn-started:1", "turn-ended:1"]);
672
+ await codex.dispose();
673
+ });
674
+
675
+ /** The turn signs the first stream of life sees, up to the first turn that ends. */
676
+ async function turnSigns(side: { life(id: SessionId): AsyncIterable<LifeEvent> }, id: SessionId): Promise<string[]> {
677
+ const signs: string[] = [];
678
+ for await (const event of side.life(id)) {
679
+ if (event.kind !== "turn-started" && event.kind !== "turn-ended") continue;
680
+ signs.push(`${event.kind}:${event.turn}`);
681
+ if (event.kind === "turn-ended") break;
682
+ }
683
+ return signs;
684
+ }
685
+
686
+ test("an ask into a busy session waits for its turn, behind what is queued", async () => {
687
+ process.env.FAKE_MODE = "hold";
688
+ process.env.FAKE_TEXT = '{"v":1}';
689
+ const codex = control();
690
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
691
+ await codex.send(id, "one");
692
+ await until(async () => (await codex.state(id)) === "running", "the turn started");
693
+
694
+ await codex.send(id, "two");
695
+ const asked = codex.ask(id, { schema: z.object({ v: z.number() }), prompt: promptFromText("question") });
696
+ expect(await startedTurns()).toEqual(["one"]);
697
+
698
+ await release(1);
699
+ await until(async () => (await startedTurns()).length === 2, "the queued message became a turn");
700
+ await release(2);
701
+ await until(async () => (await startedTurns()).length === 3, "the ask became a turn");
702
+ expect(await startedTurns()).toEqual(["one", "two", "question"]);
703
+ await release(3);
704
+ expect((await asked).value).toEqual({ v: 1 });
705
+
706
+ await codex.dispose();
707
+ });
708
+
709
+ test("a caller that asks for it gets its message into the running turn", async () => {
710
+ process.env.FAKE_MODE = "hold";
711
+ const codex = control();
712
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
713
+ const begun = await codex.send(id, "one");
714
+ expect(begun.status).toBe("started");
715
+ await until(async () => (await codex.state(id)) === "running", "the turn started");
716
+
717
+ const receipt = await codex.send(id, "into the turn", { into: "turn" });
718
+ expect(receipt.status).toBe("delivered");
719
+ expect(receipt.turn).toBe(1);
720
+
721
+ const steered = (await requests()).filter((frame) => frame.method === "turn/steer");
722
+ expect(steered).toHaveLength(1);
723
+ expect((steered[0]?.params.input as { text: string }[])[0]?.text).toBe("into the turn");
724
+ // The message was steered, not queued: no turn was started for it.
725
+ expect(await startedTurns()).toEqual(["one"]);
726
+
727
+ await codex.dispose();
728
+ });
729
+
730
+ /** The config of a thread request, as the fake server received it. */
731
+ async function threadConfig(method: string): Promise<Record<string, unknown> | undefined> {
732
+ const asked = (await requests()).filter((frame) => frame.method === method);
733
+ const last = asked[asked.length - 1];
734
+ const config = last?.params.config;
735
+ return config === undefined ? undefined : (config as Record<string, unknown>);
736
+ }
737
+
738
+ test("a message to an idle session starts a turn", async () => {
739
+ process.env.FAKE_MODE = "hold";
740
+ const codex = control();
741
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
742
+ expect(await codex.state(id)).toBe("idle");
743
+
744
+ const receipt = await codex.send(id, "one");
745
+ expect(receipt.status).toBe("started");
746
+ expect(receipt.turn).toBe(1);
747
+ await until(async () => (await startedTurns()).length === 1, "the turn started");
748
+ expect(await startedTurns()).toEqual(["one"]);
749
+
750
+ await codex.dispose();
751
+ });
752
+
753
+ test("a session is given its server and its switched-off tools in the config of the thread", async () => {
754
+ process.env.FAKE_MODE = "hold";
755
+ const codex = control({ mode: "full" });
756
+ await codex.start({
757
+ prompt: promptFromText(""),
758
+ servers: [
759
+ {
760
+ name: "niwa",
761
+ command: "/usr/bin/niwa",
762
+ args: ["/tmp/home", "--stdio"],
763
+ env: { NIWA_VISIBLE: "task.create" },
764
+ tools: ["task_create"],
765
+ },
766
+ { name: "fixture.empty", command: "fixture", args: [], env: {}, tools: [] },
767
+ { name: "fixture-unfiltered", command: "fixture", args: [], env: {} },
768
+ ],
769
+ visible: { actions: ["task.create"], disabledTools: ["shell", "web_search"] },
770
+ });
771
+
772
+ expect(await threadConfig("thread/start")).toEqual({
773
+ mcp_servers: {
774
+ niwa: {
775
+ command: "/usr/bin/niwa",
776
+ args: ["/tmp/home", "--stdio"],
777
+ env: { NIWA_VISIBLE: "task.create" },
778
+ enabled_tools: ["task_create"],
779
+ },
780
+ "fixture.empty": { command: "fixture", args: [], enabled_tools: [] },
781
+ "fixture-unfiltered": { command: "fixture", args: [] },
782
+ },
783
+ "features.shell_tool": false,
784
+ web_search: "disabled",
785
+ });
786
+
787
+ await codex.dispose();
788
+ });
789
+
790
+ test("a resumed session is given its server and its switched-off tools too", async () => {
791
+ process.env.FAKE_MODE = "hold";
792
+ const codex = control({ mode: "full" });
793
+ const { session: id } = await codex.start({
794
+ prompt: promptFromText(""),
795
+ servers: [{ name: "niwa", command: "niwa", args: [], env: {}, tools: ["task_create"] }],
796
+ });
797
+ await codex.stop(id);
798
+
799
+ await codex.resume(id, {
800
+ servers: [{ name: "niwa", command: "niwa", args: [], env: {}, tools: ["session_open_questions"] }],
801
+ visible: { actions: ["session.open-questions"], disabledTools: ["web_search"] },
802
+ });
803
+
804
+ expect(await threadConfig("thread/resume")).toEqual({
805
+ mcp_servers: { niwa: { command: "niwa", args: [], enabled_tools: ["session_open_questions"] } },
806
+ web_search: "disabled",
807
+ });
808
+
809
+ await codex.dispose();
810
+ });
811
+
812
+ test("a working mode is the approval policy and the sandbox of the thread: ask reads, auto writes its workspace with the network open, full has the machine", async () => {
813
+ process.env.FAKE_MODE = "hold";
814
+ const codex = control();
815
+ const modes = [
816
+ ["ask", { approvalPolicy: "on-request", sandbox: "read-only" }],
817
+ ["auto", { approvalPolicy: "on-request", sandbox: "workspace-write" }],
818
+ ["full", { approvalPolicy: "never", sandbox: "danger-full-access" }],
819
+ ] as const;
820
+ for (const [mode, params] of modes) {
821
+ const started = await codex.start({ prompt: promptFromText(""), mode });
822
+ expect(started.mode).toBe(mode);
823
+ const asked = (await requests()).filter((frame) => frame.method === "thread/start").at(-1);
824
+ expect(asked?.params).toMatchObject(params);
825
+ // Only auto opens the network, in the config that Codex's sandbox reads.
826
+ expect(asked?.params.config).toEqual(mode === "auto" ? { "sandbox_workspace_write.network_access": true } : undefined);
827
+ }
828
+ // None chosen: the default of every harness is auto.
829
+ const unchosen = await codex.start({ prompt: promptFromText("") });
830
+ expect(unchosen.mode).toBe("auto");
831
+ expect((await requests()).filter((frame) => frame.method === "thread/start").at(-1)?.params).toMatchObject({ sandbox: "workspace-write" });
832
+
833
+ await codex.dispose();
834
+ });
835
+
836
+ test("the control option is the mode of a start that names none, and a call's mode wins; a resumed session keeps the mode it was started in", async () => {
837
+ process.env.FAKE_MODE = "hold";
838
+ const codex = control({ mode: "ask" });
839
+ const { session: id, mode } = await codex.start({ prompt: promptFromText("") });
840
+ expect(mode).toBe("ask");
841
+ expect((await requests()).filter((frame) => frame.method === "thread/start").at(-1)?.params).toMatchObject({ sandbox: "read-only" });
842
+ await codex.stop(id);
843
+
844
+ await codex.resume(id, { mode: "full" });
845
+ expect((await requests()).filter((frame) => frame.method === "thread/resume").at(-1)?.params).toMatchObject({
846
+ approvalPolicy: "never",
847
+ sandbox: "danger-full-access",
848
+ });
849
+
850
+ await codex.dispose();
851
+ });
852
+
853
+ test("a tool Codex cannot switch off is refused by name", async () => {
854
+ process.env.FAKE_MODE = "hold";
855
+ const codex = control();
856
+
857
+ const failure = await codex
858
+ .start({
859
+ prompt: promptFromText(""),
860
+ visible: { actions: [], disabledTools: ["apply_patch"] },
861
+ })
862
+ .catch((error: unknown) => error);
863
+
864
+ expect(failure).toBeInstanceOf(Unsupported);
865
+ expect(String(failure)).toContain("apply_patch");
866
+ // Nothing was started: the refusal came before the thread was asked for.
867
+ expect((await requests()).filter((frame) => frame.method === "thread/start")).toEqual([]);
868
+
869
+ await codex.dispose();
870
+ });
871
+
872
+ test("exact tool requests fail before starting or resuming an app-server", async () => {
873
+ await writeFile(join(dir, "config.toml"), '[mcp_servers.local_fixture]\ncommand = "false"\n');
874
+ const codex = control({ home: dir });
875
+ const prompt = promptFromText("test");
876
+ for (const tools of ["niwa-only", { allow: ["apply_patch"] }, { allow: ["apply_patch", "spawn_agent"] }] as const) {
877
+ await expect(codex.start({ prompt, tools })).rejects.toBeInstanceOf(Unsupported);
878
+ await expect(codex.resume(cold, { prompt, tools })).rejects.toBeInstanceOf(Unsupported);
879
+ }
880
+ expect(await spawns()).toEqual([]);
881
+ expect(await requests()).toEqual([]);
882
+ });
883
+
884
+ test("a thread with nothing to set is given no config at all", async () => {
885
+ process.env.FAKE_MODE = "hold";
886
+ const codex = control({ mode: "full" });
887
+ await codex.start({ prompt: promptFromText("") });
888
+
889
+ const asked = (await requests()).filter((frame) => frame.method === "thread/start");
890
+ expect(asked).toHaveLength(1);
891
+ expect(asked[0]?.params.config).toBeUndefined();
892
+
893
+ await codex.dispose();
894
+ });
895
+
896
+ test("a running session cannot take another server or tool set", async () => {
897
+ process.env.FAKE_MODE = "hold";
898
+ const codex = control();
899
+ const niwa = { name: "niwa", command: "niwa", args: [], env: {}, tools: ["task_create", "task_finish"] };
900
+ const other = { name: "other", command: "other-mcp", args: [], env: {} };
901
+ const { session: id } = await codex.start({ prompt: promptFromText(""), servers: [niwa, other] });
902
+
903
+ await codex.resume(id, { servers: [other, { ...niwa, tools: ["task_finish", "task_create"] }] });
904
+ await expect(codex.resume(id, { servers: [other] })).rejects.toBeInstanceOf(Unsupported);
905
+ await expect(codex.resume(id, { servers: [{ ...niwa, tools: [] }, other] })).rejects.toBeInstanceOf(Unsupported);
906
+
907
+ // The thread was not asked for again: the refusal came first.
908
+ expect((await requests()).filter((frame) => frame.method === "thread/resume")).toEqual([]);
909
+
910
+ await codex.dispose();
911
+ });
912
+
913
+ test("the schema of an ask given as JSON reaches Codex with its objects closed, as OpenAI's structured output takes it", async () => {
914
+ process.env.FAKE_MODE = "ok";
915
+ process.env.FAKE_TEXT = '{"answer":42,"inner":{"n":1}}';
916
+ const codex = control();
917
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
918
+ // A schema given as JSON (as `session.ask` takes it) is a loose object: `additionalProperties: {}`.
919
+ const schema = z.fromJSONSchema({
920
+ type: "object",
921
+ properties: {
922
+ answer: { type: "integer" },
923
+ inner: { type: "object", properties: { n: { type: "integer" } } },
924
+ additionalProperties: { type: "object", properties: {} },
925
+ },
926
+ required: ["answer"],
927
+ });
928
+ const answer = await codex.ask(id, { schema, prompt: promptFromText("question") });
929
+ expect((answer.value as { answer: number }).answer).toBe(42);
930
+
931
+ const turn = (await requests()).filter((frame) => frame.method === "turn/start").at(-1);
932
+ const sent = turn?.params["outputSchema"] as Record<string, unknown>;
933
+ expect(sent["additionalProperties"]).toBe(false);
934
+ const properties = sent["properties"] as Record<string, Record<string, unknown>>;
935
+ // An optional property is required and may be null, as OpenAI's strict mode wants.
936
+ expect(sent["required"]).toEqual(["answer", "inner", "additionalProperties"]);
937
+ const inner = (properties["inner"]?.["anyOf"] as Array<Record<string, unknown>>)[0];
938
+ expect(inner?.["additionalProperties"]).toBe(false);
939
+ expect(properties["inner"]?.["anyOf"]).toContainEqual({ type: "null" });
940
+ // A property that is called "additionalProperties" is a property, not the keyword.
941
+ expect(plain(properties["additionalProperties"])?.["type"]).toBe("object");
942
+ expect(JSON.stringify(sent)).not.toContain('"additionalProperties":{}');
943
+ });
944
+
945
+ test("closing the objects of an ask schema reaches every schema, including a property called properties", async () => {
946
+ process.env.FAKE_MODE = "ok";
947
+ process.env.FAKE_TEXT = '{"a":1}';
948
+ const codex = control();
949
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
950
+ const schema = z.fromJSONSchema({
951
+ type: "object",
952
+ properties: {
953
+ // A property called "properties" holds a schema, and its own objects are closed.
954
+ properties: { type: "object", properties: { n: { type: "integer" } } },
955
+ list: { type: "array", items: { type: "object", properties: { n: { type: "integer" } } } },
956
+ either: { anyOf: [{ type: "object", properties: { n: { type: "integer" } } }, { type: "null" }] },
957
+ },
958
+ });
959
+ await codex.ask(id, { schema, prompt: promptFromText("question") });
960
+ const turn = (await requests()).filter((frame) => frame.method === "turn/start").at(-1);
961
+ const sent = turn?.params["outputSchema"] as Record<string, any>;
962
+ expect(sent["additionalProperties"]).toBe(false);
963
+ expect(plain(sent["properties"]["properties"])["additionalProperties"]).toBe(false);
964
+ expect(plain(sent["properties"]["list"])["items"]["additionalProperties"]).toBe(false);
965
+ expect(plain(sent["properties"]["either"])["anyOf"][0]["additionalProperties"]).toBe(false);
966
+ });
967
+
968
+ /** The schema of an optional property as the caller wrote it: strict mode wraps it as `anyOf [it, null]`. */
969
+ function plain(schema: any): any {
970
+ return Array.isArray(schema?.anyOf) && schema.anyOf.length === 2 && schema.anyOf[1]?.type === "null"
971
+ ? schema.anyOf[0]
972
+ : schema;
973
+ }
974
+
975
+ test("an optional property is sent as required and nullable, and a null for it is not part of the answer", async () => {
976
+ process.env.FAKE_MODE = "ok";
977
+ process.env.FAKE_TEXT = '{"answer":42,"note":null}';
978
+ const codex = control();
979
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
980
+ const schema = z.object({ answer: z.number().int(), note: z.string().optional() });
981
+ const answer = await codex.ask(id, { schema, prompt: promptFromText("question") });
982
+ expect(answer.value).toEqual({ answer: 42 });
983
+ const turn = (await requests()).filter((frame) => frame.method === "turn/start").at(-1);
984
+ const sent = turn?.params["outputSchema"] as Record<string, any>;
985
+ expect(sent["required"]).toEqual(["answer", "note"]);
986
+ expect(sent["properties"]["note"]).toEqual({ anyOf: [{ type: "string" }, { type: "null" }] });
987
+ });
988
+
989
+ test("nulls for optional properties are taken out under nullable objects, unions, arrays and refs; a nullable optional null stays", async () => {
990
+ process.env.FAKE_MODE = "ok";
991
+ process.env.FAKE_TEXT = JSON.stringify({
992
+ c: { n: null },
993
+ d: [{ x: null }],
994
+ e: { k: "a", p: null },
995
+ r: { properties: null, required: null },
996
+ h: null,
997
+ });
998
+ const codex = control();
999
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1000
+ const schema = z.object({
1001
+ c: z.object({ n: z.string().optional() }).nullable(),
1002
+ d: z.array(z.object({ x: z.number().optional() })),
1003
+ e: z.union([z.object({ k: z.literal("a"), p: z.string().optional() }), z.object({ k: z.literal("b") })]),
1004
+ r: z.object({ properties: z.string().optional(), required: z.string().optional() }),
1005
+ h: z.string().nullable().optional(),
1006
+ });
1007
+ const answer = await codex.ask(id, { schema, prompt: promptFromText("question") });
1008
+ expect(answer.value).toEqual({ c: {}, d: [{}], e: { k: "a" }, r: {}, h: null });
1009
+ });
1010
+
1011
+ test("a root that is not an object is asked wrapped as { value } and the answer unwrapped", async () => {
1012
+ process.env.FAKE_MODE = "ok";
1013
+ process.env.FAKE_TEXT = '{"value":[1,2,3]}';
1014
+ const codex = control();
1015
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1016
+ const answer = await codex.ask(id, { schema: z.array(z.number()), prompt: promptFromText("question") });
1017
+ expect(answer.value).toEqual([1, 2, 3]);
1018
+ const sent = (await requests()).filter((frame) => frame.method === "turn/start").at(-1)?.params["outputSchema"] as any;
1019
+ expect(sent).toMatchObject({ type: "object", required: ["value"], additionalProperties: false });
1020
+ expect(sent.properties.value.type).toBe("array");
1021
+ });
1022
+
1023
+ test("formats strict mode does not know and propertyNames are not sent; the caller's schema still checks", async () => {
1024
+ process.env.FAKE_MODE = "ok";
1025
+ process.env.FAKE_TEXT = '{"site":"https://example.com","mail":"a@b.co"}';
1026
+ const codex = control();
1027
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1028
+ const answer = await codex.ask(id, { schema: z.object({ site: z.url(), mail: z.email() }), prompt: promptFromText("q") });
1029
+ expect(answer.value).toEqual({ site: "https://example.com", mail: "a@b.co" });
1030
+ const sent = (await requests()).filter((frame) => frame.method === "turn/start").at(-1)?.params["outputSchema"] as any;
1031
+ expect(sent.properties.site.format).toBeUndefined();
1032
+ expect(sent.properties.mail.format).toBe("email");
1033
+ });
1034
+
1035
+ test("a schema strict mode cannot say (a free map) is asked in the text, and the JSON in the answer is read", async () => {
1036
+ process.env.FAKE_MODE = "ok";
1037
+ process.env.FAKE_TEXT = 'Here it is:\n```json\n{"scores":{"a":1}}\n```';
1038
+ const codex = control();
1039
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1040
+ const answer = await codex.ask(id, {
1041
+ schema: z.object({ scores: z.record(z.string(), z.number()) }),
1042
+ prompt: promptFromText("give scores"),
1043
+ });
1044
+ expect(answer.value).toEqual({ scores: { a: 1 } });
1045
+ const turn = (await requests()).filter((frame) => frame.method === "turn/start").at(-1);
1046
+ expect(turn?.params["outputSchema"]).toBeUndefined();
1047
+ const text = (turn?.params["input"] as Array<{ text: string }>).map((item) => item.text).join("");
1048
+ expect(text).toContain("Answer with JSON only");
1049
+ expect(text).toContain('"scores"');
1050
+ });
1051
+
1052
+ async function lastTurn(): Promise<{ outputSchema: any; text: string }> {
1053
+ const turn = (await requests()).filter((frame) => frame.method === "turn/start").at(-1);
1054
+ const text = (turn?.params["input"] as Array<{ text: string }>).map((item) => item.text).join("");
1055
+ return { outputSchema: turn?.params["outputSchema"], text };
1056
+ }
1057
+
1058
+ test("an object root that has a `value` key is not unwrapped", async () => {
1059
+ process.env.FAKE_MODE = "ok";
1060
+ process.env.FAKE_TEXT = '{"value":{"x":1}}';
1061
+ const codex = control();
1062
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1063
+ const schema = z.object({ value: z.object({ x: z.number() }) });
1064
+ const answer = await codex.ask(id, { schema, prompt: promptFromText("q") });
1065
+ expect(answer.value).toEqual({ value: { x: 1 } });
1066
+ expect((await lastTurn()).outputSchema.properties.value.properties.x.type).toBe("number");
1067
+ });
1068
+
1069
+ test("a non-object root strict mode cannot say is asked in the text and read as a bare value", async () => {
1070
+ process.env.FAKE_MODE = "ok";
1071
+ process.env.FAKE_TEXT = 'Sure, {not json} first. [["x",1]]';
1072
+ const codex = control();
1073
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1074
+ const answer = await codex.ask(id, {
1075
+ schema: z.array(z.tuple([z.string(), z.number()])),
1076
+ prompt: promptFromText("q"),
1077
+ });
1078
+ expect(answer.value).toEqual([["x", 1]]);
1079
+ const turn = await lastTurn();
1080
+ expect(turn.outputSchema).toBeUndefined();
1081
+ expect(turn.text).toContain("Answer with JSON only");
1082
+ });
1083
+
1084
+ test("braces in the words around a text answer do not hide the JSON", async () => {
1085
+ process.env.FAKE_MODE = "ok";
1086
+ process.env.FAKE_TEXT = 'Use {name} as the key. Result: {"scores":{"a":1}} (see [1])';
1087
+ const codex = control();
1088
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1089
+ const answer = await codex.ask(id, {
1090
+ schema: z.object({ scores: z.record(z.string(), z.number()) }),
1091
+ prompt: promptFromText("q"),
1092
+ });
1093
+ expect(answer.value).toEqual({ scores: { a: 1 } });
1094
+ });
1095
+
1096
+ test("a record of anything is asked in the text, not sent as an object that holds nothing", async () => {
1097
+ process.env.FAKE_MODE = "ok";
1098
+ process.env.FAKE_TEXT = '{"meta":{"a":1,"b":"x"}}';
1099
+ const codex = control();
1100
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1101
+ const answer = await codex.ask(id, {
1102
+ schema: z.object({ meta: z.record(z.string(), z.any()) }),
1103
+ prompt: promptFromText("q"),
1104
+ });
1105
+ expect(answer.value).toEqual({ meta: { a: 1, b: "x" } });
1106
+ expect((await lastTurn()).outputSchema).toBeUndefined();
1107
+ });
1108
+
1109
+ test("a non-object root with a ref to a place inside it is asked in the text", async () => {
1110
+ process.env.FAKE_MODE = "ok";
1111
+ process.env.FAKE_TEXT = '[{"kids":[]}]';
1112
+ const codex = control();
1113
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1114
+ const node: z.ZodType = z.lazy(() => z.array(z.object({ kids: node })));
1115
+ expect((await codex.ask(id, { schema: node, prompt: promptFromText("q") })).value).toEqual([{ kids: [] }]);
1116
+ expect((await lastTurn()).outputSchema).toBeUndefined();
1117
+ });
1118
+
1119
+ test("a non-object root with refs into $defs is wrapped, and the $defs stay at the wrapper's root", async () => {
1120
+ process.env.FAKE_MODE = "ok";
1121
+ process.env.FAKE_TEXT = '{"value":[{"x":{"a":"1"},"y":{"a":"2"}}]}';
1122
+ const codex = control();
1123
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1124
+ const shared = z.object({ a: z.string() }).meta({ id: "S" });
1125
+ const schema = z.array(z.object({ x: shared, y: shared }));
1126
+ expect((await codex.ask(id, { schema, prompt: promptFromText("q") })).value).toEqual([
1127
+ { x: { a: "1" }, y: { a: "2" } },
1128
+ ]);
1129
+ const sent = (await lastTurn()).outputSchema;
1130
+ expect(sent.$defs.S.type).toBe("object");
1131
+ expect(sent.properties.value.items.properties.x.$ref).toBe("#/$defs/S");
1132
+ });
1133
+
1134
+ test("a discriminated union is sent as anyOf", async () => {
1135
+ process.env.FAKE_MODE = "ok";
1136
+ process.env.FAKE_TEXT = '{"value":{"k":"b","n":2}}';
1137
+ const codex = control();
1138
+ const { session: id } = await codex.start({ prompt: promptFromText("") });
1139
+ const schema = z.discriminatedUnion("k", [
1140
+ z.object({ k: z.literal("a") }),
1141
+ z.object({ k: z.literal("b"), n: z.number() }),
1142
+ ]);
1143
+ expect((await codex.ask(id, { schema, prompt: promptFromText("q") })).value).toEqual({ k: "b", n: 2 });
1144
+ const sent = (await lastTurn()).outputSchema;
1145
+ expect(sent.properties.value.anyOf).toHaveLength(2);
1146
+ expect(sent.properties.value.oneOf).toBeUndefined();
1147
+ });
1148
+
1149
+ test("a model goes to the thread, and an effort goes with every turn of the session, the first and the later ones", async () => {
1150
+ process.env.FAKE_MODE = "ok";
1151
+ process.env.FAKE_MODEL = "gpt-6-real";
1152
+ const codex = control();
1153
+ const { session } = await codex.start({ prompt: promptFromText("one"), model: "gpt-6-luna", effort: "xhigh" });
1154
+ await until(async () => (await codex.state(session)) === "idle", "the first turn ends");
1155
+ await codex.send(session, "two");
1156
+ await until(async () => (await startedTurns()).length === 2, "the second turn starts");
1157
+
1158
+ const asked = await requests();
1159
+ const thread = asked.find((frame) => frame.method === "thread/start");
1160
+ expect(thread?.params.model).toBe("gpt-6-luna");
1161
+ expect(thread?.params.effort).toBeUndefined();
1162
+ expect(asked.filter((frame) => frame.method === "turn/start").map((frame) => frame.params.effort)).toEqual(["xhigh", "xhigh"]);
1163
+ // The model the thread says it runs is a sign of life, kept for the first watcher, not the one that was asked for.
1164
+ expect(await firstModel(codex, session)).toBe("gpt-6-real");
1165
+ });
1166
+
1167
+ test("a turn Codex moves to another model names that model as the one the session runs", async () => {
1168
+ process.env.FAKE_MODE = "ok";
1169
+ process.env.FAKE_MODEL = "gpt-6-real";
1170
+ process.env.FAKE_REROUTE = "gpt-6-safe";
1171
+ const codex = control();
1172
+ const { session } = await codex.start({ prompt: promptFromText("one") });
1173
+ await until(async () => (await codex.state(session)) === "idle", "the first turn ends");
1174
+ const named: string[] = [];
1175
+ for await (const event of codex.life(session)) {
1176
+ if (event.kind === "model") named.push(event.model);
1177
+ if (event.kind === "turn-ended") break;
1178
+ }
1179
+ expect(named).toEqual(["gpt-6-real", "gpt-6-safe"]);
1180
+ });
1181
+
1182
+ test("a resume pins the model the thread was named, and says again the one Codex answers", async () => {
1183
+ process.env.FAKE_MODE = "ok";
1184
+ const codex = control();
1185
+ const { session } = await codex.start({ prompt: promptFromText("") });
1186
+ await codex.dispose();
1187
+ process.env.FAKE_MODEL = "gpt-6-real";
1188
+ const again = control();
1189
+ await again.resume(session, { model: modelName("gpt-6-real") });
1190
+ expect((await requests()).filter((frame) => frame.method === "thread/resume").at(-1)?.params.model).toBe("gpt-6-real");
1191
+ expect(await firstModel(again, session)).toBe("gpt-6-real");
1192
+ });
1193
+
1194
+ /** The first model the first stream of life names. */
1195
+ async function firstModel(side: { life(id: SessionId): AsyncIterable<LifeEvent> }, id: SessionId): Promise<string | undefined> {
1196
+ for await (const event of side.life(id)) {
1197
+ if (event.kind === "model") return event.model;
1198
+ }
1199
+ return undefined;
1200
+ }
1201
+
1202
+ test("the effort of the instance is the default of a session that names none, and a session that names none and has no default sends none", async () => {
1203
+ process.env.FAKE_MODE = "ok";
1204
+ const withDefault = codexControl({ harness: "codex-fake", binary: join(dir, "codex"), effort: "low", turnTimeoutMs: 10_000 });
1205
+ controls.push(withDefault);
1206
+ await withDefault.start({ prompt: promptFromText("one") });
1207
+ await until(async () => (await startedTurns()).length === 1, "the turn starts");
1208
+ expect((await requests()).find((frame) => frame.method === "turn/start")?.params.effort).toBe("low");
1209
+
1210
+ const plain = control();
1211
+ await plain.start({ prompt: promptFromText("two") });
1212
+ await until(async () => (await startedTurns()).length === 2, "the second turn starts");
1213
+ expect((await requests()).filter((frame) => frame.method === "turn/start")[1]?.params).not.toHaveProperty("effort");
1214
+ });
1215
+
1216
+ test("an effort Codex does not take is refused before the process is started", async () => {
1217
+ process.env.FAKE_MODE = "ok";
1218
+ const codex = control();
1219
+ const failure = await codex.start({ prompt: promptFromText("one"), effort: "max" }).catch((error: unknown) => error);
1220
+
1221
+ expect(failure).toBeInstanceOf(Unsupported);
1222
+ expect(String(failure)).toContain("effort \"max\" is not one of low, medium, high, xhigh");
1223
+ expect(await requests()).toEqual([]);
1224
+ });
1225
+
1226
+ test.skipIf(process.platform !== "darwin")("a macOS nono launch uses the system CA file and preserves a supplied CA", async () => {
1227
+ const values: (string | undefined)[] = [];
1228
+ const off = onBoxManifest(({ spec }) => { values.push(spec.env["CODEX_CA_CERTIFICATE"]); });
1229
+ try {
1230
+ const codex = control();
1231
+ for (const env of [{}, { SSL_CERT_FILE: "/chosen/roots.pem" }] as Record<string, string>[]) {
1232
+ const box = { sandbox: unconfined("nono"), spec: { cwd: dir, env } };
1233
+ await codex.start({ prompt: promptFromText(""), cwd: dir, env: { FAKE_DIR: dir }, box });
1234
+ }
1235
+ expect(values).toEqual(["/etc/ssl/cert.pem", undefined]);
1236
+ } finally {
1237
+ off();
1238
+ }
1239
+ });