@niwa-labs/harness-codex 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/reading.ts ADDED
@@ -0,0 +1,940 @@
1
+ /**
2
+ * The reading side of a Codex installation.
3
+ *
4
+ * It works from the files: the sessions are listed and read whether or not Codex is running, and
5
+ * nothing is copied into niwa's journal. The format is in `format.ts`, the layout in `paths.ts`.
6
+ *
7
+ * **What a rollout holds.** One file per session, JSONL, one record per line. Two layers live in
8
+ * it. The conversation layer (`response_item`) is what the model was actually given and produced:
9
+ * messages, reasoning, tool calls, tool results. The event layer (`event_msg`) is how Codex shows
10
+ * it, and it repeats most of the conversation under its own ids. This side reads the conversation
11
+ * layer as the record of what happened, and reads the event layer for what only it knows: that a
12
+ * subagent was spawned, that the context was compacted. What the event layer repeats is not read
13
+ * twice.
14
+ *
15
+ * **The locator.** `codex:v2:<through>:<digest>:<line>:<byte>:<after>`: the revision of the file
16
+ * the position was taken from, the place in it, and how many of that line's records were already
17
+ * delivered. The revision is the digest of every byte up to the end of that line (`through` of
18
+ * them), so a file that grows keeps the locators it already gave out, and a file in which any
19
+ * earlier byte was rewritten does not answer to them. The position is counted in the *decoded*
20
+ * JSONL, which is the same space for a plain rollout and for a compressed one.
21
+ *
22
+ * **The half-written tail.** The last line of a rollout that is being written has no newline yet.
23
+ * It is marked `incomplete`, and the reader does not move past it.
24
+ *
25
+ * **The compressed rollout.** Codex compresses cold rollouts itself. The same JSONL is decoded as a
26
+ * stream, so a large archive never sits in memory.
27
+ *
28
+ * Ported from Strata's Codex source adapter, without its usage accounting and without its engine.
29
+ */
30
+
31
+ import { createHash } from "node:crypto";
32
+ import { createReadStream } from "node:fs";
33
+ import { readFile, readdir, stat } from "node:fs/promises";
34
+ import { join } from "node:path";
35
+ import { createZstdDecompress } from "node:zlib";
36
+
37
+ import { z } from "zod";
38
+
39
+ import { jsonOf, optionalJsonOf, timestamp } from "@niwa-labs/core";
40
+ import type { Timestamp } from "@niwa-labs/core";
41
+ import type {
42
+ HarnessId,
43
+ Coverage,
44
+ InsertionOrigin,
45
+ Locator,
46
+ NativeSessionId,
47
+ SessionFilter,
48
+ SessionId,
49
+ SessionInfo,
50
+ SessionReading,
51
+ SessionRecord,
52
+ } from "@niwa-labs/harness";
53
+ import {
54
+ EMPTY,
55
+ LocatorSchema,
56
+ NOT_SCANNED,
57
+ NativeSessionIdSchema,
58
+ TitleCache,
59
+ failedCoverage,
60
+ insertedRanges,
61
+ titleFromText,
62
+ unavailableCoverage,
63
+ } from "@niwa-labs/harness";
64
+
65
+ import {
66
+ argumentsOf,
67
+ compactionSummary,
68
+ instantOf,
69
+ messageOf,
70
+ outputOf,
71
+ outcomeOf,
72
+ parseNativeRecord,
73
+ payloadOf,
74
+ textsOf,
75
+ } from "./format.ts";
76
+ import type { NativeRecord } from "./format.ts";
77
+ import { baseOfArtifact, isCompressedName, nativeIdOfFileName } from "./paths.ts";
78
+ import { codexChecklist, codexToolShape } from "./tools.ts";
79
+
80
+ /** The tags Codex wraps its own insertions in, and who puts each there. */
81
+ const INSERTION_TAGS: Readonly<Record<string, InsertionOrigin>> = {
82
+ environment_context: "harness",
83
+ user_instructions: "harness",
84
+ turn_aborted: "harness",
85
+ };
86
+
87
+ export interface CodexReadingOptions {
88
+ /** The id of the installation this side serves. */
89
+ readonly harness: HarnessId;
90
+ /** The home of the installation: the directory that holds `sessions`. */
91
+ readonly home: string;
92
+ /** The directory inside the home that holds the sessions. */
93
+ readonly sessionsDir?: string;
94
+ }
95
+
96
+ /** A record before the reading side has put its locator in. */
97
+ type Unlocated = SessionRecord extends infer R ? (R extends unknown ? Omit<R, "locator"> : never) : never;
98
+
99
+ interface RecordBase {
100
+ readonly at?: number;
101
+ readonly incomplete: boolean;
102
+ }
103
+
104
+ /** A rollout file: where it is, which session it holds, and whether it is compressed. */
105
+ interface Rollout {
106
+ readonly path: string;
107
+ readonly native: NativeSessionId;
108
+ readonly compressed: boolean;
109
+ }
110
+
111
+ /** One line of a rollout, with where it sits in the file. */
112
+ interface RawLine {
113
+ /** The number of the line in the file, from one. */
114
+ readonly line: number;
115
+ /** The offset of the start of the line, in bytes from the start of the decoded JSONL. */
116
+ readonly byte: number;
117
+ /** The bytes of the line, without the newline. */
118
+ readonly bytes: Uint8Array;
119
+ /** `false` when the line has no newline after it: the tail of the file, maybe half written. */
120
+ readonly complete: boolean;
121
+ /**
122
+ * How many bytes of the file are final by the end of this line: through the newline of a line
123
+ * that is whole, before a line that is not.
124
+ *
125
+ * A line that is whole will not change, so the digest of the bytes up to its end does not
126
+ * change either when the file grows.
127
+ */
128
+ readonly through: number;
129
+ /** The SHA-256 of those `through` bytes, in hex. */
130
+ readonly digest: string;
131
+ }
132
+
133
+ /** A locator read back. */
134
+ interface Position {
135
+ readonly through: number;
136
+ readonly digest: string;
137
+ readonly line: number;
138
+ readonly byte: number;
139
+ readonly after: number;
140
+ }
141
+
142
+ /**
143
+ * The locator of one record: the revision it was taken from, and where it sits in it.
144
+ *
145
+ * The revision covers the bytes up to the end of the line the record is on, so it is the same for
146
+ * every record of that line and does not change when the file grows after it.
147
+ *
148
+ * A line that is not whole has `after = 0`: it may still change, so a resume reads it again
149
+ * instead of moving past it.
150
+ */
151
+ function makeLocator(line: RawLine, after: number): Locator {
152
+ // A line that is not whole may still change, so the revision covers only the bytes before it:
153
+ // the digest then holds when the line is completed, and the tail is read again.
154
+ const delivered = line.complete ? after : 0;
155
+ return LocatorSchema.parse(
156
+ `codex:v2:${line.through}:${line.digest}:${line.line}:${line.byte}:${delivered}`,
157
+ );
158
+ }
159
+
160
+ /** A locator that is not this side's own is refused by name. */
161
+ function decodeLocator(locator: Locator): Position {
162
+ const parts = locator.split(":");
163
+ const [tag, version, through, digest, line, byte, after] = parts;
164
+ const throughBytes = Number(through);
165
+ const lineNumber = Number(line);
166
+ const byteOffset = Number(byte);
167
+ const delivered = Number(after);
168
+ if (
169
+ parts.length !== 7 ||
170
+ tag !== "codex" ||
171
+ version !== "v2" ||
172
+ digest === undefined ||
173
+ !Number.isSafeInteger(throughBytes) ||
174
+ throughBytes < 0 ||
175
+ !Number.isSafeInteger(lineNumber) ||
176
+ lineNumber < 1 ||
177
+ !Number.isSafeInteger(byteOffset) ||
178
+ byteOffset < 0 ||
179
+ !Number.isSafeInteger(delivered) ||
180
+ delivered < 0
181
+ ) {
182
+ throw new Error(`codex: locator ${locator} is not of this side's form`);
183
+ }
184
+ return {
185
+ through: throughBytes,
186
+ digest,
187
+ line: lineNumber,
188
+ byte: byteOffset,
189
+ after: delivered,
190
+ };
191
+ }
192
+
193
+ /**
194
+ * The lines of a rollout, one at a time, from the start.
195
+ *
196
+ * A plain file is read as a stream, so a large session never sits in memory. A compressed one is
197
+ * decoded as a stream, because zstd is sequential; the offsets are in the decoded space either
198
+ * way. The scan always starts at the beginning, because the meaning of a line depends on the ones
199
+ * before it (the model of the turn, the name of a tool, the replayed history of a parent); a
200
+ * resume skips what it already delivered but still reads it.
201
+ *
202
+ * Lines are cut on raw bytes, so the offsets and the revision hash are the bytes of the file even
203
+ * when a line is not valid UTF-8.
204
+ */
205
+ async function* scanLines(rollout: Rollout): AsyncGenerator<RawLine> {
206
+ const { source, close } = openRollout(rollout);
207
+ try {
208
+ yield* linesOf(source);
209
+ } finally {
210
+ // A reader that stops early (the listing reads the head only) must not leave the file open.
211
+ close();
212
+ }
213
+ }
214
+
215
+ /**
216
+ * The decoded bytes of a rollout, and the way to close it. A piped stream does not close its
217
+ * source when it is itself closed, so both are closed by hand.
218
+ */
219
+ function openRollout(rollout: Rollout): { source: NodeJS.ReadableStream; close: () => void } {
220
+ const file = createReadStream(rollout.path);
221
+ if (!rollout.compressed) return { source: file, close: () => file.destroy() };
222
+ const decompress = createZstdDecompress();
223
+ file.on("error", (error) => decompress.destroy(error));
224
+ const source = file.pipe(decompress);
225
+ return {
226
+ source,
227
+ close: () => {
228
+ decompress.destroy();
229
+ file.destroy();
230
+ },
231
+ };
232
+ }
233
+
234
+ /** The lines of a decoded rollout, cut on raw bytes. */
235
+ async function* linesOf(source: NodeJS.ReadableStream): AsyncGenerator<RawLine> {
236
+ // The digest of every byte of the whole lines read so far, kept as the file is read.
237
+ const whole = createHash("sha256");
238
+ let through = 0;
239
+ let carry: Buffer = Buffer.alloc(0);
240
+ let lineStart = 0;
241
+ let lineNo = 1;
242
+
243
+ for await (const chunk of source) {
244
+ const bytes = Buffer.from(chunk as Uint8Array);
245
+ const data = carry.length === 0 ? bytes : Buffer.concat([carry, bytes]);
246
+ let from = 0;
247
+ for (;;) {
248
+ const lf = data.indexOf(0x0a, from);
249
+ if (lf < 0) break;
250
+ const line = data.subarray(from, lf);
251
+ whole.update(data.subarray(from, lf + 1));
252
+ through += line.length + 1;
253
+ yield { line: lineNo, byte: lineStart, bytes: line, complete: true, through, digest: whole.copy().digest("hex") };
254
+ lineNo += 1;
255
+ lineStart += line.length + 1;
256
+ from = lf + 1;
257
+ }
258
+ carry = data.subarray(from);
259
+ }
260
+ if (carry.length > 0) {
261
+ // The tail of a file that is being written: it has no newline yet, so it may be half written.
262
+ yield { line: lineNo, byte: lineStart, bytes: carry, complete: false, through, digest: whole.copy().digest("hex") };
263
+ }
264
+ }
265
+
266
+ /** A line as JSON; text that does not parse is kept as text, never an error. */
267
+ function parseLine(bytes: Uint8Array): unknown {
268
+ const text = new TextDecoder().decode(bytes);
269
+ try {
270
+ return JSON.parse(text);
271
+ } catch {
272
+ return text;
273
+ }
274
+ }
275
+
276
+ /**
277
+ * The state of the parse of one rollout, shared by all its lines.
278
+ *
279
+ * These are the things the native format holds for the file as a whole: the session it belongs
280
+ * to, the model of the current turn, the names of the tools by the id of their call, and where the
281
+ * own history of a forked session begins.
282
+ */
283
+ interface ParseState {
284
+ /** The working directory of the session. */
285
+ cwd: string | null;
286
+ /** The moment the session started. */
287
+ startedAt: Timestamp | undefined;
288
+ /** The model of the current turn. */
289
+ model: string | undefined;
290
+ /** The moment the file was last written. */
291
+ updatedAt: Timestamp | undefined;
292
+ /** The names of the tools, by the id of their call: a result does not name its tool. */
293
+ readonly toolNames: Map<string, string>;
294
+ /**
295
+ * The ordinal from which the history is this session's own.
296
+ *
297
+ * Codex writes a forked session's file with the history of the parent replayed into it, and it
298
+ * writes the parent's metadata there too. That history happened in another session, so it is not
299
+ * this session's own. The own history begins at the first `turn_context` of this session, which
300
+ * is where the replay ends. Codex says the same thing in `subagent_history_start_ordinal`, and
301
+ * when it says it, that ordinal is the boundary.
302
+ */
303
+ ownHistoryFrom: number | undefined;
304
+ /** The ordinal of the first `turn_context` of this session, once it is seen. */
305
+ ownTurnAt: number | undefined;
306
+ /** `true` while the replayed history of a parent is being read. */
307
+ awaitingOwnHistory: boolean;
308
+ /** Whether the session metadata was seen. */
309
+ metaSeen: boolean;
310
+ }
311
+
312
+ /** A fresh parse state. */
313
+ function newParseState(): ParseState {
314
+ return {
315
+ cwd: null,
316
+ startedAt: undefined,
317
+ model: undefined,
318
+ updatedAt: undefined,
319
+ toolNames: new Map(),
320
+ ownHistoryFrom: undefined,
321
+ ownTurnAt: undefined,
322
+ awaitingOwnHistory: false,
323
+ metaSeen: false,
324
+ };
325
+ }
326
+
327
+ /**
328
+ * The index of the threads Codex keeps beside its sessions: one JSON line per name it gave a
329
+ * thread, `{ "id", "thread_name", "updated_at" }`. A later line for the same id renames it.
330
+ */
331
+ const SESSION_INDEX = "session_index.jsonl";
332
+
333
+ const SessionIndexLineSchema = z.object({ id: z.string().min(1), thread_name: z.string() });
334
+
335
+ /**
336
+ * The instructions Codex puts before the first message of a person: they are what the files of
337
+ * the project say, not what the person said, so a title is never made of them.
338
+ */
339
+ const INSTRUCTIONS_PREAMBLE = /^#\s*AGENTS\.md instructions\b/;
340
+
341
+ /** How many lines of the head of a rollout a title is looked for in. */
342
+ const TITLE_LINES = 30;
343
+
344
+ /** What a line holds when it is a message of a user, or tells where the history of the session begins. */
345
+ const TITLE_MARKS = [
346
+ Buffer.from('"role":"user"'),
347
+ Buffer.from('"type":"session_meta"'),
348
+ Buffer.from('"type":"turn_context"'),
349
+ ];
350
+
351
+ /**
352
+ * Can the line change the title? Only a message of a user gives one, and the lines that say where
353
+ * a forked session's own history begins decide whether that message is its own. The others are
354
+ * not decoded: the head of a rollout holds large lines (instructions, tool lists) that say
355
+ * nothing about it.
356
+ */
357
+ function mattersForTitle(bytes: Uint8Array): boolean {
358
+ const line = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
359
+ return TITLE_MARKS.some((mark) => line.includes(mark));
360
+ }
361
+
362
+ /** The walk's signal that the position of a locator is not in the file. */
363
+ const STALE = Symbol("stale");
364
+
365
+ export class CodexReading implements SessionReading {
366
+ readonly #harness: HarnessId;
367
+ readonly #home: string;
368
+ readonly #sessionsDir: string;
369
+ /** The titles made from first messages: kept as long as this reading side lives. */
370
+ readonly #titles = new TitleCache();
371
+
372
+ constructor(options: CodexReadingOptions) {
373
+ this.#harness = options.harness;
374
+ this.#home = options.home;
375
+ this.#sessionsDir = options.sessionsDir ?? "sessions";
376
+ }
377
+
378
+ /** The directory the rollouts live in. */
379
+ get sessionsRoot(): string {
380
+ return join(this.#home, this.#sessionsDir);
381
+ }
382
+
383
+ /** A line that does not parse is returned as text, an unfamiliar shape as it is. */
384
+ readonly rawSchema: z.ZodType<unknown> = z.json();
385
+
386
+ /** The sessions of the installation, oldest first. */
387
+ async *list(filter?: SessionFilter): AsyncIterable<SessionInfo> {
388
+ const names = await this.#threadNames();
389
+ const wanted = filter?.natives === undefined ? undefined : new Set(filter.natives);
390
+ for (const rollout of await this.#discover()) {
391
+ // A session that is not wanted is left out before anything of it is read.
392
+ if (wanted !== undefined && !wanted.has(rollout.native)) continue;
393
+ const info = await this.#infoOf(rollout, names);
394
+ if (filter?.cwd !== undefined && info.cwd !== filter.cwd) continue;
395
+ if (filter?.since !== undefined) {
396
+ const seen = info.updatedAt ?? info.startedAt;
397
+ if (seen === undefined || seen < filter.since) continue;
398
+ }
399
+ yield info;
400
+ }
401
+ }
402
+
403
+ /**
404
+ * The records of a session, after `from` when given.
405
+ *
406
+ * A locator of a revision the file no longer has does not name a record any more, so the read
407
+ * then starts from the beginning and delivers everything.
408
+ */
409
+ async *read(id: SessionId, from?: Locator): AsyncIterable<SessionRecord> {
410
+ const rollout = await this.#find(id);
411
+ if (rollout === null) return;
412
+ const resume = await this.#resumeOf(rollout, from);
413
+ // A locator the file no longer holds starts the read over, and the first record says so.
414
+ const aligned = yield* this.#deliver(rollout, resume, from !== undefined && resume === undefined);
415
+ if (!aligned) yield* this.#deliver(rollout, undefined, true);
416
+ }
417
+
418
+ /**
419
+ * Walks the rollout and yields the records after `resume`, each with its locator. It returns
420
+ * `false` when the file holds no line where the locator says one starts: the position is stale,
421
+ * and nothing was yielded.
422
+ */
423
+ async *#deliver(
424
+ rollout: Rollout,
425
+ resume: Position | undefined,
426
+ restarted: boolean,
427
+ ): AsyncGenerator<SessionRecord, boolean> {
428
+ let first = restarted;
429
+ for await (const entry of this.#walk(rollout, resume)) {
430
+ if (entry === STALE) return false;
431
+ yield { ...entry.record, ...(first ? { reanchored: true } : {}), locator: makeLocator(entry.line, entry.index + 1) } as SessionRecord;
432
+ first = false;
433
+ }
434
+ return true;
435
+ }
436
+
437
+ /**
438
+ * Every line is parsed, because the state it leaves is needed by the lines after it, but only the
439
+ * records after `resume` come out. The tail line that was marked incomplete comes out again.
440
+ */
441
+ async *#walk(
442
+ rollout: Rollout,
443
+ resume: Position | undefined,
444
+ ): AsyncGenerator<{ line: RawLine; index: number; record: Unlocated } | typeof STALE> {
445
+ const state = newParseState();
446
+ let aligned = resume === undefined;
447
+ for await (const line of scanLines(rollout)) {
448
+ const records = line.bytes.length === 0 ? [] : this.#recordsOf(line, state);
449
+ if (resume === undefined) {
450
+ for (const [index, record] of records.entries()) yield { line, index, record };
451
+ continue;
452
+ }
453
+ if (line.byte < resume.byte) continue;
454
+ if (!aligned) {
455
+ if (line.byte !== resume.byte || line.line !== resume.line) {
456
+ yield STALE;
457
+ return;
458
+ }
459
+ aligned = true;
460
+ }
461
+ for (const [index, record] of records.entries()) {
462
+ if (line.line === resume.line && index + 1 <= resume.after) continue;
463
+ yield { line, index, record };
464
+ }
465
+ }
466
+ if (!aligned) yield STALE;
467
+ }
468
+
469
+ /** Whether any record follows `resume`; `STALE` when the position is not in the file. */
470
+ async #firstAfter(rollout: Rollout, resume: Position | undefined): Promise<boolean | typeof STALE> {
471
+ for await (const entry of this.#walk(rollout, resume)) return entry === STALE ? STALE : true;
472
+ return false;
473
+ }
474
+
475
+ /** The position of a locator, when the file still holds it; `undefined` for no locator or a stale one. */
476
+ async #resumeOf(rollout: Rollout, from: Locator | undefined): Promise<Position | undefined> {
477
+ if (from === undefined) return undefined;
478
+ const position = decodeLocator(from);
479
+ return (await this.#positionHolds(rollout, position)) ? position : undefined;
480
+ }
481
+
482
+ /** The harness's own record, whole, at a locator. */
483
+ async raw(id: SessionId, locator: Locator): Promise<unknown> {
484
+ const rollout = await this.#find(id);
485
+ if (rollout === null) throw new Error(`codex: session ${id.native} is not on disk`);
486
+ const position = decodeLocator(locator);
487
+ if (!(await this.#positionHolds(rollout, position))) {
488
+ throw new Error(`codex: locator ${locator} does not belong to the current file of ${id.native}`);
489
+ }
490
+ for await (const line of scanLines(rollout)) {
491
+ if (line.byte > position.byte) break;
492
+ if (line.byte === position.byte && line.line === position.line) return parseLine(line.bytes);
493
+ }
494
+ throw new Error(`codex: no record at ${locator} in session ${id.native}`);
495
+ }
496
+
497
+ /**
498
+ * `unavailable` — not on disk; `failed` — the scan broke; `empty` — nothing in the range;
499
+ * `not-scanned` — there are records niwa has not read: it copies nothing, so a range is only
500
+ * read when a caller asks `read`.
501
+ */
502
+ async coverage(id: SessionId, from?: Locator): Promise<Coverage> {
503
+ const rollout = await this.#find(id);
504
+ if (rollout === null) return unavailableCoverage(`codex: session ${id.native} is not on disk`);
505
+ try {
506
+ const resume = await this.#resumeOf(rollout, from);
507
+ let outcome = await this.#firstAfter(rollout, resume);
508
+ // A stale position names nothing, so the whole session counts.
509
+ if (outcome === STALE) outcome = await this.#firstAfter(rollout, undefined);
510
+ return outcome === false ? EMPTY : NOT_SCANNED;
511
+ } catch (error) {
512
+ return failedCoverage(`codex: session ${id.native} could not be scanned: ${reasonOf(error)}`);
513
+ }
514
+ }
515
+
516
+ /**
517
+ * The rollouts of the installation.
518
+ *
519
+ * Both copies of a session count as one session: when a plain rollout and a compressed one lie
520
+ * side by side, they are the same session, and the plain one is the one that is read. Codex
521
+ * leaves the compressed copy behind when it compresses a rollout.
522
+ *
523
+ * A path that is named like a rollout is a rollout even when it cannot be read: the session is
524
+ * still there, and reading it says so instead of pretending the session is gone.
525
+ */
526
+ async #discover(): Promise<Rollout[]> {
527
+ const found = new Map<string, Rollout>();
528
+ const walk = async (dir: string): Promise<void> => {
529
+ let entries;
530
+ try {
531
+ entries = await readdir(dir, { withFileTypes: true });
532
+ } catch {
533
+ return;
534
+ }
535
+ for (const entry of entries) {
536
+ const path = join(dir, entry.name);
537
+ const name = nativeIdOfFileName(entry.name);
538
+ if (name !== null) {
539
+ const native = NativeSessionIdSchema.parse(name);
540
+ const compressed = isCompressedName(entry.name);
541
+ const key = baseOfArtifact(path);
542
+ const existing = found.get(key);
543
+ if (existing === undefined || (existing.compressed && !compressed)) {
544
+ found.set(key, { path, native, compressed });
545
+ }
546
+ continue;
547
+ }
548
+ if (entry.isDirectory()) await walk(path);
549
+ }
550
+ };
551
+ await walk(this.sessionsRoot);
552
+ return [...found.values()].sort((a, b) => a.path.localeCompare(b.path));
553
+ }
554
+
555
+ /**
556
+ * Does the position still mean what it meant?
557
+ *
558
+ * A rollout is only appended to, so the bytes that were final stay final and their digest holds.
559
+ * A file in which any byte before the position was rewritten fails the digest, and a position
560
+ * that is not at the start of a line does not name a record at all.
561
+ */
562
+ async #positionHolds(rollout: Rollout, position: Position): Promise<boolean> {
563
+ try {
564
+ return (await this.#digestThrough(rollout, position.through)) === position.digest;
565
+ } catch {
566
+ return false;
567
+ }
568
+ }
569
+
570
+ /** The digest of the first `count` bytes of the decoded rollout; `undefined` when the file is shorter. */
571
+ async #digestThrough(rollout: Rollout, count: number): Promise<string | undefined> {
572
+ const hash = createHash("sha256");
573
+ let left = count;
574
+ if (left > 0) {
575
+ const { source, close } = openRollout(rollout);
576
+ try {
577
+ for await (const chunk of source) {
578
+ const bytes = Buffer.from(chunk as Uint8Array);
579
+ hash.update(bytes.subarray(0, left));
580
+ left -= Math.min(bytes.length, left);
581
+ if (left === 0) break;
582
+ }
583
+ } finally {
584
+ close();
585
+ }
586
+ }
587
+ return left === 0 ? hash.digest("hex") : undefined;
588
+ }
589
+
590
+ /**
591
+ * The id is the one in the file name, which is the thread id that control starts and resumes
592
+ * sessions by. The times are the file's: a session was last updated when its file was. The
593
+ * title is the name Codex gave the thread, else the first message of the person.
594
+ */
595
+ async #infoOf(rollout: Rollout, names: ReadonlyMap<string, string>): Promise<SessionInfo> {
596
+ const name = names.get(rollout.native);
597
+ const info = await stat(rollout.path).catch(() => undefined);
598
+ // The name Codex gave the thread needs no look at the file. Without one, the title kept for
599
+ // the file as it is now is used, and only a file that is new or has grown is looked into.
600
+ const key = info === undefined ? undefined : { size: info.size, mtimeMs: info.mtimeMs };
601
+ const kept = name !== undefined || key === undefined ? undefined : this.#titles.get(rollout.path, key);
602
+ const { state: header, firstMessage, read } = await this.#headerOf(
603
+ rollout,
604
+ name === undefined && kept === undefined,
605
+ );
606
+ // A head that could not be read says nothing, and is not kept as "no title".
607
+ if (name === undefined && kept === undefined && key !== undefined && read) {
608
+ this.#titles.set(rollout.path, key, firstMessage);
609
+ }
610
+ const updatedAt = info === undefined ? header.updatedAt : timestamp(Math.floor(info.mtimeMs));
611
+ const title = name ?? kept?.title ?? firstMessage;
612
+ return {
613
+ id: { harness: this.#harness, native: rollout.native },
614
+ ...(title === undefined ? {} : { title }),
615
+ ...(header.cwd === null ? {} : { cwd: header.cwd }),
616
+ ...(header.startedAt === undefined ? {} : { startedAt: header.startedAt }),
617
+ ...(updatedAt === undefined ? {} : { updatedAt }),
618
+ };
619
+ }
620
+
621
+ /**
622
+ * The names Codex gave its threads, by thread id. A missing or unreadable index names none; a
623
+ * line that is not a name is passed over.
624
+ */
625
+ async #threadNames(): Promise<Map<string, string>> {
626
+ const names = new Map<string, string>();
627
+ let text: string;
628
+ try {
629
+ text = await readFile(join(this.#home, SESSION_INDEX), "utf8");
630
+ } catch {
631
+ return names;
632
+ }
633
+ for (const line of text.split("\n")) {
634
+ if (line.trim().length === 0) continue;
635
+ let value: unknown;
636
+ try {
637
+ value = JSON.parse(line);
638
+ } catch {
639
+ continue;
640
+ }
641
+ const parsed = SessionIndexLineSchema.safeParse(value);
642
+ if (!parsed.success) continue;
643
+ const name = parsed.data.thread_name.trim();
644
+ if (name.length > 0) names.set(parsed.data.id, name);
645
+ else names.delete(parsed.data.id);
646
+ }
647
+ return names;
648
+ }
649
+
650
+ /**
651
+ * The head of a rollout: what it says about the session as a whole.
652
+ *
653
+ * The metadata is in the first lines, so only they are read. With `wantTitle`, the same pass
654
+ * goes on to the first message of the person, within the first lines only: a forked session
655
+ * replays the history of its parent first, and the listing does not walk a history for a title.
656
+ */
657
+ async #headerOf(
658
+ rollout: Rollout,
659
+ wantTitle = false,
660
+ ): Promise<{ state: ParseState; firstMessage: string | undefined; read: boolean }> {
661
+ const state = newParseState();
662
+ // The records of the head are made with a state of their own, so the metadata stays as the
663
+ // first lines left it.
664
+ const recordState = newParseState();
665
+ let firstMessage: string | undefined;
666
+ let lines = 0;
667
+ try {
668
+ for await (const line of scanLines(rollout)) {
669
+ lines += 1;
670
+ if (line.complete === false) break;
671
+ if (!state.metaSeen) {
672
+ const record = parseNativeRecord(parseLine(line.bytes));
673
+ if (record !== null) this.#applyMeta(record, state);
674
+ }
675
+ if (wantTitle && lines <= TITLE_LINES && mattersForTitle(line.bytes)) {
676
+ for (const record of this.#recordsOf(line, recordState)) {
677
+ if (record.kind !== "user" || INSTRUCTIONS_PREAMBLE.test(record.text)) continue;
678
+ firstMessage = titleFromText(record.text, INSERTION_TAGS);
679
+ if (firstMessage !== undefined) break;
680
+ }
681
+ }
682
+ const titleDone = !wantTitle || firstMessage !== undefined || lines >= TITLE_LINES;
683
+ if (state.metaSeen && titleDone) break;
684
+ }
685
+ } catch {
686
+ // A head that cannot be read is a head that says nothing; the session is still listed.
687
+ return { state, firstMessage, read: false };
688
+ }
689
+ return { state, firstMessage, read: true };
690
+ }
691
+
692
+ /** Take what a record says about the session as a whole. */
693
+ #applyMeta(record: NativeRecord, state: ParseState): void {
694
+ const payload = payloadOf(record);
695
+ const at = instantOf(record.timestamp);
696
+ if (at !== undefined) state.updatedAt = at;
697
+
698
+ if (record.type === "session_meta") {
699
+ if (state.metaSeen) {
700
+ // A second session metadata is the parent's, replayed into this file. The own history of
701
+ // the session begins at its first turn.
702
+ state.awaitingOwnHistory = true;
703
+ state.ownTurnAt = undefined;
704
+ return;
705
+ }
706
+ state.metaSeen = true;
707
+ const cwd = payload["cwd"];
708
+ if (typeof cwd === "string") state.cwd = cwd;
709
+ const stamp = instantOf(payload["timestamp"]) ?? at;
710
+ if (stamp !== undefined) state.startedAt = stamp;
711
+ const start = payload["subagent_history_start_ordinal"];
712
+ if (typeof start === "number") state.ownHistoryFrom = start;
713
+ return;
714
+ }
715
+
716
+ if (record.type === "turn_context") {
717
+ const model = payload["model"];
718
+ if (typeof model === "string") state.model = model;
719
+ const cwd = payload["cwd"];
720
+ if (state.cwd === null && typeof cwd === "string") state.cwd = cwd;
721
+ if (state.ownTurnAt === undefined) state.ownTurnAt = record.ordinal;
722
+ if (state.awaitingOwnHistory && state.ownHistoryFrom === undefined) {
723
+ // The own history of a forked session begins at its first turn.
724
+ state.ownHistoryFrom = state.ownTurnAt;
725
+ }
726
+ state.awaitingOwnHistory = false;
727
+ }
728
+ }
729
+
730
+ /** The rollout of a session, or `null` when the installation does not hold it. */
731
+ async #find(id: SessionId): Promise<Rollout | null> {
732
+ for (const rollout of await this.#discover()) {
733
+ if (rollout.native === id.native) return rollout;
734
+ }
735
+ return null;
736
+ }
737
+
738
+ /**
739
+ * The records of one native line. One line may give several: a message with a call in it gives
740
+ * an assistant record and a tool-call record.
741
+ */
742
+ #recordsOf(line: RawLine, state: ParseState): Unlocated[] {
743
+ const parsed = parseLine(line.bytes);
744
+ const record = parseNativeRecord(parsed);
745
+ const incomplete = !line.complete;
746
+ const base: RecordBase = { incomplete };
747
+ if (record === null) return [unknownRecord(parsed, base)];
748
+ const payload = payloadOf(record);
749
+ const at = instantOf(record.timestamp);
750
+ const stamped: RecordBase = { ...(at === undefined ? {} : { at }), incomplete };
751
+
752
+ this.#applyMeta(record, state);
753
+
754
+ // The replayed history of a parent is not this session's own history. It is what lies between
755
+ // the parent's metadata and the first turn of this session.
756
+ if (state.awaitingOwnHistory && record.type !== "turn_context") return [];
757
+ if (
758
+ state.ownHistoryFrom !== undefined &&
759
+ record.ordinal !== undefined &&
760
+ record.ordinal < state.ownHistoryFrom
761
+ ) {
762
+ return [];
763
+ }
764
+
765
+ switch (record.type) {
766
+ case "session_meta":
767
+ case "turn_context":
768
+ // They say what the session is, not what happened in it; the reading side holds that in
769
+ // the session itself.
770
+ return [];
771
+ case "response_item":
772
+ return this.#conversationRecords(payload, state, stamped, parsed);
773
+ case "event_msg":
774
+ return this.#eventRecords(payload, stamped);
775
+ case "compacted":
776
+ return [{ kind: "compaction", summary: compactionSummary(payload), ...stamped }];
777
+ case "realtime_item":
778
+ return this.#realtimeRecords(payload, state, stamped, parsed);
779
+ default:
780
+ // Anything else the format grows is kept whole, and never fails the read.
781
+ return [unknownRecord(parsed, stamped)];
782
+ }
783
+ }
784
+
785
+ /** The records of the conversation layer: what the model was given and produced. */
786
+ #conversationRecords(
787
+ payload: Record<string, unknown>,
788
+ state: ParseState,
789
+ base: RecordBase,
790
+ raw: unknown,
791
+ ): Unlocated[] {
792
+ const kind = payload["type"];
793
+ if (kind === "message") {
794
+ const role = payload["role"];
795
+ const { text, images } = messageOf(payload["content"]);
796
+ if (role === "user") {
797
+ return [{ kind: "user", text, inserted: insertedRanges(text, INSERTION_TAGS), ...(images.length > 0 ? { images } : {}), ...base }];
798
+ }
799
+ if (role === "assistant") {
800
+ return [
801
+ {
802
+ kind: "assistant",
803
+ text,
804
+ ...(state.model === undefined ? {} : { model: state.model }),
805
+ ...base,
806
+ },
807
+ ];
808
+ }
809
+ // A message the owner did not send and the model did not say: the harness talking to
810
+ // itself. It is neither the owner's word nor the model's answer, so it is kept whole.
811
+ return [unknownRecord(raw, base)];
812
+ }
813
+ if (kind === "function_call" || kind === "custom_tool_call") {
814
+ const callId = payload["call_id"];
815
+ const name = payload["name"];
816
+ if (typeof callId !== "string" || callId.length === 0) return [unknownRecord(raw, base)];
817
+ if (typeof name === "string" && name.length > 0) state.toolNames.set(callId, name);
818
+ const input = kind === "custom_tool_call" ? payload["input"] : argumentsOf(payload["arguments"]);
819
+ const tool = typeof name === "string" && name.length > 0 ? name : "unknown";
820
+ const checklist = codexChecklist(tool, input);
821
+ return [
822
+ {
823
+ kind: "tool-call",
824
+ callId,
825
+ name: tool,
826
+ ...codexToolShape(tool, input),
827
+ ...(input === undefined ? {} : { input: jsonOf(input) }),
828
+ ...base,
829
+ },
830
+ ...(checklist === undefined ? [] : [{ kind: "checklist" as const, callId, items: checklist.items, ...base }]),
831
+ ];
832
+ }
833
+ if (kind === "function_call_output" || kind === "custom_tool_call_output") {
834
+ const callId = payload["call_id"];
835
+ if (typeof callId !== "string" || callId.length === 0) return [unknownRecord(raw, base)];
836
+ const { output, images } = outputOf(payload["output"]);
837
+ const outcome = outcomeOf(output, payload["status"]);
838
+ const name = state.toolNames.get(callId);
839
+ return [
840
+ {
841
+ kind: "tool-result",
842
+ callId,
843
+ ...(name === undefined ? {} : { name }),
844
+ output: optionalJsonOf(output),
845
+ ...(outcome === undefined ? {} : { ok: outcome.ok }),
846
+ ...(images.length > 0 ? { images } : {}),
847
+ ...base,
848
+ },
849
+ ];
850
+ }
851
+ if (kind === "reasoning") {
852
+ // The words of the thought when Codex wrote them out, else its summaries. A thought kept
853
+ // only as sealed text has no words to read: it is a thought all the same, and redacted. The
854
+ // item itself is what `raw` gives under the locator.
855
+ const written = textsOf(payload["content"]);
856
+ const text = written !== "" ? written : textsOf(payload["summary"]);
857
+ const sealed = typeof payload["encrypted_content"] === "string" && payload["encrypted_content"] !== "";
858
+ if (text === "" && !sealed) return [unknownRecord(raw, base)];
859
+ return [{ kind: "thinking", text, redacted: text === "", ...base }];
860
+ }
861
+ // Anything else the format grows is kept whole, so that nothing of the session is lost.
862
+ return [unknownRecord(raw, base)];
863
+ }
864
+
865
+ /**
866
+ * The records of the event layer: what only it knows.
867
+ *
868
+ * The event layer repeats the conversation under its own ids. What it alone knows is that a
869
+ * subagent was spawned.
870
+ */
871
+ #eventRecords(payload: Record<string, unknown>, base: RecordBase): Unlocated[] {
872
+ switch (payload["type"]) {
873
+ case "item_completed": {
874
+ // A paginated rollout carries the item whole under `item`.
875
+ const item = payload["item"];
876
+ return typeof item === "object" && item !== null && !Array.isArray(item)
877
+ ? this.#subAgentRecords(item as Record<string, unknown>, base)
878
+ : [];
879
+ }
880
+ case "sub_agent_activity":
881
+ // A legacy rollout writes the event itself, with the same fields.
882
+ return this.#subAgentRecords(payload, base);
883
+ default:
884
+ // The rest of the event layer repeats the conversation layer, which was read already. The
885
+ // compaction events are among them: `compacted` is written next to each one.
886
+ return [];
887
+ }
888
+ }
889
+
890
+ /** A spawned subagent is a nested record; its other activity is not. */
891
+ #subAgentRecords(activity: Record<string, unknown>, base: RecordBase): Unlocated[] {
892
+ const type = activity["type"];
893
+ if (type !== "SubAgentActivity" && type !== "sub_agent_activity") return [];
894
+ if (activity["kind"] !== "started") return [];
895
+ const child = activity["agent_thread_id"];
896
+ if (typeof child !== "string" || child.length === 0) return [];
897
+ return [
898
+ { kind: "nested", child: { harness: this.#harness, native: NativeSessionIdSchema.parse(child) }, ...base },
899
+ ];
900
+ }
901
+
902
+ /**
903
+ * The records of the voice mode: the transcript of the conversation.
904
+ *
905
+ * These segments carry the conversation itself, and the conversation layer does not repeat them,
906
+ * so they are read as what they are.
907
+ */
908
+ #realtimeRecords(
909
+ payload: Record<string, unknown>,
910
+ state: ParseState,
911
+ base: RecordBase,
912
+ raw: unknown,
913
+ ): Unlocated[] {
914
+ if (payload["type"] !== "transcript_segment") return [];
915
+ const role = payload["role"];
916
+ const text = payload["text"];
917
+ if (typeof text !== "string" || text.length === 0) return [];
918
+ // What a voice transcript holds is what was said: nothing in it is inserted.
919
+ if (role === "user") return [{ kind: "user", text, inserted: [], ...base }];
920
+ if (role === "assistant") {
921
+ return [
922
+ {
923
+ kind: "assistant",
924
+ text,
925
+ ...(state.model === undefined ? {} : { model: state.model }),
926
+ ...base,
927
+ },
928
+ ];
929
+ }
930
+ return [unknownRecord(raw, base)];
931
+ }
932
+ }
933
+
934
+ function unknownRecord(raw: unknown, base: RecordBase): Unlocated {
935
+ return { kind: "unknown", raw, ...base };
936
+ }
937
+
938
+ function reasonOf(error: unknown): string {
939
+ return error instanceof Error ? error.message : String(error);
940
+ }