@yielded/agent-storage-memory 0.1.0-beta.167 → 0.1.0-beta.168

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,20 +1,79 @@
1
- import { digestCanonicalBatch, EMPTY_TAIL_DIGEST } from "@yielded/agent/digest";
2
- import { ThreadId } from "@yielded/agent/identifiers";
1
+ import { canonicalJson, utf8ByteLength } from "@yielded/agent/digest";
2
+ import { ThreadId, RunId, SubmissionId as importSubmissionId } from "@yielded/agent/identifiers";
3
+ import type { IdempotencyKey } from "@yielded/agent/receipt";
3
4
  import {
4
- type ProducerEpoch,
5
+ ExportBatch,
6
+ ExportedRecord,
7
+ ExportRecord,
8
+ ReadRecord,
9
+ } from "@yielded/agent/record-format";
10
+ import {
11
+ CURRENT_RECORD_FORMAT,
12
+ RecordEnvelope,
5
13
  type RecordId,
6
14
  CanonicalRecordEnvelope,
7
15
  CanonicalSequence,
8
16
  ObservationOffset,
9
17
  type BatchId,
10
- type Digest,
18
+ MAX_RUN_EVIDENCE_RECORDS,
19
+ MAX_RUN_EVIDENCE_BYTES,
20
+ MAX_RUN_TERMINAL_BYTES,
21
+ RUN_TERMINAL_RESERVE_RECORDS,
22
+ MAX_RUN_RECOVERY_SUFFIX_RECORDS,
23
+ MAX_RUN_RECOVERY_SUFFIX_BYTES,
24
+ PromptRecord,
25
+ type PromptRecordEnvelope,
11
26
  } from "@yielded/agent/records";
27
+ import {
28
+ canonicalRunIds,
29
+ canonicalRecordBytes,
30
+ isWorkHandoff,
31
+ prepareProgressAppend,
32
+ ProgressAppendReader,
33
+ validateProgressAppend,
34
+ verifyRunContinuations,
35
+ } from "@yielded/agent/run-continuation";
12
36
  import {
13
37
  runIdForSubmission,
14
38
  subagentLineageRecordId,
15
39
  workerOriginRecordId,
40
+ workerInputRecordId,
41
+ toolCallSettledRecordId,
16
42
  } from "@yielded/agent/run-journal";
43
+ import { StorageOwner, makeStorageOwner } from "@yielded/agent/run-storage";
44
+ import {
45
+ submissionAbortRecordId,
46
+ submissionInputRecordId,
47
+ submissionSettlementRecordId,
48
+ } from "@yielded/agent/submission-ledger";
49
+ import {
50
+ ThreadArchiveRange,
51
+ ThreadArchiveRangeRead,
52
+ ThreadArchiveRangeSeal,
53
+ ThreadArchiveRangePublish,
54
+ ThreadArchiveRangeRequest,
55
+ MAX_ARCHIVE_RANGE_RECORDS,
56
+ MAX_ARCHIVE_RANGE_BYTES,
57
+ type ThreadArchiveStorage,
58
+ } from "@yielded/agent/thread-archive-range";
17
59
  import {
60
+ makeThreadImportProgress,
61
+ captureImportSource,
62
+ prepareImportPage,
63
+ finishThreadImport,
64
+ verifyImportedReferences,
65
+ validateClosedRun,
66
+ rebuildClosedSubmission,
67
+ invalidThreadArchive,
68
+ ThreadImportReader,
69
+ ThreadDeliveryImportReader,
70
+ ThreadImport,
71
+ ThreadImportRejected,
72
+ } from "@yielded/agent/thread-import";
73
+ import { verifyThreadInvariants } from "@yielded/agent/thread-invariants";
74
+ import {
75
+ type AppendRequest,
76
+ type ExecutionCheckpoint,
18
77
  type ThreadCheckpoint,
19
78
  ThreadPeerCountRequest,
20
79
  ThreadIdentity,
@@ -22,30 +81,84 @@ import {
22
81
  AppendConflict,
23
82
  AppendResult,
24
83
  CheckpointRejected,
25
- ThreadExportRequest,
26
- ThreadExport,
84
+ ThreadExportRecord,
85
+ ThreadExportRejected,
27
86
  ThreadMaterialization,
28
87
  ThreadNotMaterialized,
29
88
  ThreadObservation,
30
89
  ThreadReadRequest,
90
+ ThreadPromptRead,
31
91
  ThreadStore,
92
+ ThreadReader,
32
93
  type ThreadCheckpoints,
33
94
  ThreadStoreError,
34
95
  ThreadTail,
35
96
  ThreadTailRequest,
36
- FenceRejected,
37
- FencedAppendRequest,
97
+ PreparedAppend,
98
+ type CommitBatch,
38
99
  LoadCheckpointRequest,
39
100
  SaveCheckpointRequest,
40
- SaveRecoveryCheckpointRequest,
41
- type ThreadRecoveryCheckpoints,
42
- MAX_THREAD_EXPORT_RECORDS,
101
+ PROMPT_EVIDENCE_TAGS,
102
+ ThreadVerificationRequest,
103
+ ThreadWorkerCapacityRequest,
104
+ ThreadWorkerCapacity,
105
+ streamExport,
106
+ ThreadExportSource,
107
+ canonicalBatchFitsTransfer,
108
+ transferPageFits,
43
109
  } from "@yielded/agent/thread-store";
44
- import { Context, Crypto, Effect, Layer, Option, PubSub, Ref, Schema, Stream } from "effect";
110
+ import {
111
+ checkExportSource,
112
+ exportThreadPage,
113
+ ThreadExporterReader,
114
+ type TransferManifest,
115
+ } from "@yielded/agent/thread-transfer";
116
+ import {
117
+ WORK_INDEX_VERSION,
118
+ MAX_WORK_REBUILD_RECORDS,
119
+ MAX_WORK_REBUILD_BYTES,
120
+ WorkDiscoveryUnavailable,
121
+ ThreadWorkRequest,
122
+ WorkThreadsRequest,
123
+ WorkIndexRebuildRequest,
124
+ decodeWorkCursor,
125
+ encodeWorkCursor,
126
+ validateWorkPage,
127
+ workIndexChanges,
128
+ advanceWorkEntry,
129
+ type CanonicalWorkEntry,
130
+ type WorkerReportingMode,
131
+ type ThreadWorkStorage,
132
+ } from "@yielded/agent/thread-work";
133
+ import {
134
+ Context,
135
+ Crypto,
136
+ Effect,
137
+ Layer,
138
+ MutableRef,
139
+ Option,
140
+ PubSub,
141
+ Ref,
142
+ Schema,
143
+ Semaphore,
144
+ Stream,
145
+ } from "effect";
45
146
  import { Base64 } from "effect/encoding";
46
147
 
148
+ import {
149
+ MemoryThreadStoreKernel,
150
+ type MemoryLedgerTransfer,
151
+ type MemoryLedgerImport,
152
+ type MemoryDeliveryTransfer,
153
+ type MemoryDeliveryImport,
154
+ type MemoryWorkOwner,
155
+ type PreparedMemoryAppend,
156
+ } from "./internal/MemoryThreadStoreKernel.ts";
157
+ import { boundedWorkSelection } from "./internal/WorkSelection.ts";
158
+
47
159
  const MAX_THREADS = 256;
48
- const MAX_RECORDS_PER_THREAD = MAX_THREAD_EXPORT_RECORDS;
160
+ const RECORD_CHUNK_SIZE = 1_024;
161
+ const READ_PAGE_SIZE = 8;
49
162
  const MAX_CHECKPOINTS_PER_THREAD = 1_024;
50
163
 
51
164
  const ThreadCapacity = Context.Reference<number>(
@@ -54,36 +167,556 @@ const ThreadCapacity = Context.Reference<number>(
54
167
  );
55
168
 
56
169
  interface StoredBatch {
57
- readonly digest: Digest;
170
+ /** Preserve the exact destination wire, including additive fields unknown to this reader. */
171
+ readonly batchJson: string;
58
172
  readonly result: AppendResult;
59
173
  }
60
174
 
61
175
  interface StoredThread {
176
+ /** Append-only owner indexes; readers must constrain them to their captured canonical tail. */
177
+ readonly runRecords: Map<string, Array<CanonicalRecordEnvelope>>;
178
+ readonly operationRecords: Map<string, Array<CanonicalRecordEnvelope>>;
179
+ readonly handoffs: Array<CanonicalRecordEnvelope>;
180
+ readonly agentUpdates: Array<CanonicalRecordEnvelope>;
181
+ readonly deliveryPredecessors: Array<CanonicalRecordEnvelope>;
182
+ readonly toolDeclarations: Map<string, CanonicalRecordEnvelope | null>;
183
+ readonly promptEvidence: Array<CanonicalRecordEnvelope>;
184
+ readonly modelCompletions: Array<CanonicalRecordEnvelope>;
185
+ readonly applicationInputs: Array<CanonicalRecordEnvelope>;
186
+ readonly durableOwners: Array<CanonicalRecordEnvelope>;
187
+ readonly windowBoundaries: Map<string, Array<CanonicalRecordEnvelope>>;
62
188
  readonly peerCount: number;
63
- readonly unverifiedWorkerInputs: ReadonlySet<string>;
64
- readonly workerRecords: ReadonlyMap<string, ReadonlyArray<CanonicalRecordEnvelope>>;
65
- readonly byId: ReadonlyMap<string, CanonicalRecordEnvelope>;
66
- readonly runInputs: ReadonlyMap<string, CanonicalRecordEnvelope | null>;
67
- readonly outstanding: ReadonlyMap<string, ReadonlyArray<CanonicalRecordEnvelope>>;
68
- readonly producerEpoch: ProducerEpoch;
189
+ readonly workerRecords: Map<string, Array<CanonicalRecordEnvelope>>;
190
+ readonly byId: Map<string, CanonicalRecordEnvelope>;
191
+ readonly runInputs: Map<string, CanonicalRecordEnvelope | null>;
69
192
  readonly tailSequence: CanonicalSequence;
70
- readonly tailDigest: Digest;
71
- readonly records: ReadonlyArray<CanonicalRecordEnvelope>;
72
- readonly recordIds: ReadonlySet<RecordId>;
73
- readonly batches: ReadonlyMap<BatchId, StoredBatch>;
74
- readonly tailDigests: ReadonlyMap<CanonicalSequence, Digest>;
75
- readonly checkpoints: ReadonlyMap<CanonicalSequence, ThreadCheckpoint>;
76
- readonly recoveryCheckpoint?: ThreadCheckpoint;
193
+ readonly ranges: Map<number, ThreadArchiveRange>;
194
+ readonly rangeStarts: Array<number>;
195
+ readonly archiveContents: Map<number, Map<BatchId, string>>;
196
+ readonly chunks: Map<
197
+ number,
198
+ { readonly records: Array<CanonicalRecordEnvelope>; readonly bytes: Array<number> }
199
+ >;
200
+ readonly batches: Map<BatchId, StoredBatch>;
201
+ readonly executions: Map<RunId, ExecutionCheckpoint>;
202
+ readonly executionCommits: Map<
203
+ RunId,
204
+ { readonly batchId: BatchId; readonly intent: string; readonly result: AppendResult }
205
+ >;
206
+ readonly batchesByFirst: Map<number, BatchId>;
207
+ readonly checkpoints: Map<CanonicalSequence, ThreadCheckpoint>;
77
208
  }
78
209
 
210
+ const storedRecord = (thread: StoredThread, sequence: number) =>
211
+ thread.chunks.get(Math.floor((sequence - 1) / RECORD_CHUNK_SIZE))?.records[
212
+ (sequence - 1) % RECORD_CHUNK_SIZE
213
+ ];
214
+
215
+ const isBatchBoundary = (thread: StoredThread, sequence: CanonicalSequence): boolean => {
216
+ if (sequence === 0) return true;
217
+ const record = storedRecord(thread, sequence);
218
+
219
+ return (
220
+ record !== undefined && thread.batches.get(record.batchId)?.result.lastSequence === sequence
221
+ );
222
+ };
223
+
224
+ const storedBytes = (thread: StoredThread, sequence: number) =>
225
+ thread.chunks.get(Math.floor((sequence - 1) / RECORD_CHUNK_SIZE))?.bytes[
226
+ (sequence - 1) % RECORD_CHUNK_SIZE
227
+ ];
228
+
229
+ const storedPage = (thread: StoredThread, after: number, limit: number) => {
230
+ const page: Array<CanonicalRecordEnvelope> = [];
231
+ let bytes = 0;
232
+
233
+ for (
234
+ let sequence = after + 1;
235
+ sequence <= Math.min(thread.tailSequence, after + limit);
236
+ sequence++
237
+ ) {
238
+ const record = storedRecord(thread, sequence);
239
+
240
+ if (record === undefined) throw new Error("Incomplete canonical chunk table");
241
+ const size = storedBytes(thread, sequence) ?? 0;
242
+
243
+ if (bytes + size > 32 * 1024 * 1024) break;
244
+ bytes += size;
245
+ page.push(record);
246
+ }
247
+
248
+ return page;
249
+ };
250
+
251
+ const retainRecords = (thread: StoredThread, records: ReadonlyArray<CanonicalRecordEnvelope>) => {
252
+ for (const entry of records) {
253
+ const chunkId = Math.floor((entry.sequence - 1) / RECORD_CHUNK_SIZE);
254
+ let chunk = thread.chunks.get(chunkId);
255
+
256
+ if (chunk === undefined) {
257
+ chunk = { records: [], bytes: [] };
258
+ thread.chunks.set(chunkId, chunk);
259
+ }
260
+ chunk.records.push(entry);
261
+ chunk.bytes.push(canonicalRecordBytes(entry.record));
262
+ }
263
+ };
264
+
79
265
  interface MemoryState {
80
- readonly threads: ReadonlyMap<ThreadId, StoredThread>;
266
+ readonly threads: Map<ThreadId, StoredThread>;
267
+ }
268
+
269
+ type NativeIndexes = Pick<
270
+ StoredThread,
271
+ | "toolDeclarations"
272
+ | "promptEvidence"
273
+ | "modelCompletions"
274
+ | "applicationInputs"
275
+ | "durableOwners"
276
+ | "windowBoundaries"
277
+ | "peerCount"
278
+ | "workerRecords"
279
+ | "byId"
280
+ | "runInputs"
281
+ | "runRecords"
282
+ | "handoffs"
283
+ | "operationRecords"
284
+ | "agentUpdates"
285
+ | "deliveryPredecessors"
286
+ >;
287
+
288
+ const emptyIndexes = (): NativeIndexes => ({
289
+ toolDeclarations: new Map(),
290
+ promptEvidence: [],
291
+ modelCompletions: [],
292
+ applicationInputs: [],
293
+ durableOwners: [],
294
+ windowBoundaries: new Map(),
295
+ peerCount: 0,
296
+ workerRecords: new Map(),
297
+ byId: new Map(),
298
+ runInputs: new Map(),
299
+ runRecords: new Map(),
300
+ handoffs: [],
301
+ agentUpdates: [],
302
+ deliveryPredecessors: [],
303
+ operationRecords: new Map(),
304
+ });
305
+
306
+ interface WorkIndex {
307
+ readonly version: number;
308
+ readonly state: "ready" | "rebuilding";
309
+ readonly through: CanonicalSequence;
310
+ readonly reporting: WorkerReportingMode;
311
+ readonly entryCount: number;
312
+ readonly entries: Map<string, CanonicalWorkEntry>;
313
+ readonly messages: Map<IdempotencyKey, Set<string>>;
314
+ readonly foreign: { child: number; worker: number; delivery: number; peer: number };
315
+ readonly workers: Map<ThreadId, { ordinary: number; update: number; effects: number }>;
316
+ readonly liveInputs: Array<LiveInputLocator>;
317
+ readonly inputsByWorker: Map<ThreadId, Array<LiveInputLocator>>;
318
+ readonly rebuildingIndexes?: NativeIndexes;
81
319
  }
82
320
 
321
+ interface LiveInputLocator {
322
+ readonly sequence: CanonicalSequence;
323
+ readonly id: string;
324
+ }
325
+
326
+ const liveInputPosition = (entries: ReadonlyArray<LiveInputLocator>, sequence: number) => {
327
+ let low = 0;
328
+ let high = entries.length;
329
+
330
+ while (low < high) {
331
+ const middle = Math.floor((low + high) / 2);
332
+
333
+ if (entries[middle].sequence < sequence) low = middle + 1;
334
+ else high = middle;
335
+ }
336
+
337
+ return low;
338
+ };
339
+
340
+ const changeLiveInput = (
341
+ entries: Array<LiveInputLocator>,
342
+ entry: CanonicalWorkEntry,
343
+ change: 1 | -1,
344
+ ) => {
345
+ const at = liveInputPosition(entries, entry.createdSequence);
346
+
347
+ if (change === 1) entries.splice(at, 0, { sequence: entry.createdSequence, id: entry.id });
348
+ else if (entries[at]?.id === entry.id) entries.splice(at, 1);
349
+ };
350
+
351
+ const emptyWorkIndex = (): WorkIndex => ({
352
+ version: WORK_INDEX_VERSION,
353
+ state: "ready",
354
+ through: ZERO_CANONICAL_SEQUENCE,
355
+ reporting: "none",
356
+ entryCount: 0,
357
+ entries: new Map(),
358
+ messages: new Map(),
359
+ foreign: { child: 0, worker: 0, delivery: 0, peer: 0 },
360
+ workers: new Map(),
361
+ liveInputs: [],
362
+ inputsByWorker: new Map(),
363
+ });
364
+
365
+ const changeWorkCounts = (index: WorkIndex, entry: CanonicalWorkEntry, change: 1 | -1) => {
366
+ const owner = entry.owner;
367
+
368
+ if (
369
+ owner._tag === "Child" ||
370
+ (owner._tag === "Handoff" &&
371
+ (owner.kind === "reservation" || owner.kind === "child-accounting"))
372
+ )
373
+ index.foreign.child += change;
374
+ if (
375
+ ["WorkerInput", "WorkerEffects", "Report"].includes(owner._tag) ||
376
+ (owner._tag === "Handoff" && owner.kind === "worker-stop")
377
+ )
378
+ index.foreign.worker += change;
379
+ if (owner._tag === "Handoff" && ["peer", "update", "report"].includes(owner.kind))
380
+ index.foreign.delivery += change;
381
+ if (owner._tag === "Handoff" && owner.kind === "peer") index.foreign.peer += change;
382
+ if (owner._tag === "WorkerInput") {
383
+ changeLiveInput(index.liveInputs, entry, change);
384
+ let inputs = index.inputsByWorker.get(owner.workerThreadId);
385
+
386
+ if (inputs === undefined) index.inputsByWorker.set(owner.workerThreadId, (inputs = []));
387
+ changeLiveInput(inputs, entry, change);
388
+ if (inputs.length === 0) index.inputsByWorker.delete(owner.workerThreadId);
389
+ let worker = index.workers.get(owner.workerThreadId);
390
+
391
+ if (worker === undefined)
392
+ index.workers.set(owner.workerThreadId, (worker = { ordinary: 0, update: 0, effects: 0 }));
393
+ if (owner.update) worker.update += change;
394
+ else worker.ordinary += change;
395
+ if (worker.ordinary + worker.update + worker.effects === 0)
396
+ index.workers.delete(owner.workerThreadId);
397
+ }
398
+ };
399
+
400
+ /** Prepare only this batch's changes; commit under the writer gate with canonical publication. */
401
+ const prepareWork = Effect.fnUntraced(function* (
402
+ previous: WorkIndex,
403
+ records: ReadonlyArray<CanonicalRecordEnvelope>,
404
+ retained: (threadId: ThreadId, messageId: IdempotencyKey) => boolean,
405
+ ) {
406
+ const changes = new Map<string, CanonicalWorkEntry | undefined>();
407
+ let reporting = previous.reporting;
408
+ let through = previous.through;
409
+
410
+ for (const record of records) {
411
+ for (const change of yield* workIndexChanges(record, reporting)) {
412
+ if (change._tag === "WorkerMode") reporting = change.mode;
413
+ else if (change._tag === "Remove") {
414
+ const entry = changes.has(change.id)
415
+ ? changes.get(change.id)
416
+ : previous.entries.get(change.id);
417
+
418
+ if (
419
+ change.stateRecordId === undefined ||
420
+ (entry?.stateReference._tag === "Canonical" &&
421
+ entry.stateReference.recordId === change.stateRecordId)
422
+ )
423
+ changes.set(change.id, undefined);
424
+ } else {
425
+ const entry = advanceWorkEntry(
426
+ changes.has(change.entry.id)
427
+ ? changes.get(change.entry.id)
428
+ : previous.entries.get(change.entry.id),
429
+ change.entry,
430
+ );
431
+
432
+ changes.set(
433
+ entry.id,
434
+ entry.owner._tag === "Handoff" &&
435
+ entry.owner.messageId !== undefined &&
436
+ retained(record.threadId, entry.owner.messageId)
437
+ ? undefined
438
+ : entry,
439
+ );
440
+ }
441
+ }
442
+ through = record.sequence;
443
+ }
444
+
445
+ return (): WorkIndex => {
446
+ const { entries, messages } = previous;
447
+
448
+ for (const [id, entry] of changes) {
449
+ const old = entries.get(id);
450
+
451
+ if (old !== undefined) changeWorkCounts(previous, old, -1);
452
+
453
+ if (old?.owner._tag === "Handoff" && old.owner.messageId !== undefined) {
454
+ const ids = messages.get(old.owner.messageId);
455
+
456
+ ids?.delete(id);
457
+ if (ids?.size === 0) messages.delete(old.owner.messageId);
458
+ }
459
+ if (entry === undefined) entries.delete(id);
460
+ else {
461
+ entries.set(id, entry);
462
+ changeWorkCounts(previous, entry, 1);
463
+ if (entry.owner._tag === "Handoff" && entry.owner.messageId !== undefined) {
464
+ const ids = messages.get(entry.owner.messageId);
465
+
466
+ if (ids === undefined) messages.set(entry.owner.messageId, new Set([id]));
467
+ else ids.add(id);
468
+ }
469
+ }
470
+ }
471
+
472
+ return { ...previous, through, reporting, entryCount: entries.size };
473
+ };
474
+ });
475
+
476
+ /** First sequence strictly after the cursor, without traversing a Run's earlier facts. */
477
+ const upperSequence = (records: ReadonlyArray<CanonicalRecordEnvelope>, sequence: number) => {
478
+ let low = 0;
479
+ let high = records.length;
480
+
481
+ while (low < high) {
482
+ const mid = Math.floor((low + high) / 2);
483
+
484
+ if (records[mid].sequence <= sequence) low = mid + 1;
485
+ else high = mid;
486
+ }
487
+
488
+ return low;
489
+ };
490
+
491
+ const appendToIndex = (
492
+ index: Map<string, Array<CanonicalRecordEnvelope>>,
493
+ key: string,
494
+ entry: CanonicalRecordEnvelope,
495
+ ) => {
496
+ const records = index.get(key);
497
+
498
+ if (records === undefined) index.set(key, [entry]);
499
+ else records.push(entry);
500
+ };
501
+
502
+ /** Only call at successful synchronous publication, or on indexes not yet reachable by readers. */
503
+ const commitIndex = (
504
+ index: Map<string, Array<CanonicalRecordEnvelope>>,
505
+ additions: ReadonlyMap<string, Array<CanonicalRecordEnvelope>>,
506
+ ) => {
507
+ for (const [key, entries] of additions) {
508
+ const records = index.get(key);
509
+
510
+ if (records === undefined) index.set(key, entries);
511
+ else for (const entry of entries) records.push(entry);
512
+ }
513
+ };
514
+
515
+ const prepareIndexes = (
516
+ previous: NativeIndexes,
517
+ records: ReadonlyArray<CanonicalRecordEnvelope>,
518
+ ) => {
519
+ let peerCount = previous.peerCount;
520
+ // Latest execution entries are replaced in this private map; owner histories only append.
521
+ // Rebuild staging is private: reuse it only in the synchronous cursor publication step.
522
+ const workerRecords = new Map<string, Array<CanonicalRecordEnvelope>>();
523
+
524
+ const wasWorker = (previous.workerRecords.get("worker") ?? []).some(
525
+ ({ record }) => record.payload._tag === "WorkerOriginRecorded",
526
+ );
527
+
528
+ const isWorker =
529
+ wasWorker || records.some(({ record }) => record.payload._tag === "WorkerOriginRecorded");
530
+
531
+ // A Thread can gain a Worker origin after its initial history. Initialize once on that boundary.
532
+ if (isWorker && !wasWorker)
533
+ for (const entry of previous.byId.values()) {
534
+ const payload = entry.record.payload;
535
+
536
+ if (
537
+ (payload._tag === "UserInputRecorded" || payload._tag === "SubmissionSettled") &&
538
+ payload.runId !== undefined
539
+ ) {
540
+ const key = `execution:${payload._tag}`;
541
+
542
+ if ((workerRecords.get(key)?.[0]?.sequence ?? 0) < entry.sequence)
543
+ workerRecords.set(key, [entry]);
544
+ }
545
+ }
546
+ const workerAppends = new Map<string, Array<CanonicalRecordEnvelope>>();
547
+ const byId = new Map<string, CanonicalRecordEnvelope>();
548
+ const runInputs = new Map<string, CanonicalRecordEnvelope | null>();
549
+ const runRecords = new Map<string, Array<CanonicalRecordEnvelope>>();
550
+ const handoffs: Array<CanonicalRecordEnvelope> = [];
551
+ const agentUpdates: Array<CanonicalRecordEnvelope> = [];
552
+
553
+ const deliveryPredecessors: Array<CanonicalRecordEnvelope> = [];
554
+
555
+ const nativeAdds = emptyIndexes();
556
+ const operationAppends = new Map<string, Array<CanonicalRecordEnvelope>>();
557
+
558
+ for (const entry of records) {
559
+ byId.set(entry.record.recordId, entry);
560
+ const payload = entry.record.payload;
561
+
562
+ if (payload._tag === "ModelResponseRecorded")
563
+ for (const operation of payload.toolOperations) {
564
+ const id = toolCallSettledRecordId(payload.runId, payload.turn, operation.toolCallId);
565
+
566
+ nativeAdds.toolDeclarations.set(
567
+ id,
568
+ previous.toolDeclarations.has(id) || nativeAdds.toolDeclarations.has(id) ? null : entry,
569
+ );
570
+ }
571
+ if (PROMPT_EVIDENCE_TAGS.some((tag) => tag === payload._tag))
572
+ nativeAdds.promptEvidence.push(entry);
573
+ if (payload._tag === "ModelCompleted") nativeAdds.modelCompletions.push(entry);
574
+ if (payload._tag === "UserInputRecorded") {
575
+ const message = payload.messageAdmission;
576
+
577
+ if (
578
+ message === undefined ||
579
+ !("_tag" in message) ||
580
+ (message._tag !== "WorkerCompletion" && message._tag !== "WorkerUpdate")
581
+ )
582
+ nativeAdds.applicationInputs.push(entry);
583
+ }
584
+ if (
585
+ (payload._tag === "UserInputRecorded" && payload.submissionId !== undefined) ||
586
+ [
587
+ "RunStarted",
588
+ "SubmissionSettled",
589
+ "AbortRequested",
590
+ "WorkerOriginRecorded",
591
+ "SubagentLineageRecorded",
592
+ ].includes(payload._tag)
593
+ )
594
+ nativeAdds.durableOwners.push(entry);
595
+ if (payload._tag === "CompactionCreated" && payload.kind === "rollover")
596
+ for (let height = 0; height < 53; height++)
597
+ appendToIndex(
598
+ nativeAdds.windowBoundaries,
599
+ `${height}:${Math.floor(payload.coversThrough / 2 ** height)}`,
600
+ entry,
601
+ );
602
+
603
+ for (const runId of canonicalRunIds(entry.record)) appendToIndex(runRecords, runId, entry);
604
+ if (isWorkHandoff(entry.record)) handoffs.push(entry);
605
+ if (payload._tag === "AgentUpdateEmitted") agentUpdates.push(entry);
606
+ if (
607
+ payload._tag === "WorkerReportPrepared" ||
608
+ (payload._tag === "AgentUpdateEmitted" && payload.delivery !== undefined)
609
+ )
610
+ deliveryPredecessors.push(entry);
611
+ if ("runId" in payload && payload.runId !== undefined && "toolCallId" in payload) {
612
+ appendToIndex(operationAppends, JSON.stringify([payload.runId, payload.toolCallId]), entry);
613
+ if (payload._tag === "ToolApprovalRequested" || payload._tag === "ToolApprovalDecided")
614
+ appendToIndex(
615
+ operationAppends,
616
+ JSON.stringify([payload.runId, payload.toolCallId, "approval"]),
617
+ entry,
618
+ );
619
+ if (payload._tag === "ToolCallSettled")
620
+ appendToIndex(
621
+ operationAppends,
622
+ JSON.stringify([payload.runId, payload.toolCallId, "settled"]),
623
+ entry,
624
+ );
625
+ }
626
+
627
+ if (payload._tag === "PeerMessagePrepared") peerCount++;
628
+ if (
629
+ isWorker &&
630
+ (payload._tag === "UserInputRecorded" || payload._tag === "SubmissionSettled") &&
631
+ payload.runId !== undefined
632
+ )
633
+ workerRecords.set(`execution:${payload._tag}`, [entry]);
634
+
635
+ const workerKey =
636
+ payload._tag === "SubtreeBudgetReserved"
637
+ ? `subtree:${payload.sourceSubmissionId ?? ""}`
638
+ : payload._tag === "SubagentJoined"
639
+ ? `joined:${payload.runId}`
640
+ : ["ThreadCreated", "WorkerOriginRecorded", "SubagentLineageRecorded"].includes(
641
+ payload._tag,
642
+ )
643
+ ? "worker"
644
+ : undefined;
645
+
646
+ if (workerKey !== undefined) appendToIndex(workerAppends, workerKey, entry);
647
+ if (payload._tag === "WorkerStopRequested")
648
+ appendToIndex(workerAppends, `stop:${payload.command.worker.threadId}`, entry);
649
+ if (
650
+ payload._tag === "WorkerInputRequested" &&
651
+ payload.admission.sourceSubmissionId !== undefined
652
+ )
653
+ appendToIndex(workerAppends, `funding:${payload.admission.sourceSubmissionId}`, entry);
654
+ if (payload._tag === "WorkerInputCompleted") {
655
+ const requested =
656
+ previous.byId.get(workerInputRecordId(payload.messageId)) ??
657
+ byId.get(workerInputRecordId(payload.messageId));
658
+
659
+ const admission = requested?.record.payload;
660
+
661
+ if (
662
+ admission?._tag === "WorkerInputRequested" &&
663
+ admission.admission.sourceSubmissionId !== undefined
664
+ )
665
+ appendToIndex(workerAppends, `funding:${admission.admission.sourceSubmissionId}`, entry);
666
+ }
667
+ if (payload._tag === "WorkerInputRefused") {
668
+ const reserved = (
669
+ previous.byId.get(payload.reservation.recordId) ?? byId.get(payload.reservation.recordId)
670
+ )?.record.payload;
671
+
672
+ const sourceSubmissionId =
673
+ reserved?._tag === "WorkerInputRequested"
674
+ ? reserved.admission.sourceSubmissionId
675
+ : reserved?._tag === "SubtreeBudgetReserved"
676
+ ? reserved.sourceSubmissionId
677
+ : undefined;
678
+
679
+ if (sourceSubmissionId !== undefined)
680
+ appendToIndex(workerAppends, `funding:${sourceSubmissionId}`, entry);
681
+ }
682
+ if (
683
+ payload._tag === "UserInputRecorded" &&
684
+ payload.kind === "user" &&
685
+ payload.runId !== undefined
686
+ )
687
+ runInputs.set(
688
+ payload.runId,
689
+ runInputs.has(payload.runId) || previous.runInputs.has(payload.runId) ? null : entry,
690
+ );
691
+ }
692
+
693
+ return {
694
+ indexes: { ...previous, peerCount },
695
+ commit: () => {
696
+ commitIndex(previous.runRecords, runRecords);
697
+ commitIndex(previous.operationRecords, operationAppends);
698
+ for (const [id, entry] of nativeAdds.toolDeclarations)
699
+ previous.toolDeclarations.set(id, entry);
700
+ for (const key of ["promptEvidence", "modelCompletions", "applicationInputs"] as const)
701
+ for (const entry of nativeAdds[key]) previous[key].push(entry);
702
+ commitIndex(previous.windowBoundaries, nativeAdds.windowBoundaries);
703
+ if (previous.durableOwners.length === 0 && nativeAdds.durableOwners[0] !== undefined)
704
+ previous.durableOwners.push(nativeAdds.durableOwners[0]);
705
+ for (const [id, entry] of byId) previous.byId.set(id, entry);
706
+ for (const [id, entry] of runInputs) previous.runInputs.set(id, entry);
707
+ for (const [key, entries] of workerRecords) previous.workerRecords.set(key, entries);
708
+ commitIndex(previous.workerRecords, workerAppends);
709
+ for (const entry of handoffs) previous.handoffs.push(entry);
710
+ for (const entry of agentUpdates) previous.agentUpdates.push(entry);
711
+ for (const entry of deliveryPredecessors) previous.deliveryPredecessors.push(entry);
712
+ },
713
+ };
714
+ };
715
+
83
716
  type AppendDecision =
84
717
  | {
85
718
  readonly _tag: "failure";
86
- readonly error: ThreadStoreError | ThreadNotMaterialized | AppendConflict | FenceRejected;
719
+ readonly error: ThreadStoreError | ThreadNotMaterialized | AppendConflict;
87
720
  }
88
721
  | {
89
722
  readonly _tag: "success";
@@ -92,7 +725,7 @@ type AppendDecision =
92
725
  };
93
726
 
94
727
  type MaterializeDecision =
95
- | { readonly _tag: "failure"; readonly error: ThreadStoreError | FenceRejected }
728
+ | { readonly _tag: "failure"; readonly error: ThreadStoreError }
96
729
  | { readonly _tag: "success" };
97
730
 
98
731
  type CheckpointDecision =
@@ -107,35 +740,33 @@ const storeError = (operation: string, message: string, cause?: unknown): Thread
107
740
  ? ThreadStoreError.make({ operation, message })
108
741
  : ThreadStoreError.make({ operation, message, cause });
109
742
 
110
- const validate = Effect.fn("MemoryThreadStore.validate")(
111
- <A, I>(
112
- schema: Schema.Codec<A, I>,
113
- operation: string,
114
- value: unknown,
115
- ): Effect.Effect<A, ThreadStoreError> =>
116
- Schema.encodeUnknownEffect(schema)(value).pipe(
117
- Effect.flatMap(Schema.decodeUnknownEffect(schema)),
118
- Effect.mapError((error) => storeError(operation, `Invalid ${operation} request`, error)),
119
- ),
120
- );
743
+ const validate = <A, I>(
744
+ schema: Schema.Codec<A, I>,
745
+ operation: string,
746
+ value: unknown,
747
+ ): Effect.Effect<A, ThreadStoreError> =>
748
+ Schema.encodeUnknownEffect(schema)(value).pipe(
749
+ Effect.flatMap(Schema.decodeUnknownEffect(schema)),
750
+ Effect.mapError((error) => storeError(operation, `Invalid ${operation} request`, error)),
751
+ );
121
752
 
122
753
  const decodeCanonicalSequence = Schema.decodeSync(CanonicalSequence);
123
754
  const ZERO_CANONICAL_SEQUENCE = decodeCanonicalSequence(0);
124
755
 
125
- const offsetSequence = Effect.fn("MemoryThreadStore.offsetSequence")((
756
+ const offsetSequence = Effect.fnUntraced(function* (
126
757
  threadId: ThreadId,
127
758
  offset: ObservationOffset | undefined,
128
- ): Effect.Effect<CanonicalSequence, ThreadStoreError> => {
129
- if (offset === undefined) return Effect.succeed(ZERO_CANONICAL_SEQUENCE);
759
+ ): Effect.fn.Return<CanonicalSequence, ThreadStoreError> {
760
+ if (offset === undefined) return yield* Effect.succeed(ZERO_CANONICAL_SEQUENCE);
130
761
  const prefix = `memory:v1:${Base64.encode(threadId)}:`;
131
762
  const encodedSequence = offset.startsWith(prefix) ? offset.slice(prefix.length) : "";
132
763
 
133
764
  if (!/^\d+$/.test(encodedSequence)) {
134
- return Effect.fail(storeError("observe", "Malformed observation offset"));
765
+ return yield* Effect.fail(storeError("observe", "Malformed observation offset"));
135
766
  }
136
767
  const sequence = Number(encodedSequence);
137
768
 
138
- return Number.isSafeInteger(sequence)
769
+ return yield* Number.isSafeInteger(sequence)
139
770
  ? Schema.decodeEffect(CanonicalSequence)(sequence).pipe(
140
771
  Effect.mapError(() => storeError("observe", "Malformed observation offset")),
141
772
  )
@@ -145,13 +776,35 @@ const offsetSequence = Effect.fn("MemoryThreadStore.offsetSequence")((
145
776
  const observationOffset = (threadId: ThreadId, sequence: CanonicalSequence): ObservationOffset =>
146
777
  Schema.decodeSync(ObservationOffset)(`memory:v1:${Base64.encode(threadId)}:${sequence}`);
147
778
 
148
- const findThread = Effect.fn("MemoryThreadStore.findThread")((
779
+ const batchEnvelopes = (
780
+ threadId: ThreadId,
781
+ batch: CommitBatch,
782
+ firstSequence: CanonicalSequence,
783
+ ): ReadonlyArray<CanonicalRecordEnvelope> =>
784
+ batch.records.map((record, index) => {
785
+ const sequence = decodeCanonicalSequence(firstSequence + index);
786
+
787
+ return CanonicalRecordEnvelope.make({
788
+ threadId,
789
+ batchId: batch.batchId,
790
+ sequence,
791
+ offset: observationOffset(threadId, sequence),
792
+ record,
793
+ });
794
+ });
795
+
796
+ const decodeStoredBatch = (batchJson: string, operation: string) =>
797
+ Schema.decodeEffect(Schema.fromJsonString(ExportBatch))(batchJson).pipe(
798
+ Effect.mapError((cause) => storeError(operation, "Invalid canonical batch JSON", cause)),
799
+ );
800
+
801
+ const findThread = Effect.fnUntraced(function* (
149
802
  state: MemoryState,
150
803
  threadId: ThreadId,
151
- ): Effect.Effect<StoredThread, ThreadNotMaterialized> => {
804
+ ): Effect.fn.Return<StoredThread, ThreadNotMaterialized> {
152
805
  const thread = state.threads.get(threadId);
153
806
 
154
- return thread === undefined
807
+ return yield* thread === undefined
155
808
  ? Effect.fail(ThreadNotMaterialized.make({ threadId }))
156
809
  : Effect.succeed(thread);
157
810
  });
@@ -163,33 +816,307 @@ const CheckpointVersionEnvelope = Schema.Struct({
163
816
  }),
164
817
  });
165
818
 
166
- const validateCheckpointVersion = Effect.fn("MemoryThreadStore.validateCheckpointVersion")(
167
- function* (value: unknown): Effect.fn.Return<void, ThreadStoreError | CheckpointRejected> {
168
- const envelope = yield* Schema.decodeUnknownEffect(CheckpointVersionEnvelope)(value).pipe(
169
- Effect.mapError(() => storeError("saveCheckpoint", "Invalid saveCheckpoint request")),
170
- );
819
+ const validateCheckpointVersion = Effect.fnUntraced(function* (
820
+ value: unknown,
821
+ ): Effect.fn.Return<void, ThreadStoreError | CheckpointRejected> {
822
+ const envelope = yield* Schema.decodeUnknownEffect(CheckpointVersionEnvelope)(value).pipe(
823
+ Effect.mapError(() => storeError("saveCheckpoint", "Invalid saveCheckpoint request")),
824
+ );
171
825
 
172
- if (envelope.checkpoint.schemaVersion !== 1) {
173
- return yield* CheckpointRejected.make({
174
- threadId: envelope.checkpoint.threadId,
175
- reason: "unsupported-version",
176
- });
177
- }
178
- },
179
- );
826
+ if (envelope.checkpoint.schemaVersion !== 1) {
827
+ return yield* CheckpointRejected.make({
828
+ threadId: envelope.checkpoint.threadId,
829
+ reason: "unsupported-version",
830
+ });
831
+ }
832
+ });
180
833
 
181
834
  const makeThreadStore = Effect.gen(function* () {
182
835
  const maxThreads = yield* ThreadCapacity;
183
836
  const crypto = yield* Crypto.Crypto;
184
837
  const state = yield* Ref.make<MemoryState>({ threads: new Map() });
838
+ const gate = yield* Semaphore.make(1);
839
+ const owner = yield* makeStorageOwner(gate.withPermits(1));
840
+ const withMutation = owner.withMutation;
185
841
  const updates = yield* PubSub.sliding<void>(1);
842
+ let ledgerTransfer: MemoryLedgerTransfer | undefined;
843
+ let deliveryTransfer: MemoryDeliveryTransfer | undefined;
844
+
845
+ const workState = yield* Ref.make(new Map<ThreadId, WorkIndex>());
846
+ const workOwners = new Map<"admissions" | "deliveries", MemoryWorkOwner>();
847
+ let deliveryLookup: ((threadId: ThreadId, messageId: IdempotencyKey) => boolean) | undefined;
848
+
849
+ const retainedDelivery = (threadId: ThreadId, messageId: IdempotencyKey) =>
850
+ deliveryLookup?.(threadId, messageId) ?? false;
851
+
852
+ const publishWork = (threadId: ThreadId, index: WorkIndex) =>
853
+ MutableRef.get(workState.ref).set(threadId, index);
854
+
855
+ const retainDelivery = (threadId: ThreadId, messageId: IdempotencyKey) => {
856
+ const index = MutableRef.get(workState.ref).get(threadId);
857
+ const ids = index?.messages.get(messageId);
858
+
859
+ if (
860
+ index === undefined ||
861
+ index.version !== WORK_INDEX_VERSION ||
862
+ index.entryCount !== index.entries.size ||
863
+ ids === undefined
864
+ )
865
+ return;
866
+ const { entries, messages } = index;
867
+
868
+ for (const id of ids) {
869
+ const entry = entries.get(id);
870
+
871
+ if (entry !== undefined) changeWorkCounts(index, entry, -1);
872
+ entries.delete(id);
873
+ }
874
+ messages.delete(messageId);
875
+ publishWork(threadId, { ...index, entryCount: entries.size });
876
+ };
877
+
878
+ const readyIndex = (
879
+ threadId: ThreadId,
880
+ tail: number,
881
+ ): Effect.Effect<WorkIndex, WorkDiscoveryUnavailable> => {
882
+ const index = MutableRef.get(workState.ref).get(threadId);
883
+
884
+ if (
885
+ index === undefined ||
886
+ index.version !== WORK_INDEX_VERSION ||
887
+ index.entries.size !== index.entryCount
888
+ )
889
+ return Effect.fail(WorkDiscoveryUnavailable.make({ threadId, reason: "missing-index" }));
890
+ if (index.state !== "ready" || index.through !== tail)
891
+ return Effect.fail(WorkDiscoveryUnavailable.make({ threadId, reason: "incomplete-rebuild" }));
892
+
893
+ return Effect.succeed(index);
894
+ };
895
+
896
+ const work: ThreadWorkStorage = {
897
+ threads: (unvalidated) =>
898
+ withMutation(
899
+ Effect.gen(function* () {
900
+ const request = yield* validate(WorkThreadsRequest, "work threads", unvalidated);
901
+ const current = yield* Ref.get(state);
902
+ const catalogue = yield* Ref.get(workState);
903
+
904
+ const candidates = boundedWorkSelection<ThreadId>(
905
+ request.limit,
906
+ (id) => id,
907
+ request.afterThreadId,
908
+ );
909
+
910
+ for (const [threadId, thread] of current.threads) {
911
+ const index = catalogue.get(threadId);
912
+
913
+ if (
914
+ index === undefined ||
915
+ index.version !== WORK_INDEX_VERSION ||
916
+ index.state !== "ready" ||
917
+ index.through !== thread.tailSequence ||
918
+ index.entryCount !== index.entries.size ||
919
+ index.entryCount > 0
920
+ )
921
+ candidates.add(threadId);
922
+ }
923
+ for (const owner of workOwners.values()) {
924
+ const page = yield* owner.threads(request);
925
+
926
+ for (const threadId of page.threadIds) candidates.add(threadId);
927
+ }
928
+ const threadIds = candidates.values;
929
+ const afterThreadId = threadIds.length === request.limit ? threadIds.at(-1) : undefined;
930
+
931
+ // A full page may have more owner rows; the next bounded call proves exhaustion.
932
+ return {
933
+ threadIds,
934
+ ...(afterThreadId === undefined ? {} : { afterThreadId }),
935
+ };
936
+ }),
937
+ ),
938
+ page: (unvalidated) =>
939
+ withMutation(
940
+ Effect.gen(function* () {
941
+ const request = yield* validate(ThreadWorkRequest, "work page", unvalidated);
942
+
943
+ const cursor = yield* decodeWorkCursor(request).pipe(
944
+ Effect.provideService(Crypto.Crypto, crypto),
945
+ );
946
+
947
+ const thread = (yield* Ref.get(state)).threads.get(request.threadId);
948
+ // Accepted input can precede canonical Thread materialization entirely.
949
+ const index = yield* readyIndex(request.threadId, thread?.tailSequence ?? 0);
950
+
951
+ if (cursor.source === "canonical") {
952
+ const candidates = boundedWorkSelection<CanonicalWorkEntry>(
953
+ request.limit + 1,
954
+ (entry) => entry.id,
955
+ cursor.after,
956
+ );
957
+
958
+ for (const entry of index.entries.values()) candidates.add(entry);
959
+ const selected = candidates.values;
960
+
961
+ const entries = selected.slice(0, request.limit);
962
+ const after = selected.length > request.limit ? entries.at(-1)?.id : undefined;
963
+
964
+ return yield* validateWorkPage(
965
+ {
966
+ entries,
967
+ cursor: encodeWorkCursor({
968
+ version: WORK_INDEX_VERSION,
969
+ threadDigest: cursor.threadDigest,
970
+ source: after === undefined ? "deliveries" : "canonical",
971
+ ...(after === undefined ? {} : { after }),
972
+ }),
973
+ },
974
+ request.limit,
975
+ );
976
+ }
977
+
978
+ const page = yield* (
979
+ workOwners.get(cursor.source)?.page(request.threadId, cursor.after, request.limit) ??
980
+ Effect.succeed<Effect.Success<ReturnType<MemoryWorkOwner["page"]>>>({ entries: [] })
981
+ );
982
+
983
+ const next =
984
+ page.after === undefined
985
+ ? cursor.source === "admissions"
986
+ ? "canonical"
987
+ : undefined
988
+ : cursor.source;
989
+
990
+ return yield* validateWorkPage(
991
+ {
992
+ entries: [...page.entries],
993
+ ...(next === undefined
994
+ ? {}
995
+ : {
996
+ cursor: encodeWorkCursor({
997
+ version: WORK_INDEX_VERSION,
998
+ threadDigest: cursor.threadDigest,
999
+ source: next,
1000
+ ...(page.after === undefined ? {} : { after: page.after }),
1001
+ }),
1002
+ }),
1003
+ },
1004
+ request.limit,
1005
+ );
1006
+ }),
1007
+ ),
1008
+ rebuild: (unvalidated) =>
1009
+ withMutation(
1010
+ Effect.gen(function* () {
1011
+ const request = yield* validate(WorkIndexRebuildRequest, "rebuild work", unvalidated);
1012
+ const current = yield* Ref.get(state);
1013
+ const thread = current.threads.get(request.threadId);
1014
+ const tail = thread?.tailSequence ?? ZERO_CANONICAL_SEQUENCE;
1015
+ const prior = (yield* Ref.get(workState)).get(request.threadId);
1016
+
1017
+ const index: WorkIndex =
1018
+ request.restart ||
1019
+ prior === undefined ||
1020
+ prior.version !== WORK_INDEX_VERSION ||
1021
+ prior.entries.size !== prior.entryCount ||
1022
+ (prior.state === "ready" && prior.through !== tail)
1023
+ ? { ...emptyWorkIndex(), state: "rebuilding", rebuildingIndexes: emptyIndexes() }
1024
+ : prior;
1025
+
1026
+ if (index.state === "ready")
1027
+ return {
1028
+ version: WORK_INDEX_VERSION,
1029
+ threadId: request.threadId,
1030
+ state: "ready",
1031
+ throughSequence: index.through,
1032
+ tailSequence: tail,
1033
+ processedRecords: 0,
1034
+ processedBytes: 0,
1035
+ };
1036
+ if (index.rebuildingIndexes === undefined)
1037
+ return yield* WorkDiscoveryUnavailable.make({
1038
+ threadId: request.threadId,
1039
+ reason: "incomplete-rebuild",
1040
+ });
1041
+ const rebuildingIndexes = index.rebuildingIndexes;
1042
+ const records: Array<CanonicalRecordEnvelope> = [];
1043
+ let bytes = 0;
1044
+ const end = Math.min(tail, index.through + (request.limit ?? MAX_WORK_REBUILD_RECORDS));
1045
+
1046
+ for (let position = index.through; position < end; position++) {
1047
+ const size = thread === undefined ? undefined : storedBytes(thread, position + 1);
1048
+
1049
+ if (size === undefined)
1050
+ return yield* storeError("rebuild work", "Canonical record byte length is missing");
1051
+ if (bytes + size > MAX_WORK_REBUILD_BYTES) break;
1052
+ const record = thread === undefined ? undefined : storedRecord(thread, position + 1);
1053
+
1054
+ if (record === undefined || record.sequence !== position + 1)
1055
+ return yield* storeError("rebuild work", "Canonical rebuild sequence is incomplete");
1056
+ bytes += size;
1057
+ records.push(record);
1058
+ }
1059
+ if (index.through < tail && records.length === 0)
1060
+ return yield* storeError(
1061
+ "rebuild work",
1062
+ "Canonical rebuild cannot advance within its byte bound",
1063
+ );
1064
+
1065
+ const commitWork = yield* prepareWork(index, records, retainedDelivery).pipe(
1066
+ Effect.provideService(Crypto.Crypto, crypto),
1067
+ );
1068
+
1069
+ // Shared writer gate fences the current tail; publish native locators and catalogue together.
1070
+ const next = yield* Effect.sync(() => {
1071
+ const native = prepareIndexes(rebuildingIndexes, records);
1072
+ const rebuilt = commitWork();
1073
+ const complete = rebuilt.through === tail;
1074
+
1075
+ const next: WorkIndex = {
1076
+ version: rebuilt.version,
1077
+ through: rebuilt.through,
1078
+ reporting: rebuilt.reporting,
1079
+ entryCount: rebuilt.entryCount,
1080
+ entries: rebuilt.entries,
1081
+ messages: rebuilt.messages,
1082
+ foreign: rebuilt.foreign,
1083
+ workers: rebuilt.workers,
1084
+ liveInputs: rebuilt.liveInputs,
1085
+ inputsByWorker: rebuilt.inputsByWorker,
1086
+ state: complete ? "ready" : "rebuilding",
1087
+ ...(complete ? {} : { rebuildingIndexes: native.indexes }),
1088
+ };
1089
+
1090
+ native.commit();
1091
+ if (thread !== undefined && complete)
1092
+ MutableRef.set(state.ref, {
1093
+ threads: new Map(current.threads).set(request.threadId, {
1094
+ ...thread,
1095
+ ...native.indexes,
1096
+ }),
1097
+ });
1098
+ publishWork(request.threadId, next);
1099
+
1100
+ return next;
1101
+ });
1102
+
1103
+ return {
1104
+ version: WORK_INDEX_VERSION,
1105
+ threadId: request.threadId,
1106
+ state: next.state,
1107
+ throughSequence: next.through,
1108
+ tailSequence: tail,
1109
+ processedRecords: records.length,
1110
+ processedBytes: bytes,
1111
+ };
1112
+ }),
1113
+ ),
1114
+ };
186
1115
 
187
1116
  yield* Effect.addFinalizer(() => PubSub.shutdown(updates));
188
1117
 
189
- const materialize: ThreadStore["Service"]["materialize"] = Effect.fn(
190
- "MemoryThreadStore.materialize",
191
- )((unvalidated) =>
192
- Effect.gen(function* () {
1118
+ const materialize: ThreadStore["Service"]["materialize"] = Effect.fnUntraced(
1119
+ function* (unvalidated) {
193
1120
  const request = yield* validate(ThreadMaterialization, "materialize", unvalidated);
194
1121
 
195
1122
  const decision = yield* Ref.modify(
@@ -197,33 +1124,11 @@ const makeThreadStore = Effect.gen(function* () {
197
1124
  (current): readonly [MaterializeDecision, MemoryState] => {
198
1125
  const existing = current.threads.get(request.threadId);
199
1126
 
200
- if (existing !== undefined) {
201
- if (request.producerEpoch < existing.producerEpoch) {
202
- return [
203
- {
204
- _tag: "failure",
205
- error: FenceRejected.make({
206
- threadId: request.threadId,
207
- actualEpoch: existing.producerEpoch,
208
- attemptedEpoch: request.producerEpoch,
209
- }),
210
- },
211
- current,
212
- ];
213
- }
214
- if (request.producerEpoch === existing.producerEpoch) {
215
- return [{ _tag: "success" }, current];
216
- }
217
- const threads = new Map(current.threads);
218
-
219
- threads.set(request.threadId, {
220
- ...existing,
221
- producerEpoch: request.producerEpoch,
222
- });
223
-
224
- return [{ _tag: "success" }, { threads }];
225
- }
226
- if (current.threads.size >= maxThreads) {
1127
+ if (existing !== undefined) return [{ _tag: "success" }, current];
1128
+ if (
1129
+ !MutableRef.get(workState.ref).has(request.threadId) &&
1130
+ MutableRef.get(workState.ref).size >= maxThreads
1131
+ ) {
227
1132
  return [
228
1133
  {
229
1134
  _tag: "failure",
@@ -232,287 +1137,383 @@ const makeThreadStore = Effect.gen(function* () {
232
1137
  current,
233
1138
  ];
234
1139
  }
1140
+ if (!transferPageFits(request.threadId))
1141
+ return [
1142
+ {
1143
+ _tag: "failure",
1144
+ error: storeError(
1145
+ "materialize",
1146
+ "Thread identity exceeds its bounded transfer representation",
1147
+ ),
1148
+ },
1149
+ current,
1150
+ ];
235
1151
  const threads = new Map(current.threads);
236
1152
 
237
1153
  threads.set(request.threadId, {
238
- producerEpoch: request.producerEpoch,
239
1154
  tailSequence: ZERO_CANONICAL_SEQUENCE,
240
- tailDigest: EMPTY_TAIL_DIGEST,
241
- unverifiedWorkerInputs: new Set(),
242
1155
  byId: new Map(),
243
1156
  workerRecords: new Map(),
244
1157
  peerCount: 0,
245
1158
  runInputs: new Map(),
246
- outstanding: new Map(),
247
- records: [],
248
- recordIds: new Set(),
1159
+ runRecords: new Map(),
1160
+ operationRecords: new Map(),
1161
+ handoffs: [],
1162
+ agentUpdates: [],
1163
+ deliveryPredecessors: [],
1164
+ toolDeclarations: new Map(),
1165
+ promptEvidence: [],
1166
+ modelCompletions: [],
1167
+ applicationInputs: [],
1168
+ durableOwners: [],
1169
+ windowBoundaries: new Map(),
1170
+ ranges: new Map(),
1171
+ rangeStarts: [],
1172
+ archiveContents: new Map(),
1173
+ chunks: new Map(),
1174
+ batchesByFirst: new Map(),
249
1175
  batches: new Map(),
250
- tailDigests: new Map([[ZERO_CANONICAL_SEQUENCE, EMPTY_TAIL_DIGEST]]),
1176
+ executions: new Map(),
1177
+ executionCommits: new Map(),
251
1178
  checkpoints: new Map(),
252
1179
  });
253
1180
 
1181
+ publishWork(request.threadId, emptyWorkIndex());
1182
+
254
1183
  return [{ _tag: "success" }, { threads }];
255
1184
  },
256
1185
  );
257
1186
 
258
1187
  if (decision._tag === "failure") return yield* decision.error;
259
- }),
1188
+ },
260
1189
  );
261
1190
 
262
- const append: ThreadStore["Service"]["append"] = Effect.fn("MemoryThreadStore.append")(
263
- (unvalidated) =>
264
- Effect.gen(function* () {
265
- const request = yield* validate(FencedAppendRequest, "append", unvalidated);
266
-
267
- const digest = yield* digestCanonicalBatch(request.expectedTailDigest, request.batch).pipe(
268
- Effect.provideService(Crypto.Crypto, crypto),
269
- Effect.mapError((error) => storeError("append", error.message, error)),
270
- );
271
-
272
- const decision = yield* Effect.uninterruptible(
273
- Ref.modify(state, (current): readonly [AppendDecision, MemoryState] => {
274
- const thread = current.threads.get(request.threadId);
275
-
276
- if (thread === undefined) {
277
- return [
278
- {
279
- _tag: "failure",
280
- error: ThreadNotMaterialized.make({
281
- threadId: request.threadId,
282
- }),
283
- },
284
- current,
285
- ];
286
- }
287
- if (request.producerEpoch !== thread.producerEpoch) {
288
- return [
289
- {
290
- _tag: "failure",
291
- error: FenceRejected.make({
292
- threadId: request.threadId,
293
- actualEpoch: thread.producerEpoch,
294
- attemptedEpoch: request.producerEpoch,
295
- }),
296
- },
297
- current,
298
- ];
299
- }
300
- const previous = thread.batches.get(request.batch.batchId);
1191
+ const prepareAppend = Effect.fnUntraced(function* (unvalidated: AppendRequest) {
1192
+ const captured = yield* PreparedAppend.capture(unvalidated);
1193
+ const request = captured;
1194
+ const batchJson = captured.batchJson;
301
1195
 
302
- if (previous !== undefined) {
303
- if (previous.digest !== digest) {
304
- return [
305
- {
306
- _tag: "failure",
307
- error: AppendConflict.make({
308
- threadId: request.threadId,
309
- batchId: request.batch.batchId,
310
- reason: "batch-digest",
311
- }),
312
- },
313
- current,
314
- ];
315
- }
1196
+ const batchBytes = new TextEncoder().encode(batchJson).byteLength;
316
1197
 
317
- return [
318
- {
319
- _tag: "success",
320
- result: AppendResult.make({
321
- firstSequence: previous.result.firstSequence,
322
- lastSequence: previous.result.lastSequence,
323
- tailDigest: previous.result.tailDigest,
324
- replayed: true,
325
- }),
326
- records: [],
327
- },
328
- current,
329
- ];
330
- }
331
- if (
332
- request.expectedTailSequence !== thread.tailSequence ||
333
- request.expectedTailDigest !== thread.tailDigest
334
- ) {
335
- return [
336
- {
337
- _tag: "failure",
338
- error: AppendConflict.make({
339
- threadId: request.threadId,
340
- batchId: request.batch.batchId,
341
- reason: "tail",
342
- actualTailSequence: thread.tailSequence,
343
- actualTailDigest: thread.tailDigest,
344
- }),
345
- },
346
- current,
347
- ];
348
- }
349
- if (thread.records.length + request.batch.records.length > MAX_RECORDS_PER_THREAD) {
350
- return [
351
- {
352
- _tag: "failure",
353
- error: storeError(
354
- "append",
355
- `In-memory record limit ${MAX_RECORDS_PER_THREAD} exceeded`,
356
- ),
357
- },
358
- current,
359
- ];
360
- }
1198
+ if (batchBytes > 16 * 1024 * 1024)
1199
+ return yield* storeError("append", "Canonical batch exceeds 16 MiB");
361
1200
 
362
- const batchRecordIds = new Set<RecordId>();
1201
+ const offsetBytes = 4 * Math.ceil(utf8ByteLength(request.threadId) / 3) + 36;
363
1202
 
364
- for (const record of request.batch.records) {
365
- if (thread.recordIds.has(record.recordId) || batchRecordIds.has(record.recordId)) {
366
- return [
367
- {
368
- _tag: "failure",
369
- error: AppendConflict.make({
370
- threadId: request.threadId,
371
- batchId: request.batch.batchId,
372
- reason: "record-identity",
373
- }),
374
- },
375
- current,
376
- ];
377
- }
378
- batchRecordIds.add(record.recordId);
379
- }
1203
+ if (!canonicalBatchFitsTransfer(request.threadId, request.batch, batchBytes, offsetBytes))
1204
+ return yield* storeError(
1205
+ "append",
1206
+ "Canonical batch exceeds its bounded transfer representation",
1207
+ );
380
1208
 
381
- const records = request.batch.records.map((record, index) => {
382
- const sequence = decodeCanonicalSequence(thread.tailSequence + index + 1);
1209
+ return { request, batchJson };
1210
+ });
383
1211
 
384
- return CanonicalRecordEnvelope.make({
385
- threadId: request.threadId,
386
- batchId: request.batch.batchId,
387
- sequence,
388
- offset: observationOffset(request.threadId, sequence),
389
- record,
390
- });
391
- });
1212
+ const appendPrepared = Effect.fnUntraced(function* ({
1213
+ request,
1214
+ batchJson,
1215
+ }: PreparedMemoryAppend) {
1216
+ // Every caller holds the shared kernel mutation gate across validation and Ref publication.
1217
+ const current = yield* Ref.get(state);
1218
+ const thread = current.threads.get(request.threadId);
1219
+
1220
+ const executionIntent = JSON.stringify({
1221
+ batch: batchJson,
1222
+ operational: (request.executions ?? []).flatMap((update) =>
1223
+ update.facts.map((fact) => Schema.encodeSync(RecordEnvelope)(fact)),
1224
+ ),
1225
+ });
392
1226
 
393
- const lastSequence = decodeCanonicalSequence(thread.tailSequence + records.length);
1227
+ const latestIntent =
1228
+ thread === undefined
1229
+ ? undefined
1230
+ : [...thread.executionCommits.values()].find(
1231
+ (entry) => entry.batchId === request.batch.batchId,
1232
+ );
394
1233
 
395
- const result = AppendResult.make({
396
- firstSequence: decodeCanonicalSequence(thread.tailSequence + 1),
397
- lastSequence,
398
- tailDigest: digest,
399
- replayed: false,
400
- });
1234
+ if (thread !== undefined && latestIntent === undefined && request.batch.records.length === 0) {
1235
+ for (const update of request.executions ?? []) {
1236
+ const prior = thread.executions.get(update.checkpoint.progress.runId);
401
1237
 
402
- const batches = new Map(thread.batches);
1238
+ if (
1239
+ prior !== undefined &&
1240
+ update.previousRevision < prior.progress.revision &&
1241
+ update.facts.every((fact) =>
1242
+ prior.operational.some((known) => known.recordId === fact.recordId),
1243
+ )
1244
+ )
1245
+ return yield* AppendConflict.make({
1246
+ threadId: request.threadId,
1247
+ batchId: request.batch.batchId,
1248
+ reason: "stale-command",
1249
+ });
1250
+ }
1251
+ }
1252
+ if (
1253
+ thread !== undefined &&
1254
+ thread.tailSequence === request.expectedTailSequence &&
1255
+ !thread.batches.has(request.batch.batchId) &&
1256
+ latestIntent === undefined
1257
+ ) {
1258
+ const executions = request.executions ?? [];
1259
+
1260
+ if (
1261
+ new Set(executions.map((update) => update.checkpoint.progress.runId)).size !==
1262
+ executions.length
1263
+ )
1264
+ return yield* storeError("append", "Duplicate active execution identity");
1265
+ for (const update of request.executions ?? []) {
1266
+ const prior = thread.executions.get(update.checkpoint.progress.runId);
403
1267
 
404
- batches.set(request.batch.batchId, { digest, result });
405
- const recordIds = new Set(thread.recordIds);
1268
+ if ((prior?.progress.revision ?? 0) !== update.previousRevision)
1269
+ return yield* AppendConflict.make({
1270
+ threadId: request.threadId,
1271
+ batchId: request.batch.batchId,
1272
+ reason: "tail",
1273
+ actualTailSequence: thread.tailSequence,
1274
+ });
1275
+ if (
1276
+ update.checkpoint.throughSequence !==
1277
+ thread.tailSequence + request.batch.records.length ||
1278
+ update.checkpoint.batchId !== request.batch.batchId ||
1279
+ (prior !== undefined && update.checkpoint.originalSequence !== prior.originalSequence)
1280
+ )
1281
+ return yield* storeError(
1282
+ "append",
1283
+ "Active checkpoint differs from its committed history boundary",
1284
+ );
1285
+ if (
1286
+ update.retire &&
1287
+ ![
1288
+ ...request.batch.records,
1289
+ ...(thread.runRecords.get(update.checkpoint.progress.runId) ?? []).map(
1290
+ (entry) => entry.record,
1291
+ ),
1292
+ ].some(
1293
+ ({ payload }) =>
1294
+ payload._tag === "SubmissionSettled" &&
1295
+ payload.runId === update.checkpoint.progress.runId &&
1296
+ payload.submissionId === update.checkpoint.progress.submissionId,
1297
+ )
1298
+ )
1299
+ return yield* storeError(
1300
+ "append",
1301
+ "Active state cannot retire without a canonical terminal outcome",
1302
+ );
1303
+ }
1304
+ yield* validateProgressAppend(
1305
+ prepareProgressAppend(request.batch.records),
1306
+ request.executions,
1307
+ ).pipe(
1308
+ Effect.provideService(ProgressAppendReader, {
1309
+ previous: (runId) => Effect.succeed(thread.executions.get(runId)?.progress),
1310
+ initial: (next) =>
1311
+ Effect.succeed((thread.runRecords.get(next.runId) ?? []).map((entry) => entry.record)),
1312
+ }),
1313
+ );
1314
+ }
406
1315
 
407
- for (const recordId of batchRecordIds) recordIds.add(recordId);
408
- const tailDigests = new Map(thread.tailDigests);
1316
+ const currentWork = (yield* Ref.get(workState)).get(request.threadId);
1317
+
1318
+ const preparedWork =
1319
+ thread !== undefined &&
1320
+ thread.tailSequence === request.expectedTailSequence &&
1321
+ !thread.batches.has(request.batch.batchId) &&
1322
+ latestIntent === undefined &&
1323
+ currentWork?.version === WORK_INDEX_VERSION &&
1324
+ currentWork.state === "ready" &&
1325
+ currentWork.through === thread.tailSequence &&
1326
+ currentWork.entryCount === currentWork.entries.size
1327
+ ? yield* prepareWork(
1328
+ currentWork,
1329
+ batchEnvelopes(
1330
+ request.threadId,
1331
+ request.batch,
1332
+ decodeCanonicalSequence(thread.tailSequence + 1),
1333
+ ),
1334
+ retainedDelivery,
1335
+ ).pipe(Effect.provideService(Crypto.Crypto, crypto))
1336
+ : undefined;
409
1337
 
410
- tailDigests.set(lastSequence, digest);
411
- const threads = new Map(current.threads);
1338
+ const decision = yield* Effect.uninterruptible(
1339
+ Ref.modify(state, (current): readonly [AppendDecision, MemoryState] => {
1340
+ const thread = current.threads.get(request.threadId);
412
1341
 
413
- const unverifiedWorkerInputs = new Set(thread.unverifiedWorkerInputs);
414
- let peerCount = thread.peerCount;
415
- const workerRecords = new Map(thread.workerRecords);
416
- const byId = new Map(thread.byId);
417
- const runInputs = new Map(thread.runInputs);
418
- const outstanding = new Map(thread.outstanding);
1342
+ if (thread === undefined) {
1343
+ return [
1344
+ {
1345
+ _tag: "failure",
1346
+ error: ThreadNotMaterialized.make({
1347
+ threadId: request.threadId,
1348
+ }),
1349
+ },
1350
+ current,
1351
+ ];
1352
+ }
1353
+ const priorExecution = latestIntent;
419
1354
 
420
- for (const entry of records) {
421
- byId.set(entry.record.recordId, entry);
422
- const payload = entry.record.payload;
1355
+ if (priorExecution !== undefined) {
1356
+ if (priorExecution.intent !== executionIntent)
1357
+ return [
1358
+ {
1359
+ _tag: "failure",
1360
+ error: AppendConflict.make({
1361
+ threadId: request.threadId,
1362
+ batchId: request.batch.batchId,
1363
+ reason: "batch-content",
1364
+ }),
1365
+ },
1366
+ current,
1367
+ ];
423
1368
 
424
- if (payload._tag === "PeerMessagePrepared") peerCount++;
425
- if (
426
- (payload._tag === "UserInputRecorded" || payload._tag === "RunStarted") &&
427
- payload.runId !== undefined
428
- )
429
- workerRecords.set(`execution:${payload._tag}`, [entry]);
430
-
431
- const workerKey =
432
- payload._tag === "SubtreeBudgetReserved"
433
- ? `subtree:${payload.sourceSubmissionId ?? ""}`
434
- : payload._tag === "SubagentJoined"
435
- ? `joined:${payload.runId}`
436
- : [
437
- "ThreadCreated",
438
- "WorkerOriginRecorded",
439
- "SubagentLineageRecorded",
440
- "WorkerInputRequested",
441
- "WorkerInputCompleted",
442
- "WorkerStopRequested",
443
- ].includes(payload._tag)
444
- ? "worker"
445
- : undefined;
446
-
447
- if (workerKey !== undefined)
448
- workerRecords.set(workerKey, [...(workerRecords.get(workerKey) ?? []), entry]);
1369
+ return [
1370
+ {
1371
+ _tag: "success",
1372
+ result: AppendResult.make({ ...priorExecution.result, replayed: true }),
1373
+ records: [],
1374
+ },
1375
+ current,
1376
+ ];
1377
+ }
1378
+ const previous = thread.batches.get(request.batch.batchId);
449
1379
 
450
- if (
451
- payload._tag === "UserInputRecorded" &&
452
- payload.kind === "user" &&
453
- payload.runId !== undefined
454
- )
455
- runInputs.set(payload.runId, runInputs.has(payload.runId) ? null : entry);
456
- if (
457
- payload._tag === "ToolCallPrepared" ||
458
- payload._tag === "ToolCallUnknown" ||
459
- payload._tag === "ToolCallSettled"
460
- ) {
461
- const key = JSON.stringify([payload.runId, payload.toolCallId]);
462
-
463
- if (payload._tag === "ToolCallSettled") outstanding.delete(key);
464
- else
465
- outstanding.set(key, [
466
- ...(outstanding.get(key) ?? []).filter(
467
- (prior) =>
468
- payload._tag !== "ToolCallUnknown" ||
469
- prior.record.payload._tag !== "ToolCallPrepared",
470
- ),
471
- entry,
472
- ]);
473
- }
474
- if (payload._tag === "WorkerInputRequested")
475
- outstanding.set(`worker:${payload.admission.messageId}`, [
476
- ...(outstanding.get(`worker:${payload.admission.messageId}`) ?? []),
477
- entry,
478
- ]);
479
- if (payload._tag === "WorkerInputCompleted") {
480
- if (payload.effectsResolved) {
481
- outstanding.delete(`worker:${payload.messageId}`);
482
- unverifiedWorkerInputs.delete(payload.messageId);
483
- } else unverifiedWorkerInputs.add(payload.messageId);
484
- }
485
- }
486
- threads.set(request.threadId, {
487
- ...thread,
488
- byId,
489
- workerRecords,
490
- peerCount,
491
- runInputs,
492
- outstanding,
493
- unverifiedWorkerInputs,
494
- tailSequence: lastSequence,
495
- tailDigest: digest,
496
- records: [...thread.records, ...records],
497
- recordIds,
498
- batches,
499
- tailDigests,
500
- });
1380
+ if (previous !== undefined) {
1381
+ if (previous.batchJson !== batchJson) {
1382
+ return [
1383
+ {
1384
+ _tag: "failure",
1385
+ error: AppendConflict.make({
1386
+ threadId: request.threadId,
1387
+ batchId: request.batch.batchId,
1388
+ reason: "batch-content",
1389
+ }),
1390
+ },
1391
+ current,
1392
+ ];
1393
+ }
501
1394
 
502
- return [{ _tag: "success", result, records }, { threads }];
503
- }).pipe(
504
- Effect.tap((decision) =>
505
- decision._tag === "success" && decision.records.length > 0
506
- ? PubSub.publish(updates, undefined)
507
- : Effect.void,
508
- ),
509
- ),
1395
+ return [
1396
+ {
1397
+ _tag: "success",
1398
+ result: AppendResult.make({
1399
+ firstSequence: previous.result.firstSequence,
1400
+ lastSequence: previous.result.lastSequence,
1401
+ replayed: true,
1402
+ }),
1403
+ records: [],
1404
+ },
1405
+ current,
1406
+ ];
1407
+ }
1408
+ if (request.expectedTailSequence !== thread.tailSequence) {
1409
+ return [
1410
+ {
1411
+ _tag: "failure",
1412
+ error: AppendConflict.make({
1413
+ threadId: request.threadId,
1414
+ batchId: request.batch.batchId,
1415
+ reason: "tail",
1416
+ actualTailSequence: thread.tailSequence,
1417
+ }),
1418
+ },
1419
+ current,
1420
+ ];
1421
+ }
1422
+ const batchRecordIds = new Set<RecordId>();
1423
+
1424
+ for (const record of request.batch.records) {
1425
+ if (thread.byId.has(record.recordId) || batchRecordIds.has(record.recordId)) {
1426
+ return [
1427
+ {
1428
+ _tag: "failure",
1429
+ error: AppendConflict.make({
1430
+ threadId: request.threadId,
1431
+ batchId: request.batch.batchId,
1432
+ reason: "record-identity",
1433
+ }),
1434
+ },
1435
+ current,
1436
+ ];
1437
+ }
1438
+ batchRecordIds.add(record.recordId);
1439
+ }
1440
+
1441
+ const records = batchEnvelopes(
1442
+ request.threadId,
1443
+ request.batch,
1444
+ decodeCanonicalSequence(thread.tailSequence + 1),
510
1445
  );
511
1446
 
512
- if (decision._tag === "failure") return yield* decision.error;
1447
+ const lastSequence = decodeCanonicalSequence(thread.tailSequence + records.length);
513
1448
 
514
- return decision.result;
515
- }),
1449
+ const result = AppendResult.make({
1450
+ firstSequence: decodeCanonicalSequence(
1451
+ thread.tailSequence + (records.length > 0 ? 1 : 0),
1452
+ ),
1453
+ lastSequence,
1454
+ replayed: false,
1455
+ });
1456
+
1457
+ const indexes = prepareIndexes(thread, records);
1458
+
1459
+ const next: StoredThread = {
1460
+ ...thread,
1461
+ ...indexes.indexes,
1462
+ tailSequence: lastSequence,
1463
+ };
1464
+
1465
+ // Every rejection and asynchronous validation precedes this synchronous publication.
1466
+ if (records.length > 0) {
1467
+ thread.batches.set(request.batch.batchId, { result, batchJson });
1468
+ thread.batchesByFirst.set(result.firstSequence, request.batch.batchId);
1469
+ retainRecords(thread, records);
1470
+ }
1471
+ for (const update of request.executions ?? []) {
1472
+ const runId = update.checkpoint.progress.runId;
1473
+
1474
+ if (update.retire) {
1475
+ thread.executions.delete(runId);
1476
+ thread.executionCommits.delete(runId);
1477
+ } else {
1478
+ thread.executions.set(runId, update.checkpoint);
1479
+ thread.executionCommits.set(runId, {
1480
+ batchId: request.batch.batchId,
1481
+ intent: executionIntent,
1482
+ result,
1483
+ });
1484
+ }
1485
+ }
1486
+ indexes.commit();
1487
+ if (preparedWork !== undefined) publishWork(request.threadId, preparedWork());
1488
+ current.threads.set(request.threadId, next);
1489
+
1490
+ const publication: readonly [AppendDecision, MemoryState] = [
1491
+ { _tag: "success", result, records },
1492
+ current,
1493
+ ];
1494
+
1495
+ return publication;
1496
+ }).pipe(
1497
+ Effect.tap((decision) =>
1498
+ decision._tag === "success" && decision.records.length > 0
1499
+ ? PubSub.publish(updates, undefined)
1500
+ : Effect.void,
1501
+ ),
1502
+ ),
1503
+ );
1504
+
1505
+ if (decision._tag === "failure") return yield* decision.error;
1506
+
1507
+ return decision.result;
1508
+ });
1509
+
1510
+ const append: ThreadStore["Service"]["append"] = Effect.fn("MemoryThreadStore.append")(
1511
+ (request) =>
1512
+ prepareAppend(request).pipe(
1513
+ Effect.flatMap((prepared) =>
1514
+ owner.withProgress(prepared.request.threadId, withMutation(appendPrepared(prepared))),
1515
+ ),
1516
+ ),
516
1517
  );
517
1518
 
518
1519
  const countPeerMessages: NonNullable<ThreadStore["Service"]["countPeerMessages"]> =
@@ -520,86 +1521,349 @@ const makeThreadStore = Effect.gen(function* () {
520
1521
  yield* validate(ThreadPeerCountRequest, "countPeerMessages", request);
521
1522
  const thread = yield* findThread(yield* Ref.get(state), request.threadId);
522
1523
 
523
- return Math.min(thread.peerCount, request.limit);
1524
+ const index = yield* readyIndex(request.threadId, thread.tailSequence).pipe(
1525
+ Effect.mapError((cause) => storeError("countPeerMessages", cause.message)),
1526
+ );
1527
+
1528
+ return Math.min(
1529
+ index.foreign.peer +
1530
+ (deliveryTransfer?.pendingPeerCount(request.threadId, request.limit) ?? 0),
1531
+ request.limit,
1532
+ );
524
1533
  });
525
1534
 
526
- const readSnapshot = Effect.fn("MemoryThreadStore.readSnapshot")(
527
- (threadId: ThreadId, afterSequence: CanonicalSequence | undefined, limit: number) =>
528
- Ref.get(state).pipe(
529
- Effect.flatMap((current) => findThread(current, threadId)),
530
- Effect.map((thread) => {
531
- // Append assigns gap-free sequences starting at 1, so the exclusive cursor is an index.
532
- const start = afterSequence ?? ZERO_CANONICAL_SEQUENCE;
1535
+ const readSnapshot = (
1536
+ threadId: ThreadId,
1537
+ afterSequence: CanonicalSequence | undefined,
1538
+ limit: number,
1539
+ ) =>
1540
+ Ref.get(state).pipe(
1541
+ Effect.flatMap((current) => findThread(current, threadId)),
1542
+ Effect.map((thread) => {
1543
+ // Append assigns gap-free sequences starting at 1, so the exclusive cursor is an index.
1544
+ const start = afterSequence ?? ZERO_CANONICAL_SEQUENCE;
1545
+
1546
+ return storedPage(thread, start, limit);
1547
+ }),
1548
+ );
533
1549
 
534
- return thread.records.slice(start, start + limit);
535
- }),
536
- ),
537
- );
1550
+ const selectedStream = (
1551
+ sources: ReadonlyArray<ReadonlyArray<CanonicalRecordEnvelope>>,
1552
+ after: number,
1553
+ through: number,
1554
+ maximum: number,
1555
+ ) =>
1556
+ Stream.paginate({ after, count: 0 }, (cursor) =>
1557
+ Effect.gen(function* () {
1558
+ const page: Array<CanonicalRecordEnvelope> = [];
1559
+ let bytes = 0;
1560
+ let last = cursor.after;
1561
+
1562
+ while (page.length < Math.min(READ_PAGE_SIZE, maximum - cursor.count)) {
1563
+ let selected: CanonicalRecordEnvelope | undefined;
1564
+
1565
+ for (const source of sources) {
1566
+ const next = source[upperSequence(source, last)];
1567
+
1568
+ if (
1569
+ next !== undefined &&
1570
+ next.sequence <= through &&
1571
+ (selected === undefined || next.sequence < selected.sequence)
1572
+ )
1573
+ selected = next;
1574
+ }
1575
+ if (selected === undefined) break;
1576
+ const size = canonicalRecordBytes(selected.record);
1577
+
1578
+ if (size > 32 * 1024 * 1024)
1579
+ return yield* storeError("read", "Canonical value exceeds the page byte bound");
1580
+ if (bytes + size > 32 * 1024 * 1024) break;
1581
+ bytes += size;
1582
+ page.push(selected);
1583
+ last = selected.sequence;
1584
+ }
1585
+ const next = { after: last, count: cursor.count + page.length };
1586
+
1587
+ return [
1588
+ page,
1589
+ page.length === 0 || next.count >= maximum
1590
+ ? Option.none<typeof next>()
1591
+ : Option.some(next),
1592
+ ] as const;
1593
+ }),
1594
+ );
538
1595
 
539
- const read: ThreadStore["Service"]["read"] = (unvalidated) =>
1596
+ const readFromThread = (threadId: ThreadId, thread: StoredThread, request: ThreadReadRequest) =>
540
1597
  Stream.unwrap(
541
1598
  Effect.gen(function* () {
542
- const request = yield* validate(ThreadReadRequest, "read", unvalidated);
1599
+ if (request.threadId !== threadId)
1600
+ return yield* storeError("read snapshot", "Thread differs from the captured snapshot");
1601
+
1602
+ if (!("selection" in request))
1603
+ return Stream.paginate(
1604
+ { after: request.afterSequence ?? ZERO_CANONICAL_SEQUENCE, count: 0 },
1605
+ (cursor) =>
1606
+ Effect.gen(function* () {
1607
+ const page = storedPage(
1608
+ thread,
1609
+ cursor.after,
1610
+ Math.min(READ_PAGE_SIZE, request.limit - cursor.count),
1611
+ );
1612
+
1613
+ const next = {
1614
+ after: page.at(-1)?.sequence ?? cursor.after,
1615
+ count: cursor.count + page.length,
1616
+ };
543
1617
 
544
- if ("selection" in request) {
545
- const thread = yield* findThread(yield* Ref.get(state), request.threadId);
546
- const selection = request.selection;
1618
+ return [
1619
+ page,
1620
+ page.length === 0 || next.count >= request.limit
1621
+ ? Option.none<typeof next>()
1622
+ : Option.some(next),
1623
+ ] as const;
1624
+ }),
1625
+ );
1626
+ const selection = request.selection;
547
1627
 
548
- if (
549
- "expectedTailSequence" in selection &&
550
- (selection.expectedTailSequence !== thread.tailSequence ||
551
- selection.expectedTailDigest !== thread.tailDigest)
552
- )
553
- return yield* storeError("selected read", "Canonical tail changed");
554
- let records: ReadonlyArray<CanonicalRecordEnvelope>;
1628
+ if (
1629
+ "expectedTailSequence" in selection &&
1630
+ selection.expectedTailSequence !== thread.tailSequence
1631
+ )
1632
+ return yield* storeError("selected read", "Canonical tail changed");
1633
+ if ("throughSequence" in selection && selection.throughSequence > thread.tailSequence)
1634
+ return yield* storeError("selected read", "Captured sequence is ahead of canonical tail");
1635
+
1636
+ const through =
1637
+ "throughSequence" in selection ? selection.throughSequence : thread.tailSequence;
1638
+
1639
+ const after = request.page.afterSequence ?? 0;
1640
+ let sources: ReadonlyArray<ReadonlyArray<CanonicalRecordEnvelope>> = [];
1641
+
1642
+ const one = (entry: CanonicalRecordEnvelope | null | undefined) =>
1643
+ entry === undefined || entry === null ? [] : [entry];
1644
+
1645
+ const latest = (entries: ReadonlyArray<CanonicalRecordEnvelope>) =>
1646
+ one(entries[upperSequence(entries, through) - 1]);
1647
+
1648
+ switch (selection._tag) {
1649
+ case "RecordId":
1650
+ sources = [one(thread.byId.get(selection.recordId))];
1651
+ break;
1652
+ case "RunInput": {
1653
+ const entry = thread.runInputs.get(selection.runId);
1654
+
1655
+ if (entry === null)
1656
+ return yield* storeError("selected read", "Ambiguous original Run input");
1657
+ sources = [one(entry)];
1658
+ break;
1659
+ }
1660
+ case "ToolDeclaration": {
1661
+ const entry = thread.toolDeclarations.get(selection.settlementRecordId);
555
1662
 
556
- switch (selection._tag) {
557
- case "RecordId": {
558
- const record = thread.byId.get(selection.recordId);
1663
+ if (entry === null)
1664
+ return yield* storeError("selected read", "Ambiguous original tool declaration");
1665
+ if (entry !== undefined) {
1666
+ const payload = entry.record.payload;
559
1667
 
560
- records = record === undefined ? [] : [record];
561
- break;
1668
+ if (
1669
+ payload._tag !== "ModelResponseRecorded" ||
1670
+ !payload.toolOperations.some(
1671
+ (operation) =>
1672
+ toolCallSettledRecordId(payload.runId, payload.turn, operation.toolCallId) ===
1673
+ selection.settlementRecordId,
1674
+ )
1675
+ )
1676
+ return yield* storeError("selected read", "Invalid exact tool declaration pointer");
562
1677
  }
563
- case "RunInput": {
564
- const input = thread.runInputs.get(selection.runId);
1678
+ sources = [one(entry)];
1679
+ break;
1680
+ }
1681
+ case "RunEvidence":
1682
+ sources = [
1683
+ thread.runRecords.get(selection.runId) ?? [],
1684
+ [
1685
+ submissionInputRecordId(selection.submissionId),
1686
+ submissionAbortRecordId(selection.submissionId),
1687
+ submissionSettlementRecordId(selection.submissionId),
1688
+ ]
1689
+ .flatMap((id) => one(thread.byId.get(id)))
1690
+ .sort((a, b) => a.sequence - b.sequence),
1691
+ ];
1692
+ break;
1693
+ case "OperationEvidence":
1694
+ sources = [
1695
+ thread.operationRecords.get(
1696
+ JSON.stringify([selection.runId, selection.toolCallId]),
1697
+ ) ?? [],
1698
+ ...selection.approvalToolCallIds.flatMap((toolCallId) =>
1699
+ toolCallId === selection.toolCallId
1700
+ ? []
1701
+ : [
1702
+ thread.operationRecords.get(
1703
+ JSON.stringify([selection.runId, toolCallId, "approval"]),
1704
+ ) ?? [],
1705
+ ],
1706
+ ),
1707
+ one(thread.byId.get(selection.originRecordId)),
1708
+ ];
1709
+ break;
1710
+ case "LastAgentUpdate":
1711
+ sources = [latest(thread.agentUpdates)];
1712
+ break;
1713
+ case "DeliveryPredecessor":
1714
+ sources = [latest(thread.deliveryPredecessors)];
1715
+ break;
1716
+ case "LatestModelCompleted":
1717
+ sources = [latest(thread.modelCompletions)];
1718
+ break;
1719
+ case "LatestApplicationInput":
1720
+ sources = [latest(thread.applicationInputs)];
1721
+ break;
1722
+ case "DurableHistoryOwner":
1723
+ sources = [one(thread.durableOwners[0])];
1724
+ break;
1725
+ case "PromptEvidence":
1726
+ sources = [thread.promptEvidence];
1727
+ break;
1728
+ case "WorkHandoffs":
1729
+ sources = [thread.handoffs];
1730
+ break;
1731
+ case "ContextWindowBoundary": {
1732
+ // Match native SQL's disjoint coverage-prefix seeks. Each bucket contains
1733
+ // canonical sequence order; captured-tail reads never assume monotone coverage.
1734
+ let start = 0;
1735
+ let selected: CanonicalRecordEnvelope | undefined;
1736
+
1737
+ while (start < selection.atSequence) {
1738
+ const remaining = selection.atSequence - start;
1739
+ let height = Math.floor(Math.log2(remaining));
1740
+
1741
+ while (2 ** height > remaining) height--;
1742
+ const width = 2 ** height;
1743
+ const entries = thread.windowBoundaries.get(`${height}:${start / width}`) ?? [];
1744
+ const candidate = entries[upperSequence(entries, through) - 1];
565
1745
 
566
- if (input === null)
567
- return yield* storeError("selected read", "Ambiguous original Run input");
568
- records = input === undefined ? [] : [input];
569
- break;
1746
+ if (
1747
+ candidate !== undefined &&
1748
+ (selected === undefined || candidate.sequence > selected.sequence)
1749
+ )
1750
+ selected = candidate;
1751
+ start += width;
570
1752
  }
571
- case "Outstanding":
572
- if (thread.unverifiedWorkerInputs.size > 0)
573
- return yield* storeError("selected read", "Unverified worker acknowledgement");
574
- records = [...thread.outstanding.values()].flat();
575
- break;
576
- case "WorkerExecution":
577
- records = ["UserInputRecorded", "RunStarted"].flatMap(
578
- (tag) => thread.workerRecords.get(`execution:${tag}`) ?? [],
579
- );
580
- break;
581
- case "WorkerState":
582
- records = [
583
- ...(thread.workerRecords.get("worker") ?? []),
584
- ...(thread.workerRecords.get(`subtree:${selection.sourceSubmissionId ?? ""}`) ??
585
- []),
586
- ...(selection.sourceSubmissionId === undefined
587
- ? []
588
- : (thread.workerRecords.get(
1753
+ sources = [one(selected)];
1754
+ break;
1755
+ }
1756
+ case "WorkerStop":
1757
+ sources = [latest(thread.workerRecords.get(`stop:${selection.workerThreadId}`) ?? [])];
1758
+ break;
1759
+ case "WorkerExecution":
1760
+ sources = [
1761
+ thread.workerRecords.get("execution:UserInputRecorded") ?? [],
1762
+ thread.workerRecords.get("execution:SubmissionSettled") ?? [],
1763
+ ];
1764
+ break;
1765
+ case "WorkerState":
1766
+ sources = [
1767
+ thread.workerRecords.get("worker") ?? [],
1768
+ ...(selection.sourceSubmissionId === undefined
1769
+ ? []
1770
+ : [
1771
+ thread.workerRecords.get(`subtree:${selection.sourceSubmissionId}`) ?? [],
1772
+ thread.workerRecords.get(
589
1773
  `joined:${runIdForSubmission(selection.sourceSubmissionId)}`,
590
- ) ?? [])),
591
- ];
592
- break;
1774
+ ) ?? [],
1775
+ thread.workerRecords.get(`funding:${selection.sourceSubmissionId}`) ?? [],
1776
+ ]),
1777
+ ];
1778
+ break;
1779
+ case "LiveWorkerInputs": {
1780
+ const index = yield* readyIndex(request.threadId, thread.tailSequence).pipe(
1781
+ Effect.mapError((e) => storeError("selected read", e.message)),
1782
+ );
1783
+
1784
+ if (selection.throughSequence !== thread.tailSequence)
1785
+ return yield* storeError(
1786
+ "selected read",
1787
+ "Live work requires the current exact tail",
1788
+ );
1789
+
1790
+ const locators =
1791
+ selection.workerThreadId === undefined
1792
+ ? index.liveInputs
1793
+ : (index.inputsByWorker.get(selection.workerThreadId) ?? []);
1794
+
1795
+ const start = liveInputPosition(locators, after + 1);
1796
+ const selected: Array<CanonicalRecordEnvelope> = [];
1797
+
1798
+ for (let at = start; at < Math.min(locators.length, start + request.page.limit); at++) {
1799
+ const entry = index.entries.get(locators[at].id);
1800
+
1801
+ const origin =
1802
+ entry === undefined ? undefined : thread.byId.get(entry.originRecordId);
1803
+
1804
+ if (
1805
+ entry?.owner._tag !== "WorkerInput" ||
1806
+ origin?.record.payload._tag !== "WorkerInputRequested" ||
1807
+ origin.sequence !== locators[at].sequence ||
1808
+ origin.record.payload.admission.origin.worker.threadId !==
1809
+ entry.owner.workerThreadId ||
1810
+ (origin.record.payload.admission.reportKind === "update") !== entry.owner.update
1811
+ )
1812
+ return yield* storeError(
1813
+ "selected read",
1814
+ "Invalid live worker input locator or origin",
1815
+ );
1816
+ selected.push(origin);
1817
+ }
1818
+ sources = [selected];
1819
+ break;
593
1820
  }
1821
+ }
1822
+
1823
+ return selectedStream(sources, after, through, request.page.limit);
1824
+ }),
1825
+ );
1826
+
1827
+ const read: ThreadStore["Service"]["read"] = (input) =>
1828
+ Stream.unwrap(
1829
+ Effect.gen(function* () {
1830
+ const request = yield* validate(ThreadReadRequest, "read", input);
1831
+ const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
1832
+
1833
+ return readFromThread(request.threadId, thread, request);
1834
+ }),
1835
+ );
1836
+
1837
+ const decodePromptRecord = Schema.decodeUnknownEffect(PromptRecord);
1838
+
1839
+ const readPrompt: ThreadStore["Service"]["readPrompt"] = (input) =>
1840
+ Stream.unwrap(
1841
+ Effect.gen(function* () {
1842
+ const request = yield* validate(ThreadPromptRead, "readPrompt", input);
1843
+ const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
1844
+
1845
+ if (request.throughSequence > thread.tailSequence)
1846
+ return yield* storeError("readPrompt", "Captured sequence is ahead of canonical tail");
1847
+ const start = upperSequence(thread.promptEvidence, request.afterSequence ?? 0);
1848
+ const records: Array<PromptRecordEnvelope> = [];
1849
+
1850
+ for (
1851
+ let at = start;
1852
+ at < Math.min(thread.promptEvidence.length, start + request.limit);
1853
+ at++
1854
+ ) {
1855
+ const entry = thread.promptEvidence[at];
594
1856
 
595
- return Stream.fromIterable(
596
- records
597
- .filter((entry) => entry.sequence > (request.page.afterSequence ?? 0))
598
- .sort((a, b) => a.sequence - b.sequence)
599
- .slice(0, request.page.limit),
1857
+ if (entry.sequence > request.throughSequence) break;
1858
+
1859
+ const record = yield* decodePromptRecord(
1860
+ entry.record instanceof ExportedRecord ? entry.record.wire : entry.record,
1861
+ ).pipe(
1862
+ Effect.mapError((cause) => storeError("readPrompt", "Invalid prompt record", cause)),
600
1863
  );
1864
+
1865
+ records.push({ threadId: request.threadId, sequence: entry.sequence, record });
601
1866
  }
602
- const records = yield* readSnapshot(request.threadId, request.afterSequence, request.limit);
603
1867
 
604
1868
  return Stream.fromIterable(records);
605
1869
  }),
@@ -609,258 +1873,564 @@ const makeThreadStore = Effect.gen(function* () {
609
1873
  Stream.unwrap(
610
1874
  Effect.gen(function* () {
611
1875
  const request = yield* validate(ThreadObservation, "observe", unvalidated);
612
- const afterSequence = yield* offsetSequence(request.threadId, request.afterOffset);
1876
+ let after = yield* offsetSequence(request.threadId, request.afterOffset);
1877
+ const subscription = yield* PubSub.subscribe(updates);
613
1878
 
614
- return Stream.unwrap(
615
- Effect.gen(function* () {
616
- const subscription = yield* PubSub.subscribe(updates);
1879
+ const drain = () =>
1880
+ Stream.paginate(after, (cursor) =>
1881
+ readSnapshot(request.threadId, cursor, READ_PAGE_SIZE).pipe(
1882
+ Effect.map((records) => {
1883
+ after = records.at(-1)?.sequence ?? cursor;
617
1884
 
618
- const initial = yield* readSnapshot(
619
- request.threadId,
620
- afterSequence,
621
- MAX_RECORDS_PER_THREAD,
622
- );
1885
+ return [
1886
+ records,
1887
+ records.length > 0 ? Option.some(after) : Option.none<CanonicalSequence>(),
1888
+ ] as const;
1889
+ }),
1890
+ ),
1891
+ );
623
1892
 
624
- const highWater =
625
- initial.length === 0 ? afterSequence : (initial.at(-1)?.sequence ?? afterSequence);
626
-
627
- const live = Stream.fromEffectRepeat(PubSub.take(subscription)).pipe(
628
- Stream.mapAccumEffect(
629
- () => highWater,
630
- (lastSequence) =>
631
- readSnapshot(request.threadId, lastSequence, MAX_RECORDS_PER_THREAD).pipe(
632
- Effect.map(
633
- (records) => [records.at(-1)?.sequence ?? lastSequence, records] as const,
634
- ),
635
- ),
1893
+ return drain().pipe(
1894
+ Stream.concat(
1895
+ Stream.fromEffectRepeat(PubSub.take(subscription)).pipe(Stream.flatMap(drain)),
1896
+ ),
1897
+ );
1898
+ }),
1899
+ );
1900
+
1901
+ const emptyThread = (): StoredThread => ({
1902
+ ...emptyIndexes(),
1903
+ tailSequence: ZERO_CANONICAL_SEQUENCE,
1904
+ ranges: new Map(),
1905
+ rangeStarts: [],
1906
+ archiveContents: new Map(),
1907
+ chunks: new Map(),
1908
+ batches: new Map(),
1909
+ executions: new Map(),
1910
+ executionCommits: new Map(),
1911
+ batchesByFirst: new Map(),
1912
+ checkpoints: new Map(),
1913
+ });
1914
+
1915
+ const makeExportReader = (idleOnly: boolean) =>
1916
+ ThreadExporterReader.of({
1917
+ snapshot: Effect.fnUntraced(function* (threadId: ThreadId, captured?: TransferManifest) {
1918
+ const thread = MutableRef.get(state.ref).threads.get(threadId);
1919
+
1920
+ const facts =
1921
+ ledgerTransfer === undefined
1922
+ ? { idle: true, admissions: 0, aborts: 0, approvals: 0, resolutions: 0 }
1923
+ : yield* ledgerTransfer.snapshot(threadId);
1924
+
1925
+ const delivered = deliveryTransfer?.snapshot(threadId) ?? { idle: true, deliveries: 0 };
1926
+
1927
+ if (thread === undefined && facts.admissions === 0 && delivered.deliveries === 0)
1928
+ return yield* ThreadNotMaterialized.make({ threadId });
1929
+ if (idleOnly) {
1930
+ const index = yield* readyIndex(threadId, thread?.tailSequence ?? 0).pipe(
1931
+ Effect.mapError((e) => storeError("export", e.message)),
1932
+ );
1933
+
1934
+ if (
1935
+ owner.hasOwner(threadId) ||
1936
+ (thread?.executions.size ?? 0) > 0 ||
1937
+ index.entries.size > 0 ||
1938
+ !facts.idle ||
1939
+ !delivered.idle
1940
+ )
1941
+ return yield* ThreadExportRejected.make({ threadId, reason: "non-idle" });
1942
+ // Only the first page performs the full closed-receipt proof. The caller keeps
1943
+ // the source quiescent; subsequent pages still check current work and the tail.
1944
+ if (captured === undefined && thread !== undefined) {
1945
+ yield* Effect.gen(function* () {
1946
+ for (const runId of thread.runRecords.keys())
1947
+ yield* validateClosedRun(threadId, Schema.decodeSync(RunId)(runId));
1948
+ if (ledgerTransfer !== undefined) {
1949
+ const ledger = ledgerTransfer;
1950
+
1951
+ yield* Stream.runForEach(ledger.submissions(threadId), (submission) =>
1952
+ Effect.gen(function* () {
1953
+ const admission = yield* ledger.admission(threadId, submission.submissionId);
1954
+
1955
+ if (admission === undefined)
1956
+ return yield* storeError("export", "Missing immutable admission");
1957
+ yield* rebuildClosedSubmission(admission);
1958
+ }),
1959
+ );
1960
+ }
1961
+ }).pipe(
1962
+ Effect.provide(ThreadImportReader.layer(importReader(threadId, thread))),
1963
+ Effect.mapError((cause) =>
1964
+ Schema.is(ThreadImportRejected)(cause)
1965
+ ? ThreadExportRejected.make({ threadId, reason: "non-idle" })
1966
+ : cause,
636
1967
  ),
637
1968
  );
1969
+ }
1970
+ }
638
1971
 
639
- return Stream.fromIterable(initial).pipe(Stream.concat(live));
640
- }),
641
- );
1972
+ return {
1973
+ format: CURRENT_RECORD_FORMAT,
1974
+ threadId,
1975
+ tailSequence: thread?.tailSequence ?? ZERO_CANONICAL_SEQUENCE,
1976
+ snapshot: {
1977
+ admissions: facts.admissions,
1978
+ aborts: facts.aborts,
1979
+ approvals: facts.approvals,
1980
+ resolutions: facts.resolutions,
1981
+ deliveries: delivered.deliveries,
1982
+ },
1983
+ ...(facts.workerSeal === undefined ? {} : { workerSeal: facts.workerSeal }),
1984
+ };
642
1985
  }),
643
- );
1986
+ batch: Effect.fnUntraced(function* (threadId: ThreadId, fromSequence: number) {
1987
+ const thread = yield* findThread(MutableRef.get(state.ref), threadId);
1988
+ const batchId = thread.batchesByFirst.get(fromSequence);
1989
+ const stored = batchId === undefined ? undefined : thread.batches.get(batchId);
644
1990
 
645
- const exportThread: ThreadStore["Service"]["export"] = Effect.fn("MemoryThreadStore.export")(
646
- (unvalidated) =>
647
- Effect.gen(function* () {
648
- const request = yield* validate(ThreadExportRequest, "export", unvalidated);
1991
+ if (
1992
+ stored === undefined ||
1993
+ new TextEncoder().encode(stored.batchJson).byteLength > 16 * 1024 * 1024
1994
+ )
1995
+ return yield* storeError("export", "Missing or oversized canonical batch wire");
1996
+ const batch = yield* decodeStoredBatch(stored.batchJson, "export");
1997
+ const records: Array<ThreadExportRecord> = [];
649
1998
 
650
- const thread = yield* Ref.get(state).pipe(
651
- Effect.flatMap((current) => findThread(current, request.threadId)),
652
- );
1999
+ for (const [index, record] of batch.records.entries()) {
2000
+ const sequence = decodeCanonicalSequence(fromSequence + index);
2001
+ const exact = storedRecord(thread, sequence);
653
2002
 
654
- return ThreadExport.make({
655
- format: "effect-agent/thread@1",
656
- threadId: request.threadId,
657
- tailSequence: thread.tailSequence,
658
- tailDigest: thread.tailDigest,
659
- records: thread.records,
660
- });
2003
+ if (
2004
+ exact === undefined ||
2005
+ exact.batchId !== batch.batchId ||
2006
+ exact.record.recordId !== record.recordId
2007
+ )
2008
+ return yield* storeError("export", "Canonical batch index differs from its wire");
2009
+ records.push(
2010
+ ThreadExportRecord.make({
2011
+ threadId,
2012
+ batchId: batch.batchId,
2013
+ sequence,
2014
+ offset: observationOffset(threadId, sequence),
2015
+ record,
2016
+ }),
2017
+ );
2018
+ }
2019
+
2020
+ return { batches: [{ batchId: batch.batchId, producerId: batch.producerId }], records };
661
2021
  }),
662
- );
2022
+ facts: (threadId, section, after) => {
2023
+ const empty = Effect.succeed({
2024
+ facts: {
2025
+ admissions: [],
2026
+ commands: { aborts: [], approvals: [], resolutions: [] },
2027
+ deliveries: [],
2028
+ },
2029
+ after: after ?? "",
2030
+ });
663
2031
 
664
- const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fn(
665
- "MemoryThreadStore.inspectTail",
666
- )((unvalidated) =>
667
- Effect.gen(function* () {
668
- const request = yield* validate(ThreadTailRequest, "inspectTail", unvalidated);
2032
+ return section === "deliveries"
2033
+ ? (deliveryTransfer?.facts(threadId, after) ?? empty)
2034
+ : (ledgerTransfer?.facts(threadId, section, after) ?? empty);
2035
+ },
2036
+ });
669
2037
 
670
- const thread = yield* Ref.get(state).pipe(
671
- Effect.flatMap((current) => findThread(current, request.threadId)),
672
- );
2038
+ const exportReader = makeExportReader(true);
2039
+ const verificationReader = makeExportReader(false);
673
2040
 
674
- return ThreadTail.make({
675
- threadId: request.threadId,
676
- tailSequence: thread.tailSequence,
677
- tailDigest: thread.tailDigest,
678
- producerEpoch: thread.producerEpoch,
679
- });
680
- }),
681
- );
2041
+ const exportThread: ThreadStore["Service"]["export"] = (request) =>
2042
+ exportThreadPage(request).pipe(Effect.provideService(ThreadExporterReader, exportReader));
682
2043
 
683
- const readIdentity: ThreadStore["Service"]["readIdentity"] = Effect.fn(
684
- "MemoryThreadStore.readIdentity",
685
- )(function* (unvalidated) {
686
- const request = yield* validate(ThreadIdentityRequest, "readIdentity", unvalidated);
2044
+ const verificationPage: ThreadStore["Service"]["export"] = (request) =>
2045
+ exportThreadPage(request).pipe(Effect.provideService(ThreadExporterReader, verificationReader));
687
2046
 
688
- const thread = yield* Ref.get(state).pipe(
689
- Effect.flatMap((current) => findThread(current, request.threadId)),
690
- );
2047
+ const exactRunRecords = (thread: StoredThread, runId: RunId) =>
2048
+ Effect.gen(function* () {
2049
+ const records: Array<CanonicalRecordEnvelope> = [];
2050
+ let bytes = 0;
2051
+ const facts = thread.runRecords.get(runId) ?? [];
691
2052
 
692
- const selected = [
693
- thread.records[0],
694
- thread.byId.get(workerOriginRecordId(request.threadId)),
695
- thread.byId.get(subagentLineageRecordId(request.threadId)),
696
- ].filter((entry) => entry !== undefined);
697
-
698
- return yield* ThreadIdentity.makeEffect({
699
- threadId: request.threadId,
700
- tailSequence: thread.tailSequence,
701
- tailDigest: thread.tailDigest,
702
- producerEpoch: thread.producerEpoch,
703
- records: selected.filter(
704
- (entry, index) =>
705
- selected.findIndex((other) => other.record.recordId === entry.record.recordId) === index,
2053
+ if (
2054
+ facts.length >
2055
+ MAX_RUN_EVIDENCE_RECORDS + RUN_TERMINAL_RESERVE_RECORDS + MAX_RUN_RECOVERY_SUFFIX_RECORDS
2056
+ )
2057
+ return yield* storeError("Run evidence", "Run record bounds exceeded");
2058
+ for (const entry of facts) {
2059
+ bytes += canonicalRecordBytes(entry.record);
2060
+ if (bytes > MAX_RUN_EVIDENCE_BYTES + MAX_RUN_TERMINAL_BYTES + MAX_RUN_RECOVERY_SUFFIX_BYTES)
2061
+ return yield* storeError("Run evidence", "Run byte bounds exceeded");
2062
+ records.push(entry);
2063
+ }
2064
+
2065
+ return records.sort((a, b) => a.sequence - b.sequence);
2066
+ });
2067
+
2068
+ const exactRecordReader = (
2069
+ threadId: ThreadId,
2070
+ thread: StoredThread,
2071
+ ): Pick<
2072
+ ThreadImportReader["Service"],
2073
+ "read" | "record" | "runRecords" | "execution" | "hasAgent"
2074
+ > => ({
2075
+ read: (input) =>
2076
+ Stream.unwrap(
2077
+ validate(ThreadReadRequest, "read snapshot", input).pipe(
2078
+ Effect.map((request) => readFromThread(threadId, thread, request)),
2079
+ ),
2080
+ ),
2081
+ record: (id) => Effect.succeed(thread.byId.get(id)),
2082
+ runRecords: (id) => exactRunRecords(thread, id),
2083
+ execution: (id) => Effect.succeed(thread.executions.get(id)),
2084
+ hasAgent: (id) =>
2085
+ Effect.succeed(
2086
+ (thread.workerRecords.get("worker") ?? []).some(
2087
+ ({ record }) => record.payload._tag === "ThreadCreated" && record.payload.agentId === id,
2088
+ ),
706
2089
  ),
707
- }).pipe(
708
- Effect.mapError((cause) => storeError("readIdentity", "Invalid canonical identity", cause)),
709
- );
710
2090
  });
711
2091
 
712
- const saveCheckpoint: ThreadCheckpoints["save"] = Effect.fn("MemoryThreadStore.saveCheckpoint")(
713
- (unvalidated) =>
714
- Effect.gen(function* () {
715
- yield* validateCheckpointVersion(unvalidated);
716
- const request = yield* validate(SaveCheckpointRequest, "saveCheckpoint", unvalidated);
717
-
718
- const decision = yield* Ref.modify(
719
- state,
720
- (current): readonly [CheckpointDecision, MemoryState] => {
721
- const checkpoint = request.checkpoint;
722
- const thread = current.threads.get(checkpoint.threadId);
723
-
724
- if (thread === undefined) {
725
- return [
726
- {
727
- _tag: "failure",
728
- error: ThreadNotMaterialized.make({
729
- threadId: checkpoint.threadId,
730
- }),
731
- },
732
- current,
733
- ];
734
- }
735
- if (checkpoint.throughSequence > thread.tailSequence) {
736
- return [
737
- {
738
- _tag: "failure",
739
- error: CheckpointRejected.make({
740
- threadId: checkpoint.threadId,
741
- reason: "ahead-of-tail",
742
- }),
743
- },
744
- current,
745
- ];
746
- }
747
- if (thread.tailDigests.get(checkpoint.throughSequence) !== checkpoint.tailDigest) {
748
- return [
749
- {
750
- _tag: "failure",
751
- error: CheckpointRejected.make({
752
- threadId: checkpoint.threadId,
753
- reason: "digest-mismatch",
2092
+ const importReader = (id: ThreadId, thread: StoredThread): ThreadImportReader["Service"] => ({
2093
+ ...exactRecordReader(id, thread),
2094
+ admission: (sid) => ledgerTransfer?.admission(id, sid) ?? Effect.succeed(undefined),
2095
+ runOwner: (runId) =>
2096
+ runId.startsWith("run:")
2097
+ ? (ledgerTransfer?.admission(id, Schema.decodeSync(importSubmissionId)(runId.slice(4))) ??
2098
+ Effect.succeed(undefined))
2099
+ : Effect.succeed(undefined),
2100
+ commands: (sid) =>
2101
+ ledgerTransfer?.commands(sid) ??
2102
+ Effect.succeed({ aborts: [], approvals: [], resolutions: [] }),
2103
+ delivery: (messageId) => deliveryTransfer?.record(id, messageId) ?? Effect.succeed(undefined),
2104
+ });
2105
+
2106
+ const importThread: ThreadImport["Service"]["import"] = (pages) =>
2107
+ captureImportSource(pages, (captured) =>
2108
+ withMutation(
2109
+ Effect.gen(function* () {
2110
+ const progress = makeThreadImportProgress();
2111
+ let staged: StoredThread | undefined;
2112
+ let stagedWork = emptyWorkIndex();
2113
+ let stagedLedger: MemoryLedgerImport | undefined;
2114
+ let stagedDelivery: MemoryDeliveryImport | undefined;
2115
+ let existing: StoredThread | undefined;
2116
+
2117
+ yield* Stream.runForEach(captured, (input) =>
2118
+ Effect.gen(function* () {
2119
+ const page = yield* prepareImportPage(progress, input).pipe(
2120
+ Effect.provideService(Crypto.Crypto, crypto),
2121
+ );
2122
+
2123
+ const id = page.archive.threadId;
2124
+
2125
+ if (staged === undefined) {
2126
+ const current = MutableRef.get(state.ref);
2127
+
2128
+ existing = current.threads.get(id);
2129
+ const existingWork = MutableRef.get(workState.ref).get(id);
2130
+
2131
+ if (
2132
+ (existing !== undefined &&
2133
+ (existing.tailSequence > 0 ||
2134
+ existing.batches.size > 0 ||
2135
+ existing.executions.size > 0 ||
2136
+ existing.executionCommits.size > 0 ||
2137
+ existing.checkpoints.size > 0 ||
2138
+ existing.ranges.size > 0 ||
2139
+ existing.archiveContents.size > 0)) ||
2140
+ (existingWork !== undefined &&
2141
+ (existingWork.entries.size > 0 ||
2142
+ existingWork.state !== "ready" ||
2143
+ existingWork.through !== 0))
2144
+ )
2145
+ return yield* ThreadImportRejected.make({
2146
+ threadId: id,
2147
+ reason: "target-not-empty",
2148
+ message: "Destination canonical Thread is not empty",
2149
+ });
2150
+ if (
2151
+ !MutableRef.get(workState.ref).has(id) &&
2152
+ MutableRef.get(workState.ref).size >= maxThreads
2153
+ )
2154
+ return yield* storeError("import", "Destination Thread capacity exceeded");
2155
+ staged = emptyThread();
2156
+ stagedLedger =
2157
+ ledgerTransfer === undefined ? undefined : yield* ledgerTransfer.startImport(id);
2158
+ stagedDelivery =
2159
+ deliveryTransfer === undefined
2160
+ ? undefined
2161
+ : yield* deliveryTransfer.startImport(id);
2162
+ if (page.archive.workerSeal !== undefined && stagedLedger === undefined)
2163
+ return yield* ThreadImportRejected.make({
2164
+ threadId: id,
2165
+ reason: "unsupported-obligations",
2166
+ message: "A paired ledger is required to preserve the worker seal",
2167
+ });
2168
+ }
2169
+ if (page.archive.admissions.length > 0 && stagedLedger === undefined)
2170
+ return yield* ThreadImportRejected.make({
2171
+ threadId: id,
2172
+ reason: "unsupported-obligations",
2173
+ message: "A paired ledger is required for admission restore",
2174
+ });
2175
+ if (page.archive.deliveries.length > 0 && stagedDelivery === undefined)
2176
+ return yield* ThreadImportRejected.make({
2177
+ threadId: id,
2178
+ reason: "unsupported-obligations",
2179
+ message: "A paired delivery store is required for delivery restore",
2180
+ });
2181
+ for (const batch of page.batches) {
2182
+ if (
2183
+ !canonicalBatchFitsTransfer(
2184
+ id,
2185
+ batch.batch,
2186
+ utf8ByteLength(batch.batchJson),
2187
+ 4 * Math.ceil(utf8ByteLength(id) / 3) + 36,
2188
+ )
2189
+ )
2190
+ return yield* ThreadImportRejected.make({
2191
+ threadId: id,
2192
+ reason: "unsupported-capacity",
2193
+ message: "Canonical batch exceeds its bounded transfer representation",
2194
+ });
2195
+
2196
+ if (staged.batches.has(batch.batch.batchId))
2197
+ return yield* invalidThreadArchive("Duplicate canonical batch identity", id);
2198
+
2199
+ const records = page.records
2200
+ .filter((entry) => entry.batchId === batch.batch.batchId)
2201
+ .map((entry) =>
2202
+ CanonicalRecordEnvelope.make({
2203
+ ...entry,
2204
+ offset: observationOffset(id, entry.sequence),
2205
+ }),
2206
+ );
2207
+
2208
+ for (const entry of records)
2209
+ if (staged.byId.has(entry.record.recordId))
2210
+ return yield* invalidThreadArchive("Duplicate canonical record identity", id);
2211
+ const native = prepareIndexes(staged, records);
2212
+
2213
+ const commitWork = yield* prepareWork(
2214
+ stagedWork,
2215
+ records,
2216
+ (_, messageId) => stagedDelivery?.has(messageId) ?? false,
2217
+ ).pipe(Effect.provideService(Crypto.Crypto, crypto));
2218
+
2219
+ retainRecords(staged, records);
2220
+ native.commit();
2221
+ staged = {
2222
+ ...staged,
2223
+ ...native.indexes,
2224
+ tailSequence: batch.lastSequence,
2225
+ };
2226
+ staged.batches.set(batch.batch.batchId, {
2227
+ batchJson: batch.batchJson,
2228
+ result: AppendResult.make({
2229
+ firstSequence: batch.firstSequence,
2230
+ lastSequence: batch.lastSequence,
2231
+ replayed: false,
754
2232
  }),
755
- },
756
- current,
757
- ];
2233
+ });
2234
+ staged.batchesByFirst.set(batch.firstSequence, batch.batch.batchId);
2235
+ stagedWork = commitWork();
2236
+ }
2237
+ if (stagedLedger !== undefined) yield* stagedLedger.stage(page);
2238
+ if (stagedDelivery !== undefined) yield* stagedDelivery.stage(page);
2239
+ for (const delivery of page.archive.deliveries) {
2240
+ for (const workId of stagedWork.messages.get(delivery.key.messageId) ?? []) {
2241
+ const entry = stagedWork.entries.get(workId);
2242
+
2243
+ if (entry !== undefined) changeWorkCounts(stagedWork, entry, -1);
2244
+ stagedWork.entries.delete(workId);
2245
+ }
2246
+ stagedWork.messages.delete(delivery.key.messageId);
2247
+ stagedWork = { ...stagedWork, entryCount: stagedWork.entries.size };
2248
+ }
2249
+ }),
2250
+ );
2251
+ const result = yield* finishThreadImport(progress);
2252
+
2253
+ if (staged === undefined) return yield* invalidThreadArchive("Missing transfer pages");
2254
+ const restoredThread = staged;
2255
+ const id = result.threadId;
2256
+
2257
+ const fallback = {
2258
+ admission: () => Effect.succeed(undefined),
2259
+ runOwner: () => Effect.succeed(undefined),
2260
+ commands: () => Effect.succeed({ aborts: [], approvals: [], resolutions: [] }),
2261
+ };
2262
+
2263
+ const reader: ThreadImportReader["Service"] = {
2264
+ ...exactRecordReader(id, restoredThread),
2265
+ ...(stagedLedger?.reader ?? fallback),
2266
+ delivery: stagedDelivery?.record ?? (() => Effect.succeed(undefined)),
2267
+ };
2268
+
2269
+ return yield* Effect.gen(function* () {
2270
+ for (const runId of restoredThread.runRecords.keys()) {
2271
+ yield* verifyRunContinuations(
2272
+ yield* reader.runRecords(Schema.decodeSync(RunId)(runId)),
2273
+ { verifyContext: false },
2274
+ ).pipe(Effect.mapError((e) => invalidThreadArchive(e.message, id)));
2275
+ yield* validateClosedRun(id, Schema.decodeSync(RunId)(runId));
758
2276
  }
759
- if (
760
- !thread.checkpoints.has(checkpoint.throughSequence) &&
761
- thread.checkpoints.size >= MAX_CHECKPOINTS_PER_THREAD
762
- ) {
763
- return [
764
- {
765
- _tag: "failure",
766
- error: storeError(
767
- "saveCheckpoint",
768
- `In-memory checkpoint limit ${MAX_CHECKPOINTS_PER_THREAD} exceeded`,
769
- ),
770
- },
771
- current,
772
- ];
2277
+ let lastInputQueue = -1;
2278
+
2279
+ for (let sequence = 1; sequence <= restoredThread.tailSequence; sequence++) {
2280
+ const entry = storedRecord(restoredThread, sequence);
2281
+
2282
+ if (entry === undefined)
2283
+ return yield* invalidThreadArchive("Incomplete canonical staging", id);
2284
+ yield* verifyImportedReferences(entry);
2285
+ const payload = entry.record.payload;
2286
+
2287
+ if (payload._tag === "UserInputRecorded" && payload.submissionId !== undefined) {
2288
+ const admission = yield* reader.admission(payload.submissionId);
2289
+
2290
+ if (admission === undefined || admission.queueSequence <= lastInputQueue)
2291
+ return yield* invalidThreadArchive(
2292
+ "Canonical inputs violate immutable admission order",
2293
+ id,
2294
+ );
2295
+ lastInputQueue = admission.queueSequence;
2296
+ }
2297
+
2298
+ if (
2299
+ payload._tag === "UserInputRecorded" &&
2300
+ payload.submissionId !== undefined &&
2301
+ entry.record.recordId !== submissionInputRecordId(payload.submissionId)
2302
+ )
2303
+ return yield* invalidThreadArchive("Canonical input identity is ambiguous", id);
2304
+ if (
2305
+ payload._tag === "SubmissionSettled" &&
2306
+ entry.record.recordId !== submissionSettlementRecordId(payload.submissionId)
2307
+ )
2308
+ return yield* invalidThreadArchive(
2309
+ "Canonical settlement identity is ambiguous",
2310
+ id,
2311
+ );
2312
+ if (
2313
+ payload._tag !== "WorkerInputCompleted" &&
2314
+ "submissionId" in payload &&
2315
+ payload.submissionId !== undefined &&
2316
+ (yield* reader.admission(payload.submissionId)) === undefined
2317
+ )
2318
+ return yield* invalidThreadArchive("Canonical submission has no admission", id);
773
2319
  }
774
- const checkpoints = new Map(thread.checkpoints);
2320
+ if (stagedWork.entries.size > 0)
2321
+ return yield* ThreadImportRejected.make({
2322
+ threadId: id,
2323
+ reason: "unsupported-obligations",
2324
+ message: "Transfer contains unresolved work",
2325
+ });
775
2326
 
776
- checkpoints.set(checkpoint.throughSequence, checkpoint);
777
- const threads = new Map(current.threads);
2327
+ const commitLedger =
2328
+ stagedLedger === undefined
2329
+ ? undefined
2330
+ : yield* stagedLedger.prepareCommit(progress.manifest?.workerSeal);
778
2331
 
779
- threads.set(checkpoint.threadId, { ...thread, checkpoints });
2332
+ const commitDelivery =
2333
+ stagedDelivery === undefined ? undefined : yield* stagedDelivery.prepareCommit();
780
2334
 
781
- return [{ _tag: "success" }, { threads }];
782
- },
783
- );
2335
+ const installed = restoredThread;
784
2336
 
785
- if (decision._tag === "failure") return yield* decision.error;
786
- }),
787
- );
2337
+ yield* Effect.uninterruptible(
2338
+ Effect.sync(() => {
2339
+ commitLedger?.();
2340
+ commitDelivery?.();
2341
+ publishWork(id, stagedWork);
2342
+ MutableRef.get(state.ref).threads.set(id, installed);
2343
+ }).pipe(Effect.andThen(PubSub.publish(updates, undefined))),
2344
+ );
788
2345
 
789
- const loadCheckpoint: ThreadCheckpoints["load"] = Effect.fn("MemoryThreadStore.loadCheckpoint")(
790
- (unvalidated) =>
791
- Effect.gen(function* () {
792
- const request = yield* validate(LoadCheckpointRequest, "loadCheckpoint", unvalidated);
2346
+ return result;
2347
+ }).pipe(
2348
+ Effect.provide(ThreadImportReader.layer(reader)),
2349
+ Effect.provideService(ThreadDeliveryImportReader, {
2350
+ admission: (threadId, sid) =>
2351
+ threadId === id
2352
+ ? reader.admission(sid)
2353
+ : (ledgerTransfer?.admission(threadId, sid) ?? Effect.succeed(undefined)),
2354
+ record: (threadId, recordId) =>
2355
+ Effect.succeed(
2356
+ threadId === id
2357
+ ? restoredThread.byId.get(recordId)?.record
2358
+ : MutableRef.get(state.ref).threads.get(threadId)?.byId.get(recordId)?.record,
2359
+ ),
2360
+ }),
2361
+ );
2362
+ }).pipe(Effect.provideService(Crypto.Crypto, crypto)),
2363
+ ),
2364
+ );
793
2365
 
794
- const thread = yield* Ref.get(state).pipe(
795
- Effect.flatMap((current) => findThread(current, request.threadId)),
796
- );
2366
+ const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fnUntraced(
2367
+ function* (unvalidated) {
2368
+ const request = yield* validate(ThreadTailRequest, "inspectTail", unvalidated);
797
2369
 
798
- const maximum = request.atOrBeforeSequence ?? thread.tailSequence;
799
- let selected: ThreadCheckpoint | undefined;
2370
+ const thread = yield* Ref.get(state).pipe(
2371
+ Effect.flatMap((current) => findThread(current, request.threadId)),
2372
+ );
800
2373
 
801
- for (const [sequence, checkpoint] of thread.checkpoints) {
802
- if (
803
- sequence <= maximum &&
804
- (selected === undefined || sequence > selected.throughSequence)
805
- ) {
806
- selected = checkpoint;
807
- }
808
- }
809
- if (
810
- selected !== undefined &&
811
- thread.tailDigests.get(selected.throughSequence) !== selected.tailDigest
812
- ) {
813
- return yield* CheckpointRejected.make({
814
- threadId: request.threadId,
815
- reason: "digest-mismatch",
816
- });
817
- }
2374
+ return ThreadTail.make({
2375
+ threadId: request.threadId,
2376
+ tailSequence: thread.tailSequence,
2377
+ });
2378
+ },
2379
+ );
818
2380
 
819
- return Option.fromNullishOr(selected);
820
- }),
2381
+ const readIdentity: ThreadStore["Service"]["readIdentity"] = Effect.fnUntraced(
2382
+ function* (unvalidated) {
2383
+ const request = yield* validate(ThreadIdentityRequest, "readIdentity", unvalidated);
2384
+
2385
+ const thread = yield* Ref.get(state).pipe(
2386
+ Effect.flatMap((current) => findThread(current, request.threadId)),
2387
+ );
2388
+
2389
+ const selected = [
2390
+ storedRecord(thread, 1),
2391
+ thread.byId.get(workerOriginRecordId(request.threadId)),
2392
+ thread.byId.get(subagentLineageRecordId(request.threadId)),
2393
+ ].filter((entry) => entry !== undefined);
2394
+
2395
+ return yield* ThreadIdentity.makeEffect({
2396
+ threadId: request.threadId,
2397
+ tailSequence: thread.tailSequence,
2398
+ hasAdmissions:
2399
+ ledgerTransfer !== undefined &&
2400
+ (yield* ledgerTransfer.snapshot(request.threadId)).admissions > 0,
2401
+ records: selected.filter(
2402
+ (entry, index) =>
2403
+ selected.findIndex((other) => other.record.recordId === entry.record.recordId) ===
2404
+ index,
2405
+ ),
2406
+ }).pipe(
2407
+ Effect.mapError((cause) => storeError("readIdentity", "Invalid canonical identity", cause)),
2408
+ );
2409
+ },
821
2410
  );
822
2411
 
823
- const saveRecoveryCheckpoint: ThreadRecoveryCheckpoints["save"] = Effect.fn(
824
- "MemoryThreadStore.saveRecoveryCheckpoint",
825
- )(function* (unvalidated) {
826
- const request = yield* validate(
827
- SaveRecoveryCheckpointRequest,
828
- "saveRecoveryCheckpoint",
829
- unvalidated,
830
- );
2412
+ const saveCheckpoint: ThreadCheckpoints["save"] = Effect.fnUntraced(function* (unvalidated) {
2413
+ yield* validateCheckpointVersion(unvalidated);
2414
+ const request = yield* validate(SaveCheckpointRequest, "saveCheckpoint", unvalidated);
831
2415
 
832
2416
  const decision = yield* Ref.modify(
833
2417
  state,
834
- (
835
- current,
836
- ): readonly [
837
- CheckpointDecision | { readonly _tag: "failure"; readonly error: FenceRejected },
838
- MemoryState,
839
- ] => {
2418
+ (current): readonly [CheckpointDecision, MemoryState] => {
840
2419
  const checkpoint = request.checkpoint;
841
2420
  const thread = current.threads.get(checkpoint.threadId);
842
2421
 
843
- if (thread === undefined)
844
- return [
845
- {
846
- _tag: "failure",
847
- error: ThreadNotMaterialized.make({ threadId: checkpoint.threadId }),
848
- },
849
- current,
850
- ];
851
- if (thread.producerEpoch !== request.producerEpoch)
2422
+ if (thread === undefined) {
852
2423
  return [
853
2424
  {
854
2425
  _tag: "failure",
855
- error: FenceRejected.make({
2426
+ error: ThreadNotMaterialized.make({
856
2427
  threadId: checkpoint.threadId,
857
- actualEpoch: thread.producerEpoch,
858
- attemptedEpoch: request.producerEpoch,
859
2428
  }),
860
2429
  },
861
2430
  current,
862
2431
  ];
863
- if (checkpoint.throughSequence > thread.tailSequence)
2432
+ }
2433
+ if (checkpoint.throughSequence > thread.tailSequence) {
864
2434
  return [
865
2435
  {
866
2436
  _tag: "failure",
@@ -871,74 +2441,485 @@ const makeThreadStore = Effect.gen(function* () {
871
2441
  },
872
2442
  current,
873
2443
  ];
874
- if (thread.tailDigests.get(checkpoint.throughSequence) !== checkpoint.tailDigest)
2444
+ }
2445
+ if (!isBatchBoundary(thread, checkpoint.throughSequence)) {
875
2446
  return [
876
2447
  {
877
2448
  _tag: "failure",
878
2449
  error: CheckpointRejected.make({
879
2450
  threadId: checkpoint.threadId,
880
- reason: "digest-mismatch",
2451
+ reason: "not-batch-boundary",
881
2452
  }),
882
2453
  },
883
2454
  current,
884
2455
  ];
885
- if ((thread.recoveryCheckpoint?.throughSequence ?? -1) > checkpoint.throughSequence)
886
- return [{ _tag: "success" }, current];
887
- const threads = new Map(current.threads);
2456
+ }
2457
+ const checkpoints = thread.checkpoints;
888
2458
 
889
- threads.set(checkpoint.threadId, { ...thread, recoveryCheckpoint: checkpoint });
2459
+ checkpoints.set(checkpoint.throughSequence, checkpoint);
2460
+ // Checkpoints are disposable cache entries; history never fails when this cache fills.
2461
+ if (checkpoints.size > MAX_CHECKPOINTS_PER_THREAD) {
2462
+ let oldest = checkpoint.throughSequence;
890
2463
 
891
- return [{ _tag: "success" }, { threads }];
2464
+ for (const sequence of checkpoints.keys()) if (sequence < oldest) oldest = sequence;
2465
+ checkpoints.delete(oldest);
2466
+ }
2467
+
2468
+ return [{ _tag: "success" }, current];
892
2469
  },
893
2470
  );
894
2471
 
895
2472
  if (decision._tag === "failure") return yield* decision.error;
896
2473
  });
897
2474
 
898
- const loadRecoveryCheckpoint: ThreadRecoveryCheckpoints["load"] = Effect.fn(
899
- "MemoryThreadStore.loadRecoveryCheckpoint",
900
- )(function* (unvalidated) {
901
- const request = yield* validate(LoadCheckpointRequest, "loadRecoveryCheckpoint", unvalidated);
2475
+ const loadCheckpoint: ThreadCheckpoints["load"] = Effect.fnUntraced(function* (unvalidated) {
2476
+ const request = yield* validate(LoadCheckpointRequest, "loadCheckpoint", unvalidated);
902
2477
 
903
2478
  const thread = yield* Ref.get(state).pipe(
904
2479
  Effect.flatMap((current) => findThread(current, request.threadId)),
905
2480
  );
906
2481
 
907
- const checkpoint = thread.recoveryCheckpoint;
2482
+ const maximum = request.atOrBeforeSequence ?? thread.tailSequence;
2483
+ let selected: ThreadCheckpoint | undefined;
908
2484
 
909
- if (
910
- checkpoint === undefined ||
911
- checkpoint.throughSequence > (request.atOrBeforeSequence ?? thread.tailSequence)
912
- )
913
- return Option.none();
914
- if (thread.tailDigests.get(checkpoint.throughSequence) !== checkpoint.tailDigest)
2485
+ for (const [sequence, checkpoint] of thread.checkpoints) {
2486
+ if (sequence <= maximum && (selected === undefined || sequence > selected.throughSequence)) {
2487
+ selected = checkpoint;
2488
+ }
2489
+ }
2490
+ if (selected !== undefined && !isBatchBoundary(thread, selected.throughSequence)) {
915
2491
  return yield* CheckpointRejected.make({
916
2492
  threadId: request.threadId,
917
- reason: "digest-mismatch",
2493
+ reason: "not-batch-boundary",
918
2494
  });
2495
+ }
919
2496
 
920
- return Option.some(checkpoint);
2497
+ return Option.fromNullishOr(selected);
921
2498
  });
922
2499
 
923
- return ThreadStore.of({
2500
+ // Range contents remain volatile RAM. Publishing an archive locator adds no durability.
2501
+ const verifyRange = Effect.fnUntraced(function* (
2502
+ thread: StoredThread,
2503
+ range: ThreadArchiveRange,
2504
+ source?: Map<BatchId, string>,
2505
+ ) {
2506
+ let sequence = range.firstSequence;
2507
+ let bytes = 0;
2508
+ let batches = 0;
2509
+ const captured = new Map<BatchId, string>();
2510
+
2511
+ while (sequence <= range.lastSequence) {
2512
+ const id = thread.batchesByFirst.get(sequence);
2513
+ const batch = id === undefined ? undefined : thread.batches.get(id);
2514
+
2515
+ const text =
2516
+ id === undefined ? undefined : source === undefined ? batch?.batchJson : source.get(id);
2517
+
2518
+ if (
2519
+ id === undefined ||
2520
+ batch === undefined ||
2521
+ text === undefined ||
2522
+ batch.result.lastSequence > range.lastSequence
2523
+ )
2524
+ return yield* storeError("verify range", "Incomplete canonical batch range");
2525
+ bytes += new TextEncoder().encode(text).byteLength;
2526
+ batches++;
2527
+ if (
2528
+ bytes > MAX_ARCHIVE_RANGE_BYTES ||
2529
+ batches > MAX_ARCHIVE_RANGE_RECORDS ||
2530
+ text !== batch.batchJson
2531
+ )
2532
+ return yield* storeError(
2533
+ "verify range",
2534
+ "Range wire or bounds differ from canonical storage",
2535
+ );
2536
+
2537
+ const wire = yield* Schema.decodeEffect(Schema.fromJsonString(ExportBatch))(text).pipe(
2538
+ Effect.mapError((cause) => storeError("verify range", "Invalid range wire", cause)),
2539
+ );
2540
+
2541
+ if (
2542
+ wire.batchId !== id ||
2543
+ batch.result.firstSequence !== sequence ||
2544
+ batch.result.lastSequence !== sequence + wire.records.length - 1
2545
+ )
2546
+ return yield* storeError("verify range", "Canonical batch span or identity differs");
2547
+
2548
+ for (const [index, record] of wire.records.entries()) {
2549
+ const expected = storedRecord(thread, sequence + index);
2550
+
2551
+ if (
2552
+ expected === undefined ||
2553
+ expected.threadId !== range.threadId ||
2554
+ expected.sequence !== sequence + index ||
2555
+ expected.batchId !== id ||
2556
+ expected.record.recordId !== record.recordId
2557
+ )
2558
+ return yield* storeError("verify range", "Canonical record identity or order differs");
2559
+
2560
+ const canonical = yield* Schema.encodeEffect(ReadRecord)(expected.record).pipe(
2561
+ Effect.mapError((cause) => storeError("verify range", "Invalid canonical record", cause)),
2562
+ );
2563
+
2564
+ const archived = yield* Schema.encodeEffect(ExportRecord)(record).pipe(
2565
+ Effect.mapError((cause) => storeError("verify range", "Invalid archive record", cause)),
2566
+ );
2567
+
2568
+ if (canonicalJson(canonical) !== canonicalJson(archived))
2569
+ return yield* storeError("verify range", "Archive and canonical record bytes differ");
2570
+ }
2571
+ captured.set(id, text);
2572
+ sequence = decodeCanonicalSequence(batch.result.lastSequence + 1);
2573
+ }
2574
+ if (
2575
+ bytes !== range.byteCount ||
2576
+ batches !== range.batchCount ||
2577
+ range.recordCount !== range.lastSequence - range.firstSequence + 1
2578
+ )
2579
+ return yield* storeError("verify range", "Range spans or counts differ");
2580
+
2581
+ return captured;
2582
+ });
2583
+
2584
+ const archiveTarget = Effect.fnUntraced(function* (
2585
+ request: import("@yielded/agent/thread-archive-range").ThreadArchiveRangeSeal,
2586
+ ) {
2587
+ const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
2588
+
2589
+ return thread;
2590
+ });
2591
+
2592
+ const archives: ThreadArchiveStorage = {
2593
+ page: Effect.fnUntraced(function* (unvalidated) {
2594
+ const request = yield* validate(ThreadArchiveRangeRead, "range page", unvalidated);
2595
+ const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
2596
+ let low = 0;
2597
+ let high = thread.rangeStarts.length;
2598
+
2599
+ while (low < high) {
2600
+ const middle = Math.floor((low + high) / 2);
2601
+ const range = thread.ranges.get(thread.rangeStarts[middle]);
2602
+
2603
+ if (range === undefined)
2604
+ return yield* storeError("page ranges", "Range locator has no exact range");
2605
+ if (range.lastSequence <= (request.afterSequence ?? 0)) low = middle + 1;
2606
+ else high = middle;
2607
+ }
2608
+ const ranges: Array<ThreadArchiveRange> = [];
2609
+ const limit = request.limit ?? 32;
2610
+
2611
+ for (let at = low; at < Math.min(thread.rangeStarts.length, low + limit); at++) {
2612
+ const range = thread.ranges.get(thread.rangeStarts[at]);
2613
+
2614
+ if (range === undefined)
2615
+ return yield* storeError("page ranges", "Range locator has no exact range");
2616
+ ranges.push(range);
2617
+ }
2618
+
2619
+ const afterSequence =
2620
+ low + ranges.length < thread.rangeStarts.length ? ranges.at(-1)?.lastSequence : undefined;
2621
+
2622
+ return { ranges, ...(afterSequence === undefined ? {} : { afterSequence }) };
2623
+ }),
2624
+ seal: (unvalidated) =>
2625
+ withMutation(
2626
+ Effect.gen(function* () {
2627
+ const request = yield* validate(ThreadArchiveRangeSeal, "seal range", unvalidated);
2628
+ const thread = yield* archiveTarget(request);
2629
+ const previousStart = thread.rangeStarts.at(-1);
2630
+
2631
+ const previous =
2632
+ previousStart === undefined ? undefined : thread.ranges.get(previousStart);
2633
+
2634
+ if (previousStart !== undefined && previous === undefined)
2635
+ return yield* storeError("seal range", "Range locator has no exact range");
2636
+ const firstSequence = (previous?.lastSequence ?? 0) + 1;
2637
+
2638
+ if (firstSequence > thread.tailSequence) return Option.none<ThreadArchiveRange>();
2639
+ let nextSequence = firstSequence;
2640
+ let byteCount = 0;
2641
+ let batchCount = 0;
2642
+
2643
+ while (nextSequence <= thread.tailSequence) {
2644
+ const batchId = thread.batchesByFirst.get(nextSequence);
2645
+ const batch = batchId === undefined ? undefined : thread.batches.get(batchId);
2646
+
2647
+ if (batch === undefined)
2648
+ return yield* storeError("seal range", "Missing exact canonical batch");
2649
+ const bytes = utf8ByteLength(batch.batchJson);
2650
+
2651
+ if (
2652
+ batchCount > 0 &&
2653
+ (batch.result.lastSequence - firstSequence + 1 > MAX_ARCHIVE_RANGE_RECORDS ||
2654
+ byteCount + bytes > MAX_ARCHIVE_RANGE_BYTES)
2655
+ )
2656
+ break;
2657
+ byteCount += bytes;
2658
+ batchCount++;
2659
+ nextSequence = batch.result.lastSequence + 1;
2660
+ }
2661
+
2662
+ const range = ThreadArchiveRange.make({
2663
+ format: "effect-agent/thread-range@2",
2664
+ threadId: request.threadId,
2665
+ firstSequence: decodeCanonicalSequence(firstSequence),
2666
+ lastSequence: decodeCanonicalSequence(nextSequence - 1),
2667
+ recordCount: nextSequence - firstSequence,
2668
+ batchCount,
2669
+ byteCount,
2670
+ state: "sealed",
2671
+ });
2672
+
2673
+ yield* verifyRange(thread, range);
2674
+ thread.ranges.set(firstSequence, range);
2675
+ thread.rangeStarts.push(firstSequence);
2676
+
2677
+ return Option.some(range);
2678
+ }),
2679
+ ),
2680
+ verify: Effect.fnUntraced(function* (unvalidated) {
2681
+ const request = yield* validate(ThreadArchiveRangeRequest, "verify range", unvalidated);
2682
+ const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
2683
+ const range = thread.ranges.get(request.firstSequence);
2684
+
2685
+ if (range === undefined) return yield* storeError("verify range", "Unknown sealed range");
2686
+ yield* verifyRange(
2687
+ thread,
2688
+ range,
2689
+ range.state === "archived" ? thread.archiveContents.get(range.firstSequence) : undefined,
2690
+ );
2691
+ if (range.state === "archived" && !thread.archiveContents.has(range.firstSequence))
2692
+ return yield* storeError("verify range", "Published archive contents are missing");
2693
+
2694
+ return range;
2695
+ }),
2696
+ archive: (unvalidated) =>
2697
+ withMutation(
2698
+ Effect.gen(function* () {
2699
+ const request = yield* validate(ThreadArchiveRangePublish, "archive range", unvalidated);
2700
+ const thread = yield* archiveTarget(request);
2701
+ const range = thread.ranges.get(request.firstSequence);
2702
+
2703
+ if (range === undefined)
2704
+ return yield* storeError("archive range", "Unknown sealed range");
2705
+ if (range.state === "archived") {
2706
+ const contents = thread.archiveContents.get(range.firstSequence);
2707
+
2708
+ if (contents === undefined)
2709
+ return yield* storeError("archive range", "Published archive contents are missing");
2710
+ yield* verifyRange(thread, range, contents);
2711
+
2712
+ return range;
2713
+ }
2714
+ const contents = yield* verifyRange(thread, range);
2715
+
2716
+ yield* verifyRange(thread, range, contents);
2717
+
2718
+ const published = ThreadArchiveRange.make({
2719
+ ...range,
2720
+ state: "archived",
2721
+ locator: `memory:${encodeURIComponent(request.threadId)}:${range.firstSequence}`,
2722
+ });
2723
+
2724
+ thread.archiveContents.set(range.firstSequence, contents);
2725
+ thread.ranges.set(range.firstSequence, published);
2726
+
2727
+ return published;
2728
+ }),
2729
+ ),
2730
+ };
2731
+
2732
+ const readWorkerCapacity: NonNullable<ThreadStore["Service"]["readWorkerCapacity"]> =
2733
+ Effect.fnUntraced(function* (unvalidated) {
2734
+ const request = yield* validate(ThreadWorkerCapacityRequest, "worker capacity", unvalidated);
2735
+ const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
2736
+
2737
+ if (request.expectedTailSequence !== thread.tailSequence)
2738
+ return yield* storeError("worker capacity", "Canonical tail changed");
2739
+
2740
+ const index = yield* readyIndex(request.threadId, thread.tailSequence).pipe(
2741
+ Effect.mapError((e) => storeError("worker capacity", e.message)),
2742
+ );
2743
+
2744
+ const worker = index.workers.get(request.workerThreadId);
2745
+
2746
+ return ThreadWorkerCapacity.make({
2747
+ threadId: request.threadId,
2748
+ tailSequence: thread.tailSequence,
2749
+ activeWorkers: Math.min(index.workers.size, request.activeLimit + 1),
2750
+ workerActive: worker !== undefined,
2751
+ pendingInputs: Math.min(
2752
+ request.update ? (worker?.update ?? 0) : (worker?.ordinary ?? 0),
2753
+ request.pendingLimit + 1,
2754
+ ),
2755
+ });
2756
+ });
2757
+
2758
+ const verification = {
2759
+ verify: (unvalidated: import("@yielded/agent/thread-store").ThreadVerificationRequest) =>
2760
+ withMutation(
2761
+ Effect.gen(function* () {
2762
+ const request = yield* validate(ThreadVerificationRequest, "verify Thread", unvalidated);
2763
+ const thread = yield* findThread(MutableRef.get(state.ref), request.threadId);
2764
+ const id = request.threadId;
2765
+
2766
+ const reader = importReader(id, thread);
2767
+
2768
+ const submissions = () => ledgerTransfer?.submissions(id) ?? Stream.empty;
2769
+
2770
+ const runs = Stream.fromIterable(thread.runRecords.keys()).pipe(
2771
+ Stream.map((runId) => Schema.decodeSync(RunId)(runId)),
2772
+ );
2773
+
2774
+ const checkpoint = yield* loadCheckpoint(
2775
+ LoadCheckpointRequest.make({ threadId: id }),
2776
+ ).pipe(
2777
+ Effect.mapError((cause) =>
2778
+ Schema.is(CheckpointRejected)(cause)
2779
+ ? storeError("verify checkpoint", "Invalid checkpoint anchor", cause)
2780
+ : cause,
2781
+ ),
2782
+ );
2783
+
2784
+ return yield* verifyThreadInvariants({
2785
+ threadId: id,
2786
+ runs,
2787
+ executions: Stream.fromIterable(thread.executions.values()),
2788
+ submissions: submissions(),
2789
+ pages: streamExport({ threadId: id }).pipe(
2790
+ Stream.provideService(ThreadExportSource, { export: verificationPage }),
2791
+ Stream.mapError((cause) =>
2792
+ Schema.is(ThreadExportRejected)(cause)
2793
+ ? storeError("verify Thread", "Internal verification snapshot changed")
2794
+ : cause,
2795
+ ),
2796
+ ),
2797
+ ...(Option.isNone(checkpoint) ? {} : { checkpoint: checkpoint.value }),
2798
+ checkpointsSupported: true,
2799
+ ...(request.requireAllSettled === undefined
2800
+ ? {}
2801
+ : { requireAllSettled: request.requireAllSettled }),
2802
+ }).pipe(
2803
+ Effect.provide(ThreadImportReader.layer(reader)),
2804
+ Effect.provideService(Crypto.Crypto, crypto),
2805
+ );
2806
+ }),
2807
+ ),
2808
+ };
2809
+
2810
+ const threadStore = ThreadStore.of({
2811
+ work,
2812
+ archives,
2813
+ verification,
2814
+ readWorkerCapacity,
924
2815
  readIdentity,
925
2816
  countPeerMessages,
926
- materialize,
2817
+ materialize: (request) => withMutation(materialize(request)),
927
2818
  append,
2819
+ readExecution: (request) =>
2820
+ withMutation(
2821
+ Effect.gen(function* () {
2822
+ const thread = (yield* Ref.get(state)).threads.get(request.threadId);
2823
+
2824
+ if (thread === undefined)
2825
+ return yield* ThreadNotMaterialized.make({ threadId: request.threadId });
2826
+
2827
+ return Option.fromUndefinedOr(thread.executions.get(request.runId));
2828
+ }),
2829
+ ),
928
2830
  read,
2831
+ readPrompt,
929
2832
  observe,
930
- export: exportThread,
2833
+ export: (request) =>
2834
+ checkExportSource(request.threadId).pipe(Effect.andThen(withMutation(exportThread(request)))),
931
2835
  inspectTail,
932
- checkpoints: { save: saveCheckpoint, load: loadCheckpoint },
933
- recoveryCheckpoints: { save: saveRecoveryCheckpoint, load: loadRecoveryCheckpoint },
2836
+ checkpoints: { save: (request) => withMutation(saveCheckpoint(request)), load: loadCheckpoint },
934
2837
  });
2838
+
2839
+ return Context.make(ThreadStore, threadStore).pipe(
2840
+ Context.add(StorageOwner, owner),
2841
+ Context.add(ThreadImport, { import: importThread }),
2842
+ Context.add(MemoryThreadStoreKernel, {
2843
+ owner,
2844
+ withMutation,
2845
+ retainDelivery,
2846
+ checkThreadCapacity: (threadId) =>
2847
+ Effect.suspend(() =>
2848
+ !MutableRef.get(workState.ref).has(threadId) &&
2849
+ MutableRef.get(workState.ref).size >= maxThreads
2850
+ ? Effect.fail(storeError("Thread capacity", "Retained Thread quota exceeded"))
2851
+ : Effect.void,
2852
+ ),
2853
+ initializeWork: (threadId) => {
2854
+ if (
2855
+ !MutableRef.get(state.ref).threads.has(threadId) &&
2856
+ !MutableRef.get(workState.ref).has(threadId)
2857
+ )
2858
+ publishWork(threadId, emptyWorkIndex());
2859
+ },
2860
+ registerWorkOwner: (kind, owner) =>
2861
+ withMutation(
2862
+ Effect.sync(() => {
2863
+ if (workOwners.has(kind))
2864
+ throw new Error(`MemoryThreadStore already has a ${kind} work owner`);
2865
+ workOwners.set(kind, owner);
2866
+ }),
2867
+ ),
2868
+ registerDeliveryLookup: (lookup) =>
2869
+ withMutation(
2870
+ Effect.sync(() => {
2871
+ if (deliveryLookup !== undefined)
2872
+ throw new Error("MemoryThreadStore already has a delivery lookup");
2873
+ deliveryLookup = lookup;
2874
+ }),
2875
+ ),
2876
+ registerLedgerTransfer: (transfer) =>
2877
+ withMutation(
2878
+ Effect.sync(() => {
2879
+ if (ledgerTransfer !== undefined)
2880
+ throw new Error("MemoryThreadStore already has a paired SubmissionLedger");
2881
+ ledgerTransfer = transfer;
2882
+ }),
2883
+ ),
2884
+ registerDeliveryTransfer: (transfer) =>
2885
+ withMutation(
2886
+ Effect.sync(() => {
2887
+ if (deliveryTransfer !== undefined)
2888
+ throw new Error("MemoryThreadStore already has a paired delivery store");
2889
+ deliveryTransfer = transfer;
2890
+ }),
2891
+ ),
2892
+ prepareAppend,
2893
+ appendPrepared,
2894
+ record: (threadId, recordId) =>
2895
+ Ref.get(state).pipe(
2896
+ Effect.map((current) => current.threads.get(threadId)?.byId.get(recordId)?.record),
2897
+ ),
2898
+ toolCallResults: (threadId, runId, toolCallId) =>
2899
+ Ref.get(state).pipe(
2900
+ Effect.map((current) =>
2901
+ (
2902
+ current.threads
2903
+ .get(threadId)
2904
+ ?.operationRecords.get(JSON.stringify([runId, toolCallId, "settled"])) ?? []
2905
+ )
2906
+ .slice(0, 2)
2907
+ .map((entry) => entry.record),
2908
+ ),
2909
+ ),
2910
+ tail: (threadId) => inspectTail(ThreadTailRequest.make({ threadId })),
2911
+ }),
2912
+ );
935
2913
  });
936
2914
 
937
2915
  /**
938
2916
  * In-memory canonical Thread persistence. Durable accepted work is served by the separate
939
- * SubmissionLedger port; this Layer deliberately provides only the ThreadStore.
2917
+ * SubmissionLedger port; this Layer provides ThreadStore, ThreadReader, and ThreadImport.
2918
+ * Importing admissions requires a MemorySubmissionLedger paired with this store.
940
2919
  */
941
- export const MemoryThreadStoreLive = Layer.effect(ThreadStore, makeThreadStore);
2920
+ export const MemoryThreadStoreLive = ThreadReader.layer().pipe(
2921
+ Layer.provideMerge(Layer.effectContext(makeThreadStore)),
2922
+ );
942
2923
 
943
2924
  /** Configure a finite retained Thread capacity. Invalid construction options throw immediately. */
944
2925
  export const memoryThreadStoreLayer = (options: { readonly maxThreads?: number } = {}) =>