@effect-agent/storage-cloudflare 0.1.0-beta.8 → 0.1.0-beta.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/DoMemoryStore.d.mts +29 -0
  2. package/dist/DoMemoryStore.mjs +55 -0
  3. package/dist/DoMemoryStore.mjs.map +1 -0
  4. package/dist/DoMessageDeliveryStore.d.mts +17 -0
  5. package/dist/DoMessageDeliveryStore.mjs +32 -0
  6. package/dist/DoMessageDeliveryStore.mjs.map +1 -0
  7. package/dist/DoScheduleStore.d.mts +36 -0
  8. package/dist/DoScheduleStore.mjs +388 -0
  9. package/dist/DoScheduleStore.mjs.map +1 -0
  10. package/dist/DoStorageConfig.d.mts +45 -0
  11. package/dist/DoStorageConfig.mjs +52 -0
  12. package/dist/DoStorageConfig.mjs.map +1 -0
  13. package/dist/DoStorageError-Cmhvl4TQ.d.mts +98 -0
  14. package/dist/DoStorageError.d.mts +2 -0
  15. package/dist/DoStorageError.mjs +164 -0
  16. package/dist/DoStorageError.mjs.map +1 -0
  17. package/dist/DoStorageFailpoint.d.mts +17 -0
  18. package/dist/DoStorageFailpoint.mjs +14 -0
  19. package/dist/DoStorageFailpoint.mjs.map +1 -0
  20. package/dist/DoStorageFailpointTesting.d.mts +34 -0
  21. package/dist/DoStorageFailpointTesting.mjs +35 -0
  22. package/dist/DoStorageFailpointTesting.mjs.map +1 -0
  23. package/dist/DoStorageVersion-x5OOurvZ.d.mts +12 -0
  24. package/dist/DoStorageVersion.d.mts +2 -0
  25. package/dist/DoStorageVersion.mjs +8 -0
  26. package/dist/DoStorageVersion.mjs.map +1 -0
  27. package/dist/DoSubmissionLedger.d.mts +23 -0
  28. package/dist/DoSubmissionLedger.mjs +1849 -0
  29. package/dist/DoSubmissionLedger.mjs.map +1 -0
  30. package/dist/DoSubscriptionStore.d.mts +25 -0
  31. package/dist/DoSubscriptionStore.mjs +183 -0
  32. package/dist/DoSubscriptionStore.mjs.map +1 -0
  33. package/dist/DoThreadStore.d.mts +59 -0
  34. package/dist/DoThreadStore.mjs +465 -0
  35. package/dist/DoThreadStore.mjs.map +1 -0
  36. package/dist/MemoryProtocol.d.mts +38160 -0
  37. package/dist/MemoryProtocol.mjs +219 -0
  38. package/dist/MemoryProtocol.mjs.map +1 -0
  39. package/dist/PortProtocol.d.mts +3207 -0
  40. package/dist/PortProtocol.mjs +176 -0
  41. package/dist/PortProtocol.mjs.map +1 -0
  42. package/dist/PortRouting.d.mts +83 -0
  43. package/dist/PortRouting.mjs +410 -0
  44. package/dist/PortRouting.mjs.map +1 -0
  45. package/dist/do-journal-DHqCQeBU.mjs +991 -0
  46. package/dist/do-journal-DHqCQeBU.mjs.map +1 -0
  47. package/dist/index.d.mts +14 -1521
  48. package/dist/index.mjs +14 -3695
  49. package/dist/migrations-Ba_kOrSx.mjs +287 -0
  50. package/dist/migrations-Ba_kOrSx.mjs.map +1 -0
  51. package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
  52. package/package.json +1 -45
  53. package/src/DoMemoryStore.ts +59 -0
  54. package/src/DoMessageDeliveryStore.ts +53 -0
  55. package/src/DoScheduleStore.ts +622 -0
  56. package/src/{errors.ts → DoStorageError.ts} +10 -3
  57. package/src/DoStorageFailpoint.ts +20 -0
  58. package/src/{do-storage-failpoint.ts → DoStorageFailpointTesting.ts} +5 -18
  59. package/src/DoStorageVersion.ts +2 -0
  60. package/src/{do-ledger.ts → DoSubmissionLedger.ts} +862 -163
  61. package/src/DoSubscriptionStore.ts +329 -0
  62. package/src/DoThreadStore.ts +1014 -0
  63. package/src/MemoryProtocol.ts +355 -0
  64. package/src/{port-protocol.ts → PortProtocol.ts} +44 -37
  65. package/src/{routing.ts → PortRouting.ts} +292 -264
  66. package/src/index.ts +13 -32
  67. package/src/{do-journal.ts → internal/do-journal.ts} +716 -232
  68. package/src/internal/message-delivery-schema.ts +24 -0
  69. package/src/{migrations.ts → internal/migrations.ts} +51 -35
  70. package/src/internal/recovery-checkpoint-schema.ts +17 -0
  71. package/dist/index.mjs.map +0 -1
  72. package/src/do-conversation-store.ts +0 -850
  73. /package/src/{do-storage-config.ts → DoStorageConfig.ts} +0 -0
@@ -1,21 +1,32 @@
1
- import { CanonicalSequence, ProducerEpoch } from "@effect-agent/session";
1
+ import { EMPTY_TAIL_DIGEST } from "@effect-agent/thread/Digest";
2
+ import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread/Records";
3
+ import { checkV2ThreadLayout } from "@effect-agent/thread/SqlStorageV2Upgrade";
4
+ import {
5
+ MAX_THREAD_EXPORT_RECORDS,
6
+ CheckpointRejected,
7
+ FenceRejected,
8
+ ThreadNotMaterialized,
9
+ type SaveRecoveryCheckpointRequest,
10
+ } from "@effect-agent/thread/ThreadStore";
2
11
  import { SqliteMigrator } from "@effect/sql-sqlite-do";
3
- import { Effect, Schema } from "effect";
12
+ import { Effect, Schema, Stream } from "effect";
4
13
  import * as SqlClient from "effect/unstable/sql/SqlClient";
5
14
  import { SqlError } from "effect/unstable/sql/SqlError";
6
15
 
7
16
  import {
17
+ type DoStorageFailpointError,
8
18
  DoAppendConflict,
9
19
  DoCheckpointConflict,
10
20
  DoFenceRejected,
11
21
  DoStorageCompatibilityError,
12
22
  DoStorageCorruptionError,
13
23
  DoStorageError,
14
- DoStorageFailpointError,
15
24
  DoValueBoundExceeded,
16
25
  type DoStorageFailpointLocation,
17
- } from "./errors.ts";
18
- import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
26
+ } from "../DoStorageError.ts";
27
+ import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
28
+ import { CurrentDoStorageVersion, createNonterminalIndex, doMigrations } from "./migrations.ts";
29
+ import { createRecoveryCheckpointTable } from "./recovery-checkpoint-schema.ts";
19
30
 
20
31
  /**
21
32
  * Static schema ceiling for stored text columns. Writes are bounded in BYTES by the
@@ -25,18 +36,23 @@ import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
25
36
  */
26
37
  const BoundedStoredText = Schema.String.check(Schema.isMaxLength(2_000_000));
27
38
  const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
28
- const MAX_RECORDS_PER_CONVERSATION = 65_536;
39
+ const MAX_RECORDS_PER_THREAD = MAX_THREAD_EXPORT_RECORDS;
40
+ const ZERO_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
29
41
  const MAX_IDENTIFIER_LENGTH = 1_024;
42
+ const MAX_READ_PAGE_JSON_BYTES = 4 * 1024 * 1024;
30
43
  /** Durable Object SQL storage allows at most 100 bound parameters per statement. */
31
44
  const MAX_BOUND_PARAMETERS = 100;
45
+ const isSqlError = Schema.is(SqlError);
32
46
 
33
47
  const storedTextBytes = (value: string): number => new TextEncoder().encode(value).byteLength;
34
48
 
35
49
  const chunked = <A>(values: ReadonlyArray<A>, size: number): Array<ReadonlyArray<A>> => {
36
50
  const chunks: Array<ReadonlyArray<A>> = [];
51
+
37
52
  for (let index = 0; index < values.length; index += size) {
38
53
  chunks.push(values.slice(index, index + size));
39
54
  }
55
+
40
56
  return chunks;
41
57
  };
42
58
 
@@ -48,8 +64,8 @@ class DoNameRow extends Schema.Class<DoNameRow>("DoNameRow")({
48
64
  name: BoundedIdentifier,
49
65
  }) {}
50
66
 
51
- class ConversationRow extends Schema.Class<ConversationRow>("ConversationRow")({
52
- conversation_id: BoundedIdentifier,
67
+ class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
68
+ thread_id: BoundedIdentifier,
53
69
  created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
54
70
  producer_epoch: ProducerEpoch,
55
71
  tail_digest: BoundedStoredText,
@@ -60,7 +76,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
60
76
  batch_digest: BoundedStoredText,
61
77
  batch_id: BoundedIdentifier,
62
78
  batch_json: BoundedStoredText,
63
- conversation_id: BoundedIdentifier,
79
+ thread_id: BoundedIdentifier,
64
80
  first_sequence: CanonicalSequence,
65
81
  last_sequence: CanonicalSequence,
66
82
  tail_digest: BoundedStoredText,
@@ -68,15 +84,22 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
68
84
 
69
85
  class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
70
86
  batch_id: BoundedIdentifier,
71
- conversation_id: BoundedIdentifier,
87
+ thread_id: BoundedIdentifier,
72
88
  record_id: BoundedIdentifier,
73
89
  record_json: BoundedStoredText,
74
90
  sequence: CanonicalSequence,
75
91
  }) {}
76
92
 
93
+ const ReadPlanRow = Schema.Struct({
94
+ sequence: CanonicalSequence,
95
+ record_json_bytes: Schema.Natural.check(Schema.isLessThanOrEqualTo(MAX_READ_PAGE_JSON_BYTES)),
96
+ });
97
+
98
+ type ReadPage = [typeof ReadPlanRow.Type, ...Array<typeof ReadPlanRow.Type>];
99
+
77
100
  class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
78
101
  checkpoint_json: BoundedStoredText,
79
- conversation_id: BoundedIdentifier,
102
+ thread_id: BoundedIdentifier,
80
103
  tail_digest: BoundedStoredText,
81
104
  through_sequence: CanonicalSequence,
82
105
  }) {}
@@ -94,7 +117,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
94
117
  batchDigest: BoundedStoredText,
95
118
  batchId: BoundedIdentifier,
96
119
  batchJson: BoundedStoredText,
97
- conversationId: BoundedIdentifier,
120
+ threadId: BoundedIdentifier,
98
121
  expectedTailDigest: BoundedStoredText,
99
122
  expectedTailSequence: CanonicalSequence,
100
123
  producerEpoch: ProducerEpoch,
@@ -114,7 +137,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
114
137
  export class RawReadRequest extends Schema.Class<RawReadRequest>(
115
138
  "@effect-agent/storage-cloudflare/RawReadRequest",
116
139
  )({
117
- conversationId: BoundedIdentifier,
140
+ threadId: BoundedIdentifier,
118
141
  fromSequenceExclusive: CanonicalSequence,
119
142
  limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
120
143
  }) {}
@@ -123,17 +146,15 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
123
146
  "@effect-agent/storage-cloudflare/RawCheckpoint",
124
147
  )({
125
148
  checkpointJson: BoundedStoredText,
126
- conversationId: BoundedIdentifier,
149
+ threadId: BoundedIdentifier,
127
150
  tailDigest: BoundedStoredText,
128
151
  throughSequence: CanonicalSequence,
129
152
  }) {}
130
153
 
131
- export class RawConversationExport extends Schema.Class<RawConversationExport>(
132
- "@effect-agent/storage-cloudflare/RawConversationExport",
154
+ export class RawThreadExport extends Schema.Class<RawThreadExport>(
155
+ "@effect-agent/storage-cloudflare/RawThreadExport",
133
156
  )({
134
- batches: Schema.Array(BatchRow),
135
- checkpoints: Schema.Array(CheckpointRow),
136
- conversation: ConversationRow,
157
+ thread: ThreadRow,
137
158
  records: Schema.Array(RecordRow),
138
159
  }) {}
139
160
 
@@ -167,7 +188,7 @@ const storageError =
167
188
  });
168
189
 
169
190
  /** Decode raw Durable Object SQLite rows against a Schema, reporting failures as typed corruption. */
170
- export const decodeRows = Effect.fn("DoJournal.decodeRows")(
191
+ export const decodeRows = Effect.fn(
171
192
  <A, I>(
172
193
  schema: Schema.Codec<ReadonlyArray<A>, ReadonlyArray<I>>,
173
194
  table: string,
@@ -186,7 +207,7 @@ export const decodeRows = Effect.fn("DoJournal.decodeRows")(
186
207
  );
187
208
 
188
209
  /** Decode exactly one raw row against a Schema, reporting failures as typed corruption. */
189
- export const decodeSingleRow = Effect.fn("DoJournal.decodeSingleRow")(
210
+ export const decodeSingleRow = Effect.fn(
190
211
  <A, I>(
191
212
  schema: Schema.Codec<ReadonlyArray<A>, ReadonlyArray<I>>,
192
213
  table: string,
@@ -208,6 +229,183 @@ export const decodeSingleRow = Effect.fn("DoJournal.decodeSingleRow")(
208
229
  ),
209
230
  );
210
231
 
232
+ /** Column inventory of the supported v3 predecessor, independent of physical column order. */
233
+ const predecessorColumns = {
234
+ effect_agent_threads: [
235
+ "thread_id",
236
+ "created_at",
237
+ "tail_sequence",
238
+ "tail_digest",
239
+ "producer_epoch",
240
+ ],
241
+ effect_agent_canonical_batches: [
242
+ "thread_id",
243
+ "batch_id",
244
+ "first_sequence",
245
+ "last_sequence",
246
+ "batch_digest",
247
+ "tail_digest",
248
+ "batch_json",
249
+ ],
250
+ effect_agent_canonical_records: ["thread_id", "sequence", "record_id", "batch_id", "record_json"],
251
+ effect_agent_checkpoints: ["thread_id", "through_sequence", "tail_digest", "checkpoint_json"],
252
+ effect_agent_submissions: [
253
+ "submission_id",
254
+ "thread_id",
255
+ "queue_sequence",
256
+ "principal",
257
+ "idempotency_key",
258
+ "agent_id",
259
+ "agent_digests_json",
260
+ "deployment_id",
261
+ "input_json",
262
+ "input_digest",
263
+ "receipt_id",
264
+ "state",
265
+ "settled_outcome",
266
+ "created_at",
267
+ "ready_at",
268
+ "input_applied_record_id",
269
+ "input_applied_sequence",
270
+ "joined_host_submission_id",
271
+ "suspended_reason_json",
272
+ "suspended_at",
273
+ "unknown_reason",
274
+ "unknown_tool_call_ids_json",
275
+ "parent_submission_id",
276
+ "parent_tool_call_id",
277
+ "admission_group",
278
+ "admission_fence_json",
279
+ ],
280
+ effect_agent_submission_ownership: [
281
+ "submission_id",
282
+ "attempt_id",
283
+ "ownership_token",
284
+ "producer_epoch",
285
+ "owner_producer_id",
286
+ "lease_expires_at",
287
+ ],
288
+ effect_agent_attempts: [
289
+ "attempt_id",
290
+ "submission_id",
291
+ "thread_id",
292
+ "owner_producer_id",
293
+ "producer_epoch",
294
+ "claimed_at",
295
+ ],
296
+ effect_agent_settlement_reservations: [
297
+ "submission_id",
298
+ "settlement_id",
299
+ "outcome",
300
+ "record_id",
301
+ "record_json",
302
+ "record_digest",
303
+ "reserved_at",
304
+ "finalized_at",
305
+ ],
306
+ effect_agent_abort_intents: [
307
+ "submission_id",
308
+ "author",
309
+ "reason",
310
+ "requested_at",
311
+ "canonical_record_id",
312
+ ],
313
+ effect_agent_approval_decisions: [
314
+ "submission_id",
315
+ "tool_call_id",
316
+ "decision",
317
+ "resolver",
318
+ "reason",
319
+ "decided_at",
320
+ ],
321
+ effect_agent_unknown_resolutions: [
322
+ "submission_id",
323
+ "tool_call_id",
324
+ "author",
325
+ "reason",
326
+ "resolution_json",
327
+ "resolved_at",
328
+ ],
329
+ effect_agent_child_reservations: [
330
+ "reservation_id",
331
+ "parent_submission_id",
332
+ "parent_tool_call_id",
333
+ "child_submission_id",
334
+ "status",
335
+ "allocation_json",
336
+ "allocation_digest",
337
+ "accounting_json",
338
+ "reserved_at",
339
+ "release_began_at",
340
+ "released_at",
341
+ ],
342
+ effect_agent_meta: ["key", "value"],
343
+ effect_agent_child_settlements: [
344
+ "parent_submission_id",
345
+ "child_submission_id",
346
+ "child_outcome",
347
+ "recorded_at",
348
+ ],
349
+ } as const;
350
+
351
+ const checkPredecessorLayout = Effect.fn("DoJournal.checkPredecessorLayout")(function* (
352
+ version: 3 | 4 | 5,
353
+ ) {
354
+ const sql = yield* SqlClient.SqlClient;
355
+
356
+ const messageColumns =
357
+ version === 3
358
+ ? predecessorColumns
359
+ : {
360
+ ...predecessorColumns,
361
+ effect_agent_submissions: [
362
+ ...predecessorColumns.effect_agent_submissions,
363
+ "worker_admission_json",
364
+ "message_admission_json",
365
+ ],
366
+ effect_agent_message_deliveries: [
367
+ "owner_thread_id",
368
+ "message_id",
369
+ "version",
370
+ "state",
371
+ "deadline_at_millis",
372
+ "record_json",
373
+ ],
374
+ };
375
+
376
+ const expectedColumns = {
377
+ ...messageColumns,
378
+ ...(version === 5
379
+ ? {
380
+ effect_agent_recovery_checkpoints: [
381
+ "thread_id",
382
+ "through_sequence",
383
+ "tail_digest",
384
+ "checkpoint_json",
385
+ ],
386
+ }
387
+ : {}),
388
+ };
389
+
390
+ for (const [table, expected] of Object.entries(expectedColumns)) {
391
+ const columns = yield* decodeRows(
392
+ Schema.Array(Schema.Struct({ name: BoundedIdentifier })),
393
+ table,
394
+ "schema",
395
+ yield* sql.unsafe(`PRAGMA table_info(${table})`),
396
+ );
397
+
398
+ const names = new Set<string>(expected);
399
+
400
+ if (columns.length !== names.size || columns.some((column) => !names.has(column.name)))
401
+ return yield* DoStorageCompatibilityError.make({
402
+ actualVersion: version,
403
+ supportedVersion: CurrentDoStorageVersion,
404
+ message: `The v${version} ${table} columns do not match the supported predecessor; no upgrade was committed.`,
405
+ });
406
+ }
407
+ });
408
+
211
409
  const REQUIRED_TABLES = [
212
410
  "effect_agent_abort_intents",
213
411
  "effect_agent_approval_decisions",
@@ -217,7 +415,7 @@ const REQUIRED_TABLES = [
217
415
  "effect_agent_checkpoints",
218
416
  "effect_agent_child_reservations",
219
417
  "effect_agent_child_settlements",
220
- "effect_agent_conversations",
418
+ "effect_agent_threads",
221
419
  "effect_agent_meta",
222
420
  "effect_agent_settlement_reservations",
223
421
  "effect_agent_submission_ownership",
@@ -226,7 +424,7 @@ const REQUIRED_TABLES = [
226
424
  ] as const;
227
425
 
228
426
  /**
229
- * Exact-or-fresh storage gate (DEPLOY-008) over `effect_agent_meta` instead of
427
+ * Supported-predecessor or fresh storage gate (DEPLOY-008) over `effect_agent_meta` instead of
230
428
  * `PRAGMA user_version` (unverified on Durable Object SQL storage; a meta table is portable
231
429
  * regardless). No WAL check (Durable Object storage owns durability and confirms writes
232
430
  * through output gates) and no busy timeout (a Durable Object has exactly one writer): the
@@ -243,6 +441,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
243
441
  WHERE type = 'table'
244
442
  AND name = 'effect_agent_meta'
245
443
  `.pipe(Effect.mapError(storageError("read storage version table")));
444
+
246
445
  const metaTables = yield* decodeRows(
247
446
  Schema.Array(DoNameRow),
248
447
  "sqlite_master",
@@ -258,6 +457,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
258
457
  AND name LIKE 'effect_agent_%'
259
458
  ORDER BY name
260
459
  `.pipe(Effect.mapError(storageError("inspect unversioned storage")));
460
+
261
461
  const existing = yield* decodeRows(
262
462
  Schema.Array(DoNameRow),
263
463
  "sqlite_master",
@@ -270,11 +470,17 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
270
470
  actualVersion: 0,
271
471
  supportedVersion: CurrentDoStorageVersion,
272
472
  message:
273
- "The Durable Object contains unversioned Effect Agent tables. Reset the development namespace explicitly; refusing to mutate ambiguous stored data.",
473
+ "The Durable Object contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer.",
274
474
  });
275
475
  }
276
476
 
277
- yield* SqliteMigrator.run({ loader: doMigrations }).pipe(
477
+ yield* SqliteMigrator.run({
478
+ loader: doMigrations,
479
+ // An application can share this SQL client and own its own migration history.
480
+ // Keep bookkeeping outside effect_agent_% so interrupted unversioned schemas
481
+ // still fail the ambiguity check above.
482
+ table: "effect_sql_migrations_agent_threads",
483
+ }).pipe(
278
484
  // SqliteMigrator depends on the generic client supplied by this adapter. The concrete
279
485
  // Durable Object client is kept at the outer Layer boundary.
280
486
  Effect.provideService(SqlClient.SqlClient, sql),
@@ -292,6 +498,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
292
498
  FROM effect_agent_meta
293
499
  WHERE key = 'storage_version'
294
500
  `.pipe(Effect.mapError(storageError("read storage version")));
501
+
295
502
  const version = yield* decodeSingleRow(
296
503
  Schema.Array(DoMetaRow),
297
504
  "effect_agent_meta",
@@ -299,18 +506,136 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
299
506
  versionRows,
300
507
  );
301
508
 
302
- // The storage version must match EXACTLY. Older private-development versions fail
303
- // closed with reset guidance rather than being migrated, and newer versions fail closed
304
- // rather than being decoded incorrectly (DEPLOY-008).
305
- if (version.value !== String(CurrentDoStorageVersion)) {
509
+ if (
510
+ version.value === "2" ||
511
+ version.value === "3" ||
512
+ version.value === "4" ||
513
+ version.value === "5"
514
+ ) {
515
+ yield* sql
516
+ .withTransaction(
517
+ Effect.gen(function* () {
518
+ const current = yield* sql<{
519
+ value: string;
520
+ }>`SELECT value FROM effect_agent_meta WHERE key='storage_version'`;
521
+
522
+ if (current.length === 1 && current[0].value === String(CurrentDoStorageVersion))
523
+ return;
524
+ if (
525
+ current.length !== 1 ||
526
+ (current[0].value !== "2" &&
527
+ current[0].value !== "3" &&
528
+ current[0].value !== "4" &&
529
+ current[0].value !== "5")
530
+ )
531
+ return yield* DoStorageCompatibilityError.make({
532
+ actualVersion: -1,
533
+ supportedVersion: CurrentDoStorageVersion,
534
+ message: "Storage version changed while acquiring the upgrade transaction.",
535
+ });
536
+
537
+ const required = yield* decodeRows(
538
+ Schema.Array(DoNameRow),
539
+ "sqlite_master",
540
+ "required_tables",
541
+ yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name IN ${sql.in([...REQUIRED_TABLES])}`,
542
+ );
543
+
544
+ if (required.length !== REQUIRED_TABLES.length)
545
+ return yield* DoStorageCompatibilityError.make({
546
+ actualVersion: Number(current[0].value),
547
+ supportedVersion: CurrentDoStorageVersion,
548
+ message:
549
+ "The predecessor store is missing required tables. Retain the original store for inspection; no upgrade was committed.",
550
+ });
551
+
552
+ const recoveryTables =
553
+ yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name='effect_agent_recovery_checkpoints'`;
554
+
555
+ if (recoveryTables.length !== (current[0].value === "5" ? 1 : 0))
556
+ return yield* DoStorageCompatibilityError.make({
557
+ actualVersion: Number(current[0].value),
558
+ supportedVersion: CurrentDoStorageVersion,
559
+ message:
560
+ "The predecessor recovery checkpoint storage does not match its version; refusing ambiguous data without mutation.",
561
+ });
562
+
563
+ const indexes =
564
+ yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_submissions_nonterminal'`;
565
+
566
+ if (indexes.length !== 0)
567
+ return yield* DoStorageCompatibilityError.make({
568
+ actualVersion: Number(current[0].value),
569
+ supportedVersion: CurrentDoStorageVersion,
570
+ message:
571
+ "The predecessor already contains the nonterminal index; refusing ambiguous storage without mutation.",
572
+ });
573
+ if (current[0].value === "2") {
574
+ yield* checkV2ThreadLayout();
575
+ for (const statement of [
576
+ sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
577
+ sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
578
+ sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`,
579
+ ]) {
580
+ yield* failpoint("upgrade:before-mutation");
581
+ yield* statement;
582
+ yield* failpoint("upgrade:after-mutation");
583
+ }
584
+ }
585
+ if (current[0].value === "3") yield* checkPredecessorLayout(3);
586
+ if (current[0].value === "4") yield* checkPredecessorLayout(4);
587
+ if (current[0].value === "5") yield* checkPredecessorLayout(5);
588
+ if (current[0].value === "2" || current[0].value === "3") {
589
+ yield* failpoint("upgrade:before-mutation");
590
+ yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
591
+ yield* failpoint("upgrade:after-mutation");
592
+ yield* failpoint("upgrade:before-mutation");
593
+ yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
594
+ yield* failpoint("upgrade:after-mutation");
595
+ yield* failpoint("upgrade:before-mutation");
596
+ yield* createMessageDeliveryTables;
597
+ yield* failpoint("upgrade:after-mutation");
598
+ }
599
+ if (current[0].value !== "5") {
600
+ yield* failpoint("upgrade:before-mutation");
601
+ yield* createRecoveryCheckpointTable;
602
+ yield* failpoint("upgrade:after-mutation");
603
+ }
604
+ yield* failpoint("upgrade:before-mutation");
605
+ yield* createNonterminalIndex;
606
+ yield* failpoint("upgrade:after-mutation");
607
+ yield* failpoint("upgrade:before-version");
608
+ yield* sql`UPDATE effect_agent_meta SET value='6' WHERE key='storage_version'`;
609
+ yield* failpoint("upgrade:after-version");
610
+ }),
611
+ )
612
+ .pipe(
613
+ Effect.catchTag("DoStorageFailpointError", (error) =>
614
+ DoStorageError.make({
615
+ cause: error,
616
+ operation: "upgrade storage",
617
+ message: error.message,
618
+ }),
619
+ ),
620
+ Effect.catchTag("StorageUpgradeError", (error) =>
621
+ DoStorageCorruptionError.make({
622
+ table: error.table,
623
+ rowKey: error.rowKey,
624
+ message: error.message,
625
+ }),
626
+ ),
627
+ Effect.catchTag("SqlError", storageError("upgrade supported thread storage")),
628
+ );
629
+ } else if (version.value !== String(CurrentDoStorageVersion)) {
306
630
  const actualVersion = Number.parseInt(version.value, 10);
631
+
307
632
  return yield* DoStorageCompatibilityError.make({
308
633
  actualVersion: Number.isSafeInteger(actualVersion) ? actualVersion : -1,
309
634
  supportedVersion: CurrentDoStorageVersion,
310
635
  message:
311
- `The Durable Object uses private-development storage version ${version.value}; ` +
636
+ `The Durable Object uses unsupported storage version ${version.value}; ` +
312
637
  `this build supports exactly version ${CurrentDoStorageVersion}. ` +
313
- "Replace the development namespace explicitly; automatic stored-data migrations are not provided during private development.",
638
+ "Only supported v2, v3, v4 and v5 can be upgraded automatically. Keep the original store and use a compatible library version.",
314
639
  });
315
640
  }
316
641
  }
@@ -318,22 +643,25 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
318
643
  const requiredRows = yield* sql<Record<string, unknown>>`
319
644
  SELECT name
320
645
  FROM sqlite_master
321
- WHERE type = 'table'
322
- AND name IN ${sql.in([...REQUIRED_TABLES])}
646
+ WHERE (type = 'table'
647
+ AND name IN ${sql.in([...REQUIRED_TABLES, "effect_agent_message_deliveries", "effect_agent_recovery_checkpoints"])}
648
+ ) OR (type = 'index' AND name = 'effect_agent_submissions_nonterminal')
323
649
  ORDER BY name
324
650
  `.pipe(Effect.mapError(storageError("verify storage tables")));
651
+
325
652
  const required = yield* decodeRows(
326
653
  Schema.Array(DoNameRow),
327
654
  "sqlite_master",
328
655
  "required_tables",
329
656
  requiredRows,
330
657
  );
331
- if (required.length !== REQUIRED_TABLES.length) {
658
+
659
+ if (required.length !== REQUIRED_TABLES.length + 3) {
332
660
  return yield* DoStorageCompatibilityError.make({
333
661
  actualVersion: CurrentDoStorageVersion,
334
662
  supportedVersion: CurrentDoStorageVersion,
335
663
  message:
336
- "The Durable Object claims the current format but is missing required tables. Reset the corrupt private-development data.",
664
+ "The Durable Object claims the current format but is missing required tables or its nonterminal index. Retain the original store for inspection.",
337
665
  });
338
666
  }
339
667
 
@@ -351,6 +679,7 @@ const makeJournal = (
351
679
  value: string,
352
680
  ): Effect.Effect<void, DoValueBoundExceeded> => {
353
681
  const actualBytes = storedTextBytes(value);
682
+
354
683
  return actualBytes > maxStoredValueBytes
355
684
  ? Effect.fail(
356
685
  DoValueBoundExceeded.make({
@@ -378,14 +707,12 @@ const makeJournal = (
378
707
  (operation: string) =>
379
708
  <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<A, E | DoStorageError> =>
380
709
  sql.withTransaction(effect).pipe(
381
- Effect.mapError((error) =>
382
- error instanceof SqlError ? storageError(operation)(error) : error,
383
- ),
710
+ Effect.mapError((error) => (isSqlError(error) ? storageError(operation)(error) : error)),
384
711
  Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
385
712
  );
386
713
 
387
714
  const materialize = Effect.fn("DoJournal.materialize")(function* (
388
- conversationId: string,
715
+ threadId: string,
389
716
  createdAt: string,
390
717
  emptyTailDigest: string,
391
718
  producerEpoch: ProducerEpoch,
@@ -393,54 +720,57 @@ const makeJournal = (
393
720
  void,
394
721
  DoFenceRejected | DoStorageCorruptionError | DoStorageError | DoValueBoundExceeded
395
722
  > {
396
- if (conversationId.length > MAX_IDENTIFIER_LENGTH) {
723
+ if (threadId.length > MAX_IDENTIFIER_LENGTH) {
397
724
  return yield* DoStorageError.make({
398
- operation: "materialize conversation",
399
- message: "Conversation identity exceeds the Durable Object storage bounds.",
725
+ operation: "materialize thread",
726
+ message: "Thread identity exceeds the Durable Object storage bounds.",
400
727
  });
401
728
  }
402
- yield* checkValueBound("materialize conversation", emptyTailDigest);
729
+ yield* checkValueBound("materialize thread", emptyTailDigest);
403
730
  yield* withWriteTransaction("materialize transaction")(
404
731
  Effect.gen(function* () {
405
732
  const existingRows = yield* sql<Record<string, unknown>>`
406
733
  SELECT
407
- conversation_id,
734
+ thread_id,
408
735
  created_at,
409
736
  tail_sequence,
410
737
  tail_digest,
411
738
  producer_epoch
412
- FROM effect_agent_conversations
413
- WHERE conversation_id = ${conversationId}
414
- `.pipe(Effect.mapError(storageError("read materialized conversation")));
739
+ FROM effect_agent_threads
740
+ WHERE thread_id = ${threadId}
741
+ `.pipe(Effect.mapError(storageError("read materialized thread")));
742
+
415
743
  const existing = yield* decodeRows(
416
- Schema.Array(ConversationRow),
417
- "effect_agent_conversations",
418
- conversationId,
744
+ Schema.Array(ThreadRow),
745
+ "effect_agent_threads",
746
+ threadId,
419
747
  existingRows,
420
748
  );
749
+
421
750
  if (existing.length > 1) {
422
751
  return yield* DoStorageCorruptionError.make({
423
- table: "effect_agent_conversations",
424
- rowKey: conversationId,
425
- message: "A conversation primary key returned more than one row.",
752
+ table: "effect_agent_threads",
753
+ rowKey: threadId,
754
+ message: "A thread primary key returned more than one row.",
426
755
  });
427
756
  }
428
757
  if (existing.length === 0) {
429
758
  yield* sql`
430
- INSERT INTO effect_agent_conversations (
431
- conversation_id,
759
+ INSERT INTO effect_agent_threads (
760
+ thread_id,
432
761
  created_at,
433
762
  tail_sequence,
434
763
  tail_digest,
435
764
  producer_epoch
436
765
  ) VALUES (
437
- ${conversationId},
766
+ ${threadId},
438
767
  ${createdAt},
439
768
  0,
440
769
  ${emptyTailDigest},
441
770
  ${producerEpoch}
442
771
  )
443
- `.pipe(Effect.mapError(storageError("materialize conversation")));
772
+ `.pipe(Effect.mapError(storageError("materialize thread")));
773
+
444
774
  return;
445
775
  }
446
776
  if (producerEpoch < existing[0].producer_epoch) {
@@ -452,41 +782,35 @@ const makeJournal = (
452
782
  }
453
783
  if (producerEpoch > existing[0].producer_epoch) {
454
784
  yield* sql`
455
- UPDATE effect_agent_conversations
785
+ UPDATE effect_agent_threads
456
786
  SET producer_epoch = ${producerEpoch}
457
- WHERE conversation_id = ${conversationId}
787
+ WHERE thread_id = ${threadId}
458
788
  `.pipe(Effect.mapError(storageError("advance materialization epoch")));
459
789
  }
460
790
  }),
461
791
  );
462
792
  });
463
793
 
464
- const getConversation = Effect.fn("DoJournal.getConversation")(function* (
465
- conversationId: string,
466
- ) {
794
+ const getThread = Effect.fn("DoJournal.getThread")(function* (threadId: string) {
467
795
  const rows = yield* sql<Record<string, unknown>>`
468
796
  SELECT
469
- conversation_id,
797
+ thread_id,
470
798
  created_at,
471
799
  tail_sequence,
472
800
  tail_digest,
473
801
  producer_epoch
474
- FROM effect_agent_conversations
475
- WHERE conversation_id = ${conversationId}
476
- `.pipe(Effect.mapError(storageError("read conversation")));
477
- return yield* decodeRows(
478
- Schema.Array(ConversationRow),
479
- "effect_agent_conversations",
480
- conversationId,
481
- rows,
482
- );
802
+ FROM effect_agent_threads
803
+ WHERE thread_id = ${threadId}
804
+ `.pipe(Effect.mapError(storageError("read thread")));
805
+
806
+ return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
483
807
  });
484
808
 
485
809
  const append = Effect.fn("DoJournal.append")(function* (
486
810
  request: RawAppendRequest,
487
811
  ): Effect.fn.Return<RawAppendResult, AppendError> {
488
812
  if (
489
- request.conversationId.length > MAX_IDENTIFIER_LENGTH ||
813
+ request.threadId.length > MAX_IDENTIFIER_LENGTH ||
490
814
  request.batchId.length > MAX_IDENTIFIER_LENGTH ||
491
815
  request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
492
816
  ) {
@@ -504,9 +828,11 @@ const makeJournal = (
504
828
  (record) => checkValueBound("append canonical record", record.recordJson),
505
829
  { discard: true },
506
830
  );
831
+
507
832
  return yield* withWriteTransaction("append transaction")(
508
833
  Effect.gen(function* () {
509
834
  const recordIds = request.records.map((record) => record.recordId);
835
+
510
836
  if (new Set(recordIds).size !== recordIds.length) {
511
837
  return yield* DoAppendConflict.make({
512
838
  message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
@@ -514,34 +840,35 @@ const makeJournal = (
514
840
  });
515
841
  }
516
842
 
517
- const conversationRows = yield* sql<Record<string, unknown>>`
843
+ const threadRows = yield* sql<Record<string, unknown>>`
518
844
  SELECT
519
- conversation_id,
845
+ thread_id,
520
846
  created_at,
521
847
  tail_sequence,
522
848
  tail_digest,
523
849
  producer_epoch
524
- FROM effect_agent_conversations
525
- WHERE conversation_id = ${request.conversationId}
850
+ FROM effect_agent_threads
851
+ WHERE thread_id = ${request.threadId}
526
852
  `.pipe(Effect.mapError(storageError("read append tail")));
527
- const conversation = yield* decodeSingleRow(
528
- Schema.Array(ConversationRow),
529
- "effect_agent_conversations",
530
- request.conversationId,
531
- conversationRows,
853
+
854
+ const thread = yield* decodeSingleRow(
855
+ Schema.Array(ThreadRow),
856
+ "effect_agent_threads",
857
+ request.threadId,
858
+ threadRows,
532
859
  );
533
860
 
534
- if (request.producerEpoch !== conversation.producer_epoch) {
861
+ if (request.producerEpoch !== thread.producer_epoch) {
535
862
  return yield* DoFenceRejected.make({
536
863
  producerEpoch: request.producerEpoch,
537
- actualEpoch: conversation.producer_epoch,
538
- message: `Producer epoch ${request.producerEpoch} is not the current epoch ${conversation.producer_epoch}.`,
864
+ actualEpoch: thread.producer_epoch,
865
+ message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
539
866
  });
540
867
  }
541
868
 
542
869
  const batchRows = yield* sql<Record<string, unknown>>`
543
870
  SELECT
544
- conversation_id,
871
+ thread_id,
545
872
  batch_id,
546
873
  first_sequence,
547
874
  last_sequence,
@@ -549,31 +876,34 @@ const makeJournal = (
549
876
  tail_digest,
550
877
  batch_json
551
878
  FROM effect_agent_canonical_batches
552
- WHERE conversation_id = ${request.conversationId}
879
+ WHERE thread_id = ${request.threadId}
553
880
  AND batch_id = ${request.batchId}
554
881
  `.pipe(Effect.mapError(storageError("read idempotent batch")));
882
+
555
883
  const batches = yield* decodeRows(
556
884
  Schema.Array(BatchRow),
557
885
  "effect_agent_canonical_batches",
558
- `${request.conversationId}/${request.batchId}`,
886
+ `${request.threadId}/${request.batchId}`,
559
887
  batchRows,
560
888
  );
561
889
 
562
890
  if (batches.length > 1) {
563
891
  return yield* DoStorageCorruptionError.make({
564
892
  table: "effect_agent_canonical_batches",
565
- rowKey: `${request.conversationId}/${request.batchId}`,
893
+ rowKey: `${request.threadId}/${request.batchId}`,
566
894
  message: "A canonical batch primary key returned more than one row.",
567
895
  });
568
896
  }
569
897
  if (batches.length === 1) {
570
898
  const existing = batches[0];
899
+
571
900
  if (existing.batch_digest !== request.batchDigest) {
572
901
  return yield* DoAppendConflict.make({
573
902
  message: `Batch ${request.batchId} already exists with different canonical content.`,
574
903
  reason: "batch-digest",
575
904
  });
576
905
  }
906
+
577
907
  return RawAppendResult.make({
578
908
  firstSequence: existing.first_sequence,
579
909
  lastSequence: existing.last_sequence,
@@ -583,46 +913,48 @@ const makeJournal = (
583
913
  }
584
914
 
585
915
  if (
586
- request.expectedTailSequence !== conversation.tail_sequence ||
587
- request.expectedTailDigest !== conversation.tail_digest
916
+ request.expectedTailSequence !== thread.tail_sequence ||
917
+ request.expectedTailDigest !== thread.tail_digest
588
918
  ) {
589
919
  return yield* DoAppendConflict.make({
590
920
  message:
591
921
  `Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
592
- `but found ${conversation.tail_sequence}/${conversation.tail_digest}.`,
922
+ `but found ${thread.tail_sequence}/${thread.tail_digest}.`,
593
923
  reason: "tail",
594
- actualTailSequence: conversation.tail_sequence,
595
- actualTailDigest: conversation.tail_digest,
924
+ actualTailSequence: thread.tail_sequence,
925
+ actualTailDigest: thread.tail_digest,
596
926
  });
597
927
  }
598
- if (conversation.tail_sequence + request.records.length > MAX_RECORDS_PER_CONVERSATION) {
928
+ if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
599
929
  return yield* DoStorageError.make({
600
930
  operation: "append canonical batch",
601
- message: `Conversation record limit ${MAX_RECORDS_PER_CONVERSATION} would be exceeded.`,
931
+ message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
602
932
  });
603
933
  }
604
934
 
605
935
  // Chunked to respect the Durable Object platform's 100-bound-parameter statement
606
936
  // limit: a batch may carry up to 256 records.
607
937
  const existingRecords: Array<RecordRow> = [];
938
+
608
939
  for (const chunk of chunked(recordIds, MAX_BOUND_PARAMETERS - 10)) {
609
940
  const existingRecordRows = yield* sql<Record<string, unknown>>`
610
941
  SELECT
611
- conversation_id,
942
+ thread_id,
612
943
  sequence,
613
944
  record_id,
614
945
  batch_id,
615
946
  record_json
616
947
  FROM effect_agent_canonical_records
617
- WHERE conversation_id = ${request.conversationId}
948
+ WHERE thread_id = ${request.threadId}
618
949
  AND record_id IN ${sql.in([...chunk])}
619
950
  ORDER BY sequence
620
951
  `.pipe(Effect.mapError(storageError("check canonical record identities")));
952
+
621
953
  existingRecords.push(
622
954
  ...(yield* decodeRows(
623
955
  Schema.Array(RecordRow),
624
956
  "effect_agent_canonical_records",
625
- `${request.conversationId}/record_ids`,
957
+ `${request.threadId}/record_ids`,
626
958
  existingRecordRows,
627
959
  )),
628
960
  );
@@ -635,7 +967,7 @@ const makeJournal = (
635
967
  }
636
968
 
637
969
  const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
638
- conversation.tail_sequence + 1,
970
+ thread.tail_sequence + 1,
639
971
  ).pipe(
640
972
  Effect.mapError((error) =>
641
973
  DoStorageError.make({
@@ -645,6 +977,7 @@ const makeJournal = (
645
977
  }),
646
978
  ),
647
979
  );
980
+
648
981
  const lastSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
649
982
  firstSequence + request.records.length - 1,
650
983
  ).pipe(
@@ -659,7 +992,7 @@ const makeJournal = (
659
992
 
660
993
  yield* sql`
661
994
  INSERT INTO effect_agent_canonical_batches (
662
- conversation_id,
995
+ thread_id,
663
996
  batch_id,
664
997
  first_sequence,
665
998
  last_sequence,
@@ -667,7 +1000,7 @@ const makeJournal = (
667
1000
  tail_digest,
668
1001
  batch_json
669
1002
  ) VALUES (
670
- ${request.conversationId},
1003
+ ${request.threadId},
671
1004
  ${request.batchId},
672
1005
  ${firstSequence},
673
1006
  ${lastSequence},
@@ -684,13 +1017,13 @@ const makeJournal = (
684
1017
  Effect.gen(function* () {
685
1018
  yield* sql`
686
1019
  INSERT INTO effect_agent_canonical_records (
687
- conversation_id,
1020
+ thread_id,
688
1021
  sequence,
689
1022
  record_id,
690
1023
  batch_id,
691
1024
  record_json
692
1025
  ) VALUES (
693
- ${request.conversationId},
1026
+ ${request.threadId},
694
1027
  ${firstSequence + index},
695
1028
  ${record.recordId},
696
1029
  ${request.batchId},
@@ -703,13 +1036,13 @@ const makeJournal = (
703
1036
  );
704
1037
 
705
1038
  yield* sql`
706
- UPDATE effect_agent_conversations
1039
+ UPDATE effect_agent_threads
707
1040
  SET
708
1041
  tail_sequence = ${lastSequence},
709
1042
  tail_digest = ${request.tailDigest},
710
1043
  producer_epoch = ${request.producerEpoch}
711
- WHERE conversation_id = ${request.conversationId}
712
- `.pipe(Effect.mapError(storageError("advance conversation tail")));
1044
+ WHERE thread_id = ${request.threadId}
1045
+ `.pipe(Effect.mapError(storageError("advance thread tail")));
713
1046
  yield* failpoint("append:after-tail-update");
714
1047
 
715
1048
  return RawAppendResult.make({
@@ -723,106 +1056,165 @@ const makeJournal = (
723
1056
  });
724
1057
 
725
1058
  const read = Effect.fn("DoJournal.read")(function* (request: RawReadRequest) {
726
- const rows = yield* sql<Record<string, unknown>>`
1059
+ // Capture membership without retaining payloads. Append-only sequences keep each later
1060
+ // payload query inside this snapshot, even when new records arrive during consumption.
1061
+ const planRows = yield* sql<Record<string, unknown>>`
727
1062
  SELECT
728
- conversation_id,
729
1063
  sequence,
730
- record_id,
731
- batch_id,
732
- record_json
1064
+ length(CAST(record_json AS BLOB)) AS record_json_bytes
733
1065
  FROM effect_agent_canonical_records
734
- WHERE conversation_id = ${request.conversationId}
1066
+ WHERE thread_id = ${request.threadId}
735
1067
  AND sequence > ${request.fromSequenceExclusive}
736
1068
  ORDER BY sequence
737
1069
  LIMIT ${request.limit}
738
1070
  `.pipe(Effect.mapError(storageError("read canonical records")));
739
- return yield* decodeRows(
740
- Schema.Array(RecordRow),
1071
+
1072
+ const plan = yield* decodeRows(
1073
+ Schema.Array(ReadPlanRow),
741
1074
  "effect_agent_canonical_records",
742
- `${request.conversationId}>${request.fromSequenceExclusive}`,
743
- rows,
1075
+ `${request.threadId}>${request.fromSequenceExclusive}`,
1076
+ planRows,
744
1077
  );
1078
+
1079
+ const mismatch = () =>
1080
+ DoStorageCorruptionError.make({
1081
+ table: "effect_agent_canonical_records",
1082
+ rowKey: `${request.threadId}>${request.fromSequenceExclusive}`,
1083
+ message: "Canonical read membership, sequence, or payload size changed from its read plan.",
1084
+ });
1085
+
1086
+ if (plan.length > request.limit) return yield* mismatch();
1087
+
1088
+ const pages: Array<ReadPage> = [];
1089
+ let page: ReadPage | undefined;
1090
+ let pageBytes = 0;
1091
+ let previousSequence = request.fromSequenceExclusive;
1092
+
1093
+ for (const row of plan) {
1094
+ if (row.sequence !== previousSequence + 1) return yield* mismatch();
1095
+ previousSequence = row.sequence;
1096
+ if (page === undefined || pageBytes + row.record_json_bytes > MAX_READ_PAGE_JSON_BYTES) {
1097
+ page = [row];
1098
+ pages.push(page);
1099
+ pageBytes = row.record_json_bytes;
1100
+ } else {
1101
+ page.push(row);
1102
+ pageBytes += row.record_json_bytes;
1103
+ }
1104
+ }
1105
+
1106
+ const readPage = Effect.fn("DoJournal.readPage")(function* (page: ReadPage) {
1107
+ const rows = yield* sql<Record<string, unknown>>`
1108
+ SELECT thread_id, sequence, record_id, batch_id, record_json
1109
+ FROM effect_agent_canonical_records
1110
+ WHERE thread_id = ${request.threadId}
1111
+ AND sequence >= ${page[0].sequence}
1112
+ AND sequence <= ${page[page.length - 1].sequence}
1113
+ ORDER BY sequence
1114
+ `.pipe(Effect.mapError(storageError("read canonical records")));
1115
+
1116
+ const decoded = yield* decodeRows(
1117
+ Schema.Array(RecordRow),
1118
+ "effect_agent_canonical_records",
1119
+ `${request.threadId}/${page[0].sequence}`,
1120
+ rows,
1121
+ );
1122
+
1123
+ if (
1124
+ decoded.length !== page.length ||
1125
+ decoded.some(
1126
+ (row, index) =>
1127
+ row.thread_id !== request.threadId ||
1128
+ row.sequence !== page[index].sequence ||
1129
+ storedTextBytes(row.record_json) !== page[index].record_json_bytes,
1130
+ )
1131
+ ) {
1132
+ return yield* mismatch();
1133
+ }
1134
+
1135
+ return decoded;
1136
+ });
1137
+
1138
+ return {
1139
+ count: plan.length,
1140
+ records: Stream.fromIterable(pages).pipe(
1141
+ Stream.flatMap((page) => Stream.fromIterableEffect(readPage(page))),
1142
+ ),
1143
+ };
745
1144
  });
746
1145
 
747
- const exportConversation = Effect.fn("DoJournal.exportConversation")(function* (
748
- conversationId: string,
749
- ) {
1146
+ const exportThread = Effect.fn("DoJournal.exportThread")(function* (threadId: string) {
750
1147
  return yield* sql
751
1148
  .withTransaction(
752
1149
  Effect.gen(function* () {
753
- const conversationRows = yield* sql<Record<string, unknown>>`
1150
+ const threadRows = yield* sql<Record<string, unknown>>`
754
1151
  SELECT
755
- conversation_id,
1152
+ thread_id,
756
1153
  created_at,
757
1154
  tail_sequence,
758
1155
  tail_digest,
759
1156
  producer_epoch
760
- FROM effect_agent_conversations
761
- WHERE conversation_id = ${conversationId}
762
- `.pipe(Effect.mapError(storageError("export conversation")));
763
- const conversation = yield* decodeSingleRow(
764
- Schema.Array(ConversationRow),
765
- "effect_agent_conversations",
766
- conversationId,
767
- conversationRows,
1157
+ FROM effect_agent_threads
1158
+ WHERE thread_id = ${threadId}
1159
+ `.pipe(Effect.mapError(storageError("export thread")));
1160
+
1161
+ const thread = yield* decodeSingleRow(
1162
+ Schema.Array(ThreadRow),
1163
+ "effect_agent_threads",
1164
+ threadId,
1165
+ threadRows,
768
1166
  );
769
- yield* failpoint("export:after-conversation-read");
770
- const batchRows = yield* sql<Record<string, unknown>>`
771
- SELECT
772
- conversation_id,
773
- batch_id,
774
- first_sequence,
775
- last_sequence,
776
- batch_digest,
777
- tail_digest,
778
- batch_json
779
- FROM effect_agent_canonical_batches
780
- WHERE conversation_id = ${conversationId}
781
- ORDER BY first_sequence
782
- `.pipe(Effect.mapError(storageError("export canonical batches")));
783
- const recordRows = yield* sql<Record<string, unknown>>`
784
- SELECT
785
- conversation_id,
786
- sequence,
787
- record_id,
788
- batch_id,
789
- record_json
790
- FROM effect_agent_canonical_records
791
- WHERE conversation_id = ${conversationId}
792
- ORDER BY sequence
793
- `.pipe(Effect.mapError(storageError("export canonical records")));
794
- const checkpointRows = yield* sql<Record<string, unknown>>`
795
- SELECT
796
- conversation_id,
797
- through_sequence,
798
- tail_digest,
799
- checkpoint_json
800
- FROM effect_agent_checkpoints
801
- WHERE conversation_id = ${conversationId}
802
- ORDER BY through_sequence
803
- `.pipe(Effect.mapError(storageError("export checkpoints")));
804
1167
 
805
- return RawConversationExport.make({
806
- conversation,
807
- batches: yield* decodeRows(
808
- Schema.Array(BatchRow),
809
- "effect_agent_canonical_batches",
810
- conversationId,
811
- batchRows,
812
- ),
813
- records: yield* decodeRows(
814
- Schema.Array(RecordRow),
815
- "effect_agent_canonical_records",
816
- conversationId,
817
- recordRows,
818
- ),
819
- checkpoints: yield* decodeRows(
820
- Schema.Array(CheckpointRow),
821
- "effect_agent_checkpoints",
822
- conversationId,
823
- checkpointRows,
824
- ),
825
- });
1168
+ yield* failpoint("export:after-thread-read");
1169
+
1170
+ if (thread.tail_sequence > MAX_RECORDS_PER_THREAD)
1171
+ return yield* DoStorageError.make({
1172
+ operation: "export thread",
1173
+ message: "The thread exceeds the current export record limit.",
1174
+ });
1175
+ const records: Array<RecordRow> = [];
1176
+ let afterSequence = ZERO_SEQUENCE;
1177
+
1178
+ while (afterSequence < thread.tail_sequence) {
1179
+ const limit = Math.min(1_024, thread.tail_sequence - afterSequence);
1180
+
1181
+ const request = RawReadRequest.make({
1182
+ threadId,
1183
+ fromSequenceExclusive: afterSequence,
1184
+ limit,
1185
+ });
1186
+
1187
+ const plan = yield* read(request);
1188
+ const page = yield* Stream.runCollect(plan.records);
1189
+
1190
+ if (
1191
+ page.length !== limit ||
1192
+ page.some((record, index) => record.sequence !== afterSequence + index + 1)
1193
+ ) {
1194
+ return yield* DoStorageCorruptionError.make({
1195
+ table: "effect_agent_canonical_records",
1196
+ rowKey: threadId,
1197
+ message:
1198
+ "The exported canonical prefix is not contiguous through its captured tail.",
1199
+ });
1200
+ }
1201
+ records.push(...page);
1202
+ afterSequence = page[page.length - 1].sequence;
1203
+ }
1204
+
1205
+ const beyondTail =
1206
+ yield* sql`SELECT sequence FROM effect_agent_canonical_records WHERE thread_id=${threadId} AND sequence > ${thread.tail_sequence} LIMIT 1`.pipe(
1207
+ Effect.mapError(storageError("verify export tail")),
1208
+ );
1209
+
1210
+ if (beyondTail.length !== 0)
1211
+ return yield* DoStorageCorruptionError.make({
1212
+ table: "effect_agent_canonical_records",
1213
+ rowKey: threadId,
1214
+ message: "Canonical records exist beyond the captured thread tail.",
1215
+ });
1216
+
1217
+ return RawThreadExport.make({ thread, records });
826
1218
  }),
827
1219
  )
828
1220
  .pipe(
@@ -835,7 +1227,7 @@ const makeJournal = (
835
1227
  const saveCheckpoint = Effect.fn("DoJournal.saveCheckpoint")(function* (
836
1228
  checkpoint: RawCheckpoint,
837
1229
  ): Effect.fn.Return<void, CheckpointError> {
838
- if (checkpoint.conversationId.length > MAX_IDENTIFIER_LENGTH) {
1230
+ if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
839
1231
  return yield* DoStorageError.make({
840
1232
  operation: "save checkpoint",
841
1233
  message: "Checkpoint identity exceeds the Durable Object storage bounds.",
@@ -844,50 +1236,54 @@ const makeJournal = (
844
1236
  yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
845
1237
  yield* withWriteTransaction("checkpoint transaction")(
846
1238
  Effect.gen(function* () {
847
- const conversationRows = yield* sql<Record<string, unknown>>`
1239
+ const threadRows = yield* sql<Record<string, unknown>>`
848
1240
  SELECT
849
- conversation_id,
1241
+ thread_id,
850
1242
  created_at,
851
1243
  tail_sequence,
852
1244
  tail_digest,
853
1245
  producer_epoch
854
- FROM effect_agent_conversations
855
- WHERE conversation_id = ${checkpoint.conversationId}
1246
+ FROM effect_agent_threads
1247
+ WHERE thread_id = ${checkpoint.threadId}
856
1248
  `.pipe(Effect.mapError(storageError("read checkpoint tail")));
857
- const conversation = yield* decodeSingleRow(
858
- Schema.Array(ConversationRow),
859
- "effect_agent_conversations",
860
- checkpoint.conversationId,
861
- conversationRows,
1249
+
1250
+ const thread = yield* decodeSingleRow(
1251
+ Schema.Array(ThreadRow),
1252
+ "effect_agent_threads",
1253
+ checkpoint.threadId,
1254
+ threadRows,
862
1255
  );
863
- if (checkpoint.throughSequence > conversation.tail_sequence) {
1256
+
1257
+ if (checkpoint.throughSequence > thread.tail_sequence) {
864
1258
  return yield* DoCheckpointConflict.make({
865
1259
  message:
866
1260
  `Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
867
- `${conversation.tail_sequence}.`,
1261
+ `${thread.tail_sequence}.`,
868
1262
  });
869
1263
  }
870
1264
 
871
1265
  const checkpointRows = yield* sql<Record<string, unknown>>`
872
1266
  SELECT
873
- conversation_id,
1267
+ thread_id,
874
1268
  through_sequence,
875
1269
  tail_digest,
876
1270
  checkpoint_json
877
1271
  FROM effect_agent_checkpoints
878
- WHERE conversation_id = ${checkpoint.conversationId}
1272
+ WHERE thread_id = ${checkpoint.threadId}
879
1273
  AND through_sequence = ${checkpoint.throughSequence}
880
1274
  `.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
1275
+
881
1276
  const existing = yield* decodeRows(
882
1277
  Schema.Array(CheckpointRow),
883
1278
  "effect_agent_checkpoints",
884
- `${checkpoint.conversationId}/${checkpoint.throughSequence}`,
1279
+ `${checkpoint.threadId}/${checkpoint.throughSequence}`,
885
1280
  checkpointRows,
886
1281
  );
1282
+
887
1283
  if (existing.length > 1) {
888
1284
  return yield* DoStorageCorruptionError.make({
889
1285
  table: "effect_agent_checkpoints",
890
- rowKey: `${checkpoint.conversationId}/${checkpoint.throughSequence}`,
1286
+ rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
891
1287
  message: "A checkpoint primary key returned more than one row.",
892
1288
  });
893
1289
  }
@@ -900,17 +1296,18 @@ const makeJournal = (
900
1296
  message: "A different checkpoint already exists at this canonical sequence.",
901
1297
  });
902
1298
  }
1299
+
903
1300
  return;
904
1301
  }
905
1302
 
906
1303
  yield* sql`
907
1304
  INSERT INTO effect_agent_checkpoints (
908
- conversation_id,
1305
+ thread_id,
909
1306
  through_sequence,
910
1307
  tail_digest,
911
1308
  checkpoint_json
912
1309
  ) VALUES (
913
- ${checkpoint.conversationId},
1310
+ ${checkpoint.threadId},
914
1311
  ${checkpoint.throughSequence},
915
1312
  ${checkpoint.tailDigest},
916
1313
  ${checkpoint.checkpointJson}
@@ -920,45 +1317,124 @@ const makeJournal = (
920
1317
  );
921
1318
  });
922
1319
 
1320
+ const saveRecoveryCheckpoint = Effect.fn("DoJournal.saveRecoveryCheckpoint")(function* (
1321
+ request: SaveRecoveryCheckpointRequest,
1322
+ checkpointJson: string,
1323
+ ) {
1324
+ const { checkpoint } = request;
1325
+
1326
+ if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
1327
+ return yield* DoStorageError.make({
1328
+ operation: "save recovery checkpoint",
1329
+ message: "Checkpoint identity exceeds the Durable Object storage bounds.",
1330
+ });
1331
+ }
1332
+ yield* checkValueBound("save recovery checkpoint", checkpointJson);
1333
+ // Keep injected waits outside the storage-backed transaction callback.
1334
+ yield* failpoint("save-recovery-checkpoint:before");
1335
+ yield* withWriteTransaction("recovery checkpoint transaction")(
1336
+ Effect.gen(function* () {
1337
+ const threads = yield* getThread(checkpoint.threadId);
1338
+ const thread = threads[0];
1339
+
1340
+ if (thread === undefined)
1341
+ return yield* ThreadNotMaterialized.make({ threadId: checkpoint.threadId });
1342
+ if (request.producerEpoch !== thread.producer_epoch)
1343
+ return yield* FenceRejected.make({
1344
+ threadId: checkpoint.threadId,
1345
+ actualEpoch: thread.producer_epoch,
1346
+ attemptedEpoch: request.producerEpoch,
1347
+ });
1348
+ if (checkpoint.throughSequence > thread.tail_sequence)
1349
+ return yield* CheckpointRejected.make({
1350
+ threadId: checkpoint.threadId,
1351
+ reason: "ahead-of-tail",
1352
+ });
1353
+
1354
+ const digests =
1355
+ checkpoint.throughSequence === 0
1356
+ ? [EMPTY_TAIL_DIGEST]
1357
+ : yield* getTailDigestAt(checkpoint.threadId, checkpoint.throughSequence);
1358
+
1359
+ if (digests.length !== 1 || digests[0] !== checkpoint.tailDigest)
1360
+ return yield* CheckpointRejected.make({
1361
+ threadId: checkpoint.threadId,
1362
+ reason: "digest-mismatch",
1363
+ });
1364
+
1365
+ yield* sql`
1366
+ INSERT INTO effect_agent_recovery_checkpoints (thread_id, through_sequence, tail_digest, checkpoint_json)
1367
+ VALUES (${checkpoint.threadId}, ${checkpoint.throughSequence}, ${checkpoint.tailDigest}, ${checkpointJson})
1368
+ ON CONFLICT (thread_id) DO UPDATE SET
1369
+ through_sequence = excluded.through_sequence,
1370
+ tail_digest = excluded.tail_digest,
1371
+ checkpoint_json = excluded.checkpoint_json
1372
+ WHERE excluded.through_sequence >= effect_agent_recovery_checkpoints.through_sequence
1373
+ `.pipe(Effect.mapError(storageError("save recovery checkpoint")));
1374
+ }),
1375
+ );
1376
+ yield* failpoint("save-recovery-checkpoint:after");
1377
+ });
1378
+
1379
+ const loadRecoveryCheckpoint = Effect.fn("DoJournal.loadRecoveryCheckpoint")(function* (
1380
+ threadId: string,
1381
+ ) {
1382
+ const rows = yield* sql<Record<string, unknown>>`
1383
+ SELECT thread_id, through_sequence, tail_digest, checkpoint_json
1384
+ FROM effect_agent_recovery_checkpoints
1385
+ WHERE thread_id = ${threadId}
1386
+ `.pipe(Effect.mapError(storageError("load recovery checkpoint")));
1387
+
1388
+ return yield* decodeRows(
1389
+ Schema.Array(CheckpointRow),
1390
+ "effect_agent_recovery_checkpoints",
1391
+ threadId,
1392
+ rows,
1393
+ );
1394
+ });
1395
+
923
1396
  const loadCheckpoint = Effect.fn("DoJournal.loadCheckpoint")(function* (
924
- conversationId: string,
1397
+ threadId: string,
925
1398
  atOrBeforeSequence: CanonicalSequence,
926
1399
  ) {
927
1400
  const rows = yield* sql<Record<string, unknown>>`
928
1401
  SELECT
929
- conversation_id,
1402
+ thread_id,
930
1403
  through_sequence,
931
1404
  tail_digest,
932
1405
  checkpoint_json
933
1406
  FROM effect_agent_checkpoints
934
- WHERE conversation_id = ${conversationId}
1407
+ WHERE thread_id = ${threadId}
935
1408
  AND through_sequence <= ${atOrBeforeSequence}
936
1409
  ORDER BY through_sequence DESC
937
1410
  LIMIT 1
938
1411
  `.pipe(Effect.mapError(storageError("load checkpoint")));
1412
+
939
1413
  return yield* decodeRows(
940
1414
  Schema.Array(CheckpointRow),
941
1415
  "effect_agent_checkpoints",
942
- `${conversationId}<=${atOrBeforeSequence}`,
1416
+ `${threadId}<=${atOrBeforeSequence}`,
943
1417
  rows,
944
1418
  );
945
1419
  });
946
1420
 
947
1421
  const getTailDigestAt = Effect.fn("DoJournal.getTailDigestAt")(function* (
948
- conversationId: string,
1422
+ threadId: string,
949
1423
  sequence: CanonicalSequence,
950
1424
  ) {
951
1425
  if (sequence === 0) {
952
- const conversations = yield* getConversation(conversationId);
953
- return conversations.length === 0
1426
+ const threads = yield* getThread(threadId);
1427
+
1428
+ return threads.length === 0
954
1429
  ? []
955
- : [conversations[0].tail_sequence === 0 ? conversations[0].tail_digest : undefined].filter(
1430
+ : [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
956
1431
  (value): value is string => value !== undefined,
957
1432
  );
958
1433
  }
1434
+
959
1435
  const rows = yield* sql<Record<string, unknown>>`
960
1436
  SELECT
961
- conversation_id,
1437
+ thread_id,
962
1438
  batch_id,
963
1439
  first_sequence,
964
1440
  last_sequence,
@@ -966,15 +1442,17 @@ const makeJournal = (
966
1442
  tail_digest,
967
1443
  batch_json
968
1444
  FROM effect_agent_canonical_batches
969
- WHERE conversation_id = ${conversationId}
1445
+ WHERE thread_id = ${threadId}
970
1446
  AND last_sequence = ${sequence}
971
1447
  `.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
1448
+
972
1449
  const batches = yield* decodeRows(
973
1450
  Schema.Array(BatchRow),
974
1451
  "effect_agent_canonical_batches",
975
- `${conversationId}/${sequence}`,
1452
+ `${threadId}/${sequence}`,
976
1453
  rows,
977
1454
  );
1455
+
978
1456
  return batches.map((batch) => batch.tail_digest);
979
1457
  });
980
1458
 
@@ -982,19 +1460,20 @@ const makeJournal = (
982
1460
  return yield* sql
983
1461
  .withTransaction(
984
1462
  Effect.gen(function* () {
985
- const conversations = yield* sql<Record<string, unknown>>`
1463
+ const threads = yield* sql<Record<string, unknown>>`
986
1464
  SELECT
987
- conversation_id,
1465
+ thread_id,
988
1466
  created_at,
989
1467
  tail_sequence,
990
1468
  tail_digest,
991
1469
  producer_epoch
992
- FROM effect_agent_conversations
993
- ORDER BY conversation_id
994
- `.pipe(Effect.mapError(storageError("scan conversations")));
1470
+ FROM effect_agent_threads
1471
+ ORDER BY thread_id
1472
+ `.pipe(Effect.mapError(storageError("scan threads")));
1473
+
995
1474
  const batches = yield* sql<Record<string, unknown>>`
996
1475
  SELECT
997
- conversation_id,
1476
+ thread_id,
998
1477
  batch_id,
999
1478
  first_sequence,
1000
1479
  last_sequence,
@@ -1002,33 +1481,36 @@ const makeJournal = (
1002
1481
  tail_digest,
1003
1482
  batch_json
1004
1483
  FROM effect_agent_canonical_batches
1005
- ORDER BY conversation_id, first_sequence
1484
+ ORDER BY thread_id, first_sequence
1006
1485
  `.pipe(Effect.mapError(storageError("scan canonical batches")));
1486
+
1007
1487
  const records = yield* sql<Record<string, unknown>>`
1008
1488
  SELECT
1009
- conversation_id,
1489
+ thread_id,
1010
1490
  sequence,
1011
1491
  record_id,
1012
1492
  batch_id,
1013
1493
  record_json
1014
1494
  FROM effect_agent_canonical_records
1015
- ORDER BY conversation_id, sequence
1495
+ ORDER BY thread_id, sequence
1016
1496
  `.pipe(Effect.mapError(storageError("scan canonical records")));
1497
+
1017
1498
  const checkpoints = yield* sql<Record<string, unknown>>`
1018
1499
  SELECT
1019
- conversation_id,
1500
+ thread_id,
1020
1501
  through_sequence,
1021
1502
  tail_digest,
1022
1503
  checkpoint_json
1023
1504
  FROM effect_agent_checkpoints
1024
- ORDER BY conversation_id, through_sequence
1505
+ ORDER BY thread_id, through_sequence
1025
1506
  `.pipe(Effect.mapError(storageError("scan checkpoints")));
1507
+
1026
1508
  return {
1027
- conversations: yield* decodeRows(
1028
- Schema.Array(ConversationRow),
1029
- "effect_agent_conversations",
1509
+ threads: yield* decodeRows(
1510
+ Schema.Array(ThreadRow),
1511
+ "effect_agent_threads",
1030
1512
  "startup_scan",
1031
- conversations,
1513
+ threads,
1032
1514
  ),
1033
1515
  batches: yield* decodeRows(
1034
1516
  Schema.Array(BatchRow),
@@ -1061,10 +1543,12 @@ const makeJournal = (
1061
1543
  return {
1062
1544
  append,
1063
1545
  checkValueBound,
1064
- exportConversation,
1065
- getConversation,
1546
+ exportThread,
1547
+ getThread,
1066
1548
  getTailDigestAt,
1067
1549
  loadCheckpoint,
1550
+ loadRecoveryCheckpoint,
1551
+ saveRecoveryCheckpoint,
1068
1552
  materialize,
1069
1553
  read,
1070
1554
  saveCheckpoint,