@effect-agent/storage-cloudflare 0.1.0-beta.6 → 0.1.0-beta.60

package/src/{do-journal.ts → internal/do-journal.ts}

@@ -1,20 +1,22 @@
-import { CanonicalSequence, ProducerEpoch } from "@effect-agent/session";
+import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread/Records";
+import { checkV2ThreadLayout } from "@effect-agent/thread/SqlStorageV2Upgrade";
 import { SqliteMigrator } from "@effect/sql-sqlite-do";
-import { Effect, Schema } from "effect";
+import { Effect, Schema, Stream } from "effect";
 import * as SqlClient from "effect/unstable/sql/SqlClient";
 import { SqlError } from "effect/unstable/sql/SqlError";
 
 import {
+  type DoStorageFailpointError,
   DoAppendConflict,
   DoCheckpointConflict,
   DoFenceRejected,
   DoStorageCompatibilityError,
   DoStorageCorruptionError,
   DoStorageError,
-  DoStorageFailpointError,
   DoValueBoundExceeded,
   type DoStorageFailpointLocation,
-} from "./errors.ts";
+} from "../DoStorageError.ts";
+import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
 import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
 
 /**
@@ -25,18 +27,22 @@ import { CurrentDoStorageVersion, doMigrations } from "./migrations.ts";
  */
 const BoundedStoredText = Schema.String.check(Schema.isMaxLength(2_000_000));
 const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
-const MAX_RECORDS_PER_CONVERSATION = 65_536;
+const MAX_RECORDS_PER_THREAD = 65_536;
 const MAX_IDENTIFIER_LENGTH = 1_024;
+const MAX_READ_PAGE_JSON_BYTES = 4 * 1024 * 1024;
 /** Durable Object SQL storage allows at most 100 bound parameters per statement. */
 const MAX_BOUND_PARAMETERS = 100;
+const isSqlError = Schema.is(SqlError);
 
 const storedTextBytes = (value: string): number => new TextEncoder().encode(value).byteLength;
 
 const chunked = <A>(values: ReadonlyArray<A>, size: number): Array<ReadonlyArray<A>> => {
   const chunks: Array<ReadonlyArray<A>> = [];
+
   for (let index = 0; index < values.length; index += size) {
     chunks.push(values.slice(index, index + size));
   }
+
   return chunks;
 };
 
@@ -48,8 +54,8 @@ class DoNameRow extends Schema.Class<DoNameRow>("DoNameRow")({
   name: BoundedIdentifier,
 }) {}
 
-class ConversationRow extends Schema.Class<ConversationRow>("ConversationRow")({
-  conversation_id: BoundedIdentifier,
+class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
+  thread_id: BoundedIdentifier,
   created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
   producer_epoch: ProducerEpoch,
   tail_digest: BoundedStoredText,
@@ -60,7 +66,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
   batch_digest: BoundedStoredText,
   batch_id: BoundedIdentifier,
   batch_json: BoundedStoredText,
-  conversation_id: BoundedIdentifier,
+  thread_id: BoundedIdentifier,
   first_sequence: CanonicalSequence,
   last_sequence: CanonicalSequence,
   tail_digest: BoundedStoredText,
@@ -68,15 +74,22 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
 
 class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
   batch_id: BoundedIdentifier,
-  conversation_id: BoundedIdentifier,
+  thread_id: BoundedIdentifier,
   record_id: BoundedIdentifier,
   record_json: BoundedStoredText,
   sequence: CanonicalSequence,
 }) {}
 
+const ReadPlanRow = Schema.Struct({
+  sequence: CanonicalSequence,
+  record_json_bytes: Schema.Natural.check(Schema.isLessThanOrEqualTo(MAX_READ_PAGE_JSON_BYTES)),
+});
+
+type ReadPage = [typeof ReadPlanRow.Type, ...Array<typeof ReadPlanRow.Type>];
+
 class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
   checkpoint_json: BoundedStoredText,
-  conversation_id: BoundedIdentifier,
+  thread_id: BoundedIdentifier,
   tail_digest: BoundedStoredText,
   through_sequence: CanonicalSequence,
 }) {}
@@ -94,7 +107,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
   batchDigest: BoundedStoredText,
   batchId: BoundedIdentifier,
   batchJson: BoundedStoredText,
-  conversationId: BoundedIdentifier,
+  threadId: BoundedIdentifier,
   expectedTailDigest: BoundedStoredText,
   expectedTailSequence: CanonicalSequence,
   producerEpoch: ProducerEpoch,
@@ -114,7 +127,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
 export class RawReadRequest extends Schema.Class<RawReadRequest>(
   "@effect-agent/storage-cloudflare/RawReadRequest",
 )({
-  conversationId: BoundedIdentifier,
+  threadId: BoundedIdentifier,
   fromSequenceExclusive: CanonicalSequence,
   limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
 }) {}
@@ -123,17 +136,15 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
   "@effect-agent/storage-cloudflare/RawCheckpoint",
 )({
   checkpointJson: BoundedStoredText,
-  conversationId: BoundedIdentifier,
+  threadId: BoundedIdentifier,
   tailDigest: BoundedStoredText,
   throughSequence: CanonicalSequence,
 }) {}
 
-export class RawConversationExport extends Schema.Class<RawConversationExport>(
-  "@effect-agent/storage-cloudflare/RawConversationExport",
+export class RawThreadExport extends Schema.Class<RawThreadExport>(
+  "@effect-agent/storage-cloudflare/RawThreadExport",
 )({
-  batches: Schema.Array(BatchRow),
-  checkpoints: Schema.Array(CheckpointRow),
-  conversation: ConversationRow,
+  thread: ThreadRow,
   records: Schema.Array(RecordRow),
 }) {}
 
@@ -167,7 +178,7 @@ const storageError =
     });
 
 /** Decode raw Durable Object SQLite rows against a Schema, reporting failures as typed corruption. */
-export const decodeRows = Effect.fn("DoJournal.decodeRows")(
+export const decodeRows = Effect.fn(
   <A, I>(
     schema: Schema.Codec<ReadonlyArray<A>, ReadonlyArray<I>>,
     table: string,
@@ -186,7 +197,7 @@ export const decodeRows = Effect.fn("DoJournal.decodeRows")(
 );
 
 /** Decode exactly one raw row against a Schema, reporting failures as typed corruption. */
-export const decodeSingleRow = Effect.fn("DoJournal.decodeSingleRow")(
+export const decodeSingleRow = Effect.fn(
   <A, I>(
     schema: Schema.Codec<ReadonlyArray<A>, ReadonlyArray<I>>,
     table: string,
@@ -208,6 +219,147 @@ export const decodeSingleRow = Effect.fn("DoJournal.decodeSingleRow")(
     ),
 );
 
+/** Column inventory of the supported v3 predecessor, independent of physical column order. */
+const predecessorColumns = {
+  effect_agent_threads: [
+    "thread_id",
+    "created_at",
+    "tail_sequence",
+    "tail_digest",
+    "producer_epoch",
+  ],
+  effect_agent_canonical_batches: [
+    "thread_id",
+    "batch_id",
+    "first_sequence",
+    "last_sequence",
+    "batch_digest",
+    "tail_digest",
+    "batch_json",
+  ],
+  effect_agent_canonical_records: ["thread_id", "sequence", "record_id", "batch_id", "record_json"],
+  effect_agent_checkpoints: ["thread_id", "through_sequence", "tail_digest", "checkpoint_json"],
+  effect_agent_submissions: [
+    "submission_id",
+    "thread_id",
+    "queue_sequence",
+    "principal",
+    "idempotency_key",
+    "agent_id",
+    "agent_digests_json",
+    "deployment_id",
+    "input_json",
+    "input_digest",
+    "receipt_id",
+    "state",
+    "settled_outcome",
+    "created_at",
+    "ready_at",
+    "input_applied_record_id",
+    "input_applied_sequence",
+    "joined_host_submission_id",
+    "suspended_reason_json",
+    "suspended_at",
+    "unknown_reason",
+    "unknown_tool_call_ids_json",
+    "parent_submission_id",
+    "parent_tool_call_id",
+    "admission_group",
+    "admission_fence_json",
+  ],
+  effect_agent_submission_ownership: [
+    "submission_id",
+    "attempt_id",
+    "ownership_token",
+    "producer_epoch",
+    "owner_producer_id",
+    "lease_expires_at",
+  ],
+  effect_agent_attempts: [
+    "attempt_id",
+    "submission_id",
+    "thread_id",
+    "owner_producer_id",
+    "producer_epoch",
+    "claimed_at",
+  ],
+  effect_agent_settlement_reservations: [
+    "submission_id",
+    "settlement_id",
+    "outcome",
+    "record_id",
+    "record_json",
+    "record_digest",
+    "reserved_at",
+    "finalized_at",
+  ],
+  effect_agent_abort_intents: [
+    "submission_id",
+    "author",
+    "reason",
+    "requested_at",
+    "canonical_record_id",
+  ],
+  effect_agent_approval_decisions: [
+    "submission_id",
+    "tool_call_id",
+    "decision",
+    "resolver",
+    "reason",
+    "decided_at",
+  ],
+  effect_agent_unknown_resolutions: [
+    "submission_id",
+    "tool_call_id",
+    "author",
+    "reason",
+    "resolution_json",
+    "resolved_at",
+  ],
+  effect_agent_child_reservations: [
+    "reservation_id",
+    "parent_submission_id",
+    "parent_tool_call_id",
+    "child_submission_id",
+    "status",
+    "allocation_json",
+    "allocation_digest",
+    "accounting_json",
+    "reserved_at",
+    "release_began_at",
+    "released_at",
+  ],
+  effect_agent_meta: ["key", "value"],
+  effect_agent_child_settlements: [
+    "parent_submission_id",
+    "child_submission_id",
+    "child_outcome",
+    "recorded_at",
+  ],
+} as const;
+
+const checkPredecessorLayout = Effect.fn("DoJournal.checkPredecessorLayout")(function* () {
+  const sql = yield* SqlClient.SqlClient;
+
+  for (const [table, expected] of Object.entries(predecessorColumns)) {
+    const columns = yield* decodeRows(
+      Schema.Array(Schema.Struct({ name: BoundedIdentifier })),
+      table,
+      "schema",
+      yield* sql.unsafe(`PRAGMA table_info(${table})`),
+    );
+
+    const names = new Set<string>(expected);
+
+    if (columns.length !== names.size || columns.some((column) => !names.has(column.name)))
+      return yield* DoStorageCompatibilityError.make({
+        actualVersion: 3,
+        supportedVersion: CurrentDoStorageVersion,
+        message: `The v3 ${table} columns do not match the supported predecessor; no upgrade was committed.`,
+      });
+  }
+});
+
 const REQUIRED_TABLES = [
   "effect_agent_abort_intents",
   "effect_agent_approval_decisions",
@@ -217,7 +369,7 @@ const REQUIRED_TABLES = [
   "effect_agent_checkpoints",
   "effect_agent_child_reservations",
   "effect_agent_child_settlements",
-  "effect_agent_conversations",
+  "effect_agent_threads",
   "effect_agent_meta",
   "effect_agent_settlement_reservations",
   "effect_agent_submission_ownership",
@@ -226,7 +378,7 @@ const REQUIRED_TABLES = [
 ] as const;
 
 /**
- * Exact-or-fresh storage gate (DEPLOY-008) over `effect_agent_meta` instead of
+ * Supported-predecessor or fresh storage gate (DEPLOY-008) over `effect_agent_meta` instead of
  * `PRAGMA user_version` (unverified on Durable Object SQL storage; a meta table is portable
  * regardless). No WAL check (Durable Object storage owns durability and confirms writes
  * through output gates) and no busy timeout (a Durable Object has exactly one writer): the
@@ -243,6 +395,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
     WHERE type = 'table'
       AND name = 'effect_agent_meta'
   `.pipe(Effect.mapError(storageError("read storage version table")));
+
   const metaTables = yield* decodeRows(
     Schema.Array(DoNameRow),
     "sqlite_master",
@@ -258,6 +411,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
         AND name LIKE 'effect_agent_%'
       ORDER BY name
     `.pipe(Effect.mapError(storageError("inspect unversioned storage")));
+
     const existing = yield* decodeRows(
       Schema.Array(DoNameRow),
       "sqlite_master",
@@ -270,7 +424,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
         actualVersion: 0,
         supportedVersion: CurrentDoStorageVersion,
         message:
-          "The Durable Object contains unversioned Effect Agent tables. Reset the development namespace explicitly; refusing to mutate ambiguous stored data.",
+          "The Durable Object contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer.",
       });
     }
 
@@ -292,6 +446,7 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
       FROM effect_agent_meta
       WHERE key = 'storage_version'
     `.pipe(Effect.mapError(storageError("read storage version")));
+
     const version = yield* decodeSingleRow(
       Schema.Array(DoMetaRow),
       "effect_agent_meta",
@@ -299,18 +454,91 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
       versionRows,
     );
 
-    // The storage version must match EXACTLY. Older private-development versions fail
-    // closed with reset guidance rather than being migrated, and newer versions fail closed
-    // rather than being decoded incorrectly (DEPLOY-008).
-    if (version.value !== String(CurrentDoStorageVersion)) {
+    if (version.value === "2" || version.value === "3") {
+      yield* sql
+        .withTransaction(
+          Effect.gen(function* () {
+            const current = yield* sql<{
+              value: string;
+            }>`SELECT value FROM effect_agent_meta WHERE key='storage_version'`;
+
+            if (current.length === 1 && current[0].value === String(CurrentDoStorageVersion))
+              return;
+            if (current.length !== 1 || (current[0].value !== "2" && current[0].value !== "3"))
+              return yield* DoStorageCompatibilityError.make({
+                actualVersion: -1,
+                supportedVersion: CurrentDoStorageVersion,
+                message: "Storage version changed while acquiring the upgrade transaction.",
+              });
+
+            const required = yield* decodeRows(
+              Schema.Array(DoNameRow),
+              "sqlite_master",
+              "required_tables",
+              yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name IN ${sql.in([...REQUIRED_TABLES])}`,
+            );
+
+            if (required.length !== REQUIRED_TABLES.length)
+              return yield* DoStorageCompatibilityError.make({
+                actualVersion: Number(current[0].value),
+                supportedVersion: CurrentDoStorageVersion,
+                message:
+                  "The predecessor store is missing required tables. Retain the original store for inspection; no upgrade was committed.",
+              });
+            if (current[0].value === "2") {
+              yield* checkV2ThreadLayout();
+              for (const statement of [
+                sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
+                sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
+                sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`,
+              ]) {
+                yield* failpoint("upgrade:before-mutation");
+                yield* statement;
+                yield* failpoint("upgrade:after-mutation");
+              }
+            }
+            if (current[0].value === "3") yield* checkPredecessorLayout();
+            yield* failpoint("upgrade:before-mutation");
+            yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
+            yield* failpoint("upgrade:after-mutation");
+            yield* failpoint("upgrade:before-mutation");
+            yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
+            yield* failpoint("upgrade:after-mutation");
+            yield* failpoint("upgrade:before-mutation");
+            yield* createMessageDeliveryTables;
+            yield* failpoint("upgrade:after-mutation");
+            yield* failpoint("upgrade:before-version");
+            yield* sql`UPDATE effect_agent_meta SET value='4' WHERE key='storage_version'`;
+            yield* failpoint("upgrade:after-version");
+          }),
+        )
+        .pipe(
+          Effect.catchTag("DoStorageFailpointError", (error) =>
+            DoStorageError.make({
+              cause: error,
+              operation: "upgrade storage",
+              message: error.message,
+            }),
+          ),
+          Effect.catchTag("StorageUpgradeError", (error) =>
+            DoStorageCorruptionError.make({
+              table: error.table,
+              rowKey: error.rowKey,
+              message: error.message,
+            }),
+          ),
+          Effect.catchTag("SqlError", storageError("upgrade supported thread storage")),
+        );
+    } else if (version.value !== String(CurrentDoStorageVersion)) {
       const actualVersion = Number.parseInt(version.value, 10);
+
       return yield* DoStorageCompatibilityError.make({
         actualVersion: Number.isSafeInteger(actualVersion) ? actualVersion : -1,
         supportedVersion: CurrentDoStorageVersion,
         message:
-          `The Durable Object uses private-development storage version ${version.value}; ` +
+          `The Durable Object uses unsupported storage version ${version.value}; ` +
           `this build supports exactly version ${CurrentDoStorageVersion}. ` +
-          "Replace the development namespace explicitly; automatic stored-data migrations are not provided during private development.",
+          "Only supported v2 and v3 can be upgraded automatically. Keep the original store and use a compatible library version.",
       });
     }
   }
@@ -319,21 +547,23 @@ const ensureCurrentStorage = Effect.fn("DoJournal.ensureCurrentStorage")(functio
     SELECT name
     FROM sqlite_master
     WHERE type = 'table'
-      AND name IN ${sql.in([...REQUIRED_TABLES])}
+      AND name IN ${sql.in([...REQUIRED_TABLES, "effect_agent_message_deliveries"])}
     ORDER BY name
   `.pipe(Effect.mapError(storageError("verify storage tables")));
+
   const required = yield* decodeRows(
     Schema.Array(DoNameRow),
     "sqlite_master",
     "required_tables",
     requiredRows,
   );
-  if (required.length !== REQUIRED_TABLES.length) {
+
+  if (required.length !== REQUIRED_TABLES.length + 1) {
     return yield* DoStorageCompatibilityError.make({
       actualVersion: CurrentDoStorageVersion,
       supportedVersion: CurrentDoStorageVersion,
       message:
-        "The Durable Object claims the current format but is missing required tables. Reset the corrupt private-development data.",
+        "The Durable Object claims the current format but is missing required tables. Retain the original store for inspection.",
     });
   }
 
@@ -351,6 +581,7 @@ const makeJournal = (
     value: string,
   ): Effect.Effect<void, DoValueBoundExceeded> => {
     const actualBytes = storedTextBytes(value);
+
     return actualBytes > maxStoredValueBytes
       ? Effect.fail(
           DoValueBoundExceeded.make({
@@ -378,14 +609,12 @@ const makeJournal = (
     (operation: string) =>
     <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<A, E | DoStorageError> =>
       sql.withTransaction(effect).pipe(
-        Effect.mapError((error) =>
-          error instanceof SqlError ? storageError(operation)(error) : error,
-        ),
+        Effect.mapError((error) => (isSqlError(error) ? storageError(operation)(error) : error)),
         Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
       );
 
   const materialize = Effect.fn("DoJournal.materialize")(function* (
-    conversationId: string,
+    threadId: string,
     createdAt: string,
     emptyTailDigest: string,
     producerEpoch: ProducerEpoch,
@@ -393,54 +622,57 @@ const makeJournal = (
     void,
     DoFenceRejected | DoStorageCorruptionError | DoStorageError | DoValueBoundExceeded
   > {
-    if (conversationId.length > MAX_IDENTIFIER_LENGTH) {
+    if (threadId.length > MAX_IDENTIFIER_LENGTH) {
       return yield* DoStorageError.make({
-        operation: "materialize conversation",
-        message: "Conversation identity exceeds the Durable Object storage bounds.",
+        operation: "materialize thread",
+        message: "Thread identity exceeds the Durable Object storage bounds.",
       });
     }
-    yield* checkValueBound("materialize conversation", emptyTailDigest);
+    yield* checkValueBound("materialize thread", emptyTailDigest);
     yield* withWriteTransaction("materialize transaction")(
       Effect.gen(function* () {
         const existingRows = yield* sql<Record<string, unknown>>`
           SELECT
-            conversation_id,
+            thread_id,
             created_at,
             tail_sequence,
             tail_digest,
             producer_epoch
-          FROM effect_agent_conversations
-          WHERE conversation_id = ${conversationId}
-        `.pipe(Effect.mapError(storageError("read materialized conversation")));
+          FROM effect_agent_threads
+          WHERE thread_id = ${threadId}
+        `.pipe(Effect.mapError(storageError("read materialized thread")));
+
         const existing = yield* decodeRows(
-          Schema.Array(ConversationRow),
-          "effect_agent_conversations",
-          conversationId,
+          Schema.Array(ThreadRow),
+          "effect_agent_threads",
+          threadId,
           existingRows,
         );
+
         if (existing.length > 1) {
           return yield* DoStorageCorruptionError.make({
-            table: "effect_agent_conversations",
-            rowKey: conversationId,
-            message: "A conversation primary key returned more than one row.",
+            table: "effect_agent_threads",
+            rowKey: threadId,
+            message: "A thread primary key returned more than one row.",
           });
         }
         if (existing.length === 0) {
           yield* sql`
-            INSERT INTO effect_agent_conversations (
-              conversation_id,
+            INSERT INTO effect_agent_threads (
+              thread_id,
               created_at,
               tail_sequence,
               tail_digest,
               producer_epoch
             ) VALUES (
-              ${conversationId},
+              ${threadId},
               ${createdAt},
               0,
               ${emptyTailDigest},
               ${producerEpoch}
             )
-          `.pipe(Effect.mapError(storageError("materialize conversation")));
+          `.pipe(Effect.mapError(storageError("materialize thread")));
+
           return;
         }
         if (producerEpoch < existing[0].producer_epoch) {
@@ -452,41 +684,35 @@ const makeJournal = (
         }
         if (producerEpoch > existing[0].producer_epoch) {
           yield* sql`
-            UPDATE effect_agent_conversations
+            UPDATE effect_agent_threads
             SET producer_epoch = ${producerEpoch}
-            WHERE conversation_id = ${conversationId}
+            WHERE thread_id = ${threadId}
           `.pipe(Effect.mapError(storageError("advance materialization epoch")));
         }
       }),
     );
   });
 
-  const getConversation = Effect.fn("DoJournal.getConversation")(function* (
-    conversationId: string,
-  ) {
+  const getThread = Effect.fn("DoJournal.getThread")(function* (threadId: string) {
     const rows = yield* sql<Record<string, unknown>>`
       SELECT
-        conversation_id,
+        thread_id,
         created_at,
         tail_sequence,
         tail_digest,
         producer_epoch
-      FROM effect_agent_conversations
-      WHERE conversation_id = ${conversationId}
-    `.pipe(Effect.mapError(storageError("read conversation")));
-    return yield* decodeRows(
-      Schema.Array(ConversationRow),
-      "effect_agent_conversations",
-      conversationId,
-      rows,
-    );
+      FROM effect_agent_threads
+      WHERE thread_id = ${threadId}
+    `.pipe(Effect.mapError(storageError("read thread")));
+
+    return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
   });
 
   const append = Effect.fn("DoJournal.append")(function* (
     request: RawAppendRequest,
   ): Effect.fn.Return<RawAppendResult, AppendError> {
     if (
-      request.conversationId.length > MAX_IDENTIFIER_LENGTH ||
+      request.threadId.length > MAX_IDENTIFIER_LENGTH ||
       request.batchId.length > MAX_IDENTIFIER_LENGTH ||
       request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
     ) {
@@ -504,9 +730,11 @@ const makeJournal = (
       (record) => checkValueBound("append canonical record", record.recordJson),
       { discard: true },
     );
+
     return yield* withWriteTransaction("append transaction")(
       Effect.gen(function* () {
         const recordIds = request.records.map((record) => record.recordId);
+
         if (new Set(recordIds).size !== recordIds.length) {
           return yield* DoAppendConflict.make({
             message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
@@ -514,34 +742,35 @@ const makeJournal = (
           });
         }
 
-        const conversationRows = yield* sql<Record<string, unknown>>`
+        const threadRows = yield* sql<Record<string, unknown>>`
           SELECT
-            conversation_id,
+            thread_id,
             created_at,
             tail_sequence,
             tail_digest,
             producer_epoch
-          FROM effect_agent_conversations
-          WHERE conversation_id = ${request.conversationId}
+          FROM effect_agent_threads
+          WHERE thread_id = ${request.threadId}
         `.pipe(Effect.mapError(storageError("read append tail")));
-        const conversation = yield* decodeSingleRow(
-          Schema.Array(ConversationRow),
-          "effect_agent_conversations",
-          request.conversationId,
-          conversationRows,
+
+        const thread = yield* decodeSingleRow(
+          Schema.Array(ThreadRow),
+          "effect_agent_threads",
+          request.threadId,
+          threadRows,
         );
 
-        if (request.producerEpoch !== conversation.producer_epoch) {
+        if (request.producerEpoch !== thread.producer_epoch) {
           return yield* DoFenceRejected.make({
             producerEpoch: request.producerEpoch,
-            actualEpoch: conversation.producer_epoch,
-            message: `Producer epoch ${request.producerEpoch} is not the current epoch ${conversation.producer_epoch}.`,
+            actualEpoch: thread.producer_epoch,
+            message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
           });
         }
 
         const batchRows = yield* sql<Record<string, unknown>>`
           SELECT
-            conversation_id,
+            thread_id,
             batch_id,
             first_sequence,
             last_sequence,
@@ -549,31 +778,34 @@ const makeJournal = (
             tail_digest,
             batch_json
           FROM effect_agent_canonical_batches
-          WHERE conversation_id = ${request.conversationId}
+          WHERE thread_id = ${request.threadId}
             AND batch_id = ${request.batchId}
         `.pipe(Effect.mapError(storageError("read idempotent batch")));
+
         const batches = yield* decodeRows(
           Schema.Array(BatchRow),
           "effect_agent_canonical_batches",
-          `${request.conversationId}/${request.batchId}`,
+          `${request.threadId}/${request.batchId}`,
           batchRows,
         );
 
         if (batches.length > 1) {
           return yield* DoStorageCorruptionError.make({
             table: "effect_agent_canonical_batches",
-            rowKey: `${request.conversationId}/${request.batchId}`,
+            rowKey: `${request.threadId}/${request.batchId}`,
             message: "A canonical batch primary key returned more than one row.",
           });
         }
         if (batches.length === 1) {
           const existing = batches[0];
+
           if (existing.batch_digest !== request.batchDigest) {
             return yield* DoAppendConflict.make({
               message: `Batch ${request.batchId} already exists with different canonical content.`,
               reason: "batch-digest",
             });
           }
+
           return RawAppendResult.make({
             firstSequence: existing.first_sequence,
             lastSequence: existing.last_sequence,
@@ -583,46 +815,48 @@ const makeJournal = (
         }
 
         if (
-          request.expectedTailSequence !== conversation.tail_sequence ||
-          request.expectedTailDigest !== conversation.tail_digest
+          request.expectedTailSequence !== thread.tail_sequence ||
+          request.expectedTailDigest !== thread.tail_digest
         ) {
           return yield* DoAppendConflict.make({
             message:
               `Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
-              `but found ${conversation.tail_sequence}/${conversation.tail_digest}.`,
+              `but found ${thread.tail_sequence}/${thread.tail_digest}.`,
             reason: "tail",
-            actualTailSequence: conversation.tail_sequence,
-            actualTailDigest: conversation.tail_digest,
+            actualTailSequence: thread.tail_sequence,
+            actualTailDigest: thread.tail_digest,
           });
         }
-        if (conversation.tail_sequence + request.records.length > MAX_RECORDS_PER_CONVERSATION) {
+        if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
           return yield* DoStorageError.make({
             operation: "append canonical batch",
-            message: `Conversation record limit ${MAX_RECORDS_PER_CONVERSATION} would be exceeded.`,
+            message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
           });
         }
 
         // Chunked to respect the Durable Object platform's 100-bound-parameter statement
         // limit: a batch may carry up to 256 records.
         const existingRecords: Array<RecordRow> = [];
+
         for (const chunk of chunked(recordIds, MAX_BOUND_PARAMETERS - 10)) {
           const existingRecordRows = yield* sql<Record<string, unknown>>`
             SELECT
-              conversation_id,
+              thread_id,
               sequence,
               record_id,
               batch_id,
               record_json
             FROM effect_agent_canonical_records
-            WHERE conversation_id = ${request.conversationId}
+            WHERE thread_id = ${request.threadId}
               AND record_id IN ${sql.in([...chunk])}
             ORDER BY sequence
           `.pipe(Effect.mapError(storageError("check canonical record identities")));
+
           existingRecords.push(
             ...(yield* decodeRows(
               Schema.Array(RecordRow),
               "effect_agent_canonical_records",
-              `${request.conversationId}/record_ids`,
+              `${request.threadId}/record_ids`,
               existingRecordRows,
             )),
           );
@@ -635,7 +869,7 @@ const makeJournal = (
         }
 
         const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
-          conversation.tail_sequence + 1,
+          thread.tail_sequence + 1,
         ).pipe(
           Effect.mapError((error) =>
             DoStorageError.make({
@@ -645,6 +879,7 @@ const makeJournal = (
             }),
           ),
         );
+
         const lastSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
           firstSequence + request.records.length - 1,
         ).pipe(
@@ -659,7 +894,7 @@ const makeJournal = (
 
         yield* sql`
           INSERT INTO effect_agent_canonical_batches (
-            conversation_id,
+            thread_id,
             batch_id,
             first_sequence,
             last_sequence,
@@ -667,7 +902,7 @@ const makeJournal = (
             tail_digest,
             batch_json
           ) VALUES (
-            ${request.conversationId},
+            ${request.threadId},
             ${request.batchId},
             ${firstSequence},
             ${lastSequence},
@@ -684,13 +919,13 @@ const makeJournal = (
             Effect.gen(function* () {
               yield* sql`
                   INSERT INTO effect_agent_canonical_records (
-                    conversation_id,
+                    thread_id,
                     sequence,
                     record_id,
                     batch_id,
                     record_json
                   ) VALUES (
-                    ${request.conversationId},
+                    ${request.threadId},
                     ${firstSequence + index},
                     ${record.recordId},
                     ${request.batchId},
@@ -703,13 +938,13 @@ const makeJournal = (
         );
 
         yield* sql`
-          UPDATE effect_agent_conversations
+          UPDATE effect_agent_threads
           SET
             tail_sequence = ${lastSequence},
             tail_digest = ${request.tailDigest},
             producer_epoch = ${request.producerEpoch}
-          WHERE conversation_id = ${request.conversationId}
-        `.pipe(Effect.mapError(storageError("advance conversation tail")));
+          WHERE thread_id = ${request.threadId}
+        `.pipe(Effect.mapError(storageError("advance thread tail")));
         yield* failpoint("append:after-tail-update");
 
         return RawAppendResult.make({
@@ -723,105 +958,137 @@ const makeJournal = (
   });
 
   const read = Effect.fn("DoJournal.read")(function* (request: RawReadRequest) {
-    const rows = yield* sql<Record<string, unknown>>`
+    // Capture membership without retaining payloads. Append-only sequences keep each later
+    // payload query inside this snapshot, even when new records arrive during consumption.
+    const planRows = yield* sql<Record<string, unknown>>`
       SELECT
-        conversation_id,
         sequence,
-        record_id,
-        batch_id,
-        record_json
+        length(CAST(record_json AS BLOB)) AS record_json_bytes
       FROM effect_agent_canonical_records
-      WHERE conversation_id = ${request.conversationId}
+      WHERE thread_id = ${request.threadId}
         AND sequence > ${request.fromSequenceExclusive}
       ORDER BY sequence
       LIMIT ${request.limit}
     `.pipe(Effect.mapError(storageError("read canonical records")));
-    return yield* decodeRows(
-      Schema.Array(RecordRow),
+
+    const plan = yield* decodeRows(
+      Schema.Array(ReadPlanRow),
       "effect_agent_canonical_records",
-      `${request.conversationId}>${request.fromSequenceExclusive}`,
-      rows,
+      `${request.threadId}>${request.fromSequenceExclusive}`,
+      planRows,
     );
+
+    const mismatch = () =>
+      DoStorageCorruptionError.make({
+        table: "effect_agent_canonical_records",
+        rowKey: `${request.threadId}>${request.fromSequenceExclusive}`,
+        message: "Canonical read membership, sequence, or payload size changed from its read plan.",
+      });
+
+    if (plan.length > request.limit) return yield* mismatch();
+
+    const pages: Array<ReadPage> = [];
+    let page: ReadPage | undefined;
+    let pageBytes = 0;
+    let previousSequence = request.fromSequenceExclusive;
+
+    for (const row of plan) {
+      if (row.sequence !== previousSequence + 1) return yield* mismatch();
+      previousSequence = row.sequence;
+      if (page === undefined || pageBytes + row.record_json_bytes > MAX_READ_PAGE_JSON_BYTES) {
+        page = [row];
+        pages.push(page);
+        pageBytes = row.record_json_bytes;
+      } else {
+        page.push(row);
+        pageBytes += row.record_json_bytes;
+      }
+    }
+
+    const readPage = Effect.fn("DoJournal.readPage")(function* (page: ReadPage) {
+      const rows = yield* sql<Record<string, unknown>>`
+        SELECT thread_id, sequence, record_id, batch_id, record_json
+        FROM effect_agent_canonical_records
+        WHERE thread_id = ${request.threadId}
+          AND sequence >= ${page[0].sequence}
+          AND sequence <= ${page[page.length - 1].sequence}
+        ORDER BY sequence
+      `.pipe(Effect.mapError(storageError("read canonical records")));
+
+      const decoded = yield* decodeRows(
+        Schema.Array(RecordRow),
+        "effect_agent_canonical_records",
+        `${request.threadId}/${page[0].sequence}`,
+        rows,
+      );
+
+      if (
+        decoded.length !== page.length ||
+        decoded.some(
+          (row, index) =>
+            row.thread_id !== request.threadId ||
+            row.sequence !== page[index].sequence ||
+            storedTextBytes(row.record_json) !== page[index].record_json_bytes,
+        )
+      ) {
+        return yield* mismatch();
+      }
+
+      return decoded;
+    });
+
+    return {
+      count: plan.length,
+      records: Stream.fromIterable(pages).pipe(
+        Stream.flatMap((page) => Stream.fromIterableEffect(readPage(page))),
+      ),
+    };
   });
 
-  const exportConversation = Effect.fn("DoJournal.exportConversation")(function* (
-    conversationId: string,
-  ) {
+  const exportThread = Effect.fn("DoJournal.exportThread")(function* (threadId: string) {
     return yield* sql
       .withTransaction(
         Effect.gen(function* () {
-          const conversationRows = yield* sql<Record<string, unknown>>`
+          const threadRows = yield* sql<Record<string, unknown>>`
             SELECT
-              conversation_id,
+              thread_id,
               created_at,
               tail_sequence,
               tail_digest,
               producer_epoch
-            FROM effect_agent_conversations
-            WHERE conversation_id = ${conversationId}
-          `.pipe(Effect.mapError(storageError("export conversation")));
-          const conversation = yield* decodeSingleRow(
-            Schema.Array(ConversationRow),
-            "effect_agent_conversations",
-            conversationId,
-            conversationRows,
+            FROM effect_agent_threads
+            WHERE thread_id = ${threadId}
+          `.pipe(Effect.mapError(storageError("export thread")));
+
+          const thread = yield* decodeSingleRow(
+            Schema.Array(ThreadRow),
+            "effect_agent_threads",
+            threadId,
+            threadRows,
           );
-          yield* failpoint("export:after-conversation-read");
-          const batchRows = yield* sql<Record<string, unknown>>`
-            SELECT
-              conversation_id,
-              batch_id,
-              first_sequence,
-              last_sequence,
-              batch_digest,
-              tail_digest,
-              batch_json
-            FROM effect_agent_canonical_batches
-            WHERE conversation_id = ${conversationId}
-            ORDER BY first_sequence
-          `.pipe(Effect.mapError(storageError("export canonical batches")));
+
+          yield* failpoint("export:after-thread-read");
+
           const recordRows = yield* sql<Record<string, unknown>>`
             SELECT
-              conversation_id,
+              thread_id,
               sequence,
               record_id,
               batch_id,
               record_json
             FROM effect_agent_canonical_records
-            WHERE conversation_id = ${conversationId}
+            WHERE thread_id = ${threadId}
             ORDER BY sequence
           `.pipe(Effect.mapError(storageError("export canonical records")));
-          const checkpointRows = yield* sql<Record<string, unknown>>`
-            SELECT
-              conversation_id,
-              through_sequence,
-              tail_digest,
-              checkpoint_json
-            FROM effect_agent_checkpoints
-            WHERE conversation_id = ${conversationId}
-            ORDER BY through_sequence
-          `.pipe(Effect.mapError(storageError("export checkpoints")));
 
-          return RawConversationExport.make({
-            conversation,
-            batches: yield* decodeRows(
-              Schema.Array(BatchRow),
-              "effect_agent_canonical_batches",
-              conversationId,
-              batchRows,
-            ),
+          return RawThreadExport.make({
+            thread,
             records: yield* decodeRows(
               Schema.Array(RecordRow),
               "effect_agent_canonical_records",
-              conversationId,
+              threadId,
               recordRows,
             ),
-            checkpoints: yield* decodeRows(
-              Schema.Array(CheckpointRow),
-              "effect_agent_checkpoints",
-              conversationId,
-              checkpointRows,
-            ),
           });
         }),
       )
@@ -835,7 +1102,7 @@ const makeJournal = (
   const saveCheckpoint = Effect.fn("DoJournal.saveCheckpoint")(function* (
     checkpoint: RawCheckpoint,
   ): Effect.fn.Return<void, CheckpointError> {
-    if (checkpoint.conversationId.length > MAX_IDENTIFIER_LENGTH) {
+    if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
       return yield* DoStorageError.make({
         operation: "save checkpoint",
         message: "Checkpoint identity exceeds the Durable Object storage bounds.",
@@ -844,50 +1111,54 @@ const makeJournal = (
     yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
     yield* withWriteTransaction("checkpoint transaction")(
       Effect.gen(function* () {
-        const conversationRows = yield* sql<Record<string, unknown>>`
+        const threadRows = yield* sql<Record<string, unknown>>`
           SELECT
-            conversation_id,
+            thread_id,
             created_at,
             tail_sequence,
             tail_digest,
             producer_epoch
-          FROM effect_agent_conversations
-          WHERE conversation_id = ${checkpoint.conversationId}
+          FROM effect_agent_threads
+          WHERE thread_id = ${checkpoint.threadId}
         `.pipe(Effect.mapError(storageError("read checkpoint tail")));
-        const conversation = yield* decodeSingleRow(
-          Schema.Array(ConversationRow),
-          "effect_agent_conversations",
-          checkpoint.conversationId,
-          conversationRows,
+
+        const thread = yield* decodeSingleRow(
+          Schema.Array(ThreadRow),
+          "effect_agent_threads",
+          checkpoint.threadId,
+          threadRows,
         );
-        if (checkpoint.throughSequence > conversation.tail_sequence) {
+
+        if (checkpoint.throughSequence > thread.tail_sequence) {
           return yield* DoCheckpointConflict.make({
             message:
               `Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
-              `${conversation.tail_sequence}.`,
+              `${thread.tail_sequence}.`,
           });
         }
 
         const checkpointRows = yield* sql<Record<string, unknown>>`
           SELECT
-            conversation_id,
+            thread_id,
             through_sequence,
             tail_digest,
             checkpoint_json
           FROM effect_agent_checkpoints
-          WHERE conversation_id = ${checkpoint.conversationId}
+          WHERE thread_id = ${checkpoint.threadId}
             AND through_sequence = ${checkpoint.throughSequence}
         `.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
+
         const existing = yield* decodeRows(
           Schema.Array(CheckpointRow),
           "effect_agent_checkpoints",
-          `${checkpoint.conversationId}/${checkpoint.throughSequence}`,
+          `${checkpoint.threadId}/${checkpoint.throughSequence}`,
           checkpointRows,
         );
+
         if (existing.length > 1) {
           return yield* DoStorageCorruptionError.make({
             table: "effect_agent_checkpoints",
-            rowKey: `${checkpoint.conversationId}/${checkpoint.throughSequence}`,
+            rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
             message: "A checkpoint primary key returned more than one row.",
           });
         }
@@ -900,17 +1171,18 @@ const makeJournal = (
               message: "A different checkpoint already exists at this canonical sequence.",
             });
           }
+
           return;
         }
 
         yield* sql`
           INSERT INTO effect_agent_checkpoints (
-            conversation_id,
+            thread_id,
             through_sequence,
             tail_digest,
             checkpoint_json
           ) VALUES (
-            ${checkpoint.conversationId},
+            ${checkpoint.threadId},
             ${checkpoint.throughSequence},
             ${checkpoint.tailDigest},
             ${checkpoint.checkpointJson}
@@ -921,44 +1193,47 @@ const makeJournal = (
   });
 
   const loadCheckpoint = Effect.fn("DoJournal.loadCheckpoint")(function* (
-    conversationId: string,
+    threadId: string,
     atOrBeforeSequence: CanonicalSequence,
   ) {
     const rows = yield* sql<Record<string, unknown>>`
       SELECT
-        conversation_id,
+        thread_id,
         through_sequence,
         tail_digest,
         checkpoint_json
       FROM effect_agent_checkpoints
-      WHERE conversation_id = ${conversationId}
+      WHERE thread_id = ${threadId}
         AND through_sequence <= ${atOrBeforeSequence}
       ORDER BY through_sequence DESC
       LIMIT 1
     `.pipe(Effect.mapError(storageError("load checkpoint")));
+
     return yield* decodeRows(
       Schema.Array(CheckpointRow),
       "effect_agent_checkpoints",
-      `${conversationId}<=${atOrBeforeSequence}`,
+      `${threadId}<=${atOrBeforeSequence}`,
       rows,
     );
   });
 
   const getTailDigestAt = Effect.fn("DoJournal.getTailDigestAt")(function* (
-    conversationId: string,
+    threadId: string,
     sequence: CanonicalSequence,
   ) {
     if (sequence === 0) {
-      const conversations = yield* getConversation(conversationId);
-      return conversations.length === 0
+      const threads = yield* getThread(threadId);
+
+      return threads.length === 0
         ? []
-        : [conversations[0].tail_sequence === 0 ? conversations[0].tail_digest : undefined].filter(
+        : [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
             (value): value is string => value !== undefined,
           );
     }
+
     const rows = yield* sql<Record<string, unknown>>`
       SELECT
-        conversation_id,
+        thread_id,
         batch_id,
         first_sequence,
         last_sequence,
@@ -966,15 +1241,17 @@ const makeJournal = (
         tail_digest,
         batch_json
       FROM effect_agent_canonical_batches
-      WHERE conversation_id = ${conversationId}
+      WHERE thread_id = ${threadId}
         AND last_sequence = ${sequence}
     `.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
+
     const batches = yield* decodeRows(
       Schema.Array(BatchRow),
       "effect_agent_canonical_batches",
-      `${conversationId}/${sequence}`,
+      `${threadId}/${sequence}`,
       rows,
     );
+
     return batches.map((batch) => batch.tail_digest);
   });
 
@@ -982,19 +1259,20 @@ const makeJournal = (
     return yield* sql
       .withTransaction(
         Effect.gen(function* () {
-          const conversations = yield* sql<Record<string, unknown>>`
+          const threads = yield* sql<Record<string, unknown>>`
             SELECT
-              conversation_id,
+              thread_id,
               created_at,
               tail_sequence,
               tail_digest,
               producer_epoch
-            FROM effect_agent_conversations
-            ORDER BY conversation_id
-          `.pipe(Effect.mapError(storageError("scan conversations")));
+            FROM effect_agent_threads
+            ORDER BY thread_id
+          `.pipe(Effect.mapError(storageError("scan threads")));
+
           const batches = yield* sql<Record<string, unknown>>`
             SELECT
-              conversation_id,
+              thread_id,
               batch_id,
               first_sequence,
               last_sequence,
@@ -1002,33 +1280,36 @@ const makeJournal = (
               tail_digest,
               batch_json
             FROM effect_agent_canonical_batches
-            ORDER BY conversation_id, first_sequence
+            ORDER BY thread_id, first_sequence
           `.pipe(Effect.mapError(storageError("scan canonical batches")));
+
           const records = yield* sql<Record<string, unknown>>`
             SELECT
-              conversation_id,
+              thread_id,
               sequence,
               record_id,
               batch_id,
               record_json
             FROM effect_agent_canonical_records
-            ORDER BY conversation_id, sequence
+            ORDER BY thread_id, sequence
           `.pipe(Effect.mapError(storageError("scan canonical records")));
+
           const checkpoints = yield* sql<Record<string, unknown>>`
             SELECT
-              conversation_id,
+              thread_id,
               through_sequence,
               tail_digest,
               checkpoint_json
             FROM effect_agent_checkpoints
-            ORDER BY conversation_id, through_sequence
+            ORDER BY thread_id, through_sequence
           `.pipe(Effect.mapError(storageError("scan checkpoints")));
+
           return {
-            conversations: yield* decodeRows(
-              Schema.Array(ConversationRow),
-              "effect_agent_conversations",
+            threads: yield* decodeRows(
+              Schema.Array(ThreadRow),
+              "effect_agent_threads",
               "startup_scan",
-              conversations,
+              threads,
             ),
             batches: yield* decodeRows(
               Schema.Array(BatchRow),
@@ -1061,8 +1342,8 @@ const makeJournal = (
   return {
     append,
     checkValueBound,
-    exportConversation,
-    getConversation,
+    exportThread,
+    getThread,
     getTailDigestAt,
     loadCheckpoint,
     materialize,