@yielded/agent-storage-cloudflare 0.1.0-beta.167 → 0.1.0-beta.169

package/src/DoThreadStore.ts

@@ -1,58 +1,46 @@
 import { BrowserCrypto } from "@effect/platform-browser";
 import { SqliteClient } from "@effect/sql-sqlite-do";
+import { makeSqlThreadImport } from "@yielded/agent-storage-sql/sql-thread-import";
 import {
   makeSelectedReads,
   SelectedReadOwner,
 } from "@yielded/agent-storage-sql/sql-thread-native-reads";
-import { digestCanonicalBatch, EMPTY_TAIL_DIGEST } from "@yielded/agent/digest";
+import { ExportRecord } from "@yielded/agent/record-format";
 import {
-  CanonicalBatch,
-  CanonicalRecord,
   CanonicalRecordEnvelope,
   CanonicalSequence,
-  Digest,
   ObservationOffset,
 } from "@yielded/agent/records";
+import { StorageOwner } from "@yielded/agent/run-storage";
 import { SqlStorageOwner } from "@yielded/agent/sql-memory-store";
-import { DEFAULT_OWNERSHIP_LEASE_DURATION } from "@yielded/agent/submission-ledger";
+import { MAX_ARCHIVE_RANGE_PAGE } from "@yielded/agent/thread-archive-range";
+import { ThreadImport } from "@yielded/agent/thread-import";
 import {
+  type ThreadExportRequest,
   AppendConflict,
   AppendResult,
   CheckpointRejected,
   ThreadCheckpoint,
-  ThreadExport,
-  ThreadExportRequest,
   ThreadMaterialization,
   ThreadNotMaterialized,
   ThreadObservation,
   ThreadReadRequest,
   ThreadStore,
+  ThreadReader,
   type ThreadCheckpoints,
   ThreadStoreError,
   ThreadStoreDiagnostic,
   ThreadTail,
   ThreadTailRequest,
-  FenceRejected,
-  FencedAppendRequest,
+  type AppendRequest,
+  PreparedAppend,
   LoadCheckpointRequest,
   SaveCheckpointRequest,
-  SaveRecoveryCheckpointRequest,
-  MAX_THREAD_EXPORT_RECORDS,
-  type ThreadRecoveryCheckpoints,
 } from "@yielded/agent/thread-store";
-import {
-  Clock,
-  Context,
-  Crypto,
-  Duration,
-  Effect,
-  Layer,
-  Option,
-  Ref,
-  Schema,
-  Stream,
-} from "effect";
+import { checkExportSource } from "@yielded/agent/thread-transfer";
+import { Clock, Context, Crypto, Effect, Layer, Option, Ref, Schema, Stream } from "effect";
 import * as SqlClientService from "effect/sql/SqlClient";
+import { isSqlError } from "effect/sql/SqlError";
 
 import {
   DEFAULT_MAX_STORED_VALUE_BYTES,
@@ -63,20 +51,21 @@ import {
   type DoStorageCompatibilityError,
   DoAppendConflict,
   DoCheckpointConflict,
-  DoFenceRejected,
   type DoStorageFailpointLocation,
   DoStorageCorruptionError,
   DoStorageError,
 } from "./DoStorageError.ts";
 import { DoStorageFailpoint, type DoStorageFailpointHandler } from "./DoStorageFailpoint.ts";
+import { DO_OFFSET_PREFIX, prepareCanonicalAppend } from "./internal/canonical-append.ts";
 import {
   initializeDoJournal,
-  RawAppendRequest,
   RawCheckpoint,
   RawReadRequest,
   type DoJournal,
 } from "./internal/do-journal.ts";
+import { readDoStorageHeader } from "./internal/migrations.ts";
 import { invalidateOwnedState, ownedState } from "./internal/owned-state.ts";
+import { makePromptCheckpoints, promptCheckpointPrefix } from "./internal/prompt-checkpoint.ts";
 import { isAppendContention, withStorageSpan } from "./internal/storage-span.ts";
 
 /**
@@ -87,20 +76,15 @@ import { isAppendContention, withStorageSpan } from "./internal/storage-span.ts"
 export interface DoStorageOptions {
   readonly storage: DurableObjectStorage;
   readonly observationPollInterval?: number | undefined;
-  /**
-   * Submission ownership lease duration in milliseconds (D5). Defaults to
-   * `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@yielded/agent/submission-ledger`.
-   */
-  readonly ownershipLeaseDuration?: number | undefined;
   /**
    * Maximum bytes for any single stored value; must stay under the platform's 2 MB
    * per-value limit. Defaults to `DEFAULT_MAX_STORED_VALUE_BYTES`.
    */
   readonly maxStoredValueBytes?: number | undefined;
   /**
-   * Re-verify every stored payload and digest chain while opening the store. Defaults to
-   * off: per-operation Schema decoding and the digest chain already fail clearly on corrupt
-   * rows without scanning the whole database on every open.
+   * Audit journal payloads, ownership, and batch coverage in bounded pages on open, including
+   * orphan rows and rows outside declared ranges. Defaults to false. Full reads validate
+   * canonical records; readPrompt trusts write-time validation of the omitted fields.
    */
   readonly verifyOnOpen?: boolean | undefined;
   readonly failpoint?: DoStorageFailpointHandler | undefined;
@@ -112,16 +96,12 @@ export type DoStorageInitializationError =
   | DoStorageError;
 
 const OffsetText = Schema.String.check(Schema.isMaxLength(4 * 1024));
-const DO_OFFSET_PREFIX = "effect-agent-do@1:";
 const ZERO_CANONICAL_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
-const isDigest = Schema.is(Digest);
-const isDoFenceRejected = Schema.is(DoFenceRejected);
 const isDoAppendConflict = Schema.is(DoAppendConflict);
 const isDoCheckpointConflict = Schema.is(DoCheckpointConflict);
 // Reuse the same AST/parser across reads. Reconstructing a JSON codec per row
 // defeats Schema's parser cache and repeatedly walks the canonical union.
-const CanonicalRecordJson = Schema.fromJsonString(CanonicalRecord);
-const CanonicalBatchJson = Schema.fromJsonString(CanonicalBatch);
+const CanonicalRecordJson = Schema.fromJsonString(ExportRecord);
 const ThreadCheckpointJson = Schema.fromJsonString(ThreadCheckpoint);
 const decodeCanonicalRecord = Schema.decodeEffect(CanonicalRecordJson);
 
@@ -147,11 +127,11 @@ const schemaStoreError = (operation: string, error: Schema.SchemaError) =>
     }),
   });
 
-const makeOffset = Effect.fn(function* (
+const makeOffset = (
   threadId: ThreadMaterialization["threadId"],
   sequence: number,
-): Effect.fn.Return<ObservationOffset, ThreadStoreError> {
-  return yield* Schema.decodeEffect(CanonicalSequence)(sequence).pipe(
+): Effect.Effect<ObservationOffset, ThreadStoreError> =>
+  Schema.decodeEffect(CanonicalSequence)(sequence).pipe(
     Effect.flatMap((validatedSequence) =>
       Schema.decodeEffect(ObservationOffset)(
         `${DO_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${validatedSequence}`,
@@ -159,9 +139,8 @@ const makeOffset = Effect.fn(function* (
     ),
     Effect.mapError((error) => schemaStoreError("encode observation offset", error)),
   );
-});
 
-const parseOffset = Effect.fn(function* (
+const parseOffset = Effect.fnUntraced(function* (
   threadId: ThreadMaterialization["threadId"],
   offset: ObservationOffset | undefined,
 ): Effect.fn.Return<CanonicalSequence, ThreadStoreError> {
@@ -193,36 +172,10 @@ const parseOffset = Effect.fn(function* (
   );
 });
 
-const mapFence = (threadId: ThreadMaterialization["threadId"], error: DoFenceRejected) =>
-  FenceRejected.make({
-    threadId,
-    actualEpoch: error.actualEpoch,
-    attemptedEpoch: error.producerEpoch,
-  });
-
-const encodeCanonicalRecord = Effect.fn(function* (
-  record: CanonicalRecord,
-): Effect.fn.Return<string, ThreadStoreError> {
-  return yield* Schema.encodeEffect(CanonicalRecordJson)(record).pipe(
-    Effect.mapError((error) => schemaStoreError("encode canonical record", error)),
-  );
-});
-
-const encodeCanonicalBatch = Effect.fn(function* (
-  batch: CanonicalBatch,
-): Effect.fn.Return<string, ThreadStoreError> {
-  return yield* Schema.encodeEffect(CanonicalBatchJson)(batch).pipe(
-    Effect.mapError((error) => schemaStoreError("encode canonical batch", error)),
-  );
-});
-
-const encodeCheckpoint = Effect.fn(function* (
-  checkpoint: ThreadCheckpoint,
-): Effect.fn.Return<string, ThreadStoreError> {
-  return yield* Schema.encodeEffect(ThreadCheckpointJson)(checkpoint).pipe(
+const encodeCheckpoint = (checkpoint: ThreadCheckpoint): Effect.Effect<string, ThreadStoreError> =>
+  Schema.encodeEffect(ThreadCheckpointJson)(checkpoint).pipe(
     Effect.mapError((error) => schemaStoreError("encode checkpoint", error)),
   );
-});
 
 const envelopes = new WeakMap<object, CanonicalRecordEnvelope>();
 
@@ -271,28 +224,24 @@ const decodeEnvelope = Effect.fnUntraced(function* (row: {
     row.batch_id,
   ).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
 
-  const envelope = CanonicalRecordEnvelope.make({
-    threadId,
-    batchId,
-    sequence: row.sequence,
-    offset,
-    record,
-  });
+  const envelope = new CanonicalRecordEnvelope(
+    { threadId, batchId, sequence: row.sequence, offset, record },
+    { disableChecks: true },
+  );
 
   envelopes.set(row, envelope);
 
   return envelope;
 });
 
-const decodeCheckpoint = Effect.fn(function* (
+const decodeCheckpoint = (
   checkpointJson: string,
-): Effect.fn.Return<ThreadCheckpoint, ThreadStoreError> {
-  return yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpointJson).pipe(
+): Effect.Effect<ThreadCheckpoint, ThreadStoreError> =>
+  Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpointJson).pipe(
     Effect.mapError((error) => schemaStoreError("decode checkpoint", error)),
   );
-});
 
-const requireThread = Effect.fn("DoThreadStore.requireThread")(function* (
+const requireThread = Effect.fnUntraced(function* (
   journal: DoJournal,
   threadId: ThreadMaterialization["threadId"],
 ) {
@@ -307,243 +256,40 @@ const requireThread = Effect.fn("DoThreadStore.requireThread")(function* (
   return rows[0];
 });
 
-const tailDigestAt = Effect.fn("DoThreadStore.tailDigestAt")(function* (
+const requireBatchBoundary = Effect.fnUntraced(function* (
   journal: DoJournal,
   threadId: ThreadMaterialization["threadId"],
   sequence: CanonicalSequence,
 ) {
-  if (sequence === 0) return EMPTY_TAIL_DIGEST;
+  const valid = yield* journal
+    .isBatchBoundary(threadId, sequence)
+    .pipe(Effect.mapError((error) => storeError("read checkpoint boundary", error)));
 
-  const digests = yield* journal
-    .getTailDigestAt(threadId, sequence)
-    .pipe(Effect.mapError((error) => storeError("read checkpoint digest", error)));
-
-  if (digests.length !== 1) {
+  if (!valid) {
     return yield* CheckpointRejected.make({
       threadId,
-      reason: "digest-mismatch",
+      reason: "not-batch-boundary",
     });
   }
-
-  return yield* Schema.decodeEffect(Digest)(digests[0]).pipe(
-    Effect.mapError((error) => schemaStoreError("decode checkpoint digest", error)),
-  );
 });
 
-const groupByKey = <A>(
-  rows: ReadonlyArray<A>,
-  key: (row: A) => string,
-): ReadonlyMap<string, ReadonlyArray<A>> => {
-  const grouped = new Map<string, Array<A>>();
+/** Audit excluded rows before paging owners and bounded ranges; never collect Thread payloads. */
+const decodeStartupPayloads = Effect.fnUntraced(function* (journal: DoJournal) {
+  yield* journal.archiveRanges.verifyCoverage;
+  let afterThreadId: string | undefined;
 
-  for (const row of rows) {
-    const existing = grouped.get(key(row));
+  while (true) {
+    const owners = yield* journal.archiveRanges.threadPage(afterThreadId);
 
-    if (existing === undefined) {
-      grouped.set(key(row), [row]);
-    } else {
-      existing.push(row);
+    for (const owner of owners) {
+      yield* journal.archiveRanges.verifyThread(owner.thread_id);
+      afterThreadId = owner.thread_id;
     }
-  }
-
-  return grouped;
-};
-
-/**
- * Opt-in integrity audit (`verifyOnOpen`) of canonical payloads, their digest chains and generic
- * projection checkpoints. Disposable recovery checkpoints are validated when loaded. Routine opens
- * skip this scan: per-operation Schema decoding fails clearly on corrupt canonical rows.
- */
-const decodeStartupPayloads = Effect.fn("DoThreadStore.decodeStartupPayloads")(function* (
-  journal: DoJournal,
-  crypto: Crypto.Crypto,
-) {
-  const stored = yield* journal.scanStoredPayloads();
-
-  const batches = yield* Effect.forEach(stored.batches, (batch) =>
-    Schema.decodeEffect(Schema.fromJsonString(CanonicalBatch))(batch.batch_json).pipe(
-      Effect.map((decoded) => ({ decoded, row: batch })),
-      Effect.mapError((error) =>
-        DoStorageCorruptionError.make({
-          table: "effect_agent_canonical_batches",
-          rowKey: `${batch.thread_id}/${batch.batch_id}`,
-          message: error.message,
-        }),
-      ),
-    ),
-  );
-
-  const records = yield* Effect.forEach(stored.records, (record) =>
-    Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(record.record_json).pipe(
-      Effect.map((decoded) => ({ decoded, row: record })),
-      Effect.mapError((error) =>
-        DoStorageCorruptionError.make({
-          table: "effect_agent_canonical_records",
-          rowKey: `${record.thread_id}/${record.sequence}`,
-          message: error.message,
-        }),
-      ),
-    ),
-  );
-
-  const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) =>
-    Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint.checkpoint_json).pipe(
-      Effect.map((decoded) => ({ decoded, row: checkpoint })),
-      Effect.mapError((error) =>
-        DoStorageCorruptionError.make({
-          table: "effect_agent_checkpoints",
-          rowKey: `${checkpoint.thread_id}/${checkpoint.through_sequence}`,
-          message: error.message,
-        }),
-      ),
-    ),
-  );
-
-  const batchesByThread = groupByKey(batches, ({ row }) => row.thread_id);
-  const recordsByThread = groupByKey(records, ({ row }) => row.thread_id);
-  const checkpointsByThread = groupByKey(checkpoints, ({ row }) => row.thread_id);
-  const materializedIds = new Set(stored.threads.map((thread) => thread.thread_id));
-
-  for (const thread of stored.threads) {
-    const threadBatches = batchesByThread.get(thread.thread_id) ?? [];
-    const threadRecords = recordsByThread.get(thread.thread_id) ?? [];
-    const threadCheckpoints = checkpointsByThread.get(thread.thread_id) ?? [];
-    const recordsByBatch = groupByKey(threadRecords, ({ row }) => row.batch_id);
-    let previousDigest = EMPTY_TAIL_DIGEST;
-    let expectedSequence = 1;
-    const tailDigests = new Map<number, string>([[0, EMPTY_TAIL_DIGEST]]);
-
-    for (const { decoded: canonicalBatch, row: batchRow } of threadBatches) {
-      const key = `${batchRow.thread_id}/${batchRow.batch_id}`;
-
-      if (
-        canonicalBatch.batchId !== batchRow.batch_id ||
-        batchRow.first_sequence !== expectedSequence ||
-        batchRow.last_sequence !== batchRow.first_sequence + canonicalBatch.records.length - 1
-      ) {
-        return yield* DoStorageCorruptionError.make({
-          table: "effect_agent_canonical_batches",
-          rowKey: key,
-          message: "Canonical batch identity, sequence, or record count is inconsistent.",
-        });
-      }
-
-      const digest = yield* digestCanonicalBatch(previousDigest, canonicalBatch).pipe(
-        Effect.provideService(Crypto.Crypto, crypto),
-        Effect.mapError((error) =>
-          DoStorageCorruptionError.make({
-            table: "effect_agent_canonical_batches",
-            rowKey: key,
-            message: error.message,
-          }),
-        ),
-      );
-
-      if (batchRow.batch_digest !== digest || batchRow.tail_digest !== digest) {
-        return yield* DoStorageCorruptionError.make({
-          table: "effect_agent_canonical_batches",
-          rowKey: key,
-          message: "Canonical batch digest does not match its decoded content and prior tail.",
-        });
-      }
-
-      const batchRecords = recordsByBatch.get(batchRow.batch_id) ?? [];
-
-      if (batchRecords.length !== canonicalBatch.records.length) {
-        return yield* DoStorageCorruptionError.make({
-          table: "effect_agent_canonical_records",
-          rowKey: key,
-          message: "Canonical batch and record-table counts differ.",
-        });
-      }
-      for (let index = 0; index < canonicalBatch.records.length; index++) {
-        const expectedRecord = canonicalBatch.records[index];
-        const storedRecord = batchRecords[index];
-
-        const expectedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
-          expectedRecord,
-        ).pipe(
-          Effect.mapError((error) =>
-            DoStorageCorruptionError.make({
-              table: "effect_agent_canonical_batches",
-              rowKey: key,
-              message: error.message,
-            }),
-          ),
-        );
-
-        const storedJson = yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(
-          storedRecord.decoded,
-        ).pipe(
-          Effect.mapError((error) =>
-            DoStorageCorruptionError.make({
-              table: "effect_agent_canonical_records",
-              rowKey: `${key}/${storedRecord.row.sequence}`,
-              message: error.message,
-            }),
-          ),
-        );
-
-        if (
-          storedRecord.row.sequence !== batchRow.first_sequence + index ||
-          storedRecord.row.record_id !== expectedRecord.recordId ||
-          expectedJson !== storedJson
-        ) {
-          return yield* DoStorageCorruptionError.make({
-            table: "effect_agent_canonical_records",
-            rowKey: `${key}/${storedRecord.row.sequence}`,
-            message: "Canonical record identity, sequence, or payload differs from its batch.",
-          });
-        }
-      }
-
-      previousDigest = digest;
-      expectedSequence = batchRow.last_sequence + 1;
-      tailDigests.set(batchRow.last_sequence, digest);
-    }
-
-    if (
-      threadRecords.length !== thread.tail_sequence ||
-      thread.tail_sequence !== expectedSequence - 1 ||
-      thread.tail_digest !== previousDigest
-    ) {
-      return yield* DoStorageCorruptionError.make({
-        table: "effect_agent_threads",
-        rowKey: thread.thread_id,
-        message: "Thread tail does not match its canonical batch chain.",
-      });
-    }
-
-    for (const checkpoint of threadCheckpoints) {
-      if (
-        checkpoint.decoded.threadId !== thread.thread_id ||
-        checkpoint.decoded.throughSequence !== checkpoint.row.through_sequence ||
-        checkpoint.decoded.tailDigest !== checkpoint.row.tail_digest ||
-        tailDigests.get(checkpoint.row.through_sequence) !== checkpoint.row.tail_digest
-      ) {
-        return yield* DoStorageCorruptionError.make({
-          table: "effect_agent_checkpoints",
-          rowKey: `${thread.thread_id}/${checkpoint.row.through_sequence}`,
-          message: "Checkpoint identity or digest is not bound to a canonical batch tail.",
-        });
-      }
-    }
-  }
-
-  if (
-    batches.some(({ row }) => !materializedIds.has(row.thread_id)) ||
-    records.some(({ row }) => !materializedIds.has(row.thread_id)) ||
-    checkpoints.some(({ row }) => !materializedIds.has(row.thread_id))
-  ) {
-    return yield* DoStorageCorruptionError.make({
-      table: "effect_agent_threads",
-      rowKey: "startup_scan",
-      message: "Canonical rows exist without a materialized Thread.",
-    });
+    if (owners.length < MAX_ARCHIVE_RANGE_PAGE) break;
   }
 });
 
-const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
+const makeServices = Effect.fnUntraced(function* () {
   const config = yield* DoStorageConfig;
   const failpoint = yield* DoStorageFailpoint;
   const sql = yield* SqlClientService.SqlClient;
@@ -551,113 +297,78 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
   const journal = yield* initializeDoJournal(sql, failpoint.hit, config.maxStoredValueBytes);
 
   if (config.verifyOnOpen) {
-    yield* decodeStartupPayloads(journal, crypto);
+    yield* journal.state.read(decodeStartupPayloads(journal)).pipe(
+      Effect.mapError((cause) =>
+        DoStorageCorruptionError.make({
+          table: "effect_agent_journal_ranges",
+          rowKey: "verification",
+          message: cause.message,
+        }),
+      ),
+    );
   }
 
-  const provideCrypto = <A, E>(effect: Effect.Effect<A, E, Crypto.Crypto>) =>
-    Effect.provideService(effect, Crypto.Crypto, crypto);
+  const hitFailpoint = (
+    location: DoStorageFailpointLocation,
+  ): Effect.Effect<void, ThreadStoreError> =>
+    failpoint
+      .hit(location)
+      .pipe(Effect.mapError((error) => storeError(`storage failpoint ${location}`, error)));
 
-  const hitFailpoint = Effect.fn(
-    (location: DoStorageFailpointLocation): Effect.Effect<void, ThreadStoreError> =>
-      failpoint
-        .hit(location)
-        .pipe(Effect.mapError((error) => storeError(`storage failpoint ${location}`, error))),
-  );
+  const materialize: ThreadStore["Service"]["materialize"] = Effect.fnUntraced(function* (
+    request: ThreadMaterialization,
+  ) {
+    const validated = yield* Schema.decodeEffect(Schema.toType(ThreadMaterialization))(
+      request,
+    ).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
 
-  const materialize: ThreadStore["Service"]["materialize"] = Effect.fnUntraced(
-    function* (request: ThreadMaterialization) {
-      const validated = yield* Schema.decodeEffect(Schema.toType(ThreadMaterialization))(
-        request,
-      ).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
-
-      const now = yield* Clock.currentTimeMillis;
-
-      yield* hitFailpoint("materialize:before");
-      yield* journal
-        .materialize(
-          validated.threadId,
-          new Date(now).toISOString(),
-          EMPTY_TAIL_DIGEST,
-          validated.producerEpoch,
-        )
-        .pipe(
-          Effect.mapError((error) =>
-            error._tag === "DoFenceRejected"
-              ? mapFence(validated.threadId, error)
-              : storeError("materialize thread", error),
-          ),
-        );
-      yield* hitFailpoint("materialize:after");
-    },
-    withStorageSpan("DoThreadStore.materialize", (error) => error._tag === "FenceRejected"),
-  );
+    const now = yield* Clock.currentTimeMillis;
 
-  const append: ThreadStore["Service"]["append"] = Effect.fnUntraced(
-    function* (request: FencedAppendRequest) {
-      const validated = yield* Schema.decodeEffect(Schema.toType(FencedAppendRequest))(
-        request,
-      ).pipe(Effect.mapError((error) => schemaStoreError("validate canonical append", error)));
+    yield* hitFailpoint("materialize:before");
+    yield* journal
+      .materialize(validated.threadId, new Date(now).toISOString())
+      .pipe(Effect.mapError((error) => storeError("materialize thread", error)));
+    yield* hitFailpoint("materialize:after");
+  }, withStorageSpan("DoThreadStore.materialize"));
 
+  const append: ThreadStore["Service"]["append"] = Effect.fnUntraced(
+    function* (request: AppendRequest) {
+      const validated = yield* PreparedAppend.capture(request);
       const observed = yield* requireThread(journal, validated.threadId);
 
-      const tailDigest = yield* provideCrypto(
-        digestCanonicalBatch(validated.expectedTailDigest, validated.batch),
-      ).pipe(Effect.mapError((error) => storeError("digest canonical append", error)));
-
-      const batchJson = yield* encodeCanonicalBatch(validated.batch);
-
-      const rawRecords = yield* Effect.forEach(validated.batch.records, (record) =>
-        encodeCanonicalRecord(record).pipe(
-          Effect.map((recordJson) => ({
-            recordId: record.recordId,
-            recordJson,
-          })),
-        ),
+      const { raw: rawRequest } = yield* prepareCanonicalAppend(validated).pipe(
+        Effect.provideService(Crypto.Crypto, crypto),
       );
 
-      const rawRequest = yield* Schema.decodeEffect(RawAppendRequest)({
-        threadId: validated.threadId,
-        batchId: validated.batch.batchId,
-        batchDigest: tailDigest,
-        batchJson,
-        expectedTailSequence: validated.expectedTailSequence,
-        expectedTailDigest: validated.expectedTailDigest,
-        producerEpoch: validated.producerEpoch,
-        records: rawRecords,
-        tailDigest,
-      }).pipe(Effect.mapError((error) => schemaStoreError("encode canonical append", error)));
-
       yield* hitFailpoint("append:before");
 
-      const result = yield* journal.append(rawRequest, observed).pipe(
-        Effect.mapError((error) => {
-          if (isDoFenceRejected(error)) {
-            return mapFence(validated.threadId, error);
-          }
-          if (isDoAppendConflict(error)) {
-            return error.actualTailSequence !== undefined && isDigest(error.actualTailDigest)
-              ? AppendConflict.make({
-                  threadId: validated.threadId,
-                  batchId: validated.batch.batchId,
-                  reason: error.reason,
-                  actualTailSequence: error.actualTailSequence,
-                  actualTailDigest: error.actualTailDigest,
-                })
-              : AppendConflict.make({
-                  threadId: validated.threadId,
-                  batchId: validated.batch.batchId,
-                  reason: error.reason,
-                });
-          }
-
-          return storeError("append canonical batch", error);
-        }),
-        Effect.flatMap((result) =>
-          Schema.decodeEffect(AppendResult)(result).pipe(
-            Effect.mapError((error) => schemaStoreError("decode append result", error)),
+      const result = yield* journal.state.owner
+        .withProgress(rawRequest.threadId, journal.append(rawRequest, observed))
+        .pipe(
+          Effect.mapError((error) => {
+            if (isDoAppendConflict(error)) {
+              return error.actualTailSequence !== undefined
+                ? AppendConflict.make({
+                    threadId: validated.threadId,
+                    batchId: validated.batch.batchId,
+                    reason: error.reason,
+                    actualTailSequence: error.actualTailSequence,
+                  })
+                : AppendConflict.make({
+                    threadId: validated.threadId,
+                    batchId: validated.batch.batchId,
+                    reason: error.reason,
+                  });
+            }
+
+            return storeError("append canonical batch", error);
+          }),
+          Effect.flatMap((result) =>
+            Schema.decodeEffect(AppendResult)(result).pipe(
+              Effect.mapError((error) => schemaStoreError("decode append result", error)),
+            ),
           ),
-        ),
-      );
+        );
 
       yield* hitFailpoint("append:after");
 
@@ -680,7 +391,7 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
     };
   });
 
-  const readEffect = Effect.fn("DoThreadStore.read")(function* (request: ThreadReadRequest) {
+  const readEffect = Effect.fnUntraced(function* (request: ThreadReadRequest) {
     const validated = yield* Schema.decodeEffect(Schema.toType(ThreadReadRequest))(request).pipe(
       Effect.mapError((error) => schemaStoreError("validate thread read", error)),
     );
@@ -758,244 +469,128 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
   const observe: ThreadStore["Service"]["observe"] = (request) =>
     Stream.unwrap(observeEffect(request)).pipe(Stream.withSpan("DoThreadStore.observe"));
 
-  const exportThread: ThreadStore["Service"]["export"] = Effect.fn("DoThreadStore.export")(
-    function* (request: ThreadExportRequest) {
-      const validated = yield* Schema.decodeEffect(Schema.toType(ThreadExportRequest))(
-        request,
-      ).pipe(Effect.mapError((error) => schemaStoreError("validate thread export", error)));
-
-      yield* requireThread(journal, validated.threadId);
-
-      const exported = yield* journal
-        .exportThread(validated.threadId)
-        .pipe(Effect.mapError((error) => storeError("export thread", error)));
-
-      const records = yield* Effect.forEach(exported.records, decodeEnvelope);
-
-      if (records.length > MAX_THREAD_EXPORT_RECORDS) {
-        return yield* ThreadStoreError.make({
-          operation: "decode thread export",
-          message: "The thread exceeds the current export record limit.",
-        });
-      }
-
-      const tailDigest = yield* Schema.decodeEffect(Digest)(exported.thread.tail_digest).pipe(
-        Effect.mapError((error) => schemaStoreError("decode export tail digest", error)),
-      );
-
-      return ThreadExport.make({
-        format: "effect-agent/thread@1",
-        threadId: validated.threadId,
-        tailSequence: exported.thread.tail_sequence,
-        tailDigest,
-        records,
-      });
-    },
-  );
-
-  const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fn("DoThreadStore.inspectTail")(
-    function* (request: ThreadTailRequest) {
-      const validated = yield* Schema.decodeEffect(Schema.toType(ThreadTailRequest))(request).pipe(
-        Effect.mapError((error) => schemaStoreError("validate tail inspection", error)),
-      );
-
-      const thread = yield* requireThread(journal, validated.threadId);
-
-      const tailDigest = yield* Schema.decodeEffect(Digest)(thread.tail_digest).pipe(
-        Effect.mapError((error) => schemaStoreError("decode tail digest", error)),
-      );
-
-      return ThreadTail.make({
-        threadId: validated.threadId,
-        tailSequence: thread.tail_sequence,
-        tailDigest,
-        producerEpoch: thread.producer_epoch,
-      });
-    },
-  );
+  const transfer = yield* makeSqlThreadImport({
+    owner: journal.state.owner,
+    offsetPrefix: DO_OFFSET_PREFIX,
+    maxValueBytes: config.maxStoredValueBytes,
+    // Import establishes a new sequence space; discard derivatives in this same transaction.
+    afterImport: ({ result }) =>
+      Effect.gen(function* () {
+        const prefix = promptCheckpointPrefix(result.threadId);
+
+        yield* sql`DELETE FROM effect_agent_meta WHERE key >= ${prefix} AND key < ${prefix + "~"}`;
+        // Closed imported history must not emit historical lifecycle notifications again.
+        if (journal.lifecycle !== undefined)
+          yield* sql`INSERT INTO effect_agent_lifecycle_cursors(thread_id, through_sequence) VALUES (${result.threadId}, ${result.tailSequence})`;
+      }).pipe(
+        Effect.catchTag("SqlError", (cause) => storeError("rebuild imported derivatives", cause)),
+      ),
+    afterThreadRead: hitFailpoint("export:after-thread-read"),
+    read: (body) =>
+      journal.owner.transaction(body).pipe(
+        Effect.provideService(SqlClientService.SqlClient, sql),
+        Effect.catchIf(isSqlError, (cause) => storeError("export transaction", cause)),
+      ),
+    write: (body) =>
+      journal.owner.transaction(body.pipe(Effect.tap(() => journal.state.invalidate))).pipe(
+        Effect.provideService(SqlClientService.SqlClient, sql),
+        Effect.catchIf(isSqlError, (cause) => storeError("import transaction", cause)),
+      ),
+  });
 
-  const saveCheckpoint: ThreadCheckpoints["save"] = Effect.fn("DoThreadStore.saveCheckpoint")(
-    function* (request: SaveCheckpointRequest) {
-      const validated = yield* Schema.decodeEffect(Schema.toType(SaveCheckpointRequest))(
-        request,
-      ).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
+  const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fnUntraced(function* (
+    request: ThreadTailRequest,
+  ) {
+    const validated = yield* Schema.decodeEffect(Schema.toType(ThreadTailRequest))(request).pipe(
+      Effect.mapError((error) => schemaStoreError("validate tail inspection", error)),
+    );
 
-      const thread = yield* requireThread(journal, validated.checkpoint.threadId);
+    const thread = yield* requireThread(journal, validated.threadId);
 
-      if (validated.checkpoint.throughSequence > thread.tail_sequence) {
-        return yield* CheckpointRejected.make({
-          threadId: validated.checkpoint.threadId,
-          reason: "ahead-of-tail",
-        });
-      }
+    return ThreadTail.make({
+      threadId: validated.threadId,
+      tailSequence: thread.tail_sequence,
+    });
+  });
 
-      const canonicalDigest = yield* tailDigestAt(
-        journal,
-        validated.checkpoint.threadId,
-        validated.checkpoint.throughSequence,
-      );
+  const saveCheckpoint: ThreadCheckpoints["save"] = Effect.fnUntraced(function* (
+    request: SaveCheckpointRequest,
+  ) {
+    const validated = yield* Schema.decodeEffect(Schema.toType(SaveCheckpointRequest))(
+      request,
+    ).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
 
-      if (canonicalDigest !== validated.checkpoint.tailDigest) {
-        return yield* CheckpointRejected.make({
-          threadId: validated.checkpoint.threadId,
-          reason: "digest-mismatch",
-        });
-      }
-      const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
+    const thread = yield* requireThread(journal, validated.checkpoint.threadId);
 
-      const raw = RawCheckpoint.make({
+    if (validated.checkpoint.throughSequence > thread.tail_sequence) {
+      return yield* CheckpointRejected.make({
         threadId: validated.checkpoint.threadId,
-        throughSequence: validated.checkpoint.throughSequence,
-        tailDigest: validated.checkpoint.tailDigest,
-        checkpointJson,
+        reason: "ahead-of-tail",
       });
+    }
 
-      yield* hitFailpoint("save-checkpoint:before");
-      yield* journal.saveCheckpoint(raw).pipe(
-        Effect.mapError((error) =>
-          isDoCheckpointConflict(error)
-            ? CheckpointRejected.make({
-                threadId: validated.checkpoint.threadId,
-                reason: "digest-mismatch",
-              })
-            : storeError("save checkpoint", error),
-        ),
-      );
-      yield* hitFailpoint("save-checkpoint:after");
-    },
-  );
-
-  const loadCheckpoint: ThreadCheckpoints["load"] = Effect.fn("DoThreadStore.loadCheckpoint")(
-    function* (request: LoadCheckpointRequest) {
-      const validated = yield* Schema.decodeEffect(Schema.toType(LoadCheckpointRequest))(
-        request,
-      ).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
-
-      const thread = yield* requireThread(journal, validated.threadId);
-
-      const rows = yield* journal
-        .loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence)
-        .pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
-
-      if (rows.length === 0) return Option.none();
-      if (rows.length !== 1) {
-        return yield* ThreadStoreError.make({
-          operation: "load checkpoint",
-          message: `Expected at most one checkpoint row but found ${rows.length}.`,
-        });
-      }
-      const row = rows[0];
-      const checkpoint = yield* decodeCheckpoint(row.checkpoint_json);
-
-      if (
-        row.thread_id !== validated.threadId ||
-        checkpoint.threadId !== row.thread_id ||
-        checkpoint.throughSequence !== row.through_sequence ||
-        checkpoint.tailDigest !== row.tail_digest
-      ) {
-        return yield* ThreadStoreError.make({
-          operation: "load checkpoint",
-          message: "Stored checkpoint metadata does not match its canonical row.",
-        });
-      }
-
-      const canonicalDigest = yield* tailDigestAt(
-        journal,
-        checkpoint.threadId,
-        checkpoint.throughSequence,
-      );
-
-      if (canonicalDigest !== checkpoint.tailDigest) {
-        return yield* CheckpointRejected.make({
-          threadId: checkpoint.threadId,
-          reason: "digest-mismatch",
-        });
-      }
-
-      return Option.some(checkpoint);
-    },
-  );
-
-  const saveRecoveryCheckpoint: ThreadRecoveryCheckpoints["save"] = Effect.fn(
-    "DoThreadStore.saveRecoveryCheckpoint",
-  )(function* (request) {
-    const validated = yield* Schema.decodeEffect(Schema.toType(SaveRecoveryCheckpointRequest))(
-      request,
-    ).pipe(Effect.mapError((error) => schemaStoreError("validate recovery checkpoint", error)));
-
+    yield* requireBatchBoundary(
+      journal,
+      validated.checkpoint.threadId,
+      validated.checkpoint.throughSequence,
+    );
     const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
 
-    yield* journal
-      .saveRecoveryCheckpoint(validated, checkpointJson)
-      .pipe(
-        Effect.mapError((error) =>
-          error._tag === "CheckpointRejected" ||
-          error._tag === "FenceRejected" ||
-          error._tag === "ThreadNotMaterialized"
-            ? error
-            : storeError("save recovery checkpoint", error),
-        ),
-      );
+    const raw = RawCheckpoint.make({
+      threadId: validated.checkpoint.threadId,
+      throughSequence: validated.checkpoint.throughSequence,
+      checkpointJson,
+    });
+
+    yield* hitFailpoint("save-checkpoint:before");
+    yield* journal.saveCheckpoint(raw).pipe(
+      Effect.mapError((error) =>
+        isDoCheckpointConflict(error)
+          ? CheckpointRejected.make({
+              threadId: validated.checkpoint.threadId,
+              reason: error.reason,
+            })
+          : storeError("save checkpoint", error),
+      ),
+    );
+    yield* hitFailpoint("save-checkpoint:after");
   });
 
-  const loadRecoveryCheckpoint: ThreadRecoveryCheckpoints["load"] = Effect.fn(
-    "DoThreadStore.loadRecoveryCheckpoint",
-  )(function* (request) {
+  const loadCheckpoint: ThreadCheckpoints["load"] = Effect.fnUntraced(function* (
+    request: LoadCheckpointRequest,
+  ) {
     const validated = yield* Schema.decodeEffect(Schema.toType(LoadCheckpointRequest))(
       request,
-    ).pipe(
-      Effect.mapError((error) => schemaStoreError("validate recovery checkpoint lookup", error)),
-    );
+    ).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
 
     const thread = yield* requireThread(journal, validated.threadId);
 
-    const corrupt = () =>
-      CheckpointRejected.make({ threadId: validated.threadId, reason: "corrupt" });
-
     const rows = yield* journal
-      .loadRecoveryCheckpoint(validated.threadId)
-      .pipe(
-        Effect.mapError((error) =>
-          error._tag === "DoStorageCorruptionError"
-            ? corrupt()
-            : storeError("load recovery checkpoint", error),
-        ),
-      );
+      .loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence)
+      .pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
 
     if (rows.length === 0) return Option.none();
-    if (rows.length !== 1) return yield* corrupt();
+    if (rows.length !== 1) {
+      return yield* ThreadStoreError.make({
+        operation: "load checkpoint",
+        message: `Expected at most one checkpoint row but found ${rows.length}.`,
+      });
+    }
     const row = rows[0];
-
-    const checkpoint = yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(
-      row.checkpoint_json,
-    ).pipe(Effect.mapError(corrupt));
+    const checkpoint = yield* decodeCheckpoint(row.checkpoint_json);
 
     if (
       row.thread_id !== validated.threadId ||
       checkpoint.threadId !== row.thread_id ||
-      checkpoint.throughSequence !== row.through_sequence ||
-      checkpoint.tailDigest !== row.tail_digest
-    )
-      return yield* corrupt();
-    if (checkpoint.throughSequence > thread.tail_sequence)
-      return yield* CheckpointRejected.make({
-        threadId: validated.threadId,
-        reason: "ahead-of-tail",
+      checkpoint.throughSequence !== row.through_sequence
+    ) {
+      return yield* ThreadStoreError.make({
+        operation: "load checkpoint",
+        message: "Stored checkpoint metadata does not match its canonical row.",
       });
-    if (checkpoint.throughSequence > (validated.atOrBeforeSequence ?? thread.tail_sequence))
-      return Option.none();
-
-    const canonicalDigest = yield* tailDigestAt(
-      journal,
-      checkpoint.threadId,
-      checkpoint.throughSequence,
-    );
+    }
 
-    if (canonicalDigest !== checkpoint.tailDigest)
-      return yield* CheckpointRejected.make({
-        threadId: validated.threadId,
-        reason: "digest-mismatch",
-      });
+    yield* requireBatchBoundary(journal, checkpoint.threadId, checkpoint.throughSequence);
 
     return Option.some(checkpoint);
   });
@@ -1010,8 +605,6 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
               Schema.UndefinedOr(
                 Schema.Struct({
                   tail_sequence: CanonicalSequence,
-                  tail_digest: Digest,
-                  producer_epoch: ThreadMaterialization.fields.producerEpoch,
                 }),
               ),
             )(rows[0]),
@@ -1022,22 +615,36 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
   );
 
   const threadStore = ThreadStore.of({
+    archives: journal.archives,
+    work: journal.work.storage,
     ...(journal.lifecycle === undefined
       ? {}
       : { lifecyclePublications: journal.lifecycle.storage }),
     readIdentity: selectedReads.readIdentity,
+    readExecution: (request) =>
+      journal.state.read(
+        requireThread(journal, request.threadId).pipe(
+          Effect.andThen(journal.readExecution(request)),
+        ),
+      ),
+    readWorkerCapacity: selectedReads.readWorkerCapacity,
+    verification: transfer.verification,
     countPeerMessages: selectedReads.countPeerMessages,
     append,
-    export: exportThread,
+    export: transfer.export,
     inspectTail,
     materialize,
     observe,
     read,
+    readPrompt: (request) => Stream.fromIterableEffect(selectedReads.readPrompt(request)),
     checkpoints: { save: saveCheckpoint, load: loadCheckpoint },
-    recoveryCheckpoints: { save: saveRecoveryCheckpoint, load: loadRecoveryCheckpoint },
+    promptCheckpoints: makePromptCheckpoints(sql, journal, config.maxStoredValueBytes),
   });
 
-  return Context.make(ThreadStore, threadStore);
+  return Context.make(ThreadStore, threadStore).pipe(
+    Context.add(ThreadImport, transfer.importer),
+    Context.add(StorageOwner, journal.state.owner),
+  );
 });
 
 /**
@@ -1045,10 +652,10 @@ const makeServices = Effect.fn("DoThreadStore.makeServices")(function* () {
  * Crypto authority kept visible in its input channel.
  */
 export const threadStoreLayer: Layer.Layer<
-  ThreadStore,
+  ThreadStore | ThreadReader | ThreadImport | StorageOwner,
   DoStorageInitializationError,
   DoStorageConfig | DoStorageFailpoint | SqlClientService.SqlClient | Crypto.Crypto
-> = Layer.effectContext(makeServices());
+> = ThreadReader.layer().pipe(Layer.provideMerge(Layer.effectContext(makeServices())));
 
 /**
  * Validated Durable Object storage configuration Layer with the documented defaults applied.
@@ -1061,8 +668,6 @@ export const storageConfigLayer = (
   Layer.effect(DoStorageConfig)(
     Schema.decodeEffect(DoStorageConfigValue)({
       observationPollInterval: options.observationPollInterval ?? 25,
-      ownershipLeaseDuration:
-        options.ownershipLeaseDuration ?? Duration.toMillis(DEFAULT_OWNERSHIP_LEASE_DURATION),
       maxStoredValueBytes: options.maxStoredValueBytes ?? DEFAULT_MAX_STORED_VALUE_BYTES,
       verifyOnOpen: options.verifyOnOpen ?? false,
     }).pipe(
@@ -1087,12 +692,15 @@ export const storageFailpointLayer = (
 /**
  * A composition-root convenience Layer for canonical Threads inside one Durable Object,
  * built over `ctx.storage`. Durable accepted work is served by the separate SubmissionLedger
- * port; point both at the SAME `ctx.storage` so claims fence the same producer epochs
+ * port; point both at the SAME `ctx.storage` so both share the same scoped storage owner
  * (ADR-0011 D7's "same file" rule, transposed to one object's private database).
  */
 export const layer = (
   options: DoStorageOptions,
-): Layer.Layer<ThreadStore, DoStorageInitializationError> =>
+): Layer.Layer<
+  ThreadStore | ThreadReader | ThreadImport | StorageOwner,
+  DoStorageInitializationError
+> =>
   Layer.unwrap(
     Effect.map(DoStorageConfig, (config) =>
       threadStoreLayer.pipe(
@@ -1108,6 +716,48 @@ export const layer = (
     ),
   ).pipe(Layer.provide(storageConfigLayer(options)));
 
+/** Read a quiesced current-layout Object without initializing its layout or ownership. */
+export const exportThread = Effect.fn("DoThreadStore.exportThread")(function* (
+  options: Pick<DoStorageOptions, "storage">,
+  request: ThreadExportRequest,
+) {
+  yield* checkExportSource(request.threadId);
+
+  return yield* Effect.gen(function* () {
+    const sql = yield* SqlClientService.SqlClient;
+    const state = yield* ownedState(sql);
+
+    return yield* state
+      .transaction(
+        Effect.gen(function* () {
+          yield* readDoStorageHeader();
+
+          const transfer = yield* makeSqlThreadImport({
+            owner: state.owner,
+            offsetPrefix: DO_OFFSET_PREFIX,
+            read: (body) => body,
+            write: () => Effect.die("A read-only exporter cannot import"),
+          });
+
+          return yield* transfer.export(request);
+        }),
+      )
+      .pipe(
+        Effect.catchTag("SqlError", (cause) =>
+          DoStorageError.make({
+            operation: "export Thread snapshot",
+            message: cause.message,
+            cause,
+          }),
+        ),
+      );
+  }).pipe(
+    Effect.provide(
+      Layer.merge(SqliteClient.layer({ storage: options.storage }), BrowserCrypto.layer),
+    ),
+  );
+});
+
 /**
  * Discard the Object's derived thread and ledger state after direct SQL maintenance.
  * Quiesce port operations during the raw write and invalidation, and enroll the write with
@@ -1116,6 +766,10 @@ export const layer = (
 export const invalidate = invalidateOwnedState;
 
 /** Share the Object's transaction gate with owner-local SQL Memory compositions. */
-export const sqlOwnerLayer = Layer.effect(SqlStorageOwner)(
-  Effect.flatMap(SqlClientService.SqlClient, ownedState),
+export const sqlOwnerLayer = Layer.effectContext(
+  Effect.flatMap(SqlClientService.SqlClient, ownedState).pipe(
+    Effect.map((state) =>
+      Context.make(SqlStorageOwner, state).pipe(Context.add(StorageOwner, state.owner)),
+    ),
+  ),
 );