@ixo/oracle-runtime-workers 0.4.1 → 0.10.0

package/src/do/user-oracle-do.ts

@@ -37,6 +37,8 @@ import {
   type AttachmentViewSurface,
   type MatrixMediaSource,
   type SandboxUploadConfig,
+  MAX_FILE_SIZE,
+  readBytesCapped,
 } from '../attachments';
 import type { RuntimeCore } from '../core';
 import {
@@ -81,12 +83,17 @@ import {
   finishCompaction,
   type CompactCursors,
 } from '../sqlite/blob-compactor';
-import { createHash } from 'node:crypto';
 import {
   cachePagesForBytes,
+  parseByteSize,
   parseChunkCacheBytes,
 } from '../sqlite/cache-config';
-import { DoSqliteDatabase } from '../sqlite/database';
+import { DoSqliteDatabase, type DoSqliteOpenOptions } from '../sqlite/database';
+import type { TierFlushResult } from '../sqlite/do-vfs';
+import {
+  DEFAULT_TIER_HOT_BUDGET_BYTES,
+  type PageTierOptions,
+} from '../sqlite/page-tier';
 import {
   shouldVacuum,
   vacuumWanted,
@@ -103,7 +110,9 @@ import {
   UNTITLED_SESSION,
   type SessionRecord,
 } from '../sqlite/sessions-store';
+import { createAttachmentDecryptor } from '@ixo/matrix-bot-workers-sdk';
 import { MatrixMediaOwnerStore } from '../owner-store/matrix-media-store';
+import { measureForSave } from '../owner-store/measure';
 import { MigratingOwnerStore } from '../owner-store/migrating-store';
 import type { OwnerCopy } from '../owner-store/types';
 import {
@@ -135,7 +144,13 @@ import {
   type TurnRequest,
   type TurnResult,
   type MemorySchemaDebug,
+  TURN_INTERRUPTED_MARKER,
 } from './contracts';
+import { turnRecursionLimit } from './turn-config';
+import { decideMatrixTurn, MatrixTurnLedger } from './matrix-turn-ledger';
+import { decideBootDirty } from './boot-dirty';
+import { mirrorTxnId, RoomMirror } from './room-mirror';
+import { ReauthPrompter } from './reauth-prompt';
 import {
   createTaskScheduler,
   TASK_SESSION_PREFIX,
@@ -215,6 +230,14 @@ interface MatrixUserIdCache {
 /** Consecutive flush failures (cleared on success). */
 const META_FLUSH_FAILURES = 'meta:flushFailures';
 const META_LAST_VACUUM = 'meta:lastVacuumAt';
+/** When the last R2 page-tier eviction pass ran (see `tierTick`). */
+const META_LAST_TIER_PASS = 'meta:lastTierPassAt';
+/**
+ * Eviction passes at most this often per object. The horizon is two daily
+ * periods, so four checks a day catch chunks the moment they age out
+ * without re-reading the hot set more than that.
+ */
+const TIER_PASS_INTERVAL_MS = 6 * 60 * 60_000;
 /**
  * Persisted twin of the in-memory `dirty` flag. The flag alone dies with an
  * eviction, and evictions are routine — without the marker an alarm waking a
@@ -239,6 +262,16 @@ const DELEGATION_MISS_TTL_MS = 60_000;
 const META_REAUTH_PROMPT_AT = 'meta:reauthPromptAt';
 /** Node's `AgentBuilder.DEFAULT_REAUTH_THROTTLE_SECONDS`. */
 const DEFAULT_REAUTH_THROTTLE_SECONDS = 6 * 60 * 60;
+
+/** `UCAN_REAUTH_PROMPT_THROTTLE_SECONDS` as a positive number, else the default. */
+function reauthThrottleSeconds(env: OracleWorkerEnv): number {
+  const raw = env.UCAN_REAUTH_PROMPT_THROTTLE_SECONDS;
+  return typeof raw === 'string' &&
+    Number.isFinite(Number(raw)) &&
+    Number(raw) > 0
+    ? Number(raw)
+    : DEFAULT_REAUTH_THROTTLE_SECONDS;
+}
 /** Highest whole-GB size watermark already alerted to Slack (0 = none). */
 const META_SIZE_ALERT_GB = 'meta:sizeAlertGb';
 /** Blob re-compression bookkeeping (see sqlite/blob-compactor.ts). */
@@ -295,22 +328,6 @@ interface FlushResult {
 }
 
 /** Hex SHA-256 of a byte stream, hashed as it flows. */
-async function sha256OfStream(
-  stream: ReadableStream<Uint8Array>,
-): Promise<string> {
-  const hash = createHash('sha256');
-  const reader = stream.getReader();
-  try {
-    for (;;) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      hash.update(value);
-    }
-  } finally {
-    reader.releaseLock();
-  }
-  return hash.digest('hex');
-}
 
 /** Does this adapter expose provider config + role resolution (platform adapters do)? */
 function isProviderAdapter(llm: LlmAdapter): llm is OpenRouterLlmAdapter {
@@ -433,6 +450,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
 
     /** In-flight Matrix turn per session (sessionId → requestId), for supersede. */
     private readonly matrixTurns = new Map<string, string>();
+    /** What this object remembers about each room message it answered (see matrix-turn-ledger.ts). */
+    private matrixLedger: MatrixTurnLedger | null = null;
+    /** Room mirrors of HTTP turns, serialised per session and retried across gateway restarts (see room-mirror.ts). */
+    private mirror: RoomMirror | null = null;
+    /** The oracle room of each session this instance resolved: spares the mirrors a row read per send. */
+    private readonly sessionRooms = new Map<string, string>();
+    /** The throttled `delegation_required` prompt (see reauth-prompt.ts). */
+    private reauthPrompter: ReauthPrompter | null = null;
+    /** Room turns running right now, by Matrix event id: a gateway that asks again attaches instead of re-running. */
+    private readonly matrixTurnRuns = new Map<string, Promise<TurnResult>>();
 
     /** Session-history → memory-engine indexing (built lazily, see `scheduleHistoryIndexing`). */
     private historyIndexer: SessionHistoryIndexer | null = null;
@@ -447,6 +474,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
     >();
     private initPromise: Promise<void> | null = null;
     private reloadedFromOwnerStore = false;
+    /** Set by `adoptOwnerCopy` for a legacy copy: `ready()` flushes it to the system of record right after. */
+    private pendingLegacyMigration: { bytes: number } | null = null;
 
     // ── lifecycle ───────────────────────────────────────────────────────────
 
@@ -792,7 +821,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
           const imported = await this.adoptOwnerCopy(db, loaded);
           this.reloadedFromOwnerStore = true;
           console.log(
-            `[user-do] imported ${imported} bytes from ${this.ownerStore.kind} for ${userDid}`,
+            `[user-do] imported ${imported} bytes from ${loaded.fromLegacy ? 'legacy Matrix media' : this.ownerStore.kind} for ${userDid}`,
           );
         }
       } else {
@@ -926,6 +955,31 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
 
       this.db ??= db;
       const liveDb = this.db;
+      if (this.pendingLegacyMigration) {
+        const { bytes } = this.pendingLegacyMigration;
+        this.pendingLegacyMigration = null;
+        // Second half of the one-time migration: the legacy copy just
+        // imported goes to the system of record from the working copy —
+        // streamed from a snapshot like every flush, so a Node-era history of
+        // any size costs a few chunks of memory. A failure (no
+        // `ixo:filesystem` delegation yet, a VFS outage) keeps the copy dirty
+        // and retried every 10 min; the object serves the legacy history
+        // meanwhile, and the Matrix copy is only redacted once VFS holds it.
+        try {
+          const flushed = await this.flushToOwnerStore();
+          console.log(
+            `[user-do] migrated ${bytes} bytes from legacy Matrix media to ${this.ownerStore.kind} for ${userDid} (${flushed.etag ?? '-'})`,
+          );
+        } catch (err) {
+          console.warn(
+            `[user-do] VFS migration write failed for ${userDid} — serving the legacy Matrix copy for now: ${err instanceof Error ? err.message : String(err)}`,
+          );
+          // Sooner than the 24 h debounce `markDirty` armed: the system of
+          // record is still empty for this user.
+          this.requestAlarm(Date.now() + FLUSH_RETRY_DELAY_MS);
+        }
+      }
+      await this.reconcileDirtyOnBoot(liveDb, userDid);
       {
         // One line per boot so an operator can tell what a user's object
         // actually holds without a debug route: which store, how big, how
@@ -959,6 +1013,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       await this.saver.setup();
       this.sessions = new SessionsStore(liveDb);
       await this.sessions.setup();
+      this.matrixLedger = new MatrixTurnLedger(liveDb);
+      await this.matrixLedger.setup();
 
       const core = this.core;
       // Boot-time plugin hooks + collision checks, once per object (memoised).
@@ -1003,8 +1059,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         logger: console,
       });
       this.workStatus = new WorkStatusProducer({
-        postEvent: (roomId, type, content) =>
-          this.gateway.sendEvent(roomId, type, JSON.stringify(content)),
+        postEvent: (roomId, type, content, opts) =>
+          this.gateway.sendEvent(
+            roomId,
+            type,
+            JSON.stringify(content),
+            opts ?? {},
+          ),
         logger: console,
       });
       this.ambient = createAmbientServices({
@@ -1022,6 +1083,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         delegationFor: (did) => this.delegations.get(did),
         events: this.events,
         secrets: createSecretsAdapter(secretsService),
+        background: (work) => this.ctx.waitUntil(work),
       });
     }
 
@@ -1067,7 +1129,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       if (db) {
         await db.close().catch(() => undefined);
       }
-      await DoSqliteDatabase.wipe(this.ctx, DB_FILE);
+      const tier = this.tierOptions();
+      await DoSqliteDatabase.wipe(this.ctx, DB_FILE, tier ? { tier } : {});
       await this.ctx.storage.delete([
         META_OWNER_ETAG,
         META_LAST_CHECKSUM,
@@ -1078,12 +1141,40 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       ]);
     }
 
-    /** `CHUNK_CACHE_BYTES` → the VFS clean-chunk budget for this object. */
-    private sqliteOpenOptions(): { cachePages: number } {
+    /**
+     * `CHUNK_CACHE_BYTES` → the VFS clean-chunk budget for this object, and
+     * the R2 page tier when the script binds `TIER_BUCKET` (the object's id
+     * is its key prefix; the knobs are documented in `contracts.ts`).
+     */
+    private sqliteOpenOptions(): DoSqliteOpenOptions {
       const bytes = parseChunkCacheBytes(this.env.CHUNK_CACHE_BYTES, (m) =>
         console.warn(`[user-do] ${m}`),
       );
-      return { cachePages: cachePagesForBytes(bytes) };
+      const tier = this.tierOptions();
+      return { cachePages: cachePagesForBytes(bytes), ...(tier && { tier }) };
+    }
+
+    private tierOptions(): PageTierOptions | undefined {
+      const bucket = this.env.TIER_BUCKET;
+      if (!bucket) return undefined;
+      const warn = (m: string): void => console.warn(`[user-do] ${m}`);
+      const hotBudgetBytes = parseByteSize(this.env.TIER_HOT_BUDGET_BYTES, {
+        name: 'TIER_HOT_BUDGET_BYTES',
+        fallback: DEFAULT_TIER_HOT_BUDGET_BYTES,
+        min: 1024 * 1024,
+        max: 1024 * 1024 * 1024,
+        warn,
+      });
+      const periods = Number(this.env.TIER_EVICT_AFTER_PERIODS);
+      const periodMs = Number(this.env.TIER_PERIOD_MS);
+      return {
+        bucket,
+        prefix: this.ctx.id.toString(),
+        hotBudgetBytes,
+        ...(Number.isInteger(periods) &&
+          periods >= 1 && { evictAfterPeriods: periods }),
+        ...(Number.isFinite(periodMs) && periodMs >= 1000 && { periodMs }),
+      };
     }
 
     /**
@@ -1094,10 +1185,25 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
      */
     private async adoptOwnerCopy(
       db: DoSqliteDatabase,
-      copy: { stream: ReadableStream<Uint8Array>; etag: string },
+      copy: OwnerCopy,
     ): Promise<number> {
       const bytes = await db.importFromStream(copy.stream);
       await this.ctx.storage.put(META_OWNER_ETAG, copy.etag);
+      if (copy.fromLegacy) {
+        // Streamed in from the legacy Matrix media, not in the system of
+        // record yet: forget any earlier upload markers so the flush that
+        // `ready()` runs next (`pendingLegacyMigration`) actually sends it,
+        // and forget "no legacy copy, stop looking" — a copy evidently
+        // exists now, and the flush must be able to redact it.
+        await this.ctx.storage.delete([
+          META_LAST_CHECKSUM,
+          META_UPLOADED_GEN,
+          META_LEGACY_CLEARED,
+        ]);
+        this.markDirty();
+        this.pendingLegacyMigration = { bytes };
+        return bytes;
+      }
       await this.ctx.storage.put(META_LAST_CHECKSUM, await db.checksum());
       await this.ctx.storage.put(META_UPLOADED_GEN, db.writeGeneration);
       return bytes;
@@ -1226,8 +1332,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       }
 
       // Legacy-blob compaction: one bounded batch per tick until the whole
-      // file is codec-compressed (see sqlite/blob-compactor.ts).
-      if (this.db) {
+      // file is codec-compressed (see sqlite/blob-compactor.ts). A flush in
+      // flight holds a snapshot of the file, which the compactor's swap-in
+      // cannot replace — a large migration flush (streamed, seconds long)
+      // routinely overlaps the first tick, so wait for it instead of erroring.
+      if (this.db && this.flushInFlight) {
+        deadlines.push(now + 5000);
+      } else if (this.db) {
         try {
           if (await this.compactionTick(this.db)) deadlines.push(now + 2000);
         } catch (err) {
@@ -1249,6 +1360,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         if (next !== null) deadlines.push(next);
       }
 
+      // R2 page tier: move chunks no turn touched for a couple of days out
+      // of DO storage (see sqlite/page-tier.ts). After the flush above (the
+      // upload reads hot rows, not R2) and never over an in-flight one (it
+      // holds a snapshot). A pass is capped; more work re-arms soon.
+      if (this.db?.tierEnabled && !this.flushInFlight) {
+        const next = await this.tierTick(this.db, now);
+        if (next !== null) deadlines.push(next);
+      }
+
       // Idle housekeeping: a user silent for a day loses the cached pages —
       // their file is safe upstream, and we stop paying to store a copy. Never
       // evict while tasks are scheduled: their runs ARE activity. The flush
@@ -1277,6 +1397,71 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       );
     }
 
+    /**
+     * One page-tier eviction pass, at most once per `TIER_PASS_INTERVAL_MS`
+     * (a pass that hit its segment cap re-arms in a minute). Returns the next
+     * time a pass is due, or null when nothing is pending.
+     */
+    private async tierTick(
+      db: DoSqliteDatabase,
+      now: number,
+    ): Promise<number | null> {
+      const last =
+        (await this.ctx.storage.get<number>(META_LAST_TIER_PASS)) ?? 0;
+      if (now - last < TIER_PASS_INTERVAL_MS)
+        return last + TIER_PASS_INTERVAL_MS;
+      try {
+        const pass = await db.tierFlush();
+        await this.ctx.storage.put(META_LAST_TIER_PASS, now);
+        if (pass.skipped) {
+          if (pass.skipped !== 'no-tier')
+            console.log(
+              `[user-do] tier pass skipped for ${this.userDid}: ${pass.skipped}`,
+            );
+          return now + 60_000;
+        }
+        if (pass.evictedChunks > 0 || pass.remaining > 0) {
+          console.log(
+            `[user-do] tier pass for ${this.userDid}: ${pass.evictedChunks} chunks → R2 in ${pass.segmentsRewritten} segment(s), ${pass.hotRows} hot rows (${Math.round(pass.hotBytes / 1024 / 1024)} MB) kept, ${pass.remaining} segment(s) pending`,
+          );
+        }
+        const budget = this.tierOptions()?.hotBudgetBytes;
+        if (
+          budget !== undefined &&
+          pass.remaining === 0 &&
+          pass.hotBytes > budget
+        ) {
+          // Everything left is recent: the user's real working set exceeds
+          // the target — a sizing signal, not a failure.
+          console.warn(
+            `[user-do] tier hot set of ${this.userDid} is ${Math.round(pass.hotBytes / 1024 / 1024)} MB, over the ${Math.round(budget / 1024 / 1024)} MB target (all of it touched within the eviction horizon)`,
+          );
+        }
+        return pass.remaining > 0 ? now + 60_000 : now + TIER_PASS_INTERVAL_MS;
+      } catch (err) {
+        console.error(
+          `[user-do] tier pass failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+        return now + FLUSH_RETRY_DELAY_MS;
+      }
+    }
+
+    /** Operator/test entry: run an eviction pass now (debug routes). */
+    async tierFlush(
+      opts: { force?: boolean; maxSegments?: number } = {},
+    ): Promise<TierFlushResult> {
+      if (!this.db) {
+        const userDid =
+          this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
+        if (userDid) await this.ready({ userDid });
+      }
+      if (!this.db) throw new Error('no working copy');
+      if (this.flushInFlight) await this.flushInFlight.catch(() => undefined);
+      const pass = await this.db.tierFlush(opts);
+      await this.ctx.storage.put(META_LAST_TIER_PASS, Date.now());
+      return pass;
+    }
+
     /**
      * One VACUUM policy check. Returns the time of a follow-up check when a
      * rebuild is wanted but the object is not quiet yet, else null.
@@ -1579,42 +1764,70 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
     }
 
     /**
-     * Emit a single throttled `ixo.oracle.delegation_required` event into the
-     * user's room when a Matrix turn finds no valid delegation; the web app
-     * listens for it and opens the authorize modal in place. Throttled per
-     * object (`UCAN_REAUTH_PROMPT_THROTTLE_SECONDS`, default 6 h). Never throws.
+     * Ask the web app to open its "authorize for Matrix" modal in place when a
+     * Matrix turn finds no valid delegation — the Node AgentBuilder's
+     * `maybePromptReauth`, throttled per object
+     * (`UCAN_REAUTH_PROMPT_THROTTLE_SECONDS`, default 6 h). Never throws; the
+     * prompt outlives the turn under `waitUntil` while it is retried across a
+     * gateway restart (see reauth-prompt.ts for the ordering that matters).
      */
-    private async maybePromptReauth(
+    private promptReauth(userDid: string, roomId: string): void {
+      this.reauthPrompter ??= new ReauthPrompter({
+        throttleMs: reauthThrottleSeconds(this.env) * 1000,
+        getStamp: () => this.ctx.storage.get<number>(META_REAUTH_PROMPT_AT),
+        setStamp: (at) => this.ctx.storage.put(META_REAUTH_PROMPT_AT, at),
+        send: (room, txnId) =>
+          this.gateway.sendEvent(
+            room,
+            'ixo.oracle.delegation_required',
+            JSON.stringify({
+              oracleEntityDid: this.core.identity.entityDid,
+              oracleDid: this.env.ORACLE_DID,
+            }),
+            { txnId },
+          ),
+        keepAlive: (work) => this.ctx.waitUntil(work),
+        log: (message) => console.log(message),
+        warn: (message) => console.warn(message),
+      });
+      void this.reauthPrompter.prompt(userDid, roomId);
+    }
+
+    /** Debug route: forget when the last re-authorise prompt was posted, so a drill can trigger the next one. */
+    async debugResetReauthThrottle(): Promise<void> {
+      await this.ctx.storage.delete(META_REAUTH_PROMPT_AT);
+    }
+
+    /**
+     * A turn that died mid-way leaves committed checkpoint steps with no dirty
+     * mark (the mark is set at the end of a turn). Compare the file's write
+     * generation with the last upload's on boot and mark the copy dirty when
+     * it moved on; one batched storage read per boot (see boot-dirty.ts).
+     */
+    private async reconcileDirtyOnBoot(
+      db: DoSqliteDatabase,
       userDid: string,
-      roomId: string,
     ): Promise<void> {
-      try {
-        const raw = this.env.UCAN_REAUTH_PROMPT_THROTTLE_SECONDS;
-        const throttleSeconds =
-          typeof raw === 'string' &&
-          Number.isFinite(Number(raw)) &&
-          Number(raw) > 0
-            ? Number(raw)
-            : DEFAULT_REAUTH_THROTTLE_SECONDS;
-        const last = await this.ctx.storage.get<number>(META_REAUTH_PROMPT_AT);
-        if (last !== undefined && Date.now() - last < throttleSeconds * 1000)
-          return;
-        await this.ctx.storage.put(META_REAUTH_PROMPT_AT, Date.now());
-        await this.gateway.sendEvent(
-          roomId,
-          'ixo.oracle.delegation_required',
-          JSON.stringify({
-            oracleEntityDid: this.core.identity.entityDid,
-            oracleDid: this.env.ORACLE_DID,
-          }),
-        );
+      if (this.dirty) return;
+      const got = await this.ctx.storage.get<unknown>([
+        META_DIRTY,
+        META_UPLOADED_GEN,
+      ]);
+      const decision = decideBootDirty({
+        dirtyInMemory: this.dirty,
+        dirtyFlag: got.get(META_DIRTY),
+        uploadedGen: got.get(META_UPLOADED_GEN),
+        writeGeneration: db.writeGeneration,
+      });
+      if (decision === 'flagged') {
+        // The mark is on disk already (its flush alarm too); just adopt it.
+        this.dirty = true;
+        this.requestAlarm(Date.now() + FLUSH_DEBOUNCE_MS);
+      } else if (decision === 'behind-upload') {
         console.log(
-          `[user-do] sent ixo.oracle.delegation_required to ${roomId} for ${userDid}`,
-        );
-      } catch (err) {
-        console.warn(
-          `[user-do] delegation-required event failed for ${userDid}: ${err instanceof Error ? err.message : String(err)}`,
+          `[user-do] working copy of ${userDid} is at generation ${db.writeGeneration}, last upload at ${String(got.get(META_UPLOADED_GEN))} — a turn ended without marking it; flushing`,
         );
+        this.markDirty();
       }
     }
 
@@ -1656,8 +1869,68 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       return true;
     }
 
+    /**
+     * A room turn runs at most once per Matrix event, however many times the
+     * gateway asks (it asks again after a reset, see the inbox in
+     * `src/matrix/gateway-do.ts`). The ledger and the in-flight map decide:
+     * answered → stored reply; running → attach; started but lost in a reset
+     * of this object → refused (tools may have run), the user is told to
+     * retry; unknown → run once. HTTP and task turns carry no event id and
+     * always run.
+     */
     async runTurn(req: TurnRequest): Promise<TurnResult> {
       await this.ready(req.identity);
+      const eventId = req.client === 'matrix' ? req.eventId : undefined;
+      const ledger = this.matrixLedger;
+      if (!eventId || !ledger) return this.runTurnOnce(req);
+
+      const running = this.matrixTurnRuns.get(eventId);
+      if (running) {
+        console.log(
+          `[user-do] turn for ${eventId} is still running; the gateway asked again and attaches to it`,
+        );
+        return running;
+      }
+      const existing = await ledger.get(eventId);
+      switch (decideMatrixTurn(existing)) {
+        case 'answered':
+          console.log(
+            `[user-do] turn for ${eventId} already answered; returning the stored reply`,
+          );
+          return {
+            sessionId: req.sessionId,
+            requestId: req.requestId,
+            text: existing?.replyText ?? '',
+            toolCalls: [],
+            replayed: true,
+          };
+        case 'interrupted':
+          throw new Error(`${TURN_INTERRUPTED_MARKER} (${eventId})`);
+        case 'run':
+          break;
+      }
+      await ledger.start(eventId, req.sessionId, req.requestId);
+      const run = this.runTurnOnce(req)
+        .then(async (result) => {
+          await ledger.answer(eventId, result.text);
+          return result;
+        })
+        .finally(() => {
+          this.matrixTurnRuns.delete(eventId);
+        });
+      this.matrixTurnRuns.set(eventId, run);
+      // The turn's I/O belongs to this RPC's request context. If the gateway
+      // that made the call is reset, the context would be cancelled with it:
+      // fetches started afterwards (the next model call, the next tool) never
+      // settle, and the turn hangs forever — with a replay attached to it.
+      // `waitUntil` keeps the context alive until the turn has ended, so the
+      // turn finishes, the ledger records the reply, and the replayed request
+      // returns it.
+      this.ctx.waitUntil(run.catch(() => undefined));
+      return run;
+    }
+
+    private async runTurnOnce(req: TurnRequest): Promise<TurnResult> {
       // Matrix turns drive a `work_status` card in the user's thread; a newer
       // message on the same session supersedes the previous card (the
       // previous turn itself is aborted by prepareTurn).
@@ -1809,6 +2082,13 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
      * while one runs joins it.
      */
     async flushToOwnerStore(): Promise<FlushResult> {
+      if (!this.db) {
+        // A cold (evicted) object: boot from the persisted identity first, so
+        // an operator's `POST /debug/storage/flush` is never a silent no-op.
+        const userDid =
+          this.userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
+        if (userDid) await this.ready({ userDid });
+      }
       if (this.flushInFlight) return this.flushInFlight;
       const run = this.flushOnce().finally(() => {
         this.flushInFlight = null;
@@ -1836,7 +2116,11 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
 
       const snapshot = await db.snapshot();
       try {
-        const checksum = await sha256OfStream(snapshot.open());
+        // One pass yields the change gate (hash) AND the length the store
+        // needs to upload; the upload is the only other read of the file.
+        const { sha256Hex: checksum, gzippedLength } = await measureForSave(
+          snapshot.open(),
+        );
         const last = await this.ctx.storage.get<string>(META_LAST_CHECKSUM);
         if (last === checksum) {
           await this.ctx.storage.put(META_UPLOADED_GEN, snapshot.generation);
@@ -1851,7 +2135,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
 
         let saved: { etag: string; bytes: number };
         try {
-          saved = await store.save(snapshot);
+          saved = await store.save(snapshot, { gzippedLength });
         } catch (err) {
           // Never diverted anywhere else: the working copy in Durable Object
           // storage is durable, stays dirty, and is retried. Loud from the
@@ -1976,6 +2260,8 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       this.db = null;
       this.saver = null;
       this.sessions = null;
+      this.matrixLedger = null;
+      this.sessionRooms.clear();
       this.ambient = null;
       this.secretsService = null;
       this.byo = null;
@@ -1994,9 +2280,21 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
     async debugSession(
       userDid: string,
       sessionId: string,
-    ): Promise<SessionRecord | null> {
+    ): Promise<Record<string, unknown> | null> {
       await this.ready({ userDid });
-      return (await this.sessions!.getSession(sessionId)) ?? null;
+      const session = await this.sessions!.getSession(sessionId);
+      if (!session) return null;
+      // Whether the thread's agent context was condensed: the summarization
+      // middleware's bookkeeping message is stored with the history but
+      // never listed, so `GET /messages/:id` alone cannot tell.
+      const all = await this.saver!.listThreadMessages(sessionId);
+      return {
+        ...session,
+        threadMessages: all.length,
+        summaryMessages: all.filter(isSummarizationMessage).length,
+        // Folded into their assistant messages by the listing, not shown alone.
+        toolMessages: all.filter((m) => m.type === 'tool').length,
+      };
     }
 
     /**
@@ -2004,6 +2302,16 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
      * this user's credentials and return `search_memory_engine`'s raw schema
      * plus whether it converts to Zod. Same headers the memory plugin sends.
      */
+    /**
+     * Operator / testing aid behind `ORACLE_DEBUG_ROUTES`: reset this object
+     * the way an unplanned platform reset does — in-memory state and every
+     * in-flight turn are gone, committed storage survives, the next request
+     * boots a fresh instance. The RPC rejects by design.
+     */
+    async debugAbortObject(): Promise<void> {
+      this.ctx.abort('debug reset requested');
+    }
+
     async debugMemorySchema(userDid: string): Promise<MemorySchemaDebug> {
       await this.ready({ userDid });
       const ambient = this.ambient;
@@ -2092,10 +2400,17 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
               rowsWritten: vfs.rowsWritten,
             }
           : undefined,
+        tier: db
+          ? {
+              ...db.tierStatus(),
+              lastPassAt:
+                await this.ctx.storage.get<number>(META_LAST_TIER_PASS),
+            }
+          : undefined,
       };
     }
 
-    async tasksStatus(): Promise<{
+    async tasksStatus(userDid?: string): Promise<{
       now: number;
       alarm: number | null;
       schedulerActive: boolean;
@@ -2107,14 +2422,41 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         lastResult: { ok: boolean; summary: string; at: string } | null;
         consecutiveFailures: number;
       }>;
+      openRuns: Array<{
+        runId: string;
+        taskId: string;
+        startedAt: string;
+        state: 'running' | 'delivering';
+        attempts: number;
+        retryAt: number | null;
+      }>;
     }> {
+      // A cold (evicted or reset) object has no scheduler until it boots;
+      // reporting "no tasks" for it would be wrong, so boot like the alarm
+      // does when the caller — or the stored identity — says whose object.
+      if (!this.taskScheduler) {
+        const did =
+          userDid ?? (await this.ctx.storage.get<string>(META_USER_DID));
+        if (did) await this.ready({ userDid: did });
+      }
       const records = this.taskScheduler
         ? await this.taskScheduler.surface.list()
         : [];
+      const openRuns = this.taskScheduler
+        ? await this.taskScheduler.openRuns()
+        : [];
       return {
         now: Date.now(),
         alarm: await this.ctx.storage.getAlarm(),
         schedulerActive: this.taskScheduler !== null,
+        openRuns: openRuns.map((r) => ({
+          runId: r.runId,
+          taskId: r.taskId,
+          startedAt: r.startedAt,
+          state: r.state,
+          attempts: r.attempts,
+          retryAt: r.retryAt ?? null,
+        })),
         tasks: records.map((t) => ({
           id: t.id,
           title: t.title,
@@ -2392,12 +2734,36 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
 
     // ── turn plumbing ──────────────────────────────────────────────────────
 
-    /** Matrix media access for the attachments pipeline, through the gateway. */
+    /**
+     * Matrix media access for the attachments pipeline, through the gateway.
+     * Both transfers stream across the object boundary and are cut off at
+     * the attachment cap; an encrypted attachment is decrypted HERE, not in
+     * the gateway (its whole-buffer download grew the crypto WASM heap for
+     * good, and a hash mismatch on the far side of the RPC would only reach
+     * this object as a disconnect).
+     */
     private matrixMediaSource(): MatrixMediaSource {
       return {
-        downloadMxc: (mxc) => this.gateway.downloadMxcMedia(mxc),
-        downloadEvent: (roomId, eventId) =>
-          this.gateway.downloadEventMedia(roomId, eventId),
+        downloadMxc: async (mxc, maxBytes = MAX_FILE_SIZE) =>
+          readBytesCapped(
+            await this.gateway.downloadMxcMediaStream(mxc),
+            maxBytes,
+          ),
+        downloadEvent: async (roomId, eventId, maxBytes = MAX_FILE_SIZE) => {
+          const media = await this.gateway.downloadEventMediaStream(
+            roomId,
+            eventId,
+          );
+          if (!media) return null;
+          const plain = media.file
+            ? media.stream.pipeThrough(createAttachmentDecryptor(media.file))
+            : media.stream;
+          return {
+            bytes: await readBytesCapped(plain, maxBytes),
+            ...(media.mimetype ? { mimetype: media.mimetype } : {}),
+            ...(media.filename ? { filename: media.filename } : {}),
+          };
+        },
       };
     }
 
@@ -2672,6 +3038,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
           `[user-do] no oracle room resolved for ${req.identity.userDid}; room-scoped plugins (memory) will be unavailable this turn`,
         );
       }
+      if (sessionRoomId) this.sessionRooms.set(req.sessionId, sessionRoomId);
 
       // A Matrix turn with no usable delegation: ask the web app to open its
       // "authorize for Matrix" modal (throttled per user, best-effort) — the
@@ -2681,7 +3048,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         sessionRoomId &&
         !this.delegations.get(req.identity.userDid)?.raw
       ) {
-        void this.maybePromptReauth(req.identity.userDid, sessionRoomId);
+        this.promptReauth(req.identity.userDid, sessionRoomId);
       }
 
       // Per-room user preferences (tone, language, what to call whom) —
@@ -2873,7 +3240,7 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
         configurable: { thread_id: req.sessionId },
         context,
         signal: abortController.signal,
-        recursionLimit: 60,
+        recursionLimit: turnRecursionLimit(this.env),
         metadata: tracing.metadata,
         ...(tracing.callbacks ? { callbacks: tracing.callbacks } : {}),
       };
@@ -2909,40 +3276,57 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
       if (!req.sessionId.startsWith('$')) return;
       if (!text.trim()) return;
       const label = who === 'user' ? 'user message' : 'AI response';
-      // `waitUntil`: the non-streaming handler returns its Response right
-      // after the turn, and un-awaited I/O started inside a request is torn
-      // down with it ("Network connection lost") unless the object is told
-      // to keep the work alive.
-      this.ctx.waitUntil(
-        (async () => {
+      // Serialised per session, retried across a gateway restart with a fixed
+      // transaction id, kept alive past the request (room-mirror.ts).
+      void this.roomMirror().enqueue(
+        req.sessionId,
+        async () => {
           const roomId =
+            this.sessionRooms.get(req.sessionId) ??
             req.roomId ??
             (await this.sessions?.getSession(req.sessionId))?.roomId ??
-            (await this.gateway.resolveUserRoom(req.identity.userDid))?.roomId;
+            (
+              await retryGateway(() =>
+                this.gateway.resolveUserRoom(req.identity.userDid),
+              )
+            )?.roomId;
           if (!roomId) throw new Error('no oracle room for this user');
+          this.sessionRooms.set(req.sessionId, roomId);
           const { body, formattedBody } = formatReplay({
             message: text,
             isOracle: who === 'oracle',
             oracleName: this.core.identity.name,
           });
-          const eventId = await this.gateway.sendText(roomId, body, {
+          return {
+            roomId,
+            body,
+            ...(formattedBody ? { formattedBody } : {}),
             threadId: req.sessionId,
-            formattedBody,
-            priority: 'background',
-          });
-          console.log(
-            `[user-do] Matrix replay (${label}) → ${eventId} in thread ${req.sessionId}`,
-          );
-        })().catch((err: unknown) => {
-          console.error(
-            `[user-do] Matrix replay (${label}) failed — session=${req.sessionId}: ${
-              err instanceof Error ? err.message : String(err)
-            }`,
-          );
-        }),
+            txnId: mirrorTxnId(req.sessionId, req.requestId, who),
+          };
+        },
+        label,
       );
     }
 
+    private roomMirror(): RoomMirror {
+      this.mirror ??= new RoomMirror({
+        sendText: (send) =>
+          this.gateway.sendText(send.roomId, send.body, {
+            threadId: send.threadId,
+            ...(send.formattedBody
+              ? { formattedBody: send.formattedBody }
+              : {}),
+            priority: 'background',
+            txnId: send.txnId,
+          }),
+        keepAlive: (work) => this.ctx.waitUntil(work),
+        log: (message) => console.log(message),
+        warn: (message) => console.warn(message),
+      });
+      return this.mirror;
+    }
+
     private async afterTurn(
       sessionId: string,
       messages: BaseMessage[],
@@ -2963,6 +3347,15 @@ export function createUserOracleDO(opts: UserOracleDOOptions) {
           await sessions.setTitle(sessionId, title, { onlyIfUntitled: true });
       }
       this.markDirty();
+      // Which chunks this turn touched, for the page tier's eviction policy
+      // (one row write; a no-op without a tier).
+      try {
+        this.db?.recordAccess();
+      } catch (err) {
+        console.warn(
+          `[user-do] tier access record failed for ${this.userDid}: ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
     }
 
     private async generateTitle(