@ixo/oracle-runtime-workers 0.4.1 → 0.10.0

package/src/shell/app.ts

@@ -26,6 +26,18 @@ import {
   type AuthResult,
   type RouteExclusion,
 } from './auth';
+import { turnBodyTooLarge } from './turn-body-cap';
+import { z } from 'zod';
+
+const DebugEventBody = z.object({
+  type: z
+    .string()
+    .min(1)
+    .max(255)
+    .refine((t) => t !== 'm.room.redaction', 'redactions are not events'),
+  content: z.record(z.string(), z.unknown()).default({}),
+  txnId: z.string().min(1).max(255).optional(),
+});
 
 export interface PluginRoute {
   method: 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'ALL';
@@ -54,6 +66,17 @@ export interface ShellOptions {
 
 type Variables = { auth: AuthResult };
 
+function errorText(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+/** The rejection an in-flight RPC gets when `ctx.abort()` resets its object. */
+function isAbortRejection(err: unknown): boolean {
+  return /debug reset requested|reset requested by the debug route|durable object reset|no longer active|aborted/i.test(
+    errorText(err),
+  );
+}
+
 const BUILTIN_EXCLUSIONS: RouteExclusion[] = [
   { path: '/', method: 'GET' },
   { path: '/health', method: 'ALL' },
@@ -66,9 +89,6 @@ const BUILTIN_EXCLUSIONS: RouteExclusion[] = [
   { path: '/debug/matrix/stop', method: 'POST' },
 ];
 
-/** Express body-parser default in the Node runtime: JSON bodies over 100 KiB are a 413. */
-const MAX_TURN_BODY_BYTES = 100 * 1024;
-
 export function createShell(
   opts: ShellOptions = {},
 ): Hono<{ Bindings: OracleWorkerEnv; Variables: Variables }> {
@@ -267,9 +287,9 @@ export function createShell(
     const identity = identityOf(c.get('auth'), c.req.raw.headers);
     const sessionId = c.req.param('sessionId');
     const body = await c.req.text();
-    // Node (express body-parser default) refuses JSON bodies over 100 KiB
-    // with a 413; attachments are references, so real turns never get near it.
-    if (new TextEncoder().encode(body).byteLength > MAX_TURN_BODY_BYTES) {
+    // Bounds the memory one request can pin (see turn-body-cap.ts); the
+    // status and message are the ones Node's body parser uses.
+    if (turnBodyTooLarge(body)) {
       return c.json(
         { statusCode: 413, message: 'request entity too large' },
         413,
@@ -451,9 +471,33 @@ export function createShell(
   app.post('/debug/storage/flush', async (c) =>
     c.json(await userStub(c.env, c.get('auth').userDid).flushToOwnerStore()),
   );
+  app.post('/debug/storage/tier-flush', async (c) => {
+    const body: unknown = await c.req.json().catch(() => ({}));
+    const rec = typeof body === 'object' && body !== null ? body : {};
+    const force = Reflect.get(rec, 'force') === true;
+    const max = Number(Reflect.get(rec, 'maxSegments'));
+    return c.json(
+      await userStub(c.env, c.get('auth').userDid).tierFlush({
+        force,
+        ...(Number.isInteger(max) && max > 0 && { maxSegments: max }),
+      }),
+    );
+  });
   app.post('/debug/storage/reset', async (c) =>
     c.json(await userStub(c.env, c.get('auth').userDid).resetWorkingCopy()),
   );
+  app.post('/debug/object/abort', async (c) => {
+    try {
+      await userStub(c.env, c.get('auth').userDid).debugAbortObject();
+    } catch (err) {
+      // The abort tears the object down under the call: that rejection is
+      // the expected signal. Anything else (an older build without the
+      // method, a transport error) is a real failure.
+      if (!isAbortRejection(err))
+        return c.json({ aborted: false, error: errorText(err) }, 500);
+    }
+    return c.json({ aborted: true });
+  });
   app.get('/debug/memory-schema', async (c) => {
     const userDid = c.get('auth').userDid;
     return c.json(await userStub(c.env, userDid).debugMemorySchema(userDid));
@@ -469,11 +513,19 @@ export function createShell(
       : c.json({ statusCode: 404, message: 'Session not found' }, 404);
   });
   app.get('/debug/tasks', async (c) =>
-    c.json(await userStub(c.env, c.get('auth').userDid).tasksStatus()),
+    c.json(
+      await userStub(c.env, c.get('auth').userDid).tasksStatus(
+        c.get('auth').userDid,
+      ),
+    ),
   );
   app.get('/debug/realtime', async (c) =>
     c.json(await userStub(c.env, c.get('auth').userDid).realtimeStatus()),
   );
+  app.post('/debug/reauth-prompt/reset', async (c) => {
+    await userStub(c.env, c.get('auth').userDid).debugResetReauthThrottle();
+    return c.json({ reset: true });
+  });
   app.get('/debug/delegation', async (c) =>
     c.json(
       await userStub(c.env, c.get('auth').userDid).delegationStatus(
@@ -487,6 +539,39 @@ export function createShell(
   app.get('/debug/matrix/outbox', async (c) =>
     c.json({ rows: await gateway(c.env).listOutbox() }),
   );
+  // Drill: post a custom event into the caller's own room under a
+  // caller-pinned transaction id. The same id twice returns the same event
+  // (the homeserver deduplicates per device), which is what makes a
+  // best-effort post retried across a gateway restart land once.
+  app.post('/debug/matrix/event', async (c) => {
+    const parsed = DebugEventBody.safeParse(
+      await c.req.json().catch(() => null),
+    );
+    if (!parsed.success)
+      return c.json(
+        { error: 'expected { type: string, content: object, txnId?: string }' },
+        400,
+      );
+    const room = await gateway(c.env).resolveUserRoom(c.get('auth').userDid);
+    if (!room) return c.json({ error: 'no room for this user' }, 404);
+    const { type, content, txnId } = parsed.data;
+    const eventId = await gateway(c.env).sendEvent(
+      room.roomId,
+      type,
+      JSON.stringify(content),
+      txnId === undefined ? {} : { txnId },
+    );
+    return c.json({ roomId: room.roomId, eventId });
+  });
+  app.post('/debug/matrix/abort', async (c) => {
+    try {
+      await gateway(c.env).debugAbortObject();
+    } catch (err) {
+      if (!isAbortRejection(err))
+        return c.json({ aborted: false, error: errorText(err) }, 500);
+    }
+    return c.json({ aborted: true });
+  });
   app.post('/debug/matrix/restart', async (c) =>
     c.json(await gateway(c.env).restart()),
   );

package/src/shell/turn-body-cap.ts

@@ -0,0 +1,19 @@
+/**
+ * Upper bound on a chat turn's request body (`POST /messages/:sessionId`).
+ *
+ * The cap bounds the memory one request can pin: the shell reads the body in
+ * full before forwarding it to the user object. It no longer mirrors the Node
+ * runtime, which keeps Express body-parser's 100 kb default — a normal Portal
+ * turn now exceeds that. Every turn carries the browser-tool catalogue (26–32
+ * tools, 65–70 KiB of JSON schema) and the AG-UI action schemas (~27 KiB)
+ * before the message text, and a compose-topic turn adds the pinned text in
+ * `metadata.contextToolCall`; on devnet turns of 102–115 KiB were refused.
+ * 256 KiB leaves room for those. Attachments never count: files go to Matrix
+ * media first and the body carries only their references.
+ */
+export const MAX_TURN_BODY_BYTES = 256 * 1024;
+
+/** Whether a raw body is over the cap, measured as the UTF-8 bytes the wire carried. */
+export function turnBodyTooLarge(body: string): boolean {
+  return new TextEncoder().encode(body).byteLength > MAX_TURN_BODY_BYTES;
+}

package/src/sqlite/cache-config.ts

@@ -19,22 +19,43 @@ export function parseChunkCacheBytes(
   raw: string | undefined,
   warn: (message: string) => void = () => undefined,
 ): number {
-  if (raw === undefined || raw.trim() === '') return DEFAULT_CHUNK_CACHE_BYTES;
+  return parseByteSize(raw, {
+    name: 'CHUNK_CACHE_BYTES',
+    fallback: DEFAULT_CHUNK_CACHE_BYTES,
+    min: MIN_CHUNK_CACHE_BYTES,
+    max: MAX_CHUNK_CACHE_BYTES,
+    warn,
+  });
+}
+
+/** Same grammar for any byte-sized env knob (`TIER_HOT_BUDGET_BYTES`, ...). */
+export function parseByteSize(
+  raw: string | undefined,
+  options: {
+    name: string;
+    fallback: number;
+    min: number;
+    max: number;
+    warn?: (message: string) => void;
+  },
+): number {
+  const warn = options.warn ?? (() => undefined);
+  if (raw === undefined || raw.trim() === '') return options.fallback;
   const match = /^\s*(\d+)\s*([kKmM]?)\s*$/.exec(raw);
   if (!match) {
     warn(
-      `CHUNK_CACHE_BYTES=${JSON.stringify(raw)} is not a byte count — using the default ${DEFAULT_CHUNK_CACHE_BYTES}`,
+      `${options.name}=${JSON.stringify(raw)} is not a byte count — using the default ${options.fallback}`,
     );
-    return DEFAULT_CHUNK_CACHE_BYTES;
+    return options.fallback;
   }
   const unit = match[2]?.toLowerCase();
   const multiplier = unit === 'k' ? 1024 : unit === 'm' ? 1024 * 1024 : 1;
   const bytes = Number(match[1]) * multiplier;
-  if (bytes < MIN_CHUNK_CACHE_BYTES || bytes > MAX_CHUNK_CACHE_BYTES) {
+  if (bytes < options.min || bytes > options.max) {
     warn(
-      `CHUNK_CACHE_BYTES=${raw} is outside ${MIN_CHUNK_CACHE_BYTES}..${MAX_CHUNK_CACHE_BYTES} — using the default ${DEFAULT_CHUNK_CACHE_BYTES}`,
+      `${options.name}=${raw} is outside ${options.min}..${options.max} — using the default ${options.fallback}`,
     );
-    return DEFAULT_CHUNK_CACHE_BYTES;
+    return options.fallback;
   }
   return bytes;
 }

package/src/sqlite/chunk-layout.ts

@@ -0,0 +1,14 @@
+/**
+ * The storage grain shared by the DO VFS and the R2 page tier. Kept in its
+ * own module so `page-tier.ts` can import it without a cycle through
+ * `do-vfs.ts`.
+ */
+
+/** Bytes per SQLite page. Matches `PRAGMA page_size=4096`. */
+export const VFS_PAGE_SIZE = 4096;
+
+/** SQLite pages packed into one stored row (billing grain). */
+export const PAGES_PER_CHUNK = 16;
+
+/** Bytes per stored chunk row (64 KiB — well under the 2 MB DO value cap). */
+export const CHUNK_SIZE = VFS_PAGE_SIZE * PAGES_PER_CHUNK;

package/src/sqlite/database.ts

@@ -31,14 +31,28 @@ import {
   VFS_PAGE_SIZE,
   type DoVfsOptions,
   type DoVfsStats,
+  type TierFlushResult,
+  type TierStatus,
   type VfsSnapshot,
 } from './do-vfs';
+import type { PageTierOptions } from './page-tier';
 import {
   getOrRegisterVfs,
   loadSqlite,
   type SqliteRuntime,
 } from './wa-sqlite-loader';
 
+/**
+ * Cold-miss retries. A statement outside a transaction re-runs after each
+ * resolve; a transaction rolls back, resolves, and re-runs its callback.
+ * Every resolve fetches whole 1 MiB segments (plus one of read-ahead) and
+ * pins them for the retry, so even a scan across a cold file converges in
+ * a handful of rounds; the caps only guard against a bug that keeps
+ * recording misses.
+ */
+const MAX_STATEMENT_MISS_RETRIES = 64;
+const MAX_TRANSACTION_MISS_RETRIES = 32;
+
 export type SqlValue = null | number | bigint | string | Uint8Array;
 export type SqlParam = SqlValue | boolean | undefined;
 export type SqlParams =
@@ -136,7 +150,10 @@ export class DoSqliteDatabase {
       }
       vfs.attach(ctx.storage);
     }
-    if (!created) vfs.configureCache(options.cachePages);
+    if (!created) {
+      vfs.configureCache(options.cachePages);
+      vfs.configureTier(options.tier);
+    }
     const pragmas = options.pragmas ?? DEFAULT_PRAGMAS;
     const db = await DoSqliteDatabase.openConnection(
       runtime,
@@ -152,6 +169,38 @@ export class DoSqliteDatabase {
     vfs: DoVfs,
     fileName: string,
     pragmas: readonly string[],
+  ): Promise<number> {
+    // Opening reads the file header and the pragmas read page 1 — before
+    // any statement of ours could retry a cold miss. Chunk 0 is never
+    // evicted, but a pin-less retry here keeps a reopen robust regardless.
+    let pinned = false;
+    try {
+      for (let attempt = 0; ; attempt++) {
+        try {
+          return await DoSqliteDatabase.openConnectionOnce(
+            runtime,
+            vfs,
+            fileName,
+            pragmas,
+          );
+        } catch (error) {
+          if (!vfs.hasMisses() || attempt >= MAX_STATEMENT_MISS_RETRIES)
+            throw error;
+          await vfs.resolveMisses();
+          pinned = true;
+          vfs.missRetries++;
+        }
+      }
+    } finally {
+      if (pinned) vfs.releasePins();
+    }
+  }
+
+  private static async openConnectionOnce(
+    runtime: SqliteRuntime,
+    vfs: DoVfs,
+    fileName: string,
+    pragmas: readonly string[],
   ): Promise<number> {
     const db = await runtime.sqlite3.open_v2(
       fileName,
@@ -203,12 +252,54 @@ export class DoSqliteDatabase {
     sql: string,
     params?: SqlParams,
   ): Promise<T[]> {
-    return this.serialized(() => this.execUnlocked<T>(sql, params));
+    return this.serialized(() =>
+      this.retryingColdMisses(() => this.execUnlocked<T>(sql, params)),
+    );
   }
 
   /** Run a statement that produces no rows of interest; returns the change count. */
   async run(sql: string, params?: SqlParams): Promise<RunResult> {
-    return this.serialized(() => this.runUnlocked(sql, params));
+    return this.serialized(() =>
+      this.retryingColdMisses(() => this.runUnlocked(sql, params)),
+    );
+  }
+
+  /**
+   * Run one statement, re-running it after a cold-miss resolve. Only
+   * outside a transaction: inside one, SQLite may have put the pager into
+   * its error state, so the transaction as a whole is rolled back and
+   * retried by `transaction()` instead.
+   */
+  private async retryingColdMisses<T>(op: () => Promise<T>): Promise<T> {
+    if (this.txContext.getStore() !== undefined || this.txDepth > 0)
+      return op();
+    let pinned = false;
+    try {
+      for (let attempt = 0; ; attempt++) {
+        try {
+          const result = await op();
+          // A few statements swallow an I/O error into their result instead
+          // of failing (`PRAGMA integrity_check` reports it as corruption):
+          // misses left behind by a "successful" statement mean exactly
+          // that, so resolve and run it again.
+          if (this.vfs.hasMisses() && attempt < MAX_STATEMENT_MISS_RETRIES) {
+            await this.vfs.resolveMisses();
+            pinned = true;
+            this.vfs.missRetries++;
+            continue;
+          }
+          return result;
+        } catch (error) {
+          if (!this.vfs.hasMisses() || attempt >= MAX_STATEMENT_MISS_RETRIES)
+            throw error;
+          await this.vfs.resolveMisses();
+          pinned = true;
+          this.vfs.missRetries++;
+        }
+      }
+    } finally {
+      if (pinned) this.vfs.releasePins();
+    }
   }
 
   /**
@@ -329,24 +420,50 @@ export class DoSqliteDatabase {
     });
     await previous;
     try {
-      return await this.txContext.run({ depth: 1 }, async () => {
-        await this.run('BEGIN IMMEDIATE');
-        this.txDepth++;
-        try {
-          const result = await fn();
-          this.txDepth--;
-          await this.run('COMMIT');
-          return result;
-        } catch (error) {
-          this.txDepth--;
+      // A cold page inside `fn` fails a statement with SQLITE_IOERR; the
+      // transaction is rolled back, the segment(s) fetched and pinned, and
+      // `fn` re-run from the top (it must be re-runnable: the checkpointer's
+      // and stores' callbacks only issue statements). The pins are released
+      // to the LRU once the transaction ends either way.
+      let pinned = false;
+      try {
+        for (let attempt = 0; ; attempt++) {
           try {
-            await this.run('ROLLBACK');
-          } catch {
-            // The original error is more useful than a rollback failure.
+            return await this.txContext.run({ depth: 1 }, async () => {
+              await this.run('BEGIN IMMEDIATE');
+              this.txDepth++;
+              try {
+                const result = await fn();
+                this.txDepth--;
+                await this.run('COMMIT');
+                // Misses a statement swallowed (see `retryingColdMisses`)
+                // must not trigger a retry of some later, unrelated error.
+                this.vfs.clearMisses();
+                return result;
+              } catch (error) {
+                this.txDepth--;
+                try {
+                  await this.run('ROLLBACK');
+                } catch {
+                  // The original error is more useful than a rollback failure.
+                }
+                throw error;
+              }
+            });
+          } catch (error) {
+            if (
+              !this.vfs.hasMisses() ||
+              attempt >= MAX_TRANSACTION_MISS_RETRIES
+            )
+              throw error;
+            await this.vfs.resolveMisses();
+            pinned = true;
+            this.vfs.missRetries++;
           }
-          throw error;
         }
-      });
+      } finally {
+        if (pinned) this.vfs.releasePins();
+      }
     } finally {
       release();
     }
@@ -387,15 +504,17 @@ export class DoSqliteDatabase {
   static async wipe(
     ctx: DoSqliteContext,
     fileName = 'oracle.db',
+    options: { tier?: PageTierOptions } = {},
   ): Promise<void> {
     const runtime = await loadSqlite();
     const vfsName = vfsNameForObject(ctx);
     const vfs = getOrRegisterVfs(
       runtime,
       vfsName,
-      () => new DoVfs(vfsName, ctx.storage),
+      () => new DoVfs(vfsName, ctx.storage, { tier: options.tier }),
       isDoVfs,
     );
+    vfs.configureTier(options.tier);
     for (const handle of vfs.takeConnections()) {
       try {
         await runtime.sqlite3.close(handle);
@@ -410,6 +529,53 @@ export class DoSqliteDatabase {
     // leaked xOpen handle cannot block the delete.
     vfs.attach(ctx.storage);
     vfs.deleteFile(fileName);
+    // The R2 side of the working copy goes with it (nothing references it
+    // any more; the owner's file upstream is untouched).
+    await vfs.deleteTierObjects();
+  }
+
+  // ---------------------------------------------------------------------------
+  // R2 page tier controls (see do-vfs.ts / page-tier.ts)
+  // ---------------------------------------------------------------------------
+
+  get tierEnabled(): boolean {
+    return this.vfs.tierEnabled;
+  }
+
+  /** One eviction pass; holds the transaction mutex so no write interleaves. */
+  async tierFlush(
+    opts: { force?: boolean; maxSegments?: number } = {},
+  ): Promise<TierFlushResult> {
+    if (this.txDepth > 0)
+      throw new DoVfsError('tierFlush() is not allowed inside a transaction');
+    return this.withoutTransactions(() =>
+      this.vfs.tierFlush(this.fileName, opts),
+    );
+  }
+
+  /** Persist the chunks touched since the last call (call at the end of a turn). */
+  recordAccess(): void {
+    this.vfs.recordAccess(this.fileName);
+  }
+
+  tierStatus(): TierStatus {
+    return this.vfs.tierStatus(this.fileName);
+  }
+
+  /** Delete queued/orphaned R2 objects. */
+  async tierMaintenance(
+    opts: { sweep?: boolean } = {},
+  ): Promise<{ deleted: number }> {
+    return this.vfs.tierMaintenance(opts);
+  }
+
+  /** Pull every cold chunk back into storage (the file is then fully hot). */
+  async materializeTier(): Promise<{ chunks: number }> {
+    if (this.txDepth > 0)
+      throw new DoVfsError(
+        'materializeTier() is not allowed inside a transaction',
+      );
+    return this.withoutTransactions(() => this.vfs.materialize(this.fileName));
   }
 
   /** Close the connection. Buffered pages are flushed by the VFS on close. */
@@ -535,6 +701,9 @@ export class DoSqliteDatabase {
     return this.withoutTransactions(async () => {
       const beforeBytes = this.fileSize;
       if (this.vfs.fileExists(target)) this.vfs.deleteFile(target);
+      // VACUUM reads the whole file through the synchronous path; a cold
+      // chunk would fail and restart it. Bring everything home first.
+      if (this.vfs.tierEnabled) await this.vfs.materialize(this.fileName);
       // `fileName` is a constant of the runtime, never user input.
       await this.run(`VACUUM INTO '${target.replace(/'/g, "''")}'`);
       await this.close();