@ixo/oracle-runtime-workers 0.4.1 → 0.10.0

package/src/do/contracts.ts

@@ -1,6 +1,7 @@
-import type { BotStatus } from '@ixo/matrix-bot-workers-sdk';
+import type { BotStatus, EncryptedFileInfo } from '@ixo/matrix-bot-workers-sdk';
 import type { AttachmentInput } from '../attachments/types';
 import type { RealtimeStatus } from '../realtime/realtime-endpoint';
+import type { TierFlushResult, TierStatus } from '../sqlite/do-vfs';
 /**
  * Cross-object contracts for the Workers runtime.
  *
@@ -82,6 +83,20 @@ export interface OracleWorkerEnv {
    * for the hit/miss counters that show what a budget buys.
    */
   CHUNK_CACHE_BYTES?: string;
+  /**
+   * R2 page tier for the users' working copies (see `sqlite/page-tier.ts`):
+   * chunks no turn touched for `TIER_EVICT_AFTER_PERIODS` periods move
+   * from DO SQLite ($0.20/GB-month, 10 GB cap per object) into 1 MiB R2
+   * segment objects ($0.015/GB-month, no cap) under a per-object prefix.
+   * Absent = every chunk stays in DO storage (the pre-tier behaviour).
+   */
+  TIER_BUCKET?: R2Bucket;
+  /** Soft target for hot bytes per user object (bytes or `16m`; default 16 MiB). */
+  TIER_HOT_BUDGET_BYTES?: string;
+  /** Periods (of `TIER_PERIOD_MS`) a chunk must go untouched before eviction (default 2). */
+  TIER_EVICT_AFTER_PERIODS?: string;
+  /** Length of one access-tracking period in ms (default one day; tests shorten it). */
+  TIER_PERIOD_MS?: string;
   /** IXO VFS worker base URL when `OWNER_STORE=vfs` (defaults per NETWORK). */
   VFS_BASE_URL?: string;
   /** UCAN store worker base URL when `OWNER_STORE=vfs` (defaults per NETWORK). */
@@ -107,6 +122,8 @@ export interface OracleWorkerEnv {
   /** Display markup for `GET /models` prices (default 1.6). */
   MODEL_PRICE_MARKUP?: string;
   MAIN_REASONING_EFFORT?: string;
+  /** LangGraph steps one turn may take before `GraphRecursionError` (default 600; Node hard-codes 200). */
+  TURN_RECURSION_LIMIT?: string;
   /** Platform model provider. Default `openrouter`; `nebius` for self-hosted. */
   LLM_PROVIDER?: 'openrouter' | 'nebius';
   /** Nebius Token Factory API key — required when `LLM_PROVIDER=nebius`. */
@@ -205,6 +222,21 @@ export interface TurnResult {
   messageId?: string;
   /** Tool calls made during the turn (name + summarized status), for logging. */
   toolCalls: Array<{ name: string; status: 'done' | 'error' }>;
+  /** The stored reply of a turn that had already finished (a gateway asked again after a reset). */
+  replayed?: boolean;
+}
+
+/**
+ * Prefix of the error a user object raises when a gateway asks again about a
+ * room message whose turn the object itself lost mid-run (see
+ * `src/do/matrix-turn-ledger.ts`): not re-run, the user is told to retry.
+ */
+export const TURN_INTERRUPTED_MARKER =
+  'turn interrupted by a runtime reset before it finished; not re-run';
+
+export function isInterruptedTurnError(err: unknown): boolean {
+  const message = err instanceof Error ? err.message : String(err);
+  return message.includes(TURN_INTERRUPTED_MARKER);
 }
 
 /** Row shape served by `GET /sessions`. */
@@ -267,6 +299,8 @@ export interface StorageStatus {
   freelistCount?: number;
   /** SQLite's own page cache (`PRAGMA cache_size`; negative = KiB). */
   sqliteCacheSize?: number;
+  /** The R2 page tier of this object's working copy (see sqlite/page-tier.ts). */
+  tier?: TierStatus & { lastPassAt?: number };
   /** The DO VFS clean-chunk cache and its storage counters (isolate lifetime). */
   chunkCache?: {
     budgetBytes: number;
@@ -339,8 +373,17 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
    * the "user deleted their file upstream" path.
    */
   resetWorkingCopy(): Promise<{ reloadedFromOwnerStore: boolean }>;
+  /** Run one R2 page-tier eviction pass now (debug routes; `force` ignores recency). */
+  tierFlush(opts?: {
+    force?: boolean;
+    maxSegments?: number;
+  }): Promise<TierFlushResult>;
   /** Raw upstream schema of the memory search tool as fetched right now. */
   debugMemorySchema(userDid: string): Promise<MemorySchemaDebug>;
+  /** Forcibly reset the object like a platform host drain (debug routes only). The call itself rejects. */
+  debugAbortObject(): Promise<void>;
+  /** Forget when the last re-authorise prompt was posted (debug routes only). */
+  debugResetReauthThrottle(): Promise<void>;
   /** Raw session row for `GET /debug/sessions/:id` (null when unknown). */
   debugSession(
     userDid: string,
@@ -354,7 +397,8 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
    * Diagnostics: the object's current alarm and the user's task records — the
    * pair that answers "is this task going to fire, and when?".
    */
-  tasksStatus(): Promise<{
+  /** Task records, open runs and the alarm; boots the object when cold (`userDid` = the caller). */
+  tasksStatus(userDid?: string): Promise<{
     now: number;
     alarm: number | null;
     schedulerActive: boolean;
@@ -367,6 +411,15 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
       consecutiveFailures: number;
       deliveryRoomId: string | null;
     }>;
+    /** Runs no incarnation has finished: what an alarm will re-deliver or close. */
+    openRuns: Array<{
+      runId: string;
+      taskId: string;
+      startedAt: string;
+      state: 'running' | 'delivering';
+      attempts: number;
+      retryAt: number | null;
+    }>;
   }>;
 }
 
@@ -388,17 +441,21 @@ export interface CreateRoomOptions {
  * rest is the oracle's.
  */
 export interface MatrixGatewayObject extends Rpc.DurableObjectBranded {
-  /** Bytes of a media event (decrypted when it carries an encrypted `file`); null when unknown. */
-  downloadEventMedia(
+  /**
+   * A media event's content as a stream, AS STORED: in an E2EE room the
+   * ciphertext plus the `EncryptedFile` fields to pipe through the SDK's
+   * `createAttachmentDecryptor` on the caller's side (a stream that errors
+   * on the far side of an RPC only surfaces here as a disconnect, so the
+   * hash check belongs to the consumer). Null when unknown or redacted.
+   */
+  downloadEventMediaStream(
     roomId: string,
     eventId: string,
-  ): Promise<{
-    bytes: Uint8Array;
-    mimetype?: string;
-    filename?: string;
-  } | null>;
-  /** Authenticated media download of an `mxc://` URI. */
-  downloadMxcMedia(mxc: string): Promise<Uint8Array>;
+  ): Promise<MediaStream | null>;
+  /** Authenticated media download of an `mxc://` URI, as a stream. */
+  downloadMxcMediaStream(mxc: string): Promise<ReadableStream<Uint8Array>>;
+  /** Forcibly reset the gateway object like a platform host drain (debug routes only). The call itself rejects. */
+  debugAbortObject(): Promise<void>;
   /** Stop the bot (operator/debug; `ensureStarted`/cron brings it back). */
   stop(): Promise<void>;
   /** Idempotent — starts the bot + keep-alive alarm if not running. */
@@ -451,8 +508,18 @@ export interface MatrixGatewayObject extends Rpc.DurableObjectBranded {
       txnId?: string;
     },
   ): Promise<string>;
-  /** Send an arbitrary timeline event (`content` JSON-encoded). Returns the event id. */
-  sendEvent(roomId: string, type: string, content: JsonString): Promise<string>;
+  /**
+   * Send an arbitrary timeline event (`content` JSON-encoded). Returns the
+   * event id. `txnId` pins the transaction id, as for `sendText`: the
+   * homeserver deduplicates by it per device, so a send retried after a lost
+   * response returns the original event instead of posting a second one.
+   */
+  sendEvent(
+    roomId: string,
+    type: string,
+    content: JsonString,
+    opts?: { txnId?: string },
+  ): Promise<string>;
   /** Resolve the canonical user↔oracle room for a user DID (alias lookup, cached). */
   resolveUserRoom(
     userDid: string,
@@ -465,17 +532,23 @@ export interface MatrixGatewayObject extends Rpc.DurableObjectBranded {
    * Returns the new event id. Wire-compatible with the Node runtime so
    * existing users migrate transparently.
    */
-  uploadUserSnapshot(
+  uploadUserSnapshotStream(
     userDid: string,
     storageKey: string,
-    bytes: Uint8Array,
+    /** The gzipped file; encrypted chunk by chunk in an E2EE room, never buffered. */
+    body: ReadableStream<Uint8Array>,
     filename: string,
+    /** Exact byte length of `body` (the homeserver needs a Content-Length). */
+    size: number,
   ): Promise<{ eventId: string }>;
-  /** Download the latest snapshot for `storageKey`, or null when none exists. */
-  downloadUserSnapshot(
+  /**
+   * The latest snapshot for `storageKey` as a stream, as stored (see
+   * `downloadEventMediaStream`), or null when none exists.
+   */
+  downloadUserSnapshotStream(
     userDid: string,
     storageKey: string,
-  ): Promise<{ bytes: Uint8Array; eventId: string } | null>;
+  ): Promise<SnapshotStream | null>;
   /** Read a room state event's content, JSON-encoded (state is never E2EE). Null when absent. */
   getRoomStateEvent(
     roomId: string,
@@ -533,10 +606,32 @@ export interface OutboxRow {
 }
 
 /** `MatrixBotDO.status()` plus the gateway's own turn bookkeeping. */
+/**
+ * Media handed across the Durable Object boundary as a stream, as stored on
+ * the homeserver: `file` is present when the event carried an encrypted
+ * attachment — pipe `stream` through the SDK's `createAttachmentDecryptor`
+ * with it; without `file` the stream is the plain upload. `size` is the
+ * sender's `info.size` when it set one.
+ */
+export interface MediaStream {
+  stream: ReadableStream<Uint8Array>;
+  file?: EncryptedFileInfo;
+  size?: number;
+  mimetype?: string;
+  filename?: string;
+}
+
+/** `MediaStream` of a user's snapshot, plus the media event that carries it. */
+export interface SnapshotStream extends MediaStream {
+  eventId: string;
+}
+
 export interface GatewayStatus extends BotStatus {
   turns: { inFlight: number; waiting: number };
   /** Debounce buffers waiting to become turns. */
   ingestPending: number;
+  /** Room messages whose turn has not ended yet (durable; replayed after a reset). */
+  inbox: number;
 }
 
 /** Derive the durable-object name for a user's object. */

package/src/do/matrix-turn-ledger.ts

@@ -0,0 +1,126 @@
+/**
+ * What the user object remembers about each room message it was asked to
+ * answer, so the gateway can ask again after a reset without the turn ever
+ * running twice (`src/matrix/inbox-store.ts` is the asking side).
+ *
+ * One row per Matrix event id, in the user's own SQLite next to the
+ * checkpoints: `started_at` is written before the turn runs, `reply_text` +
+ * `answered_at` when it has finished. A second request for the same event:
+ *
+ * - answered → the stored text is returned; no model, no tools;
+ * - running in this instance → the caller attaches to the running turn
+ *   (kept in memory by the object, not here);
+ * - started, not answered, not running → the object itself reset mid-turn;
+ *   tools may have run, so the turn is NOT re-run: the request is refused
+ *   with `TURN_INTERRUPTED_MARKER` and the gateway tells the user to try
+ *   again — the same thing that happens today when the object dies while
+ *   the gateway is alive;
+ * - unknown → run once.
+ *
+ * Rows older than `RETENTION_MS` are pruned at setup; a replay that late
+ * cannot happen (the gateway inbox gives up after `MAX_TURN_REPLAYS`).
+ */
+import type { DoSqliteDatabase } from '../sqlite/database';
+
+export interface MatrixTurnRecord {
+  eventId: string;
+  sessionId: string;
+  requestId: string;
+  startedAt: string;
+  answeredAt: string | null;
+  replyText: string | null;
+}
+
+export type MatrixTurnDecision = 'run' | 'answered' | 'interrupted';
+
+/** Rows are kept for a week; the gateway never replays anything that old. */
+export const RETENTION_MS = 7 * 24 * 3600 * 1000;
+
+export function decideMatrixTurn(
+  existing: MatrixTurnRecord | undefined,
+): MatrixTurnDecision {
+  if (!existing) return 'run';
+  return existing.replyText !== null ? 'answered' : 'interrupted';
+}
+
+type LedgerRow = {
+  event_id: string;
+  session_id: string;
+  request_id: string;
+  started_at: string;
+  answered_at: string | null;
+  reply_text: string | null;
+} & Record<string, string | null>;
+
+export class MatrixTurnLedger {
+  private setupPromise: Promise<void> | undefined;
+
+  constructor(
+    readonly db: DoSqliteDatabase,
+    private readonly now: () => number = () => Date.now(),
+  ) {}
+
+  setup(): Promise<void> {
+    this.setupPromise ??= this.doSetup().catch((error: unknown) => {
+      this.setupPromise = undefined;
+      throw error;
+    });
+    return this.setupPromise;
+  }
+
+  private async doSetup(): Promise<void> {
+    await this.db.run(`
+      CREATE TABLE IF NOT EXISTS matrix_turns (
+        event_id TEXT PRIMARY KEY,
+        session_id TEXT NOT NULL,
+        request_id TEXT NOT NULL,
+        started_at TEXT NOT NULL,
+        answered_at TEXT,
+        reply_text TEXT
+      )`);
+    await this.db.run(`DELETE FROM matrix_turns WHERE started_at < ?`, [
+      new Date(this.now() - RETENTION_MS).toISOString(),
+    ]);
+  }
+
+  async get(eventId: string): Promise<MatrixTurnRecord | undefined> {
+    await this.setup();
+    const row = await this.db.get<LedgerRow>(
+      `SELECT event_id, session_id, request_id, started_at, answered_at, reply_text
+       FROM matrix_turns WHERE event_id = ?`,
+      [eventId],
+    );
+    if (!row) return undefined;
+    return {
+      eventId: row.event_id,
+      sessionId: row.session_id,
+      requestId: row.request_id,
+      startedAt: row.started_at,
+      answeredAt: row.answered_at,
+      replyText: row.reply_text,
+    };
+  }
+
+  /** Mark the turn as started. Durable before any model or tool runs. */
+  async start(
+    eventId: string,
+    sessionId: string,
+    requestId: string,
+  ): Promise<void> {
+    await this.setup();
+    await this.db.run(
+      `INSERT OR IGNORE INTO matrix_turns (event_id, session_id, request_id, started_at)
+       VALUES (?, ?, ?, ?)`,
+      [eventId, sessionId, requestId, new Date(this.now()).toISOString()],
+    );
+  }
+
+  /** Record the reply; from now on the event is answered, never re-run. */
+  async answer(eventId: string, replyText: string): Promise<void> {
+    await this.setup();
+    await this.db.run(
+      `UPDATE matrix_turns SET reply_text = ?, answered_at = ? WHERE event_id = ?`,
+      [replyText, new Date(this.now()).toISOString(), eventId],
+    );
+  }
+}

package/src/do/reauth-prompt.ts

@@ -0,0 +1,76 @@
+/**
+ * The `ixo.oracle.delegation_required` prompt: when a room message arrives
+ * and the user has no usable delegation, the object posts an event the web
+ * app listens for to open its "authorize for Matrix" modal in place. It is
+ * throttled per user (`UCAN_REAUTH_PROMPT_THROTTLE_SECONDS`) so the room is
+ * not spammed while the user is away.
+ *
+ * The throttle is stamped only AFTER the homeserver accepted the event. The
+ * earlier order — stamp, then send — turned a send lost to a gateway restart
+ * into six hours of silence: the user kept messaging an oracle that could
+ * not act, and nothing told them why. The send is retried across a restart
+ * under one transaction id, so a response lost after the homeserver accepted
+ * the event is deduplicated rather than posted twice; a second turn arriving
+ * while a prompt is in flight joins it instead of posting another.
+ */
+import { retryTxnId } from '../matrix/txn-id';
+import { retryGateway, type RetryGatewayOptions } from './gateway-retry';
+
+export interface ReauthPromptDeps {
+  throttleMs: number;
+  /** When the last prompt was accepted (undefined = never). */
+  getStamp(): Promise<number | undefined>;
+  setStamp(at: number): Promise<void>;
+  /** Post the event under the transaction id; resolves to its event id. */
+  send(roomId: string, txnId: string): Promise<string>;
+  /** Keeps the (un-awaited) prompt alive past the request that started it: `ctx.waitUntil`. */
+  keepAlive(work: Promise<unknown>): void;
+  log(message: string): void;
+  warn(message: string): void;
+  now?: () => number;
+  retry?: Pick<RetryGatewayOptions, 'delaysMs' | 'sleep' | 'isTransient'>;
+}
+
+const errorText = (err: unknown): string =>
+  err instanceof Error ? err.message : String(err);
+
+export class ReauthPrompter {
+  private inFlight: Promise<void> | null = null;
+
+  constructor(private readonly deps: ReauthPromptDeps) {}
+
+  /** Prompt unless throttled or already in flight. Never rejects. */
+  prompt(userDid: string, roomId: string): Promise<void> {
+    if (this.inFlight) return this.inFlight;
+    const run = this.run(userDid, roomId)
+      .catch((err: unknown) => {
+        this.deps.warn(
+          `[user-do] delegation-required event failed for ${userDid}: ${errorText(err)}`,
+        );
+      })
+      .finally(() => {
+        this.inFlight = null;
+      });
+    this.inFlight = run;
+    this.deps.keepAlive(run);
+    return run;
+  }
+
+  private async run(userDid: string, roomId: string): Promise<void> {
+    const now = this.deps.now?.() ?? Date.now();
+    const last = await this.deps.getStamp();
+    if (last !== undefined && now - last < this.deps.throttleMs) return;
+    const txnId = retryTxnId('reauth');
+    const eventId = await retryGateway(() => this.deps.send(roomId, txnId), {
+      ...this.deps.retry,
+      onRetry: (err, attempt, delayMs) =>
+        this.deps.warn(
+          `[user-do] delegation-required event send failed (attempt ${attempt}) — retrying in ${delayMs} ms: ${errorText(err)}`,
+        ),
+    });
+    await this.deps.setStamp(this.deps.now?.() ?? Date.now());
+    this.deps.log(
+      `[user-do] sent ixo.oracle.delegation_required (${eventId}) to ${roomId} for ${userDid}`,
+    );
+  }
+}

package/src/do/room-mirror.ts

@@ -0,0 +1,111 @@
+/**
+ * The room mirror of HTTP chats: every Portal turn is replayed into the
+ * user's oracle room as a thread under the session's marker event, the user
+ * message when the turn starts and the reply when it ends (Node's
+ * `MessagesService`). Node fires these and forgets them; on Workers the
+ * gateway object is replaced on every deploy and can be drained at any time,
+ * so a mirror sent at that instant fails with a transport error and, unless
+ * retried, is gone for good — the room transcript then disagrees with the
+ * Portal's for ever.
+ *
+ * Two rules make the retry safe:
+ * - each send carries a transaction id derived from the session, the request
+ *   and the author, so a send whose response was lost is deduplicated by the
+ *   homeserver rather than posted twice;
+ * - sends are serialised per session (parallel across sessions), so a reply
+ *   being retried can never overtake the message it answers, and a later
+ *   turn's mirrors queue behind an earlier turn's.
+ */
+import { matrixTxnId } from '../matrix/txn-id';
+import { retryGateway, type RetryGatewayOptions } from './gateway-retry';
+
+export type MirrorAuthor = 'user' | 'oracle';
+
+export function mirrorTxnId(
+  sessionId: string,
+  requestId: string,
+  who: MirrorAuthor,
+): string {
+  return matrixTxnId(
+    'replay',
+    sessionId,
+    requestId,
+    who === 'user' ? 'u' : 'o',
+  );
+}
+
+export interface MirrorSend {
+  roomId: string;
+  body: string;
+  formattedBody?: string;
+  /** The session's marker event — the thread the mirror is posted under. */
+  threadId: string;
+  txnId: string;
+}
+
+export interface RoomMirrorDeps {
+  sendText(send: MirrorSend): Promise<string>;
+  /** Keeps the (un-awaited) send alive past the request that started it: `ctx.waitUntil`. */
+  keepAlive(work: Promise<unknown>): void;
+  log(message: string): void;
+  warn(message: string): void;
+  /** Retry policy override (tests shrink the delays). */
+  retry?: Pick<RetryGatewayOptions, 'delaysMs' | 'sleep' | 'isTransient'>;
+}
+
+const errorText = (err: unknown): string =>
+  err instanceof Error ? err.message : String(err);
+
+export class RoomMirror {
+  /** Per session: the tail of its send chain. Removed once the chain drains. */
+  private readonly chains = new Map<string, Promise<void>>();
+
+  constructor(private readonly deps: RoomMirrorDeps) {}
+
+  /** Sessions with a mirror still in flight. */
+  get pending(): number {
+    return this.chains.size;
+  }
+
+  /**
+   * Queue one mirror behind the session's previous ones. `prepare` runs when
+   * the session's earlier mirrors have settled and resolves what to send
+   * (`null` = nothing to mirror, e.g. no room). Never rejects: a mirror that
+   * cannot be posted is logged and the chain moves on.
+   */
+  enqueue(
+    sessionId: string,
+    prepare: () => Promise<MirrorSend | null>,
+    label: string,
+  ): Promise<void> {
+    const previous = this.chains.get(sessionId) ?? Promise.resolve();
+    const tracked: Promise<void> = previous
+      .then(() => prepare())
+      .then((send) => (send ? this.send(send, label) : undefined))
+      .catch((err: unknown) => {
+        this.deps.warn(
+          `[user-do] Matrix replay (${label}) failed — session=${sessionId}: ${errorText(err)}`,
+        );
+      })
+      .finally(() => {
+        if (this.chains.get(sessionId) === tracked)
+          this.chains.delete(sessionId);
+      });
+    this.chains.set(sessionId, tracked);
+    this.deps.keepAlive(tracked);
+    return tracked;
+  }
+
+  private async send(send: MirrorSend, label: string): Promise<void> {
+    const eventId = await retryGateway(() => this.deps.sendText(send), {
+      ...this.deps.retry,
+      onRetry: (err, attempt, delayMs) =>
+        this.deps.warn(
+          `[user-do] Matrix replay (${label}) send failed (attempt ${attempt}) — retrying in ${delayMs} ms with the same transaction id: ${errorText(err)}`,
+        ),
+    });
+    this.deps.log(
+      `[user-do] Matrix replay (${label}) → ${eventId} in thread ${send.threadId}`,
+    );
+  }
+}