@ixo/oracle-runtime-workers 0.12.0 → 0.15.0

package/src/matrix/gateway-do.ts

@@ -57,6 +57,7 @@ import {
   SIGNING_MNEMONIC_STATE_TYPE,
 } from '../secrets/signing-mnemonic';
 import {
+  attributeUser,
   IngestPipeline,
   userDidFromRoomAlias,
   type InboundAttachment,
@@ -75,6 +76,7 @@ import {
   MAX_TURN_REPLAYS,
   planInboxReplay,
   replyTxnId,
+  settleOfferedRow,
   updateInboxThread,
 } from './inbox-store';
 import {
@@ -90,14 +92,48 @@ import {
   resolveReplyChainRoot,
   ThreadRootCache,
 } from './reply-chain';
-import { fetchUserMatrixServerName } from './user-homeserver';
+import {
+  type CachedUserServerName,
+  lookupUserServerName,
+  type UserServerNameLookup,
+} from './user-homeserver';
+import { replyPartTxnId, roomReplyMessages } from './reply-parts';
+import { VerifiedRoomAliases } from './room-alias';
+import {
+  boundedSignal,
+  raceSignal,
+  streamWithDeadline,
+} from './media-deadline';
 
 const TYPING_REFRESH_MS = 20_000;
 const TYPING_TIMEOUT_MS = 30_000;
-/** User DID → room id memo (aliases never move; a miss re-resolves). */
+/**
+ * User DID → room id memo (aliases never move; a miss re-resolves), and the
+ * lifetime of a room's verified canonical alias (`room-alias.ts`).
+ */
 const ALIAS_CACHE_TTL_MS = 30 * 60_000;
 /** User DID → Matrix server name (from the DID document via Blocksync). */
 const HOMESERVER_CACHE_TTL_MS = 6 * 60 * 60_000;
+/** Entries each in-memory memo in front of the `hs:` / `alias:` storage keys holds. */
+const MEMO_MAX_ENTRIES = 5_000;
+/**
+ * The full `status()` is served from memory for this long: it scans several
+ * tables, and the route that asks for it is public.
+ */
+const STATUS_CACHE_MS = 5_000;
+/**
+ * Upper bound on one media download (response and body), unless the caller
+ * asks for less. A stalled homeserver stream must not hold the reader — and
+ * with it the user's turn — open indefinitely.
+ */
+export const MEDIA_DOWNLOAD_TIMEOUT_MS = 60_000;
+/**
+ * A live message whose sender's homeserver could not be looked up stays in
+ * the inbox and is replayed on its own after this delay (each replay is
+ * charged to the row, so a long Blocksync outage ends in the "try again"
+ * notice, not a silent drop).
+ */
+const SENDER_CHECK_RETRY_MS = 60_000;
 /**
  * A room created by a user seconds ago may not have delivered its invite
  * through sync yet when the user's object asks for its state; within this
@@ -161,9 +197,37 @@ interface AliasCacheEntry {
   at: number;
 }
 
-interface HomeserverCacheEntry {
-  serverName: string;
-  at: number;
+/** A media download's bound. `signal` only works for a caller in this isolate: an `AbortSignal` cannot cross an RPC boundary. */
+export interface MediaDownloadOptions {
+  /** Upper bound on the whole transfer (default `MEDIA_DOWNLOAD_TIMEOUT_MS`). */
+  timeoutMs?: number;
+  signal?: AbortSignal;
+}
+
+/** What `GET /health` on the gateway answers: from memory only, no storage read. */
+export interface GatewayHealth {
+  running: boolean;
+}
+
+/**
+ * A small bounded memo with insertion-order eviction, in front of the
+ * gateway's storage keys (the values carry their own timestamps; freshness
+ * is the caller's rule, the same one the stored copy is judged by).
+ */
+class BoundedMemo<V> {
+  private readonly map = new Map<string, V>();
+  get(key: string): V | undefined {
+    return this.map.get(key);
+  }
+  set(key: string, value: V): void {
+    this.map.delete(key);
+    while (this.map.size >= MEMO_MAX_ENTRIES) {
+      const oldest = this.map.keys().next().value;
+      if (oldest === undefined) break;
+      this.map.delete(oldest);
+    }
+    this.map.set(key, value);
+  }
 }
 
 function gateSizeFromEnv(raw: string | undefined, fallback: number): number {
@@ -240,8 +304,33 @@ export class MatrixGatewayDO
 {
   private config: OracleGatewayConfig | null = null;
   private ingest: IngestPipeline | null = null;
-  /** `m.room.canonical_alias` per room (null = none), read once per instance. */
-  private readonly roomAliases = new Map<string, string | null>();
+  /** How long a row without a verdict waits before it is offered again. */
+  protected readonly senderCheckRetryMs: number = SENDER_CHECK_RETRY_MS;
+  /**
+   * The verified user↔oracle alias per room (`room-alias.ts`): a canonical
+   * alias counts only when its homeserver resolves it to that same room.
+   */
+  private readonly roomAliases = new VerifiedRoomAliases({
+    readCanonicalAlias: (roomId) => this.readOracleRoomAlias(roomId),
+    resolveAlias: (alias) => this.resolveAlias(alias),
+    ttlMs: ALIAS_CACHE_TTL_MS,
+  });
+  /** In front of `hs:<did>` in storage (same entries, same TTL). */
+  private readonly homeserverMemo = new BoundedMemo<CachedUserServerName>();
+  /** In front of `alias:<did>` in storage (same entries, same TTL). */
+  private readonly roomMemo = new BoundedMemo<AliasCacheEntry>();
+  /** The storage-scanning half of `status()`, served for `STATUS_CACHE_MS`. */
+  private statusCache: {
+    at: number;
+    value: Promise<Omit<GatewayStatus, 'turns' | 'ingestPending'>>;
+  } | null = null;
+  /** Whether the bot was last seen started (set by a start, cleared by `stop`); read by `GET /health`. */
+  private botRunning = false;
+  /**
+   * User DID → registered Matrix server name (lowercased), refreshed before
+   * every offer from `userServerName` so the pipeline can check it synchronously.
+   */
+  private readonly userServers = new Map<string, string>();
   /** Quote-reply chain → thread root memo (see reply-chain.ts). */
   private readonly threadRoots = new ThreadRootCache();
   private readonly turnGate = new Semaphore(
@@ -254,7 +343,16 @@ export class MatrixGatewayDO
   private inboxTableReady = false;
   /** Event ids of turns running in this instance (a graceful restart must not replay them). */
   private readonly inFlightEvents = new Set<string>();
+  /**
+   * Event ids waiting in the ingest debounce buffer: their turn exists in
+   * this instance already, so a start in place (a device rotation) must not
+   * replay them. Cleared with the buffers on `stop`, which drops them.
+   */
+  private readonly debouncing = new Set<string>();
   private inboxReplayRunning = false;
+  /** Inbox rows whose sender check had no verdict, waiting for `unverifiedRetry`. */
+  private readonly unverifiedEvents = new Set<string>();
+  private unverifiedRetry: ReturnType<typeof setTimeout> | null = null;
   /** Group rooms: the gate and the per-room channel memory (see group-chat.ts). */
   private groupChatService: GroupChatService | null = null;
 
@@ -310,6 +408,23 @@ export class MatrixGatewayDO
         env.MATRIX_RECYCLE_AFTER_SENDS ?? String(DEFAULT_RECYCLE_AFTER_SENDS),
       MATRIX_HOT_ROOMS: env.MATRIX_HOT_ROOMS,
       MATRIX_BACKFILL_MAX_EVENTS: env.MATRIX_BACKFILL_MAX_EVENTS,
+      // Memory guards (SDK 0.5+): byte-capped /sync and catch-up pages and
+      // an optional ciphertext cap on decryption. Unset = the SDK defaults.
+      MATRIX_SYNC_TIMELINE_LIMIT: env.MATRIX_SYNC_TIMELINE_LIMIT,
+      MATRIX_SYNC_BYTES_CAP: env.MATRIX_SYNC_BYTES_CAP,
+      MATRIX_CATCHUP_PAGE_SIZE: env.MATRIX_CATCHUP_PAGE_SIZE,
+      MATRIX_PAGE_BYTES_CAP: env.MATRIX_PAGE_BYTES_CAP,
+      MATRIX_MAX_DECRYPT_BYTES: env.MATRIX_MAX_DECRYPT_BYTES,
+      // Opt-in bulk restore of the key backup on a device's first start
+      // (SDK 0.6+); off by default — the gateway fetches room keys from the
+      // backup on demand, one session at a time.
+      MATRIX_BACKUP_BULK_RESTORE: env.MATRIX_BACKUP_BULK_RESTORE,
+      MATRIX_BACKUP_BULK_RESTORE_MAX_KEYS:
+        env.MATRIX_BACKUP_BULK_RESTORE_MAX_KEYS,
+      // Keep-alive fuse (SDK 0.7+): the alarm is re-armed a few seconds
+      // ahead while the object works, so a host drain brings it back within
+      // the fuse instead of at the next cycle.
+      MATRIX_KEEPALIVE_FUSE_MS: env.MATRIX_KEEPALIVE_FUSE_MS,
       LOG_LEVEL: env.LOG_LEVEL,
     });
     return {
@@ -330,7 +445,8 @@ export class MatrixGatewayDO
     this.ingest = new IngestPipeline({
       oracleDid: cfg.oracleRoomDid,
       botUserId: cfg.userId,
-      canonicalAlias: (roomId) => this.roomAliases.get(roomId) ?? null,
+      canonicalAlias: (roomId) => this.roomAliases.known(roomId),
+      userServerName: (userDid) => this.userServers.get(userDid) ?? null,
       dispatch: (turn) => this.dispatchTurn(turn),
       onError: (err, context) =>
         this.log('error', `ingest failed (${context})`, err),
@@ -370,38 +486,131 @@ export class MatrixGatewayDO
     // message back if the instance dies before its reply is in the outbox.
     const sql = this.inboxSql();
     insertInboxRow(sql, inbound, Date.now());
-    // Resolved before the offer so the pipeline's synchronous alias lookup
-    // (room alias → user DID) can answer from the memo.
-    await this.canonicalAliasOf(message.roomId);
-    const threadRootId = await this.threadRootFor(
-      eventId,
-      message.roomId,
-      readRelatesTo(message.content),
-      message.threadRootId,
-    );
-    if (threadRootId !== inbound.threadRootId) {
-      inbound.threadRootId = threadRootId;
-      updateInboxThread(sql, eventId, threadRootId);
+    try {
+      // The thread (a quote-reply chain walk) and the sender's identity (the
+      // verified alias, then the user's registered homeserver) are
+      // independent lookups: they run side by side.
+      const [alias, threadRootId] = await Promise.all([
+        this.attributionFor(inbound),
+        this.threadRootFor(
+          eventId,
+          message.roomId,
+          readRelatesTo(message.content),
+          message.threadRootId,
+        ),
+      ]);
+      if (threadRootId !== inbound.threadRootId) {
+        inbound.threadRootId = threadRootId;
+        updateInboxThread(sql, eventId, threadRootId);
+      }
+      this.offerInbound(inbound, alias);
+    } catch (err) {
+      // A homeserver error while preparing the offer (the alias lookup, a
+      // rate limit) is not a verdict on the message. The SDK has already
+      // marked the event processed, so the row comes back on its own timer.
+      this.log(
+        'warn',
+        `ingest: could not prepare ${eventId} in ${message.roomId}; keeping it in the inbox for a replay`,
+        err,
+      );
+      this.retryUnverifiedLater(eventId);
     }
-    this.offerInbound(inbound);
   }
 
-  /** Feed a message into the ingest pipeline; a message it will not turn into a turn leaves the inbox at once. */
-  private offerInbound(inbound: InboundMessage): void {
-    const outcome = this.ingestPipeline().offer(inbound);
-    if (outcome !== 'queued') {
-      deleteInboxRows(this.inboxSql(), [inbound.eventId]);
+  /**
+   * Feed a message into the ingest pipeline. A message it will not turn into
+   * a turn leaves the inbox at once; one whose sender could not be checked
+   * stays there and is replayed after `SENDER_CHECK_RETRY_MS`.
+   */
+  private offerInbound(inbound: InboundMessage, alias: string | null): void {
+    const outcome = this.ingestPipeline().offer(inbound, alias);
+    const row = settleOfferedRow(this.inboxSql(), inbound.eventId, outcome);
+    if (outcome === 'queued') this.debouncing.add(inbound.eventId);
+    else if (outcome === 'unverified') {
       this.log(
-        'debug',
-        `ingest dropped ${inbound.eventId} in ${inbound.roomId}: ${outcome}`,
+        'warn',
+        `ingest: no homeserver verdict for ${inbound.sender}; keeping ${inbound.eventId} in ${inbound.roomId} for a replay`,
+      );
+      this.retryUnverifiedLater(inbound.eventId);
+    } else if (row === 'deleted') {
+      this.log(
+        // A DID-shaped identity from a server that is not the DID's own is
+        // someone presenting a DID they do not control.
+        outcome === 'foreign' ? 'warn' : 'debug',
+        `ingest dropped ${inbound.eventId} in ${inbound.roomId} from ${inbound.sender}: ${outcome}`,
+      );
+    }
+  }
+
+  /** Replay the unverified rows once `SENDER_CHECK_RETRY_MS` has passed (one timer for all of them). */
+  private retryUnverifiedLater(eventId: string): void {
+    this.unverifiedEvents.add(eventId);
+    if (this.unverifiedRetry) return;
+    this.unverifiedRetry = setTimeout(() => {
+      this.unverifiedRetry = null;
+      const only = new Set(this.unverifiedEvents);
+      this.unverifiedEvents.clear();
+      this.scheduleInboxReplay(only);
+    }, this.senderCheckRetryMs);
+  }
+
+  /**
+   * The room's verified alias, with the homeserver of the user the message
+   * is then attributed to looked up for the sender check. The alias is
+   * handed on as checked here: re-reading the memo later could find it
+   * expired or invalidated meanwhile, and a legacy sender would read as
+   * unmapped. Throws when the alias could not be checked.
+   */
+  private async attributionFor(
+    inbound: InboundMessage,
+  ): Promise<string | null> {
+    const alias = await this.canonicalAliasOf(inbound.roomId);
+    await this.resolveSenderServer(inbound, alias);
+    return alias;
+  }
+
+  /**
+   * Look up the registered homeserver of the user a message is attributed
+   * to, for the pipeline's sender check. A DID document that names no
+   * homeserver answers the oracle's own server. A Blocksync failure answers
+   * the last cached server even when it has expired; with nothing cached the
+   * memo is cleared, so the pipeline gives no verdict (`'unverified'`) and
+   * the message stays in the inbox instead of being dropped as foreign.
+   */
+  private async resolveSenderServer(
+    inbound: InboundMessage,
+    alias: string | null,
+  ): Promise<void> {
+    const attribution = attributeUser({
+      alias,
+      sender: inbound.sender,
+      oracleDid: this.cfg().oracleRoomDid,
+    });
+    if (!attribution) return;
+    try {
+      const { serverName } = await this.lookupUserServer(attribution.userDid);
+      this.userServers.set(attribution.userDid, serverName.toLowerCase());
+    } catch (err) {
+      this.userServers.delete(attribution.userDid);
+      this.log(
+        'warn',
+        `could not resolve ${attribution.userDid}'s homeserver from Blocksync and nothing is cached; ${inbound.eventId} waits for a replay`,
+        err,
       );
     }
   }
 
-  /** A membership change invalidates what the group-chat gate knows about the room. */
+  /**
+   * A membership change invalidates what the group-chat gate knows about the
+   * room; an alias change also invalidates the room's verified alias (the
+   * gate's "direct" verdict rests on it too).
+   */
   protected override async onEvent(event: BotTimelineEvent): Promise<void> {
     await super.onEvent(event);
-    if (event.type === 'm.room.member')
+    if (event.type === 'm.room.canonical_alias') {
+      this.roomAliases.invalidate(event.roomId);
+      this.groupChat().invalidateRoom(event.roomId);
+    } else if (event.type === 'm.room.member')
       this.groupChat().invalidateRoom(event.roomId);
   }
 
@@ -498,10 +707,18 @@ export class MatrixGatewayDO
     return this.threadRootFor(row.eventId, row.roomId, relatesTo);
   }
 
-  /** `m.room.canonical_alias` of a room, memoised per instance (null = none). */
+  /**
+   * The room's verified user↔oracle alias (null = none): its canonical alias,
+   * when that is an alias of this oracle AND its homeserver resolves it to
+   * this room. Memoised (`room-alias.ts`). Throws when the homeserver could
+   * not be asked — no verdict, never "accept".
+   */
   private async canonicalAliasOf(roomId: string): Promise<string | null> {
-    const memo = this.roomAliases.get(roomId);
-    if (memo !== undefined) return memo;
+    return this.roomAliases.verify(roomId);
+  }
+
+  /** `m.room.canonical_alias` of a room when it is a user↔oracle alias of this oracle. */
+  private async readOracleRoomAlias(roomId: string): Promise<string | null> {
     const json = await this.getRoomStateEvent(roomId, 'm.room.canonical_alias');
     const parsed: unknown = json ? JSON.parse(json) : null;
     const alias =
@@ -512,12 +729,25 @@ export class MatrixGatewayDO
       parsed.alias
         ? parsed.alias
         : null;
-    this.roomAliases.set(roomId, alias);
-    return alias;
+    return alias !== null &&
+      userDidFromRoomAlias(alias, this.cfg().oracleRoomDid) !== null
+      ? alias
+      : null;
   }
 
   private async dispatchTurn(turn: IngestTurn): Promise<void> {
-    return this.turnGate.run(() => this.dispatchTurnNow(turn));
+    // In flight from the moment the turn exists, not only once it holds a
+    // gate slot: an in-place restart replays the inbox, and a row still
+    // queued behind the gate must not be offered (and charged) again.
+    for (const id of turn.eventIds) {
+      this.debouncing.delete(id);
+      this.inFlightEvents.add(id);
+    }
+    try {
+      await this.turnGate.run(() => this.dispatchTurnNow(turn));
+    } finally {
+      for (const id of turn.eventIds) this.inFlightEvents.delete(id);
+    }
   }
 
   private async dispatchTurnNow(turn: IngestTurn): Promise<void> {
@@ -563,10 +793,14 @@ export class MatrixGatewayDO
       'info',
       `turn ${requestId}: ${turn.userDid} in ${turn.roomId} (thread ${turn.threadId}${decision.roomKind === 'group' ? `, group room: ${decision.reason}` : ''})`,
     );
-    for (const id of turn.eventIds) this.inFlightEvents.add(id);
     let typing: ReturnType<typeof setInterval> | null = null;
     try {
-      await this.setTyping(turn.roomId, true, TYPING_TIMEOUT_MS);
+      // Cosmetic: a homeserver hiccup on the typing notice must not cost the
+      // user their turn.
+      await this.setTyping(turn.roomId, true, TYPING_TIMEOUT_MS).catch(
+        (err: unknown) =>
+          this.log('debug', `turn ${requestId}: typing notice failed`, err),
+      );
       typing = setInterval(() => {
         this.setTyping(turn.roomId, true, TYPING_TIMEOUT_MS).catch(
           () => undefined,
@@ -578,19 +812,30 @@ export class MatrixGatewayDO
           'info',
           `turn ${requestId}: reply served from the user object's ledger (turn had already finished)`,
         );
-      if (result.text.trim()) {
-        // The transaction id is derived from the event id, so this reply is
-        // deduplicated by the homeserver if a previous incarnation already
-        // sent it before it died (see inbox-store.ts).
-        // Always inside the thread: the reply to a bare message opens the
-        // thread on it (Node's listener bridge does the same).
-        await this.sendText(turn.roomId, result.text, {
+      const messages = roomReplyMessages(result);
+      if (messages.length === 0)
+        this.log('info', `turn ${requestId}: empty reply, nothing sent`);
+      // Transaction ids are derived from the event id (one per plan part), so
+      // a reply is deduplicated by the homeserver if a previous incarnation
+      // already sent it before it died (see inbox-store.ts). Always inside the
+      // thread: the reply to a bare message opens the thread on it (Node's
+      // listener bridge does the same).
+      const eventId = turn.eventIds[0];
+      for (const message of messages)
+        await this.sendText(turn.roomId, message.body, {
           threadId: turn.threadId,
-          ...(turn.eventIds[0] ? { txnId: replyTxnId(turn.eventIds[0]) } : {}),
+          ...(message.formattedBody
+            ? { formattedBody: message.formattedBody }
+            : {}),
+          ...(eventId
+            ? {
+                txnId:
+                  message.partId !== undefined
+                    ? replyPartTxnId(eventId, message.partId)
+                    : replyTxnId(eventId),
+              }
+            : {}),
         });
-      } else {
-        this.log('info', `turn ${requestId}: empty reply, nothing sent`);
-      }
       // The reply is in the durable outbox (or there is none): the turn
       // has ended, the inbox row has done its job.
       deleteInboxRows(this.inboxSql(), turn.eventIds);
@@ -625,23 +870,29 @@ export class MatrixGatewayDO
       // Told the user (or could not): either way this turn is over.
       deleteInboxRows(this.inboxSql(), turn.eventIds);
     } finally {
-      for (const id of turn.eventIds) this.inFlightEvents.delete(id);
       if (typing) clearInterval(typing);
       await this.setTyping(turn.roomId, false).catch(() => undefined);
       // Close the card on every exit — reply posted, empty reply, thrown
       // error — so no turn can leave a spinner running in the room.
-      await this.userStub(turn.userDid)
-        .finishWorkStatus(requestId, 'done')
-        .catch((err: unknown) =>
+      await this.finishWorkStatus(turn.userDid, requestId).catch(
+        (err: unknown) =>
           this.log(
             'warn',
             `turn ${requestId}: could not close the status card`,
             err,
           ),
-        );
+      );
     }
   }
 
+  /** Close the turn's work-status card in the user's object. */
+  protected async finishWorkStatus(
+    userDid: string,
+    requestId: string,
+  ): Promise<void> {
+    await this.userStub(userDid).finishWorkStatus(requestId, 'done');
+  }
+
   private userStub(userDid: string): DurableObjectStub<UserOracleObject> {
     return this.env.USER_ORACLE.get(
       this.env.USER_ORACLE.idFromName(
@@ -685,20 +936,38 @@ export class MatrixGatewayDO
   }
 
   /**
-   * Every start of this instance — first request, keep-alive alarm, RPC —
-   * goes through here. A start that actually brought the bot up is the
+   * After every successful start of this instance — a request, the
+   * keep-alive alarm bringing it back after a reset, a device rotation
+   * (the SDK's `onStarted` hook, detached from the caller). That is the
    * moment to re-dispatch the turns a previous incarnation left unfinished.
    */
+  protected override async onStarted(_result: StartResult): Promise<void> {
+    this.noteRunning(true);
+    this.scheduleInboxReplay();
+  }
+
   override async ensureStarted(): Promise<StartResult> {
     const result = await super.ensureStarted();
-    if (result.started) this.scheduleInboxReplay();
+    this.noteRunning(true);
     return result;
   }
 
-  private scheduleInboxReplay(): void {
-    if (this.inboxReplayRunning) return;
+  /** A lifecycle change: `GET /health` follows it, the cached status is stale. */
+  private noteRunning(running: boolean): void {
+    if (this.botRunning !== running) this.statusCache = null;
+    this.botRunning = running;
+  }
+
+  /** `only`: replay just these rows (the unverified-sender retry); omitted, every row. */
+  private scheduleInboxReplay(only?: ReadonlySet<string>): void {
+    if (this.inboxReplayRunning) {
+      // A full replay already listed every row; a targeted one may not have,
+      // so the rows asked for here wait for the next retry.
+      if (only) for (const id of only) this.retryUnverifiedLater(id);
+      return;
+    }
     this.inboxReplayRunning = true;
-    void this.replayInbox()
+    void this.replayInbox(only)
       .catch((err: unknown) => this.log('error', 'inbox replay failed', err))
       .finally(() => {
         this.inboxReplayRunning = false;
@@ -712,12 +981,14 @@ export class MatrixGatewayDO
    * object decides what a replay means for it (`matrix-turn-ledger.ts`):
    * stored reply, attach to the running turn, or refuse — never a second run.
    */
-  private async replayInbox(): Promise<void> {
+  private async replayInbox(only?: ReadonlySet<string>): Promise<void> {
     const sql = this.inboxSql();
-    const rows = listInboxRows(sql);
+    const rows = listInboxRows(sql).filter(
+      (row) => !only || only.has(row.eventId),
+    );
     if (rows.length === 0) return;
     const plan = planInboxReplay(rows, {
-      inFlight: this.inFlightEvents,
+      inFlight: new Set([...this.inFlightEvents, ...this.debouncing]),
       maxReplays: MAX_TURN_REPLAYS,
     });
     for (const row of plan.exhausted) {
@@ -746,17 +1017,30 @@ export class MatrixGatewayDO
       );
     for (const row of plan.replay) {
       try {
-        await this.canonicalAliasOf(row.roomId);
+        // Without an alias verdict a legacy sender would read as unmapped
+        // and the row would be dropped: the row waits for the next retry
+        // instead (its attempts were bumped above, so an outage still ends
+        // in the "try again" notice).
+        // A failed homeserver lookup leaves no verdict: the offer answers
+        // 'unverified' and the row stays for the next replay.
+        const inbound = inboundOfRow(row);
+        const [alias, threadRootId] = await Promise.all([
+          this.attributionFor(inbound),
+          this.threadRootForRow(row),
+        ]);
+        if (threadRootId !== inbound.threadRootId) {
+          inbound.threadRootId = threadRootId;
+          updateInboxThread(sql, row.eventId, threadRootId);
+        }
+        this.offerInbound(inbound, alias);
       } catch (err) {
-        this.log('warn', `inbox: alias lookup failed for ${row.roomId}`, err);
-      }
-      const inbound = inboundOfRow(row);
-      const threadRootId = await this.threadRootForRow(row);
-      if (threadRootId !== inbound.threadRootId) {
-        inbound.threadRootId = threadRootId;
-        updateInboxThread(sql, row.eventId, threadRootId);
+        this.log(
+          'warn',
+          `inbox: could not prepare ${row.eventId} in ${row.roomId}; retrying later`,
+          err,
+        );
+        this.retryUnverifiedLater(row.eventId);
       }
-      this.offerInbound(inbound);
     }
     this.log(
       'info',
@@ -780,6 +1064,11 @@ export class MatrixGatewayDO
     // A debounce buffer firing after the stop would send through
     // `startedClient()`, which restarts the bot.
     this.ingest?.clear();
+    this.debouncing.clear();
+    if (this.unverifiedRetry) clearTimeout(this.unverifiedRetry);
+    this.unverifiedRetry = null;
+    this.unverifiedEvents.clear();
+    this.noteRunning(false);
     await super.stop();
   }
 
@@ -787,36 +1076,66 @@ export class MatrixGatewayDO
   // Rooms
   // -------------------------------------------------------------------------
 
-  private async userServerName(userDid: string): Promise<string> {
+  /**
+   * The user's registered homeserver (`lookupUserServerName`: a 6 h cache
+   * over Blocksync, an expired entry when Blocksync fails). Throws when
+   * Blocksync fails and nothing was ever cached.
+   */
+  private async lookupUserServer(
+    userDid: string,
+  ): Promise<UserServerNameLookup> {
     const cfg = this.cfg();
     const cacheKey = `hs:${userDid}`;
-    const cached = await this.ctx.storage.get<HomeserverCacheEntry>(cacheKey);
-    if (cached && Date.now() - cached.at < HOMESERVER_CACHE_TTL_MS) {
-      return cached.serverName;
-    }
-    const blocksync = this.env.BLOCKSYNC_GRAPHQL_URL;
-    if (!blocksync) return cfg.serverName;
+    const lookup = await lookupUserServerName(userDid, {
+      blocksyncGraphqlUrl: this.env.BLOCKSYNC_GRAPHQL_URL,
+      defaultServerName: cfg.serverName,
+      ttlMs: HOMESERVER_CACHE_TTL_MS,
+      readCache: async () => {
+        const memo = this.homeserverMemo.get(userDid);
+        if (memo) return memo;
+        const stored =
+          await this.ctx.storage.get<CachedUserServerName>(cacheKey);
+        if (stored) this.homeserverMemo.set(userDid, stored);
+        return stored;
+      },
+      writeCache: async (_did, entry) => {
+        this.homeserverMemo.set(userDid, entry);
+        await this.ctx.storage.put(cacheKey, entry);
+      },
+    });
+    if (lookup.source === 'stale')
+      this.log(
+        'warn',
+        `could not refresh ${userDid}'s homeserver from Blocksync; using the expired cached ${lookup.serverName}`,
+        lookup.error,
+      );
+    else if (
+      lookup.source === 'blocksync' &&
+      lookup.serverName !== cfg.serverName
+    )
+      this.log(
+        'info',
+        `user ${userDid} lives on ${lookup.serverName} (oracle is on ${cfg.serverName}); room alias uses the user's server`,
+      );
+    return lookup;
+  }
+
+  /**
+   * The server half of the user's room alias. When the lookup has no answer
+   * the alias is tried on the oracle's own server — a wrong guess only
+   * misses the room — unlike the sender check, which gives no verdict then.
+   */
+  private async userServerName(userDid: string): Promise<string> {
     try {
-      const resolved = await fetchUserMatrixServerName(blocksync, userDid);
-      if (!resolved) return cfg.serverName;
-      await this.ctx.storage.put(cacheKey, {
-        serverName: resolved,
-        at: Date.now(),
-      } satisfies HomeserverCacheEntry);
-      if (resolved !== cfg.serverName) {
-        this.log(
-          'info',
-          `user ${userDid} lives on ${resolved} (oracle is on ${cfg.serverName}); room alias uses the user's server`,
-        );
-      }
-      return resolved;
+      return (await this.lookupUserServer(userDid)).serverName;
     } catch (err) {
+      const fallback = this.cfg().serverName;
       this.log(
         'warn',
-        `could not resolve ${userDid}'s homeserver from Blocksync; using ${cfg.serverName}`,
+        `could not resolve ${userDid}'s homeserver from Blocksync; using ${fallback} for the room alias`,
         err,
       );
-      return cfg.serverName;
+      return fallback;
     }
   }
 
@@ -829,21 +1148,24 @@ export class MatrixGatewayDO
       await this.userServerName(userDid),
     );
     const cacheKey = `alias:${userDid}`;
+    const fresh = (
+      entry: AliasCacheEntry | undefined,
+    ): entry is AliasCacheEntry =>
+      entry !== undefined &&
+      entry.alias === alias &&
+      Date.now() - entry.at < ALIAS_CACHE_TTL_MS;
+    const memo = this.roomMemo.get(userDid);
+    if (fresh(memo)) return { roomId: memo.roomId, alias };
     const cached = await this.ctx.storage.get<AliasCacheEntry>(cacheKey);
-    if (
-      cached &&
-      cached.alias === alias &&
-      Date.now() - cached.at < ALIAS_CACHE_TTL_MS
-    ) {
+    if (fresh(cached)) {
+      this.roomMemo.set(userDid, cached);
       return { roomId: cached.roomId, alias };
     }
     const roomId = await this.resolveAlias(alias);
     if (!roomId) return null;
-    await this.ctx.storage.put(cacheKey, {
-      roomId,
-      alias,
-      at: Date.now(),
-    } satisfies AliasCacheEntry);
+    const entry: AliasCacheEntry = { roomId, alias, at: Date.now() };
+    this.roomMemo.set(userDid, entry);
+    await this.ctx.storage.put(cacheKey, entry);
     return { roomId, alias };
   }
 
@@ -1156,13 +1478,58 @@ export class MatrixGatewayDO
   async downloadEventMediaStream(
     roomId: string,
     eventId: string,
+    opts: MediaDownloadOptions = {},
   ): Promise<MediaStream | null> {
-    return this.mediaStream(roomId, eventId);
+    const bound = boundedSignal(
+      opts.timeoutMs ?? MEDIA_DOWNLOAD_TIMEOUT_MS,
+      opts.signal,
+    );
+    let media: MediaStream | null;
+    try {
+      // The SDK opens the download without a signal: race it, and release a
+      // stream that arrives after the bound.
+      media = await raceSignal(
+        this.mediaStream(roomId, eventId),
+        bound.signal,
+        (late) => void late?.stream.cancel().catch(() => undefined),
+      );
+    } catch (err) {
+      bound.clear();
+      throw err;
+    }
+    if (!media) {
+      bound.clear();
+      return null;
+    }
+    return { ...media, stream: streamWithDeadline(media.stream, bound) };
   }
 
-  /** Authenticated download of an `mxc://` URI as a stream (`/_matrix/client/v1/media/download`, legacy fallback on 404). */
+  /**
+   * Authenticated download of an `mxc://` URI as a stream
+   * (`/_matrix/client/v1/media/download`, legacy fallback on 404), bounded
+   * by `opts.timeoutMs` (default `MEDIA_DOWNLOAD_TIMEOUT_MS`) from the
+   * request to the last byte.
+   */
   async downloadMxcMediaStream(
     mxc: string,
+    opts: MediaDownloadOptions = {},
+  ): Promise<ReadableStream<Uint8Array>> {
+    const bound = boundedSignal(
+      opts.timeoutMs ?? MEDIA_DOWNLOAD_TIMEOUT_MS,
+      opts.signal,
+    );
+    try {
+      const body = await this.fetchMxc(mxc, bound.signal);
+      return streamWithDeadline(body, bound);
+    } catch (err) {
+      bound.clear();
+      throw err;
+    }
+  }
+
+  private async fetchMxc(
+    mxc: string,
+    signal: AbortSignal,
   ): Promise<ReadableStream<Uint8Array>> {
     const client = await this.startedClient();
     const token = client.getAccessToken();
@@ -1179,7 +1546,7 @@ export class MatrixGatewayDO
       true,
     );
     if (!authed) throw new Error(`MatrixGatewayDO: invalid mxc url ${mxc}`);
-    let res = await fetch(authed, { headers });
+    let res = await fetch(authed, { headers, signal });
     if (res.status === 404) {
       const legacy = client.mxcUrlToHttp(
         mxc,
@@ -1190,12 +1557,17 @@ export class MatrixGatewayDO
         true,
         false,
       );
-      if (legacy) res = await fetch(legacy, { headers });
+      if (legacy) {
+        await res.body?.cancel().catch(() => undefined);
+        res = await fetch(legacy, { headers, signal });
+      }
     }
-    if (!res.ok)
+    if (!res.ok) {
+      await res.body?.cancel().catch(() => undefined);
       throw new Error(
         `MatrixGatewayDO: media download failed: ${res.status} ${res.statusText}`,
       );
+    }
     if (!res.body)
       throw new Error('MatrixGatewayDO: media download has no body');
     return res.body;
@@ -1358,15 +1730,55 @@ export class MatrixGatewayDO
     return this.groupChat().unpinFact(roomId, factId);
   }
 
+  /**
+   * The bot's status. The half that scans storage (the SDK's room, outbox
+   * and catch-up counts, crypto key counts, channel memory, the inbox) is
+   * computed at most once per `STATUS_CACHE_MS` and whenever the bot starts
+   * or stops; the in-memory turn counters are always live.
+   */
   override async status(): Promise<GatewayStatus> {
+    const now = Date.now();
+    if (!this.statusCache || now - this.statusCache.at >= STATUS_CACHE_MS) {
+      const value = this.scanStatus();
+      this.statusCache = { at: now, value };
+      // A failed scan is not served to the next caller.
+      value.catch(() => {
+        if (this.statusCache?.value === value) this.statusCache = null;
+      });
+    }
     return {
-      ...(await super.status()),
+      ...(await this.statusCache.value),
       turns: { inFlight: this.turnGate.inUse, waiting: this.turnGate.queued },
-      groupChat: this.groupChat().stats(),
       ingestPending: this.ingest?.pendingCount ?? 0,
+    };
+  }
+
+  private async scanStatus(): Promise<
+    Omit<GatewayStatus, 'turns' | 'ingestPending'>
+  > {
+    return {
+      ...(await super.status()),
+      groupChat: this.groupChat().stats(),
       inbox: countInboxRows(this.inboxSql()),
     };
   }
+
+  /** Whether the bot is running, from memory only (no storage read, no start). */
+  health(): GatewayHealth {
+    return { running: this.botRunning };
+  }
+
+  /**
+   * The gateway's HTTP surface: `GET /health` answers `health()` — what the
+   * shell's public `GET /health/matrix` asks, at the price of a request to
+   * this object and nothing more.
+   */
+  async fetch(request: Request): Promise<Response> {
+    const { pathname } = new URL(request.url);
+    if (request.method === 'GET' && pathname === '/health')
+      return Response.json(this.health());
+    return new Response('Not found', { status: 404 });
+  }
 }
 
 /** Attachment descriptor of a media message for the ingest (Node bridge shape). */