@ixo/oracle-runtime-workers 0.37.2 → 0.39.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ixo/oracle-runtime-workers",
-  "version": "0.37.2",
+  "version": "0.39.0",
   "description": "QiForge oracle runtime for Cloudflare Workers — Durable Object per user (SQLite in WASM over DO storage), Matrix E2EE gateway Durable Object, Hono HTTP/SSE shell.",
   "private": false,
   "publishConfig": {
@@ -67,9 +67,9 @@
     "protobufjs": "^7.6.5",
     "@ixo/topic-protocol": "1.0.0-rc.8",
     "survey-core": "2.3.11",
-    "@ixo/common": "1.7.2",
-    "@ixo/ixo-transaction": "0.2.0",
-    "@ixo/ucan": "2.3.0"
+    "@ixo/common": "1.8.0",
+    "@ixo/ucan": "2.3.0",
+    "@ixo/ixo-transaction": "0.2.0"
   },
   "devDependencies": {
     "@cloudflare/vitest-pool-workers": "^0.22.0",

package/src/attachments/channel-links.ts

@@ -3,10 +3,17 @@
  * user's own SQLite file in their own table, apart from the messages: the
  * messages, the model and the transcript carry only the `channel:`
  * reference, and `view_attachment` resolves it here once the turn that
- * brought the file has ended. A row lives until its link expires (the
- * gateway signs links for 30 days; `exp` in the link) and goes with its
+ * brought the file has ended. A link is served until it expires (the
+ * gateway signs links for 30 days; `exp` in the link); a row goes with its
  * session (`forgetSession`). Failures are logged and treated as a miss: a
  * missing link makes the file unavailable, never the turn.
+ *
+ * A row also keeps the Matrix event the file was posted as in the session
+ * thread (`event_id`, `attachments/room-media.ts`): that room copy serves
+ * the file when the link fails or has expired, so a row that carries one
+ * outlives its link (the expiry sweep only removes rows without it). A
+ * resend of the same file in a later turn refreshes the link and keeps the
+ * event until the new post records its own.
  */
 import type { DoSqliteDatabase } from '../sqlite/database';
 import type { Logger } from '../plugin-api/types';
@@ -32,7 +39,7 @@ export class ChannelAttachmentLinkStore {
   private setupPromise: Promise<void> | undefined;
 
   constructor(
-    private readonly db: Pick<DoSqliteDatabase, 'run' | 'get'>,
+    private readonly db: Pick<DoSqliteDatabase, 'run' | 'get' | 'exec'>,
     private readonly log: Pick<Logger, 'warn'>,
     private readonly now: () => number = Date.now,
   ) {}
@@ -54,16 +61,30 @@ export class ChannelAttachmentLinkStore {
          url TEXT NOT NULL,
          sha256 TEXT NOT NULL,
          expires_at INTEGER NOT NULL,
+         event_id TEXT,
          UNIQUE (session_id, ref)
        )`,
     );
+    // A table created before `event_id` existed (CREATE TABLE IF NOT
+    // EXISTS leaves it as it was).
+    const columns = await this.db.exec<{ name: string }>(
+      'PRAGMA table_info(channel_attachment_links)',
+    );
+    if (!columns.some((column) => column.name === 'event_id'))
+      await this.db.run(
+        'ALTER TABLE channel_attachment_links ADD COLUMN event_id TEXT',
+      );
     await this.db.run(
       `CREATE INDEX IF NOT EXISTS idx_channel_attachment_links_expiry
        ON channel_attachment_links(expires_at)`,
     );
   }
 
-  /** Keep the links of a turn's files for its session (a resend refreshes them). */
+  /**
+   * Keep the links of a turn's files for its session (a resend refreshes
+   * them, keeping a recorded room event), then sweep expired rows that have
+   * no room copy.
+   */
   async put(
     sessionId: string,
     links: ReadonlyArray<ChannelMediaLink & { ref: string }>,
@@ -74,9 +95,12 @@ export class ChannelAttachmentLinkStore {
       const now = this.now();
       for (const link of links)
         await this.db.run(
-          `INSERT OR REPLACE INTO channel_attachment_links
+          `INSERT INTO channel_attachment_links
              (session_id, ref, binding_id, url, sha256, expires_at)
-           VALUES (?, ?, ?, ?, ?, ?)`,
+           VALUES (?, ?, ?, ?, ?, ?)
+           ON CONFLICT (session_id, ref) DO UPDATE SET
+             binding_id = excluded.binding_id, url = excluded.url,
+             sha256 = excluded.sha256, expires_at = excluded.expires_at`,
           [
             sessionId,
             link.ref,
@@ -87,7 +111,7 @@ export class ChannelAttachmentLinkStore {
           ],
         );
       await this.db.run(
-        'DELETE FROM channel_attachment_links WHERE expires_at <= ?',
+        'DELETE FROM channel_attachment_links WHERE expires_at <= ? AND event_id IS NULL',
         [now],
       );
     } catch (error) {
@@ -124,6 +148,49 @@ export class ChannelAttachmentLinkStore {
     }
   }
 
+  /**
+   * Record the session-thread events the turn's files were posted as
+   * (reference → event id). A failure is logged: the files are then served
+   * by their links alone.
+   */
+  async setEventIds(
+    sessionId: string,
+    eventIds: ReadonlyMap<string, string>,
+  ): Promise<void> {
+    if (eventIds.size === 0) return;
+    try {
+      await this.setup();
+      for (const [ref, eventId] of eventIds)
+        await this.db.run(
+          `UPDATE channel_attachment_links SET event_id = ?
+           WHERE session_id = ? AND ref = ?`,
+          [eventId, sessionId, ref],
+        );
+    } catch (error) {
+      this.log.warn(
+        `[attachments] could not keep the thread events of ${sessionId}'s channel files: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+
+  /** The session-thread event recorded for `ref`, expired link or not. */
+  async eventIdOf(sessionId: string, ref: string): Promise<string | undefined> {
+    try {
+      await this.setup();
+      const row = await this.db.get<{ event_id: string | null }>(
+        `SELECT event_id FROM channel_attachment_links
+         WHERE session_id = ? AND ref = ?`,
+        [sessionId, ref],
+      );
+      return row?.event_id ?? undefined;
+    } catch (error) {
+      this.log.warn(
+        `[attachments] channel file thread event read failed: ${error instanceof Error ? error.message : String(error)}`,
+      );
+      return undefined;
+    }
+  }
+
   /** Drop every link of a deleted session. */
   async forgetSession(sessionId: string): Promise<void> {
     await this.setup();

package/src/attachments/channel.ts

@@ -27,8 +27,9 @@ export const CHANNEL_REF_PREFIX = 'channel:';
 /**
  * The gateway's per-file bound for a video or a document
  * (`maxLargeMediaBytes`, 50 MiB); photos are at most 16 MB there. A file
- * above `MAX_FILE_SIZE` is archived to the sandbox but never handed to a
- * model (see the pipeline).
+ * above `MAX_FILE_SIZE` is posted to the session thread (`room-media.ts`)
+ * but never read by the turn: no model is handed it and the sandbox
+ * archive refuses it (see the pipeline).
  */
 export const CHANNEL_MAX_FILE_BYTES = 52_428_800;
 
@@ -94,7 +95,8 @@ export function isGatewayMediaUrl(
   );
 }
 
-function hex(bytes: ArrayBuffer): string {
+/** Lowercase hex of a digest. */
+export function hex(bytes: ArrayBuffer): string {
   return Array.from(new Uint8Array(bytes), (byte) =>
     byte.toString(16).padStart(2, '0'),
   ).join('');
@@ -163,15 +165,23 @@ export interface ChannelMediaSource {
       timeoutMs?: number;
     },
   ): Promise<{ bytes: Uint8Array; contentType?: string }>;
+  /**
+   * The media event the file was posted as in the session thread
+   * (`room-media.ts`), when one is recorded: the room copy that serves the
+   * file once its link has expired or the gateway no longer has it.
+   */
+  roomEventOf?(ref: string): Promise<string | undefined>;
 }
 
 /**
  * A `ChannelMediaSource` over `resolve` (the turn's files, then the
- * session's stored links) for the gateways in `origins`.
+ * session's stored links) for the gateways in `origins`; `roomEventOf`
+ * names the room copy of a file, when the owner keeps one.
  */
 export function createChannelMediaSource(input: {
   resolve: (ref: string) => Promise<ChannelMediaLink | undefined>;
   origins: readonly string[];
+  roomEventOf?: (ref: string) => Promise<string | undefined>;
 }): ChannelMediaSource {
   return {
     load: async (ref, opts) => {
@@ -182,6 +192,7 @@ export function createChannelMediaSource(input: {
         );
       return downloadChannelFile(link, { ...opts, origins: input.origins });
     },
+    ...(input.roomEventOf ? { roomEventOf: input.roomEventOf } : {}),
   };
 }
 

package/src/attachments/content-blocks.ts

@@ -30,22 +30,25 @@ export function buildUserMessageContent(
 
   const content: MessageContent = [
     { type: 'text', text },
-    ...natives.map((native) =>
-      native.kind === 'image'
-        ? {
-            type: 'image',
-            source_type: 'base64',
-            mime_type: native.mimeType,
-            data: native.base64,
-          }
-        : {
-            type: 'file',
-            source_type: 'base64',
-            mime_type: native.mimeType,
-            data: native.base64,
-            filename: native.filename,
-          },
-    ),
+    ...natives.map(nativeContentBlock),
   ];
   return content;
 }
+
+/** The content block of one native attachment (the shape described above). */
+export function nativeContentBlock(native: NativeAttachment) {
+  return native.kind === 'image'
+    ? {
+        type: 'image',
+        source_type: 'base64',
+        mime_type: native.mimeType,
+        data: native.base64,
+      }
+    : {
+        type: 'file',
+        source_type: 'base64',
+        mime_type: native.mimeType,
+        data: native.base64,
+        filename: native.filename,
+      };
+}

package/src/attachments/download.ts

@@ -1,9 +1,11 @@
 /**
- * Fetching attachment bytes — Matrix media (through the gateway, which holds
- * the authenticated client and the room keys), channel files (through the
- * owner object's `ChannelMediaSource`, see `channel.ts`) and plain `http(s)`
- * URLs with the Node runtime's SSRF guard, manual redirect validation and
- * streaming size cap.
+ * Fetching attachment bytes — Matrix media (the owner object asks the
+ * gateway for the media event's small JSON — it alone holds the room keys —
+ * and reads the bytes from the homeserver itself, `openHomeserverMedia`),
+ * channel files (through the owner object's `ChannelMediaSource`, see
+ * `channel.ts`, with the file's room copy as the fallback) and plain
+ * `http(s)` URLs with the Node runtime's SSRF guard, manual redirect
+ * validation and streaming size cap.
  */
 import type { ChannelMediaSource } from './channel';
 import { containerMatchesClaim, detectMimeFromMagicBytes } from './magic';
@@ -53,7 +55,7 @@ function tooLarge(maxBytes: number): AttachmentTooLargeError {
 }
 
 /** The signal's reason as an Error (an aborter may pass any value). */
-function abortError(signal: AbortSignal): Error {
+export function abortError(signal: AbortSignal): Error {
   const reason: unknown = signal.reason;
   if (reason instanceof Error) return reason;
   return new Error(
@@ -65,7 +67,10 @@ function abortError(signal: AbortSignal): Error {
  * `work`, or the signal's reason as soon as it aborts — so a transfer that
  * ignores the signal still cannot hold the caller past it.
  */
-function untilAborted<T>(work: Promise<T>, signal: AbortSignal): Promise<T> {
+export function untilAborted<T>(
+  work: Promise<T>,
+  signal: AbortSignal,
+): Promise<T> {
   if (signal.aborted) {
     work.catch(() => undefined);
     return Promise.reject(abortError(signal));
@@ -105,8 +110,9 @@ export function validateUrlTarget(targetUrl: string): void {
 }
 
 /**
- * Matrix media access the pipeline needs. Implemented over the gateway
- * Durable Object in the user object; tests pass an in-memory fake.
+ * Matrix media access the pipeline needs. Implemented in the user object
+ * (the event's media info from the gateway, the bytes from the homeserver);
+ * tests pass an in-memory fake.
  *
  * `signal` aborts at the download deadline or when the turn is aborted: an
  * implementation stops the transfer then (`readBytesCapped` with the signal
@@ -148,15 +154,69 @@ export interface DownloadOptions {
   maxRedirects?: number;
   /** Where a `channelRef` attachment's bytes come from; absent = none can be read. */
   channel?: ChannelMediaSource;
+  /**
+   * The attachment's bytes as this turn already read them (a channel file
+   * read while it was posted to the session thread): used in place of a
+   * download, under the same size, emptiness and type checks.
+   */
+  preloaded?: { bytes: Uint8Array; contentType?: string };
+}
+
+/** The homeserver answered 401: the access token it was given is no longer valid. */
+export class HomeserverUnauthorizedError extends Error {
+  override readonly name: string = 'HomeserverUnauthorizedError';
+}
+
+/** `mxc://<server>/<media id>`, the parts checked before they go into a URL. */
+const MXC_PATTERN = /^mxc:\/\/([A-Za-z0-9.:[\]-]+)\/([A-Za-z0-9_-]+)$/;
+
+/**
+ * Open the media behind `mxc` on the homeserver as the account behind
+ * `accessToken`: the authenticated endpoint
+ * (`/_matrix/client/v1/media/download`), then the legacy one
+ * (`/_matrix/media/v3/download`) when the server answers 404 — the order
+ * matrix-js-sdk's `mxcUrlToHttp` and the bot SDK use. The body is returned
+ * unread; a 401 is `HomeserverUnauthorizedError`.
+ */
+export async function openHomeserverMedia(
+  mxc: string,
+  homeserver: { baseUrl: string; accessToken: string },
+  opts: { signal?: AbortSignal; fetchImpl?: typeof fetch } = {},
+): Promise<ReadableStream<Uint8Array>> {
+  const match = MXC_PATTERN.exec(mxc);
+  if (!match) throw new Error('Invalid mxc URI');
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const path = `${match[1]}/${match[2]}?allow_redirect=true`;
+  const headers = { authorization: `Bearer ${homeserver.accessToken}` };
+  const open = (prefix: string): Promise<Response> =>
+    fetchImpl(new URL(`${prefix}/${path}`, homeserver.baseUrl), {
+      headers,
+      ...(opts.signal ? { signal: opts.signal } : {}),
+    });
+  let res = await open('/_matrix/client/v1/media/download');
+  if (res.status === 404) {
+    await res.body?.cancel().catch(() => undefined);
+    res = await open('/_matrix/media/v3/download');
+  }
+  if (!res.ok || !res.body) {
+    await res.body?.cancel().catch(() => undefined);
+    if (res.status === 401)
+      throw new HomeserverUnauthorizedError(
+        'the homeserver rejected the access token (401)',
+      );
+    throw new Error(`Matrix media download failed: HTTP ${res.status}`);
+  }
+  return res.body;
 }
 
 /**
  * A controller that aborts on `signal` (with its reason) OR after
- * `timeoutMs` (with a timeout error), whichever comes first.
+ * `timeoutMs` (with a timeout error naming `what`), whichever comes first.
  */
-function boundedSignal(
+export function boundedSignal(
   signal: AbortSignal | undefined,
   timeoutMs: number,
+  what = 'Attachment download',
 ): { signal: AbortSignal; release: () => void } {
   const controller = new AbortController();
   const onAbort = (): void => {
@@ -167,9 +227,7 @@ function boundedSignal(
   const timer = setTimeout(
     () =>
       controller.abort(
-        new Error(
-          `Attachment download timed out after ${Math.round(timeoutMs / 1000)} s`,
-        ),
+        new Error(`${what} timed out after ${Math.round(timeoutMs / 1000)} s`),
       ),
     timeoutMs,
   );
@@ -297,11 +355,87 @@ export async function downloadFromUrl(
   }
 }
 
+/**
+ * The Matrix media event `eventId` of `roomId`, under the download deadline
+ * (`opts.timeoutMs`, default `DOWNLOAD_TIMEOUT_MS`) and `opts.signal`.
+ */
+async function loadMatrixEvent(
+  eventId: string,
+  roomId: string | undefined,
+  source: MatrixMediaSource,
+  maxBytes: number,
+  opts: DownloadOptions,
+): Promise<{ bytes: Uint8Array; httpType?: string }> {
+  if (!roomId)
+    throw new Error(
+      `Cannot fetch event ${eventId}: no Matrix room for this turn`,
+    );
+  const bound = boundedSignal(
+    opts.signal,
+    opts.timeoutMs ?? DOWNLOAD_TIMEOUT_MS,
+  );
+  try {
+    const media = await untilAborted(
+      source.downloadEvent(roomId, eventId, maxBytes, bound.signal),
+      bound.signal,
+    );
+    if (!media) throw new Error(`Matrix event ${eventId} not found`);
+    return {
+      bytes: media.bytes,
+      ...(media.mimetype ? { httpType: media.mimetype } : {}),
+    };
+  } finally {
+    bound.release();
+  }
+}
+
+/**
+ * A channel file's bytes: from its link, else — when the link is gone or
+ * fails for any reason but an abort or the size cap — from the copy posted
+ * in the session thread (`postedAs`, else the event the channel source
+ * recorded), which outlives the gateway's 30-day link.
+ */
+async function loadChannelFile(
+  ref: string,
+  postedAs: string | undefined,
+  roomId: string | undefined,
+  source: MatrixMediaSource,
+  maxBytes: number,
+  opts: DownloadOptions,
+): Promise<{ bytes: Uint8Array; httpType?: string }> {
+  try {
+    if (!opts.channel)
+      throw new Error('Channel files cannot be read on this host');
+    const loaded = await opts.channel.load(ref, {
+      maxBytes,
+      ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
+      ...(opts.signal ? { signal: opts.signal } : {}),
+      ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
+    });
+    return {
+      bytes: loaded.bytes,
+      ...(loaded.contentType ? { httpType: loaded.contentType } : {}),
+    };
+  } catch (error) {
+    if (
+      opts.signal?.aborted ||
+      error instanceof AttachmentTooLargeError ||
+      !roomId
+    )
+      throw error;
+    const eventId = postedAs ?? (await opts.channel?.roomEventOf?.(ref));
+    if (!eventId) throw error;
+    return loadMatrixEvent(eventId, roomId, source, maxBytes, opts);
+  }
+}
+
 /**
  * Bytes for one attachment from whichever source it names, with the per-file
  * cap (`opts.maxBytes`, default `MAX_FILE_SIZE`) enforced before and during
  * the transfer and the `DOWNLOAD_TIMEOUT_MS` deadline (or `opts.timeoutMs`)
- * on every source — Matrix media included. `mimetype` is the best-known
+ * on every source — Matrix media included; `opts.preloaded` bytes are
+ * checked the same way without a download. A channel file whose link fails
+ * is read from its room copy when one is known (`loadChannelFile`). `mimetype` is the best-known
  * type: content sniffing (`sniffed` is then true; a container signature
  * consistent with the claim, such as a zip for a `.docx`, keeps the claimed
  * type), then the HTTP `content-type`, then the client's claim. An empty
@@ -326,47 +460,36 @@ export async function loadAttachmentBytes(
 
   let bytes: Uint8Array;
   let httpType: string | undefined;
-  if (attachment.channelRef) {
-    if (!opts.channel)
-      throw new Error('Channel files cannot be read on this host');
-    const loaded = await opts.channel.load(attachment.channelRef, {
+  if (opts.preloaded) {
+    bytes = opts.preloaded.bytes;
+    httpType = opts.preloaded.contentType;
+  } else if (attachment.channelRef) {
+    ({ bytes, httpType } = await loadChannelFile(
+      attachment.channelRef,
+      attachment.eventId,
+      roomId,
+      source,
       maxBytes,
-      ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
-      ...(opts.signal ? { signal: opts.signal } : {}),
-      ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
-    });
-    bytes = loaded.bytes;
-    httpType = loaded.contentType;
-  } else if (attachment.eventId || attachment.mxcUri!.startsWith('mxc://')) {
+      opts,
+    ));
+  } else if (attachment.eventId) {
+    ({ bytes, httpType } = await loadMatrixEvent(
+      attachment.eventId,
+      roomId,
+      source,
+      maxBytes,
+      opts,
+    ));
+  } else if (attachment.mxcUri!.startsWith('mxc://')) {
     const bound = boundedSignal(
       opts.signal,
       opts.timeoutMs ?? DOWNLOAD_TIMEOUT_MS,
     );
     try {
-      if (attachment.eventId) {
-        if (!roomId)
-          throw new Error(
-            `Cannot fetch event ${attachment.eventId}: no Matrix room for this turn`,
-          );
-        const media = await untilAborted(
-          source.downloadEvent(
-            roomId,
-            attachment.eventId,
-            maxBytes,
-            bound.signal,
-          ),
-          bound.signal,
-        );
-        if (!media)
-          throw new Error(`Matrix event ${attachment.eventId} not found`);
-        bytes = media.bytes;
-        httpType = media.mimetype;
-      } else {
-        bytes = await untilAborted(
-          source.downloadMxc(attachment.mxcUri!, maxBytes, bound.signal),
-          bound.signal,
-        );
-      }
+      bytes = await untilAborted(
+        source.downloadMxc(attachment.mxcUri!, maxBytes, bound.signal),
+        bound.signal,
+      );
     } finally {
       bound.release();
     }

package/src/attachments/index.ts

@@ -73,7 +73,12 @@ export {
   MIN_CHARS_PER_PAGE,
   type PdfTextResult,
 } from './pdf-text';
-export { routeAttachment, type AttachmentStrategy } from './route';
+export { resizeForModel, RESIZE_JPEG_QUALITY, RESIZE_MAX_EDGE } from './resize';
+export {
+  NATIVE_DOCUMENT_MAX_BYTES,
+  routeAttachment,
+  type AttachmentStrategy,
+} from './route';
 export {
   buildAnalysisMarkdown,
   uploadToSandbox,
@@ -94,21 +99,17 @@ export {
   type AttachmentInput,
 } from './types';
 export {
-  applyAttachmentRetention,
-  ATTACHMENT_PAYLOAD_TURNS,
   ATTACHMENT_PLACEHOLDER_PREFIX,
   ATTACHMENT_VIEW_SOURCE,
   attachmentMetas,
   attachmentRef,
   buildPlaceholderText,
   hasInlinePayload,
-  INLINE_PAYLOAD_NEEDLE,
   isAttachmentPlaceholderText,
   isAttachmentViewMessage,
   isInlinePayloadBlock,
   offloadInlinePayloads,
   VIEW_ATTACHMENT_TOOL_NAME,
-  type AttachmentRetentionResult,
 } from './retention';
 export {
   createAttachmentViewSurface,