@ixo/oracle-runtime-workers 0.4.1 → 0.10.0

package/src/matrix/work-status.ts

@@ -15,6 +15,7 @@
  * replaces it — which also guarantees a terminal phase is never dropped.
  * Fire-and-forget throughout: a failed post is logged, never thrown.
  */
+import { retryTxnId } from './txn-id';
 
 export const ORACLE_COMPONENT_EVENT_TYPE = 'ixo.oracle.component';
 
@@ -129,11 +130,16 @@ interface TurnEntry extends WorkStatusTurn {
 }
 
 export interface WorkStatusProducerDeps {
-  /** Posts a room event and resolves its event id (the gateway's `sendEvent`). */
+  /**
+   * Posts a room event and resolves its event id (the gateway's `sendEvent`).
+   * `txnId` pins the transaction id so the anchor's single retry cannot post
+   * a second card when only the response was lost.
+   */
   postEvent: (
     roomId: string,
     eventType: string,
     content: object,
+    opts?: { txnId?: string },
   ) => Promise<string>;
   clock?: () => Date;
   logger?: { warn: (message: string) => void };
@@ -205,14 +211,20 @@ export class WorkStatusProducer {
     content: object,
     retryOnce: boolean,
   ): Promise<string> {
+    const opts = { txnId: retryTxnId('work-status') };
     try {
-      return await this.postEvent(roomId, ORACLE_COMPONENT_EVENT_TYPE, content);
+      return await this.postEvent(
+        roomId,
+        ORACLE_COMPONENT_EVENT_TYPE,
+        content,
+        opts,
+      );
     } catch (error) {
       if (!retryOnce) throw error;
       await new Promise((resolve) =>
         setTimeout(resolve, this.anchorRetryDelayMs),
       );
-      return this.postEvent(roomId, ORACLE_COMPONENT_EVENT_TYPE, content);
+      return this.postEvent(roomId, ORACLE_COMPONENT_EVENT_TYPE, content, opts);
     }
   }
 

package/src/owner-store/ixo-vfs-store.ts

@@ -50,6 +50,7 @@ import {
   type OwnerCopy,
   type OwnerStore,
   type SaveResult,
+  type SaveHints,
 } from './types';
 
 const VFS_RESOURCE = 'ixo:filesystem';
@@ -233,9 +234,11 @@ export class IxoVfsOwnerStore implements OwnerStore {
    * Atomic replace, always delete + re-create, never a new version:
    *
    *   1. drop stale `.uploading-*` temp files a crashed flush left behind;
-   *   2. measure the gzipped length in a counting pass (the VFS needs
-   *      `Upload-Length` at session creation — `Upload-Defer-Length` is not
-   *      supported — and gzip output is deterministic for identical input);
+   *   2. take the gzipped length from `hints` (the object measured it in the
+   *      same pass as its checksum) or, without a hint, count it in a pass
+   *      of its own (the VFS needs `Upload-Length` at session creation —
+   *      `Upload-Defer-Length` is not supported — and gzip output is
+   *      deterministic for identical input);
    *   3. upload gzip → a TEMP path next to the real one: one `POST /files`
    *      when it fits in a single part, else a tus session in 5 MiB parts
    *      (`tusUpload`, resumable per part);
@@ -247,7 +250,10 @@ export class IxoVfsOwnerStore implements OwnerStore {
    * file is cleaned up then). Peak memory is one part (5 MiB) plus fixed
    * overhead, whatever the file size.
    */
-  async save(snapshot: FileSnapshot): Promise<SaveResult> {
+  async save(
+    snapshot: FileSnapshot,
+    hints: SaveHints = {},
+  ): Promise<SaveResult> {
     const listing = await this.listOracleFiles();
     const existing = listing.filter((f) => f.path === this.path);
     const stale = listing.filter((f) =>
@@ -261,7 +267,8 @@ export class IxoVfsOwnerStore implements OwnerStore {
       });
     }
 
-    const gzLength = await countStream(gzipStream(snapshot.open()));
+    const gzLength =
+      hints.gzippedLength ?? (await countStream(gzipStream(snapshot.open())));
     const tempPath = `${this.path}${TEMP_PATH_INFIX}${Date.now().toString(36)}`;
     const uploaded =
       gzLength <= SINGLE_SHOT_MAX_BYTES

package/src/owner-store/matrix-media-store.ts

@@ -4,41 +4,50 @@
  * Wire format is the Node runtime's (`m.ixo.media_upload` timeline event +
  * `m.ixo.media_state[storageKey]` pointer), so a user migrating from the
  * Node runtime finds their checkpoint file where they left it. The gateway
- * handles encryption and media transfer; this class owns gzip and the
- * SQLite sanity check.
+ * moves the media across the object boundary as a stream, as stored; this
+ * class decrypts, gunzips and header-checks it chunk by chunk (and gzips +
+ * hands over a stream on the way up), so the file is never held whole in
+ * either isolate — a 50 MB Node-era history used to be materialised three
+ * times over in the user object and reset its 128 MB isolate.
  */
 
+import { createAttachmentDecryptor } from '@ixo/matrix-bot-workers-sdk';
 import type { MatrixGatewayObject } from '../do/contracts';
 import {
-  bytesOfStream,
-  gunzip,
-  gzip,
-  isSqliteFile,
-  streamOfBytes,
+  assertSqliteStream,
+  countStream,
+  gunzipStreamIfNeeded,
+  gzipStream,
   type FileSnapshot,
   type OwnerCopy,
   type OwnerStore,
   type SaveResult,
+  type SaveHints,
 } from './types';
 
-const GZIP_MAGIC_0 = 0x1f;
-const GZIP_MAGIC_1 = 0x8b;
-
-function looksGzipped(bytes: Uint8Array): boolean {
-  return (
-    bytes.length >= 2 && bytes[0] === GZIP_MAGIC_0 && bytes[1] === GZIP_MAGIC_1
-  );
-}
+/**
+ * The slice of the gateway RPC surface this store needs — structurally
+ * satisfied by `DurableObjectStub<MatrixGatewayObject>` and faked in tests.
+ */
+export type SnapshotGateway = Pick<
+  MatrixGatewayObject,
+  | 'downloadUserSnapshotStream'
+  | 'uploadUserSnapshotStream'
+  | 'resolveUserRoom'
+  | 'getRoomStateEvent'
+  | 'sendEvent'
+  | 'sendStateEvent'
+>;
 
 export interface MatrixMediaOwnerStoreOptions {
-  gateway: DurableObjectStub<MatrixGatewayObject>;
+  gateway: SnapshotGateway;
   userDid: string;
   storageKey: string;
 }
 
 export class MatrixMediaOwnerStore implements OwnerStore {
   readonly kind = 'matrix' as const;
-  private readonly gateway: DurableObjectStub<MatrixGatewayObject>;
+  private readonly gateway: SnapshotGateway;
   private readonly userDid: string;
   private readonly storageKey: string;
 
@@ -53,43 +62,59 @@ export class MatrixMediaOwnerStore implements OwnerStore {
   }
 
   /**
-   * The gateway RPC hands the whole (encrypted, then decrypted) media blob
-   * over; this path is the legacy one and inherently materializes the file.
+   * Streamed end to end: the gateway hands the media over as stored (the
+   * ciphertext plus its `EncryptedFile` fields in an E2EE room, the plain
+   * upload otherwise), this side decrypts and gunzips as the bytes arrive and
+   * rejects anything that does not start like a SQLite file before the object
+   * writes a single chunk. A hash mismatch surfaces at the END of the stream
+   * (the SDK verifies the ciphertext SHA-256 once the last byte is in), which
+   * fails the import and leaves the working copy untouched. Peak memory is a
+   * few chunks, whatever the file size.
    */
   async load(): Promise<OwnerCopy | null> {
-    const found = await this.gateway.downloadUserSnapshot(
+    const found = await this.gateway.downloadUserSnapshotStream(
       this.userDid,
       this.storageKey,
     );
     if (!found) return null;
+    const plain = found.file
+      ? found.stream.pipeThrough(createAttachmentDecryptor(found.file))
+      : found.stream;
     // Old Node-runtime uploads were always gzipped, but tolerate a raw file
     // so a hand-uploaded database still loads.
-    const bytes = looksGzipped(found.bytes)
-      ? await gunzip(found.bytes)
-      : found.bytes;
-    if (!isSqliteFile(bytes)) {
-      throw new Error(
-        `MatrixMediaOwnerStore: snapshot ${found.eventId} for ${this.storageKey} is not a SQLite file (${bytes.length} bytes)`,
-      );
-    }
-    return { stream: streamOfBytes(bytes), etag: found.eventId };
+    const stream = await assertSqliteStream(
+      await gunzipStreamIfNeeded(plain),
+      `MatrixMediaOwnerStore: snapshot ${found.eventId} for ${this.storageKey}`,
+    );
+    return { stream, etag: found.eventId };
   }
 
-  async save(snapshot: FileSnapshot): Promise<SaveResult> {
-    const bytes = await bytesOfStream(snapshot.open());
-    if (!isSqliteFile(bytes)) {
-      throw new Error(
-        `MatrixMediaOwnerStore: refusing to save ${bytes.length} bytes that are not a SQLite file`,
-      );
-    }
-    const compressed = await gzip(bytes);
-    const { eventId } = await this.gateway.uploadUserSnapshot(
+  /**
+   * Streamed as well: the snapshot is opened for a header peek, for a gzip
+   * pass that only counts unless the caller measured the length already
+   * (`hints`; the homeserver needs the exact ciphertext length up front and
+   * gzip output is deterministic for identical input) and for the gzip pass
+   * that is uploaded — and never held whole.
+   */
+  async save(
+    snapshot: FileSnapshot,
+    hints: SaveHints = {},
+  ): Promise<SaveResult> {
+    const checked = await assertSqliteStream(
+      snapshot.open(),
+      `MatrixMediaOwnerStore: refusing to save — the working copy (${snapshot.size} bytes)`,
+    );
+    await checked.cancel().catch(() => undefined);
+    const gzLength =
+      hints.gzippedLength ?? (await countStream(gzipStream(snapshot.open())));
+    const { eventId } = await this.gateway.uploadUserSnapshotStream(
       this.userDid,
       this.storageKey,
-      compressed,
+      gzipStream(snapshot.open()),
       this.filename,
+      gzLength,
     );
-    return { etag: eventId, bytes: compressed.byteLength };
+    return { etag: eventId, bytes: gzLength };
   }
 
   async head(): Promise<{ etag: string } | null> {

package/src/owner-store/measure.ts

@@ -0,0 +1,39 @@
+/**
+ * One streamed pass over a snapshot that yields BOTH what a flush needs
+ * before it uploads: the SHA-256 of the raw SQLite bytes (the "did anything
+ * change" gate against the last upload) and the exact gzipped length (the
+ * VFS needs `Upload-Length` when the tus session is created). Before this
+ * helper the object hashed in one pass and the store counted the gzip in a
+ * second one; with the R2 page tier every pass over a cold file is one R2
+ * GET per segment, so the two are folded together and the upload itself is
+ * the only other read.
+ *
+ * The length is exact for the store's own gzip: `gzipStream` is the same
+ * `CompressionStream('gzip')` the stores pipe the upload through, and its
+ * output is deterministic for identical input.
+ */
+import { createHash } from 'node:crypto';
+import { countStream, gzipStream } from './types';
+
+export interface SnapshotMeasure {
+  /** Hex SHA-256 of the raw (uncompressed) bytes. */
+  sha256Hex: string;
+  /** Byte length of `gzipStream(<the same bytes>)`. */
+  gzippedLength: number;
+}
+
+export async function measureForSave(
+  stream: ReadableStream<Uint8Array>,
+): Promise<SnapshotMeasure> {
+  const hash = createHash('sha256');
+  const tapped = stream.pipeThrough(
+    new TransformStream<Uint8Array, Uint8Array>({
+      transform(chunk, controller) {
+        hash.update(chunk);
+        controller.enqueue(chunk);
+      },
+    }),
+  );
+  const gzippedLength = await countStream(gzipStream(tapped));
+  return { sha256Hex: hash.digest('hex'), gzippedLength };
+}

package/src/owner-store/migrating-store.ts

@@ -3,10 +3,12 @@
  * Matrix room media is only a read-only *legacy* source, checked when VFS has
  * no file yet so users coming from the Node runtime keep their history.
  *
- *   load():  VFS first. If absent there but present in Matrix media, import
- *            the legacy file AND write it to VFS immediately — from that
- *            moment the VFS copy is authoritative and Matrix is never
- *            consulted again (head() sees the VFS file).
+ *   load():  VFS first. If absent there but present in Matrix media, hand
+ *            the legacy stream over flagged `fromLegacy`: the object imports
+ *            it chunk by chunk and writes it to VFS from its working copy at
+ *            once (`flushToOwnerStore`) — from that moment the VFS copy is
+ *            authoritative and Matrix is never consulted again (head() sees
+ *            the VFS file). Nothing on this path holds the file in memory.
  *   save():  VFS, and only VFS. Nothing is ever written to Matrix media any
  *            more: a failed VFS write (no `ixo:filesystem` delegation yet, a
  *            VFS outage) propagates, and the object keeps its working copy
@@ -18,14 +20,12 @@
  *            Matrix copy so no second copy of the user's history lingers.
  */
 import { VfsNoDelegationError } from './ixo-vfs-store';
-import {
-  bytesOfStream,
-  snapshotOfBytes,
-  streamOfBytes,
-  type FileSnapshot,
-  type OwnerCopy,
-  type OwnerStore,
-  type SaveResult,
+import type {
+  FileSnapshot,
+  OwnerCopy,
+  OwnerStore,
+  SaveHints,
+  SaveResult,
 } from './types';
 
 export interface MigratingOwnerStoreOptions {
@@ -93,24 +93,16 @@ export class MigratingOwnerStore implements OwnerStore {
       return null;
     }
 
-    // One-time migration: the legacy file becomes the first VFS version. The
-    // legacy copy is already in memory (gateway RPCs hand over bytes), so
-    // re-opening it for the VFS upload costs nothing extra.
-    const legacyBytes = await bytesOfStream(fromLegacy.stream);
-    try {
-      const { etag } = await this.primary.save(snapshotOfBytes(legacyBytes));
-      this.log(
-        'log',
-        `[owner-store] migrated ${legacyBytes.byteLength} bytes from Matrix media to VFS (${etag})`,
-      );
-      return { stream: streamOfBytes(legacyBytes), etag };
-    } catch (err) {
-      this.log(
-        'warn',
-        `[owner-store] VFS migration write failed — serving the legacy Matrix copy for now: ${err instanceof Error ? err.message : String(err)}`,
-      );
-      return { stream: streamOfBytes(legacyBytes), etag: fromLegacy.etag };
-    }
+    // One-time migration, streamed: the object imports this copy chunk by
+    // chunk and then flushes its working copy to VFS (`fromLegacy`). Writing
+    // VFS from here would need the whole file in memory twice over (the VFS
+    // store measures the gzip before it uploads) — what used to reset the
+    // isolate on large Node-era histories.
+    this.log(
+      'log',
+      `[owner-store] legacy Matrix copy ${fromLegacy.etag} found and no VFS copy yet — importing it for migration`,
+    );
+    return { ...fromLegacy, fromLegacy: true };
   }
 
   /**
@@ -118,8 +110,8 @@ export class MigratingOwnerStore implements OwnerStore {
    * "not on VFS yet" (no delegation) — so the object keeps the copy dirty
    * and retries; Matrix media is never written.
    */
-  async save(snapshot: FileSnapshot): Promise<SaveResult> {
-    return this.primary.save(snapshot);
+  async save(snapshot: FileSnapshot, hints?: SaveHints): Promise<SaveResult> {
+    return this.primary.save(snapshot, hints);
   }
 
   /**
@@ -159,13 +151,16 @@ export class MigratingOwnerStore implements OwnerStore {
 
   /** The legacy Matrix copy regardless of what VFS holds (see `OwnerStore.loadLegacy`). */
   async loadLegacy(): Promise<OwnerCopy | null> {
-    return this.legacy.load().catch((err) => {
+    try {
+      const copy = await this.legacy.load();
+      return copy ? { ...copy, fromLegacy: true } : null;
+    } catch (err) {
       this.log(
         'warn',
         `[owner-store] legacy Matrix load failed: ${err instanceof Error ? err.message : String(err)}`,
       );
       return null;
-    });
+    }
   }
 
   async remove(): Promise<void> {

package/src/plugin-api/types.ts

@@ -415,6 +415,17 @@ export interface PluginContext<TConfig = MergedConfig> {
  * Passed to tool handlers, sub-agent handlers, and to plugin middlewares' hook
  * functions. Built fresh per graph invocation.
  */
+/**
+ * Options of a room post made through `RuntimeContext.matrix`. A plugin
+ * that retries a post after a lost response should pin `txnId` (one id for
+ * the whole retry loop, see `retryTxnId`): the homeserver deduplicates sends
+ * by transaction id per device, so the retried post lands once. Printable
+ * ASCII without `/`, at most 255 characters.
+ */
+export interface MatrixPostOpts {
+  txnId?: string;
+}
+
 export interface RuntimeContext<TConfig = MergedConfig> {
   /** Authenticated user identity (validated by core auth middleware). */
   user: {
@@ -492,7 +503,11 @@ export interface RuntimeContext<TConfig = MergedConfig> {
 
   /** Matrix client, scoped operations only. */
   matrix: {
-    postToRoom: (roomId: string, content: unknown) => Promise<string>;
+    postToRoom: (
+      roomId: string,
+      content: unknown,
+      opts?: MatrixPostOpts,
+    ) => Promise<string>;
     /**
      * Post a timeline event with a caller-chosen event type (e.g. the
      * `ixo.oracle.*` protocol events). `postToRoom` is the `m.room.message`
@@ -502,6 +517,7 @@ export interface RuntimeContext<TConfig = MergedConfig> {
       roomId: string,
       eventType: string,
       content: object,
+      opts?: MatrixPostOpts,
     ) => Promise<string>;
     getRoomState: (roomId: string) => Promise<RoomStateSnapshot>;
     getEventById: (roomId: string, eventId: string) => Promise<MatrixEvent>;
@@ -637,6 +653,14 @@ export interface RuntimeContext<TConfig = MergedConfig> {
    */
   onTurnEnd?: (dispose: () => void | Promise<void>) => void;
 
+  /**
+   * Keep un-awaited work alive past the end of the request that started it
+   * (a best-effort room post retried across a gateway restart). On Workers
+   * this is the Durable Object's `waitUntil`; absent on hosts where pending
+   * promises run to completion on their own.
+   */
+  background?: (work: Promise<unknown>) => void;
+
   /** Propagates from the HTTP request / graph invocation. */
   abortSignal: AbortSignal;
 

package/src/plugins/portal/action-log.ts

@@ -2,7 +2,16 @@
  * The Node runtime's `logActionToMatrix`: a best-effort `ixo.action.log`
  * room event recording a frontend action (browser tool / AG-UI action) and
  * its outcome, posted only when the turn has a room. Never throws.
+ *
+ * Retried across a gateway restart (the gateway object is replaced on every
+ * deploy and can be drained mid-turn) and handed to the host to keep alive
+ * past the turn, so the audit trail does not lose entries to resets. The
+ * post carries one transaction id for the life of its retry loop, so a
+ * response lost between the homeserver's ack and the reply to the gateway
+ * is deduplicated instead of logged twice.
  */
+import { retryGateway, type RetryGatewayOptions } from '../../do/gateway-retry';
+import { retryTxnId } from '../../matrix/txn-id';
 import type { RuntimeContext } from '../../plugin-api/types';
 
 export const ACTION_LOG_EVENT_TYPE = 'ixo.action.log';
@@ -16,19 +25,37 @@ export interface LoggedAction {
 }
 
 export function logActionToMatrix(
-  ctx: Pick<RuntimeContext, 'matrix' | 'session' | 'logger'>,
+  ctx: Pick<RuntimeContext, 'matrix' | 'session' | 'logger' | 'background'>,
   action: LoggedAction,
+  retry?: Pick<RetryGatewayOptions, 'delaysMs' | 'sleep' | 'isTransient'>,
 ): void {
   const roomId = ctx.session.roomId;
   if (!roomId) return;
-  void ctx.matrix
-    .postEvent(roomId, ACTION_LOG_EVENT_TYPE, {
-      action,
-      threadId: ctx.session.id,
-    })
-    .catch((err: unknown) => {
+  const errorText = (err: unknown): string =>
+    err instanceof Error ? err.message : String(err);
+  const txnId = retryTxnId('action-log');
+  const work = retryGateway(
+    () =>
+      ctx.matrix.postEvent(
+        roomId,
+        ACTION_LOG_EVENT_TYPE,
+        { action, threadId: ctx.session.id },
+        { txnId },
+      ),
+    {
+      ...retry,
+      onRetry: (err, attempt, delayMs) =>
+        ctx.logger.warn?.(
+          `[portal] action log for ${action.name} failed (attempt ${attempt}) — retrying in ${delayMs} ms: ${errorText(err)}`,
+        ),
+    },
+  ).then(
+    () => undefined,
+    (err: unknown) => {
       ctx.logger.warn?.(
-        `[portal] action log for ${action.name} not posted: ${err instanceof Error ? err.message : String(err)}`,
+        `[portal] action log for ${action.name} not posted: ${errorText(err)}`,
       );
-    });
+    },
+  );
+  ctx.background?.(work);
 }

package/src/plugins/tasks/tasks-tools.ts

@@ -17,11 +17,8 @@ import type {
   RuntimeContext,
 } from '../../plugin-api/types';
 import { summarizeSchedule } from '../../tasks/schedule';
-import {
-  TASK_ID_PATTERN,
-  TASK_STATUSES,
-  TaskScheduleSchema,
-} from '../../tasks/spec';
+import { TASK_ID_PATTERN, TASK_STATUSES } from '../../tasks/spec';
+import { ScheduleInputToScheduleSchema } from '../../tasks/schedule-input';
 import { pendingApprovalOf } from '../../tasks/store';
 
 const TASKS_UNAVAILABLE = {
@@ -51,7 +48,10 @@ const taskInputShape = {
     .describe(
       'Markdown instructions the run executes. Scheduled runs are BACKGROUND sessions with NO memory of this conversation — put every ID, URL, and name the run needs into this text, never assume the run "knows" anything discussed here.',
     ),
-  schedule: TaskScheduleSchema,
+  // Flat at the tool boundary (a `kind` enum + optional per-kind fields),
+  // converted to the store's discriminated union on parse — see
+  // tasks/schedule-input.ts for why the union itself cannot be the input.
+  schedule: ScheduleInputToScheduleSchema,
   approval: z
     .enum(['never', 'before-action'])
     .default('never')
@@ -259,7 +259,7 @@ const updateInput = z.object({
   taskId: taskIdSchema.shape.taskId,
   title: z.string().min(1).max(120).optional(),
   intent: taskInputShape.intent.optional(),
-  schedule: TaskScheduleSchema.optional(),
+  schedule: ScheduleInputToScheduleSchema.optional(),
   approval: z
     .enum(['never', 'before-action'])
     .optional()