@ixo/oracle-runtime-workers 0.12.0 → 0.15.0

package/src/do/result-store.ts

@@ -18,7 +18,11 @@
  * R2 objects are removed by `sweep()` at boot; a session's results go with
  * the session. Operators may add an R2 lifecycle rule on the `results/`
  * prefix as a belt on top of the sweep. Identical results de-duplicate by
- * hash (the row's expiry is refreshed instead of a second copy).
+ * hash (the row's expiry is refreshed instead of a second copy); every
+ * session that stored one is recorded in `tool_result_sessions`, and
+ * deleting a session removes only the results no other session holds.
+ * `tool_results.session_id` keeps the first session (the row format is
+ * unchanged).
  */
 import type { DoSqliteDatabase } from '../sqlite/database';
 import type { Logger } from '../plugin-api/types';
@@ -64,6 +68,38 @@ export const DEFAULT_RESULT_R2_MIN_BYTES = 1_000_000;
 export const SQLITE_RESULT_MAX_BYTES = 1_500_000;
 /** Longest chunk `read` returns (bytes). */
 export const MAX_READ_BYTES = 64 * 1024;
+/** Ids per `IN (…)` statement. */
+const ID_BATCH = 100;
+/** Longest UTF-8 sequence minus its lead byte. */
+const UTF8_MAX_CONTINUATION = 3;
+
+/**
+ * Move a byte range of UTF-8 text onto character boundaries: the start back
+ * to the lead byte of its character, the end back to the start of the
+ * character it splits (or, when that would leave nothing, forward past it).
+ * `bytes` holds the text from byte `base` on and covers the range plus up
+ * to three bytes on either side.
+ */
+export function alignUtf8Range(
+  bytes: Uint8Array,
+  base: number,
+  size: number,
+  start: number,
+  end: number,
+): { start: number; end: number } {
+  const limit = Math.min(size, base + bytes.length);
+  const continues = (at: number) =>
+    at > base && at < limit && (bytes[at - base]! & 0xc0) === 0x80;
+  let from = start;
+  while (continues(from)) from -= 1;
+  let to = Math.max(from, Math.min(end, limit));
+  while (to > from && continues(to)) to -= 1;
+  if (to === from && from < limit) {
+    to = from + 1;
+    while (continues(to)) to += 1;
+  }
+  return { start: from, end: to };
+}
 
 type ResultRow = {
   id: string;
@@ -86,6 +122,13 @@ export async function sha256Hex(bytes: Uint8Array): Promise<string> {
 export class ResultStore {
   private setupPromise: Promise<void> | undefined;
 
+  /**
+   * Stores and removals, one at a time: a removal deletes R2 objects before
+   * their rows, so a put of the same content must not land in between (it
+   * would keep a row whose object is then deleted).
+   */
+  private writes: Promise<unknown> = Promise.resolve();
+
   private readonly now: () => number;
 
   private readonly ttlMs: number;
@@ -130,6 +173,28 @@ export class ResultStore {
     await this.db.run(
       `CREATE INDEX IF NOT EXISTS idx_tool_results_expires ON tool_results(expires_at)`,
     );
+    const hasRefs = await this.db.get(
+      `SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'tool_result_sessions'`,
+    );
+    if (!hasRefs)
+      // Rows stored before results were shared across sessions name their
+      // one session; they become its references in the same transaction
+      // that creates the table, so a crash leaves either both or neither.
+      await this.db.transaction(async () => {
+        await this.db.run(`
+          CREATE TABLE IF NOT EXISTS tool_result_sessions (
+            id TEXT NOT NULL,
+            session_id TEXT NOT NULL,
+            PRIMARY KEY (id, session_id)
+          )`);
+        await this.db.run(
+          `CREATE INDEX IF NOT EXISTS idx_tool_result_sessions_session ON tool_result_sessions(session_id)`,
+        );
+        await this.db.run(
+          `INSERT OR IGNORE INTO tool_result_sessions (id, session_id)
+           SELECT id, session_id FROM tool_results`,
+        );
+      });
     await this.sweep();
   }
 
@@ -150,16 +215,32 @@ export class ResultStore {
     await this.setup();
     const bytes = new TextEncoder().encode(input.content);
     const id = await sha256Hex(bytes);
+    return this.serialized(() => this.store(input, bytes, id));
+  }
+
+  private async store(
+    input: { sessionId: string; toolName: string; content: string },
+    bytes: Uint8Array,
+    id: string,
+  ): Promise<StoredResultRef | undefined> {
     const expiresAt = this.now() + this.ttlMs;
+    // An expired row is not reused: it may be one kept only because its R2
+    // delete failed, with its object already gone. It is stored afresh.
     const existing = await this.db.get<ResultRow>(
-      `SELECT id, tier, size FROM tool_results WHERE id = ?`,
-      [id],
+      `SELECT id, tier, size FROM tool_results WHERE id = ? AND expires_at > ?`,
+      [id, this.now()],
     );
     if (existing) {
-      await this.db.run(`UPDATE tool_results SET expires_at = ? WHERE id = ?`, [
-        expiresAt,
-        id,
-      ]);
+      await this.db.transaction(async () => {
+        await this.db.run(
+          `UPDATE tool_results SET expires_at = ? WHERE id = ?`,
+          [expiresAt, id],
+        );
+        await this.db.run(
+          `INSERT OR IGNORE INTO tool_result_sessions (id, session_id) VALUES (?, ?)`,
+          [id, input.sessionId],
+        );
+      });
       return {
         id,
         size: Number(existing.size),
@@ -183,24 +264,39 @@ export class ResultStore {
         },
       });
     }
-    await this.db.run(
-      `INSERT OR IGNORE INTO tool_results (id, session_id, tool_name, size, tier, content, created_at, expires_at)
-       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
-      [
-        id,
-        input.sessionId,
-        input.toolName,
-        bytes.byteLength,
-        useR2 ? 'r2' : 'sqlite',
-        useR2 ? null : input.content,
-        new Date(this.now()).toISOString(),
-        expiresAt,
-      ],
-    );
+    await this.db.transaction(async () => {
+      await this.db.run(
+        `INSERT INTO tool_results (id, session_id, tool_name, size, tier, content, created_at, expires_at)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT (id) DO UPDATE SET
+           tool_name = excluded.tool_name, size = excluded.size,
+           tier = excluded.tier, content = excluded.content,
+           created_at = excluded.created_at, expires_at = excluded.expires_at`,
+        [
+          id,
+          input.sessionId,
+          input.toolName,
+          bytes.byteLength,
+          useR2 ? 'r2' : 'sqlite',
+          useR2 ? null : input.content,
+          new Date(this.now()).toISOString(),
+          expiresAt,
+        ],
+      );
+      await this.db.run(
+        `INSERT OR IGNORE INTO tool_result_sessions (id, session_id) VALUES (?, ?)`,
+        [id, input.sessionId],
+      );
+    });
     return { id, size: bytes.byteLength, tier: useR2 ? 'r2' : 'sqlite' };
   }
 
-  /** A chunk of a stored result, by byte range. */
+  /**
+   * A chunk of a stored result, by byte range, on UTF-8 character
+   * boundaries: an offset inside a character starts at that character, and
+   * a chunk never ends inside one (`length` and `next` say where it really
+   * ended), so concatenating the chunks gives the text back exactly.
+   */
   async read(
     id: string,
     offset = 0,
@@ -213,80 +309,165 @@ export class ResultStore {
     );
     if (!row) return { status: 'not-found' };
     if (Number(row.expires_at) <= this.now()) {
-      await this.remove([row]);
+      await this.serialized(async () => {
+        // Still expired once it is this removal's turn (a put may have
+        // stored the same content again meanwhile).
+        const expired = await this.db.exec<ResultRow>(
+          `SELECT id, tier FROM tool_results WHERE id = ? AND expires_at <= ?`,
+          [id, this.now()],
+        );
+        await this.remove(expired);
+      });
       return { status: 'expired' };
     }
     const size = Number(row.size);
-    const start = Math.max(0, Math.min(Math.floor(offset), size));
+    const requested = Math.max(0, Math.min(Math.floor(offset), size));
     const span = Math.max(
       1,
-      Math.min(Math.floor(length), MAX_READ_BYTES, size - start),
+      Math.min(Math.floor(length), MAX_READ_BYTES, size - requested),
     );
     const tier: ResultTier = row.tier === 'r2' ? 'r2' : 'sqlite';
-    let slice: Uint8Array;
+    // The bytes around the range too, to find the character boundaries.
+    let base: number;
+    let bytes: Uint8Array;
     if (tier === 'r2') {
+      base = Math.max(0, requested - UTF8_MAX_CONTINUATION);
+      const until = Math.min(size, requested + span + UTF8_MAX_CONTINUATION);
       const object = await this.options.bucket?.get(this.key(id), {
-        range: { offset: start, length: span },
+        range: { offset: base, length: Math.max(1, until - base) },
       });
       if (!object) return { status: 'not-found' };
-      slice = new Uint8Array(await object.arrayBuffer());
+      bytes = new Uint8Array(await object.arrayBuffer());
     } else {
-      slice = new TextEncoder()
-        .encode(row.content ?? '')
-        .subarray(start, start + span);
+      base = 0;
+      bytes = new TextEncoder().encode(row.content ?? '');
     }
-    const text = new TextDecoder().decode(slice);
-    const next = start + span < size ? start + span : null;
+    const { start, end } = alignUtf8Range(
+      bytes,
+      base,
+      size,
+      requested,
+      Math.min(size, requested + span),
+    );
+    const text = new TextDecoder().decode(
+      bytes.subarray(start - base, end - base),
+    );
     return {
       status: 'ok',
       id,
       tier,
       size,
       offset: start,
-      length: span,
+      length: end - start,
       text,
-      next,
+      next: end < size ? end : null,
     };
   }
 
+  /**
+   * Drop the session's references; remove the results no other session
+   * holds (and their R2 objects). Returns how many results were removed.
+   */
   async deleteForSession(sessionId: string): Promise<number> {
     await this.setup();
-    const rows = await this.db.exec<ResultRow>(
-      `SELECT id, tier FROM tool_results WHERE session_id = ?`,
-      [sessionId],
-    );
+    return this.serialized(() => this.removeSession(sessionId));
+  }
+
+  private async removeSession(sessionId: string): Promise<number> {
+    const rows = await this.db.transaction(async () => {
+      const orphaned = await this.db.exec<ResultRow>(
+        `SELECT r.id, r.tier FROM tool_results r
+         WHERE (r.session_id = ?
+                OR EXISTS (SELECT 1 FROM tool_result_sessions s
+                           WHERE s.id = r.id AND s.session_id = ?))
+           AND NOT EXISTS (SELECT 1 FROM tool_result_sessions s
+                           WHERE s.id = r.id AND s.session_id <> ?)`,
+        [sessionId, sessionId, sessionId],
+      );
+      await this.db.run(
+        `DELETE FROM tool_result_sessions WHERE session_id = ?`,
+        [sessionId],
+      );
+      return orphaned;
+    });
     await this.remove(rows);
     return rows.length;
   }
 
   /** Drop expired rows (and their R2 objects). Returns how many. */
-  async sweep(): Promise<number> {
-    const rows = await this.db.exec<ResultRow>(
-      `SELECT id, tier FROM tool_results WHERE expires_at <= ?`,
-      [this.now()],
-    );
-    await this.remove(rows);
-    return rows.length;
+  sweep(): Promise<number> {
+    return this.serialized(async () => {
+      const rows = await this.db.exec<ResultRow>(
+        `SELECT id, tier FROM tool_results WHERE expires_at <= ?`,
+        [this.now()],
+      );
+      await this.remove(rows);
+      return rows.length;
+    });
+  }
+
+  /** Run `work` after every store or removal already in progress. */
+  private serialized<T>(work: () => Promise<T>): Promise<T> {
+    const next = this.writes.then(work);
+    this.writes = next.catch(() => undefined);
+    return next;
   }
 
+  /**
+   * Remove rows and their R2 objects. An R2-tier row whose object could not
+   * be deleted stays, marked expired: it is the only record of the object,
+   * so the next sweep (or read) tries again instead of leaking it.
+   */
   private async remove(
     rows: Array<Pick<ResultRow, 'id' | 'tier'>>,
   ): Promise<void> {
     if (rows.length === 0) return;
-    const r2Keys = rows
-      .filter((r) => r.tier === 'r2')
-      .map((r) => this.key(r.id));
-    if (r2Keys.length > 0 && this.options.bucket) {
-      try {
-        await this.options.bucket.delete(r2Keys);
-      } catch (error) {
-        this.logger.warn(
-          `[results] could not delete ${r2Keys.length} R2 object(s): ${error instanceof Error ? error.message : String(error)}`,
-        );
+    const r2Rows = rows.filter((r) => r.tier === 'r2');
+    const kept = new Set<string>();
+    if (r2Rows.length > 0 && this.options.bucket) {
+      // R2 deletes at most 1,000 keys per call; a failed call keeps only
+      // its own rows.
+      for (let i = 0; i < r2Rows.length; i += 1_000) {
+        const batch = r2Rows.slice(i, i + 1_000);
+        try {
+          await this.options.bucket.delete(batch.map((r) => this.key(r.id)));
+        } catch (error) {
+          this.logger.warn(
+            `[results] could not delete ${batch.length} R2 object(s); their rows stay for the next sweep: ${error instanceof Error ? error.message : String(error)}`,
+          );
+          for (const row of batch) kept.add(row.id);
+        }
       }
     }
-    for (const row of rows)
-      await this.db.run(`DELETE FROM tool_results WHERE id = ?`, [row.id]);
+    const gone = rows.filter((r) => !kept.has(r.id)).map((r) => r.id);
+    await this.db.transaction(async () => {
+      for (let i = 0; i < gone.length; i += ID_BATCH) {
+        const ids = gone.slice(i, i + ID_BATCH);
+        const marks = ids.map(() => '?').join(', ');
+        await this.db.run(
+          `DELETE FROM tool_results WHERE id IN (${marks})`,
+          ids,
+        );
+        await this.db.run(
+          `DELETE FROM tool_result_sessions WHERE id IN (${marks})`,
+          ids,
+        );
+      }
+      const retry = [...kept];
+      for (let i = 0; i < retry.length; i += ID_BATCH) {
+        const ids = retry.slice(i, i + ID_BATCH);
+        const marks = ids.map(() => '?').join(', ');
+        await this.db.run(
+          `UPDATE tool_results SET expires_at = 0 WHERE id IN (${marks})`,
+          ids,
+        );
+        // No session holds it any more; the row only tracks the object.
+        await this.db.run(
+          `DELETE FROM tool_result_sessions WHERE id IN (${marks})`,
+          ids,
+        );
+      }
+    });
   }
 
   async stats(): Promise<{

package/src/do/room-mirror.ts

@@ -1,3 +1,4 @@
+import { channelOrigin } from '../channels/contract';
 /**
  * The room mirror of HTTP chats: every Portal turn is replayed into the
  * user's oracle room as a thread under the session's marker event, the user
@@ -41,6 +42,11 @@ export interface MirrorSend {
   /** The session's marker event — the thread the mirror is posted under. */
   threadId: string;
   txnId: string;
+  origin?: {
+    provider: 'whatsapp';
+    bindingId: string;
+    remoteMessageRef: string;
+  };
 }
 
 export interface RoomMirrorDeps {
@@ -53,6 +59,27 @@ export interface RoomMirrorDeps {
   retry?: Pick<RetryGatewayOptions, 'delaysMs' | 'sleep' | 'isTransient'>;
 }
 
+export function mirrorEventContent(send: MirrorSend) {
+  return {
+    msgtype: 'm.text',
+    body: send.body,
+    ...(send.formattedBody
+      ? { format: 'org.matrix.custom.html', formatted_body: send.formattedBody }
+      : {}),
+    'm.relates_to': {
+      rel_type: 'm.thread',
+      event_id: send.threadId,
+      is_falling_back: true,
+      'm.in_reply_to': { event_id: send.threadId },
+    },
+    ...(send.origin
+      ? {
+          'org.ixo.qi.origin': channelOrigin(send.origin),
+        }
+      : {}),
+  };
+}
+
 const errorText = (err: unknown): string =>
   err instanceof Error ? err.message : String(err);
 
@@ -77,12 +104,14 @@ export class RoomMirror {
     sessionId: string,
     prepare: () => Promise<MirrorSend | null>,
     label: string,
+    required = false,
   ): Promise<void> {
     const previous = this.chains.get(sessionId) ?? Promise.resolve();
     const tracked: Promise<void> = previous
       .then(() => prepare())
       .then((send) => (send ? this.send(send, label) : undefined))
       .catch((err: unknown) => {
+        if (required) throw err;
         this.deps.warn(
           `[user-do] Matrix replay (${label}) failed — session=${sessionId}: ${errorText(err)}`,
         );

package/src/do/run-buffer.ts

@@ -89,9 +89,20 @@ export class RunBuffer {
   /** Frames not yet packed into a segment. */
   private readonly tail: RunFrame[] = [];
 
+  /**
+   * Frames taken out of the tail whose segment write has not settled yet,
+   * one batch per pending pack, in order. A re-join reads the store and then
+   * the tail; without these, a frame packed between the two would be in
+   * neither.
+   */
+  private readonly inFlight: RunFrame[][] = [];
+
   private tailBytes = 0;
 
-  private readonly subscribers = new Set<RunSubscriber>();
+  private readonly subscribers = new Map<RunSubscriber, (() => void) | null>();
+
+  /** The `done` frame, once pushed (a closed buffer keeps it for late readers). */
+  private doneFrame: RunFrame | null = null;
 
   private timer: unknown = null;
 
@@ -115,13 +126,19 @@ export class RunBuffer {
     return this.closed;
   }
 
+  /** A `done` frame has been pushed: the run told its subscribers it ended. */
+  get hasDone(): boolean {
+    return this.doneFrame !== null;
+  }
+
   /** Append a frame, fan it out, and schedule or force a pack. */
   push(event: string, data: unknown): RunFrame {
     if (this.closed) throw new Error('run buffer is closed');
     const frame: RunFrame = { seq: ++this.seq, event, data };
+    if (event === 'done' && this.doneFrame === null) this.doneFrame = frame;
     this.tail.push(frame);
     this.tailBytes += JSON.stringify(frame).length;
-    for (const subscriber of this.subscribers) {
+    for (const subscriber of this.subscribers.keys()) {
       try {
         subscriber(frame);
       } catch {
@@ -144,19 +161,40 @@ export class RunBuffer {
   }
 
   /**
-   * The unflushed frames after `after` — what a re-join needs on top of the
-   * segments it read from the store.
+   * The frames after `after` that a store read may not hold yet — the ones
+   * whose segment write is still pending, then the unflushed tail — what a
+   * re-join needs on top of the segments it read from the store. A closed
+   * buffer also keeps its `done` frame, so a reader that arrives after the
+   * run ended still learns how it ended.
    */
   tailAfter(after: number): RunFrame[] {
-    return this.tail.filter((frame) => frame.seq > after);
+    const out: RunFrame[] = [];
+    for (const batch of this.inFlight)
+      for (const frame of batch) if (frame.seq > after) out.push(frame);
+    for (const frame of this.tail) if (frame.seq > after) out.push(frame);
+    const done = this.doneFrame;
+    if (
+      this.closed &&
+      done !== null &&
+      done.seq > after &&
+      !out.some((frame) => frame.seq === done.seq)
+    )
+      out.push(done);
+    return out;
   }
 
   /**
    * Attach a live subscriber. The caller replays the store's segments and
    * `tailAfter(cursor)` first; from then on every new frame is delivered.
+   * `onEnd` runs once when the buffer closes (at once when it already is),
+   * after the subscriber received every frame.
    */
-  subscribe(subscriber: RunSubscriber): () => void {
-    this.subscribers.add(subscriber);
+  subscribe(subscriber: RunSubscriber, onEnd?: () => void): () => void {
+    if (this.closed) {
+      onEnd?.();
+      return () => undefined;
+    }
+    this.subscribers.set(subscriber, onEnd ?? null);
     return () => {
       this.subscribers.delete(subscriber);
     };
@@ -175,6 +213,7 @@ export class RunBuffer {
     if (this.tail.length === 0) return this.packing;
     const frames = this.tail.splice(0, this.tail.length);
     this.tailBytes = 0;
+    this.inFlight.push(frames);
     const segment: PackedSegment = {
       seqFrom: frames[0]!.seq,
       seqTo: frames[frames.length - 1]!.seq,
@@ -190,15 +229,31 @@ export class RunBuffer {
         // copy is missing. Report and carry on — the next pack retries
         // nothing (the frames are gone) but the run itself is unaffected.
         this.options.onPackError?.(error);
+      })
+      .finally(() => {
+        // Packs settle in order: this batch is the oldest one pending.
+        this.inFlight.shift();
       });
     return this.packing;
   }
 
-  /** Final pack; no more frames accepted afterwards. */
+  /**
+   * Final pack; no more frames accepted afterwards. Every subscriber is
+   * told the buffer ended (after the frames it was already given) and
+   * detached.
+   */
   async close(): Promise<void> {
     if (this.closed) return this.packing;
     this.closed = true;
-    await this.flush();
+    const ended = [...this.subscribers.values()];
     this.subscribers.clear();
+    for (const onEnd of ended) {
+      try {
+        onEnd?.();
+      } catch {
+        /* a broken subscriber never breaks the run */
+      }
+    }
+    await this.flush();
   }
 }