@ixo/oracle-runtime-workers 0.21.1 → 0.22.0

package/src/core/tool-scheduler.ts

@@ -2,18 +2,61 @@
  * Bounded tool concurrency for one user object, shared by every session and
  * every turn that object runs:
  *
- *   - writes run one at a time (a mutation never races another mutation of
- *     the same user's data, even from two simultaneous sessions);
- *   - reads run up to `maxReads` at a time;
- *   - sub-agent dispatches run up to `maxSubagents` at a time, in their own
- *     lane so a child waiting for one of the parent's slots can never
- *     deadlock the parent.
+ *   - `write`: one at a time across every session (a mutation never races
+ *     another mutation of the same user's data, even from two simultaneous
+ *     sessions);
+ *   - `read`: up to `maxReads` at a time across every session;
+ *   - `frontend:<sessionId>`: a tool that acts in the user's browser
+ *     (`PluginTool.surface: 'frontend'`), one at a time per session and
+ *     never in the user-wide lanes above, so a call parked while its chat
+ *     has no tab holds no slot another chat needs;
+ *   - `subagent:<sessionId>`: sub-agent dispatches, up to `maxSubagents` at
+ *     a time per session, in their own lane so a child waiting for one of
+ *     the parent's slots can never deadlock the parent, and a dispatch
+ *     waiting in one chat (an inner frontend call parked) never holds up
+ *     another chat's.
+ *
+ * The per-session lanes are created on demand and removed once idle (no
+ * call in flight or queued), so only busy ones exist.
  *
  * A wait is cancelled by the turn's abort signal: the waiter leaves the
  * queue and the signal's reason is thrown, so an aborted turn never holds a
  * slot it will not use.
  */
-export type ToolLane = 'read' | 'write' | 'subagent';
+export type ToolLane = 'read' | 'write' | 'frontend' | 'subagent';
+
+/** The lanes kept per session (`frontend:<sessionId>`, `subagent:<sessionId>`). */
+export type SessionToolLane = Extract<ToolLane, 'frontend' | 'subagent'>;
+
+const isSessionLane = (lane: ToolLane): lane is SessionToolLane =>
+  lane === 'frontend' || lane === 'subagent';
+
+/** The key of a per-session lane: `frontend:<sessionId>` or `subagent:<sessionId>`. */
+export function sessionLaneKey(
+  lane: SessionToolLane,
+  sessionId: string,
+): string {
+  return `${lane}:${sessionId}`;
+}
+
+export interface LaneOccupancy {
+  inFlight: number;
+  queued: number;
+}
+
+export interface ToolSchedulerSnapshot {
+  read: LaneOccupancy;
+  write: LaneOccupancy;
+  /** Summed over every session's frontend lane. */
+  frontend: LaneOccupancy;
+  /** Summed over every session's sub-agent lane. */
+  subagent: LaneOccupancy;
+  /**
+   * Per-session lanes by key (`sessionLaneKey`); only lanes with a call in
+   * flight or queued exist.
+   */
+  sessionLanes: Record<string, LaneOccupancy>;
+}
 
 class Lane {
   private active = 0;
@@ -65,6 +108,14 @@ class Lane {
   get queued(): number {
     return this.waiting.length;
   }
+
+  get idle(): boolean {
+    return this.active === 0 && this.waiting.length === 0;
+  }
+
+  get occupancy(): LaneOccupancy {
+    return { inFlight: this.active, queued: this.waiting.length };
+  }
 }
 
 /** The signal's reason as an Error (an aborter may pass any value). */
@@ -86,40 +137,94 @@ export interface ToolSchedulerOptions {
 }
 
 export class ToolScheduler {
-  private readonly lanes: Record<ToolLane, Lane>;
+  private readonly read: Lane;
+  private readonly write = new Lane(1);
+  private readonly maxSubagents: number;
+  /** Per-session lanes by `sessionLaneKey`; an idle lane is removed. */
+  private readonly sessionLanes = new Map<
+    string,
+    { kind: SessionToolLane; lane: Lane }
+  >();
 
   constructor(options: ToolSchedulerOptions = {}) {
-    this.lanes = {
-      write: new Lane(1),
-      read: new Lane(options.maxReads ?? 4),
-      subagent: new Lane(options.maxSubagents ?? 4),
-    };
+    this.read = new Lane(options.maxReads ?? 4);
+    this.maxSubagents = options.maxSubagents ?? 4;
   }
 
+  /**
+   * Run `action` once a slot of `lane` is free. `sessionId` keys the
+   * per-session lanes (`frontend`, `subagent`) and is required for them; the
+   * user-wide `read` and `write` lanes ignore it.
+   */
   async run<T>(
     lane: ToolLane,
     signal: AbortSignal | undefined,
     action: () => Promise<T>,
+    sessionId?: string,
   ): Promise<T> {
-    const slot = this.lanes[lane];
-    await slot.acquire(signal);
+    const key = isSessionLane(lane)
+      ? sessionLaneKey(lane, requireSessionId(lane, sessionId))
+      : undefined;
+    const slot = this.laneFor(lane, key);
+    try {
+      await slot.acquire(signal);
+    } catch (error) {
+      this.dropIfIdle(key, slot);
+      throw error;
+    }
     try {
       return await action();
     } finally {
       slot.release();
+      this.dropIfIdle(key, slot);
     }
   }
 
-  /** Occupancy per lane (diagnostics). */
-  snapshot(): Record<ToolLane, { inFlight: number; queued: number }> {
-    const entry = (lane: Lane) => ({
-      inFlight: lane.inFlight,
-      queued: lane.queued,
-    });
+  /** Occupancy per lane (diagnostics: `GET /debug/runs`). */
+  snapshot(): ToolSchedulerSnapshot {
+    const sessionLanes: Record<string, LaneOccupancy> = {};
+    const summed: Record<SessionToolLane, LaneOccupancy> = {
+      frontend: { inFlight: 0, queued: 0 },
+      subagent: { inFlight: 0, queued: 0 },
+    };
+    for (const [key, { kind, lane }] of this.sessionLanes) {
+      sessionLanes[key] = lane.occupancy;
+      summed[kind].inFlight += lane.inFlight;
+      summed[kind].queued += lane.queued;
+    }
     return {
-      read: entry(this.lanes.read),
-      write: entry(this.lanes.write),
-      subagent: entry(this.lanes.subagent),
+      read: this.read.occupancy,
+      write: this.write.occupancy,
+      frontend: summed.frontend,
+      subagent: summed.subagent,
+      sessionLanes,
     };
   }
+
+  private laneFor(lane: ToolLane, key: string | undefined): Lane {
+    if (!isSessionLane(lane) || key === undefined)
+      return lane === 'read' ? this.read : this.write;
+    const existing = this.sessionLanes.get(key);
+    if (existing) return existing.lane;
+    const created = new Lane(lane === 'frontend' ? 1 : this.maxSubagents);
+    this.sessionLanes.set(key, { kind: lane, lane: created });
+    return created;
+  }
+
+  private dropIfIdle(key: string | undefined, slot: Lane): void {
+    if (key === undefined || !slot.idle) return;
+    if (this.sessionLanes.get(key)?.lane === slot)
+      this.sessionLanes.delete(key);
+  }
+}
+
+function requireSessionId(
+  lane: SessionToolLane,
+  sessionId: string | undefined,
+): string {
+  if (sessionId === undefined)
+    throw new TypeError(
+      `ToolScheduler.run: the ${lane} lane is kept per session; pass the call's sessionId.`,
+    );
+  return sessionId;
 }

package/src/do/active-runs.ts

@@ -0,0 +1,171 @@
+/**
+ * The user-wide view of runs: which of the user's chats are busy right now
+ * (`GET /runs/active`, and `activeRun` on the rows of `GET /sessions`).
+ * Task runs (`task:<id>` sessions) are not chats and stay out, as they stay
+ * out of the session list; Matrix and channel runs are the user's chats
+ * whichever client sent the message, and are in.
+ */
+import type { FrontendWaitingFor } from '../plugin-api/types';
+import { TASK_SESSION_PREFIX } from '../tasks/scheduler';
+import type { RunSummary, SessionSummary } from './contracts';
+import { runSummaryOf } from './run-request';
+import { ACTIVE_RUN_STATUSES, type RunRecord } from './run-store';
+
+/**
+ * Every active (queued / running / recovering) chat run, oldest first (run
+ * id breaks a tie). `live` are the coordinator's records, `stored` the
+ * store's active rows (`RunStore.listActive()`): a run in both is listed
+ * once with its live record, which is ahead of the row between writes (a
+ * dequeued run is `running` in memory before its row says so). After a boot
+ * whose recovery scan succeeded every active row is live and the host
+ * passes no `stored` rows: a run leaves the live map before its row is
+ * closed, so a row read then would still say it runs.
+ */
+export function activeChatRuns(
+  live: Iterable<RunRecord>,
+  stored: Iterable<RunRecord>,
+): RunRecord[] {
+  const byRunId = new Map<string, RunRecord>();
+  for (const record of stored) byRunId.set(record.runId, record);
+  for (const record of live) byRunId.set(record.runId, record);
+  return [...byRunId.values()]
+    .filter(
+      (record) =>
+        ACTIVE_RUN_STATUSES.has(record.status) &&
+        !record.sessionId.startsWith(TASK_SESSION_PREFIX),
+    )
+    .sort(
+      (a, b) =>
+        (a.startedAt < b.startedAt ? -1 : a.startedAt > b.startedAt ? 1 : 0) ||
+        a.runId.localeCompare(b.runId),
+    );
+}
+
+/** The room a run's stored request names (`turn.roomId`), if any. */
+export function storedTurnRoomId(request: string): string | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(request);
+  } catch {
+    return undefined;
+  }
+  const turn: unknown =
+    typeof parsed === 'object' && parsed !== null
+      ? Reflect.get(parsed, 'turn')
+      : undefined;
+  const roomId: unknown =
+    typeof turn === 'object' && turn !== null
+      ? Reflect.get(turn, 'roomId')
+      : undefined;
+  return typeof roomId === 'string' ? roomId : undefined;
+}
+
+/**
+ * The runs of the chats `GET /sessions` lists: with a main oracle room, only
+ * sessions in that room (`sessionRooms`: the room of each run's session
+ * that has a row). A run whose session has no row yet — a Matrix thread is
+ * recorded inside its first attempt — is placed by the room its request
+ * names; a Portal run without one is listed (the Portal opens its sessions
+ * in the main room). Without a main room every chat is listed, as
+ * `listSessions` lists every session then. Task runs are already out
+ * (`activeChatRuns`).
+ */
+export function listedChatRuns(
+  runs: readonly RunRecord[],
+  o: {
+    mainRoomId: string | null;
+    sessionRooms: ReadonlyMap<string, string | null>;
+  },
+): RunRecord[] {
+  const { mainRoomId, sessionRooms } = o;
+  if (mainRoomId === null) return [...runs];
+  return runs.filter((run) => {
+    if (sessionRooms.has(run.sessionId))
+      return sessionRooms.get(run.sessionId) === mainRoomId;
+    const roomId = storedTurnRoomId(run.request);
+    if (roomId !== undefined) return roomId === mainRoomId;
+    return run.client === 'portal';
+  });
+}
+
+/** `running` before `recovering` before `queued`, as `RunStore.activeForSession` ranks them. */
+const STATUS_RANK: Partial<Record<RunRecord['status'], number>> = {
+  running: 0,
+  recovering: 1,
+  queued: 2,
+};
+
+/**
+ * The run each session is busy with, from `activeChatRuns` (oldest first):
+ * the running or recovering one, else the oldest queued one — the run
+ * `GET /sessions/:id/run` reports for that session.
+ */
+export function activeRunBySession(
+  runs: readonly RunRecord[],
+): Map<string, RunRecord> {
+  const bySession = new Map<string, RunRecord>();
+  for (const run of runs) {
+    const current = bySession.get(run.sessionId);
+    if (
+      !current ||
+      (STATUS_RANK[run.status] ?? 3) < (STATUS_RANK[current.status] ?? 3)
+    )
+      bySession.set(run.sessionId, run);
+  }
+  return bySession;
+}
+
+/**
+ * The session's parked frontend call, if any (`ctx.frontend.parkedFor`):
+ * the object's live realtime state, never stored.
+ */
+export type ParkedCallLookup = (sessionId: string) => FrontendWaitingFor | null;
+
+/**
+ * `summary` with `waitingFor` set when the run executes (`running` or
+ * `recovering`) and its session has a parked call: only the executing run
+ * of a session can have made one. No field at all otherwise (a queued run,
+ * a run that is over).
+ */
+export function withWaitingFor(
+  summary: RunSummary,
+  parkedFor?: ParkedCallLookup,
+): RunSummary {
+  if (
+    !parkedFor ||
+    (summary.status !== 'running' && summary.status !== 'recovering')
+  )
+    return summary;
+  const waitingFor = parkedFor(summary.sessionId);
+  return waitingFor ? { ...summary, waitingFor } : summary;
+}
+
+/** A run as the user-wide listings report it: without the kept reply text. */
+export function activeRunSummaryOf(
+  record: RunRecord,
+  parkedFor?: ParkedCallLookup,
+): RunSummary {
+  return withWaitingFor(
+    runSummaryOf(record, { withPartialText: false }),
+    parkedFor,
+  );
+}
+
+/**
+ * Session rows with `activeRun` set on the ones that have an active run.
+ * A row without one gets no field at all (never `null`), so clients that
+ * predate it see the rows they always did.
+ */
+export function withActiveRuns(
+  sessions: readonly SessionSummary[],
+  runs: readonly RunRecord[],
+  parkedFor?: ParkedCallLookup,
+): SessionSummary[] {
+  const bySession = activeRunBySession(runs);
+  return sessions.map((session) => {
+    const run = bySession.get(session.sessionId);
+    return run
+      ? { ...session, activeRun: activeRunSummaryOf(run, parkedFor) }
+      : session;
+  });
+}

package/src/do/contracts.ts

@@ -14,9 +14,11 @@ import type {
 } from '@ixo/common/ai/decisions';
 import type { TranscriptPageOptions } from './transcript';
 import type { TurnUsage } from '../core/turn-budget';
+import type { ToolSchedulerSnapshot } from '../core/tool-scheduler';
 import type { BotStatus, EncryptedFileInfo } from '@ixo/matrix-bot-workers-sdk';
 import type { AttachmentInput } from '../attachments/types';
 import type { RealtimeStatus } from '../realtime/realtime-endpoint';
+import type { FrontendWaitingFor } from '../plugin-api/types';
 import type { TierFlushResult, TierStatus } from '../sqlite/do-vfs';
 import type { OwnedArtifact } from '../artifacts/store';
 import type {
@@ -251,6 +253,8 @@ export interface OracleWorkerEnv {
   TURN_MAX_IDENTICAL_READS?: string;
   /** Time limit per request-admission handler in ms (default 2000). */
   REQUEST_ADMISSION_TIMEOUT_MS?: string;
+  /** How long a frontend call waits unsent for a browser, in ms (default 300000). */
+  FRONTEND_PARK_TIMEOUT_MS?: string;
   /** Durable-run knobs (docs/plans/durable-runs.md); see `runDurabilityConfig`. */
   RUN_KEEPALIVE_MS?: string;
   RUN_SEGMENT_FLUSH_MS?: string;
@@ -405,7 +409,6 @@ export interface TurnRequest {
   taskRunId?: string;
 }
 
-/** A run as the shell reports it (`GET /sessions/:id/run`, `GET /debug/runs`). */
 /** `GET /debug/context` — the context budget a model gets on this object. */
 /**
  * Per-session context counters (`GET /debug/context?session=`): what the
@@ -500,6 +503,10 @@ export interface DomainContextRouterUpdate {
   domainContext: DomainContextProvenance[];
 }
 
+/**
+ * A run as the shell reports it (`GET /sessions/:id/run`, `GET /runs/active`,
+ * `activeRun` on `GET /sessions` rows, `GET /debug/runs`).
+ */
 export interface RunSummary {
   runId: string;
   sessionId: string;
@@ -513,13 +520,27 @@ export interface RunSummary {
     | 'aborted'
     | 'interrupted'
     | 'failed';
+  /**
+   * When the run's row was created (`RunStore.create`), unchanged when a
+   * queued run starts: a dequeued run's elapsed time includes its wait.
+   */
   startedAt: string;
   updatedAt: string;
-  /** Highest frame sequence a re-join can ask `after`. */
+  /**
+   * Highest frame sequence packed when the row was last written; written
+   * when the run ends, so not a live cursor for an active run.
+   */
   lastSeq: number;
   messageId?: string;
   /** The reply so far, for a run that ended without a committed reply. */
   partialText?: string;
+  /**
+   * The browser tool / AG-UI action the run is waiting on, parked because no
+   * browser of the session that can run it is connected: opening the chat
+   * sends it. Live state of this object only, never persisted; absent when
+   * the run waits on nothing.
+   */
+  waitingFor?: FrontendWaitingFor;
 }
 
 /** `GET /debug/runs`: the durable-run configuration, live runs and recent rows. */
@@ -573,6 +594,12 @@ export interface RunsStatus {
     startedAt: string;
     state: 'pending' | 'warned';
   }>;
+  /**
+   * Tool lanes of the object right now (tool-scheduler.ts): the user-wide
+   * `read` / `write` lanes, the per-session `frontend` and `subagent` lanes
+   * summed, and each busy per-session lane by key (`sessionLanes`).
+   */
+  tools: ToolSchedulerSnapshot;
 }
 
 /** Non-streaming turn result. */
@@ -611,6 +638,12 @@ export interface SessionSummary {
   createdAt: string;
   lastUpdatedAt: string;
   roomId?: string;
+  /**
+   * The run the session is busy with (queued / running / recovering), as
+   * `GET /runs/active` reports it: without `partialText`. Absent, never
+   * null, when the session has none.
+   */
+  activeRun?: RunSummary;
 }
 
 /** `search_memory_engine`'s raw upstream schema + whether it converts to Zod. */
@@ -761,12 +794,23 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
     target: FeedbackTarget,
     outcome: FeedbackSettlement,
   ): Promise<void>;
-  abortTurn(sessionId: string): Promise<boolean>;
+  /**
+   * Stop the session's current message. With `identity` (the HTTP route)
+   * the object boots first and the answer waits until the stopped runs
+   * have ended (at most the supersede grace).
+   */
+  abortTurn(sessionId: string, identity?: TurnIdentity): Promise<boolean>;
   /** The session's active (queued/running/recovering) run, if any. */
   sessionRun(
     identity: TurnIdentity,
     sessionId: string,
   ): Promise<RunSummary | null>;
+  /**
+   * `GET /runs/active` — every active run of the chats `GET /sessions`
+   * lists (same main-room rule, task runs excluded), oldest first, without
+   * `partialText`.
+   */
+  activeRuns(identity: TurnIdentity): Promise<RunSummary[]>;
   /** Diagnostics: recent runs with their tool marks and live state. */
   runsStatus(): Promise<RunsStatus>;
   /** `GET /debug/context` — the context budget a model gets on this object. */

package/src/do/run-coordinator.ts

@@ -108,6 +108,8 @@ export interface RunCoordinatorHost {
   checkpointIdOf: (sessionId: string) => Promise<string | null>;
   /** After a run reached a terminal state and its row was closed. */
   onRunEnded?: (record: RunRecord, outcome: RunOutcome) => Promise<void>;
+  /** Before a run's row is created (the host marks that runs may be active). */
+  beforeCreate?: () => Promise<void>;
   /** How long a superseded or aborted attempt gets to wind down (default 5 s). */
   supersedeGraceMs?: number;
 }
@@ -215,6 +217,11 @@ export class RunCoordinator {
     return candidate;
   }
 
+  /** The record of every run this coordinator holds (not ended yet). */
+  records(): RunRecord[] {
+    return [...this.live.values()].map((run) => run.record);
+  }
+
   byRequestId(requestId: string): LiveRun | undefined {
     for (const run of this.live.values())
       if (run.requestId === requestId) return run;
@@ -367,6 +374,7 @@ export class RunCoordinator {
       }
     }
     const checkpointId = await this.host.checkpointIdOf(sessionId);
+    await this.host.beforeCreate?.();
     const record = await this.host.store.create({
       runId: input.runId,
       sessionId,
@@ -423,6 +431,28 @@ export class RunCoordinator {
     return true;
   }
 
+  /**
+   * `abortSession`, resolving once every run it stopped has ended — out of
+   * the live map, its row closed — or after the supersede grace, whichever
+   * comes first. Runs enqueued behind the active one are not stopped and
+   * not waited for.
+   */
+  async abortSessionSettled(sessionId: string): Promise<boolean> {
+    const before = [...this.live.values()].filter(
+      (run) => run.sessionId === sessionId,
+    );
+    const stopped = this.abortSession(sessionId);
+    const ending = before.filter(
+      (run) => this.live.get(run.runId) !== run || run.abort.signal.aborted,
+    );
+    if (ending.length > 0)
+      await withinGrace(
+        this.graceMs,
+        Promise.all(ending.map((run) => run.done)),
+      );
+    return stopped;
+  }
+
   /**
    * End every run of the session: the executing attempt is aborted, queued
    * runs and runs waiting for a recovery attempt are closed, and messages

package/src/do/run-request.ts

@@ -78,7 +78,20 @@ export function clientSurfaceFor(
   };
 }
 
-export function runSummaryOf(record: RunRecord): RunSummary {
+export interface RunSummaryOptions {
+  /**
+   * Carry the run's kept reply text (default true). The user-wide listings
+   * (`GET /runs/active`, `activeRun` on session rows) pass false: the text
+   * can be large and a client polling every chat does not need it; the
+   * session's own `GET /sessions/:id/run` still has it.
+   */
+  withPartialText?: boolean;
+}
+
+export function runSummaryOf(
+  record: RunRecord,
+  { withPartialText = true }: RunSummaryOptions = {},
+): RunSummary {
   return {
     runId: record.runId,
     sessionId: record.sessionId,
@@ -89,7 +102,7 @@ export function runSummaryOf(record: RunRecord): RunSummary {
     updatedAt: record.updatedAt,
     lastSeq: record.lastSeq,
     ...(record.messageId ? { messageId: record.messageId } : {}),
-    ...(record.partialText && record.status !== 'finished'
+    ...(withPartialText && record.partialText && record.status !== 'finished'
       ? { partialText: record.partialText }
       : {}),
   };

package/src/do/run-store-test-do.ts

@@ -5,6 +5,7 @@
  */
 import { DurableObject } from 'cloudflare:workers';
 import { DoSqliteDatabase } from '../sqlite/database';
+import { activeChatRuns } from './active-runs';
 import type { RunSummary } from './contracts';
 import type { PackedSegment } from './run-buffer';
 import { RunCoordinator } from './run-coordinator';
@@ -172,6 +173,11 @@ export class RunStoreTestDO extends DurableObject {
     return (await this.runStore()).listActive();
   }
 
+  /** `activeChatRuns` over the given live records and this store's `listActive()`. */
+  async activeChatRuns(live: RunRecord[]): Promise<RunRecord[]> {
+    return activeChatRuns(live, await (await this.runStore()).listActive());
+  }
+
   async queuedForSession(sessionId: string): Promise<RunRecord[]> {
     return (await this.runStore()).queuedForSession(sessionId);
   }
@@ -202,6 +208,10 @@ export class RunStoreTestDO extends DurableObject {
     await (await this.runStore()).bumpMark(runId, toolCallId);
   }
 
+  async markSent(runId: string, toolCallId: string): Promise<void> {
+    await (await this.runStore()).markSent(runId, toolCallId);
+  }
+
   async finishMark(
     runId: string,
     toolCallId: string,