@ixo/oracle-runtime-workers 0.21.2 → 0.23.0

package/src/do/contracts.ts

@@ -1,3 +1,9 @@
+import type {
+  BeginTopicStep,
+  TopicReceiptReferences,
+} from '@ixo/ixo-transaction/topic-proposals';
+import type { ProposalCommand } from '../action-proposals/store';
+import type { ActionOutcomeV1 } from '@ixo/action-protocol';
 import type {
   TopicDeliverableCommand,
   TopicDeliverableResult,
@@ -14,9 +20,11 @@ import type {
 } from '@ixo/common/ai/decisions';
 import type { TranscriptPageOptions } from './transcript';
 import type { TurnUsage } from '../core/turn-budget';
+import type { ToolSchedulerSnapshot } from '../core/tool-scheduler';
 import type { BotStatus, EncryptedFileInfo } from '@ixo/matrix-bot-workers-sdk';
 import type { AttachmentInput } from '../attachments/types';
 import type { RealtimeStatus } from '../realtime/realtime-endpoint';
+import type { FrontendWaitingFor } from '../plugin-api/types';
 import type { TierFlushResult, TierStatus } from '../sqlite/do-vfs';
 import type { OwnedArtifact } from '../artifacts/store';
 import type {
@@ -48,6 +56,12 @@ import type { DomainContextProvenance } from '../core/domain-context';
 /** Bindings every oracle Worker must declare (see the example `wrangler.jsonc`). */
 export interface OracleWorkerEnv {
   TOPIC_DELIVERABLES_ENABLED?: string;
+  ACTION_PROPOSALS_RPC_TESTNET?: string;
+  ACTION_PROPOSALS_RPC_DEVNET?: string;
+  ACTION_PROPOSALS_RPC_MAINNET?: string;
+  ACTION_PROPOSALS_ENABLED?: string;
+  ACTION_TOPIC_PROPOSALS_ENABLED?: string;
+  ACTION_PROPOSALS_PORTAL_ORIGIN?: string;
   USER_ORACLE: DurableObjectNamespace<UserOracleObject>;
   MATRIX_GATEWAY: DurableObjectNamespace<MatrixGatewayObject>;
 
@@ -70,6 +84,8 @@ export interface OracleWorkerEnv {
   AUTH_HUB?: Fetcher;
   AUTH_HUB_URL?: string;
   AUTH_HUB_CHANNEL_SERVICE_KEY?: string;
+  HUB_ACTIONS_ENABLED?: string;
+  AUTH_HUB_ACTION_OUTCOME_KEY?: string;
 
   // --- matrix -------------------------------------------------------------
   MATRIX_BASE_URL: string;
@@ -251,6 +267,8 @@ export interface OracleWorkerEnv {
   TURN_MAX_IDENTICAL_READS?: string;
   /** Time limit per request-admission handler in ms (default 2000). */
   REQUEST_ADMISSION_TIMEOUT_MS?: string;
+  /** How long a frontend call waits unsent for a browser, in ms (default 300000). */
+  FRONTEND_PARK_TIMEOUT_MS?: string;
   /** Durable-run knobs (docs/plans/durable-runs.md); see `runDurabilityConfig`. */
   RUN_KEEPALIVE_MS?: string;
   RUN_SEGMENT_FLUSH_MS?: string;
@@ -367,6 +385,7 @@ export interface TurnRequest {
     /** A provider id (`/^[a-z][a-z0-9_-]{0,31}$/`), e.g. `whatsapp`. */
     provider: string;
     bindingId: string;
+    bindingRevision?: number;
     remoteMessageRef: string;
   };
   executionProfile?: TaskExecutionProfile;
@@ -405,7 +424,6 @@ export interface TurnRequest {
   taskRunId?: string;
 }
 
-/** A run as the shell reports it (`GET /sessions/:id/run`, `GET /debug/runs`). */
 /** `GET /debug/context` — the context budget a model gets on this object. */
 /**
  * Per-session context counters (`GET /debug/context?session=`): what the
@@ -500,6 +518,10 @@ export interface DomainContextRouterUpdate {
   domainContext: DomainContextProvenance[];
 }
 
+/**
+ * A run as the shell reports it (`GET /sessions/:id/run`, `GET /runs/active`,
+ * `activeRun` on `GET /sessions` rows, `GET /debug/runs`).
+ */
 export interface RunSummary {
   runId: string;
   sessionId: string;
@@ -513,13 +535,27 @@ export interface RunSummary {
     | 'aborted'
     | 'interrupted'
     | 'failed';
+  /**
+   * When the run's row was created (`RunStore.create`), unchanged when a
+   * queued run starts: a dequeued run's elapsed time includes its wait.
+   */
   startedAt: string;
   updatedAt: string;
-  /** Highest frame sequence a re-join can ask `after`. */
+  /**
+   * Highest frame sequence packed when the row was last written; written
+   * when the run ends, so not a live cursor for an active run.
+   */
   lastSeq: number;
   messageId?: string;
   /** The reply so far, for a run that ended without a committed reply. */
   partialText?: string;
+  /**
+   * The browser tool / AG-UI action the run is waiting on, parked because no
+   * browser of the session that can run it is connected: opening the chat
+   * sends it. Live state of this object only, never persisted; absent when
+   * the run waits on nothing.
+   */
+  waitingFor?: FrontendWaitingFor;
 }
 
 /** `GET /debug/runs`: the durable-run configuration, live runs and recent rows. */
@@ -573,6 +609,12 @@ export interface RunsStatus {
     startedAt: string;
     state: 'pending' | 'warned';
   }>;
+  /**
+   * Tool lanes of the object right now (tool-scheduler.ts): the user-wide
+   * `read` / `write` lanes, the per-session `frontend` and `subagent` lanes
+   * summed, and each busy per-session lane by key (`sessionLanes`).
+   */
+  tools: ToolSchedulerSnapshot;
 }
 
 /** Non-streaming turn result. */
@@ -611,6 +653,12 @@ export interface SessionSummary {
   createdAt: string;
   lastUpdatedAt: string;
   roomId?: string;
+  /**
+   * The run the session is busy with (queued / running / recovering), as
+   * `GET /runs/active` reports it: without `partialText`. Absent, never
+   * null, when the session has none.
+   */
+  activeRun?: RunSummary;
 }
 
 /** `search_memory_engine`'s raw upstream schema + whether it converts to Zod. */
@@ -688,6 +736,7 @@ export interface StorageStatus {
  * piped straight to the client.
  */
 export interface UserOracleObject extends Rpc.DurableObjectBranded {
+  recordHubActionOutcome(outcome: ActionOutcomeV1): Promise<void>;
   topicDeliverable(
     identity: TurnIdentity,
     operationId: string,
@@ -726,6 +775,18 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
     opts?: { limit?: number; offset?: number },
   ): Promise<{ sessions: SessionSummary[]; total: number }>;
   deleteSession(identity: TurnIdentity, sessionId: string): Promise<boolean>;
+  actionProposal(
+    identity: TurnIdentity,
+    id: string,
+    command?:
+      | ProposalCommand
+      | { kind: 'topic-reconcile'; input: TopicReceiptReferences },
+  ): Promise<JsonString | null>;
+  beginTopicStep(
+    identity: TurnIdentity,
+    id: string,
+    input: BeginTopicStep,
+  ): Promise<JsonString | null>;
   /** `GET /artifacts/:id` — the owner's canonical copy, or null. */
   artifact(
     identity: TurnIdentity,
@@ -761,12 +822,23 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
     target: FeedbackTarget,
     outcome: FeedbackSettlement,
   ): Promise<void>;
-  abortTurn(sessionId: string): Promise<boolean>;
+  /**
+   * Stop the session's current message. With `identity` (the HTTP route)
+   * the object boots first and the answer waits until the stopped runs
+   * have ended (at most the supersede grace).
+   */
+  abortTurn(sessionId: string, identity?: TurnIdentity): Promise<boolean>;
   /** The session's active (queued/running/recovering) run, if any. */
   sessionRun(
     identity: TurnIdentity,
     sessionId: string,
   ): Promise<RunSummary | null>;
+  /**
+   * `GET /runs/active` — every active run of the chats `GET /sessions`
+   * lists (same main-room rule, task runs excluded), oldest first, without
+   * `partialText`.
+   */
+  activeRuns(identity: TurnIdentity): Promise<RunSummary[]>;
   /** Diagnostics: recent runs with their tool marks and live state. */
   runsStatus(): Promise<RunsStatus>;
   /** `GET /debug/context` — the context budget a model gets on this object. */
@@ -1022,6 +1094,17 @@ export interface MatrixGatewayObject extends Rpc.DurableObjectBranded {
   ): Promise<string>;
   /** Full room state as a JSON-encoded array of `{ type, state_key, content, sender, event_id }`. */
   getRoomState(roomId: string): Promise<JsonString>;
+  getTopicRoomState(roomId: string): Promise<JsonString>;
+  getTopicEvent(
+    roomId: string,
+    eventId: string,
+    opts?: { deadlineAt?: number },
+  ): Promise<JsonString | null>;
+  getTopicHistory(
+    roomId: string,
+    root: string,
+    opts?: { deadlineAt?: number; sender?: string },
+  ): Promise<JsonString>;
   /** One timeline event (decrypted when E2EE) as JSON `{ event_id, type, content, sender, origin_server_ts }`, or null. */
   getEvent(roomId: string, eventId: string): Promise<JsonString | null>;
   /**

package/src/do/run-coordinator.ts

@@ -108,6 +108,8 @@ export interface RunCoordinatorHost {
   checkpointIdOf: (sessionId: string) => Promise<string | null>;
   /** After a run reached a terminal state and its row was closed. */
   onRunEnded?: (record: RunRecord, outcome: RunOutcome) => Promise<void>;
+  /** Before a run's row is created (the host marks that runs may be active). */
+  beforeCreate?: () => Promise<void>;
   /** How long a superseded or aborted attempt gets to wind down (default 5 s). */
   supersedeGraceMs?: number;
 }
@@ -215,6 +217,11 @@ export class RunCoordinator {
     return candidate;
   }
 
+  /** The record of every run this coordinator holds (not ended yet). */
+  records(): RunRecord[] {
+    return [...this.live.values()].map((run) => run.record);
+  }
+
   byRequestId(requestId: string): LiveRun | undefined {
     for (const run of this.live.values())
       if (run.requestId === requestId) return run;
@@ -367,6 +374,7 @@ export class RunCoordinator {
       }
     }
     const checkpointId = await this.host.checkpointIdOf(sessionId);
+    await this.host.beforeCreate?.();
     const record = await this.host.store.create({
       runId: input.runId,
       sessionId,
@@ -423,6 +431,28 @@ export class RunCoordinator {
     return true;
   }
 
+  /**
+   * `abortSession`, resolving once every run it stopped has ended — out of
+   * the live map, its row closed — or after the supersede grace, whichever
+   * comes first. Runs enqueued behind the active one are not stopped and
+   * not waited for.
+   */
+  async abortSessionSettled(sessionId: string): Promise<boolean> {
+    const before = [...this.live.values()].filter(
+      (run) => run.sessionId === sessionId,
+    );
+    const stopped = this.abortSession(sessionId);
+    const ending = before.filter(
+      (run) => this.live.get(run.runId) !== run || run.abort.signal.aborted,
+    );
+    if (ending.length > 0)
+      await withinGrace(
+        this.graceMs,
+        Promise.all(ending.map((run) => run.done)),
+      );
+    return stopped;
+  }
+
   /**
    * End every run of the session: the executing attempt is aborted, queued
    * runs and runs waiting for a recovery attempt are closed, and messages

package/src/do/run-request.ts

@@ -78,7 +78,20 @@ export function clientSurfaceFor(
   };
 }
 
-export function runSummaryOf(record: RunRecord): RunSummary {
+export interface RunSummaryOptions {
+  /**
+   * Carry the run's kept reply text (default true). The user-wide listings
+   * (`GET /runs/active`, `activeRun` on session rows) pass false: the text
+   * can be large and a client polling every chat does not need it; the
+   * session's own `GET /sessions/:id/run` still has it.
+   */
+  withPartialText?: boolean;
+}
+
+export function runSummaryOf(
+  record: RunRecord,
+  { withPartialText = true }: RunSummaryOptions = {},
+): RunSummary {
   return {
     runId: record.runId,
     sessionId: record.sessionId,
@@ -89,7 +102,7 @@ export function runSummaryOf(record: RunRecord): RunSummary {
     updatedAt: record.updatedAt,
     lastSeq: record.lastSeq,
     ...(record.messageId ? { messageId: record.messageId } : {}),
-    ...(record.partialText && record.status !== 'finished'
+    ...(withPartialText && record.partialText && record.status !== 'finished'
       ? { partialText: record.partialText }
       : {}),
   };

package/src/do/run-store-test-do.ts

@@ -5,6 +5,7 @@
  */
 import { DurableObject } from 'cloudflare:workers';
 import { DoSqliteDatabase } from '../sqlite/database';
+import { activeChatRuns } from './active-runs';
 import type { RunSummary } from './contracts';
 import type { PackedSegment } from './run-buffer';
 import { RunCoordinator } from './run-coordinator';
@@ -172,6 +173,11 @@ export class RunStoreTestDO extends DurableObject {
     return (await this.runStore()).listActive();
   }
 
+  /** `activeChatRuns` over the given live records and this store's `listActive()`. */
+  async activeChatRuns(live: RunRecord[]): Promise<RunRecord[]> {
+    return activeChatRuns(live, await (await this.runStore()).listActive());
+  }
+
   async queuedForSession(sessionId: string): Promise<RunRecord[]> {
     return (await this.runStore()).queuedForSession(sessionId);
   }
@@ -202,6 +208,10 @@ export class RunStoreTestDO extends DurableObject {
     await (await this.runStore()).bumpMark(runId, toolCallId);
   }
 
+  async markSent(runId: string, toolCallId: string): Promise<void> {
+    await (await this.runStore()).markSent(runId, toolCallId);
+  }
+
   async finishMark(
     runId: string,
     toolCallId: string,

package/src/do/run-store.ts

@@ -14,6 +14,8 @@
  *   - `turn_tool_marks`: one row per tool call, written BEFORE the tool runs
  *     (`started`) and updated when it returns (`done`). This is what makes a
  *     resume safe: a started-but-unfinished write call is never re-executed.
+ *     A frontend call also records when it reached a browser (`sent_at`):
+ *     one still parked at the restart never left the object and runs again.
  *   - `turn_write_claims`: one row per write whose outcome is not known yet
  *     (tool-execution.ts), keyed by the fingerprint of the tool name and
  *     arguments. Released by a returned outcome; kept by an abort, a deadline
@@ -22,7 +24,8 @@
  *
  * Cost shape (SQLite-backed DO storage bills rows written or deleted): a run
  * costs 1 insert + ~2 updates, a segment 1 insert + 1 delete, a tool call
- * 1 insert + 1 update, a write 1 insert + 1 delete more.
+ * 1 insert + 1 update (a frontend call 1 update more, when it is sent), a
+ * write 1 insert + 1 delete more.
  */
 import type { DoSqliteDatabase } from '../sqlite/database';
 import type { PackedSegment } from './run-buffer';
@@ -169,6 +172,14 @@ export interface ToolMark {
   outcome: string | null;
   /** Executions started for this call id (a read re-run bumps it). */
   attempts: number;
+  /**
+   * When the call was handed to its executor, for tools that run outside
+   * the object (frontend tools: the moment a browser socket received it).
+   * `null` while a frontend call is parked waiting for a socket, so a
+   * restart knows the call never left the object. Always `null` for tools
+   * that run in the object.
+   */
+  sentAt: string | null;
 }
 
 /** Rows of ended runs are kept a week (a re-join that late gets 404). */
@@ -331,6 +342,7 @@ type MarkRow = {
   done_at: string | null;
   outcome: string | null;
   attempts: number;
+  sent_at: string | null;
 } & Record<string, string | number | null>;
 
 type SegmentRow = {
@@ -393,6 +405,7 @@ function toMark(row: MarkRow): ToolMark {
     doneAt: row.done_at,
     outcome: row.outcome,
     attempts: Number(row.attempts),
+    sentAt: row.sent_at,
   };
 }
 
@@ -474,8 +487,21 @@ export class RunStore {
         done_at TEXT,
         outcome TEXT,
         attempts INTEGER NOT NULL DEFAULT 1,
+        sent_at TEXT,
         PRIMARY KEY (run_id, tool_call_id)
       ) WITHOUT ROWID`);
+    // When a call left the object (frontend calls; see `markSent`). Marks
+    // from before the column cannot tell, so they count as sent: a resume
+    // keeps treating them as it did when they were written.
+    const markColumns = await this.db.exec<{ name: string }>(
+      'PRAGMA table_info(turn_tool_marks)',
+    );
+    if (!markColumns.some((c) => c.name === 'sent_at')) {
+      await this.db.run('ALTER TABLE turn_tool_marks ADD COLUMN sent_at TEXT');
+      await this.db.run(
+        'UPDATE turn_tool_marks SET sent_at = started_at WHERE sent_at IS NULL',
+      );
+    }
     await this.db.run(`
       CREATE TABLE IF NOT EXISTS turn_write_claims (
         fingerprint TEXT PRIMARY KEY,
@@ -705,12 +731,12 @@ export class RunStore {
     return row ? toRecord(row) : undefined;
   }
 
-  /** Every run that is not over, oldest first. */
+  /** Every run that is not over, oldest first (run id breaks a tie). */
   async listActive(): Promise<RunRecord[]> {
     await this.setup();
     const rows = await this.db.exec<RunRow>(
       `SELECT ${RUN_COLUMNS} FROM turn_runs
-       WHERE status IN ('running','recovering','queued') ORDER BY started_at ASC`,
+       WHERE status IN ('running','recovering','queued') ORDER BY started_at ASC, run_id ASC`,
     );
     return rows.map(toRecord);
   }
@@ -853,16 +879,32 @@ export class RunStore {
     return undefined;
   }
 
-  /** A read-only call re-executed on resume: count the execution, keep the row open. */
+  /**
+   * A call re-executed on resume: count the execution, keep the row open.
+   * The new execution has not been handed to an executor yet, so `sent_at`
+   * is cleared with it.
+   */
   async bumpMark(runId: string, toolCallId: string): Promise<void> {
     await this.setup();
     await this.db.run(
-      `UPDATE turn_tool_marks SET attempts = attempts + 1, started_at = ?, done_at = NULL, outcome = NULL
+      `UPDATE turn_tool_marks SET attempts = attempts + 1, started_at = ?, done_at = NULL, outcome = NULL, sent_at = NULL
        WHERE run_id = ? AND tool_call_id = ?`,
       [this.iso(), runId, toolCallId],
     );
   }
 
+  /**
+   * The call left the object: its executor (a browser socket for a frontend
+   * tool) received it. From here on a restart cannot tell whether it ran.
+   */
+  async markSent(runId: string, toolCallId: string): Promise<void> {
+    await this.setup();
+    await this.db.run(
+      `UPDATE turn_tool_marks SET sent_at = ? WHERE run_id = ? AND tool_call_id = ?`,
+      [this.iso(), runId, toolCallId],
+    );
+  }
+
   async finishMark(
     runId: string,
     toolCallId: string,
@@ -881,7 +923,7 @@ export class RunStore {
   ): Promise<ToolMark | undefined> {
     await this.setup();
     const row = await this.db.get<MarkRow>(
-      `SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts
+      `SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts, sent_at
        FROM turn_tool_marks WHERE run_id = ? AND tool_call_id = ?`,
       [runId, toolCallId],
     );
@@ -891,7 +933,7 @@ export class RunStore {
   async listMarks(runId: string): Promise<ToolMark[]> {
     await this.setup();
     const rows = await this.db.exec<MarkRow>(
-      `SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts
+      `SELECT run_id, tool_call_id, tool_name, effect, started_at, done_at, outcome, attempts, sent_at
        FROM turn_tool_marks WHERE run_id = ? ORDER BY started_at ASC`,
       [runId],
     );
@@ -902,10 +944,14 @@ export class RunStore {
 
   /**
    * Claim a write before it runs. `claimed` when no identical write is
-   * outstanding. `blocked` when one is: a `pending` claim becomes `warned`
-   * (owned by this run, so the same turn stays blocked); a claim already
-   * `warned` by an earlier run is released to this run — the user was told
-   * and asked again.
+   * outstanding. `blocked` when one is: a `pending` claim of this run, or of
+   * a run that is over, becomes `warned` (owned by this run, so the same
+   * turn stays blocked); a `pending` claim of ANOTHER run that is still
+   * active (queued / running / recovering) is left exactly as it is — its
+   * owner may still release it when its write returns, and a takeover would
+   * make that release delete nothing; a claim already `warned` by an
+   * earlier run is released to this run — the user was told and asked
+   * again.
    */
   async claimWrite(input: {
     fingerprint: string;
@@ -939,9 +985,16 @@ export class RunStore {
       return { status: 'claimed' };
     }
     if (existing.state === 'pending')
+      // One statement: the owner's liveness is read where the row is
+      // written, so no release or run end can slip in between.
       await this.db.run(
-        `UPDATE turn_write_claims SET state = 'warned', run_id = ? WHERE fingerprint = ? AND state = 'pending'`,
-        [input.runId, input.fingerprint],
+        `UPDATE turn_write_claims SET state = 'warned', run_id = ?
+         WHERE fingerprint = ? AND state = 'pending'
+           AND (run_id = ? OR NOT EXISTS (
+             SELECT 1 FROM turn_runs r WHERE r.run_id = turn_write_claims.run_id
+               AND r.status IN ('running','recovering','queued')
+           ))`,
+        [input.runId, input.fingerprint, input.runId],
       );
     return {
       status: 'blocked',

package/src/do/turn-body.ts

@@ -5,6 +5,8 @@
  * missing `message` are all 400s, never a failed turn or a 500.
  */
 
+import { declaredToolEffect } from '../plugins/portal/declared-tools';
+
 /** Top-level fields of Node's `SendMessageDto`; anything else is rejected. */
 export const TURN_BODY_KEYS: ReadonlySet<string> = new Set([
   'message',
@@ -45,6 +47,8 @@ export interface TurnBody {
     name: string;
     description: string;
     schema: Record<string, unknown>;
+    /** Declared effect; any value but `'read'` / `'write'` is dropped. */
+    effect?: 'read' | 'write';
   }>;
   agActions?: Array<{
     name: string;
@@ -87,6 +91,14 @@ function toolDeclaration(value: unknown): ToolDeclaration | null {
   };
 }
 
+/** A browser tool declaration: a tool declaration plus its optional `effect`. */
+function browserToolDeclaration(value: unknown): ToolDeclaration | null {
+  const declaration = toolDeclaration(value);
+  if (!declaration || !isRecord(value)) return declaration;
+  const effect = declaredToolEffect(value.effect);
+  return effect ? { ...declaration, effect } : declaration;
+}
+
 /**
  * Parse and validate a raw request body. Never throws. Every field the
  * runtime reads is checked for its type here, so a wrong one is a 400 and
@@ -147,7 +159,7 @@ export function parseTurnBody(raw: string): ParsedTurnBody {
     if (!Array.isArray(record.tools)) return reject('tools must be an array');
     tools = [];
     for (const item of record.tools) {
-      const tool = toolDeclaration(item);
+      const tool = browserToolDeclaration(item);
       if (!tool)
         return reject(
           'each tool needs a string name, description and an object schema',