@ixo/oracle-runtime-workers 0.12.0 → 0.15.0

package/src/realtime/realtime-endpoint.ts

@@ -16,12 +16,21 @@
  *   42["ping"] / 42["status"] / 42["list-events"]
  *     → Node's diagnostic events, same payloads.
  *   42["tool_result",{toolCallId,result,error?}] / 42["action_call_result",…]
- *     → settle the pending browser-tool / AG-UI call with that id.
+ *     → settle the browser-tool / AG-UI invocation with that id, when this
+ *       is the socket it was sent to (see "Frontend calls" below).
  *
  * Outbound, every event the runtime emits for a session (`ctx.emit.*`,
- * `browser_tool_call`, `action_call`, …) is fanned out to the session's
+ * `tool_call`, `render_component`, …) is fanned out to the session's
  * authenticated sockets — the router's tap.
  *
+ * Frontend calls (`ctx.frontend.callBrowserTool` / `callAgAction`) are not
+ * fanned out: each invocation gets its own id, is sent to ONE socket of its
+ * session (the one most recently active), and only that socket's answer
+ * settles it. When the
+ * answer cannot come — deadline, or that socket went — the call resolves
+ * with `FRONTEND_OUTCOME_UNKNOWN` and is never re-sent. See
+ * docs/frontend-bridge.md.
+ *
  * Heartbeat: engine.io's server-initiated ping, driven by the OBJECT'S
  * ALARM rather than a timer. A timer would keep the object resident (and
  * billed) for as long as a browser tab is open; with the alarm the object
@@ -40,9 +49,12 @@ import type {
   FrontendCallSurface,
 } from '../plugin-api/types';
 import type { RawEventPayload } from '../core/runtime-context';
+import { frontendInvocationId } from '@ixo/common/ai/frontend-bridge';
 import {
+  FRONTEND_CALL_LABEL,
   FrontendCallRegistry,
   type FrontendCallKind,
+  type FrontendResultRejection,
   type PendingFrontendCall,
 } from './frontend-call-registry';
 import { SessionSocketHub, type SocketAttachment } from './session-socket-hub';
@@ -99,16 +111,38 @@ const SERVER_EVENTS = [
 
 /** Socket event every runtime event travels under, carrying `{ eventName, payload }` (Node's wire). */
 const ENVELOPE_EVENT = 'event';
-/** Frontend calls the client SDK listens for by name with the raw payload. */
-const RAW_NAMED_EVENTS: ReadonlySet<string> = new Set([
+/**
+ * Frontend calls: the client SDK listens for them by name with the raw
+ * payload and EXECUTES them, so they reach a socket only as a dispatched
+ * invocation (`call`), never through the fan-out.
+ */
+const FRONTEND_CALL_EVENTS: ReadonlySet<string> = new Set([
   'browser_tool_call',
   'action_call',
 ]);
 
+/** Warning text per rejected result (identifiers only, never the payload). */
+const REJECTION_TEXT: Record<FrontendResultRejection, string> = {
+  'not-issued': 'no such invocation was issued (or its record expired)',
+  'already-settled': 'the invocation already settled (duplicate or late)',
+  'wrong-socket': 'this is not the socket it was sent to',
+  'wrong-kind': 'the invocation is of the other kind',
+};
+
 /** Close codes in the 4000–4999 application range. */
 const CLOSE_UNAUTHORIZED = 4401;
 const CLOSE_HANDSHAKE_TIMEOUT = 4408;
 const CLOSE_PROTOCOL = 4400;
+/** RFC 6455 "message too big". */
+const CLOSE_TOO_LARGE = 1009;
+
+/** Whether a text frame is over `MAX_PAYLOAD_BYTES` in UTF-8 bytes (encoded only when its length leaves it in doubt). */
+function exceedsPayloadLimit(message: string): boolean {
+  // A UTF-16 unit is at least one UTF-8 byte and at most three.
+  if (message.length > MAX_PAYLOAD_BYTES) return true;
+  if (message.length * 3 <= MAX_PAYLOAD_BYTES) return false;
+  return new TextEncoder().encode(message).byteLength > MAX_PAYLOAD_BYTES;
+}
 
 export interface RealtimeEndpointDeps {
   /** `ctx.acceptWebSocket` / `ctx.getWebSockets` of the owning object. */
@@ -156,6 +190,8 @@ export interface RealtimeStatus {
   authenticated: number;
   sessions: number;
   pendingCalls: PendingFrontendCall[];
+  /** Finished invocations remembered to reject replays (bounded). */
+  completedCalls: number;
   pingIntervalMs: number;
   pingTimeoutMs: number;
   /** When the next heartbeat round is due (null with no sockets). */
@@ -189,7 +225,7 @@ function readString(
 export class RealtimeEndpoint {
   readonly hub: SessionSocketHub<WebSocket>;
 
-  readonly calls = new FrontendCallRegistry();
+  readonly calls: FrontendCallRegistry;
 
   readonly frontend: FrontendCallSurface;
 
@@ -198,11 +234,18 @@ export class RealtimeEndpoint {
     ReturnType<typeof setTimeout>
   >();
 
+  /** Sockets whose CONNECT is being validated: a second CONNECT meanwhile is ignored. */
+  private readonly authenticating = new Set<WebSocket>();
+
   private readonly now: () => number;
 
   constructor(private readonly deps: RealtimeEndpointDeps) {
     this.now = deps.now ?? (() => Date.now());
-    this.hub = new SessionSocketHub<WebSocket>({ now: this.now });
+    this.calls = new FrontendCallRegistry({ now: this.now });
+    this.hub = new SessionSocketHub<WebSocket>({
+      now: this.now,
+      onDropped: (attachment) => this.executorGone(attachment.sid),
+    });
     const tap: EventSink = {
       emit: (eventName, payload) => this.fanOut(eventName, payload),
     };
@@ -223,6 +266,14 @@ export class RealtimeEndpoint {
     const restored = this.hub.restore(deps.ctx.getWebSockets());
     if (restored > 0) {
       deps.logger.log(`[realtime] re-adopted ${restored} hibernated socket(s)`);
+      // A socket that had not authenticated lost its handshake timer with
+      // the old instance: it gets what is left of its window.
+      for (const { socket, attachment } of this.hub.entries())
+        if (!attachment.userDid)
+          this.armHandshakeDeadline(
+            socket,
+            attachment.openedAt + HANDSHAKE_DEADLINE_MS - this.now(),
+          );
       this.scheduleHeartbeat();
     }
   }
@@ -272,16 +323,27 @@ export class RealtimeEndpoint {
         maxPayloadBytes: MAX_PAYLOAD_BYTES,
       }),
     );
+    // Until CONNECT succeeds the handshake timer is the socket's only clock:
+    // no heartbeat alarm is armed for a caller who has not authenticated.
+    this.armHandshakeDeadline(server, HANDSHAKE_DEADLINE_MS);
+    return new Response(null, { status: 101, webSocket: client });
+  }
+
+  /** Close the socket unless it has authenticated within `ms`. */
+  private armHandshakeDeadline(ws: WebSocket, ms: number): void {
+    const existing = this.handshakeTimers.get(ws);
+    if (existing) clearTimeout(existing);
     this.handshakeTimers.set(
-      server,
-      setTimeout(() => {
-        this.handshakeTimers.delete(server);
-        if (this.hub.get(server)?.userDid) return;
-        this.drop(server, CLOSE_HANDSHAKE_TIMEOUT, 'handshake timeout');
-      }, HANDSHAKE_DEADLINE_MS),
+      ws,
+      setTimeout(
+        () => {
+          this.handshakeTimers.delete(ws);
+          if (this.hub.get(ws)?.userDid) return;
+          this.drop(ws, CLOSE_HANDSHAKE_TIMEOUT, 'handshake timeout');
+        },
+        Math.max(0, ms),
+      ),
     );
-    this.scheduleHeartbeat();
-    return new Response(null, { status: 101, webSocket: client });
   }
 
   // ── Hibernatable WebSocket handlers ─────────────────────────────────────
@@ -296,6 +358,11 @@ export class RealtimeEndpoint {
       this.drop(ws, CLOSE_PROTOCOL, 'binary frames are not supported');
       return;
     }
+    if (exceedsPayloadLimit(message)) {
+      // The limit the OPEN handshake advertised; nothing larger is parsed.
+      this.drop(ws, CLOSE_TOO_LARGE, 'frame exceeds maxPayload');
+      return;
+    }
     const frame = decodeClientFrame(message);
     switch (frame.kind) {
       case 'pong':
@@ -317,6 +384,8 @@ export class RealtimeEndpoint {
           this.drop(ws, CLOSE_UNAUTHORIZED, 'event before CONNECT');
           return;
         }
+        // Client activity picks the tab frontend calls run on.
+        this.hub.noteActivity(ws);
         this.handleEvent(ws, meta, frame.name, frame.args[0]);
         if (frame.ackId !== undefined)
           this.hub.send(ws, encodeAck(frame.ackId));
@@ -351,6 +420,7 @@ export class RealtimeEndpoint {
       authenticated: this.hub.connectionCount(),
       sessions: this.hub.sessionCount(),
       pendingCalls: this.calls.list(),
+      completedCalls: this.calls.completedCount,
       pingIntervalMs: PING_INTERVAL_MS,
       pingTimeoutMs: PING_TIMEOUT_MS,
       nextPingAt: this.nextPingAt(),
@@ -400,6 +470,22 @@ export class RealtimeEndpoint {
       this.hub.send(ws, encodeConnectAck(meta.sid));
       return;
     }
+    // One validation per socket at a time: each one may resolve keys from
+    // Blocksync, and a client has no reason to send CONNECT twice.
+    if (this.authenticating.has(ws)) return;
+    this.authenticating.add(ws);
+    try {
+      await this.authenticateConnect(ws, meta, auth);
+    } finally {
+      this.authenticating.delete(ws);
+    }
+  }
+
+  private async authenticateConnect(
+    ws: WebSocket,
+    meta: SocketAttachment,
+    auth: unknown,
+  ): Promise<void> {
     const record = isRecord(auth) ? auth : {};
     const invocation = readString(record, 'invocation');
     const ucanDelegation = readString(record, 'ucanDelegation');
@@ -434,9 +520,27 @@ export class RealtimeEndpoint {
       this.refuse(ws, 'Unauthorized: token does not belong to the routed user');
       return;
     }
-    if (
-      !(await this.deps.sessionExists(outcome.auth.userDid, meta.sessionId))
-    ) {
+    if (outcome.auth.via === 'delegation')
+      // The shell logs the same line per HTTP request: operators find the
+      // clients that still need UCAN_ALLOW_BARE_DELEGATION_AUTH by it.
+      this.deps.logger.warn(
+        `[auth] socket CONNECT ${meta.sessionId}: ${outcome.auth.userDid} authenticated with a bare delegation (UCAN_ALLOW_BARE_DELEGATION_AUTH); the client must send a UCAN invocation before the fallback is turned off`,
+      );
+    let owned: boolean;
+    try {
+      owned = await this.deps.sessionExists(
+        outcome.auth.userDid,
+        meta.sessionId,
+      );
+    } catch (err) {
+      // Ownership that cannot be checked is not granted.
+      this.deps.logger.warn(
+        `[realtime] session check for ${meta.sessionId} failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      this.refuse(ws, 'Unauthorized: session check failed');
+      return;
+    }
+    if (!owned) {
       this.refuse(ws, `Session ${meta.sessionId} not found`);
       return;
     }
@@ -460,6 +564,8 @@ export class RealtimeEndpoint {
     this.deps.logger.log(
       `[realtime] ${outcome.auth.userDid} connected to session ${meta.sessionId} (${meta.sid})`,
     );
+    // From here the heartbeat (the object's alarm) keeps the socket honest.
+    this.scheduleHeartbeat();
   }
 
   private refuse(ws: WebSocket, message: string): void {
@@ -534,21 +640,33 @@ export class RealtimeEndpoint {
   ): void {
     const record = isRecord(data) ? data : {};
     const toolCallId = readString(record, 'toolCallId');
-    if (!toolCallId) {
+    const userDid = meta.userDid;
+    if (!toolCallId || !userDid) {
       this.deps.logger.warn(
         `[realtime] ${kind} result without toolCallId from ${meta.sid}`,
       );
       return;
     }
+    // Routing comes from the socket alone. A client that names another
+    // session is making a claim it cannot make; the result is dropped and
+    // the call keeps waiting for its own socket's answer.
+    const claimed = record.sessionId;
+    if (claimed !== undefined && claimed !== meta.sessionId) {
+      this.deps.logger.warn(
+        `[realtime] ${kind} result for ${toolCallId} rejected: socket ${meta.sid} of session ${meta.sessionId} names session ${typeof claimed === 'string' ? claimed : typeof claimed}`,
+      );
+      return;
+    }
     const error = readString(record, 'error');
-    const settled = this.calls.settle(kind, {
+    const outcome = this.calls.settle(kind, {
       toolCallId,
+      from: { sid: meta.sid, sessionId: meta.sessionId, userDid },
       result: record.result,
       ...(error ? { error } : {}),
     });
-    if (!settled) {
+    if (!outcome.settled) {
       this.deps.logger.warn(
-        `[realtime] ${kind} result for ${toolCallId} had no pending call (late or duplicate)`,
+        `[realtime] ${kind} result for ${toolCallId} from socket ${meta.sid} (session ${meta.sessionId}) rejected: ${REJECTION_TEXT[outcome.reason]}`,
       );
     }
   }
@@ -562,51 +680,102 @@ export class RealtimeEndpoint {
    * SDK's catch-all listener validates exactly that (`payload.sessionId` and
    * `payload.requestId`) and drops a bare payload, while its named
    * `tool_call` / `render_component` listeners crash on one. The two calls
-   * the SDK answers (`browser_tool_call`, `action_call`) are the exception:
-   * it listens for them by name with the raw payload, on both runtimes.
+   * the SDK EXECUTES (`browser_tool_call`, `action_call`) never travel this
+   * way: a frame of theirs here is the SSE stream's mirror of a sub-agent
+   * tool call or a plugin's one-way emit — not an invocation — and a browser
+   * that ran it would act again. Invocations go out through `call`.
    */
   private fanOut(eventName: string, payload: RawEventPayload): void {
     const sessionId =
       typeof payload.sessionId === 'string' ? payload.sessionId : undefined;
-    if (!sessionId) return;
-    if (RAW_NAMED_EVENTS.has(eventName)) {
-      this.hub.emitToSession(sessionId, eventName, payload);
-      return;
-    }
+    if (!sessionId || FRONTEND_CALL_EVENTS.has(eventName)) return;
     this.hub.emitToSession(sessionId, ENVELOPE_EVENT, { eventName, payload });
   }
 
+  /**
+   * One frontend invocation: a fresh id, registered before it is sent (an
+   * immediate answer cannot be missed), sent to the most recently active
+   * authenticated socket of the session (`executorForSession`) and bound to
+   * it. A call that reaches no socket is
+   * a definite failure; one whose answer never comes is an unknown outcome.
+   */
   private call(
     kind: FrontendCallKind,
     eventName: 'browser_tool_call' | 'action_call',
     params: FrontendCallParams,
     defaultTimeoutMs: number,
   ): Promise<unknown> {
-    const timeoutMs = params.timeoutMs ?? defaultTimeoutMs;
-    // Register BEFORE emitting so a synchronous answer cannot be missed.
-    const pending = this.calls.wait(
-      {
-        kind,
-        toolCallId: params.toolCallId,
-        toolName: params.toolName,
-        sessionId: params.sessionId,
-      },
-      { timeoutMs },
-    );
+    const toolCallId = frontendInvocationId(params.toolCallId);
+    const label = FRONTEND_CALL_LABEL[kind];
+    let pending: Promise<unknown>;
+    try {
+      pending = this.calls.open(
+        {
+          kind,
+          toolCallId,
+          toolName: params.toolName,
+          sessionId: params.sessionId,
+        },
+        {
+          timeoutMs: params.timeoutMs ?? defaultTimeoutMs,
+          ...(params.signal ? { signal: params.signal } : {}),
+        },
+      );
+    } catch (err) {
+      return Promise.reject(
+        err instanceof Error ? err : new Error(String(err)),
+      );
+    }
+    // A turn already aborted rejected the call as not sent: send nothing.
+    if (!this.calls.isPending(toolCallId)) return pending;
+    params.onInvocation?.(toolCallId);
     const payload: RawEventPayload = {
       sessionId: params.sessionId,
       requestId: params.toolCallId,
-      toolCallId: params.toolCallId,
+      toolCallId,
       toolName: params.toolName,
       args: params.args,
       ...(kind === 'agui' ? { status: 'isRunning' } : {}),
     };
-    // Through the router: the session's SSE stream sees it too (Node emits
-    // it on the root emitter, which feeds both channels).
-    this.deps.router.emit(eventName, payload);
+    const target = this.hub.executorForSession(params.sessionId);
+    const userDid = target?.attachment.userDid;
+    if (
+      !target ||
+      !userDid ||
+      !this.hub.send(target.socket, encodeEvent(eventName, payload))
+    ) {
+      this.calls.fail(
+        toolCallId,
+        new Error(
+          `${label} ${params.toolName} was not sent: no browser is connected to session ${params.sessionId}`,
+        ),
+      );
+      return pending;
+    }
+    this.calls.dispatched(toolCallId, {
+      sid: target.attachment.sid,
+      sessionId: params.sessionId,
+      userDid,
+    });
+    // A browser tool call is shown on the session's SSE stream too (session
+    // sinks; the socket tap skips frontend calls, see `fanOut`). An AG-UI
+    // invocation is not: the SSE stream already reports the action as an
+    // `action_call` under the model's call id and closes it when the tool
+    // ends, and a second frame under the invocation id would never be
+    // closed — the client would show a card spinning forever.
+    if (kind === 'browser') this.deps.router.emit(eventName, payload);
     return pending;
   }
 
+  /** The socket `sid` is gone: its invocations end as unknown, never re-sent. */
+  private executorGone(sid: string): void {
+    const ended = this.calls.executorGone(sid);
+    if (ended > 0)
+      this.deps.logger.warn(
+        `[realtime] socket ${sid} went with ${ended} frontend call(s) in flight; reported as unknown outcome, not re-sent`,
+      );
+  }
+
   // ── housekeeping ────────────────────────────────────────────────────────
 
   private drop(ws: WebSocket, code: number, reason: string): void {
@@ -624,8 +793,10 @@ export class RealtimeEndpoint {
       clearTimeout(timer);
       this.handshakeTimers.delete(ws);
     }
+    this.authenticating.delete(ws);
     const attachment = this.hub.get(ws);
     this.hub.remove(ws);
+    if (attachment) this.executorGone(attachment.sid);
     // Only an authenticated socket ever counted for its session; a socket
     // that never passed CONNECT leaves nothing to drain.
     if (attachment?.userDid && !this.hub.hasSession(attachment.sessionId)) {
@@ -633,7 +804,7 @@ export class RealtimeEndpoint {
     }
   }
 
-  /** Make sure an alarm is set for the next heartbeat round. */
+  /** Make sure an alarm is set for the next heartbeat round (authenticated sockets only). */
   private scheduleHeartbeat(): void {
     const at = this.nextPingAt();
     if (at !== null) this.deps.requestAlarm?.(at);

package/src/realtime/session-socket-hub.ts

@@ -27,6 +27,11 @@ export interface SocketAttachment {
   lastPongAt: number;
   /** Last engine.io ping (`2`) sent; unset until the first heartbeat round. */
   lastPingAt?: number;
+  /**
+   * Last client-originated socket.io event (a result, `ping`, `status`, …);
+   * engine.io pongs do not count — a background tab answers those too.
+   */
+  lastActiveAt?: number;
 }
 
 /** The subset of the Workers `WebSocket` the hub needs (tests use fakes). */
@@ -40,6 +45,11 @@ export interface HubSocket {
 export interface SessionSocketHubOptions {
   logger?: { warn: (message: string) => void };
   now?: () => number;
+  /**
+   * A socket the hub dropped on its own — its send threw, or the heartbeat
+   * timed it out. (`remove` by the owner does not call it.)
+   */
+  onDropped?: (attachment: SocketAttachment) => void;
 }
 
 function isRecord(value: unknown): value is Record<string, unknown> {
@@ -74,9 +84,12 @@ export class SessionSocketHub<S extends HubSocket = HubSocket> {
 
   private readonly now: () => number;
 
+  private readonly onDropped: (attachment: SocketAttachment) => void;
+
   constructor(options: SessionSocketHubOptions = {}) {
     this.logger = options.logger ?? { warn: () => undefined };
     this.now = options.now ?? (() => Date.now());
+    this.onDropped = options.onDropped ?? (() => undefined);
   }
 
   /** Register a freshly accepted socket and persist its identity on it. */
@@ -155,6 +168,33 @@ export class SessionSocketHub<S extends HubSocket = HubSocket> {
     return this.forSession(sessionId).length > 0;
   }
 
+  /** A client event arrived on the socket: persist the time (see `executorForSession`). */
+  noteActivity(socket: S): void {
+    if (this.sockets.has(socket))
+      this.update(socket, { lastActiveAt: this.now() });
+  }
+
+  /**
+   * The authenticated socket of `sessionId` a frontend call runs on: the one
+   * with the most recent client event, else the one opened last. The tab the
+   * user acted in last is the likeliest to be in front; the tab that
+   * connected last is often just a background tab that reconnected after a
+   * laptop woke. Ties go to the socket registered later.
+   */
+  executorForSession(
+    sessionId: string,
+  ): { socket: S; attachment: SocketAttachment } | undefined {
+    const recency = (attachment: SocketAttachment): number =>
+      attachment.lastActiveAt ?? attachment.openedAt;
+    let chosen: { socket: S; attachment: SocketAttachment } | undefined;
+    for (const [socket, attachment] of this.sockets) {
+      if (!attachment.userDid || attachment.sessionId !== sessionId) continue;
+      if (!chosen || recency(attachment) >= recency(chosen.attachment))
+        chosen = { socket, attachment };
+    }
+    return chosen;
+  }
+
   /** Fan an event out to every authenticated socket of the session. */
   emitToSession(
     sessionId: string,
@@ -192,6 +232,7 @@ export class SessionSocketHub<S extends HubSocket = HubSocket> {
           // already gone
         }
         closed.push(socket);
+        this.onDropped(meta);
         continue;
       }
       if (this.trySend(socket, ENGINE_PING))
@@ -202,12 +243,14 @@ export class SessionSocketHub<S extends HubSocket = HubSocket> {
 
   /**
    * When the next heartbeat round is due: `pingIntervalMs` after the most
-   * overdue socket's last ping (its open time before the first round), or
-   * null with no sockets attached.
+   * overdue authenticated socket's last ping (its open time before the first
+   * round), or null with none attached. A socket still in its handshake is
+   * on the handshake deadline, not the heartbeat: it never arms an alarm.
    */
   nextPingAt(pingIntervalMs: number): number | null {
     let next: number | null = null;
     for (const meta of this.sockets.values()) {
+      if (!meta.userDid) continue;
       const due = (meta.lastPingAt ?? meta.openedAt) + pingIntervalMs;
       if (next === null || due < next) next = due;
     }
@@ -242,7 +285,9 @@ export class SessionSocketHub<S extends HubSocket = HubSocket> {
       this.logger.warn(
         `socket send failed, dropping socket: ${err instanceof Error ? err.message : String(err)}`,
       );
+      const attachment = this.sockets.get(socket);
       this.remove(socket);
+      if (attachment) this.onDropped(attachment);
       return false;
     }
   }

package/src/secrets/secrets-service.ts

@@ -165,12 +165,18 @@ export class WorkersSecretsService {
     const index = (await this.getIndex(roomId)).filter((entry) =>
       requested.has(entry.name),
     );
-    return this.loadValues(roomId, index);
+    return this.getValuesFor(roomId, index);
   }
 
-  private async loadValues(
+  /**
+   * Decrypted values for index entries the caller already read with
+   * {@link getIndex} — the values of exactly those entries, without reading
+   * room state again. The entries' eventIds decide cache hits, so a secret
+   * rotated after that read is still fetched fresh by the next one.
+   */
+  async getValuesFor(
     roomId: string,
-    index: SecretIndexEntry[],
+    index: ReadonlyArray<Pick<SecretIndexEntry, 'name' | 'eventId'>>,
   ): Promise<Record<string, string>> {
     const result: Record<string, string> = {};
     const now = Date.now();