@ixo/oracle-runtime-workers 0.2.0

package/src/do/fresh-stub.ts

@@ -0,0 +1,32 @@
+/**
+ * A Durable Object stub that never goes stale.
+ *
+ * A stub is a connection to one incarnation of the target object: once that
+ * object restarts (deploy, eviction, memory reset, `restart()`), calls on the
+ * old stub fail with "Network connection lost" until the CALLER is itself
+ * recreated. The user object keeps one gateway handle for its whole life and
+ * hands it to long-lived helpers (owner store, preferences, status card,
+ * ambient matrix adapter), so a single gateway restart silently degraded
+ * every best-effort gateway call for hours.
+ *
+ * `freshStub` returns a proxy with the stub's interface whose every property
+ * access resolves against a NEW stub from `get()` — `namespace.get(id)` is a
+ * local, allocation-only operation, so this costs nothing per call and each
+ * RPC lands on whatever incarnation is current.
+ */
+export function freshStub<T extends object>(get: () => T): T {
+  const handler: ProxyHandler<object> = {
+    get(_target, prop) {
+      // Hand the property back untouched: an RPC property carries its own
+      // target, and touching it in any other way (e.g. `.bind`) is itself
+      // treated as an RPC call by the runtime ("The RPC receiver does not
+      // implement the method \"bind\"").
+      const stub = get();
+      return Reflect.get(stub, prop, stub) as unknown;
+    },
+    has(_target, prop) {
+      return Reflect.has(get(), prop);
+    },
+  };
+  return new Proxy({}, handler) as T;
+}

package/src/do/gateway-retry.ts

@@ -0,0 +1,73 @@
+/**
+ * Retry a gateway RPC across a gateway restart.
+ *
+ * The Matrix gateway object is replaced on every deploy and can be evicted;
+ * an RPC issued in that window fails with a transport error ("Network
+ * connection lost", "Durable Object reset", …) rather than a Matrix error.
+ * Waited sends that must not be replayed blindly (the session marker: its
+ * event id becomes the session id) instead retry the SAME transaction id a
+ * few times over ~10 s, so the homeserver deduplicates a send whose response
+ * was lost and a restart never produces a second marker.
+ */
+
+export const GATEWAY_RETRY_DELAYS_MS: readonly number[] = [
+  // Must outlast a gateway restart or idle recycle: the SDK's stop, alarm
+  // re-arm and boot take a few seconds normally, longer on a cold start.
+  500, 1000, 2000, 4000, 8000, 8000, 8000,
+];
+
+const TRANSIENT_PATTERNS = [
+  /network connection lost/i,
+  /durable object (reset|storage operation exceeded|has been)/i,
+  /object (was|has been) (reset|evicted)/i,
+  /internal error in durable object/i,
+  /not (yet )?started/i,
+  /client is not running/i,
+  /matrix client.*(stopped|not ready|not started)/i,
+  /timed out waiting for the matrix client/i,
+  /failed to fetch/i,
+  /econnreset|econnrefused|socket hang up/i,
+  // `ctx.abort(reason)` surfaces the reason to in-flight RPC callers.
+  /gateway reset/i,
+  /planned recycle/i,
+  /\babort(ed)?\b/i,
+  // matrix-js-sdk's wording while the bot stops or recycles.
+  /shutting down/i,
+  /OutgoingRequestsManager was stopped/i,
+  /null pointer passed to rust/i,
+  // A retry that raced the stopping client's pending list (see startedClient).
+  /known txnId/i,
+];
+
+export function isTransientGatewayError(err: unknown): boolean {
+  const message = err instanceof Error ? err.message : String(err);
+  return TRANSIENT_PATTERNS.some((p) => p.test(message));
+}
+
+export interface RetryGatewayOptions {
+  delaysMs?: readonly number[];
+  isTransient?: (err: unknown) => boolean;
+  onRetry?: (err: unknown, attempt: number, delayMs: number) => void;
+  sleep?: (ms: number) => Promise<void>;
+}
+
+/** Run `fn`, retrying transient gateway failures with the given delays. */
+export async function retryGateway<T>(
+  fn: () => Promise<T>,
+  opts: RetryGatewayOptions = {},
+): Promise<T> {
+  const delays = opts.delaysMs ?? GATEWAY_RETRY_DELAYS_MS;
+  const isTransient = opts.isTransient ?? isTransientGatewayError;
+  const sleep =
+    opts.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
+  for (let attempt = 0; ; attempt += 1) {
+    try {
+      return await fn();
+    } catch (err) {
+      const delay = delays[attempt];
+      if (delay === undefined || !isTransient(err)) throw err;
+      opts.onRetry?.(err, attempt + 1, delay);
+      await sleep(delay);
+    }
+  }
+}

package/src/do/secrets-adapter.ts

@@ -0,0 +1,32 @@
+/**
+ * Adapts `WorkersSecretsService` (per-room JWE secrets over the Matrix
+ * gateway) to the core's `SecretsAdapter` shape — the same mapping the Node
+ * runtime's `ambient-factory` performs over its `SecretsService`:
+ * `getIndex` lists names (plugins never see event ids), `getValues` decrypts
+ * only the requested names. Degrades exactly like Node when the oracle has no
+ * P-256 key seated: the index still lists, values come back empty.
+ */
+import type { SecretsAdapter } from '../core/runtime-context';
+import type { SecretIndex } from '../plugin-api/types';
+import type { WorkersSecretsService } from '../secrets/secrets-service';
+
+export function createSecretsAdapter(
+  service: WorkersSecretsService,
+): SecretsAdapter {
+  return {
+    async getIndex(roomId: string): Promise<SecretIndex> {
+      const entries = await service.getIndex(roomId);
+      const index: Record<string, { key: string }> = {};
+      for (const entry of entries) {
+        index[entry.name] = { key: entry.name };
+      }
+      return index satisfies SecretIndex;
+    },
+    async getValues(
+      roomId: string,
+      keys: string[],
+    ): Promise<Record<string, string>> {
+      return service.getValues(roomId, keys);
+    },
+  };
+}

package/src/do/size-alerts.ts

@@ -0,0 +1,76 @@
+/**
+ * Storage watermark alerts: a Slack webhook message the first time a user's
+ * working-copy SQLite file crosses each whole-GB threshold from
+ * `START_ALERT_GB` up (1 GB, then 2 GB, ... 9 GB is the last crossing below
+ * the cap). One alert per threshold — the caller persists the highest GB
+ * already alerted and passes it back in.
+ *
+ * Exists for two reasons: the DO storage cap is 10 GB per user object, and
+ * resident storage is the runtime's dominant cost ($0.20/GB-month) — see the
+ * cost-planning notes in the package README. Early crossings are the signal
+ * to schedule the R2 page-tier work with runway, not under fire.
+ */
+
+export const START_ALERT_GB = 1;
+const GB = 1024 * 1024 * 1024;
+
+/**
+ * The threshold this file size newly crosses, or null when no alert is due.
+ * `lastAlertedGb` is the highest threshold already alerted (0 = none).
+ * A file that jumped several GB since the last check (e.g. a big legacy
+ * import) alerts once at its CURRENT floor, not once per skipped step.
+ */
+export function crossedGbThreshold(
+  fileBytes: number,
+  lastAlertedGb: number,
+  startGb = START_ALERT_GB,
+): number | null {
+  const floorGb = Math.floor(fileBytes / GB);
+  if (floorGb < startGb || floorGb <= lastAlertedGb) return null;
+  return floorGb;
+}
+
+export function storageAlertText(args: {
+  oracleName: string;
+  oracleDid: string;
+  userDid: string;
+  fileBytes: number;
+  gb: number;
+}): string {
+  const sizeGb = (args.fileBytes / GB).toFixed(2);
+  return (
+    `:warning: *Oracle user storage crossed ${args.gb} GB*\n` +
+    `• Oracle: ${args.oracleName} (\`${args.oracleDid}\`)\n` +
+    `• User: \`${args.userDid}\`\n` +
+    `• Working-copy SQLite file: ${sizeGb} GB of the 10 GB Durable Object cap\n` +
+    `At 10 GB this user's turns and exports start failing — ship the storage tier before then.`
+  );
+}
+
+/**
+ * Fire the webhook. Failures are logged and swallowed — an alert must never
+ * take down the flush path it rides on.
+ */
+export async function postSlackAlert(
+  webhookUrl: string,
+  text: string,
+): Promise<void> {
+  try {
+    const res = await fetch(webhookUrl, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ text }),
+    });
+    if (!res.ok) {
+      console.warn(
+        `[size-alerts] Slack webhook responded ${res.status}: ${await res
+          .text()
+          .catch(() => '')}`,
+      );
+    }
+  } catch (err) {
+    console.warn(
+      `[size-alerts] Slack webhook failed: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+}

package/src/do/sse-stream.ts

@@ -0,0 +1,472 @@
+/**
+ * LangGraph `streamEvents` → Server-Sent Events, as a Web `ReadableStream`.
+ *
+ * Wire-compatible port of the Node runtime's `SseStreamRunner`: the same
+ * event names and payload shapes the Portal / client SDK parse
+ * (`message`, `reasoning`, `tool_call`, `action_call`, `error`, `done`),
+ * the same "instant Thinking…" ack, the same 15 s heartbeat comment.
+ *
+ * On Workers the response body is a stream we write into; the client
+ * disconnecting cancels the stream, which aborts the run.
+ */
+import {
+  classifyLlmError,
+  isOperatorFault,
+  redactOperatorFault,
+} from '../llm/provider-error';
+import type { AIMessageChunk, ToolMessage } from '@langchain/core/messages';
+
+export interface SseTurnRunnerInput {
+  /** `agent.streamEvents(stateInput, config)` async iterable. */
+  events: AsyncIterable<unknown>;
+  sessionId: string;
+  requestId: string;
+  abortController: AbortController;
+  /** Names declared as client-side AG-UI actions (rendered as `action_call`). */
+  agActionNames?: ReadonlySet<string>;
+  onComplete?: (fullText: string) => void | Promise<void>;
+  onError?: (error: unknown) => void;
+  log?: (msg: string) => void;
+  /** BYO provider of this turn, when it ran on the user's own credential. */
+  byoProvider?: string;
+  /**
+   * Receives every `tool_call` / `action_call` / `router_update` frame the
+   * stream writes, for the session's realtime sockets (the SSE consumer
+   * already has them). Payloads carry `sessionId`.
+   */
+  mirror?: (eventName: string, payload: Record<string, unknown>) => void;
+}
+
+const THINKING_PHRASES = [
+  'Thinking...',
+  'Working...',
+  'Analyzing...',
+  'Processing...',
+  'Reasoning...',
+  'Reading...',
+  'Synthesizing...',
+  'Considering...',
+  'Investigating...',
+  'Solving...',
+  'Reviewing...',
+  'Reflecting...',
+];
+
+export function pickThinkingPhrase(): string {
+  return (
+    THINKING_PHRASES[Math.floor(Math.random() * THINKING_PHRASES.length)] ??
+    'Thinking...'
+  );
+}
+
+export function formatSSE(eventType: string, data: unknown): string {
+  return `event: ${eventType}\ndata: ${JSON.stringify(data)}\n\n`;
+}
+
+export const SSE_HEADERS: Record<string, string> = {
+  'Content-Type': 'text/event-stream',
+  'Cache-Control': 'no-cache, no-transform',
+  Connection: 'keep-alive',
+  'X-Accel-Buffering': 'no',
+};
+
+interface ToolCallPayload {
+  sessionId: string;
+  requestId: string;
+  toolName: string;
+  args: Record<string, unknown>;
+  status: 'isRunning' | 'done';
+  output?: string;
+  eventId?: string;
+}
+
+interface ActionCallPayload {
+  sessionId: string;
+  requestId: string;
+  toolName: string;
+  toolCallId: string;
+  args?: Record<string, unknown>;
+  status: 'isRunning' | 'done' | 'error';
+  output?: string;
+  error?: string;
+}
+
+function extractToolArgs(input: unknown): Record<string, unknown> {
+  if (input && typeof input === 'object') {
+    const maybe = input as { input?: unknown };
+    if (maybe.input && typeof maybe.input === 'object')
+      return maybe.input as Record<string, unknown>;
+    return input as Record<string, unknown>;
+  }
+  return {};
+}
+
+function collectBlockStrings(
+  content: unknown,
+  type: string,
+  field: string,
+): string {
+  if (!Array.isArray(content)) return '';
+  return content
+    .map((block) => {
+      if (
+        block &&
+        typeof block === 'object' &&
+        (block as { type?: unknown }).type === type
+      ) {
+        const v = (block as Record<string, unknown>)[field];
+        return typeof v === 'string' ? v : '';
+      }
+      return '';
+    })
+    .join('');
+}
+
+function safeParseToolContent(
+  content: unknown,
+): { success?: boolean; error?: unknown } | null {
+  if (typeof content !== 'string') return null;
+  try {
+    const parsed = JSON.parse(content);
+    return parsed && typeof parsed === 'object'
+      ? (parsed as { success?: boolean; error?: unknown })
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+function toText(content: unknown): string {
+  if (typeof content === 'string') return content;
+  if (Array.isArray(content))
+    return collectBlockStrings(content, 'text', 'text');
+  return content == null ? '' : JSON.stringify(content);
+}
+
+/**
+ * Build the SSE body for one turn. The returned stream starts emitting the
+ * Thinking… ack immediately; `events` is consumed lazily once the consumer
+ * reads. Errors inside the run become an `error` frame followed by `done`.
+ */
+/** SSE frame name → socket event name (Node's `serverEvents` spelling). */
+const MIRRORED_EVENTS: Record<string, string | undefined> = {
+  tool_call: 'tool_call',
+  action_call: 'action_call',
+  'router.update': 'router_update',
+};
+
+export function createSseTurnStream(
+  input: SseTurnRunnerInput,
+): ReadableStream<Uint8Array> {
+  const encoder = new TextEncoder();
+  const { sessionId, requestId, abortController } = input;
+  const toolCallMap = new Map<string, ToolCallPayload>();
+  const actionCallMap = new Map<string, ActionCallPayload>();
+  let closed = false;
+  let heartbeat: ReturnType<typeof setInterval> | undefined;
+
+  return new ReadableStream<Uint8Array>({
+    start: (controller) => {
+      const write = (
+        eventName: string,
+        payload: unknown,
+        opts: { force?: boolean } = {},
+      ) => {
+        if (closed || (abortController.signal.aborted && !opts.force)) return;
+        try {
+          controller.enqueue(encoder.encode(formatSSE(eventName, payload)));
+        } catch {
+          closed = true;
+        }
+        // Node's WebSocket gateway relays these to the session's sockets as
+        // well; message chunks and `done` stay SSE-only, as there.
+        const mirrored = MIRRORED_EVENTS[eventName];
+        if (mirrored && input.mirror && payload && typeof payload === 'object')
+          input.mirror(mirrored, payload as Record<string, unknown>);
+      };
+      const finish = () => {
+        if (closed) return;
+        closed = true;
+        if (heartbeat) clearInterval(heartbeat);
+        try {
+          controller.close();
+        } catch {
+          /* already closed */
+        }
+      };
+
+      write('router.update', {
+        step: pickThinkingPhrase(),
+        sessionId,
+        requestId,
+      });
+      heartbeat = setInterval(() => {
+        if (closed) return;
+        try {
+          controller.enqueue(encoder.encode(': heartbeat\n\n'));
+        } catch {
+          closed = true;
+        }
+      }, 15_000);
+
+      const flushOrphans = () => {
+        for (const [runId, evt] of actionCallMap) {
+          write('action_call', {
+            ...evt,
+            status: 'error',
+            error: 'Action did not complete',
+            toolCallId: runId,
+          });
+        }
+        actionCallMap.clear();
+        for (const [runId, evt] of toolCallMap) {
+          write('tool_call', {
+            ...evt,
+            status: 'done',
+            output: '⏱️ Tool did not complete',
+            eventId: runId,
+          });
+        }
+        toolCallMap.clear();
+      };
+
+      const run = async () => {
+        let fullContent = '';
+        try {
+          for await (const raw of input.events) {
+            if (abortController.signal.aborted) break;
+            const evt = raw as {
+              event: string;
+              run_id: string;
+              name?: string;
+              data: unknown;
+            };
+            if (evt.event === 'on_tool_start') {
+              const toolName = evt.name ?? 'tool';
+              const args = extractToolArgs(
+                (evt.data as { input?: unknown })?.input,
+              );
+              if (input.agActionNames?.has(toolName)) {
+                const payload: ActionCallPayload = {
+                  requestId,
+                  sessionId,
+                  toolCallId: evt.run_id,
+                  toolName,
+                  args,
+                  status: 'isRunning',
+                };
+                actionCallMap.set(evt.run_id, payload);
+                write('action_call', payload);
+              } else {
+                const payload: ToolCallPayload = {
+                  requestId,
+                  sessionId,
+                  toolName,
+                  args: { ...args, toolName },
+                  status: 'isRunning',
+                  eventId: evt.run_id,
+                };
+                toolCallMap.set(evt.run_id, payload);
+                write('tool_call', payload);
+              }
+              continue;
+            }
+            if (evt.event === 'on_tool_end') {
+              const output = (evt.data as { output?: ToolMessage })?.output;
+              const text = toText(output?.content);
+              const action = actionCallMap.get(evt.run_id);
+              if (action) {
+                const parsed = safeParseToolContent(output?.content);
+                const failed =
+                  parsed?.success === false || Boolean(parsed?.error);
+                write('action_call', {
+                  ...action,
+                  output: text,
+                  status: failed ? 'error' : 'done',
+                  ...(failed && {
+                    error:
+                      typeof parsed?.error === 'string'
+                        ? parsed.error
+                        : 'Action failed',
+                  }),
+                });
+                actionCallMap.delete(evt.run_id);
+                continue;
+              }
+              const tool = toolCallMap.get(evt.run_id);
+              if (tool) {
+                write('tool_call', {
+                  ...tool,
+                  status: 'done',
+                  output: text,
+                  eventId: evt.run_id,
+                  args: {
+                    ...tool.args,
+                    toolName: output?.name ?? tool.toolName,
+                  },
+                });
+                toolCallMap.delete(evt.run_id);
+              }
+              continue;
+            }
+            if (evt.event === 'on_tool_error') {
+              // A tool that threw (retries exhausted, upstream down, MCP
+              // timeout) never emits `on_tool_end`; without this branch the
+              // client only learns "did not complete" at stream end.
+              const raw = (evt.data as { error?: unknown })?.error;
+              const message =
+                raw instanceof Error
+                  ? raw.message
+                  : String(raw ?? 'Tool failed');
+              const action = actionCallMap.get(evt.run_id);
+              if (action) {
+                write('action_call', {
+                  ...action,
+                  output: message,
+                  status: 'error',
+                  error: message,
+                });
+                actionCallMap.delete(evt.run_id);
+                continue;
+              }
+              const tool = toolCallMap.get(evt.run_id);
+              if (tool) {
+                write('tool_call', {
+                  ...tool,
+                  status: 'done',
+                  output: `⚠️ ${message}`,
+                  eventId: evt.run_id,
+                });
+                toolCallMap.delete(evt.run_id);
+              }
+              continue;
+            }
+            if (evt.event === 'on_chat_model_stream') {
+              const chunk = (evt.data as { chunk?: AIMessageChunk })?.chunk;
+              if (!chunk) continue;
+              const raw = chunk.additional_kwargs?.__raw_response as
+                | {
+                    choices?: Array<{
+                      delta?: {
+                        reasoning?: string;
+                        reasoning_content?: string;
+                        reasoning_details?: unknown[];
+                      };
+                    }>;
+                  }
+                | undefined;
+              const delta = raw?.choices?.[0]?.delta;
+              const deltaReasoning =
+                delta?.reasoning ?? delta?.reasoning_content;
+              const reasoning = deltaReasoning?.trim()
+                ? deltaReasoning
+                : collectBlockStrings(chunk.content, 'reasoning', 'reasoning');
+              if (reasoning && reasoning.trim()) {
+                const details = Array.isArray(delta?.reasoning_details)
+                  ? delta.reasoning_details
+                      .filter(
+                        (d): d is { type: string; text: string } =>
+                          !!d &&
+                          typeof d === 'object' &&
+                          typeof (d as { type?: unknown }).type === 'string' &&
+                          typeof (d as { text?: unknown }).text === 'string' &&
+                          (d as { text: string }).text.trim().length > 0,
+                      )
+                      .map((d) => ({
+                        type: d.type,
+                        text: d.text,
+                        format: 'unknown',
+                        index: 0,
+                      }))
+                  : undefined;
+                write('reasoning', {
+                  sessionId,
+                  requestId,
+                  reasoning,
+                  ...(details && { reasoningDetails: details }),
+                  isComplete: false,
+                  timestamp: new Date().toISOString(),
+                });
+              }
+              const text =
+                typeof chunk.content === 'string'
+                  ? chunk.content
+                  : collectBlockStrings(chunk.content, 'text', 'text');
+              if (text) {
+                fullContent += text;
+                write('message', {
+                  content: text,
+                  timestamp: new Date().toISOString(),
+                });
+              }
+            }
+          }
+          if (!abortController.signal.aborted) {
+            flushOrphans();
+            write('reasoning', {
+              sessionId,
+              requestId,
+              reasoning: '',
+              isComplete: true,
+              timestamp: new Date().toISOString(),
+            });
+            write('done', {});
+            await input.onComplete?.(fullContent);
+          } else {
+            // Aborted by the client (POST /messages/abort): close the stream
+            // cleanly so the UI leaves its "thinking" state.
+            write('done', {}, { force: true });
+          }
+        } catch (error) {
+          const aborted =
+            error instanceof Error &&
+            (error.name === 'AbortError' || /abort/i.test(error.message));
+          if (!aborted) {
+            input.onError?.(error);
+            flushOrphans();
+            write('reasoning', {
+              sessionId,
+              requestId,
+              reasoning: '',
+              isComplete: true,
+              timestamp: new Date().toISOString(),
+            });
+            // Same wire shape as the Node runtime's `sendSSEError`: the
+            // classification is redacted here — the one place an LLM failure
+            // becomes bytes on a client's wire.
+            const classified = classifyLlmError(error, {
+              byoProvider: input.byoProvider,
+            });
+            if (isOperatorFault(classified))
+              input.log?.(
+                `OPERATOR FAULT (platform ${classified.kind}) on turn ${requestId}: ${classified.detail}`,
+              );
+            const safe = redactOperatorFault(classified);
+            write('error', {
+              error: safe.message,
+              kind: safe.kind,
+              source: safe.source,
+              ...(safe.provider && { provider: safe.provider }),
+              ...(safe.providerLabel && { providerLabel: safe.providerLabel }),
+              ...(safe.status !== undefined && { status: safe.status }),
+              retryable: safe.retryable,
+              detail: safe.detail,
+              sessionId,
+              requestId,
+              timestamp: new Date().toISOString(),
+            });
+          }
+          write('done', {}, { force: true });
+        } finally {
+          finish();
+        }
+      };
+      void run();
+    },
+    cancel: () => {
+      closed = true;
+      if (heartbeat) clearInterval(heartbeat);
+      abortController.abort();
+    },
+  });
+}