@kuralle-syrinx/cf-agents 3.1.0 → 4.1.0

package/README.md

@@ -61,15 +61,43 @@ export class SupportVoiceAgent extends withVoice<Env, typeof Agent<Env>>(Agent<E
 }) {}
 ```
 
+## The Responder-Thinker primitive
+
+`withVoice` packages Syrinx's bi-model **Responder-Thinker** shape turnkey (RFC
+`docs/rfc-bimodel-delegate-seam.md`): wire a realtime front + a `Reasoner` and the delegate seam
+comes with —
+
+- **Structured result envelope (G1, default).** The reasoner's answer reaches the front model as
+  `{ response_text, require_repeat_verbatim: true, render? }` so it repeats facts faithfully
+  instead of paraphrasing. Configure per pipeline: `toolResultFormat: "envelope" | "string"`,
+  `renderDirective: "translate_faithfully"`.
+- **Delegate observability (G2).** `onDelegateQuery` / `onDelegateResult` hooks fire around every
+  reasoner run with the query, answer, `durationMs`, and `grounded` — log or persist the Q&A pair
+  without wrapping the `Reasoner`.
+- **Typed "thinking" cues (G3).** Clients automatically receive `tool_call_started` /
+  `tool_call_delayed` (after `delayCueAfterMs`) / `tool_call_complete` / `tool_call_failed` wire
+  messages around the reasoner-latency window — key earcons/indicators on these instead of
+  inventing an app message (`@kuralle-syrinx/browser-client` parses them).
+- **Durable session + resume (G4, default on).** The conversation persists to the Agent's
+  DO-SQLite and survives eviction/hibernation: cascaded pipelines re-seed the `ReasoningBridge`;
+  realtime pipelines feed the durable transcript to delegate turns and expose `ctx.resume` to the
+  `front()` factory — `resumeHistory: ctx.resume.history` on replay providers (OpenAI),
+  `sessionResumptionHandle: ctx.resume.providerHandle` on native-resume providers (Gemini; never
+  replay on top of a handle).
+
 ## Options
 
 | Option | Description |
 | --- | --- |
 | `transport` | `"edge"` (default — Syrinx browser/edge protocol over `/ws`) or `"twilio"` (Twilio Media Streams, μ-law 8 kHz, for a PSTN leg). One transport per Agent class. |
-| `pipeline` | `{ kind: "realtime", front, delegateToolName? }` or `{ kind: "cascaded", stt, tts, vad?, eos?, endpointingOwner?, sttForceFinalizeTimeoutMs? }`. |
-| `reasoner` | `(env, { sessionId }) => Reasoner`. Defaults to `fromKuralleRuntime(this.runtime)` when the Agent exposes a kuralle `runtime`. Required for cascaded agents without one. |
+| `pipeline` | `{ kind: "realtime", front, delegateToolName?, toolResultFormat?, renderDirective? }` or `{ kind: "cascaded", stt, tts, vad?, eos?, endpointingOwner?, sttForceFinalizeTimeoutMs? }`. |
+| `reasoner` | `(env, ctx) => Reasoner` (ctx: `{ sessionId, resume? }`). Defaults to `fromKuralleRuntime(this.runtime)` when the Agent exposes a kuralle `runtime`. Required for cascaded agents without one. |
 | `recorder` | `(env, { sessionId }) => EdgeRecorder \| undefined` — optional per-call recorder (e.g. the R2 recorder at `@kuralle-syrinx/cf-agents/r2-recorder`). Edge transport. |
-| `onToolCallStart` | `(ctx: { toolName, args, sessionId, connection }) => void \| Promise<void>` — fired the instant the front model invokes the delegate tool, **before** the reasoner runs. The seam for a deterministic latency-masking preamble / "thinking" earcon: `ctx.connection.send(...)` to trigger a cached client-side cue. A throwing callback never affects the call. |
+| `onToolCallStart` | `(ctx: { toolName, args, sessionId, connection }) => void \| Promise<void>` — fired the instant the front model invokes the delegate tool, **before** the reasoner runs — for app-specific cues beyond the standard `tool_call_*` wire messages. A throwing callback never affects the call. |
+| `onDelegateQuery` / `onDelegateResult` | G2 observability hooks around the reasoner run. `onDelegateResult` is self-contained (`{ query, answer, durationMs, grounded, toolId?, toolName?, turnId, sessionId, connection }`) — the one hook for logging/persisting grounded Q&A pairs. Throwing never affects the call. |
+| `durableHistory` | G4 durable session state over the Agent's SQLite (default `true`). Set `false` for ephemeral pre-G4 behavior. |
+| `delayCueAfterMs` | G3: ms before a pending tool call fires the `tool_call_delayed` ("still working") cue. 0 disables. Default 2000. |
+| `backgroundAudio` | `{ ambient?, thinking?, duckWhileSpeaking? }` — looped ambient bed + thinking loop (raw mono PCM16 sources), mixed (ducked) under assistant speech; on the `"twilio"` transport the bed also fills between-turn gaps as comfort noise. Thinking follows the G3 cues. |
 | `inputSampleRateHz` / `outputSampleRateHz` | Edge audio rates (default 16000). |
 | `resumeWindowMs` | How long a dropped connection can resume its session. |
 | `sessionId` | `(request, agentName) => string`. Defaults to the `?sessionId=` query param (so a reconnecting client can resume), else a per-connection random id. (Not the Agent name — concurrent connections to one instance must not share a session.) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/cf-agents",
-  "version": "3.1.0",
+  "version": "4.1.0",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -12,13 +12,13 @@
     "./r2-recorder": "./src/r2-recorder.ts"
   },
   "dependencies": {
-    "@kuralle-syrinx/core": "3.1.0",
-    "@kuralle-syrinx/realtime": "3.1.0",
-    "@kuralle-syrinx/aisdk": "3.1.0",
-    "@kuralle-syrinx/kuralle": "3.1.0",
-    "@kuralle-syrinx/server-websocket": "3.1.0",
-    "@kuralle-syrinx/ws": "3.1.0",
-    "@kuralle-syrinx/recorder": "3.1.0"
+    "@kuralle-syrinx/core": "4.1.0",
+    "@kuralle-syrinx/aisdk": "4.1.0",
+    "@kuralle-syrinx/server-websocket": "4.1.0",
+    "@kuralle-syrinx/ws": "4.1.0",
+    "@kuralle-syrinx/recorder": "4.1.0",
+    "@kuralle-syrinx/realtime": "4.1.0",
+    "@kuralle-syrinx/kuralle": "4.1.0"
   },
   "peerDependencies": {
     "agents": ">=0.14.0 <1.0.0"

package/src/build-session.ts

@@ -3,6 +3,8 @@
 import {
   VoiceAgentSession,
   type Reasoner,
+  type ReasonerMessage,
+  type ReasonerSessionStore,
   type VoicePlugin,
   type PluginConfig,
 } from "@kuralle-syrinx/core";
@@ -12,6 +14,28 @@ import { ReasoningBridge } from "@kuralle-syrinx/aisdk";
 /** Per-session context handed to every pipeline factory. */
 export interface VoicePipelineContext {
   readonly sessionId: string;
+  /**
+   * G4 resume state for the session, present when durable history is on. The
+   * `front()` factory wires it into the adapter: `resumeHistory: ctx.resume.history`
+   * on replay providers (OpenAI), `sessionResumptionHandle: ctx.resume.providerHandle`
+   * on native-resume providers (Gemini — do NOT also replay, R6).
+   */
+  readonly resume?: {
+    /** Live view of the durable transcript (call again on reconnect for the current state). */
+    readonly history: () => readonly { readonly role: "user" | "assistant"; readonly content: string }[];
+    /** Latest provider-native resume handle, when one was issued. */
+    readonly providerHandle?: string;
+  };
+}
+
+/** Host wiring (withVoice) threaded into the assembled session. */
+export interface VoiceSessionWiring {
+  /** G4: durable store for the cascaded ReasoningBridge's conversation history. */
+  readonly reasonerSessionStore?: ReasonerSessionStore;
+  /** G4: prior-context provider for realtime delegate turns (live view of durable history). */
+  readonly contextProvider?: () => readonly ReasonerMessage[];
+  /** G3: ms before a pending tool call fires its "delayed" (still-working) cue. */
+  readonly delayCueAfterMs?: number;
 }
 
 /**
@@ -26,6 +50,14 @@ export interface RealtimePipeline<Env> {
   readonly front: (env: Env, ctx: VoicePipelineContext) => RealtimeAdapter;
   /** Name of the front-model tool routed to the reasoner. @default "consult_knowledge" */
   readonly delegateToolName?: string;
+  /**
+   * How the delegate answer reaches the front model (G1): `"envelope"` (default) wraps
+   * it as a `DelegateResultEnvelope` (`response_text` + `require_repeat_verbatim`);
+   * `"string"` injects the raw answer.
+   */
+  readonly toolResultFormat?: "envelope" | "string";
+  /** Optional `render` directive included in the envelope, e.g. `"translate_faithfully"`. */
+  readonly renderDirective?: string;
 }
 
 /** A cascaded-stage plugin plus its `VoiceAgentSession` plugin config. */
@@ -57,6 +89,13 @@ export interface CascadedPipeline<Env> {
    * (engine default 7000). Set it when a provider-endpointed cascade tunes this (e.g. Deepgram at 3500).
    */
   readonly sttForceFinalizeTimeoutMs?: number;
+  /**
+   * Speculative generation: start the reasoner on an eager end-of-turn signal
+   * (`eos.interim` — Deepgram Flux `eager_eot_threshold`, smart-turn interim) and
+   * commit/discard when the endpoint confirms. Trades extra LLM calls for
+   * parallelizing LLM TTFT with endpoint confirmation. @default false
+   */
+  readonly speculative?: boolean;
 }
 
 export type VoicePipeline<Env> = RealtimePipeline<Env> | CascadedPipeline<Env>;
@@ -71,13 +110,19 @@ export function buildVoiceSession<Env>(
   env: Env,
   reasoner: Reasoner | undefined,
   ctx: VoicePipelineContext,
+  wiring: VoiceSessionWiring = {},
 ): VoiceAgentSession {
   if (pipeline.kind === "realtime") {
     const front = pipeline.front(env, ctx);
-    const bridge = new RealtimeBridge(front, reasoner, pipeline.delegateToolName);
+    const bridge = new RealtimeBridge(front, reasoner, pipeline.delegateToolName, {
+      ...(pipeline.toolResultFormat !== undefined ? { toolResultFormat: pipeline.toolResultFormat } : {}),
+      ...(pipeline.renderDirective !== undefined ? { renderDirective: pipeline.renderDirective } : {}),
+      ...(wiring.contextProvider ? { contextProvider: wiring.contextProvider } : {}),
+    });
     const session = new VoiceAgentSession({
       plugins: { realtime: {} },
       endpointingOwner: "timer",
+      ...(wiring.delayCueAfterMs !== undefined ? { delayCueAfterMs: wiring.delayCueAfterMs } : {}),
     });
     session.registerPlugin("realtime", bridge);
     return session;
@@ -116,9 +161,18 @@ export function buildVoiceSession<Env>(
     ...(pipeline.sttForceFinalizeTimeoutMs !== undefined
       ? { sttForceFinalizeTimeoutMs: pipeline.sttForceFinalizeTimeoutMs }
       : {}),
+    ...(wiring.delayCueAfterMs !== undefined ? { delayCueAfterMs: wiring.delayCueAfterMs } : {}),
   });
   session.registerPlugin("stt", stt.plugin);
-  session.registerPlugin("bridge", new ReasoningBridge(reasoner));
+  session.registerPlugin(
+    "bridge",
+    new ReasoningBridge(reasoner, {
+      ...(wiring.reasonerSessionStore
+        ? { sessionStore: wiring.reasonerSessionStore, sessionId: ctx.sessionId }
+        : {}),
+      ...(pipeline.speculative ? { speculative: true } : {}),
+    }),
+  );
   session.registerPlugin("tts", tts.plugin);
   if (vad) session.registerPlugin("vad", vad.plugin);
   if (eos) session.registerPlugin("eos", eos.plugin);

package/src/durable-history.ts

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: MIT
+//
+// G4 (RFC bimodel-delegate-seam) — durable reasoner session state over the
+// Agent's DO-SQLite. Survives Durable Object eviction/hibernation, replacing
+// consumer-side module-global memory stores (the SLIIT "F8" finding).
+
+import type { ReasonerMessage, ReasonerSessionStore } from "@kuralle-syrinx/core";
+
+/** The agents-SDK `Agent.sql` tagged-template surface (synchronous in the DO). */
+export type SqlTag = <T = Record<string, unknown>>(
+  strings: TemplateStringsArray,
+  ...values: (string | number | boolean | null)[]
+) => T[];
+
+interface HistoryRow {
+  role: string;
+  content: string;
+  tool_call_id: string | null;
+}
+
+interface HandleRow {
+  handle: string;
+}
+
+/**
+ * `ReasonerSessionStore` backed by the Agent's SQLite. Also persists the latest
+ * provider-native resume handle (Gemini `sessionResumption`) per session.
+ */
+export class SqliteReasonerSessionStore implements ReasonerSessionStore {
+  constructor(private readonly sql: SqlTag) {
+    this.sql`CREATE TABLE IF NOT EXISTS syrinx_reasoner_history (
+      session_id TEXT NOT NULL,
+      seq INTEGER NOT NULL,
+      role TEXT NOT NULL,
+      content TEXT NOT NULL,
+      tool_call_id TEXT,
+      PRIMARY KEY (session_id, seq)
+    )`;
+    this.sql`CREATE TABLE IF NOT EXISTS syrinx_resume_handle (
+      session_id TEXT PRIMARY KEY,
+      handle TEXT NOT NULL
+    )`;
+  }
+
+  load(sessionId: string): readonly ReasonerMessage[] {
+    const rows = this.sql<HistoryRow>`
+      SELECT role, content, tool_call_id FROM syrinx_reasoner_history
+      WHERE session_id = ${sessionId} ORDER BY seq ASC`;
+    return rows.map((row) => ({
+      role: row.role as ReasonerMessage["role"],
+      content: row.content,
+      ...(row.tool_call_id ? { toolCallId: row.tool_call_id } : {}),
+    }));
+  }
+
+  save(sessionId: string, messages: readonly ReasonerMessage[]): void {
+    this.sql`DELETE FROM syrinx_reasoner_history WHERE session_id = ${sessionId}`;
+    messages.forEach((message, seq) => {
+      this.sql`INSERT INTO syrinx_reasoner_history (session_id, seq, role, content, tool_call_id)
+        VALUES (${sessionId}, ${seq}, ${message.role}, ${message.content}, ${message.toolCallId ?? null})`;
+    });
+  }
+
+  clear(sessionId: string): void {
+    this.sql`DELETE FROM syrinx_reasoner_history WHERE session_id = ${sessionId}`;
+    this.sql`DELETE FROM syrinx_resume_handle WHERE session_id = ${sessionId}`;
+  }
+
+  loadResumeHandle(sessionId: string): string | undefined {
+    const rows = this.sql<HandleRow>`
+      SELECT handle FROM syrinx_resume_handle WHERE session_id = ${sessionId}`;
+    return rows[0]?.handle;
+  }
+
+  saveResumeHandle(sessionId: string, handle: string): void {
+    this.sql`DELETE FROM syrinx_resume_handle WHERE session_id = ${sessionId}`;
+    this.sql`INSERT INTO syrinx_resume_handle (session_id, handle) VALUES (${sessionId}, ${handle})`;
+  }
+}

package/src/index.ts

@@ -8,6 +8,8 @@ export {
   type WithVoiceOptions,
   type WithVoiceMembers,
   type ToolCallStartContext,
+  type DelegateQueryContext,
+  type DelegateResultContext,
 } from "./with-voice.js";
 export type {
   VoicePipeline,
@@ -15,6 +17,8 @@ export type {
   CascadedPipeline,
   CascadedStage,
   VoicePipelineContext,
+  VoiceSessionWiring,
 } from "./build-session.js";
+export { SqliteReasonerSessionStore, type SqlTag } from "./durable-history.js";
 export { connectionManagedSocket } from "./connection-socket.js";
 export type { VoiceConnection, ConnectionSocketController } from "./connection-socket.js";

package/src/r2-recorder.test.ts

@@ -8,15 +8,54 @@ interface PutCall {
   body: Uint8Array | string;
 }
 
+interface MpuRec {
+  key: string;
+  parts: { partNumber: number; size: number }[];
+  first?: Uint8Array; // retained bytes of part 1 (the header-bearing part)
+  completed: boolean;
+}
+
 function fakeBucket() {
   const puts: PutCall[] = [];
+  const mpus = new Map<string, MpuRec>();
+  let seq = 0;
+
+  function handle(uploadId: string) {
+    const rec = mpus.get(uploadId)!;
+    return {
+      key: rec.key,
+      uploadId,
+      async uploadPart(partNumber: number, value: Uint8Array) {
+        rec.parts.push({ partNumber, size: value.byteLength });
+        if (partNumber === 1) rec.first = value.slice();
+        return { partNumber, etag: `etag-${partNumber}` };
+      },
+      async complete(_parts: unknown) {
+        rec.completed = true;
+        return { key: rec.key } as unknown;
+      },
+      async abort() {
+        /* no-op */
+      },
+    };
+  }
+
   const bucket = {
     async put(key: string, body: Uint8Array | string) {
-      puts.push({ key, body });
+      puts.push({ key, body: typeof body === "string" ? body : body.slice() });
       return {} as unknown;
     },
+    async createMultipartUpload(key: string) {
+      const uploadId = `u${++seq}`;
+      mpus.set(uploadId, { key, parts: [], completed: false });
+      return handle(uploadId);
+    },
+    resumeMultipartUpload(_key: string, uploadId: string) {
+      return handle(uploadId);
+    },
   } as unknown as R2Bucket;
-  return { bucket, puts };
+
+  return { bucket, puts, mpus };
 }
 
 function asString(body: Uint8Array | string): string {
@@ -51,9 +90,10 @@ describe("R2EdgeRecorder", () => {
     expect(wavChannels(puts.find((p) => p.key.endsWith("user.wav"))!.body)).toBe(1);
 
     const manifest = JSON.parse(asString(puts.find((p) => p.key.endsWith("manifest.json"))!.body)) as {
-      conversation: { channels: number };
+      conversation: { channels: number; omitted: boolean };
     };
     expect(manifest.conversation.channels).toBe(2);
+    expect(manifest.conversation.omitted).toBe(false);
   });
 
   it("time-aligns the assistant after the user instead of stacking at 0", async () => {
@@ -97,4 +137,49 @@ describe("R2EdgeRecorder", () => {
     expect(manifest.user.byteLength).toBe(800);
     expect(manifest.user.truncated).toBe(true);
   });
+
+  it("streams a long, mostly-silent call to R2 in bounded parts instead of one giant buffer", async () => {
+    const { bucket, puts, mpus } = fakeBucket();
+    let now = 0;
+    const rec = new R2EdgeRecorder({ bucket, sessionId: "long", startedAtMs: 0, now: () => now });
+
+    // A 20-minute call: a blip of audio, a 20-minute wall-clock gap, then another blip.
+    // The old recorder would gap-fill the full ~38 MB wall-clock length in one allocation
+    // at finalize (OOMing the 128 MB DO). The stream must instead flush it as R2 parts.
+    now = 0;
+    rec.onUserAudio("c", new Uint8Array(320), 16000);
+    now = 20 * 60 * 1000; // +20 min
+    rec.onUserAudio("c", new Uint8Array(320), 16000);
+    await rec.finalize({ sessionId: "long", closedAtMs: now + 100 });
+
+    const expectedDataBytes = 20 * 60 * 16000 * 2 + 320; // wall-clock timeline + the final blip
+
+    // (1) The user stem streamed out incrementally: multiple parts, not one buffer.
+    const userMpu = [...mpus.values()].find((m) => m.key === "recordings/long/0/user.wav")!;
+    expect(userMpu).toBeDefined();
+    expect(userMpu.parts.length).toBeGreaterThan(1);
+    expect(userMpu.completed).toBe(true);
+    // Every part except the last carries at least R2's 5 MiB minimum (part 1 also holds the header).
+    const byNumber = [...userMpu.parts].sort((a, b) => a.partNumber - b.partNumber);
+    for (const p of byNumber.slice(0, -1)) expect(p.size).toBeGreaterThanOrEqual(5 * 1024 * 1024);
+
+    // (2) The stem decodes to the right duration: part 1's WAV header declares the full
+    // data length, and the parts sum to header(44) + that length.
+    const header = new DataView(userMpu.first!.buffer, userMpu.first!.byteOffset, userMpu.first!.byteLength);
+    expect(header.getUint32(40, true)).toBe(expectedDataBytes); // WAV data-chunk size field
+    const uploaded = userMpu.parts.reduce((n, p) => n + p.size, 0);
+    expect(uploaded).toBe(44 + expectedDataBytes);
+
+    // (3) finalize still writes the manifest; the stem length/duration match, and the
+    // stereo mix is flagged omitted (best-effort) once a stem has streamed out.
+    const manifest = JSON.parse(asString(puts.find((p) => p.key.endsWith("manifest.json"))!.body)) as {
+      user: { byteLength: number; durationMs: number; truncated: boolean };
+      conversation: { omitted: boolean };
+    };
+    expect(manifest.user.byteLength).toBe(expectedDataBytes);
+    expect(manifest.user.durationMs).toBe(Math.round((expectedDataBytes / 2 / 16000) * 1000));
+    expect(manifest.user.truncated).toBe(false);
+    expect(manifest.conversation.omitted).toBe(true);
+    expect(puts.some((p) => p.key.endsWith("conversation.wav"))).toBe(false);
+  });
 });