@kuralle-syrinx/cf-agents 3.0.0 → 3.1.0

package/README.md

@@ -65,9 +65,11 @@ export class SupportVoiceAgent extends withVoice<Env, typeof Agent<Env>>(Agent<E
 
 | Option | Description |
 | --- | --- |
-| `pipeline` | `{ kind: "realtime", front, delegateToolName? }` or `{ kind: "cascaded", stt, tts, vad?, eos?, endpointingOwner? }`. |
+| `transport` | `"edge"` (default — Syrinx browser/edge protocol over `/ws`) or `"twilio"` (Twilio Media Streams, μ-law 8 kHz, for a PSTN leg). One transport per Agent class. |
+| `pipeline` | `{ kind: "realtime", front, delegateToolName? }` or `{ kind: "cascaded", stt, tts, vad?, eos?, endpointingOwner?, sttForceFinalizeTimeoutMs? }`. |
 | `reasoner` | `(env, { sessionId }) => Reasoner`. Defaults to `fromKuralleRuntime(this.runtime)` when the Agent exposes a kuralle `runtime`. Required for cascaded agents without one. |
-| `recorder` | `(env, { sessionId }) => EdgeRecorder \| undefined` — optional per-call recorder (e.g. R2). |
+| `recorder` | `(env, { sessionId }) => EdgeRecorder \| undefined` — optional per-call recorder (e.g. the R2 recorder at `@kuralle-syrinx/cf-agents/r2-recorder`). Edge transport. |
+| `onToolCallStart` | `(ctx: { toolName, args, sessionId, connection }) => void \| Promise<void>` — fired the instant the front model invokes the delegate tool, **before** the reasoner runs. The seam for a deterministic latency-masking preamble / "thinking" earcon: `ctx.connection.send(...)` to trigger a cached client-side cue. A throwing callback never affects the call. |
 | `inputSampleRateHz` / `outputSampleRateHz` | Edge audio rates (default 16000). |
 | `resumeWindowMs` | How long a dropped connection can resume its session. |
 | `sessionId` | `(request, agentName) => string`. Defaults to the `?sessionId=` query param (so a reconnecting client can resume), else a per-connection random id. (Not the Agent name — concurrent connections to one instance must not share a session.) |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kuralle-syrinx/cf-agents",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "private": false,
   "type": "module",
   "license": "MIT",
@@ -12,13 +12,13 @@
     "./r2-recorder": "./src/r2-recorder.ts"
   },
   "dependencies": {
-    "@kuralle-syrinx/core": "3.0.0",
-    "@kuralle-syrinx/realtime": "3.0.0",
-    "@kuralle-syrinx/aisdk": "3.0.0",
-    "@kuralle-syrinx/server-websocket": "3.0.0",
-    "@kuralle-syrinx/ws": "3.0.0",
-    "@kuralle-syrinx/recorder": "3.0.0",
-    "@kuralle-syrinx/kuralle": "3.0.0"
+    "@kuralle-syrinx/core": "3.1.0",
+    "@kuralle-syrinx/realtime": "3.1.0",
+    "@kuralle-syrinx/aisdk": "3.1.0",
+    "@kuralle-syrinx/kuralle": "3.1.0",
+    "@kuralle-syrinx/server-websocket": "3.1.0",
+    "@kuralle-syrinx/ws": "3.1.0",
+    "@kuralle-syrinx/recorder": "3.1.0"
   },
   "peerDependencies": {
     "agents": ">=0.14.0 <1.0.0"

package/src/index.ts

@@ -3,7 +3,12 @@
 // @kuralle-syrinx/cf-agents — add a Syrinx voice pipeline (realtime or cascaded) to
 // a Cloudflare `agents` SDK Agent via the `withVoice(Agent, options)` mixin.
 
-export { withVoice, type WithVoiceOptions, type WithVoiceMembers } from "./with-voice.js";
+export {
+  withVoice,
+  type WithVoiceOptions,
+  type WithVoiceMembers,
+  type ToolCallStartContext,
+} from "./with-voice.js";
 export type {
   VoicePipeline,
   RealtimePipeline,

package/src/with-voice.test.ts

@@ -8,9 +8,47 @@
 import { describe, it, expect, vi } from "vitest";
 import type { PipelineBus, Reasoner, UserAudioReceivedPacket, VoicePlugin } from "@kuralle-syrinx/core";
 import { encodePcm16ToMuLaw } from "@kuralle-syrinx/core/audio";
-import { withVoice } from "./with-voice.js";
+import type { RealtimeAdapter, RealtimeEvent } from "@kuralle-syrinx/realtime";
+import { withVoice, type ToolCallStartContext } from "./with-voice.js";
 import type { VoicePipeline } from "./build-session.js";
 
+/** Controllable fake realtime front — `emit()` pushes provider events into the bridge. */
+class FakeFront implements RealtimeAdapter {
+  readonly caps = {
+    inputSampleRateHz: 24_000,
+    outputSampleRateHz: 24_000,
+    supportsConcurrentToolAudio: true,
+    supportsTruncate: true,
+    emitsServerSpeechStarted: true,
+  } as const;
+  #queued: RealtimeEvent[] = [];
+  #waiters: Array<(e: RealtimeEvent | null) => void> = [];
+  #closed = false;
+  readonly events: AsyncIterable<RealtimeEvent> = {
+    [Symbol.asyncIterator]: () => ({
+      next: async (): Promise<IteratorResult<RealtimeEvent>> => {
+        if (this.#queued.length) return { value: this.#queued.shift()!, done: false };
+        if (this.#closed) return { value: undefined, done: true };
+        const e = await new Promise<RealtimeEvent | null>((r) => this.#waiters.push(r));
+        return e === null ? { value: undefined, done: true } : { value: e, done: false };
+      },
+    }),
+  };
+  async open(): Promise<void> {}
+  sendAudio(): void {}
+  cancelResponse(): void {}
+  injectToolResult(): void {}
+  async close(): Promise<void> {
+    this.#closed = true;
+    for (const w of this.#waiters.splice(0)) w(null);
+  }
+  emit(e: RealtimeEvent): void {
+    const w = this.#waiters.shift();
+    if (w) w(e);
+    else this.#queued.push(e);
+  }
+}
+
 // --- Fakes ---------------------------------------------------------------
 
 class FakeAgentBase {
@@ -263,6 +301,65 @@ describe("withVoice(Agent)", () => {
     expect(jsonFrames(conn).some((f) => f["type"] === "error")).toBe(false);
   });
 
+  it("fires onToolCallStart with the tool + the live connection when the delegate tool is invoked", async () => {
+    const front = new FakeFront();
+    const calls: ToolCallStartContext[] = [];
+    const realtimePipeline: VoicePipeline<Record<string, unknown>> = {
+      kind: "realtime",
+      front: () => front,
+      delegateToolName: "consult_knowledge",
+    };
+    const VoiceAgent = withVoice<Record<string, unknown>, ReturnType<typeof asBase>>(
+      asBase(FakeAgentBase),
+      {
+        pipeline: realtimePipeline,
+        reasoner: () => stubReasoner(),
+        onToolCallStart: (c) => { calls.push(c); },
+      },
+    );
+    const agent = new VoiceAgent({});
+    const conn = fakeConnection();
+
+    agent.onConnect(conn, ctx());
+    await vi.waitFor(() => expect(jsonFrames(conn).some((f) => f["type"] === "ready")).toBe(true));
+
+    // The front model starts the turn, then invokes the delegate tool — the latency-mask moment.
+    front.emit({ type: "response_started" });
+    front.emit({ type: "tool_call", toolId: "t1", toolName: "consult_knowledge", args: { query: "fees" } });
+
+    await vi.waitFor(() => expect(calls.length).toBeGreaterThan(0));
+    expect(calls[0]!.toolName).toBe("consult_knowledge");
+    expect(calls[0]!.args).toEqual({ query: "fees" });
+    expect(calls[0]!.sessionId).toBe("test-session");
+    // The app gets the real connection — it can `connection.send(...)` a preamble/earcon trigger.
+    expect(calls[0]!.connection).toBe(conn);
+  });
+
+  it("a throwing onToolCallStart never breaks the call", async () => {
+    const front = new FakeFront();
+    const VoiceAgent = withVoice<Record<string, unknown>, ReturnType<typeof asBase>>(
+      asBase(FakeAgentBase),
+      {
+        pipeline: { kind: "realtime", front: () => front, delegateToolName: "consult_knowledge" },
+        reasoner: () => stubReasoner(),
+        onToolCallStart: () => { throw new Error("app hook blew up"); },
+      },
+    );
+    const agent = new VoiceAgent({});
+    const conn = fakeConnection();
+
+    agent.onConnect(conn, ctx());
+    await vi.waitFor(() => expect(jsonFrames(conn).some((f) => f["type"] === "ready")).toBe(true));
+
+    front.emit({ type: "response_started" });
+    expect(() => {
+      front.emit({ type: "tool_call", toolId: "t1", toolName: "consult_knowledge", args: { query: "x" } });
+    }).not.toThrow();
+    // The connection stays usable (a ping after the throwing hook is still handled).
+    await new Promise((r) => setTimeout(r, 10));
+    expect(() => agent.onMessage(conn, JSON.stringify({ type: "ping" }))).not.toThrow();
+  });
+
   it("forceEndVoice closes the connection", () => {
     const VoiceAgent = withVoice<Record<string, unknown>, ReturnType<typeof asBase>>(
       asBase(FakeAgentBase),

package/src/with-voice.ts

@@ -44,6 +44,21 @@ type AgentLike = Constructor<
   Pick<Agent<Record<string, unknown>>, "sql" | "getConnections" | "keepAlive">
 >;
 
+/**
+ * Fired the instant the front model invokes the delegate tool — BEFORE the reasoner runs.
+ * Lets an app emit a deterministic, in-language preamble or a "thinking" earcon that masks the
+ * reasoner's wait, instead of relying on the realtime front LLM to remember to speak one (cf.
+ * Vapi/Pipecat `on_function_calls_started`). Use `connection.send(...)` to signal the client
+ * (the idiomatic agents-SDK pattern) — e.g. trigger a cached client-side earcon/preamble.
+ */
+export interface ToolCallStartContext {
+  readonly toolName: string;
+  readonly args: Record<string, unknown>;
+  readonly sessionId: string;
+  /** The live agents-SDK connection — `connection.send(json)` to message the client. */
+  readonly connection: VoiceConnection;
+}
+
 export interface WithVoiceOptions<Env> {
   /**
    * The connection wire protocol this host speaks. `"edge"` (default) is the Syrinx
@@ -63,6 +78,12 @@ export interface WithVoiceOptions<Env> {
   readonly reasoner?: (env: Env, ctx: VoicePipelineContext) => Reasoner | Promise<Reasoner>;
   /** Optional per-call recorder (e.g. an R2-backed `EdgeRecorder`). Applies to the `"edge"` transport. */
   readonly recorder?: (env: Env, ctx: VoicePipelineContext) => EdgeRecorder | undefined;
+  /**
+   * Fired when the front model starts a delegate tool call, before the reasoner runs — the seam
+   * for a deterministic latency-masking preamble / "thinking" earcon. Throwing here never affects
+   * the call. See {@link ToolCallStartContext}.
+   */
+  readonly onToolCallStart?: (ctx: ToolCallStartContext) => void | Promise<void>;
   readonly inputSampleRateHz?: number;
   readonly outputSampleRateHz?: number;
   readonly resumeWindowMs?: number;
@@ -156,9 +177,8 @@ export function withVoice<Env, TBase extends AgentLike>(
       // (workerd's WebSocket.send accepts string/ArrayBuffer/ArrayBufferView), but
       // its lib type is nominally stricter (ArrayBuffer- vs ArrayBufferLike-backed
       // views). Bridge that single boundary structurally.
-      const { socket, controller } = connectionManagedSocket(
-        connection as unknown as VoiceConnection,
-      );
+      const voiceConnection = connection as unknown as VoiceConnection;
+      const { socket, controller } = connectionManagedSocket(voiceConnection);
       this.#controllers.set(connection.id, controller);
 
       // Release the keepAlive lease + drop the controller on ANY close — a client
@@ -187,7 +207,22 @@ export function withVoice<Env, TBase extends AgentLike>(
       // Both runners assemble the session the same way — pipeline + (resolved) reasoner.
       const createSession = async () => {
         const reasoner = await this.#resolveReasoner(env, sessionId);
-        return buildVoiceSession(options.pipeline, env, reasoner, { sessionId });
+        const session = buildVoiceSession(options.pipeline, env, reasoner, { sessionId });
+        // Surface the tool-call-start seam: VoiceAgentSession emits `agent_tool_call` the instant
+        // the front model invokes the delegate tool, before the reasoner runs. A throwing app
+        // callback must never break the call, so it is fully isolated.
+        if (options.onToolCallStart) {
+          session.on("agent_tool_call", (e) => {
+            try {
+              void Promise.resolve(
+                options.onToolCallStart!({ toolName: e.name, args: e.args, sessionId, connection: voiceConnection }),
+              ).catch(() => undefined);
+            } catch {
+              /* app hook threw synchronously — ignore */
+            }
+          });
+        }
+        return session;
       };
       // The runner reports startup failures to the client and disposes the socket
       // itself; nothing to do here beyond not crashing the isolate.