@kuralle-agents/cf-agent 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -191,7 +191,7 @@ export class KuralleAgent extends AIChatAgent {
191
191
  let text = '';
192
192
  for await (const part of handle.events) {
193
193
  if (part.type === 'text-delta')
194
- text += part.delta;
194
+ text += part.payload.delta;
195
195
  }
196
196
  await handle;
197
197
  if (text.trim()) {
@@ -277,7 +277,7 @@ export class KuralleAgent extends AIChatAgent {
277
277
  let text = '';
278
278
  for await (const part of handle.events) {
279
279
  if (part.type === 'text-delta')
280
- text += part.delta;
280
+ text += part.payload.delta;
281
281
  }
282
282
  await handle;
283
283
  if (text.trim()) {
@@ -1,7 +1,7 @@
1
- import type { HarnessStreamPart } from '@kuralle-agents/core';
1
+ import type { StreamPart } from '@kuralle-agents/core';
2
2
  import type { StreamAdapterConfig } from './types.js';
3
3
  /**
4
- * Convert Kuralle's HarnessStreamPart generator into an SSE Response
4
+ * Convert Kuralle's StreamPart generator into an SSE Response
5
5
  * that CF's AIChatAgent._reply() can parse.
6
6
  *
7
7
  * CF's _streamSSEReply reads lines in the format:
@@ -19,10 +19,10 @@ type SSETextState = {
19
19
  open: boolean;
20
20
  canceledIds: Set<string>;
21
21
  };
22
- export declare function createSSEResponse(stream: AsyncGenerator<HarnessStreamPart>, config?: StreamAdapterConfig): Response;
22
+ export declare function createSSEResponse(stream: AsyncGenerator<StreamPart>, config?: StreamAdapterConfig): Response;
23
23
  /**
24
- * Convert a single HarnessStreamPart to SSE data lines.
24
+ * Convert a single StreamPart to SSE data lines.
25
25
  * Returns 0+ formatted SSE lines ("data: {...}\n\n").
26
26
  */
27
- export declare function convertToSSELines(part: HarnessStreamPart, config: StreamAdapterConfig, textState: SSETextState): string[];
27
+ export declare function convertToSSELines(part: StreamPart, config: StreamAdapterConfig, textState: SSETextState): string[];
28
28
  export {};
@@ -30,7 +30,7 @@ export function createSSEResponse(stream, config = DEFAULT_STREAM_CONFIG) {
30
30
  });
31
31
  }
32
32
  /**
33
- * Convert a single HarnessStreamPart to SSE data lines.
33
+ * Convert a single StreamPart to SSE data lines.
34
34
  * Returns 0+ formatted SSE lines ("data: {...}\n\n").
35
35
  */
36
36
  export function convertToSSELines(part, config, textState) {
@@ -49,19 +49,19 @@ export function convertToSSELines(part, config, textState) {
49
49
  closeTextSegment();
50
50
  break;
51
51
  case 'text-cancel': {
52
- textState.canceledIds.add(part.id);
52
+ textState.canceledIds.add(part.payload.id);
53
53
  closeTextSegment();
54
54
  break;
55
55
  }
56
56
  case 'text-delta': {
57
- if (textState.canceledIds.has(part.id)) {
57
+ if (textState.canceledIds.has(part.payload.id)) {
58
58
  break;
59
59
  }
60
60
  if (!textState.open) {
61
61
  lines.push(sse({ type: 'text-start' }));
62
62
  textState.open = true;
63
63
  }
64
- lines.push(sse({ type: 'text-delta', delta: part.delta }));
64
+ lines.push(sse({ type: 'text-delta', delta: part.payload.delta }));
65
65
  break;
66
66
  }
67
67
  case 'tool-call': {
@@ -69,17 +69,17 @@ export function convertToSSELines(part, config, textState) {
69
69
  // CF expects tool-input-available with toolCallId, toolName, input
70
70
  lines.push(sse({
71
71
  type: 'tool-input-available',
72
- toolCallId: part.toolCallId,
73
- toolName: part.toolName,
74
- input: config.includeToolArgs ? part.args : undefined,
72
+ toolCallId: part.payload.toolCallId,
73
+ toolName: part.payload.toolName,
74
+ input: config.includeToolArgs ? part.payload.args : undefined,
75
75
  }));
76
76
  break;
77
77
  }
78
78
  case 'tool-result': {
79
79
  lines.push(sse({
80
80
  type: 'tool-output-available',
81
- toolCallId: part.toolCallId,
82
- output: part.result,
81
+ toolCallId: part.payload.toolCallId,
82
+ output: part.payload.result,
83
83
  }));
84
84
  break;
85
85
  }
@@ -89,8 +89,8 @@ export function convertToSSELines(part, config, textState) {
89
89
  lines.push(sse({
90
90
  type: 'data-handoff',
91
91
  data: {
92
- to: part.targetAgent,
93
- reason: part.reason,
92
+ to: part.payload.targetAgent,
93
+ reason: part.payload.reason,
94
94
  },
95
95
  }));
96
96
  break;
@@ -100,7 +100,7 @@ export function convertToSSELines(part, config, textState) {
100
100
  break;
101
101
  lines.push(sse({
102
102
  type: 'data-flow-enter',
103
- data: { flow: part.flow },
103
+ data: { flow: part.payload.flow },
104
104
  }));
105
105
  break;
106
106
  }
@@ -109,7 +109,7 @@ export function convertToSSELines(part, config, textState) {
109
109
  break;
110
110
  lines.push(sse({
111
111
  type: 'data-flow-node',
112
- data: { node: part.nodeName, event: 'enter' },
112
+ data: { node: part.payload.nodeName, event: 'enter' },
113
113
  }));
114
114
  break;
115
115
  }
@@ -118,7 +118,7 @@ export function convertToSSELines(part, config, textState) {
118
118
  break;
119
119
  lines.push(sse({
120
120
  type: 'data-flow-node',
121
- data: { node: part.nodeName, event: 'exit' },
121
+ data: { node: part.payload.nodeName, event: 'exit' },
122
122
  }));
123
123
  break;
124
124
  }
@@ -127,7 +127,7 @@ export function convertToSSELines(part, config, textState) {
127
127
  break;
128
128
  lines.push(sse({
129
129
  type: 'data-flow-transition',
130
- data: { from: part.from, to: part.to },
130
+ data: { from: part.payload.from, to: part.payload.to },
131
131
  }));
132
132
  break;
133
133
  }
@@ -136,14 +136,14 @@ export function convertToSSELines(part, config, textState) {
136
136
  break;
137
137
  lines.push(sse({
138
138
  type: 'data-flow-end',
139
- data: { reason: part.reason },
139
+ data: { reason: part.payload.reason },
140
140
  }));
141
141
  break;
142
142
  }
143
143
  case 'error': {
144
144
  lines.push(sse({
145
145
  type: 'data-error',
146
- data: { error: part.error },
146
+ data: { error: part.payload.error },
147
147
  }));
148
148
  break;
149
149
  }
@@ -51,8 +51,8 @@ function turnResult(parts) {
51
51
  const paused = parts.find((part) => part.type === 'paused');
52
52
  return {
53
53
  parts,
54
- suspended: paused ? { signalId: paused.waitingFor } : undefined,
55
- handoffToHuman: parts.some((part) => part.type === 'handoff' && part.targetAgent === 'human'),
54
+ suspended: paused ? { signalId: paused.payload.waitingFor } : undefined,
55
+ handoffToHuman: parts.some((part) => part.type === 'handoff' && part.payload.targetAgent === 'human'),
56
56
  };
57
57
  }
58
58
  export class SqlInboundLedger {
package/dist/index.d.ts CHANGED
@@ -35,4 +35,4 @@ export { AgentScheduleCoalesceScheduler, QueuedTurnRunner, RuntimeTurnRunner, Sq
35
35
  export type { StreamAdapterConfig, OrchestrationState, SqlExecutor, } from './types.js';
36
36
  export type { DurableObjectInboundRuntimeOptions, ScheduleHost, } from './inbound-runtime.js';
37
37
  export { DEFAULT_STREAM_CONFIG } from './types.js';
38
- export type { HarnessConfig, HarnessHooks, HarnessStreamPart, Session, } from '@kuralle-agents/core';
38
+ export type { HarnessConfig, StreamPart, Session, } from '@kuralle-agents/core';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@kuralle-agents/cf-agent",
3
- "version": "0.13.0",
3
+ "version": "0.14.0",
4
4
  "description": "Kuralle agent integration for Cloudflare Workers with AIChatAgent",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -9,10 +9,6 @@
9
9
  ".": {
10
10
  "import": "./dist/index.js",
11
11
  "types": "./dist/index.d.ts"
12
- },
13
- "./voice": {
14
- "import": "./dist/voice/index.js",
15
- "types": "./dist/voice/index.d.ts"
16
12
  }
17
13
  },
18
14
  "files": [
@@ -21,22 +17,19 @@
21
17
  "dependencies": {
22
18
  "@cloudflare/ai-chat": "^0.8.4",
23
19
  "ai": "^6.0.90",
24
- "@kuralle-agents/core": "0.13.0",
25
- "@kuralle-agents/realtime-audio": "0.13.0",
26
- "@kuralle-agents/messaging": "0.13.0"
20
+ "@kuralle-agents/core": "0.14.0",
21
+ "@kuralle-agents/messaging": "0.14.0"
27
22
  },
28
23
  "peerDependencies": {
29
24
  "agents": ">=0.14.0 <1.0.0",
30
- "zod": "^4.0.0",
31
- "@kuralle-agents/voice-protocol": "0.13.0"
25
+ "zod": "^4.0.0"
32
26
  },
33
27
  "devDependencies": {
34
28
  "@cloudflare/vitest-pool-workers": "^0.12.7",
35
29
  "@cloudflare/workers-types": "^4.0.0",
36
30
  "agents": "^0.15.0",
37
31
  "typescript": "^5.7.0",
38
- "vitest": "^3.2.4",
39
- "@kuralle-agents/voice-protocol": "0.13.0"
32
+ "vitest": "^3.2.4"
40
33
  },
41
34
  "engines": {
42
35
  "node": ">=20"
@@ -60,7 +53,7 @@
60
53
  "prebuild": "rm -rf dist",
61
54
  "build": "tsc",
62
55
  "dev": "tsc --watch",
63
- "test": "bun test src/voice/__tests__ src/__tests__ && vitest run --config vitest.config.ts",
56
+ "test": "bun test src/__tests__ && vitest run --config vitest.config.ts",
64
57
  "test:sql-memory-workers": "vitest run --config vitest.config.ts",
65
58
  "typecheck": "tsc --noEmit",
66
59
  "clean": "rm -rf dist"
@@ -1,43 +0,0 @@
1
- /**
2
- * Vendored, trimmed port of AudioConnectionManager + sendVoiceJSON from
3
- * `@cloudflare/voice` (cloudflare/agents, packages/voice/src/audio-pipeline.ts,
4
- * MIT License, © 2025 Cloudflare, Inc.).
5
- *
6
- * Original provides cascaded-pipeline state (audio buffers, transcriber
7
- * sessions, abort pipelines). The realtime variant needs none of that —
8
- * audio is pass-through, turn detection happens provider-side, there is no
9
- * separate STT session to manage. We keep only the connection-level
10
- * in-call tracking primitive.
11
- *
12
- * Upstream source:
13
- * https://github.com/cloudflare/agents/blob/main/packages/voice/src/audio-pipeline.ts
14
- */
15
- /**
16
- * Minimal per-connection state manager for the realtime voice mixin.
17
- *
18
- * Tracks which connection IDs currently have an open realtime session.
19
- * Owning the Map here (rather than inline in the mixin) keeps the mixin
20
- * class body focused on protocol handling and matches the shape the
21
- * cascaded lane uses.
22
- */
23
- export declare class AudioConnectionManager {
24
- #private;
25
- /** Mark a connection as having an active in-call state. */
26
- initConnection(connectionId: string): void;
27
- /** True if this connection currently has an active realtime session. */
28
- isInCall(connectionId: string): boolean;
29
- /** Clear in-call state for a connection (end_call or onClose). */
30
- cleanup(connectionId: string): void;
31
- /** Count of active in-call connections. Used for maxConcurrentSessions gate. */
32
- size(): number;
33
- }
34
- /**
35
- * Serialize and push a voice-protocol frame to a client connection.
36
- *
37
- * Upstream accepts a log prefix; we drop it since the realtime path has no
38
- * chatty log calls. Signature intentionally minimal so the mixin's send
39
- * sites stay readable.
40
- */
41
- export declare function sendVoiceJSON(connection: {
42
- send(data: string | ArrayBuffer): void;
43
- }, data: unknown): void;
@@ -1,51 +0,0 @@
1
- /**
2
- * Vendored, trimmed port of AudioConnectionManager + sendVoiceJSON from
3
- * `@cloudflare/voice` (cloudflare/agents, packages/voice/src/audio-pipeline.ts,
4
- * MIT License, © 2025 Cloudflare, Inc.).
5
- *
6
- * Original provides cascaded-pipeline state (audio buffers, transcriber
7
- * sessions, abort pipelines). The realtime variant needs none of that —
8
- * audio is pass-through, turn detection happens provider-side, there is no
9
- * separate STT session to manage. We keep only the connection-level
10
- * in-call tracking primitive.
11
- *
12
- * Upstream source:
13
- * https://github.com/cloudflare/agents/blob/main/packages/voice/src/audio-pipeline.ts
14
- */
15
- /**
16
- * Minimal per-connection state manager for the realtime voice mixin.
17
- *
18
- * Tracks which connection IDs currently have an open realtime session.
19
- * Owning the Map here (rather than inline in the mixin) keeps the mixin
20
- * class body focused on protocol handling and matches the shape the
21
- * cascaded lane uses.
22
- */
23
- export class AudioConnectionManager {
24
- #inCall = new Set();
25
- /** Mark a connection as having an active in-call state. */
26
- initConnection(connectionId) {
27
- this.#inCall.add(connectionId);
28
- }
29
- /** True if this connection currently has an active realtime session. */
30
- isInCall(connectionId) {
31
- return this.#inCall.has(connectionId);
32
- }
33
- /** Clear in-call state for a connection (end_call or onClose). */
34
- cleanup(connectionId) {
35
- this.#inCall.delete(connectionId);
36
- }
37
- /** Count of active in-call connections. Used for maxConcurrentSessions gate. */
38
- size() {
39
- return this.#inCall.size;
40
- }
41
- }
42
- /**
43
- * Serialize and push a voice-protocol frame to a client connection.
44
- *
45
- * Upstream accepts a log prefix; we drop it since the realtime path has no
46
- * chatty log calls. Signature intentionally minimal so the mixin's send
47
- * sites stay readable.
48
- */
49
- export function sendVoiceJSON(connection, data) {
50
- connection.send(JSON.stringify(data));
51
- }
@@ -1,48 +0,0 @@
1
- /**
2
- * KuralleRealtimeVoiceAgent — thin concrete class applying the
3
- * {@link withRealtimeVoice} mixin to {@link KuralleAgent}.
4
- *
5
- * Subclasses must either set `realtimeModel` to a constructed
6
- * {@link RealtimeAudioClient}, or override `createRealtimeModel(env)` so the
7
- * mixin can lazily build one (usually a `CloudflareGeminiLiveClient` with
8
- * `this.env.GEMINI_API_KEY`).
9
- *
10
- * @example
11
- * ```ts
12
- * import { CloudflareGeminiLiveClient } from "@kuralle-agents/realtime-audio";
13
- * import { KuralleRealtimeVoiceAgent } from "@kuralle-agents/cf-agent/voice";
14
- *
15
- * export class MyVoiceAgent extends KuralleRealtimeVoiceAgent<Env> {
16
- * createRealtimeModel(env: Env) {
17
- * return new CloudflareGeminiLiveClient({ apiKey: env.GEMINI_API_KEY });
18
- * }
19
- * protected getAgents() { return [{ id: "assistant", name: "Assistant", tools: {} }]; }
20
- * protected getDefaultAgentId() { return "assistant"; }
21
- * }
22
- * ```
23
- */
24
- import { KuralleAgent } from "../KuralleAgent.js";
25
- declare const RealtimeVoiceKuralleAgentBase: typeof KuralleAgent;
26
- export declare abstract class KuralleRealtimeVoiceAgent<Env = unknown, State = unknown> extends RealtimeVoiceKuralleAgentBase<Env, State> {
27
- /**
28
- * Hibernate between calls; stay awake during active calls via `keepAlive()`.
29
- *
30
- * Reasoning:
31
- * - Client WS uses the agents-SDK `Connection` abstraction (hibernation-
32
- * aware), so a reconnect after eviction lands on a rehydrated DO with
33
- * `this.messages` + `this.state` restored from SQLite.
34
- * - Outbound provider WS is not hibernation-aware, but `keepAlive()`
35
- * is held for the duration of a call → no eviction during calls.
36
- * - On `end_call` / teardown we close the outbound cleanly, then release
37
- * keepAlive → DO can safely hibernate (free) between calls.
38
- *
39
- * Contrast: `cloudflare/agents/openai-sdk/call-my-agent` sets
40
- * `hibernate: false` because its `TwilioRealtimeTransportLayer` attaches
41
- * raw `.addEventListener` handlers bypassing the hibernation path — that
42
- * doesn't apply to us.
43
- */
44
- static options: {
45
- hibernate: boolean;
46
- };
47
- }
48
- export {};
@@ -1,46 +0,0 @@
1
- /**
2
- * KuralleRealtimeVoiceAgent — thin concrete class applying the
3
- * {@link withRealtimeVoice} mixin to {@link KuralleAgent}.
4
- *
5
- * Subclasses must either set `realtimeModel` to a constructed
6
- * {@link RealtimeAudioClient}, or override `createRealtimeModel(env)` so the
7
- * mixin can lazily build one (usually a `CloudflareGeminiLiveClient` with
8
- * `this.env.GEMINI_API_KEY`).
9
- *
10
- * @example
11
- * ```ts
12
- * import { CloudflareGeminiLiveClient } from "@kuralle-agents/realtime-audio";
13
- * import { KuralleRealtimeVoiceAgent } from "@kuralle-agents/cf-agent/voice";
14
- *
15
- * export class MyVoiceAgent extends KuralleRealtimeVoiceAgent<Env> {
16
- * createRealtimeModel(env: Env) {
17
- * return new CloudflareGeminiLiveClient({ apiKey: env.GEMINI_API_KEY });
18
- * }
19
- * protected getAgents() { return [{ id: "assistant", name: "Assistant", tools: {} }]; }
20
- * protected getDefaultAgentId() { return "assistant"; }
21
- * }
22
- * ```
23
- */
24
- import { KuralleAgent } from "../KuralleAgent.js";
25
- import { withRealtimeVoice } from "./withRealtimeVoice.js";
26
- const RealtimeVoiceKuralleAgentBase = withRealtimeVoice(KuralleAgent);
27
- export class KuralleRealtimeVoiceAgent extends RealtimeVoiceKuralleAgentBase {
28
- /**
29
- * Hibernate between calls; stay awake during active calls via `keepAlive()`.
30
- *
31
- * Reasoning:
32
- * - Client WS uses the agents-SDK `Connection` abstraction (hibernation-
33
- * aware), so a reconnect after eviction lands on a rehydrated DO with
34
- * `this.messages` + `this.state` restored from SQLite.
35
- * - Outbound provider WS is not hibernation-aware, but `keepAlive()`
36
- * is held for the duration of a call → no eviction during calls.
37
- * - On `end_call` / teardown we close the outbound cleanly, then release
38
- * keepAlive → DO can safely hibernate (free) between calls.
39
- *
40
- * Contrast: `cloudflare/agents/openai-sdk/call-my-agent` sets
41
- * `hibernate: false` because its `TwilioRealtimeTransportLayer` attaches
42
- * raw `.addEventListener` handlers bypassing the hibernation path — that
43
- * doesn't apply to us.
44
- */
45
- static options = { hibernate: true };
46
- }
@@ -1,13 +0,0 @@
1
- /**
2
- * @kuralle-agents/cf-agent/voice — realtime voice mixin + agent base for
3
- * Cloudflare Durable Objects. Pairs with any
4
- * {@link RealtimeAudioClient} (v2) implementation — Gemini Live,
5
- * OpenAI Realtime, xAI Grok, etc.
6
- *
7
- * For the cascaded (STT → LLM → TTS) variant, see the sibling
8
- * `KuralleCascadedVoiceAgent` export.
9
- */
10
- export { withRealtimeVoice } from "./withRealtimeVoice.js";
11
- export type { RealtimeVoiceOptions, RealtimeVoiceMixinMembers, } from "./withRealtimeVoice.js";
12
- export { KuralleRealtimeVoiceAgent } from "./RealtimeVoiceAgent.js";
13
- export { AudioConnectionManager, sendVoiceJSON } from "./AudioConnectionManager.js";
@@ -1,12 +0,0 @@
1
- /**
2
- * @kuralle-agents/cf-agent/voice — realtime voice mixin + agent base for
3
- * Cloudflare Durable Objects. Pairs with any
4
- * {@link RealtimeAudioClient} (v2) implementation — Gemini Live,
5
- * OpenAI Realtime, xAI Grok, etc.
6
- *
7
- * For the cascaded (STT → LLM → TTS) variant, see the sibling
8
- * `KuralleCascadedVoiceAgent` export.
9
- */
10
- export { withRealtimeVoice } from "./withRealtimeVoice.js";
11
- export { KuralleRealtimeVoiceAgent } from "./RealtimeVoiceAgent.js";
12
- export { AudioConnectionManager, sendVoiceJSON } from "./AudioConnectionManager.js";
@@ -1,63 +0,0 @@
1
- /**
2
- * withRealtimeVoice — functional mixin adding realtime voice to an
3
- * AIChatAgent-derived Durable Object class.
4
- *
5
- * Realtime voice convergence: this mixin is now pure transport glue. All
6
- * orchestration — tool dispatch, flow transitions, hooks, handoffs, session
7
- * persistence, reconnect strategy, chat_ctx replay — lives in Kuralle's
8
- * `Runtime` and `CloudflareRealtimeAdapter`.
9
- *
10
- * Responsibilities that stay in the mixin:
11
- * - Intercept voice-protocol frames on the browser ↔ DO WebSocket.
12
- * - Per-connection concurrent-session cap (`maxConcurrentSessions`).
13
- * - DO-specific lifecycle (`keepAlive()` during an active call).
14
- * - Expose subclass-override hooks (`beforeSessionStart`, `onSessionStart`,
15
- * `onSessionEnd`, `onInterrupt`, `onUserTranscript`, `onModelTranscript`).
16
- * - Build the `Runtime` per call (via `createRuntime(...)`),
17
- * wire in a `BridgeSessionStore`, and hand control to the adapter.
18
- *
19
- * Responsibilities moved OUT of the mixin:
20
- * - Tool execution → the `Runtime` effect path (`ctx.tool`)
21
- * - Flow reconfigure / handoff → same, via the runtime's transition handling
22
- * - Transcript persistence → the `Runtime` session + event log +
23
- * existing `AIChatAgent.persistMessages`
24
- * - Resumption handle + chat_ctx replay → provider client (in-memory) +
25
- * adapter (reconnect dispatch)
26
- * - SQLite tables `cf_realtime_resumption` and `cf_realtime_chat_ctx`
27
- * — DELETED. Storage flows through `SessionStore` (BridgeSessionStore on CF).
28
- *
29
- * Capture-and-patch lifecycle wrap mirrors `@cloudflare/voice`'s pattern
30
- * (voice.ts:251-334, MIT, © 2025 Cloudflare, Inc.).
31
- */
32
- import type { Connection } from "agents";
33
- import type { RealtimeAudioClient } from "@kuralle-agents/core/realtime";
34
- type Constructor<T = object> = new (...args: unknown[]) => T;
35
- type AbstractConstructor<T = object> = abstract new (...args: unknown[]) => T;
36
- type AgentLike = Constructor | AbstractConstructor;
37
- export interface RealtimeVoiceOptions {
38
- /** Audio wire format. Fixed to PCM16 today; future provider-specific variants. */
39
- audioFormat?: "pcm16";
40
- /** Per-DO concurrent-session cap. 5th `start_call` rejected with an `error` frame. Default 4. */
41
- maxConcurrentSessions?: number;
42
- /** Max outbound-reconnect attempts under backoff before giving up. Default 3. */
43
- reconnectMaxAttempts?: number;
44
- /** Base delay for backoff (delay ∈ [0, min(cap, base * 2^attempt)]). Default 500ms. */
45
- reconnectBaseDelayMs?: number;
46
- /** Cap on the backoff delay. Default 8000ms. */
47
- reconnectCapDelayMs?: number;
48
- /** Bytes of PCM to ring-buffer per-connection across a reconnect window. Default 48_000. */
49
- reconnectAudioBufferBytes?: number;
50
- }
51
- export interface RealtimeVoiceMixinMembers {
52
- realtimeModel?: RealtimeAudioClient;
53
- createRealtimeModel?(env: unknown): RealtimeAudioClient;
54
- beforeSessionStart(conn: Connection): boolean | Promise<boolean>;
55
- onSessionStart(conn: Connection): void | Promise<void>;
56
- onSessionEnd(conn: Connection): void | Promise<void>;
57
- onInterrupt(conn: Connection): void | Promise<void>;
58
- onUserTranscript(text: string, isFinal: boolean, conn: Connection): void | Promise<void>;
59
- onModelTranscript(text: string, isFinal: boolean, conn: Connection): void | Promise<void>;
60
- forceEndSession(conn: Connection): void;
61
- }
62
- export declare function withRealtimeVoice<TBase extends AgentLike>(Base: TBase, opts?: RealtimeVoiceOptions): TBase & Constructor<RealtimeVoiceMixinMembers>;
63
- export {};
@@ -1,451 +0,0 @@
1
- /**
2
- * withRealtimeVoice — functional mixin adding realtime voice to an
3
- * AIChatAgent-derived Durable Object class.
4
- *
5
- * Realtime voice convergence: this mixin is now pure transport glue. All
6
- * orchestration — tool dispatch, flow transitions, hooks, handoffs, session
7
- * persistence, reconnect strategy, chat_ctx replay — lives in Kuralle's
8
- * `Runtime` and `CloudflareRealtimeAdapter`.
9
- *
10
- * Responsibilities that stay in the mixin:
11
- * - Intercept voice-protocol frames on the browser ↔ DO WebSocket.
12
- * - Per-connection concurrent-session cap (`maxConcurrentSessions`).
13
- * - DO-specific lifecycle (`keepAlive()` during an active call).
14
- * - Expose subclass-override hooks (`beforeSessionStart`, `onSessionStart`,
15
- * `onSessionEnd`, `onInterrupt`, `onUserTranscript`, `onModelTranscript`).
16
- * - Build the `Runtime` per call (via `createRuntime(...)`),
17
- * wire in a `BridgeSessionStore`, and hand control to the adapter.
18
- *
19
- * Responsibilities moved OUT of the mixin:
20
- * - Tool execution → the `Runtime` effect path (`ctx.tool`)
21
- * - Flow reconfigure / handoff → same, via the runtime's transition handling
22
- * - Transcript persistence → the `Runtime` session + event log +
23
- * existing `AIChatAgent.persistMessages`
24
- * - Resumption handle + chat_ctx replay → provider client (in-memory) +
25
- * adapter (reconnect dispatch)
26
- * - SQLite tables `cf_realtime_resumption` and `cf_realtime_chat_ctx`
27
- * — DELETED. Storage flows through `SessionStore` (BridgeSessionStore on CF).
28
- *
29
- * Capture-and-patch lifecycle wrap mirrors `@cloudflare/voice`'s pattern
30
- * (voice.ts:251-334, MIT, © 2025 Cloudflare, Inc.).
31
- */
32
- import { createRuntime } from "@kuralle-agents/core";
33
- import { VOICE_PROTOCOL_VERSION } from "@kuralle-agents/voice-protocol";
34
- import { CloudflareRealtimeAdapter, } from "@kuralle-agents/realtime-audio";
35
- import { AudioConnectionManager, sendVoiceJSON, } from "./AudioConnectionManager.js";
36
- import { BridgeSessionStore } from "../BridgeSessionStore.js";
37
- import { durableAgentSurface, } from "../durable-agent-surface.js";
38
- function voiceHost(self) {
39
- return durableAgentSurface(self);
40
- }
41
- function resolveSql(self) {
42
- const { sql } = durableAgentSurface(self);
43
- if (typeof sql !== "function")
44
- return undefined;
45
- return sql.bind(self);
46
- }
47
- function inferModelPolicy(client) {
48
- const provider = (client.provider ?? "unknown").toLowerCase();
49
- const supportsInstructionUpdate = client.capabilities?.midSessionInstructionsUpdate === true ||
50
- client.capabilities?.midSessionChatCtxUpdate === true;
51
- if (provider.startsWith("gemini") || provider.startsWith("google")) {
52
- return { provider: "google", supportsInstructionUpdate };
53
- }
54
- if (provider.startsWith("openai")) {
55
- return { provider: "openai", supportsInstructionUpdate };
56
- }
57
- if (provider.startsWith("azure")) {
58
- return { provider: "azure-openai", supportsInstructionUpdate };
59
- }
60
- if (provider.startsWith("xai")) {
61
- return { provider: "xai", supportsInstructionUpdate };
62
- }
63
- if (provider.startsWith("phonic")) {
64
- return { provider: "phonic", supportsInstructionUpdate };
65
- }
66
- if (provider.startsWith("workers")) {
67
- return { provider: "workers-ai", supportsInstructionUpdate };
68
- }
69
- return { provider: "unknown", supportsInstructionUpdate };
70
- }
71
- export function withRealtimeVoice(Base, opts = {}) {
72
- const maxConcurrent = opts.maxConcurrentSessions ?? 4;
73
- const reconnectMaxAttempts = opts.reconnectMaxAttempts ?? 3;
74
- const reconnectBaseDelayMs = opts.reconnectBaseDelayMs ?? 500;
75
- const reconnectCapDelayMs = opts.reconnectCapDelayMs ?? 8000;
76
- const reconnectAudioBufferBytes = opts.reconnectAudioBufferBytes ?? 48_000;
77
- class RealtimeVoiceMixin extends Base {
78
- realtimeModel;
79
- createRealtimeModel(_env) {
80
- throw new Error("Subclass must override createRealtimeModel or set realtimeModel.");
81
- }
82
- #cm = new AudioConnectionManager();
83
- #adapters = new Map();
84
- #agents = new Map();
85
- #keepAlive = new Map();
86
- #starting = new Set();
87
- /** One-shot guard: emit `session_lost` at most once per conn. */
88
- #sessionLossSignalled = new Set();
89
- /** Warn once per instance when host lacks `keepAlive()` — calls will die at 70-140s idle. */
90
- #keepAliveWarned = false;
91
- // eslint-disable-next-line @typescript-eslint/no-explicit-any -- reason: TypeScript TS2545 mixin constructors require `any[]` rest params
92
- constructor(...args) {
93
- super(...args);
94
- const host = voiceHost(this);
95
- const _onConnect = host.onConnect?.bind(this);
96
- const _onClose = host.onClose?.bind(this);
97
- const _onMessage = host.onMessage?.bind(this);
98
- host.onConnect = (c, ...rest) => {
99
- sendVoiceJSON(c, { type: "welcome", protocol_version: VOICE_PROTOCOL_VERSION });
100
- sendVoiceJSON(c, { type: "status", status: "idle" });
101
- return _onConnect?.(c, ...rest);
102
- };
103
- host.onClose = (c, ...rest) => {
104
- void this.#teardown(c);
105
- return _onClose?.(c, ...rest);
106
- };
107
- host.onMessage = (c, m) => {
108
- if (m instanceof ArrayBuffer) {
109
- const frame = new Uint8Array(m);
110
- const adapter = this.#adapters.get(c.id);
111
- if (adapter) {
112
- adapter.sendUserAudio(frame);
113
- return;
114
- }
115
- // No adapter for this conn. Three plausible causes:
116
- // 1. Mid-call DO eviction — isolate died, `#adapters` wiped, client
117
- // WS re-hydrated by hibernation, audio now lands on a fresh
118
- // mixin with no session to forward to.
119
- // 2. Teardown race — client sent last 20ms of audio after
120
- // `end_call`. `@cloudflare/voice` stops streaming on end so
121
- // this is near-zero in practice.
122
- // 3. Buggy client streaming pre-`start_call`.
123
- // In all three cases, silently dropping audio is worse than
124
- // signalling. Emit once + close so partysocket reconnects cleanly
125
- // and the user can restart.
126
- if (this.#starting.has(c.id))
127
- return;
128
- if (this.#sessionLossSignalled.has(c.id))
129
- return;
130
- this.#sessionLossSignalled.add(c.id);
131
- sendVoiceJSON(c, {
132
- type: "error",
133
- code: "session_lost",
134
- message: "Voice session lost (server restart, eviction, or no active call). " +
135
- "Please start a new call.",
136
- });
137
- try {
138
- c.close(1011, "session_lost");
139
- }
140
- catch {
141
- /* connection already closing */
142
- }
143
- return;
144
- }
145
- if (typeof m !== "string")
146
- return _onMessage?.(c, m);
147
- let parsed;
148
- try {
149
- parsed = JSON.parse(m);
150
- }
151
- catch {
152
- return _onMessage?.(c, m);
153
- }
154
- switch (parsed.type) {
155
- case "start_call":
156
- this.#handleStartCall(c).catch((err) => {
157
- console.error("[withRealtimeVoice] #handleStartCall REJECTED:", String(err), err instanceof Error ? err.stack : "");
158
- const raw = err instanceof Error ? err.message : String(err);
159
- sendVoiceJSON(c, { type: "error", message: `start_call failed: ${raw}` });
160
- });
161
- return;
162
- case "end_call":
163
- void this.#teardown(c);
164
- return;
165
- case "interrupt":
166
- void this.#handleInterrupt(c);
167
- return;
168
- case "text_message": {
169
- const text = typeof parsed.text === "string" ? parsed.text : undefined;
170
- if (text)
171
- this.#adapters.get(c.id)?.sendUserText(text);
172
- return;
173
- }
174
- case "start_of_speech":
175
- case "end_of_speech":
176
- case "hello":
177
- return; // acknowledge but no-op
178
- default:
179
- return _onMessage?.(c, m);
180
- }
181
- };
182
- }
183
- // ─── Hook defaults (overridable by subclasses) ──────────────────────────
184
- async beforeSessionStart(_c) {
185
- return true;
186
- }
187
- async onSessionStart(_c) { }
188
- async onSessionEnd(_c) { }
189
- async onInterrupt(_c) { }
190
- async onUserTranscript(_t, _f, _c) { }
191
- async onModelTranscript(_t, _f, _c) { }
192
- /** Imperative end-session helper for subclasses (e.g. timeout, moderation). */
193
- forceEndSession(c) {
194
- void this.#teardown(c);
195
- }
196
- // ─── Internals ───────────────────────────────────────────────────────────
197
- #resolveModel() {
198
- if (this.realtimeModel)
199
- return this.realtimeModel;
200
- if (this.createRealtimeModel) {
201
- const { env } = voiceHost(this);
202
- return this.createRealtimeModel(env);
203
- }
204
- throw new Error("withRealtimeVoice: subclass must provide `realtimeModel` or override `createRealtimeModel(env)`.");
205
- }
206
- #resolveDefaultAgent() {
207
- const host = voiceHost(this);
208
- const agents = host.getAgents?.() ?? [];
209
- const defaultId = host.getDefaultAgentId?.() ?? agents[0]?.id;
210
- const agent = agents.find((a) => a.id === defaultId) ?? agents[0];
211
- if (!agent) {
212
- throw new Error("withRealtimeVoice: subclass must implement getAgents()/getDefaultAgentId() returning at least one agent.");
213
- }
214
- return agent;
215
- }
216
- #buildRuntime(sessionId) {
217
- const host = voiceHost(this);
218
- const defaultAgentId = host.getDefaultAgentId?.() ?? "";
219
- const extra = host.getRuntimeConfig?.() ?? {};
220
- const agents = host.getAgents?.() ?? [];
221
- const sql = resolveSql(this);
222
- // BridgeSessionStore is the CF-native SessionStore — messages live in
223
- // AIChatAgent's `cf_ai_chat_agent_messages`, orchestration state in the
224
- // `kuralle_orchestration` row. No voice-specific tables.
225
- const sessionStore = sql
226
- ? new BridgeSessionStore({
227
- sqlExecutor: sql,
228
- cfMessages: host.messages ?? [],
229
- sessionId,
230
- defaultAgentId,
231
- })
232
- : undefined;
233
- return createRuntime({
234
- ...extra,
235
- agents,
236
- defaultAgentId,
237
- sessionStore,
238
- });
239
- }
240
- /**
241
- * Generate a **call-scoped** session id. Each `start_call` gets its own
242
- * UUID so orchestration state (currentAgent, flow node, handoffHistory)
243
- * is isolated per call — the 2nd call doesn't inherit the 1st call's
244
- * flow-node position. DO-scoped state that survives across calls
245
- * (Gemini resumption handle, long-term memory) belongs in `this.state`.
246
- *
247
- * Prior version returned `ctx.id.toString()` which collapsed every
248
- * call in a DO into one orchestration row — fine for demo but wrong
249
- * for any agent with flow transitions.
250
- */
251
- #mintCallId() {
252
- return crypto.randomUUID();
253
- }
254
- /** DO-stable identity, distinct from per-call session id. */
255
- #getDoId() {
256
- const id = voiceHost(this).ctx?.id;
257
- return id?.toString?.() ?? "unknown-do";
258
- }
259
- /**
260
- * Read a previously-captured provider resumption handle from DO-scoped
261
- * `this.state`. Returns undefined if missing, from a different provider,
262
- * or older than the provider's TTL (Gemini: 2h). DO-scoped means
263
- * `hibernate: true` + fresh wake + new call will still see it.
264
- */
265
- #readDoResumption(provider) {
266
- const state = (voiceHost(this).state ?? {});
267
- const r = state.voiceResumption;
268
- if (!r || r.provider !== provider)
269
- return undefined;
270
- const TWO_HOURS = 2 * 60 * 60 * 1000;
271
- if (Date.now() - r.issuedAt > TWO_HOURS)
272
- return undefined;
273
- return r.handle;
274
- }
275
- #writeDoResumption(handle, provider) {
276
- const host = voiceHost(this);
277
- if (typeof host.setState !== 'function')
278
- return;
279
- const prev = (host.state ?? {});
280
- host.setState({
281
- ...prev,
282
- voiceResumption: { handle, provider, issuedAt: Date.now() },
283
- });
284
- }
285
- async #handleStartCall(conn) {
286
- if (this.#adapters.has(conn.id) || this.#starting.has(conn.id)) {
287
- // Duplicate start_call on the same connection — ignore.
288
- return;
289
- }
290
- // Count in-flight starts toward the cap — otherwise concurrent start_call
291
- // frames from N distinct connections can all read size=0 before any has
292
- // populated the adapter map, over-subscribing the DO.
293
- if (this.#adapters.size + this.#starting.size >= maxConcurrent) {
294
- sendVoiceJSON(conn, {
295
- type: "error",
296
- code: "session_cap_exceeded",
297
- message: `Max concurrent realtime sessions (${maxConcurrent}) reached.`,
298
- });
299
- return;
300
- }
301
- this.#starting.add(conn.id);
302
- let client;
303
- try {
304
- client = this.#resolveModel();
305
- }
306
- catch (err) {
307
- this.#starting.delete(conn.id);
308
- sendVoiceJSON(conn, { type: "error", message: String(err) });
309
- return;
310
- }
311
- const canStart = await this.beforeSessionStart(conn);
312
- if (!canStart) {
313
- this.#starting.delete(conn.id);
314
- return;
315
- }
316
- const agent = this.#resolveDefaultAgent();
317
- this.#agents.set(conn.id, agent);
318
- // Keep the DO alive for the duration of the call (matches @cloudflare/voice).
319
- const { keepAlive } = voiceHost(this);
320
- if (keepAlive) {
321
- try {
322
- const dispose = await keepAlive.call(this);
323
- this.#keepAlive.set(conn.id, dispose);
324
- }
325
- catch (err) {
326
- // Host exposes keepAlive but it threw — that's a real bug, surface it.
327
- console.error("[withRealtimeVoice] keepAlive() threw — call will not be protected " +
328
- "from idle eviction (70-140s):", err instanceof Error ? err.message : String(err));
329
- }
330
- }
331
- else if (!this.#keepAliveWarned) {
332
- // Host lacks `keepAlive()` entirely — this is an Kuralle usage error.
333
- // The SDK-shipped Agent / AIChatAgent classes both provide it; a
334
- // missing implementation means the mixin was applied to a custom base
335
- // that didn't extend those. Without keepAlive, calls die at ~70-140s
336
- // of audio-flow idle (VAD silence during a turn).
337
- this.#keepAliveWarned = true;
338
- console.warn("[withRealtimeVoice] host does not expose `keepAlive()` — realtime " +
339
- "calls will be killed by DO idle eviction at ~70-140s. Extend " +
340
- "`KuralleRealtimeVoiceAgent` (which inherits from `AIChatAgent`) " +
341
- "rather than applying the mixin to a custom base.");
342
- }
343
- this.#cm.initConnection(conn.id);
344
- // Per-call session id. DO identity (`this.ctx.id`) gives scalability
345
- // across users (distinct `name` on the client → distinct DO); call id
346
- // gives isolation between successive calls within the same DO.
347
- const sessionId = this.#mintCallId();
348
- const runtime = this.#buildRuntime(sessionId);
349
- // DO-scoped resumption handle (if the same user previously called and
350
- // their Gemini session handle is still within the 2h TTL, pass it in
351
- // so the provider resumes server-side state — audio/text context).
352
- const initialResumptionHandle = this.#readDoResumption(client.provider);
353
- const adapter = new CloudflareRealtimeAdapter({
354
- runtime,
355
- client,
356
- sessionId,
357
- agentId: agent.id,
358
- modelPolicy: inferModelPolicy(client),
359
- reconnectMaxAttempts,
360
- reconnectBaseDelayMs,
361
- reconnectCapDelayMs,
362
- reconnectAudioBufferBytes,
363
- initialResumptionHandle,
364
- sendJson: (frame) => sendVoiceJSON(conn, frame),
365
- sendBinary: (data) => conn.send(data),
366
- onResumptionHandle: (handle, meta) => {
367
- // Persist DO-scoped (`this.state`), not call-scoped. Survives the
368
- // end of this call and is available to the next `start_call` on
369
- // the same DO.
370
- this.#writeDoResumption(handle, meta.provider);
371
- },
372
- onReconnecting: () => {
373
- sendVoiceJSON(conn, { type: "status", status: "connecting" });
374
- },
375
- onReconnected: () => {
376
- sendVoiceJSON(conn, { type: "status", status: "listening" });
377
- },
378
- onEnd: () => {
379
- void this.#teardown(conn);
380
- },
381
- onError: (err) => {
382
- console.error("[withRealtimeVoice] adapter error:", err.message);
383
- },
384
- });
385
- this.#adapters.set(conn.id, adapter);
386
- try {
387
- await adapter.start();
388
- }
389
- catch (err) {
390
- this.#adapters.delete(conn.id);
391
- this.#starting.delete(conn.id);
392
- throw err;
393
- }
394
- this.#starting.delete(conn.id);
395
- sendVoiceJSON(conn, { type: "audio_config", format: "pcm16", sampleRate: 24000 });
396
- sendVoiceJSON(conn, { type: "status", status: "listening" });
397
- // Wrap adapter transcript frames so subclass hooks fire alongside UI events.
398
- // The adapter already sent `{type:"transcript", role, text}` via sendJson;
399
- // we surface to the hook here. isFinal is always true in the current
400
- // RealtimeAudioClient transcript stream (every emission is cumulative).
401
- const userHook = this.onUserTranscript.bind(this);
402
- const modelHook = this.onModelTranscript.bind(this);
403
- const clientForHooks = client;
404
- clientForHooks.on("transcript", (text, role) => {
405
- sendVoiceJSON(conn, { type: "transcript", role, text });
406
- const fn = role === "user" ? userHook : modelHook;
407
- void fn(text, true, conn);
408
- });
409
- await this.onSessionStart(conn);
410
- }
411
- async #handleInterrupt(conn) {
412
- this.#adapters.get(conn.id)?.sendInterrupt();
413
- sendVoiceJSON(conn, { type: "status", status: "listening" });
414
- await this.onInterrupt(conn);
415
- }
416
- async #teardown(conn) {
417
- this.#starting.delete(conn.id);
418
- this.#sessionLossSignalled.delete(conn.id);
419
- const adapter = this.#adapters.get(conn.id);
420
- if (adapter) {
421
- this.#adapters.delete(conn.id);
422
- try {
423
- await adapter.stop("teardown");
424
- }
425
- catch {
426
- /* already closed */
427
- }
428
- }
429
- this.#agents.delete(conn.id);
430
- this.#cm.cleanup(conn.id);
431
- const dispose = this.#keepAlive.get(conn.id);
432
- if (dispose) {
433
- this.#keepAlive.delete(conn.id);
434
- try {
435
- dispose();
436
- }
437
- catch {
438
- /* ignore */
439
- }
440
- }
441
- sendVoiceJSON(conn, { type: "status", status: "idle" });
442
- try {
443
- await this.onSessionEnd(conn);
444
- }
445
- catch {
446
- /* ignore */
447
- }
448
- }
449
- }
450
- return RealtimeVoiceMixin;
451
- }