@kuralle-syrinx/cf-agents 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/package.json +9 -9
- package/src/build-session.ts +23 -5
- package/src/with-voice.ts +44 -9
- package/src/build-session.test.ts +0 -83
- package/src/connection-socket.test.ts +0 -114
- package/src/r2-recorder.test.ts +0 -185
- package/src/with-voice.test.ts +0 -695
- package/tsconfig.json +0 -22
package/README.md
CHANGED
|
@@ -71,9 +71,11 @@ comes with —
|
|
|
71
71
|
`{ response_text, require_repeat_verbatim: true, render? }` so it repeats facts faithfully
|
|
72
72
|
instead of paraphrasing. Configure per pipeline: `toolResultFormat: "envelope" | "string"`,
|
|
73
73
|
`renderDirective: "translate_faithfully"`.
|
|
74
|
-
- **Delegate observability (G2).** `onDelegateQuery` / `onDelegateResult` hooks
|
|
75
|
-
reasoner run with the query, answer, `durationMs`, and `grounded` — log or persist
|
|
76
|
-
without wrapping the `Reasoner`.
|
|
74
|
+
- **Delegate observability and client messaging (G2).** `onDelegateQuery` / `onDelegateResult` hooks
|
|
75
|
+
fire around every reasoner run with the query, answer, `durationMs`, and `grounded` — log or persist
|
|
76
|
+
the Q&A pair without wrapping the `Reasoner`. `onDelegateResult` also carries the originating live
|
|
77
|
+
`connection`, so a consumer can call `connection.send(...)` for a post-result app message without a
|
|
78
|
+
session-to-connection registry.
|
|
77
79
|
- **Typed "thinking" cues (G3).** Clients automatically receive `tool_call_started` /
|
|
78
80
|
`tool_call_delayed` (after `delayCueAfterMs`) / `tool_call_complete` / `tool_call_failed` wire
|
|
79
81
|
messages around the reasoner-latency window — key earcons/indicators on these instead of
|
|
@@ -94,7 +96,7 @@ comes with —
|
|
|
94
96
|
| `reasoner` | `(env, ctx) => Reasoner` (ctx: `{ sessionId, resume? }`). Defaults to `fromKuralleRuntime(this.runtime)` when the Agent exposes a kuralle `runtime`. Required for cascaded agents without one. |
|
|
95
97
|
| `recorder` | `(env, { sessionId }) => EdgeRecorder \| undefined` — optional per-call recorder (e.g. the R2 recorder at `@kuralle-syrinx/cf-agents/r2-recorder`). Edge transport. |
|
|
96
98
|
| `onToolCallStart` | `(ctx: { toolName, args, sessionId, connection }) => void \| Promise<void>` — fired the instant the front model invokes the delegate tool, **before** the reasoner runs — for app-specific cues beyond the standard `tool_call_*` wire messages. A throwing callback never affects the call. |
|
|
97
|
-
| `onDelegateQuery` / `onDelegateResult` | G2
|
|
99
|
+
| `onDelegateQuery` / `onDelegateResult` | G2 hooks around the reasoner run. `onDelegateResult` is self-contained (`{ query, answer, durationMs, grounded, toolId?, toolName?, turnId, sessionId, connection }`) — log/persist the grounded Q&A pair or use `connection.send(...)` to message the originating client. Throwing never affects the call. |
|
|
98
100
|
| `durableHistory` | G4 durable session state over the Agent's SQLite (default `true`). Set `false` for ephemeral pre-G4 behavior. |
|
|
99
101
|
| `delayCueAfterMs` | G3: ms before a pending tool call fires the `tool_call_delayed` ("still working") cue. 0 disables. Default 2000. |
|
|
100
102
|
| `backgroundAudio` | `{ ambient?, thinking?, duckWhileSpeaking? }` — looped ambient bed + thinking loop (raw mono PCM16 sources), mixed (ducked) under assistant speech; on the `"twilio"` transport the bed also fills between-turn gaps as comfort noise. Thinking follows the G3 cues. |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/cf-agents",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.4.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "withVoice(Agent) — add a Syrinx realtime or cascaded voice pipeline to a Cloudflare agents SDK Agent",
|
|
6
6
|
"keywords": [
|
|
@@ -33,13 +33,13 @@
|
|
|
33
33
|
"./r2-recorder": "./src/r2-recorder.ts"
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@kuralle-syrinx/kuralle": "4.
|
|
37
|
-
"@kuralle-syrinx/
|
|
38
|
-
"@kuralle-syrinx/
|
|
39
|
-
"@kuralle-syrinx/aisdk": "4.
|
|
40
|
-
"@kuralle-syrinx/
|
|
41
|
-
"@kuralle-syrinx/
|
|
42
|
-
"@kuralle-syrinx/
|
|
36
|
+
"@kuralle-syrinx/kuralle": "4.4.0",
|
|
37
|
+
"@kuralle-syrinx/realtime": "4.4.0",
|
|
38
|
+
"@kuralle-syrinx/ws": "4.4.0",
|
|
39
|
+
"@kuralle-syrinx/aisdk": "4.4.0",
|
|
40
|
+
"@kuralle-syrinx/core": "4.4.0",
|
|
41
|
+
"@kuralle-syrinx/server-websocket": "4.4.0",
|
|
42
|
+
"@kuralle-syrinx/recorder": "4.4.0"
|
|
43
43
|
},
|
|
44
44
|
"peerDependencies": {
|
|
45
45
|
"agents": ">=0.14.0 <1.0.0"
|
|
@@ -48,7 +48,7 @@
|
|
|
48
48
|
"@cloudflare/workers-types": "^4.20260601.0",
|
|
49
49
|
"agents": "0.14.0",
|
|
50
50
|
"typescript": "^5.7.0",
|
|
51
|
-
"vitest": "^2.
|
|
51
|
+
"vitest": "^3.2.6"
|
|
52
52
|
},
|
|
53
53
|
"scripts": {
|
|
54
54
|
"typecheck": "tsc --noEmit",
|
package/src/build-session.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
4
|
VoiceAgentSession,
|
|
5
|
+
type IdleTimeoutConfig,
|
|
5
6
|
type Reasoner,
|
|
6
7
|
type ReasonerMessage,
|
|
7
8
|
type ReasonerSessionStore,
|
|
@@ -36,6 +37,12 @@ export interface VoiceSessionWiring {
|
|
|
36
37
|
readonly contextProvider?: () => readonly ReasonerMessage[];
|
|
37
38
|
/** G3: ms before a pending tool call fires its "delayed" (still-working) cue. */
|
|
38
39
|
readonly delayCueAfterMs?: number;
|
|
40
|
+
/**
|
|
41
|
+
* Idle-timeout override. Defaults to the 15s telephony re-engagement. Browser/edge
|
|
42
|
+
* hosts pass `{ durationMs: 0 }` to disable it — a playground/demo user granting mic
|
|
43
|
+
* or reading the UI should never be nagged ("Are you still there?") or disconnected.
|
|
44
|
+
*/
|
|
45
|
+
readonly idleTimeout?: Partial<IdleTimeoutConfig>;
|
|
39
46
|
}
|
|
40
47
|
|
|
41
48
|
/**
|
|
@@ -72,17 +79,22 @@ export interface CascadedStage {
|
|
|
72
79
|
* `plugins: { stt, vad?, eos?, bridge, tts }`, reasoner wrapped in a
|
|
73
80
|
* `ReasoningBridge` registered as `"bridge"`.
|
|
74
81
|
*/
|
|
82
|
+
export type CascadedEndpointingOwner = "provider_stt" | "smart_turn";
|
|
83
|
+
|
|
75
84
|
export interface CascadedPipeline<Env> {
|
|
76
85
|
readonly kind: "cascaded";
|
|
77
86
|
readonly stt: (env: Env, ctx: VoicePipelineContext) => CascadedStage;
|
|
78
87
|
readonly tts: (env: Env, ctx: VoicePipelineContext) => CascadedStage;
|
|
79
88
|
readonly vad?: (env: Env, ctx: VoicePipelineContext) => CascadedStage;
|
|
80
|
-
|
|
89
|
+
/** Optional EOS stage. Returning `undefined` is equivalent to omitting the stage. */
|
|
90
|
+
readonly eos?: (env: Env, ctx: VoicePipelineContext) => CascadedStage | undefined;
|
|
81
91
|
/**
|
|
82
92
|
* Which component owns end-of-speech. @default "provider_stt". Set to
|
|
83
|
-
* "smart_turn" when supplying an `eos` stage.
|
|
93
|
+
* "smart_turn" when supplying an `eos` stage. May be a static literal or an
|
|
94
|
+
* `(env) => owner` factory for per-request selection (e.g. smart_turn only
|
|
95
|
+
* when Workers AI is bound).
|
|
84
96
|
*/
|
|
85
|
-
readonly endpointingOwner?:
|
|
97
|
+
readonly endpointingOwner?: CascadedEndpointingOwner | ((env: Env) => CascadedEndpointingOwner);
|
|
86
98
|
/**
|
|
87
99
|
* Fallback timeout (ms) before the engine force-finalizes a turn when the STT provider's own
|
|
88
100
|
* endpointing/finalize never fires. Maps to `VoiceAgentSession`'s `sttForceFinalizeTimeoutMs`
|
|
@@ -123,6 +135,7 @@ export function buildVoiceSession<Env>(
|
|
|
123
135
|
plugins: { realtime: {} },
|
|
124
136
|
endpointingOwner: "timer",
|
|
125
137
|
...(wiring.delayCueAfterMs !== undefined ? { delayCueAfterMs: wiring.delayCueAfterMs } : {}),
|
|
138
|
+
...(wiring.idleTimeout !== undefined ? { idleTimeout: wiring.idleTimeout } : {}),
|
|
126
139
|
});
|
|
127
140
|
session.registerPlugin("realtime", bridge);
|
|
128
141
|
return session;
|
|
@@ -139,8 +152,12 @@ export function buildVoiceSession<Env>(
|
|
|
139
152
|
const tts = pipeline.tts(env, ctx);
|
|
140
153
|
const vad = pipeline.vad?.(env, ctx);
|
|
141
154
|
const eos = pipeline.eos?.(env, ctx);
|
|
155
|
+
const endpointingOwner =
|
|
156
|
+
typeof pipeline.endpointingOwner === "function"
|
|
157
|
+
? pipeline.endpointingOwner(env)
|
|
158
|
+
: pipeline.endpointingOwner;
|
|
142
159
|
|
|
143
|
-
if (
|
|
160
|
+
if (endpointingOwner === "smart_turn" && !eos) {
|
|
144
161
|
throw new Error(
|
|
145
162
|
'withVoice: a cascaded pipeline with endpointingOwner "smart_turn" must provide an `eos` stage ' +
|
|
146
163
|
"(e.g. a PipecatEOSPlugin); otherwise no component owns end-of-speech and turns never complete.",
|
|
@@ -157,11 +174,12 @@ export function buildVoiceSession<Env>(
|
|
|
157
174
|
|
|
158
175
|
const session = new VoiceAgentSession({
|
|
159
176
|
plugins,
|
|
160
|
-
endpointingOwner:
|
|
177
|
+
endpointingOwner: endpointingOwner ?? "provider_stt",
|
|
161
178
|
...(pipeline.sttForceFinalizeTimeoutMs !== undefined
|
|
162
179
|
? { sttForceFinalizeTimeoutMs: pipeline.sttForceFinalizeTimeoutMs }
|
|
163
180
|
: {}),
|
|
164
181
|
...(wiring.delayCueAfterMs !== undefined ? { delayCueAfterMs: wiring.delayCueAfterMs } : {}),
|
|
182
|
+
...(wiring.idleTimeout !== undefined ? { idleTimeout: wiring.idleTimeout } : {}),
|
|
165
183
|
});
|
|
166
184
|
session.registerPlugin("stt", stt.plugin);
|
|
167
185
|
session.registerPlugin(
|
package/src/with-voice.ts
CHANGED
|
@@ -23,8 +23,9 @@ import {
|
|
|
23
23
|
type EdgeRecorder,
|
|
24
24
|
} from "@kuralle-syrinx/server-websocket/edge";
|
|
25
25
|
import { runTwilioEdgeWebSocketConnection } from "@kuralle-syrinx/server-websocket/edge-twilio";
|
|
26
|
+
import { runTelnyxEdgeWebSocketConnection } from "@kuralle-syrinx/server-websocket/edge-telnyx";
|
|
26
27
|
import { InMemorySessionStore } from "@kuralle-syrinx/server-websocket/session-store";
|
|
27
|
-
import type { Reasoner, ReasonerMessage } from "@kuralle-syrinx/core";
|
|
28
|
+
import type { IdleTimeoutConfig, Reasoner, ReasonerMessage } from "@kuralle-syrinx/core";
|
|
28
29
|
import { fromKuralleRuntime, type KuralleRuntimeLike } from "@kuralle-syrinx/kuralle";
|
|
29
30
|
import {
|
|
30
31
|
connectionManagedSocket,
|
|
@@ -84,7 +85,7 @@ export interface DelegateQueryContext<Env = unknown> {
|
|
|
84
85
|
|
|
85
86
|
/**
|
|
86
87
|
* Fired when the Reasoner produced the turn's final answer. Self-contained (carries the
|
|
87
|
-
* query again) so a consumer can log/persist the grounded Q&A pair
|
|
88
|
+
* query again) so a consumer can log/persist the grounded Q&A pair or message its client.
|
|
88
89
|
*/
|
|
89
90
|
export interface DelegateResultContext<Env = unknown> {
|
|
90
91
|
readonly query: string;
|
|
@@ -93,8 +94,11 @@ export interface DelegateResultContext<Env = unknown> {
|
|
|
93
94
|
readonly grounded: boolean;
|
|
94
95
|
readonly toolId?: string;
|
|
95
96
|
readonly toolName?: string;
|
|
97
|
+
readonly control?: { name: string; payload: unknown };
|
|
98
|
+
readonly blocked?: { userFacingMessage: string; payload?: unknown };
|
|
96
99
|
readonly turnId: string;
|
|
97
100
|
readonly sessionId: string;
|
|
101
|
+
/** The connection that initiated this delegate; use `send(...)` for an app message to that client. */
|
|
98
102
|
readonly connection: VoiceConnection;
|
|
99
103
|
/** The Worker env — bindings for logging/persistence (e.g. an R2 bucket). */
|
|
100
104
|
readonly env: Env;
|
|
@@ -105,10 +109,12 @@ export interface WithVoiceOptions<Env> {
|
|
|
105
109
|
* The connection wire protocol this host speaks. `"edge"` (default) is the Syrinx
|
|
106
110
|
* browser/edge JSON+envelope protocol (`runVoiceEdgeWebSocketConnection`). `"twilio"`
|
|
107
111
|
* speaks the Twilio Media Streams protocol (μ-law 8 kHz both ways,
|
|
108
|
-
* `runTwilioEdgeWebSocketConnection`) for a PSTN leg.
|
|
109
|
-
*
|
|
112
|
+
* `runTwilioEdgeWebSocketConnection`) for a PSTN leg. `"telnyx"` speaks the Telnyx
|
|
113
|
+
* Media Streaming protocol (PCMU/PCMA/G722/L16, `runTelnyxEdgeWebSocketConnection`).
|
|
114
|
+
* One transport per Agent class — route `/ws` to an `"edge"` agent, `/twilio` to a
|
|
115
|
+
* `"twilio"` agent, and `/telnyx` to a `"telnyx"` agent.
|
|
110
116
|
*/
|
|
111
|
-
readonly transport?: "edge" | "twilio";
|
|
117
|
+
readonly transport?: "edge" | "twilio" | "telnyx";
|
|
112
118
|
/** The voice pipeline: `{ kind: "realtime", ... }` or `{ kind: "cascaded", ... }`. */
|
|
113
119
|
readonly pipeline: VoicePipeline<Env>;
|
|
114
120
|
/**
|
|
@@ -140,8 +146,9 @@ export interface WithVoiceOptions<Env> {
|
|
|
140
146
|
readonly onDelegateQuery?: (ctx: DelegateQueryContext<Env>) => void | Promise<void>;
|
|
141
147
|
/**
|
|
142
148
|
* Delegate observability (G2): fired when the Reasoner produced the turn's final answer —
|
|
143
|
-
* the hook for logging/persisting the grounded Q&A pair
|
|
144
|
-
* grounded). Throwing here never affects
|
|
149
|
+
* the hook for logging/persisting the grounded Q&A pair or sending a post-result app message
|
|
150
|
+
* through `ctx.connection` (query + answer + durationMs + grounded). Throwing here never affects
|
|
151
|
+
* the call.
|
|
145
152
|
*/
|
|
146
153
|
readonly onDelegateResult?: (ctx: DelegateResultContext<Env>) => void | Promise<void>;
|
|
147
154
|
/**
|
|
@@ -158,12 +165,21 @@ export interface WithVoiceOptions<Env> {
|
|
|
158
165
|
* ("still working") wire cue fires. 0 disables the delayed phase. Default: 2000.
|
|
159
166
|
*/
|
|
160
167
|
readonly delayCueAfterMs?: number;
|
|
168
|
+
/**
|
|
169
|
+
* Idle-timeout override for this host. Omitted → the 15s telephony re-engagement
|
|
170
|
+
* ("Are you still there?" then disconnect). Browser/edge hosts should pass
|
|
171
|
+
* `{ durationMs: 0 }` to disable it: a demo/playground user granting mic or reading
|
|
172
|
+
* the UI must not be nagged or hung up on. Leave the default on `"twilio"` hosts.
|
|
173
|
+
*/
|
|
174
|
+
readonly idleTimeout?: Partial<IdleTimeoutConfig>;
|
|
161
175
|
readonly inputSampleRateHz?: number;
|
|
162
176
|
readonly outputSampleRateHz?: number;
|
|
163
177
|
readonly resumeWindowMs?: number;
|
|
164
178
|
/**
|
|
165
|
-
* Derive the Syrinx session id.
|
|
166
|
-
* the
|
|
179
|
+
* Derive the Syrinx session id. When supplied, this resolver is authoritative;
|
|
180
|
+
* otherwise the `?sessionId=` query param is used, then a per-connection random id.
|
|
181
|
+
* The Agent instance `name` is not used because concurrent connections to one DO
|
|
182
|
+
* must not silently share a session.
|
|
167
183
|
*/
|
|
168
184
|
readonly sessionId?: (request: Request, agentName: string) => string;
|
|
169
185
|
}
|
|
@@ -302,6 +318,7 @@ export function withVoice<Env, TBase extends AgentLike>(
|
|
|
302
318
|
};
|
|
303
319
|
const wiring: VoiceSessionWiring = {
|
|
304
320
|
...(options.delayCueAfterMs !== undefined ? { delayCueAfterMs: options.delayCueAfterMs } : {}),
|
|
321
|
+
...(options.idleTimeout !== undefined ? { idleTimeout: options.idleTimeout } : {}),
|
|
305
322
|
...(durable
|
|
306
323
|
? options.pipeline.kind === "realtime"
|
|
307
324
|
? { contextProvider: () => liveHistory }
|
|
@@ -397,6 +414,8 @@ export function withVoice<Env, TBase extends AgentLike>(
|
|
|
397
414
|
grounded: e.grounded,
|
|
398
415
|
toolId: e.toolId,
|
|
399
416
|
toolName: e.toolName,
|
|
417
|
+
control: e.control,
|
|
418
|
+
blocked: e.blocked,
|
|
400
419
|
turnId: e.turnId,
|
|
401
420
|
sessionId,
|
|
402
421
|
connection: voiceConnection,
|
|
@@ -430,6 +449,22 @@ export function withVoice<Env, TBase extends AgentLike>(
|
|
|
430
449
|
return;
|
|
431
450
|
}
|
|
432
451
|
|
|
452
|
+
if (options.transport === "telnyx") {
|
|
453
|
+
// Telnyx Media Streaming: PCMU/PCMA/G722/L16 per negotiated start.media_format.
|
|
454
|
+
// Same lease/heartbeat pattern as Twilio; recorder is edge-only.
|
|
455
|
+
// Live streaming_start → /telnyx is carrier-gated / unit-tested only.
|
|
456
|
+
void runTelnyxEdgeWebSocketConnection(socket, request, {
|
|
457
|
+
sessionStore: this.#store,
|
|
458
|
+
createSession,
|
|
459
|
+
...(options.inputSampleRateHz !== undefined
|
|
460
|
+
? { engineSampleRateHz: options.inputSampleRateHz }
|
|
461
|
+
: {}),
|
|
462
|
+
...(options.resumeWindowMs !== undefined ? { resumeWindowMs: options.resumeWindowMs } : {}),
|
|
463
|
+
...(options.backgroundAudio ? { backgroundAudio: options.backgroundAudio } : {}),
|
|
464
|
+
}).catch(onRunnerSettled);
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
|
|
433
468
|
let recorder: EdgeRecorder | undefined;
|
|
434
469
|
try {
|
|
435
470
|
recorder = options.recorder?.(env, { sessionId });
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { describe, it, expect } from "vitest";
|
|
4
|
-
import { VoiceAgentSession, type Reasoner, type VoicePlugin } from "@kuralle-syrinx/core";
|
|
5
|
-
import type { RealtimeAdapter } from "@kuralle-syrinx/realtime";
|
|
6
|
-
import { buildVoiceSession, type VoicePipeline } from "./build-session.js";
|
|
7
|
-
|
|
8
|
-
const stubPlugin = (): VoicePlugin => ({
|
|
9
|
-
initialize: async () => {},
|
|
10
|
-
close: async () => {},
|
|
11
|
-
});
|
|
12
|
-
|
|
13
|
-
const stubReasoner = (): Reasoner => ({
|
|
14
|
-
// eslint-disable-next-line require-yield
|
|
15
|
-
stream: async function* () {
|
|
16
|
-
return;
|
|
17
|
-
},
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
const stubFront = (): RealtimeAdapter => ({}) as unknown as RealtimeAdapter;
|
|
21
|
-
|
|
22
|
-
const ctx = { sessionId: "s1" };
|
|
23
|
-
|
|
24
|
-
describe("buildVoiceSession", () => {
|
|
25
|
-
it("builds a realtime session from a realtime pipeline", () => {
|
|
26
|
-
const pipeline: VoicePipeline<unknown> = {
|
|
27
|
-
kind: "realtime",
|
|
28
|
-
front: () => stubFront(),
|
|
29
|
-
delegateToolName: "consult_knowledge",
|
|
30
|
-
};
|
|
31
|
-
const session = buildVoiceSession(pipeline, {}, stubReasoner(), ctx);
|
|
32
|
-
expect(session).toBeInstanceOf(VoiceAgentSession);
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
it("allows a realtime session with no reasoner (front-only)", () => {
|
|
36
|
-
const pipeline: VoicePipeline<unknown> = { kind: "realtime", front: () => stubFront() };
|
|
37
|
-
const session = buildVoiceSession(pipeline, {}, undefined, ctx);
|
|
38
|
-
expect(session).toBeInstanceOf(VoiceAgentSession);
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
it("builds a cascaded session from a cascaded pipeline", () => {
|
|
42
|
-
const pipeline: VoicePipeline<unknown> = {
|
|
43
|
-
kind: "cascaded",
|
|
44
|
-
stt: () => ({ plugin: stubPlugin(), config: { model: "nova-3" } }),
|
|
45
|
-
tts: () => ({ plugin: stubPlugin(), config: { voice_id: "v" } }),
|
|
46
|
-
};
|
|
47
|
-
const session = buildVoiceSession(pipeline, {}, stubReasoner(), ctx);
|
|
48
|
-
expect(session).toBeInstanceOf(VoiceAgentSession);
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
it("carries sttForceFinalizeTimeoutMs through to the cascaded session", () => {
|
|
52
|
-
// Provider-endpointed cascades (e.g. Deepgram) tune this below the engine default; the mixin
|
|
53
|
-
// must thread it through instead of silently reverting to 7000ms.
|
|
54
|
-
const pipeline: VoicePipeline<unknown> = {
|
|
55
|
-
kind: "cascaded",
|
|
56
|
-
stt: () => ({ plugin: stubPlugin(), config: { model: "nova-3" } }),
|
|
57
|
-
tts: () => ({ plugin: stubPlugin(), config: { voice_id: "v" } }),
|
|
58
|
-
endpointingOwner: "provider_stt",
|
|
59
|
-
sttForceFinalizeTimeoutMs: 3500,
|
|
60
|
-
};
|
|
61
|
-
const session = buildVoiceSession(pipeline, {}, stubReasoner(), ctx);
|
|
62
|
-
expect(session).toBeInstanceOf(VoiceAgentSession);
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
it("throws a clear error when a cascaded pipeline has no reasoner", () => {
|
|
66
|
-
const pipeline: VoicePipeline<unknown> = {
|
|
67
|
-
kind: "cascaded",
|
|
68
|
-
stt: () => ({ plugin: stubPlugin() }),
|
|
69
|
-
tts: () => ({ plugin: stubPlugin() }),
|
|
70
|
-
};
|
|
71
|
-
expect(() => buildVoiceSession(pipeline, {}, undefined, ctx)).toThrow(/cascaded pipeline needs a reasoner/);
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
it('throws when endpointingOwner is "smart_turn" but no eos stage is provided', () => {
|
|
75
|
-
const pipeline: VoicePipeline<unknown> = {
|
|
76
|
-
kind: "cascaded",
|
|
77
|
-
stt: () => ({ plugin: stubPlugin() }),
|
|
78
|
-
tts: () => ({ plugin: stubPlugin() }),
|
|
79
|
-
endpointingOwner: "smart_turn",
|
|
80
|
-
};
|
|
81
|
-
expect(() => buildVoiceSession(pipeline, {}, stubReasoner(), ctx)).toThrow(/smart_turn/);
|
|
82
|
-
});
|
|
83
|
-
});
|
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { describe, it, expect, vi } from "vitest";
|
|
4
|
-
import { connectionManagedSocket, type VoiceConnection } from "./connection-socket.js";
|
|
5
|
-
|
|
6
|
-
function fakeConnection(readyState = 1): VoiceConnection & { sent: Array<string | ArrayBuffer | ArrayBufferView>; closed: boolean } {
|
|
7
|
-
const sent: Array<string | ArrayBuffer | ArrayBufferView> = [];
|
|
8
|
-
let closed = false;
|
|
9
|
-
return {
|
|
10
|
-
id: "c1",
|
|
11
|
-
get readyState() {
|
|
12
|
-
return closed ? 3 : readyState;
|
|
13
|
-
},
|
|
14
|
-
send(data) {
|
|
15
|
-
sent.push(data);
|
|
16
|
-
},
|
|
17
|
-
close() {
|
|
18
|
-
closed = true;
|
|
19
|
-
},
|
|
20
|
-
sent,
|
|
21
|
-
get closed() {
|
|
22
|
-
return closed;
|
|
23
|
-
},
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
describe("connectionManagedSocket", () => {
|
|
28
|
-
it("forwards send() to the connection and reflects readyState via isOpen", () => {
|
|
29
|
-
const conn = fakeConnection();
|
|
30
|
-
const { socket } = connectionManagedSocket(conn);
|
|
31
|
-
expect(socket.isOpen).toBe(true);
|
|
32
|
-
socket.send("hello");
|
|
33
|
-
socket.send(new Uint8Array([1, 2, 3]));
|
|
34
|
-
expect(conn.sent).toEqual(["hello", new Uint8Array([1, 2, 3])]);
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
it("isOpen is false when the connection is not OPEN", () => {
|
|
38
|
-
const conn = fakeConnection(0);
|
|
39
|
-
const { socket } = connectionManagedSocket(conn);
|
|
40
|
-
expect(socket.isOpen).toBe(false);
|
|
41
|
-
});
|
|
42
|
-
|
|
43
|
-
it("dispose() closes the underlying connection", () => {
|
|
44
|
-
const conn = fakeConnection();
|
|
45
|
-
const { socket } = connectionManagedSocket(conn);
|
|
46
|
-
socket.dispose();
|
|
47
|
-
expect(conn.closed).toBe(true);
|
|
48
|
-
expect(socket.isOpen).toBe(false);
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
it("controller.message pumps string frames as text and ArrayBuffer as binary", () => {
|
|
52
|
-
const { socket, controller } = connectionManagedSocket(fakeConnection());
|
|
53
|
-
const received: Array<{ data: unknown; isBinary: boolean }> = [];
|
|
54
|
-
socket.onMessage((data, isBinary) => received.push({ data, isBinary }));
|
|
55
|
-
|
|
56
|
-
controller.message("a-json-frame");
|
|
57
|
-
const buf = new Uint8Array([9, 8, 7]).buffer;
|
|
58
|
-
controller.message(buf);
|
|
59
|
-
|
|
60
|
-
expect(received).toHaveLength(2);
|
|
61
|
-
expect(received[0]).toEqual({ data: "a-json-frame", isBinary: false });
|
|
62
|
-
expect(received[1]?.isBinary).toBe(true);
|
|
63
|
-
expect(received[1]?.data).toBeInstanceOf(Uint8Array);
|
|
64
|
-
expect(Array.from(received[1]?.data as Uint8Array)).toEqual([9, 8, 7]);
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
it("controller.close and controller.error fan out to registered handlers", () => {
|
|
68
|
-
const { socket, controller } = connectionManagedSocket(fakeConnection());
|
|
69
|
-
const onClose = vi.fn();
|
|
70
|
-
const onError = vi.fn();
|
|
71
|
-
socket.onClose(onClose);
|
|
72
|
-
socket.onError(onError);
|
|
73
|
-
|
|
74
|
-
controller.close(1011, "boom");
|
|
75
|
-
controller.error(new Error("network"));
|
|
76
|
-
|
|
77
|
-
expect(onClose).toHaveBeenCalledWith(1011, "boom");
|
|
78
|
-
expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: "network" }));
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
it("dispose() fires the close handlers (so edge teardown runs) and closes the connection", () => {
|
|
82
|
-
const conn = fakeConnection();
|
|
83
|
-
const { socket } = connectionManagedSocket(conn);
|
|
84
|
-
const onClose = vi.fn();
|
|
85
|
-
socket.onClose(onClose);
|
|
86
|
-
|
|
87
|
-
socket.dispose();
|
|
88
|
-
|
|
89
|
-
expect(onClose).toHaveBeenCalledTimes(1);
|
|
90
|
-
expect(conn.closed).toBe(true);
|
|
91
|
-
expect(socket.isOpen).toBe(false);
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
it("fires close at most once across dispose() and controller.close()", () => {
|
|
95
|
-
const { socket, controller } = connectionManagedSocket(fakeConnection());
|
|
96
|
-
const onClose = vi.fn();
|
|
97
|
-
socket.onClose(onClose);
|
|
98
|
-
|
|
99
|
-
socket.dispose();
|
|
100
|
-
controller.close(1000, "later");
|
|
101
|
-
controller.close(1000, "again");
|
|
102
|
-
|
|
103
|
-
expect(onClose).toHaveBeenCalledTimes(1);
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
it("onOpen fires on the next microtask when already open", async () => {
|
|
107
|
-
const { socket } = connectionManagedSocket(fakeConnection());
|
|
108
|
-
const onOpen = vi.fn();
|
|
109
|
-
socket.onOpen(onOpen);
|
|
110
|
-
expect(onOpen).not.toHaveBeenCalled();
|
|
111
|
-
await Promise.resolve();
|
|
112
|
-
expect(onOpen).toHaveBeenCalledOnce();
|
|
113
|
-
});
|
|
114
|
-
});
|
package/src/r2-recorder.test.ts
DELETED
|
@@ -1,185 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { describe, expect, it } from "vitest";
|
|
4
|
-
import { R2EdgeRecorder } from "./r2-recorder.js";
|
|
5
|
-
|
|
6
|
-
interface PutCall {
|
|
7
|
-
key: string;
|
|
8
|
-
body: Uint8Array | string;
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
interface MpuRec {
|
|
12
|
-
key: string;
|
|
13
|
-
parts: { partNumber: number; size: number }[];
|
|
14
|
-
first?: Uint8Array; // retained bytes of part 1 (the header-bearing part)
|
|
15
|
-
completed: boolean;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
function fakeBucket() {
|
|
19
|
-
const puts: PutCall[] = [];
|
|
20
|
-
const mpus = new Map<string, MpuRec>();
|
|
21
|
-
let seq = 0;
|
|
22
|
-
|
|
23
|
-
function handle(uploadId: string) {
|
|
24
|
-
const rec = mpus.get(uploadId)!;
|
|
25
|
-
return {
|
|
26
|
-
key: rec.key,
|
|
27
|
-
uploadId,
|
|
28
|
-
async uploadPart(partNumber: number, value: Uint8Array) {
|
|
29
|
-
rec.parts.push({ partNumber, size: value.byteLength });
|
|
30
|
-
if (partNumber === 1) rec.first = value.slice();
|
|
31
|
-
return { partNumber, etag: `etag-${partNumber}` };
|
|
32
|
-
},
|
|
33
|
-
async complete(_parts: unknown) {
|
|
34
|
-
rec.completed = true;
|
|
35
|
-
return { key: rec.key } as unknown;
|
|
36
|
-
},
|
|
37
|
-
async abort() {
|
|
38
|
-
/* no-op */
|
|
39
|
-
},
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
const bucket = {
|
|
44
|
-
async put(key: string, body: Uint8Array | string) {
|
|
45
|
-
puts.push({ key, body: typeof body === "string" ? body : body.slice() });
|
|
46
|
-
return {} as unknown;
|
|
47
|
-
},
|
|
48
|
-
async createMultipartUpload(key: string) {
|
|
49
|
-
const uploadId = `u${++seq}`;
|
|
50
|
-
mpus.set(uploadId, { key, parts: [], completed: false });
|
|
51
|
-
return handle(uploadId);
|
|
52
|
-
},
|
|
53
|
-
resumeMultipartUpload(_key: string, uploadId: string) {
|
|
54
|
-
return handle(uploadId);
|
|
55
|
-
},
|
|
56
|
-
} as unknown as R2Bucket;
|
|
57
|
-
|
|
58
|
-
return { bucket, puts, mpus };
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function asString(body: Uint8Array | string): string {
|
|
62
|
-
return typeof body === "string" ? body : new TextDecoder().decode(body);
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
function wavChannels(body: Uint8Array | string): number {
|
|
66
|
-
const wav = body as Uint8Array;
|
|
67
|
-
return new DataView(wav.buffer, wav.byteOffset, wav.byteLength).getUint16(22, true);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
describe("R2EdgeRecorder", () => {
|
|
71
|
-
it("writes a stereo conversation.wav plus user/assistant stems and manifest", async () => {
|
|
72
|
-
const { bucket, puts } = fakeBucket();
|
|
73
|
-
const rec = new R2EdgeRecorder({ bucket, sessionId: "s1", startedAtMs: 1000, now: () => 1000 });
|
|
74
|
-
|
|
75
|
-
rec.onUserAudio("c1", new Uint8Array(640), 16000);
|
|
76
|
-
rec.onAssistantAudio("c1", new Uint8Array(320), 24000);
|
|
77
|
-
await rec.finalize({ sessionId: "s1", closedAtMs: 2000 });
|
|
78
|
-
|
|
79
|
-
const keys = puts.map((p) => p.key).sort();
|
|
80
|
-
expect(keys).toEqual([
|
|
81
|
-
"recordings/s1/1000/assistant.wav",
|
|
82
|
-
"recordings/s1/1000/conversation.wav",
|
|
83
|
-
"recordings/s1/1000/manifest.json",
|
|
84
|
-
"recordings/s1/1000/user.wav",
|
|
85
|
-
]);
|
|
86
|
-
|
|
87
|
-
const conversation = puts.find((p) => p.key.endsWith("conversation.wav"))!.body;
|
|
88
|
-
expect(asString((conversation as Uint8Array).subarray(0, 4))).toBe("RIFF");
|
|
89
|
-
expect(wavChannels(conversation)).toBe(2); // stereo: user L / assistant R
|
|
90
|
-
expect(wavChannels(puts.find((p) => p.key.endsWith("user.wav"))!.body)).toBe(1);
|
|
91
|
-
|
|
92
|
-
const manifest = JSON.parse(asString(puts.find((p) => p.key.endsWith("manifest.json"))!.body)) as {
|
|
93
|
-
conversation: { channels: number; omitted: boolean };
|
|
94
|
-
};
|
|
95
|
-
expect(manifest.conversation.channels).toBe(2);
|
|
96
|
-
expect(manifest.conversation.omitted).toBe(false);
|
|
97
|
-
});
|
|
98
|
-
|
|
99
|
-
it("time-aligns the assistant after the user instead of stacking at 0", async () => {
|
|
100
|
-
const { bucket, puts } = fakeBucket();
|
|
101
|
-
let now = 0;
|
|
102
|
-
const rec = new R2EdgeRecorder({ bucket, sessionId: "t", startedAtMs: 0, now: () => now });
|
|
103
|
-
|
|
104
|
-
now = 0;
|
|
105
|
-
rec.onUserAudio("c", new Uint8Array(640), 16000); // 20ms of user at t=0
|
|
106
|
-
now = 1000;
|
|
107
|
-
rec.onAssistantAudio("c", new Uint8Array(320), 16000); // assistant starts at t=1000ms
|
|
108
|
-
await rec.finalize({ sessionId: "t", closedAtMs: 1100 });
|
|
109
|
-
|
|
110
|
-
const manifest = JSON.parse(asString(puts.find((p) => p.key.endsWith("manifest.json"))!.body)) as {
|
|
111
|
-
conversation: { durationMs: number };
|
|
112
|
-
assistant: { durationMs: number };
|
|
113
|
-
};
|
|
114
|
-
// Assistant anchored at ~1000ms, so both the assistant stem and the merged
|
|
115
|
-
// conversation run ~1010ms — not the ~20ms they'd be if stacked at offset 0.
|
|
116
|
-
expect(manifest.assistant.durationMs).toBe(1010);
|
|
117
|
-
expect(manifest.conversation.durationMs).toBe(1010);
|
|
118
|
-
});
|
|
119
|
-
|
|
120
|
-
it("does not write anything when no audio was captured", async () => {
|
|
121
|
-
const { bucket, puts } = fakeBucket();
|
|
122
|
-
const rec = new R2EdgeRecorder({ bucket, sessionId: "empty", startedAtMs: 1 });
|
|
123
|
-
await rec.finalize({ sessionId: "empty", closedAtMs: 2 });
|
|
124
|
-
expect(puts).toHaveLength(0);
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
it("flags truncation past the per-stream cap instead of buffering unbounded", async () => {
|
|
128
|
-
const { bucket, puts } = fakeBucket();
|
|
129
|
-
const rec = new R2EdgeRecorder({ bucket, sessionId: "big", startedAtMs: 1, maxBytesPerStream: 1000, now: () => 1 });
|
|
130
|
-
rec.onUserAudio("c", new Uint8Array(800), 16000);
|
|
131
|
-
rec.onUserAudio("c", new Uint8Array(800), 16000); // would exceed 1000 -> dropped
|
|
132
|
-
await rec.finalize({ sessionId: "big", closedAtMs: 2 });
|
|
133
|
-
|
|
134
|
-
const manifest = JSON.parse(asString(puts.find((p) => p.key.endsWith("manifest.json"))!.body)) as {
|
|
135
|
-
user: { byteLength: number; truncated: boolean };
|
|
136
|
-
};
|
|
137
|
-
expect(manifest.user.byteLength).toBe(800);
|
|
138
|
-
expect(manifest.user.truncated).toBe(true);
|
|
139
|
-
});
|
|
140
|
-
|
|
141
|
-
it("streams a long, mostly-silent call to R2 in bounded parts instead of one giant buffer", async () => {
|
|
142
|
-
const { bucket, puts, mpus } = fakeBucket();
|
|
143
|
-
let now = 0;
|
|
144
|
-
const rec = new R2EdgeRecorder({ bucket, sessionId: "long", startedAtMs: 0, now: () => now });
|
|
145
|
-
|
|
146
|
-
// A 20-minute call: a blip of audio, a 20-minute wall-clock gap, then another blip.
|
|
147
|
-
// The old recorder would gap-fill the full ~38 MB wall-clock length in one allocation
|
|
148
|
-
// at finalize (OOMing the 128 MB DO). The stream must instead flush it as R2 parts.
|
|
149
|
-
now = 0;
|
|
150
|
-
rec.onUserAudio("c", new Uint8Array(320), 16000);
|
|
151
|
-
now = 20 * 60 * 1000; // +20 min
|
|
152
|
-
rec.onUserAudio("c", new Uint8Array(320), 16000);
|
|
153
|
-
await rec.finalize({ sessionId: "long", closedAtMs: now + 100 });
|
|
154
|
-
|
|
155
|
-
const expectedDataBytes = 20 * 60 * 16000 * 2 + 320; // wall-clock timeline + the final blip
|
|
156
|
-
|
|
157
|
-
// (1) The user stem streamed out incrementally: multiple parts, not one buffer.
|
|
158
|
-
const userMpu = [...mpus.values()].find((m) => m.key === "recordings/long/0/user.wav")!;
|
|
159
|
-
expect(userMpu).toBeDefined();
|
|
160
|
-
expect(userMpu.parts.length).toBeGreaterThan(1);
|
|
161
|
-
expect(userMpu.completed).toBe(true);
|
|
162
|
-
// Every part except the last carries at least R2's 5 MiB minimum (part 1 also holds the header).
|
|
163
|
-
const byNumber = [...userMpu.parts].sort((a, b) => a.partNumber - b.partNumber);
|
|
164
|
-
for (const p of byNumber.slice(0, -1)) expect(p.size).toBeGreaterThanOrEqual(5 * 1024 * 1024);
|
|
165
|
-
|
|
166
|
-
// (2) The stem decodes to the right duration: part 1's WAV header declares the full
|
|
167
|
-
// data length, and the parts sum to header(44) + that length.
|
|
168
|
-
const header = new DataView(userMpu.first!.buffer, userMpu.first!.byteOffset, userMpu.first!.byteLength);
|
|
169
|
-
expect(header.getUint32(40, true)).toBe(expectedDataBytes); // WAV data-chunk size field
|
|
170
|
-
const uploaded = userMpu.parts.reduce((n, p) => n + p.size, 0);
|
|
171
|
-
expect(uploaded).toBe(44 + expectedDataBytes);
|
|
172
|
-
|
|
173
|
-
// (3) finalize still writes the manifest; the stem length/duration match, and the
|
|
174
|
-
// stereo mix is flagged omitted (best-effort) once a stem has streamed out.
|
|
175
|
-
const manifest = JSON.parse(asString(puts.find((p) => p.key.endsWith("manifest.json"))!.body)) as {
|
|
176
|
-
user: { byteLength: number; durationMs: number; truncated: boolean };
|
|
177
|
-
conversation: { omitted: boolean };
|
|
178
|
-
};
|
|
179
|
-
expect(manifest.user.byteLength).toBe(expectedDataBytes);
|
|
180
|
-
expect(manifest.user.durationMs).toBe(Math.round((expectedDataBytes / 2 / 16000) * 1000));
|
|
181
|
-
expect(manifest.user.truncated).toBe(false);
|
|
182
|
-
expect(manifest.conversation.omitted).toBe(true);
|
|
183
|
-
expect(puts.some((p) => p.key.endsWith("conversation.wav"))).toBe(false);
|
|
184
|
-
});
|
|
185
|
-
});
|