@kuralle-syrinx/server-workers 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +34 -14
- package/src/live-session.ts +20 -6
- package/src/worker.ts +78 -0
- package/src/workers-ai-smart-turn.ts +49 -0
- package/wrangler.jsonc +8 -0
- package/wrangler.smartturn-spike.jsonc +44 -0
- package/src/live-realtime-session.test.ts +0 -88
- package/src/live-session.test.ts +0 -94
- package/src/worker-runtime.test.ts +0 -409
- package/tsconfig.json +0 -22
package/package.json
CHANGED
|
@@ -1,9 +1,28 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/server-workers",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.3.0",
|
|
4
4
|
"private": false,
|
|
5
|
-
"
|
|
5
|
+
"description": "Cloudflare Workers voice host for Syrinx — the full STT→LLM→TTS engine in one hibernatable Durable Object per session, with R2 call recording",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"voice",
|
|
8
|
+
"voice-agent",
|
|
9
|
+
"speech",
|
|
10
|
+
"syrinx",
|
|
11
|
+
"cloudflare-workers",
|
|
12
|
+
"durable-objects",
|
|
13
|
+
"edge"
|
|
14
|
+
],
|
|
6
15
|
"license": "MIT",
|
|
16
|
+
"homepage": "https://github.com/kuralle/syrinx#readme",
|
|
17
|
+
"repository": {
|
|
18
|
+
"type": "git",
|
|
19
|
+
"url": "git+https://github.com/kuralle/syrinx.git",
|
|
20
|
+
"directory": "packages/server-workers"
|
|
21
|
+
},
|
|
22
|
+
"bugs": {
|
|
23
|
+
"url": "https://github.com/kuralle/syrinx/issues"
|
|
24
|
+
},
|
|
25
|
+
"type": "module",
|
|
7
26
|
"main": "./src/worker.ts",
|
|
8
27
|
"types": "./src/worker.ts",
|
|
9
28
|
"dependencies": {
|
|
@@ -15,23 +34,24 @@
|
|
|
15
34
|
"agents": "0.14.0",
|
|
16
35
|
"ai": "^6.0.0",
|
|
17
36
|
"zod": "^4.1.8",
|
|
18
|
-
"@kuralle-syrinx/core": "4.
|
|
19
|
-
"@kuralle-syrinx/aisdk": "4.
|
|
20
|
-
"@kuralle-syrinx/
|
|
21
|
-
"@kuralle-syrinx/
|
|
22
|
-
"@kuralle-syrinx/
|
|
23
|
-
"@kuralle-syrinx/
|
|
24
|
-
"@kuralle-syrinx/
|
|
25
|
-
"@kuralle-syrinx/kuralle": "4.
|
|
26
|
-
"@kuralle-syrinx/
|
|
27
|
-
"@kuralle-syrinx/
|
|
37
|
+
"@kuralle-syrinx/core": "4.3.0",
|
|
38
|
+
"@kuralle-syrinx/aisdk": "4.3.0",
|
|
39
|
+
"@kuralle-syrinx/deepgram": "4.3.0",
|
|
40
|
+
"@kuralle-syrinx/server-websocket": "4.3.0",
|
|
41
|
+
"@kuralle-syrinx/cf-agents": "4.3.0",
|
|
42
|
+
"@kuralle-syrinx/realtime": "4.3.0",
|
|
43
|
+
"@kuralle-syrinx/cartesia": "4.3.0",
|
|
44
|
+
"@kuralle-syrinx/kuralle": "4.3.0",
|
|
45
|
+
"@kuralle-syrinx/recorder": "4.3.0",
|
|
46
|
+
"@kuralle-syrinx/pipecat-smart-turn": "4.3.0",
|
|
47
|
+
"@kuralle-syrinx/ws": "4.3.0"
|
|
28
48
|
},
|
|
29
49
|
"devDependencies": {
|
|
30
50
|
"@cloudflare/workers-types": "^4.20260601.0",
|
|
31
|
-
"esbuild": "^0.
|
|
51
|
+
"esbuild": "^0.28.1",
|
|
32
52
|
"miniflare": "^4.20260601.0",
|
|
33
53
|
"typescript": "^5.7.0",
|
|
34
|
-
"vitest": "^2.
|
|
54
|
+
"vitest": "^3.2.6",
|
|
35
55
|
"wrangler": "^4.19.0"
|
|
36
56
|
},
|
|
37
57
|
"scripts": {
|
package/src/live-session.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
//
|
|
3
3
|
// Cascaded voice pipeline for the Cloudflare Workers host: real Deepgram STT
|
|
4
|
-
// (nova-3
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
4
|
+
// (nova-3) + kuralle (Vectorize RAG) Reasoner + Deepgram Aura TTS, all dialed
|
|
5
|
+
// over the Workers fetch-upgrade socket (createWorkersSocket) so no Node `ws`
|
|
6
|
+
// is pulled into the edge bundle. Default turn-taking is Deepgram endpointing
|
|
7
|
+
// (endpointingOwner: "provider_stt"); when `env.AI` is bound, hosted Workers AI
|
|
8
|
+
// Smart Turn owns EOS via the edge-safe PipecatEOSPlugin (`/eos` subpath, no ONNX).
|
|
9
9
|
//
|
|
10
10
|
// This is the pipeline/brain layer — the host (worker.ts) composes it via
|
|
11
11
|
// `withVoice(Agent)`; this module owns the plugin slots and the reasoner, not the
|
|
@@ -14,9 +14,11 @@
|
|
|
14
14
|
import type { Reasoner } from "@kuralle-syrinx/core";
|
|
15
15
|
import type { CascadedPipeline, VoicePipelineContext } from "@kuralle-syrinx/cf-agents";
|
|
16
16
|
import { DeepgramSTTPlugin, DeepgramTTSPlugin } from "@kuralle-syrinx/deepgram";
|
|
17
|
+
import { PipecatEOSPlugin } from "@kuralle-syrinx/pipecat-smart-turn/eos";
|
|
17
18
|
import { createWorkersSocket } from "@kuralle-syrinx/ws/workers";
|
|
18
19
|
import type { VectorizeIndex } from "@cloudflare/workers-types";
|
|
19
20
|
import { createRealtimeKuralleReasoner } from "./kuralle-realtime-agent.js";
|
|
21
|
+
import { WorkersAiSmartTurnPredictor, type Ai } from "./workers-ai-smart-turn.js";
|
|
20
22
|
|
|
21
23
|
/** Provider secrets + optional tuning, supplied as Workers env/secret bindings. */
|
|
22
24
|
export interface LiveSessionEnv {
|
|
@@ -24,6 +26,8 @@ export interface LiveSessionEnv {
|
|
|
24
26
|
readonly OPENAI_API_KEY?: string;
|
|
25
27
|
readonly OPENAI_MODEL?: string;
|
|
26
28
|
readonly VECTORIZE: VectorizeIndex;
|
|
29
|
+
/** Optional Cloudflare Workers AI binding for hosted Smart Turn. */
|
|
30
|
+
readonly AI?: Ai;
|
|
27
31
|
}
|
|
28
32
|
|
|
29
33
|
const DEFAULT_DEEPGRAM_TTS_MODEL = "aura-2-thalia-en";
|
|
@@ -77,7 +81,11 @@ export const liveCascadedPipeline: CascadedPipeline<LiveSessionEnv> = {
|
|
|
77
81
|
sample_rate: OUTPUT_SAMPLE_RATE_HZ,
|
|
78
82
|
},
|
|
79
83
|
}),
|
|
80
|
-
|
|
84
|
+
eos: (env) => {
|
|
85
|
+
const predictor = createWorkersAiSmartTurnPredictor(env);
|
|
86
|
+
return predictor ? { plugin: new PipecatEOSPlugin(predictor), config: {} } : undefined;
|
|
87
|
+
},
|
|
88
|
+
endpointingOwner: (env) => (env.AI ? "smart_turn" : "provider_stt"),
|
|
81
89
|
sttForceFinalizeTimeoutMs: 3500,
|
|
82
90
|
};
|
|
83
91
|
|
|
@@ -88,6 +96,12 @@ export async function createLiveReasoner(env: LiveSessionEnv, ctx: VoicePipeline
|
|
|
88
96
|
return createRealtimeKuralleReasoner(env, { sessionId: ctx.sessionId });
|
|
89
97
|
}
|
|
90
98
|
|
|
99
|
+
/** Factory: returns a hosted Smart Turn predictor when `env.AI` is present, otherwise undefined. */
|
|
100
|
+
export function createWorkersAiSmartTurnPredictor(env: LiveSessionEnv): WorkersAiSmartTurnPredictor | undefined {
|
|
101
|
+
if (!env.AI) return undefined;
|
|
102
|
+
return new WorkersAiSmartTurnPredictor(env.AI);
|
|
103
|
+
}
|
|
104
|
+
|
|
91
105
|
function requireKey(value: string | undefined, name: string): string {
|
|
92
106
|
const trimmed = value?.trim();
|
|
93
107
|
if (!trimmed) throw new Error(`${name} is required to start a live voice session`);
|
package/src/worker.ts
CHANGED
|
@@ -19,8 +19,11 @@ import {
|
|
|
19
19
|
export interface Env extends LiveSessionEnv {
|
|
20
20
|
VOICE_CONVERSATIONS: DurableObjectNamespace;
|
|
21
21
|
TWILIO_VOICE_CONVERSATIONS: DurableObjectNamespace;
|
|
22
|
+
TELNYX_VOICE_CONVERSATIONS: DurableObjectNamespace;
|
|
22
23
|
/** Optional: when bound, full call audio is recorded to this bucket. */
|
|
23
24
|
RECORDINGS?: R2Bucket;
|
|
25
|
+
/** Optional: Cloudflare Workers AI binding for hosted Smart Turn endpointing. */
|
|
26
|
+
AI?: import("./workers-ai-smart-turn.js").Ai;
|
|
24
27
|
}
|
|
25
28
|
|
|
26
29
|
const INPUT_SAMPLE_RATE_HZ = 16000;
|
|
@@ -37,6 +40,11 @@ export class VoiceConversation extends withVoice<Env, typeof Agent<Env>>(Agent<E
|
|
|
37
40
|
inputSampleRateHz: INPUT_SAMPLE_RATE_HZ,
|
|
38
41
|
outputSampleRateHz: OUTPUT_SAMPLE_RATE_HZ,
|
|
39
42
|
resumeWindowMs: 15_000,
|
|
43
|
+
// Browser/studio clients: no telephony-style idle re-engagement. The 15s default nags
|
|
44
|
+
// ("Are you still there?") and disconnects a user who is granting mic or reading the UI
|
|
45
|
+
// before speaking; the DO hibernates abandoned browser sessions anyway. Telephony keeps
|
|
46
|
+
// the default on TwilioVoiceConversation below.
|
|
47
|
+
idleTimeout: { durationMs: 0 },
|
|
40
48
|
}) {}
|
|
41
49
|
|
|
42
50
|
/** Telephony cascaded host (Twilio Media Streams μ-law 8 kHz over /twilio). Same pipeline/brain. */
|
|
@@ -48,6 +56,19 @@ export class TwilioVoiceConversation extends withVoice<Env, typeof Agent<Env>>(A
|
|
|
48
56
|
resumeWindowMs: 15_000,
|
|
49
57
|
}) {}
|
|
50
58
|
|
|
59
|
+
/**
|
|
60
|
+
* Telephony cascaded host (Telnyx Media Streaming over /telnyx). Same pipeline/brain;
|
|
61
|
+
* idle timeout keeps the telephony re-engagement default. Live streaming_start →
|
|
62
|
+
* /telnyx is carrier-gated / unit-tested only.
|
|
63
|
+
*/
|
|
64
|
+
export class TelnyxVoiceConversation extends withVoice<Env, typeof Agent<Env>>(Agent<Env>, {
|
|
65
|
+
transport: "telnyx",
|
|
66
|
+
pipeline: liveCascadedPipeline,
|
|
67
|
+
reasoner: (env, ctx) => createLiveReasoner(env, ctx),
|
|
68
|
+
inputSampleRateHz: INPUT_SAMPLE_RATE_HZ,
|
|
69
|
+
resumeWindowMs: 15_000,
|
|
70
|
+
}) {}
|
|
71
|
+
|
|
51
72
|
export default {
|
|
52
73
|
async fetch(request: Request, env: Env): Promise<Response> {
|
|
53
74
|
const url = new URL(request.url);
|
|
@@ -57,6 +78,9 @@ export default {
|
|
|
57
78
|
// worker's own /twilio Media Streams endpoint over a bidirectional <Stream>. Point a
|
|
58
79
|
// Twilio number's "A call comes in" webhook at https://<host>/incoming-call.
|
|
59
80
|
if (url.pathname === "/incoming-call") return await twilioIncomingCallResponse(request, url);
|
|
81
|
+
// Telnyx TeXML / Call Control stream-start helper: pure payload construction.
|
|
82
|
+
// Live dispatch of streaming_start to a real trunk is carrier-gated.
|
|
83
|
+
if (url.pathname === "/telnyx-stream-start") return await telnyxStreamStartResponse(request, url);
|
|
60
84
|
// Name-addressed routing: one DO per sessionId (callSid for telephony). The Agent
|
|
61
85
|
// (partyserver) resolves its name from ctx.id.name, so a direct stub.fetch() upgrade
|
|
62
86
|
// is valid for both transports.
|
|
@@ -65,6 +89,10 @@ export default {
|
|
|
65
89
|
const id = env.TWILIO_VOICE_CONVERSATIONS.idFromName(sessionId);
|
|
66
90
|
return await env.TWILIO_VOICE_CONVERSATIONS.get(id).fetch(request);
|
|
67
91
|
}
|
|
92
|
+
if (url.pathname === "/telnyx") {
|
|
93
|
+
const id = env.TELNYX_VOICE_CONVERSATIONS.idFromName(sessionId);
|
|
94
|
+
return await env.TELNYX_VOICE_CONVERSATIONS.get(id).fetch(request);
|
|
95
|
+
}
|
|
68
96
|
if (url.pathname !== "/ws") return new Response("not found", { status: 404 });
|
|
69
97
|
const id = env.VOICE_CONVERSATIONS.idFromName(sessionId);
|
|
70
98
|
return await env.VOICE_CONVERSATIONS.get(id).fetch(request);
|
|
@@ -97,6 +125,56 @@ async function twilioIncomingCallResponse(request: Request, url: URL): Promise<R
|
|
|
97
125
|
return new Response(twiml, { headers: { "content-type": "text/xml; charset=utf-8" } });
|
|
98
126
|
}
|
|
99
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Telnyx stream-start helper — pure payload construction (no live carrier dispatch).
|
|
130
|
+
*
|
|
131
|
+
* - Default (`format=texml` or omitted): TeXML `<Stream url="wss://…/telnyx?sessionId=…">`
|
|
132
|
+
* for TeXML applications that start bidirectional media via the Stream verb.
|
|
133
|
+
* - `format=streaming_start`: JSON body matching Call Control `streaming_start`
|
|
134
|
+
* (`stream_url`, `stream_bidirectional_mode`, `stream_bidirectional_codec`, …).
|
|
135
|
+
* Live POST to `/v2/calls/{call_control_id}/actions/streaming_start` is carrier-gated.
|
|
136
|
+
*
|
|
137
|
+
* Query: `sessionId` (optional), `codec` (PCMU|PCMA|G722|L16, default PCMU),
|
|
138
|
+
* `sampleRate` (default from codec), `format` (texml|streaming_start).
|
|
139
|
+
*/
|
|
140
|
+
async function telnyxStreamStartResponse(request: Request, url: URL): Promise<Response> {
|
|
141
|
+
let sessionId = url.searchParams.get("sessionId")?.trim() ?? "";
|
|
142
|
+
if (!sessionId && request.method === "POST") {
|
|
143
|
+
const form = await request.formData().catch(() => null);
|
|
144
|
+
const value = form?.get("sessionId") ?? form?.get("CallControlId") ?? form?.get("call_control_id");
|
|
145
|
+
if (typeof value === "string") sessionId = value.trim();
|
|
146
|
+
}
|
|
147
|
+
if (!sessionId) sessionId = crypto.randomUUID();
|
|
148
|
+
|
|
149
|
+
const codec = (url.searchParams.get("codec")?.trim().toUpperCase() || "PCMU");
|
|
150
|
+
const defaultRate = codec === "L16" || codec === "G722" ? "16000" : "8000";
|
|
151
|
+
const sampleRate = url.searchParams.get("sampleRate")?.trim() || defaultRate;
|
|
152
|
+
const format = url.searchParams.get("format")?.trim().toLowerCase() || "texml";
|
|
153
|
+
|
|
154
|
+
const wsScheme = url.protocol === "https:" ? "wss" : "ws";
|
|
155
|
+
const streamUrl = `${wsScheme}://${url.host}/telnyx?sessionId=${encodeURIComponent(sessionId)}`;
|
|
156
|
+
|
|
157
|
+
if (format === "streaming_start") {
|
|
158
|
+
// Call Control streaming_start body — pure; live dispatch carrier-gated.
|
|
159
|
+
return Response.json({
|
|
160
|
+
stream_url: streamUrl,
|
|
161
|
+
stream_track: "inbound_track",
|
|
162
|
+
stream_bidirectional_mode: "rtp",
|
|
163
|
+
stream_bidirectional_codec: codec,
|
|
164
|
+
stream_bidirectional_sampling_rate: Number(sampleRate),
|
|
165
|
+
send_silence_when_idle: true,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const texml = [
|
|
170
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
171
|
+
"<Response>",
|
|
172
|
+
` <Stream url="${xmlEscape(streamUrl)}" bidirectionalMode="rtp" bidirectionalCodec="${xmlEscape(codec)}" bidirectionalSamplingRate="${xmlEscape(sampleRate)}" />`,
|
|
173
|
+
"</Response>",
|
|
174
|
+
].join("\n");
|
|
175
|
+
return new Response(texml, { headers: { "content-type": "text/xml; charset=utf-8" } });
|
|
176
|
+
}
|
|
177
|
+
|
|
100
178
|
function xmlEscape(value: string): string {
|
|
101
179
|
return value.replace(/[<>&"']/g, (c) =>
|
|
102
180
|
({ "<": "<", ">": ">", "&": "&", '"': """, "'": "'" })[c] ?? c,
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// SPDX-License-Identifier: MIT
|
|
2
|
+
|
|
3
|
+
import type { SmartTurnPredictor } from "@kuralle-syrinx/pipecat-smart-turn/eos";
|
|
4
|
+
import type { PluginConfig } from "@kuralle-syrinx/core";
|
|
5
|
+
|
|
6
|
+
/** Minimal Cloudflare Workers AI binding surface for the Smart Turn v2 model. */
|
|
7
|
+
export interface Ai {
|
|
8
|
+
run(model: string, input: unknown): Promise<{
|
|
9
|
+
is_complete?: boolean;
|
|
10
|
+
probability?: number;
|
|
11
|
+
}>;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const MODEL_ID = "@cf/pipecat-ai/smart-turn-v2";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Cloudflare Workers AI-hosted Smart Turn predictor.
|
|
18
|
+
*
|
|
19
|
+
* Uses the hosted `@cf/pipecat-ai/smart-turn-v2` model (v3 is NOT available in the
|
|
20
|
+
* Workers AI catalog). No ONNX is loaded locally — the model runs in Cloudflare's
|
|
21
|
+
* inference cluster, so `initialize`/`close` are no-ops.
|
|
22
|
+
*/
|
|
23
|
+
export class WorkersAiSmartTurnPredictor implements SmartTurnPredictor {
|
|
24
|
+
constructor(private readonly ai: Ai) {}
|
|
25
|
+
|
|
26
|
+
async initialize(_config: PluginConfig): Promise<void> {
|
|
27
|
+
// Hosted model — nothing to load.
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async predict(audio: Float32Array): Promise<number> {
|
|
31
|
+
const base64 = float32ArrayToBase64(audio);
|
|
32
|
+
const result = await this.ai.run(MODEL_ID, { audio: base64, dtype: "float32" });
|
|
33
|
+
return typeof result.probability === "number" ? result.probability : 0;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async close(): Promise<void> {
|
|
37
|
+
// Nothing to release.
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function float32ArrayToBase64(arr: Float32Array): string {
|
|
42
|
+
const bytes = new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength);
|
|
43
|
+
const chunks: string[] = [];
|
|
44
|
+
const chunkSize = 0x8000; // 32 KB — safe for String.fromCharCode spread
|
|
45
|
+
for (let i = 0; i < bytes.length; i += chunkSize) {
|
|
46
|
+
chunks.push(String.fromCharCode(...bytes.subarray(i, i + chunkSize)));
|
|
47
|
+
}
|
|
48
|
+
return btoa(chunks.join(""));
|
|
49
|
+
}
|
package/wrangler.jsonc
CHANGED
|
@@ -13,6 +13,10 @@
|
|
|
13
13
|
{
|
|
14
14
|
"name": "TWILIO_VOICE_CONVERSATIONS",
|
|
15
15
|
"class_name": "TwilioVoiceConversation"
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"name": "TELNYX_VOICE_CONVERSATIONS",
|
|
19
|
+
"class_name": "TelnyxVoiceConversation"
|
|
16
20
|
}
|
|
17
21
|
]
|
|
18
22
|
},
|
|
@@ -30,6 +34,10 @@
|
|
|
30
34
|
{
|
|
31
35
|
"tag": "v2",
|
|
32
36
|
"new_sqlite_classes": ["TwilioVoiceConversation"]
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"tag": "v3",
|
|
40
|
+
"new_sqlite_classes": ["TelnyxVoiceConversation"]
|
|
33
41
|
}
|
|
34
42
|
],
|
|
35
43
|
"vectorize": [
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "node_modules/wrangler/config-schema.json",
|
|
3
|
+
"name": "syrinx-voice-smartturn-spike",
|
|
4
|
+
"main": "src/worker.ts",
|
|
5
|
+
"compatibility_date": "2026-06-01",
|
|
6
|
+
"compatibility_flags": ["nodejs_compat"],
|
|
7
|
+
"durable_objects": {
|
|
8
|
+
"bindings": [
|
|
9
|
+
{
|
|
10
|
+
"name": "VOICE_CONVERSATIONS",
|
|
11
|
+
"class_name": "VoiceConversation"
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"name": "TWILIO_VOICE_CONVERSATIONS",
|
|
15
|
+
"class_name": "TwilioVoiceConversation"
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
},
|
|
19
|
+
"r2_buckets": [
|
|
20
|
+
{
|
|
21
|
+
"binding": "RECORDINGS",
|
|
22
|
+
"bucket_name": "syrinx-voice-recordings"
|
|
23
|
+
}
|
|
24
|
+
],
|
|
25
|
+
"migrations": [
|
|
26
|
+
{
|
|
27
|
+
"tag": "v1",
|
|
28
|
+
"new_sqlite_classes": ["VoiceConversation"]
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"tag": "v2",
|
|
32
|
+
"new_sqlite_classes": ["TwilioVoiceConversation"]
|
|
33
|
+
}
|
|
34
|
+
],
|
|
35
|
+
"vectorize": [
|
|
36
|
+
{
|
|
37
|
+
"binding": "VECTORIZE",
|
|
38
|
+
"index_name": "kuralle-university-kb"
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
"ai": {
|
|
42
|
+
"binding": "AI"
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { readFileSync, readdirSync } from "node:fs";
|
|
4
|
-
import path from "node:path";
|
|
5
|
-
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { describe, expect, it } from "vitest";
|
|
7
|
-
|
|
8
|
-
import {
|
|
9
|
-
hasRealtimeSessionCredentials,
|
|
10
|
-
realtimeVoicePipeline,
|
|
11
|
-
resolveRealtimeFront,
|
|
12
|
-
} from "./live-realtime-session.js";
|
|
13
|
-
|
|
14
|
-
const srcDir = path.dirname(fileURLToPath(import.meta.url));
|
|
15
|
-
|
|
16
|
-
const mockVectorize = {} as import("@cloudflare/workers-types").VectorizeIndex;
|
|
17
|
-
const ctx = { sessionId: "s1" };
|
|
18
|
-
|
|
19
|
-
describe("realtimeVoicePipeline", () => {
|
|
20
|
-
it("is a realtime pipeline routed through the ask_university delegate tool", () => {
|
|
21
|
-
expect(realtimeVoicePipeline.kind).toBe("realtime");
|
|
22
|
-
expect(realtimeVoicePipeline.delegateToolName).toBe("ask_university");
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
it("builds an OpenAI front by default and a Gemini front when REALTIME_FRONT=gemini", () => {
|
|
26
|
-
const openai = realtimeVoicePipeline.front({ OPENAI_API_KEY: "k", VECTORIZE: mockVectorize }, ctx);
|
|
27
|
-
expect(openai).toBeTruthy();
|
|
28
|
-
const gemini = realtimeVoicePipeline.front(
|
|
29
|
-
{ REALTIME_FRONT: "gemini", GEMINI_API_KEY: "g", VECTORIZE: mockVectorize },
|
|
30
|
-
ctx,
|
|
31
|
-
);
|
|
32
|
-
expect(gemini).toBeTruthy();
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
it("requires the front model's key (OPENAI_API_KEY default; GEMINI_API_KEY for gemini)", () => {
|
|
36
|
-
expect(() => realtimeVoicePipeline.front({ VECTORIZE: mockVectorize }, ctx)).toThrow(/OPENAI_API_KEY/);
|
|
37
|
-
expect(() =>
|
|
38
|
-
realtimeVoicePipeline.front({ REALTIME_FRONT: "gemini", VECTORIZE: mockVectorize }, ctx),
|
|
39
|
-
).toThrow(/GEMINI_API_KEY/);
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
it("reports credential presence via hasRealtimeSessionCredentials", () => {
|
|
43
|
-
expect(hasRealtimeSessionCredentials({ OPENAI_API_KEY: "k", VECTORIZE: mockVectorize })).toBe(true);
|
|
44
|
-
expect(hasRealtimeSessionCredentials({ VECTORIZE: mockVectorize })).toBe(false);
|
|
45
|
-
expect(hasRealtimeSessionCredentials({
|
|
46
|
-
REALTIME_FRONT: "gemini",
|
|
47
|
-
GEMINI_API_KEY: "g",
|
|
48
|
-
VECTORIZE: mockVectorize,
|
|
49
|
-
})).toBe(true);
|
|
50
|
-
expect(hasRealtimeSessionCredentials({
|
|
51
|
-
REALTIME_FRONT: "gemini",
|
|
52
|
-
VECTORIZE: mockVectorize,
|
|
53
|
-
})).toBe(false);
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
it("defaults REALTIME_FRONT to openai", () => {
|
|
57
|
-
expect(resolveRealtimeFront({ VECTORIZE: mockVectorize })).toBe("openai");
|
|
58
|
-
expect(resolveRealtimeFront({ REALTIME_FRONT: "gemini", VECTORIZE: mockVectorize })).toBe("gemini");
|
|
59
|
-
});
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
describe("realtime worker edge safety", () => {
|
|
63
|
-
it("live-realtime-session and worker-realtime stay free of Node-only primitives", () => {
|
|
64
|
-
const files = ["live-realtime-session.ts", "worker-realtime.ts", "kuralle-realtime-agent.ts"];
|
|
65
|
-
const banned = /\bBuffer\b|from "node:|process\./;
|
|
66
|
-
for (const file of files) {
|
|
67
|
-
const source = readFileSync(path.join(srcDir, file), "utf8");
|
|
68
|
-
for (const [index, line] of source.split("\n").entries()) {
|
|
69
|
-
expect(line, `${file}:${index + 1}`).not.toMatch(banned);
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
it("does not pull node: imports into the realtime worker src tree", () => {
|
|
75
|
-
const nodeImports = readdirSync(srcDir)
|
|
76
|
-
.filter((name) => name.endsWith(".ts") && !name.endsWith(".test.ts"))
|
|
77
|
-
.flatMap((name) => {
|
|
78
|
-
const source = readFileSync(path.join(srcDir, name), "utf8");
|
|
79
|
-
return source
|
|
80
|
-
.split("\n")
|
|
81
|
-
.filter((line) => line.includes('from "node:'))
|
|
82
|
-
.map((line) => `${name}: ${line.trim()}`);
|
|
83
|
-
});
|
|
84
|
-
expect(nodeImports.filter((entry) => entry.startsWith("live-realtime-session")
|
|
85
|
-
|| entry.startsWith("worker-realtime")
|
|
86
|
-
|| entry.startsWith("kuralle-realtime-agent"))).toEqual([]);
|
|
87
|
-
});
|
|
88
|
-
});
|
package/src/live-session.test.ts
DELETED
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { readFileSync, readdirSync } from "node:fs";
|
|
4
|
-
import path from "node:path";
|
|
5
|
-
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { describe, expect, it } from "vitest";
|
|
7
|
-
import { DeepgramSTTPlugin, DeepgramTTSPlugin } from "@kuralle-syrinx/deepgram";
|
|
8
|
-
|
|
9
|
-
import {
|
|
10
|
-
createLiveReasoner,
|
|
11
|
-
hasLiveSessionCredentials,
|
|
12
|
-
liveCascadedPipeline,
|
|
13
|
-
} from "./live-session.js";
|
|
14
|
-
|
|
15
|
-
const srcDir = path.dirname(fileURLToPath(import.meta.url));
|
|
16
|
-
const mockVectorize = {} as import("@cloudflare/workers-types").VectorizeIndex;
|
|
17
|
-
const ctx = { sessionId: "s1" };
|
|
18
|
-
|
|
19
|
-
describe("liveCascadedPipeline", () => {
|
|
20
|
-
it("is a provider-endpointed cascade with a tightened force-finalize timeout", () => {
|
|
21
|
-
expect(liveCascadedPipeline.kind).toBe("cascaded");
|
|
22
|
-
expect(liveCascadedPipeline.endpointingOwner).toBe("provider_stt");
|
|
23
|
-
expect(liveCascadedPipeline.sttForceFinalizeTimeoutMs).toBe(3500);
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
it("builds Deepgram Nova-3 STT (VAD events) and Aura TTS stages from the env", () => {
|
|
27
|
-
const env = { DEEPGRAM_API_KEY: "dg-key", OPENAI_API_KEY: "oa-key", VECTORIZE: mockVectorize };
|
|
28
|
-
const stt = liveCascadedPipeline.stt(env, ctx);
|
|
29
|
-
expect(stt.plugin).toBeInstanceOf(DeepgramSTTPlugin);
|
|
30
|
-
expect(stt.config).toMatchObject({ model: "nova-3", endpointing: 500, vad_events: true, utterance_end_ms: 1000, api_key: "dg-key" });
|
|
31
|
-
|
|
32
|
-
const tts = liveCascadedPipeline.tts(env, ctx);
|
|
33
|
-
expect(tts.plugin).toBeInstanceOf(DeepgramTTSPlugin);
|
|
34
|
-
expect(tts.config).toMatchObject({ model: "aura-2-thalia-en", api_key: "dg-key" });
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
it("requires DEEPGRAM_API_KEY to build the stt/tts stages", () => {
|
|
38
|
-
const env = { OPENAI_API_KEY: "k", VECTORIZE: mockVectorize };
|
|
39
|
-
expect(() => liveCascadedPipeline.stt(env, ctx)).toThrow(/DEEPGRAM_API_KEY/);
|
|
40
|
-
expect(() => liveCascadedPipeline.tts(env, ctx)).toThrow(/DEEPGRAM_API_KEY/);
|
|
41
|
-
});
|
|
42
|
-
});
|
|
43
|
-
|
|
44
|
-
describe("createLiveReasoner", () => {
|
|
45
|
-
it("requires OPENAI_API_KEY and VECTORIZE", async () => {
|
|
46
|
-
await expect(createLiveReasoner({ VECTORIZE: mockVectorize }, ctx)).rejects.toThrow(/OPENAI_API_KEY/);
|
|
47
|
-
await expect(
|
|
48
|
-
createLiveReasoner(
|
|
49
|
-
{ OPENAI_API_KEY: "k", VECTORIZE: undefined as unknown as typeof mockVectorize },
|
|
50
|
-
ctx,
|
|
51
|
-
),
|
|
52
|
-
).rejects.toThrow(/VECTORIZE/);
|
|
53
|
-
});
|
|
54
|
-
|
|
55
|
-
it("reports credential presence via hasLiveSessionCredentials", () => {
|
|
56
|
-
expect(hasLiveSessionCredentials({
|
|
57
|
-
DEEPGRAM_API_KEY: "d",
|
|
58
|
-
OPENAI_API_KEY: "o",
|
|
59
|
-
VECTORIZE: mockVectorize,
|
|
60
|
-
})).toBe(true);
|
|
61
|
-
expect(hasLiveSessionCredentials({
|
|
62
|
-
OPENAI_API_KEY: "o",
|
|
63
|
-
VECTORIZE: mockVectorize,
|
|
64
|
-
})).toBe(false);
|
|
65
|
-
});
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
describe("cascade worker edge safety", () => {
|
|
69
|
-
it("live-session and worker stay free of Node-only primitives", () => {
|
|
70
|
-
const files = ["live-session.ts", "worker.ts", "kuralle-realtime-agent.ts"];
|
|
71
|
-
const banned = /\bBuffer\b|from "node:|process\./;
|
|
72
|
-
for (const file of files) {
|
|
73
|
-
const source = readFileSync(path.join(srcDir, file), "utf8");
|
|
74
|
-
for (const [index, line] of source.split("\n").entries()) {
|
|
75
|
-
expect(line, `${file}:${index + 1}`).not.toMatch(banned);
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
});
|
|
79
|
-
|
|
80
|
-
it("does not pull node: imports into the cascade worker src tree", () => {
|
|
81
|
-
const nodeImports = readdirSync(srcDir)
|
|
82
|
-
.filter((name) => name.endsWith(".ts") && !name.endsWith(".test.ts"))
|
|
83
|
-
.flatMap((name) => {
|
|
84
|
-
const source = readFileSync(path.join(srcDir, name), "utf8");
|
|
85
|
-
return source
|
|
86
|
-
.split("\n")
|
|
87
|
-
.filter((line) => line.includes('from "node:'))
|
|
88
|
-
.map((line) => `${name}: ${line.trim()}`);
|
|
89
|
-
});
|
|
90
|
-
expect(nodeImports.filter((entry) => entry.startsWith("live-session")
|
|
91
|
-
|| entry.startsWith("worker.ts")
|
|
92
|
-
|| entry.startsWith("kuralle-realtime-agent"))).toEqual([]);
|
|
93
|
-
});
|
|
94
|
-
});
|
|
@@ -1,409 +0,0 @@
|
|
|
1
|
-
// SPDX-License-Identifier: MIT
|
|
2
|
-
|
|
3
|
-
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
4
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
5
|
-
import { tmpdir } from "node:os";
|
|
6
|
-
import { join, dirname } from "node:path";
|
|
7
|
-
import { fileURLToPath } from "node:url";
|
|
8
|
-
import { execFile } from "node:child_process";
|
|
9
|
-
import { promisify } from "node:util";
|
|
10
|
-
import { Miniflare } from "miniflare";
|
|
11
|
-
import { afterEach, describe, expect, it } from "vitest";
|
|
12
|
-
import { decodeMuLawToPcm16, encodePcm16ToMuLaw } from "@kuralle-syrinx/core/audio";
|
|
13
|
-
|
|
14
|
-
const execFileAsync = promisify(execFile);
|
|
15
|
-
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
|
|
16
|
-
|
|
17
|
-
type WorkersWebSocket = WebSocket & { accept(): void };
|
|
18
|
-
|
|
19
|
-
const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url));
|
|
20
|
-
const TURN_DETECTION_FIXTURE = join(
|
|
21
|
-
REPO_ROOT,
|
|
22
|
-
"examples/02-hello-voice-headless/test/fixtures/gemini-turn-detection/02-reset-password.wav",
|
|
23
|
-
);
|
|
24
|
-
const WORKER_INPUT_RATE_HZ = 16000;
|
|
25
|
-
|
|
26
|
-
loadRepoEnv();
|
|
27
|
-
const liveEnv = {
|
|
28
|
-
DEEPGRAM_API_KEY: process.env["DEEPGRAM_API_KEY"]?.trim() ?? "",
|
|
29
|
-
OPENAI_API_KEY: process.env["OPENAI_API_KEY"]?.trim() ?? "",
|
|
30
|
-
VECTORIZE: {},
|
|
31
|
-
};
|
|
32
|
-
const hasLiveKeys = Boolean(liveEnv.DEEPGRAM_API_KEY && liveEnv.OPENAI_API_KEY);
|
|
33
|
-
// The live turn hits paid, non-deterministic provider APIs, so it is opt-in
|
|
34
|
-
// (SYRINX_LIVE_WORKER_TEST=1) rather than running on every `pnpm -r test`.
|
|
35
|
-
// Run it with: pnpm --filter @kuralle-syrinx/server-workers test:live
|
|
36
|
-
const liveTurnEnabled = hasLiveKeys && process.env["SYRINX_LIVE_WORKER_TEST"] === "1";
|
|
37
|
-
|
|
38
|
-
const tempDirs: string[] = [];
|
|
39
|
-
afterEach(async () => {
|
|
40
|
-
await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
|
|
41
|
-
});
|
|
42
|
-
|
|
43
|
-
async function buildWorker(): Promise<string> {
|
|
44
|
-
const tmp = await mkdtemp(join(tmpdir(), "syrinx-worker-"));
|
|
45
|
-
tempDirs.push(tmp);
|
|
46
|
-
await execFileAsync(
|
|
47
|
-
"npx",
|
|
48
|
-
["wrangler", "deploy", "--dry-run", "--outdir", tmp, "-c", "wrangler.jsonc"],
|
|
49
|
-
{ cwd: PKG_ROOT },
|
|
50
|
-
);
|
|
51
|
-
return readFile(join(tmp, "worker.js"), "utf8");
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function newMiniflare(script: string, bindings: Record<string, unknown>): Miniflare {
|
|
55
|
-
return new Miniflare({
|
|
56
|
-
modules: true,
|
|
57
|
-
script,
|
|
58
|
-
compatibilityDate: "2026-06-01",
|
|
59
|
-
compatibilityFlags: ["nodejs_compat"],
|
|
60
|
-
durableObjects: {
|
|
61
|
-
VOICE_CONVERSATIONS: { className: "VoiceConversation", useSQLite: true },
|
|
62
|
-
TWILIO_VOICE_CONVERSATIONS: { className: "TwilioVoiceConversation", useSQLite: true },
|
|
63
|
-
},
|
|
64
|
-
vectorize: {
|
|
65
|
-
VECTORIZE: { dimensions: 1536, metric: "cosine", index_name: "kuralle-university-kb" },
|
|
66
|
-
},
|
|
67
|
-
bindings,
|
|
68
|
-
});
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
describe("VoiceConversation worker runtime", () => {
|
|
72
|
-
// Deterministic, no provider keys: proves the edge bundle boots inside workerd
|
|
73
|
-
// and accepts a WebSocketPair upgrade before any provider session starts.
|
|
74
|
-
it("boots in workerd and accepts a WebSocket upgrade", async () => {
|
|
75
|
-
const script = await buildWorker();
|
|
76
|
-
const mf = newMiniflare(script, {});
|
|
77
|
-
try {
|
|
78
|
-
const health = await mf.dispatchFetch("http://localhost/health");
|
|
79
|
-
expect(health.status).toBe(200);
|
|
80
|
-
expect(await health.text()).toBe("ok");
|
|
81
|
-
|
|
82
|
-
const response = await mf.dispatchFetch("http://localhost/ws?sessionId=boot-smoke", {
|
|
83
|
-
headers: { Upgrade: "websocket" },
|
|
84
|
-
});
|
|
85
|
-
expect(response.status).toBe(101);
|
|
86
|
-
const ws = (response as unknown as Response & { webSocket?: WorkersWebSocket }).webSocket;
|
|
87
|
-
expect(ws).toBeTruthy();
|
|
88
|
-
ws!.accept();
|
|
89
|
-
ws!.close();
|
|
90
|
-
} finally {
|
|
91
|
-
await mf.dispose();
|
|
92
|
-
}
|
|
93
|
-
}, 20_000);
|
|
94
|
-
|
|
95
|
-
// Live: drives a real STT -> LLM -> TTS turn through real providers inside
|
|
96
|
-
// workerd. Skipped when provider keys are absent so offline CI stays green.
|
|
97
|
-
it.skipIf(!liveTurnEnabled)(
|
|
98
|
-
"drives a real audio turn through Deepgram STT + kuralle + Deepgram TTS in workerd",
|
|
99
|
-
async () => {
|
|
100
|
-
expect(existsSync(TURN_DETECTION_FIXTURE)).toBe(true);
|
|
101
|
-
const pcm16 = readWav16kMono(TURN_DETECTION_FIXTURE);
|
|
102
|
-
const script = await buildWorker();
|
|
103
|
-
const mf = newMiniflare(script, liveEnv);
|
|
104
|
-
try {
|
|
105
|
-
const response = await mf.dispatchFetch("http://localhost/ws?sessionId=live-turn", {
|
|
106
|
-
headers: { Upgrade: "websocket" },
|
|
107
|
-
});
|
|
108
|
-
expect(response.status).toBe(101);
|
|
109
|
-
const ws = (response as unknown as Response & { webSocket?: WorkersWebSocket }).webSocket;
|
|
110
|
-
expect(ws).toBeTruthy();
|
|
111
|
-
|
|
112
|
-
const messages: Array<string | ArrayBuffer> = [];
|
|
113
|
-
ws!.addEventListener("message", (event) => {
|
|
114
|
-
messages.push(event.data as string | ArrayBuffer);
|
|
115
|
-
});
|
|
116
|
-
ws!.accept();
|
|
117
|
-
|
|
118
|
-
// Wait for the session to come up (provider sockets connected) before audio.
|
|
119
|
-
// Surface a provider init error (e.g. bad key, unreachable socket) instead
|
|
120
|
-
// of a bare timeout.
|
|
121
|
-
try {
|
|
122
|
-
await waitFor(() => messages.some((m) => typeof m === "string" && m.includes('"type":"ready"')), 20_000);
|
|
123
|
-
} catch {
|
|
124
|
-
throw new Error(`session never became ready: ${firstSessionError(messages) ?? "no error reported"}`);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// Stream the fixture as 20ms PCM16 frames, base64 JSON audio messages.
|
|
128
|
-
const frameSamples = (WORKER_INPUT_RATE_HZ / 1000) * 20; // 320 samples = 640 bytes
|
|
129
|
-
let sequence = 0;
|
|
130
|
-
for (let offset = 0; offset < pcm16.length; offset += frameSamples) {
|
|
131
|
-
const frame = pcm16.subarray(offset, Math.min(offset + frameSamples, pcm16.length));
|
|
132
|
-
sequence += 1;
|
|
133
|
-
ws!.send(JSON.stringify({
|
|
134
|
-
type: "audio",
|
|
135
|
-
audio: int16ToBase64(frame),
|
|
136
|
-
sampleRateHz: WORKER_INPUT_RATE_HZ,
|
|
137
|
-
sequence,
|
|
138
|
-
}));
|
|
139
|
-
await sleep(20);
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
// Real Deepgram transcript for the "reset password" utterance.
|
|
143
|
-
await waitFor(
|
|
144
|
-
() => messages.some((m) => typeof m === "string" && m.includes('"type":"stt_output"')),
|
|
145
|
-
15_000,
|
|
146
|
-
);
|
|
147
|
-
// Real Deepgram TTS audio frames back from the kuralle reply.
|
|
148
|
-
await waitFor(
|
|
149
|
-
() => messages.some((m) => typeof m === "string" && m.includes('"type":"tts_chunk"')),
|
|
150
|
-
30_000,
|
|
151
|
-
);
|
|
152
|
-
|
|
153
|
-
const transcript = extractTranscript(messages);
|
|
154
|
-
expect(transcript.length).toBeGreaterThan(0);
|
|
155
|
-
expect(messages.some((m) => m instanceof ArrayBuffer && m.byteLength > 0)).toBe(true);
|
|
156
|
-
|
|
157
|
-
// eslint-disable-next-line no-console
|
|
158
|
-
console.log(`[live-turn] STT transcript: ${JSON.stringify(transcript)}`);
|
|
159
|
-
ws!.close();
|
|
160
|
-
} finally {
|
|
161
|
-
await mf.dispose();
|
|
162
|
-
}
|
|
163
|
-
},
|
|
164
|
-
90_000,
|
|
165
|
-
);
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
describe("TwilioVoiceConversation worker runtime", () => {
|
|
169
|
-
// Deterministic, no keys: the Twilio Media Streams front accepts a WS upgrade at /twilio
|
|
170
|
-
// through withVoice(Agent, { transport: "twilio" }) — the cf-agents telephony front bundles
|
|
171
|
-
// and boots in workerd.
|
|
172
|
-
it("accepts a Twilio Media Streams WebSocket upgrade at /twilio", async () => {
|
|
173
|
-
const script = await buildWorker();
|
|
174
|
-
const mf = newMiniflare(script, {});
|
|
175
|
-
try {
|
|
176
|
-
const response = await mf.dispatchFetch("http://localhost/twilio?sessionId=twilio-boot", {
|
|
177
|
-
headers: { Upgrade: "websocket" },
|
|
178
|
-
});
|
|
179
|
-
expect(response.status).toBe(101);
|
|
180
|
-
const ws = (response as unknown as Response & { webSocket?: WorkersWebSocket }).webSocket;
|
|
181
|
-
expect(ws).toBeTruthy();
|
|
182
|
-
ws!.accept();
|
|
183
|
-
ws!.close();
|
|
184
|
-
} finally {
|
|
185
|
-
await mf.dispose();
|
|
186
|
-
}
|
|
187
|
-
}, 20_000);
|
|
188
|
-
|
|
189
|
-
// Deterministic: the Twilio Voice webhook returns TwiML that bridges the call to /twilio.
|
|
190
|
-
it("returns <Connect><Stream> TwiML from /incoming-call pointing at /twilio", async () => {
|
|
191
|
-
const script = await buildWorker();
|
|
192
|
-
const mf = newMiniflare(script, {});
|
|
193
|
-
try {
|
|
194
|
-
const res = await mf.dispatchFetch("http://localhost/incoming-call", {
|
|
195
|
-
method: "POST",
|
|
196
|
-
headers: { "content-type": "application/x-www-form-urlencoded" },
|
|
197
|
-
body: "CallSid=CA0123456789abcdef&From=%2B15551234567",
|
|
198
|
-
});
|
|
199
|
-
expect(res.status).toBe(200);
|
|
200
|
-
expect(res.headers.get("content-type")).toContain("text/xml");
|
|
201
|
-
const twiml = await res.text();
|
|
202
|
-
expect(twiml).toContain("<Connect>");
|
|
203
|
-
expect(twiml).toContain("<Stream");
|
|
204
|
-
// The Twilio CallSid becomes the /twilio sessionId.
|
|
205
|
-
expect(twiml).toContain("/twilio?sessionId=CA0123456789abcdef");
|
|
206
|
-
} finally {
|
|
207
|
-
await mf.dispose();
|
|
208
|
-
}
|
|
209
|
-
}, 20_000);
|
|
210
|
-
|
|
211
|
-
// Live: EMULATE A PSTN CALL — speak the Twilio Media Streams protocol (connected/start/media
|
|
212
|
-
// as base64 μ-law 8 kHz) at /twilio and assert the agent answers with non-silent μ-law media
|
|
213
|
-
// frames on the phone leg. No carrier, no phone number — the protocol is just a WebSocket.
|
|
214
|
-
it.skipIf(!liveTurnEnabled)(
|
|
215
|
-
"answers an emulated Twilio Media Streams call end-to-end in workerd",
|
|
216
|
-
async () => {
|
|
217
|
-
expect(existsSync(TURN_DETECTION_FIXTURE)).toBe(true);
|
|
218
|
-
const pcm16k = readWav16kMono(TURN_DETECTION_FIXTURE);
|
|
219
|
-
const pcm8k = downsampleTo8k(pcm16k);
|
|
220
|
-
const script = await buildWorker();
|
|
221
|
-
const mf = newMiniflare(script, liveEnv);
|
|
222
|
-
try {
|
|
223
|
-
const response = await mf.dispatchFetch("http://localhost/twilio?sessionId=twilio-live", {
|
|
224
|
-
headers: { Upgrade: "websocket" },
|
|
225
|
-
});
|
|
226
|
-
expect(response.status).toBe(101);
|
|
227
|
-
const ws = (response as unknown as Response & { webSocket?: WorkersWebSocket }).webSocket;
|
|
228
|
-
expect(ws).toBeTruthy();
|
|
229
|
-
|
|
230
|
-
let downlinkFrames = 0;
|
|
231
|
-
let downlinkPeak = 0;
|
|
232
|
-
let clearReceived = false;
|
|
233
|
-
ws!.addEventListener("message", (event) => {
|
|
234
|
-
const data = event.data as string | ArrayBuffer;
|
|
235
|
-
if (typeof data !== "string" || !data.startsWith("{")) return;
|
|
236
|
-
const msg = JSON.parse(data) as Record<string, unknown>;
|
|
237
|
-
if (msg.event === "media") {
|
|
238
|
-
downlinkFrames += 1;
|
|
239
|
-
const media = msg.media as { payload?: string } | undefined;
|
|
240
|
-
if (media?.payload) {
|
|
241
|
-
const pcm = decodeMuLawToPcm16(base64ToBytes(media.payload));
|
|
242
|
-
for (const s of pcm) downlinkPeak = Math.max(downlinkPeak, Math.abs(s));
|
|
243
|
-
}
|
|
244
|
-
} else if (msg.event === "clear") {
|
|
245
|
-
clearReceived = true;
|
|
246
|
-
}
|
|
247
|
-
});
|
|
248
|
-
ws!.accept();
|
|
249
|
-
|
|
250
|
-
const streamSid = "MZtwiliolivesmoke";
|
|
251
|
-
ws!.send(JSON.stringify({ event: "connected", protocol: "Call", version: "1.0.0" }));
|
|
252
|
-
ws!.send(JSON.stringify({
|
|
253
|
-
event: "start",
|
|
254
|
-
streamSid,
|
|
255
|
-
start: { streamSid, callSid: "CAtwiliolivesmoke", mediaFormat: { encoding: "audio/x-mulaw", sampleRate: 8000, channels: 1 } },
|
|
256
|
-
}));
|
|
257
|
-
|
|
258
|
-
const sendMulaw = (frame: Int16Array): void => {
|
|
259
|
-
ws!.send(JSON.stringify({
|
|
260
|
-
event: "media",
|
|
261
|
-
streamSid,
|
|
262
|
-
media: { payload: bytesToBase64(encodePcm16ToMuLaw(frame)) },
|
|
263
|
-
}));
|
|
264
|
-
};
|
|
265
|
-
|
|
266
|
-
const frameSamples8k = 160; // 20ms at 8 kHz
|
|
267
|
-
for (let offset = 0; offset < pcm8k.length; offset += frameSamples8k) {
|
|
268
|
-
const frame = new Int16Array(frameSamples8k);
|
|
269
|
-
frame.set(pcm8k.subarray(offset, Math.min(offset + frameSamples8k, pcm8k.length)));
|
|
270
|
-
sendMulaw(frame);
|
|
271
|
-
await sleep(20);
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
// Pad silence until the agent answers (Deepgram endpointing + kuralle + TTS).
|
|
275
|
-
const deadline = Date.now() + 60_000;
|
|
276
|
-
while (downlinkFrames === 0 && Date.now() < deadline) {
|
|
277
|
-
sendMulaw(new Int16Array(frameSamples8k));
|
|
278
|
-
await sleep(20);
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
ws!.send(JSON.stringify({ event: "stop", streamSid }));
|
|
282
|
-
ws!.close();
|
|
283
|
-
|
|
284
|
-
// eslint-disable-next-line no-console
|
|
285
|
-
console.log(`[twilio-live] downlink media frames=${downlinkFrames} peak=${downlinkPeak} clear=${clearReceived}`);
|
|
286
|
-
expect(downlinkFrames).toBeGreaterThan(0);
|
|
287
|
-
expect(downlinkPeak).toBeGreaterThan(100); // non-silent answer on the phone leg
|
|
288
|
-
} finally {
|
|
289
|
-
await mf.dispose();
|
|
290
|
-
}
|
|
291
|
-
},
|
|
292
|
-
120_000,
|
|
293
|
-
);
|
|
294
|
-
});
|
|
295
|
-
|
|
296
|
-
function downsampleTo8k(samples: Int16Array): Int16Array {
|
|
297
|
-
const out = new Int16Array(Math.floor(samples.length / 2));
|
|
298
|
-
for (let i = 0; i < out.length; i += 1) out[i] = samples[i * 2]!;
|
|
299
|
-
return out;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
function base64ToBytes(value: string): Uint8Array {
|
|
303
|
-
return new Uint8Array(Buffer.from(value, "base64"));
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
function bytesToBase64(bytes: Uint8Array): string {
|
|
307
|
-
return Buffer.from(bytes).toString("base64");
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
function firstSessionError(messages: ReadonlyArray<string | ArrayBuffer>): string | null {
|
|
311
|
-
for (const m of messages) {
|
|
312
|
-
if (typeof m !== "string") continue;
|
|
313
|
-
try {
|
|
314
|
-
const parsed = JSON.parse(m) as { type?: string; message?: string };
|
|
315
|
-
if (parsed.type === "error" && typeof parsed.message === "string") return parsed.message;
|
|
316
|
-
} catch {
|
|
317
|
-
// not JSON — ignore
|
|
318
|
-
}
|
|
319
|
-
}
|
|
320
|
-
return null;
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
function extractTranscript(messages: ReadonlyArray<string | ArrayBuffer>): string {
|
|
324
|
-
for (const m of messages) {
|
|
325
|
-
if (typeof m !== "string") continue;
|
|
326
|
-
try {
|
|
327
|
-
const parsed = JSON.parse(m) as { type?: string; transcript?: string };
|
|
328
|
-
if (parsed.type === "stt_output" && typeof parsed.transcript === "string") return parsed.transcript;
|
|
329
|
-
} catch {
|
|
330
|
-
// not JSON — ignore
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
return "";
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
async function waitFor(predicate: () => boolean, timeoutMs: number): Promise<void> {
|
|
337
|
-
const deadline = Date.now() + timeoutMs;
|
|
338
|
-
while (Date.now() < deadline) {
|
|
339
|
-
if (predicate()) return;
|
|
340
|
-
await sleep(25);
|
|
341
|
-
}
|
|
342
|
-
throw new Error("timed out waiting for worker websocket output");
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
function sleep(ms: number): Promise<void> {
|
|
346
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
function int16ToBase64(samples: Int16Array): string {
|
|
350
|
-
const bytes = new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength);
|
|
351
|
-
return Buffer.from(bytes).toString("base64");
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
/** Minimal PCM16 WAV reader that returns samples resampled to 16 kHz mono. */
|
|
355
|
-
function readWav16kMono(path: string): Int16Array {
|
|
356
|
-
const buf = readFileSync(path);
|
|
357
|
-
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
358
|
-
let sampleRate = 24000;
|
|
359
|
-
let dataStart = -1;
|
|
360
|
-
let dataLen = 0;
|
|
361
|
-
let pos = 12; // skip RIFF/WAVE header
|
|
362
|
-
while (pos + 8 <= buf.byteLength) {
|
|
363
|
-
const id = String.fromCharCode(buf[pos]!, buf[pos + 1]!, buf[pos + 2]!, buf[pos + 3]!);
|
|
364
|
-
const size = view.getUint32(pos + 4, true);
|
|
365
|
-
if (id === "fmt ") sampleRate = view.getUint32(pos + 8 + 4, true);
|
|
366
|
-
if (id === "data") {
|
|
367
|
-
dataStart = pos + 8;
|
|
368
|
-
dataLen = size;
|
|
369
|
-
break;
|
|
370
|
-
}
|
|
371
|
-
pos += 8 + size + (size % 2);
|
|
372
|
-
}
|
|
373
|
-
if (dataStart < 0) throw new Error(`no data chunk in WAV: ${path}`);
|
|
374
|
-
const sampleCount = Math.floor(dataLen / 2);
|
|
375
|
-
const src = new Int16Array(sampleCount);
|
|
376
|
-
for (let i = 0; i < sampleCount; i += 1) src[i] = view.getInt16(dataStart + i * 2, true);
|
|
377
|
-
return sampleRate === WORKER_INPUT_RATE_HZ ? src : resampleLinear(src, sampleRate, WORKER_INPUT_RATE_HZ);
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
function resampleLinear(src: Int16Array, fromHz: number, toHz: number): Int16Array {
|
|
381
|
-
const outLen = Math.max(1, Math.round((src.length * toHz) / fromHz));
|
|
382
|
-
const out = new Int16Array(outLen);
|
|
383
|
-
const ratio = (src.length - 1) / Math.max(1, outLen - 1);
|
|
384
|
-
for (let i = 0; i < outLen; i += 1) {
|
|
385
|
-
const x = i * ratio;
|
|
386
|
-
const i0 = Math.floor(x);
|
|
387
|
-
const i1 = Math.min(src.length - 1, i0 + 1);
|
|
388
|
-
const frac = x - i0;
|
|
389
|
-
out[i] = Math.round(src[i0]! * (1 - frac) + src[i1]! * frac);
|
|
390
|
-
}
|
|
391
|
-
return out;
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
/** Load DEEPGRAM/OPENAI keys from the repo-root .env if not already set. */
|
|
395
|
-
function loadRepoEnv(): void {
|
|
396
|
-
const envPath = join(REPO_ROOT, ".env");
|
|
397
|
-
if (!existsSync(envPath)) return;
|
|
398
|
-
for (const line of readFileSync(envPath, "utf8").split(/\r?\n/)) {
|
|
399
|
-
const match = /^([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line.trim());
|
|
400
|
-
if (!match) continue;
|
|
401
|
-
const key = match[1]!;
|
|
402
|
-
if (process.env[key] !== undefined) continue;
|
|
403
|
-
let value = match[2]!.trim();
|
|
404
|
-
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
405
|
-
value = value.slice(1, -1);
|
|
406
|
-
}
|
|
407
|
-
process.env[key] = value;
|
|
408
|
-
}
|
|
409
|
-
}
|
package/tsconfig.json
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "ES2022",
|
|
4
|
-
"module": "NodeNext",
|
|
5
|
-
"moduleResolution": "NodeNext",
|
|
6
|
-
"lib": ["ES2022"],
|
|
7
|
-
"types": ["@cloudflare/workers-types"],
|
|
8
|
-
"strict": true,
|
|
9
|
-
"noUncheckedIndexedAccess": true,
|
|
10
|
-
"noImplicitReturns": true,
|
|
11
|
-
"declaration": true,
|
|
12
|
-
"declarationMap": true,
|
|
13
|
-
"sourceMap": true,
|
|
14
|
-
"outDir": "./dist",
|
|
15
|
-
"rootDir": "./src",
|
|
16
|
-
"esModuleInterop": true,
|
|
17
|
-
"skipLibCheck": true,
|
|
18
|
-
"forceConsistentCasingInFileNames": true
|
|
19
|
-
},
|
|
20
|
-
"include": ["src/**/*.ts"],
|
|
21
|
-
"exclude": ["node_modules", "dist"]
|
|
22
|
-
}
|