@kuralle-syrinx/server-workers 2.1.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dev.vars.example +17 -0
- package/package.json +12 -9
- package/src/live-realtime-session.test.ts +22 -23
- package/src/live-realtime-session.ts +37 -40
- package/src/live-session.test.ts +36 -31
- package/src/live-session.ts +52 -53
- package/src/worker-realtime.ts +16 -68
- package/src/worker-runtime.test.ts +144 -0
- package/src/worker.ts +81 -94
- package/wrangler.jsonc +8 -0
- package/src/alarm-scheduler.test.ts +0 -44
- package/src/alarm-scheduler.ts +0 -67
- package/src/durable-session-store.test.ts +0 -60
- package/src/durable-session-store.ts +0 -148
- package/src/r2-recorder.test.ts +0 -100
- package/src/r2-recorder.ts +0 -218
- package/src/test-storage.ts +0 -80
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Syrinx on Cloudflare Workers — local dev secrets.
|
|
2
|
+
# Copy to `.dev.vars` for `wrangler dev`; for production use `wrangler secret put <NAME>`.
|
|
3
|
+
# .dev.vars is gitignored — never commit real keys.
|
|
4
|
+
|
|
5
|
+
# --- Cascaded host (wrangler.jsonc → VoiceConversation / TwilioVoiceConversation) ---
|
|
6
|
+
# Deepgram Nova-3 STT + Deepgram Aura TTS.
|
|
7
|
+
DEEPGRAM_API_KEY=
|
|
8
|
+
|
|
9
|
+
# OpenAI (the kuralle reasoner's model).
|
|
10
|
+
OPENAI_API_KEY=
|
|
11
|
+
|
|
12
|
+
# --- Realtime host (wrangler.realtime.jsonc → RealtimeVoiceConversation) ---
|
|
13
|
+
# Front model select: "openai" (default) or "gemini".
|
|
14
|
+
# REALTIME_FRONT=openai
|
|
15
|
+
# OPENAI_API_KEY is reused for the OpenAI realtime front.
|
|
16
|
+
# GEMINI_API_KEY= # required when REALTIME_FRONT=gemini
|
|
17
|
+
# GEMINI_LIVE_MODEL=gemini-3.1-flash-live-preview
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kuralle-syrinx/server-workers",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.1.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -12,16 +12,19 @@
|
|
|
12
12
|
"@kuralle-agents/rag": "^0.8.5",
|
|
13
13
|
"@kuralle-agents/skills": "^0.8.5",
|
|
14
14
|
"@kuralle-agents/vectorize-store": "^0.8.5",
|
|
15
|
+
"agents": "0.14.0",
|
|
15
16
|
"ai": "^6.0.0",
|
|
16
17
|
"zod": "^4.1.8",
|
|
17
|
-
"@kuralle-syrinx/core": "
|
|
18
|
-
"@kuralle-syrinx/aisdk": "
|
|
19
|
-
"@kuralle-syrinx/
|
|
20
|
-
"@kuralle-syrinx/
|
|
21
|
-
"@kuralle-syrinx/
|
|
22
|
-
"@kuralle-syrinx/
|
|
23
|
-
"@kuralle-syrinx/
|
|
24
|
-
"@kuralle-syrinx/
|
|
18
|
+
"@kuralle-syrinx/core": "3.1.0",
|
|
19
|
+
"@kuralle-syrinx/aisdk": "3.1.0",
|
|
20
|
+
"@kuralle-syrinx/cf-agents": "3.1.0",
|
|
21
|
+
"@kuralle-syrinx/server-websocket": "3.1.0",
|
|
22
|
+
"@kuralle-syrinx/kuralle": "3.1.0",
|
|
23
|
+
"@kuralle-syrinx/recorder": "3.1.0",
|
|
24
|
+
"@kuralle-syrinx/realtime": "3.1.0",
|
|
25
|
+
"@kuralle-syrinx/ws": "3.1.0",
|
|
26
|
+
"@kuralle-syrinx/deepgram": "3.1.0",
|
|
27
|
+
"@kuralle-syrinx/cartesia": "3.1.0"
|
|
25
28
|
},
|
|
26
29
|
"devDependencies": {
|
|
27
30
|
"@cloudflare/workers-types": "^4.20260601.0",
|
|
@@ -3,34 +3,40 @@
|
|
|
3
3
|
import { readFileSync, readdirSync } from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { describe, expect, it
|
|
7
|
-
import { VoiceAgentSession } from "@kuralle-syrinx/core";
|
|
8
|
-
import { RealtimeBridge } from "@kuralle-syrinx/realtime";
|
|
6
|
+
import { describe, expect, it } from "vitest";
|
|
9
7
|
|
|
10
8
|
import {
|
|
11
|
-
createRealtimeVoiceAgentSession,
|
|
12
9
|
hasRealtimeSessionCredentials,
|
|
10
|
+
realtimeVoicePipeline,
|
|
13
11
|
resolveRealtimeFront,
|
|
14
12
|
} from "./live-realtime-session.js";
|
|
15
13
|
|
|
16
14
|
const srcDir = path.dirname(fileURLToPath(import.meta.url));
|
|
17
15
|
|
|
18
16
|
const mockVectorize = {} as import("@cloudflare/workers-types").VectorizeIndex;
|
|
17
|
+
const ctx = { sessionId: "s1" };
|
|
19
18
|
|
|
20
|
-
describe("
|
|
21
|
-
it("
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
expect(
|
|
29
|
-
|
|
19
|
+
describe("realtimeVoicePipeline", () => {
|
|
20
|
+
it("is a realtime pipeline routed through the ask_university delegate tool", () => {
|
|
21
|
+
expect(realtimeVoicePipeline.kind).toBe("realtime");
|
|
22
|
+
expect(realtimeVoicePipeline.delegateToolName).toBe("ask_university");
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("builds an OpenAI front by default and a Gemini front when REALTIME_FRONT=gemini", () => {
|
|
26
|
+
const openai = realtimeVoicePipeline.front({ OPENAI_API_KEY: "k", VECTORIZE: mockVectorize }, ctx);
|
|
27
|
+
expect(openai).toBeTruthy();
|
|
28
|
+
const gemini = realtimeVoicePipeline.front(
|
|
29
|
+
{ REALTIME_FRONT: "gemini", GEMINI_API_KEY: "g", VECTORIZE: mockVectorize },
|
|
30
|
+
ctx,
|
|
31
|
+
);
|
|
32
|
+
expect(gemini).toBeTruthy();
|
|
30
33
|
});
|
|
31
34
|
|
|
32
|
-
it("requires OPENAI_API_KEY",
|
|
33
|
-
|
|
35
|
+
it("requires the front model's key (OPENAI_API_KEY default; GEMINI_API_KEY for gemini)", () => {
|
|
36
|
+
expect(() => realtimeVoicePipeline.front({ VECTORIZE: mockVectorize }, ctx)).toThrow(/OPENAI_API_KEY/);
|
|
37
|
+
expect(() =>
|
|
38
|
+
realtimeVoicePipeline.front({ REALTIME_FRONT: "gemini", VECTORIZE: mockVectorize }, ctx),
|
|
39
|
+
).toThrow(/GEMINI_API_KEY/);
|
|
34
40
|
});
|
|
35
41
|
|
|
36
42
|
it("reports credential presence via hasRealtimeSessionCredentials", () => {
|
|
@@ -47,13 +53,6 @@ describe("createRealtimeVoiceAgentSession", () => {
|
|
|
47
53
|
})).toBe(false);
|
|
48
54
|
});
|
|
49
55
|
|
|
50
|
-
it("requires GEMINI_API_KEY when REALTIME_FRONT=gemini", async () => {
|
|
51
|
-
await expect(createRealtimeVoiceAgentSession({
|
|
52
|
-
REALTIME_FRONT: "gemini",
|
|
53
|
-
VECTORIZE: mockVectorize,
|
|
54
|
-
})).rejects.toThrow(/GEMINI_API_KEY/);
|
|
55
|
-
});
|
|
56
|
-
|
|
57
56
|
it("defaults REALTIME_FRONT to openai", () => {
|
|
58
57
|
expect(resolveRealtimeFront({ VECTORIZE: mockVectorize })).toBe("openai");
|
|
59
58
|
expect(resolveRealtimeFront({ REALTIME_FRONT: "gemini", VECTORIZE: mockVectorize })).toBe("gemini");
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
//
|
|
3
|
-
// Bi-model
|
|
4
|
-
// with a kuralle Vectorize-backed Reasoner back model
|
|
3
|
+
// Bi-model voice pipeline for the Cloudflare Workers host: a gpt-realtime or Gemini
|
|
4
|
+
// Live front model with a kuralle Vectorize-backed Reasoner back model, consulted
|
|
5
|
+
// via the `ask_university` front-model delegate tool. This is the pipeline/brain
|
|
6
|
+
// layer — the host (worker-realtime.ts) composes it via `withVoice(Agent)`.
|
|
5
7
|
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import
|
|
8
|
+
import type { Reasoner } from "@kuralle-syrinx/core";
|
|
9
|
+
import type { RealtimePipeline, VoicePipelineContext } from "@kuralle-syrinx/cf-agents";
|
|
10
|
+
import { fromGeminiLive, fromOpenAIRealtime } from "@kuralle-syrinx/realtime";
|
|
11
|
+
import type { RealtimeAdapter, RealtimeToolDef } from "@kuralle-syrinx/realtime";
|
|
9
12
|
import { createWorkersSocket } from "@kuralle-syrinx/ws/workers";
|
|
10
13
|
import type { VectorizeIndex } from "@cloudflare/workers-types";
|
|
11
14
|
import { createRealtimeKuralleReasoner } from "./kuralle-realtime-agent.js";
|
|
@@ -21,12 +24,6 @@ export interface RealtimeSessionEnv {
|
|
|
21
24
|
readonly VECTORIZE: VectorizeIndex;
|
|
22
25
|
}
|
|
23
26
|
|
|
24
|
-
export interface RealtimeSessionOptions {
|
|
25
|
-
readonly sessionId?: string;
|
|
26
|
-
readonly inputSampleRateHz?: number;
|
|
27
|
-
readonly outputSampleRateHz?: number;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
27
|
const DEFAULT_GEMINI_LIVE_MODEL = "gemini-3.1-flash-live-preview";
|
|
31
28
|
|
|
32
29
|
const REALTIME_SYSTEM_INSTRUCTION =
|
|
@@ -53,38 +50,38 @@ export function hasRealtimeSessionCredentials(env: RealtimeSessionEnv): boolean
|
|
|
53
50
|
return Boolean(env.OPENAI_API_KEY?.trim());
|
|
54
51
|
}
|
|
55
52
|
|
|
56
|
-
|
|
57
|
-
env: RealtimeSessionEnv,
|
|
58
|
-
options: RealtimeSessionOptions = {},
|
|
59
|
-
): Promise<VoiceAgentSession> {
|
|
60
|
-
const sessionId = options.sessionId?.trim() || crypto.randomUUID();
|
|
53
|
+
function buildRealtimeFront(env: RealtimeSessionEnv): RealtimeAdapter {
|
|
61
54
|
const front = resolveRealtimeFront(env);
|
|
55
|
+
if (front === "gemini") {
|
|
56
|
+
return fromGeminiLive({
|
|
57
|
+
apiKey: requireKey(env.GEMINI_API_KEY, "GEMINI_API_KEY"),
|
|
58
|
+
model: env.GEMINI_LIVE_MODEL?.trim() || DEFAULT_GEMINI_LIVE_MODEL,
|
|
59
|
+
systemInstruction: REALTIME_SYSTEM_INSTRUCTION,
|
|
60
|
+
tools: [ASK_UNIVERSITY_TOOL],
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
return fromOpenAIRealtime({
|
|
64
|
+
apiKey: requireKey(env.OPENAI_API_KEY, "OPENAI_API_KEY"),
|
|
65
|
+
socketFactory: createWorkersSocket,
|
|
66
|
+
turnDetection: { type: "server_vad", silence_duration_ms: 500 },
|
|
67
|
+
inputTranscription: true,
|
|
68
|
+
tools: [ASK_UNIVERSITY_TOOL],
|
|
69
|
+
});
|
|
70
|
+
}
|
|
62
71
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
})
|
|
70
|
-
: fromOpenAIRealtime({
|
|
71
|
-
apiKey: requireKey(env.OPENAI_API_KEY, "OPENAI_API_KEY"),
|
|
72
|
-
socketFactory: createWorkersSocket,
|
|
73
|
-
turnDetection: { type: "server_vad", silence_duration_ms: 500 },
|
|
74
|
-
inputTranscription: true,
|
|
75
|
-
tools: [ASK_UNIVERSITY_TOOL],
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
const universityReasoner = await createRealtimeKuralleReasoner(env, { sessionId });
|
|
79
|
-
|
|
80
|
-
const bridge = new RealtimeBridge(adapter, universityReasoner, ASK_UNIVERSITY_TOOL.name);
|
|
72
|
+
/** Realtime pipeline descriptor consumed by `withVoice(Agent)`. */
|
|
73
|
+
export const realtimeVoicePipeline: RealtimePipeline<RealtimeSessionEnv> = {
|
|
74
|
+
kind: "realtime",
|
|
75
|
+
front: (env) => buildRealtimeFront(env),
|
|
76
|
+
delegateToolName: ASK_UNIVERSITY_TOOL.name,
|
|
77
|
+
};
|
|
81
78
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
return
|
|
79
|
+
/** The back model: a kuralle Vectorize-backed Reasoner, keyed by the session id. */
|
|
80
|
+
export function createRealtimeReasoner(
|
|
81
|
+
env: RealtimeSessionEnv,
|
|
82
|
+
ctx: VoicePipelineContext,
|
|
83
|
+
): Promise<Reasoner> {
|
|
84
|
+
return createRealtimeKuralleReasoner(env, { sessionId: ctx.sessionId });
|
|
88
85
|
}
|
|
89
86
|
|
|
90
87
|
function requireKey(value: string | undefined, name: string): string {
|
package/src/live-session.test.ts
CHANGED
|
@@ -3,48 +3,53 @@
|
|
|
3
3
|
import { readFileSync, readdirSync } from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { describe, expect, it
|
|
7
|
-
import { VoiceAgentSession } from "@kuralle-syrinx/core";
|
|
8
|
-
import { ReasoningBridge } from "@kuralle-syrinx/aisdk";
|
|
6
|
+
import { describe, expect, it } from "vitest";
|
|
9
7
|
import { DeepgramSTTPlugin, DeepgramTTSPlugin } from "@kuralle-syrinx/deepgram";
|
|
10
8
|
|
|
11
9
|
import {
|
|
12
|
-
|
|
10
|
+
createLiveReasoner,
|
|
13
11
|
hasLiveSessionCredentials,
|
|
12
|
+
liveCascadedPipeline,
|
|
14
13
|
} from "./live-session.js";
|
|
15
14
|
|
|
16
15
|
const srcDir = path.dirname(fileURLToPath(import.meta.url));
|
|
17
16
|
const mockVectorize = {} as import("@cloudflare/workers-types").VectorizeIndex;
|
|
17
|
+
const ctx = { sessionId: "s1" };
|
|
18
18
|
|
|
19
|
-
describe("
|
|
20
|
-
it("
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
OPENAI_API_KEY: "oa-key",
|
|
25
|
-
VECTORIZE: mockVectorize,
|
|
26
|
-
});
|
|
27
|
-
expect(session).toBeInstanceOf(VoiceAgentSession);
|
|
28
|
-
expect(registerSpy).toHaveBeenCalledWith("stt", expect.any(DeepgramSTTPlugin));
|
|
29
|
-
expect(registerSpy).toHaveBeenCalledWith("bridge", expect.any(ReasoningBridge));
|
|
30
|
-
expect(registerSpy).toHaveBeenCalledWith("tts", expect.any(DeepgramTTSPlugin));
|
|
31
|
-
registerSpy.mockRestore();
|
|
19
|
+
describe("liveCascadedPipeline", () => {
|
|
20
|
+
it("is a provider-endpointed cascade with a tightened force-finalize timeout", () => {
|
|
21
|
+
expect(liveCascadedPipeline.kind).toBe("cascaded");
|
|
22
|
+
expect(liveCascadedPipeline.endpointingOwner).toBe("provider_stt");
|
|
23
|
+
expect(liveCascadedPipeline.sttForceFinalizeTimeoutMs).toBe(3500);
|
|
32
24
|
});
|
|
33
25
|
|
|
34
|
-
it("
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
26
|
+
it("builds Deepgram Nova-3 STT (VAD events) and Aura TTS stages from the env", () => {
|
|
27
|
+
const env = { DEEPGRAM_API_KEY: "dg-key", OPENAI_API_KEY: "oa-key", VECTORIZE: mockVectorize };
|
|
28
|
+
const stt = liveCascadedPipeline.stt(env, ctx);
|
|
29
|
+
expect(stt.plugin).toBeInstanceOf(DeepgramSTTPlugin);
|
|
30
|
+
expect(stt.config).toMatchObject({ model: "nova-3", endpointing: 300, vad_events: true, api_key: "dg-key" });
|
|
31
|
+
|
|
32
|
+
const tts = liveCascadedPipeline.tts(env, ctx);
|
|
33
|
+
expect(tts.plugin).toBeInstanceOf(DeepgramTTSPlugin);
|
|
34
|
+
expect(tts.config).toMatchObject({ model: "aura-2-thalia-en", api_key: "dg-key" });
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("requires DEEPGRAM_API_KEY to build the stt/tts stages", () => {
|
|
38
|
+
const env = { OPENAI_API_KEY: "k", VECTORIZE: mockVectorize };
|
|
39
|
+
expect(() => liveCascadedPipeline.stt(env, ctx)).toThrow(/DEEPGRAM_API_KEY/);
|
|
40
|
+
expect(() => liveCascadedPipeline.tts(env, ctx)).toThrow(/DEEPGRAM_API_KEY/);
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
describe("createLiveReasoner", () => {
|
|
45
|
+
it("requires OPENAI_API_KEY and VECTORIZE", async () => {
|
|
46
|
+
await expect(createLiveReasoner({ VECTORIZE: mockVectorize }, ctx)).rejects.toThrow(/OPENAI_API_KEY/);
|
|
47
|
+
await expect(
|
|
48
|
+
createLiveReasoner(
|
|
49
|
+
{ OPENAI_API_KEY: "k", VECTORIZE: undefined as unknown as typeof mockVectorize },
|
|
50
|
+
ctx,
|
|
51
|
+
),
|
|
52
|
+
).rejects.toThrow(/VECTORIZE/);
|
|
48
53
|
});
|
|
49
54
|
|
|
50
55
|
it("reports credential presence via hasLiveSessionCredentials", () => {
|
package/src/live-session.ts
CHANGED
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
2
|
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// the Workers fetch-upgrade socket (createWorkersSocket)
|
|
6
|
-
// into the edge bundle. Turn-taking is owned by Deepgram
|
|
7
|
-
// (endpointingOwner: "provider_stt"), so no Silero VAD / Smart Turn
|
|
8
|
-
// needed on the hot path.
|
|
3
|
+
// Cascaded voice pipeline for the Cloudflare Workers host: real Deepgram STT
|
|
4
|
+
// (nova-3, provider-endpointed) + kuralle (Vectorize RAG) Reasoner + Deepgram
|
|
5
|
+
// Aura TTS, all dialed over the Workers fetch-upgrade socket (createWorkersSocket)
|
|
6
|
+
// so no Node `ws` is pulled into the edge bundle. Turn-taking is owned by Deepgram
|
|
7
|
+
// endpointing (endpointingOwner: "provider_stt"), so no Silero VAD / Smart Turn
|
|
8
|
+
// ONNX is needed on the hot path.
|
|
9
|
+
//
|
|
10
|
+
// This is the pipeline/brain layer — the host (worker.ts) composes it via
|
|
11
|
+
// `withVoice(Agent)`; this module owns the plugin slots and the reasoner, not the
|
|
12
|
+
// connection lifecycle.
|
|
9
13
|
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
14
|
+
import type { Reasoner } from "@kuralle-syrinx/core";
|
|
15
|
+
import type { CascadedPipeline, VoicePipelineContext } from "@kuralle-syrinx/cf-agents";
|
|
12
16
|
import { DeepgramSTTPlugin, DeepgramTTSPlugin } from "@kuralle-syrinx/deepgram";
|
|
13
17
|
import { createWorkersSocket } from "@kuralle-syrinx/ws/workers";
|
|
14
18
|
import type { VectorizeIndex } from "@cloudflare/workers-types";
|
|
@@ -22,60 +26,55 @@ export interface LiveSessionEnv {
|
|
|
22
26
|
readonly VECTORIZE: VectorizeIndex;
|
|
23
27
|
}
|
|
24
28
|
|
|
25
|
-
export interface LiveSessionOptions {
|
|
26
|
-
readonly sessionId?: string;
|
|
27
|
-
readonly inputSampleRateHz?: number;
|
|
28
|
-
readonly outputSampleRateHz?: number;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
29
|
const DEFAULT_DEEPGRAM_TTS_MODEL = "aura-2-thalia-en";
|
|
30
|
+
const INPUT_SAMPLE_RATE_HZ = 16000;
|
|
31
|
+
const OUTPUT_SAMPLE_RATE_HZ = 16000;
|
|
32
32
|
|
|
33
33
|
/** True when every provider secret needed for a live turn is present. */
|
|
34
34
|
export function hasLiveSessionCredentials(env: LiveSessionEnv): boolean {
|
|
35
35
|
return Boolean(env.DEEPGRAM_API_KEY?.trim() && env.OPENAI_API_KEY?.trim() && env.VECTORIZE);
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
sample_rate: outputSampleRateHz,
|
|
68
|
-
},
|
|
38
|
+
/**
|
|
39
|
+
* Cascaded pipeline descriptor consumed by `withVoice(Agent)`. Deepgram Nova-3 STT
|
|
40
|
+
* (provider-endpointed, VAD events for barge-in) → kuralle Reasoner → Deepgram Aura
|
|
41
|
+
* TTS. `sttForceFinalizeTimeoutMs: 3500` keeps the engine's force-finalize tighter
|
|
42
|
+
* than the 7000ms default for the provider-endpointed cascade.
|
|
43
|
+
*/
|
|
44
|
+
export const liveCascadedPipeline: CascadedPipeline<LiveSessionEnv> = {
|
|
45
|
+
kind: "cascaded",
|
|
46
|
+
stt: (env) => ({
|
|
47
|
+
plugin: new DeepgramSTTPlugin(createWorkersSocket),
|
|
48
|
+
config: {
|
|
49
|
+
api_key: requireKey(env.DEEPGRAM_API_KEY, "DEEPGRAM_API_KEY"),
|
|
50
|
+
sample_rate: INPUT_SAMPLE_RATE_HZ,
|
|
51
|
+
model: "nova-3",
|
|
52
|
+
language: "en-US",
|
|
53
|
+
endpointing: 300,
|
|
54
|
+
provider_finalize_timeout_ms: 2500,
|
|
55
|
+
finalize_timeout_fallback: true,
|
|
56
|
+
// No local VAD on the edge: Deepgram SpeechStarted is the barge-in
|
|
57
|
+
// speech-start signal (vad.speech_started producer).
|
|
58
|
+
vad_events: true,
|
|
59
|
+
},
|
|
60
|
+
}),
|
|
61
|
+
tts: (env) => ({
|
|
62
|
+
plugin: new DeepgramTTSPlugin(createWorkersSocket),
|
|
63
|
+
config: {
|
|
64
|
+
api_key: requireKey(env.DEEPGRAM_API_KEY, "DEEPGRAM_API_KEY"),
|
|
65
|
+
model: DEFAULT_DEEPGRAM_TTS_MODEL,
|
|
66
|
+
sample_rate: OUTPUT_SAMPLE_RATE_HZ,
|
|
69
67
|
},
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
68
|
+
}),
|
|
69
|
+
endpointingOwner: "provider_stt",
|
|
70
|
+
sttForceFinalizeTimeoutMs: 3500,
|
|
71
|
+
};
|
|
73
72
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return
|
|
73
|
+
/** The brain: a kuralle Vectorize-backed Reasoner, keyed by the session id. */
|
|
74
|
+
export async function createLiveReasoner(env: LiveSessionEnv, ctx: VoicePipelineContext): Promise<Reasoner> {
|
|
75
|
+
requireKey(env.OPENAI_API_KEY, "OPENAI_API_KEY");
|
|
76
|
+
if (!env.VECTORIZE) throw new Error("VECTORIZE binding is required to start a live voice session");
|
|
77
|
+
return createRealtimeKuralleReasoner(env, { sessionId: ctx.sessionId });
|
|
79
78
|
}
|
|
80
79
|
|
|
81
80
|
function requireKey(value: string | undefined, name: string): string {
|
package/src/worker-realtime.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MIT
|
|
2
|
+
//
|
|
3
|
+
// Cloudflare Workers realtime (bi-model) voice host, built on `withVoice(Agent)`
|
|
4
|
+
// (issue #10 / W1). Same host machinery as the cascaded worker — the only
|
|
5
|
+
// difference is the pipeline (realtime front + kuralle back) and the reasoner.
|
|
2
6
|
|
|
7
|
+
import { Agent } from "agents";
|
|
8
|
+
import { withVoice } from "@kuralle-syrinx/cf-agents";
|
|
3
9
|
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
} from "@kuralle-syrinx/server-websocket/edge";
|
|
7
|
-
import type { WorkersInboundSocketController } from "@kuralle-syrinx/ws/workers";
|
|
8
|
-
import { DurableObjectAlarmScheduler } from "./alarm-scheduler.js";
|
|
9
|
-
import { DurableObjectSessionStore } from "./durable-session-store.js";
|
|
10
|
-
import {
|
|
11
|
-
createRealtimeVoiceAgentSession,
|
|
10
|
+
createRealtimeReasoner,
|
|
11
|
+
realtimeVoicePipeline,
|
|
12
12
|
type RealtimeSessionEnv,
|
|
13
13
|
} from "./live-realtime-session.js";
|
|
14
14
|
|
|
@@ -19,6 +19,14 @@ export interface Env extends RealtimeSessionEnv {
|
|
|
19
19
|
const INPUT_SAMPLE_RATE_HZ = 16000;
|
|
20
20
|
const OUTPUT_SAMPLE_RATE_HZ = 16000;
|
|
21
21
|
|
|
22
|
+
export class RealtimeVoiceConversation extends withVoice<Env, typeof Agent<Env>>(Agent<Env>, {
|
|
23
|
+
pipeline: realtimeVoicePipeline,
|
|
24
|
+
reasoner: (env, ctx) => createRealtimeReasoner(env, ctx),
|
|
25
|
+
inputSampleRateHz: INPUT_SAMPLE_RATE_HZ,
|
|
26
|
+
outputSampleRateHz: OUTPUT_SAMPLE_RATE_HZ,
|
|
27
|
+
resumeWindowMs: 15_000,
|
|
28
|
+
}) {}
|
|
29
|
+
|
|
22
30
|
export default {
|
|
23
31
|
async fetch(request: Request, env: Env): Promise<Response> {
|
|
24
32
|
const url = new URL(request.url);
|
|
@@ -29,63 +37,3 @@ export default {
|
|
|
29
37
|
return await env.REALTIME_VOICE_CONVERSATIONS.get(id).fetch(request);
|
|
30
38
|
},
|
|
31
39
|
};
|
|
32
|
-
|
|
33
|
-
export class RealtimeVoiceConversation {
|
|
34
|
-
private readonly scheduler: DurableObjectAlarmScheduler;
|
|
35
|
-
private readonly store: DurableObjectSessionStore;
|
|
36
|
-
private activeUpgrade: VoiceEdgeWebSocketUpgrade | null = null;
|
|
37
|
-
|
|
38
|
-
constructor(
|
|
39
|
-
private readonly ctx: DurableObjectState,
|
|
40
|
-
private readonly env: Env,
|
|
41
|
-
) {
|
|
42
|
-
this.scheduler = new DurableObjectAlarmScheduler(ctx.storage);
|
|
43
|
-
this.store = new DurableObjectSessionStore(ctx.storage, this.scheduler);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
async fetch(request: Request): Promise<Response> {
|
|
47
|
-
if (request.headers.get("Upgrade")?.toLowerCase() !== "websocket") {
|
|
48
|
-
return new Response("expected websocket", { status: 426 });
|
|
49
|
-
}
|
|
50
|
-
const sessionId = new URL(request.url).searchParams.get("sessionId") ?? crypto.randomUUID();
|
|
51
|
-
const upgrade = createVoiceEdgeWebSocketUpgrade(request, {
|
|
52
|
-
sessionStore: this.store,
|
|
53
|
-
scheduler: this.scheduler,
|
|
54
|
-
createSession: () =>
|
|
55
|
-
createRealtimeVoiceAgentSession(this.env, {
|
|
56
|
-
sessionId,
|
|
57
|
-
inputSampleRateHz: INPUT_SAMPLE_RATE_HZ,
|
|
58
|
-
outputSampleRateHz: OUTPUT_SAMPLE_RATE_HZ,
|
|
59
|
-
}),
|
|
60
|
-
sessionId: () => sessionId,
|
|
61
|
-
inputSampleRateHz: INPUT_SAMPLE_RATE_HZ,
|
|
62
|
-
outputSampleRateHz: OUTPUT_SAMPLE_RATE_HZ,
|
|
63
|
-
resumeWindowMs: 15_000,
|
|
64
|
-
}, {
|
|
65
|
-
acceptWebSocket: (socket) => this.ctx.acceptWebSocket(socket as WebSocket),
|
|
66
|
-
});
|
|
67
|
-
this.activeUpgrade = upgrade;
|
|
68
|
-
return upgrade.response;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
async alarm(): Promise<void> {
|
|
72
|
-
await this.scheduler.runDue();
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
webSocketMessage(_ws: WebSocket, message: string | ArrayBuffer): void {
|
|
76
|
-
this.controller?.message(message);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
webSocketClose(_ws: WebSocket, code: number, reason: string): void {
|
|
80
|
-
this.controller?.close(code, reason);
|
|
81
|
-
this.activeUpgrade = null;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
webSocketError(_ws: WebSocket, error: unknown): void {
|
|
85
|
-
this.controller?.error(error instanceof Error ? error : new Error(String(error)));
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
private get controller(): WorkersInboundSocketController | undefined {
|
|
89
|
-
return this.activeUpgrade?.controller;
|
|
90
|
-
}
|
|
91
|
-
}
|