@crewhaus/voice-runtime 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +163 -0
- package/dist/index.js +450 -0
- package/package.json +9 -6
- package/src/index.coverage.test.ts +0 -590
- package/src/index.test.ts +0 -296
- package/src/index.ts +0 -577
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Catalog R16 `voice-runtime` — Section 24 VOICE.
|
|
3
|
+
*
|
|
4
|
+
* Provider-agnostic realtime audio adapter. The user-facing surface is
|
|
5
|
+
* a small interface — `RealtimeAdapter` — with one concrete adapter in
|
|
6
|
+
* v0 (`createOpenAIRealtimeAdapter`) and a Vapi stub that lazy-loads.
|
|
7
|
+
*
|
|
8
|
+
* Audio framing: every adapter normalises to 16-bit signed-int PCM at
|
|
9
|
+
* 24kHz mono. The OpenAI adapter speaks this format natively; the Vapi
|
|
10
|
+
* adapter (when implemented) will resample its native Opus stream
|
|
11
|
+
* through a server-side helper. Frames are passed as `Int16Array` so
|
|
12
|
+
* downstream consumers (vad-engine) don't have to re-decode bytes.
|
|
13
|
+
*
|
|
14
|
+
* Event surface:
|
|
15
|
+
* - `transcript_partial { text }` — incremental transcript chunks
|
|
16
|
+
* - `transcript_final { text }` — when the user's utterance ends
|
|
17
|
+
* - `audio_chunk { pcm }` — agent's TTS audio (pcm16 24kHz)
|
|
18
|
+
* - `tool_use { id, name, input }` — agent function calls
|
|
19
|
+
* - `interrupt { reason }` — emitted on barge-in
|
|
20
|
+
* - `disconnect { code?, reason? }` — final
|
|
21
|
+
*
|
|
22
|
+
* The OpenAI adapter is HEADLESS — no microphone, no speakers. The
|
|
23
|
+
* caller pumps PCM in via `sendAudio(pcm)`; the adapter emits audio +
|
|
24
|
+
* transcript events back. This keeps the runtime usable in three
|
|
25
|
+
* deployment modes:
|
|
26
|
+
* 1. Browser → daemon over WebRTC (browser has the mic/speaker).
|
|
27
|
+
* 2. SIP gateway → daemon (telephony provides PCM).
|
|
28
|
+
* 3. Node-only smoke (synthesize PCM in TS, pipe in, assert events).
|
|
29
|
+
*/
|
|
30
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
31
|
+
export declare class VoiceRuntimeError extends CrewhausError {
|
|
32
|
+
readonly name = "VoiceRuntimeError";
|
|
33
|
+
constructor(message: string, cause?: unknown);
|
|
34
|
+
}
|
|
35
|
+
export type ProviderId = "openai" | "vapi";
|
|
36
|
+
export type RealtimeEvent = {
|
|
37
|
+
kind: "session_created";
|
|
38
|
+
sessionId: string;
|
|
39
|
+
} | {
|
|
40
|
+
kind: "transcript_partial";
|
|
41
|
+
text: string;
|
|
42
|
+
itemId?: string;
|
|
43
|
+
} | {
|
|
44
|
+
kind: "transcript_final";
|
|
45
|
+
text: string;
|
|
46
|
+
itemId?: string;
|
|
47
|
+
} | {
|
|
48
|
+
kind: "audio_chunk";
|
|
49
|
+
pcm: Int16Array;
|
|
50
|
+
sampleRate: number;
|
|
51
|
+
} | {
|
|
52
|
+
kind: "tool_use";
|
|
53
|
+
id: string;
|
|
54
|
+
name: string;
|
|
55
|
+
input: unknown;
|
|
56
|
+
} | {
|
|
57
|
+
kind: "interrupt";
|
|
58
|
+
reason: string;
|
|
59
|
+
} | {
|
|
60
|
+
kind: "disconnect";
|
|
61
|
+
code?: number;
|
|
62
|
+
reason?: string;
|
|
63
|
+
} | {
|
|
64
|
+
kind: "error";
|
|
65
|
+
message: string;
|
|
66
|
+
cause?: unknown;
|
|
67
|
+
}
|
|
68
|
+
/** Raw provider frame, surfaced verbatim (Vapi adapter passthrough). */
|
|
69
|
+
| {
|
|
70
|
+
kind: "raw";
|
|
71
|
+
provider: ProviderId;
|
|
72
|
+
ts: string;
|
|
73
|
+
payload: unknown;
|
|
74
|
+
};
|
|
75
|
+
export type RealtimeEventHandler = (event: RealtimeEvent) => void;
|
|
76
|
+
export type RealtimeConnectOptions = {
|
|
77
|
+
readonly model: string;
|
|
78
|
+
readonly instructions?: string;
|
|
79
|
+
readonly voice?: string;
|
|
80
|
+
/** Tools (Anthropic-style schemas) the model may call mid-conversation. */
|
|
81
|
+
readonly tools?: ReadonlyArray<{
|
|
82
|
+
readonly name: string;
|
|
83
|
+
readonly description: string;
|
|
84
|
+
readonly inputSchema: unknown;
|
|
85
|
+
}>;
|
|
86
|
+
/** Server-side VAD vs none. v0 defaults to "server" (OpenAI's built-in). */
|
|
87
|
+
readonly vad?: "server" | "none";
|
|
88
|
+
/** Max response audio length in seconds. */
|
|
89
|
+
readonly maxResponseSeconds?: number;
|
|
90
|
+
};
|
|
91
|
+
export interface RealtimeAdapter {
|
|
92
|
+
readonly providerId: ProviderId;
|
|
93
|
+
readonly sampleRate: number;
|
|
94
|
+
/** True after `connect()` resolves and before `disconnect()` is called. */
|
|
95
|
+
readonly connected: boolean;
|
|
96
|
+
connect(opts: RealtimeConnectOptions): Promise<void>;
|
|
97
|
+
/** Push PCM 16-bit signed mono frames at the adapter's sample rate. */
|
|
98
|
+
sendAudio(pcm: Int16Array): void;
|
|
99
|
+
/**
|
|
100
|
+
* Tell the server "this is the end of the user's utterance — process it
|
|
101
|
+
* now". Used by callers that bypass server-side VAD (or when server VAD
|
|
102
|
+
* timing is too slow for a one-shot smoke). Idempotent.
|
|
103
|
+
*/
|
|
104
|
+
commitInput(): void;
|
|
105
|
+
/** Inject a text turn (no audio). Useful for tests + chat-only mixed mode. */
|
|
106
|
+
sendText(text: string): void;
|
|
107
|
+
/** Cancel the in-flight response. Used by `barge-in-controller`. */
|
|
108
|
+
interrupt(reason: string): void;
|
|
109
|
+
on(handler: RealtimeEventHandler): () => void;
|
|
110
|
+
disconnect(): Promise<void>;
|
|
111
|
+
}
|
|
112
|
+
/** Test injection: a factory that returns a WebSocket-like (skips `new`). */
|
|
113
|
+
export type WebSocketFactory = (url: string, init?: {
|
|
114
|
+
headers?: Record<string, string>;
|
|
115
|
+
}) => WebSocketLike;
|
|
116
|
+
export type WebSocketLike = {
|
|
117
|
+
readyState: number;
|
|
118
|
+
send(data: string | ArrayBuffer): void;
|
|
119
|
+
close(code?: number, reason?: string): void;
|
|
120
|
+
addEventListener(type: "open" | "close" | "message" | "error", handler: (ev: unknown) => void): void;
|
|
121
|
+
};
|
|
122
|
+
export type CreateOpenAIRealtimeAdapterOptions = {
|
|
123
|
+
readonly apiKey?: string;
|
|
124
|
+
readonly url?: string;
|
|
125
|
+
/** Override the URL→WebSocket factory; tests pass a stub. Default uses globalThis.WebSocket. */
|
|
126
|
+
readonly _ws?: WebSocketFactory;
|
|
127
|
+
readonly _now?: () => Date;
|
|
128
|
+
};
|
|
129
|
+
export declare function createOpenAIRealtimeAdapter(opts?: CreateOpenAIRealtimeAdapterOptions): RealtimeAdapter;
|
|
130
|
+
export type CreateVapiRealtimeAdapterOptions = {
|
|
131
|
+
readonly apiKey?: string;
|
|
132
|
+
readonly assistantId?: string;
|
|
133
|
+
/** Test injection — same shape as OpenAI's. */
|
|
134
|
+
readonly _ws?: WebSocketFactory;
|
|
135
|
+
readonly _now?: () => Date;
|
|
136
|
+
/** Override the WebSocket URL. Default: `wss://api.vapi.ai/v1/realtime`. */
|
|
137
|
+
readonly url?: string;
|
|
138
|
+
};
|
|
139
|
+
/**
|
|
140
|
+
* Section 30 — Vapi realtime adapter. Maps to the same `RealtimeAdapter`
|
|
141
|
+
* interface as OpenAI Realtime so callers can swap providers via spec
|
|
142
|
+
* config. The connect handshake takes an `assistantId` (Vapi's preset)
|
|
143
|
+
* plus the same instructions/voice/temperature knobs the OpenAI path
|
|
144
|
+
* supports.
|
|
145
|
+
*
|
|
146
|
+
* Without `VAPI_API_KEY` set or an `_ws` injection, `connect()` throws
|
|
147
|
+
* with a clear message — production deployments enable this when the
|
|
148
|
+
* env var is present.
|
|
149
|
+
*/
|
|
150
|
+
export declare function createVapiRealtimeAdapter(opts?: CreateVapiRealtimeAdapterOptions): RealtimeAdapter;
|
|
151
|
+
/**
|
|
152
|
+
* Resolve a provider string to an adapter factory. Mirrors the
|
|
153
|
+
* `model-router` pattern — the spec carries `voice.provider: "openai" |
|
|
154
|
+
* "vapi"` and the daemon constructs the right adapter at boot.
|
|
155
|
+
*/
|
|
156
|
+
export declare function createRealtimeAdapter(provider: ProviderId, opts?: CreateOpenAIRealtimeAdapterOptions): RealtimeAdapter;
|
|
157
|
+
/**
|
|
158
|
+
* Encode an Int16Array of PCM 16-bit samples (little-endian) as base64.
|
|
159
|
+
* Bun's Buffer is a polyfill; this implementation works in Node, Bun,
|
|
160
|
+
* and the browser when used with the global `btoa` fallback.
|
|
161
|
+
*/
|
|
162
|
+
export declare function encodePcm16Base64(pcm: Int16Array): string;
|
|
163
|
+
export declare function decodePcm16Base64(b64: string): Int16Array;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,450 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Catalog R16 `voice-runtime` — Section 24 VOICE.
|
|
3
|
+
*
|
|
4
|
+
* Provider-agnostic realtime audio adapter. The user-facing surface is
|
|
5
|
+
* a small interface — `RealtimeAdapter` — with one concrete adapter in
|
|
6
|
+
* v0 (`createOpenAIRealtimeAdapter`) and a Vapi stub that lazy-loads.
|
|
7
|
+
*
|
|
8
|
+
* Audio framing: every adapter normalises to 16-bit signed-int PCM at
|
|
9
|
+
* 24kHz mono. The OpenAI adapter speaks this format natively; the Vapi
|
|
10
|
+
* adapter (when implemented) will resample its native Opus stream
|
|
11
|
+
* through a server-side helper. Frames are passed as `Int16Array` so
|
|
12
|
+
* downstream consumers (vad-engine) don't have to re-decode bytes.
|
|
13
|
+
*
|
|
14
|
+
* Event surface:
|
|
15
|
+
* - `transcript_partial { text }` — incremental transcript chunks
|
|
16
|
+
* - `transcript_final { text }` — when the user's utterance ends
|
|
17
|
+
* - `audio_chunk { pcm }` — agent's TTS audio (pcm16 24kHz)
|
|
18
|
+
* - `tool_use { id, name, input }` — agent function calls
|
|
19
|
+
* - `interrupt { reason }` — emitted on barge-in
|
|
20
|
+
* - `disconnect { code?, reason? }` — final
|
|
21
|
+
*
|
|
22
|
+
* The OpenAI adapter is HEADLESS — no microphone, no speakers. The
|
|
23
|
+
* caller pumps PCM in via `sendAudio(pcm)`; the adapter emits audio +
|
|
24
|
+
* transcript events back. This keeps the runtime usable in three
|
|
25
|
+
* deployment modes:
|
|
26
|
+
* 1. Browser → daemon over WebRTC (browser has the mic/speaker).
|
|
27
|
+
* 2. SIP gateway → daemon (telephony provides PCM).
|
|
28
|
+
* 3. Node-only smoke (synthesize PCM in TS, pipe in, assert events).
|
|
29
|
+
*/
|
|
30
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
31
|
+
export class VoiceRuntimeError extends CrewhausError {
|
|
32
|
+
name = "VoiceRuntimeError";
|
|
33
|
+
constructor(message, cause) {
|
|
34
|
+
super("runtime", message, cause);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// OpenAI Realtime adapter — speaks the v1 WebSocket protocol.
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
const OPENAI_REALTIME_SAMPLE_RATE = 24_000;
|
|
41
|
+
const DEFAULT_OPENAI_REALTIME_MODEL = "gpt-4o-realtime-preview";
|
|
42
|
+
const DEFAULT_OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime";
|
|
43
|
+
export function createOpenAIRealtimeAdapter(opts = {}) {
|
|
44
|
+
let ws;
|
|
45
|
+
let connected = false;
|
|
46
|
+
let disconnected = false;
|
|
47
|
+
const handlers = new Set();
|
|
48
|
+
function emit(event) {
|
|
49
|
+
for (const h of handlers) {
|
|
50
|
+
try {
|
|
51
|
+
h(event);
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
/* subscribers must not crash the adapter */
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function send(message) {
|
|
59
|
+
if (ws === undefined || ws.readyState !== 1 /* OPEN */) {
|
|
60
|
+
throw new VoiceRuntimeError("adapter is not connected (or socket is not open)");
|
|
61
|
+
}
|
|
62
|
+
ws.send(JSON.stringify(message));
|
|
63
|
+
}
|
|
64
|
+
function handleServerEvent(raw) {
|
|
65
|
+
if (typeof raw !== "object" || raw === null)
|
|
66
|
+
return;
|
|
67
|
+
const ev = raw;
|
|
68
|
+
switch (ev.type) {
|
|
69
|
+
case "session.created": {
|
|
70
|
+
const sess = ev["session"];
|
|
71
|
+
emit({ kind: "session_created", sessionId: sess?.id ?? "(unknown)" });
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
case "response.created": {
|
|
75
|
+
// No-op for now; reserved for future response-id tracking when
|
|
76
|
+
// response.cancel needs the most recent response_id.
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
case "response.audio_transcript.delta": {
|
|
80
|
+
const delta = ev["delta"];
|
|
81
|
+
const itemId = ev["item_id"];
|
|
82
|
+
if (typeof delta === "string") {
|
|
83
|
+
emit({
|
|
84
|
+
kind: "transcript_partial",
|
|
85
|
+
text: delta,
|
|
86
|
+
...(typeof itemId === "string" ? { itemId } : {}),
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
case "response.audio_transcript.done": {
|
|
92
|
+
const transcript = ev["transcript"];
|
|
93
|
+
const itemId = ev["item_id"];
|
|
94
|
+
if (typeof transcript === "string") {
|
|
95
|
+
emit({
|
|
96
|
+
kind: "transcript_final",
|
|
97
|
+
text: transcript,
|
|
98
|
+
...(typeof itemId === "string" ? { itemId } : {}),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
case "response.audio.delta": {
|
|
104
|
+
const b64 = ev["delta"];
|
|
105
|
+
if (typeof b64 === "string") {
|
|
106
|
+
const pcm = decodePcm16Base64(b64);
|
|
107
|
+
emit({ kind: "audio_chunk", pcm, sampleRate: OPENAI_REALTIME_SAMPLE_RATE });
|
|
108
|
+
}
|
|
109
|
+
break;
|
|
110
|
+
}
|
|
111
|
+
case "response.function_call_arguments.done": {
|
|
112
|
+
const id = ev["call_id"];
|
|
113
|
+
const name = ev["name"];
|
|
114
|
+
const argsStr = ev["arguments"];
|
|
115
|
+
let parsed = {};
|
|
116
|
+
if (typeof argsStr === "string") {
|
|
117
|
+
try {
|
|
118
|
+
parsed = JSON.parse(argsStr);
|
|
119
|
+
}
|
|
120
|
+
catch {
|
|
121
|
+
parsed = argsStr;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
emit({
|
|
125
|
+
kind: "tool_use",
|
|
126
|
+
id: typeof id === "string" ? id : `call_${Date.now()}`,
|
|
127
|
+
name: typeof name === "string" ? name : "(unknown)",
|
|
128
|
+
input: parsed,
|
|
129
|
+
});
|
|
130
|
+
break;
|
|
131
|
+
}
|
|
132
|
+
case "response.cancelled":
|
|
133
|
+
case "response.cancel":
|
|
134
|
+
emit({ kind: "interrupt", reason: "response cancelled" });
|
|
135
|
+
break;
|
|
136
|
+
case "error": {
|
|
137
|
+
const errObj = ev["error"];
|
|
138
|
+
emit({ kind: "error", message: errObj?.message ?? "unknown error" });
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
default:
|
|
142
|
+
// unknown event — ignore for forward compat
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
providerId: "openai",
|
|
148
|
+
sampleRate: OPENAI_REALTIME_SAMPLE_RATE,
|
|
149
|
+
get connected() {
|
|
150
|
+
return connected && !disconnected;
|
|
151
|
+
},
|
|
152
|
+
async connect(connectOpts) {
|
|
153
|
+
if (connected) {
|
|
154
|
+
throw new VoiceRuntimeError("already connected");
|
|
155
|
+
}
|
|
156
|
+
const apiKey = opts.apiKey ?? process.env["OPENAI_API_KEY"];
|
|
157
|
+
if (!apiKey) {
|
|
158
|
+
throw new VoiceRuntimeError("OpenAI Realtime adapter: OPENAI_API_KEY is not set (pass `apiKey` or env)");
|
|
159
|
+
}
|
|
160
|
+
const baseUrl = opts.url ?? DEFAULT_OPENAI_REALTIME_URL;
|
|
161
|
+
const url = `${baseUrl}?model=${encodeURIComponent(connectOpts.model || DEFAULT_OPENAI_REALTIME_MODEL)}`;
|
|
162
|
+
const factory = opts._ws ??
|
|
163
|
+
((u, init) => new globalThis.WebSocket(u, init));
|
|
164
|
+
ws = factory(url, {
|
|
165
|
+
headers: {
|
|
166
|
+
Authorization: `Bearer ${apiKey}`,
|
|
167
|
+
"OpenAI-Beta": "realtime=v1",
|
|
168
|
+
},
|
|
169
|
+
});
|
|
170
|
+
const socket = ws;
|
|
171
|
+
await new Promise((resolve, reject) => {
|
|
172
|
+
const onOpen = () => {
|
|
173
|
+
connected = true;
|
|
174
|
+
resolve();
|
|
175
|
+
};
|
|
176
|
+
const onError = (ev) => {
|
|
177
|
+
const message = ev && typeof ev === "object" && "message" in ev
|
|
178
|
+
? String(ev.message)
|
|
179
|
+
: "websocket error";
|
|
180
|
+
if (!connected)
|
|
181
|
+
reject(new VoiceRuntimeError(`OpenAI Realtime: ${message}`));
|
|
182
|
+
emit({ kind: "error", message });
|
|
183
|
+
};
|
|
184
|
+
const onClose = (ev) => {
|
|
185
|
+
const close = ev;
|
|
186
|
+
disconnected = true;
|
|
187
|
+
emit({
|
|
188
|
+
kind: "disconnect",
|
|
189
|
+
...(typeof close.code === "number" ? { code: close.code } : {}),
|
|
190
|
+
...(typeof close.reason === "string" ? { reason: close.reason } : {}),
|
|
191
|
+
});
|
|
192
|
+
};
|
|
193
|
+
const onMessage = (ev) => {
|
|
194
|
+
const data = ev.data;
|
|
195
|
+
if (typeof data !== "string")
|
|
196
|
+
return;
|
|
197
|
+
try {
|
|
198
|
+
handleServerEvent(JSON.parse(data));
|
|
199
|
+
}
|
|
200
|
+
catch (err) {
|
|
201
|
+
emit({ kind: "error", message: `server event parse: ${err.message}` });
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
socket.addEventListener("open", onOpen);
|
|
205
|
+
socket.addEventListener("error", onError);
|
|
206
|
+
socket.addEventListener("close", onClose);
|
|
207
|
+
socket.addEventListener("message", onMessage);
|
|
208
|
+
});
|
|
209
|
+
// Configure session.
|
|
210
|
+
send({
|
|
211
|
+
type: "session.update",
|
|
212
|
+
session: {
|
|
213
|
+
modalities: ["audio", "text"],
|
|
214
|
+
...(connectOpts.instructions !== undefined
|
|
215
|
+
? { instructions: connectOpts.instructions }
|
|
216
|
+
: {}),
|
|
217
|
+
...(connectOpts.voice !== undefined ? { voice: connectOpts.voice } : {}),
|
|
218
|
+
input_audio_format: "pcm16",
|
|
219
|
+
output_audio_format: "pcm16",
|
|
220
|
+
turn_detection: connectOpts.vad === "none" ? null : { type: "server_vad" },
|
|
221
|
+
...(connectOpts.tools !== undefined && connectOpts.tools.length > 0
|
|
222
|
+
? {
|
|
223
|
+
tools: connectOpts.tools.map((t) => ({
|
|
224
|
+
type: "function",
|
|
225
|
+
name: t.name,
|
|
226
|
+
description: t.description,
|
|
227
|
+
parameters: t.inputSchema,
|
|
228
|
+
})),
|
|
229
|
+
}
|
|
230
|
+
: {}),
|
|
231
|
+
},
|
|
232
|
+
});
|
|
233
|
+
},
|
|
234
|
+
sendAudio(pcm) {
|
|
235
|
+
const b64 = encodePcm16Base64(pcm);
|
|
236
|
+
send({ type: "input_audio_buffer.append", audio: b64 });
|
|
237
|
+
},
|
|
238
|
+
commitInput() {
|
|
239
|
+
try {
|
|
240
|
+
send({ type: "input_audio_buffer.commit" });
|
|
241
|
+
send({ type: "response.create" });
|
|
242
|
+
}
|
|
243
|
+
catch {
|
|
244
|
+
/* tolerate not-connected */
|
|
245
|
+
}
|
|
246
|
+
},
|
|
247
|
+
sendText(text) {
|
|
248
|
+
send({
|
|
249
|
+
type: "conversation.item.create",
|
|
250
|
+
item: {
|
|
251
|
+
type: "message",
|
|
252
|
+
role: "user",
|
|
253
|
+
content: [{ type: "input_text", text }],
|
|
254
|
+
},
|
|
255
|
+
});
|
|
256
|
+
send({ type: "response.create" });
|
|
257
|
+
},
|
|
258
|
+
interrupt(reason) {
|
|
259
|
+
try {
|
|
260
|
+
send({ type: "response.cancel" });
|
|
261
|
+
}
|
|
262
|
+
catch {
|
|
263
|
+
/* tolerate cancel-not-supported edge cases */
|
|
264
|
+
}
|
|
265
|
+
emit({ kind: "interrupt", reason });
|
|
266
|
+
},
|
|
267
|
+
on(handler) {
|
|
268
|
+
handlers.add(handler);
|
|
269
|
+
return () => {
|
|
270
|
+
handlers.delete(handler);
|
|
271
|
+
};
|
|
272
|
+
},
|
|
273
|
+
async disconnect() {
|
|
274
|
+
if (disconnected)
|
|
275
|
+
return;
|
|
276
|
+
disconnected = true;
|
|
277
|
+
try {
|
|
278
|
+
ws?.close(1000, "client closed");
|
|
279
|
+
}
|
|
280
|
+
catch {
|
|
281
|
+
/* tolerate */
|
|
282
|
+
}
|
|
283
|
+
},
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
const DEFAULT_VAPI_REALTIME_URL = "wss://api.vapi.ai/v1/realtime";
|
|
287
|
+
/**
|
|
288
|
+
* Section 30 — Vapi realtime adapter. Maps to the same `RealtimeAdapter`
|
|
289
|
+
* interface as OpenAI Realtime so callers can swap providers via spec
|
|
290
|
+
* config. The connect handshake takes an `assistantId` (Vapi's preset)
|
|
291
|
+
* plus the same instructions/voice/temperature knobs the OpenAI path
|
|
292
|
+
* supports.
|
|
293
|
+
*
|
|
294
|
+
* Without `VAPI_API_KEY` set or an `_ws` injection, `connect()` throws
|
|
295
|
+
* with a clear message — production deployments enable this when the
|
|
296
|
+
* env var is present.
|
|
297
|
+
*/
|
|
298
|
+
export function createVapiRealtimeAdapter(opts = {}) {
|
|
299
|
+
const apiKey = opts.apiKey ?? process.env["VAPI_API_KEY"];
|
|
300
|
+
let ws;
|
|
301
|
+
let connected = false;
|
|
302
|
+
let disconnected = false;
|
|
303
|
+
const handlers = new Set();
|
|
304
|
+
function emit(event) {
|
|
305
|
+
for (const h of handlers) {
|
|
306
|
+
try {
|
|
307
|
+
h(event);
|
|
308
|
+
}
|
|
309
|
+
catch {
|
|
310
|
+
/* subscribers must not crash the adapter */
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
function send(message) {
|
|
315
|
+
if (ws === undefined || ws.readyState !== 1) {
|
|
316
|
+
throw new VoiceRuntimeError("vapi adapter is not connected (or socket is not open)");
|
|
317
|
+
}
|
|
318
|
+
ws.send(JSON.stringify(message));
|
|
319
|
+
}
|
|
320
|
+
return {
|
|
321
|
+
providerId: "vapi",
|
|
322
|
+
sampleRate: OPENAI_REALTIME_SAMPLE_RATE,
|
|
323
|
+
get connected() {
|
|
324
|
+
return connected;
|
|
325
|
+
},
|
|
326
|
+
async connect(connectOpts) {
|
|
327
|
+
if (!apiKey) {
|
|
328
|
+
throw new VoiceRuntimeError("voice-runtime: vapi adapter requires VAPI_API_KEY");
|
|
329
|
+
}
|
|
330
|
+
if (disconnected)
|
|
331
|
+
throw new VoiceRuntimeError("vapi adapter has been disconnected");
|
|
332
|
+
const wsFactory = opts._ws ??
|
|
333
|
+
((url, init) => new globalThis.WebSocket(url, init));
|
|
334
|
+
const url = opts.url ?? DEFAULT_VAPI_REALTIME_URL;
|
|
335
|
+
ws = wsFactory(url, {
|
|
336
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
337
|
+
});
|
|
338
|
+
await new Promise((resolve, reject) => {
|
|
339
|
+
if (!ws)
|
|
340
|
+
return reject(new VoiceRuntimeError("ws factory returned undefined"));
|
|
341
|
+
ws.addEventListener("open", () => {
|
|
342
|
+
connected = true;
|
|
343
|
+
// Vapi configuration handshake.
|
|
344
|
+
send({
|
|
345
|
+
type: "session.update",
|
|
346
|
+
session: {
|
|
347
|
+
voice: connectOpts.voice,
|
|
348
|
+
instructions: connectOpts.instructions,
|
|
349
|
+
...(opts.assistantId !== undefined ? { assistant_id: opts.assistantId } : {}),
|
|
350
|
+
},
|
|
351
|
+
});
|
|
352
|
+
resolve();
|
|
353
|
+
});
|
|
354
|
+
ws.addEventListener("error", (ev) => {
|
|
355
|
+
reject(new VoiceRuntimeError(`vapi ws error: ${String(ev?.message ?? ev)}`));
|
|
356
|
+
});
|
|
357
|
+
ws.addEventListener("close", () => {
|
|
358
|
+
connected = false;
|
|
359
|
+
});
|
|
360
|
+
ws.addEventListener("message", (ev) => {
|
|
361
|
+
const raw = ev.data ?? "";
|
|
362
|
+
try {
|
|
363
|
+
const parsed = JSON.parse(raw);
|
|
364
|
+
if (typeof parsed.type === "string") {
|
|
365
|
+
emit({
|
|
366
|
+
kind: "raw",
|
|
367
|
+
provider: "vapi",
|
|
368
|
+
ts: (opts._now ?? (() => new Date()))().toISOString(),
|
|
369
|
+
payload: parsed,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
catch {
|
|
374
|
+
/* swallow malformed frames */
|
|
375
|
+
}
|
|
376
|
+
});
|
|
377
|
+
});
|
|
378
|
+
},
|
|
379
|
+
sendAudio(pcm) {
|
|
380
|
+
// Vapi accepts base64-encoded PCM in `input_audio_buffer.append`.
|
|
381
|
+
const buf = Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
|
382
|
+
send({ type: "input_audio_buffer.append", audio: buf.toString("base64") });
|
|
383
|
+
},
|
|
384
|
+
commitInput() {
|
|
385
|
+
send({ type: "input_audio_buffer.commit" });
|
|
386
|
+
},
|
|
387
|
+
sendText(text) {
|
|
388
|
+
send({ type: "response.create", response: { modalities: ["text", "audio"], input: text } });
|
|
389
|
+
},
|
|
390
|
+
interrupt(_reason) {
|
|
391
|
+
try {
|
|
392
|
+
send({ type: "response.cancel" });
|
|
393
|
+
}
|
|
394
|
+
catch {
|
|
395
|
+
/* idempotent */
|
|
396
|
+
}
|
|
397
|
+
},
|
|
398
|
+
on(handler) {
|
|
399
|
+
handlers.add(handler);
|
|
400
|
+
return () => {
|
|
401
|
+
handlers.delete(handler);
|
|
402
|
+
};
|
|
403
|
+
},
|
|
404
|
+
async disconnect() {
|
|
405
|
+
if (ws) {
|
|
406
|
+
try {
|
|
407
|
+
ws.close();
|
|
408
|
+
}
|
|
409
|
+
catch {
|
|
410
|
+
/* best effort */
|
|
411
|
+
}
|
|
412
|
+
ws = undefined;
|
|
413
|
+
}
|
|
414
|
+
disconnected = true;
|
|
415
|
+
connected = false;
|
|
416
|
+
},
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Resolve a provider string to an adapter factory. Mirrors the
|
|
421
|
+
* `model-router` pattern — the spec carries `voice.provider: "openai" |
|
|
422
|
+
* "vapi"` and the daemon constructs the right adapter at boot.
|
|
423
|
+
*/
|
|
424
|
+
export function createRealtimeAdapter(provider, opts = {}) {
|
|
425
|
+
if (provider === "openai")
|
|
426
|
+
return createOpenAIRealtimeAdapter(opts);
|
|
427
|
+
if (provider === "vapi")
|
|
428
|
+
return createVapiRealtimeAdapter(opts);
|
|
429
|
+
throw new VoiceRuntimeError(`unknown realtime provider: ${provider}`);
|
|
430
|
+
}
|
|
431
|
+
// ---------------------------------------------------------------------------
|
|
432
|
+
// PCM ↔ base64 helpers. Exported because vad-engine + tests use them.
|
|
433
|
+
// ---------------------------------------------------------------------------
|
|
434
|
+
/**
|
|
435
|
+
* Encode an Int16Array of PCM 16-bit samples (little-endian) as base64.
|
|
436
|
+
* Bun's Buffer is a polyfill; this implementation works in Node, Bun,
|
|
437
|
+
* and the browser when used with the global `btoa` fallback.
|
|
438
|
+
*/
|
|
439
|
+
export function encodePcm16Base64(pcm) {
|
|
440
|
+
const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
|
441
|
+
return Buffer.from(bytes).toString("base64");
|
|
442
|
+
}
|
|
443
|
+
export function decodePcm16Base64(b64) {
|
|
444
|
+
const buf = Buffer.from(b64, "base64");
|
|
445
|
+
// Copy to a new ArrayBuffer aligned for Int16Array (Buffer's underlying
|
|
446
|
+
// ArrayBuffer can have an unaligned offset; alignment matters here).
|
|
447
|
+
const aligned = new ArrayBuffer(buf.byteLength);
|
|
448
|
+
new Uint8Array(aligned).set(buf);
|
|
449
|
+
return new Int16Array(aligned);
|
|
450
|
+
}
|
package/package.json
CHANGED
|
@@ -1,18 +1,21 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/voice-runtime",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Realtime voice loop — provider-agnostic RealtimeAdapter + OpenAI Realtime WebSocket adapter (Section 24 VOICE)",
|
|
6
|
-
"main": "
|
|
7
|
-
"types": "
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
8
|
"exports": {
|
|
9
|
-
".":
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js"
|
|
12
|
+
}
|
|
10
13
|
},
|
|
11
14
|
"scripts": {
|
|
12
15
|
"test": "bun test src"
|
|
13
16
|
},
|
|
14
17
|
"dependencies": {
|
|
15
|
-
"@crewhaus/errors": "0.1.
|
|
18
|
+
"@crewhaus/errors": "0.1.6"
|
|
16
19
|
},
|
|
17
20
|
"license": "Apache-2.0",
|
|
18
21
|
"author": {
|
|
@@ -32,5 +35,5 @@
|
|
|
32
35
|
"publishConfig": {
|
|
33
36
|
"access": "public"
|
|
34
37
|
},
|
|
35
|
-
"files": ["
|
|
38
|
+
"files": ["dist", "README.md", "LICENSE", "NOTICE"]
|
|
36
39
|
}
|