@crewhaus/voice-runtime 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts DELETED
@@ -1,577 +0,0 @@
1
- /**
2
- * Catalog R16 `voice-runtime` — Section 24 VOICE.
3
- *
4
- * Provider-agnostic realtime audio adapter. The user-facing surface is
5
- * a small interface — `RealtimeAdapter` — with one concrete adapter in
6
- * v0 (`createOpenAIRealtimeAdapter`) and a Vapi stub that lazy-loads.
7
- *
8
- * Audio framing: every adapter normalises to 16-bit signed-int PCM at
9
- * 24kHz mono. The OpenAI adapter speaks this format natively; the Vapi
10
- * adapter (when implemented) will resample its native Opus stream
11
- * through a server-side helper. Frames are passed as `Int16Array` so
12
- * downstream consumers (vad-engine) don't have to re-decode bytes.
13
- *
14
- * Event surface:
15
- * - `transcript_partial { text }` — incremental transcript chunks
16
- * - `transcript_final { text }` — when the user's utterance ends
17
- * - `audio_chunk { pcm }` — agent's TTS audio (pcm16 24kHz)
18
- * - `tool_use { id, name, input }` — agent function calls
19
- * - `interrupt { reason }` — emitted on barge-in
20
- * - `disconnect { code?, reason? }` — final
21
- *
22
- * The OpenAI adapter is HEADLESS — no microphone, no speakers. The
23
- * caller pumps PCM in via `sendAudio(pcm)`; the adapter emits audio +
24
- * transcript events back. This keeps the runtime usable in three
25
- * deployment modes:
26
- * 1. Browser → daemon over WebRTC (browser has the mic/speaker).
27
- * 2. SIP gateway → daemon (telephony provides PCM).
28
- * 3. Node-only smoke (synthesize PCM in TS, pipe in, assert events).
29
- */
30
- import { CrewhausError } from "@crewhaus/errors";
31
-
32
- export class VoiceRuntimeError extends CrewhausError {
33
- override readonly name = "VoiceRuntimeError";
34
- constructor(message: string, cause?: unknown) {
35
- super("runtime", message, cause);
36
- }
37
- }
38
-
39
- export type ProviderId = "openai" | "vapi";
40
-
41
- export type RealtimeEvent =
42
- | { kind: "session_created"; sessionId: string }
43
- | { kind: "transcript_partial"; text: string; itemId?: string }
44
- | { kind: "transcript_final"; text: string; itemId?: string }
45
- | { kind: "audio_chunk"; pcm: Int16Array; sampleRate: number }
46
- | { kind: "tool_use"; id: string; name: string; input: unknown }
47
- | { kind: "interrupt"; reason: string }
48
- | { kind: "disconnect"; code?: number; reason?: string }
49
- | { kind: "error"; message: string; cause?: unknown }
50
- /** Raw provider frame, surfaced verbatim (Vapi adapter passthrough). */
51
- | { kind: "raw"; provider: ProviderId; ts: string; payload: unknown };
52
-
53
- export type RealtimeEventHandler = (event: RealtimeEvent) => void;
54
-
55
- export type RealtimeConnectOptions = {
56
- readonly model: string;
57
- readonly instructions?: string;
58
- readonly voice?: string;
59
- /** Tools (Anthropic-style schemas) the model may call mid-conversation. */
60
- readonly tools?: ReadonlyArray<{
61
- readonly name: string;
62
- readonly description: string;
63
- readonly inputSchema: unknown;
64
- }>;
65
- /** Server-side VAD vs none. v0 defaults to "server" (OpenAI's built-in). */
66
- readonly vad?: "server" | "none";
67
- /** Max response audio length in seconds. */
68
- readonly maxResponseSeconds?: number;
69
- };
70
-
71
- export interface RealtimeAdapter {
72
- readonly providerId: ProviderId;
73
- readonly sampleRate: number;
74
- /** True after `connect()` resolves and before `disconnect()` is called. */
75
- readonly connected: boolean;
76
- connect(opts: RealtimeConnectOptions): Promise<void>;
77
- /** Push PCM 16-bit signed mono frames at the adapter's sample rate. */
78
- sendAudio(pcm: Int16Array): void;
79
- /**
80
- * Tell the server "this is the end of the user's utterance — process it
81
- * now". Used by callers that bypass server-side VAD (or when server VAD
82
- * timing is too slow for a one-shot smoke). Idempotent.
83
- */
84
- commitInput(): void;
85
- /** Inject a text turn (no audio). Useful for tests + chat-only mixed mode. */
86
- sendText(text: string): void;
87
- /** Cancel the in-flight response. Used by `barge-in-controller`. */
88
- interrupt(reason: string): void;
89
- on(handler: RealtimeEventHandler): () => void;
90
- disconnect(): Promise<void>;
91
- }
92
-
93
- // ---------------------------------------------------------------------------
94
- // OpenAI Realtime adapter — speaks the v1 WebSocket protocol.
95
- // ---------------------------------------------------------------------------
96
-
97
- const OPENAI_REALTIME_SAMPLE_RATE = 24_000;
98
- const DEFAULT_OPENAI_REALTIME_MODEL = "gpt-4o-realtime-preview";
99
- const DEFAULT_OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime";
100
-
101
- /** Test injection: a factory that returns a WebSocket-like (skips `new`). */
102
- export type WebSocketFactory = (
103
- url: string,
104
- init?: { headers?: Record<string, string> },
105
- ) => WebSocketLike;
106
-
107
- export type WebSocketLike = {
108
- readyState: number;
109
- send(data: string | ArrayBuffer): void;
110
- close(code?: number, reason?: string): void;
111
- addEventListener(
112
- type: "open" | "close" | "message" | "error",
113
- handler: (ev: unknown) => void,
114
- ): void;
115
- };
116
-
117
- export type CreateOpenAIRealtimeAdapterOptions = {
118
- readonly apiKey?: string;
119
- readonly url?: string;
120
- /** Override the URL→WebSocket factory; tests pass a stub. Default uses globalThis.WebSocket. */
121
- readonly _ws?: WebSocketFactory;
122
- readonly _now?: () => Date;
123
- };
124
-
125
- export function createOpenAIRealtimeAdapter(
126
- opts: CreateOpenAIRealtimeAdapterOptions = {},
127
- ): RealtimeAdapter {
128
- let ws: WebSocketLike | undefined;
129
- let connected = false;
130
- let disconnected = false;
131
- const handlers = new Set<RealtimeEventHandler>();
132
-
133
- function emit(event: RealtimeEvent): void {
134
- for (const h of handlers) {
135
- try {
136
- h(event);
137
- } catch {
138
- /* subscribers must not crash the adapter */
139
- }
140
- }
141
- }
142
-
143
- function send(message: Record<string, unknown>): void {
144
- if (ws === undefined || ws.readyState !== 1 /* OPEN */) {
145
- throw new VoiceRuntimeError("adapter is not connected (or socket is not open)");
146
- }
147
- ws.send(JSON.stringify(message));
148
- }
149
-
150
- function handleServerEvent(raw: unknown): void {
151
- if (typeof raw !== "object" || raw === null) return;
152
- const ev = raw as { type?: string; [k: string]: unknown };
153
- switch (ev.type) {
154
- case "session.created": {
155
- const sess = ev["session"] as { id?: string } | undefined;
156
- emit({ kind: "session_created", sessionId: sess?.id ?? "(unknown)" });
157
- break;
158
- }
159
- case "response.created": {
160
- // No-op for now; reserved for future response-id tracking when
161
- // response.cancel needs the most recent response_id.
162
- break;
163
- }
164
- case "response.audio_transcript.delta": {
165
- const delta = ev["delta"];
166
- const itemId = ev["item_id"];
167
- if (typeof delta === "string") {
168
- emit({
169
- kind: "transcript_partial",
170
- text: delta,
171
- ...(typeof itemId === "string" ? { itemId } : {}),
172
- });
173
- }
174
- break;
175
- }
176
- case "response.audio_transcript.done": {
177
- const transcript = ev["transcript"];
178
- const itemId = ev["item_id"];
179
- if (typeof transcript === "string") {
180
- emit({
181
- kind: "transcript_final",
182
- text: transcript,
183
- ...(typeof itemId === "string" ? { itemId } : {}),
184
- });
185
- }
186
- break;
187
- }
188
- case "response.audio.delta": {
189
- const b64 = ev["delta"];
190
- if (typeof b64 === "string") {
191
- const pcm = decodePcm16Base64(b64);
192
- emit({ kind: "audio_chunk", pcm, sampleRate: OPENAI_REALTIME_SAMPLE_RATE });
193
- }
194
- break;
195
- }
196
- case "response.function_call_arguments.done": {
197
- const id = ev["call_id"];
198
- const name = ev["name"];
199
- const argsStr = ev["arguments"];
200
- let parsed: unknown = {};
201
- if (typeof argsStr === "string") {
202
- try {
203
- parsed = JSON.parse(argsStr);
204
- } catch {
205
- parsed = argsStr;
206
- }
207
- }
208
- emit({
209
- kind: "tool_use",
210
- id: typeof id === "string" ? id : `call_${Date.now()}`,
211
- name: typeof name === "string" ? name : "(unknown)",
212
- input: parsed,
213
- });
214
- break;
215
- }
216
- case "response.cancelled":
217
- case "response.cancel":
218
- emit({ kind: "interrupt", reason: "response cancelled" });
219
- break;
220
- case "error": {
221
- const errObj = ev["error"] as { message?: string } | undefined;
222
- emit({ kind: "error", message: errObj?.message ?? "unknown error" });
223
- break;
224
- }
225
- default:
226
- // unknown event — ignore for forward compat
227
- break;
228
- }
229
- }
230
-
231
- return {
232
- providerId: "openai",
233
- sampleRate: OPENAI_REALTIME_SAMPLE_RATE,
234
-
235
- get connected() {
236
- return connected && !disconnected;
237
- },
238
-
239
- async connect(connectOpts) {
240
- if (connected) {
241
- throw new VoiceRuntimeError("already connected");
242
- }
243
- const apiKey = opts.apiKey ?? process.env["OPENAI_API_KEY"];
244
- if (!apiKey) {
245
- throw new VoiceRuntimeError(
246
- "OpenAI Realtime adapter: OPENAI_API_KEY is not set (pass `apiKey` or env)",
247
- );
248
- }
249
- const baseUrl = opts.url ?? DEFAULT_OPENAI_REALTIME_URL;
250
- const url = `${baseUrl}?model=${encodeURIComponent(connectOpts.model || DEFAULT_OPENAI_REALTIME_MODEL)}`;
251
- const factory: WebSocketFactory =
252
- opts._ws ??
253
- ((u, init) =>
254
- new (
255
- globalThis.WebSocket as unknown as new (
256
- url: string,
257
- protocolsOrInit?: string | string[] | { headers?: Record<string, string> },
258
- ) => WebSocketLike
259
- )(u, init));
260
- ws = factory(url, {
261
- headers: {
262
- Authorization: `Bearer ${apiKey}`,
263
- "OpenAI-Beta": "realtime=v1",
264
- },
265
- });
266
-
267
- const socket = ws;
268
- await new Promise<void>((resolve, reject) => {
269
- const onOpen = (): void => {
270
- connected = true;
271
- resolve();
272
- };
273
- const onError = (ev: unknown): void => {
274
- const message =
275
- ev && typeof ev === "object" && "message" in ev
276
- ? String((ev as { message: unknown }).message)
277
- : "websocket error";
278
- if (!connected) reject(new VoiceRuntimeError(`OpenAI Realtime: ${message}`));
279
- emit({ kind: "error", message });
280
- };
281
- const onClose = (ev: unknown): void => {
282
- const close = ev as { code?: number; reason?: string };
283
- disconnected = true;
284
- emit({
285
- kind: "disconnect",
286
- ...(typeof close.code === "number" ? { code: close.code } : {}),
287
- ...(typeof close.reason === "string" ? { reason: close.reason } : {}),
288
- });
289
- };
290
- const onMessage = (ev: unknown): void => {
291
- const data = (ev as { data?: unknown }).data;
292
- if (typeof data !== "string") return;
293
- try {
294
- handleServerEvent(JSON.parse(data));
295
- } catch (err) {
296
- emit({ kind: "error", message: `server event parse: ${(err as Error).message}` });
297
- }
298
- };
299
- socket.addEventListener("open", onOpen);
300
- socket.addEventListener("error", onError);
301
- socket.addEventListener("close", onClose);
302
- socket.addEventListener("message", onMessage);
303
- });
304
-
305
- // Configure session.
306
- send({
307
- type: "session.update",
308
- session: {
309
- modalities: ["audio", "text"],
310
- ...(connectOpts.instructions !== undefined
311
- ? { instructions: connectOpts.instructions }
312
- : {}),
313
- ...(connectOpts.voice !== undefined ? { voice: connectOpts.voice } : {}),
314
- input_audio_format: "pcm16",
315
- output_audio_format: "pcm16",
316
- turn_detection: connectOpts.vad === "none" ? null : { type: "server_vad" },
317
- ...(connectOpts.tools !== undefined && connectOpts.tools.length > 0
318
- ? {
319
- tools: connectOpts.tools.map((t) => ({
320
- type: "function",
321
- name: t.name,
322
- description: t.description,
323
- parameters: t.inputSchema,
324
- })),
325
- }
326
- : {}),
327
- },
328
- });
329
- },
330
-
331
- sendAudio(pcm) {
332
- const b64 = encodePcm16Base64(pcm);
333
- send({ type: "input_audio_buffer.append", audio: b64 });
334
- },
335
-
336
- commitInput() {
337
- try {
338
- send({ type: "input_audio_buffer.commit" });
339
- send({ type: "response.create" });
340
- } catch {
341
- /* tolerate not-connected */
342
- }
343
- },
344
-
345
- sendText(text) {
346
- send({
347
- type: "conversation.item.create",
348
- item: {
349
- type: "message",
350
- role: "user",
351
- content: [{ type: "input_text", text }],
352
- },
353
- });
354
- send({ type: "response.create" });
355
- },
356
-
357
- interrupt(reason) {
358
- try {
359
- send({ type: "response.cancel" });
360
- } catch {
361
- /* tolerate cancel-not-supported edge cases */
362
- }
363
- emit({ kind: "interrupt", reason });
364
- },
365
-
366
- on(handler) {
367
- handlers.add(handler);
368
- return () => {
369
- handlers.delete(handler);
370
- };
371
- },
372
-
373
- async disconnect() {
374
- if (disconnected) return;
375
- disconnected = true;
376
- try {
377
- ws?.close(1000, "client closed");
378
- } catch {
379
- /* tolerate */
380
- }
381
- },
382
- };
383
- }
384
-
385
- // ---------------------------------------------------------------------------
386
- // Vapi adapter — stub that lazy-loads. Throws a clean diagnostic when invoked
387
- // without a VAPI_API_KEY or before the package is implemented.
388
- // ---------------------------------------------------------------------------
389
-
390
- export type CreateVapiRealtimeAdapterOptions = {
391
- readonly apiKey?: string;
392
- readonly assistantId?: string;
393
- /** Test injection — same shape as OpenAI's. */
394
- readonly _ws?: WebSocketFactory;
395
- readonly _now?: () => Date;
396
- /** Override the WebSocket URL. Default: `wss://api.vapi.ai/v1/realtime`. */
397
- readonly url?: string;
398
- };
399
-
400
- const DEFAULT_VAPI_REALTIME_URL = "wss://api.vapi.ai/v1/realtime";
401
-
402
- /**
403
- * Section 30 — Vapi realtime adapter. Maps to the same `RealtimeAdapter`
404
- * interface as OpenAI Realtime so callers can swap providers via spec
405
- * config. The connect handshake takes an `assistantId` (Vapi's preset)
406
- * plus the same instructions/voice/temperature knobs the OpenAI path
407
- * supports.
408
- *
409
- * Without `VAPI_API_KEY` set or an `_ws` injection, `connect()` throws
410
- * with a clear message — production deployments enable this when the
411
- * env var is present.
412
- */
413
- export function createVapiRealtimeAdapter(
414
- opts: CreateVapiRealtimeAdapterOptions = {},
415
- ): RealtimeAdapter {
416
- const apiKey = opts.apiKey ?? process.env["VAPI_API_KEY"];
417
- let ws: WebSocketLike | undefined;
418
- let connected = false;
419
- let disconnected = false;
420
- const handlers = new Set<RealtimeEventHandler>();
421
-
422
- function emit(event: RealtimeEvent): void {
423
- for (const h of handlers) {
424
- try {
425
- h(event);
426
- } catch {
427
- /* subscribers must not crash the adapter */
428
- }
429
- }
430
- }
431
-
432
- function send(message: Record<string, unknown>): void {
433
- if (ws === undefined || ws.readyState !== 1) {
434
- throw new VoiceRuntimeError("vapi adapter is not connected (or socket is not open)");
435
- }
436
- ws.send(JSON.stringify(message));
437
- }
438
-
439
- return {
440
- providerId: "vapi",
441
- sampleRate: OPENAI_REALTIME_SAMPLE_RATE,
442
- get connected() {
443
- return connected;
444
- },
445
- async connect(connectOpts: RealtimeConnectOptions): Promise<void> {
446
- if (!apiKey) {
447
- throw new VoiceRuntimeError("voice-runtime: vapi adapter requires VAPI_API_KEY");
448
- }
449
- if (disconnected) throw new VoiceRuntimeError("vapi adapter has been disconnected");
450
- const wsFactory =
451
- opts._ws ??
452
- (((url: string, init: { headers?: Record<string, string> }) =>
453
- new (
454
- globalThis as { WebSocket: new (u: string, init?: unknown) => WebSocketLike }
455
- ).WebSocket(url, init) as unknown as WebSocketLike) as WebSocketFactory);
456
- const url = opts.url ?? DEFAULT_VAPI_REALTIME_URL;
457
- ws = wsFactory(url, {
458
- headers: { Authorization: `Bearer ${apiKey}` },
459
- });
460
- await new Promise<void>((resolve, reject) => {
461
- if (!ws) return reject(new VoiceRuntimeError("ws factory returned undefined"));
462
- ws.addEventListener("open", () => {
463
- connected = true;
464
- // Vapi configuration handshake.
465
- send({
466
- type: "session.update",
467
- session: {
468
- voice: connectOpts.voice,
469
- instructions: connectOpts.instructions,
470
- ...(opts.assistantId !== undefined ? { assistant_id: opts.assistantId } : {}),
471
- },
472
- });
473
- resolve();
474
- });
475
- ws.addEventListener("error", (ev) => {
476
- reject(
477
- new VoiceRuntimeError(
478
- `vapi ws error: ${String((ev as { message?: string })?.message ?? ev)}`,
479
- ),
480
- );
481
- });
482
- ws.addEventListener("close", () => {
483
- connected = false;
484
- });
485
- ws.addEventListener("message", (ev) => {
486
- const raw = (ev as { data?: string }).data ?? "";
487
- try {
488
- const parsed = JSON.parse(raw) as { type?: string };
489
- if (typeof parsed.type === "string") {
490
- emit({
491
- kind: "raw",
492
- provider: "vapi",
493
- ts: (opts._now ?? ((): Date => new Date()))().toISOString(),
494
- payload: parsed,
495
- });
496
- }
497
- } catch {
498
- /* swallow malformed frames */
499
- }
500
- });
501
- });
502
- },
503
- sendAudio(pcm: Int16Array): void {
504
- // Vapi accepts base64-encoded PCM in `input_audio_buffer.append`.
505
- const buf = Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength);
506
- send({ type: "input_audio_buffer.append", audio: buf.toString("base64") });
507
- },
508
- commitInput(): void {
509
- send({ type: "input_audio_buffer.commit" });
510
- },
511
- sendText(text: string): void {
512
- send({ type: "response.create", response: { modalities: ["text", "audio"], input: text } });
513
- },
514
- interrupt(_reason: string): void {
515
- try {
516
- send({ type: "response.cancel" });
517
- } catch {
518
- /* idempotent */
519
- }
520
- },
521
- on(handler): () => void {
522
- handlers.add(handler);
523
- return () => {
524
- handlers.delete(handler);
525
- };
526
- },
527
- async disconnect(): Promise<void> {
528
- if (ws) {
529
- try {
530
- ws.close();
531
- } catch {
532
- /* best effort */
533
- }
534
- ws = undefined;
535
- }
536
- disconnected = true;
537
- connected = false;
538
- },
539
- };
540
- }
541
-
542
- /**
543
- * Resolve a provider string to an adapter factory. Mirrors the
544
- * `model-router` pattern — the spec carries `voice.provider: "openai" |
545
- * "vapi"` and the daemon constructs the right adapter at boot.
546
- */
547
- export function createRealtimeAdapter(
548
- provider: ProviderId,
549
- opts: CreateOpenAIRealtimeAdapterOptions = {},
550
- ): RealtimeAdapter {
551
- if (provider === "openai") return createOpenAIRealtimeAdapter(opts);
552
- if (provider === "vapi") return createVapiRealtimeAdapter(opts);
553
- throw new VoiceRuntimeError(`unknown realtime provider: ${provider}`);
554
- }
555
-
556
- // ---------------------------------------------------------------------------
557
- // PCM ↔ base64 helpers. Exported because vad-engine + tests use them.
558
- // ---------------------------------------------------------------------------
559
-
560
- /**
561
- * Encode an Int16Array of PCM 16-bit samples (little-endian) as base64.
562
- * Bun's Buffer is a polyfill; this implementation works in Node, Bun,
563
- * and the browser when used with the global `btoa` fallback.
564
- */
565
- export function encodePcm16Base64(pcm: Int16Array): string {
566
- const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
567
- return Buffer.from(bytes).toString("base64");
568
- }
569
-
570
- export function decodePcm16Base64(b64: string): Int16Array {
571
- const buf = Buffer.from(b64, "base64");
572
- // Copy to a new ArrayBuffer aligned for Int16Array (Buffer's underlying
573
- // ArrayBuffer can have an unaligned offset; alignment matters here).
574
- const aligned = new ArrayBuffer(buf.byteLength);
575
- new Uint8Array(aligned).set(buf);
576
- return new Int16Array(aligned);
577
- }