@cogitator-ai/voice 0.1.13 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -77
- package/dist/audio.d.ts +7 -0
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +40 -3
- package/dist/audio.js.map +1 -1
- package/dist/cogitator-runner.d.ts +34 -0
- package/dist/cogitator-runner.d.ts.map +1 -0
- package/dist/cogitator-runner.js +28 -0
- package/dist/cogitator-runner.js.map +1 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -4
- package/dist/index.js.map +1 -1
- package/dist/pipeline/index.d.ts +2 -2
- package/dist/pipeline/index.d.ts.map +1 -1
- package/dist/pipeline/index.js.map +1 -1
- package/dist/pipeline/pipeline-session.d.ts +20 -2
- package/dist/pipeline/pipeline-session.d.ts.map +1 -1
- package/dist/pipeline/pipeline-session.js +187 -63
- package/dist/pipeline/pipeline-session.js.map +1 -1
- package/dist/pipeline/voice-pipeline.d.ts +12 -7
- package/dist/pipeline/voice-pipeline.d.ts.map +1 -1
- package/dist/pipeline/voice-pipeline.js +14 -4
- package/dist/pipeline/voice-pipeline.js.map +1 -1
- package/dist/realtime/gemini-realtime.d.ts +15 -3
- package/dist/realtime/gemini-realtime.d.ts.map +1 -1
- package/dist/realtime/gemini-realtime.js +140 -94
- package/dist/realtime/gemini-realtime.js.map +1 -1
- package/dist/realtime/openai-realtime.d.ts +10 -1
- package/dist/realtime/openai-realtime.d.ts.map +1 -1
- package/dist/realtime/openai-realtime.js +140 -69
- package/dist/realtime/openai-realtime.js.map +1 -1
- package/dist/realtime/realtime-session.d.ts +10 -0
- package/dist/realtime/realtime-session.d.ts.map +1 -1
- package/dist/realtime/realtime-session.js +60 -7
- package/dist/realtime/realtime-session.js.map +1 -1
- package/dist/stt/deepgram-stt.d.ts +3 -0
- package/dist/stt/deepgram-stt.d.ts.map +1 -1
- package/dist/stt/deepgram-stt.js +104 -55
- package/dist/stt/deepgram-stt.js.map +1 -1
- package/dist/stt/openai-stt.d.ts +5 -0
- package/dist/stt/openai-stt.d.ts.map +1 -1
- package/dist/stt/openai-stt.js +70 -13
- package/dist/stt/openai-stt.js.map +1 -1
- package/dist/tools.d.ts +3 -3
- package/dist/transport/ws-transport.d.ts +6 -7
- package/dist/transport/ws-transport.d.ts.map +1 -1
- package/dist/transport/ws-transport.js +58 -20
- package/dist/transport/ws-transport.js.map +1 -1
- package/dist/tts/elevenlabs-tts.d.ts.map +1 -1
- package/dist/tts/elevenlabs-tts.js +26 -16
- package/dist/tts/elevenlabs-tts.js.map +1 -1
- package/dist/types.d.ts +38 -16
- package/dist/types.d.ts.map +1 -1
- package/dist/vad/silero-vad.d.ts +14 -1
- package/dist/vad/silero-vad.d.ts.map +1 -1
- package/dist/vad/silero-vad.js +101 -25
- package/dist/vad/silero-vad.js.map +1 -1
- package/dist/voice-agent.d.ts +14 -0
- package/dist/voice-agent.d.ts.map +1 -1
- package/dist/voice-agent.js +112 -23
- package/dist/voice-agent.js.map +1 -1
- package/package.json +5 -5
|
@@ -1,73 +1,95 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import { WebSocket } from 'ws';
|
|
3
|
-
const DEFAULT_MODEL = 'gpt-
|
|
3
|
+
const DEFAULT_MODEL = 'gpt-realtime-mini';
|
|
4
|
+
const DEFAULT_VOICE = 'marin';
|
|
5
|
+
const DEFAULT_TRANSCRIPTION_MODEL = 'gpt-4o-mini-transcribe';
|
|
4
6
|
const BASE_URL = 'wss://api.openai.com/v1/realtime';
|
|
5
7
|
const CONNECT_TIMEOUT_MS = 30_000;
|
|
8
|
+
const PCM_FORMAT = { type: 'audio/pcm', rate: 24000 };
|
|
9
|
+
function rawDataToString(data) {
|
|
10
|
+
if (Array.isArray(data))
|
|
11
|
+
return Buffer.concat(data).toString();
|
|
12
|
+
if (data instanceof ArrayBuffer)
|
|
13
|
+
return Buffer.from(new Uint8Array(data)).toString();
|
|
14
|
+
return data.toString();
|
|
15
|
+
}
|
|
16
|
+
function isRecord(value) {
|
|
17
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* OpenAI Realtime API (GA interface) adapter. Audio in both directions is
|
|
21
|
+
* PCM16 mono at 24kHz.
|
|
22
|
+
*/
|
|
6
23
|
export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
7
24
|
config;
|
|
8
25
|
model;
|
|
9
26
|
ws = null;
|
|
27
|
+
connected = false;
|
|
28
|
+
responseActive = false;
|
|
10
29
|
constructor(config) {
|
|
11
30
|
super();
|
|
12
31
|
this.config = config;
|
|
13
32
|
this.model = config.model ?? DEFAULT_MODEL;
|
|
14
33
|
}
|
|
34
|
+
get isConnected() {
|
|
35
|
+
return this.connected;
|
|
36
|
+
}
|
|
15
37
|
async connect() {
|
|
16
38
|
if (this.ws) {
|
|
17
39
|
throw new Error('Already connected or connecting — call close() first');
|
|
18
40
|
}
|
|
19
|
-
const url = `${BASE_URL}?model=${this.model}`;
|
|
41
|
+
const url = `${BASE_URL}?model=${encodeURIComponent(this.model)}`;
|
|
20
42
|
return new Promise((resolve, reject) => {
|
|
21
43
|
let settled = false;
|
|
22
|
-
const
|
|
23
|
-
if (settled)
|
|
24
|
-
return;
|
|
25
|
-
settled = true;
|
|
26
|
-
clearTimeout(timer);
|
|
27
|
-
resolve();
|
|
28
|
-
};
|
|
29
|
-
const settleReject = (err) => {
|
|
44
|
+
const settle = (err) => {
|
|
30
45
|
if (settled)
|
|
31
46
|
return;
|
|
32
47
|
settled = true;
|
|
33
48
|
clearTimeout(timer);
|
|
34
|
-
|
|
49
|
+
if (err)
|
|
50
|
+
reject(err);
|
|
51
|
+
else
|
|
52
|
+
resolve();
|
|
35
53
|
};
|
|
36
54
|
const ws = new WebSocket(url, {
|
|
37
|
-
headers: {
|
|
38
|
-
Authorization: `Bearer ${this.config.apiKey}`,
|
|
39
|
-
'OpenAI-Beta': 'realtime=v1',
|
|
40
|
-
},
|
|
55
|
+
headers: { Authorization: `Bearer ${this.config.apiKey}` },
|
|
41
56
|
});
|
|
42
57
|
this.ws = ws;
|
|
43
58
|
const timer = setTimeout(() => {
|
|
44
|
-
this.ws
|
|
59
|
+
if (this.ws === ws)
|
|
60
|
+
this.ws = null;
|
|
45
61
|
ws.removeAllListeners();
|
|
46
|
-
ws.
|
|
47
|
-
|
|
62
|
+
ws.on('error', () => { });
|
|
63
|
+
ws.terminate();
|
|
64
|
+
settle(new Error(`Connect timed out after ${CONNECT_TIMEOUT_MS}ms`));
|
|
48
65
|
}, CONNECT_TIMEOUT_MS);
|
|
49
66
|
ws.on('open', () => {
|
|
67
|
+
this.connected = true;
|
|
50
68
|
this.sendSessionUpdate();
|
|
51
69
|
this.emit('connected');
|
|
52
|
-
|
|
70
|
+
settle();
|
|
53
71
|
});
|
|
54
72
|
ws.on('message', (data) => {
|
|
55
|
-
|
|
56
|
-
? Buffer.concat(data)
|
|
57
|
-
: data instanceof ArrayBuffer
|
|
58
|
-
? Buffer.from(new Uint8Array(data))
|
|
59
|
-
: data;
|
|
60
|
-
this.handleMessage(buf.toString());
|
|
73
|
+
this.handleMessage(rawDataToString(data));
|
|
61
74
|
});
|
|
62
75
|
ws.on('error', (err) => {
|
|
63
|
-
|
|
64
|
-
|
|
76
|
+
if (settled) {
|
|
77
|
+
this.emit('error', err);
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
settle(err);
|
|
81
|
+
}
|
|
65
82
|
});
|
|
66
83
|
ws.on('close', (code, reason) => {
|
|
67
84
|
const reasonStr = reason.toString() || 'connection closed';
|
|
68
|
-
this.ws
|
|
69
|
-
|
|
70
|
-
|
|
85
|
+
if (this.ws === ws)
|
|
86
|
+
this.ws = null;
|
|
87
|
+
const wasConnected = this.connected;
|
|
88
|
+
this.connected = false;
|
|
89
|
+
this.responseActive = false;
|
|
90
|
+
settle(new Error(`WebSocket closed before connect (code ${code}): ${reasonStr}`));
|
|
91
|
+
if (wasConnected)
|
|
92
|
+
this.emit('disconnected', code, reasonStr);
|
|
71
93
|
});
|
|
72
94
|
});
|
|
73
95
|
}
|
|
@@ -89,23 +111,36 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
89
111
|
this.send({ type: 'response.create' });
|
|
90
112
|
}
|
|
91
113
|
interrupt() {
|
|
114
|
+
if (!this.responseActive)
|
|
115
|
+
return;
|
|
92
116
|
this.send({ type: 'response.cancel' });
|
|
93
117
|
}
|
|
94
118
|
close() {
|
|
95
119
|
const ws = this.ws;
|
|
120
|
+
this.connected = false;
|
|
121
|
+
this.responseActive = false;
|
|
96
122
|
if (!ws)
|
|
97
123
|
return;
|
|
98
124
|
this.ws = null;
|
|
99
125
|
ws.removeAllListeners();
|
|
126
|
+
ws.on('error', () => { });
|
|
100
127
|
ws.close();
|
|
101
128
|
}
|
|
102
129
|
sendSessionUpdate() {
|
|
103
130
|
const session = {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
131
|
+
type: 'realtime',
|
|
132
|
+
output_modalities: ['audio'],
|
|
133
|
+
audio: {
|
|
134
|
+
input: {
|
|
135
|
+
format: PCM_FORMAT,
|
|
136
|
+
transcription: { model: DEFAULT_TRANSCRIPTION_MODEL },
|
|
137
|
+
turn_detection: { type: 'server_vad' },
|
|
138
|
+
},
|
|
139
|
+
output: {
|
|
140
|
+
format: PCM_FORMAT,
|
|
141
|
+
voice: this.config.voice ?? DEFAULT_VOICE,
|
|
142
|
+
},
|
|
143
|
+
},
|
|
109
144
|
};
|
|
110
145
|
if (this.config.instructions) {
|
|
111
146
|
session.instructions = this.config.instructions;
|
|
@@ -117,24 +152,27 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
117
152
|
description: t.description,
|
|
118
153
|
parameters: t.parameters,
|
|
119
154
|
}));
|
|
155
|
+
session.tool_choice = 'auto';
|
|
120
156
|
}
|
|
121
157
|
this.send({ type: 'session.update', session });
|
|
122
158
|
}
|
|
123
159
|
handleMessage(raw) {
|
|
124
|
-
let
|
|
160
|
+
let parsed;
|
|
125
161
|
try {
|
|
126
|
-
|
|
162
|
+
parsed = JSON.parse(raw);
|
|
127
163
|
}
|
|
128
164
|
catch {
|
|
129
165
|
this.emit('error', new Error('Failed to parse WebSocket message'));
|
|
130
166
|
return;
|
|
131
167
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
168
|
+
if (!isRecord(parsed))
|
|
169
|
+
return;
|
|
170
|
+
const event = parsed;
|
|
171
|
+
switch (event.type) {
|
|
172
|
+
case 'response.output_audio.delta': {
|
|
135
173
|
const delta = event.delta;
|
|
136
174
|
if (typeof delta !== 'string') {
|
|
137
|
-
this.emit('error', new Error('Malformed response.
|
|
175
|
+
this.emit('error', new Error('Malformed response.output_audio.delta: missing or invalid delta'));
|
|
138
176
|
break;
|
|
139
177
|
}
|
|
140
178
|
this.emit('audio', Buffer.from(delta, 'base64'));
|
|
@@ -145,7 +183,7 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
145
183
|
this.emit('transcript', event.transcript, 'user');
|
|
146
184
|
}
|
|
147
185
|
break;
|
|
148
|
-
case 'response.
|
|
186
|
+
case 'response.output_audio_transcript.done':
|
|
149
187
|
if (typeof event.transcript === 'string') {
|
|
150
188
|
this.emit('transcript', event.transcript, 'assistant');
|
|
151
189
|
}
|
|
@@ -153,54 +191,87 @@ export class OpenAIRealtimeAdapter extends EventEmitter {
|
|
|
153
191
|
case 'input_audio_buffer.speech_started':
|
|
154
192
|
this.emit('speech_start');
|
|
155
193
|
break;
|
|
156
|
-
case 'response.
|
|
157
|
-
|
|
194
|
+
case 'response.created':
|
|
195
|
+
this.responseActive = true;
|
|
158
196
|
break;
|
|
159
197
|
case 'response.done':
|
|
160
|
-
this.
|
|
198
|
+
this.responseActive = false;
|
|
199
|
+
this.handleResponseDone(event.response);
|
|
161
200
|
break;
|
|
162
201
|
case 'error': {
|
|
163
202
|
const errObj = event.error;
|
|
164
|
-
const message = errObj && typeof errObj === '
|
|
165
|
-
?
|
|
203
|
+
const message = isRecord(errObj) && typeof errObj.message === 'string'
|
|
204
|
+
? errObj.message
|
|
166
205
|
: 'Unknown server error';
|
|
167
206
|
this.emit('error', new Error(message));
|
|
168
207
|
break;
|
|
169
208
|
}
|
|
170
209
|
}
|
|
171
210
|
}
|
|
172
|
-
|
|
173
|
-
const
|
|
174
|
-
|
|
211
|
+
handleResponseDone(response) {
|
|
212
|
+
const calls = [];
|
|
213
|
+
if (isRecord(response) && response.status === 'completed' && Array.isArray(response.output)) {
|
|
214
|
+
for (const item of response.output) {
|
|
215
|
+
if (isRecord(item) &&
|
|
216
|
+
item.type === 'function_call' &&
|
|
217
|
+
typeof item.name === 'string' &&
|
|
218
|
+
typeof item.call_id === 'string') {
|
|
219
|
+
calls.push({
|
|
220
|
+
name: item.name,
|
|
221
|
+
callId: item.call_id,
|
|
222
|
+
arguments: typeof item.arguments === 'string' ? item.arguments : '{}',
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
if (calls.length === 0) {
|
|
228
|
+
this.emit('turn_end');
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
void this.handleToolCalls(calls);
|
|
232
|
+
}
|
|
233
|
+
async handleToolCalls(calls) {
|
|
234
|
+
const ws = this.ws;
|
|
235
|
+
const outputs = await Promise.all(calls.map((call) => this.executeToolCall(call)));
|
|
236
|
+
if (this.ws !== ws)
|
|
237
|
+
return;
|
|
238
|
+
for (const { callId, output } of outputs) {
|
|
239
|
+
this.send({
|
|
240
|
+
type: 'conversation.item.create',
|
|
241
|
+
item: { type: 'function_call_output', call_id: callId, output },
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
this.send({ type: 'response.create' });
|
|
245
|
+
}
|
|
246
|
+
async executeToolCall(call) {
|
|
175
247
|
let args;
|
|
176
248
|
try {
|
|
177
|
-
args = JSON.parse(
|
|
249
|
+
args = JSON.parse(call.arguments || '{}');
|
|
178
250
|
}
|
|
179
251
|
catch {
|
|
180
|
-
|
|
181
|
-
|
|
252
|
+
return {
|
|
253
|
+
callId: call.callId,
|
|
254
|
+
output: JSON.stringify({ error: `Invalid JSON arguments for tool ${call.name}` }),
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
this.emit('tool_call', call.name, args);
|
|
258
|
+
const tool = this.config.tools?.find((t) => t.name === call.name);
|
|
259
|
+
if (!tool) {
|
|
260
|
+
return {
|
|
261
|
+
callId: call.callId,
|
|
262
|
+
output: JSON.stringify({ error: `Unknown tool: ${call.name}` }),
|
|
263
|
+
};
|
|
182
264
|
}
|
|
183
|
-
this.emit('tool_call', name, args);
|
|
184
|
-
const tool = this.config.tools?.find((t) => t.name === name);
|
|
185
|
-
if (!tool)
|
|
186
|
-
return;
|
|
187
|
-
let output;
|
|
188
265
|
try {
|
|
189
266
|
const result = await tool.execute(args);
|
|
190
|
-
output
|
|
267
|
+
return { callId: call.callId, output: JSON.stringify(result ?? null) };
|
|
191
268
|
}
|
|
192
269
|
catch (err) {
|
|
193
|
-
|
|
270
|
+
return {
|
|
271
|
+
callId: call.callId,
|
|
272
|
+
output: JSON.stringify({ error: err instanceof Error ? err.message : String(err) }),
|
|
273
|
+
};
|
|
194
274
|
}
|
|
195
|
-
this.send({
|
|
196
|
-
type: 'conversation.item.create',
|
|
197
|
-
item: {
|
|
198
|
-
type: 'function_call_output',
|
|
199
|
-
call_id: callId,
|
|
200
|
-
output,
|
|
201
|
-
},
|
|
202
|
-
});
|
|
203
|
-
this.send({ type: 'response.create' });
|
|
204
275
|
}
|
|
205
276
|
send(data) {
|
|
206
277
|
const ws = this.ws;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai-realtime.js","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,
|
|
1
|
+
{"version":3,"file":"openai-realtime.js","sourceRoot":"","sources":["../../src/realtime/openai-realtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAG/B,MAAM,aAAa,GAAG,mBAAmB,CAAC;AAC1C,MAAM,aAAa,GAAG,OAAO,CAAC;AAC9B,MAAM,2BAA2B,GAAG,wBAAwB,CAAC;AAC7D,MAAM,QAAQ,GAAG,kCAAkC,CAAC;AACpD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,UAAU,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAW,CAAC;AAmB/D,SAAS,eAAe,CAAC,IAAuB;IAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC/D,IAAI,IAAI,YAAY,WAAW;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACrF,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;AACzB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,qBAAsB,SAAQ,YAAkC;IAC1D,MAAM,CAAwB;IAC9B,KAAK,CAAS;IACvB,EAAE,GAAqB,IAAI,CAAC;IAC5B,SAAS,GAAG,KAAK,CAAC;IAClB,cAAc,GAAG,KAAK,CAAC;IAE/B,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,GAAG,GAAG,GAAG,QAAQ,UAAU,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAElE,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,MAAM,MAAM,GAAG,CAAC,GAAW,EAAE,EAAE;gBAC7B,IAAI,OAAO;oBAAE,OAAO;gBACpB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,IAAI,GAAG;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;oBAChB,OAAO,EAAE,CAAC;YACjB,CAAC,CAAC;YAEF,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE;aAC3D,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YAEb,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;oBAAE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACnC,EAAE,CAAC,kBAAkB,EAAE,CAAC;gBACxB,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBACzB,EAAE,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,kBAAkB,IAAI,CAAC,CAAC,CAAC;YACvE,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAEvB,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;gBACtB,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACvB,MAAM,EAAE,CAAC;YACX,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAuB,EAAE,EAAE;gBAC3C,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5C,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;gBAC5B,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC1B,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE;gBAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,mBAAmB,CAAC;gBAC3D,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;oBAAE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;gBACnC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;gBACpC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBACvB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC5B,MAAM,CAAC,IAAI,KAAK,CAAC,yCAAyC,IAAI,MAAM,SAAS,EAAE,CAAC,CAAC,CAAC;gBAClF,IAAI,YAAY;oBAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,2BAA2B;YACjC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;SAChC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,IAAI,CAAC;YACR,IAAI,EAAE,0BAA0B;YAChC,IAAI,EAAE;gBACJ,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;aACxC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,SAAS;QACP,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO;QACjC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,KAAK;QACH,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,EAAE;YAAE,OAAO;QAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACf,EAAE,CAAC,kBAAkB,EAAE,CAAC;QACxB,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzB,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;IAEO,iBAAiB;QACvB,MAAM,OAAO,GAA4B;YACvC,IAAI,EAAE,UAAU;YAChB,iBAAiB,EAAE,CAAC,OAAO,CAAC;YAC5B,KAAK,EAAE;gBACL,KAAK,EAAE;oBACL,MAAM,EAAE,UAAU;oBAClB,aAAa,EAAE,EAAE,KAAK,EAAE,2BAA2B,EAAE;oBACrD,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;iBACvC;gBACD,MAAM,EAAE;oBACN,MAAM,EAAE,UAAU;oBAClB,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,aAAa;iBAC1C;aACF;SACF,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7B,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC;QAClD,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YAC9B,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC5C,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;aACzB,CAAC,CAAC,CAAC;YACJ,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC;IACjD,CAAC;IAEO,aAAa,CAAC,GAAW;QAC/B,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;YACnE,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC;QAErB,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,6BAA6B,CAAC,CAAC,CAAC;gBACnC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;gBAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAC9B,IAAI,CAAC,IAAI,CACP,OAAO,EACP,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAC7E,CAAC;oBACF,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;gBACjD,MAAM;YACR,CAAC;YAED,KAAK,uDAAuD;gBAC1D,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;oBACzC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACpD,CAAC;gBACD,MAAM;YAER,KAAK,uCAAuC;gBAC1C,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;oBACzC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACzD,CAAC;gBACD,MAAM;YAER,KAAK,mCAAmC;gBACtC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC1B,MAAM;YAER,KAAK,kBAAkB;gBACrB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC3B,MAAM;YAER,KAAK,eAAe;gBAClB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;gBAC5B,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBACxC,MAAM;YAER,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;gBAC3B,MAAM,OAAO,GACX,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ;oBACpD,CAAC,CAAC,MAAM,CAAC,OAAO;oBAChB,CAAC,CAAC,sBAAsB,CAAC;gBAC7B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACvC,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAEO,kBAAkB,CAAC,QAAiB;QAC1C,MAAM,KAAK,GAAuB,EAAE,CAAC;QACrC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5F,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACnC,IACE,QAAQ,CAAC,IAAI,CAAC;oBACd,IAAI,CAAC,IAAI,KAAK,eAAe;oBAC7B,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ;oBAC7B,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAChC,CAAC;oBACD,KAAK,CAAC,IAAI,CAAC;wBACT,IAAI,EAAE,IAAI,CAAC,IAAI;wBACf,MAAM,EAAE,IAAI,CAAC,OAAO;wBACpB,SAAS,EAAE,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;qBACtE,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACtB,OAAO;QACT,CAAC;QAED,KAAK,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,KAAyB;QACrD,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnF,IAAI,IAAI,CAAC,EAAE,KAAK,EAAE;YAAE,OAAO;QAE3B,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,OAAO,EAAE,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,0BAA0B;gBAChC,IAAI,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;aAChE,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,IAAsB;QAEtB,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,mCAAmC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;aAClF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAExC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC;QAClE,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,iBAAiB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;aAChE,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACxC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;QACzE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;aACpF,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,IAAI,CAAC,IAA6B;QACxC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QACnB,IAAI,EAAE,EAAE,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -15,13 +15,23 @@ export declare class RealtimeSession extends EventEmitter<RealtimeSessionEvents>
|
|
|
15
15
|
private readonly _provider;
|
|
16
16
|
private readonly forwarders;
|
|
17
17
|
private _closed;
|
|
18
|
+
private pending;
|
|
19
|
+
private pendingAudioBytes;
|
|
18
20
|
constructor(config: RealtimeSessionConfig);
|
|
19
21
|
get provider(): RealtimeSessionConfig['provider'];
|
|
22
|
+
get isConnected(): boolean;
|
|
20
23
|
connect(): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Send audio to the model. Audio pushed before the connection is established is
|
|
26
|
+
* buffered (up to 1MB, oldest audio dropped first) and flushed once connected.
|
|
27
|
+
*/
|
|
21
28
|
pushAudio(chunk: Buffer): void;
|
|
29
|
+
/** Send a user text turn. Text sent before the connection is established is queued. */
|
|
22
30
|
sendText(text: string): void;
|
|
23
31
|
interrupt(): void;
|
|
24
32
|
close(): void;
|
|
33
|
+
private addForwarder;
|
|
34
|
+
private flushPending;
|
|
25
35
|
}
|
|
26
36
|
export {};
|
|
27
37
|
//# sourceMappingURL=realtime-session.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"realtime-session.d.ts","sourceRoot":"","sources":["../../src/realtime/realtime-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"realtime-session.d.ts","sourceRoot":"","sources":["../../src/realtime/realtime-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAoBzD,UAAU,qBAAqB;IAC7B,SAAS,EAAE,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,KAAK,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACvB,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACvD,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,YAAY,EAAE,EAAE,CAAC;IACjB,QAAQ,EAAE,EAAE,CAAC;IACb,KAAK,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;CACvB;AAED,qBAAa,eAAgB,SAAQ,YAAY,CAAC,qBAAqB,CAAC;IACtE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAoC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmD;IAC9E,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,iBAAiB,CAAK;gBAElB,MAAM,EAAE,qBAAqB;IA0BzC,IAAI,QAAQ,IAAI,qBAAqB,CAAC,UAAU,CAAC,CAEhD;IAED,IAAI,WAAW,IAAI,OAAO,CAEzB;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAO9B;;;OAGG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAiB9B,uFAAuF;IACvF,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAS5B,SAAS,IAAI,IAAI;IAIjB,KAAK,IAAI,IAAI;IAYb,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,YAAY;CASrB"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { EventEmitter } from 'node:events';
|
|
2
2
|
import { OpenAIRealtimeAdapter } from './openai-realtime.js';
|
|
3
3
|
import { GeminiRealtimeAdapter } from './gemini-realtime.js';
|
|
4
|
+
const MAX_PENDING_AUDIO_BYTES = 1024 * 1024;
|
|
4
5
|
const FORWARDED_EVENTS = [
|
|
5
|
-
'connected',
|
|
6
6
|
'disconnected',
|
|
7
7
|
'speech_start',
|
|
8
8
|
'transcript',
|
|
@@ -16,6 +16,8 @@ export class RealtimeSession extends EventEmitter {
|
|
|
16
16
|
_provider;
|
|
17
17
|
forwarders = new Map();
|
|
18
18
|
_closed = false;
|
|
19
|
+
pending = [];
|
|
20
|
+
pendingAudioBytes = 0;
|
|
19
21
|
constructor(config) {
|
|
20
22
|
super();
|
|
21
23
|
this._provider = config.provider;
|
|
@@ -30,24 +32,58 @@ export class RealtimeSession extends EventEmitter {
|
|
|
30
32
|
throw new Error(`Unknown realtime provider: ${config.provider}`);
|
|
31
33
|
}
|
|
32
34
|
for (const event of FORWARDED_EVENTS) {
|
|
33
|
-
|
|
35
|
+
this.addForwarder(event, (...args) => {
|
|
34
36
|
this.emit(event, ...args);
|
|
35
|
-
};
|
|
36
|
-
this.forwarders.set(event, forwarder);
|
|
37
|
-
this.adapter.on(event, forwarder);
|
|
37
|
+
});
|
|
38
38
|
}
|
|
39
|
+
this.addForwarder('connected', () => {
|
|
40
|
+
this.flushPending();
|
|
41
|
+
this.emit('connected');
|
|
42
|
+
});
|
|
39
43
|
}
|
|
40
44
|
get provider() {
|
|
41
45
|
return this._provider;
|
|
42
46
|
}
|
|
47
|
+
get isConnected() {
|
|
48
|
+
return this.adapter.isConnected;
|
|
49
|
+
}
|
|
43
50
|
async connect() {
|
|
51
|
+
if (this._closed) {
|
|
52
|
+
throw new Error('RealtimeSession is closed');
|
|
53
|
+
}
|
|
44
54
|
return this.adapter.connect();
|
|
45
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* Send audio to the model. Audio pushed before the connection is established is
|
|
58
|
+
* buffered (up to 1MB, oldest audio dropped first) and flushed once connected.
|
|
59
|
+
*/
|
|
46
60
|
pushAudio(chunk) {
|
|
47
|
-
this.
|
|
61
|
+
if (this._closed)
|
|
62
|
+
return;
|
|
63
|
+
if (this.adapter.isConnected) {
|
|
64
|
+
this.adapter.pushAudio(chunk);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
this.pending.push({ kind: 'audio', chunk });
|
|
68
|
+
this.pendingAudioBytes += chunk.length;
|
|
69
|
+
while (this.pendingAudioBytes > MAX_PENDING_AUDIO_BYTES) {
|
|
70
|
+
const index = this.pending.findIndex((input) => input.kind === 'audio');
|
|
71
|
+
if (index === -1)
|
|
72
|
+
break;
|
|
73
|
+
const [dropped] = this.pending.splice(index, 1);
|
|
74
|
+
if (dropped?.kind === 'audio')
|
|
75
|
+
this.pendingAudioBytes -= dropped.chunk.length;
|
|
76
|
+
}
|
|
48
77
|
}
|
|
78
|
+
/** Send a user text turn. Text sent before the connection is established is queued. */
|
|
49
79
|
sendText(text) {
|
|
50
|
-
this.
|
|
80
|
+
if (this._closed)
|
|
81
|
+
return;
|
|
82
|
+
if (this.adapter.isConnected) {
|
|
83
|
+
this.adapter.sendText(text);
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
this.pending.push({ kind: 'text', text });
|
|
51
87
|
}
|
|
52
88
|
interrupt() {
|
|
53
89
|
this.adapter.interrupt();
|
|
@@ -56,11 +92,28 @@ export class RealtimeSession extends EventEmitter {
|
|
|
56
92
|
if (this._closed)
|
|
57
93
|
return;
|
|
58
94
|
this._closed = true;
|
|
95
|
+
this.pending = [];
|
|
96
|
+
this.pendingAudioBytes = 0;
|
|
59
97
|
for (const [event, forwarder] of this.forwarders) {
|
|
60
98
|
this.adapter.off(event, forwarder);
|
|
61
99
|
}
|
|
62
100
|
this.forwarders.clear();
|
|
63
101
|
this.adapter.close();
|
|
64
102
|
}
|
|
103
|
+
addForwarder(event, forwarder) {
|
|
104
|
+
this.forwarders.set(event, forwarder);
|
|
105
|
+
this.adapter.on(event, forwarder);
|
|
106
|
+
}
|
|
107
|
+
flushPending() {
|
|
108
|
+
const pending = this.pending;
|
|
109
|
+
this.pending = [];
|
|
110
|
+
this.pendingAudioBytes = 0;
|
|
111
|
+
for (const input of pending) {
|
|
112
|
+
if (input.kind === 'audio')
|
|
113
|
+
this.adapter.pushAudio(input.chunk);
|
|
114
|
+
else
|
|
115
|
+
this.adapter.sendText(input.text);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
65
118
|
}
|
|
66
119
|
//# sourceMappingURL=realtime-session.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"realtime-session.js","sourceRoot":"","sources":["../../src/realtime/realtime-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAI7D,MAAM,gBAAgB,GAAG;IACvB,
|
|
1
|
+
{"version":3,"file":"realtime-session.js","sourceRoot":"","sources":["../../src/realtime/realtime-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAI7D,MAAM,uBAAuB,GAAG,IAAI,GAAG,IAAI,CAAC;AAI5C,MAAM,gBAAgB,GAAG;IACvB,cAAc;IACd,cAAc;IACd,YAAY;IACZ,OAAO;IACP,WAAW;IACX,UAAU;IACV,OAAO;CACC,CAAC;AAaX,MAAM,OAAO,eAAgB,SAAQ,YAAmC;IACrD,OAAO,CAAkB;IACzB,SAAS,CAAoC;IAC7C,UAAU,GAAG,IAAI,GAAG,EAAwC,CAAC;IACtE,OAAO,GAAG,KAAK,CAAC;IAChB,OAAO,GAAmB,EAAE,CAAC;IAC7B,iBAAiB,GAAG,CAAC,CAAC;IAE9B,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxB,KAAK,QAAQ;gBACX,IAAI,CAAC,OAAO,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAAC;gBACjD,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,CAAC,OAAO,GAAG,IAAI,qBAAqB,CAAC,MAAM,CAAC,CAAC;gBACjD,MAAM;YACR;gBACE,MAAM,IAAI,KAAK,CAAC,8BAA8B,MAAM,CAAC,QAAkB,EAAE,CAAC,CAAC;QAC/E,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;YACrC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,GAAG,IAAe,EAAE,EAAE;gBAC9C,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAI,IAAc,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,GAAG,EAAE;YAClC,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAChC,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,KAAa;QACrB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO;QACT,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,iBAAiB,IAAI,KAAK,CAAC,MAAM,CAAC;QACvC,OAAO,IAAI,CAAC,iBAAiB,GAAG,uBAAuB,EAAE,CAAC;YACxD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;YACxE,IAAI,KAAK,KAAK,CAAC,CAAC;gBAAE,MAAM;YACxB,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAChD,IAAI,OAAO,EAAE,IAAI,KAAK,OAAO;gBAAE,IAAI,CAAC,iBAAiB,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;QAChF,CAAC;IACH,CAAC;IAED,uFAAuF;IACvF,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,SAAS;QACP,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACjD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAoC,EAAE,SAAS,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAEO,YAAY,CAClB,KAAkC,EAClC,SAAuC;QAEvC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QACtC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpC,CAAC;IAEO,YAAY;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;gBAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;;gBAC3D,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -3,12 +3,15 @@ export interface DeepgramSTTConfig {
|
|
|
3
3
|
apiKey: string;
|
|
4
4
|
model?: string;
|
|
5
5
|
language?: string;
|
|
6
|
+
/** Sample rate of raw PCM16 audio (streams and headerless batch input). Defaults to 16000. */
|
|
7
|
+
sampleRate?: number;
|
|
6
8
|
}
|
|
7
9
|
export declare class DeepgramSTT implements STTProvider {
|
|
8
10
|
readonly name = "deepgram";
|
|
9
11
|
private readonly apiKey;
|
|
10
12
|
private readonly model;
|
|
11
13
|
private readonly language?;
|
|
14
|
+
private readonly sampleRate;
|
|
12
15
|
constructor(config: DeepgramSTTConfig);
|
|
13
16
|
transcribe(audio: Buffer, options?: STTOptions): Promise<TranscribeResult>;
|
|
14
17
|
createStream(options?: STTStreamOptions): STTStream;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deepgram-stt.d.ts","sourceRoot":"","sources":["../../src/stt/deepgram-stt.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"deepgram-stt.d.ts","sourceRoot":"","sources":["../../src/stt/deepgram-stt.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,WAAW,EACX,UAAU,EACV,gBAAgB,EAChB,SAAS,EACT,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8FAA8F;IAC9F,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AA+CD,qBAAa,WAAY,YAAW,WAAW;IAC7C,QAAQ,CAAC,IAAI,cAAc;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,MAAM,EAAE,iBAAiB;IAO/B,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuChF,YAAY,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,SAAS;IAOnD,OAAO,CAAC,gBAAgB;CAgBzB"}
|