@byokit/realtime 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/LICENSE +202 -0
- package/README.md +41 -0
- package/dist/adapter.d.ts +13 -0
- package/dist/adapter.js +1 -0
- package/dist/catalogue.d.ts +14 -0
- package/dist/catalogue.js +2 -0
- package/dist/catalogue.json +6 -0
- package/dist/chatgpt.d.ts +10 -0
- package/dist/chatgpt.js +146 -0
- package/dist/child.d.ts +1 -0
- package/dist/child.js +50 -0
- package/dist/client.d.ts +8 -0
- package/dist/client.js +276 -0
- package/dist/frames.d.ts +107 -0
- package/dist/frames.js +188 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/node.d.ts +22 -0
- package/dist/node.js +163 -0
- package/dist/policy.d.ts +7 -0
- package/dist/policy.js +24 -0
- package/dist/prose.d.ts +7 -0
- package/dist/prose.js +42 -0
- package/dist/providers/gemini.d.ts +6 -0
- package/dist/providers/gemini.js +429 -0
- package/dist/providers/openai.d.ts +6 -0
- package/dist/providers/openai.js +313 -0
- package/dist/providers/xai.d.ts +6 -0
- package/dist/providers/xai.js +542 -0
- package/dist/rn-webrtc.d.ts +5 -0
- package/dist/rn-webrtc.js +12 -0
- package/dist/tools.d.ts +30 -0
- package/dist/tools.js +93 -0
- package/dist/types.d.ts +106 -0
- package/dist/types.js +12 -0
- package/dist/webrtc.d.ts +24 -0
- package/dist/webrtc.js +133 -0
- package/dist/words.d.ts +8 -0
- package/dist/words.js +5 -0
- package/dist/words.json +8 -0
- package/package.json +56 -0
|
@@ -0,0 +1,542 @@
|
|
|
1
|
+
import WebSocket from 'ws';
|
|
2
|
+
import { responseFence } from "../policy.js";
|
|
3
|
+
import { cleanProse, providerRefusal as refusal } from "../prose.js";
|
|
4
|
+
// Lifted wire behavior; this module is loaded only by the kit child process.
|
|
5
|
+
export function createAdapter(options) {
|
|
6
|
+
const MODEL = options.model ?? 'grok-voice-think-fast-2.0';
|
|
7
|
+
const RATE = 24000;
|
|
8
|
+
const PROVIDER_URL = options.endpoint ?? `wss://api.x.ai/v1/realtime?model=${MODEL}`;
|
|
9
|
+
let endAfterResponse = false;
|
|
10
|
+
const providerTools = options.tools.map((tool) => ({ type: 'function', ...tool }));
|
|
11
|
+
const currentContext = '';
|
|
12
|
+
const isExplicitHangup = options.hangup ?? (() => false);
|
|
13
|
+
const cleanProviderProse = (value, fallback, max) => cleanProse(value, fallback, max, options.redact);
|
|
14
|
+
const providerRefusal = (status, body) => refusal(status, body, options.redact);
|
|
15
|
+
const MAX_STDOUT_BYTES = 256 * 1024;
|
|
16
|
+
const MAX_REFUSAL_BODY_BYTES = 16 * 1024;
|
|
17
|
+
const stdoutQueue = [];
|
|
18
|
+
let stdoutBytes = 0;
|
|
19
|
+
let stdoutBlocked = false;
|
|
20
|
+
let stdoutOverflowed = false;
|
|
21
|
+
function syncProviderReadState() {
|
|
22
|
+
if (ws?.readyState !== WebSocket.OPEN)
|
|
23
|
+
return;
|
|
24
|
+
if (stdoutBlocked || stdoutQueue.length > 0 || outputPausedByClient)
|
|
25
|
+
ws.pause();
|
|
26
|
+
else
|
|
27
|
+
ws.resume();
|
|
28
|
+
}
|
|
29
|
+
const flushStdout = () => {
|
|
30
|
+
while (!stdoutBlocked && stdoutQueue.length > 0) {
|
|
31
|
+
const item = stdoutQueue.shift();
|
|
32
|
+
stdoutBytes -= item.line.length;
|
|
33
|
+
stdoutBlocked = !process.stdout.write(item.line, item.done);
|
|
34
|
+
}
|
|
35
|
+
syncProviderReadState();
|
|
36
|
+
};
|
|
37
|
+
process.stdout.on('drain', () => { stdoutBlocked = false; flushStdout(); });
|
|
38
|
+
const emit = (frame, done, force = false) => {
|
|
39
|
+
const line = `${JSON.stringify(frame)}\n`;
|
|
40
|
+
if (!force && stdoutBytes + line.length > MAX_STDOUT_BYTES) {
|
|
41
|
+
if (!stdoutOverflowed) {
|
|
42
|
+
stdoutOverflowed = true;
|
|
43
|
+
stopped = true;
|
|
44
|
+
ws?.close();
|
|
45
|
+
close('Voice output buffer overflowed.', true);
|
|
46
|
+
}
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
if (!stdoutBlocked && stdoutQueue.length === 0) {
|
|
50
|
+
stdoutBlocked = !process.stdout.write(line, done);
|
|
51
|
+
syncProviderReadState();
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
stdoutQueue.push({ line, done, audio: frame.type === 'realtime.audio' });
|
|
55
|
+
stdoutBytes += line.length;
|
|
56
|
+
syncProviderReadState();
|
|
57
|
+
}
|
|
58
|
+
return true;
|
|
59
|
+
};
|
|
60
|
+
// Provider deltas may exceed the public frame bound after tool calls. Base64 is
|
|
61
|
+
// independently decodable when split on four-character boundaries.
|
|
62
|
+
function chunkAudio(audio) {
|
|
63
|
+
const chunks = [];
|
|
64
|
+
const chunkSize = 64 * 1024;
|
|
65
|
+
for (let offset = 0; offset < audio.length; offset += chunkSize)
|
|
66
|
+
chunks.push(audio.slice(offset, offset + chunkSize));
|
|
67
|
+
return chunks;
|
|
68
|
+
}
|
|
69
|
+
const emitAudio = (audio) => {
|
|
70
|
+
let emitted = false;
|
|
71
|
+
for (const data of chunkAudio(audio))
|
|
72
|
+
emitted = emit({ type: 'realtime.audio', data }) || emitted;
|
|
73
|
+
return emitted;
|
|
74
|
+
};
|
|
75
|
+
const state = (value, detail) => tools.state(value, detail);
|
|
76
|
+
const PROMPT = options.instructions;
|
|
77
|
+
const tools = options.bridge;
|
|
78
|
+
const fence = responseFence();
|
|
79
|
+
let closing = false;
|
|
80
|
+
const close = (reason, forceExit = false) => {
|
|
81
|
+
if (closing)
|
|
82
|
+
return;
|
|
83
|
+
closing = true;
|
|
84
|
+
fence.interrupt();
|
|
85
|
+
clearTimeout(reconnectTimer);
|
|
86
|
+
clearTimeout(stableTimer);
|
|
87
|
+
tools.close();
|
|
88
|
+
clearTimeout(inputPoll);
|
|
89
|
+
const timer = setTimeout(() => process.exit(forceExit ? 1 : 0), 1000);
|
|
90
|
+
emit({ type: 'realtime.closed', reason }, () => {
|
|
91
|
+
clearTimeout(timer);
|
|
92
|
+
process.exit(forceExit ? 1 : 0);
|
|
93
|
+
}, true);
|
|
94
|
+
flushStdout();
|
|
95
|
+
};
|
|
96
|
+
let ws;
|
|
97
|
+
let stopped = false;
|
|
98
|
+
let providerReconnects = 0;
|
|
99
|
+
let reconnectTimer;
|
|
100
|
+
let stableTimer;
|
|
101
|
+
let providerEpoch = 0;
|
|
102
|
+
let providerReady = false;
|
|
103
|
+
let inputPoll;
|
|
104
|
+
let outputFenced = false;
|
|
105
|
+
let responseActive = false;
|
|
106
|
+
let responseGeneration = 0;
|
|
107
|
+
let clearedGeneration = -1;
|
|
108
|
+
let deferredClearGeneration;
|
|
109
|
+
let deferredClearContinuation;
|
|
110
|
+
let deferredClearPending = false;
|
|
111
|
+
const playbackGenerations = new Set();
|
|
112
|
+
const playbackDrainQueue = [];
|
|
113
|
+
let gracefulEndPending = false;
|
|
114
|
+
let outputPausedByClient = false;
|
|
115
|
+
const MAX_INPUT_BYTES = 96000;
|
|
116
|
+
const PROVIDER_BUFFER_LIMIT = 512 * 1024;
|
|
117
|
+
const clientQueue = [];
|
|
118
|
+
let clientQueueBytes = 0;
|
|
119
|
+
let inputOverflowPending = false;
|
|
120
|
+
/** A provider link alive this long was healthy; forget its retries. */
|
|
121
|
+
const PROVIDER_STABLE_AFTER_MS = 30000;
|
|
122
|
+
const text = (value) => String(value ?? '').trim();
|
|
123
|
+
function providerError(error) {
|
|
124
|
+
const raw = typeof error === 'string' ? error : error?.message ?? error?.code ?? 'provider error';
|
|
125
|
+
const detail = cleanProviderProse(raw, 'provider error', 200);
|
|
126
|
+
return { detail, terminal: /api key|auth|credit|quota|billing|permission|forbidden|invalid json|invalid payload|unknown name|unsupported/i.test(detail) };
|
|
127
|
+
}
|
|
128
|
+
const runTool = (name, input, operationId) => tools.run(name, input, operationId);
|
|
129
|
+
const currentProvider = (current, epoch) => !stopped && ws === current && providerEpoch === epoch;
|
|
130
|
+
const finishGracefulEndIfDrained = () => {
|
|
131
|
+
if (!gracefulEndPending || playbackDrainQueue.length > 0 || deferredClearPending)
|
|
132
|
+
return;
|
|
133
|
+
stopped = true;
|
|
134
|
+
ws?.close();
|
|
135
|
+
close('ended');
|
|
136
|
+
};
|
|
137
|
+
const continueAfterClear = (continuation) => {
|
|
138
|
+
if (gracefulEndPending)
|
|
139
|
+
finishGracefulEndIfDrained();
|
|
140
|
+
else
|
|
141
|
+
continuation?.();
|
|
142
|
+
};
|
|
143
|
+
const finishDeferredClearIfDrained = () => {
|
|
144
|
+
if (!deferredClearPending || playbackDrainQueue.length > 0)
|
|
145
|
+
return;
|
|
146
|
+
const generation = deferredClearGeneration;
|
|
147
|
+
const continuation = deferredClearContinuation;
|
|
148
|
+
deferredClearPending = false;
|
|
149
|
+
deferredClearGeneration = undefined;
|
|
150
|
+
deferredClearContinuation = undefined;
|
|
151
|
+
if (generation !== undefined)
|
|
152
|
+
playbackGenerations.delete(generation);
|
|
153
|
+
if (generation !== undefined && clearedGeneration !== generation) {
|
|
154
|
+
clearedGeneration = generation;
|
|
155
|
+
emit({ type: 'realtime.audio.clear' });
|
|
156
|
+
}
|
|
157
|
+
syncProviderReadState();
|
|
158
|
+
continueAfterClear(continuation);
|
|
159
|
+
};
|
|
160
|
+
const clearIncompleteOutput = () => {
|
|
161
|
+
fence.interrupt();
|
|
162
|
+
for (let index = stdoutQueue.length - 1; index >= 0; index--) {
|
|
163
|
+
if (stdoutQueue[index].audio) {
|
|
164
|
+
stdoutBytes -= stdoutQueue[index].line.length;
|
|
165
|
+
stdoutQueue.splice(index, 1);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
outputFenced = true;
|
|
169
|
+
responseActive = false;
|
|
170
|
+
const continuation = deferredClearContinuation;
|
|
171
|
+
deferredClearPending = false;
|
|
172
|
+
deferredClearGeneration = undefined;
|
|
173
|
+
deferredClearContinuation = undefined;
|
|
174
|
+
playbackGenerations.clear();
|
|
175
|
+
playbackDrainQueue.length = 0;
|
|
176
|
+
if (clearedGeneration !== responseGeneration) {
|
|
177
|
+
clearedGeneration = responseGeneration;
|
|
178
|
+
emit({ type: 'realtime.audio.clear' });
|
|
179
|
+
}
|
|
180
|
+
syncProviderReadState();
|
|
181
|
+
continueAfterClear(continuation);
|
|
182
|
+
};
|
|
183
|
+
const fenceActiveResponse = (continuation) => {
|
|
184
|
+
fence.interrupt();
|
|
185
|
+
outputFenced = true;
|
|
186
|
+
if (!responseActive) {
|
|
187
|
+
if (deferredClearPending || playbackDrainQueue.length > 0) {
|
|
188
|
+
deferredClearPending = true;
|
|
189
|
+
deferredClearContinuation = continuation;
|
|
190
|
+
}
|
|
191
|
+
else
|
|
192
|
+
continueAfterClear(continuation);
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
responseActive = false;
|
|
196
|
+
if (!playbackGenerations.has(responseGeneration)) {
|
|
197
|
+
if (playbackDrainQueue.length > 0) {
|
|
198
|
+
deferredClearPending = true;
|
|
199
|
+
deferredClearContinuation = continuation;
|
|
200
|
+
}
|
|
201
|
+
else
|
|
202
|
+
continueAfterClear(continuation);
|
|
203
|
+
}
|
|
204
|
+
else if (playbackDrainQueue.length > 0) {
|
|
205
|
+
deferredClearPending = true;
|
|
206
|
+
deferredClearGeneration = responseGeneration;
|
|
207
|
+
deferredClearContinuation = continuation;
|
|
208
|
+
syncProviderReadState();
|
|
209
|
+
}
|
|
210
|
+
else {
|
|
211
|
+
playbackGenerations.delete(responseGeneration);
|
|
212
|
+
if (clearedGeneration !== responseGeneration) {
|
|
213
|
+
clearedGeneration = responseGeneration;
|
|
214
|
+
emit({ type: 'realtime.audio.clear' });
|
|
215
|
+
}
|
|
216
|
+
continueAfterClear(continuation);
|
|
217
|
+
}
|
|
218
|
+
};
|
|
219
|
+
const deliverAfterClear = (continuation) => {
|
|
220
|
+
if (deferredClearPending || playbackDrainQueue.length > 0) {
|
|
221
|
+
deferredClearPending = true;
|
|
222
|
+
deferredClearContinuation = continuation;
|
|
223
|
+
}
|
|
224
|
+
else
|
|
225
|
+
continueAfterClear(continuation);
|
|
226
|
+
};
|
|
227
|
+
function handleXaiEvent(raw, current, epoch) {
|
|
228
|
+
if (!currentProvider(current, epoch))
|
|
229
|
+
return;
|
|
230
|
+
let message;
|
|
231
|
+
try {
|
|
232
|
+
message = JSON.parse(raw);
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
if (deferredClearPending && message.type !== 'input_audio_buffer.speech_started' && message.type !== 'error')
|
|
238
|
+
return;
|
|
239
|
+
switch (message.type) {
|
|
240
|
+
case 'input_audio_buffer.speech_started':
|
|
241
|
+
clearIncompleteOutput();
|
|
242
|
+
break;
|
|
243
|
+
case 'response.created':
|
|
244
|
+
if (!fence.begin(message.response?.id))
|
|
245
|
+
break;
|
|
246
|
+
responseGeneration += 1;
|
|
247
|
+
responseActive = true;
|
|
248
|
+
outputFenced = false;
|
|
249
|
+
state('thinking');
|
|
250
|
+
break;
|
|
251
|
+
case 'response.output_audio.delta':
|
|
252
|
+
case 'response.audio.delta': {
|
|
253
|
+
const audio = typeof message.delta === 'string' ? message.delta : typeof message.audio === 'string' ? message.audio : '';
|
|
254
|
+
if (audio && !outputFenced && fence.accepts(message.response_id) && emitAudio(audio))
|
|
255
|
+
playbackGenerations.add(responseGeneration);
|
|
256
|
+
break;
|
|
257
|
+
}
|
|
258
|
+
case 'response.done':
|
|
259
|
+
if (!fence.accepts(message.response?.id))
|
|
260
|
+
break;
|
|
261
|
+
if (!responseActive)
|
|
262
|
+
break;
|
|
263
|
+
responseActive = false;
|
|
264
|
+
if (playbackGenerations.has(responseGeneration) && !playbackDrainQueue.includes(responseGeneration)) {
|
|
265
|
+
playbackDrainQueue.push(responseGeneration);
|
|
266
|
+
}
|
|
267
|
+
state('connected');
|
|
268
|
+
if (endAfterResponse) {
|
|
269
|
+
gracefulEndPending = true;
|
|
270
|
+
finishGracefulEndIfDrained();
|
|
271
|
+
}
|
|
272
|
+
break;
|
|
273
|
+
case 'conversation.item.input_audio_transcription.completed':
|
|
274
|
+
if (typeof message.transcript === 'string' && message.transcript.trim() !== '') {
|
|
275
|
+
if (isExplicitHangup(message.transcript))
|
|
276
|
+
endAfterResponse = true;
|
|
277
|
+
emit({ type: 'realtime.transcript', role: 'user', text: message.transcript });
|
|
278
|
+
}
|
|
279
|
+
break;
|
|
280
|
+
case 'response.output_audio_transcript.done':
|
|
281
|
+
if (!fence.accepts(message.response_id))
|
|
282
|
+
break;
|
|
283
|
+
if (typeof message.transcript === 'string' && message.transcript.trim() !== '') {
|
|
284
|
+
tools.answered();
|
|
285
|
+
emit({ type: 'realtime.transcript', role: 'agent', text: message.transcript });
|
|
286
|
+
}
|
|
287
|
+
break;
|
|
288
|
+
case 'response.function_call_arguments.done': {
|
|
289
|
+
if (!fence.accepts(message.response_id))
|
|
290
|
+
break;
|
|
291
|
+
const signal = fence.signal;
|
|
292
|
+
const callGeneration = responseGeneration;
|
|
293
|
+
void (async () => {
|
|
294
|
+
let args = {};
|
|
295
|
+
try {
|
|
296
|
+
args = JSON.parse(message.arguments || '{}');
|
|
297
|
+
}
|
|
298
|
+
catch { /* interrupted arguments */ }
|
|
299
|
+
let output;
|
|
300
|
+
try {
|
|
301
|
+
output = await tools.run(message.name, args, message.call_id, signal);
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
output = 'The request could not be completed. Please try again.';
|
|
305
|
+
}
|
|
306
|
+
if (signal.aborted || !currentProvider(current, epoch) || current.readyState !== WebSocket.OPEN || outputFenced || callGeneration !== responseGeneration)
|
|
307
|
+
return;
|
|
308
|
+
current.send(JSON.stringify({
|
|
309
|
+
type: 'conversation.item.create',
|
|
310
|
+
item: { type: 'function_call_output', call_id: message.call_id, output: text(output).slice(0, 24000) },
|
|
311
|
+
}));
|
|
312
|
+
if (currentProvider(current, epoch) && !outputFenced && callGeneration === responseGeneration)
|
|
313
|
+
current.send(JSON.stringify({ type: 'response.create' }));
|
|
314
|
+
})();
|
|
315
|
+
break;
|
|
316
|
+
}
|
|
317
|
+
case 'error': {
|
|
318
|
+
const { detail, terminal } = providerError(message.error);
|
|
319
|
+
if (terminal) {
|
|
320
|
+
providerReady = false;
|
|
321
|
+
if (endAfterResponse)
|
|
322
|
+
gracefulEndPending = true;
|
|
323
|
+
fenceActiveResponse(() => close(`Voice provider error: ${detail}`));
|
|
324
|
+
stopped = true;
|
|
325
|
+
current.close();
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
if (endAfterResponse)
|
|
329
|
+
gracefulEndPending = true;
|
|
330
|
+
fenceActiveResponse(() => state('connected', detail));
|
|
331
|
+
}
|
|
332
|
+
break;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
function sendClientFrame(current, frame) {
|
|
337
|
+
if (frame.type === 'realtime.audio') {
|
|
338
|
+
current.send(JSON.stringify({ type: 'input_audio_buffer.append', audio: frame.data }));
|
|
339
|
+
}
|
|
340
|
+
else if (frame.type === 'realtime.say') {
|
|
341
|
+
current.send(JSON.stringify({
|
|
342
|
+
type: 'conversation.item.create',
|
|
343
|
+
item: { type: 'message', role: 'user', content: [{ type: 'input_text', text: frame.text }] },
|
|
344
|
+
}));
|
|
345
|
+
current.send(JSON.stringify({ type: 'response.create' }));
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
function flushClientFrames(current = ws, epoch = providerEpoch) {
|
|
349
|
+
clearTimeout(inputPoll);
|
|
350
|
+
inputPoll = undefined;
|
|
351
|
+
if (!current || !currentProvider(current, epoch) || !providerReady || current.readyState !== WebSocket.OPEN)
|
|
352
|
+
return;
|
|
353
|
+
while (clientQueue.length > 0 && current.bufferedAmount <= PROVIDER_BUFFER_LIMIT) {
|
|
354
|
+
const item = clientQueue.shift();
|
|
355
|
+
clientQueueBytes -= item.bytes;
|
|
356
|
+
sendClientFrame(current, item.frame);
|
|
357
|
+
}
|
|
358
|
+
if (clientQueue.length > 0)
|
|
359
|
+
inputPoll = setTimeout(() => flushClientFrames(current, epoch), 20);
|
|
360
|
+
}
|
|
361
|
+
const clientFrameBytes = (frame) => frame.type === 'realtime.audio'
|
|
362
|
+
? Math.floor(frame.data.length * 3 / 4)
|
|
363
|
+
: Buffer.byteLength(JSON.stringify(frame));
|
|
364
|
+
function handleClientFrame(frame) {
|
|
365
|
+
if (tools.receive(frame))
|
|
366
|
+
return;
|
|
367
|
+
if (frame.type === 'realtime.control') {
|
|
368
|
+
if (frame.action === 'stop') {
|
|
369
|
+
stopped = true;
|
|
370
|
+
ws?.close();
|
|
371
|
+
close('ended');
|
|
372
|
+
}
|
|
373
|
+
else if (frame.action === 'interrupt') {
|
|
374
|
+
const cancel = responseActive;
|
|
375
|
+
clearIncompleteOutput();
|
|
376
|
+
if (cancel && ws?.readyState === WebSocket.OPEN)
|
|
377
|
+
ws.send(JSON.stringify({ type: 'response.cancel' }));
|
|
378
|
+
}
|
|
379
|
+
else if (frame.action === 'pause_output') {
|
|
380
|
+
outputPausedByClient = true;
|
|
381
|
+
syncProviderReadState();
|
|
382
|
+
}
|
|
383
|
+
else if (frame.action === 'resume_output') {
|
|
384
|
+
outputPausedByClient = false;
|
|
385
|
+
syncProviderReadState();
|
|
386
|
+
}
|
|
387
|
+
else if (frame.action === 'output_drained') {
|
|
388
|
+
const drainedGeneration = playbackDrainQueue.shift();
|
|
389
|
+
if (drainedGeneration !== undefined)
|
|
390
|
+
playbackGenerations.delete(drainedGeneration);
|
|
391
|
+
finishDeferredClearIfDrained();
|
|
392
|
+
finishGracefulEndIfDrained();
|
|
393
|
+
}
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
if (inputOverflowPending)
|
|
397
|
+
return;
|
|
398
|
+
if (frame.type !== 'realtime.audio' && frame.type !== 'realtime.say')
|
|
399
|
+
return;
|
|
400
|
+
const bytes = clientFrameBytes(frame);
|
|
401
|
+
if (clientQueueBytes + bytes > MAX_INPUT_BYTES) {
|
|
402
|
+
inputOverflowPending = true;
|
|
403
|
+
providerReady = false;
|
|
404
|
+
clearTimeout(inputPoll);
|
|
405
|
+
inputPoll = undefined;
|
|
406
|
+
fenceActiveResponse(() => close('Voice input buffer overflowed.'));
|
|
407
|
+
stopped = true;
|
|
408
|
+
ws?.close();
|
|
409
|
+
return;
|
|
410
|
+
}
|
|
411
|
+
clientQueue.push({ frame, bytes });
|
|
412
|
+
clientQueueBytes += bytes;
|
|
413
|
+
flushClientFrames();
|
|
414
|
+
// mute/unmute are enforced on the phone's capture side; nothing to forward.
|
|
415
|
+
}
|
|
416
|
+
function connectProvider(key) {
|
|
417
|
+
if (stopped)
|
|
418
|
+
return;
|
|
419
|
+
clearTimeout(reconnectTimer);
|
|
420
|
+
reconnectTimer = undefined;
|
|
421
|
+
clearTimeout(stableTimer);
|
|
422
|
+
stableTimer = undefined;
|
|
423
|
+
providerReady = false;
|
|
424
|
+
const epoch = ++providerEpoch;
|
|
425
|
+
deliverAfterClear(() => state('connecting', providerReconnects === 0 ? undefined : 'Voice provider reconnecting'));
|
|
426
|
+
const current = new WebSocket(PROVIDER_URL, {
|
|
427
|
+
headers: { Authorization: `Bearer ${key}` },
|
|
428
|
+
maxPayload: 4 * 1024 * 1024,
|
|
429
|
+
});
|
|
430
|
+
ws = current;
|
|
431
|
+
let downHandled = false;
|
|
432
|
+
const onDown = (reason) => {
|
|
433
|
+
if (downHandled || stopped || ws !== current || providerEpoch !== epoch)
|
|
434
|
+
return;
|
|
435
|
+
downHandled = true;
|
|
436
|
+
providerReady = false;
|
|
437
|
+
clearTimeout(stableTimer);
|
|
438
|
+
stableTimer = undefined;
|
|
439
|
+
clearTimeout(inputPoll);
|
|
440
|
+
inputPoll = undefined;
|
|
441
|
+
if (endAfterResponse)
|
|
442
|
+
gracefulEndPending = true;
|
|
443
|
+
fenceActiveResponse(providerReconnects >= 2 ? () => close(reason) : undefined);
|
|
444
|
+
if (!gracefulEndPending && providerReconnects < 2) {
|
|
445
|
+
providerReconnects += 1;
|
|
446
|
+
clearTimeout(reconnectTimer);
|
|
447
|
+
reconnectTimer = setTimeout(() => connectProvider(key), providerReconnects * 500);
|
|
448
|
+
}
|
|
449
|
+
};
|
|
450
|
+
current.on('open', () => {
|
|
451
|
+
if (!currentProvider(current, epoch))
|
|
452
|
+
return;
|
|
453
|
+
syncProviderReadState();
|
|
454
|
+
current.send(JSON.stringify({
|
|
455
|
+
type: 'session.update',
|
|
456
|
+
session: {
|
|
457
|
+
instructions: PROMPT + currentContext,
|
|
458
|
+
voice: options.voice ?? 'ara',
|
|
459
|
+
reasoning: { effort: 'none' },
|
|
460
|
+
turn_detection: { type: 'server_vad', threshold: 0.9, silence_duration_ms: 700, prefix_padding_ms: 300 },
|
|
461
|
+
audio: {
|
|
462
|
+
input: { format: { type: 'audio/pcm', rate: RATE }, transport: 'json' },
|
|
463
|
+
output: { format: { type: 'audio/pcm', rate: RATE }, transport: 'json' },
|
|
464
|
+
},
|
|
465
|
+
tools: providerTools,
|
|
466
|
+
},
|
|
467
|
+
}), (error) => {
|
|
468
|
+
if (error) {
|
|
469
|
+
onDown('Voice provider session setup failed.');
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
if (!currentProvider(current, epoch))
|
|
473
|
+
return;
|
|
474
|
+
providerReady = true;
|
|
475
|
+
flushClientFrames(current, epoch);
|
|
476
|
+
deliverAfterClear(() => {
|
|
477
|
+
emit({ type: 'realtime.ready', inputRate: RATE, outputRate: RATE });
|
|
478
|
+
state('connected', providerReconnects === 0 ? undefined : 'Voice provider reconnected');
|
|
479
|
+
});
|
|
480
|
+
// Reconnect budget is consecutive, not cumulative: a long healthy
|
|
481
|
+
// call gets the full budget again after its next transient drop.
|
|
482
|
+
clearTimeout(stableTimer);
|
|
483
|
+
stableTimer = setTimeout(() => {
|
|
484
|
+
if (currentProvider(current, epoch))
|
|
485
|
+
providerReconnects = 0;
|
|
486
|
+
}, PROVIDER_STABLE_AFTER_MS);
|
|
487
|
+
});
|
|
488
|
+
});
|
|
489
|
+
current.on('message', (data) => handleXaiEvent(String(data), current, epoch));
|
|
490
|
+
// ws suppresses 'error' and 'close' once this is handled, so the failure
|
|
491
|
+
// path is driven from here. Auth and permission refusals are terminal:
|
|
492
|
+
// retrying an out-of-credits account only delays the real message.
|
|
493
|
+
current.on('unexpected-response', (_request, response) => {
|
|
494
|
+
const bodyBuffer = Buffer.alloc(MAX_REFUSAL_BODY_BYTES);
|
|
495
|
+
let bodyBytes = 0;
|
|
496
|
+
response.on('data', (chunk) => {
|
|
497
|
+
const admitted = Math.min(chunk.length, MAX_REFUSAL_BODY_BYTES - bodyBytes);
|
|
498
|
+
if (admitted > 0)
|
|
499
|
+
bodyBytes += chunk.copy(bodyBuffer, bodyBytes, 0, admitted);
|
|
500
|
+
});
|
|
501
|
+
response.on('end', () => {
|
|
502
|
+
if (!currentProvider(current, epoch))
|
|
503
|
+
return;
|
|
504
|
+
const body = bodyBuffer.subarray(0, bodyBytes).toString('utf8');
|
|
505
|
+
const reason = providerRefusal(response.statusCode, body);
|
|
506
|
+
if (response.statusCode === 401 || response.statusCode === 403) {
|
|
507
|
+
providerReady = false;
|
|
508
|
+
if (endAfterResponse)
|
|
509
|
+
gracefulEndPending = true;
|
|
510
|
+
fenceActiveResponse(() => close(reason));
|
|
511
|
+
stopped = true;
|
|
512
|
+
}
|
|
513
|
+
else
|
|
514
|
+
onDown(reason);
|
|
515
|
+
current.terminate();
|
|
516
|
+
});
|
|
517
|
+
});
|
|
518
|
+
current.on('close', (code, reasonBuffer) => {
|
|
519
|
+
if (stopped || ws !== current)
|
|
520
|
+
return;
|
|
521
|
+
const reason = cleanProviderProse(String(reasonBuffer), '', 160);
|
|
522
|
+
const detail = `The voice provider disconnected (${code})${reason ? `: ${reason}` : '.'}`;
|
|
523
|
+
if (providerError(reason).terminal) {
|
|
524
|
+
providerReady = false;
|
|
525
|
+
if (endAfterResponse)
|
|
526
|
+
gracefulEndPending = true;
|
|
527
|
+
fenceActiveResponse(() => close(detail));
|
|
528
|
+
stopped = true;
|
|
529
|
+
}
|
|
530
|
+
else {
|
|
531
|
+
onDown(detail);
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
current.on('error', (error) => {
|
|
535
|
+
if (currentProvider(current, epoch)) {
|
|
536
|
+
deliverAfterClear(() => state('connecting', `Voice provider connection interrupted: ${cleanProviderProse(error.message, 'connection error', 160)}`));
|
|
537
|
+
}
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
connectProvider(options.key);
|
|
541
|
+
return { receive: handleClientFrame, close: (reason = 'ended') => { stopped = true; ws?.close(); close(reason); } };
|
|
542
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type WebRtcOptions } from './webrtc.ts';
|
|
2
|
+
export type { WebRtcOptions, WebRtcHandle } from './webrtc.ts';
|
|
3
|
+
/** Native media stays unloaded until the app actually starts a voice session. */
|
|
4
|
+
export declare function webRtcPeer(options: WebRtcOptions): Promise<import("./webrtc.ts").WebRtcHandle>;
|
|
5
|
+
export { webRtcPeer as reactNativeWebRtcPeer };
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { webRtcPeer as createPeer } from "./webrtc.js";
|
|
2
|
+
/** Native media stays unloaded until the app actually starts a voice session. */
|
|
3
|
+
export async function webRtcPeer(options) {
|
|
4
|
+
if (options.platform)
|
|
5
|
+
return createPeer(options);
|
|
6
|
+
const native = await import('react-native-webrtc');
|
|
7
|
+
return createPeer({ ...options, platform: {
|
|
8
|
+
createPeer: () => new native.RTCPeerConnection({ bundlePolicy: 'max-bundle', rtcpMuxPolicy: 'require' }),
|
|
9
|
+
getUserMedia: () => native.mediaDevices.getUserMedia({ audio: true, video: false }),
|
|
10
|
+
} });
|
|
11
|
+
}
|
|
12
|
+
export { webRtcPeer as reactNativeWebRtcPeer };
|
package/dist/tools.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { RealtimeTool } from './types.ts';
|
|
2
|
+
import type { RealtimeHostFrame, RealtimeClientFrame, RealtimeAppAction, RealtimeState } from './frames.ts';
|
|
3
|
+
export type ToolHandler = (args: Record<string, unknown>, context: {
|
|
4
|
+
id: string;
|
|
5
|
+
signal: AbortSignal;
|
|
6
|
+
}) => Promise<string>;
|
|
7
|
+
export type ToolBridgeOptions = {
|
|
8
|
+
emit(frame: RealtimeHostFrame): void;
|
|
9
|
+
tools: RealtimeTool[];
|
|
10
|
+
handlers: Record<string, ToolHandler>;
|
|
11
|
+
timeoutFor?: (name: string) => number;
|
|
12
|
+
answerTimeoutMs?: number;
|
|
13
|
+
failure(name: string, error: unknown, timedOut: boolean): string;
|
|
14
|
+
};
|
|
15
|
+
/** Bounded, deduplicated tool execution. Product authorization stays in handlers. */
|
|
16
|
+
export declare function toolBridge(options: ToolBridgeOptions): {
|
|
17
|
+
run: (name: string, args?: unknown, id?: string, signal?: AbortSignal) => Promise<string>;
|
|
18
|
+
state: (value: RealtimeState, detail?: string) => void;
|
|
19
|
+
answered: () => void;
|
|
20
|
+
receive: (_frame: RealtimeClientFrame) => boolean;
|
|
21
|
+
close(): void;
|
|
22
|
+
};
|
|
23
|
+
export declare function appBridge(emit: (frame: RealtimeHostFrame) => void | boolean, options?: {
|
|
24
|
+
timeoutMs?: number;
|
|
25
|
+
maxPending?: number;
|
|
26
|
+
}): {
|
|
27
|
+
run(action: RealtimeAppAction, target?: string, signal?: AbortSignal): Promise<string>;
|
|
28
|
+
receive(frame: RealtimeClientFrame): boolean;
|
|
29
|
+
close(): void;
|
|
30
|
+
};
|
package/dist/tools.js
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
/** Bounded, deduplicated tool execution. Product authorization stays in handlers. */
|
|
3
|
+
export function toolBridge(options) {
|
|
4
|
+
const lifetime = new AbortController();
|
|
5
|
+
const requests = new Map();
|
|
6
|
+
let active = 0, waiting = false;
|
|
7
|
+
let answerTimer;
|
|
8
|
+
const state = (value, detail) => options.emit({ type: 'realtime.state', state: value === 'connected' && (active > 0 || waiting) ? 'thinking' : value, ...(detail ? { detail } : {}) });
|
|
9
|
+
const answered = () => { if (active)
|
|
10
|
+
return; clearTimeout(answerTimer); waiting = false; };
|
|
11
|
+
const run = (name, args = {}, id = randomUUID(), signal) => {
|
|
12
|
+
if (lifetime.signal.aborted)
|
|
13
|
+
return Promise.resolve('The request was cancelled.');
|
|
14
|
+
let key;
|
|
15
|
+
try {
|
|
16
|
+
key = JSON.stringify([name, args]);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return Promise.resolve('That request is invalid.');
|
|
20
|
+
}
|
|
21
|
+
if (!options.tools.some(tool => tool.name === name) || !Object.hasOwn(options.handlers, name) || !args || typeof args !== 'object' || Array.isArray(args) || typeof id !== 'string' || id.length > 160 || Buffer.byteLength(JSON.stringify(args)) > 16000)
|
|
22
|
+
return Promise.resolve('That request is invalid.');
|
|
23
|
+
const prior = requests.get(id);
|
|
24
|
+
if (prior)
|
|
25
|
+
return prior.key === key ? prior.promise : Promise.resolve('Conflicting repeated request. No additional action was performed.');
|
|
26
|
+
if (active >= 8 || requests.size >= 128)
|
|
27
|
+
return Promise.resolve('Requests are at the session limit.');
|
|
28
|
+
const controller = new AbortController();
|
|
29
|
+
const combined = AbortSignal.any([lifetime.signal, controller.signal, ...(signal ? [signal] : [])]);
|
|
30
|
+
active++;
|
|
31
|
+
clearTimeout(answerTimer);
|
|
32
|
+
waiting = false;
|
|
33
|
+
state('thinking');
|
|
34
|
+
const promise = (async () => {
|
|
35
|
+
let abort = () => { };
|
|
36
|
+
const timer = setTimeout(() => controller.abort(), options.timeoutFor?.(name) ?? 20000);
|
|
37
|
+
try {
|
|
38
|
+
const aborted = new Promise((_, reject) => { abort = () => reject(new Error('cancelled')); combined.addEventListener('abort', abort, { once: true }); if (combined.aborted)
|
|
39
|
+
abort(); });
|
|
40
|
+
const operation = Promise.resolve().then(() => { combined.throwIfAborted(); return options.handlers[name](args, { id, signal: combined }); });
|
|
41
|
+
return String(await Promise.race([operation, aborted])).slice(0, 8000);
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
return options.failure(name, error, controller.signal.aborted).slice(0, 8000);
|
|
45
|
+
}
|
|
46
|
+
finally {
|
|
47
|
+
clearTimeout(timer);
|
|
48
|
+
combined.removeEventListener('abort', abort);
|
|
49
|
+
active--;
|
|
50
|
+
if (!combined.aborted && !lifetime.signal.aborted) {
|
|
51
|
+
waiting = true;
|
|
52
|
+
clearTimeout(answerTimer);
|
|
53
|
+
answerTimer = setTimeout(() => { waiting = false; state('connected', 'The voice provider did not finish answering the request.'); }, options.answerTimeoutMs ?? 20000);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
})();
|
|
57
|
+
requests.set(id, { key, promise });
|
|
58
|
+
return promise;
|
|
59
|
+
};
|
|
60
|
+
return { run, state, answered, receive: (_frame) => false, close() { lifetime.abort(); clearTimeout(answerTimer); waiting = false; } };
|
|
61
|
+
}
|
|
62
|
+
export function appBridge(emit, options = {}) {
|
|
63
|
+
const pending = new Map();
|
|
64
|
+
return {
|
|
65
|
+
run(action, target, signal) {
|
|
66
|
+
if (signal?.aborted)
|
|
67
|
+
return Promise.resolve('The app request was cancelled.');
|
|
68
|
+
if (!['view', 'navigate', 'activate'].includes(action) || action !== 'view' && (!target?.trim() || Buffer.byteLength(target) > 160))
|
|
69
|
+
return Promise.resolve('That app request is invalid.');
|
|
70
|
+
if (pending.size >= (options.maxPending ?? 8))
|
|
71
|
+
return Promise.resolve('The app is busy.');
|
|
72
|
+
return new Promise(resolve => {
|
|
73
|
+
const requestId = randomUUID();
|
|
74
|
+
const finish = (text) => { clearTimeout(timer); signal?.removeEventListener('abort', abort); pending.delete(requestId); resolve(text); };
|
|
75
|
+
const abort = () => finish('The app request was cancelled.');
|
|
76
|
+
const timer = setTimeout(() => finish('The app did not answer that request.'), options.timeoutMs ?? 15000);
|
|
77
|
+
pending.set(requestId, finish);
|
|
78
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
79
|
+
try {
|
|
80
|
+
if (emit({ type: 'realtime.app.request', requestId, action, ...(action === 'view' ? {} : { target }) }) === false)
|
|
81
|
+
finish('The app could not receive that request.');
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
finish('The app is unavailable.');
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
},
|
|
88
|
+
receive(frame) { if (frame.type !== 'realtime.app.result')
|
|
89
|
+
return false; pending.get(frame.requestId)?.(frame.ok ? frame.text.slice(0, 4000) : 'The app could not complete that request.'); return true; },
|
|
90
|
+
close() { for (const finish of pending.values())
|
|
91
|
+
finish('The app request was cancelled.'); },
|
|
92
|
+
};
|
|
93
|
+
}
|