@byokit/realtime 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/LICENSE +202 -0
- package/README.md +41 -0
- package/dist/adapter.d.ts +13 -0
- package/dist/adapter.js +1 -0
- package/dist/catalogue.d.ts +14 -0
- package/dist/catalogue.js +2 -0
- package/dist/catalogue.json +6 -0
- package/dist/chatgpt.d.ts +10 -0
- package/dist/chatgpt.js +146 -0
- package/dist/child.d.ts +1 -0
- package/dist/child.js +50 -0
- package/dist/client.d.ts +8 -0
- package/dist/client.js +276 -0
- package/dist/frames.d.ts +107 -0
- package/dist/frames.js +188 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/node.d.ts +22 -0
- package/dist/node.js +163 -0
- package/dist/policy.d.ts +7 -0
- package/dist/policy.js +24 -0
- package/dist/prose.d.ts +7 -0
- package/dist/prose.js +42 -0
- package/dist/providers/gemini.d.ts +6 -0
- package/dist/providers/gemini.js +429 -0
- package/dist/providers/openai.d.ts +6 -0
- package/dist/providers/openai.js +313 -0
- package/dist/providers/xai.d.ts +6 -0
- package/dist/providers/xai.js +542 -0
- package/dist/rn-webrtc.d.ts +5 -0
- package/dist/rn-webrtc.js +12 -0
- package/dist/tools.d.ts +30 -0
- package/dist/tools.js +93 -0
- package/dist/types.d.ts +106 -0
- package/dist/types.js +12 -0
- package/dist/webrtc.d.ts +24 -0
- package/dist/webrtc.js +133 -0
- package/dist/words.d.ts +8 -0
- package/dist/words.js +5 -0
- package/dist/words.json +8 -0
- package/package.json +56 -0
package/dist/client.js
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import { parseRealtimeClientFrame, parseRealtimeHostFrame, realtimePcm16ByteLength } from "./frames.js";
|
|
2
|
+
/** Provider-blind client with bounded reconnects and exclusive media ownership. */
|
|
3
|
+
export function realtimeClient(options) {
|
|
4
|
+
const audio = options.audio;
|
|
5
|
+
let stopped = false, muted = false, epoch = 0, reconnects = 0;
|
|
6
|
+
let retry, reconnectTimer, stableTimer;
|
|
7
|
+
let attempt;
|
|
8
|
+
const mic = [], speech = [];
|
|
9
|
+
let micBytes = 0;
|
|
10
|
+
const stats = { micCaptured: 0, micSent: 0, micQueued: 0, micDropped: 0, transportReconnects: 0, providerReconnects: 0 };
|
|
11
|
+
const cleanup = async (current) => {
|
|
12
|
+
if (!current)
|
|
13
|
+
return;
|
|
14
|
+
current.signal.abort();
|
|
15
|
+
current.stream?.close();
|
|
16
|
+
if (current.stream)
|
|
17
|
+
audio.player.unbind(current.stream);
|
|
18
|
+
await current.media?.catch(() => { });
|
|
19
|
+
current.peer?.stop();
|
|
20
|
+
await current.routeRelease;
|
|
21
|
+
try {
|
|
22
|
+
await current.capture?.release();
|
|
23
|
+
}
|
|
24
|
+
finally {
|
|
25
|
+
if (current.acquired) {
|
|
26
|
+
current.acquired = false;
|
|
27
|
+
audio.microphone.release();
|
|
28
|
+
await audio.unroute();
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
const reset = () => { clearTimeout(retry); retry = undefined; clearTimeout(stableTimer); audio.player.clear(); mic.length = 0; micBytes = 0; };
|
|
33
|
+
const stop = (reason) => {
|
|
34
|
+
if (stopped)
|
|
35
|
+
return;
|
|
36
|
+
stopped = true;
|
|
37
|
+
epoch++;
|
|
38
|
+
clearTimeout(reconnectTimer);
|
|
39
|
+
reset();
|
|
40
|
+
const current = attempt;
|
|
41
|
+
attempt = undefined;
|
|
42
|
+
current?.stream?.send({ type: 'realtime.control', action: 'stop' });
|
|
43
|
+
void cleanup(current).catch(() => { });
|
|
44
|
+
audio.player.stop();
|
|
45
|
+
audio.player.release();
|
|
46
|
+
speech.length = 0;
|
|
47
|
+
options.onStats?.({ ...stats });
|
|
48
|
+
options.onStatus('disconnected', reason);
|
|
49
|
+
};
|
|
50
|
+
const fail = (error) => stop(error instanceof Error ? error.message : String(error));
|
|
51
|
+
const connected = () => { clearTimeout(stableTimer); stableTimer = setTimeout(() => { reconnects = 0; }, 30000); };
|
|
52
|
+
const schedule = () => { if (!retry && !stopped)
|
|
53
|
+
retry = setTimeout(() => { retry = undefined; flush(); }, 20); };
|
|
54
|
+
const flush = () => {
|
|
55
|
+
const current = attempt;
|
|
56
|
+
if (!current?.stream || stopped || !current.ready)
|
|
57
|
+
return;
|
|
58
|
+
while (mic.length) {
|
|
59
|
+
const next = mic[0];
|
|
60
|
+
if (!current.stream.send({ type: 'realtime.audio', data: next.data })) {
|
|
61
|
+
schedule();
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
mic.shift();
|
|
65
|
+
micBytes -= next.bytes;
|
|
66
|
+
stats.micSent++;
|
|
67
|
+
}
|
|
68
|
+
if (speech.length && audio.player.afterDrain('speech', flush))
|
|
69
|
+
return;
|
|
70
|
+
while (speech.length) {
|
|
71
|
+
if (!current.stream.send({ type: 'realtime.say', text: speech[0] })) {
|
|
72
|
+
schedule();
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
speech.shift();
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
const reconnect = (current, reason) => {
|
|
79
|
+
if (stopped || attempt !== current)
|
|
80
|
+
return;
|
|
81
|
+
if (options.retryableClose?.(reason) === false || reconnects >= 2) {
|
|
82
|
+
stop(reason);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
attempt = undefined;
|
|
86
|
+
epoch++;
|
|
87
|
+
reset();
|
|
88
|
+
reconnects++;
|
|
89
|
+
stats.transportReconnects++;
|
|
90
|
+
options.onStatus('connecting', reason);
|
|
91
|
+
// A slow microphone release cannot overlap the next acquisition.
|
|
92
|
+
void cleanup(current).then(() => {
|
|
93
|
+
if (!stopped)
|
|
94
|
+
reconnectTimer = setTimeout(connect, reconnects * 500);
|
|
95
|
+
}).catch(fail);
|
|
96
|
+
};
|
|
97
|
+
const connect = () => {
|
|
98
|
+
if (stopped)
|
|
99
|
+
return;
|
|
100
|
+
reconnectTimer = undefined;
|
|
101
|
+
const current = { acquired: false, ready: false, signal: new AbortController() };
|
|
102
|
+
attempt = current;
|
|
103
|
+
const generation = ++epoch;
|
|
104
|
+
const active = () => !stopped && attempt === current && epoch === generation;
|
|
105
|
+
const onAudio = (data) => {
|
|
106
|
+
if (!active() || muted)
|
|
107
|
+
return;
|
|
108
|
+
try {
|
|
109
|
+
const bytes = realtimePcm16ByteLength(data);
|
|
110
|
+
stats.micCaptured++;
|
|
111
|
+
options.onActivity?.();
|
|
112
|
+
options.onLevel?.('input', data);
|
|
113
|
+
if (micBytes + bytes > 96000) {
|
|
114
|
+
stats.micDropped++;
|
|
115
|
+
throw new Error('Microphone buffer overflowed.');
|
|
116
|
+
}
|
|
117
|
+
mic.push({ data, bytes });
|
|
118
|
+
micBytes += bytes;
|
|
119
|
+
if (!current.ready)
|
|
120
|
+
stats.micQueued++;
|
|
121
|
+
flush();
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
fail(error);
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
const handle = async (raw) => {
|
|
128
|
+
if (!active())
|
|
129
|
+
return;
|
|
130
|
+
const frame = parseRealtimeHostFrame(raw);
|
|
131
|
+
options.onActivity?.();
|
|
132
|
+
switch (frame.type) {
|
|
133
|
+
case 'realtime.ready':
|
|
134
|
+
if (current.media) {
|
|
135
|
+
current.ready = true;
|
|
136
|
+
stats.providerReconnects++;
|
|
137
|
+
connected();
|
|
138
|
+
flush();
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
current.media = (async () => {
|
|
142
|
+
await audio.microphone.acquire();
|
|
143
|
+
current.acquired = true;
|
|
144
|
+
if (!active())
|
|
145
|
+
return;
|
|
146
|
+
await audio.route();
|
|
147
|
+
if (!active())
|
|
148
|
+
return;
|
|
149
|
+
audio.player.ensure(frame.outputRate);
|
|
150
|
+
current.capture = await audio.capture(frame.inputRate, onAudio);
|
|
151
|
+
if (!active())
|
|
152
|
+
return;
|
|
153
|
+
current.ready = true;
|
|
154
|
+
for (const data of current.capture.pending)
|
|
155
|
+
onAudio(data);
|
|
156
|
+
connected();
|
|
157
|
+
options.onStatus('connected');
|
|
158
|
+
flush();
|
|
159
|
+
})();
|
|
160
|
+
await current.media;
|
|
161
|
+
break;
|
|
162
|
+
case 'realtime.webrtc.start':
|
|
163
|
+
if (current.media)
|
|
164
|
+
break;
|
|
165
|
+
if (!options.webrtc)
|
|
166
|
+
throw new Error('Voice media is unsupported.');
|
|
167
|
+
current.media = options.webrtc({ label: frame.dataChannelLabel, audio: { ...audio, unroute() { return current.routeRelease = audio.unroute(); } }, signal: current.signal.signal,
|
|
168
|
+
onOffer(sdp) { if (active() && !current.stream?.send({ type: 'realtime.webrtc.offer', sdp }))
|
|
169
|
+
reconnect(current, 'Voice offer could not be sent.'); },
|
|
170
|
+
onData(data) { if (active() && !current.stream?.send({ type: 'realtime.webrtc.data', data }))
|
|
171
|
+
reconnect(current, 'Voice channel overflowed.'); },
|
|
172
|
+
onRemoteAudio(speaking) { if (active())
|
|
173
|
+
options.onStatus(speaking ? 'speaking' : 'connected'); },
|
|
174
|
+
onConnectionState(state) { if (!active())
|
|
175
|
+
return; if (state === 'connected') {
|
|
176
|
+
current.ready = true;
|
|
177
|
+
connected();
|
|
178
|
+
flush();
|
|
179
|
+
}
|
|
180
|
+
else if (state === 'disconnected') {
|
|
181
|
+
reconnect(current, 'Voice connection ended.');
|
|
182
|
+
return;
|
|
183
|
+
} options.onStatus(state); },
|
|
184
|
+
onInterruption(interrupted) { if (active())
|
|
185
|
+
options.onStatus(interrupted ? 'connecting' : 'connected'); },
|
|
186
|
+
onError(error) { if (active())
|
|
187
|
+
reconnect(current, error.message); },
|
|
188
|
+
}).then(peer => { current.peer = peer; if (active())
|
|
189
|
+
peer.setMuted(muted); });
|
|
190
|
+
await current.media;
|
|
191
|
+
break;
|
|
192
|
+
case 'realtime.webrtc.answer':
|
|
193
|
+
await current.media;
|
|
194
|
+
if (active())
|
|
195
|
+
await current.peer?.acceptAnswer(frame.sdp);
|
|
196
|
+
break;
|
|
197
|
+
case 'realtime.webrtc.data':
|
|
198
|
+
await current.media;
|
|
199
|
+
if (active() && !current.peer?.sendData(frame.data))
|
|
200
|
+
throw new Error('Voice channel overflowed.');
|
|
201
|
+
break;
|
|
202
|
+
case 'realtime.audio':
|
|
203
|
+
options.onLevel?.('output', frame.data);
|
|
204
|
+
if (audio.player.admit(frame.data) !== 'ok')
|
|
205
|
+
throw new Error('Voice playback could not accept audio.');
|
|
206
|
+
options.onStatus('speaking');
|
|
207
|
+
break;
|
|
208
|
+
case 'realtime.audio.clear':
|
|
209
|
+
audio.player.clear();
|
|
210
|
+
break;
|
|
211
|
+
case 'realtime.transcript':
|
|
212
|
+
options.onTurn(frame.role, frame.text);
|
|
213
|
+
break;
|
|
214
|
+
case 'realtime.state':
|
|
215
|
+
if (frame.state === 'connecting') {
|
|
216
|
+
current.ready = false;
|
|
217
|
+
reset();
|
|
218
|
+
}
|
|
219
|
+
if (frame.state === 'connected')
|
|
220
|
+
audio.player.finish(() => { if (active())
|
|
221
|
+
options.onStatus('connected', frame.detail); });
|
|
222
|
+
else
|
|
223
|
+
options.onStatus(frame.state, frame.detail);
|
|
224
|
+
break;
|
|
225
|
+
case 'realtime.closed':
|
|
226
|
+
if (frame.retryable)
|
|
227
|
+
reconnect(current, frame.reason);
|
|
228
|
+
else
|
|
229
|
+
stop(frame.reason);
|
|
230
|
+
break;
|
|
231
|
+
case 'realtime.app.request': {
|
|
232
|
+
const result = await options.onAppRequest?.(frame.action, frame.target) ?? { ok: false, text: 'The app cannot answer that request.' };
|
|
233
|
+
if (active() && !current.stream?.send(parseRealtimeClientFrame({ type: 'realtime.app.result', requestId: frame.requestId, ...result })))
|
|
234
|
+
throw new Error('App answer could not be sent.');
|
|
235
|
+
break;
|
|
236
|
+
}
|
|
237
|
+
case 'realtime.usage':
|
|
238
|
+
options.onUsage?.(frame.usage);
|
|
239
|
+
break;
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
void options.open().then(stream => {
|
|
243
|
+
if (!active()) {
|
|
244
|
+
stream.close();
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
current.stream = stream;
|
|
248
|
+
audio.player.bind(stream);
|
|
249
|
+
stream.onFrame(frame => { void handle(frame).catch(error => { if (active())
|
|
250
|
+
fail(error); }); });
|
|
251
|
+
stream.onClose(reason => reconnect(current, reason));
|
|
252
|
+
stream.start();
|
|
253
|
+
}).catch(error => { if (active())
|
|
254
|
+
reconnect(current, error instanceof Error ? error.message : String(error)); });
|
|
255
|
+
};
|
|
256
|
+
options.onStatus('connecting');
|
|
257
|
+
connect();
|
|
258
|
+
return {
|
|
259
|
+
stop,
|
|
260
|
+
interrupt() { if (stopped)
|
|
261
|
+
return; audio.player.clear(); speech.length = 0; if (!attempt?.stream?.send({ type: 'realtime.control', action: 'interrupt' })) {
|
|
262
|
+
if (attempt)
|
|
263
|
+
reconnect(attempt, 'Voice interruption could not be sent.');
|
|
264
|
+
} },
|
|
265
|
+
setMuted(value) { muted = value; attempt?.peer?.setMuted(value); if (value) {
|
|
266
|
+
mic.length = 0;
|
|
267
|
+
micBytes = 0;
|
|
268
|
+
} },
|
|
269
|
+
speak(text) { if (stopped)
|
|
270
|
+
return; const frame = parseRealtimeClientFrame({ type: 'realtime.say', text }); if (frame.type !== 'realtime.say')
|
|
271
|
+
return; if (speech.length >= 16) {
|
|
272
|
+
fail(new Error('Voice request buffer overflowed.'));
|
|
273
|
+
return;
|
|
274
|
+
} speech.push(frame.text); flush(); },
|
|
275
|
+
};
|
|
276
|
+
}
|
package/dist/frames.d.ts
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import type { RealtimeUsage } from './types.ts';
|
|
2
|
+
/**
|
|
3
|
+
* Provider-neutral realtime voice channel.
|
|
4
|
+
*
|
|
5
|
+
* The phone owns only microphone capture, PCM playback and these generic frames.
|
|
6
|
+
* Provider authentication, event vocabularies, models, tools and prompts live in
|
|
7
|
+
* the host plugin process behind the channel.
|
|
8
|
+
*/
|
|
9
|
+
export declare const REALTIME_INPUT_RATE = 24000;
|
|
10
|
+
export declare const REALTIME_OUTPUT_RATE = 24000;
|
|
11
|
+
export declare const MAX_REALTIME_AUDIO_BASE64_BYTES: number;
|
|
12
|
+
export declare const MAX_REALTIME_TEXT_BYTES: number;
|
|
13
|
+
export declare const MAX_REALTIME_SDP_BYTES: number;
|
|
14
|
+
export declare const MAX_REALTIME_WEBRTC_DATA_BYTES: number;
|
|
15
|
+
export declare const MAX_REALTIME_CLOSE_REASON_BYTES: number;
|
|
16
|
+
export type RealtimeOpen = {
|
|
17
|
+
type: 'realtime.open';
|
|
18
|
+
context?: unknown;
|
|
19
|
+
};
|
|
20
|
+
export type RealtimeState = 'connecting' | 'connected' | 'thinking' | 'speaking';
|
|
21
|
+
export type RealtimeControlAction = 'mute' | 'unmute' | 'interrupt' | 'stop' | 'pause_output' | 'resume_output' | 'output_drained';
|
|
22
|
+
export type RealtimeAppAction = 'view' | 'navigate' | 'activate';
|
|
23
|
+
export interface RealtimeAudioClientFrame {
|
|
24
|
+
type: 'realtime.audio';
|
|
25
|
+
/** base64 PCM16 mono at the rate named by realtime.ready. */
|
|
26
|
+
data: string;
|
|
27
|
+
}
|
|
28
|
+
export interface RealtimeControlFrame {
|
|
29
|
+
type: 'realtime.control';
|
|
30
|
+
action: RealtimeControlAction;
|
|
31
|
+
}
|
|
32
|
+
export interface RealtimeSayFrame {
|
|
33
|
+
type: 'realtime.say';
|
|
34
|
+
text: string;
|
|
35
|
+
}
|
|
36
|
+
export interface RealtimeAppResultFrame {
|
|
37
|
+
type: 'realtime.app.result';
|
|
38
|
+
requestId: string;
|
|
39
|
+
ok: boolean;
|
|
40
|
+
text: string;
|
|
41
|
+
}
|
|
42
|
+
export interface RealtimeWebRtcOfferFrame {
|
|
43
|
+
type: 'realtime.webrtc.offer';
|
|
44
|
+
sdp: string;
|
|
45
|
+
}
|
|
46
|
+
export interface RealtimeWebRtcDataClientFrame {
|
|
47
|
+
type: 'realtime.webrtc.data';
|
|
48
|
+
data: string;
|
|
49
|
+
}
|
|
50
|
+
export type RealtimeClientFrame = RealtimeAudioClientFrame | RealtimeControlFrame | RealtimeSayFrame | RealtimeAppResultFrame | RealtimeWebRtcOfferFrame | RealtimeWebRtcDataClientFrame;
|
|
51
|
+
export interface RealtimeReadyFrame {
|
|
52
|
+
type: 'realtime.ready';
|
|
53
|
+
inputRate: number;
|
|
54
|
+
outputRate: number;
|
|
55
|
+
}
|
|
56
|
+
export interface RealtimeAudioHostFrame {
|
|
57
|
+
type: 'realtime.audio';
|
|
58
|
+
/** base64 PCM16 mono at realtime.ready's outputRate. */
|
|
59
|
+
data: string;
|
|
60
|
+
}
|
|
61
|
+
export interface RealtimeAudioClearFrame {
|
|
62
|
+
type: 'realtime.audio.clear';
|
|
63
|
+
}
|
|
64
|
+
export interface RealtimeStateFrame {
|
|
65
|
+
type: 'realtime.state';
|
|
66
|
+
state: RealtimeState;
|
|
67
|
+
detail?: string;
|
|
68
|
+
}
|
|
69
|
+
export interface RealtimeTranscriptFrame {
|
|
70
|
+
type: 'realtime.transcript';
|
|
71
|
+
role: 'user' | 'agent';
|
|
72
|
+
text: string;
|
|
73
|
+
}
|
|
74
|
+
export interface RealtimeClosedFrame {
|
|
75
|
+
type: 'realtime.closed';
|
|
76
|
+
reason?: string;
|
|
77
|
+
retryable?: boolean;
|
|
78
|
+
}
|
|
79
|
+
export interface RealtimeAppRequestFrame {
|
|
80
|
+
type: 'realtime.app.request';
|
|
81
|
+
requestId: string;
|
|
82
|
+
action: RealtimeAppAction;
|
|
83
|
+
target?: string;
|
|
84
|
+
}
|
|
85
|
+
export interface RealtimeWebRtcStartFrame {
|
|
86
|
+
type: 'realtime.webrtc.start';
|
|
87
|
+
dataChannelLabel: string;
|
|
88
|
+
}
|
|
89
|
+
export interface RealtimeWebRtcAnswerFrame {
|
|
90
|
+
type: 'realtime.webrtc.answer';
|
|
91
|
+
sdp: string;
|
|
92
|
+
}
|
|
93
|
+
export interface RealtimeWebRtcDataHostFrame {
|
|
94
|
+
type: 'realtime.webrtc.data';
|
|
95
|
+
data: string;
|
|
96
|
+
}
|
|
97
|
+
export type RealtimeUsageFrame = {
|
|
98
|
+
type: 'realtime.usage';
|
|
99
|
+
usage: RealtimeUsage;
|
|
100
|
+
};
|
|
101
|
+
export type RealtimeHostFrame = RealtimeUsageFrame | RealtimeReadyFrame | RealtimeAudioHostFrame | RealtimeAudioClearFrame | RealtimeStateFrame | RealtimeTranscriptFrame | RealtimeAppRequestFrame | RealtimeClosedFrame | RealtimeWebRtcStartFrame | RealtimeWebRtcAnswerFrame | RealtimeWebRtcDataHostFrame;
|
|
102
|
+
export declare function realtimePcm16ByteLength(value: unknown): number;
|
|
103
|
+
export declare function parseRealtimeClientFrame(value: unknown): RealtimeClientFrame;
|
|
104
|
+
export declare function parseRealtimeHostFrame(value: unknown): RealtimeHostFrame;
|
|
105
|
+
export declare function encodeRealtimeFrame(frame: RealtimeClientFrame | RealtimeHostFrame): string;
|
|
106
|
+
/** Random channel id for a link voice stream. */
|
|
107
|
+
export declare function newRealtimeChannel(): string;
|
package/dist/frames.js
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider-neutral realtime voice channel.
|
|
3
|
+
*
|
|
4
|
+
* The phone owns only microphone capture, PCM playback and these generic frames.
|
|
5
|
+
* Provider authentication, event vocabularies, models, tools and prompts live in
|
|
6
|
+
* the host plugin process behind the channel.
|
|
7
|
+
*/
|
|
8
|
+
export const REALTIME_INPUT_RATE = 24_000;
|
|
9
|
+
export const REALTIME_OUTPUT_RATE = 24_000;
|
|
10
|
+
export const MAX_REALTIME_AUDIO_BASE64_BYTES = 96 * 1024;
|
|
11
|
+
export const MAX_REALTIME_TEXT_BYTES = 4 * 1024;
|
|
12
|
+
export const MAX_REALTIME_SDP_BYTES = 128 * 1024;
|
|
13
|
+
export const MAX_REALTIME_WEBRTC_DATA_BYTES = 32 * 1024;
|
|
14
|
+
export const MAX_REALTIME_CLOSE_REASON_BYTES = 2 * 1024;
|
|
15
|
+
const REALTIME_CONTROL_ACTIONS = new Set(['mute', 'unmute', 'interrupt', 'stop', 'pause_output', 'resume_output', 'output_drained']);
|
|
16
|
+
const REALTIME_APP_ACTIONS = { view: true, navigate: true, activate: true };
|
|
17
|
+
const REALTIME_STATES = new Set(['connecting', 'connected', 'thinking', 'speaking']);
|
|
18
|
+
function record(value) {
|
|
19
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
20
|
+
throw new Error('invalid realtime frame');
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
function boundedText(value, max, label) {
|
|
24
|
+
if (typeof value !== 'string')
|
|
25
|
+
throw new Error(`invalid realtime ${label}`);
|
|
26
|
+
const clean = value.replace(/[\u0000-\u001F\u007F]/g, '').trim();
|
|
27
|
+
if (clean.length === 0 || new TextEncoder().encode(clean).length > max)
|
|
28
|
+
throw new Error(`invalid realtime ${label}`);
|
|
29
|
+
return clean;
|
|
30
|
+
}
|
|
31
|
+
function requestId(value) {
|
|
32
|
+
if (typeof value !== 'string' || !/^[A-Za-z0-9._:-]{1,160}$/.test(value))
|
|
33
|
+
throw new Error('invalid realtime request id');
|
|
34
|
+
return value;
|
|
35
|
+
}
|
|
36
|
+
const BASE64_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
|
|
37
|
+
export function realtimePcm16ByteLength(value) {
|
|
38
|
+
if (typeof value !== 'string' || value.length === 0 || value.length > MAX_REALTIME_AUDIO_BASE64_BYTES || value.length % 4 !== 0
|
|
39
|
+
|| !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value)) {
|
|
40
|
+
throw new Error('invalid realtime audio');
|
|
41
|
+
}
|
|
42
|
+
let padding = 0;
|
|
43
|
+
if (value.endsWith('=='))
|
|
44
|
+
padding = 2;
|
|
45
|
+
else if (value.endsWith('='))
|
|
46
|
+
padding = 1;
|
|
47
|
+
const trailingSextet = BASE64_ALPHABET.indexOf(value.charAt(value.length - padding - 1));
|
|
48
|
+
let nonCanonicalPadding = false;
|
|
49
|
+
if (padding === 2)
|
|
50
|
+
nonCanonicalPadding = (trailingSextet & 15) !== 0;
|
|
51
|
+
else if (padding === 1)
|
|
52
|
+
nonCanonicalPadding = (trailingSextet & 3) !== 0;
|
|
53
|
+
const bytes = value.length / 4 * 3 - padding;
|
|
54
|
+
if (nonCanonicalPadding || bytes === 0 || bytes % 2 !== 0)
|
|
55
|
+
throw new Error('invalid realtime audio');
|
|
56
|
+
return bytes;
|
|
57
|
+
}
|
|
58
|
+
function audio(value) {
|
|
59
|
+
realtimePcm16ByteLength(value);
|
|
60
|
+
return value;
|
|
61
|
+
}
|
|
62
|
+
function rate(value) {
|
|
63
|
+
if (typeof value !== 'number' || !Number.isSafeInteger(value) || ![8_000, 16_000, 22_050, 24_000, 32_000, 44_100, 48_000].includes(value)) {
|
|
64
|
+
throw new Error('invalid realtime audio rate');
|
|
65
|
+
}
|
|
66
|
+
return value;
|
|
67
|
+
}
|
|
68
|
+
function webRtcSdp(value) {
|
|
69
|
+
if (typeof value !== 'string' || !value.startsWith('v=0') || value.includes('\u0000')
|
|
70
|
+
|| new TextEncoder().encode(value).length > MAX_REALTIME_SDP_BYTES) {
|
|
71
|
+
throw new Error('invalid realtime WebRTC SDP');
|
|
72
|
+
}
|
|
73
|
+
return value;
|
|
74
|
+
}
|
|
75
|
+
function webRtcData(value) {
|
|
76
|
+
if (typeof value !== 'string' || value.length === 0 || value.includes('\u0000')
|
|
77
|
+
|| new TextEncoder().encode(value).length > MAX_REALTIME_WEBRTC_DATA_BYTES) {
|
|
78
|
+
throw new Error('invalid realtime WebRTC data');
|
|
79
|
+
}
|
|
80
|
+
return value;
|
|
81
|
+
}
|
|
82
|
+
function dataChannelLabel(value) {
|
|
83
|
+
if (typeof value !== 'string' || !/^[A-Za-z0-9._-]{1,64}$/.test(value)) {
|
|
84
|
+
throw new Error('invalid realtime WebRTC data channel');
|
|
85
|
+
}
|
|
86
|
+
return value;
|
|
87
|
+
}
|
|
88
|
+
export function parseRealtimeClientFrame(value) {
|
|
89
|
+
const frame = record(value);
|
|
90
|
+
if (frame.type === 'realtime.audio')
|
|
91
|
+
return { type: 'realtime.audio', data: audio(frame.data) };
|
|
92
|
+
if (frame.type === 'realtime.control') {
|
|
93
|
+
if (typeof frame.action !== 'string' || !REALTIME_CONTROL_ACTIONS.has(frame.action)) {
|
|
94
|
+
throw new Error('invalid realtime control');
|
|
95
|
+
}
|
|
96
|
+
return { type: 'realtime.control', action: frame.action };
|
|
97
|
+
}
|
|
98
|
+
if (frame.type === 'realtime.say')
|
|
99
|
+
return { type: 'realtime.say', text: boundedText(frame.text, MAX_REALTIME_TEXT_BYTES, 'speech') };
|
|
100
|
+
if (frame.type === 'realtime.app.result') {
|
|
101
|
+
if (typeof frame.ok !== 'boolean')
|
|
102
|
+
throw new Error('invalid realtime app result');
|
|
103
|
+
return {
|
|
104
|
+
type: 'realtime.app.result',
|
|
105
|
+
requestId: requestId(frame.requestId),
|
|
106
|
+
ok: frame.ok,
|
|
107
|
+
text: boundedText(frame.text, MAX_REALTIME_TEXT_BYTES, 'app result'),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
if (frame.type === 'realtime.webrtc.offer')
|
|
111
|
+
return { type: 'realtime.webrtc.offer', sdp: webRtcSdp(frame.sdp) };
|
|
112
|
+
if (frame.type === 'realtime.webrtc.data')
|
|
113
|
+
return { type: 'realtime.webrtc.data', data: webRtcData(frame.data) };
|
|
114
|
+
throw new Error('unknown realtime client frame');
|
|
115
|
+
}
|
|
116
|
+
export function parseRealtimeHostFrame(value) {
|
|
117
|
+
const frame = record(value);
|
|
118
|
+
if (frame.type === 'realtime.usage') {
|
|
119
|
+
const usage = record(frame.usage);
|
|
120
|
+
if (!['tokens', 'minutes', 'subscription'].includes(String(usage.basis)) || typeof usage.seconds !== 'number' || !Number.isFinite(usage.seconds) || usage.seconds < 0)
|
|
121
|
+
throw new Error('invalid realtime usage');
|
|
122
|
+
const result = { basis: usage.basis, seconds: usage.seconds };
|
|
123
|
+
for (const key of ['inputTokens', 'outputTokens', 'audioInTokens', 'audioOutTokens', 'cachedTokens']) {
|
|
124
|
+
if (usage[key] === undefined)
|
|
125
|
+
continue;
|
|
126
|
+
if (typeof usage[key] !== 'number' || !Number.isSafeInteger(usage[key]) || usage[key] < 0)
|
|
127
|
+
throw new Error('invalid realtime usage');
|
|
128
|
+
result[key] = usage[key];
|
|
129
|
+
}
|
|
130
|
+
return { type: 'realtime.usage', usage: result };
|
|
131
|
+
}
|
|
132
|
+
if (frame.type === 'realtime.ready')
|
|
133
|
+
return { type: 'realtime.ready', inputRate: rate(frame.inputRate), outputRate: rate(frame.outputRate) };
|
|
134
|
+
if (frame.type === 'realtime.audio')
|
|
135
|
+
return { type: 'realtime.audio', data: audio(frame.data) };
|
|
136
|
+
if (frame.type === 'realtime.audio.clear')
|
|
137
|
+
return { type: 'realtime.audio.clear' };
|
|
138
|
+
if (frame.type === 'realtime.state') {
|
|
139
|
+
if (typeof frame.state !== 'string' || !REALTIME_STATES.has(frame.state)) {
|
|
140
|
+
throw new Error('invalid realtime state');
|
|
141
|
+
}
|
|
142
|
+
return {
|
|
143
|
+
type: 'realtime.state', state: frame.state,
|
|
144
|
+
...(frame.detail === undefined ? {} : { detail: boundedText(frame.detail, 500, 'state detail') }),
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
if (frame.type === 'realtime.transcript') {
|
|
148
|
+
if (frame.role !== 'user' && frame.role !== 'agent')
|
|
149
|
+
throw new Error('invalid realtime transcript role');
|
|
150
|
+
return { type: 'realtime.transcript', role: frame.role, text: boundedText(frame.text, MAX_REALTIME_TEXT_BYTES, 'transcript') };
|
|
151
|
+
}
|
|
152
|
+
if (frame.type === 'realtime.app.request') {
|
|
153
|
+
if (typeof frame.action !== 'string' || REALTIME_APP_ACTIONS[frame.action] !== true) {
|
|
154
|
+
throw new Error('invalid realtime app action');
|
|
155
|
+
}
|
|
156
|
+
const action = frame.action;
|
|
157
|
+
if (action === 'view' && frame.target !== undefined)
|
|
158
|
+
throw new Error('invalid realtime app target');
|
|
159
|
+
if (action !== 'view' && frame.target === undefined)
|
|
160
|
+
throw new Error('invalid realtime app target');
|
|
161
|
+
return {
|
|
162
|
+
type: 'realtime.app.request',
|
|
163
|
+
requestId: requestId(frame.requestId),
|
|
164
|
+
action,
|
|
165
|
+
...(frame.target === undefined ? {} : { target: boundedText(frame.target, 160, 'app target') }),
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
if (frame.type === 'realtime.closed') {
|
|
169
|
+
if (frame.retryable !== undefined && typeof frame.retryable !== 'boolean')
|
|
170
|
+
throw new Error('invalid realtime retry policy');
|
|
171
|
+
return { type: 'realtime.closed', ...(frame.retryable === undefined ? {} : { retryable: frame.retryable }), ...(frame.reason === undefined ? {} : { reason: boundedText(frame.reason, MAX_REALTIME_CLOSE_REASON_BYTES, 'close reason') }) };
|
|
172
|
+
}
|
|
173
|
+
if (frame.type === 'realtime.webrtc.start') {
|
|
174
|
+
return { type: 'realtime.webrtc.start', dataChannelLabel: dataChannelLabel(frame.dataChannelLabel) };
|
|
175
|
+
}
|
|
176
|
+
if (frame.type === 'realtime.webrtc.answer')
|
|
177
|
+
return { type: 'realtime.webrtc.answer', sdp: webRtcSdp(frame.sdp) };
|
|
178
|
+
if (frame.type === 'realtime.webrtc.data')
|
|
179
|
+
return { type: 'realtime.webrtc.data', data: webRtcData(frame.data) };
|
|
180
|
+
throw new Error('unknown realtime host frame');
|
|
181
|
+
}
|
|
182
|
+
export function encodeRealtimeFrame(frame) {
|
|
183
|
+
return JSON.stringify(frame);
|
|
184
|
+
}
|
|
185
|
+
/** Random channel id for a link voice stream. */
|
|
186
|
+
export function newRealtimeChannel() {
|
|
187
|
+
return `rs_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`;
|
|
188
|
+
}
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
package/dist/node.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { RealtimeClientFrame, RealtimeHostFrame } from './frames.ts';
|
|
2
|
+
import type { RealtimeAuth, RealtimeConfig, RealtimeProviderId, RealtimeUsage } from './types.ts';
|
|
3
|
+
import { toolBridge } from './tools.ts';
|
|
4
|
+
export { toolBridge, appBridge } from './tools.ts';
|
|
5
|
+
export type { ToolHandler, ToolBridgeOptions } from './tools.ts';
|
|
6
|
+
export type RealtimeEngineOptions = RealtimeConfig & {
|
|
7
|
+
engine: RealtimeProviderId;
|
|
8
|
+
auth: RealtimeAuth;
|
|
9
|
+
bridge: ReturnType<typeof toolBridge>;
|
|
10
|
+
emit(frame: RealtimeHostFrame): void;
|
|
11
|
+
/** Only loopback overrides are accepted, for fake-provider flows. */
|
|
12
|
+
endpoint?: string;
|
|
13
|
+
redact?: RegExp[];
|
|
14
|
+
onUsage?(usage: RealtimeUsage): void;
|
|
15
|
+
};
|
|
16
|
+
/** Provider sockets and SDP signaling run in a kit-owned child, never in the app process. */
|
|
17
|
+
export declare function realtimeEngine(options: RealtimeEngineOptions): {
|
|
18
|
+
ready: Promise<void>;
|
|
19
|
+
usage: RealtimeUsage;
|
|
20
|
+
receive(raw: RealtimeClientFrame): boolean;
|
|
21
|
+
close(reason?: string): void;
|
|
22
|
+
};
|