codex-grok-bridge 1.7.2 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +82 -16
- package/package.json +5 -1
- package/scripts/install-codex-grok-app.ps1 +30 -0
- package/scripts/install-codex-grok-app.sh +26 -0
- package/src/bridge.mjs +99 -2
- package/src/imagine.mjs +42 -8
- package/src/realtime.mjs +212 -0
- package/src/runtime.mjs +3 -0
- package/src/tools.mjs +292 -22
- package/src/videogen.mjs +157 -8
- package/src/voice.mjs +1401 -0
package/src/voice.mjs
ADDED
|
@@ -0,0 +1,1401 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
let OpusScript;
|
|
4
|
+
let MediaStreamTrack;
|
|
5
|
+
let RTCPeerConnection;
|
|
6
|
+
let RtpHeader;
|
|
7
|
+
let RtpPacket;
|
|
8
|
+
let useOPUS;
|
|
9
|
+
let usePCMU;
|
|
10
|
+
let voicePackagesError;
|
|
11
|
+
|
|
12
|
+
try {
|
|
13
|
+
const opusMod = await import("opusscript");
|
|
14
|
+
const weriftMod = await import("werift");
|
|
15
|
+
OpusScript = opusMod.default ?? opusMod;
|
|
16
|
+
({ MediaStreamTrack, RTCPeerConnection, RtpHeader, RtpPacket, useOPUS, usePCMU } = weriftMod);
|
|
17
|
+
if (
|
|
18
|
+
typeof OpusScript !== "function" ||
|
|
19
|
+
typeof MediaStreamTrack !== "function" ||
|
|
20
|
+
typeof RTCPeerConnection !== "function" ||
|
|
21
|
+
typeof RtpHeader !== "function" ||
|
|
22
|
+
typeof RtpPacket !== "function" ||
|
|
23
|
+
typeof useOPUS !== "function" ||
|
|
24
|
+
typeof usePCMU !== "function"
|
|
25
|
+
) {
|
|
26
|
+
throw new Error("opusscript or werift did not export the voice runtime");
|
|
27
|
+
}
|
|
28
|
+
} catch (error) {
|
|
29
|
+
OpusScript = undefined;
|
|
30
|
+
MediaStreamTrack = undefined;
|
|
31
|
+
RTCPeerConnection = undefined;
|
|
32
|
+
RtpHeader = undefined;
|
|
33
|
+
RtpPacket = undefined;
|
|
34
|
+
useOPUS = undefined;
|
|
35
|
+
usePCMU = undefined;
|
|
36
|
+
voicePackagesError = error;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function voicePackages() {
|
|
40
|
+
if (OpusScript && MediaStreamTrack && RTCPeerConnection && RtpHeader && RtpPacket && useOPUS && usePCMU) {
|
|
41
|
+
return { OpusScript, MediaStreamTrack, RTCPeerConnection, RtpHeader, RtpPacket, useOPUS, usePCMU };
|
|
42
|
+
}
|
|
43
|
+
const detail = voicePackagesError instanceof Error ? voicePackagesError.message : "missing";
|
|
44
|
+
throw new Error(
|
|
45
|
+
`Voice needs opusscript and werift installed in the bridge directory (${detail}). Re-run the Codex Grok installer.`,
|
|
46
|
+
voicePackagesError instanceof Error ? { cause: voicePackagesError } : undefined,
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export const VOICE_SOCKET_URL = "wss://api.x.ai/v1/realtime?model=grok-voice-latest";
|
|
51
|
+
|
|
52
|
+
export function voiceSocketUrl(conversationId) {
|
|
53
|
+
if (!conversationId) return VOICE_SOCKET_URL;
|
|
54
|
+
const url = new URL(VOICE_SOCKET_URL);
|
|
55
|
+
url.searchParams.set("conversation_id", conversationId);
|
|
56
|
+
return url.toString();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const localCalls = new Set();
|
|
60
|
+
const active = new Set();
|
|
61
|
+
const sessionsById = new Map();
|
|
62
|
+
|
|
63
|
+
const OPUS_FRAME = 960;
|
|
64
|
+
|
|
65
|
+
function peerConfig() {
|
|
66
|
+
const { useOPUS, usePCMU } = voicePackages();
|
|
67
|
+
return {
|
|
68
|
+
iceServers: [],
|
|
69
|
+
iceUseIpv4: true,
|
|
70
|
+
iceUseIpv6: false,
|
|
71
|
+
iceUseTcp: false,
|
|
72
|
+
iceAdditionalHostAddresses: ["127.0.0.1"],
|
|
73
|
+
codecs: { audio: [useOPUS(), usePCMU()] },
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Codex voice-host rejects an answer with more than 32 candidate lines and
|
|
78
|
+
// waits until the offered oai-events channel opens. A public srflx address
|
|
79
|
+
// is not the local bridge. Trailing "generation" and "ufrag" tokens are not
|
|
80
|
+
// part of the candidate the helper parses, so they are removed.
|
|
81
|
+
// webrtc-rs opens SCTP only when it is the DTLS client, and only after the
|
|
82
|
+
// handshake. An actpass offer lets this answer be the server, so the desktop
|
|
83
|
+
// sends the INIT. A werift offerer still sends it: that stack initiates SCTP
|
|
84
|
+
// when it is ICE-controlling, which the offerer is.
|
|
85
|
+
function answerSdpForOffer(offer, answerSdp) {
|
|
86
|
+
const text = String(answerSdp ?? "");
|
|
87
|
+
if (!String(offer ?? "").includes("a=setup:actpass")) return text;
|
|
88
|
+
return text.replaceAll("a=setup:active", "a=setup:passive");
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function hostAnswerSdp(sdp) {
|
|
92
|
+
const lines = String(sdp ?? "").split("\r\n");
|
|
93
|
+
const hosts = [];
|
|
94
|
+
lines.forEach((line, index) => {
|
|
95
|
+
if (line.startsWith("a=candidate:") && line.includes(" typ host") && line.includes(" 1 udp ")) {
|
|
96
|
+
hosts.push(index);
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
const picked = new Set(
|
|
100
|
+
hosts
|
|
101
|
+
.sort((left, right) => {
|
|
102
|
+
const loopback = Number(lines[right].includes(" 127.0.0.1 ")) - Number(lines[left].includes(" 127.0.0.1 "));
|
|
103
|
+
return loopback || left - right;
|
|
104
|
+
})
|
|
105
|
+
.slice(0, 24),
|
|
106
|
+
);
|
|
107
|
+
return lines
|
|
108
|
+
.map((line, index) => {
|
|
109
|
+
if (!picked.has(index)) return line;
|
|
110
|
+
const canonical = line.match(/^(a=candidate:\S+ \d+ \S+ \d+ \S+ \d+ typ host)\b/);
|
|
111
|
+
return canonical ? canonical[1] : line;
|
|
112
|
+
})
|
|
113
|
+
.filter((line, index) => !lines[index].startsWith("a=candidate:") || picked.has(index))
|
|
114
|
+
.join("\r\n");
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function callBodyText(body) {
|
|
118
|
+
return Buffer.isBuffer(body) ? body.toString("utf8") : String(body ?? "");
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function partHeaderEnd(part) {
|
|
122
|
+
const crlf = part.indexOf("\r\n\r\n");
|
|
123
|
+
const lf = part.indexOf("\n\n");
|
|
124
|
+
if (crlf === -1) return lf === -1 ? null : { index: lf, length: 2 };
|
|
125
|
+
if (lf === -1 || crlf < lf) return { index: crlf, length: 4 };
|
|
126
|
+
return { index: lf, length: 2 };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function partFromCallBody(body, contentType, name) {
|
|
130
|
+
const text = callBodyText(body);
|
|
131
|
+
const type = String(contentType ?? "");
|
|
132
|
+
if (!type.toLowerCase().includes("multipart/form-data")) return null;
|
|
133
|
+
const boundaryMatch = type.match(/boundary="?([^";]+)"?/i);
|
|
134
|
+
if (!boundaryMatch) return null;
|
|
135
|
+
const boundary = boundaryMatch[1];
|
|
136
|
+
for (const part of text.split(`--${boundary}`)) {
|
|
137
|
+
const header = partHeaderEnd(part);
|
|
138
|
+
if (!header) continue;
|
|
139
|
+
const headers = part.slice(0, header.index).toLowerCase();
|
|
140
|
+
if (!headers.includes(`name="${name}"`) && !headers.includes(`name=${name}`)) continue;
|
|
141
|
+
let value = part.slice(header.index + header.length);
|
|
142
|
+
if (value.endsWith("\r\n")) value = value.slice(0, -2);
|
|
143
|
+
else if (value.endsWith("\n")) value = value.slice(0, -1);
|
|
144
|
+
return value;
|
|
145
|
+
}
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function offerFromCallBody(body, contentType) {
|
|
150
|
+
return partFromCallBody(body, contentType, "sdp") ?? callBodyText(body);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function sessionObject(body, contentType) {
|
|
154
|
+
const raw = partFromCallBody(body, contentType, "session");
|
|
155
|
+
if (!raw) return null;
|
|
156
|
+
try {
|
|
157
|
+
const session = JSON.parse(raw);
|
|
158
|
+
return session?.session && typeof session.session === "object" ? session.session : session;
|
|
159
|
+
} catch {
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export function instructionsFromCallBody(body, contentType) {
|
|
165
|
+
const session = sessionObject(body, contentType);
|
|
166
|
+
return typeof session?.instructions === "string" ? session.instructions.trim() : "";
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function ackFillerFromCallBody(body, contentType) {
|
|
170
|
+
return sessionObject(body, contentType)?.delegation?.ack_filler === true;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function initialItemsFromCallBody(body, contentType) {
|
|
174
|
+
const items = sessionObject(body, contentType)?.initial_items;
|
|
175
|
+
if (!Array.isArray(items)) return [];
|
|
176
|
+
const seeded = [];
|
|
177
|
+
for (const item of items) {
|
|
178
|
+
if (item?.type !== "message" || !Array.isArray(item.content)) continue;
|
|
179
|
+
const text = item.content
|
|
180
|
+
.map((part) => (typeof part?.text === "string" ? part.text : ""))
|
|
181
|
+
.join("")
|
|
182
|
+
.trim();
|
|
183
|
+
if (!text) continue;
|
|
184
|
+
if (item.role === "developer") continue;
|
|
185
|
+
const assistant = item.role === "assistant";
|
|
186
|
+
seeded.push({
|
|
187
|
+
type: "conversation.item.create",
|
|
188
|
+
item: {
|
|
189
|
+
type: "message",
|
|
190
|
+
role: assistant ? "assistant" : "user",
|
|
191
|
+
content: [{ type: assistant ? "output_text" : "input_text", text }],
|
|
192
|
+
},
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
return seeded;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export function developerTextFromCallBody(body, contentType) {
|
|
199
|
+
const items = sessionObject(body, contentType)?.initial_items;
|
|
200
|
+
if (!Array.isArray(items)) return "";
|
|
201
|
+
const parts = [];
|
|
202
|
+
for (const item of items) {
|
|
203
|
+
if (item?.role !== "developer" || item.type !== "message" || !Array.isArray(item.content)) continue;
|
|
204
|
+
const text = item.content
|
|
205
|
+
.map((part) => (typeof part?.text === "string" ? part.text : ""))
|
|
206
|
+
.join("")
|
|
207
|
+
.trim();
|
|
208
|
+
if (text) parts.push(text);
|
|
209
|
+
}
|
|
210
|
+
return parts.join("\n\n");
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function remoteAudioTracks(pc) {
|
|
214
|
+
const found = [];
|
|
215
|
+
for (const transceiver of pc.getTransceivers?.() ?? []) {
|
|
216
|
+
for (const track of transceiver.receiver?.tracks ?? []) {
|
|
217
|
+
if (track?.remote && track.kind === "audio") found.push(track);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return found;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export function audioCodecFromSdp(sdp) {
|
|
224
|
+
const opus = String(sdp ?? "").match(/a=rtpmap:(\d+) opus\/48000/i);
|
|
225
|
+
if (opus) return { kind: "opus", payloadType: Number(opus[1]) };
|
|
226
|
+
return { kind: "pcmu", payloadType: 0 };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function createOpus() {
|
|
230
|
+
const { OpusScript } = voicePackages();
|
|
231
|
+
return new OpusScript(48000, 2, OpusScript.Application.VOIP);
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function opusPayloadToMono(decoder, payload) {
|
|
235
|
+
const decoded = Buffer.from(decoder.decode(Buffer.from(payload)));
|
|
236
|
+
const frames = Math.floor(decoded.length / 4);
|
|
237
|
+
const mono = Buffer.alloc(frames * 2);
|
|
238
|
+
for (let i = 0; i < frames; i += 1) {
|
|
239
|
+
const left = decoded.readInt16LE(i * 4);
|
|
240
|
+
const right = decoded.readInt16LE(i * 4 + 2);
|
|
241
|
+
mono.writeInt16LE((left + right) >> 1, i * 2);
|
|
242
|
+
}
|
|
243
|
+
return mono;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function monoToStereo(pcm) {
|
|
247
|
+
const samples = Math.floor(pcm.length / 2);
|
|
248
|
+
const stereo = Buffer.alloc(samples * 4);
|
|
249
|
+
for (let i = 0; i < samples; i += 1) {
|
|
250
|
+
const sample = pcm.readInt16LE(i * 2);
|
|
251
|
+
stereo.writeInt16LE(sample, i * 4);
|
|
252
|
+
stereo.writeInt16LE(sample, i * 4 + 2);
|
|
253
|
+
}
|
|
254
|
+
return stereo;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Codex reads Location for rtc_* or a 36-character UUID and rejects anything else.
|
|
258
|
+
export function rememberLocalCall(id = randomUUID()) {
|
|
259
|
+
localCalls.add(id);
|
|
260
|
+
return id;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function callIdIn(url) {
|
|
264
|
+
const text = String(url ?? "");
|
|
265
|
+
for (const id of localCalls) {
|
|
266
|
+
if (text.includes(id)) return id;
|
|
267
|
+
}
|
|
268
|
+
return null;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export function isLocalVoiceSideband(url) {
|
|
272
|
+
return callIdIn(url) != null;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
export function encodeServerFrame(opcode, payload) {
|
|
276
|
+
const data = Buffer.isBuffer(payload) ? payload : Buffer.from(String(payload));
|
|
277
|
+
let header;
|
|
278
|
+
if (data.length < 126) header = Buffer.from([0x80 | opcode, data.length]);
|
|
279
|
+
else header = Buffer.from([0x80 | opcode, 126, data.length >> 8, data.length & 0xff]);
|
|
280
|
+
return Buffer.concat([header, data]);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function attachClientFrames(socket, onText) {
|
|
284
|
+
let buf = Buffer.alloc(0);
|
|
285
|
+
let parts = [];
|
|
286
|
+
let partOpcode = null;
|
|
287
|
+
socket.on("data", (chunk) => {
|
|
288
|
+
buf = Buffer.concat([buf, chunk]);
|
|
289
|
+
while (buf.length >= 2) {
|
|
290
|
+
const fin = (buf[0] & 0x80) !== 0;
|
|
291
|
+
const opcode = buf[0] & 0x0f;
|
|
292
|
+
const masked = (buf[1] & 0x80) !== 0;
|
|
293
|
+
let length = buf[1] & 0x7f;
|
|
294
|
+
let offset = 2;
|
|
295
|
+
if (length === 126) {
|
|
296
|
+
if (buf.length < 4) return;
|
|
297
|
+
length = buf.readUInt16BE(2);
|
|
298
|
+
offset = 4;
|
|
299
|
+
} else if (length === 127) {
|
|
300
|
+
socket.end(encodeServerFrame(0x8, Buffer.alloc(0)));
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
const maskLen = masked ? 4 : 0;
|
|
304
|
+
if (buf.length < offset + maskLen + length) return;
|
|
305
|
+
let payload = buf.subarray(offset + maskLen, offset + maskLen + length);
|
|
306
|
+
if (masked) {
|
|
307
|
+
const mask = buf.subarray(offset, offset + 4);
|
|
308
|
+
payload = Buffer.from(payload);
|
|
309
|
+
for (let i = 0; i < payload.length; i += 1) payload[i] ^= mask[i % 4];
|
|
310
|
+
}
|
|
311
|
+
buf = buf.subarray(offset + maskLen + length);
|
|
312
|
+
if (opcode === 0x8) {
|
|
313
|
+
socket.end(encodeServerFrame(0x8, Buffer.alloc(0)));
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
if (opcode === 0x9) {
|
|
317
|
+
socket.write(encodeServerFrame(0xa, payload));
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
if (opcode === 0x1 || opcode === 0x0) {
|
|
321
|
+
if (opcode === 0x1) {
|
|
322
|
+
parts = [payload];
|
|
323
|
+
partOpcode = opcode;
|
|
324
|
+
} else if (partOpcode != null) parts.push(payload);
|
|
325
|
+
if (fin && partOpcode === 0x1) onText(Buffer.concat(parts).toString("utf8"));
|
|
326
|
+
if (fin) {
|
|
327
|
+
parts = [];
|
|
328
|
+
partOpcode = null;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
export function acceptLocalSideband(socket, key, url, head = Buffer.alloc(0)) {
|
|
336
|
+
const accept = createHash("sha1")
|
|
337
|
+
.update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
|
|
338
|
+
.digest("base64");
|
|
339
|
+
socket.write(
|
|
340
|
+
"HTTP/1.1 101 Switching Protocols\r\n" +
|
|
341
|
+
"Upgrade: websocket\r\n" +
|
|
342
|
+
"Connection: Upgrade\r\n" +
|
|
343
|
+
`Sec-WebSocket-Accept: ${accept}\r\n\r\n`,
|
|
344
|
+
);
|
|
345
|
+
const id = callIdIn(url);
|
|
346
|
+
const session = sessionsById.get(id);
|
|
347
|
+
const hangup = () => session?.close();
|
|
348
|
+
socket.on("error", () => {});
|
|
349
|
+
socket.on("close", hangup);
|
|
350
|
+
socket.on("end", hangup);
|
|
351
|
+
if (session?.attachSideband) session.attachSideband(socket);
|
|
352
|
+
else socket.on("data", () => {});
|
|
353
|
+
if (head?.length) socket.unshift(head);
|
|
354
|
+
socket.resume();
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
function waitForHostCandidate(pc) {
|
|
358
|
+
const sdp = () => hostAnswerSdp(pc.localDescription?.sdp ?? "");
|
|
359
|
+
const hasHost = () => (pc.localDescription?.sdp ?? "").includes(" typ host");
|
|
360
|
+
const hasLoopback = () => (pc.localDescription?.sdp ?? "").includes(" 127.0.0.1 ");
|
|
361
|
+
if (hasHost() && hasLoopback()) return Promise.resolve(sdp());
|
|
362
|
+
return new Promise((resolve) => {
|
|
363
|
+
let done = false;
|
|
364
|
+
const finish = () => {
|
|
365
|
+
if (done) return;
|
|
366
|
+
done = true;
|
|
367
|
+
clearTimeout(limit);
|
|
368
|
+
clearTimeout(loopbackGrace);
|
|
369
|
+
resolve(sdp());
|
|
370
|
+
};
|
|
371
|
+
const limit = setTimeout(finish, 500);
|
|
372
|
+
const loopbackGrace = setTimeout(() => {
|
|
373
|
+
if (hasHost()) finish();
|
|
374
|
+
}, 100);
|
|
375
|
+
const check = () => {
|
|
376
|
+
if (hasHost() && hasLoopback()) finish();
|
|
377
|
+
};
|
|
378
|
+
if (typeof pc.iceGatheringStateChange?.subscribe === "function") {
|
|
379
|
+
pc.iceGatheringStateChange.subscribe(check);
|
|
380
|
+
}
|
|
381
|
+
if (typeof pc.onIceCandidate?.subscribe === "function") pc.onIceCandidate.subscribe(check);
|
|
382
|
+
check();
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
const MULAW_BIAS = 0x84;
|
|
387
|
+
const MULAW_CLIP = 32635;
|
|
388
|
+
|
|
389
|
+
export function mulawToPcm16(mulaw) {
|
|
390
|
+
const input = Buffer.from(mulaw);
|
|
391
|
+
const out = Buffer.alloc(input.length * 2);
|
|
392
|
+
for (let i = 0; i < input.length; i += 1) {
|
|
393
|
+
const sample = ~input[i] & 0xff;
|
|
394
|
+
const sign = sample & 0x80;
|
|
395
|
+
const exponent = (sample >> 4) & 0x07;
|
|
396
|
+
const mantissa = sample & 0x0f;
|
|
397
|
+
let pcm = ((mantissa << 3) + MULAW_BIAS) << exponent;
|
|
398
|
+
pcm -= MULAW_BIAS;
|
|
399
|
+
out.writeInt16LE(sign ? -pcm : pcm, i * 2);
|
|
400
|
+
}
|
|
401
|
+
return out;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
export function pcm16ToMulaw(pcm) {
|
|
405
|
+
const input = Buffer.from(pcm);
|
|
406
|
+
const samples = Math.floor(input.length / 2);
|
|
407
|
+
const out = Buffer.alloc(samples);
|
|
408
|
+
for (let i = 0; i < samples; i += 1) {
|
|
409
|
+
let sample = input.readInt16LE(i * 2);
|
|
410
|
+
const sign = sample < 0 ? 0x80 : 0;
|
|
411
|
+
if (sample < 0) sample = -sample;
|
|
412
|
+
if (sample > MULAW_CLIP) sample = MULAW_CLIP;
|
|
413
|
+
sample += MULAW_BIAS;
|
|
414
|
+
let exponent = 7;
|
|
415
|
+
for (let mask = 0x4000; (sample & mask) === 0 && exponent > 0; mask >>= 1) exponent -= 1;
|
|
416
|
+
const mantissa = (sample >> (exponent + 3)) & 0x0f;
|
|
417
|
+
out[i] = ~(sign | (exponent << 4) | mantissa) & 0xff;
|
|
418
|
+
}
|
|
419
|
+
return out;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
const XAI_PCM_RATE = 24000;
|
|
423
|
+
|
|
424
|
+
function resampleInt16(pcm, factor, direction) {
|
|
425
|
+
const samples = Math.floor(Buffer.from(pcm).length / 2);
|
|
426
|
+
const input = Buffer.from(pcm);
|
|
427
|
+
if (direction === "down") {
|
|
428
|
+
const outSamples = Math.floor(samples / factor);
|
|
429
|
+
const out = Buffer.alloc(outSamples * 2);
|
|
430
|
+
for (let i = 0; i < outSamples; i += 1) out.writeInt16LE(input.readInt16LE(i * factor * 2), i * 2);
|
|
431
|
+
return out;
|
|
432
|
+
}
|
|
433
|
+
const out = Buffer.alloc(samples * factor * 2);
|
|
434
|
+
for (let i = 0; i < samples; i += 1) {
|
|
435
|
+
const sample = input.readInt16LE(i * 2);
|
|
436
|
+
for (let step = 0; step < factor; step += 1) out.writeInt16LE(sample, (i * factor + step) * 2);
|
|
437
|
+
}
|
|
438
|
+
return out;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function pcmForXai(payload, codec, decoder) {
|
|
442
|
+
if (codec.kind === "opus" && decoder) return resampleInt16(opusPayloadToMono(decoder, payload), 2, "down");
|
|
443
|
+
return resampleInt16(mulawToPcm16(payload), 3, "up");
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
export function appendFromRtp(send, payload, codec = { kind: "pcmu", payloadType: 0 }, decoder) {
|
|
447
|
+
const pcm = pcmForXai(payload, codec, decoder);
|
|
448
|
+
if (pcm.length === 0) return;
|
|
449
|
+
send({
|
|
450
|
+
type: "input_audio_buffer.append",
|
|
451
|
+
audio: pcm.toString("base64"),
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
// A late 20 ms timer is still one talkspurt. Only a real pause moves the clock,
|
|
456
|
+
// or the helper treats the next packet as a new, late frame and can drop it.
|
|
457
|
+
export function playoutGap(state, now) {
|
|
458
|
+
if (state.lastSentAt == null) return 0;
|
|
459
|
+
const frames = Math.floor((now - state.lastSentAt) / 20);
|
|
460
|
+
if (frames < 4) return 0;
|
|
461
|
+
return frames - 1;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
function writeAudio(track, state, payload, clockStep, forceMarker = false) {
|
|
465
|
+
if (payload.length === 0) return;
|
|
466
|
+
const { RtpHeader, RtpPacket } = voicePackages();
|
|
467
|
+
const now = Date.now();
|
|
468
|
+
const extra = playoutGap(state, now);
|
|
469
|
+
const sequence = (state.sequence + 1) & 0xffff;
|
|
470
|
+
const timestamp = (state.timestamp + (extra + 1) * clockStep) >>> 0;
|
|
471
|
+
track.writeRtp(
|
|
472
|
+
new RtpPacket(
|
|
473
|
+
new RtpHeader({
|
|
474
|
+
marker: forceMarker || extra > 0 || !state.talking,
|
|
475
|
+
payloadType: state.payloadType,
|
|
476
|
+
sequenceNumber: sequence,
|
|
477
|
+
timestamp,
|
|
478
|
+
ssrc: state.ssrc,
|
|
479
|
+
}),
|
|
480
|
+
payload,
|
|
481
|
+
),
|
|
482
|
+
);
|
|
483
|
+
state.talking = true;
|
|
484
|
+
state.sequence = sequence;
|
|
485
|
+
state.timestamp = timestamp;
|
|
486
|
+
state.lastSentAt = now;
|
|
487
|
+
const rate = state.kind === "opus" ? 48000 : 8000;
|
|
488
|
+
state.playedMs = (state.playedMs ?? 0) + Math.round((clockStep / rate) * 1000);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
export function voiceAudioDelta(event) {
|
|
492
|
+
if (event?.type !== "response.output_audio.delta" && event?.type !== "response.audio.delta") return "";
|
|
493
|
+
if (typeof event.delta === "string" && event.delta) return event.delta;
|
|
494
|
+
if (typeof event.audio === "string" && event.audio) return event.audio;
|
|
495
|
+
return "";
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
export function playbackFromDelta(track, state, delta) {
|
|
499
|
+
const pcm = Buffer.from(String(delta ?? ""), "base64");
|
|
500
|
+
if (state.kind === "opus" && state.opus) {
|
|
501
|
+
if (state.endTalk) state.armMarker = true;
|
|
502
|
+
state.endTalk = false;
|
|
503
|
+
const wide = resampleInt16(pcm, 2, "up");
|
|
504
|
+
state.pending = Buffer.concat([state.pending ?? Buffer.alloc(0), wide]);
|
|
505
|
+
const frameBytes = OPUS_FRAME * 2;
|
|
506
|
+
while (state.pending.length >= frameBytes) {
|
|
507
|
+
const frame = state.pending.subarray(0, frameBytes);
|
|
508
|
+
state.pending = state.pending.subarray(frameBytes);
|
|
509
|
+
const encoded = Buffer.from(state.opus.encode(monoToStereo(frame), OPUS_FRAME));
|
|
510
|
+
enqueueAudio(state, encoded, OPUS_FRAME);
|
|
511
|
+
}
|
|
512
|
+
kickAudio(track, state);
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
const mulaw = pcm16ToMulaw(resampleInt16(pcm, 3, "down"));
|
|
516
|
+
writeAudio(track, state, mulaw, mulaw.length);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
function enqueueAudio(state, payload, clockStep) {
|
|
520
|
+
if (!state.playout) state.playout = [];
|
|
521
|
+
const marker = state.armMarker === true;
|
|
522
|
+
if (marker) state.armMarker = false;
|
|
523
|
+
state.playout.push({ payload, clockStep, marker });
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
function scheduleAudio(track, state) {
|
|
527
|
+
if (state.stopped) return;
|
|
528
|
+
state.pumping = true;
|
|
529
|
+
const timer = setTimeout(() => {
|
|
530
|
+
state.pumping = false;
|
|
531
|
+
state.playoutTimer = null;
|
|
532
|
+
kickAudio(track, state);
|
|
533
|
+
}, 20);
|
|
534
|
+
timer.unref?.();
|
|
535
|
+
state.playoutTimer = timer;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function kickAudio(track, state) {
|
|
539
|
+
if (state.stopped || state.pumping) return;
|
|
540
|
+
const frame = state.playout?.[0];
|
|
541
|
+
if (!frame) {
|
|
542
|
+
if (state.endTalk) state.talking = false;
|
|
543
|
+
state.onDrained?.();
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
try {
|
|
547
|
+
writeAudio(track, state, frame.payload, frame.clockStep, frame.marker);
|
|
548
|
+
} catch {
|
|
549
|
+
scheduleAudio(track, state);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
state.playout.shift();
|
|
553
|
+
if (!state.playout.length) {
|
|
554
|
+
if (state.endTalk) state.talking = false;
|
|
555
|
+
state.onDrained?.();
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
558
|
+
scheduleAudio(track, state);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
export function stopPlayback(state, options = {}) {
|
|
562
|
+
state.playout = [];
|
|
563
|
+
state.pending = Buffer.alloc(0);
|
|
564
|
+
state.pumping = false;
|
|
565
|
+
state.dropping = true;
|
|
566
|
+
state.userSpeaking = options.userSpeaking !== false;
|
|
567
|
+
state.endTalk = true;
|
|
568
|
+
state.talking = false;
|
|
569
|
+
if (state.playoutTimer) {
|
|
570
|
+
clearTimeout(state.playoutTimer);
|
|
571
|
+
state.playoutTimer = null;
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
// The socket is gone, so frames still queued here will never belong to the next
|
|
576
|
+
// response. Leave dropping and userSpeaking alone; the caller decides those.
|
|
577
|
+
function discardUnsentAudio(state) {
|
|
578
|
+
state.playout = [];
|
|
579
|
+
state.pending = Buffer.alloc(0);
|
|
580
|
+
state.pumping = false;
|
|
581
|
+
state.endTalk = true;
|
|
582
|
+
state.talking = false;
|
|
583
|
+
if (state.playoutTimer) {
|
|
584
|
+
clearTimeout(state.playoutTimer);
|
|
585
|
+
state.playoutTimer = null;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
function flushPlayback(track, state) {
|
|
590
|
+
if (state.dropping) return;
|
|
591
|
+
if (state.kind !== "opus" || !state.opus || !state.pending?.length) return;
|
|
592
|
+
const frameBytes = OPUS_FRAME * 2;
|
|
593
|
+
const padded = Buffer.alloc(frameBytes);
|
|
594
|
+
state.pending.copy(padded);
|
|
595
|
+
state.pending = Buffer.alloc(0);
|
|
596
|
+
const encoded = Buffer.from(state.opus.encode(monoToStereo(padded), OPUS_FRAME));
|
|
597
|
+
enqueueAudio(state, encoded, OPUS_FRAME);
|
|
598
|
+
kickAudio(track, state);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
const CODEX_TOOL = {
|
|
602
|
+
type: "function",
|
|
603
|
+
name: "codex",
|
|
604
|
+
description:
|
|
605
|
+
"Ask the Codex coding agent to carry out work in the user's workspace. Use this when the user asks to read, edit, run, or inspect the project. Do not use it for ordinary conversation.",
|
|
606
|
+
parameters: {
|
|
607
|
+
type: "object",
|
|
608
|
+
properties: {
|
|
609
|
+
request: { type: "string", description: "What the user wants done, in their words." },
|
|
610
|
+
},
|
|
611
|
+
required: ["request"],
|
|
612
|
+
},
|
|
613
|
+
};
|
|
614
|
+
|
|
615
|
+
const ACK_FILLER =
|
|
616
|
+
"When you call the codex tool, say one short sentence first so the user hears that work has started.";
|
|
617
|
+
|
|
618
|
+
function sessionUpdate(instructions, ackFiller = false, developerContext = "") {
|
|
619
|
+
let base = typeof instructions === "string" && instructions.trim() ? instructions.trim() : "Say ready.";
|
|
620
|
+
const context = typeof developerContext === "string" ? developerContext.trim() : "";
|
|
621
|
+
if (context && !base.includes(context)) base = `${base}\n\n${context}`;
|
|
622
|
+
const text = ackFiller && !base.includes(ACK_FILLER) ? `${base}\n\n${ACK_FILLER}` : base;
|
|
623
|
+
return {
|
|
624
|
+
type: "session.update",
|
|
625
|
+
session: {
|
|
626
|
+
voice: "eve",
|
|
627
|
+
instructions: text,
|
|
628
|
+
turn_detection: { type: "server_vad" },
|
|
629
|
+
resumption: { enabled: true },
|
|
630
|
+
tools: [CODEX_TOOL],
|
|
631
|
+
audio: {
|
|
632
|
+
input: {
|
|
633
|
+
format: { type: "audio/pcm", rate: XAI_PCM_RATE },
|
|
634
|
+
transcription: { model: "grok-transcribe" },
|
|
635
|
+
},
|
|
636
|
+
output: { format: { type: "audio/pcm", rate: XAI_PCM_RATE } },
|
|
637
|
+
},
|
|
638
|
+
},
|
|
639
|
+
};
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
function textFromContent(content) {
|
|
643
|
+
if (!Array.isArray(content)) return "";
|
|
644
|
+
return content
|
|
645
|
+
.filter((part) => part?.type === "input_text" && typeof part.text === "string")
|
|
646
|
+
.map((part) => part.text)
|
|
647
|
+
.join("");
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
function contextChannel(event) {
|
|
651
|
+
return event?.channel ?? null;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// Desktop splits one append into 500-byte frames. Join a back-to-back burst
|
|
655
|
+
// before the tool result and the spoken line are built.
|
|
656
|
+
export function mergeVoiceClientBurst(events) {
|
|
657
|
+
const merged = [];
|
|
658
|
+
for (const event of events) {
|
|
659
|
+
const prev = merged.at(-1);
|
|
660
|
+
const type = event?.type;
|
|
661
|
+
const sameDelegation =
|
|
662
|
+
prev?.type === "delegation.context.append" &&
|
|
663
|
+
type === "delegation.context.append" &&
|
|
664
|
+
prev.delegation_item_id === event.delegation_item_id &&
|
|
665
|
+
contextChannel(prev) === contextChannel(event);
|
|
666
|
+
const sameSession =
|
|
667
|
+
prev?.type === "session.context.append" &&
|
|
668
|
+
type === "session.context.append" &&
|
|
669
|
+
contextChannel(prev) === contextChannel(event);
|
|
670
|
+
if (sameDelegation || sameSession) {
|
|
671
|
+
merged[merged.length - 1] = {
|
|
672
|
+
...prev,
|
|
673
|
+
content: [{ type: "input_text", text: textFromContent(prev.content) + textFromContent(event.content) }],
|
|
674
|
+
};
|
|
675
|
+
continue;
|
|
676
|
+
}
|
|
677
|
+
merged.push(structuredClone(event));
|
|
678
|
+
}
|
|
679
|
+
return merged;
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// v3 puts instructions in the call's session part and later sends frameless
|
|
683
|
+
// appends. xAI accepts session.update, conversation.item.create, and force_message.
|
|
684
|
+
function codexRequest(argumentsText) {
|
|
685
|
+
if (typeof argumentsText !== "string" || !argumentsText.trim()) return "";
|
|
686
|
+
try {
|
|
687
|
+
const args = JSON.parse(argumentsText);
|
|
688
|
+
if (typeof args?.request === "string" && args.request.trim()) return args.request.trim();
|
|
689
|
+
} catch {
|
|
690
|
+
// The model sometimes sends plain text instead of JSON.
|
|
691
|
+
}
|
|
692
|
+
return argumentsText.trim();
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
export function voiceClientEvents(event, state = {}) {
|
|
696
|
+
const type = event?.type;
|
|
697
|
+
if (type === "response.create" || type === "response.cancel" || type === "conversation.item.create") {
|
|
698
|
+
return [event];
|
|
699
|
+
}
|
|
700
|
+
if (type === "input_audio.append" && typeof event.audio === "string" && event.audio) {
|
|
701
|
+
return [{ type: "input_audio_buffer.append", audio: event.audio }];
|
|
702
|
+
}
|
|
703
|
+
if (type === "session.update") {
|
|
704
|
+
const instructions = typeof event.session?.instructions === "string" ? event.session.instructions.trim() : "";
|
|
705
|
+
return instructions
|
|
706
|
+
? [sessionUpdate(instructions, state.ackFiller === true, state.developerContext)]
|
|
707
|
+
: [];
|
|
708
|
+
}
|
|
709
|
+
if (type === "session.context.append" || type === "delegation.context.append") {
|
|
710
|
+
const text = textFromContent(event.content);
|
|
711
|
+
if (!text) return [];
|
|
712
|
+
if (event.channel === "commentary") {
|
|
713
|
+
return [
|
|
714
|
+
{
|
|
715
|
+
type: "conversation.item.create",
|
|
716
|
+
item: { type: "message", role: "assistant", content: [{ type: "input_text", text }] },
|
|
717
|
+
},
|
|
718
|
+
];
|
|
719
|
+
}
|
|
720
|
+
const spoken = {
|
|
721
|
+
type: "conversation.item.create",
|
|
722
|
+
item: { type: "force_message", role: "assistant", content: [{ type: "output_text", text }] },
|
|
723
|
+
};
|
|
724
|
+
const callId = event.delegation_item_id;
|
|
725
|
+
if (typeof callId === "string" && state.pendingCalls?.has(callId)) {
|
|
726
|
+
state.pendingCalls.delete(callId);
|
|
727
|
+
return [
|
|
728
|
+
{
|
|
729
|
+
type: "conversation.item.create",
|
|
730
|
+
item: { type: "function_call_output", call_id: callId, output: JSON.stringify(text) },
|
|
731
|
+
},
|
|
732
|
+
spoken,
|
|
733
|
+
];
|
|
734
|
+
}
|
|
735
|
+
return [spoken];
|
|
736
|
+
}
|
|
737
|
+
return [];
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
// Desktop streams a codex reply in pieces about 200 ms apart. Hold them until
|
|
741
|
+
// the stream pauses, then close the tool and speak the whole reply once.
|
|
742
|
+
export const DELEGATION_SPEECH_WAIT_MS = 350;
|
|
743
|
+
export const MIC_HOLD_MS = 2000;
|
|
744
|
+
export const MIC_HOLD_FRAMES = MIC_HOLD_MS / 20;
|
|
745
|
+
|
|
746
|
+
export function queueDelegationSpeech(state, event) {
|
|
747
|
+
if (event?.type !== "delegation.context.append") return false;
|
|
748
|
+
const id = event.delegation_item_id;
|
|
749
|
+
if (typeof id !== "string") return false;
|
|
750
|
+
const text = textFromContent(event.content);
|
|
751
|
+
if (event.channel === "commentary" && text.startsWith("[STATUS]")) return false;
|
|
752
|
+
if (!state.pendingCalls?.has(id) && !state.speechParts?.has(id)) return false;
|
|
753
|
+
if (!state.speechParts) state.speechParts = new Map();
|
|
754
|
+
const prior = state.speechParts.get(id) ?? { text: "", commentary: event.channel === "commentary" };
|
|
755
|
+
if (event.channel !== "commentary") prior.commentary = false;
|
|
756
|
+
if (text) prior.text += text;
|
|
757
|
+
state.speechParts.set(id, prior);
|
|
758
|
+
return true;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
export function flushDelegationSpeech(state) {
|
|
762
|
+
const parts = state.speechParts;
|
|
763
|
+
if (!parts?.size) return [];
|
|
764
|
+
const events = [];
|
|
765
|
+
const spoken = [];
|
|
766
|
+
let commentary = false;
|
|
767
|
+
for (const [id, record] of parts) {
|
|
768
|
+
const text = record.text ?? "";
|
|
769
|
+
if (record.commentary) {
|
|
770
|
+
commentary = true;
|
|
771
|
+
if (state.pendingCalls?.has(id)) {
|
|
772
|
+
state.pendingCalls.delete(id);
|
|
773
|
+
events.push({
|
|
774
|
+
type: "conversation.item.create",
|
|
775
|
+
item: { type: "function_call_output", call_id: id, output: JSON.stringify(text) },
|
|
776
|
+
});
|
|
777
|
+
}
|
|
778
|
+
if (text) {
|
|
779
|
+
events.push({
|
|
780
|
+
type: "conversation.item.create",
|
|
781
|
+
item: { type: "message", role: "assistant", content: [{ type: "input_text", text }] },
|
|
782
|
+
});
|
|
783
|
+
}
|
|
784
|
+
continue;
|
|
785
|
+
}
|
|
786
|
+
for (const event of voiceClientEvents(
|
|
787
|
+
{
|
|
788
|
+
type: "delegation.context.append",
|
|
789
|
+
delegation_item_id: id,
|
|
790
|
+
content: [{ type: "input_text", text }],
|
|
791
|
+
},
|
|
792
|
+
state,
|
|
793
|
+
)) {
|
|
794
|
+
if (event?.item?.type === "force_message") spoken.push(event);
|
|
795
|
+
else events.push(event);
|
|
796
|
+
}
|
|
797
|
+
}
|
|
798
|
+
parts.clear();
|
|
799
|
+
events.push(...spoken);
|
|
800
|
+
// force_message is itself the turn. A commentary batch continues once, after every result.
|
|
801
|
+
if (commentary && spoken.length === 0) events.push({ type: "response.create" });
|
|
802
|
+
return events;
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// Desktop v3 reads frameless sideband events. xAI voice uses different names.
|
|
806
|
+
// Cumulative user captions are reduced to the new suffix. A rewrite that no
|
|
807
|
+
// longer extends the previous caption is sent as the completed turn text.
|
|
808
|
+
function assistantTextDelta(event) {
|
|
809
|
+
if (event?.type !== "response.output_text.delta" && event?.type !== "response.text.delta") return "";
|
|
810
|
+
if (typeof event.delta === "string" && event.delta) return event.delta;
|
|
811
|
+
if (typeof event.text === "string" && event.text) return event.text;
|
|
812
|
+
return "";
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
export function voiceSidebandEvent(event, state) {
|
|
816
|
+
const type = event?.type;
|
|
817
|
+
if (type === "response.created") {
|
|
818
|
+
state.outputText = "";
|
|
819
|
+
state.audioText = "";
|
|
820
|
+
state.audioTranscript = false;
|
|
821
|
+
state.assistantTurnClosed = false;
|
|
822
|
+
state.interrupted = false;
|
|
823
|
+
return null;
|
|
824
|
+
}
|
|
825
|
+
const text = assistantTextDelta(event);
|
|
826
|
+
if (text) {
|
|
827
|
+
if (state.interrupted || state.audioTranscript) return null;
|
|
828
|
+
state.outputText = `${state.outputText ?? ""}${text}`;
|
|
829
|
+
return { type: "output_transcript.added", item: { text } };
|
|
830
|
+
}
|
|
831
|
+
if (type === "response.output_text.done" || type === "response.text.done") {
|
|
832
|
+
if (state.assistantTurnClosed && !state.interrupted) return null;
|
|
833
|
+
const streamed = state.outputText ?? "";
|
|
834
|
+
const full = state.interrupted
|
|
835
|
+
? streamed
|
|
836
|
+
: typeof event.text === "string" && event.text
|
|
837
|
+
? event.text
|
|
838
|
+
: typeof event.transcript === "string" && event.transcript
|
|
839
|
+
? event.transcript
|
|
840
|
+
: streamed;
|
|
841
|
+
state.outputText = "";
|
|
842
|
+
state.assistantTurnClosed = true;
|
|
843
|
+
return full ? { type: "turn.done", turn: { role: "assistant", transcript: full } } : null;
|
|
844
|
+
}
|
|
845
|
+
if (type === "response.output_audio_transcript.delta" && typeof event.delta === "string" && event.delta) {
|
|
846
|
+
if (state.interrupted) return null;
|
|
847
|
+
state.audioTranscript = true;
|
|
848
|
+
state.outputText = "";
|
|
849
|
+
state.audioText = `${state.audioText ?? ""}${event.delta}`;
|
|
850
|
+
return { type: "output_transcript.added", item: { text: event.delta } };
|
|
851
|
+
}
|
|
852
|
+
if (type === "response.output_audio_transcript.done" && typeof event.transcript === "string" && event.transcript) {
|
|
853
|
+
if (state.assistantTurnClosed && !state.interrupted) return null;
|
|
854
|
+
const full = state.interrupted ? (state.audioText ?? "") : event.transcript;
|
|
855
|
+
state.outputText = "";
|
|
856
|
+
state.audioText = "";
|
|
857
|
+
state.assistantTurnClosed = true;
|
|
858
|
+
return full ? { type: "turn.done", turn: { role: "assistant", transcript: full } } : null;
|
|
859
|
+
}
|
|
860
|
+
if (type === "response.done" && !state.assistantTurnClosed) {
|
|
861
|
+
const full = state.audioTranscript ? (state.audioText ?? "") : (state.outputText ?? "");
|
|
862
|
+
state.outputText = "";
|
|
863
|
+
state.audioText = "";
|
|
864
|
+
if (!full) return null;
|
|
865
|
+
state.assistantTurnClosed = true;
|
|
866
|
+
return { type: "turn.done", turn: { role: "assistant", transcript: full } };
|
|
867
|
+
}
|
|
868
|
+
if (type === "input_audio_buffer.speech_started") {
|
|
869
|
+
if (!state.assistantTurnClosed) state.interrupted = true;
|
|
870
|
+
if (!state.inputTranscript) return null;
|
|
871
|
+
const transcript = state.inputTranscript;
|
|
872
|
+
state.inputTranscript = "";
|
|
873
|
+
state.userClosed = transcript;
|
|
874
|
+
return { type: "turn.done", turn: { role: "user", transcript } };
|
|
875
|
+
}
|
|
876
|
+
if (type === "conversation.item.input_audio_transcription.updated" && typeof event.transcript === "string") {
|
|
877
|
+
const next = event.transcript;
|
|
878
|
+
const prev = state.inputTranscript ?? "";
|
|
879
|
+
state.inputTranscript = next;
|
|
880
|
+
if (!next || next === prev) return null;
|
|
881
|
+
if (next.startsWith(prev)) {
|
|
882
|
+
const suffix = next.slice(prev.length);
|
|
883
|
+
return suffix ? { type: "input_transcript.added", item: { text: suffix } } : null;
|
|
884
|
+
}
|
|
885
|
+
return { type: "turn.done", turn: { role: "user", transcript: next } };
|
|
886
|
+
}
|
|
887
|
+
if (
|
|
888
|
+
type === "conversation.item.input_audio_transcription.completed" &&
|
|
889
|
+
typeof event.transcript === "string" &&
|
|
890
|
+
event.transcript
|
|
891
|
+
) {
|
|
892
|
+
const closed = state.userClosed ?? "";
|
|
893
|
+
state.inputTranscript = "";
|
|
894
|
+
state.userClosed = "";
|
|
895
|
+
if (closed && (event.transcript === closed || event.transcript.startsWith(closed))) return null;
|
|
896
|
+
return { type: "turn.done", turn: { role: "user", transcript: event.transcript } };
|
|
897
|
+
}
|
|
898
|
+
if ((type === "session.created" || type === "session.updated") && typeof event.session?.id === "string") {
|
|
899
|
+
const session = { id: event.session.id };
|
|
900
|
+
if (typeof event.session.instructions === "string") session.instructions = event.session.instructions;
|
|
901
|
+
return { type: "session.updated", session };
|
|
902
|
+
}
|
|
903
|
+
if (type === "response.function_call_arguments.done" && event.name === "codex" && typeof event.call_id === "string") {
|
|
904
|
+
const request = codexRequest(event.arguments);
|
|
905
|
+
if (!request) return null;
|
|
906
|
+
if (!state.pendingCalls) state.pendingCalls = new Set();
|
|
907
|
+
state.pendingCalls.add(event.call_id);
|
|
908
|
+
return {
|
|
909
|
+
type: "delegation.created",
|
|
910
|
+
item: {
|
|
911
|
+
type: "delegation",
|
|
912
|
+
target: "client",
|
|
913
|
+
id: event.call_id,
|
|
914
|
+
content: [{ type: "input_text", text: request }],
|
|
915
|
+
},
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
// Codex treats every sideband error as fatal and ends the call. xAI errors
|
|
919
|
+
// are usually recoverable, so they stay here. A dead socket reports its own error.
|
|
920
|
+
return null;
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
// The desktop waits for session.updated before it will read any other sideband event.
|
|
924
|
+
export function announceSession(state, pending) {
|
|
925
|
+
if (state.sessionAnnounced || !state.sessionId) return pending;
|
|
926
|
+
state.sessionAnnounced = true;
|
|
927
|
+
return [JSON.stringify({ type: "session.updated", session: { id: state.sessionId } }), ...pending];
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
// The desktop rejects a frameless call whose first event is not session.updated.
|
|
931
|
+
export function sidebandFrames(state, text) {
|
|
932
|
+
const frames = [];
|
|
933
|
+
let event = null;
|
|
934
|
+
try {
|
|
935
|
+
event = JSON.parse(text);
|
|
936
|
+
} catch {
|
|
937
|
+
event = null;
|
|
938
|
+
}
|
|
939
|
+
const announced = event?.type === "session.updated" && typeof event.session?.id === "string";
|
|
940
|
+
if (!state.sessionAnnounced && !announced && state.sessionId) {
|
|
941
|
+
frames.push(JSON.stringify({ type: "session.updated", session: { id: state.sessionId } }));
|
|
942
|
+
}
|
|
943
|
+
if (announced || frames.length) state.sessionAnnounced = true;
|
|
944
|
+
frames.push(text);
|
|
945
|
+
return frames;
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
export function startVoiceBridge({
|
|
949
|
+
track,
|
|
950
|
+
socket,
|
|
951
|
+
codec = { kind: "pcmu", payloadType: 0 },
|
|
952
|
+
onEvent,
|
|
953
|
+
instructions,
|
|
954
|
+
ackFiller = false,
|
|
955
|
+
developerContext = "",
|
|
956
|
+
initialItems = [],
|
|
957
|
+
voiceState,
|
|
958
|
+
receiveTracks = [],
|
|
959
|
+
openSocket,
|
|
960
|
+
}) {
|
|
961
|
+
const opus = codec.kind === "opus" ? createOpus() : null;
|
|
962
|
+
const state = {
|
|
963
|
+
sequence: 0,
|
|
964
|
+
timestamp: 0,
|
|
965
|
+
ssrc: 1,
|
|
966
|
+
kind: codec.kind,
|
|
967
|
+
payloadType: codec.payloadType,
|
|
968
|
+
opus,
|
|
969
|
+
pending: Buffer.alloc(0),
|
|
970
|
+
};
|
|
971
|
+
const queued = [];
|
|
972
|
+
const held = [];
|
|
973
|
+
const sidebandState = voiceState ?? { inputTranscript: "", pendingCalls: new Set() };
|
|
974
|
+
if (developerContext && !sidebandState.developerContext) sidebandState.developerContext = developerContext;
|
|
975
|
+
let current = socket;
|
|
976
|
+
let opened = socket.readyState === 1;
|
|
977
|
+
let responseActive = false;
|
|
978
|
+
let pendingCreate = false;
|
|
979
|
+
let awaitingCreated = false;
|
|
980
|
+
let resumed = false;
|
|
981
|
+
let sessionReady = false;
|
|
982
|
+
const pendingAudio = [];
|
|
983
|
+
const send = (event) => {
|
|
984
|
+
if (!sessionReady && event?.type === "input_audio_buffer.append") {
|
|
985
|
+
pendingAudio.push(event);
|
|
986
|
+
if (pendingAudio.length > MIC_HOLD_FRAMES) pendingAudio.shift();
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
const text = JSON.stringify(event);
|
|
990
|
+
if (!opened) {
|
|
991
|
+
queued.push(text);
|
|
992
|
+
return;
|
|
993
|
+
}
|
|
994
|
+
current.send(text);
|
|
995
|
+
};
|
|
996
|
+
let readyTimer = null;
|
|
997
|
+
const armReadyTimer = () => {
|
|
998
|
+
if (readyTimer) clearTimeout(readyTimer);
|
|
999
|
+
readyTimer = setTimeout(() => markSessionReady(), MIC_HOLD_MS);
|
|
1000
|
+
readyTimer.unref();
|
|
1001
|
+
};
|
|
1002
|
+
const markSessionReady = () => {
|
|
1003
|
+
if (sessionReady) return;
|
|
1004
|
+
sessionReady = true;
|
|
1005
|
+
if (readyTimer) clearTimeout(readyTimer);
|
|
1006
|
+
for (const event of pendingAudio.splice(0)) send(event);
|
|
1007
|
+
releaseHeld();
|
|
1008
|
+
};
|
|
1009
|
+
armReadyTimer();
|
|
1010
|
+
const holdsForIdleResponse = (event) => {
|
|
1011
|
+
const item = event?.item;
|
|
1012
|
+
return (
|
|
1013
|
+
event?.type === "response.create" ||
|
|
1014
|
+
(event?.type === "conversation.item.create" &&
|
|
1015
|
+
(item?.type === "force_message" || item?.type === "function_call_output"))
|
|
1016
|
+
);
|
|
1017
|
+
};
|
|
1018
|
+
const playbackBusy = () => Boolean(state.playout?.length || state.pumping);
|
|
1019
|
+
const startsResponse = (event) =>
|
|
1020
|
+
event?.type === "response.create" ||
|
|
1021
|
+
(event?.type === "conversation.item.create" && event.item?.type === "force_message");
|
|
1022
|
+
const releaseHeld = () => {
|
|
1023
|
+
if (responseActive || playbackBusy() || state.userSpeaking) return;
|
|
1024
|
+
while (held.length) {
|
|
1025
|
+
const event = held[0];
|
|
1026
|
+
if (startsResponse(event) && responseActive) return;
|
|
1027
|
+
held.shift();
|
|
1028
|
+
send(event);
|
|
1029
|
+
if (startsResponse(event)) {
|
|
1030
|
+
responseActive = true;
|
|
1031
|
+
awaitingCreated = true;
|
|
1032
|
+
if (event?.item?.type === "force_message") pendingCreate = false;
|
|
1033
|
+
return;
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
};
|
|
1037
|
+
const interruptAssistant = (userSpeaking) => {
|
|
1038
|
+
const open = !sidebandState.assistantTurnClosed;
|
|
1039
|
+
if (open) sidebandState.interrupted = true;
|
|
1040
|
+
const unsent = Boolean(state.playout?.length || state.pending?.length);
|
|
1041
|
+
if (!open && !unsent) {
|
|
1042
|
+
state.userSpeaking = userSpeaking;
|
|
1043
|
+
return;
|
|
1044
|
+
}
|
|
1045
|
+
stopPlayback(state, { userSpeaking });
|
|
1046
|
+
if (unsent && state.audioItemId && !state.truncated) {
|
|
1047
|
+
state.truncated = true;
|
|
1048
|
+
send({
|
|
1049
|
+
type: "conversation.item.truncate",
|
|
1050
|
+
item_id: state.audioItemId,
|
|
1051
|
+
content_index: state.audioContentIndex ?? 0,
|
|
1052
|
+
audio_end_ms: state.playedMs ?? 0,
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
};
|
|
1056
|
+
state.onDrained = releaseHeld;
|
|
1057
|
+
const flush = () => {
|
|
1058
|
+
opened = true;
|
|
1059
|
+
for (const text of queued.splice(0)) current.send(text);
|
|
1060
|
+
};
|
|
1061
|
+
const onRtp = (rtp) => {
|
|
1062
|
+
const payload = rtp?.payload ?? rtp;
|
|
1063
|
+
const payloadType = rtp?.header?.payloadType;
|
|
1064
|
+
const incoming =
|
|
1065
|
+
payloadType === 0
|
|
1066
|
+
? { kind: "pcmu", payloadType: 0 }
|
|
1067
|
+
: payloadType == null
|
|
1068
|
+
? codec
|
|
1069
|
+
: { kind: "opus", payloadType };
|
|
1070
|
+
try {
|
|
1071
|
+
appendFromRtp(send, payload, incoming, opus);
|
|
1072
|
+
} catch {
|
|
1073
|
+
// A bad frame is dropped. The call stays up.
|
|
1074
|
+
}
|
|
1075
|
+
};
|
|
1076
|
+
const onUpstream = (raw) => {
|
|
1077
|
+
let event;
|
|
1078
|
+
try {
|
|
1079
|
+
event = JSON.parse(typeof raw === "string" ? raw : Buffer.from(raw?.data ?? raw).toString("utf8"));
|
|
1080
|
+
} catch {
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
const errorMessage = event?.error?.message ?? event?.message;
|
|
1084
|
+
if (
|
|
1085
|
+
event?.type === "error" &&
|
|
1086
|
+
typeof errorMessage === "string" &&
|
|
1087
|
+
errorMessage.startsWith("Conversation already has an active response in progress")
|
|
1088
|
+
) {
|
|
1089
|
+
responseActive = true;
|
|
1090
|
+
pendingCreate = true;
|
|
1091
|
+
awaitingCreated = false;
|
|
1092
|
+
} else if (event?.type === "error" && awaitingCreated) {
|
|
1093
|
+
responseActive = false;
|
|
1094
|
+
awaitingCreated = false;
|
|
1095
|
+
releaseHeld();
|
|
1096
|
+
}
|
|
1097
|
+
if (event?.type === "conversation.created" && typeof event.conversation?.id === "string") {
|
|
1098
|
+
sidebandState.conversationId = event.conversation.id;
|
|
1099
|
+
}
|
|
1100
|
+
if (event?.type === "session.updated") markSessionReady();
|
|
1101
|
+
if (event?.type === "input_audio_buffer.speech_started") interruptAssistant(true);
|
|
1102
|
+
if (event?.type === "input_audio_buffer.speech_stopped") {
|
|
1103
|
+
state.userSpeaking = false;
|
|
1104
|
+
releaseHeld();
|
|
1105
|
+
}
|
|
1106
|
+
const spoken = voiceAudioDelta(event);
|
|
1107
|
+
if (spoken) {
|
|
1108
|
+
if (!state.dropping) {
|
|
1109
|
+
if (typeof event.item_id === "string" && event.item_id !== state.audioItemId) {
|
|
1110
|
+
state.audioItemId = event.item_id;
|
|
1111
|
+
state.audioContentIndex = Number.isInteger(event.content_index) ? event.content_index : 0;
|
|
1112
|
+
state.playedMs = 0;
|
|
1113
|
+
state.truncated = false;
|
|
1114
|
+
}
|
|
1115
|
+
try {
|
|
1116
|
+
playbackFromDelta(track, state, spoken);
|
|
1117
|
+
} catch {
|
|
1118
|
+
// The peer may not be connected yet. Keep the socket.
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
if (event?.type === "response.created") {
|
|
1124
|
+
responseActive = true;
|
|
1125
|
+
awaitingCreated = false;
|
|
1126
|
+
state.dropping = false;
|
|
1127
|
+
}
|
|
1128
|
+
if (event?.type === "response.output_audio.done" || event?.type === "response.done") {
|
|
1129
|
+
try {
|
|
1130
|
+
flushPlayback(track, state);
|
|
1131
|
+
} catch {
|
|
1132
|
+
// A short tail is dropped. The call stays up.
|
|
1133
|
+
}
|
|
1134
|
+
state.endTalk = true;
|
|
1135
|
+
if (!state.playout?.length) state.talking = false;
|
|
1136
|
+
if (event.type === "response.done") {
|
|
1137
|
+
responseActive = false;
|
|
1138
|
+
awaitingCreated = false;
|
|
1139
|
+
releaseHeld();
|
|
1140
|
+
if (pendingCreate && !responseActive) {
|
|
1141
|
+
pendingCreate = false;
|
|
1142
|
+
send({ type: "response.create" });
|
|
1143
|
+
responseActive = true;
|
|
1144
|
+
awaitingCreated = true;
|
|
1145
|
+
}
|
|
1146
|
+
state.dropping = true;
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
const sideband = voiceSidebandEvent(event, sidebandState);
|
|
1150
|
+
if (sideband) onEvent?.(JSON.stringify(sideband));
|
|
1151
|
+
};
|
|
1152
|
+
// The local track reports packets we send. The microphone is the remote track.
|
|
1153
|
+
for (const remote of receiveTracks) remote.onReceiveRtp?.subscribe?.(onRtp);
|
|
1154
|
+
const onSocketClose = () => {
|
|
1155
|
+
if (sidebandState.closing || state.stopped) return;
|
|
1156
|
+
if (!resumed && sidebandState.conversationId && openSocket) {
|
|
1157
|
+
resumed = true;
|
|
1158
|
+
try {
|
|
1159
|
+
const pendingHeld = held.splice(0);
|
|
1160
|
+
responseActive = false;
|
|
1161
|
+
pendingCreate = false;
|
|
1162
|
+
awaitingCreated = false;
|
|
1163
|
+
state.dropping = false;
|
|
1164
|
+
state.userSpeaking = false;
|
|
1165
|
+
const resumeTruncate =
|
|
1166
|
+
Boolean(state.playout?.length || state.pending?.length) && state.audioItemId && !state.truncated
|
|
1167
|
+
? {
|
|
1168
|
+
type: "conversation.item.truncate",
|
|
1169
|
+
item_id: state.audioItemId,
|
|
1170
|
+
content_index: state.audioContentIndex ?? 0,
|
|
1171
|
+
audio_end_ms: state.playedMs ?? 0,
|
|
1172
|
+
}
|
|
1173
|
+
: null;
|
|
1174
|
+
discardUnsentAudio(state);
|
|
1175
|
+
state.truncated = false;
|
|
1176
|
+
state.playedMs = 0;
|
|
1177
|
+
state.audioItemId = undefined;
|
|
1178
|
+
queued.length = 0;
|
|
1179
|
+
if (resumeTruncate) pendingHeld.unshift(resumeTruncate);
|
|
1180
|
+
current = openSocket(sidebandState.conversationId);
|
|
1181
|
+
opened = current.readyState === 1;
|
|
1182
|
+
sessionReady = false;
|
|
1183
|
+
armReadyTimer();
|
|
1184
|
+
bindSocket(current);
|
|
1185
|
+
send(sessionUpdate(instructions, ackFiller, developerContext));
|
|
1186
|
+
held.push(...pendingHeld);
|
|
1187
|
+
return;
|
|
1188
|
+
} catch {
|
|
1189
|
+
// The resume dial failed. Tell the desktop below.
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
onEvent?.(JSON.stringify({ type: "error", error: { message: "Voice connection closed." } }));
|
|
1193
|
+
};
|
|
1194
|
+
const bindSocket = (next) => {
|
|
1195
|
+
next.onopen = flush;
|
|
1196
|
+
next.onmessage = (event) => onUpstream(event?.data ?? event);
|
|
1197
|
+
next.onclose = onSocketClose;
|
|
1198
|
+
next.onerror = () => {};
|
|
1199
|
+
};
|
|
1200
|
+
bindSocket(socket);
|
|
1201
|
+
send(sessionUpdate(instructions, ackFiller, developerContext));
|
|
1202
|
+
for (const item of initialItems) send(item);
|
|
1203
|
+
if (opened) flush();
|
|
1204
|
+
return {
|
|
1205
|
+
onRtp,
|
|
1206
|
+
onUpstream,
|
|
1207
|
+
sendClient(event) {
|
|
1208
|
+
try {
|
|
1209
|
+
if (event?.type === "response.cancel") {
|
|
1210
|
+
pendingCreate = false;
|
|
1211
|
+
interruptAssistant(state.userSpeaking === true);
|
|
1212
|
+
send(event);
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
if ((responseActive || playbackBusy() || state.userSpeaking) && holdsForIdleResponse(event)) {
|
|
1216
|
+
held.push(event);
|
|
1217
|
+
return;
|
|
1218
|
+
}
|
|
1219
|
+
send(event);
|
|
1220
|
+
if (startsResponse(event)) {
|
|
1221
|
+
responseActive = true;
|
|
1222
|
+
awaitingCreated = true;
|
|
1223
|
+
if (event?.item?.type === "force_message") pendingCreate = false;
|
|
1224
|
+
}
|
|
1225
|
+
} catch {
|
|
1226
|
+
// The xAI socket may already be closed.
|
|
1227
|
+
}
|
|
1228
|
+
},
|
|
1229
|
+
close() {
|
|
1230
|
+
state.stopped = true;
|
|
1231
|
+
if (readyTimer) clearTimeout(readyTimer);
|
|
1232
|
+
if (state.playoutTimer) clearTimeout(state.playoutTimer);
|
|
1233
|
+
try {
|
|
1234
|
+
opus?.delete?.();
|
|
1235
|
+
} catch {
|
|
1236
|
+
// The decoder is already gone.
|
|
1237
|
+
}
|
|
1238
|
+
},
|
|
1239
|
+
};
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
export function openVoiceSocket(url, token) {
|
|
1243
|
+
return new WebSocket(url || VOICE_SOCKET_URL, {
|
|
1244
|
+
headers: { authorization: `Bearer ${token}` },
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
export async function answerVoiceCall({
|
|
1249
|
+
offer,
|
|
1250
|
+
token,
|
|
1251
|
+
webSocketFactory,
|
|
1252
|
+
instructions,
|
|
1253
|
+
ackFiller = false,
|
|
1254
|
+
developerContext = "",
|
|
1255
|
+
initialItems,
|
|
1256
|
+
}) {
|
|
1257
|
+
const { MediaStreamTrack, RTCPeerConnection } = voicePackages();
|
|
1258
|
+
const pc = new RTCPeerConnection(peerConfig());
|
|
1259
|
+
const track = new MediaStreamTrack({ kind: "audio" });
|
|
1260
|
+
let socket;
|
|
1261
|
+
let bridge;
|
|
1262
|
+
let closed = false;
|
|
1263
|
+
let appendTimer = null;
|
|
1264
|
+
let speechTimer = null;
|
|
1265
|
+
const close = () => {
|
|
1266
|
+
if (closed) return;
|
|
1267
|
+
closed = true;
|
|
1268
|
+
if (appendTimer) clearTimeout(appendTimer);
|
|
1269
|
+
if (speechTimer) clearTimeout(speechTimer);
|
|
1270
|
+
voiceState.closing = true;
|
|
1271
|
+
active.delete(session);
|
|
1272
|
+
if (session.id) sessionsById.delete(session.id);
|
|
1273
|
+
try {
|
|
1274
|
+
bridge?.close();
|
|
1275
|
+
socket?.close();
|
|
1276
|
+
} catch {
|
|
1277
|
+
// Already closed.
|
|
1278
|
+
}
|
|
1279
|
+
pc.close().catch(() => {});
|
|
1280
|
+
};
|
|
1281
|
+
const pendingEvents = [];
|
|
1282
|
+
const voiceState = { inputTranscript: "", pendingCalls: new Set(), ackFiller, developerContext };
|
|
1283
|
+
const appendBurst = [];
|
|
1284
|
+
const flushAppends = () => {
|
|
1285
|
+
appendTimer = null;
|
|
1286
|
+
const burst = appendBurst.splice(0);
|
|
1287
|
+
for (const event of mergeVoiceClientBurst(burst)) {
|
|
1288
|
+
for (const outbound of voiceClientEvents(event, voiceState)) bridge?.sendClient?.(outbound);
|
|
1289
|
+
}
|
|
1290
|
+
};
|
|
1291
|
+
let sidebandSocket = null;
|
|
1292
|
+
const session = {
|
|
1293
|
+
close,
|
|
1294
|
+
id: null,
|
|
1295
|
+
attachSideband(sock) {
|
|
1296
|
+
sidebandSocket = sock;
|
|
1297
|
+
attachClientFrames(sock, (text) => {
|
|
1298
|
+
let event;
|
|
1299
|
+
try {
|
|
1300
|
+
event = JSON.parse(text);
|
|
1301
|
+
} catch {
|
|
1302
|
+
return;
|
|
1303
|
+
}
|
|
1304
|
+
const type = event?.type;
|
|
1305
|
+
if (type === "session.close") {
|
|
1306
|
+
queueMicrotask(() => close());
|
|
1307
|
+
return;
|
|
1308
|
+
}
|
|
1309
|
+
if (queueDelegationSpeech(voiceState, event)) {
|
|
1310
|
+
if (speechTimer) clearTimeout(speechTimer);
|
|
1311
|
+
speechTimer = setTimeout(() => {
|
|
1312
|
+
speechTimer = null;
|
|
1313
|
+
for (const outbound of flushDelegationSpeech(voiceState)) bridge?.sendClient?.(outbound);
|
|
1314
|
+
}, DELEGATION_SPEECH_WAIT_MS);
|
|
1315
|
+
speechTimer.unref();
|
|
1316
|
+
return;
|
|
1317
|
+
}
|
|
1318
|
+
if (type === "session.context.append" || type === "delegation.context.append") {
|
|
1319
|
+
appendBurst.push(event);
|
|
1320
|
+
if (!appendTimer) {
|
|
1321
|
+
appendTimer = setTimeout(flushAppends, 30);
|
|
1322
|
+
appendTimer.unref();
|
|
1323
|
+
}
|
|
1324
|
+
return;
|
|
1325
|
+
}
|
|
1326
|
+
if (appendTimer) {
|
|
1327
|
+
clearTimeout(appendTimer);
|
|
1328
|
+
flushAppends();
|
|
1329
|
+
}
|
|
1330
|
+
for (const outbound of voiceClientEvents(event, voiceState)) bridge?.sendClient?.(outbound);
|
|
1331
|
+
});
|
|
1332
|
+
for (const text of announceSession(noteState, pendingEvents.splice(0))) {
|
|
1333
|
+
sock.write(encodeServerFrame(0x1, text));
|
|
1334
|
+
}
|
|
1335
|
+
},
|
|
1336
|
+
};
|
|
1337
|
+
const noteState = { sessionAnnounced: false, sessionId: null };
|
|
1338
|
+
const writeSideband = (text) => {
|
|
1339
|
+
for (const frame of sidebandFrames(noteState, text)) {
|
|
1340
|
+
if (sidebandSocket && !sidebandSocket.destroyed) sidebandSocket.write(encodeServerFrame(0x1, frame));
|
|
1341
|
+
else pendingEvents.push(frame);
|
|
1342
|
+
}
|
|
1343
|
+
};
|
|
1344
|
+
try {
|
|
1345
|
+
pc.addTrack(track);
|
|
1346
|
+
const seenRemote = new Set();
|
|
1347
|
+
const receiveTracks = [];
|
|
1348
|
+
const watchRemote = (remote) => {
|
|
1349
|
+
if (!remote?.remote || remote.kind !== "audio" || seenRemote.has(remote)) return;
|
|
1350
|
+
seenRemote.add(remote);
|
|
1351
|
+
if (bridge?.onRtp) remote.onReceiveRtp?.subscribe?.(bridge.onRtp);
|
|
1352
|
+
else receiveTracks.push(remote);
|
|
1353
|
+
};
|
|
1354
|
+
pc.onTrack?.subscribe?.(watchRemote);
|
|
1355
|
+
await pc.setRemoteDescription({ type: "offer", sdp: offer });
|
|
1356
|
+
for (const remote of remoteAudioTracks(pc)) watchRemote(remote);
|
|
1357
|
+
const answer = await pc.createAnswer();
|
|
1358
|
+
await pc.setLocalDescription({ type: "answer", sdp: answerSdpForOffer(offer, answer?.sdp) });
|
|
1359
|
+
let sdp = pc.localDescription?.sdp ?? "";
|
|
1360
|
+
if (!sdp.startsWith("v=0") || !sdp.includes("a=fingerprint:")) {
|
|
1361
|
+
throw new Error("incomplete answer");
|
|
1362
|
+
}
|
|
1363
|
+
sdp = await waitForHostCandidate(pc);
|
|
1364
|
+
const id = rememberLocalCall();
|
|
1365
|
+
session.id = id;
|
|
1366
|
+
noteState.sessionId = id;
|
|
1367
|
+
sessionsById.set(id, session);
|
|
1368
|
+
const dial = webSocketFactory ?? openVoiceSocket;
|
|
1369
|
+
socket = dial(VOICE_SOCKET_URL, token);
|
|
1370
|
+
socket.onerror = () => {};
|
|
1371
|
+
bridge = startVoiceBridge({
|
|
1372
|
+
track,
|
|
1373
|
+
socket,
|
|
1374
|
+
codec: audioCodecFromSdp(sdp),
|
|
1375
|
+
onEvent: writeSideband,
|
|
1376
|
+
instructions,
|
|
1377
|
+
ackFiller,
|
|
1378
|
+
developerContext,
|
|
1379
|
+
initialItems,
|
|
1380
|
+
voiceState,
|
|
1381
|
+
receiveTracks,
|
|
1382
|
+
openSocket(conversationId) {
|
|
1383
|
+
socket = dial(voiceSocketUrl(conversationId), token);
|
|
1384
|
+
return socket;
|
|
1385
|
+
},
|
|
1386
|
+
});
|
|
1387
|
+
active.add(session);
|
|
1388
|
+
return { sdp, location: `/v1/realtime/calls/${id}`, close };
|
|
1389
|
+
} catch (error) {
|
|
1390
|
+
close();
|
|
1391
|
+
throw error;
|
|
1392
|
+
}
|
|
1393
|
+
}
|
|
1394
|
+
|
|
1395
|
+
export function closeVoiceCalls() {
|
|
1396
|
+
for (const session of [...active]) session.close();
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
export function voiceCallCount() {
|
|
1400
|
+
return active.size;
|
|
1401
|
+
}
|