@charivo/realtime 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/{chunk-FABRM56W.mjs → chunk-3O3COMUY.mjs} +216 -17
- package/dist/chunk-3O3COMUY.mjs.map +1 -0
- package/dist/{chunk-WLTIMNHQ.mjs → chunk-J5L3BPMG.mjs} +248 -13
- package/dist/chunk-J5L3BPMG.mjs.map +1 -0
- package/dist/chunk-ZJZO4QYA.mjs +123 -0
- package/dist/chunk-ZJZO4QYA.mjs.map +1 -0
- package/dist/index.d.mts +15 -3
- package/dist/index.d.ts +15 -3
- package/dist/index.js +240 -21
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +237 -21
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.d.mts +29 -1
- package/dist/openai/index.d.ts +29 -1
- package/dist/openai/index.js +326 -13
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +2 -2
- package/dist/openai-agents/index.d.mts +24 -1
- package/dist/openai-agents/index.d.ts +24 -1
- package/dist/openai-agents/index.js +293 -17
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +2 -2
- package/dist/remote/index.d.mts +2 -1
- package/dist/remote/index.d.ts +2 -1
- package/dist/remote/index.js +572 -54
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +38 -31
- package/dist/remote/index.mjs.map +1 -1
- package/dist/{types-Bc2tcYr1.d.mts → types-bmYuc_Vs.d.mts} +6 -1
- package/dist/{types-Bc2tcYr1.d.ts → types-bmYuc_Vs.d.ts} +6 -1
- package/package.json +2 -2
- package/dist/chunk-FABRM56W.mjs.map +0 -1
- package/dist/chunk-WLTIMNHQ.mjs.map +0 -1
- package/dist/chunk-XJNT4ZTF.mjs +0 -41
- package/dist/chunk-XJNT4ZTF.mjs.map +0 -1
package/dist/remote/index.js
CHANGED
|
@@ -5,7 +5,83 @@ var agentsRealtime = require('@openai/agents-realtime');
|
|
|
5
5
|
|
|
6
6
|
// src/remote/index.ts
|
|
7
7
|
|
|
8
|
-
// src/internal/
|
|
8
|
+
// src/internal/microphone.ts
|
|
9
|
+
var DEFAULT_MICROPHONE_CONSTRAINTS = {
|
|
10
|
+
echoCancellation: true,
|
|
11
|
+
noiseSuppression: true,
|
|
12
|
+
autoGainControl: true
|
|
13
|
+
};
|
|
14
|
+
async function acquireMicrophoneStream() {
|
|
15
|
+
return navigator.mediaDevices.getUserMedia({
|
|
16
|
+
audio: DEFAULT_MICROPHONE_CONSTRAINTS
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
var DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS = 1e3;
|
|
20
|
+
function bindTransportLifecycle(callbacks) {
|
|
21
|
+
const teardownBrowserLifecycle = core.subscribeBrowserLifecycle({
|
|
22
|
+
onHidden: callbacks.onHidden,
|
|
23
|
+
onOnline: callbacks.onOnline,
|
|
24
|
+
onPageHide: callbacks.onPageHide,
|
|
25
|
+
onPageShow: callbacks.onPageShow,
|
|
26
|
+
onVisible: callbacks.onVisible
|
|
27
|
+
});
|
|
28
|
+
if (callbacks.onDeviceChange) {
|
|
29
|
+
navigator.mediaDevices?.addEventListener?.(
|
|
30
|
+
"devicechange",
|
|
31
|
+
callbacks.onDeviceChange
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
return () => {
|
|
35
|
+
teardownBrowserLifecycle();
|
|
36
|
+
if (callbacks.onDeviceChange) {
|
|
37
|
+
navigator.mediaDevices?.removeEventListener?.(
|
|
38
|
+
"devicechange",
|
|
39
|
+
callbacks.onDeviceChange
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
async function replaceMicrophoneTrack(state) {
|
|
45
|
+
if (!state.peerConnection) {
|
|
46
|
+
throw new Error("Realtime transport not active");
|
|
47
|
+
}
|
|
48
|
+
const nextStream = await acquireMicrophoneStream();
|
|
49
|
+
const nextTrack = nextStream.getAudioTracks()[0];
|
|
50
|
+
if (!nextTrack) {
|
|
51
|
+
throw new Error("Microphone access required for Realtime API");
|
|
52
|
+
}
|
|
53
|
+
const sender = state.audioSender ?? (typeof state.peerConnection.getSenders === "function" ? state.peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null);
|
|
54
|
+
if (!sender) {
|
|
55
|
+
throw new Error("No outbound audio sender available");
|
|
56
|
+
}
|
|
57
|
+
await sender.replaceTrack(nextTrack);
|
|
58
|
+
state.mediaStream?.getTracks().forEach((track) => track.stop());
|
|
59
|
+
return {
|
|
60
|
+
audioSender: sender,
|
|
61
|
+
mediaStream: nextStream
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function createIceDisconnectDebouncer(onDisconnect, delayMs = DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS) {
|
|
65
|
+
let pending = null;
|
|
66
|
+
return {
|
|
67
|
+
schedule() {
|
|
68
|
+
if (pending) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
pending = setTimeout(() => {
|
|
72
|
+
pending = null;
|
|
73
|
+
onDisconnect();
|
|
74
|
+
}, delayMs);
|
|
75
|
+
},
|
|
76
|
+
cancel() {
|
|
77
|
+
if (!pending) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
clearTimeout(pending);
|
|
81
|
+
pending = null;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
9
85
|
var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
10
86
|
function isRecord(value) {
|
|
11
87
|
return typeof value === "object" && value !== null;
|
|
@@ -32,9 +108,11 @@ async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT
|
|
|
32
108
|
});
|
|
33
109
|
} catch (error) {
|
|
34
110
|
if (isAbortError(error)) {
|
|
35
|
-
throw new
|
|
111
|
+
throw new core.CharivoTimeoutError(timeoutMessage, { cause: error });
|
|
36
112
|
}
|
|
37
|
-
throw
|
|
113
|
+
throw new core.CharivoTransportError("Realtime request failed", {
|
|
114
|
+
cause: error instanceof Error ? error : void 0
|
|
115
|
+
});
|
|
38
116
|
} finally {
|
|
39
117
|
clearTimeout(timeoutId);
|
|
40
118
|
}
|
|
@@ -42,15 +120,13 @@ async function fetchWithTimeout(input, init, timeoutMessage, timeoutMs = DEFAULT
|
|
|
42
120
|
function isAbortError(error) {
|
|
43
121
|
return error instanceof DOMException && error.name === "AbortError" || error instanceof Error && error.name === "AbortError";
|
|
44
122
|
}
|
|
45
|
-
|
|
46
|
-
// src/openai-agents/bootstrap.ts
|
|
47
123
|
async function getOpenAIRealtimeAgentsBootstrap(options, request) {
|
|
48
124
|
if (options.sessionBootstrap) {
|
|
49
125
|
return options.sessionBootstrap(request);
|
|
50
126
|
}
|
|
51
127
|
const apiEndpoint = options.apiEndpoint;
|
|
52
128
|
if (!apiEndpoint) {
|
|
53
|
-
throw new
|
|
129
|
+
throw new core.CharivoStateError(
|
|
54
130
|
"OpenAI agents realtime client requires apiEndpoint or sessionBootstrap"
|
|
55
131
|
);
|
|
56
132
|
}
|
|
@@ -67,11 +143,15 @@ async function getOpenAIRealtimeAgentsBootstrap(options, request) {
|
|
|
67
143
|
);
|
|
68
144
|
if (!response.ok) {
|
|
69
145
|
const errorText = await response.text();
|
|
70
|
-
throw new
|
|
146
|
+
throw new core.CharivoProviderError(
|
|
147
|
+
`Failed to create Realtime session: ${errorText}`
|
|
148
|
+
);
|
|
71
149
|
}
|
|
72
150
|
const bootstrap = await response.json();
|
|
73
151
|
if (!isRealtimeSessionBootstrap(bootstrap)) {
|
|
74
|
-
throw new
|
|
152
|
+
throw new core.CharivoProviderError(
|
|
153
|
+
"Invalid realtime session bootstrap response"
|
|
154
|
+
);
|
|
75
155
|
}
|
|
76
156
|
return bootstrap;
|
|
77
157
|
}
|
|
@@ -83,11 +163,22 @@ var LipSyncAnalyzer = class {
|
|
|
83
163
|
}
|
|
84
164
|
audioAnalysisStream = null;
|
|
85
165
|
audioContext = null;
|
|
166
|
+
audioSource = null;
|
|
86
167
|
analyser = null;
|
|
87
168
|
lipSyncInterval = null;
|
|
88
169
|
pendingAudioElementPoll = null;
|
|
89
170
|
observedAudioElement = null;
|
|
90
171
|
observedAudioElementListener = null;
|
|
172
|
+
async prepareAudioContext() {
|
|
173
|
+
if (this.audioContext) {
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
177
|
+
if (!audioContextConstructor) {
|
|
178
|
+
throw new Error("AudioContext is not supported in this browser");
|
|
179
|
+
}
|
|
180
|
+
this.audioContext = new audioContextConstructor();
|
|
181
|
+
}
|
|
91
182
|
observeAudioElement(audioElement) {
|
|
92
183
|
this.detachObservedAudioElement();
|
|
93
184
|
const tryAttachStream = () => {
|
|
@@ -109,16 +200,20 @@ var LipSyncAnalyzer = class {
|
|
|
109
200
|
this.stopLipSyncAnalysis();
|
|
110
201
|
this.audioAnalysisStream = stream;
|
|
111
202
|
try {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
203
|
+
if (!this.audioContext) {
|
|
204
|
+
void this.prepareAudioContext().catch((error) => {
|
|
205
|
+
this.options.onError?.(error);
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
if (!this.audioContext) {
|
|
209
|
+
return;
|
|
115
210
|
}
|
|
116
|
-
this.
|
|
117
|
-
|
|
211
|
+
this.audioSource?.disconnect?.();
|
|
212
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
118
213
|
this.analyser = this.audioContext.createAnalyser();
|
|
119
214
|
this.analyser.fftSize = 256;
|
|
120
215
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
121
|
-
|
|
216
|
+
this.audioSource.connect(this.analyser);
|
|
122
217
|
this.startLipSyncAnalysis();
|
|
123
218
|
} catch (error) {
|
|
124
219
|
this.options.onError?.(error);
|
|
@@ -127,10 +222,21 @@ var LipSyncAnalyzer = class {
|
|
|
127
222
|
stopOutput() {
|
|
128
223
|
this.stopLipSyncAnalysis();
|
|
129
224
|
}
|
|
225
|
+
pause() {
|
|
226
|
+
this.stopLipSyncAnalysis();
|
|
227
|
+
}
|
|
228
|
+
resume() {
|
|
229
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
this.startLipSyncAnalysis();
|
|
233
|
+
}
|
|
130
234
|
cleanup() {
|
|
131
235
|
this.stopLipSyncAnalysis();
|
|
132
236
|
this.stopAudioElementPolling();
|
|
133
237
|
this.detachObservedAudioElement();
|
|
238
|
+
this.audioSource?.disconnect?.();
|
|
239
|
+
this.audioSource = null;
|
|
134
240
|
if (this.audioContext) {
|
|
135
241
|
void this.audioContext.close();
|
|
136
242
|
this.audioContext = null;
|
|
@@ -265,9 +371,21 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
265
371
|
session = null;
|
|
266
372
|
transport = null;
|
|
267
373
|
audioElement = null;
|
|
374
|
+
mediaStream = null;
|
|
375
|
+
peerConnection = null;
|
|
376
|
+
audioSender = null;
|
|
268
377
|
connectionWasActive = false;
|
|
378
|
+
connectionLossNotified = false;
|
|
379
|
+
isExplicitDisconnect = false;
|
|
380
|
+
isRecovering = false;
|
|
381
|
+
isCleaningUp = false;
|
|
269
382
|
assistant = { text: "", started: false };
|
|
270
383
|
latestHistory = [];
|
|
384
|
+
currentSessionConfig;
|
|
385
|
+
teardownTransportLifecycle;
|
|
386
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
387
|
+
this.emitConnectionLost("ice-disconnected");
|
|
388
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
271
389
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
272
390
|
pendingToolCalls = /* @__PURE__ */ new Map();
|
|
273
391
|
lipSyncAnalyzer = new LipSyncAnalyzer({
|
|
@@ -280,13 +398,23 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
280
398
|
});
|
|
281
399
|
async connect(config) {
|
|
282
400
|
try {
|
|
401
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
402
|
+
throw new Error("Realtime session disconnect requested");
|
|
403
|
+
}
|
|
283
404
|
this.log("Starting OpenAI Agents Realtime WebRTC connection");
|
|
405
|
+
this.isExplicitDisconnect = false;
|
|
406
|
+
this.isRecovering = false;
|
|
407
|
+
this.connectionLossNotified = false;
|
|
408
|
+
this.currentSessionConfig = config;
|
|
284
409
|
this.audioElement = document.createElement("audio");
|
|
285
410
|
this.audioElement.autoplay = true;
|
|
286
411
|
this.audioElement.setAttribute("playsinline", "true");
|
|
412
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
413
|
+
this.bindTransportLifecycleEvents();
|
|
414
|
+
await this.lipSyncAnalyzer.prepareAudioContext();
|
|
287
415
|
const agent = this.createAgent(config);
|
|
288
416
|
this.transport = new agentsRealtime.OpenAIRealtimeWebRTC(
|
|
289
|
-
this.createTransportOptions(this.audioElement)
|
|
417
|
+
this.createTransportOptions(this.audioElement, this.mediaStream)
|
|
290
418
|
);
|
|
291
419
|
this.session = new agentsRealtime.RealtimeSession(agent, {
|
|
292
420
|
transport: this.transport,
|
|
@@ -318,6 +446,8 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
318
446
|
}
|
|
319
447
|
async disconnect() {
|
|
320
448
|
this.log("Disconnecting OpenAI Agents Realtime WebRTC");
|
|
449
|
+
this.isExplicitDisconnect = true;
|
|
450
|
+
this.isRecovering = false;
|
|
321
451
|
this.cleanup();
|
|
322
452
|
}
|
|
323
453
|
async updateSession(config) {
|
|
@@ -326,8 +456,27 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
326
456
|
}
|
|
327
457
|
this.session.options.config = toOpenAIRealtimeAgentsSessionConfig(config);
|
|
328
458
|
await this.session.updateAgent(this.createAgent(config));
|
|
459
|
+
this.currentSessionConfig = config;
|
|
460
|
+
}
|
|
461
|
+
async recover(config) {
|
|
462
|
+
this.isRecovering = true;
|
|
463
|
+
this.connectionLossNotified = true;
|
|
464
|
+
this.currentSessionConfig = config;
|
|
465
|
+
try {
|
|
466
|
+
this.cleanupPendingToolCalls(
|
|
467
|
+
new Error("Realtime session interrupted during reconnect")
|
|
468
|
+
);
|
|
469
|
+
this.cleanup();
|
|
470
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
471
|
+
this.connectionLossNotified = false;
|
|
472
|
+
} finally {
|
|
473
|
+
this.isRecovering = false;
|
|
474
|
+
}
|
|
329
475
|
}
|
|
330
476
|
async sendText(text) {
|
|
477
|
+
if (this.isRecovering) {
|
|
478
|
+
throw new Error("Realtime transport reconnecting");
|
|
479
|
+
}
|
|
331
480
|
if (!this.session) {
|
|
332
481
|
throw new Error("Realtime session not active");
|
|
333
482
|
}
|
|
@@ -335,11 +484,17 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
335
484
|
this.session.sendMessage(text);
|
|
336
485
|
}
|
|
337
486
|
async sendAudio(_audio) {
|
|
487
|
+
if (this.isRecovering) {
|
|
488
|
+
throw new Error("Realtime transport reconnecting");
|
|
489
|
+
}
|
|
338
490
|
console.warn(
|
|
339
491
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
340
492
|
);
|
|
341
493
|
}
|
|
342
494
|
async sendToolResult(callId, output) {
|
|
495
|
+
if (this.isRecovering) {
|
|
496
|
+
throw new Error("Realtime transport reconnecting");
|
|
497
|
+
}
|
|
343
498
|
const pendingCall = this.pendingToolCalls.get(callId);
|
|
344
499
|
if (!pendingCall) {
|
|
345
500
|
throw new Error(`No pending realtime tool call for "${callId}"`);
|
|
@@ -348,6 +503,9 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
348
503
|
pendingCall.resolve(output);
|
|
349
504
|
}
|
|
350
505
|
async interrupt() {
|
|
506
|
+
if (this.isRecovering) {
|
|
507
|
+
throw new Error("Realtime transport reconnecting");
|
|
508
|
+
}
|
|
351
509
|
if (!this.session) {
|
|
352
510
|
throw new Error("Realtime session not active");
|
|
353
511
|
}
|
|
@@ -411,7 +569,7 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
411
569
|
if (status === "disconnected" && this.connectionWasActive) {
|
|
412
570
|
this.cleanupPendingToolCalls();
|
|
413
571
|
this.connectionWasActive = false;
|
|
414
|
-
this.
|
|
572
|
+
this.emitConnectionLost("connection-failed");
|
|
415
573
|
}
|
|
416
574
|
});
|
|
417
575
|
}
|
|
@@ -513,10 +671,14 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
513
671
|
});
|
|
514
672
|
});
|
|
515
673
|
}
|
|
516
|
-
createTransportOptions(audioElement) {
|
|
674
|
+
createTransportOptions(audioElement, mediaStream) {
|
|
517
675
|
return {
|
|
518
676
|
audioElement,
|
|
677
|
+
mediaStream,
|
|
519
678
|
changePeerConnection: async (peerConnection) => {
|
|
679
|
+
this.peerConnection = peerConnection;
|
|
680
|
+
this.audioSender = typeof peerConnection.getSenders === "function" ? peerConnection.getSenders().find((candidate) => candidate.track?.kind === "audio") ?? null : null;
|
|
681
|
+
this.bindPeerConnectionEvents(peerConnection);
|
|
520
682
|
peerConnection.addEventListener("track", (event) => {
|
|
521
683
|
const stream = event.streams[0];
|
|
522
684
|
if (stream) {
|
|
@@ -527,7 +689,113 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
527
689
|
}
|
|
528
690
|
};
|
|
529
691
|
}
|
|
692
|
+
bindPeerConnectionEvents(peerConnection) {
|
|
693
|
+
peerConnection.addEventListener("iceconnectionstatechange", () => {
|
|
694
|
+
const iceState = peerConnection.iceConnectionState;
|
|
695
|
+
if (iceState === "failed") {
|
|
696
|
+
this.iceDisconnectDebouncer.cancel();
|
|
697
|
+
this.emitConnectionLost("ice-failed");
|
|
698
|
+
} else if (iceState === "disconnected") {
|
|
699
|
+
this.iceDisconnectDebouncer.schedule();
|
|
700
|
+
} else {
|
|
701
|
+
this.iceDisconnectDebouncer.cancel();
|
|
702
|
+
}
|
|
703
|
+
});
|
|
704
|
+
peerConnection.addEventListener("connectionstatechange", () => {
|
|
705
|
+
if (peerConnection.connectionState === "failed" || peerConnection.connectionState === "closed") {
|
|
706
|
+
this.iceDisconnectDebouncer.cancel();
|
|
707
|
+
this.emitConnectionLost("connection-failed");
|
|
708
|
+
}
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
bindTransportLifecycleEvents() {
|
|
712
|
+
if (this.teardownTransportLifecycle) {
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
716
|
+
onHidden: this.handleHidden,
|
|
717
|
+
onOnline: this.handleOnline,
|
|
718
|
+
onPageHide: this.handlePageHide,
|
|
719
|
+
onPageShow: this.handlePageShow,
|
|
720
|
+
onVisible: this.handleVisible,
|
|
721
|
+
onDeviceChange: this.handleDeviceChange
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
unbindTransportLifecycleEvents() {
|
|
725
|
+
this.teardownTransportLifecycle?.();
|
|
726
|
+
this.teardownTransportLifecycle = void 0;
|
|
727
|
+
}
|
|
728
|
+
handleOnline = () => {
|
|
729
|
+
this.emitConnectionLost("online");
|
|
730
|
+
};
|
|
731
|
+
handleHidden = () => {
|
|
732
|
+
this.lipSyncAnalyzer.pause();
|
|
733
|
+
};
|
|
734
|
+
handleVisible = () => {
|
|
735
|
+
this.lipSyncAnalyzer.resume();
|
|
736
|
+
this.emitConnectionLost("visibility");
|
|
737
|
+
};
|
|
738
|
+
handlePageHide = () => {
|
|
739
|
+
this.lipSyncAnalyzer.pause();
|
|
740
|
+
};
|
|
741
|
+
handlePageShow = (event) => {
|
|
742
|
+
this.lipSyncAnalyzer.resume();
|
|
743
|
+
if (event.persisted) {
|
|
744
|
+
this.emitConnectionLost("pageshow");
|
|
745
|
+
}
|
|
746
|
+
};
|
|
747
|
+
handleDeviceChange = () => {
|
|
748
|
+
void this.refreshMicrophoneTrack();
|
|
749
|
+
};
|
|
750
|
+
async refreshMicrophoneTrack() {
|
|
751
|
+
if (!this.peerConnection || this.isExplicitDisconnect) {
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
try {
|
|
755
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
756
|
+
audioSender: this.audioSender,
|
|
757
|
+
mediaStream: this.mediaStream,
|
|
758
|
+
peerConnection: this.peerConnection
|
|
759
|
+
});
|
|
760
|
+
this.audioSender = audioSender;
|
|
761
|
+
this.mediaStream = mediaStream;
|
|
762
|
+
} catch (error) {
|
|
763
|
+
this.emitEvent({
|
|
764
|
+
type: "error",
|
|
765
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
766
|
+
});
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
emitConnectionLost(cause, error) {
|
|
770
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.shouldRecover(cause)) {
|
|
771
|
+
return;
|
|
772
|
+
}
|
|
773
|
+
this.connectionLossNotified = true;
|
|
774
|
+
this.cleanupPendingToolCalls(
|
|
775
|
+
new Error("Realtime session interrupted during reconnect")
|
|
776
|
+
);
|
|
777
|
+
this.resetAssistantTracking();
|
|
778
|
+
this.lipSyncAnalyzer.stopOutput();
|
|
779
|
+
this.emitEvent({
|
|
780
|
+
type: "connection.lost",
|
|
781
|
+
cause,
|
|
782
|
+
error
|
|
783
|
+
});
|
|
784
|
+
}
|
|
785
|
+
shouldRecover(cause) {
|
|
786
|
+
if (!this.peerConnection) {
|
|
787
|
+
return false;
|
|
788
|
+
}
|
|
789
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
790
|
+
return this.peerConnection.connectionState === "failed" || this.peerConnection.connectionState === "closed" || this.peerConnection.iceConnectionState === "failed" || this.peerConnection.iceConnectionState === "disconnected";
|
|
791
|
+
}
|
|
792
|
+
return true;
|
|
793
|
+
}
|
|
530
794
|
cleanup(error) {
|
|
795
|
+
this.connectionWasActive = false;
|
|
796
|
+
this.isCleaningUp = true;
|
|
797
|
+
this.iceDisconnectDebouncer.cancel();
|
|
798
|
+
this.unbindTransportLifecycleEvents();
|
|
531
799
|
this.cleanupPendingToolCalls(error);
|
|
532
800
|
this.lipSyncAnalyzer.cleanup();
|
|
533
801
|
if (this.transport) {
|
|
@@ -542,7 +810,15 @@ var OpenAIRealtimeAgentsClient = class {
|
|
|
542
810
|
this.audioElement.srcObject = null;
|
|
543
811
|
this.audioElement = null;
|
|
544
812
|
}
|
|
813
|
+
if (this.mediaStream) {
|
|
814
|
+
this.mediaStream.getTracks?.().forEach((track) => track.stop());
|
|
815
|
+
this.mediaStream = null;
|
|
816
|
+
}
|
|
817
|
+
this.peerConnection = null;
|
|
818
|
+
this.audioSender = null;
|
|
545
819
|
this.latestHistory = [];
|
|
820
|
+
this.connectionLossNotified = false;
|
|
821
|
+
this.isCleaningUp = false;
|
|
546
822
|
this.resetAssistantTracking();
|
|
547
823
|
}
|
|
548
824
|
resetAssistantTracking() {
|
|
@@ -597,7 +873,9 @@ var OpenAIRealtimeClient = class {
|
|
|
597
873
|
dc = null;
|
|
598
874
|
audioElement = null;
|
|
599
875
|
mediaStream = null;
|
|
876
|
+
audioSender = null;
|
|
600
877
|
audioContext = null;
|
|
878
|
+
audioSource = null;
|
|
601
879
|
analyser = null;
|
|
602
880
|
lipSyncInterval = null;
|
|
603
881
|
isResponseInProgress = false;
|
|
@@ -607,13 +885,33 @@ var OpenAIRealtimeClient = class {
|
|
|
607
885
|
assistantText = "";
|
|
608
886
|
nextSessionUpdateId = 1;
|
|
609
887
|
pendingSessionUpdate = null;
|
|
888
|
+
currentSessionConfig;
|
|
889
|
+
connectionLossNotified = false;
|
|
890
|
+
isExplicitDisconnect = false;
|
|
891
|
+
isRecovering = false;
|
|
892
|
+
isCleaningUp = false;
|
|
893
|
+
teardownTransportLifecycle;
|
|
894
|
+
iceDisconnectDebouncer = createIceDisconnectDebouncer(() => {
|
|
895
|
+
this.emitConnectionLost("ice-disconnected");
|
|
896
|
+
}, DEFAULT_ICE_DISCONNECTED_DEBOUNCE_MS);
|
|
610
897
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
611
898
|
async connect(config) {
|
|
612
899
|
try {
|
|
900
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
901
|
+
throw new Error("Realtime session disconnect requested");
|
|
902
|
+
}
|
|
613
903
|
this.log("Starting OpenAI Realtime WebRTC connection");
|
|
904
|
+
this.isExplicitDisconnect = false;
|
|
905
|
+
this.isRecovering = false;
|
|
906
|
+
this.connectionLossNotified = false;
|
|
907
|
+
this.currentSessionConfig = config;
|
|
614
908
|
this.pc = new RTCPeerConnection();
|
|
615
909
|
this.audioElement = document.createElement("audio");
|
|
616
910
|
this.audioElement.autoplay = true;
|
|
911
|
+
this.audioElement.setAttribute("playsinline", "true");
|
|
912
|
+
this.prepareAudioAnalysis();
|
|
913
|
+
this.bindConnectionEvents();
|
|
914
|
+
this.bindTransportLifecycleEvents();
|
|
617
915
|
this.pc.ontrack = (event) => {
|
|
618
916
|
if (this.audioElement) {
|
|
619
917
|
this.audioElement.srcObject = event.streams[0];
|
|
@@ -621,11 +919,9 @@ var OpenAIRealtimeClient = class {
|
|
|
621
919
|
this.setupAudioAnalysis(event.streams[0]);
|
|
622
920
|
};
|
|
623
921
|
try {
|
|
624
|
-
this.mediaStream = await
|
|
625
|
-
audio: true
|
|
626
|
-
});
|
|
922
|
+
this.mediaStream = await acquireMicrophoneStream();
|
|
627
923
|
const audioTrack = this.mediaStream.getTracks()[0];
|
|
628
|
-
this.pc.addTrack(audioTrack, this.mediaStream);
|
|
924
|
+
this.audioSender = this.pc.addTrack(audioTrack, this.mediaStream) ?? this.audioSender;
|
|
629
925
|
} catch {
|
|
630
926
|
throw new Error("Microphone access required for Realtime API");
|
|
631
927
|
}
|
|
@@ -664,6 +960,8 @@ var OpenAIRealtimeClient = class {
|
|
|
664
960
|
}
|
|
665
961
|
async disconnect() {
|
|
666
962
|
this.log("Disconnecting OpenAI Realtime WebRTC");
|
|
963
|
+
this.isExplicitDisconnect = true;
|
|
964
|
+
this.isRecovering = false;
|
|
667
965
|
this.cleanup();
|
|
668
966
|
}
|
|
669
967
|
async updateSession(config) {
|
|
@@ -699,8 +997,27 @@ var OpenAIRealtimeClient = class {
|
|
|
699
997
|
timeoutId
|
|
700
998
|
};
|
|
701
999
|
});
|
|
1000
|
+
this.currentSessionConfig = config;
|
|
1001
|
+
}
|
|
1002
|
+
async recover(config) {
|
|
1003
|
+
this.isRecovering = true;
|
|
1004
|
+
this.connectionLossNotified = true;
|
|
1005
|
+
this.currentSessionConfig = config;
|
|
1006
|
+
try {
|
|
1007
|
+
if (await this.tryIceRestartRecovery(config)) {
|
|
1008
|
+
this.connectionLossNotified = false;
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
await this.rebuildConnection(config);
|
|
1012
|
+
this.connectionLossNotified = false;
|
|
1013
|
+
} finally {
|
|
1014
|
+
this.isRecovering = false;
|
|
1015
|
+
}
|
|
702
1016
|
}
|
|
703
1017
|
async sendText(text) {
|
|
1018
|
+
if (this.isRecovering) {
|
|
1019
|
+
throw new Error("Realtime transport reconnecting");
|
|
1020
|
+
}
|
|
704
1021
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
705
1022
|
throw new Error("DataChannel not ready");
|
|
706
1023
|
}
|
|
@@ -727,11 +1044,17 @@ var OpenAIRealtimeClient = class {
|
|
|
727
1044
|
this.beginResponseRequest();
|
|
728
1045
|
}
|
|
729
1046
|
async sendAudio(_audio) {
|
|
1047
|
+
if (this.isRecovering) {
|
|
1048
|
+
throw new Error("Realtime transport reconnecting");
|
|
1049
|
+
}
|
|
730
1050
|
console.warn(
|
|
731
1051
|
"sendAudio is not needed with WebRTC - audio is automatically transmitted"
|
|
732
1052
|
);
|
|
733
1053
|
}
|
|
734
1054
|
async sendToolResult(callId, output) {
|
|
1055
|
+
if (this.isRecovering) {
|
|
1056
|
+
throw new Error("Realtime transport reconnecting");
|
|
1057
|
+
}
|
|
735
1058
|
const dc = this.requireOpenDataChannel();
|
|
736
1059
|
dc.send(
|
|
737
1060
|
JSON.stringify({
|
|
@@ -747,6 +1070,9 @@ var OpenAIRealtimeClient = class {
|
|
|
747
1070
|
this.beginResponseRequest();
|
|
748
1071
|
}
|
|
749
1072
|
async interrupt() {
|
|
1073
|
+
if (this.isRecovering) {
|
|
1074
|
+
throw new Error("Realtime transport reconnecting");
|
|
1075
|
+
}
|
|
750
1076
|
const dc = this.requireOpenDataChannel();
|
|
751
1077
|
if (!this.isResponseInProgress) {
|
|
752
1078
|
return;
|
|
@@ -943,18 +1269,133 @@ var OpenAIRealtimeClient = class {
|
|
|
943
1269
|
}
|
|
944
1270
|
return bootstrap;
|
|
945
1271
|
}
|
|
1272
|
+
bindConnectionEvents() {
|
|
1273
|
+
if (!this.pc) {
|
|
1274
|
+
return;
|
|
1275
|
+
}
|
|
1276
|
+
this.pc.addEventListener("iceconnectionstatechange", () => {
|
|
1277
|
+
const iceState = this.pc?.iceConnectionState;
|
|
1278
|
+
if (iceState === "failed") {
|
|
1279
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1280
|
+
this.emitConnectionLost("ice-failed");
|
|
1281
|
+
} else if (iceState === "disconnected") {
|
|
1282
|
+
this.iceDisconnectDebouncer.schedule();
|
|
1283
|
+
} else {
|
|
1284
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1285
|
+
}
|
|
1286
|
+
});
|
|
1287
|
+
this.pc.addEventListener("connectionstatechange", () => {
|
|
1288
|
+
const connectionState = this.pc?.connectionState;
|
|
1289
|
+
if (connectionState === "failed" || connectionState === "closed") {
|
|
1290
|
+
this.iceDisconnectDebouncer.cancel();
|
|
1291
|
+
this.emitConnectionLost("connection-failed");
|
|
1292
|
+
}
|
|
1293
|
+
});
|
|
1294
|
+
}
|
|
1295
|
+
bindTransportLifecycleEvents() {
|
|
1296
|
+
if (this.teardownTransportLifecycle) {
|
|
1297
|
+
return;
|
|
1298
|
+
}
|
|
1299
|
+
this.teardownTransportLifecycle = bindTransportLifecycle({
|
|
1300
|
+
onHidden: this.handleHidden,
|
|
1301
|
+
onOnline: this.handleOnline,
|
|
1302
|
+
onPageHide: this.handlePageHide,
|
|
1303
|
+
onPageShow: this.handlePageShow,
|
|
1304
|
+
onVisible: this.handleVisible,
|
|
1305
|
+
onDeviceChange: this.handleDeviceChange
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
unbindTransportLifecycleEvents() {
|
|
1309
|
+
this.teardownTransportLifecycle?.();
|
|
1310
|
+
this.teardownTransportLifecycle = void 0;
|
|
1311
|
+
}
|
|
1312
|
+
handleOnline = () => {
|
|
1313
|
+
this.emitConnectionLost("online");
|
|
1314
|
+
};
|
|
1315
|
+
handleHidden = () => {
|
|
1316
|
+
this.pauseAudioAnalysis();
|
|
1317
|
+
};
|
|
1318
|
+
handleVisible = () => {
|
|
1319
|
+
this.resumeAudioAnalysis();
|
|
1320
|
+
this.emitConnectionLost("visibility");
|
|
1321
|
+
};
|
|
1322
|
+
handlePageHide = () => {
|
|
1323
|
+
this.pauseAudioAnalysis();
|
|
1324
|
+
};
|
|
1325
|
+
handlePageShow = (event) => {
|
|
1326
|
+
this.resumeAudioAnalysis();
|
|
1327
|
+
if (event.persisted) {
|
|
1328
|
+
this.emitConnectionLost("pageshow");
|
|
1329
|
+
}
|
|
1330
|
+
};
|
|
1331
|
+
handleDeviceChange = () => {
|
|
1332
|
+
void this.refreshMicrophoneTrack();
|
|
1333
|
+
};
|
|
1334
|
+
async refreshMicrophoneTrack() {
|
|
1335
|
+
if (!this.pc || this.isExplicitDisconnect) {
|
|
1336
|
+
return;
|
|
1337
|
+
}
|
|
1338
|
+
try {
|
|
1339
|
+
const { audioSender, mediaStream } = await replaceMicrophoneTrack({
|
|
1340
|
+
audioSender: this.audioSender,
|
|
1341
|
+
mediaStream: this.mediaStream,
|
|
1342
|
+
peerConnection: this.pc
|
|
1343
|
+
});
|
|
1344
|
+
this.audioSender = audioSender;
|
|
1345
|
+
this.mediaStream = mediaStream;
|
|
1346
|
+
} catch (error) {
|
|
1347
|
+
this.emitEvent({
|
|
1348
|
+
type: "error",
|
|
1349
|
+
error: error instanceof Error ? error : new Error(String(error))
|
|
1350
|
+
});
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
emitConnectionLost(cause, error) {
|
|
1354
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp || !this.pc || !this.dc || !this.needsRecovery(cause)) {
|
|
1355
|
+
return;
|
|
1356
|
+
}
|
|
1357
|
+
this.connectionLossNotified = true;
|
|
1358
|
+
this.isResponseInProgress = false;
|
|
1359
|
+
this.cancelInFlight = false;
|
|
1360
|
+
this.resetResponseTracking();
|
|
1361
|
+
this.stopLipSyncAnalysis();
|
|
1362
|
+
this.emitEvent({
|
|
1363
|
+
type: "connection.lost",
|
|
1364
|
+
cause,
|
|
1365
|
+
error
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1368
|
+
needsRecovery(cause) {
|
|
1369
|
+
if (!this.pc || !this.dc) {
|
|
1370
|
+
return false;
|
|
1371
|
+
}
|
|
1372
|
+
if (cause === "visibility" || cause === "pageshow" || cause === "offline" || cause === "online") {
|
|
1373
|
+
return this.dc.readyState !== "open" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed" || this.pc.iceConnectionState === "disconnected";
|
|
1374
|
+
}
|
|
1375
|
+
return true;
|
|
1376
|
+
}
|
|
1377
|
+
prepareAudioAnalysis() {
|
|
1378
|
+
if (this.audioContext) {
|
|
1379
|
+
return;
|
|
1380
|
+
}
|
|
1381
|
+
const audioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
|
1382
|
+
if (!audioContextConstructor) {
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
this.audioContext = new audioContextConstructor();
|
|
1386
|
+
}
|
|
946
1387
|
setupAudioAnalysis(stream) {
|
|
947
1388
|
try {
|
|
948
|
-
|
|
949
|
-
if (!
|
|
950
|
-
|
|
1389
|
+
this.prepareAudioAnalysis();
|
|
1390
|
+
if (!this.audioContext) {
|
|
1391
|
+
return;
|
|
951
1392
|
}
|
|
952
|
-
this.
|
|
953
|
-
|
|
1393
|
+
this.audioSource?.disconnect();
|
|
1394
|
+
this.audioSource = this.audioContext.createMediaStreamSource(stream);
|
|
954
1395
|
this.analyser = this.audioContext.createAnalyser();
|
|
955
1396
|
this.analyser.fftSize = 256;
|
|
956
1397
|
this.analyser.smoothingTimeConstant = 0.8;
|
|
957
|
-
|
|
1398
|
+
this.audioSource.connect(this.analyser);
|
|
958
1399
|
this.startLipSyncAnalysis();
|
|
959
1400
|
} catch (error) {
|
|
960
1401
|
console.error("Failed to setup audio analysis:", error);
|
|
@@ -990,7 +1431,19 @@ var OpenAIRealtimeClient = class {
|
|
|
990
1431
|
}
|
|
991
1432
|
this.emitEvent({ type: "audio.lipsync", rms: 0 });
|
|
992
1433
|
}
|
|
1434
|
+
pauseAudioAnalysis() {
|
|
1435
|
+
this.stopLipSyncAnalysis();
|
|
1436
|
+
}
|
|
1437
|
+
resumeAudioAnalysis() {
|
|
1438
|
+
if (!this.analyser || this.lipSyncInterval) {
|
|
1439
|
+
return;
|
|
1440
|
+
}
|
|
1441
|
+
this.startLipSyncAnalysis();
|
|
1442
|
+
}
|
|
993
1443
|
cleanup() {
|
|
1444
|
+
this.isCleaningUp = true;
|
|
1445
|
+
this.unbindTransportLifecycleEvents();
|
|
1446
|
+
this.iceDisconnectDebouncer.cancel();
|
|
994
1447
|
this.stopLipSyncAnalysis();
|
|
995
1448
|
this.rejectPendingSessionUpdate(
|
|
996
1449
|
new Error("Realtime session ended before session update completed")
|
|
@@ -1015,9 +1468,13 @@ var OpenAIRealtimeClient = class {
|
|
|
1015
1468
|
this.audioContext.close();
|
|
1016
1469
|
this.audioContext = null;
|
|
1017
1470
|
}
|
|
1471
|
+
this.audioSource = null;
|
|
1472
|
+
this.audioSender = null;
|
|
1018
1473
|
this.analyser = null;
|
|
1019
1474
|
this.isResponseInProgress = false;
|
|
1020
1475
|
this.cancelInFlight = false;
|
|
1476
|
+
this.connectionLossNotified = false;
|
|
1477
|
+
this.isCleaningUp = false;
|
|
1021
1478
|
this.resetResponseTracking();
|
|
1022
1479
|
}
|
|
1023
1480
|
beginResponseRequest() {
|
|
@@ -1026,11 +1483,60 @@ var OpenAIRealtimeClient = class {
|
|
|
1026
1483
|
this.cancelInFlight = false;
|
|
1027
1484
|
}
|
|
1028
1485
|
requireOpenDataChannel() {
|
|
1486
|
+
if (this.isRecovering) {
|
|
1487
|
+
throw new Error("Realtime transport reconnecting");
|
|
1488
|
+
}
|
|
1029
1489
|
if (!this.dc || this.dc.readyState !== "open") {
|
|
1030
1490
|
throw new Error("DataChannel not ready");
|
|
1031
1491
|
}
|
|
1032
1492
|
return this.dc;
|
|
1033
1493
|
}
|
|
1494
|
+
async tryIceRestartRecovery(config) {
|
|
1495
|
+
const pc = this.pc;
|
|
1496
|
+
if (!pc || this.shouldRebuildConnection()) {
|
|
1497
|
+
return false;
|
|
1498
|
+
}
|
|
1499
|
+
if (pc.iceConnectionState === "connected") {
|
|
1500
|
+
return true;
|
|
1501
|
+
}
|
|
1502
|
+
if (pc.iceConnectionState === "disconnected") {
|
|
1503
|
+
await delay(1500);
|
|
1504
|
+
if (pc.iceConnectionState !== "disconnected") {
|
|
1505
|
+
return true;
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
if (pc.iceConnectionState === "failed" || pc.connectionState === "failed" || pc.connectionState === "closed") {
|
|
1509
|
+
return false;
|
|
1510
|
+
}
|
|
1511
|
+
if (typeof pc.restartIce === "function") {
|
|
1512
|
+
pc.restartIce();
|
|
1513
|
+
}
|
|
1514
|
+
const offer = await pc.createOffer({ iceRestart: true });
|
|
1515
|
+
await pc.setLocalDescription(offer);
|
|
1516
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
1517
|
+
transport: "webrtc",
|
|
1518
|
+
session: config ?? this.currentSessionConfig ?? {},
|
|
1519
|
+
sdpOffer: offer.sdp
|
|
1520
|
+
});
|
|
1521
|
+
if (bootstrap.adapter !== core.OPENAI_REALTIME_ADAPTER || bootstrap.transport !== "webrtc" || !("answerSdp" in bootstrap)) {
|
|
1522
|
+
throw new Error(
|
|
1523
|
+
`OpenAI realtime client only supports ${core.OPENAI_REALTIME_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1524
|
+
);
|
|
1525
|
+
}
|
|
1526
|
+
await pc.setRemoteDescription({
|
|
1527
|
+
type: "answer",
|
|
1528
|
+
sdp: bootstrap.answerSdp
|
|
1529
|
+
});
|
|
1530
|
+
return true;
|
|
1531
|
+
}
|
|
1532
|
+
shouldRebuildConnection() {
|
|
1533
|
+
return !this.pc || !this.dc || this.dc.readyState === "closed" || this.pc.connectionState === "failed" || this.pc.connectionState === "closed" || this.pc.iceConnectionState === "failed";
|
|
1534
|
+
}
|
|
1535
|
+
async rebuildConnection(config) {
|
|
1536
|
+
this.connectionLossNotified = true;
|
|
1537
|
+
this.cleanup();
|
|
1538
|
+
await this.connect(config ?? this.currentSessionConfig);
|
|
1539
|
+
}
|
|
1034
1540
|
resolvePendingSessionUpdate() {
|
|
1035
1541
|
if (!this.pendingSessionUpdate) {
|
|
1036
1542
|
return;
|
|
@@ -1105,6 +1611,11 @@ function toOpenAIRealtimeSessionUpdate(config) {
|
|
|
1105
1611
|
}
|
|
1106
1612
|
return session;
|
|
1107
1613
|
}
|
|
1614
|
+
function delay(ms) {
|
|
1615
|
+
return new Promise((resolve) => {
|
|
1616
|
+
setTimeout(resolve, ms);
|
|
1617
|
+
});
|
|
1618
|
+
}
|
|
1108
1619
|
|
|
1109
1620
|
// src/remote/index.ts
|
|
1110
1621
|
var DEFAULT_REMOTE_REALTIME_ADAPTERS = {
|
|
@@ -1131,30 +1642,33 @@ var RemoteRealtimeClient = class {
|
|
|
1131
1642
|
adapters;
|
|
1132
1643
|
eventCallbacks = /* @__PURE__ */ new Set();
|
|
1133
1644
|
async connect(config) {
|
|
1134
|
-
const adapterId = this.resolveAdapterId(config);
|
|
1135
|
-
const factory = this.adapters.get(adapterId);
|
|
1136
|
-
if (!factory) {
|
|
1137
|
-
throw new Error(
|
|
1138
|
-
`No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
|
|
1139
|
-
);
|
|
1140
|
-
}
|
|
1141
|
-
const transportClient = factory({
|
|
1142
|
-
debug: this.config.debug,
|
|
1143
|
-
requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
|
|
1144
|
-
});
|
|
1145
|
-
for (const callback of this.eventCallbacks) {
|
|
1146
|
-
transportClient.onEvent(callback);
|
|
1147
|
-
}
|
|
1148
|
-
this.transportClient = transportClient;
|
|
1149
1645
|
try {
|
|
1646
|
+
const adapterId = this.resolveAdapterId(config);
|
|
1647
|
+
const factory = this.adapters.get(adapterId);
|
|
1648
|
+
if (!factory) {
|
|
1649
|
+
throw new core.CharivoStateError(
|
|
1650
|
+
`No realtime adapter registered for "${adapterId}". Registered adapters: ${Array.from(this.adapters.keys()).join(", ") || "(none)"}`
|
|
1651
|
+
);
|
|
1652
|
+
}
|
|
1653
|
+
const transportClient = factory({
|
|
1654
|
+
debug: this.config.debug,
|
|
1655
|
+
requestBootstrap: (request) => this.requestBootstrap(request, { expectedAdapterId: adapterId })
|
|
1656
|
+
});
|
|
1657
|
+
for (const callback of this.eventCallbacks) {
|
|
1658
|
+
transportClient.onEvent(callback);
|
|
1659
|
+
}
|
|
1660
|
+
this.transportClient = transportClient;
|
|
1150
1661
|
await transportClient.connect(config);
|
|
1151
1662
|
} catch (error) {
|
|
1152
1663
|
this.transportClient = null;
|
|
1153
|
-
throw error;
|
|
1664
|
+
throw core.toCharivoError("transport", error);
|
|
1154
1665
|
}
|
|
1155
1666
|
}
|
|
1156
1667
|
async updateSession(config) {
|
|
1157
|
-
await this.getActiveTransportClient().updateSession(config);
|
|
1668
|
+
await this.getActiveTransportClient().updateSession(config).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
1669
|
+
}
|
|
1670
|
+
async recover(config) {
|
|
1671
|
+
await this.getActiveTransportClient().recover(config).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
1158
1672
|
}
|
|
1159
1673
|
async disconnect() {
|
|
1160
1674
|
if (!this.transportClient) {
|
|
@@ -1162,19 +1676,19 @@ var RemoteRealtimeClient = class {
|
|
|
1162
1676
|
}
|
|
1163
1677
|
const transportClient = this.transportClient;
|
|
1164
1678
|
this.transportClient = null;
|
|
1165
|
-
await transportClient.disconnect();
|
|
1679
|
+
await transportClient.disconnect().catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
1166
1680
|
}
|
|
1167
1681
|
async sendText(text) {
|
|
1168
|
-
await this.getActiveTransportClient().sendText(text);
|
|
1682
|
+
await this.getActiveTransportClient().sendText(text).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
1169
1683
|
}
|
|
1170
1684
|
async sendAudio(audio) {
|
|
1171
|
-
await this.getActiveTransportClient().sendAudio(audio);
|
|
1685
|
+
await this.getActiveTransportClient().sendAudio(audio).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
1172
1686
|
}
|
|
1173
1687
|
async sendToolResult(callId, output) {
|
|
1174
|
-
await this.getActiveTransportClient().sendToolResult(callId, output);
|
|
1688
|
+
await this.getActiveTransportClient().sendToolResult(callId, output).catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
1175
1689
|
}
|
|
1176
1690
|
async interrupt() {
|
|
1177
|
-
await this.getActiveTransportClient().interrupt();
|
|
1691
|
+
await this.getActiveTransportClient().interrupt().catch((error) => Promise.reject(core.toCharivoError("transport", error)));
|
|
1178
1692
|
}
|
|
1179
1693
|
onEvent(callback) {
|
|
1180
1694
|
this.eventCallbacks.add(callback);
|
|
@@ -1182,7 +1696,7 @@ var RemoteRealtimeClient = class {
|
|
|
1182
1696
|
}
|
|
1183
1697
|
getActiveTransportClient() {
|
|
1184
1698
|
if (!this.transportClient) {
|
|
1185
|
-
throw new
|
|
1699
|
+
throw new core.CharivoStateError("Realtime session not active");
|
|
1186
1700
|
}
|
|
1187
1701
|
return this.transportClient;
|
|
1188
1702
|
}
|
|
@@ -1194,7 +1708,7 @@ var RemoteRealtimeClient = class {
|
|
|
1194
1708
|
if (config?.provider === "openai" && transport === "webrtc") {
|
|
1195
1709
|
return core.OPENAI_REALTIME_AGENTS_ADAPTER;
|
|
1196
1710
|
}
|
|
1197
|
-
throw new
|
|
1711
|
+
throw new core.CharivoStateError(
|
|
1198
1712
|
`No remote realtime adapter resolver for provider "${config?.provider ?? "(unspecified)"}" and transport "${transport}"`
|
|
1199
1713
|
);
|
|
1200
1714
|
}
|
|
@@ -1215,14 +1729,18 @@ var RemoteRealtimeClient = class {
|
|
|
1215
1729
|
);
|
|
1216
1730
|
if (!response.ok) {
|
|
1217
1731
|
const errorText = await response.text();
|
|
1218
|
-
throw new
|
|
1732
|
+
throw new core.CharivoProviderError(
|
|
1733
|
+
`Failed to create Realtime session: ${errorText}`
|
|
1734
|
+
);
|
|
1219
1735
|
}
|
|
1220
1736
|
const bootstrap = await response.json();
|
|
1221
1737
|
if (!isRealtimeSessionBootstrap(bootstrap)) {
|
|
1222
|
-
throw new
|
|
1738
|
+
throw new core.CharivoProviderError(
|
|
1739
|
+
"Invalid realtime session bootstrap response"
|
|
1740
|
+
);
|
|
1223
1741
|
}
|
|
1224
1742
|
if (bootstrap.adapter !== options.expectedAdapterId) {
|
|
1225
|
-
throw new
|
|
1743
|
+
throw new core.CharivoProviderError(
|
|
1226
1744
|
`Realtime session bootstrap adapter mismatch: expected ${options.expectedAdapterId}, received ${bootstrap.adapter}`
|
|
1227
1745
|
);
|
|
1228
1746
|
}
|