@voxevolv/client 0.3.10 → 0.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +7 -0
- package/dist/browser/index.js +6 -6
- package/dist/browser/index.js.map +4 -4
- package/dist/browser/index.js.sha384 +1 -1
- package/dist/browser/manifest.json +1 -1
- package/dist/diagnostics.d.ts +4 -1
- package/dist/diagnostics.d.ts.map +1 -1
- package/dist/diagnostics.js +10 -1
- package/dist/diagnostics.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2379 -2284
- package/dist/index.js.map +7 -1
- package/dist/vendor/livekit-client.LICENSE +175 -0
- package/dist/vendor/livekit-client.esm.mjs +33222 -0
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,2370 +1,2465 @@
|
|
|
1
|
-
|
|
2
|
-
import {
|
|
1
|
+
// src/index.ts
|
|
2
|
+
import {
|
|
3
|
+
ConnectionState as LiveKitConnectionState,
|
|
4
|
+
DataPacket_Kind,
|
|
5
|
+
DisconnectReason,
|
|
6
|
+
LocalAudioTrack,
|
|
7
|
+
Room,
|
|
8
|
+
RoomEvent,
|
|
9
|
+
RpcError,
|
|
10
|
+
Track
|
|
11
|
+
} from "./vendor/livekit-client.esm.mjs";
|
|
12
|
+
import {
|
|
13
|
+
createClientMessage,
|
|
14
|
+
CommandError,
|
|
15
|
+
EventCursor,
|
|
16
|
+
isCommandAckPayload,
|
|
17
|
+
isCommandErrorPayload,
|
|
18
|
+
isContextRejectedPayload,
|
|
19
|
+
isContextUpdatePayload,
|
|
20
|
+
isContextUpdatedPayload,
|
|
21
|
+
isRealtimeEvent
|
|
22
|
+
} from "./protocol.js";
|
|
3
23
|
import { ContextDeliveryError, contextRetryDelay, ReplacementContextQueue } from "./context-queue.js";
|
|
4
|
-
import {
|
|
24
|
+
import {
|
|
25
|
+
ClientDiagnosticBuffer,
|
|
26
|
+
DiagnosticDeliveryError
|
|
27
|
+
} from "./diagnostics.js";
|
|
5
28
|
import { TranscriptStore } from "./transcripts.js";
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
29
|
+
import { TranscriptStore as TranscriptStore2 } from "./transcripts.js";
|
|
30
|
+
import {
|
|
31
|
+
CommandError as CommandError2,
|
|
32
|
+
EventCursor as EventCursor2,
|
|
33
|
+
createClientMessage as createClientMessage2,
|
|
34
|
+
isCommandAckPayload as isCommandAckPayload2,
|
|
35
|
+
isCommandErrorPayload as isCommandErrorPayload2,
|
|
36
|
+
isContextRejectedPayload as isContextRejectedPayload2,
|
|
37
|
+
isContextUpdatePayload as isContextUpdatePayload2,
|
|
38
|
+
isContextUpdatedPayload as isContextUpdatedPayload2,
|
|
39
|
+
isRealtimeEvent as isRealtimeEvent2
|
|
40
|
+
} from "./protocol.js";
|
|
41
|
+
import {
|
|
42
|
+
decodeAgentSnapshotPayload,
|
|
43
|
+
decodeClientHelloPayload,
|
|
44
|
+
decodeClientMessage,
|
|
45
|
+
decodeCommandAckPayload,
|
|
46
|
+
decodeCommandErrorPayload,
|
|
47
|
+
decodeContextRejectedPayload,
|
|
48
|
+
decodeContextUpdatePayload,
|
|
49
|
+
decodeContextUpdatedPayload,
|
|
50
|
+
decodeRealtimeEvent
|
|
51
|
+
} from "./generated/protocol-codecs.js";
|
|
52
|
+
var SDK_VERSION = "0.3.12";
|
|
53
|
+
var CLIENT_HELLO_TIMEOUT_MS = 3e4;
|
|
54
|
+
var RECOVERY_BUFFER_STALL_MS = 5e3;
|
|
55
|
+
var RECOVERY_BUFFER_POLL_MS = 500;
|
|
56
|
+
var DIAGNOSTIC_REPORT_TIMEOUT_MS = 5e3;
|
|
57
|
+
var DEFAULT_STARTUP_ATTEMPTS = [
|
|
58
|
+
{ delayBeforeMs: 0, readyTimeoutMs: 15e3 },
|
|
59
|
+
{ delayBeforeMs: 1e3, readyTimeoutMs: 2e4 },
|
|
60
|
+
{ delayBeforeMs: 3e3, readyTimeoutMs: 5e4 }
|
|
16
61
|
];
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
62
|
+
var DEFAULT_STARTUP_TIMEOUT_MS = 9e4;
|
|
63
|
+
var RecoveryDataChannelStalledError = class extends Error {
|
|
64
|
+
constructor() {
|
|
65
|
+
super("Recovery data channel stopped draining before the agent handshake completed");
|
|
66
|
+
this.name = "RecoveryDataChannelStalledError";
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
var RealtimeSessionHttpError = class extends Error {
|
|
70
|
+
status;
|
|
71
|
+
code;
|
|
72
|
+
requestId;
|
|
73
|
+
details;
|
|
74
|
+
constructor(message, options) {
|
|
75
|
+
super(message);
|
|
76
|
+
this.name = "RealtimeSessionHttpError";
|
|
77
|
+
this.status = options.status;
|
|
78
|
+
this.code = options.code ?? null;
|
|
79
|
+
this.requestId = options.requestId ?? null;
|
|
80
|
+
this.details = options.details ?? [];
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
var SessionConfigurationError = class extends TypeError {
|
|
84
|
+
};
|
|
85
|
+
var SessionErrorEventError = class extends Error {
|
|
86
|
+
retryable;
|
|
87
|
+
constructor(event) {
|
|
88
|
+
const payload = event.payload;
|
|
89
|
+
const message = typeof payload.message === "string" ? payload.message : "VoxEvolv reported a session error";
|
|
90
|
+
super(message);
|
|
91
|
+
this.name = "SessionErrorEventError";
|
|
92
|
+
this.retryable = payload.retryable === true;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
var RealtimeSession = class _RealtimeSession {
|
|
96
|
+
id;
|
|
97
|
+
room;
|
|
98
|
+
accessToken;
|
|
99
|
+
credentials;
|
|
100
|
+
stateValue = "idle";
|
|
101
|
+
listeners = /* @__PURE__ */ new Set();
|
|
102
|
+
stateListeners = /* @__PURE__ */ new Set();
|
|
103
|
+
gapListeners = /* @__PURE__ */ new Set();
|
|
104
|
+
connectionQualityListeners = /* @__PURE__ */ new Set();
|
|
105
|
+
recoveryListeners = /* @__PURE__ */ new Set();
|
|
106
|
+
startupAttemptListeners = /* @__PURE__ */ new Set();
|
|
107
|
+
remoteAudioTrackListeners = /* @__PURE__ */ new Set();
|
|
108
|
+
remoteAudioTrackUnsubscribedListeners = /* @__PURE__ */ new Set();
|
|
109
|
+
readyListeners = /* @__PURE__ */ new Set();
|
|
110
|
+
userTranscriptListeners = /* @__PURE__ */ new Set();
|
|
111
|
+
agentTranscriptListeners = /* @__PURE__ */ new Set();
|
|
112
|
+
transcriptListeners = /* @__PURE__ */ new Set();
|
|
113
|
+
transcriptStore = new TranscriptStore();
|
|
114
|
+
errorListeners = /* @__PURE__ */ new Set();
|
|
115
|
+
endedListeners = /* @__PURE__ */ new Set();
|
|
116
|
+
toolCallListeners = /* @__PURE__ */ new Set();
|
|
117
|
+
audioPlaybackBlockedListeners = /* @__PURE__ */ new Set();
|
|
118
|
+
remoteAudioStreams = /* @__PURE__ */ new WeakMap();
|
|
119
|
+
outputMediaStream = null;
|
|
120
|
+
remoteAudioTrackResolve = null;
|
|
121
|
+
cursor = new EventCursor();
|
|
122
|
+
connectionId = `conn_${Math.random().toString(36).slice(2)}_${Date.now().toString(36)}`;
|
|
123
|
+
clientSeq = 0;
|
|
124
|
+
lastHelloMessageId = null;
|
|
125
|
+
reconnectStartedAt = null;
|
|
126
|
+
pendingReconnect = null;
|
|
127
|
+
agentReady = false;
|
|
128
|
+
agentStreamId = null;
|
|
129
|
+
contextGeneration = 0;
|
|
130
|
+
replacementContext = new ReplacementContextQueue(
|
|
131
|
+
(update, timeoutMs) => this.sendContextUpdate(update, timeoutMs)
|
|
132
|
+
);
|
|
133
|
+
readyResolve = null;
|
|
134
|
+
replayPromise = null;
|
|
135
|
+
pendingCommands = /* @__PURE__ */ new Map();
|
|
136
|
+
commandPublishTail = Promise.resolve();
|
|
137
|
+
toolHandlers = /* @__PURE__ */ new Map();
|
|
138
|
+
apiBase = "/v1";
|
|
139
|
+
hasConnectedValue = false;
|
|
140
|
+
sessionEnded = false;
|
|
141
|
+
pendingEndEvent = null;
|
|
142
|
+
agentDisconnected = false;
|
|
143
|
+
terminalSessionError = null;
|
|
144
|
+
endNotificationDelivered = false;
|
|
145
|
+
disconnectMode = null;
|
|
146
|
+
activeRecovery = null;
|
|
147
|
+
recoveryStartedAt = null;
|
|
148
|
+
livekitResumeController = null;
|
|
149
|
+
autoRejoinPromise = null;
|
|
150
|
+
autoRejoinController = null;
|
|
151
|
+
manualRejoinController = null;
|
|
152
|
+
startupConnectController = null;
|
|
153
|
+
startupAttemptController = null;
|
|
154
|
+
useManagedMicrophone = true;
|
|
155
|
+
microphoneEnabled = true;
|
|
156
|
+
applicationAudioTrack = null;
|
|
157
|
+
localApplicationAudioTrack = null;
|
|
158
|
+
connectDefaults;
|
|
159
|
+
recoveryReadiness;
|
|
160
|
+
credentialRefreshGeneration = 0;
|
|
161
|
+
fetcher;
|
|
162
|
+
endEndpoint;
|
|
163
|
+
refreshCredentialsCallback;
|
|
164
|
+
autoRejoinOptions;
|
|
165
|
+
startupRetryOptions;
|
|
166
|
+
lifecycleCredentials;
|
|
167
|
+
diagnostics = new ClientDiagnosticBuffer({ sdkVersion: SDK_VERSION });
|
|
168
|
+
activeContextTransfers = 0;
|
|
169
|
+
activeContextBytes = 0;
|
|
170
|
+
diagnosticLoopTimer;
|
|
171
|
+
diagnosticLoopExpectedAt = 0;
|
|
172
|
+
diagnosticLoopDelayMs;
|
|
173
|
+
diagnosticReportTimeoutMs = DIAGNOSTIC_REPORT_TIMEOUT_MS;
|
|
174
|
+
onDiagnosticVisibilityChange = () => {
|
|
175
|
+
if (globalThis.document?.visibilityState === "hidden") this.flushDiagnostics("page_hidden");
|
|
176
|
+
};
|
|
177
|
+
constructor(credentials, options = {}) {
|
|
178
|
+
validateSessionCredentials(credentials);
|
|
179
|
+
this.id = credentials.id;
|
|
180
|
+
this.accessToken = credentials.access_token;
|
|
181
|
+
this.credentials = credentials.connection;
|
|
182
|
+
this.connectDefaults = options.connect?.apiBase === void 0 ? options.connect ?? {} : { ...options.connect, apiBase: normalizeApiBase(options.connect.apiBase) };
|
|
183
|
+
this.apiBase = this.connectDefaults.apiBase ?? this.apiBase;
|
|
184
|
+
this.recoveryReadiness = {
|
|
185
|
+
readyTimeoutMs: this.connectDefaults.readyTimeoutMs ?? 15e3,
|
|
186
|
+
waitForRemoteAudio: this.connectDefaults.waitForRemoteAudio ?? true
|
|
124
187
|
};
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
});
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
if (options.signal?.aborted) {
|
|
298
|
-
return Promise.reject(options.signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
299
|
-
}
|
|
300
|
-
return new Promise((resolve, reject) => {
|
|
301
|
-
let settled = false;
|
|
302
|
-
const finish = (state, error) => {
|
|
303
|
-
if (settled)
|
|
304
|
-
return;
|
|
305
|
-
settled = true;
|
|
306
|
-
globalThis.clearTimeout(timeout);
|
|
307
|
-
this.stateListeners.delete(onState);
|
|
308
|
-
options.signal?.removeEventListener("abort", onAbort);
|
|
309
|
-
if (error !== undefined) {
|
|
310
|
-
reject(error);
|
|
311
|
-
}
|
|
312
|
-
else if (state !== null) {
|
|
313
|
-
resolve(state);
|
|
314
|
-
}
|
|
315
|
-
};
|
|
316
|
-
const onState = (state) => {
|
|
317
|
-
if (targets.has(state))
|
|
318
|
-
finish(state);
|
|
319
|
-
};
|
|
320
|
-
const onAbort = () => {
|
|
321
|
-
finish(null, options.signal?.reason ?? new DOMException("Aborted", "AbortError"));
|
|
322
|
-
};
|
|
323
|
-
const timeout = globalThis.setTimeout(() => {
|
|
324
|
-
finish(null, new Error(`Session did not reach ${states.join(" or ")} within ${timeoutMs}ms`));
|
|
325
|
-
}, timeoutMs);
|
|
326
|
-
this.stateListeners.add(onState);
|
|
327
|
-
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
328
|
-
});
|
|
329
|
-
}
|
|
330
|
-
onRemoteAudioTrack(listener) {
|
|
331
|
-
this.remoteAudioTrackListeners.add(listener);
|
|
332
|
-
return () => this.remoteAudioTrackListeners.delete(listener);
|
|
333
|
-
}
|
|
334
|
-
onRemoteAudioTrackUnsubscribed(listener) {
|
|
335
|
-
this.remoteAudioTrackUnsubscribedListeners.add(listener);
|
|
336
|
-
return () => this.remoteAudioTrackUnsubscribedListeners.delete(listener);
|
|
337
|
-
}
|
|
338
|
-
onReady(listener) {
|
|
339
|
-
this.readyListeners.add(listener);
|
|
340
|
-
return () => this.readyListeners.delete(listener);
|
|
341
|
-
}
|
|
342
|
-
onUserTranscript(listener) {
|
|
343
|
-
this.userTranscriptListeners.add(listener);
|
|
344
|
-
return () => this.userTranscriptListeners.delete(listener);
|
|
345
|
-
}
|
|
346
|
-
onAgentTranscript(listener) {
|
|
347
|
-
this.agentTranscriptListeners.add(listener);
|
|
348
|
-
return () => this.agentTranscriptListeners.delete(listener);
|
|
349
|
-
}
|
|
350
|
-
/** Latest message snapshots, retained across transport recovery and replay. */
|
|
351
|
-
get transcript() {
|
|
352
|
-
return this.transcriptStore.snapshot;
|
|
353
|
-
}
|
|
354
|
-
onTranscriptUpdated(listener) {
|
|
355
|
-
this.transcriptListeners.add(listener);
|
|
356
|
-
return () => this.transcriptListeners.delete(listener);
|
|
357
|
-
}
|
|
358
|
-
onError(listener) {
|
|
359
|
-
this.errorListeners.add(listener);
|
|
360
|
-
return () => this.errorListeners.delete(listener);
|
|
361
|
-
}
|
|
362
|
-
onEnded(listener) {
|
|
363
|
-
this.endedListeners.add(listener);
|
|
364
|
-
return () => this.endedListeners.delete(listener);
|
|
188
|
+
this.fetcher = options.fetch ?? globalThis.fetch.bind(globalThis);
|
|
189
|
+
this.endEndpoint = options.endEndpoint ?? null;
|
|
190
|
+
this.refreshCredentialsCallback = options.refreshCredentials ?? null;
|
|
191
|
+
this.autoRejoinOptions = resolveAutoRejoinOptions(options.autoRejoin);
|
|
192
|
+
this.startupRetryOptions = resolveStartupRetryOptions(options.startupRetry);
|
|
193
|
+
this.lifecycleCredentials = options.lifecycleCredentials ?? "same-origin";
|
|
194
|
+
this.room = new Room({ adaptiveStream: true, dynacast: true });
|
|
195
|
+
for (const [name, handler] of Object.entries(options.tools ?? {})) {
|
|
196
|
+
this.registerTool(name, handler);
|
|
197
|
+
}
|
|
198
|
+
this.room.registerRpcMethod("voxevolv.tool.invoke", (invocation) => this.invokeBrowserTool(invocation));
|
|
199
|
+
this.room.registerRpcMethod("voxevolv.diagnostics.flush", async (invocation) => {
|
|
200
|
+
const payload = JSON.parse(invocation.payload);
|
|
201
|
+
if (typeof payload?.call_id !== "string" || payload.call_id.length > 200) {
|
|
202
|
+
throw new RpcError(1500, "Invalid diagnostic request");
|
|
203
|
+
}
|
|
204
|
+
this.recordDiagnostic({
|
|
205
|
+
operation: "tool_rpc",
|
|
206
|
+
status: "started",
|
|
207
|
+
stage: "flush_requested",
|
|
208
|
+
call_id: payload.call_id,
|
|
209
|
+
rpc_request_id: invocation.requestId
|
|
210
|
+
});
|
|
211
|
+
this.flushDiagnostics("tool_rpc_worker_request");
|
|
212
|
+
return "{}";
|
|
213
|
+
});
|
|
214
|
+
this.bindRoomEvents();
|
|
215
|
+
}
|
|
216
|
+
async invokeBrowserTool(invocation) {
|
|
217
|
+
const receivedAt = performance.now();
|
|
218
|
+
let callId;
|
|
219
|
+
let toolName;
|
|
220
|
+
let slowTimer;
|
|
221
|
+
const record = (entry) => this.recordDiagnostic({
|
|
222
|
+
operation: "tool_rpc",
|
|
223
|
+
status: "started",
|
|
224
|
+
rpc_request_id: invocation.requestId,
|
|
225
|
+
call_id: callId,
|
|
226
|
+
tool_name: toolName,
|
|
227
|
+
duration_ms: performance.now() - receivedAt,
|
|
228
|
+
...entry
|
|
229
|
+
});
|
|
230
|
+
record({ stage: "received" });
|
|
231
|
+
try {
|
|
232
|
+
let payload;
|
|
233
|
+
try {
|
|
234
|
+
payload = JSON.parse(invocation.payload);
|
|
235
|
+
} catch (error) {
|
|
236
|
+
throw new RpcError(1500, "Tool invocation payload was not valid JSON", void 0, { cause: error });
|
|
237
|
+
}
|
|
238
|
+
if (typeof payload?.name !== "string") {
|
|
239
|
+
throw new RpcError(1500, "Tool invocation did not include a valid name");
|
|
240
|
+
}
|
|
241
|
+
toolName = payload.name;
|
|
242
|
+
callId = typeof payload.call_id === "string" ? payload.call_id : invocation.requestId;
|
|
243
|
+
const handler = this.toolHandlers.get(toolName);
|
|
244
|
+
if (!handler) throw new RpcError(1500, `No browser handler is registered for ${toolName}`);
|
|
245
|
+
if (!payload.arguments || typeof payload.arguments !== "object" || Array.isArray(payload.arguments)) {
|
|
246
|
+
throw new RpcError(1500, "Tool invocation arguments must be an object");
|
|
247
|
+
}
|
|
248
|
+
const arguments_ = payload.arguments;
|
|
249
|
+
this.notifyListeners(this.toolCallListeners, { name: toolName, callId, arguments: arguments_ });
|
|
250
|
+
record({ stage: "handler_started" });
|
|
251
|
+
this.flushDiagnostics("tool_rpc_received");
|
|
252
|
+
slowTimer = globalThis.setTimeout(() => {
|
|
253
|
+
record({ stage: "handler_started", reason: "handler_slow" });
|
|
254
|
+
this.flushDiagnostics("tool_rpc_slow");
|
|
255
|
+
}, 2e3);
|
|
256
|
+
let response;
|
|
257
|
+
try {
|
|
258
|
+
const result = await handler(arguments_, { callId });
|
|
259
|
+
response = JSON.stringify(result ?? null);
|
|
260
|
+
} catch (error) {
|
|
261
|
+
const message = error instanceof Error ? error.message : "Browser tool handler failed";
|
|
262
|
+
throw new RpcError(1500, message, void 0, { cause: error });
|
|
263
|
+
}
|
|
264
|
+
record({ stage: "response_ready", status: "succeeded" });
|
|
265
|
+
return response;
|
|
266
|
+
} catch (error) {
|
|
267
|
+
record({ status: "failed", rpc_error_code: error instanceof RpcError ? error.code : void 0 });
|
|
268
|
+
throw error;
|
|
269
|
+
} finally {
|
|
270
|
+
globalThis.clearTimeout(slowTimer);
|
|
271
|
+
this.flushDiagnostics("tool_rpc_finished");
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
static async create(options) {
|
|
275
|
+
if ("connect" in options) {
|
|
276
|
+
throw new TypeError(
|
|
277
|
+
"RealtimeSession.create() accepts apiBase and media options at the top level, not in connect"
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
const {
|
|
281
|
+
endpoint,
|
|
282
|
+
request,
|
|
283
|
+
microphone,
|
|
284
|
+
audioTrack,
|
|
285
|
+
readyTimeoutMs,
|
|
286
|
+
waitForRemoteAudio,
|
|
287
|
+
apiBase,
|
|
288
|
+
...sessionOptions
|
|
289
|
+
} = options;
|
|
290
|
+
if (!endpoint) throw new TypeError("endpoint is required");
|
|
291
|
+
if (!apiBase) {
|
|
292
|
+
throw new TypeError(
|
|
293
|
+
"apiBase is required so rejoin, replay, and end use the intended VoxEvolv API or application proxy"
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
const normalizedApiBase = normalizeApiBase(apiBase);
|
|
297
|
+
const fetcher = sessionOptions.fetch ?? globalThis.fetch.bind(globalThis);
|
|
298
|
+
const response = await fetcher(endpoint, { ...request, method: "POST" });
|
|
299
|
+
if (!response.ok) throw await httpError(response, "Could not create realtime session");
|
|
300
|
+
const credentials = await response.json();
|
|
301
|
+
return new _RealtimeSession(credentials, {
|
|
302
|
+
...sessionOptions,
|
|
303
|
+
fetch: fetcher,
|
|
304
|
+
connect: { microphone, audioTrack, readyTimeoutMs, waitForRemoteAudio, apiBase: normalizedApiBase }
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
get state() {
|
|
308
|
+
return this.stateValue;
|
|
309
|
+
}
|
|
310
|
+
get hasConnectedBefore() {
|
|
311
|
+
return this.hasConnectedValue;
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Stable stream holding the agent's current remote audio tracks. The stream
|
|
315
|
+
* object never changes across track arrival, reconnects, or rejoin; tracks
|
|
316
|
+
* are added and removed as the room subscribes and unsubscribes them.
|
|
317
|
+
*/
|
|
318
|
+
get outputStream() {
|
|
319
|
+
this.outputMediaStream ??= new MediaStream();
|
|
320
|
+
return this.outputMediaStream;
|
|
321
|
+
}
|
|
322
|
+
onEvent(listener) {
|
|
323
|
+
this.listeners.add(listener);
|
|
324
|
+
return () => this.listeners.delete(listener);
|
|
325
|
+
}
|
|
326
|
+
onStateChange(listener) {
|
|
327
|
+
this.stateListeners.add(listener);
|
|
328
|
+
this.notifyListeners(/* @__PURE__ */ new Set([listener]), this.stateValue);
|
|
329
|
+
return () => this.stateListeners.delete(listener);
|
|
330
|
+
}
|
|
331
|
+
onProtocolGap(listener) {
|
|
332
|
+
this.gapListeners.add(listener);
|
|
333
|
+
return () => this.gapListeners.delete(listener);
|
|
334
|
+
}
|
|
335
|
+
onConnectionQuality(listener) {
|
|
336
|
+
this.connectionQualityListeners.add(listener);
|
|
337
|
+
this.notifyListeners(
|
|
338
|
+
/* @__PURE__ */ new Set([listener]),
|
|
339
|
+
this.room.localParticipant.connectionQuality
|
|
340
|
+
);
|
|
341
|
+
return () => this.connectionQualityListeners.delete(listener);
|
|
342
|
+
}
|
|
343
|
+
onRecovery(listener) {
|
|
344
|
+
this.recoveryListeners.add(listener);
|
|
345
|
+
return () => this.recoveryListeners.delete(listener);
|
|
346
|
+
}
|
|
347
|
+
onStartupAttempt(listener) {
|
|
348
|
+
this.startupAttemptListeners.add(listener);
|
|
349
|
+
return () => this.startupAttemptListeners.delete(listener);
|
|
350
|
+
}
|
|
351
|
+
waitForState(states, timeoutMs, options = {}) {
|
|
352
|
+
if (!states.length) throw new TypeError("states must contain at least one connection state");
|
|
353
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
|
354
|
+
throw new TypeError("timeoutMs must be a non-negative finite number");
|
|
355
|
+
}
|
|
356
|
+
const targets = new Set(states);
|
|
357
|
+
if (targets.has(this.stateValue)) return Promise.resolve(this.stateValue);
|
|
358
|
+
if (options.signal?.aborted) {
|
|
359
|
+
return Promise.reject(options.signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
365
360
|
}
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
361
|
+
return new Promise((resolve, reject) => {
|
|
362
|
+
let settled = false;
|
|
363
|
+
const finish = (state, error) => {
|
|
364
|
+
if (settled) return;
|
|
365
|
+
settled = true;
|
|
366
|
+
globalThis.clearTimeout(timeout);
|
|
367
|
+
this.stateListeners.delete(onState);
|
|
368
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
369
|
+
if (error !== void 0) {
|
|
370
|
+
reject(error);
|
|
371
|
+
} else if (state !== null) {
|
|
372
|
+
resolve(state);
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
const onState = (state) => {
|
|
376
|
+
if (targets.has(state)) finish(state);
|
|
377
|
+
};
|
|
378
|
+
const onAbort = () => {
|
|
379
|
+
finish(null, options.signal?.reason ?? new DOMException("Aborted", "AbortError"));
|
|
380
|
+
};
|
|
381
|
+
const timeout = globalThis.setTimeout(() => {
|
|
382
|
+
finish(
|
|
383
|
+
null,
|
|
384
|
+
new Error(`Session did not reach ${states.join(" or ")} within ${timeoutMs}ms`)
|
|
385
|
+
);
|
|
386
|
+
}, timeoutMs);
|
|
387
|
+
this.stateListeners.add(onState);
|
|
388
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
onRemoteAudioTrack(listener) {
|
|
392
|
+
this.remoteAudioTrackListeners.add(listener);
|
|
393
|
+
return () => this.remoteAudioTrackListeners.delete(listener);
|
|
394
|
+
}
|
|
395
|
+
onRemoteAudioTrackUnsubscribed(listener) {
|
|
396
|
+
this.remoteAudioTrackUnsubscribedListeners.add(listener);
|
|
397
|
+
return () => this.remoteAudioTrackUnsubscribedListeners.delete(listener);
|
|
398
|
+
}
|
|
399
|
+
onReady(listener) {
|
|
400
|
+
this.readyListeners.add(listener);
|
|
401
|
+
return () => this.readyListeners.delete(listener);
|
|
402
|
+
}
|
|
403
|
+
onUserTranscript(listener) {
|
|
404
|
+
this.userTranscriptListeners.add(listener);
|
|
405
|
+
return () => this.userTranscriptListeners.delete(listener);
|
|
406
|
+
}
|
|
407
|
+
onAgentTranscript(listener) {
|
|
408
|
+
this.agentTranscriptListeners.add(listener);
|
|
409
|
+
return () => this.agentTranscriptListeners.delete(listener);
|
|
410
|
+
}
|
|
411
|
+
/** Latest message snapshots, retained across transport recovery and replay. */
|
|
412
|
+
get transcript() {
|
|
413
|
+
return this.transcriptStore.snapshot;
|
|
414
|
+
}
|
|
415
|
+
onTranscriptUpdated(listener) {
|
|
416
|
+
this.transcriptListeners.add(listener);
|
|
417
|
+
return () => this.transcriptListeners.delete(listener);
|
|
418
|
+
}
|
|
419
|
+
onError(listener) {
|
|
420
|
+
this.errorListeners.add(listener);
|
|
421
|
+
return () => this.errorListeners.delete(listener);
|
|
422
|
+
}
|
|
423
|
+
onEnded(listener) {
|
|
424
|
+
this.endedListeners.add(listener);
|
|
425
|
+
return () => this.endedListeners.delete(listener);
|
|
426
|
+
}
|
|
427
|
+
onToolCall(listener) {
|
|
428
|
+
this.toolCallListeners.add(listener);
|
|
429
|
+
return () => this.toolCallListeners.delete(listener);
|
|
430
|
+
}
|
|
431
|
+
onAudioPlaybackBlocked(listener) {
|
|
432
|
+
this.audioPlaybackBlockedListeners.add(listener);
|
|
433
|
+
return () => this.audioPlaybackBlockedListeners.delete(listener);
|
|
434
|
+
}
|
|
435
|
+
registerTool(name, handler) {
|
|
436
|
+
if (["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"].includes(this.stateValue)) {
|
|
437
|
+
throw new Error("Browser tools must be registered before connect()");
|
|
438
|
+
}
|
|
439
|
+
if (!/^[a-z][a-z0-9_]{0,79}$/.test(name)) {
|
|
440
|
+
throw new TypeError("Tool name must be lowercase letters, numbers, and underscores");
|
|
441
|
+
}
|
|
442
|
+
if (this.toolHandlers.has(name)) throw new TypeError(`Tool ${name} is already registered`);
|
|
443
|
+
this.toolHandlers.set(name, handler);
|
|
444
|
+
return () => {
|
|
445
|
+
if (this.toolHandlers.get(name) === handler) this.toolHandlers.delete(name);
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
async connect(options = {}) {
|
|
449
|
+
if (this.startupConnectController || this.manualRejoinController) {
|
|
450
|
+
throw new Error("A session connection attempt is already in progress");
|
|
369
451
|
}
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
return () => this.audioPlaybackBlockedListeners.delete(listener);
|
|
452
|
+
if (["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"].includes(this.stateValue)) {
|
|
453
|
+
throw new Error(`Cannot connect while session is ${this.stateValue}`);
|
|
373
454
|
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
455
|
+
this.cancelRecovery();
|
|
456
|
+
const controller = new AbortController();
|
|
457
|
+
this.startupConnectController = controller;
|
|
458
|
+
this.credentialRefreshGeneration += 1;
|
|
459
|
+
const resolved = { ...this.connectDefaults, ...options };
|
|
460
|
+
this.recoveryReadiness = {
|
|
461
|
+
readyTimeoutMs: resolved.readyTimeoutMs ?? 15e3,
|
|
462
|
+
waitForRemoteAudio: resolved.waitForRemoteAudio ?? true
|
|
463
|
+
};
|
|
464
|
+
this.microphoneEnabled = resolved.audioTrack ? resolved.audioTrack.enabled !== false : resolved.microphone ?? true;
|
|
465
|
+
this.setState("connecting");
|
|
466
|
+
try {
|
|
467
|
+
if (this.startupRetryOptions) {
|
|
468
|
+
await this.connectWithStartupRetry(options, controller);
|
|
469
|
+
} else {
|
|
470
|
+
this.startupAttemptController = controller;
|
|
471
|
+
await this.connectWithMode(options, false, controller.signal, false);
|
|
472
|
+
}
|
|
473
|
+
} catch (error) {
|
|
474
|
+
if (this.startupConnectController === controller && this.terminalSessionError) {
|
|
475
|
+
this.sessionEnded = true;
|
|
476
|
+
this.setState("failed");
|
|
477
|
+
} else if (this.startupConnectController === controller && !this.sessionEnded) {
|
|
478
|
+
this.setState("failed");
|
|
479
|
+
}
|
|
480
|
+
throw error;
|
|
481
|
+
} finally {
|
|
482
|
+
if (this.startupAttemptController === controller) {
|
|
483
|
+
this.startupAttemptController = null;
|
|
484
|
+
}
|
|
485
|
+
if (this.startupConnectController === controller) {
|
|
486
|
+
this.startupConnectController = null;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
async connectWithStartupRetry(options, controller) {
|
|
491
|
+
const startupOptions = this.startupRetryOptions;
|
|
492
|
+
if (!startupOptions) return;
|
|
493
|
+
const startupStartedAt = performance.now();
|
|
494
|
+
const timeout = globalThis.setTimeout(() => {
|
|
495
|
+
controller.abort(new Error(
|
|
496
|
+
`Session startup did not complete within ${startupOptions.timeoutMs}ms`
|
|
497
|
+
));
|
|
498
|
+
}, startupOptions.timeoutMs);
|
|
499
|
+
let lastError = new Error("Session startup failed");
|
|
500
|
+
let startupRegistered = this.hasConnectedValue;
|
|
501
|
+
this.apiBase = normalizeApiBase(options.apiBase ?? this.connectDefaults.apiBase ?? this.apiBase);
|
|
502
|
+
try {
|
|
503
|
+
for (const [index, attempt] of startupOptions.attempts.entries()) {
|
|
504
|
+
const attemptNumber = index + 1;
|
|
505
|
+
if (attempt.delayBeforeMs > 0) {
|
|
506
|
+
this.setState("connecting");
|
|
507
|
+
await abortableDelay(attempt.delayBeforeMs, controller.signal);
|
|
508
|
+
}
|
|
509
|
+
throwIfAborted(controller.signal);
|
|
510
|
+
const attemptController = createLinkedAbortController(controller.signal);
|
|
511
|
+
this.startupAttemptController = attemptController;
|
|
512
|
+
this.notifyStartupAttempt("started", attemptNumber, attempt.readyTimeoutMs);
|
|
412
513
|
try {
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
514
|
+
if (!startupRegistered) {
|
|
515
|
+
const remainingMs = startupOptions.timeoutMs - (performance.now() - startupStartedAt);
|
|
516
|
+
await this.registerStartupRecovery(remainingMs, attemptController.signal);
|
|
517
|
+
startupRegistered = true;
|
|
518
|
+
}
|
|
519
|
+
throwIfAborted(attemptController.signal);
|
|
520
|
+
if (index === 0) {
|
|
521
|
+
await this.connectWithMode(
|
|
522
|
+
{ ...options, readyTimeoutMs: attempt.readyTimeoutMs },
|
|
523
|
+
false,
|
|
524
|
+
attemptController.signal,
|
|
525
|
+
false
|
|
526
|
+
);
|
|
527
|
+
} else {
|
|
528
|
+
await this.performRejoin(this.apiBase, {
|
|
529
|
+
signal: attemptController.signal,
|
|
530
|
+
recovering: false,
|
|
531
|
+
connect: { ...options, readyTimeoutMs: attempt.readyTimeoutMs },
|
|
532
|
+
markFailureTerminal: false
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
throwIfAborted(controller.signal);
|
|
536
|
+
this.notifyStartupAttempt("succeeded", attemptNumber, attempt.readyTimeoutMs);
|
|
537
|
+
return;
|
|
538
|
+
} catch (error) {
|
|
539
|
+
lastError = toError(error);
|
|
540
|
+
this.notifyStartupAttempt("failed", attemptNumber, attempt.readyTimeoutMs, lastError);
|
|
541
|
+
if (controller.signal.aborted) throw abortError(controller.signal);
|
|
542
|
+
if (isTerminalStartupError(error) || attemptNumber === startupOptions.attempts.length) {
|
|
429
543
|
throw error;
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
544
|
+
}
|
|
545
|
+
this.setState("connecting");
|
|
546
|
+
} finally {
|
|
547
|
+
if (this.startupAttemptController === attemptController) {
|
|
548
|
+
this.startupAttemptController = null;
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
throw lastError;
|
|
553
|
+
} finally {
|
|
554
|
+
globalThis.clearTimeout(timeout);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
async registerStartupRecovery(timeoutMs, signal) {
|
|
558
|
+
const reserveMs = Math.min(5e3, timeoutMs / 10);
|
|
559
|
+
const controller = createLinkedAbortController(signal);
|
|
560
|
+
const requestTimeout = globalThis.setTimeout(() => {
|
|
561
|
+
controller.abort(new Error("Startup registration timed out"));
|
|
562
|
+
}, Math.max(1, reserveMs));
|
|
563
|
+
try {
|
|
564
|
+
const send = () => raceWithAbort(this.fetcher(`${this.apiBase}/sessions/${this.id}/startup`, {
|
|
565
|
+
method: "POST",
|
|
566
|
+
headers: { Authorization: `Bearer ${this.accessToken}`, "Content-Type": "application/json" },
|
|
567
|
+
credentials: this.lifecycleCredentials,
|
|
568
|
+
body: JSON.stringify({ timeout_ms: Math.max(1, Math.floor(Math.min(timeoutMs - reserveMs, 85e3))) }),
|
|
569
|
+
signal: controller.signal
|
|
570
|
+
}), controller.signal);
|
|
571
|
+
let response = await send();
|
|
572
|
+
if (response.status === 401 && this.refreshCredentialsCallback) {
|
|
573
|
+
await this.refreshCredentials(controller.signal);
|
|
574
|
+
response = await send();
|
|
575
|
+
}
|
|
576
|
+
if (!response.ok) throw await httpError(response, "Could not register session startup");
|
|
577
|
+
const body = await raceWithAbort(response.json(), controller.signal);
|
|
578
|
+
if (typeof body.remaining_ms !== "number" || body.remaining_ms <= 0) {
|
|
579
|
+
throw new Error("Session startup window has expired");
|
|
580
|
+
}
|
|
581
|
+
} finally {
|
|
582
|
+
globalThis.clearTimeout(requestTimeout);
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
notifyStartupAttempt(status, attempt, readyTimeoutMs, error) {
|
|
586
|
+
const maxAttempts = this.startupRetryOptions?.attempts.length ?? 1;
|
|
587
|
+
this.notifyListeners(this.startupAttemptListeners, {
|
|
588
|
+
status,
|
|
589
|
+
attempt,
|
|
590
|
+
maxAttempts,
|
|
591
|
+
readyTimeoutMs,
|
|
592
|
+
...error ? { error } : {}
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
async connectWithMode(options, recovering, signal, markFailureTerminal = true) {
|
|
596
|
+
throwIfAborted(signal);
|
|
597
|
+
const blockedStates = recovering ? ["connecting", "connected_not_ready", "connected", "reconnecting"] : ["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"];
|
|
598
|
+
const continuingStartupRetry = !recovering && !markFailureTerminal && this.stateValue === "connecting";
|
|
599
|
+
if (blockedStates.includes(this.stateValue) && !continuingStartupRetry) {
|
|
600
|
+
throw new Error(`Cannot connect while session is ${this.stateValue}`);
|
|
601
|
+
}
|
|
602
|
+
const generation = this.contextGeneration;
|
|
603
|
+
const resolved = { ...this.connectDefaults, ...options };
|
|
604
|
+
const audioTrack = resolved.audioTrack ?? null;
|
|
605
|
+
const microphone = resolved.microphone ?? audioTrack === null;
|
|
606
|
+
const readyTimeoutMs = resolved.readyTimeoutMs ?? 15e3;
|
|
607
|
+
const waitForRemoteAudio = resolved.waitForRemoteAudio ?? true;
|
|
608
|
+
const apiBase = normalizeApiBase(resolved.apiBase ?? this.apiBase);
|
|
609
|
+
this.validateAudioOptions(microphone, audioTrack);
|
|
610
|
+
this.useManagedMicrophone = microphone;
|
|
611
|
+
this.applicationAudioTrack = audioTrack;
|
|
612
|
+
if (!recovering) {
|
|
613
|
+
this.sessionEnded = false;
|
|
614
|
+
this.agentStreamId = null;
|
|
615
|
+
this.reconnectStartedAt = null;
|
|
616
|
+
this.pendingEndEvent = null;
|
|
617
|
+
this.agentDisconnected = false;
|
|
618
|
+
this.terminalSessionError = null;
|
|
619
|
+
this.endNotificationDelivered = false;
|
|
620
|
+
this.setState("connecting");
|
|
621
|
+
} else {
|
|
622
|
+
this.setState("resuming");
|
|
623
|
+
}
|
|
624
|
+
this.apiBase = apiBase;
|
|
625
|
+
this.agentReady = false;
|
|
626
|
+
try {
|
|
627
|
+
await this.measureRecoveryStage(
|
|
628
|
+
"room_connect",
|
|
629
|
+
() => this.connectRoom(signal, recovering),
|
|
630
|
+
recovering
|
|
631
|
+
);
|
|
632
|
+
throwIfAborted(signal);
|
|
633
|
+
if (audioTrack) {
|
|
634
|
+
await raceWithAbort(this.publishApplicationAudioTrack(audioTrack), signal);
|
|
635
|
+
} else if (microphone) {
|
|
636
|
+
await raceWithAbort(
|
|
637
|
+
this.room.localParticipant.setMicrophoneEnabled(this.microphoneEnabled),
|
|
638
|
+
signal
|
|
639
|
+
);
|
|
640
|
+
}
|
|
641
|
+
throwIfAborted(signal);
|
|
642
|
+
this.setState(recovering ? "resuming" : "connected_not_ready");
|
|
643
|
+
await this.measureRecoveryStage("hello", () => recovering ? this.sendRecoveryHello(signal) : raceWithAbort(this.sendHello(), signal), recovering);
|
|
644
|
+
const readyDeadline = Date.now() + readyTimeoutMs;
|
|
645
|
+
await this.measureRecoveryStage(
|
|
646
|
+
"agent_ready",
|
|
647
|
+
() => this.waitForAgentReady(readyTimeoutMs, signal),
|
|
648
|
+
recovering
|
|
649
|
+
);
|
|
650
|
+
if (waitForRemoteAudio) {
|
|
651
|
+
await this.measureRecoveryStage(
|
|
652
|
+
"remote_audio",
|
|
653
|
+
() => this.waitForRemoteAudioTrack(readyDeadline, signal),
|
|
654
|
+
recovering
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
throwIfAborted(signal);
|
|
658
|
+
if (this.sessionEnded || generation !== this.contextGeneration) {
|
|
659
|
+
throw new Error("Connection restoration interrupted");
|
|
660
|
+
}
|
|
661
|
+
this.resumeContextDelivery(recovering);
|
|
662
|
+
throwIfAborted(signal);
|
|
663
|
+
if (this.sessionEnded || generation !== this.contextGeneration) {
|
|
664
|
+
throw new Error("Connection restoration interrupted");
|
|
665
|
+
}
|
|
666
|
+
this.hasConnectedValue = true;
|
|
667
|
+
this.setState("connected");
|
|
668
|
+
} catch (error) {
|
|
669
|
+
if (signal?.aborted) this.rejectPendingCommands(toError(abortError(signal)));
|
|
670
|
+
this.disconnectMode = recovering ? "rejoin" : "connect_failure";
|
|
671
|
+
try {
|
|
672
|
+
await this.room.disconnect();
|
|
673
|
+
} catch {
|
|
674
|
+
} finally {
|
|
675
|
+
this.disconnectMode = null;
|
|
676
|
+
}
|
|
677
|
+
if (!recovering && markFailureTerminal) this.setState("failed");
|
|
678
|
+
throw error;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
connectRoom(signal, recovering) {
|
|
682
|
+
throwIfAborted(signal);
|
|
683
|
+
const connection = this.room.connect(this.credentials.url, this.credentials.token);
|
|
684
|
+
if (!signal) return connection;
|
|
685
|
+
return new Promise((resolve, reject) => {
|
|
686
|
+
let settled = false;
|
|
687
|
+
const resolveOnce = () => {
|
|
688
|
+
if (settled) return;
|
|
689
|
+
settled = true;
|
|
690
|
+
signal.removeEventListener("abort", onAbort);
|
|
691
|
+
resolve();
|
|
692
|
+
};
|
|
693
|
+
const rejectOnce = (error) => {
|
|
694
|
+
if (settled) return;
|
|
695
|
+
settled = true;
|
|
696
|
+
signal.removeEventListener("abort", onAbort);
|
|
697
|
+
reject(error);
|
|
698
|
+
};
|
|
699
|
+
const onAbort = () => {
|
|
700
|
+
this.disconnectMode = recovering ? "rejoin" : "connect_failure";
|
|
701
|
+
void this.room.disconnect().catch(() => {
|
|
546
702
|
});
|
|
703
|
+
rejectOnce(abortError(signal));
|
|
704
|
+
};
|
|
705
|
+
if (signal.aborted) {
|
|
706
|
+
onAbort();
|
|
707
|
+
} else {
|
|
708
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
709
|
+
}
|
|
710
|
+
connection.then(resolveOnce, rejectOnce);
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
async rejoin(apiBase = this.apiBase) {
|
|
714
|
+
if (this.startupConnectController || this.manualRejoinController) {
|
|
715
|
+
throw new Error("A session connection attempt is already in progress");
|
|
547
716
|
}
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
const blockedStates = recovering
|
|
551
|
-
? ["connecting", "connected_not_ready", "connected", "reconnecting"]
|
|
552
|
-
: ["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"];
|
|
553
|
-
const continuingStartupRetry = (!recovering
|
|
554
|
-
&& !markFailureTerminal
|
|
555
|
-
&& this.stateValue === "connecting");
|
|
556
|
-
if (blockedStates.includes(this.stateValue) && !continuingStartupRetry) {
|
|
557
|
-
throw new Error(`Cannot connect while session is ${this.stateValue}`);
|
|
558
|
-
}
|
|
559
|
-
const generation = this.contextGeneration;
|
|
560
|
-
const resolved = { ...this.connectDefaults, ...options };
|
|
561
|
-
const audioTrack = resolved.audioTrack ?? null;
|
|
562
|
-
const microphone = resolved.microphone ?? audioTrack === null;
|
|
563
|
-
const readyTimeoutMs = resolved.readyTimeoutMs ?? 15_000;
|
|
564
|
-
const waitForRemoteAudio = resolved.waitForRemoteAudio ?? true;
|
|
565
|
-
const apiBase = normalizeApiBase(resolved.apiBase ?? this.apiBase);
|
|
566
|
-
this.validateAudioOptions(microphone, audioTrack);
|
|
567
|
-
this.useManagedMicrophone = microphone;
|
|
568
|
-
this.applicationAudioTrack = audioTrack;
|
|
569
|
-
if (!recovering) {
|
|
570
|
-
this.sessionEnded = false;
|
|
571
|
-
this.agentStreamId = null;
|
|
572
|
-
this.reconnectStartedAt = null;
|
|
573
|
-
this.pendingEndEvent = null;
|
|
574
|
-
this.agentDisconnected = false;
|
|
575
|
-
this.terminalSessionError = null;
|
|
576
|
-
this.endNotificationDelivered = false;
|
|
577
|
-
this.setState("connecting");
|
|
578
|
-
}
|
|
579
|
-
else {
|
|
580
|
-
this.setState("resuming");
|
|
581
|
-
}
|
|
582
|
-
this.apiBase = apiBase;
|
|
583
|
-
this.agentReady = false;
|
|
584
|
-
try {
|
|
585
|
-
await this.connectRoom(signal, recovering);
|
|
586
|
-
throwIfAborted(signal);
|
|
587
|
-
if (audioTrack) {
|
|
588
|
-
await raceWithAbort(this.publishApplicationAudioTrack(audioTrack), signal);
|
|
589
|
-
}
|
|
590
|
-
else if (microphone) {
|
|
591
|
-
await raceWithAbort(this.room.localParticipant.setMicrophoneEnabled(this.microphoneEnabled), signal);
|
|
592
|
-
}
|
|
593
|
-
throwIfAborted(signal);
|
|
594
|
-
this.setState(recovering ? "resuming" : "connected_not_ready");
|
|
595
|
-
await raceWithAbort(this.sendHello(), signal);
|
|
596
|
-
const readyDeadline = Date.now() + readyTimeoutMs;
|
|
597
|
-
await this.waitForAgentReady(readyTimeoutMs, signal);
|
|
598
|
-
if (waitForRemoteAudio)
|
|
599
|
-
await this.waitForRemoteAudioTrack(readyDeadline, signal);
|
|
600
|
-
throwIfAborted(signal);
|
|
601
|
-
if (this.sessionEnded || generation !== this.contextGeneration) {
|
|
602
|
-
throw new Error("Connection restoration interrupted");
|
|
603
|
-
}
|
|
604
|
-
this.resumeContextDelivery(recovering);
|
|
605
|
-
throwIfAborted(signal);
|
|
606
|
-
if (this.sessionEnded || generation !== this.contextGeneration) {
|
|
607
|
-
throw new Error("Connection restoration interrupted");
|
|
608
|
-
}
|
|
609
|
-
this.hasConnectedValue = true;
|
|
610
|
-
this.setState("connected");
|
|
611
|
-
}
|
|
612
|
-
catch (error) {
|
|
613
|
-
if (signal?.aborted)
|
|
614
|
-
this.rejectPendingCommands(toError(abortError(signal)));
|
|
615
|
-
this.disconnectMode = recovering ? "rejoin" : "connect_failure";
|
|
616
|
-
try {
|
|
617
|
-
await this.room.disconnect();
|
|
618
|
-
}
|
|
619
|
-
catch {
|
|
620
|
-
// Preserve the original connection/readiness failure.
|
|
621
|
-
}
|
|
622
|
-
finally {
|
|
623
|
-
this.disconnectMode = null;
|
|
624
|
-
}
|
|
625
|
-
if (!recovering && markFailureTerminal)
|
|
626
|
-
this.setState("failed");
|
|
627
|
-
throw error;
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
connectRoom(signal, recovering) {
|
|
631
|
-
throwIfAborted(signal);
|
|
632
|
-
const connection = this.room.connect(this.credentials.url, this.credentials.token);
|
|
633
|
-
if (!signal)
|
|
634
|
-
return connection;
|
|
635
|
-
return new Promise((resolve, reject) => {
|
|
636
|
-
let settled = false;
|
|
637
|
-
const resolveOnce = () => {
|
|
638
|
-
if (settled)
|
|
639
|
-
return;
|
|
640
|
-
settled = true;
|
|
641
|
-
signal.removeEventListener("abort", onAbort);
|
|
642
|
-
resolve();
|
|
643
|
-
};
|
|
644
|
-
const rejectOnce = (error) => {
|
|
645
|
-
if (settled)
|
|
646
|
-
return;
|
|
647
|
-
settled = true;
|
|
648
|
-
signal.removeEventListener("abort", onAbort);
|
|
649
|
-
reject(error);
|
|
650
|
-
};
|
|
651
|
-
const onAbort = () => {
|
|
652
|
-
this.disconnectMode = recovering ? "rejoin" : "connect_failure";
|
|
653
|
-
void this.room.disconnect().catch(() => { });
|
|
654
|
-
rejectOnce(abortError(signal));
|
|
655
|
-
};
|
|
656
|
-
if (signal.aborted) {
|
|
657
|
-
onAbort();
|
|
658
|
-
}
|
|
659
|
-
else {
|
|
660
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
661
|
-
}
|
|
662
|
-
connection.then(resolveOnce, rejectOnce);
|
|
663
|
-
});
|
|
717
|
+
if (this.sessionEnded) {
|
|
718
|
+
throw this.terminalSessionError ?? new Error("Cannot rejoin a session that has ended");
|
|
664
719
|
}
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
throw new Error("A session connection attempt is already in progress");
|
|
668
|
-
}
|
|
669
|
-
if (this.sessionEnded) {
|
|
670
|
-
throw this.terminalSessionError ?? new Error("Cannot rejoin a session that has ended");
|
|
671
|
-
}
|
|
672
|
-
if (["connecting", "connected_not_ready", "reconnecting", "resuming"].includes(this.stateValue)) {
|
|
673
|
-
throw new Error(`Cannot rejoin while session is ${this.stateValue}`);
|
|
674
|
-
}
|
|
675
|
-
this.cancelRecovery();
|
|
676
|
-
const normalizedApiBase = normalizeApiBase(apiBase);
|
|
677
|
-
const recovering = this.hasConnectedValue;
|
|
678
|
-
const controller = new AbortController();
|
|
679
|
-
this.manualRejoinController = controller;
|
|
680
|
-
if (recovering) {
|
|
681
|
-
this.beginRecovery("rejoin", 1, { trigger: "manual_rejoin" });
|
|
682
|
-
this.setState("resuming");
|
|
683
|
-
}
|
|
684
|
-
try {
|
|
685
|
-
await this.performRejoin(normalizedApiBase, { recovering, signal: controller.signal });
|
|
686
|
-
if (this.sessionEnded)
|
|
687
|
-
return;
|
|
688
|
-
if (recovering)
|
|
689
|
-
this.completeRecovery();
|
|
690
|
-
}
|
|
691
|
-
catch (error) {
|
|
692
|
-
if (controller.signal.aborted && !this.terminalSessionError)
|
|
693
|
-
return;
|
|
694
|
-
const normalizedError = toError(error);
|
|
695
|
-
if (recovering)
|
|
696
|
-
this.failRecovery(normalizedError);
|
|
697
|
-
this.setState("failed");
|
|
698
|
-
throw error;
|
|
699
|
-
}
|
|
700
|
-
finally {
|
|
701
|
-
if (this.manualRejoinController === controller) {
|
|
702
|
-
this.manualRejoinController = null;
|
|
703
|
-
}
|
|
704
|
-
}
|
|
720
|
+
if (["connecting", "connected_not_ready", "reconnecting", "resuming"].includes(this.stateValue)) {
|
|
721
|
+
throw new Error(`Cannot rejoin while session is ${this.stateValue}`);
|
|
705
722
|
}
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
throwIfAborted(signal);
|
|
715
|
-
response = await this.fetchRoomToken(apiBase, signal);
|
|
716
|
-
}
|
|
717
|
-
// A live ending may arrive while token renewal is in flight.
|
|
718
|
-
if (this.sessionEnded) {
|
|
719
|
-
if (this.terminalSessionError)
|
|
720
|
-
throw this.terminalSessionError;
|
|
721
|
-
return;
|
|
722
|
-
}
|
|
723
|
-
if (!response.ok) {
|
|
724
|
-
if (response.status === 409 && await this.recoverSessionEnd(apiBase, signal))
|
|
725
|
-
return;
|
|
726
|
-
throw await raceWithAbort(httpError(response, "Could not renew room token"), signal);
|
|
727
|
-
}
|
|
728
|
-
const payload = (await raceWithAbort(response.json(), signal));
|
|
729
|
-
throwIfAborted(signal);
|
|
730
|
-
this.credentials = payload.connection;
|
|
731
|
-
if (this.room.state !== LiveKitConnectionState.Disconnected) {
|
|
732
|
-
this.disconnectMode = "rejoin";
|
|
733
|
-
try {
|
|
734
|
-
await raceWithAbort(this.room.disconnect(), signal);
|
|
735
|
-
}
|
|
736
|
-
finally {
|
|
737
|
-
this.disconnectMode = null;
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
throwIfAborted(signal);
|
|
741
|
-
if (recovering && this.reconnectStartedAt !== null) {
|
|
742
|
-
this.pendingReconnect = {
|
|
743
|
-
started_at: new Date(this.reconnectStartedAt).toISOString(),
|
|
744
|
-
recovered_at: new Date().toISOString(),
|
|
745
|
-
duration_ms: Date.now() - this.reconnectStartedAt,
|
|
746
|
-
};
|
|
747
|
-
}
|
|
748
|
-
// Startup retries resolve the original caller options with constructor defaults.
|
|
749
|
-
// A later rejoin restores the audio settings saved by the previous connection.
|
|
750
|
-
const connectionOptions = connect ?? {
|
|
751
|
-
...this.recoveryReadiness,
|
|
752
|
-
microphone: this.useManagedMicrophone,
|
|
753
|
-
...(this.applicationAudioTrack ? { audioTrack: this.applicationAudioTrack } : {}),
|
|
754
|
-
};
|
|
755
|
-
await this.connectWithMode({ ...connectionOptions, apiBase }, recovering, signal, markFailureTerminal);
|
|
756
|
-
}
|
|
757
|
-
async setMicrophoneEnabled(enabled) {
|
|
758
|
-
// Recovery must see the user's latest intent while the media operation waits.
|
|
759
|
-
this.microphoneEnabled = enabled;
|
|
760
|
-
if (this.applicationAudioTrack) {
|
|
761
|
-
// The room may not know this track yet while publication is pending.
|
|
762
|
-
const track = this.localApplicationAudioTrack;
|
|
763
|
-
if (enabled)
|
|
764
|
-
await track?.unmute();
|
|
765
|
-
else
|
|
766
|
-
await track?.mute();
|
|
767
|
-
return;
|
|
768
|
-
}
|
|
769
|
-
await this.room.localParticipant.setMicrophoneEnabled(enabled);
|
|
723
|
+
this.cancelRecovery();
|
|
724
|
+
const normalizedApiBase = normalizeApiBase(apiBase);
|
|
725
|
+
const recovering = this.hasConnectedValue;
|
|
726
|
+
const controller = new AbortController();
|
|
727
|
+
this.manualRejoinController = controller;
|
|
728
|
+
if (recovering) {
|
|
729
|
+
this.beginRecovery("rejoin", 1, { trigger: "manual_rejoin" });
|
|
730
|
+
this.setState("resuming");
|
|
770
731
|
}
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
732
|
+
try {
|
|
733
|
+
await this.performRejoin(normalizedApiBase, { recovering, signal: controller.signal });
|
|
734
|
+
if (this.sessionEnded) return;
|
|
735
|
+
if (recovering) this.completeRecovery();
|
|
736
|
+
} catch (error) {
|
|
737
|
+
if (controller.signal.aborted && !this.terminalSessionError) return;
|
|
738
|
+
const normalizedError = toError(error);
|
|
739
|
+
if (recovering) this.failRecovery(normalizedError);
|
|
740
|
+
this.setState("failed");
|
|
741
|
+
throw error;
|
|
742
|
+
} finally {
|
|
743
|
+
if (this.manualRejoinController === controller) {
|
|
744
|
+
this.manualRejoinController = null;
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
async performRejoin(apiBase, options = {}) {
|
|
749
|
+
const {
|
|
750
|
+
signal,
|
|
751
|
+
recovering = true,
|
|
752
|
+
connect,
|
|
753
|
+
markFailureTerminal = true
|
|
754
|
+
} = options;
|
|
755
|
+
this.contextGeneration += 1;
|
|
756
|
+
this.replacementContext.pause();
|
|
757
|
+
this.rejectPendingCommands(new Error("Restoring the session"));
|
|
758
|
+
let response = await this.fetchRoomToken(apiBase, signal, recovering);
|
|
759
|
+
if (response.status === 401 && this.refreshCredentialsCallback) {
|
|
760
|
+
await this.refreshCredentials(signal);
|
|
761
|
+
throwIfAborted(signal);
|
|
762
|
+
response = await this.fetchRoomToken(apiBase, signal, recovering);
|
|
763
|
+
}
|
|
764
|
+
if (this.sessionEnded) {
|
|
765
|
+
if (this.terminalSessionError) throw this.terminalSessionError;
|
|
766
|
+
return;
|
|
767
|
+
}
|
|
768
|
+
if (!response.ok) {
|
|
769
|
+
if (response.status === 409 && await this.recoverSessionEnd(apiBase, signal)) return;
|
|
770
|
+
throw await raceWithAbort(httpError(response, "Could not renew room token"), signal);
|
|
771
|
+
}
|
|
772
|
+
const payload = await raceWithAbort(response.json(), signal);
|
|
773
|
+
throwIfAborted(signal);
|
|
774
|
+
this.credentials = payload.connection;
|
|
775
|
+
if (this.room.state !== LiveKitConnectionState.Disconnected) {
|
|
776
|
+
this.disconnectMode = "rejoin";
|
|
777
|
+
try {
|
|
778
|
+
await this.measureRecoveryStage(
|
|
779
|
+
"room_disconnect",
|
|
780
|
+
() => raceWithAbort(this.room.disconnect(), signal),
|
|
781
|
+
recovering
|
|
782
|
+
);
|
|
783
|
+
} finally {
|
|
784
|
+
this.disconnectMode = null;
|
|
785
|
+
}
|
|
774
786
|
}
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
787
|
+
throwIfAborted(signal);
|
|
788
|
+
if (recovering && this.reconnectStartedAt !== null) {
|
|
789
|
+
this.pendingReconnect = {
|
|
790
|
+
started_at: new Date(this.reconnectStartedAt).toISOString(),
|
|
791
|
+
recovered_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
792
|
+
duration_ms: Date.now() - this.reconnectStartedAt
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
const connectionOptions = connect ?? {
|
|
796
|
+
...this.recoveryReadiness,
|
|
797
|
+
microphone: this.useManagedMicrophone,
|
|
798
|
+
...this.applicationAudioTrack ? { audioTrack: this.applicationAudioTrack } : {}
|
|
799
|
+
};
|
|
800
|
+
await this.connectWithMode({ ...connectionOptions, apiBase }, recovering, signal, markFailureTerminal);
|
|
801
|
+
}
|
|
802
|
+
async setMicrophoneEnabled(enabled) {
|
|
803
|
+
this.microphoneEnabled = enabled;
|
|
804
|
+
if (this.applicationAudioTrack) {
|
|
805
|
+
const track = this.localApplicationAudioTrack;
|
|
806
|
+
if (enabled) await track?.unmute();
|
|
807
|
+
else await track?.mute();
|
|
808
|
+
return;
|
|
809
|
+
}
|
|
810
|
+
await this.room.localParticipant.setMicrophoneEnabled(enabled);
|
|
811
|
+
}
|
|
812
|
+
async sendCommand(type, payload, timeoutMs = 1e4) {
|
|
813
|
+
const message = createClientMessage(
|
|
814
|
+
this.connectionId,
|
|
815
|
+
++this.clientSeq,
|
|
816
|
+
type,
|
|
817
|
+
payload
|
|
818
|
+
);
|
|
819
|
+
return this.sendTrackedCommand(message, timeoutMs);
|
|
820
|
+
}
|
|
821
|
+
async updateContext(update, timeoutMs = 2e4) {
|
|
822
|
+
if (this.sessionEnded || ["failed", "disconnected"].includes(this.stateValue)) {
|
|
823
|
+
throw new Error("Cannot update context while the session is inactive");
|
|
824
|
+
}
|
|
825
|
+
if (update.mode === "replace") return this.replacementContext.submit(update, timeoutMs);
|
|
826
|
+
if (this.activeRecovery) throw new Error("Cannot append context during recovery");
|
|
827
|
+
return this.sendContextUpdate(update, timeoutMs);
|
|
828
|
+
}
|
|
829
|
+
async sendContextUpdate(update, timeoutMs) {
|
|
830
|
+
const generation = this.contextGeneration;
|
|
831
|
+
const payloadBytes = contextUpdateBytes(update);
|
|
832
|
+
const transferStartedAt = Date.now();
|
|
833
|
+
let transferTimedOut = false;
|
|
834
|
+
this.recordDiagnostic({
|
|
835
|
+
operation: "context_transfer",
|
|
836
|
+
status: "started",
|
|
837
|
+
context_key: update.key,
|
|
838
|
+
payload_bytes: payloadBytes
|
|
839
|
+
});
|
|
840
|
+
this.activeContextTransfers += 1;
|
|
841
|
+
this.activeContextBytes += payloadBytes;
|
|
842
|
+
const uploading = ("image" in update ? this.uploadImageContext(update) : this.uploadTextContext(update)).finally(() => {
|
|
843
|
+
this.activeContextTransfers -= 1;
|
|
844
|
+
this.activeContextBytes -= payloadBytes;
|
|
845
|
+
});
|
|
846
|
+
const transferSettled = uploading.then(() => {
|
|
847
|
+
}, () => {
|
|
848
|
+
});
|
|
849
|
+
let transferTimeout;
|
|
850
|
+
const transferExpired = new Promise((_resolve, reject) => {
|
|
851
|
+
transferTimeout = globalThis.setTimeout(() => {
|
|
852
|
+
transferTimedOut = true;
|
|
853
|
+
const error = new ContextDeliveryError(`Context transfer timed out after ${timeoutMs}ms`, {
|
|
854
|
+
retryAfter: transferSettled
|
|
823
855
|
});
|
|
824
|
-
let transfer;
|
|
825
|
-
try {
|
|
826
|
-
transfer = await Promise.race([
|
|
827
|
-
uploading,
|
|
828
|
-
transferExpired,
|
|
829
|
-
]);
|
|
830
|
-
}
|
|
831
|
-
catch (error) {
|
|
832
|
-
if (!transferTimedOut) {
|
|
833
|
-
this.recordDiagnostic({
|
|
834
|
-
operation: "context_transfer",
|
|
835
|
-
status: "failed",
|
|
836
|
-
reason: "context_transfer_failed",
|
|
837
|
-
context_key: update.key,
|
|
838
|
-
payload_bytes: payloadBytes,
|
|
839
|
-
duration_ms: Date.now() - transferStartedAt,
|
|
840
|
-
});
|
|
841
|
-
this.flushDiagnostics("command_failed");
|
|
842
|
-
}
|
|
843
|
-
if (error instanceof TypeError || error instanceof ContextDeliveryError)
|
|
844
|
-
throw error;
|
|
845
|
-
throw new ContextDeliveryError(toError(error).message, { cause: error });
|
|
846
|
-
}
|
|
847
|
-
finally {
|
|
848
|
-
globalThis.clearTimeout(transferTimeout);
|
|
849
|
-
}
|
|
850
|
-
const transferDurationMs = Date.now() - transferStartedAt;
|
|
851
856
|
this.recordDiagnostic({
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
this.flushDiagnostics("slow_operation");
|
|
860
|
-
if (generation !== this.contextGeneration)
|
|
861
|
-
throw new Error("Context transfer interrupted by recovery");
|
|
862
|
-
const { payload } = transfer;
|
|
863
|
-
if (!isContextUpdatePayload(payload)) {
|
|
864
|
-
throw new TypeError("Invalid context update payload");
|
|
865
|
-
}
|
|
866
|
-
const message = createClientMessage(this.connectionId, ++this.clientSeq, "context.update", { ...payload });
|
|
867
|
-
return this.sendTrackedCommand(message, timeoutMs, transfer.resend);
|
|
868
|
-
}
|
|
869
|
-
resumeContextDelivery(restore) {
|
|
870
|
-
// Audio readiness owns connection recovery. Context delivery has its own
|
|
871
|
-
// deadlines and retries, and may be interrupted by a later disconnect.
|
|
872
|
-
void this.replacementContext.resume(restore).catch(() => { });
|
|
873
|
-
}
|
|
874
|
-
async uploadTextContext(update) {
|
|
875
|
-
const data = new TextEncoder().encode(update.text);
|
|
876
|
-
if (data.byteLength === 0)
|
|
877
|
-
throw new TypeError("Context text cannot be empty");
|
|
878
|
-
if (data.byteLength > 262_144) {
|
|
879
|
-
throw new TypeError("Context text must be at most 262144 UTF-8 bytes");
|
|
880
|
-
}
|
|
881
|
-
const base = {
|
|
882
|
-
key: update.key,
|
|
883
|
-
mode: update.mode,
|
|
884
|
-
label: update.label ?? "",
|
|
885
|
-
respond: update.respond ?? false,
|
|
886
|
-
};
|
|
887
|
-
if (data.byteLength <= 32_768)
|
|
888
|
-
return { payload: { ...base, text: update.text } };
|
|
889
|
-
const sha256 = await sha256Hex(data);
|
|
890
|
-
const streamId = contextStreamId();
|
|
891
|
-
const send = async () => {
|
|
892
|
-
const writer = await this.room.localParticipant.streamText({
|
|
893
|
-
streamId,
|
|
894
|
-
topic: "voxevolv.context.text",
|
|
895
|
-
totalSize: data.byteLength,
|
|
896
|
-
attributes: { sha256 },
|
|
897
|
-
});
|
|
898
|
-
await writer.write(update.text);
|
|
899
|
-
await writer.close();
|
|
900
|
-
};
|
|
901
|
-
await send();
|
|
902
|
-
return {
|
|
903
|
-
payload: {
|
|
904
|
-
...base,
|
|
905
|
-
text_stream: { stream_id: streamId, bytes: data.byteLength, sha256 },
|
|
906
|
-
},
|
|
907
|
-
resend: send,
|
|
908
|
-
};
|
|
909
|
-
}
|
|
910
|
-
async uploadImageContext(update) {
|
|
911
|
-
const allowed = new Set(["image/jpeg", "image/png", "image/webp"]);
|
|
912
|
-
if (!allowed.has(update.image.type)) {
|
|
913
|
-
throw new TypeError("Image context must be JPEG, PNG, or WebP");
|
|
914
|
-
}
|
|
915
|
-
const data = new Uint8Array(await update.image.arrayBuffer());
|
|
916
|
-
if (data.byteLength === 0 || data.byteLength > 5_242_880) {
|
|
917
|
-
throw new TypeError("Image context must be between 1 byte and 5 MiB");
|
|
918
|
-
}
|
|
919
|
-
const sha256 = await sha256Hex(data);
|
|
920
|
-
const streamId = contextStreamId();
|
|
921
|
-
const send = async () => {
|
|
922
|
-
const writer = await this.room.localParticipant.streamBytes({
|
|
923
|
-
streamId,
|
|
924
|
-
topic: "voxevolv.context.image",
|
|
925
|
-
name: "context-image",
|
|
926
|
-
mimeType: update.image.type,
|
|
927
|
-
totalSize: data.byteLength,
|
|
928
|
-
attributes: { sha256 },
|
|
929
|
-
});
|
|
930
|
-
await writer.write(data);
|
|
931
|
-
await writer.close();
|
|
932
|
-
};
|
|
933
|
-
await send();
|
|
934
|
-
return {
|
|
935
|
-
payload: {
|
|
936
|
-
key: update.key,
|
|
937
|
-
mode: "replace",
|
|
938
|
-
label: update.label ?? "",
|
|
939
|
-
image: {
|
|
940
|
-
stream_id: streamId,
|
|
941
|
-
mime: update.image.type,
|
|
942
|
-
bytes: data.byteLength,
|
|
943
|
-
sha256,
|
|
944
|
-
...(update.width === undefined ? {} : { width: update.width }),
|
|
945
|
-
...(update.height === undefined ? {} : { height: update.height }),
|
|
946
|
-
detail: update.detail ?? "low",
|
|
947
|
-
},
|
|
948
|
-
respond: update.respond ?? false,
|
|
949
|
-
},
|
|
950
|
-
resend: send,
|
|
951
|
-
};
|
|
952
|
-
}
|
|
953
|
-
async disconnect() {
|
|
954
|
-
this.abortStartupConnect(new Error("Session disconnected"));
|
|
955
|
-
this.startupConnectController = null;
|
|
956
|
-
this.failRecovery(new Error("Session disconnected during recovery"));
|
|
957
|
-
this.cancelRecovery();
|
|
958
|
-
this.contextGeneration += 1;
|
|
959
|
-
this.localApplicationAudioTrack = null;
|
|
960
|
-
this.replacementContext.clear(new Error("Session disconnected"));
|
|
961
|
-
if (this.stateValue === "disconnected")
|
|
962
|
-
return;
|
|
963
|
-
this.rejectPendingCommands(new Error("Session disconnected before command completion"));
|
|
964
|
-
this.disconnectMode = "intentional";
|
|
965
|
-
try {
|
|
966
|
-
await this.room.disconnect();
|
|
967
|
-
}
|
|
968
|
-
finally {
|
|
969
|
-
this.disconnectMode = null;
|
|
970
|
-
this.activeRecovery = null;
|
|
971
|
-
this.setState("disconnected");
|
|
972
|
-
}
|
|
973
|
-
}
|
|
974
|
-
async end(options = {}) {
|
|
975
|
-
this.credentialRefreshGeneration += 1;
|
|
976
|
-
this.abortStartupConnect(new Error("Session ended"));
|
|
977
|
-
this.startupConnectController = null;
|
|
978
|
-
this.failRecovery(new Error("Session ended during recovery"));
|
|
979
|
-
this.cancelRecovery();
|
|
980
|
-
this.sessionEnded = true;
|
|
981
|
-
const endpoint = options.endpoint ?? this.endEndpoint ?? `${this.apiBase}/sessions/${this.id}`;
|
|
982
|
-
const request = options.request ?? {};
|
|
983
|
-
const headers = new Headers(request.headers);
|
|
984
|
-
const usesScopedSessionEndpoint = !options.endpoint && !this.endEndpoint;
|
|
985
|
-
if (!headers.has("Authorization") && usesScopedSessionEndpoint) {
|
|
986
|
-
headers.set("Authorization", `Bearer ${this.accessToken}`);
|
|
987
|
-
}
|
|
988
|
-
let response = await this.fetcher(endpoint, {
|
|
989
|
-
...request,
|
|
990
|
-
method: request.method ?? "DELETE",
|
|
991
|
-
headers,
|
|
992
|
-
credentials: request.credentials ?? this.lifecycleCredentials,
|
|
993
|
-
});
|
|
994
|
-
if (response.status === 401 && usesScopedSessionEndpoint && this.refreshCredentialsCallback) {
|
|
995
|
-
await this.refreshCredentials(request.signal ?? undefined);
|
|
996
|
-
headers.set("Authorization", `Bearer ${this.accessToken}`);
|
|
997
|
-
response = await this.fetcher(endpoint, {
|
|
998
|
-
...request,
|
|
999
|
-
method: request.method ?? "DELETE",
|
|
1000
|
-
headers,
|
|
1001
|
-
credentials: request.credentials ?? this.lifecycleCredentials,
|
|
1002
|
-
});
|
|
1003
|
-
}
|
|
1004
|
-
if (!response.ok)
|
|
1005
|
-
throw await httpError(response, "Could not end realtime session");
|
|
1006
|
-
if (options.disconnect ?? true)
|
|
1007
|
-
await this.disconnect();
|
|
1008
|
-
}
|
|
1009
|
-
async replay(apiBase = this.apiBase, afterSeq = this.cursor.lastServerSeq) {
|
|
1010
|
-
if (this.replayPromise)
|
|
1011
|
-
return this.replayPromise;
|
|
1012
|
-
const normalizedApiBase = normalizeApiBase(apiBase);
|
|
1013
|
-
this.replayPromise = this.fetchReplay(normalizedApiBase, afterSeq).finally(() => {
|
|
1014
|
-
this.replayPromise = null;
|
|
1015
|
-
});
|
|
1016
|
-
return this.replayPromise;
|
|
1017
|
-
}
|
|
1018
|
-
setState(state) {
|
|
1019
|
-
if (this.stateValue === state)
|
|
1020
|
-
return;
|
|
1021
|
-
this.stateValue = state;
|
|
1022
|
-
if (state === "connecting" || state === "resuming") {
|
|
1023
|
-
this.startDiagnosticLoopMonitor();
|
|
1024
|
-
globalThis.document?.addEventListener?.("visibilitychange", this.onDiagnosticVisibilityChange);
|
|
1025
|
-
}
|
|
1026
|
-
if (state === "disconnected" || state === "failed") {
|
|
1027
|
-
globalThis.clearInterval(this.diagnosticLoopTimer);
|
|
1028
|
-
this.diagnosticLoopTimer = undefined;
|
|
1029
|
-
this.diagnosticLoopDelayMs = undefined;
|
|
1030
|
-
globalThis.document?.removeEventListener?.("visibilitychange", this.onDiagnosticVisibilityChange);
|
|
1031
|
-
this.flushDiagnostics("session_finished");
|
|
1032
|
-
const error = new Error(`Session ${state}`);
|
|
1033
|
-
this.contextGeneration += 1;
|
|
1034
|
-
this.replacementContext.clear(error);
|
|
1035
|
-
this.rejectPendingCommands(error);
|
|
1036
|
-
}
|
|
1037
|
-
this.notifyListeners(this.stateListeners, state);
|
|
1038
|
-
}
|
|
1039
|
-
bindRoomEvents() {
|
|
1040
|
-
this.room.on(RoomEvent.ParticipantDisconnected, (participant) => {
|
|
1041
|
-
if (participant.isAgent && !this.disconnectMode) {
|
|
1042
|
-
this.agentDisconnected = true;
|
|
1043
|
-
if (this.terminalSessionError)
|
|
1044
|
-
return;
|
|
1045
|
-
if (this.sessionEnded) {
|
|
1046
|
-
this.finishPendingEnd();
|
|
1047
|
-
return;
|
|
1048
|
-
}
|
|
1049
|
-
this.startAutoRejoin(new Error("Agent disconnected from the room"), { trigger: "agent_disconnected" });
|
|
1050
|
-
}
|
|
1051
|
-
});
|
|
1052
|
-
this.room.on(RoomEvent.SignalReconnecting, () => this.handleLiveKitReconnecting());
|
|
1053
|
-
this.room.on(RoomEvent.Reconnecting, () => this.handleLiveKitReconnecting());
|
|
1054
|
-
this.room.on(RoomEvent.Reconnected, () => {
|
|
1055
|
-
void this.handleLiveKitReconnected();
|
|
1056
|
-
});
|
|
1057
|
-
this.room.on(RoomEvent.Disconnected, (reason) => {
|
|
1058
|
-
this.handleLiveKitDisconnected(reason);
|
|
1059
|
-
});
|
|
1060
|
-
this.room.on(RoomEvent.ConnectionStateChanged, (state) => {
|
|
1061
|
-
if (this.sessionEnded || this.terminalSessionError)
|
|
1062
|
-
return;
|
|
1063
|
-
if (state === LiveKitConnectionState.Connected) {
|
|
1064
|
-
const recovering = this.activeRecovery !== null || this.stateValue === "resuming";
|
|
1065
|
-
this.setState(this.agentReady
|
|
1066
|
-
? "connected"
|
|
1067
|
-
: recovering
|
|
1068
|
-
? "resuming"
|
|
1069
|
-
: "connected_not_ready");
|
|
1070
|
-
}
|
|
1071
|
-
});
|
|
1072
|
-
this.room.on(RoomEvent.ConnectionQualityChanged, (quality, participant) => {
|
|
1073
|
-
if (!participant.isLocal)
|
|
1074
|
-
return;
|
|
1075
|
-
this.notifyListeners(this.connectionQualityListeners, quality);
|
|
1076
|
-
});
|
|
1077
|
-
this.room.on(RoomEvent.TrackSubscribed, (track) => {
|
|
1078
|
-
if (track.kind !== Track.Kind.Audio)
|
|
1079
|
-
return;
|
|
1080
|
-
const element = track.attach();
|
|
1081
|
-
element.dataset.voxevolvRemoteAudio = "true";
|
|
1082
|
-
element.style.display = "none";
|
|
1083
|
-
document.body.appendChild(element);
|
|
1084
|
-
try {
|
|
1085
|
-
const playback = element.play();
|
|
1086
|
-
void playback.catch((error) => {
|
|
1087
|
-
this.notifyListeners(this.audioPlaybackBlockedListeners, {
|
|
1088
|
-
error,
|
|
1089
|
-
element,
|
|
1090
|
-
track: track.mediaStreamTrack,
|
|
1091
|
-
});
|
|
1092
|
-
});
|
|
1093
|
-
}
|
|
1094
|
-
catch (error) {
|
|
1095
|
-
this.notifyListeners(this.audioPlaybackBlockedListeners, {
|
|
1096
|
-
error,
|
|
1097
|
-
element,
|
|
1098
|
-
track: track.mediaStreamTrack,
|
|
1099
|
-
});
|
|
1100
|
-
}
|
|
1101
|
-
const stream = new MediaStream([track.mediaStreamTrack]);
|
|
1102
|
-
this.remoteAudioStreams.set(track, stream);
|
|
1103
|
-
this.outputStream.addTrack(track.mediaStreamTrack);
|
|
1104
|
-
this.notifyRemoteAudioListeners(this.remoteAudioTrackListeners, track.mediaStreamTrack, stream);
|
|
1105
|
-
this.remoteAudioTrackResolve?.();
|
|
1106
|
-
});
|
|
1107
|
-
this.room.on(RoomEvent.TrackUnsubscribed, (track) => {
|
|
1108
|
-
if (track.kind !== Track.Kind.Audio)
|
|
1109
|
-
return;
|
|
1110
|
-
const stream = this.remoteAudioStreams.get(track) ?? new MediaStream([track.mediaStreamTrack]);
|
|
1111
|
-
this.notifyRemoteAudioListeners(this.remoteAudioTrackUnsubscribedListeners, track.mediaStreamTrack, stream);
|
|
1112
|
-
this.remoteAudioStreams.delete(track);
|
|
1113
|
-
this.outputMediaStream?.removeTrack(track.mediaStreamTrack);
|
|
1114
|
-
track.detach().forEach((element) => element.remove());
|
|
1115
|
-
});
|
|
1116
|
-
this.room.on(RoomEvent.DataReceived, (data, _participant, kind, topic) => {
|
|
1117
|
-
if (topic !== "voxevolv.events" || kind !== DataPacket_Kind.RELIABLE)
|
|
1118
|
-
return;
|
|
1119
|
-
let value;
|
|
1120
|
-
try {
|
|
1121
|
-
value = JSON.parse(new TextDecoder().decode(data));
|
|
1122
|
-
}
|
|
1123
|
-
catch {
|
|
1124
|
-
// Ignore data from other room participants that is not a VoxEvolv envelope.
|
|
1125
|
-
return;
|
|
1126
|
-
}
|
|
1127
|
-
if (!isRealtimeEvent(value))
|
|
1128
|
-
return;
|
|
1129
|
-
const result = this.cursor.accept(value);
|
|
1130
|
-
if (result.gap) {
|
|
1131
|
-
this.notifyListeners(this.gapListeners, result.gap);
|
|
1132
|
-
void this.replay(this.apiBase, result.gap.after_server_seq);
|
|
1133
|
-
}
|
|
1134
|
-
if (!result.deliver)
|
|
1135
|
-
return;
|
|
1136
|
-
this.observeReadiness(value);
|
|
1137
|
-
this.observeCommandResult(value);
|
|
1138
|
-
this.observeConvenienceEvents(value);
|
|
1139
|
-
this.notifyListeners(this.listeners, value);
|
|
857
|
+
operation: "context_transfer",
|
|
858
|
+
status: "timed_out",
|
|
859
|
+
reason: "context_transfer_timeout",
|
|
860
|
+
context_key: update.key,
|
|
861
|
+
payload_bytes: payloadBytes,
|
|
862
|
+
duration_ms: Date.now() - transferStartedAt,
|
|
863
|
+
timeout_ms: timeoutMs
|
|
1140
864
|
});
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
return;
|
|
1154
|
-
}
|
|
1155
|
-
this.contextGeneration += 1;
|
|
1156
|
-
this.replacementContext.pause();
|
|
1157
|
-
this.reconnectStartedAt ??= Date.now();
|
|
1158
|
-
this.rejectPendingCommands(new Error("Connection interrupted before command completion"), true);
|
|
1159
|
-
if (this.activeRecovery === null) {
|
|
1160
|
-
this.beginRecovery("livekit", 1, { trigger: "livekit_reconnecting" });
|
|
1161
|
-
}
|
|
1162
|
-
this.setState("reconnecting");
|
|
1163
|
-
}
|
|
1164
|
-
async handleLiveKitReconnected() {
|
|
1165
|
-
if (this.sessionEnded || this.disconnectMode || this.autoRejoinPromise)
|
|
1166
|
-
return;
|
|
1167
|
-
const generation = this.contextGeneration;
|
|
1168
|
-
const recoveredAt = Date.now();
|
|
1169
|
-
if (this.reconnectStartedAt !== null) {
|
|
1170
|
-
this.pendingReconnect = {
|
|
1171
|
-
started_at: new Date(this.reconnectStartedAt).toISOString(),
|
|
1172
|
-
recovered_at: new Date(recoveredAt).toISOString(),
|
|
1173
|
-
duration_ms: Math.max(recoveredAt - this.reconnectStartedAt, 0),
|
|
1174
|
-
};
|
|
1175
|
-
this.reconnectStartedAt = null;
|
|
1176
|
-
}
|
|
1177
|
-
if (this.hasConnectedValue && this.activeRecovery === null) {
|
|
1178
|
-
this.beginRecovery("livekit", 1, { trigger: "livekit_reconnecting" });
|
|
1179
|
-
}
|
|
1180
|
-
this.connectionId = `conn_${Math.random().toString(36).slice(2)}_${Date.now().toString(36)}`;
|
|
1181
|
-
this.clientSeq = 0;
|
|
1182
|
-
this.setState(this.hasConnectedValue ? "resuming" : "connected_not_ready");
|
|
1183
|
-
try {
|
|
1184
|
-
await this.sendHello();
|
|
1185
|
-
const deadline = Date.now() + this.recoveryReadiness.readyTimeoutMs;
|
|
1186
|
-
await this.waitForAgentReady(Math.max(deadline - Date.now(), 0));
|
|
1187
|
-
if (this.recoveryReadiness.waitForRemoteAudio)
|
|
1188
|
-
await this.waitForRemoteAudioTrack(deadline);
|
|
1189
|
-
if (this.sessionEnded || generation !== this.contextGeneration)
|
|
1190
|
-
return;
|
|
1191
|
-
this.resumeContextDelivery(true);
|
|
1192
|
-
if (this.sessionEnded || generation !== this.contextGeneration)
|
|
1193
|
-
return;
|
|
1194
|
-
this.setState("connected");
|
|
1195
|
-
this.completeRecovery();
|
|
1196
|
-
}
|
|
1197
|
-
catch (error) {
|
|
1198
|
-
if (this.sessionEnded || generation !== this.contextGeneration)
|
|
1199
|
-
return;
|
|
1200
|
-
const normalizedError = toError(error);
|
|
1201
|
-
if (this.shouldAutoRejoin()) {
|
|
1202
|
-
this.startAutoRejoin(normalizedError, { trigger: "livekit_resume_failed" });
|
|
1203
|
-
}
|
|
1204
|
-
else {
|
|
1205
|
-
this.failRecovery(normalizedError);
|
|
1206
|
-
this.setState("failed");
|
|
1207
|
-
}
|
|
1208
|
-
}
|
|
1209
|
-
}
|
|
1210
|
-
handleLiveKitDisconnected(reason) {
|
|
1211
|
-
this.agentReady = false;
|
|
1212
|
-
if (this.disconnectMode !== "rejoin") {
|
|
1213
|
-
this.pendingReconnect = null;
|
|
1214
|
-
}
|
|
1215
|
-
// Drop any remaining remote tracks so a stale, ended track cannot
|
|
1216
|
-
// satisfy the next connect()'s wait for agent audio.
|
|
1217
|
-
this.outputMediaStream?.getTracks().forEach((track) => {
|
|
1218
|
-
this.outputMediaStream?.removeTrack(track);
|
|
1219
|
-
});
|
|
1220
|
-
this.rejectPendingCommands(new Error("Session disconnected before command completion"));
|
|
1221
|
-
if (this.disconnectMode === "rejoin")
|
|
1222
|
-
return;
|
|
1223
|
-
if (this.disconnectMode === "connect_failure")
|
|
1224
|
-
return;
|
|
1225
|
-
if (this.disconnectMode === "intentional") {
|
|
1226
|
-
this.activeRecovery = null;
|
|
1227
|
-
this.deliverPendingEndEvent();
|
|
1228
|
-
this.setState("disconnected");
|
|
1229
|
-
return;
|
|
1230
|
-
}
|
|
1231
|
-
if (this.terminalSessionError)
|
|
1232
|
-
return;
|
|
1233
|
-
if (this.shouldAutoRejoin(reason)) {
|
|
1234
|
-
this.startAutoRejoin(new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`), { trigger: "livekit_disconnected" });
|
|
1235
|
-
return;
|
|
1236
|
-
}
|
|
1237
|
-
if (this.activeRecovery !== null) {
|
|
1238
|
-
this.failRecovery(new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`));
|
|
1239
|
-
}
|
|
1240
|
-
this.deliverPendingEndEvent();
|
|
1241
|
-
this.setState("disconnected");
|
|
1242
|
-
}
|
|
1243
|
-
// The terminal session.ended event is recorded durably after room teardown,
|
|
1244
|
-
// so turn the earlier live announcement into the browser completion signal.
|
|
1245
|
-
deliverPendingEndEvent() {
|
|
1246
|
-
const ending = this.pendingEndEvent;
|
|
1247
|
-
if (!ending)
|
|
1248
|
-
return;
|
|
1249
|
-
this.pendingEndEvent = null;
|
|
1250
|
-
this.notifyEnded({ ...ending, type: "session.ended" });
|
|
1251
|
-
}
|
|
1252
|
-
finishPendingEnd() {
|
|
1253
|
-
this.deliverPendingEndEvent();
|
|
1254
|
-
// The agent can leave while the browser remains in the room. Close that
|
|
1255
|
-
// transport ourselves, after notifying the application of the true reason.
|
|
1256
|
-
void this.disconnect().catch(() => { });
|
|
1257
|
-
}
|
|
1258
|
-
async recoverSessionEnd(apiBase, signal) {
|
|
1259
|
-
const controller = new AbortController();
|
|
1260
|
-
const cancel = () => controller.abort(signal?.reason);
|
|
1261
|
-
signal?.addEventListener("abort", cancel, { once: true });
|
|
1262
|
-
if (signal?.aborted)
|
|
1263
|
-
cancel();
|
|
1264
|
-
const timeout = globalThis.setTimeout(() => controller.abort(), 5_000);
|
|
1265
|
-
try {
|
|
1266
|
-
// A later live event may have advanced the cursor past a missed ending.
|
|
1267
|
-
// Replay from the start; event IDs suppress already delivered messages.
|
|
1268
|
-
await this.fetchReplay(apiBase, 0, { signal: controller.signal, terminalOnly: true });
|
|
1269
|
-
}
|
|
1270
|
-
catch {
|
|
1271
|
-
// A 409 also covers failed sessions. Without an authoritative ending,
|
|
1272
|
-
// retain the original HTTP failure rather than invent a normal end.
|
|
1273
|
-
}
|
|
1274
|
-
finally {
|
|
1275
|
-
globalThis.clearTimeout(timeout);
|
|
1276
|
-
signal?.removeEventListener("abort", cancel);
|
|
1277
|
-
}
|
|
1278
|
-
if (!this.sessionEnded || this.terminalSessionError)
|
|
1279
|
-
return false;
|
|
1280
|
-
this.finishPendingEnd();
|
|
1281
|
-
return true;
|
|
1282
|
-
}
|
|
1283
|
-
notifyEnded(event) {
|
|
1284
|
-
if (this.endNotificationDelivered)
|
|
1285
|
-
return;
|
|
1286
|
-
this.endNotificationDelivered = true;
|
|
1287
|
-
this.notifyListeners(this.endedListeners, event);
|
|
1288
|
-
}
|
|
1289
|
-
beginRecovery(strategy, attempt, details) {
|
|
1290
|
-
this.activeRecovery = { strategy, attempt, details };
|
|
865
|
+
this.flushDiagnostics("context_transfer_timeout");
|
|
866
|
+
reject(error);
|
|
867
|
+
}, timeoutMs);
|
|
868
|
+
});
|
|
869
|
+
let transfer;
|
|
870
|
+
try {
|
|
871
|
+
transfer = await Promise.race([
|
|
872
|
+
uploading,
|
|
873
|
+
transferExpired
|
|
874
|
+
]);
|
|
875
|
+
} catch (error) {
|
|
876
|
+
if (!transferTimedOut) {
|
|
1291
877
|
this.recordDiagnostic({
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
878
|
+
operation: "context_transfer",
|
|
879
|
+
status: "failed",
|
|
880
|
+
reason: "context_transfer_failed",
|
|
881
|
+
context_key: update.key,
|
|
882
|
+
payload_bytes: payloadBytes,
|
|
883
|
+
duration_ms: Date.now() - transferStartedAt
|
|
1298
884
|
});
|
|
1299
|
-
this.flushDiagnostics("
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
885
|
+
this.flushDiagnostics("command_failed");
|
|
886
|
+
}
|
|
887
|
+
if (error instanceof TypeError || error instanceof ContextDeliveryError) throw error;
|
|
888
|
+
throw new ContextDeliveryError(toError(error).message, { cause: error });
|
|
889
|
+
} finally {
|
|
890
|
+
globalThis.clearTimeout(transferTimeout);
|
|
891
|
+
}
|
|
892
|
+
const transferDurationMs = Date.now() - transferStartedAt;
|
|
893
|
+
this.recordDiagnostic({
|
|
894
|
+
operation: "context_transfer",
|
|
895
|
+
status: "succeeded",
|
|
896
|
+
context_key: update.key,
|
|
897
|
+
payload_bytes: payloadBytes,
|
|
898
|
+
duration_ms: transferDurationMs
|
|
899
|
+
});
|
|
900
|
+
if (transferDurationMs > 2e3) this.flushDiagnostics("slow_operation");
|
|
901
|
+
if (generation !== this.contextGeneration) throw new Error("Context transfer interrupted by recovery");
|
|
902
|
+
const { payload } = transfer;
|
|
903
|
+
if (!isContextUpdatePayload(payload)) {
|
|
904
|
+
throw new TypeError("Invalid context update payload");
|
|
905
|
+
}
|
|
906
|
+
const message = createClientMessage(
|
|
907
|
+
this.connectionId,
|
|
908
|
+
++this.clientSeq,
|
|
909
|
+
"context.update",
|
|
910
|
+
{ ...payload }
|
|
911
|
+
);
|
|
912
|
+
return this.sendTrackedCommand(
|
|
913
|
+
message,
|
|
914
|
+
timeoutMs,
|
|
915
|
+
transfer.resend
|
|
916
|
+
);
|
|
917
|
+
}
|
|
918
|
+
resumeContextDelivery(restore) {
|
|
919
|
+
void this.replacementContext.resume(restore).catch(() => {
|
|
920
|
+
});
|
|
921
|
+
}
|
|
922
|
+
async uploadTextContext(update) {
|
|
923
|
+
const data = new TextEncoder().encode(update.text);
|
|
924
|
+
if (data.byteLength === 0) throw new TypeError("Context text cannot be empty");
|
|
925
|
+
if (data.byteLength > 262144) {
|
|
926
|
+
throw new TypeError("Context text must be at most 262144 UTF-8 bytes");
|
|
927
|
+
}
|
|
928
|
+
const base = {
|
|
929
|
+
key: update.key,
|
|
930
|
+
mode: update.mode,
|
|
931
|
+
label: update.label ?? "",
|
|
932
|
+
respond: update.respond ?? false
|
|
933
|
+
};
|
|
934
|
+
if (data.byteLength <= 32768) return { payload: { ...base, text: update.text } };
|
|
935
|
+
const sha256 = await sha256Hex(data);
|
|
936
|
+
const streamId = contextStreamId();
|
|
937
|
+
const send = async () => {
|
|
938
|
+
const writer = await this.room.localParticipant.streamText({
|
|
939
|
+
streamId,
|
|
940
|
+
topic: "voxevolv.context.text",
|
|
941
|
+
totalSize: data.byteLength,
|
|
942
|
+
attributes: { sha256 }
|
|
943
|
+
});
|
|
944
|
+
await writer.write(update.text);
|
|
945
|
+
await writer.close();
|
|
946
|
+
};
|
|
947
|
+
await send();
|
|
948
|
+
return {
|
|
949
|
+
payload: {
|
|
950
|
+
...base,
|
|
951
|
+
text_stream: { stream_id: streamId, bytes: data.byteLength, sha256 }
|
|
952
|
+
},
|
|
953
|
+
resend: send
|
|
954
|
+
};
|
|
955
|
+
}
|
|
956
|
+
async uploadImageContext(update) {
|
|
957
|
+
const allowed = /* @__PURE__ */ new Set(["image/jpeg", "image/png", "image/webp"]);
|
|
958
|
+
if (!allowed.has(update.image.type)) {
|
|
959
|
+
throw new TypeError("Image context must be JPEG, PNG, or WebP");
|
|
960
|
+
}
|
|
961
|
+
const data = new Uint8Array(await update.image.arrayBuffer());
|
|
962
|
+
if (data.byteLength === 0 || data.byteLength > 5242880) {
|
|
963
|
+
throw new TypeError("Image context must be between 1 byte and 5 MiB");
|
|
964
|
+
}
|
|
965
|
+
const sha256 = await sha256Hex(data);
|
|
966
|
+
const streamId = contextStreamId();
|
|
967
|
+
const send = async () => {
|
|
968
|
+
const writer = await this.room.localParticipant.streamBytes({
|
|
969
|
+
streamId,
|
|
970
|
+
topic: "voxevolv.context.image",
|
|
971
|
+
name: "context-image",
|
|
972
|
+
mimeType: update.image.type,
|
|
973
|
+
totalSize: data.byteLength,
|
|
974
|
+
attributes: { sha256 }
|
|
975
|
+
});
|
|
976
|
+
await writer.write(data);
|
|
977
|
+
await writer.close();
|
|
978
|
+
};
|
|
979
|
+
await send();
|
|
980
|
+
return {
|
|
981
|
+
payload: {
|
|
982
|
+
key: update.key,
|
|
983
|
+
mode: "replace",
|
|
984
|
+
label: update.label ?? "",
|
|
985
|
+
image: {
|
|
986
|
+
stream_id: streamId,
|
|
987
|
+
mime: update.image.type,
|
|
988
|
+
bytes: data.byteLength,
|
|
989
|
+
sha256,
|
|
990
|
+
...update.width === void 0 ? {} : { width: update.width },
|
|
991
|
+
...update.height === void 0 ? {} : { height: update.height },
|
|
992
|
+
detail: update.detail ?? "low"
|
|
993
|
+
},
|
|
994
|
+
respond: update.respond ?? false
|
|
995
|
+
},
|
|
996
|
+
resend: send
|
|
997
|
+
};
|
|
998
|
+
}
|
|
999
|
+
async disconnect() {
|
|
1000
|
+
this.abortStartupConnect(new Error("Session disconnected"));
|
|
1001
|
+
this.startupConnectController = null;
|
|
1002
|
+
this.failRecovery(new Error("Session disconnected during recovery"));
|
|
1003
|
+
this.cancelRecovery();
|
|
1004
|
+
this.contextGeneration += 1;
|
|
1005
|
+
this.localApplicationAudioTrack = null;
|
|
1006
|
+
this.replacementContext.clear(new Error("Session disconnected"));
|
|
1007
|
+
if (this.stateValue === "disconnected") return;
|
|
1008
|
+
this.rejectPendingCommands(new Error("Session disconnected before command completion"));
|
|
1009
|
+
this.disconnectMode = "intentional";
|
|
1010
|
+
try {
|
|
1011
|
+
await this.room.disconnect();
|
|
1012
|
+
} finally {
|
|
1013
|
+
this.disconnectMode = null;
|
|
1014
|
+
this.activeRecovery = null;
|
|
1015
|
+
this.setState("disconnected");
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
async end(options = {}) {
|
|
1019
|
+
this.credentialRefreshGeneration += 1;
|
|
1020
|
+
this.abortStartupConnect(new Error("Session ended"));
|
|
1021
|
+
this.startupConnectController = null;
|
|
1022
|
+
this.failRecovery(new Error("Session ended during recovery"));
|
|
1023
|
+
this.cancelRecovery();
|
|
1024
|
+
this.sessionEnded = true;
|
|
1025
|
+
const endpoint = options.endpoint ?? this.endEndpoint ?? `${this.apiBase}/sessions/${this.id}`;
|
|
1026
|
+
const request = options.request ?? {};
|
|
1027
|
+
const headers = new Headers(request.headers);
|
|
1028
|
+
const usesScopedSessionEndpoint = !options.endpoint && !this.endEndpoint;
|
|
1029
|
+
if (!headers.has("Authorization") && usesScopedSessionEndpoint) {
|
|
1030
|
+
headers.set("Authorization", `Bearer ${this.accessToken}`);
|
|
1031
|
+
}
|
|
1032
|
+
let response = await this.fetcher(endpoint, {
|
|
1033
|
+
...request,
|
|
1034
|
+
method: request.method ?? "DELETE",
|
|
1035
|
+
headers,
|
|
1036
|
+
credentials: request.credentials ?? this.lifecycleCredentials
|
|
1037
|
+
});
|
|
1038
|
+
if (response.status === 401 && usesScopedSessionEndpoint && this.refreshCredentialsCallback) {
|
|
1039
|
+
await this.refreshCredentials(request.signal ?? void 0);
|
|
1040
|
+
headers.set("Authorization", `Bearer ${this.accessToken}`);
|
|
1041
|
+
response = await this.fetcher(endpoint, {
|
|
1042
|
+
...request,
|
|
1043
|
+
method: request.method ?? "DELETE",
|
|
1044
|
+
headers,
|
|
1045
|
+
credentials: request.credentials ?? this.lifecycleCredentials
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
if (!response.ok) throw await httpError(response, "Could not end realtime session");
|
|
1049
|
+
if (options.disconnect ?? true) await this.disconnect();
|
|
1050
|
+
}
|
|
1051
|
+
async replay(apiBase = this.apiBase, afterSeq = this.cursor.lastServerSeq) {
|
|
1052
|
+
if (this.replayPromise) return this.replayPromise;
|
|
1053
|
+
const normalizedApiBase = normalizeApiBase(apiBase);
|
|
1054
|
+
this.replayPromise = this.fetchReplay(normalizedApiBase, afterSeq).finally(() => {
|
|
1055
|
+
this.replayPromise = null;
|
|
1056
|
+
});
|
|
1057
|
+
return this.replayPromise;
|
|
1058
|
+
}
|
|
1059
|
+
setState(state) {
|
|
1060
|
+
if (this.stateValue === state) return;
|
|
1061
|
+
this.stateValue = state;
|
|
1062
|
+
if (state === "connecting" || state === "resuming") {
|
|
1063
|
+
this.startDiagnosticLoopMonitor();
|
|
1064
|
+
globalThis.document?.addEventListener?.("visibilitychange", this.onDiagnosticVisibilityChange);
|
|
1065
|
+
}
|
|
1066
|
+
if (state === "disconnected" || state === "failed") {
|
|
1067
|
+
globalThis.clearInterval(this.diagnosticLoopTimer);
|
|
1068
|
+
this.diagnosticLoopTimer = void 0;
|
|
1069
|
+
this.diagnosticLoopDelayMs = void 0;
|
|
1070
|
+
globalThis.document?.removeEventListener?.("visibilitychange", this.onDiagnosticVisibilityChange);
|
|
1071
|
+
this.flushDiagnostics("session_finished");
|
|
1072
|
+
const error = new Error(`Session ${state}`);
|
|
1073
|
+
this.contextGeneration += 1;
|
|
1074
|
+
this.replacementContext.clear(error);
|
|
1075
|
+
this.rejectPendingCommands(error);
|
|
1076
|
+
}
|
|
1077
|
+
this.notifyListeners(this.stateListeners, state);
|
|
1078
|
+
}
|
|
1079
|
+
bindRoomEvents() {
|
|
1080
|
+
this.room.on(RoomEvent.ParticipantDisconnected, (participant) => {
|
|
1081
|
+
if (participant.isAgent && !this.disconnectMode) {
|
|
1082
|
+
queueMicrotask(() => {
|
|
1083
|
+
if (this.disconnectMode || this.terminalSessionError) return;
|
|
1084
|
+
if (this.sessionEnded) {
|
|
1085
|
+
this.agentDisconnected = true;
|
|
1086
|
+
this.finishPendingEnd();
|
|
1310
1087
|
return;
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1088
|
+
}
|
|
1089
|
+
if (this.room.state === LiveKitConnectionState.Reconnecting || this.activeRecovery !== null || this.autoRejoinPromise !== null || this.manualRejoinController !== null) return;
|
|
1090
|
+
if ([...this.room.remoteParticipants.values()].some((remote) => remote.isAgent)) return;
|
|
1091
|
+
this.agentDisconnected = true;
|
|
1092
|
+
this.startAutoRejoin(
|
|
1093
|
+
new Error("Agent disconnected from the room"),
|
|
1094
|
+
{ trigger: "agent_disconnected" }
|
|
1095
|
+
);
|
|
1317
1096
|
});
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
this.room.on(RoomEvent.SignalReconnecting, () => this.handleLiveKitReconnecting());
|
|
1100
|
+
this.room.on(RoomEvent.Reconnecting, () => this.handleLiveKitReconnecting());
|
|
1101
|
+
this.room.on(RoomEvent.Reconnected, () => {
|
|
1102
|
+
void this.handleLiveKitReconnected();
|
|
1103
|
+
});
|
|
1104
|
+
this.room.on(RoomEvent.Disconnected, (reason) => {
|
|
1105
|
+
this.handleLiveKitDisconnected(reason);
|
|
1106
|
+
});
|
|
1107
|
+
this.room.on(RoomEvent.ConnectionStateChanged, (state) => {
|
|
1108
|
+
if (this.sessionEnded || this.terminalSessionError) return;
|
|
1109
|
+
if (state === LiveKitConnectionState.Connected) {
|
|
1110
|
+
const recovering = this.activeRecovery !== null || this.stateValue === "resuming";
|
|
1111
|
+
this.setState(
|
|
1112
|
+
this.agentReady ? "connected" : recovering ? "resuming" : "connected_not_ready"
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
});
|
|
1116
|
+
this.room.on(RoomEvent.ConnectionQualityChanged, (quality, participant) => {
|
|
1117
|
+
if (!participant.isLocal) return;
|
|
1118
|
+
this.notifyListeners(
|
|
1119
|
+
this.connectionQualityListeners,
|
|
1120
|
+
quality
|
|
1121
|
+
);
|
|
1122
|
+
});
|
|
1123
|
+
this.room.on(RoomEvent.TrackSubscribed, (track) => {
|
|
1124
|
+
if (track.kind !== Track.Kind.Audio) return;
|
|
1125
|
+
const element = track.attach();
|
|
1126
|
+
element.dataset.voxevolvRemoteAudio = "true";
|
|
1127
|
+
element.style.display = "none";
|
|
1128
|
+
document.body.appendChild(element);
|
|
1129
|
+
try {
|
|
1130
|
+
const playback = element.play();
|
|
1131
|
+
void playback.catch((error) => {
|
|
1132
|
+
this.notifyListeners(this.audioPlaybackBlockedListeners, {
|
|
1329
1133
|
error,
|
|
1134
|
+
element,
|
|
1135
|
+
track: track.mediaStreamTrack
|
|
1136
|
+
});
|
|
1330
1137
|
});
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
&& !["disconnected", "failed"].includes(this.stateValue)
|
|
1337
|
-
&& !isTerminalDisconnectReason(reason));
|
|
1338
|
-
}
|
|
1339
|
-
startAutoRejoin(initialError, details) {
|
|
1340
|
-
if (this.autoRejoinPromise || !this.shouldAutoRejoin())
|
|
1341
|
-
return;
|
|
1342
|
-
this.agentReady = false;
|
|
1343
|
-
this.contextGeneration += 1;
|
|
1344
|
-
this.replacementContext.pause();
|
|
1345
|
-
this.rejectPendingCommands(initialError);
|
|
1346
|
-
this.reconnectStartedAt ??= Date.now();
|
|
1347
|
-
const controller = new AbortController();
|
|
1348
|
-
this.autoRejoinController = controller;
|
|
1349
|
-
const timeout = globalThis.setTimeout(() => {
|
|
1350
|
-
if (controller.signal.aborted)
|
|
1351
|
-
return;
|
|
1352
|
-
const error = new Error("Session recovery timed out");
|
|
1353
|
-
controller.abort(error);
|
|
1354
|
-
this.contextGeneration += 1;
|
|
1355
|
-
this.rejectPendingCommands(error);
|
|
1356
|
-
this.replacementContext.clear(error);
|
|
1357
|
-
this.failRecovery(error);
|
|
1358
|
-
this.setState("failed");
|
|
1359
|
-
}, this.autoRejoinOptions.timeoutMs);
|
|
1360
|
-
const promise = this.runAutoRejoin(initialError, controller.signal, details)
|
|
1361
|
-
.finally(() => {
|
|
1362
|
-
globalThis.clearTimeout(timeout);
|
|
1363
|
-
if (this.autoRejoinController === controller) {
|
|
1364
|
-
this.autoRejoinController = null;
|
|
1365
|
-
this.autoRejoinPromise = null;
|
|
1366
|
-
}
|
|
1138
|
+
} catch (error) {
|
|
1139
|
+
this.notifyListeners(this.audioPlaybackBlockedListeners, {
|
|
1140
|
+
error,
|
|
1141
|
+
element,
|
|
1142
|
+
track: track.mediaStreamTrack
|
|
1367
1143
|
});
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
this.
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1144
|
+
}
|
|
1145
|
+
const stream = new MediaStream([track.mediaStreamTrack]);
|
|
1146
|
+
this.remoteAudioStreams.set(track, stream);
|
|
1147
|
+
this.outputStream.addTrack(track.mediaStreamTrack);
|
|
1148
|
+
this.notifyRemoteAudioListeners(this.remoteAudioTrackListeners, track.mediaStreamTrack, stream);
|
|
1149
|
+
this.remoteAudioTrackResolve?.();
|
|
1150
|
+
});
|
|
1151
|
+
this.room.on(RoomEvent.TrackUnsubscribed, (track) => {
|
|
1152
|
+
if (track.kind !== Track.Kind.Audio) return;
|
|
1153
|
+
const stream = this.remoteAudioStreams.get(track) ?? new MediaStream([track.mediaStreamTrack]);
|
|
1154
|
+
this.notifyRemoteAudioListeners(
|
|
1155
|
+
this.remoteAudioTrackUnsubscribedListeners,
|
|
1156
|
+
track.mediaStreamTrack,
|
|
1157
|
+
stream
|
|
1158
|
+
);
|
|
1159
|
+
this.remoteAudioStreams.delete(track);
|
|
1160
|
+
this.outputMediaStream?.removeTrack(track.mediaStreamTrack);
|
|
1161
|
+
track.detach().forEach((element) => element.remove());
|
|
1162
|
+
});
|
|
1163
|
+
this.room.on(RoomEvent.DataReceived, (data, _participant, kind, topic) => {
|
|
1164
|
+
if (topic !== "voxevolv.events" || kind !== DataPacket_Kind.RELIABLE) return;
|
|
1165
|
+
let value;
|
|
1166
|
+
try {
|
|
1167
|
+
value = JSON.parse(new TextDecoder().decode(data));
|
|
1168
|
+
} catch {
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
1171
|
+
if (!isRealtimeEvent(value)) return;
|
|
1172
|
+
const result = this.cursor.accept(value);
|
|
1173
|
+
if (result.gap) {
|
|
1174
|
+
this.notifyListeners(this.gapListeners, result.gap);
|
|
1175
|
+
void this.replay(this.apiBase, result.gap.after_server_seq);
|
|
1176
|
+
}
|
|
1177
|
+
if (!result.deliver) return;
|
|
1178
|
+
this.observeReadiness(value);
|
|
1179
|
+
this.observeCommandResult(value);
|
|
1180
|
+
this.observeConvenienceEvents(value);
|
|
1181
|
+
this.notifyListeners(this.listeners, value);
|
|
1182
|
+
});
|
|
1183
|
+
}
|
|
1184
|
+
handleLiveKitReconnecting() {
|
|
1185
|
+
if (this.sessionEnded || this.disconnectMode === "intentional") return;
|
|
1186
|
+
this.agentReady = false;
|
|
1187
|
+
if (this.autoRejoinPromise !== null || this.activeRecovery?.strategy === "rejoin" || this.disconnectMode === "rejoin") {
|
|
1188
|
+
this.setState("resuming");
|
|
1189
|
+
return;
|
|
1190
|
+
}
|
|
1191
|
+
this.contextGeneration += 1;
|
|
1192
|
+
this.livekitResumeController?.abort(new Error("LiveKit connection interrupted again"));
|
|
1193
|
+
this.livekitResumeController = null;
|
|
1194
|
+
this.replacementContext.pause();
|
|
1195
|
+
this.reconnectStartedAt ??= Date.now();
|
|
1196
|
+
this.rejectPendingCommands(
|
|
1197
|
+
new Error("Connection interrupted before command completion"),
|
|
1198
|
+
true
|
|
1199
|
+
);
|
|
1200
|
+
if (this.activeRecovery === null) {
|
|
1201
|
+
this.beginRecovery("livekit", 1, { trigger: "livekit_reconnecting" });
|
|
1202
|
+
this.recordDiagnostic({ operation: "connection_recovery", stage: "room_connect", status: "started" });
|
|
1203
|
+
}
|
|
1204
|
+
this.setState("reconnecting");
|
|
1205
|
+
}
|
|
1206
|
+
async handleLiveKitReconnected() {
|
|
1207
|
+
if (this.sessionEnded || this.disconnectMode || this.autoRejoinPromise || this.livekitResumeController) return;
|
|
1208
|
+
const generation = this.contextGeneration;
|
|
1209
|
+
const controller = new AbortController();
|
|
1210
|
+
this.livekitResumeController = controller;
|
|
1211
|
+
const recoveredAt = Date.now();
|
|
1212
|
+
if (this.reconnectStartedAt !== null) {
|
|
1213
|
+
this.pendingReconnect = {
|
|
1214
|
+
started_at: new Date(this.reconnectStartedAt).toISOString(),
|
|
1215
|
+
recovered_at: new Date(recoveredAt).toISOString(),
|
|
1216
|
+
duration_ms: Math.max(recoveredAt - this.reconnectStartedAt, 0)
|
|
1217
|
+
};
|
|
1218
|
+
this.reconnectStartedAt = null;
|
|
1219
|
+
}
|
|
1220
|
+
if (this.hasConnectedValue && this.activeRecovery === null) {
|
|
1221
|
+
this.beginRecovery("livekit", 1, { trigger: "livekit_reconnecting" });
|
|
1222
|
+
}
|
|
1223
|
+
this.recordDiagnostic({
|
|
1224
|
+
operation: "connection_recovery",
|
|
1225
|
+
stage: "room_connect",
|
|
1226
|
+
status: "succeeded",
|
|
1227
|
+
duration_ms: Math.max(Date.now() - (this.recoveryStartedAt ?? Date.now()), 0)
|
|
1228
|
+
});
|
|
1229
|
+
this.connectionId = `conn_${Math.random().toString(36).slice(2)}_${Date.now().toString(36)}`;
|
|
1230
|
+
this.clientSeq = 0;
|
|
1231
|
+
this.setState(this.hasConnectedValue ? "resuming" : "connected_not_ready");
|
|
1232
|
+
try {
|
|
1233
|
+
await this.measureRecoveryStage("hello", () => this.sendRecoveryHello(controller.signal));
|
|
1234
|
+
const deadline = Date.now() + this.recoveryReadiness.readyTimeoutMs;
|
|
1235
|
+
await this.measureRecoveryStage(
|
|
1236
|
+
"agent_ready",
|
|
1237
|
+
() => this.waitForAgentReady(Math.max(deadline - Date.now(), 0), controller.signal)
|
|
1238
|
+
);
|
|
1239
|
+
if (this.recoveryReadiness.waitForRemoteAudio) {
|
|
1240
|
+
await this.measureRecoveryStage(
|
|
1241
|
+
"remote_audio",
|
|
1242
|
+
() => this.waitForRemoteAudioTrack(deadline, controller.signal)
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
if (this.sessionEnded || generation !== this.contextGeneration) return;
|
|
1246
|
+
this.resumeContextDelivery(true);
|
|
1247
|
+
if (this.sessionEnded || generation !== this.contextGeneration) return;
|
|
1248
|
+
this.setState("connected");
|
|
1249
|
+
this.completeRecovery();
|
|
1250
|
+
} catch (error) {
|
|
1251
|
+
if (controller.signal.aborted || this.sessionEnded || generation !== this.contextGeneration) return;
|
|
1252
|
+
const normalizedError = toError(error);
|
|
1253
|
+
if (this.shouldAutoRejoin()) {
|
|
1254
|
+
this.startAutoRejoin(normalizedError, { trigger: "livekit_resume_failed" });
|
|
1255
|
+
} else {
|
|
1256
|
+
this.failRecovery(normalizedError);
|
|
1408
1257
|
this.setState("failed");
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1258
|
+
}
|
|
1259
|
+
} finally {
|
|
1260
|
+
if (this.livekitResumeController === controller) this.livekitResumeController = null;
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
handleLiveKitDisconnected(reason) {
|
|
1264
|
+
this.agentReady = false;
|
|
1265
|
+
if (this.disconnectMode !== "rejoin") {
|
|
1266
|
+
this.pendingReconnect = null;
|
|
1267
|
+
}
|
|
1268
|
+
this.outputMediaStream?.getTracks().forEach((track) => {
|
|
1269
|
+
this.outputMediaStream?.removeTrack(track);
|
|
1270
|
+
});
|
|
1271
|
+
this.rejectPendingCommands(new Error("Session disconnected before command completion"));
|
|
1272
|
+
if (this.disconnectMode === "rejoin") return;
|
|
1273
|
+
if (this.disconnectMode === "connect_failure") return;
|
|
1274
|
+
if (this.disconnectMode === "intentional") {
|
|
1275
|
+
this.activeRecovery = null;
|
|
1276
|
+
this.deliverPendingEndEvent();
|
|
1277
|
+
this.setState("disconnected");
|
|
1278
|
+
return;
|
|
1279
|
+
}
|
|
1280
|
+
if (this.terminalSessionError) return;
|
|
1281
|
+
if (this.shouldAutoRejoin(reason)) {
|
|
1282
|
+
this.startAutoRejoin(
|
|
1283
|
+
new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`),
|
|
1284
|
+
{ trigger: "livekit_disconnected" }
|
|
1285
|
+
);
|
|
1286
|
+
return;
|
|
1287
|
+
}
|
|
1288
|
+
if (this.activeRecovery !== null) {
|
|
1289
|
+
this.failRecovery(
|
|
1290
|
+
new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`)
|
|
1291
|
+
);
|
|
1292
|
+
}
|
|
1293
|
+
this.deliverPendingEndEvent();
|
|
1294
|
+
this.setState("disconnected");
|
|
1295
|
+
}
|
|
1296
|
+
// The terminal session.ended event is recorded durably after room teardown,
|
|
1297
|
+
// so turn the earlier live announcement into the browser completion signal.
|
|
1298
|
+
deliverPendingEndEvent() {
|
|
1299
|
+
const ending = this.pendingEndEvent;
|
|
1300
|
+
if (!ending) return;
|
|
1301
|
+
this.pendingEndEvent = null;
|
|
1302
|
+
this.notifyEnded({ ...ending, type: "session.ended" });
|
|
1303
|
+
}
|
|
1304
|
+
finishPendingEnd() {
|
|
1305
|
+
this.deliverPendingEndEvent();
|
|
1306
|
+
void this.disconnect().catch(() => {
|
|
1307
|
+
});
|
|
1308
|
+
}
|
|
1309
|
+
async recoverSessionEnd(apiBase, signal) {
|
|
1310
|
+
const controller = new AbortController();
|
|
1311
|
+
const cancel = () => controller.abort(signal?.reason);
|
|
1312
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
1313
|
+
if (signal?.aborted) cancel();
|
|
1314
|
+
const timeout = globalThis.setTimeout(() => controller.abort(), 5e3);
|
|
1315
|
+
try {
|
|
1316
|
+
await this.fetchReplay(apiBase, 0, { signal: controller.signal, terminalOnly: true });
|
|
1317
|
+
} catch {
|
|
1318
|
+
} finally {
|
|
1319
|
+
globalThis.clearTimeout(timeout);
|
|
1320
|
+
signal?.removeEventListener("abort", cancel);
|
|
1321
|
+
}
|
|
1322
|
+
if (!this.sessionEnded || this.terminalSessionError) return false;
|
|
1323
|
+
this.finishPendingEnd();
|
|
1324
|
+
return true;
|
|
1325
|
+
}
|
|
1326
|
+
notifyEnded(event) {
|
|
1327
|
+
if (this.endNotificationDelivered) return;
|
|
1328
|
+
this.endNotificationDelivered = true;
|
|
1329
|
+
this.notifyListeners(this.endedListeners, event);
|
|
1330
|
+
}
|
|
1331
|
+
beginRecovery(strategy, attempt, details) {
|
|
1332
|
+
this.recoveryStartedAt ??= Date.now();
|
|
1333
|
+
this.activeRecovery = { strategy, attempt, details };
|
|
1334
|
+
this.recordDiagnostic({
|
|
1335
|
+
operation: "connection_recovery",
|
|
1336
|
+
status: "started",
|
|
1337
|
+
reason: details.trigger,
|
|
1338
|
+
message_id: details.messageId,
|
|
1339
|
+
context_key: details.contextKey,
|
|
1340
|
+
timeout_ms: details.timeoutMs,
|
|
1341
|
+
retry_count: attempt - 1
|
|
1342
|
+
});
|
|
1343
|
+
this.flushDiagnostics("recovery_started");
|
|
1344
|
+
this.notifyListeners(this.recoveryListeners, {
|
|
1345
|
+
status: "started",
|
|
1346
|
+
strategy,
|
|
1347
|
+
attempt,
|
|
1348
|
+
...details
|
|
1349
|
+
});
|
|
1350
|
+
}
|
|
1351
|
+
completeRecovery() {
|
|
1352
|
+
const recovery = this.activeRecovery;
|
|
1353
|
+
if (!recovery) return;
|
|
1354
|
+
this.recordDiagnostic({
|
|
1355
|
+
operation: "connection_recovery",
|
|
1356
|
+
stage: "complete",
|
|
1357
|
+
status: "succeeded",
|
|
1358
|
+
duration_ms: Math.max(Date.now() - (this.recoveryStartedAt ?? Date.now()), 0),
|
|
1359
|
+
retry_count: recovery.attempt - 1,
|
|
1360
|
+
reason: recovery.details.trigger
|
|
1361
|
+
});
|
|
1362
|
+
this.flushDiagnostics("recovery_completed");
|
|
1363
|
+
this.recoveryStartedAt = null;
|
|
1364
|
+
this.activeRecovery = null;
|
|
1365
|
+
this.notifyListeners(this.recoveryListeners, {
|
|
1366
|
+
status: "succeeded",
|
|
1367
|
+
strategy: recovery.strategy,
|
|
1368
|
+
attempt: recovery.attempt,
|
|
1369
|
+
...recovery.details
|
|
1370
|
+
});
|
|
1371
|
+
}
|
|
1372
|
+
failRecovery(error) {
|
|
1373
|
+
const recovery = this.activeRecovery;
|
|
1374
|
+
if (!recovery) return;
|
|
1375
|
+
this.recordDiagnostic({
|
|
1376
|
+
operation: "connection_recovery",
|
|
1377
|
+
stage: "complete",
|
|
1378
|
+
status: "failed",
|
|
1379
|
+
duration_ms: Math.max(Date.now() - (this.recoveryStartedAt ?? Date.now()), 0),
|
|
1380
|
+
retry_count: recovery.attempt - 1,
|
|
1381
|
+
reason: recovery.details.trigger
|
|
1382
|
+
});
|
|
1383
|
+
this.flushDiagnostics("recovery_failed");
|
|
1384
|
+
this.activeRecovery = null;
|
|
1385
|
+
this.notifyListeners(this.recoveryListeners, {
|
|
1386
|
+
status: "failed",
|
|
1387
|
+
strategy: recovery.strategy,
|
|
1388
|
+
attempt: recovery.attempt,
|
|
1389
|
+
...recovery.details,
|
|
1390
|
+
error
|
|
1391
|
+
});
|
|
1392
|
+
}
|
|
1393
|
+
shouldAutoRejoin(reason) {
|
|
1394
|
+
return this.autoRejoinOptions !== null && this.hasConnectedValue && !this.sessionEnded && !["disconnected", "failed"].includes(this.stateValue) && !isTerminalDisconnectReason(reason);
|
|
1395
|
+
}
|
|
1396
|
+
startAutoRejoin(initialError, details) {
|
|
1397
|
+
if (this.autoRejoinPromise || !this.shouldAutoRejoin()) return;
|
|
1398
|
+
this.recoveryStartedAt ??= Date.now();
|
|
1399
|
+
this.agentReady = false;
|
|
1400
|
+
this.contextGeneration += 1;
|
|
1401
|
+
this.livekitResumeController?.abort(initialError);
|
|
1402
|
+
this.replacementContext.pause();
|
|
1403
|
+
this.rejectPendingCommands(initialError);
|
|
1404
|
+
this.reconnectStartedAt ??= Date.now();
|
|
1405
|
+
const controller = new AbortController();
|
|
1406
|
+
this.autoRejoinController = controller;
|
|
1407
|
+
const timeout = globalThis.setTimeout(() => {
|
|
1408
|
+
if (controller.signal.aborted) return;
|
|
1409
|
+
const error = new Error("Session recovery timed out");
|
|
1410
|
+
controller.abort(error);
|
|
1411
|
+
this.contextGeneration += 1;
|
|
1412
|
+
this.rejectPendingCommands(error);
|
|
1413
|
+
this.replacementContext.clear(error);
|
|
1414
|
+
this.failRecovery(error);
|
|
1415
|
+
this.setState("failed");
|
|
1416
|
+
}, this.autoRejoinOptions.timeoutMs);
|
|
1417
|
+
const promise = this.runAutoRejoin(initialError, controller.signal, details).finally(() => {
|
|
1418
|
+
globalThis.clearTimeout(timeout);
|
|
1419
|
+
if (this.autoRejoinController === controller) {
|
|
1412
1420
|
this.autoRejoinController = null;
|
|
1413
|
-
this.manualRejoinController?.abort();
|
|
1414
|
-
this.manualRejoinController = null;
|
|
1415
1421
|
this.autoRejoinPromise = null;
|
|
1416
|
-
|
|
1417
|
-
}
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
}
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
}
|
|
1437
|
-
}
|
|
1438
|
-
validateAudioOptions(microphone, audioTrack) {
|
|
1439
|
-
if (microphone && audioTrack) {
|
|
1440
|
-
throw new SessionConfigurationError("microphone and audioTrack cannot both be enabled");
|
|
1441
|
-
}
|
|
1442
|
-
if (!audioTrack)
|
|
1443
|
-
return;
|
|
1444
|
-
if (audioTrack.kind !== "audio") {
|
|
1445
|
-
throw new SessionConfigurationError("audioTrack must be an audio MediaStreamTrack");
|
|
1446
|
-
}
|
|
1447
|
-
if (audioTrack.readyState === "ended") {
|
|
1448
|
-
throw new SessionConfigurationError("audioTrack has already ended");
|
|
1449
|
-
}
|
|
1450
|
-
}
|
|
1451
|
-
async fetchRoomToken(apiBase, signal) {
|
|
1452
|
-
const controller = new AbortController();
|
|
1453
|
-
const cancel = () => controller.abort(signal?.reason);
|
|
1454
|
-
signal?.addEventListener("abort", cancel, { once: true });
|
|
1455
|
-
const timeout = globalThis.setTimeout(() => controller.abort(), 10_000);
|
|
1456
|
-
try {
|
|
1457
|
-
throwIfAborted(signal);
|
|
1458
|
-
return await raceWithAbort(this.fetcher(`${apiBase}/sessions/${this.id}/room-tokens`, {
|
|
1459
|
-
method: "POST",
|
|
1460
|
-
headers: { Authorization: `Bearer ${this.accessToken}` },
|
|
1461
|
-
credentials: this.lifecycleCredentials,
|
|
1462
|
-
signal: controller.signal,
|
|
1463
|
-
}), controller.signal);
|
|
1464
|
-
}
|
|
1465
|
-
finally {
|
|
1466
|
-
globalThis.clearTimeout(timeout);
|
|
1467
|
-
signal?.removeEventListener("abort", cancel);
|
|
1422
|
+
}
|
|
1423
|
+
});
|
|
1424
|
+
this.autoRejoinPromise = promise;
|
|
1425
|
+
void promise;
|
|
1426
|
+
}
|
|
1427
|
+
async runAutoRejoin(initialError, signal, details) {
|
|
1428
|
+
const options = this.autoRejoinOptions;
|
|
1429
|
+
if (!options) return;
|
|
1430
|
+
let lastError = initialError;
|
|
1431
|
+
if (this.activeRecovery?.strategy === "livekit") {
|
|
1432
|
+
this.failRecovery(initialError);
|
|
1433
|
+
}
|
|
1434
|
+
this.setState("resuming");
|
|
1435
|
+
for (let attempt = 1; attempt <= options.maxAttempts; attempt += 1) {
|
|
1436
|
+
try {
|
|
1437
|
+
await waitUntilOnline(signal);
|
|
1438
|
+
if (attempt > 1) {
|
|
1439
|
+
const exponentialDelay = options.baseDelayMs * 2 ** (attempt - 2);
|
|
1440
|
+
const delayMs = Math.min(exponentialDelay, options.maxDelayMs);
|
|
1441
|
+
await abortableDelay(delayMs + Math.random() * Math.min(delayMs, 500), signal);
|
|
1468
1442
|
}
|
|
1469
|
-
}
|
|
1470
|
-
async refreshCredentials(signal) {
|
|
1471
|
-
if (!this.refreshCredentialsCallback)
|
|
1472
|
-
return;
|
|
1473
1443
|
throwIfAborted(signal);
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1444
|
+
this.beginRecovery("rejoin", attempt, details);
|
|
1445
|
+
await this.performRejoin(this.apiBase, { signal });
|
|
1446
|
+
if (this.sessionEnded) return;
|
|
1477
1447
|
throwIfAborted(signal);
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
const transcript = transcriptEvent(event);
|
|
1566
|
-
if (!transcript)
|
|
1567
|
-
return;
|
|
1568
|
-
this.notifyListeners(transcript.role === "user" ? this.userTranscriptListeners : this.agentTranscriptListeners, transcript);
|
|
1569
|
-
}
|
|
1570
|
-
abortStartupConnect(error) {
|
|
1571
|
-
if (!this.startupConnectController?.signal.aborted) {
|
|
1572
|
-
this.startupConnectController?.abort(error);
|
|
1573
|
-
}
|
|
1574
|
-
}
|
|
1575
|
-
async publishApplicationAudioTrack(audioTrack) {
|
|
1576
|
-
const clonedTrack = audioTrack.clone();
|
|
1577
|
-
let publishedTrack = null;
|
|
1578
|
-
try {
|
|
1579
|
-
publishedTrack = new LocalAudioTrack(clonedTrack, undefined, true);
|
|
1580
|
-
this.localApplicationAudioTrack = publishedTrack;
|
|
1581
|
-
if (!this.microphoneEnabled) {
|
|
1582
|
-
// LiveKit can re-enable a raw track when wrapping it. Mute the wrapper
|
|
1583
|
-
// before publication so recovery cannot briefly transmit muted input.
|
|
1584
|
-
await publishedTrack.mute();
|
|
1585
|
-
}
|
|
1586
|
-
await this.room.localParticipant.publishTrack(publishedTrack, {
|
|
1587
|
-
source: Track.Source.Microphone,
|
|
1588
|
-
name: "voxevolv-application-audio",
|
|
1589
|
-
});
|
|
1590
|
-
}
|
|
1591
|
-
catch (error) {
|
|
1592
|
-
if (this.localApplicationAudioTrack === publishedTrack) {
|
|
1593
|
-
this.localApplicationAudioTrack = null;
|
|
1594
|
-
}
|
|
1595
|
-
clonedTrack.stop();
|
|
1596
|
-
throw error;
|
|
1597
|
-
}
|
|
1448
|
+
this.reconnectStartedAt = null;
|
|
1449
|
+
this.completeRecovery();
|
|
1450
|
+
return;
|
|
1451
|
+
} catch (error) {
|
|
1452
|
+
if (signal.aborted) return;
|
|
1453
|
+
lastError = toError(error);
|
|
1454
|
+
if (isTerminalRejoinError(error)) break;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
this.failRecovery(lastError);
|
|
1458
|
+
this.replacementContext.clear(lastError);
|
|
1459
|
+
this.setState("failed");
|
|
1460
|
+
}
|
|
1461
|
+
cancelRecovery() {
|
|
1462
|
+
this.livekitResumeController?.abort();
|
|
1463
|
+
this.livekitResumeController = null;
|
|
1464
|
+
this.recoveryStartedAt = null;
|
|
1465
|
+
this.autoRejoinController?.abort();
|
|
1466
|
+
this.autoRejoinController = null;
|
|
1467
|
+
this.manualRejoinController?.abort();
|
|
1468
|
+
this.manualRejoinController = null;
|
|
1469
|
+
this.autoRejoinPromise = null;
|
|
1470
|
+
this.activeRecovery = null;
|
|
1471
|
+
}
|
|
1472
|
+
notifyListeners(listeners, value) {
|
|
1473
|
+
for (const listener of listeners) {
|
|
1474
|
+
try {
|
|
1475
|
+
listener(value);
|
|
1476
|
+
} catch (error) {
|
|
1477
|
+
console.error("VoxEvolv event listener failed", error);
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
notifyRemoteAudioListeners(listeners, track, stream) {
|
|
1482
|
+
for (const listener of listeners) {
|
|
1483
|
+
try {
|
|
1484
|
+
listener(track, stream);
|
|
1485
|
+
} catch (error) {
|
|
1486
|
+
console.error("VoxEvolv remote audio listener failed", error);
|
|
1487
|
+
}
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
validateAudioOptions(microphone, audioTrack) {
|
|
1491
|
+
if (microphone && audioTrack) {
|
|
1492
|
+
throw new SessionConfigurationError("microphone and audioTrack cannot both be enabled");
|
|
1493
|
+
}
|
|
1494
|
+
if (!audioTrack) return;
|
|
1495
|
+
if (audioTrack.kind !== "audio") {
|
|
1496
|
+
throw new SessionConfigurationError("audioTrack must be an audio MediaStreamTrack");
|
|
1497
|
+
}
|
|
1498
|
+
if (audioTrack.readyState === "ended") {
|
|
1499
|
+
throw new SessionConfigurationError("audioTrack has already ended");
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
async measureRecoveryStage(stage, operation, recovering = true) {
|
|
1503
|
+
if (!recovering) return operation();
|
|
1504
|
+
const startedAt = Date.now();
|
|
1505
|
+
this.recordDiagnostic({ operation: "connection_recovery", stage, status: "started" });
|
|
1506
|
+
try {
|
|
1507
|
+
const result = await operation();
|
|
1508
|
+
this.recordDiagnostic({
|
|
1509
|
+
operation: "connection_recovery",
|
|
1510
|
+
stage,
|
|
1511
|
+
status: "succeeded",
|
|
1512
|
+
duration_ms: Date.now() - startedAt
|
|
1513
|
+
});
|
|
1514
|
+
return result;
|
|
1515
|
+
} catch (error) {
|
|
1516
|
+
this.recordDiagnostic({
|
|
1517
|
+
operation: "connection_recovery",
|
|
1518
|
+
stage,
|
|
1519
|
+
status: "failed",
|
|
1520
|
+
duration_ms: Date.now() - startedAt,
|
|
1521
|
+
reason: error instanceof RecoveryDataChannelStalledError ? "data_channel_stalled" : "stage_failed"
|
|
1522
|
+
});
|
|
1523
|
+
this.flushDiagnostics("recovery_stage_failed");
|
|
1524
|
+
throw error;
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
async fetchRoomToken(apiBase, signal, recovering = true) {
|
|
1528
|
+
const controller = new AbortController();
|
|
1529
|
+
const cancel = () => controller.abort(signal?.reason);
|
|
1530
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
1531
|
+
const timeout = globalThis.setTimeout(() => controller.abort(), 1e4);
|
|
1532
|
+
const startedAt = Date.now();
|
|
1533
|
+
if (recovering) {
|
|
1534
|
+
this.recordDiagnostic({ operation: "connection_recovery", stage: "token_request", status: "started" });
|
|
1598
1535
|
}
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
await this.sendTrackedCommand(message, CLIENT_HELLO_TIMEOUT_MS);
|
|
1609
|
-
if (this.pendingReconnect === reconnect)
|
|
1610
|
-
this.pendingReconnect = null;
|
|
1611
|
-
}
|
|
1612
|
-
async sendTrackedCommand(message, timeoutMs, retryTransfer) {
|
|
1613
|
-
const contextKey = contextKeyFromMessage(message);
|
|
1614
|
-
const payloadBytes = contextPayloadBytes(message.payload);
|
|
1615
|
-
const startedAt = Date.now();
|
|
1536
|
+
try {
|
|
1537
|
+
throwIfAborted(signal);
|
|
1538
|
+
const response = await raceWithAbort(this.fetcher(`${apiBase}/sessions/${this.id}/room-tokens`, {
|
|
1539
|
+
method: "POST",
|
|
1540
|
+
headers: { Authorization: `Bearer ${this.accessToken}` },
|
|
1541
|
+
credentials: this.lifecycleCredentials,
|
|
1542
|
+
signal: controller.signal
|
|
1543
|
+
}), controller.signal);
|
|
1544
|
+
if (recovering) {
|
|
1616
1545
|
this.recordDiagnostic({
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
context_key: contextKey,
|
|
1623
|
-
payload_bytes: payloadBytes,
|
|
1624
|
-
retry_count: 0,
|
|
1625
|
-
timeout_ms: timeoutMs,
|
|
1626
|
-
event_type: diagnosticEventType(message.type),
|
|
1627
|
-
});
|
|
1628
|
-
let markSettled = () => { };
|
|
1629
|
-
const settled = new Promise((resolve) => { markSettled = resolve; });
|
|
1630
|
-
const response = new Promise((resolve, reject) => {
|
|
1631
|
-
const timeout = globalThis.setTimeout(() => {
|
|
1632
|
-
const pending = this.pendingCommands.get(message.message_id);
|
|
1633
|
-
if (pending)
|
|
1634
|
-
globalThis.clearTimeout(pending.retry);
|
|
1635
|
-
pending?.markSettled();
|
|
1636
|
-
this.pendingCommands.delete(message.message_id);
|
|
1637
|
-
const detail = `Command ${message.type} timed out after ${timeoutMs}ms`;
|
|
1638
|
-
const error = message.type === "context.update"
|
|
1639
|
-
? new ContextDeliveryError(detail, { retryAfter: pending?.transferSettled })
|
|
1640
|
-
: new Error(detail);
|
|
1641
|
-
this.recordDiagnostic({
|
|
1642
|
-
operation: "command_send",
|
|
1643
|
-
status: "timed_out",
|
|
1644
|
-
reason: message.type === "context.update"
|
|
1645
|
-
? "context_command_timeout"
|
|
1646
|
-
: "command_timeout",
|
|
1647
|
-
message_id: message.message_id,
|
|
1648
|
-
connection_id: message.connection_id,
|
|
1649
|
-
client_seq: message.client_seq,
|
|
1650
|
-
context_key: contextKey,
|
|
1651
|
-
payload_bytes: payloadBytes,
|
|
1652
|
-
duration_ms: Date.now() - startedAt,
|
|
1653
|
-
retry_count: pending?.retryCount ?? 0,
|
|
1654
|
-
timeout_ms: timeoutMs,
|
|
1655
|
-
event_type: diagnosticEventType(message.type),
|
|
1656
|
-
});
|
|
1657
|
-
this.flushDiagnostics("command_timeout");
|
|
1658
|
-
reject(error);
|
|
1659
|
-
}, timeoutMs);
|
|
1660
|
-
this.pendingCommands.set(message.message_id, {
|
|
1661
|
-
commandType: message.type,
|
|
1662
|
-
message,
|
|
1663
|
-
startedAt,
|
|
1664
|
-
contextKey,
|
|
1665
|
-
payloadBytes,
|
|
1666
|
-
retryCount: 0,
|
|
1667
|
-
publishQueued: false,
|
|
1668
|
-
resolve,
|
|
1669
|
-
reject,
|
|
1670
|
-
timeout,
|
|
1671
|
-
settled,
|
|
1672
|
-
markSettled,
|
|
1673
|
-
retryTransfer,
|
|
1674
|
-
transferRetrying: false,
|
|
1675
|
-
transferRetryRequested: false,
|
|
1676
|
-
});
|
|
1677
|
-
this.scheduleCommandRetry(message);
|
|
1546
|
+
operation: "connection_recovery",
|
|
1547
|
+
stage: "token_request",
|
|
1548
|
+
status: response.ok ? "succeeded" : "failed",
|
|
1549
|
+
http_status: response.status,
|
|
1550
|
+
duration_ms: Date.now() - startedAt
|
|
1678
1551
|
});
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
const delayMs = message.type === "context.update"
|
|
1691
|
-
? contextRetryDelay(pending.retryCount + 1)
|
|
1692
|
-
: 500;
|
|
1693
|
-
pending.retry = globalThis.setTimeout(() => {
|
|
1694
|
-
void this.retryCommand(message).finally(() => this.scheduleCommandRetry(message));
|
|
1695
|
-
}, delayMs);
|
|
1696
|
-
}
|
|
1697
|
-
async retryCommand(message) {
|
|
1698
|
-
const pending = this.pendingCommands.get(message.message_id);
|
|
1699
|
-
if (!pending)
|
|
1700
|
-
return;
|
|
1701
|
-
if (message.type === "context.update" && (this.sessionEnded || ["reconnecting", "resuming", "disconnected", "failed"].includes(this.stateValue)))
|
|
1702
|
-
return;
|
|
1703
|
-
try {
|
|
1704
|
-
if (pending.transferRetryRequested && pending.retryTransfer) {
|
|
1705
|
-
pending.transferRetryRequested = false;
|
|
1706
|
-
pending.transferRetrying = true;
|
|
1707
|
-
const transfer = pending.retryTransfer();
|
|
1708
|
-
pending.transferSettled = transfer.then(() => { }, () => { });
|
|
1709
|
-
if (pending.contextKey) {
|
|
1710
|
-
this.replacementContext.blockUntilTransferSettles(pending.contextKey, pending.transferSettled);
|
|
1711
|
-
}
|
|
1712
|
-
await Promise.race([transfer, pending.settled]);
|
|
1713
|
-
}
|
|
1714
|
-
await this.publishCommandAttempt(message, true);
|
|
1715
|
-
}
|
|
1716
|
-
catch {
|
|
1717
|
-
// Failed writes retain their original deadline and retry with backoff.
|
|
1718
|
-
}
|
|
1719
|
-
finally {
|
|
1720
|
-
pending.transferRetrying = false;
|
|
1721
|
-
}
|
|
1722
|
-
}
|
|
1723
|
-
publishCommandAttempt(message, retrying) {
|
|
1724
|
-
const pending = this.pendingCommands.get(message.message_id);
|
|
1725
|
-
if (!pending || pending.publishQueued)
|
|
1726
|
-
return Promise.resolve();
|
|
1727
|
-
pending.publishQueued = true;
|
|
1728
|
-
// Context keys drain independently, but client_seq is global. Serialize the
|
|
1729
|
-
// asynchronous LiveKit publish boundary so a later sequence cannot overtake
|
|
1730
|
-
// an earlier one before the reliable data channel receives it.
|
|
1731
|
-
const attempt = this.commandPublishTail
|
|
1732
|
-
.then(() => this.publishCommandAttemptNow(message, retrying))
|
|
1733
|
-
.finally(() => {
|
|
1734
|
-
const current = this.pendingCommands.get(message.message_id);
|
|
1735
|
-
if (current === pending)
|
|
1736
|
-
current.publishQueued = false;
|
|
1552
|
+
if (!response.ok) this.flushDiagnostics("recovery_stage_failed");
|
|
1553
|
+
}
|
|
1554
|
+
return response;
|
|
1555
|
+
} catch (error) {
|
|
1556
|
+
if (recovering) {
|
|
1557
|
+
this.recordDiagnostic({
|
|
1558
|
+
operation: "connection_recovery",
|
|
1559
|
+
stage: "token_request",
|
|
1560
|
+
status: "failed",
|
|
1561
|
+
duration_ms: Date.now() - startedAt,
|
|
1562
|
+
reason: signal?.aborted ? "cancelled" : controller.signal.aborted ? "request_timeout" : "request_failed"
|
|
1737
1563
|
});
|
|
1738
|
-
this.
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1564
|
+
this.flushDiagnostics("recovery_stage_failed");
|
|
1565
|
+
}
|
|
1566
|
+
throw error;
|
|
1567
|
+
} finally {
|
|
1568
|
+
globalThis.clearTimeout(timeout);
|
|
1569
|
+
signal?.removeEventListener("abort", cancel);
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
async refreshCredentials(signal) {
|
|
1573
|
+
if (!this.refreshCredentialsCallback) return;
|
|
1574
|
+
throwIfAborted(signal);
|
|
1575
|
+
const connectionGeneration = this.contextGeneration;
|
|
1576
|
+
const refreshGeneration = ++this.credentialRefreshGeneration;
|
|
1577
|
+
const refreshed = await raceWithAbort(this.refreshCredentialsCallback(), signal);
|
|
1578
|
+
throwIfAborted(signal);
|
|
1579
|
+
if (connectionGeneration !== this.contextGeneration || refreshGeneration !== this.credentialRefreshGeneration) {
|
|
1580
|
+
throw new Error("Credential refresh was superseded");
|
|
1581
|
+
}
|
|
1582
|
+
validateSessionCredentials(refreshed);
|
|
1583
|
+
if (refreshed.id !== this.id) {
|
|
1584
|
+
throw new Error("Refreshed credentials belong to a different session");
|
|
1585
|
+
}
|
|
1586
|
+
this.accessToken = refreshed.access_token;
|
|
1587
|
+
this.credentials = refreshed.connection;
|
|
1588
|
+
}
|
|
1589
|
+
async finishSessionFailure(error) {
|
|
1590
|
+
this.sessionEnded = true;
|
|
1591
|
+
this.contextGeneration += 1;
|
|
1592
|
+
this.rejectPendingCommands(error);
|
|
1593
|
+
this.replacementContext.clear(error);
|
|
1594
|
+
this.failRecovery(error);
|
|
1595
|
+
this.cancelRecovery();
|
|
1596
|
+
this.disconnectMode = "connect_failure";
|
|
1597
|
+
try {
|
|
1598
|
+
await this.room.disconnect();
|
|
1599
|
+
} finally {
|
|
1600
|
+
this.disconnectMode = null;
|
|
1601
|
+
this.setState("failed");
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
observeConvenienceEvents(event) {
|
|
1605
|
+
if (event.type === "session.error") {
|
|
1606
|
+
if (this.sessionEnded) return;
|
|
1607
|
+
const error = new SessionErrorEventError(event);
|
|
1608
|
+
if (error.retryable && this.startupAttemptController) {
|
|
1609
|
+
this.startupAttemptController.abort(error);
|
|
1610
|
+
} else {
|
|
1611
|
+
this.terminalSessionError = error;
|
|
1612
|
+
this.abortStartupConnect(error);
|
|
1613
|
+
if (!this.startupConnectController) {
|
|
1614
|
+
void this.finishSessionFailure(error).catch(() => {
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
if (event.type === "session.ending" || event.type === "session.ended") {
|
|
1620
|
+
if (this.terminalSessionError) return;
|
|
1621
|
+
this.abortStartupConnect(new Error("Session ended during startup"));
|
|
1622
|
+
this.contextGeneration += 1;
|
|
1623
|
+
this.rejectPendingCommands(new Error("Session ended"));
|
|
1624
|
+
this.failRecovery(new Error("Session ended during recovery"));
|
|
1625
|
+
}
|
|
1626
|
+
if (event.type === "session.ending") {
|
|
1627
|
+
this.sessionEnded = true;
|
|
1628
|
+
this.cancelRecovery();
|
|
1629
|
+
this.replacementContext.clear(new Error("Session ended"));
|
|
1630
|
+
this.pendingEndEvent ??= event;
|
|
1631
|
+
if (!this.hasConnectedValue || this.agentDisconnected || this.room.state === LiveKitConnectionState.Disconnected) {
|
|
1632
|
+
this.finishPendingEnd();
|
|
1633
|
+
}
|
|
1634
|
+
}
|
|
1635
|
+
if (event.type === "session.ended") {
|
|
1636
|
+
const ending = this.pendingEndEvent;
|
|
1637
|
+
this.pendingEndEvent = null;
|
|
1638
|
+
this.sessionEnded = true;
|
|
1639
|
+
this.replacementContext.clear(new Error("Session ended"));
|
|
1640
|
+
this.cancelRecovery();
|
|
1641
|
+
this.notifyEnded(ending ? { ...ending, type: "session.ended" } : event);
|
|
1642
|
+
void this.disconnect().catch(() => {
|
|
1643
|
+
});
|
|
1644
|
+
}
|
|
1645
|
+
if (event.type.endsWith(".error")) this.notifyListeners(this.errorListeners, event);
|
|
1646
|
+
const message = this.transcriptStore.accept(event);
|
|
1647
|
+
if (message) this.notifyListeners(this.transcriptListeners, message);
|
|
1648
|
+
const transcript = transcriptEvent(event);
|
|
1649
|
+
if (!transcript) return;
|
|
1650
|
+
this.notifyListeners(
|
|
1651
|
+
transcript.role === "user" ? this.userTranscriptListeners : this.agentTranscriptListeners,
|
|
1652
|
+
transcript
|
|
1653
|
+
);
|
|
1654
|
+
}
|
|
1655
|
+
abortStartupConnect(error) {
|
|
1656
|
+
if (!this.startupConnectController?.signal.aborted) {
|
|
1657
|
+
this.startupConnectController?.abort(error);
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
async publishApplicationAudioTrack(audioTrack) {
|
|
1661
|
+
const clonedTrack = audioTrack.clone();
|
|
1662
|
+
let publishedTrack = null;
|
|
1663
|
+
try {
|
|
1664
|
+
publishedTrack = new LocalAudioTrack(clonedTrack, void 0, true);
|
|
1665
|
+
this.localApplicationAudioTrack = publishedTrack;
|
|
1666
|
+
if (!this.microphoneEnabled) {
|
|
1667
|
+
await publishedTrack.mute();
|
|
1668
|
+
}
|
|
1669
|
+
await this.room.localParticipant.publishTrack(publishedTrack, {
|
|
1670
|
+
source: Track.Source.Microphone,
|
|
1671
|
+
name: "voxevolv-application-audio"
|
|
1672
|
+
});
|
|
1673
|
+
} catch (error) {
|
|
1674
|
+
if (this.localApplicationAudioTrack === publishedTrack) {
|
|
1675
|
+
this.localApplicationAudioTrack = null;
|
|
1676
|
+
}
|
|
1677
|
+
clonedTrack.stop();
|
|
1678
|
+
throw error;
|
|
1679
|
+
}
|
|
1680
|
+
}
|
|
1681
|
+
async sendHello() {
|
|
1682
|
+
const reconnect = this.pendingReconnect;
|
|
1683
|
+
const message = createClientMessage(
|
|
1684
|
+
this.connectionId,
|
|
1685
|
+
++this.clientSeq,
|
|
1686
|
+
"client.hello",
|
|
1687
|
+
{
|
|
1688
|
+
last_server_seq: this.cursor.lastServerSeq,
|
|
1689
|
+
sdk: `@voxevolv/client@${SDK_VERSION}`,
|
|
1690
|
+
tool_handlers: [...this.toolHandlers.keys()].sort(),
|
|
1691
|
+
...reconnect ? { reconnect } : {}
|
|
1692
|
+
}
|
|
1693
|
+
);
|
|
1694
|
+
this.lastHelloMessageId = message.message_id;
|
|
1695
|
+
await this.sendTrackedCommand(message, CLIENT_HELLO_TIMEOUT_MS);
|
|
1696
|
+
if (this.pendingReconnect === reconnect) this.pendingReconnect = null;
|
|
1697
|
+
}
|
|
1698
|
+
async sendRecoveryHello(signal) {
|
|
1699
|
+
throwIfAborted(signal);
|
|
1700
|
+
let previousBytes = this.reliableDataChannelBufferedBytes();
|
|
1701
|
+
let lastProgressAt = Date.now();
|
|
1702
|
+
let timer;
|
|
1703
|
+
const stalled = new Promise((_resolve, reject) => {
|
|
1704
|
+
const checkProgress = () => {
|
|
1705
|
+
const bufferedBytes = this.reliableDataChannelBufferedBytes();
|
|
1706
|
+
if (bufferedBytes === void 0 || bufferedBytes === 0 || previousBytes === void 0 || previousBytes === 0 || bufferedBytes < previousBytes) {
|
|
1707
|
+
lastProgressAt = Date.now();
|
|
1708
|
+
} else if (Date.now() - lastProgressAt >= RECOVERY_BUFFER_STALL_MS) {
|
|
1709
|
+
reject(new RecoveryDataChannelStalledError());
|
|
1710
|
+
return;
|
|
1711
|
+
}
|
|
1712
|
+
previousBytes = bufferedBytes;
|
|
1713
|
+
timer = globalThis.setTimeout(checkProgress, RECOVERY_BUFFER_POLL_MS);
|
|
1714
|
+
};
|
|
1715
|
+
timer = globalThis.setTimeout(checkProgress, RECOVERY_BUFFER_POLL_MS);
|
|
1716
|
+
});
|
|
1717
|
+
try {
|
|
1718
|
+
await raceWithAbort(Promise.race([this.sendHello(), stalled]), signal);
|
|
1719
|
+
} catch (error) {
|
|
1720
|
+
if (error instanceof RecoveryDataChannelStalledError) {
|
|
1721
|
+
this.rejectPendingCommands(error);
|
|
1722
|
+
}
|
|
1723
|
+
throw error;
|
|
1724
|
+
} finally {
|
|
1725
|
+
if (timer !== void 0) globalThis.clearTimeout(timer);
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
async sendTrackedCommand(message, timeoutMs, retryTransfer) {
|
|
1729
|
+
const contextKey = contextKeyFromMessage(message);
|
|
1730
|
+
const payloadBytes = contextPayloadBytes(message.payload);
|
|
1731
|
+
const startedAt = Date.now();
|
|
1732
|
+
this.recordDiagnostic({
|
|
1733
|
+
operation: "command_send",
|
|
1734
|
+
status: "started",
|
|
1735
|
+
message_id: message.message_id,
|
|
1736
|
+
connection_id: message.connection_id,
|
|
1737
|
+
client_seq: message.client_seq,
|
|
1738
|
+
context_key: contextKey,
|
|
1739
|
+
payload_bytes: payloadBytes,
|
|
1740
|
+
retry_count: 0,
|
|
1741
|
+
timeout_ms: timeoutMs,
|
|
1742
|
+
event_type: diagnosticEventType(message.type)
|
|
1743
|
+
});
|
|
1744
|
+
let markSettled = () => {
|
|
1745
|
+
};
|
|
1746
|
+
const settled = new Promise((resolve) => {
|
|
1747
|
+
markSettled = resolve;
|
|
1748
|
+
});
|
|
1749
|
+
const response = new Promise((resolve, reject) => {
|
|
1750
|
+
const timeout = globalThis.setTimeout(() => {
|
|
1742
1751
|
const pending = this.pendingCommands.get(message.message_id);
|
|
1743
|
-
if (
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
operation: "command_send",
|
|
1749
|
-
status: "retrying",
|
|
1750
|
-
message_id: message.message_id,
|
|
1751
|
-
connection_id: message.connection_id,
|
|
1752
|
-
client_seq: message.client_seq,
|
|
1753
|
-
context_key: pending.contextKey,
|
|
1754
|
-
payload_bytes: pending.payloadBytes,
|
|
1755
|
-
retry_count: pending.retryCount,
|
|
1756
|
-
event_type: diagnosticEventType(message.type),
|
|
1757
|
-
});
|
|
1758
|
-
}
|
|
1759
|
-
const attemptStartedAt = Date.now();
|
|
1760
|
-
try {
|
|
1761
|
-
let publishFinished = false;
|
|
1762
|
-
const publish = this.publishCommand(message).then(() => { publishFinished = true; });
|
|
1763
|
-
await Promise.race([publish, pending.settled]);
|
|
1764
|
-
if (!publishFinished)
|
|
1765
|
-
return;
|
|
1766
|
-
}
|
|
1767
|
-
catch (error) {
|
|
1768
|
-
this.recordDiagnostic({
|
|
1769
|
-
operation: "command_send",
|
|
1770
|
-
status: "failed",
|
|
1771
|
-
reason: "command_publish_failed",
|
|
1772
|
-
message_id: message.message_id,
|
|
1773
|
-
connection_id: message.connection_id,
|
|
1774
|
-
client_seq: message.client_seq,
|
|
1775
|
-
context_key: pending.contextKey,
|
|
1776
|
-
payload_bytes: pending.payloadBytes,
|
|
1777
|
-
duration_ms: Date.now() - attemptStartedAt,
|
|
1778
|
-
retry_count: pending.retryCount,
|
|
1779
|
-
event_type: diagnosticEventType(message.type),
|
|
1780
|
-
});
|
|
1781
|
-
this.flushDiagnostics("command_failed");
|
|
1782
|
-
throw error;
|
|
1783
|
-
}
|
|
1784
|
-
const durationMs = Date.now() - attemptStartedAt;
|
|
1752
|
+
if (pending) globalThis.clearTimeout(pending.retry);
|
|
1753
|
+
pending?.markSettled();
|
|
1754
|
+
this.pendingCommands.delete(message.message_id);
|
|
1755
|
+
const detail = `Command ${message.type} timed out after ${timeoutMs}ms`;
|
|
1756
|
+
const error = message.type === "context.update" ? new ContextDeliveryError(detail, { retryAfter: pending?.transferSettled }) : new Error(detail);
|
|
1785
1757
|
this.recordDiagnostic({
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1758
|
+
operation: "command_send",
|
|
1759
|
+
status: "timed_out",
|
|
1760
|
+
reason: message.type === "context.update" ? "context_command_timeout" : "command_timeout",
|
|
1761
|
+
message_id: message.message_id,
|
|
1762
|
+
connection_id: message.connection_id,
|
|
1763
|
+
client_seq: message.client_seq,
|
|
1764
|
+
context_key: contextKey,
|
|
1765
|
+
payload_bytes: payloadBytes,
|
|
1766
|
+
duration_ms: Date.now() - startedAt,
|
|
1767
|
+
retry_count: pending?.retryCount ?? 0,
|
|
1768
|
+
timeout_ms: timeoutMs,
|
|
1769
|
+
event_type: diagnosticEventType(message.type)
|
|
1796
1770
|
});
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
const timeoutExpired = new Promise((_resolve, reject) => {
|
|
1846
|
-
timeout = globalThis.setTimeout(() => {
|
|
1847
|
-
controller.abort();
|
|
1848
|
-
reject(new Error(`Diagnostic report timed out after ${this.diagnosticReportTimeoutMs}ms`));
|
|
1849
|
-
}, this.diagnosticReportTimeoutMs);
|
|
1850
|
-
});
|
|
1851
|
-
try {
|
|
1852
|
-
const upload = async () => {
|
|
1853
|
-
const response = await this.fetcher(`${this.apiBase}/sessions/${this.id}/diagnostics`, {
|
|
1854
|
-
method: "POST",
|
|
1855
|
-
headers: {
|
|
1856
|
-
Authorization: `Bearer ${this.accessToken}`,
|
|
1857
|
-
"Content-Type": "application/json",
|
|
1858
|
-
},
|
|
1859
|
-
credentials: this.lifecycleCredentials,
|
|
1860
|
-
body: JSON.stringify(report),
|
|
1861
|
-
signal: controller.signal,
|
|
1862
|
-
keepalive: true,
|
|
1863
|
-
});
|
|
1864
|
-
const body = await response.json().catch(() => null);
|
|
1865
|
-
if (!response.ok) {
|
|
1866
|
-
const retryable = response.status === 408 || response.status === 429 || response.status >= 500;
|
|
1867
|
-
const headerDelay = Number(response.headers.get("Retry-After")) * 1000;
|
|
1868
|
-
const retryAfterMs = typeof body?.retry_after_ms === "number"
|
|
1869
|
-
? body.retry_after_ms : headerDelay;
|
|
1870
|
-
throw new DiagnosticDeliveryError(`Diagnostic report failed (${response.status})`, retryable, Number.isFinite(retryAfterMs) ? Math.max(retryAfterMs, 0) : 0);
|
|
1871
|
-
}
|
|
1872
|
-
if (body?.report_id !== report.report_id) {
|
|
1873
|
-
throw new DiagnosticDeliveryError("Diagnostic report was not acknowledged");
|
|
1874
|
-
}
|
|
1875
|
-
};
|
|
1876
|
-
// The receipt body is part of the timeout too, not just the response headers.
|
|
1877
|
-
await Promise.race([upload(), timeoutExpired]);
|
|
1878
|
-
}
|
|
1879
|
-
finally {
|
|
1880
|
-
if (timeout !== undefined)
|
|
1881
|
-
globalThis.clearTimeout(timeout);
|
|
1882
|
-
}
|
|
1883
|
-
}
|
|
1884
|
-
waitForRemoteAudioTrack(deadline, signal) {
|
|
1885
|
-
if ((this.outputMediaStream?.getAudioTracks().length ?? 0) > 0)
|
|
1886
|
-
return Promise.resolve();
|
|
1887
|
-
throwIfAborted(signal);
|
|
1888
|
-
return new Promise((resolve, reject) => {
|
|
1889
|
-
const cleanup = () => {
|
|
1890
|
-
globalThis.clearTimeout(timeout);
|
|
1891
|
-
signal?.removeEventListener("abort", onAbort);
|
|
1892
|
-
this.remoteAudioTrackResolve = null;
|
|
1893
|
-
};
|
|
1894
|
-
const timeout = globalThis.setTimeout(() => {
|
|
1895
|
-
cleanup();
|
|
1896
|
-
reject(new Error("Agent audio track did not arrive within the ready timeout;"
|
|
1897
|
-
+ " pass waitForRemoteAudio: false to resolve connect() on signaling readiness alone"));
|
|
1898
|
-
}, Math.max(deadline - Date.now(), 0));
|
|
1899
|
-
const onAbort = () => {
|
|
1900
|
-
cleanup();
|
|
1901
|
-
reject(abortError(signal));
|
|
1902
|
-
};
|
|
1903
|
-
this.remoteAudioTrackResolve = () => {
|
|
1904
|
-
cleanup();
|
|
1905
|
-
resolve();
|
|
1906
|
-
};
|
|
1907
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1908
|
-
});
|
|
1909
|
-
}
|
|
1910
|
-
waitForAgentReady(timeoutMs, signal) {
|
|
1911
|
-
if (this.agentReady)
|
|
1912
|
-
return Promise.resolve();
|
|
1913
|
-
throwIfAborted(signal);
|
|
1914
|
-
return new Promise((resolve, reject) => {
|
|
1915
|
-
const cleanup = () => {
|
|
1916
|
-
globalThis.clearTimeout(timeout);
|
|
1917
|
-
signal?.removeEventListener("abort", onAbort);
|
|
1918
|
-
this.readyResolve = null;
|
|
1919
|
-
};
|
|
1920
|
-
const timeout = globalThis.setTimeout(() => {
|
|
1921
|
-
cleanup();
|
|
1922
|
-
reject(new Error(`Agent did not become ready within ${timeoutMs}ms`));
|
|
1923
|
-
}, timeoutMs);
|
|
1924
|
-
const onAbort = () => {
|
|
1925
|
-
cleanup();
|
|
1926
|
-
reject(abortError(signal));
|
|
1927
|
-
};
|
|
1928
|
-
this.readyResolve = () => {
|
|
1929
|
-
cleanup();
|
|
1930
|
-
resolve();
|
|
1931
|
-
};
|
|
1932
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1771
|
+
this.flushDiagnostics("command_timeout");
|
|
1772
|
+
reject(error);
|
|
1773
|
+
}, timeoutMs);
|
|
1774
|
+
this.pendingCommands.set(
|
|
1775
|
+
message.message_id,
|
|
1776
|
+
{
|
|
1777
|
+
commandType: message.type,
|
|
1778
|
+
message,
|
|
1779
|
+
startedAt,
|
|
1780
|
+
contextKey,
|
|
1781
|
+
payloadBytes,
|
|
1782
|
+
retryCount: 0,
|
|
1783
|
+
publishQueued: false,
|
|
1784
|
+
resolve,
|
|
1785
|
+
reject,
|
|
1786
|
+
timeout,
|
|
1787
|
+
settled,
|
|
1788
|
+
markSettled,
|
|
1789
|
+
retryTransfer,
|
|
1790
|
+
transferRetrying: false,
|
|
1791
|
+
transferRetryRequested: false
|
|
1792
|
+
}
|
|
1793
|
+
);
|
|
1794
|
+
this.scheduleCommandRetry(message);
|
|
1795
|
+
});
|
|
1796
|
+
void this.publishCommandAttempt(message, false).catch(() => {
|
|
1797
|
+
});
|
|
1798
|
+
return response;
|
|
1799
|
+
}
|
|
1800
|
+
scheduleCommandRetry(message) {
|
|
1801
|
+
const pending = this.pendingCommands.get(message.message_id);
|
|
1802
|
+
if (!pending) return;
|
|
1803
|
+
const delayMs = message.type === "context.update" ? contextRetryDelay(pending.retryCount + 1) : 500;
|
|
1804
|
+
pending.retry = globalThis.setTimeout(() => {
|
|
1805
|
+
void this.retryCommand(message).finally(() => this.scheduleCommandRetry(message));
|
|
1806
|
+
}, delayMs);
|
|
1807
|
+
}
|
|
1808
|
+
async retryCommand(message) {
|
|
1809
|
+
const pending = this.pendingCommands.get(message.message_id);
|
|
1810
|
+
if (!pending) return;
|
|
1811
|
+
if (message.type === "context.update" && (this.sessionEnded || ["reconnecting", "resuming", "disconnected", "failed"].includes(this.stateValue))) return;
|
|
1812
|
+
try {
|
|
1813
|
+
if (pending.transferRetryRequested && pending.retryTransfer) {
|
|
1814
|
+
pending.transferRetryRequested = false;
|
|
1815
|
+
pending.transferRetrying = true;
|
|
1816
|
+
const transfer = pending.retryTransfer();
|
|
1817
|
+
pending.transferSettled = transfer.then(() => {
|
|
1818
|
+
}, () => {
|
|
1933
1819
|
});
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
1820
|
+
if (pending.contextKey) {
|
|
1821
|
+
this.replacementContext.blockUntilTransferSettles(pending.contextKey, pending.transferSettled);
|
|
1822
|
+
}
|
|
1823
|
+
await Promise.race([transfer, pending.settled]);
|
|
1824
|
+
}
|
|
1825
|
+
await this.publishCommandAttempt(message, true);
|
|
1826
|
+
} catch {
|
|
1827
|
+
} finally {
|
|
1828
|
+
pending.transferRetrying = false;
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
publishCommandAttempt(message, retrying) {
|
|
1832
|
+
const pending = this.pendingCommands.get(message.message_id);
|
|
1833
|
+
if (!pending || pending.publishQueued) return Promise.resolve();
|
|
1834
|
+
pending.publishQueued = true;
|
|
1835
|
+
const attempt = this.commandPublishTail.then(() => this.publishCommandAttemptNow(message, retrying)).finally(() => {
|
|
1836
|
+
const current = this.pendingCommands.get(message.message_id);
|
|
1837
|
+
if (current === pending) current.publishQueued = false;
|
|
1838
|
+
});
|
|
1839
|
+
this.commandPublishTail = attempt.catch(() => {
|
|
1840
|
+
});
|
|
1841
|
+
return attempt;
|
|
1842
|
+
}
|
|
1843
|
+
async publishCommandAttemptNow(message, retrying) {
|
|
1844
|
+
const pending = this.pendingCommands.get(message.message_id);
|
|
1845
|
+
if (!pending) return;
|
|
1846
|
+
if (retrying) {
|
|
1847
|
+
pending.retryCount += 1;
|
|
1848
|
+
this.recordDiagnostic({
|
|
1849
|
+
operation: "command_send",
|
|
1850
|
+
status: "retrying",
|
|
1851
|
+
message_id: message.message_id,
|
|
1852
|
+
connection_id: message.connection_id,
|
|
1853
|
+
client_seq: message.client_seq,
|
|
1854
|
+
context_key: pending.contextKey,
|
|
1855
|
+
payload_bytes: pending.payloadBytes,
|
|
1856
|
+
retry_count: pending.retryCount,
|
|
1857
|
+
event_type: diagnosticEventType(message.type)
|
|
1858
|
+
});
|
|
1859
|
+
}
|
|
1860
|
+
const attemptStartedAt = Date.now();
|
|
1861
|
+
try {
|
|
1862
|
+
let publishFinished = false;
|
|
1863
|
+
const publish = this.publishCommand(message).then(() => {
|
|
1864
|
+
publishFinished = true;
|
|
1865
|
+
});
|
|
1866
|
+
await Promise.race([publish, pending.settled]);
|
|
1867
|
+
if (!publishFinished) return;
|
|
1868
|
+
} catch (error) {
|
|
1869
|
+
this.recordDiagnostic({
|
|
1870
|
+
operation: "command_send",
|
|
1871
|
+
status: "failed",
|
|
1872
|
+
reason: "command_publish_failed",
|
|
1873
|
+
message_id: message.message_id,
|
|
1874
|
+
connection_id: message.connection_id,
|
|
1875
|
+
client_seq: message.client_seq,
|
|
1876
|
+
context_key: pending.contextKey,
|
|
1877
|
+
payload_bytes: pending.payloadBytes,
|
|
1878
|
+
duration_ms: Date.now() - attemptStartedAt,
|
|
1879
|
+
retry_count: pending.retryCount,
|
|
1880
|
+
event_type: diagnosticEventType(message.type)
|
|
1881
|
+
});
|
|
1882
|
+
this.flushDiagnostics("command_failed");
|
|
1883
|
+
throw error;
|
|
1884
|
+
}
|
|
1885
|
+
const durationMs = Date.now() - attemptStartedAt;
|
|
1886
|
+
this.recordDiagnostic({
|
|
1887
|
+
operation: "command_send",
|
|
1888
|
+
status: "succeeded",
|
|
1889
|
+
message_id: message.message_id,
|
|
1890
|
+
connection_id: message.connection_id,
|
|
1891
|
+
client_seq: message.client_seq,
|
|
1892
|
+
context_key: pending.contextKey,
|
|
1893
|
+
payload_bytes: pending.payloadBytes,
|
|
1894
|
+
duration_ms: durationMs,
|
|
1895
|
+
retry_count: pending.retryCount,
|
|
1896
|
+
event_type: diagnosticEventType(message.type)
|
|
1897
|
+
});
|
|
1898
|
+
if (durationMs > 2e3) this.flushDiagnostics("slow_operation");
|
|
1899
|
+
}
|
|
1900
|
+
async publishCommand(message) {
|
|
1901
|
+
await this.room.localParticipant.publishData(
|
|
1902
|
+
new TextEncoder().encode(JSON.stringify(message)),
|
|
1903
|
+
{ reliable: true, topic: "voxevolv.commands" }
|
|
1904
|
+
);
|
|
1905
|
+
}
|
|
1906
|
+
startDiagnosticLoopMonitor() {
|
|
1907
|
+
if (this.diagnosticLoopTimer !== void 0) return;
|
|
1908
|
+
this.diagnosticLoopExpectedAt = performance.now() + 1e3;
|
|
1909
|
+
this.diagnosticLoopTimer = globalThis.setInterval(() => {
|
|
1910
|
+
const now = performance.now();
|
|
1911
|
+
this.diagnosticLoopDelayMs = Math.max(0, now - this.diagnosticLoopExpectedAt);
|
|
1912
|
+
this.diagnosticLoopExpectedAt = now + 1e3;
|
|
1913
|
+
}, 1e3);
|
|
1914
|
+
this.diagnosticLoopTimer.unref?.();
|
|
1915
|
+
}
|
|
1916
|
+
recordDiagnostic(entry) {
|
|
1917
|
+
const snapshot = {
|
|
1918
|
+
...entry,
|
|
1919
|
+
connection_state: this.stateValue,
|
|
1920
|
+
buffered_bytes: this.reliableDataChannelBufferedBytes()
|
|
1921
|
+
};
|
|
1922
|
+
if (entry.operation === "tool_rpc") {
|
|
1923
|
+
snapshot.page_visibility = globalThis.document?.visibilityState;
|
|
1924
|
+
snapshot.active_context_transfers = this.activeContextTransfers;
|
|
1925
|
+
snapshot.active_context_bytes = this.activeContextBytes;
|
|
1926
|
+
if (this.diagnosticLoopTimer !== void 0) {
|
|
1927
|
+
snapshot.event_loop_delay_ms = Math.max(
|
|
1928
|
+
this.diagnosticLoopDelayMs ?? 0,
|
|
1929
|
+
performance.now() - this.diagnosticLoopExpectedAt
|
|
1930
|
+
);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
this.diagnostics.record(snapshot);
|
|
1934
|
+
}
|
|
1935
|
+
reliableDataChannelBufferedBytes() {
|
|
1936
|
+
try {
|
|
1937
|
+
const bufferedAmount = this.room.engine.dataChannelForKind(0)?.bufferedAmount;
|
|
1938
|
+
return typeof bufferedAmount === "number" ? bufferedAmount : void 0;
|
|
1939
|
+
} catch {
|
|
1940
|
+
return void 0;
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
flushDiagnostics(trigger) {
|
|
1944
|
+
void this.diagnostics.flush(trigger, (report) => this.postDiagnostics(report));
|
|
1945
|
+
}
|
|
1946
|
+
async postDiagnostics(report) {
|
|
1947
|
+
const controller = new AbortController();
|
|
1948
|
+
let timeout;
|
|
1949
|
+
const timeoutExpired = new Promise((_resolve, reject) => {
|
|
1950
|
+
timeout = globalThis.setTimeout(() => {
|
|
1951
|
+
controller.abort();
|
|
1952
|
+
reject(new Error(
|
|
1953
|
+
`Diagnostic report timed out after ${this.diagnosticReportTimeoutMs}ms`
|
|
1954
|
+
));
|
|
1955
|
+
}, this.diagnosticReportTimeoutMs);
|
|
1956
|
+
});
|
|
1957
|
+
try {
|
|
1958
|
+
const upload = async () => {
|
|
1959
|
+
const response = await this.fetcher(`${this.apiBase}/sessions/${this.id}/diagnostics`, {
|
|
1960
|
+
method: "POST",
|
|
1961
|
+
headers: {
|
|
1962
|
+
Authorization: `Bearer ${this.accessToken}`,
|
|
1963
|
+
"Content-Type": "application/json"
|
|
1964
|
+
},
|
|
1965
|
+
credentials: this.lifecycleCredentials,
|
|
1966
|
+
body: JSON.stringify(report),
|
|
1967
|
+
signal: controller.signal,
|
|
1968
|
+
keepalive: true
|
|
2032
1969
|
});
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
1970
|
+
const body = await response.json().catch(() => null);
|
|
1971
|
+
if (!response.ok) {
|
|
1972
|
+
const retryable = response.status === 408 || response.status === 429 || response.status >= 500;
|
|
1973
|
+
const headerDelay = Number(response.headers.get("Retry-After")) * 1e3;
|
|
1974
|
+
const retryAfterMs = typeof body?.retry_after_ms === "number" ? body.retry_after_ms : headerDelay;
|
|
1975
|
+
throw new DiagnosticDeliveryError(
|
|
1976
|
+
`Diagnostic report failed (${response.status})`,
|
|
1977
|
+
retryable,
|
|
1978
|
+
Number.isFinite(retryAfterMs) ? Math.max(retryAfterMs, 0) : 0
|
|
1979
|
+
);
|
|
1980
|
+
}
|
|
1981
|
+
if (body?.report_id !== report.report_id) {
|
|
1982
|
+
throw new DiagnosticDeliveryError("Diagnostic report was not acknowledged");
|
|
1983
|
+
}
|
|
1984
|
+
};
|
|
1985
|
+
await Promise.race([upload(), timeoutExpired]);
|
|
1986
|
+
} finally {
|
|
1987
|
+
if (timeout !== void 0) globalThis.clearTimeout(timeout);
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
waitForRemoteAudioTrack(deadline, signal) {
|
|
1991
|
+
if ((this.outputMediaStream?.getAudioTracks().length ?? 0) > 0) return Promise.resolve();
|
|
1992
|
+
throwIfAborted(signal);
|
|
1993
|
+
return new Promise((resolve, reject) => {
|
|
1994
|
+
const cleanup = () => {
|
|
1995
|
+
globalThis.clearTimeout(timeout);
|
|
1996
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1997
|
+
this.remoteAudioTrackResolve = null;
|
|
1998
|
+
};
|
|
1999
|
+
const timeout = globalThis.setTimeout(() => {
|
|
2000
|
+
cleanup();
|
|
2001
|
+
reject(new Error(
|
|
2002
|
+
"Agent audio track did not arrive within the ready timeout; pass waitForRemoteAudio: false to resolve connect() on signaling readiness alone"
|
|
2003
|
+
));
|
|
2004
|
+
}, Math.max(deadline - Date.now(), 0));
|
|
2005
|
+
const onAbort = () => {
|
|
2006
|
+
cleanup();
|
|
2007
|
+
reject(abortError(signal));
|
|
2008
|
+
};
|
|
2009
|
+
this.remoteAudioTrackResolve = () => {
|
|
2010
|
+
cleanup();
|
|
2011
|
+
resolve();
|
|
2012
|
+
};
|
|
2013
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2014
|
+
});
|
|
2015
|
+
}
|
|
2016
|
+
waitForAgentReady(timeoutMs, signal) {
|
|
2017
|
+
if (this.agentReady) return Promise.resolve();
|
|
2018
|
+
throwIfAborted(signal);
|
|
2019
|
+
return new Promise((resolve, reject) => {
|
|
2020
|
+
const cleanup = () => {
|
|
2021
|
+
globalThis.clearTimeout(timeout);
|
|
2022
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2023
|
+
this.readyResolve = null;
|
|
2024
|
+
};
|
|
2025
|
+
const timeout = globalThis.setTimeout(() => {
|
|
2026
|
+
cleanup();
|
|
2027
|
+
reject(new Error(`Agent did not become ready within ${timeoutMs}ms`));
|
|
2028
|
+
}, timeoutMs);
|
|
2029
|
+
const onAbort = () => {
|
|
2030
|
+
cleanup();
|
|
2031
|
+
reject(abortError(signal));
|
|
2032
|
+
};
|
|
2033
|
+
this.readyResolve = () => {
|
|
2034
|
+
cleanup();
|
|
2035
|
+
resolve();
|
|
2036
|
+
};
|
|
2037
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2038
|
+
});
|
|
2039
|
+
}
|
|
2040
|
+
observeReadiness(event) {
|
|
2041
|
+
if (event.stream_id === "durable-replay") return;
|
|
2042
|
+
const payload = event.payload;
|
|
2043
|
+
const snapshotReady = event.type === "agent.snapshot" && payload.status === "ready" && payload.connection_id === this.connectionId && payload.ack_message_id === this.lastHelloMessageId;
|
|
2044
|
+
if (event.type !== "agent.ready" && !snapshotReady) return;
|
|
2045
|
+
if (this.agentStreamId && this.agentStreamId !== event.stream_id && !this.activeRecovery) {
|
|
2046
|
+
this.startAutoRejoin(
|
|
2047
|
+
new Error("Agent worker was replaced"),
|
|
2048
|
+
{ trigger: "worker_replaced" }
|
|
2049
|
+
);
|
|
2050
|
+
return;
|
|
2051
|
+
}
|
|
2052
|
+
this.agentStreamId = event.stream_id;
|
|
2053
|
+
this.agentDisconnected = false;
|
|
2054
|
+
const wasReady = this.agentReady;
|
|
2055
|
+
this.agentReady = true;
|
|
2056
|
+
this.readyResolve?.();
|
|
2057
|
+
if (!wasReady) this.notifyListeners(this.readyListeners, event);
|
|
2058
|
+
}
|
|
2059
|
+
observeCommandResult(event) {
|
|
2060
|
+
if (event.type !== "command.ack" && event.type !== "command.error" && event.type !== "context.updated" && event.type !== "context.rejected") return;
|
|
2061
|
+
const pending = this.pendingCommands.get(String(event.payload.message_id ?? ""));
|
|
2062
|
+
if (!pending) return;
|
|
2063
|
+
if (event.type === "command.ack" && isCommandAckPayload(event.payload)) {
|
|
2064
|
+
this.recordCommandResult(pending, event.type, "acknowledged");
|
|
2065
|
+
globalThis.clearTimeout(pending.timeout);
|
|
2066
|
+
globalThis.clearTimeout(pending.retry);
|
|
2067
|
+
pending.markSettled();
|
|
2068
|
+
this.pendingCommands.delete(event.payload.message_id);
|
|
2069
|
+
pending.resolve(event.payload);
|
|
2070
|
+
return;
|
|
2071
|
+
}
|
|
2072
|
+
if (event.type === "context.updated" && isContextUpdatedPayload(event.payload)) {
|
|
2073
|
+
this.recordCommandResult(pending, event.type, "acknowledged");
|
|
2074
|
+
globalThis.clearTimeout(pending.timeout);
|
|
2075
|
+
globalThis.clearTimeout(pending.retry);
|
|
2076
|
+
pending.markSettled();
|
|
2077
|
+
this.pendingCommands.delete(event.payload.message_id);
|
|
2078
|
+
pending.resolve(event.payload);
|
|
2079
|
+
return;
|
|
2080
|
+
}
|
|
2081
|
+
if (event.type === "command.error" && isCommandErrorPayload(event.payload) || event.type === "context.rejected" && isContextRejectedPayload(event.payload)) {
|
|
2082
|
+
this.recordCommandResult(pending, event.type, "rejected", event.payload.code);
|
|
2083
|
+
this.flushDiagnostics("command_rejected");
|
|
2084
|
+
if (event.type === "context.rejected" && event.payload.retryable) {
|
|
2085
|
+
if (event.payload.code === "stream_unavailable" && pending.retryTransfer && !pending.transferRetrying) {
|
|
2086
|
+
pending.transferRetryRequested = true;
|
|
2087
|
+
}
|
|
2088
|
+
return;
|
|
2089
|
+
}
|
|
2090
|
+
globalThis.clearTimeout(pending.timeout);
|
|
2091
|
+
globalThis.clearTimeout(pending.retry);
|
|
2092
|
+
pending.markSettled();
|
|
2093
|
+
this.pendingCommands.delete(event.payload.message_id);
|
|
2094
|
+
pending.reject(new CommandError(event.payload));
|
|
2095
|
+
}
|
|
2096
|
+
}
|
|
2097
|
+
rejectPendingCommands(error, preserveContext = false) {
|
|
2098
|
+
for (const [messageId, pending] of this.pendingCommands) {
|
|
2099
|
+
if (preserveContext && pending.commandType === "context.update") continue;
|
|
2100
|
+
this.recordCommandResult(
|
|
2101
|
+
pending,
|
|
2102
|
+
pending.commandType === "context.update" ? "context.rejected" : "command.error",
|
|
2103
|
+
"failed",
|
|
2104
|
+
"pending_command_cancelled"
|
|
2105
|
+
);
|
|
2106
|
+
globalThis.clearTimeout(pending.timeout);
|
|
2107
|
+
globalThis.clearTimeout(pending.retry);
|
|
2108
|
+
pending.markSettled();
|
|
2109
|
+
pending.reject(pending.commandType === "context.update" ? new ContextDeliveryError(error.message, { cause: error, retryAfter: pending.transferSettled }) : error);
|
|
2110
|
+
this.pendingCommands.delete(messageId);
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
recordCommandResult(pending, eventType, status, reason) {
|
|
2114
|
+
const durationMs = Date.now() - pending.startedAt;
|
|
2115
|
+
this.recordDiagnostic({
|
|
2116
|
+
operation: "command_result",
|
|
2117
|
+
status,
|
|
2118
|
+
reason,
|
|
2119
|
+
message_id: pending.message.message_id,
|
|
2120
|
+
connection_id: pending.message.connection_id,
|
|
2121
|
+
client_seq: pending.message.client_seq,
|
|
2122
|
+
context_key: pending.contextKey,
|
|
2123
|
+
payload_bytes: pending.payloadBytes,
|
|
2124
|
+
duration_ms: durationMs,
|
|
2125
|
+
retry_count: pending.retryCount,
|
|
2126
|
+
event_type: eventType
|
|
2127
|
+
});
|
|
2128
|
+
if (durationMs > 2e3) this.flushDiagnostics("slow_operation");
|
|
2129
|
+
}
|
|
2130
|
+
async fetchReplay(apiBase, afterSeq, options = {}) {
|
|
2131
|
+
const { signal, terminalOnly = false } = options;
|
|
2132
|
+
let nextAfter = afterSeq;
|
|
2133
|
+
while (true) {
|
|
2134
|
+
throwIfAborted(signal);
|
|
2135
|
+
const url = `${apiBase}/sessions/${this.id}/events?after_seq=${nextAfter}&limit=200` + (terminalOnly ? "&terminal_only=true" : "");
|
|
2136
|
+
let response = await raceWithAbort(this.fetcher(
|
|
2137
|
+
url,
|
|
2138
|
+
{
|
|
2139
|
+
headers: { Authorization: `Bearer ${this.accessToken}` },
|
|
2140
|
+
credentials: this.lifecycleCredentials,
|
|
2141
|
+
signal
|
|
2142
|
+
}
|
|
2143
|
+
), signal);
|
|
2144
|
+
if (response.status === 401 && this.refreshCredentialsCallback) {
|
|
2145
|
+
await this.refreshCredentials(signal);
|
|
2146
|
+
response = await raceWithAbort(this.fetcher(
|
|
2147
|
+
url,
|
|
2148
|
+
{
|
|
2149
|
+
headers: { Authorization: `Bearer ${this.accessToken}` },
|
|
2150
|
+
credentials: this.lifecycleCredentials,
|
|
2151
|
+
signal
|
|
2152
|
+
}
|
|
2153
|
+
), signal);
|
|
2154
|
+
}
|
|
2155
|
+
if (!response.ok) {
|
|
2156
|
+
throw await raceWithAbort(httpError(response, "Could not replay session events"), signal);
|
|
2157
|
+
}
|
|
2158
|
+
const page = await raceWithAbort(response.json(), signal);
|
|
2159
|
+
throwIfAborted(signal);
|
|
2160
|
+
for (const stored of page.items) {
|
|
2161
|
+
nextAfter = Math.max(nextAfter, stored.server_seq);
|
|
2162
|
+
const event = {
|
|
2163
|
+
protocol_version: 1,
|
|
2164
|
+
event_id: stored.event_id,
|
|
2165
|
+
stream_id: "durable-replay",
|
|
2166
|
+
stream_seq: Math.max(this.cursor.lastStreamSeq, 1),
|
|
2167
|
+
server_seq: stored.server_seq,
|
|
2168
|
+
ts: stored.created_at,
|
|
2169
|
+
type: stored.type,
|
|
2170
|
+
payload: stored.payload
|
|
2171
|
+
};
|
|
2172
|
+
if (!this.cursor.acceptReplay(event)) continue;
|
|
2173
|
+
this.observeReadiness(event);
|
|
2174
|
+
this.observeConvenienceEvents(event);
|
|
2175
|
+
this.notifyListeners(this.listeners, event);
|
|
2176
|
+
}
|
|
2177
|
+
nextAfter = Math.max(nextAfter, page.next_after_seq);
|
|
2178
|
+
if (!page.has_more) return;
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
};
|
|
2085
2182
|
function resolveAutoRejoinOptions(value) {
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
return resolved;
|
|
2183
|
+
if (!value) return null;
|
|
2184
|
+
const options = value === true ? {} : value;
|
|
2185
|
+
const resolved = {
|
|
2186
|
+
maxAttempts: options.maxAttempts ?? Number.MAX_SAFE_INTEGER,
|
|
2187
|
+
baseDelayMs: options.baseDelayMs ?? 750,
|
|
2188
|
+
maxDelayMs: options.maxDelayMs ?? 5e3,
|
|
2189
|
+
timeoutMs: options.timeoutMs ?? 18e4
|
|
2190
|
+
};
|
|
2191
|
+
if (!Number.isFinite(resolved.timeoutMs) || resolved.timeoutMs <= 0) {
|
|
2192
|
+
throw new TypeError("autoRejoin.timeoutMs must be a positive finite number");
|
|
2193
|
+
}
|
|
2194
|
+
if (!Number.isInteger(resolved.maxAttempts) || resolved.maxAttempts < 1) {
|
|
2195
|
+
throw new TypeError("autoRejoin.maxAttempts must be a positive integer");
|
|
2196
|
+
}
|
|
2197
|
+
if (!Number.isFinite(resolved.baseDelayMs) || resolved.baseDelayMs < 0) {
|
|
2198
|
+
throw new TypeError("autoRejoin.baseDelayMs must be a non-negative finite number");
|
|
2199
|
+
}
|
|
2200
|
+
if (!Number.isFinite(resolved.maxDelayMs) || resolved.maxDelayMs < 0) {
|
|
2201
|
+
throw new TypeError("autoRejoin.maxDelayMs must be a non-negative finite number");
|
|
2202
|
+
}
|
|
2203
|
+
return resolved;
|
|
2108
2204
|
}
|
|
2109
2205
|
function resolveStartupRetryOptions(value) {
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2206
|
+
if (value === false) return null;
|
|
2207
|
+
const options = typeof value === "object" ? value : {};
|
|
2208
|
+
const attempts = (options.attempts ?? DEFAULT_STARTUP_ATTEMPTS).map((attempt) => ({
|
|
2209
|
+
readyTimeoutMs: attempt.readyTimeoutMs,
|
|
2210
|
+
delayBeforeMs: attempt.delayBeforeMs ?? 0
|
|
2211
|
+
}));
|
|
2212
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS;
|
|
2213
|
+
if (attempts.length === 0) {
|
|
2214
|
+
throw new TypeError("startupRetry.attempts must contain at least one attempt");
|
|
2215
|
+
}
|
|
2216
|
+
for (const attempt of attempts) {
|
|
2217
|
+
if (!Number.isFinite(attempt.readyTimeoutMs) || attempt.readyTimeoutMs <= 0) {
|
|
2218
|
+
throw new TypeError(
|
|
2219
|
+
"startupRetry.attempts readyTimeoutMs must be a positive finite number"
|
|
2220
|
+
);
|
|
2221
|
+
}
|
|
2222
|
+
if (!Number.isFinite(attempt.delayBeforeMs) || attempt.delayBeforeMs < 0) {
|
|
2223
|
+
throw new TypeError(
|
|
2224
|
+
"startupRetry.attempts delayBeforeMs must be a non-negative finite number"
|
|
2225
|
+
);
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2228
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
2229
|
+
throw new TypeError("startupRetry.timeoutMs must be a positive finite number");
|
|
2230
|
+
}
|
|
2231
|
+
return { attempts, timeoutMs };
|
|
2133
2232
|
}
|
|
2134
2233
|
function isTerminalDisconnectReason(reason) {
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2234
|
+
return reason !== void 0 && [
|
|
2235
|
+
DisconnectReason.CLIENT_INITIATED,
|
|
2236
|
+
DisconnectReason.DUPLICATE_IDENTITY,
|
|
2237
|
+
DisconnectReason.PARTICIPANT_REMOVED,
|
|
2238
|
+
DisconnectReason.ROOM_DELETED,
|
|
2239
|
+
DisconnectReason.ROOM_CLOSED,
|
|
2240
|
+
DisconnectReason.USER_UNAVAILABLE,
|
|
2241
|
+
DisconnectReason.USER_REJECTED
|
|
2242
|
+
].includes(reason);
|
|
2144
2243
|
}
|
|
2145
2244
|
function disconnectReasonName(reason) {
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
return DisconnectReason[reason] ?? String(reason);
|
|
2245
|
+
if (reason === void 0) return "unknown";
|
|
2246
|
+
return DisconnectReason[reason] ?? String(reason);
|
|
2149
2247
|
}
|
|
2150
2248
|
function isTerminalRejoinError(error) {
|
|
2151
|
-
|
|
2152
|
-
&& [401, 403, 409].includes(error.status);
|
|
2249
|
+
return error instanceof RealtimeSessionHttpError && [401, 403, 409].includes(error.status);
|
|
2153
2250
|
}
|
|
2154
2251
|
function isTerminalStartupError(error) {
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
"NotReadableError",
|
|
2173
|
-
"NotSupportedError",
|
|
2174
|
-
"OverconstrainedError",
|
|
2175
|
-
"SecurityError",
|
|
2176
|
-
"TrackStartError",
|
|
2177
|
-
].includes(error.name);
|
|
2252
|
+
if (error instanceof SessionErrorEventError) return !error.retryable;
|
|
2253
|
+
if (error instanceof RealtimeSessionHttpError) {
|
|
2254
|
+
return error.status >= 400 && error.status < 500 && ![408, 425, 429].includes(error.status);
|
|
2255
|
+
}
|
|
2256
|
+
if (error instanceof SessionConfigurationError) return true;
|
|
2257
|
+
return error instanceof Error && [
|
|
2258
|
+
"ConstraintNotSatisfiedError",
|
|
2259
|
+
"DevicesNotFoundError",
|
|
2260
|
+
"InvalidStateError",
|
|
2261
|
+
"NotAllowedError",
|
|
2262
|
+
"NotFoundError",
|
|
2263
|
+
"NotReadableError",
|
|
2264
|
+
"NotSupportedError",
|
|
2265
|
+
"OverconstrainedError",
|
|
2266
|
+
"SecurityError",
|
|
2267
|
+
"TrackStartError"
|
|
2268
|
+
].includes(error.name);
|
|
2178
2269
|
}
|
|
2179
2270
|
function toError(error) {
|
|
2180
|
-
|
|
2271
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
2181
2272
|
}
|
|
2182
2273
|
function abortError(signal) {
|
|
2183
|
-
|
|
2274
|
+
return signal.reason ?? new DOMException("Aborted", "AbortError");
|
|
2184
2275
|
}
|
|
2185
2276
|
function createLinkedAbortController(parentSignal) {
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2277
|
+
const controller = new AbortController();
|
|
2278
|
+
if (parentSignal.aborted) {
|
|
2279
|
+
controller.abort(abortError(parentSignal));
|
|
2280
|
+
} else {
|
|
2281
|
+
parentSignal.addEventListener(
|
|
2282
|
+
"abort",
|
|
2283
|
+
() => controller.abort(abortError(parentSignal)),
|
|
2284
|
+
{ once: true }
|
|
2285
|
+
);
|
|
2286
|
+
}
|
|
2287
|
+
return controller;
|
|
2194
2288
|
}
|
|
2195
2289
|
function throwIfAborted(signal) {
|
|
2196
|
-
|
|
2197
|
-
throw abortError(signal);
|
|
2290
|
+
if (signal?.aborted) throw abortError(signal);
|
|
2198
2291
|
}
|
|
2199
2292
|
function raceWithAbort(promise, signal) {
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
2224
|
-
}
|
|
2225
|
-
// Always observe the operation, including when it aborted the signal before
|
|
2226
|
-
// returning its promise, so a later rejection cannot escape to the caller.
|
|
2227
|
-
promise.then(resolveOnce, rejectOnce);
|
|
2228
|
-
});
|
|
2293
|
+
if (!signal) return promise;
|
|
2294
|
+
return new Promise((resolve, reject) => {
|
|
2295
|
+
let settled = false;
|
|
2296
|
+
const resolveOnce = (value) => {
|
|
2297
|
+
if (settled) return;
|
|
2298
|
+
settled = true;
|
|
2299
|
+
signal.removeEventListener("abort", onAbort);
|
|
2300
|
+
resolve(value);
|
|
2301
|
+
};
|
|
2302
|
+
const rejectOnce = (error) => {
|
|
2303
|
+
if (settled) return;
|
|
2304
|
+
settled = true;
|
|
2305
|
+
signal.removeEventListener("abort", onAbort);
|
|
2306
|
+
reject(error);
|
|
2307
|
+
};
|
|
2308
|
+
const onAbort = () => rejectOnce(abortError(signal));
|
|
2309
|
+
if (signal.aborted) {
|
|
2310
|
+
onAbort();
|
|
2311
|
+
} else {
|
|
2312
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2313
|
+
}
|
|
2314
|
+
promise.then(resolveOnce, rejectOnce);
|
|
2315
|
+
});
|
|
2229
2316
|
}
|
|
2230
2317
|
function abortableDelay(delayMs, signal) {
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
});
|
|
2318
|
+
throwIfAborted(signal);
|
|
2319
|
+
if (delayMs <= 0) return Promise.resolve();
|
|
2320
|
+
return new Promise((resolve, reject) => {
|
|
2321
|
+
const timeout = globalThis.setTimeout(() => {
|
|
2322
|
+
signal.removeEventListener("abort", onAbort);
|
|
2323
|
+
resolve();
|
|
2324
|
+
}, delayMs);
|
|
2325
|
+
const onAbort = () => {
|
|
2326
|
+
globalThis.clearTimeout(timeout);
|
|
2327
|
+
reject(abortError(signal));
|
|
2328
|
+
};
|
|
2329
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2330
|
+
});
|
|
2245
2331
|
}
|
|
2246
2332
|
function waitUntilOnline(signal) {
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
});
|
|
2333
|
+
throwIfAborted(signal);
|
|
2334
|
+
if (typeof navigator === "undefined" || navigator.onLine !== false) {
|
|
2335
|
+
return Promise.resolve();
|
|
2336
|
+
}
|
|
2337
|
+
if (typeof globalThis.addEventListener !== "function") return Promise.resolve();
|
|
2338
|
+
return new Promise((resolve, reject) => {
|
|
2339
|
+
const cleanup = () => {
|
|
2340
|
+
globalThis.removeEventListener("online", onOnline);
|
|
2341
|
+
signal.removeEventListener("abort", onAbort);
|
|
2342
|
+
};
|
|
2343
|
+
const onOnline = () => {
|
|
2344
|
+
cleanup();
|
|
2345
|
+
resolve();
|
|
2346
|
+
};
|
|
2347
|
+
const onAbort = () => {
|
|
2348
|
+
cleanup();
|
|
2349
|
+
reject(abortError(signal));
|
|
2350
|
+
};
|
|
2351
|
+
globalThis.addEventListener("online", onOnline, { once: true });
|
|
2352
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
2353
|
+
});
|
|
2269
2354
|
}
|
|
2270
2355
|
async function sha256Hex(data) {
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
return [...new Uint8Array(digest)]
|
|
2275
|
-
.map((byte) => byte.toString(16).padStart(2, "0"))
|
|
2276
|
-
.join("");
|
|
2356
|
+
if (!globalThis.crypto?.subtle) throw new Error("Web Crypto SHA-256 is unavailable");
|
|
2357
|
+
const digest = await globalThis.crypto.subtle.digest("SHA-256", Uint8Array.from(data).buffer);
|
|
2358
|
+
return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
2277
2359
|
}
|
|
2278
2360
|
function contextStreamId() {
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
return `ctx_${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`;
|
|
2361
|
+
if (globalThis.crypto?.randomUUID) return globalThis.crypto.randomUUID();
|
|
2362
|
+
return `ctx_${Date.now().toString(36)}_${Math.random().toString(36).slice(2)}`;
|
|
2282
2363
|
}
|
|
2283
2364
|
function contextUpdateBytes(update) {
|
|
2284
|
-
|
|
2285
|
-
? update.image.size
|
|
2286
|
-
: new TextEncoder().encode(update.text).byteLength;
|
|
2365
|
+
return "image" in update ? update.image.size : new TextEncoder().encode(update.text).byteLength;
|
|
2287
2366
|
}
|
|
2288
2367
|
function contextPayloadBytes(payload) {
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
catch {
|
|
2308
|
-
return undefined;
|
|
2309
|
-
}
|
|
2368
|
+
const image = payload.image;
|
|
2369
|
+
if (image && typeof image === "object" && "bytes" in image) {
|
|
2370
|
+
const bytes = image.bytes;
|
|
2371
|
+
if (typeof bytes === "number") return bytes;
|
|
2372
|
+
}
|
|
2373
|
+
const textStream = payload.text_stream;
|
|
2374
|
+
if (textStream && typeof textStream === "object" && "bytes" in textStream) {
|
|
2375
|
+
const bytes = textStream.bytes;
|
|
2376
|
+
if (typeof bytes === "number") return bytes;
|
|
2377
|
+
}
|
|
2378
|
+
if (typeof payload.text === "string") {
|
|
2379
|
+
return new TextEncoder().encode(payload.text).byteLength;
|
|
2380
|
+
}
|
|
2381
|
+
try {
|
|
2382
|
+
return new TextEncoder().encode(JSON.stringify(payload)).byteLength;
|
|
2383
|
+
} catch {
|
|
2384
|
+
return void 0;
|
|
2385
|
+
}
|
|
2310
2386
|
}
|
|
2311
2387
|
function contextKeyFromMessage(message) {
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
return typeof key === "string" ? key : undefined;
|
|
2388
|
+
if (message.type !== "context.update") return void 0;
|
|
2389
|
+
const key = message.payload.key;
|
|
2390
|
+
return typeof key === "string" ? key : void 0;
|
|
2316
2391
|
}
|
|
2317
2392
|
function diagnosticEventType(type) {
|
|
2318
|
-
|
|
2393
|
+
return type === "client.hello" || type === "context.update" ? type : void 0;
|
|
2319
2394
|
}
|
|
2320
2395
|
function validateSessionCredentials(value) {
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|| !value.connection.room_name) {
|
|
2331
|
-
throw new SessionConfigurationError("Session connection credentials are incomplete");
|
|
2332
|
-
}
|
|
2396
|
+
if (!value || typeof value !== "object") {
|
|
2397
|
+
throw new SessionConfigurationError("Session credentials must be an object");
|
|
2398
|
+
}
|
|
2399
|
+
if (!value.id || !value.access_token || !value.access_token_expires_at) {
|
|
2400
|
+
throw new SessionConfigurationError("Session credentials are incomplete");
|
|
2401
|
+
}
|
|
2402
|
+
if (!value.connection || !value.connection.url || !value.connection.token || !value.connection.room_name) {
|
|
2403
|
+
throw new SessionConfigurationError("Session connection credentials are incomplete");
|
|
2404
|
+
}
|
|
2333
2405
|
}
|
|
2334
2406
|
function normalizeApiBase(value) {
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2407
|
+
const normalized = value.trim().replace(/\/+$/, "");
|
|
2408
|
+
if (!normalized) {
|
|
2409
|
+
throw new SessionConfigurationError("apiBase must be a non-root relative path or an absolute URL");
|
|
2410
|
+
}
|
|
2411
|
+
return normalized;
|
|
2340
2412
|
}
|
|
2341
2413
|
function transcriptEvent(event) {
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
event,
|
|
2353
|
-
};
|
|
2414
|
+
const match = /^transcript\.(user|agent)\.(interim|final)$/.exec(event.type);
|
|
2415
|
+
if (!match) return null;
|
|
2416
|
+
const text = event.payload.text;
|
|
2417
|
+
if (typeof text !== "string") return null;
|
|
2418
|
+
return {
|
|
2419
|
+
role: match[1],
|
|
2420
|
+
text,
|
|
2421
|
+
final: match[2] === "final",
|
|
2422
|
+
event
|
|
2423
|
+
};
|
|
2354
2424
|
}
|
|
2355
2425
|
async function httpError(response, fallback) {
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
}
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
}
|
|
2426
|
+
let payload = {};
|
|
2427
|
+
try {
|
|
2428
|
+
payload = await response.json();
|
|
2429
|
+
} catch {
|
|
2430
|
+
}
|
|
2431
|
+
return new RealtimeSessionHttpError(
|
|
2432
|
+
typeof payload.message === "string" ? payload.message : `${fallback} (${response.status})`,
|
|
2433
|
+
{
|
|
2434
|
+
status: response.status,
|
|
2435
|
+
code: typeof payload.error === "string" ? payload.error : null,
|
|
2436
|
+
requestId: typeof payload.request_id === "string" ? payload.request_id : null,
|
|
2437
|
+
details: Array.isArray(payload.details) ? payload.details : []
|
|
2438
|
+
}
|
|
2439
|
+
);
|
|
2369
2440
|
}
|
|
2370
|
-
|
|
2441
|
+
export {
|
|
2442
|
+
CommandError2 as CommandError,
|
|
2443
|
+
EventCursor2 as EventCursor,
|
|
2444
|
+
RealtimeSession,
|
|
2445
|
+
RealtimeSessionHttpError,
|
|
2446
|
+
SDK_VERSION,
|
|
2447
|
+
TranscriptStore2 as TranscriptStore,
|
|
2448
|
+
createClientMessage2 as createClientMessage,
|
|
2449
|
+
decodeAgentSnapshotPayload,
|
|
2450
|
+
decodeClientHelloPayload,
|
|
2451
|
+
decodeClientMessage,
|
|
2452
|
+
decodeCommandAckPayload,
|
|
2453
|
+
decodeCommandErrorPayload,
|
|
2454
|
+
decodeContextRejectedPayload,
|
|
2455
|
+
decodeContextUpdatePayload,
|
|
2456
|
+
decodeContextUpdatedPayload,
|
|
2457
|
+
decodeRealtimeEvent,
|
|
2458
|
+
isCommandAckPayload2 as isCommandAckPayload,
|
|
2459
|
+
isCommandErrorPayload2 as isCommandErrorPayload,
|
|
2460
|
+
isContextRejectedPayload2 as isContextRejectedPayload,
|
|
2461
|
+
isContextUpdatePayload2 as isContextUpdatePayload,
|
|
2462
|
+
isContextUpdatedPayload2 as isContextUpdatedPayload,
|
|
2463
|
+
isRealtimeEvent2 as isRealtimeEvent
|
|
2464
|
+
};
|
|
2465
|
+
//# sourceMappingURL=index.js.map
|