@voxevolv/client 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +46 -11
- package/dist/browser/index.js +6 -6
- package/dist/browser/index.js.map +3 -3
- package/dist/browser/index.js.sha384 +1 -1
- package/dist/browser/manifest.json +1 -1
- package/dist/generated/protocol-codecs.d.ts.map +1 -1
- package/dist/generated/protocol-codecs.js +41 -0
- package/dist/generated/protocol-codecs.js.map +1 -1
- package/dist/index.d.ts +50 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +499 -60
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { ConnectionState as LiveKitConnectionState, DataPacket_Kind, Room, RoomEvent, RpcError, Track, } from "livekit-client";
|
|
1
|
+
import { ConnectionState as LiveKitConnectionState, DataPacket_Kind, DisconnectReason, Room, RoomEvent, RpcError, Track, } from "livekit-client";
|
|
2
2
|
import { createClientMessage, CommandError, EventCursor, isCommandAckPayload, isCommandErrorPayload, isContextRejectedPayload, isContextUpdatePayload, isContextUpdatedPayload, isRealtimeEvent, } from "./protocol.js";
|
|
3
3
|
export { CommandError, EventCursor, createClientMessage, isCommandAckPayload, isCommandErrorPayload, isContextRejectedPayload, isContextUpdatePayload, isContextUpdatedPayload, isRealtimeEvent, } from "./protocol.js";
|
|
4
4
|
export { decodeAgentSnapshotPayload, decodeClientHelloPayload, decodeClientMessage, decodeCommandAckPayload, decodeCommandErrorPayload, decodeContextRejectedPayload, decodeContextUpdatePayload, decodeContextUpdatedPayload, decodeRealtimeEvent, } from "./generated/protocol-codecs.js";
|
|
5
|
-
export const SDK_VERSION = "0.
|
|
5
|
+
export const SDK_VERSION = "0.3.0";
|
|
6
6
|
export class RealtimeSessionHttpError extends Error {
|
|
7
7
|
status;
|
|
8
8
|
code;
|
|
@@ -26,6 +26,8 @@ export class RealtimeSession {
|
|
|
26
26
|
listeners = new Set();
|
|
27
27
|
stateListeners = new Set();
|
|
28
28
|
gapListeners = new Set();
|
|
29
|
+
connectionQualityListeners = new Set();
|
|
30
|
+
recoveryListeners = new Set();
|
|
29
31
|
remoteAudioTrackListeners = new Set();
|
|
30
32
|
remoteAudioTrackUnsubscribedListeners = new Set();
|
|
31
33
|
readyListeners = new Set();
|
|
@@ -42,18 +44,29 @@ export class RealtimeSession {
|
|
|
42
44
|
connectionId = `conn_${Math.random().toString(36).slice(2)}_${Date.now().toString(36)}`;
|
|
43
45
|
clientSeq = 0;
|
|
44
46
|
lastHelloMessageId = null;
|
|
47
|
+
reconnectStartedAt = null;
|
|
48
|
+
pendingReconnect = null;
|
|
45
49
|
agentReady = false;
|
|
46
50
|
readyResolve = null;
|
|
47
51
|
replayPromise = null;
|
|
48
52
|
pendingCommands = new Map();
|
|
49
53
|
toolHandlers = new Map();
|
|
50
54
|
apiBase = "/v1";
|
|
55
|
+
hasConnectedValue = false;
|
|
56
|
+
sessionEnded = false;
|
|
57
|
+
disconnectMode = null;
|
|
58
|
+
activeRecovery = null;
|
|
59
|
+
autoRejoinPromise = null;
|
|
60
|
+
autoRejoinController = null;
|
|
51
61
|
useManagedMicrophone = true;
|
|
62
|
+
managedMicrophoneEnabled = true;
|
|
52
63
|
applicationAudioTrack = null;
|
|
53
64
|
connectDefaults;
|
|
54
65
|
fetcher;
|
|
55
66
|
endEndpoint;
|
|
56
67
|
refreshCredentialsCallback;
|
|
68
|
+
autoRejoinOptions;
|
|
69
|
+
lifecycleCredentials;
|
|
57
70
|
constructor(credentials, options = {}) {
|
|
58
71
|
validateSessionCredentials(credentials);
|
|
59
72
|
this.id = credentials.id;
|
|
@@ -66,6 +79,8 @@ export class RealtimeSession {
|
|
|
66
79
|
this.fetcher = options.fetch ?? globalThis.fetch.bind(globalThis);
|
|
67
80
|
this.endEndpoint = options.endEndpoint ?? null;
|
|
68
81
|
this.refreshCredentialsCallback = options.refreshCredentials ?? null;
|
|
82
|
+
this.autoRejoinOptions = resolveAutoRejoinOptions(options.autoRejoin);
|
|
83
|
+
this.lifecycleCredentials = options.lifecycleCredentials ?? "same-origin";
|
|
69
84
|
this.room = new Room({ adaptiveStream: true, dynacast: true });
|
|
70
85
|
for (const [name, handler] of Object.entries(options.tools ?? {})) {
|
|
71
86
|
this.registerTool(name, handler);
|
|
@@ -130,6 +145,9 @@ export class RealtimeSession {
|
|
|
130
145
|
get state() {
|
|
131
146
|
return this.stateValue;
|
|
132
147
|
}
|
|
148
|
+
get hasConnectedBefore() {
|
|
149
|
+
return this.hasConnectedValue;
|
|
150
|
+
}
|
|
133
151
|
/**
|
|
134
152
|
* Stable stream holding the agent's current remote audio tracks. The stream
|
|
135
153
|
* object never changes across track arrival, reconnects, or rejoin; tracks
|
|
@@ -152,6 +170,57 @@ export class RealtimeSession {
|
|
|
152
170
|
this.gapListeners.add(listener);
|
|
153
171
|
return () => this.gapListeners.delete(listener);
|
|
154
172
|
}
|
|
173
|
+
onConnectionQuality(listener) {
|
|
174
|
+
this.connectionQualityListeners.add(listener);
|
|
175
|
+
this.notifyListeners(new Set([listener]), this.room.localParticipant.connectionQuality);
|
|
176
|
+
return () => this.connectionQualityListeners.delete(listener);
|
|
177
|
+
}
|
|
178
|
+
onRecovery(listener) {
|
|
179
|
+
this.recoveryListeners.add(listener);
|
|
180
|
+
return () => this.recoveryListeners.delete(listener);
|
|
181
|
+
}
|
|
182
|
+
waitForState(states, timeoutMs, options = {}) {
|
|
183
|
+
if (!states.length)
|
|
184
|
+
throw new TypeError("states must contain at least one connection state");
|
|
185
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
|
186
|
+
throw new TypeError("timeoutMs must be a non-negative finite number");
|
|
187
|
+
}
|
|
188
|
+
const targets = new Set(states);
|
|
189
|
+
if (targets.has(this.stateValue))
|
|
190
|
+
return Promise.resolve(this.stateValue);
|
|
191
|
+
if (options.signal?.aborted) {
|
|
192
|
+
return Promise.reject(options.signal.reason ?? new DOMException("Aborted", "AbortError"));
|
|
193
|
+
}
|
|
194
|
+
return new Promise((resolve, reject) => {
|
|
195
|
+
let settled = false;
|
|
196
|
+
const finish = (state, error) => {
|
|
197
|
+
if (settled)
|
|
198
|
+
return;
|
|
199
|
+
settled = true;
|
|
200
|
+
globalThis.clearTimeout(timeout);
|
|
201
|
+
this.stateListeners.delete(onState);
|
|
202
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
203
|
+
if (error !== undefined) {
|
|
204
|
+
reject(error);
|
|
205
|
+
}
|
|
206
|
+
else if (state !== null) {
|
|
207
|
+
resolve(state);
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
const onState = (state) => {
|
|
211
|
+
if (targets.has(state))
|
|
212
|
+
finish(state);
|
|
213
|
+
};
|
|
214
|
+
const onAbort = () => {
|
|
215
|
+
finish(null, options.signal?.reason ?? new DOMException("Aborted", "AbortError"));
|
|
216
|
+
};
|
|
217
|
+
const timeout = globalThis.setTimeout(() => {
|
|
218
|
+
finish(null, new Error(`Session did not reach ${states.join(" or ")} within ${timeoutMs}ms`));
|
|
219
|
+
}, timeoutMs);
|
|
220
|
+
this.stateListeners.add(onState);
|
|
221
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
222
|
+
});
|
|
223
|
+
}
|
|
155
224
|
onRemoteAudioTrack(listener) {
|
|
156
225
|
this.remoteAudioTrackListeners.add(listener);
|
|
157
226
|
return () => this.remoteAudioTrackListeners.delete(listener);
|
|
@@ -189,7 +258,8 @@ export class RealtimeSession {
|
|
|
189
258
|
return () => this.audioPlaybackBlockedListeners.delete(listener);
|
|
190
259
|
}
|
|
191
260
|
registerTool(name, handler) {
|
|
192
|
-
if (["connecting", "connected_not_ready", "connected", "reconnecting"]
|
|
261
|
+
if (["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"]
|
|
262
|
+
.includes(this.stateValue)) {
|
|
193
263
|
throw new Error("Browser tools must be registered before connect()");
|
|
194
264
|
}
|
|
195
265
|
if (!/^[a-z][a-z0-9_]{0,79}$/.test(name)) {
|
|
@@ -204,7 +274,18 @@ export class RealtimeSession {
|
|
|
204
274
|
};
|
|
205
275
|
}
|
|
206
276
|
async connect(options = {}) {
|
|
207
|
-
if (["connecting", "connected_not_ready", "connected", "reconnecting"]
|
|
277
|
+
if (["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"]
|
|
278
|
+
.includes(this.stateValue)) {
|
|
279
|
+
throw new Error(`Cannot connect while session is ${this.stateValue}`);
|
|
280
|
+
}
|
|
281
|
+
this.cancelAutoRejoin();
|
|
282
|
+
await this.connectWithMode(options, false);
|
|
283
|
+
}
|
|
284
|
+
async connectWithMode(options, recovering, signal) {
|
|
285
|
+
const blockedStates = recovering
|
|
286
|
+
? ["connecting", "connected_not_ready", "connected", "reconnecting"]
|
|
287
|
+
: ["connecting", "connected_not_ready", "connected", "reconnecting", "resuming"];
|
|
288
|
+
if (blockedStates.includes(this.stateValue)) {
|
|
208
289
|
throw new Error(`Cannot connect while session is ${this.stateValue}`);
|
|
209
290
|
}
|
|
210
291
|
const resolved = { ...this.connectDefaults, ...options };
|
|
@@ -215,62 +296,108 @@ export class RealtimeSession {
|
|
|
215
296
|
const apiBase = normalizeApiBase(resolved.apiBase ?? this.apiBase);
|
|
216
297
|
this.validateAudioOptions(microphone, audioTrack);
|
|
217
298
|
this.useManagedMicrophone = microphone;
|
|
299
|
+
if (!recovering)
|
|
300
|
+
this.managedMicrophoneEnabled = microphone;
|
|
218
301
|
this.applicationAudioTrack = audioTrack;
|
|
219
|
-
|
|
302
|
+
if (!recovering) {
|
|
303
|
+
this.sessionEnded = false;
|
|
304
|
+
this.setState("connecting");
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
this.setState("resuming");
|
|
308
|
+
}
|
|
220
309
|
this.apiBase = apiBase;
|
|
221
310
|
this.agentReady = false;
|
|
222
311
|
try {
|
|
223
312
|
await this.room.connect(this.credentials.url, this.credentials.token);
|
|
313
|
+
throwIfAborted(signal);
|
|
224
314
|
if (audioTrack) {
|
|
225
315
|
await this.publishApplicationAudioTrack(audioTrack);
|
|
226
316
|
}
|
|
227
317
|
else if (microphone) {
|
|
228
|
-
await this.room.localParticipant.setMicrophoneEnabled(
|
|
318
|
+
await this.room.localParticipant.setMicrophoneEnabled(this.managedMicrophoneEnabled);
|
|
229
319
|
}
|
|
230
|
-
|
|
320
|
+
throwIfAborted(signal);
|
|
321
|
+
this.setState(recovering ? "resuming" : "connected_not_ready");
|
|
231
322
|
await this.sendHello();
|
|
232
323
|
const readyDeadline = Date.now() + readyTimeoutMs;
|
|
233
|
-
await this.waitForAgentReady(readyTimeoutMs);
|
|
324
|
+
await this.waitForAgentReady(readyTimeoutMs, signal);
|
|
234
325
|
if (waitForRemoteAudio)
|
|
235
|
-
await this.waitForRemoteAudioTrack(readyDeadline);
|
|
326
|
+
await this.waitForRemoteAudioTrack(readyDeadline, signal);
|
|
327
|
+
throwIfAborted(signal);
|
|
328
|
+
this.hasConnectedValue = true;
|
|
236
329
|
this.setState("connected");
|
|
237
330
|
}
|
|
238
331
|
catch (error) {
|
|
332
|
+
this.disconnectMode = recovering ? "rejoin" : "connect_failure";
|
|
239
333
|
try {
|
|
240
334
|
await this.room.disconnect();
|
|
241
335
|
}
|
|
242
336
|
catch {
|
|
243
337
|
// Preserve the original connection/readiness failure.
|
|
244
338
|
}
|
|
245
|
-
|
|
339
|
+
finally {
|
|
340
|
+
this.disconnectMode = null;
|
|
341
|
+
}
|
|
342
|
+
if (!recovering)
|
|
343
|
+
this.setState("failed");
|
|
246
344
|
throw error;
|
|
247
345
|
}
|
|
248
346
|
}
|
|
249
347
|
async rejoin(apiBase = this.apiBase) {
|
|
250
|
-
if (["connecting", "reconnecting"].includes(this.stateValue)) {
|
|
348
|
+
if (["connecting", "reconnecting", "resuming"].includes(this.stateValue)) {
|
|
251
349
|
throw new Error(`Cannot rejoin while session is ${this.stateValue}`);
|
|
252
350
|
}
|
|
351
|
+
this.cancelAutoRejoin();
|
|
253
352
|
const normalizedApiBase = normalizeApiBase(apiBase);
|
|
254
|
-
|
|
353
|
+
const recovering = this.hasConnectedValue;
|
|
354
|
+
if (recovering) {
|
|
355
|
+
this.beginRecovery("rejoin", 1);
|
|
356
|
+
this.setState("resuming");
|
|
357
|
+
}
|
|
358
|
+
try {
|
|
359
|
+
await this.performRejoin(normalizedApiBase, undefined, recovering);
|
|
360
|
+
if (recovering)
|
|
361
|
+
this.completeRecovery();
|
|
362
|
+
}
|
|
363
|
+
catch (error) {
|
|
364
|
+
const normalizedError = toError(error);
|
|
365
|
+
if (recovering)
|
|
366
|
+
this.failRecovery(normalizedError);
|
|
367
|
+
this.setState("failed");
|
|
368
|
+
throw error;
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
async performRejoin(apiBase, signal, recovering = true) {
|
|
372
|
+
let response = await this.fetchRoomToken(apiBase, signal);
|
|
255
373
|
if (response.status === 401 && this.refreshCredentialsCallback) {
|
|
256
374
|
await this.refreshCredentials();
|
|
257
|
-
|
|
375
|
+
throwIfAborted(signal);
|
|
376
|
+
response = await this.fetchRoomToken(apiBase, signal);
|
|
258
377
|
}
|
|
259
378
|
if (!response.ok)
|
|
260
379
|
throw await httpError(response, "Could not renew room token");
|
|
261
380
|
const payload = (await response.json());
|
|
262
381
|
this.credentials = payload.connection;
|
|
263
|
-
if (
|
|
264
|
-
|
|
382
|
+
if (this.room.state !== LiveKitConnectionState.Disconnected) {
|
|
383
|
+
this.disconnectMode = "rejoin";
|
|
384
|
+
try {
|
|
385
|
+
await this.room.disconnect();
|
|
386
|
+
}
|
|
387
|
+
finally {
|
|
388
|
+
this.disconnectMode = null;
|
|
389
|
+
}
|
|
265
390
|
}
|
|
266
|
-
|
|
267
|
-
|
|
391
|
+
throwIfAborted(signal);
|
|
392
|
+
await this.connectWithMode({
|
|
393
|
+
apiBase,
|
|
268
394
|
microphone: this.useManagedMicrophone,
|
|
269
395
|
...(this.applicationAudioTrack ? { audioTrack: this.applicationAudioTrack } : {}),
|
|
270
|
-
});
|
|
396
|
+
}, recovering, signal);
|
|
271
397
|
}
|
|
272
398
|
async setMicrophoneEnabled(enabled) {
|
|
273
399
|
await this.room.localParticipant.setMicrophoneEnabled(enabled);
|
|
400
|
+
this.managedMicrophoneEnabled = enabled;
|
|
274
401
|
}
|
|
275
402
|
async sendCommand(type, payload, timeoutMs = 10_000) {
|
|
276
403
|
const message = createClientMessage(this.connectionId, ++this.clientSeq, type, payload);
|
|
@@ -367,13 +494,23 @@ export class RealtimeSession {
|
|
|
367
494
|
};
|
|
368
495
|
}
|
|
369
496
|
async disconnect() {
|
|
497
|
+
this.cancelAutoRejoin();
|
|
370
498
|
if (this.stateValue === "disconnected")
|
|
371
499
|
return;
|
|
372
500
|
this.rejectPendingCommands(new Error("Session disconnected before command completion"));
|
|
373
|
-
|
|
374
|
-
|
|
501
|
+
this.disconnectMode = "intentional";
|
|
502
|
+
try {
|
|
503
|
+
await this.room.disconnect();
|
|
504
|
+
}
|
|
505
|
+
finally {
|
|
506
|
+
this.disconnectMode = null;
|
|
507
|
+
this.activeRecovery = null;
|
|
508
|
+
this.setState("disconnected");
|
|
509
|
+
}
|
|
375
510
|
}
|
|
376
511
|
async end(options = {}) {
|
|
512
|
+
this.cancelAutoRejoin();
|
|
513
|
+
this.sessionEnded = true;
|
|
377
514
|
const endpoint = options.endpoint ?? this.endEndpoint ?? `${this.apiBase}/sessions/${this.id}`;
|
|
378
515
|
const request = options.request ?? {};
|
|
379
516
|
const headers = new Headers(request.headers);
|
|
@@ -385,6 +522,7 @@ export class RealtimeSession {
|
|
|
385
522
|
...request,
|
|
386
523
|
method: request.method ?? "DELETE",
|
|
387
524
|
headers,
|
|
525
|
+
credentials: request.credentials ?? this.lifecycleCredentials,
|
|
388
526
|
});
|
|
389
527
|
if (response.status === 401 && usesScopedSessionEndpoint && this.refreshCredentialsCallback) {
|
|
390
528
|
await this.refreshCredentials();
|
|
@@ -393,6 +531,7 @@ export class RealtimeSession {
|
|
|
393
531
|
...request,
|
|
394
532
|
method: request.method ?? "DELETE",
|
|
395
533
|
headers,
|
|
534
|
+
credentials: request.credentials ?? this.lifecycleCredentials,
|
|
396
535
|
});
|
|
397
536
|
}
|
|
398
537
|
if (!response.ok)
|
|
@@ -416,32 +555,29 @@ export class RealtimeSession {
|
|
|
416
555
|
this.notifyListeners(this.stateListeners, state);
|
|
417
556
|
}
|
|
418
557
|
bindRoomEvents() {
|
|
419
|
-
this.room.on(RoomEvent.
|
|
420
|
-
|
|
421
|
-
this.rejectPendingCommands(new Error("Connection interrupted before command completion"), true);
|
|
422
|
-
this.setState("reconnecting");
|
|
423
|
-
});
|
|
558
|
+
this.room.on(RoomEvent.SignalReconnecting, () => this.handleLiveKitReconnecting());
|
|
559
|
+
this.room.on(RoomEvent.Reconnecting, () => this.handleLiveKitReconnecting());
|
|
424
560
|
this.room.on(RoomEvent.Reconnected, () => {
|
|
425
|
-
this.
|
|
426
|
-
this.clientSeq = 0;
|
|
427
|
-
void this.sendHello().catch(() => this.setState("failed"));
|
|
428
|
-
this.setState("connected_not_ready");
|
|
561
|
+
void this.handleLiveKitReconnected();
|
|
429
562
|
});
|
|
430
|
-
this.room.on(RoomEvent.Disconnected, () => {
|
|
431
|
-
this.
|
|
432
|
-
// Drop any remaining remote tracks so a stale, ended track cannot
|
|
433
|
-
// satisfy the next connect()'s wait for agent audio.
|
|
434
|
-
this.outputMediaStream?.getTracks().forEach((track) => {
|
|
435
|
-
this.outputMediaStream?.removeTrack(track);
|
|
436
|
-
});
|
|
437
|
-
this.rejectPendingCommands(new Error("Session disconnected before command completion"));
|
|
438
|
-
this.setState("disconnected");
|
|
563
|
+
this.room.on(RoomEvent.Disconnected, (reason) => {
|
|
564
|
+
this.handleLiveKitDisconnected(reason);
|
|
439
565
|
});
|
|
440
566
|
this.room.on(RoomEvent.ConnectionStateChanged, (state) => {
|
|
441
567
|
if (state === LiveKitConnectionState.Connected) {
|
|
442
|
-
this.
|
|
568
|
+
const recovering = this.activeRecovery !== null || this.stateValue === "resuming";
|
|
569
|
+
this.setState(this.agentReady
|
|
570
|
+
? "connected"
|
|
571
|
+
: recovering
|
|
572
|
+
? "resuming"
|
|
573
|
+
: "connected_not_ready");
|
|
443
574
|
}
|
|
444
575
|
});
|
|
576
|
+
this.room.on(RoomEvent.ConnectionQualityChanged, (quality, participant) => {
|
|
577
|
+
if (!participant.isLocal)
|
|
578
|
+
return;
|
|
579
|
+
this.notifyListeners(this.connectionQualityListeners, quality);
|
|
580
|
+
});
|
|
445
581
|
this.room.on(RoomEvent.TrackSubscribed, (track) => {
|
|
446
582
|
if (track.kind !== Track.Kind.Audio)
|
|
447
583
|
return;
|
|
@@ -507,6 +643,175 @@ export class RealtimeSession {
|
|
|
507
643
|
this.notifyListeners(this.listeners, value);
|
|
508
644
|
});
|
|
509
645
|
}
|
|
646
|
+
handleLiveKitReconnecting() {
|
|
647
|
+
this.agentReady = false;
|
|
648
|
+
// Once full rejoin owns recovery, late LiveKit reconnect notifications
|
|
649
|
+
// from the old transport must not move the wrapper back to reconnecting.
|
|
650
|
+
// Doing so makes connectWithMode() reject the replacement connection.
|
|
651
|
+
if (this.autoRejoinPromise !== null
|
|
652
|
+
|| this.activeRecovery?.strategy === "rejoin"
|
|
653
|
+
|| this.disconnectMode === "rejoin") {
|
|
654
|
+
this.setState("resuming");
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
this.reconnectStartedAt ??= Date.now();
|
|
658
|
+
this.rejectPendingCommands(new Error("Connection interrupted before command completion"), true);
|
|
659
|
+
if (this.activeRecovery === null) {
|
|
660
|
+
this.beginRecovery("livekit", 1);
|
|
661
|
+
}
|
|
662
|
+
this.setState("reconnecting");
|
|
663
|
+
}
|
|
664
|
+
async handleLiveKitReconnected() {
|
|
665
|
+
const recoveredAt = Date.now();
|
|
666
|
+
if (this.reconnectStartedAt !== null) {
|
|
667
|
+
this.pendingReconnect = {
|
|
668
|
+
started_at: new Date(this.reconnectStartedAt).toISOString(),
|
|
669
|
+
recovered_at: new Date(recoveredAt).toISOString(),
|
|
670
|
+
duration_ms: Math.max(recoveredAt - this.reconnectStartedAt, 0),
|
|
671
|
+
};
|
|
672
|
+
this.reconnectStartedAt = null;
|
|
673
|
+
}
|
|
674
|
+
if (this.hasConnectedValue && this.activeRecovery === null) {
|
|
675
|
+
this.beginRecovery("livekit", 1);
|
|
676
|
+
}
|
|
677
|
+
this.connectionId = `conn_${Math.random().toString(36).slice(2)}_${Date.now().toString(36)}`;
|
|
678
|
+
this.clientSeq = 0;
|
|
679
|
+
this.setState(this.hasConnectedValue ? "resuming" : "connected_not_ready");
|
|
680
|
+
try {
|
|
681
|
+
await this.sendHello();
|
|
682
|
+
}
|
|
683
|
+
catch (error) {
|
|
684
|
+
const normalizedError = toError(error);
|
|
685
|
+
if (this.shouldAutoRejoin()) {
|
|
686
|
+
this.startAutoRejoin(normalizedError);
|
|
687
|
+
}
|
|
688
|
+
else {
|
|
689
|
+
this.failRecovery(normalizedError);
|
|
690
|
+
this.setState("failed");
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
handleLiveKitDisconnected(reason) {
|
|
695
|
+
this.agentReady = false;
|
|
696
|
+
this.reconnectStartedAt = null;
|
|
697
|
+
this.pendingReconnect = null;
|
|
698
|
+
// Drop any remaining remote tracks so a stale, ended track cannot
|
|
699
|
+
// satisfy the next connect()'s wait for agent audio.
|
|
700
|
+
this.outputMediaStream?.getTracks().forEach((track) => {
|
|
701
|
+
this.outputMediaStream?.removeTrack(track);
|
|
702
|
+
});
|
|
703
|
+
this.rejectPendingCommands(new Error("Session disconnected before command completion"));
|
|
704
|
+
if (this.disconnectMode === "rejoin")
|
|
705
|
+
return;
|
|
706
|
+
if (this.disconnectMode === "connect_failure")
|
|
707
|
+
return;
|
|
708
|
+
if (this.disconnectMode === "intentional") {
|
|
709
|
+
this.activeRecovery = null;
|
|
710
|
+
this.setState("disconnected");
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
713
|
+
if (this.shouldAutoRejoin(reason)) {
|
|
714
|
+
this.startAutoRejoin(new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`));
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
if (this.activeRecovery !== null) {
|
|
718
|
+
this.failRecovery(new Error(`LiveKit disconnected (${disconnectReasonName(reason)})`));
|
|
719
|
+
}
|
|
720
|
+
this.setState("disconnected");
|
|
721
|
+
}
|
|
722
|
+
beginRecovery(strategy, attempt) {
|
|
723
|
+
this.activeRecovery = { strategy, attempt };
|
|
724
|
+
this.notifyListeners(this.recoveryListeners, {
|
|
725
|
+
status: "started",
|
|
726
|
+
strategy,
|
|
727
|
+
attempt,
|
|
728
|
+
});
|
|
729
|
+
}
|
|
730
|
+
completeRecovery() {
|
|
731
|
+
const recovery = this.activeRecovery;
|
|
732
|
+
if (!recovery)
|
|
733
|
+
return;
|
|
734
|
+
this.activeRecovery = null;
|
|
735
|
+
this.notifyListeners(this.recoveryListeners, {
|
|
736
|
+
status: "succeeded",
|
|
737
|
+
strategy: recovery.strategy,
|
|
738
|
+
attempt: recovery.attempt,
|
|
739
|
+
});
|
|
740
|
+
}
|
|
741
|
+
failRecovery(error) {
|
|
742
|
+
const recovery = this.activeRecovery;
|
|
743
|
+
if (!recovery)
|
|
744
|
+
return;
|
|
745
|
+
this.activeRecovery = null;
|
|
746
|
+
this.notifyListeners(this.recoveryListeners, {
|
|
747
|
+
status: "failed",
|
|
748
|
+
strategy: recovery.strategy,
|
|
749
|
+
attempt: recovery.attempt,
|
|
750
|
+
error,
|
|
751
|
+
});
|
|
752
|
+
}
|
|
753
|
+
shouldAutoRejoin(reason) {
|
|
754
|
+
return (this.autoRejoinOptions !== null
|
|
755
|
+
&& this.hasConnectedValue
|
|
756
|
+
&& !this.sessionEnded
|
|
757
|
+
&& !isTerminalDisconnectReason(reason));
|
|
758
|
+
}
|
|
759
|
+
startAutoRejoin(initialError) {
|
|
760
|
+
if (this.autoRejoinPromise || !this.shouldAutoRejoin())
|
|
761
|
+
return;
|
|
762
|
+
const controller = new AbortController();
|
|
763
|
+
this.autoRejoinController = controller;
|
|
764
|
+
const promise = this.runAutoRejoin(initialError, controller.signal)
|
|
765
|
+
.finally(() => {
|
|
766
|
+
if (this.autoRejoinController === controller) {
|
|
767
|
+
this.autoRejoinController = null;
|
|
768
|
+
this.autoRejoinPromise = null;
|
|
769
|
+
}
|
|
770
|
+
});
|
|
771
|
+
this.autoRejoinPromise = promise;
|
|
772
|
+
void promise;
|
|
773
|
+
}
|
|
774
|
+
async runAutoRejoin(initialError, signal) {
|
|
775
|
+
const options = this.autoRejoinOptions;
|
|
776
|
+
if (!options)
|
|
777
|
+
return;
|
|
778
|
+
let lastError = initialError;
|
|
779
|
+
if (this.activeRecovery?.strategy === "livekit") {
|
|
780
|
+
this.failRecovery(initialError);
|
|
781
|
+
}
|
|
782
|
+
this.setState("resuming");
|
|
783
|
+
for (let attempt = 1; attempt <= options.maxAttempts; attempt += 1) {
|
|
784
|
+
try {
|
|
785
|
+
await waitUntilOnline(signal);
|
|
786
|
+
if (attempt > 1) {
|
|
787
|
+
const exponentialDelay = options.baseDelayMs * (2 ** (attempt - 2));
|
|
788
|
+
const delayMs = Math.min(exponentialDelay, options.maxDelayMs);
|
|
789
|
+
await abortableDelay(delayMs + Math.random() * Math.min(delayMs, 500), signal);
|
|
790
|
+
}
|
|
791
|
+
throwIfAborted(signal);
|
|
792
|
+
this.beginRecovery("rejoin", attempt);
|
|
793
|
+
await this.performRejoin(this.apiBase, signal);
|
|
794
|
+
throwIfAborted(signal);
|
|
795
|
+
this.completeRecovery();
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
catch (error) {
|
|
799
|
+
if (signal.aborted)
|
|
800
|
+
return;
|
|
801
|
+
lastError = toError(error);
|
|
802
|
+
if (isTerminalRejoinError(error))
|
|
803
|
+
break;
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
this.failRecovery(lastError);
|
|
807
|
+
this.setState("failed");
|
|
808
|
+
}
|
|
809
|
+
cancelAutoRejoin() {
|
|
810
|
+
this.autoRejoinController?.abort();
|
|
811
|
+
this.autoRejoinController = null;
|
|
812
|
+
this.autoRejoinPromise = null;
|
|
813
|
+
this.activeRecovery = null;
|
|
814
|
+
}
|
|
510
815
|
notifyListeners(listeners, value) {
|
|
511
816
|
for (const listener of listeners) {
|
|
512
817
|
try {
|
|
@@ -540,10 +845,12 @@ export class RealtimeSession {
|
|
|
540
845
|
throw new TypeError("audioTrack has already ended");
|
|
541
846
|
}
|
|
542
847
|
}
|
|
543
|
-
async fetchRoomToken(apiBase) {
|
|
848
|
+
async fetchRoomToken(apiBase, signal) {
|
|
544
849
|
return this.fetcher(`${apiBase}/sessions/${this.id}/room-tokens`, {
|
|
545
850
|
method: "POST",
|
|
546
851
|
headers: { Authorization: `Bearer ${this.accessToken}` },
|
|
852
|
+
credentials: this.lifecycleCredentials,
|
|
853
|
+
signal,
|
|
547
854
|
});
|
|
548
855
|
}
|
|
549
856
|
async refreshCredentials() {
|
|
@@ -558,8 +865,20 @@ export class RealtimeSession {
|
|
|
558
865
|
this.credentials = refreshed.connection;
|
|
559
866
|
}
|
|
560
867
|
observeConvenienceEvents(event) {
|
|
561
|
-
if (event.type === "session.ended")
|
|
868
|
+
if (event.type === "session.ended") {
|
|
869
|
+
const recoveryWasActive = (this.autoRejoinPromise !== null
|
|
870
|
+
|| this.activeRecovery !== null
|
|
871
|
+
|| this.stateValue === "reconnecting"
|
|
872
|
+
|| this.stateValue === "resuming");
|
|
873
|
+
this.sessionEnded = true;
|
|
874
|
+
if (this.activeRecovery !== null) {
|
|
875
|
+
this.failRecovery(new Error("Session ended during recovery"));
|
|
876
|
+
}
|
|
877
|
+
this.cancelAutoRejoin();
|
|
878
|
+
if (recoveryWasActive)
|
|
879
|
+
this.setState("disconnected");
|
|
562
880
|
this.notifyListeners(this.endedListeners, event);
|
|
881
|
+
}
|
|
563
882
|
if (event.type.endsWith(".error"))
|
|
564
883
|
this.notifyListeners(this.errorListeners, event);
|
|
565
884
|
const transcript = transcriptEvent(event);
|
|
@@ -581,13 +900,17 @@ export class RealtimeSession {
|
|
|
581
900
|
}
|
|
582
901
|
}
|
|
583
902
|
async sendHello() {
|
|
903
|
+
const reconnect = this.pendingReconnect;
|
|
584
904
|
const message = createClientMessage(this.connectionId, ++this.clientSeq, "client.hello", {
|
|
585
905
|
last_server_seq: this.cursor.lastServerSeq,
|
|
586
906
|
sdk: `@voxevolv/client@${SDK_VERSION}`,
|
|
587
907
|
tool_handlers: [...this.toolHandlers.keys()].sort(),
|
|
908
|
+
...(reconnect ? { reconnect } : {}),
|
|
588
909
|
});
|
|
589
910
|
this.lastHelloMessageId = message.message_id;
|
|
590
911
|
await this.sendTrackedCommand(message, 10_000);
|
|
912
|
+
if (this.pendingReconnect === reconnect)
|
|
913
|
+
this.pendingReconnect = null;
|
|
591
914
|
}
|
|
592
915
|
async sendTrackedCommand(message, timeoutMs, retryTransfer) {
|
|
593
916
|
const response = new Promise((resolve, reject) => {
|
|
@@ -621,49 +944,65 @@ export class RealtimeSession {
|
|
|
621
944
|
try {
|
|
622
945
|
await this.publishCommand(message);
|
|
623
946
|
}
|
|
624
|
-
catch
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
globalThis.clearInterval(pending.retry);
|
|
629
|
-
this.pendingCommands.delete(message.message_id);
|
|
630
|
-
pending.reject(error instanceof Error ? error : new Error(String(error)));
|
|
631
|
-
}
|
|
947
|
+
catch {
|
|
948
|
+
// The reliable data channel may still be warming immediately after a
|
|
949
|
+
// reconnect. Keep the command pending so the retry interval can publish
|
|
950
|
+
// it once the transport accepts writes, or reject at the normal timeout.
|
|
632
951
|
}
|
|
633
952
|
return response;
|
|
634
953
|
}
|
|
635
954
|
async publishCommand(message) {
|
|
636
955
|
await this.room.localParticipant.publishData(new TextEncoder().encode(JSON.stringify(message)), { reliable: true, topic: "voxevolv.commands" });
|
|
637
956
|
}
|
|
638
|
-
waitForRemoteAudioTrack(deadline) {
|
|
957
|
+
waitForRemoteAudioTrack(deadline, signal) {
|
|
639
958
|
if ((this.outputMediaStream?.getAudioTracks().length ?? 0) > 0)
|
|
640
959
|
return Promise.resolve();
|
|
960
|
+
throwIfAborted(signal);
|
|
641
961
|
return new Promise((resolve, reject) => {
|
|
642
|
-
const
|
|
962
|
+
const cleanup = () => {
|
|
963
|
+
globalThis.clearTimeout(timeout);
|
|
964
|
+
signal?.removeEventListener("abort", onAbort);
|
|
643
965
|
this.remoteAudioTrackResolve = null;
|
|
966
|
+
};
|
|
967
|
+
const timeout = globalThis.setTimeout(() => {
|
|
968
|
+
cleanup();
|
|
644
969
|
reject(new Error("Agent audio track did not arrive within the ready timeout;"
|
|
645
970
|
+ " pass waitForRemoteAudio: false to resolve connect() on signaling readiness alone"));
|
|
646
971
|
}, Math.max(deadline - Date.now(), 0));
|
|
972
|
+
const onAbort = () => {
|
|
973
|
+
cleanup();
|
|
974
|
+
reject(abortError(signal));
|
|
975
|
+
};
|
|
647
976
|
this.remoteAudioTrackResolve = () => {
|
|
648
|
-
|
|
649
|
-
this.remoteAudioTrackResolve = null;
|
|
977
|
+
cleanup();
|
|
650
978
|
resolve();
|
|
651
979
|
};
|
|
980
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
652
981
|
});
|
|
653
982
|
}
|
|
654
|
-
waitForAgentReady(timeoutMs) {
|
|
983
|
+
waitForAgentReady(timeoutMs, signal) {
|
|
655
984
|
if (this.agentReady)
|
|
656
985
|
return Promise.resolve();
|
|
986
|
+
throwIfAborted(signal);
|
|
657
987
|
return new Promise((resolve, reject) => {
|
|
658
|
-
const
|
|
988
|
+
const cleanup = () => {
|
|
989
|
+
globalThis.clearTimeout(timeout);
|
|
990
|
+
signal?.removeEventListener("abort", onAbort);
|
|
659
991
|
this.readyResolve = null;
|
|
992
|
+
};
|
|
993
|
+
const timeout = globalThis.setTimeout(() => {
|
|
994
|
+
cleanup();
|
|
660
995
|
reject(new Error(`Agent did not become ready within ${timeoutMs}ms`));
|
|
661
996
|
}, timeoutMs);
|
|
997
|
+
const onAbort = () => {
|
|
998
|
+
cleanup();
|
|
999
|
+
reject(abortError(signal));
|
|
1000
|
+
};
|
|
662
1001
|
this.readyResolve = () => {
|
|
663
|
-
|
|
664
|
-
this.readyResolve = null;
|
|
1002
|
+
cleanup();
|
|
665
1003
|
resolve();
|
|
666
1004
|
};
|
|
1005
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
667
1006
|
});
|
|
668
1007
|
}
|
|
669
1008
|
observeReadiness(event) {
|
|
@@ -676,8 +1015,12 @@ export class RealtimeSession {
|
|
|
676
1015
|
return;
|
|
677
1016
|
const wasReady = this.agentReady;
|
|
678
1017
|
this.agentReady = true;
|
|
679
|
-
if (this.room.state === LiveKitConnectionState.Connected)
|
|
1018
|
+
if (this.room.state === LiveKitConnectionState.Connected) {
|
|
1019
|
+
this.hasConnectedValue = true;
|
|
680
1020
|
this.setState("connected");
|
|
1021
|
+
if (this.activeRecovery?.strategy === "livekit")
|
|
1022
|
+
this.completeRecovery();
|
|
1023
|
+
}
|
|
681
1024
|
this.readyResolve?.();
|
|
682
1025
|
if (!wasReady)
|
|
683
1026
|
this.notifyListeners(this.readyListeners, event);
|
|
@@ -737,10 +1080,16 @@ export class RealtimeSession {
|
|
|
737
1080
|
async fetchReplay(apiBase, afterSeq) {
|
|
738
1081
|
let nextAfter = afterSeq;
|
|
739
1082
|
while (true) {
|
|
740
|
-
let response = await this.fetcher(`${apiBase}/sessions/${this.id}/events?after_seq=${nextAfter}&limit=200`, {
|
|
1083
|
+
let response = await this.fetcher(`${apiBase}/sessions/${this.id}/events?after_seq=${nextAfter}&limit=200`, {
|
|
1084
|
+
headers: { Authorization: `Bearer ${this.accessToken}` },
|
|
1085
|
+
credentials: this.lifecycleCredentials,
|
|
1086
|
+
});
|
|
741
1087
|
if (response.status === 401 && this.refreshCredentialsCallback) {
|
|
742
1088
|
await this.refreshCredentials();
|
|
743
|
-
response = await this.fetcher(`${apiBase}/sessions/${this.id}/events?after_seq=${nextAfter}&limit=200`, {
|
|
1089
|
+
response = await this.fetcher(`${apiBase}/sessions/${this.id}/events?after_seq=${nextAfter}&limit=200`, {
|
|
1090
|
+
headers: { Authorization: `Bearer ${this.accessToken}` },
|
|
1091
|
+
credentials: this.lifecycleCredentials,
|
|
1092
|
+
});
|
|
744
1093
|
}
|
|
745
1094
|
if (!response.ok)
|
|
746
1095
|
throw await httpError(response, "Could not replay session events");
|
|
@@ -769,6 +1118,96 @@ export class RealtimeSession {
|
|
|
769
1118
|
}
|
|
770
1119
|
}
|
|
771
1120
|
}
|
|
1121
|
+
function resolveAutoRejoinOptions(value) {
|
|
1122
|
+
if (!value)
|
|
1123
|
+
return null;
|
|
1124
|
+
const options = value === true ? {} : value;
|
|
1125
|
+
const resolved = {
|
|
1126
|
+
maxAttempts: options.maxAttempts ?? 3,
|
|
1127
|
+
baseDelayMs: options.baseDelayMs ?? 750,
|
|
1128
|
+
maxDelayMs: options.maxDelayMs ?? 5_000,
|
|
1129
|
+
};
|
|
1130
|
+
if (!Number.isInteger(resolved.maxAttempts) || resolved.maxAttempts < 1) {
|
|
1131
|
+
throw new TypeError("autoRejoin.maxAttempts must be a positive integer");
|
|
1132
|
+
}
|
|
1133
|
+
if (!Number.isFinite(resolved.baseDelayMs) || resolved.baseDelayMs < 0) {
|
|
1134
|
+
throw new TypeError("autoRejoin.baseDelayMs must be a non-negative finite number");
|
|
1135
|
+
}
|
|
1136
|
+
if (!Number.isFinite(resolved.maxDelayMs) || resolved.maxDelayMs < 0) {
|
|
1137
|
+
throw new TypeError("autoRejoin.maxDelayMs must be a non-negative finite number");
|
|
1138
|
+
}
|
|
1139
|
+
return resolved;
|
|
1140
|
+
}
|
|
1141
|
+
function isTerminalDisconnectReason(reason) {
|
|
1142
|
+
return reason !== undefined && [
|
|
1143
|
+
DisconnectReason.CLIENT_INITIATED,
|
|
1144
|
+
DisconnectReason.DUPLICATE_IDENTITY,
|
|
1145
|
+
DisconnectReason.PARTICIPANT_REMOVED,
|
|
1146
|
+
DisconnectReason.ROOM_DELETED,
|
|
1147
|
+
DisconnectReason.ROOM_CLOSED,
|
|
1148
|
+
DisconnectReason.USER_UNAVAILABLE,
|
|
1149
|
+
DisconnectReason.USER_REJECTED,
|
|
1150
|
+
].includes(reason);
|
|
1151
|
+
}
|
|
1152
|
+
function disconnectReasonName(reason) {
|
|
1153
|
+
if (reason === undefined)
|
|
1154
|
+
return "unknown";
|
|
1155
|
+
return DisconnectReason[reason] ?? String(reason);
|
|
1156
|
+
}
|
|
1157
|
+
function isTerminalRejoinError(error) {
|
|
1158
|
+
return error instanceof RealtimeSessionHttpError
|
|
1159
|
+
&& [401, 403, 409].includes(error.status);
|
|
1160
|
+
}
|
|
1161
|
+
function toError(error) {
|
|
1162
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1163
|
+
}
|
|
1164
|
+
function abortError(signal) {
|
|
1165
|
+
return signal.reason ?? new DOMException("Aborted", "AbortError");
|
|
1166
|
+
}
|
|
1167
|
+
function throwIfAborted(signal) {
|
|
1168
|
+
if (signal?.aborted)
|
|
1169
|
+
throw abortError(signal);
|
|
1170
|
+
}
|
|
1171
|
+
function abortableDelay(delayMs, signal) {
|
|
1172
|
+
throwIfAborted(signal);
|
|
1173
|
+
if (delayMs <= 0)
|
|
1174
|
+
return Promise.resolve();
|
|
1175
|
+
return new Promise((resolve, reject) => {
|
|
1176
|
+
const timeout = globalThis.setTimeout(() => {
|
|
1177
|
+
signal.removeEventListener("abort", onAbort);
|
|
1178
|
+
resolve();
|
|
1179
|
+
}, delayMs);
|
|
1180
|
+
const onAbort = () => {
|
|
1181
|
+
globalThis.clearTimeout(timeout);
|
|
1182
|
+
reject(abortError(signal));
|
|
1183
|
+
};
|
|
1184
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1187
|
+
function waitUntilOnline(signal) {
|
|
1188
|
+
throwIfAborted(signal);
|
|
1189
|
+
if (typeof navigator === "undefined" || navigator.onLine !== false) {
|
|
1190
|
+
return Promise.resolve();
|
|
1191
|
+
}
|
|
1192
|
+
if (typeof globalThis.addEventListener !== "function")
|
|
1193
|
+
return Promise.resolve();
|
|
1194
|
+
return new Promise((resolve, reject) => {
|
|
1195
|
+
const cleanup = () => {
|
|
1196
|
+
globalThis.removeEventListener("online", onOnline);
|
|
1197
|
+
signal.removeEventListener("abort", onAbort);
|
|
1198
|
+
};
|
|
1199
|
+
const onOnline = () => {
|
|
1200
|
+
cleanup();
|
|
1201
|
+
resolve();
|
|
1202
|
+
};
|
|
1203
|
+
const onAbort = () => {
|
|
1204
|
+
cleanup();
|
|
1205
|
+
reject(abortError(signal));
|
|
1206
|
+
};
|
|
1207
|
+
globalThis.addEventListener("online", onOnline, { once: true });
|
|
1208
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1209
|
+
});
|
|
1210
|
+
}
|
|
772
1211
|
async function sha256Hex(data) {
|
|
773
1212
|
if (!globalThis.crypto?.subtle)
|
|
774
1213
|
throw new Error("Web Crypto SHA-256 is unavailable");
|