@d-id/client-sdk 1.1.61 → 1.1.63-staging.289
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index-BkSUCJW5.js +1640 -0
- package/dist/index.js +27 -2032
- package/dist/index.umd.cjs +13 -13
- package/dist/livekit-manager-DylcG4TH.js +427 -0
- package/dist/src/services/agent-manager/connect-to-manager.d.ts +1 -3
- package/dist/src/services/streaming-manager/common.d.ts +17 -1
- package/dist/src/types/entities/agents/manager.d.ts +10 -0
- package/dist/src/types/stream/stream.d.ts +0 -2
- package/package.json +1 -1
- package/dist/src/services/interrupt/index.d.ts +0 -6
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
import { c as De, C as O, S as u, a as Fe, b as Ne, A as m, d as Ue, l as de, e as v, f as F, g as je, h as xe, i as A, n as Oe } from "./index-BkSUCJW5.js";
|
|
2
|
+
function Je(T, L, g, i) {
|
|
3
|
+
const k = De(T, `${L}/v2/agents/${g}`, i);
|
|
4
|
+
return {
|
|
5
|
+
async createStream(l) {
|
|
6
|
+
return k.post("/sessions", l);
|
|
7
|
+
}
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
const Be = {
|
|
11
|
+
[u.ChatAnswer]: O.Answer,
|
|
12
|
+
[u.ChatPartial]: O.Partial
|
|
13
|
+
}, ue = 2e4;
|
|
14
|
+
async function Qe() {
|
|
15
|
+
try {
|
|
16
|
+
return await import("./livekit-client.esm-RGXA_gNu.js");
|
|
17
|
+
} catch {
|
|
18
|
+
throw new Error(
|
|
19
|
+
"LiveKit client is required for this streaming manager. Please install it using: npm install livekit-client"
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
const qe = {
|
|
24
|
+
excellent: F.Strong,
|
|
25
|
+
good: F.Strong,
|
|
26
|
+
poor: F.Weak,
|
|
27
|
+
lost: F.Unknown,
|
|
28
|
+
unknown: F.Unknown
|
|
29
|
+
}, N = JSON.stringify({
|
|
30
|
+
kind: "InternalServerError",
|
|
31
|
+
description: "Stream Error"
|
|
32
|
+
});
|
|
33
|
+
function le(T, L, g) {
|
|
34
|
+
var i, k;
|
|
35
|
+
throw L("Failed to connect to LiveKit room:", T), (i = g.onConnectionStateChange) == null || i.call(g, v.Fail, "internal:init-error"), (k = g.onError) == null || k.call(g, T, { sessionId: "" }), T;
|
|
36
|
+
}
|
|
37
|
+
async function We(T, L, g) {
|
|
38
|
+
var re;
|
|
39
|
+
const i = Fe(g.debug || !1, "LiveKitStreamingManager"), { Room: k, RoomEvent: l, ConnectionState: M, Track: H } = await Qe(), { callbacks: n, auth: pe, baseURL: he, analytics: G } = g;
|
|
40
|
+
let a = null, w = !1;
|
|
41
|
+
const X = Ne.Fluent;
|
|
42
|
+
let R = null;
|
|
43
|
+
const P = { isPublishing: !1, publication: null }, J = { isPublishing: !1, publication: null };
|
|
44
|
+
let h = null, f = null, U = null, V = !1;
|
|
45
|
+
a = new k({
|
|
46
|
+
adaptiveStream: !1,
|
|
47
|
+
// Must be false to use mediaStreamTrack directly
|
|
48
|
+
dynacast: !0
|
|
49
|
+
});
|
|
50
|
+
let E = null, b = m.Idle, j = !0;
|
|
51
|
+
const ge = Je(pe, he || Ue, T, n.onError);
|
|
52
|
+
let C, I, $, Y = !0;
|
|
53
|
+
try {
|
|
54
|
+
const e = await ge.createStream({
|
|
55
|
+
transport: L.transport,
|
|
56
|
+
chat_persist: L.chat_persist ?? !0
|
|
57
|
+
}), { id: t, session_token: r, session_url: o, interrupt_enabled: c } = e;
|
|
58
|
+
(re = n.onStreamCreated) == null || re.call(n, { session_id: t, stream_id: t, agent_id: T }), C = t, I = r, $ = o, Y = c ?? !0, await a.prepareConnection($, I);
|
|
59
|
+
} catch (e) {
|
|
60
|
+
le(e, i, n);
|
|
61
|
+
}
|
|
62
|
+
if (!$ || !I || !C)
|
|
63
|
+
return Promise.reject(new Error("Failed to initialize LiveKit stream"));
|
|
64
|
+
a.on(l.ConnectionStateChanged, me).on(l.ConnectionQualityChanged, Ce).on(l.ParticipantConnected, Se).on(l.ParticipantDisconnected, Te).on(l.TrackSubscribed, ve).on(l.TrackUnsubscribed, we).on(l.DataReceived, Pe).on(l.MediaDevicesError, Le).on(l.TranscriptionReceived, fe).on(l.EncryptionError, ke).on(l.TrackSubscriptionFailed, Me);
|
|
65
|
+
function fe(e, t) {
|
|
66
|
+
t != null && t.isLocal && (de.update(), b === m.Talking && (b = m.Idle));
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
await a.connect($, I), i("LiveKit room joined successfully"), E = setTimeout(() => {
|
|
70
|
+
var e;
|
|
71
|
+
i(
|
|
72
|
+
`Track subscription timeout - no track subscribed within ${ue / 1e3} seconds after connect`
|
|
73
|
+
), E = null, G.track("connectivity-error", {
|
|
74
|
+
error: "Track subscription timeout",
|
|
75
|
+
sessionId: C
|
|
76
|
+
}), (e = n.onError) == null || e.call(n, new Error("Track subscription timeout"), { sessionId: C }), W("internal:track-subscription-timeout");
|
|
77
|
+
}, ue);
|
|
78
|
+
} catch (e) {
|
|
79
|
+
le(e, i, n);
|
|
80
|
+
}
|
|
81
|
+
G.enrich({
|
|
82
|
+
"stream-type": X
|
|
83
|
+
});
|
|
84
|
+
function me(e) {
|
|
85
|
+
var t, r, o, c;
|
|
86
|
+
switch (i("Connection state changed:", e), e) {
|
|
87
|
+
case M.Connecting:
|
|
88
|
+
i("CALLBACK: onConnectionStateChange(Connecting)"), (t = n.onConnectionStateChange) == null || t.call(n, v.Connecting, "livekit:connecting");
|
|
89
|
+
break;
|
|
90
|
+
case M.Connected:
|
|
91
|
+
i("LiveKit room connected successfully"), w = !0;
|
|
92
|
+
break;
|
|
93
|
+
case M.Disconnected:
|
|
94
|
+
i("LiveKit room disconnected"), w = !1, V = !1, P.publication = null, J.publication = null, (r = n.onConnectionStateChange) == null || r.call(n, v.Disconnected, "livekit:disconnected");
|
|
95
|
+
break;
|
|
96
|
+
case M.Reconnecting:
|
|
97
|
+
i("LiveKit room reconnecting..."), (o = n.onConnectionStateChange) == null || o.call(n, v.Connecting, "livekit:reconnecting");
|
|
98
|
+
break;
|
|
99
|
+
case M.SignalReconnecting:
|
|
100
|
+
i("LiveKit room signal reconnecting..."), (c = n.onConnectionStateChange) == null || c.call(n, v.Connecting, "livekit:signal-reconnecting");
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
function Ce(e, t) {
|
|
105
|
+
var r;
|
|
106
|
+
i("Connection quality:", e), t != null && t.isLocal && ((r = n.onConnectivityStateChange) == null || r.call(n, qe[e]));
|
|
107
|
+
}
|
|
108
|
+
function Se(e) {
|
|
109
|
+
i("Participant connected:", e.identity);
|
|
110
|
+
}
|
|
111
|
+
function Te(e) {
|
|
112
|
+
i("Participant disconnected:", e.identity), W("livekit:participant-disconnected");
|
|
113
|
+
}
|
|
114
|
+
function ye() {
|
|
115
|
+
var e;
|
|
116
|
+
U !== A.Start && (i("CALLBACK: onVideoStateChange(Start)"), U = A.Start, (e = n.onVideoStateChange) == null || e.call(n, A.Start));
|
|
117
|
+
}
|
|
118
|
+
function Z(e) {
|
|
119
|
+
var t;
|
|
120
|
+
U !== A.Stop && (i("CALLBACK: onVideoStateChange(Stop)"), U = A.Stop, (t = n.onVideoStateChange) == null || t.call(n, A.Stop, e));
|
|
121
|
+
}
|
|
122
|
+
function ve(e, t, r) {
|
|
123
|
+
var c, s, p;
|
|
124
|
+
i(`Track subscribed: ${e.kind} from ${r.identity}`);
|
|
125
|
+
const o = e.mediaStreamTrack;
|
|
126
|
+
if (!o) {
|
|
127
|
+
i(`No mediaStreamTrack available for ${e.kind}`);
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
R ? (R.addTrack(o), i(`Added ${e.kind} track to shared MediaStream`)) : (R = new MediaStream([o]), i(`Created shared MediaStream with ${e.kind} track`)), e.kind === "audio" && (f = xe(
|
|
131
|
+
() => e.getRTCStatsReport(),
|
|
132
|
+
({ sttLatency: d, serviceLatency: S }) => {
|
|
133
|
+
var oe, ae, ce;
|
|
134
|
+
const _ = de.get(!0);
|
|
135
|
+
let D = 0;
|
|
136
|
+
if (d) {
|
|
137
|
+
const se = ((ae = (oe = h == null ? void 0 : h.getReport()) == null ? void 0 : oe.webRTCStats) == null ? void 0 : ae.avgRtt) ?? 0;
|
|
138
|
+
D = se > 0 ? Math.round(se * 1e3) : 0;
|
|
139
|
+
}
|
|
140
|
+
const K = _ > 0 ? _ + (d ?? 0) + D : void 0, y = K !== void 0 && S !== void 0 ? K - S : void 0;
|
|
141
|
+
(ce = n.onFirstAudioDetected) == null || ce.call(n, { latency: K, networkLatency: y });
|
|
142
|
+
}
|
|
143
|
+
)), e.kind === "video" && ((c = n.onStreamReady) == null || c.call(n), i("CALLBACK: onSrcObjectReady"), (s = n.onSrcObjectReady) == null || s.call(n, R), V || (V = !0, i("CALLBACK: onConnectionStateChange(Connected)"), (p = n.onConnectionStateChange) == null || p.call(n, v.Connected, "livekit:track-subscribed")), h = je(
|
|
144
|
+
() => e.getRTCStatsReport(),
|
|
145
|
+
() => w,
|
|
146
|
+
Oe,
|
|
147
|
+
(d, S) => {
|
|
148
|
+
i(`Video state change: ${d}`), d === A.Start ? (E && (clearTimeout(E), E = null, i("Track subscription timeout cleared")), ye()) : d === A.Stop && Z(S);
|
|
149
|
+
}
|
|
150
|
+
), h.start());
|
|
151
|
+
}
|
|
152
|
+
function we(e, t, r) {
|
|
153
|
+
i(`Track unsubscribed: ${e.kind} from ${r.identity}`), e.kind === "audio" && (f == null || f.destroy(), f = null), e.kind === "video" && (Z(h == null ? void 0 : h.getReport()), h == null || h.stop(), h = null);
|
|
154
|
+
}
|
|
155
|
+
function ee(e, t) {
|
|
156
|
+
var o;
|
|
157
|
+
const r = Be[e];
|
|
158
|
+
r && ((o = n.onMessage) == null || o.call(n, r, { event: r, ...t }));
|
|
159
|
+
}
|
|
160
|
+
function B(e, t) {
|
|
161
|
+
var r, o, c, s;
|
|
162
|
+
if (e === u.ToolCallStarted) {
|
|
163
|
+
b = m.ToolActive, (r = n.onAgentActivityStateChange) == null || r.call(n, m.ToolActive), (o = n.onToolEvent) == null || o.call(n, u.ToolCallStarted, t);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
if (e === u.ToolCallDone) {
|
|
167
|
+
(c = n.onToolEvent) == null || c.call(n, u.ToolCallDone, t);
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
e === u.ToolCallError && ((s = n.onToolEvent) == null || s.call(n, u.ToolCallError, t));
|
|
171
|
+
}
|
|
172
|
+
function Ee(e, t) {
|
|
173
|
+
var r, o, c, s, p;
|
|
174
|
+
if (j = ((r = t.metadata) == null ? void 0 : r.interruptible) !== !1, (o = n.onInterruptibleChange) == null || o.call(n, j), e === u.StreamVideoCreated) {
|
|
175
|
+
b = m.Talking, (c = n.onAgentActivityStateChange) == null || c.call(n, m.Talking), f == null || f.arm({
|
|
176
|
+
sttLatency: (s = t == null ? void 0 : t.stt) == null ? void 0 : s.latency,
|
|
177
|
+
serviceLatency: t == null ? void 0 : t.serviceLatency
|
|
178
|
+
});
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
j && (b = m.Idle, (p = n.onAgentActivityStateChange) == null || p.call(n, m.Idle));
|
|
182
|
+
}
|
|
183
|
+
function x(e, t) {
|
|
184
|
+
var s, p, d, S;
|
|
185
|
+
const r = ((p = (s = h == null ? void 0 : h.getReport()) == null ? void 0 : s.webRTCStats) == null ? void 0 : p.avgRtt) ?? 0, o = r > 0 ? Math.round(r / 2 * 1e3) : 0, c = { ...t, downstreamNetworkLatency: o };
|
|
186
|
+
g.debug && ((d = t == null ? void 0 : t.metadata) != null && d.sentiment) && (c.sentiment = {
|
|
187
|
+
id: t.metadata.sentiment.id,
|
|
188
|
+
name: t.metadata.sentiment.sentiment
|
|
189
|
+
}), (S = n.onMessage) == null || S.call(n, e, c), Ee(e, t);
|
|
190
|
+
}
|
|
191
|
+
function Ae(e, t) {
|
|
192
|
+
var r;
|
|
193
|
+
(r = n.onMessage) == null || r.call(n, O.Transcribe, { event: O.Transcribe, ...t }), queueMicrotask(() => {
|
|
194
|
+
var o;
|
|
195
|
+
(o = n.onAgentActivityStateChange) == null || o.call(n, m.Loading);
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
const Re = {
|
|
199
|
+
[u.ChatAnswer]: ee,
|
|
200
|
+
[u.ChatPartial]: ee,
|
|
201
|
+
[u.ToolCallStarted]: B,
|
|
202
|
+
[u.ToolCallDone]: B,
|
|
203
|
+
[u.ToolCallError]: B,
|
|
204
|
+
[u.StreamVideoCreated]: x,
|
|
205
|
+
[u.StreamVideoDone]: x,
|
|
206
|
+
[u.StreamVideoError]: x,
|
|
207
|
+
[u.StreamVideoRejected]: x,
|
|
208
|
+
[u.ChatAudioTranscribed]: Ae
|
|
209
|
+
};
|
|
210
|
+
function Pe(e, t, r, o) {
|
|
211
|
+
const c = new TextDecoder().decode(e);
|
|
212
|
+
try {
|
|
213
|
+
const s = JSON.parse(c), p = o || s.subject;
|
|
214
|
+
if (i("Data received:", { subject: p, data: s }), !p) return;
|
|
215
|
+
const d = Re[p];
|
|
216
|
+
d == null || d(p, s);
|
|
217
|
+
} catch (s) {
|
|
218
|
+
i("Failed to parse data channel message:", s);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
function Le(e) {
|
|
222
|
+
var t;
|
|
223
|
+
i("Media devices error:", e), (t = n.onError) == null || t.call(n, new Error(N), { sessionId: C });
|
|
224
|
+
}
|
|
225
|
+
function ke(e) {
|
|
226
|
+
var t;
|
|
227
|
+
i("Encryption error:", e), (t = n.onError) == null || t.call(n, new Error(N), { sessionId: C });
|
|
228
|
+
}
|
|
229
|
+
function Me(e, t, r) {
|
|
230
|
+
i("Track subscription failed:", { trackSid: e, participant: t, reason: r });
|
|
231
|
+
}
|
|
232
|
+
function be(e, t, r) {
|
|
233
|
+
for (const [o, c] of r)
|
|
234
|
+
if (c.source === t && c.track) {
|
|
235
|
+
const s = c.track.mediaStreamTrack;
|
|
236
|
+
if (s === e || (s == null ? void 0 : s.id) === e.id)
|
|
237
|
+
return c;
|
|
238
|
+
}
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
async function ne(e, t, r, o, c, s) {
|
|
242
|
+
var _, D, K;
|
|
243
|
+
if (!w || !a)
|
|
244
|
+
throw i(`Room is not connected, cannot publish ${o} stream`), new Error("Room is not connected");
|
|
245
|
+
if (e.isPublishing) {
|
|
246
|
+
i(`${o} publish already in progress, skipping`);
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
const p = r(t);
|
|
250
|
+
if (p.length === 0)
|
|
251
|
+
throw new Error(`No ${o} track found in the provided MediaStream`);
|
|
252
|
+
const d = p[0], S = be(d, o, c());
|
|
253
|
+
if (S) {
|
|
254
|
+
i(`${o} track is already published, skipping`, {
|
|
255
|
+
trackId: d.id,
|
|
256
|
+
publishedTrackId: (D = (_ = S.track) == null ? void 0 : _.mediaStreamTrack) == null ? void 0 : D.id
|
|
257
|
+
}), e.publication = S;
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
if ((K = e.publication) != null && K.track) {
|
|
261
|
+
const y = e.publication.track.mediaStreamTrack;
|
|
262
|
+
y !== d && (y == null ? void 0 : y.id) !== d.id && (i(`Unpublishing existing ${o} track before publishing new one`), await s());
|
|
263
|
+
}
|
|
264
|
+
i(`Publishing ${o} track from provided MediaStream`, { trackId: d.id }), e.isPublishing = !0;
|
|
265
|
+
try {
|
|
266
|
+
e.publication = await a.localParticipant.publishTrack(d, { source: o }), i(`${o} track published successfully`, { trackSid: e.publication.trackSid });
|
|
267
|
+
} catch (y) {
|
|
268
|
+
throw i(`Failed to publish ${o} track:`, y), y;
|
|
269
|
+
} finally {
|
|
270
|
+
e.isPublishing = !1;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
async function te(e, t) {
|
|
274
|
+
if (!(!e.publication || !e.publication.track))
|
|
275
|
+
try {
|
|
276
|
+
a && (await a.localParticipant.unpublishTrack(e.publication.track, !1), i(`${t} track unpublished`));
|
|
277
|
+
} catch (r) {
|
|
278
|
+
i(`Error unpublishing ${t} track:`, r);
|
|
279
|
+
} finally {
|
|
280
|
+
e.publication = null;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
async function Ie(e) {
|
|
284
|
+
return ne(
|
|
285
|
+
P,
|
|
286
|
+
e,
|
|
287
|
+
(t) => t.getAudioTracks(),
|
|
288
|
+
H.Source.Microphone,
|
|
289
|
+
() => a.localParticipant.audioTrackPublications,
|
|
290
|
+
Q
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
async function Q() {
|
|
294
|
+
return te(P, "Microphone");
|
|
295
|
+
}
|
|
296
|
+
async function $e(e) {
|
|
297
|
+
if (!w || !a)
|
|
298
|
+
throw i("Cannot replace microphone track: room is not connected"), new Error("Room is not connected");
|
|
299
|
+
if (e.kind !== "audio")
|
|
300
|
+
throw i("Cannot replace microphone track: not an audio track", { kind: e.kind }), new Error("Microphone track must be an audio track");
|
|
301
|
+
if (P.isPublishing)
|
|
302
|
+
throw i("Cannot replace microphone track: publish in progress"), new Error("Microphone publish in progress");
|
|
303
|
+
const t = P.publication;
|
|
304
|
+
if (!t || !t.track)
|
|
305
|
+
throw i("Cannot replace microphone track: no publication to replace"), new Error("No microphone publication to replace");
|
|
306
|
+
try {
|
|
307
|
+
P.isPublishing = !0, await t.track.replaceTrack(e), i("Microphone track replaced", { trackId: e.id, trackSid: t.trackSid });
|
|
308
|
+
} finally {
|
|
309
|
+
P.isPublishing = !1;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
async function Ke(e) {
|
|
313
|
+
return ne(
|
|
314
|
+
J,
|
|
315
|
+
e,
|
|
316
|
+
(t) => t.getVideoTracks(),
|
|
317
|
+
H.Source.Camera,
|
|
318
|
+
() => a.localParticipant.videoTrackPublications,
|
|
319
|
+
q
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
async function q() {
|
|
323
|
+
return te(J, "Camera");
|
|
324
|
+
}
|
|
325
|
+
function Ve() {
|
|
326
|
+
R && (R.getTracks().forEach((e) => e.stop()), R = null);
|
|
327
|
+
}
|
|
328
|
+
async function z(e, t) {
|
|
329
|
+
var r, o;
|
|
330
|
+
if (!w || !a) {
|
|
331
|
+
i("Room is not connected for sending messages"), (r = n.onError) == null || r.call(n, new Error(N), {
|
|
332
|
+
sessionId: C
|
|
333
|
+
});
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
try {
|
|
337
|
+
await a.localParticipant.sendText(e, { topic: t }), i("Message sent successfully:", e);
|
|
338
|
+
} catch (c) {
|
|
339
|
+
i("Failed to send message:", c), (o = n.onError) == null || o.call(n, new Error(N), { sessionId: C });
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
async function ie(e) {
|
|
343
|
+
var t;
|
|
344
|
+
try {
|
|
345
|
+
const o = JSON.parse(e).topic;
|
|
346
|
+
return z("", o);
|
|
347
|
+
} catch (r) {
|
|
348
|
+
i("Failed to send data channel message:", r), (t = n.onError) == null || t.call(n, new Error(N), { sessionId: C });
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
function _e(e) {
|
|
352
|
+
return z(
|
|
353
|
+
e,
|
|
354
|
+
"lk.chat"
|
|
355
|
+
/* Chat */
|
|
356
|
+
);
|
|
357
|
+
}
|
|
358
|
+
async function W(e) {
|
|
359
|
+
var t, r;
|
|
360
|
+
E && (clearTimeout(E), E = null), f == null || f.destroy(), f = null, a && ((t = n.onConnectionStateChange) == null || t.call(n, v.Disconnecting, e), await Promise.all([Q(), q()]), await a.disconnect()), Ve(), w = !1, V = !1, (r = n.onAgentActivityStateChange) == null || r.call(n, m.Idle), b = m.Idle;
|
|
361
|
+
}
|
|
362
|
+
return {
|
|
363
|
+
speak(e) {
|
|
364
|
+
const t = typeof e == "string" ? e : JSON.stringify(e);
|
|
365
|
+
return z(
|
|
366
|
+
t,
|
|
367
|
+
"did.speak"
|
|
368
|
+
/* Speak */
|
|
369
|
+
);
|
|
370
|
+
},
|
|
371
|
+
disconnect: () => W("user:disconnect"),
|
|
372
|
+
async reconnect() {
|
|
373
|
+
var e, t;
|
|
374
|
+
if ((a == null ? void 0 : a.state) === M.Connected) {
|
|
375
|
+
i("Room is already connected");
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (!a || !$ || !I)
|
|
379
|
+
throw i("Cannot reconnect: missing room, URL or token"), new Error("Cannot reconnect: session not available");
|
|
380
|
+
i("Reconnecting to LiveKit room, state:", a.state), V = !1, (e = n.onConnectionStateChange) == null || e.call(n, v.Connecting, "user:reconnect");
|
|
381
|
+
try {
|
|
382
|
+
if (await a.connect($, I), i("Room reconnected"), w = !0, a.remoteParticipants.size === 0) {
|
|
383
|
+
if (i("Waiting for agent to join..."), !await new Promise((o) => {
|
|
384
|
+
const c = setTimeout(() => {
|
|
385
|
+
a == null || a.off(l.ParticipantConnected, s), o(!1);
|
|
386
|
+
}, 5e3), s = () => {
|
|
387
|
+
clearTimeout(c), a == null || a.off(l.ParticipantConnected, s), o(!0);
|
|
388
|
+
};
|
|
389
|
+
a == null || a.on(l.ParticipantConnected, s);
|
|
390
|
+
}))
|
|
391
|
+
throw i("Agent did not join within timeout"), await a.disconnect(), new Error("Agent did not rejoin the room");
|
|
392
|
+
i("Agent joined, reconnection successful");
|
|
393
|
+
}
|
|
394
|
+
} catch (r) {
|
|
395
|
+
throw i("Failed to reconnect:", r), (t = n.onConnectionStateChange) == null || t.call(n, v.Fail, "user:reconnect-failed"), r;
|
|
396
|
+
}
|
|
397
|
+
},
|
|
398
|
+
sendDataChannelMessage: ie,
|
|
399
|
+
sendTextMessage: _e,
|
|
400
|
+
publishMicrophoneStream: Ie,
|
|
401
|
+
unpublishMicrophoneStream: Q,
|
|
402
|
+
replaceMicrophoneTrack: $e,
|
|
403
|
+
publishCameraStream: Ke,
|
|
404
|
+
unpublishCameraStream: q,
|
|
405
|
+
interrupt(e) {
|
|
406
|
+
e !== "text" && ie(JSON.stringify({
|
|
407
|
+
topic: "did.interrupt"
|
|
408
|
+
/* Interrupt */
|
|
409
|
+
}));
|
|
410
|
+
},
|
|
411
|
+
registerRpcMethod(e, t) {
|
|
412
|
+
a == null || a.registerRpcMethod(e, t);
|
|
413
|
+
},
|
|
414
|
+
unregisterRpcMethod(e) {
|
|
415
|
+
a == null || a.unregisterRpcMethod(e);
|
|
416
|
+
},
|
|
417
|
+
sessionId: C,
|
|
418
|
+
streamId: C,
|
|
419
|
+
streamType: X,
|
|
420
|
+
interruptAvailable: Y,
|
|
421
|
+
isInterruptible: j
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
export {
|
|
425
|
+
We as createLiveKitStreamingManager,
|
|
426
|
+
le as handleInitError
|
|
427
|
+
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { StreamingManager } from '../streaming-manager';
|
|
2
|
-
import { Agent, AgentManagerOptions, AgentsAPI, AudioDetectionMetrics, Chat, ChatProgressCallback, CreateSessionV2Options, CreateStreamOptions
|
|
2
|
+
import { Agent, AgentManagerOptions, AgentsAPI, AudioDetectionMetrics, Chat, ChatProgressCallback, CreateSessionV2Options, CreateStreamOptions } from '../../types';
|
|
3
3
|
import { Analytics } from '../analytics/mixpanel';
|
|
4
4
|
|
|
5
5
|
type ConnectToManagerOptions = AgentManagerOptions & {
|
|
@@ -7,8 +7,6 @@ type ConnectToManagerOptions = AgentManagerOptions & {
|
|
|
7
7
|
onVideoIdChange?: (videoId: string | null) => void;
|
|
8
8
|
/** Internal callback for livekit-manager data channel events */
|
|
9
9
|
onMessage?: ChatProgressCallback;
|
|
10
|
-
/** Internal callback for when interrupt is detected by streaming manager */
|
|
11
|
-
onInterruptDetected?: (interrupt: Interrupt) => void;
|
|
12
10
|
onFirstAudioDetected?: (metrics: AudioDetectionMetrics) => void;
|
|
13
11
|
};
|
|
14
12
|
chatId?: string;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CreateSessionV2Options, CreateStreamOptions, PayloadType, StreamType } from '../../types';
|
|
1
|
+
import { CreateSessionV2Options, CreateStreamOptions, Interrupt, PayloadType, StreamType } from '../../types';
|
|
2
2
|
|
|
3
3
|
export declare const createStreamingLogger: (debug: boolean, prefix: string) => (message: string, extra?: any) => false | void;
|
|
4
4
|
/**
|
|
@@ -39,6 +39,15 @@ export type StreamingManager<T extends CreateStreamOptions | CreateSessionV2Opti
|
|
|
39
39
|
* supported only for livekit manager
|
|
40
40
|
*/
|
|
41
41
|
unpublishMicrophoneStream?(): Promise<void>;
|
|
42
|
+
/**
|
|
43
|
+
* Replace the currently published microphone MediaStreamTrack without
|
|
44
|
+
* unpublishing. Preserves the LiveKit publication (SSRC, trackSid) so the
|
|
45
|
+
* server sees continuous audio. Resolves once LiveKit has switched the
|
|
46
|
+
* underlying RTCRtpSender's track. Rejects if no publication exists —
|
|
47
|
+
* caller should fall back to `publishMicrophoneStream` in that case.
|
|
48
|
+
* supported only for livekit manager
|
|
49
|
+
*/
|
|
50
|
+
replaceMicrophoneTrack?(track: MediaStreamTrack): Promise<void>;
|
|
42
51
|
/**
|
|
43
52
|
* Publish a camera video stream to the LiveKit room.
|
|
44
53
|
* Can be called after connection to enable vision.
|
|
@@ -71,6 +80,13 @@ export type StreamingManager<T extends CreateStreamOptions | CreateSessionV2Opti
|
|
|
71
80
|
* Whether the current stream segment can be interrupted by the user
|
|
72
81
|
*/
|
|
73
82
|
isInterruptible: boolean;
|
|
83
|
+
/**
|
|
84
|
+
* Send an interrupt for the current stream segment.
|
|
85
|
+
* Each implementation owns the validation/transport details (e.g. V1
|
|
86
|
+
* sends a `stream/interrupt` payload over the data channel; V2 sends
|
|
87
|
+
* `did.interrupt` and ignores `text` interrupts to avoid races).
|
|
88
|
+
*/
|
|
89
|
+
interrupt(type: Interrupt['type']): void;
|
|
74
90
|
/**
|
|
75
91
|
* Register an RPC method handler on the LiveKit room.
|
|
76
92
|
* Used internally by the agent-manager for client tool delegation.
|
|
@@ -203,6 +203,16 @@ export interface AgentManager {
|
|
|
203
203
|
* supported only for livekit manager
|
|
204
204
|
*/
|
|
205
205
|
unpublishMicrophoneStream?: () => Promise<void>;
|
|
206
|
+
/**
|
|
207
|
+
* Replace the live microphone track on the current publication without
|
|
208
|
+
* unpublishing. Preserves the LiveKit publication (SSRC, trackSid) so the
|
|
209
|
+
* server sees continuous audio across mic device swaps. Resolves once
|
|
210
|
+
* LiveKit has switched the underlying RTCRtpSender's track.
|
|
211
|
+
* Rejects if there is no active publication — callers should fall back to
|
|
212
|
+
* `publishMicrophoneStream` in that case.
|
|
213
|
+
* Supported only for the LiveKit streaming manager.
|
|
214
|
+
*/
|
|
215
|
+
replaceMicrophoneTrack?: (track: MediaStreamTrack) => Promise<void>;
|
|
206
216
|
/**
|
|
207
217
|
* Publish a camera video stream to the LiveKit room.
|
|
208
218
|
* Can be called after connection to enable vision.
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Analytics } from '../../services/analytics/mixpanel';
|
|
2
2
|
import { VideoRTCStatsReport } from '../../services/streaming-manager/stats/report';
|
|
3
3
|
import { Auth } from '../auth';
|
|
4
|
-
import { Interrupt } from '../entities';
|
|
5
4
|
import { ChatProgressCallback } from '../entities/agents/manager';
|
|
6
5
|
import { CreateClipStreamRequest, CreateTalkStreamRequest, SendClipStreamPayload, SendTalkStreamPayload } from './api';
|
|
7
6
|
import { ICreateStreamRequestResponse, IceCandidate, SendStreamPayloadResponse, Status } from './rtc';
|
|
@@ -69,7 +68,6 @@ export interface ManagerCallbacks {
|
|
|
69
68
|
agent_id: string;
|
|
70
69
|
}) => void;
|
|
71
70
|
onStreamReady?: () => void;
|
|
72
|
-
onInterruptDetected?: (interrupt: Interrupt) => void;
|
|
73
71
|
onToolEvent?: ToolEventCallback;
|
|
74
72
|
onInterruptibleChange?: (interruptible: boolean) => void;
|
|
75
73
|
onFirstAudioDetected?: (metrics: AudioDetectionMetrics) => void;
|
package/package.json
CHANGED
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
import { CreateSessionV2Options, CreateStreamOptions, StreamType } from '../../types';
|
|
2
|
-
import { StreamingManager } from '../streaming-manager';
|
|
3
|
-
|
|
4
|
-
export declare function validateInterrupt(streamingManager: StreamingManager<CreateStreamOptions> | undefined, streamType: StreamType | undefined, videoId: string | null): void;
|
|
5
|
-
export declare function sendInterrupt(streamingManager: StreamingManager<CreateStreamOptions>, videoId: string): Promise<void>;
|
|
6
|
-
export declare function sendInterruptV2(streamingManager: StreamingManager<CreateSessionV2Options>): Promise<void>;
|