@d-id/client-sdk 3.0.0-staging.412 → 3.0.0-staging.414
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-BLrKDuV_.js → index-DBWnR72S.js} +1628 -1615
- package/dist/index.js +20 -20
- package/dist/index.umd.cjs +7 -7
- package/dist/livekit-manager-BM3JMzT5.js +531 -0
- package/dist/src/api/agents.d.ts +5 -5
- package/dist/src/api/streams/streamApi.test.d.ts +1 -0
- package/dist/src/index.d.ts +6 -6
- package/dist/src/services/streaming-manager/common.d.ts +4 -4
- package/dist/src/types/entities/agents/chat.d.ts +9 -9
- package/dist/src/types/entities/agents/manager.d.ts +31 -30
- package/dist/src/types/stream/data-channel.d.ts +2 -2
- package/dist/src/types/stream/rtc.d.ts +14 -3
- package/dist/src/types/stream/stream.d.ts +114 -43
- package/dist/src/types/stream-script.d.ts +5 -5
- package/dist/src/types/voice/stt.d.ts +2 -2
- package/dist/src/types/voice/tts.d.ts +1 -1
- package/package.json +1 -1
- package/dist/livekit-manager-Cfb9x2ei.js +0 -506
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
import { c as qe, C as J, S as p, a as Ge, b as He, A as C, d as Xe, l as me, e as Ze, t as Ye, f as w, g as U, h as en, I as Ce, i as nn, j as R, T as ee, n as tn } from "./index-DBWnR72S.js";
|
|
2
|
+
function on() {
|
|
3
|
+
try {
|
|
4
|
+
return Intl.DateTimeFormat().resolvedOptions().timeZone ?? "";
|
|
5
|
+
} catch {
|
|
6
|
+
return "";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
function rn() {
|
|
10
|
+
if (typeof navigator > "u")
|
|
11
|
+
return "";
|
|
12
|
+
const c = navigator.userAgent ?? "", S = (navigator.platform ?? "").toLowerCase();
|
|
13
|
+
let d = "Unknown";
|
|
14
|
+
/android/i.test(c) ? d = "Android" : /iphone|ipad|ipod/i.test(c) ? d = "iOS" : S.includes("win") ? d = "Windows" : S.includes("mac") ? d = "Mac OS X" : S.includes("linux") && (d = "Linux");
|
|
15
|
+
const i = d === "Android" || d === "iOS" || /Mobi/i.test(c);
|
|
16
|
+
return `${d}, ${i ? "Mobile" : "Desktop"}`;
|
|
17
|
+
}
|
|
18
|
+
function an() {
|
|
19
|
+
const c = {}, S = on();
|
|
20
|
+
S && (c.timezone = S);
|
|
21
|
+
const d = rn();
|
|
22
|
+
return d && (c.device = d), c;
|
|
23
|
+
}
|
|
24
|
+
function cn(c, S, d, i) {
|
|
25
|
+
const M = qe(c, `${S}/v2/agents/${d}`, i);
|
|
26
|
+
return {
|
|
27
|
+
async createStream(h) {
|
|
28
|
+
return M.post("/sessions", h);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
const sn = {
|
|
33
|
+
[p.ChatAnswer]: J.Answer,
|
|
34
|
+
[p.ChatPartial]: J.Partial
|
|
35
|
+
}, Se = 2e4, un = [];
|
|
36
|
+
async function dn() {
|
|
37
|
+
try {
|
|
38
|
+
return await import("./index-DBWnR72S.js").then((c) => c.x);
|
|
39
|
+
} catch {
|
|
40
|
+
throw new Error(
|
|
41
|
+
"LiveKit client is required for this streaming manager. Please install it using: npm install livekit-client"
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const ln = {
|
|
46
|
+
excellent: U.Strong,
|
|
47
|
+
good: U.Strong,
|
|
48
|
+
poor: U.Weak,
|
|
49
|
+
lost: U.Unknown,
|
|
50
|
+
unknown: U.Unknown
|
|
51
|
+
}, j = (c = "Stream Error") => new Ze(c);
|
|
52
|
+
function pn(c) {
|
|
53
|
+
return {
|
|
54
|
+
callId: c.call_id,
|
|
55
|
+
name: c.name,
|
|
56
|
+
input: c.input,
|
|
57
|
+
output: c.output,
|
|
58
|
+
// The server omits the field on some started events; the public payload declares a boolean.
|
|
59
|
+
interruptible: c.interruptible === !0,
|
|
60
|
+
// Spread, so an event that carries neither does not gain the keys with an undefined value.
|
|
61
|
+
...c.execution_mode !== void 0 ? { executionMode: c.execution_mode } : {},
|
|
62
|
+
...c.turn_id !== void 0 ? { turnId: c.turn_id } : {},
|
|
63
|
+
timestamp: c.timestamp
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function Te(c) {
|
|
67
|
+
return {
|
|
68
|
+
callId: c.call_id,
|
|
69
|
+
name: c.name,
|
|
70
|
+
input: c.input,
|
|
71
|
+
output: c.output,
|
|
72
|
+
durationMs: c.duration_ms,
|
|
73
|
+
extra: c.extra,
|
|
74
|
+
timestamp: c.timestamp
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function ve(c, S, d) {
|
|
78
|
+
var i, M;
|
|
79
|
+
throw S("Failed to connect to LiveKit room:", c), (i = d.onConnectionStateChange) == null || i.call(d, w.Fail, "internal:init-error"), (M = d.onError) == null || M.call(d, c, { sessionId: "" }), c;
|
|
80
|
+
}
|
|
81
|
+
async function hn(c, S, d) {
|
|
82
|
+
var le;
|
|
83
|
+
const i = Ge(d.debug || !1, "LiveKitStreamingManager"), { Room: M, RoomEvent: h, ConnectionState: k, Track: ne } = await dn(), { callbacks: n, auth: ye, baseURL: Ae, analytics: te } = d;
|
|
84
|
+
let a = null, A = !1;
|
|
85
|
+
const ie = He.Fluent;
|
|
86
|
+
let L = null;
|
|
87
|
+
const P = { isPublishing: !1, publication: null }, Q = { isPublishing: !1, publication: null };
|
|
88
|
+
let g = null, T = null, O = null, D = !1;
|
|
89
|
+
a = new M({
|
|
90
|
+
adaptiveStream: !1,
|
|
91
|
+
// Must be false to use mediaStreamTrack directly
|
|
92
|
+
dynacast: !0
|
|
93
|
+
});
|
|
94
|
+
for (const [e, t] of d.rpcMethods ?? [])
|
|
95
|
+
a.registerRpcMethod(e, t);
|
|
96
|
+
let b = null, I = C.Idle, W = !0;
|
|
97
|
+
const v = /* @__PURE__ */ new Map();
|
|
98
|
+
let K = null, q = null;
|
|
99
|
+
const Ee = cn(ye, Ae || Xe, c, n.onError);
|
|
100
|
+
let y, _, x, oe = !0;
|
|
101
|
+
try {
|
|
102
|
+
const e = await Ee.createStream({
|
|
103
|
+
transport: S.transport,
|
|
104
|
+
chat_persist: S.chat_persist ?? !1,
|
|
105
|
+
verbose: d.verbose ?? !1
|
|
106
|
+
}), { id: t, session_token: o, session_url: r, interrupt_enabled: s } = e;
|
|
107
|
+
(le = n.onStreamCreated) == null || le.call(n, { sessionId: t, streamId: t, agentId: c }), y = t, _ = o, x = r, oe = s ?? !0, await a.prepareConnection(x, _);
|
|
108
|
+
} catch (e) {
|
|
109
|
+
ve(e, i, n);
|
|
110
|
+
}
|
|
111
|
+
if (!x || !_ || !y)
|
|
112
|
+
return Promise.reject(new Error("Failed to initialize LiveKit stream"));
|
|
113
|
+
a.on(h.ConnectionStateChanged, Re).on(h.ConnectionQualityChanged, Le).on(h.ParticipantConnected, Pe).on(h.ParticipantDisconnected, Ie).on(h.TrackSubscribed, ke).on(h.TrackUnsubscribed, _e).on(h.DataReceived, Ue).on(h.MediaDevicesError, je).on(h.TranscriptionReceived, we).on(h.EncryptionError, Oe).on(h.TrackSubscriptionFailed, Ne);
|
|
114
|
+
function we(e, t) {
|
|
115
|
+
t != null && t.isLocal && me.update();
|
|
116
|
+
}
|
|
117
|
+
async function be() {
|
|
118
|
+
const e = an();
|
|
119
|
+
if (!(!a || Object.keys(e).length === 0))
|
|
120
|
+
try {
|
|
121
|
+
await a.localParticipant.setAttributes(e);
|
|
122
|
+
} catch (t) {
|
|
123
|
+
i("Failed to set user context attributes", t);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
await a.connect(x, _), i("LiveKit room joined successfully"), be(), b = setTimeout(() => {
|
|
128
|
+
var t;
|
|
129
|
+
i(
|
|
130
|
+
`Track subscription timeout - no track subscribed within ${Se / 1e3} seconds after connect`
|
|
131
|
+
), b = null;
|
|
132
|
+
const e = j("Track subscription timeout");
|
|
133
|
+
te.track("connectivity-error", {
|
|
134
|
+
error: Ye(e),
|
|
135
|
+
sessionId: y
|
|
136
|
+
}), (t = n.onError) == null || t.call(n, e, { sessionId: y }), Y("internal:track-subscription-timeout");
|
|
137
|
+
}, Se);
|
|
138
|
+
} catch (e) {
|
|
139
|
+
ve(e, i, n);
|
|
140
|
+
}
|
|
141
|
+
te.enrich({
|
|
142
|
+
"stream-type": ie
|
|
143
|
+
});
|
|
144
|
+
function Re(e) {
|
|
145
|
+
var t, o, r, s;
|
|
146
|
+
switch (i("Connection state changed:", e), e) {
|
|
147
|
+
case k.Connecting:
|
|
148
|
+
i("CALLBACK: onConnectionStateChange(Connecting)"), (t = n.onConnectionStateChange) == null || t.call(n, w.Connecting, "livekit:connecting");
|
|
149
|
+
break;
|
|
150
|
+
case k.Connected:
|
|
151
|
+
i("LiveKit room connected successfully"), A = !0;
|
|
152
|
+
break;
|
|
153
|
+
case k.Disconnected:
|
|
154
|
+
i("LiveKit room disconnected"), A = !1, D = !1, P.publication = null, Q.publication = null, (o = n.onConnectionStateChange) == null || o.call(
|
|
155
|
+
n,
|
|
156
|
+
w.Disconnected,
|
|
157
|
+
q ?? "livekit:disconnected"
|
|
158
|
+
);
|
|
159
|
+
break;
|
|
160
|
+
case k.Reconnecting:
|
|
161
|
+
i("LiveKit room reconnecting..."), (r = n.onConnectionStateChange) == null || r.call(n, w.Connecting, "livekit:reconnecting");
|
|
162
|
+
break;
|
|
163
|
+
case k.SignalReconnecting:
|
|
164
|
+
i("LiveKit room signal reconnecting..."), (s = n.onConnectionStateChange) == null || s.call(n, w.Connecting, "livekit:signal-reconnecting");
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
function Le(e, t) {
|
|
169
|
+
var o;
|
|
170
|
+
i("Connection quality:", e), t != null && t.isLocal && ((o = n.onConnectivityStateChange) == null || o.call(n, ln[e]));
|
|
171
|
+
}
|
|
172
|
+
function Pe(e) {
|
|
173
|
+
i("Participant connected:", e.identity);
|
|
174
|
+
}
|
|
175
|
+
function Ie(e) {
|
|
176
|
+
i("Participant disconnected:", e.identity), Y("livekit:participant-disconnected");
|
|
177
|
+
}
|
|
178
|
+
function Me() {
|
|
179
|
+
var e;
|
|
180
|
+
O !== R.Start && (i("CALLBACK: onVideoStateChange(Start)"), O = R.Start, (e = n.onVideoStateChange) == null || e.call(n, R.Start));
|
|
181
|
+
}
|
|
182
|
+
function re(e) {
|
|
183
|
+
var t;
|
|
184
|
+
O !== R.Stop && (i("CALLBACK: onVideoStateChange(Stop)"), O = R.Stop, (t = n.onVideoStateChange) == null || t.call(n, R.Stop, e));
|
|
185
|
+
}
|
|
186
|
+
function ke(e, t, o) {
|
|
187
|
+
var s, l, u;
|
|
188
|
+
i(`Track subscribed: ${e.kind} from ${o.identity}`);
|
|
189
|
+
const r = e.mediaStreamTrack;
|
|
190
|
+
if (!r) {
|
|
191
|
+
i(`No mediaStreamTrack available for ${e.kind}`);
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
L ? (L.addTrack(r), i(`Added ${e.kind} track to shared MediaStream`)) : (L = new MediaStream([r]), i(`Created shared MediaStream with ${e.kind} track`)), e.kind === "audio" && (T = nn(
|
|
195
|
+
() => e.getRTCStatsReport(),
|
|
196
|
+
({ sttLatency: f, serviceLatency: m }) => {
|
|
197
|
+
var pe, fe, he;
|
|
198
|
+
const F = me.get(!0);
|
|
199
|
+
let V = 0;
|
|
200
|
+
if (f) {
|
|
201
|
+
const ge = ((fe = (pe = g == null ? void 0 : g.getReport()) == null ? void 0 : pe.webRTCStats) == null ? void 0 : fe.avgRtt) ?? 0;
|
|
202
|
+
V = ge > 0 ? Math.round(ge * 1e3) : 0;
|
|
203
|
+
}
|
|
204
|
+
const $ = F > 0 ? F + (f ?? 0) + V : void 0, E = $ !== void 0 && m !== void 0 ? $ - m : void 0;
|
|
205
|
+
(he = n.onFirstAudioDetected) == null || he.call(n, { latency: $, networkLatency: E });
|
|
206
|
+
}
|
|
207
|
+
)), e.kind === "video" && ((s = n.onStreamReady) == null || s.call(n), i("CALLBACK: onSrcObjectReady"), (l = n.onSrcObjectReady) == null || l.call(n, L), D || (D = !0, i("CALLBACK: onConnectionStateChange(Connected)"), (u = n.onConnectionStateChange) == null || u.call(n, w.Connected, "livekit:track-subscribed")), g = en(
|
|
208
|
+
() => e.getRTCStatsReport(),
|
|
209
|
+
() => A,
|
|
210
|
+
tn,
|
|
211
|
+
(f, m) => {
|
|
212
|
+
i(`Video state change: ${f}`), f === R.Start ? (b && (clearTimeout(b), b = null, i("Track subscription timeout cleared")), Me()) : f === R.Stop && re(m);
|
|
213
|
+
}
|
|
214
|
+
), g.start());
|
|
215
|
+
}
|
|
216
|
+
function _e(e, t, o) {
|
|
217
|
+
i(`Track unsubscribed: ${e.kind} from ${o.identity}`), e.kind === "audio" && (T == null || T.destroy(), T = null), e.kind === "video" && (re((g == null ? void 0 : g.getReport()) ?? void 0), g == null || g.stop(), g = null);
|
|
218
|
+
}
|
|
219
|
+
function ae(e, t) {
|
|
220
|
+
var r;
|
|
221
|
+
const o = sn[e];
|
|
222
|
+
o && ((r = n.onMessage) == null || r.call(n, o, { event: o, ...t }));
|
|
223
|
+
}
|
|
224
|
+
function G(e, t) {
|
|
225
|
+
var o, r, s, l;
|
|
226
|
+
if (e === p.ToolCallStarted) {
|
|
227
|
+
const u = pn(t);
|
|
228
|
+
v.set(u.callId, {
|
|
229
|
+
call: {
|
|
230
|
+
callId: u.callId,
|
|
231
|
+
name: u.name,
|
|
232
|
+
executionMode: u.executionMode === "async" ? "async" : "blocking"
|
|
233
|
+
},
|
|
234
|
+
interruptible: u.interruptible === !0,
|
|
235
|
+
turnId: u.turnId ?? K
|
|
236
|
+
}), N(), z(), I = C.ToolActive, (o = n.onAgentActivityStateChange) == null || o.call(n, C.ToolActive), (r = n.onToolEvent) == null || r.call(n, ee.Started, u);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
if (e === p.ToolCallDone) {
|
|
240
|
+
const u = Te(t);
|
|
241
|
+
ce(u.callId), (s = n.onToolEvent) == null || s.call(n, ee.Done, u);
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
if (e === p.ToolCallError) {
|
|
245
|
+
const u = Te(t);
|
|
246
|
+
ce(u.callId), (l = n.onToolEvent) == null || l.call(n, ee.Error, u);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function ce(e) {
|
|
250
|
+
v.delete(e) && (N(), z());
|
|
251
|
+
}
|
|
252
|
+
function xe(e) {
|
|
253
|
+
let t = !1;
|
|
254
|
+
for (const [o, r] of v)
|
|
255
|
+
r.call.executionMode === "blocking" && (e !== null && r.turnId !== null && r.turnId !== e || (v.delete(o), t = !0));
|
|
256
|
+
t && (N(), z());
|
|
257
|
+
}
|
|
258
|
+
function N() {
|
|
259
|
+
var t;
|
|
260
|
+
const e = ![...v.values()].some(({ call: o }) => o.executionMode === "blocking");
|
|
261
|
+
e !== W && (W = e, (t = n.onInterruptibleChange) == null || t.call(n, e));
|
|
262
|
+
}
|
|
263
|
+
function z() {
|
|
264
|
+
var e;
|
|
265
|
+
(e = n.onRunningToolCallsChange) == null || e.call(
|
|
266
|
+
n,
|
|
267
|
+
v.size === 0 ? un : [...v.values()].map(({ call: t }) => t)
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
function $e(e, t) {
|
|
271
|
+
var o, r, s;
|
|
272
|
+
if (e === p.StreamVideoCreated) {
|
|
273
|
+
I = C.Talking, (o = n.onAgentActivityStateChange) == null || o.call(n, C.Talking), T == null || T.arm({
|
|
274
|
+
sttLatency: (r = t == null ? void 0 : t.stt) == null ? void 0 : r.latency,
|
|
275
|
+
serviceLatency: t == null ? void 0 : t.serviceLatency
|
|
276
|
+
});
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
if (v.size > 0) {
|
|
280
|
+
I !== C.ToolActive && (I = C.ToolActive, (s = n.onAgentActivityStateChange) == null || s.call(n, C.ToolActive));
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
function B(e, t) {
|
|
285
|
+
var l, u, f, m;
|
|
286
|
+
const o = ((u = (l = g == null ? void 0 : g.getReport()) == null ? void 0 : l.webRTCStats) == null ? void 0 : u.avgRtt) ?? 0, r = o > 0 ? Math.round(o / 2 * 1e3) : 0, s = { ...t, downstreamNetworkLatency: r };
|
|
287
|
+
d.debug && ((f = t == null ? void 0 : t.metadata) != null && f.sentiment) && (s.sentiment = {
|
|
288
|
+
id: t.metadata.sentiment.id,
|
|
289
|
+
name: t.metadata.sentiment.sentiment
|
|
290
|
+
}), (m = n.onMessage) == null || m.call(n, e, s), $e(e, t);
|
|
291
|
+
}
|
|
292
|
+
function De(e, t) {
|
|
293
|
+
var o;
|
|
294
|
+
(o = n.onMessage) == null || o.call(n, J.Transcribe, { event: J.Transcribe, ...t }), queueMicrotask(() => {
|
|
295
|
+
var r;
|
|
296
|
+
(r = n.onAgentActivityStateChange) == null || r.call(n, C.Loading);
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
function Ke(e, t) {
|
|
300
|
+
var o;
|
|
301
|
+
K = (t == null ? void 0 : t.turn_id) ?? null, I = C.Loading, (o = n.onAgentActivityStateChange) == null || o.call(n, C.Loading);
|
|
302
|
+
}
|
|
303
|
+
function Fe(e, t) {
|
|
304
|
+
var r;
|
|
305
|
+
const o = (t == null ? void 0 : t.turn_id) ?? null;
|
|
306
|
+
K !== null && o !== null && o < K || (xe(o), I = C.Idle, (r = n.onAgentActivityStateChange) == null || r.call(n, C.Idle));
|
|
307
|
+
}
|
|
308
|
+
function se(e, t) {
|
|
309
|
+
q = (t == null ? void 0 : t.reason) ?? null;
|
|
310
|
+
}
|
|
311
|
+
const Ve = {
|
|
312
|
+
[p.ChatAnswer]: ae,
|
|
313
|
+
[p.ChatPartial]: ae,
|
|
314
|
+
[p.ToolCallStarted]: G,
|
|
315
|
+
[p.ToolCallDone]: G,
|
|
316
|
+
[p.ToolCallError]: G,
|
|
317
|
+
[p.StreamVideoCreated]: B,
|
|
318
|
+
[p.StreamVideoDone]: B,
|
|
319
|
+
[p.StreamVideoError]: B,
|
|
320
|
+
[p.StreamVideoRejected]: B,
|
|
321
|
+
[p.ChatAudioTranscribed]: De,
|
|
322
|
+
[p.TurnStarted]: Ke,
|
|
323
|
+
[p.TurnEnded]: Fe,
|
|
324
|
+
[p.StreamDone]: se,
|
|
325
|
+
[p.StreamFailed]: se
|
|
326
|
+
};
|
|
327
|
+
function Ue(e, t, o, r) {
|
|
328
|
+
const s = new TextDecoder().decode(e);
|
|
329
|
+
let l;
|
|
330
|
+
try {
|
|
331
|
+
l = JSON.parse(s);
|
|
332
|
+
} catch (m) {
|
|
333
|
+
i("Failed to parse data channel message:", m);
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
const u = r || l.subject;
|
|
337
|
+
if (i("Data received:", { subject: u, data: l }), !u) return;
|
|
338
|
+
const f = Ve[u];
|
|
339
|
+
if (f)
|
|
340
|
+
try {
|
|
341
|
+
f(u, l);
|
|
342
|
+
} catch (m) {
|
|
343
|
+
console.warn("[LiveKitStreamingManager] Data channel handler failed", { subject: u, error: m });
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
function je(e) {
|
|
347
|
+
var t;
|
|
348
|
+
i("Media devices error:", e), (t = n.onError) == null || t.call(n, j(), { sessionId: y });
|
|
349
|
+
}
|
|
350
|
+
function Oe(e) {
|
|
351
|
+
var t;
|
|
352
|
+
i("Encryption error:", e), (t = n.onError) == null || t.call(n, j(), { sessionId: y });
|
|
353
|
+
}
|
|
354
|
+
function Ne(e, t, o) {
|
|
355
|
+
i("Track subscription failed:", { trackSid: e, participant: t, reason: o });
|
|
356
|
+
}
|
|
357
|
+
function ze(e, t, o) {
|
|
358
|
+
for (const [r, s] of o)
|
|
359
|
+
if (s.source === t && s.track) {
|
|
360
|
+
const l = s.track.mediaStreamTrack;
|
|
361
|
+
if (l === e || (l == null ? void 0 : l.id) === e.id)
|
|
362
|
+
return s;
|
|
363
|
+
}
|
|
364
|
+
return null;
|
|
365
|
+
}
|
|
366
|
+
async function ue(e, t, o, r, s, l) {
|
|
367
|
+
var F, V, $;
|
|
368
|
+
if (!A || !a)
|
|
369
|
+
throw i(`Room is not connected, cannot publish ${r} stream`), new Error("Room is not connected");
|
|
370
|
+
if (e.isPublishing) {
|
|
371
|
+
i(`${r} publish already in progress, skipping`);
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
const u = o(t);
|
|
375
|
+
if (u.length === 0)
|
|
376
|
+
throw new Error(`No ${r} track found in the provided MediaStream`);
|
|
377
|
+
const f = u[0], m = ze(f, r, s());
|
|
378
|
+
if (m) {
|
|
379
|
+
i(`${r} track is already published, skipping`, {
|
|
380
|
+
trackId: f.id,
|
|
381
|
+
publishedTrackId: (V = (F = m.track) == null ? void 0 : F.mediaStreamTrack) == null ? void 0 : V.id
|
|
382
|
+
}), e.publication = m;
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
if (($ = e.publication) != null && $.track) {
|
|
386
|
+
const E = e.publication.track.mediaStreamTrack;
|
|
387
|
+
E !== f && (E == null ? void 0 : E.id) !== f.id && (i(`Unpublishing existing ${r} track before publishing new one`), await l());
|
|
388
|
+
}
|
|
389
|
+
i(`Publishing ${r} track from provided MediaStream`, { trackId: f.id }), e.isPublishing = !0;
|
|
390
|
+
try {
|
|
391
|
+
e.publication = await a.localParticipant.publishTrack(f, { source: r }), i(`${r} track published successfully`, { trackSid: e.publication.trackSid });
|
|
392
|
+
} catch (E) {
|
|
393
|
+
throw i(`Failed to publish ${r} track:`, E), E;
|
|
394
|
+
} finally {
|
|
395
|
+
e.isPublishing = !1;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
async function de(e, t) {
|
|
399
|
+
if (!(!e.publication || !e.publication.track))
|
|
400
|
+
try {
|
|
401
|
+
a && (await a.localParticipant.unpublishTrack(e.publication.track, !1), i(`${t} track unpublished`));
|
|
402
|
+
} catch (o) {
|
|
403
|
+
i(`Error unpublishing ${t} track:`, o);
|
|
404
|
+
} finally {
|
|
405
|
+
e.publication = null;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
async function Be(e) {
|
|
409
|
+
return ue(
|
|
410
|
+
P,
|
|
411
|
+
e,
|
|
412
|
+
(t) => t.getAudioTracks(),
|
|
413
|
+
ne.Source.Microphone,
|
|
414
|
+
() => a.localParticipant.audioTrackPublications,
|
|
415
|
+
H
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
async function H() {
|
|
419
|
+
return de(P, "Microphone");
|
|
420
|
+
}
|
|
421
|
+
async function Je(e) {
|
|
422
|
+
if (!A || !a)
|
|
423
|
+
throw i("Cannot replace microphone track: room is not connected"), new Error("Room is not connected");
|
|
424
|
+
if (e.kind !== "audio")
|
|
425
|
+
throw i("Cannot replace microphone track: not an audio track", { kind: e.kind }), new Error("Microphone track must be an audio track");
|
|
426
|
+
if (P.isPublishing)
|
|
427
|
+
throw i("Cannot replace microphone track: publish in progress"), new Error("Microphone publish in progress");
|
|
428
|
+
const t = P.publication;
|
|
429
|
+
if (!t || !t.track)
|
|
430
|
+
throw i("Cannot replace microphone track: no publication to replace"), new Error("No microphone publication to replace");
|
|
431
|
+
try {
|
|
432
|
+
P.isPublishing = !0, await t.track.replaceTrack(e), i("Microphone track replaced", { trackId: e.id, trackSid: t.trackSid });
|
|
433
|
+
} finally {
|
|
434
|
+
P.isPublishing = !1;
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
async function Qe(e) {
|
|
438
|
+
return ue(
|
|
439
|
+
Q,
|
|
440
|
+
e,
|
|
441
|
+
(t) => t.getVideoTracks(),
|
|
442
|
+
ne.Source.Camera,
|
|
443
|
+
() => a.localParticipant.videoTrackPublications,
|
|
444
|
+
X
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
async function X() {
|
|
448
|
+
return de(Q, "Camera");
|
|
449
|
+
}
|
|
450
|
+
function We() {
|
|
451
|
+
L && (L.getTracks().forEach((e) => e.stop()), L = null);
|
|
452
|
+
}
|
|
453
|
+
async function Z(e, t) {
|
|
454
|
+
var o, r;
|
|
455
|
+
if (!A || !a) {
|
|
456
|
+
i("Room is not connected for sending messages"), (o = n.onError) == null || o.call(n, j(), {
|
|
457
|
+
sessionId: y
|
|
458
|
+
});
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
try {
|
|
462
|
+
await a.localParticipant.sendText(t, { topic: e }), i("Message sent successfully:", t);
|
|
463
|
+
} catch (s) {
|
|
464
|
+
i("Failed to send message:", s), (r = n.onError) == null || r.call(n, j(), { sessionId: y });
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
async function Y(e) {
|
|
468
|
+
var t, o;
|
|
469
|
+
b && (clearTimeout(b), b = null), T == null || T.destroy(), T = null, a && ((t = n.onConnectionStateChange) == null || t.call(n, w.Disconnecting, e), await Promise.all([H(), X()]), await a.disconnect()), We(), A = !1, D = !1, v.size > 0 && (v.clear(), N(), z()), K = null, (o = n.onAgentActivityStateChange) == null || o.call(n, C.Idle), I = C.Idle;
|
|
470
|
+
}
|
|
471
|
+
return {
|
|
472
|
+
speak(e) {
|
|
473
|
+
const t = typeof e == "string" ? e : JSON.stringify(e);
|
|
474
|
+
return Z(Ce.Speak, t);
|
|
475
|
+
},
|
|
476
|
+
disconnect: () => Y("user:disconnect"),
|
|
477
|
+
async reconnect() {
|
|
478
|
+
var e, t;
|
|
479
|
+
if ((a == null ? void 0 : a.state) === k.Connected) {
|
|
480
|
+
i("Room is already connected");
|
|
481
|
+
return;
|
|
482
|
+
}
|
|
483
|
+
if (!a || !x || !_)
|
|
484
|
+
throw i("Cannot reconnect: missing room, URL or token"), new Error("Cannot reconnect: session not available");
|
|
485
|
+
i("Reconnecting to LiveKit room, state:", a.state), D = !1, q = null, (e = n.onConnectionStateChange) == null || e.call(n, w.Connecting, "user:reconnect");
|
|
486
|
+
try {
|
|
487
|
+
if (await a.connect(x, _), i("Room reconnected"), A = !0, a.remoteParticipants.size === 0) {
|
|
488
|
+
if (i("Waiting for agent to join..."), !await new Promise((r) => {
|
|
489
|
+
const s = setTimeout(() => {
|
|
490
|
+
a == null || a.off(h.ParticipantConnected, l), r(!1);
|
|
491
|
+
}, 5e3), l = () => {
|
|
492
|
+
clearTimeout(s), a == null || a.off(h.ParticipantConnected, l), r(!0);
|
|
493
|
+
};
|
|
494
|
+
a == null || a.on(h.ParticipantConnected, l);
|
|
495
|
+
}))
|
|
496
|
+
throw i("Agent did not join within timeout"), await a.disconnect(), new Error("Agent did not rejoin the room");
|
|
497
|
+
i("Agent joined, reconnection successful");
|
|
498
|
+
}
|
|
499
|
+
} catch (o) {
|
|
500
|
+
throw i("Failed to reconnect:", o), (t = n.onConnectionStateChange) == null || t.call(n, w.Fail, "user:reconnect-failed"), o;
|
|
501
|
+
}
|
|
502
|
+
},
|
|
503
|
+
sendDataChannelMessage: Z,
|
|
504
|
+
publishMicrophoneStream: Be,
|
|
505
|
+
unpublishMicrophoneStream: H,
|
|
506
|
+
replaceMicrophoneTrack: Je,
|
|
507
|
+
publishCameraStream: Qe,
|
|
508
|
+
unpublishCameraStream: X,
|
|
509
|
+
interrupt(e) {
|
|
510
|
+
return e === "text" || !A || !a ? !1 : (Z(Ce.Interrupt, ""), !0);
|
|
511
|
+
},
|
|
512
|
+
registerRpcMethod(e, t) {
|
|
513
|
+
a == null || a.registerRpcMethod(e, t);
|
|
514
|
+
},
|
|
515
|
+
unregisterRpcMethod(e) {
|
|
516
|
+
a == null || a.unregisterRpcMethod(e);
|
|
517
|
+
},
|
|
518
|
+
sessionId: y,
|
|
519
|
+
streamId: y,
|
|
520
|
+
streamType: ie,
|
|
521
|
+
interruptAvailable: oe,
|
|
522
|
+
// A getter, not a snapshot: currentInterruptible changes as blocking tool calls come and go.
|
|
523
|
+
get isInterruptible() {
|
|
524
|
+
return W;
|
|
525
|
+
}
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
export {
|
|
529
|
+
hn as createLiveKitStreamingManager,
|
|
530
|
+
ve as handleInitError
|
|
531
|
+
};
|
package/dist/src/api/agents.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Agent, Auth, Chat, ChatPayload, ChatResponse,
|
|
1
|
+
import { Agent, Auth, Chat, ChatPayload, ChatResponse, Rating, RatingPayload, SttTokenResponse, SubmitFeedbackResponse } from '../types/index';
|
|
2
2
|
import { RequestOptions } from './apiClient';
|
|
3
3
|
|
|
4
4
|
export declare function createAgentsApi(auth: Auth, host?: string, onError?: (error: Error, errorData: Record<string, unknown>) => void, externalId?: string): {
|
|
@@ -7,12 +7,12 @@ export declare function createAgentsApi(auth: Auth, host?: string, onError?: (er
|
|
|
7
7
|
persist: boolean;
|
|
8
8
|
}, options?: RequestOptions): Promise<Chat>;
|
|
9
9
|
chat(agentId: string, chatId: string, payload: ChatPayload, options?: RequestOptions): Promise<ChatResponse>;
|
|
10
|
-
createRating(agentId: string, chatId: string, payload: RatingPayload, options?: RequestOptions): Promise<
|
|
11
|
-
updateRating(agentId: string, chatId: string, ratingId: string, payload: Partial<RatingPayload>, options?: RequestOptions): Promise<
|
|
12
|
-
deleteRating(agentId: string, chatId: string, ratingId: string, options?: RequestOptions): Promise<
|
|
10
|
+
createRating(agentId: string, chatId: string, payload: RatingPayload, options?: RequestOptions): Promise<Rating>;
|
|
11
|
+
updateRating(agentId: string, chatId: string, ratingId: string, payload: Partial<RatingPayload>, options?: RequestOptions): Promise<Rating>;
|
|
12
|
+
deleteRating(agentId: string, chatId: string, ratingId: string, options?: RequestOptions): Promise<Rating>;
|
|
13
13
|
submitFeedback(agentId: string, chatId: string, payload: {
|
|
14
14
|
rating: number;
|
|
15
15
|
answer?: string;
|
|
16
16
|
}, options?: RequestOptions): Promise<SubmitFeedbackResponse>;
|
|
17
|
-
|
|
17
|
+
getSttToken(agentId: string, options?: RequestOptions): Promise<SttTokenResponse>;
|
|
18
18
|
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/src/index.d.ts
CHANGED
|
@@ -3,18 +3,18 @@ export type { Auth, BasicAuth, BearerToken, ClientKeyAuth } from './types/auth';
|
|
|
3
3
|
export type { Agent, AgentAvatar, EndOfCallFeedbackConfig } from './types/entities/agents/agent';
|
|
4
4
|
export type { AgentManager, AgentManagerCallbacks, AgentManagerOptions, StreamOptions, } from './types/entities/agents/manager';
|
|
5
5
|
export { AvatarType } from './types/entities/avatar';
|
|
6
|
-
export type {
|
|
7
|
-
export { AgentActivityState, ConnectionState, ConnectivityState,
|
|
6
|
+
export type { SttTokenResponse } from './types/voice/stt';
|
|
7
|
+
export { AgentActivityState, ConnectionState, ConnectivityState, DataChannelTopic, StreamEndReason, StreamingState, ToolCallEvent, } from './types/stream/stream';
|
|
8
8
|
export type { ClientToolHandler, RunningToolCall, StreamCreatedInfo, ToolCallDonePayload, ToolCallErrorPayload, ToolCallStartedPayload, ToolEventCallback, ToolExecutionMode, } from './types/stream/stream';
|
|
9
|
-
export type {
|
|
9
|
+
export type { SpeakResponse } from './types/stream/rtc';
|
|
10
10
|
export { StreamType } from './types/stream/stream';
|
|
11
11
|
export type { CompatibilityMode } from './types/stream/stream';
|
|
12
|
-
export type { AudioStreamScript,
|
|
12
|
+
export type { AudioStreamScript, SpeakScript, TextStreamScript } from './types/stream-script';
|
|
13
13
|
export { ChatMode } from './types/entities/agents/chat';
|
|
14
|
-
export type { ChatResponse,
|
|
14
|
+
export type { ChatResponse, InterruptOptions, Message, MessagePart, MessageSentiment, Rating, RetrievalMetadata, SubmitFeedbackResponse, } from './types/entities/agents/chat';
|
|
15
15
|
export { parseMessageParts } from './utils/content-parser';
|
|
16
16
|
export { isAwaitingTool } from './utils/tool-calls';
|
|
17
17
|
export { Providers, VoiceAccess } from './types/voice/tts';
|
|
18
|
-
export type { AmazonTtsProvider, AzureOpenAiTtsProvider, ElevenlabsTtsProvider, MicrosoftTtsProvider,
|
|
18
|
+
export type { AmazonTtsProvider, AzureOpenAiTtsProvider, ElevenlabsTtsProvider, MicrosoftTtsProvider, TtsProvider, Voice, VoiceConfigElevenlabs, VoiceConfigMicrosoft, } from './types/voice/tts';
|
|
19
19
|
export { BaseError, ChatCreationFailed, ChatModeDowngraded, HttpError, NetworkError, StreamError, ValidationError, WsError, isDIDError, } from './errors';
|
|
20
20
|
export type { ErrorJson } from './errors';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { CreateSessionV2Options, CreateStreamOptions,
|
|
2
|
-
import {
|
|
1
|
+
import { CreateSessionV2Options, CreateStreamOptions, InterruptOptions, PayloadType, StreamType } from '../../types';
|
|
2
|
+
import { InternalDataChannelTopic } from '../../types/stream/data-channel';
|
|
3
3
|
|
|
4
4
|
export declare const createStreamingLogger: (debug: boolean, prefix: string) => (message: string, extra?: any) => false | void;
|
|
5
5
|
/**
|
|
@@ -23,7 +23,7 @@ export type StreamingManager<T extends CreateStreamOptions | CreateSessionV2Opti
|
|
|
23
23
|
* @param topic Data-channel topic to send on
|
|
24
24
|
* @param payload The message payload to send, already serialized
|
|
25
25
|
*/
|
|
26
|
-
sendDataChannelMessage(topic:
|
|
26
|
+
sendDataChannelMessage(topic: InternalDataChannelTopic, payload: string): Promise<void>;
|
|
27
27
|
/**
|
|
28
28
|
* Publish a microphone stream to the DataChannel
|
|
29
29
|
* Can be called after connection to add microphone input
|
|
@@ -86,7 +86,7 @@ export type StreamingManager<T extends CreateStreamOptions | CreateSessionV2Opti
|
|
|
86
86
|
* interrupt type is one this transport drops), and `false` when the transport could not carry
|
|
87
87
|
* it because the connection or the data channel is not up.
|
|
88
88
|
*/
|
|
89
|
-
interrupt(type:
|
|
89
|
+
interrupt(type: InterruptOptions['type']): boolean;
|
|
90
90
|
/**
|
|
91
91
|
* Register an RPC method handler on the LiveKit room.
|
|
92
92
|
* Used internally by the agent-manager for client tool delegation.
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
* A rating stored against one message of a chat, as the Agents API returns it.
|
|
3
3
|
*
|
|
4
4
|
* Produced by {@link AgentManager.rate | rate()} and returned again by
|
|
5
|
-
* {@link AgentManager.deleteRate | deleteRate()}. Keep {@link
|
|
5
|
+
* {@link AgentManager.deleteRate | deleteRate()}. Keep {@link Rating.id | id} if the user may
|
|
6
6
|
* change their mind: pass it back to {@link AgentManager.rate | rate()} as `rateId` to update the
|
|
7
7
|
* rating, or to {@link AgentManager.deleteRate | deleteRate()} to remove it.
|
|
8
8
|
*
|
|
9
9
|
* @category Chat
|
|
10
10
|
*/
|
|
11
|
-
export interface
|
|
11
|
+
export interface Rating {
|
|
12
12
|
/**
|
|
13
13
|
* Id of this rating.
|
|
14
14
|
*
|
|
@@ -56,10 +56,10 @@ export interface RatingEntity {
|
|
|
56
56
|
message_id: string;
|
|
57
57
|
}
|
|
58
58
|
/**
|
|
59
|
-
* Request payload for rating a chat message, derived from `
|
|
59
|
+
* Request payload for rating a chat message, derived from `Rating`.
|
|
60
60
|
* @internal Implementation type; not part of the public SDK surface.
|
|
61
61
|
*/
|
|
62
|
-
export type RatingPayload = Omit<
|
|
62
|
+
export type RatingPayload = Omit<Rating, 'owner_id' | 'id' | 'created_at' | 'modified_at' | 'created_by' | 'external_id' | 'agent_id' | 'chat_id'>;
|
|
63
63
|
/**
|
|
64
64
|
* What the Agents API stored when end-of-call feedback was submitted.
|
|
65
65
|
*
|
|
@@ -188,7 +188,7 @@ export interface Message {
|
|
|
188
188
|
*/
|
|
189
189
|
parts: MessagePart[];
|
|
190
190
|
/** When the message was added, as an ISO 8601 timestamp. */
|
|
191
|
-
|
|
191
|
+
createdAt?: string;
|
|
192
192
|
/**
|
|
193
193
|
* The knowledge citations the answer was drawn from, as {@link RetrievalMetadata} entries.
|
|
194
194
|
*
|
|
@@ -267,14 +267,14 @@ export interface ChatPayload {
|
|
|
267
267
|
*/
|
|
268
268
|
export interface RetrievalMetadata {
|
|
269
269
|
/** Id of this match. Sent back with a rating as the second half of a
|
|
270
|
-
* {@link
|
|
270
|
+
* {@link Rating.matches} pair. */
|
|
271
271
|
id: string;
|
|
272
272
|
/** The matched passage itself, as stored in the knowledge base. */
|
|
273
273
|
data: string;
|
|
274
274
|
/** Title of the document the passage comes from. */
|
|
275
275
|
title: string;
|
|
276
276
|
/** Id of the source document. Sent back with a rating as the first half of a
|
|
277
|
-
* {@link
|
|
277
|
+
* {@link Rating.matches} pair. */
|
|
278
278
|
document_id: string;
|
|
279
279
|
/** Id of the knowledge base the document belongs to — the agent's {@link Agent.knowledge}. */
|
|
280
280
|
knowledge_id: string;
|
|
@@ -311,7 +311,7 @@ export declare enum ChatMode {
|
|
|
311
311
|
* {@link AgentManager.chat | chat()} still returns answers through
|
|
312
312
|
* {@link AgentManagerCallbacks.onNewMessage | onNewMessage}, and
|
|
313
313
|
* {@link AgentManager.speak | speak()} adds a text script to the transcript but streams no
|
|
314
|
-
* video: it resolves with a `duration` of `0` and an empty `
|
|
314
|
+
* video: it resolves with a `duration` of `0` and an empty `videoId`.
|
|
315
315
|
*/
|
|
316
316
|
TextOnly = "TextOnly",
|
|
317
317
|
/**
|
|
@@ -461,7 +461,7 @@ export interface Chat {
|
|
|
461
461
|
*
|
|
462
462
|
* @category Chat
|
|
463
463
|
*/
|
|
464
|
-
export interface
|
|
464
|
+
export interface InterruptOptions {
|
|
465
465
|
/**
|
|
466
466
|
* The cause: `text` when the user typed over the answer, `audio` when they started speaking,
|
|
467
467
|
* `click` when they pressed a stop control, and `manual` for an interruption the application
|