@decartai/sdk 0.0.68 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -9
- package/dist/index.d.ts +6 -4
- package/dist/index.js +43 -28
- package/dist/process/client.js +1 -3
- package/dist/process/request.js +1 -3
- package/dist/queue/client.js +1 -3
- package/dist/queue/polling.js +1 -2
- package/dist/queue/request.js +1 -3
- package/dist/realtime/client.d.ts +17 -11
- package/dist/realtime/client.js +71 -155
- package/dist/realtime/config-realtime.js +49 -0
- package/dist/realtime/event-buffer.js +1 -3
- package/dist/realtime/initial-state-gate.js +21 -0
- package/dist/realtime/media-channel.js +82 -0
- package/dist/realtime/methods.js +12 -42
- package/dist/realtime/mirror-stream.js +1 -2
- package/dist/realtime/observability/diagnostics.d.ts +14 -53
- package/dist/realtime/observability/livekit-stats-provider.js +25 -0
- package/dist/realtime/observability/realtime-observability.js +70 -6
- package/dist/realtime/observability/telemetry-reporter.js +9 -28
- package/dist/realtime/observability/webrtc-stats.d.ts +5 -4
- package/dist/realtime/observability/webrtc-stats.js +3 -5
- package/dist/realtime/signaling-channel.js +286 -0
- package/dist/realtime/stream-session.js +252 -0
- package/dist/realtime/subscribe-client.d.ts +2 -3
- package/dist/realtime/subscribe-client.js +115 -11
- package/dist/realtime/types.d.ts +25 -1
- package/dist/shared/model.d.ts +11 -1
- package/dist/shared/model.js +51 -14
- package/dist/shared/request.js +1 -3
- package/dist/shared/types.js +1 -3
- package/dist/tokens/client.js +1 -3
- package/dist/utils/env.js +1 -2
- package/dist/utils/errors.js +1 -2
- package/dist/utils/logger.js +1 -2
- package/dist/utils/media.js +43 -0
- package/dist/utils/platform.js +13 -0
- package/dist/utils/user-agent.js +1 -3
- package/dist/version.js +1 -2
- package/package.json +2 -1
- package/dist/realtime/webrtc-connection.js +0 -500
- package/dist/realtime/webrtc-manager.js +0 -210
|
@@ -1,500 +0,0 @@
|
|
|
1
|
-
import { buildUserAgent } from "../utils/user-agent.js";
|
|
2
|
-
import mitt from "mitt";
|
|
3
|
-
|
|
4
|
-
//#region src/realtime/webrtc-connection.ts
|
|
5
|
-
const ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }];
|
|
6
|
-
const SETUP_TIMEOUT_MS = 3e4;
|
|
7
|
-
var WebRTCConnection = class {
|
|
8
|
-
pc = null;
|
|
9
|
-
ws = null;
|
|
10
|
-
localStream = null;
|
|
11
|
-
connectionReject = null;
|
|
12
|
-
logger;
|
|
13
|
-
observability;
|
|
14
|
-
state = "disconnected";
|
|
15
|
-
websocketMessagesEmitter = mitt();
|
|
16
|
-
constructor(callbacks) {
|
|
17
|
-
this.callbacks = callbacks;
|
|
18
|
-
this.logger = callbacks.logger ?? {
|
|
19
|
-
debug() {},
|
|
20
|
-
info() {},
|
|
21
|
-
warn() {},
|
|
22
|
-
error() {}
|
|
23
|
-
};
|
|
24
|
-
this.observability = callbacks.observability;
|
|
25
|
-
}
|
|
26
|
-
getPeerConnection() {
|
|
27
|
-
return this.pc;
|
|
28
|
-
}
|
|
29
|
-
async connect(url, localStream, timeout, integration) {
|
|
30
|
-
const deadline = Date.now() + timeout;
|
|
31
|
-
this.localStream = localStream;
|
|
32
|
-
const userAgent = encodeURIComponent(buildUserAgent(integration));
|
|
33
|
-
const wsUrl = `${url}${url.includes("?") ? "&" : "?"}user_agent=${userAgent}`;
|
|
34
|
-
let rejectConnect;
|
|
35
|
-
const connectAbort = new Promise((_, reject) => {
|
|
36
|
-
rejectConnect = reject;
|
|
37
|
-
});
|
|
38
|
-
connectAbort.catch(() => {});
|
|
39
|
-
this.connectionReject = (error) => rejectConnect(error);
|
|
40
|
-
const totalStart = performance.now();
|
|
41
|
-
try {
|
|
42
|
-
const wsStart = performance.now();
|
|
43
|
-
await Promise.race([new Promise((resolve, reject) => {
|
|
44
|
-
const timer = setTimeout(() => reject(/* @__PURE__ */ new Error("WebSocket timeout")), timeout);
|
|
45
|
-
this.ws = new WebSocket(wsUrl);
|
|
46
|
-
this.ws.onopen = () => {
|
|
47
|
-
clearTimeout(timer);
|
|
48
|
-
this.observability.diagnostic("phaseTiming", {
|
|
49
|
-
phase: "websocket",
|
|
50
|
-
durationMs: performance.now() - wsStart,
|
|
51
|
-
success: true
|
|
52
|
-
});
|
|
53
|
-
resolve();
|
|
54
|
-
};
|
|
55
|
-
this.ws.onmessage = (e) => {
|
|
56
|
-
try {
|
|
57
|
-
this.handleSignalingMessage(JSON.parse(e.data));
|
|
58
|
-
} catch (err) {
|
|
59
|
-
this.logger.error("Signaling message parse error", { error: String(err) });
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
this.ws.onerror = () => {
|
|
63
|
-
clearTimeout(timer);
|
|
64
|
-
const error = /* @__PURE__ */ new Error("WebSocket error");
|
|
65
|
-
this.observability.diagnostic("phaseTiming", {
|
|
66
|
-
phase: "websocket",
|
|
67
|
-
durationMs: performance.now() - wsStart,
|
|
68
|
-
success: false,
|
|
69
|
-
error: error.message
|
|
70
|
-
});
|
|
71
|
-
reject(error);
|
|
72
|
-
rejectConnect(error);
|
|
73
|
-
};
|
|
74
|
-
this.ws.onclose = () => {
|
|
75
|
-
this.setState("disconnected");
|
|
76
|
-
clearTimeout(timer);
|
|
77
|
-
reject(/* @__PURE__ */ new Error("WebSocket closed before connection was established"));
|
|
78
|
-
rejectConnect(/* @__PURE__ */ new Error("WebSocket closed"));
|
|
79
|
-
};
|
|
80
|
-
}), connectAbort]);
|
|
81
|
-
if (this.callbacks.initialImage) {
|
|
82
|
-
const imageStart = performance.now();
|
|
83
|
-
await Promise.race([this.setImageBase64(this.callbacks.initialImage, {
|
|
84
|
-
prompt: this.callbacks.initialPrompt?.text,
|
|
85
|
-
enhance: this.callbacks.initialPrompt?.enhance
|
|
86
|
-
}), connectAbort]);
|
|
87
|
-
this.observability.diagnostic("phaseTiming", {
|
|
88
|
-
phase: "avatar-image",
|
|
89
|
-
durationMs: performance.now() - imageStart,
|
|
90
|
-
success: true
|
|
91
|
-
});
|
|
92
|
-
} else if (this.callbacks.initialPrompt) {
|
|
93
|
-
const promptStart = performance.now();
|
|
94
|
-
await Promise.race([this.sendInitialPrompt(this.callbacks.initialPrompt), connectAbort]);
|
|
95
|
-
this.observability.diagnostic("phaseTiming", {
|
|
96
|
-
phase: "initial-prompt",
|
|
97
|
-
durationMs: performance.now() - promptStart,
|
|
98
|
-
success: true
|
|
99
|
-
});
|
|
100
|
-
} else if (localStream) {
|
|
101
|
-
const nullStart = performance.now();
|
|
102
|
-
await Promise.race([this.setImageBase64(null, { prompt: null }), connectAbort]);
|
|
103
|
-
this.observability.diagnostic("phaseTiming", {
|
|
104
|
-
phase: "initial-prompt",
|
|
105
|
-
durationMs: performance.now() - nullStart,
|
|
106
|
-
success: true
|
|
107
|
-
});
|
|
108
|
-
}
|
|
109
|
-
const handshakeStart = performance.now();
|
|
110
|
-
await this.setupNewPeerConnection();
|
|
111
|
-
await Promise.race([new Promise((resolve, reject) => {
|
|
112
|
-
const checkConnection = setInterval(() => {
|
|
113
|
-
if (this.state === "connected" || this.state === "generating") {
|
|
114
|
-
clearInterval(checkConnection);
|
|
115
|
-
this.observability.diagnostic("phaseTiming", {
|
|
116
|
-
phase: "webrtc-handshake",
|
|
117
|
-
durationMs: performance.now() - handshakeStart,
|
|
118
|
-
success: true
|
|
119
|
-
});
|
|
120
|
-
resolve();
|
|
121
|
-
} else if (this.state === "disconnected") {
|
|
122
|
-
clearInterval(checkConnection);
|
|
123
|
-
this.observability.diagnostic("phaseTiming", {
|
|
124
|
-
phase: "webrtc-handshake",
|
|
125
|
-
durationMs: performance.now() - handshakeStart,
|
|
126
|
-
success: false,
|
|
127
|
-
error: "Connection lost during handshake"
|
|
128
|
-
});
|
|
129
|
-
reject(/* @__PURE__ */ new Error("Connection lost during WebRTC handshake"));
|
|
130
|
-
} else if (Date.now() >= deadline) {
|
|
131
|
-
clearInterval(checkConnection);
|
|
132
|
-
this.observability.diagnostic("phaseTiming", {
|
|
133
|
-
phase: "webrtc-handshake",
|
|
134
|
-
durationMs: performance.now() - handshakeStart,
|
|
135
|
-
success: false,
|
|
136
|
-
error: "Timeout"
|
|
137
|
-
});
|
|
138
|
-
reject(/* @__PURE__ */ new Error("Connection timeout"));
|
|
139
|
-
}
|
|
140
|
-
}, 100);
|
|
141
|
-
connectAbort.catch(() => clearInterval(checkConnection));
|
|
142
|
-
}), connectAbort]);
|
|
143
|
-
this.observability.diagnostic("phaseTiming", {
|
|
144
|
-
phase: "total",
|
|
145
|
-
durationMs: performance.now() - totalStart,
|
|
146
|
-
success: true
|
|
147
|
-
});
|
|
148
|
-
} finally {
|
|
149
|
-
this.connectionReject = null;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
async handleSignalingMessage(msg) {
|
|
153
|
-
try {
|
|
154
|
-
if (msg.type === "error") {
|
|
155
|
-
const error = new Error(msg.error);
|
|
156
|
-
error.source = "server";
|
|
157
|
-
this.callbacks.onError?.(error);
|
|
158
|
-
if (this.connectionReject) {
|
|
159
|
-
this.connectionReject(error);
|
|
160
|
-
this.connectionReject = null;
|
|
161
|
-
}
|
|
162
|
-
return;
|
|
163
|
-
}
|
|
164
|
-
if (msg.type === "set_image_ack") {
|
|
165
|
-
this.websocketMessagesEmitter.emit("setImageAck", msg);
|
|
166
|
-
return;
|
|
167
|
-
}
|
|
168
|
-
if (msg.type === "prompt_ack") {
|
|
169
|
-
this.websocketMessagesEmitter.emit("promptAck", msg);
|
|
170
|
-
return;
|
|
171
|
-
}
|
|
172
|
-
if (msg.type === "generation_started") {
|
|
173
|
-
this.setState("generating");
|
|
174
|
-
return;
|
|
175
|
-
}
|
|
176
|
-
if (msg.type === "generation_tick") {
|
|
177
|
-
this.websocketMessagesEmitter.emit("generationTick", msg);
|
|
178
|
-
return;
|
|
179
|
-
}
|
|
180
|
-
if (msg.type === "generation_ended") return;
|
|
181
|
-
if (msg.type === "session_id") {
|
|
182
|
-
this.websocketMessagesEmitter.emit("sessionId", msg);
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
if (!this.pc) return;
|
|
186
|
-
switch (msg.type) {
|
|
187
|
-
case "ready": {
|
|
188
|
-
await this.applyCodecPreference("video/VP8");
|
|
189
|
-
const offer = await this.pc.createOffer();
|
|
190
|
-
this.modifyVP8Bitrate(offer);
|
|
191
|
-
await this.callbacks.customizeOffer?.(offer);
|
|
192
|
-
await this.pc.setLocalDescription(offer);
|
|
193
|
-
this.send({
|
|
194
|
-
type: "offer",
|
|
195
|
-
sdp: offer.sdp || ""
|
|
196
|
-
});
|
|
197
|
-
break;
|
|
198
|
-
}
|
|
199
|
-
case "offer": {
|
|
200
|
-
await this.pc.setRemoteDescription({
|
|
201
|
-
type: "offer",
|
|
202
|
-
sdp: msg.sdp
|
|
203
|
-
});
|
|
204
|
-
const answer = await this.pc.createAnswer();
|
|
205
|
-
await this.pc.setLocalDescription(answer);
|
|
206
|
-
this.send({
|
|
207
|
-
type: "answer",
|
|
208
|
-
sdp: answer.sdp || ""
|
|
209
|
-
});
|
|
210
|
-
break;
|
|
211
|
-
}
|
|
212
|
-
case "answer":
|
|
213
|
-
await this.pc.setRemoteDescription({
|
|
214
|
-
type: "answer",
|
|
215
|
-
sdp: msg.sdp
|
|
216
|
-
});
|
|
217
|
-
break;
|
|
218
|
-
case "ice-candidate":
|
|
219
|
-
if (msg.candidate) {
|
|
220
|
-
await this.pc.addIceCandidate(msg.candidate);
|
|
221
|
-
this.observability.diagnostic("iceCandidate", {
|
|
222
|
-
source: "remote",
|
|
223
|
-
candidateType: msg.candidate.candidate?.match(/typ (\w+)/)?.[1] ?? "unknown",
|
|
224
|
-
protocol: msg.candidate.candidate?.match(/udp|tcp/i)?.[0]?.toLowerCase() ?? "unknown"
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
|
-
break;
|
|
228
|
-
}
|
|
229
|
-
} catch (error) {
|
|
230
|
-
this.logger.error("Signaling handler error", { error: String(error) });
|
|
231
|
-
this.callbacks.onError?.(error);
|
|
232
|
-
this.connectionReject?.(error);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
send(message) {
|
|
236
|
-
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
237
|
-
this.ws.send(JSON.stringify(message));
|
|
238
|
-
return true;
|
|
239
|
-
}
|
|
240
|
-
this.logger.warn("Message dropped: WebSocket is not open");
|
|
241
|
-
return false;
|
|
242
|
-
}
|
|
243
|
-
async setImageBase64(imageBase64, options) {
|
|
244
|
-
return new Promise((resolve, reject) => {
|
|
245
|
-
const timeoutId = setTimeout(() => {
|
|
246
|
-
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
247
|
-
reject(/* @__PURE__ */ new Error("Image send timed out"));
|
|
248
|
-
}, options?.timeout ?? SETUP_TIMEOUT_MS);
|
|
249
|
-
const listener = (msg) => {
|
|
250
|
-
clearTimeout(timeoutId);
|
|
251
|
-
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
252
|
-
if (msg.success) resolve();
|
|
253
|
-
else reject(new Error(msg.error ?? "Failed to send image"));
|
|
254
|
-
};
|
|
255
|
-
this.websocketMessagesEmitter.on("setImageAck", listener);
|
|
256
|
-
const message = {
|
|
257
|
-
type: "set_image",
|
|
258
|
-
image_data: imageBase64
|
|
259
|
-
};
|
|
260
|
-
if (options?.prompt !== void 0) message.prompt = options.prompt;
|
|
261
|
-
if (options?.enhance !== void 0) message.enhance_prompt = options.enhance;
|
|
262
|
-
if (!this.send(message)) {
|
|
263
|
-
clearTimeout(timeoutId);
|
|
264
|
-
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
265
|
-
reject(/* @__PURE__ */ new Error("WebSocket is not open"));
|
|
266
|
-
}
|
|
267
|
-
});
|
|
268
|
-
}
|
|
269
|
-
/**
|
|
270
|
-
* Send the initial prompt to the server before WebRTC handshake.
|
|
271
|
-
*/
|
|
272
|
-
async sendInitialPrompt(prompt) {
|
|
273
|
-
return new Promise((resolve, reject) => {
|
|
274
|
-
const timeoutId = setTimeout(() => {
|
|
275
|
-
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
276
|
-
reject(/* @__PURE__ */ new Error("Prompt send timed out"));
|
|
277
|
-
}, SETUP_TIMEOUT_MS);
|
|
278
|
-
const listener = (msg) => {
|
|
279
|
-
if (msg.prompt === prompt.text) {
|
|
280
|
-
clearTimeout(timeoutId);
|
|
281
|
-
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
282
|
-
if (msg.success) resolve();
|
|
283
|
-
else reject(new Error(msg.error ?? "Failed to send prompt"));
|
|
284
|
-
}
|
|
285
|
-
};
|
|
286
|
-
this.websocketMessagesEmitter.on("promptAck", listener);
|
|
287
|
-
if (!this.send({
|
|
288
|
-
type: "prompt",
|
|
289
|
-
prompt: prompt.text,
|
|
290
|
-
enhance_prompt: prompt.enhance ?? true
|
|
291
|
-
})) {
|
|
292
|
-
clearTimeout(timeoutId);
|
|
293
|
-
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
294
|
-
reject(/* @__PURE__ */ new Error("WebSocket is not open"));
|
|
295
|
-
}
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
setState(state) {
|
|
299
|
-
if (this.state !== state) {
|
|
300
|
-
this.state = state;
|
|
301
|
-
this.callbacks.onStateChange?.(state);
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
async setupNewPeerConnection() {
|
|
305
|
-
if (this.pc) {
|
|
306
|
-
this.pc.getSenders().forEach((sender) => {
|
|
307
|
-
if (sender.track && this.pc) this.pc.removeTrack(sender);
|
|
308
|
-
});
|
|
309
|
-
this.pc.close();
|
|
310
|
-
}
|
|
311
|
-
this.pc = new RTCPeerConnection({ iceServers: ICE_SERVERS });
|
|
312
|
-
this.setState("connecting");
|
|
313
|
-
if (this.localStream) this.localStream.getTracks().forEach((track) => {
|
|
314
|
-
if (this.pc && this.localStream) this.pc.addTrack(track, this.localStream);
|
|
315
|
-
});
|
|
316
|
-
else {
|
|
317
|
-
this.pc.addTransceiver("video", { direction: "recvonly" });
|
|
318
|
-
this.pc.addTransceiver("audio", { direction: "recvonly" });
|
|
319
|
-
}
|
|
320
|
-
let fallbackStream = null;
|
|
321
|
-
this.pc.ontrack = (e) => {
|
|
322
|
-
if (e.streams?.[0]) this.callbacks.onRemoteStream?.(e.streams[0]);
|
|
323
|
-
else {
|
|
324
|
-
if (!fallbackStream) fallbackStream = new MediaStream();
|
|
325
|
-
fallbackStream.addTrack(e.track);
|
|
326
|
-
this.callbacks.onRemoteStream?.(fallbackStream);
|
|
327
|
-
}
|
|
328
|
-
};
|
|
329
|
-
this.pc.onicecandidate = (e) => {
|
|
330
|
-
this.send({
|
|
331
|
-
type: "ice-candidate",
|
|
332
|
-
candidate: e.candidate
|
|
333
|
-
});
|
|
334
|
-
if (e.candidate) this.observability.diagnostic("iceCandidate", {
|
|
335
|
-
source: "local",
|
|
336
|
-
candidateType: e.candidate.type ?? "unknown",
|
|
337
|
-
protocol: e.candidate.protocol ?? "unknown",
|
|
338
|
-
address: e.candidate.address ?? void 0,
|
|
339
|
-
port: e.candidate.port ?? void 0
|
|
340
|
-
});
|
|
341
|
-
};
|
|
342
|
-
let prevPcState = "new";
|
|
343
|
-
this.pc.onconnectionstatechange = () => {
|
|
344
|
-
if (!this.pc) return;
|
|
345
|
-
const s = this.pc.connectionState;
|
|
346
|
-
this.observability.diagnostic("peerConnectionStateChange", {
|
|
347
|
-
state: s,
|
|
348
|
-
previousState: prevPcState,
|
|
349
|
-
timestampMs: performance.now()
|
|
350
|
-
});
|
|
351
|
-
prevPcState = s;
|
|
352
|
-
if (s === "connected") this.emitSelectedCandidatePair();
|
|
353
|
-
const nextState = s === "connected" ? "connected" : ["connecting", "new"].includes(s) ? "connecting" : "disconnected";
|
|
354
|
-
if (this.state === "generating" && nextState !== "disconnected") return;
|
|
355
|
-
this.setState(nextState);
|
|
356
|
-
};
|
|
357
|
-
let prevIceState = "new";
|
|
358
|
-
this.pc.oniceconnectionstatechange = () => {
|
|
359
|
-
if (!this.pc) return;
|
|
360
|
-
const newIceState = this.pc.iceConnectionState;
|
|
361
|
-
this.observability.diagnostic("iceStateChange", {
|
|
362
|
-
state: newIceState,
|
|
363
|
-
previousState: prevIceState,
|
|
364
|
-
timestampMs: performance.now()
|
|
365
|
-
});
|
|
366
|
-
prevIceState = newIceState;
|
|
367
|
-
if (newIceState === "failed") {
|
|
368
|
-
this.setState("disconnected");
|
|
369
|
-
this.callbacks.onError?.(/* @__PURE__ */ new Error("ICE connection failed"));
|
|
370
|
-
}
|
|
371
|
-
};
|
|
372
|
-
let prevSignalingState = "stable";
|
|
373
|
-
this.pc.onsignalingstatechange = () => {
|
|
374
|
-
if (!this.pc) return;
|
|
375
|
-
const newState = this.pc.signalingState;
|
|
376
|
-
this.observability.diagnostic("signalingStateChange", {
|
|
377
|
-
state: newState,
|
|
378
|
-
previousState: prevSignalingState,
|
|
379
|
-
timestampMs: performance.now()
|
|
380
|
-
});
|
|
381
|
-
prevSignalingState = newState;
|
|
382
|
-
};
|
|
383
|
-
this.handleSignalingMessage({ type: "ready" });
|
|
384
|
-
}
|
|
385
|
-
async emitSelectedCandidatePair() {
|
|
386
|
-
if (!this.pc) return;
|
|
387
|
-
try {
|
|
388
|
-
const stats = await this.pc.getStats();
|
|
389
|
-
let found = false;
|
|
390
|
-
stats.forEach((report) => {
|
|
391
|
-
if (found) return;
|
|
392
|
-
if (report.type === "candidate-pair" && report.state === "succeeded") {
|
|
393
|
-
found = true;
|
|
394
|
-
let localCandidate;
|
|
395
|
-
let remoteCandidate;
|
|
396
|
-
stats.forEach((r) => {
|
|
397
|
-
if (r.id === report.localCandidateId) localCandidate = r;
|
|
398
|
-
if (r.id === report.remoteCandidateId) remoteCandidate = r;
|
|
399
|
-
});
|
|
400
|
-
if (localCandidate && remoteCandidate) this.observability.diagnostic("selectedCandidatePair", {
|
|
401
|
-
local: {
|
|
402
|
-
candidateType: String(localCandidate.candidateType ?? "unknown"),
|
|
403
|
-
protocol: String(localCandidate.protocol ?? "unknown"),
|
|
404
|
-
address: localCandidate.address,
|
|
405
|
-
port: localCandidate.port
|
|
406
|
-
},
|
|
407
|
-
remote: {
|
|
408
|
-
candidateType: String(remoteCandidate.candidateType ?? "unknown"),
|
|
409
|
-
protocol: String(remoteCandidate.protocol ?? "unknown"),
|
|
410
|
-
address: remoteCandidate.address,
|
|
411
|
-
port: remoteCandidate.port
|
|
412
|
-
}
|
|
413
|
-
});
|
|
414
|
-
}
|
|
415
|
-
});
|
|
416
|
-
} catch {}
|
|
417
|
-
}
|
|
418
|
-
cleanup() {
|
|
419
|
-
this.pc?.close();
|
|
420
|
-
this.pc = null;
|
|
421
|
-
this.ws?.close();
|
|
422
|
-
this.ws = null;
|
|
423
|
-
this.localStream = null;
|
|
424
|
-
this.setState("disconnected");
|
|
425
|
-
}
|
|
426
|
-
applyCodecPreference(preferredCodecName) {
|
|
427
|
-
if (!this.pc) return;
|
|
428
|
-
if (typeof RTCRtpSender === "undefined" || typeof RTCRtpSender.getCapabilities !== "function") {
|
|
429
|
-
this.logger.debug("RTCRtpSender capabilities not available in this environment");
|
|
430
|
-
return;
|
|
431
|
-
}
|
|
432
|
-
const videoTransceiver = this.pc.getTransceivers().find((r) => r.sender.track?.kind === "video" || r.receiver.track?.kind === "video");
|
|
433
|
-
if (!videoTransceiver) {
|
|
434
|
-
this.logger.warn("Video transceiver not found for codec preference");
|
|
435
|
-
return;
|
|
436
|
-
}
|
|
437
|
-
const capabilities = RTCRtpSender.getCapabilities("video");
|
|
438
|
-
if (!capabilities) {
|
|
439
|
-
this.logger.warn("Video sender capabilities unavailable");
|
|
440
|
-
return;
|
|
441
|
-
}
|
|
442
|
-
const preferredCodecs = [];
|
|
443
|
-
const otherCodecs = [];
|
|
444
|
-
capabilities.codecs.forEach((codec) => {
|
|
445
|
-
if (codec.mimeType.toLowerCase() === preferredCodecName.toLowerCase()) preferredCodecs.push(codec);
|
|
446
|
-
else otherCodecs.push(codec);
|
|
447
|
-
});
|
|
448
|
-
const orderedCodecs = [...preferredCodecs, ...otherCodecs];
|
|
449
|
-
if (orderedCodecs.length === 0) {
|
|
450
|
-
this.logger.debug("No video codecs found for preference setting");
|
|
451
|
-
return;
|
|
452
|
-
}
|
|
453
|
-
try {
|
|
454
|
-
videoTransceiver.setCodecPreferences(orderedCodecs);
|
|
455
|
-
} catch {
|
|
456
|
-
this.logger.debug("setCodecPreferences not supported, skipping");
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
|
-
modifyVP8Bitrate(offer) {
|
|
460
|
-
if (!offer.sdp) return;
|
|
461
|
-
const minBitrateInKbps = this.callbacks.vp8MinBitrate;
|
|
462
|
-
const startBitrateInKbps = this.callbacks.vp8StartBitrate;
|
|
463
|
-
if (minBitrateInKbps === void 0 || startBitrateInKbps === void 0) return;
|
|
464
|
-
if (minBitrateInKbps === 0 && startBitrateInKbps === 0) return;
|
|
465
|
-
const bitrateParams = `x-google-min-bitrate=${minBitrateInKbps};x-google-start-bitrate=${startBitrateInKbps}`;
|
|
466
|
-
const sdpLines = offer.sdp.split("\r\n");
|
|
467
|
-
const modifiedLines = [];
|
|
468
|
-
for (let i = 0; i < sdpLines.length; i++) {
|
|
469
|
-
if (sdpLines[i].includes("VP8/90000")) {
|
|
470
|
-
const match = sdpLines[i].match(/a=rtpmap:(\d+) VP8/);
|
|
471
|
-
if (match) {
|
|
472
|
-
const payloadType = match[1];
|
|
473
|
-
let fmtpIndex = -1;
|
|
474
|
-
let insertAfterIndex = i;
|
|
475
|
-
for (let j = i + 1; j < sdpLines.length && sdpLines[j].startsWith("a="); j++) {
|
|
476
|
-
if (sdpLines[j].startsWith(`a=fmtp:${payloadType}`)) {
|
|
477
|
-
fmtpIndex = j;
|
|
478
|
-
break;
|
|
479
|
-
}
|
|
480
|
-
if (sdpLines[j].startsWith(`a=rtcp-fb:${payloadType}`)) insertAfterIndex = j;
|
|
481
|
-
if (sdpLines[j].startsWith("a=rtpmap:")) break;
|
|
482
|
-
}
|
|
483
|
-
if (fmtpIndex !== -1) {
|
|
484
|
-
if (!sdpLines[fmtpIndex].includes("x-google-min-bitrate")) sdpLines[fmtpIndex] += `;${bitrateParams}`;
|
|
485
|
-
} else {
|
|
486
|
-
for (let k = i; k <= insertAfterIndex; k++) modifiedLines.push(sdpLines[k]);
|
|
487
|
-
modifiedLines.push(`a=fmtp:${payloadType} ${bitrateParams}`);
|
|
488
|
-
i = insertAfterIndex;
|
|
489
|
-
continue;
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
modifiedLines.push(sdpLines[i]);
|
|
494
|
-
}
|
|
495
|
-
offer.sdp = modifiedLines.join("\r\n");
|
|
496
|
-
}
|
|
497
|
-
};
|
|
498
|
-
|
|
499
|
-
//#endregion
|
|
500
|
-
export { WebRTCConnection };
|
|
@@ -1,210 +0,0 @@
|
|
|
1
|
-
import { WebRTCConnection } from "./webrtc-connection.js";
|
|
2
|
-
import pRetry, { AbortError } from "p-retry";
|
|
3
|
-
|
|
4
|
-
//#region src/realtime/webrtc-manager.ts
|
|
5
|
-
const PERMANENT_ERRORS = [
|
|
6
|
-
"permission denied",
|
|
7
|
-
"not allowed",
|
|
8
|
-
"invalid session",
|
|
9
|
-
"401",
|
|
10
|
-
"invalid api key",
|
|
11
|
-
"unauthorized"
|
|
12
|
-
];
|
|
13
|
-
const CONNECTION_TIMEOUT = 6e4 * 5;
|
|
14
|
-
const RETRY_OPTIONS = {
|
|
15
|
-
retries: 5,
|
|
16
|
-
factor: 2,
|
|
17
|
-
minTimeout: 1e3,
|
|
18
|
-
maxTimeout: 1e4
|
|
19
|
-
};
|
|
20
|
-
var WebRTCManager = class {
|
|
21
|
-
connection;
|
|
22
|
-
config;
|
|
23
|
-
logger;
|
|
24
|
-
observability;
|
|
25
|
-
localStream = null;
|
|
26
|
-
subscribeMode = false;
|
|
27
|
-
managerState = "disconnected";
|
|
28
|
-
hasConnected = false;
|
|
29
|
-
isReconnecting = false;
|
|
30
|
-
intentionalDisconnect = false;
|
|
31
|
-
reconnectGeneration = 0;
|
|
32
|
-
statsProviderConnection = null;
|
|
33
|
-
constructor(config) {
|
|
34
|
-
this.config = config;
|
|
35
|
-
this.logger = config.logger ?? {
|
|
36
|
-
debug() {},
|
|
37
|
-
info() {},
|
|
38
|
-
warn() {},
|
|
39
|
-
error() {}
|
|
40
|
-
};
|
|
41
|
-
this.observability = config.observability;
|
|
42
|
-
this.connection = new WebRTCConnection({
|
|
43
|
-
onRemoteStream: config.onRemoteStream,
|
|
44
|
-
onStateChange: (state) => this.handleConnectionStateChange(state),
|
|
45
|
-
onError: config.onError,
|
|
46
|
-
customizeOffer: config.customizeOffer,
|
|
47
|
-
vp8MinBitrate: config.vp8MinBitrate,
|
|
48
|
-
vp8StartBitrate: config.vp8StartBitrate,
|
|
49
|
-
initialImage: config.initialImage,
|
|
50
|
-
initialPrompt: config.initialPrompt,
|
|
51
|
-
logger: this.logger,
|
|
52
|
-
observability: this.observability
|
|
53
|
-
});
|
|
54
|
-
}
|
|
55
|
-
emitState(state) {
|
|
56
|
-
if (this.managerState !== state) {
|
|
57
|
-
this.managerState = state;
|
|
58
|
-
if (state === "connected" || state === "generating") this.hasConnected = true;
|
|
59
|
-
this.config.onConnectionStateChange?.(state);
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
syncStatsProvider() {
|
|
63
|
-
const pc = this.getPeerConnection();
|
|
64
|
-
const isLive = this.managerState === "connected" || this.managerState === "generating";
|
|
65
|
-
if (isLive && pc && pc !== this.statsProviderConnection) {
|
|
66
|
-
this.statsProviderConnection = pc;
|
|
67
|
-
this.observability.setStatsProvider(pc);
|
|
68
|
-
} else if (!isLive && this.statsProviderConnection) {
|
|
69
|
-
this.statsProviderConnection = null;
|
|
70
|
-
this.observability.setStatsProvider(null);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
handleConnectionStateChange(state) {
|
|
74
|
-
if (this.intentionalDisconnect) {
|
|
75
|
-
this.emitState("disconnected");
|
|
76
|
-
this.syncStatsProvider();
|
|
77
|
-
return;
|
|
78
|
-
}
|
|
79
|
-
if (this.isReconnecting) {
|
|
80
|
-
if (state === "connected" || state === "generating") {
|
|
81
|
-
this.isReconnecting = false;
|
|
82
|
-
this.emitState(state);
|
|
83
|
-
this.syncStatsProvider();
|
|
84
|
-
}
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
if (state === "disconnected" && !this.intentionalDisconnect && this.hasConnected) {
|
|
88
|
-
this.reconnect();
|
|
89
|
-
return;
|
|
90
|
-
}
|
|
91
|
-
this.emitState(state);
|
|
92
|
-
this.syncStatsProvider();
|
|
93
|
-
}
|
|
94
|
-
async reconnect() {
|
|
95
|
-
if (this.isReconnecting || this.intentionalDisconnect) return;
|
|
96
|
-
if (!this.subscribeMode && !this.localStream) return;
|
|
97
|
-
const reconnectGeneration = ++this.reconnectGeneration;
|
|
98
|
-
this.isReconnecting = true;
|
|
99
|
-
this.emitState("reconnecting");
|
|
100
|
-
this.observability.setStatsProvider(null);
|
|
101
|
-
this.statsProviderConnection = null;
|
|
102
|
-
const reconnectStart = performance.now();
|
|
103
|
-
try {
|
|
104
|
-
let attemptCount = 0;
|
|
105
|
-
await pRetry(async () => {
|
|
106
|
-
attemptCount++;
|
|
107
|
-
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) throw new AbortError("Reconnect cancelled");
|
|
108
|
-
if (!this.subscribeMode && !this.localStream) throw new AbortError("Reconnect cancelled: no local stream");
|
|
109
|
-
this.connection.cleanup();
|
|
110
|
-
await this.connection.connect(this.config.webrtcUrl, this.localStream, CONNECTION_TIMEOUT, this.config.integration);
|
|
111
|
-
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) {
|
|
112
|
-
this.connection.cleanup();
|
|
113
|
-
throw new AbortError("Reconnect cancelled");
|
|
114
|
-
}
|
|
115
|
-
}, {
|
|
116
|
-
...RETRY_OPTIONS,
|
|
117
|
-
onFailedAttempt: (error) => {
|
|
118
|
-
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) return;
|
|
119
|
-
this.logger.warn("Reconnect attempt failed", {
|
|
120
|
-
error: error.message,
|
|
121
|
-
attempt: error.attemptNumber
|
|
122
|
-
});
|
|
123
|
-
this.observability.diagnostic("reconnect", {
|
|
124
|
-
attempt: error.attemptNumber,
|
|
125
|
-
maxAttempts: RETRY_OPTIONS.retries + 1,
|
|
126
|
-
durationMs: performance.now() - reconnectStart,
|
|
127
|
-
success: false,
|
|
128
|
-
error: error.message
|
|
129
|
-
});
|
|
130
|
-
this.connection.cleanup();
|
|
131
|
-
},
|
|
132
|
-
shouldRetry: (error) => {
|
|
133
|
-
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) return false;
|
|
134
|
-
const msg = error.message.toLowerCase();
|
|
135
|
-
return !PERMANENT_ERRORS.some((err) => msg.includes(err));
|
|
136
|
-
}
|
|
137
|
-
});
|
|
138
|
-
this.observability.diagnostic("reconnect", {
|
|
139
|
-
attempt: attemptCount,
|
|
140
|
-
maxAttempts: RETRY_OPTIONS.retries + 1,
|
|
141
|
-
durationMs: performance.now() - reconnectStart,
|
|
142
|
-
success: true
|
|
143
|
-
});
|
|
144
|
-
} catch (error) {
|
|
145
|
-
this.isReconnecting = false;
|
|
146
|
-
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) return;
|
|
147
|
-
this.emitState("disconnected");
|
|
148
|
-
this.config.onError?.(error instanceof Error ? error : new Error(String(error)));
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
async connect(localStream) {
|
|
152
|
-
this.localStream = localStream;
|
|
153
|
-
this.subscribeMode = localStream === null;
|
|
154
|
-
this.intentionalDisconnect = false;
|
|
155
|
-
this.hasConnected = false;
|
|
156
|
-
this.isReconnecting = false;
|
|
157
|
-
this.reconnectGeneration += 1;
|
|
158
|
-
this.emitState("connecting");
|
|
159
|
-
return pRetry(async () => {
|
|
160
|
-
if (this.intentionalDisconnect) throw new AbortError("Connect cancelled");
|
|
161
|
-
await this.connection.connect(this.config.webrtcUrl, localStream, CONNECTION_TIMEOUT, this.config.integration);
|
|
162
|
-
return true;
|
|
163
|
-
}, {
|
|
164
|
-
...RETRY_OPTIONS,
|
|
165
|
-
onFailedAttempt: (error) => {
|
|
166
|
-
this.logger.warn("Connection attempt failed", {
|
|
167
|
-
error: error.message,
|
|
168
|
-
attempt: error.attemptNumber
|
|
169
|
-
});
|
|
170
|
-
this.connection.cleanup();
|
|
171
|
-
},
|
|
172
|
-
shouldRetry: (error) => {
|
|
173
|
-
if (this.intentionalDisconnect) return false;
|
|
174
|
-
const msg = error.message.toLowerCase();
|
|
175
|
-
return !PERMANENT_ERRORS.some((err) => msg.includes(err));
|
|
176
|
-
}
|
|
177
|
-
});
|
|
178
|
-
}
|
|
179
|
-
sendMessage(message) {
|
|
180
|
-
return this.connection.send(message);
|
|
181
|
-
}
|
|
182
|
-
cleanup() {
|
|
183
|
-
this.intentionalDisconnect = true;
|
|
184
|
-
this.isReconnecting = false;
|
|
185
|
-
this.reconnectGeneration += 1;
|
|
186
|
-
this.connection.cleanup();
|
|
187
|
-
this.localStream = null;
|
|
188
|
-
this.statsProviderConnection = null;
|
|
189
|
-
this.observability.setStatsProvider(null);
|
|
190
|
-
this.emitState("disconnected");
|
|
191
|
-
}
|
|
192
|
-
isConnected() {
|
|
193
|
-
return this.managerState === "connected" || this.managerState === "generating";
|
|
194
|
-
}
|
|
195
|
-
getConnectionState() {
|
|
196
|
-
return this.managerState;
|
|
197
|
-
}
|
|
198
|
-
getPeerConnection() {
|
|
199
|
-
return this.connection.getPeerConnection();
|
|
200
|
-
}
|
|
201
|
-
getWebsocketMessageEmitter() {
|
|
202
|
-
return this.connection.websocketMessagesEmitter;
|
|
203
|
-
}
|
|
204
|
-
setImage(imageBase64, options) {
|
|
205
|
-
return this.connection.setImageBase64(imageBase64, options);
|
|
206
|
-
}
|
|
207
|
-
};
|
|
208
|
-
|
|
209
|
-
//#endregion
|
|
210
|
-
export { WebRTCManager };
|