@decartai/sdk 0.0.68 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -9
- package/dist/files/client.d.ts +33 -0
- package/dist/files/client.js +65 -0
- package/dist/files/types.d.ts +22 -0
- package/dist/files/types.js +4 -0
- package/dist/index.d.ts +19 -4
- package/dist/index.js +50 -28
- package/dist/process/client.js +1 -3
- package/dist/process/request.js +1 -3
- package/dist/queue/client.js +1 -3
- package/dist/queue/polling.js +1 -2
- package/dist/queue/request.js +1 -3
- package/dist/realtime/client.d.ts +23 -11
- package/dist/realtime/client.js +85 -156
- package/dist/realtime/config-realtime.js +49 -0
- package/dist/realtime/event-buffer.js +1 -3
- package/dist/realtime/initial-state-gate.js +21 -0
- package/dist/realtime/media-channel.js +82 -0
- package/dist/realtime/methods.js +25 -43
- package/dist/realtime/mirror-stream.js +1 -2
- package/dist/realtime/observability/diagnostics.d.ts +14 -53
- package/dist/realtime/observability/livekit-stats-provider.js +25 -0
- package/dist/realtime/observability/realtime-observability.js +70 -6
- package/dist/realtime/observability/telemetry-reporter.js +9 -28
- package/dist/realtime/observability/webrtc-stats.d.ts +5 -4
- package/dist/realtime/observability/webrtc-stats.js +3 -5
- package/dist/realtime/signaling-channel.js +302 -0
- package/dist/realtime/stream-session.js +257 -0
- package/dist/realtime/subscribe-client.d.ts +2 -3
- package/dist/realtime/subscribe-client.js +115 -11
- package/dist/realtime/types.d.ts +25 -1
- package/dist/shared/model.d.ts +11 -1
- package/dist/shared/model.js +51 -14
- package/dist/shared/request.js +1 -3
- package/dist/shared/types.js +1 -3
- package/dist/tokens/client.js +1 -3
- package/dist/utils/env.js +1 -2
- package/dist/utils/errors.d.ts +3 -0
- package/dist/utils/errors.js +4 -2
- package/dist/utils/logger.js +1 -2
- package/dist/utils/media.js +43 -0
- package/dist/utils/platform.js +13 -0
- package/dist/utils/user-agent.js +1 -3
- package/dist/version.js +1 -2
- package/package.json +2 -1
- package/dist/realtime/webrtc-connection.js +0 -500
- package/dist/realtime/webrtc-manager.js +0 -210
package/dist/utils/env.js
CHANGED
|
@@ -4,6 +4,5 @@ const readEnv = (env) => {
|
|
|
4
4
|
if (typeof globalThisAny.process !== "undefined") return globalThisAny.process.env?.[env]?.trim();
|
|
5
5
|
if (typeof globalThisAny.Deno !== "undefined") return globalThisAny.Deno.env?.get?.(env)?.trim();
|
|
6
6
|
};
|
|
7
|
-
|
|
8
7
|
//#endregion
|
|
9
|
-
export { readEnv };
|
|
8
|
+
export { readEnv };
|
package/dist/utils/errors.d.ts
CHANGED
|
@@ -17,6 +17,9 @@ declare const ERROR_CODES: {
|
|
|
17
17
|
readonly QUEUE_RESULT_ERROR: "QUEUE_RESULT_ERROR";
|
|
18
18
|
readonly JOB_NOT_COMPLETED: "JOB_NOT_COMPLETED";
|
|
19
19
|
readonly TOKEN_CREATE_ERROR: "TOKEN_CREATE_ERROR";
|
|
20
|
+
readonly FILES_UPLOAD_ERROR: "FILES_UPLOAD_ERROR";
|
|
21
|
+
readonly FILES_GET_ERROR: "FILES_GET_ERROR";
|
|
22
|
+
readonly FILES_DELETE_ERROR: "FILES_DELETE_ERROR";
|
|
20
23
|
readonly WEBRTC_WEBSOCKET_ERROR: "WEBRTC_WEBSOCKET_ERROR";
|
|
21
24
|
readonly WEBRTC_ICE_ERROR: "WEBRTC_ICE_ERROR";
|
|
22
25
|
readonly WEBRTC_TIMEOUT_ERROR: "WEBRTC_TIMEOUT_ERROR";
|
package/dist/utils/errors.js
CHANGED
|
@@ -11,6 +11,9 @@ const ERROR_CODES = {
|
|
|
11
11
|
QUEUE_RESULT_ERROR: "QUEUE_RESULT_ERROR",
|
|
12
12
|
JOB_NOT_COMPLETED: "JOB_NOT_COMPLETED",
|
|
13
13
|
TOKEN_CREATE_ERROR: "TOKEN_CREATE_ERROR",
|
|
14
|
+
FILES_UPLOAD_ERROR: "FILES_UPLOAD_ERROR",
|
|
15
|
+
FILES_GET_ERROR: "FILES_GET_ERROR",
|
|
16
|
+
FILES_DELETE_ERROR: "FILES_DELETE_ERROR",
|
|
14
17
|
WEBRTC_WEBSOCKET_ERROR: "WEBRTC_WEBSOCKET_ERROR",
|
|
15
18
|
WEBRTC_ICE_ERROR: "WEBRTC_ICE_ERROR",
|
|
16
19
|
WEBRTC_TIMEOUT_ERROR: "WEBRTC_TIMEOUT_ERROR",
|
|
@@ -79,6 +82,5 @@ function createQueueStatusError(message, status) {
|
|
|
79
82
|
function createQueueResultError(message, status) {
|
|
80
83
|
return createSDKError(ERROR_CODES.QUEUE_RESULT_ERROR, message, { status });
|
|
81
84
|
}
|
|
82
|
-
|
|
83
85
|
//#endregion
|
|
84
|
-
export { ERROR_CODES, classifyWebrtcError, createInvalidApiKeyError, createInvalidBaseUrlError, createInvalidInputError, createModelNotFoundError, createQueueResultError, createQueueStatusError, createQueueSubmitError, createSDKError };
|
|
86
|
+
export { ERROR_CODES, classifyWebrtcError, createInvalidApiKeyError, createInvalidBaseUrlError, createInvalidInputError, createModelNotFoundError, createQueueResultError, createQueueStatusError, createQueueSubmitError, createSDKError };
|
package/dist/utils/logger.js
CHANGED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
//#region src/utils/media.ts
|
|
2
|
+
async function blobToBase64(blob) {
|
|
3
|
+
return new Promise((resolve, reject) => {
|
|
4
|
+
const reader = new FileReader();
|
|
5
|
+
reader.onloadend = () => {
|
|
6
|
+
const result = reader.result;
|
|
7
|
+
if (typeof result !== "string") {
|
|
8
|
+
reject(/* @__PURE__ */ new Error("FileReader did not return a string"));
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
const base64 = result.split(",")[1];
|
|
12
|
+
if (!base64) {
|
|
13
|
+
reject(/* @__PURE__ */ new Error("Invalid data URL format"));
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
resolve(base64);
|
|
17
|
+
};
|
|
18
|
+
reader.onerror = reject;
|
|
19
|
+
reader.readAsDataURL(blob);
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
async function imageToBase64(image) {
|
|
23
|
+
if (typeof image === "string") {
|
|
24
|
+
let url = null;
|
|
25
|
+
try {
|
|
26
|
+
url = new URL(image);
|
|
27
|
+
} catch {}
|
|
28
|
+
if (url?.protocol === "data:") {
|
|
29
|
+
const [, base64] = image.split(",", 2);
|
|
30
|
+
if (!base64) throw new Error("Invalid data URL image");
|
|
31
|
+
return base64;
|
|
32
|
+
}
|
|
33
|
+
if (url?.protocol === "http:" || url?.protocol === "https:") {
|
|
34
|
+
const response = await fetch(image);
|
|
35
|
+
if (!response.ok) throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
|
|
36
|
+
return blobToBase64(await response.blob());
|
|
37
|
+
}
|
|
38
|
+
return image;
|
|
39
|
+
}
|
|
40
|
+
return blobToBase64(image);
|
|
41
|
+
}
|
|
42
|
+
//#endregion
|
|
43
|
+
export { imageToBase64 };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
//#region src/utils/platform.ts
|
|
2
|
+
function isDesktopSafari() {
|
|
3
|
+
const g = globalThis;
|
|
4
|
+
const ua = g?.navigator?.userAgent ?? "";
|
|
5
|
+
const platform = g?.navigator?.platform ?? "";
|
|
6
|
+
const maxTouchPoints = g?.navigator?.maxTouchPoints ?? 0;
|
|
7
|
+
if (!/^((?!chrome|chromium|crios|fxios|edg|firefox|opr|opera|android).)*safari/i.test(ua)) return false;
|
|
8
|
+
if (/iPad|iPhone|iPod/.test(ua)) return false;
|
|
9
|
+
if (platform === "MacIntel" && maxTouchPoints > 1) return false;
|
|
10
|
+
return true;
|
|
11
|
+
}
|
|
12
|
+
//#endregion
|
|
13
|
+
export { isDesktopSafari };
|
package/dist/utils/user-agent.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { VERSION } from "../version.js";
|
|
2
|
-
|
|
3
2
|
//#region src/utils/user-agent.ts
|
|
4
3
|
function getRuntimeEnvironment(globalThisAny = globalThis) {
|
|
5
4
|
if (globalThisAny.window) return "runtime/browser";
|
|
@@ -27,6 +26,5 @@ function buildUserAgent(integration, globalThisAny = globalThis) {
|
|
|
27
26
|
getRuntimeEnvironment(globalThisAny)
|
|
28
27
|
].join(" ");
|
|
29
28
|
}
|
|
30
|
-
|
|
31
29
|
//#endregion
|
|
32
|
-
export { buildUserAgent };
|
|
30
|
+
export { buildUserAgent };
|
package/dist/version.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@decartai/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Decart's JavaScript SDK",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -43,6 +43,7 @@
|
|
|
43
43
|
"vitest": "^4.0.18"
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
|
+
"livekit-client": "^2.0.0",
|
|
46
47
|
"mitt": "^3.0.1",
|
|
47
48
|
"p-retry": "^6.2.1",
|
|
48
49
|
"zod": "^4.0.17"
|
|
@@ -1,500 +0,0 @@
|
|
|
1
|
-
import { buildUserAgent } from "../utils/user-agent.js";
|
|
2
|
-
import mitt from "mitt";
|
|
3
|
-
|
|
4
|
-
//#region src/realtime/webrtc-connection.ts
|
|
5
|
-
const ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }];
|
|
6
|
-
const SETUP_TIMEOUT_MS = 3e4;
|
|
7
|
-
var WebRTCConnection = class {
|
|
8
|
-
pc = null;
|
|
9
|
-
ws = null;
|
|
10
|
-
localStream = null;
|
|
11
|
-
connectionReject = null;
|
|
12
|
-
logger;
|
|
13
|
-
observability;
|
|
14
|
-
state = "disconnected";
|
|
15
|
-
websocketMessagesEmitter = mitt();
|
|
16
|
-
constructor(callbacks) {
|
|
17
|
-
this.callbacks = callbacks;
|
|
18
|
-
this.logger = callbacks.logger ?? {
|
|
19
|
-
debug() {},
|
|
20
|
-
info() {},
|
|
21
|
-
warn() {},
|
|
22
|
-
error() {}
|
|
23
|
-
};
|
|
24
|
-
this.observability = callbacks.observability;
|
|
25
|
-
}
|
|
26
|
-
getPeerConnection() {
|
|
27
|
-
return this.pc;
|
|
28
|
-
}
|
|
29
|
-
async connect(url, localStream, timeout, integration) {
|
|
30
|
-
const deadline = Date.now() + timeout;
|
|
31
|
-
this.localStream = localStream;
|
|
32
|
-
const userAgent = encodeURIComponent(buildUserAgent(integration));
|
|
33
|
-
const wsUrl = `${url}${url.includes("?") ? "&" : "?"}user_agent=${userAgent}`;
|
|
34
|
-
let rejectConnect;
|
|
35
|
-
const connectAbort = new Promise((_, reject) => {
|
|
36
|
-
rejectConnect = reject;
|
|
37
|
-
});
|
|
38
|
-
connectAbort.catch(() => {});
|
|
39
|
-
this.connectionReject = (error) => rejectConnect(error);
|
|
40
|
-
const totalStart = performance.now();
|
|
41
|
-
try {
|
|
42
|
-
const wsStart = performance.now();
|
|
43
|
-
await Promise.race([new Promise((resolve, reject) => {
|
|
44
|
-
const timer = setTimeout(() => reject(/* @__PURE__ */ new Error("WebSocket timeout")), timeout);
|
|
45
|
-
this.ws = new WebSocket(wsUrl);
|
|
46
|
-
this.ws.onopen = () => {
|
|
47
|
-
clearTimeout(timer);
|
|
48
|
-
this.observability.diagnostic("phaseTiming", {
|
|
49
|
-
phase: "websocket",
|
|
50
|
-
durationMs: performance.now() - wsStart,
|
|
51
|
-
success: true
|
|
52
|
-
});
|
|
53
|
-
resolve();
|
|
54
|
-
};
|
|
55
|
-
this.ws.onmessage = (e) => {
|
|
56
|
-
try {
|
|
57
|
-
this.handleSignalingMessage(JSON.parse(e.data));
|
|
58
|
-
} catch (err) {
|
|
59
|
-
this.logger.error("Signaling message parse error", { error: String(err) });
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
this.ws.onerror = () => {
|
|
63
|
-
clearTimeout(timer);
|
|
64
|
-
const error = /* @__PURE__ */ new Error("WebSocket error");
|
|
65
|
-
this.observability.diagnostic("phaseTiming", {
|
|
66
|
-
phase: "websocket",
|
|
67
|
-
durationMs: performance.now() - wsStart,
|
|
68
|
-
success: false,
|
|
69
|
-
error: error.message
|
|
70
|
-
});
|
|
71
|
-
reject(error);
|
|
72
|
-
rejectConnect(error);
|
|
73
|
-
};
|
|
74
|
-
this.ws.onclose = () => {
|
|
75
|
-
this.setState("disconnected");
|
|
76
|
-
clearTimeout(timer);
|
|
77
|
-
reject(/* @__PURE__ */ new Error("WebSocket closed before connection was established"));
|
|
78
|
-
rejectConnect(/* @__PURE__ */ new Error("WebSocket closed"));
|
|
79
|
-
};
|
|
80
|
-
}), connectAbort]);
|
|
81
|
-
if (this.callbacks.initialImage) {
|
|
82
|
-
const imageStart = performance.now();
|
|
83
|
-
await Promise.race([this.setImageBase64(this.callbacks.initialImage, {
|
|
84
|
-
prompt: this.callbacks.initialPrompt?.text,
|
|
85
|
-
enhance: this.callbacks.initialPrompt?.enhance
|
|
86
|
-
}), connectAbort]);
|
|
87
|
-
this.observability.diagnostic("phaseTiming", {
|
|
88
|
-
phase: "avatar-image",
|
|
89
|
-
durationMs: performance.now() - imageStart,
|
|
90
|
-
success: true
|
|
91
|
-
});
|
|
92
|
-
} else if (this.callbacks.initialPrompt) {
|
|
93
|
-
const promptStart = performance.now();
|
|
94
|
-
await Promise.race([this.sendInitialPrompt(this.callbacks.initialPrompt), connectAbort]);
|
|
95
|
-
this.observability.diagnostic("phaseTiming", {
|
|
96
|
-
phase: "initial-prompt",
|
|
97
|
-
durationMs: performance.now() - promptStart,
|
|
98
|
-
success: true
|
|
99
|
-
});
|
|
100
|
-
} else if (localStream) {
|
|
101
|
-
const nullStart = performance.now();
|
|
102
|
-
await Promise.race([this.setImageBase64(null, { prompt: null }), connectAbort]);
|
|
103
|
-
this.observability.diagnostic("phaseTiming", {
|
|
104
|
-
phase: "initial-prompt",
|
|
105
|
-
durationMs: performance.now() - nullStart,
|
|
106
|
-
success: true
|
|
107
|
-
});
|
|
108
|
-
}
|
|
109
|
-
const handshakeStart = performance.now();
|
|
110
|
-
await this.setupNewPeerConnection();
|
|
111
|
-
await Promise.race([new Promise((resolve, reject) => {
|
|
112
|
-
const checkConnection = setInterval(() => {
|
|
113
|
-
if (this.state === "connected" || this.state === "generating") {
|
|
114
|
-
clearInterval(checkConnection);
|
|
115
|
-
this.observability.diagnostic("phaseTiming", {
|
|
116
|
-
phase: "webrtc-handshake",
|
|
117
|
-
durationMs: performance.now() - handshakeStart,
|
|
118
|
-
success: true
|
|
119
|
-
});
|
|
120
|
-
resolve();
|
|
121
|
-
} else if (this.state === "disconnected") {
|
|
122
|
-
clearInterval(checkConnection);
|
|
123
|
-
this.observability.diagnostic("phaseTiming", {
|
|
124
|
-
phase: "webrtc-handshake",
|
|
125
|
-
durationMs: performance.now() - handshakeStart,
|
|
126
|
-
success: false,
|
|
127
|
-
error: "Connection lost during handshake"
|
|
128
|
-
});
|
|
129
|
-
reject(/* @__PURE__ */ new Error("Connection lost during WebRTC handshake"));
|
|
130
|
-
} else if (Date.now() >= deadline) {
|
|
131
|
-
clearInterval(checkConnection);
|
|
132
|
-
this.observability.diagnostic("phaseTiming", {
|
|
133
|
-
phase: "webrtc-handshake",
|
|
134
|
-
durationMs: performance.now() - handshakeStart,
|
|
135
|
-
success: false,
|
|
136
|
-
error: "Timeout"
|
|
137
|
-
});
|
|
138
|
-
reject(/* @__PURE__ */ new Error("Connection timeout"));
|
|
139
|
-
}
|
|
140
|
-
}, 100);
|
|
141
|
-
connectAbort.catch(() => clearInterval(checkConnection));
|
|
142
|
-
}), connectAbort]);
|
|
143
|
-
this.observability.diagnostic("phaseTiming", {
|
|
144
|
-
phase: "total",
|
|
145
|
-
durationMs: performance.now() - totalStart,
|
|
146
|
-
success: true
|
|
147
|
-
});
|
|
148
|
-
} finally {
|
|
149
|
-
this.connectionReject = null;
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
async handleSignalingMessage(msg) {
|
|
153
|
-
try {
|
|
154
|
-
if (msg.type === "error") {
|
|
155
|
-
const error = new Error(msg.error);
|
|
156
|
-
error.source = "server";
|
|
157
|
-
this.callbacks.onError?.(error);
|
|
158
|
-
if (this.connectionReject) {
|
|
159
|
-
this.connectionReject(error);
|
|
160
|
-
this.connectionReject = null;
|
|
161
|
-
}
|
|
162
|
-
return;
|
|
163
|
-
}
|
|
164
|
-
if (msg.type === "set_image_ack") {
|
|
165
|
-
this.websocketMessagesEmitter.emit("setImageAck", msg);
|
|
166
|
-
return;
|
|
167
|
-
}
|
|
168
|
-
if (msg.type === "prompt_ack") {
|
|
169
|
-
this.websocketMessagesEmitter.emit("promptAck", msg);
|
|
170
|
-
return;
|
|
171
|
-
}
|
|
172
|
-
if (msg.type === "generation_started") {
|
|
173
|
-
this.setState("generating");
|
|
174
|
-
return;
|
|
175
|
-
}
|
|
176
|
-
if (msg.type === "generation_tick") {
|
|
177
|
-
this.websocketMessagesEmitter.emit("generationTick", msg);
|
|
178
|
-
return;
|
|
179
|
-
}
|
|
180
|
-
if (msg.type === "generation_ended") return;
|
|
181
|
-
if (msg.type === "session_id") {
|
|
182
|
-
this.websocketMessagesEmitter.emit("sessionId", msg);
|
|
183
|
-
return;
|
|
184
|
-
}
|
|
185
|
-
if (!this.pc) return;
|
|
186
|
-
switch (msg.type) {
|
|
187
|
-
case "ready": {
|
|
188
|
-
await this.applyCodecPreference("video/VP8");
|
|
189
|
-
const offer = await this.pc.createOffer();
|
|
190
|
-
this.modifyVP8Bitrate(offer);
|
|
191
|
-
await this.callbacks.customizeOffer?.(offer);
|
|
192
|
-
await this.pc.setLocalDescription(offer);
|
|
193
|
-
this.send({
|
|
194
|
-
type: "offer",
|
|
195
|
-
sdp: offer.sdp || ""
|
|
196
|
-
});
|
|
197
|
-
break;
|
|
198
|
-
}
|
|
199
|
-
case "offer": {
|
|
200
|
-
await this.pc.setRemoteDescription({
|
|
201
|
-
type: "offer",
|
|
202
|
-
sdp: msg.sdp
|
|
203
|
-
});
|
|
204
|
-
const answer = await this.pc.createAnswer();
|
|
205
|
-
await this.pc.setLocalDescription(answer);
|
|
206
|
-
this.send({
|
|
207
|
-
type: "answer",
|
|
208
|
-
sdp: answer.sdp || ""
|
|
209
|
-
});
|
|
210
|
-
break;
|
|
211
|
-
}
|
|
212
|
-
case "answer":
|
|
213
|
-
await this.pc.setRemoteDescription({
|
|
214
|
-
type: "answer",
|
|
215
|
-
sdp: msg.sdp
|
|
216
|
-
});
|
|
217
|
-
break;
|
|
218
|
-
case "ice-candidate":
|
|
219
|
-
if (msg.candidate) {
|
|
220
|
-
await this.pc.addIceCandidate(msg.candidate);
|
|
221
|
-
this.observability.diagnostic("iceCandidate", {
|
|
222
|
-
source: "remote",
|
|
223
|
-
candidateType: msg.candidate.candidate?.match(/typ (\w+)/)?.[1] ?? "unknown",
|
|
224
|
-
protocol: msg.candidate.candidate?.match(/udp|tcp/i)?.[0]?.toLowerCase() ?? "unknown"
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
|
-
break;
|
|
228
|
-
}
|
|
229
|
-
} catch (error) {
|
|
230
|
-
this.logger.error("Signaling handler error", { error: String(error) });
|
|
231
|
-
this.callbacks.onError?.(error);
|
|
232
|
-
this.connectionReject?.(error);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
send(message) {
|
|
236
|
-
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
237
|
-
this.ws.send(JSON.stringify(message));
|
|
238
|
-
return true;
|
|
239
|
-
}
|
|
240
|
-
this.logger.warn("Message dropped: WebSocket is not open");
|
|
241
|
-
return false;
|
|
242
|
-
}
|
|
243
|
-
async setImageBase64(imageBase64, options) {
|
|
244
|
-
return new Promise((resolve, reject) => {
|
|
245
|
-
const timeoutId = setTimeout(() => {
|
|
246
|
-
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
247
|
-
reject(/* @__PURE__ */ new Error("Image send timed out"));
|
|
248
|
-
}, options?.timeout ?? SETUP_TIMEOUT_MS);
|
|
249
|
-
const listener = (msg) => {
|
|
250
|
-
clearTimeout(timeoutId);
|
|
251
|
-
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
252
|
-
if (msg.success) resolve();
|
|
253
|
-
else reject(new Error(msg.error ?? "Failed to send image"));
|
|
254
|
-
};
|
|
255
|
-
this.websocketMessagesEmitter.on("setImageAck", listener);
|
|
256
|
-
const message = {
|
|
257
|
-
type: "set_image",
|
|
258
|
-
image_data: imageBase64
|
|
259
|
-
};
|
|
260
|
-
if (options?.prompt !== void 0) message.prompt = options.prompt;
|
|
261
|
-
if (options?.enhance !== void 0) message.enhance_prompt = options.enhance;
|
|
262
|
-
if (!this.send(message)) {
|
|
263
|
-
clearTimeout(timeoutId);
|
|
264
|
-
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
265
|
-
reject(/* @__PURE__ */ new Error("WebSocket is not open"));
|
|
266
|
-
}
|
|
267
|
-
});
|
|
268
|
-
}
|
|
269
|
-
/**
|
|
270
|
-
* Send the initial prompt to the server before WebRTC handshake.
|
|
271
|
-
*/
|
|
272
|
-
async sendInitialPrompt(prompt) {
|
|
273
|
-
return new Promise((resolve, reject) => {
|
|
274
|
-
const timeoutId = setTimeout(() => {
|
|
275
|
-
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
276
|
-
reject(/* @__PURE__ */ new Error("Prompt send timed out"));
|
|
277
|
-
}, SETUP_TIMEOUT_MS);
|
|
278
|
-
const listener = (msg) => {
|
|
279
|
-
if (msg.prompt === prompt.text) {
|
|
280
|
-
clearTimeout(timeoutId);
|
|
281
|
-
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
282
|
-
if (msg.success) resolve();
|
|
283
|
-
else reject(new Error(msg.error ?? "Failed to send prompt"));
|
|
284
|
-
}
|
|
285
|
-
};
|
|
286
|
-
this.websocketMessagesEmitter.on("promptAck", listener);
|
|
287
|
-
if (!this.send({
|
|
288
|
-
type: "prompt",
|
|
289
|
-
prompt: prompt.text,
|
|
290
|
-
enhance_prompt: prompt.enhance ?? true
|
|
291
|
-
})) {
|
|
292
|
-
clearTimeout(timeoutId);
|
|
293
|
-
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
294
|
-
reject(/* @__PURE__ */ new Error("WebSocket is not open"));
|
|
295
|
-
}
|
|
296
|
-
});
|
|
297
|
-
}
|
|
298
|
-
setState(state) {
|
|
299
|
-
if (this.state !== state) {
|
|
300
|
-
this.state = state;
|
|
301
|
-
this.callbacks.onStateChange?.(state);
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
async setupNewPeerConnection() {
|
|
305
|
-
if (this.pc) {
|
|
306
|
-
this.pc.getSenders().forEach((sender) => {
|
|
307
|
-
if (sender.track && this.pc) this.pc.removeTrack(sender);
|
|
308
|
-
});
|
|
309
|
-
this.pc.close();
|
|
310
|
-
}
|
|
311
|
-
this.pc = new RTCPeerConnection({ iceServers: ICE_SERVERS });
|
|
312
|
-
this.setState("connecting");
|
|
313
|
-
if (this.localStream) this.localStream.getTracks().forEach((track) => {
|
|
314
|
-
if (this.pc && this.localStream) this.pc.addTrack(track, this.localStream);
|
|
315
|
-
});
|
|
316
|
-
else {
|
|
317
|
-
this.pc.addTransceiver("video", { direction: "recvonly" });
|
|
318
|
-
this.pc.addTransceiver("audio", { direction: "recvonly" });
|
|
319
|
-
}
|
|
320
|
-
let fallbackStream = null;
|
|
321
|
-
this.pc.ontrack = (e) => {
|
|
322
|
-
if (e.streams?.[0]) this.callbacks.onRemoteStream?.(e.streams[0]);
|
|
323
|
-
else {
|
|
324
|
-
if (!fallbackStream) fallbackStream = new MediaStream();
|
|
325
|
-
fallbackStream.addTrack(e.track);
|
|
326
|
-
this.callbacks.onRemoteStream?.(fallbackStream);
|
|
327
|
-
}
|
|
328
|
-
};
|
|
329
|
-
this.pc.onicecandidate = (e) => {
|
|
330
|
-
this.send({
|
|
331
|
-
type: "ice-candidate",
|
|
332
|
-
candidate: e.candidate
|
|
333
|
-
});
|
|
334
|
-
if (e.candidate) this.observability.diagnostic("iceCandidate", {
|
|
335
|
-
source: "local",
|
|
336
|
-
candidateType: e.candidate.type ?? "unknown",
|
|
337
|
-
protocol: e.candidate.protocol ?? "unknown",
|
|
338
|
-
address: e.candidate.address ?? void 0,
|
|
339
|
-
port: e.candidate.port ?? void 0
|
|
340
|
-
});
|
|
341
|
-
};
|
|
342
|
-
let prevPcState = "new";
|
|
343
|
-
this.pc.onconnectionstatechange = () => {
|
|
344
|
-
if (!this.pc) return;
|
|
345
|
-
const s = this.pc.connectionState;
|
|
346
|
-
this.observability.diagnostic("peerConnectionStateChange", {
|
|
347
|
-
state: s,
|
|
348
|
-
previousState: prevPcState,
|
|
349
|
-
timestampMs: performance.now()
|
|
350
|
-
});
|
|
351
|
-
prevPcState = s;
|
|
352
|
-
if (s === "connected") this.emitSelectedCandidatePair();
|
|
353
|
-
const nextState = s === "connected" ? "connected" : ["connecting", "new"].includes(s) ? "connecting" : "disconnected";
|
|
354
|
-
if (this.state === "generating" && nextState !== "disconnected") return;
|
|
355
|
-
this.setState(nextState);
|
|
356
|
-
};
|
|
357
|
-
let prevIceState = "new";
|
|
358
|
-
this.pc.oniceconnectionstatechange = () => {
|
|
359
|
-
if (!this.pc) return;
|
|
360
|
-
const newIceState = this.pc.iceConnectionState;
|
|
361
|
-
this.observability.diagnostic("iceStateChange", {
|
|
362
|
-
state: newIceState,
|
|
363
|
-
previousState: prevIceState,
|
|
364
|
-
timestampMs: performance.now()
|
|
365
|
-
});
|
|
366
|
-
prevIceState = newIceState;
|
|
367
|
-
if (newIceState === "failed") {
|
|
368
|
-
this.setState("disconnected");
|
|
369
|
-
this.callbacks.onError?.(/* @__PURE__ */ new Error("ICE connection failed"));
|
|
370
|
-
}
|
|
371
|
-
};
|
|
372
|
-
let prevSignalingState = "stable";
|
|
373
|
-
this.pc.onsignalingstatechange = () => {
|
|
374
|
-
if (!this.pc) return;
|
|
375
|
-
const newState = this.pc.signalingState;
|
|
376
|
-
this.observability.diagnostic("signalingStateChange", {
|
|
377
|
-
state: newState,
|
|
378
|
-
previousState: prevSignalingState,
|
|
379
|
-
timestampMs: performance.now()
|
|
380
|
-
});
|
|
381
|
-
prevSignalingState = newState;
|
|
382
|
-
};
|
|
383
|
-
this.handleSignalingMessage({ type: "ready" });
|
|
384
|
-
}
|
|
385
|
-
async emitSelectedCandidatePair() {
|
|
386
|
-
if (!this.pc) return;
|
|
387
|
-
try {
|
|
388
|
-
const stats = await this.pc.getStats();
|
|
389
|
-
let found = false;
|
|
390
|
-
stats.forEach((report) => {
|
|
391
|
-
if (found) return;
|
|
392
|
-
if (report.type === "candidate-pair" && report.state === "succeeded") {
|
|
393
|
-
found = true;
|
|
394
|
-
let localCandidate;
|
|
395
|
-
let remoteCandidate;
|
|
396
|
-
stats.forEach((r) => {
|
|
397
|
-
if (r.id === report.localCandidateId) localCandidate = r;
|
|
398
|
-
if (r.id === report.remoteCandidateId) remoteCandidate = r;
|
|
399
|
-
});
|
|
400
|
-
if (localCandidate && remoteCandidate) this.observability.diagnostic("selectedCandidatePair", {
|
|
401
|
-
local: {
|
|
402
|
-
candidateType: String(localCandidate.candidateType ?? "unknown"),
|
|
403
|
-
protocol: String(localCandidate.protocol ?? "unknown"),
|
|
404
|
-
address: localCandidate.address,
|
|
405
|
-
port: localCandidate.port
|
|
406
|
-
},
|
|
407
|
-
remote: {
|
|
408
|
-
candidateType: String(remoteCandidate.candidateType ?? "unknown"),
|
|
409
|
-
protocol: String(remoteCandidate.protocol ?? "unknown"),
|
|
410
|
-
address: remoteCandidate.address,
|
|
411
|
-
port: remoteCandidate.port
|
|
412
|
-
}
|
|
413
|
-
});
|
|
414
|
-
}
|
|
415
|
-
});
|
|
416
|
-
} catch {}
|
|
417
|
-
}
|
|
418
|
-
cleanup() {
|
|
419
|
-
this.pc?.close();
|
|
420
|
-
this.pc = null;
|
|
421
|
-
this.ws?.close();
|
|
422
|
-
this.ws = null;
|
|
423
|
-
this.localStream = null;
|
|
424
|
-
this.setState("disconnected");
|
|
425
|
-
}
|
|
426
|
-
applyCodecPreference(preferredCodecName) {
|
|
427
|
-
if (!this.pc) return;
|
|
428
|
-
if (typeof RTCRtpSender === "undefined" || typeof RTCRtpSender.getCapabilities !== "function") {
|
|
429
|
-
this.logger.debug("RTCRtpSender capabilities not available in this environment");
|
|
430
|
-
return;
|
|
431
|
-
}
|
|
432
|
-
const videoTransceiver = this.pc.getTransceivers().find((r) => r.sender.track?.kind === "video" || r.receiver.track?.kind === "video");
|
|
433
|
-
if (!videoTransceiver) {
|
|
434
|
-
this.logger.warn("Video transceiver not found for codec preference");
|
|
435
|
-
return;
|
|
436
|
-
}
|
|
437
|
-
const capabilities = RTCRtpSender.getCapabilities("video");
|
|
438
|
-
if (!capabilities) {
|
|
439
|
-
this.logger.warn("Video sender capabilities unavailable");
|
|
440
|
-
return;
|
|
441
|
-
}
|
|
442
|
-
const preferredCodecs = [];
|
|
443
|
-
const otherCodecs = [];
|
|
444
|
-
capabilities.codecs.forEach((codec) => {
|
|
445
|
-
if (codec.mimeType.toLowerCase() === preferredCodecName.toLowerCase()) preferredCodecs.push(codec);
|
|
446
|
-
else otherCodecs.push(codec);
|
|
447
|
-
});
|
|
448
|
-
const orderedCodecs = [...preferredCodecs, ...otherCodecs];
|
|
449
|
-
if (orderedCodecs.length === 0) {
|
|
450
|
-
this.logger.debug("No video codecs found for preference setting");
|
|
451
|
-
return;
|
|
452
|
-
}
|
|
453
|
-
try {
|
|
454
|
-
videoTransceiver.setCodecPreferences(orderedCodecs);
|
|
455
|
-
} catch {
|
|
456
|
-
this.logger.debug("setCodecPreferences not supported, skipping");
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
|
-
modifyVP8Bitrate(offer) {
|
|
460
|
-
if (!offer.sdp) return;
|
|
461
|
-
const minBitrateInKbps = this.callbacks.vp8MinBitrate;
|
|
462
|
-
const startBitrateInKbps = this.callbacks.vp8StartBitrate;
|
|
463
|
-
if (minBitrateInKbps === void 0 || startBitrateInKbps === void 0) return;
|
|
464
|
-
if (minBitrateInKbps === 0 && startBitrateInKbps === 0) return;
|
|
465
|
-
const bitrateParams = `x-google-min-bitrate=${minBitrateInKbps};x-google-start-bitrate=${startBitrateInKbps}`;
|
|
466
|
-
const sdpLines = offer.sdp.split("\r\n");
|
|
467
|
-
const modifiedLines = [];
|
|
468
|
-
for (let i = 0; i < sdpLines.length; i++) {
|
|
469
|
-
if (sdpLines[i].includes("VP8/90000")) {
|
|
470
|
-
const match = sdpLines[i].match(/a=rtpmap:(\d+) VP8/);
|
|
471
|
-
if (match) {
|
|
472
|
-
const payloadType = match[1];
|
|
473
|
-
let fmtpIndex = -1;
|
|
474
|
-
let insertAfterIndex = i;
|
|
475
|
-
for (let j = i + 1; j < sdpLines.length && sdpLines[j].startsWith("a="); j++) {
|
|
476
|
-
if (sdpLines[j].startsWith(`a=fmtp:${payloadType}`)) {
|
|
477
|
-
fmtpIndex = j;
|
|
478
|
-
break;
|
|
479
|
-
}
|
|
480
|
-
if (sdpLines[j].startsWith(`a=rtcp-fb:${payloadType}`)) insertAfterIndex = j;
|
|
481
|
-
if (sdpLines[j].startsWith("a=rtpmap:")) break;
|
|
482
|
-
}
|
|
483
|
-
if (fmtpIndex !== -1) {
|
|
484
|
-
if (!sdpLines[fmtpIndex].includes("x-google-min-bitrate")) sdpLines[fmtpIndex] += `;${bitrateParams}`;
|
|
485
|
-
} else {
|
|
486
|
-
for (let k = i; k <= insertAfterIndex; k++) modifiedLines.push(sdpLines[k]);
|
|
487
|
-
modifiedLines.push(`a=fmtp:${payloadType} ${bitrateParams}`);
|
|
488
|
-
i = insertAfterIndex;
|
|
489
|
-
continue;
|
|
490
|
-
}
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
modifiedLines.push(sdpLines[i]);
|
|
494
|
-
}
|
|
495
|
-
offer.sdp = modifiedLines.join("\r\n");
|
|
496
|
-
}
|
|
497
|
-
};
|
|
498
|
-
|
|
499
|
-
//#endregion
|
|
500
|
-
export { WebRTCConnection };
|