@decartai/sdk 0.0.43 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -2
- package/dist/realtime/audio-stream-manager.js +1 -1
- package/dist/realtime/client.d.ts +4 -3
- package/dist/realtime/client.js +103 -63
- package/dist/realtime/methods.js +14 -7
- package/dist/realtime/types.d.ts +5 -0
- package/dist/realtime/webrtc-connection.js +97 -44
- package/dist/realtime/webrtc-manager.js +94 -10
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -4,8 +4,9 @@ import { ProcessClient } from "./process/client.js";
|
|
|
4
4
|
import { JobStatus, JobStatusResponse, JobSubmitResponse, QueueJobResult, QueueSubmitAndPollOptions, QueueSubmitOptions } from "./queue/types.js";
|
|
5
5
|
import { QueueClient } from "./queue/client.js";
|
|
6
6
|
import { DecartSDKError, ERROR_CODES } from "./utils/errors.js";
|
|
7
|
+
import { ConnectionState } from "./realtime/types.js";
|
|
7
8
|
import { SetInput } from "./realtime/methods.js";
|
|
8
|
-
import { AvatarOptions, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState } from "./realtime/client.js";
|
|
9
|
+
import { AvatarOptions, Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState } from "./realtime/client.js";
|
|
9
10
|
import { ModelState } from "./shared/types.js";
|
|
10
11
|
import { CreateTokenResponse, TokensClient } from "./tokens/client.js";
|
|
11
12
|
|
|
@@ -118,4 +119,4 @@ declare const createDecartClient: (options?: DecartClientOptions) => {
|
|
|
118
119
|
tokens: TokensClient;
|
|
119
120
|
};
|
|
120
121
|
//#endregion
|
|
121
|
-
export { type AvatarOptions, type CreateTokenResponse, DecartClientOptions, type DecartSDKError, ERROR_CODES, type FileInput, type ImageModelDefinition, type ImageModels, type JobStatus, type JobStatusResponse, type JobSubmitResponse, type Model, type ModelDefinition, type ModelState, type ProcessClient, type ProcessOptions, type QueueClient, type QueueJobResult, type QueueSubmitAndPollOptions, type QueueSubmitOptions, type ReactNativeFile, type RealTimeClient, type RealTimeClientConnectOptions, type RealTimeClientInitialState, type RealTimeModels, type SetInput, type TokensClient, type VideoModelDefinition, type VideoModels, createDecartClient, isImageModel, isRealtimeModel, isVideoModel, models };
|
|
122
|
+
export { type AvatarOptions, type ConnectionState, type CreateTokenResponse, DecartClientOptions, type DecartSDKError, ERROR_CODES, type FileInput, type ImageModelDefinition, type ImageModels, type JobStatus, type JobStatusResponse, type JobSubmitResponse, type Model, type ModelDefinition, type ModelState, type ProcessClient, type ProcessOptions, type QueueClient, type QueueJobResult, type QueueSubmitAndPollOptions, type QueueSubmitOptions, type ReactNativeFile, type RealTimeClient, type RealTimeClientConnectOptions, type RealTimeClientInitialState, type Events as RealTimeEvents, type RealTimeModels, type SetInput, type TokensClient, type VideoModelDefinition, type VideoModels, createDecartClient, isImageModel, isRealtimeModel, isVideoModel, models };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DecartSDKError } from "../utils/errors.js";
|
|
2
|
+
import { ConnectionState } from "./types.js";
|
|
2
3
|
import { SetInput } from "./methods.js";
|
|
3
4
|
import { z } from "zod";
|
|
4
5
|
|
|
@@ -40,7 +41,7 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
|
40
41
|
}, z.core.$strip>;
|
|
41
42
|
type RealTimeClientConnectOptions = z.infer<typeof realTimeClientConnectOptionsSchema>;
|
|
42
43
|
type Events = {
|
|
43
|
-
connectionChange:
|
|
44
|
+
connectionChange: ConnectionState;
|
|
44
45
|
error: DecartSDKError;
|
|
45
46
|
};
|
|
46
47
|
type RealTimeClient = {
|
|
@@ -51,7 +52,7 @@ type RealTimeClient = {
|
|
|
51
52
|
enhance?: boolean;
|
|
52
53
|
}) => Promise<void>;
|
|
53
54
|
isConnected: () => boolean;
|
|
54
|
-
getConnectionState: () =>
|
|
55
|
+
getConnectionState: () => ConnectionState;
|
|
55
56
|
disconnect: () => void;
|
|
56
57
|
on: <K extends keyof Events>(event: K, listener: (data: Events[K]) => void) => void;
|
|
57
58
|
off: <K extends keyof Events>(event: K, listener: (data: Events[K]) => void) => void;
|
|
@@ -64,4 +65,4 @@ type RealTimeClient = {
|
|
|
64
65
|
playAudio?: (audio: Blob | File | ArrayBuffer) => Promise<void>;
|
|
65
66
|
};
|
|
66
67
|
//#endregion
|
|
67
|
-
export { AvatarOptions, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
|
|
68
|
+
export { AvatarOptions, Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
|
package/dist/realtime/client.js
CHANGED
|
@@ -13,7 +13,16 @@ async function blobToBase64(blob) {
|
|
|
13
13
|
return new Promise((resolve, reject) => {
|
|
14
14
|
const reader = new FileReader();
|
|
15
15
|
reader.onloadend = () => {
|
|
16
|
-
const
|
|
16
|
+
const result = reader.result;
|
|
17
|
+
if (typeof result !== "string") {
|
|
18
|
+
reject(/* @__PURE__ */ new Error("FileReader did not return a string"));
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
const base64 = result.split(",")[1];
|
|
22
|
+
if (!base64) {
|
|
23
|
+
reject(/* @__PURE__ */ new Error("Invalid data URL format"));
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
17
26
|
resolve(base64);
|
|
18
27
|
};
|
|
19
28
|
reader.onerror = reject;
|
|
@@ -26,8 +35,16 @@ async function imageToBase64(image) {
|
|
|
26
35
|
try {
|
|
27
36
|
url = new URL(image);
|
|
28
37
|
} catch {}
|
|
29
|
-
if (url?.protocol === "data:")
|
|
30
|
-
|
|
38
|
+
if (url?.protocol === "data:") {
|
|
39
|
+
const [, base64] = image.split(",", 2);
|
|
40
|
+
if (!base64) throw new Error("Invalid data URL image");
|
|
41
|
+
return base64;
|
|
42
|
+
}
|
|
43
|
+
if (url?.protocol === "http:" || url?.protocol === "https:") {
|
|
44
|
+
const response = await fetch(image);
|
|
45
|
+
if (!response.ok) throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
|
|
46
|
+
return blobToBase64(await response.blob());
|
|
47
|
+
}
|
|
31
48
|
return image;
|
|
32
49
|
}
|
|
33
50
|
return blobToBase64(image);
|
|
@@ -61,68 +78,91 @@ const createRealTimeClient = (opts) => {
|
|
|
61
78
|
audioStreamManager = new AudioStreamManager();
|
|
62
79
|
inputStream = audioStreamManager.getStream();
|
|
63
80
|
} else inputStream = stream ?? new MediaStream();
|
|
64
|
-
let
|
|
65
|
-
|
|
66
|
-
let
|
|
67
|
-
if (typeof avatar.avatarImage === "string")
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
eventEmitter.emit(
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
const base64 = await imageToBase64(image);
|
|
118
|
-
return webrtcManager.setImage(base64, options$1);
|
|
81
|
+
let webrtcManager;
|
|
82
|
+
try {
|
|
83
|
+
let avatarImageBase64;
|
|
84
|
+
if (isAvatarLive && avatar?.avatarImage) if (typeof avatar.avatarImage === "string") {
|
|
85
|
+
const response = await fetch(avatar.avatarImage);
|
|
86
|
+
if (!response.ok) throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
|
|
87
|
+
avatarImageBase64 = await blobToBase64(await response.blob());
|
|
88
|
+
} else avatarImageBase64 = await blobToBase64(avatar.avatarImage);
|
|
89
|
+
const initialPrompt = isAvatarLive && initialState?.prompt ? {
|
|
90
|
+
text: initialState.prompt.text,
|
|
91
|
+
enhance: initialState.prompt.enhance
|
|
92
|
+
} : void 0;
|
|
93
|
+
const url = `${baseUrl}${options.model.urlPath}`;
|
|
94
|
+
const eventBuffer = [];
|
|
95
|
+
let buffering = true;
|
|
96
|
+
const emitOrBuffer = (event, data) => {
|
|
97
|
+
if (buffering) eventBuffer.push({
|
|
98
|
+
event,
|
|
99
|
+
data
|
|
100
|
+
});
|
|
101
|
+
else eventEmitter.emit(event, data);
|
|
102
|
+
};
|
|
103
|
+
const flushBufferedEvents = () => {
|
|
104
|
+
setTimeout(() => {
|
|
105
|
+
buffering = false;
|
|
106
|
+
for (const { event, data } of eventBuffer) eventEmitter.emit(event, data);
|
|
107
|
+
eventBuffer.length = 0;
|
|
108
|
+
}, 0);
|
|
109
|
+
};
|
|
110
|
+
webrtcManager = new WebRTCManager({
|
|
111
|
+
webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}`,
|
|
112
|
+
integration,
|
|
113
|
+
onRemoteStream,
|
|
114
|
+
onConnectionStateChange: (state) => {
|
|
115
|
+
emitOrBuffer("connectionChange", state);
|
|
116
|
+
},
|
|
117
|
+
onError: (error) => {
|
|
118
|
+
console.error("WebRTC error:", error);
|
|
119
|
+
emitOrBuffer("error", createWebrtcError(error));
|
|
120
|
+
},
|
|
121
|
+
customizeOffer: options.customizeOffer,
|
|
122
|
+
vp8MinBitrate: 300,
|
|
123
|
+
vp8StartBitrate: 600,
|
|
124
|
+
isAvatarLive,
|
|
125
|
+
avatarImageBase64,
|
|
126
|
+
initialPrompt
|
|
127
|
+
});
|
|
128
|
+
const manager = webrtcManager;
|
|
129
|
+
await manager.connect(inputStream);
|
|
130
|
+
const methods = realtimeMethods(manager, imageToBase64);
|
|
131
|
+
if (!isAvatarLive && initialState?.prompt) {
|
|
132
|
+
const { text, enhance } = initialState.prompt;
|
|
133
|
+
await methods.setPrompt(text, { enhance });
|
|
119
134
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
135
|
+
const client = {
|
|
136
|
+
set: methods.set,
|
|
137
|
+
setPrompt: methods.setPrompt,
|
|
138
|
+
isConnected: () => manager.isConnected(),
|
|
139
|
+
getConnectionState: () => manager.getConnectionState(),
|
|
140
|
+
disconnect: () => {
|
|
141
|
+
buffering = false;
|
|
142
|
+
eventBuffer.length = 0;
|
|
143
|
+
manager.cleanup();
|
|
144
|
+
audioStreamManager?.cleanup();
|
|
145
|
+
},
|
|
146
|
+
on: eventEmitter.on,
|
|
147
|
+
off: eventEmitter.off,
|
|
148
|
+
sessionId,
|
|
149
|
+
setImage: async (image, options$1) => {
|
|
150
|
+
if (image === null) return manager.setImage(null, options$1);
|
|
151
|
+
const base64 = await imageToBase64(image);
|
|
152
|
+
return manager.setImage(base64, options$1);
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
if (isAvatarLive && audioStreamManager) {
|
|
156
|
+
const manager$1 = audioStreamManager;
|
|
157
|
+
client.playAudio = (audio) => manager$1.playAudio(audio);
|
|
158
|
+
}
|
|
159
|
+
flushBufferedEvents();
|
|
160
|
+
return client;
|
|
161
|
+
} catch (error) {
|
|
162
|
+
webrtcManager?.cleanup();
|
|
163
|
+
audioStreamManager?.cleanup();
|
|
164
|
+
throw error;
|
|
124
165
|
}
|
|
125
|
-
return client;
|
|
126
166
|
};
|
|
127
167
|
return { connect };
|
|
128
168
|
};
|
package/dist/realtime/methods.js
CHANGED
|
@@ -13,8 +13,17 @@ const setInputSchema = z.object({
|
|
|
13
13
|
z.null()
|
|
14
14
|
]).optional()
|
|
15
15
|
}).refine((data) => data.prompt !== void 0 || data.image !== void 0, { message: "At least one of 'prompt' or 'image' must be provided" });
|
|
16
|
+
const setPromptInputSchema = z.object({
|
|
17
|
+
prompt: z.string().min(1),
|
|
18
|
+
enhance: z.boolean().optional().default(true)
|
|
19
|
+
});
|
|
16
20
|
const realtimeMethods = (webrtcManager, imageToBase64) => {
|
|
21
|
+
const assertConnected = () => {
|
|
22
|
+
const state = webrtcManager.getConnectionState();
|
|
23
|
+
if (state !== "connected" && state !== "generating") throw new Error(`Cannot send message: connection is ${state}`);
|
|
24
|
+
};
|
|
17
25
|
const set = async (input) => {
|
|
26
|
+
assertConnected();
|
|
18
27
|
const parsed = setInputSchema.safeParse(input);
|
|
19
28
|
if (!parsed.success) throw parsed.error;
|
|
20
29
|
const { prompt, enhance, image } = parsed.data;
|
|
@@ -27,10 +36,8 @@ const realtimeMethods = (webrtcManager, imageToBase64) => {
|
|
|
27
36
|
});
|
|
28
37
|
};
|
|
29
38
|
const setPrompt = async (prompt, { enhance } = {}) => {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
enhance: z.boolean().optional().default(true)
|
|
33
|
-
}).safeParse({
|
|
39
|
+
assertConnected();
|
|
40
|
+
const parsedInput = setPromptInputSchema.safeParse({
|
|
34
41
|
prompt,
|
|
35
42
|
enhance
|
|
36
43
|
});
|
|
@@ -42,15 +49,15 @@ const realtimeMethods = (webrtcManager, imageToBase64) => {
|
|
|
42
49
|
const ackPromise = new Promise((resolve, reject) => {
|
|
43
50
|
promptAckListener = (promptAckMessage) => {
|
|
44
51
|
if (promptAckMessage.prompt === parsedInput.data.prompt) if (promptAckMessage.success) resolve();
|
|
45
|
-
else reject(promptAckMessage.error);
|
|
52
|
+
else reject(new Error(promptAckMessage.error ?? "Failed to send prompt"));
|
|
46
53
|
};
|
|
47
54
|
emitter.on("promptAck", promptAckListener);
|
|
48
55
|
});
|
|
49
|
-
webrtcManager.sendMessage({
|
|
56
|
+
if (!webrtcManager.sendMessage({
|
|
50
57
|
type: "prompt",
|
|
51
58
|
prompt: parsedInput.data.prompt,
|
|
52
59
|
enhance_prompt: parsedInput.data.enhance
|
|
53
|
-
});
|
|
60
|
+
})) throw new Error("WebSocket is not open");
|
|
54
61
|
const timeoutPromise = new Promise((_, reject) => {
|
|
55
62
|
timeoutId = setTimeout(() => reject(/* @__PURE__ */ new Error("Prompt timed out")), PROMPT_TIMEOUT_MS);
|
|
56
63
|
});
|
|
@@ -19,43 +19,61 @@ var WebRTCConnection = class {
|
|
|
19
19
|
this.localStream = localStream;
|
|
20
20
|
const userAgent = encodeURIComponent(buildUserAgent(integration));
|
|
21
21
|
const wsUrl = `${url}${url.includes("?") ? "&" : "?"}user_agent=${userAgent}`;
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
this.ws = new WebSocket(wsUrl);
|
|
26
|
-
this.ws.onopen = () => {
|
|
27
|
-
clearTimeout(timer);
|
|
28
|
-
resolve();
|
|
29
|
-
};
|
|
30
|
-
this.ws.onmessage = (e) => {
|
|
31
|
-
try {
|
|
32
|
-
this.handleSignalingMessage(JSON.parse(e.data));
|
|
33
|
-
} catch (err) {
|
|
34
|
-
console.error("[WebRTC] Parse error:", err);
|
|
35
|
-
}
|
|
36
|
-
};
|
|
37
|
-
this.ws.onerror = () => {
|
|
38
|
-
clearTimeout(timer);
|
|
39
|
-
};
|
|
40
|
-
this.ws.onclose = () => this.setState("disconnected");
|
|
22
|
+
let rejectConnect;
|
|
23
|
+
const connectAbort = new Promise((_, reject) => {
|
|
24
|
+
rejectConnect = reject;
|
|
41
25
|
});
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
this.connectionReject = null;
|
|
26
|
+
connectAbort.catch(() => {});
|
|
27
|
+
this.connectionReject = (error) => rejectConnect(error);
|
|
28
|
+
try {
|
|
29
|
+
await Promise.race([new Promise((resolve, reject) => {
|
|
30
|
+
const timer = setTimeout(() => reject(/* @__PURE__ */ new Error("WebSocket timeout")), timeout);
|
|
31
|
+
this.ws = new WebSocket(wsUrl);
|
|
32
|
+
this.ws.onopen = () => {
|
|
33
|
+
clearTimeout(timer);
|
|
51
34
|
resolve();
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
35
|
+
};
|
|
36
|
+
this.ws.onmessage = (e) => {
|
|
37
|
+
try {
|
|
38
|
+
this.handleSignalingMessage(JSON.parse(e.data));
|
|
39
|
+
} catch (err) {
|
|
40
|
+
console.error("[WebRTC] Parse error:", err);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
this.ws.onerror = () => {
|
|
44
|
+
clearTimeout(timer);
|
|
45
|
+
const error = /* @__PURE__ */ new Error("WebSocket error");
|
|
46
|
+
reject(error);
|
|
47
|
+
rejectConnect(error);
|
|
48
|
+
};
|
|
49
|
+
this.ws.onclose = () => {
|
|
50
|
+
this.setState("disconnected");
|
|
51
|
+
clearTimeout(timer);
|
|
52
|
+
reject(/* @__PURE__ */ new Error("WebSocket closed before connection was established"));
|
|
53
|
+
rejectConnect(/* @__PURE__ */ new Error("WebSocket closed"));
|
|
54
|
+
};
|
|
55
|
+
}), connectAbort]);
|
|
56
|
+
if (this.callbacks.avatarImageBase64) await Promise.race([this.sendAvatarImage(this.callbacks.avatarImageBase64), connectAbort]);
|
|
57
|
+
if (this.callbacks.initialPrompt) await Promise.race([this.sendInitialPrompt(this.callbacks.initialPrompt), connectAbort]);
|
|
58
|
+
await this.setupNewPeerConnection();
|
|
59
|
+
await Promise.race([new Promise((resolve, reject) => {
|
|
60
|
+
const checkConnection = setInterval(() => {
|
|
61
|
+
if (this.state === "connected" || this.state === "generating") {
|
|
62
|
+
clearInterval(checkConnection);
|
|
63
|
+
resolve();
|
|
64
|
+
} else if (this.state === "disconnected") {
|
|
65
|
+
clearInterval(checkConnection);
|
|
66
|
+
reject(/* @__PURE__ */ new Error("Connection lost during WebRTC handshake"));
|
|
67
|
+
} else if (Date.now() >= deadline) {
|
|
68
|
+
clearInterval(checkConnection);
|
|
69
|
+
reject(/* @__PURE__ */ new Error("Connection timeout"));
|
|
70
|
+
}
|
|
71
|
+
}, 100);
|
|
72
|
+
connectAbort.catch(() => clearInterval(checkConnection));
|
|
73
|
+
}), connectAbort]);
|
|
74
|
+
} finally {
|
|
75
|
+
this.connectionReject = null;
|
|
76
|
+
}
|
|
59
77
|
}
|
|
60
78
|
async handleSignalingMessage(msg) {
|
|
61
79
|
try {
|
|
@@ -76,6 +94,10 @@ var WebRTCConnection = class {
|
|
|
76
94
|
this.websocketMessagesEmitter.emit("promptAck", msg);
|
|
77
95
|
return;
|
|
78
96
|
}
|
|
97
|
+
if (msg.type === "generation_started") {
|
|
98
|
+
this.setState("generating");
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
79
101
|
if (!this.pc) return;
|
|
80
102
|
switch (msg.type) {
|
|
81
103
|
case "ready": {
|
|
@@ -121,10 +143,16 @@ var WebRTCConnection = class {
|
|
|
121
143
|
} catch (error) {
|
|
122
144
|
console.error("[WebRTC] Error:", error);
|
|
123
145
|
this.callbacks.onError?.(error);
|
|
146
|
+
this.connectionReject?.(error);
|
|
124
147
|
}
|
|
125
148
|
}
|
|
126
149
|
send(message) {
|
|
127
|
-
if (this.ws?.readyState === WebSocket.OPEN)
|
|
150
|
+
if (this.ws?.readyState === WebSocket.OPEN) {
|
|
151
|
+
this.ws.send(JSON.stringify(message));
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
console.warn("[WebRTC] Message dropped: WebSocket is not open");
|
|
155
|
+
return false;
|
|
128
156
|
}
|
|
129
157
|
async sendAvatarImage(imageBase64) {
|
|
130
158
|
return this.setImageBase64(imageBase64);
|
|
@@ -154,7 +182,11 @@ var WebRTCConnection = class {
|
|
|
154
182
|
};
|
|
155
183
|
if (options?.prompt !== void 0) message.prompt = options.prompt;
|
|
156
184
|
if (options?.enhance !== void 0) message.enhance_prompt = options.enhance;
|
|
157
|
-
this.send(message)
|
|
185
|
+
if (!this.send(message)) {
|
|
186
|
+
clearTimeout(timeoutId);
|
|
187
|
+
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
188
|
+
reject(/* @__PURE__ */ new Error("WebSocket is not open"));
|
|
189
|
+
}
|
|
158
190
|
});
|
|
159
191
|
}
|
|
160
192
|
/**
|
|
@@ -175,11 +207,15 @@ var WebRTCConnection = class {
|
|
|
175
207
|
}
|
|
176
208
|
};
|
|
177
209
|
this.websocketMessagesEmitter.on("promptAck", listener);
|
|
178
|
-
this.send({
|
|
210
|
+
if (!this.send({
|
|
179
211
|
type: "prompt",
|
|
180
212
|
prompt: prompt.text,
|
|
181
213
|
enhance_prompt: prompt.enhance ?? true
|
|
182
|
-
})
|
|
214
|
+
})) {
|
|
215
|
+
clearTimeout(timeoutId);
|
|
216
|
+
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
217
|
+
reject(/* @__PURE__ */ new Error("WebSocket is not open"));
|
|
218
|
+
}
|
|
183
219
|
});
|
|
184
220
|
}
|
|
185
221
|
setState(state) {
|
|
@@ -196,13 +232,14 @@ var WebRTCConnection = class {
|
|
|
196
232
|
});
|
|
197
233
|
this.pc.close();
|
|
198
234
|
}
|
|
199
|
-
const iceServers = ICE_SERVERS;
|
|
235
|
+
const iceServers = [...ICE_SERVERS];
|
|
200
236
|
if (turnConfig) iceServers.push({
|
|
201
237
|
urls: turnConfig.server_url,
|
|
202
238
|
credential: turnConfig.credential,
|
|
203
239
|
username: turnConfig.username
|
|
204
240
|
});
|
|
205
241
|
this.pc = new RTCPeerConnection({ iceServers });
|
|
242
|
+
this.setState("connecting");
|
|
206
243
|
if (this.callbacks.isAvatarLive) this.pc.addTransceiver("video", { direction: "recvonly" });
|
|
207
244
|
this.localStream.getTracks().forEach((track) => {
|
|
208
245
|
if (this.pc && this.localStream) this.pc.addTrack(track, this.localStream);
|
|
@@ -219,9 +256,16 @@ var WebRTCConnection = class {
|
|
|
219
256
|
this.pc.onconnectionstatechange = () => {
|
|
220
257
|
if (!this.pc) return;
|
|
221
258
|
const s = this.pc.connectionState;
|
|
222
|
-
|
|
259
|
+
const nextState = s === "connected" ? "connected" : ["connecting", "new"].includes(s) ? "connecting" : "disconnected";
|
|
260
|
+
if (this.state === "generating" && nextState !== "disconnected") return;
|
|
261
|
+
this.setState(nextState);
|
|
262
|
+
};
|
|
263
|
+
this.pc.oniceconnectionstatechange = () => {
|
|
264
|
+
if (this.pc?.iceConnectionState === "failed") {
|
|
265
|
+
this.setState("disconnected");
|
|
266
|
+
this.callbacks.onError?.(/* @__PURE__ */ new Error("ICE connection failed"));
|
|
267
|
+
}
|
|
223
268
|
};
|
|
224
|
-
this.pc.oniceconnectionstatechange = () => {};
|
|
225
269
|
this.handleSignalingMessage({ type: "ready" });
|
|
226
270
|
}
|
|
227
271
|
cleanup() {
|
|
@@ -232,8 +276,12 @@ var WebRTCConnection = class {
|
|
|
232
276
|
this.localStream = null;
|
|
233
277
|
this.setState("disconnected");
|
|
234
278
|
}
|
|
235
|
-
|
|
279
|
+
applyCodecPreference(preferredCodecName) {
|
|
236
280
|
if (!this.pc) return;
|
|
281
|
+
if (typeof RTCRtpSender === "undefined" || typeof RTCRtpSender.getCapabilities !== "function") {
|
|
282
|
+
console.warn("RTCRtpSender capabilities are not available in this environment.");
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
237
285
|
const videoTransceiver = this.pc.getTransceivers().find((r) => r.sender.track?.kind === "video" || r.receiver.track?.kind === "video");
|
|
238
286
|
if (!videoTransceiver) {
|
|
239
287
|
console.error("Could not find video transceiver. Ensure track is added to peer connection.");
|
|
@@ -255,12 +303,17 @@ var WebRTCConnection = class {
|
|
|
255
303
|
console.warn("No video codecs found to set preferences for.");
|
|
256
304
|
return;
|
|
257
305
|
}
|
|
258
|
-
|
|
306
|
+
try {
|
|
307
|
+
videoTransceiver.setCodecPreferences(orderedCodecs);
|
|
308
|
+
} catch {
|
|
309
|
+
console.warn("[WebRTC] setCodecPreferences not supported, skipping codec preference.");
|
|
310
|
+
}
|
|
259
311
|
}
|
|
260
312
|
modifyVP8Bitrate(offer) {
|
|
261
313
|
if (!offer.sdp) return;
|
|
262
314
|
const minBitrateInKbps = this.callbacks.vp8MinBitrate;
|
|
263
315
|
const startBitrateInKbps = this.callbacks.vp8StartBitrate;
|
|
316
|
+
if (minBitrateInKbps === void 0 || startBitrateInKbps === void 0) return;
|
|
264
317
|
if (minBitrateInKbps === 0 && startBitrateInKbps === 0) return;
|
|
265
318
|
const bitrateParams = `x-google-min-bitrate=${minBitrateInKbps};x-google-start-bitrate=${startBitrateInKbps}`;
|
|
266
319
|
const sdpLines = offer.sdp.split("\r\n");
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { WebRTCConnection } from "./webrtc-connection.js";
|
|
2
|
-
import pRetry from "p-retry";
|
|
2
|
+
import pRetry, { AbortError } from "p-retry";
|
|
3
3
|
|
|
4
4
|
//#region src/realtime/webrtc-manager.ts
|
|
5
5
|
const PERMANENT_ERRORS = [
|
|
@@ -10,14 +10,27 @@ const PERMANENT_ERRORS = [
|
|
|
10
10
|
"invalid api key",
|
|
11
11
|
"unauthorized"
|
|
12
12
|
];
|
|
13
|
+
const CONNECTION_TIMEOUT = 6e4 * 5;
|
|
14
|
+
const RETRY_OPTIONS = {
|
|
15
|
+
retries: 5,
|
|
16
|
+
factor: 2,
|
|
17
|
+
minTimeout: 1e3,
|
|
18
|
+
maxTimeout: 1e4
|
|
19
|
+
};
|
|
13
20
|
var WebRTCManager = class {
|
|
14
21
|
connection;
|
|
15
22
|
config;
|
|
23
|
+
localStream = null;
|
|
24
|
+
managerState = "disconnected";
|
|
25
|
+
hasConnected = false;
|
|
26
|
+
isReconnecting = false;
|
|
27
|
+
intentionalDisconnect = false;
|
|
28
|
+
reconnectGeneration = 0;
|
|
16
29
|
constructor(config) {
|
|
17
30
|
this.config = config;
|
|
18
31
|
this.connection = new WebRTCConnection({
|
|
19
32
|
onRemoteStream: config.onRemoteStream,
|
|
20
|
-
onStateChange:
|
|
33
|
+
onStateChange: (state) => this.handleConnectionStateChange(state),
|
|
21
34
|
onError: config.onError,
|
|
22
35
|
customizeOffer: config.customizeOffer,
|
|
23
36
|
vp8MinBitrate: config.vp8MinBitrate,
|
|
@@ -27,36 +40,107 @@ var WebRTCManager = class {
|
|
|
27
40
|
initialPrompt: config.initialPrompt
|
|
28
41
|
});
|
|
29
42
|
}
|
|
43
|
+
emitState(state) {
|
|
44
|
+
if (this.managerState !== state) {
|
|
45
|
+
this.managerState = state;
|
|
46
|
+
if (state === "connected" || state === "generating") this.hasConnected = true;
|
|
47
|
+
this.config.onConnectionStateChange?.(state);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
handleConnectionStateChange(state) {
|
|
51
|
+
if (this.intentionalDisconnect) {
|
|
52
|
+
this.emitState("disconnected");
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
if (this.isReconnecting) {
|
|
56
|
+
if (state === "connected" || state === "generating") {
|
|
57
|
+
this.isReconnecting = false;
|
|
58
|
+
this.emitState(state);
|
|
59
|
+
}
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
if (state === "disconnected" && !this.intentionalDisconnect && this.hasConnected) {
|
|
63
|
+
this.reconnect();
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
this.emitState(state);
|
|
67
|
+
}
|
|
68
|
+
async reconnect() {
|
|
69
|
+
if (this.isReconnecting || this.intentionalDisconnect || !this.localStream) return;
|
|
70
|
+
const reconnectGeneration = ++this.reconnectGeneration;
|
|
71
|
+
this.isReconnecting = true;
|
|
72
|
+
this.emitState("reconnecting");
|
|
73
|
+
try {
|
|
74
|
+
await pRetry(async () => {
|
|
75
|
+
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) throw new AbortError("Reconnect cancelled");
|
|
76
|
+
const stream = this.localStream;
|
|
77
|
+
if (!stream) throw new AbortError("Reconnect cancelled: no local stream");
|
|
78
|
+
this.connection.cleanup();
|
|
79
|
+
await this.connection.connect(this.config.webrtcUrl, stream, CONNECTION_TIMEOUT, this.config.integration);
|
|
80
|
+
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) {
|
|
81
|
+
this.connection.cleanup();
|
|
82
|
+
throw new AbortError("Reconnect cancelled");
|
|
83
|
+
}
|
|
84
|
+
}, {
|
|
85
|
+
...RETRY_OPTIONS,
|
|
86
|
+
onFailedAttempt: (error) => {
|
|
87
|
+
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) return;
|
|
88
|
+
console.error(`[WebRTC] Reconnect attempt failed: ${error.message}`);
|
|
89
|
+
this.connection.cleanup();
|
|
90
|
+
},
|
|
91
|
+
shouldRetry: (error) => {
|
|
92
|
+
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) return false;
|
|
93
|
+
const msg = error.message.toLowerCase();
|
|
94
|
+
return !PERMANENT_ERRORS.some((err) => msg.includes(err));
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
} catch (error) {
|
|
98
|
+
this.isReconnecting = false;
|
|
99
|
+
if (this.intentionalDisconnect || reconnectGeneration !== this.reconnectGeneration) return;
|
|
100
|
+
this.emitState("disconnected");
|
|
101
|
+
this.config.onError?.(error instanceof Error ? error : new Error(String(error)));
|
|
102
|
+
}
|
|
103
|
+
}
|
|
30
104
|
async connect(localStream) {
|
|
105
|
+
this.localStream = localStream;
|
|
106
|
+
this.intentionalDisconnect = false;
|
|
107
|
+
this.hasConnected = false;
|
|
108
|
+
this.isReconnecting = false;
|
|
109
|
+
this.reconnectGeneration += 1;
|
|
110
|
+
this.emitState("connecting");
|
|
31
111
|
return pRetry(async () => {
|
|
32
|
-
|
|
112
|
+
if (this.intentionalDisconnect) throw new AbortError("Connect cancelled");
|
|
113
|
+
await this.connection.connect(this.config.webrtcUrl, localStream, CONNECTION_TIMEOUT, this.config.integration);
|
|
33
114
|
return true;
|
|
34
115
|
}, {
|
|
35
|
-
|
|
36
|
-
factor: 2,
|
|
37
|
-
minTimeout: 1e3,
|
|
38
|
-
maxTimeout: 1e4,
|
|
116
|
+
...RETRY_OPTIONS,
|
|
39
117
|
onFailedAttempt: (error) => {
|
|
40
118
|
console.error(`[WebRTC] Failed to connect: ${error.message}`);
|
|
41
119
|
this.connection.cleanup();
|
|
42
120
|
},
|
|
43
121
|
shouldRetry: (error) => {
|
|
122
|
+
if (this.intentionalDisconnect) return false;
|
|
44
123
|
const msg = error.message.toLowerCase();
|
|
45
124
|
return !PERMANENT_ERRORS.some((err) => msg.includes(err));
|
|
46
125
|
}
|
|
47
126
|
});
|
|
48
127
|
}
|
|
49
128
|
sendMessage(message) {
|
|
50
|
-
this.connection.send(message);
|
|
129
|
+
return this.connection.send(message);
|
|
51
130
|
}
|
|
52
131
|
cleanup() {
|
|
132
|
+
this.intentionalDisconnect = true;
|
|
133
|
+
this.isReconnecting = false;
|
|
134
|
+
this.reconnectGeneration += 1;
|
|
53
135
|
this.connection.cleanup();
|
|
136
|
+
this.localStream = null;
|
|
137
|
+
this.emitState("disconnected");
|
|
54
138
|
}
|
|
55
139
|
isConnected() {
|
|
56
|
-
return this.
|
|
140
|
+
return this.managerState === "connected" || this.managerState === "generating";
|
|
57
141
|
}
|
|
58
142
|
getConnectionState() {
|
|
59
|
-
return this.
|
|
143
|
+
return this.managerState;
|
|
60
144
|
}
|
|
61
145
|
getWebsocketMessageEmitter() {
|
|
62
146
|
return this.connection.websocketMessagesEmitter;
|