@decartai/sdk 0.0.68 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -9
- package/dist/index.d.ts +6 -4
- package/dist/index.js +43 -28
- package/dist/process/client.js +1 -3
- package/dist/process/request.js +1 -3
- package/dist/queue/client.js +1 -3
- package/dist/queue/polling.js +1 -2
- package/dist/queue/request.js +1 -3
- package/dist/realtime/client.d.ts +17 -11
- package/dist/realtime/client.js +71 -155
- package/dist/realtime/config-realtime.js +49 -0
- package/dist/realtime/event-buffer.js +1 -3
- package/dist/realtime/initial-state-gate.js +21 -0
- package/dist/realtime/media-channel.js +82 -0
- package/dist/realtime/methods.js +12 -42
- package/dist/realtime/mirror-stream.js +1 -2
- package/dist/realtime/observability/diagnostics.d.ts +14 -53
- package/dist/realtime/observability/livekit-stats-provider.js +25 -0
- package/dist/realtime/observability/realtime-observability.js +70 -6
- package/dist/realtime/observability/telemetry-reporter.js +9 -28
- package/dist/realtime/observability/webrtc-stats.d.ts +5 -4
- package/dist/realtime/observability/webrtc-stats.js +3 -5
- package/dist/realtime/signaling-channel.js +286 -0
- package/dist/realtime/stream-session.js +252 -0
- package/dist/realtime/subscribe-client.d.ts +2 -3
- package/dist/realtime/subscribe-client.js +115 -11
- package/dist/realtime/types.d.ts +25 -1
- package/dist/shared/model.d.ts +11 -1
- package/dist/shared/model.js +51 -14
- package/dist/shared/request.js +1 -3
- package/dist/shared/types.js +1 -3
- package/dist/tokens/client.js +1 -3
- package/dist/utils/env.js +1 -2
- package/dist/utils/errors.js +1 -2
- package/dist/utils/logger.js +1 -2
- package/dist/utils/media.js +43 -0
- package/dist/utils/platform.js +13 -0
- package/dist/utils/user-agent.js +1 -3
- package/dist/version.js +1 -2
- package/package.json +2 -1
- package/dist/realtime/webrtc-connection.js +0 -500
- package/dist/realtime/webrtc-manager.js +0 -210
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import { buildUserAgent } from "../utils/user-agent.js";
|
|
2
|
+
import { createConsoleLogger } from "../utils/logger.js";
|
|
3
|
+
import { REALTIME_CONFIG } from "./config-realtime.js";
|
|
4
|
+
import mitt from "mitt";
|
|
5
|
+
//#region src/realtime/signaling-channel.ts
|
|
6
|
+
var SignalingChannel = class {
|
|
7
|
+
ws = null;
|
|
8
|
+
events = mitt();
|
|
9
|
+
pendingAcks = [];
|
|
10
|
+
pendingRoomInfo = null;
|
|
11
|
+
connected = false;
|
|
12
|
+
closing = false;
|
|
13
|
+
logger;
|
|
14
|
+
constructor(config) {
|
|
15
|
+
this.config = config;
|
|
16
|
+
this.logger = config.logger ?? createConsoleLogger("warn");
|
|
17
|
+
}
|
|
18
|
+
on(event, handler) {
|
|
19
|
+
this.events.on(event, handler);
|
|
20
|
+
}
|
|
21
|
+
off(event, handler) {
|
|
22
|
+
this.events.off(event, handler);
|
|
23
|
+
}
|
|
24
|
+
async openAndJoin(opts = {}) {
|
|
25
|
+
const connectTimeout = opts.connectTimeout ?? REALTIME_CONFIG.signaling.connectTimeoutMs;
|
|
26
|
+
const handshakeTimeout = opts.handshakeTimeout ?? REALTIME_CONFIG.signaling.handshakeTimeoutMs;
|
|
27
|
+
this.config.observability?.startPhase("websocket-open");
|
|
28
|
+
await this.openSocket(connectTimeout);
|
|
29
|
+
this.config.observability?.endPhase("websocket-open", { success: true });
|
|
30
|
+
this.config.observability?.startPhase("room-join");
|
|
31
|
+
const roomInfoWait = this.waitForRoomInfo(handshakeTimeout);
|
|
32
|
+
if (!this.writeMessage({ type: "livekit_join" })) {
|
|
33
|
+
roomInfoWait.cancel();
|
|
34
|
+
throw new Error("WebSocket is not open");
|
|
35
|
+
}
|
|
36
|
+
let roomInfo;
|
|
37
|
+
try {
|
|
38
|
+
roomInfo = await roomInfoWait.promise;
|
|
39
|
+
} catch (error) {
|
|
40
|
+
this.rejectAllPending(error instanceof Error ? error : new Error(String(error)));
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
this.config.observability?.endPhase("room-join", { success: true });
|
|
44
|
+
this.connected = true;
|
|
45
|
+
const initialStateAck = this.sendInitialStateTracked(opts.initialState);
|
|
46
|
+
initialStateAck.catch(() => {});
|
|
47
|
+
return {
|
|
48
|
+
roomInfo,
|
|
49
|
+
initialStateAck
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
async sendInitialStateTracked(initialState) {
|
|
53
|
+
if (!initialState) return;
|
|
54
|
+
this.config.observability?.startPhase("initial-state-handshake");
|
|
55
|
+
await this.sendInitialState(initialState);
|
|
56
|
+
this.config.observability?.endPhase("initial-state-handshake", { success: true });
|
|
57
|
+
}
|
|
58
|
+
close() {
|
|
59
|
+
this.closing = true;
|
|
60
|
+
this.connected = false;
|
|
61
|
+
const ws = this.ws;
|
|
62
|
+
this.ws = null;
|
|
63
|
+
if (ws) try {
|
|
64
|
+
ws.close();
|
|
65
|
+
} catch {}
|
|
66
|
+
this.rejectPendingRoomInfo(/* @__PURE__ */ new Error("Control channel closed"));
|
|
67
|
+
this.rejectAllPending(/* @__PURE__ */ new Error("Control channel closed"));
|
|
68
|
+
}
|
|
69
|
+
async sendPrompt(text, opts = {}) {
|
|
70
|
+
const ack = await this.request({
|
|
71
|
+
message: {
|
|
72
|
+
type: "prompt",
|
|
73
|
+
prompt: text,
|
|
74
|
+
enhance_prompt: opts.enhance ?? true
|
|
75
|
+
},
|
|
76
|
+
matchAck: (msg) => msg.type === "prompt_ack" && msg.prompt === text,
|
|
77
|
+
timeoutMs: opts.timeout ?? REALTIME_CONFIG.signaling.requestTimeoutMs,
|
|
78
|
+
label: "Prompt send"
|
|
79
|
+
});
|
|
80
|
+
if (!ack.success) throw new Error(ack.error ?? "Failed to send prompt");
|
|
81
|
+
}
|
|
82
|
+
async setImage(image, opts = {}) {
|
|
83
|
+
const message = {
|
|
84
|
+
type: "set_image",
|
|
85
|
+
image_data: image
|
|
86
|
+
};
|
|
87
|
+
if (opts.prompt !== void 0) message.prompt = opts.prompt;
|
|
88
|
+
if (opts.enhance !== void 0) message.enhance_prompt = opts.enhance;
|
|
89
|
+
const ack = await this.request({
|
|
90
|
+
message,
|
|
91
|
+
matchAck: (msg) => msg.type === "set_image_ack",
|
|
92
|
+
timeoutMs: opts.timeout ?? REALTIME_CONFIG.signaling.requestTimeoutMs,
|
|
93
|
+
label: "Image send"
|
|
94
|
+
});
|
|
95
|
+
if (!ack.success) throw new Error(ack.error ?? "Failed to send image");
|
|
96
|
+
}
|
|
97
|
+
async openSocket(timeout) {
|
|
98
|
+
const userAgent = encodeURIComponent(buildUserAgent(this.config.integration));
|
|
99
|
+
const separator = this.config.url.includes("?") ? "&" : "?";
|
|
100
|
+
const wsUrl = `${this.config.url}${separator}user_agent=${userAgent}`;
|
|
101
|
+
this.closing = false;
|
|
102
|
+
await new Promise((resolve, reject) => {
|
|
103
|
+
const timer = setTimeout(() => reject(/* @__PURE__ */ new Error(`WebSocket open timeout (${timeout}ms)`)), timeout);
|
|
104
|
+
const ws = new WebSocket(wsUrl);
|
|
105
|
+
this.ws = ws;
|
|
106
|
+
ws.onopen = () => {
|
|
107
|
+
clearTimeout(timer);
|
|
108
|
+
resolve();
|
|
109
|
+
};
|
|
110
|
+
ws.onclose = (e) => {
|
|
111
|
+
clearTimeout(timer);
|
|
112
|
+
const wasConnected = this.connected;
|
|
113
|
+
const pendingCount = this.pendingAcks.length;
|
|
114
|
+
this.connected = false;
|
|
115
|
+
this.ws = null;
|
|
116
|
+
this.logger.warn("signaling: websocket closed", {
|
|
117
|
+
code: e.code,
|
|
118
|
+
reason: e.reason,
|
|
119
|
+
wasConnected,
|
|
120
|
+
closing: this.closing,
|
|
121
|
+
pendingAcks: pendingCount
|
|
122
|
+
});
|
|
123
|
+
const error = /* @__PURE__ */ new Error(`WebSocket closed: ${e.code} ${e.reason}`);
|
|
124
|
+
this.rejectPendingRoomInfo(error);
|
|
125
|
+
this.rejectAllPending(error);
|
|
126
|
+
if (wasConnected || this.closing) this.events.emit("closed", {
|
|
127
|
+
code: e.code,
|
|
128
|
+
reason: e.reason
|
|
129
|
+
});
|
|
130
|
+
else reject(error);
|
|
131
|
+
};
|
|
132
|
+
ws.onerror = () => {};
|
|
133
|
+
ws.onmessage = (e) => {
|
|
134
|
+
try {
|
|
135
|
+
this.handleMessage(JSON.parse(e.data));
|
|
136
|
+
} catch {}
|
|
137
|
+
};
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
waitForRoomInfo(timeoutMs) {
|
|
141
|
+
let cleanup = () => {};
|
|
142
|
+
return {
|
|
143
|
+
promise: new Promise((resolve, reject) => {
|
|
144
|
+
let timer = setTimeout(() => {
|
|
145
|
+
cleanup();
|
|
146
|
+
this.logger.warn("signaling: livekit_room_info timeout", { timeoutMs });
|
|
147
|
+
reject(/* @__PURE__ */ new Error(`livekit_room_info timeout (${timeoutMs}ms)`));
|
|
148
|
+
}, timeoutMs);
|
|
149
|
+
const pendingRoomInfo = {
|
|
150
|
+
resolve: (info) => {
|
|
151
|
+
cleanup();
|
|
152
|
+
resolve(info);
|
|
153
|
+
},
|
|
154
|
+
reject: (err) => {
|
|
155
|
+
cleanup();
|
|
156
|
+
reject(err);
|
|
157
|
+
},
|
|
158
|
+
cancel: () => {
|
|
159
|
+
cleanup();
|
|
160
|
+
},
|
|
161
|
+
pauseTimeout: () => {
|
|
162
|
+
if (timer) {
|
|
163
|
+
clearTimeout(timer);
|
|
164
|
+
timer = null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
cleanup = () => {
|
|
169
|
+
if (timer) {
|
|
170
|
+
clearTimeout(timer);
|
|
171
|
+
timer = null;
|
|
172
|
+
}
|
|
173
|
+
if (this.pendingRoomInfo === pendingRoomInfo) this.pendingRoomInfo = null;
|
|
174
|
+
};
|
|
175
|
+
this.pendingRoomInfo = pendingRoomInfo;
|
|
176
|
+
}),
|
|
177
|
+
cancel: cleanup
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
async sendInitialState(initialState) {
|
|
181
|
+
if (!initialState) return;
|
|
182
|
+
if (initialState.image !== void 0) {
|
|
183
|
+
await this.setImage(initialState.image, {
|
|
184
|
+
prompt: initialState.prompt,
|
|
185
|
+
enhance: initialState.enhance
|
|
186
|
+
});
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
if (initialState.prompt !== void 0 && initialState.prompt !== null) await this.sendPrompt(initialState.prompt, { enhance: initialState.enhance });
|
|
190
|
+
}
|
|
191
|
+
async request({ message, matchAck, timeoutMs, label }) {
|
|
192
|
+
return new Promise((resolve, reject) => {
|
|
193
|
+
const timer = setTimeout(() => {
|
|
194
|
+
cleanup();
|
|
195
|
+
this.logger.warn("signaling: ack timed out", {
|
|
196
|
+
label,
|
|
197
|
+
timeoutMs
|
|
198
|
+
});
|
|
199
|
+
reject(/* @__PURE__ */ new Error(`${label} timed out`));
|
|
200
|
+
}, timeoutMs);
|
|
201
|
+
const entry = {
|
|
202
|
+
matches: matchAck,
|
|
203
|
+
onMatch: (msg) => {
|
|
204
|
+
cleanup();
|
|
205
|
+
resolve(msg);
|
|
206
|
+
},
|
|
207
|
+
reject: (err) => {
|
|
208
|
+
cleanup();
|
|
209
|
+
reject(err);
|
|
210
|
+
}
|
|
211
|
+
};
|
|
212
|
+
const cleanup = () => {
|
|
213
|
+
clearTimeout(timer);
|
|
214
|
+
this.pendingAcks = this.pendingAcks.filter((e) => e !== entry);
|
|
215
|
+
};
|
|
216
|
+
this.pendingAcks.push(entry);
|
|
217
|
+
if (!this.writeMessage(message)) {
|
|
218
|
+
cleanup();
|
|
219
|
+
reject(/* @__PURE__ */ new Error("WebSocket is not open"));
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
writeMessage(message) {
|
|
224
|
+
if (this.ws?.readyState !== WebSocket.OPEN) return false;
|
|
225
|
+
this.ws.send(JSON.stringify(message));
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
handleMessage(msg) {
|
|
229
|
+
for (const ack of [...this.pendingAcks]) if (ack.matches(msg)) {
|
|
230
|
+
ack.onMatch(msg);
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
switch (msg.type) {
|
|
234
|
+
case "livekit_room_info":
|
|
235
|
+
this.resolvePendingRoomInfo({
|
|
236
|
+
livekitUrl: msg.livekit_url,
|
|
237
|
+
token: msg.token,
|
|
238
|
+
roomName: msg.room_name,
|
|
239
|
+
sessionId: msg.session_id
|
|
240
|
+
});
|
|
241
|
+
break;
|
|
242
|
+
case "queue_position":
|
|
243
|
+
this.pendingRoomInfo?.pauseTimeout();
|
|
244
|
+
this.events.emit("queuePosition", {
|
|
245
|
+
position: msg.position,
|
|
246
|
+
queueSize: msg.queue_size
|
|
247
|
+
});
|
|
248
|
+
break;
|
|
249
|
+
case "generation_tick":
|
|
250
|
+
this.events.emit("generationTick", { seconds: msg.seconds });
|
|
251
|
+
break;
|
|
252
|
+
case "generation_ended":
|
|
253
|
+
this.events.emit("generationEnded", {
|
|
254
|
+
seconds: msg.seconds,
|
|
255
|
+
reason: msg.reason
|
|
256
|
+
});
|
|
257
|
+
break;
|
|
258
|
+
case "error": {
|
|
259
|
+
const error = new Error(msg.error);
|
|
260
|
+
error.source = "server";
|
|
261
|
+
this.logger.error("signaling: server error received", { error: msg.error });
|
|
262
|
+
this.events.emit("serverError", error);
|
|
263
|
+
this.rejectPendingRoomInfo(error);
|
|
264
|
+
this.rejectAllPending(error);
|
|
265
|
+
break;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
resolvePendingRoomInfo(info) {
|
|
270
|
+
const pending = this.pendingRoomInfo;
|
|
271
|
+
if (!pending) return;
|
|
272
|
+
pending.resolve(info);
|
|
273
|
+
}
|
|
274
|
+
rejectPendingRoomInfo(error) {
|
|
275
|
+
const pending = this.pendingRoomInfo;
|
|
276
|
+
if (!pending) return;
|
|
277
|
+
pending.reject(error);
|
|
278
|
+
}
|
|
279
|
+
rejectAllPending(error) {
|
|
280
|
+
const pending = this.pendingAcks;
|
|
281
|
+
this.pendingAcks = [];
|
|
282
|
+
for (const entry of pending) entry.reject(error);
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
//#endregion
|
|
286
|
+
export { SignalingChannel };
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { createConsoleLogger } from "../utils/logger.js";
|
|
2
|
+
import { REALTIME_CONFIG } from "./config-realtime.js";
|
|
3
|
+
import { InitialStateGate } from "./initial-state-gate.js";
|
|
4
|
+
import { MediaChannel } from "./media-channel.js";
|
|
5
|
+
import { SignalingChannel } from "./signaling-channel.js";
|
|
6
|
+
import mitt from "mitt";
|
|
7
|
+
import pRetry, { AbortError } from "p-retry";
|
|
8
|
+
//#region src/realtime/stream-session.ts
|
|
9
|
+
function encodeSubscribeToken(roomName) {
|
|
10
|
+
return btoa(JSON.stringify({ room_name: roomName }));
|
|
11
|
+
}
|
|
12
|
+
function getInitialImageSizeKb(image) {
|
|
13
|
+
if (!image) return null;
|
|
14
|
+
const commaIdx = image.indexOf(",");
|
|
15
|
+
const base64 = commaIdx >= 0 && image.startsWith("data:") ? image.slice(commaIdx + 1) : image;
|
|
16
|
+
const padding = base64.endsWith("==") ? 2 : base64.endsWith("=") ? 1 : 0;
|
|
17
|
+
const bytes = Math.floor(base64.length * 3 / 4) - padding;
|
|
18
|
+
return Math.max(0, Math.round(bytes / 1024));
|
|
19
|
+
}
|
|
20
|
+
var StreamSession = class {
|
|
21
|
+
signaling;
|
|
22
|
+
media;
|
|
23
|
+
events = mitt();
|
|
24
|
+
state = "disconnected";
|
|
25
|
+
queue = null;
|
|
26
|
+
disposed = false;
|
|
27
|
+
currentAttempt = 0;
|
|
28
|
+
initialStateGate = new InitialStateGate();
|
|
29
|
+
logger;
|
|
30
|
+
constructor(config) {
|
|
31
|
+
this.config = config;
|
|
32
|
+
this.logger = config.logger ?? createConsoleLogger("warn");
|
|
33
|
+
this.createTransport();
|
|
34
|
+
}
|
|
35
|
+
on(event, handler) {
|
|
36
|
+
this.events.on(event, handler);
|
|
37
|
+
}
|
|
38
|
+
off(event, handler) {
|
|
39
|
+
this.events.off(event, handler);
|
|
40
|
+
}
|
|
41
|
+
getStatus() {
|
|
42
|
+
return {
|
|
43
|
+
connection: this.state,
|
|
44
|
+
queue: this.queue
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
getConnectionState() {
|
|
48
|
+
return this.state;
|
|
49
|
+
}
|
|
50
|
+
isConnected() {
|
|
51
|
+
return this.state === "connected" || this.state === "generating";
|
|
52
|
+
}
|
|
53
|
+
async connect() {
|
|
54
|
+
this.disposed = false;
|
|
55
|
+
const attempt = ++this.currentAttempt;
|
|
56
|
+
this.setState("connecting");
|
|
57
|
+
this.logger.info("realtime connect: starting", { attemptCycle: attempt });
|
|
58
|
+
try {
|
|
59
|
+
await pRetry(() => this.runOneConnect(attempt), this.retryOptionsFor(attempt));
|
|
60
|
+
} catch (error) {
|
|
61
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
62
|
+
this.logger.error("realtime connect: exhausted all retries", { error: message });
|
|
63
|
+
if (this.currentAttempt === attempt && !this.disposed) this.setState("disconnected");
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async sendPrompt(text, opts) {
|
|
68
|
+
this.assertConnected();
|
|
69
|
+
return this.signaling.sendPrompt(text, opts);
|
|
70
|
+
}
|
|
71
|
+
async setImage(image, opts) {
|
|
72
|
+
this.assertConnected();
|
|
73
|
+
return this.signaling.setImage(image, opts);
|
|
74
|
+
}
|
|
75
|
+
disconnect() {
|
|
76
|
+
this.disposed = true;
|
|
77
|
+
this.tearDown();
|
|
78
|
+
this.setState("disconnected");
|
|
79
|
+
}
|
|
80
|
+
assertConnected() {
|
|
81
|
+
if (!this.isConnected()) throw new Error(`Cannot send message: connection is ${this.state}`);
|
|
82
|
+
}
|
|
83
|
+
retryOptionsFor(attempt) {
|
|
84
|
+
return {
|
|
85
|
+
...REALTIME_CONFIG.session.retry,
|
|
86
|
+
onFailedAttempt: (_error) => {
|
|
87
|
+
this.tearDown();
|
|
88
|
+
},
|
|
89
|
+
shouldRetry: (error) => {
|
|
90
|
+
if (this.disposed || this.currentAttempt !== attempt) return false;
|
|
91
|
+
const msg = error.message.toLowerCase();
|
|
92
|
+
const permanent = REALTIME_CONFIG.session.permanentErrorSubstrings.some((err) => msg.includes(err));
|
|
93
|
+
if (permanent) this.logger.error("realtime connect: permanent error, not retrying", { error: error.message });
|
|
94
|
+
return !permanent;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
async runOneConnect(attempt) {
|
|
99
|
+
if (this.disposed || this.currentAttempt !== attempt) throw new AbortError("Stale connect attempt");
|
|
100
|
+
try {
|
|
101
|
+
this.resetHandshakeState();
|
|
102
|
+
const initialState = this.getInitialState();
|
|
103
|
+
this.config.observability?.beginConnectionBreakdown(attempt, getInitialImageSizeKb(initialState?.image));
|
|
104
|
+
const gateAttempt = this.initialStateGate.startAttempt(initialState);
|
|
105
|
+
const { roomInfo, initialStateAck } = await this.signaling.openAndJoin({
|
|
106
|
+
connectTimeout: REALTIME_CONFIG.session.connectionTimeoutMs,
|
|
107
|
+
initialState
|
|
108
|
+
});
|
|
109
|
+
if (this.disposed || this.currentAttempt !== attempt) {
|
|
110
|
+
this.tearDown();
|
|
111
|
+
throw new AbortError("Stale connect attempt");
|
|
112
|
+
}
|
|
113
|
+
this.queue = null;
|
|
114
|
+
try {
|
|
115
|
+
await this.media.connect({
|
|
116
|
+
url: roomInfo.livekitUrl,
|
|
117
|
+
token: roomInfo.token
|
|
118
|
+
});
|
|
119
|
+
if (!await gateAttempt.waitForReadiness(initialStateAck)) throw new AbortError("Stale connect attempt");
|
|
120
|
+
await this.media.publishLocalTracks();
|
|
121
|
+
} catch (error) {
|
|
122
|
+
this.tearDown();
|
|
123
|
+
throw error;
|
|
124
|
+
}
|
|
125
|
+
if (this.disposed || this.currentAttempt !== attempt) {
|
|
126
|
+
this.tearDown();
|
|
127
|
+
throw new AbortError("Stale connect attempt");
|
|
128
|
+
}
|
|
129
|
+
this.config.observability?.finishConnectionBreakdown({ success: true });
|
|
130
|
+
this.setState("connected");
|
|
131
|
+
this.events.emit("sessionStarted", {
|
|
132
|
+
sessionId: roomInfo.sessionId,
|
|
133
|
+
subscribeToken: encodeSubscribeToken(roomInfo.roomName)
|
|
134
|
+
});
|
|
135
|
+
} catch (error) {
|
|
136
|
+
this.config.observability?.finishConnectionBreakdown({
|
|
137
|
+
success: false,
|
|
138
|
+
error: error instanceof Error ? error.message : String(error)
|
|
139
|
+
});
|
|
140
|
+
throw error;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
getInitialState() {
|
|
144
|
+
if (this.config.initialImage !== void 0) return {
|
|
145
|
+
image: this.config.initialImage,
|
|
146
|
+
prompt: this.config.initialPrompt?.text,
|
|
147
|
+
enhance: this.config.initialPrompt?.enhance
|
|
148
|
+
};
|
|
149
|
+
if (this.config.initialPrompt) return {
|
|
150
|
+
prompt: this.config.initialPrompt.text,
|
|
151
|
+
enhance: this.config.initialPrompt.enhance
|
|
152
|
+
};
|
|
153
|
+
if (this.config.localStream) return {
|
|
154
|
+
image: null,
|
|
155
|
+
prompt: null
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
wireSignalingEvents() {
|
|
159
|
+
this.signaling.on("queuePosition", (qp) => {
|
|
160
|
+
this.queue = qp;
|
|
161
|
+
this.events.emit("queuePosition", qp);
|
|
162
|
+
});
|
|
163
|
+
this.signaling.on("generationTick", (e) => this.events.emit("generationTick", e));
|
|
164
|
+
this.signaling.on("generationEnded", (e) => this.events.emit("generationEnded", e));
|
|
165
|
+
this.signaling.on("serverError", (err) => this.events.emit("error", err));
|
|
166
|
+
this.signaling.on("closed", (info) => this.handleConnectionLoss({
|
|
167
|
+
source: "signaling",
|
|
168
|
+
...info
|
|
169
|
+
}));
|
|
170
|
+
}
|
|
171
|
+
wireMediaEvents() {
|
|
172
|
+
this.media.on("remoteStream", (stream) => this.events.emit("remoteStream", stream));
|
|
173
|
+
this.media.on("firstFrame", () => {
|
|
174
|
+
if (this.state === "connected") this.setState("generating");
|
|
175
|
+
});
|
|
176
|
+
this.media.on("disconnected", (info) => this.handleConnectionLoss({
|
|
177
|
+
source: "media",
|
|
178
|
+
reason: info.reason
|
|
179
|
+
}));
|
|
180
|
+
}
|
|
181
|
+
handleConnectionLoss(cause) {
|
|
182
|
+
if (this.disposed) return;
|
|
183
|
+
if (this.state !== "connected" && this.state !== "generating") {
|
|
184
|
+
this.logger.debug("connection loss ignored (not connected)", {
|
|
185
|
+
state: this.state,
|
|
186
|
+
...cause
|
|
187
|
+
});
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
this.logger.warn("realtime connection lost; scheduling reconnect", {
|
|
191
|
+
state: this.state,
|
|
192
|
+
...cause
|
|
193
|
+
});
|
|
194
|
+
this.scheduleReconnect();
|
|
195
|
+
}
|
|
196
|
+
scheduleReconnect() {
|
|
197
|
+
const attempt = ++this.currentAttempt;
|
|
198
|
+
this.setState("reconnecting");
|
|
199
|
+
pRetry(async () => {
|
|
200
|
+
if (this.disposed || this.currentAttempt !== attempt) throw new AbortError("Reconnect cancelled");
|
|
201
|
+
this.tearDown();
|
|
202
|
+
this.createTransport();
|
|
203
|
+
await this.runOneConnect(attempt);
|
|
204
|
+
}, this.retryOptionsFor(attempt)).then(() => {
|
|
205
|
+
if (this.disposed || this.currentAttempt !== attempt) return;
|
|
206
|
+
this.logger.info("realtime reconnect: succeeded");
|
|
207
|
+
}).catch((error) => {
|
|
208
|
+
if (this.disposed || this.currentAttempt !== attempt) return;
|
|
209
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
210
|
+
this.logger.error("realtime reconnect: failed permanently", { error: message });
|
|
211
|
+
this.tearDown();
|
|
212
|
+
this.setState("disconnected");
|
|
213
|
+
this.events.emit("error", error instanceof Error ? error : new Error(String(error)));
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
createTransport() {
|
|
217
|
+
this.signaling = new SignalingChannel({
|
|
218
|
+
url: this.config.url,
|
|
219
|
+
integration: this.config.integration,
|
|
220
|
+
logger: this.logger,
|
|
221
|
+
observability: this.config.observability
|
|
222
|
+
});
|
|
223
|
+
this.media = new MediaChannel({
|
|
224
|
+
observability: this.config.observability,
|
|
225
|
+
localStream: this.config.localStream,
|
|
226
|
+
logger: this.logger,
|
|
227
|
+
videoCodec: this.config.videoCodec
|
|
228
|
+
});
|
|
229
|
+
this.wireSignalingEvents();
|
|
230
|
+
this.wireMediaEvents();
|
|
231
|
+
}
|
|
232
|
+
tearDown() {
|
|
233
|
+
this.signaling.close();
|
|
234
|
+
this.media.disconnect();
|
|
235
|
+
this.initialStateGate.reset();
|
|
236
|
+
this.resetHandshakeState();
|
|
237
|
+
}
|
|
238
|
+
resetHandshakeState() {
|
|
239
|
+
this.queue = null;
|
|
240
|
+
}
|
|
241
|
+
setState(state) {
|
|
242
|
+
if (this.state === state) return;
|
|
243
|
+
this.logger.debug("realtime state change", {
|
|
244
|
+
from: this.state,
|
|
245
|
+
to: state
|
|
246
|
+
});
|
|
247
|
+
this.state = state;
|
|
248
|
+
this.events.emit("connectionChange", state);
|
|
249
|
+
}
|
|
250
|
+
};
|
|
251
|
+
//#endregion
|
|
252
|
+
export { StreamSession };
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { DecartSDKError } from "../utils/errors.js";
|
|
2
|
-
import { ConnectionState } from "./types.js";
|
|
3
2
|
import { DiagnosticEvent } from "./observability/diagnostics.js";
|
|
4
|
-
import {
|
|
3
|
+
import { ConnectionState } from "./types.js";
|
|
5
4
|
|
|
6
5
|
//#region src/realtime/subscribe-client.d.ts
|
|
7
6
|
|
|
@@ -9,7 +8,6 @@ type SubscribeEvents = {
|
|
|
9
8
|
connectionChange: ConnectionState;
|
|
10
9
|
error: DecartSDKError;
|
|
11
10
|
diagnostic: DiagnosticEvent;
|
|
12
|
-
stats: WebRTCStats;
|
|
13
11
|
};
|
|
14
12
|
type RealTimeSubscribeClient = {
|
|
15
13
|
isConnected: () => boolean;
|
|
@@ -21,6 +19,7 @@ type RealTimeSubscribeClient = {
|
|
|
21
19
|
type SubscribeOptions = {
|
|
22
20
|
token: string;
|
|
23
21
|
onRemoteStream: (stream: MediaStream) => void;
|
|
22
|
+
onConnectionChange?: (state: ConnectionState) => void;
|
|
24
23
|
};
|
|
25
24
|
//#endregion
|
|
26
25
|
export { RealTimeSubscribeClient, SubscribeEvents, SubscribeOptions };
|
|
@@ -1,20 +1,124 @@
|
|
|
1
|
+
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
|
+
import { createConsoleLogger } from "../utils/logger.js";
|
|
3
|
+
import { createEventBuffer } from "./event-buffer.js";
|
|
4
|
+
import { REALTIME_CONFIG } from "./config-realtime.js";
|
|
5
|
+
import { RealtimeObservability } from "./observability/realtime-observability.js";
|
|
6
|
+
import { ConnectionState, Room, RoomEvent, Track } from "livekit-client";
|
|
1
7
|
//#region src/realtime/subscribe-client.ts
|
|
2
|
-
function encodeSubscribeToken(sessionId, serverIp, serverPort) {
|
|
3
|
-
return btoa(JSON.stringify({
|
|
4
|
-
sid: sessionId,
|
|
5
|
-
ip: serverIp,
|
|
6
|
-
port: serverPort
|
|
7
|
-
}));
|
|
8
|
-
}
|
|
9
8
|
function decodeSubscribeToken(token) {
|
|
10
9
|
try {
|
|
11
10
|
const payload = JSON.parse(atob(token));
|
|
12
|
-
if (!payload.
|
|
13
|
-
return payload;
|
|
11
|
+
if (!payload.room_name || typeof payload.room_name !== "string") throw new Error("Invalid subscribe token format");
|
|
12
|
+
return { room_name: payload.room_name };
|
|
14
13
|
} catch {
|
|
15
14
|
throw new Error("Invalid subscribe token");
|
|
16
15
|
}
|
|
17
16
|
}
|
|
18
|
-
|
|
17
|
+
function mapLiveKitState(state) {
|
|
18
|
+
switch (state) {
|
|
19
|
+
case ConnectionState.Connecting: return "connecting";
|
|
20
|
+
case ConnectionState.Connected: return "connected";
|
|
21
|
+
case ConnectionState.Reconnecting:
|
|
22
|
+
case ConnectionState.SignalReconnecting: return "reconnecting";
|
|
23
|
+
case ConnectionState.Disconnected: return "disconnected";
|
|
24
|
+
default: return "disconnected";
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
async function fetchWatchStreamCredentials(opts) {
|
|
28
|
+
if (!/^https?:\/\//i.test(opts.baseUrl)) throw new Error(`watch-stream baseUrl must use http(s); got ${opts.baseUrl}`);
|
|
29
|
+
const url = `${opts.baseUrl}/watch-stream/${encodeURIComponent(opts.roomName)}`;
|
|
30
|
+
const res = await fetch(url, {
|
|
31
|
+
method: "POST",
|
|
32
|
+
headers: {
|
|
33
|
+
"x-api-key": opts.apiKey,
|
|
34
|
+
"content-type": "application/json"
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
if (!res.ok) {
|
|
38
|
+
const body = await res.text().catch(() => "");
|
|
39
|
+
throw new Error(`watch-stream request failed (${res.status}): ${body || res.statusText}`);
|
|
40
|
+
}
|
|
41
|
+
const json = await res.json();
|
|
42
|
+
if (!json.livekit_url || !json.token || !json.room_name) throw new Error("watch-stream response missing required fields");
|
|
43
|
+
return {
|
|
44
|
+
livekit_url: json.livekit_url,
|
|
45
|
+
token: json.token,
|
|
46
|
+
room_name: json.room_name
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
const createRealTimeSubscribeClient = (opts) => {
|
|
50
|
+
const { baseUrl, apiKey, integration } = opts;
|
|
51
|
+
const logger = opts.logger ?? createConsoleLogger("info");
|
|
52
|
+
const subscribe = async (options) => {
|
|
53
|
+
const { room_name: roomName } = decodeSubscribeToken(options.token);
|
|
54
|
+
const { emitter, emitOrBuffer, flush, stop } = createEventBuffer();
|
|
55
|
+
let observability;
|
|
56
|
+
let room;
|
|
57
|
+
let currentState = "connecting";
|
|
58
|
+
let remoteStream = null;
|
|
59
|
+
const setState = (state) => {
|
|
60
|
+
if (currentState === state) return;
|
|
61
|
+
currentState = state;
|
|
62
|
+
options.onConnectionChange?.(state);
|
|
63
|
+
emitOrBuffer("connectionChange", state);
|
|
64
|
+
};
|
|
65
|
+
try {
|
|
66
|
+
observability = new RealtimeObservability({
|
|
67
|
+
telemetryEnabled: false,
|
|
68
|
+
apiKey,
|
|
69
|
+
integration,
|
|
70
|
+
logger,
|
|
71
|
+
onDiagnostic: (event) => emitOrBuffer("diagnostic", event)
|
|
72
|
+
});
|
|
73
|
+
setState("connecting");
|
|
74
|
+
const creds = await fetchWatchStreamCredentials({
|
|
75
|
+
baseUrl,
|
|
76
|
+
apiKey,
|
|
77
|
+
roomName
|
|
78
|
+
});
|
|
79
|
+
room = new Room(REALTIME_CONFIG.livekit.roomOptions);
|
|
80
|
+
const activeRoom = room;
|
|
81
|
+
activeRoom.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
|
|
82
|
+
if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
|
|
83
|
+
if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
|
|
84
|
+
track.attach();
|
|
85
|
+
const mediaStreamTrack = track.mediaStreamTrack;
|
|
86
|
+
if (!mediaStreamTrack) return;
|
|
87
|
+
remoteStream ??= new MediaStream();
|
|
88
|
+
if (!remoteStream.getTracks().includes(mediaStreamTrack)) remoteStream.addTrack(mediaStreamTrack);
|
|
89
|
+
options.onRemoteStream(remoteStream);
|
|
90
|
+
});
|
|
91
|
+
activeRoom.on(RoomEvent.ConnectionStateChanged, (state) => {
|
|
92
|
+
setState(mapLiveKitState(state));
|
|
93
|
+
});
|
|
94
|
+
activeRoom.on(RoomEvent.Disconnected, () => {
|
|
95
|
+
setState("disconnected");
|
|
96
|
+
});
|
|
97
|
+
await activeRoom.connect(creds.livekit_url, creds.token);
|
|
98
|
+
observability.setLiveKitRoom(activeRoom);
|
|
99
|
+
setState("connected");
|
|
100
|
+
const client = {
|
|
101
|
+
isConnected: () => activeRoom.state === ConnectionState.Connected,
|
|
102
|
+
getConnectionState: () => mapLiveKitState(activeRoom.state),
|
|
103
|
+
disconnect: () => {
|
|
104
|
+
observability?.stop();
|
|
105
|
+
stop();
|
|
106
|
+
activeRoom.disconnect().catch(() => {});
|
|
107
|
+
},
|
|
108
|
+
on: emitter.on,
|
|
109
|
+
off: emitter.off
|
|
110
|
+
};
|
|
111
|
+
flush();
|
|
112
|
+
return client;
|
|
113
|
+
} catch (error) {
|
|
114
|
+
observability?.stop();
|
|
115
|
+
if (room) room.disconnect().catch(() => {});
|
|
116
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
117
|
+
logger.error("Realtime subscribe error", { error: err.message });
|
|
118
|
+
throw classifyWebrtcError(err);
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
return { subscribe };
|
|
122
|
+
};
|
|
19
123
|
//#endregion
|
|
20
|
-
export {
|
|
124
|
+
export { createRealTimeSubscribeClient };
|