@decartai/sdk 0.0.68 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -9
- package/dist/files/client.d.ts +33 -0
- package/dist/files/client.js +65 -0
- package/dist/files/types.d.ts +22 -0
- package/dist/files/types.js +4 -0
- package/dist/index.d.ts +19 -4
- package/dist/index.js +50 -28
- package/dist/process/client.js +1 -3
- package/dist/process/request.js +1 -3
- package/dist/queue/client.js +1 -3
- package/dist/queue/polling.js +1 -2
- package/dist/queue/request.js +1 -3
- package/dist/realtime/client.d.ts +23 -11
- package/dist/realtime/client.js +85 -156
- package/dist/realtime/config-realtime.js +49 -0
- package/dist/realtime/event-buffer.js +1 -3
- package/dist/realtime/initial-state-gate.js +21 -0
- package/dist/realtime/media-channel.js +82 -0
- package/dist/realtime/methods.js +25 -43
- package/dist/realtime/mirror-stream.js +1 -2
- package/dist/realtime/observability/diagnostics.d.ts +14 -53
- package/dist/realtime/observability/livekit-stats-provider.js +25 -0
- package/dist/realtime/observability/realtime-observability.js +70 -6
- package/dist/realtime/observability/telemetry-reporter.js +9 -28
- package/dist/realtime/observability/webrtc-stats.d.ts +5 -4
- package/dist/realtime/observability/webrtc-stats.js +3 -5
- package/dist/realtime/signaling-channel.js +302 -0
- package/dist/realtime/stream-session.js +257 -0
- package/dist/realtime/subscribe-client.d.ts +2 -3
- package/dist/realtime/subscribe-client.js +115 -11
- package/dist/realtime/types.d.ts +25 -1
- package/dist/shared/model.d.ts +11 -1
- package/dist/shared/model.js +51 -14
- package/dist/shared/request.js +1 -3
- package/dist/shared/types.js +1 -3
- package/dist/tokens/client.js +1 -3
- package/dist/utils/env.js +1 -2
- package/dist/utils/errors.d.ts +3 -0
- package/dist/utils/errors.js +4 -2
- package/dist/utils/logger.js +1 -2
- package/dist/utils/media.js +43 -0
- package/dist/utils/platform.js +13 -0
- package/dist/utils/user-agent.js +1 -3
- package/dist/version.js +1 -2
- package/package.json +2 -1
- package/dist/realtime/webrtc-connection.js +0 -500
- package/dist/realtime/webrtc-manager.js +0 -210
package/dist/realtime/client.js
CHANGED
|
@@ -1,145 +1,117 @@
|
|
|
1
1
|
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
|
-
import {
|
|
2
|
+
import { isFileRefId } from "../files/types.js";
|
|
3
|
+
import { modelDefinitionSchema, resolveFpsNumber } from "../shared/model.js";
|
|
3
4
|
import { modelStateSchema } from "../shared/types.js";
|
|
5
|
+
import { createConsoleLogger } from "../utils/logger.js";
|
|
6
|
+
import { imageToBase64 } from "../utils/media.js";
|
|
7
|
+
import { isDesktopSafari } from "../utils/platform.js";
|
|
4
8
|
import { createEventBuffer } from "./event-buffer.js";
|
|
5
9
|
import { realtimeMethods } from "./methods.js";
|
|
6
10
|
import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
|
|
7
11
|
import { RealtimeObservability } from "./observability/realtime-observability.js";
|
|
8
|
-
import {
|
|
9
|
-
import { WebRTCManager } from "./webrtc-manager.js";
|
|
12
|
+
import { StreamSession } from "./stream-session.js";
|
|
10
13
|
import { z } from "zod";
|
|
11
|
-
|
|
12
14
|
//#region src/realtime/client.ts
|
|
13
|
-
async function blobToBase64(blob) {
|
|
14
|
-
return new Promise((resolve, reject) => {
|
|
15
|
-
const reader = new FileReader();
|
|
16
|
-
reader.onloadend = () => {
|
|
17
|
-
const result = reader.result;
|
|
18
|
-
if (typeof result !== "string") {
|
|
19
|
-
reject(/* @__PURE__ */ new Error("FileReader did not return a string"));
|
|
20
|
-
return;
|
|
21
|
-
}
|
|
22
|
-
const base64 = result.split(",")[1];
|
|
23
|
-
if (!base64) {
|
|
24
|
-
reject(/* @__PURE__ */ new Error("Invalid data URL format"));
|
|
25
|
-
return;
|
|
26
|
-
}
|
|
27
|
-
resolve(base64);
|
|
28
|
-
};
|
|
29
|
-
reader.onerror = reject;
|
|
30
|
-
reader.readAsDataURL(blob);
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
async function imageToBase64(image) {
|
|
34
|
-
if (typeof image === "string") {
|
|
35
|
-
let url = null;
|
|
36
|
-
try {
|
|
37
|
-
url = new URL(image);
|
|
38
|
-
} catch {}
|
|
39
|
-
if (url?.protocol === "data:") {
|
|
40
|
-
const [, base64] = image.split(",", 2);
|
|
41
|
-
if (!base64) throw new Error("Invalid data URL image");
|
|
42
|
-
return base64;
|
|
43
|
-
}
|
|
44
|
-
if (url?.protocol === "http:" || url?.protocol === "https:") {
|
|
45
|
-
const response = await fetch(image);
|
|
46
|
-
if (!response.ok) throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
|
|
47
|
-
return blobToBase64(await response.blob());
|
|
48
|
-
}
|
|
49
|
-
return image;
|
|
50
|
-
}
|
|
51
|
-
return blobToBase64(image);
|
|
52
|
-
}
|
|
53
15
|
const realTimeClientInitialStateSchema = modelStateSchema;
|
|
54
|
-
const createAsyncFunctionSchema = (schema) => z.custom((fn) => schema.implementAsync(fn));
|
|
55
16
|
const realTimeClientConnectOptionsSchema = z.object({
|
|
56
17
|
model: modelDefinitionSchema,
|
|
57
18
|
onRemoteStream: z.custom((val) => typeof val === "function", { message: "onRemoteStream must be a function" }),
|
|
19
|
+
onConnectionChange: z.custom((val) => typeof val === "function", { message: "onConnectionChange must be a function" }).optional(),
|
|
20
|
+
onQueuePosition: z.custom((val) => typeof val === "function", { message: "onQueuePosition must be a function" }).optional(),
|
|
58
21
|
initialState: realTimeClientInitialStateSchema.optional(),
|
|
59
|
-
|
|
22
|
+
queryParams: z.record(z.string(), z.string()).optional(),
|
|
60
23
|
mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
|
|
61
24
|
resolution: z.enum(["720p", "1080p"]).optional()
|
|
62
25
|
});
|
|
63
26
|
const createRealTimeClient = (opts) => {
|
|
64
|
-
const { baseUrl, apiKey, integration
|
|
27
|
+
const { baseUrl, apiKey, integration } = opts;
|
|
28
|
+
const logger = opts.logger ?? createConsoleLogger("info");
|
|
65
29
|
const connect = async (stream, options) => {
|
|
66
30
|
const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
|
|
67
31
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
68
|
-
const { onRemoteStream, initialState, resolution } = parsedOptions.data;
|
|
32
|
+
const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution } = parsedOptions.data;
|
|
69
33
|
const mirror = parsedOptions.data.mirror ?? false;
|
|
70
34
|
let inputStream = stream ?? new MediaStream();
|
|
71
35
|
let mirroredStream;
|
|
72
36
|
if (mirror !== false) try {
|
|
73
37
|
const firstVideoTrack = inputStream.getVideoTracks?.()[0];
|
|
74
38
|
if (firstVideoTrack && (mirror === true || shouldMirrorTrack(firstVideoTrack))) {
|
|
75
|
-
mirroredStream = createMirroredStream(inputStream, { fps: options.model.fps });
|
|
39
|
+
mirroredStream = createMirroredStream(inputStream, { fps: resolveFpsNumber(options.model.fps) });
|
|
76
40
|
inputStream = mirroredStream.stream;
|
|
77
41
|
} else if (mirror === true && !firstVideoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
|
|
78
42
|
} catch (error) {
|
|
79
43
|
logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
|
|
80
44
|
}
|
|
81
|
-
let
|
|
82
|
-
|
|
83
|
-
const observability = new RealtimeObservability({
|
|
84
|
-
telemetryEnabled: opts.telemetryEnabled,
|
|
85
|
-
apiKey,
|
|
86
|
-
model: options.model.name,
|
|
87
|
-
integration,
|
|
88
|
-
logger,
|
|
89
|
-
onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
|
|
90
|
-
onStats: opts.telemetryEnabled ? (stats) => emitOrBuffer("stats", stats) : void 0
|
|
91
|
-
});
|
|
45
|
+
let session;
|
|
46
|
+
let observability;
|
|
92
47
|
try {
|
|
93
|
-
const
|
|
48
|
+
const initialImageRef = isFileRefId(initialState?.image) ? initialState.image : void 0;
|
|
49
|
+
const initialImage = initialImageRef === void 0 && initialState?.image ? await imageToBase64(initialState.image) : void 0;
|
|
94
50
|
const initialPrompt = initialState?.prompt ? {
|
|
95
51
|
text: initialState.prompt.text,
|
|
96
52
|
enhance: initialState.prompt.enhance
|
|
97
53
|
} : void 0;
|
|
98
54
|
const url = `${baseUrl}${options.model.urlPath}`;
|
|
99
|
-
const
|
|
100
|
-
|
|
101
|
-
|
|
55
|
+
const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
|
|
56
|
+
observability = new RealtimeObservability({
|
|
57
|
+
telemetryEnabled: opts.telemetryEnabled,
|
|
58
|
+
apiKey,
|
|
59
|
+
model: options.model.name,
|
|
102
60
|
integration,
|
|
103
61
|
logger,
|
|
62
|
+
onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
|
|
63
|
+
onStats: (stats) => emitOrBuffer("stats", stats)
|
|
64
|
+
});
|
|
65
|
+
const safariCodec = isDesktopSafari() ? "vp8" : void 0;
|
|
66
|
+
session = new StreamSession({
|
|
67
|
+
url: `${url}?${new URLSearchParams({
|
|
68
|
+
...safariCodec ? { livekit_server_codec: safariCodec } : {},
|
|
69
|
+
...options.queryParams ?? {},
|
|
70
|
+
api_key: apiKey,
|
|
71
|
+
model: options.model.name,
|
|
72
|
+
...resolution ? { resolution } : {}
|
|
73
|
+
}).toString()}`,
|
|
74
|
+
integration,
|
|
104
75
|
observability,
|
|
105
|
-
|
|
106
|
-
onConnectionStateChange: (state) => {
|
|
107
|
-
emitOrBuffer("connectionChange", state);
|
|
108
|
-
},
|
|
109
|
-
onError: (error) => {
|
|
110
|
-
logger.error("WebRTC error", { error: error.message });
|
|
111
|
-
emitOrBuffer("error", classifyWebrtcError(error));
|
|
112
|
-
},
|
|
113
|
-
customizeOffer: options.customizeOffer,
|
|
114
|
-
vp8MinBitrate: 300,
|
|
115
|
-
vp8StartBitrate: 600,
|
|
76
|
+
localStream: inputStream,
|
|
116
77
|
initialImage,
|
|
117
|
-
|
|
78
|
+
initialImageRef,
|
|
79
|
+
initialPrompt,
|
|
80
|
+
logger,
|
|
81
|
+
videoCodec: safariCodec
|
|
118
82
|
});
|
|
119
|
-
const manager = webrtcManager;
|
|
120
83
|
let sessionId = null;
|
|
121
84
|
let subscribeToken = null;
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
};
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
};
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
85
|
+
session.on("remoteStream", onRemoteStream);
|
|
86
|
+
session.on("connectionChange", (state) => {
|
|
87
|
+
emitOrBuffer("connectionChange", state);
|
|
88
|
+
onConnectionChange?.(state);
|
|
89
|
+
});
|
|
90
|
+
session.on("queuePosition", (qp) => {
|
|
91
|
+
emitOrBuffer("queuePosition", qp);
|
|
92
|
+
onQueuePosition?.(qp);
|
|
93
|
+
});
|
|
94
|
+
session.on("sessionStarted", ({ sessionId: id, subscribeToken: token }) => {
|
|
95
|
+
sessionId = id;
|
|
96
|
+
subscribeToken = token;
|
|
97
|
+
observability?.sessionStarted(id);
|
|
98
|
+
});
|
|
99
|
+
session.on("generationTick", (e) => emitOrBuffer("generationTick", e));
|
|
100
|
+
session.on("generationEnded", (e) => emitOrBuffer("generationEnded", e));
|
|
101
|
+
session.on("error", (error) => {
|
|
102
|
+
logger.error("Realtime error", { error: error.message });
|
|
103
|
+
emitOrBuffer("error", classifyWebrtcError(error));
|
|
104
|
+
});
|
|
105
|
+
const activeSession = session;
|
|
106
|
+
await activeSession.connect();
|
|
134
107
|
const client = {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
getConnectionState: () => manager.getConnectionState(),
|
|
108
|
+
...realtimeMethods(activeSession, imageToBase64),
|
|
109
|
+
isConnected: () => activeSession.isConnected(),
|
|
110
|
+
getConnectionState: () => activeSession.getConnectionState(),
|
|
139
111
|
disconnect: () => {
|
|
140
|
-
observability
|
|
112
|
+
observability?.stop();
|
|
141
113
|
stop();
|
|
142
|
-
|
|
114
|
+
activeSession.disconnect();
|
|
143
115
|
mirroredStream?.dispose();
|
|
144
116
|
},
|
|
145
117
|
on: eventEmitter.on,
|
|
@@ -150,76 +122,33 @@ const createRealTimeClient = (opts) => {
|
|
|
150
122
|
get subscribeToken() {
|
|
151
123
|
return subscribeToken;
|
|
152
124
|
},
|
|
153
|
-
|
|
154
|
-
|
|
125
|
+
getSubscribeToken: () => subscribeToken,
|
|
126
|
+
setImage: async (image, imgOptions) => {
|
|
127
|
+
if (isFileRefId(image)) return activeSession.setImage({
|
|
128
|
+
kind: "ref",
|
|
129
|
+
ref: image
|
|
130
|
+
}, imgOptions);
|
|
131
|
+
if (image === null) return activeSession.setImage({
|
|
132
|
+
kind: "data",
|
|
133
|
+
data: null
|
|
134
|
+
}, imgOptions);
|
|
155
135
|
const base64 = await imageToBase64(image);
|
|
156
|
-
return
|
|
136
|
+
return activeSession.setImage({
|
|
137
|
+
kind: "data",
|
|
138
|
+
data: base64
|
|
139
|
+
}, imgOptions);
|
|
157
140
|
}
|
|
158
141
|
};
|
|
159
142
|
flush();
|
|
160
143
|
return client;
|
|
161
144
|
} catch (error) {
|
|
162
|
-
observability
|
|
163
|
-
|
|
145
|
+
observability?.stop();
|
|
146
|
+
session?.disconnect();
|
|
164
147
|
mirroredStream?.dispose();
|
|
165
148
|
throw error;
|
|
166
149
|
}
|
|
167
150
|
};
|
|
168
|
-
|
|
169
|
-
const { sid, ip, port } = decodeSubscribeToken(options.token);
|
|
170
|
-
const subscribeUrl = `${baseUrl}/subscribe/${encodeURIComponent(sid)}?IP=${encodeURIComponent(ip)}&port=${encodeURIComponent(port)}&api_key=${encodeURIComponent(apiKey)}`;
|
|
171
|
-
const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
|
|
172
|
-
let webrtcManager;
|
|
173
|
-
const observability = new RealtimeObservability({
|
|
174
|
-
telemetryEnabled: opts.telemetryEnabled,
|
|
175
|
-
apiKey,
|
|
176
|
-
integration,
|
|
177
|
-
logger,
|
|
178
|
-
onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
|
|
179
|
-
onStats: opts.telemetryEnabled ? (stats) => emitOrBuffer("stats", stats) : void 0
|
|
180
|
-
});
|
|
181
|
-
observability.sessionStarted(sid);
|
|
182
|
-
try {
|
|
183
|
-
webrtcManager = new WebRTCManager({
|
|
184
|
-
webrtcUrl: subscribeUrl,
|
|
185
|
-
integration,
|
|
186
|
-
logger,
|
|
187
|
-
observability,
|
|
188
|
-
onRemoteStream: options.onRemoteStream,
|
|
189
|
-
onConnectionStateChange: (state) => {
|
|
190
|
-
emitOrBuffer("connectionChange", state);
|
|
191
|
-
},
|
|
192
|
-
onError: (error) => {
|
|
193
|
-
logger.error("WebRTC subscribe error", { error: error.message });
|
|
194
|
-
emitOrBuffer("error", classifyWebrtcError(error));
|
|
195
|
-
}
|
|
196
|
-
});
|
|
197
|
-
const manager = webrtcManager;
|
|
198
|
-
await manager.connect(null);
|
|
199
|
-
const client = {
|
|
200
|
-
isConnected: () => manager.isConnected(),
|
|
201
|
-
getConnectionState: () => manager.getConnectionState(),
|
|
202
|
-
disconnect: () => {
|
|
203
|
-
observability.stop();
|
|
204
|
-
stop();
|
|
205
|
-
manager.cleanup();
|
|
206
|
-
},
|
|
207
|
-
on: eventEmitter.on,
|
|
208
|
-
off: eventEmitter.off
|
|
209
|
-
};
|
|
210
|
-
flush();
|
|
211
|
-
return client;
|
|
212
|
-
} catch (error) {
|
|
213
|
-
observability.stop();
|
|
214
|
-
webrtcManager?.cleanup();
|
|
215
|
-
throw error;
|
|
216
|
-
}
|
|
217
|
-
};
|
|
218
|
-
return {
|
|
219
|
-
connect,
|
|
220
|
-
subscribe
|
|
221
|
-
};
|
|
151
|
+
return { connect };
|
|
222
152
|
};
|
|
223
|
-
|
|
224
153
|
//#endregion
|
|
225
|
-
export { createRealTimeClient };
|
|
154
|
+
export { createRealTimeClient };
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
//#region src/realtime/config-realtime.ts
|
|
2
|
+
const REALTIME_CONFIG = {
|
|
3
|
+
signaling: {
|
|
4
|
+
connectTimeoutMs: 6e4,
|
|
5
|
+
handshakeTimeoutMs: 15e3,
|
|
6
|
+
requestTimeoutMs: 3e4
|
|
7
|
+
},
|
|
8
|
+
session: {
|
|
9
|
+
connectionTimeoutMs: 6e4 * 5,
|
|
10
|
+
retry: {
|
|
11
|
+
retries: 5,
|
|
12
|
+
factor: 2,
|
|
13
|
+
minTimeout: 1e3,
|
|
14
|
+
maxTimeout: 1e4
|
|
15
|
+
},
|
|
16
|
+
permanentErrorSubstrings: [
|
|
17
|
+
"permission denied",
|
|
18
|
+
"not allowed",
|
|
19
|
+
"invalid session",
|
|
20
|
+
"401",
|
|
21
|
+
"invalid api key",
|
|
22
|
+
"unauthorized"
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
methods: {
|
|
26
|
+
promptTimeoutMs: 15e3,
|
|
27
|
+
updateTimeoutMs: 3e4
|
|
28
|
+
},
|
|
29
|
+
livekit: {
|
|
30
|
+
inferenceServerIdentityPrefix: "inference-server-",
|
|
31
|
+
roomOptions: {
|
|
32
|
+
adaptiveStream: false,
|
|
33
|
+
dynacast: false
|
|
34
|
+
},
|
|
35
|
+
defaultVideoCodec: "h264",
|
|
36
|
+
defaultMaxVideoBitrateBps: 35e5,
|
|
37
|
+
defaultPublishFps: 30
|
|
38
|
+
},
|
|
39
|
+
observability: {
|
|
40
|
+
stallFpsThreshold: .5,
|
|
41
|
+
statsDefaultIntervalMs: 1e3,
|
|
42
|
+
statsMinIntervalMs: 500,
|
|
43
|
+
telemetryReportIntervalMs: 1e4,
|
|
44
|
+
telemetryUrl: "https://platform.decart.ai/api/v1/telemetry",
|
|
45
|
+
telemetryMaxItemsPerReport: 120
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
//#endregion
|
|
49
|
+
export { REALTIME_CONFIG };
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import mitt from "mitt";
|
|
2
|
-
|
|
3
2
|
//#region src/realtime/event-buffer.ts
|
|
4
3
|
function createEventBuffer() {
|
|
5
4
|
const emitter = mitt();
|
|
@@ -30,6 +29,5 @@ function createEventBuffer() {
|
|
|
30
29
|
stop
|
|
31
30
|
};
|
|
32
31
|
}
|
|
33
|
-
|
|
34
32
|
//#endregion
|
|
35
|
-
export { createEventBuffer };
|
|
33
|
+
export { createEventBuffer };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
//#region src/realtime/initial-state-gate.ts
|
|
2
|
+
var InitialStateGate = class {
|
|
3
|
+
attemptId = 0;
|
|
4
|
+
startAttempt(initialState) {
|
|
5
|
+
const attemptId = ++this.attemptId;
|
|
6
|
+
const shouldWait = hasCallerProvidedInitialState(initialState);
|
|
7
|
+
return { waitForReadiness: async (initialStateAck) => {
|
|
8
|
+
if (shouldWait) await initialStateAck;
|
|
9
|
+
return this.attemptId === attemptId;
|
|
10
|
+
} };
|
|
11
|
+
}
|
|
12
|
+
reset() {
|
|
13
|
+
this.attemptId++;
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
function hasCallerProvidedInitialState(state) {
|
|
17
|
+
if (!state) return false;
|
|
18
|
+
return state.image !== void 0 && state.image !== null || state.prompt !== void 0 && state.prompt !== null;
|
|
19
|
+
}
|
|
20
|
+
//#endregion
|
|
21
|
+
export { InitialStateGate };
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { createConsoleLogger } from "../utils/logger.js";
|
|
2
|
+
import { REALTIME_CONFIG } from "./config-realtime.js";
|
|
3
|
+
import mitt from "mitt";
|
|
4
|
+
import { Room, RoomEvent, Track, TrackEvent } from "livekit-client";
|
|
5
|
+
//#region src/realtime/media-channel.ts
|
|
6
|
+
function getDefaultVideoPublishOptions(videoCodec) {
|
|
7
|
+
const videoEncoding = {
|
|
8
|
+
maxBitrate: REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps,
|
|
9
|
+
maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps
|
|
10
|
+
};
|
|
11
|
+
return {
|
|
12
|
+
source: Track.Source.Camera,
|
|
13
|
+
videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec,
|
|
14
|
+
simulcast: true,
|
|
15
|
+
videoEncoding
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
var MediaChannel = class {
|
|
19
|
+
room = null;
|
|
20
|
+
remoteStream = null;
|
|
21
|
+
events = mitt();
|
|
22
|
+
logger;
|
|
23
|
+
constructor(config) {
|
|
24
|
+
this.config = config;
|
|
25
|
+
this.logger = config.logger ?? createConsoleLogger("warn");
|
|
26
|
+
}
|
|
27
|
+
get localStream() {
|
|
28
|
+
return this.config.localStream;
|
|
29
|
+
}
|
|
30
|
+
on(event, handler) {
|
|
31
|
+
this.events.on(event, handler);
|
|
32
|
+
}
|
|
33
|
+
off(event, handler) {
|
|
34
|
+
this.events.off(event, handler);
|
|
35
|
+
}
|
|
36
|
+
async connect(opts) {
|
|
37
|
+
this.room ??= new Room(REALTIME_CONFIG.livekit.roomOptions);
|
|
38
|
+
const room = this.room;
|
|
39
|
+
room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
|
|
40
|
+
if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
|
|
41
|
+
if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
|
|
42
|
+
track.attach();
|
|
43
|
+
const mediaStreamTrack = track.mediaStreamTrack;
|
|
44
|
+
if (mediaStreamTrack) {
|
|
45
|
+
this.remoteStream ??= new MediaStream();
|
|
46
|
+
if (!this.remoteStream.getTracks().includes(mediaStreamTrack)) this.remoteStream.addTrack(mediaStreamTrack);
|
|
47
|
+
this.events.emit("remoteStream", this.remoteStream);
|
|
48
|
+
}
|
|
49
|
+
track.on(TrackEvent.VideoPlaybackStarted, () => {
|
|
50
|
+
this.events.emit("firstFrame");
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
room.on(RoomEvent.Disconnected, (reason) => {
|
|
54
|
+
this.logger.warn("livekit: room disconnected", { reason });
|
|
55
|
+
this.events.emit("disconnected", { reason });
|
|
56
|
+
});
|
|
57
|
+
this.config.observability?.startPhase("webrtc-handshake");
|
|
58
|
+
await room.connect(opts.url, opts.token);
|
|
59
|
+
this.config.observability?.endPhase("webrtc-handshake", { success: true });
|
|
60
|
+
this.config.observability?.setLiveKitRoom(room);
|
|
61
|
+
}
|
|
62
|
+
async publishLocalTracks() {
|
|
63
|
+
if (!this.config.localStream) return;
|
|
64
|
+
this.config.observability?.startPhase("publish-local-track");
|
|
65
|
+
await this.publishTracks(this.config.localStream);
|
|
66
|
+
this.config.observability?.endPhase("publish-local-track", { success: true });
|
|
67
|
+
}
|
|
68
|
+
disconnect() {
|
|
69
|
+
const room = this.room;
|
|
70
|
+
this.room = null;
|
|
71
|
+
this.remoteStream = null;
|
|
72
|
+
this.config.observability?.setLiveKitRoom(null);
|
|
73
|
+
if (room) room.disconnect().catch(() => {});
|
|
74
|
+
}
|
|
75
|
+
async publishTracks(stream) {
|
|
76
|
+
if (!this.room) return;
|
|
77
|
+
for (const track of stream.getTracks()) if (track.kind === "video") await this.room.localParticipant.publishTrack(track, getDefaultVideoPublishOptions(this.config.videoCodec));
|
|
78
|
+
else await this.room.localParticipant.publishTrack(track);
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
//#endregion
|
|
82
|
+
export { MediaChannel };
|
package/dist/realtime/methods.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
+
import { isFileRefId } from "../files/types.js";
|
|
2
|
+
import { REALTIME_CONFIG } from "./config-realtime.js";
|
|
1
3
|
import { z } from "zod";
|
|
2
|
-
|
|
3
4
|
//#region src/realtime/methods.ts
|
|
4
|
-
const PROMPT_TIMEOUT_MS = 15 * 1e3;
|
|
5
|
-
const UPDATE_TIMEOUT_MS = 30 * 1e3;
|
|
6
5
|
const setInputSchema = z.object({
|
|
7
6
|
prompt: z.string().min(1).optional(),
|
|
8
7
|
enhance: z.boolean().optional().default(true),
|
|
@@ -17,61 +16,44 @@ const setPromptInputSchema = z.object({
|
|
|
17
16
|
prompt: z.string().min(1),
|
|
18
17
|
enhance: z.boolean().optional().default(true)
|
|
19
18
|
});
|
|
20
|
-
const realtimeMethods = (
|
|
21
|
-
const assertConnected = () => {
|
|
22
|
-
const state = webrtcManager.getConnectionState();
|
|
23
|
-
if (state !== "connected" && state !== "generating") throw new Error(`Cannot send message: connection is ${state}`);
|
|
24
|
-
};
|
|
19
|
+
const realtimeMethods = (session, imageToBase64) => {
|
|
25
20
|
const set = async (input) => {
|
|
26
|
-
assertConnected();
|
|
27
21
|
const parsed = setInputSchema.safeParse(input);
|
|
28
22
|
if (!parsed.success) throw parsed.error;
|
|
29
23
|
const { prompt, enhance, image } = parsed.data;
|
|
30
|
-
|
|
31
|
-
if (image !== void 0 && image !== null) imageBase64 = await imageToBase64(image);
|
|
32
|
-
await webrtcManager.setImage(imageBase64, {
|
|
24
|
+
const options = {
|
|
33
25
|
prompt,
|
|
34
26
|
enhance,
|
|
35
|
-
timeout:
|
|
36
|
-
}
|
|
27
|
+
timeout: REALTIME_CONFIG.methods.updateTimeoutMs
|
|
28
|
+
};
|
|
29
|
+
if (isFileRefId(image)) {
|
|
30
|
+
await session.setImage({
|
|
31
|
+
kind: "ref",
|
|
32
|
+
ref: image
|
|
33
|
+
}, options);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const imageBase64 = image !== void 0 && image !== null ? await imageToBase64(image) : null;
|
|
37
|
+
await session.setImage({
|
|
38
|
+
kind: "data",
|
|
39
|
+
data: imageBase64
|
|
40
|
+
}, options);
|
|
37
41
|
};
|
|
38
42
|
const setPrompt = async (prompt, { enhance } = {}) => {
|
|
39
|
-
|
|
40
|
-
const parsedInput = setPromptInputSchema.safeParse({
|
|
43
|
+
const parsed = setPromptInputSchema.safeParse({
|
|
41
44
|
prompt,
|
|
42
45
|
enhance
|
|
43
46
|
});
|
|
44
|
-
if (!
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
const ackPromise = new Promise((resolve, reject) => {
|
|
50
|
-
promptAckListener = (promptAckMessage) => {
|
|
51
|
-
if (promptAckMessage.prompt === parsedInput.data.prompt) if (promptAckMessage.success) resolve();
|
|
52
|
-
else reject(new Error(promptAckMessage.error ?? "Failed to send prompt"));
|
|
53
|
-
};
|
|
54
|
-
emitter.on("promptAck", promptAckListener);
|
|
55
|
-
});
|
|
56
|
-
if (!webrtcManager.sendMessage({
|
|
57
|
-
type: "prompt",
|
|
58
|
-
prompt: parsedInput.data.prompt,
|
|
59
|
-
enhance_prompt: parsedInput.data.enhance
|
|
60
|
-
})) throw new Error("WebSocket is not open");
|
|
61
|
-
const timeoutPromise = new Promise((_, reject) => {
|
|
62
|
-
timeoutId = setTimeout(() => reject(/* @__PURE__ */ new Error("Prompt timed out")), PROMPT_TIMEOUT_MS);
|
|
63
|
-
});
|
|
64
|
-
await Promise.race([ackPromise, timeoutPromise]);
|
|
65
|
-
} finally {
|
|
66
|
-
if (promptAckListener) emitter.off("promptAck", promptAckListener);
|
|
67
|
-
if (timeoutId) clearTimeout(timeoutId);
|
|
68
|
-
}
|
|
47
|
+
if (!parsed.success) throw parsed.error;
|
|
48
|
+
await session.sendPrompt(parsed.data.prompt, {
|
|
49
|
+
enhance: parsed.data.enhance,
|
|
50
|
+
timeout: REALTIME_CONFIG.methods.promptTimeoutMs
|
|
51
|
+
});
|
|
69
52
|
};
|
|
70
53
|
return {
|
|
71
54
|
set,
|
|
72
55
|
setPrompt
|
|
73
56
|
};
|
|
74
57
|
};
|
|
75
|
-
|
|
76
58
|
//#endregion
|
|
77
|
-
export { realtimeMethods };
|
|
59
|
+
export { realtimeMethods };
|
|
@@ -1,47 +1,17 @@
|
|
|
1
1
|
//#region src/realtime/observability/diagnostics.d.ts
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
type PhaseTimingEvent = {
|
|
5
|
-
phase: ConnectionPhase;
|
|
2
|
+
type ClientSessionConnectionBreakdownPhase = {
|
|
3
|
+
phase: string;
|
|
6
4
|
durationMs: number;
|
|
7
5
|
success: boolean;
|
|
8
6
|
error?: string;
|
|
9
7
|
};
|
|
10
|
-
type
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
type IceStateEvent = {
|
|
18
|
-
state: string;
|
|
19
|
-
previousState: string;
|
|
20
|
-
timestampMs: number;
|
|
21
|
-
};
|
|
22
|
-
type PeerConnectionStateEvent = {
|
|
23
|
-
state: string;
|
|
24
|
-
previousState: string;
|
|
25
|
-
timestampMs: number;
|
|
26
|
-
};
|
|
27
|
-
type SignalingStateEvent = {
|
|
28
|
-
state: string;
|
|
29
|
-
previousState: string;
|
|
30
|
-
timestampMs: number;
|
|
31
|
-
};
|
|
32
|
-
type SelectedCandidatePairEvent = {
|
|
33
|
-
local: {
|
|
34
|
-
candidateType: string;
|
|
35
|
-
protocol: string;
|
|
36
|
-
address?: string;
|
|
37
|
-
port?: number;
|
|
38
|
-
};
|
|
39
|
-
remote: {
|
|
40
|
-
candidateType: string;
|
|
41
|
-
protocol: string;
|
|
42
|
-
address?: string;
|
|
43
|
-
port?: number;
|
|
44
|
-
};
|
|
8
|
+
type ClientSessionConnectionBreakdownEvent = {
|
|
9
|
+
attempt: number;
|
|
10
|
+
success: boolean;
|
|
11
|
+
totalDurationMs: number;
|
|
12
|
+
initialImageSizeKb: number | null;
|
|
13
|
+
phases: ClientSessionConnectionBreakdownPhase[];
|
|
14
|
+
error?: string;
|
|
45
15
|
};
|
|
46
16
|
type ReconnectEvent = {
|
|
47
17
|
attempt: number;
|
|
@@ -51,28 +21,19 @@ type ReconnectEvent = {
|
|
|
51
21
|
error?: string;
|
|
52
22
|
};
|
|
53
23
|
type VideoStallEvent = {
|
|
54
|
-
/** True when a stall is detected, false when recovered. */
|
|
55
24
|
stalled: boolean;
|
|
56
|
-
/** Duration of the stall in ms (0 when stall first detected, actual duration on recovery). */
|
|
57
25
|
durationMs: number;
|
|
58
26
|
};
|
|
59
|
-
/** All diagnostic event types keyed by name. */
|
|
60
27
|
type DiagnosticEvents = {
|
|
61
|
-
|
|
62
|
-
iceCandidate: IceCandidateEvent;
|
|
63
|
-
iceStateChange: IceStateEvent;
|
|
64
|
-
peerConnectionStateChange: PeerConnectionStateEvent;
|
|
65
|
-
signalingStateChange: SignalingStateEvent;
|
|
66
|
-
selectedCandidatePair: SelectedCandidatePairEvent;
|
|
28
|
+
"client-session-connection-breakdown": ClientSessionConnectionBreakdownEvent;
|
|
67
29
|
reconnect: ReconnectEvent;
|
|
68
30
|
videoStall: VideoStallEvent;
|
|
69
31
|
};
|
|
70
32
|
type DiagnosticEventName = keyof DiagnosticEvents;
|
|
71
|
-
|
|
72
|
-
type DiagnosticEvent = { [K in DiagnosticEventName]: {
|
|
33
|
+
type DiagnosticEventForName<K extends DiagnosticEventName> = {
|
|
73
34
|
name: K;
|
|
74
35
|
data: DiagnosticEvents[K];
|
|
75
|
-
}
|
|
76
|
-
|
|
36
|
+
};
|
|
37
|
+
type DiagnosticEvent = { [K in DiagnosticEventName]: DiagnosticEventForName<K> }[DiagnosticEventName];
|
|
77
38
|
//#endregion
|
|
78
|
-
export {
|
|
39
|
+
export { ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ReconnectEvent, VideoStallEvent };
|