@decartai/sdk 0.0.68 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +49 -9
  2. package/dist/files/client.d.ts +33 -0
  3. package/dist/files/client.js +65 -0
  4. package/dist/files/types.d.ts +22 -0
  5. package/dist/files/types.js +4 -0
  6. package/dist/index.d.ts +19 -4
  7. package/dist/index.js +50 -28
  8. package/dist/process/client.js +1 -3
  9. package/dist/process/request.js +1 -3
  10. package/dist/queue/client.js +1 -3
  11. package/dist/queue/polling.js +1 -2
  12. package/dist/queue/request.js +1 -3
  13. package/dist/realtime/client.d.ts +23 -11
  14. package/dist/realtime/client.js +85 -156
  15. package/dist/realtime/config-realtime.js +49 -0
  16. package/dist/realtime/event-buffer.js +1 -3
  17. package/dist/realtime/initial-state-gate.js +21 -0
  18. package/dist/realtime/media-channel.js +82 -0
  19. package/dist/realtime/methods.js +25 -43
  20. package/dist/realtime/mirror-stream.js +1 -2
  21. package/dist/realtime/observability/diagnostics.d.ts +14 -53
  22. package/dist/realtime/observability/livekit-stats-provider.js +25 -0
  23. package/dist/realtime/observability/realtime-observability.js +70 -6
  24. package/dist/realtime/observability/telemetry-reporter.js +9 -28
  25. package/dist/realtime/observability/webrtc-stats.d.ts +5 -4
  26. package/dist/realtime/observability/webrtc-stats.js +3 -5
  27. package/dist/realtime/signaling-channel.js +302 -0
  28. package/dist/realtime/stream-session.js +257 -0
  29. package/dist/realtime/subscribe-client.d.ts +2 -3
  30. package/dist/realtime/subscribe-client.js +115 -11
  31. package/dist/realtime/types.d.ts +25 -1
  32. package/dist/shared/model.d.ts +11 -1
  33. package/dist/shared/model.js +51 -14
  34. package/dist/shared/request.js +1 -3
  35. package/dist/shared/types.js +1 -3
  36. package/dist/tokens/client.js +1 -3
  37. package/dist/utils/env.js +1 -2
  38. package/dist/utils/errors.d.ts +3 -0
  39. package/dist/utils/errors.js +4 -2
  40. package/dist/utils/logger.js +1 -2
  41. package/dist/utils/media.js +43 -0
  42. package/dist/utils/platform.js +13 -0
  43. package/dist/utils/user-agent.js +1 -3
  44. package/dist/version.js +1 -2
  45. package/package.json +2 -1
  46. package/dist/realtime/webrtc-connection.js +0 -500
  47. package/dist/realtime/webrtc-manager.js +0 -210
@@ -1,145 +1,117 @@
1
1
  import { classifyWebrtcError } from "../utils/errors.js";
2
- import { modelDefinitionSchema } from "../shared/model.js";
2
+ import { isFileRefId } from "../files/types.js";
3
+ import { modelDefinitionSchema, resolveFpsNumber } from "../shared/model.js";
3
4
  import { modelStateSchema } from "../shared/types.js";
5
+ import { createConsoleLogger } from "../utils/logger.js";
6
+ import { imageToBase64 } from "../utils/media.js";
7
+ import { isDesktopSafari } from "../utils/platform.js";
4
8
  import { createEventBuffer } from "./event-buffer.js";
5
9
  import { realtimeMethods } from "./methods.js";
6
10
  import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
7
11
  import { RealtimeObservability } from "./observability/realtime-observability.js";
8
- import { decodeSubscribeToken, encodeSubscribeToken } from "./subscribe-client.js";
9
- import { WebRTCManager } from "./webrtc-manager.js";
12
+ import { StreamSession } from "./stream-session.js";
10
13
  import { z } from "zod";
11
-
12
14
  //#region src/realtime/client.ts
13
- async function blobToBase64(blob) {
14
- return new Promise((resolve, reject) => {
15
- const reader = new FileReader();
16
- reader.onloadend = () => {
17
- const result = reader.result;
18
- if (typeof result !== "string") {
19
- reject(/* @__PURE__ */ new Error("FileReader did not return a string"));
20
- return;
21
- }
22
- const base64 = result.split(",")[1];
23
- if (!base64) {
24
- reject(/* @__PURE__ */ new Error("Invalid data URL format"));
25
- return;
26
- }
27
- resolve(base64);
28
- };
29
- reader.onerror = reject;
30
- reader.readAsDataURL(blob);
31
- });
32
- }
33
- async function imageToBase64(image) {
34
- if (typeof image === "string") {
35
- let url = null;
36
- try {
37
- url = new URL(image);
38
- } catch {}
39
- if (url?.protocol === "data:") {
40
- const [, base64] = image.split(",", 2);
41
- if (!base64) throw new Error("Invalid data URL image");
42
- return base64;
43
- }
44
- if (url?.protocol === "http:" || url?.protocol === "https:") {
45
- const response = await fetch(image);
46
- if (!response.ok) throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
47
- return blobToBase64(await response.blob());
48
- }
49
- return image;
50
- }
51
- return blobToBase64(image);
52
- }
53
15
  const realTimeClientInitialStateSchema = modelStateSchema;
54
- const createAsyncFunctionSchema = (schema) => z.custom((fn) => schema.implementAsync(fn));
55
16
  const realTimeClientConnectOptionsSchema = z.object({
56
17
  model: modelDefinitionSchema,
57
18
  onRemoteStream: z.custom((val) => typeof val === "function", { message: "onRemoteStream must be a function" }),
19
+ onConnectionChange: z.custom((val) => typeof val === "function", { message: "onConnectionChange must be a function" }).optional(),
20
+ onQueuePosition: z.custom((val) => typeof val === "function", { message: "onQueuePosition must be a function" }).optional(),
58
21
  initialState: realTimeClientInitialStateSchema.optional(),
59
- customizeOffer: createAsyncFunctionSchema(z.function()).optional(),
22
+ queryParams: z.record(z.string(), z.string()).optional(),
60
23
  mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
61
24
  resolution: z.enum(["720p", "1080p"]).optional()
62
25
  });
63
26
  const createRealTimeClient = (opts) => {
64
- const { baseUrl, apiKey, integration, logger } = opts;
27
+ const { baseUrl, apiKey, integration } = opts;
28
+ const logger = opts.logger ?? createConsoleLogger("info");
65
29
  const connect = async (stream, options) => {
66
30
  const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
67
31
  if (!parsedOptions.success) throw parsedOptions.error;
68
- const { onRemoteStream, initialState, resolution } = parsedOptions.data;
32
+ const { onRemoteStream, onConnectionChange, onQueuePosition, initialState, resolution } = parsedOptions.data;
69
33
  const mirror = parsedOptions.data.mirror ?? false;
70
34
  let inputStream = stream ?? new MediaStream();
71
35
  let mirroredStream;
72
36
  if (mirror !== false) try {
73
37
  const firstVideoTrack = inputStream.getVideoTracks?.()[0];
74
38
  if (firstVideoTrack && (mirror === true || shouldMirrorTrack(firstVideoTrack))) {
75
- mirroredStream = createMirroredStream(inputStream, { fps: options.model.fps });
39
+ mirroredStream = createMirroredStream(inputStream, { fps: resolveFpsNumber(options.model.fps) });
76
40
  inputStream = mirroredStream.stream;
77
41
  } else if (mirror === true && !firstVideoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
78
42
  } catch (error) {
79
43
  logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
80
44
  }
81
- let webrtcManager;
82
- const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
83
- const observability = new RealtimeObservability({
84
- telemetryEnabled: opts.telemetryEnabled,
85
- apiKey,
86
- model: options.model.name,
87
- integration,
88
- logger,
89
- onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
90
- onStats: opts.telemetryEnabled ? (stats) => emitOrBuffer("stats", stats) : void 0
91
- });
45
+ let session;
46
+ let observability;
92
47
  try {
93
- const initialImage = initialState?.image ? await imageToBase64(initialState.image) : void 0;
48
+ const initialImageRef = isFileRefId(initialState?.image) ? initialState.image : void 0;
49
+ const initialImage = initialImageRef === void 0 && initialState?.image ? await imageToBase64(initialState.image) : void 0;
94
50
  const initialPrompt = initialState?.prompt ? {
95
51
  text: initialState.prompt.text,
96
52
  enhance: initialState.prompt.enhance
97
53
  } : void 0;
98
54
  const url = `${baseUrl}${options.model.urlPath}`;
99
- const resolutionQs = resolution ? `&resolution=${encodeURIComponent(resolution)}` : "";
100
- webrtcManager = new WebRTCManager({
101
- webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}${resolutionQs}`,
55
+ const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
56
+ observability = new RealtimeObservability({
57
+ telemetryEnabled: opts.telemetryEnabled,
58
+ apiKey,
59
+ model: options.model.name,
102
60
  integration,
103
61
  logger,
62
+ onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
63
+ onStats: (stats) => emitOrBuffer("stats", stats)
64
+ });
65
+ const safariCodec = isDesktopSafari() ? "vp8" : void 0;
66
+ session = new StreamSession({
67
+ url: `${url}?${new URLSearchParams({
68
+ ...safariCodec ? { livekit_server_codec: safariCodec } : {},
69
+ ...options.queryParams ?? {},
70
+ api_key: apiKey,
71
+ model: options.model.name,
72
+ ...resolution ? { resolution } : {}
73
+ }).toString()}`,
74
+ integration,
104
75
  observability,
105
- onRemoteStream,
106
- onConnectionStateChange: (state) => {
107
- emitOrBuffer("connectionChange", state);
108
- },
109
- onError: (error) => {
110
- logger.error("WebRTC error", { error: error.message });
111
- emitOrBuffer("error", classifyWebrtcError(error));
112
- },
113
- customizeOffer: options.customizeOffer,
114
- vp8MinBitrate: 300,
115
- vp8StartBitrate: 600,
76
+ localStream: inputStream,
116
77
  initialImage,
117
- initialPrompt
78
+ initialImageRef,
79
+ initialPrompt,
80
+ logger,
81
+ videoCodec: safariCodec
118
82
  });
119
- const manager = webrtcManager;
120
83
  let sessionId = null;
121
84
  let subscribeToken = null;
122
- const sessionIdListener = (msg) => {
123
- subscribeToken = encodeSubscribeToken(msg.session_id, msg.server_ip, msg.server_port);
124
- sessionId = msg.session_id;
125
- observability.sessionStarted(msg.session_id);
126
- };
127
- manager.getWebsocketMessageEmitter().on("sessionId", sessionIdListener);
128
- const tickListener = (msg) => {
129
- emitOrBuffer("generationTick", { seconds: msg.seconds });
130
- };
131
- manager.getWebsocketMessageEmitter().on("generationTick", tickListener);
132
- await manager.connect(inputStream);
133
- const methods = realtimeMethods(manager, imageToBase64);
85
+ session.on("remoteStream", onRemoteStream);
86
+ session.on("connectionChange", (state) => {
87
+ emitOrBuffer("connectionChange", state);
88
+ onConnectionChange?.(state);
89
+ });
90
+ session.on("queuePosition", (qp) => {
91
+ emitOrBuffer("queuePosition", qp);
92
+ onQueuePosition?.(qp);
93
+ });
94
+ session.on("sessionStarted", ({ sessionId: id, subscribeToken: token }) => {
95
+ sessionId = id;
96
+ subscribeToken = token;
97
+ observability?.sessionStarted(id);
98
+ });
99
+ session.on("generationTick", (e) => emitOrBuffer("generationTick", e));
100
+ session.on("generationEnded", (e) => emitOrBuffer("generationEnded", e));
101
+ session.on("error", (error) => {
102
+ logger.error("Realtime error", { error: error.message });
103
+ emitOrBuffer("error", classifyWebrtcError(error));
104
+ });
105
+ const activeSession = session;
106
+ await activeSession.connect();
134
107
  const client = {
135
- set: methods.set,
136
- setPrompt: methods.setPrompt,
137
- isConnected: () => manager.isConnected(),
138
- getConnectionState: () => manager.getConnectionState(),
108
+ ...realtimeMethods(activeSession, imageToBase64),
109
+ isConnected: () => activeSession.isConnected(),
110
+ getConnectionState: () => activeSession.getConnectionState(),
139
111
  disconnect: () => {
140
- observability.stop();
112
+ observability?.stop();
141
113
  stop();
142
- manager.cleanup();
114
+ activeSession.disconnect();
143
115
  mirroredStream?.dispose();
144
116
  },
145
117
  on: eventEmitter.on,
@@ -150,76 +122,33 @@ const createRealTimeClient = (opts) => {
150
122
  get subscribeToken() {
151
123
  return subscribeToken;
152
124
  },
153
- setImage: async (image, options) => {
154
- if (image === null) return manager.setImage(null, options);
125
+ getSubscribeToken: () => subscribeToken,
126
+ setImage: async (image, imgOptions) => {
127
+ if (isFileRefId(image)) return activeSession.setImage({
128
+ kind: "ref",
129
+ ref: image
130
+ }, imgOptions);
131
+ if (image === null) return activeSession.setImage({
132
+ kind: "data",
133
+ data: null
134
+ }, imgOptions);
155
135
  const base64 = await imageToBase64(image);
156
- return manager.setImage(base64, options);
136
+ return activeSession.setImage({
137
+ kind: "data",
138
+ data: base64
139
+ }, imgOptions);
157
140
  }
158
141
  };
159
142
  flush();
160
143
  return client;
161
144
  } catch (error) {
162
- observability.stop();
163
- webrtcManager?.cleanup();
145
+ observability?.stop();
146
+ session?.disconnect();
164
147
  mirroredStream?.dispose();
165
148
  throw error;
166
149
  }
167
150
  };
168
- const subscribe = async (options) => {
169
- const { sid, ip, port } = decodeSubscribeToken(options.token);
170
- const subscribeUrl = `${baseUrl}/subscribe/${encodeURIComponent(sid)}?IP=${encodeURIComponent(ip)}&port=${encodeURIComponent(port)}&api_key=${encodeURIComponent(apiKey)}`;
171
- const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
172
- let webrtcManager;
173
- const observability = new RealtimeObservability({
174
- telemetryEnabled: opts.telemetryEnabled,
175
- apiKey,
176
- integration,
177
- logger,
178
- onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
179
- onStats: opts.telemetryEnabled ? (stats) => emitOrBuffer("stats", stats) : void 0
180
- });
181
- observability.sessionStarted(sid);
182
- try {
183
- webrtcManager = new WebRTCManager({
184
- webrtcUrl: subscribeUrl,
185
- integration,
186
- logger,
187
- observability,
188
- onRemoteStream: options.onRemoteStream,
189
- onConnectionStateChange: (state) => {
190
- emitOrBuffer("connectionChange", state);
191
- },
192
- onError: (error) => {
193
- logger.error("WebRTC subscribe error", { error: error.message });
194
- emitOrBuffer("error", classifyWebrtcError(error));
195
- }
196
- });
197
- const manager = webrtcManager;
198
- await manager.connect(null);
199
- const client = {
200
- isConnected: () => manager.isConnected(),
201
- getConnectionState: () => manager.getConnectionState(),
202
- disconnect: () => {
203
- observability.stop();
204
- stop();
205
- manager.cleanup();
206
- },
207
- on: eventEmitter.on,
208
- off: eventEmitter.off
209
- };
210
- flush();
211
- return client;
212
- } catch (error) {
213
- observability.stop();
214
- webrtcManager?.cleanup();
215
- throw error;
216
- }
217
- };
218
- return {
219
- connect,
220
- subscribe
221
- };
151
+ return { connect };
222
152
  };
223
-
224
153
  //#endregion
225
- export { createRealTimeClient };
154
+ export { createRealTimeClient };
@@ -0,0 +1,49 @@
1
+ //#region src/realtime/config-realtime.ts
2
+ const REALTIME_CONFIG = {
3
+ signaling: {
4
+ connectTimeoutMs: 6e4,
5
+ handshakeTimeoutMs: 15e3,
6
+ requestTimeoutMs: 3e4
7
+ },
8
+ session: {
9
+ connectionTimeoutMs: 6e4 * 5,
10
+ retry: {
11
+ retries: 5,
12
+ factor: 2,
13
+ minTimeout: 1e3,
14
+ maxTimeout: 1e4
15
+ },
16
+ permanentErrorSubstrings: [
17
+ "permission denied",
18
+ "not allowed",
19
+ "invalid session",
20
+ "401",
21
+ "invalid api key",
22
+ "unauthorized"
23
+ ]
24
+ },
25
+ methods: {
26
+ promptTimeoutMs: 15e3,
27
+ updateTimeoutMs: 3e4
28
+ },
29
+ livekit: {
30
+ inferenceServerIdentityPrefix: "inference-server-",
31
+ roomOptions: {
32
+ adaptiveStream: false,
33
+ dynacast: false
34
+ },
35
+ defaultVideoCodec: "h264",
36
+ defaultMaxVideoBitrateBps: 35e5,
37
+ defaultPublishFps: 30
38
+ },
39
+ observability: {
40
+ stallFpsThreshold: .5,
41
+ statsDefaultIntervalMs: 1e3,
42
+ statsMinIntervalMs: 500,
43
+ telemetryReportIntervalMs: 1e4,
44
+ telemetryUrl: "https://platform.decart.ai/api/v1/telemetry",
45
+ telemetryMaxItemsPerReport: 120
46
+ }
47
+ };
48
+ //#endregion
49
+ export { REALTIME_CONFIG };
@@ -1,5 +1,4 @@
1
1
  import mitt from "mitt";
2
-
3
2
  //#region src/realtime/event-buffer.ts
4
3
  function createEventBuffer() {
5
4
  const emitter = mitt();
@@ -30,6 +29,5 @@ function createEventBuffer() {
30
29
  stop
31
30
  };
32
31
  }
33
-
34
32
  //#endregion
35
- export { createEventBuffer };
33
+ export { createEventBuffer };
@@ -0,0 +1,21 @@
1
+ //#region src/realtime/initial-state-gate.ts
2
+ var InitialStateGate = class {
3
+ attemptId = 0;
4
+ startAttempt(initialState) {
5
+ const attemptId = ++this.attemptId;
6
+ const shouldWait = hasCallerProvidedInitialState(initialState);
7
+ return { waitForReadiness: async (initialStateAck) => {
8
+ if (shouldWait) await initialStateAck;
9
+ return this.attemptId === attemptId;
10
+ } };
11
+ }
12
+ reset() {
13
+ this.attemptId++;
14
+ }
15
+ };
16
+ function hasCallerProvidedInitialState(state) {
17
+ if (!state) return false;
18
+ return state.image !== void 0 && state.image !== null || state.prompt !== void 0 && state.prompt !== null;
19
+ }
20
+ //#endregion
21
+ export { InitialStateGate };
@@ -0,0 +1,82 @@
1
+ import { createConsoleLogger } from "../utils/logger.js";
2
+ import { REALTIME_CONFIG } from "./config-realtime.js";
3
+ import mitt from "mitt";
4
+ import { Room, RoomEvent, Track, TrackEvent } from "livekit-client";
5
+ //#region src/realtime/media-channel.ts
6
+ function getDefaultVideoPublishOptions(videoCodec) {
7
+ const videoEncoding = {
8
+ maxBitrate: REALTIME_CONFIG.livekit.defaultMaxVideoBitrateBps,
9
+ maxFramerate: REALTIME_CONFIG.livekit.defaultPublishFps
10
+ };
11
+ return {
12
+ source: Track.Source.Camera,
13
+ videoCodec: videoCodec ?? REALTIME_CONFIG.livekit.defaultVideoCodec,
14
+ simulcast: true,
15
+ videoEncoding
16
+ };
17
+ }
18
+ var MediaChannel = class {
19
+ room = null;
20
+ remoteStream = null;
21
+ events = mitt();
22
+ logger;
23
+ constructor(config) {
24
+ this.config = config;
25
+ this.logger = config.logger ?? createConsoleLogger("warn");
26
+ }
27
+ get localStream() {
28
+ return this.config.localStream;
29
+ }
30
+ on(event, handler) {
31
+ this.events.on(event, handler);
32
+ }
33
+ off(event, handler) {
34
+ this.events.off(event, handler);
35
+ }
36
+ async connect(opts) {
37
+ this.room ??= new Room(REALTIME_CONFIG.livekit.roomOptions);
38
+ const room = this.room;
39
+ room.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
40
+ if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
41
+ if (track.kind !== Track.Kind.Video && track.kind !== Track.Kind.Audio) return;
42
+ track.attach();
43
+ const mediaStreamTrack = track.mediaStreamTrack;
44
+ if (mediaStreamTrack) {
45
+ this.remoteStream ??= new MediaStream();
46
+ if (!this.remoteStream.getTracks().includes(mediaStreamTrack)) this.remoteStream.addTrack(mediaStreamTrack);
47
+ this.events.emit("remoteStream", this.remoteStream);
48
+ }
49
+ track.on(TrackEvent.VideoPlaybackStarted, () => {
50
+ this.events.emit("firstFrame");
51
+ });
52
+ });
53
+ room.on(RoomEvent.Disconnected, (reason) => {
54
+ this.logger.warn("livekit: room disconnected", { reason });
55
+ this.events.emit("disconnected", { reason });
56
+ });
57
+ this.config.observability?.startPhase("webrtc-handshake");
58
+ await room.connect(opts.url, opts.token);
59
+ this.config.observability?.endPhase("webrtc-handshake", { success: true });
60
+ this.config.observability?.setLiveKitRoom(room);
61
+ }
62
+ async publishLocalTracks() {
63
+ if (!this.config.localStream) return;
64
+ this.config.observability?.startPhase("publish-local-track");
65
+ await this.publishTracks(this.config.localStream);
66
+ this.config.observability?.endPhase("publish-local-track", { success: true });
67
+ }
68
+ disconnect() {
69
+ const room = this.room;
70
+ this.room = null;
71
+ this.remoteStream = null;
72
+ this.config.observability?.setLiveKitRoom(null);
73
+ if (room) room.disconnect().catch(() => {});
74
+ }
75
+ async publishTracks(stream) {
76
+ if (!this.room) return;
77
+ for (const track of stream.getTracks()) if (track.kind === "video") await this.room.localParticipant.publishTrack(track, getDefaultVideoPublishOptions(this.config.videoCodec));
78
+ else await this.room.localParticipant.publishTrack(track);
79
+ }
80
+ };
81
+ //#endregion
82
+ export { MediaChannel };
@@ -1,8 +1,7 @@
1
+ import { isFileRefId } from "../files/types.js";
2
+ import { REALTIME_CONFIG } from "./config-realtime.js";
1
3
  import { z } from "zod";
2
-
3
4
  //#region src/realtime/methods.ts
4
- const PROMPT_TIMEOUT_MS = 15 * 1e3;
5
- const UPDATE_TIMEOUT_MS = 30 * 1e3;
6
5
  const setInputSchema = z.object({
7
6
  prompt: z.string().min(1).optional(),
8
7
  enhance: z.boolean().optional().default(true),
@@ -17,61 +16,44 @@ const setPromptInputSchema = z.object({
17
16
  prompt: z.string().min(1),
18
17
  enhance: z.boolean().optional().default(true)
19
18
  });
20
- const realtimeMethods = (webrtcManager, imageToBase64) => {
21
- const assertConnected = () => {
22
- const state = webrtcManager.getConnectionState();
23
- if (state !== "connected" && state !== "generating") throw new Error(`Cannot send message: connection is ${state}`);
24
- };
19
+ const realtimeMethods = (session, imageToBase64) => {
25
20
  const set = async (input) => {
26
- assertConnected();
27
21
  const parsed = setInputSchema.safeParse(input);
28
22
  if (!parsed.success) throw parsed.error;
29
23
  const { prompt, enhance, image } = parsed.data;
30
- let imageBase64 = null;
31
- if (image !== void 0 && image !== null) imageBase64 = await imageToBase64(image);
32
- await webrtcManager.setImage(imageBase64, {
24
+ const options = {
33
25
  prompt,
34
26
  enhance,
35
- timeout: UPDATE_TIMEOUT_MS
36
- });
27
+ timeout: REALTIME_CONFIG.methods.updateTimeoutMs
28
+ };
29
+ if (isFileRefId(image)) {
30
+ await session.setImage({
31
+ kind: "ref",
32
+ ref: image
33
+ }, options);
34
+ return;
35
+ }
36
+ const imageBase64 = image !== void 0 && image !== null ? await imageToBase64(image) : null;
37
+ await session.setImage({
38
+ kind: "data",
39
+ data: imageBase64
40
+ }, options);
37
41
  };
38
42
  const setPrompt = async (prompt, { enhance } = {}) => {
39
- assertConnected();
40
- const parsedInput = setPromptInputSchema.safeParse({
43
+ const parsed = setPromptInputSchema.safeParse({
41
44
  prompt,
42
45
  enhance
43
46
  });
44
- if (!parsedInput.success) throw parsedInput.error;
45
- const emitter = webrtcManager.getWebsocketMessageEmitter();
46
- let promptAckListener;
47
- let timeoutId;
48
- try {
49
- const ackPromise = new Promise((resolve, reject) => {
50
- promptAckListener = (promptAckMessage) => {
51
- if (promptAckMessage.prompt === parsedInput.data.prompt) if (promptAckMessage.success) resolve();
52
- else reject(new Error(promptAckMessage.error ?? "Failed to send prompt"));
53
- };
54
- emitter.on("promptAck", promptAckListener);
55
- });
56
- if (!webrtcManager.sendMessage({
57
- type: "prompt",
58
- prompt: parsedInput.data.prompt,
59
- enhance_prompt: parsedInput.data.enhance
60
- })) throw new Error("WebSocket is not open");
61
- const timeoutPromise = new Promise((_, reject) => {
62
- timeoutId = setTimeout(() => reject(/* @__PURE__ */ new Error("Prompt timed out")), PROMPT_TIMEOUT_MS);
63
- });
64
- await Promise.race([ackPromise, timeoutPromise]);
65
- } finally {
66
- if (promptAckListener) emitter.off("promptAck", promptAckListener);
67
- if (timeoutId) clearTimeout(timeoutId);
68
- }
47
+ if (!parsed.success) throw parsed.error;
48
+ await session.sendPrompt(parsed.data.prompt, {
49
+ enhance: parsed.data.enhance,
50
+ timeout: REALTIME_CONFIG.methods.promptTimeoutMs
51
+ });
69
52
  };
70
53
  return {
71
54
  set,
72
55
  setPrompt
73
56
  };
74
57
  };
75
-
76
58
  //#endregion
77
- export { realtimeMethods };
59
+ export { realtimeMethods };
@@ -111,6 +111,5 @@ function createWithCanvas(sourceVideo, audioTracks, fps) {
111
111
  }
112
112
  };
113
113
  }
114
-
115
114
  //#endregion
116
- export { createMirroredStream, shouldMirrorTrack };
115
+ export { createMirroredStream, shouldMirrorTrack };
@@ -1,47 +1,17 @@
1
1
  //#region src/realtime/observability/diagnostics.d.ts
2
- /** Connection phase names for timing events. */
3
- type ConnectionPhase = "websocket" | "avatar-image" | "initial-prompt" | "webrtc-handshake" | "total";
4
- type PhaseTimingEvent = {
5
- phase: ConnectionPhase;
2
+ type ClientSessionConnectionBreakdownPhase = {
3
+ phase: string;
6
4
  durationMs: number;
7
5
  success: boolean;
8
6
  error?: string;
9
7
  };
10
- type IceCandidateEvent = {
11
- source: "local" | "remote";
12
- candidateType: "host" | "srflx" | "prflx" | "relay" | "unknown";
13
- protocol: "udp" | "tcp" | "unknown";
14
- address?: string;
15
- port?: number;
16
- };
17
- type IceStateEvent = {
18
- state: string;
19
- previousState: string;
20
- timestampMs: number;
21
- };
22
- type PeerConnectionStateEvent = {
23
- state: string;
24
- previousState: string;
25
- timestampMs: number;
26
- };
27
- type SignalingStateEvent = {
28
- state: string;
29
- previousState: string;
30
- timestampMs: number;
31
- };
32
- type SelectedCandidatePairEvent = {
33
- local: {
34
- candidateType: string;
35
- protocol: string;
36
- address?: string;
37
- port?: number;
38
- };
39
- remote: {
40
- candidateType: string;
41
- protocol: string;
42
- address?: string;
43
- port?: number;
44
- };
8
+ type ClientSessionConnectionBreakdownEvent = {
9
+ attempt: number;
10
+ success: boolean;
11
+ totalDurationMs: number;
12
+ initialImageSizeKb: number | null;
13
+ phases: ClientSessionConnectionBreakdownPhase[];
14
+ error?: string;
45
15
  };
46
16
  type ReconnectEvent = {
47
17
  attempt: number;
@@ -51,28 +21,19 @@ type ReconnectEvent = {
51
21
  error?: string;
52
22
  };
53
23
  type VideoStallEvent = {
54
- /** True when a stall is detected, false when recovered. */
55
24
  stalled: boolean;
56
- /** Duration of the stall in ms (0 when stall first detected, actual duration on recovery). */
57
25
  durationMs: number;
58
26
  };
59
- /** All diagnostic event types keyed by name. */
60
27
  type DiagnosticEvents = {
61
- phaseTiming: PhaseTimingEvent;
62
- iceCandidate: IceCandidateEvent;
63
- iceStateChange: IceStateEvent;
64
- peerConnectionStateChange: PeerConnectionStateEvent;
65
- signalingStateChange: SignalingStateEvent;
66
- selectedCandidatePair: SelectedCandidatePairEvent;
28
+ "client-session-connection-breakdown": ClientSessionConnectionBreakdownEvent;
67
29
  reconnect: ReconnectEvent;
68
30
  videoStall: VideoStallEvent;
69
31
  };
70
32
  type DiagnosticEventName = keyof DiagnosticEvents;
71
- /** A single diagnostic event with its name and typed data. */
72
- type DiagnosticEvent = { [K in DiagnosticEventName]: {
33
+ type DiagnosticEventForName<K extends DiagnosticEventName> = {
73
34
  name: K;
74
35
  data: DiagnosticEvents[K];
75
- } }[DiagnosticEventName];
76
- /** Callback for emitting diagnostic events from the connection/manager layers. */
36
+ };
37
+ type DiagnosticEvent = { [K in DiagnosticEventName]: DiagnosticEventForName<K> }[DiagnosticEventName];
77
38
  //#endregion
78
- export { ConnectionPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, IceCandidateEvent, IceStateEvent, PeerConnectionStateEvent, PhaseTimingEvent, ReconnectEvent, SelectedCandidatePairEvent, SignalingStateEvent, VideoStallEvent };
39
+ export { ClientSessionConnectionBreakdownEvent, ClientSessionConnectionBreakdownPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, ReconnectEvent, VideoStallEvent };