@decartai/sdk 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/README.md +64 -0
  2. package/dist/index.d.ts +28 -2
  3. package/dist/index.js +7 -1
  4. package/dist/process/types.d.ts +1 -1
  5. package/dist/realtime/client.d.ts +8 -1
  6. package/dist/realtime/client.js +14 -3
  7. package/dist/realtime/config-realtime.js +56 -1
  8. package/dist/realtime/media-channel.js +1 -0
  9. package/dist/realtime/mirror-stream.js +41 -31
  10. package/dist/realtime/observability/connection-quality.d.ts +54 -0
  11. package/dist/realtime/observability/connection-quality.js +219 -0
  12. package/dist/realtime/observability/glass-to-glass.d.ts +41 -0
  13. package/dist/realtime/observability/glass-to-glass.js +229 -0
  14. package/dist/realtime/observability/pixel-marker.js +144 -0
  15. package/dist/realtime/observability/realtime-observability.js +51 -1
  16. package/dist/realtime/observability/telemetry-reporter.js +16 -9
  17. package/dist/realtime/observability/webrtc-stats.d.ts +9 -0
  18. package/dist/realtime/observability/webrtc-stats.js +2 -1
  19. package/dist/realtime/preflight.d.ts +59 -0
  20. package/dist/realtime/preflight.js +311 -0
  21. package/dist/realtime/signaling-channel.js +63 -32
  22. package/dist/realtime/stream-session.js +1 -0
  23. package/dist/shared/model.d.ts +22 -3
  24. package/dist/shared/model.js +26 -0
  25. package/package.json +1 -1
package/dist/realtime/signaling-channel.js CHANGED
@@ -3,10 +3,43 @@ import { createConsoleLogger } from "../utils/logger.js";
3
3
  import { REALTIME_CONFIG } from "./config-realtime.js";
4
4
  import mitt from "mitt";
5
5
  //#region src/realtime/signaling-channel.ts
6
+ function buildInitialStateRequest(initialState) {
7
+ if (!initialState) return null;
8
+ if (initialState.imageRef !== void 0 || initialState.image !== void 0) {
9
+ const message = initialState.imageRef !== void 0 ? {
10
+ type: "set_image",
11
+ image_ref: initialState.imageRef
12
+ } : {
13
+ type: "set_image",
14
+ image_data: initialState.image ?? null
15
+ };
16
+ if (initialState.prompt !== void 0) message.prompt = initialState.prompt;
17
+ if (initialState.enhance !== void 0) message.enhance_prompt = initialState.enhance;
18
+ return {
19
+ message,
20
+ matchAck: (msg) => msg.type === "set_image_ack",
21
+ label: "Image send"
22
+ };
23
+ }
24
+ if (initialState.prompt !== void 0 && initialState.prompt !== null) {
25
+ const text = initialState.prompt;
26
+ return {
27
+ message: {
28
+ type: "prompt",
29
+ prompt: text,
30
+ enhance_prompt: initialState.enhance ?? true
31
+ },
32
+ matchAck: (msg) => msg.type === "prompt_ack" && msg.prompt === text,
33
+ label: "Prompt send"
34
+ };
35
+ }
36
+ return null;
37
+ }
6
38
  var SignalingChannel = class {
7
39
  ws = null;
8
40
  events = mitt();
9
41
  pendingAcks = [];
42
+ bufferedAcks = [];
10
43
  pendingRoomInfo = null;
11
44
  connected = false;
12
45
  closing = false;
@@ -29,7 +62,12 @@ var SignalingChannel = class {
29
62
  this.config.observability?.endPhase("websocket-open", { success: true });
30
63
  this.config.observability?.startPhase("room-join");
31
64
  const roomInfoWait = this.waitForRoomInfo(handshakeTimeout);
32
- if (!this.writeMessage({ type: "livekit_join" })) {
65
+ const initialStateRequest = buildInitialStateRequest(opts.initialState);
66
+ const joinMessage = {
67
+ type: "livekit_join",
68
+ initial_state: initialStateRequest ? initialStateRequest.message : null
69
+ };
70
+ if (!this.writeMessage(joinMessage)) {
33
71
  roomInfoWait.cancel();
34
72
  throw new Error("WebSocket is not open");
35
73
  }
@@ -42,18 +80,24 @@ var SignalingChannel = class {
42
80
  }
43
81
  this.config.observability?.endPhase("room-join", { success: true });
44
82
  this.connected = true;
45
- const initialStateAck = this.sendInitialStateTracked(opts.initialState);
83
+ const initialStateAck = initialStateRequest ? this.flushInitialState(initialStateRequest) : Promise.resolve();
46
84
  initialStateAck.catch(() => {});
47
85
  return {
48
86
  roomInfo,
49
87
  initialStateAck
50
88
  };
51
89
  }
52
- async sendInitialStateTracked(initialState) {
53
- if (!initialState) return;
90
+ async flushInitialState(request) {
54
91
  this.config.observability?.startPhase("initial-state-handshake");
55
- await this.sendInitialState(initialState);
92
+ const ack = await this.request({
93
+ message: request.message,
94
+ matchAck: request.matchAck,
95
+ timeoutMs: REALTIME_CONFIG.signaling.requestTimeoutMs,
96
+ label: request.label,
97
+ write: false
98
+ });
56
99
  this.config.observability?.endPhase("initial-state-handshake", { success: true });
100
+ if (!ack.success) throw new Error(ack.error ?? `Failed: ${request.label}`);
57
101
  }
58
102
  close() {
59
103
  this.closing = true;
@@ -180,32 +224,14 @@ var SignalingChannel = class {
180
224
  cancel: cleanup
181
225
  };
182
226
  }
183
- async sendInitialState(initialState) {
184
- if (!initialState) return;
185
- if (initialState.imageRef !== void 0) {
186
- await this.setImage({
187
- kind: "ref",
188
- ref: initialState.imageRef
189
- }, {
190
- prompt: initialState.prompt,
191
- enhance: initialState.enhance
192
- });
193
- return;
194
- }
195
- if (initialState.image !== void 0) {
196
- await this.setImage({
197
- kind: "data",
198
- data: initialState.image
199
- }, {
200
- prompt: initialState.prompt,
201
- enhance: initialState.enhance
202
- });
203
- return;
204
- }
205
- if (initialState.prompt !== void 0 && initialState.prompt !== null) await this.sendPrompt(initialState.prompt, { enhance: initialState.enhance });
206
- }
207
- async request({ message, matchAck, timeoutMs, label }) {
227
+ async request({ message, matchAck, timeoutMs, label, write = true }) {
208
228
  return new Promise((resolve, reject) => {
229
+ const buffered = this.bufferedAcks.findIndex((m) => matchAck(m));
230
+ if (buffered !== -1) {
231
+ const [claimed] = this.bufferedAcks.splice(buffered, 1);
232
+ resolve(claimed);
233
+ return;
234
+ }
209
235
  const timer = setTimeout(() => {
210
236
  cleanup();
211
237
  this.logger.warn("signaling: ack timed out", {
@@ -230,7 +256,7 @@ var SignalingChannel = class {
230
256
  this.pendingAcks = this.pendingAcks.filter((e) => e !== entry);
231
257
  };
232
258
  this.pendingAcks.push(entry);
233
- if (!this.writeMessage(message)) {
259
+ if (write && !this.writeMessage(message)) {
234
260
  cleanup();
235
261
  reject(/* @__PURE__ */ new Error("WebSocket is not open"));
236
262
  }
@@ -244,7 +270,11 @@ var SignalingChannel = class {
244
270
  handleMessage(msg) {
245
271
  for (const ack of [...this.pendingAcks]) if (ack.matches(msg)) {
246
272
  ack.onMatch(msg);
247
- break;
273
+ return;
274
+ }
275
+ if (!this.connected && (msg.type === "set_image_ack" || msg.type === "prompt_ack")) {
276
+ this.bufferedAcks.push(msg);
277
+ return;
248
278
  }
249
279
  switch (msg.type) {
250
280
  case "livekit_room_info":
@@ -298,6 +328,7 @@ var SignalingChannel = class {
298
328
  rejectAllPending(error) {
299
329
  const pending = this.pendingAcks;
300
330
  this.pendingAcks = [];
331
+ this.bufferedAcks = [];
301
332
  for (const entry of pending) entry.reject(error);
302
333
  }
303
334
  };
package/dist/realtime/stream-session.js CHANGED
@@ -101,6 +101,7 @@ var StreamSession = class {
101
101
  this.resetHandshakeState();
102
102
  const initialState = this.getInitialState();
103
103
  this.config.observability?.beginConnectionBreakdown(attempt, getInitialImageSizeKb(initialState?.image));
104
+ this.config.observability?.markGlassToGlassStart();
104
105
  const gateAttempt = this.initialStateGate.startAttempt(initialState);
105
106
  const { roomInfo, initialStateAck } = await this.signaling.openAndJoin({
106
107
  connectTimeout: REALTIME_CONFIG.session.connectionTimeoutMs,
package/dist/shared/model.d.ts CHANGED
@@ -6,6 +6,7 @@ declare const canonicalModelSchema: z.ZodEnum<{
6
6
  "lucy-2.1": "lucy-2.1";
7
7
  "lucy-2.1-vton": "lucy-2.1-vton";
8
8
  "lucy-vton-2": "lucy-vton-2";
9
+ "lucy-vton-3": "lucy-vton-3";
9
10
  "lucy-restyle-2": "lucy-restyle-2";
10
11
  "lucy-clip": "lucy-clip";
11
12
  "lucy-image-2": "lucy-image-2";
@@ -25,10 +26,10 @@ declare const modelAliases: {
25
26
  };
26
27
  /** @internal Test-only helper to reset deprecation warning tracking */
27
28
 
28
- declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">]>;
29
- declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>;
29
+ declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">]>;
30
+ declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>;
30
31
  declare const imageModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>;
31
- declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
32
+ declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
32
33
  type Model = z.infer<typeof modelSchema>;
33
34
  type RealTimeModels = z.infer<typeof realtimeModels>;
34
35
  type VideoModels = z.infer<typeof videoModels>;
@@ -150,6 +151,22 @@ declare const modelInputSchemas: {
150
151
  resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
151
152
  enhance_prompt: z.ZodOptional<z.ZodBoolean>;
152
153
  }, z.core.$strip>;
154
+ readonly "lucy-vton-3": z.ZodObject<{
155
+ prompt: z.ZodString;
156
+ reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
157
+ uri: z.ZodString;
158
+ type: z.ZodString;
159
+ name: z.ZodString;
160
+ }, z.core.$strip>]>>;
161
+ data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
162
+ uri: z.ZodString;
163
+ type: z.ZodString;
164
+ name: z.ZodString;
165
+ }, z.core.$strip>]>;
166
+ seed: z.ZodOptional<z.ZodNumber>;
167
+ resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
168
+ enhance_prompt: z.ZodOptional<z.ZodBoolean>;
169
+ }, z.core.$strip>;
153
170
  readonly "lucy-latest": z.ZodObject<{
154
171
  prompt: z.ZodString;
155
172
  reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
@@ -380,6 +397,7 @@ declare const models: {
380
397
  * - `"lucy-2.1"` - Lucy 2.1 realtime video editing
381
398
  * - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on
382
399
  * - `"lucy-vton-2"` - Lucy virtual try-on 2
400
+ * - `"lucy-vton-3"` - Lucy virtual try-on 3
383
401
  * - `"lucy-restyle-2"` - Realtime video restyling
384
402
  */
385
403
  realtime: <T extends RealTimeModels>(model: T) => ModelDefinition<T>;
@@ -391,6 +409,7 @@ declare const models: {
391
409
  * - `"lucy-2.1"` - Long-form video editing (Lucy 2.1)
392
410
  * - `"lucy-2.1-vton"` - Virtual try-on video editing
393
411
  * - `"lucy-vton-2"` - Virtual try-on 2 video editing
412
+ * - `"lucy-vton-3"` - Virtual try-on 3 video editing
394
413
  * - `"lucy-restyle-2"` - Video restyling
395
414
  */
396
415
  video: <T extends VideoModels>(model: T) => ModelDefinition<T> & {
package/dist/shared/model.js CHANGED
@@ -5,6 +5,7 @@ const CANONICAL_MODEL_NAMES = [
5
5
  "lucy-2.1",
6
6
  "lucy-2.1-vton",
7
7
  "lucy-vton-2",
8
+ "lucy-vton-3",
8
9
  "lucy-restyle-2",
9
10
  "lucy-clip",
10
11
  "lucy-image-2"
@@ -13,6 +14,7 @@ const CANONICAL_REALTIME_MODEL_NAMES = [
13
14
  "lucy-2.1",
14
15
  "lucy-2.1-vton",
15
16
  "lucy-vton-2",
17
+ "lucy-vton-3",
16
18
  "lucy-restyle-2"
17
19
  ];
18
20
  const CANONICAL_VIDEO_MODEL_NAMES = [
@@ -20,6 +22,7 @@ const CANONICAL_VIDEO_MODEL_NAMES = [
20
22
  "lucy-2.1",
21
23
  "lucy-2.1-vton",
22
24
  "lucy-vton-2",
25
+ "lucy-vton-3",
23
26
  "lucy-restyle-2"
24
27
  ];
25
28
  const CANONICAL_IMAGE_MODEL_NAMES = ["lucy-image-2"];
@@ -51,6 +54,7 @@ const realtimeModels = z.union([
51
54
  z.literal("lucy-2.1"),
52
55
  z.literal("lucy-2.1-vton"),
53
56
  z.literal("lucy-vton-2"),
57
+ z.literal("lucy-vton-3"),
54
58
  z.literal("lucy-restyle-2"),
55
59
  z.literal("lucy-latest"),
56
60
  z.literal("lucy-vton-latest"),
@@ -64,6 +68,7 @@ const videoModels = z.union([
64
68
  z.literal("lucy-2.1"),
65
69
  z.literal("lucy-2.1-vton"),
66
70
  z.literal("lucy-vton-2"),
71
+ z.literal("lucy-vton-3"),
67
72
  z.literal("lucy-restyle-2"),
68
73
  z.literal("lucy-latest"),
69
74
  z.literal("lucy-vton-latest"),
@@ -183,6 +188,7 @@ const modelInputSchemas = {
183
188
  "lucy-2.1": videoEdit2Schema,
184
189
  "lucy-2.1-vton": videoEdit2Schema,
185
190
  "lucy-vton-2": videoEdit2Schema,
191
+ "lucy-vton-3": videoEdit2Schema,
186
192
  "lucy-latest": videoEdit2Schema,
187
193
  "lucy-vton-latest": videoEdit2Schema,
188
194
  "lucy-restyle-latest": restyleSchema,
@@ -247,6 +253,17 @@ const _models = {
247
253
  height: 624,
248
254
  inputSchema: z.object({})
249
255
  },
256
+ "lucy-vton-3": {
257
+ urlPath: "/v1/stream",
258
+ name: "lucy-vton-3",
259
+ fps: {
260
+ ideal: 30,
261
+ max: 30
262
+ },
263
+ width: 1088,
264
+ height: 624,
265
+ inputSchema: z.object({})
266
+ },
250
267
  "lucy-restyle-2": {
251
268
  urlPath: "/v1/stream",
252
269
  name: "lucy-restyle-2",
@@ -391,6 +408,15 @@ const _models = {
391
408
  height: 624,
392
409
  inputSchema: modelInputSchemas["lucy-vton-2"]
393
410
  },
411
+ "lucy-vton-3": {
412
+ urlPath: "/v1/generate/lucy-vton-3",
413
+ queueUrlPath: "/v1/jobs/lucy-vton-3",
414
+ name: "lucy-vton-3",
415
+ fps: 20,
416
+ width: 1088,
417
+ height: 624,
418
+ inputSchema: modelInputSchemas["lucy-vton-3"]
419
+ },
394
420
  "lucy-restyle-2": {
395
421
  urlPath: "/v1/generate/lucy-restyle-2",
396
422
  queueUrlPath: "/v1/jobs/lucy-restyle-2",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@decartai/sdk",
3
- "version": "0.1.4",
3
+ "version": "0.1.6",
4
4
  "description": "Decart's JavaScript SDK",
5
5
  "type": "module",
6
6
  "license": "MIT",