@decartai/sdk 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -21
- package/dist/package.js +1 -1
- package/dist/process/types.d.ts +1 -1
- package/dist/realtime/browser/frame-metadata-diagnostics.js +3 -12
- package/dist/realtime/browser/index.js +1 -4
- package/dist/realtime/browser/preflight.js +1 -0
- package/dist/realtime/browser/prepare-connection.js +4 -5
- package/dist/realtime/client.js +7 -3
- package/dist/realtime/config-realtime.js +1 -2
- package/dist/realtime/media-channel.js +1 -3
- package/dist/realtime/observability/connection-quality.d.ts +4 -4
- package/dist/realtime/observability/webrtc-stats.d.ts +3 -2
- package/dist/realtime/react-native/prepare-connection.js +2 -1
- package/dist/realtime/stream-session.js +3 -6
- package/dist/realtime/subscribe-client.js +4 -41
- package/dist/shared/model.d.ts +22 -3
- package/dist/shared/model.js +32 -4
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -88,18 +88,6 @@ A connected realtime session exposes an SDK `subscribeToken` once it reaches a
|
|
|
88
88
|
connected state. Share that SDK token with viewers — `client.realtime.subscribe`
|
|
89
89
|
uses it to request receive-only LiveKit credentials from Decart, then connects to
|
|
90
90
|
the LiveKit room for the styled output stream. No viewer camera is required.
|
|
91
|
-
Browser-produced streams can include LiveKit frame metadata for glass-to-glass
|
|
92
|
-
latency, so viewers also need a browser runtime that can create the SDK's
|
|
93
|
-
frame-metadata worker; unsupported runtimes reject those tokens before joining.
|
|
94
|
-
|
|
95
|
-
> **Viewers must be on this SDK version or newer.** A browser publisher now
|
|
96
|
-
> advertises frame timing by default, which makes the server append a packet
|
|
97
|
-
> trailer to every frame in the room. Viewers on an older `@decartai/sdk`
|
|
98
|
-
> ignore the token's `frame_timing` flag and join without a strip worker — they
|
|
99
|
-
> connect successfully but decode nothing, so the video element stays black with
|
|
100
|
-
> no error raised. React Native viewers cannot strip trailers at all and are
|
|
101
|
-
> rejected with `UNSUPPORTED_PLATFORM_FEATURE`. Upgrade viewers before upgrading
|
|
102
|
-
> publishers.
|
|
103
91
|
|
|
104
92
|
**Producer** — capture the token from the active session:
|
|
105
93
|
|
|
@@ -187,12 +175,12 @@ realtimeClient.getConnectionQuality(); // latest report, or null before the firs
|
|
|
187
175
|
> setInterval(() => console.log(realtimeClient.getConnectionQuality()?.metrics.g2gMs), 1000);
|
|
188
176
|
> ```
|
|
189
177
|
|
|
190
|
-
**Glass-to-glass latency.** Network RTT hides the dominant cost in
|
|
191
|
-
inference — so a session can read "good" while actually feeling laggy.
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
This surfaces **startup** (`ttffMs`) and **steady-state**
|
|
195
|
-
glass-to-glass drives the latency verdict instead of RTT.
|
|
178
|
+
**Glass-to-glass latency (opt-in, diagnostic).** Network RTT hides the dominant cost in
|
|
179
|
+
real-time video — model inference — so a session can read "good" while actually feeling laggy.
|
|
180
|
+
Set `debugQuality: true` to measure the *real* camera→display latency. The SDK attaches a capture
|
|
181
|
+
timestamp using LiveKit frame metadata; the server propagates it through inference and the SDK
|
|
182
|
+
matches it to output playout. This surfaces **startup** (`ttffMs`) and **steady-state**
|
|
183
|
+
(`g2gMs`) latency. When present, glass-to-glass drives the latency verdict instead of RTT.
|
|
196
184
|
|
|
197
185
|
> Frame metadata is currently experimental in LiveKit and requires encoded-transform support.
|
|
198
186
|
> It does not alter visible pixels. `g2gDropRatio` remains `null` until frame IDs are propagated
|
|
@@ -201,6 +189,7 @@ glass-to-glass drives the latency verdict instead of RTT.
|
|
|
201
189
|
```typescript
|
|
202
190
|
const realtimeClient = await client.realtime.connect(stream, {
|
|
203
191
|
model,
|
|
192
|
+
debugQuality: true,
|
|
204
193
|
});
|
|
205
194
|
|
|
206
195
|
// g2g updates every stats tick — read it from the `stats` event (the
|
|
@@ -349,9 +338,9 @@ Add them to `app.json`:
|
|
|
349
338
|
Run `npx expo prebuild` and rebuild the native app. LiveKit does not run in Expo
|
|
350
339
|
Go. Bare React Native apps must follow LiveKit's native setup instructions.
|
|
351
340
|
|
|
352
|
-
Outgoing `mirror` and deep connectivity preflight are browser-only
|
|
353
|
-
`UNSUPPORTED_PLATFORM_FEATURE` on React Native. Mirror the local
|
|
354
|
-
native video view instead.
|
|
341
|
+
Outgoing `mirror`, `debugQuality`, and deep connectivity preflight are browser-only
|
|
342
|
+
and fail with `UNSUPPORTED_PLATFORM_FEATURE` on React Native. Mirror the local
|
|
343
|
+
preview with your native video view instead.
|
|
355
344
|
|
|
356
345
|
## Development
|
|
357
346
|
|
package/dist/package.js
CHANGED
package/dist/process/types.d.ts
CHANGED
|
@@ -143,7 +143,7 @@ interface PromptInput {
|
|
|
143
143
|
* This allows different models to have field-specific documentation while maintaining type safety.
|
|
144
144
|
* Specific models are checked first, then falls back to category-based selection.
|
|
145
145
|
*/
|
|
146
|
-
type ModelSpecificInputs<T extends ModelDefinition> = T["name"] extends "lucy-pro-i2i" | "lucy-image-2" ? ImageEditingInputs : T["name"] extends "lucy-restyle-v2v" | "lucy-restyle-2" ? VideoRestyleInputs : T["name"] extends "lucy-2.1" | "lucy-2.5" | "lucy-vton-2" | "lucy-vton-3" | "lucy-2.1-vton-2" | "lucy-vton-latest" ? VideoEdit2Inputs : T["name"] extends "lucy-pro-v2v" | "lucy-clip" ? VideoEditInputs : T["name"] extends ImageModels ? ImageGenerationInputs : T["name"] extends VideoModels ? VideoModelInputs : PromptInput;
|
|
146
|
+
type ModelSpecificInputs<T extends ModelDefinition> = T["name"] extends "lucy-pro-i2i" | "lucy-image-2" ? ImageEditingInputs : T["name"] extends "lucy-restyle-v2v" | "lucy-restyle-2" ? VideoRestyleInputs : T["name"] extends "lucy-2.1" | "lucy-2.5" | "lucy-vton-2" | "lucy-vton-3" | "lucy-vton-3.5" | "lucy-2.1-vton-2" | "lucy-vton-latest" ? VideoEdit2Inputs : T["name"] extends "lucy-pro-v2v" | "lucy-clip" ? VideoEditInputs : T["name"] extends ImageModels ? ImageGenerationInputs : T["name"] extends VideoModels ? VideoModelInputs : PromptInput;
|
|
147
147
|
interface ProcessInputs {
|
|
148
148
|
/**
|
|
149
149
|
* Random seed for reproducible results.
|
|
@@ -52,8 +52,8 @@ const TIME_SYNC_UPDATE = "timeSyncUpdate";
|
|
|
52
52
|
function createFrameReader(tracker) {
|
|
53
53
|
let attachedTrack = null;
|
|
54
54
|
const onTimeSyncUpdate = ({ timestamp, rtpTimestamp }) => {
|
|
55
|
-
const frameMetadata = attachedTrack?.lookupFrameMetadata
|
|
56
|
-
if (frameMetadata) tracker.recordFrame(frameMetadata.userTimestamp, timestamp);
|
|
55
|
+
const frameMetadata = attachedTrack?.lookupFrameMetadata({ rtpTimestamp });
|
|
56
|
+
if (frameMetadata) tracker.recordFrame(frameMetadata.userTimestamp, performance.timeOrigin + timestamp);
|
|
57
57
|
};
|
|
58
58
|
const detach = () => {
|
|
59
59
|
attachedTrack?.off(TIME_SYNC_UPDATE, onTimeSyncUpdate);
|
|
@@ -86,14 +86,5 @@ function createBrowserFrameMetadataDiagnostics() {
|
|
|
86
86
|
function createFrameMetadataWorker() {
|
|
87
87
|
return new Worker(new URL("./frame-metadata-worker.js", import.meta.url));
|
|
88
88
|
}
|
|
89
|
-
function isFrameMetadataRuntimeSupported() {
|
|
90
|
-
if (typeof window === "undefined") return false;
|
|
91
|
-
const maybeWindow = window;
|
|
92
|
-
const userAgent = window.navigator?.userAgent?.toLowerCase() ?? "";
|
|
93
|
-
const isChromiumBased = /(?:chrome|chromium|crmo)\//.test(userAgent) && !/crios\//.test(userAgent);
|
|
94
|
-
const scriptTransformSupported = typeof maybeWindow.RTCRtpScriptTransform !== "undefined" && !isChromiumBased;
|
|
95
|
-
const insertableStreamsSupported = typeof maybeWindow.RTCRtpSender?.prototype?.createEncodedStreams !== "undefined" && typeof maybeWindow.RTCRtpReceiver?.prototype?.createEncodedStreams !== "undefined";
|
|
96
|
-
return scriptTransformSupported || insertableStreamsSupported;
|
|
97
|
-
}
|
|
98
89
|
//#endregion
|
|
99
|
-
export { FrameMetadataTracker, createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker
|
|
90
|
+
export { FrameMetadataTracker, createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker };
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { createRealTimeClient } from "../client.js";
|
|
2
2
|
import { createRealTimeSubscribeClient } from "../subscribe-client.js";
|
|
3
|
-
import { createFrameMetadataWorker, isFrameMetadataRuntimeSupported } from "./frame-metadata-diagnostics.js";
|
|
4
3
|
import { createPreflight } from "./preflight.js";
|
|
5
4
|
import { prepareBrowserConnection } from "./prepare-connection.js";
|
|
6
5
|
//#region src/realtime/browser/index.ts
|
|
@@ -17,9 +16,7 @@ const createBrowserRealtime = (options) => {
|
|
|
17
16
|
baseUrl: options.subscribeBaseUrl,
|
|
18
17
|
apiKey: options.apiKey,
|
|
19
18
|
integration: options.integration,
|
|
20
|
-
logger: options.logger
|
|
21
|
-
createFrameMetadataWorker,
|
|
22
|
-
isFrameMetadataRuntimeSupported
|
|
19
|
+
logger: options.logger
|
|
23
20
|
});
|
|
24
21
|
const preflight = createPreflight({
|
|
25
22
|
logger: options.logger,
|
|
@@ -166,6 +166,7 @@ async function runActiveProbe(args) {
|
|
|
166
166
|
source = createSyntheticSource(model.width, model.height, resolveFpsNumber(model.fps));
|
|
167
167
|
const connectTask = connect(source.stream, {
|
|
168
168
|
model,
|
|
169
|
+
debugQuality: true,
|
|
169
170
|
onRemoteStream: () => {}
|
|
170
171
|
});
|
|
171
172
|
signal?.addEventListener("abort", () => connectTask.then((c) => c.disconnect()).catch(() => {}), { once: true });
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { RealtimeObservability } from "../observability/realtime-observability.js";
|
|
2
2
|
import { isDesktopSafari } from "../../utils/platform.js";
|
|
3
3
|
import { createLiveKitMediaChannel } from "../media-channel.js";
|
|
4
|
-
import { createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker
|
|
4
|
+
import { createBrowserFrameMetadataDiagnostics, createFrameMetadataWorker } from "./frame-metadata-diagnostics.js";
|
|
5
5
|
import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
|
|
6
6
|
//#region src/realtime/browser/prepare-connection.ts
|
|
7
|
-
const prepareBrowserConnection = ({ stream, mirror, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
|
|
7
|
+
const prepareBrowserConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, fps, logger, observability: observabilityOptions }) => {
|
|
8
8
|
let inputStream = stream ?? new MediaStream();
|
|
9
9
|
let disposeMirroring = () => {};
|
|
10
10
|
if (mirror !== false) try {
|
|
@@ -18,12 +18,11 @@ const prepareBrowserConnection = ({ stream, mirror, preferredVideoCodec, fps, lo
|
|
|
18
18
|
logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
|
|
19
19
|
}
|
|
20
20
|
let pendingWorker;
|
|
21
|
-
if (
|
|
21
|
+
if (debugQuality) try {
|
|
22
22
|
pendingWorker = createFrameMetadataWorker();
|
|
23
23
|
} catch (error) {
|
|
24
|
-
logger.
|
|
24
|
+
logger.warn("Frame-metadata worker unavailable; glass-to-glass latency measurement disabled", { error: error instanceof Error ? error.message : String(error) });
|
|
25
25
|
}
|
|
26
|
-
else logger.debug("Frame-metadata runtime unavailable; glass-to-glass latency measurement disabled");
|
|
27
26
|
const frameTiming = pendingWorker !== void 0;
|
|
28
27
|
const takeFrameMetadataWorker = () => {
|
|
29
28
|
if (pendingWorker) {
|
package/dist/realtime/client.js
CHANGED
|
@@ -27,9 +27,11 @@ const realTimeClientConnectOptionsSchema = z.object({
|
|
|
27
27
|
"vp9"
|
|
28
28
|
]).optional(),
|
|
29
29
|
/**
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* Opt-in quality measurement using LiveKit frame metadata. The capture
|
|
31
|
+
* timestamp is propagated through inference and matched to output playout to
|
|
32
|
+
* surface true glass-to-glass `g2gMs` / `ttffMs` on the `stats` and
|
|
33
|
+
* `connectionQuality` signals. Browser-only and experimental because it
|
|
34
|
+
* relies on LiveKit's frame-metadata worker and encoded transforms.
|
|
33
35
|
*/
|
|
34
36
|
debugQuality: z.boolean().optional()
|
|
35
37
|
});
|
|
@@ -41,6 +43,7 @@ const createRealTimeClient = (opts) => {
|
|
|
41
43
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
42
44
|
const { onRemoteStream, onConnectionChange, onConnectionQuality, onQueuePosition, initialState, resolution, preferredVideoCodec } = parsedOptions.data;
|
|
43
45
|
const mirror = parsedOptions.data.mirror ?? false;
|
|
46
|
+
const debugQuality = parsedOptions.data.debugQuality ?? false;
|
|
44
47
|
let session;
|
|
45
48
|
let observability;
|
|
46
49
|
let preparedConnection;
|
|
@@ -56,6 +59,7 @@ const createRealTimeClient = (opts) => {
|
|
|
56
59
|
preparedConnection = opts.prepareConnection({
|
|
57
60
|
stream,
|
|
58
61
|
mirror,
|
|
62
|
+
debugQuality,
|
|
59
63
|
preferredVideoCodec,
|
|
60
64
|
fps: resolveFpsNumber(options.model.fps),
|
|
61
65
|
logger,
|
|
@@ -46,9 +46,7 @@ var LiveKitMediaChannel = class {
|
|
|
46
46
|
if (this.config.createFrameMetadataWorker) try {
|
|
47
47
|
worker = this.config.createFrameMetadataWorker();
|
|
48
48
|
} catch (error) {
|
|
49
|
-
|
|
50
|
-
this.logger.warn("Failed to create LiveKit frame-metadata worker", { error: detail });
|
|
51
|
-
throw new Error(`Failed to create LiveKit frame-metadata worker: ${detail}`, { cause: error });
|
|
49
|
+
this.logger.warn("Failed to create LiveKit frame-metadata worker; continuing without latency metrics", { error: error instanceof Error ? error.message : String(error) });
|
|
52
50
|
}
|
|
53
51
|
this.frameMetadataEnabled = worker !== void 0;
|
|
54
52
|
try {
|
|
@@ -16,10 +16,10 @@ type ConnectionQualityMetrics = {
|
|
|
16
16
|
rttMs: number | null;
|
|
17
17
|
/**
|
|
18
18
|
* Mid-stream (steady-state) glass-to-glass latency (ms) — the real per-frame
|
|
19
|
-
* camera→display latency through the model, excluding startup.
|
|
20
|
-
*
|
|
21
|
-
* null otherwise. When present it drives the latency verdict
|
|
22
|
-
* `rttMs`.
|
|
19
|
+
* camera→display latency through the model, excluding startup. Only populated
|
|
20
|
+
* when the opt-in frame-metadata measurement is on (`connect({ debugQuality: true })`)
|
|
21
|
+
* and past warm-up; null otherwise. When present it drives the latency verdict
|
|
22
|
+
* instead of `rttMs`.
|
|
23
23
|
*/
|
|
24
24
|
g2gMs: number | null;
|
|
25
25
|
/**
|
|
@@ -133,8 +133,9 @@ type WebRTCStats = {
|
|
|
133
133
|
};
|
|
134
134
|
/**
|
|
135
135
|
* True glass-to-glass latency + end-to-end drop signal, merged in by
|
|
136
|
-
* `RealtimeObservability` when frame-metadata measurement is active.
|
|
137
|
-
* otherwise — the stats collector does not
|
|
136
|
+
* `RealtimeObservability` when the opt-in frame-metadata measurement is active.
|
|
137
|
+
* Null otherwise — the stats collector does not
|
|
138
|
+
* populate it.
|
|
138
139
|
*/
|
|
139
140
|
glassToGlass: G2GMetrics | null;
|
|
140
141
|
};
|
|
@@ -10,9 +10,10 @@ function assertReactNativeReady() {
|
|
|
10
10
|
function unsupportedReactNativeFeature(feature) {
|
|
11
11
|
throw createUnsupportedPlatformFeatureError(feature, "React Native");
|
|
12
12
|
}
|
|
13
|
-
const prepareReactNativeConnection = ({ stream, mirror, preferredVideoCodec, observability: observabilityOptions }) => {
|
|
13
|
+
const prepareReactNativeConnection = ({ stream, mirror, debugQuality, preferredVideoCodec, observability: observabilityOptions }) => {
|
|
14
14
|
assertReactNativeReady();
|
|
15
15
|
if (mirror !== false) unsupportedReactNativeFeature("Outgoing video mirroring");
|
|
16
|
+
if (debugQuality) unsupportedReactNativeFeature("debugQuality");
|
|
16
17
|
return {
|
|
17
18
|
stream: stream ?? new MediaStream(),
|
|
18
19
|
observability: new RealtimeObservability(observabilityOptions),
|
|
@@ -4,11 +4,8 @@ import { SignalingChannel } from "./signaling-channel.js";
|
|
|
4
4
|
import mitt from "mitt";
|
|
5
5
|
import pRetry, { AbortError } from "p-retry";
|
|
6
6
|
//#region src/realtime/stream-session.ts
|
|
7
|
-
function encodeSubscribeToken(roomName
|
|
8
|
-
return btoa(JSON.stringify({
|
|
9
|
-
room_name: roomName,
|
|
10
|
-
...options.frameTiming ? { frame_timing: true } : {}
|
|
11
|
-
}));
|
|
7
|
+
function encodeSubscribeToken(roomName) {
|
|
8
|
+
return btoa(JSON.stringify({ room_name: roomName }));
|
|
12
9
|
}
|
|
13
10
|
function getInitialImageSizeKb(image) {
|
|
14
11
|
if (!image) return null;
|
|
@@ -140,7 +137,7 @@ var StreamSession = class {
|
|
|
140
137
|
this.setState("connected");
|
|
141
138
|
this.events.emit("sessionStarted", {
|
|
142
139
|
sessionId: roomInfo.sessionId,
|
|
143
|
-
subscribeToken: encodeSubscribeToken(roomInfo.roomName
|
|
140
|
+
subscribeToken: encodeSubscribeToken(roomInfo.roomName)
|
|
144
141
|
});
|
|
145
142
|
} catch (error) {
|
|
146
143
|
this.config.observability?.finishConnectionBreakdown({
|
|
@@ -1,28 +1,15 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
2
|
import { createConsoleLogger } from "../utils/logger.js";
|
|
3
3
|
import { createEventBuffer } from "./event-buffer.js";
|
|
4
4
|
import { REALTIME_CONFIG } from "./config-realtime.js";
|
|
5
5
|
import { loadLiveKitClient } from "./livekit.js";
|
|
6
6
|
import { RealtimeObservability } from "./observability/realtime-observability.js";
|
|
7
7
|
//#region src/realtime/subscribe-client.ts
|
|
8
|
-
function isDecartSDKError(error) {
|
|
9
|
-
return typeof error === "object" && error !== null && typeof error.code === "string" && typeof error.message === "string";
|
|
10
|
-
}
|
|
11
|
-
function createFrameMetadataSubscribeUnsupportedError(detail) {
|
|
12
|
-
const suffix = detail ? `: ${detail}` : "";
|
|
13
|
-
return createSDKError(ERROR_CODES.UNSUPPORTED_PLATFORM_FEATURE, `This realtime stream requires LiveKit frame metadata, which this SDK environment does not support${suffix}.`, {
|
|
14
|
-
feature: "LiveKit frame metadata subscribe",
|
|
15
|
-
platform: "this SDK environment"
|
|
16
|
-
});
|
|
17
|
-
}
|
|
18
8
|
function decodeSubscribeToken(token) {
|
|
19
9
|
try {
|
|
20
10
|
const payload = JSON.parse(atob(token));
|
|
21
11
|
if (!payload.room_name || typeof payload.room_name !== "string") throw new Error("Invalid subscribe token format");
|
|
22
|
-
return {
|
|
23
|
-
room_name: payload.room_name,
|
|
24
|
-
...payload.frame_timing === true ? { frame_timing: true } : {}
|
|
25
|
-
};
|
|
12
|
+
return { room_name: payload.room_name };
|
|
26
13
|
} catch {
|
|
27
14
|
throw new Error("Invalid subscribe token");
|
|
28
15
|
}
|
|
@@ -63,11 +50,10 @@ const createRealTimeSubscribeClient = (opts) => {
|
|
|
63
50
|
const { baseUrl, apiKey, integration } = opts;
|
|
64
51
|
const logger = opts.logger ?? createConsoleLogger("info");
|
|
65
52
|
const subscribe = async (options) => {
|
|
66
|
-
const { room_name: roomName
|
|
53
|
+
const { room_name: roomName } = decodeSubscribeToken(options.token);
|
|
67
54
|
const { emitter, emitOrBuffer, flush, stop } = createEventBuffer();
|
|
68
55
|
let observability;
|
|
69
56
|
let room;
|
|
70
|
-
let frameMetadataWorker;
|
|
71
57
|
let currentState = "connecting";
|
|
72
58
|
let remoteStream = null;
|
|
73
59
|
const setState = (state) => {
|
|
@@ -91,25 +77,7 @@ const createRealTimeSubscribeClient = (opts) => {
|
|
|
91
77
|
apiKey,
|
|
92
78
|
roomName
|
|
93
79
|
});
|
|
94
|
-
|
|
95
|
-
if (!opts.createFrameMetadataWorker) throw createFrameMetadataSubscribeUnsupportedError("this platform has no frame-metadata worker");
|
|
96
|
-
if (!(opts.isFrameMetadataRuntimeSupported?.() ?? false)) throw createFrameMetadataSubscribeUnsupportedError("encoded transforms are unavailable");
|
|
97
|
-
try {
|
|
98
|
-
frameMetadataWorker = opts.createFrameMetadataWorker();
|
|
99
|
-
} catch (error) {
|
|
100
|
-
throw createFrameMetadataSubscribeUnsupportedError(`failed to create the required worker (${error instanceof Error ? error.message : String(error)})`);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
try {
|
|
104
|
-
room = new LiveKitRoom({
|
|
105
|
-
...REALTIME_CONFIG.livekit.roomOptions,
|
|
106
|
-
...frameMetadataWorker ? { frameMetadata: { worker: frameMetadataWorker } } : {}
|
|
107
|
-
});
|
|
108
|
-
} catch (error) {
|
|
109
|
-
frameMetadataWorker?.terminate();
|
|
110
|
-
frameMetadataWorker = void 0;
|
|
111
|
-
throw error;
|
|
112
|
-
}
|
|
80
|
+
room = new LiveKitRoom(REALTIME_CONFIG.livekit.roomOptions);
|
|
113
81
|
const activeRoom = room;
|
|
114
82
|
activeRoom.on(RoomEvent.TrackSubscribed, (track, _pub, participant) => {
|
|
115
83
|
if (!participant.identity.startsWith(REALTIME_CONFIG.livekit.inferenceServerIdentityPrefix)) return;
|
|
@@ -146,11 +114,6 @@ const createRealTimeSubscribeClient = (opts) => {
|
|
|
146
114
|
} catch (error) {
|
|
147
115
|
observability?.stop();
|
|
148
116
|
if (room) room.disconnect().catch(() => {});
|
|
149
|
-
frameMetadataWorker?.terminate();
|
|
150
|
-
if (isDecartSDKError(error)) {
|
|
151
|
-
logger.error("Realtime subscribe error", { error: error.message });
|
|
152
|
-
throw error;
|
|
153
|
-
}
|
|
154
117
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
155
118
|
logger.error("Realtime subscribe error", { error: err.message });
|
|
156
119
|
throw classifyWebrtcError(err);
|
package/dist/shared/model.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ declare const canonicalModelSchema: z.ZodEnum<{
|
|
|
7
7
|
"lucy-2.5": "lucy-2.5";
|
|
8
8
|
"lucy-vton-2": "lucy-vton-2";
|
|
9
9
|
"lucy-vton-3": "lucy-vton-3";
|
|
10
|
+
"lucy-vton-3.5": "lucy-vton-3.5";
|
|
10
11
|
"lucy-restyle-2": "lucy-restyle-2";
|
|
11
12
|
"lucy-clip": "lucy-clip";
|
|
12
13
|
"lucy-image-2": "lucy-image-2";
|
|
@@ -24,10 +25,10 @@ declare const modelAliases: {
|
|
|
24
25
|
};
|
|
25
26
|
/** @internal Test-only helper to reset deprecation warning tracking */
|
|
26
27
|
|
|
27
|
-
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-2.1-vton-2">]>;
|
|
28
|
-
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>;
|
|
28
|
+
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-vton-3.5">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-2.1-vton-2">]>;
|
|
29
|
+
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-vton-3.5">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>;
|
|
29
30
|
declare const imageModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>;
|
|
30
|
-
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-2.1-vton-2">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
|
|
31
|
+
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-vton-3.5">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-2.1-vton-2">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.5">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-vton-3">, z.ZodLiteral<"lucy-vton-3.5">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
|
|
31
32
|
type Model = z.infer<typeof modelSchema>;
|
|
32
33
|
type RealTimeModels = z.infer<typeof realtimeModels>;
|
|
33
34
|
type VideoModels = z.infer<typeof videoModels>;
|
|
@@ -165,6 +166,22 @@ declare const modelInputSchemas: {
|
|
|
165
166
|
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
166
167
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
167
168
|
}, z.core.$strip>;
|
|
169
|
+
readonly "lucy-vton-3.5": z.ZodObject<{
|
|
170
|
+
prompt: z.ZodString;
|
|
171
|
+
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<ReadableStream<any>, unknown, z.core.$ZodTypeInternals<ReadableStream<any>, unknown>>, z.ZodType<URL, unknown, z.core.$ZodTypeInternals<URL, unknown>>, z.ZodURL, z.ZodObject<{
|
|
172
|
+
uri: z.ZodString;
|
|
173
|
+
type: z.ZodString;
|
|
174
|
+
name: z.ZodString;
|
|
175
|
+
}, z.core.$strip>]>>;
|
|
176
|
+
data: z.ZodUnion<readonly [z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<ReadableStream<any>, unknown, z.core.$ZodTypeInternals<ReadableStream<any>, unknown>>, z.ZodType<URL, unknown, z.core.$ZodTypeInternals<URL, unknown>>, z.ZodURL, z.ZodObject<{
|
|
177
|
+
uri: z.ZodString;
|
|
178
|
+
type: z.ZodString;
|
|
179
|
+
name: z.ZodString;
|
|
180
|
+
}, z.core.$strip>]>;
|
|
181
|
+
seed: z.ZodOptional<z.ZodNumber>;
|
|
182
|
+
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
183
|
+
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
184
|
+
}, z.core.$strip>;
|
|
168
185
|
readonly "lucy-latest": z.ZodObject<{
|
|
169
186
|
prompt: z.ZodString;
|
|
170
187
|
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodType<File, unknown, z.core.$ZodTypeInternals<File, unknown>>, z.ZodType<Blob, unknown, z.core.$ZodTypeInternals<Blob, unknown>>, z.ZodType<ReadableStream<any>, unknown, z.core.$ZodTypeInternals<ReadableStream<any>, unknown>>, z.ZodType<URL, unknown, z.core.$ZodTypeInternals<URL, unknown>>, z.ZodURL, z.ZodObject<{
|
|
@@ -381,6 +398,7 @@ declare const models: {
|
|
|
381
398
|
* - `"lucy-2.5"` - Lucy 2.5 realtime video editing
|
|
382
399
|
* - `"lucy-vton-2"` - Lucy virtual try-on 2
|
|
383
400
|
* - `"lucy-vton-3"` - Lucy virtual try-on 3
|
|
401
|
+
* - `"lucy-vton-3.5"` - Lucy virtual try-on 3.5 (latest)
|
|
384
402
|
* - `"lucy-restyle-2"` - Realtime video restyling
|
|
385
403
|
*/
|
|
386
404
|
realtime: <T extends RealTimeModels>(model: T) => ModelDefinition<T>;
|
|
@@ -393,6 +411,7 @@ declare const models: {
|
|
|
393
411
|
* - `"lucy-2.5"` - Long-form video editing (Lucy 2.5)
|
|
394
412
|
* - `"lucy-vton-2"` - Virtual try-on 2 video editing
|
|
395
413
|
* - `"lucy-vton-3"` - Virtual try-on 3 video editing
|
|
414
|
+
* - `"lucy-vton-3.5"` - Virtual try-on 3.5 video editing (latest)
|
|
396
415
|
* - `"lucy-restyle-2"` - Video restyling
|
|
397
416
|
*/
|
|
398
417
|
video: <T extends VideoModels>(model: T) => ModelDefinition<T> & {
|
package/dist/shared/model.js
CHANGED
|
@@ -7,6 +7,7 @@ const CANONICAL_MODEL_NAMES = [
|
|
|
7
7
|
"lucy-2.5",
|
|
8
8
|
"lucy-vton-2",
|
|
9
9
|
"lucy-vton-3",
|
|
10
|
+
"lucy-vton-3.5",
|
|
10
11
|
"lucy-restyle-2",
|
|
11
12
|
"lucy-clip",
|
|
12
13
|
"lucy-image-2"
|
|
@@ -16,6 +17,7 @@ const CANONICAL_REALTIME_MODEL_NAMES = [
|
|
|
16
17
|
"lucy-2.5",
|
|
17
18
|
"lucy-vton-2",
|
|
18
19
|
"lucy-vton-3",
|
|
20
|
+
"lucy-vton-3.5",
|
|
19
21
|
"lucy-restyle-2"
|
|
20
22
|
];
|
|
21
23
|
const CANONICAL_VIDEO_MODEL_NAMES = [
|
|
@@ -24,6 +26,7 @@ const CANONICAL_VIDEO_MODEL_NAMES = [
|
|
|
24
26
|
"lucy-2.5",
|
|
25
27
|
"lucy-vton-2",
|
|
26
28
|
"lucy-vton-3",
|
|
29
|
+
"lucy-vton-3.5",
|
|
27
30
|
"lucy-restyle-2"
|
|
28
31
|
];
|
|
29
32
|
const CANONICAL_IMAGE_MODEL_NAMES = ["lucy-image-2"];
|
|
@@ -54,6 +57,7 @@ const realtimeModels = z.union([
|
|
|
54
57
|
z.literal("lucy-2.5"),
|
|
55
58
|
z.literal("lucy-vton-2"),
|
|
56
59
|
z.literal("lucy-vton-3"),
|
|
60
|
+
z.literal("lucy-vton-3.5"),
|
|
57
61
|
z.literal("lucy-restyle-2"),
|
|
58
62
|
z.literal("lucy-latest"),
|
|
59
63
|
z.literal("lucy-vton-latest"),
|
|
@@ -66,6 +70,7 @@ const videoModels = z.union([
|
|
|
66
70
|
z.literal("lucy-2.5"),
|
|
67
71
|
z.literal("lucy-vton-2"),
|
|
68
72
|
z.literal("lucy-vton-3"),
|
|
73
|
+
z.literal("lucy-vton-3.5"),
|
|
69
74
|
z.literal("lucy-restyle-2"),
|
|
70
75
|
z.literal("lucy-latest"),
|
|
71
76
|
z.literal("lucy-vton-latest"),
|
|
@@ -185,6 +190,7 @@ const modelInputSchemas = {
|
|
|
185
190
|
"lucy-2.5": videoEdit2Schema,
|
|
186
191
|
"lucy-vton-2": videoEdit2Schema,
|
|
187
192
|
"lucy-vton-3": videoEdit2Schema,
|
|
193
|
+
"lucy-vton-3.5": videoEdit2Schema,
|
|
188
194
|
"lucy-latest": videoEdit2Schema,
|
|
189
195
|
"lucy-vton-latest": videoEdit2Schema,
|
|
190
196
|
"lucy-restyle-latest": restyleSchema,
|
|
@@ -259,6 +265,17 @@ const _models = {
|
|
|
259
265
|
height: 624,
|
|
260
266
|
inputSchema: z.object({})
|
|
261
267
|
},
|
|
268
|
+
"lucy-vton-3.5": {
|
|
269
|
+
urlPath: "/v1/stream",
|
|
270
|
+
name: "lucy-vton-3.5",
|
|
271
|
+
fps: {
|
|
272
|
+
ideal: 30,
|
|
273
|
+
max: 30
|
|
274
|
+
},
|
|
275
|
+
width: 1280,
|
|
276
|
+
height: 720,
|
|
277
|
+
inputSchema: z.object({})
|
|
278
|
+
},
|
|
262
279
|
"lucy-restyle-2": {
|
|
263
280
|
urlPath: "/v1/stream",
|
|
264
281
|
name: "lucy-restyle-2",
|
|
@@ -288,8 +305,8 @@ const _models = {
|
|
|
288
305
|
ideal: 30,
|
|
289
306
|
max: 30
|
|
290
307
|
},
|
|
291
|
-
width:
|
|
292
|
-
height:
|
|
308
|
+
width: 1280,
|
|
309
|
+
height: 720,
|
|
293
310
|
inputSchema: z.object({})
|
|
294
311
|
},
|
|
295
312
|
"lucy-restyle-latest": {
|
|
@@ -390,6 +407,15 @@ const _models = {
|
|
|
390
407
|
height: 624,
|
|
391
408
|
inputSchema: modelInputSchemas["lucy-vton-3"]
|
|
392
409
|
},
|
|
410
|
+
"lucy-vton-3.5": {
|
|
411
|
+
urlPath: "/v1/generate/lucy-vton-3.5",
|
|
412
|
+
queueUrlPath: "/v1/jobs/lucy-vton-3.5",
|
|
413
|
+
name: "lucy-vton-3.5",
|
|
414
|
+
fps: 20,
|
|
415
|
+
width: 1280,
|
|
416
|
+
height: 720,
|
|
417
|
+
inputSchema: modelInputSchemas["lucy-vton-3.5"]
|
|
418
|
+
},
|
|
393
419
|
"lucy-restyle-2": {
|
|
394
420
|
urlPath: "/v1/generate/lucy-restyle-2",
|
|
395
421
|
queueUrlPath: "/v1/jobs/lucy-restyle-2",
|
|
@@ -413,8 +439,8 @@ const _models = {
|
|
|
413
439
|
queueUrlPath: "/v1/jobs/lucy-vton-latest",
|
|
414
440
|
name: "lucy-vton-latest",
|
|
415
441
|
fps: 20,
|
|
416
|
-
width:
|
|
417
|
-
height:
|
|
442
|
+
width: 1280,
|
|
443
|
+
height: 720,
|
|
418
444
|
inputSchema: modelInputSchemas["lucy-vton-latest"]
|
|
419
445
|
},
|
|
420
446
|
"lucy-restyle-latest": {
|
|
@@ -496,6 +522,7 @@ const models = {
|
|
|
496
522
|
* - `"lucy-2.5"` - Lucy 2.5 realtime video editing
|
|
497
523
|
* - `"lucy-vton-2"` - Lucy virtual try-on 2
|
|
498
524
|
* - `"lucy-vton-3"` - Lucy virtual try-on 3
|
|
525
|
+
* - `"lucy-vton-3.5"` - Lucy virtual try-on 3.5 (latest)
|
|
499
526
|
* - `"lucy-restyle-2"` - Realtime video restyling
|
|
500
527
|
*/
|
|
501
528
|
realtime: (model) => {
|
|
@@ -513,6 +540,7 @@ const models = {
|
|
|
513
540
|
* - `"lucy-2.5"` - Long-form video editing (Lucy 2.5)
|
|
514
541
|
* - `"lucy-vton-2"` - Virtual try-on 2 video editing
|
|
515
542
|
* - `"lucy-vton-3"` - Virtual try-on 3 video editing
|
|
543
|
+
* - `"lucy-vton-3.5"` - Virtual try-on 3.5 video editing (latest)
|
|
516
544
|
* - `"lucy-restyle-2"` - Video restyling
|
|
517
545
|
*/
|
|
518
546
|
video: (model) => {
|