@decartai/sdk 0.0.62 → 0.0.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/realtime/client.d.ts +1 -14
- package/dist/realtime/client.js +3 -32
- package/dist/realtime/telemetry-reporter.js +6 -7
- package/dist/realtime/webrtc-connection.js +7 -17
- package/dist/realtime/webrtc-manager.js +0 -1
- package/dist/shared/model.d.ts +3 -35
- package/dist/shared/model.js +2 -103
- package/dist/tokens/client.d.ts +12 -3
- package/package.json +1 -1
- package/dist/realtime/audio-stream-manager.js +0 -90
package/README.md
CHANGED
|
@@ -24,7 +24,7 @@ For complete documentation, guides, and examples, visit:
|
|
|
24
24
|
```typescript
|
|
25
25
|
import { createDecartClient, models } from "@decartai/sdk";
|
|
26
26
|
|
|
27
|
-
const model = models.realtime("
|
|
27
|
+
const model = models.realtime("lucy-restyle-2");
|
|
28
28
|
|
|
29
29
|
// Get user's camera stream
|
|
30
30
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
@@ -16,11 +16,6 @@ declare const realTimeClientInitialStateSchema: z.ZodObject<{
|
|
|
16
16
|
image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
|
|
17
17
|
}, z.core.$strip>;
|
|
18
18
|
type OnRemoteStreamFn = (stream: MediaStream) => void;
|
|
19
|
-
type OnStatusFn = (status: string) => void;
|
|
20
|
-
type OnQueuePositionFn = (data: {
|
|
21
|
-
position: number;
|
|
22
|
-
queueSize: number;
|
|
23
|
-
}) => void;
|
|
24
19
|
type RealTimeClientInitialState = z.infer<typeof realTimeClientInitialStateSchema>;
|
|
25
20
|
declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
26
21
|
model: z.ZodObject<{
|
|
@@ -30,7 +25,7 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
|
30
25
|
fps: z.ZodNumber;
|
|
31
26
|
width: z.ZodNumber;
|
|
32
27
|
height: z.ZodNumber;
|
|
33
|
-
inputSchema: z.ZodAny
|
|
28
|
+
inputSchema: z.ZodOptional<z.ZodAny>;
|
|
34
29
|
}, z.core.$strip>;
|
|
35
30
|
onRemoteStream: z.ZodCustom<OnRemoteStreamFn, OnRemoteStreamFn>;
|
|
36
31
|
initialState: z.ZodOptional<z.ZodObject<{
|
|
@@ -41,8 +36,6 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
|
41
36
|
image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
|
|
42
37
|
}, z.core.$strip>>;
|
|
43
38
|
customizeOffer: z.ZodOptional<z.ZodCustom<z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>, z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>>>;
|
|
44
|
-
onStatus: z.ZodOptional<z.ZodCustom<OnStatusFn, OnStatusFn>>;
|
|
45
|
-
onQueuePosition: z.ZodOptional<z.ZodCustom<OnQueuePositionFn, OnQueuePositionFn>>;
|
|
46
39
|
}, z.core.$strip>;
|
|
47
40
|
type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
|
|
48
41
|
model: ModelDefinition | CustomModelDefinition;
|
|
@@ -53,11 +46,6 @@ type Events = {
|
|
|
53
46
|
generationTick: {
|
|
54
47
|
seconds: number;
|
|
55
48
|
};
|
|
56
|
-
status: string;
|
|
57
|
-
queuePosition: {
|
|
58
|
-
position: number;
|
|
59
|
-
queueSize: number;
|
|
60
|
-
};
|
|
61
49
|
diagnostic: DiagnosticEvent;
|
|
62
50
|
stats: WebRTCStats;
|
|
63
51
|
};
|
|
@@ -80,7 +68,6 @@ type RealTimeClient = {
|
|
|
80
68
|
enhance?: boolean;
|
|
81
69
|
timeout?: number;
|
|
82
70
|
}) => Promise<void>;
|
|
83
|
-
playAudio?: (audio: Blob | File | ArrayBuffer) => Promise<void>;
|
|
84
71
|
};
|
|
85
72
|
//#endregion
|
|
86
73
|
export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
|
package/dist/realtime/client.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
2
|
import { modelDefinitionSchema } from "../shared/model.js";
|
|
3
3
|
import { modelStateSchema } from "../shared/types.js";
|
|
4
|
-
import { AudioStreamManager } from "./audio-stream-manager.js";
|
|
5
4
|
import { createEventBuffer } from "./event-buffer.js";
|
|
6
5
|
import { realtimeMethods } from "./methods.js";
|
|
7
6
|
import { decodeSubscribeToken, encodeSubscribeToken } from "./subscribe-client.js";
|
|
@@ -57,23 +56,15 @@ const realTimeClientConnectOptionsSchema = z.object({
|
|
|
57
56
|
model: modelDefinitionSchema,
|
|
58
57
|
onRemoteStream: z.custom((val) => typeof val === "function", { message: "onRemoteStream must be a function" }),
|
|
59
58
|
initialState: realTimeClientInitialStateSchema.optional(),
|
|
60
|
-
customizeOffer: createAsyncFunctionSchema(z.function()).optional()
|
|
61
|
-
onStatus: z.custom((val) => typeof val === "function", { message: "onStatus must be a function" }).optional(),
|
|
62
|
-
onQueuePosition: z.custom((val) => typeof val === "function", { message: "onQueuePosition must be a function" }).optional()
|
|
59
|
+
customizeOffer: createAsyncFunctionSchema(z.function()).optional()
|
|
63
60
|
});
|
|
64
61
|
const createRealTimeClient = (opts) => {
|
|
65
62
|
const { baseUrl, apiKey, integration, logger } = opts;
|
|
66
63
|
const connect = async (stream, options) => {
|
|
67
64
|
const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
|
|
68
65
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
69
|
-
const
|
|
70
|
-
const
|
|
71
|
-
let audioStreamManager;
|
|
72
|
-
let inputStream;
|
|
73
|
-
if (isAvatarLive && !stream) {
|
|
74
|
-
audioStreamManager = new AudioStreamManager();
|
|
75
|
-
inputStream = audioStreamManager.getStream();
|
|
76
|
-
} else inputStream = stream ?? new MediaStream();
|
|
66
|
+
const { onRemoteStream, initialState } = parsedOptions.data;
|
|
67
|
+
const inputStream = stream ?? new MediaStream();
|
|
77
68
|
let webrtcManager;
|
|
78
69
|
let telemetryReporter = new NullTelemetryReporter();
|
|
79
70
|
let handleConnectionStateChange = null;
|
|
@@ -108,7 +99,6 @@ const createRealTimeClient = (opts) => {
|
|
|
108
99
|
customizeOffer: options.customizeOffer,
|
|
109
100
|
vp8MinBitrate: 300,
|
|
110
101
|
vp8StartBitrate: 600,
|
|
111
|
-
modelName: options.model.name,
|
|
112
102
|
initialImage,
|
|
113
103
|
initialPrompt
|
|
114
104
|
});
|
|
@@ -157,19 +147,6 @@ const createRealTimeClient = (opts) => {
|
|
|
157
147
|
emitOrBuffer("generationTick", { seconds: msg.seconds });
|
|
158
148
|
};
|
|
159
149
|
manager.getWebsocketMessageEmitter().on("generationTick", tickListener);
|
|
160
|
-
const wsEmitter = manager.getWebsocketMessageEmitter();
|
|
161
|
-
wsEmitter.on("status", (msg) => {
|
|
162
|
-
emitOrBuffer("status", msg.status);
|
|
163
|
-
onStatus?.(msg.status);
|
|
164
|
-
});
|
|
165
|
-
wsEmitter.on("queuePosition", (msg) => {
|
|
166
|
-
const data = {
|
|
167
|
-
position: msg.position,
|
|
168
|
-
queueSize: msg.queue_size
|
|
169
|
-
};
|
|
170
|
-
emitOrBuffer("queuePosition", data);
|
|
171
|
-
onQueuePosition?.(data);
|
|
172
|
-
});
|
|
173
150
|
await manager.connect(inputStream);
|
|
174
151
|
const methods = realtimeMethods(manager, imageToBase64);
|
|
175
152
|
let statsCollector = null;
|
|
@@ -242,7 +219,6 @@ const createRealTimeClient = (opts) => {
|
|
|
242
219
|
telemetryReporter.stop();
|
|
243
220
|
stop();
|
|
244
221
|
manager.cleanup();
|
|
245
|
-
audioStreamManager?.cleanup();
|
|
246
222
|
},
|
|
247
223
|
on: eventEmitter.on,
|
|
248
224
|
off: eventEmitter.off,
|
|
@@ -258,16 +234,11 @@ const createRealTimeClient = (opts) => {
|
|
|
258
234
|
return manager.setImage(base64, options);
|
|
259
235
|
}
|
|
260
236
|
};
|
|
261
|
-
if (isAvatarLive && audioStreamManager) {
|
|
262
|
-
const manager = audioStreamManager;
|
|
263
|
-
client.playAudio = (audio) => manager.playAudio(audio);
|
|
264
|
-
}
|
|
265
237
|
flush();
|
|
266
238
|
return client;
|
|
267
239
|
} catch (error) {
|
|
268
240
|
telemetryReporter.stop();
|
|
269
241
|
webrtcManager?.cleanup();
|
|
270
|
-
audioStreamManager?.cleanup();
|
|
271
242
|
throw error;
|
|
272
243
|
}
|
|
273
244
|
};
|
|
@@ -50,15 +50,16 @@ var TelemetryReporter = class {
|
|
|
50
50
|
}
|
|
51
51
|
/** Flush buffered data immediately. */
|
|
52
52
|
flush() {
|
|
53
|
-
this.sendReport(
|
|
53
|
+
this.sendReport();
|
|
54
54
|
}
|
|
55
|
-
/** Stop the reporter and
|
|
55
|
+
/** Stop the reporter and discard any buffered data. */
|
|
56
56
|
stop() {
|
|
57
57
|
if (this.intervalId !== null) {
|
|
58
58
|
clearInterval(this.intervalId);
|
|
59
59
|
this.intervalId = null;
|
|
60
60
|
}
|
|
61
|
-
this.
|
|
61
|
+
this.statsBuffer = [];
|
|
62
|
+
this.diagnosticsBuffer = [];
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
64
65
|
* Build a single chunk from the front of the buffers, respecting MAX_ITEMS_PER_REPORT.
|
|
@@ -82,7 +83,7 @@ var TelemetryReporter = class {
|
|
|
82
83
|
diagnostics: this.diagnosticsBuffer.splice(0, MAX_ITEMS_PER_REPORT)
|
|
83
84
|
};
|
|
84
85
|
}
|
|
85
|
-
sendReport(
|
|
86
|
+
sendReport() {
|
|
86
87
|
if (this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0) return;
|
|
87
88
|
try {
|
|
88
89
|
const commonHeaders = {
|
|
@@ -94,12 +95,10 @@ var TelemetryReporter = class {
|
|
|
94
95
|
};
|
|
95
96
|
let chunk = this.createReportChunk();
|
|
96
97
|
while (chunk !== null) {
|
|
97
|
-
const isLast = this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0;
|
|
98
98
|
fetch(TELEMETRY_URL, {
|
|
99
99
|
method: "POST",
|
|
100
100
|
headers: commonHeaders,
|
|
101
|
-
body: JSON.stringify(chunk)
|
|
102
|
-
keepalive: keepalive && isLast
|
|
101
|
+
body: JSON.stringify(chunk)
|
|
103
102
|
}).then((response) => {
|
|
104
103
|
if (!response.ok) this.logger.warn("Telemetry report rejected", {
|
|
105
104
|
status: response.status,
|
|
@@ -3,7 +3,7 @@ import mitt from "mitt";
|
|
|
3
3
|
|
|
4
4
|
//#region src/realtime/webrtc-connection.ts
|
|
5
5
|
const ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }];
|
|
6
|
-
const
|
|
6
|
+
const SETUP_TIMEOUT_MS = 3e4;
|
|
7
7
|
const noopDiagnostic = () => {};
|
|
8
8
|
var WebRTCConnection = class {
|
|
9
9
|
pc = null;
|
|
@@ -183,14 +183,6 @@ var WebRTCConnection = class {
|
|
|
183
183
|
this.websocketMessagesEmitter.emit("sessionId", msg);
|
|
184
184
|
return;
|
|
185
185
|
}
|
|
186
|
-
if (msg.type === "status") {
|
|
187
|
-
this.websocketMessagesEmitter.emit("status", msg);
|
|
188
|
-
return;
|
|
189
|
-
}
|
|
190
|
-
if (msg.type === "queue_position") {
|
|
191
|
-
this.websocketMessagesEmitter.emit("queuePosition", msg);
|
|
192
|
-
return;
|
|
193
|
-
}
|
|
194
186
|
if (!this.pc) return;
|
|
195
187
|
switch (msg.type) {
|
|
196
188
|
case "ready": {
|
|
@@ -254,7 +246,7 @@ var WebRTCConnection = class {
|
|
|
254
246
|
const timeoutId = setTimeout(() => {
|
|
255
247
|
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
256
248
|
reject(/* @__PURE__ */ new Error("Image send timed out"));
|
|
257
|
-
}, options?.timeout ??
|
|
249
|
+
}, options?.timeout ?? SETUP_TIMEOUT_MS);
|
|
258
250
|
const listener = (msg) => {
|
|
259
251
|
clearTimeout(timeoutId);
|
|
260
252
|
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
@@ -283,7 +275,7 @@ var WebRTCConnection = class {
|
|
|
283
275
|
const timeoutId = setTimeout(() => {
|
|
284
276
|
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
285
277
|
reject(/* @__PURE__ */ new Error("Prompt send timed out"));
|
|
286
|
-
},
|
|
278
|
+
}, SETUP_TIMEOUT_MS);
|
|
287
279
|
const listener = (msg) => {
|
|
288
280
|
if (msg.prompt === prompt.text) {
|
|
289
281
|
clearTimeout(timeoutId);
|
|
@@ -319,12 +311,10 @@ var WebRTCConnection = class {
|
|
|
319
311
|
}
|
|
320
312
|
this.pc = new RTCPeerConnection({ iceServers: ICE_SERVERS });
|
|
321
313
|
this.setState("connecting");
|
|
322
|
-
if (this.localStream) {
|
|
323
|
-
if (this.
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
});
|
|
327
|
-
} else {
|
|
314
|
+
if (this.localStream) this.localStream.getTracks().forEach((track) => {
|
|
315
|
+
if (this.pc && this.localStream) this.pc.addTrack(track, this.localStream);
|
|
316
|
+
});
|
|
317
|
+
else {
|
|
328
318
|
this.pc.addTransceiver("video", { direction: "recvonly" });
|
|
329
319
|
this.pc.addTransceiver("audio", { direction: "recvonly" });
|
|
330
320
|
}
|
|
@@ -43,7 +43,6 @@ var WebRTCManager = class {
|
|
|
43
43
|
customizeOffer: config.customizeOffer,
|
|
44
44
|
vp8MinBitrate: config.vp8MinBitrate,
|
|
45
45
|
vp8StartBitrate: config.vp8StartBitrate,
|
|
46
|
-
modelName: config.modelName,
|
|
47
46
|
initialImage: config.initialImage,
|
|
48
47
|
initialPrompt: config.initialPrompt,
|
|
49
48
|
logger: this.logger,
|
package/dist/shared/model.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
3
|
//#region src/shared/model.d.ts
|
|
4
|
-
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy
|
|
5
|
-
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-
|
|
4
|
+
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">]>;
|
|
5
|
+
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>;
|
|
6
6
|
declare const imageModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>;
|
|
7
|
-
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy
|
|
7
|
+
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
|
|
8
8
|
type Model = z.infer<typeof modelSchema>;
|
|
9
9
|
type RealTimeModels = z.infer<typeof realtimeModels>;
|
|
10
10
|
type VideoModels = z.infer<typeof videoModels>;
|
|
@@ -96,20 +96,6 @@ declare const modelInputSchemas: {
|
|
|
96
96
|
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
97
97
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
98
98
|
}, z.core.$strip>;
|
|
99
|
-
readonly "lucy-motion": z.ZodObject<{
|
|
100
|
-
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
101
|
-
uri: z.ZodString;
|
|
102
|
-
type: z.ZodString;
|
|
103
|
-
name: z.ZodString;
|
|
104
|
-
}, z.core.$strip>]>;
|
|
105
|
-
trajectory: z.ZodArray<z.ZodObject<{
|
|
106
|
-
frame: z.ZodNumber;
|
|
107
|
-
x: z.ZodNumber;
|
|
108
|
-
y: z.ZodNumber;
|
|
109
|
-
}, z.core.$strip>>;
|
|
110
|
-
seed: z.ZodOptional<z.ZodNumber>;
|
|
111
|
-
resolution: z.ZodOptional<z.ZodDefault<z.ZodLiteral<"720p">>>;
|
|
112
|
-
}, z.core.$strip>;
|
|
113
99
|
readonly "lucy-latest": z.ZodObject<{
|
|
114
100
|
prompt: z.ZodString;
|
|
115
101
|
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
@@ -174,20 +160,6 @@ declare const modelInputSchemas: {
|
|
|
174
160
|
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
175
161
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
176
162
|
}, z.core.$strip>;
|
|
177
|
-
readonly "lucy-motion-latest": z.ZodObject<{
|
|
178
|
-
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
179
|
-
uri: z.ZodString;
|
|
180
|
-
type: z.ZodString;
|
|
181
|
-
name: z.ZodString;
|
|
182
|
-
}, z.core.$strip>]>;
|
|
183
|
-
trajectory: z.ZodArray<z.ZodObject<{
|
|
184
|
-
frame: z.ZodNumber;
|
|
185
|
-
x: z.ZodNumber;
|
|
186
|
-
y: z.ZodNumber;
|
|
187
|
-
}, z.core.$strip>>;
|
|
188
|
-
seed: z.ZodOptional<z.ZodNumber>;
|
|
189
|
-
resolution: z.ZodOptional<z.ZodDefault<z.ZodLiteral<"720p">>>;
|
|
190
|
-
}, z.core.$strip>;
|
|
191
163
|
readonly "lucy-image-latest": z.ZodObject<{
|
|
192
164
|
prompt: z.ZodString;
|
|
193
165
|
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
@@ -301,9 +273,6 @@ declare const models: {
|
|
|
301
273
|
* - `"lucy-2.1"` - Lucy 2.1 realtime video editing
|
|
302
274
|
* - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on
|
|
303
275
|
* - `"lucy-restyle-2"` - Realtime video restyling
|
|
304
|
-
* - `"lucy-restyle"` - Legacy realtime restyling
|
|
305
|
-
* - `"lucy"` - Legacy Lucy realtime
|
|
306
|
-
* - `"live-avatar"` - Live avatar
|
|
307
276
|
*/
|
|
308
277
|
realtime: <T extends RealTimeModels>(model: T) => ModelDefinition<T>;
|
|
309
278
|
/**
|
|
@@ -314,7 +283,6 @@ declare const models: {
|
|
|
314
283
|
* - `"lucy-2.1"` - Long-form video editing (Lucy 2.1)
|
|
315
284
|
* - `"lucy-2.1-vton"` - Virtual try-on video editing
|
|
316
285
|
* - `"lucy-restyle-2"` - Video restyling
|
|
317
|
-
* - `"lucy-motion"` - Motion generation
|
|
318
286
|
*/
|
|
319
287
|
video: <T extends VideoModels>(model: T) => ModelDefinition<T> & {
|
|
320
288
|
queueUrlPath: string;
|
package/dist/shared/model.js
CHANGED
|
@@ -7,10 +7,7 @@ import { z } from "zod";
|
|
|
7
7
|
* Old names still work but will log a deprecation warning.
|
|
8
8
|
*/
|
|
9
9
|
const MODEL_ALIASES = {
|
|
10
|
-
mirage: "lucy-restyle",
|
|
11
10
|
mirage_v2: "lucy-restyle-2",
|
|
12
|
-
lucy_v2v_720p_rt: "lucy",
|
|
13
|
-
live_avatar: "live-avatar",
|
|
14
11
|
"lucy-pro-v2v": "lucy-clip",
|
|
15
12
|
"lucy-restyle-v2v": "lucy-restyle-2",
|
|
16
13
|
"lucy-pro-i2i": "lucy-image-2"
|
|
@@ -24,31 +21,23 @@ function warnDeprecated(model) {
|
|
|
24
21
|
}
|
|
25
22
|
}
|
|
26
23
|
const realtimeModels = z.union([
|
|
27
|
-
z.literal("lucy"),
|
|
28
24
|
z.literal("lucy-2.1"),
|
|
29
25
|
z.literal("lucy-2.1-vton"),
|
|
30
|
-
z.literal("lucy-restyle"),
|
|
31
26
|
z.literal("lucy-restyle-2"),
|
|
32
|
-
z.literal("live-avatar"),
|
|
33
27
|
z.literal("lucy-latest"),
|
|
34
28
|
z.literal("lucy-vton-latest"),
|
|
35
29
|
z.literal("lucy-restyle-latest"),
|
|
36
|
-
z.literal("
|
|
37
|
-
z.literal("mirage_v2"),
|
|
38
|
-
z.literal("lucy_v2v_720p_rt"),
|
|
39
|
-
z.literal("live_avatar")
|
|
30
|
+
z.literal("mirage_v2")
|
|
40
31
|
]);
|
|
41
32
|
const videoModels = z.union([
|
|
42
33
|
z.literal("lucy-clip"),
|
|
43
34
|
z.literal("lucy-2.1"),
|
|
44
35
|
z.literal("lucy-2.1-vton"),
|
|
45
36
|
z.literal("lucy-restyle-2"),
|
|
46
|
-
z.literal("lucy-motion"),
|
|
47
37
|
z.literal("lucy-latest"),
|
|
48
38
|
z.literal("lucy-vton-latest"),
|
|
49
39
|
z.literal("lucy-restyle-latest"),
|
|
50
40
|
z.literal("lucy-clip-latest"),
|
|
51
|
-
z.literal("lucy-motion-latest"),
|
|
52
41
|
z.literal("lucy-pro-v2v"),
|
|
53
42
|
z.literal("lucy-restyle-v2v")
|
|
54
43
|
]);
|
|
@@ -91,10 +80,6 @@ const proResolutionSchema = () => {
|
|
|
91
80
|
return z.enum(["720p", "480p"]).optional().describe("The resolution to use for the generation").default("720p");
|
|
92
81
|
};
|
|
93
82
|
/**
|
|
94
|
-
* Resolution schema for lucy-motion.
|
|
95
|
-
*/
|
|
96
|
-
const motionResolutionSchema = z.literal("720p").default("720p").optional().describe("The resolution to use for the generation");
|
|
97
|
-
/**
|
|
98
83
|
* Resolution schema for video-to-video models (supports 720p).
|
|
99
84
|
*/
|
|
100
85
|
const v2vResolutionSchema = z.literal("720p").optional().describe("The resolution to use for the generation").default("720p");
|
|
@@ -136,30 +121,10 @@ const modelInputSchemas = {
|
|
|
136
121
|
"lucy-restyle-2": restyleSchema,
|
|
137
122
|
"lucy-2.1": videoEdit2Schema,
|
|
138
123
|
"lucy-2.1-vton": videoEdit2Schema,
|
|
139
|
-
"lucy-motion": z.object({
|
|
140
|
-
data: fileInputSchema.describe("The image data to use for generation (File, Blob, ReadableStream, URL, or string URL). Output video is limited to 5 seconds."),
|
|
141
|
-
trajectory: z.array(z.object({
|
|
142
|
-
frame: z.number().min(0),
|
|
143
|
-
x: z.number().min(0),
|
|
144
|
-
y: z.number().min(0)
|
|
145
|
-
})).min(2).max(1e3).describe("The trajectory of the desired movement of the object in the image"),
|
|
146
|
-
seed: z.number().optional().describe("The seed to use for the generation"),
|
|
147
|
-
resolution: motionResolutionSchema
|
|
148
|
-
}),
|
|
149
124
|
"lucy-latest": videoEdit2Schema,
|
|
150
125
|
"lucy-vton-latest": videoEdit2Schema,
|
|
151
126
|
"lucy-restyle-latest": restyleSchema,
|
|
152
127
|
"lucy-clip-latest": videoEditSchema,
|
|
153
|
-
"lucy-motion-latest": z.object({
|
|
154
|
-
data: fileInputSchema.describe("The image data to use for generation (File, Blob, ReadableStream, URL, or string URL). Output video is limited to 5 seconds."),
|
|
155
|
-
trajectory: z.array(z.object({
|
|
156
|
-
frame: z.number().min(0),
|
|
157
|
-
x: z.number().min(0),
|
|
158
|
-
y: z.number().min(0)
|
|
159
|
-
})).min(2).max(1e3).describe("The trajectory of the desired movement of the object in the image"),
|
|
160
|
-
seed: z.number().optional().describe("The seed to use for the generation"),
|
|
161
|
-
resolution: motionResolutionSchema
|
|
162
|
-
}),
|
|
163
128
|
"lucy-image-latest": imageEditSchema,
|
|
164
129
|
"lucy-pro-v2v": videoEditSchema,
|
|
165
130
|
"lucy-pro-i2i": imageEditSchema,
|
|
@@ -172,18 +137,10 @@ const modelDefinitionSchema = z.object({
|
|
|
172
137
|
fps: z.number().min(1),
|
|
173
138
|
width: z.number().min(1),
|
|
174
139
|
height: z.number().min(1),
|
|
175
|
-
inputSchema: z.any()
|
|
140
|
+
inputSchema: z.any().optional()
|
|
176
141
|
});
|
|
177
142
|
const _models = {
|
|
178
143
|
realtime: {
|
|
179
|
-
lucy: {
|
|
180
|
-
urlPath: "/v1/stream",
|
|
181
|
-
name: "lucy",
|
|
182
|
-
fps: 25,
|
|
183
|
-
width: 1280,
|
|
184
|
-
height: 704,
|
|
185
|
-
inputSchema: z.object({})
|
|
186
|
-
},
|
|
187
144
|
"lucy-2.1": {
|
|
188
145
|
urlPath: "/v1/stream",
|
|
189
146
|
name: "lucy-2.1",
|
|
@@ -200,14 +157,6 @@ const _models = {
|
|
|
200
157
|
height: 624,
|
|
201
158
|
inputSchema: z.object({})
|
|
202
159
|
},
|
|
203
|
-
"lucy-restyle": {
|
|
204
|
-
urlPath: "/v1/stream",
|
|
205
|
-
name: "lucy-restyle",
|
|
206
|
-
fps: 25,
|
|
207
|
-
width: 1280,
|
|
208
|
-
height: 704,
|
|
209
|
-
inputSchema: z.object({})
|
|
210
|
-
},
|
|
211
160
|
"lucy-restyle-2": {
|
|
212
161
|
urlPath: "/v1/stream",
|
|
213
162
|
name: "lucy-restyle-2",
|
|
@@ -216,14 +165,6 @@ const _models = {
|
|
|
216
165
|
height: 704,
|
|
217
166
|
inputSchema: z.object({})
|
|
218
167
|
},
|
|
219
|
-
"live-avatar": {
|
|
220
|
-
urlPath: "/v1/stream",
|
|
221
|
-
name: "live-avatar",
|
|
222
|
-
fps: 25,
|
|
223
|
-
width: 1280,
|
|
224
|
-
height: 720,
|
|
225
|
-
inputSchema: z.object({})
|
|
226
|
-
},
|
|
227
168
|
"lucy-latest": {
|
|
228
169
|
urlPath: "/v1/stream",
|
|
229
170
|
name: "lucy-latest",
|
|
@@ -248,14 +189,6 @@ const _models = {
|
|
|
248
189
|
height: 704,
|
|
249
190
|
inputSchema: z.object({})
|
|
250
191
|
},
|
|
251
|
-
mirage: {
|
|
252
|
-
urlPath: "/v1/stream",
|
|
253
|
-
name: "mirage",
|
|
254
|
-
fps: 25,
|
|
255
|
-
width: 1280,
|
|
256
|
-
height: 704,
|
|
257
|
-
inputSchema: z.object({})
|
|
258
|
-
},
|
|
259
192
|
mirage_v2: {
|
|
260
193
|
urlPath: "/v1/stream",
|
|
261
194
|
name: "mirage_v2",
|
|
@@ -263,22 +196,6 @@ const _models = {
|
|
|
263
196
|
width: 1280,
|
|
264
197
|
height: 704,
|
|
265
198
|
inputSchema: z.object({})
|
|
266
|
-
},
|
|
267
|
-
lucy_v2v_720p_rt: {
|
|
268
|
-
urlPath: "/v1/stream",
|
|
269
|
-
name: "lucy_v2v_720p_rt",
|
|
270
|
-
fps: 25,
|
|
271
|
-
width: 1280,
|
|
272
|
-
height: 704,
|
|
273
|
-
inputSchema: z.object({})
|
|
274
|
-
},
|
|
275
|
-
live_avatar: {
|
|
276
|
-
urlPath: "/v1/stream",
|
|
277
|
-
name: "live_avatar",
|
|
278
|
-
fps: 25,
|
|
279
|
-
width: 1280,
|
|
280
|
-
height: 720,
|
|
281
|
-
inputSchema: z.object({})
|
|
282
199
|
}
|
|
283
200
|
},
|
|
284
201
|
image: {
|
|
@@ -347,15 +264,6 @@ const _models = {
|
|
|
347
264
|
height: 704,
|
|
348
265
|
inputSchema: modelInputSchemas["lucy-restyle-2"]
|
|
349
266
|
},
|
|
350
|
-
"lucy-motion": {
|
|
351
|
-
urlPath: "/v1/generate/lucy-motion",
|
|
352
|
-
queueUrlPath: "/v1/jobs/lucy-motion",
|
|
353
|
-
name: "lucy-motion",
|
|
354
|
-
fps: 25,
|
|
355
|
-
width: 1280,
|
|
356
|
-
height: 704,
|
|
357
|
-
inputSchema: modelInputSchemas["lucy-motion"]
|
|
358
|
-
},
|
|
359
267
|
"lucy-latest": {
|
|
360
268
|
urlPath: "/v1/generate/lucy-latest",
|
|
361
269
|
queueUrlPath: "/v1/jobs/lucy-latest",
|
|
@@ -392,15 +300,6 @@ const _models = {
|
|
|
392
300
|
height: 704,
|
|
393
301
|
inputSchema: modelInputSchemas["lucy-clip-latest"]
|
|
394
302
|
},
|
|
395
|
-
"lucy-motion-latest": {
|
|
396
|
-
urlPath: "/v1/generate/lucy-motion-latest",
|
|
397
|
-
queueUrlPath: "/v1/jobs/lucy-motion-latest",
|
|
398
|
-
name: "lucy-motion-latest",
|
|
399
|
-
fps: 25,
|
|
400
|
-
width: 1280,
|
|
401
|
-
height: 704,
|
|
402
|
-
inputSchema: modelInputSchemas["lucy-motion-latest"]
|
|
403
|
-
},
|
|
404
303
|
"lucy-pro-v2v": {
|
|
405
304
|
urlPath: "/v1/generate/lucy-pro-v2v",
|
|
406
305
|
queueUrlPath: "/v1/jobs/lucy-pro-v2v",
|
package/dist/tokens/client.d.ts
CHANGED
|
@@ -9,6 +9,13 @@ type CreateTokenOptions = {
|
|
|
9
9
|
expiresIn?: number;
|
|
10
10
|
/** Restrict which models this token can access (max 20 items). */
|
|
11
11
|
allowedModels?: (Model | (string & {}))[];
|
|
12
|
+
/**
|
|
13
|
+
* Restrict which web origins this token can be used from (max 20 items).
|
|
14
|
+
* Each entry must be a full origin including scheme, e.g. `https://example.com`.
|
|
15
|
+
* Enforced on realtime sessions by matching the WebSocket `Origin` header
|
|
16
|
+
* verbatim. Defense-in-depth — only effective for browser-based clients.
|
|
17
|
+
*/
|
|
18
|
+
allowedOrigins?: string[];
|
|
12
19
|
/** Operational limits for the token. */
|
|
13
20
|
constraints?: {
|
|
14
21
|
realtime?: {
|
|
@@ -19,9 +26,10 @@ type CreateTokenOptions = {
|
|
|
19
26
|
type CreateTokenResponse = {
|
|
20
27
|
apiKey: string;
|
|
21
28
|
expiresAt: string;
|
|
22
|
-
/** Present when `allowedModels`
|
|
29
|
+
/** Present when `allowedModels` and/or `allowedOrigins` were set on the request. */
|
|
23
30
|
permissions?: {
|
|
24
|
-
models
|
|
31
|
+
models?: (Model | (string & {}))[];
|
|
32
|
+
origins?: string[];
|
|
25
33
|
} | null;
|
|
26
34
|
/** Present when `constraints` was set on the request. */
|
|
27
35
|
constraints?: {
|
|
@@ -45,10 +53,11 @@ type TokensClient = {
|
|
|
45
53
|
* // With metadata:
|
|
46
54
|
* const token = await client.tokens.create({ metadata: { role: "viewer" } });
|
|
47
55
|
*
|
|
48
|
-
* // With expiry, model restrictions, and constraints:
|
|
56
|
+
* // With expiry, model restrictions, origin restrictions, and constraints:
|
|
49
57
|
* const token = await client.tokens.create({
|
|
50
58
|
* expiresIn: 300,
|
|
51
59
|
* allowedModels: ["lucy-pro-v2v", "lucy-restyle-v2v"],
|
|
60
|
+
* allowedOrigins: ["https://example.com"],
|
|
52
61
|
* constraints: { realtime: { maxSessionDuration: 120 } },
|
|
53
62
|
* });
|
|
54
63
|
* ```
|
package/package.json
CHANGED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
//#region src/realtime/audio-stream-manager.ts
|
|
2
|
-
/**
|
|
3
|
-
* Manages an audio stream for live_avatar mode.
|
|
4
|
-
* Creates a continuous audio stream that outputs silence by default,
|
|
5
|
-
* and allows playing audio files through the stream.
|
|
6
|
-
*/
|
|
7
|
-
var AudioStreamManager = class {
|
|
8
|
-
audioContext;
|
|
9
|
-
destination;
|
|
10
|
-
silenceOscillator;
|
|
11
|
-
silenceGain;
|
|
12
|
-
currentSource = null;
|
|
13
|
-
_isPlaying = false;
|
|
14
|
-
constructor() {
|
|
15
|
-
this.audioContext = new AudioContext();
|
|
16
|
-
this.destination = this.audioContext.createMediaStreamDestination();
|
|
17
|
-
this.silenceOscillator = this.audioContext.createOscillator();
|
|
18
|
-
this.silenceGain = this.audioContext.createGain();
|
|
19
|
-
this.silenceGain.gain.value = 0;
|
|
20
|
-
this.silenceOscillator.connect(this.silenceGain);
|
|
21
|
-
this.silenceGain.connect(this.destination);
|
|
22
|
-
this.silenceOscillator.start();
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Get the MediaStream to pass to WebRTC.
|
|
26
|
-
* This stream outputs silence when no audio is playing.
|
|
27
|
-
*/
|
|
28
|
-
getStream() {
|
|
29
|
-
return this.destination.stream;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Check if audio is currently playing.
|
|
33
|
-
*/
|
|
34
|
-
isPlaying() {
|
|
35
|
-
return this._isPlaying;
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Play audio through the stream.
|
|
39
|
-
* When the audio ends, the stream automatically reverts to silence.
|
|
40
|
-
* @param audio - Audio data as Blob, File, or ArrayBuffer
|
|
41
|
-
* @returns Promise that resolves when audio finishes playing
|
|
42
|
-
*/
|
|
43
|
-
async playAudio(audio) {
|
|
44
|
-
if (this.audioContext.state === "suspended") await this.audioContext.resume();
|
|
45
|
-
if (this._isPlaying) this.stopAudio();
|
|
46
|
-
let arrayBuffer;
|
|
47
|
-
if (audio instanceof ArrayBuffer) arrayBuffer = audio;
|
|
48
|
-
else arrayBuffer = await audio.arrayBuffer();
|
|
49
|
-
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
|
50
|
-
const source = this.audioContext.createBufferSource();
|
|
51
|
-
source.buffer = audioBuffer;
|
|
52
|
-
source.connect(this.destination);
|
|
53
|
-
this.currentSource = source;
|
|
54
|
-
this._isPlaying = true;
|
|
55
|
-
return new Promise((resolve) => {
|
|
56
|
-
source.onended = () => {
|
|
57
|
-
this._isPlaying = false;
|
|
58
|
-
this.currentSource = null;
|
|
59
|
-
resolve();
|
|
60
|
-
};
|
|
61
|
-
source.start();
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Stop currently playing audio immediately.
|
|
66
|
-
* The stream will revert to silence.
|
|
67
|
-
*/
|
|
68
|
-
stopAudio() {
|
|
69
|
-
if (this.currentSource) {
|
|
70
|
-
try {
|
|
71
|
-
this.currentSource.stop();
|
|
72
|
-
} catch {}
|
|
73
|
-
this.currentSource = null;
|
|
74
|
-
}
|
|
75
|
-
this._isPlaying = false;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Clean up resources.
|
|
79
|
-
*/
|
|
80
|
-
cleanup() {
|
|
81
|
-
this.stopAudio();
|
|
82
|
-
try {
|
|
83
|
-
this.silenceOscillator.stop();
|
|
84
|
-
} catch {}
|
|
85
|
-
this.audioContext.close().catch(() => {});
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
|
|
89
|
-
//#endregion
|
|
90
|
-
export { AudioStreamManager };
|