@decartai/sdk 0.0.63 → 0.0.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/realtime/client.d.ts +0 -1
- package/dist/realtime/client.js +1 -15
- package/dist/realtime/telemetry-reporter.js +6 -7
- package/dist/realtime/webrtc-connection.js +7 -9
- package/dist/realtime/webrtc-manager.js +0 -1
- package/dist/shared/model.d.ts +3 -35
- package/dist/shared/model.js +1 -102
- package/package.json +1 -1
- package/dist/realtime/audio-stream-manager.js +0 -90
package/README.md
CHANGED
|
@@ -24,7 +24,7 @@ For complete documentation, guides, and examples, visit:
|
|
|
24
24
|
```typescript
|
|
25
25
|
import { createDecartClient, models } from "@decartai/sdk";
|
|
26
26
|
|
|
27
|
-
const model = models.realtime("
|
|
27
|
+
const model = models.realtime("lucy-restyle-2");
|
|
28
28
|
|
|
29
29
|
// Get user's camera stream
|
|
30
30
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
@@ -68,7 +68,6 @@ type RealTimeClient = {
|
|
|
68
68
|
enhance?: boolean;
|
|
69
69
|
timeout?: number;
|
|
70
70
|
}) => Promise<void>;
|
|
71
|
-
playAudio?: (audio: Blob | File | ArrayBuffer) => Promise<void>;
|
|
72
71
|
};
|
|
73
72
|
//#endregion
|
|
74
73
|
export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
|
package/dist/realtime/client.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
2
|
import { modelDefinitionSchema } from "../shared/model.js";
|
|
3
3
|
import { modelStateSchema } from "../shared/types.js";
|
|
4
|
-
import { AudioStreamManager } from "./audio-stream-manager.js";
|
|
5
4
|
import { createEventBuffer } from "./event-buffer.js";
|
|
6
5
|
import { realtimeMethods } from "./methods.js";
|
|
7
6
|
import { decodeSubscribeToken, encodeSubscribeToken } from "./subscribe-client.js";
|
|
@@ -64,14 +63,8 @@ const createRealTimeClient = (opts) => {
|
|
|
64
63
|
const connect = async (stream, options) => {
|
|
65
64
|
const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
|
|
66
65
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
67
|
-
const isAvatarLive = options.model.name === "live_avatar" || options.model.name === "live-avatar";
|
|
68
66
|
const { onRemoteStream, initialState } = parsedOptions.data;
|
|
69
|
-
|
|
70
|
-
let inputStream;
|
|
71
|
-
if (isAvatarLive && !stream) {
|
|
72
|
-
audioStreamManager = new AudioStreamManager();
|
|
73
|
-
inputStream = audioStreamManager.getStream();
|
|
74
|
-
} else inputStream = stream ?? new MediaStream();
|
|
67
|
+
const inputStream = stream ?? new MediaStream();
|
|
75
68
|
let webrtcManager;
|
|
76
69
|
let telemetryReporter = new NullTelemetryReporter();
|
|
77
70
|
let handleConnectionStateChange = null;
|
|
@@ -106,7 +99,6 @@ const createRealTimeClient = (opts) => {
|
|
|
106
99
|
customizeOffer: options.customizeOffer,
|
|
107
100
|
vp8MinBitrate: 300,
|
|
108
101
|
vp8StartBitrate: 600,
|
|
109
|
-
modelName: options.model.name,
|
|
110
102
|
initialImage,
|
|
111
103
|
initialPrompt
|
|
112
104
|
});
|
|
@@ -227,7 +219,6 @@ const createRealTimeClient = (opts) => {
|
|
|
227
219
|
telemetryReporter.stop();
|
|
228
220
|
stop();
|
|
229
221
|
manager.cleanup();
|
|
230
|
-
audioStreamManager?.cleanup();
|
|
231
222
|
},
|
|
232
223
|
on: eventEmitter.on,
|
|
233
224
|
off: eventEmitter.off,
|
|
@@ -243,16 +234,11 @@ const createRealTimeClient = (opts) => {
|
|
|
243
234
|
return manager.setImage(base64, options);
|
|
244
235
|
}
|
|
245
236
|
};
|
|
246
|
-
if (isAvatarLive && audioStreamManager) {
|
|
247
|
-
const manager = audioStreamManager;
|
|
248
|
-
client.playAudio = (audio) => manager.playAudio(audio);
|
|
249
|
-
}
|
|
250
237
|
flush();
|
|
251
238
|
return client;
|
|
252
239
|
} catch (error) {
|
|
253
240
|
telemetryReporter.stop();
|
|
254
241
|
webrtcManager?.cleanup();
|
|
255
|
-
audioStreamManager?.cleanup();
|
|
256
242
|
throw error;
|
|
257
243
|
}
|
|
258
244
|
};
|
|
@@ -50,15 +50,16 @@ var TelemetryReporter = class {
|
|
|
50
50
|
}
|
|
51
51
|
/** Flush buffered data immediately. */
|
|
52
52
|
flush() {
|
|
53
|
-
this.sendReport(
|
|
53
|
+
this.sendReport();
|
|
54
54
|
}
|
|
55
|
-
/** Stop the reporter and
|
|
55
|
+
/** Stop the reporter and discard any buffered data. */
|
|
56
56
|
stop() {
|
|
57
57
|
if (this.intervalId !== null) {
|
|
58
58
|
clearInterval(this.intervalId);
|
|
59
59
|
this.intervalId = null;
|
|
60
60
|
}
|
|
61
|
-
this.
|
|
61
|
+
this.statsBuffer = [];
|
|
62
|
+
this.diagnosticsBuffer = [];
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
64
65
|
* Build a single chunk from the front of the buffers, respecting MAX_ITEMS_PER_REPORT.
|
|
@@ -82,7 +83,7 @@ var TelemetryReporter = class {
|
|
|
82
83
|
diagnostics: this.diagnosticsBuffer.splice(0, MAX_ITEMS_PER_REPORT)
|
|
83
84
|
};
|
|
84
85
|
}
|
|
85
|
-
sendReport(
|
|
86
|
+
sendReport() {
|
|
86
87
|
if (this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0) return;
|
|
87
88
|
try {
|
|
88
89
|
const commonHeaders = {
|
|
@@ -94,12 +95,10 @@ var TelemetryReporter = class {
|
|
|
94
95
|
};
|
|
95
96
|
let chunk = this.createReportChunk();
|
|
96
97
|
while (chunk !== null) {
|
|
97
|
-
const isLast = this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0;
|
|
98
98
|
fetch(TELEMETRY_URL, {
|
|
99
99
|
method: "POST",
|
|
100
100
|
headers: commonHeaders,
|
|
101
|
-
body: JSON.stringify(chunk)
|
|
102
|
-
keepalive: keepalive && isLast
|
|
101
|
+
body: JSON.stringify(chunk)
|
|
103
102
|
}).then((response) => {
|
|
104
103
|
if (!response.ok) this.logger.warn("Telemetry report rejected", {
|
|
105
104
|
status: response.status,
|
|
@@ -3,7 +3,7 @@ import mitt from "mitt";
|
|
|
3
3
|
|
|
4
4
|
//#region src/realtime/webrtc-connection.ts
|
|
5
5
|
const ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }];
|
|
6
|
-
const
|
|
6
|
+
const SETUP_TIMEOUT_MS = 3e4;
|
|
7
7
|
const noopDiagnostic = () => {};
|
|
8
8
|
var WebRTCConnection = class {
|
|
9
9
|
pc = null;
|
|
@@ -246,7 +246,7 @@ var WebRTCConnection = class {
|
|
|
246
246
|
const timeoutId = setTimeout(() => {
|
|
247
247
|
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
248
248
|
reject(/* @__PURE__ */ new Error("Image send timed out"));
|
|
249
|
-
}, options?.timeout ??
|
|
249
|
+
}, options?.timeout ?? SETUP_TIMEOUT_MS);
|
|
250
250
|
const listener = (msg) => {
|
|
251
251
|
clearTimeout(timeoutId);
|
|
252
252
|
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
@@ -275,7 +275,7 @@ var WebRTCConnection = class {
|
|
|
275
275
|
const timeoutId = setTimeout(() => {
|
|
276
276
|
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
277
277
|
reject(/* @__PURE__ */ new Error("Prompt send timed out"));
|
|
278
|
-
},
|
|
278
|
+
}, SETUP_TIMEOUT_MS);
|
|
279
279
|
const listener = (msg) => {
|
|
280
280
|
if (msg.prompt === prompt.text) {
|
|
281
281
|
clearTimeout(timeoutId);
|
|
@@ -311,12 +311,10 @@ var WebRTCConnection = class {
|
|
|
311
311
|
}
|
|
312
312
|
this.pc = new RTCPeerConnection({ iceServers: ICE_SERVERS });
|
|
313
313
|
this.setState("connecting");
|
|
314
|
-
if (this.localStream) {
|
|
315
|
-
if (this.
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
});
|
|
319
|
-
} else {
|
|
314
|
+
if (this.localStream) this.localStream.getTracks().forEach((track) => {
|
|
315
|
+
if (this.pc && this.localStream) this.pc.addTrack(track, this.localStream);
|
|
316
|
+
});
|
|
317
|
+
else {
|
|
320
318
|
this.pc.addTransceiver("video", { direction: "recvonly" });
|
|
321
319
|
this.pc.addTransceiver("audio", { direction: "recvonly" });
|
|
322
320
|
}
|
|
@@ -43,7 +43,6 @@ var WebRTCManager = class {
|
|
|
43
43
|
customizeOffer: config.customizeOffer,
|
|
44
44
|
vp8MinBitrate: config.vp8MinBitrate,
|
|
45
45
|
vp8StartBitrate: config.vp8StartBitrate,
|
|
46
|
-
modelName: config.modelName,
|
|
47
46
|
initialImage: config.initialImage,
|
|
48
47
|
initialPrompt: config.initialPrompt,
|
|
49
48
|
logger: this.logger,
|
package/dist/shared/model.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
3
|
//#region src/shared/model.d.ts
|
|
4
|
-
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy
|
|
5
|
-
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-
|
|
4
|
+
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">]>;
|
|
5
|
+
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>;
|
|
6
6
|
declare const imageModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>;
|
|
7
|
-
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy
|
|
7
|
+
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
|
|
8
8
|
type Model = z.infer<typeof modelSchema>;
|
|
9
9
|
type RealTimeModels = z.infer<typeof realtimeModels>;
|
|
10
10
|
type VideoModels = z.infer<typeof videoModels>;
|
|
@@ -96,20 +96,6 @@ declare const modelInputSchemas: {
|
|
|
96
96
|
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
97
97
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
98
98
|
}, z.core.$strip>;
|
|
99
|
-
readonly "lucy-motion": z.ZodObject<{
|
|
100
|
-
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
101
|
-
uri: z.ZodString;
|
|
102
|
-
type: z.ZodString;
|
|
103
|
-
name: z.ZodString;
|
|
104
|
-
}, z.core.$strip>]>;
|
|
105
|
-
trajectory: z.ZodArray<z.ZodObject<{
|
|
106
|
-
frame: z.ZodNumber;
|
|
107
|
-
x: z.ZodNumber;
|
|
108
|
-
y: z.ZodNumber;
|
|
109
|
-
}, z.core.$strip>>;
|
|
110
|
-
seed: z.ZodOptional<z.ZodNumber>;
|
|
111
|
-
resolution: z.ZodOptional<z.ZodDefault<z.ZodLiteral<"720p">>>;
|
|
112
|
-
}, z.core.$strip>;
|
|
113
99
|
readonly "lucy-latest": z.ZodObject<{
|
|
114
100
|
prompt: z.ZodString;
|
|
115
101
|
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
@@ -174,20 +160,6 @@ declare const modelInputSchemas: {
|
|
|
174
160
|
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
175
161
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
176
162
|
}, z.core.$strip>;
|
|
177
|
-
readonly "lucy-motion-latest": z.ZodObject<{
|
|
178
|
-
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
179
|
-
uri: z.ZodString;
|
|
180
|
-
type: z.ZodString;
|
|
181
|
-
name: z.ZodString;
|
|
182
|
-
}, z.core.$strip>]>;
|
|
183
|
-
trajectory: z.ZodArray<z.ZodObject<{
|
|
184
|
-
frame: z.ZodNumber;
|
|
185
|
-
x: z.ZodNumber;
|
|
186
|
-
y: z.ZodNumber;
|
|
187
|
-
}, z.core.$strip>>;
|
|
188
|
-
seed: z.ZodOptional<z.ZodNumber>;
|
|
189
|
-
resolution: z.ZodOptional<z.ZodDefault<z.ZodLiteral<"720p">>>;
|
|
190
|
-
}, z.core.$strip>;
|
|
191
163
|
readonly "lucy-image-latest": z.ZodObject<{
|
|
192
164
|
prompt: z.ZodString;
|
|
193
165
|
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
@@ -301,9 +273,6 @@ declare const models: {
|
|
|
301
273
|
* - `"lucy-2.1"` - Lucy 2.1 realtime video editing
|
|
302
274
|
* - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on
|
|
303
275
|
* - `"lucy-restyle-2"` - Realtime video restyling
|
|
304
|
-
* - `"lucy-restyle"` - Legacy realtime restyling
|
|
305
|
-
* - `"lucy"` - Legacy Lucy realtime
|
|
306
|
-
* - `"live-avatar"` - Live avatar
|
|
307
276
|
*/
|
|
308
277
|
realtime: <T extends RealTimeModels>(model: T) => ModelDefinition<T>;
|
|
309
278
|
/**
|
|
@@ -314,7 +283,6 @@ declare const models: {
|
|
|
314
283
|
* - `"lucy-2.1"` - Long-form video editing (Lucy 2.1)
|
|
315
284
|
* - `"lucy-2.1-vton"` - Virtual try-on video editing
|
|
316
285
|
* - `"lucy-restyle-2"` - Video restyling
|
|
317
|
-
* - `"lucy-motion"` - Motion generation
|
|
318
286
|
*/
|
|
319
287
|
video: <T extends VideoModels>(model: T) => ModelDefinition<T> & {
|
|
320
288
|
queueUrlPath: string;
|
package/dist/shared/model.js
CHANGED
|
@@ -7,10 +7,7 @@ import { z } from "zod";
|
|
|
7
7
|
* Old names still work but will log a deprecation warning.
|
|
8
8
|
*/
|
|
9
9
|
const MODEL_ALIASES = {
|
|
10
|
-
mirage: "lucy-restyle",
|
|
11
10
|
mirage_v2: "lucy-restyle-2",
|
|
12
|
-
lucy_v2v_720p_rt: "lucy",
|
|
13
|
-
live_avatar: "live-avatar",
|
|
14
11
|
"lucy-pro-v2v": "lucy-clip",
|
|
15
12
|
"lucy-restyle-v2v": "lucy-restyle-2",
|
|
16
13
|
"lucy-pro-i2i": "lucy-image-2"
|
|
@@ -24,31 +21,23 @@ function warnDeprecated(model) {
|
|
|
24
21
|
}
|
|
25
22
|
}
|
|
26
23
|
const realtimeModels = z.union([
|
|
27
|
-
z.literal("lucy"),
|
|
28
24
|
z.literal("lucy-2.1"),
|
|
29
25
|
z.literal("lucy-2.1-vton"),
|
|
30
|
-
z.literal("lucy-restyle"),
|
|
31
26
|
z.literal("lucy-restyle-2"),
|
|
32
|
-
z.literal("live-avatar"),
|
|
33
27
|
z.literal("lucy-latest"),
|
|
34
28
|
z.literal("lucy-vton-latest"),
|
|
35
29
|
z.literal("lucy-restyle-latest"),
|
|
36
|
-
z.literal("
|
|
37
|
-
z.literal("mirage_v2"),
|
|
38
|
-
z.literal("lucy_v2v_720p_rt"),
|
|
39
|
-
z.literal("live_avatar")
|
|
30
|
+
z.literal("mirage_v2")
|
|
40
31
|
]);
|
|
41
32
|
const videoModels = z.union([
|
|
42
33
|
z.literal("lucy-clip"),
|
|
43
34
|
z.literal("lucy-2.1"),
|
|
44
35
|
z.literal("lucy-2.1-vton"),
|
|
45
36
|
z.literal("lucy-restyle-2"),
|
|
46
|
-
z.literal("lucy-motion"),
|
|
47
37
|
z.literal("lucy-latest"),
|
|
48
38
|
z.literal("lucy-vton-latest"),
|
|
49
39
|
z.literal("lucy-restyle-latest"),
|
|
50
40
|
z.literal("lucy-clip-latest"),
|
|
51
|
-
z.literal("lucy-motion-latest"),
|
|
52
41
|
z.literal("lucy-pro-v2v"),
|
|
53
42
|
z.literal("lucy-restyle-v2v")
|
|
54
43
|
]);
|
|
@@ -91,10 +80,6 @@ const proResolutionSchema = () => {
|
|
|
91
80
|
return z.enum(["720p", "480p"]).optional().describe("The resolution to use for the generation").default("720p");
|
|
92
81
|
};
|
|
93
82
|
/**
|
|
94
|
-
* Resolution schema for lucy-motion.
|
|
95
|
-
*/
|
|
96
|
-
const motionResolutionSchema = z.literal("720p").default("720p").optional().describe("The resolution to use for the generation");
|
|
97
|
-
/**
|
|
98
83
|
* Resolution schema for video-to-video models (supports 720p).
|
|
99
84
|
*/
|
|
100
85
|
const v2vResolutionSchema = z.literal("720p").optional().describe("The resolution to use for the generation").default("720p");
|
|
@@ -136,30 +121,10 @@ const modelInputSchemas = {
|
|
|
136
121
|
"lucy-restyle-2": restyleSchema,
|
|
137
122
|
"lucy-2.1": videoEdit2Schema,
|
|
138
123
|
"lucy-2.1-vton": videoEdit2Schema,
|
|
139
|
-
"lucy-motion": z.object({
|
|
140
|
-
data: fileInputSchema.describe("The image data to use for generation (File, Blob, ReadableStream, URL, or string URL). Output video is limited to 5 seconds."),
|
|
141
|
-
trajectory: z.array(z.object({
|
|
142
|
-
frame: z.number().min(0),
|
|
143
|
-
x: z.number().min(0),
|
|
144
|
-
y: z.number().min(0)
|
|
145
|
-
})).min(2).max(1e3).describe("The trajectory of the desired movement of the object in the image"),
|
|
146
|
-
seed: z.number().optional().describe("The seed to use for the generation"),
|
|
147
|
-
resolution: motionResolutionSchema
|
|
148
|
-
}),
|
|
149
124
|
"lucy-latest": videoEdit2Schema,
|
|
150
125
|
"lucy-vton-latest": videoEdit2Schema,
|
|
151
126
|
"lucy-restyle-latest": restyleSchema,
|
|
152
127
|
"lucy-clip-latest": videoEditSchema,
|
|
153
|
-
"lucy-motion-latest": z.object({
|
|
154
|
-
data: fileInputSchema.describe("The image data to use for generation (File, Blob, ReadableStream, URL, or string URL). Output video is limited to 5 seconds."),
|
|
155
|
-
trajectory: z.array(z.object({
|
|
156
|
-
frame: z.number().min(0),
|
|
157
|
-
x: z.number().min(0),
|
|
158
|
-
y: z.number().min(0)
|
|
159
|
-
})).min(2).max(1e3).describe("The trajectory of the desired movement of the object in the image"),
|
|
160
|
-
seed: z.number().optional().describe("The seed to use for the generation"),
|
|
161
|
-
resolution: motionResolutionSchema
|
|
162
|
-
}),
|
|
163
128
|
"lucy-image-latest": imageEditSchema,
|
|
164
129
|
"lucy-pro-v2v": videoEditSchema,
|
|
165
130
|
"lucy-pro-i2i": imageEditSchema,
|
|
@@ -176,14 +141,6 @@ const modelDefinitionSchema = z.object({
|
|
|
176
141
|
});
|
|
177
142
|
const _models = {
|
|
178
143
|
realtime: {
|
|
179
|
-
lucy: {
|
|
180
|
-
urlPath: "/v1/stream",
|
|
181
|
-
name: "lucy",
|
|
182
|
-
fps: 25,
|
|
183
|
-
width: 1280,
|
|
184
|
-
height: 704,
|
|
185
|
-
inputSchema: z.object({})
|
|
186
|
-
},
|
|
187
144
|
"lucy-2.1": {
|
|
188
145
|
urlPath: "/v1/stream",
|
|
189
146
|
name: "lucy-2.1",
|
|
@@ -200,14 +157,6 @@ const _models = {
|
|
|
200
157
|
height: 624,
|
|
201
158
|
inputSchema: z.object({})
|
|
202
159
|
},
|
|
203
|
-
"lucy-restyle": {
|
|
204
|
-
urlPath: "/v1/stream",
|
|
205
|
-
name: "lucy-restyle",
|
|
206
|
-
fps: 25,
|
|
207
|
-
width: 1280,
|
|
208
|
-
height: 704,
|
|
209
|
-
inputSchema: z.object({})
|
|
210
|
-
},
|
|
211
160
|
"lucy-restyle-2": {
|
|
212
161
|
urlPath: "/v1/stream",
|
|
213
162
|
name: "lucy-restyle-2",
|
|
@@ -216,14 +165,6 @@ const _models = {
|
|
|
216
165
|
height: 704,
|
|
217
166
|
inputSchema: z.object({})
|
|
218
167
|
},
|
|
219
|
-
"live-avatar": {
|
|
220
|
-
urlPath: "/v1/stream",
|
|
221
|
-
name: "live-avatar",
|
|
222
|
-
fps: 25,
|
|
223
|
-
width: 1280,
|
|
224
|
-
height: 720,
|
|
225
|
-
inputSchema: z.object({})
|
|
226
|
-
},
|
|
227
168
|
"lucy-latest": {
|
|
228
169
|
urlPath: "/v1/stream",
|
|
229
170
|
name: "lucy-latest",
|
|
@@ -248,14 +189,6 @@ const _models = {
|
|
|
248
189
|
height: 704,
|
|
249
190
|
inputSchema: z.object({})
|
|
250
191
|
},
|
|
251
|
-
mirage: {
|
|
252
|
-
urlPath: "/v1/stream",
|
|
253
|
-
name: "mirage",
|
|
254
|
-
fps: 25,
|
|
255
|
-
width: 1280,
|
|
256
|
-
height: 704,
|
|
257
|
-
inputSchema: z.object({})
|
|
258
|
-
},
|
|
259
192
|
mirage_v2: {
|
|
260
193
|
urlPath: "/v1/stream",
|
|
261
194
|
name: "mirage_v2",
|
|
@@ -263,22 +196,6 @@ const _models = {
|
|
|
263
196
|
width: 1280,
|
|
264
197
|
height: 704,
|
|
265
198
|
inputSchema: z.object({})
|
|
266
|
-
},
|
|
267
|
-
lucy_v2v_720p_rt: {
|
|
268
|
-
urlPath: "/v1/stream",
|
|
269
|
-
name: "lucy_v2v_720p_rt",
|
|
270
|
-
fps: 25,
|
|
271
|
-
width: 1280,
|
|
272
|
-
height: 704,
|
|
273
|
-
inputSchema: z.object({})
|
|
274
|
-
},
|
|
275
|
-
live_avatar: {
|
|
276
|
-
urlPath: "/v1/stream",
|
|
277
|
-
name: "live_avatar",
|
|
278
|
-
fps: 25,
|
|
279
|
-
width: 1280,
|
|
280
|
-
height: 720,
|
|
281
|
-
inputSchema: z.object({})
|
|
282
199
|
}
|
|
283
200
|
},
|
|
284
201
|
image: {
|
|
@@ -347,15 +264,6 @@ const _models = {
|
|
|
347
264
|
height: 704,
|
|
348
265
|
inputSchema: modelInputSchemas["lucy-restyle-2"]
|
|
349
266
|
},
|
|
350
|
-
"lucy-motion": {
|
|
351
|
-
urlPath: "/v1/generate/lucy-motion",
|
|
352
|
-
queueUrlPath: "/v1/jobs/lucy-motion",
|
|
353
|
-
name: "lucy-motion",
|
|
354
|
-
fps: 25,
|
|
355
|
-
width: 1280,
|
|
356
|
-
height: 704,
|
|
357
|
-
inputSchema: modelInputSchemas["lucy-motion"]
|
|
358
|
-
},
|
|
359
267
|
"lucy-latest": {
|
|
360
268
|
urlPath: "/v1/generate/lucy-latest",
|
|
361
269
|
queueUrlPath: "/v1/jobs/lucy-latest",
|
|
@@ -392,15 +300,6 @@ const _models = {
|
|
|
392
300
|
height: 704,
|
|
393
301
|
inputSchema: modelInputSchemas["lucy-clip-latest"]
|
|
394
302
|
},
|
|
395
|
-
"lucy-motion-latest": {
|
|
396
|
-
urlPath: "/v1/generate/lucy-motion-latest",
|
|
397
|
-
queueUrlPath: "/v1/jobs/lucy-motion-latest",
|
|
398
|
-
name: "lucy-motion-latest",
|
|
399
|
-
fps: 25,
|
|
400
|
-
width: 1280,
|
|
401
|
-
height: 704,
|
|
402
|
-
inputSchema: modelInputSchemas["lucy-motion-latest"]
|
|
403
|
-
},
|
|
404
303
|
"lucy-pro-v2v": {
|
|
405
304
|
urlPath: "/v1/generate/lucy-pro-v2v",
|
|
406
305
|
queueUrlPath: "/v1/jobs/lucy-pro-v2v",
|
package/package.json
CHANGED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
//#region src/realtime/audio-stream-manager.ts
|
|
2
|
-
/**
|
|
3
|
-
* Manages an audio stream for live_avatar mode.
|
|
4
|
-
* Creates a continuous audio stream that outputs silence by default,
|
|
5
|
-
* and allows playing audio files through the stream.
|
|
6
|
-
*/
|
|
7
|
-
var AudioStreamManager = class {
|
|
8
|
-
audioContext;
|
|
9
|
-
destination;
|
|
10
|
-
silenceOscillator;
|
|
11
|
-
silenceGain;
|
|
12
|
-
currentSource = null;
|
|
13
|
-
_isPlaying = false;
|
|
14
|
-
constructor() {
|
|
15
|
-
this.audioContext = new AudioContext();
|
|
16
|
-
this.destination = this.audioContext.createMediaStreamDestination();
|
|
17
|
-
this.silenceOscillator = this.audioContext.createOscillator();
|
|
18
|
-
this.silenceGain = this.audioContext.createGain();
|
|
19
|
-
this.silenceGain.gain.value = 0;
|
|
20
|
-
this.silenceOscillator.connect(this.silenceGain);
|
|
21
|
-
this.silenceGain.connect(this.destination);
|
|
22
|
-
this.silenceOscillator.start();
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Get the MediaStream to pass to WebRTC.
|
|
26
|
-
* This stream outputs silence when no audio is playing.
|
|
27
|
-
*/
|
|
28
|
-
getStream() {
|
|
29
|
-
return this.destination.stream;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Check if audio is currently playing.
|
|
33
|
-
*/
|
|
34
|
-
isPlaying() {
|
|
35
|
-
return this._isPlaying;
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Play audio through the stream.
|
|
39
|
-
* When the audio ends, the stream automatically reverts to silence.
|
|
40
|
-
* @param audio - Audio data as Blob, File, or ArrayBuffer
|
|
41
|
-
* @returns Promise that resolves when audio finishes playing
|
|
42
|
-
*/
|
|
43
|
-
async playAudio(audio) {
|
|
44
|
-
if (this.audioContext.state === "suspended") await this.audioContext.resume();
|
|
45
|
-
if (this._isPlaying) this.stopAudio();
|
|
46
|
-
let arrayBuffer;
|
|
47
|
-
if (audio instanceof ArrayBuffer) arrayBuffer = audio;
|
|
48
|
-
else arrayBuffer = await audio.arrayBuffer();
|
|
49
|
-
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
|
50
|
-
const source = this.audioContext.createBufferSource();
|
|
51
|
-
source.buffer = audioBuffer;
|
|
52
|
-
source.connect(this.destination);
|
|
53
|
-
this.currentSource = source;
|
|
54
|
-
this._isPlaying = true;
|
|
55
|
-
return new Promise((resolve) => {
|
|
56
|
-
source.onended = () => {
|
|
57
|
-
this._isPlaying = false;
|
|
58
|
-
this.currentSource = null;
|
|
59
|
-
resolve();
|
|
60
|
-
};
|
|
61
|
-
source.start();
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Stop currently playing audio immediately.
|
|
66
|
-
* The stream will revert to silence.
|
|
67
|
-
*/
|
|
68
|
-
stopAudio() {
|
|
69
|
-
if (this.currentSource) {
|
|
70
|
-
try {
|
|
71
|
-
this.currentSource.stop();
|
|
72
|
-
} catch {}
|
|
73
|
-
this.currentSource = null;
|
|
74
|
-
}
|
|
75
|
-
this._isPlaying = false;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Clean up resources.
|
|
79
|
-
*/
|
|
80
|
-
cleanup() {
|
|
81
|
-
this.stopAudio();
|
|
82
|
-
try {
|
|
83
|
-
this.silenceOscillator.stop();
|
|
84
|
-
} catch {}
|
|
85
|
-
this.audioContext.close().catch(() => {});
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
|
|
89
|
-
//#endregion
|
|
90
|
-
export { AudioStreamManager };
|