@decartai/sdk 0.0.63 → 0.0.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/process/types.d.ts +1 -1
- package/dist/realtime/client.d.ts +0 -1
- package/dist/realtime/client.js +1 -15
- package/dist/realtime/telemetry-reporter.js +6 -7
- package/dist/realtime/webrtc-connection.js +7 -9
- package/dist/realtime/webrtc-manager.js +0 -1
- package/dist/shared/model.d.ts +44 -26
- package/dist/shared/model.js +52 -91
- package/package.json +1 -1
- package/dist/realtime/audio-stream-manager.js +0 -90
package/README.md
CHANGED
|
@@ -24,7 +24,7 @@ For complete documentation, guides, and examples, visit:
|
|
|
24
24
|
```typescript
|
|
25
25
|
import { createDecartClient, models } from "@decartai/sdk";
|
|
26
26
|
|
|
27
|
-
const model = models.realtime("
|
|
27
|
+
const model = models.realtime("lucy-restyle-2");
|
|
28
28
|
|
|
29
29
|
// Get user's camera stream
|
|
30
30
|
const stream = await navigator.mediaDevices.getUserMedia({
|
package/dist/process/types.d.ts
CHANGED
|
@@ -143,7 +143,7 @@ interface PromptInput {
|
|
|
143
143
|
* This allows different models to have field-specific documentation while maintaining type safety.
|
|
144
144
|
* Specific models are checked first, then falls back to category-based selection.
|
|
145
145
|
*/
|
|
146
|
-
type ModelSpecificInputs<T extends ModelDefinition> = T["name"] extends "lucy-pro-i2i" | "lucy-image-2" ? ImageEditingInputs : T["name"] extends "lucy-restyle-v2v" | "lucy-restyle-2" ? VideoRestyleInputs : T["name"] extends "lucy-2.1" | "lucy-2.1-vton" ? VideoEdit2Inputs : T["name"] extends "lucy-pro-v2v" | "lucy-clip" ? VideoEditInputs : T["name"] extends ImageModels ? ImageGenerationInputs : T["name"] extends VideoModels ? VideoModelInputs : PromptInput;
|
|
146
|
+
type ModelSpecificInputs<T extends ModelDefinition> = T["name"] extends "lucy-pro-i2i" | "lucy-image-2" ? ImageEditingInputs : T["name"] extends "lucy-restyle-v2v" | "lucy-restyle-2" ? VideoRestyleInputs : T["name"] extends "lucy-2.1" | "lucy-2.1-vton" | "lucy-vton-2" | "lucy-vton" | "lucy-2.1-vton-2" | "lucy-vton-latest" ? VideoEdit2Inputs : T["name"] extends "lucy-pro-v2v" | "lucy-clip" ? VideoEditInputs : T["name"] extends ImageModels ? ImageGenerationInputs : T["name"] extends VideoModels ? VideoModelInputs : PromptInput;
|
|
147
147
|
interface ProcessInputs {
|
|
148
148
|
/**
|
|
149
149
|
* Random seed for reproducible results.
|
|
@@ -68,7 +68,6 @@ type RealTimeClient = {
|
|
|
68
68
|
enhance?: boolean;
|
|
69
69
|
timeout?: number;
|
|
70
70
|
}) => Promise<void>;
|
|
71
|
-
playAudio?: (audio: Blob | File | ArrayBuffer) => Promise<void>;
|
|
72
71
|
};
|
|
73
72
|
//#endregion
|
|
74
73
|
export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
|
package/dist/realtime/client.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
2
|
import { modelDefinitionSchema } from "../shared/model.js";
|
|
3
3
|
import { modelStateSchema } from "../shared/types.js";
|
|
4
|
-
import { AudioStreamManager } from "./audio-stream-manager.js";
|
|
5
4
|
import { createEventBuffer } from "./event-buffer.js";
|
|
6
5
|
import { realtimeMethods } from "./methods.js";
|
|
7
6
|
import { decodeSubscribeToken, encodeSubscribeToken } from "./subscribe-client.js";
|
|
@@ -64,14 +63,8 @@ const createRealTimeClient = (opts) => {
|
|
|
64
63
|
const connect = async (stream, options) => {
|
|
65
64
|
const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
|
|
66
65
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
67
|
-
const isAvatarLive = options.model.name === "live_avatar" || options.model.name === "live-avatar";
|
|
68
66
|
const { onRemoteStream, initialState } = parsedOptions.data;
|
|
69
|
-
|
|
70
|
-
let inputStream;
|
|
71
|
-
if (isAvatarLive && !stream) {
|
|
72
|
-
audioStreamManager = new AudioStreamManager();
|
|
73
|
-
inputStream = audioStreamManager.getStream();
|
|
74
|
-
} else inputStream = stream ?? new MediaStream();
|
|
67
|
+
const inputStream = stream ?? new MediaStream();
|
|
75
68
|
let webrtcManager;
|
|
76
69
|
let telemetryReporter = new NullTelemetryReporter();
|
|
77
70
|
let handleConnectionStateChange = null;
|
|
@@ -106,7 +99,6 @@ const createRealTimeClient = (opts) => {
|
|
|
106
99
|
customizeOffer: options.customizeOffer,
|
|
107
100
|
vp8MinBitrate: 300,
|
|
108
101
|
vp8StartBitrate: 600,
|
|
109
|
-
modelName: options.model.name,
|
|
110
102
|
initialImage,
|
|
111
103
|
initialPrompt
|
|
112
104
|
});
|
|
@@ -227,7 +219,6 @@ const createRealTimeClient = (opts) => {
|
|
|
227
219
|
telemetryReporter.stop();
|
|
228
220
|
stop();
|
|
229
221
|
manager.cleanup();
|
|
230
|
-
audioStreamManager?.cleanup();
|
|
231
222
|
},
|
|
232
223
|
on: eventEmitter.on,
|
|
233
224
|
off: eventEmitter.off,
|
|
@@ -243,16 +234,11 @@ const createRealTimeClient = (opts) => {
|
|
|
243
234
|
return manager.setImage(base64, options);
|
|
244
235
|
}
|
|
245
236
|
};
|
|
246
|
-
if (isAvatarLive && audioStreamManager) {
|
|
247
|
-
const manager = audioStreamManager;
|
|
248
|
-
client.playAudio = (audio) => manager.playAudio(audio);
|
|
249
|
-
}
|
|
250
237
|
flush();
|
|
251
238
|
return client;
|
|
252
239
|
} catch (error) {
|
|
253
240
|
telemetryReporter.stop();
|
|
254
241
|
webrtcManager?.cleanup();
|
|
255
|
-
audioStreamManager?.cleanup();
|
|
256
242
|
throw error;
|
|
257
243
|
}
|
|
258
244
|
};
|
|
@@ -50,15 +50,16 @@ var TelemetryReporter = class {
|
|
|
50
50
|
}
|
|
51
51
|
/** Flush buffered data immediately. */
|
|
52
52
|
flush() {
|
|
53
|
-
this.sendReport(
|
|
53
|
+
this.sendReport();
|
|
54
54
|
}
|
|
55
|
-
/** Stop the reporter and
|
|
55
|
+
/** Stop the reporter and discard any buffered data. */
|
|
56
56
|
stop() {
|
|
57
57
|
if (this.intervalId !== null) {
|
|
58
58
|
clearInterval(this.intervalId);
|
|
59
59
|
this.intervalId = null;
|
|
60
60
|
}
|
|
61
|
-
this.
|
|
61
|
+
this.statsBuffer = [];
|
|
62
|
+
this.diagnosticsBuffer = [];
|
|
62
63
|
}
|
|
63
64
|
/**
|
|
64
65
|
* Build a single chunk from the front of the buffers, respecting MAX_ITEMS_PER_REPORT.
|
|
@@ -82,7 +83,7 @@ var TelemetryReporter = class {
|
|
|
82
83
|
diagnostics: this.diagnosticsBuffer.splice(0, MAX_ITEMS_PER_REPORT)
|
|
83
84
|
};
|
|
84
85
|
}
|
|
85
|
-
sendReport(
|
|
86
|
+
sendReport() {
|
|
86
87
|
if (this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0) return;
|
|
87
88
|
try {
|
|
88
89
|
const commonHeaders = {
|
|
@@ -94,12 +95,10 @@ var TelemetryReporter = class {
|
|
|
94
95
|
};
|
|
95
96
|
let chunk = this.createReportChunk();
|
|
96
97
|
while (chunk !== null) {
|
|
97
|
-
const isLast = this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0;
|
|
98
98
|
fetch(TELEMETRY_URL, {
|
|
99
99
|
method: "POST",
|
|
100
100
|
headers: commonHeaders,
|
|
101
|
-
body: JSON.stringify(chunk)
|
|
102
|
-
keepalive: keepalive && isLast
|
|
101
|
+
body: JSON.stringify(chunk)
|
|
103
102
|
}).then((response) => {
|
|
104
103
|
if (!response.ok) this.logger.warn("Telemetry report rejected", {
|
|
105
104
|
status: response.status,
|
|
@@ -3,7 +3,7 @@ import mitt from "mitt";
|
|
|
3
3
|
|
|
4
4
|
//#region src/realtime/webrtc-connection.ts
|
|
5
5
|
const ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }];
|
|
6
|
-
const
|
|
6
|
+
const SETUP_TIMEOUT_MS = 3e4;
|
|
7
7
|
const noopDiagnostic = () => {};
|
|
8
8
|
var WebRTCConnection = class {
|
|
9
9
|
pc = null;
|
|
@@ -246,7 +246,7 @@ var WebRTCConnection = class {
|
|
|
246
246
|
const timeoutId = setTimeout(() => {
|
|
247
247
|
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
248
248
|
reject(/* @__PURE__ */ new Error("Image send timed out"));
|
|
249
|
-
}, options?.timeout ??
|
|
249
|
+
}, options?.timeout ?? SETUP_TIMEOUT_MS);
|
|
250
250
|
const listener = (msg) => {
|
|
251
251
|
clearTimeout(timeoutId);
|
|
252
252
|
this.websocketMessagesEmitter.off("setImageAck", listener);
|
|
@@ -275,7 +275,7 @@ var WebRTCConnection = class {
|
|
|
275
275
|
const timeoutId = setTimeout(() => {
|
|
276
276
|
this.websocketMessagesEmitter.off("promptAck", listener);
|
|
277
277
|
reject(/* @__PURE__ */ new Error("Prompt send timed out"));
|
|
278
|
-
},
|
|
278
|
+
}, SETUP_TIMEOUT_MS);
|
|
279
279
|
const listener = (msg) => {
|
|
280
280
|
if (msg.prompt === prompt.text) {
|
|
281
281
|
clearTimeout(timeoutId);
|
|
@@ -311,12 +311,10 @@ var WebRTCConnection = class {
|
|
|
311
311
|
}
|
|
312
312
|
this.pc = new RTCPeerConnection({ iceServers: ICE_SERVERS });
|
|
313
313
|
this.setState("connecting");
|
|
314
|
-
if (this.localStream) {
|
|
315
|
-
if (this.
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
});
|
|
319
|
-
} else {
|
|
314
|
+
if (this.localStream) this.localStream.getTracks().forEach((track) => {
|
|
315
|
+
if (this.pc && this.localStream) this.pc.addTrack(track, this.localStream);
|
|
316
|
+
});
|
|
317
|
+
else {
|
|
320
318
|
this.pc.addTransceiver("video", { direction: "recvonly" });
|
|
321
319
|
this.pc.addTransceiver("audio", { direction: "recvonly" });
|
|
322
320
|
}
|
|
@@ -43,7 +43,6 @@ var WebRTCManager = class {
|
|
|
43
43
|
customizeOffer: config.customizeOffer,
|
|
44
44
|
vp8MinBitrate: config.vp8MinBitrate,
|
|
45
45
|
vp8StartBitrate: config.vp8StartBitrate,
|
|
46
|
-
modelName: config.modelName,
|
|
47
46
|
initialImage: config.initialImage,
|
|
48
47
|
initialPrompt: config.initialPrompt,
|
|
49
48
|
logger: this.logger,
|
package/dist/shared/model.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
3
|
//#region src/shared/model.d.ts
|
|
4
|
-
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy
|
|
5
|
-
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-
|
|
4
|
+
declare const realtimeModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">]>;
|
|
5
|
+
declare const videoModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>;
|
|
6
6
|
declare const imageModels: z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>;
|
|
7
|
-
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy
|
|
7
|
+
declare const modelSchema: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"mirage_v2">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-clip">, z.ZodLiteral<"lucy-2.1">, z.ZodLiteral<"lucy-2.1-vton">, z.ZodLiteral<"lucy-vton-2">, z.ZodLiteral<"lucy-restyle-2">, z.ZodLiteral<"lucy-latest">, z.ZodLiteral<"lucy-vton-latest">, z.ZodLiteral<"lucy-restyle-latest">, z.ZodLiteral<"lucy-clip-latest">, z.ZodLiteral<"lucy-vton">, z.ZodLiteral<"lucy-2.1-vton-2">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-image-2">, z.ZodLiteral<"lucy-image-latest">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
|
|
8
8
|
type Model = z.infer<typeof modelSchema>;
|
|
9
9
|
type RealTimeModels = z.infer<typeof realtimeModels>;
|
|
10
10
|
type VideoModels = z.infer<typeof videoModels>;
|
|
@@ -96,19 +96,21 @@ declare const modelInputSchemas: {
|
|
|
96
96
|
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
97
97
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
98
98
|
}, z.core.$strip>;
|
|
99
|
-
readonly "lucy-
|
|
99
|
+
readonly "lucy-vton-2": z.ZodObject<{
|
|
100
|
+
prompt: z.ZodString;
|
|
101
|
+
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
102
|
+
uri: z.ZodString;
|
|
103
|
+
type: z.ZodString;
|
|
104
|
+
name: z.ZodString;
|
|
105
|
+
}, z.core.$strip>]>>;
|
|
100
106
|
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
101
107
|
uri: z.ZodString;
|
|
102
108
|
type: z.ZodString;
|
|
103
109
|
name: z.ZodString;
|
|
104
110
|
}, z.core.$strip>]>;
|
|
105
|
-
trajectory: z.ZodArray<z.ZodObject<{
|
|
106
|
-
frame: z.ZodNumber;
|
|
107
|
-
x: z.ZodNumber;
|
|
108
|
-
y: z.ZodNumber;
|
|
109
|
-
}, z.core.$strip>>;
|
|
110
111
|
seed: z.ZodOptional<z.ZodNumber>;
|
|
111
|
-
resolution: z.
|
|
112
|
+
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
113
|
+
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
112
114
|
}, z.core.$strip>;
|
|
113
115
|
readonly "lucy-latest": z.ZodObject<{
|
|
114
116
|
prompt: z.ZodString;
|
|
@@ -174,37 +176,55 @@ declare const modelInputSchemas: {
|
|
|
174
176
|
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
175
177
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
176
178
|
}, z.core.$strip>;
|
|
177
|
-
readonly "lucy-
|
|
179
|
+
readonly "lucy-image-latest": z.ZodObject<{
|
|
180
|
+
prompt: z.ZodString;
|
|
178
181
|
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
179
182
|
uri: z.ZodString;
|
|
180
183
|
type: z.ZodString;
|
|
181
184
|
name: z.ZodString;
|
|
182
185
|
}, z.core.$strip>]>;
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
}, z.core.$strip>>;
|
|
186
|
+
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
187
|
+
uri: z.ZodString;
|
|
188
|
+
type: z.ZodString;
|
|
189
|
+
name: z.ZodString;
|
|
190
|
+
}, z.core.$strip>]>>;
|
|
188
191
|
seed: z.ZodOptional<z.ZodNumber>;
|
|
189
|
-
resolution: z.
|
|
192
|
+
resolution: z.ZodDefault<z.ZodOptional<z.ZodEnum<{
|
|
193
|
+
"720p": "720p";
|
|
194
|
+
"480p": "480p";
|
|
195
|
+
}>>>;
|
|
196
|
+
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
190
197
|
}, z.core.$strip>;
|
|
191
|
-
readonly "lucy-
|
|
198
|
+
readonly "lucy-vton": z.ZodObject<{
|
|
192
199
|
prompt: z.ZodString;
|
|
200
|
+
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
201
|
+
uri: z.ZodString;
|
|
202
|
+
type: z.ZodString;
|
|
203
|
+
name: z.ZodString;
|
|
204
|
+
}, z.core.$strip>]>>;
|
|
193
205
|
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
194
206
|
uri: z.ZodString;
|
|
195
207
|
type: z.ZodString;
|
|
196
208
|
name: z.ZodString;
|
|
197
209
|
}, z.core.$strip>]>;
|
|
210
|
+
seed: z.ZodOptional<z.ZodNumber>;
|
|
211
|
+
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
212
|
+
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
213
|
+
}, z.core.$strip>;
|
|
214
|
+
readonly "lucy-2.1-vton-2": z.ZodObject<{
|
|
215
|
+
prompt: z.ZodString;
|
|
198
216
|
reference_image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
199
217
|
uri: z.ZodString;
|
|
200
218
|
type: z.ZodString;
|
|
201
219
|
name: z.ZodString;
|
|
202
220
|
}, z.core.$strip>]>>;
|
|
221
|
+
data: z.ZodUnion<readonly [z.ZodCustom<File, File>, z.ZodCustom<Blob, Blob>, z.ZodCustom<ReadableStream<unknown>, ReadableStream<unknown>>, z.ZodCustom<URL, URL>, z.ZodURL, z.ZodObject<{
|
|
222
|
+
uri: z.ZodString;
|
|
223
|
+
type: z.ZodString;
|
|
224
|
+
name: z.ZodString;
|
|
225
|
+
}, z.core.$strip>]>;
|
|
203
226
|
seed: z.ZodOptional<z.ZodNumber>;
|
|
204
|
-
resolution: z.ZodDefault<z.ZodOptional<z.
|
|
205
|
-
"720p": "720p";
|
|
206
|
-
"480p": "480p";
|
|
207
|
-
}>>>;
|
|
227
|
+
resolution: z.ZodDefault<z.ZodOptional<z.ZodLiteral<"720p">>>;
|
|
208
228
|
enhance_prompt: z.ZodOptional<z.ZodBoolean>;
|
|
209
229
|
}, z.core.$strip>;
|
|
210
230
|
readonly "lucy-pro-v2v": z.ZodObject<{
|
|
@@ -300,10 +320,8 @@ declare const models: {
|
|
|
300
320
|
* Available options:
|
|
301
321
|
* - `"lucy-2.1"` - Lucy 2.1 realtime video editing
|
|
302
322
|
* - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on
|
|
323
|
+
* - `"lucy-vton-2"` - Lucy virtual try-on 2
|
|
303
324
|
* - `"lucy-restyle-2"` - Realtime video restyling
|
|
304
|
-
* - `"lucy-restyle"` - Legacy realtime restyling
|
|
305
|
-
* - `"lucy"` - Legacy Lucy realtime
|
|
306
|
-
* - `"live-avatar"` - Live avatar
|
|
307
325
|
*/
|
|
308
326
|
realtime: <T extends RealTimeModels>(model: T) => ModelDefinition<T>;
|
|
309
327
|
/**
|
|
@@ -313,8 +331,8 @@ declare const models: {
|
|
|
313
331
|
* - `"lucy-clip"` - Video-to-video editing
|
|
314
332
|
* - `"lucy-2.1"` - Long-form video editing (Lucy 2.1)
|
|
315
333
|
* - `"lucy-2.1-vton"` - Virtual try-on video editing
|
|
334
|
+
* - `"lucy-vton-2"` - Virtual try-on 2 video editing
|
|
316
335
|
* - `"lucy-restyle-2"` - Video restyling
|
|
317
|
-
* - `"lucy-motion"` - Motion generation
|
|
318
336
|
*/
|
|
319
337
|
video: <T extends VideoModels>(model: T) => ModelDefinition<T> & {
|
|
320
338
|
queueUrlPath: string;
|
package/dist/shared/model.js
CHANGED
|
@@ -7,10 +7,9 @@ import { z } from "zod";
|
|
|
7
7
|
* Old names still work but will log a deprecation warning.
|
|
8
8
|
*/
|
|
9
9
|
const MODEL_ALIASES = {
|
|
10
|
-
mirage: "lucy-restyle",
|
|
11
10
|
mirage_v2: "lucy-restyle-2",
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
"lucy-vton": "lucy-2.1-vton",
|
|
12
|
+
"lucy-2.1-vton-2": "lucy-vton-2",
|
|
14
13
|
"lucy-pro-v2v": "lucy-clip",
|
|
15
14
|
"lucy-restyle-v2v": "lucy-restyle-2",
|
|
16
15
|
"lucy-pro-i2i": "lucy-image-2"
|
|
@@ -24,31 +23,29 @@ function warnDeprecated(model) {
|
|
|
24
23
|
}
|
|
25
24
|
}
|
|
26
25
|
const realtimeModels = z.union([
|
|
27
|
-
z.literal("lucy"),
|
|
28
26
|
z.literal("lucy-2.1"),
|
|
29
27
|
z.literal("lucy-2.1-vton"),
|
|
30
|
-
z.literal("lucy-
|
|
28
|
+
z.literal("lucy-vton-2"),
|
|
31
29
|
z.literal("lucy-restyle-2"),
|
|
32
|
-
z.literal("live-avatar"),
|
|
33
30
|
z.literal("lucy-latest"),
|
|
34
31
|
z.literal("lucy-vton-latest"),
|
|
35
32
|
z.literal("lucy-restyle-latest"),
|
|
36
|
-
z.literal("mirage"),
|
|
37
33
|
z.literal("mirage_v2"),
|
|
38
|
-
z.literal("
|
|
39
|
-
z.literal("
|
|
34
|
+
z.literal("lucy-vton"),
|
|
35
|
+
z.literal("lucy-2.1-vton-2")
|
|
40
36
|
]);
|
|
41
37
|
const videoModels = z.union([
|
|
42
38
|
z.literal("lucy-clip"),
|
|
43
39
|
z.literal("lucy-2.1"),
|
|
44
40
|
z.literal("lucy-2.1-vton"),
|
|
41
|
+
z.literal("lucy-vton-2"),
|
|
45
42
|
z.literal("lucy-restyle-2"),
|
|
46
|
-
z.literal("lucy-motion"),
|
|
47
43
|
z.literal("lucy-latest"),
|
|
48
44
|
z.literal("lucy-vton-latest"),
|
|
49
45
|
z.literal("lucy-restyle-latest"),
|
|
50
46
|
z.literal("lucy-clip-latest"),
|
|
51
|
-
z.literal("lucy-
|
|
47
|
+
z.literal("lucy-vton"),
|
|
48
|
+
z.literal("lucy-2.1-vton-2"),
|
|
52
49
|
z.literal("lucy-pro-v2v"),
|
|
53
50
|
z.literal("lucy-restyle-v2v")
|
|
54
51
|
]);
|
|
@@ -91,10 +88,6 @@ const proResolutionSchema = () => {
|
|
|
91
88
|
return z.enum(["720p", "480p"]).optional().describe("The resolution to use for the generation").default("720p");
|
|
92
89
|
};
|
|
93
90
|
/**
|
|
94
|
-
* Resolution schema for lucy-motion.
|
|
95
|
-
*/
|
|
96
|
-
const motionResolutionSchema = z.literal("720p").default("720p").optional().describe("The resolution to use for the generation");
|
|
97
|
-
/**
|
|
98
91
|
* Resolution schema for video-to-video models (supports 720p).
|
|
99
92
|
*/
|
|
100
93
|
const v2vResolutionSchema = z.literal("720p").optional().describe("The resolution to use for the generation").default("720p");
|
|
@@ -136,31 +129,14 @@ const modelInputSchemas = {
|
|
|
136
129
|
"lucy-restyle-2": restyleSchema,
|
|
137
130
|
"lucy-2.1": videoEdit2Schema,
|
|
138
131
|
"lucy-2.1-vton": videoEdit2Schema,
|
|
139
|
-
"lucy-
|
|
140
|
-
data: fileInputSchema.describe("The image data to use for generation (File, Blob, ReadableStream, URL, or string URL). Output video is limited to 5 seconds."),
|
|
141
|
-
trajectory: z.array(z.object({
|
|
142
|
-
frame: z.number().min(0),
|
|
143
|
-
x: z.number().min(0),
|
|
144
|
-
y: z.number().min(0)
|
|
145
|
-
})).min(2).max(1e3).describe("The trajectory of the desired movement of the object in the image"),
|
|
146
|
-
seed: z.number().optional().describe("The seed to use for the generation"),
|
|
147
|
-
resolution: motionResolutionSchema
|
|
148
|
-
}),
|
|
132
|
+
"lucy-vton-2": videoEdit2Schema,
|
|
149
133
|
"lucy-latest": videoEdit2Schema,
|
|
150
134
|
"lucy-vton-latest": videoEdit2Schema,
|
|
151
135
|
"lucy-restyle-latest": restyleSchema,
|
|
152
136
|
"lucy-clip-latest": videoEditSchema,
|
|
153
|
-
"lucy-motion-latest": z.object({
|
|
154
|
-
data: fileInputSchema.describe("The image data to use for generation (File, Blob, ReadableStream, URL, or string URL). Output video is limited to 5 seconds."),
|
|
155
|
-
trajectory: z.array(z.object({
|
|
156
|
-
frame: z.number().min(0),
|
|
157
|
-
x: z.number().min(0),
|
|
158
|
-
y: z.number().min(0)
|
|
159
|
-
})).min(2).max(1e3).describe("The trajectory of the desired movement of the object in the image"),
|
|
160
|
-
seed: z.number().optional().describe("The seed to use for the generation"),
|
|
161
|
-
resolution: motionResolutionSchema
|
|
162
|
-
}),
|
|
163
137
|
"lucy-image-latest": imageEditSchema,
|
|
138
|
+
"lucy-vton": videoEdit2Schema,
|
|
139
|
+
"lucy-2.1-vton-2": videoEdit2Schema,
|
|
164
140
|
"lucy-pro-v2v": videoEditSchema,
|
|
165
141
|
"lucy-pro-i2i": imageEditSchema,
|
|
166
142
|
"lucy-restyle-v2v": restyleSchema
|
|
@@ -176,14 +152,6 @@ const modelDefinitionSchema = z.object({
|
|
|
176
152
|
});
|
|
177
153
|
const _models = {
|
|
178
154
|
realtime: {
|
|
179
|
-
lucy: {
|
|
180
|
-
urlPath: "/v1/stream",
|
|
181
|
-
name: "lucy",
|
|
182
|
-
fps: 25,
|
|
183
|
-
width: 1280,
|
|
184
|
-
height: 704,
|
|
185
|
-
inputSchema: z.object({})
|
|
186
|
-
},
|
|
187
155
|
"lucy-2.1": {
|
|
188
156
|
urlPath: "/v1/stream",
|
|
189
157
|
name: "lucy-2.1",
|
|
@@ -200,12 +168,12 @@ const _models = {
|
|
|
200
168
|
height: 624,
|
|
201
169
|
inputSchema: z.object({})
|
|
202
170
|
},
|
|
203
|
-
"lucy-
|
|
171
|
+
"lucy-vton-2": {
|
|
204
172
|
urlPath: "/v1/stream",
|
|
205
|
-
name: "lucy-
|
|
206
|
-
fps:
|
|
207
|
-
width:
|
|
208
|
-
height:
|
|
173
|
+
name: "lucy-vton-2",
|
|
174
|
+
fps: 20,
|
|
175
|
+
width: 1088,
|
|
176
|
+
height: 624,
|
|
209
177
|
inputSchema: z.object({})
|
|
210
178
|
},
|
|
211
179
|
"lucy-restyle-2": {
|
|
@@ -216,14 +184,6 @@ const _models = {
|
|
|
216
184
|
height: 704,
|
|
217
185
|
inputSchema: z.object({})
|
|
218
186
|
},
|
|
219
|
-
"live-avatar": {
|
|
220
|
-
urlPath: "/v1/stream",
|
|
221
|
-
name: "live-avatar",
|
|
222
|
-
fps: 25,
|
|
223
|
-
width: 1280,
|
|
224
|
-
height: 720,
|
|
225
|
-
inputSchema: z.object({})
|
|
226
|
-
},
|
|
227
187
|
"lucy-latest": {
|
|
228
188
|
urlPath: "/v1/stream",
|
|
229
189
|
name: "lucy-latest",
|
|
@@ -248,14 +208,6 @@ const _models = {
|
|
|
248
208
|
height: 704,
|
|
249
209
|
inputSchema: z.object({})
|
|
250
210
|
},
|
|
251
|
-
mirage: {
|
|
252
|
-
urlPath: "/v1/stream",
|
|
253
|
-
name: "mirage",
|
|
254
|
-
fps: 25,
|
|
255
|
-
width: 1280,
|
|
256
|
-
height: 704,
|
|
257
|
-
inputSchema: z.object({})
|
|
258
|
-
},
|
|
259
211
|
mirage_v2: {
|
|
260
212
|
urlPath: "/v1/stream",
|
|
261
213
|
name: "mirage_v2",
|
|
@@ -264,20 +216,20 @@ const _models = {
|
|
|
264
216
|
height: 704,
|
|
265
217
|
inputSchema: z.object({})
|
|
266
218
|
},
|
|
267
|
-
|
|
219
|
+
"lucy-vton": {
|
|
268
220
|
urlPath: "/v1/stream",
|
|
269
|
-
name: "
|
|
270
|
-
fps:
|
|
271
|
-
width:
|
|
272
|
-
height:
|
|
221
|
+
name: "lucy-vton",
|
|
222
|
+
fps: 20,
|
|
223
|
+
width: 1088,
|
|
224
|
+
height: 624,
|
|
273
225
|
inputSchema: z.object({})
|
|
274
226
|
},
|
|
275
|
-
|
|
227
|
+
"lucy-2.1-vton-2": {
|
|
276
228
|
urlPath: "/v1/stream",
|
|
277
|
-
name: "
|
|
278
|
-
fps:
|
|
279
|
-
width:
|
|
280
|
-
height:
|
|
229
|
+
name: "lucy-2.1-vton-2",
|
|
230
|
+
fps: 20,
|
|
231
|
+
width: 1088,
|
|
232
|
+
height: 624,
|
|
281
233
|
inputSchema: z.object({})
|
|
282
234
|
}
|
|
283
235
|
},
|
|
@@ -338,6 +290,15 @@ const _models = {
|
|
|
338
290
|
height: 624,
|
|
339
291
|
inputSchema: modelInputSchemas["lucy-2.1-vton"]
|
|
340
292
|
},
|
|
293
|
+
"lucy-vton-2": {
|
|
294
|
+
urlPath: "/v1/generate/lucy-vton-2",
|
|
295
|
+
queueUrlPath: "/v1/jobs/lucy-vton-2",
|
|
296
|
+
name: "lucy-vton-2",
|
|
297
|
+
fps: 20,
|
|
298
|
+
width: 1088,
|
|
299
|
+
height: 624,
|
|
300
|
+
inputSchema: modelInputSchemas["lucy-vton-2"]
|
|
301
|
+
},
|
|
341
302
|
"lucy-restyle-2": {
|
|
342
303
|
urlPath: "/v1/generate/lucy-restyle-2",
|
|
343
304
|
queueUrlPath: "/v1/jobs/lucy-restyle-2",
|
|
@@ -347,15 +308,6 @@ const _models = {
|
|
|
347
308
|
height: 704,
|
|
348
309
|
inputSchema: modelInputSchemas["lucy-restyle-2"]
|
|
349
310
|
},
|
|
350
|
-
"lucy-motion": {
|
|
351
|
-
urlPath: "/v1/generate/lucy-motion",
|
|
352
|
-
queueUrlPath: "/v1/jobs/lucy-motion",
|
|
353
|
-
name: "lucy-motion",
|
|
354
|
-
fps: 25,
|
|
355
|
-
width: 1280,
|
|
356
|
-
height: 704,
|
|
357
|
-
inputSchema: modelInputSchemas["lucy-motion"]
|
|
358
|
-
},
|
|
359
311
|
"lucy-latest": {
|
|
360
312
|
urlPath: "/v1/generate/lucy-latest",
|
|
361
313
|
queueUrlPath: "/v1/jobs/lucy-latest",
|
|
@@ -392,14 +344,23 @@ const _models = {
|
|
|
392
344
|
height: 704,
|
|
393
345
|
inputSchema: modelInputSchemas["lucy-clip-latest"]
|
|
394
346
|
},
|
|
395
|
-
"lucy-
|
|
396
|
-
urlPath: "/v1/generate/lucy-
|
|
397
|
-
queueUrlPath: "/v1/jobs/lucy-
|
|
398
|
-
name: "lucy-
|
|
399
|
-
fps:
|
|
400
|
-
width:
|
|
401
|
-
height:
|
|
402
|
-
inputSchema: modelInputSchemas["lucy-
|
|
347
|
+
"lucy-vton": {
|
|
348
|
+
urlPath: "/v1/generate/lucy-vton",
|
|
349
|
+
queueUrlPath: "/v1/jobs/lucy-vton",
|
|
350
|
+
name: "lucy-vton",
|
|
351
|
+
fps: 20,
|
|
352
|
+
width: 1088,
|
|
353
|
+
height: 624,
|
|
354
|
+
inputSchema: modelInputSchemas["lucy-vton"]
|
|
355
|
+
},
|
|
356
|
+
"lucy-2.1-vton-2": {
|
|
357
|
+
urlPath: "/v1/generate/lucy-2.1-vton-2",
|
|
358
|
+
queueUrlPath: "/v1/jobs/lucy-2.1-vton-2",
|
|
359
|
+
name: "lucy-2.1-vton-2",
|
|
360
|
+
fps: 20,
|
|
361
|
+
width: 1088,
|
|
362
|
+
height: 624,
|
|
363
|
+
inputSchema: modelInputSchemas["lucy-2.1-vton-2"]
|
|
403
364
|
},
|
|
404
365
|
"lucy-pro-v2v": {
|
|
405
366
|
urlPath: "/v1/generate/lucy-pro-v2v",
|
package/package.json
CHANGED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
//#region src/realtime/audio-stream-manager.ts
|
|
2
|
-
/**
|
|
3
|
-
* Manages an audio stream for live_avatar mode.
|
|
4
|
-
* Creates a continuous audio stream that outputs silence by default,
|
|
5
|
-
* and allows playing audio files through the stream.
|
|
6
|
-
*/
|
|
7
|
-
var AudioStreamManager = class {
|
|
8
|
-
audioContext;
|
|
9
|
-
destination;
|
|
10
|
-
silenceOscillator;
|
|
11
|
-
silenceGain;
|
|
12
|
-
currentSource = null;
|
|
13
|
-
_isPlaying = false;
|
|
14
|
-
constructor() {
|
|
15
|
-
this.audioContext = new AudioContext();
|
|
16
|
-
this.destination = this.audioContext.createMediaStreamDestination();
|
|
17
|
-
this.silenceOscillator = this.audioContext.createOscillator();
|
|
18
|
-
this.silenceGain = this.audioContext.createGain();
|
|
19
|
-
this.silenceGain.gain.value = 0;
|
|
20
|
-
this.silenceOscillator.connect(this.silenceGain);
|
|
21
|
-
this.silenceGain.connect(this.destination);
|
|
22
|
-
this.silenceOscillator.start();
|
|
23
|
-
}
|
|
24
|
-
/**
|
|
25
|
-
* Get the MediaStream to pass to WebRTC.
|
|
26
|
-
* This stream outputs silence when no audio is playing.
|
|
27
|
-
*/
|
|
28
|
-
getStream() {
|
|
29
|
-
return this.destination.stream;
|
|
30
|
-
}
|
|
31
|
-
/**
|
|
32
|
-
* Check if audio is currently playing.
|
|
33
|
-
*/
|
|
34
|
-
isPlaying() {
|
|
35
|
-
return this._isPlaying;
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Play audio through the stream.
|
|
39
|
-
* When the audio ends, the stream automatically reverts to silence.
|
|
40
|
-
* @param audio - Audio data as Blob, File, or ArrayBuffer
|
|
41
|
-
* @returns Promise that resolves when audio finishes playing
|
|
42
|
-
*/
|
|
43
|
-
async playAudio(audio) {
|
|
44
|
-
if (this.audioContext.state === "suspended") await this.audioContext.resume();
|
|
45
|
-
if (this._isPlaying) this.stopAudio();
|
|
46
|
-
let arrayBuffer;
|
|
47
|
-
if (audio instanceof ArrayBuffer) arrayBuffer = audio;
|
|
48
|
-
else arrayBuffer = await audio.arrayBuffer();
|
|
49
|
-
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
|
50
|
-
const source = this.audioContext.createBufferSource();
|
|
51
|
-
source.buffer = audioBuffer;
|
|
52
|
-
source.connect(this.destination);
|
|
53
|
-
this.currentSource = source;
|
|
54
|
-
this._isPlaying = true;
|
|
55
|
-
return new Promise((resolve) => {
|
|
56
|
-
source.onended = () => {
|
|
57
|
-
this._isPlaying = false;
|
|
58
|
-
this.currentSource = null;
|
|
59
|
-
resolve();
|
|
60
|
-
};
|
|
61
|
-
source.start();
|
|
62
|
-
});
|
|
63
|
-
}
|
|
64
|
-
/**
|
|
65
|
-
* Stop currently playing audio immediately.
|
|
66
|
-
* The stream will revert to silence.
|
|
67
|
-
*/
|
|
68
|
-
stopAudio() {
|
|
69
|
-
if (this.currentSource) {
|
|
70
|
-
try {
|
|
71
|
-
this.currentSource.stop();
|
|
72
|
-
} catch {}
|
|
73
|
-
this.currentSource = null;
|
|
74
|
-
}
|
|
75
|
-
this._isPlaying = false;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Clean up resources.
|
|
79
|
-
*/
|
|
80
|
-
cleanup() {
|
|
81
|
-
this.stopAudio();
|
|
82
|
-
try {
|
|
83
|
-
this.silenceOscillator.stop();
|
|
84
|
-
} catch {}
|
|
85
|
-
this.audioContext.close().catch(() => {});
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
|
|
89
|
-
//#endregion
|
|
90
|
-
export { AudioStreamManager };
|