@remotion/video-matting 4.0.527 → 4.0.528
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/create-video-layer-output.d.ts +1 -0
- package/dist/create-video-matting-frames.d.ts +29 -0
- package/dist/download-video-matting-model.d.ts +16 -0
- package/dist/esm/index.mjs +875 -220
- package/dist/import-node-module.d.ts +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/load-video-matting-model.d.ts +9 -7
- package/dist/node-video-matting-frames.d.ts +2 -0
- package/dist/output-target.d.ts +1 -1
- package/dist/prepare-audio.d.ts +1 -1
- package/dist/probe-node-webgpu.d.ts +3 -0
- package/dist/separate-video-layers.d.ts +14 -1
- package/dist/symbol-async-dispose.d.ts +0 -0
- package/dist/with-remotion-model-host.d.ts +2 -1
- package/package.json +4 -3
package/dist/esm/index.mjs
CHANGED
|
@@ -14,6 +14,16 @@ var __toESM = (mod, isNodeMode, target) => {
|
|
|
14
14
|
});
|
|
15
15
|
return to;
|
|
16
16
|
};
|
|
17
|
+
var __export = (target, all) => {
|
|
18
|
+
for (var name in all)
|
|
19
|
+
__defProp(target, name, {
|
|
20
|
+
get: all[name],
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
set: (newValue) => all[name] = () => newValue
|
|
24
|
+
});
|
|
25
|
+
};
|
|
26
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
17
27
|
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
18
28
|
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
19
29
|
}) : x)(function(x) {
|
|
@@ -22,6 +32,363 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
|
|
|
22
32
|
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
23
33
|
});
|
|
24
34
|
|
|
35
|
+
// src/import-node-module.ts
|
|
36
|
+
var importNodeModule = (specifier) => {
|
|
37
|
+
const dynamicImport = new Function("specifier", "return import(specifier)");
|
|
38
|
+
return dynamicImport(specifier);
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
// src/probe-node-webgpu.ts
|
|
42
|
+
var exports_probe_node_webgpu = {};
|
|
43
|
+
__export(exports_probe_node_webgpu, {
|
|
44
|
+
probeNodeWebGpu: () => probeNodeWebGpu
|
|
45
|
+
});
|
|
46
|
+
var webGpuProbeModel, webGpuFp16ProbeModel, probeNodeWebGpu = async ({
|
|
47
|
+
requiresShaderF16
|
|
48
|
+
}) => {
|
|
49
|
+
const { InferenceSession, Tensor } = await importNodeModule("onnxruntime-node");
|
|
50
|
+
const session = await InferenceSession.create(requiresShaderF16 ? webGpuFp16ProbeModel : webGpuProbeModel, {
|
|
51
|
+
executionProviders: ["webgpu"],
|
|
52
|
+
extra: { session: { disable_cpu_ep_fallback: "1" } }
|
|
53
|
+
});
|
|
54
|
+
try {
|
|
55
|
+
await session.run({ X: new Tensor("float32", new Float32Array([1]), [1]) });
|
|
56
|
+
} finally {
|
|
57
|
+
await session.release();
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
var init_probe_node_webgpu = __esm(() => {
|
|
61
|
+
webGpuProbeModel = new Uint8Array([
|
|
62
|
+
8,
|
|
63
|
+
8,
|
|
64
|
+
58,
|
|
65
|
+
64,
|
|
66
|
+
10,
|
|
67
|
+
14,
|
|
68
|
+
10,
|
|
69
|
+
1,
|
|
70
|
+
88,
|
|
71
|
+
10,
|
|
72
|
+
1,
|
|
73
|
+
88,
|
|
74
|
+
18,
|
|
75
|
+
1,
|
|
76
|
+
89,
|
|
77
|
+
34,
|
|
78
|
+
3,
|
|
79
|
+
77,
|
|
80
|
+
117,
|
|
81
|
+
108,
|
|
82
|
+
18,
|
|
83
|
+
12,
|
|
84
|
+
119,
|
|
85
|
+
101,
|
|
86
|
+
98,
|
|
87
|
+
103,
|
|
88
|
+
112,
|
|
89
|
+
117,
|
|
90
|
+
45,
|
|
91
|
+
112,
|
|
92
|
+
114,
|
|
93
|
+
111,
|
|
94
|
+
98,
|
|
95
|
+
101,
|
|
96
|
+
90,
|
|
97
|
+
15,
|
|
98
|
+
10,
|
|
99
|
+
1,
|
|
100
|
+
88,
|
|
101
|
+
18,
|
|
102
|
+
10,
|
|
103
|
+
10,
|
|
104
|
+
8,
|
|
105
|
+
8,
|
|
106
|
+
1,
|
|
107
|
+
18,
|
|
108
|
+
4,
|
|
109
|
+
10,
|
|
110
|
+
2,
|
|
111
|
+
8,
|
|
112
|
+
1,
|
|
113
|
+
98,
|
|
114
|
+
15,
|
|
115
|
+
10,
|
|
116
|
+
1,
|
|
117
|
+
89,
|
|
118
|
+
18,
|
|
119
|
+
10,
|
|
120
|
+
10,
|
|
121
|
+
8,
|
|
122
|
+
8,
|
|
123
|
+
1,
|
|
124
|
+
18,
|
|
125
|
+
4,
|
|
126
|
+
10,
|
|
127
|
+
2,
|
|
128
|
+
8,
|
|
129
|
+
1,
|
|
130
|
+
66,
|
|
131
|
+
2,
|
|
132
|
+
16,
|
|
133
|
+
13
|
|
134
|
+
]);
|
|
135
|
+
webGpuFp16ProbeModel = new Uint8Array([
|
|
136
|
+
8,
|
|
137
|
+
8,
|
|
138
|
+
58,
|
|
139
|
+
114,
|
|
140
|
+
10,
|
|
141
|
+
23,
|
|
142
|
+
10,
|
|
143
|
+
1,
|
|
144
|
+
88,
|
|
145
|
+
18,
|
|
146
|
+
1,
|
|
147
|
+
72,
|
|
148
|
+
34,
|
|
149
|
+
4,
|
|
150
|
+
67,
|
|
151
|
+
97,
|
|
152
|
+
115,
|
|
153
|
+
116,
|
|
154
|
+
42,
|
|
155
|
+
9,
|
|
156
|
+
10,
|
|
157
|
+
2,
|
|
158
|
+
116,
|
|
159
|
+
111,
|
|
160
|
+
24,
|
|
161
|
+
10,
|
|
162
|
+
160,
|
|
163
|
+
1,
|
|
164
|
+
2,
|
|
165
|
+
10,
|
|
166
|
+
14,
|
|
167
|
+
10,
|
|
168
|
+
1,
|
|
169
|
+
72,
|
|
170
|
+
10,
|
|
171
|
+
1,
|
|
172
|
+
72,
|
|
173
|
+
18,
|
|
174
|
+
1,
|
|
175
|
+
83,
|
|
176
|
+
34,
|
|
177
|
+
3,
|
|
178
|
+
77,
|
|
179
|
+
117,
|
|
180
|
+
108,
|
|
181
|
+
10,
|
|
182
|
+
23,
|
|
183
|
+
10,
|
|
184
|
+
1,
|
|
185
|
+
83,
|
|
186
|
+
18,
|
|
187
|
+
1,
|
|
188
|
+
89,
|
|
189
|
+
34,
|
|
190
|
+
4,
|
|
191
|
+
67,
|
|
192
|
+
97,
|
|
193
|
+
115,
|
|
194
|
+
116,
|
|
195
|
+
42,
|
|
196
|
+
9,
|
|
197
|
+
10,
|
|
198
|
+
2,
|
|
199
|
+
116,
|
|
200
|
+
111,
|
|
201
|
+
24,
|
|
202
|
+
1,
|
|
203
|
+
160,
|
|
204
|
+
1,
|
|
205
|
+
2,
|
|
206
|
+
18,
|
|
207
|
+
12,
|
|
208
|
+
119,
|
|
209
|
+
101,
|
|
210
|
+
98,
|
|
211
|
+
103,
|
|
212
|
+
112,
|
|
213
|
+
117,
|
|
214
|
+
45,
|
|
215
|
+
112,
|
|
216
|
+
114,
|
|
217
|
+
111,
|
|
218
|
+
98,
|
|
219
|
+
101,
|
|
220
|
+
90,
|
|
221
|
+
15,
|
|
222
|
+
10,
|
|
223
|
+
1,
|
|
224
|
+
88,
|
|
225
|
+
18,
|
|
226
|
+
10,
|
|
227
|
+
10,
|
|
228
|
+
8,
|
|
229
|
+
8,
|
|
230
|
+
1,
|
|
231
|
+
18,
|
|
232
|
+
4,
|
|
233
|
+
10,
|
|
234
|
+
2,
|
|
235
|
+
8,
|
|
236
|
+
1,
|
|
237
|
+
98,
|
|
238
|
+
15,
|
|
239
|
+
10,
|
|
240
|
+
1,
|
|
241
|
+
89,
|
|
242
|
+
18,
|
|
243
|
+
10,
|
|
244
|
+
10,
|
|
245
|
+
8,
|
|
246
|
+
8,
|
|
247
|
+
1,
|
|
248
|
+
18,
|
|
249
|
+
4,
|
|
250
|
+
10,
|
|
251
|
+
2,
|
|
252
|
+
8,
|
|
253
|
+
1,
|
|
254
|
+
66,
|
|
255
|
+
2,
|
|
256
|
+
16,
|
|
257
|
+
13
|
|
258
|
+
]);
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
// src/node-video-matting-frames.ts
|
|
262
|
+
var exports_node_video_matting_frames = {};
|
|
263
|
+
__export(exports_node_video_matting_frames, {
|
|
264
|
+
createNodeVideoMattingFrames: () => createNodeVideoMattingFrames
|
|
265
|
+
});
|
|
266
|
+
import { VideoSample, VideoSampleSink, VideoSampleSource } from "mediabunny";
|
|
267
|
+
var createNodeVideoMattingFrames = ({
|
|
268
|
+
videoTrack,
|
|
269
|
+
width,
|
|
270
|
+
height,
|
|
271
|
+
videoQuality,
|
|
272
|
+
keyframeIntervalInSeconds,
|
|
273
|
+
videoStartTimestamp,
|
|
274
|
+
videoEndTimestamp,
|
|
275
|
+
includeBase
|
|
276
|
+
}) => {
|
|
277
|
+
const sink = new VideoSampleSink(videoTrack);
|
|
278
|
+
const baseSource = includeBase ? new VideoSampleSource({
|
|
279
|
+
codec: "vp9",
|
|
280
|
+
quality: videoQuality,
|
|
281
|
+
keyFrameInterval: keyframeIntervalInSeconds,
|
|
282
|
+
alpha: "discard"
|
|
283
|
+
}) : null;
|
|
284
|
+
const foregroundSource = new VideoSampleSource({
|
|
285
|
+
codec: "vp9",
|
|
286
|
+
quality: videoQuality,
|
|
287
|
+
keyFrameInterval: keyframeIntervalInSeconds,
|
|
288
|
+
alpha: "keep"
|
|
289
|
+
});
|
|
290
|
+
return {
|
|
291
|
+
frames: async function* () {
|
|
292
|
+
for await (const sample of sink.samples(videoStartTimestamp, videoEndTimestamp)) {
|
|
293
|
+
const data = new Uint8ClampedArray(width * height * 4);
|
|
294
|
+
try {
|
|
295
|
+
const transformed = await sample.transform({
|
|
296
|
+
width,
|
|
297
|
+
height,
|
|
298
|
+
fit: "fill"
|
|
299
|
+
});
|
|
300
|
+
try {
|
|
301
|
+
await transformed.copyTo(data, { format: "RGBA" });
|
|
302
|
+
} finally {
|
|
303
|
+
transformed.close();
|
|
304
|
+
}
|
|
305
|
+
} finally {
|
|
306
|
+
sample.close();
|
|
307
|
+
}
|
|
308
|
+
yield {
|
|
309
|
+
image: { data, width, height, channels: 4 },
|
|
310
|
+
timestamp: sample.timestamp,
|
|
311
|
+
duration: sample.duration
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
}(),
|
|
315
|
+
baseSource,
|
|
316
|
+
foregroundSource,
|
|
317
|
+
addFrame: async ({ frame, foreground, timestamp, duration }) => {
|
|
318
|
+
if (!("data" in frame.image)) {
|
|
319
|
+
throw new Error("Expected an RGBA frame.");
|
|
320
|
+
}
|
|
321
|
+
const baseData = includeBase ? frame.image.data.slice() : null;
|
|
322
|
+
let foregroundData;
|
|
323
|
+
if (foreground.width === width && foreground.height === height) {
|
|
324
|
+
foregroundData = foreground.data.slice();
|
|
325
|
+
} else {
|
|
326
|
+
foregroundData = new Uint8ClampedArray(width * height * 4);
|
|
327
|
+
const foregroundSample = new VideoSample(foreground.data, {
|
|
328
|
+
format: "RGBA",
|
|
329
|
+
codedWidth: foreground.width,
|
|
330
|
+
codedHeight: foreground.height,
|
|
331
|
+
timestamp,
|
|
332
|
+
duration
|
|
333
|
+
});
|
|
334
|
+
try {
|
|
335
|
+
const resized = await foregroundSample.transform({
|
|
336
|
+
width,
|
|
337
|
+
height,
|
|
338
|
+
fit: "fill"
|
|
339
|
+
});
|
|
340
|
+
try {
|
|
341
|
+
await resized.copyTo(foregroundData, { format: "RGBA" });
|
|
342
|
+
} finally {
|
|
343
|
+
resized.close();
|
|
344
|
+
}
|
|
345
|
+
} finally {
|
|
346
|
+
foregroundSample.close();
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
for (let i = 0;i < foregroundData.length; i += 4) {
|
|
350
|
+
const alpha = frame.image.data[i + 3] / 255;
|
|
351
|
+
if (baseData) {
|
|
352
|
+
baseData[i] = Math.round(baseData[i] * alpha);
|
|
353
|
+
baseData[i + 1] = Math.round(baseData[i + 1] * alpha);
|
|
354
|
+
baseData[i + 2] = Math.round(baseData[i + 2] * alpha);
|
|
355
|
+
baseData[i + 3] = 255;
|
|
356
|
+
}
|
|
357
|
+
foregroundData[i + 3] = Math.round(foregroundData[i + 3] * alpha);
|
|
358
|
+
}
|
|
359
|
+
const base = baseData ? new VideoSample(baseData, {
|
|
360
|
+
format: "RGBA",
|
|
361
|
+
codedWidth: width,
|
|
362
|
+
codedHeight: height,
|
|
363
|
+
timestamp,
|
|
364
|
+
duration
|
|
365
|
+
}) : null;
|
|
366
|
+
const foregroundFrame = new VideoSample(foregroundData, {
|
|
367
|
+
format: "RGBA",
|
|
368
|
+
codedWidth: width,
|
|
369
|
+
codedHeight: height,
|
|
370
|
+
timestamp,
|
|
371
|
+
duration
|
|
372
|
+
});
|
|
373
|
+
try {
|
|
374
|
+
const results = await Promise.allSettled([
|
|
375
|
+
baseSource?.add(base),
|
|
376
|
+
foregroundSource.add(foregroundFrame)
|
|
377
|
+
]);
|
|
378
|
+
for (const result of results) {
|
|
379
|
+
if (result.status === "rejected") {
|
|
380
|
+
throw result.reason;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
} finally {
|
|
384
|
+
base?.close();
|
|
385
|
+
foregroundFrame.close();
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
};
|
|
389
|
+
};
|
|
390
|
+
var init_node_video_matting_frames = () => {};
|
|
391
|
+
|
|
25
392
|
// src/models.ts
|
|
26
393
|
var VIDEO_MATTING_MODELS = ["modnet", "ben2-base"];
|
|
27
394
|
var MODEL_INFO = {
|
|
@@ -81,11 +448,25 @@ var canUseVideoMatting = async ({
|
|
|
81
448
|
model = "modnet"
|
|
82
449
|
} = {}) => {
|
|
83
450
|
const modelInfo = getVideoMattingModelInfo(model);
|
|
451
|
+
if (typeof window === "undefined" && typeof process !== "undefined" && process.release?.name === "node") {
|
|
452
|
+
try {
|
|
453
|
+
const { probeNodeWebGpu: probeNodeWebGpu2 } = await Promise.resolve().then(() => (init_probe_node_webgpu(), exports_probe_node_webgpu));
|
|
454
|
+
await probeNodeWebGpu2({ requiresShaderF16: modelInfo.requiresShaderF16 });
|
|
455
|
+
return { supported: true };
|
|
456
|
+
} catch (error) {
|
|
457
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
458
|
+
return {
|
|
459
|
+
supported: false,
|
|
460
|
+
reason: "webgpu-unavailable" /* WebGpuUnavailable */,
|
|
461
|
+
detailedReason: `ONNX Runtime could not run WebGPU inference for "${model}": ${message}`
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
}
|
|
84
465
|
if (typeof window === "undefined" && typeof OffscreenCanvas === "undefined") {
|
|
85
466
|
return {
|
|
86
467
|
supported: false,
|
|
87
468
|
reason: "window-undefined" /* WindowUndefined */,
|
|
88
|
-
detailedReason: "No browser
|
|
469
|
+
detailedReason: "No browser, worker canvas, or Node.js environment is available."
|
|
89
470
|
};
|
|
90
471
|
}
|
|
91
472
|
const isSecureContext = typeof window === "undefined" ? globalThis.isSecureContext : window.isSecureContext;
|
|
@@ -126,6 +507,13 @@ var canUseVideoMatting = async ({
|
|
|
126
507
|
}
|
|
127
508
|
return { supported: true };
|
|
128
509
|
};
|
|
510
|
+
// src/symbol-async-dispose.ts
|
|
511
|
+
if (typeof Symbol.asyncDispose !== "symbol") {
|
|
512
|
+
Object.defineProperty(Symbol, "asyncDispose", {
|
|
513
|
+
value: Symbol.for("asyncDispose")
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
|
|
129
517
|
// src/with-remotion-model-host.ts
|
|
130
518
|
var REMOTION_MODEL_HOST = "https://remotion.media/";
|
|
131
519
|
var REMOTION_MODEL_PATH_TEMPLATE = "models/{model}/";
|
|
@@ -237,7 +625,8 @@ var getOrCreateVideoMattingPipeline = ({
|
|
|
237
625
|
activeUses: 0,
|
|
238
626
|
onIdle: [],
|
|
239
627
|
progressListeners: new Set,
|
|
240
|
-
lastProgress: null
|
|
628
|
+
lastProgress: null,
|
|
629
|
+
retainedLoadHandles: 0
|
|
241
630
|
};
|
|
242
631
|
const pendingDisposal = disposals.get(model) ?? null;
|
|
243
632
|
state.loading = Promise.resolve().then(async () => {
|
|
@@ -247,39 +636,24 @@ var getOrCreateVideoMattingPipeline = ({
|
|
|
247
636
|
return withRemotionModelHost(async ({
|
|
248
637
|
AutoModelForImageSegmentation,
|
|
249
638
|
AutoProcessor,
|
|
250
|
-
BackgroundRemovalPipeline
|
|
639
|
+
BackgroundRemovalPipeline,
|
|
640
|
+
ModelRegistry,
|
|
641
|
+
RawImage
|
|
251
642
|
}) => {
|
|
252
643
|
const hostedModelId = getHostedVideoMattingModelId(model);
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
let lastLoadedBytes = 0;
|
|
644
|
+
if (!await ModelRegistry.is_pipeline_cached("background-removal", hostedModelId, { device: "webgpu", dtype: modelInfo.dtype })) {
|
|
645
|
+
throw new Error(`The video matting model "${model}" is not downloaded. Call downloadVideoMattingModel() first.`);
|
|
646
|
+
}
|
|
257
647
|
notifyProgress(state, {
|
|
258
|
-
status: "
|
|
648
|
+
status: "initializing",
|
|
259
649
|
file: null,
|
|
260
|
-
progress:
|
|
261
|
-
loadedBytes:
|
|
262
|
-
totalBytes
|
|
650
|
+
progress: null,
|
|
651
|
+
loadedBytes: null,
|
|
652
|
+
totalBytes: null
|
|
263
653
|
});
|
|
264
654
|
const pretrainedOptions = {
|
|
265
655
|
device: "webgpu",
|
|
266
|
-
dtype: modelInfo.dtype
|
|
267
|
-
progress_callback: (event) => {
|
|
268
|
-
const record = event;
|
|
269
|
-
if (record.status === "progress" && typeof record.file === "string" && typeof record.loaded === "number" && Number.isFinite(record.loaded)) {
|
|
270
|
-
loadedByFile.set(record.file, Math.max(loadedByFile.get(record.file) ?? 0, record.loaded));
|
|
271
|
-
const loadedBytes = [...loadedByFile.values()].reduce((sum, loaded) => sum + loaded, 0);
|
|
272
|
-
lastProgress = Math.max(lastProgress, Math.min(loadedBytes / totalBytes, 0.99));
|
|
273
|
-
lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
|
|
274
|
-
notifyProgress(state, {
|
|
275
|
-
status: "loading",
|
|
276
|
-
file: null,
|
|
277
|
-
progress: lastProgress,
|
|
278
|
-
loadedBytes: Math.min(lastLoadedBytes, totalBytes),
|
|
279
|
-
totalBytes
|
|
280
|
-
});
|
|
281
|
-
}
|
|
282
|
-
}
|
|
656
|
+
dtype: modelInfo.dtype
|
|
283
657
|
};
|
|
284
658
|
const transformerProcessor = await AutoProcessor.from_pretrained(hostedModelId, pretrainedOptions);
|
|
285
659
|
const transformerModel = await AutoModelForImageSegmentation.from_pretrained(hostedModelId, pretrainedOptions);
|
|
@@ -296,12 +670,12 @@ var getOrCreateVideoMattingPipeline = ({
|
|
|
296
670
|
status: "ready",
|
|
297
671
|
file: null,
|
|
298
672
|
progress: 1,
|
|
299
|
-
loadedBytes:
|
|
300
|
-
totalBytes
|
|
673
|
+
loadedBytes: null,
|
|
674
|
+
totalBytes: null
|
|
301
675
|
});
|
|
302
676
|
const loadedPipeline = {
|
|
303
677
|
run: async (image) => {
|
|
304
|
-
const result = await loadedTransformerPipeline(image);
|
|
678
|
+
const result = await loadedTransformerPipeline("data" in image ? new RawImage(image.data, image.width, image.height, image.channels) : image);
|
|
305
679
|
if (result.channels !== 4) {
|
|
306
680
|
throw new Error(`The video matting model "${model}" returned ${result.channels} channels instead of RGBA.`);
|
|
307
681
|
}
|
|
@@ -355,18 +729,94 @@ var subscribeToProgress = (state, onProgress) => {
|
|
|
355
729
|
state.progressListeners.delete(listener);
|
|
356
730
|
};
|
|
357
731
|
};
|
|
732
|
+
var disposeVideoMattingModel = async ({
|
|
733
|
+
model
|
|
734
|
+
} = {}) => {
|
|
735
|
+
if (model !== undefined) {
|
|
736
|
+
getVideoMattingModelInfo(model);
|
|
737
|
+
}
|
|
738
|
+
const matching = [...pipelines.entries()].filter(([loadedModel]) => {
|
|
739
|
+
return model === undefined || loadedModel === model;
|
|
740
|
+
});
|
|
741
|
+
for (const [key, state] of matching) {
|
|
742
|
+
if (pipelines.get(key) === state) {
|
|
743
|
+
pipelines.delete(key);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
const pending = new Map;
|
|
747
|
+
for (const [key, disposal] of disposals) {
|
|
748
|
+
if (model === undefined || model === key) {
|
|
749
|
+
pending.set(key, disposal);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
for (const [key, state] of matching) {
|
|
753
|
+
const previousDisposal = disposals.get(key) ?? Promise.resolve();
|
|
754
|
+
const disposal = previousDisposal.then(async () => {
|
|
755
|
+
const loadedPipeline = await state.loading;
|
|
756
|
+
if (state.pendingUses > 0 || state.activeUses > 0) {
|
|
757
|
+
await new Promise((resolve) => {
|
|
758
|
+
state.onIdle.push(resolve);
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
await loadedPipeline.dispose();
|
|
762
|
+
});
|
|
763
|
+
const disposalBarrier = disposal.then(() => {
|
|
764
|
+
return;
|
|
765
|
+
}, () => {
|
|
766
|
+
return;
|
|
767
|
+
});
|
|
768
|
+
disposals.set(key, disposalBarrier);
|
|
769
|
+
disposalBarrier.then(() => {
|
|
770
|
+
if (disposals.get(key) === disposalBarrier) {
|
|
771
|
+
disposals.delete(key);
|
|
772
|
+
}
|
|
773
|
+
});
|
|
774
|
+
pending.set(key, disposal);
|
|
775
|
+
}
|
|
776
|
+
await Promise.all(pending.values());
|
|
777
|
+
};
|
|
358
778
|
var loadVideoMattingModel = async ({
|
|
359
779
|
model,
|
|
780
|
+
signal,
|
|
360
781
|
onProgress
|
|
361
782
|
}) => {
|
|
783
|
+
signal?.throwIfAborted();
|
|
362
784
|
const { state, alreadyLoaded } = getOrCreateVideoMattingPipeline({ model });
|
|
785
|
+
state.pendingUses++;
|
|
363
786
|
const unsubscribe = subscribeToProgress(state, onProgress);
|
|
364
787
|
try {
|
|
365
|
-
await
|
|
788
|
+
await waitForLoadingOrAbort({
|
|
789
|
+
loading: state.loading,
|
|
790
|
+
signal: signal ?? null
|
|
791
|
+
});
|
|
792
|
+
signal?.throwIfAborted();
|
|
793
|
+
state.retainedLoadHandles++;
|
|
366
794
|
} finally {
|
|
367
795
|
unsubscribe();
|
|
796
|
+
state.pendingUses--;
|
|
797
|
+
notifyIfIdle(state);
|
|
798
|
+
if (signal?.aborted && state.pendingUses === 0 && state.activeUses === 0 && state.retainedLoadHandles === 0 && pipelines.get(model) === state) {
|
|
799
|
+
disposeVideoMattingModel({ model }).catch(() => {
|
|
800
|
+
return;
|
|
801
|
+
});
|
|
802
|
+
}
|
|
368
803
|
}
|
|
369
|
-
|
|
804
|
+
let released = false;
|
|
805
|
+
const result = { alreadyLoaded };
|
|
806
|
+
Object.defineProperty(result, Symbol.asyncDispose, {
|
|
807
|
+
enumerable: false,
|
|
808
|
+
value: async () => {
|
|
809
|
+
if (released) {
|
|
810
|
+
return;
|
|
811
|
+
}
|
|
812
|
+
released = true;
|
|
813
|
+
state.retainedLoadHandles--;
|
|
814
|
+
if (state.retainedLoadHandles === 0 && pipelines.get(model) === state) {
|
|
815
|
+
await disposeVideoMattingModel({ model });
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
});
|
|
819
|
+
return result;
|
|
370
820
|
};
|
|
371
821
|
var withLoadedVideoMattingPipeline = async ({
|
|
372
822
|
model,
|
|
@@ -397,51 +847,152 @@ var withLoadedVideoMattingPipeline = async ({
|
|
|
397
847
|
notifyIfIdle(state);
|
|
398
848
|
}
|
|
399
849
|
};
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
850
|
+
// src/download-video-matting-model.ts
|
|
851
|
+
var downloadVideoMattingModel = ({
|
|
852
|
+
model,
|
|
853
|
+
signal,
|
|
854
|
+
onProgress
|
|
855
|
+
}) => {
|
|
856
|
+
const modelInfo = getVideoMattingModelInfo(model);
|
|
857
|
+
const totalBytes = modelInfo.webGpuDownloadSize;
|
|
858
|
+
const hostedModelId = getHostedVideoMattingModelId(model);
|
|
859
|
+
if (signal?.aborted) {
|
|
860
|
+
return Promise.reject(signal.reason);
|
|
405
861
|
}
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
if (
|
|
411
|
-
|
|
862
|
+
return withRemotionModelHost(async ({ env, ModelRegistry }) => {
|
|
863
|
+
signal?.throwIfAborted();
|
|
864
|
+
const cache = env.useCustomCache ? env.customCache : env.useBrowserCache && typeof caches !== "undefined" ? await caches.open(env.cacheKey) : null;
|
|
865
|
+
const useFileCache = cache === null && env.useFSCache;
|
|
866
|
+
if (cache === null && !useFileCache) {
|
|
867
|
+
throw new Error("A Transformers.js model cache is required to download a video matting model.");
|
|
412
868
|
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
869
|
+
const registryOptions = {
|
|
870
|
+
device: "webgpu",
|
|
871
|
+
dtype: modelInfo.dtype
|
|
872
|
+
};
|
|
873
|
+
const cacheStatus = await ModelRegistry.is_pipeline_cached_files("background-removal", hostedModelId, registryOptions);
|
|
874
|
+
signal?.throwIfAborted();
|
|
875
|
+
if (cacheStatus.allCached) {
|
|
876
|
+
onProgress?.({
|
|
877
|
+
file: null,
|
|
878
|
+
progress: 1,
|
|
879
|
+
loadedBytes: totalBytes,
|
|
880
|
+
totalBytes
|
|
881
|
+
});
|
|
882
|
+
signal?.throwIfAborted();
|
|
883
|
+
return { alreadyDownloaded: true };
|
|
418
884
|
}
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
885
|
+
let loadedBytes = 0;
|
|
886
|
+
const emitProgress = (file) => {
|
|
887
|
+
onProgress?.({
|
|
888
|
+
file,
|
|
889
|
+
progress: Math.min(loadedBytes / totalBytes, 0.99),
|
|
890
|
+
loadedBytes,
|
|
891
|
+
totalBytes
|
|
892
|
+
});
|
|
893
|
+
};
|
|
894
|
+
emitProgress(null);
|
|
895
|
+
for (const { file, cached } of cacheStatus.files) {
|
|
896
|
+
signal?.throwIfAborted();
|
|
897
|
+
if (cached) {
|
|
898
|
+
continue;
|
|
428
899
|
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
return;
|
|
435
|
-
});
|
|
436
|
-
disposals.set(key, disposalBarrier);
|
|
437
|
-
disposalBarrier.then(() => {
|
|
438
|
-
if (disposals.get(key) === disposalBarrier) {
|
|
439
|
-
disposals.delete(key);
|
|
900
|
+
const url = `${env.remoteHost}${env.remotePathTemplate.replace("{model}", hostedModelId)}${file}`;
|
|
901
|
+
const cacheKey = cache === null ? `${hostedModelId}/${file}` : url;
|
|
902
|
+
const response = await env.fetch(url, { signal });
|
|
903
|
+
if (!response.ok || response.body === null) {
|
|
904
|
+
throw new Error(`Could not download ${url}: HTTP ${response.status}.`);
|
|
440
905
|
}
|
|
906
|
+
const reader = response.body.getReader();
|
|
907
|
+
const onAbort = () => {
|
|
908
|
+
reader.cancel(signal?.reason).catch(() => {
|
|
909
|
+
return;
|
|
910
|
+
});
|
|
911
|
+
};
|
|
912
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
913
|
+
try {
|
|
914
|
+
signal?.throwIfAborted();
|
|
915
|
+
if (cache !== null) {
|
|
916
|
+
const stream = new ReadableStream({
|
|
917
|
+
async pull(controller) {
|
|
918
|
+
try {
|
|
919
|
+
const { done, value } = await reader.read();
|
|
920
|
+
signal?.throwIfAborted();
|
|
921
|
+
if (done) {
|
|
922
|
+
controller.close();
|
|
923
|
+
return;
|
|
924
|
+
}
|
|
925
|
+
loadedBytes += value.byteLength;
|
|
926
|
+
emitProgress(file);
|
|
927
|
+
signal?.throwIfAborted();
|
|
928
|
+
controller.enqueue(value);
|
|
929
|
+
} catch (error) {
|
|
930
|
+
controller.error(error);
|
|
931
|
+
}
|
|
932
|
+
},
|
|
933
|
+
cancel: () => reader.cancel()
|
|
934
|
+
});
|
|
935
|
+
await cache.put(cacheKey, new Response(stream, { headers: response.headers }));
|
|
936
|
+
} else {
|
|
937
|
+
const [{ mkdir, open, rename, unlink }, { join, dirname }, { randomUUID }] = await Promise.all([
|
|
938
|
+
importNodeModule("node:fs/promises"),
|
|
939
|
+
importNodeModule("node:path"),
|
|
940
|
+
importNodeModule("node:crypto")
|
|
941
|
+
]);
|
|
942
|
+
if (env.cacheDir === null) {
|
|
943
|
+
throw new Error("Transformers.js env.cacheDir must be set.");
|
|
944
|
+
}
|
|
945
|
+
const destination = join(env.cacheDir, cacheKey);
|
|
946
|
+
const temporary = `${destination}.tmp.${randomUUID()}`;
|
|
947
|
+
await mkdir(dirname(destination), { recursive: true });
|
|
948
|
+
const handle = await open(temporary, "wx");
|
|
949
|
+
try {
|
|
950
|
+
while (true) {
|
|
951
|
+
const { done, value } = await reader.read();
|
|
952
|
+
signal?.throwIfAborted();
|
|
953
|
+
if (done) {
|
|
954
|
+
break;
|
|
955
|
+
}
|
|
956
|
+
await handle.writeFile(value);
|
|
957
|
+
loadedBytes += value.byteLength;
|
|
958
|
+
emitProgress(file);
|
|
959
|
+
}
|
|
960
|
+
await handle.close();
|
|
961
|
+
signal?.throwIfAborted();
|
|
962
|
+
await rename(temporary, destination);
|
|
963
|
+
} catch (error) {
|
|
964
|
+
await Promise.allSettled([handle.close(), reader.cancel(error)]);
|
|
965
|
+
await unlink(temporary).catch(() => {
|
|
966
|
+
return;
|
|
967
|
+
});
|
|
968
|
+
throw error;
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
} finally {
|
|
972
|
+
signal?.removeEventListener("abort", onAbort);
|
|
973
|
+
await reader.cancel().catch(() => {
|
|
974
|
+
return;
|
|
975
|
+
});
|
|
976
|
+
reader.releaseLock();
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
signal?.throwIfAborted();
|
|
980
|
+
if (!await ModelRegistry.is_pipeline_cached("background-removal", hostedModelId, registryOptions)) {
|
|
981
|
+
throw new Error(`The video matting model "${model}" was not fully cached.`);
|
|
982
|
+
}
|
|
983
|
+
signal?.throwIfAborted();
|
|
984
|
+
onProgress?.({
|
|
985
|
+
file: null,
|
|
986
|
+
progress: 1,
|
|
987
|
+
loadedBytes: totalBytes,
|
|
988
|
+
totalBytes
|
|
441
989
|
});
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
990
|
+
signal?.throwIfAborted();
|
|
991
|
+
return { alreadyDownloaded: false };
|
|
992
|
+
}).catch((error) => {
|
|
993
|
+
signal?.throwIfAborted();
|
|
994
|
+
throw error;
|
|
995
|
+
});
|
|
445
996
|
};
|
|
446
997
|
// src/is-video-matting-model-cached.ts
|
|
447
998
|
var isVideoMattingModelCached = ({
|
|
@@ -472,8 +1023,7 @@ var removeVideoMattingModel = async ({
|
|
|
472
1023
|
import {
|
|
473
1024
|
ALL_FORMATS,
|
|
474
1025
|
BlobSource,
|
|
475
|
-
|
|
476
|
-
CanvasSource,
|
|
1026
|
+
FilePathSource,
|
|
477
1027
|
Input,
|
|
478
1028
|
UrlSource,
|
|
479
1029
|
WebMOutputFormat,
|
|
@@ -487,7 +1037,7 @@ import {
|
|
|
487
1037
|
StreamTarget
|
|
488
1038
|
} from "mediabunny";
|
|
489
1039
|
// package.json
|
|
490
|
-
var version = "4.0.
|
|
1040
|
+
var version = "4.0.528";
|
|
491
1041
|
|
|
492
1042
|
// src/web-fs-target.ts
|
|
493
1043
|
var filePrefix = "__remotion_video_matting:";
|
|
@@ -595,6 +1145,13 @@ var createWebFsVideoLayerTarget = async () => {
|
|
|
595
1145
|
};
|
|
596
1146
|
|
|
597
1147
|
// src/create-video-layer-output.ts
|
|
1148
|
+
var withAsyncDispose = (result) => {
|
|
1149
|
+
Object.defineProperty(result, Symbol.asyncDispose, {
|
|
1150
|
+
enumerable: false,
|
|
1151
|
+
value: result.dispose
|
|
1152
|
+
});
|
|
1153
|
+
return result;
|
|
1154
|
+
};
|
|
598
1155
|
var createVideoLayerOutput = async ({
|
|
599
1156
|
format,
|
|
600
1157
|
options
|
|
@@ -754,17 +1311,17 @@ var createVideoLayerOutput = async ({
|
|
|
754
1311
|
}
|
|
755
1312
|
finalized = true;
|
|
756
1313
|
if (outputMode === "writable") {
|
|
757
|
-
return {
|
|
1314
|
+
return withAsyncDispose({
|
|
758
1315
|
dispose: () => Promise.resolve(),
|
|
759
1316
|
getBlob: () => Promise.reject(new Error("getBlob() is unavailable when outputWritable is used"))
|
|
760
|
-
};
|
|
1317
|
+
});
|
|
761
1318
|
}
|
|
762
1319
|
if (outputMode === "web-fs") {
|
|
763
1320
|
if (webFsTarget === null) {
|
|
764
1321
|
throw new Error("Expected an OPFS-backed output target");
|
|
765
1322
|
}
|
|
766
1323
|
let blobPromise = null;
|
|
767
|
-
return {
|
|
1324
|
+
return withAsyncDispose({
|
|
768
1325
|
dispose: discard,
|
|
769
1326
|
getBlob: () => {
|
|
770
1327
|
blobPromise ??= (async () => {
|
|
@@ -774,12 +1331,12 @@ var createVideoLayerOutput = async ({
|
|
|
774
1331
|
})();
|
|
775
1332
|
return blobPromise;
|
|
776
1333
|
}
|
|
777
|
-
};
|
|
1334
|
+
});
|
|
778
1335
|
}
|
|
779
1336
|
if (!(target instanceof BufferTarget)) {
|
|
780
1337
|
throw new Error("Expected an in-memory output target");
|
|
781
1338
|
}
|
|
782
|
-
return {
|
|
1339
|
+
return withAsyncDispose({
|
|
783
1340
|
dispose: () => Promise.resolve(),
|
|
784
1341
|
getBlob: () => {
|
|
785
1342
|
if (target.buffer === null) {
|
|
@@ -787,7 +1344,7 @@ var createVideoLayerOutput = async ({
|
|
|
787
1344
|
}
|
|
788
1345
|
return Promise.resolve(new Blob([target.buffer], { type: mimeType }));
|
|
789
1346
|
}
|
|
790
|
-
};
|
|
1347
|
+
});
|
|
791
1348
|
} catch (error) {
|
|
792
1349
|
await Promise.allSettled([cancelMediabunnyOutput(), discard()]);
|
|
793
1350
|
throw error;
|
|
@@ -798,6 +1355,156 @@ var createVideoLayerOutput = async ({
|
|
|
798
1355
|
return { output, finalize, cancel, discard };
|
|
799
1356
|
};
|
|
800
1357
|
|
|
1358
|
+
// src/create-video-matting-frames.ts
|
|
1359
|
+
import { CanvasSink, CanvasSource } from "mediabunny";
|
|
1360
|
+
|
|
1361
|
+
// src/video-matting-canvas.ts
|
|
1362
|
+
var createVideoMattingCanvas = ({
|
|
1363
|
+
width,
|
|
1364
|
+
height
|
|
1365
|
+
}) => {
|
|
1366
|
+
if (typeof OffscreenCanvas !== "undefined") {
|
|
1367
|
+
return new OffscreenCanvas(width, height);
|
|
1368
|
+
}
|
|
1369
|
+
if (typeof document !== "undefined") {
|
|
1370
|
+
const canvas = document.createElement("canvas");
|
|
1371
|
+
canvas.width = width;
|
|
1372
|
+
canvas.height = height;
|
|
1373
|
+
return canvas;
|
|
1374
|
+
}
|
|
1375
|
+
throw new Error("Could not create a canvas. This API must run in a browser with OffscreenCanvas or the DOM available.");
|
|
1376
|
+
};
|
|
1377
|
+
var getVideoMattingCanvasContext = (canvas) => {
|
|
1378
|
+
const context = canvas.getContext("2d", { willReadFrequently: true });
|
|
1379
|
+
if (!context) {
|
|
1380
|
+
throw new Error("Could not create a 2D canvas context.");
|
|
1381
|
+
}
|
|
1382
|
+
return context;
|
|
1383
|
+
};
|
|
1384
|
+
var drawOpaqueBaseFrame = ({
|
|
1385
|
+
context,
|
|
1386
|
+
source,
|
|
1387
|
+
width,
|
|
1388
|
+
height
|
|
1389
|
+
}) => {
|
|
1390
|
+
context.save();
|
|
1391
|
+
context.globalCompositeOperation = "copy";
|
|
1392
|
+
context.fillStyle = "#000000";
|
|
1393
|
+
context.fillRect(0, 0, width, height);
|
|
1394
|
+
context.globalCompositeOperation = "source-over";
|
|
1395
|
+
context.drawImage(source, 0, 0, width, height);
|
|
1396
|
+
context.restore();
|
|
1397
|
+
};
|
|
1398
|
+
var drawForegroundFrame = ({
|
|
1399
|
+
context,
|
|
1400
|
+
result,
|
|
1401
|
+
source,
|
|
1402
|
+
targetWidth,
|
|
1403
|
+
targetHeight
|
|
1404
|
+
}) => {
|
|
1405
|
+
if (!Number.isInteger(result.width) || result.width <= 0 || !Number.isInteger(result.height) || result.height <= 0) {
|
|
1406
|
+
throw new Error("The video matting model returned invalid dimensions.");
|
|
1407
|
+
}
|
|
1408
|
+
const expectedLength = result.width * result.height * result.channels;
|
|
1409
|
+
if (result.data.length !== expectedLength) {
|
|
1410
|
+
throw new Error(`The video matting model returned ${result.data.length} bytes, but ${expectedLength} RGBA bytes were expected.`);
|
|
1411
|
+
}
|
|
1412
|
+
const imageData = new ImageData(result.data, result.width, result.height);
|
|
1413
|
+
context.clearRect(0, 0, targetWidth, targetHeight);
|
|
1414
|
+
if (result.width === targetWidth && result.height === targetHeight) {
|
|
1415
|
+
context.putImageData(imageData, 0, 0);
|
|
1416
|
+
} else {
|
|
1417
|
+
const intermediateCanvas = createVideoMattingCanvas({
|
|
1418
|
+
width: result.width,
|
|
1419
|
+
height: result.height
|
|
1420
|
+
});
|
|
1421
|
+
const intermediateContext = getVideoMattingCanvasContext(intermediateCanvas);
|
|
1422
|
+
intermediateContext.putImageData(imageData, 0, 0);
|
|
1423
|
+
context.drawImage(intermediateCanvas, 0, 0, targetWidth, targetHeight);
|
|
1424
|
+
}
|
|
1425
|
+
context.save();
|
|
1426
|
+
context.globalCompositeOperation = "destination-in";
|
|
1427
|
+
context.drawImage(source, 0, 0, targetWidth, targetHeight);
|
|
1428
|
+
context.restore();
|
|
1429
|
+
};
|
|
1430
|
+
|
|
1431
|
+
// src/create-video-matting-frames.ts
|
|
1432
|
+
var createVideoMattingFrames = async (options) => {
|
|
1433
|
+
if (typeof window === "undefined" && typeof process !== "undefined" && process.release?.name === "node") {
|
|
1434
|
+
const { createNodeVideoMattingFrames: createNodeVideoMattingFrames2 } = await Promise.resolve().then(() => (init_node_video_matting_frames(), exports_node_video_matting_frames));
|
|
1435
|
+
return createNodeVideoMattingFrames2(options);
|
|
1436
|
+
}
|
|
1437
|
+
const {
|
|
1438
|
+
videoTrack,
|
|
1439
|
+
width,
|
|
1440
|
+
height,
|
|
1441
|
+
videoQuality,
|
|
1442
|
+
keyframeIntervalInSeconds,
|
|
1443
|
+
videoStartTimestamp,
|
|
1444
|
+
videoEndTimestamp,
|
|
1445
|
+
includeBase
|
|
1446
|
+
} = options;
|
|
1447
|
+
const sink = new CanvasSink(videoTrack, {
|
|
1448
|
+
alpha: true,
|
|
1449
|
+
width,
|
|
1450
|
+
height,
|
|
1451
|
+
fit: "fill",
|
|
1452
|
+
poolSize: 1
|
|
1453
|
+
});
|
|
1454
|
+
const baseCanvas = includeBase ? createVideoMattingCanvas({ width, height }) : null;
|
|
1455
|
+
const foregroundCanvas = createVideoMattingCanvas({ width, height });
|
|
1456
|
+
const baseContext = baseCanvas ? getVideoMattingCanvasContext(baseCanvas) : null;
|
|
1457
|
+
const foregroundContext = getVideoMattingCanvasContext(foregroundCanvas);
|
|
1458
|
+
const baseSource = baseCanvas ? new CanvasSource(baseCanvas, {
|
|
1459
|
+
codec: "vp9",
|
|
1460
|
+
quality: videoQuality,
|
|
1461
|
+
keyFrameInterval: keyframeIntervalInSeconds,
|
|
1462
|
+
alpha: "discard"
|
|
1463
|
+
}) : null;
|
|
1464
|
+
const foregroundSource = new CanvasSource(foregroundCanvas, {
|
|
1465
|
+
codec: "vp9",
|
|
1466
|
+
quality: videoQuality,
|
|
1467
|
+
keyFrameInterval: keyframeIntervalInSeconds,
|
|
1468
|
+
alpha: "keep"
|
|
1469
|
+
});
|
|
1470
|
+
return {
|
|
1471
|
+
frames: async function* () {
|
|
1472
|
+
for await (const frame of sink.canvases(videoStartTimestamp, videoEndTimestamp)) {
|
|
1473
|
+
yield {
|
|
1474
|
+
image: frame.canvas,
|
|
1475
|
+
timestamp: frame.timestamp,
|
|
1476
|
+
duration: frame.duration
|
|
1477
|
+
};
|
|
1478
|
+
}
|
|
1479
|
+
}(),
|
|
1480
|
+
baseSource,
|
|
1481
|
+
foregroundSource,
|
|
1482
|
+
addFrame: async ({ frame, foreground, timestamp, duration }) => {
|
|
1483
|
+
if ("data" in frame.image) {
|
|
1484
|
+
throw new Error("Expected a canvas frame.");
|
|
1485
|
+
}
|
|
1486
|
+
if (baseContext) {
|
|
1487
|
+
drawOpaqueBaseFrame({
|
|
1488
|
+
context: baseContext,
|
|
1489
|
+
source: frame.image,
|
|
1490
|
+
width,
|
|
1491
|
+
height
|
|
1492
|
+
});
|
|
1493
|
+
}
|
|
1494
|
+
drawForegroundFrame({
|
|
1495
|
+
context: foregroundContext,
|
|
1496
|
+
result: foreground,
|
|
1497
|
+
source: frame.image,
|
|
1498
|
+
targetWidth: width,
|
|
1499
|
+
targetHeight: height
|
|
1500
|
+
});
|
|
1501
|
+
await Promise.all([
|
|
1502
|
+
baseSource?.add(timestamp, duration),
|
|
1503
|
+
foregroundSource.add(timestamp, duration)
|
|
1504
|
+
]);
|
|
1505
|
+
}
|
|
1506
|
+
};
|
|
1507
|
+
};
|
|
801
1508
|
// src/prepare-audio.ts
|
|
802
1509
|
import {
|
|
803
1510
|
AudioSampleSink,
|
|
@@ -858,6 +1565,9 @@ var prepareAudio = async ({
|
|
|
858
1565
|
cancel: () => Promise.resolve()
|
|
859
1566
|
};
|
|
860
1567
|
}
|
|
1568
|
+
if ((destination === "base" || destination === "both") && baseOutput === null) {
|
|
1569
|
+
throw new Error("A base output is required for this audio destination.");
|
|
1570
|
+
}
|
|
861
1571
|
const outputs = destination === "both" ? [baseOutput, foregroundOutput] : destination === "base" ? [baseOutput] : [foregroundOutput];
|
|
862
1572
|
const sourceCodec = await audioTrack.getCodec();
|
|
863
1573
|
const canCopyPackets = !forceTranscode && sourceCodec === "opus" && firstAudioTimestamp >= videoStartTimestamp;
|
|
@@ -1151,76 +1861,6 @@ var prepareAudio = async ({
|
|
|
1151
1861
|
};
|
|
1152
1862
|
};
|
|
1153
1863
|
|
|
1154
|
-
// src/video-matting-canvas.ts
|
|
1155
|
-
var createVideoMattingCanvas = ({
|
|
1156
|
-
width,
|
|
1157
|
-
height
|
|
1158
|
-
}) => {
|
|
1159
|
-
if (typeof OffscreenCanvas !== "undefined") {
|
|
1160
|
-
return new OffscreenCanvas(width, height);
|
|
1161
|
-
}
|
|
1162
|
-
if (typeof document !== "undefined") {
|
|
1163
|
-
const canvas = document.createElement("canvas");
|
|
1164
|
-
canvas.width = width;
|
|
1165
|
-
canvas.height = height;
|
|
1166
|
-
return canvas;
|
|
1167
|
-
}
|
|
1168
|
-
throw new Error("Could not create a canvas. This API must run in a browser with OffscreenCanvas or the DOM available.");
|
|
1169
|
-
};
|
|
1170
|
-
var getVideoMattingCanvasContext = (canvas) => {
|
|
1171
|
-
const context = canvas.getContext("2d", { willReadFrequently: true });
|
|
1172
|
-
if (!context) {
|
|
1173
|
-
throw new Error("Could not create a 2D canvas context.");
|
|
1174
|
-
}
|
|
1175
|
-
return context;
|
|
1176
|
-
};
|
|
1177
|
-
var drawOpaqueBaseFrame = ({
|
|
1178
|
-
context,
|
|
1179
|
-
source,
|
|
1180
|
-
width,
|
|
1181
|
-
height
|
|
1182
|
-
}) => {
|
|
1183
|
-
context.save();
|
|
1184
|
-
context.globalCompositeOperation = "copy";
|
|
1185
|
-
context.fillStyle = "#000000";
|
|
1186
|
-
context.fillRect(0, 0, width, height);
|
|
1187
|
-
context.globalCompositeOperation = "source-over";
|
|
1188
|
-
context.drawImage(source, 0, 0, width, height);
|
|
1189
|
-
context.restore();
|
|
1190
|
-
};
|
|
1191
|
-
var drawForegroundFrame = ({
|
|
1192
|
-
context,
|
|
1193
|
-
result,
|
|
1194
|
-
source,
|
|
1195
|
-
targetWidth,
|
|
1196
|
-
targetHeight
|
|
1197
|
-
}) => {
|
|
1198
|
-
if (!Number.isInteger(result.width) || result.width <= 0 || !Number.isInteger(result.height) || result.height <= 0) {
|
|
1199
|
-
throw new Error("The video matting model returned invalid dimensions.");
|
|
1200
|
-
}
|
|
1201
|
-
const expectedLength = result.width * result.height * result.channels;
|
|
1202
|
-
if (result.data.length !== expectedLength) {
|
|
1203
|
-
throw new Error(`The video matting model returned ${result.data.length} bytes, but ${expectedLength} RGBA bytes were expected.`);
|
|
1204
|
-
}
|
|
1205
|
-
const imageData = new ImageData(result.data, result.width, result.height);
|
|
1206
|
-
context.clearRect(0, 0, targetWidth, targetHeight);
|
|
1207
|
-
if (result.width === targetWidth && result.height === targetHeight) {
|
|
1208
|
-
context.putImageData(imageData, 0, 0);
|
|
1209
|
-
} else {
|
|
1210
|
-
const intermediateCanvas = createVideoMattingCanvas({
|
|
1211
|
-
width: result.width,
|
|
1212
|
-
height: result.height
|
|
1213
|
-
});
|
|
1214
|
-
const intermediateContext = getVideoMattingCanvasContext(intermediateCanvas);
|
|
1215
|
-
intermediateContext.putImageData(imageData, 0, 0);
|
|
1216
|
-
context.drawImage(intermediateCanvas, 0, 0, targetWidth, targetHeight);
|
|
1217
|
-
}
|
|
1218
|
-
context.save();
|
|
1219
|
-
context.globalCompositeOperation = "destination-in";
|
|
1220
|
-
context.drawImage(source, 0, 0, targetWidth, targetHeight);
|
|
1221
|
-
context.restore();
|
|
1222
|
-
};
|
|
1223
|
-
|
|
1224
1864
|
// src/video-matting-quality.ts
|
|
1225
1865
|
import { Quality as Quality2 } from "mediabunny";
|
|
1226
1866
|
var VIDEO_MATTING_QUALITIES = [
|
|
@@ -1316,7 +1956,7 @@ var createAbortError = (signal) => {
|
|
|
1316
1956
|
if (signal.reason !== undefined) {
|
|
1317
1957
|
return signal.reason;
|
|
1318
1958
|
}
|
|
1319
|
-
const error = new Error("Video
|
|
1959
|
+
const error = new Error("Video matting was aborted.");
|
|
1320
1960
|
error.name = "AbortError";
|
|
1321
1961
|
return error;
|
|
1322
1962
|
};
|
|
@@ -1350,9 +1990,9 @@ var validateLayerOutputOptions = ({
|
|
|
1350
1990
|
}
|
|
1351
1991
|
}
|
|
1352
1992
|
};
|
|
1353
|
-
var validateOptions = (options) => {
|
|
1993
|
+
var validateOptions = (options, operationName) => {
|
|
1354
1994
|
if (!options || typeof options !== "object") {
|
|
1355
|
-
throw new TypeError(
|
|
1995
|
+
throw new TypeError(`${operationName}() expects an options object.`);
|
|
1356
1996
|
}
|
|
1357
1997
|
const isBlob = typeof Blob !== "undefined" && options.src instanceof Blob;
|
|
1358
1998
|
const isUrl = options.src instanceof URL;
|
|
@@ -1392,13 +2032,30 @@ var validateOptions = (options) => {
|
|
|
1392
2032
|
throw new TypeError("onModelLoadProgress must be a function.");
|
|
1393
2033
|
}
|
|
1394
2034
|
};
|
|
1395
|
-
var makeInput = (src) => {
|
|
2035
|
+
var makeInput = async (src) => {
|
|
2036
|
+
if (typeof window === "undefined" && typeof process !== "undefined" && process.release?.name === "node" && (typeof src === "string" || src instanceof URL)) {
|
|
2037
|
+
const path = String(src);
|
|
2038
|
+
if (path.startsWith("file:")) {
|
|
2039
|
+
const { fileURLToPath } = await importNodeModule("node:url");
|
|
2040
|
+
return new Input({
|
|
2041
|
+
formats: ALL_FORMATS,
|
|
2042
|
+
source: new FilePathSource(fileURLToPath(path))
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
if (/^[a-z]:[/\\]/i.test(path) || !/^[a-z][a-z\d+.-]*:/i.test(path)) {
|
|
2046
|
+
return new Input({
|
|
2047
|
+
formats: ALL_FORMATS,
|
|
2048
|
+
source: new FilePathSource(path)
|
|
2049
|
+
});
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
1396
2052
|
const source = typeof src === "string" || src instanceof URL ? new UrlSource(src) : new BlobSource(src);
|
|
1397
2053
|
return new Input({ formats: ALL_FORMATS, source });
|
|
1398
2054
|
};
|
|
1399
2055
|
var probeVideoInput = async ({
|
|
1400
2056
|
input,
|
|
1401
|
-
videoQuality
|
|
2057
|
+
videoQuality,
|
|
2058
|
+
includeBase
|
|
1402
2059
|
}) => {
|
|
1403
2060
|
if (!await input.canRead()) {
|
|
1404
2061
|
throw new Error("The input is not a supported media file.");
|
|
@@ -1418,12 +2075,12 @@ var probeVideoInput = async ({
|
|
|
1418
2075
|
throw new Error("The input video has invalid dimensions.");
|
|
1419
2076
|
}
|
|
1420
2077
|
const [canEncodeBase, canEncodeForeground] = await Promise.all([
|
|
1421
|
-
canEncodeVideo("vp9", {
|
|
2078
|
+
includeBase ? canEncodeVideo("vp9", {
|
|
1422
2079
|
width,
|
|
1423
2080
|
height,
|
|
1424
2081
|
quality: videoQuality,
|
|
1425
2082
|
alpha: "discard"
|
|
1426
|
-
}),
|
|
2083
|
+
}) : Promise.resolve(true),
|
|
1427
2084
|
canEncodeVideo("vp9", {
|
|
1428
2085
|
width,
|
|
1429
2086
|
height,
|
|
@@ -1432,25 +2089,27 @@ var probeVideoInput = async ({
|
|
|
1432
2089
|
})
|
|
1433
2090
|
]);
|
|
1434
2091
|
if (!canEncodeBase || !canEncodeForeground) {
|
|
1435
|
-
throw new Error("This
|
|
2092
|
+
throw new Error("This environment cannot encode the VP9 video streams required for video matting.");
|
|
1436
2093
|
}
|
|
1437
2094
|
return { videoTrack, width, height };
|
|
1438
2095
|
};
|
|
1439
|
-
var
|
|
1440
|
-
validateOptions(options);
|
|
2096
|
+
var runVideoMatting = async (options, includeBase) => {
|
|
2097
|
+
validateOptions(options, includeBase ? "separateVideoLayers" : "removeVideoBackground");
|
|
1441
2098
|
throwIfAborted(options.signal);
|
|
1442
2099
|
const model = options.model ?? "modnet";
|
|
1443
|
-
const audio = options.audio ?? "base";
|
|
2100
|
+
const audio = options.audio ?? (includeBase ? "base" : "foreground");
|
|
1444
2101
|
const videoQuality = resolveVideoMattingQuality(options.videoBitrate ?? "very-high");
|
|
1445
2102
|
const audioQuality = resolveVideoMattingQuality(options.audioBitrate ?? "medium");
|
|
1446
2103
|
const keyframeIntervalInSeconds = options.keyframeIntervalInSeconds ?? 1;
|
|
1447
|
-
const input = makeInput(options.src);
|
|
2104
|
+
const input = await makeInput(options.src);
|
|
1448
2105
|
const onInputAbort = () => input.dispose();
|
|
1449
2106
|
options.signal?.addEventListener("abort", onInputAbort, { once: true });
|
|
1450
2107
|
try {
|
|
2108
|
+
throwIfAborted(options.signal);
|
|
1451
2109
|
const { videoTrack, width, height } = await probeVideoInput({
|
|
1452
2110
|
input,
|
|
1453
|
-
videoQuality
|
|
2111
|
+
videoQuality,
|
|
2112
|
+
includeBase
|
|
1454
2113
|
});
|
|
1455
2114
|
const [inputFirstVideoTimestamp, videoEndTimestamp] = await Promise.all([
|
|
1456
2115
|
videoTrack.getFirstTimestamp(),
|
|
@@ -1468,11 +2127,9 @@ var separateVideoLayers = async (options) => {
|
|
|
1468
2127
|
signal: options.signal ?? null,
|
|
1469
2128
|
run: async (pipeline) => {
|
|
1470
2129
|
throwIfAborted(options.signal);
|
|
1471
|
-
let
|
|
2130
|
+
let videoFrames = null;
|
|
1472
2131
|
let baseOutput = null;
|
|
1473
2132
|
let foregroundOutput = null;
|
|
1474
|
-
let baseVideoSource = null;
|
|
1475
|
-
let foregroundVideoSource = null;
|
|
1476
2133
|
let audioWriter = null;
|
|
1477
2134
|
let completed = false;
|
|
1478
2135
|
let abortCleanupPromise = null;
|
|
@@ -1492,50 +2149,41 @@ var separateVideoLayers = async (options) => {
|
|
|
1492
2149
|
};
|
|
1493
2150
|
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
1494
2151
|
try {
|
|
1495
|
-
|
|
1496
|
-
|
|
2152
|
+
videoFrames = await createVideoMattingFrames({
|
|
2153
|
+
videoTrack,
|
|
1497
2154
|
width,
|
|
1498
2155
|
height,
|
|
1499
|
-
|
|
1500
|
-
|
|
2156
|
+
videoQuality,
|
|
2157
|
+
keyframeIntervalInSeconds,
|
|
2158
|
+
videoStartTimestamp,
|
|
2159
|
+
videoEndTimestamp,
|
|
2160
|
+
includeBase
|
|
1501
2161
|
});
|
|
1502
|
-
iterator =
|
|
2162
|
+
const iterator = videoFrames.frames;
|
|
1503
2163
|
let nextFrame = await iterator.next();
|
|
1504
2164
|
if (nextFrame.done) {
|
|
1505
2165
|
throw new Error("The primary video track contains no presentable decodable frames.");
|
|
1506
2166
|
}
|
|
1507
2167
|
throwIfAborted(options.signal);
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
options: options.outputs?.base
|
|
1515
|
-
});
|
|
2168
|
+
if (includeBase) {
|
|
2169
|
+
baseOutput = await createVideoLayerOutput({
|
|
2170
|
+
format: new WebMOutputFormat,
|
|
2171
|
+
options: options.outputs?.base
|
|
2172
|
+
});
|
|
2173
|
+
}
|
|
1516
2174
|
throwIfAborted(options.signal);
|
|
1517
2175
|
foregroundOutput = await createVideoLayerOutput({
|
|
1518
2176
|
format: new WebMOutputFormat,
|
|
1519
2177
|
options: options.outputs?.foreground
|
|
1520
2178
|
});
|
|
1521
2179
|
throwIfAborted(options.signal);
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
alpha: "discard"
|
|
1527
|
-
});
|
|
1528
|
-
foregroundVideoSource = new CanvasSource(foregroundCanvas, {
|
|
1529
|
-
codec: "vp9",
|
|
1530
|
-
quality: videoQuality,
|
|
1531
|
-
keyFrameInterval: keyframeIntervalInSeconds,
|
|
1532
|
-
alpha: "keep"
|
|
1533
|
-
});
|
|
1534
|
-
baseOutput.output.addVideoTrack(baseVideoSource);
|
|
1535
|
-
foregroundOutput.output.addVideoTrack(foregroundVideoSource);
|
|
2180
|
+
if (baseOutput && videoFrames.baseSource) {
|
|
2181
|
+
baseOutput.output.addVideoTrack(videoFrames.baseSource);
|
|
2182
|
+
}
|
|
2183
|
+
foregroundOutput.output.addVideoTrack(videoFrames.foregroundSource);
|
|
1536
2184
|
audioWriter = await prepareAudio({
|
|
1537
2185
|
input,
|
|
1538
|
-
baseOutput: baseOutput
|
|
2186
|
+
baseOutput: baseOutput?.output ?? null,
|
|
1539
2187
|
foregroundOutput: foregroundOutput.output,
|
|
1540
2188
|
destination: audio,
|
|
1541
2189
|
videoStartTimestamp,
|
|
@@ -1545,7 +2193,7 @@ var separateVideoLayers = async (options) => {
|
|
|
1545
2193
|
});
|
|
1546
2194
|
throwIfAborted(options.signal);
|
|
1547
2195
|
await Promise.all([
|
|
1548
|
-
baseOutput
|
|
2196
|
+
baseOutput?.output.start(),
|
|
1549
2197
|
foregroundOutput.output.start()
|
|
1550
2198
|
]);
|
|
1551
2199
|
throwIfAborted(options.signal);
|
|
@@ -1573,24 +2221,14 @@ var separateVideoLayers = async (options) => {
|
|
|
1573
2221
|
nextFrame = await iterator.next();
|
|
1574
2222
|
continue;
|
|
1575
2223
|
}
|
|
1576
|
-
const foregroundFrame = await pipeline(frame.
|
|
2224
|
+
const foregroundFrame = await pipeline(frame.image);
|
|
1577
2225
|
throwIfAborted(options.signal);
|
|
1578
|
-
drawOpaqueBaseFrame({
|
|
1579
|
-
context: baseContext,
|
|
1580
|
-
source: frame.canvas,
|
|
1581
|
-
width,
|
|
1582
|
-
height
|
|
1583
|
-
});
|
|
1584
|
-
drawForegroundFrame({
|
|
1585
|
-
context: foregroundContext,
|
|
1586
|
-
result: foregroundFrame,
|
|
1587
|
-
source: frame.canvas,
|
|
1588
|
-
targetWidth: width,
|
|
1589
|
-
targetHeight: height
|
|
1590
|
-
});
|
|
1591
2226
|
await Promise.all([
|
|
1592
|
-
|
|
1593
|
-
|
|
2227
|
+
videoFrames.addFrame({
|
|
2228
|
+
frame,
|
|
2229
|
+
foreground: foregroundFrame,
|
|
2230
|
+
...timing
|
|
2231
|
+
}),
|
|
1594
2232
|
audioWriter.writeAudioUntil(timing.timestamp + timing.duration)
|
|
1595
2233
|
]);
|
|
1596
2234
|
processedFrames++;
|
|
@@ -1618,23 +2256,29 @@ var separateVideoLayers = async (options) => {
|
|
|
1618
2256
|
durationInSeconds
|
|
1619
2257
|
});
|
|
1620
2258
|
await audioWriter.finishAudio();
|
|
1621
|
-
|
|
1622
|
-
|
|
2259
|
+
videoFrames.baseSource?.close();
|
|
2260
|
+
videoFrames.foregroundSource.close();
|
|
1623
2261
|
const [base, foreground] = await Promise.all([
|
|
1624
|
-
baseOutput
|
|
2262
|
+
baseOutput?.finalize() ?? Promise.resolve(null),
|
|
1625
2263
|
foregroundOutput.finalize()
|
|
1626
2264
|
]);
|
|
1627
2265
|
throwIfAborted(options.signal);
|
|
1628
2266
|
completed = true;
|
|
1629
|
-
|
|
1630
|
-
base,
|
|
1631
|
-
foreground,
|
|
2267
|
+
const separated = {
|
|
2268
|
+
...base ? { base, foreground } : { video: foreground },
|
|
1632
2269
|
model,
|
|
1633
2270
|
width,
|
|
1634
2271
|
height,
|
|
1635
2272
|
durationInSeconds,
|
|
1636
2273
|
processedFrames
|
|
1637
2274
|
};
|
|
2275
|
+
Object.defineProperty(separated, Symbol.asyncDispose, {
|
|
2276
|
+
enumerable: false,
|
|
2277
|
+
value: async () => {
|
|
2278
|
+
await Promise.all([base?.dispose(), foreground.dispose()]);
|
|
2279
|
+
}
|
|
2280
|
+
});
|
|
2281
|
+
return separated;
|
|
1638
2282
|
} catch (error) {
|
|
1639
2283
|
await cancelPendingMedia();
|
|
1640
2284
|
if (options.signal?.aborted) {
|
|
@@ -1646,7 +2290,7 @@ var separateVideoLayers = async (options) => {
|
|
|
1646
2290
|
await abortCleanupPromise;
|
|
1647
2291
|
if (!completed) {
|
|
1648
2292
|
try {
|
|
1649
|
-
await
|
|
2293
|
+
await videoFrames?.frames.return();
|
|
1650
2294
|
} catch {}
|
|
1651
2295
|
}
|
|
1652
2296
|
}
|
|
@@ -1663,13 +2307,24 @@ var separateVideoLayers = async (options) => {
|
|
|
1663
2307
|
input.dispose();
|
|
1664
2308
|
}
|
|
1665
2309
|
};
|
|
2310
|
+
var separateVideoLayers = async (options) => await runVideoMatting(options, true);
|
|
2311
|
+
var removeVideoBackground = async (options) => await runVideoMatting({
|
|
2312
|
+
...options,
|
|
2313
|
+
audio: options.audio === "none" ? "none" : "foreground",
|
|
2314
|
+
outputs: { foreground: options.output }
|
|
2315
|
+
}, false);
|
|
2316
|
+
|
|
2317
|
+
// src/index.ts
|
|
2318
|
+
var VideoMattingInternals = { removeVideoBackground };
|
|
1666
2319
|
export {
|
|
1667
2320
|
separateVideoLayers,
|
|
1668
2321
|
removeVideoMattingModel,
|
|
1669
2322
|
loadVideoMattingModel,
|
|
1670
2323
|
isVideoMattingModelCached,
|
|
1671
2324
|
getAvailableModels,
|
|
2325
|
+
downloadVideoMattingModel,
|
|
1672
2326
|
disposeVideoMattingModel,
|
|
1673
2327
|
canUseVideoMatting,
|
|
1674
|
-
VideoMattingUnsupportedReason
|
|
2328
|
+
VideoMattingUnsupportedReason,
|
|
2329
|
+
VideoMattingInternals
|
|
1675
2330
|
};
|