@remotion/video-matting 4.0.523
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/can-use-video-matting.d.ts +18 -0
- package/dist/create-video-layer-output.d.ts +14 -0
- package/dist/esm/index.mjs +1675 -0
- package/dist/index.d.ts +14 -0
- package/dist/is-video-matting-model-cached.d.ts +5 -0
- package/dist/load-video-matting-model.d.ts +35 -0
- package/dist/models.d.ts +17 -0
- package/dist/output-target.d.ts +13 -0
- package/dist/prepare-audio.d.ts +28 -0
- package/dist/remove-video-matting-model.d.ts +5 -0
- package/dist/separate-video-layers.d.ts +43 -0
- package/dist/video-matting-canvas.d.ts +22 -0
- package/dist/video-matting-quality.d.ts +5 -0
- package/dist/video-timing.d.ts +22 -0
- package/dist/web-fs-target.d.ts +8 -0
- package/dist/with-remotion-model-host.d.ts +10 -0
- package/package.json +56 -0
|
@@ -0,0 +1,1675 @@
|
|
|
1
|
+
var __create = Object.create;
|
|
2
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __toESM = (mod, isNodeMode, target) => {
|
|
7
|
+
target = mod != null ? __create(__getProtoOf(mod)) : {};
|
|
8
|
+
const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
|
|
9
|
+
for (let key of __getOwnPropNames(mod))
|
|
10
|
+
if (!__hasOwnProp.call(to, key))
|
|
11
|
+
__defProp(to, key, {
|
|
12
|
+
get: () => mod[key],
|
|
13
|
+
enumerable: true
|
|
14
|
+
});
|
|
15
|
+
return to;
|
|
16
|
+
};
|
|
17
|
+
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
|
|
18
|
+
get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
|
|
19
|
+
}) : x)(function(x) {
|
|
20
|
+
if (typeof require !== "undefined")
|
|
21
|
+
return require.apply(this, arguments);
|
|
22
|
+
throw Error('Dynamic require of "' + x + '" is not supported');
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
// src/models.ts
|
|
26
|
+
var VIDEO_MATTING_MODELS = ["modnet", "ben2-base"];
|
|
27
|
+
var MODEL_INFO = {
|
|
28
|
+
modnet: {
|
|
29
|
+
name: "modnet",
|
|
30
|
+
modelId: "Xenova/modnet",
|
|
31
|
+
purpose: "person",
|
|
32
|
+
webGpuDownloadSize: 25889088,
|
|
33
|
+
revision: "fa2fa546052fba4c08921230a26cc69a333fca12",
|
|
34
|
+
dtype: "fp32",
|
|
35
|
+
requiresShaderF16: false
|
|
36
|
+
},
|
|
37
|
+
"ben2-base": {
|
|
38
|
+
name: "ben2-base",
|
|
39
|
+
modelId: "onnx-community/BEN2-ONNX",
|
|
40
|
+
purpose: "general",
|
|
41
|
+
webGpuDownloadSize: 219122146,
|
|
42
|
+
revision: "c552aa82688edce09f0ac9d2e31ad53d9d629010",
|
|
43
|
+
dtype: "fp16",
|
|
44
|
+
requiresShaderF16: true
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
var HOSTED_MODEL_IDS = {
|
|
48
|
+
modnet: "modnet-v1",
|
|
49
|
+
"ben2-base": "ben2-base-v1"
|
|
50
|
+
};
|
|
51
|
+
var getAvailableModels = () => {
|
|
52
|
+
return VIDEO_MATTING_MODELS.map((model) => {
|
|
53
|
+
const {
|
|
54
|
+
revision: _revision,
|
|
55
|
+
dtype: _dtype,
|
|
56
|
+
requiresShaderF16: _requiresShaderF16,
|
|
57
|
+
...info
|
|
58
|
+
} = MODEL_INFO[model];
|
|
59
|
+
return { ...info };
|
|
60
|
+
});
|
|
61
|
+
};
|
|
62
|
+
var getVideoMattingModelInfo = (model) => {
|
|
63
|
+
if (!Object.hasOwn(MODEL_INFO, model)) {
|
|
64
|
+
throw new Error(`Unsupported video matting model "${model}". Available models: ${VIDEO_MATTING_MODELS.join(", ")}.`);
|
|
65
|
+
}
|
|
66
|
+
return MODEL_INFO[model];
|
|
67
|
+
};
|
|
68
|
+
var getHostedVideoMattingModelId = (model) => {
|
|
69
|
+
return HOSTED_MODEL_IDS[model];
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
// src/can-use-video-matting.ts
|
|
73
|
+
var VideoMattingUnsupportedReason;
|
|
74
|
+
((VideoMattingUnsupportedReason2) => {
|
|
75
|
+
VideoMattingUnsupportedReason2["WindowUndefined"] = "window-undefined";
|
|
76
|
+
VideoMattingUnsupportedReason2["WebGpuUnavailable"] = "webgpu-unavailable";
|
|
77
|
+
VideoMattingUnsupportedReason2["WebGpuRequiresSecureContext"] = "webgpu-requires-secure-context";
|
|
78
|
+
VideoMattingUnsupportedReason2["ShaderF16Unavailable"] = "shader-f16-unavailable";
|
|
79
|
+
})(VideoMattingUnsupportedReason ||= {});
|
|
80
|
+
var canUseVideoMatting = async ({
|
|
81
|
+
model = "modnet"
|
|
82
|
+
} = {}) => {
|
|
83
|
+
const modelInfo = getVideoMattingModelInfo(model);
|
|
84
|
+
if (typeof window === "undefined" && typeof OffscreenCanvas === "undefined") {
|
|
85
|
+
return {
|
|
86
|
+
supported: false,
|
|
87
|
+
reason: "window-undefined" /* WindowUndefined */,
|
|
88
|
+
detailedReason: "No browser window or worker canvas environment is available. @remotion/video-matting is intended for browser environments."
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
const isSecureContext = typeof window === "undefined" ? globalThis.isSecureContext : window.isSecureContext;
|
|
92
|
+
if (isSecureContext === false) {
|
|
93
|
+
return {
|
|
94
|
+
supported: false,
|
|
95
|
+
reason: "webgpu-requires-secure-context" /* WebGpuRequiresSecureContext */,
|
|
96
|
+
detailedReason: "WebGPU requires HTTPS in production or localhost during development."
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
if (typeof navigator === "undefined" || !("gpu" in navigator)) {
|
|
100
|
+
return {
|
|
101
|
+
supported: false,
|
|
102
|
+
reason: "webgpu-unavailable" /* WebGpuUnavailable */,
|
|
103
|
+
detailedReason: "WebGPU is not available in this browser."
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
let adapter;
|
|
107
|
+
try {
|
|
108
|
+
const { gpu } = navigator;
|
|
109
|
+
adapter = await gpu.requestAdapter();
|
|
110
|
+
} catch {
|
|
111
|
+
adapter = null;
|
|
112
|
+
}
|
|
113
|
+
if (!adapter) {
|
|
114
|
+
return {
|
|
115
|
+
supported: false,
|
|
116
|
+
reason: "webgpu-unavailable" /* WebGpuUnavailable */,
|
|
117
|
+
detailedReason: "No usable WebGPU adapter is available in this browser."
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
if (modelInfo.requiresShaderF16 && adapter.features?.has("shader-f16") !== true) {
|
|
121
|
+
return {
|
|
122
|
+
supported: false,
|
|
123
|
+
reason: "shader-f16-unavailable" /* ShaderF16Unavailable */,
|
|
124
|
+
detailedReason: `The video matting model "${model}" requires the WebGPU shader-f16 feature, which is unavailable on this adapter.`
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
return { supported: true };
|
|
128
|
+
};
|
|
129
|
+
// src/with-remotion-model-host.ts
|
|
130
|
+
var REMOTION_MODEL_HOST = "https://remotion.media/";
|
|
131
|
+
var REMOTION_MODEL_PATH_TEMPLATE = "models/{model}/";
|
|
132
|
+
var REMOTION_MODEL_HOST_STATE = Symbol.for("@remotion/transformers/model-host-state");
|
|
133
|
+
var WHISPER_MODEL_HOST_STATE = Symbol.for("@remotion/whisper-webgpu/model-host-state");
|
|
134
|
+
var withRemotionModelHost = async (operation) => {
|
|
135
|
+
const transformers = await import("@huggingface/transformers");
|
|
136
|
+
const { env } = transformers;
|
|
137
|
+
const environmentWithState = env;
|
|
138
|
+
let state = environmentWithState[REMOTION_MODEL_HOST_STATE] ?? environmentWithState[WHISPER_MODEL_HOST_STATE];
|
|
139
|
+
if (!state) {
|
|
140
|
+
state = {
|
|
141
|
+
activeOperations: 0,
|
|
142
|
+
previousRemoteConfiguration: null
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
if (!environmentWithState[REMOTION_MODEL_HOST_STATE]) {
|
|
146
|
+
Object.defineProperty(environmentWithState, REMOTION_MODEL_HOST_STATE, {
|
|
147
|
+
value: state
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
if (!environmentWithState[WHISPER_MODEL_HOST_STATE]) {
|
|
151
|
+
Object.defineProperty(environmentWithState, WHISPER_MODEL_HOST_STATE, {
|
|
152
|
+
value: state
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
if (state.activeOperations === 0) {
|
|
156
|
+
state.previousRemoteConfiguration = {
|
|
157
|
+
remoteHost: env.remoteHost,
|
|
158
|
+
remotePathTemplate: env.remotePathTemplate
|
|
159
|
+
};
|
|
160
|
+
env.remoteHost = REMOTION_MODEL_HOST;
|
|
161
|
+
env.remotePathTemplate = REMOTION_MODEL_PATH_TEMPLATE;
|
|
162
|
+
}
|
|
163
|
+
state.activeOperations++;
|
|
164
|
+
try {
|
|
165
|
+
return await operation(transformers);
|
|
166
|
+
} finally {
|
|
167
|
+
state.activeOperations--;
|
|
168
|
+
if (state.activeOperations === 0) {
|
|
169
|
+
const configurationToRestore = state.previousRemoteConfiguration;
|
|
170
|
+
state.previousRemoteConfiguration = null;
|
|
171
|
+
if (configurationToRestore) {
|
|
172
|
+
if (env.remoteHost === REMOTION_MODEL_HOST) {
|
|
173
|
+
env.remoteHost = configurationToRestore.remoteHost;
|
|
174
|
+
}
|
|
175
|
+
if (env.remotePathTemplate === REMOTION_MODEL_PATH_TEMPLATE) {
|
|
176
|
+
env.remotePathTemplate = configurationToRestore.remotePathTemplate;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
// src/load-video-matting-model.ts
|
|
184
|
+
var pipelines = new Map;
|
|
185
|
+
var disposals = new Map;
|
|
186
|
+
var notifyProgress = (state, progress) => {
|
|
187
|
+
state.lastProgress = progress;
|
|
188
|
+
for (const listener of state.progressListeners) {
|
|
189
|
+
try {
|
|
190
|
+
listener(progress);
|
|
191
|
+
} catch {}
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
var notifyIfIdle = (state) => {
|
|
195
|
+
if (state.pendingUses === 0 && state.activeUses === 0) {
|
|
196
|
+
for (const resolve of state.onIdle.splice(0)) {
|
|
197
|
+
resolve();
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
var waitForLoadingOrAbort = ({
|
|
202
|
+
loading,
|
|
203
|
+
signal
|
|
204
|
+
}) => {
|
|
205
|
+
if (signal === null) {
|
|
206
|
+
return loading;
|
|
207
|
+
}
|
|
208
|
+
if (signal.aborted) {
|
|
209
|
+
return Promise.reject(signal.reason);
|
|
210
|
+
}
|
|
211
|
+
return new Promise((resolve, reject) => {
|
|
212
|
+
const onAbort = () => {
|
|
213
|
+
signal.removeEventListener("abort", onAbort);
|
|
214
|
+
reject(signal.reason);
|
|
215
|
+
};
|
|
216
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
217
|
+
loading.then((value) => {
|
|
218
|
+
signal.removeEventListener("abort", onAbort);
|
|
219
|
+
resolve(value);
|
|
220
|
+
}, (error) => {
|
|
221
|
+
signal.removeEventListener("abort", onAbort);
|
|
222
|
+
reject(error);
|
|
223
|
+
});
|
|
224
|
+
});
|
|
225
|
+
};
|
|
226
|
+
var getOrCreateVideoMattingPipeline = ({
|
|
227
|
+
model
|
|
228
|
+
}) => {
|
|
229
|
+
const modelInfo = getVideoMattingModelInfo(model);
|
|
230
|
+
const existing = pipelines.get(model);
|
|
231
|
+
if (existing) {
|
|
232
|
+
return { state: existing, alreadyLoaded: true };
|
|
233
|
+
}
|
|
234
|
+
const state = {
|
|
235
|
+
loading: Promise.resolve(null),
|
|
236
|
+
pendingUses: 0,
|
|
237
|
+
activeUses: 0,
|
|
238
|
+
onIdle: [],
|
|
239
|
+
progressListeners: new Set,
|
|
240
|
+
lastProgress: null
|
|
241
|
+
};
|
|
242
|
+
const pendingDisposal = disposals.get(model) ?? null;
|
|
243
|
+
state.loading = Promise.resolve().then(async () => {
|
|
244
|
+
if (pendingDisposal !== null) {
|
|
245
|
+
await pendingDisposal;
|
|
246
|
+
}
|
|
247
|
+
return withRemotionModelHost(async ({
|
|
248
|
+
AutoModelForImageSegmentation,
|
|
249
|
+
AutoProcessor,
|
|
250
|
+
BackgroundRemovalPipeline
|
|
251
|
+
}) => {
|
|
252
|
+
const hostedModelId = getHostedVideoMattingModelId(model);
|
|
253
|
+
const totalBytes = modelInfo.webGpuDownloadSize;
|
|
254
|
+
const loadedByFile = new Map;
|
|
255
|
+
let lastProgress = 0;
|
|
256
|
+
let lastLoadedBytes = 0;
|
|
257
|
+
notifyProgress(state, {
|
|
258
|
+
status: "loading",
|
|
259
|
+
file: null,
|
|
260
|
+
progress: 0,
|
|
261
|
+
loadedBytes: 0,
|
|
262
|
+
totalBytes
|
|
263
|
+
});
|
|
264
|
+
const pretrainedOptions = {
|
|
265
|
+
device: "webgpu",
|
|
266
|
+
dtype: modelInfo.dtype,
|
|
267
|
+
progress_callback: (event) => {
|
|
268
|
+
const record = event;
|
|
269
|
+
if (record.status === "progress" && typeof record.file === "string" && typeof record.loaded === "number" && Number.isFinite(record.loaded)) {
|
|
270
|
+
loadedByFile.set(record.file, Math.max(loadedByFile.get(record.file) ?? 0, record.loaded));
|
|
271
|
+
const loadedBytes = [...loadedByFile.values()].reduce((sum, loaded) => sum + loaded, 0);
|
|
272
|
+
lastProgress = Math.max(lastProgress, Math.min(loadedBytes / totalBytes, 0.99));
|
|
273
|
+
lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
|
|
274
|
+
notifyProgress(state, {
|
|
275
|
+
status: "loading",
|
|
276
|
+
file: null,
|
|
277
|
+
progress: lastProgress,
|
|
278
|
+
loadedBytes: Math.min(lastLoadedBytes, totalBytes),
|
|
279
|
+
totalBytes
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
const transformerProcessor = await AutoProcessor.from_pretrained(hostedModelId, pretrainedOptions);
|
|
285
|
+
const transformerModel = await AutoModelForImageSegmentation.from_pretrained(hostedModelId, pretrainedOptions);
|
|
286
|
+
let transformerPipeline = null;
|
|
287
|
+
let returnedPipeline = false;
|
|
288
|
+
try {
|
|
289
|
+
transformerPipeline = new BackgroundRemovalPipeline({
|
|
290
|
+
task: "background-removal",
|
|
291
|
+
model: transformerModel,
|
|
292
|
+
processor: transformerProcessor
|
|
293
|
+
});
|
|
294
|
+
const loadedTransformerPipeline = transformerPipeline;
|
|
295
|
+
notifyProgress(state, {
|
|
296
|
+
status: "ready",
|
|
297
|
+
file: null,
|
|
298
|
+
progress: 1,
|
|
299
|
+
loadedBytes: totalBytes,
|
|
300
|
+
totalBytes
|
|
301
|
+
});
|
|
302
|
+
const loadedPipeline = {
|
|
303
|
+
run: async (image) => {
|
|
304
|
+
const result = await loadedTransformerPipeline(image);
|
|
305
|
+
if (result.channels !== 4) {
|
|
306
|
+
throw new Error(`The video matting model "${model}" returned ${result.channels} channels instead of RGBA.`);
|
|
307
|
+
}
|
|
308
|
+
const data = result.data instanceof Uint8ClampedArray && result.data.buffer instanceof ArrayBuffer ? result.data : new Uint8ClampedArray(result.data);
|
|
309
|
+
return {
|
|
310
|
+
data,
|
|
311
|
+
width: result.width,
|
|
312
|
+
height: result.height,
|
|
313
|
+
channels: 4
|
|
314
|
+
};
|
|
315
|
+
},
|
|
316
|
+
dispose: () => loadedTransformerPipeline.dispose()
|
|
317
|
+
};
|
|
318
|
+
returnedPipeline = true;
|
|
319
|
+
return loadedPipeline;
|
|
320
|
+
} finally {
|
|
321
|
+
if (!returnedPipeline) {
|
|
322
|
+
if (transformerPipeline === null) {
|
|
323
|
+
await transformerModel.dispose();
|
|
324
|
+
} else {
|
|
325
|
+
await transformerPipeline.dispose();
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
});
|
|
331
|
+
pipelines.set(model, state);
|
|
332
|
+
state.loading.catch(() => {
|
|
333
|
+
if (pipelines.get(model) === state) {
|
|
334
|
+
pipelines.delete(model);
|
|
335
|
+
}
|
|
336
|
+
});
|
|
337
|
+
return { state, alreadyLoaded: false };
|
|
338
|
+
};
|
|
339
|
+
var subscribeToProgress = (state, onProgress) => {
|
|
340
|
+
if (!onProgress) {
|
|
341
|
+
return () => {
|
|
342
|
+
return;
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
const listener = (progress) => {
|
|
346
|
+
onProgress(progress);
|
|
347
|
+
};
|
|
348
|
+
state.progressListeners.add(listener);
|
|
349
|
+
if (state.lastProgress) {
|
|
350
|
+
try {
|
|
351
|
+
listener(state.lastProgress);
|
|
352
|
+
} catch {}
|
|
353
|
+
}
|
|
354
|
+
return () => {
|
|
355
|
+
state.progressListeners.delete(listener);
|
|
356
|
+
};
|
|
357
|
+
};
|
|
358
|
+
var loadVideoMattingModel = async ({
|
|
359
|
+
model,
|
|
360
|
+
onProgress
|
|
361
|
+
}) => {
|
|
362
|
+
const { state, alreadyLoaded } = getOrCreateVideoMattingPipeline({ model });
|
|
363
|
+
const unsubscribe = subscribeToProgress(state, onProgress);
|
|
364
|
+
try {
|
|
365
|
+
await state.loading;
|
|
366
|
+
} finally {
|
|
367
|
+
unsubscribe();
|
|
368
|
+
}
|
|
369
|
+
return { alreadyLoaded };
|
|
370
|
+
};
|
|
371
|
+
var withLoadedVideoMattingPipeline = async ({
|
|
372
|
+
model,
|
|
373
|
+
onProgress,
|
|
374
|
+
signal,
|
|
375
|
+
run
|
|
376
|
+
}) => {
|
|
377
|
+
const { state } = getOrCreateVideoMattingPipeline({ model });
|
|
378
|
+
const unsubscribe = subscribeToProgress(state, onProgress);
|
|
379
|
+
state.pendingUses++;
|
|
380
|
+
let isActive = false;
|
|
381
|
+
try {
|
|
382
|
+
const loaded = await waitForLoadingOrAbort({
|
|
383
|
+
loading: state.loading,
|
|
384
|
+
signal
|
|
385
|
+
});
|
|
386
|
+
state.pendingUses--;
|
|
387
|
+
state.activeUses++;
|
|
388
|
+
isActive = true;
|
|
389
|
+
return await run(loaded.run);
|
|
390
|
+
} finally {
|
|
391
|
+
unsubscribe();
|
|
392
|
+
if (isActive) {
|
|
393
|
+
state.activeUses--;
|
|
394
|
+
} else {
|
|
395
|
+
state.pendingUses--;
|
|
396
|
+
}
|
|
397
|
+
notifyIfIdle(state);
|
|
398
|
+
}
|
|
399
|
+
};
|
|
400
|
+
var disposeVideoMattingModel = async ({
|
|
401
|
+
model
|
|
402
|
+
} = {}) => {
|
|
403
|
+
if (model !== undefined) {
|
|
404
|
+
getVideoMattingModelInfo(model);
|
|
405
|
+
}
|
|
406
|
+
const matching = [...pipelines.entries()].filter(([loadedModel]) => {
|
|
407
|
+
return model === undefined || loadedModel === model;
|
|
408
|
+
});
|
|
409
|
+
for (const [key, state] of matching) {
|
|
410
|
+
if (pipelines.get(key) === state) {
|
|
411
|
+
pipelines.delete(key);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
const pending = new Map;
|
|
415
|
+
for (const [key, disposal] of disposals) {
|
|
416
|
+
if (model === undefined || model === key) {
|
|
417
|
+
pending.set(key, disposal);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
for (const [key, state] of matching) {
|
|
421
|
+
const previousDisposal = disposals.get(key) ?? Promise.resolve();
|
|
422
|
+
const disposal = previousDisposal.then(async () => {
|
|
423
|
+
const loadedPipeline = await state.loading;
|
|
424
|
+
if (state.pendingUses > 0 || state.activeUses > 0) {
|
|
425
|
+
await new Promise((resolve) => {
|
|
426
|
+
state.onIdle.push(resolve);
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
await loadedPipeline.dispose();
|
|
430
|
+
});
|
|
431
|
+
const disposalBarrier = disposal.then(() => {
|
|
432
|
+
return;
|
|
433
|
+
}, () => {
|
|
434
|
+
return;
|
|
435
|
+
});
|
|
436
|
+
disposals.set(key, disposalBarrier);
|
|
437
|
+
disposalBarrier.then(() => {
|
|
438
|
+
if (disposals.get(key) === disposalBarrier) {
|
|
439
|
+
disposals.delete(key);
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
pending.set(key, disposal);
|
|
443
|
+
}
|
|
444
|
+
await Promise.all(pending.values());
|
|
445
|
+
};
|
|
446
|
+
// src/is-video-matting-model-cached.ts
|
|
447
|
+
var isVideoMattingModelCached = ({
|
|
448
|
+
model
|
|
449
|
+
}) => {
|
|
450
|
+
const modelInfo = getVideoMattingModelInfo(model);
|
|
451
|
+
return withRemotionModelHost(({ ModelRegistry }) => {
|
|
452
|
+
return ModelRegistry.is_pipeline_cached("background-removal", getHostedVideoMattingModelId(model), {
|
|
453
|
+
device: "webgpu",
|
|
454
|
+
dtype: modelInfo.dtype
|
|
455
|
+
});
|
|
456
|
+
});
|
|
457
|
+
};
|
|
458
|
+
// src/remove-video-matting-model.ts
|
|
459
|
+
var removeVideoMattingModel = async ({
|
|
460
|
+
model
|
|
461
|
+
}) => {
|
|
462
|
+
const modelInfo = getVideoMattingModelInfo(model);
|
|
463
|
+
await disposeVideoMattingModel({ model });
|
|
464
|
+
await withRemotionModelHost(({ ModelRegistry }) => {
|
|
465
|
+
return ModelRegistry.clear_pipeline_cache("background-removal", getHostedVideoMattingModelId(model), {
|
|
466
|
+
device: "webgpu",
|
|
467
|
+
dtype: modelInfo.dtype
|
|
468
|
+
});
|
|
469
|
+
});
|
|
470
|
+
};
|
|
471
|
+
// src/separate-video-layers.ts
|
|
472
|
+
import {
|
|
473
|
+
ALL_FORMATS,
|
|
474
|
+
BlobSource,
|
|
475
|
+
CanvasSink,
|
|
476
|
+
CanvasSource,
|
|
477
|
+
Input,
|
|
478
|
+
UrlSource,
|
|
479
|
+
WebMOutputFormat,
|
|
480
|
+
canEncodeVideo
|
|
481
|
+
} from "mediabunny";
|
|
482
|
+
|
|
483
|
+
// src/create-video-layer-output.ts
|
|
484
|
+
import {
|
|
485
|
+
BufferTarget,
|
|
486
|
+
Output,
|
|
487
|
+
StreamTarget
|
|
488
|
+
} from "mediabunny";
|
|
489
|
+
// package.json
|
|
490
|
+
var version = "4.0.523";
|
|
491
|
+
|
|
492
|
+
// src/web-fs-target.ts
|
|
493
|
+
var filePrefix = "__remotion_video_matting:";
|
|
494
|
+
var sessionId = null;
|
|
495
|
+
var getSessionPrefix = () => {
|
|
496
|
+
if (sessionId === null) {
|
|
497
|
+
sessionId = crypto.randomUUID();
|
|
498
|
+
}
|
|
499
|
+
return `${filePrefix}${sessionId}:`;
|
|
500
|
+
};
|
|
501
|
+
var canUseWebFsWriter = async () => {
|
|
502
|
+
if (typeof navigator === "undefined" || !("storage" in navigator)) {
|
|
503
|
+
return false;
|
|
504
|
+
}
|
|
505
|
+
if (!("getDirectory" in navigator.storage)) {
|
|
506
|
+
return false;
|
|
507
|
+
}
|
|
508
|
+
const probeName = `${filePrefix}probe:${crypto.randomUUID()}`;
|
|
509
|
+
try {
|
|
510
|
+
const directoryHandle = await navigator.storage.getDirectory();
|
|
511
|
+
const fileHandle = await directoryHandle.getFileHandle(probeName, {
|
|
512
|
+
create: true
|
|
513
|
+
});
|
|
514
|
+
const writable = await fileHandle.createWritable();
|
|
515
|
+
await writable.close();
|
|
516
|
+
await directoryHandle.removeEntry(probeName);
|
|
517
|
+
return true;
|
|
518
|
+
} catch {
|
|
519
|
+
try {
|
|
520
|
+
const directoryHandle = await navigator.storage.getDirectory();
|
|
521
|
+
await directoryHandle.removeEntry(probeName);
|
|
522
|
+
} catch {}
|
|
523
|
+
return false;
|
|
524
|
+
}
|
|
525
|
+
};
|
|
526
|
+
var createWebFsVideoLayerTarget = async () => {
|
|
527
|
+
const directoryHandle = await navigator.storage.getDirectory();
|
|
528
|
+
const filename = `${getSessionPrefix()}${crypto.randomUUID()}`;
|
|
529
|
+
const fileHandle = await directoryHandle.getFileHandle(filename, {
|
|
530
|
+
create: true
|
|
531
|
+
});
|
|
532
|
+
let writable;
|
|
533
|
+
try {
|
|
534
|
+
writable = await fileHandle.createWritable();
|
|
535
|
+
} catch (error) {
|
|
536
|
+
try {
|
|
537
|
+
await directoryHandle.removeEntry(filename);
|
|
538
|
+
} catch {}
|
|
539
|
+
throw error;
|
|
540
|
+
}
|
|
541
|
+
let closed = false;
|
|
542
|
+
let removed = false;
|
|
543
|
+
let removalPromise = null;
|
|
544
|
+
const close = async () => {
|
|
545
|
+
if (closed) {
|
|
546
|
+
return;
|
|
547
|
+
}
|
|
548
|
+
closed = true;
|
|
549
|
+
await writable.close();
|
|
550
|
+
};
|
|
551
|
+
const abort = async (reason) => {
|
|
552
|
+
if (closed) {
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
closed = true;
|
|
556
|
+
await writable.abort(reason);
|
|
557
|
+
};
|
|
558
|
+
const stream = new WritableStream({
|
|
559
|
+
async write(chunk) {
|
|
560
|
+
await writable.seek(chunk.position);
|
|
561
|
+
await writable.write(chunk);
|
|
562
|
+
},
|
|
563
|
+
close,
|
|
564
|
+
abort
|
|
565
|
+
});
|
|
566
|
+
const getBlob = async () => {
|
|
567
|
+
const currentFileHandle = await directoryHandle.getFileHandle(filename);
|
|
568
|
+
return currentFileHandle.getFile();
|
|
569
|
+
};
|
|
570
|
+
const remove = () => {
|
|
571
|
+
if (removed) {
|
|
572
|
+
return Promise.resolve();
|
|
573
|
+
}
|
|
574
|
+
if (removalPromise === null) {
|
|
575
|
+
removalPromise = (async () => {
|
|
576
|
+
try {
|
|
577
|
+
await abort(new Error("Video layer output was discarded"));
|
|
578
|
+
} catch {}
|
|
579
|
+
try {
|
|
580
|
+
await directoryHandle.removeEntry(filename);
|
|
581
|
+
} catch (error) {
|
|
582
|
+
if (typeof error !== "object" || error === null || !("name" in error) || error.name !== "NotFoundError") {
|
|
583
|
+
throw error;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
removed = true;
|
|
587
|
+
})().catch((error) => {
|
|
588
|
+
removalPromise = null;
|
|
589
|
+
throw error;
|
|
590
|
+
});
|
|
591
|
+
}
|
|
592
|
+
return removalPromise;
|
|
593
|
+
};
|
|
594
|
+
return { stream, getBlob, remove };
|
|
595
|
+
};
|
|
596
|
+
|
|
597
|
+
// src/create-video-layer-output.ts
|
|
598
|
+
var createVideoLayerOutput = async ({
|
|
599
|
+
format,
|
|
600
|
+
options
|
|
601
|
+
}) => {
|
|
602
|
+
if (options?.outputTarget !== undefined && options.outputWritable !== undefined) {
|
|
603
|
+
throw new Error("outputTarget and outputWritable cannot both be specified for a video layer");
|
|
604
|
+
}
|
|
605
|
+
let webFsTarget = null;
|
|
606
|
+
let target;
|
|
607
|
+
let outputMode;
|
|
608
|
+
const outputWritable = options?.outputWritable;
|
|
609
|
+
let outputWritableWriter = null;
|
|
610
|
+
let outputWritableClosePromise = null;
|
|
611
|
+
let outputWritableAbortPromise = null;
|
|
612
|
+
let outputWritableWasClosed = false;
|
|
613
|
+
const getOutputWritableWriter = () => {
|
|
614
|
+
if (outputWritable === undefined) {
|
|
615
|
+
throw new Error("Expected a caller-provided output WritableStream");
|
|
616
|
+
}
|
|
617
|
+
outputWritableWriter ??= outputWritable.getWriter();
|
|
618
|
+
return outputWritableWriter;
|
|
619
|
+
};
|
|
620
|
+
const releaseOutputWritableWriter = () => {
|
|
621
|
+
if (outputWritableWriter === null) {
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
outputWritableWriter.releaseLock();
|
|
625
|
+
outputWritableWriter = null;
|
|
626
|
+
};
|
|
627
|
+
const closeOutputWritable = () => {
|
|
628
|
+
if (outputWritable === undefined || outputWritableWasClosed) {
|
|
629
|
+
return Promise.resolve();
|
|
630
|
+
}
|
|
631
|
+
if (outputWritableAbortPromise !== null) {
|
|
632
|
+
return outputWritableAbortPromise;
|
|
633
|
+
}
|
|
634
|
+
if (outputWritableClosePromise === null) {
|
|
635
|
+
const writer = getOutputWritableWriter();
|
|
636
|
+
outputWritableClosePromise = (async () => {
|
|
637
|
+
try {
|
|
638
|
+
await writer.close();
|
|
639
|
+
outputWritableWasClosed = true;
|
|
640
|
+
} finally {
|
|
641
|
+
releaseOutputWritableWriter();
|
|
642
|
+
}
|
|
643
|
+
})();
|
|
644
|
+
}
|
|
645
|
+
return outputWritableClosePromise;
|
|
646
|
+
};
|
|
647
|
+
const abortOutputWritable = (reason) => {
|
|
648
|
+
if (outputWritable === undefined || outputWritableWasClosed) {
|
|
649
|
+
return Promise.resolve();
|
|
650
|
+
}
|
|
651
|
+
if (outputWritableAbortPromise === null) {
|
|
652
|
+
const writer = getOutputWritableWriter();
|
|
653
|
+
outputWritableAbortPromise = (async () => {
|
|
654
|
+
try {
|
|
655
|
+
await writer.abort(reason);
|
|
656
|
+
} finally {
|
|
657
|
+
releaseOutputWritableWriter();
|
|
658
|
+
}
|
|
659
|
+
})();
|
|
660
|
+
}
|
|
661
|
+
return outputWritableAbortPromise;
|
|
662
|
+
};
|
|
663
|
+
if (outputWritable !== undefined) {
|
|
664
|
+
target = new StreamTarget(new WritableStream({
|
|
665
|
+
write: (chunk) => getOutputWritableWriter().write(chunk),
|
|
666
|
+
close: () => Promise.resolve(),
|
|
667
|
+
abort: (reason) => abortOutputWritable(reason)
|
|
668
|
+
}));
|
|
669
|
+
outputMode = "writable";
|
|
670
|
+
} else {
|
|
671
|
+
const outputTarget = options?.outputTarget ?? (await canUseWebFsWriter() ? "web-fs" : "arraybuffer");
|
|
672
|
+
if (outputTarget === "web-fs") {
|
|
673
|
+
webFsTarget = await createWebFsVideoLayerTarget();
|
|
674
|
+
target = new StreamTarget(webFsTarget.stream);
|
|
675
|
+
outputMode = "web-fs";
|
|
676
|
+
} else {
|
|
677
|
+
target = new BufferTarget;
|
|
678
|
+
outputMode = "arraybuffer";
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
const output = new Output({ format, target });
|
|
682
|
+
output.setMetadataTags({
|
|
683
|
+
comment: `Separated with @remotion/video-matting ${version}`
|
|
684
|
+
});
|
|
685
|
+
let finalizationPromise = null;
|
|
686
|
+
let cancellationPromise = null;
|
|
687
|
+
let finalized = false;
|
|
688
|
+
let cancellationRequested = false;
|
|
689
|
+
const discardWithReason = (reason) => {
|
|
690
|
+
if (webFsTarget !== null) {
|
|
691
|
+
return webFsTarget.remove();
|
|
692
|
+
}
|
|
693
|
+
return abortOutputWritable(reason);
|
|
694
|
+
};
|
|
695
|
+
const discard = () => discardWithReason(new Error("Video layer output was discarded"));
|
|
696
|
+
const cancelMediabunnyOutput = async () => {
|
|
697
|
+
if (output.state === "canceled" || output.state === "finalized") {
|
|
698
|
+
return;
|
|
699
|
+
}
|
|
700
|
+
await output.cancel();
|
|
701
|
+
};
|
|
702
|
+
const cancel = () => {
|
|
703
|
+
if (finalized) {
|
|
704
|
+
return Promise.resolve();
|
|
705
|
+
}
|
|
706
|
+
if (cancellationPromise !== null) {
|
|
707
|
+
return cancellationPromise;
|
|
708
|
+
}
|
|
709
|
+
cancellationRequested = true;
|
|
710
|
+
cancellationPromise = (async () => {
|
|
711
|
+
const discardPromise = discardWithReason(new Error("Video layer output was canceled"));
|
|
712
|
+
let cancellationError = null;
|
|
713
|
+
try {
|
|
714
|
+
await cancelMediabunnyOutput();
|
|
715
|
+
} catch (error) {
|
|
716
|
+
cancellationError = error;
|
|
717
|
+
}
|
|
718
|
+
if (finalizationPromise !== null) {
|
|
719
|
+
try {
|
|
720
|
+
await finalizationPromise;
|
|
721
|
+
} catch {}
|
|
722
|
+
}
|
|
723
|
+
try {
|
|
724
|
+
await discardPromise;
|
|
725
|
+
} catch (error) {
|
|
726
|
+
cancellationError ??= error;
|
|
727
|
+
}
|
|
728
|
+
if (cancellationError !== null) {
|
|
729
|
+
throw cancellationError;
|
|
730
|
+
}
|
|
731
|
+
})();
|
|
732
|
+
return cancellationPromise;
|
|
733
|
+
};
|
|
734
|
+
const finalize = () => {
|
|
735
|
+
if (cancellationRequested) {
|
|
736
|
+
return Promise.reject(new Error("Video layer output was canceled"));
|
|
737
|
+
}
|
|
738
|
+
if (finalizationPromise !== null) {
|
|
739
|
+
return finalizationPromise;
|
|
740
|
+
}
|
|
741
|
+
finalizationPromise = (async () => {
|
|
742
|
+
try {
|
|
743
|
+
await output.finalize();
|
|
744
|
+
if (cancellationRequested) {
|
|
745
|
+
throw new Error("Video layer output was canceled");
|
|
746
|
+
}
|
|
747
|
+
const mimeType = await output.getMimeType();
|
|
748
|
+
if (cancellationRequested) {
|
|
749
|
+
throw new Error("Video layer output was canceled");
|
|
750
|
+
}
|
|
751
|
+
await closeOutputWritable();
|
|
752
|
+
if (cancellationRequested) {
|
|
753
|
+
throw new Error("Video layer output was canceled");
|
|
754
|
+
}
|
|
755
|
+
finalized = true;
|
|
756
|
+
if (outputMode === "writable") {
|
|
757
|
+
return {
|
|
758
|
+
dispose: () => Promise.resolve(),
|
|
759
|
+
getBlob: () => Promise.reject(new Error("getBlob() is unavailable when outputWritable is used"))
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
if (outputMode === "web-fs") {
|
|
763
|
+
if (webFsTarget === null) {
|
|
764
|
+
throw new Error("Expected an OPFS-backed output target");
|
|
765
|
+
}
|
|
766
|
+
let blobPromise = null;
|
|
767
|
+
return {
|
|
768
|
+
dispose: discard,
|
|
769
|
+
getBlob: () => {
|
|
770
|
+
blobPromise ??= (async () => {
|
|
771
|
+
const file = await webFsTarget.getBlob();
|
|
772
|
+
const buffer = await file.arrayBuffer();
|
|
773
|
+
return new Blob([buffer], { type: mimeType });
|
|
774
|
+
})();
|
|
775
|
+
return blobPromise;
|
|
776
|
+
}
|
|
777
|
+
};
|
|
778
|
+
}
|
|
779
|
+
if (!(target instanceof BufferTarget)) {
|
|
780
|
+
throw new Error("Expected an in-memory output target");
|
|
781
|
+
}
|
|
782
|
+
return {
|
|
783
|
+
dispose: () => Promise.resolve(),
|
|
784
|
+
getBlob: () => {
|
|
785
|
+
if (target.buffer === null) {
|
|
786
|
+
return Promise.reject(new Error("The resulting buffer is empty"));
|
|
787
|
+
}
|
|
788
|
+
return Promise.resolve(new Blob([target.buffer], { type: mimeType }));
|
|
789
|
+
}
|
|
790
|
+
};
|
|
791
|
+
} catch (error) {
|
|
792
|
+
await Promise.allSettled([cancelMediabunnyOutput(), discard()]);
|
|
793
|
+
throw error;
|
|
794
|
+
}
|
|
795
|
+
})();
|
|
796
|
+
return finalizationPromise;
|
|
797
|
+
};
|
|
798
|
+
return { output, finalize, cancel, discard };
|
|
799
|
+
};
|
|
800
|
+
|
|
801
|
+
// src/prepare-audio.ts
|
|
802
|
+
import {
|
|
803
|
+
AudioSampleSink,
|
|
804
|
+
AudioSampleSource,
|
|
805
|
+
canEncodeAudio,
|
|
806
|
+
EncodedAudioPacketSource,
|
|
807
|
+
EncodedPacketSink,
|
|
808
|
+
Quality
|
|
809
|
+
} from "mediabunny";
|
|
810
|
+
var prepareAudio = async ({
|
|
811
|
+
input,
|
|
812
|
+
baseOutput,
|
|
813
|
+
foregroundOutput,
|
|
814
|
+
destination,
|
|
815
|
+
videoStartTimestamp,
|
|
816
|
+
videoEndTimestamp,
|
|
817
|
+
audioQuality,
|
|
818
|
+
forceTranscode
|
|
819
|
+
}) => {
|
|
820
|
+
if (!Number.isFinite(videoStartTimestamp)) {
|
|
821
|
+
throw new TypeError("videoStartTimestamp must be a finite number.");
|
|
822
|
+
}
|
|
823
|
+
if (!Number.isFinite(videoEndTimestamp)) {
|
|
824
|
+
throw new TypeError("videoEndTimestamp must be a finite number.");
|
|
825
|
+
}
|
|
826
|
+
if (videoEndTimestamp < videoStartTimestamp) {
|
|
827
|
+
throw new RangeError("videoEndTimestamp must be greater than or equal to videoStartTimestamp.");
|
|
828
|
+
}
|
|
829
|
+
if (typeof forceTranscode !== "boolean") {
|
|
830
|
+
throw new TypeError("forceTranscode must be a boolean.");
|
|
831
|
+
}
|
|
832
|
+
if (destination === "none") {
|
|
833
|
+
return {
|
|
834
|
+
prime: () => Promise.resolve(),
|
|
835
|
+
writeAudioUntil: () => Promise.resolve(),
|
|
836
|
+
finishAudio: () => Promise.resolve(),
|
|
837
|
+
cancel: () => Promise.resolve()
|
|
838
|
+
};
|
|
839
|
+
}
|
|
840
|
+
const audioTrack = await input.getPrimaryAudioTrack();
|
|
841
|
+
if (audioTrack === null) {
|
|
842
|
+
return {
|
|
843
|
+
prime: () => Promise.resolve(),
|
|
844
|
+
writeAudioUntil: () => Promise.resolve(),
|
|
845
|
+
finishAudio: () => Promise.resolve(),
|
|
846
|
+
cancel: () => Promise.resolve()
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
const [firstAudioTimestamp, audioEndTimestamp] = await Promise.all([
|
|
850
|
+
audioTrack.getFirstTimestamp(),
|
|
851
|
+
audioTrack.computeDuration()
|
|
852
|
+
]);
|
|
853
|
+
if (firstAudioTimestamp >= videoEndTimestamp || audioEndTimestamp <= videoStartTimestamp) {
|
|
854
|
+
return {
|
|
855
|
+
prime: () => Promise.resolve(),
|
|
856
|
+
writeAudioUntil: () => Promise.resolve(),
|
|
857
|
+
finishAudio: () => Promise.resolve(),
|
|
858
|
+
cancel: () => Promise.resolve()
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
const outputs = destination === "both" ? [baseOutput, foregroundOutput] : destination === "base" ? [baseOutput] : [foregroundOutput];
|
|
862
|
+
const sourceCodec = await audioTrack.getCodec();
|
|
863
|
+
const canCopyPackets = !forceTranscode && sourceCodec === "opus" && firstAudioTimestamp >= videoStartTimestamp;
|
|
864
|
+
let lastWriteTimestamp = -Infinity;
|
|
865
|
+
let primed = false;
|
|
866
|
+
let finished = false;
|
|
867
|
+
let canceled = false;
|
|
868
|
+
if (canCopyPackets) {
|
|
869
|
+
const decoderConfig = await audioTrack.getDecoderConfig();
|
|
870
|
+
const packetSources = outputs.map((output) => {
|
|
871
|
+
const source = new EncodedAudioPacketSource("opus");
|
|
872
|
+
output.addAudioTrack(source, {
|
|
873
|
+
decoderConfig: decoderConfig ?? undefined
|
|
874
|
+
});
|
|
875
|
+
return source;
|
|
876
|
+
});
|
|
877
|
+
const packetIterator = new EncodedPacketSink(audioTrack).packets();
|
|
878
|
+
let pendingPacket;
|
|
879
|
+
let packetIteratorDone = false;
|
|
880
|
+
let packetSourcesClosed = false;
|
|
881
|
+
const closePacketSources = () => {
|
|
882
|
+
if (packetSourcesClosed || outputs.some((output) => output.state !== "started")) {
|
|
883
|
+
return;
|
|
884
|
+
}
|
|
885
|
+
packetSourcesClosed = true;
|
|
886
|
+
for (const source of packetSources) {
|
|
887
|
+
source.close();
|
|
888
|
+
}
|
|
889
|
+
};
|
|
890
|
+
const stopPacketIterator = async () => {
|
|
891
|
+
if (packetIteratorDone) {
|
|
892
|
+
return;
|
|
893
|
+
}
|
|
894
|
+
packetIteratorDone = true;
|
|
895
|
+
try {
|
|
896
|
+
await packetIterator.return(undefined);
|
|
897
|
+
} catch {}
|
|
898
|
+
};
|
|
899
|
+
const writePacketsUntil = async ({
|
|
900
|
+
outputTimestamp,
|
|
901
|
+
writeAtLeastOne
|
|
902
|
+
}) => {
|
|
903
|
+
let packetsWritten = 0;
|
|
904
|
+
while (!packetIteratorDone) {
|
|
905
|
+
if (pendingPacket === undefined) {
|
|
906
|
+
const next = await packetIterator.next();
|
|
907
|
+
if (next.done) {
|
|
908
|
+
packetIteratorDone = true;
|
|
909
|
+
closePacketSources();
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
912
|
+
pendingPacket = next.value;
|
|
913
|
+
}
|
|
914
|
+
if (pendingPacket.timestamp >= videoEndTimestamp) {
|
|
915
|
+
pendingPacket = undefined;
|
|
916
|
+
await stopPacketIterator();
|
|
917
|
+
closePacketSources();
|
|
918
|
+
return;
|
|
919
|
+
}
|
|
920
|
+
const rebasedTimestamp = pendingPacket.timestamp - videoStartTimestamp;
|
|
921
|
+
if (rebasedTimestamp > outputTimestamp && (!writeAtLeastOne || packetsWritten > 0)) {
|
|
922
|
+
return;
|
|
923
|
+
}
|
|
924
|
+
const packet = pendingPacket.clone({
|
|
925
|
+
timestamp: rebasedTimestamp
|
|
926
|
+
});
|
|
927
|
+
pendingPacket = undefined;
|
|
928
|
+
const metadata = {
|
|
929
|
+
decoderConfig: decoderConfig ?? undefined
|
|
930
|
+
};
|
|
931
|
+
await Promise.all(packetSources.map((source) => source.add(packet, metadata)));
|
|
932
|
+
packetsWritten++;
|
|
933
|
+
if (writeAtLeastOne) {
|
|
934
|
+
return;
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
};
|
|
938
|
+
return {
|
|
939
|
+
prime: async () => {
|
|
940
|
+
if (primed || finished || canceled) {
|
|
941
|
+
return;
|
|
942
|
+
}
|
|
943
|
+
primed = true;
|
|
944
|
+
await writePacketsUntil({
|
|
945
|
+
outputTimestamp: 0,
|
|
946
|
+
writeAtLeastOne: true
|
|
947
|
+
});
|
|
948
|
+
},
|
|
949
|
+
writeAudioUntil: async (outputTimestamp) => {
|
|
950
|
+
if (!Number.isFinite(outputTimestamp) || outputTimestamp < 0) {
|
|
951
|
+
throw new TypeError("outputTimestamp must be a finite, non-negative number.");
|
|
952
|
+
}
|
|
953
|
+
if (outputTimestamp < lastWriteTimestamp) {
|
|
954
|
+
throw new RangeError("writeAudioUntil() timestamps must be monotonically increasing.");
|
|
955
|
+
}
|
|
956
|
+
if (finished || canceled) {
|
|
957
|
+
return;
|
|
958
|
+
}
|
|
959
|
+
lastWriteTimestamp = outputTimestamp;
|
|
960
|
+
await writePacketsUntil({
|
|
961
|
+
outputTimestamp: Math.min(outputTimestamp, videoEndTimestamp - videoStartTimestamp),
|
|
962
|
+
writeAtLeastOne: false
|
|
963
|
+
});
|
|
964
|
+
},
|
|
965
|
+
finishAudio: async () => {
|
|
966
|
+
if (finished || canceled) {
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
finished = true;
|
|
970
|
+
try {
|
|
971
|
+
await writePacketsUntil({
|
|
972
|
+
outputTimestamp: videoEndTimestamp - videoStartTimestamp,
|
|
973
|
+
writeAtLeastOne: false
|
|
974
|
+
});
|
|
975
|
+
} finally {
|
|
976
|
+
await stopPacketIterator();
|
|
977
|
+
closePacketSources();
|
|
978
|
+
}
|
|
979
|
+
},
|
|
980
|
+
cancel: async () => {
|
|
981
|
+
if (finished || canceled) {
|
|
982
|
+
return;
|
|
983
|
+
}
|
|
984
|
+
canceled = true;
|
|
985
|
+
pendingPacket = undefined;
|
|
986
|
+
await stopPacketIterator();
|
|
987
|
+
}
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
if (!await audioTrack.canDecode()) {
|
|
991
|
+
throw new Error(`The primary audio track uses ${sourceCodec ?? "an unknown codec"}, which ` + "cannot be decoded in this browser. The audio cannot be converted to Opus.");
|
|
992
|
+
}
|
|
993
|
+
const originalNumberOfChannels = await audioTrack.getNumberOfChannels();
|
|
994
|
+
const originalSampleRate = await audioTrack.getSampleRate();
|
|
995
|
+
const quality = audioQuality ?? new Quality("medium");
|
|
996
|
+
let outputNumberOfChannels = originalNumberOfChannels;
|
|
997
|
+
let outputSampleRate = originalSampleRate;
|
|
998
|
+
if (!await canEncodeAudio("opus", {
|
|
999
|
+
numberOfChannels: outputNumberOfChannels,
|
|
1000
|
+
sampleRate: outputSampleRate,
|
|
1001
|
+
quality
|
|
1002
|
+
})) {
|
|
1003
|
+
outputNumberOfChannels = Math.min(originalNumberOfChannels, 2);
|
|
1004
|
+
outputSampleRate = 48000;
|
|
1005
|
+
if (!await canEncodeAudio("opus", {
|
|
1006
|
+
numberOfChannels: outputNumberOfChannels,
|
|
1007
|
+
sampleRate: outputSampleRate,
|
|
1008
|
+
quality
|
|
1009
|
+
})) {
|
|
1010
|
+
throw new Error("The primary audio track must be transcoded to Opus, but this browser " + "does not support a compatible Opus encoder.");
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
const sampleSources = outputs.map((output) => {
|
|
1014
|
+
const source = new AudioSampleSource({
|
|
1015
|
+
codec: "opus",
|
|
1016
|
+
quality,
|
|
1017
|
+
transform: {
|
|
1018
|
+
numberOfChannels: outputNumberOfChannels === originalNumberOfChannels ? undefined : outputNumberOfChannels,
|
|
1019
|
+
sampleRate: outputSampleRate === originalSampleRate ? undefined : outputSampleRate
|
|
1020
|
+
}
|
|
1021
|
+
});
|
|
1022
|
+
output.addAudioTrack(source);
|
|
1023
|
+
return source;
|
|
1024
|
+
});
|
|
1025
|
+
const sampleIterator = new AudioSampleSink(audioTrack).samples(videoStartTimestamp, videoEndTimestamp);
|
|
1026
|
+
let pendingSample;
|
|
1027
|
+
let iteratorDone = false;
|
|
1028
|
+
let sourcesClosed = false;
|
|
1029
|
+
const closeSources = () => {
|
|
1030
|
+
if (sourcesClosed || outputs.some((output) => output.state !== "started")) {
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
1033
|
+
sourcesClosed = true;
|
|
1034
|
+
for (const source of sampleSources) {
|
|
1035
|
+
source.close();
|
|
1036
|
+
}
|
|
1037
|
+
};
|
|
1038
|
+
const stopSampleIterator = async () => {
|
|
1039
|
+
if (iteratorDone) {
|
|
1040
|
+
return;
|
|
1041
|
+
}
|
|
1042
|
+
iteratorDone = true;
|
|
1043
|
+
try {
|
|
1044
|
+
await sampleIterator.return(undefined);
|
|
1045
|
+
} catch {}
|
|
1046
|
+
};
|
|
1047
|
+
const writeUntil = async ({
|
|
1048
|
+
outputTimestamp,
|
|
1049
|
+
writeAtLeastOne
|
|
1050
|
+
}) => {
|
|
1051
|
+
let samplesWritten = 0;
|
|
1052
|
+
while (!iteratorDone) {
|
|
1053
|
+
if (pendingSample === undefined) {
|
|
1054
|
+
const next = await sampleIterator.next();
|
|
1055
|
+
if (next.done) {
|
|
1056
|
+
iteratorDone = true;
|
|
1057
|
+
closeSources();
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
pendingSample = next.value;
|
|
1061
|
+
}
|
|
1062
|
+
const effectiveTimestamp = Math.max(pendingSample.timestamp, videoStartTimestamp) - videoStartTimestamp;
|
|
1063
|
+
if (effectiveTimestamp > outputTimestamp && (!writeAtLeastOne || samplesWritten > 0)) {
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
const originalSample = pendingSample;
|
|
1067
|
+
pendingSample = undefined;
|
|
1068
|
+
let startFrame = 0;
|
|
1069
|
+
let endFrame = originalSample.numberOfFrames;
|
|
1070
|
+
if (originalSample.timestamp < videoStartTimestamp) {
|
|
1071
|
+
startFrame = Math.max(0, Math.min(originalSample.numberOfFrames, Math.round((videoStartTimestamp - originalSample.timestamp) * originalSample.sampleRate)));
|
|
1072
|
+
}
|
|
1073
|
+
if (originalSample.timestamp + originalSample.duration > videoEndTimestamp) {
|
|
1074
|
+
endFrame = Math.max(0, Math.min(originalSample.numberOfFrames, Math.round((videoEndTimestamp - originalSample.timestamp) * originalSample.sampleRate)));
|
|
1075
|
+
}
|
|
1076
|
+
if (endFrame <= startFrame) {
|
|
1077
|
+
originalSample.close();
|
|
1078
|
+
continue;
|
|
1079
|
+
}
|
|
1080
|
+
const sample = startFrame === 0 && endFrame === originalSample.numberOfFrames ? originalSample : originalSample.trim(startFrame, endFrame);
|
|
1081
|
+
if (sample !== originalSample) {
|
|
1082
|
+
originalSample.close();
|
|
1083
|
+
}
|
|
1084
|
+
sample.setTimestamp(Math.max(0, sample.timestamp - videoStartTimestamp));
|
|
1085
|
+
const samples = sampleSources.map((_, index) => index === sampleSources.length - 1 ? sample : sample.clone());
|
|
1086
|
+
try {
|
|
1087
|
+
await Promise.all(sampleSources.map((source, index) => source.add(samples[index])));
|
|
1088
|
+
} finally {
|
|
1089
|
+
for (const sampleToClose of samples) {
|
|
1090
|
+
sampleToClose.close();
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
samplesWritten++;
|
|
1094
|
+
if (writeAtLeastOne) {
|
|
1095
|
+
return;
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
};
|
|
1099
|
+
return {
|
|
1100
|
+
prime: async () => {
|
|
1101
|
+
if (primed || finished || canceled) {
|
|
1102
|
+
return;
|
|
1103
|
+
}
|
|
1104
|
+
primed = true;
|
|
1105
|
+
await writeUntil({ outputTimestamp: 0, writeAtLeastOne: true });
|
|
1106
|
+
},
|
|
1107
|
+
writeAudioUntil: async (outputTimestamp) => {
|
|
1108
|
+
if (!Number.isFinite(outputTimestamp) || outputTimestamp < 0) {
|
|
1109
|
+
throw new TypeError("outputTimestamp must be a finite, non-negative number.");
|
|
1110
|
+
}
|
|
1111
|
+
if (outputTimestamp < lastWriteTimestamp) {
|
|
1112
|
+
throw new RangeError("writeAudioUntil() timestamps must be monotonically increasing.");
|
|
1113
|
+
}
|
|
1114
|
+
if (finished || canceled) {
|
|
1115
|
+
return;
|
|
1116
|
+
}
|
|
1117
|
+
lastWriteTimestamp = outputTimestamp;
|
|
1118
|
+
await writeUntil({
|
|
1119
|
+
outputTimestamp: Math.min(outputTimestamp, videoEndTimestamp - videoStartTimestamp),
|
|
1120
|
+
writeAtLeastOne: false
|
|
1121
|
+
});
|
|
1122
|
+
},
|
|
1123
|
+
finishAudio: async () => {
|
|
1124
|
+
if (finished || canceled) {
|
|
1125
|
+
return;
|
|
1126
|
+
}
|
|
1127
|
+
finished = true;
|
|
1128
|
+
try {
|
|
1129
|
+
await writeUntil({
|
|
1130
|
+
outputTimestamp: videoEndTimestamp - videoStartTimestamp,
|
|
1131
|
+
writeAtLeastOne: false
|
|
1132
|
+
});
|
|
1133
|
+
} finally {
|
|
1134
|
+
pendingSample?.close();
|
|
1135
|
+
pendingSample = undefined;
|
|
1136
|
+
await stopSampleIterator();
|
|
1137
|
+
closeSources();
|
|
1138
|
+
}
|
|
1139
|
+
},
|
|
1140
|
+
cancel: async () => {
|
|
1141
|
+
if (finished || canceled) {
|
|
1142
|
+
return;
|
|
1143
|
+
}
|
|
1144
|
+
canceled = true;
|
|
1145
|
+
try {
|
|
1146
|
+
pendingSample?.close();
|
|
1147
|
+
} catch {}
|
|
1148
|
+
pendingSample = undefined;
|
|
1149
|
+
await stopSampleIterator();
|
|
1150
|
+
}
|
|
1151
|
+
};
|
|
1152
|
+
};
|
|
1153
|
+
|
|
1154
|
+
// src/video-matting-canvas.ts
|
|
1155
|
+
var createVideoMattingCanvas = ({
|
|
1156
|
+
width,
|
|
1157
|
+
height
|
|
1158
|
+
}) => {
|
|
1159
|
+
if (typeof OffscreenCanvas !== "undefined") {
|
|
1160
|
+
return new OffscreenCanvas(width, height);
|
|
1161
|
+
}
|
|
1162
|
+
if (typeof document !== "undefined") {
|
|
1163
|
+
const canvas = document.createElement("canvas");
|
|
1164
|
+
canvas.width = width;
|
|
1165
|
+
canvas.height = height;
|
|
1166
|
+
return canvas;
|
|
1167
|
+
}
|
|
1168
|
+
throw new Error("Could not create a canvas. This API must run in a browser with OffscreenCanvas or the DOM available.");
|
|
1169
|
+
};
|
|
1170
|
+
var getVideoMattingCanvasContext = (canvas) => {
|
|
1171
|
+
const context = canvas.getContext("2d", { willReadFrequently: true });
|
|
1172
|
+
if (!context) {
|
|
1173
|
+
throw new Error("Could not create a 2D canvas context.");
|
|
1174
|
+
}
|
|
1175
|
+
return context;
|
|
1176
|
+
};
|
|
1177
|
+
var drawOpaqueBaseFrame = ({
|
|
1178
|
+
context,
|
|
1179
|
+
source,
|
|
1180
|
+
width,
|
|
1181
|
+
height
|
|
1182
|
+
}) => {
|
|
1183
|
+
context.save();
|
|
1184
|
+
context.globalCompositeOperation = "copy";
|
|
1185
|
+
context.fillStyle = "#000000";
|
|
1186
|
+
context.fillRect(0, 0, width, height);
|
|
1187
|
+
context.globalCompositeOperation = "source-over";
|
|
1188
|
+
context.drawImage(source, 0, 0, width, height);
|
|
1189
|
+
context.restore();
|
|
1190
|
+
};
|
|
1191
|
+
var drawForegroundFrame = ({
|
|
1192
|
+
context,
|
|
1193
|
+
result,
|
|
1194
|
+
source,
|
|
1195
|
+
targetWidth,
|
|
1196
|
+
targetHeight
|
|
1197
|
+
}) => {
|
|
1198
|
+
if (!Number.isInteger(result.width) || result.width <= 0 || !Number.isInteger(result.height) || result.height <= 0) {
|
|
1199
|
+
throw new Error("The video matting model returned invalid dimensions.");
|
|
1200
|
+
}
|
|
1201
|
+
const expectedLength = result.width * result.height * result.channels;
|
|
1202
|
+
if (result.data.length !== expectedLength) {
|
|
1203
|
+
throw new Error(`The video matting model returned ${result.data.length} bytes, but ${expectedLength} RGBA bytes were expected.`);
|
|
1204
|
+
}
|
|
1205
|
+
const imageData = new ImageData(result.data, result.width, result.height);
|
|
1206
|
+
context.clearRect(0, 0, targetWidth, targetHeight);
|
|
1207
|
+
if (result.width === targetWidth && result.height === targetHeight) {
|
|
1208
|
+
context.putImageData(imageData, 0, 0);
|
|
1209
|
+
} else {
|
|
1210
|
+
const intermediateCanvas = createVideoMattingCanvas({
|
|
1211
|
+
width: result.width,
|
|
1212
|
+
height: result.height
|
|
1213
|
+
});
|
|
1214
|
+
const intermediateContext = getVideoMattingCanvasContext(intermediateCanvas);
|
|
1215
|
+
intermediateContext.putImageData(imageData, 0, 0);
|
|
1216
|
+
context.drawImage(intermediateCanvas, 0, 0, targetWidth, targetHeight);
|
|
1217
|
+
}
|
|
1218
|
+
context.save();
|
|
1219
|
+
context.globalCompositeOperation = "destination-in";
|
|
1220
|
+
context.drawImage(source, 0, 0, targetWidth, targetHeight);
|
|
1221
|
+
context.restore();
|
|
1222
|
+
};
|
|
1223
|
+
|
|
1224
|
+
// src/video-matting-quality.ts
|
|
1225
|
+
import { Quality as Quality2 } from "mediabunny";
|
|
1226
|
+
var VIDEO_MATTING_QUALITIES = [
|
|
1227
|
+
"very-low",
|
|
1228
|
+
"low",
|
|
1229
|
+
"medium",
|
|
1230
|
+
"high",
|
|
1231
|
+
"very-high"
|
|
1232
|
+
];
|
|
1233
|
+
var resolveVideoMattingQuality = (value) => {
|
|
1234
|
+
if (typeof value === "number") {
|
|
1235
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
1236
|
+
throw new TypeError("A numeric bitrate must be a positive integer.");
|
|
1237
|
+
}
|
|
1238
|
+
return new Quality2({ bitrate: value });
|
|
1239
|
+
}
|
|
1240
|
+
if (!VIDEO_MATTING_QUALITIES.includes(value)) {
|
|
1241
|
+
throw new TypeError(`The bitrate quality must be one of ${VIDEO_MATTING_QUALITIES.join(", ")}.`);
|
|
1242
|
+
}
|
|
1243
|
+
return new Quality2({ quality: value, preferBitrate: true });
|
|
1244
|
+
};
|
|
1245
|
+
|
|
1246
|
+
// src/video-timing.ts
|
|
1247
|
+
var rebaseVideoTimestamp = ({
|
|
1248
|
+
timestamp,
|
|
1249
|
+
firstVideoTimestamp
|
|
1250
|
+
}) => {
|
|
1251
|
+
if (!Number.isFinite(timestamp) || !Number.isFinite(firstVideoTimestamp)) {
|
|
1252
|
+
throw new TypeError("Video timestamps must be finite numbers.");
|
|
1253
|
+
}
|
|
1254
|
+
const rebased = timestamp - firstVideoTimestamp;
|
|
1255
|
+
if (rebased < -Number.EPSILON * 16) {
|
|
1256
|
+
throw new Error("Video frames must be in presentation order.");
|
|
1257
|
+
}
|
|
1258
|
+
return Math.max(0, rebased);
|
|
1259
|
+
};
|
|
1260
|
+
var getClippedVideoFrameTiming = ({
|
|
1261
|
+
timestamp,
|
|
1262
|
+
duration,
|
|
1263
|
+
videoStartTimestamp,
|
|
1264
|
+
videoEndTimestamp
|
|
1265
|
+
}) => {
|
|
1266
|
+
if (!Number.isFinite(timestamp)) {
|
|
1267
|
+
throw new TypeError("The video frame timestamp must be finite.");
|
|
1268
|
+
}
|
|
1269
|
+
if (!Number.isFinite(duration) || duration < 0) {
|
|
1270
|
+
throw new TypeError("The video frame duration must be non-negative.");
|
|
1271
|
+
}
|
|
1272
|
+
if (!Number.isFinite(videoStartTimestamp) || !Number.isFinite(videoEndTimestamp) || videoEndTimestamp < videoStartTimestamp) {
|
|
1273
|
+
throw new TypeError("The video presentation range is invalid.");
|
|
1274
|
+
}
|
|
1275
|
+
const visibleStart = Math.max(timestamp, videoStartTimestamp);
|
|
1276
|
+
const visibleEnd = Math.min(timestamp + duration, videoEndTimestamp);
|
|
1277
|
+
if (visibleEnd <= visibleStart) {
|
|
1278
|
+
return null;
|
|
1279
|
+
}
|
|
1280
|
+
return {
|
|
1281
|
+
timestamp: visibleStart - videoStartTimestamp,
|
|
1282
|
+
duration: visibleEnd - visibleStart
|
|
1283
|
+
};
|
|
1284
|
+
};
|
|
1285
|
+
var getVideoProcessingProgress = ({
|
|
1286
|
+
timestamp,
|
|
1287
|
+
duration,
|
|
1288
|
+
firstVideoTimestamp,
|
|
1289
|
+
durationInSeconds
|
|
1290
|
+
}) => {
|
|
1291
|
+
if (!Number.isFinite(duration) || duration < 0) {
|
|
1292
|
+
throw new TypeError("Video frame durations must be non-negative.");
|
|
1293
|
+
}
|
|
1294
|
+
if (!Number.isFinite(durationInSeconds) || durationInSeconds < 0) {
|
|
1295
|
+
throw new TypeError("The video duration must be non-negative.");
|
|
1296
|
+
}
|
|
1297
|
+
const rebasedTimestamp = rebaseVideoTimestamp({
|
|
1298
|
+
timestamp,
|
|
1299
|
+
firstVideoTimestamp
|
|
1300
|
+
});
|
|
1301
|
+
const processedDurationInSeconds = Math.min(durationInSeconds, Math.max(0, rebasedTimestamp + duration));
|
|
1302
|
+
return {
|
|
1303
|
+
processedDurationInSeconds,
|
|
1304
|
+
progress: durationInSeconds === 0 ? 1 : Math.min(1, processedDurationInSeconds / durationInSeconds)
|
|
1305
|
+
};
|
|
1306
|
+
};
|
|
1307
|
+
|
|
1308
|
+
// src/separate-video-layers.ts
|
|
1309
|
+
var AUDIO_DESTINATIONS = [
|
|
1310
|
+
"base",
|
|
1311
|
+
"foreground",
|
|
1312
|
+
"both",
|
|
1313
|
+
"none"
|
|
1314
|
+
];
|
|
1315
|
+
var createAbortError = (signal) => {
|
|
1316
|
+
if (signal.reason !== undefined) {
|
|
1317
|
+
return signal.reason;
|
|
1318
|
+
}
|
|
1319
|
+
const error = new Error("Video layer separation was aborted.");
|
|
1320
|
+
error.name = "AbortError";
|
|
1321
|
+
return error;
|
|
1322
|
+
};
|
|
1323
|
+
var throwIfAborted = (signal) => {
|
|
1324
|
+
if (signal?.aborted) {
|
|
1325
|
+
throw createAbortError(signal);
|
|
1326
|
+
}
|
|
1327
|
+
};
|
|
1328
|
+
var validateLayerOutputOptions = ({
|
|
1329
|
+
layer,
|
|
1330
|
+
output
|
|
1331
|
+
}) => {
|
|
1332
|
+
if (output === undefined) {
|
|
1333
|
+
return;
|
|
1334
|
+
}
|
|
1335
|
+
if (!output || typeof output !== "object" || Array.isArray(output)) {
|
|
1336
|
+
throw new TypeError(`outputs.${layer} must be an object.`);
|
|
1337
|
+
}
|
|
1338
|
+
if (output.outputTarget !== undefined && output.outputTarget !== "arraybuffer" && output.outputTarget !== "web-fs") {
|
|
1339
|
+
throw new TypeError(`outputs.${layer}.outputTarget must be arraybuffer or web-fs.`);
|
|
1340
|
+
}
|
|
1341
|
+
if (output.outputTarget !== undefined && output.outputWritable !== undefined) {
|
|
1342
|
+
throw new TypeError(`outputs.${layer} cannot specify both outputTarget and outputWritable.`);
|
|
1343
|
+
}
|
|
1344
|
+
if (output.outputWritable !== undefined) {
|
|
1345
|
+
if (typeof WritableStream === "undefined" || !(output.outputWritable instanceof WritableStream)) {
|
|
1346
|
+
throw new TypeError(`outputs.${layer}.outputWritable must be a WritableStream.`);
|
|
1347
|
+
}
|
|
1348
|
+
if (output.outputWritable.locked) {
|
|
1349
|
+
throw new TypeError(`outputs.${layer}.outputWritable must not already be locked.`);
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
};
|
|
1353
|
+
var validateOptions = (options) => {
|
|
1354
|
+
if (!options || typeof options !== "object") {
|
|
1355
|
+
throw new TypeError("separateVideoLayers() expects an options object.");
|
|
1356
|
+
}
|
|
1357
|
+
const isBlob = typeof Blob !== "undefined" && options.src instanceof Blob;
|
|
1358
|
+
const isUrl = options.src instanceof URL;
|
|
1359
|
+
if (typeof options.src !== "string" && !isUrl && !isBlob) {
|
|
1360
|
+
throw new TypeError("src must be a string, URL, or Blob.");
|
|
1361
|
+
}
|
|
1362
|
+
if (typeof options.src === "string" && options.src.length === 0) {
|
|
1363
|
+
throw new TypeError("src must not be an empty string.");
|
|
1364
|
+
}
|
|
1365
|
+
if (options.outputs !== undefined && (!options.outputs || typeof options.outputs !== "object" || Array.isArray(options.outputs))) {
|
|
1366
|
+
throw new TypeError("outputs must be an object.");
|
|
1367
|
+
}
|
|
1368
|
+
validateLayerOutputOptions({
|
|
1369
|
+
layer: "base",
|
|
1370
|
+
output: options.outputs?.base
|
|
1371
|
+
});
|
|
1372
|
+
validateLayerOutputOptions({
|
|
1373
|
+
layer: "foreground",
|
|
1374
|
+
output: options.outputs?.foreground
|
|
1375
|
+
});
|
|
1376
|
+
if (options.outputs?.base?.outputWritable !== undefined && options.outputs.base.outputWritable === options.outputs.foreground?.outputWritable) {
|
|
1377
|
+
throw new TypeError("outputs.base and outputs.foreground must not use the same outputWritable.");
|
|
1378
|
+
}
|
|
1379
|
+
getVideoMattingModelInfo(options.model ?? "modnet");
|
|
1380
|
+
if (options.audio !== undefined && !AUDIO_DESTINATIONS.includes(options.audio)) {
|
|
1381
|
+
throw new TypeError("audio must be one of base, foreground, both, or none.");
|
|
1382
|
+
}
|
|
1383
|
+
resolveVideoMattingQuality(options.videoBitrate ?? "very-high");
|
|
1384
|
+
resolveVideoMattingQuality(options.audioBitrate ?? "medium");
|
|
1385
|
+
if (options.keyframeIntervalInSeconds !== undefined && (!Number.isFinite(options.keyframeIntervalInSeconds) || options.keyframeIntervalInSeconds <= 0)) {
|
|
1386
|
+
throw new TypeError("keyframeIntervalInSeconds must be a positive finite number.");
|
|
1387
|
+
}
|
|
1388
|
+
if (options.onProgress !== undefined && typeof options.onProgress !== "function") {
|
|
1389
|
+
throw new TypeError("onProgress must be a function.");
|
|
1390
|
+
}
|
|
1391
|
+
if (options.onModelLoadProgress !== undefined && typeof options.onModelLoadProgress !== "function") {
|
|
1392
|
+
throw new TypeError("onModelLoadProgress must be a function.");
|
|
1393
|
+
}
|
|
1394
|
+
};
|
|
1395
|
+
var makeInput = (src) => {
|
|
1396
|
+
const source = typeof src === "string" || src instanceof URL ? new UrlSource(src) : new BlobSource(src);
|
|
1397
|
+
return new Input({ formats: ALL_FORMATS, source });
|
|
1398
|
+
};
|
|
1399
|
+
var probeVideoInput = async ({
|
|
1400
|
+
input,
|
|
1401
|
+
videoQuality
|
|
1402
|
+
}) => {
|
|
1403
|
+
if (!await input.canRead()) {
|
|
1404
|
+
throw new Error("The input is not a supported media file.");
|
|
1405
|
+
}
|
|
1406
|
+
const videoTrack = await input.getPrimaryVideoTrack();
|
|
1407
|
+
if (videoTrack === null) {
|
|
1408
|
+
throw new Error("The input does not contain a video track.");
|
|
1409
|
+
}
|
|
1410
|
+
if (!await videoTrack.canDecode()) {
|
|
1411
|
+
throw new Error("The primary video track cannot be decoded.");
|
|
1412
|
+
}
|
|
1413
|
+
const [width, height] = await Promise.all([
|
|
1414
|
+
videoTrack.getDisplayWidth(),
|
|
1415
|
+
videoTrack.getDisplayHeight()
|
|
1416
|
+
]);
|
|
1417
|
+
if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) {
|
|
1418
|
+
throw new Error("The input video has invalid dimensions.");
|
|
1419
|
+
}
|
|
1420
|
+
const [canEncodeBase, canEncodeForeground] = await Promise.all([
|
|
1421
|
+
canEncodeVideo("vp9", {
|
|
1422
|
+
width,
|
|
1423
|
+
height,
|
|
1424
|
+
quality: videoQuality,
|
|
1425
|
+
alpha: "discard"
|
|
1426
|
+
}),
|
|
1427
|
+
canEncodeVideo("vp9", {
|
|
1428
|
+
width,
|
|
1429
|
+
height,
|
|
1430
|
+
quality: videoQuality,
|
|
1431
|
+
alpha: "keep"
|
|
1432
|
+
})
|
|
1433
|
+
]);
|
|
1434
|
+
if (!canEncodeBase || !canEncodeForeground) {
|
|
1435
|
+
throw new Error("This browser cannot encode the VP9 video streams required for video layer separation.");
|
|
1436
|
+
}
|
|
1437
|
+
return { videoTrack, width, height };
|
|
1438
|
+
};
|
|
1439
|
+
var separateVideoLayers = async (options) => {
|
|
1440
|
+
validateOptions(options);
|
|
1441
|
+
throwIfAborted(options.signal);
|
|
1442
|
+
const model = options.model ?? "modnet";
|
|
1443
|
+
const audio = options.audio ?? "base";
|
|
1444
|
+
const videoQuality = resolveVideoMattingQuality(options.videoBitrate ?? "very-high");
|
|
1445
|
+
const audioQuality = resolveVideoMattingQuality(options.audioBitrate ?? "medium");
|
|
1446
|
+
const keyframeIntervalInSeconds = options.keyframeIntervalInSeconds ?? 1;
|
|
1447
|
+
const input = makeInput(options.src);
|
|
1448
|
+
const onInputAbort = () => input.dispose();
|
|
1449
|
+
options.signal?.addEventListener("abort", onInputAbort, { once: true });
|
|
1450
|
+
try {
|
|
1451
|
+
const { videoTrack, width, height } = await probeVideoInput({
|
|
1452
|
+
input,
|
|
1453
|
+
videoQuality
|
|
1454
|
+
});
|
|
1455
|
+
const [inputFirstVideoTimestamp, videoEndTimestamp] = await Promise.all([
|
|
1456
|
+
videoTrack.getFirstTimestamp(),
|
|
1457
|
+
videoTrack.computeDuration()
|
|
1458
|
+
]);
|
|
1459
|
+
const videoStartTimestamp = Math.max(inputFirstVideoTimestamp, 0);
|
|
1460
|
+
if (videoEndTimestamp <= videoStartTimestamp) {
|
|
1461
|
+
throw new Error("The primary video track contains no presentable video duration.");
|
|
1462
|
+
}
|
|
1463
|
+
const durationInSeconds = videoEndTimestamp - videoStartTimestamp;
|
|
1464
|
+
throwIfAborted(options.signal);
|
|
1465
|
+
const result = await withLoadedVideoMattingPipeline({
|
|
1466
|
+
model,
|
|
1467
|
+
onProgress: options.onModelLoadProgress,
|
|
1468
|
+
signal: options.signal ?? null,
|
|
1469
|
+
run: async (pipeline) => {
|
|
1470
|
+
throwIfAborted(options.signal);
|
|
1471
|
+
let iterator = null;
|
|
1472
|
+
let baseOutput = null;
|
|
1473
|
+
let foregroundOutput = null;
|
|
1474
|
+
let baseVideoSource = null;
|
|
1475
|
+
let foregroundVideoSource = null;
|
|
1476
|
+
let audioWriter = null;
|
|
1477
|
+
let completed = false;
|
|
1478
|
+
let abortCleanupPromise = null;
|
|
1479
|
+
const cancelPendingMedia = async () => {
|
|
1480
|
+
await Promise.allSettled([
|
|
1481
|
+
baseOutput?.cancel(),
|
|
1482
|
+
foregroundOutput?.cancel()
|
|
1483
|
+
]);
|
|
1484
|
+
await Promise.allSettled([audioWriter?.cancel()]);
|
|
1485
|
+
await Promise.allSettled([
|
|
1486
|
+
baseOutput?.discard(),
|
|
1487
|
+
foregroundOutput?.discard()
|
|
1488
|
+
]);
|
|
1489
|
+
};
|
|
1490
|
+
const onAbort = () => {
|
|
1491
|
+
abortCleanupPromise = cancelPendingMedia();
|
|
1492
|
+
};
|
|
1493
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
1494
|
+
try {
|
|
1495
|
+
const canvasSink = new CanvasSink(videoTrack, {
|
|
1496
|
+
alpha: true,
|
|
1497
|
+
width,
|
|
1498
|
+
height,
|
|
1499
|
+
fit: "fill",
|
|
1500
|
+
poolSize: 1
|
|
1501
|
+
});
|
|
1502
|
+
iterator = canvasSink.canvases(videoStartTimestamp, videoEndTimestamp);
|
|
1503
|
+
let nextFrame = await iterator.next();
|
|
1504
|
+
if (nextFrame.done) {
|
|
1505
|
+
throw new Error("The primary video track contains no presentable decodable frames.");
|
|
1506
|
+
}
|
|
1507
|
+
throwIfAborted(options.signal);
|
|
1508
|
+
const baseCanvas = createVideoMattingCanvas({ width, height });
|
|
1509
|
+
const foregroundCanvas = createVideoMattingCanvas({ width, height });
|
|
1510
|
+
const baseContext = getVideoMattingCanvasContext(baseCanvas);
|
|
1511
|
+
const foregroundContext = getVideoMattingCanvasContext(foregroundCanvas);
|
|
1512
|
+
baseOutput = await createVideoLayerOutput({
|
|
1513
|
+
format: new WebMOutputFormat,
|
|
1514
|
+
options: options.outputs?.base
|
|
1515
|
+
});
|
|
1516
|
+
throwIfAborted(options.signal);
|
|
1517
|
+
foregroundOutput = await createVideoLayerOutput({
|
|
1518
|
+
format: new WebMOutputFormat,
|
|
1519
|
+
options: options.outputs?.foreground
|
|
1520
|
+
});
|
|
1521
|
+
throwIfAborted(options.signal);
|
|
1522
|
+
baseVideoSource = new CanvasSource(baseCanvas, {
|
|
1523
|
+
codec: "vp9",
|
|
1524
|
+
quality: videoQuality,
|
|
1525
|
+
keyFrameInterval: keyframeIntervalInSeconds,
|
|
1526
|
+
alpha: "discard"
|
|
1527
|
+
});
|
|
1528
|
+
foregroundVideoSource = new CanvasSource(foregroundCanvas, {
|
|
1529
|
+
codec: "vp9",
|
|
1530
|
+
quality: videoQuality,
|
|
1531
|
+
keyFrameInterval: keyframeIntervalInSeconds,
|
|
1532
|
+
alpha: "keep"
|
|
1533
|
+
});
|
|
1534
|
+
baseOutput.output.addVideoTrack(baseVideoSource);
|
|
1535
|
+
foregroundOutput.output.addVideoTrack(foregroundVideoSource);
|
|
1536
|
+
audioWriter = await prepareAudio({
|
|
1537
|
+
input,
|
|
1538
|
+
baseOutput: baseOutput.output,
|
|
1539
|
+
foregroundOutput: foregroundOutput.output,
|
|
1540
|
+
destination: audio,
|
|
1541
|
+
videoStartTimestamp,
|
|
1542
|
+
videoEndTimestamp,
|
|
1543
|
+
audioQuality,
|
|
1544
|
+
forceTranscode: options.audioBitrate !== undefined
|
|
1545
|
+
});
|
|
1546
|
+
throwIfAborted(options.signal);
|
|
1547
|
+
await Promise.all([
|
|
1548
|
+
baseOutput.output.start(),
|
|
1549
|
+
foregroundOutput.output.start()
|
|
1550
|
+
]);
|
|
1551
|
+
throwIfAborted(options.signal);
|
|
1552
|
+
await audioWriter.prime();
|
|
1553
|
+
throwIfAborted(options.signal);
|
|
1554
|
+
let processedFrames = 0;
|
|
1555
|
+
let processedDurationInSeconds = 0;
|
|
1556
|
+
options.onProgress?.({
|
|
1557
|
+
stage: "processing",
|
|
1558
|
+
progress: 0,
|
|
1559
|
+
processedFrames,
|
|
1560
|
+
processedDurationInSeconds,
|
|
1561
|
+
durationInSeconds
|
|
1562
|
+
});
|
|
1563
|
+
while (!nextFrame.done) {
|
|
1564
|
+
throwIfAborted(options.signal);
|
|
1565
|
+
const frame = nextFrame.value;
|
|
1566
|
+
const timing = getClippedVideoFrameTiming({
|
|
1567
|
+
timestamp: frame.timestamp,
|
|
1568
|
+
duration: frame.duration,
|
|
1569
|
+
videoStartTimestamp,
|
|
1570
|
+
videoEndTimestamp
|
|
1571
|
+
});
|
|
1572
|
+
if (timing === null) {
|
|
1573
|
+
nextFrame = await iterator.next();
|
|
1574
|
+
continue;
|
|
1575
|
+
}
|
|
1576
|
+
const foregroundFrame = await pipeline(frame.canvas);
|
|
1577
|
+
throwIfAborted(options.signal);
|
|
1578
|
+
drawOpaqueBaseFrame({
|
|
1579
|
+
context: baseContext,
|
|
1580
|
+
source: frame.canvas,
|
|
1581
|
+
width,
|
|
1582
|
+
height
|
|
1583
|
+
});
|
|
1584
|
+
drawForegroundFrame({
|
|
1585
|
+
context: foregroundContext,
|
|
1586
|
+
result: foregroundFrame,
|
|
1587
|
+
source: frame.canvas,
|
|
1588
|
+
targetWidth: width,
|
|
1589
|
+
targetHeight: height
|
|
1590
|
+
});
|
|
1591
|
+
await Promise.all([
|
|
1592
|
+
baseVideoSource.add(timing.timestamp, timing.duration),
|
|
1593
|
+
foregroundVideoSource.add(timing.timestamp, timing.duration),
|
|
1594
|
+
audioWriter.writeAudioUntil(timing.timestamp + timing.duration)
|
|
1595
|
+
]);
|
|
1596
|
+
processedFrames++;
|
|
1597
|
+
const processingProgress = getVideoProcessingProgress({
|
|
1598
|
+
timestamp: videoStartTimestamp + timing.timestamp,
|
|
1599
|
+
duration: timing.duration,
|
|
1600
|
+
firstVideoTimestamp: videoStartTimestamp,
|
|
1601
|
+
durationInSeconds
|
|
1602
|
+
});
|
|
1603
|
+
processedDurationInSeconds = processingProgress.processedDurationInSeconds;
|
|
1604
|
+
options.onProgress?.({
|
|
1605
|
+
stage: "processing",
|
|
1606
|
+
progress: processingProgress.progress,
|
|
1607
|
+
processedFrames,
|
|
1608
|
+
processedDurationInSeconds,
|
|
1609
|
+
durationInSeconds
|
|
1610
|
+
});
|
|
1611
|
+
nextFrame = await iterator.next();
|
|
1612
|
+
}
|
|
1613
|
+
options.onProgress?.({
|
|
1614
|
+
stage: "finalizing",
|
|
1615
|
+
progress: null,
|
|
1616
|
+
processedFrames,
|
|
1617
|
+
processedDurationInSeconds,
|
|
1618
|
+
durationInSeconds
|
|
1619
|
+
});
|
|
1620
|
+
await audioWriter.finishAudio();
|
|
1621
|
+
baseVideoSource.close();
|
|
1622
|
+
foregroundVideoSource.close();
|
|
1623
|
+
const [base, foreground] = await Promise.all([
|
|
1624
|
+
baseOutput.finalize(),
|
|
1625
|
+
foregroundOutput.finalize()
|
|
1626
|
+
]);
|
|
1627
|
+
throwIfAborted(options.signal);
|
|
1628
|
+
completed = true;
|
|
1629
|
+
return {
|
|
1630
|
+
base,
|
|
1631
|
+
foreground,
|
|
1632
|
+
model,
|
|
1633
|
+
width,
|
|
1634
|
+
height,
|
|
1635
|
+
durationInSeconds,
|
|
1636
|
+
processedFrames
|
|
1637
|
+
};
|
|
1638
|
+
} catch (error) {
|
|
1639
|
+
await cancelPendingMedia();
|
|
1640
|
+
if (options.signal?.aborted) {
|
|
1641
|
+
throw createAbortError(options.signal);
|
|
1642
|
+
}
|
|
1643
|
+
throw error;
|
|
1644
|
+
} finally {
|
|
1645
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
1646
|
+
await abortCleanupPromise;
|
|
1647
|
+
if (!completed) {
|
|
1648
|
+
try {
|
|
1649
|
+
await iterator?.return();
|
|
1650
|
+
} catch {}
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1653
|
+
}
|
|
1654
|
+
});
|
|
1655
|
+
return result;
|
|
1656
|
+
} catch (error) {
|
|
1657
|
+
if (options.signal?.aborted) {
|
|
1658
|
+
throw createAbortError(options.signal);
|
|
1659
|
+
}
|
|
1660
|
+
throw error;
|
|
1661
|
+
} finally {
|
|
1662
|
+
options.signal?.removeEventListener("abort", onInputAbort);
|
|
1663
|
+
input.dispose();
|
|
1664
|
+
}
|
|
1665
|
+
};
|
|
1666
|
+
export {
|
|
1667
|
+
separateVideoLayers,
|
|
1668
|
+
removeVideoMattingModel,
|
|
1669
|
+
loadVideoMattingModel,
|
|
1670
|
+
isVideoMattingModelCached,
|
|
1671
|
+
getAvailableModels,
|
|
1672
|
+
disposeVideoMattingModel,
|
|
1673
|
+
canUseVideoMatting,
|
|
1674
|
+
VideoMattingUnsupportedReason
|
|
1675
|
+
};
|