@mcut/media 0.1.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +187 -0
- package/README.md +11 -0
- package/dist/export-core-B8z3duRc.js +605 -0
- package/dist/export-worker.d.ts +1 -0
- package/dist/export-worker.js +87 -0
- package/dist/index.d.ts +431 -0
- package/dist/index.js +1384 -0
- package/package.json +56 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,1384 @@
|
|
|
1
|
+
import { _ as pruneMediaBlobs, a as listContainerFormats, c as ensureFallbackAudioEncoders, d as probeImage, f as probeMedia, g as loadMediaBlob, h as isMediaStoreSupported, i as getContainerFormat, l as createAssetFromFile, m as hashBlob, n as resolveContainerFormat, o as registerContainerFormat, p as MAX_HASHABLE_BYTES, r as runExportPipeline, s as AUDIO_SAMPLE_RATE, t as getExportSupport, u as inputFor, v as saveMediaBlob } from "./export-core-B8z3duRc.js";
|
|
2
|
+
import { AudioBufferSink, BufferTarget, CanvasSink, Conversion, Output, WavOutputFormat } from "mediabunny";
|
|
3
|
+
import { getEffectiveVolume, getMulticamAudioSource, getMulticamSourceTimeMs, getProjectDurationMs, getRenderableElements, getSourceSpanMs, getSourceTimeMs, getSpeedAt, hasFades, hasKeyframes, interpolateTrack, isElementActiveAt } from "@mcut/timeline";
|
|
4
|
+
//#region src/thumbnails.ts
|
|
5
|
+
/** Extract a single poster frame from a video. Returns `null` for audio-only files. */
|
|
6
|
+
async function getVideoThumbnail(src, options = {}) {
|
|
7
|
+
const input = inputFor(src);
|
|
8
|
+
try {
|
|
9
|
+
const track = await input.getPrimaryVideoTrack();
|
|
10
|
+
if (!track) return null;
|
|
11
|
+
return (await new CanvasSink(track, { width: options.width ?? 160 }).getCanvas((options.timeMs ?? 0) / 1e3))?.canvas ?? null;
|
|
12
|
+
} finally {
|
|
13
|
+
input.dispose();
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** A poster frame as a data URL (handy for `<img>` in media bins). */
|
|
17
|
+
async function getVideoThumbnailUrl(src, options = {}) {
|
|
18
|
+
const canvas = await getVideoThumbnail(src, options);
|
|
19
|
+
if (!canvas) return null;
|
|
20
|
+
if (canvas instanceof OffscreenCanvas) {
|
|
21
|
+
const blob = await canvas.convertToBlob({
|
|
22
|
+
type: "image/jpeg",
|
|
23
|
+
quality: .7
|
|
24
|
+
});
|
|
25
|
+
return URL.createObjectURL(blob);
|
|
26
|
+
}
|
|
27
|
+
return canvas.toDataURL("image/jpeg", .7);
|
|
28
|
+
}
|
|
29
|
+
//#endregion
|
|
30
|
+
//#region src/extract-audio.ts
|
|
31
|
+
/** The file's audio exists but this browser has no decoder for its codec. */
|
|
32
|
+
var AudioNotDecodableError = class extends Error {
|
|
33
|
+
constructor(codec) {
|
|
34
|
+
super(`This browser cannot decode the clip's audio${codec ? ` (${codec})` : ""}. Try re-encoding the file as MP4/AAC.`);
|
|
35
|
+
this.name = "AudioNotDecodableError";
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
async function runWavConversion(src, audio, onProgress) {
|
|
39
|
+
const input = inputFor(src);
|
|
40
|
+
try {
|
|
41
|
+
const target = new BufferTarget();
|
|
42
|
+
const output = new Output({
|
|
43
|
+
format: new WavOutputFormat(),
|
|
44
|
+
target
|
|
45
|
+
});
|
|
46
|
+
const conversion = await Conversion.init({
|
|
47
|
+
input,
|
|
48
|
+
output,
|
|
49
|
+
video: { discard: true },
|
|
50
|
+
audio,
|
|
51
|
+
showWarnings: false
|
|
52
|
+
});
|
|
53
|
+
if (!conversion.isValid) {
|
|
54
|
+
const audioDiscard = conversion.discardedTracks.find((d) => d.track.type === "audio");
|
|
55
|
+
if (audioDiscard?.reason === "undecodable_source_codec") throw new AudioNotDecodableError(audioDiscard.track.codec ?? void 0);
|
|
56
|
+
throw new Error(`Audio conversion is not possible for this file` + (audioDiscard ? ` (${audioDiscard.reason})` : ""));
|
|
57
|
+
}
|
|
58
|
+
if (onProgress) conversion.onProgress = onProgress;
|
|
59
|
+
await conversion.execute();
|
|
60
|
+
if (!target.buffer) return null;
|
|
61
|
+
return new Blob([target.buffer], { type: "audio/wav" });
|
|
62
|
+
} finally {
|
|
63
|
+
input.dispose();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Extract a file's audio track to a PCM WAV blob, fully client-side.
|
|
68
|
+
* Returns `null` when the file has no audio track. The default
|
|
69
|
+
* 16 kHz/mono output keeps uploads to transcription APIs small.
|
|
70
|
+
*
|
|
71
|
+
* Resilience: when the resampled conversion fails (Mediabunny's resampler /
|
|
72
|
+
* channel mixer can throw "Assertion failed." on unusual source layouts),
|
|
73
|
+
* retry once WITHOUT resampling — the WAV is bigger but transcription APIs
|
|
74
|
+
* accept any PCM rate. Undecodable codecs fail fast with a clear error.
|
|
75
|
+
*/
|
|
76
|
+
async function extractAudioToWav(src, options = {}) {
|
|
77
|
+
const probe = inputFor(src);
|
|
78
|
+
try {
|
|
79
|
+
if (!await probe.getPrimaryAudioTrack()) return null;
|
|
80
|
+
} finally {
|
|
81
|
+
probe.dispose();
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
return await runWavConversion(src, {
|
|
85
|
+
codec: "pcm-s16",
|
|
86
|
+
sampleRate: options.sampleRate ?? 16e3,
|
|
87
|
+
numberOfChannels: options.numberOfChannels ?? 1
|
|
88
|
+
}, options.onProgress);
|
|
89
|
+
} catch (error) {
|
|
90
|
+
if (error instanceof AudioNotDecodableError) throw error;
|
|
91
|
+
return await runWavConversion(src, { codec: "pcm-s16" }, options.onProgress);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
//#endregion
|
|
95
|
+
//#region src/scrub-cache.ts
|
|
96
|
+
/** ≈576² area cap per cached frame — small enough to keep 150 around. */
|
|
97
|
+
const MAX_FRAME_AREA = 331776;
|
|
98
|
+
var ScrubFrameCache = class {
|
|
99
|
+
maxFrames;
|
|
100
|
+
minGapMs;
|
|
101
|
+
/** Sorted by timeMs for binary search. */
|
|
102
|
+
frames = [];
|
|
103
|
+
/** Insertion order for FIFO eviction. */
|
|
104
|
+
order = [];
|
|
105
|
+
constructor(maxFrames = 150, minGapMs = 90) {
|
|
106
|
+
this.maxFrames = maxFrames;
|
|
107
|
+
this.minGapMs = minGapMs;
|
|
108
|
+
}
|
|
109
|
+
/** Capture the element's current frame if this instant isn't cached yet. */
|
|
110
|
+
capture(source, timeMs) {
|
|
111
|
+
if (typeof OffscreenCanvas === "undefined") return;
|
|
112
|
+
const sw = source.videoWidth;
|
|
113
|
+
const sh = source.videoHeight;
|
|
114
|
+
if (sw <= 0 || sh <= 0) return;
|
|
115
|
+
const index = this.indexAtOrAfter(timeMs);
|
|
116
|
+
const before = this.frames[index - 1];
|
|
117
|
+
const at = this.frames[index];
|
|
118
|
+
if (before && timeMs - before.timeMs < this.minGapMs || at && at.timeMs - timeMs < this.minGapMs) return;
|
|
119
|
+
const scale = Math.min(1, Math.sqrt(MAX_FRAME_AREA / (sw * sh)));
|
|
120
|
+
const canvas = new OffscreenCanvas(Math.max(1, Math.round(sw * scale)), Math.max(1, Math.round(sh * scale)));
|
|
121
|
+
const ctx = canvas.getContext("2d");
|
|
122
|
+
if (!ctx) return;
|
|
123
|
+
try {
|
|
124
|
+
ctx.drawImage(source, 0, 0, canvas.width, canvas.height);
|
|
125
|
+
} catch {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
const frame = {
|
|
129
|
+
timeMs,
|
|
130
|
+
canvas
|
|
131
|
+
};
|
|
132
|
+
this.frames.splice(index, 0, frame);
|
|
133
|
+
this.order.push(frame);
|
|
134
|
+
if (this.order.length > this.maxFrames) {
|
|
135
|
+
const evicted = this.order.shift();
|
|
136
|
+
const i = this.frames.indexOf(evicted);
|
|
137
|
+
if (i !== -1) this.frames.splice(i, 1);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
/** The cached frame nearest `timeMs`, or null when the cache is empty. */
|
|
141
|
+
nearest(timeMs) {
|
|
142
|
+
if (this.frames.length === 0) return null;
|
|
143
|
+
const index = this.indexAtOrAfter(timeMs);
|
|
144
|
+
const before = this.frames[index - 1];
|
|
145
|
+
const at = this.frames[index];
|
|
146
|
+
if (!before) return at.canvas;
|
|
147
|
+
if (!at) return before.canvas;
|
|
148
|
+
return timeMs - before.timeMs <= at.timeMs - timeMs ? before.canvas : at.canvas;
|
|
149
|
+
}
|
|
150
|
+
get size() {
|
|
151
|
+
return this.frames.length;
|
|
152
|
+
}
|
|
153
|
+
clear() {
|
|
154
|
+
this.frames = [];
|
|
155
|
+
this.order = [];
|
|
156
|
+
}
|
|
157
|
+
/** First index whose frame time is >= timeMs. */
|
|
158
|
+
indexAtOrAfter(timeMs) {
|
|
159
|
+
let lo = 0;
|
|
160
|
+
let hi = this.frames.length;
|
|
161
|
+
while (lo < hi) {
|
|
162
|
+
const mid = lo + hi >> 1;
|
|
163
|
+
if (this.frames[mid].timeMs < timeMs) lo = mid + 1;
|
|
164
|
+
else hi = mid;
|
|
165
|
+
}
|
|
166
|
+
return lo;
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
//#endregion
|
|
170
|
+
//#region src/preview-pool.ts
|
|
171
|
+
const SAME_SOURCE_TOLERANCE_MS = 40;
|
|
172
|
+
const SAME_RATE_TOLERANCE = .001;
|
|
173
|
+
function hasSameMediaClock(a, b) {
|
|
174
|
+
return Math.abs(a.sourceTimeMs - b.sourceTimeMs) <= SAME_SOURCE_TOLERANCE_MS && Math.abs(a.rate - b.rate) <= SAME_RATE_TOLERANCE && Boolean(a.reversed) === Boolean(b.reversed);
|
|
175
|
+
}
|
|
176
|
+
function mergeActiveMediaItems(current, next) {
|
|
177
|
+
const kind = current.kind === "video" || next.kind === "video" ? "video" : "audio";
|
|
178
|
+
const currentAudible = current.volume > 0;
|
|
179
|
+
const nextAudible = next.volume > 0;
|
|
180
|
+
if (currentAudible && nextAudible) return {
|
|
181
|
+
...next.volume > current.volume ? next : current,
|
|
182
|
+
kind,
|
|
183
|
+
volume: hasSameMediaClock(current, next) ? current.volume + next.volume : Math.max(current.volume, next.volume)
|
|
184
|
+
};
|
|
185
|
+
if (nextAudible && !currentAudible) return {
|
|
186
|
+
...next,
|
|
187
|
+
kind
|
|
188
|
+
};
|
|
189
|
+
if (currentAudible && !nextAudible) return {
|
|
190
|
+
...current,
|
|
191
|
+
kind
|
|
192
|
+
};
|
|
193
|
+
if (current.kind !== "video" && next.kind === "video") return {
|
|
194
|
+
...next,
|
|
195
|
+
kind
|
|
196
|
+
};
|
|
197
|
+
return {
|
|
198
|
+
...current,
|
|
199
|
+
kind
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* The pool has one native media element per asset. Collapse duplicate active
|
|
204
|
+
* references so muted transition/visual items do not fight audible timeline
|
|
205
|
+
* items over volume, rate, and currentTime in the same animation frame.
|
|
206
|
+
*/
|
|
207
|
+
function coalesceActiveMediaItems(items) {
|
|
208
|
+
const byAsset = /* @__PURE__ */ new Map();
|
|
209
|
+
for (const item of items) {
|
|
210
|
+
const current = byAsset.get(item.assetId);
|
|
211
|
+
byAsset.set(item.assetId, current ? mergeActiveMediaItems(current, item) : item);
|
|
212
|
+
}
|
|
213
|
+
return [...byAsset.values()];
|
|
214
|
+
}
|
|
215
|
+
/** The media items the preview pool should have live at `timeMs`. */
|
|
216
|
+
function getActiveMediaItems(project, timeMs) {
|
|
217
|
+
const items = [];
|
|
218
|
+
for (const { track, element } of getRenderableElements(project, timeMs)) {
|
|
219
|
+
if (element.type === "multicam") {
|
|
220
|
+
const audible = isElementActiveAt(element, timeMs);
|
|
221
|
+
const speedShim = {
|
|
222
|
+
startMs: element.startMs,
|
|
223
|
+
durationMs: element.durationMs,
|
|
224
|
+
trimStartMs: 0,
|
|
225
|
+
timeMap: element.timeMap
|
|
226
|
+
};
|
|
227
|
+
for (const source of element.sources) {
|
|
228
|
+
const isAudio = source.key === element.audioSource;
|
|
229
|
+
items.push({
|
|
230
|
+
assetId: source.assetId,
|
|
231
|
+
kind: "video",
|
|
232
|
+
sourceTimeMs: getMulticamSourceTimeMs(element, source, timeMs),
|
|
233
|
+
rate: getSpeedAt(speedShim, timeMs - element.startMs),
|
|
234
|
+
volume: isAudio && audible && !track.muted && !element.muted ? getEffectiveVolume(element, timeMs) : 0
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
if (element.type !== "video" && element.type !== "audio") continue;
|
|
240
|
+
if (element.type === "video" && track.hidden && (track.muted || element.muted)) continue;
|
|
241
|
+
const localMs = timeMs - element.startMs;
|
|
242
|
+
const audible = isElementActiveAt(element, timeMs);
|
|
243
|
+
if (element.type === "audio" && !audible) continue;
|
|
244
|
+
items.push({
|
|
245
|
+
assetId: element.assetId,
|
|
246
|
+
kind: element.type,
|
|
247
|
+
sourceTimeMs: Math.max(0, getSourceTimeMs(element, localMs)),
|
|
248
|
+
rate: getSpeedAt(element, localMs),
|
|
249
|
+
volume: !audible || track.muted || element.muted || element.reversed ? 0 : getEffectiveVolume(element, timeMs),
|
|
250
|
+
...element.reversed ? { reversed: true } : {}
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
return items;
|
|
254
|
+
}
|
|
255
|
+
/**
|
|
256
|
+
* Playing drift the rate bias absorbs; beyond this we re-seek. Re-seeking a
|
|
257
|
+
* long-GOP source decodes from the previous keyframe, so chasing the playhead
|
|
258
|
+
* with seeks degrades playback to blurry scrub-cache frames — within this
|
|
259
|
+
* window a speed bias converges without ever interrupting decode.
|
|
260
|
+
*/
|
|
261
|
+
const MAX_CATCHUP_DRIFT_S = 1;
|
|
262
|
+
/** Drift below this is noise (≈1 frame); don't bias the rate for it. */
|
|
263
|
+
const MIN_CATCHUP_DRIFT_S = .05;
|
|
264
|
+
const CATCHUP_RATE_MAX_BIAS = 1.5;
|
|
265
|
+
const CATCHUP_RATE_MIN_BIAS = .75;
|
|
266
|
+
/** Drift beyond which a paused media element gets re-seeked (scrubbing). */
|
|
267
|
+
const PAUSED_DRIFT_TOLERANCE_S = .04;
|
|
268
|
+
const DECODED_FRAME_STEP_MS = 100;
|
|
269
|
+
const DECODED_FRAME_NEARBY_MS = 750;
|
|
270
|
+
/** Upper bound on the predictive seek lead (runaway-estimate guard). */
|
|
271
|
+
const MAX_SEEK_LEAD_S = 2;
|
|
272
|
+
/** A seek in flight this long is wedged (lost decoder, dead src); re-issue. */
|
|
273
|
+
const STUCK_SEEK_MS = 4e3;
|
|
274
|
+
/** Minimum spacing between load() recovery attempts on an errored element. */
|
|
275
|
+
const RECOVERY_INTERVAL_MS = 3e3;
|
|
276
|
+
/** Back-off before re-trying a failed decoded-path (CanvasSink) init. */
|
|
277
|
+
const DECODED_INIT_RETRY_MS = 3e3;
|
|
278
|
+
function matroskaLike(asset) {
|
|
279
|
+
const name = asset.name?.toLowerCase() ?? "";
|
|
280
|
+
const mime = asset.mimeType?.toLowerCase() ?? "";
|
|
281
|
+
return name.endsWith(".mkv") || name.endsWith(".mk3d") || mime.includes("matroska") || mime === "video/x-matroska";
|
|
282
|
+
}
|
|
283
|
+
/** canPlayType per mimeType — this runs on every getFrame/sync call. */
|
|
284
|
+
const canPlayTypeCache = /* @__PURE__ */ new Map();
|
|
285
|
+
function canUseNativeVideoPreview(asset) {
|
|
286
|
+
if (asset.nativePreview === false || matroskaLike(asset)) return false;
|
|
287
|
+
if (asset.nativePreview === true) return true;
|
|
288
|
+
if (typeof document === "undefined") return true;
|
|
289
|
+
const mimeType = asset.mimeType;
|
|
290
|
+
if (!mimeType?.startsWith("video/")) return true;
|
|
291
|
+
let playable = canPlayTypeCache.get(mimeType);
|
|
292
|
+
if (playable === void 0) {
|
|
293
|
+
playable = document.createElement("video").canPlayType(mimeType) !== "";
|
|
294
|
+
canPlayTypeCache.set(mimeType, playable);
|
|
295
|
+
}
|
|
296
|
+
return playable;
|
|
297
|
+
}
|
|
298
|
+
function decodedFrameKey(sourceTimeMs) {
|
|
299
|
+
return Math.max(0, Math.round(sourceTimeMs / DECODED_FRAME_STEP_MS) * DECODED_FRAME_STEP_MS);
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* The approximate, low-latency {@link FrameSource} used for interactive
|
|
303
|
+
* preview: one pooled `<video>`/`<audio>` element per media asset, kept in
|
|
304
|
+
* sync with the playback clock, plus decoded `ImageBitmap`s for images.
|
|
305
|
+
* Audio plays through the media elements themselves (no Web Audio graph);
|
|
306
|
+
* the deterministic export pipeline is a separate implementation.
|
|
307
|
+
*
|
|
308
|
+
* Known approximation: two simultaneously-active elements sharing one asset
|
|
309
|
+
* share one media element, so they render the same source frame.
|
|
310
|
+
*/
|
|
311
|
+
var PreviewMediaPool = class {
|
|
312
|
+
resolveAsset;
|
|
313
|
+
media = /* @__PURE__ */ new Map();
|
|
314
|
+
images = /* @__PURE__ */ new Map();
|
|
315
|
+
scrubCaches = /* @__PURE__ */ new Map();
|
|
316
|
+
decodedVideos = /* @__PURE__ */ new Map();
|
|
317
|
+
disposed = false;
|
|
318
|
+
/** Transport state from the last sync(); steers mid-seek frame choice. */
|
|
319
|
+
playing = false;
|
|
320
|
+
constructor(resolveAsset) {
|
|
321
|
+
this.resolveAsset = resolveAsset;
|
|
322
|
+
}
|
|
323
|
+
getFrame(assetId, sourceTimeMs) {
|
|
324
|
+
const asset = this.resolveAsset(assetId);
|
|
325
|
+
if (!asset) return null;
|
|
326
|
+
if (asset.kind === "image") {
|
|
327
|
+
const cached = this.images.get(assetId);
|
|
328
|
+
if (cached === void 0) {
|
|
329
|
+
this.loadImage(assetId, asset.src);
|
|
330
|
+
return null;
|
|
331
|
+
}
|
|
332
|
+
return cached instanceof ImageBitmap ? cached : null;
|
|
333
|
+
}
|
|
334
|
+
if (asset.kind === "video") {
|
|
335
|
+
if (!canUseNativeVideoPreview(asset)) return this.getDecodedVideoFrame(assetId, asset, sourceTimeMs);
|
|
336
|
+
const element = this.media.get(assetId)?.el;
|
|
337
|
+
if (!(element instanceof HTMLVideoElement)) return null;
|
|
338
|
+
const cache = this.ensureScrubCache(assetId);
|
|
339
|
+
if (element.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && !element.seeking) {
|
|
340
|
+
cache.capture(element, element.currentTime * 1e3);
|
|
341
|
+
return element;
|
|
342
|
+
}
|
|
343
|
+
if (element.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA && this.playing) return element;
|
|
344
|
+
const nearby = cache.nearest(sourceTimeMs);
|
|
345
|
+
if (nearby) return nearby;
|
|
346
|
+
return element.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA ? element : null;
|
|
347
|
+
}
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* Reconcile pooled media elements with the items active under the
|
|
352
|
+
* playhead. Called by the playback loop every frame and on seeks.
|
|
353
|
+
*/
|
|
354
|
+
sync(items, options) {
|
|
355
|
+
if (this.disposed) return;
|
|
356
|
+
this.playing = options.isPlaying && options.playbackRate > 0;
|
|
357
|
+
const activeItems = coalesceActiveMediaItems(items);
|
|
358
|
+
const activeIds = new Set(activeItems.map((item) => item.assetId));
|
|
359
|
+
for (const [assetId, pooled] of this.media) {
|
|
360
|
+
this.settleSeek(pooled);
|
|
361
|
+
if (!activeIds.has(assetId) && !pooled.el.paused) pooled.el.pause();
|
|
362
|
+
}
|
|
363
|
+
for (const item of activeItems) {
|
|
364
|
+
const pooled = this.ensureMediaElement(item.assetId, item.kind);
|
|
365
|
+
if (!pooled) continue;
|
|
366
|
+
const element = pooled.el;
|
|
367
|
+
if (element.error) {
|
|
368
|
+
this.recoverMediaElement(item.assetId, pooled);
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
const targetSeconds = item.sourceTimeMs / 1e3;
|
|
372
|
+
element.volume = Math.max(0, Math.min(1, item.volume * options.masterVolume));
|
|
373
|
+
element.muted = options.muted || item.volume <= 0;
|
|
374
|
+
const frozen = item.rate <= .01;
|
|
375
|
+
const forwardRate = Math.max(.0625, options.playbackRate * (frozen ? 1 : item.rate));
|
|
376
|
+
if (options.isPlaying && options.playbackRate > 0 && !frozen && !item.reversed) {
|
|
377
|
+
const drift = targetSeconds - element.currentTime;
|
|
378
|
+
let rate = forwardRate;
|
|
379
|
+
if (Math.abs(drift) > MAX_CATCHUP_DRIFT_S) {
|
|
380
|
+
const lead = drift > 0 ? Math.min(MAX_SEEK_LEAD_S, pooled.seekLatencyS * forwardRate) : 0;
|
|
381
|
+
this.requestSeek(pooled, targetSeconds + lead);
|
|
382
|
+
} else if (Math.abs(drift) > MIN_CATCHUP_DRIFT_S && !element.seeking) rate = forwardRate * Math.min(CATCHUP_RATE_MAX_BIAS, Math.max(CATCHUP_RATE_MIN_BIAS, 1 + drift));
|
|
383
|
+
if (element.playbackRate !== rate) element.playbackRate = rate;
|
|
384
|
+
if (element.paused) element.play().catch(() => {});
|
|
385
|
+
} else {
|
|
386
|
+
if (options.playbackRate > 0 && element.playbackRate !== forwardRate) element.playbackRate = forwardRate;
|
|
387
|
+
if (!element.paused) element.pause();
|
|
388
|
+
if (Math.abs(element.currentTime - targetSeconds) > PAUSED_DRIFT_TOLERANCE_S) this.requestSeek(pooled, targetSeconds);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
/** Reload an errored element from the asset's current src, rate-limited. */
|
|
393
|
+
recoverMediaElement(assetId, pooled) {
|
|
394
|
+
const now = performance.now();
|
|
395
|
+
if (now - pooled.lastRecoveryAt < RECOVERY_INTERVAL_MS) return;
|
|
396
|
+
pooled.lastRecoveryAt = now;
|
|
397
|
+
const asset = this.resolveAsset(assetId);
|
|
398
|
+
if (!asset) return;
|
|
399
|
+
if (pooled.src !== asset.src) {
|
|
400
|
+
pooled.src = asset.src;
|
|
401
|
+
pooled.el.src = asset.src;
|
|
402
|
+
}
|
|
403
|
+
pooled.el.load();
|
|
404
|
+
pooled.seekStartedAt = null;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Seek unless one is already in flight. Restarting an in-flight seek aborts
|
|
408
|
+
* its decode, and on long-GOP sources (seek latency above the drift
|
|
409
|
+
* tolerance) that loops forever: no seek ever completes, playback degrades
|
|
410
|
+
* to scrub-cache frames, and a paused preview can stay black. Letting the
|
|
411
|
+
* seek land also coalesces scrubbing to the latest playhead position.
|
|
412
|
+
*/
|
|
413
|
+
requestSeek(pooled, targetSeconds) {
|
|
414
|
+
if (pooled.el.seeking) {
|
|
415
|
+
const startedAt = pooled.seekStartedAt;
|
|
416
|
+
if (startedAt !== null && performance.now() - startedAt < STUCK_SEEK_MS) return;
|
|
417
|
+
}
|
|
418
|
+
pooled.seekStartedAt = performance.now();
|
|
419
|
+
pooled.el.currentTime = Math.max(0, targetSeconds);
|
|
420
|
+
}
|
|
421
|
+
/** Fold a completed seek into the element's latency estimate. */
|
|
422
|
+
settleSeek(pooled) {
|
|
423
|
+
if (pooled.seekStartedAt === null || pooled.el.seeking) return;
|
|
424
|
+
const latency = (performance.now() - pooled.seekStartedAt) / 1e3;
|
|
425
|
+
pooled.seekLatencyS = pooled.seekLatencyS === 0 ? latency : pooled.seekLatencyS * .5 + latency * .5;
|
|
426
|
+
pooled.seekStartedAt = null;
|
|
427
|
+
}
|
|
428
|
+
/** Pause everything (e.g. when the player unmounts a project). */
|
|
429
|
+
pauseAll() {
|
|
430
|
+
for (const { el } of this.media.values()) if (!el.paused) el.pause();
|
|
431
|
+
}
|
|
432
|
+
dispose() {
|
|
433
|
+
this.disposed = true;
|
|
434
|
+
for (const { el } of this.media.values()) {
|
|
435
|
+
el.pause();
|
|
436
|
+
el.removeAttribute("src");
|
|
437
|
+
el.load();
|
|
438
|
+
}
|
|
439
|
+
this.media.clear();
|
|
440
|
+
for (const cache of this.scrubCaches.values()) cache.clear();
|
|
441
|
+
this.scrubCaches.clear();
|
|
442
|
+
for (const image of this.images.values()) if (image instanceof ImageBitmap) image.close();
|
|
443
|
+
this.images.clear();
|
|
444
|
+
for (const state of this.decodedVideos.values()) {
|
|
445
|
+
state.input?.dispose();
|
|
446
|
+
for (const frame of state.frames.values()) if (typeof ImageBitmap !== "undefined" && frame instanceof ImageBitmap) frame.close();
|
|
447
|
+
}
|
|
448
|
+
this.decodedVideos.clear();
|
|
449
|
+
}
|
|
450
|
+
ensureScrubCache(assetId) {
|
|
451
|
+
let cache = this.scrubCaches.get(assetId);
|
|
452
|
+
if (!cache) {
|
|
453
|
+
cache = new ScrubFrameCache();
|
|
454
|
+
this.scrubCaches.set(assetId, cache);
|
|
455
|
+
}
|
|
456
|
+
return cache;
|
|
457
|
+
}
|
|
458
|
+
ensureMediaElement(assetId, kind) {
|
|
459
|
+
const asset = this.resolveAsset(assetId);
|
|
460
|
+
if (!asset) return null;
|
|
461
|
+
let existing = this.media.get(assetId);
|
|
462
|
+
const audioOnly = kind === "video" && !canUseNativeVideoPreview(asset);
|
|
463
|
+
const wantsVideoElement = kind === "video" && !audioOnly;
|
|
464
|
+
if (existing) {
|
|
465
|
+
const existingIsVideo = existing.el instanceof HTMLVideoElement;
|
|
466
|
+
if (wantsVideoElement && !existingIsVideo || audioOnly && existingIsVideo) {
|
|
467
|
+
existing.el.pause();
|
|
468
|
+
existing.el.removeAttribute("src");
|
|
469
|
+
existing.el.load();
|
|
470
|
+
this.media.delete(assetId);
|
|
471
|
+
existing = void 0;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
if (existing) {
|
|
475
|
+
if (existing.src !== asset.src) {
|
|
476
|
+
existing.src = asset.src;
|
|
477
|
+
existing.el.src = asset.src;
|
|
478
|
+
existing.el.load();
|
|
479
|
+
existing.seekStartedAt = null;
|
|
480
|
+
}
|
|
481
|
+
return existing;
|
|
482
|
+
}
|
|
483
|
+
const element = kind === "video" && !audioOnly ? document.createElement("video") : document.createElement("audio");
|
|
484
|
+
element.src = asset.src;
|
|
485
|
+
element.preload = "auto";
|
|
486
|
+
element.crossOrigin = "anonymous";
|
|
487
|
+
if (element instanceof HTMLVideoElement) {
|
|
488
|
+
element.playsInline = true;
|
|
489
|
+
element.muted = true;
|
|
490
|
+
}
|
|
491
|
+
const pooled = {
|
|
492
|
+
el: element,
|
|
493
|
+
src: asset.src,
|
|
494
|
+
seekStartedAt: null,
|
|
495
|
+
seekLatencyS: 0,
|
|
496
|
+
lastRecoveryAt: 0
|
|
497
|
+
};
|
|
498
|
+
this.media.set(assetId, pooled);
|
|
499
|
+
return pooled;
|
|
500
|
+
}
|
|
501
|
+
getDecodedVideoFrame(assetId, asset, sourceTimeMs) {
|
|
502
|
+
const state = this.ensureDecodedVideoState(assetId, asset);
|
|
503
|
+
if (state.failed) return null;
|
|
504
|
+
const key = decodedFrameKey(sourceTimeMs);
|
|
505
|
+
const exact = state.frames.get(key);
|
|
506
|
+
if (exact) return exact;
|
|
507
|
+
this.requestDecodedVideoFrame(assetId, asset, key);
|
|
508
|
+
let nearest = null;
|
|
509
|
+
for (const [frameKey, frame] of state.frames) {
|
|
510
|
+
const distance = Math.abs(frameKey - sourceTimeMs);
|
|
511
|
+
if (distance > DECODED_FRAME_NEARBY_MS) continue;
|
|
512
|
+
if (!nearest || distance < nearest.distance) nearest = {
|
|
513
|
+
distance,
|
|
514
|
+
frame
|
|
515
|
+
};
|
|
516
|
+
}
|
|
517
|
+
return nearest?.frame ?? null;
|
|
518
|
+
}
|
|
519
|
+
ensureDecodedVideoState(assetId, asset) {
|
|
520
|
+
let state = this.decodedVideos.get(assetId);
|
|
521
|
+
if (state && asset && state.src !== null && state.src !== asset.src) {
|
|
522
|
+
state.input?.dispose();
|
|
523
|
+
for (const frame of state.frames.values()) if (typeof ImageBitmap !== "undefined" && frame instanceof ImageBitmap) frame.close();
|
|
524
|
+
state = void 0;
|
|
525
|
+
}
|
|
526
|
+
if (!state) {
|
|
527
|
+
state = {
|
|
528
|
+
src: asset?.src ?? null,
|
|
529
|
+
input: null,
|
|
530
|
+
sink: null,
|
|
531
|
+
frames: /* @__PURE__ */ new Map(),
|
|
532
|
+
pendingKey: null,
|
|
533
|
+
failed: false,
|
|
534
|
+
lastInitFailureAt: 0
|
|
535
|
+
};
|
|
536
|
+
this.decodedVideos.set(assetId, state);
|
|
537
|
+
}
|
|
538
|
+
return state;
|
|
539
|
+
}
|
|
540
|
+
requestDecodedVideoFrame(assetId, asset, key) {
|
|
541
|
+
const state = this.ensureDecodedVideoState(assetId, asset);
|
|
542
|
+
if (state.failed || state.pendingKey === key || state.frames.has(key)) return;
|
|
543
|
+
if (!state.sink && performance.now() - state.lastInitFailureAt < DECODED_INIT_RETRY_MS) return;
|
|
544
|
+
state.pendingKey = key;
|
|
545
|
+
this.decodeVideoFrame(asset, key).then((frame) => {
|
|
546
|
+
if (this.disposed) return;
|
|
547
|
+
const current = this.decodedVideos.get(assetId);
|
|
548
|
+
if (!current) return;
|
|
549
|
+
if (frame) {
|
|
550
|
+
current.frames.set(key, frame);
|
|
551
|
+
this.trimDecodedVideoFrames(current, key);
|
|
552
|
+
}
|
|
553
|
+
}).catch(() => {}).finally(() => {
|
|
554
|
+
const current = this.decodedVideos.get(assetId);
|
|
555
|
+
if (current?.pendingKey === key) current.pendingKey = null;
|
|
556
|
+
});
|
|
557
|
+
}
|
|
558
|
+
async decodeVideoFrame(asset, sourceTimeMs) {
|
|
559
|
+
const state = this.ensureDecodedVideoState(asset.id, asset);
|
|
560
|
+
if (!state.sink) {
|
|
561
|
+
const input = inputFor(asset.src);
|
|
562
|
+
try {
|
|
563
|
+
const track = await input.getPrimaryVideoTrack();
|
|
564
|
+
if (!track || !await track.canDecode()) {
|
|
565
|
+
state.failed = true;
|
|
566
|
+
console.warn(`[mcut] no WebCodecs decoder for "${asset.name ?? asset.id}" (${track?.codec ?? "no video track"}); preview frames unavailable`);
|
|
567
|
+
input.dispose();
|
|
568
|
+
return null;
|
|
569
|
+
}
|
|
570
|
+
state.sink = new CanvasSink(track, {
|
|
571
|
+
width: Math.min(1280, asset.width ?? 1280),
|
|
572
|
+
fit: "contain"
|
|
573
|
+
});
|
|
574
|
+
state.input = input;
|
|
575
|
+
} catch (error) {
|
|
576
|
+
state.lastInitFailureAt = performance.now();
|
|
577
|
+
input.dispose();
|
|
578
|
+
throw error;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
return (await state.sink.getCanvas(sourceTimeMs / 1e3))?.canvas ?? null;
|
|
582
|
+
}
|
|
583
|
+
trimDecodedVideoFrames(state, centerKey) {
|
|
584
|
+
if (state.frames.size <= 80) return;
|
|
585
|
+
const keep = new Set([...state.frames.keys()].sort((a, b) => Math.abs(a - centerKey) - Math.abs(b - centerKey)).slice(0, 60));
|
|
586
|
+
for (const key of state.frames.keys()) if (!keep.has(key)) state.frames.delete(key);
|
|
587
|
+
}
|
|
588
|
+
loadImage(assetId, src) {
|
|
589
|
+
this.images.set(assetId, "loading");
|
|
590
|
+
fetch(src).then((response) => response.blob()).then((blob) => createImageBitmap(blob)).then((bitmap) => {
|
|
591
|
+
if (this.disposed) {
|
|
592
|
+
bitmap.close();
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
this.images.set(assetId, bitmap);
|
|
596
|
+
}).catch(() => {
|
|
597
|
+
this.images.set(assetId, "error");
|
|
598
|
+
});
|
|
599
|
+
}
|
|
600
|
+
};
|
|
601
|
+
//#endregion
|
|
602
|
+
//#region src/signalsmith-offline.ts
|
|
603
|
+
/**
|
|
604
|
+
* Offline buffer driver for signalsmith-stretch.
|
|
605
|
+
*
|
|
606
|
+
* The official npm build only ships an AudioWorklet wrapper, but export is
|
|
607
|
+
* offline: we want the stretch as a pure samples-in/samples-out call that
|
|
608
|
+
* works anywhere — main thread, dedicated workers (no OfflineAudioContext),
|
|
609
|
+
* and Bun tests (no Web Audio at all). The worklet file detects its scope by
|
|
610
|
+
* looking for `AudioWorkletProcessor`/`registerProcessor` globals, so we
|
|
611
|
+
* shim those for the import, capture the registered processor class, and
|
|
612
|
+
* pump its `process()` blocks ourselves instead of letting an AudioContext
|
|
613
|
+
* drive it. The WASM engine and scheduling logic are untouched — only the
|
|
614
|
+
* realtime callback is replaced with a loop.
|
|
615
|
+
*
|
|
616
|
+
* The processor reads the worklet globals `sampleRate` and `currentTime` as
|
|
617
|
+
* free variables, so renders are serialized through a queue while those sit
|
|
618
|
+
* on `globalThis`.
|
|
619
|
+
*/
|
|
620
|
+
/** Worklets process audio in fixed 128-frame quanta; the processor assumes the same. */
|
|
621
|
+
const BLOCK_FRAMES = 128;
|
|
622
|
+
/** A same-realm MessageChannel stand-in delivering on microtasks. */
|
|
623
|
+
function createPortPair() {
|
|
624
|
+
const node = {
|
|
625
|
+
onmessage: null,
|
|
626
|
+
postMessage: (data) => queueMicrotask(() => processor.onmessage?.({ data }))
|
|
627
|
+
};
|
|
628
|
+
const processor = {
|
|
629
|
+
onmessage: null,
|
|
630
|
+
postMessage: (data) => queueMicrotask(() => node.onmessage?.({ data }))
|
|
631
|
+
};
|
|
632
|
+
return {
|
|
633
|
+
node,
|
|
634
|
+
processor
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
const globals = globalThis;
|
|
638
|
+
/** Port handed to the next FakeAudioWorkletProcessor constructed. */
|
|
639
|
+
let nextProcessorPort = null;
|
|
640
|
+
let processorClassPromise = null;
|
|
641
|
+
function loadProcessorClass() {
|
|
642
|
+
processorClassPromise ??= (async () => {
|
|
643
|
+
let captured;
|
|
644
|
+
const hadProcessor = "AudioWorkletProcessor" in globals;
|
|
645
|
+
const hadRegister = "registerProcessor" in globals;
|
|
646
|
+
const previousProcessor = globals.AudioWorkletProcessor;
|
|
647
|
+
const previousRegister = globals.registerProcessor;
|
|
648
|
+
globals.AudioWorkletProcessor = class FakeAudioWorkletProcessor {
|
|
649
|
+
port;
|
|
650
|
+
constructor() {
|
|
651
|
+
const port = nextProcessorPort;
|
|
652
|
+
nextProcessorPort = null;
|
|
653
|
+
if (!port) throw new Error("FakeAudioWorkletProcessor constructed without a port");
|
|
654
|
+
this.port = port;
|
|
655
|
+
}
|
|
656
|
+
};
|
|
657
|
+
globals.registerProcessor = (_name, cls) => {
|
|
658
|
+
captured = cls;
|
|
659
|
+
};
|
|
660
|
+
try {
|
|
661
|
+
await import("signalsmith-stretch");
|
|
662
|
+
} finally {
|
|
663
|
+
if (hadProcessor) globals.AudioWorkletProcessor = previousProcessor;
|
|
664
|
+
else delete globals.AudioWorkletProcessor;
|
|
665
|
+
if (hadRegister) globals.registerProcessor = previousRegister;
|
|
666
|
+
else delete globals.registerProcessor;
|
|
667
|
+
}
|
|
668
|
+
if (!captured) throw new Error("signalsmith-stretch did not register its worklet processor");
|
|
669
|
+
return captured;
|
|
670
|
+
})();
|
|
671
|
+
return processorClassPromise;
|
|
672
|
+
}
|
|
673
|
+
/** Renders are serialized: the processor reads sampleRate/currentTime off globalThis. */
|
|
674
|
+
let renderQueue = Promise.resolve();
|
|
675
|
+
/**
|
|
676
|
+
* Stretch `channels` (equal-length planar PCM) by `tempo` (2 = twice as
|
|
677
|
+
* fast), pitch preserved, producing exactly `outputFrames` frames per
|
|
678
|
+
* channel. Same scheduling as the realtime node: play from input 0 at
|
|
679
|
+
* `tempo` starting at output time 0.
|
|
680
|
+
*/
|
|
681
|
+
function renderStretchOffline(channels, sampleRate, tempo, outputFrames) {
|
|
682
|
+
const run = renderQueue.then(() => doRender(channels, sampleRate, tempo, outputFrames));
|
|
683
|
+
renderQueue = run.catch(() => {});
|
|
684
|
+
return run;
|
|
685
|
+
}
|
|
686
|
+
async function doRender(channels, sampleRate, tempo, outputFrames) {
|
|
687
|
+
if (channels.length === 0 || outputFrames <= 0) return channels.map(() => new Float32Array(0));
|
|
688
|
+
const Processor = await loadProcessorClass();
|
|
689
|
+
const hadSampleRate = "sampleRate" in globals;
|
|
690
|
+
const hadCurrentTime = "currentTime" in globals;
|
|
691
|
+
const previousSampleRate = globals.sampleRate;
|
|
692
|
+
const previousCurrentTime = globals.currentTime;
|
|
693
|
+
globals.sampleRate = sampleRate;
|
|
694
|
+
globals.currentTime = 0;
|
|
695
|
+
try {
|
|
696
|
+
const { node, processor } = createPortPair();
|
|
697
|
+
const pending = /* @__PURE__ */ new Map();
|
|
698
|
+
let readyResolve;
|
|
699
|
+
const ready = new Promise((resolve) => {
|
|
700
|
+
readyResolve = resolve;
|
|
701
|
+
});
|
|
702
|
+
node.onmessage = ({ data }) => {
|
|
703
|
+
const [id, value] = data;
|
|
704
|
+
if (id === "ready") readyResolve();
|
|
705
|
+
else if (typeof id === "number") {
|
|
706
|
+
pending.get(id)?.(value);
|
|
707
|
+
pending.delete(id);
|
|
708
|
+
}
|
|
709
|
+
};
|
|
710
|
+
nextProcessorPort = processor;
|
|
711
|
+
const instance = new Processor({
|
|
712
|
+
numberOfInputs: 0,
|
|
713
|
+
numberOfOutputs: 1,
|
|
714
|
+
outputChannelCount: [channels.length]
|
|
715
|
+
});
|
|
716
|
+
await ready;
|
|
717
|
+
let idCounter = 0;
|
|
718
|
+
const call = (method, ...args) => new Promise((resolve) => {
|
|
719
|
+
const id = idCounter++;
|
|
720
|
+
pending.set(id, resolve);
|
|
721
|
+
node.postMessage([
|
|
722
|
+
id,
|
|
723
|
+
method,
|
|
724
|
+
...args
|
|
725
|
+
]);
|
|
726
|
+
});
|
|
727
|
+
await call("addBuffers", channels);
|
|
728
|
+
await call("schedule", {
|
|
729
|
+
active: true,
|
|
730
|
+
input: 0,
|
|
731
|
+
output: 0,
|
|
732
|
+
rate: tempo
|
|
733
|
+
});
|
|
734
|
+
const out = channels.map(() => new Float32Array(outputFrames));
|
|
735
|
+
const block = channels.map(() => new Float32Array(BLOCK_FRAMES));
|
|
736
|
+
let written = 0;
|
|
737
|
+
while (written < outputFrames) {
|
|
738
|
+
globals.currentTime = written / sampleRate;
|
|
739
|
+
instance.process([[]], [block], {});
|
|
740
|
+
const take = Math.min(BLOCK_FRAMES, outputFrames - written);
|
|
741
|
+
for (let c = 0; c < channels.length; c++) {
|
|
742
|
+
const source = take === BLOCK_FRAMES ? block[c] : block[c].subarray(0, take);
|
|
743
|
+
out[c].set(source, written);
|
|
744
|
+
}
|
|
745
|
+
written += take;
|
|
746
|
+
}
|
|
747
|
+
return out;
|
|
748
|
+
} finally {
|
|
749
|
+
if (hadSampleRate) globals.sampleRate = previousSampleRate;
|
|
750
|
+
else delete globals.sampleRate;
|
|
751
|
+
if (hadCurrentTime) globals.currentTime = previousCurrentTime;
|
|
752
|
+
else delete globals.currentTime;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
//#endregion
|
|
756
|
+
//#region src/time-stretch.ts
|
|
757
|
+
/** The constant speed a timeMap encodes, or null when it's a ramp/freeze. */
|
|
758
|
+
function constantSpeedOf(timeMap) {
|
|
759
|
+
if (!timeMap || timeMap.length !== 2) return null;
|
|
760
|
+
const [from, to] = [timeMap[0], timeMap[1]];
|
|
761
|
+
if (from.timeMs !== 0) return null;
|
|
762
|
+
if (from.easing !== void 0 && from.easing !== "linear") return null;
|
|
763
|
+
const sourceSpanMs = to.value - from.value;
|
|
764
|
+
const outputMs = to.timeMs - from.timeMs;
|
|
765
|
+
if (sourceSpanMs <= 0 || outputMs <= 0) return null;
|
|
766
|
+
return {
|
|
767
|
+
rate: sourceSpanMs / outputMs,
|
|
768
|
+
sourceStartOffsetMs: from.value,
|
|
769
|
+
sourceSpanMs
|
|
770
|
+
};
|
|
771
|
+
}
|
|
772
|
+
/**
|
|
773
|
+
* Stretch stereo PCM by `tempo` (2 = twice as fast, half as long) with
|
|
774
|
+
* pitch preserved. Output length ≈ input / tempo.
|
|
775
|
+
*/
|
|
776
|
+
async function stretchStereo(data, tempo) {
|
|
777
|
+
const inputFrames = data.left.length;
|
|
778
|
+
const expectedFrames = Math.max(1, Math.round(inputFrames / tempo));
|
|
779
|
+
const [left, right] = await renderStretchOffline([data.left, data.right], data.sampleRate, tempo, expectedFrames);
|
|
780
|
+
return {
|
|
781
|
+
left,
|
|
782
|
+
right,
|
|
783
|
+
sampleRate: data.sampleRate
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
//#endregion
|
|
787
|
+
//#region src/export-audio.ts
|
|
788
|
+
function buildRemapPlan(timeMap, durationMs) {
|
|
789
|
+
const stepMs = 10;
|
|
790
|
+
const steps = Math.max(2, Math.ceil(durationMs / stepMs) + 1);
|
|
791
|
+
const grid = new Float64Array(steps);
|
|
792
|
+
for (let i = 0; i < steps; i++) grid[i] = interpolateTrack(timeMap, Math.min(durationMs, i * stepMs));
|
|
793
|
+
return {
|
|
794
|
+
grid,
|
|
795
|
+
stepMs
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
/**
|
|
799
|
+
* Where (output ms) and how fast (source-ms per output-ms) a source offset
|
|
800
|
+
* plays. Returns null inside freezes — frozen spans consume no source audio.
|
|
801
|
+
*/
|
|
802
|
+
function remapSourceToOutput(plan, sourceOffsetMs) {
|
|
803
|
+
const { grid, stepMs } = plan;
|
|
804
|
+
const last = grid.length - 1;
|
|
805
|
+
if (sourceOffsetMs >= grid[last]) {
|
|
806
|
+
const seg = grid[last] - grid[last - 1];
|
|
807
|
+
if (seg <= 1e-6) return null;
|
|
808
|
+
return {
|
|
809
|
+
outputMs: last * stepMs,
|
|
810
|
+
rate: seg / stepMs
|
|
811
|
+
};
|
|
812
|
+
}
|
|
813
|
+
let lo = 0;
|
|
814
|
+
let hi = last;
|
|
815
|
+
while (hi - lo > 1) {
|
|
816
|
+
const mid = lo + hi >> 1;
|
|
817
|
+
if (grid[mid] <= sourceOffsetMs) lo = mid;
|
|
818
|
+
else hi = mid;
|
|
819
|
+
}
|
|
820
|
+
const seg = grid[hi] - grid[lo];
|
|
821
|
+
if (seg <= 1e-6) return null;
|
|
822
|
+
const frac = (sourceOffsetMs - grid[lo]) / seg;
|
|
823
|
+
return {
|
|
824
|
+
outputMs: (lo + frac) * stepMs,
|
|
825
|
+
rate: seg / stepMs
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
/** Sample an armed volume track into a Web Audio value curve (~50ms steps). */
|
|
829
|
+
function sampleVolumeCurve(element, getValue) {
|
|
830
|
+
const steps = Math.min(2e3, Math.max(2, Math.ceil(element.durationMs / 50) + 1));
|
|
831
|
+
const curve = new Float32Array(steps);
|
|
832
|
+
for (let i = 0; i < steps; i++) {
|
|
833
|
+
const timelineMs = element.startMs + i / (steps - 1) * element.durationMs;
|
|
834
|
+
curve[i] = Math.max(0, getValue(timelineMs));
|
|
835
|
+
}
|
|
836
|
+
return curve;
|
|
837
|
+
}
|
|
838
|
+
function collectAudibleSegments(project) {
|
|
839
|
+
const segments = [];
|
|
840
|
+
for (const track of project.tracks) {
|
|
841
|
+
if (track.muted) continue;
|
|
842
|
+
for (const element of track.elements) {
|
|
843
|
+
if (element.type === "multicam" && !element.muted) {
|
|
844
|
+
const source = getMulticamAudioSource(element);
|
|
845
|
+
const asset = source ? project.assets[source.assetId] : void 0;
|
|
846
|
+
if (!source || !asset) continue;
|
|
847
|
+
const curved = hasKeyframes(element, "volume") || hasFades(element);
|
|
848
|
+
if (element.volume <= 0 && !curved) continue;
|
|
849
|
+
segments.push({
|
|
850
|
+
src: asset.src,
|
|
851
|
+
startMs: element.startMs,
|
|
852
|
+
durationMs: element.durationMs,
|
|
853
|
+
trimStartMs: source.trimStartMs,
|
|
854
|
+
sourceSpanMs: getSourceSpanMs(element),
|
|
855
|
+
...element.timeMap ? { timeMap: element.timeMap } : {},
|
|
856
|
+
volume: element.volume,
|
|
857
|
+
...curved ? { volumeCurve: sampleVolumeCurve(element, (timelineMs) => getEffectiveVolume(element, timelineMs)) } : {}
|
|
858
|
+
});
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
861
|
+
if (element.type !== "video" && element.type !== "audio" || element.muted) continue;
|
|
862
|
+
const curved = hasKeyframes(element, "volume") || hasFades(element);
|
|
863
|
+
if (element.volume <= 0 && !curved) continue;
|
|
864
|
+
const asset = project.assets[element.assetId];
|
|
865
|
+
if (!asset) continue;
|
|
866
|
+
segments.push({
|
|
867
|
+
src: asset.src,
|
|
868
|
+
startMs: element.startMs,
|
|
869
|
+
durationMs: element.durationMs,
|
|
870
|
+
trimStartMs: element.trimStartMs,
|
|
871
|
+
sourceSpanMs: getSourceSpanMs(element),
|
|
872
|
+
...element.timeMap ? { timeMap: element.timeMap } : {},
|
|
873
|
+
...element.reversed ? { reversed: true } : {},
|
|
874
|
+
volume: element.volume,
|
|
875
|
+
...curved ? { volumeCurve: sampleVolumeCurve(element, (timelineMs) => getEffectiveVolume(element, timelineMs)) } : {}
|
|
876
|
+
});
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
return segments;
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* Mix every audible segment offline and return transferable planar stereo
|
|
883
|
+
* PCM, or null when the project has no audible segments. Main-thread only.
|
|
884
|
+
*/
|
|
885
|
+
async function mixProjectAudio(project, totalDurationMs, signal) {
|
|
886
|
+
const segments = collectAudibleSegments(project);
|
|
887
|
+
if (segments.length === 0) return null;
|
|
888
|
+
const buffer = await mixAudioSegments(segments, totalDurationMs, signal);
|
|
889
|
+
return {
|
|
890
|
+
left: buffer.getChannelData(0),
|
|
891
|
+
right: buffer.numberOfChannels > 1 ? buffer.getChannelData(1) : buffer.getChannelData(0),
|
|
892
|
+
sampleRate: buffer.sampleRate
|
|
893
|
+
};
|
|
894
|
+
}
|
|
895
|
+
async function mixAudioSegments(segments, totalDurationMs, signal) {
|
|
896
|
+
const length = Math.ceil(totalDurationMs / 1e3 * AUDIO_SAMPLE_RATE);
|
|
897
|
+
const offline = new OfflineAudioContext(2, length, AUDIO_SAMPLE_RATE);
|
|
898
|
+
for (const segment of segments) {
|
|
899
|
+
signal?.throwIfAborted();
|
|
900
|
+
const input = inputFor(segment.src);
|
|
901
|
+
try {
|
|
902
|
+
const track = await input.getPrimaryAudioTrack();
|
|
903
|
+
if (!track) continue;
|
|
904
|
+
const sink = new AudioBufferSink(track);
|
|
905
|
+
const segmentStartS = segment.startMs / 1e3;
|
|
906
|
+
const segmentEndS = (segment.startMs + segment.durationMs) / 1e3;
|
|
907
|
+
const trimS = segment.trimStartMs / 1e3;
|
|
908
|
+
const gain = offline.createGain();
|
|
909
|
+
if (segment.volumeCurve) gain.gain.setValueCurveAtTime(segment.volumeCurve, segmentStartS, segment.durationMs / 1e3);
|
|
910
|
+
else gain.gain.value = segment.volume;
|
|
911
|
+
gain.connect(offline.destination);
|
|
912
|
+
if (segment.reversed) {
|
|
913
|
+
await scheduleReversedSegment(offline, gain, sink, segment, signal);
|
|
914
|
+
continue;
|
|
915
|
+
}
|
|
916
|
+
const constant = constantSpeedOf(segment.timeMap);
|
|
917
|
+
if (constant && Math.abs(constant.rate - 1) > 1e-6) {
|
|
918
|
+
if (await scheduleStretchedSegment(offline, gain, sink, segment, constant, signal)) continue;
|
|
919
|
+
}
|
|
920
|
+
const plan = segment.timeMap ? buildRemapPlan(segment.timeMap, segment.durationMs) : null;
|
|
921
|
+
for await (const { buffer, timestamp } of sink.buffers(trimS, trimS + segment.sourceSpanMs / 1e3)) {
|
|
922
|
+
signal?.throwIfAborted();
|
|
923
|
+
let rate = 1;
|
|
924
|
+
let when;
|
|
925
|
+
if (plan) {
|
|
926
|
+
const remapped = remapSourceToOutput(plan, (timestamp - trimS) * 1e3);
|
|
927
|
+
if (!remapped || remapped.rate <= .01) continue;
|
|
928
|
+
rate = remapped.rate;
|
|
929
|
+
when = segmentStartS + remapped.outputMs / 1e3;
|
|
930
|
+
} else when = segmentStartS + (timestamp - trimS);
|
|
931
|
+
let offset = 0;
|
|
932
|
+
if (when < segmentStartS) {
|
|
933
|
+
offset = (segmentStartS - when) * rate;
|
|
934
|
+
when = segmentStartS;
|
|
935
|
+
}
|
|
936
|
+
const playDuration = Math.min(buffer.duration - offset, (segmentEndS - when) * rate);
|
|
937
|
+
if (playDuration <= 0) continue;
|
|
938
|
+
const node = offline.createBufferSource();
|
|
939
|
+
node.buffer = buffer;
|
|
940
|
+
node.playbackRate.value = rate;
|
|
941
|
+
node.connect(gain);
|
|
942
|
+
node.start(when, offset, playDuration);
|
|
943
|
+
}
|
|
944
|
+
} finally {
|
|
945
|
+
input.dispose();
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
return offline.startRendering();
|
|
949
|
+
}
|
|
950
|
+
/** Source-frame ceiling for whole-segment stretching (~11 min at 48kHz). */
|
|
951
|
+
const MAX_STRETCH_SOURCE_FRAMES = 32e6;
|
|
952
|
+
/**
|
|
953
|
+
* Decode a contiguous source range into one stereo buffer. Returns null when
|
|
954
|
+
* the range is too large (see {@link MAX_STRETCH_SOURCE_FRAMES}) or yields
|
|
955
|
+
* no buffers.
|
|
956
|
+
*/
|
|
957
|
+
async function decodeCompositeRange(sink, startS, spanS, signal) {
|
|
958
|
+
let composite = null;
|
|
959
|
+
for await (const { buffer, timestamp } of sink.buffers(startS, startS + spanS)) {
|
|
960
|
+
signal?.throwIfAborted();
|
|
961
|
+
if (!composite) {
|
|
962
|
+
const sampleRate = buffer.sampleRate;
|
|
963
|
+
const frames = Math.ceil(spanS * sampleRate);
|
|
964
|
+
if (frames > MAX_STRETCH_SOURCE_FRAMES) return null;
|
|
965
|
+
composite = {
|
|
966
|
+
left: new Float32Array(frames),
|
|
967
|
+
right: new Float32Array(frames),
|
|
968
|
+
sampleRate
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
const offset = Math.max(0, Math.round((timestamp - startS) * composite.sampleRate));
|
|
972
|
+
if (offset >= composite.left.length) continue;
|
|
973
|
+
const left = buffer.getChannelData(0);
|
|
974
|
+
const right = buffer.numberOfChannels > 1 ? buffer.getChannelData(1) : left;
|
|
975
|
+
const count = Math.min(left.length, composite.left.length - offset);
|
|
976
|
+
composite.left.set(count === left.length ? left : left.subarray(0, count), offset);
|
|
977
|
+
composite.right.set(count === right.length ? right : right.subarray(0, count), offset);
|
|
978
|
+
}
|
|
979
|
+
return composite;
|
|
980
|
+
}
|
|
981
|
+
/** Wrap a composite in an AudioBuffer and schedule it across the segment. */
|
|
982
|
+
function scheduleComposite(offline, gain, segment, left, right, sampleRate) {
|
|
983
|
+
const out = offline.createBuffer(2, left.length, sampleRate);
|
|
984
|
+
out.getChannelData(0).set(left);
|
|
985
|
+
out.getChannelData(1).set(right);
|
|
986
|
+
const node = offline.createBufferSource();
|
|
987
|
+
node.buffer = out;
|
|
988
|
+
node.connect(gain);
|
|
989
|
+
node.start(segment.startMs / 1e3, 0, Math.min(out.duration, segment.durationMs / 1e3));
|
|
990
|
+
}
|
|
991
|
+
/**
|
|
992
|
+
* Decode a constant-speed segment's full source range, time-stretch it with
|
|
993
|
+
* pitch preserved, and schedule it as one node. Returns false (caller falls
|
|
994
|
+
* back to per-buffer playbackRate) when the range is too large or decoding
|
|
995
|
+
* misbehaves.
|
|
996
|
+
*/
|
|
997
|
+
async function scheduleStretchedSegment(offline, gain, sink, segment, constant, signal) {
|
|
998
|
+
try {
|
|
999
|
+
const composite = await decodeCompositeRange(sink, (segment.trimStartMs + constant.sourceStartOffsetMs) / 1e3, constant.sourceSpanMs / 1e3, signal);
|
|
1000
|
+
if (!composite) return false;
|
|
1001
|
+
const stretched = await stretchStereo(composite, constant.rate);
|
|
1002
|
+
if (stretched.left.length === 0) return false;
|
|
1003
|
+
scheduleComposite(offline, gain, segment, stretched.left, stretched.right, composite.sampleRate);
|
|
1004
|
+
return true;
|
|
1005
|
+
} catch (error) {
|
|
1006
|
+
if (signal?.aborted) throw error;
|
|
1007
|
+
return false;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
/**
|
|
1011
|
+
* Decode a reversed segment's full source span, flip the samples, and
|
|
1012
|
+
* schedule it as one node — pitch-preserving stretch when the clip is also
|
|
1013
|
+
* speed-changed. Speed RAMPS on reversed clips render at their average rate
|
|
1014
|
+
* (a v1 limit). Returns false when decoding misbehaves; the caller leaves
|
|
1015
|
+
* the segment silent rather than playing it forward.
|
|
1016
|
+
*/
|
|
1017
|
+
async function scheduleReversedSegment(offline, gain, sink, segment, signal) {
|
|
1018
|
+
try {
|
|
1019
|
+
const composite = await decodeCompositeRange(sink, segment.trimStartMs / 1e3, segment.sourceSpanMs / 1e3, signal);
|
|
1020
|
+
if (!composite) return false;
|
|
1021
|
+
composite.left.reverse();
|
|
1022
|
+
composite.right.reverse();
|
|
1023
|
+
const rate = segment.sourceSpanMs / Math.max(1, segment.durationMs);
|
|
1024
|
+
let { left, right } = composite;
|
|
1025
|
+
if (Math.abs(rate - 1) > 1e-6) {
|
|
1026
|
+
const stretched = await stretchStereo(composite, rate);
|
|
1027
|
+
if (stretched.left.length === 0) return false;
|
|
1028
|
+
left = stretched.left;
|
|
1029
|
+
right = stretched.right;
|
|
1030
|
+
}
|
|
1031
|
+
scheduleComposite(offline, gain, segment, left, right, composite.sampleRate);
|
|
1032
|
+
return true;
|
|
1033
|
+
} catch (error) {
|
|
1034
|
+
if (signal?.aborted) throw error;
|
|
1035
|
+
return false;
|
|
1036
|
+
}
|
|
1037
|
+
}
|
|
1038
|
+
//#endregion
|
|
1039
|
+
//#region src/export.ts
|
|
1040
|
+
/** How the last `exportProject` call ran — observability for tests/debugging. */
|
|
1041
|
+
function noteExportMode(mode) {
|
|
1042
|
+
globalThis.__mcutLastExportMode = mode;
|
|
1043
|
+
}
|
|
1044
|
+
/** Worker spawn → first message budget (dev bundlers compile on demand). */
|
|
1045
|
+
const WORKER_READY_TIMEOUT_MS = 15e3;
|
|
1046
|
+
/** The worker failed before its 'ready' handshake — safe to run locally. */
|
|
1047
|
+
var WorkerStartError = class extends Error {};
|
|
1048
|
+
/**
|
|
1049
|
+
* Render a project to a video file, fully client-side and deterministically.
|
|
1050
|
+
*
|
|
1051
|
+
* The audio mix renders first on the main thread (`OfflineAudioContext` and
|
|
1052
|
+
* the time-stretch worklet don't exist in workers), then the frame
|
|
1053
|
+
* decode→composite→encode→mux pipeline runs in a dedicated worker so the
|
|
1054
|
+
* editor stays responsive; environments without workers (Node/Bun, spawn
|
|
1055
|
+
* failure) fall back to running the same pipeline in-context.
|
|
1056
|
+
*/
|
|
1057
|
+
async function exportProject(project, options = {}) {
|
|
1058
|
+
const { onProgress, signal } = options;
|
|
1059
|
+
signal?.throwIfAborted();
|
|
1060
|
+
const durationMs = getProjectDurationMs(project);
|
|
1061
|
+
if (durationMs <= 0) throw new Error("Cannot export an empty project");
|
|
1062
|
+
const container = resolveContainerFormat(options.format);
|
|
1063
|
+
let mixedAudio = null;
|
|
1064
|
+
if ((await getExportSupport(options.format)).audio) {
|
|
1065
|
+
onProgress?.({
|
|
1066
|
+
phase: "audio",
|
|
1067
|
+
progress: 0
|
|
1068
|
+
});
|
|
1069
|
+
mixedAudio = await mixProjectAudio(project, durationMs, signal);
|
|
1070
|
+
onProgress?.({
|
|
1071
|
+
phase: "audio",
|
|
1072
|
+
progress: .1
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
const worker = options.videoBitrate === void 0 || typeof options.videoBitrate === "number" ? spawnExportWorker() : null;
|
|
1076
|
+
if (worker) try {
|
|
1077
|
+
noteExportMode("worker");
|
|
1078
|
+
return await runInWorker(worker, project, options, mixedAudio, container.extension);
|
|
1079
|
+
} catch (error) {
|
|
1080
|
+
if (!(error instanceof WorkerStartError)) throw error;
|
|
1081
|
+
console.warn(`mcut export: ${error.message}; falling back to main-thread export`);
|
|
1082
|
+
} finally {
|
|
1083
|
+
worker.terminate();
|
|
1084
|
+
}
|
|
1085
|
+
noteExportMode("local");
|
|
1086
|
+
const result = await runExportPipeline(project, {
|
|
1087
|
+
...options.format ? { format: options.format } : {},
|
|
1088
|
+
...options.videoBitrate !== void 0 ? { videoBitrate: options.videoBitrate } : {},
|
|
1089
|
+
mixedAudio,
|
|
1090
|
+
...onProgress ? { onProgress } : {},
|
|
1091
|
+
...signal ? { signal } : {}
|
|
1092
|
+
});
|
|
1093
|
+
return {
|
|
1094
|
+
blob: new Blob([result.buffer], { type: result.mimeType }),
|
|
1095
|
+
extension: result.extension
|
|
1096
|
+
};
|
|
1097
|
+
}
|
|
1098
|
+
/**
|
|
1099
|
+
* Spawn the export worker, or null where workers can't run the pipeline
|
|
1100
|
+
* (Node/Bun, no Worker global, spawn throws). The `new Worker(new URL(...))`
|
|
1101
|
+
* form is load-bearing: bundlers (Turbopack/webpack/Vite) statically detect
|
|
1102
|
+
* it and emit `export-worker.js` as a worker entry.
|
|
1103
|
+
*/
|
|
1104
|
+
function spawnExportWorker() {
|
|
1105
|
+
if (typeof Worker === "undefined" || typeof window === "undefined") return null;
|
|
1106
|
+
try {
|
|
1107
|
+
return new Worker(new URL("./export-worker.js", import.meta.url), { type: "module" });
|
|
1108
|
+
} catch {
|
|
1109
|
+
return null;
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
function runInWorker(worker, project, options, mixedAudio, extension) {
|
|
1113
|
+
const { onProgress, signal } = options;
|
|
1114
|
+
return new Promise((resolve, reject) => {
|
|
1115
|
+
let settled = false;
|
|
1116
|
+
let started = false;
|
|
1117
|
+
const settle = (fn) => {
|
|
1118
|
+
if (settled) return;
|
|
1119
|
+
settled = true;
|
|
1120
|
+
clearTimeout(readyTimer);
|
|
1121
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1122
|
+
fn();
|
|
1123
|
+
};
|
|
1124
|
+
const onAbort = () => settle(() => reject(signal?.reason ?? new DOMException("Aborted", "AbortError")));
|
|
1125
|
+
signal?.addEventListener("abort", onAbort);
|
|
1126
|
+
const readyTimer = setTimeout(() => settle(() => reject(new WorkerStartError("export worker did not start in time"))), WORKER_READY_TIMEOUT_MS);
|
|
1127
|
+
worker.onerror = (event) => settle(() => {
|
|
1128
|
+
const detail = event.message || "unknown error";
|
|
1129
|
+
reject(started ? /* @__PURE__ */ new Error(`Export worker crashed: ${detail}`) : new WorkerStartError(`export worker failed to load (${detail})`));
|
|
1130
|
+
});
|
|
1131
|
+
worker.onmessage = (event) => {
|
|
1132
|
+
const message = event.data;
|
|
1133
|
+
switch (message.type) {
|
|
1134
|
+
case "ready": {
|
|
1135
|
+
clearTimeout(readyTimer);
|
|
1136
|
+
started = true;
|
|
1137
|
+
const start = {
|
|
1138
|
+
type: "start",
|
|
1139
|
+
project,
|
|
1140
|
+
options: {
|
|
1141
|
+
...options.format ? { format: options.format } : {},
|
|
1142
|
+
...typeof options.videoBitrate === "number" ? { videoBitrate: options.videoBitrate } : {}
|
|
1143
|
+
},
|
|
1144
|
+
mixedAudio,
|
|
1145
|
+
fonts: options.fonts ?? []
|
|
1146
|
+
};
|
|
1147
|
+
worker.postMessage(start, collectTransfers(mixedAudio, options.fonts));
|
|
1148
|
+
break;
|
|
1149
|
+
}
|
|
1150
|
+
case "progress":
|
|
1151
|
+
if (!settled) onProgress?.({
|
|
1152
|
+
progress: message.progress,
|
|
1153
|
+
phase: message.phase
|
|
1154
|
+
});
|
|
1155
|
+
break;
|
|
1156
|
+
case "done":
|
|
1157
|
+
settle(() => {
|
|
1158
|
+
onProgress?.({
|
|
1159
|
+
phase: "finalize",
|
|
1160
|
+
progress: 1
|
|
1161
|
+
});
|
|
1162
|
+
resolve({
|
|
1163
|
+
blob: new Blob([message.buffer], { type: message.mimeType }),
|
|
1164
|
+
extension: message.extension || extension
|
|
1165
|
+
});
|
|
1166
|
+
});
|
|
1167
|
+
break;
|
|
1168
|
+
case "error":
|
|
1169
|
+
settle(() => reject(new Error(message.message)));
|
|
1170
|
+
break;
|
|
1171
|
+
}
|
|
1172
|
+
};
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
function collectTransfers(mixedAudio, fonts) {
|
|
1176
|
+
const transfers = /* @__PURE__ */ new Set();
|
|
1177
|
+
if (mixedAudio) {
|
|
1178
|
+
transfers.add(mixedAudio.left.buffer);
|
|
1179
|
+
transfers.add(mixedAudio.right.buffer);
|
|
1180
|
+
}
|
|
1181
|
+
for (const font of fonts ?? []) if (typeof font.source !== "string") transfers.add(font.source);
|
|
1182
|
+
return [...transfers];
|
|
1183
|
+
}
|
|
1184
|
+
//#endregion
|
|
1185
|
+
//#region src/filmstrip.ts
|
|
1186
|
+
function createCanvas(width, height) {
|
|
1187
|
+
if (typeof document !== "undefined") {
|
|
1188
|
+
const canvas = document.createElement("canvas");
|
|
1189
|
+
canvas.width = width;
|
|
1190
|
+
canvas.height = height;
|
|
1191
|
+
return canvas;
|
|
1192
|
+
}
|
|
1193
|
+
return new OffscreenCanvas(width, height);
|
|
1194
|
+
}
|
|
1195
|
+
/**
|
|
1196
|
+
* Sample evenly spaced poster frames into one horizontal strip — the
|
|
1197
|
+
* filmstrip background of timeline video clips. Returns `null` for files
|
|
1198
|
+
* without a video track.
|
|
1199
|
+
*/
|
|
1200
|
+
async function getFilmstrip(src, options) {
|
|
1201
|
+
const frameWidth = options.frameWidth ?? 80;
|
|
1202
|
+
const frameCount = Math.max(1, Math.round(options.frameCount));
|
|
1203
|
+
const input = inputFor(src);
|
|
1204
|
+
try {
|
|
1205
|
+
const track = await input.getPrimaryVideoTrack();
|
|
1206
|
+
if (!track) return null;
|
|
1207
|
+
const durationMs = options.endMs ?? await input.computeDuration() * 1e3;
|
|
1208
|
+
const startMs = options.startMs ?? 0;
|
|
1209
|
+
const spanMs = Math.max(1, durationMs - startMs);
|
|
1210
|
+
const timestampsMs = Array.from({ length: frameCount }, (_, i) => startMs + (i + .5) / frameCount * spanMs);
|
|
1211
|
+
const sink = new CanvasSink(track, {
|
|
1212
|
+
width: frameWidth,
|
|
1213
|
+
fit: "cover"
|
|
1214
|
+
});
|
|
1215
|
+
let strip = null;
|
|
1216
|
+
let ctx = null;
|
|
1217
|
+
let frameHeight = 0;
|
|
1218
|
+
let index = 0;
|
|
1219
|
+
for await (const wrapped of sink.canvasesAtTimestamps(timestampsMs.map((ms) => ms / 1e3))) {
|
|
1220
|
+
if (wrapped) {
|
|
1221
|
+
if (!strip) {
|
|
1222
|
+
frameHeight = wrapped.canvas.height;
|
|
1223
|
+
strip = createCanvas(frameWidth * frameCount, frameHeight);
|
|
1224
|
+
ctx = strip.getContext("2d");
|
|
1225
|
+
}
|
|
1226
|
+
ctx?.drawImage(wrapped.canvas, index * frameWidth, 0);
|
|
1227
|
+
}
|
|
1228
|
+
index++;
|
|
1229
|
+
}
|
|
1230
|
+
if (!strip) return null;
|
|
1231
|
+
return {
|
|
1232
|
+
canvas: strip,
|
|
1233
|
+
frameWidth,
|
|
1234
|
+
frameHeight,
|
|
1235
|
+
frameCount,
|
|
1236
|
+
timestampsMs
|
|
1237
|
+
};
|
|
1238
|
+
} finally {
|
|
1239
|
+
input.dispose();
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
//#endregion
|
|
1243
|
+
//#region src/audio-peaks.ts
|
|
1244
|
+
/** Fold samples into `buckets` max-|amplitude| bins (pure; unit-tested). */
|
|
1245
|
+
function bucketPeaks(samples, buckets) {
|
|
1246
|
+
const peaks = new Float32Array(Math.max(1, buckets));
|
|
1247
|
+
if (samples.length === 0) return peaks;
|
|
1248
|
+
const perBucket = samples.length / peaks.length;
|
|
1249
|
+
for (let i = 0; i < samples.length; i++) {
|
|
1250
|
+
const bucket = Math.min(peaks.length - 1, Math.floor(i / perBucket));
|
|
1251
|
+
const value = Math.abs(samples[i]);
|
|
1252
|
+
if (value > peaks[bucket]) peaks[bucket] = value;
|
|
1253
|
+
}
|
|
1254
|
+
return peaks;
|
|
1255
|
+
}
|
|
1256
|
+
/**
|
|
1257
|
+
* Decode a file's audio and reduce it to waveform peaks for timeline clip
|
|
1258
|
+
* rendering. Returns `null` when the file has no audio track. Browser-only
|
|
1259
|
+
* (WebCodecs decode via Mediabunny).
|
|
1260
|
+
*/
|
|
1261
|
+
async function extractAudioPeaks(src, options = {}) {
|
|
1262
|
+
const bucketCount = options.buckets ?? 256;
|
|
1263
|
+
const input = inputFor(src);
|
|
1264
|
+
try {
|
|
1265
|
+
const track = await input.getPrimaryAudioTrack();
|
|
1266
|
+
if (!track) return null;
|
|
1267
|
+
const durationMs = options.endMs ?? await input.computeDuration() * 1e3;
|
|
1268
|
+
const startMs = options.startMs ?? 0;
|
|
1269
|
+
const spanMs = Math.max(1, durationMs - startMs);
|
|
1270
|
+
const peaks = new Float32Array(bucketCount);
|
|
1271
|
+
const sink = new AudioBufferSink(track);
|
|
1272
|
+
for await (const { buffer, timestamp } of sink.buffers(startMs / 1e3, durationMs / 1e3)) {
|
|
1273
|
+
const channel = buffer.getChannelData(0);
|
|
1274
|
+
const bufferStartMs = timestamp * 1e3;
|
|
1275
|
+
const msPerSample = 1e3 / buffer.sampleRate;
|
|
1276
|
+
const stride = Math.max(1, Math.floor(channel.length / 4096));
|
|
1277
|
+
for (let i = 0; i < channel.length; i += stride) {
|
|
1278
|
+
const timeMs = bufferStartMs + i * msPerSample;
|
|
1279
|
+
const bucket = Math.floor((timeMs - startMs) / spanMs * bucketCount);
|
|
1280
|
+
if (bucket < 0 || bucket >= bucketCount) continue;
|
|
1281
|
+
const value = Math.abs(channel[i]);
|
|
1282
|
+
if (value > peaks[bucket]) peaks[bucket] = value;
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
return {
|
|
1286
|
+
peaks,
|
|
1287
|
+
durationMs: spanMs
|
|
1288
|
+
};
|
|
1289
|
+
} finally {
|
|
1290
|
+
input.dispose();
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
//#endregion
|
|
1294
|
+
//#region src/audio-sync.ts
|
|
1295
|
+
/**
|
|
1296
|
+
* Normalized cross-correlation of two zero-meaned envelopes. Returns the lag
|
|
1297
|
+
* (in buckets) that best aligns `b` to `a`: positive lag means b's content
|
|
1298
|
+
* happens LATER in its own file, i.e. b started recording earlier.
|
|
1299
|
+
* Pure — unit-testable without decoding.
|
|
1300
|
+
*/
|
|
1301
|
+
function crossCorrelateEnvelopes(a, b, maxLagBuckets) {
|
|
1302
|
+
const center = (env) => {
|
|
1303
|
+
let mean = 0;
|
|
1304
|
+
for (const v of env) mean += v;
|
|
1305
|
+
mean /= env.length || 1;
|
|
1306
|
+
const out = new Float32Array(env.length);
|
|
1307
|
+
for (let i = 0; i < env.length; i++) out[i] = env[i] - mean;
|
|
1308
|
+
return out;
|
|
1309
|
+
};
|
|
1310
|
+
const ca = center(a);
|
|
1311
|
+
const cb = center(b);
|
|
1312
|
+
let bestLag = 0;
|
|
1313
|
+
let best = -Infinity;
|
|
1314
|
+
let secondBest = -Infinity;
|
|
1315
|
+
for (let lag = -maxLagBuckets; lag <= maxLagBuckets; lag++) {
|
|
1316
|
+
let dot = 0;
|
|
1317
|
+
let na = 0;
|
|
1318
|
+
let nb = 0;
|
|
1319
|
+
for (let i = 0; i < ca.length; i++) {
|
|
1320
|
+
const j = i + lag;
|
|
1321
|
+
if (j < 0 || j >= cb.length) continue;
|
|
1322
|
+
dot += ca[i] * cb[j];
|
|
1323
|
+
na += ca[i] * ca[i];
|
|
1324
|
+
nb += cb[j] * cb[j];
|
|
1325
|
+
}
|
|
1326
|
+
const score = na > 0 && nb > 0 ? dot / Math.sqrt(na * nb) : 0;
|
|
1327
|
+
if (score > best) {
|
|
1328
|
+
if (Math.abs(lag - bestLag) > 4) secondBest = best;
|
|
1329
|
+
best = score;
|
|
1330
|
+
bestLag = lag;
|
|
1331
|
+
} else if (score > secondBest && Math.abs(lag - bestLag) > 4) secondBest = score;
|
|
1332
|
+
}
|
|
1333
|
+
const confidence = secondBest > 0 ? best / secondBest : best > 0 ? 99 : 0;
|
|
1334
|
+
return {
|
|
1335
|
+
lag: bestLag,
|
|
1336
|
+
confidence
|
|
1337
|
+
};
|
|
1338
|
+
}
|
|
1339
|
+
/** RMS envelope of the first `windowS` seconds at `rateHz` buckets/second. */
|
|
1340
|
+
async function extractEnvelope(src, { windowS = 60, rateHz = 100, signal } = {}) {
|
|
1341
|
+
const input = inputFor(src);
|
|
1342
|
+
try {
|
|
1343
|
+
const track = await input.getPrimaryAudioTrack();
|
|
1344
|
+
if (!track) return null;
|
|
1345
|
+
const buckets = Math.ceil(windowS * rateHz);
|
|
1346
|
+
const sums = new Float64Array(buckets);
|
|
1347
|
+
const counts = new Float64Array(buckets);
|
|
1348
|
+
const sink = new AudioBufferSink(track);
|
|
1349
|
+
for await (const { buffer, timestamp } of sink.buffers(0, windowS)) {
|
|
1350
|
+
signal?.throwIfAborted();
|
|
1351
|
+
const data = buffer.getChannelData(0);
|
|
1352
|
+
const sampleRate = buffer.sampleRate;
|
|
1353
|
+
for (let i = 0; i < data.length; i += 4) {
|
|
1354
|
+
const t = timestamp + i / sampleRate;
|
|
1355
|
+
const bucket = Math.floor(t * rateHz);
|
|
1356
|
+
if (bucket < 0 || bucket >= buckets) continue;
|
|
1357
|
+
sums[bucket] += data[i] * data[i];
|
|
1358
|
+
counts[bucket] += 1;
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
const envelope = new Float32Array(buckets);
|
|
1362
|
+
for (let i = 0; i < buckets; i++) envelope[i] = counts[i] > 0 ? Math.sqrt(sums[i] / counts[i]) : 0;
|
|
1363
|
+
return envelope;
|
|
1364
|
+
} finally {
|
|
1365
|
+
input.dispose();
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
/**
|
|
1369
|
+
* The sync offset between two recordings: how many ms after A's recording
|
|
1370
|
+
* started did B's start. Null when either source has no audio.
|
|
1371
|
+
*/
|
|
1372
|
+
async function findSyncOffsetMs(a, b, options = {}) {
|
|
1373
|
+
const rateHz = options.rateHz ?? 100;
|
|
1374
|
+
const maxLagS = options.maxLagS ?? 30;
|
|
1375
|
+
const [envA, envB] = await Promise.all([extractEnvelope(a, options), extractEnvelope(b, options)]);
|
|
1376
|
+
if (!envA || !envB) return null;
|
|
1377
|
+
const { lag, confidence } = crossCorrelateEnvelopes(envA, envB, Math.round(maxLagS * rateHz));
|
|
1378
|
+
return {
|
|
1379
|
+
offsetMs: Math.round(-lag * 1e3 / rateHz),
|
|
1380
|
+
confidence
|
|
1381
|
+
};
|
|
1382
|
+
}
|
|
1383
|
+
//#endregion
|
|
1384
|
+
export { AudioNotDecodableError, MAX_HASHABLE_BYTES, PreviewMediaPool, ScrubFrameCache, bucketPeaks, constantSpeedOf, createAssetFromFile, crossCorrelateEnvelopes, ensureFallbackAudioEncoders, exportProject, extractAudioPeaks, extractAudioToWav, extractEnvelope, findSyncOffsetMs, getActiveMediaItems, getContainerFormat, getExportSupport, getFilmstrip, getVideoThumbnail, getVideoThumbnailUrl, hashBlob, inputFor, isMediaStoreSupported, listContainerFormats, loadMediaBlob, probeImage, probeMedia, pruneMediaBlobs, registerContainerFormat, saveMediaBlob, stretchStereo };
|