@bendyline/squisq-video-react 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -5
- package/dist/chunk-6DKLHXX3.js +47 -0
- package/dist/chunk-6DKLHXX3.js.map +1 -0
- package/dist/index.d.ts +120 -38
- package/dist/index.js +320 -25
- package/dist/index.js.map +1 -1
- package/dist/workers/encode.worker.js +3 -22
- package/dist/workers/encode.worker.js.map +1 -1
- package/package.json +8 -5
- package/src/VideoExportButton.tsx +14 -2
- package/src/VideoExportModal.tsx +37 -7
- package/src/__tests__/audioTrack.test.ts +108 -0
- package/src/__tests__/mp4MuxAudio.test.ts +72 -0
- package/src/__tests__/videoExportProps.test.tsx +73 -0
- package/src/audioTrack.ts +323 -0
- package/src/hooks/useFrameCapture.ts +1 -1
- package/src/hooks/useVideoExport.ts +187 -6
- package/src/index.ts +5 -2
- package/src/mainThreadEncoder.ts +26 -13
- package/src/mp4Mux.ts +77 -3
- package/src/workers/encode.worker.ts +6 -9
- package/dist/chunk-LZYWP4HB.js +0 -28
- package/dist/chunk-LZYWP4HB.js.map +0 -1
- package/src/__tests__/encodeParams.test.ts +0 -33
- package/src/workers/encodeParams.ts +0 -24
|
@@ -0,0 +1,323 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* audioTrack — In-browser audio rendering + AAC encoding for MP4 export.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the CLI's ffmpeg `adelay`/`atrim`/`amix` graph with browser-native
|
|
5
|
+
* primitives:
|
|
6
|
+
*
|
|
7
|
+
* - {@link supportsWebCodecsAac} probes whether the browser can encode AAC
|
|
8
|
+
* (WebCodecs `AudioEncoder`).
|
|
9
|
+
* - {@link renderAudioTimeline} mixes an {@link AudioTimelineClip} list into a
|
|
10
|
+
* single `AudioBuffer` using an `OfflineAudioContext` (decode → schedule →
|
|
11
|
+
* render).
|
|
12
|
+
* - {@link encodeAacTrack} feeds that buffer through a WebCodecs
|
|
13
|
+
* `AudioEncoder` and hands the chunks to an mp4-muxer audio track (tier 1).
|
|
14
|
+
* - {@link audioBufferToWav} + {@link muxAudioWithFfmpegWasm} cover the
|
|
15
|
+
* ffmpeg.wasm fallback (tier 2).
|
|
16
|
+
* - {@link selectAudioTier} is the pure capability→tier decision shared with
|
|
17
|
+
* `useVideoExport` (and unit-tested at the module boundary).
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import type { AudioTimelineClip } from '@bendyline/squisq-video';
|
|
21
|
+
|
|
22
|
+
/** Sample rate the export renders + encodes audio at. */
|
|
23
|
+
export const EXPORT_AUDIO_SAMPLE_RATE = 48_000;
|
|
24
|
+
/** Channel count the export renders + encodes audio at (stereo). */
|
|
25
|
+
export const EXPORT_AUDIO_CHANNELS = 2;
|
|
26
|
+
|
|
27
|
+
/** Tier-3 reason used when AAC encoding is impossible and no ffmpeg fallback. */
|
|
28
|
+
export const REASON_NO_AAC_NO_SAB =
|
|
29
|
+
'This browser cannot encode AAC audio (WebCodecs AudioEncoder unavailable) ' +
|
|
30
|
+
'and the ffmpeg.wasm fallback requires cross-origin isolation (SharedArrayBuffer).';
|
|
31
|
+
|
|
32
|
+
// ── Capability probe ───────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Whether the browser can encode AAC (`mp4a.40.2`) via WebCodecs.
|
|
36
|
+
* Returns false when `AudioEncoder` is undefined (e.g. Node/jsdom, Safari,
|
|
37
|
+
* older Chromium) or the config is unsupported.
|
|
38
|
+
*/
|
|
39
|
+
export async function supportsWebCodecsAac(
|
|
40
|
+
sampleRate: number = EXPORT_AUDIO_SAMPLE_RATE,
|
|
41
|
+
channels: number = EXPORT_AUDIO_CHANNELS,
|
|
42
|
+
): Promise<boolean> {
|
|
43
|
+
if (typeof AudioEncoder === 'undefined') return false;
|
|
44
|
+
try {
|
|
45
|
+
const support = await AudioEncoder.isConfigSupported({
|
|
46
|
+
codec: 'mp4a.40.2',
|
|
47
|
+
sampleRate,
|
|
48
|
+
numberOfChannels: channels,
|
|
49
|
+
bitrate: 128_000,
|
|
50
|
+
});
|
|
51
|
+
return support.supported === true;
|
|
52
|
+
} catch {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// ── Tier selection (pure) ──────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
export interface AudioTierInputs {
|
|
60
|
+
/** Whether the doc has any audio timeline clips to include at all. */
|
|
61
|
+
hasClips: boolean;
|
|
62
|
+
/** Whether WebCodecs AAC encoding is available. */
|
|
63
|
+
aacSupported: boolean;
|
|
64
|
+
/** Whether `SharedArrayBuffer` (ffmpeg.wasm path) is available. */
|
|
65
|
+
sharedArrayBufferAvailable: boolean;
|
|
66
|
+
/** Whether the main-thread WebCodecs H.264 encoder is in use (tier-1 host). */
|
|
67
|
+
canUseMainThreadWebCodecs: boolean;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface AudioTierDecision {
|
|
71
|
+
/**
|
|
72
|
+
* 1 → inline AAC into the video muxer.
|
|
73
|
+
* 2 → finalize video-only then ffmpeg.wasm mux pass.
|
|
74
|
+
* 3 → video-only (no audio).
|
|
75
|
+
*/
|
|
76
|
+
tier: 1 | 2 | 3;
|
|
77
|
+
/**
|
|
78
|
+
* Reason audio will be absent, when tier 3. `null` means "there was simply
|
|
79
|
+
* no audio to include" (not a failure). Non-null for a genuine capability
|
|
80
|
+
* shortfall. Always `null` for tiers 1 and 2 (the attempt is expected to
|
|
81
|
+
* succeed; the caller downgrades with a fresh reason on runtime failure).
|
|
82
|
+
*/
|
|
83
|
+
reason: string | null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Decide which audio tier to run from capability booleans. Pure — the single
|
|
88
|
+
* source of truth for tier selection, shared with `useVideoExport`.
|
|
89
|
+
*/
|
|
90
|
+
export function selectAudioTier(inputs: AudioTierInputs): AudioTierDecision {
|
|
91
|
+
if (!inputs.hasClips) {
|
|
92
|
+
return { tier: 3, reason: null };
|
|
93
|
+
}
|
|
94
|
+
if (inputs.aacSupported && inputs.canUseMainThreadWebCodecs) {
|
|
95
|
+
return { tier: 1, reason: null };
|
|
96
|
+
}
|
|
97
|
+
if (inputs.sharedArrayBufferAvailable) {
|
|
98
|
+
return { tier: 2, reason: null };
|
|
99
|
+
}
|
|
100
|
+
return { tier: 3, reason: REASON_NO_AAC_NO_SAB };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ── OfflineAudioContext rendering ──────────────────────────────────
|
|
104
|
+
|
|
105
|
+
type OfflineCtor = new (
|
|
106
|
+
numberOfChannels: number,
|
|
107
|
+
length: number,
|
|
108
|
+
sampleRate: number,
|
|
109
|
+
) => OfflineAudioContext;
|
|
110
|
+
|
|
111
|
+
function resolveOfflineAudioContext(): OfflineCtor {
|
|
112
|
+
const g = globalThis as unknown as {
|
|
113
|
+
OfflineAudioContext?: OfflineCtor;
|
|
114
|
+
webkitOfflineAudioContext?: OfflineCtor;
|
|
115
|
+
};
|
|
116
|
+
const ctor = g.OfflineAudioContext ?? g.webkitOfflineAudioContext;
|
|
117
|
+
if (!ctor) {
|
|
118
|
+
throw new Error('OfflineAudioContext is not available in this environment.');
|
|
119
|
+
}
|
|
120
|
+
return ctor;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Mix a list of timeline clips into one `AudioBuffer`.
|
|
125
|
+
*
|
|
126
|
+
* Decodes each unique source once, then schedules an `AudioBufferSourceNode`
|
|
127
|
+
* per clip at its `startSec` with the trim window (`offset = sourceInSec`,
|
|
128
|
+
* `duration = durationSec`). Undecodable sources are skipped rather than
|
|
129
|
+
* failing the whole render.
|
|
130
|
+
*/
|
|
131
|
+
export async function renderAudioTimeline(
|
|
132
|
+
clips: AudioTimelineClip[],
|
|
133
|
+
buffers: Map<string, ArrayBuffer>,
|
|
134
|
+
totalDurationSec: number,
|
|
135
|
+
sampleRate: number = EXPORT_AUDIO_SAMPLE_RATE,
|
|
136
|
+
): Promise<AudioBuffer> {
|
|
137
|
+
const Ctor = resolveOfflineAudioContext();
|
|
138
|
+
const channels = EXPORT_AUDIO_CHANNELS;
|
|
139
|
+
const length = Math.max(1, Math.ceil(totalDurationSec * sampleRate));
|
|
140
|
+
const ctx = new Ctor(channels, length, sampleRate);
|
|
141
|
+
|
|
142
|
+
// Decode each unique source once. decodeAudioData detaches the buffer, so
|
|
143
|
+
// hand it a copy to keep the caller's ArrayBuffer reusable.
|
|
144
|
+
const decoded = new Map<string, AudioBuffer>();
|
|
145
|
+
for (const [src, data] of buffers) {
|
|
146
|
+
try {
|
|
147
|
+
decoded.set(src, await ctx.decodeAudioData(data.slice(0)));
|
|
148
|
+
} catch {
|
|
149
|
+
// Skip undecodable sources; the clip is simply omitted.
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
for (const clip of clips) {
|
|
154
|
+
const buffer = decoded.get(clip.src);
|
|
155
|
+
if (!buffer) continue;
|
|
156
|
+
const node = ctx.createBufferSource();
|
|
157
|
+
node.buffer = buffer;
|
|
158
|
+
node.connect(ctx.destination);
|
|
159
|
+
const when = Math.max(0, clip.startSec);
|
|
160
|
+
const offset = Math.max(0, clip.sourceInSec);
|
|
161
|
+
const duration = Math.max(0, clip.durationSec);
|
|
162
|
+
node.start(when, offset, duration);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return ctx.startRendering();
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// ── WebCodecs AAC encode (tier 1) ──────────────────────────────────
|
|
169
|
+
|
|
170
|
+
/** Minimal muxer surface {@link encodeAacTrack} needs. */
|
|
171
|
+
export interface AudioChunkSink {
|
|
172
|
+
addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Encode an `AudioBuffer` to AAC via WebCodecs and feed the chunks to `sink`.
|
|
177
|
+
* Resolves once every chunk has been flushed to the muxer. Throws only if the
|
|
178
|
+
* `AudioEncoder` errors — the caller wraps this so audio failure never aborts
|
|
179
|
+
* the video export.
|
|
180
|
+
*/
|
|
181
|
+
export async function encodeAacTrack(
|
|
182
|
+
audioBuffer: AudioBuffer,
|
|
183
|
+
sink: AudioChunkSink,
|
|
184
|
+
bitrate: number,
|
|
185
|
+
): Promise<void> {
|
|
186
|
+
if (typeof AudioEncoder === 'undefined' || typeof AudioData === 'undefined') {
|
|
187
|
+
throw new Error('WebCodecs AudioEncoder is not available.');
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const sampleRate = audioBuffer.sampleRate;
|
|
191
|
+
const channels = audioBuffer.numberOfChannels;
|
|
192
|
+
|
|
193
|
+
let encodeError: Error | null = null;
|
|
194
|
+
const encoder = new AudioEncoder({
|
|
195
|
+
output: (chunk, meta) => sink.addAudioChunk(chunk, meta ?? undefined),
|
|
196
|
+
error: (err) => {
|
|
197
|
+
encodeError = err instanceof Error ? err : new Error(String(err));
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
encoder.configure({ codec: 'mp4a.40.2', sampleRate, numberOfChannels: channels, bitrate });
|
|
202
|
+
|
|
203
|
+
const FRAME = 1024; // samples per AudioData chunk
|
|
204
|
+
const total = audioBuffer.length;
|
|
205
|
+
const channelData: Float32Array[] = [];
|
|
206
|
+
for (let ch = 0; ch < channels; ch++) {
|
|
207
|
+
channelData.push(audioBuffer.getChannelData(ch));
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
for (let offset = 0; offset < total; offset += FRAME) {
|
|
211
|
+
if (encodeError) break;
|
|
212
|
+
const count = Math.min(FRAME, total - offset);
|
|
213
|
+
// Planar layout: all of channel 0, then all of channel 1, …
|
|
214
|
+
const planar = new Float32Array(count * channels);
|
|
215
|
+
for (let ch = 0; ch < channels; ch++) {
|
|
216
|
+
planar.set(channelData[ch].subarray(offset, offset + count), ch * count);
|
|
217
|
+
}
|
|
218
|
+
const timestamp = Math.round((offset / sampleRate) * 1_000_000);
|
|
219
|
+
const audioData = new AudioData({
|
|
220
|
+
format: 'f32-planar',
|
|
221
|
+
sampleRate,
|
|
222
|
+
numberOfFrames: count,
|
|
223
|
+
numberOfChannels: channels,
|
|
224
|
+
timestamp,
|
|
225
|
+
data: planar,
|
|
226
|
+
});
|
|
227
|
+
encoder.encode(audioData);
|
|
228
|
+
audioData.close();
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
await encoder.flush();
|
|
232
|
+
encoder.close();
|
|
233
|
+
if (encodeError) throw encodeError;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// ── WAV serialization (tier 2 input) ───────────────────────────────
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Serialize an `AudioBuffer` to a 16-bit PCM WAV byte array. Used to hand the
|
|
240
|
+
* rendered audio to the ffmpeg.wasm fallback for muxing.
|
|
241
|
+
*/
|
|
242
|
+
export function audioBufferToWav(buffer: AudioBuffer): Uint8Array {
|
|
243
|
+
const channels = buffer.numberOfChannels;
|
|
244
|
+
const sampleRate = buffer.sampleRate;
|
|
245
|
+
const frames = buffer.length;
|
|
246
|
+
const bytesPerSample = 2;
|
|
247
|
+
const blockAlign = channels * bytesPerSample;
|
|
248
|
+
const dataSize = frames * blockAlign;
|
|
249
|
+
const out = new ArrayBuffer(44 + dataSize);
|
|
250
|
+
const view = new DataView(out);
|
|
251
|
+
|
|
252
|
+
const writeStr = (offset: number, str: string) => {
|
|
253
|
+
for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));
|
|
254
|
+
};
|
|
255
|
+
|
|
256
|
+
writeStr(0, 'RIFF');
|
|
257
|
+
view.setUint32(4, 36 + dataSize, true);
|
|
258
|
+
writeStr(8, 'WAVE');
|
|
259
|
+
writeStr(12, 'fmt ');
|
|
260
|
+
view.setUint32(16, 16, true); // PCM chunk size
|
|
261
|
+
view.setUint16(20, 1, true); // PCM format
|
|
262
|
+
view.setUint16(22, channels, true);
|
|
263
|
+
view.setUint32(24, sampleRate, true);
|
|
264
|
+
view.setUint32(28, sampleRate * blockAlign, true); // byte rate
|
|
265
|
+
view.setUint16(32, blockAlign, true);
|
|
266
|
+
view.setUint16(34, bytesPerSample * 8, true);
|
|
267
|
+
writeStr(36, 'data');
|
|
268
|
+
view.setUint32(40, dataSize, true);
|
|
269
|
+
|
|
270
|
+
const channelData: Float32Array[] = [];
|
|
271
|
+
for (let ch = 0; ch < channels; ch++) channelData.push(buffer.getChannelData(ch));
|
|
272
|
+
|
|
273
|
+
let pos = 44;
|
|
274
|
+
for (let i = 0; i < frames; i++) {
|
|
275
|
+
for (let ch = 0; ch < channels; ch++) {
|
|
276
|
+
let sample = channelData[ch][i];
|
|
277
|
+
sample = Math.max(-1, Math.min(1, sample));
|
|
278
|
+
view.setInt16(pos, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
|
|
279
|
+
pos += 2;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
return new Uint8Array(out);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// ── ffmpeg.wasm mux fallback (tier 2) ──────────────────────────────
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Mux a pre-encoded video-only MP4 with a WAV audio track in a single
|
|
290
|
+
* ffmpeg.wasm pass (`-c:v copy -c:a aac -shortest`). Requires
|
|
291
|
+
* `SharedArrayBuffer` (cross-origin isolation). Returns the muxed MP4 bytes.
|
|
292
|
+
*/
|
|
293
|
+
export async function muxAudioWithFfmpegWasm(
|
|
294
|
+
videoMp4: Uint8Array,
|
|
295
|
+
wav: Uint8Array,
|
|
296
|
+
audioBitrate: number,
|
|
297
|
+
): Promise<Uint8Array> {
|
|
298
|
+
const { FFmpeg } = await import('@ffmpeg/ffmpeg');
|
|
299
|
+
const ffmpeg = new FFmpeg();
|
|
300
|
+
await ffmpeg.load();
|
|
301
|
+
try {
|
|
302
|
+
await ffmpeg.writeFile('video.mp4', videoMp4);
|
|
303
|
+
await ffmpeg.writeFile('audio.wav', wav);
|
|
304
|
+
await ffmpeg.exec([
|
|
305
|
+
'-i',
|
|
306
|
+
'video.mp4',
|
|
307
|
+
'-i',
|
|
308
|
+
'audio.wav',
|
|
309
|
+
'-c:v',
|
|
310
|
+
'copy',
|
|
311
|
+
'-c:a',
|
|
312
|
+
'aac',
|
|
313
|
+
'-b:a',
|
|
314
|
+
String(audioBitrate),
|
|
315
|
+
'-shortest',
|
|
316
|
+
'out.mp4',
|
|
317
|
+
]);
|
|
318
|
+
const data = await ffmpeg.readFile('out.mp4');
|
|
319
|
+
return data instanceof Uint8Array ? data : new TextEncoder().encode(data as string);
|
|
320
|
+
} finally {
|
|
321
|
+
ffmpeg.terminate();
|
|
322
|
+
}
|
|
323
|
+
}
|
|
@@ -152,7 +152,7 @@ export function useFrameCapture(): FrameCaptureHandle {
|
|
|
152
152
|
const captionStyle: CaptionStyle = captionMode === 'social' ? 'social' : 'standard';
|
|
153
153
|
|
|
154
154
|
const playerElement = createElement(DocPlayer, {
|
|
155
|
-
|
|
155
|
+
doc,
|
|
156
156
|
basePath: '.',
|
|
157
157
|
renderMode: true,
|
|
158
158
|
showControls: false,
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
import { useState, useRef, useCallback, useEffect } from 'react';
|
|
19
19
|
import type { Doc } from '@bendyline/squisq/schemas';
|
|
20
20
|
import type { MediaProvider } from '@bendyline/squisq/schemas';
|
|
21
|
-
import type { VideoQuality, VideoOrientation } from '@bendyline/squisq-video';
|
|
22
|
-
import { resolveDimensions } from '@bendyline/squisq-video';
|
|
21
|
+
import type { VideoQuality, VideoOrientation, AudioTimelineClip } from '@bendyline/squisq-video';
|
|
22
|
+
import { resolveDimensions, computeAudioTimeline, QUALITY_PRESETS } from '@bendyline/squisq-video';
|
|
23
23
|
import type { CaptionMode } from '@bendyline/squisq-react';
|
|
24
24
|
import {
|
|
25
25
|
createEncoder,
|
|
@@ -28,8 +28,52 @@ import {
|
|
|
28
28
|
type MainThreadEncoder,
|
|
29
29
|
} from '../mainThreadEncoder.js';
|
|
30
30
|
import { createWorkerEncoder } from '../workerEncoder.js';
|
|
31
|
+
import {
|
|
32
|
+
supportsWebCodecsAac,
|
|
33
|
+
selectAudioTier,
|
|
34
|
+
renderAudioTimeline,
|
|
35
|
+
encodeAacTrack,
|
|
36
|
+
audioBufferToWav,
|
|
37
|
+
muxAudioWithFfmpegWasm,
|
|
38
|
+
EXPORT_AUDIO_SAMPLE_RATE,
|
|
39
|
+
EXPORT_AUDIO_CHANNELS,
|
|
40
|
+
} from '../audioTrack.js';
|
|
31
41
|
import { useFrameCapture } from './useFrameCapture.js';
|
|
32
42
|
|
|
43
|
+
// ── Audio resolution ───────────────────────────────────────────────
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Resolve the raw bytes for every unique source in the audio timeline from
|
|
47
|
+
* (in order) the pre-collected audio map, the images map, then the
|
|
48
|
+
* MediaProvider. Sources that can't be resolved are simply omitted — the
|
|
49
|
+
* caller degrades gracefully rather than failing.
|
|
50
|
+
*/
|
|
51
|
+
async function resolveAudioBuffers(
|
|
52
|
+
clips: AudioTimelineClip[],
|
|
53
|
+
sources: {
|
|
54
|
+
audio?: Map<string, ArrayBuffer>;
|
|
55
|
+
images?: Map<string, ArrayBuffer>;
|
|
56
|
+
mediaProvider?: MediaProvider;
|
|
57
|
+
},
|
|
58
|
+
): Promise<Map<string, ArrayBuffer>> {
|
|
59
|
+
const srcs = new Set(clips.map((c) => c.src));
|
|
60
|
+
const out = new Map<string, ArrayBuffer>();
|
|
61
|
+
for (const src of srcs) {
|
|
62
|
+
let data = sources.audio?.get(src) ?? sources.images?.get(src);
|
|
63
|
+
if (!data && sources.mediaProvider) {
|
|
64
|
+
try {
|
|
65
|
+
const url = await sources.mediaProvider.resolveUrl(src);
|
|
66
|
+
const res = await fetch(url);
|
|
67
|
+
if (res.ok) data = await res.arrayBuffer();
|
|
68
|
+
} catch {
|
|
69
|
+
// Unresolvable source; skip it.
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (data) out.set(src, data);
|
|
73
|
+
}
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
76
|
+
|
|
33
77
|
// ── Types ──────────────────────────────────────────────────────────
|
|
34
78
|
|
|
35
79
|
export type VideoExportState =
|
|
@@ -80,6 +124,19 @@ export interface VideoExportResult {
|
|
|
80
124
|
downloadUrl: string | null;
|
|
81
125
|
/** File size in bytes (populated when state === 'complete') */
|
|
82
126
|
fileSize: number;
|
|
127
|
+
/**
|
|
128
|
+
* Whether an audio track was muxed into the exported MP4 (populated when
|
|
129
|
+
* state === 'complete'). False when the doc had no audio or when audio was
|
|
130
|
+
* skipped/degraded — see {@link audioSkippedReason}.
|
|
131
|
+
*/
|
|
132
|
+
audioIncluded: boolean;
|
|
133
|
+
/**
|
|
134
|
+
* Why audio is absent, when `audioIncluded` is false. `null` means the doc
|
|
135
|
+
* simply had no audio (not a failure); a non-null string explains a genuine
|
|
136
|
+
* capability shortfall or runtime failure. Audio problems never fail the
|
|
137
|
+
* export — the video always completes.
|
|
138
|
+
*/
|
|
139
|
+
audioSkippedReason: string | null;
|
|
83
140
|
/** Error message (populated when state === 'error') */
|
|
84
141
|
error: string | null;
|
|
85
142
|
/** Seconds elapsed since export started */
|
|
@@ -104,6 +161,8 @@ export function useVideoExport(): VideoExportResult {
|
|
|
104
161
|
const [backend, setBackend] = useState<'webcodecs' | 'ffmpeg-wasm' | null>(null);
|
|
105
162
|
const [downloadUrl, setDownloadUrl] = useState<string | null>(null);
|
|
106
163
|
const [fileSize, setFileSize] = useState(0);
|
|
164
|
+
const [audioIncluded, setAudioIncluded] = useState(false);
|
|
165
|
+
const [audioSkippedReason, setAudioSkippedReason] = useState<string | null>(null);
|
|
107
166
|
const [error, setError] = useState<string | null>(null);
|
|
108
167
|
|
|
109
168
|
const [elapsed, setElapsed] = useState(0);
|
|
@@ -148,6 +207,8 @@ export function useVideoExport(): VideoExportResult {
|
|
|
148
207
|
setBackend(null);
|
|
149
208
|
setDownloadUrl(null);
|
|
150
209
|
setFileSize(0);
|
|
210
|
+
setAudioIncluded(false);
|
|
211
|
+
setAudioSkippedReason(null);
|
|
151
212
|
setError(null);
|
|
152
213
|
setElapsed(0);
|
|
153
214
|
setEstimatedRemaining(0);
|
|
@@ -178,6 +239,8 @@ export function useVideoExport(): VideoExportResult {
|
|
|
178
239
|
}
|
|
179
240
|
setDownloadUrl(null);
|
|
180
241
|
setFileSize(0);
|
|
242
|
+
setAudioIncluded(false);
|
|
243
|
+
setAudioSkippedReason(null);
|
|
181
244
|
setError(null);
|
|
182
245
|
|
|
183
246
|
const quality = config.quality ?? 'normal';
|
|
@@ -250,9 +313,79 @@ export function useVideoExport(): VideoExportResult {
|
|
|
250
313
|
const canUseWebCodecs =
|
|
251
314
|
webCodecsAvailable && (await supportsWebCodecsH264({ width, height, fps, quality }));
|
|
252
315
|
|
|
316
|
+
// ── Audio: tier selection + (best-effort) render ──────────
|
|
317
|
+
// The browser frame-capture path has no cover pre-roll (Playwright
|
|
318
|
+
// only), so the timeline is unshifted. Every audio operation below is
|
|
319
|
+
// wrapped so a failure degrades to a silent video with a reason —
|
|
320
|
+
// audio never aborts the export.
|
|
321
|
+
const audioBitrate = (QUALITY_PRESETS[quality] ?? QUALITY_PRESETS.normal).audioBitrate;
|
|
322
|
+
const timeline = computeAudioTimeline(doc, 0);
|
|
323
|
+
const aacSupported =
|
|
324
|
+
timeline.length > 0
|
|
325
|
+
? await supportsWebCodecsAac(EXPORT_AUDIO_SAMPLE_RATE, EXPORT_AUDIO_CHANNELS)
|
|
326
|
+
: false;
|
|
327
|
+
const tierDecision = selectAudioTier({
|
|
328
|
+
hasClips: timeline.length > 0,
|
|
329
|
+
aacSupported,
|
|
330
|
+
sharedArrayBufferAvailable,
|
|
331
|
+
canUseMainThreadWebCodecs: canUseWebCodecs,
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
let renderedAudio: AudioBuffer | null = null;
|
|
335
|
+
let audioIncludedLocal = false;
|
|
336
|
+
let audioReasonLocal: string | null = tierDecision.reason;
|
|
337
|
+
|
|
338
|
+
if (tierDecision.tier === 1 || tierDecision.tier === 2) {
|
|
339
|
+
setPhase('Preparing audio…');
|
|
340
|
+
try {
|
|
341
|
+
const buffers = await resolveAudioBuffers(timeline, {
|
|
342
|
+
audio: config.audio,
|
|
343
|
+
images,
|
|
344
|
+
mediaProvider: config.mediaProvider,
|
|
345
|
+
});
|
|
346
|
+
if (buffers.size === 0) {
|
|
347
|
+
audioReasonLocal = 'Audio files for this document could not be loaded.';
|
|
348
|
+
} else {
|
|
349
|
+
const totalAudioDur = timeline.reduce(
|
|
350
|
+
(max, c) => Math.max(max, c.startSec + c.durationSec),
|
|
351
|
+
docDuration,
|
|
352
|
+
);
|
|
353
|
+
renderedAudio = await renderAudioTimeline(
|
|
354
|
+
timeline,
|
|
355
|
+
buffers,
|
|
356
|
+
totalAudioDur,
|
|
357
|
+
EXPORT_AUDIO_SAMPLE_RATE,
|
|
358
|
+
);
|
|
359
|
+
}
|
|
360
|
+
} catch (audioErr: unknown) {
|
|
361
|
+
renderedAudio = null;
|
|
362
|
+
audioReasonLocal = `Audio could not be prepared: ${
|
|
363
|
+
audioErr instanceof Error ? audioErr.message : String(audioErr)
|
|
364
|
+
}`;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
const useInlineAudio = renderedAudio !== null && tierDecision.tier === 1;
|
|
369
|
+
const useFfmpegAudio = renderedAudio !== null && tierDecision.tier === 2;
|
|
370
|
+
|
|
371
|
+
if (cancelledRef.current) return;
|
|
372
|
+
|
|
253
373
|
let encoder: MainThreadEncoder;
|
|
254
374
|
if (canUseWebCodecs) {
|
|
255
|
-
encoder = createEncoder({
|
|
375
|
+
encoder = createEncoder({
|
|
376
|
+
width,
|
|
377
|
+
height,
|
|
378
|
+
fps,
|
|
379
|
+
quality,
|
|
380
|
+
...(useInlineAudio && renderedAudio
|
|
381
|
+
? {
|
|
382
|
+
audio: {
|
|
383
|
+
numberOfChannels: renderedAudio.numberOfChannels,
|
|
384
|
+
sampleRate: renderedAudio.sampleRate,
|
|
385
|
+
},
|
|
386
|
+
}
|
|
387
|
+
: {}),
|
|
388
|
+
});
|
|
256
389
|
setBackend('webcodecs');
|
|
257
390
|
} else if (sharedArrayBufferAvailable) {
|
|
258
391
|
const workerEncoder = createWorkerEncoder({ width, height, fps, quality });
|
|
@@ -315,23 +448,69 @@ export function useVideoExport(): VideoExportResult {
|
|
|
315
448
|
|
|
316
449
|
if (cancelledRef.current) return;
|
|
317
450
|
|
|
451
|
+
// ── Step 4a: Inline audio (tier 1) ────────────────────────
|
|
452
|
+
// Encode the rendered audio into the same muxer before finalizing.
|
|
453
|
+
if (useInlineAudio && renderedAudio && encoder.addAudioChunk) {
|
|
454
|
+
setState('encoding');
|
|
455
|
+
setPhase('Encoding audio…');
|
|
456
|
+
try {
|
|
457
|
+
await encodeAacTrack(
|
|
458
|
+
renderedAudio,
|
|
459
|
+
{ addAudioChunk: encoder.addAudioChunk.bind(encoder) },
|
|
460
|
+
audioBitrate,
|
|
461
|
+
);
|
|
462
|
+
audioIncludedLocal = true;
|
|
463
|
+
} catch (audioErr: unknown) {
|
|
464
|
+
audioIncludedLocal = false;
|
|
465
|
+
audioReasonLocal = `Audio encoding failed: ${
|
|
466
|
+
audioErr instanceof Error ? audioErr.message : String(audioErr)
|
|
467
|
+
}`;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
318
471
|
// ── Step 4: Finalize MP4 ──────────────────────────────────
|
|
319
472
|
setState('encoding');
|
|
320
473
|
setPhase('Finalizing video…');
|
|
321
474
|
setProgress(95);
|
|
322
475
|
|
|
323
|
-
|
|
476
|
+
let outputBytes: ArrayBuffer | Uint8Array = await encoder.finalize();
|
|
324
477
|
encoderRef.current = null;
|
|
325
478
|
|
|
326
479
|
if (cancelledRef.current) return;
|
|
327
480
|
|
|
481
|
+
// ── Step 4b: ffmpeg.wasm audio mux (tier 2) ───────────────
|
|
482
|
+
// Video is finalized; add the audio in a single copy-video pass.
|
|
483
|
+
if (useFfmpegAudio && renderedAudio) {
|
|
484
|
+
setPhase('Muxing audio…');
|
|
485
|
+
try {
|
|
486
|
+
const wav = audioBufferToWav(renderedAudio);
|
|
487
|
+
const videoOnly =
|
|
488
|
+
outputBytes instanceof Uint8Array ? outputBytes : new Uint8Array(outputBytes);
|
|
489
|
+
outputBytes = await muxAudioWithFfmpegWasm(videoOnly, wav, audioBitrate);
|
|
490
|
+
audioIncludedLocal = true;
|
|
491
|
+
} catch (audioErr: unknown) {
|
|
492
|
+
audioIncludedLocal = false;
|
|
493
|
+
audioReasonLocal = `Audio muxing failed: ${
|
|
494
|
+
audioErr instanceof Error ? audioErr.message : String(audioErr)
|
|
495
|
+
}`;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
if (cancelledRef.current) return;
|
|
500
|
+
|
|
328
501
|
// ── Step 5: Create download URL ───────────────────────────
|
|
329
|
-
|
|
502
|
+
// Normalize to a plain ArrayBuffer-backed view for Blob (ffmpeg.wasm
|
|
503
|
+
// output may be typed over SharedArrayBuffer).
|
|
504
|
+
const finalBytes =
|
|
505
|
+
outputBytes instanceof Uint8Array ? outputBytes.slice() : new Uint8Array(outputBytes);
|
|
506
|
+
const blob = new Blob([finalBytes], { type: 'video/mp4' });
|
|
330
507
|
const url = URL.createObjectURL(blob);
|
|
331
508
|
downloadUrlRef.current = url;
|
|
332
509
|
|
|
333
510
|
setDownloadUrl(url);
|
|
334
|
-
setFileSize(
|
|
511
|
+
setFileSize(finalBytes.byteLength);
|
|
512
|
+
setAudioIncluded(audioIncludedLocal);
|
|
513
|
+
setAudioSkippedReason(audioIncludedLocal ? null : audioReasonLocal);
|
|
335
514
|
setState('complete');
|
|
336
515
|
setProgress(100);
|
|
337
516
|
setPhase('Export complete');
|
|
@@ -367,6 +546,8 @@ export function useVideoExport(): VideoExportResult {
|
|
|
367
546
|
backend,
|
|
368
547
|
downloadUrl,
|
|
369
548
|
fileSize,
|
|
549
|
+
audioIncluded,
|
|
550
|
+
audioSkippedReason,
|
|
370
551
|
error,
|
|
371
552
|
elapsed,
|
|
372
553
|
estimatedRemaining,
|
package/src/index.ts
CHANGED
|
@@ -31,5 +31,8 @@ export { useFrameCapture } from './hooks/useFrameCapture.js';
|
|
|
31
31
|
export type { FrameCaptureHandle } from './hooks/useFrameCapture.js';
|
|
32
32
|
|
|
33
33
|
// ── Encoder Utilities (for advanced usage) ─────────────────────────
|
|
34
|
-
export { supportsWebCodecs, createEncoder } from './mainThreadEncoder.js';
|
|
35
|
-
export type { MainThreadEncoder } from './mainThreadEncoder.js';
|
|
34
|
+
export { supportsWebCodecs, supportsWebCodecsH264, createEncoder } from './mainThreadEncoder.js';
|
|
35
|
+
export type { MainThreadEncoder, EncoderConfig } from './mainThreadEncoder.js';
|
|
36
|
+
|
|
37
|
+
// ── Audio (capability probe) ───────────────────────────────────────
|
|
38
|
+
export { supportsWebCodecsAac } from './audioTrack.js';
|