@bendyline/squisq-video-react 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,323 @@
1
+ /**
2
+ * audioTrack — In-browser audio rendering + AAC encoding for MP4 export.
3
+ *
4
+ * Replaces the CLI's ffmpeg `adelay`/`atrim`/`amix` graph with browser-native
5
+ * primitives:
6
+ *
7
+ * - {@link supportsWebCodecsAac} probes whether the browser can encode AAC
8
+ * (WebCodecs `AudioEncoder`).
9
+ * - {@link renderAudioTimeline} mixes an {@link AudioTimelineClip} list into a
10
+ * single `AudioBuffer` using an `OfflineAudioContext` (decode → schedule →
11
+ * render).
12
+ * - {@link encodeAacTrack} feeds that buffer through a WebCodecs
13
+ * `AudioEncoder` and hands the chunks to an mp4-muxer audio track (tier 1).
14
+ * - {@link audioBufferToWav} + {@link muxAudioWithFfmpegWasm} cover the
15
+ * ffmpeg.wasm fallback (tier 2).
16
+ * - {@link selectAudioTier} is the pure capability→tier decision shared with
17
+ * `useVideoExport` (and unit-tested at the module boundary).
18
+ */
19
+
20
+ import type { AudioTimelineClip } from '@bendyline/squisq-video';
21
+
22
+ /** Sample rate the export renders + encodes audio at. */
23
+ export const EXPORT_AUDIO_SAMPLE_RATE = 48_000;
24
+ /** Channel count the export renders + encodes audio at (stereo). */
25
+ export const EXPORT_AUDIO_CHANNELS = 2;
26
+
27
+ /** Tier-3 reason used when AAC encoding is impossible and no ffmpeg fallback. */
28
+ export const REASON_NO_AAC_NO_SAB =
29
+ 'This browser cannot encode AAC audio (WebCodecs AudioEncoder unavailable) ' +
30
+ 'and the ffmpeg.wasm fallback requires cross-origin isolation (SharedArrayBuffer).';
31
+
32
+ // ── Capability probe ───────────────────────────────────────────────
33
+
34
+ /**
35
+ * Whether the browser can encode AAC (`mp4a.40.2`) via WebCodecs.
36
+ * Returns false when `AudioEncoder` is undefined (e.g. Node/jsdom, Safari,
37
+ * older Chromium) or the config is unsupported.
38
+ */
39
+ export async function supportsWebCodecsAac(
40
+ sampleRate: number = EXPORT_AUDIO_SAMPLE_RATE,
41
+ channels: number = EXPORT_AUDIO_CHANNELS,
42
+ ): Promise<boolean> {
43
+ if (typeof AudioEncoder === 'undefined') return false;
44
+ try {
45
+ const support = await AudioEncoder.isConfigSupported({
46
+ codec: 'mp4a.40.2',
47
+ sampleRate,
48
+ numberOfChannels: channels,
49
+ bitrate: 128_000,
50
+ });
51
+ return support.supported === true;
52
+ } catch {
53
+ return false;
54
+ }
55
+ }
56
+
57
+ // ── Tier selection (pure) ──────────────────────────────────────────
58
+
59
+ export interface AudioTierInputs {
60
+ /** Whether the doc has any audio timeline clips to include at all. */
61
+ hasClips: boolean;
62
+ /** Whether WebCodecs AAC encoding is available. */
63
+ aacSupported: boolean;
64
+ /** Whether `SharedArrayBuffer` (ffmpeg.wasm path) is available. */
65
+ sharedArrayBufferAvailable: boolean;
66
+ /** Whether the main-thread WebCodecs H.264 encoder is in use (tier-1 host). */
67
+ canUseMainThreadWebCodecs: boolean;
68
+ }
69
+
70
+ export interface AudioTierDecision {
71
+ /**
72
+ * 1 → inline AAC into the video muxer.
73
+ * 2 → finalize video-only then ffmpeg.wasm mux pass.
74
+ * 3 → video-only (no audio).
75
+ */
76
+ tier: 1 | 2 | 3;
77
+ /**
78
+ * Reason audio will be absent, when tier 3. `null` means "there was simply
79
+ * no audio to include" (not a failure). Non-null for a genuine capability
80
+ * shortfall. Always `null` for tiers 1 and 2 (the attempt is expected to
81
+ * succeed; the caller downgrades with a fresh reason on runtime failure).
82
+ */
83
+ reason: string | null;
84
+ }
85
+
86
+ /**
87
+ * Decide which audio tier to run from capability booleans. Pure — the single
88
+ * source of truth for tier selection, shared with `useVideoExport`.
89
+ */
90
+ export function selectAudioTier(inputs: AudioTierInputs): AudioTierDecision {
91
+ if (!inputs.hasClips) {
92
+ return { tier: 3, reason: null };
93
+ }
94
+ if (inputs.aacSupported && inputs.canUseMainThreadWebCodecs) {
95
+ return { tier: 1, reason: null };
96
+ }
97
+ if (inputs.sharedArrayBufferAvailable) {
98
+ return { tier: 2, reason: null };
99
+ }
100
+ return { tier: 3, reason: REASON_NO_AAC_NO_SAB };
101
+ }
102
+
103
+ // ── OfflineAudioContext rendering ──────────────────────────────────
104
+
105
+ type OfflineCtor = new (
106
+ numberOfChannels: number,
107
+ length: number,
108
+ sampleRate: number,
109
+ ) => OfflineAudioContext;
110
+
111
+ function resolveOfflineAudioContext(): OfflineCtor {
112
+ const g = globalThis as unknown as {
113
+ OfflineAudioContext?: OfflineCtor;
114
+ webkitOfflineAudioContext?: OfflineCtor;
115
+ };
116
+ const ctor = g.OfflineAudioContext ?? g.webkitOfflineAudioContext;
117
+ if (!ctor) {
118
+ throw new Error('OfflineAudioContext is not available in this environment.');
119
+ }
120
+ return ctor;
121
+ }
122
+
123
+ /**
124
+ * Mix a list of timeline clips into one `AudioBuffer`.
125
+ *
126
+ * Decodes each unique source once, then schedules an `AudioBufferSourceNode`
127
+ * per clip at its `startSec` with the trim window (`offset = sourceInSec`,
128
+ * `duration = durationSec`). Undecodable sources are skipped rather than
129
+ * failing the whole render.
130
+ */
131
+ export async function renderAudioTimeline(
132
+ clips: AudioTimelineClip[],
133
+ buffers: Map<string, ArrayBuffer>,
134
+ totalDurationSec: number,
135
+ sampleRate: number = EXPORT_AUDIO_SAMPLE_RATE,
136
+ ): Promise<AudioBuffer> {
137
+ const Ctor = resolveOfflineAudioContext();
138
+ const channels = EXPORT_AUDIO_CHANNELS;
139
+ const length = Math.max(1, Math.ceil(totalDurationSec * sampleRate));
140
+ const ctx = new Ctor(channels, length, sampleRate);
141
+
142
+ // Decode each unique source once. decodeAudioData detaches the buffer, so
143
+ // hand it a copy to keep the caller's ArrayBuffer reusable.
144
+ const decoded = new Map<string, AudioBuffer>();
145
+ for (const [src, data] of buffers) {
146
+ try {
147
+ decoded.set(src, await ctx.decodeAudioData(data.slice(0)));
148
+ } catch {
149
+ // Skip undecodable sources; the clip is simply omitted.
150
+ }
151
+ }
152
+
153
+ for (const clip of clips) {
154
+ const buffer = decoded.get(clip.src);
155
+ if (!buffer) continue;
156
+ const node = ctx.createBufferSource();
157
+ node.buffer = buffer;
158
+ node.connect(ctx.destination);
159
+ const when = Math.max(0, clip.startSec);
160
+ const offset = Math.max(0, clip.sourceInSec);
161
+ const duration = Math.max(0, clip.durationSec);
162
+ node.start(when, offset, duration);
163
+ }
164
+
165
+ return ctx.startRendering();
166
+ }
167
+
168
+ // ── WebCodecs AAC encode (tier 1) ──────────────────────────────────
169
+
170
+ /** Minimal muxer surface {@link encodeAacTrack} needs. */
171
+ export interface AudioChunkSink {
172
+ addAudioChunk(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void;
173
+ }
174
+
175
+ /**
176
+ * Encode an `AudioBuffer` to AAC via WebCodecs and feed the chunks to `sink`.
177
+ * Resolves once every chunk has been flushed to the muxer. Throws only if the
178
+ * `AudioEncoder` errors — the caller wraps this so audio failure never aborts
179
+ * the video export.
180
+ */
181
+ export async function encodeAacTrack(
182
+ audioBuffer: AudioBuffer,
183
+ sink: AudioChunkSink,
184
+ bitrate: number,
185
+ ): Promise<void> {
186
+ if (typeof AudioEncoder === 'undefined' || typeof AudioData === 'undefined') {
187
+ throw new Error('WebCodecs AudioEncoder is not available.');
188
+ }
189
+
190
+ const sampleRate = audioBuffer.sampleRate;
191
+ const channels = audioBuffer.numberOfChannels;
192
+
193
+ let encodeError: Error | null = null;
194
+ const encoder = new AudioEncoder({
195
+ output: (chunk, meta) => sink.addAudioChunk(chunk, meta ?? undefined),
196
+ error: (err) => {
197
+ encodeError = err instanceof Error ? err : new Error(String(err));
198
+ },
199
+ });
200
+
201
+ encoder.configure({ codec: 'mp4a.40.2', sampleRate, numberOfChannels: channels, bitrate });
202
+
203
+ const FRAME = 1024; // samples per AudioData chunk
204
+ const total = audioBuffer.length;
205
+ const channelData: Float32Array[] = [];
206
+ for (let ch = 0; ch < channels; ch++) {
207
+ channelData.push(audioBuffer.getChannelData(ch));
208
+ }
209
+
210
+ for (let offset = 0; offset < total; offset += FRAME) {
211
+ if (encodeError) break;
212
+ const count = Math.min(FRAME, total - offset);
213
+ // Planar layout: all of channel 0, then all of channel 1, …
214
+ const planar = new Float32Array(count * channels);
215
+ for (let ch = 0; ch < channels; ch++) {
216
+ planar.set(channelData[ch].subarray(offset, offset + count), ch * count);
217
+ }
218
+ const timestamp = Math.round((offset / sampleRate) * 1_000_000);
219
+ const audioData = new AudioData({
220
+ format: 'f32-planar',
221
+ sampleRate,
222
+ numberOfFrames: count,
223
+ numberOfChannels: channels,
224
+ timestamp,
225
+ data: planar,
226
+ });
227
+ encoder.encode(audioData);
228
+ audioData.close();
229
+ }
230
+
231
+ await encoder.flush();
232
+ encoder.close();
233
+ if (encodeError) throw encodeError;
234
+ }
235
+
236
+ // ── WAV serialization (tier 2 input) ───────────────────────────────
237
+
238
+ /**
239
+ * Serialize an `AudioBuffer` to a 16-bit PCM WAV byte array. Used to hand the
240
+ * rendered audio to the ffmpeg.wasm fallback for muxing.
241
+ */
242
+ export function audioBufferToWav(buffer: AudioBuffer): Uint8Array {
243
+ const channels = buffer.numberOfChannels;
244
+ const sampleRate = buffer.sampleRate;
245
+ const frames = buffer.length;
246
+ const bytesPerSample = 2;
247
+ const blockAlign = channels * bytesPerSample;
248
+ const dataSize = frames * blockAlign;
249
+ const out = new ArrayBuffer(44 + dataSize);
250
+ const view = new DataView(out);
251
+
252
+ const writeStr = (offset: number, str: string) => {
253
+ for (let i = 0; i < str.length; i++) view.setUint8(offset + i, str.charCodeAt(i));
254
+ };
255
+
256
+ writeStr(0, 'RIFF');
257
+ view.setUint32(4, 36 + dataSize, true);
258
+ writeStr(8, 'WAVE');
259
+ writeStr(12, 'fmt ');
260
+ view.setUint32(16, 16, true); // PCM chunk size
261
+ view.setUint16(20, 1, true); // PCM format
262
+ view.setUint16(22, channels, true);
263
+ view.setUint32(24, sampleRate, true);
264
+ view.setUint32(28, sampleRate * blockAlign, true); // byte rate
265
+ view.setUint16(32, blockAlign, true);
266
+ view.setUint16(34, bytesPerSample * 8, true);
267
+ writeStr(36, 'data');
268
+ view.setUint32(40, dataSize, true);
269
+
270
+ const channelData: Float32Array[] = [];
271
+ for (let ch = 0; ch < channels; ch++) channelData.push(buffer.getChannelData(ch));
272
+
273
+ let pos = 44;
274
+ for (let i = 0; i < frames; i++) {
275
+ for (let ch = 0; ch < channels; ch++) {
276
+ let sample = channelData[ch][i];
277
+ sample = Math.max(-1, Math.min(1, sample));
278
+ view.setInt16(pos, sample < 0 ? sample * 0x8000 : sample * 0x7fff, true);
279
+ pos += 2;
280
+ }
281
+ }
282
+
283
+ return new Uint8Array(out);
284
+ }
285
+
286
+ // ── ffmpeg.wasm mux fallback (tier 2) ──────────────────────────────
287
+
288
+ /**
289
+ * Mux a pre-encoded video-only MP4 with a WAV audio track in a single
290
+ * ffmpeg.wasm pass (`-c:v copy -c:a aac -shortest`). Requires
291
+ * `SharedArrayBuffer` (cross-origin isolation). Returns the muxed MP4 bytes.
292
+ */
293
+ export async function muxAudioWithFfmpegWasm(
294
+ videoMp4: Uint8Array,
295
+ wav: Uint8Array,
296
+ audioBitrate: number,
297
+ ): Promise<Uint8Array> {
298
+ const { FFmpeg } = await import('@ffmpeg/ffmpeg');
299
+ const ffmpeg = new FFmpeg();
300
+ await ffmpeg.load();
301
+ try {
302
+ await ffmpeg.writeFile('video.mp4', videoMp4);
303
+ await ffmpeg.writeFile('audio.wav', wav);
304
+ await ffmpeg.exec([
305
+ '-i',
306
+ 'video.mp4',
307
+ '-i',
308
+ 'audio.wav',
309
+ '-c:v',
310
+ 'copy',
311
+ '-c:a',
312
+ 'aac',
313
+ '-b:a',
314
+ String(audioBitrate),
315
+ '-shortest',
316
+ 'out.mp4',
317
+ ]);
318
+ const data = await ffmpeg.readFile('out.mp4');
319
+ return data instanceof Uint8Array ? data : new TextEncoder().encode(data as string);
320
+ } finally {
321
+ ffmpeg.terminate();
322
+ }
323
+ }
@@ -152,7 +152,7 @@ export function useFrameCapture(): FrameCaptureHandle {
152
152
  const captionStyle: CaptionStyle = captionMode === 'social' ? 'social' : 'standard';
153
153
 
154
154
  const playerElement = createElement(DocPlayer, {
155
- script: doc,
155
+ doc,
156
156
  basePath: '.',
157
157
  renderMode: true,
158
158
  showControls: false,
@@ -18,8 +18,8 @@
18
18
  import { useState, useRef, useCallback, useEffect } from 'react';
19
19
  import type { Doc } from '@bendyline/squisq/schemas';
20
20
  import type { MediaProvider } from '@bendyline/squisq/schemas';
21
- import type { VideoQuality, VideoOrientation } from '@bendyline/squisq-video';
22
- import { resolveDimensions } from '@bendyline/squisq-video';
21
+ import type { VideoQuality, VideoOrientation, AudioTimelineClip } from '@bendyline/squisq-video';
22
+ import { resolveDimensions, computeAudioTimeline, QUALITY_PRESETS } from '@bendyline/squisq-video';
23
23
  import type { CaptionMode } from '@bendyline/squisq-react';
24
24
  import {
25
25
  createEncoder,
@@ -28,8 +28,52 @@ import {
28
28
  type MainThreadEncoder,
29
29
  } from '../mainThreadEncoder.js';
30
30
  import { createWorkerEncoder } from '../workerEncoder.js';
31
+ import {
32
+ supportsWebCodecsAac,
33
+ selectAudioTier,
34
+ renderAudioTimeline,
35
+ encodeAacTrack,
36
+ audioBufferToWav,
37
+ muxAudioWithFfmpegWasm,
38
+ EXPORT_AUDIO_SAMPLE_RATE,
39
+ EXPORT_AUDIO_CHANNELS,
40
+ } from '../audioTrack.js';
31
41
  import { useFrameCapture } from './useFrameCapture.js';
32
42
 
43
+ // ── Audio resolution ───────────────────────────────────────────────
44
+
45
+ /**
46
+ * Resolve the raw bytes for every unique source in the audio timeline from
47
+ * (in order) the pre-collected audio map, the images map, then the
48
+ * MediaProvider. Sources that can't be resolved are simply omitted — the
49
+ * caller degrades gracefully rather than failing.
50
+ */
51
+ async function resolveAudioBuffers(
52
+ clips: AudioTimelineClip[],
53
+ sources: {
54
+ audio?: Map<string, ArrayBuffer>;
55
+ images?: Map<string, ArrayBuffer>;
56
+ mediaProvider?: MediaProvider;
57
+ },
58
+ ): Promise<Map<string, ArrayBuffer>> {
59
+ const srcs = new Set(clips.map((c) => c.src));
60
+ const out = new Map<string, ArrayBuffer>();
61
+ for (const src of srcs) {
62
+ let data = sources.audio?.get(src) ?? sources.images?.get(src);
63
+ if (!data && sources.mediaProvider) {
64
+ try {
65
+ const url = await sources.mediaProvider.resolveUrl(src);
66
+ const res = await fetch(url);
67
+ if (res.ok) data = await res.arrayBuffer();
68
+ } catch {
69
+ // Unresolvable source; skip it.
70
+ }
71
+ }
72
+ if (data) out.set(src, data);
73
+ }
74
+ return out;
75
+ }
76
+
33
77
  // ── Types ──────────────────────────────────────────────────────────
34
78
 
35
79
  export type VideoExportState =
@@ -80,6 +124,19 @@ export interface VideoExportResult {
80
124
  downloadUrl: string | null;
81
125
  /** File size in bytes (populated when state === 'complete') */
82
126
  fileSize: number;
127
+ /**
128
+ * Whether an audio track was muxed into the exported MP4 (populated when
129
+ * state === 'complete'). False when the doc had no audio or when audio was
130
+ * skipped/degraded — see {@link audioSkippedReason}.
131
+ */
132
+ audioIncluded: boolean;
133
+ /**
134
+ * Why audio is absent, when `audioIncluded` is false. `null` means the doc
135
+ * simply had no audio (not a failure); a non-null string explains a genuine
136
+ * capability shortfall or runtime failure. Audio problems never fail the
137
+ * export — the video always completes.
138
+ */
139
+ audioSkippedReason: string | null;
83
140
  /** Error message (populated when state === 'error') */
84
141
  error: string | null;
85
142
  /** Seconds elapsed since export started */
@@ -104,6 +161,8 @@ export function useVideoExport(): VideoExportResult {
104
161
  const [backend, setBackend] = useState<'webcodecs' | 'ffmpeg-wasm' | null>(null);
105
162
  const [downloadUrl, setDownloadUrl] = useState<string | null>(null);
106
163
  const [fileSize, setFileSize] = useState(0);
164
+ const [audioIncluded, setAudioIncluded] = useState(false);
165
+ const [audioSkippedReason, setAudioSkippedReason] = useState<string | null>(null);
107
166
  const [error, setError] = useState<string | null>(null);
108
167
 
109
168
  const [elapsed, setElapsed] = useState(0);
@@ -148,6 +207,8 @@ export function useVideoExport(): VideoExportResult {
148
207
  setBackend(null);
149
208
  setDownloadUrl(null);
150
209
  setFileSize(0);
210
+ setAudioIncluded(false);
211
+ setAudioSkippedReason(null);
151
212
  setError(null);
152
213
  setElapsed(0);
153
214
  setEstimatedRemaining(0);
@@ -178,6 +239,8 @@ export function useVideoExport(): VideoExportResult {
178
239
  }
179
240
  setDownloadUrl(null);
180
241
  setFileSize(0);
242
+ setAudioIncluded(false);
243
+ setAudioSkippedReason(null);
181
244
  setError(null);
182
245
 
183
246
  const quality = config.quality ?? 'normal';
@@ -250,9 +313,79 @@ export function useVideoExport(): VideoExportResult {
250
313
  const canUseWebCodecs =
251
314
  webCodecsAvailable && (await supportsWebCodecsH264({ width, height, fps, quality }));
252
315
 
316
+ // ── Audio: tier selection + (best-effort) render ──────────
317
+ // The browser frame-capture path has no cover pre-roll (Playwright
318
+ // only), so the timeline is unshifted. Every audio operation below is
319
+ // wrapped so a failure degrades to a silent video with a reason —
320
+ // audio never aborts the export.
321
+ const audioBitrate = (QUALITY_PRESETS[quality] ?? QUALITY_PRESETS.normal).audioBitrate;
322
+ const timeline = computeAudioTimeline(doc, 0);
323
+ const aacSupported =
324
+ timeline.length > 0
325
+ ? await supportsWebCodecsAac(EXPORT_AUDIO_SAMPLE_RATE, EXPORT_AUDIO_CHANNELS)
326
+ : false;
327
+ const tierDecision = selectAudioTier({
328
+ hasClips: timeline.length > 0,
329
+ aacSupported,
330
+ sharedArrayBufferAvailable,
331
+ canUseMainThreadWebCodecs: canUseWebCodecs,
332
+ });
333
+
334
+ let renderedAudio: AudioBuffer | null = null;
335
+ let audioIncludedLocal = false;
336
+ let audioReasonLocal: string | null = tierDecision.reason;
337
+
338
+ if (tierDecision.tier === 1 || tierDecision.tier === 2) {
339
+ setPhase('Preparing audio…');
340
+ try {
341
+ const buffers = await resolveAudioBuffers(timeline, {
342
+ audio: config.audio,
343
+ images,
344
+ mediaProvider: config.mediaProvider,
345
+ });
346
+ if (buffers.size === 0) {
347
+ audioReasonLocal = 'Audio files for this document could not be loaded.';
348
+ } else {
349
+ const totalAudioDur = timeline.reduce(
350
+ (max, c) => Math.max(max, c.startSec + c.durationSec),
351
+ docDuration,
352
+ );
353
+ renderedAudio = await renderAudioTimeline(
354
+ timeline,
355
+ buffers,
356
+ totalAudioDur,
357
+ EXPORT_AUDIO_SAMPLE_RATE,
358
+ );
359
+ }
360
+ } catch (audioErr: unknown) {
361
+ renderedAudio = null;
362
+ audioReasonLocal = `Audio could not be prepared: ${
363
+ audioErr instanceof Error ? audioErr.message : String(audioErr)
364
+ }`;
365
+ }
366
+ }
367
+
368
+ const useInlineAudio = renderedAudio !== null && tierDecision.tier === 1;
369
+ const useFfmpegAudio = renderedAudio !== null && tierDecision.tier === 2;
370
+
371
+ if (cancelledRef.current) return;
372
+
253
373
  let encoder: MainThreadEncoder;
254
374
  if (canUseWebCodecs) {
255
- encoder = createEncoder({ width, height, fps, quality });
375
+ encoder = createEncoder({
376
+ width,
377
+ height,
378
+ fps,
379
+ quality,
380
+ ...(useInlineAudio && renderedAudio
381
+ ? {
382
+ audio: {
383
+ numberOfChannels: renderedAudio.numberOfChannels,
384
+ sampleRate: renderedAudio.sampleRate,
385
+ },
386
+ }
387
+ : {}),
388
+ });
256
389
  setBackend('webcodecs');
257
390
  } else if (sharedArrayBufferAvailable) {
258
391
  const workerEncoder = createWorkerEncoder({ width, height, fps, quality });
@@ -315,23 +448,69 @@ export function useVideoExport(): VideoExportResult {
315
448
 
316
449
  if (cancelledRef.current) return;
317
450
 
451
+ // ── Step 4a: Inline audio (tier 1) ────────────────────────
452
+ // Encode the rendered audio into the same muxer before finalizing.
453
+ if (useInlineAudio && renderedAudio && encoder.addAudioChunk) {
454
+ setState('encoding');
455
+ setPhase('Encoding audio…');
456
+ try {
457
+ await encodeAacTrack(
458
+ renderedAudio,
459
+ { addAudioChunk: encoder.addAudioChunk.bind(encoder) },
460
+ audioBitrate,
461
+ );
462
+ audioIncludedLocal = true;
463
+ } catch (audioErr: unknown) {
464
+ audioIncludedLocal = false;
465
+ audioReasonLocal = `Audio encoding failed: ${
466
+ audioErr instanceof Error ? audioErr.message : String(audioErr)
467
+ }`;
468
+ }
469
+ }
470
+
318
471
  // ── Step 4: Finalize MP4 ──────────────────────────────────
319
472
  setState('encoding');
320
473
  setPhase('Finalizing video…');
321
474
  setProgress(95);
322
475
 
323
- const mp4Buffer = await encoder.finalize();
476
+ let outputBytes: ArrayBuffer | Uint8Array = await encoder.finalize();
324
477
  encoderRef.current = null;
325
478
 
326
479
  if (cancelledRef.current) return;
327
480
 
481
+ // ── Step 4b: ffmpeg.wasm audio mux (tier 2) ───────────────
482
+ // Video is finalized; add the audio in a single copy-video pass.
483
+ if (useFfmpegAudio && renderedAudio) {
484
+ setPhase('Muxing audio…');
485
+ try {
486
+ const wav = audioBufferToWav(renderedAudio);
487
+ const videoOnly =
488
+ outputBytes instanceof Uint8Array ? outputBytes : new Uint8Array(outputBytes);
489
+ outputBytes = await muxAudioWithFfmpegWasm(videoOnly, wav, audioBitrate);
490
+ audioIncludedLocal = true;
491
+ } catch (audioErr: unknown) {
492
+ audioIncludedLocal = false;
493
+ audioReasonLocal = `Audio muxing failed: ${
494
+ audioErr instanceof Error ? audioErr.message : String(audioErr)
495
+ }`;
496
+ }
497
+ }
498
+
499
+ if (cancelledRef.current) return;
500
+
328
501
  // ── Step 5: Create download URL ───────────────────────────
329
- const blob = new Blob([mp4Buffer], { type: 'video/mp4' });
502
+ // Normalize to a plain ArrayBuffer-backed view for Blob (ffmpeg.wasm
503
+ // output may be typed over SharedArrayBuffer).
504
+ const finalBytes =
505
+ outputBytes instanceof Uint8Array ? outputBytes.slice() : new Uint8Array(outputBytes);
506
+ const blob = new Blob([finalBytes], { type: 'video/mp4' });
330
507
  const url = URL.createObjectURL(blob);
331
508
  downloadUrlRef.current = url;
332
509
 
333
510
  setDownloadUrl(url);
334
- setFileSize(mp4Buffer.byteLength);
511
+ setFileSize(finalBytes.byteLength);
512
+ setAudioIncluded(audioIncludedLocal);
513
+ setAudioSkippedReason(audioIncludedLocal ? null : audioReasonLocal);
335
514
  setState('complete');
336
515
  setProgress(100);
337
516
  setPhase('Export complete');
@@ -367,6 +546,8 @@ export function useVideoExport(): VideoExportResult {
367
546
  backend,
368
547
  downloadUrl,
369
548
  fileSize,
549
+ audioIncluded,
550
+ audioSkippedReason,
370
551
  error,
371
552
  elapsed,
372
553
  estimatedRemaining,
package/src/index.ts CHANGED
@@ -31,5 +31,8 @@ export { useFrameCapture } from './hooks/useFrameCapture.js';
31
31
  export type { FrameCaptureHandle } from './hooks/useFrameCapture.js';
32
32
 
33
33
  // ── Encoder Utilities (for advanced usage) ─────────────────────────
34
- export { supportsWebCodecs, createEncoder } from './mainThreadEncoder.js';
35
- export type { MainThreadEncoder } from './mainThreadEncoder.js';
34
+ export { supportsWebCodecs, supportsWebCodecsH264, createEncoder } from './mainThreadEncoder.js';
35
+ export type { MainThreadEncoder, EncoderConfig } from './mainThreadEncoder.js';
36
+
37
+ // ── Audio (capability probe) ───────────────────────────────────────
38
+ export { supportsWebCodecsAac } from './audioTrack.js';