@remotion/media 4.0.378 → 4.0.380

package/dist/audio/audio-for-preview.js

@@ -261,7 +261,7 @@ const AudioForPreviewAssertedShowing = ({ src, playbackRate, logLevel, muted, vo
         mediaPlayer.setIsPostmounting(isPostmounting);
     }, [isPostmounting, mediaPlayerReady]);
     if (shouldFallbackToNativeAudio && !disallowFallbackToHtml5Audio) {
-        return (_jsx(RemotionAudio, { src: src, muted: muted, volume: volume, startFrom: trimBefore, endAt: trimAfter, playbackRate: playbackRate, loopVolumeCurveBehavior: loopVolumeCurveBehavior, name: name, loop: loop, showInTimeline: showInTimeline, stack: stack ?? undefined, toneFrequency: toneFrequency, audioStreamIndex: audioStreamIndex, pauseWhenBuffering: fallbackHtml5AudioProps?.pauseWhenBuffering, ...fallbackHtml5AudioProps }));
+        return (_jsx(RemotionAudio, { src: src, muted: muted, volume: volume, startFrom: trimBefore, endAt: trimAfter, playbackRate: playbackRate, loopVolumeCurveBehavior: loopVolumeCurveBehavior, name: name, loop: loop, showInTimeline: showInTimeline, stack: stack ?? undefined, toneFrequency: toneFrequency, audioStreamIndex: audioStreamIndex, pauseWhenBuffering: fallbackHtml5AudioProps?.pauseWhenBuffering, crossOrigin: fallbackHtml5AudioProps?.crossOrigin, ...fallbackHtml5AudioProps }));
     }
     return null;
 };

package/dist/audio/audio-preview-iterator.js

@@ -6,6 +6,7 @@ export const makeAudioIterator = (audioSink, startFromSecond) => {
     const iterator = audioSink.buffers(startFromSecond);
     const queuedAudioNodes = [];
     const audioChunksForAfterResuming = [];
+    let mostRecentTimestamp = -Infinity;
     const cleanupAudioQueue = () => {
         for (const node of queuedAudioNodes) {
             node.node.stop();
@@ -31,12 +32,25 @@ export const makeAudioIterator = (audioSink, startFromSecond) => {
                 },
             };
         }
+        if (result.value) {
+            mostRecentTimestamp = Math.max(mostRecentTimestamp, result.value.timestamp + result.value.duration);
+            return {
+                type: 'got-buffer',
+                buffer: result.value,
+            };
+        }
         return {
-            type: 'got-buffer-or-end',
-            buffer: result.value ?? null,
+            type: 'got-end',
+            mostRecentTimestamp,
         };
     };
     const tryToSatisfySeek = async (time, allowWait, onBufferScheduled) => {
+        if (time < startFromSecond) {
+            return {
+                type: 'not-satisfied',
+                reason: `time requested is before the start of the iterator`,
+            };
+        }
         while (true) {
             const buffer = await getNextOrNullIfNotAvailable(allowWait);
             if (buffer.type === 'need-to-wait-for-it') {
@@ -45,12 +59,18 @@ export const makeAudioIterator = (audioSink, startFromSecond) => {
                     reason: 'iterator did not have buffer ready',
                 };
             }
-            if (buffer.type === 'got-buffer-or-end') {
-                if (buffer.buffer === null) {
+            if (buffer.type === 'got-end') {
+                if (time >= mostRecentTimestamp) {
                     return {
                         type: 'ended',
                     };
                 }
+                return {
+                    type: 'not-satisfied',
+                    reason: `iterator ended before the requested time`,
+                };
+            }
+            if (buffer.type === 'got-buffer') {
                 const bufferTimestamp = roundTo4Digits(buffer.buffer.timestamp);
                 const bufferEndTimestamp = roundTo4Digits(buffer.buffer.timestamp + buffer.buffer.duration);
                 const timestamp = roundTo4Digits(time);
@@ -101,6 +121,9 @@ export const makeAudioIterator = (audioSink, startFromSecond) => {
         },
         getNext: async () => {
             const next = await iterator.next();
+            if (next.value) {
+                mostRecentTimestamp = Math.max(mostRecentTimestamp, next.value.timestamp + next.value.duration);
+            }
             return next;
         },
         isDestroyed: () => {

package/dist/audio/props.d.ts

@@ -1,5 +1,6 @@
 import type { LogLevel, LoopVolumeCurveBehavior, VolumeProp } from 'remotion';
 export type FallbackHtml5AudioProps = {
+    crossOrigin?: '' | 'anonymous' | 'use-credentials' | undefined;
     onError?: (err: Error) => void;
     useWebAudioApi?: boolean;
     acceptableTimeShiftInSeconds?: number;

package/dist/audio-extraction/extract-audio.d.ts

@@ -17,6 +17,6 @@ type ExtractAudioParams = {
 declare const extractAudioInternal: ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds: durationNotYetApplyingPlaybackRate, logLevel, loop, playbackRate, audioStreamIndex, trimBefore, trimAfter, fps, maxCacheSize, }: ExtractAudioParams) => Promise<{
     data: PcmS16AudioData | null;
     durationInSeconds: number | null;
-} | "cannot-decode" | "unknown-container-format">;
+} | "cannot-decode" | "unknown-container-format" | "network-error">;
 export declare const extractAudio: (params: ExtractAudioParams) => Promise<ExtractAudioReturnType>;
 export {};

package/dist/audio-extraction/extract-audio.js

@@ -11,6 +11,9 @@ const extractAudioInternal = async ({ src, timeInSeconds: unloopedTimeInSeconds,
         mediaDurationInSeconds = await getDuration();
     }
     const audio = await getAudio(audioStreamIndex);
+    if (audio === 'network-error') {
+        return 'network-error';
+    }
     if (audio === 'no-audio-track') {
         return { data: null, durationInSeconds: null };
     }

package/dist/audio-for-rendering.d.ts

@@ -0,0 +1,3 @@
+import type React from 'react';
+import type { AudioProps } from './props';
+export declare const AudioForRendering: React.FC<AudioProps>;

package/dist/audio-for-rendering.js

@@ -0,0 +1,94 @@
+import { useContext, useLayoutEffect, useMemo, useState } from 'react';
+import { cancelRender, Internals, useCurrentFrame, useDelayRender, useRemotionEnvironment, } from 'remotion';
+import { extractFrameViaBroadcastChannel } from './extract-frame-via-broadcast-channel';
+export const AudioForRendering = ({ volume: volumeProp, playbackRate, src, muted, loopVolumeCurveBehavior, delayRenderRetries, delayRenderTimeoutInMilliseconds, logLevel = window.remotion_logLevel, }) => {
+    const absoluteFrame = Internals.useTimelinePosition();
+    const videoConfig = Internals.useUnsafeVideoConfig();
+    const { registerRenderAsset, unregisterRenderAsset } = useContext(Internals.RenderAssetManager);
+    const frame = useCurrentFrame();
+    const volumePropsFrame = Internals.useFrameForVolumeProp(loopVolumeCurveBehavior ?? 'repeat');
+    const environment = useRemotionEnvironment();
+    const [id] = useState(() => `${Math.random()}`.replace('0.', ''));
+    if (!videoConfig) {
+        throw new Error('No video config found');
+    }
+    if (!src) {
+        throw new TypeError('No `src` was passed to <Video>.');
+    }
+    const volume = Internals.evaluateVolume({
+        volume: volumeProp,
+        frame: volumePropsFrame,
+        mediaVolume: 1,
+    });
+    Internals.warnAboutTooHighVolume(volume);
+    const shouldRenderAudio = useMemo(() => {
+        if (!window.remotion_audioEnabled) {
+            return false;
+        }
+        if (muted) {
+            return false;
+        }
+        if (volume <= 0) {
+            return false;
+        }
+        return true;
+    }, [muted, volume]);
+    const { fps } = videoConfig;
+    const { delayRender, continueRender } = useDelayRender();
+    useLayoutEffect(() => {
+        const actualFps = playbackRate ? fps / playbackRate : fps;
+        const timestamp = frame / actualFps;
+        const durationInSeconds = 1 / actualFps;
+        const newHandle = delayRender(`Extracting frame number ${frame}`, {
+            retries: delayRenderRetries ?? undefined,
+            timeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? undefined,
+        });
+        extractFrameViaBroadcastChannel({
+            src,
+            timeInSeconds: timestamp,
+            durationInSeconds,
+            logLevel: logLevel ?? 'info',
+            shouldRenderAudio,
+            isClientSideRendering: environment.isClientSideRendering,
+        })
+            .then(({ audio }) => {
+            if (audio) {
+                registerRenderAsset({
+                    type: 'inline-audio',
+                    id,
+                    audio: Array.from(audio.data),
+                    sampleRate: audio.sampleRate,
+                    numberOfChannels: audio.numberOfChannels,
+                    frame: absoluteFrame,
+                    timestamp: audio.timestamp,
+                    duration: (audio.numberOfFrames / audio.sampleRate) * 1000000,
+                });
+            }
+            continueRender(newHandle);
+        })
+            .catch((error) => {
+            cancelRender(error);
+        });
+        return () => {
+            continueRender(newHandle);
+            unregisterRenderAsset(id);
+        };
+    }, [
+        absoluteFrame,
+        continueRender,
+        delayRender,
+        delayRenderRetries,
+        delayRenderTimeoutInMilliseconds,
+        environment.isClientSideRendering,
+        fps,
+        frame,
+        id,
+        logLevel,
+        playbackRate,
+        registerRenderAsset,
+        shouldRenderAudio,
+        src,
+        unregisterRenderAsset,
+    ]);
+    return null;
+};

package/dist/audio.d.ts

@@ -0,0 +1,3 @@
+import React from 'react';
+import type { AudioProps } from './props';
+export declare const Audio: React.FC<AudioProps>;

package/dist/audio.js

@@ -0,0 +1,60 @@
+import { jsx as _jsx } from "react/jsx-runtime";
+import { useCallback, useContext } from 'react';
+import { cancelRender, Internals, Sequence, useRemotionEnvironment, } from 'remotion';
+import { SharedAudioContext } from '../../core/src/audio/shared-audio-tags';
+import { AudioForRendering } from './audio-for-rendering';
+const { validateMediaTrimProps, resolveTrimProps, validateMediaProps, AudioForPreview, } = Internals;
+export const Audio = (props) => {
+    const audioContext = useContext(SharedAudioContext);
+    // Should only destruct `trimBefore` and `trimAfter` from props,
+    // rest gets drilled down
+    const { trimBefore, trimAfter, name, pauseWhenBuffering, stack, showInTimeline, onError: onRemotionError, loop, ...otherProps } = props;
+    const environment = useRemotionEnvironment();
+    const onDuration = useCallback(() => undefined, []);
+    if (typeof props.src !== 'string') {
+        throw new TypeError(`The \`<Audio>\` tag requires a string for \`src\`, but got ${JSON.stringify(props.src)} instead.`);
+    }
+    validateMediaTrimProps({
+        startFrom: undefined,
+        endAt: undefined,
+        trimBefore,
+        trimAfter,
+    });
+    const { trimBeforeValue, trimAfterValue } = resolveTrimProps({
+        startFrom: undefined,
+        endAt: undefined,
+        trimBefore,
+        trimAfter,
+    });
+    const onError = useCallback((e) => {
+        // eslint-disable-next-line no-console
+        console.log(e.currentTarget.error);
+        // If there is no `loop` property, we don't need to get the duration
+        // and this does not need to be a fatal error
+        const errMessage = `Could not play audio: ${e.currentTarget.error}. See https://remotion.dev/docs/media-playback-error for help.`;
+        if (loop) {
+            if (onRemotionError) {
+                onRemotionError(new Error(errMessage));
+                return;
+            }
+            cancelRender(new Error(errMessage));
+        }
+        else {
+            onRemotionError?.(new Error(errMessage));
+            // eslint-disable-next-line no-console
+            console.warn(errMessage);
+        }
+    }, [onRemotionError, loop]);
+    if (typeof trimBeforeValue !== 'undefined' ||
+        typeof trimAfterValue !== 'undefined') {
+        return (_jsx(Sequence, { layout: "none", from: 0 - (trimBeforeValue ?? 0), showInTimeline: false, durationInFrames: trimAfterValue, name: name, children: _jsx(Audio, { pauseWhenBuffering: pauseWhenBuffering ?? false, ...otherProps }) }));
+    }
+    validateMediaProps(props, 'Video');
+    if (environment.isRendering) {
+        return _jsx(AudioForRendering, { ...otherProps });
+    }
+    const { onAutoPlayError, crossOrigin, delayRenderRetries, delayRenderTimeoutInMilliseconds, ...propsForPreview } = otherProps;
+    return (_jsx(AudioForPreview, { _remotionInternalNativeLoopPassed: props._remotionInternalNativeLoopPassed ?? false, _remotionInternalStack: stack ?? null, shouldPreMountAudioTags: audioContext !== null && audioContext.numberOfAudioTags > 0, ...propsForPreview, onNativeError: onError, onDuration: onDuration, 
+        // Proposal: Make this default to true in v5
+        pauseWhenBuffering: pauseWhenBuffering ?? false, _remotionInternalNeedsDurationCalculation: Boolean(loop), showInTimeline: showInTimeline ?? true }));
+};

package/dist/audiodata-to-array.js

@@ -0,0 +1 @@
+"use strict";

package/dist/convert-audiodata/data-types.d.ts

@@ -0,0 +1 @@
+export declare const getDataTypeForAudioFormat: (format: AudioSampleFormat) => Float32ArrayConstructor | Int16ArrayConstructor | Uint8ArrayConstructor | Int32ArrayConstructor;

package/dist/convert-audiodata/data-types.js

@@ -0,0 +1,22 @@
+export const getDataTypeForAudioFormat = (format) => {
+    switch (format) {
+        case 'f32':
+            return Float32Array;
+        case 'f32-planar':
+            return Float32Array;
+        case 's16':
+            return Int16Array;
+        case 's16-planar':
+            return Int16Array;
+        case 'u8':
+            return Uint8Array;
+        case 'u8-planar':
+            return Uint8Array;
+        case 's32':
+            return Int32Array;
+        case 's32-planar':
+            return Int32Array;
+        default:
+            throw new Error(`Unsupported audio format: ${format}`);
+    }
+};

package/dist/convert-audiodata/is-planar-format.d.ts

@@ -0,0 +1 @@
+export declare const isPlanarFormat: (format: AudioSampleFormat) => boolean;

package/dist/convert-audiodata/is-planar-format.js

@@ -0,0 +1,3 @@
+export const isPlanarFormat = (format) => {
+    return format.includes('-planar');
+};

package/dist/convert-audiodata/log-audiodata.d.ts

@@ -0,0 +1 @@
+export declare const logAudioData: (audioData: AudioData) => string;

package/dist/convert-audiodata/log-audiodata.js

@@ -0,0 +1,8 @@
+export const logAudioData = (audioData) => {
+    const srcChannels = new Int16Array(audioData.numberOfFrames * audioData.numberOfChannels);
+    audioData.copyTo(srcChannels, {
+        planeIndex: 0,
+        format: 's16',
+    });
+    return srcChannels.slice(0, 10).join(',');
+};

package/dist/convert-audiodata/trim-audiodata.js

@@ -0,0 +1 @@
+"use strict";

package/dist/deserialized-audiodata.d.ts

@@ -0,0 +1,15 @@
+import type { DataType } from './convert-audiodata/data-types';
+export type SerializableAudioData = {
+    data: DataType[];
+    format: AudioSampleFormat;
+    numberOfChannels: number;
+    numberOfFrames: number;
+    sampleRate: number;
+};
+export declare const turnAudioDataIntoSerializableData: (audioData: AudioData) => {
+    data: (Float32Array<ArrayBuffer> | Int32Array<ArrayBuffer> | Int16Array<ArrayBuffer> | Uint8Array<ArrayBuffer>)[];
+    format: AudioSampleFormat;
+    numberOfChannels: number;
+    numberOfFrames: number;
+    sampleRate: number;
+};

package/dist/deserialized-audiodata.js

@@ -0,0 +1,26 @@
+import { getDataTypeForAudioFormat } from './convert-audiodata/data-types';
+import { isPlanarFormat } from './convert-audiodata/is-planar-format';
+export const turnAudioDataIntoSerializableData = (audioData) => {
+    if (!audioData.format) {
+        throw new Error('AudioData format is not set');
+    }
+    const DataType = getDataTypeForAudioFormat(audioData.format);
+    const isPlanar = isPlanarFormat(audioData.format);
+    const planes = isPlanar ? audioData.numberOfChannels : 1;
+    const srcChannels = new Array(planes)
+        .fill(true)
+        .map(() => new DataType((isPlanar ? 1 : audioData.numberOfChannels) *
+        audioData.numberOfFrames));
+    for (let i = 0; i < planes; i++) {
+        audioData.copyTo(srcChannels[i], {
+            planeIndex: i,
+        });
+    }
+    return {
+        data: srcChannels,
+        format: audioData.format,
+        numberOfChannels: audioData.numberOfChannels,
+        numberOfFrames: audioData.numberOfFrames,
+        sampleRate: audioData.sampleRate,
+    };
+};

package/dist/esm/index.mjs

@@ -85,6 +85,7 @@ var makeAudioIterator = (audioSink, startFromSecond) => {
   const iterator = audioSink.buffers(startFromSecond);
   const queuedAudioNodes = [];
   const audioChunksForAfterResuming = [];
+  let mostRecentTimestamp = -Infinity;
   const cleanupAudioQueue = () => {
     for (const node of queuedAudioNodes) {
       node.node.stop();
@@ -108,12 +109,25 @@ var makeAudioIterator = (audioSink, startFromSecond) => {
         }
       };
     }
+    if (result.value) {
+      mostRecentTimestamp = Math.max(mostRecentTimestamp, result.value.timestamp + result.value.duration);
+      return {
+        type: "got-buffer",
+        buffer: result.value
+      };
+    }
     return {
-      type: "got-buffer-or-end",
-      buffer: result.value ?? null
+      type: "got-end",
+      mostRecentTimestamp
     };
   };
   const tryToSatisfySeek = async (time, allowWait, onBufferScheduled) => {
+    if (time < startFromSecond) {
+      return {
+        type: "not-satisfied",
+        reason: `time requested is before the start of the iterator`
+      };
+    }
     while (true) {
       const buffer = await getNextOrNullIfNotAvailable(allowWait);
       if (buffer.type === "need-to-wait-for-it") {
@@ -122,12 +136,18 @@ var makeAudioIterator = (audioSink, startFromSecond) => {
           reason: "iterator did not have buffer ready"
         };
       }
-      if (buffer.type === "got-buffer-or-end") {
-        if (buffer.buffer === null) {
+      if (buffer.type === "got-end") {
+        if (time >= mostRecentTimestamp) {
           return {
             type: "ended"
           };
         }
+        return {
+          type: "not-satisfied",
+          reason: `iterator ended before the requested time`
+        };
+      }
+      if (buffer.type === "got-buffer") {
         const bufferTimestamp = roundTo4Digits(buffer.buffer.timestamp);
         const bufferEndTimestamp = roundTo4Digits(buffer.buffer.timestamp + buffer.buffer.duration);
         const timestamp = roundTo4Digits(time);
@@ -180,6 +200,9 @@ var makeAudioIterator = (audioSink, startFromSecond) => {
     },
     getNext: async () => {
       const next = await iterator.next();
+      if (next.value) {
+        mostRecentTimestamp = Math.max(mostRecentTimestamp, next.value.timestamp + next.value.duration);
+      }
       return next;
     },
     isDestroyed: () => {
@@ -1580,6 +1603,7 @@ var AudioForPreviewAssertedShowing = ({
       toneFrequency,
       audioStreamIndex,
       pauseWhenBuffering: fallbackHtml5AudioProps?.pauseWhenBuffering,
+      crossOrigin: fallbackHtml5AudioProps?.crossOrigin,
       ...fallbackHtml5AudioProps
     });
   }
@@ -2208,10 +2232,13 @@ var rememberActualMatroskaTimestamps = (isMatroska) => {
 var getRetryDelay = () => {
   return null;
 };
-var getFormatOrNull = async (input) => {
+var getFormatOrNullOrNetworkError = async (input) => {
   try {
     return await input.getFormat();
-  } catch {
+  } catch (err) {
+    if (isNetworkError(err)) {
+      return "network-error";
+    }
     return null;
   }
 };
@@ -2222,9 +2249,12 @@ var getSinks = async (src) => {
       getRetryDelay
     })
   });
-  const format = await getFormatOrNull(input);
+  const format = await getFormatOrNullOrNetworkError(input);
   const isMatroska = format === MATROSKA || format === WEBM;
   const getVideoSinks = async () => {
+    if (format === "network-error") {
+      return "network-error";
+    }
     if (format === null) {
       return "unknown-container-format";
     }
@@ -2254,6 +2284,9 @@ var getSinks = async (src) => {
     if (format === null) {
       return "unknown-container-format";
     }
+    if (format === "network-error") {
+      return "network-error";
+    }
     const audioTracks = await input.getAudioTracks();
     const audioTrack = audioTracks[index];
     if (!audioTrack) {
@@ -2833,6 +2866,9 @@ var extractAudioInternal = async ({
     mediaDurationInSeconds = await getDuration();
   }
   const audio = await getAudio(audioStreamIndex);
+  if (audio === "network-error") {
+    return "network-error";
+  }
   if (audio === "no-audio-track") {
     return { data: null, durationInSeconds: null };
   }
@@ -2951,6 +2987,9 @@ var extractFrameInternal = async ({
   if (video === "unknown-container-format") {
     return { type: "unknown-container-format" };
   }
+  if (video === "network-error") {
+    return { type: "network-error" };
+  }
   let mediaDurationInSeconds = null;
   if (loop) {
     mediaDurationInSeconds = await sink.getDuration();
@@ -3094,12 +3133,21 @@ var extractFrameAndAudio = async ({
         durationInSeconds: frame.durationInSeconds
       };
     }
+    if (frame?.type === "network-error") {
+      return { type: "network-error" };
+    }
     if (audio === "unknown-container-format") {
       if (frame !== null) {
         frame?.frame?.close();
       }
       return { type: "unknown-container-format" };
     }
+    if (audio === "network-error") {
+      if (frame !== null) {
+        frame?.frame?.close();
+      }
+      return { type: "network-error" };
+    }
     if (audio === "cannot-decode") {
       if (frame?.type === "success" && frame.frame !== null) {
         frame?.frame.close();
@@ -4091,7 +4139,7 @@ var VideoForRendering = ({
           cancelRender3(new Error(`Cannot decode ${src}, and 'disallowFallbackToOffthreadVideo' was set. Failing the render.`));
         }
         if (window.remotion_isMainTab) {
-          Internals16.Log.info({ logLevel, tag: "@remotion/media" }, `Network error fetching ${src}, falling back to <OffthreadVideo>`);
+          Internals16.Log.warn({ logLevel, tag: "@remotion/media" }, `Network error fetching ${src} (no CORS?), falling back to <OffthreadVideo>`);
         }
         setReplaceWithOffthreadVideo({ durationInSeconds: null });
         return;

package/dist/extract-audio.d.ts

@@ -0,0 +1,7 @@
+import type { PcmS16AudioData } from './convert-audiodata/convert-audiodata';
+export declare const extractAudio: ({ src, timeInSeconds, durationInSeconds, volume, }: {
+    src: string;
+    timeInSeconds: number;
+    durationInSeconds: number;
+    volume: number;
+}) => Promise<PcmS16AudioData | null>;

package/dist/extract-audio.js

@@ -0,0 +1,98 @@
+import { combineAudioDataAndClosePrevious } from './convert-audiodata/combine-audiodata';
+import { convertAudioData } from './convert-audiodata/convert-audiodata';
+import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from './convert-audiodata/resample-audiodata';
+import { sinkPromises } from './extract-frame';
+import { getSinks } from './video-extraction/get-frames-since-keyframe';
+export const extractAudio = async ({ src, timeInSeconds, durationInSeconds, volume, }) => {
+    console.time('extractAudio');
+    if (!sinkPromises[src]) {
+        sinkPromises[src] = getSinks(src);
+    }
+    const { audio, actualMatroskaTimestamps, isMatroska } = await sinkPromises[src];
+    if (audio === null) {
+        console.timeEnd('extractAudio');
+        return null;
+    }
+    // https://discord.com/channels/@me/1409810025844838481/1415028953093111870
+    // Audio frames might have dependencies on previous and next frames so we need to decode a bit more
+    // and then discard it.
+    // The worst case seems to be FLAC files with a 65'535 sample window, which would be 1486.0ms at 44.1Khz.
+    // So let's set a threshold of 1.5 seconds.
+    const extraThreshold = 1.5;
+    // Matroska timestamps are not accurate unless we start from the beginning
+    // So for matroska, we need to decode all samples :(
+    // https://github.com/Vanilagy/mediabunny/issues/105
+    const sampleIterator = audio.sampleSink.samples(isMatroska ? 0 : Math.max(0, timeInSeconds - extraThreshold), timeInSeconds + durationInSeconds);
+    const samples = [];
+    for await (const sample of sampleIterator) {
+        const realTimestamp = actualMatroskaTimestamps.getRealTimestamp(sample.timestamp);
+        if (realTimestamp !== null && realTimestamp !== sample.timestamp) {
+            sample.setTimestamp(realTimestamp);
+        }
+        actualMatroskaTimestamps.observeTimestamp(sample.timestamp);
+        actualMatroskaTimestamps.observeTimestamp(sample.timestamp + sample.duration);
+        if (sample.timestamp + sample.duration - 0.0000000001 <= timeInSeconds) {
+            continue;
+        }
+        if (sample.timestamp >= timeInSeconds + durationInSeconds - 0.0000000001) {
+            continue;
+        }
+        samples.push(sample);
+    }
+    const audioDataArray = [];
+    for (let i = 0; i < samples.length; i++) {
+        const sample = samples[i];
+        // Less than 1 sample would be included - we did not need it after all!
+        if (Math.abs(sample.timestamp - (timeInSeconds + durationInSeconds)) *
+            sample.sampleRate <
+            1) {
+            sample.close();
+            continue;
+        }
+        // Less than 1 sample would be included - we did not need it after all!
+        if (sample.timestamp + sample.duration <= timeInSeconds) {
+            sample.close();
+            continue;
+        }
+        const isFirstSample = i === 0;
+        const isLastSample = i === samples.length - 1;
+        const audioDataRaw = sample.toAudioData();
+        // amount of samples to shave from start and end
+        let trimStartInSeconds = 0;
+        let trimEndInSeconds = 0;
+        // TODO: Apply playback rate
+        // TODO: Apply tone frequency
+        if (isFirstSample) {
+            trimStartInSeconds = timeInSeconds - sample.timestamp;
+        }
+        if (isLastSample) {
+            trimEndInSeconds =
+                // clamp to 0 in case the audio ends early
+                Math.max(0, sample.timestamp +
+                    sample.duration -
+                    (timeInSeconds + durationInSeconds));
+        }
+        const audioData = convertAudioData({
+            audioData: audioDataRaw,
+            newSampleRate: TARGET_SAMPLE_RATE,
+            trimStartInSeconds,
+            trimEndInSeconds,
+            targetNumberOfChannels: TARGET_NUMBER_OF_CHANNELS,
+            volume,
+        });
+        audioDataRaw.close();
+        if (audioData.numberOfFrames === 0) {
+            sample.close();
+            continue;
+        }
+        audioDataArray.push(audioData);
+        sample.close();
+    }
+    if (audioDataArray.length === 0) {
+        console.timeEnd('extractAudio');
+        return null;
+    }
+    const combined = combineAudioDataAndClosePrevious(audioDataArray);
+    console.timeEnd('extractAudio');
+    return combined;
+};

package/dist/extract-frame-and-audio.js

@@ -49,12 +49,21 @@ export const extractFrameAndAudio = async ({ src, timeInSeconds, logLevel, durat
                 durationInSeconds: frame.durationInSeconds,
             };
         }
+        if (frame?.type === 'network-error') {
+            return { type: 'network-error' };
+        }
         if (audio === 'unknown-container-format') {
             if (frame !== null) {
                 frame?.frame?.close();
             }
             return { type: 'unknown-container-format' };
         }
+        if (audio === 'network-error') {
+            if (frame !== null) {
+                frame?.frame?.close();
+            }
+            return { type: 'network-error' };
+        }
         if (audio === 'cannot-decode') {
             if (frame?.type === 'success' && frame.frame !== null) {
                 frame?.frame.close();