@remotion/media 4.0.351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +49 -0
- package/README.md +18 -0
- package/dist/audio/audio-for-rendering.d.ts +3 -0
- package/dist/audio/audio-for-rendering.js +99 -0
- package/dist/audio/audio.d.ts +3 -0
- package/dist/audio/audio.js +60 -0
- package/dist/audio/props.d.ts +29 -0
- package/dist/audio/props.js +1 -0
- package/dist/audio-extraction/audio-cache.d.ts +11 -0
- package/dist/audio-extraction/audio-cache.js +58 -0
- package/dist/audio-extraction/audio-iterator.d.ts +24 -0
- package/dist/audio-extraction/audio-iterator.js +106 -0
- package/dist/audio-extraction/audio-manager.d.ts +64 -0
- package/dist/audio-extraction/audio-manager.js +83 -0
- package/dist/audio-extraction/extract-audio.d.ts +10 -0
- package/dist/audio-extraction/extract-audio.js +77 -0
- package/dist/audio-for-rendering.d.ts +3 -0
- package/dist/audio-for-rendering.js +94 -0
- package/dist/audio.d.ts +3 -0
- package/dist/audio.js +60 -0
- package/dist/audiodata-to-array.d.ts +0 -0
- package/dist/audiodata-to-array.js +1 -0
- package/dist/caches.d.ts +86 -0
- package/dist/caches.js +15 -0
- package/dist/convert-audiodata/combine-audiodata.d.ts +2 -0
- package/dist/convert-audiodata/combine-audiodata.js +40 -0
- package/dist/convert-audiodata/convert-audiodata.d.ts +16 -0
- package/dist/convert-audiodata/convert-audiodata.js +53 -0
- package/dist/convert-audiodata/data-types.d.ts +1 -0
- package/dist/convert-audiodata/data-types.js +22 -0
- package/dist/convert-audiodata/is-planar-format.d.ts +1 -0
- package/dist/convert-audiodata/is-planar-format.js +3 -0
- package/dist/convert-audiodata/log-audiodata.d.ts +1 -0
- package/dist/convert-audiodata/log-audiodata.js +8 -0
- package/dist/convert-audiodata/resample-audiodata.d.ts +10 -0
- package/dist/convert-audiodata/resample-audiodata.js +72 -0
- package/dist/convert-audiodata/trim-audiodata.d.ts +0 -0
- package/dist/convert-audiodata/trim-audiodata.js +1 -0
- package/dist/deserialized-audiodata.d.ts +15 -0
- package/dist/deserialized-audiodata.js +26 -0
- package/dist/esm/index.mjs +14487 -0
- package/dist/extract-audio.d.ts +7 -0
- package/dist/extract-audio.js +98 -0
- package/dist/extract-frame-and-audio.d.ts +15 -0
- package/dist/extract-frame-and-audio.js +28 -0
- package/dist/extract-frame-via-broadcast-channel.d.ts +15 -0
- package/dist/extract-frame-via-broadcast-channel.js +104 -0
- package/dist/extract-frame.d.ts +27 -0
- package/dist/extract-frame.js +21 -0
- package/dist/extrct-audio.d.ts +7 -0
- package/dist/extrct-audio.js +94 -0
- package/dist/get-frames-since-keyframe.d.ts +22 -0
- package/dist/get-frames-since-keyframe.js +41 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +2 -0
- package/dist/keyframe-bank.d.ts +25 -0
- package/dist/keyframe-bank.js +120 -0
- package/dist/keyframe-manager.d.ts +23 -0
- package/dist/keyframe-manager.js +170 -0
- package/dist/log.d.ts +10 -0
- package/dist/log.js +33 -0
- package/dist/new-video-for-rendering.d.ts +3 -0
- package/dist/new-video-for-rendering.js +108 -0
- package/dist/new-video.d.ts +3 -0
- package/dist/new-video.js +37 -0
- package/dist/props.d.ts +29 -0
- package/dist/props.js +1 -0
- package/dist/remember-actual-matroska-timestamps.d.ts +4 -0
- package/dist/remember-actual-matroska-timestamps.js +19 -0
- package/dist/serialize-videoframe.d.ts +0 -0
- package/dist/serialize-videoframe.js +1 -0
- package/dist/video/props.d.ts +28 -0
- package/dist/video/props.js +1 -0
- package/dist/video/video-for-rendering.d.ts +3 -0
- package/dist/video/video-for-rendering.js +115 -0
- package/dist/video/video.d.ts +3 -0
- package/dist/video/video.js +37 -0
- package/dist/video-extraction/extract-frame-via-broadcast-channel.d.ts +16 -0
- package/dist/video-extraction/extract-frame-via-broadcast-channel.js +107 -0
- package/dist/video-extraction/extract-frame.d.ts +9 -0
- package/dist/video-extraction/extract-frame.js +21 -0
- package/dist/video-extraction/get-frames-since-keyframe.d.ts +23 -0
- package/dist/video-extraction/get-frames-since-keyframe.js +42 -0
- package/dist/video-extraction/keyframe-bank.d.ts +25 -0
- package/dist/video-extraction/keyframe-bank.js +121 -0
- package/dist/video-extraction/keyframe-manager.d.ts +23 -0
- package/dist/video-extraction/keyframe-manager.js +171 -0
- package/dist/video-extraction/remember-actual-matroska-timestamps.d.ts +5 -0
- package/dist/video-extraction/remember-actual-matroska-timestamps.js +19 -0
- package/dist/video-for-rendering.d.ts +3 -0
- package/dist/video-for-rendering.js +108 -0
- package/dist/video.d.ts +3 -0
- package/dist/video.js +37 -0
- package/package.json +55 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { audioManager } from '../caches';
|
|
2
|
+
import { combineAudioDataAndClosePrevious } from '../convert-audiodata/combine-audiodata';
|
|
3
|
+
import { convertAudioData } from '../convert-audiodata/convert-audiodata';
|
|
4
|
+
import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from '../convert-audiodata/resample-audiodata';
|
|
5
|
+
import { sinkPromises } from '../video-extraction/extract-frame';
|
|
6
|
+
import { getSinks } from '../video-extraction/get-frames-since-keyframe';
|
|
7
|
+
export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds, volume, logLevel, loop, }) => {
|
|
8
|
+
if (!sinkPromises[src]) {
|
|
9
|
+
sinkPromises[src] = getSinks(src);
|
|
10
|
+
}
|
|
11
|
+
const { audio, actualMatroskaTimestamps, isMatroska, getDuration } = await sinkPromises[src];
|
|
12
|
+
if (audio === null) {
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
15
|
+
const timeInSeconds = loop
|
|
16
|
+
? unloopedTimeInSeconds % (await getDuration())
|
|
17
|
+
: unloopedTimeInSeconds;
|
|
18
|
+
const sampleIterator = await audioManager.getIterator({
|
|
19
|
+
src,
|
|
20
|
+
timeInSeconds,
|
|
21
|
+
audioSampleSink: audio.sampleSink,
|
|
22
|
+
isMatroska,
|
|
23
|
+
actualMatroskaTimestamps,
|
|
24
|
+
});
|
|
25
|
+
const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
|
|
26
|
+
audioManager.logOpenFrames(logLevel);
|
|
27
|
+
const audioDataArray = [];
|
|
28
|
+
for (let i = 0; i < samples.length; i++) {
|
|
29
|
+
const sample = samples[i];
|
|
30
|
+
// Less than 1 sample would be included - we did not need it after all!
|
|
31
|
+
if (Math.abs(sample.timestamp - (timeInSeconds + durationInSeconds)) *
|
|
32
|
+
sample.sampleRate <
|
|
33
|
+
1) {
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
// Less than 1 sample would be included - we did not need it after all!
|
|
37
|
+
if (sample.timestamp + sample.duration <= timeInSeconds) {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
const isFirstSample = i === 0;
|
|
41
|
+
const isLastSample = i === samples.length - 1;
|
|
42
|
+
const audioDataRaw = sample.toAudioData();
|
|
43
|
+
// amount of samples to shave from start and end
|
|
44
|
+
let trimStartInSeconds = 0;
|
|
45
|
+
let trimEndInSeconds = 0;
|
|
46
|
+
// TODO: Apply playback rate
|
|
47
|
+
// TODO: Apply tone frequency
|
|
48
|
+
if (isFirstSample) {
|
|
49
|
+
trimStartInSeconds = timeInSeconds - sample.timestamp;
|
|
50
|
+
}
|
|
51
|
+
if (isLastSample) {
|
|
52
|
+
trimEndInSeconds =
|
|
53
|
+
// clamp to 0 in case the audio ends early
|
|
54
|
+
Math.max(0, sample.timestamp +
|
|
55
|
+
sample.duration -
|
|
56
|
+
(timeInSeconds + durationInSeconds));
|
|
57
|
+
}
|
|
58
|
+
const audioData = convertAudioData({
|
|
59
|
+
audioData: audioDataRaw,
|
|
60
|
+
newSampleRate: TARGET_SAMPLE_RATE,
|
|
61
|
+
trimStartInSeconds,
|
|
62
|
+
trimEndInSeconds,
|
|
63
|
+
targetNumberOfChannels: TARGET_NUMBER_OF_CHANNELS,
|
|
64
|
+
volume,
|
|
65
|
+
});
|
|
66
|
+
audioDataRaw.close();
|
|
67
|
+
if (audioData.numberOfFrames === 0) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
audioDataArray.push(audioData);
|
|
71
|
+
}
|
|
72
|
+
if (audioDataArray.length === 0) {
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
const combined = combineAudioDataAndClosePrevious(audioDataArray);
|
|
76
|
+
return combined;
|
|
77
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { useContext, useLayoutEffect, useMemo, useState } from 'react';
|
|
2
|
+
import { cancelRender, Internals, useCurrentFrame, useDelayRender, useRemotionEnvironment, } from 'remotion';
|
|
3
|
+
import { extractFrameViaBroadcastChannel } from './extract-frame-via-broadcast-channel';
|
|
4
|
+
export const AudioForRendering = ({ volume: volumeProp, playbackRate, src, muted, loopVolumeCurveBehavior, delayRenderRetries, delayRenderTimeoutInMilliseconds, logLevel = window.remotion_logLevel, }) => {
|
|
5
|
+
const absoluteFrame = Internals.useTimelinePosition();
|
|
6
|
+
const videoConfig = Internals.useUnsafeVideoConfig();
|
|
7
|
+
const { registerRenderAsset, unregisterRenderAsset } = useContext(Internals.RenderAssetManager);
|
|
8
|
+
const frame = useCurrentFrame();
|
|
9
|
+
const volumePropsFrame = Internals.useFrameForVolumeProp(loopVolumeCurveBehavior ?? 'repeat');
|
|
10
|
+
const environment = useRemotionEnvironment();
|
|
11
|
+
const [id] = useState(() => `${Math.random()}`.replace('0.', ''));
|
|
12
|
+
if (!videoConfig) {
|
|
13
|
+
throw new Error('No video config found');
|
|
14
|
+
}
|
|
15
|
+
if (!src) {
|
|
16
|
+
throw new TypeError('No `src` was passed to <Video>.');
|
|
17
|
+
}
|
|
18
|
+
const volume = Internals.evaluateVolume({
|
|
19
|
+
volume: volumeProp,
|
|
20
|
+
frame: volumePropsFrame,
|
|
21
|
+
mediaVolume: 1,
|
|
22
|
+
});
|
|
23
|
+
Internals.warnAboutTooHighVolume(volume);
|
|
24
|
+
const shouldRenderAudio = useMemo(() => {
|
|
25
|
+
if (!window.remotion_audioEnabled) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (muted) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
if (volume <= 0) {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
return true;
|
|
35
|
+
}, [muted, volume]);
|
|
36
|
+
const { fps } = videoConfig;
|
|
37
|
+
const { delayRender, continueRender } = useDelayRender();
|
|
38
|
+
useLayoutEffect(() => {
|
|
39
|
+
const actualFps = playbackRate ? fps / playbackRate : fps;
|
|
40
|
+
const timestamp = frame / actualFps;
|
|
41
|
+
const durationInSeconds = 1 / actualFps;
|
|
42
|
+
const newHandle = delayRender(`Extracting frame number ${frame}`, {
|
|
43
|
+
retries: delayRenderRetries ?? undefined,
|
|
44
|
+
timeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? undefined,
|
|
45
|
+
});
|
|
46
|
+
extractFrameViaBroadcastChannel({
|
|
47
|
+
src,
|
|
48
|
+
timeInSeconds: timestamp,
|
|
49
|
+
durationInSeconds,
|
|
50
|
+
logLevel: logLevel ?? 'info',
|
|
51
|
+
shouldRenderAudio,
|
|
52
|
+
isClientSideRendering: environment.isClientSideRendering,
|
|
53
|
+
})
|
|
54
|
+
.then(({ audio }) => {
|
|
55
|
+
if (audio) {
|
|
56
|
+
registerRenderAsset({
|
|
57
|
+
type: 'inline-audio',
|
|
58
|
+
id,
|
|
59
|
+
audio: Array.from(audio.data),
|
|
60
|
+
sampleRate: audio.sampleRate,
|
|
61
|
+
numberOfChannels: audio.numberOfChannels,
|
|
62
|
+
frame: absoluteFrame,
|
|
63
|
+
timestamp: audio.timestamp,
|
|
64
|
+
duration: (audio.numberOfFrames / audio.sampleRate) * 1000000,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
continueRender(newHandle);
|
|
68
|
+
})
|
|
69
|
+
.catch((error) => {
|
|
70
|
+
cancelRender(error);
|
|
71
|
+
});
|
|
72
|
+
return () => {
|
|
73
|
+
continueRender(newHandle);
|
|
74
|
+
unregisterRenderAsset(id);
|
|
75
|
+
};
|
|
76
|
+
}, [
|
|
77
|
+
absoluteFrame,
|
|
78
|
+
continueRender,
|
|
79
|
+
delayRender,
|
|
80
|
+
delayRenderRetries,
|
|
81
|
+
delayRenderTimeoutInMilliseconds,
|
|
82
|
+
environment.isClientSideRendering,
|
|
83
|
+
fps,
|
|
84
|
+
frame,
|
|
85
|
+
id,
|
|
86
|
+
logLevel,
|
|
87
|
+
playbackRate,
|
|
88
|
+
registerRenderAsset,
|
|
89
|
+
shouldRenderAudio,
|
|
90
|
+
src,
|
|
91
|
+
unregisterRenderAsset,
|
|
92
|
+
]);
|
|
93
|
+
return null;
|
|
94
|
+
};
|
package/dist/audio.d.ts
ADDED
package/dist/audio.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { jsx as _jsx } from "react/jsx-runtime";
|
|
2
|
+
import { useCallback, useContext } from 'react';
|
|
3
|
+
import { cancelRender, Internals, Sequence, useRemotionEnvironment, } from 'remotion';
|
|
4
|
+
import { SharedAudioContext } from '../../core/src/audio/shared-audio-tags';
|
|
5
|
+
import { AudioForRendering } from './audio-for-rendering';
|
|
6
|
+
const { validateMediaTrimProps, resolveTrimProps, validateMediaProps, AudioForPreview, } = Internals;
|
|
7
|
+
export const Audio = (props) => {
|
|
8
|
+
const audioContext = useContext(SharedAudioContext);
|
|
9
|
+
// Should only destruct `trimBefore` and `trimAfter` from props,
|
|
10
|
+
// rest gets drilled down
|
|
11
|
+
const { trimBefore, trimAfter, name, pauseWhenBuffering, stack, showInTimeline, onError: onRemotionError, loop, ...otherProps } = props;
|
|
12
|
+
const environment = useRemotionEnvironment();
|
|
13
|
+
const onDuration = useCallback(() => undefined, []);
|
|
14
|
+
if (typeof props.src !== 'string') {
|
|
15
|
+
throw new TypeError(`The \`<Audio>\` tag requires a string for \`src\`, but got ${JSON.stringify(props.src)} instead.`);
|
|
16
|
+
}
|
|
17
|
+
validateMediaTrimProps({
|
|
18
|
+
startFrom: undefined,
|
|
19
|
+
endAt: undefined,
|
|
20
|
+
trimBefore,
|
|
21
|
+
trimAfter,
|
|
22
|
+
});
|
|
23
|
+
const { trimBeforeValue, trimAfterValue } = resolveTrimProps({
|
|
24
|
+
startFrom: undefined,
|
|
25
|
+
endAt: undefined,
|
|
26
|
+
trimBefore,
|
|
27
|
+
trimAfter,
|
|
28
|
+
});
|
|
29
|
+
const onError = useCallback((e) => {
|
|
30
|
+
// eslint-disable-next-line no-console
|
|
31
|
+
console.log(e.currentTarget.error);
|
|
32
|
+
// If there is no `loop` property, we don't need to get the duration
|
|
33
|
+
// and this does not need to be a fatal error
|
|
34
|
+
const errMessage = `Could not play audio: ${e.currentTarget.error}. See https://remotion.dev/docs/media-playback-error for help.`;
|
|
35
|
+
if (loop) {
|
|
36
|
+
if (onRemotionError) {
|
|
37
|
+
onRemotionError(new Error(errMessage));
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
cancelRender(new Error(errMessage));
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
onRemotionError?.(new Error(errMessage));
|
|
44
|
+
// eslint-disable-next-line no-console
|
|
45
|
+
console.warn(errMessage);
|
|
46
|
+
}
|
|
47
|
+
}, [onRemotionError, loop]);
|
|
48
|
+
if (typeof trimBeforeValue !== 'undefined' ||
|
|
49
|
+
typeof trimAfterValue !== 'undefined') {
|
|
50
|
+
return (_jsx(Sequence, { layout: "none", from: 0 - (trimBeforeValue ?? 0), showInTimeline: false, durationInFrames: trimAfterValue, name: name, children: _jsx(Audio, { pauseWhenBuffering: pauseWhenBuffering ?? false, ...otherProps }) }));
|
|
51
|
+
}
|
|
52
|
+
validateMediaProps(props, 'Video');
|
|
53
|
+
if (environment.isRendering) {
|
|
54
|
+
return _jsx(AudioForRendering, { ...otherProps });
|
|
55
|
+
}
|
|
56
|
+
const { onAutoPlayError, crossOrigin, delayRenderRetries, delayRenderTimeoutInMilliseconds, ...propsForPreview } = otherProps;
|
|
57
|
+
return (_jsx(AudioForPreview, { _remotionInternalNativeLoopPassed: props._remotionInternalNativeLoopPassed ?? false, _remotionInternalStack: stack ?? null, shouldPreMountAudioTags: audioContext !== null && audioContext.numberOfAudioTags > 0, ...propsForPreview, onNativeError: onError, onDuration: onDuration,
|
|
58
|
+
// Proposal: Make this default to true in v5
|
|
59
|
+
pauseWhenBuffering: pauseWhenBuffering ?? false, _remotionInternalNeedsDurationCalculation: Boolean(loop), showInTimeline: showInTimeline ?? true }));
|
|
60
|
+
};
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";
|
package/dist/caches.d.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
export declare const MAX_CACHE_SIZE: number;
|
|
2
|
+
export declare const SAFE_BACK_WINDOW_IN_SECONDS = 1;
|
|
3
|
+
export declare const keyframeManager: {
|
|
4
|
+
requestKeyframeBank: ({ packetSink, timestamp, videoSampleSink, src, logLevel, }: {
|
|
5
|
+
timestamp: number;
|
|
6
|
+
packetSink: import("mediabunny").EncodedPacketSink;
|
|
7
|
+
videoSampleSink: import("mediabunny").VideoSampleSink;
|
|
8
|
+
src: string;
|
|
9
|
+
logLevel: import("./log").LogLevel;
|
|
10
|
+
}) => Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
|
|
11
|
+
addKeyframeBank: ({ src, bank, startTimestampInSeconds, }: {
|
|
12
|
+
src: string;
|
|
13
|
+
bank: Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
|
|
14
|
+
startTimestampInSeconds: number;
|
|
15
|
+
}) => void;
|
|
16
|
+
getCacheStats: () => Promise<{
|
|
17
|
+
count: number;
|
|
18
|
+
totalSize: number;
|
|
19
|
+
}>;
|
|
20
|
+
clearAll: () => Promise<void>;
|
|
21
|
+
};
|
|
22
|
+
export declare const audioManager: {
|
|
23
|
+
makeIterator: ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
|
|
24
|
+
timeInSeconds: number;
|
|
25
|
+
src: string;
|
|
26
|
+
audioSampleSink: import("mediabunny").AudioSampleSink;
|
|
27
|
+
isMatroska: boolean;
|
|
28
|
+
actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
|
|
29
|
+
}) => {
|
|
30
|
+
src: string;
|
|
31
|
+
getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
|
|
32
|
+
waitForCompletion: () => Promise<boolean>;
|
|
33
|
+
canSatisfyRequestedTime: (timestamp: number) => boolean;
|
|
34
|
+
logOpenFrames: (logLevel: import("./log").LogLevel) => void;
|
|
35
|
+
getCacheStats: () => {
|
|
36
|
+
count: number;
|
|
37
|
+
size: number;
|
|
38
|
+
};
|
|
39
|
+
getLastUsed: () => number;
|
|
40
|
+
prepareForDeletion: () => Promise<void>;
|
|
41
|
+
startTimestamp: number;
|
|
42
|
+
};
|
|
43
|
+
getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
|
|
44
|
+
src: string;
|
|
45
|
+
timeInSeconds: number;
|
|
46
|
+
audioSampleSink: import("mediabunny").AudioSampleSink;
|
|
47
|
+
isMatroska: boolean;
|
|
48
|
+
actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
|
|
49
|
+
}) => Promise<{
|
|
50
|
+
src: string;
|
|
51
|
+
getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
|
|
52
|
+
waitForCompletion: () => Promise<boolean>;
|
|
53
|
+
canSatisfyRequestedTime: (timestamp: number) => boolean;
|
|
54
|
+
logOpenFrames: (logLevel: import("./log").LogLevel) => void;
|
|
55
|
+
getCacheStats: () => {
|
|
56
|
+
count: number;
|
|
57
|
+
size: number;
|
|
58
|
+
};
|
|
59
|
+
getLastUsed: () => number;
|
|
60
|
+
prepareForDeletion: () => Promise<void>;
|
|
61
|
+
startTimestamp: number;
|
|
62
|
+
}>;
|
|
63
|
+
getCacheStats: () => {
|
|
64
|
+
count: number;
|
|
65
|
+
totalSize: number;
|
|
66
|
+
};
|
|
67
|
+
getIteratorMostInThePast: () => {
|
|
68
|
+
src: string;
|
|
69
|
+
getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
|
|
70
|
+
waitForCompletion: () => Promise<boolean>;
|
|
71
|
+
canSatisfyRequestedTime: (timestamp: number) => boolean;
|
|
72
|
+
logOpenFrames: (logLevel: import("./log").LogLevel) => void;
|
|
73
|
+
getCacheStats: () => {
|
|
74
|
+
count: number;
|
|
75
|
+
size: number;
|
|
76
|
+
};
|
|
77
|
+
getLastUsed: () => number;
|
|
78
|
+
prepareForDeletion: () => Promise<void>;
|
|
79
|
+
startTimestamp: number;
|
|
80
|
+
} | null;
|
|
81
|
+
logOpenFrames: (logLevel: import("./log").LogLevel) => void;
|
|
82
|
+
};
|
|
83
|
+
export declare const getTotalCacheStats: () => Promise<{
|
|
84
|
+
count: number;
|
|
85
|
+
totalSize: number;
|
|
86
|
+
}>;
|
package/dist/caches.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { makeAudioManager } from './audio-extraction/audio-manager';
|
|
2
|
+
import { makeKeyframeManager } from './video-extraction/keyframe-manager';
|
|
3
|
+
export const MAX_CACHE_SIZE = 1000 * 1000 * 1000; // 1GB
|
|
4
|
+
// TODO: make it dependent on the fps and concurrency
|
|
5
|
+
export const SAFE_BACK_WINDOW_IN_SECONDS = 1;
|
|
6
|
+
export const keyframeManager = makeKeyframeManager();
|
|
7
|
+
export const audioManager = makeAudioManager();
|
|
8
|
+
export const getTotalCacheStats = async () => {
|
|
9
|
+
const keyframeManagerCacheStats = await keyframeManager.getCacheStats();
|
|
10
|
+
const audioManagerCacheStats = audioManager.getCacheStats();
|
|
11
|
+
return {
|
|
12
|
+
count: keyframeManagerCacheStats.count + audioManagerCacheStats.count,
|
|
13
|
+
totalSize: keyframeManagerCacheStats.totalSize + audioManagerCacheStats.totalSize,
|
|
14
|
+
};
|
|
15
|
+
};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export const combineAudioDataAndClosePrevious = (audioDataArray) => {
|
|
2
|
+
let numberOfFrames = 0;
|
|
3
|
+
let numberOfChannels = null;
|
|
4
|
+
let sampleRate = null;
|
|
5
|
+
const { timestamp } = audioDataArray[0];
|
|
6
|
+
for (const audioData of audioDataArray) {
|
|
7
|
+
numberOfFrames += audioData.numberOfFrames;
|
|
8
|
+
if (!numberOfChannels) {
|
|
9
|
+
numberOfChannels = audioData.numberOfChannels;
|
|
10
|
+
}
|
|
11
|
+
else if (numberOfChannels !== audioData.numberOfChannels) {
|
|
12
|
+
throw new Error('Number of channels do not match');
|
|
13
|
+
}
|
|
14
|
+
if (!sampleRate) {
|
|
15
|
+
sampleRate = audioData.sampleRate;
|
|
16
|
+
}
|
|
17
|
+
else if (sampleRate !== audioData.sampleRate) {
|
|
18
|
+
throw new Error('Sample rates do not match');
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
if (!numberOfChannels) {
|
|
22
|
+
throw new Error('Number of channels is not set');
|
|
23
|
+
}
|
|
24
|
+
if (!sampleRate) {
|
|
25
|
+
throw new Error('Sample rate is not set');
|
|
26
|
+
}
|
|
27
|
+
const arr = new Int16Array(numberOfFrames * numberOfChannels);
|
|
28
|
+
let offset = 0;
|
|
29
|
+
for (const audioData of audioDataArray) {
|
|
30
|
+
arr.set(audioData.data, offset);
|
|
31
|
+
offset += audioData.data.length;
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
data: arr,
|
|
35
|
+
numberOfChannels,
|
|
36
|
+
numberOfFrames,
|
|
37
|
+
sampleRate,
|
|
38
|
+
timestamp,
|
|
39
|
+
};
|
|
40
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export type ConvertAudioDataOptions = {
|
|
2
|
+
audioData: AudioData;
|
|
3
|
+
newSampleRate: number;
|
|
4
|
+
trimStartInSeconds: number;
|
|
5
|
+
trimEndInSeconds: number;
|
|
6
|
+
targetNumberOfChannels: number;
|
|
7
|
+
volume: number;
|
|
8
|
+
};
|
|
9
|
+
export type PcmS16AudioData = {
|
|
10
|
+
data: Int16Array;
|
|
11
|
+
sampleRate: number;
|
|
12
|
+
numberOfChannels: number;
|
|
13
|
+
numberOfFrames: number;
|
|
14
|
+
timestamp: number;
|
|
15
|
+
};
|
|
16
|
+
export declare const convertAudioData: ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, volume, }: ConvertAudioDataOptions) => PcmS16AudioData;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { resampleAudioData } from './resample-audiodata';
|
|
2
|
+
const FORMAT = 's16';
|
|
3
|
+
export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, volume, }) => {
|
|
4
|
+
const { numberOfChannels: srcNumberOfChannels, sampleRate: currentSampleRate, numberOfFrames, } = audioData;
|
|
5
|
+
const ratio = currentSampleRate / newSampleRate;
|
|
6
|
+
const frameOffset = Math.round(trimStartInSeconds * audioData.sampleRate);
|
|
7
|
+
const frameCount = numberOfFrames -
|
|
8
|
+
Math.round((trimEndInSeconds + trimStartInSeconds) * audioData.sampleRate);
|
|
9
|
+
const newNumberOfFrames = Math.round(frameCount / ratio);
|
|
10
|
+
if (newNumberOfFrames === 0) {
|
|
11
|
+
throw new Error('Cannot resample - the given sample rate would result in less than 1 sample');
|
|
12
|
+
}
|
|
13
|
+
if (newSampleRate < 3000 || newSampleRate > 768000) {
|
|
14
|
+
throw new Error('newSampleRate must be between 3000 and 768000');
|
|
15
|
+
}
|
|
16
|
+
const srcChannels = new Int16Array(srcNumberOfChannels * frameCount);
|
|
17
|
+
audioData.copyTo(srcChannels, {
|
|
18
|
+
planeIndex: 0,
|
|
19
|
+
format: FORMAT,
|
|
20
|
+
frameOffset,
|
|
21
|
+
frameCount,
|
|
22
|
+
});
|
|
23
|
+
const data = new Int16Array(newNumberOfFrames * targetNumberOfChannels);
|
|
24
|
+
const chunkSize = frameCount / newNumberOfFrames;
|
|
25
|
+
if (newNumberOfFrames === frameCount &&
|
|
26
|
+
targetNumberOfChannels === srcNumberOfChannels &&
|
|
27
|
+
volume === 1) {
|
|
28
|
+
return {
|
|
29
|
+
data: srcChannels,
|
|
30
|
+
numberOfChannels: targetNumberOfChannels,
|
|
31
|
+
numberOfFrames: newNumberOfFrames,
|
|
32
|
+
sampleRate: newSampleRate,
|
|
33
|
+
timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
resampleAudioData({
|
|
37
|
+
srcNumberOfChannels,
|
|
38
|
+
sourceChannels: srcChannels,
|
|
39
|
+
destination: data,
|
|
40
|
+
targetFrames: newNumberOfFrames,
|
|
41
|
+
chunkSize,
|
|
42
|
+
volume,
|
|
43
|
+
});
|
|
44
|
+
const newAudioData = {
|
|
45
|
+
data,
|
|
46
|
+
format: FORMAT,
|
|
47
|
+
numberOfChannels: targetNumberOfChannels,
|
|
48
|
+
numberOfFrames: newNumberOfFrames,
|
|
49
|
+
sampleRate: newSampleRate,
|
|
50
|
+
timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
|
|
51
|
+
};
|
|
52
|
+
return newAudioData;
|
|
53
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const getDataTypeForAudioFormat: (format: AudioSampleFormat) => Float32ArrayConstructor | Int16ArrayConstructor | Uint8ArrayConstructor | Int32ArrayConstructor;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export const getDataTypeForAudioFormat = (format) => {
|
|
2
|
+
switch (format) {
|
|
3
|
+
case 'f32':
|
|
4
|
+
return Float32Array;
|
|
5
|
+
case 'f32-planar':
|
|
6
|
+
return Float32Array;
|
|
7
|
+
case 's16':
|
|
8
|
+
return Int16Array;
|
|
9
|
+
case 's16-planar':
|
|
10
|
+
return Int16Array;
|
|
11
|
+
case 'u8':
|
|
12
|
+
return Uint8Array;
|
|
13
|
+
case 'u8-planar':
|
|
14
|
+
return Uint8Array;
|
|
15
|
+
case 's32':
|
|
16
|
+
return Int32Array;
|
|
17
|
+
case 's32-planar':
|
|
18
|
+
return Int32Array;
|
|
19
|
+
default:
|
|
20
|
+
throw new Error(`Unsupported audio format: ${format}`);
|
|
21
|
+
}
|
|
22
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const isPlanarFormat: (format: AudioSampleFormat) => boolean;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const logAudioData: (audioData: AudioData) => string;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export declare const TARGET_NUMBER_OF_CHANNELS = 2;
|
|
2
|
+
export declare const TARGET_SAMPLE_RATE = 48000;
|
|
3
|
+
export declare const resampleAudioData: ({ srcNumberOfChannels, sourceChannels, destination, targetFrames, chunkSize, volume, }: {
|
|
4
|
+
srcNumberOfChannels: number;
|
|
5
|
+
sourceChannels: Int16Array;
|
|
6
|
+
destination: Int16Array;
|
|
7
|
+
targetFrames: number;
|
|
8
|
+
chunkSize: number;
|
|
9
|
+
volume: number;
|
|
10
|
+
}) => void;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Remotion exports all videos with 2 channels.
|
|
2
|
+
export const TARGET_NUMBER_OF_CHANNELS = 2;
|
|
3
|
+
// Remotion exports all videos with 48kHz sample rate.
|
|
4
|
+
export const TARGET_SAMPLE_RATE = 48000;
|
|
5
|
+
export const resampleAudioData = ({ srcNumberOfChannels, sourceChannels, destination, targetFrames, chunkSize, volume, }) => {
|
|
6
|
+
const getSourceValues = (start, end, channelIndex) => {
|
|
7
|
+
const sampleCountAvg = end - start;
|
|
8
|
+
let itemSum = 0;
|
|
9
|
+
let itemCount = 0;
|
|
10
|
+
for (let k = 0; k < sampleCountAvg; k++) {
|
|
11
|
+
const num = sourceChannels[(start + k) * srcNumberOfChannels + channelIndex];
|
|
12
|
+
itemSum += num;
|
|
13
|
+
itemCount++;
|
|
14
|
+
}
|
|
15
|
+
const average = itemSum / itemCount;
|
|
16
|
+
const averageVolume = average * volume;
|
|
17
|
+
if (averageVolume < -32768) {
|
|
18
|
+
return -32768;
|
|
19
|
+
}
|
|
20
|
+
if (averageVolume > 32767) {
|
|
21
|
+
return 32767;
|
|
22
|
+
}
|
|
23
|
+
return averageVolume;
|
|
24
|
+
};
|
|
25
|
+
for (let newFrameIndex = 0; newFrameIndex < targetFrames; newFrameIndex++) {
|
|
26
|
+
const start = Math.floor(newFrameIndex * chunkSize);
|
|
27
|
+
const end = Math.max(Math.floor(start + chunkSize), start + 1);
|
|
28
|
+
if (TARGET_NUMBER_OF_CHANNELS === srcNumberOfChannels) {
|
|
29
|
+
for (let i = 0; i < srcNumberOfChannels; i++) {
|
|
30
|
+
destination[newFrameIndex * srcNumberOfChannels + i] = getSourceValues(start, end, i);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
// The following formulas were taken from Mediabunnys audio resampler:
|
|
34
|
+
// https://github.com/Vanilagy/mediabunny/blob/b9f7ab2fa2b9167784cbded044d466185308999f/src/conversion.ts
|
|
35
|
+
// Mono to Stereo: M -> L, M -> R
|
|
36
|
+
if (srcNumberOfChannels === 1) {
|
|
37
|
+
const m = getSourceValues(start, end, 0);
|
|
38
|
+
destination[newFrameIndex * 2 + 0] = m;
|
|
39
|
+
destination[newFrameIndex * 2 + 1] = m;
|
|
40
|
+
}
|
|
41
|
+
// Quad to Stereo: 0.5 * (L + SL), 0.5 * (R + SR)
|
|
42
|
+
else if (srcNumberOfChannels === 4) {
|
|
43
|
+
const l = getSourceValues(start, end, 0);
|
|
44
|
+
const r = getSourceValues(start, end, 1);
|
|
45
|
+
const sl = getSourceValues(start, end, 2);
|
|
46
|
+
const sr = getSourceValues(start, end, 3);
|
|
47
|
+
const l2 = 0.5 * (l + sl);
|
|
48
|
+
const r2 = 0.5 * (r + sr);
|
|
49
|
+
destination[newFrameIndex * 2 + 0] = l2;
|
|
50
|
+
destination[newFrameIndex * 2 + 1] = r2;
|
|
51
|
+
}
|
|
52
|
+
// 5.1 to Stereo: L + sqrt(1/2) * (C + SL), R + sqrt(1/2) * (C + SR)
|
|
53
|
+
else if (srcNumberOfChannels === 6) {
|
|
54
|
+
const l = getSourceValues(start, end, 0);
|
|
55
|
+
const r = getSourceValues(start, end, 1);
|
|
56
|
+
const c = getSourceValues(start, end, 2);
|
|
57
|
+
const sl = getSourceValues(start, end, 3);
|
|
58
|
+
const sr = getSourceValues(start, end, 4);
|
|
59
|
+
const l2 = l + Math.sqrt(1 / 2) * (c + sl);
|
|
60
|
+
const r2 = r + Math.sqrt(1 / 2) * (c + sr);
|
|
61
|
+
destination[newFrameIndex * 2 + 0] = l2;
|
|
62
|
+
destination[newFrameIndex * 2 + 1] = r2;
|
|
63
|
+
}
|
|
64
|
+
// Discrete fallback: direct mapping with zero-fill or drop
|
|
65
|
+
else {
|
|
66
|
+
for (let i = 0; i < srcNumberOfChannels; i++) {
|
|
67
|
+
destination[newFrameIndex * TARGET_NUMBER_OF_CHANNELS + i] =
|
|
68
|
+
getSourceValues(start, end, i);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
};
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { DataType } from './convert-audiodata/data-types';
|
|
2
|
+
export type SerializableAudioData = {
|
|
3
|
+
data: DataType[];
|
|
4
|
+
format: AudioSampleFormat;
|
|
5
|
+
numberOfChannels: number;
|
|
6
|
+
numberOfFrames: number;
|
|
7
|
+
sampleRate: number;
|
|
8
|
+
};
|
|
9
|
+
export declare const turnAudioDataIntoSerializableData: (audioData: AudioData) => {
|
|
10
|
+
data: (Float32Array<ArrayBuffer> | Int32Array<ArrayBuffer> | Int16Array<ArrayBuffer> | Uint8Array<ArrayBuffer>)[];
|
|
11
|
+
format: AudioSampleFormat;
|
|
12
|
+
numberOfChannels: number;
|
|
13
|
+
numberOfFrames: number;
|
|
14
|
+
sampleRate: number;
|
|
15
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { getDataTypeForAudioFormat } from './convert-audiodata/data-types';
|
|
2
|
+
import { isPlanarFormat } from './convert-audiodata/is-planar-format';
|
|
3
|
+
export const turnAudioDataIntoSerializableData = (audioData) => {
|
|
4
|
+
if (!audioData.format) {
|
|
5
|
+
throw new Error('AudioData format is not set');
|
|
6
|
+
}
|
|
7
|
+
const DataType = getDataTypeForAudioFormat(audioData.format);
|
|
8
|
+
const isPlanar = isPlanarFormat(audioData.format);
|
|
9
|
+
const planes = isPlanar ? audioData.numberOfChannels : 1;
|
|
10
|
+
const srcChannels = new Array(planes)
|
|
11
|
+
.fill(true)
|
|
12
|
+
.map(() => new DataType((isPlanar ? 1 : audioData.numberOfChannels) *
|
|
13
|
+
audioData.numberOfFrames));
|
|
14
|
+
for (let i = 0; i < planes; i++) {
|
|
15
|
+
audioData.copyTo(srcChannels[i], {
|
|
16
|
+
planeIndex: i,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
data: srcChannels,
|
|
21
|
+
format: audioData.format,
|
|
22
|
+
numberOfChannels: audioData.numberOfChannels,
|
|
23
|
+
numberOfFrames: audioData.numberOfFrames,
|
|
24
|
+
sampleRate: audioData.sampleRate,
|
|
25
|
+
};
|
|
26
|
+
};
|