@remotion/media 4.0.351

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/LICENSE.md +49 -0
  2. package/README.md +18 -0
  3. package/dist/audio/audio-for-rendering.d.ts +3 -0
  4. package/dist/audio/audio-for-rendering.js +99 -0
  5. package/dist/audio/audio.d.ts +3 -0
  6. package/dist/audio/audio.js +60 -0
  7. package/dist/audio/props.d.ts +29 -0
  8. package/dist/audio/props.js +1 -0
  9. package/dist/audio-extraction/audio-cache.d.ts +11 -0
  10. package/dist/audio-extraction/audio-cache.js +58 -0
  11. package/dist/audio-extraction/audio-iterator.d.ts +24 -0
  12. package/dist/audio-extraction/audio-iterator.js +106 -0
  13. package/dist/audio-extraction/audio-manager.d.ts +64 -0
  14. package/dist/audio-extraction/audio-manager.js +83 -0
  15. package/dist/audio-extraction/extract-audio.d.ts +10 -0
  16. package/dist/audio-extraction/extract-audio.js +77 -0
  17. package/dist/audio-for-rendering.d.ts +3 -0
  18. package/dist/audio-for-rendering.js +94 -0
  19. package/dist/audio.d.ts +3 -0
  20. package/dist/audio.js +60 -0
  21. package/dist/audiodata-to-array.d.ts +0 -0
  22. package/dist/audiodata-to-array.js +1 -0
  23. package/dist/caches.d.ts +86 -0
  24. package/dist/caches.js +15 -0
  25. package/dist/convert-audiodata/combine-audiodata.d.ts +2 -0
  26. package/dist/convert-audiodata/combine-audiodata.js +40 -0
  27. package/dist/convert-audiodata/convert-audiodata.d.ts +16 -0
  28. package/dist/convert-audiodata/convert-audiodata.js +53 -0
  29. package/dist/convert-audiodata/data-types.d.ts +1 -0
  30. package/dist/convert-audiodata/data-types.js +22 -0
  31. package/dist/convert-audiodata/is-planar-format.d.ts +1 -0
  32. package/dist/convert-audiodata/is-planar-format.js +3 -0
  33. package/dist/convert-audiodata/log-audiodata.d.ts +1 -0
  34. package/dist/convert-audiodata/log-audiodata.js +8 -0
  35. package/dist/convert-audiodata/resample-audiodata.d.ts +10 -0
  36. package/dist/convert-audiodata/resample-audiodata.js +72 -0
  37. package/dist/convert-audiodata/trim-audiodata.d.ts +0 -0
  38. package/dist/convert-audiodata/trim-audiodata.js +1 -0
  39. package/dist/deserialized-audiodata.d.ts +15 -0
  40. package/dist/deserialized-audiodata.js +26 -0
  41. package/dist/esm/index.mjs +14487 -0
  42. package/dist/extract-audio.d.ts +7 -0
  43. package/dist/extract-audio.js +98 -0
  44. package/dist/extract-frame-and-audio.d.ts +15 -0
  45. package/dist/extract-frame-and-audio.js +28 -0
  46. package/dist/extract-frame-via-broadcast-channel.d.ts +15 -0
  47. package/dist/extract-frame-via-broadcast-channel.js +104 -0
  48. package/dist/extract-frame.d.ts +27 -0
  49. package/dist/extract-frame.js +21 -0
  50. package/dist/extrct-audio.d.ts +7 -0
  51. package/dist/extrct-audio.js +94 -0
  52. package/dist/get-frames-since-keyframe.d.ts +22 -0
  53. package/dist/get-frames-since-keyframe.js +41 -0
  54. package/dist/index.d.ts +4 -0
  55. package/dist/index.js +2 -0
  56. package/dist/keyframe-bank.d.ts +25 -0
  57. package/dist/keyframe-bank.js +120 -0
  58. package/dist/keyframe-manager.d.ts +23 -0
  59. package/dist/keyframe-manager.js +170 -0
  60. package/dist/log.d.ts +10 -0
  61. package/dist/log.js +33 -0
  62. package/dist/new-video-for-rendering.d.ts +3 -0
  63. package/dist/new-video-for-rendering.js +108 -0
  64. package/dist/new-video.d.ts +3 -0
  65. package/dist/new-video.js +37 -0
  66. package/dist/props.d.ts +29 -0
  67. package/dist/props.js +1 -0
  68. package/dist/remember-actual-matroska-timestamps.d.ts +4 -0
  69. package/dist/remember-actual-matroska-timestamps.js +19 -0
  70. package/dist/serialize-videoframe.d.ts +0 -0
  71. package/dist/serialize-videoframe.js +1 -0
  72. package/dist/video/props.d.ts +28 -0
  73. package/dist/video/props.js +1 -0
  74. package/dist/video/video-for-rendering.d.ts +3 -0
  75. package/dist/video/video-for-rendering.js +115 -0
  76. package/dist/video/video.d.ts +3 -0
  77. package/dist/video/video.js +37 -0
  78. package/dist/video-extraction/extract-frame-via-broadcast-channel.d.ts +16 -0
  79. package/dist/video-extraction/extract-frame-via-broadcast-channel.js +107 -0
  80. package/dist/video-extraction/extract-frame.d.ts +9 -0
  81. package/dist/video-extraction/extract-frame.js +21 -0
  82. package/dist/video-extraction/get-frames-since-keyframe.d.ts +23 -0
  83. package/dist/video-extraction/get-frames-since-keyframe.js +42 -0
  84. package/dist/video-extraction/keyframe-bank.d.ts +25 -0
  85. package/dist/video-extraction/keyframe-bank.js +121 -0
  86. package/dist/video-extraction/keyframe-manager.d.ts +23 -0
  87. package/dist/video-extraction/keyframe-manager.js +171 -0
  88. package/dist/video-extraction/remember-actual-matroska-timestamps.d.ts +5 -0
  89. package/dist/video-extraction/remember-actual-matroska-timestamps.js +19 -0
  90. package/dist/video-for-rendering.d.ts +3 -0
  91. package/dist/video-for-rendering.js +108 -0
  92. package/dist/video.d.ts +3 -0
  93. package/dist/video.js +37 -0
  94. package/package.json +55 -0
@@ -0,0 +1,77 @@
1
+ import { audioManager } from '../caches';
2
+ import { combineAudioDataAndClosePrevious } from '../convert-audiodata/combine-audiodata';
3
+ import { convertAudioData } from '../convert-audiodata/convert-audiodata';
4
+ import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from '../convert-audiodata/resample-audiodata';
5
+ import { sinkPromises } from '../video-extraction/extract-frame';
6
+ import { getSinks } from '../video-extraction/get-frames-since-keyframe';
7
+ export const extractAudio = async ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds, volume, logLevel, loop, }) => {
8
+ if (!sinkPromises[src]) {
9
+ sinkPromises[src] = getSinks(src);
10
+ }
11
+ const { audio, actualMatroskaTimestamps, isMatroska, getDuration } = await sinkPromises[src];
12
+ if (audio === null) {
13
+ return null;
14
+ }
15
+ const timeInSeconds = loop
16
+ ? unloopedTimeInSeconds % (await getDuration())
17
+ : unloopedTimeInSeconds;
18
+ const sampleIterator = await audioManager.getIterator({
19
+ src,
20
+ timeInSeconds,
21
+ audioSampleSink: audio.sampleSink,
22
+ isMatroska,
23
+ actualMatroskaTimestamps,
24
+ });
25
+ const samples = await sampleIterator.getSamples(timeInSeconds, durationInSeconds);
26
+ audioManager.logOpenFrames(logLevel);
27
+ const audioDataArray = [];
28
+ for (let i = 0; i < samples.length; i++) {
29
+ const sample = samples[i];
30
+ // Less than 1 sample would be included - we did not need it after all!
31
+ if (Math.abs(sample.timestamp - (timeInSeconds + durationInSeconds)) *
32
+ sample.sampleRate <
33
+ 1) {
34
+ continue;
35
+ }
36
+ // Less than 1 sample would be included - we did not need it after all!
37
+ if (sample.timestamp + sample.duration <= timeInSeconds) {
38
+ continue;
39
+ }
40
+ const isFirstSample = i === 0;
41
+ const isLastSample = i === samples.length - 1;
42
+ const audioDataRaw = sample.toAudioData();
43
+ // amount of samples to shave from start and end
44
+ let trimStartInSeconds = 0;
45
+ let trimEndInSeconds = 0;
46
+ // TODO: Apply playback rate
47
+ // TODO: Apply tone frequency
48
+ if (isFirstSample) {
49
+ trimStartInSeconds = timeInSeconds - sample.timestamp;
50
+ }
51
+ if (isLastSample) {
52
+ trimEndInSeconds =
53
+ // clamp to 0 in case the audio ends early
54
+ Math.max(0, sample.timestamp +
55
+ sample.duration -
56
+ (timeInSeconds + durationInSeconds));
57
+ }
58
+ const audioData = convertAudioData({
59
+ audioData: audioDataRaw,
60
+ newSampleRate: TARGET_SAMPLE_RATE,
61
+ trimStartInSeconds,
62
+ trimEndInSeconds,
63
+ targetNumberOfChannels: TARGET_NUMBER_OF_CHANNELS,
64
+ volume,
65
+ });
66
+ audioDataRaw.close();
67
+ if (audioData.numberOfFrames === 0) {
68
+ continue;
69
+ }
70
+ audioDataArray.push(audioData);
71
+ }
72
+ if (audioDataArray.length === 0) {
73
+ return null;
74
+ }
75
+ const combined = combineAudioDataAndClosePrevious(audioDataArray);
76
+ return combined;
77
+ };
@@ -0,0 +1,3 @@
1
+ import type React from 'react';
2
+ import type { AudioProps } from './props';
3
+ export declare const AudioForRendering: React.FC<AudioProps>;
@@ -0,0 +1,94 @@
1
+ import { useContext, useLayoutEffect, useMemo, useState } from 'react';
2
+ import { cancelRender, Internals, useCurrentFrame, useDelayRender, useRemotionEnvironment, } from 'remotion';
3
+ import { extractFrameViaBroadcastChannel } from './extract-frame-via-broadcast-channel';
4
+ export const AudioForRendering = ({ volume: volumeProp, playbackRate, src, muted, loopVolumeCurveBehavior, delayRenderRetries, delayRenderTimeoutInMilliseconds, logLevel = window.remotion_logLevel, }) => {
5
+ const absoluteFrame = Internals.useTimelinePosition();
6
+ const videoConfig = Internals.useUnsafeVideoConfig();
7
+ const { registerRenderAsset, unregisterRenderAsset } = useContext(Internals.RenderAssetManager);
8
+ const frame = useCurrentFrame();
9
+ const volumePropsFrame = Internals.useFrameForVolumeProp(loopVolumeCurveBehavior ?? 'repeat');
10
+ const environment = useRemotionEnvironment();
11
+ const [id] = useState(() => `${Math.random()}`.replace('0.', ''));
12
+ if (!videoConfig) {
13
+ throw new Error('No video config found');
14
+ }
15
+ if (!src) {
16
+ throw new TypeError('No `src` was passed to <Video>.');
17
+ }
18
+ const volume = Internals.evaluateVolume({
19
+ volume: volumeProp,
20
+ frame: volumePropsFrame,
21
+ mediaVolume: 1,
22
+ });
23
+ Internals.warnAboutTooHighVolume(volume);
24
+ const shouldRenderAudio = useMemo(() => {
25
+ if (!window.remotion_audioEnabled) {
26
+ return false;
27
+ }
28
+ if (muted) {
29
+ return false;
30
+ }
31
+ if (volume <= 0) {
32
+ return false;
33
+ }
34
+ return true;
35
+ }, [muted, volume]);
36
+ const { fps } = videoConfig;
37
+ const { delayRender, continueRender } = useDelayRender();
38
+ useLayoutEffect(() => {
39
+ const actualFps = playbackRate ? fps / playbackRate : fps;
40
+ const timestamp = frame / actualFps;
41
+ const durationInSeconds = 1 / actualFps;
42
+ const newHandle = delayRender(`Extracting frame number ${frame}`, {
43
+ retries: delayRenderRetries ?? undefined,
44
+ timeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? undefined,
45
+ });
46
+ extractFrameViaBroadcastChannel({
47
+ src,
48
+ timeInSeconds: timestamp,
49
+ durationInSeconds,
50
+ logLevel: logLevel ?? 'info',
51
+ shouldRenderAudio,
52
+ isClientSideRendering: environment.isClientSideRendering,
53
+ })
54
+ .then(({ audio }) => {
55
+ if (audio) {
56
+ registerRenderAsset({
57
+ type: 'inline-audio',
58
+ id,
59
+ audio: Array.from(audio.data),
60
+ sampleRate: audio.sampleRate,
61
+ numberOfChannels: audio.numberOfChannels,
62
+ frame: absoluteFrame,
63
+ timestamp: audio.timestamp,
64
+ duration: (audio.numberOfFrames / audio.sampleRate) * 1000000,
65
+ });
66
+ }
67
+ continueRender(newHandle);
68
+ })
69
+ .catch((error) => {
70
+ cancelRender(error);
71
+ });
72
+ return () => {
73
+ continueRender(newHandle);
74
+ unregisterRenderAsset(id);
75
+ };
76
+ }, [
77
+ absoluteFrame,
78
+ continueRender,
79
+ delayRender,
80
+ delayRenderRetries,
81
+ delayRenderTimeoutInMilliseconds,
82
+ environment.isClientSideRendering,
83
+ fps,
84
+ frame,
85
+ id,
86
+ logLevel,
87
+ playbackRate,
88
+ registerRenderAsset,
89
+ shouldRenderAudio,
90
+ src,
91
+ unregisterRenderAsset,
92
+ ]);
93
+ return null;
94
+ };
@@ -0,0 +1,3 @@
1
+ import React from 'react';
2
+ import type { AudioProps } from './props';
3
+ export declare const Audio: React.FC<AudioProps>;
package/dist/audio.js ADDED
@@ -0,0 +1,60 @@
1
+ import { jsx as _jsx } from "react/jsx-runtime";
2
+ import { useCallback, useContext } from 'react';
3
+ import { cancelRender, Internals, Sequence, useRemotionEnvironment, } from 'remotion';
4
+ import { SharedAudioContext } from '../../core/src/audio/shared-audio-tags';
5
+ import { AudioForRendering } from './audio-for-rendering';
6
+ const { validateMediaTrimProps, resolveTrimProps, validateMediaProps, AudioForPreview, } = Internals;
7
+ export const Audio = (props) => {
8
+ const audioContext = useContext(SharedAudioContext);
9
+ // Should only destruct `trimBefore` and `trimAfter` from props,
10
+ // rest gets drilled down
11
+ const { trimBefore, trimAfter, name, pauseWhenBuffering, stack, showInTimeline, onError: onRemotionError, loop, ...otherProps } = props;
12
+ const environment = useRemotionEnvironment();
13
+ const onDuration = useCallback(() => undefined, []);
14
+ if (typeof props.src !== 'string') {
15
+ throw new TypeError(`The \`<Audio>\` tag requires a string for \`src\`, but got ${JSON.stringify(props.src)} instead.`);
16
+ }
17
+ validateMediaTrimProps({
18
+ startFrom: undefined,
19
+ endAt: undefined,
20
+ trimBefore,
21
+ trimAfter,
22
+ });
23
+ const { trimBeforeValue, trimAfterValue } = resolveTrimProps({
24
+ startFrom: undefined,
25
+ endAt: undefined,
26
+ trimBefore,
27
+ trimAfter,
28
+ });
29
+ const onError = useCallback((e) => {
30
+ // eslint-disable-next-line no-console
31
+ console.log(e.currentTarget.error);
32
+ // If there is no `loop` property, we don't need to get the duration
33
+ // and this does not need to be a fatal error
34
+ const errMessage = `Could not play audio: ${e.currentTarget.error}. See https://remotion.dev/docs/media-playback-error for help.`;
35
+ if (loop) {
36
+ if (onRemotionError) {
37
+ onRemotionError(new Error(errMessage));
38
+ return;
39
+ }
40
+ cancelRender(new Error(errMessage));
41
+ }
42
+ else {
43
+ onRemotionError?.(new Error(errMessage));
44
+ // eslint-disable-next-line no-console
45
+ console.warn(errMessage);
46
+ }
47
+ }, [onRemotionError, loop]);
48
+ if (typeof trimBeforeValue !== 'undefined' ||
49
+ typeof trimAfterValue !== 'undefined') {
50
+ return (_jsx(Sequence, { layout: "none", from: 0 - (trimBeforeValue ?? 0), showInTimeline: false, durationInFrames: trimAfterValue, name: name, children: _jsx(Audio, { pauseWhenBuffering: pauseWhenBuffering ?? false, ...otherProps }) }));
51
+ }
52
+ validateMediaProps(props, 'Video');
53
+ if (environment.isRendering) {
54
+ return _jsx(AudioForRendering, { ...otherProps });
55
+ }
56
+ const { onAutoPlayError, crossOrigin, delayRenderRetries, delayRenderTimeoutInMilliseconds, ...propsForPreview } = otherProps;
57
+ return (_jsx(AudioForPreview, { _remotionInternalNativeLoopPassed: props._remotionInternalNativeLoopPassed ?? false, _remotionInternalStack: stack ?? null, shouldPreMountAudioTags: audioContext !== null && audioContext.numberOfAudioTags > 0, ...propsForPreview, onNativeError: onError, onDuration: onDuration,
58
+ // Proposal: Make this default to true in v5
59
+ pauseWhenBuffering: pauseWhenBuffering ?? false, _remotionInternalNeedsDurationCalculation: Boolean(loop), showInTimeline: showInTimeline ?? true }));
60
+ };
File without changes
@@ -0,0 +1 @@
1
+ "use strict";
@@ -0,0 +1,86 @@
1
+ export declare const MAX_CACHE_SIZE: number;
2
+ export declare const SAFE_BACK_WINDOW_IN_SECONDS = 1;
3
+ export declare const keyframeManager: {
4
+ requestKeyframeBank: ({ packetSink, timestamp, videoSampleSink, src, logLevel, }: {
5
+ timestamp: number;
6
+ packetSink: import("mediabunny").EncodedPacketSink;
7
+ videoSampleSink: import("mediabunny").VideoSampleSink;
8
+ src: string;
9
+ logLevel: import("./log").LogLevel;
10
+ }) => Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
11
+ addKeyframeBank: ({ src, bank, startTimestampInSeconds, }: {
12
+ src: string;
13
+ bank: Promise<import("./video-extraction/keyframe-bank").KeyframeBank>;
14
+ startTimestampInSeconds: number;
15
+ }) => void;
16
+ getCacheStats: () => Promise<{
17
+ count: number;
18
+ totalSize: number;
19
+ }>;
20
+ clearAll: () => Promise<void>;
21
+ };
22
+ export declare const audioManager: {
23
+ makeIterator: ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
24
+ timeInSeconds: number;
25
+ src: string;
26
+ audioSampleSink: import("mediabunny").AudioSampleSink;
27
+ isMatroska: boolean;
28
+ actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
29
+ }) => {
30
+ src: string;
31
+ getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
32
+ waitForCompletion: () => Promise<boolean>;
33
+ canSatisfyRequestedTime: (timestamp: number) => boolean;
34
+ logOpenFrames: (logLevel: import("./log").LogLevel) => void;
35
+ getCacheStats: () => {
36
+ count: number;
37
+ size: number;
38
+ };
39
+ getLastUsed: () => number;
40
+ prepareForDeletion: () => Promise<void>;
41
+ startTimestamp: number;
42
+ };
43
+ getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
44
+ src: string;
45
+ timeInSeconds: number;
46
+ audioSampleSink: import("mediabunny").AudioSampleSink;
47
+ isMatroska: boolean;
48
+ actualMatroskaTimestamps: import("./video-extraction/remember-actual-matroska-timestamps").RememberActualMatroskaTimestamps;
49
+ }) => Promise<{
50
+ src: string;
51
+ getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
52
+ waitForCompletion: () => Promise<boolean>;
53
+ canSatisfyRequestedTime: (timestamp: number) => boolean;
54
+ logOpenFrames: (logLevel: import("./log").LogLevel) => void;
55
+ getCacheStats: () => {
56
+ count: number;
57
+ size: number;
58
+ };
59
+ getLastUsed: () => number;
60
+ prepareForDeletion: () => Promise<void>;
61
+ startTimestamp: number;
62
+ }>;
63
+ getCacheStats: () => {
64
+ count: number;
65
+ totalSize: number;
66
+ };
67
+ getIteratorMostInThePast: () => {
68
+ src: string;
69
+ getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
70
+ waitForCompletion: () => Promise<boolean>;
71
+ canSatisfyRequestedTime: (timestamp: number) => boolean;
72
+ logOpenFrames: (logLevel: import("./log").LogLevel) => void;
73
+ getCacheStats: () => {
74
+ count: number;
75
+ size: number;
76
+ };
77
+ getLastUsed: () => number;
78
+ prepareForDeletion: () => Promise<void>;
79
+ startTimestamp: number;
80
+ } | null;
81
+ logOpenFrames: (logLevel: import("./log").LogLevel) => void;
82
+ };
83
+ export declare const getTotalCacheStats: () => Promise<{
84
+ count: number;
85
+ totalSize: number;
86
+ }>;
package/dist/caches.js ADDED
@@ -0,0 +1,15 @@
1
+ import { makeAudioManager } from './audio-extraction/audio-manager';
2
+ import { makeKeyframeManager } from './video-extraction/keyframe-manager';
3
+ export const MAX_CACHE_SIZE = 1000 * 1000 * 1000; // 1GB
4
+ // TODO: make it dependent on the fps and concurrency
5
+ export const SAFE_BACK_WINDOW_IN_SECONDS = 1;
6
+ export const keyframeManager = makeKeyframeManager();
7
+ export const audioManager = makeAudioManager();
8
+ export const getTotalCacheStats = async () => {
9
+ const keyframeManagerCacheStats = await keyframeManager.getCacheStats();
10
+ const audioManagerCacheStats = audioManager.getCacheStats();
11
+ return {
12
+ count: keyframeManagerCacheStats.count + audioManagerCacheStats.count,
13
+ totalSize: keyframeManagerCacheStats.totalSize + audioManagerCacheStats.totalSize,
14
+ };
15
+ };
@@ -0,0 +1,2 @@
1
+ import type { PcmS16AudioData } from './convert-audiodata';
2
+ export declare const combineAudioDataAndClosePrevious: (audioDataArray: PcmS16AudioData[]) => PcmS16AudioData;
@@ -0,0 +1,40 @@
1
+ export const combineAudioDataAndClosePrevious = (audioDataArray) => {
2
+ let numberOfFrames = 0;
3
+ let numberOfChannels = null;
4
+ let sampleRate = null;
5
+ const { timestamp } = audioDataArray[0];
6
+ for (const audioData of audioDataArray) {
7
+ numberOfFrames += audioData.numberOfFrames;
8
+ if (!numberOfChannels) {
9
+ numberOfChannels = audioData.numberOfChannels;
10
+ }
11
+ else if (numberOfChannels !== audioData.numberOfChannels) {
12
+ throw new Error('Number of channels do not match');
13
+ }
14
+ if (!sampleRate) {
15
+ sampleRate = audioData.sampleRate;
16
+ }
17
+ else if (sampleRate !== audioData.sampleRate) {
18
+ throw new Error('Sample rates do not match');
19
+ }
20
+ }
21
+ if (!numberOfChannels) {
22
+ throw new Error('Number of channels is not set');
23
+ }
24
+ if (!sampleRate) {
25
+ throw new Error('Sample rate is not set');
26
+ }
27
+ const arr = new Int16Array(numberOfFrames * numberOfChannels);
28
+ let offset = 0;
29
+ for (const audioData of audioDataArray) {
30
+ arr.set(audioData.data, offset);
31
+ offset += audioData.data.length;
32
+ }
33
+ return {
34
+ data: arr,
35
+ numberOfChannels,
36
+ numberOfFrames,
37
+ sampleRate,
38
+ timestamp,
39
+ };
40
+ };
@@ -0,0 +1,16 @@
1
+ export type ConvertAudioDataOptions = {
2
+ audioData: AudioData;
3
+ newSampleRate: number;
4
+ trimStartInSeconds: number;
5
+ trimEndInSeconds: number;
6
+ targetNumberOfChannels: number;
7
+ volume: number;
8
+ };
9
+ export type PcmS16AudioData = {
10
+ data: Int16Array;
11
+ sampleRate: number;
12
+ numberOfChannels: number;
13
+ numberOfFrames: number;
14
+ timestamp: number;
15
+ };
16
+ export declare const convertAudioData: ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, volume, }: ConvertAudioDataOptions) => PcmS16AudioData;
@@ -0,0 +1,53 @@
1
+ import { resampleAudioData } from './resample-audiodata';
2
+ const FORMAT = 's16';
3
+ export const convertAudioData = ({ audioData, newSampleRate, trimStartInSeconds, trimEndInSeconds, targetNumberOfChannels, volume, }) => {
4
+ const { numberOfChannels: srcNumberOfChannels, sampleRate: currentSampleRate, numberOfFrames, } = audioData;
5
+ const ratio = currentSampleRate / newSampleRate;
6
+ const frameOffset = Math.round(trimStartInSeconds * audioData.sampleRate);
7
+ const frameCount = numberOfFrames -
8
+ Math.round((trimEndInSeconds + trimStartInSeconds) * audioData.sampleRate);
9
+ const newNumberOfFrames = Math.round(frameCount / ratio);
10
+ if (newNumberOfFrames === 0) {
11
+ throw new Error('Cannot resample - the given sample rate would result in less than 1 sample');
12
+ }
13
+ if (newSampleRate < 3000 || newSampleRate > 768000) {
14
+ throw new Error('newSampleRate must be between 3000 and 768000');
15
+ }
16
+ const srcChannels = new Int16Array(srcNumberOfChannels * frameCount);
17
+ audioData.copyTo(srcChannels, {
18
+ planeIndex: 0,
19
+ format: FORMAT,
20
+ frameOffset,
21
+ frameCount,
22
+ });
23
+ const data = new Int16Array(newNumberOfFrames * targetNumberOfChannels);
24
+ const chunkSize = frameCount / newNumberOfFrames;
25
+ if (newNumberOfFrames === frameCount &&
26
+ targetNumberOfChannels === srcNumberOfChannels &&
27
+ volume === 1) {
28
+ return {
29
+ data: srcChannels,
30
+ numberOfChannels: targetNumberOfChannels,
31
+ numberOfFrames: newNumberOfFrames,
32
+ sampleRate: newSampleRate,
33
+ timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
34
+ };
35
+ }
36
+ resampleAudioData({
37
+ srcNumberOfChannels,
38
+ sourceChannels: srcChannels,
39
+ destination: data,
40
+ targetFrames: newNumberOfFrames,
41
+ chunkSize,
42
+ volume,
43
+ });
44
+ const newAudioData = {
45
+ data,
46
+ format: FORMAT,
47
+ numberOfChannels: targetNumberOfChannels,
48
+ numberOfFrames: newNumberOfFrames,
49
+ sampleRate: newSampleRate,
50
+ timestamp: audioData.timestamp + trimStartInSeconds * 1000000,
51
+ };
52
+ return newAudioData;
53
+ };
@@ -0,0 +1 @@
1
+ export declare const getDataTypeForAudioFormat: (format: AudioSampleFormat) => Float32ArrayConstructor | Int16ArrayConstructor | Uint8ArrayConstructor | Int32ArrayConstructor;
@@ -0,0 +1,22 @@
1
+ export const getDataTypeForAudioFormat = (format) => {
2
+ switch (format) {
3
+ case 'f32':
4
+ return Float32Array;
5
+ case 'f32-planar':
6
+ return Float32Array;
7
+ case 's16':
8
+ return Int16Array;
9
+ case 's16-planar':
10
+ return Int16Array;
11
+ case 'u8':
12
+ return Uint8Array;
13
+ case 'u8-planar':
14
+ return Uint8Array;
15
+ case 's32':
16
+ return Int32Array;
17
+ case 's32-planar':
18
+ return Int32Array;
19
+ default:
20
+ throw new Error(`Unsupported audio format: ${format}`);
21
+ }
22
+ };
@@ -0,0 +1 @@
1
+ export declare const isPlanarFormat: (format: AudioSampleFormat) => boolean;
@@ -0,0 +1,3 @@
1
+ export const isPlanarFormat = (format) => {
2
+ return format.includes('-planar');
3
+ };
@@ -0,0 +1 @@
1
+ export declare const logAudioData: (audioData: AudioData) => string;
@@ -0,0 +1,8 @@
1
+ export const logAudioData = (audioData) => {
2
+ const srcChannels = new Int16Array(audioData.numberOfFrames * audioData.numberOfChannels);
3
+ audioData.copyTo(srcChannels, {
4
+ planeIndex: 0,
5
+ format: 's16',
6
+ });
7
+ return srcChannels.slice(0, 10).join(',');
8
+ };
@@ -0,0 +1,10 @@
1
+ export declare const TARGET_NUMBER_OF_CHANNELS = 2;
2
+ export declare const TARGET_SAMPLE_RATE = 48000;
3
+ export declare const resampleAudioData: ({ srcNumberOfChannels, sourceChannels, destination, targetFrames, chunkSize, volume, }: {
4
+ srcNumberOfChannels: number;
5
+ sourceChannels: Int16Array;
6
+ destination: Int16Array;
7
+ targetFrames: number;
8
+ chunkSize: number;
9
+ volume: number;
10
+ }) => void;
@@ -0,0 +1,72 @@
1
+ // Remotion exports all videos with 2 channels.
2
+ export const TARGET_NUMBER_OF_CHANNELS = 2;
3
+ // Remotion exports all videos with 48kHz sample rate.
4
+ export const TARGET_SAMPLE_RATE = 48000;
5
+ export const resampleAudioData = ({ srcNumberOfChannels, sourceChannels, destination, targetFrames, chunkSize, volume, }) => {
6
+ const getSourceValues = (start, end, channelIndex) => {
7
+ const sampleCountAvg = end - start;
8
+ let itemSum = 0;
9
+ let itemCount = 0;
10
+ for (let k = 0; k < sampleCountAvg; k++) {
11
+ const num = sourceChannels[(start + k) * srcNumberOfChannels + channelIndex];
12
+ itemSum += num;
13
+ itemCount++;
14
+ }
15
+ const average = itemSum / itemCount;
16
+ const averageVolume = average * volume;
17
+ if (averageVolume < -32768) {
18
+ return -32768;
19
+ }
20
+ if (averageVolume > 32767) {
21
+ return 32767;
22
+ }
23
+ return averageVolume;
24
+ };
25
+ for (let newFrameIndex = 0; newFrameIndex < targetFrames; newFrameIndex++) {
26
+ const start = Math.floor(newFrameIndex * chunkSize);
27
+ const end = Math.max(Math.floor(start + chunkSize), start + 1);
28
+ if (TARGET_NUMBER_OF_CHANNELS === srcNumberOfChannels) {
29
+ for (let i = 0; i < srcNumberOfChannels; i++) {
30
+ destination[newFrameIndex * srcNumberOfChannels + i] = getSourceValues(start, end, i);
31
+ }
32
+ }
33
+ // The following formulas were taken from Mediabunnys audio resampler:
34
+ // https://github.com/Vanilagy/mediabunny/blob/b9f7ab2fa2b9167784cbded044d466185308999f/src/conversion.ts
35
+ // Mono to Stereo: M -> L, M -> R
36
+ if (srcNumberOfChannels === 1) {
37
+ const m = getSourceValues(start, end, 0);
38
+ destination[newFrameIndex * 2 + 0] = m;
39
+ destination[newFrameIndex * 2 + 1] = m;
40
+ }
41
+ // Quad to Stereo: 0.5 * (L + SL), 0.5 * (R + SR)
42
+ else if (srcNumberOfChannels === 4) {
43
+ const l = getSourceValues(start, end, 0);
44
+ const r = getSourceValues(start, end, 1);
45
+ const sl = getSourceValues(start, end, 2);
46
+ const sr = getSourceValues(start, end, 3);
47
+ const l2 = 0.5 * (l + sl);
48
+ const r2 = 0.5 * (r + sr);
49
+ destination[newFrameIndex * 2 + 0] = l2;
50
+ destination[newFrameIndex * 2 + 1] = r2;
51
+ }
52
+ // 5.1 to Stereo: L + sqrt(1/2) * (C + SL), R + sqrt(1/2) * (C + SR)
53
+ else if (srcNumberOfChannels === 6) {
54
+ const l = getSourceValues(start, end, 0);
55
+ const r = getSourceValues(start, end, 1);
56
+ const c = getSourceValues(start, end, 2);
57
+ const sl = getSourceValues(start, end, 3);
58
+ const sr = getSourceValues(start, end, 4);
59
+ const l2 = l + Math.sqrt(1 / 2) * (c + sl);
60
+ const r2 = r + Math.sqrt(1 / 2) * (c + sr);
61
+ destination[newFrameIndex * 2 + 0] = l2;
62
+ destination[newFrameIndex * 2 + 1] = r2;
63
+ }
64
+ // Discrete fallback: direct mapping with zero-fill or drop
65
+ else {
66
+ for (let i = 0; i < srcNumberOfChannels; i++) {
67
+ destination[newFrameIndex * TARGET_NUMBER_OF_CHANNELS + i] =
68
+ getSourceValues(start, end, i);
69
+ }
70
+ }
71
+ }
72
+ };
File without changes
@@ -0,0 +1 @@
1
+ "use strict";
@@ -0,0 +1,15 @@
1
+ import type { DataType } from './convert-audiodata/data-types';
2
+ export type SerializableAudioData = {
3
+ data: DataType[];
4
+ format: AudioSampleFormat;
5
+ numberOfChannels: number;
6
+ numberOfFrames: number;
7
+ sampleRate: number;
8
+ };
9
+ export declare const turnAudioDataIntoSerializableData: (audioData: AudioData) => {
10
+ data: (Float32Array<ArrayBuffer> | Int32Array<ArrayBuffer> | Int16Array<ArrayBuffer> | Uint8Array<ArrayBuffer>)[];
11
+ format: AudioSampleFormat;
12
+ numberOfChannels: number;
13
+ numberOfFrames: number;
14
+ sampleRate: number;
15
+ };
@@ -0,0 +1,26 @@
1
+ import { getDataTypeForAudioFormat } from './convert-audiodata/data-types';
2
+ import { isPlanarFormat } from './convert-audiodata/is-planar-format';
3
+ export const turnAudioDataIntoSerializableData = (audioData) => {
4
+ if (!audioData.format) {
5
+ throw new Error('AudioData format is not set');
6
+ }
7
+ const DataType = getDataTypeForAudioFormat(audioData.format);
8
+ const isPlanar = isPlanarFormat(audioData.format);
9
+ const planes = isPlanar ? audioData.numberOfChannels : 1;
10
+ const srcChannels = new Array(planes)
11
+ .fill(true)
12
+ .map(() => new DataType((isPlanar ? 1 : audioData.numberOfChannels) *
13
+ audioData.numberOfFrames));
14
+ for (let i = 0; i < planes; i++) {
15
+ audioData.copyTo(srcChannels[i], {
16
+ planeIndex: i,
17
+ });
18
+ }
19
+ return {
20
+ data: srcChannels,
21
+ format: audioData.format,
22
+ numberOfChannels: audioData.numberOfChannels,
23
+ numberOfFrames: audioData.numberOfFrames,
24
+ sampleRate: audioData.sampleRate,
25
+ };
26
+ };