@remotion/media 4.0.351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +49 -0
- package/README.md +18 -0
- package/dist/audio/audio-for-rendering.d.ts +3 -0
- package/dist/audio/audio-for-rendering.js +99 -0
- package/dist/audio/audio.d.ts +3 -0
- package/dist/audio/audio.js +60 -0
- package/dist/audio/props.d.ts +29 -0
- package/dist/audio/props.js +1 -0
- package/dist/audio-extraction/audio-cache.d.ts +11 -0
- package/dist/audio-extraction/audio-cache.js +58 -0
- package/dist/audio-extraction/audio-iterator.d.ts +24 -0
- package/dist/audio-extraction/audio-iterator.js +106 -0
- package/dist/audio-extraction/audio-manager.d.ts +64 -0
- package/dist/audio-extraction/audio-manager.js +83 -0
- package/dist/audio-extraction/extract-audio.d.ts +10 -0
- package/dist/audio-extraction/extract-audio.js +77 -0
- package/dist/audio-for-rendering.d.ts +3 -0
- package/dist/audio-for-rendering.js +94 -0
- package/dist/audio.d.ts +3 -0
- package/dist/audio.js +60 -0
- package/dist/audiodata-to-array.d.ts +0 -0
- package/dist/audiodata-to-array.js +1 -0
- package/dist/caches.d.ts +86 -0
- package/dist/caches.js +15 -0
- package/dist/convert-audiodata/combine-audiodata.d.ts +2 -0
- package/dist/convert-audiodata/combine-audiodata.js +40 -0
- package/dist/convert-audiodata/convert-audiodata.d.ts +16 -0
- package/dist/convert-audiodata/convert-audiodata.js +53 -0
- package/dist/convert-audiodata/data-types.d.ts +1 -0
- package/dist/convert-audiodata/data-types.js +22 -0
- package/dist/convert-audiodata/is-planar-format.d.ts +1 -0
- package/dist/convert-audiodata/is-planar-format.js +3 -0
- package/dist/convert-audiodata/log-audiodata.d.ts +1 -0
- package/dist/convert-audiodata/log-audiodata.js +8 -0
- package/dist/convert-audiodata/resample-audiodata.d.ts +10 -0
- package/dist/convert-audiodata/resample-audiodata.js +72 -0
- package/dist/convert-audiodata/trim-audiodata.d.ts +0 -0
- package/dist/convert-audiodata/trim-audiodata.js +1 -0
- package/dist/deserialized-audiodata.d.ts +15 -0
- package/dist/deserialized-audiodata.js +26 -0
- package/dist/esm/index.mjs +14487 -0
- package/dist/extract-audio.d.ts +7 -0
- package/dist/extract-audio.js +98 -0
- package/dist/extract-frame-and-audio.d.ts +15 -0
- package/dist/extract-frame-and-audio.js +28 -0
- package/dist/extract-frame-via-broadcast-channel.d.ts +15 -0
- package/dist/extract-frame-via-broadcast-channel.js +104 -0
- package/dist/extract-frame.d.ts +27 -0
- package/dist/extract-frame.js +21 -0
- package/dist/extrct-audio.d.ts +7 -0
- package/dist/extrct-audio.js +94 -0
- package/dist/get-frames-since-keyframe.d.ts +22 -0
- package/dist/get-frames-since-keyframe.js +41 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +2 -0
- package/dist/keyframe-bank.d.ts +25 -0
- package/dist/keyframe-bank.js +120 -0
- package/dist/keyframe-manager.d.ts +23 -0
- package/dist/keyframe-manager.js +170 -0
- package/dist/log.d.ts +10 -0
- package/dist/log.js +33 -0
- package/dist/new-video-for-rendering.d.ts +3 -0
- package/dist/new-video-for-rendering.js +108 -0
- package/dist/new-video.d.ts +3 -0
- package/dist/new-video.js +37 -0
- package/dist/props.d.ts +29 -0
- package/dist/props.js +1 -0
- package/dist/remember-actual-matroska-timestamps.d.ts +4 -0
- package/dist/remember-actual-matroska-timestamps.js +19 -0
- package/dist/serialize-videoframe.d.ts +0 -0
- package/dist/serialize-videoframe.js +1 -0
- package/dist/video/props.d.ts +28 -0
- package/dist/video/props.js +1 -0
- package/dist/video/video-for-rendering.d.ts +3 -0
- package/dist/video/video-for-rendering.js +115 -0
- package/dist/video/video.d.ts +3 -0
- package/dist/video/video.js +37 -0
- package/dist/video-extraction/extract-frame-via-broadcast-channel.d.ts +16 -0
- package/dist/video-extraction/extract-frame-via-broadcast-channel.js +107 -0
- package/dist/video-extraction/extract-frame.d.ts +9 -0
- package/dist/video-extraction/extract-frame.js +21 -0
- package/dist/video-extraction/get-frames-since-keyframe.d.ts +23 -0
- package/dist/video-extraction/get-frames-since-keyframe.js +42 -0
- package/dist/video-extraction/keyframe-bank.d.ts +25 -0
- package/dist/video-extraction/keyframe-bank.js +121 -0
- package/dist/video-extraction/keyframe-manager.d.ts +23 -0
- package/dist/video-extraction/keyframe-manager.js +171 -0
- package/dist/video-extraction/remember-actual-matroska-timestamps.d.ts +5 -0
- package/dist/video-extraction/remember-actual-matroska-timestamps.js +19 -0
- package/dist/video-for-rendering.d.ts +3 -0
- package/dist/video-for-rendering.js +108 -0
- package/dist/video.d.ts +3 -0
- package/dist/video.js +37 -0
- package/package.json +55 -0
package/LICENSE.md
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Remotion License
|
|
2
|
+
|
|
3
|
+
In Remotion 5.0, the license will slightly change. [View the changes here](https://github.com/remotion-dev/remotion/pull/3750).
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Depending on the type of your legal entity, you are granted permission to use Remotion for your project. Individuals and small companies are allowed to use Remotion to create videos for free (even commercial), while a company license is required for for-profit organizations of a certain size. This two-tier system was designed to ensure funding for this project while still allowing the source code to be available and the program to be free for most. Read below for the exact terms of use.
|
|
8
|
+
|
|
9
|
+
- [Free License](#free-license)
|
|
10
|
+
- [Company License](#company-license)
|
|
11
|
+
|
|
12
|
+
## Free License
|
|
13
|
+
|
|
14
|
+
Copyright © 2025 [Remotion](https://www.remotion.dev)
|
|
15
|
+
|
|
16
|
+
### Eligibility
|
|
17
|
+
|
|
18
|
+
You are eligible to use Remotion for free if you are:
|
|
19
|
+
|
|
20
|
+
- an individual
|
|
21
|
+
- a for-profit organization with up to 3 employees
|
|
22
|
+
- a non-profit or not-for-profit organization
|
|
23
|
+
- evaluating whether Remotion is a good fit, and are not yet using it in a commercial way
|
|
24
|
+
|
|
25
|
+
### Allowed use cases
|
|
26
|
+
|
|
27
|
+
Permission is hereby granted, free of charge, to any person eligible for the "Free License", to use the software non-commercially or commercially for the purpose of creating videos and images and to modify the software to their own liking, for the purpose of fulfilling their custom use case or to contribute bug fixes or improvements back to Remotion.
|
|
28
|
+
|
|
29
|
+
### Disallowed use cases
|
|
30
|
+
|
|
31
|
+
It is not allowed to copy or modify Remotion code for the purpose of selling, renting, licensing, relicensing, or sublicensing your own derivate of Remotion.
|
|
32
|
+
|
|
33
|
+
### Warranty notice
|
|
34
|
+
|
|
35
|
+
The software is provided "as is", without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and non-infringement. In no event shall the author or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the software.
|
|
36
|
+
|
|
37
|
+
### Support
|
|
38
|
+
|
|
39
|
+
Support is provided on a best-we-can-do basis via GitHub Issues and Discord.
|
|
40
|
+
|
|
41
|
+
## Company License
|
|
42
|
+
|
|
43
|
+
You are required to obtain a Company License to use Remotion if you are not within the group of entities eligible for a Free License. This license will enable you to use Remotion for the allowed use cases specified in the Free License, and give you access to prioritized support (read the [Support Policy](https://www.remotion.dev/docs/support)).
|
|
44
|
+
|
|
45
|
+
Visit [remotion.pro](https://www.remotion.pro/license) for pricing and to buy a license.
|
|
46
|
+
|
|
47
|
+
### FAQs
|
|
48
|
+
|
|
49
|
+
Are you not sure whether you need a Company License because of an edge case? Here are some [frequently asked questions](https://www.remotion.pro/faq).
|
package/README.md
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# @remotion/media
|
|
2
|
+
|
|
3
|
+
Experimental WebCodecs-based media tags
|
|
4
|
+
|
|
5
|
+
[](https://npmcharts.com/compare/@remotion/media?minimal=true)
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @remotion/media --save-exact
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
When installing a Remotion package, make sure to align the version of all `remotion` and `@remotion/*` packages to the same version.
|
|
14
|
+
Remove the `^` character from the version number to use the exact version.
|
|
15
|
+
|
|
16
|
+
## Usage
|
|
17
|
+
|
|
18
|
+
See the [documentation](https://remotion.dev/docs/media) for more information.
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { useContext, useLayoutEffect, useMemo, useState } from 'react';
|
|
2
|
+
import { cancelRender, Internals, useCurrentFrame, useDelayRender, useRemotionEnvironment, } from 'remotion';
|
|
3
|
+
import { extractFrameViaBroadcastChannel } from '../video-extraction/extract-frame-via-broadcast-channel';
|
|
4
|
+
export const AudioForRendering = ({ volume: volumeProp, playbackRate, src, muted, loopVolumeCurveBehavior, delayRenderRetries, delayRenderTimeoutInMilliseconds, logLevel = window.remotion_logLevel, loop, }) => {
|
|
5
|
+
const absoluteFrame = Internals.useTimelinePosition();
|
|
6
|
+
const videoConfig = Internals.useUnsafeVideoConfig();
|
|
7
|
+
const { registerRenderAsset, unregisterRenderAsset } = useContext(Internals.RenderAssetManager);
|
|
8
|
+
const frame = useCurrentFrame();
|
|
9
|
+
const volumePropsFrame = Internals.useFrameForVolumeProp(loopVolumeCurveBehavior ?? 'repeat');
|
|
10
|
+
const environment = useRemotionEnvironment();
|
|
11
|
+
const [id] = useState(() => `${Math.random()}`.replace('0.', ''));
|
|
12
|
+
if (!videoConfig) {
|
|
13
|
+
throw new Error('No video config found');
|
|
14
|
+
}
|
|
15
|
+
if (!src) {
|
|
16
|
+
throw new TypeError('No `src` was passed to <Audio>.');
|
|
17
|
+
}
|
|
18
|
+
const volume = Internals.evaluateVolume({
|
|
19
|
+
volume: volumeProp,
|
|
20
|
+
frame: volumePropsFrame,
|
|
21
|
+
mediaVolume: 1,
|
|
22
|
+
});
|
|
23
|
+
Internals.warnAboutTooHighVolume(volume);
|
|
24
|
+
const shouldRenderAudio = useMemo(() => {
|
|
25
|
+
if (!window.remotion_audioEnabled) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
if (muted) {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
if (volume <= 0) {
|
|
32
|
+
return false;
|
|
33
|
+
}
|
|
34
|
+
return true;
|
|
35
|
+
}, [muted, volume]);
|
|
36
|
+
const { fps } = videoConfig;
|
|
37
|
+
const { delayRender, continueRender } = useDelayRender();
|
|
38
|
+
useLayoutEffect(() => {
|
|
39
|
+
const actualFps = playbackRate ? fps / playbackRate : fps;
|
|
40
|
+
const timestamp = frame / actualFps;
|
|
41
|
+
const durationInSeconds = 1 / actualFps;
|
|
42
|
+
const newHandle = delayRender(`Extracting audio for frame ${frame}`, {
|
|
43
|
+
retries: delayRenderRetries ?? undefined,
|
|
44
|
+
timeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? undefined,
|
|
45
|
+
});
|
|
46
|
+
extractFrameViaBroadcastChannel({
|
|
47
|
+
src,
|
|
48
|
+
timeInSeconds: timestamp,
|
|
49
|
+
durationInSeconds,
|
|
50
|
+
logLevel: logLevel ?? 'info',
|
|
51
|
+
includeAudio: shouldRenderAudio,
|
|
52
|
+
includeVideo: false,
|
|
53
|
+
isClientSideRendering: environment.isClientSideRendering,
|
|
54
|
+
volume,
|
|
55
|
+
loop: loop ?? false,
|
|
56
|
+
})
|
|
57
|
+
.then(({ audio }) => {
|
|
58
|
+
if (audio) {
|
|
59
|
+
registerRenderAsset({
|
|
60
|
+
type: 'inline-audio',
|
|
61
|
+
id,
|
|
62
|
+
audio: Array.from(audio.data),
|
|
63
|
+
sampleRate: audio.sampleRate,
|
|
64
|
+
numberOfChannels: audio.numberOfChannels,
|
|
65
|
+
frame: absoluteFrame,
|
|
66
|
+
timestamp: audio.timestamp,
|
|
67
|
+
duration: (audio.numberOfFrames / audio.sampleRate) * 1000000,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
continueRender(newHandle);
|
|
71
|
+
})
|
|
72
|
+
.catch((error) => {
|
|
73
|
+
cancelRender(error);
|
|
74
|
+
});
|
|
75
|
+
return () => {
|
|
76
|
+
continueRender(newHandle);
|
|
77
|
+
unregisterRenderAsset(id);
|
|
78
|
+
};
|
|
79
|
+
}, [
|
|
80
|
+
absoluteFrame,
|
|
81
|
+
continueRender,
|
|
82
|
+
delayRender,
|
|
83
|
+
delayRenderRetries,
|
|
84
|
+
delayRenderTimeoutInMilliseconds,
|
|
85
|
+
environment.isClientSideRendering,
|
|
86
|
+
fps,
|
|
87
|
+
frame,
|
|
88
|
+
id,
|
|
89
|
+
logLevel,
|
|
90
|
+
playbackRate,
|
|
91
|
+
registerRenderAsset,
|
|
92
|
+
shouldRenderAudio,
|
|
93
|
+
src,
|
|
94
|
+
unregisterRenderAsset,
|
|
95
|
+
volume,
|
|
96
|
+
loop,
|
|
97
|
+
]);
|
|
98
|
+
return null;
|
|
99
|
+
};
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { jsx as _jsx } from "react/jsx-runtime";
|
|
2
|
+
import { useCallback, useContext } from 'react';
|
|
3
|
+
import { cancelRender, Internals, Sequence, useRemotionEnvironment, } from 'remotion';
|
|
4
|
+
import { SharedAudioContext } from '../../../core/src/audio/shared-audio-tags';
|
|
5
|
+
import { AudioForRendering } from './audio-for-rendering';
|
|
6
|
+
const { validateMediaTrimProps, resolveTrimProps, validateMediaProps, AudioForPreview, } = Internals;
|
|
7
|
+
export const Audio = (props) => {
|
|
8
|
+
const audioContext = useContext(SharedAudioContext);
|
|
9
|
+
// Should only destruct `trimBefore` and `trimAfter` from props,
|
|
10
|
+
// rest gets drilled down
|
|
11
|
+
const { trimBefore, trimAfter, name, pauseWhenBuffering, stack, showInTimeline, onError: onRemotionError, loop, ...otherProps } = props;
|
|
12
|
+
const environment = useRemotionEnvironment();
|
|
13
|
+
const onDuration = useCallback(() => undefined, []);
|
|
14
|
+
if (typeof props.src !== 'string') {
|
|
15
|
+
throw new TypeError(`The \`<Audio>\` tag requires a string for \`src\`, but got ${JSON.stringify(props.src)} instead.`);
|
|
16
|
+
}
|
|
17
|
+
validateMediaTrimProps({
|
|
18
|
+
startFrom: undefined,
|
|
19
|
+
endAt: undefined,
|
|
20
|
+
trimBefore,
|
|
21
|
+
trimAfter,
|
|
22
|
+
});
|
|
23
|
+
const { trimBeforeValue, trimAfterValue } = resolveTrimProps({
|
|
24
|
+
startFrom: undefined,
|
|
25
|
+
endAt: undefined,
|
|
26
|
+
trimBefore,
|
|
27
|
+
trimAfter,
|
|
28
|
+
});
|
|
29
|
+
const onError = useCallback((e) => {
|
|
30
|
+
// eslint-disable-next-line no-console
|
|
31
|
+
console.log(e.currentTarget.error);
|
|
32
|
+
// If there is no `loop` property, we don't need to get the duration
|
|
33
|
+
// and this does not need to be a fatal error
|
|
34
|
+
const errMessage = `Could not play audio: ${e.currentTarget.error}. See https://remotion.dev/docs/media-playback-error for help.`;
|
|
35
|
+
if (loop) {
|
|
36
|
+
if (onRemotionError) {
|
|
37
|
+
onRemotionError(new Error(errMessage));
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
cancelRender(new Error(errMessage));
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
onRemotionError?.(new Error(errMessage));
|
|
44
|
+
// eslint-disable-next-line no-console
|
|
45
|
+
console.warn(errMessage);
|
|
46
|
+
}
|
|
47
|
+
}, [onRemotionError, loop]);
|
|
48
|
+
if (typeof trimBeforeValue !== 'undefined' ||
|
|
49
|
+
typeof trimAfterValue !== 'undefined') {
|
|
50
|
+
return (_jsx(Sequence, { layout: "none", from: 0 - (trimBeforeValue ?? 0), showInTimeline: false, durationInFrames: trimAfterValue, name: name, children: _jsx(Audio, { pauseWhenBuffering: pauseWhenBuffering ?? false, ...otherProps }) }));
|
|
51
|
+
}
|
|
52
|
+
validateMediaProps(props, 'Video');
|
|
53
|
+
if (environment.isRendering) {
|
|
54
|
+
return _jsx(AudioForRendering, { ...otherProps });
|
|
55
|
+
}
|
|
56
|
+
const { onAutoPlayError, crossOrigin, delayRenderRetries, delayRenderTimeoutInMilliseconds, ...propsForPreview } = otherProps;
|
|
57
|
+
return (_jsx(AudioForPreview, { _remotionInternalNativeLoopPassed: props._remotionInternalNativeLoopPassed ?? false, _remotionInternalStack: stack ?? null, shouldPreMountAudioTags: audioContext !== null && audioContext.numberOfAudioTags > 0, ...propsForPreview, onNativeError: onError, onDuration: onDuration,
|
|
58
|
+
// Proposal: Make this default to true in v5
|
|
59
|
+
pauseWhenBuffering: pauseWhenBuffering ?? false, _remotionInternalNeedsDurationCalculation: Boolean(loop), showInTimeline: showInTimeline ?? true }));
|
|
60
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { LoopVolumeCurveBehavior, VolumeProp } from 'remotion';
|
|
2
|
+
import type { LogLevel } from '../log';
|
|
3
|
+
export type AudioProps = {
|
|
4
|
+
src: string;
|
|
5
|
+
trimBefore?: number;
|
|
6
|
+
trimAfter?: number;
|
|
7
|
+
volume?: VolumeProp;
|
|
8
|
+
loopVolumeCurveBehavior?: LoopVolumeCurveBehavior;
|
|
9
|
+
name?: string;
|
|
10
|
+
pauseWhenBuffering?: boolean;
|
|
11
|
+
showInTimeline?: boolean;
|
|
12
|
+
onAutoPlayError?: null | (() => void);
|
|
13
|
+
playbackRate?: number;
|
|
14
|
+
muted?: boolean;
|
|
15
|
+
delayRenderRetries?: number;
|
|
16
|
+
delayRenderTimeoutInMilliseconds?: number;
|
|
17
|
+
crossOrigin?: '' | 'anonymous' | 'use-credentials';
|
|
18
|
+
style?: React.CSSProperties;
|
|
19
|
+
onError?: (err: Error) => void;
|
|
20
|
+
useWebAudioApi?: boolean;
|
|
21
|
+
acceptableTimeShiftInSeconds?: number;
|
|
22
|
+
/**
|
|
23
|
+
* @deprecated For internal use only
|
|
24
|
+
*/
|
|
25
|
+
stack?: string;
|
|
26
|
+
logLevel?: LogLevel;
|
|
27
|
+
loop?: boolean;
|
|
28
|
+
_remotionInternalNativeLoopPassed?: boolean;
|
|
29
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { AudioSample } from 'mediabunny';
|
|
2
|
+
export declare const makeAudioCache: () => {
|
|
3
|
+
addFrame: (sample: AudioSample) => void;
|
|
4
|
+
clearBeforeThreshold: (threshold: number) => void;
|
|
5
|
+
deleteAll: () => void;
|
|
6
|
+
getSamples: (timestamp: number, durationInSeconds: number) => AudioSample[];
|
|
7
|
+
getOldestTimestamp: () => number;
|
|
8
|
+
getNewestTimestamp: () => number;
|
|
9
|
+
getOpenTimestamps: () => number[];
|
|
10
|
+
};
|
|
11
|
+
export type AudioCache = ReturnType<typeof makeAudioCache>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
export const makeAudioCache = () => {
|
|
2
|
+
const timestamps = [];
|
|
3
|
+
const samples = {};
|
|
4
|
+
const addFrame = (sample) => {
|
|
5
|
+
timestamps.push(sample.timestamp);
|
|
6
|
+
samples[sample.timestamp] = sample;
|
|
7
|
+
};
|
|
8
|
+
const clearBeforeThreshold = (threshold) => {
|
|
9
|
+
for (const timestamp of timestamps) {
|
|
10
|
+
const endTimestamp = timestamp + samples[timestamp].duration;
|
|
11
|
+
if (endTimestamp < threshold) {
|
|
12
|
+
samples[timestamp].close();
|
|
13
|
+
delete samples[timestamp];
|
|
14
|
+
timestamps.splice(timestamps.indexOf(timestamp), 1);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
const deleteAll = () => {
|
|
19
|
+
for (const timestamp of timestamps) {
|
|
20
|
+
delete samples[timestamp];
|
|
21
|
+
}
|
|
22
|
+
timestamps.length = 0;
|
|
23
|
+
};
|
|
24
|
+
const getSamples = (timestamp, durationInSeconds) => {
|
|
25
|
+
const selected = [];
|
|
26
|
+
for (let i = 0; i < timestamps.length; i++) {
|
|
27
|
+
const sampleTimestamp = timestamps[i];
|
|
28
|
+
const sample = samples[sampleTimestamp];
|
|
29
|
+
if (sample.timestamp + sample.duration - 0.0000000001 <= timestamp) {
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
if (sample.timestamp >= timestamp + durationInSeconds - 0.0000000001) {
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
selected.push(sample);
|
|
36
|
+
}
|
|
37
|
+
return selected;
|
|
38
|
+
};
|
|
39
|
+
const getOpenTimestamps = () => {
|
|
40
|
+
return timestamps;
|
|
41
|
+
};
|
|
42
|
+
const getOldestTimestamp = () => {
|
|
43
|
+
return timestamps[0];
|
|
44
|
+
};
|
|
45
|
+
const getNewestTimestamp = () => {
|
|
46
|
+
const sample = samples[timestamps[timestamps.length - 1]];
|
|
47
|
+
return sample.timestamp + sample.duration;
|
|
48
|
+
};
|
|
49
|
+
return {
|
|
50
|
+
addFrame,
|
|
51
|
+
clearBeforeThreshold,
|
|
52
|
+
deleteAll,
|
|
53
|
+
getSamples,
|
|
54
|
+
getOldestTimestamp,
|
|
55
|
+
getNewestTimestamp,
|
|
56
|
+
getOpenTimestamps,
|
|
57
|
+
};
|
|
58
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { AudioSample, AudioSampleSink } from 'mediabunny';
|
|
2
|
+
import type { LogLevel } from '../log';
|
|
3
|
+
import type { RememberActualMatroskaTimestamps } from '../video-extraction/remember-actual-matroska-timestamps';
|
|
4
|
+
export declare const makeAudioIterator: ({ audioSampleSink, isMatroska, startTimestamp, src, actualMatroskaTimestamps, }: {
|
|
5
|
+
audioSampleSink: AudioSampleSink;
|
|
6
|
+
isMatroska: boolean;
|
|
7
|
+
startTimestamp: number;
|
|
8
|
+
src: string;
|
|
9
|
+
actualMatroskaTimestamps: RememberActualMatroskaTimestamps;
|
|
10
|
+
}) => {
|
|
11
|
+
src: string;
|
|
12
|
+
getSamples: (ts: number, dur: number) => Promise<AudioSample[]>;
|
|
13
|
+
waitForCompletion: () => Promise<boolean>;
|
|
14
|
+
canSatisfyRequestedTime: (timestamp: number) => boolean;
|
|
15
|
+
logOpenFrames: (logLevel: LogLevel) => void;
|
|
16
|
+
getCacheStats: () => {
|
|
17
|
+
count: number;
|
|
18
|
+
size: number;
|
|
19
|
+
};
|
|
20
|
+
getLastUsed: () => number;
|
|
21
|
+
prepareForDeletion: () => Promise<void>;
|
|
22
|
+
startTimestamp: number;
|
|
23
|
+
};
|
|
24
|
+
export type AudioSampleIterator = ReturnType<typeof makeAudioIterator>;
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { SAFE_BACK_WINDOW_IN_SECONDS } from '../caches';
|
|
2
|
+
import { Log } from '../log';
|
|
3
|
+
import { makeAudioCache } from './audio-cache';
|
|
4
|
+
// https://discord.com/channels/@me/1409810025844838481/1415028953093111870
|
|
5
|
+
// Audio frames might have dependencies on previous and next frames so we need to decode a bit more
|
|
6
|
+
// and then discard it.
|
|
7
|
+
// The worst case seems to be FLAC files with a 65'535 sample window, which would be 1486.0ms at 44.1Khz.
|
|
8
|
+
// So let's set a threshold of 1.5 seconds.
|
|
9
|
+
const extraThreshold = 1.5;
|
|
10
|
+
export const makeAudioIterator = ({ audioSampleSink, isMatroska, startTimestamp, src, actualMatroskaTimestamps, }) => {
|
|
11
|
+
// Matroska timestamps are not accurate unless we start from the beginning
|
|
12
|
+
// So for matroska, we need to decode all samples :(
|
|
13
|
+
// https://github.com/Vanilagy/mediabunny/issues/105
|
|
14
|
+
const sampleIterator = audioSampleSink.samples(isMatroska ? 0 : Math.max(0, startTimestamp - extraThreshold));
|
|
15
|
+
let fullDuration = null;
|
|
16
|
+
const cache = makeAudioCache();
|
|
17
|
+
let lastUsed = Date.now();
|
|
18
|
+
const getNextSample = async () => {
|
|
19
|
+
lastUsed = Date.now();
|
|
20
|
+
const { value: sample, done } = await sampleIterator.next();
|
|
21
|
+
if (done) {
|
|
22
|
+
fullDuration = cache.getNewestTimestamp() ?? null;
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
const realTimestamp = actualMatroskaTimestamps.getRealTimestamp(sample.timestamp);
|
|
26
|
+
if (realTimestamp !== null && realTimestamp !== sample.timestamp) {
|
|
27
|
+
sample.setTimestamp(realTimestamp);
|
|
28
|
+
}
|
|
29
|
+
actualMatroskaTimestamps.observeTimestamp(sample.timestamp);
|
|
30
|
+
actualMatroskaTimestamps.observeTimestamp(sample.timestamp + sample.duration);
|
|
31
|
+
cache.addFrame(sample);
|
|
32
|
+
return sample;
|
|
33
|
+
};
|
|
34
|
+
const getSamples = async (timestamp, durationInSeconds) => {
|
|
35
|
+
lastUsed = Date.now();
|
|
36
|
+
if (fullDuration !== null && timestamp > fullDuration) {
|
|
37
|
+
// Clear all samples before the timestamp
|
|
38
|
+
// Do this in the while loop because samples might start from 0
|
|
39
|
+
cache.clearBeforeThreshold(fullDuration - SAFE_BACK_WINDOW_IN_SECONDS);
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
const samples = cache.getSamples(timestamp, durationInSeconds);
|
|
43
|
+
while (true) {
|
|
44
|
+
// Clear all samples before the timestamp
|
|
45
|
+
// Do this in the while loop because samples might start from 0
|
|
46
|
+
cache.clearBeforeThreshold(timestamp - SAFE_BACK_WINDOW_IN_SECONDS);
|
|
47
|
+
const sample = await getNextSample();
|
|
48
|
+
if (sample === null) {
|
|
49
|
+
break;
|
|
50
|
+
}
|
|
51
|
+
if (sample.timestamp + sample.duration - 0.0000000001 <= timestamp) {
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (sample.timestamp >= timestamp + durationInSeconds - 0.0000000001) {
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
samples.push(sample);
|
|
58
|
+
}
|
|
59
|
+
return samples;
|
|
60
|
+
};
|
|
61
|
+
const logOpenFrames = (logLevel) => {
|
|
62
|
+
Log.verbose(logLevel, '[Audio] Open samples for src', src, cache
|
|
63
|
+
.getOpenTimestamps()
|
|
64
|
+
.map((t) => t.toFixed(3))
|
|
65
|
+
.join(', '));
|
|
66
|
+
};
|
|
67
|
+
const getCacheStats = () => {
|
|
68
|
+
return {
|
|
69
|
+
count: cache.getOpenTimestamps().length,
|
|
70
|
+
size: cache.getOpenTimestamps().reduce((acc, t) => acc + t, 0),
|
|
71
|
+
};
|
|
72
|
+
};
|
|
73
|
+
const canSatisfyRequestedTime = (timestamp) => {
|
|
74
|
+
const oldestTimestamp = cache.getOldestTimestamp() ?? startTimestamp;
|
|
75
|
+
if (fullDuration !== null && timestamp > fullDuration) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
return (oldestTimestamp < timestamp && Math.abs(oldestTimestamp - timestamp) < 10);
|
|
79
|
+
};
|
|
80
|
+
const prepareForDeletion = async () => {
|
|
81
|
+
cache.deleteAll();
|
|
82
|
+
const { value } = await sampleIterator.return();
|
|
83
|
+
if (value) {
|
|
84
|
+
value.close();
|
|
85
|
+
}
|
|
86
|
+
fullDuration = null;
|
|
87
|
+
};
|
|
88
|
+
let op = Promise.resolve([]);
|
|
89
|
+
return {
|
|
90
|
+
src,
|
|
91
|
+
getSamples: (ts, dur) => {
|
|
92
|
+
op = op.then(() => getSamples(ts, dur));
|
|
93
|
+
return op;
|
|
94
|
+
},
|
|
95
|
+
waitForCompletion: async () => {
|
|
96
|
+
await op;
|
|
97
|
+
return true;
|
|
98
|
+
},
|
|
99
|
+
canSatisfyRequestedTime,
|
|
100
|
+
logOpenFrames,
|
|
101
|
+
getCacheStats,
|
|
102
|
+
getLastUsed: () => lastUsed,
|
|
103
|
+
prepareForDeletion,
|
|
104
|
+
startTimestamp,
|
|
105
|
+
};
|
|
106
|
+
};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { AudioSampleSink } from 'mediabunny';
|
|
2
|
+
import type { LogLevel } from '../log';
|
|
3
|
+
import type { RememberActualMatroskaTimestamps } from '../video-extraction/remember-actual-matroska-timestamps';
|
|
4
|
+
export declare const makeAudioManager: () => {
|
|
5
|
+
makeIterator: ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
|
|
6
|
+
timeInSeconds: number;
|
|
7
|
+
src: string;
|
|
8
|
+
audioSampleSink: AudioSampleSink;
|
|
9
|
+
isMatroska: boolean;
|
|
10
|
+
actualMatroskaTimestamps: RememberActualMatroskaTimestamps;
|
|
11
|
+
}) => {
|
|
12
|
+
src: string;
|
|
13
|
+
getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
|
|
14
|
+
waitForCompletion: () => Promise<boolean>;
|
|
15
|
+
canSatisfyRequestedTime: (timestamp: number) => boolean;
|
|
16
|
+
logOpenFrames: (logLevel: LogLevel) => void;
|
|
17
|
+
getCacheStats: () => {
|
|
18
|
+
count: number;
|
|
19
|
+
size: number;
|
|
20
|
+
};
|
|
21
|
+
getLastUsed: () => number;
|
|
22
|
+
prepareForDeletion: () => Promise<void>;
|
|
23
|
+
startTimestamp: number;
|
|
24
|
+
};
|
|
25
|
+
getIterator: ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, }: {
|
|
26
|
+
src: string;
|
|
27
|
+
timeInSeconds: number;
|
|
28
|
+
audioSampleSink: AudioSampleSink;
|
|
29
|
+
isMatroska: boolean;
|
|
30
|
+
actualMatroskaTimestamps: RememberActualMatroskaTimestamps;
|
|
31
|
+
}) => Promise<{
|
|
32
|
+
src: string;
|
|
33
|
+
getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
|
|
34
|
+
waitForCompletion: () => Promise<boolean>;
|
|
35
|
+
canSatisfyRequestedTime: (timestamp: number) => boolean;
|
|
36
|
+
logOpenFrames: (logLevel: LogLevel) => void;
|
|
37
|
+
getCacheStats: () => {
|
|
38
|
+
count: number;
|
|
39
|
+
size: number;
|
|
40
|
+
};
|
|
41
|
+
getLastUsed: () => number;
|
|
42
|
+
prepareForDeletion: () => Promise<void>;
|
|
43
|
+
startTimestamp: number;
|
|
44
|
+
}>;
|
|
45
|
+
getCacheStats: () => {
|
|
46
|
+
count: number;
|
|
47
|
+
totalSize: number;
|
|
48
|
+
};
|
|
49
|
+
getIteratorMostInThePast: () => {
|
|
50
|
+
src: string;
|
|
51
|
+
getSamples: (ts: number, dur: number) => Promise<import("mediabunny").AudioSample[]>;
|
|
52
|
+
waitForCompletion: () => Promise<boolean>;
|
|
53
|
+
canSatisfyRequestedTime: (timestamp: number) => boolean;
|
|
54
|
+
logOpenFrames: (logLevel: LogLevel) => void;
|
|
55
|
+
getCacheStats: () => {
|
|
56
|
+
count: number;
|
|
57
|
+
size: number;
|
|
58
|
+
};
|
|
59
|
+
getLastUsed: () => number;
|
|
60
|
+
prepareForDeletion: () => Promise<void>;
|
|
61
|
+
startTimestamp: number;
|
|
62
|
+
} | null;
|
|
63
|
+
logOpenFrames: (logLevel: LogLevel) => void;
|
|
64
|
+
};
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { getTotalCacheStats, MAX_CACHE_SIZE } from '../caches';
|
|
2
|
+
import { makeAudioIterator } from './audio-iterator';
|
|
3
|
+
export const makeAudioManager = () => {
|
|
4
|
+
const iterators = [];
|
|
5
|
+
const makeIterator = ({ timeInSeconds, src, audioSampleSink, isMatroska, actualMatroskaTimestamps, }) => {
|
|
6
|
+
const iterator = makeAudioIterator({
|
|
7
|
+
audioSampleSink,
|
|
8
|
+
isMatroska,
|
|
9
|
+
startTimestamp: timeInSeconds,
|
|
10
|
+
src,
|
|
11
|
+
actualMatroskaTimestamps,
|
|
12
|
+
});
|
|
13
|
+
iterators.push(iterator);
|
|
14
|
+
return iterator;
|
|
15
|
+
};
|
|
16
|
+
const getIteratorMostInThePast = () => {
|
|
17
|
+
let mostInThePast = null;
|
|
18
|
+
let mostInThePastIterator = null;
|
|
19
|
+
for (const iterator of iterators) {
|
|
20
|
+
const lastUsed = iterator.getLastUsed();
|
|
21
|
+
if (mostInThePast === null || lastUsed < mostInThePast) {
|
|
22
|
+
mostInThePast = lastUsed;
|
|
23
|
+
mostInThePastIterator = iterator;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return mostInThePastIterator;
|
|
27
|
+
};
|
|
28
|
+
const deleteOldestIterator = async () => {
|
|
29
|
+
const iterator = getIteratorMostInThePast();
|
|
30
|
+
if (iterator) {
|
|
31
|
+
await iterator.prepareForDeletion();
|
|
32
|
+
iterators.splice(iterators.indexOf(iterator), 1);
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
const getIterator = async ({ src, timeInSeconds, audioSampleSink, isMatroska, actualMatroskaTimestamps, }) => {
|
|
36
|
+
while ((await getTotalCacheStats()).totalSize > MAX_CACHE_SIZE) {
|
|
37
|
+
await deleteOldestIterator();
|
|
38
|
+
}
|
|
39
|
+
for (const iterator of iterators) {
|
|
40
|
+
if (iterator.src === src &&
|
|
41
|
+
(await iterator.waitForCompletion()) &&
|
|
42
|
+
iterator.canSatisfyRequestedTime(timeInSeconds)) {
|
|
43
|
+
return iterator;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
for (const iterator of iterators) {
|
|
47
|
+
// delete iterator with same starting timestamp
|
|
48
|
+
if (iterator.src === src && iterator.startTimestamp === timeInSeconds) {
|
|
49
|
+
await iterator.prepareForDeletion();
|
|
50
|
+
iterators.splice(iterators.indexOf(iterator), 1);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return makeIterator({
|
|
54
|
+
src,
|
|
55
|
+
timeInSeconds,
|
|
56
|
+
audioSampleSink,
|
|
57
|
+
isMatroska,
|
|
58
|
+
actualMatroskaTimestamps,
|
|
59
|
+
});
|
|
60
|
+
};
|
|
61
|
+
const getCacheStats = () => {
|
|
62
|
+
let totalCount = 0;
|
|
63
|
+
let totalSize = 0;
|
|
64
|
+
for (const iterator of iterators) {
|
|
65
|
+
const { count, size } = iterator.getCacheStats();
|
|
66
|
+
totalCount += count;
|
|
67
|
+
totalSize += size;
|
|
68
|
+
}
|
|
69
|
+
return { count: totalCount, totalSize };
|
|
70
|
+
};
|
|
71
|
+
const logOpenFrames = (logLevel) => {
|
|
72
|
+
for (const iterator of iterators) {
|
|
73
|
+
iterator.logOpenFrames(logLevel);
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
return {
|
|
77
|
+
makeIterator,
|
|
78
|
+
getIterator,
|
|
79
|
+
getCacheStats,
|
|
80
|
+
getIteratorMostInThePast,
|
|
81
|
+
logOpenFrames,
|
|
82
|
+
};
|
|
83
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { PcmS16AudioData } from '../convert-audiodata/convert-audiodata';
|
|
2
|
+
import type { LogLevel } from '../log';
|
|
3
|
+
export declare const extractAudio: ({ src, timeInSeconds: unloopedTimeInSeconds, durationInSeconds, volume, logLevel, loop, }: {
|
|
4
|
+
src: string;
|
|
5
|
+
timeInSeconds: number;
|
|
6
|
+
durationInSeconds: number;
|
|
7
|
+
volume: number;
|
|
8
|
+
logLevel: LogLevel;
|
|
9
|
+
loop: boolean;
|
|
10
|
+
}) => Promise<PcmS16AudioData | null>;
|