@remotion/media 4.0.354 → 4.0.355
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio-extraction/extract-audio.js +2 -6
- package/dist/esm/index.mjs +223 -13130
- package/dist/get-sink-weak.d.ts +18 -0
- package/dist/get-sink-weak.js +23 -0
- package/dist/video-extraction/extract-frame.d.ts +0 -2
- package/dist/video-extraction/extract-frame.js +3 -6
- package/dist/video-extraction/get-frames-since-keyframe.d.ts +2 -2
- package/dist/video-extraction/get-frames-since-keyframe.js +2 -2
- package/package.json +5 -5
- package/dist/audio-for-rendering.d.ts +0 -3
- package/dist/audio-for-rendering.js +0 -94
- package/dist/audio.d.ts +0 -3
- package/dist/audio.js +0 -60
- package/dist/audiodata-to-array.d.ts +0 -0
- package/dist/audiodata-to-array.js +0 -1
- package/dist/convert-audiodata/data-types.d.ts +0 -1
- package/dist/convert-audiodata/data-types.js +0 -22
- package/dist/convert-audiodata/is-planar-format.d.ts +0 -1
- package/dist/convert-audiodata/is-planar-format.js +0 -3
- package/dist/convert-audiodata/log-audiodata.d.ts +0 -1
- package/dist/convert-audiodata/log-audiodata.js +0 -8
- package/dist/convert-audiodata/trim-audiodata.d.ts +0 -0
- package/dist/convert-audiodata/trim-audiodata.js +0 -1
- package/dist/deserialized-audiodata.d.ts +0 -15
- package/dist/deserialized-audiodata.js +0 -26
- package/dist/extract-audio.d.ts +0 -7
- package/dist/extract-audio.js +0 -98
- package/dist/extract-frame-via-broadcast-channel.d.ts +0 -15
- package/dist/extract-frame-via-broadcast-channel.js +0 -104
- package/dist/extract-frame.d.ts +0 -27
- package/dist/extract-frame.js +0 -21
- package/dist/extrct-audio.d.ts +0 -7
- package/dist/extrct-audio.js +0 -94
- package/dist/get-frames-since-keyframe.d.ts +0 -22
- package/dist/get-frames-since-keyframe.js +0 -41
- package/dist/keyframe-bank.d.ts +0 -25
- package/dist/keyframe-bank.js +0 -120
- package/dist/keyframe-manager.d.ts +0 -23
- package/dist/keyframe-manager.js +0 -170
- package/dist/new-video-for-rendering.d.ts +0 -3
- package/dist/new-video-for-rendering.js +0 -108
- package/dist/new-video.d.ts +0 -3
- package/dist/new-video.js +0 -37
- package/dist/props.d.ts +0 -29
- package/dist/props.js +0 -1
- package/dist/remember-actual-matroska-timestamps.d.ts +0 -4
- package/dist/remember-actual-matroska-timestamps.js +0 -19
- package/dist/serialize-videoframe.d.ts +0 -0
- package/dist/serialize-videoframe.js +0 -1
- package/dist/video/media-player.d.ts +0 -62
- package/dist/video/media-player.js +0 -361
- package/dist/video/new-video-for-preview.d.ts +0 -10
- package/dist/video/new-video-for-preview.js +0 -108
- package/dist/video/timeout-utils.d.ts +0 -2
- package/dist/video/timeout-utils.js +0 -18
- package/dist/video/video-for-preview.d.ts +0 -11
- package/dist/video/video-for-preview.js +0 -113
- package/dist/video-extraction/media-player.d.ts +0 -64
- package/dist/video-extraction/media-player.js +0 -501
- package/dist/video-extraction/new-video-for-preview.d.ts +0 -10
- package/dist/video-extraction/new-video-for-preview.js +0 -114
- package/dist/video-for-rendering.d.ts +0 -3
- package/dist/video-for-rendering.js +0 -108
- package/dist/video.d.ts +0 -3
- package/dist/video.js +0 -37
package/dist/extract-frame.js
DELETED
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import { getSinks, } from './video-extraction/get-frames-since-keyframe';
|
|
2
|
-
import { makeKeyframeManager } from './video-extraction/keyframe-manager';
|
|
3
|
-
export const keyframeManager = makeKeyframeManager();
|
|
4
|
-
export const sinkPromises = {};
|
|
5
|
-
export const extractFrame = async ({ src, timeInSeconds, logLevel, }) => {
|
|
6
|
-
console.time('extractFrame');
|
|
7
|
-
if (!sinkPromises[src]) {
|
|
8
|
-
sinkPromises[src] = getSinks(src);
|
|
9
|
-
}
|
|
10
|
-
const { video } = await sinkPromises[src];
|
|
11
|
-
const keyframeBank = await keyframeManager.requestKeyframeBank({
|
|
12
|
-
packetSink: video.packetSink,
|
|
13
|
-
videoSampleSink: video.sampleSink,
|
|
14
|
-
timestamp: timeInSeconds,
|
|
15
|
-
src,
|
|
16
|
-
logLevel,
|
|
17
|
-
});
|
|
18
|
-
const frame = await keyframeBank.getFrameFromTimestamp(timeInSeconds);
|
|
19
|
-
console.timeEnd('extractFrame');
|
|
20
|
-
return frame;
|
|
21
|
-
};
|
package/dist/extrct-audio.d.ts
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import type { PcmS16AudioData } from './convert-audiodata/convert-audiodata';
|
|
2
|
-
export declare const extractAudio: ({ src, timeInSeconds, durationInSeconds, volume, }: {
|
|
3
|
-
src: string;
|
|
4
|
-
timeInSeconds: number;
|
|
5
|
-
durationInSeconds: number;
|
|
6
|
-
volume: number;
|
|
7
|
-
}) => Promise<PcmS16AudioData | null>;
|
package/dist/extrct-audio.js
DELETED
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
import { combineAudioDataAndClosePrevious } from './convert-audiodata/combine-audiodata';
|
|
2
|
-
import { convertAudioData } from './convert-audiodata/convert-audiodata';
|
|
3
|
-
import { TARGET_NUMBER_OF_CHANNELS, TARGET_SAMPLE_RATE, } from './convert-audiodata/resample-audiodata';
|
|
4
|
-
import { sinkPromises } from './extract-frame';
|
|
5
|
-
import { getSinks } from './get-frames-since-keyframe';
|
|
6
|
-
export const extractAudio = async ({ src, timeInSeconds, durationInSeconds, volume, }) => {
|
|
7
|
-
if (!sinkPromises[src]) {
|
|
8
|
-
sinkPromises[src] = getSinks(src);
|
|
9
|
-
}
|
|
10
|
-
const { audio, actualMatroskaTimestamps, isMatroska } = await sinkPromises[src];
|
|
11
|
-
if (audio === null) {
|
|
12
|
-
return null;
|
|
13
|
-
}
|
|
14
|
-
// https://discord.com/channels/@me/1409810025844838481/1415028953093111870
|
|
15
|
-
// Audio frames might have dependencies on previous and next frames so we need to decode a bit more
|
|
16
|
-
// and then discard it.
|
|
17
|
-
// The worst case seems to be FLAC files with a 65'535 sample window, which would be 1486.0ms at 44.1Khz.
|
|
18
|
-
// So let's set a threshold of 1.5 seconds.
|
|
19
|
-
const extraThreshold = 1.5;
|
|
20
|
-
// Matroska timestamps are not accurate unless we start from the beginning
|
|
21
|
-
// So for matroska, we need to decode all samples :(
|
|
22
|
-
// https://github.com/Vanilagy/mediabunny/issues/105
|
|
23
|
-
const sampleIterator = audio.sampleSink.samples(isMatroska ? 0 : Math.max(0, timeInSeconds - extraThreshold), timeInSeconds + durationInSeconds);
|
|
24
|
-
const samples = [];
|
|
25
|
-
for await (const sample of sampleIterator) {
|
|
26
|
-
const realTimestamp = actualMatroskaTimestamps.getRealTimestamp(sample.timestamp);
|
|
27
|
-
if (realTimestamp !== null && realTimestamp !== sample.timestamp) {
|
|
28
|
-
sample.setTimestamp(realTimestamp);
|
|
29
|
-
}
|
|
30
|
-
actualMatroskaTimestamps.observeTimestamp(sample.timestamp);
|
|
31
|
-
actualMatroskaTimestamps.observeTimestamp(sample.timestamp + sample.duration);
|
|
32
|
-
if (sample.timestamp + sample.duration - 0.0000000001 <= timeInSeconds) {
|
|
33
|
-
continue;
|
|
34
|
-
}
|
|
35
|
-
if (sample.timestamp >= timeInSeconds + durationInSeconds - 0.0000000001) {
|
|
36
|
-
continue;
|
|
37
|
-
}
|
|
38
|
-
samples.push(sample);
|
|
39
|
-
}
|
|
40
|
-
const audioDataArray = [];
|
|
41
|
-
for (let i = 0; i < samples.length; i++) {
|
|
42
|
-
const sample = samples[i];
|
|
43
|
-
// Less than 1 sample would be included - we did not need it after all!
|
|
44
|
-
if (Math.abs(sample.timestamp - (timeInSeconds + durationInSeconds)) *
|
|
45
|
-
sample.sampleRate <
|
|
46
|
-
1) {
|
|
47
|
-
sample.close();
|
|
48
|
-
continue;
|
|
49
|
-
}
|
|
50
|
-
// Less than 1 sample would be included - we did not need it after all!
|
|
51
|
-
if (sample.timestamp + sample.duration <= timeInSeconds) {
|
|
52
|
-
sample.close();
|
|
53
|
-
continue;
|
|
54
|
-
}
|
|
55
|
-
const isFirstSample = i === 0;
|
|
56
|
-
const isLastSample = i === samples.length - 1;
|
|
57
|
-
const audioDataRaw = sample.toAudioData();
|
|
58
|
-
// amount of samples to shave from start and end
|
|
59
|
-
let trimStartInSeconds = 0;
|
|
60
|
-
let trimEndInSeconds = 0;
|
|
61
|
-
// TODO: Apply playback rate
|
|
62
|
-
// TODO: Apply tone frequency
|
|
63
|
-
if (isFirstSample) {
|
|
64
|
-
trimStartInSeconds = timeInSeconds - sample.timestamp;
|
|
65
|
-
}
|
|
66
|
-
if (isLastSample) {
|
|
67
|
-
trimEndInSeconds =
|
|
68
|
-
// clamp to 0 in case the audio ends early
|
|
69
|
-
Math.max(0, sample.timestamp +
|
|
70
|
-
sample.duration -
|
|
71
|
-
(timeInSeconds + durationInSeconds));
|
|
72
|
-
}
|
|
73
|
-
const audioData = convertAudioData({
|
|
74
|
-
audioData: audioDataRaw,
|
|
75
|
-
newSampleRate: TARGET_SAMPLE_RATE,
|
|
76
|
-
trimStartInSeconds,
|
|
77
|
-
trimEndInSeconds,
|
|
78
|
-
targetNumberOfChannels: TARGET_NUMBER_OF_CHANNELS,
|
|
79
|
-
volume,
|
|
80
|
-
});
|
|
81
|
-
audioDataRaw.close();
|
|
82
|
-
if (audioData.numberOfFrames === 0) {
|
|
83
|
-
sample.close();
|
|
84
|
-
continue;
|
|
85
|
-
}
|
|
86
|
-
audioDataArray.push(audioData);
|
|
87
|
-
sample.close();
|
|
88
|
-
}
|
|
89
|
-
if (audioDataArray.length === 0) {
|
|
90
|
-
return null;
|
|
91
|
-
}
|
|
92
|
-
const combined = combineAudioDataAndClosePrevious(audioDataArray);
|
|
93
|
-
return combined;
|
|
94
|
-
};
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import type { EncodedPacket } from 'mediabunny';
|
|
2
|
-
import { AudioSampleSink, EncodedPacketSink, VideoSampleSink } from 'mediabunny';
|
|
3
|
-
export declare const getSinks: (src: string) => Promise<{
|
|
4
|
-
video: {
|
|
5
|
-
sampleSink: VideoSampleSink;
|
|
6
|
-
packetSink: EncodedPacketSink;
|
|
7
|
-
};
|
|
8
|
-
audio: {
|
|
9
|
-
sampleSink: AudioSampleSink;
|
|
10
|
-
} | null;
|
|
11
|
-
actualMatroskaTimestamps: {
|
|
12
|
-
observeTimestamp: (startTime: number) => void;
|
|
13
|
-
getRealTimestamp: (observedTimestamp: number) => number | null;
|
|
14
|
-
};
|
|
15
|
-
isMatroska: boolean;
|
|
16
|
-
}>;
|
|
17
|
-
export type GetSink = Awaited<ReturnType<typeof getSinks>>;
|
|
18
|
-
export declare const getFramesSinceKeyframe: ({ packetSink, videoSampleSink, startPacket, }: {
|
|
19
|
-
packetSink: EncodedPacketSink;
|
|
20
|
-
videoSampleSink: VideoSampleSink;
|
|
21
|
-
startPacket: EncodedPacket;
|
|
22
|
-
}) => Promise<any>;
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
import { ALL_FORMATS, AudioSampleSink, EncodedPacketSink, Input, MATROSKA, UrlSource, VideoSampleSink, } from 'mediabunny';
|
|
2
|
-
import { makeKeyframeBank } from './keyframe-bank';
|
|
3
|
-
import { rememberActualMatroskaTimestamps } from './remember-actual-matroska-timestamps';
|
|
4
|
-
export const getSinks = async (src) => {
|
|
5
|
-
const input = new Input({
|
|
6
|
-
formats: ALL_FORMATS,
|
|
7
|
-
source: new UrlSource(src),
|
|
8
|
-
});
|
|
9
|
-
const format = await input.getFormat();
|
|
10
|
-
const videoTrack = await input.getPrimaryVideoTrack();
|
|
11
|
-
if (!videoTrack) {
|
|
12
|
-
throw new Error(`No video track found for ${src}`);
|
|
13
|
-
}
|
|
14
|
-
const audioTrack = await input.getPrimaryAudioTrack();
|
|
15
|
-
const isMatroska = format === MATROSKA;
|
|
16
|
-
return {
|
|
17
|
-
video: {
|
|
18
|
-
sampleSink: new VideoSampleSink(videoTrack),
|
|
19
|
-
packetSink: new EncodedPacketSink(videoTrack),
|
|
20
|
-
},
|
|
21
|
-
audio: audioTrack
|
|
22
|
-
? {
|
|
23
|
-
sampleSink: new AudioSampleSink(audioTrack),
|
|
24
|
-
}
|
|
25
|
-
: null,
|
|
26
|
-
actualMatroskaTimestamps: rememberActualMatroskaTimestamps(isMatroska),
|
|
27
|
-
isMatroska,
|
|
28
|
-
};
|
|
29
|
-
};
|
|
30
|
-
export const getFramesSinceKeyframe = async ({ packetSink, videoSampleSink, startPacket, }) => {
|
|
31
|
-
const nextKeyPacket = await packetSink.getNextKeyPacket(startPacket, {
|
|
32
|
-
verifyKeyPackets: false,
|
|
33
|
-
});
|
|
34
|
-
const sampleIterator = videoSampleSink.samples(startPacket.timestamp, nextKeyPacket ? nextKeyPacket.timestamp : Infinity);
|
|
35
|
-
const keyframeBank = makeKeyframeBank({
|
|
36
|
-
startTimestampInSeconds: startPacket.timestamp,
|
|
37
|
-
endTimestampInSeconds: nextKeyPacket ? nextKeyPacket.timestamp : Infinity,
|
|
38
|
-
sampleIterator,
|
|
39
|
-
});
|
|
40
|
-
return keyframeBank;
|
|
41
|
-
};
|
package/dist/keyframe-bank.d.ts
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
import type { VideoSample } from 'mediabunny';
|
|
2
|
-
import type { LogLevel } from './log';
|
|
3
|
-
export type KeyframeBank = {
|
|
4
|
-
startTimestampInSeconds: number;
|
|
5
|
-
endTimestampInSeconds: number;
|
|
6
|
-
getFrameFromTimestamp: (timestamp: number) => Promise<VideoSample | null>;
|
|
7
|
-
prepareForDeletion: () => Promise<void>;
|
|
8
|
-
deleteFramesBeforeTimestamp: ({ logLevel, src, timestampInSeconds, }: {
|
|
9
|
-
timestampInSeconds: number;
|
|
10
|
-
logLevel: LogLevel;
|
|
11
|
-
src: string;
|
|
12
|
-
}) => void;
|
|
13
|
-
hasTimestampInSecond: (timestamp: number) => Promise<boolean>;
|
|
14
|
-
addFrame: (frame: VideoSample) => void;
|
|
15
|
-
getOpenFrameCount: () => {
|
|
16
|
-
size: number;
|
|
17
|
-
timestamps: number[];
|
|
18
|
-
};
|
|
19
|
-
getLastUsed: () => number;
|
|
20
|
-
};
|
|
21
|
-
export declare const makeKeyframeBank: ({ startTimestampInSeconds, endTimestampInSeconds, sampleIterator, }: {
|
|
22
|
-
startTimestampInSeconds: number;
|
|
23
|
-
endTimestampInSeconds: number;
|
|
24
|
-
sampleIterator: AsyncGenerator<VideoSample, void, unknown>;
|
|
25
|
-
}) => KeyframeBank;
|
package/dist/keyframe-bank.js
DELETED
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
import { Log } from './log';
|
|
2
|
-
// Round to only 4 digits, because WebM has a timescale of 1_000, e.g. framer.webm
|
|
3
|
-
const roundTo4Digits = (timestamp) => {
|
|
4
|
-
return Math.round(timestamp * 1000) / 1000;
|
|
5
|
-
};
|
|
6
|
-
export const makeKeyframeBank = ({ startTimestampInSeconds, endTimestampInSeconds, sampleIterator, }) => {
|
|
7
|
-
const frames = {};
|
|
8
|
-
const frameTimestamps = [];
|
|
9
|
-
let lastUsed = Date.now();
|
|
10
|
-
let alloctionSize = 0;
|
|
11
|
-
const hasDecodedEnoughForTimestamp = (timestamp) => {
|
|
12
|
-
const lastFrameTimestamp = frameTimestamps[frameTimestamps.length - 1];
|
|
13
|
-
if (!lastFrameTimestamp) {
|
|
14
|
-
return false;
|
|
15
|
-
}
|
|
16
|
-
const lastFrame = frames[lastFrameTimestamp];
|
|
17
|
-
// Don't decode more, will probably have to re-decode everything
|
|
18
|
-
if (!lastFrame) {
|
|
19
|
-
return true;
|
|
20
|
-
}
|
|
21
|
-
return (roundTo4Digits(lastFrame.timestamp + lastFrame.duration) >
|
|
22
|
-
roundTo4Digits(timestamp));
|
|
23
|
-
};
|
|
24
|
-
const addFrame = (frame) => {
|
|
25
|
-
frames[frame.timestamp] = frame;
|
|
26
|
-
frameTimestamps.push(frame.timestamp);
|
|
27
|
-
alloctionSize += frame.allocationSize();
|
|
28
|
-
lastUsed = Date.now();
|
|
29
|
-
};
|
|
30
|
-
const ensureEnoughFramesForTimestamp = async (timestamp) => {
|
|
31
|
-
while (!hasDecodedEnoughForTimestamp(timestamp)) {
|
|
32
|
-
const sample = await sampleIterator.next();
|
|
33
|
-
if (sample.value) {
|
|
34
|
-
addFrame(sample.value);
|
|
35
|
-
}
|
|
36
|
-
if (sample.done) {
|
|
37
|
-
break;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
lastUsed = Date.now();
|
|
41
|
-
};
|
|
42
|
-
const getFrameFromTimestamp = async (timestampInSeconds) => {
|
|
43
|
-
lastUsed = Date.now();
|
|
44
|
-
if (timestampInSeconds < startTimestampInSeconds) {
|
|
45
|
-
return Promise.reject(new Error(`Timestamp is before start timestamp (requested: ${timestampInSeconds}sec, start: ${startTimestampInSeconds})`));
|
|
46
|
-
}
|
|
47
|
-
if (timestampInSeconds > endTimestampInSeconds) {
|
|
48
|
-
return Promise.reject(new Error(`Timestamp is after end timestamp (requested: ${timestampInSeconds}sec, end: ${endTimestampInSeconds})`));
|
|
49
|
-
}
|
|
50
|
-
await ensureEnoughFramesForTimestamp(timestampInSeconds);
|
|
51
|
-
for (let i = frameTimestamps.length - 1; i >= 0; i--) {
|
|
52
|
-
const sample = frames[frameTimestamps[i]];
|
|
53
|
-
if (!sample) {
|
|
54
|
-
return null;
|
|
55
|
-
}
|
|
56
|
-
if (roundTo4Digits(sample.timestamp) <= roundTo4Digits(timestampInSeconds)) {
|
|
57
|
-
return sample;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
throw new Error('No frame found for timestamp ' + timestampInSeconds);
|
|
61
|
-
};
|
|
62
|
-
const hasTimestampInSecond = async (timestamp) => {
|
|
63
|
-
return (await getFrameFromTimestamp(timestamp)) !== null;
|
|
64
|
-
};
|
|
65
|
-
const prepareForDeletion = async () => {
|
|
66
|
-
// Cleanup frames that have been extracted that might not have been retrieved yet
|
|
67
|
-
const { value } = await sampleIterator.return();
|
|
68
|
-
if (value) {
|
|
69
|
-
value.close();
|
|
70
|
-
}
|
|
71
|
-
for (const frameTimestamp of frameTimestamps) {
|
|
72
|
-
if (!frames[frameTimestamp]) {
|
|
73
|
-
continue;
|
|
74
|
-
}
|
|
75
|
-
alloctionSize -= frames[frameTimestamp].allocationSize();
|
|
76
|
-
frames[frameTimestamp].close();
|
|
77
|
-
delete frames[frameTimestamp];
|
|
78
|
-
}
|
|
79
|
-
frameTimestamps.length = 0;
|
|
80
|
-
};
|
|
81
|
-
const deleteFramesBeforeTimestamp = ({ logLevel, src, timestampInSeconds, }) => {
|
|
82
|
-
for (const frameTimestamp of frameTimestamps) {
|
|
83
|
-
const isLast = frameTimestamp === frameTimestamps[frameTimestamps.length - 1];
|
|
84
|
-
// Don't delete the last frame, since it may be the last one in the video!
|
|
85
|
-
if (isLast) {
|
|
86
|
-
continue;
|
|
87
|
-
}
|
|
88
|
-
if (frameTimestamp < timestampInSeconds) {
|
|
89
|
-
if (!frames[frameTimestamp]) {
|
|
90
|
-
continue;
|
|
91
|
-
}
|
|
92
|
-
alloctionSize -= frames[frameTimestamp].allocationSize();
|
|
93
|
-
frames[frameTimestamp].close();
|
|
94
|
-
delete frames[frameTimestamp];
|
|
95
|
-
Log.verbose(logLevel, `[Video] Deleted frame ${frameTimestamp} for src ${src}`);
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
};
|
|
99
|
-
const getOpenFrameCount = () => {
|
|
100
|
-
return {
|
|
101
|
-
size: alloctionSize,
|
|
102
|
-
timestamps: frameTimestamps,
|
|
103
|
-
};
|
|
104
|
-
};
|
|
105
|
-
const getLastUsed = () => {
|
|
106
|
-
return lastUsed;
|
|
107
|
-
};
|
|
108
|
-
const keyframeBank = {
|
|
109
|
-
startTimestampInSeconds,
|
|
110
|
-
endTimestampInSeconds,
|
|
111
|
-
getFrameFromTimestamp,
|
|
112
|
-
prepareForDeletion,
|
|
113
|
-
hasTimestampInSecond,
|
|
114
|
-
addFrame,
|
|
115
|
-
deleteFramesBeforeTimestamp,
|
|
116
|
-
getOpenFrameCount,
|
|
117
|
-
getLastUsed,
|
|
118
|
-
};
|
|
119
|
-
return keyframeBank;
|
|
120
|
-
};
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import type { EncodedPacketSink, VideoSampleSink } from 'mediabunny';
|
|
2
|
-
import { type KeyframeBank } from './keyframe-bank';
|
|
3
|
-
import type { LogLevel } from './log';
|
|
4
|
-
export declare const makeKeyframeManager: () => {
|
|
5
|
-
requestKeyframeBank: ({ packetSink, timestamp, videoSampleSink, src, logLevel, }: {
|
|
6
|
-
timestamp: number;
|
|
7
|
-
packetSink: EncodedPacketSink;
|
|
8
|
-
videoSampleSink: VideoSampleSink;
|
|
9
|
-
src: string;
|
|
10
|
-
logLevel: LogLevel;
|
|
11
|
-
}) => Promise<KeyframeBank>;
|
|
12
|
-
addKeyframeBank: ({ src, bank, startTimestampInSeconds, }: {
|
|
13
|
-
src: string;
|
|
14
|
-
bank: Promise<KeyframeBank>;
|
|
15
|
-
startTimestampInSeconds: number;
|
|
16
|
-
}) => void;
|
|
17
|
-
getCacheStats: () => Promise<{
|
|
18
|
-
count: number;
|
|
19
|
-
totalSize: number;
|
|
20
|
-
}>;
|
|
21
|
-
clearAll: () => Promise<void>;
|
|
22
|
-
};
|
|
23
|
-
export type KeyframeManager = Awaited<ReturnType<typeof makeKeyframeManager>>;
|
package/dist/keyframe-manager.js
DELETED
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
import { getFramesSinceKeyframe } from './get-frames-since-keyframe';
|
|
2
|
-
import { Log } from './log';
|
|
3
|
-
const MAX_CACHE_SIZE = 1000 * 1000 * 1000; // 1GB
|
|
4
|
-
export const makeKeyframeManager = () => {
|
|
5
|
-
// src => {[startTimestampInSeconds]: KeyframeBank
|
|
6
|
-
const sources = {};
|
|
7
|
-
const addKeyframeBank = ({ src, bank, startTimestampInSeconds, }) => {
|
|
8
|
-
sources[src] = sources[src] ?? {};
|
|
9
|
-
sources[src][startTimestampInSeconds] = bank;
|
|
10
|
-
};
|
|
11
|
-
const logCacheStats = async (logLevel) => {
|
|
12
|
-
let count = 0;
|
|
13
|
-
let totalSize = 0;
|
|
14
|
-
for (const src in sources) {
|
|
15
|
-
for (const bank in sources[src]) {
|
|
16
|
-
const v = await sources[src][bank];
|
|
17
|
-
const { size, timestamps } = v.getOpenFrameCount();
|
|
18
|
-
count += timestamps.length;
|
|
19
|
-
totalSize += size;
|
|
20
|
-
if (size === 0) {
|
|
21
|
-
continue;
|
|
22
|
-
}
|
|
23
|
-
Log.verbose(logLevel, `[Video] Open frames for src ${src}: ${timestamps.join(', ')}}`);
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
Log.verbose(logLevel, `[Video] Cache stats: ${count} open frames, ${totalSize} bytes`);
|
|
27
|
-
};
|
|
28
|
-
const getCacheStats = async () => {
|
|
29
|
-
let count = 0;
|
|
30
|
-
let totalSize = 0;
|
|
31
|
-
for (const src in sources) {
|
|
32
|
-
for (const bank in sources[src]) {
|
|
33
|
-
const v = await sources[src][bank];
|
|
34
|
-
const { timestamps, size } = v.getOpenFrameCount();
|
|
35
|
-
count += timestamps.length;
|
|
36
|
-
totalSize += size;
|
|
37
|
-
if (size === 0) {
|
|
38
|
-
continue;
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
return { count, totalSize };
|
|
43
|
-
};
|
|
44
|
-
const getTheKeyframeBankMostInThePast = async () => {
|
|
45
|
-
let mostInThePast = null;
|
|
46
|
-
let mostInThePastBank = null;
|
|
47
|
-
for (const src in sources) {
|
|
48
|
-
for (const b in sources[src]) {
|
|
49
|
-
const bank = await sources[src][b];
|
|
50
|
-
const lastUsed = bank.getLastUsed();
|
|
51
|
-
if (mostInThePast === null || lastUsed < mostInThePast) {
|
|
52
|
-
mostInThePast = lastUsed;
|
|
53
|
-
mostInThePastBank = { src, bank };
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
if (!mostInThePastBank) {
|
|
58
|
-
throw new Error('No keyframe bank found');
|
|
59
|
-
}
|
|
60
|
-
return mostInThePastBank;
|
|
61
|
-
};
|
|
62
|
-
const ensureToStayUnderMaxCacheSize = async (logLevel) => {
|
|
63
|
-
let cacheStats = await getCacheStats();
|
|
64
|
-
while (cacheStats.totalSize > MAX_CACHE_SIZE) {
|
|
65
|
-
const { bank: mostInThePastBank, src: mostInThePastSrc } = await getTheKeyframeBankMostInThePast();
|
|
66
|
-
if (mostInThePastBank) {
|
|
67
|
-
await mostInThePastBank.prepareForDeletion();
|
|
68
|
-
delete sources[mostInThePastSrc][mostInThePastBank.startTimestampInSeconds];
|
|
69
|
-
Log.verbose(logLevel, `[Video] Deleted frames for src ${mostInThePastSrc} from ${mostInThePastBank.startTimestampInSeconds}sec to ${mostInThePastBank.endTimestampInSeconds}sec to free up memory.`);
|
|
70
|
-
}
|
|
71
|
-
cacheStats = await getCacheStats();
|
|
72
|
-
}
|
|
73
|
-
};
|
|
74
|
-
const clearKeyframeBanksBeforeTime = async ({ timestampInSeconds, src, logLevel, }) => {
|
|
75
|
-
// TODO: make it dependent on the fps and concurrency
|
|
76
|
-
const SAFE_BACK_WINDOW_IN_SECONDS = 1;
|
|
77
|
-
const threshold = timestampInSeconds - SAFE_BACK_WINDOW_IN_SECONDS;
|
|
78
|
-
if (!sources[src]) {
|
|
79
|
-
return;
|
|
80
|
-
}
|
|
81
|
-
const banks = Object.keys(sources[src]);
|
|
82
|
-
for (const startTimeInSeconds of banks) {
|
|
83
|
-
const bank = await sources[src][startTimeInSeconds];
|
|
84
|
-
const { endTimestampInSeconds, startTimestampInSeconds } = bank;
|
|
85
|
-
if (endTimestampInSeconds < threshold) {
|
|
86
|
-
await bank.prepareForDeletion();
|
|
87
|
-
Log.verbose(logLevel, `[Video] Cleared frames for src ${src} from ${startTimestampInSeconds}sec to ${endTimestampInSeconds}sec`);
|
|
88
|
-
delete sources[src][startTimeInSeconds];
|
|
89
|
-
}
|
|
90
|
-
else {
|
|
91
|
-
bank.deleteFramesBeforeTimestamp({
|
|
92
|
-
timestampInSeconds: threshold,
|
|
93
|
-
logLevel,
|
|
94
|
-
src,
|
|
95
|
-
});
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
await logCacheStats(logLevel);
|
|
99
|
-
};
|
|
100
|
-
const getKeyframeBankOrRefetch = async ({ packetSink, timestamp, videoSampleSink, src, logLevel, }) => {
|
|
101
|
-
const startPacket = await packetSink.getKeyPacket(timestamp, {
|
|
102
|
-
verifyKeyPackets: true,
|
|
103
|
-
});
|
|
104
|
-
if (!startPacket) {
|
|
105
|
-
throw new Error(`No key packet found for timestamp ${timestamp}`);
|
|
106
|
-
}
|
|
107
|
-
const startTimestampInSeconds = startPacket.timestamp;
|
|
108
|
-
const existingBank = sources[src]?.[startTimestampInSeconds];
|
|
109
|
-
// Bank does not yet exist, we need to fetch
|
|
110
|
-
if (!existingBank) {
|
|
111
|
-
const newKeyframeBank = getFramesSinceKeyframe({
|
|
112
|
-
packetSink,
|
|
113
|
-
videoSampleSink,
|
|
114
|
-
startPacket,
|
|
115
|
-
});
|
|
116
|
-
addKeyframeBank({ src, bank: newKeyframeBank, startTimestampInSeconds });
|
|
117
|
-
return newKeyframeBank;
|
|
118
|
-
}
|
|
119
|
-
// Bank exists and still has the frame we want
|
|
120
|
-
if (await (await existingBank).hasTimestampInSecond(timestamp)) {
|
|
121
|
-
return existingBank;
|
|
122
|
-
}
|
|
123
|
-
Log.verbose(logLevel, `[Video] Bank exists but frames have already been evicted!`);
|
|
124
|
-
// Bank exists but frames have already been evicted!
|
|
125
|
-
// First delete it entirely
|
|
126
|
-
await (await existingBank).prepareForDeletion();
|
|
127
|
-
delete sources[src][startTimestampInSeconds];
|
|
128
|
-
// Then refetch
|
|
129
|
-
const replacementKeybank = getFramesSinceKeyframe({
|
|
130
|
-
packetSink,
|
|
131
|
-
videoSampleSink,
|
|
132
|
-
startPacket,
|
|
133
|
-
});
|
|
134
|
-
addKeyframeBank({ src, bank: replacementKeybank, startTimestampInSeconds });
|
|
135
|
-
return replacementKeybank;
|
|
136
|
-
};
|
|
137
|
-
const requestKeyframeBank = async ({ packetSink, timestamp, videoSampleSink, src, logLevel, }) => {
|
|
138
|
-
await ensureToStayUnderMaxCacheSize(logLevel);
|
|
139
|
-
await clearKeyframeBanksBeforeTime({
|
|
140
|
-
timestampInSeconds: timestamp,
|
|
141
|
-
src,
|
|
142
|
-
logLevel,
|
|
143
|
-
});
|
|
144
|
-
const keyframeBank = await getKeyframeBankOrRefetch({
|
|
145
|
-
packetSink,
|
|
146
|
-
timestamp,
|
|
147
|
-
videoSampleSink,
|
|
148
|
-
src,
|
|
149
|
-
logLevel,
|
|
150
|
-
});
|
|
151
|
-
return keyframeBank;
|
|
152
|
-
};
|
|
153
|
-
const clearAll = async () => {
|
|
154
|
-
const srcs = Object.keys(sources);
|
|
155
|
-
for (const src of srcs) {
|
|
156
|
-
const banks = Object.keys(sources[src]);
|
|
157
|
-
for (const startTimeInSeconds of banks) {
|
|
158
|
-
const bank = await sources[src][startTimeInSeconds];
|
|
159
|
-
await bank.prepareForDeletion();
|
|
160
|
-
delete sources[src][startTimeInSeconds];
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
};
|
|
164
|
-
return {
|
|
165
|
-
requestKeyframeBank,
|
|
166
|
-
addKeyframeBank,
|
|
167
|
-
getCacheStats,
|
|
168
|
-
clearAll,
|
|
169
|
-
};
|
|
170
|
-
};
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
import { jsx as _jsx } from "react/jsx-runtime";
|
|
2
|
-
import { useContext, useLayoutEffect, useMemo, useRef, useState, } from 'react';
|
|
3
|
-
import { cancelRender, Internals, useCurrentFrame, useDelayRender, useRemotionEnvironment, } from 'remotion';
|
|
4
|
-
import { extractFrameViaBroadcastChannel } from './extract-frame-via-broadcast-channel';
|
|
5
|
-
export const VideoForRendering = ({ volume: volumeProp, playbackRate, src, muted, loopVolumeCurveBehavior, delayRenderRetries, delayRenderTimeoutInMilliseconds,
|
|
6
|
-
// call when a frame of the video, i.e. frame drawn on canvas
|
|
7
|
-
onVideoFrame, logLevel = window.remotion_logLevel, }) => {
|
|
8
|
-
const absoluteFrame = Internals.useTimelinePosition();
|
|
9
|
-
const videoConfig = Internals.useUnsafeVideoConfig();
|
|
10
|
-
const canvasRef = useRef(null);
|
|
11
|
-
const { registerRenderAsset, unregisterRenderAsset } = useContext(Internals.RenderAssetManager);
|
|
12
|
-
const frame = useCurrentFrame();
|
|
13
|
-
const volumePropsFrame = Internals.useFrameForVolumeProp(loopVolumeCurveBehavior ?? 'repeat');
|
|
14
|
-
const environment = useRemotionEnvironment();
|
|
15
|
-
const [id] = useState(() => `${Math.random()}`.replace('0.', ''));
|
|
16
|
-
if (!videoConfig) {
|
|
17
|
-
throw new Error('No video config found');
|
|
18
|
-
}
|
|
19
|
-
if (!src) {
|
|
20
|
-
throw new TypeError('No `src` was passed to <Video>.');
|
|
21
|
-
}
|
|
22
|
-
const volume = Internals.evaluateVolume({
|
|
23
|
-
volume: volumeProp,
|
|
24
|
-
frame: volumePropsFrame,
|
|
25
|
-
mediaVolume: 1,
|
|
26
|
-
});
|
|
27
|
-
Internals.warnAboutTooHighVolume(volume);
|
|
28
|
-
const shouldRenderAudio = useMemo(() => {
|
|
29
|
-
if (!window.remotion_audioEnabled) {
|
|
30
|
-
return false;
|
|
31
|
-
}
|
|
32
|
-
if (muted) {
|
|
33
|
-
return false;
|
|
34
|
-
}
|
|
35
|
-
if (volume <= 0) {
|
|
36
|
-
return false;
|
|
37
|
-
}
|
|
38
|
-
return true;
|
|
39
|
-
}, [muted, volume]);
|
|
40
|
-
const { fps } = videoConfig;
|
|
41
|
-
const { delayRender, continueRender } = useDelayRender();
|
|
42
|
-
useLayoutEffect(() => {
|
|
43
|
-
if (!canvasRef.current) {
|
|
44
|
-
return;
|
|
45
|
-
}
|
|
46
|
-
const actualFps = playbackRate ? fps / playbackRate : fps;
|
|
47
|
-
const timestamp = frame / actualFps;
|
|
48
|
-
const durationInSeconds = 1 / actualFps;
|
|
49
|
-
const newHandle = delayRender(`Extracting frame number ${frame}`, {
|
|
50
|
-
retries: delayRenderRetries ?? undefined,
|
|
51
|
-
timeoutInMilliseconds: delayRenderTimeoutInMilliseconds ?? undefined,
|
|
52
|
-
});
|
|
53
|
-
extractFrameViaBroadcastChannel({
|
|
54
|
-
src,
|
|
55
|
-
timeInSeconds: timestamp,
|
|
56
|
-
durationInSeconds,
|
|
57
|
-
logLevel: logLevel ?? 'info',
|
|
58
|
-
shouldRenderAudio,
|
|
59
|
-
isClientSideRendering: environment.isClientSideRendering,
|
|
60
|
-
})
|
|
61
|
-
.then(({ frame: imageBitmap, audio }) => {
|
|
62
|
-
if (!imageBitmap) {
|
|
63
|
-
cancelRender(new Error('No video frame found'));
|
|
64
|
-
}
|
|
65
|
-
onVideoFrame?.(imageBitmap);
|
|
66
|
-
canvasRef.current?.getContext('2d')?.drawImage(imageBitmap, 0, 0);
|
|
67
|
-
imageBitmap.close();
|
|
68
|
-
if (audio) {
|
|
69
|
-
registerRenderAsset({
|
|
70
|
-
type: 'inline-audio',
|
|
71
|
-
id,
|
|
72
|
-
audio: Array.from(audio.data),
|
|
73
|
-
sampleRate: audio.sampleRate,
|
|
74
|
-
numberOfChannels: audio.numberOfChannels,
|
|
75
|
-
frame: absoluteFrame,
|
|
76
|
-
timestamp: audio.timestamp,
|
|
77
|
-
duration: (audio.numberOfFrames / audio.sampleRate) * 1000000,
|
|
78
|
-
});
|
|
79
|
-
}
|
|
80
|
-
continueRender(newHandle);
|
|
81
|
-
})
|
|
82
|
-
.catch((error) => {
|
|
83
|
-
cancelRender(error);
|
|
84
|
-
});
|
|
85
|
-
return () => {
|
|
86
|
-
continueRender(newHandle);
|
|
87
|
-
unregisterRenderAsset(id);
|
|
88
|
-
};
|
|
89
|
-
}, [
|
|
90
|
-
absoluteFrame,
|
|
91
|
-
continueRender,
|
|
92
|
-
delayRender,
|
|
93
|
-
delayRenderRetries,
|
|
94
|
-
delayRenderTimeoutInMilliseconds,
|
|
95
|
-
environment.isClientSideRendering,
|
|
96
|
-
fps,
|
|
97
|
-
frame,
|
|
98
|
-
id,
|
|
99
|
-
logLevel,
|
|
100
|
-
onVideoFrame,
|
|
101
|
-
playbackRate,
|
|
102
|
-
registerRenderAsset,
|
|
103
|
-
shouldRenderAudio,
|
|
104
|
-
src,
|
|
105
|
-
unregisterRenderAsset,
|
|
106
|
-
]);
|
|
107
|
-
return (_jsx("canvas", { ref: canvasRef, width: videoConfig.width, height: videoConfig.height }));
|
|
108
|
-
};
|
package/dist/new-video.d.ts
DELETED