@remotion/media 4.0.352 → 4.0.354
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio/audio-for-rendering.js +37 -27
- package/dist/audio/audio.js +6 -3
- package/dist/audio/props.d.ts +1 -7
- package/dist/audio-extraction/audio-iterator.d.ts +1 -1
- package/dist/audio-extraction/audio-iterator.js +2 -2
- package/dist/audio-extraction/audio-manager.d.ts +1 -1
- package/dist/audio-extraction/extract-audio.d.ts +7 -4
- package/dist/audio-extraction/extract-audio.js +16 -7
- package/dist/caches.d.ts +6 -6
- package/dist/caches.js +5 -6
- package/dist/convert-audiodata/apply-volume.d.ts +1 -0
- package/dist/convert-audiodata/apply-volume.js +17 -0
- package/dist/convert-audiodata/convert-audiodata.d.ts +2 -2
- package/dist/convert-audiodata/convert-audiodata.js +13 -7
- package/dist/convert-audiodata/resample-audiodata.d.ts +1 -2
- package/dist/convert-audiodata/resample-audiodata.js +42 -20
- package/dist/esm/index.mjs +242 -182
- package/dist/extract-frame-and-audio.d.ts +3 -2
- package/dist/extract-frame-and-audio.js +4 -3
- package/dist/looped-frame.d.ts +9 -0
- package/dist/looped-frame.js +10 -0
- package/dist/video/media-player.d.ts +28 -30
- package/dist/video/media-player.js +174 -314
- package/dist/video/new-video-for-preview.d.ts +1 -1
- package/dist/video/new-video-for-preview.js +12 -18
- package/dist/video/props.d.ts +0 -5
- package/dist/video/timeout-utils.d.ts +2 -0
- package/dist/video/timeout-utils.js +18 -0
- package/dist/video/video-for-preview.d.ts +11 -0
- package/dist/video/video-for-preview.js +113 -0
- package/dist/video/video-for-rendering.js +41 -31
- package/dist/video/video.js +2 -2
- package/dist/video-extraction/extract-frame-via-broadcast-channel.d.ts +4 -3
- package/dist/video-extraction/extract-frame-via-broadcast-channel.js +9 -5
- package/dist/video-extraction/extract-frame.d.ts +1 -1
- package/dist/video-extraction/extract-frame.js +3 -0
- package/dist/video-extraction/get-frames-since-keyframe.d.ts +1 -1
- package/dist/video-extraction/get-frames-since-keyframe.js +7 -8
- package/dist/video-extraction/keyframe-bank.d.ts +1 -1
- package/dist/video-extraction/keyframe-bank.js +7 -7
- package/dist/video-extraction/keyframe-manager.d.ts +1 -1
- package/dist/video-extraction/keyframe-manager.js +6 -6
- package/package.json +3 -3
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ALL_FORMATS, AudioBufferSink, CanvasSink, Input, UrlSource, } from 'mediabunny';
|
|
2
|
-
import {
|
|
3
|
-
|
|
2
|
+
import { Internals } from 'remotion';
|
|
3
|
+
import { sleep, withTimeout } from './timeout-utils';
|
|
4
|
+
export const SEEK_THRESHOLD = 0.05;
|
|
4
5
|
export class MediaPlayer {
|
|
5
6
|
constructor({ canvas, src, logLevel, sharedAudioContext, }) {
|
|
6
7
|
this.canvasSink = null;
|
|
@@ -10,37 +11,26 @@ export class MediaPlayer {
|
|
|
10
11
|
this.audioBufferIterator = null;
|
|
11
12
|
this.queuedAudioNodes = new Set();
|
|
12
13
|
this.gainNode = null;
|
|
13
|
-
|
|
14
|
-
this.
|
|
15
|
-
this.mediaTimeOffset = 0;
|
|
14
|
+
// audioDelay = mediaTimestamp + audioSyncAnchor - sharedAudioContext.currentTime
|
|
15
|
+
this.audioSyncAnchor = 0;
|
|
16
16
|
this.playing = false;
|
|
17
17
|
this.animationFrameId = null;
|
|
18
|
-
this.
|
|
18
|
+
this.videoAsyncId = 0;
|
|
19
19
|
this.initialized = false;
|
|
20
20
|
this.totalDuration = 0;
|
|
21
|
-
this.actualFps = null;
|
|
22
21
|
// for remotion buffer state
|
|
23
|
-
this.
|
|
24
|
-
this.
|
|
25
|
-
this.
|
|
26
|
-
this.
|
|
27
|
-
this.
|
|
28
|
-
// A/V sync coordination
|
|
29
|
-
this.canStartAudio = false;
|
|
22
|
+
this.isBuffering = false;
|
|
23
|
+
this.audioBufferHealth = 0;
|
|
24
|
+
this.audioIteratorStarted = false;
|
|
25
|
+
this.HEALTHY_BUFER_THRESHOLD_SECONDS = 1;
|
|
26
|
+
this.input = null;
|
|
30
27
|
this.render = () => {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if (!this.audioSink) {
|
|
37
|
-
this.resetAudioProgressStopwatch();
|
|
38
|
-
}
|
|
39
|
-
this.nextFrame = null;
|
|
40
|
-
this.updateNextFrame();
|
|
28
|
+
if (this.isBuffering) {
|
|
29
|
+
this.maybeForceResumeFromBuffering();
|
|
30
|
+
}
|
|
31
|
+
if (this.shouldRenderFrame()) {
|
|
32
|
+
this.drawCurrentFrame();
|
|
41
33
|
}
|
|
42
|
-
this.updateStalledState();
|
|
43
|
-
// continue render loop only if playing
|
|
44
34
|
if (this.playing) {
|
|
45
35
|
this.animationFrameId = requestAnimationFrame(this.render);
|
|
46
36
|
}
|
|
@@ -48,122 +38,107 @@ export class MediaPlayer {
|
|
|
48
38
|
this.animationFrameId = null;
|
|
49
39
|
}
|
|
50
40
|
};
|
|
41
|
+
this.startAudioIterator = async (timeToSeek) => {
|
|
42
|
+
if (!this.hasAudio())
|
|
43
|
+
return;
|
|
44
|
+
// Clean up existing audio iterator
|
|
45
|
+
await this.audioBufferIterator?.return();
|
|
46
|
+
this.audioIteratorStarted = false;
|
|
47
|
+
this.audioBufferHealth = 0;
|
|
48
|
+
try {
|
|
49
|
+
this.audioBufferIterator = this.audioSink.buffers(timeToSeek);
|
|
50
|
+
this.runAudioIterator();
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start audio iterator', error);
|
|
54
|
+
}
|
|
55
|
+
};
|
|
51
56
|
this.startVideoIterator = async (timeToSeek) => {
|
|
52
57
|
if (!this.canvasSink) {
|
|
53
58
|
return;
|
|
54
59
|
}
|
|
55
|
-
this.
|
|
56
|
-
const currentAsyncId = this.
|
|
60
|
+
this.videoAsyncId++;
|
|
61
|
+
const currentAsyncId = this.videoAsyncId;
|
|
57
62
|
await this.videoFrameIterator?.return();
|
|
58
63
|
this.videoFrameIterator = this.canvasSink.canvases(timeToSeek);
|
|
59
64
|
try {
|
|
60
65
|
const firstFrame = (await this.videoFrameIterator.next()).value ?? null;
|
|
61
66
|
const secondFrame = (await this.videoFrameIterator.next()).value ?? null;
|
|
62
|
-
if (currentAsyncId !== this.
|
|
63
|
-
Log.trace(this.logLevel, `[MediaPlayer] Race condition detected, aborting startVideoIterator for ${timeToSeek.toFixed(3)}s`);
|
|
67
|
+
if (currentAsyncId !== this.videoAsyncId) {
|
|
64
68
|
return;
|
|
65
69
|
}
|
|
66
70
|
if (firstFrame) {
|
|
67
|
-
Log.trace(this.logLevel, `[MediaPlayer] Drew initial frame ${firstFrame.timestamp.toFixed(3)}s`);
|
|
71
|
+
Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Drew initial frame ${firstFrame.timestamp.toFixed(3)}s`);
|
|
68
72
|
this.context.drawImage(firstFrame.canvas, 0, 0);
|
|
69
|
-
// For video-only content, track video progress as audio progress
|
|
70
|
-
if (!this.audioSink) {
|
|
71
|
-
this.resetAudioProgressStopwatch();
|
|
72
|
-
}
|
|
73
|
-
this.canStartAudio = true;
|
|
74
|
-
this.isSeeking = false;
|
|
75
|
-
this.tryStartAudio();
|
|
76
73
|
}
|
|
77
74
|
this.nextFrame = secondFrame ?? null;
|
|
78
75
|
if (secondFrame) {
|
|
79
|
-
Log.trace(this.logLevel, `[MediaPlayer] Buffered next frame ${secondFrame.timestamp.toFixed(3)}s`);
|
|
80
|
-
// For video-only content, track video progress as audio progress
|
|
81
|
-
if (!this.audioSink) {
|
|
82
|
-
this.resetAudioProgressStopwatch();
|
|
83
|
-
}
|
|
84
|
-
if (!this.canStartAudio) {
|
|
85
|
-
this.canStartAudio = true;
|
|
86
|
-
this.tryStartAudio();
|
|
87
|
-
}
|
|
76
|
+
Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Buffered next frame ${secondFrame.timestamp.toFixed(3)}s`);
|
|
88
77
|
}
|
|
89
|
-
this.updateStalledState();
|
|
90
78
|
}
|
|
91
79
|
catch (error) {
|
|
92
|
-
Log.error('[MediaPlayer] Failed to start video iterator', error);
|
|
80
|
+
Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to start video iterator', error);
|
|
93
81
|
}
|
|
94
82
|
};
|
|
95
83
|
this.updateNextFrame = async () => {
|
|
96
84
|
if (!this.videoFrameIterator) {
|
|
97
85
|
return;
|
|
98
86
|
}
|
|
99
|
-
const currentAsyncId = this.asyncId;
|
|
100
87
|
try {
|
|
101
88
|
while (true) {
|
|
102
89
|
const newNextFrame = (await this.videoFrameIterator.next()).value ?? null;
|
|
103
90
|
if (!newNextFrame) {
|
|
104
91
|
break;
|
|
105
92
|
}
|
|
106
|
-
if (currentAsyncId !== this.asyncId) {
|
|
107
|
-
Log.trace(this.logLevel, `[MediaPlayer] Race condition detected in updateNextFrame`);
|
|
108
|
-
break;
|
|
109
|
-
}
|
|
110
93
|
if (newNextFrame.timestamp <= this.getPlaybackTime()) {
|
|
111
|
-
|
|
112
|
-
this.context.drawImage(newNextFrame.canvas, 0, 0);
|
|
113
|
-
// For video-only content, track video progress as audio progress
|
|
114
|
-
if (!this.audioSink) {
|
|
115
|
-
this.resetAudioProgressStopwatch();
|
|
116
|
-
}
|
|
94
|
+
continue;
|
|
117
95
|
}
|
|
118
96
|
else {
|
|
119
97
|
this.nextFrame = newNextFrame;
|
|
120
|
-
Log.trace(this.logLevel, `[MediaPlayer] Buffered next frame ${newNextFrame.timestamp.toFixed(3)}s`);
|
|
121
|
-
// For video-only content, track video progress as audio progress
|
|
122
|
-
if (!this.audioSink) {
|
|
123
|
-
this.resetAudioProgressStopwatch();
|
|
124
|
-
}
|
|
125
|
-
// Open audio gate when new frames become available
|
|
126
|
-
if (!this.canStartAudio) {
|
|
127
|
-
this.canStartAudio = true;
|
|
128
|
-
this.tryStartAudio();
|
|
129
|
-
}
|
|
98
|
+
Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Buffered next frame ${newNextFrame.timestamp.toFixed(3)}s`);
|
|
130
99
|
break;
|
|
131
100
|
}
|
|
132
101
|
}
|
|
133
102
|
}
|
|
134
103
|
catch (error) {
|
|
135
|
-
Log.error('[MediaPlayer] Failed to update next frame', error);
|
|
104
|
+
Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to update next frame', error);
|
|
136
105
|
}
|
|
137
|
-
this.updateStalledState();
|
|
138
106
|
};
|
|
107
|
+
this.bufferingStartedAtMs = null;
|
|
108
|
+
this.minBufferingTimeoutMs = 500;
|
|
139
109
|
this.runAudioIterator = async () => {
|
|
140
|
-
if (!this.
|
|
141
|
-
!this.sharedAudioContext ||
|
|
142
|
-
!this.audioBufferIterator ||
|
|
143
|
-
!this.gainNode) {
|
|
110
|
+
if (!this.hasAudio() || !this.audioBufferIterator)
|
|
144
111
|
return;
|
|
145
|
-
}
|
|
146
112
|
try {
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
113
|
+
let totalBufferDuration = 0;
|
|
114
|
+
let isFirstBuffer = true;
|
|
115
|
+
this.audioIteratorStarted = true;
|
|
116
|
+
while (true) {
|
|
117
|
+
const BUFFERING_TIMEOUT_MS = 50;
|
|
118
|
+
let result;
|
|
119
|
+
try {
|
|
120
|
+
result = await withTimeout(this.audioBufferIterator.next(), BUFFERING_TIMEOUT_MS, 'Iterator timeout');
|
|
154
121
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
122
|
+
catch {
|
|
123
|
+
this.setBufferingState(true);
|
|
124
|
+
await sleep(10);
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (result.done || !result.value) {
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
const { buffer, timestamp, duration } = result.value;
|
|
131
|
+
totalBufferDuration += duration;
|
|
132
|
+
this.audioBufferHealth = Math.max(0, totalBufferDuration);
|
|
133
|
+
this.maybeResumeFromBuffering(totalBufferDuration);
|
|
134
|
+
if (this.playing) {
|
|
135
|
+
if (isFirstBuffer) {
|
|
136
|
+
this.audioSyncAnchor =
|
|
137
|
+
this.sharedAudioContext.currentTime - timestamp;
|
|
138
|
+
isFirstBuffer = false;
|
|
139
|
+
}
|
|
140
|
+
this.scheduleAudioChunk(buffer, timestamp);
|
|
158
141
|
}
|
|
159
|
-
this.queuedAudioNodes.add(node);
|
|
160
|
-
node.onended = () => {
|
|
161
|
-
this.queuedAudioNodes.delete(node);
|
|
162
|
-
};
|
|
163
|
-
this.expectedAudioTime += buffer.duration;
|
|
164
|
-
this.updateStalledState();
|
|
165
|
-
// If we're more than a second ahead of the current playback time, let's slow down the loop until time has
|
|
166
|
-
// passed. Use timestamp for throttling logic as it represents media time.
|
|
167
142
|
if (timestamp - this.getPlaybackTime() >= 1) {
|
|
168
143
|
await new Promise((resolve) => {
|
|
169
144
|
const check = () => {
|
|
@@ -180,13 +155,13 @@ export class MediaPlayer {
|
|
|
180
155
|
}
|
|
181
156
|
}
|
|
182
157
|
catch (error) {
|
|
183
|
-
Log.error('[MediaPlayer] Failed to run audio iterator', error);
|
|
158
|
+
Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to run audio iterator', error);
|
|
184
159
|
}
|
|
185
160
|
};
|
|
186
161
|
this.canvas = canvas;
|
|
187
162
|
this.src = src;
|
|
188
163
|
this.logLevel = logLevel ?? 'info';
|
|
189
|
-
this.sharedAudioContext = sharedAudioContext
|
|
164
|
+
this.sharedAudioContext = sharedAudioContext;
|
|
190
165
|
const context = canvas.getContext('2d', {
|
|
191
166
|
alpha: false,
|
|
192
167
|
desynchronized: true,
|
|
@@ -195,26 +170,24 @@ export class MediaPlayer {
|
|
|
195
170
|
throw new Error('Could not get 2D context from canvas');
|
|
196
171
|
}
|
|
197
172
|
this.context = context;
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
173
|
+
}
|
|
174
|
+
isReady() {
|
|
175
|
+
return this.initialized && Boolean(this.sharedAudioContext);
|
|
176
|
+
}
|
|
177
|
+
hasAudio() {
|
|
178
|
+
return Boolean(this.audioSink && this.sharedAudioContext && this.gainNode);
|
|
179
|
+
}
|
|
180
|
+
isCurrentlyBuffering() {
|
|
181
|
+
return this.isBuffering && Boolean(this.bufferingStartedAtMs);
|
|
201
182
|
}
|
|
202
183
|
async initialize(startTime = 0) {
|
|
203
|
-
if (this.initialized) {
|
|
204
|
-
Log.trace(this.logLevel, `[MediaPlayer] Already initialized, skipping`);
|
|
205
|
-
return;
|
|
206
|
-
}
|
|
207
184
|
try {
|
|
208
|
-
Log.trace(this.logLevel, `[MediaPlayer] Initializing at startTime: ${startTime.toFixed(3)}s...`);
|
|
209
185
|
const urlSource = new UrlSource(this.src);
|
|
210
|
-
urlSource.onread = () => {
|
|
211
|
-
this.lastNetworkActivityAtMs = this.getCurrentTimeMs();
|
|
212
|
-
this.isNetworkActive = true;
|
|
213
|
-
};
|
|
214
186
|
const input = new Input({
|
|
215
187
|
source: urlSource,
|
|
216
188
|
formats: ALL_FORMATS,
|
|
217
189
|
});
|
|
190
|
+
this.input = input;
|
|
218
191
|
this.totalDuration = await input.computeDuration();
|
|
219
192
|
const videoTrack = await input.getPrimaryVideoTrack();
|
|
220
193
|
const audioTrack = await input.getPrimaryAudioTrack();
|
|
@@ -228,274 +201,161 @@ export class MediaPlayer {
|
|
|
228
201
|
});
|
|
229
202
|
this.canvas.width = videoTrack.displayWidth;
|
|
230
203
|
this.canvas.height = videoTrack.displayHeight;
|
|
231
|
-
// Extract actual FPS for stall detection
|
|
232
|
-
const packetStats = await videoTrack.computePacketStats();
|
|
233
|
-
this.actualFps = packetStats.averagePacketRate;
|
|
234
|
-
Log.trace(this.logLevel, `[MediaPlayer] Detected video FPS: ${this.actualFps}`);
|
|
235
204
|
}
|
|
236
205
|
if (audioTrack && this.sharedAudioContext) {
|
|
237
206
|
this.audioSink = new AudioBufferSink(audioTrack);
|
|
238
207
|
this.gainNode = this.sharedAudioContext.createGain();
|
|
239
208
|
this.gainNode.connect(this.sharedAudioContext.destination);
|
|
240
209
|
}
|
|
241
|
-
// For audio-only content, allow audio to start immediately
|
|
242
|
-
if (!videoTrack && audioTrack) {
|
|
243
|
-
this.canStartAudio = true;
|
|
244
|
-
}
|
|
245
|
-
// Initialize timing offset based on actual starting position
|
|
246
210
|
if (this.sharedAudioContext) {
|
|
247
|
-
this.
|
|
248
|
-
Log.trace(this.logLevel, `[MediaPlayer] Set mediaTimeOffset to ${this.mediaTimeOffset.toFixed(3)}s (audioContext: ${this.sharedAudioContext.currentTime.toFixed(3)}s, startTime: ${startTime.toFixed(3)}s)`);
|
|
249
|
-
this.lastAudioProgressAtMs = this.getCurrentTimeMs();
|
|
250
|
-
this.lastNetworkActivityAtMs = this.getCurrentTimeMs();
|
|
211
|
+
this.audioSyncAnchor = this.sharedAudioContext.currentTime - startTime;
|
|
251
212
|
}
|
|
252
213
|
this.initialized = true;
|
|
214
|
+
await this.startAudioIterator(startTime);
|
|
253
215
|
await this.startVideoIterator(startTime);
|
|
254
216
|
this.startRenderLoop();
|
|
255
|
-
Log.trace(this.logLevel, `[MediaPlayer] Initialized successfully with iterators started, duration: ${this.totalDuration}s`);
|
|
256
217
|
}
|
|
257
218
|
catch (error) {
|
|
258
|
-
Log.error('[MediaPlayer] Failed to initialize', error);
|
|
219
|
+
Internals.Log.error({ logLevel: this.logLevel, tag: '@remotion/media' }, '[MediaPlayer] Failed to initialize', error);
|
|
259
220
|
throw error;
|
|
260
221
|
}
|
|
261
222
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
}
|
|
266
|
-
// Ensure mediaTimeOffset is initialized (safety fallback)
|
|
267
|
-
if (this.mediaTimeOffset === 0) {
|
|
268
|
-
this.mediaTimeOffset = this.sharedAudioContext.currentTime - time;
|
|
269
|
-
Log.trace(this.logLevel, `[MediaPlayer] Late-initialized mediaTimeOffset to ${this.mediaTimeOffset.toFixed(3)}s`);
|
|
223
|
+
cleanupAudioQueue() {
|
|
224
|
+
for (const node of this.queuedAudioNodes) {
|
|
225
|
+
node.stop();
|
|
270
226
|
}
|
|
227
|
+
this.queuedAudioNodes.clear();
|
|
228
|
+
}
|
|
229
|
+
async cleanAudioIteratorAndNodes() {
|
|
230
|
+
await this.audioBufferIterator?.return();
|
|
231
|
+
this.audioBufferIterator = null;
|
|
232
|
+
this.audioIteratorStarted = false;
|
|
233
|
+
this.audioBufferHealth = 0;
|
|
234
|
+
this.cleanupAudioQueue();
|
|
235
|
+
}
|
|
236
|
+
async seekTo(time) {
|
|
237
|
+
if (!this.isReady())
|
|
238
|
+
return;
|
|
271
239
|
const newTime = Math.max(0, Math.min(time, this.totalDuration));
|
|
272
240
|
const currentPlaybackTime = this.getPlaybackTime();
|
|
273
241
|
const isSignificantSeek = Math.abs(newTime - currentPlaybackTime) > SEEK_THRESHOLD;
|
|
274
|
-
// Update offset to make audio context time correspond to new media time
|
|
275
|
-
this.mediaTimeOffset = this.sharedAudioContext.currentTime - newTime;
|
|
276
242
|
if (isSignificantSeek) {
|
|
277
|
-
|
|
278
|
-
this.
|
|
279
|
-
|
|
280
|
-
this.updateStalledState();
|
|
281
|
-
// Stop existing audio first
|
|
282
|
-
if (this.playing && this.audioSink) {
|
|
283
|
-
this.audioBufferIterator?.return();
|
|
284
|
-
this.audioBufferIterator = null;
|
|
285
|
-
// Stop current audio nodes
|
|
286
|
-
for (const node of this.queuedAudioNodes) {
|
|
287
|
-
node.stop();
|
|
288
|
-
}
|
|
289
|
-
this.queuedAudioNodes.clear();
|
|
290
|
-
}
|
|
291
|
-
// Start video iterator (which will open audio gate when ready)
|
|
292
|
-
this.startVideoIterator(newTime);
|
|
293
|
-
}
|
|
294
|
-
else {
|
|
295
|
-
Log.trace(this.logLevel, `[MediaPlayer] Minor time update to ${newTime.toFixed(3)}s - using existing iterator`);
|
|
296
|
-
// if paused, trigger a single frame update to show current position
|
|
297
|
-
if (!this.playing) {
|
|
298
|
-
this.renderSingleFrame();
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
async drawInitialFrame(time = 0) {
|
|
303
|
-
if (!this.initialized || !this.canvasSink) {
|
|
304
|
-
Log.trace(this.logLevel, `[MediaPlayer] Cannot draw initial frame - not initialized or no canvas sink`);
|
|
305
|
-
return;
|
|
306
|
-
}
|
|
307
|
-
try {
|
|
308
|
-
Log.trace(this.logLevel, `[MediaPlayer] Drawing initial frame at ${time.toFixed(3)}s`);
|
|
309
|
-
// create temporary iterator just to get the first frame
|
|
310
|
-
const tempIterator = this.canvasSink.canvases(time);
|
|
311
|
-
const firstFrame = (await tempIterator.next()).value;
|
|
312
|
-
if (firstFrame) {
|
|
313
|
-
this.context.drawImage(firstFrame.canvas, 0, 0);
|
|
314
|
-
Log.trace(this.logLevel, `[MediaPlayer] Drew initial frame at timestamp ${firstFrame.timestamp.toFixed(3)}s`);
|
|
315
|
-
}
|
|
316
|
-
else {
|
|
317
|
-
Log.trace(this.logLevel, `[MediaPlayer] No frame available at ${time.toFixed(3)}s`);
|
|
243
|
+
this.audioSyncAnchor = this.sharedAudioContext.currentTime - newTime;
|
|
244
|
+
if (this.audioSink) {
|
|
245
|
+
await this.cleanAudioIteratorAndNodes();
|
|
318
246
|
}
|
|
319
|
-
|
|
320
|
-
await
|
|
247
|
+
await this.startAudioIterator(newTime);
|
|
248
|
+
await this.startVideoIterator(newTime);
|
|
321
249
|
}
|
|
322
|
-
|
|
323
|
-
|
|
250
|
+
if (!this.playing) {
|
|
251
|
+
this.render();
|
|
324
252
|
}
|
|
325
253
|
}
|
|
326
254
|
async play() {
|
|
327
|
-
if (!this.
|
|
255
|
+
if (!this.isReady())
|
|
328
256
|
return;
|
|
329
|
-
}
|
|
330
257
|
if (!this.playing) {
|
|
331
258
|
if (this.sharedAudioContext.state === 'suspended') {
|
|
332
259
|
await this.sharedAudioContext.resume();
|
|
333
260
|
}
|
|
334
261
|
this.playing = true;
|
|
335
|
-
Log.trace(this.logLevel, `[MediaPlayer] Play - starting render loop`);
|
|
336
262
|
this.startRenderLoop();
|
|
337
|
-
// Audio will start automatically when video signals readiness via tryStartAudio()
|
|
338
|
-
this.tryStartAudio();
|
|
339
263
|
}
|
|
340
264
|
}
|
|
341
265
|
pause() {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
this.audioBufferIterator?.return();
|
|
346
|
-
this.audioBufferIterator = null;
|
|
347
|
-
// stop all playing audio nodes
|
|
348
|
-
for (const node of this.queuedAudioNodes) {
|
|
349
|
-
node.stop();
|
|
350
|
-
}
|
|
351
|
-
this.queuedAudioNodes.clear();
|
|
352
|
-
Log.trace(this.logLevel, `[MediaPlayer] Pause - stopping render loop`);
|
|
353
|
-
this.stopRenderLoop();
|
|
354
|
-
}
|
|
266
|
+
this.playing = false;
|
|
267
|
+
this.cleanupAudioQueue();
|
|
268
|
+
this.stopRenderLoop();
|
|
355
269
|
}
|
|
356
270
|
dispose() {
|
|
357
|
-
|
|
271
|
+
this.input?.dispose();
|
|
358
272
|
this.stopRenderLoop();
|
|
359
|
-
// clean up video resources
|
|
360
273
|
this.videoFrameIterator?.return();
|
|
361
|
-
this.
|
|
362
|
-
this.
|
|
363
|
-
this.canvasSink = null;
|
|
364
|
-
// Clean up audio resources
|
|
365
|
-
for (const node of this.queuedAudioNodes) {
|
|
366
|
-
node.stop();
|
|
367
|
-
}
|
|
368
|
-
this.queuedAudioNodes.clear();
|
|
369
|
-
this.audioBufferIterator?.return();
|
|
370
|
-
this.audioBufferIterator = null;
|
|
371
|
-
this.audioSink = null;
|
|
372
|
-
this.gainNode = null;
|
|
373
|
-
this.initialized = false;
|
|
374
|
-
this.asyncId++;
|
|
375
|
-
}
|
|
376
|
-
get currentTime() {
|
|
377
|
-
return this.getPlaybackTime();
|
|
274
|
+
this.cleanAudioIteratorAndNodes();
|
|
275
|
+
this.videoAsyncId++;
|
|
378
276
|
}
|
|
379
|
-
// current position in the media
|
|
380
277
|
getPlaybackTime() {
|
|
381
|
-
|
|
382
|
-
return 0;
|
|
383
|
-
}
|
|
384
|
-
// Audio context is single source of truth
|
|
385
|
-
return this.sharedAudioContext.currentTime - this.mediaTimeOffset;
|
|
386
|
-
}
|
|
387
|
-
get duration() {
|
|
388
|
-
return this.totalDuration;
|
|
389
|
-
}
|
|
390
|
-
get isPlaying() {
|
|
391
|
-
return this.playing;
|
|
278
|
+
return this.sharedAudioContext.currentTime - this.audioSyncAnchor;
|
|
392
279
|
}
|
|
393
|
-
|
|
394
|
-
|
|
280
|
+
scheduleAudioChunk(buffer, mediaTimestamp) {
|
|
281
|
+
const targetTime = mediaTimestamp + this.audioSyncAnchor;
|
|
282
|
+
const delay = targetTime - this.sharedAudioContext.currentTime;
|
|
283
|
+
const node = this.sharedAudioContext.createBufferSource();
|
|
284
|
+
node.buffer = buffer;
|
|
285
|
+
node.connect(this.gainNode);
|
|
286
|
+
if (delay >= 0) {
|
|
287
|
+
node.start(targetTime);
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
node.start(this.sharedAudioContext.currentTime, -delay);
|
|
291
|
+
}
|
|
292
|
+
this.queuedAudioNodes.add(node);
|
|
293
|
+
node.onended = () => this.queuedAudioNodes.delete(node);
|
|
395
294
|
}
|
|
396
|
-
|
|
397
|
-
this.
|
|
295
|
+
onBufferingChange(callback) {
|
|
296
|
+
this.onBufferingChangeCallback = callback;
|
|
398
297
|
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
Log.trace(this.logLevel, `[MediaPlayer] Single frame update at ${this.nextFrame.timestamp.toFixed(3)}s`);
|
|
403
|
-
this.context.drawImage(this.nextFrame.canvas, 0, 0);
|
|
404
|
-
// For video-only content, track video progress as audio progress
|
|
405
|
-
if (!this.audioSink) {
|
|
406
|
-
this.resetAudioProgressStopwatch();
|
|
407
|
-
}
|
|
408
|
-
this.nextFrame = null;
|
|
409
|
-
this.updateNextFrame();
|
|
410
|
-
}
|
|
298
|
+
canRenderVideo() {
|
|
299
|
+
return (this.audioIteratorStarted &&
|
|
300
|
+
this.audioBufferHealth >= this.HEALTHY_BUFER_THRESHOLD_SECONDS);
|
|
411
301
|
}
|
|
412
302
|
startRenderLoop() {
|
|
413
303
|
if (this.animationFrameId !== null) {
|
|
414
304
|
return;
|
|
415
305
|
}
|
|
416
|
-
Log.trace(this.logLevel, `[MediaPlayer] Starting render loop`);
|
|
417
306
|
this.render();
|
|
418
307
|
}
|
|
419
308
|
stopRenderLoop() {
|
|
420
309
|
if (this.animationFrameId !== null) {
|
|
421
310
|
cancelAnimationFrame(this.animationFrameId);
|
|
422
311
|
this.animationFrameId = null;
|
|
423
|
-
Log.trace(this.logLevel, `[MediaPlayer] Stopped render loop`);
|
|
424
312
|
}
|
|
425
313
|
}
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
this.
|
|
431
|
-
this.canStartAudio &&
|
|
432
|
-
!this.audioBufferIterator) {
|
|
433
|
-
this.audioBufferIterator = this.audioSink.buffers(this.getPlaybackTime());
|
|
434
|
-
this.runAudioIterator();
|
|
435
|
-
this.resetAudioProgressStopwatch();
|
|
436
|
-
Log.trace(this.logLevel, '[MediaPlayer] Audio started - A/V sync established');
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
// Unified time reference for stall detection
|
|
440
|
-
getCurrentTimeMs() {
|
|
441
|
-
if (!this.sharedAudioContext) {
|
|
442
|
-
return performance.now();
|
|
443
|
-
}
|
|
444
|
-
return this.sharedAudioContext.currentTime * 1000;
|
|
314
|
+
shouldRenderFrame() {
|
|
315
|
+
return (!this.isBuffering &&
|
|
316
|
+
this.canRenderVideo() &&
|
|
317
|
+
this.nextFrame !== null &&
|
|
318
|
+
this.nextFrame.timestamp <= this.getPlaybackTime());
|
|
445
319
|
}
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
this.
|
|
449
|
-
|
|
450
|
-
getAudioLookaheadSec() {
|
|
451
|
-
if (!this.sharedAudioContext)
|
|
452
|
-
return 0;
|
|
453
|
-
return this.expectedAudioTime - this.sharedAudioContext.currentTime;
|
|
454
|
-
}
|
|
455
|
-
calculateAudioStallThresholdSec() {
|
|
456
|
-
return 0.2; // Need 200ms of audio scheduled ahead
|
|
320
|
+
drawCurrentFrame() {
|
|
321
|
+
this.context.drawImage(this.nextFrame.canvas, 0, 0);
|
|
322
|
+
this.nextFrame = null;
|
|
323
|
+
this.updateNextFrame();
|
|
457
324
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
325
|
+
setBufferingState(isBuffering) {
|
|
326
|
+
if (this.isBuffering !== isBuffering) {
|
|
327
|
+
this.isBuffering = isBuffering;
|
|
328
|
+
if (isBuffering) {
|
|
329
|
+
this.bufferingStartedAtMs = performance.now();
|
|
330
|
+
this.onBufferingChangeCallback?.(true);
|
|
331
|
+
}
|
|
332
|
+
else {
|
|
333
|
+
this.bufferingStartedAtMs = null;
|
|
334
|
+
this.onBufferingChangeCallback?.(false);
|
|
335
|
+
}
|
|
463
336
|
}
|
|
464
|
-
return !this.isNetworkActive && timeSinceNetworkMs >= 500;
|
|
465
337
|
}
|
|
466
|
-
|
|
467
|
-
if (!this.
|
|
468
|
-
return
|
|
469
|
-
const
|
|
470
|
-
const
|
|
471
|
-
const
|
|
472
|
-
const
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
// Use a separate video progress tracker for video-only content
|
|
477
|
-
const timeSinceVideoProgressMs = nowMs - this.lastAudioProgressAtMs; // Reuse for now
|
|
478
|
-
return (!this.nextFrame &&
|
|
479
|
-
timeSinceVideoProgressMs > threshold &&
|
|
480
|
-
this.playing &&
|
|
481
|
-
this.currentTime < this.duration);
|
|
482
|
-
}
|
|
483
|
-
checkIfStalled() {
|
|
484
|
-
// Only check what matters for playback readiness
|
|
485
|
-
if (this.audioSink && this.playing) {
|
|
486
|
-
const audioLookaheadSec = this.getAudioLookaheadSec();
|
|
487
|
-
const isAudioStarved = audioLookaheadSec < this.calculateAudioStallThresholdSec();
|
|
488
|
-
return isAudioStarved && this.isNetworkStalled();
|
|
489
|
-
}
|
|
490
|
-
// Video-only fallback
|
|
491
|
-
if (!this.audioSink) {
|
|
492
|
-
return this.checkVideoStall() && this.isNetworkStalled();
|
|
338
|
+
maybeResumeFromBuffering(currentBufferDuration) {
|
|
339
|
+
if (!this.isCurrentlyBuffering())
|
|
340
|
+
return;
|
|
341
|
+
const now = performance.now();
|
|
342
|
+
const bufferingDuration = now - this.bufferingStartedAtMs;
|
|
343
|
+
const minTimeElapsed = bufferingDuration >= this.minBufferingTimeoutMs;
|
|
344
|
+
const bufferHealthy = currentBufferDuration >= this.HEALTHY_BUFER_THRESHOLD_SECONDS;
|
|
345
|
+
if (minTimeElapsed && bufferHealthy) {
|
|
346
|
+
Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Resuming from buffering after ${bufferingDuration}ms - buffer recovered`);
|
|
347
|
+
this.setBufferingState(false);
|
|
493
348
|
}
|
|
494
|
-
return false; // Remove: return this.isSeeking;
|
|
495
349
|
}
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
350
|
+
maybeForceResumeFromBuffering() {
|
|
351
|
+
if (!this.isCurrentlyBuffering())
|
|
352
|
+
return;
|
|
353
|
+
const now = performance.now();
|
|
354
|
+
const bufferingDuration = now - this.bufferingStartedAtMs;
|
|
355
|
+
const forceTimeout = bufferingDuration > this.minBufferingTimeoutMs * 10;
|
|
356
|
+
if (forceTimeout) {
|
|
357
|
+
Internals.Log.trace({ logLevel: this.logLevel, tag: '@remotion/media' }, `[MediaPlayer] Force resuming from buffering after ${bufferingDuration}ms`);
|
|
358
|
+
this.setBufferingState(false);
|
|
359
|
+
}
|
|
500
360
|
}
|
|
501
361
|
}
|