@meframe/core 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache/CacheManager.d.ts +5 -3
- package/dist/cache/CacheManager.d.ts.map +1 -1
- package/dist/cache/CacheManager.js +3 -10
- package/dist/cache/CacheManager.js.map +1 -1
- package/dist/cache/l1/AudioL1Cache.d.ts +5 -0
- package/dist/cache/l1/AudioL1Cache.d.ts.map +1 -1
- package/dist/cache/l1/AudioL1Cache.js +31 -3
- package/dist/cache/l1/AudioL1Cache.js.map +1 -1
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +1 -0
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/types.d.ts +1 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/controllers/PlaybackController.d.ts +0 -4
- package/dist/controllers/PlaybackController.d.ts.map +1 -1
- package/dist/controllers/PlaybackController.js +11 -54
- package/dist/controllers/PlaybackController.js.map +1 -1
- package/dist/orchestrator/Orchestrator.d.ts +0 -1
- package/dist/orchestrator/Orchestrator.d.ts.map +1 -1
- package/dist/orchestrator/Orchestrator.js +25 -37
- package/dist/orchestrator/Orchestrator.js.map +1 -1
- package/dist/orchestrator/VideoClipSession.d.ts +1 -0
- package/dist/orchestrator/VideoClipSession.d.ts.map +1 -1
- package/dist/orchestrator/VideoClipSession.js +114 -92
- package/dist/orchestrator/VideoClipSession.js.map +1 -1
- package/dist/stages/compose/GlobalAudioSession.d.ts +20 -25
- package/dist/stages/compose/GlobalAudioSession.d.ts.map +1 -1
- package/dist/stages/compose/GlobalAudioSession.js +252 -67
- package/dist/stages/compose/GlobalAudioSession.js.map +1 -1
- package/dist/stages/compose/OfflineAudioMixer.d.ts.map +1 -1
- package/dist/stages/compose/OfflineAudioMixer.js +26 -8
- package/dist/stages/compose/OfflineAudioMixer.js.map +1 -1
- package/dist/stages/demux/MP4Demuxer.d.ts.map +1 -1
- package/dist/stages/load/ResourceLoader.js +1 -1
- package/dist/stages/load/ResourceLoader.js.map +1 -1
- package/dist/worker/WorkerPool.d.ts.map +1 -1
- package/dist/worker/WorkerPool.js +6 -2
- package/dist/worker/WorkerPool.js.map +1 -1
- package/dist/worker/types.d.ts +1 -1
- package/dist/worker/types.d.ts.map +1 -1
- package/dist/worker/types.js.map +1 -1
- package/dist/worker/worker-event-whitelist.d.ts.map +1 -1
- package/dist/workers/BaseDecoder.js +130 -0
- package/dist/workers/BaseDecoder.js.map +1 -0
- package/dist/workers/MP4Demuxer.js +4 -3
- package/dist/workers/MP4Demuxer.js.map +1 -1
- package/dist/workers/WorkerChannel.js.map +1 -1
- package/dist/workers/stages/compose/video-compose.worker.js +13 -9
- package/dist/workers/stages/compose/video-compose.worker.js.map +1 -1
- package/dist/workers/stages/decode/audio-decode.worker.js +243 -0
- package/dist/workers/stages/decode/audio-decode.worker.js.map +1 -0
- package/dist/workers/stages/decode/video-decode.worker.js +346 -0
- package/dist/workers/stages/decode/video-decode.worker.js.map +1 -0
- package/dist/workers/stages/demux/audio-demux.worker.js +28 -18
- package/dist/workers/stages/demux/audio-demux.worker.js.map +1 -1
- package/dist/workers/stages/demux/video-demux.worker.js +5 -2
- package/dist/workers/stages/demux/video-demux.worker.js.map +1 -1
- package/dist/workers/stages/encode/video-encode.worker.js +325 -0
- package/dist/workers/stages/encode/video-encode.worker.js.map +1 -0
- package/package.json +1 -1
- package/dist/cache/l1/MixedAudioL1Cache.d.ts +0 -13
- package/dist/cache/l1/MixedAudioL1Cache.d.ts.map +0 -1
- package/dist/cache/l1/MixedAudioL1Cache.js +0 -52
- package/dist/cache/l1/MixedAudioL1Cache.js.map +0 -1
- package/dist/workers/stages/decode/decode.worker.js +0 -826
- package/dist/workers/stages/decode/decode.worker.js.map +0 -1
- package/dist/workers/stages/encode/encode.worker.js +0 -547
- package/dist/workers/stages/encode/encode.worker.js.map +0 -1
|
@@ -2,10 +2,16 @@ import { OfflineAudioMixer } from "./OfflineAudioMixer.js";
|
|
|
2
2
|
import { MeframeEvent } from "../../event/events.js";
|
|
3
3
|
import { AudioChunkEncoder } from "../encode/AudioChunkEncoder.js";
|
|
4
4
|
class GlobalAudioSession {
|
|
5
|
-
mixWindowUs = 3e6;
|
|
6
5
|
mixer;
|
|
7
6
|
activeClips = /* @__PURE__ */ new Set();
|
|
7
|
+
streamEndedClips = /* @__PURE__ */ new Set();
|
|
8
8
|
deps;
|
|
9
|
+
audioContext = null;
|
|
10
|
+
audioSources = [];
|
|
11
|
+
audioGainNodes = [];
|
|
12
|
+
volume = 1;
|
|
13
|
+
playbackRate = 1;
|
|
14
|
+
isPlaying = false;
|
|
9
15
|
constructor(deps) {
|
|
10
16
|
this.deps = deps;
|
|
11
17
|
this.mixer = new OfflineAudioMixer(deps.cacheManager, deps.getModel);
|
|
@@ -14,41 +20,6 @@ class GlobalAudioSession {
|
|
|
14
20
|
const { sessionId, audioData, clipStartUs, clipDurationUs } = message;
|
|
15
21
|
this.deps.cacheManager.putClipAudioData(sessionId, audioData, clipStartUs, clipDurationUs);
|
|
16
22
|
}
|
|
17
|
-
async ensureMixedPCM(startUs) {
|
|
18
|
-
const model = this.deps.getModel();
|
|
19
|
-
if (!model) {
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
const windowStartUs = this.alignToWindow(startUs);
|
|
23
|
-
const windowEndUs = windowStartUs + this.mixWindowUs;
|
|
24
|
-
const cached = this.deps.cacheManager.getMixedAudio(windowStartUs, windowEndUs);
|
|
25
|
-
if (cached) {
|
|
26
|
-
return cached;
|
|
27
|
-
}
|
|
28
|
-
try {
|
|
29
|
-
const mixedBuffer = await this.mixer.mix(windowStartUs, windowEndUs);
|
|
30
|
-
this.deps.cacheManager.putMixedAudio(windowStartUs, windowEndUs, mixedBuffer);
|
|
31
|
-
return mixedBuffer;
|
|
32
|
-
} catch (error) {
|
|
33
|
-
console.error("[GlobalAudioSession] Mix failed:", error);
|
|
34
|
-
return null;
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
async prepareAudioForPlayback(startUs) {
|
|
38
|
-
const buffer = await this.ensureMixedPCM(startUs);
|
|
39
|
-
if (!buffer) {
|
|
40
|
-
return null;
|
|
41
|
-
}
|
|
42
|
-
const windowStartUs = this.alignToWindow(startUs);
|
|
43
|
-
const offsetUs = startUs - windowStartUs;
|
|
44
|
-
const offsetSeconds = offsetUs / 1e6;
|
|
45
|
-
const durationSeconds = buffer.duration - offsetSeconds;
|
|
46
|
-
return {
|
|
47
|
-
buffer,
|
|
48
|
-
offsetSeconds,
|
|
49
|
-
durationSeconds
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
23
|
async activateAllAudioClips() {
|
|
53
24
|
const model = this.deps.getModel();
|
|
54
25
|
if (!model) {
|
|
@@ -79,6 +50,7 @@ class GlobalAudioSession {
|
|
|
79
50
|
try {
|
|
80
51
|
const { done, value } = await reader.read();
|
|
81
52
|
if (done) {
|
|
53
|
+
this.streamEndedClips.add(sessionId);
|
|
82
54
|
reader.releaseLock();
|
|
83
55
|
return;
|
|
84
56
|
}
|
|
@@ -96,14 +68,41 @@ class GlobalAudioSession {
|
|
|
96
68
|
};
|
|
97
69
|
pump();
|
|
98
70
|
}
|
|
71
|
+
async startPlayback(timeUs, audioContext) {
|
|
72
|
+
this.audioContext = audioContext;
|
|
73
|
+
this.isPlaying = true;
|
|
74
|
+
this.startAllActiveClips(timeUs);
|
|
75
|
+
}
|
|
76
|
+
stopPlayback() {
|
|
77
|
+
this.isPlaying = false;
|
|
78
|
+
this.stopAllAudioSources();
|
|
79
|
+
}
|
|
80
|
+
updateTime(timeUs) {
|
|
81
|
+
if (!this.isPlaying) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
this.checkAndStartNewClips(timeUs);
|
|
85
|
+
}
|
|
86
|
+
setVolume(volume) {
|
|
87
|
+
this.volume = volume;
|
|
88
|
+
for (const gainNode of this.audioGainNodes) {
|
|
89
|
+
gainNode.gain.value = this.volume;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
setPlaybackRate(rate) {
|
|
93
|
+
this.playbackRate = rate;
|
|
94
|
+
for (const source of this.audioSources) {
|
|
95
|
+
source.playbackRate.value = this.playbackRate;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
99
98
|
reset() {
|
|
99
|
+
this.stopAllAudioSources();
|
|
100
100
|
this.deps.cacheManager.resetAudioCache();
|
|
101
101
|
this.activeClips.clear();
|
|
102
|
+
this.streamEndedClips.clear();
|
|
102
103
|
}
|
|
103
104
|
/**
|
|
104
|
-
* Create export encoded audio stream
|
|
105
|
-
* Mixes all audio clips and encodes to EncodedAudioChunk for export
|
|
106
|
-
* Extracts metadata from first chunk and invokes callback
|
|
105
|
+
* Create export encoded audio stream
|
|
107
106
|
*/
|
|
108
107
|
async createExportEncodedStream(config, onFirstMetadata) {
|
|
109
108
|
const audioDataStream = await this.createExportAudioStream();
|
|
@@ -129,7 +128,6 @@ class GlobalAudioSession {
|
|
|
129
128
|
}
|
|
130
129
|
/**
|
|
131
130
|
* Create export audio stream
|
|
132
|
-
* Mixes all audio clips for the entire composition and outputs AudioData stream
|
|
133
131
|
*/
|
|
134
132
|
async createExportAudioStream() {
|
|
135
133
|
const model = this.deps.getModel();
|
|
@@ -141,7 +139,7 @@ class GlobalAudioSession {
|
|
|
141
139
|
await this.waitForAudioClipsReady();
|
|
142
140
|
return new ReadableStream({
|
|
143
141
|
start: async (controller) => {
|
|
144
|
-
const windowSize =
|
|
142
|
+
const windowSize = 5e6;
|
|
145
143
|
let currentUs = 0;
|
|
146
144
|
while (currentUs < totalDurationUs) {
|
|
147
145
|
const windowEndUs = Math.min(currentUs + windowSize, totalDurationUs);
|
|
@@ -156,33 +154,48 @@ class GlobalAudioSession {
|
|
|
156
154
|
}
|
|
157
155
|
});
|
|
158
156
|
}
|
|
159
|
-
/**
|
|
160
|
-
* Wait for audio clips to be decoded and cached
|
|
161
|
-
*/
|
|
162
157
|
async waitForAudioClipsReady() {
|
|
163
158
|
const model = this.deps.getModel();
|
|
164
159
|
if (!model) return;
|
|
165
160
|
const audioClips = model.tracks.filter((track) => track.kind === "audio").flatMap((track) => track.clips);
|
|
166
|
-
const waitPromises = audioClips.map(
|
|
167
|
-
(clip) => this.waitForClipPCM(clip.id, 1e3)
|
|
168
|
-
// Wait up to 1s per clip
|
|
169
|
-
);
|
|
161
|
+
const waitPromises = audioClips.map((clip) => this.waitForClipPCM(clip.id, 1e4));
|
|
170
162
|
await Promise.allSettled(waitPromises);
|
|
171
163
|
}
|
|
172
|
-
/**
|
|
173
|
-
* Wait for clip PCM data to be available
|
|
174
|
-
*/
|
|
175
164
|
waitForClipPCM(clipId, timeoutMs) {
|
|
176
165
|
return new Promise((resolve) => {
|
|
177
|
-
const checkInterval =
|
|
166
|
+
const checkInterval = 100;
|
|
178
167
|
let elapsed = 0;
|
|
168
|
+
let lastFrameCount = 0;
|
|
169
|
+
let stableCount = 0;
|
|
170
|
+
let streamEndDetected = false;
|
|
179
171
|
const check = () => {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
172
|
+
const pcm = this.deps.cacheManager.getClipPCM(clipId, 0, Number.MAX_SAFE_INTEGER);
|
|
173
|
+
if (pcm && pcm.length > 0) {
|
|
174
|
+
const currentFrameCount = pcm[0]?.length ?? 0;
|
|
175
|
+
if (this.streamEndedClips.has(clipId)) {
|
|
176
|
+
streamEndDetected = true;
|
|
177
|
+
}
|
|
178
|
+
if (streamEndDetected) {
|
|
179
|
+
resolve(true);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
if (currentFrameCount === lastFrameCount) {
|
|
183
|
+
stableCount++;
|
|
184
|
+
if (stableCount >= 5) {
|
|
185
|
+
resolve(true);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
} else {
|
|
189
|
+
stableCount = 0;
|
|
190
|
+
lastFrameCount = currentFrameCount;
|
|
191
|
+
}
|
|
183
192
|
}
|
|
184
193
|
elapsed += checkInterval;
|
|
185
194
|
if (elapsed >= timeoutMs) {
|
|
195
|
+
console.warn("[GlobalAudioSession] Timeout waiting for clip", clipId, {
|
|
196
|
+
frames: lastFrameCount,
|
|
197
|
+
elapsed
|
|
198
|
+
});
|
|
186
199
|
resolve(false);
|
|
187
200
|
return;
|
|
188
201
|
}
|
|
@@ -191,9 +204,179 @@ class GlobalAudioSession {
|
|
|
191
204
|
check();
|
|
192
205
|
});
|
|
193
206
|
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
207
|
+
startAllActiveClips(timeUs) {
|
|
208
|
+
if (!this.audioContext) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const currentClips = this.getActiveAudioClips(timeUs);
|
|
212
|
+
for (const clip of currentClips) {
|
|
213
|
+
this.startClipPlayback(clip, timeUs);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
checkAndStartNewClips(timeUs) {
|
|
217
|
+
if (!this.audioContext) {
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
const currentClips = this.getActiveAudioClips(timeUs);
|
|
221
|
+
const activeClipIds = new Set(
|
|
222
|
+
this.audioSources.map((source) => source._meframeClipId).filter(Boolean)
|
|
223
|
+
);
|
|
224
|
+
for (const clip of currentClips) {
|
|
225
|
+
const clipEndUs = clip.startUs + clip.durationUs;
|
|
226
|
+
if (timeUs >= clipEndUs) {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
if (!activeClipIds.has(clip.id)) {
|
|
230
|
+
this.startClipPlayback(clip, timeUs);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
startClipPlayback(clip, currentTimeUs) {
|
|
235
|
+
if (!this.audioContext) {
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
const clipPCMData = this.deps.cacheManager.getClipPCMWithMetadata(
|
|
239
|
+
clip.id,
|
|
240
|
+
clip.startUs,
|
|
241
|
+
clip.startUs + clip.durationUs
|
|
242
|
+
);
|
|
243
|
+
if (!clipPCMData || clipPCMData.planes.length === 0) {
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
const buffer = this.pcmToAudioBuffer(clipPCMData.planes, clipPCMData.sampleRate);
|
|
247
|
+
const source = this.audioContext.createBufferSource();
|
|
248
|
+
source.buffer = buffer;
|
|
249
|
+
source.playbackRate.value = this.playbackRate;
|
|
250
|
+
source._meframeClipId = clip.id;
|
|
251
|
+
source._playedToUs = clip.startUs + buffer.duration * 1e6;
|
|
252
|
+
const gainNode = this.audioContext.createGain();
|
|
253
|
+
gainNode.gain.value = this.volume;
|
|
254
|
+
source.connect(gainNode);
|
|
255
|
+
gainNode.connect(this.audioContext.destination);
|
|
256
|
+
const offsetUs = Math.max(0, currentTimeUs - clip.startUs);
|
|
257
|
+
const offsetSeconds = offsetUs / 1e6;
|
|
258
|
+
const actualDurationSeconds = buffer.duration - offsetSeconds;
|
|
259
|
+
if (actualDurationSeconds <= 0) {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
source.start(0, offsetSeconds, actualDurationSeconds);
|
|
263
|
+
source.onended = () => {
|
|
264
|
+
const index = this.audioSources.indexOf(source);
|
|
265
|
+
if (index >= 0) {
|
|
266
|
+
this.audioSources.splice(index, 1);
|
|
267
|
+
this.audioGainNodes.splice(index, 1);
|
|
268
|
+
}
|
|
269
|
+
const playedToUs = source._playedToUs;
|
|
270
|
+
const clipEndUs = clip.startUs + clip.durationUs;
|
|
271
|
+
if (playedToUs < clipEndUs && this.isPlaying) {
|
|
272
|
+
setTimeout(() => {
|
|
273
|
+
if (this.isPlaying) {
|
|
274
|
+
this.continueClipPlayback(clip, playedToUs);
|
|
275
|
+
}
|
|
276
|
+
}, 50);
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
this.audioSources.push(source);
|
|
280
|
+
this.audioGainNodes.push(gainNode);
|
|
281
|
+
}
|
|
282
|
+
continueClipPlayback(clip, fromUs) {
|
|
283
|
+
if (!this.audioContext) {
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
const clipPCMData = this.deps.cacheManager.getClipPCMWithMetadata(
|
|
287
|
+
clip.id,
|
|
288
|
+
clip.startUs,
|
|
289
|
+
clip.startUs + clip.durationUs
|
|
290
|
+
);
|
|
291
|
+
if (!clipPCMData || clipPCMData.planes.length === 0) {
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
const buffer = this.pcmToAudioBuffer(clipPCMData.planes, clipPCMData.sampleRate);
|
|
295
|
+
const bufferEndUs = clip.startUs + buffer.duration * 1e6;
|
|
296
|
+
if (bufferEndUs <= fromUs + 1e5) {
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
const source = this.audioContext.createBufferSource();
|
|
300
|
+
source.buffer = buffer;
|
|
301
|
+
source.playbackRate.value = this.playbackRate;
|
|
302
|
+
source._meframeClipId = clip.id;
|
|
303
|
+
source._playedToUs = bufferEndUs;
|
|
304
|
+
const gainNode = this.audioContext.createGain();
|
|
305
|
+
gainNode.gain.value = this.volume;
|
|
306
|
+
source.connect(gainNode);
|
|
307
|
+
gainNode.connect(this.audioContext.destination);
|
|
308
|
+
const offsetUs = Math.max(0, fromUs - clip.startUs);
|
|
309
|
+
const offsetSeconds = offsetUs / 1e6;
|
|
310
|
+
const actualDurationSeconds = buffer.duration - offsetSeconds;
|
|
311
|
+
if (actualDurationSeconds <= 0) {
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
source.start(0, offsetSeconds, actualDurationSeconds);
|
|
315
|
+
source.onended = () => {
|
|
316
|
+
const index = this.audioSources.indexOf(source);
|
|
317
|
+
if (index >= 0) {
|
|
318
|
+
this.audioSources.splice(index, 1);
|
|
319
|
+
this.audioGainNodes.splice(index, 1);
|
|
320
|
+
}
|
|
321
|
+
const playedToUs = source._playedToUs;
|
|
322
|
+
const clipEndUs = clip.startUs + clip.durationUs;
|
|
323
|
+
if (playedToUs < clipEndUs && this.isPlaying) {
|
|
324
|
+
setTimeout(() => {
|
|
325
|
+
if (this.isPlaying) {
|
|
326
|
+
this.continueClipPlayback(clip, playedToUs);
|
|
327
|
+
}
|
|
328
|
+
}, 50);
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
this.audioSources.push(source);
|
|
332
|
+
this.audioGainNodes.push(gainNode);
|
|
333
|
+
}
|
|
334
|
+
stopAllAudioSources() {
|
|
335
|
+
for (const source of this.audioSources) {
|
|
336
|
+
try {
|
|
337
|
+
source.stop();
|
|
338
|
+
source.disconnect();
|
|
339
|
+
} catch (error) {
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
for (const gainNode of this.audioGainNodes) {
|
|
343
|
+
try {
|
|
344
|
+
gainNode.disconnect();
|
|
345
|
+
} catch (error) {
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
this.audioSources = [];
|
|
349
|
+
this.audioGainNodes = [];
|
|
350
|
+
}
|
|
351
|
+
getActiveAudioClips(timeUs) {
|
|
352
|
+
const model = this.deps.getModel();
|
|
353
|
+
if (!model) {
|
|
354
|
+
return [];
|
|
355
|
+
}
|
|
356
|
+
const audioTracks = model.getTracksByKind("audio");
|
|
357
|
+
const clips = [];
|
|
358
|
+
for (const track of audioTracks) {
|
|
359
|
+
const trackClips = model.getClipsAtTime(timeUs, track.id);
|
|
360
|
+
if (trackClips.length > 0) {
|
|
361
|
+
clips.push(...trackClips);
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return clips;
|
|
365
|
+
}
|
|
366
|
+
pcmToAudioBuffer(planes, sampleRate) {
|
|
367
|
+
const numberOfChannels = planes.length;
|
|
368
|
+
const numberOfFrames = planes[0]?.length ?? 0;
|
|
369
|
+
const ctx = new OfflineAudioContext(numberOfChannels, 1, sampleRate);
|
|
370
|
+
const buffer = ctx.createBuffer(numberOfChannels, numberOfFrames, sampleRate);
|
|
371
|
+
for (let channel = 0; channel < numberOfChannels; channel++) {
|
|
372
|
+
const plane = planes[channel];
|
|
373
|
+
if (plane) {
|
|
374
|
+
const channelData = buffer.getChannelData(channel);
|
|
375
|
+
channelData.set(plane);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return buffer;
|
|
379
|
+
}
|
|
197
380
|
audioBufferToAudioData(buffer, timestampUs) {
|
|
198
381
|
const sampleRate = buffer.sampleRate;
|
|
199
382
|
const numberOfChannels = buffer.numberOfChannels;
|
|
@@ -212,9 +395,6 @@ class GlobalAudioSession {
|
|
|
212
395
|
data: this.interleavePlanarData(planes)
|
|
213
396
|
});
|
|
214
397
|
}
|
|
215
|
-
/**
|
|
216
|
-
* Interleave planar audio data for AudioData constructor
|
|
217
|
-
*/
|
|
218
398
|
interleavePlanarData(planes) {
|
|
219
399
|
const numberOfChannels = planes.length;
|
|
220
400
|
const numberOfFrames = planes[0]?.length ?? 0;
|
|
@@ -230,25 +410,33 @@ class GlobalAudioSession {
|
|
|
230
410
|
async setupAudioPipeline(clip) {
|
|
231
411
|
const { id: clipId, resourceId, startUs, durationUs } = clip;
|
|
232
412
|
const audioDemuxWorker = await this.deps.workers.get("audioDemux", clipId, { lazy: true });
|
|
233
|
-
const
|
|
413
|
+
const audioDecodeWorker = await this.deps.workers.get("audioDecode", clipId, { lazy: true });
|
|
234
414
|
const demuxToDecodeChannel = new MessageChannel();
|
|
235
415
|
await audioDemuxWorker.send(
|
|
236
416
|
"connect",
|
|
237
417
|
{ direction: "downstream", port: demuxToDecodeChannel.port1, streamType: "audio", clipId },
|
|
238
418
|
{ transfer: [demuxToDecodeChannel.port1] }
|
|
239
419
|
);
|
|
240
|
-
await
|
|
420
|
+
await audioDecodeWorker.send(
|
|
241
421
|
"connect",
|
|
242
422
|
{
|
|
243
423
|
direction: "upstream",
|
|
244
424
|
port: demuxToDecodeChannel.port2,
|
|
245
425
|
streamType: "audio",
|
|
246
|
-
clipId,
|
|
426
|
+
sessionId: clipId,
|
|
247
427
|
clipStartUs: startUs || 0,
|
|
248
428
|
clipDurationUs: durationUs || 0
|
|
249
429
|
},
|
|
250
430
|
{ transfer: [demuxToDecodeChannel.port2] }
|
|
251
431
|
);
|
|
432
|
+
audioDecodeWorker.receiveStream((stream, metadata) => {
|
|
433
|
+
this.handleAudioStream(stream, {
|
|
434
|
+
sessionId: clipId,
|
|
435
|
+
clipStartUs: startUs || 0,
|
|
436
|
+
clipDurationUs: durationUs || 0,
|
|
437
|
+
...metadata
|
|
438
|
+
});
|
|
439
|
+
});
|
|
252
440
|
const demuxConfig = this.deps.buildWorkerConfigs().audioDemux;
|
|
253
441
|
await audioDemuxWorker.send("configure", {
|
|
254
442
|
initial: true,
|
|
@@ -257,9 +445,6 @@ class GlobalAudioSession {
|
|
|
257
445
|
config: demuxConfig
|
|
258
446
|
});
|
|
259
447
|
}
|
|
260
|
-
alignToWindow(timeUs) {
|
|
261
|
-
return Math.floor(timeUs / this.mixWindowUs) * this.mixWindowUs;
|
|
262
|
-
}
|
|
263
448
|
}
|
|
264
449
|
export {
|
|
265
450
|
GlobalAudioSession
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GlobalAudioSession.js","sources":["../../../src/stages/compose/GlobalAudioSession.ts"],"sourcesContent":["import type { TimeUs } from '../../model/types';\nimport { OfflineAudioMixer } from './OfflineAudioMixer';\nimport type { CompositionModel, Clip } from '../../model';\nimport type { WorkerPool } from '../../worker/WorkerPool';\nimport type { ResourceLoader } from '../load/ResourceLoader';\nimport type { EventBus } from '../../event/EventBus';\nimport type { EventPayloadMap } from '../../event/events';\nimport { MeframeEvent } from '../../event/events';\nimport type { CacheManager } from '../../cache/CacheManager';\nimport { AudioChunkEncoder } from '../encode/AudioChunkEncoder';\n\ninterface AudioDataMessage {\n sessionId: string;\n audioData: AudioData;\n clipStartUs: TimeUs;\n clipDurationUs: TimeUs;\n}\n\nexport interface AudioPlaybackInfo {\n buffer: AudioBuffer;\n offsetSeconds: number;\n durationSeconds: number;\n}\n\ninterface AudioSessionDeps {\n cacheManager: CacheManager;\n workers: WorkerPool;\n resourceLoader: ResourceLoader;\n eventBus: EventBus<EventPayloadMap>;\n getModel: () => CompositionModel | null;\n buildWorkerConfigs: () => any;\n}\n\nexport class GlobalAudioSession {\n private mixWindowUs = 3_000_000;\n private mixer: OfflineAudioMixer;\n private activeClips = new Set<string>();\n private deps: AudioSessionDeps;\n\n constructor(deps: AudioSessionDeps) {\n this.deps = deps;\n this.mixer = new OfflineAudioMixer(deps.cacheManager, deps.getModel);\n }\n\n onAudioData(message: AudioDataMessage): void {\n const { sessionId, audioData, clipStartUs, clipDurationUs } = message;\n this.deps.cacheManager.putClipAudioData(sessionId, audioData, clipStartUs, clipDurationUs);\n }\n\n async ensureMixedPCM(startUs: TimeUs): Promise<AudioBuffer | null> {\n const model = this.deps.getModel();\n if (!model) {\n return null;\n }\n\n const windowStartUs = this.alignToWindow(startUs);\n const windowEndUs = windowStartUs + this.mixWindowUs;\n\n const cached = this.deps.cacheManager.getMixedAudio(windowStartUs, windowEndUs);\n if (cached) {\n return cached;\n }\n\n try {\n const mixedBuffer = await this.mixer.mix(windowStartUs, windowEndUs);\n this.deps.cacheManager.putMixedAudio(windowStartUs, windowEndUs, mixedBuffer);\n return mixedBuffer;\n } catch (error) {\n console.error('[GlobalAudioSession] Mix failed:', error);\n return null;\n }\n }\n\n async prepareAudioForPlayback(startUs: TimeUs): Promise<AudioPlaybackInfo | null> {\n const buffer = await this.ensureMixedPCM(startUs);\n if (!buffer) {\n return null;\n }\n\n const windowStartUs = this.alignToWindow(startUs);\n const offsetUs = startUs - windowStartUs;\n const offsetSeconds = offsetUs / 1_000_000;\n const durationSeconds = buffer.duration - offsetSeconds;\n\n return {\n buffer,\n offsetSeconds,\n durationSeconds,\n };\n }\n\n async activateAllAudioClips(): Promise<void> {\n const model = this.deps.getModel();\n if (!model) {\n return;\n }\n\n const audioTracks = model.tracks.filter((track) => track.kind === 'audio');\n\n for (const track of audioTracks) {\n for (const clip of track.clips) {\n if (!this.activeClips.has(clip.id)) {\n await this.setupAudioPipeline(clip);\n this.activeClips.add(clip.id);\n\n await this.deps.resourceLoader.fetch(clip.resourceId, {\n priority: 'high',\n sessionId: clip.id,\n trackId: track.id,\n });\n\n this.deps.eventBus.emit(MeframeEvent.ClipActivated, { clipId: clip.id });\n }\n }\n }\n }\n\n handleAudioStream(stream: ReadableStream<AudioData>, metadata: Record<string, any>): void {\n const sessionId = metadata.sessionId || 'unknown';\n const clipStartUs = metadata.clipStartUs ?? 0;\n const clipDurationUs = metadata.clipDurationUs ?? 0;\n\n const reader = stream.getReader();\n const pump = async (): Promise<void> => {\n try {\n const { done, value } = await reader.read();\n if (done) {\n reader.releaseLock();\n return;\n }\n\n this.onAudioData({\n sessionId,\n audioData: value,\n clipStartUs,\n clipDurationUs,\n });\n\n await pump();\n } catch (error) {\n console.error('[GlobalAudioSession] Audio stream error:', error);\n reader.releaseLock();\n }\n };\n\n pump();\n }\n\n reset(): void {\n this.deps.cacheManager.resetAudioCache();\n this.activeClips.clear();\n }\n\n /**\n * Create export encoded audio stream with metadata callback\n * Mixes all audio clips and encodes to EncodedAudioChunk for export\n * Extracts metadata from first chunk and invokes callback\n */\n async createExportEncodedStream(\n config?: Partial<AudioEncoderConfig>,\n onFirstMetadata?: (metadata: EncodedAudioChunkMetadata) => void\n ): Promise<ReadableStream<EncodedAudioChunk> | null> {\n const audioDataStream = await this.createExportAudioStream();\n if (!audioDataStream) {\n return null;\n }\n\n const encoder = new AudioChunkEncoder(config);\n await encoder.initialize();\n\n const encodingTransform = encoder.createStream();\n const encodedStream = audioDataStream.pipeThrough(encodingTransform);\n\n let firstMetadataExtracted = false;\n\n // Extract chunk from EncoderChunk and capture first metadata\n return encodedStream.pipeThrough(\n new TransformStream({\n transform(encoderChunk, controller) {\n // Extract metadata from first chunk\n if (!firstMetadataExtracted && onFirstMetadata) {\n onFirstMetadata(encoderChunk.metadata as EncodedAudioChunkMetadata);\n firstMetadataExtracted = true;\n }\n controller.enqueue(encoderChunk.chunk as EncodedAudioChunk);\n },\n })\n );\n }\n\n /**\n * Create export audio stream\n * Mixes all audio clips for the entire composition and outputs AudioData stream\n */\n async createExportAudioStream(): Promise<ReadableStream<AudioData> | null> {\n const model = this.deps.getModel();\n if (!model) {\n return null;\n }\n\n const totalDurationUs = model.durationUs;\n\n // Ensure all audio clips are active and cached\n await this.activateAllAudioClips();\n\n // Wait for PCM data to be available\n await this.waitForAudioClipsReady();\n\n return new ReadableStream<AudioData>({\n start: async (controller) => {\n const windowSize = 3_000_000; // 3 seconds per chunk\n let currentUs = 0;\n\n while (currentUs < totalDurationUs) {\n const windowEndUs = Math.min(currentUs + windowSize, totalDurationUs);\n\n // Mix this window\n const mixedBuffer = await this.mixer.mix(currentUs, windowEndUs);\n\n // Convert AudioBuffer to AudioData\n const audioData = this.audioBufferToAudioData(mixedBuffer, currentUs);\n if (audioData) {\n controller.enqueue(audioData);\n }\n\n currentUs = windowEndUs;\n }\n\n controller.close();\n },\n });\n }\n\n /**\n * Wait for audio clips to be decoded and cached\n */\n private async waitForAudioClipsReady(): Promise<void> {\n const model = this.deps.getModel();\n if (!model) return;\n\n const audioClips = model.tracks\n .filter((track) => track.kind === 'audio')\n .flatMap((track) => track.clips);\n\n // Wait for each clip to have PCM data\n const waitPromises = audioClips.map(\n (clip) => this.waitForClipPCM(clip.id, 1000) // Wait up to 1s per clip\n );\n\n await Promise.allSettled(waitPromises);\n }\n\n /**\n * Wait for clip PCM data to be available\n */\n private waitForClipPCM(clipId: string, timeoutMs: number): Promise<boolean> {\n return new Promise((resolve) => {\n const checkInterval = 50;\n let elapsed = 0;\n\n const check = () => {\n if (this.deps.cacheManager.hasClipPCM(clipId)) {\n resolve(true);\n return;\n }\n\n elapsed += checkInterval;\n if (elapsed >= timeoutMs) {\n resolve(false);\n return;\n }\n\n setTimeout(check, checkInterval);\n };\n\n check();\n });\n }\n\n /**\n * Convert AudioBuffer to AudioData for encoding\n */\n private audioBufferToAudioData(buffer: AudioBuffer, timestampUs: TimeUs): AudioData | null {\n const sampleRate = buffer.sampleRate;\n const numberOfChannels = buffer.numberOfChannels;\n const numberOfFrames = buffer.length;\n\n // Interleave channels into planar format\n const planes: Float32Array[] = [];\n for (let channel = 0; channel < numberOfChannels; channel++) {\n planes.push(buffer.getChannelData(channel));\n }\n\n return new AudioData({\n format: 'f32', // interleaved format\n sampleRate,\n numberOfFrames,\n numberOfChannels,\n timestamp: timestampUs,\n data: this.interleavePlanarData(planes),\n });\n }\n\n /**\n * Interleave planar audio data for AudioData constructor\n */\n private interleavePlanarData(planes: Float32Array[]): ArrayBuffer {\n const numberOfChannels = planes.length;\n const numberOfFrames = planes[0]?.length ?? 0;\n const totalSamples = numberOfChannels * numberOfFrames;\n\n const interleaved = new Float32Array(totalSamples);\n\n for (let frame = 0; frame < numberOfFrames; frame++) {\n for (let channel = 0; channel < numberOfChannels; channel++) {\n interleaved[frame * numberOfChannels + channel] = planes[channel]![frame]!;\n }\n }\n\n return interleaved.buffer;\n }\n\n private async setupAudioPipeline(clip: Clip): Promise<void> {\n const { id: clipId, resourceId, startUs, durationUs } = clip;\n const audioDemuxWorker = await this.deps.workers.get('audioDemux', clipId, { lazy: true });\n const decodeWorker = await this.deps.workers.get('decode');\n\n const demuxToDecodeChannel = new MessageChannel();\n await audioDemuxWorker.send(\n 'connect',\n { direction: 'downstream', port: demuxToDecodeChannel.port1, streamType: 'audio', clipId },\n { transfer: [demuxToDecodeChannel.port1] }\n );\n await decodeWorker.send(\n 'connect',\n {\n direction: 'upstream',\n port: demuxToDecodeChannel.port2,\n streamType: 'audio',\n clipId,\n clipStartUs: startUs || 0,\n clipDurationUs: durationUs || 0,\n },\n { transfer: [demuxToDecodeChannel.port2] }\n );\n\n const demuxConfig = this.deps.buildWorkerConfigs().audioDemux;\n await audioDemuxWorker.send('configure', {\n initial: true,\n resourceId,\n clipId,\n config: demuxConfig,\n });\n }\n\n private alignToWindow(timeUs: TimeUs): TimeUs {\n return Math.floor(timeUs / this.mixWindowUs) * this.mixWindowUs;\n }\n}\n"],"names":[],"mappings":";;;AAiCO,MAAM,mBAAmB;AAAA,EACtB,cAAc;AAAA,EACd;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB;AAAA,EAER,YAAY,MAAwB;AAClC,SAAK,OAAO;AACZ,SAAK,QAAQ,IAAI,kBAAkB,KAAK,cAAc,KAAK,QAAQ;AAAA,EACrE;AAAA,EAEA,YAAY,SAAiC;AAC3C,UAAM,EAAE,WAAW,WAAW,aAAa,mBAAmB;AAC9D,SAAK,KAAK,aAAa,iBAAiB,WAAW,WAAW,aAAa,cAAc;AAAA,EAC3F;AAAA,EAEA,MAAM,eAAe,SAA8C;AACjE,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV,aAAO;AAAA,IACT;AAEA,UAAM,gBAAgB,KAAK,cAAc,OAAO;AAChD,UAAM,cAAc,gBAAgB,KAAK;AAEzC,UAAM,SAAS,KAAK,KAAK,aAAa,cAAc,eAAe,WAAW;AAC9E,QAAI,QAAQ;AACV,aAAO;AAAA,IACT;AAEA,QAAI;AACF,YAAM,cAAc,MAAM,KAAK,MAAM,IAAI,eAAe,WAAW;AACnE,WAAK,KAAK,aAAa,cAAc,eAAe,aAAa,WAAW;AAC5E,aAAO;AAAA,IACT,SAAS,OAAO;AACd,cAAQ,MAAM,oCAAoC,KAAK;AACvD,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,MAAM,wBAAwB,SAAoD;AAChF,UAAM,SAAS,MAAM,KAAK,eAAe,OAAO;AAChD,QAAI,CAAC,QAAQ;AACX,aAAO;AAAA,IACT;AAEA,UAAM,gBAAgB,KAAK,cAAc,OAAO;AAChD,UAAM,WAAW,UAAU;AAC3B,UAAM,gBAAgB,WAAW;AACjC,UAAM,kBAAkB,OAAO,WAAW;AAE1C,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,IAAA;AAAA,EAEJ;AAAA,EAEA,MAAM,wBAAuC;AAC3C,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV;AAAA,IACF;AAEA,UAAM,cAAc,MAAM,OAAO,OAAO,CAAC,UAAU,MAAM,SAAS,OAAO;AAEzE,eAAW,SAAS,aAAa;AAC/B,iBAAW,QAAQ,MAAM,OAAO;AAC9B,YAAI,CAAC,KAAK,YAAY,IAAI,KAAK,EAAE,GAAG;AAClC,gBAAM,KAAK,mBAAmB,IAAI;AAClC,eAAK,YAAY,IAAI,KAAK,EAAE;AAE5B,gBAAM,KAAK,KAAK,eAAe,MAAM,KAAK,YAAY;AAAA,YACpD,UAAU;AAAA,YACV,WAAW,KAAK;AAAA,YAChB,SAAS,MAAM;AAAA,UAAA,CAChB;AAED,eAAK,KAAK,SAAS,KAAK,aAAa,eAAe,EAAE,QAAQ,KAAK,IAAI;AAAA,QACzE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBAAkB,QAAmC,UAAqC;AACxF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,cAAc,SAAS,eAAe;AAC5C,UAAM,iBAAiB,SAAS,kBAAkB;AAElD,UAAM,SAAS,OAAO,UAAA;AACtB,UAAM,OAAO,YAA2B;AACtC,UAAI;AACF,cAAM,EAAE,MAAM,MAAA,IAAU,MAAM,OAAO,KAAA;AACrC,YAAI,MAAM;AACR,iBAAO,YAAA;AACP;AAAA,QACF;AAEA,aAAK,YAAY;AAAA,UACf;AAAA,UACA,WAAW;AAAA,UACX;AAAA,UACA;AAAA,QAAA,CACD;AAED,cAAM,KAAA;AAAA,MACR,SAAS,OAAO;AACd,gBAAQ,MAAM,4CAA4C,KAAK;AAC/D,eAAO,YAAA;AAAA,MACT;AAAA,IACF;AAEA,SAAA;AAAA,EACF;AAAA,EAEA,QAAc;AACZ,SAAK,KAAK,aAAa,gBAAA;AACvB,SAAK,YAAY,MAAA;AAAA,EACnB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,0BACJ,QACA,iBACmD;AACnD,UAAM,kBAAkB,MAAM,KAAK,wBAAA;AACnC,QAAI,CAAC,iBAAiB;AACpB,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,kBAAkB,MAAM;AAC5C,UAAM,QAAQ,WAAA;AAEd,UAAM,oBAAoB,QAAQ,aAAA;AAClC,UAAM,gBAAgB,gBAAgB,YAAY,iBAAiB;AAEnE,QAAI,yBAAyB;AAG7B,WAAO,cAAc;AAAA,MACnB,IAAI,gBAAgB;AAAA,QAClB,UAAU,cAAc,YAAY;AAElC,cAAI,CAAC,0BAA0B,iBAAiB;AAC9C,4BAAgB,aAAa,QAAqC;AAClE,qCAAyB;AAAA,UAC3B;AACA,qBAAW,QAAQ,aAAa,KAA0B;AAAA,QAC5D;AAAA,MAAA,CACD;AAAA,IAAA;AAAA,EAEL;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,0BAAqE;AACzE,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB,MAAM;AAG9B,UAAM,KAAK,sBAAA;AAGX,UAAM,KAAK,uBAAA;AAEX,WAAO,IAAI,eAA0B;AAAA,MACnC,OAAO,OAAO,eAAe;AAC3B,cAAM,aAAa;AACnB,YAAI,YAAY;AAEhB,eAAO,YAAY,iBAAiB;AAClC,gBAAM,cAAc,KAAK,IAAI,YAAY,YAAY,eAAe;AAGpE,gBAAM,cAAc,MAAM,KAAK,MAAM,IAAI,WAAW,WAAW;AAG/D,gBAAM,YAAY,KAAK,uBAAuB,aAAa,SAAS;AACpE,cAAI,WAAW;AACb,uBAAW,QAAQ,SAAS;AAAA,UAC9B;AAEA,sBAAY;AAAA,QACd;AAEA,mBAAW,MAAA;AAAA,MACb;AAAA,IAAA,CACD;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,yBAAwC;AACpD,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,MAAO;AAEZ,UAAM,aAAa,MAAM,OACtB,OAAO,CAAC,UAAU,MAAM,SAAS,OAAO,EACxC,QAAQ,CAAC,UAAU,MAAM,KAAK;AAGjC,UAAM,eAAe,WAAW;AAAA,MAC9B,CAAC,SAAS,KAAK,eAAe,KAAK,IAAI,GAAI;AAAA;AAAA,IAAA;AAG7C,UAAM,QAAQ,WAAW,YAAY;AAAA,EACvC;AAAA;AAAA;AAAA;AAAA,EAKQ,eAAe,QAAgB,WAAqC;AAC1E,WAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,YAAM,gBAAgB;AACtB,UAAI,UAAU;AAEd,YAAM,QAAQ,MAAM;AAClB,YAAI,KAAK,KAAK,aAAa,WAAW,MAAM,GAAG;AAC7C,kBAAQ,IAAI;AACZ;AAAA,QACF;AAEA,mBAAW;AACX,YAAI,WAAW,WAAW;AACxB,kBAAQ,KAAK;AACb;AAAA,QACF;AAEA,mBAAW,OAAO,aAAa;AAAA,MACjC;AAEA,YAAA;AAAA,IACF,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKQ,uBAAuB,QAAqB,aAAuC;AACzF,UAAM,aAAa,OAAO;AAC1B,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO;AAG9B,UAAM,SAAyB,CAAA;AAC/B,aAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,aAAO,KAAK,OAAO,eAAe,OAAO,CAAC;AAAA,IAC5C;AAEA,WAAO,IAAI,UAAU;AAAA,MACnB,QAAQ;AAAA;AAAA,MACR;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,MAAM,KAAK,qBAAqB,MAAM;AAAA,IAAA,CACvC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKQ,qBAAqB,QAAqC;AAChE,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO,CAAC,GAAG,UAAU;AAC5C,UAAM,eAAe,mBAAmB;AAExC,UAAM,cAAc,IAAI,aAAa,YAAY;AAEjD,aAAS,QAAQ,GAAG,QAAQ,gBAAgB,SAAS;AACnD,eAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,oBAAY,QAAQ,mBAAmB,OAAO,IAAI,OAAO,OAAO,EAAG,KAAK;AAAA,MAC1E;AAAA,IACF;AAEA,WAAO,YAAY;AAAA,EACrB;AAAA,EAEA,MAAc,mBAAmB,MAA2B;AAC1D,UAAM,EAAE,IAAI,QAAQ,YAAY,SAAS,eAAe;AACxD,UAAM,mBAAmB,MAAM,KAAK,KAAK,QAAQ,IAAI,cAAc,QAAQ,EAAE,MAAM,KAAA,CAAM;AACzF,UAAM,eAAe,MAAM,KAAK,KAAK,QAAQ,IAAI,QAAQ;AAEzD,UAAM,uBAAuB,IAAI,eAAA;AACjC,UAAM,iBAAiB;AAAA,MACrB;AAAA,MACA,EAAE,WAAW,cAAc,MAAM,qBAAqB,OAAO,YAAY,SAAS,OAAA;AAAA,MAClF,EAAE,UAAU,CAAC,qBAAqB,KAAK,EAAA;AAAA,IAAE;AAE3C,UAAM,aAAa;AAAA,MACjB;AAAA,MACA;AAAA,QACE,WAAW;AAAA,QACX,MAAM,qBAAqB;AAAA,QAC3B,YAAY;AAAA,QACZ;AAAA,QACA,aAAa,WAAW;AAAA,QACxB,gBAAgB,cAAc;AAAA,MAAA;AAAA,MAEhC,EAAE,UAAU,CAAC,qBAAqB,KAAK,EAAA;AAAA,IAAE;AAG3C,UAAM,cAAc,KAAK,KAAK,mBAAA,EAAqB;AACnD,UAAM,iBAAiB,KAAK,aAAa;AAAA,MACvC,SAAS;AAAA,MACT;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AAAA,EACH;AAAA,EAEQ,cAAc,QAAwB;AAC5C,WAAO,KAAK,MAAM,SAAS,KAAK,WAAW,IAAI,KAAK;AAAA,EACtD;AACF;"}
|
|
1
|
+
{"version":3,"file":"GlobalAudioSession.js","sources":["../../../src/stages/compose/GlobalAudioSession.ts"],"sourcesContent":["import type { TimeUs } from '../../model/types';\nimport { OfflineAudioMixer } from './OfflineAudioMixer';\nimport type { CompositionModel, Clip } from '../../model';\nimport type { WorkerPool } from '../../worker/WorkerPool';\nimport type { ResourceLoader } from '../load/ResourceLoader';\nimport type { EventBus } from '../../event/EventBus';\nimport type { EventPayloadMap } from '../../event/events';\nimport { MeframeEvent } from '../../event/events';\nimport type { CacheManager } from '../../cache/CacheManager';\nimport { AudioChunkEncoder } from '../encode/AudioChunkEncoder';\n\ninterface AudioDataMessage {\n sessionId: string;\n audioData: AudioData;\n clipStartUs: TimeUs;\n clipDurationUs: TimeUs;\n}\n\ninterface AudioSessionDeps {\n cacheManager: CacheManager;\n workers: WorkerPool;\n resourceLoader: ResourceLoader;\n eventBus: EventBus<EventPayloadMap>;\n getModel: () => CompositionModel | null;\n buildWorkerConfigs: () => any;\n}\n\nexport class GlobalAudioSession {\n private mixer: OfflineAudioMixer;\n private activeClips = new Set<string>();\n private streamEndedClips = new Set<string>();\n private deps: AudioSessionDeps;\n private audioContext: AudioContext | null = null;\n private audioSources: AudioBufferSourceNode[] = [];\n private audioGainNodes: GainNode[] = [];\n private volume = 1.0;\n private playbackRate = 1.0;\n private isPlaying = false;\n\n constructor(deps: AudioSessionDeps) {\n this.deps = deps;\n this.mixer = new OfflineAudioMixer(deps.cacheManager, deps.getModel);\n }\n\n onAudioData(message: AudioDataMessage): void {\n const { sessionId, audioData, clipStartUs, clipDurationUs } = message;\n this.deps.cacheManager.putClipAudioData(sessionId, audioData, clipStartUs, clipDurationUs);\n }\n\n async activateAllAudioClips(): Promise<void> {\n const model = this.deps.getModel();\n if (!model) {\n return;\n }\n\n const audioTracks = model.tracks.filter((track) => track.kind === 'audio');\n\n for (const track of audioTracks) {\n for (const clip of track.clips) {\n if (!this.activeClips.has(clip.id)) {\n await this.setupAudioPipeline(clip);\n this.activeClips.add(clip.id);\n\n await this.deps.resourceLoader.fetch(clip.resourceId, {\n priority: 'high',\n sessionId: clip.id,\n trackId: track.id,\n });\n\n this.deps.eventBus.emit(MeframeEvent.ClipActivated, { clipId: clip.id });\n }\n }\n }\n }\n\n handleAudioStream(stream: ReadableStream<AudioData>, metadata: Record<string, any>): void {\n const sessionId = metadata.sessionId || 'unknown';\n const clipStartUs = metadata.clipStartUs ?? 0;\n const clipDurationUs = metadata.clipDurationUs ?? 0;\n\n const reader = stream.getReader();\n const pump = async (): Promise<void> => {\n try {\n const { done, value } = await reader.read();\n if (done) {\n this.streamEndedClips.add(sessionId);\n reader.releaseLock();\n return;\n }\n\n this.onAudioData({\n sessionId,\n audioData: value,\n clipStartUs,\n clipDurationUs,\n });\n\n await pump();\n } catch (error) {\n console.error('[GlobalAudioSession] Audio stream error:', error);\n reader.releaseLock();\n }\n };\n\n pump();\n }\n\n async startPlayback(timeUs: TimeUs, audioContext: AudioContext): Promise<void> {\n this.audioContext = audioContext;\n this.isPlaying = true;\n this.startAllActiveClips(timeUs);\n }\n\n stopPlayback(): void {\n this.isPlaying = false;\n this.stopAllAudioSources();\n }\n\n updateTime(timeUs: TimeUs): void {\n if (!this.isPlaying) {\n return;\n }\n this.checkAndStartNewClips(timeUs);\n }\n\n setVolume(volume: number): void {\n this.volume = volume;\n for (const gainNode of this.audioGainNodes) {\n gainNode.gain.value = this.volume;\n }\n }\n\n setPlaybackRate(rate: number): void {\n this.playbackRate = rate;\n for (const source of this.audioSources) {\n source.playbackRate.value = this.playbackRate;\n }\n }\n\n reset(): void {\n this.stopAllAudioSources();\n this.deps.cacheManager.resetAudioCache();\n this.activeClips.clear();\n this.streamEndedClips.clear();\n }\n\n /**\n * Create export encoded audio stream\n */\n async createExportEncodedStream(\n config?: Partial<AudioEncoderConfig>,\n onFirstMetadata?: (metadata: EncodedAudioChunkMetadata) => void\n ): Promise<ReadableStream<EncodedAudioChunk> | null> {\n const audioDataStream = await this.createExportAudioStream();\n if (!audioDataStream) {\n return null;\n }\n\n const encoder = new AudioChunkEncoder(config);\n await encoder.initialize();\n\n const encodingTransform = encoder.createStream();\n const encodedStream = audioDataStream.pipeThrough(encodingTransform);\n\n let firstMetadataExtracted = false;\n\n return encodedStream.pipeThrough(\n new TransformStream({\n transform(encoderChunk, controller) {\n if (!firstMetadataExtracted && onFirstMetadata) {\n onFirstMetadata(encoderChunk.metadata as EncodedAudioChunkMetadata);\n firstMetadataExtracted = true;\n }\n controller.enqueue(encoderChunk.chunk as EncodedAudioChunk);\n },\n })\n );\n }\n\n /**\n * Create export audio stream\n */\n async createExportAudioStream(): Promise<ReadableStream<AudioData> | null> {\n const model = this.deps.getModel();\n if (!model) {\n return null;\n }\n\n const totalDurationUs = model.durationUs;\n\n await this.activateAllAudioClips();\n await this.waitForAudioClipsReady();\n\n return new ReadableStream<AudioData>({\n start: async (controller) => {\n const windowSize = 5_000_000;\n let currentUs = 0;\n\n while (currentUs < totalDurationUs) {\n const windowEndUs = Math.min(currentUs + windowSize, totalDurationUs);\n const mixedBuffer = await this.mixer.mix(currentUs, windowEndUs);\n const audioData = this.audioBufferToAudioData(mixedBuffer, currentUs);\n if (audioData) {\n controller.enqueue(audioData);\n }\n currentUs = windowEndUs;\n }\n\n controller.close();\n },\n });\n }\n\n private async waitForAudioClipsReady(): Promise<void> {\n const model = this.deps.getModel();\n if (!model) return;\n\n const audioClips = model.tracks\n .filter((track) => track.kind === 'audio')\n .flatMap((track) => track.clips);\n\n const waitPromises = audioClips.map((clip) => this.waitForClipPCM(clip.id, 10000)); // 10s timeout\n await Promise.allSettled(waitPromises);\n }\n\n private waitForClipPCM(clipId: string, timeoutMs: number): Promise<boolean> {\n return new Promise((resolve) => {\n const checkInterval = 100;\n let elapsed = 0;\n let lastFrameCount = 0;\n let stableCount = 0;\n let streamEndDetected = false;\n\n const check = () => {\n const pcm = this.deps.cacheManager.getClipPCM(clipId, 0, Number.MAX_SAFE_INTEGER);\n\n if (pcm && pcm.length > 0) {\n const currentFrameCount = pcm[0]?.length ?? 0;\n\n // Check if we have received stream end signal\n if (this.streamEndedClips.has(clipId)) {\n streamEndDetected = true;\n }\n\n // If stream has ended, we're done\n if (streamEndDetected) {\n resolve(true);\n return;\n }\n\n // Otherwise, check if frame count is stable (no new data for 500ms)\n if (currentFrameCount === lastFrameCount) {\n stableCount++;\n if (stableCount >= 5) {\n // 5 * 100ms = 500ms\n resolve(true);\n return;\n }\n } else {\n stableCount = 0;\n lastFrameCount = currentFrameCount;\n }\n }\n\n elapsed += checkInterval;\n if (elapsed >= timeoutMs) {\n console.warn('[GlobalAudioSession] Timeout waiting for clip', clipId, {\n frames: lastFrameCount,\n elapsed,\n });\n resolve(false);\n return;\n }\n\n setTimeout(check, checkInterval);\n };\n\n check();\n });\n }\n\n private startAllActiveClips(timeUs: TimeUs): void {\n if (!this.audioContext) {\n return;\n }\n\n const currentClips = this.getActiveAudioClips(timeUs);\n\n for (const clip of currentClips) {\n this.startClipPlayback(clip, timeUs);\n }\n }\n\n private checkAndStartNewClips(timeUs: TimeUs): void {\n if (!this.audioContext) {\n return;\n }\n\n const currentClips = this.getActiveAudioClips(timeUs);\n const activeClipIds = new Set(\n this.audioSources.map((source) => (source as any)._meframeClipId).filter(Boolean)\n );\n\n for (const clip of currentClips) {\n // Check if clip should be playing at current time\n const clipEndUs = clip.startUs + clip.durationUs;\n if (timeUs >= clipEndUs) {\n // Clip has already ended, skip\n continue;\n }\n\n if (!activeClipIds.has(clip.id)) {\n this.startClipPlayback(clip, timeUs);\n }\n }\n }\n\n private startClipPlayback(clip: Clip, currentTimeUs: TimeUs): void {\n if (!this.audioContext) {\n return;\n }\n\n const clipPCMData = this.deps.cacheManager.getClipPCMWithMetadata(\n clip.id,\n clip.startUs,\n clip.startUs + clip.durationUs\n );\n\n if (!clipPCMData || clipPCMData.planes.length === 0) {\n // No data yet, will retry later via checkAndStartNewClips\n return;\n }\n\n const buffer = this.pcmToAudioBuffer(clipPCMData.planes, clipPCMData.sampleRate);\n\n const source = this.audioContext.createBufferSource();\n source.buffer = buffer;\n source.playbackRate.value = this.playbackRate;\n (source as any)._meframeClipId = clip.id;\n (source as any)._playedToUs = clip.startUs + buffer.duration * 1_000_000; // Track where it played to\n\n const gainNode = this.audioContext.createGain();\n gainNode.gain.value = this.volume;\n\n source.connect(gainNode);\n gainNode.connect(this.audioContext.destination);\n\n const offsetUs = Math.max(0, currentTimeUs - clip.startUs);\n const offsetSeconds = offsetUs / 1_000_000;\n\n // Use actual buffer duration instead of clip.durationUs\n const actualDurationSeconds = buffer.duration - offsetSeconds;\n\n if (actualDurationSeconds <= 0) {\n return;\n }\n\n source.start(0, offsetSeconds, actualDurationSeconds);\n\n source.onended = () => {\n const index = this.audioSources.indexOf(source);\n if (index >= 0) {\n this.audioSources.splice(index, 1);\n this.audioGainNodes.splice(index, 1);\n }\n\n // Check if more data has arrived and continue playing\n const playedToUs = (source as any)._playedToUs;\n const clipEndUs = clip.startUs + clip.durationUs;\n\n if (playedToUs < clipEndUs && this.isPlaying) {\n // There might be more data, try to continue\n setTimeout(() => {\n if (this.isPlaying) {\n this.continueClipPlayback(clip, playedToUs);\n }\n }, 50);\n }\n };\n\n this.audioSources.push(source);\n this.audioGainNodes.push(gainNode);\n }\n\n private continueClipPlayback(clip: Clip, fromUs: TimeUs): void {\n if (!this.audioContext) {\n return;\n }\n\n // Check if there's new data\n const clipPCMData = this.deps.cacheManager.getClipPCMWithMetadata(\n clip.id,\n clip.startUs,\n clip.startUs + clip.durationUs\n );\n\n if (!clipPCMData || clipPCMData.planes.length === 0) {\n return;\n }\n\n const buffer = this.pcmToAudioBuffer(clipPCMData.planes, clipPCMData.sampleRate);\n const bufferEndUs = clip.startUs + buffer.duration * 1_000_000;\n\n // Check if there's new data beyond where we played to\n if (bufferEndUs <= fromUs + 100_000) {\n // 100ms tolerance\n // No significant new data\n return;\n }\n\n // Continue playback from where it left off\n const source = this.audioContext.createBufferSource();\n source.buffer = buffer;\n source.playbackRate.value = this.playbackRate;\n (source as any)._meframeClipId = clip.id;\n (source as any)._playedToUs = bufferEndUs;\n\n const gainNode = this.audioContext.createGain();\n gainNode.gain.value = this.volume;\n\n source.connect(gainNode);\n gainNode.connect(this.audioContext.destination);\n\n const offsetUs = Math.max(0, fromUs - clip.startUs);\n const offsetSeconds = offsetUs / 1_000_000;\n const actualDurationSeconds = buffer.duration - offsetSeconds;\n\n if (actualDurationSeconds <= 0) {\n return;\n }\n\n source.start(0, offsetSeconds, actualDurationSeconds);\n\n source.onended = () => {\n const index = this.audioSources.indexOf(source);\n if (index >= 0) {\n this.audioSources.splice(index, 1);\n this.audioGainNodes.splice(index, 1);\n }\n\n // Check if more data has arrived\n const playedToUs = (source as any)._playedToUs;\n const clipEndUs = clip.startUs + clip.durationUs;\n\n if (playedToUs < clipEndUs && this.isPlaying) {\n setTimeout(() => {\n if (this.isPlaying) {\n this.continueClipPlayback(clip, playedToUs);\n }\n }, 50);\n }\n };\n\n this.audioSources.push(source);\n this.audioGainNodes.push(gainNode);\n }\n\n private stopAllAudioSources(): void {\n for (const source of this.audioSources) {\n try {\n source.stop();\n source.disconnect();\n } catch (error) {\n // Ignore\n }\n }\n\n for (const gainNode of this.audioGainNodes) {\n try {\n gainNode.disconnect();\n } catch (error) {\n // Ignore\n }\n }\n\n this.audioSources = [];\n this.audioGainNodes = [];\n }\n\n private getActiveAudioClips(timeUs: TimeUs): Clip[] {\n const model = this.deps.getModel();\n if (!model) {\n return [];\n }\n\n const audioTracks = model.getTracksByKind('audio');\n const clips: Clip[] = [];\n\n for (const track of audioTracks) {\n const trackClips = model.getClipsAtTime(timeUs, track.id);\n if (trackClips.length > 0) {\n clips.push(...trackClips);\n }\n }\n\n return clips;\n }\n\n private pcmToAudioBuffer(planes: Float32Array[], sampleRate: number): AudioBuffer {\n const numberOfChannels = planes.length;\n const numberOfFrames = planes[0]?.length ?? 0;\n\n const ctx = new OfflineAudioContext(numberOfChannels, 1, sampleRate);\n const buffer = ctx.createBuffer(numberOfChannels, numberOfFrames, sampleRate);\n\n for (let channel = 0; channel < numberOfChannels; channel++) {\n const plane = planes[channel];\n if (plane) {\n const channelData = buffer.getChannelData(channel);\n channelData.set(plane);\n }\n }\n\n return buffer;\n }\n\n private audioBufferToAudioData(buffer: AudioBuffer, timestampUs: TimeUs): AudioData | null {\n const sampleRate = buffer.sampleRate;\n const numberOfChannels = buffer.numberOfChannels;\n const numberOfFrames = buffer.length;\n\n const planes: Float32Array[] = [];\n for (let channel = 0; channel < numberOfChannels; channel++) {\n planes.push(buffer.getChannelData(channel));\n }\n\n return new AudioData({\n format: 'f32', // interleaved format\n sampleRate,\n numberOfFrames,\n numberOfChannels,\n timestamp: timestampUs,\n data: this.interleavePlanarData(planes),\n });\n }\n\n private interleavePlanarData(planes: Float32Array[]): ArrayBuffer {\n const numberOfChannels = planes.length;\n const numberOfFrames = planes[0]?.length ?? 0;\n const totalSamples = numberOfChannels * numberOfFrames;\n\n const interleaved = new Float32Array(totalSamples);\n\n for (let frame = 0; frame < numberOfFrames; frame++) {\n for (let channel = 0; channel < numberOfChannels; channel++) {\n interleaved[frame * numberOfChannels + channel] = planes[channel]![frame]!;\n }\n }\n\n return interleaved.buffer;\n }\n\n private async setupAudioPipeline(clip: Clip): Promise<void> {\n const { id: clipId, resourceId, startUs, durationUs } = clip;\n const audioDemuxWorker = await this.deps.workers.get('audioDemux', clipId, { lazy: true });\n const audioDecodeWorker = await this.deps.workers.get('audioDecode', clipId, { lazy: true });\n\n const demuxToDecodeChannel = new MessageChannel();\n await audioDemuxWorker.send(\n 'connect',\n { direction: 'downstream', port: demuxToDecodeChannel.port1, streamType: 'audio', clipId },\n { transfer: [demuxToDecodeChannel.port1] }\n );\n await audioDecodeWorker.send(\n 'connect',\n {\n direction: 'upstream',\n port: demuxToDecodeChannel.port2,\n streamType: 'audio',\n sessionId: clipId,\n clipStartUs: startUs || 0,\n clipDurationUs: durationUs || 0,\n },\n { transfer: [demuxToDecodeChannel.port2] }\n );\n\n audioDecodeWorker.receiveStream((stream, metadata) => {\n this.handleAudioStream(stream as ReadableStream<AudioData>, {\n sessionId: clipId,\n clipStartUs: startUs || 0,\n clipDurationUs: durationUs || 0,\n ...metadata,\n });\n });\n\n const demuxConfig = this.deps.buildWorkerConfigs().audioDemux;\n await audioDemuxWorker.send('configure', {\n initial: true,\n resourceId,\n clipId,\n config: demuxConfig,\n });\n }\n}\n"],"names":[],"mappings":";;;AA2BO,MAAM,mBAAmB;AAAA,EACtB;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB,uCAAuB,IAAA;AAAA,EACvB;AAAA,EACA,eAAoC;AAAA,EACpC,eAAwC,CAAA;AAAA,EACxC,iBAA6B,CAAA;AAAA,EAC7B,SAAS;AAAA,EACT,eAAe;AAAA,EACf,YAAY;AAAA,EAEpB,YAAY,MAAwB;AAClC,SAAK,OAAO;AACZ,SAAK,QAAQ,IAAI,kBAAkB,KAAK,cAAc,KAAK,QAAQ;AAAA,EACrE;AAAA,EAEA,YAAY,SAAiC;AAC3C,UAAM,EAAE,WAAW,WAAW,aAAa,mBAAmB;AAC9D,SAAK,KAAK,aAAa,iBAAiB,WAAW,WAAW,aAAa,cAAc;AAAA,EAC3F;AAAA,EAEA,MAAM,wBAAuC;AAC3C,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV;AAAA,IACF;AAEA,UAAM,cAAc,MAAM,OAAO,OAAO,CAAC,UAAU,MAAM,SAAS,OAAO;AAEzE,eAAW,SAAS,aAAa;AAC/B,iBAAW,QAAQ,MAAM,OAAO;AAC9B,YAAI,CAAC,KAAK,YAAY,IAAI,KAAK,EAAE,GAAG;AAClC,gBAAM,KAAK,mBAAmB,IAAI;AAClC,eAAK,YAAY,IAAI,KAAK,EAAE;AAE5B,gBAAM,KAAK,KAAK,eAAe,MAAM,KAAK,YAAY;AAAA,YACpD,UAAU;AAAA,YACV,WAAW,KAAK;AAAA,YAChB,SAAS,MAAM;AAAA,UAAA,CAChB;AAED,eAAK,KAAK,SAAS,KAAK,aAAa,eAAe,EAAE,QAAQ,KAAK,IAAI;AAAA,QACzE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBAAkB,QAAmC,UAAqC;AACxF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,cAAc,SAAS,eAAe;AAC5C,UAAM,iBAAiB,SAAS,kBAAkB;AAElD,UAAM,SAAS,OAAO,UAAA;AACtB,UAAM,OAAO,YAA2B;AACtC,UAAI;AACF,cAAM,EAAE,MAAM,MAAA,IAAU,MAAM,OAAO,KAAA;AACrC,YAAI,MAAM;AACR,eAAK,iBAAiB,IAAI,SAAS;AACnC,iBAAO,YAAA;AACP;AAAA,QACF;AAEA,aAAK,YAAY;AAAA,UACf;AAAA,UACA,WAAW;AAAA,UACX;AAAA,UACA;AAAA,QAAA,CACD;AAED,cAAM,KAAA;AAAA,MACR,SAAS,OAAO;AACd,gBAAQ,MAAM,4CAA4C,KAAK;AAC/D,eAAO,YAAA;AAAA,MACT;AAAA,IACF;AAEA,SAAA;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,QAAgB,cAA2C;AAC7E,SAAK,eAAe;AACpB,SAAK,YAAY;AACjB,SAAK,oBAAoB,MAAM;AAAA,EACjC;AAAA,EAEA,eAAqB;AACnB,SAAK,YAAY;AACjB,SAAK,oBAAA;AAAA,EACP;AAAA,EAEA,WAAW,QAAsB;AAC/B,QAAI,CAAC,KAAK,WAAW;AACnB;AAAA,IACF;AACA,SAAK,sBAAsB,MAAM;AAAA,EACnC;AAAA,EAEA,UAAU,QAAsB;AAC9B,SAAK,SAAS;AACd,eAAW,YAAY,KAAK,gBAAgB;AAC1C,eAAS,KAAK,QAAQ,KAAK;AAAA,IAC7B;AAAA,EACF;AAAA,EAEA,gBAAgB,MAAoB;AAClC,SAAK,eAAe;AACpB,eAAW,UAAU,KAAK,cAAc;AACtC,aAAO,aAAa,QAAQ,KAAK;AAAA,IACnC;AAAA,EACF;AAAA,EAEA,QAAc;AACZ,SAAK,oBAAA;AACL,SAAK,KAAK,aAAa,gBAAA;AACvB,SAAK,YAAY,MAAA;AACjB,SAAK,iBAAiB,MAAA;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,0BACJ,QACA,iBACmD;AACnD,UAAM,kBAAkB,MAAM,KAAK,wBAAA;AACnC,QAAI,CAAC,iBAAiB;AACpB,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,kBAAkB,MAAM;AAC5C,UAAM,QAAQ,WAAA;AAEd,UAAM,oBAAoB,QAAQ,aAAA;AAClC,UAAM,gBAAgB,gBAAgB,YAAY,iBAAiB;AAEnE,QAAI,yBAAyB;AAE7B,WAAO,cAAc;AAAA,MACnB,IAAI,gBAAgB;AAAA,QAClB,UAAU,cAAc,YAAY;AAClC,cAAI,CAAC,0BAA0B,iBAAiB;AAC9C,4BAAgB,aAAa,QAAqC;AAClE,qCAAyB;AAAA,UAC3B;AACA,qBAAW,QAAQ,aAAa,KAA0B;AAAA,QAC5D;AAAA,MAAA,CACD;AAAA,IAAA;AAAA,EAEL;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,0BAAqE;AACzE,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB,MAAM;AAE9B,UAAM,KAAK,sBAAA;AACX,UAAM,KAAK,uBAAA;AAEX,WAAO,IAAI,eAA0B;AAAA,MACnC,OAAO,OAAO,eAAe;AAC3B,cAAM,aAAa;AACnB,YAAI,YAAY;AAEhB,eAAO,YAAY,iBAAiB;AAClC,gBAAM,cAAc,KAAK,IAAI,YAAY,YAAY,eAAe;AACpE,gBAAM,cAAc,MAAM,KAAK,MAAM,IAAI,WAAW,WAAW;AAC/D,gBAAM,YAAY,KAAK,uBAAuB,aAAa,SAAS;AACpE,cAAI,WAAW;AACb,uBAAW,QAAQ,SAAS;AAAA,UAC9B;AACA,sBAAY;AAAA,QACd;AAEA,mBAAW,MAAA;AAAA,MACb;AAAA,IAAA,CACD;AAAA,EACH;AAAA,EAEA,MAAc,yBAAwC;AACpD,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,MAAO;AAEZ,UAAM,aAAa,MAAM,OACtB,OAAO,CAAC,UAAU,MAAM,SAAS,OAAO,EACxC,QAAQ,CAAC,UAAU,MAAM,KAAK;AAEjC,UAAM,eAAe,WAAW,IAAI,CAAC,SAAS,KAAK,eAAe,KAAK,IAAI,GAAK,CAAC;AACjF,UAAM,QAAQ,WAAW,YAAY;AAAA,EACvC;AAAA,EAEQ,eAAe,QAAgB,WAAqC;AAC1E,WAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,YAAM,gBAAgB;AACtB,UAAI,UAAU;AACd,UAAI,iBAAiB;AACrB,UAAI,cAAc;AAClB,UAAI,oBAAoB;AAExB,YAAM,QAAQ,MAAM;AAClB,cAAM,MAAM,KAAK,KAAK,aAAa,WAAW,QAAQ,GAAG,OAAO,gBAAgB;AAEhF,YAAI,OAAO,IAAI,SAAS,GAAG;AACzB,gBAAM,oBAAoB,IAAI,CAAC,GAAG,UAAU;AAG5C,cAAI,KAAK,iBAAiB,IAAI,MAAM,GAAG;AACrC,gCAAoB;AAAA,UACtB;AAGA,cAAI,mBAAmB;AACrB,oBAAQ,IAAI;AACZ;AAAA,UACF;AAGA,cAAI,sBAAsB,gBAAgB;AACxC;AACA,gBAAI,eAAe,GAAG;AAEpB,sBAAQ,IAAI;AACZ;AAAA,YACF;AAAA,UACF,OAAO;AACL,0BAAc;AACd,6BAAiB;AAAA,UACnB;AAAA,QACF;AAEA,mBAAW;AACX,YAAI,WAAW,WAAW;AACxB,kBAAQ,KAAK,iDAAiD,QAAQ;AAAA,YACpE,QAAQ;AAAA,YACR;AAAA,UAAA,CACD;AACD,kBAAQ,KAAK;AACb;AAAA,QACF;AAEA,mBAAW,OAAO,aAAa;AAAA,MACjC;AAEA,YAAA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,oBAAoB,QAAsB;AAChD,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,UAAM,eAAe,KAAK,oBAAoB,MAAM;AAEpD,eAAW,QAAQ,cAAc;AAC/B,WAAK,kBAAkB,MAAM,MAAM;AAAA,IACrC;AAAA,EACF;AAAA,EAEQ,sBAAsB,QAAsB;AAClD,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,UAAM,eAAe,KAAK,oBAAoB,MAAM;AACpD,UAAM,gBAAgB,IAAI;AAAA,MACxB,KAAK,aAAa,IAAI,CAAC,WAAY,OAAe,cAAc,EAAE,OAAO,OAAO;AAAA,IAAA;AAGlF,eAAW,QAAQ,cAAc;AAE/B,YAAM,YAAY,KAAK,UAAU,KAAK;AACtC,UAAI,UAAU,WAAW;AAEvB;AAAA,MACF;AAEA,UAAI,CAAC,cAAc,IAAI,KAAK,EAAE,GAAG;AAC/B,aAAK,kBAAkB,MAAM,MAAM;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kBAAkB,MAAY,eAA6B;AACjE,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,UAAM,cAAc,KAAK,KAAK,aAAa;AAAA,MACzC,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK,UAAU,KAAK;AAAA,IAAA;AAGtB,QAAI,CAAC,eAAe,YAAY,OAAO,WAAW,GAAG;AAEnD;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,iBAAiB,YAAY,QAAQ,YAAY,UAAU;AAE/E,UAAM,SAAS,KAAK,aAAa,mBAAA;AACjC,WAAO,SAAS;AAChB,WAAO,aAAa,QAAQ,KAAK;AAChC,WAAe,iBAAiB,KAAK;AACrC,WAAe,cAAc,KAAK,UAAU,OAAO,WAAW;AAE/D,UAAM,WAAW,KAAK,aAAa,WAAA;AACnC,aAAS,KAAK,QAAQ,KAAK;AAE3B,WAAO,QAAQ,QAAQ;AACvB,aAAS,QAAQ,KAAK,aAAa,WAAW;AAE9C,UAAM,WAAW,KAAK,IAAI,GAAG,gBAAgB,KAAK,OAAO;AACzD,UAAM,gBAAgB,WAAW;AAGjC,UAAM,wBAAwB,OAAO,WAAW;AAEhD,QAAI,yBAAyB,GAAG;AAC9B;AAAA,IACF;AAEA,WAAO,MAAM,GAAG,eAAe,qBAAqB;AAEpD,WAAO,UAAU,MAAM;AACrB,YAAM,QAAQ,KAAK,aAAa,QAAQ,MAAM;AAC9C,UAAI,SAAS,GAAG;AACd,aAAK,aAAa,OAAO,OAAO,CAAC;AACjC,aAAK,eAAe,OAAO,OAAO,CAAC;AAAA,MACrC;AAGA,YAAM,aAAc,OAAe;AACnC,YAAM,YAAY,KAAK,UAAU,KAAK;AAEtC,UAAI,aAAa,aAAa,KAAK,WAAW;AAE5C,mBAAW,MAAM;AACf,cAAI,KAAK,WAAW;AAClB,iBAAK,qBAAqB,MAAM,UAAU;AAAA,UAC5C;AAAA,QACF,GAAG,EAAE;AAAA,MACP;AAAA,IACF;AAEA,SAAK,aAAa,KAAK,MAAM;AAC7B,SAAK,eAAe,KAAK,QAAQ;AAAA,EACnC;AAAA,EAEQ,qBAAqB,MAAY,QAAsB;AAC7D,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAGA,UAAM,cAAc,KAAK,KAAK,aAAa;AAAA,MACzC,KAAK;AAAA,MACL,KAAK;AAAA,MACL,KAAK,UAAU,KAAK;AAAA,IAAA;AAGtB,QAAI,CAAC,eAAe,YAAY,OAAO,WAAW,GAAG;AACnD;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,iBAAiB,YAAY,QAAQ,YAAY,UAAU;AAC/E,UAAM,cAAc,KAAK,UAAU,OAAO,WAAW;AAGrD,QAAI,eAAe,SAAS,KAAS;AAGnC;AAAA,IACF;AAGA,UAAM,SAAS,KAAK,aAAa,mBAAA;AACjC,WAAO,SAAS;AAChB,WAAO,aAAa,QAAQ,KAAK;AAChC,WAAe,iBAAiB,KAAK;AACrC,WAAe,cAAc;AAE9B,UAAM,WAAW,KAAK,aAAa,WAAA;AACnC,aAAS,KAAK,QAAQ,KAAK;AAE3B,WAAO,QAAQ,QAAQ;AACvB,aAAS,QAAQ,KAAK,aAAa,WAAW;AAE9C,UAAM,WAAW,KAAK,IAAI,GAAG,SAAS,KAAK,OAAO;AAClD,UAAM,gBAAgB,WAAW;AACjC,UAAM,wBAAwB,OAAO,WAAW;AAEhD,QAAI,yBAAyB,GAAG;AAC9B;AAAA,IACF;AAEA,WAAO,MAAM,GAAG,eAAe,qBAAqB;AAEpD,WAAO,UAAU,MAAM;AACrB,YAAM,QAAQ,KAAK,aAAa,QAAQ,MAAM;AAC9C,UAAI,SAAS,GAAG;AACd,aAAK,aAAa,OAAO,OAAO,CAAC;AACjC,aAAK,eAAe,OAAO,OAAO,CAAC;AAAA,MACrC;AAGA,YAAM,aAAc,OAAe;AACnC,YAAM,YAAY,KAAK,UAAU,KAAK;AAEtC,UAAI,aAAa,aAAa,KAAK,WAAW;AAC5C,mBAAW,MAAM;AACf,cAAI,KAAK,WAAW;AAClB,iBAAK,qBAAqB,MAAM,UAAU;AAAA,UAC5C;AAAA,QACF,GAAG,EAAE;AAAA,MACP;AAAA,IACF;AAEA,SAAK,aAAa,KAAK,MAAM;AAC7B,SAAK,eAAe,KAAK,QAAQ;AAAA,EACnC;AAAA,EAEQ,sBAA4B;AAClC,eAAW,UAAU,KAAK,cAAc;AACtC,UAAI;AACF,eAAO,KAAA;AACP,eAAO,WAAA;AAAA,MACT,SAAS,OAAO;AAAA,MAEhB;AAAA,IACF;AAEA,eAAW,YAAY,KAAK,gBAAgB;AAC1C,UAAI;AACF,iBAAS,WAAA;AAAA,MACX,SAAS,OAAO;AAAA,MAEhB;AAAA,IACF;AAEA,SAAK,eAAe,CAAA;AACpB,SAAK,iBAAiB,CAAA;AAAA,EACxB;AAAA,EAEQ,oBAAoB,QAAwB;AAClD,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV,aAAO,CAAA;AAAA,IACT;AAEA,UAAM,cAAc,MAAM,gBAAgB,OAAO;AACjD,UAAM,QAAgB,CAAA;AAEtB,eAAW,SAAS,aAAa;AAC/B,YAAM,aAAa,MAAM,eAAe,QAAQ,MAAM,EAAE;AACxD,UAAI,WAAW,SAAS,GAAG;AACzB,cAAM,KAAK,GAAG,UAAU;AAAA,MAC1B;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,iBAAiB,QAAwB,YAAiC;AAChF,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO,CAAC,GAAG,UAAU;AAE5C,UAAM,MAAM,IAAI,oBAAoB,kBAAkB,GAAG,UAAU;AACnE,UAAM,SAAS,IAAI,aAAa,kBAAkB,gBAAgB,UAAU;AAE5E,aAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,YAAM,QAAQ,OAAO,OAAO;AAC5B,UAAI,OAAO;AACT,cAAM,cAAc,OAAO,eAAe,OAAO;AACjD,oBAAY,IAAI,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB,QAAqB,aAAuC;AACzF,UAAM,aAAa,OAAO;AAC1B,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO;AAE9B,UAAM,SAAyB,CAAA;AAC/B,aAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,aAAO,KAAK,OAAO,eAAe,OAAO,CAAC;AAAA,IAC5C;AAEA,WAAO,IAAI,UAAU;AAAA,MACnB,QAAQ;AAAA;AAAA,MACR;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,MAAM,KAAK,qBAAqB,MAAM;AAAA,IAAA,CACvC;AAAA,EACH;AAAA,EAEQ,qBAAqB,QAAqC;AAChE,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO,CAAC,GAAG,UAAU;AAC5C,UAAM,eAAe,mBAAmB;AAExC,UAAM,cAAc,IAAI,aAAa,YAAY;AAEjD,aAAS,QAAQ,GAAG,QAAQ,gBAAgB,SAAS;AACnD,eAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,oBAAY,QAAQ,mBAAmB,OAAO,IAAI,OAAO,OAAO,EAAG,KAAK;AAAA,MAC1E;AAAA,IACF;AAEA,WAAO,YAAY;AAAA,EACrB;AAAA,EAEA,MAAc,mBAAmB,MAA2B;AAC1D,UAAM,EAAE,IAAI,QAAQ,YAAY,SAAS,eAAe;AACxD,UAAM,mBAAmB,MAAM,KAAK,KAAK,QAAQ,IAAI,cAAc,QAAQ,EAAE,MAAM,KAAA,CAAM;AACzF,UAAM,oBAAoB,MAAM,KAAK,KAAK,QAAQ,IAAI,eAAe,QAAQ,EAAE,MAAM,KAAA,CAAM;AAE3F,UAAM,uBAAuB,IAAI,eAAA;AACjC,UAAM,iBAAiB;AAAA,MACrB;AAAA,MACA,EAAE,WAAW,cAAc,MAAM,qBAAqB,OAAO,YAAY,SAAS,OAAA;AAAA,MAClF,EAAE,UAAU,CAAC,qBAAqB,KAAK,EAAA;AAAA,IAAE;AAE3C,UAAM,kBAAkB;AAAA,MACtB;AAAA,MACA;AAAA,QACE,WAAW;AAAA,QACX,MAAM,qBAAqB;AAAA,QAC3B,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,aAAa,WAAW;AAAA,QACxB,gBAAgB,cAAc;AAAA,MAAA;AAAA,MAEhC,EAAE,UAAU,CAAC,qBAAqB,KAAK,EAAA;AAAA,IAAE;AAG3C,sBAAkB,cAAc,CAAC,QAAQ,aAAa;AACpD,WAAK,kBAAkB,QAAqC;AAAA,QAC1D,WAAW;AAAA,QACX,aAAa,WAAW;AAAA,QACxB,gBAAgB,cAAc;AAAA,QAC9B,GAAG;AAAA,MAAA,CACJ;AAAA,IACH,CAAC;AAED,UAAM,cAAc,KAAK,KAAK,mBAAA,EAAqB;AACnD,UAAM,iBAAiB,KAAK,aAAa;AAAA,MACvC,SAAS;AAAA,MACT;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AAAA,EACH;AACF;"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OfflineAudioMixer.d.ts","sourceRoot":"","sources":["../../../src/stages/compose/OfflineAudioMixer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAS7D,qBAAa,iBAAiB;IAK1B,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ;IALlB,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,gBAAgB,CAAK;gBAGnB,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,MAAM,gBAAgB,GAAG,IAAI;IAG3C,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"OfflineAudioMixer.d.ts","sourceRoot":"","sources":["../../../src/stages/compose/OfflineAudioMixer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACpD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AAS7D,qBAAa,iBAAiB;IAK1B,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ;IALlB,OAAO,CAAC,UAAU,CAAU;IAC5B,OAAO,CAAC,gBAAgB,CAAK;gBAGnB,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,MAAM,gBAAgB,GAAG,IAAI;IAG3C,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAkE3E,OAAO,CAAC,gBAAgB;CA2BzB"}
|
|
@@ -11,17 +11,35 @@ class OfflineAudioMixer {
|
|
|
11
11
|
const ctx = new OfflineAudioContext(this.numberOfChannels, frameCount, this.sampleRate);
|
|
12
12
|
const clips = this.getClipsInWindow(windowStartUs, windowEndUs);
|
|
13
13
|
for (const clip of clips) {
|
|
14
|
-
const
|
|
15
|
-
|
|
14
|
+
const pcmData = this.cacheManager.getClipPCMWithMetadata(
|
|
15
|
+
clip.clipId,
|
|
16
|
+
clip.startUs,
|
|
17
|
+
clip.startUs + clip.durationUs
|
|
18
|
+
);
|
|
19
|
+
if (!pcmData || pcmData.planes.length === 0) {
|
|
16
20
|
console.warn("[OfflineAudioMixer] Missing PCM for clip", clip.clipId);
|
|
17
21
|
continue;
|
|
18
22
|
}
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
+
const clipIntersectStartUs = Math.max(windowStartUs, clip.startUs);
|
|
24
|
+
const clipIntersectEndUs = Math.min(windowEndUs, clip.startUs + clip.durationUs);
|
|
25
|
+
const offsetInClipUs = clipIntersectStartUs - clip.startUs;
|
|
26
|
+
const offsetFrames = Math.floor(offsetInClipUs / 1e6 * pcmData.sampleRate);
|
|
27
|
+
const intersectDurationUs = clipIntersectEndUs - clipIntersectStartUs;
|
|
28
|
+
const intersectFrames = Math.ceil(intersectDurationUs / 1e6 * pcmData.sampleRate);
|
|
29
|
+
const buffer = ctx.createBuffer(
|
|
30
|
+
pcmData.numberOfChannels,
|
|
31
|
+
intersectFrames,
|
|
32
|
+
pcmData.sampleRate
|
|
33
|
+
// Use actual sample rate, OfflineAudioContext will resample
|
|
34
|
+
);
|
|
35
|
+
for (let channel = 0; channel < pcmData.planes.length; channel++) {
|
|
36
|
+
const plane = pcmData.planes[channel];
|
|
23
37
|
if (plane) {
|
|
24
|
-
buffer.
|
|
38
|
+
const channelData = buffer.getChannelData(channel);
|
|
39
|
+
const copyLength = Math.min(intersectFrames, plane.length - offsetFrames);
|
|
40
|
+
for (let i = 0; i < copyLength; i++) {
|
|
41
|
+
channelData[i] = plane[offsetFrames + i] ?? 0;
|
|
42
|
+
}
|
|
25
43
|
}
|
|
26
44
|
}
|
|
27
45
|
const source = ctx.createBufferSource();
|
|
@@ -30,7 +48,7 @@ class OfflineAudioMixer {
|
|
|
30
48
|
gainNode.gain.value = clip.volume;
|
|
31
49
|
source.connect(gainNode);
|
|
32
50
|
gainNode.connect(ctx.destination);
|
|
33
|
-
const relativeStartUs =
|
|
51
|
+
const relativeStartUs = clipIntersectStartUs - windowStartUs;
|
|
34
52
|
const startTime = relativeStartUs / 1e6;
|
|
35
53
|
source.start(startTime);
|
|
36
54
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OfflineAudioMixer.js","sources":["../../../src/stages/compose/OfflineAudioMixer.ts"],"sourcesContent":["import type { TimeUs } from '../../model/types';\nimport type { CompositionModel } from '../../model';\nimport type { CacheManager } from '../../cache/CacheManager';\n\ninterface MixClipInfo {\n clipId: string;\n startUs: TimeUs;\n durationUs: TimeUs;\n volume: number;\n}\n\nexport class OfflineAudioMixer {\n private sampleRate = 48_000;\n private numberOfChannels = 2;\n\n constructor(\n private cacheManager: CacheManager,\n private getModel: () => CompositionModel | null\n ) {}\n\n async mix(windowStartUs: TimeUs, windowEndUs: TimeUs): Promise<AudioBuffer> {\n const durationUs = windowEndUs - windowStartUs;\n const frameCount = Math.ceil((durationUs / 1_000_000) * this.sampleRate);\n\n const ctx = new OfflineAudioContext(this.numberOfChannels, frameCount, this.sampleRate);\n\n const clips = this.getClipsInWindow(windowStartUs, windowEndUs);\n\n for (const clip of clips) {\n const
|
|
1
|
+
{"version":3,"file":"OfflineAudioMixer.js","sources":["../../../src/stages/compose/OfflineAudioMixer.ts"],"sourcesContent":["import type { TimeUs } from '../../model/types';\nimport type { CompositionModel } from '../../model';\nimport type { CacheManager } from '../../cache/CacheManager';\n\ninterface MixClipInfo {\n clipId: string;\n startUs: TimeUs;\n durationUs: TimeUs;\n volume: number;\n}\n\nexport class OfflineAudioMixer {\n private sampleRate = 48_000;\n private numberOfChannels = 2;\n\n constructor(\n private cacheManager: CacheManager,\n private getModel: () => CompositionModel | null\n ) {}\n\n async mix(windowStartUs: TimeUs, windowEndUs: TimeUs): Promise<AudioBuffer> {\n const durationUs = windowEndUs - windowStartUs;\n const frameCount = Math.ceil((durationUs / 1_000_000) * this.sampleRate);\n\n const ctx = new OfflineAudioContext(this.numberOfChannels, frameCount, this.sampleRate);\n\n const clips = this.getClipsInWindow(windowStartUs, windowEndUs);\n\n for (const clip of clips) {\n // Get clip's PCM data with metadata (including actual sample rate)\n const pcmData = this.cacheManager.getClipPCMWithMetadata(\n clip.clipId,\n clip.startUs,\n clip.startUs + clip.durationUs\n );\n\n if (!pcmData || pcmData.planes.length === 0) {\n console.warn('[OfflineAudioMixer] Missing PCM for clip', clip.clipId);\n continue;\n }\n\n // Extract the portion within the window\n const clipIntersectStartUs = Math.max(windowStartUs, clip.startUs);\n const clipIntersectEndUs = Math.min(windowEndUs, clip.startUs + clip.durationUs);\n\n const offsetInClipUs = clipIntersectStartUs - clip.startUs;\n const offsetFrames = Math.floor((offsetInClipUs / 1_000_000) * pcmData.sampleRate);\n const intersectDurationUs = clipIntersectEndUs - clipIntersectStartUs;\n const intersectFrames = Math.ceil((intersectDurationUs / 1_000_000) * pcmData.sampleRate);\n\n // Create AudioBuffer with correct sample rate for resampling\n const buffer = ctx.createBuffer(\n pcmData.numberOfChannels,\n intersectFrames,\n pcmData.sampleRate // Use actual sample rate, OfflineAudioContext will resample\n );\n\n for (let channel = 0; channel < pcmData.planes.length; channel++) {\n const plane = pcmData.planes[channel];\n if (plane) {\n const channelData = buffer.getChannelData(channel);\n const copyLength = Math.min(intersectFrames, plane.length - offsetFrames);\n for (let i = 0; i < copyLength; i++) {\n channelData[i] = plane[offsetFrames + i] ?? 0;\n }\n }\n }\n\n const source = ctx.createBufferSource();\n source.buffer = buffer;\n\n const gainNode = ctx.createGain();\n gainNode.gain.value = clip.volume;\n\n source.connect(gainNode);\n gainNode.connect(ctx.destination);\n\n const relativeStartUs = clipIntersectStartUs - windowStartUs;\n const startTime = relativeStartUs / 1_000_000;\n source.start(startTime);\n }\n\n const mixedBuffer = await ctx.startRendering();\n return mixedBuffer;\n }\n\n private getClipsInWindow(windowStartUs: TimeUs, windowEndUs: TimeUs): MixClipInfo[] {\n const clips: MixClipInfo[] = [];\n const model = this.getModel();\n if (!model) {\n return clips;\n }\n\n for (const track of model.tracks) {\n if (track.kind !== 'audio') {\n continue;\n }\n\n for (const clip of track.clips) {\n const clipEndUs = clip.startUs + clip.durationUs;\n if (clip.startUs < windowEndUs && clipEndUs > windowStartUs) {\n clips.push({\n clipId: clip.id,\n startUs: clip.startUs,\n durationUs: clip.durationUs,\n volume: 1.0,\n });\n }\n }\n }\n\n return clips;\n }\n}\n"],"names":[],"mappings":"AAWO,MAAM,kBAAkB;AAAA,EAI7B,YACU,cACA,UACR;AAFQ,SAAA,eAAA;AACA,SAAA,WAAA;AAAA,EACP;AAAA,EANK,aAAa;AAAA,EACb,mBAAmB;AAAA,EAO3B,MAAM,IAAI,eAAuB,aAA2C;AAC1E,UAAM,aAAa,cAAc;AACjC,UAAM,aAAa,KAAK,KAAM,aAAa,MAAa,KAAK,UAAU;AAEvE,UAAM,MAAM,IAAI,oBAAoB,KAAK,kBAAkB,YAAY,KAAK,UAAU;AAEtF,UAAM,QAAQ,KAAK,iBAAiB,eAAe,WAAW;AAE9D,eAAW,QAAQ,OAAO;AAExB,YAAM,UAAU,KAAK,aAAa;AAAA,QAChC,KAAK;AAAA,QACL,KAAK;AAAA,QACL,KAAK,UAAU,KAAK;AAAA,MAAA;AAGtB,UAAI,CAAC,WAAW,QAAQ,OAAO,WAAW,GAAG;AAC3C,gBAAQ,KAAK,4CAA4C,KAAK,MAAM;AACpE;AAAA,MACF;AAGA,YAAM,uBAAuB,KAAK,IAAI,eAAe,KAAK,OAAO;AACjE,YAAM,qBAAqB,KAAK,IAAI,aAAa,KAAK,UAAU,KAAK,UAAU;AAE/E,YAAM,iBAAiB,uBAAuB,KAAK;AACnD,YAAM,eAAe,KAAK,MAAO,iBAAiB,MAAa,QAAQ,UAAU;AACjF,YAAM,sBAAsB,qBAAqB;AACjD,YAAM,kBAAkB,KAAK,KAAM,sBAAsB,MAAa,QAAQ,UAAU;AAGxF,YAAM,SAAS,IAAI;AAAA,QACjB,QAAQ;AAAA,QACR;AAAA,QACA,QAAQ;AAAA;AAAA,MAAA;AAGV,eAAS,UAAU,GAAG,UAAU,QAAQ,OAAO,QAAQ,WAAW;AAChE,cAAM,QAAQ,QAAQ,OAAO,OAAO;AACpC,YAAI,OAAO;AACT,gBAAM,cAAc,OAAO,eAAe,OAAO;AACjD,gBAAM,aAAa,KAAK,IAAI,iBAAiB,MAAM,SAAS,YAAY;AACxE,mBAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,wBAAY,CAAC,IAAI,MAAM,eAAe,CAAC,KAAK;AAAA,UAC9C;AAAA,QACF;AAAA,MACF;AAEA,YAAM,SAAS,IAAI,mBAAA;AACnB,aAAO,SAAS;AAEhB,YAAM,WAAW,IAAI,WAAA;AACrB,eAAS,KAAK,QAAQ,KAAK;AAE3B,aAAO,QAAQ,QAAQ;AACvB,eAAS,QAAQ,IAAI,WAAW;AAEhC,YAAM,kBAAkB,uBAAuB;AAC/C,YAAM,YAAY,kBAAkB;AACpC,aAAO,MAAM,SAAS;AAAA,IACxB;AAEA,UAAM,cAAc,MAAM,IAAI,eAAA;AAC9B,WAAO;AAAA,EACT;AAAA,EAEQ,iBAAiB,eAAuB,aAAoC;AAClF,UAAM,QAAuB,CAAA;AAC7B,UAAM,QAAQ,KAAK,SAAA;AACnB,QAAI,CAAC,OAAO;AACV,aAAO;AAAA,IACT;AAEA,eAAW,SAAS,MAAM,QAAQ;AAChC,UAAI,MAAM,SAAS,SAAS;AAC1B;AAAA,MACF;AAEA,iBAAW,QAAQ,MAAM,OAAO;AAC9B,cAAM,YAAY,KAAK,UAAU,KAAK;AACtC,YAAI,KAAK,UAAU,eAAe,YAAY,eAAe;AAC3D,gBAAM,KAAK;AAAA,YACT,QAAQ,KAAK;AAAA,YACb,SAAS,KAAK;AAAA,YACd,YAAY,KAAK;AAAA,YACjB,QAAQ;AAAA,UAAA,CACT;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MP4Demuxer.d.ts","sourceRoot":"","sources":["../../../src/stages/demux/MP4Demuxer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEtD;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAM;IACxB,MAAM,yBAAgC;IACtC,OAAO,UAAS;IAChB,OAAO,CAAC,eAAe,CAAC,CAAsD;IAC9E,OAAO,CAAC,eAAe,CAAC,CAAsD;IAC9E,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,eAAe,CAAC,CAAa;IACrC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,oBAAoB,CAAuB;gBAEvC,MAAM,GAAE,WAAW,GAAG;QAAE,OAAO,CAAC,EAAE,MAAM,IAAI,CAAA;KAAO;IAW/D,YAAY,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAIvC,OAAO,CAAC,aAAa;
|
|
1
|
+
{"version":3,"file":"MP4Demuxer.d.ts","sourceRoot":"","sources":["../../../src/stages/demux/MP4Demuxer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEtD;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAM;IACxB,MAAM,yBAAgC;IACtC,OAAO,UAAS;IAChB,OAAO,CAAC,eAAe,CAAC,CAAsD;IAC9E,OAAO,CAAC,eAAe,CAAC,CAAsD;IAC9E,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,eAAe,CAAC,CAAa;IACrC,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,oBAAoB,CAAuB;gBAEvC,MAAM,GAAE,WAAW,GAAG;QAAE,OAAO,CAAC,EAAE,MAAM,IAAI,CAAA;KAAO;IAW/D,YAAY,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAIvC,OAAO,CAAC,aAAa;IA4BrB,OAAO,CAAC,aAAa;IA8BrB,OAAO,CAAC,cAAc;IAmDtB,OAAO,CAAC,mBAAmB;IA0B3B,OAAO,CAAC,mBAAmB;IAgB3B;;OAEG;IACH,iBAAiB,IAAI,eAAe,CAAC,UAAU,EAAE,iBAAiB,CAAC;IAgCnE;;OAEG;IACH,iBAAiB,IAAI,eAAe,CAAC,UAAU,EAAE,iBAAiB,CAAC,GAAG,IAAI;IA6B1E,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAOrC;;OAEG;IACH,IAAI,cAAc,IAAI,SAAS,GAAG,SAAS,CAE1C;IAED;;OAEG;IACH,IAAI,cAAc,IAAI,SAAS,GAAG,SAAS,CAE1C;IAED,OAAO,IAAI,IAAI;CAQhB"}
|
|
@@ -13,7 +13,7 @@ class ResourceLoader {
|
|
|
13
13
|
onStateChange;
|
|
14
14
|
byteRangeResolver;
|
|
15
15
|
constructor(options) {
|
|
16
|
-
const maxConcurrent = options?.config?.maxConcurrent ??
|
|
16
|
+
const maxConcurrent = options?.config?.maxConcurrent ?? 2;
|
|
17
17
|
this.taskManager = new TaskManager(maxConcurrent);
|
|
18
18
|
this.streamFactory = new StreamFactory(options?.onProgress, options?.config);
|
|
19
19
|
this.eventBus = options?.eventBus;
|