@meframe/core 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/Meframe.d.ts.map +1 -1
  2. package/dist/Meframe.js +1 -0
  3. package/dist/Meframe.js.map +1 -1
  4. package/dist/controllers/PlaybackController.d.ts.map +1 -1
  5. package/dist/controllers/PlaybackController.js +1 -1
  6. package/dist/controllers/PlaybackController.js.map +1 -1
  7. package/dist/orchestrator/AudioExportSession.d.ts +28 -0
  8. package/dist/orchestrator/AudioExportSession.d.ts.map +1 -0
  9. package/dist/orchestrator/AudioExportSession.js +95 -0
  10. package/dist/orchestrator/AudioExportSession.js.map +1 -0
  11. package/dist/orchestrator/AudioPreviewSession.d.ts +61 -0
  12. package/dist/orchestrator/AudioPreviewSession.d.ts.map +1 -0
  13. package/dist/orchestrator/AudioPreviewSession.js +340 -0
  14. package/dist/orchestrator/AudioPreviewSession.js.map +1 -0
  15. package/dist/orchestrator/AudioWindowPreparer.d.ts +62 -0
  16. package/dist/orchestrator/AudioWindowPreparer.d.ts.map +1 -0
  17. package/dist/orchestrator/AudioWindowPreparer.js +259 -0
  18. package/dist/orchestrator/AudioWindowPreparer.js.map +1 -0
  19. package/dist/orchestrator/ExportScheduler.d.ts +2 -2
  20. package/dist/orchestrator/ExportScheduler.d.ts.map +1 -1
  21. package/dist/orchestrator/ExportScheduler.js.map +1 -1
  22. package/dist/orchestrator/Orchestrator.d.ts +8 -2
  23. package/dist/orchestrator/Orchestrator.d.ts.map +1 -1
  24. package/dist/orchestrator/Orchestrator.js +22 -16
  25. package/dist/orchestrator/Orchestrator.js.map +1 -1
  26. package/dist/stages/mux/MP4Muxer.js.map +1 -1
  27. package/dist/stages/mux/MuxManager.d.ts +1 -4
  28. package/dist/stages/mux/MuxManager.d.ts.map +1 -1
  29. package/dist/stages/mux/MuxManager.js +1 -1
  30. package/dist/stages/mux/MuxManager.js.map +1 -1
  31. package/package.json +1 -1
  32. package/dist/orchestrator/GlobalAudioSession.d.ts +0 -139
  33. package/dist/orchestrator/GlobalAudioSession.d.ts.map +0 -1
  34. package/dist/orchestrator/GlobalAudioSession.js +0 -683
  35. package/dist/orchestrator/GlobalAudioSession.js.map +0 -1
@@ -1,683 +0,0 @@
1
- import { OfflineAudioMixer } from "../stages/compose/OfflineAudioMixer.js";
2
- import { MeframeEvent } from "../event/events.js";
3
- import { AudioMixBlockCache } from "../cache/AudioMixBlockCache.js";
4
- import { AudioChunkEncoder } from "../stages/encode/AudioChunkEncoder.js";
5
- import { AudioChunkDecoder } from "../stages/decode/AudioChunkDecoder.js";
6
- import { hasResourceId, isAudioClip } from "../model/types.js";
7
- class GlobalAudioSession {
8
- mixer;
9
- activeClips = /* @__PURE__ */ new Set();
10
- deps;
11
- model = null;
12
- audioContext = null;
13
- volume = 1;
14
- playbackRate = 1;
15
- isPlaying = false;
16
- // Preview strategy (unified):
17
- // - Always schedule audio in fixed 60s "mix blocks"
18
- // - Cache 2~3 mixed AudioBuffer blocks (LRU) to accelerate seek
19
- // - Schedule ahead using AudioContext clock to avoid underrun
20
- PREVIEW_BLOCK_DURATION_US = 60 * 1e6;
21
- // 60s
22
- PREVIEW_BLOCK_CACHE_SIZE = 3;
23
- PREVIEW_SCHEDULE_AHEAD_SEC = 12;
24
- // keep enough scheduled audio to hide mixing latency
25
- PREVIEW_BUFFER_GUARD_US = 2e6;
26
- // if next block isn't ready near boundary -> buffering
27
- PREVIEW_BLOCK_FADE_SEC = 0.01;
28
- // 10ms fade-in/out to avoid click at boundaries
29
- previewBlockCache = new AudioMixBlockCache(this.PREVIEW_BLOCK_CACHE_SIZE);
30
- previewScheduleTask = null;
31
- previewScheduleToken = 0;
32
- previewMixToken = 0;
33
- previewNextBlockIndex = 0;
34
- previewNextScheduleTime = 0;
35
- // AudioContext time
36
- previewFirstBlockOffsetUs = 0;
37
- // seek offset within the first block
38
- previewLastTimelineUs = 0;
39
- previewLastStallWarnAt = 0;
40
- // AudioContext time (sec)
41
- previewScheduledSources = /* @__PURE__ */ new Set();
42
- previewMixQueue = Promise.resolve();
43
- constructor(deps) {
44
- this.deps = deps;
45
- this.mixer = new OfflineAudioMixer(deps.cacheManager, () => this.model);
46
- }
47
- enqueuePreviewMix(work, token) {
48
- const run = () => work();
49
- const next = this.previewMixQueue.then(
50
- () => {
51
- if (this.previewMixToken !== token) {
52
- return null;
53
- }
54
- return run();
55
- },
56
- () => {
57
- if (this.previewMixToken !== token) {
58
- return null;
59
- }
60
- return run();
61
- }
62
- );
63
- this.previewMixQueue = next.then(
64
- () => void 0,
65
- () => void 0
66
- );
67
- return next;
68
- }
69
- setModel(model) {
70
- this.model = model;
71
- }
72
- onAudioData(message) {
73
- const { sessionId, audioData, clipStartUs, clipDurationUs } = message;
74
- const globalTimeUs = clipStartUs + (audioData.timestamp ?? 0);
75
- this.deps.cacheManager.putClipAudioData(sessionId, audioData, clipDurationUs, globalTimeUs);
76
- }
77
- async ensureAudioForTime(timeUs, options) {
78
- const model = this.model;
79
- if (!model) return;
80
- const mode = options?.mode ?? "blocking";
81
- const blockIndex = Math.floor(Math.max(0, timeUs) / this.PREVIEW_BLOCK_DURATION_US);
82
- if (mode === "probe") {
83
- void this.getOrCreateMixedBlock(blockIndex);
84
- const blockEndUs2 = (blockIndex + 1) * this.PREVIEW_BLOCK_DURATION_US;
85
- const remainingToBoundaryUs2 = blockEndUs2 - Math.max(0, timeUs);
86
- const lookaheadUs = Math.floor(this.PREVIEW_SCHEDULE_AHEAD_SEC * 1e6);
87
- if (remainingToBoundaryUs2 > 0 && remainingToBoundaryUs2 <= lookaheadUs) {
88
- void this.getOrCreateMixedBlock(blockIndex + 1);
89
- }
90
- return;
91
- }
92
- await this.getOrCreateMixedBlock(blockIndex);
93
- const blockEndUs = (blockIndex + 1) * this.PREVIEW_BLOCK_DURATION_US;
94
- const remainingToBoundaryUs = blockEndUs - Math.max(0, timeUs);
95
- if (remainingToBoundaryUs > 0 && remainingToBoundaryUs <= this.PREVIEW_BUFFER_GUARD_US) {
96
- await this.getOrCreateMixedBlock(blockIndex + 1);
97
- }
98
- }
99
- isPreviewMixBlockCached(timeUs) {
100
- const blockIndex = Math.floor(Math.max(0, timeUs) / this.PREVIEW_BLOCK_DURATION_US);
101
- return this.previewBlockCache.get(blockIndex) !== null;
102
- }
103
- shouldEnterBufferingForUpcomingPreviewAudio(timeUs) {
104
- const model = this.model;
105
- if (!model) return false;
106
- const clampedUs = Math.max(0, timeUs);
107
- const blockIndex = Math.floor(clampedUs / this.PREVIEW_BLOCK_DURATION_US);
108
- const nextBlockStartUs = (blockIndex + 1) * this.PREVIEW_BLOCK_DURATION_US;
109
- if (nextBlockStartUs >= model.durationUs) return false;
110
- const remainingToBoundaryUs = nextBlockStartUs - clampedUs;
111
- if (remainingToBoundaryUs > this.PREVIEW_BUFFER_GUARD_US) return false;
112
- return !this.isPreviewMixBlockCached(nextBlockStartUs);
113
- }
114
- /**
115
- * Fast readiness probe for preview playback.
116
- *
117
- * This is intentionally synchronous and lightweight:
118
- * - Only checks resource-level readiness (download + MP4 index parsing).
119
- * - If any relevant resource isn't ready yet, return false.
120
- * - Does NOT require audio samples / PCM window coverage (probe is resource-level only).
121
- *
122
- * Note: This probe does NOT gate on PCM coverage to avoid frequent buffering oscillation.
123
- * PCM is prepared incrementally by scheduleAudio() / ensureAudioForTimeRange().
124
- */
125
- isAudioResourceWindowReady(startUs, endUs) {
126
- const model = this.model;
127
- if (!model) return true;
128
- const activeClips = model.getActiveClips(startUs, endUs);
129
- for (const clip of activeClips) {
130
- if (clip.trackKind !== "audio" && clip.trackKind !== "video") continue;
131
- if (!hasResourceId(clip)) continue;
132
- const resource = model.getResource(clip.resourceId);
133
- if (resource?.state !== "ready") {
134
- return false;
135
- }
136
- }
137
- return true;
138
- }
139
- async activateAllAudioClips() {
140
- const model = this.model;
141
- if (!model) {
142
- return;
143
- }
144
- const audioTracks = model.tracks.filter((track) => track.kind === "audio");
145
- if (audioTracks.length === 0) return;
146
- const maxClipCount = Math.max(...audioTracks.map((track) => track.clips.length));
147
- for (let clipIndex = 0; clipIndex < maxClipCount; clipIndex++) {
148
- for (const track of audioTracks) {
149
- const clip = track.clips[clipIndex];
150
- if (!clip || this.activeClips.has(clip.id)) continue;
151
- if (!isAudioClip(clip)) {
152
- throw new Error(`Clip ${clip.id} in audio track is not an audio clip`);
153
- }
154
- if (this.deps.cacheManager.audioSampleCache.has(clip.resourceId)) {
155
- this.activeClips.add(clip.id);
156
- this.deps.eventBus.emit(MeframeEvent.ClipActivated, { clipId: clip.id });
157
- continue;
158
- }
159
- await this.deps.resourceLoader.load(clip.resourceId, {
160
- isPreload: false,
161
- clipId: clip.id,
162
- trackId: track.id
163
- });
164
- this.activeClips.add(clip.id);
165
- this.deps.eventBus.emit(MeframeEvent.ClipActivated, { clipId: clip.id });
166
- }
167
- }
168
- }
169
- async deactivateClip(clipId) {
170
- if (!this.activeClips.has(clipId)) {
171
- return;
172
- }
173
- this.activeClips.delete(clipId);
174
- this.deps.cacheManager.clearClipAudioData(clipId);
175
- }
176
- async startPlayback(timeUs, audioContext) {
177
- this.audioContext = audioContext;
178
- if (audioContext.state === "suspended") {
179
- await audioContext.resume();
180
- }
181
- await this.ensureAudioForTime(timeUs, { mode: "blocking" });
182
- this.isPlaying = true;
183
- this.startPreviewBlockScheduling(timeUs, audioContext);
184
- await this.scheduleNextPreviewBlock(audioContext, this.previewScheduleToken);
185
- void this.scheduleAudio(timeUs, audioContext);
186
- }
187
- stopPlayback() {
188
- this.isPlaying = false;
189
- this.stopAllPreviewSources();
190
- this.cancelPreviewBlockScheduling();
191
- this.previewMixToken += 1;
192
- }
193
- updateTime(_timeUs) {
194
- }
195
- /**
196
- * Schedule audio chunks ahead of playback cursor
197
- * Uses OfflineAudioMixer for proper mixing, then plays the result
198
- */
199
- async scheduleAudio(currentTimelineUs, audioContext) {
200
- if (!this.isPlaying || !this.model || !this.audioContext) {
201
- return;
202
- }
203
- this.previewLastTimelineUs = currentTimelineUs;
204
- if (this.previewScheduleTask) return;
205
- if (this.previewNextScheduleTime === 0) {
206
- this.startPreviewBlockScheduling(currentTimelineUs, audioContext);
207
- }
208
- const token = this.previewScheduleToken;
209
- this.previewScheduleTask = this.runPreviewBlockSchedulingLoop(audioContext, token).finally(
210
- () => {
211
- if (this.previewScheduleToken === token) {
212
- this.previewScheduleTask = null;
213
- }
214
- }
215
- );
216
- }
217
- /**
218
- * Reset playback states (called on seek)
219
- */
220
- resetPlaybackStates() {
221
- this.stopAllPreviewSources();
222
- this.cancelPreviewBlockScheduling();
223
- this.previewMixToken += 1;
224
- }
225
- setVolume(volume) {
226
- this.volume = volume;
227
- const audioContext = this.audioContext;
228
- if (!audioContext) return;
229
- const t = audioContext.currentTime;
230
- for (const { gain } of this.previewScheduledSources) {
231
- try {
232
- gain.gain.cancelScheduledValues(t);
233
- gain.gain.setValueAtTime(gain.gain.value, t);
234
- gain.gain.linearRampToValueAtTime(volume, t + 0.01);
235
- } catch {
236
- }
237
- }
238
- }
239
- setPlaybackRate(rate) {
240
- this.playbackRate = rate;
241
- this.resetPlaybackStates();
242
- }
243
- startPreviewBlockScheduling(startTimelineUs, audioContext) {
244
- this.cancelPreviewBlockScheduling();
245
- this.stopAllPreviewSources();
246
- const clampedUs = Math.max(0, startTimelineUs);
247
- const blockIndex = Math.floor(clampedUs / this.PREVIEW_BLOCK_DURATION_US);
248
- const blockStartUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
249
- this.previewFirstBlockOffsetUs = clampedUs - blockStartUs;
250
- this.previewNextBlockIndex = blockIndex;
251
- this.previewNextScheduleTime = audioContext.currentTime + 0.02;
252
- this.previewLastTimelineUs = startTimelineUs;
253
- }
254
- cancelPreviewBlockScheduling() {
255
- this.previewScheduleToken += 1;
256
- this.previewScheduleTask = null;
257
- this.previewNextBlockIndex = 0;
258
- this.previewNextScheduleTime = 0;
259
- this.previewFirstBlockOffsetUs = 0;
260
- this.previewLastTimelineUs = 0;
261
- }
262
- realignPreviewSchedulingToTimeline(audioContext) {
263
- const model = this.model;
264
- if (!model) return;
265
- const clampedUs = Math.max(0, this.previewLastTimelineUs);
266
- const blockIndex = Math.floor(clampedUs / this.PREVIEW_BLOCK_DURATION_US);
267
- const blockStartUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
268
- if (blockStartUs >= model.durationUs) return;
269
- this.stopAllPreviewSources();
270
- this.previewFirstBlockOffsetUs = clampedUs - blockStartUs;
271
- this.previewNextBlockIndex = blockIndex;
272
- this.previewNextScheduleTime = audioContext.currentTime + 0.02;
273
- }
274
- async runPreviewBlockSchedulingLoop(audioContext, token) {
275
- while (this.isPlaying && this.previewScheduleToken === token) {
276
- const model = this.model;
277
- if (!model) return;
278
- const nextBlockStartUs = this.previewNextBlockIndex * this.PREVIEW_BLOCK_DURATION_US;
279
- if (nextBlockStartUs >= model.durationUs) {
280
- this.previewNextScheduleTime = audioContext.currentTime + this.PREVIEW_SCHEDULE_AHEAD_SEC;
281
- return;
282
- }
283
- const scheduleAheadTime = audioContext.currentTime + this.PREVIEW_SCHEDULE_AHEAD_SEC;
284
- if (this.previewNextScheduleTime >= scheduleAheadTime) return;
285
- const prevBlockIndex = this.previewNextBlockIndex;
286
- const prevScheduleTime = this.previewNextScheduleTime;
287
- await this.scheduleNextPreviewBlock(audioContext, token);
288
- if (this.previewScheduleToken === token && this.previewNextBlockIndex === prevBlockIndex && this.previewNextScheduleTime === prevScheduleTime) {
289
- const now = audioContext.currentTime;
290
- if (now - this.previewLastStallWarnAt >= 1) {
291
- this.previewLastStallWarnAt = now;
292
- console.warn(
293
- "[GlobalAudioSession][preview] scheduling stalled; stop loop to avoid spin",
294
- {
295
- prevBlockIndex,
296
- prevScheduleTime,
297
- now
298
- }
299
- );
300
- }
301
- return;
302
- }
303
- }
304
- }
305
- async getOrCreateMixedBlock(blockIndex) {
306
- const model = this.model;
307
- if (!model) return null;
308
- const token = this.previewMixToken;
309
- return await this.previewBlockCache.getOrCreate(blockIndex, async () => {
310
- return await this.enqueuePreviewMix(async () => {
311
- const startUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
312
- const endUs = Math.min(startUs + this.PREVIEW_BLOCK_DURATION_US, model.durationUs);
313
- await this.ensureAudioForTimeRange(startUs, endUs, {
314
- mode: "blocking",
315
- loadResource: true
316
- });
317
- const mixed = await this.mixer.mix(startUs, endUs);
318
- this.deps.cacheManager.clearAudioCache();
319
- return mixed;
320
- }, token);
321
- });
322
- }
323
- async scheduleNextPreviewBlock(audioContext, token) {
324
- const model = this.model;
325
- if (!this.isPlaying || !model) return;
326
- if (this.previewScheduleToken !== token) return;
327
- if (this.previewNextScheduleTime < audioContext.currentTime + 0.01) {
328
- this.realignPreviewSchedulingToTimeline(audioContext);
329
- }
330
- const blockIndex = this.previewNextBlockIndex;
331
- const blockStartUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
332
- if (blockStartUs >= model.durationUs) {
333
- this.previewFirstBlockOffsetUs = 0;
334
- this.previewNextBlockIndex = blockIndex + 1;
335
- this.previewNextScheduleTime = audioContext.currentTime + this.PREVIEW_SCHEDULE_AHEAD_SEC;
336
- return;
337
- }
338
- const buffer = await this.getOrCreateMixedBlock(blockIndex);
339
- if (!buffer) {
340
- this.previewFirstBlockOffsetUs = 0;
341
- this.previewNextBlockIndex = blockIndex + 1;
342
- this.previewNextScheduleTime = audioContext.currentTime + 0.02;
343
- return;
344
- }
345
- if (this.previewScheduleToken !== token) return;
346
- const rate = this.playbackRate || 1;
347
- const offsetUs = this.previewFirstBlockOffsetUs;
348
- let startTime = this.previewNextScheduleTime;
349
- let offsetSec = offsetUs > 0 ? offsetUs / 1e6 : 0;
350
- const now = audioContext.currentTime;
351
- if (startTime < now + 0.01) {
352
- const targetStartTime = now + 0.02;
353
- const skippedSec = Math.max(0, (targetStartTime - startTime) * rate);
354
- startTime = targetStartTime;
355
- offsetSec += skippedSec;
356
- }
357
- if (offsetSec >= buffer.duration) {
358
- this.previewFirstBlockOffsetUs = 0;
359
- this.previewNextBlockIndex = blockIndex + 1;
360
- this.previewNextScheduleTime = startTime;
361
- return;
362
- }
363
- const remainingSec = Math.max(0, buffer.duration - offsetSec);
364
- if (remainingSec <= 0) return;
365
- const source = audioContext.createBufferSource();
366
- source.buffer = buffer;
367
- source.playbackRate.value = rate;
368
- const gainNode = audioContext.createGain();
369
- const volume = this.volume;
370
- const fadeSec = Math.min(this.PREVIEW_BLOCK_FADE_SEC, remainingSec / 2);
371
- gainNode.gain.setValueAtTime(0, startTime);
372
- gainNode.gain.linearRampToValueAtTime(volume, startTime + fadeSec);
373
- gainNode.gain.setValueAtTime(volume, startTime + Math.max(fadeSec, remainingSec - fadeSec));
374
- gainNode.gain.linearRampToValueAtTime(0, startTime + remainingSec);
375
- source.connect(gainNode);
376
- gainNode.connect(audioContext.destination);
377
- source.start(startTime, offsetSec);
378
- const entry = { source, gain: gainNode };
379
- this.previewScheduledSources.add(entry);
380
- source.onended = () => {
381
- try {
382
- source.disconnect();
383
- gainNode.disconnect();
384
- } catch {
385
- }
386
- this.previewScheduledSources.delete(entry);
387
- };
388
- this.previewFirstBlockOffsetUs = 0;
389
- this.previewNextBlockIndex = blockIndex + 1;
390
- this.previewNextScheduleTime = startTime + remainingSec / rate;
391
- }
392
- stopAllPreviewSources() {
393
- for (const { source, gain } of this.previewScheduledSources) {
394
- try {
395
- source.disconnect();
396
- } catch {
397
- }
398
- try {
399
- source.stop(0);
400
- } catch {
401
- }
402
- try {
403
- gain.disconnect();
404
- } catch {
405
- }
406
- }
407
- this.previewScheduledSources.clear();
408
- }
409
- reset() {
410
- this.stopAllPreviewSources();
411
- this.deps.cacheManager.clearAudioCache();
412
- this.activeClips.clear();
413
- this.previewBlockCache.clear();
414
- this.cancelPreviewBlockScheduling();
415
- this.previewMixToken += 1;
416
- }
417
- /**
418
- * Mix and encode audio for a specific segment (used by ExportScheduler)
419
- */
420
- async mixAndEncodeSegment(startUs, endUs, onChunk) {
421
- await this.ensureAudioForSegment(startUs, endUs);
422
- const mixedBuffer = await this.mixer.mix(startUs, endUs);
423
- const audioData = this.audioBufferToAudioData(mixedBuffer, startUs);
424
- if (!audioData) return;
425
- if (!this.exportEncoder) {
426
- this.exportEncoder = new AudioChunkEncoder();
427
- await this.exportEncoder.initialize();
428
- this.exportEncoderStream = this.exportEncoder.createStream();
429
- this.exportEncoderWriter = this.exportEncoderStream.writable.getWriter();
430
- void this.startExportEncoderReader(this.exportEncoderStream.readable, onChunk);
431
- await new Promise((resolve) => setTimeout(resolve, 10));
432
- }
433
- await this.exportEncoderWriter?.write(audioData);
434
- }
435
- /**
436
- * Ensure audio clips in time range are decoded (for export)
437
- * Decodes from AudioSampleCache (replaces Worker pipeline)
438
- */
439
- async ensureAudioForSegment(startUs, endUs) {
440
- await this.ensureAudioForTimeRange(startUs, endUs, {
441
- mode: "blocking",
442
- loadResource: false,
443
- strictMode: true
444
- });
445
- }
446
- exportEncoder = null;
447
- exportEncoderStream = null;
448
- exportEncoderWriter = null;
449
- async startExportEncoderReader(stream, onChunk) {
450
- const reader = stream.getReader();
451
- try {
452
- while (true) {
453
- const { done, value } = await reader.read();
454
- if (done) break;
455
- if (value) {
456
- onChunk(value.chunk, value.metadata);
457
- }
458
- }
459
- } catch (e) {
460
- console.error("Export encoder reader error", e);
461
- }
462
- }
463
- async finalizeExportAudio() {
464
- if (this.exportEncoderWriter) {
465
- await this.exportEncoderWriter.close();
466
- this.exportEncoderWriter = null;
467
- }
468
- this.exportEncoder = null;
469
- this.exportEncoderStream = null;
470
- }
471
- // Preview source scheduling is managed by stopAllPreviewSources()/previewScheduledSources.
472
- /**
473
- * Core method to ensure audio for all clips in a time range
474
- * Unified implementation used by ensureAudioForTime, scheduleAudio, and export
475
- */
476
- async ensureAudioForTimeRange(startUs, endUs, options) {
477
- const model = this.model;
478
- if (!model) return;
479
- const { mode = "blocking", loadResource = true, strictMode = false } = options;
480
- const activeClips = model.getActiveClips(startUs, endUs);
481
- const ensurePromises = activeClips.map(async (clip) => {
482
- if (clip.trackKind !== "audio" && clip.trackKind !== "video") return;
483
- if (!hasResourceId(clip)) return;
484
- const resource = model.getResource(clip.resourceId);
485
- if (resource?.state === "ready" && !this.deps.cacheManager.audioSampleCache.has(clip.resourceId)) {
486
- return;
487
- }
488
- if (!this.deps.cacheManager.audioSampleCache.has(clip.resourceId)) {
489
- if (!loadResource) {
490
- return;
491
- }
492
- const resource2 = model.getResource(clip.resourceId);
493
- if (resource2?.state !== "ready") {
494
- await this.deps.resourceLoader.load(clip.resourceId, {
495
- isPreload: false,
496
- clipId: clip.id,
497
- trackId: clip.trackId
498
- });
499
- }
500
- }
501
- const clipRelativeStartUs = Math.max(0, startUs - clip.startUs);
502
- const clipRelativeEndUs = Math.min(clip.durationUs, endUs - clip.startUs);
503
- const trimStartUs = clip.trimStartUs ?? 0;
504
- const resourceStartUs = clipRelativeStartUs + trimStartUs;
505
- const resourceEndUs = clipRelativeEndUs + trimStartUs;
506
- await this.ensureAudioWindow(clip.id, resourceStartUs, resourceEndUs, strictMode);
507
- });
508
- if (mode === "probe") {
509
- void Promise.all(ensurePromises);
510
- return;
511
- }
512
- await Promise.all(ensurePromises);
513
- }
514
- /**
515
- * Ensure audio window for a clip (aligned with video architecture)
516
- *
517
- * Note: Unlike video getFrame(), this method doesn't need a 'preheat' parameter
518
- * Why: Audio cache check is window-level (range query) via hasWindowPCM()
519
- * It verifies the entire window has ≥95% data (preview) or ≥99% (export)
520
- * This naturally prevents premature return during preheating
521
- */
522
- async ensureAudioWindow(clipId, startUs, endUs, strictMode = false) {
523
- if (this.deps.cacheManager.hasWindowPCM(clipId, startUs, endUs, strictMode)) {
524
- return;
525
- }
526
- await this.decodeAudioWindow(clipId, startUs, endUs);
527
- }
528
- /**
529
- * Decode audio window for a clip (aligned with video architecture)
530
- * Incremental decoding strategy with smart fallback:
531
- * - High coverage (≥80%): Skip decoding
532
- * - Low coverage (<30%): Full decode (avoid fragmentation)
533
- * - Medium coverage (30%-80%): Incremental decode
534
- */
535
- async decodeAudioWindow(clipId, startUs, endUs) {
536
- const clip = this.model?.findClip(clipId);
537
- if (!clip || !hasResourceId(clip)) {
538
- return;
539
- }
540
- const audioRecord = this.deps.cacheManager.audioSampleCache.get(clip.resourceId);
541
- if (!audioRecord) {
542
- return;
543
- }
544
- const windowChunks = audioRecord.samples.filter((s) => {
545
- const sampleEndUs = s.timestamp + (s.duration ?? 0);
546
- return s.timestamp < endUs && sampleEndUs > startUs;
547
- });
548
- if (windowChunks.length === 0) {
549
- return;
550
- }
551
- const INCREMENTAL_THRESHOLD = 0.95;
552
- const FULL_FALLBACK_THRESHOLD = 0.3;
553
- const coverage = this.deps.cacheManager.getAudioRangeCoverage(
554
- clipId,
555
- startUs,
556
- endUs,
557
- INCREMENTAL_THRESHOLD
558
- );
559
- if (coverage.covered) {
560
- return;
561
- }
562
- if (coverage.coverageRatio < FULL_FALLBACK_THRESHOLD) {
563
- await this.decodeAudioSamples(
564
- clipId,
565
- windowChunks,
566
- audioRecord.metadata,
567
- clip.durationUs,
568
- clip.startUs
569
- );
570
- return;
571
- }
572
- const chunksToDecode = windowChunks.filter((chunk) => {
573
- const chunkEndUs = chunk.timestamp + (chunk.duration ?? 0);
574
- const chunkCoverage = this.deps.cacheManager.getAudioRangeCoverage(
575
- clipId,
576
- chunk.timestamp,
577
- chunkEndUs,
578
- 0.95
579
- // Stricter threshold for individual chunks
580
- );
581
- return !chunkCoverage.covered;
582
- });
583
- if (chunksToDecode.length === 0) {
584
- return;
585
- }
586
- await this.decodeAudioSamples(
587
- clipId,
588
- chunksToDecode,
589
- audioRecord.metadata,
590
- clip.durationUs,
591
- clip.startUs
592
- );
593
- }
594
- /**
595
- * Decode audio samples to PCM and cache
596
- * Uses AudioChunkDecoder for consistency with project architecture
597
- * Resamples to AudioContext sample rate if needed for better quality
598
- */
599
- async decodeAudioSamples(clipId, samples, config, clipDurationUs, clipStartUs) {
600
- let description;
601
- if (config.description) {
602
- if (config.description instanceof ArrayBuffer) {
603
- description = config.description;
604
- } else if (ArrayBuffer.isView(config.description)) {
605
- const view = config.description;
606
- const newBuffer = new ArrayBuffer(view.byteLength);
607
- new Uint8Array(newBuffer).set(
608
- new Uint8Array(view.buffer, view.byteOffset, view.byteLength)
609
- );
610
- description = newBuffer;
611
- }
612
- }
613
- const decoderConfig = {
614
- codec: config.codec,
615
- sampleRate: config.sampleRate,
616
- numberOfChannels: config.numberOfChannels,
617
- description
618
- };
619
- const decoder = new AudioChunkDecoder(`audio-${clipId}`, decoderConfig);
620
- try {
621
- const chunkStream = new ReadableStream({
622
- start(controller) {
623
- for (const sample of samples) {
624
- controller.enqueue(sample);
625
- }
626
- controller.close();
627
- }
628
- });
629
- const audioDataStream = chunkStream.pipeThrough(decoder.createStream());
630
- const reader = audioDataStream.getReader();
631
- try {
632
- while (true) {
633
- const { done, value } = await reader.read();
634
- if (done) break;
635
- if (value) {
636
- const globalTimeUs = clipStartUs + (value.timestamp ?? 0);
637
- this.deps.cacheManager.putClipAudioData(clipId, value, clipDurationUs, globalTimeUs);
638
- }
639
- }
640
- } finally {
641
- reader.releaseLock();
642
- }
643
- } catch (error) {
644
- console.error(`[GlobalAudioSession] Decoder error for clip ${clipId}:`, error);
645
- throw error;
646
- } finally {
647
- await decoder.close();
648
- }
649
- }
650
- audioBufferToAudioData(buffer, timestampUs) {
651
- const sampleRate = buffer.sampleRate;
652
- const numberOfChannels = buffer.numberOfChannels;
653
- const numberOfFrames = buffer.length;
654
- const planes = [];
655
- for (let channel = 0; channel < numberOfChannels; channel++) {
656
- planes.push(buffer.getChannelData(channel));
657
- }
658
- return new AudioData({
659
- format: "f32-planar",
660
- sampleRate,
661
- numberOfFrames,
662
- numberOfChannels,
663
- timestamp: timestampUs,
664
- data: this.packPlanarF32Data(planes)
665
- });
666
- }
667
- packPlanarF32Data(planes) {
668
- const numberOfChannels = planes.length;
669
- const numberOfFrames = planes[0]?.length ?? 0;
670
- const totalSamples = numberOfChannels * numberOfFrames;
671
- const packed = new Float32Array(totalSamples);
672
- for (let channel = 0; channel < numberOfChannels; channel++) {
673
- const plane = planes[channel];
674
- if (!plane) continue;
675
- packed.set(plane, channel * numberOfFrames);
676
- }
677
- return packed.buffer;
678
- }
679
- }
680
- export {
681
- GlobalAudioSession
682
- };
683
- //# sourceMappingURL=GlobalAudioSession.js.map