@meframe/core 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Meframe.d.ts.map +1 -1
- package/dist/Meframe.js +1 -0
- package/dist/Meframe.js.map +1 -1
- package/dist/controllers/PlaybackController.d.ts.map +1 -1
- package/dist/controllers/PlaybackController.js +1 -1
- package/dist/controllers/PlaybackController.js.map +1 -1
- package/dist/orchestrator/AudioExportSession.d.ts +28 -0
- package/dist/orchestrator/AudioExportSession.d.ts.map +1 -0
- package/dist/orchestrator/AudioExportSession.js +95 -0
- package/dist/orchestrator/AudioExportSession.js.map +1 -0
- package/dist/orchestrator/AudioPreviewSession.d.ts +61 -0
- package/dist/orchestrator/AudioPreviewSession.d.ts.map +1 -0
- package/dist/orchestrator/AudioPreviewSession.js +340 -0
- package/dist/orchestrator/AudioPreviewSession.js.map +1 -0
- package/dist/orchestrator/AudioWindowPreparer.d.ts +62 -0
- package/dist/orchestrator/AudioWindowPreparer.d.ts.map +1 -0
- package/dist/orchestrator/AudioWindowPreparer.js +259 -0
- package/dist/orchestrator/AudioWindowPreparer.js.map +1 -0
- package/dist/orchestrator/ExportScheduler.d.ts +2 -2
- package/dist/orchestrator/ExportScheduler.d.ts.map +1 -1
- package/dist/orchestrator/ExportScheduler.js.map +1 -1
- package/dist/orchestrator/Orchestrator.d.ts +8 -2
- package/dist/orchestrator/Orchestrator.d.ts.map +1 -1
- package/dist/orchestrator/Orchestrator.js +22 -16
- package/dist/orchestrator/Orchestrator.js.map +1 -1
- package/dist/stages/mux/MP4Muxer.js.map +1 -1
- package/dist/stages/mux/MuxManager.d.ts +1 -4
- package/dist/stages/mux/MuxManager.d.ts.map +1 -1
- package/dist/stages/mux/MuxManager.js +1 -1
- package/dist/stages/mux/MuxManager.js.map +1 -1
- package/package.json +1 -1
- package/dist/orchestrator/GlobalAudioSession.d.ts +0 -139
- package/dist/orchestrator/GlobalAudioSession.d.ts.map +0 -1
- package/dist/orchestrator/GlobalAudioSession.js +0 -683
- package/dist/orchestrator/GlobalAudioSession.js.map +0 -1
|
@@ -1,683 +0,0 @@
|
|
|
1
|
-
import { OfflineAudioMixer } from "../stages/compose/OfflineAudioMixer.js";
|
|
2
|
-
import { MeframeEvent } from "../event/events.js";
|
|
3
|
-
import { AudioMixBlockCache } from "../cache/AudioMixBlockCache.js";
|
|
4
|
-
import { AudioChunkEncoder } from "../stages/encode/AudioChunkEncoder.js";
|
|
5
|
-
import { AudioChunkDecoder } from "../stages/decode/AudioChunkDecoder.js";
|
|
6
|
-
import { hasResourceId, isAudioClip } from "../model/types.js";
|
|
7
|
-
class GlobalAudioSession {
|
|
8
|
-
mixer;
|
|
9
|
-
activeClips = /* @__PURE__ */ new Set();
|
|
10
|
-
deps;
|
|
11
|
-
model = null;
|
|
12
|
-
audioContext = null;
|
|
13
|
-
volume = 1;
|
|
14
|
-
playbackRate = 1;
|
|
15
|
-
isPlaying = false;
|
|
16
|
-
// Preview strategy (unified):
|
|
17
|
-
// - Always schedule audio in fixed 60s "mix blocks"
|
|
18
|
-
// - Cache 2~3 mixed AudioBuffer blocks (LRU) to accelerate seek
|
|
19
|
-
// - Schedule ahead using AudioContext clock to avoid underrun
|
|
20
|
-
PREVIEW_BLOCK_DURATION_US = 60 * 1e6;
|
|
21
|
-
// 60s
|
|
22
|
-
PREVIEW_BLOCK_CACHE_SIZE = 3;
|
|
23
|
-
PREVIEW_SCHEDULE_AHEAD_SEC = 12;
|
|
24
|
-
// keep enough scheduled audio to hide mixing latency
|
|
25
|
-
PREVIEW_BUFFER_GUARD_US = 2e6;
|
|
26
|
-
// if next block isn't ready near boundary -> buffering
|
|
27
|
-
PREVIEW_BLOCK_FADE_SEC = 0.01;
|
|
28
|
-
// 10ms fade-in/out to avoid click at boundaries
|
|
29
|
-
previewBlockCache = new AudioMixBlockCache(this.PREVIEW_BLOCK_CACHE_SIZE);
|
|
30
|
-
previewScheduleTask = null;
|
|
31
|
-
previewScheduleToken = 0;
|
|
32
|
-
previewMixToken = 0;
|
|
33
|
-
previewNextBlockIndex = 0;
|
|
34
|
-
previewNextScheduleTime = 0;
|
|
35
|
-
// AudioContext time
|
|
36
|
-
previewFirstBlockOffsetUs = 0;
|
|
37
|
-
// seek offset within the first block
|
|
38
|
-
previewLastTimelineUs = 0;
|
|
39
|
-
previewLastStallWarnAt = 0;
|
|
40
|
-
// AudioContext time (sec)
|
|
41
|
-
previewScheduledSources = /* @__PURE__ */ new Set();
|
|
42
|
-
previewMixQueue = Promise.resolve();
|
|
43
|
-
constructor(deps) {
|
|
44
|
-
this.deps = deps;
|
|
45
|
-
this.mixer = new OfflineAudioMixer(deps.cacheManager, () => this.model);
|
|
46
|
-
}
|
|
47
|
-
enqueuePreviewMix(work, token) {
|
|
48
|
-
const run = () => work();
|
|
49
|
-
const next = this.previewMixQueue.then(
|
|
50
|
-
() => {
|
|
51
|
-
if (this.previewMixToken !== token) {
|
|
52
|
-
return null;
|
|
53
|
-
}
|
|
54
|
-
return run();
|
|
55
|
-
},
|
|
56
|
-
() => {
|
|
57
|
-
if (this.previewMixToken !== token) {
|
|
58
|
-
return null;
|
|
59
|
-
}
|
|
60
|
-
return run();
|
|
61
|
-
}
|
|
62
|
-
);
|
|
63
|
-
this.previewMixQueue = next.then(
|
|
64
|
-
() => void 0,
|
|
65
|
-
() => void 0
|
|
66
|
-
);
|
|
67
|
-
return next;
|
|
68
|
-
}
|
|
69
|
-
setModel(model) {
|
|
70
|
-
this.model = model;
|
|
71
|
-
}
|
|
72
|
-
onAudioData(message) {
|
|
73
|
-
const { sessionId, audioData, clipStartUs, clipDurationUs } = message;
|
|
74
|
-
const globalTimeUs = clipStartUs + (audioData.timestamp ?? 0);
|
|
75
|
-
this.deps.cacheManager.putClipAudioData(sessionId, audioData, clipDurationUs, globalTimeUs);
|
|
76
|
-
}
|
|
77
|
-
async ensureAudioForTime(timeUs, options) {
|
|
78
|
-
const model = this.model;
|
|
79
|
-
if (!model) return;
|
|
80
|
-
const mode = options?.mode ?? "blocking";
|
|
81
|
-
const blockIndex = Math.floor(Math.max(0, timeUs) / this.PREVIEW_BLOCK_DURATION_US);
|
|
82
|
-
if (mode === "probe") {
|
|
83
|
-
void this.getOrCreateMixedBlock(blockIndex);
|
|
84
|
-
const blockEndUs2 = (blockIndex + 1) * this.PREVIEW_BLOCK_DURATION_US;
|
|
85
|
-
const remainingToBoundaryUs2 = blockEndUs2 - Math.max(0, timeUs);
|
|
86
|
-
const lookaheadUs = Math.floor(this.PREVIEW_SCHEDULE_AHEAD_SEC * 1e6);
|
|
87
|
-
if (remainingToBoundaryUs2 > 0 && remainingToBoundaryUs2 <= lookaheadUs) {
|
|
88
|
-
void this.getOrCreateMixedBlock(blockIndex + 1);
|
|
89
|
-
}
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
await this.getOrCreateMixedBlock(blockIndex);
|
|
93
|
-
const blockEndUs = (blockIndex + 1) * this.PREVIEW_BLOCK_DURATION_US;
|
|
94
|
-
const remainingToBoundaryUs = blockEndUs - Math.max(0, timeUs);
|
|
95
|
-
if (remainingToBoundaryUs > 0 && remainingToBoundaryUs <= this.PREVIEW_BUFFER_GUARD_US) {
|
|
96
|
-
await this.getOrCreateMixedBlock(blockIndex + 1);
|
|
97
|
-
}
|
|
98
|
-
}
|
|
99
|
-
isPreviewMixBlockCached(timeUs) {
|
|
100
|
-
const blockIndex = Math.floor(Math.max(0, timeUs) / this.PREVIEW_BLOCK_DURATION_US);
|
|
101
|
-
return this.previewBlockCache.get(blockIndex) !== null;
|
|
102
|
-
}
|
|
103
|
-
shouldEnterBufferingForUpcomingPreviewAudio(timeUs) {
|
|
104
|
-
const model = this.model;
|
|
105
|
-
if (!model) return false;
|
|
106
|
-
const clampedUs = Math.max(0, timeUs);
|
|
107
|
-
const blockIndex = Math.floor(clampedUs / this.PREVIEW_BLOCK_DURATION_US);
|
|
108
|
-
const nextBlockStartUs = (blockIndex + 1) * this.PREVIEW_BLOCK_DURATION_US;
|
|
109
|
-
if (nextBlockStartUs >= model.durationUs) return false;
|
|
110
|
-
const remainingToBoundaryUs = nextBlockStartUs - clampedUs;
|
|
111
|
-
if (remainingToBoundaryUs > this.PREVIEW_BUFFER_GUARD_US) return false;
|
|
112
|
-
return !this.isPreviewMixBlockCached(nextBlockStartUs);
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Fast readiness probe for preview playback.
|
|
116
|
-
*
|
|
117
|
-
* This is intentionally synchronous and lightweight:
|
|
118
|
-
* - Only checks resource-level readiness (download + MP4 index parsing).
|
|
119
|
-
* - If any relevant resource isn't ready yet, return false.
|
|
120
|
-
* - Does NOT require audio samples / PCM window coverage (probe is resource-level only).
|
|
121
|
-
*
|
|
122
|
-
* Note: This probe does NOT gate on PCM coverage to avoid frequent buffering oscillation.
|
|
123
|
-
* PCM is prepared incrementally by scheduleAudio() / ensureAudioForTimeRange().
|
|
124
|
-
*/
|
|
125
|
-
isAudioResourceWindowReady(startUs, endUs) {
|
|
126
|
-
const model = this.model;
|
|
127
|
-
if (!model) return true;
|
|
128
|
-
const activeClips = model.getActiveClips(startUs, endUs);
|
|
129
|
-
for (const clip of activeClips) {
|
|
130
|
-
if (clip.trackKind !== "audio" && clip.trackKind !== "video") continue;
|
|
131
|
-
if (!hasResourceId(clip)) continue;
|
|
132
|
-
const resource = model.getResource(clip.resourceId);
|
|
133
|
-
if (resource?.state !== "ready") {
|
|
134
|
-
return false;
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
return true;
|
|
138
|
-
}
|
|
139
|
-
async activateAllAudioClips() {
|
|
140
|
-
const model = this.model;
|
|
141
|
-
if (!model) {
|
|
142
|
-
return;
|
|
143
|
-
}
|
|
144
|
-
const audioTracks = model.tracks.filter((track) => track.kind === "audio");
|
|
145
|
-
if (audioTracks.length === 0) return;
|
|
146
|
-
const maxClipCount = Math.max(...audioTracks.map((track) => track.clips.length));
|
|
147
|
-
for (let clipIndex = 0; clipIndex < maxClipCount; clipIndex++) {
|
|
148
|
-
for (const track of audioTracks) {
|
|
149
|
-
const clip = track.clips[clipIndex];
|
|
150
|
-
if (!clip || this.activeClips.has(clip.id)) continue;
|
|
151
|
-
if (!isAudioClip(clip)) {
|
|
152
|
-
throw new Error(`Clip ${clip.id} in audio track is not an audio clip`);
|
|
153
|
-
}
|
|
154
|
-
if (this.deps.cacheManager.audioSampleCache.has(clip.resourceId)) {
|
|
155
|
-
this.activeClips.add(clip.id);
|
|
156
|
-
this.deps.eventBus.emit(MeframeEvent.ClipActivated, { clipId: clip.id });
|
|
157
|
-
continue;
|
|
158
|
-
}
|
|
159
|
-
await this.deps.resourceLoader.load(clip.resourceId, {
|
|
160
|
-
isPreload: false,
|
|
161
|
-
clipId: clip.id,
|
|
162
|
-
trackId: track.id
|
|
163
|
-
});
|
|
164
|
-
this.activeClips.add(clip.id);
|
|
165
|
-
this.deps.eventBus.emit(MeframeEvent.ClipActivated, { clipId: clip.id });
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
async deactivateClip(clipId) {
|
|
170
|
-
if (!this.activeClips.has(clipId)) {
|
|
171
|
-
return;
|
|
172
|
-
}
|
|
173
|
-
this.activeClips.delete(clipId);
|
|
174
|
-
this.deps.cacheManager.clearClipAudioData(clipId);
|
|
175
|
-
}
|
|
176
|
-
async startPlayback(timeUs, audioContext) {
|
|
177
|
-
this.audioContext = audioContext;
|
|
178
|
-
if (audioContext.state === "suspended") {
|
|
179
|
-
await audioContext.resume();
|
|
180
|
-
}
|
|
181
|
-
await this.ensureAudioForTime(timeUs, { mode: "blocking" });
|
|
182
|
-
this.isPlaying = true;
|
|
183
|
-
this.startPreviewBlockScheduling(timeUs, audioContext);
|
|
184
|
-
await this.scheduleNextPreviewBlock(audioContext, this.previewScheduleToken);
|
|
185
|
-
void this.scheduleAudio(timeUs, audioContext);
|
|
186
|
-
}
|
|
187
|
-
stopPlayback() {
|
|
188
|
-
this.isPlaying = false;
|
|
189
|
-
this.stopAllPreviewSources();
|
|
190
|
-
this.cancelPreviewBlockScheduling();
|
|
191
|
-
this.previewMixToken += 1;
|
|
192
|
-
}
|
|
193
|
-
updateTime(_timeUs) {
|
|
194
|
-
}
|
|
195
|
-
/**
|
|
196
|
-
* Schedule audio chunks ahead of playback cursor
|
|
197
|
-
* Uses OfflineAudioMixer for proper mixing, then plays the result
|
|
198
|
-
*/
|
|
199
|
-
async scheduleAudio(currentTimelineUs, audioContext) {
|
|
200
|
-
if (!this.isPlaying || !this.model || !this.audioContext) {
|
|
201
|
-
return;
|
|
202
|
-
}
|
|
203
|
-
this.previewLastTimelineUs = currentTimelineUs;
|
|
204
|
-
if (this.previewScheduleTask) return;
|
|
205
|
-
if (this.previewNextScheduleTime === 0) {
|
|
206
|
-
this.startPreviewBlockScheduling(currentTimelineUs, audioContext);
|
|
207
|
-
}
|
|
208
|
-
const token = this.previewScheduleToken;
|
|
209
|
-
this.previewScheduleTask = this.runPreviewBlockSchedulingLoop(audioContext, token).finally(
|
|
210
|
-
() => {
|
|
211
|
-
if (this.previewScheduleToken === token) {
|
|
212
|
-
this.previewScheduleTask = null;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
);
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Reset playback states (called on seek)
|
|
219
|
-
*/
|
|
220
|
-
resetPlaybackStates() {
|
|
221
|
-
this.stopAllPreviewSources();
|
|
222
|
-
this.cancelPreviewBlockScheduling();
|
|
223
|
-
this.previewMixToken += 1;
|
|
224
|
-
}
|
|
225
|
-
setVolume(volume) {
|
|
226
|
-
this.volume = volume;
|
|
227
|
-
const audioContext = this.audioContext;
|
|
228
|
-
if (!audioContext) return;
|
|
229
|
-
const t = audioContext.currentTime;
|
|
230
|
-
for (const { gain } of this.previewScheduledSources) {
|
|
231
|
-
try {
|
|
232
|
-
gain.gain.cancelScheduledValues(t);
|
|
233
|
-
gain.gain.setValueAtTime(gain.gain.value, t);
|
|
234
|
-
gain.gain.linearRampToValueAtTime(volume, t + 0.01);
|
|
235
|
-
} catch {
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
setPlaybackRate(rate) {
|
|
240
|
-
this.playbackRate = rate;
|
|
241
|
-
this.resetPlaybackStates();
|
|
242
|
-
}
|
|
243
|
-
startPreviewBlockScheduling(startTimelineUs, audioContext) {
|
|
244
|
-
this.cancelPreviewBlockScheduling();
|
|
245
|
-
this.stopAllPreviewSources();
|
|
246
|
-
const clampedUs = Math.max(0, startTimelineUs);
|
|
247
|
-
const blockIndex = Math.floor(clampedUs / this.PREVIEW_BLOCK_DURATION_US);
|
|
248
|
-
const blockStartUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
|
|
249
|
-
this.previewFirstBlockOffsetUs = clampedUs - blockStartUs;
|
|
250
|
-
this.previewNextBlockIndex = blockIndex;
|
|
251
|
-
this.previewNextScheduleTime = audioContext.currentTime + 0.02;
|
|
252
|
-
this.previewLastTimelineUs = startTimelineUs;
|
|
253
|
-
}
|
|
254
|
-
cancelPreviewBlockScheduling() {
|
|
255
|
-
this.previewScheduleToken += 1;
|
|
256
|
-
this.previewScheduleTask = null;
|
|
257
|
-
this.previewNextBlockIndex = 0;
|
|
258
|
-
this.previewNextScheduleTime = 0;
|
|
259
|
-
this.previewFirstBlockOffsetUs = 0;
|
|
260
|
-
this.previewLastTimelineUs = 0;
|
|
261
|
-
}
|
|
262
|
-
realignPreviewSchedulingToTimeline(audioContext) {
|
|
263
|
-
const model = this.model;
|
|
264
|
-
if (!model) return;
|
|
265
|
-
const clampedUs = Math.max(0, this.previewLastTimelineUs);
|
|
266
|
-
const blockIndex = Math.floor(clampedUs / this.PREVIEW_BLOCK_DURATION_US);
|
|
267
|
-
const blockStartUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
|
|
268
|
-
if (blockStartUs >= model.durationUs) return;
|
|
269
|
-
this.stopAllPreviewSources();
|
|
270
|
-
this.previewFirstBlockOffsetUs = clampedUs - blockStartUs;
|
|
271
|
-
this.previewNextBlockIndex = blockIndex;
|
|
272
|
-
this.previewNextScheduleTime = audioContext.currentTime + 0.02;
|
|
273
|
-
}
|
|
274
|
-
async runPreviewBlockSchedulingLoop(audioContext, token) {
|
|
275
|
-
while (this.isPlaying && this.previewScheduleToken === token) {
|
|
276
|
-
const model = this.model;
|
|
277
|
-
if (!model) return;
|
|
278
|
-
const nextBlockStartUs = this.previewNextBlockIndex * this.PREVIEW_BLOCK_DURATION_US;
|
|
279
|
-
if (nextBlockStartUs >= model.durationUs) {
|
|
280
|
-
this.previewNextScheduleTime = audioContext.currentTime + this.PREVIEW_SCHEDULE_AHEAD_SEC;
|
|
281
|
-
return;
|
|
282
|
-
}
|
|
283
|
-
const scheduleAheadTime = audioContext.currentTime + this.PREVIEW_SCHEDULE_AHEAD_SEC;
|
|
284
|
-
if (this.previewNextScheduleTime >= scheduleAheadTime) return;
|
|
285
|
-
const prevBlockIndex = this.previewNextBlockIndex;
|
|
286
|
-
const prevScheduleTime = this.previewNextScheduleTime;
|
|
287
|
-
await this.scheduleNextPreviewBlock(audioContext, token);
|
|
288
|
-
if (this.previewScheduleToken === token && this.previewNextBlockIndex === prevBlockIndex && this.previewNextScheduleTime === prevScheduleTime) {
|
|
289
|
-
const now = audioContext.currentTime;
|
|
290
|
-
if (now - this.previewLastStallWarnAt >= 1) {
|
|
291
|
-
this.previewLastStallWarnAt = now;
|
|
292
|
-
console.warn(
|
|
293
|
-
"[GlobalAudioSession][preview] scheduling stalled; stop loop to avoid spin",
|
|
294
|
-
{
|
|
295
|
-
prevBlockIndex,
|
|
296
|
-
prevScheduleTime,
|
|
297
|
-
now
|
|
298
|
-
}
|
|
299
|
-
);
|
|
300
|
-
}
|
|
301
|
-
return;
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
async getOrCreateMixedBlock(blockIndex) {
|
|
306
|
-
const model = this.model;
|
|
307
|
-
if (!model) return null;
|
|
308
|
-
const token = this.previewMixToken;
|
|
309
|
-
return await this.previewBlockCache.getOrCreate(blockIndex, async () => {
|
|
310
|
-
return await this.enqueuePreviewMix(async () => {
|
|
311
|
-
const startUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
|
|
312
|
-
const endUs = Math.min(startUs + this.PREVIEW_BLOCK_DURATION_US, model.durationUs);
|
|
313
|
-
await this.ensureAudioForTimeRange(startUs, endUs, {
|
|
314
|
-
mode: "blocking",
|
|
315
|
-
loadResource: true
|
|
316
|
-
});
|
|
317
|
-
const mixed = await this.mixer.mix(startUs, endUs);
|
|
318
|
-
this.deps.cacheManager.clearAudioCache();
|
|
319
|
-
return mixed;
|
|
320
|
-
}, token);
|
|
321
|
-
});
|
|
322
|
-
}
|
|
323
|
-
async scheduleNextPreviewBlock(audioContext, token) {
|
|
324
|
-
const model = this.model;
|
|
325
|
-
if (!this.isPlaying || !model) return;
|
|
326
|
-
if (this.previewScheduleToken !== token) return;
|
|
327
|
-
if (this.previewNextScheduleTime < audioContext.currentTime + 0.01) {
|
|
328
|
-
this.realignPreviewSchedulingToTimeline(audioContext);
|
|
329
|
-
}
|
|
330
|
-
const blockIndex = this.previewNextBlockIndex;
|
|
331
|
-
const blockStartUs = blockIndex * this.PREVIEW_BLOCK_DURATION_US;
|
|
332
|
-
if (blockStartUs >= model.durationUs) {
|
|
333
|
-
this.previewFirstBlockOffsetUs = 0;
|
|
334
|
-
this.previewNextBlockIndex = blockIndex + 1;
|
|
335
|
-
this.previewNextScheduleTime = audioContext.currentTime + this.PREVIEW_SCHEDULE_AHEAD_SEC;
|
|
336
|
-
return;
|
|
337
|
-
}
|
|
338
|
-
const buffer = await this.getOrCreateMixedBlock(blockIndex);
|
|
339
|
-
if (!buffer) {
|
|
340
|
-
this.previewFirstBlockOffsetUs = 0;
|
|
341
|
-
this.previewNextBlockIndex = blockIndex + 1;
|
|
342
|
-
this.previewNextScheduleTime = audioContext.currentTime + 0.02;
|
|
343
|
-
return;
|
|
344
|
-
}
|
|
345
|
-
if (this.previewScheduleToken !== token) return;
|
|
346
|
-
const rate = this.playbackRate || 1;
|
|
347
|
-
const offsetUs = this.previewFirstBlockOffsetUs;
|
|
348
|
-
let startTime = this.previewNextScheduleTime;
|
|
349
|
-
let offsetSec = offsetUs > 0 ? offsetUs / 1e6 : 0;
|
|
350
|
-
const now = audioContext.currentTime;
|
|
351
|
-
if (startTime < now + 0.01) {
|
|
352
|
-
const targetStartTime = now + 0.02;
|
|
353
|
-
const skippedSec = Math.max(0, (targetStartTime - startTime) * rate);
|
|
354
|
-
startTime = targetStartTime;
|
|
355
|
-
offsetSec += skippedSec;
|
|
356
|
-
}
|
|
357
|
-
if (offsetSec >= buffer.duration) {
|
|
358
|
-
this.previewFirstBlockOffsetUs = 0;
|
|
359
|
-
this.previewNextBlockIndex = blockIndex + 1;
|
|
360
|
-
this.previewNextScheduleTime = startTime;
|
|
361
|
-
return;
|
|
362
|
-
}
|
|
363
|
-
const remainingSec = Math.max(0, buffer.duration - offsetSec);
|
|
364
|
-
if (remainingSec <= 0) return;
|
|
365
|
-
const source = audioContext.createBufferSource();
|
|
366
|
-
source.buffer = buffer;
|
|
367
|
-
source.playbackRate.value = rate;
|
|
368
|
-
const gainNode = audioContext.createGain();
|
|
369
|
-
const volume = this.volume;
|
|
370
|
-
const fadeSec = Math.min(this.PREVIEW_BLOCK_FADE_SEC, remainingSec / 2);
|
|
371
|
-
gainNode.gain.setValueAtTime(0, startTime);
|
|
372
|
-
gainNode.gain.linearRampToValueAtTime(volume, startTime + fadeSec);
|
|
373
|
-
gainNode.gain.setValueAtTime(volume, startTime + Math.max(fadeSec, remainingSec - fadeSec));
|
|
374
|
-
gainNode.gain.linearRampToValueAtTime(0, startTime + remainingSec);
|
|
375
|
-
source.connect(gainNode);
|
|
376
|
-
gainNode.connect(audioContext.destination);
|
|
377
|
-
source.start(startTime, offsetSec);
|
|
378
|
-
const entry = { source, gain: gainNode };
|
|
379
|
-
this.previewScheduledSources.add(entry);
|
|
380
|
-
source.onended = () => {
|
|
381
|
-
try {
|
|
382
|
-
source.disconnect();
|
|
383
|
-
gainNode.disconnect();
|
|
384
|
-
} catch {
|
|
385
|
-
}
|
|
386
|
-
this.previewScheduledSources.delete(entry);
|
|
387
|
-
};
|
|
388
|
-
this.previewFirstBlockOffsetUs = 0;
|
|
389
|
-
this.previewNextBlockIndex = blockIndex + 1;
|
|
390
|
-
this.previewNextScheduleTime = startTime + remainingSec / rate;
|
|
391
|
-
}
|
|
392
|
-
stopAllPreviewSources() {
|
|
393
|
-
for (const { source, gain } of this.previewScheduledSources) {
|
|
394
|
-
try {
|
|
395
|
-
source.disconnect();
|
|
396
|
-
} catch {
|
|
397
|
-
}
|
|
398
|
-
try {
|
|
399
|
-
source.stop(0);
|
|
400
|
-
} catch {
|
|
401
|
-
}
|
|
402
|
-
try {
|
|
403
|
-
gain.disconnect();
|
|
404
|
-
} catch {
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
this.previewScheduledSources.clear();
|
|
408
|
-
}
|
|
409
|
-
reset() {
|
|
410
|
-
this.stopAllPreviewSources();
|
|
411
|
-
this.deps.cacheManager.clearAudioCache();
|
|
412
|
-
this.activeClips.clear();
|
|
413
|
-
this.previewBlockCache.clear();
|
|
414
|
-
this.cancelPreviewBlockScheduling();
|
|
415
|
-
this.previewMixToken += 1;
|
|
416
|
-
}
|
|
417
|
-
/**
|
|
418
|
-
* Mix and encode audio for a specific segment (used by ExportScheduler)
|
|
419
|
-
*/
|
|
420
|
-
async mixAndEncodeSegment(startUs, endUs, onChunk) {
|
|
421
|
-
await this.ensureAudioForSegment(startUs, endUs);
|
|
422
|
-
const mixedBuffer = await this.mixer.mix(startUs, endUs);
|
|
423
|
-
const audioData = this.audioBufferToAudioData(mixedBuffer, startUs);
|
|
424
|
-
if (!audioData) return;
|
|
425
|
-
if (!this.exportEncoder) {
|
|
426
|
-
this.exportEncoder = new AudioChunkEncoder();
|
|
427
|
-
await this.exportEncoder.initialize();
|
|
428
|
-
this.exportEncoderStream = this.exportEncoder.createStream();
|
|
429
|
-
this.exportEncoderWriter = this.exportEncoderStream.writable.getWriter();
|
|
430
|
-
void this.startExportEncoderReader(this.exportEncoderStream.readable, onChunk);
|
|
431
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
432
|
-
}
|
|
433
|
-
await this.exportEncoderWriter?.write(audioData);
|
|
434
|
-
}
|
|
435
|
-
/**
|
|
436
|
-
* Ensure audio clips in time range are decoded (for export)
|
|
437
|
-
* Decodes from AudioSampleCache (replaces Worker pipeline)
|
|
438
|
-
*/
|
|
439
|
-
async ensureAudioForSegment(startUs, endUs) {
|
|
440
|
-
await this.ensureAudioForTimeRange(startUs, endUs, {
|
|
441
|
-
mode: "blocking",
|
|
442
|
-
loadResource: false,
|
|
443
|
-
strictMode: true
|
|
444
|
-
});
|
|
445
|
-
}
|
|
446
|
-
exportEncoder = null;
|
|
447
|
-
exportEncoderStream = null;
|
|
448
|
-
exportEncoderWriter = null;
|
|
449
|
-
async startExportEncoderReader(stream, onChunk) {
|
|
450
|
-
const reader = stream.getReader();
|
|
451
|
-
try {
|
|
452
|
-
while (true) {
|
|
453
|
-
const { done, value } = await reader.read();
|
|
454
|
-
if (done) break;
|
|
455
|
-
if (value) {
|
|
456
|
-
onChunk(value.chunk, value.metadata);
|
|
457
|
-
}
|
|
458
|
-
}
|
|
459
|
-
} catch (e) {
|
|
460
|
-
console.error("Export encoder reader error", e);
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
async finalizeExportAudio() {
|
|
464
|
-
if (this.exportEncoderWriter) {
|
|
465
|
-
await this.exportEncoderWriter.close();
|
|
466
|
-
this.exportEncoderWriter = null;
|
|
467
|
-
}
|
|
468
|
-
this.exportEncoder = null;
|
|
469
|
-
this.exportEncoderStream = null;
|
|
470
|
-
}
|
|
471
|
-
// Preview source scheduling is managed by stopAllPreviewSources()/previewScheduledSources.
|
|
472
|
-
/**
|
|
473
|
-
* Core method to ensure audio for all clips in a time range
|
|
474
|
-
* Unified implementation used by ensureAudioForTime, scheduleAudio, and export
|
|
475
|
-
*/
|
|
476
|
-
async ensureAudioForTimeRange(startUs, endUs, options) {
|
|
477
|
-
const model = this.model;
|
|
478
|
-
if (!model) return;
|
|
479
|
-
const { mode = "blocking", loadResource = true, strictMode = false } = options;
|
|
480
|
-
const activeClips = model.getActiveClips(startUs, endUs);
|
|
481
|
-
const ensurePromises = activeClips.map(async (clip) => {
|
|
482
|
-
if (clip.trackKind !== "audio" && clip.trackKind !== "video") return;
|
|
483
|
-
if (!hasResourceId(clip)) return;
|
|
484
|
-
const resource = model.getResource(clip.resourceId);
|
|
485
|
-
if (resource?.state === "ready" && !this.deps.cacheManager.audioSampleCache.has(clip.resourceId)) {
|
|
486
|
-
return;
|
|
487
|
-
}
|
|
488
|
-
if (!this.deps.cacheManager.audioSampleCache.has(clip.resourceId)) {
|
|
489
|
-
if (!loadResource) {
|
|
490
|
-
return;
|
|
491
|
-
}
|
|
492
|
-
const resource2 = model.getResource(clip.resourceId);
|
|
493
|
-
if (resource2?.state !== "ready") {
|
|
494
|
-
await this.deps.resourceLoader.load(clip.resourceId, {
|
|
495
|
-
isPreload: false,
|
|
496
|
-
clipId: clip.id,
|
|
497
|
-
trackId: clip.trackId
|
|
498
|
-
});
|
|
499
|
-
}
|
|
500
|
-
}
|
|
501
|
-
const clipRelativeStartUs = Math.max(0, startUs - clip.startUs);
|
|
502
|
-
const clipRelativeEndUs = Math.min(clip.durationUs, endUs - clip.startUs);
|
|
503
|
-
const trimStartUs = clip.trimStartUs ?? 0;
|
|
504
|
-
const resourceStartUs = clipRelativeStartUs + trimStartUs;
|
|
505
|
-
const resourceEndUs = clipRelativeEndUs + trimStartUs;
|
|
506
|
-
await this.ensureAudioWindow(clip.id, resourceStartUs, resourceEndUs, strictMode);
|
|
507
|
-
});
|
|
508
|
-
if (mode === "probe") {
|
|
509
|
-
void Promise.all(ensurePromises);
|
|
510
|
-
return;
|
|
511
|
-
}
|
|
512
|
-
await Promise.all(ensurePromises);
|
|
513
|
-
}
|
|
514
|
-
/**
|
|
515
|
-
* Ensure audio window for a clip (aligned with video architecture)
|
|
516
|
-
*
|
|
517
|
-
* Note: Unlike video getFrame(), this method doesn't need a 'preheat' parameter
|
|
518
|
-
* Why: Audio cache check is window-level (range query) via hasWindowPCM()
|
|
519
|
-
* It verifies the entire window has ≥95% data (preview) or ≥99% (export)
|
|
520
|
-
* This naturally prevents premature return during preheating
|
|
521
|
-
*/
|
|
522
|
-
async ensureAudioWindow(clipId, startUs, endUs, strictMode = false) {
|
|
523
|
-
if (this.deps.cacheManager.hasWindowPCM(clipId, startUs, endUs, strictMode)) {
|
|
524
|
-
return;
|
|
525
|
-
}
|
|
526
|
-
await this.decodeAudioWindow(clipId, startUs, endUs);
|
|
527
|
-
}
|
|
528
|
-
/**
|
|
529
|
-
* Decode audio window for a clip (aligned with video architecture)
|
|
530
|
-
* Incremental decoding strategy with smart fallback:
|
|
531
|
-
* - High coverage (≥80%): Skip decoding
|
|
532
|
-
* - Low coverage (<30%): Full decode (avoid fragmentation)
|
|
533
|
-
* - Medium coverage (30%-80%): Incremental decode
|
|
534
|
-
*/
|
|
535
|
-
async decodeAudioWindow(clipId, startUs, endUs) {
|
|
536
|
-
const clip = this.model?.findClip(clipId);
|
|
537
|
-
if (!clip || !hasResourceId(clip)) {
|
|
538
|
-
return;
|
|
539
|
-
}
|
|
540
|
-
const audioRecord = this.deps.cacheManager.audioSampleCache.get(clip.resourceId);
|
|
541
|
-
if (!audioRecord) {
|
|
542
|
-
return;
|
|
543
|
-
}
|
|
544
|
-
const windowChunks = audioRecord.samples.filter((s) => {
|
|
545
|
-
const sampleEndUs = s.timestamp + (s.duration ?? 0);
|
|
546
|
-
return s.timestamp < endUs && sampleEndUs > startUs;
|
|
547
|
-
});
|
|
548
|
-
if (windowChunks.length === 0) {
|
|
549
|
-
return;
|
|
550
|
-
}
|
|
551
|
-
const INCREMENTAL_THRESHOLD = 0.95;
|
|
552
|
-
const FULL_FALLBACK_THRESHOLD = 0.3;
|
|
553
|
-
const coverage = this.deps.cacheManager.getAudioRangeCoverage(
|
|
554
|
-
clipId,
|
|
555
|
-
startUs,
|
|
556
|
-
endUs,
|
|
557
|
-
INCREMENTAL_THRESHOLD
|
|
558
|
-
);
|
|
559
|
-
if (coverage.covered) {
|
|
560
|
-
return;
|
|
561
|
-
}
|
|
562
|
-
if (coverage.coverageRatio < FULL_FALLBACK_THRESHOLD) {
|
|
563
|
-
await this.decodeAudioSamples(
|
|
564
|
-
clipId,
|
|
565
|
-
windowChunks,
|
|
566
|
-
audioRecord.metadata,
|
|
567
|
-
clip.durationUs,
|
|
568
|
-
clip.startUs
|
|
569
|
-
);
|
|
570
|
-
return;
|
|
571
|
-
}
|
|
572
|
-
const chunksToDecode = windowChunks.filter((chunk) => {
|
|
573
|
-
const chunkEndUs = chunk.timestamp + (chunk.duration ?? 0);
|
|
574
|
-
const chunkCoverage = this.deps.cacheManager.getAudioRangeCoverage(
|
|
575
|
-
clipId,
|
|
576
|
-
chunk.timestamp,
|
|
577
|
-
chunkEndUs,
|
|
578
|
-
0.95
|
|
579
|
-
// Stricter threshold for individual chunks
|
|
580
|
-
);
|
|
581
|
-
return !chunkCoverage.covered;
|
|
582
|
-
});
|
|
583
|
-
if (chunksToDecode.length === 0) {
|
|
584
|
-
return;
|
|
585
|
-
}
|
|
586
|
-
await this.decodeAudioSamples(
|
|
587
|
-
clipId,
|
|
588
|
-
chunksToDecode,
|
|
589
|
-
audioRecord.metadata,
|
|
590
|
-
clip.durationUs,
|
|
591
|
-
clip.startUs
|
|
592
|
-
);
|
|
593
|
-
}
|
|
594
|
-
/**
|
|
595
|
-
* Decode audio samples to PCM and cache
|
|
596
|
-
* Uses AudioChunkDecoder for consistency with project architecture
|
|
597
|
-
* Resamples to AudioContext sample rate if needed for better quality
|
|
598
|
-
*/
|
|
599
|
-
async decodeAudioSamples(clipId, samples, config, clipDurationUs, clipStartUs) {
|
|
600
|
-
let description;
|
|
601
|
-
if (config.description) {
|
|
602
|
-
if (config.description instanceof ArrayBuffer) {
|
|
603
|
-
description = config.description;
|
|
604
|
-
} else if (ArrayBuffer.isView(config.description)) {
|
|
605
|
-
const view = config.description;
|
|
606
|
-
const newBuffer = new ArrayBuffer(view.byteLength);
|
|
607
|
-
new Uint8Array(newBuffer).set(
|
|
608
|
-
new Uint8Array(view.buffer, view.byteOffset, view.byteLength)
|
|
609
|
-
);
|
|
610
|
-
description = newBuffer;
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
const decoderConfig = {
|
|
614
|
-
codec: config.codec,
|
|
615
|
-
sampleRate: config.sampleRate,
|
|
616
|
-
numberOfChannels: config.numberOfChannels,
|
|
617
|
-
description
|
|
618
|
-
};
|
|
619
|
-
const decoder = new AudioChunkDecoder(`audio-${clipId}`, decoderConfig);
|
|
620
|
-
try {
|
|
621
|
-
const chunkStream = new ReadableStream({
|
|
622
|
-
start(controller) {
|
|
623
|
-
for (const sample of samples) {
|
|
624
|
-
controller.enqueue(sample);
|
|
625
|
-
}
|
|
626
|
-
controller.close();
|
|
627
|
-
}
|
|
628
|
-
});
|
|
629
|
-
const audioDataStream = chunkStream.pipeThrough(decoder.createStream());
|
|
630
|
-
const reader = audioDataStream.getReader();
|
|
631
|
-
try {
|
|
632
|
-
while (true) {
|
|
633
|
-
const { done, value } = await reader.read();
|
|
634
|
-
if (done) break;
|
|
635
|
-
if (value) {
|
|
636
|
-
const globalTimeUs = clipStartUs + (value.timestamp ?? 0);
|
|
637
|
-
this.deps.cacheManager.putClipAudioData(clipId, value, clipDurationUs, globalTimeUs);
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
} finally {
|
|
641
|
-
reader.releaseLock();
|
|
642
|
-
}
|
|
643
|
-
} catch (error) {
|
|
644
|
-
console.error(`[GlobalAudioSession] Decoder error for clip ${clipId}:`, error);
|
|
645
|
-
throw error;
|
|
646
|
-
} finally {
|
|
647
|
-
await decoder.close();
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
audioBufferToAudioData(buffer, timestampUs) {
|
|
651
|
-
const sampleRate = buffer.sampleRate;
|
|
652
|
-
const numberOfChannels = buffer.numberOfChannels;
|
|
653
|
-
const numberOfFrames = buffer.length;
|
|
654
|
-
const planes = [];
|
|
655
|
-
for (let channel = 0; channel < numberOfChannels; channel++) {
|
|
656
|
-
planes.push(buffer.getChannelData(channel));
|
|
657
|
-
}
|
|
658
|
-
return new AudioData({
|
|
659
|
-
format: "f32-planar",
|
|
660
|
-
sampleRate,
|
|
661
|
-
numberOfFrames,
|
|
662
|
-
numberOfChannels,
|
|
663
|
-
timestamp: timestampUs,
|
|
664
|
-
data: this.packPlanarF32Data(planes)
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
packPlanarF32Data(planes) {
|
|
668
|
-
const numberOfChannels = planes.length;
|
|
669
|
-
const numberOfFrames = planes[0]?.length ?? 0;
|
|
670
|
-
const totalSamples = numberOfChannels * numberOfFrames;
|
|
671
|
-
const packed = new Float32Array(totalSamples);
|
|
672
|
-
for (let channel = 0; channel < numberOfChannels; channel++) {
|
|
673
|
-
const plane = planes[channel];
|
|
674
|
-
if (!plane) continue;
|
|
675
|
-
packed.set(plane, channel * numberOfFrames);
|
|
676
|
-
}
|
|
677
|
-
return packed.buffer;
|
|
678
|
-
}
|
|
679
|
-
}
|
|
680
|
-
export {
|
|
681
|
-
GlobalAudioSession
|
|
682
|
-
};
|
|
683
|
-
//# sourceMappingURL=GlobalAudioSession.js.map
|