@meframe/core 0.0.17 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/controllers/PlaybackController.d.ts +1 -1
- package/dist/controllers/PlaybackController.d.ts.map +1 -1
- package/dist/controllers/PlaybackController.js.map +1 -1
- package/dist/{stages/compose → orchestrator}/GlobalAudioSession.d.ts +13 -7
- package/dist/orchestrator/GlobalAudioSession.d.ts.map +1 -0
- package/dist/{stages/compose → orchestrator}/GlobalAudioSession.js +56 -4
- package/dist/orchestrator/GlobalAudioSession.js.map +1 -0
- package/dist/orchestrator/Orchestrator.d.ts +2 -8
- package/dist/orchestrator/Orchestrator.d.ts.map +1 -1
- package/dist/orchestrator/Orchestrator.js +17 -9
- package/dist/orchestrator/Orchestrator.js.map +1 -1
- package/dist/orchestrator/VideoClipSession.d.ts.map +1 -1
- package/dist/orchestrator/VideoClipSession.js +15 -1
- package/dist/orchestrator/VideoClipSession.js.map +1 -1
- package/dist/stages/decode/AudioChunkDecoder.js +169 -0
- package/dist/stages/decode/AudioChunkDecoder.js.map +1 -0
- package/dist/stages/demux/types.d.ts +6 -0
- package/dist/stages/demux/types.d.ts.map +1 -1
- package/dist/stages/mux/MuxManager.d.ts +1 -1
- package/dist/stages/mux/MuxManager.d.ts.map +1 -1
- package/dist/stages/mux/MuxManager.js.map +1 -1
- package/dist/workers/stages/demux/video-demux.worker.js +92 -54
- package/dist/workers/stages/demux/video-demux.worker.js.map +1 -1
- package/package.json +1 -1
- package/dist/stages/compose/GlobalAudioSession.d.ts.map +0 -1
- package/dist/stages/compose/GlobalAudioSession.js.map +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { IPlaybackController, PlaybackOptions, IEventBus, PreviewHandle, TimeUs } from './types';
|
|
2
|
-
import { GlobalAudioSession } from '../
|
|
2
|
+
import { GlobalAudioSession } from '../orchestrator/GlobalAudioSession';
|
|
3
3
|
import { Orchestrator } from '../orchestrator';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PlaybackController.d.ts","sourceRoot":"","sources":["../../src/controllers/PlaybackController.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EAEnB,eAAe,EACf,SAAS,EACT,aAAa,EACb,MAAM,EACP,MAAM,SAAS,CAAC;AAEjB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"PlaybackController.d.ts","sourceRoot":"","sources":["../../src/controllers/PlaybackController.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EAEnB,eAAe,EACf,SAAS,EACT,aAAa,EACb,MAAM,EACP,MAAM,SAAS,CAAC;AAEjB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAGpD;;;GAGG;AACH,qBAAa,kBAAmB,YAAW,mBAAmB,EAAE,aAAa;IAC3E,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAY;IAC5B,OAAO,CAAC,MAAM,CAAsC;IACpD,OAAO,CAAC,GAAG,CAA+D;IAG1E,aAAa,EAAE,MAAM,CAAK;IAC1B,OAAO,CAAC,KAAK,CAAyB;IACtC,OAAO,CAAC,YAAY,CAAO;IAC3B,OAAO,CAAC,MAAM,CAAO;IACrB,OAAO,CAAC,IAAI,CAAS;IAGrB,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,SAAS,CAAK;IAGtB,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,GAAG,CAAK;IAChB,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,YAAY,CAAmC;IAGvD,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,oBAAoB,CAAS;gBAEzB,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe;IAyCrF,IAAI,IAAI,IAAI;YAOE,aAAa;IA6B3B,KAAK,IAAI,IAAI;IAgBb,IAAI,IAAI,IAAI;IAiBN,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CzC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAW3B,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAO/B,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAQ7B,OAAO,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI;IAI5B,IAAI,QAAQ,IAAI,MAAM,CAOrB;IAED,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,eAAe,CAAC,OAAO,EAAE,kBAAkB,GAAG,IAAI;IAKlD,MAAM,IAAI,IAAI;IAId,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI;IAIxD,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,OAAO,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI;IAIzD,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,YAAY;IA2CpB,OAAO,CAAC,UAAU;IAuBZ,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,UAAO,GAAG,OAAO,CAAC,IAAI,CAAC;YAsB3D,uBAAuB;IAsDrC,OAAO,CAAC,SAAS;IAKjB,OAAO,IAAI,IAAI;YAID,kBAAkB;CAOjC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PlaybackController.js","sources":["../../src/controllers/PlaybackController.ts"],"sourcesContent":["import type {\n IPlaybackController,\n PlaybackState,\n PlaybackOptions,\n IEventBus,\n PreviewHandle,\n TimeUs,\n} from './types';\nimport { MeframeEvent } from '../event/events';\nimport type { GlobalAudioSession } from '../stages/compose/GlobalAudioSession';\nimport type { Orchestrator } from '../orchestrator';\nimport { WaiterReplacedError } from '../utils/errors';\n\n/**\n * Playback controller for preview\n * Internal implementation - not exposed directly to external consumers\n */\nexport class PlaybackController implements IPlaybackController, PreviewHandle {\n private orchestrator: Orchestrator;\n private eventBus: IEventBus;\n private canvas: HTMLCanvasElement | OffscreenCanvas;\n private ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;\n\n // Playback state\n currentTimeUs: TimeUs = 0;\n private state: PlaybackState = 'idle';\n private playbackRate = 1.0;\n private volume = 1.0;\n private loop = false;\n\n // Animation loop\n private rafId: number | null = null;\n private startTime = 0;\n\n // Frame tracking\n private frameCount = 0;\n private lastFrameTime = 0;\n private fps = 0;\n private audioContext: AudioContext | null = null;\n private audioSession: GlobalAudioSession | null = null;\n\n // Buffering state\n private isBuffering = false;\n private currentSeekId = 0;\n private wasPlayingBeforeSeek = false;\n\n constructor(orchestrator: Orchestrator, eventBus: IEventBus, options: PlaybackOptions) {\n this.orchestrator = orchestrator;\n this.eventBus = eventBus;\n this.canvas = options.canvas;\n\n // Get 2D context with high quality settings\n const ctx = this.canvas.getContext('2d', {\n alpha: false,\n desynchronized: true,\n colorSpace: 'srgb',\n } as any);\n if (!ctx) {\n throw new Error('Failed to get 2D context from canvas');\n }\n this.ctx = ctx as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;\n\n // Configure high quality rendering\n this.ctx.imageSmoothingEnabled = true;\n this.ctx.imageSmoothingQuality = 'high';\n\n // Set initial time if provided\n if (options.startUs !== undefined) {\n this.currentTimeUs = options.startUs;\n }\n\n if (options.rate !== undefined) {\n this.playbackRate = options.rate;\n }\n\n if (options.loop !== undefined) {\n this.loop = options.loop;\n }\n\n if (options.autoStart) {\n this.play();\n }\n\n this.setupListeners();\n }\n\n // Playback control\n play(): void {\n if (this.state === 'playing') return;\n\n this.wasPlayingBeforeSeek = true; // User wants to play\n void this.startPlayback();\n }\n\n private async startPlayback(): Promise<void> {\n const wasIdle = this.state === 'idle';\n const seekId = this.currentSeekId;\n\n try {\n // Render first frame (may trigger buffering if cache miss)\n await this.renderCurrentFrame(this.currentTimeUs, true);\n\n // Check if seek happened during render\n if (seekId !== this.currentSeekId) {\n return;\n }\n\n this.state = 'playing';\n this.startTime = performance.now() - this.currentTimeUs / 1000 / this.playbackRate;\n await this.ensureAudioContext();\n if (this.audioSession && this.audioContext) {\n await this.audioSession.startPlayback(this.currentTimeUs, this.audioContext);\n }\n this.playbackLoop();\n\n this.eventBus.emit(MeframeEvent.PlaybackPlay);\n } catch (error) {\n console.error('[PlaybackController] Failed to start playback:', error);\n this.state = wasIdle ? 'idle' : 'paused';\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n }\n }\n\n pause(): void {\n if (this.state !== 'playing') return;\n\n this.state = 'paused';\n this.wasPlayingBeforeSeek = false; // User explicitly paused\n\n if (this.rafId !== null) {\n cancelAnimationFrame(this.rafId);\n this.rafId = null;\n }\n\n this.audioSession?.stopPlayback();\n\n this.eventBus.emit(MeframeEvent.PlaybackPause);\n }\n\n stop(): void {\n this.pause();\n this.currentTimeUs = 0;\n this.state = 'idle';\n this.wasPlayingBeforeSeek = false; // Reset seek state\n this.frameCount = 0; // Reset frame counter\n this.lastFrameTime = 0; // Reset frame timing\n this.fps = 0; // Reset FPS\n\n // Clear canvas\n this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n\n this.audioSession?.reset();\n\n this.eventBus.emit(MeframeEvent.PlaybackStop);\n }\n\n async seek(timeUs: TimeUs): Promise<void> {\n const previousState = this.state;\n\n // Stop playback without changing wasPlayingBeforeSeek\n if (this.rafId !== null) {\n cancelAnimationFrame(this.rafId);\n this.rafId = null;\n }\n this.audioSession?.stopPlayback();\n\n const clamped = this.clampTime(timeUs);\n this.currentTimeUs = clamped;\n this.currentSeekId++; // Invalidate previous seek operations\n this.isBuffering = false; // Reset buffering flag\n\n this.state = 'seeking';\n\n const seekId = this.currentSeekId;\n\n try {\n await this.renderCurrentFrame(clamped, false);\n\n // Check if another seek happened during render\n if (seekId !== this.currentSeekId) {\n return;\n }\n\n this.eventBus.emit(MeframeEvent.PlaybackSeek, { timeUs: this.currentTimeUs });\n\n if (this.wasPlayingBeforeSeek) {\n await this.startPlayback();\n } else {\n this.state = previousState === 'idle' ? 'idle' : 'paused';\n }\n } catch (error) {\n // Check if this seek is still current\n if (seekId !== this.currentSeekId) {\n return;\n }\n console.error('[PlaybackController] Seek error:', error);\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n this.state = previousState === 'idle' ? 'idle' : 'paused';\n }\n }\n\n // Playback properties\n setRate(rate: number): void {\n // Adjust start time to maintain current position\n const elapsed = performance.now() - this.startTime;\n this.playbackRate = rate;\n this.startTime = performance.now() - elapsed / rate;\n\n this.eventBus.emit(MeframeEvent.PlaybackRateChange, { rate });\n\n this.audioSession?.setPlaybackRate(this.playbackRate);\n }\n\n setVolume(volume: number): void {\n this.volume = Math.max(0, Math.min(1, volume));\n this.eventBus.emit(MeframeEvent.PlaybackVolumeChange, { volume: this.volume });\n\n this.audioSession?.setVolume(this.volume);\n }\n\n setMute(muted: boolean): void {\n if (muted) {\n this.audioSession?.stopPlayback();\n } else if (this.state === 'playing' && this.audioContext) {\n this.audioSession?.startPlayback(this.currentTimeUs, this.audioContext);\n }\n }\n\n setLoop(loop: boolean): void {\n this.loop = loop;\n }\n\n get duration(): TimeUs {\n const modelDuration = this.orchestrator.compositionModel?.durationUs;\n if (modelDuration !== undefined) {\n return modelDuration;\n }\n\n return 0;\n }\n\n get isPlaying(): boolean {\n return this.state === 'playing';\n }\n\n setAudioSession(session: GlobalAudioSession): void {\n this.audioSession = session;\n }\n\n // Resume is just an alias for play\n resume(): void {\n this.play();\n }\n\n on(event: string, handler: (payload: any) => void): void {\n this.eventBus.on(event as MeframeEvent, handler);\n }\n\n off(event: string, handler: (payload: any) => void): void {\n this.eventBus.off(event as MeframeEvent, handler);\n }\n\n private setupListeners(): void {\n this.orchestrator.on(MeframeEvent.CacheCover, (event) => {\n // Only render cover in idle/paused state, not during playback\n if (this.state === 'playing' || this.state === 'buffering') {\n return;\n }\n // CacheCover event now contains global timeUs\n this.renderCurrentFrame(event.timeUs);\n });\n }\n\n // Private methods\n private playbackLoop(): void {\n // Only continue loop if actively playing (not buffering/paused/etc)\n if (this.state !== 'playing') {\n if (this.rafId !== null) {\n cancelAnimationFrame(this.rafId);\n this.rafId = null;\n }\n return;\n }\n\n this.rafId = requestAnimationFrame(async () => {\n // Check state again after async boundary\n if (this.state !== 'playing') {\n return;\n }\n\n this.updateTime();\n\n // Update audio clips based on current time\n this.audioSession?.updateTime(this.currentTimeUs);\n\n await this.renderCurrentFrame(this.currentTimeUs, true);\n\n // Check if still playing after render (might have entered buffering)\n if (this.state !== 'playing') {\n return;\n }\n\n // Calculate FPS based on actual frame timing\n const now = performance.now();\n if (this.lastFrameTime > 0) {\n const deltaTime = now - this.lastFrameTime;\n const instantFps = 1000 / deltaTime;\n this.fps = this.fps > 0 ? this.fps * 0.9 + instantFps * 0.1 : instantFps;\n }\n this.lastFrameTime = now;\n\n this.frameCount++;\n\n this.playbackLoop();\n });\n }\n\n private updateTime(): void {\n const elapsed = (performance.now() - this.startTime) * this.playbackRate;\n const rawTimeUs = elapsed * 1000;\n // Use continuous time instead of quantizing to frame boundaries\n // This ensures smooth time progression and prevents stalling at clip boundaries\n this.currentTimeUs = rawTimeUs;\n // Check if reached end\n if (this.currentTimeUs >= this.duration) {\n if (this.loop) {\n this.currentTimeUs = 0;\n this.startTime = performance.now();\n } else {\n this.currentTimeUs = this.duration;\n this.pause();\n this.state = 'ended';\n this.eventBus.emit(MeframeEvent.PlaybackEnded, { timeUs: this.currentTimeUs });\n }\n }\n\n // Emit time update\n this.eventBus.emit(MeframeEvent.PlaybackTimeUpdate, { timeUs: this.currentTimeUs });\n }\n\n async renderCurrentFrame(timeUs: TimeUs, immediate = true): Promise<void> {\n try {\n const rcFrame = await this.orchestrator.renderFrame(timeUs, { immediate });\n if (!rcFrame) {\n // Cache miss during playback - trigger buffering\n if (this.state === 'playing') {\n await this.handlePlaybackBuffering(timeUs);\n }\n return;\n }\n await rcFrame.use((frame) => {\n // Ensure high quality rendering for every frame\n this.ctx.imageSmoothingEnabled = true;\n this.ctx.imageSmoothingQuality = 'high';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n });\n } catch (error) {\n console.error('Render error:', error);\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n }\n }\n\n private async handlePlaybackBuffering(timeUs: TimeUs): Promise<void> {\n if (this.isBuffering) {\n return;\n }\n\n const wasPlaying = this.state === 'playing';\n if (!wasPlaying) return;\n\n const seekId = this.currentSeekId;\n this.isBuffering = true;\n this.state = 'buffering';\n this.eventBus.emit(MeframeEvent.PlaybackBuffering);\n\n try {\n // await this.orchestrator.ensureClipCache(timeUs);\n\n const ready = await this.orchestrator.waitForClipReady(timeUs, {\n minFrameCount: 3, // 等待 3 帧,确保连续播放不会立即再次 miss\n timeoutMs: 5_000,\n });\n\n // Check if seek happened during buffering\n if (seekId !== this.currentSeekId) {\n return;\n }\n\n if (!ready) {\n console.warn('[PlaybackController] Buffering timeout during playback', timeUs);\n }\n\n this.state = 'playing';\n this.startTime = performance.now() - timeUs / 1000 / this.playbackRate;\n this.eventBus.emit(MeframeEvent.PlaybackPlay);\n\n if (!this.rafId) {\n this.playbackLoop();\n }\n } catch (error) {\n // Ignore WaiterReplacedError (happens during fast seeks)\n if (error instanceof WaiterReplacedError) {\n return;\n }\n // Check if seek happened during error handling\n if (seekId !== this.currentSeekId) {\n return;\n }\n console.error('[PlaybackController] Buffering error:', error);\n this.state = 'paused';\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n } finally {\n this.isBuffering = false;\n }\n }\n\n private clampTime(timeUs: TimeUs): TimeUs {\n return Math.max(0, Math.min(timeUs, this.duration));\n }\n\n // Cleanup\n dispose(): void {\n this.stop();\n }\n\n private async ensureAudioContext(): Promise<void> {\n if (this.audioContext) {\n return;\n }\n\n this.audioContext = new AudioContext();\n }\n}\n"],"names":[],"mappings":";;AAiBO,MAAM,mBAAiE;AAAA,EACpE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR,gBAAwB;AAAA,EAChB,QAAuB;AAAA,EACvB,eAAe;AAAA,EACf,SAAS;AAAA,EACT,OAAO;AAAA;AAAA,EAGP,QAAuB;AAAA,EACvB,YAAY;AAAA;AAAA,EAGZ,aAAa;AAAA,EACb,gBAAgB;AAAA,EAChB,MAAM;AAAA,EACN,eAAoC;AAAA,EACpC,eAA0C;AAAA;AAAA,EAG1C,cAAc;AAAA,EACd,gBAAgB;AAAA,EAChB,uBAAuB;AAAA,EAE/B,YAAY,cAA4B,UAAqB,SAA0B;AACrF,SAAK,eAAe;AACpB,SAAK,WAAW;AAChB,SAAK,SAAS,QAAQ;AAGtB,UAAM,MAAM,KAAK,OAAO,WAAW,MAAM;AAAA,MACvC,OAAO;AAAA,MACP,gBAAgB;AAAA,MAChB,YAAY;AAAA,IAAA,CACN;AACR,QAAI,CAAC,KAAK;AACR,YAAM,IAAI,MAAM,sCAAsC;AAAA,IACxD;AACA,SAAK,MAAM;AAGX,SAAK,IAAI,wBAAwB;AACjC,SAAK,IAAI,wBAAwB;AAGjC,QAAI,QAAQ,YAAY,QAAW;AACjC,WAAK,gBAAgB,QAAQ;AAAA,IAC/B;AAEA,QAAI,QAAQ,SAAS,QAAW;AAC9B,WAAK,eAAe,QAAQ;AAAA,IAC9B;AAEA,QAAI,QAAQ,SAAS,QAAW;AAC9B,WAAK,OAAO,QAAQ;AAAA,IACtB;AAEA,QAAI,QAAQ,WAAW;AACrB,WAAK,KAAA;AAAA,IACP;AAEA,SAAK,eAAA;AAAA,EACP;AAAA;AAAA,EAGA,OAAa;AACX,QAAI,KAAK,UAAU,UAAW;AAE9B,SAAK,uBAAuB;AAC5B,SAAK,KAAK,cAAA;AAAA,EACZ;AAAA,EAEA,MAAc,gBAA+B;AAC3C,UAAM,UAAU,KAAK,UAAU;AAC/B,UAAM,SAAS,KAAK;AAEpB,QAAI;AAEF,YAAM,KAAK,mBAAmB,KAAK,eAAe,IAAI;AAGtD,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AAEA,WAAK,QAAQ;AACb,WAAK,YAAY,YAAY,IAAA,IAAQ,KAAK,gBAAgB,MAAO,KAAK;AACtE,YAAM,KAAK,mBAAA;AACX,UAAI,KAAK,gBAAgB,KAAK,cAAc;AAC1C,cAAM,KAAK,aAAa,cAAc,KAAK,eAAe,KAAK,YAAY;AAAA,MAC7E;AACA,WAAK,aAAA;AAEL,WAAK,SAAS,KAAK,aAAa,YAAY;AAAA,IAC9C,SAAS,OAAO;AACd,cAAQ,MAAM,kDAAkD,KAAK;AACrE,WAAK,QAAQ,UAAU,SAAS;AAChC,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,QAAc;AACZ,QAAI,KAAK,UAAU,UAAW;AAE9B,SAAK,QAAQ;AACb,SAAK,uBAAuB;AAE5B,QAAI,KAAK,UAAU,MAAM;AACvB,2BAAqB,KAAK,KAAK;AAC/B,WAAK,QAAQ;AAAA,IACf;AAEA,SAAK,cAAc,aAAA;AAEnB,SAAK,SAAS,KAAK,aAAa,aAAa;AAAA,EAC/C;AAAA,EAEA,OAAa;AACX,SAAK,MAAA;AACL,SAAK,gBAAgB;AACrB,SAAK,QAAQ;AACb,SAAK,uBAAuB;AAC5B,SAAK,aAAa;AAClB,SAAK,gBAAgB;AACrB,SAAK,MAAM;AAGX,SAAK,IAAI,UAAU,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAE9D,SAAK,cAAc,MAAA;AAEnB,SAAK,SAAS,KAAK,aAAa,YAAY;AAAA,EAC9C;AAAA,EAEA,MAAM,KAAK,QAA+B;AACxC,UAAM,gBAAgB,KAAK;AAG3B,QAAI,KAAK,UAAU,MAAM;AACvB,2BAAqB,KAAK,KAAK;AAC/B,WAAK,QAAQ;AAAA,IACf;AACA,SAAK,cAAc,aAAA;AAEnB,UAAM,UAAU,KAAK,UAAU,MAAM;AACrC,SAAK,gBAAgB;AACrB,SAAK;AACL,SAAK,cAAc;AAEnB,SAAK,QAAQ;AAEb,UAAM,SAAS,KAAK;AAEpB,QAAI;AACF,YAAM,KAAK,mBAAmB,SAAS,KAAK;AAG5C,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AAEA,WAAK,SAAS,KAAK,aAAa,cAAc,EAAE,QAAQ,KAAK,eAAe;AAE5E,UAAI,KAAK,sBAAsB;AAC7B,cAAM,KAAK,cAAA;AAAA,MACb,OAAO;AACL,aAAK,QAAQ,kBAAkB,SAAS,SAAS;AAAA,MACnD;AAAA,IACF,SAAS,OAAO;AAEd,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AACA,cAAQ,MAAM,oCAAoC,KAAK;AACvD,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAC7D,WAAK,QAAQ,kBAAkB,SAAS,SAAS;AAAA,IACnD;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ,MAAoB;AAE1B,UAAM,UAAU,YAAY,IAAA,IAAQ,KAAK;AACzC,SAAK,eAAe;AACpB,SAAK,YAAY,YAAY,IAAA,IAAQ,UAAU;AAE/C,SAAK,SAAS,KAAK,aAAa,oBAAoB,EAAE,MAAM;AAE5D,SAAK,cAAc,gBAAgB,KAAK,YAAY;AAAA,EACtD;AAAA,EAEA,UAAU,QAAsB;AAC9B,SAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;AAC7C,SAAK,SAAS,KAAK,aAAa,sBAAsB,EAAE,QAAQ,KAAK,QAAQ;AAE7E,SAAK,cAAc,UAAU,KAAK,MAAM;AAAA,EAC1C;AAAA,EAEA,QAAQ,OAAsB;AAC5B,QAAI,OAAO;AACT,WAAK,cAAc,aAAA;AAAA,IACrB,WAAW,KAAK,UAAU,aAAa,KAAK,cAAc;AACxD,WAAK,cAAc,cAAc,KAAK,eAAe,KAAK,YAAY;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,QAAQ,MAAqB;AAC3B,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,IAAI,WAAmB;AACrB,UAAM,gBAAgB,KAAK,aAAa,kBAAkB;AAC1D,QAAI,kBAAkB,QAAW;AAC/B,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA,EAEA,gBAAgB,SAAmC;AACjD,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,SAAe;AACb,SAAK,KAAA;AAAA,EACP;AAAA,EAEA,GAAG,OAAe,SAAuC;AACvD,SAAK,SAAS,GAAG,OAAuB,OAAO;AAAA,EACjD;AAAA,EAEA,IAAI,OAAe,SAAuC;AACxD,SAAK,SAAS,IAAI,OAAuB,OAAO;AAAA,EAClD;AAAA,EAEQ,iBAAuB;AAC7B,SAAK,aAAa,GAAG,aAAa,YAAY,CAAC,UAAU;AAEvD,UAAI,KAAK,UAAU,aAAa,KAAK,UAAU,aAAa;AAC1D;AAAA,MACF;AAEA,WAAK,mBAAmB,MAAM,MAAM;AAAA,IACtC,CAAC;AAAA,EACH;AAAA;AAAA,EAGQ,eAAqB;AAE3B,QAAI,KAAK,UAAU,WAAW;AAC5B,UAAI,KAAK,UAAU,MAAM;AACvB,6BAAqB,KAAK,KAAK;AAC/B,aAAK,QAAQ;AAAA,MACf;AACA;AAAA,IACF;AAEA,SAAK,QAAQ,sBAAsB,YAAY;AAE7C,UAAI,KAAK,UAAU,WAAW;AAC5B;AAAA,MACF;AAEA,WAAK,WAAA;AAGL,WAAK,cAAc,WAAW,KAAK,aAAa;AAEhD,YAAM,KAAK,mBAAmB,KAAK,eAAe,IAAI;AAGtD,UAAI,KAAK,UAAU,WAAW;AAC5B;AAAA,MACF;AAGA,YAAM,MAAM,YAAY,IAAA;AACxB,UAAI,KAAK,gBAAgB,GAAG;AAC1B,cAAM,YAAY,MAAM,KAAK;AAC7B,cAAM,aAAa,MAAO;AAC1B,aAAK,MAAM,KAAK,MAAM,IAAI,KAAK,MAAM,MAAM,aAAa,MAAM;AAAA,MAChE;AACA,WAAK,gBAAgB;AAErB,WAAK;AAEL,WAAK,aAAA;AAAA,IACP,CAAC;AAAA,EACH;AAAA,EAEQ,aAAmB;AACzB,UAAM,WAAW,YAAY,IAAA,IAAQ,KAAK,aAAa,KAAK;AAC5D,UAAM,YAAY,UAAU;AAG5B,SAAK,gBAAgB;AAErB,QAAI,KAAK,iBAAiB,KAAK,UAAU;AACvC,UAAI,KAAK,MAAM;AACb,aAAK,gBAAgB;AACrB,aAAK,YAAY,YAAY,IAAA;AAAA,MAC/B,OAAO;AACL,aAAK,gBAAgB,KAAK;AAC1B,aAAK,MAAA;AACL,aAAK,QAAQ;AACb,aAAK,SAAS,KAAK,aAAa,eAAe,EAAE,QAAQ,KAAK,eAAe;AAAA,MAC/E;AAAA,IACF;AAGA,SAAK,SAAS,KAAK,aAAa,oBAAoB,EAAE,QAAQ,KAAK,eAAe;AAAA,EACpF;AAAA,EAEA,MAAM,mBAAmB,QAAgB,YAAY,MAAqB;AACxE,QAAI;AACF,YAAM,UAAU,MAAM,KAAK,aAAa,YAAY,QAAQ,EAAE,WAAW;AACzE,UAAI,CAAC,SAAS;AAEZ,YAAI,KAAK,UAAU,WAAW;AAC5B,gBAAM,KAAK,wBAAwB,MAAM;AAAA,QAC3C;AACA;AAAA,MACF;AACA,YAAM,QAAQ,IAAI,CAAC,UAAU;AAE3B,aAAK,IAAI,wBAAwB;AACjC,aAAK,IAAI,wBAAwB;AACjC,aAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MACvE,CAAC;AAAA,IACH,SAAS,OAAO;AACd,cAAQ,MAAM,iBAAiB,KAAK;AACpC,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,MAAc,wBAAwB,QAA+B;AACnE,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AAEA,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,CAAC,WAAY;AAEjB,UAAM,SAAS,KAAK;AACpB,SAAK,cAAc;AACnB,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,aAAa,iBAAiB;AAEjD,QAAI;AAGF,YAAM,QAAQ,MAAM,KAAK,aAAa,iBAAiB,QAAQ;AAAA,QAC7D,eAAe;AAAA;AAAA,QACf,WAAW;AAAA,MAAA,CACZ;AAGD,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AAEA,UAAI,CAAC,OAAO;AACV,gBAAQ,KAAK,0DAA0D,MAAM;AAAA,MAC/E;AAEA,WAAK,QAAQ;AACb,WAAK,YAAY,YAAY,IAAA,IAAQ,SAAS,MAAO,KAAK;AAC1D,WAAK,SAAS,KAAK,aAAa,YAAY;AAE5C,UAAI,CAAC,KAAK,OAAO;AACf,aAAK,aAAA;AAAA,MACP;AAAA,IACF,SAAS,OAAO;AAEd,UAAI,iBAAiB,qBAAqB;AACxC;AAAA,MACF;AAEA,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AACA,cAAQ,MAAM,yCAAyC,KAAK;AAC5D,WAAK,QAAQ;AACb,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAAA,IAC/D,UAAA;AACE,WAAK,cAAc;AAAA,IACrB;AAAA,EACF;AAAA,EAEQ,UAAU,QAAwB;AACxC,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,KAAK,QAAQ,CAAC;AAAA,EACpD;AAAA;AAAA,EAGA,UAAgB;AACd,SAAK,KAAA;AAAA,EACP;AAAA,EAEA,MAAc,qBAAoC;AAChD,QAAI,KAAK,cAAc;AACrB;AAAA,IACF;AAEA,SAAK,eAAe,IAAI,aAAA;AAAA,EAC1B;AACF;"}
|
|
1
|
+
{"version":3,"file":"PlaybackController.js","sources":["../../src/controllers/PlaybackController.ts"],"sourcesContent":["import type {\n IPlaybackController,\n PlaybackState,\n PlaybackOptions,\n IEventBus,\n PreviewHandle,\n TimeUs,\n} from './types';\nimport { MeframeEvent } from '../event/events';\nimport type { GlobalAudioSession } from '../orchestrator/GlobalAudioSession';\nimport type { Orchestrator } from '../orchestrator';\nimport { WaiterReplacedError } from '../utils/errors';\n\n/**\n * Playback controller for preview\n * Internal implementation - not exposed directly to external consumers\n */\nexport class PlaybackController implements IPlaybackController, PreviewHandle {\n private orchestrator: Orchestrator;\n private eventBus: IEventBus;\n private canvas: HTMLCanvasElement | OffscreenCanvas;\n private ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;\n\n // Playback state\n currentTimeUs: TimeUs = 0;\n private state: PlaybackState = 'idle';\n private playbackRate = 1.0;\n private volume = 1.0;\n private loop = false;\n\n // Animation loop\n private rafId: number | null = null;\n private startTime = 0;\n\n // Frame tracking\n private frameCount = 0;\n private lastFrameTime = 0;\n private fps = 0;\n private audioContext: AudioContext | null = null;\n private audioSession: GlobalAudioSession | null = null;\n\n // Buffering state\n private isBuffering = false;\n private currentSeekId = 0;\n private wasPlayingBeforeSeek = false;\n\n constructor(orchestrator: Orchestrator, eventBus: IEventBus, options: PlaybackOptions) {\n this.orchestrator = orchestrator;\n this.eventBus = eventBus;\n this.canvas = options.canvas;\n\n // Get 2D context with high quality settings\n const ctx = this.canvas.getContext('2d', {\n alpha: false,\n desynchronized: true,\n colorSpace: 'srgb',\n } as any);\n if (!ctx) {\n throw new Error('Failed to get 2D context from canvas');\n }\n this.ctx = ctx as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;\n\n // Configure high quality rendering\n this.ctx.imageSmoothingEnabled = true;\n this.ctx.imageSmoothingQuality = 'high';\n\n // Set initial time if provided\n if (options.startUs !== undefined) {\n this.currentTimeUs = options.startUs;\n }\n\n if (options.rate !== undefined) {\n this.playbackRate = options.rate;\n }\n\n if (options.loop !== undefined) {\n this.loop = options.loop;\n }\n\n if (options.autoStart) {\n this.play();\n }\n\n this.setupListeners();\n }\n\n // Playback control\n play(): void {\n if (this.state === 'playing') return;\n\n this.wasPlayingBeforeSeek = true; // User wants to play\n void this.startPlayback();\n }\n\n private async startPlayback(): Promise<void> {\n const wasIdle = this.state === 'idle';\n const seekId = this.currentSeekId;\n\n try {\n // Render first frame (may trigger buffering if cache miss)\n await this.renderCurrentFrame(this.currentTimeUs, true);\n\n // Check if seek happened during render\n if (seekId !== this.currentSeekId) {\n return;\n }\n\n this.state = 'playing';\n this.startTime = performance.now() - this.currentTimeUs / 1000 / this.playbackRate;\n await this.ensureAudioContext();\n if (this.audioSession && this.audioContext) {\n await this.audioSession.startPlayback(this.currentTimeUs, this.audioContext);\n }\n this.playbackLoop();\n\n this.eventBus.emit(MeframeEvent.PlaybackPlay);\n } catch (error) {\n console.error('[PlaybackController] Failed to start playback:', error);\n this.state = wasIdle ? 'idle' : 'paused';\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n }\n }\n\n pause(): void {\n if (this.state !== 'playing') return;\n\n this.state = 'paused';\n this.wasPlayingBeforeSeek = false; // User explicitly paused\n\n if (this.rafId !== null) {\n cancelAnimationFrame(this.rafId);\n this.rafId = null;\n }\n\n this.audioSession?.stopPlayback();\n\n this.eventBus.emit(MeframeEvent.PlaybackPause);\n }\n\n stop(): void {\n this.pause();\n this.currentTimeUs = 0;\n this.state = 'idle';\n this.wasPlayingBeforeSeek = false; // Reset seek state\n this.frameCount = 0; // Reset frame counter\n this.lastFrameTime = 0; // Reset frame timing\n this.fps = 0; // Reset FPS\n\n // Clear canvas\n this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);\n\n this.audioSession?.reset();\n\n this.eventBus.emit(MeframeEvent.PlaybackStop);\n }\n\n async seek(timeUs: TimeUs): Promise<void> {\n const previousState = this.state;\n\n // Stop playback without changing wasPlayingBeforeSeek\n if (this.rafId !== null) {\n cancelAnimationFrame(this.rafId);\n this.rafId = null;\n }\n this.audioSession?.stopPlayback();\n\n const clamped = this.clampTime(timeUs);\n this.currentTimeUs = clamped;\n this.currentSeekId++; // Invalidate previous seek operations\n this.isBuffering = false; // Reset buffering flag\n\n this.state = 'seeking';\n\n const seekId = this.currentSeekId;\n\n try {\n await this.renderCurrentFrame(clamped, false);\n\n // Check if another seek happened during render\n if (seekId !== this.currentSeekId) {\n return;\n }\n\n this.eventBus.emit(MeframeEvent.PlaybackSeek, { timeUs: this.currentTimeUs });\n\n if (this.wasPlayingBeforeSeek) {\n await this.startPlayback();\n } else {\n this.state = previousState === 'idle' ? 'idle' : 'paused';\n }\n } catch (error) {\n // Check if this seek is still current\n if (seekId !== this.currentSeekId) {\n return;\n }\n console.error('[PlaybackController] Seek error:', error);\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n this.state = previousState === 'idle' ? 'idle' : 'paused';\n }\n }\n\n // Playback properties\n setRate(rate: number): void {\n // Adjust start time to maintain current position\n const elapsed = performance.now() - this.startTime;\n this.playbackRate = rate;\n this.startTime = performance.now() - elapsed / rate;\n\n this.eventBus.emit(MeframeEvent.PlaybackRateChange, { rate });\n\n this.audioSession?.setPlaybackRate(this.playbackRate);\n }\n\n setVolume(volume: number): void {\n this.volume = Math.max(0, Math.min(1, volume));\n this.eventBus.emit(MeframeEvent.PlaybackVolumeChange, { volume: this.volume });\n\n this.audioSession?.setVolume(this.volume);\n }\n\n setMute(muted: boolean): void {\n if (muted) {\n this.audioSession?.stopPlayback();\n } else if (this.state === 'playing' && this.audioContext) {\n this.audioSession?.startPlayback(this.currentTimeUs, this.audioContext);\n }\n }\n\n setLoop(loop: boolean): void {\n this.loop = loop;\n }\n\n get duration(): TimeUs {\n const modelDuration = this.orchestrator.compositionModel?.durationUs;\n if (modelDuration !== undefined) {\n return modelDuration;\n }\n\n return 0;\n }\n\n get isPlaying(): boolean {\n return this.state === 'playing';\n }\n\n setAudioSession(session: GlobalAudioSession): void {\n this.audioSession = session;\n }\n\n // Resume is just an alias for play\n resume(): void {\n this.play();\n }\n\n on(event: string, handler: (payload: any) => void): void {\n this.eventBus.on(event as MeframeEvent, handler);\n }\n\n off(event: string, handler: (payload: any) => void): void {\n this.eventBus.off(event as MeframeEvent, handler);\n }\n\n private setupListeners(): void {\n this.orchestrator.on(MeframeEvent.CacheCover, (event) => {\n // Only render cover in idle/paused state, not during playback\n if (this.state === 'playing' || this.state === 'buffering') {\n return;\n }\n // CacheCover event now contains global timeUs\n this.renderCurrentFrame(event.timeUs);\n });\n }\n\n // Private methods\n private playbackLoop(): void {\n // Only continue loop if actively playing (not buffering/paused/etc)\n if (this.state !== 'playing') {\n if (this.rafId !== null) {\n cancelAnimationFrame(this.rafId);\n this.rafId = null;\n }\n return;\n }\n\n this.rafId = requestAnimationFrame(async () => {\n // Check state again after async boundary\n if (this.state !== 'playing') {\n return;\n }\n\n this.updateTime();\n\n // Update audio clips based on current time\n this.audioSession?.updateTime(this.currentTimeUs);\n\n await this.renderCurrentFrame(this.currentTimeUs, true);\n\n // Check if still playing after render (might have entered buffering)\n if (this.state !== 'playing') {\n return;\n }\n\n // Calculate FPS based on actual frame timing\n const now = performance.now();\n if (this.lastFrameTime > 0) {\n const deltaTime = now - this.lastFrameTime;\n const instantFps = 1000 / deltaTime;\n this.fps = this.fps > 0 ? this.fps * 0.9 + instantFps * 0.1 : instantFps;\n }\n this.lastFrameTime = now;\n\n this.frameCount++;\n\n this.playbackLoop();\n });\n }\n\n private updateTime(): void {\n const elapsed = (performance.now() - this.startTime) * this.playbackRate;\n const rawTimeUs = elapsed * 1000;\n // Use continuous time instead of quantizing to frame boundaries\n // This ensures smooth time progression and prevents stalling at clip boundaries\n this.currentTimeUs = rawTimeUs;\n // Check if reached end\n if (this.currentTimeUs >= this.duration) {\n if (this.loop) {\n this.currentTimeUs = 0;\n this.startTime = performance.now();\n } else {\n this.currentTimeUs = this.duration;\n this.pause();\n this.state = 'ended';\n this.eventBus.emit(MeframeEvent.PlaybackEnded, { timeUs: this.currentTimeUs });\n }\n }\n\n // Emit time update\n this.eventBus.emit(MeframeEvent.PlaybackTimeUpdate, { timeUs: this.currentTimeUs });\n }\n\n async renderCurrentFrame(timeUs: TimeUs, immediate = true): Promise<void> {\n try {\n const rcFrame = await this.orchestrator.renderFrame(timeUs, { immediate });\n if (!rcFrame) {\n // Cache miss during playback - trigger buffering\n if (this.state === 'playing') {\n await this.handlePlaybackBuffering(timeUs);\n }\n return;\n }\n await rcFrame.use((frame) => {\n // Ensure high quality rendering for every frame\n this.ctx.imageSmoothingEnabled = true;\n this.ctx.imageSmoothingQuality = 'high';\n this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);\n });\n } catch (error) {\n console.error('Render error:', error);\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n }\n }\n\n private async handlePlaybackBuffering(timeUs: TimeUs): Promise<void> {\n if (this.isBuffering) {\n return;\n }\n\n const wasPlaying = this.state === 'playing';\n if (!wasPlaying) return;\n\n const seekId = this.currentSeekId;\n this.isBuffering = true;\n this.state = 'buffering';\n this.eventBus.emit(MeframeEvent.PlaybackBuffering);\n\n try {\n // await this.orchestrator.ensureClipCache(timeUs);\n\n const ready = await this.orchestrator.waitForClipReady(timeUs, {\n minFrameCount: 3, // 等待 3 帧,确保连续播放不会立即再次 miss\n timeoutMs: 5_000,\n });\n\n // Check if seek happened during buffering\n if (seekId !== this.currentSeekId) {\n return;\n }\n\n if (!ready) {\n console.warn('[PlaybackController] Buffering timeout during playback', timeUs);\n }\n\n this.state = 'playing';\n this.startTime = performance.now() - timeUs / 1000 / this.playbackRate;\n this.eventBus.emit(MeframeEvent.PlaybackPlay);\n\n if (!this.rafId) {\n this.playbackLoop();\n }\n } catch (error) {\n // Ignore WaiterReplacedError (happens during fast seeks)\n if (error instanceof WaiterReplacedError) {\n return;\n }\n // Check if seek happened during error handling\n if (seekId !== this.currentSeekId) {\n return;\n }\n console.error('[PlaybackController] Buffering error:', error);\n this.state = 'paused';\n this.eventBus.emit(MeframeEvent.PlaybackError, error as Error);\n } finally {\n this.isBuffering = false;\n }\n }\n\n private clampTime(timeUs: TimeUs): TimeUs {\n return Math.max(0, Math.min(timeUs, this.duration));\n }\n\n // Cleanup\n dispose(): void {\n this.stop();\n }\n\n private async ensureAudioContext(): Promise<void> {\n if (this.audioContext) {\n return;\n }\n\n this.audioContext = new AudioContext();\n }\n}\n"],"names":[],"mappings":";;AAiBO,MAAM,mBAAiE;AAAA,EACpE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAGR,gBAAwB;AAAA,EAChB,QAAuB;AAAA,EACvB,eAAe;AAAA,EACf,SAAS;AAAA,EACT,OAAO;AAAA;AAAA,EAGP,QAAuB;AAAA,EACvB,YAAY;AAAA;AAAA,EAGZ,aAAa;AAAA,EACb,gBAAgB;AAAA,EAChB,MAAM;AAAA,EACN,eAAoC;AAAA,EACpC,eAA0C;AAAA;AAAA,EAG1C,cAAc;AAAA,EACd,gBAAgB;AAAA,EAChB,uBAAuB;AAAA,EAE/B,YAAY,cAA4B,UAAqB,SAA0B;AACrF,SAAK,eAAe;AACpB,SAAK,WAAW;AAChB,SAAK,SAAS,QAAQ;AAGtB,UAAM,MAAM,KAAK,OAAO,WAAW,MAAM;AAAA,MACvC,OAAO;AAAA,MACP,gBAAgB;AAAA,MAChB,YAAY;AAAA,IAAA,CACN;AACR,QAAI,CAAC,KAAK;AACR,YAAM,IAAI,MAAM,sCAAsC;AAAA,IACxD;AACA,SAAK,MAAM;AAGX,SAAK,IAAI,wBAAwB;AACjC,SAAK,IAAI,wBAAwB;AAGjC,QAAI,QAAQ,YAAY,QAAW;AACjC,WAAK,gBAAgB,QAAQ;AAAA,IAC/B;AAEA,QAAI,QAAQ,SAAS,QAAW;AAC9B,WAAK,eAAe,QAAQ;AAAA,IAC9B;AAEA,QAAI,QAAQ,SAAS,QAAW;AAC9B,WAAK,OAAO,QAAQ;AAAA,IACtB;AAEA,QAAI,QAAQ,WAAW;AACrB,WAAK,KAAA;AAAA,IACP;AAEA,SAAK,eAAA;AAAA,EACP;AAAA;AAAA,EAGA,OAAa;AACX,QAAI,KAAK,UAAU,UAAW;AAE9B,SAAK,uBAAuB;AAC5B,SAAK,KAAK,cAAA;AAAA,EACZ;AAAA,EAEA,MAAc,gBAA+B;AAC3C,UAAM,UAAU,KAAK,UAAU;AAC/B,UAAM,SAAS,KAAK;AAEpB,QAAI;AAEF,YAAM,KAAK,mBAAmB,KAAK,eAAe,IAAI;AAGtD,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AAEA,WAAK,QAAQ;AACb,WAAK,YAAY,YAAY,IAAA,IAAQ,KAAK,gBAAgB,MAAO,KAAK;AACtE,YAAM,KAAK,mBAAA;AACX,UAAI,KAAK,gBAAgB,KAAK,cAAc;AAC1C,cAAM,KAAK,aAAa,cAAc,KAAK,eAAe,KAAK,YAAY;AAAA,MAC7E;AACA,WAAK,aAAA;AAEL,WAAK,SAAS,KAAK,aAAa,YAAY;AAAA,IAC9C,SAAS,OAAO;AACd,cAAQ,MAAM,kDAAkD,KAAK;AACrE,WAAK,QAAQ,UAAU,SAAS;AAChC,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,QAAc;AACZ,QAAI,KAAK,UAAU,UAAW;AAE9B,SAAK,QAAQ;AACb,SAAK,uBAAuB;AAE5B,QAAI,KAAK,UAAU,MAAM;AACvB,2BAAqB,KAAK,KAAK;AAC/B,WAAK,QAAQ;AAAA,IACf;AAEA,SAAK,cAAc,aAAA;AAEnB,SAAK,SAAS,KAAK,aAAa,aAAa;AAAA,EAC/C;AAAA,EAEA,OAAa;AACX,SAAK,MAAA;AACL,SAAK,gBAAgB;AACrB,SAAK,QAAQ;AACb,SAAK,uBAAuB;AAC5B,SAAK,aAAa;AAClB,SAAK,gBAAgB;AACrB,SAAK,MAAM;AAGX,SAAK,IAAI,UAAU,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAE9D,SAAK,cAAc,MAAA;AAEnB,SAAK,SAAS,KAAK,aAAa,YAAY;AAAA,EAC9C;AAAA,EAEA,MAAM,KAAK,QAA+B;AACxC,UAAM,gBAAgB,KAAK;AAG3B,QAAI,KAAK,UAAU,MAAM;AACvB,2BAAqB,KAAK,KAAK;AAC/B,WAAK,QAAQ;AAAA,IACf;AACA,SAAK,cAAc,aAAA;AAEnB,UAAM,UAAU,KAAK,UAAU,MAAM;AACrC,SAAK,gBAAgB;AACrB,SAAK;AACL,SAAK,cAAc;AAEnB,SAAK,QAAQ;AAEb,UAAM,SAAS,KAAK;AAEpB,QAAI;AACF,YAAM,KAAK,mBAAmB,SAAS,KAAK;AAG5C,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AAEA,WAAK,SAAS,KAAK,aAAa,cAAc,EAAE,QAAQ,KAAK,eAAe;AAE5E,UAAI,KAAK,sBAAsB;AAC7B,cAAM,KAAK,cAAA;AAAA,MACb,OAAO;AACL,aAAK,QAAQ,kBAAkB,SAAS,SAAS;AAAA,MACnD;AAAA,IACF,SAAS,OAAO;AAEd,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AACA,cAAQ,MAAM,oCAAoC,KAAK;AACvD,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAC7D,WAAK,QAAQ,kBAAkB,SAAS,SAAS;AAAA,IACnD;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ,MAAoB;AAE1B,UAAM,UAAU,YAAY,IAAA,IAAQ,KAAK;AACzC,SAAK,eAAe;AACpB,SAAK,YAAY,YAAY,IAAA,IAAQ,UAAU;AAE/C,SAAK,SAAS,KAAK,aAAa,oBAAoB,EAAE,MAAM;AAE5D,SAAK,cAAc,gBAAgB,KAAK,YAAY;AAAA,EACtD;AAAA,EAEA,UAAU,QAAsB;AAC9B,SAAK,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,CAAC;AAC7C,SAAK,SAAS,KAAK,aAAa,sBAAsB,EAAE,QAAQ,KAAK,QAAQ;AAE7E,SAAK,cAAc,UAAU,KAAK,MAAM;AAAA,EAC1C;AAAA,EAEA,QAAQ,OAAsB;AAC5B,QAAI,OAAO;AACT,WAAK,cAAc,aAAA;AAAA,IACrB,WAAW,KAAK,UAAU,aAAa,KAAK,cAAc;AACxD,WAAK,cAAc,cAAc,KAAK,eAAe,KAAK,YAAY;AAAA,IACxE;AAAA,EACF;AAAA,EAEA,QAAQ,MAAqB;AAC3B,SAAK,OAAO;AAAA,EACd;AAAA,EAEA,IAAI,WAAmB;AACrB,UAAM,gBAAgB,KAAK,aAAa,kBAAkB;AAC1D,QAAI,kBAAkB,QAAW;AAC/B,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,EACT;AAAA,EAEA,IAAI,YAAqB;AACvB,WAAO,KAAK,UAAU;AAAA,EACxB;AAAA,EAEA,gBAAgB,SAAmC;AACjD,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA,EAGA,SAAe;AACb,SAAK,KAAA;AAAA,EACP;AAAA,EAEA,GAAG,OAAe,SAAuC;AACvD,SAAK,SAAS,GAAG,OAAuB,OAAO;AAAA,EACjD;AAAA,EAEA,IAAI,OAAe,SAAuC;AACxD,SAAK,SAAS,IAAI,OAAuB,OAAO;AAAA,EAClD;AAAA,EAEQ,iBAAuB;AAC7B,SAAK,aAAa,GAAG,aAAa,YAAY,CAAC,UAAU;AAEvD,UAAI,KAAK,UAAU,aAAa,KAAK,UAAU,aAAa;AAC1D;AAAA,MACF;AAEA,WAAK,mBAAmB,MAAM,MAAM;AAAA,IACtC,CAAC;AAAA,EACH;AAAA;AAAA,EAGQ,eAAqB;AAE3B,QAAI,KAAK,UAAU,WAAW;AAC5B,UAAI,KAAK,UAAU,MAAM;AACvB,6BAAqB,KAAK,KAAK;AAC/B,aAAK,QAAQ;AAAA,MACf;AACA;AAAA,IACF;AAEA,SAAK,QAAQ,sBAAsB,YAAY;AAE7C,UAAI,KAAK,UAAU,WAAW;AAC5B;AAAA,MACF;AAEA,WAAK,WAAA;AAGL,WAAK,cAAc,WAAW,KAAK,aAAa;AAEhD,YAAM,KAAK,mBAAmB,KAAK,eAAe,IAAI;AAGtD,UAAI,KAAK,UAAU,WAAW;AAC5B;AAAA,MACF;AAGA,YAAM,MAAM,YAAY,IAAA;AACxB,UAAI,KAAK,gBAAgB,GAAG;AAC1B,cAAM,YAAY,MAAM,KAAK;AAC7B,cAAM,aAAa,MAAO;AAC1B,aAAK,MAAM,KAAK,MAAM,IAAI,KAAK,MAAM,MAAM,aAAa,MAAM;AAAA,MAChE;AACA,WAAK,gBAAgB;AAErB,WAAK;AAEL,WAAK,aAAA;AAAA,IACP,CAAC;AAAA,EACH;AAAA,EAEQ,aAAmB;AACzB,UAAM,WAAW,YAAY,IAAA,IAAQ,KAAK,aAAa,KAAK;AAC5D,UAAM,YAAY,UAAU;AAG5B,SAAK,gBAAgB;AAErB,QAAI,KAAK,iBAAiB,KAAK,UAAU;AACvC,UAAI,KAAK,MAAM;AACb,aAAK,gBAAgB;AACrB,aAAK,YAAY,YAAY,IAAA;AAAA,MAC/B,OAAO;AACL,aAAK,gBAAgB,KAAK;AAC1B,aAAK,MAAA;AACL,aAAK,QAAQ;AACb,aAAK,SAAS,KAAK,aAAa,eAAe,EAAE,QAAQ,KAAK,eAAe;AAAA,MAC/E;AAAA,IACF;AAGA,SAAK,SAAS,KAAK,aAAa,oBAAoB,EAAE,QAAQ,KAAK,eAAe;AAAA,EACpF;AAAA,EAEA,MAAM,mBAAmB,QAAgB,YAAY,MAAqB;AACxE,QAAI;AACF,YAAM,UAAU,MAAM,KAAK,aAAa,YAAY,QAAQ,EAAE,WAAW;AACzE,UAAI,CAAC,SAAS;AAEZ,YAAI,KAAK,UAAU,WAAW;AAC5B,gBAAM,KAAK,wBAAwB,MAAM;AAAA,QAC3C;AACA;AAAA,MACF;AACA,YAAM,QAAQ,IAAI,CAAC,UAAU;AAE3B,aAAK,IAAI,wBAAwB;AACjC,aAAK,IAAI,wBAAwB;AACjC,aAAK,IAAI,UAAU,OAAO,GAAG,GAAG,KAAK,OAAO,OAAO,KAAK,OAAO,MAAM;AAAA,MACvE,CAAC;AAAA,IACH,SAAS,OAAO;AACd,cAAQ,MAAM,iBAAiB,KAAK;AACpC,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,MAAc,wBAAwB,QAA+B;AACnE,QAAI,KAAK,aAAa;AACpB;AAAA,IACF;AAEA,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,CAAC,WAAY;AAEjB,UAAM,SAAS,KAAK;AACpB,SAAK,cAAc;AACnB,SAAK,QAAQ;AACb,SAAK,SAAS,KAAK,aAAa,iBAAiB;AAEjD,QAAI;AAGF,YAAM,QAAQ,MAAM,KAAK,aAAa,iBAAiB,QAAQ;AAAA,QAC7D,eAAe;AAAA;AAAA,QACf,WAAW;AAAA,MAAA,CACZ;AAGD,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AAEA,UAAI,CAAC,OAAO;AACV,gBAAQ,KAAK,0DAA0D,MAAM;AAAA,MAC/E;AAEA,WAAK,QAAQ;AACb,WAAK,YAAY,YAAY,IAAA,IAAQ,SAAS,MAAO,KAAK;AAC1D,WAAK,SAAS,KAAK,aAAa,YAAY;AAE5C,UAAI,CAAC,KAAK,OAAO;AACf,aAAK,aAAA;AAAA,MACP;AAAA,IACF,SAAS,OAAO;AAEd,UAAI,iBAAiB,qBAAqB;AACxC;AAAA,MACF;AAEA,UAAI,WAAW,KAAK,eAAe;AACjC;AAAA,MACF;AACA,cAAQ,MAAM,yCAAyC,KAAK;AAC5D,WAAK,QAAQ;AACb,WAAK,SAAS,KAAK,aAAa,eAAe,KAAc;AAAA,IAC/D,UAAA;AACE,WAAK,cAAc;AAAA,IACrB;AAAA,EACF;AAAA,EAEQ,UAAU,QAAwB;AACxC,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,KAAK,QAAQ,CAAC;AAAA,EACpD;AAAA;AAAA,EAGA,UAAgB;AACd,SAAK,KAAA;AAAA,EACP;AAAA,EAEA,MAAc,qBAAoC;AAChD,QAAI,KAAK,cAAc;AACrB;AAAA,IACF;AAEA,SAAK,eAAe,IAAI,aAAA;AAAA,EAC1B;AACF;"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { TimeUs } from '
|
|
2
|
-
import { CompositionModel } from '
|
|
3
|
-
import { WorkerPool } from '
|
|
4
|
-
import { ResourceLoader } from '../load/ResourceLoader';
|
|
5
|
-
import { EventBus } from '
|
|
6
|
-
import { EventPayloadMap } from '
|
|
7
|
-
import { CacheManager } from '
|
|
1
|
+
import { TimeUs } from '../model/types';
|
|
2
|
+
import { CompositionModel } from '../model';
|
|
3
|
+
import { WorkerPool } from '../worker/WorkerPool';
|
|
4
|
+
import { ResourceLoader } from '../stages/load/ResourceLoader';
|
|
5
|
+
import { EventBus } from '../event/EventBus';
|
|
6
|
+
import { EventPayloadMap } from '../event/events';
|
|
7
|
+
import { CacheManager } from '../cache/CacheManager';
|
|
8
8
|
|
|
9
9
|
interface AudioDataMessage {
|
|
10
10
|
sessionId: string;
|
|
@@ -32,6 +32,7 @@ export declare class GlobalAudioSession {
|
|
|
32
32
|
private playbackRate;
|
|
33
33
|
private isPlaying;
|
|
34
34
|
private currentPlaybackTimeUs;
|
|
35
|
+
private ensuringFromL2;
|
|
35
36
|
constructor(deps: AudioSessionDeps);
|
|
36
37
|
onAudioData(message: AudioDataMessage): void;
|
|
37
38
|
activateAllAudioClips(): Promise<void>;
|
|
@@ -61,6 +62,11 @@ export declare class GlobalAudioSession {
|
|
|
61
62
|
private continueClipPlayback;
|
|
62
63
|
private stopAllAudioSources;
|
|
63
64
|
private getActiveAudioClips;
|
|
65
|
+
/**
|
|
66
|
+
* Ensure PCM for a clip is available by decoding from L2 encoded audio
|
|
67
|
+
* No-op if PCM already exists or L2 lacks audio
|
|
68
|
+
*/
|
|
69
|
+
ensureClipAudioFromL2(clipId: string): Promise<void>;
|
|
64
70
|
private pcmToAudioBuffer;
|
|
65
71
|
private audioBufferToAudioData;
|
|
66
72
|
private interleavePlanarData;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GlobalAudioSession.d.ts","sourceRoot":"","sources":["../../src/orchestrator/GlobalAudioSession.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAa,MAAM,gBAAgB,CAAC;AAExD,OAAO,KAAK,EAAE,gBAAgB,EAAQ,MAAM,UAAU,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAK1D,UAAU,gBAAgB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,SAAS,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,gBAAgB;IACxB,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,EAAE,UAAU,CAAC;IACpB,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,CAAC;IACpC,QAAQ,EAAE,MAAM,gBAAgB,GAAG,IAAI,CAAC;IACxC,kBAAkB,EAAE,MAAM,GAAG,CAAC;CAC/B;AAED,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAoB;IACjC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,IAAI,CAAmB;IAC/B,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,MAAM,CAAO;IACrB,OAAO,CAAC,YAAY,CAAO;IAC3B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,qBAAqB,CAAa;IAC1C,OAAO,CAAC,cAAc,CAAqB;gBAE/B,IAAI,EAAE,gBAAgB;IAKlC,WAAW,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAKtC,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IA8BtC,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBnD,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI;IA2BhE,OAAO,CAAC,oBAAoB;IAoC5B,iBAAiB,CAAC,MAAM,EAAE,cAAc,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IAgCnF,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAY9E,YAAY,IAAI,IAAI;IAKpB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAQhC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAqB/B,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAOnC,KAAK,IAAI,IAAI;IAQb;;OAEG;IACG,yBAAyB,CAC7B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,EACpC,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,yBAAyB,KAAK,IAAI,GAC9D,OAAO,CAAC,cAAc,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC;IA2BpD;;OAEG;IACG,uBAAuB,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;YA+B5D,sBAAsB;IAYpC,OAAO,CAAC,cAAc;IAwDtB,OAAO,CAAC,mBAAmB;IAY3B,OAAO,CAAC,qBAAqB;IAqC7B,OAAO,CAAC,iBAAiB;IAiFzB,OAAO,CAAC,oBAAoB;IAiF5B,OAAO,CAAC,mBAAmB;IAsB3B,OAAO,CAAC,mBAAmB;IAsB3B;;;OAGG;IACG,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmD1D,OAAO,CAAC,gBAAgB;IAkBxB,OAAO,CAAC,sBAAsB;IAoB9B,OAAO,CAAC,oBAAoB;YAgBd,kBAAkB;CA6CjC"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { OfflineAudioMixer } from "
|
|
2
|
-
import { MeframeEvent } from "
|
|
3
|
-
import { AudioChunkEncoder } from "../encode/AudioChunkEncoder.js";
|
|
4
|
-
import {
|
|
1
|
+
import { OfflineAudioMixer } from "../stages/compose/OfflineAudioMixer.js";
|
|
2
|
+
import { MeframeEvent } from "../event/events.js";
|
|
3
|
+
import { AudioChunkEncoder } from "../stages/encode/AudioChunkEncoder.js";
|
|
4
|
+
import { AudioChunkDecoder } from "../stages/decode/AudioChunkDecoder.js";
|
|
5
|
+
import { isAudioClip, hasAudioConfig } from "../model/types.js";
|
|
5
6
|
class GlobalAudioSession {
|
|
6
7
|
mixer;
|
|
7
8
|
activeClips = /* @__PURE__ */ new Set();
|
|
@@ -14,6 +15,7 @@ class GlobalAudioSession {
|
|
|
14
15
|
playbackRate = 1;
|
|
15
16
|
isPlaying = false;
|
|
16
17
|
currentPlaybackTimeUs = 0;
|
|
18
|
+
ensuringFromL2 = /* @__PURE__ */ new Set();
|
|
17
19
|
constructor(deps) {
|
|
18
20
|
this.deps = deps;
|
|
19
21
|
this.mixer = new OfflineAudioMixer(deps.cacheManager, deps.getModel);
|
|
@@ -178,6 +180,7 @@ class GlobalAudioSession {
|
|
|
178
180
|
this.deps.cacheManager.resetAudioCache();
|
|
179
181
|
this.activeClips.clear();
|
|
180
182
|
this.streamEndedClips.clear();
|
|
183
|
+
this.ensuringFromL2.clear();
|
|
181
184
|
}
|
|
182
185
|
/**
|
|
183
186
|
* Create export encoded audio stream
|
|
@@ -468,6 +471,55 @@ class GlobalAudioSession {
|
|
|
468
471
|
}
|
|
469
472
|
return clips;
|
|
470
473
|
}
|
|
474
|
+
/**
|
|
475
|
+
* Ensure PCM for a clip is available by decoding from L2 encoded audio
|
|
476
|
+
* No-op if PCM already exists or L2 lacks audio
|
|
477
|
+
*/
|
|
478
|
+
async ensureClipAudioFromL2(clipId) {
|
|
479
|
+
console.log("[GlobalAudioSession] ensureClipAudioFromL2", clipId);
|
|
480
|
+
if (this.deps.cacheManager.hasClipPCM(clipId)) {
|
|
481
|
+
return;
|
|
482
|
+
}
|
|
483
|
+
if (this.ensuringFromL2.has(clipId)) {
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
const model = this.deps.getModel();
|
|
487
|
+
const clip = model?.findClip(clipId);
|
|
488
|
+
if (!clip) return;
|
|
489
|
+
const l2Meta = await this.deps.cacheManager.getL2Metadata(clipId, "audio");
|
|
490
|
+
const chunkStream = await this.deps.cacheManager.l2Cache.createReadStream(clipId, "audio");
|
|
491
|
+
if (!l2Meta || !chunkStream) {
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
this.ensuringFromL2.add(clipId);
|
|
495
|
+
const decoder = new AudioChunkDecoder(`l2-audio-${clipId}`, {
|
|
496
|
+
codec: l2Meta?.codec,
|
|
497
|
+
sampleRate: l2Meta?.sampleRate,
|
|
498
|
+
numberOfChannels: l2Meta?.numberOfChannels,
|
|
499
|
+
description: l2Meta?.description
|
|
500
|
+
});
|
|
501
|
+
try {
|
|
502
|
+
const decodeStream = chunkStream.pipeThrough(decoder.createStream());
|
|
503
|
+
const reader = decodeStream.getReader();
|
|
504
|
+
const pump = async () => {
|
|
505
|
+
const { done, value } = await reader.read();
|
|
506
|
+
if (done) {
|
|
507
|
+
reader.releaseLock();
|
|
508
|
+
return;
|
|
509
|
+
}
|
|
510
|
+
if (value) {
|
|
511
|
+
this.deps.cacheManager.putClipAudioData(clipId, value, clip.durationUs);
|
|
512
|
+
}
|
|
513
|
+
await pump();
|
|
514
|
+
};
|
|
515
|
+
await pump();
|
|
516
|
+
} catch (error) {
|
|
517
|
+
console.error("[GlobalAudioSession] ensureClipAudioFromL2 error:", error);
|
|
518
|
+
} finally {
|
|
519
|
+
this.ensuringFromL2.delete(clipId);
|
|
520
|
+
await decoder.close();
|
|
521
|
+
}
|
|
522
|
+
}
|
|
471
523
|
pcmToAudioBuffer(planes, sampleRate) {
|
|
472
524
|
const numberOfChannels = planes.length;
|
|
473
525
|
const numberOfFrames = planes[0]?.length ?? 0;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GlobalAudioSession.js","sources":["../../src/orchestrator/GlobalAudioSession.ts"],"sourcesContent":["import type { TimeUs, AudioClip } from '../model/types';\nimport { OfflineAudioMixer } from '../stages/compose/OfflineAudioMixer';\nimport type { CompositionModel, Clip } from '../model';\nimport type { WorkerPool } from '../worker/WorkerPool';\nimport type { ResourceLoader } from '../stages/load/ResourceLoader';\nimport type { EventBus } from '../event/EventBus';\nimport type { EventPayloadMap } from '../event/events';\nimport { MeframeEvent } from '../event/events';\nimport type { CacheManager } from '../cache/CacheManager';\nimport { AudioChunkEncoder } from '../stages/encode/AudioChunkEncoder';\nimport { AudioChunkDecoder } from '../stages/decode/AudioChunkDecoder';\nimport { isAudioClip, hasAudioConfig } from '../model/types';\n\ninterface AudioDataMessage {\n sessionId: string;\n audioData: AudioData;\n clipStartUs: TimeUs;\n clipDurationUs: TimeUs;\n}\n\ninterface AudioSessionDeps {\n cacheManager: CacheManager;\n workers: WorkerPool;\n resourceLoader: ResourceLoader;\n eventBus: EventBus<EventPayloadMap>;\n getModel: () => CompositionModel | null;\n buildWorkerConfigs: () => any;\n}\n\nexport class GlobalAudioSession {\n private mixer: OfflineAudioMixer;\n private activeClips = new Set<string>();\n private streamEndedClips = new Set<string>();\n private deps: AudioSessionDeps;\n private audioContext: AudioContext | null = null;\n private audioSources: AudioBufferSourceNode[] = [];\n private audioGainNodes: GainNode[] = [];\n private volume = 1.0;\n private playbackRate = 1.0;\n private isPlaying = false;\n private currentPlaybackTimeUs: TimeUs = 0;\n private ensuringFromL2 = new Set<string>();\n\n constructor(deps: AudioSessionDeps) {\n this.deps = deps;\n this.mixer = new OfflineAudioMixer(deps.cacheManager, deps.getModel);\n }\n\n onAudioData(message: AudioDataMessage): void {\n const { sessionId, audioData, clipDurationUs } = message;\n this.deps.cacheManager.putClipAudioData(sessionId, audioData, clipDurationUs);\n }\n\n async activateAllAudioClips(): Promise<void> {\n const model = this.deps.getModel();\n if (!model) {\n return;\n }\n\n const audioTracks = model.tracks.filter((track) => track.kind === 'audio');\n\n for (const track of audioTracks) {\n for (const clip of track.clips) {\n if (!this.activeClips.has(clip.id)) {\n if (!isAudioClip(clip)) {\n throw new Error(`Clip ${clip.id} in audio track is not an audio clip`);\n }\n\n await this.setupAudioPipeline(clip);\n this.activeClips.add(clip.id);\n\n await this.deps.resourceLoader.fetch(clip.resourceId, {\n priority: 'high',\n sessionId: clip.id,\n trackId: track.id,\n });\n\n this.deps.eventBus.emit(MeframeEvent.ClipActivated, { clipId: clip.id });\n }\n }\n }\n }\n\n async deactivateClip(clipId: string): Promise<void> {\n if (!this.activeClips.has(clipId)) {\n return;\n }\n\n // Stop any playing audio sources for this clip\n this.stopClipAudioSources(clipId);\n\n this.deps.workers.terminate('audioDemux', clipId);\n this.deps.workers.terminate('audioDecode', clipId);\n\n this.activeClips.delete(clipId);\n\n this.deps.cacheManager.clearClipAudioData(clipId);\n }\n\n restartPlayingClip(clipId: string, currentTimeUs?: TimeUs): void {\n if (!this.isPlaying || !this.audioContext) {\n return;\n }\n\n const timeUs = currentTimeUs ?? this.currentPlaybackTimeUs;\n\n const model = this.deps.getModel();\n if (!model) {\n return;\n }\n\n const clip = model.findClip(clipId);\n if (!clip) {\n return;\n }\n\n // Check if clip should be playing at current time\n const clipEndUs = clip.startUs + clip.durationUs;\n if (timeUs < clip.startUs || timeUs >= clipEndUs) {\n return;\n }\n\n // Start playback from current time\n this.startClipPlayback(clip, timeUs);\n }\n\n private stopClipAudioSources(clipId: string): void {\n const sourcesToStop: AudioBufferSourceNode[] = [];\n const gainNodesToDisconnect: GainNode[] = [];\n\n for (let i = this.audioSources.length - 1; i >= 0; i--) {\n const source = this.audioSources[i];\n if (source && (source as any)._meframeClipId === clipId) {\n sourcesToStop.push(source);\n this.audioSources.splice(i, 1);\n\n const gainNode = this.audioGainNodes[i];\n if (gainNode) {\n gainNodesToDisconnect.push(gainNode);\n this.audioGainNodes.splice(i, 1);\n }\n }\n }\n\n for (const source of sourcesToStop) {\n try {\n source.stop();\n source.disconnect();\n } catch (error) {\n // Ignore - source may have already stopped\n }\n }\n\n for (const gainNode of gainNodesToDisconnect) {\n try {\n gainNode.disconnect();\n } catch (error) {\n // Ignore\n }\n }\n }\n\n handleAudioStream(stream: ReadableStream<AudioData>, metadata: Record<string, any>): void {\n const sessionId = metadata.sessionId || 'unknown';\n const clipStartUs = metadata.clipStartUs ?? 0;\n const clipDurationUs = metadata.clipDurationUs ?? 0;\n\n const reader = stream.getReader();\n const pump = async (): Promise<void> => {\n try {\n const { done, value } = await reader.read();\n if (done) {\n this.streamEndedClips.add(sessionId);\n reader.releaseLock();\n return;\n }\n\n this.onAudioData({\n sessionId,\n audioData: value,\n clipStartUs,\n clipDurationUs,\n });\n\n await pump();\n } catch (error) {\n console.error('[GlobalAudioSession] Audio stream error:', error);\n reader.releaseLock();\n }\n };\n\n pump();\n }\n\n async startPlayback(timeUs: TimeUs, audioContext: AudioContext): Promise<void> {\n this.audioContext = audioContext;\n\n // Resume AudioContext if suspended (required by modern browsers)\n if (audioContext.state === 'suspended') {\n await audioContext.resume();\n }\n\n this.isPlaying = true;\n this.startAllActiveClips(timeUs);\n }\n\n stopPlayback(): void {\n this.isPlaying = false;\n this.stopAllAudioSources();\n }\n\n updateTime(timeUs: TimeUs): void {\n this.currentPlaybackTimeUs = timeUs;\n if (!this.isPlaying) {\n return;\n }\n this.checkAndStartNewClips(timeUs);\n }\n\n setVolume(volume: number): void {\n this.volume = volume;\n\n // Update existing gain nodes with clip-level config\n for (let i = 0; i < this.audioGainNodes.length; i++) {\n const gainNode = this.audioGainNodes[i];\n const source = this.audioSources[i];\n const clipId = (source as any)._meframeClipId;\n\n if (clipId && gainNode) {\n const model = this.deps.getModel();\n const clip = model?.findClip(clipId);\n if (clip && hasAudioConfig(clip)) {\n const clipVolume = clip.audioConfig?.volume ?? 1.0;\n const muted = clip.audioConfig?.muted ?? false;\n gainNode.gain.value = muted ? 0 : clipVolume * this.volume;\n }\n }\n }\n }\n\n setPlaybackRate(rate: number): void {\n this.playbackRate = rate;\n for (const source of this.audioSources) {\n source.playbackRate.value = this.playbackRate;\n }\n }\n\n reset(): void {\n this.stopAllAudioSources();\n this.deps.cacheManager.resetAudioCache();\n this.activeClips.clear();\n this.streamEndedClips.clear();\n this.ensuringFromL2.clear();\n }\n\n /**\n * Create export encoded audio stream\n */\n async createExportEncodedStream(\n config?: Partial<AudioEncoderConfig>,\n onFirstMetadata?: (metadata: EncodedAudioChunkMetadata) => void\n ): Promise<ReadableStream<EncodedAudioChunk> | null> {\n const audioDataStream = await this.createExportAudioStream();\n if (!audioDataStream) {\n return null;\n }\n\n const encoder = new AudioChunkEncoder(config);\n await encoder.initialize();\n\n const encodingTransform = encoder.createStream();\n const encodedStream = audioDataStream.pipeThrough(encodingTransform);\n\n let firstMetadataExtracted = false;\n\n return encodedStream.pipeThrough(\n new TransformStream({\n transform(encoderChunk, controller) {\n if (!firstMetadataExtracted && onFirstMetadata) {\n onFirstMetadata(encoderChunk.metadata as EncodedAudioChunkMetadata);\n firstMetadataExtracted = true;\n }\n controller.enqueue(encoderChunk.chunk as EncodedAudioChunk);\n },\n })\n );\n }\n\n /**\n * Create export audio stream\n */\n async createExportAudioStream(): Promise<ReadableStream<AudioData> | null> {\n const model = this.deps.getModel();\n if (!model) {\n return null;\n }\n\n const totalDurationUs = model.durationUs;\n\n await this.activateAllAudioClips();\n await this.waitForAudioClipsReady();\n\n return new ReadableStream<AudioData>({\n start: async (controller) => {\n const windowSize = 5_000_000;\n let currentUs = 0;\n\n while (currentUs < totalDurationUs) {\n const windowEndUs = Math.min(currentUs + windowSize, totalDurationUs);\n const mixedBuffer = await this.mixer.mix(currentUs, windowEndUs);\n const audioData = this.audioBufferToAudioData(mixedBuffer, currentUs);\n if (audioData) {\n controller.enqueue(audioData);\n }\n currentUs = windowEndUs;\n }\n\n controller.close();\n },\n });\n }\n\n private async waitForAudioClipsReady(): Promise<void> {\n const model = this.deps.getModel();\n if (!model) return;\n\n const audioClips = model.tracks\n .filter((track) => track.kind === 'audio')\n .flatMap((track) => track.clips);\n\n const waitPromises = audioClips.map((clip) => this.waitForClipPCM(clip.id, 10000)); // 10s timeout\n await Promise.allSettled(waitPromises);\n }\n\n private waitForClipPCM(clipId: string, timeoutMs: number): Promise<boolean> {\n return new Promise((resolve) => {\n const checkInterval = 100;\n let elapsed = 0;\n let lastFrameCount = 0;\n let stableCount = 0;\n let streamEndDetected = false;\n\n const check = () => {\n const pcm = this.deps.cacheManager.getClipPCM(clipId, 0, Number.MAX_SAFE_INTEGER);\n\n if (pcm && pcm.length > 0) {\n const currentFrameCount = pcm[0]?.length ?? 0;\n\n // Check if we have received stream end signal\n if (this.streamEndedClips.has(clipId)) {\n streamEndDetected = true;\n }\n\n // If stream has ended, we're done\n if (streamEndDetected) {\n resolve(true);\n return;\n }\n\n // Otherwise, check if frame count is stable (no new data for 500ms)\n if (currentFrameCount === lastFrameCount) {\n stableCount++;\n if (stableCount >= 5) {\n // 5 * 100ms = 500ms\n resolve(true);\n return;\n }\n } else {\n stableCount = 0;\n lastFrameCount = currentFrameCount;\n }\n }\n\n elapsed += checkInterval;\n if (elapsed >= timeoutMs) {\n console.warn('[GlobalAudioSession] Timeout waiting for clip', clipId, {\n frames: lastFrameCount,\n elapsed,\n });\n resolve(false);\n return;\n }\n\n setTimeout(check, checkInterval);\n };\n\n check();\n });\n }\n\n private startAllActiveClips(timeUs: TimeUs): void {\n if (!this.audioContext) {\n return;\n }\n\n const currentClips = this.getActiveAudioClips(timeUs);\n\n for (const clip of currentClips) {\n this.startClipPlayback(clip, timeUs);\n }\n }\n\n private checkAndStartNewClips(timeUs: TimeUs): void {\n if (!this.audioContext) {\n return;\n }\n\n const currentClips = this.getActiveAudioClips(timeUs);\n const activeClipIds = new Set(\n this.audioSources.map((source) => (source as any)._meframeClipId).filter(Boolean)\n );\n\n for (const clip of currentClips) {\n // Check if clip should be playing at current time\n const clipEndUs = clip.startUs + clip.durationUs;\n if (timeUs >= clipEndUs) {\n // Clip has already ended, skip\n continue;\n }\n\n // Check if current time is before clip starts\n if (timeUs < clip.startUs) {\n // Not yet time to play this clip\n continue;\n }\n\n // Check if clip is too close to ending (avoid glitches from very short playback)\n const MIN_REMAINING_TIME_US = 30000; // 30ms in microseconds\n if (clipEndUs - timeUs < MIN_REMAINING_TIME_US) {\n // Too close to the end, skip to avoid audio glitches\n continue;\n }\n\n if (!activeClipIds.has(clip.id)) {\n this.startClipPlayback(clip, timeUs);\n }\n }\n }\n\n private startClipPlayback(clip: Clip, currentTimeUs: TimeUs): void {\n if (!this.audioContext) {\n console.warn('[GlobalAudioSession] No audioContext, cannot start playback');\n return;\n }\n\n // Use clip-relative time (0-based) like video cache\n const clipPCMData = this.deps.cacheManager.getClipPCMWithMetadata(\n clip.id,\n 0, // Start from beginning of clip (0-based)\n clip.durationUs // Full clip duration\n );\n\n if (!clipPCMData || clipPCMData.planes.length === 0) {\n // No data yet, will retry later via checkAndStartNewClips\n console.warn('[GlobalAudioSession] No PCM data for clip, will retry later', clip.id);\n return;\n }\n\n const buffer = this.pcmToAudioBuffer(clipPCMData.planes, clipPCMData.sampleRate);\n\n const offsetUs = Math.max(0, currentTimeUs - clip.startUs);\n const offsetSeconds = offsetUs / 1_000_000;\n\n // Use actual buffer duration instead of clip.durationUs\n const actualDurationSeconds = buffer.duration - offsetSeconds;\n\n // Early check: if remaining duration is too short, skip\n // (Note: checkAndStartNewClips should already filter these out)\n const MIN_PLAYBACK_DURATION_SECONDS = 0.03; // 30ms\n if (actualDurationSeconds < MIN_PLAYBACK_DURATION_SECONDS) {\n return;\n }\n\n const source = this.audioContext.createBufferSource();\n source.buffer = buffer;\n source.playbackRate.value = this.playbackRate;\n (source as any)._meframeClipId = clip.id;\n (source as any)._playedToUs = clip.startUs + buffer.duration * 1_000_000; // Track where it played to\n\n const gainNode = this.audioContext.createGain();\n\n // Apply audio config\n if (hasAudioConfig(clip)) {\n const volume = clip.audioConfig?.volume ?? 1.0;\n const muted = clip.audioConfig?.muted ?? false;\n gainNode.gain.value = muted ? 0 : volume * this.volume;\n } else {\n gainNode.gain.value = this.volume;\n }\n\n source.connect(gainNode);\n gainNode.connect(this.audioContext.destination);\n\n source.start(0, offsetSeconds, actualDurationSeconds);\n\n source.onended = () => {\n const index = this.audioSources.indexOf(source);\n if (index >= 0) {\n this.audioSources.splice(index, 1);\n this.audioGainNodes.splice(index, 1);\n }\n\n // Check if more data has arrived and continue playing\n const playedToUs = (source as any)._playedToUs;\n const clipEndUs = clip.startUs + clip.durationUs;\n\n if (playedToUs < clipEndUs && this.isPlaying) {\n // There might be more data, try to continue\n setTimeout(() => {\n if (this.isPlaying) {\n this.continueClipPlayback(clip, playedToUs);\n }\n }, 50);\n }\n };\n\n this.audioSources.push(source);\n this.audioGainNodes.push(gainNode);\n }\n\n private continueClipPlayback(clip: Clip, fromUs: TimeUs): void {\n if (!this.audioContext) {\n return;\n }\n\n // Use clip-relative time (0-based) like video cache\n const clipPCMData = this.deps.cacheManager.getClipPCMWithMetadata(\n clip.id,\n 0, // 0-based start\n clip.durationUs // clip duration\n );\n\n if (!clipPCMData || clipPCMData.planes.length === 0) {\n return;\n }\n\n const buffer = this.pcmToAudioBuffer(clipPCMData.planes, clipPCMData.sampleRate);\n const bufferEndUs = clip.startUs + buffer.duration * 1_000_000;\n\n // Check if there's new data beyond where we played to\n if (bufferEndUs <= fromUs + 100_000) {\n // 100ms tolerance\n // No significant new data\n return;\n }\n\n // Continue playback from where it left off\n const source = this.audioContext.createBufferSource();\n source.buffer = buffer;\n source.playbackRate.value = this.playbackRate;\n (source as any)._meframeClipId = clip.id;\n (source as any)._playedToUs = bufferEndUs;\n\n const gainNode = this.audioContext.createGain();\n\n // Apply audio config\n if (hasAudioConfig(clip)) {\n const volume = clip.audioConfig?.volume ?? 1.0;\n const muted = clip.audioConfig?.muted ?? false;\n gainNode.gain.value = muted ? 0 : volume * this.volume;\n } else {\n gainNode.gain.value = this.volume;\n }\n\n source.connect(gainNode);\n gainNode.connect(this.audioContext.destination);\n\n const offsetUs = Math.max(0, fromUs - clip.startUs);\n const offsetSeconds = offsetUs / 1_000_000;\n const actualDurationSeconds = buffer.duration - offsetSeconds;\n\n if (actualDurationSeconds <= 0) {\n return;\n }\n\n source.start(0, offsetSeconds, actualDurationSeconds);\n\n source.onended = () => {\n const index = this.audioSources.indexOf(source);\n if (index >= 0) {\n this.audioSources.splice(index, 1);\n this.audioGainNodes.splice(index, 1);\n }\n\n // Check if more data has arrived\n const playedToUs = (source as any)._playedToUs;\n const clipEndUs = clip.startUs + clip.durationUs;\n\n if (playedToUs < clipEndUs && this.isPlaying) {\n setTimeout(() => {\n if (this.isPlaying) {\n this.continueClipPlayback(clip, playedToUs);\n }\n }, 50);\n }\n };\n\n this.audioSources.push(source);\n this.audioGainNodes.push(gainNode);\n }\n\n private stopAllAudioSources(): void {\n for (const source of this.audioSources) {\n try {\n source.stop();\n source.disconnect();\n } catch (error) {\n // Ignore\n }\n }\n\n for (const gainNode of this.audioGainNodes) {\n try {\n gainNode.disconnect();\n } catch (error) {\n // Ignore\n }\n }\n\n this.audioSources = [];\n this.audioGainNodes = [];\n }\n\n private getActiveAudioClips(timeUs: TimeUs): Clip[] {\n const model = this.deps.getModel();\n if (!model) {\n return [];\n }\n\n const clips: Clip[] = [];\n\n // Get clips from all tracks (video clips can have audio!)\n for (const track of model.tracks) {\n const trackClips = model.getClipsAtTime(timeUs, track.id);\n for (const clip of trackClips) {\n // Check if this clip has audio data in cache\n if (this.deps.cacheManager.hasClipPCM(clip.id)) {\n clips.push(clip);\n }\n }\n }\n\n return clips;\n }\n\n /**\n * Ensure PCM for a clip is available by decoding from L2 encoded audio\n * No-op if PCM already exists or L2 lacks audio\n */\n async ensureClipAudioFromL2(clipId: string): Promise<void> {\n console.log('[GlobalAudioSession] ensureClipAudioFromL2', clipId);\n if (this.deps.cacheManager.hasClipPCM(clipId)) {\n return;\n }\n if (this.ensuringFromL2.has(clipId)) {\n return;\n }\n\n const model = this.deps.getModel();\n const clip = model?.findClip(clipId);\n if (!clip) return;\n\n const l2Meta = await this.deps.cacheManager.getL2Metadata(clipId, 'audio');\n const chunkStream = await this.deps.cacheManager.l2Cache.createReadStream(clipId, 'audio');\n if (!l2Meta || !chunkStream) {\n return;\n }\n\n this.ensuringFromL2.add(clipId);\n\n const decoder = new AudioChunkDecoder(`l2-audio-${clipId}`, {\n codec: l2Meta?.codec,\n sampleRate: l2Meta?.sampleRate,\n numberOfChannels: l2Meta?.numberOfChannels,\n description: l2Meta?.description,\n } as any);\n\n try {\n const decodeStream = chunkStream.pipeThrough(decoder.createStream());\n const reader = decodeStream.getReader();\n const pump = async (): Promise<void> => {\n const { done, value } = await reader.read();\n if (done) {\n reader.releaseLock();\n return;\n }\n if (value) {\n this.deps.cacheManager.putClipAudioData(clipId, value, clip.durationUs);\n }\n await pump();\n };\n await pump();\n } catch (error) {\n console.error('[GlobalAudioSession] ensureClipAudioFromL2 error:', error);\n } finally {\n this.ensuringFromL2.delete(clipId);\n await decoder.close();\n }\n }\n\n private pcmToAudioBuffer(planes: Float32Array[], sampleRate: number): AudioBuffer {\n const numberOfChannels = planes.length;\n const numberOfFrames = planes[0]?.length ?? 0;\n\n const ctx = new OfflineAudioContext(numberOfChannels, 1, sampleRate);\n const buffer = ctx.createBuffer(numberOfChannels, numberOfFrames, sampleRate);\n\n for (let channel = 0; channel < numberOfChannels; channel++) {\n const plane = planes[channel];\n if (plane) {\n const channelData = buffer.getChannelData(channel);\n channelData.set(plane);\n }\n }\n\n return buffer;\n }\n\n private audioBufferToAudioData(buffer: AudioBuffer, timestampUs: TimeUs): AudioData | null {\n const sampleRate = buffer.sampleRate;\n const numberOfChannels = buffer.numberOfChannels;\n const numberOfFrames = buffer.length;\n\n const planes: Float32Array[] = [];\n for (let channel = 0; channel < numberOfChannels; channel++) {\n planes.push(buffer.getChannelData(channel));\n }\n\n return new AudioData({\n format: 'f32', // interleaved format\n sampleRate,\n numberOfFrames,\n numberOfChannels,\n timestamp: timestampUs,\n data: this.interleavePlanarData(planes),\n });\n }\n\n private interleavePlanarData(planes: Float32Array[]): ArrayBuffer {\n const numberOfChannels = planes.length;\n const numberOfFrames = planes[0]?.length ?? 0;\n const totalSamples = numberOfChannels * numberOfFrames;\n\n const interleaved = new Float32Array(totalSamples);\n\n for (let frame = 0; frame < numberOfFrames; frame++) {\n for (let channel = 0; channel < numberOfChannels; channel++) {\n interleaved[frame * numberOfChannels + channel] = planes[channel]![frame]!;\n }\n }\n\n return interleaved.buffer;\n }\n\n private async setupAudioPipeline(clip: AudioClip): Promise<void> {\n const { id: clipId, resourceId, startUs, durationUs } = clip;\n const audioDemuxWorker = await this.deps.workers.getOrCreate('audioDemux', clipId, {\n lazy: true,\n });\n const audioDecodeWorker = await this.deps.workers.getOrCreate('audioDecode', clipId, {\n lazy: true,\n });\n\n const demuxToDecodeChannel = new MessageChannel();\n await audioDemuxWorker.send(\n 'connect',\n { direction: 'downstream', port: demuxToDecodeChannel.port1, streamType: 'audio', clipId },\n { transfer: [demuxToDecodeChannel.port1] }\n );\n await audioDecodeWorker.send(\n 'connect',\n {\n direction: 'upstream',\n port: demuxToDecodeChannel.port2,\n streamType: 'audio',\n sessionId: clipId,\n clipStartUs: startUs || 0,\n clipDurationUs: durationUs || 0,\n },\n { transfer: [demuxToDecodeChannel.port2] }\n );\n\n audioDecodeWorker.receiveStream((stream, metadata) => {\n this.handleAudioStream(stream as ReadableStream<AudioData>, {\n sessionId: clipId,\n clipStartUs: startUs || 0,\n clipDurationUs: durationUs || 0,\n ...metadata,\n });\n });\n\n const demuxConfig = this.deps.buildWorkerConfigs().audioDemux;\n await audioDemuxWorker.send('configure', {\n initial: true,\n resourceId,\n clipId,\n config: demuxConfig,\n });\n }\n}\n"],"names":[],"mappings":";;;;;AA6BO,MAAM,mBAAmB;AAAA,EACtB;AAAA,EACA,kCAAkB,IAAA;AAAA,EAClB,uCAAuB,IAAA;AAAA,EACvB;AAAA,EACA,eAAoC;AAAA,EACpC,eAAwC,CAAA;AAAA,EACxC,iBAA6B,CAAA;AAAA,EAC7B,SAAS;AAAA,EACT,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,wBAAgC;AAAA,EAChC,qCAAqB,IAAA;AAAA,EAE7B,YAAY,MAAwB;AAClC,SAAK,OAAO;AACZ,SAAK,QAAQ,IAAI,kBAAkB,KAAK,cAAc,KAAK,QAAQ;AAAA,EACrE;AAAA,EAEA,YAAY,SAAiC;AAC3C,UAAM,EAAE,WAAW,WAAW,eAAA,IAAmB;AACjD,SAAK,KAAK,aAAa,iBAAiB,WAAW,WAAW,cAAc;AAAA,EAC9E;AAAA,EAEA,MAAM,wBAAuC;AAC3C,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV;AAAA,IACF;AAEA,UAAM,cAAc,MAAM,OAAO,OAAO,CAAC,UAAU,MAAM,SAAS,OAAO;AAEzE,eAAW,SAAS,aAAa;AAC/B,iBAAW,QAAQ,MAAM,OAAO;AAC9B,YAAI,CAAC,KAAK,YAAY,IAAI,KAAK,EAAE,GAAG;AAClC,cAAI,CAAC,YAAY,IAAI,GAAG;AACtB,kBAAM,IAAI,MAAM,QAAQ,KAAK,EAAE,sCAAsC;AAAA,UACvE;AAEA,gBAAM,KAAK,mBAAmB,IAAI;AAClC,eAAK,YAAY,IAAI,KAAK,EAAE;AAE5B,gBAAM,KAAK,KAAK,eAAe,MAAM,KAAK,YAAY;AAAA,YACpD,UAAU;AAAA,YACV,WAAW,KAAK;AAAA,YAChB,SAAS,MAAM;AAAA,UAAA,CAChB;AAED,eAAK,KAAK,SAAS,KAAK,aAAa,eAAe,EAAE,QAAQ,KAAK,IAAI;AAAA,QACzE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,eAAe,QAA+B;AAClD,QAAI,CAAC,KAAK,YAAY,IAAI,MAAM,GAAG;AACjC;AAAA,IACF;AAGA,SAAK,qBAAqB,MAAM;AAEhC,SAAK,KAAK,QAAQ,UAAU,cAAc,MAAM;AAChD,SAAK,KAAK,QAAQ,UAAU,eAAe,MAAM;AAEjD,SAAK,YAAY,OAAO,MAAM;AAE9B,SAAK,KAAK,aAAa,mBAAmB,MAAM;AAAA,EAClD;AAAA,EAEA,mBAAmB,QAAgB,eAA8B;AAC/D,QAAI,CAAC,KAAK,aAAa,CAAC,KAAK,cAAc;AACzC;AAAA,IACF;AAEA,UAAM,SAAS,iBAAiB,KAAK;AAErC,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV;AAAA,IACF;AAEA,UAAM,OAAO,MAAM,SAAS,MAAM;AAClC,QAAI,CAAC,MAAM;AACT;AAAA,IACF;AAGA,UAAM,YAAY,KAAK,UAAU,KAAK;AACtC,QAAI,SAAS,KAAK,WAAW,UAAU,WAAW;AAChD;AAAA,IACF;AAGA,SAAK,kBAAkB,MAAM,MAAM;AAAA,EACrC;AAAA,EAEQ,qBAAqB,QAAsB;AACjD,UAAM,gBAAyC,CAAA;AAC/C,UAAM,wBAAoC,CAAA;AAE1C,aAAS,IAAI,KAAK,aAAa,SAAS,GAAG,KAAK,GAAG,KAAK;AACtD,YAAM,SAAS,KAAK,aAAa,CAAC;AAClC,UAAI,UAAW,OAAe,mBAAmB,QAAQ;AACvD,sBAAc,KAAK,MAAM;AACzB,aAAK,aAAa,OAAO,GAAG,CAAC;AAE7B,cAAM,WAAW,KAAK,eAAe,CAAC;AACtC,YAAI,UAAU;AACZ,gCAAsB,KAAK,QAAQ;AACnC,eAAK,eAAe,OAAO,GAAG,CAAC;AAAA,QACjC;AAAA,MACF;AAAA,IACF;AAEA,eAAW,UAAU,eAAe;AAClC,UAAI;AACF,eAAO,KAAA;AACP,eAAO,WAAA;AAAA,MACT,SAAS,OAAO;AAAA,MAEhB;AAAA,IACF;AAEA,eAAW,YAAY,uBAAuB;AAC5C,UAAI;AACF,iBAAS,WAAA;AAAA,MACX,SAAS,OAAO;AAAA,MAEhB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,kBAAkB,QAAmC,UAAqC;AACxF,UAAM,YAAY,SAAS,aAAa;AACxC,UAAM,cAAc,SAAS,eAAe;AAC5C,UAAM,iBAAiB,SAAS,kBAAkB;AAElD,UAAM,SAAS,OAAO,UAAA;AACtB,UAAM,OAAO,YAA2B;AACtC,UAAI;AACF,cAAM,EAAE,MAAM,MAAA,IAAU,MAAM,OAAO,KAAA;AACrC,YAAI,MAAM;AACR,eAAK,iBAAiB,IAAI,SAAS;AACnC,iBAAO,YAAA;AACP;AAAA,QACF;AAEA,aAAK,YAAY;AAAA,UACf;AAAA,UACA,WAAW;AAAA,UACX;AAAA,UACA;AAAA,QAAA,CACD;AAED,cAAM,KAAA;AAAA,MACR,SAAS,OAAO;AACd,gBAAQ,MAAM,4CAA4C,KAAK;AAC/D,eAAO,YAAA;AAAA,MACT;AAAA,IACF;AAEA,SAAA;AAAA,EACF;AAAA,EAEA,MAAM,cAAc,QAAgB,cAA2C;AAC7E,SAAK,eAAe;AAGpB,QAAI,aAAa,UAAU,aAAa;AACtC,YAAM,aAAa,OAAA;AAAA,IACrB;AAEA,SAAK,YAAY;AACjB,SAAK,oBAAoB,MAAM;AAAA,EACjC;AAAA,EAEA,eAAqB;AACnB,SAAK,YAAY;AACjB,SAAK,oBAAA;AAAA,EACP;AAAA,EAEA,WAAW,QAAsB;AAC/B,SAAK,wBAAwB;AAC7B,QAAI,CAAC,KAAK,WAAW;AACnB;AAAA,IACF;AACA,SAAK,sBAAsB,MAAM;AAAA,EACnC;AAAA,EAEA,UAAU,QAAsB;AAC9B,SAAK,SAAS;AAGd,aAAS,IAAI,GAAG,IAAI,KAAK,eAAe,QAAQ,KAAK;AACnD,YAAM,WAAW,KAAK,eAAe,CAAC;AACtC,YAAM,SAAS,KAAK,aAAa,CAAC;AAClC,YAAM,SAAU,OAAe;AAE/B,UAAI,UAAU,UAAU;AACtB,cAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,cAAM,OAAO,OAAO,SAAS,MAAM;AACnC,YAAI,QAAQ,eAAe,IAAI,GAAG;AAChC,gBAAM,aAAa,KAAK,aAAa,UAAU;AAC/C,gBAAM,QAAQ,KAAK,aAAa,SAAS;AACzC,mBAAS,KAAK,QAAQ,QAAQ,IAAI,aAAa,KAAK;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,gBAAgB,MAAoB;AAClC,SAAK,eAAe;AACpB,eAAW,UAAU,KAAK,cAAc;AACtC,aAAO,aAAa,QAAQ,KAAK;AAAA,IACnC;AAAA,EACF;AAAA,EAEA,QAAc;AACZ,SAAK,oBAAA;AACL,SAAK,KAAK,aAAa,gBAAA;AACvB,SAAK,YAAY,MAAA;AACjB,SAAK,iBAAiB,MAAA;AACtB,SAAK,eAAe,MAAA;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,0BACJ,QACA,iBACmD;AACnD,UAAM,kBAAkB,MAAM,KAAK,wBAAA;AACnC,QAAI,CAAC,iBAAiB;AACpB,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,IAAI,kBAAkB,MAAM;AAC5C,UAAM,QAAQ,WAAA;AAEd,UAAM,oBAAoB,QAAQ,aAAA;AAClC,UAAM,gBAAgB,gBAAgB,YAAY,iBAAiB;AAEnE,QAAI,yBAAyB;AAE7B,WAAO,cAAc;AAAA,MACnB,IAAI,gBAAgB;AAAA,QAClB,UAAU,cAAc,YAAY;AAClC,cAAI,CAAC,0BAA0B,iBAAiB;AAC9C,4BAAgB,aAAa,QAAqC;AAClE,qCAAyB;AAAA,UAC3B;AACA,qBAAW,QAAQ,aAAa,KAA0B;AAAA,QAC5D;AAAA,MAAA,CACD;AAAA,IAAA;AAAA,EAEL;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,0BAAqE;AACzE,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB,MAAM;AAE9B,UAAM,KAAK,sBAAA;AACX,UAAM,KAAK,uBAAA;AAEX,WAAO,IAAI,eAA0B;AAAA,MACnC,OAAO,OAAO,eAAe;AAC3B,cAAM,aAAa;AACnB,YAAI,YAAY;AAEhB,eAAO,YAAY,iBAAiB;AAClC,gBAAM,cAAc,KAAK,IAAI,YAAY,YAAY,eAAe;AACpE,gBAAM,cAAc,MAAM,KAAK,MAAM,IAAI,WAAW,WAAW;AAC/D,gBAAM,YAAY,KAAK,uBAAuB,aAAa,SAAS;AACpE,cAAI,WAAW;AACb,uBAAW,QAAQ,SAAS;AAAA,UAC9B;AACA,sBAAY;AAAA,QACd;AAEA,mBAAW,MAAA;AAAA,MACb;AAAA,IAAA,CACD;AAAA,EACH;AAAA,EAEA,MAAc,yBAAwC;AACpD,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,MAAO;AAEZ,UAAM,aAAa,MAAM,OACtB,OAAO,CAAC,UAAU,MAAM,SAAS,OAAO,EACxC,QAAQ,CAAC,UAAU,MAAM,KAAK;AAEjC,UAAM,eAAe,WAAW,IAAI,CAAC,SAAS,KAAK,eAAe,KAAK,IAAI,GAAK,CAAC;AACjF,UAAM,QAAQ,WAAW,YAAY;AAAA,EACvC;AAAA,EAEQ,eAAe,QAAgB,WAAqC;AAC1E,WAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,YAAM,gBAAgB;AACtB,UAAI,UAAU;AACd,UAAI,iBAAiB;AACrB,UAAI,cAAc;AAClB,UAAI,oBAAoB;AAExB,YAAM,QAAQ,MAAM;AAClB,cAAM,MAAM,KAAK,KAAK,aAAa,WAAW,QAAQ,GAAG,OAAO,gBAAgB;AAEhF,YAAI,OAAO,IAAI,SAAS,GAAG;AACzB,gBAAM,oBAAoB,IAAI,CAAC,GAAG,UAAU;AAG5C,cAAI,KAAK,iBAAiB,IAAI,MAAM,GAAG;AACrC,gCAAoB;AAAA,UACtB;AAGA,cAAI,mBAAmB;AACrB,oBAAQ,IAAI;AACZ;AAAA,UACF;AAGA,cAAI,sBAAsB,gBAAgB;AACxC;AACA,gBAAI,eAAe,GAAG;AAEpB,sBAAQ,IAAI;AACZ;AAAA,YACF;AAAA,UACF,OAAO;AACL,0BAAc;AACd,6BAAiB;AAAA,UACnB;AAAA,QACF;AAEA,mBAAW;AACX,YAAI,WAAW,WAAW;AACxB,kBAAQ,KAAK,iDAAiD,QAAQ;AAAA,YACpE,QAAQ;AAAA,YACR;AAAA,UAAA,CACD;AACD,kBAAQ,KAAK;AACb;AAAA,QACF;AAEA,mBAAW,OAAO,aAAa;AAAA,MACjC;AAEA,YAAA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,oBAAoB,QAAsB;AAChD,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,UAAM,eAAe,KAAK,oBAAoB,MAAM;AAEpD,eAAW,QAAQ,cAAc;AAC/B,WAAK,kBAAkB,MAAM,MAAM;AAAA,IACrC;AAAA,EACF;AAAA,EAEQ,sBAAsB,QAAsB;AAClD,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAEA,UAAM,eAAe,KAAK,oBAAoB,MAAM;AACpD,UAAM,gBAAgB,IAAI;AAAA,MACxB,KAAK,aAAa,IAAI,CAAC,WAAY,OAAe,cAAc,EAAE,OAAO,OAAO;AAAA,IAAA;AAGlF,eAAW,QAAQ,cAAc;AAE/B,YAAM,YAAY,KAAK,UAAU,KAAK;AACtC,UAAI,UAAU,WAAW;AAEvB;AAAA,MACF;AAGA,UAAI,SAAS,KAAK,SAAS;AAEzB;AAAA,MACF;AAGA,YAAM,wBAAwB;AAC9B,UAAI,YAAY,SAAS,uBAAuB;AAE9C;AAAA,MACF;AAEA,UAAI,CAAC,cAAc,IAAI,KAAK,EAAE,GAAG;AAC/B,aAAK,kBAAkB,MAAM,MAAM;AAAA,MACrC;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,kBAAkB,MAAY,eAA6B;AACjE,QAAI,CAAC,KAAK,cAAc;AACtB,cAAQ,KAAK,6DAA6D;AAC1E;AAAA,IACF;AAGA,UAAM,cAAc,KAAK,KAAK,aAAa;AAAA,MACzC,KAAK;AAAA,MACL;AAAA;AAAA,MACA,KAAK;AAAA;AAAA,IAAA;AAGP,QAAI,CAAC,eAAe,YAAY,OAAO,WAAW,GAAG;AAEnD,cAAQ,KAAK,+DAA+D,KAAK,EAAE;AACnF;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,iBAAiB,YAAY,QAAQ,YAAY,UAAU;AAE/E,UAAM,WAAW,KAAK,IAAI,GAAG,gBAAgB,KAAK,OAAO;AACzD,UAAM,gBAAgB,WAAW;AAGjC,UAAM,wBAAwB,OAAO,WAAW;AAIhD,UAAM,gCAAgC;AACtC,QAAI,wBAAwB,+BAA+B;AACzD;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,aAAa,mBAAA;AACjC,WAAO,SAAS;AAChB,WAAO,aAAa,QAAQ,KAAK;AAChC,WAAe,iBAAiB,KAAK;AACrC,WAAe,cAAc,KAAK,UAAU,OAAO,WAAW;AAE/D,UAAM,WAAW,KAAK,aAAa,WAAA;AAGnC,QAAI,eAAe,IAAI,GAAG;AACxB,YAAM,SAAS,KAAK,aAAa,UAAU;AAC3C,YAAM,QAAQ,KAAK,aAAa,SAAS;AACzC,eAAS,KAAK,QAAQ,QAAQ,IAAI,SAAS,KAAK;AAAA,IAClD,OAAO;AACL,eAAS,KAAK,QAAQ,KAAK;AAAA,IAC7B;AAEA,WAAO,QAAQ,QAAQ;AACvB,aAAS,QAAQ,KAAK,aAAa,WAAW;AAE9C,WAAO,MAAM,GAAG,eAAe,qBAAqB;AAEpD,WAAO,UAAU,MAAM;AACrB,YAAM,QAAQ,KAAK,aAAa,QAAQ,MAAM;AAC9C,UAAI,SAAS,GAAG;AACd,aAAK,aAAa,OAAO,OAAO,CAAC;AACjC,aAAK,eAAe,OAAO,OAAO,CAAC;AAAA,MACrC;AAGA,YAAM,aAAc,OAAe;AACnC,YAAM,YAAY,KAAK,UAAU,KAAK;AAEtC,UAAI,aAAa,aAAa,KAAK,WAAW;AAE5C,mBAAW,MAAM;AACf,cAAI,KAAK,WAAW;AAClB,iBAAK,qBAAqB,MAAM,UAAU;AAAA,UAC5C;AAAA,QACF,GAAG,EAAE;AAAA,MACP;AAAA,IACF;AAEA,SAAK,aAAa,KAAK,MAAM;AAC7B,SAAK,eAAe,KAAK,QAAQ;AAAA,EACnC;AAAA,EAEQ,qBAAqB,MAAY,QAAsB;AAC7D,QAAI,CAAC,KAAK,cAAc;AACtB;AAAA,IACF;AAGA,UAAM,cAAc,KAAK,KAAK,aAAa;AAAA,MACzC,KAAK;AAAA,MACL;AAAA;AAAA,MACA,KAAK;AAAA;AAAA,IAAA;AAGP,QAAI,CAAC,eAAe,YAAY,OAAO,WAAW,GAAG;AACnD;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,iBAAiB,YAAY,QAAQ,YAAY,UAAU;AAC/E,UAAM,cAAc,KAAK,UAAU,OAAO,WAAW;AAGrD,QAAI,eAAe,SAAS,KAAS;AAGnC;AAAA,IACF;AAGA,UAAM,SAAS,KAAK,aAAa,mBAAA;AACjC,WAAO,SAAS;AAChB,WAAO,aAAa,QAAQ,KAAK;AAChC,WAAe,iBAAiB,KAAK;AACrC,WAAe,cAAc;AAE9B,UAAM,WAAW,KAAK,aAAa,WAAA;AAGnC,QAAI,eAAe,IAAI,GAAG;AACxB,YAAM,SAAS,KAAK,aAAa,UAAU;AAC3C,YAAM,QAAQ,KAAK,aAAa,SAAS;AACzC,eAAS,KAAK,QAAQ,QAAQ,IAAI,SAAS,KAAK;AAAA,IAClD,OAAO;AACL,eAAS,KAAK,QAAQ,KAAK;AAAA,IAC7B;AAEA,WAAO,QAAQ,QAAQ;AACvB,aAAS,QAAQ,KAAK,aAAa,WAAW;AAE9C,UAAM,WAAW,KAAK,IAAI,GAAG,SAAS,KAAK,OAAO;AAClD,UAAM,gBAAgB,WAAW;AACjC,UAAM,wBAAwB,OAAO,WAAW;AAEhD,QAAI,yBAAyB,GAAG;AAC9B;AAAA,IACF;AAEA,WAAO,MAAM,GAAG,eAAe,qBAAqB;AAEpD,WAAO,UAAU,MAAM;AACrB,YAAM,QAAQ,KAAK,aAAa,QAAQ,MAAM;AAC9C,UAAI,SAAS,GAAG;AACd,aAAK,aAAa,OAAO,OAAO,CAAC;AACjC,aAAK,eAAe,OAAO,OAAO,CAAC;AAAA,MACrC;AAGA,YAAM,aAAc,OAAe;AACnC,YAAM,YAAY,KAAK,UAAU,KAAK;AAEtC,UAAI,aAAa,aAAa,KAAK,WAAW;AAC5C,mBAAW,MAAM;AACf,cAAI,KAAK,WAAW;AAClB,iBAAK,qBAAqB,MAAM,UAAU;AAAA,UAC5C;AAAA,QACF,GAAG,EAAE;AAAA,MACP;AAAA,IACF;AAEA,SAAK,aAAa,KAAK,MAAM;AAC7B,SAAK,eAAe,KAAK,QAAQ;AAAA,EACnC;AAAA,EAEQ,sBAA4B;AAClC,eAAW,UAAU,KAAK,cAAc;AACtC,UAAI;AACF,eAAO,KAAA;AACP,eAAO,WAAA;AAAA,MACT,SAAS,OAAO;AAAA,MAEhB;AAAA,IACF;AAEA,eAAW,YAAY,KAAK,gBAAgB;AAC1C,UAAI;AACF,iBAAS,WAAA;AAAA,MACX,SAAS,OAAO;AAAA,MAEhB;AAAA,IACF;AAEA,SAAK,eAAe,CAAA;AACpB,SAAK,iBAAiB,CAAA;AAAA,EACxB;AAAA,EAEQ,oBAAoB,QAAwB;AAClD,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,QAAI,CAAC,OAAO;AACV,aAAO,CAAA;AAAA,IACT;AAEA,UAAM,QAAgB,CAAA;AAGtB,eAAW,SAAS,MAAM,QAAQ;AAChC,YAAM,aAAa,MAAM,eAAe,QAAQ,MAAM,EAAE;AACxD,iBAAW,QAAQ,YAAY;AAE7B,YAAI,KAAK,KAAK,aAAa,WAAW,KAAK,EAAE,GAAG;AAC9C,gBAAM,KAAK,IAAI;AAAA,QACjB;AAAA,MACF;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,sBAAsB,QAA+B;AACzD,YAAQ,IAAI,8CAA8C,MAAM;AAChE,QAAI,KAAK,KAAK,aAAa,WAAW,MAAM,GAAG;AAC7C;AAAA,IACF;AACA,QAAI,KAAK,eAAe,IAAI,MAAM,GAAG;AACnC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,KAAK,SAAA;AACxB,UAAM,OAAO,OAAO,SAAS,MAAM;AACnC,QAAI,CAAC,KAAM;AAEX,UAAM,SAAS,MAAM,KAAK,KAAK,aAAa,cAAc,QAAQ,OAAO;AACzE,UAAM,cAAc,MAAM,KAAK,KAAK,aAAa,QAAQ,iBAAiB,QAAQ,OAAO;AACzF,QAAI,CAAC,UAAU,CAAC,aAAa;AAC3B;AAAA,IACF;AAEA,SAAK,eAAe,IAAI,MAAM;AAE9B,UAAM,UAAU,IAAI,kBAAkB,YAAY,MAAM,IAAI;AAAA,MAC1D,OAAO,QAAQ;AAAA,MACf,YAAY,QAAQ;AAAA,MACpB,kBAAkB,QAAQ;AAAA,MAC1B,aAAa,QAAQ;AAAA,IAAA,CACf;AAER,QAAI;AACF,YAAM,eAAe,YAAY,YAAY,QAAQ,cAAc;AACnE,YAAM,SAAS,aAAa,UAAA;AAC5B,YAAM,OAAO,YAA2B;AACtC,cAAM,EAAE,MAAM,MAAA,IAAU,MAAM,OAAO,KAAA;AACrC,YAAI,MAAM;AACR,iBAAO,YAAA;AACP;AAAA,QACF;AACA,YAAI,OAAO;AACT,eAAK,KAAK,aAAa,iBAAiB,QAAQ,OAAO,KAAK,UAAU;AAAA,QACxE;AACA,cAAM,KAAA;AAAA,MACR;AACA,YAAM,KAAA;AAAA,IACR,SAAS,OAAO;AACd,cAAQ,MAAM,qDAAqD,KAAK;AAAA,IAC1E,UAAA;AACE,WAAK,eAAe,OAAO,MAAM;AACjC,YAAM,QAAQ,MAAA;AAAA,IAChB;AAAA,EACF;AAAA,EAEQ,iBAAiB,QAAwB,YAAiC;AAChF,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO,CAAC,GAAG,UAAU;AAE5C,UAAM,MAAM,IAAI,oBAAoB,kBAAkB,GAAG,UAAU;AACnE,UAAM,SAAS,IAAI,aAAa,kBAAkB,gBAAgB,UAAU;AAE5E,aAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,YAAM,QAAQ,OAAO,OAAO;AAC5B,UAAI,OAAO;AACT,cAAM,cAAc,OAAO,eAAe,OAAO;AACjD,oBAAY,IAAI,KAAK;AAAA,MACvB;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA,EAEQ,uBAAuB,QAAqB,aAAuC;AACzF,UAAM,aAAa,OAAO;AAC1B,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO;AAE9B,UAAM,SAAyB,CAAA;AAC/B,aAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,aAAO,KAAK,OAAO,eAAe,OAAO,CAAC;AAAA,IAC5C;AAEA,WAAO,IAAI,UAAU;AAAA,MACnB,QAAQ;AAAA;AAAA,MACR;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW;AAAA,MACX,MAAM,KAAK,qBAAqB,MAAM;AAAA,IAAA,CACvC;AAAA,EACH;AAAA,EAEQ,qBAAqB,QAAqC;AAChE,UAAM,mBAAmB,OAAO;AAChC,UAAM,iBAAiB,OAAO,CAAC,GAAG,UAAU;AAC5C,UAAM,eAAe,mBAAmB;AAExC,UAAM,cAAc,IAAI,aAAa,YAAY;AAEjD,aAAS,QAAQ,GAAG,QAAQ,gBAAgB,SAAS;AACnD,eAAS,UAAU,GAAG,UAAU,kBAAkB,WAAW;AAC3D,oBAAY,QAAQ,mBAAmB,OAAO,IAAI,OAAO,OAAO,EAAG,KAAK;AAAA,MAC1E;AAAA,IACF;AAEA,WAAO,YAAY;AAAA,EACrB;AAAA,EAEA,MAAc,mBAAmB,MAAgC;AAC/D,UAAM,EAAE,IAAI,QAAQ,YAAY,SAAS,eAAe;AACxD,UAAM,mBAAmB,MAAM,KAAK,KAAK,QAAQ,YAAY,cAAc,QAAQ;AAAA,MACjF,MAAM;AAAA,IAAA,CACP;AACD,UAAM,oBAAoB,MAAM,KAAK,KAAK,QAAQ,YAAY,eAAe,QAAQ;AAAA,MACnF,MAAM;AAAA,IAAA,CACP;AAED,UAAM,uBAAuB,IAAI,eAAA;AACjC,UAAM,iBAAiB;AAAA,MACrB;AAAA,MACA,EAAE,WAAW,cAAc,MAAM,qBAAqB,OAAO,YAAY,SAAS,OAAA;AAAA,MAClF,EAAE,UAAU,CAAC,qBAAqB,KAAK,EAAA;AAAA,IAAE;AAE3C,UAAM,kBAAkB;AAAA,MACtB;AAAA,MACA;AAAA,QACE,WAAW;AAAA,QACX,MAAM,qBAAqB;AAAA,QAC3B,YAAY;AAAA,QACZ,WAAW;AAAA,QACX,aAAa,WAAW;AAAA,QACxB,gBAAgB,cAAc;AAAA,MAAA;AAAA,MAEhC,EAAE,UAAU,CAAC,qBAAqB,KAAK,EAAA;AAAA,IAAE;AAG3C,sBAAkB,cAAc,CAAC,QAAQ,aAAa;AACpD,WAAK,kBAAkB,QAAqC;AAAA,QAC1D,WAAW;AAAA,QACX,aAAa,WAAW;AAAA,QACxB,gBAAgB,cAAc;AAAA,QAC9B,GAAG;AAAA,MAAA,CACJ;AAAA,IACH,CAAC;AAED,UAAM,cAAc,KAAK,KAAK,mBAAA,EAAqB;AACnD,UAAM,iBAAiB,KAAK,aAAa;AAAA,MACvC,SAAS;AAAA,MACT;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IAAA,CACT;AAAA,EACH;AACF;"}
|
|
@@ -7,7 +7,7 @@ import { WorkerStatus, WorkerType } from '../worker/types';
|
|
|
7
7
|
import { CompositionModel, CompositionPatch, TimeUs, RcFrame } from '../model';
|
|
8
8
|
import { EventPayloadMap } from '../event/events';
|
|
9
9
|
import { CompositionPlanner } from './CompositionPlanner';
|
|
10
|
-
import { GlobalAudioSession } from '
|
|
10
|
+
import { GlobalAudioSession } from './GlobalAudioSession';
|
|
11
11
|
import { MuxManager } from '../stages/mux/MuxManager';
|
|
12
12
|
import { ExportOptions } from '../types';
|
|
13
13
|
|
|
@@ -39,6 +39,7 @@ export declare class Orchestrator implements IOrchestrator {
|
|
|
39
39
|
private handleResourceStateChange;
|
|
40
40
|
restartWorker(type: WorkerType, clipId?: string): Promise<void>;
|
|
41
41
|
renderFrame(timeUs: TimeUs, options?: RenderFrameOptions): Promise<RcFrame | null>;
|
|
42
|
+
private ensureAudioFromL2;
|
|
42
43
|
private decodeFromL2;
|
|
43
44
|
/**
|
|
44
45
|
* Ensure clips are cached using 2-Clip strategy
|
|
@@ -55,14 +56,7 @@ export declare class Orchestrator implements IOrchestrator {
|
|
|
55
56
|
minFrameCount?: number;
|
|
56
57
|
timeoutMs?: number;
|
|
57
58
|
}): Promise<boolean>;
|
|
58
|
-
/**
|
|
59
|
-
* Render a clip completely for L2 cache (bypass ClipSessionManager)
|
|
60
|
-
* Returns a promise that resolves when encoding is complete
|
|
61
|
-
*/
|
|
62
59
|
renderClipForL2(clipId: string): Promise<boolean>;
|
|
63
|
-
/**
|
|
64
|
-
* Create a new session for a clip
|
|
65
|
-
*/
|
|
66
60
|
private createSession;
|
|
67
61
|
dispose(): Promise<void>;
|
|
68
62
|
private buildWorkerConfigs;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Orchestrator.d.ts","sourceRoot":"","sources":["../../src/orchestrator/Orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAyB,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACtF,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AACrF,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAY,MAAM,EAAE,OAAO,EAAQ,MAAM,UAAU,CAAC;AAE/F,OAAO,EAAgB,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"Orchestrator.d.ts","sourceRoot":"","sources":["../../src/orchestrator/Orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAyB,cAAc,EAAE,MAAM,+BAA+B,CAAC;AACtF,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AACrF,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAY,MAAM,EAAE,OAAO,EAAQ,MAAM,UAAU,CAAC;AAE/F,OAAO,EAAgB,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAIxC,qBAAa,YAAa,YAAW,aAAa;IAChD,OAAO,EAAE,UAAU,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC,eAAe,CAAC,CAAC;IACpC,gBAAgB,EAAE,gBAAgB,GAAG,IAAI,CAAQ;IACjD,cAAc,EAAE,cAAc,CAAC;IAC/B,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,EAAE,kBAAkB,CAAC;IAC5B,YAAY,EAAE,kBAAkB,CAAC;IACjC,UAAU,EAAE,UAAU,CAAC;IAEvB,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,MAAM,CAA0C;IACxD,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,wBAAwB,CAAuB;IACvD,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAO;IAChD,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC,CAAC;gBAE5D,MAAM,EAAE,kBAAkB;IAqEtC,IAAI,YAAY,IAAI,YAAY,CAsB/B;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IASjC,EAAE,CAAC,CAAC,SAAS,MAAM,eAAe,EAChC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,IAAI,GAC7C,IAAI;IAIP,GAAG,CAAC,CAAC,SAAS,MAAM,eAAe,EACjC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,IAAI,GAC7C,IAAI;IAIP,IAAI,CAAC,CAAC,SAAS,MAAM,eAAe,EAClC,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC,CAAC,KAAK,IAAI,GAC7C,IAAI;IAID,mBAAmB,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB3D,UAAU,CAAC,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoDxD,OAAO,CAAC,yBAAyB;IAiD3B,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA+B/D,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC;YA4D1E,iBAAiB;YAiBjB,YAAY;IA0C1B;;;;;OAKG;IACG,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BvE;;;OAGG;IACG,gBAAgB,CACpB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GACvD,OAAO,CAAC,OAAO,CAAC;IA0Bb,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;YAoCzC,aAAa;IAyGrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAkB9B,OAAO,CAAC,kBAAkB;IA0CpB,MAAM,CAAC,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;CAG7E"}
|
|
@@ -9,7 +9,7 @@ import { MeframeEvent } from "../event/events.js";
|
|
|
9
9
|
import { CompositionPlanner } from "./CompositionPlanner.js";
|
|
10
10
|
import { VideoClipSession } from "./VideoClipSession.js";
|
|
11
11
|
import { ClipSessionManager } from "./ClipSessionManager.js";
|
|
12
|
-
import { GlobalAudioSession } from "
|
|
12
|
+
import { GlobalAudioSession } from "./GlobalAudioSession.js";
|
|
13
13
|
import { MuxManager } from "../stages/mux/MuxManager.js";
|
|
14
14
|
import { VideoChunkDecoder } from "../stages/decode/VideoChunkDecoder.js";
|
|
15
15
|
import { quantizeTimestampToFrame } from "../utils/time-utils.js";
|
|
@@ -267,10 +267,24 @@ class Orchestrator {
|
|
|
267
267
|
}
|
|
268
268
|
const l2Frame = await this.decodeFromL2(relativeTimeUs, clip);
|
|
269
269
|
if (l2Frame) {
|
|
270
|
+
void this.ensureAudioFromL2(clip.id);
|
|
270
271
|
return l2Frame;
|
|
271
272
|
}
|
|
272
273
|
return null;
|
|
273
274
|
}
|
|
275
|
+
async ensureAudioFromL2(clipId) {
|
|
276
|
+
try {
|
|
277
|
+
if (this.cacheManager.hasClipPCM(clipId)) {
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const hasAudio = await this.cacheManager.hasClipInL2(clipId, "audio");
|
|
281
|
+
if (!hasAudio) return;
|
|
282
|
+
await this.audioSession.ensureClipAudioFromL2(clipId);
|
|
283
|
+
} catch (error) {
|
|
284
|
+
console.warn("[Orchestrator] ensureAudioFromL2IfNeeded error:", error);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
// TODO: move to @ClipSessionManager
|
|
274
288
|
async decodeFromL2(timeUs, clip) {
|
|
275
289
|
const { id, trackId, startUs } = clip;
|
|
276
290
|
const [chunkStream, metadata] = await Promise.all([
|
|
@@ -356,10 +370,7 @@ class Orchestrator {
|
|
|
356
370
|
// Don't pass startTimeUs - count all frames in the clip
|
|
357
371
|
});
|
|
358
372
|
}
|
|
359
|
-
|
|
360
|
-
* Render a clip completely for L2 cache (bypass ClipSessionManager)
|
|
361
|
-
* Returns a promise that resolves when encoding is complete
|
|
362
|
-
*/
|
|
373
|
+
// TODO: move to @ClipSessionManager
|
|
363
374
|
async renderClipForL2(clipId) {
|
|
364
375
|
const sessionId = `${clipId}#l2`;
|
|
365
376
|
let session = null;
|
|
@@ -389,9 +400,7 @@ class Orchestrator {
|
|
|
389
400
|
});
|
|
390
401
|
});
|
|
391
402
|
}
|
|
392
|
-
|
|
393
|
-
* Create a new session for a clip
|
|
394
|
-
*/
|
|
403
|
+
// TODO: move to @ClipSessionManager
|
|
395
404
|
async createSession(sessionId, options) {
|
|
396
405
|
const clipId = options?.clipId ?? sessionId;
|
|
397
406
|
const clip = this.compositionModel?.findClip(clipId);
|
|
@@ -448,7 +457,6 @@ class Orchestrator {
|
|
|
448
457
|
}
|
|
449
458
|
},
|
|
450
459
|
onAudioStreamReady: (stream, metadata) => {
|
|
451
|
-
console.log("[Orchestrator] Received audio stream", metadata, options?.forL2Only);
|
|
452
460
|
if (options?.forL2Only) {
|
|
453
461
|
stream.cancel();
|
|
454
462
|
} else {
|