@editframe/elements 0.17.6-beta.0 → 0.18.3-beta.0

package/src/gui/services/PlaybackController.ts

@@ -0,0 +1,521 @@
+export interface PlaybackControllerOptions {
+  fps?: number;
+  onTimeUpdate?: (timeMs: number) => void;
+  onPlayStateChange?: (playing: boolean) => void;
+  onError?: (error: Error) => void;
+}
+
+/**
+ * Manages playback timing, AudioContext lifecycle, and timeline synchronization
+ * Extracted from ContextMixin to improve separation of concerns and testability
+ */
+export class PlaybackController {
+  private playbackAudioContext: AudioContext | null = null;
+  private animationFrameRequest: number | null = null;
+  private options: Required<PlaybackControllerOptions>;
+  private playing = false;
+  private currentTimeMs = 0;
+  private msPerFrame: number;
+  private audioStartTime = 0;
+  private playbackStartTimeMs = 0; // Track the actual timeline position where playback started
+
+  // Progressive chunking properties
+  private activeChunks = new Map<number, AudioBufferSourceNode>();
+  private chunkDurationMs = 4000; // 4-second chunks
+  private lookaheadChunks = 2; // Always have 2 chunks ready
+  private currentChunkIndex = 0;
+  private renderingChunks = new Set<number>(); // Track chunks being rendered
+
+  constructor(options: PlaybackControllerOptions = {}) {
+    this.options = {
+      fps: 30,
+      onTimeUpdate: () => {},
+      onPlayStateChange: () => {},
+      onError: () => {},
+      ...options,
+    };
+    this.msPerFrame = 1000 / this.options.fps;
+  }
+
+  /**
+   * Start playback for the given timegroup
+   */
+  async startPlayback(timegroup: any, fromMs?: number): Promise<void> {
+    await this.stopPlayback();
+
+    if (!timegroup) {
+      this.setPlaying(false);
+      this.options.onPlayStateChange(false); // Always notify on failed start attempt
+      this.options.onError(new Error("No timegroup provided"));
+      return;
+    }
+
+    await timegroup.waitForMediaDurations?.();
+
+    const currentMs = fromMs ?? timegroup.currentTimeMs ?? 0;
+    const toMs = timegroup.endTimeMs;
+
+    if (currentMs >= toMs) {
+      this.setPlaying(false);
+      this.options.onPlayStateChange(false); // Always notify on failed start attempt
+      return;
+    }
+
+    try {
+      this.playbackAudioContext = new AudioContext({
+        latencyHint: "playback",
+      });
+
+      if (this.playbackAudioContext.state === "suspended") {
+        console.warn("AudioContext is suspended, attempting to resume...");
+
+        try {
+          // Add timeout for resume operation to prevent hanging in browser tests
+          await Promise.race([
+            this.playbackAudioContext.resume(),
+            new Promise((_, reject) =>
+              setTimeout(
+                () => reject(new Error("AudioContext resume timeout")),
+                2000,
+              ),
+            ),
+          ]);
+        } catch (error) {
+          console.warn("AudioContext resume failed:", error);
+          // Continue anyway - in test environments this is often expected
+        }
+      } else {
+        await this.playbackAudioContext.resume();
+      }
+
+      // Initialize progressive playback
+      this.audioStartTime = this.playbackAudioContext.currentTime;
+      this.playbackStartTimeMs = currentMs; // Store where we started in the timeline
+      this.currentChunkIndex = Math.floor(currentMs / this.chunkDurationMs);
+
+      // Start progressive chunk rendering and playback
+      await this.startProgressivePlayback(timegroup, currentMs, toMs);
+
+      // Set playing state based on successful setup
+      // In browser test environments, AudioContext may remain suspended but playback should still be considered active
+      if (this.isAudioContextReady()) {
+        this.setPlaying(true);
+        // Ensure onTimeUpdate is called at least once during startup
+        this.currentTimeMs = currentMs;
+        this.options.onTimeUpdate(currentMs);
+        this.syncPlayheadToAudioBuffer(timegroup, currentMs);
+      } else {
+        this.setPlaying(false);
+        this.options.onPlayStateChange(false); // Always notify on failed start attempt
+        console.warn(
+          "AudioContext not ready for playback, state:",
+          this.playbackAudioContext?.state,
+        );
+      }
+    } catch (error) {
+      console.error(
+        "🎵 [PLAYBACK_ERROR] Failed to setup progressive audio playback:",
+        error,
+      );
+      this.setPlaying(false);
+      this.options.onPlayStateChange(false); // Always notify on failed start attempt
+      this.options.onError(error as Error);
+    }
+  }
+
+  /**
+   * Stop playback and clean up resources
+   */
+  async stopPlayback(): Promise<void> {
+    // Stop all active chunks
+    for (const [_chunkIndex, bufferSource] of this.activeChunks.entries()) {
+      try {
+        bufferSource.stop();
+      } catch (_error) {
+        // Ignore errors when stopping already stopped sources
+      }
+    }
+    this.activeChunks.clear();
+    this.renderingChunks.clear();
+
+    if (this.playbackAudioContext) {
+      if (this.playbackAudioContext.state !== "closed") {
+        await this.playbackAudioContext.close();
+      }
+    }
+
+    if (this.animationFrameRequest) {
+      cancelAnimationFrame(this.animationFrameRequest);
+      this.animationFrameRequest = null;
+    }
+
+    this.playbackAudioContext = null;
+    this.setPlaying(false);
+  }
+
+  /**
+   * Pause playback (can be resumed)
+   */
+  async pausePlayback(): Promise<void> {
+    if (
+      this.playbackAudioContext &&
+      this.playbackAudioContext.state === "running"
+    ) {
+      try {
+        // Add timeout for suspend operation to prevent hanging in browser tests
+        await Promise.race([
+          this.playbackAudioContext.suspend(),
+          new Promise((_, reject) =>
+            setTimeout(
+              () => reject(new Error("AudioContext suspend timeout")),
+              2000,
+            ),
+          ),
+        ]);
+      } catch (error) {
+        console.warn("AudioContext suspend failed:", error);
+        // Continue anyway - in test environments this is often expected
+      }
+    }
+
+    if (this.animationFrameRequest) {
+      cancelAnimationFrame(this.animationFrameRequest);
+      this.animationFrameRequest = null;
+    }
+
+    this.setPlaying(false);
+  }
+
+  /**
+   * Resume paused playback
+   */
+  async resumePlayback(): Promise<void> {
+    if (
+      this.playbackAudioContext &&
+      this.playbackAudioContext.state === "suspended"
+    ) {
+      try {
+        // Add timeout for resume operation to prevent hanging in browser tests
+        await Promise.race([
+          this.playbackAudioContext.resume(),
+          new Promise((_, reject) =>
+            setTimeout(
+              () => reject(new Error("AudioContext resume timeout")),
+              2000,
+            ),
+          ),
+        ]);
+      } catch (error) {
+        console.warn("AudioContext resume failed:", error);
+        // Continue anyway - in test environments this is often expected
+      }
+      this.setPlaying(true);
+    }
+  }
+
+  /**
+   * Seek to a specific time (restarts progressive playback from new position)
+   */
+  async seekTo(timeMs: number, timegroup?: any): Promise<void> {
+    this.currentTimeMs = timeMs;
+    this.options.onTimeUpdate(timeMs);
+
+    // For progressive chunks, we need to restart playback from the new position
+    if (this.playing && timegroup) {
+      // Stop current chunks
+      for (const bufferSource of this.activeChunks.values()) {
+        try {
+          bufferSource.stop();
+        } catch (_error) {
+          // Ignore stop errors
+        }
+      }
+      this.activeChunks.clear();
+      this.renderingChunks.clear();
+
+      // Restart progressive playback from new position with proper timing
+      this.audioStartTime = this.playbackAudioContext?.currentTime ?? 0;
+      this.playbackStartTimeMs = timeMs; // Set the new playback start position
+      this.currentChunkIndex = Math.floor(timeMs / this.chunkDurationMs);
+
+      await this.startProgressivePlayback(
+        timegroup,
+        timeMs,
+        timegroup.endTimeMs,
+      );
+    }
+  }
+
+  /**
+   * Internal method to sync playhead with unified audio buffer timing
+   */
+  private syncPlayheadToAudioBuffer(timegroup: any, startMs: number): void {
+    if (!this.playbackAudioContext || !this.playing) {
+      return;
+    }
+
+    const elapsedAudioTime =
+      this.playbackAudioContext.currentTime - this.audioStartTime;
+    const rawTimeMs = startMs + elapsedAudioTime * 1000;
+    const nextTimeMs =
+      Math.round(rawTimeMs / this.msPerFrame) * this.msPerFrame;
+
+    if (nextTimeMs !== this.currentTimeMs) {
+      this.currentTimeMs = nextTimeMs;
+      this.options.onTimeUpdate(nextTimeMs);
+
+      // Update the timegroup's currentTimeMs, which will automatically
+      // sync all child elements (video, animations, etc.)
+      if (timegroup && timegroup.currentTimeMs !== nextTimeMs) {
+        timegroup.currentTimeMs = nextTimeMs;
+      }
+
+      // Update progressive chunks as playhead advances
+      this.updateProgressiveChunks(timegroup, nextTimeMs, timegroup.endTimeMs);
+    }
+
+    this.animationFrameRequest = requestAnimationFrame(() => {
+      this.syncPlayheadToAudioBuffer(timegroup, startMs);
+    });
+  }
+
+  /**
+   * Update playing state and notify observers
+   */
+  private setPlaying(playing: boolean): void {
+    if (this.playing !== playing) {
+      this.playing = playing;
+      this.options.onPlayStateChange(playing);
+    }
+  }
+
+  /**
+   * Get current playback state
+   */
+  isPlaying(): boolean {
+    return this.playing;
+  }
+
+  /**
+   * Get current time
+   */
+  getCurrentTime(): number {
+    return this.currentTimeMs;
+  }
+
+  /**
+   * Get current AudioContext
+   */
+  getAudioContext(): AudioContext | null {
+    return this.playbackAudioContext;
+  }
+
+  /**
+   * Check if AudioContext is ready
+   */
+  isAudioContextReady(): boolean {
+    // In test environments, AudioContext might remain suspended, so be more lenient
+    return (
+      this.playbackAudioContext != null &&
+      this.playbackAudioContext.state !== "closed"
+    );
+  }
+
+  /**
+   * Get playback statistics for debugging
+   */
+  getPlaybackInfo(): {
+    playing: boolean;
+    currentTimeMs: number;
+    audioContextState: string | null;
+    hasElementManager: boolean;
+  } {
+    return {
+      playing: this.playing,
+      currentTimeMs: this.currentTimeMs,
+      audioContextState: this.playbackAudioContext?.state || null,
+      hasElementManager: false, // No longer tracking ElementConnectionManager
+    };
+  }
+
+  /**
+   * Update playback options
+   */
+  updateOptions(options: Partial<PlaybackControllerOptions>): void {
+    Object.assign(this.options, options);
+    if (options.fps) {
+      this.msPerFrame = 1000 / options.fps;
+    }
+  }
+
+  /**
+   * Start progressive chunk rendering and playback
+   */
+  private async startProgressivePlayback(
+    timegroup: any,
+    fromMs: number,
+    _toMs: number,
+  ): Promise<void> {
+    // Calculate the starting chunk and offset
+    const firstChunkIndex = Math.floor(fromMs / this.chunkDurationMs);
+    const firstChunkStart = firstChunkIndex * this.chunkDurationMs;
+    const offsetInChunk = fromMs - firstChunkStart;
+
+    // Render the first chunk immediately to ensure it starts playing right away
+    await this.renderAndScheduleChunk(
+      timegroup,
+      firstChunkStart,
+      firstChunkIndex,
+      offsetInChunk,
+    );
+  }
+
+  /**
+   * Render and schedule a single audio chunk
+   */
+  private async renderAndScheduleChunk(
+    timegroup: any,
+    chunkStartMs: number,
+    chunkIndex: number,
+    offsetInChunk = 0,
+  ): Promise<void> {
+    if (
+      this.renderingChunks.has(chunkIndex) ||
+      this.activeChunks.has(chunkIndex)
+    ) {
+      return; // Already rendering or scheduled
+    }
+
+    this.renderingChunks.add(chunkIndex);
+
+    try {
+      const chunkEndMs = chunkStartMs + this.chunkDurationMs;
+
+      // Render the chunk using the existing audio composition logic
+      const chunkBuffer = await timegroup.renderAudio(chunkStartMs, chunkEndMs);
+
+      // Create and schedule the buffer source
+      const bufferSource = this.playbackAudioContext?.createBufferSource();
+      if (!bufferSource || !this.playbackAudioContext?.destination) {
+        throw new Error("Audio context or buffer source not available");
+      }
+      bufferSource.buffer = chunkBuffer;
+      bufferSource.connect(this.playbackAudioContext.destination);
+
+      // Calculate precise timing: chunks should be scheduled relative to playback start
+      const chunkTimelineStartMs = chunkIndex * this.chunkDurationMs;
+      const relativeDelayMs = Math.max(
+        0,
+        chunkTimelineStartMs - this.playbackStartTimeMs,
+      );
+      const chunkStartTime = this.audioStartTime + relativeDelayMs / 1000;
+      const startOffset = offsetInChunk / 1000;
+
+      // Schedule the chunk with precise timing
+      const now = this.playbackAudioContext?.currentTime ?? 0;
+      if (chunkStartTime <= now) {
+        console.warn(
+          `🎵 [CHUNK_TIMING_WARNING] Chunk ${chunkIndex} scheduled in the past! startTime=${chunkStartTime.toFixed(3)}s, currentTime=${now.toFixed(3)}s`,
+        );
+      }
+
+      bufferSource.start(chunkStartTime, startOffset);
+      this.activeChunks.set(chunkIndex, bufferSource);
+
+      // Handle chunk completion and trigger next chunk rendering
+      bufferSource.onended = () => {
+        this.activeChunks.delete(chunkIndex);
+
+        // Don't trigger more rendering here to avoid race conditions
+        // Let the sync loop handle chunk advancement
+      };
+
+      // Remove automatic async rendering to eliminate race conditions
+      // this.ensureAheadChunksReady(timegroup, chunkIndex);
+    } catch (error) {
+      console.error(
+        `🎵 [CHUNK_ERROR] Failed to render chunk ${chunkIndex}:`,
+        error,
+      );
+    } finally {
+      this.renderingChunks.delete(chunkIndex);
+    }
+  }
+
+  /**
+   * Update chunk rendering as playhead advances - now handles all chunk management
+   */
+  private updateProgressiveChunks(
+    timegroup: any,
+    currentTimeMs: number,
+    maxTimeMs: number,
+  ): void {
+    const newChunkIndex = Math.floor(currentTimeMs / this.chunkDurationMs);
+
+    if (newChunkIndex !== this.currentChunkIndex) {
+      this.currentChunkIndex = newChunkIndex;
+
+      // Cleanup old chunks
+      this.cleanupOldChunks();
+    }
+
+    // Always ensure we have enough chunks ahead (check every frame)
+    this.ensureChunksAhead(timegroup, maxTimeMs);
+  }
+
+  /**
+   * Systematically ensure chunks are ready ahead of current playback (synchronous)
+   */
+  private ensureChunksAhead(timegroup: any, maxTimeMs: number): void {
+    // Start from chunk 1 ahead of current (chunk 0 was handled at startup)
+    for (let i = 1; i <= this.lookaheadChunks; i++) {
+      const targetChunkIndex = this.currentChunkIndex + i;
+      const targetChunkStartMs = targetChunkIndex * this.chunkDurationMs;
+
+      // Don't render beyond timegroup end
+      if (targetChunkStartMs >= maxTimeMs) {
+        break;
+      }
+
+      // Check if this chunk needs to be rendered
+      if (
+        !this.renderingChunks.has(targetChunkIndex) &&
+        !this.activeChunks.has(targetChunkIndex)
+      ) {
+        // Future chunks don't need offset since they start from the beginning
+        const offsetInChunk = 0;
+
+        // Render chunk asynchronously to avoid blocking the sync loop
+        this.renderAndScheduleChunk(
+          timegroup,
+          targetChunkStartMs,
+          targetChunkIndex,
+          offsetInChunk,
+        ).catch((error) => {
+          console.error(
+            `🎵 [ENSURE_CHUNKS_ERROR] Failed to render chunk ${targetChunkIndex}:`,
+            error,
+          );
+        });
+      }
+    }
+  }
+
+  /**
+   * Clean up chunks that are behind the current playhead
+   */
+  private cleanupOldChunks(): void {
+    const cutoffChunkIndex = this.currentChunkIndex - 1; // Keep 1 chunk behind for safety
+
+    for (const [chunkIndex, bufferSource] of this.activeChunks.entries()) {
+      if (chunkIndex < cutoffChunkIndex) {
+        try {
+          bufferSource.stop();
+        } catch (_error) {
+          // Ignore stop errors for already stopped sources
+        }
+        this.activeChunks.delete(chunkIndex);
+      }
+    }
+  }
+}