@editframe/elements 0.17.6-beta.0 → 0.18.3-beta.0

package/src/elements/EFVideo.ts

@@ -1,14 +1,19 @@
-import { VideoAsset } from "@editframe/assets/EncodedAsset.js";
 import { Task } from "@lit/task";
 import debug from "debug";
 import { css, html, type PropertyValueMap } from "lit";
-import { customElement, state } from "lit/decorators.js";
+import { customElement, property, state } from "lit/decorators.js";
 import { createRef, ref } from "lit/directives/ref.js";
+
 import { DelayedLoadingState } from "../DelayedLoadingState.js";
 import { TWMixin } from "../gui/TWMixin.js";
-import { type CacheStats, ScrubTrackManager } from "../ScrubTrackManager.js";
+import type { CacheStats, ScrubTrackManager } from "../ScrubTrackManager.js";
+import { makeVideoBufferTask } from "./EFMedia/videoTasks/makeVideoBufferTask.ts";
+import { makeVideoInitSegmentFetchTask } from "./EFMedia/videoTasks/makeVideoInitSegmentFetchTask.ts";
+import { makeVideoInputTask } from "./EFMedia/videoTasks/makeVideoInputTask.ts";
+import { makeVideoSeekTask } from "./EFMedia/videoTasks/makeVideoSeekTask.ts";
+import { makeVideoSegmentFetchTask } from "./EFMedia/videoTasks/makeVideoSegmentFetchTask.ts";
+import { makeVideoSegmentIdTask } from "./EFMedia/videoTasks/makeVideoSegmentIdTask.ts";
 import { EFMedia } from "./EFMedia.js";
-import { printTaskStatus } from "./printTaskStatus.ts";
 
 // EF_FRAMEGEN is a global instance created in EF_FRAMEGEN.ts
 declare global {
@@ -25,6 +30,16 @@ interface LoadingState {
 
 @customElement("ef-video")
 export class EFVideo extends TWMixin(EFMedia) {
+  static get observedAttributes() {
+    const parentAttributes = EFMedia.observedAttributes || [];
+    return [
+      ...parentAttributes,
+      "video-buffer-duration",
+      "max-video-buffer-fetches",
+      "enable-video-buffering",
+    ];
+  }
+
   static styles = [
     /**
      *
@@ -92,14 +107,38 @@ export class EFVideo extends TWMixin(EFMedia) {
   canvasRef = createRef<HTMLCanvasElement>();
 
   /**
-   * Scrub track manager for fast timeline navigation
+   * Duration in milliseconds for video buffering ahead of current time
+   * @domAttribute "video-buffer-duration"
    */
-  scrubTrackManager?: ScrubTrackManager;
+  @property({ type: Number, attribute: "video-buffer-duration" })
+  videoBufferDurationMs = 60000; // 60 seconds
 
   /**
-   * Track last seek time for fast seeking detection
+   * Maximum number of concurrent video segment fetches for buffering
+   * @domAttribute "max-video-buffer-fetches"
    */
-  private lastSeekTimeMs = 0;
+  @property({ type: Number, attribute: "max-video-buffer-fetches" })
+  maxVideoBufferFetches = 2;
+
+  /**
+   * Enable/disable video buffering system
+   * @domAttribute "enable-video-buffering"
+   */
+  @property({ type: Boolean, attribute: "enable-video-buffering" })
+  enableVideoBuffering = true;
+
+  // Video-specific tasks
+  videoSegmentIdTask = makeVideoSegmentIdTask(this);
+  videoInitSegmentFetchTask = makeVideoInitSegmentFetchTask(this);
+  videoSegmentFetchTask = makeVideoSegmentFetchTask(this);
+  videoInputTask = makeVideoInputTask(this);
+  videoSeekTask = makeVideoSeekTask(this);
+  videoBufferTask = makeVideoBufferTask(this);
+
+  /**
+   * Scrub track manager for fast timeline navigation
+   */
+  scrubTrackManager?: ScrubTrackManager;
 
   /**
    * Delayed loading state manager for user feedback
@@ -153,35 +192,14 @@ export class EFVideo extends TWMixin(EFMedia) {
     return this.canvasRef.value;
   }
 
-  // The underlying video decoder MUST NOT be used concurrently.
-  // If frames are fed in out of order, the decoder may crash.
-  #decoderLock = false;
-
-  // Track if decoder needs reset due to errors
-  #decoderNeedsReset = false;
-
   frameTask = new Task(this, {
     args: () => [this.desiredSeekTimeMs] as const,
     onError: (error) => {
       console.error("frameTask error", error);
     },
+    onComplete: () => {},
     task: async ([_desiredSeekTimeMs], { signal }) => {
-      await this.seekTask.taskComplete;
-      if (signal.aborted) {
-        return;
-      }
-      await this.fragmentIndexTask.taskComplete;
-      if (signal.aborted) {
-        return;
-      }
-      await this.mediaSegmentsTask.taskComplete;
-      if (signal.aborted) {
-        return;
-      }
-      await this.videoAssetTask.taskComplete;
-      if (signal.aborted) {
-        return;
-      }
+      await this.videoSeekTask.taskComplete;
       await this.paintTask.taskComplete;
       if (signal.aborted) {
         return;
@@ -189,70 +207,10 @@ export class EFVideo extends TWMixin(EFMedia) {
     },
   });
 
-  get frameTaskStatus() {
-    return {
-      desiredSeekTimeMs: this.desiredSeekTimeMs,
-      fragmentIndexTask: printTaskStatus(this.fragmentIndexTask.status),
-      seekTask: printTaskStatus(this.seekTask.status),
-      mediaSegmentsTask: printTaskStatus(this.mediaSegmentsTask.status),
-      assetSegmentLoader: printTaskStatus(this.assetSegmentLoader.status),
-      assetSegmentKeysTask: printTaskStatus(this.assetSegmentKeysTask.status),
-      assetInitSegmentsTask: printTaskStatus(this.assetInitSegmentsTask.status),
-      videoAssetTask: printTaskStatus(this.videoAssetTask.status),
-      paintTask: printTaskStatus(this.paintTask.status),
-      frameTask: printTaskStatus(this.frameTask.status),
-    };
-  }
-
-  #lastVideoAsset: any = null;
-
   protected updated(
     changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
   ): void {
     super.updated(changedProperties);
-
-    const currentVideoAsset = this.videoAssetTask.value;
-    if (currentVideoAsset !== this.#lastVideoAsset) {
-      // Track video asset changes for reference, but don't reset decoder
-      // Decoder resets should only happen due to actual decoder errors, not normal asset transitions
-      this.#lastVideoAsset = currentVideoAsset;
-    }
-
-    // Initialize scrub track manager for JIT transcode mode
-    this.initializeScrubTrackManager();
-  }
-
-  /**
-   * Initialize scrub track manager if needed
-   */
-  private async initializeScrubTrackManager(): Promise<void> {
-    const mode = this.effectiveMode;
-
-    // Only initialize for JIT transcode mode with valid src
-    if (mode === "jit-transcode" && this.src && !this.scrubTrackManager) {
-      const jitClient = this.jitClientTask.value;
-      if (jitClient) {
-        try {
-          this.scrubTrackManager = new ScrubTrackManager(this.src, jitClient, {
-            onLoadingStateChange: (isLoading: boolean, message?: string) => {
-              if (isLoading) {
-                // Only show loading for user-visible operations (non-background)
-                this.startDelayedLoading(
-                  "scrub-segment",
-                  message || "Loading scrub track...",
-                );
-              } else {
-                this.clearDelayedLoading("scrub-segment");
-              }
-            },
-          });
-
-          await this.scrubTrackManager.initialize();
-        } catch (error) {
-          console.warn("Failed to initialize scrub track manager:", error);
-        }
-      }
-    }
   }
 
   /**
@@ -288,107 +246,24 @@ export class EFVideo extends TWMixin(EFMedia) {
     };
   }
 
-  videoAssetTask = new Task(this, {
-    autoRun: true,
-    args: () => [this.effectiveMode, this.mediaSegmentsTask.value] as const,
-    onError: (error) => {
-      console.error("videoAsset task error", error);
-    },
-    task: async ([mode, _files], { signal: _signal }) => {
-      await this.mediaSegmentsTask.taskComplete;
-      if (_signal.aborted) {
-        return undefined;
-      }
-
-      await this.fragmentIndexTask.taskComplete;
-      if (_signal.aborted) {
-        return undefined;
-      }
-
-      // Get fresh values
-      const files = this.mediaSegmentsTask.value;
-      const fragmentIndex = this.fragmentIndexTask.value;
-
-      if (!files) {
-        log("trace: videoAsset task aborted - no files");
-        throw new Error(
-          `Video asset creation failed: No media segment files available. This indicates a problem with media segment loading for source: "${this.src}"`,
-        );
-      }
-
-      const computedVideoTrackId = Object.values(fragmentIndex ?? {}).find(
-        (track) => track.type === "video",
-      )?.track;
-
-      if (computedVideoTrackId === undefined) {
-        log("trace: videoAsset task aborted - no video track");
-        throw new Error(
-          `Video asset creation failed: No video track found in media segments. Source may not contain video content: "${this.src}"`,
-        );
-      }
-
-      const videoFile = files[computedVideoTrackId];
-      if (!videoFile) {
-        log("trace: videoAsset task aborted - no video file");
-        throw new Error(
-          `Video asset creation failed: Video file not available for track ${computedVideoTrackId}. Media segment loading may have failed for source: "${this.src}"`,
-        );
-      }
-
-      // Cleanup existing asset
-      const existingAsset = this.videoAssetTask.value;
-      if (existingAsset) {
-        for (const frame of existingAsset?.decodedFrames || []) {
-          frame.close();
-        }
-        const decoder = existingAsset?.videoDecoder;
-        if (decoder && decoder.state !== "closed") {
-          decoder.close();
-        }
-      }
-
-      if (_signal.aborted) {
-        return undefined;
-      }
-
-      log("trace: creating video asset", { mode });
-
-      // Get start time offset from fragment index (timing correction for FFmpeg processing)
-      const videoTrackFragmentIndex = Object.values(fragmentIndex ?? {}).find(
-        (track) => track.type === "video",
-      );
-      const startTimeOffsetMs = Number(
-        (videoTrackFragmentIndex?.startTimeOffsetMs ?? 0).toFixed(5),
-      );
-
-      // Single branching point for creation method
-      if (mode === "jit-transcode") {
-        const result = await VideoAsset.createFromCompleteMP4(
-          `jit-segment-${computedVideoTrackId}`,
-          videoFile,
-          { startTimeOffsetMs },
-        );
-        return result;
-      }
-      const result = await VideoAsset.createFromReadableStream(
-        "video.mp4",
-        videoFile.stream(),
-        videoFile,
-        { startTimeOffsetMs },
-      );
-      return result;
-    },
-  });
-
   paintTask = new Task(this, {
     args: () => [this.desiredSeekTimeMs] as const,
     onError: (error) => {
       console.error("paintTask error", error);
     },
+    onComplete: () => {},
     task: async ([_seekToMs], { signal }) => {
+      await this.videoSeekTask.taskComplete;
       // Check if we're in production rendering mode vs preview mode
       const isProductionRendering = this.isInProductionRenderingMode();
 
+      const sample = this.videoSeekTask.value;
+      if (sample) {
+        const videoFrame = sample.toVideoFrame();
+        this.displayFrame(videoFrame, _seekToMs);
+        videoFrame.close();
+      }
+
       // EF_FRAMEGEN-aware rendering mode detection
       if (!isProductionRendering) {
         // Preview mode: skip rendering during initialization to prevent artifacts
@@ -417,277 +292,9 @@ export class EFVideo extends TWMixin(EFMedia) {
       if (signal.aborted) {
         return;
       }
-
-      // CRITICAL: For segment transitions, ensure we wait for the correct mediaSegmentsTask
-      // This prevents using stale VideoAssets from previous segments
-
-      await this.mediaSegmentsTask.taskComplete;
-      if (signal.aborted) {
-        return;
-      }
-
-      // CRITICAL: Always await fresh videoAsset just before using it
-      // This prevents race conditions where old VideoAssets with closed decoders are used
-      await this.videoAssetTask.taskComplete;
-      if (signal.aborted) {
-        return;
-      }
-
-      // Get fresh values after await - ensures we use current VideoAsset
-      const videoAsset = this.videoAssetTask.value;
-      const currentSeekToMs = this.desiredSeekTimeMs; // Use current seek time, not captured
-
-      if (!videoAsset) {
-        log("trace: paintTask aborted - no video asset");
-        throw new Error(
-          `Frame rendering failed: No video asset available. This may indicate a problem with video loading or an invalid source: "${this.src}"`,
-        );
-      }
-
-      // Check if decoder needs reset due to previous errors
-      if (this.#decoderNeedsReset) {
-        try {
-          // Reset the video decoder
-          if (videoAsset?.videoDecoder) {
-            videoAsset.configureDecoder();
-          } else {
-            console.warn("No video decoder available for reset");
-          }
-
-          // Clear the flag after successful reset
-          this.#decoderNeedsReset = false;
-        } catch (resetError) {
-          console.error("reset error", resetError);
-          // Keep the flag set if reset fails
-          throw new Error(
-            `Frame rendering failed: Unable to reset video decoder after previous error. Decoder state: ${resetError instanceof Error ? resetError.message : "Unknown error"}. Try refreshing the page or reloading the video.`,
-          );
-        }
-      }
-      if (signal.aborted) {
-        return;
-      }
-
-      if (this.#decoderLock) {
-        return;
-      }
-
-      try {
-        this.#decoderLock = true;
-
-        // Validate VideoAsset is still current and decoder is in valid state
-        const currentVideoAsset = this.videoAssetTask.value;
-        if (videoAsset !== currentVideoAsset) {
-          return; // Skip render with stale videoAsset
-        }
-
-        // Check decoder state before using it
-        const decoderState = videoAsset?.videoDecoder?.state;
-        if (decoderState === "closed") {
-          return; // Skip render with closed decoder
-        }
-
-        // Try scrub track first for JIT transcode mode
-        if (this.effectiveMode === "jit-transcode" && this.scrubTrackManager) {
-          const shouldUseScrub =
-            this.scrubTrackManager.shouldUseScrubTrack(currentSeekToMs);
-          const isFastSeeking = this.scrubTrackManager.isFastSeeking(
-            this.lastSeekTimeMs,
-            currentSeekToMs,
-          );
-
-          if (false || shouldUseScrub || isFastSeeking) {
-            try {
-              // Use delayed loading instead of immediate loading
-              this.startDelayedLoading(
-                "scrub-segment-load",
-                "Loading scrub segment...",
-              );
-
-              const scrubFrame =
-                await this.scrubTrackManager.getScrubFrame(currentSeekToMs);
-
-              if (scrubFrame && this.canvasElement) {
-                this.scrubTrackManager.recordCacheMiss();
-                this.lastSeekTimeMs = currentSeekToMs;
-
-                // Clear loading and display scrub frame
-                this.clearDelayedLoading("scrub-segment-load");
-                return this.displayFrame(scrubFrame, currentSeekToMs);
-              }
-
-              // Scrub frame was null/failed - fall back to normal video
-              console.warn(
-                "Scrub track returned null frame, falling back to normal video",
-              );
-              this.clearDelayedLoading("scrub-segment-load");
-              this.startDelayedLoading(
-                "video-segment-fallback",
-                "Loading high quality video...",
-              );
-            } catch (error) {
-              this.clearDelayedLoading("scrub-segment-load");
-              console.warn(
-                "Scrub track failed, falling back to normal video:",
-                error,
-              );
-
-              // Show loading for normal video fallback
-              this.startDelayedLoading(
-                "video-segment-fallback",
-                "Loading high quality video...",
-              );
-            }
-          } else {
-            // Cache hit for normal video - scrub track manager exists, record the hit
-            this.scrubTrackManager?.recordCacheHit();
-          }
-        }
-
-        // Normal video rendering path (for all cases where scrub track isn't used)
-        // Check if we need to show loading for normal video operations
-        const shouldShowLoading =
-          !this.delayedLoadingState.isLoading &&
-          (this.effectiveMode !== "asset" || !videoAsset);
-
-        if (shouldShowLoading) {
-          this.startDelayedLoading("video-segment", "Loading video segment...");
-        }
-
-        // Render normal video - pass fresh VideoAsset reference
-        this.lastSeekTimeMs = currentSeekToMs;
-        const result = await this.renderNormalVideo(
-          videoAsset,
-          currentSeekToMs,
-        );
-
-        // Clear loading state after normal video renders
-        this.clearDelayedLoading("video-segment");
-        this.clearDelayedLoading("video-segment-fallback");
-
-        return result;
-      } catch (error) {
-        // Clear all loading states on error
-        this.clearDelayedLoading("scrub-segment-load");
-        this.clearDelayedLoading("video-segment");
-        this.clearDelayedLoading("video-segment-fallback");
-
-        // Handle errors with proper error propagation
-        if (error instanceof Error) {
-          if (
-            error.name === "DataError" &&
-            error.message.includes("key frame is required")
-          ) {
-            console.warn(
-              "Decoder reset during VideoAsset due to key frame requirement",
-            );
-            this.#decoderNeedsReset = true;
-
-            if (this.effectiveMode === "jit-transcode") {
-              this.requestUpdate();
-            }
-            throw error;
-          }
-          if (error.name === "AbortError") {
-            // AbortError is expected behavior when tasks are cancelled
-            throw new Error(
-              "Frame rendering cancelled: Operation was aborted, likely due to a new seek request or component unmounting.",
-            );
-          }
-          if (
-            error.message.includes("VideoAsset decoder closed") ||
-            error.message.includes("recreation in progress")
-          ) {
-            // This is now expected behavior during VideoAsset transitions - don't treat as error
-            return; // Gracefully abort instead of throwing
-          }
-          if (
-            error.name === "InvalidStateError" &&
-            error.message.includes("closed codec")
-          ) {
-            // Expected during VideoAsset recreation - gracefully abort
-            return; // Gracefully abort instead of throwing
-          }
-          console.warn("Decoder reset during VideoAsset recreation", error);
-          this.#decoderNeedsReset = true;
-          throw error;
-        }
-
-        // For non-Error objects, still provide descriptive error
-        throw new Error(
-          `Frame rendering failed: Unknown error during video rendering at ${currentSeekToMs}ms. Error: ${String(error)}`,
-        );
-      } finally {
-        this.#decoderLock = false;
-      }
     },
   });
 
-  /**
-   * Render normal video using existing logic
-   */
-  private async renderNormalVideo(
-    videoAsset: VideoAsset,
-    seekToMs: number,
-  ): Promise<number> {
-    let targetSeekTimeSeconds = seekToMs / 1000;
-
-    try {
-      // Validate VideoAsset is still current before seeking
-      const currentVideoAsset = this.videoAssetTask.value;
-      if (videoAsset !== currentVideoAsset) {
-        throw new Error(
-          "VideoAsset decoder closed during seek - recreation in progress",
-        );
-      }
-
-      // Check decoder state immediately before seeking
-      const decoderState = videoAsset?.videoDecoder?.state;
-      if (decoderState === "closed") {
-        throw new Error(
-          "VideoAsset decoder closed during seek - recreation in progress",
-        );
-      }
-      if (this.effectiveMode === "jit-transcode") {
-        targetSeekTimeSeconds %= 2;
-      }
-
-      const frame = await videoAsset.seekToTime(targetSeekTimeSeconds);
-
-      if (frame) {
-        // Final validation that VideoAsset is still current before displaying
-        const finalVideoAsset = this.videoAssetTask.value;
-        if (videoAsset !== finalVideoAsset) {
-          frame.close(); // Clean up the frame
-          throw new Error(
-            "VideoAsset decoder closed during seek - recreation in progress",
-          );
-        }
-
-        // Read fresh time right before displaying - final safeguard against stale values
-        const finalSeekToMs = this.desiredSeekTimeMs;
-        return this.displayFrame(frame, finalSeekToMs);
-      }
-
-      log("trace: no frame returned from seekToTime");
-      throw new Error(
-        `Frame rendering failed: No frame available at time ${seekToMs}ms (${targetSeekTimeSeconds}s). This may indicate seeking beyond video duration, corrupted video data, or an incompatible video format.`,
-      );
-    } catch (error) {
-      if (
-        error instanceof Error &&
-        (error.message.includes("VideoAsset decoder closed") ||
-          error.message.includes("recreation in progress"))
-      ) {
-        // This is the expected narrow timing window race condition during VideoAsset transitions
-        throw error; // Re-throw to let paintTask handle it gracefully
-      }
-
-      // Re-throw other unexpected errors
-      throw error;
-    }
-  }
-
   /**
    * Display a video frame on the canvas
    */
@@ -729,7 +336,7 @@ export class EFVideo extends TWMixin(EFMedia) {
       );
     }
 
-    log("trace: drawing frame to canvas");
+    log("trace: drawing frame to canvas", frame.timestamp / 1000);
     ctx.drawImage(
       frame,
       0,
@@ -792,6 +399,70 @@ export class EFVideo extends TWMixin(EFMedia) {
     return this.scrubTrackManager?.getCacheStats() || null;
   }
 
+  // Getter properties for backward compatibility with tests
+  /**
+   * Effective mode - always returns "asset" for EFVideo
+   */
+  get effectiveMode(): string {
+    return "asset";
+  }
+
+  /**
+   * Legacy getter for asset index loader (maps to mediaEngine task)
+   */
+  get assetIndexLoader() {
+    return this.mediaEngineTask;
+  }
+
+  /**
+   * Legacy getter for fragment index task (maps to videoSegmentIdTask)
+   */
+  get fragmentIndexTask() {
+    return this.videoSegmentIdTask;
+  }
+
+  /**
+   * Legacy getter for seek task (maps to videoSeekTask)
+   */
+  get seekTask() {
+    return this.videoSeekTask;
+  }
+
+  /**
+   * Legacy getter for media segments task (maps to videoSegmentFetchTask)
+   */
+  get mediaSegmentsTask() {
+    return this.videoSegmentFetchTask;
+  }
+
+  /**
+   * Legacy getter for asset segment keys task (maps to videoSegmentIdTask)
+   */
+  get assetSegmentKeysTask() {
+    return this.videoSegmentIdTask;
+  }
+
+  /**
+   * Legacy getter for asset init segments task (maps to videoInitSegmentFetchTask)
+   */
+  get assetInitSegmentsTask() {
+    return this.videoInitSegmentFetchTask;
+  }
+
+  /**
+   * Legacy getter for asset segment loader (maps to videoSegmentFetchTask)
+   */
+  get assetSegmentLoader() {
+    return this.videoSegmentFetchTask;
+  }
+
+  /**
+   * Legacy getter for video asset task (maps to videoBufferTask)
+   */
+  get videoAssetTask() {
+    return this.videoBufferTask;
+  }
+
   /**
    * Clean up resources when component is disconnected
    */

package/src/elements/EFWaveform.ts

@@ -16,8 +16,8 @@ import { TargetController } from "./TargetController.ts";
 export class EFWaveform extends EFTemporal(TWMixin(LitElement)) {
   static styles = css`
       :host {
-        display: block;
         all: inherit;
+        display: block;
         position: relative;
       }