@editframe/elements 0.16.8-beta.0 → 0.18.3-beta.0

package/src/elements/EFVideo.ts

@@ -1,13 +1,45 @@
 import { Task } from "@lit/task";
-import { css, html } from "lit";
-import { customElement } from "lit/decorators.js";
+import debug from "debug";
+import { css, html, type PropertyValueMap } from "lit";
+import { customElement, property, state } from "lit/decorators.js";
 import { createRef, ref } from "lit/directives/ref.js";
 
+import { DelayedLoadingState } from "../DelayedLoadingState.js";
 import { TWMixin } from "../gui/TWMixin.js";
+import type { CacheStats, ScrubTrackManager } from "../ScrubTrackManager.js";
+import { makeVideoBufferTask } from "./EFMedia/videoTasks/makeVideoBufferTask.ts";
+import { makeVideoInitSegmentFetchTask } from "./EFMedia/videoTasks/makeVideoInitSegmentFetchTask.ts";
+import { makeVideoInputTask } from "./EFMedia/videoTasks/makeVideoInputTask.ts";
+import { makeVideoSeekTask } from "./EFMedia/videoTasks/makeVideoSeekTask.ts";
+import { makeVideoSegmentFetchTask } from "./EFMedia/videoTasks/makeVideoSegmentFetchTask.ts";
+import { makeVideoSegmentIdTask } from "./EFMedia/videoTasks/makeVideoSegmentIdTask.ts";
 import { EFMedia } from "./EFMedia.js";
 
+// EF_FRAMEGEN is a global instance created in EF_FRAMEGEN.ts
+declare global {
+  var EF_FRAMEGEN: import("../EF_FRAMEGEN.js").EFFramegen;
+}
+
+const log = debug("ef:elements:EFVideo");
+
+interface LoadingState {
+  isLoading: boolean;
+  operation: "scrub-segment" | "video-segment" | "seeking" | "decoding" | null;
+  message: string;
+}
+
 @customElement("ef-video")
 export class EFVideo extends TWMixin(EFMedia) {
+  static get observedAttributes() {
+    const parentAttributes = EFMedia.observedAttributes || [];
+    return [
+      ...parentAttributes,
+      "video-buffer-duration",
+      "max-video-buffer-fetches",
+      "enable-video-buffering",
+    ];
+  }
+
   static styles = [
     /**
      *
@@ -15,6 +47,7 @@ export class EFVideo extends TWMixin(EFMedia) {
     css`
       :host {
         display: block;
+        position: relative;
       }
       canvas {
         all: inherit;
@@ -29,12 +62,129 @@ export class EFVideo extends TWMixin(EFMedia) {
         outline: none;
         box-shadow: none;
       }
+      .loading-overlay {
+        position: absolute;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        background: rgba(0, 0, 0, 0.6);
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        z-index: 10;
+        backdrop-filter: blur(2px);
+      }
+      .loading-content {
+        background: rgba(0, 0, 0, 0.8);
+        border-radius: 8px;
+        padding: 16px 24px;
+        display: flex;
+        align-items: center;
+        gap: 12px;
+        color: white;
+        font-size: 14px;
+        font-weight: 500;
+      }
+      .loading-spinner {
+        width: 20px;
+        height: 20px;
+        border: 2px solid rgba(255, 255, 255, 0.2);
+        border-left: 2px solid #fff;
+        border-radius: 50%;
+        animation: spin 1s linear infinite;
+      }
+      @keyframes spin {
+        0% { transform: rotate(0deg); }
+        100% { transform: rotate(360deg); }
+      }
+      .loading-message {
+        font-size: 12px;
+        opacity: 0.8;
+      }
     `,
   ];
   canvasRef = createRef<HTMLCanvasElement>();
+
+  /**
+   * Duration in milliseconds for video buffering ahead of current time
+   * @domAttribute "video-buffer-duration"
+   */
+  @property({ type: Number, attribute: "video-buffer-duration" })
+  videoBufferDurationMs = 60000; // 60 seconds
+
+  /**
+   * Maximum number of concurrent video segment fetches for buffering
+   * @domAttribute "max-video-buffer-fetches"
+   */
+  @property({ type: Number, attribute: "max-video-buffer-fetches" })
+  maxVideoBufferFetches = 2;
+
+  /**
+   * Enable/disable video buffering system
+   * @domAttribute "enable-video-buffering"
+   */
+  @property({ type: Boolean, attribute: "enable-video-buffering" })
+  enableVideoBuffering = true;
+
+  // Video-specific tasks
+  videoSegmentIdTask = makeVideoSegmentIdTask(this);
+  videoInitSegmentFetchTask = makeVideoInitSegmentFetchTask(this);
+  videoSegmentFetchTask = makeVideoSegmentFetchTask(this);
+  videoInputTask = makeVideoInputTask(this);
+  videoSeekTask = makeVideoSeekTask(this);
+  videoBufferTask = makeVideoBufferTask(this);
+
+  /**
+   * Scrub track manager for fast timeline navigation
+   */
+  scrubTrackManager?: ScrubTrackManager;
+
+  /**
+   * Delayed loading state manager for user feedback
+   */
+  private delayedLoadingState: DelayedLoadingState;
+
+  /**
+   * Loading state for user feedback
+   */
+  @state()
+  loadingState = {
+    isLoading: false,
+    operation: null as LoadingState["operation"],
+    message: "",
+  };
+
+  constructor() {
+    super();
+
+    // Initialize delayed loading state with callback to update UI
+    this.delayedLoadingState = new DelayedLoadingState(
+      250,
+      (isLoading, message) => {
+        this.setLoadingState(isLoading, null, message);
+      },
+    );
+  }
+
   render() {
     return html`
       <canvas ${ref(this.canvasRef)}></canvas>
+      ${
+        this.loadingState.isLoading
+          ? html`
+        <div class="loading-overlay">
+          <div class="loading-content">
+            <div class="loading-spinner"></div>
+            <div>
+              <div>Loading Video...</div>
+              <div class="loading-message">${this.loadingState.message}</div>
+            </div>
+          </div>
+        </div>
+      `
+          : ""
+      }
     `;
   }
 
@@ -42,89 +192,289 @@ export class EFVideo extends TWMixin(EFMedia) {
     return this.canvasRef.value;
   }
 
-  // The underlying video decoder MUST NOT be used concurrently.
-  // If frames are fed in out of order, the decoder may crash.
-  #decoderLock = false;
-
   frameTask = new Task(this, {
-    args: () =>
-      [
-        this.trackFragmentIndexLoader.status,
-        this.initSegmentsLoader.status,
-        this.seekTask.status,
-        this.fetchSeekTask.status,
-        this.videoAssetTask.status,
-        this.paintTask.status,
-      ] as const,
-    task: async () => {
-      await this.trackFragmentIndexLoader.taskComplete;
-      await this.initSegmentsLoader.taskComplete;
-      await this.seekTask.taskComplete;
-      await this.fetchSeekTask.taskComplete;
-      await this.videoAssetTask.taskComplete;
+    args: () => [this.desiredSeekTimeMs] as const,
+    onError: (error) => {
+      console.error("frameTask error", error);
+    },
+    onComplete: () => {},
+    task: async ([_desiredSeekTimeMs], { signal }) => {
+      await this.videoSeekTask.taskComplete;
       await this.paintTask.taskComplete;
+      if (signal.aborted) {
+        return;
+      }
     },
   });
 
+  protected updated(
+    changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+  ): void {
+    super.updated(changedProperties);
+  }
+
+  /**
+   * Start a delayed loading operation for testing
+   */
+  startDelayedLoading(
+    operationId: string,
+    message: string,
+    options: { background?: boolean } = {},
+  ): void {
+    this.delayedLoadingState.startLoading(operationId, message, options);
+  }
+
+  /**
+   * Clear a delayed loading operation for testing
+   */
+  clearDelayedLoading(operationId: string): void {
+    this.delayedLoadingState.clearLoading(operationId);
+  }
+
+  /**
+   * Set loading state for user feedback
+   */
+  private setLoadingState(
+    isLoading: boolean,
+    operation: LoadingState["operation"] = null,
+    message = "",
+  ): void {
+    this.loadingState = {
+      isLoading,
+      operation,
+      message,
+    };
+  }
+
   paintTask = new Task(this, {
-    args: () => [this.videoAssetTask.value, this.desiredSeekTimeMs] as const,
-    task: async (
-      [videoAsset, seekToMs],
-      {
-        signal:
-          _signal /** Aborting seeks is counter-productive. It is better to drop seeks. */,
-      },
-    ) => {
-      if (!videoAsset) {
-        return;
-      }
-      if (this.#decoderLock) {
-        return;
+    args: () => [this.desiredSeekTimeMs] as const,
+    onError: (error) => {
+      console.error("paintTask error", error);
+    },
+    onComplete: () => {},
+    task: async ([_seekToMs], { signal }) => {
+      await this.videoSeekTask.taskComplete;
+      // Check if we're in production rendering mode vs preview mode
+      const isProductionRendering = this.isInProductionRenderingMode();
+
+      const sample = this.videoSeekTask.value;
+      if (sample) {
+        const videoFrame = sample.toVideoFrame();
+        this.displayFrame(videoFrame, _seekToMs);
+        videoFrame.close();
       }
-      try {
-        this.#decoderLock = true;
-        const frame = await videoAsset.seekToTime(seekToMs / 1000);
 
-        if (!this.canvasElement) {
-          return;
+      // EF_FRAMEGEN-aware rendering mode detection
+      if (!isProductionRendering) {
+        // Preview mode: skip rendering during initialization to prevent artifacts
+        if (
+          !this.rootTimegroup ||
+          (this.rootTimegroup.currentTimeMs === 0 &&
+            this.desiredSeekTimeMs === 0)
+        ) {
+          return; // Skip initialization frame in preview mode
         }
-        const ctx = this.canvasElement.getContext("2d");
-        if (!(frame && ctx)) {
+        // Preview mode: proceed with rendering
+      } else {
+        // Production rendering mode: only render when EF_FRAMEGEN has explicitly started frame rendering
+        // This prevents initialization frames before the actual render sequence begins
+        if (!this.rootTimegroup) {
           return;
         }
 
-        if (frame?.codedWidth && frame?.codedHeight) {
-          if (
-            this.canvasElement.width !== frame.codedWidth ||
-            this.canvasElement.height !== frame.codedHeight
-          ) {
-            this.canvasElement.width = frame.codedWidth;
-            this.canvasElement.height = frame.codedHeight;
-          }
+        if (!this.isFrameRenderingActive()) {
+          return; // Wait for EF_FRAMEGEN to start frame sequence
         }
 
-        if (frame.format === null) {
-          console.warn("Frame format is null", frame);
-          return seekToMs;
-        }
-        ctx.drawImage(
-          frame,
-          0,
-          0,
-          this.canvasElement.width,
-          this.canvasElement.height,
-        );
-
-        return seekToMs;
-      } catch (error) {
-        console.trace("Unexpected error while seeking video", error);
-        // As a practical matter, we should probably just re-create the VideoAsset if decoding fails.
-        // The decoder is probably in an invalid state anyway.
-      } finally {
-        this.#decoderLock = false;
+        // Production mode: EF_FRAMEGEN has started frame sequence, proceed with rendering
+      }
+
+      if (signal.aborted) {
+        return;
       }
     },
   });
+
+  /**
+   * Display a video frame on the canvas
+   */
+  private displayFrame(frame: VideoFrame, seekToMs: number): number {
+    log("trace: displayFrame start", { seekToMs, frameFormat: frame.format });
+    if (!this.canvasElement) {
+      log("trace: displayFrame aborted - no canvas element");
+      throw new Error(
+        `Frame display failed: Canvas element is not available at time ${seekToMs}ms. The video component may not be properly initialized.`,
+      );
+    }
+
+    const ctx = this.canvasElement.getContext("2d");
+    if (!ctx) {
+      log("trace: displayFrame aborted - no canvas context");
+      throw new Error(
+        `Frame display failed: Unable to get 2D canvas context at time ${seekToMs}ms. This may indicate a browser compatibility issue or canvas corruption.`,
+      );
+    }
+
+    if (frame?.codedWidth && frame?.codedHeight) {
+      if (
+        this.canvasElement.width !== frame.codedWidth ||
+        this.canvasElement.height !== frame.codedHeight
+      ) {
+        log("trace: updating canvas dimensions", {
+          width: frame.codedWidth,
+          height: frame.codedHeight,
+        });
+        this.canvasElement.width = frame.codedWidth;
+        this.canvasElement.height = frame.codedHeight;
+      }
+    }
+
+    if (frame.format === null) {
+      log("trace: displayFrame aborted - null frame format");
+      throw new Error(
+        `Frame display failed: Video frame has null format at time ${seekToMs}ms. This indicates corrupted or incompatible video data.`,
+      );
+    }
+
+    log("trace: drawing frame to canvas", frame.timestamp / 1000);
+    ctx.drawImage(
+      frame,
+      0,
+      0,
+      this.canvasElement.width,
+      this.canvasElement.height,
+    );
+    log("trace: frame drawn to canvas", { seekToMs });
+
+    return seekToMs;
+  }
+
+  /**
+   * Check if we're in production rendering mode (EF_FRAMEGEN active) vs preview mode
+   */
+  private isInProductionRenderingMode(): boolean {
+    // Check if EF_RENDERING function exists and returns true (production rendering)
+    if (typeof window.EF_RENDERING === "function") {
+      return window.EF_RENDERING();
+    }
+
+    // Check if workbench is in rendering mode
+    const workbench = document.querySelector("ef-workbench") as any;
+    if (workbench?.rendering) {
+      return true;
+    }
+
+    // Check if EF_FRAMEGEN exists and has render options (indicates active rendering)
+    if (window.EF_FRAMEGEN?.renderOptions) {
+      return true;
+    }
+
+    // Default to preview mode
+    return false;
+  }
+
+  /**
+   * Check if EF_FRAMEGEN has explicitly started frame rendering (not just initialization)
+   */
+  private isFrameRenderingActive(): boolean {
+    if (!window.EF_FRAMEGEN?.renderOptions) {
+      return false;
+    }
+
+    // In production mode, only render when EF_FRAMEGEN has actually begun frame sequence
+    // Check if we're past the initialization phase by looking for explicit frame control
+    const renderOptions = window.EF_FRAMEGEN.renderOptions;
+    const renderStartTime = renderOptions.encoderOptions.fromMs;
+    const currentTime = this.rootTimegroup?.currentTimeMs || 0;
+
+    // We're in active frame rendering if:
+    // 1. currentTime >= renderStartTime (includes the starting frame)
+    return currentTime >= renderStartTime;
+  }
+
+  /**
+   * Get scrub track performance statistics
+   */
+  getScrubTrackStats(): CacheStats | null {
+    return this.scrubTrackManager?.getCacheStats() || null;
+  }
+
+  // Getter properties for backward compatibility with tests
+  /**
+   * Effective mode - always returns "asset" for EFVideo
+   */
+  get effectiveMode(): string {
+    return "asset";
+  }
+
+  /**
+   * Legacy getter for asset index loader (maps to mediaEngine task)
+   */
+  get assetIndexLoader() {
+    return this.mediaEngineTask;
+  }
+
+  /**
+   * Legacy getter for fragment index task (maps to videoSegmentIdTask)
+   */
+  get fragmentIndexTask() {
+    return this.videoSegmentIdTask;
+  }
+
+  /**
+   * Legacy getter for seek task (maps to videoSeekTask)
+   */
+  get seekTask() {
+    return this.videoSeekTask;
+  }
+
+  /**
+   * Legacy getter for media segments task (maps to videoSegmentFetchTask)
+   */
+  get mediaSegmentsTask() {
+    return this.videoSegmentFetchTask;
+  }
+
+  /**
+   * Legacy getter for asset segment keys task (maps to videoSegmentIdTask)
+   */
+  get assetSegmentKeysTask() {
+    return this.videoSegmentIdTask;
+  }
+
+  /**
+   * Legacy getter for asset init segments task (maps to videoInitSegmentFetchTask)
+   */
+  get assetInitSegmentsTask() {
+    return this.videoInitSegmentFetchTask;
+  }
+
+  /**
+   * Legacy getter for asset segment loader (maps to videoSegmentFetchTask)
+   */
+  get assetSegmentLoader() {
+    return this.videoSegmentFetchTask;
+  }
+
+  /**
+   * Legacy getter for video asset task (maps to videoBufferTask)
+   */
+  get videoAssetTask() {
+    return this.videoBufferTask;
+  }
+
+  /**
+   * Clean up resources when component is disconnected
+   */
+  disconnectedCallback(): void {
+    super.disconnectedCallback();
+
+    // Clean up scrub track manager
+    this.scrubTrackManager?.cleanup();
+
+    // Clean up delayed loading state
+    this.delayedLoadingState.clearAllLoading();
+  }
 }
 
 declare global {

package/src/elements/EFWaveform.ts

@@ -1,8 +1,8 @@
 import { CSSStyleObserver } from "@bramus/style-observer";
 import { Task } from "@lit/task";
-import { LitElement, type PropertyValueMap, css, html } from "lit";
+import { css, html, LitElement, type PropertyValueMap } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
-import { type Ref, createRef, ref } from "lit/directives/ref.js";
+import { createRef, type Ref, ref } from "lit/directives/ref.js";
 import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
 import { EF_RENDERING } from "../EF_RENDERING.js";
 import { TWMixin } from "../gui/TWMixin.js";
@@ -16,8 +16,8 @@ import { TargetController } from "./TargetController.ts";
 export class EFWaveform extends EFTemporal(TWMixin(LitElement)) {
   static styles = css`
       :host {
-        display: block;
         all: inherit;
+        display: block;
         position: relative;
       }
 
@@ -38,7 +38,7 @@ export class EFWaveform extends EFTemporal(TWMixin(LitElement)) {
   private mutationObserver?: MutationObserver;
 
   render() {
-    return html`<canvas ${ref(this.canvasRef)}></canvas>`;
+    return html`<canvas ${ref(this.canvasRef)}></canvas><div class="text-5xl inline-block bg-red-500">`;
   }
 
   @property({
@@ -78,7 +78,7 @@ export class EFWaveform extends EFTemporal(TWMixin(LitElement)) {
       if (this.targetElement) {
         new CrossUpdateController(this.targetElement, this);
       }
-    } catch (e) {
+    } catch (_error) {
       // TODO: determine if this is a critical error, or if we should just ignore it
       // currenty evidence suggests everything still works
       // no target element, no cross update controller

package/src/elements/MediaController.ts

@@ -0,0 +1,98 @@
+import type { ReactiveController, ReactiveControllerHost } from "lit";
+
+export class MediaController implements ReactiveController {
+  #src: string | null = null;
+  #assetId: string | null = null;
+
+  constructor(
+    private host: ReactiveControllerHost & {
+      src: string;
+      assetId: string | null;
+      mode: "asset" | "jit-transcode" | "auto";
+    },
+  ) {
+    this.host.addController(this);
+  }
+
+  get mode() {
+    const actualMode = this.host.mode;
+    if (actualMode === "asset" || actualMode === "jit-transcode") {
+      return actualMode;
+    }
+
+    if (this.host.assetId) {
+      return "asset";
+    }
+
+    if (!this.host.src) {
+      return "asset";
+    }
+
+    return "asset";
+  }
+
+  get src() {
+    return this.#src;
+  }
+
+  set src(value: string | null) {
+    if (this.#src !== value) {
+      this.trackUpdate("src", this.#src, value);
+      this.#src = value;
+    }
+  }
+
+  get assetId() {
+    return this.#assetId;
+  }
+  set assetId(value: string | null) {
+    if (this.#assetId !== value) {
+      this.trackUpdate("assetId", this.#assetId, value);
+      this.#assetId = value;
+    }
+  }
+
+  changes: Map<string, { oldValue: any; newValue: any }> | null = null;
+  updatePending = false;
+
+  trackUpdate(key: string, oldValue: any, newValue: any) {
+    this.changes ??= new Map<string, { oldValue: any; newValue: any }>();
+    this.changes.set(key, { oldValue, newValue });
+    if (!this.updatePending) {
+      this.updatePending = true;
+      queueMicrotask(() => {
+        this.endUpdate();
+      });
+    }
+  }
+
+  endUpdate() {
+    try {
+      if (this.changes) {
+        this.handleUpdate(this.changes);
+      }
+    } finally {
+      this.changes = null;
+      this.updatePending = false;
+    }
+  }
+
+  handleUpdate(changes: Map<string, { oldValue: any; newValue: any }>) {
+    if (changes.has("src") || changes.has("assetId") || changes.has("mode")) {
+    }
+  }
+
+  hostUpdated(): void {
+    this.src = this.host.src;
+    this.assetId = this.host.assetId;
+  }
+
+  hostConnected() {
+    this.src = this.host.src;
+    this.assetId = this.host.assetId;
+  }
+
+  hostDisconnected() {}
+
+  updated() {}
+}