@editframe/elements 0.5.0-beta.6 → 0.5.0-beta.8

package/src/elements/EFTimegroup.ts

@@ -0,0 +1,394 @@
+import { LitElement, html, css, PropertyValueMap } from "lit";
+import { provide } from "@lit/context";
+import { customElement, property } from "lit/decorators.js";
+import {
+  EFTemporal,
+  isEFTemporal,
+  shallowGetTemporalElements,
+  timegroupContext,
+} from "./EFTemporal";
+import { TimegroupController } from "./TimegroupController";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+import { deepGetMediaElements } from "./EFMedia";
+import { Task } from "@lit/task";
+
+function audioBufferToFloat32(buffer: AudioBuffer) {
+  const numOfChan = buffer.numberOfChannels;
+  const length = buffer.length * numOfChan * 4; // 4 bytes per sample for 32-bit float
+  const resultBuffer = new ArrayBuffer(length);
+  const view = new DataView(resultBuffer);
+  const channels = [];
+
+  for (let i = 0; i < numOfChan; i++) {
+    channels.push(buffer.getChannelData(i));
+  }
+
+  let offset = 0;
+  for (let i = 0; i < buffer.length; i++) {
+    for (let channel = 0; channel < numOfChan; channel++) {
+      view.setFloat32(offset, channels[channel][i], true); // true for little-endian
+      offset += 4;
+    }
+  }
+
+  return new Blob([resultBuffer], { type: "application/octet-stream" });
+}
+
+@customElement("ef-timegroup")
+export class EFTimegroup extends EFTemporal(LitElement) {
+  static styles = css`
+    :host {
+      display: block;
+      width: 100%;
+      height: 100%;
+      position: relative;
+      top: 0;
+    }
+  `;
+
+  @provide({ context: timegroupContext })
+  _timeGroupContext = this;
+
+  #currentTime = 0;
+
+  @property({
+    type: String,
+    attribute: "mode",
+  })
+  mode: "fixed" | "sequence" | "contain" = "sequence";
+
+  @property({ type: Number })
+  set currentTime(time: number) {
+    this.#currentTime = Math.max(0, Math.min(time, this.durationMs / 1000));
+    try {
+      if (this.id) {
+        if (this.isConnected) {
+          localStorage.setItem(this.storageKey, time.toString());
+        }
+      }
+    } catch (error) {
+      console.warn("Failed to save time to localStorage", error);
+    }
+  }
+  get currentTime() {
+    return this.#currentTime;
+  }
+  get currentTimeMs() {
+    return this.currentTime * 1000;
+  }
+  set currentTimeMs(ms: number) {
+    this.currentTime = ms / 1000;
+  }
+
+  @property({
+    attribute: "crossover",
+    converter: {
+      fromAttribute: (value: string): number => {
+        if (value.endsWith("ms")) {
+          return parseFloat(value);
+        }
+        if (value.endsWith("s")) {
+          return parseFloat(value) * 1000;
+        } else {
+          throw new Error(
+            "`crossover` MUST be in milliseconds or seconds (10s, 10000ms)",
+          );
+        }
+      },
+      toAttribute: (value: number) => `${value}ms`,
+    },
+  })
+  crossoverMs = 0;
+
+  render() {
+    return html`<slot></slot> `;
+  }
+
+  maybeLoadTimeFromLocalStorage() {
+    if (this.id) {
+      try {
+        return parseFloat(localStorage.getItem(this.storageKey) || "0");
+      } catch (error) {
+        console.warn("Failed to load time from localStorage", error);
+      }
+    }
+    return 0;
+  }
+
+  connectedCallback() {
+    super.connectedCallback();
+    if (this.id) {
+      this.#currentTime = this.maybeLoadTimeFromLocalStorage();
+    }
+
+    if (this.parentTimegroup) {
+      new TimegroupController(this.parentTimegroup, this);
+    }
+
+    if (this.shouldWrapWithWorkbench()) {
+      this.wrapWithWorkbench();
+    }
+  }
+
+  get storageKey() {
+    if (!this.id) {
+      throw new Error("Timegroup must have an id to use localStorage.");
+    }
+    return `ef-timegroup-${this.id}`;
+  }
+
+  get crossoverStartMs() {
+    const parentTimeGroup = this.parentTimegroup;
+    if (!parentTimeGroup || !this.previousElementSibling) {
+      return 0;
+    }
+
+    return parentTimeGroup.crossoverMs;
+  }
+  get crossoverEndMs() {
+    const parentTimeGroup = this.parentTimegroup;
+    if (!parentTimeGroup || !this.nextElementSibling) {
+      return 0;
+    }
+
+    return parentTimeGroup.crossoverMs;
+  }
+
+  get durationMs() {
+    switch (this.mode) {
+      case "fixed":
+        return super.durationMs || 0;
+      case "sequence":
+        let duration = 0;
+        this.childTemporals.forEach((node) => {
+          duration += node.durationMs;
+        });
+        return duration;
+      case "contain":
+        let maxDuration = 0;
+        this.childTemporals.forEach((node) => {
+          if (node.hasOwnDuration) {
+            maxDuration = Math.max(maxDuration, node.durationMs);
+          }
+        });
+        return maxDuration;
+      default:
+        throw new Error(`Invalid time mode: ${this.mode}`);
+    }
+  }
+
+  async waitForMediaDurations() {
+    return await Promise.all(
+      deepGetMediaElements(this).map(
+        (media) => media.trackFragmentIndexLoader.taskComplete,
+      ),
+    );
+  }
+
+  get childTemporals() {
+    return shallowGetTemporalElements(this);
+  }
+
+  protected updated(
+    changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+  ): void {
+    super.updated(changedProperties);
+    if (
+      changedProperties.has("currentTime") ||
+      changedProperties.has("ownCurrentTimeMs")
+    ) {
+      console.log("Updating animations to time", this.ownCurrentTimeMs);
+      const animations = this.getAnimations({ subtree: true });
+      this.style.setProperty(
+        "--ef-duration",
+        `${this.durationMs + this.crossoverEndMs + this.crossoverStartMs}ms`,
+      );
+      animations.forEach((animation) => {
+        if (animation.playState === "running") {
+          animation.pause();
+        }
+        const effect = animation.effect;
+        if (!(effect && effect instanceof KeyframeEffect)) {
+          return;
+        }
+        const target = effect.target;
+        // Important to avoid going to the end of the animation
+        // or it will reset awkwardly.
+        if (isEFTemporal(target)) {
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(target.ownCurrentTimeMs, duration - 1 + delay),
+          );
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        } else if (target) {
+          const nearestTimegroup = target.closest("ef-timegroup");
+          if (!nearestTimegroup) {
+            return;
+          }
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(nearestTimegroup.ownCurrentTimeMs, duration - 1 + delay),
+          );
+
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        }
+      });
+    }
+  }
+
+  shouldWrapWithWorkbench() {
+    return (
+      EF_INTERACTIVE &&
+      this.closest("ef-timegroup") === this &&
+      this.closest("ef-workbench") === null
+    );
+  }
+
+  wrapWithWorkbench() {
+    const workbench = document.createElement("ef-workbench");
+    document.body.append(workbench);
+    if (!this.hasAttribute("id")) {
+      this.setAttribute("id", "root-this");
+    }
+    this.setAttribute("slot", "canvas");
+    workbench.append(this as unknown as Element);
+
+    const filmstrip = document.createElement("ef-filmstrip");
+    filmstrip.setAttribute("slot", "timeline");
+    filmstrip.setAttribute("target", `#${this.id}`);
+    workbench.append(filmstrip);
+  }
+
+  get hasOwnDuration() {
+    return true;
+  }
+
+  get efElements() {
+    return Array.from(
+      this.querySelectorAll(
+        "ef-audio, ef-video, ef-image, ef-captions, ef-waveform",
+      ),
+    );
+  }
+
+  async renderAudio(fromMs: number, toMs: number) {
+    await this.waitForMediaDurations();
+
+    const durationMs = toMs - fromMs;
+    const audioContext = new OfflineAudioContext(
+      2,
+      Math.round((48000 * durationMs) / 1000),
+      48000,
+    );
+
+    console.log("RENDERING AUDIO");
+    console.log(
+      `renderAudio fromMs=${fromMs} toMs=${toMs} durationMs=${durationMs} ctxSize=${audioContext.length}`,
+    );
+
+    await Promise.all(
+      deepGetMediaElements(this).map(async (mediaElement) => {
+        await mediaElement.trackFragmentIndexLoader.taskComplete;
+
+        const mediaStartsBeforeEnd = mediaElement.startTimeMs <= toMs;
+        const mediaEndsAfterStart = mediaElement.endTimeMs >= fromMs;
+        const mediaOverlaps = mediaStartsBeforeEnd && mediaEndsAfterStart;
+        if (!mediaOverlaps || mediaElement.defaultAudioTrackId === undefined) {
+          console.log("Skipping audio element due to lack of overlap");
+          return;
+        }
+
+        const audio = await mediaElement.fetchAudioSpanningTime(fromMs, toMs);
+        if (!audio) {
+          throw new Error("Failed to fetch audio");
+        }
+
+        const ctxStartMs = Math.max(0, mediaElement.startTimeMs - fromMs);
+        const ctxEndMs = Math.min(durationMs, mediaElement.endTimeMs - fromMs);
+        const ctxDurationMs = ctxEndMs - ctxStartMs;
+
+        const offset =
+          Math.max(0, fromMs - mediaElement.startTimeMs) - audio.startMs;
+
+        // console.log("Fetching audio spanning time", audio.startMs, audio.endMs);
+        console.log(
+          "AUDIO SPAN",
+          JSON.stringify({
+            fromMs,
+            toMs,
+            audio: {
+              startMs: audio.startMs,
+              endMs: audio.endMs,
+            },
+            elementStart: mediaElement.startTimeMs,
+            elementEnd: mediaElement.endTimeMs,
+            ctxStart: ctxStartMs,
+            ctxEnd: ctxEndMs,
+            offset,
+          }),
+        );
+
+        const bufferSource = audioContext.createBufferSource();
+        bufferSource.buffer = await audioContext.decodeAudioData(
+          await audio.blob.arrayBuffer(),
+        );
+        bufferSource.connect(audioContext.destination);
+
+        bufferSource.start(
+          ctxStartMs / 1000,
+          offset / 1000,
+          ctxDurationMs / 1000,
+        );
+      }),
+    );
+
+    return await audioContext.startRendering();
+  }
+
+  async loadMd5Sums() {
+    const efElements = this.efElements;
+    const loaderTasks: Promise<any>[] = [];
+    efElements.forEach((el) => {
+      const md5SumLoader = (el as any).md5SumLoader;
+      if (md5SumLoader instanceof Task) {
+        md5SumLoader.run();
+        loaderTasks.push(md5SumLoader.taskComplete);
+      }
+    });
+
+    await Promise.all(loaderTasks);
+
+    efElements.map((el) => {
+      if ("productionSrc" in el && el.productionSrc instanceof Function) {
+        el.setAttribute("src", el.productionSrc());
+      }
+    });
+  }
+
+  frameTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.ownCurrentTimeMs, this.currentTimeMs] as const,
+    task: async ([], { signal }) => {
+      let fullyUpdated = await this.updateComplete;
+      while (!fullyUpdated) {
+        fullyUpdated = await this.updateComplete;
+      }
+    },
+  });
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "ef-timegroup": EFTimegroup & Element;
+  }
+}

package/src/elements/EFTimeline.ts

@@ -0,0 +1,13 @@
+class EFTimeline extends HTMLElement {
+  get durationMs() {
+    let duration = 0;
+    this.childNodes.forEach((node) => {
+      if ("durationMs" in node && typeof node.durationMs === "number") {
+        duration += node.durationMs;
+      }
+    });
+    return duration;
+  }
+}
+
+window.customElements.define("ef-timeline", EFTimeline);

package/src/elements/EFVideo.ts

@@ -0,0 +1,114 @@
+import { html, css } from "lit";
+import { Task } from "@lit/task";
+import { createRef, ref } from "lit/directives/ref.js";
+import { customElement } from "lit/decorators.js";
+
+import { EFMedia } from "./EFMedia";
+import { TWMixin } from "../gui/TWMixin";
+
+@customElement("ef-video")
+export class EFVideo extends TWMixin(EFMedia) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+    `,
+  ];
+  canvasRef = createRef<HTMLCanvasElement>();
+
+  render() {
+    return html` <canvas
+      class="h-full w-full object-fill"
+      ${ref(this.canvasRef)}
+    ></canvas>`;
+  }
+
+  get canvasElement() {
+    return this.canvasRef.value;
+  }
+
+  // The underlying video decoder MUST NOT be used concurrently.
+  // If frames are fed in out of order, the decoder may crash.
+  #decoderLock = false;
+
+  frameTask = new Task(this, {
+    args: () =>
+      [
+        this.trackFragmentIndexLoader.status,
+        this.initSegmentsLoader.status,
+        this.seekTask.status,
+        this.fetchSeekTask.status,
+        this.videoAssetTask.status,
+        this.paintTask.status,
+      ] as const,
+    task: async () => {
+      console.log("EFVideo frameTask", this.ownCurrentTimeMs);
+      await this.trackFragmentIndexLoader.taskComplete;
+      await this.initSegmentsLoader.taskComplete;
+      await this.seekTask.taskComplete;
+      await this.fetchSeekTask.taskComplete;
+      await this.videoAssetTask.taskComplete;
+      await this.paintTask.taskComplete;
+      console.log("EFVideo frameTask complete", this.ownCurrentTimeMs);
+      this.rootTimegroup?.requestUpdate();
+    },
+  });
+
+  paintTask = new Task(this, {
+    args: () => [this.videoAssetTask.value, this.desiredSeekTimeMs] as const,
+    task: async (
+      [videoAsset, seekToMs],
+      {
+        signal:
+          _signal /** Aborting seeks is counter-productive. It is better to drop seeks. */,
+      },
+    ) => {
+      console.log(`EFVideo paintTask decoderLock=${this.#decoderLock}`);
+      if (!videoAsset) {
+        return;
+      }
+      if (this.#decoderLock) {
+        return;
+      }
+      try {
+        this.#decoderLock = true;
+        const frame = await videoAsset.seekToTime(seekToMs / 1000);
+        console.log(
+          "Painting frame",
+          "seekToMs",
+          seekToMs,
+          "timestamp",
+          frame?.timestamp,
+        );
+
+        if (!this.canvasElement) {
+          return;
+        }
+        const ctx = this.canvasElement.getContext("2d");
+        if (!(frame && ctx)) {
+          return;
+        }
+
+        this.canvasElement.width = frame?.codedWidth;
+        this.canvasElement.height = frame?.codedHeight;
+
+        ctx.drawImage(
+          frame,
+          0,
+          0,
+          this.canvasElement.width,
+          this.canvasElement.height,
+        );
+
+        return seekToMs;
+      } catch (error) {
+        console.trace("Unexpected error while seeking video", error);
+        // As a practical matter, we should probably just re-create the VideoAsset if decoding fails.
+        // The decoder is probably in an invalid state anyway.
+      } finally {
+        this.#decoderLock = false;
+      }
+    },
+  });
+}