@editframe/elements 0.6.0-beta.16 → 0.6.0-beta.17

package/src/elements/EFTimeline.ts

@@ -0,0 +1,13 @@
+export class EFTimeline extends HTMLElement {
+  get durationMs() {
+    let duration = 0;
+    for (const node of this.childNodes) {
+      if ("durationMs" in node && typeof node.durationMs === "number") {
+        duration += node.durationMs;
+      }
+    }
+    return duration;
+  }
+}
+
+window.customElements.define("ef-timeline", EFTimeline);

package/src/elements/EFVideo.ts

@@ -0,0 +1,103 @@
+import { html, css } from "lit";
+import { Task } from "@lit/task";
+import { createRef, ref } from "lit/directives/ref.js";
+import { customElement } from "lit/decorators.js";
+
+import { EFMedia } from "./EFMedia";
+import { TWMixin } from "../gui/TWMixin";
+
+@customElement("ef-video")
+export class EFVideo extends TWMixin(EFMedia) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+    `,
+  ];
+  canvasRef = createRef<HTMLCanvasElement>();
+
+  render() {
+    return html` <canvas
+      class="h-full w-full object-fill"
+      ${ref(this.canvasRef)}
+    ></canvas>`;
+  }
+
+  get canvasElement() {
+    return this.canvasRef.value;
+  }
+
+  // The underlying video decoder MUST NOT be used concurrently.
+  // If frames are fed in out of order, the decoder may crash.
+  #decoderLock = false;
+
+  frameTask = new Task(this, {
+    args: () =>
+      [
+        this.trackFragmentIndexLoader.status,
+        this.initSegmentsLoader.status,
+        this.seekTask.status,
+        this.fetchSeekTask.status,
+        this.videoAssetTask.status,
+        this.paintTask.status,
+      ] as const,
+    task: async () => {
+      await this.trackFragmentIndexLoader.taskComplete;
+      await this.initSegmentsLoader.taskComplete;
+      await this.seekTask.taskComplete;
+      await this.fetchSeekTask.taskComplete;
+      await this.videoAssetTask.taskComplete;
+      await this.paintTask.taskComplete;
+    },
+  });
+
+  paintTask = new Task(this, {
+    args: () => [this.videoAssetTask.value, this.desiredSeekTimeMs] as const,
+    task: async (
+      [videoAsset, seekToMs],
+      {
+        signal:
+          _signal /** Aborting seeks is counter-productive. It is better to drop seeks. */,
+      },
+    ) => {
+      if (!videoAsset) {
+        return;
+      }
+      if (this.#decoderLock) {
+        return;
+      }
+      try {
+        this.#decoderLock = true;
+        const frame = await videoAsset.seekToTime(seekToMs / 1000);
+
+        if (!this.canvasElement) {
+          return;
+        }
+        const ctx = this.canvasElement.getContext("2d");
+        if (!(frame && ctx)) {
+          return;
+        }
+
+        this.canvasElement.width = frame?.codedWidth;
+        this.canvasElement.height = frame?.codedHeight;
+
+        ctx.drawImage(
+          frame,
+          0,
+          0,
+          this.canvasElement.width,
+          this.canvasElement.height,
+        );
+
+        return seekToMs;
+      } catch (error) {
+        console.trace("Unexpected error while seeking video", error);
+        // As a practical matter, we should probably just re-create the VideoAsset if decoding fails.
+        // The decoder is probably in an invalid state anyway.
+      } finally {
+        this.#decoderLock = false;
+      }
+    },
+  });
+}

package/src/elements/EFWaveform.ts

@@ -0,0 +1,409 @@
+import { EFAudio } from "./EFAudio";
+
+import { LitElement, html } from "lit";
+import { customElement, property } from "lit/decorators.js";
+import { EFVideo } from "./EFVideo";
+import { EFTemporal } from "./EFTemporal";
+import { CrossUpdateController } from "./CrossUpdateController";
+import { TWMixin } from "../gui/TWMixin";
+import { Task } from "@lit/task";
+import * as d3 from "d3";
+import { type Ref, createRef, ref } from "lit/directives/ref.js";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+
+@customElement("ef-waveform")
+export class EFWaveform extends EFTemporal(TWMixin(LitElement)) {
+  static styles = [];
+  svgRef: Ref<SVGElement> = createRef();
+  createRenderRoot() {
+    return this;
+  }
+  render() {
+    return html` <svg ${ref(this.svgRef)} class="h-full w-full" store></svg> `;
+  }
+
+  @property({
+    type: String,
+    attribute: "mode",
+  })
+  mode:
+    | "roundBars"
+    | "bars"
+    | "bricks"
+    | "equalizer"
+    | "curve"
+    | "line"
+    | "pixel"
+    | "wave" = "bars";
+
+  @property({ type: String })
+  color = "currentColor";
+
+  connectedCallback() {
+    super.connectedCallback();
+    if (this.target) {
+      new CrossUpdateController(this.target, this);
+    }
+  }
+
+  protected drawBars(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight;
+    const waveLeft = 0;
+    const waveRight = waveWidth;
+
+    const barX = d3
+      .scaleBand()
+      .paddingInner(0.5)
+      .paddingOuter(0.01)
+      .domain(d3.range(frequencyData.length).map((n) => String(n)))
+      .rangeRound([waveLeft, waveRight]);
+
+    const height = d3
+      .scaleLinear()
+      .domain([0, 255])
+      .range([0, waveHeight / 2]);
+
+    const baseline = waveHeight / 2;
+
+    const bars = d3.select(svg).selectAll("rect").data(frequencyData);
+    const minBarHeight = 2;
+
+    bars
+      .enter()
+      .append("rect")
+      // @ts-ignore Not sure why this doesn't pass typechecks.
+      .merge(bars)
+      .attr("x", (_, i) => barX(String(i)) || 0)
+      .attr("y", (value) => baseline - height(value))
+      .attr("width", barX.bandwidth() / 1.2)
+      .attr("height", (value) => Math.max(height(value) * 2, minBarHeight));
+
+    bars.exit().remove();
+  }
+
+  protected drawBricks(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight * devicePixelRatio;
+    const brickWidth = waveWidth / frequencyData.length;
+    const brickHeightFactor = waveHeight / 255 / 2;
+    const brickPadding = 2;
+    const midHeight = waveHeight / 2;
+
+    d3.select(svg)
+      .selectAll("line.brickBaseLine")
+      .data([0])
+      .join("line")
+      .attr("class", "brickBaseLine")
+      .attr("x1", 0)
+      .attr("x2", waveWidth)
+      .attr("y1", midHeight)
+      .attr("y2", midHeight)
+      .attr("stroke", "currentColor")
+      .attr("stroke-width", 4)
+      .attr("stroke-dasharray", "2");
+
+    d3.select(svg)
+      .selectAll("rect.brick")
+      .data(frequencyData)
+      .join("rect")
+      .attr("class", "brick")
+      .attr("x", (_d, i) => i * brickWidth)
+      .attr("y", (d) => midHeight - d * brickHeightFactor)
+      .attr("width", brickWidth - brickPadding)
+      .attr("height", (d) => d * brickHeightFactor * 2);
+  }
+
+  protected drawLine(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight * devicePixelRatio;
+
+    const xScale = d3
+      .scaleLinear()
+      .domain([0, frequencyData.length - 1])
+      .range([0, waveWidth]);
+
+    const yScale = d3.scaleLinear().domain([0, 255]).range([waveHeight, 0]);
+
+    const lineGenerator = d3
+      .line<Uint8Array[number]>()
+      .x((_, i) => xScale(i))
+      .y((d) => yScale(d));
+
+    const pathData = lineGenerator(frequencyData);
+
+    d3.select(svg)
+      .selectAll("path.line")
+      .data([frequencyData])
+      .join("path")
+      .attr("class", "line")
+      .attr("d", pathData)
+      .attr("fill", "none")
+      .attr("stroke", "currentColor")
+      .attr("stroke-width", 4);
+  }
+
+  protected drawRoundBars(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight;
+    const waveLeft = 0;
+    const waveRight = waveWidth;
+
+    const barX = d3
+      .scaleBand()
+      .paddingInner(0.5)
+      .paddingOuter(0.01)
+      .domain(d3.range(frequencyData.length).map((n) => String(n)))
+      .rangeRound([waveLeft, waveRight]);
+
+    const height = d3
+      .scaleLinear()
+      .domain([0, 255])
+      .range([0, waveHeight / 2]);
+
+    const baseline = waveHeight / 2;
+
+    const bars = d3.select(svg).selectAll("rect").data(frequencyData);
+    const minBarHeight = 2;
+
+    bars
+      .enter()
+      .append("rect")
+      // @ts-ignore Not sure why this doesn't pass typechecks.
+      .merge(bars)
+      .attr("x", (_, i) => barX(String(i)) || 0)
+      .attr("y", (value) => baseline - height(value))
+      .attr("width", barX.bandwidth() / 1.2)
+      .attr("height", (value) => Math.max(height(value) * 2, minBarHeight))
+      .attr("rx", barX.bandwidth())
+      .attr("ry", barX.bandwidth());
+
+    bars.exit().remove();
+  }
+  protected drawEqualizer(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight * devicePixelRatio;
+    const barWidth = waveWidth / frequencyData.length;
+    const barPadding = 1;
+    const minHeight = 1;
+
+    const heightScale = d3
+      .scaleLinear()
+      .domain([0, 255])
+      .range([0, waveHeight / 2]);
+
+    d3.select(svg)
+      .selectAll("line.equalizerBaseLine")
+      .data([0])
+      .join("line")
+      .attr("class", "equalizerBaseLine")
+      .attr("x1", 0)
+      .attr("x2", waveWidth)
+      .attr("y1", waveHeight / 2)
+      .attr("y2", waveHeight / 2)
+      .attr("stroke-width", 2);
+
+    d3.select(svg)
+      .selectAll("rect.equalizerBar")
+      .data(frequencyData)
+      .join("rect")
+      .attr("class", "equalizerBar")
+      .attr("x", (_d, i) => i * barWidth + barPadding / 2)
+      .attr(
+        "y",
+        (d) => waveHeight / 2 - Math.max(heightScale(d) / 2, minHeight),
+      )
+      .attr("width", barWidth - barPadding)
+      .attr("height", (d) => Math.max(heightScale(d), minHeight));
+
+    d3.select(svg)
+      .selectAll("rect.equalizerBar")
+      .transition()
+      .duration(100)
+      .attr(
+        "y",
+        (d) => waveHeight / 2 - Math.max(heightScale(Number(d)) / 2, minHeight),
+      )
+      .attr("height", (d) => Math.max(heightScale(Number(d)), minHeight));
+  }
+
+  protected drawCurve(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight * devicePixelRatio;
+
+    const xScale = d3
+      .scaleLinear()
+      .domain([0, frequencyData.length])
+      .range([0, waveWidth]);
+
+    const yScale = d3.scaleLinear().domain([0, 255]).range([waveHeight, 0]);
+
+    const curveGenerator = d3
+      .line<Uint8Array[number]>()
+      .x((_, i) => xScale(i))
+      .y((d) => yScale(d))
+      .curve(d3.curveNatural);
+
+    const pathData = curveGenerator(frequencyData);
+
+    d3.select(svg)
+      .selectAll("path.curve")
+      .data([frequencyData])
+      .join("path")
+      .attr("class", "curve")
+      .attr("d", pathData)
+      .attr("fill", "none")
+      .attr("stroke", "currentColor")
+      .attr("stroke-width", 4);
+  }
+
+  protected drawPixel(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight;
+    const baseline = waveHeight / 2;
+
+    const barX = d3
+      .scaleBand()
+      .domain(d3.range(frequencyData.length).map(String))
+      .rangeRound([0, waveWidth])
+      .paddingInner(0.03)
+      .paddingOuter(0.02);
+
+    const height = d3.scaleLinear().domain([0, 255]).range([0, baseline]);
+
+    const bars = d3.select(svg).selectAll("rect").data(frequencyData);
+
+    bars
+      .enter()
+      .append("rect")
+      // @ts-ignore Not sure why this doesn't pass typechecks.
+      .merge(bars)
+      .attr("x", (_, i) => barX(String(i)) || 0)
+      .attr("y", (value) => baseline - height(value))
+      .attr("width", barX.bandwidth())
+      .attr("height", (value) => height(value) * 2);
+
+    bars.exit().remove();
+  }
+  protected drawWave(svg: SVGElement, frequencyData: Uint8Array) {
+    const waveWidth = svg.clientWidth * devicePixelRatio;
+    const waveHeight = svg.clientHeight;
+
+    const barX = d3
+      .scaleBand()
+      .domain(d3.range(frequencyData.length).map(String))
+      .rangeRound([0, waveWidth])
+      .paddingInner(0.03)
+      .paddingOuter(0.02);
+
+    const height = d3
+      .scaleLinear()
+      .domain([0, 255])
+      .range([0, waveHeight / 2]);
+
+    d3.select(svg)
+      .selectAll("line.baseline")
+      .data([0])
+      .join("line")
+      .attr("class", "baseline")
+      .attr("x1", (_, i) => barX(String(i)) || 0)
+      .attr("x2", waveWidth)
+      .attr("y1", waveHeight / 2)
+      .attr("y2", waveHeight / 2)
+      .attr("stroke", "currentColor")
+      .attr("stroke-width", 2);
+
+    const bars = d3.select(svg).selectAll("rect").data(frequencyData);
+
+    bars
+      .enter()
+      .append("rect")
+      // @ts-ignore Not sure why this doesn't pass typechecks.
+      .merge(bars)
+      .attr("x", (_, i) => barX(String(i)) || 0)
+      .attr("y", (value) => waveHeight / 2 - height(value))
+      .attr("width", barX.bandwidth())
+      .attr("height", (value) => height(value) * 2);
+
+    bars.exit().remove();
+  }
+
+  frameTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.target.audioBufferTask.status] as const,
+    task: async () => {
+      await this.target.audioBufferTask.taskComplete;
+    },
+  });
+
+  protected async updated() {
+    const svg = this.svgRef.value;
+    if (!svg) {
+      return;
+    }
+    if (!this.target.audioBufferTask.value) {
+      return;
+    }
+    if (this.target.ownCurrentTimeMs > 0) {
+      const audioContext = new OfflineAudioContext(2, 48000 / 25, 48000);
+      const audioBufferSource = audioContext.createBufferSource();
+      audioBufferSource.buffer = this.target.audioBufferTask.value.buffer;
+      const analyser = audioContext.createAnalyser();
+      analyser.fftSize = 256;
+      audioBufferSource.connect(analyser);
+
+      audioBufferSource.start(
+        0,
+        Math.max(
+          0,
+          (this.target.ownCurrentTimeMs -
+            this.target.audioBufferTask.value.startOffsetMs) /
+            1000,
+        ),
+        48000 / 1000,
+      );
+      await audioContext.startRendering();
+      const frequencyData = new Uint8Array(analyser.frequencyBinCount);
+      analyser.getByteFrequencyData(frequencyData);
+      const rect = this.getBoundingClientRect();
+
+      svg.setAttribute("width", (rect.width * devicePixelRatio).toString());
+      svg.setAttribute("height", (rect.height * devicePixelRatio).toString());
+
+      switch (this.mode) {
+        case "bars":
+          this.drawBars(svg, frequencyData);
+          break;
+        case "bricks":
+          this.drawBricks(svg, frequencyData);
+          break;
+        case "curve":
+          this.drawCurve(svg, frequencyData);
+          break;
+        case "line":
+          this.drawLine(svg, frequencyData);
+          break;
+        case "pixel":
+          this.drawPixel(svg, frequencyData);
+          break;
+        case "wave":
+          this.drawWave(svg, frequencyData);
+          break;
+        case "roundBars":
+          this.drawRoundBars(svg, frequencyData);
+          break;
+        case "equalizer":
+          this.drawEqualizer(svg, frequencyData);
+          break;
+      }
+    }
+  }
+
+  get target() {
+    const target = document.querySelector(this.getAttribute("target") ?? "");
+    if (target instanceof EFAudio || target instanceof EFVideo) {
+      return target;
+    }
+    throw new Error("Invalid target, must be an EFAudio element");
+  }
+}

package/src/elements/FetchMixin.ts

@@ -0,0 +1,19 @@
+import { consume } from "@lit/context";
+import type { LitElement } from "lit";
+import { fetchContext } from "../gui/EFWorkbench";
+import { state } from "lit/decorators/state.js";
+
+export declare class FetchMixinInterface {
+  fetch: typeof fetch;
+}
+
+type Constructor<T = {}> = new (...args: any[]) => T;
+export function FetchMixin<T extends Constructor<LitElement>>(superClass: T) {
+  class FetchElement extends superClass {
+    @consume({ context: fetchContext, subscribe: true })
+    @state()
+    fetch = fetch.bind(window);
+  }
+
+  return FetchElement as Constructor<FetchMixinInterface> & T;
+}

package/src/elements/TimegroupController.ts

@@ -0,0 +1,25 @@
+import type { ReactiveController, LitElement } from "lit";
+import type { EFTimegroup } from "./EFTimegroup";
+
+export class TimegroupController implements ReactiveController {
+  constructor(
+    private host: EFTimegroup,
+    private child: { currentTimeMs: number; startTimeMs?: number } & LitElement,
+  ) {
+    this.host.addController(this);
+  }
+
+  remove() {
+    this.host.removeController(this);
+  }
+
+  hostDisconnected(): void {
+    this.host.removeController(this);
+  }
+
+  hostUpdated(): void {
+    this.child.requestUpdate();
+    this.child.currentTimeMs =
+      this.host.currentTimeMs - (this.child.startTimeMs ?? 0);
+  }
+}

package/src/elements/durationConverter.ts

@@ -0,0 +1,6 @@
+import { parseTimeToMs } from "./parseTimeToMs";
+
+export const durationConverter = {
+  fromAttribute: (value: string): number => parseTimeToMs(value),
+  toAttribute: (value: number) => `${value}s`,
+};

package/src/elements/parseTimeToMs.ts

@@ -0,0 +1,9 @@
+export const parseTimeToMs = (time: string) => {
+  if (time.endsWith("ms")) {
+    return Number.parseFloat(time);
+  }
+  if (time.endsWith("s")) {
+    return Number.parseFloat(time) * 1000;
+  }
+  throw new Error("Time must be in milliseconds or seconds (10s, 10000ms)");
+};

package/src/elements/util.ts

@@ -0,0 +1,24 @@
+import { EFTimegroup } from "./EFTimegroup";
+
+export const getRootTimeGroup = (element: Element): EFTimegroup | null => {
+  let bestCandidate: EFTimegroup | null = null;
+
+  let currentElement: Element | null = element;
+  while (currentElement) {
+    if (currentElement instanceof EFTimegroup) {
+      bestCandidate = currentElement;
+    }
+    currentElement = currentElement.parentElement;
+  }
+
+  return bestCandidate;
+};
+
+export const getStartTimeMs = (element: Element): number => {
+  const nearestTimeGroup = element.closest("ef-timegroup");
+  if (!(nearestTimeGroup instanceof EFTimegroup)) {
+    return 0;
+  }
+
+  return nearestTimeGroup.startTimeMs;
+};