@editframe/elements 0.6.0-beta.16 → 0.6.0-beta.17

package/src/elements/EFTimegroup.browsertest.ts

@@ -0,0 +1,333 @@
+import { describe, test, assert, beforeEach } from "vitest";
+import {
+  LitElement,
+  type TemplateResult,
+  html,
+  render as litRender,
+} from "lit";
+import { EFTimegroup } from "./EFTimegroup";
+import "./EFTimegroup";
+import { customElement } from "lit/decorators/custom-element.js";
+import { EFTemporal } from "./EFTemporal";
+
+beforeEach(() => {
+  for (let i = 0; i < localStorage.length; i++) {
+    const key = localStorage.key(i);
+    if (typeof key !== "string") continue;
+    localStorage.removeItem(key);
+  }
+  while (document.body.children.length) {
+    document.body.children[0]?.remove();
+  }
+});
+
+@customElement("test-temporal")
+class TestTemporal extends EFTemporal(LitElement) {
+  get hasOwnDuration(): boolean {
+    return true;
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "test-temporal": TestTemporal;
+  }
+}
+
+const renderTimegroup = (result: TemplateResult) => {
+  const container = document.createElement("div");
+  litRender(result, container);
+  const firstChild = container.firstElementChild;
+  if (!firstChild) {
+    throw new Error("No first child found");
+  }
+  if (!(firstChild instanceof EFTimegroup)) {
+    throw new Error("First child is not an EFTimegroup");
+  }
+  document.body.appendChild(firstChild);
+  return firstChild;
+};
+
+describe(`<ef-timegroup mode="fixed">`, () => {
+  test("can explicitly set a duration in seconds", async () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup mode="fixed" duration="10s"></ef-timegroup>`,
+    );
+    assert.equal(timegroup.durationMs, 10_000);
+  });
+
+  test("can explicitly set a duration in milliseconds", async () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup mode="fixed" duration="10ms"></ef-timegroup>`,
+    );
+    assert.equal(timegroup.durationMs, 10);
+  });
+});
+
+describe(`<ef-timegroup mode="sequence">`, () => {
+  test("fixed duration is ignored", () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup mode="sequence" duration="10s"></ef-timegroup>`,
+    );
+    assert.equal(timegroup.durationMs, 0);
+  });
+
+  test("duration is the sum of child time groups", async () => {
+    const timegroup = renderTimegroup(
+      html`
+        <ef-timegroup mode="sequence">
+          <ef-timegroup mode="fixed" duration="5s"></ef-timegroup>
+          <ef-timegroup mode="fixed" duration="5s"></ef-timegroup>
+        </ef-timegroup>
+      `,
+    );
+    assert.equal(timegroup.durationMs, 10_000);
+  });
+
+  test("duration can include any element with a durationMs value", async () => {
+    const timegroup = renderTimegroup(
+      html`
+        <ef-timegroup mode="sequence">
+          <ef-timegroup mode="fixed" duration="5s"></ef-timegroup>
+          <test-temporal duration="5s"></test-temporal>
+        </ef-timegroup>
+      `,
+    );
+
+    assert.equal(timegroup.durationMs, 10_000);
+  });
+
+  test("arbitrary html does not factor into the calculation of a sequence duration", () => {
+    const timegroup = renderTimegroup(
+      html`
+        <ef-timegroup mode="sequence">
+          <div>
+            <ef-timegroup mode="fixed" duration="5s"></ef-timegroup>
+          </div>
+        </ef-timegroup>
+      `,
+    );
+    assert.equal(timegroup.durationMs, 5_000);
+  });
+
+  test("nested time groups do not factor into the calculation of a sequence duration", async () => {
+    const timegroup = renderTimegroup(
+      html`
+        <ef-timegroup mode="sequence">
+          <ef-timegroup mode="fixed" duration="5s">
+            <ef-timegroup mode="fixed" duration="5s"></ef-timegroup>
+          </ef-timegroup>
+        </ef-timegroup>
+      `,
+    );
+    assert.equal(timegroup.durationMs, 5_000);
+  });
+});
+
+describe(`<ef-timegroup mode="contain">`, () => {
+  test("fixed duration is ignored", () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup mode="contain" duration="10s"></ef-timegroup>`,
+    );
+    assert.equal(timegroup.durationMs, 0);
+  });
+
+  test("duration is the maximum of it's child time groups", async () => {
+    const timegroup = renderTimegroup(
+      html`
+        <ef-timegroup mode="contain">
+          <ef-timegroup mode="fixed" duration="5s"></ef-timegroup>
+          <ef-timegroup mode="fixed" duration="10s"></ef-timegroup>
+        </ef-timegroup>
+      `,
+    );
+    assert.equal(timegroup.durationMs, 10_000);
+  });
+});
+
+describe("startTimeMs", () => {
+  test("is computed relative to the root time group", async () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup id="root" mode="sequence">
+        <ef-timegroup id="a" mode="fixed" duration="5s"></ef-timegroup>
+        <ef-timegroup id="b" mode="sequence">
+          <ef-timegroup id="c" mode="fixed" duration="5s"></ef-timegroup>
+          <ef-timegroup id="d" mode="contain">
+            <ef-timegroup id="e" mode="fixed" duration="5s"></ef-timegroup>
+            <ef-timegroup id="f" mode="fixed" duration="5s"></ef-timegroup>
+          </ef-timegroup>
+        </ef-timegroup>
+      </ef-timegroup>`,
+    );
+
+    const a = timegroup.querySelector("#a") as EFTimegroup;
+    const b = timegroup.querySelector("#b") as EFTimegroup;
+    const c = timegroup.querySelector("#c") as EFTimegroup;
+    const d = timegroup.querySelector("#d") as EFTimegroup;
+    const e = timegroup.querySelector("#e") as EFTimegroup;
+    const f = timegroup.querySelector("#f") as EFTimegroup;
+
+    assert.equal(a.startTimeMs, 0);
+    assert.equal(b.startTimeMs, 5_000);
+    assert.equal(c.startTimeMs, 5_000);
+    assert.equal(d.startTimeMs, 10_000);
+    assert.equal(e.startTimeMs, 10_000);
+    assert.equal(f.startTimeMs, 10_000);
+  });
+
+  //   // TODO: Rethink offset math, it shouldn't effect the duration of a temporal item
+  //   // but actually change the start and end time.
+  //   litTest.skip("can be offset with offset attribute", async ({ container }) => {
+  //     render(
+  //       html`<ef-timegroup id="root" mode="contain">
+  //         <test-temporal id="a" duration="5s" offset="5s"></test-temporal>
+  //       </ef-timegroup> `,
+  //       container,
+  //     );
+
+  //     const root = container.querySelector("#root") as EFTimegroup;
+  //     const a = container.querySelector("#a") as TestTemporal;
+
+  //     assert.equal(a.durationMs, 5_000);
+  //     assert.equal(root.durationMs, 10_000);
+  //     assert.equal(a.startTimeMs, 5_000);
+  //   });
+
+  // litTest.skip(
+  //   "offsets do not affect start time when in a sequence group",
+  //   async ({ container }) => {
+  //     render(
+  //       html`<ef-timegroup id="root" mode="sequence">
+  //         <test-temporal id="a" duration="5s"></test-temporal>
+  //         <test-temporal id="b" duration="5s" offset="5s"></test-temporal>
+  //       </ef-timegroup> `,
+  //       container,
+  //     );
+
+  //     const root = container.querySelector("#root") as EFTimegroup;
+  //     const a = container.querySelector("#a") as TestTemporal;
+  //     const b = container.querySelector("#b") as TestTemporal;
+
+  //     assert.equal(root.durationMs, 10_000);
+  //     assert.equal(a.startTimeMs, 0);
+  //     assert.equal(b.startTimeMs, 5_000);
+  //   },
+  // );
+
+  test("temporal elements default to expand to fill a timegroup", async () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup id="root" mode="fixed" duration="10s">
+        <test-temporal id="a"></test-temporal>
+      </ef-timegroup> `,
+    );
+
+    const a = timegroup.querySelector("#a") as TestTemporal;
+
+    assert.equal(timegroup.durationMs, 10_000);
+    assert.equal(a.durationMs, 10_000);
+  });
+
+  test("element's parentTimegroup updates as they move", async () => {
+    const container = document.createElement("div");
+    document.body.appendChild(container);
+    const timegroup1 = document.createElement("ef-timegroup");
+    timegroup1.setAttribute("mode", "fixed");
+    timegroup1.setAttribute("duration", "5s");
+
+    const timegroup2 = document.createElement("ef-timegroup");
+    timegroup2.setAttribute("mode", "fixed");
+    timegroup2.setAttribute("duration", "5s");
+
+    container.appendChild(timegroup1);
+    container.appendChild(timegroup2);
+
+    const temporal = document.createElement("test-temporal");
+
+    timegroup1.appendChild(temporal);
+
+    assert.equal(temporal.parentTimegroup, timegroup1);
+
+    timegroup2.appendChild(temporal);
+    assert.equal(temporal.parentTimegroup, timegroup2);
+  });
+
+  test("elements can access their root temporal element", async () => {
+    const root = renderTimegroup(
+      html`<ef-timegroup id="root" mode="contain" duration="10s">
+        <ef-timegroup id="a" mode="contain">
+          <div>
+            <ef-timegroup id="b" mode="contain">
+              <div>
+                <test-temporal id="c" duration="5s"></test-temporal>
+              </div>
+            </ef-timegroup>
+          </div>
+        </ef-timegroup>
+      </ef-timegroup> `,
+    );
+
+    const a = root.querySelector("#a") as EFTimegroup;
+    const b = root.querySelector("#b") as EFTimegroup;
+    const c = root.querySelector("#c") as TestTemporal;
+
+    assert.equal(root.rootTimegroup, root);
+
+    assert.equal(a.rootTimegroup, root);
+    assert.equal(b.rootTimegroup, root);
+    assert.equal(c.rootTimegroup, root);
+  });
+});
+
+describe("setting currentTime", () => {
+  test("persists in localStorage if the timegroup has an id and is in the dom", async () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup id="root" mode="fixed" duration="10s"></ef-timegroup>`,
+    );
+    document.body.appendChild(timegroup);
+    assert.isNull(localStorage.getItem(timegroup.storageKey));
+    timegroup.currentTime = 5_000;
+    assert.equal(localStorage.getItem(timegroup.storageKey), "5000");
+    timegroup.remove();
+  });
+
+  test("does not persist in localStorage if the timegroup has no id", async () => {
+    const timegroup = renderTimegroup(
+      html`<ef-timegroup mode="fixed" duration="10s"></ef-timegroup>`,
+    );
+    document.body.appendChild(timegroup);
+    timegroup.currentTime = 5_000;
+    timegroup.removeAttribute("id");
+    assert.throws(() => {
+      assert.isNull(localStorage.getItem(timegroup.storageKey));
+    }, "Timegroup must have an id to use localStorage");
+    timegroup.remove();
+  });
+
+  test("nested items derive their ownCurrentTimeMs", async () => {
+    const timegroup = renderTimegroup(
+      html`
+        <ef-timegroup id="root" mode="sequence">
+          <ef-timegroup id="a" mode="fixed" duration="5s"> </ef-timegroup>
+          <ef-timegroup id="b" mode="fixed" duration="5s"> </ef-timegroup>
+        </ef-timegroup>
+      `,
+    );
+
+    const root = timegroup;
+    const a = timegroup.querySelector("#a") as EFTimegroup;
+    const b = timegroup.querySelector("#b") as EFTimegroup;
+
+    assert.equal(a.ownCurrentTimeMs, 0);
+    assert.equal(b.ownCurrentTimeMs, 0);
+
+    root.currentTimeMs = 2_500;
+
+    assert.equal(a.ownCurrentTimeMs, 2_500);
+    assert.equal(b.ownCurrentTimeMs, 0);
+
+    root.currentTimeMs = 7_500;
+
+    assert.equal(a.ownCurrentTimeMs, 5_000);
+    assert.equal(b.ownCurrentTimeMs, 2_500);
+  });
+});

package/src/elements/EFTimegroup.ts

@@ -0,0 +1,381 @@
+import { LitElement, html, css, type PropertyValueMap } from "lit";
+import { provide } from "@lit/context";
+import { customElement, property } from "lit/decorators.js";
+import {
+  EFTemporal,
+  isEFTemporal,
+  shallowGetTemporalElements,
+  timegroupContext,
+} from "./EFTemporal";
+import { TimegroupController } from "./TimegroupController";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+import { deepGetMediaElements } from "./EFMedia";
+import { Task } from "@lit/task";
+
+export const shallowGetTimegroups = (
+  element: Element,
+  groups: EFTimegroup[] = [],
+) => {
+  for (const child of Array.from(element.children)) {
+    if (child instanceof EFTimegroup) {
+      groups.push(child);
+    } else {
+      shallowGetTimegroups(child, groups);
+    }
+  }
+  return groups;
+};
+
+@customElement("ef-timegroup")
+export class EFTimegroup extends EFTemporal(LitElement) {
+  static styles = css`
+    :host {
+      display: block;
+      width: 100%;
+      height: 100%;
+      position: relative;
+      top: 0;
+    }
+  `;
+
+  @provide({ context: timegroupContext })
+  _timeGroupContext = this;
+
+  #currentTime = 0;
+
+  @property({
+    type: String,
+    attribute: "mode",
+  })
+  mode: "fixed" | "sequence" | "contain" = "sequence";
+
+  @property({ type: Number })
+  set currentTime(time: number) {
+    this.#currentTime = Math.max(0, Math.min(time, this.durationMs / 1000));
+    try {
+      if (this.id) {
+        if (this.isConnected) {
+          localStorage.setItem(this.storageKey, time.toString());
+        }
+      }
+    } catch (error) {
+      console.warn("Failed to save time to localStorage", error);
+    }
+  }
+  get currentTime() {
+    return this.#currentTime;
+  }
+  get currentTimeMs() {
+    return this.currentTime * 1000;
+  }
+  set currentTimeMs(ms: number) {
+    this.currentTime = ms / 1000;
+  }
+
+  @property({
+    attribute: "crossover",
+    converter: {
+      fromAttribute: (value: string): number => {
+        if (value.endsWith("ms")) {
+          return Number.parseFloat(value);
+        }
+        if (value.endsWith("s")) {
+          return Number.parseFloat(value) * 1000;
+        }
+        throw new Error(
+          "`crossover` MUST be in milliseconds or seconds (10s, 10000ms)",
+        );
+      },
+      toAttribute: (value: number) => `${value}ms`,
+    },
+  })
+  crossoverMs = 0;
+
+  render() {
+    return html`<slot></slot> `;
+  }
+
+  maybeLoadTimeFromLocalStorage() {
+    if (this.id) {
+      try {
+        return Number.parseFloat(localStorage.getItem(this.storageKey) || "0");
+      } catch (error) {
+        console.warn("Failed to load time from localStorage", error);
+      }
+    }
+    return 0;
+  }
+
+  connectedCallback() {
+    super.connectedCallback();
+    if (this.id) {
+      this.#currentTime = this.maybeLoadTimeFromLocalStorage();
+    }
+
+    if (this.parentTimegroup) {
+      new TimegroupController(this.parentTimegroup, this);
+    }
+
+    if (this.shouldWrapWithWorkbench()) {
+      this.wrapWithWorkbench();
+    }
+  }
+
+  get storageKey() {
+    if (!this.id) {
+      throw new Error("Timegroup must have an id to use localStorage.");
+    }
+    return `ef-timegroup-${this.id}`;
+  }
+
+  get crossoverStartMs() {
+    const parentTimeGroup = this.parentTimegroup;
+    if (!parentTimeGroup || !this.previousElementSibling) {
+      return 0;
+    }
+
+    return parentTimeGroup.crossoverMs;
+  }
+  get crossoverEndMs() {
+    const parentTimeGroup = this.parentTimegroup;
+    if (!parentTimeGroup || !this.nextElementSibling) {
+      return 0;
+    }
+
+    return parentTimeGroup.crossoverMs;
+  }
+
+  get durationMs() {
+    switch (this.mode) {
+      case "fixed":
+        return super.durationMs || 0;
+      case "sequence": {
+        let duration = 0;
+        for (const node of this.childTemporals) {
+          duration += node.durationMs;
+        }
+        return duration;
+      }
+      case "contain": {
+        let maxDuration = 0;
+        for (const node of this.childTemporals) {
+          if (node.hasOwnDuration) {
+            maxDuration = Math.max(maxDuration, node.durationMs);
+          }
+        }
+        return maxDuration;
+      }
+      default:
+        throw new Error(`Invalid time mode: ${this.mode}`);
+    }
+  }
+
+  async waitForMediaDurations() {
+    return await Promise.all(
+      deepGetMediaElements(this).map(
+        (media) => media.trackFragmentIndexLoader.taskComplete,
+      ),
+    );
+  }
+
+  get childTemporals() {
+    return shallowGetTemporalElements(this);
+  }
+
+  protected updated(
+    changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+  ): void {
+    super.updated(changedProperties);
+    if (
+      changedProperties.has("currentTime") ||
+      changedProperties.has("ownCurrentTimeMs")
+    ) {
+      const timelineTimeMs = (this.rootTimegroup ?? this).currentTimeMs;
+      if (
+        this.startTimeMs > timelineTimeMs ||
+        this.endTimeMs < timelineTimeMs
+      ) {
+        this.style.display = "none";
+        return;
+      }
+      this.style.display = "";
+
+      const animations = this.getAnimations({ subtree: true });
+      this.style.setProperty(
+        "--ef-duration",
+        `${this.durationMs + this.crossoverEndMs + this.crossoverStartMs}ms`,
+      );
+
+      for (const animation of animations) {
+        if (animation.playState === "running") {
+          animation.pause();
+        }
+        const effect = animation.effect;
+        if (!(effect && effect instanceof KeyframeEffect)) {
+          return;
+        }
+        const target = effect.target;
+        // TODO: better generalize work avoidance for temporal elements
+        if (!target) {
+          return;
+        }
+        if (target.closest("ef-timegroup") !== this) {
+          return;
+        }
+
+        // Important to avoid going to the end of the animation
+        // or it will reset awkwardly.
+        if (isEFTemporal(target)) {
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(target.ownCurrentTimeMs, duration - 1 + delay),
+          );
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        } else if (target) {
+          const nearestTimegroup = target.closest("ef-timegroup");
+          if (!nearestTimegroup) {
+            return;
+          }
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(nearestTimegroup.ownCurrentTimeMs, duration - 1 + delay),
+          );
+
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        }
+      }
+    }
+  }
+
+  shouldWrapWithWorkbench() {
+    return (
+      EF_INTERACTIVE &&
+      this.closest("ef-timegroup") === this &&
+      this.closest("ef-workbench") === null
+    );
+  }
+
+  wrapWithWorkbench() {
+    const workbench = document.createElement("ef-workbench");
+    document.body.append(workbench);
+    if (!this.hasAttribute("id")) {
+      this.setAttribute("id", "root-this");
+    }
+    this.setAttribute("slot", "canvas");
+    workbench.append(this as unknown as Element);
+
+    const filmstrip = document.createElement("ef-filmstrip");
+    filmstrip.setAttribute("slot", "timeline");
+    filmstrip.setAttribute("target", `#${this.id}`);
+    workbench.append(filmstrip);
+  }
+
+  get hasOwnDuration() {
+    return true;
+  }
+
+  get efElements() {
+    return Array.from(
+      this.querySelectorAll(
+        "ef-audio, ef-video, ef-image, ef-captions, ef-waveform",
+      ),
+    );
+  }
+
+  async renderAudio(fromMs: number, toMs: number) {
+    await this.waitForMediaDurations();
+
+    const durationMs = toMs - fromMs;
+    const audioContext = new OfflineAudioContext(
+      2,
+      Math.round((48000 * durationMs) / 1000),
+      48000,
+    );
+
+    await Promise.all(
+      deepGetMediaElements(this).map(async (mediaElement) => {
+        await mediaElement.trackFragmentIndexLoader.taskComplete;
+
+        const mediaStartsBeforeEnd = mediaElement.startTimeMs <= toMs;
+        const mediaEndsAfterStart = mediaElement.endTimeMs >= fromMs;
+        const mediaOverlaps = mediaStartsBeforeEnd && mediaEndsAfterStart;
+        if (!mediaOverlaps || mediaElement.defaultAudioTrackId === undefined) {
+          return;
+        }
+
+        const audio = await mediaElement.fetchAudioSpanningTime(fromMs, toMs);
+        if (!audio) {
+          throw new Error("Failed to fetch audio");
+        }
+
+        const ctxStartMs = Math.max(0, mediaElement.startTimeMs - fromMs);
+        const ctxEndMs = Math.min(durationMs, mediaElement.endTimeMs - fromMs);
+        const ctxDurationMs = ctxEndMs - ctxStartMs;
+
+        const offset =
+          Math.max(0, fromMs - mediaElement.startTimeMs) - audio.startMs;
+
+        const bufferSource = audioContext.createBufferSource();
+        bufferSource.buffer = await audioContext.decodeAudioData(
+          await audio.blob.arrayBuffer(),
+        );
+        bufferSource.connect(audioContext.destination);
+
+        bufferSource.start(
+          ctxStartMs / 1000,
+          offset / 1000,
+          ctxDurationMs / 1000,
+        );
+      }),
+    );
+
+    return await audioContext.startRendering();
+  }
+
+  async loadMd5Sums() {
+    const efElements = this.efElements;
+    const loaderTasks: Promise<any>[] = [];
+    for (const el of efElements) {
+      const md5SumLoader = (el as any).md5SumLoader;
+      if (md5SumLoader instanceof Task) {
+        md5SumLoader.run();
+        loaderTasks.push(md5SumLoader.taskComplete);
+      }
+    }
+
+    await Promise.all(loaderTasks);
+
+    efElements.map((el) => {
+      if ("productionSrc" in el && el.productionSrc instanceof Function) {
+        el.setAttribute("src", el.productionSrc());
+      }
+    });
+  }
+
+  frameTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.ownCurrentTimeMs, this.currentTimeMs] as const,
+    task: async ([], { signal: _signal }) => {
+      let fullyUpdated = await this.updateComplete;
+      while (!fullyUpdated) {
+        fullyUpdated = await this.updateComplete;
+      }
+    },
+  });
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "ef-timegroup": EFTimegroup & Element;
+  }
+}