@editframe/elements 0.6.0-beta.16 → 0.6.0-beta.17

package/src/gui/EFFilmstrip.ts

@@ -0,0 +1,766 @@
+import { EFTimegroup } from "../elements/EFTimegroup";
+import {
+  LitElement,
+  html,
+  css,
+  nothing,
+  type TemplateResult,
+  type ReactiveController,
+  type PropertyValueMap,
+} from "lit";
+import {
+  customElement,
+  property,
+  eventOptions,
+  state,
+} from "lit/decorators.js";
+import { styleMap } from "lit/directives/style-map.js";
+import { ref, createRef } from "lit/directives/ref.js";
+import { EFImage } from "../elements/EFImage";
+import { EFAudio } from "../elements/EFAudio";
+import { EFVideo } from "../elements/EFVideo";
+import { EFCaptions, EFCaptionsActiveWord } from "../elements/EFCaptions";
+import { EFWaveform } from "../elements/EFWaveform";
+import type { TemporalMixinInterface } from "../elements/EFTemporal";
+import { TimegroupController } from "../elements/TimegroupController";
+import { consume } from "@lit/context";
+import { type FocusContext, focusContext, focusedElement } from "./EFWorkbench";
+import { TWMixin } from "./TWMixin";
+import { msToTimeCode } from "@/av/msToTimeCode";
+
+class ElementFilmstripController implements ReactiveController {
+  constructor(
+    private host: LitElement,
+    private filmstrip: FilmstripItem,
+  ) {
+    this.host.addController(this);
+  }
+
+  remove() {
+    this.host.removeController(this);
+  }
+
+  hostDisconnected() {
+    this.host.removeController(this);
+  }
+
+  hostUpdated(): void {
+    this.filmstrip.requestUpdate();
+  }
+}
+
+const CommonEffectKeys = new Set([
+  "offset",
+  "easing",
+  "composite",
+  "computedOffset",
+]);
+
+class FilmstripItem extends TWMixin(LitElement) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+    `,
+  ];
+
+  @consume({ context: focusContext })
+  focusContext?: FocusContext;
+
+  @consume({ context: focusedElement, subscribe: true })
+  focusedElement?: HTMLElement | null;
+
+  get isFocused() {
+    return this.element && this.focusedElement === this.element;
+  }
+
+  @property({ type: HTMLElement, attribute: false })
+  element: TemporalMixinInterface & LitElement = new EFTimegroup();
+
+  @property({ type: Number })
+  pixelsPerMs = 0.04;
+
+  get styles() {
+    return {
+      position: "relative",
+      left: `${this.pixelsPerMs * this.element.startTimeWithinParentMs}px`,
+      width: `${this.pixelsPerMs * this.element.durationMs}px`,
+    };
+  }
+
+  render() {
+    return html` <div class="" style=${styleMap(this.styles)}>
+      <div
+        @mouseenter=${() => {
+          if (this.focusContext) {
+            this.focusContext.focusedElement = this.element;
+          }
+        }}
+        @mouseleave=${() => {
+          if (this.focusContext) {
+            this.focusContext.focusedElement = null;
+          }
+        }}
+        ?data-focused=${this.isFocused}
+        class="border-outset relative mb-[1px] block h-[1.25rem] text-nowrap border border-slate-500 bg-blue-200 text-sm data-[focused]:bg-slate-400"
+      >
+        ${this.animations()}
+      </div>
+      ${this.renderChildren()}
+    </div>`;
+  }
+
+  renderChildren(): Array<TemplateResult<1> | typeof nothing> | typeof nothing {
+    return renderFilmStripChildren(
+      Array.from(this.element.children),
+      this.pixelsPerMs,
+    );
+  }
+
+  contents() {
+    return html``;
+  }
+
+  animations() {
+    const animations = this.element.getAnimations();
+    return animations.map((animation) => {
+      const effect = animation.effect;
+      if (!(effect instanceof KeyframeEffect)) {
+        return nothing;
+      }
+      const start = effect.getTiming().delay ?? 0;
+      const duration = effect.getTiming().duration;
+      if (duration === null) {
+        return nothing;
+      }
+      const keyframes = effect.getKeyframes();
+      const firstKeyframe = keyframes[0];
+      if (!firstKeyframe) {
+        return nothing;
+      }
+      const properties = new Set(Object.keys(firstKeyframe));
+      for (const key of CommonEffectKeys) {
+        properties.delete(key);
+      }
+
+      return html`<div
+        class="relative h-[5px] bg-blue-500 opacity-50"
+        label="animation"
+        style=${styleMap({
+          left: `${this.pixelsPerMs * start}px`,
+          width: `${this.pixelsPerMs * Number(duration)}px`,
+        })}
+      >
+        <!-- <div class="text-nowrap">${Array.from(properties).join(" ")}</div> -->
+        ${effect.getKeyframes().map((keyframe) => {
+          return html`<div
+            class="absolute top-0 h-full w-1 bg-red-500"
+            style=${styleMap({
+              left: `${
+                this.pixelsPerMs * keyframe.computedOffset * Number(duration)
+              }px`,
+            })}
+          ></div>`;
+        })}
+      </div>`;
+    });
+  }
+
+  protected filmstripController?: ElementFilmstripController;
+
+  update(changedProperties: Map<string | number | symbol, unknown>) {
+    if (
+      changedProperties.has("element") &&
+      this.element instanceof LitElement
+    ) {
+      this.filmstripController?.remove();
+      this.filmstripController = new ElementFilmstripController(
+        this.element,
+        this,
+      );
+    }
+    super.update(changedProperties);
+  }
+}
+
+@customElement("ef-audio-filmstrip")
+export class EFAudioFilmstrip extends FilmstripItem {
+  contents() {
+    return html``;
+  }
+}
+
+@customElement("ef-video-filmstrip")
+export class EFVideoFilmstrip extends FilmstripItem {
+  contents() {
+    return html` 📼 `;
+  }
+}
+
+@customElement("ef-captions-filmstrip")
+export class EFCaptionsFilmstrip extends FilmstripItem {
+  contents() {
+    return html` 📝 `;
+  }
+}
+
+@customElement("ef-waveform-filmstrip")
+export class EFWaveformFilmstrip extends FilmstripItem {
+  contents() {
+    return html` 🌊 `;
+  }
+
+  renderChildren(): typeof nothing {
+    return nothing;
+  }
+}
+
+@customElement("ef-image-filmstrip")
+export class EFImageFilmstrip extends FilmstripItem {
+  contents() {
+    return html` 🖼️ `;
+  }
+}
+
+@customElement("ef-timegroup-filmstrip")
+export class EFTimegroupFilmstrip extends FilmstripItem {
+  contents() {
+    return html`
+      <span>TIME GROUP</span>
+      ${renderFilmStripChildren(
+        Array.from(this.element.children || []),
+        this.pixelsPerMs,
+      )}
+    </div>
+    `;
+  }
+}
+
+@customElement("ef-html-filmstrip")
+export class EFHTMLFilmstrip extends FilmstripItem {
+  contents() {
+    return html`
+      <span>${this.element.tagName}</span>
+      ${renderFilmStripChildren(
+        Array.from(this.element.children || []),
+        this.pixelsPerMs,
+      )}
+    `;
+  }
+}
+
+@customElement("ef-hierarchy-item")
+class EFHierarchyItem<
+  ElementType extends HTMLElement = HTMLElement,
+> extends TWMixin(LitElement) {
+  @property({ type: HTMLElement, attribute: false })
+  // @ts-expect-error This could be initialzed with any HTMLElement
+  element: ElementType = new EFTimegroup();
+
+  @consume({ context: focusContext })
+  focusContext?: FocusContext;
+
+  @consume({ context: focusedElement, subscribe: true })
+  focusedElement?: HTMLElement | null;
+
+  get icon(): TemplateResult<1> | string {
+    return "📼";
+  }
+
+  get isFocused() {
+    return this.element && this.focusedElement === this.element;
+  }
+
+  displayLabel(): TemplateResult<1> | string | typeof nothing {
+    return nothing;
+  }
+
+  render() {
+    return html` 
+      <div>
+        <div
+          ?data-focused=${this.isFocused}
+          class="peer 
+            flex h-[1.25rem] items-center overflow-hidden text-nowrap  border border-slate-500
+          bg-slate-200 pl-2 text-sm hover:bg-slate-400 data-[focused]:bg-slate-400"
+          @mouseenter=${() => {
+            if (this.focusContext) {
+              this.focusContext.focusedElement = this.element;
+            }
+          }}
+          @mouseleave=${() => {
+            if (this.focusContext) {
+              this.focusContext.focusedElement = null;
+            }
+          }}
+        >
+          ${this.icon} ${this.displayLabel()}
+        </div>
+        <div
+          class="p-[1px] pb-0 pl-2 pr-0 peer-hover:bg-slate-300 peer-data-[focused]:bg-slate-300 peer-hover:border-slate-400 peer-data-[focused]:border-slate-400""
+        >
+          ${this.renderChildren()}
+        </div>
+      </div>`;
+  }
+
+  renderChildren(): Array<TemplateResult<1>> | typeof nothing {
+    return renderHierarchyChildren(Array.from(this.element.children));
+  }
+}
+
+@customElement("ef-timegroup-hierarchy-item")
+class EFTimegroupHierarchyItem extends EFHierarchyItem<EFTimegroup> {
+  get icon() {
+    return "🕒";
+  }
+
+  displayLabel(): string | TemplateResult<1> | typeof nothing {
+    return this.element.mode ?? "(no mode)";
+  }
+}
+
+@customElement("ef-audio-hierarchy-item")
+class EFAudioHierarchyItem extends EFHierarchyItem {
+  get icon() {
+    return "🔊";
+  }
+
+  displayLabel() {
+    return this.element.getAttribute("src") ?? "(no src)";
+  }
+}
+
+@customElement("ef-video-hierarchy-item")
+class EFVideoHierarchyItem extends EFHierarchyItem {
+  get icon() {
+    return "📼";
+  }
+
+  displayLabel() {
+    return this.element.getAttribute("src") ?? "(no src)";
+  }
+}
+
+@customElement("ef-captions-hierarchy-item")
+class EFCaptionsHierarchyItem extends EFHierarchyItem {
+  get icon() {
+    return "📝 Captions";
+  }
+}
+
+@customElement("ef-captions-active-word-hierarchy-item")
+class EFCaptionsActiveWordHierarchyItem extends EFHierarchyItem {
+  get icon() {
+    return "🗣️ Active Word";
+  }
+}
+
+@customElement("ef-waveform-hierarchy-item")
+class EFWaveformHierarchyItem extends EFHierarchyItem {
+  get icon() {
+    return "🌊";
+  }
+
+  renderChildren(): typeof nothing {
+    return nothing;
+  }
+}
+
+@customElement("ef-image-hierarchy-item")
+class EFImageHierarchyItem extends EFHierarchyItem {
+  get icon() {
+    return "🖼️";
+  }
+
+  displayLabel() {
+    return this.element.getAttribute("src") ?? "(no src)";
+  }
+}
+
+@customElement("ef-html-hierarchy-item")
+class EFHTMLHierarchyItem extends EFHierarchyItem {
+  get icon() {
+    return html`<code>${`<${this.element.tagName.toLowerCase()}>`}</code>`;
+  }
+}
+
+const renderHierarchyChildren = (
+  children: Element[],
+): Array<TemplateResult<1>> => {
+  return children.map((child) => {
+    if (child instanceof EFTimegroup) {
+      return html`<ef-timegroup-hierarchy-item
+        .element=${child}
+      ></ef-timegroup-hierarchy-item>`;
+    }
+    if (child instanceof EFImage) {
+      return html`<ef-image-hierarchy-item
+        .element=${child}
+      ></ef-image-hierarchy-item>`;
+    }
+    if (child instanceof EFAudio) {
+      return html`<ef-audio-hierarchy-item
+        .element=${child}
+      ></ef-audio-hierarchy-item>`;
+    }
+    if (child instanceof EFVideo) {
+      return html`<ef-video-hierarchy-item
+        .element=${child}
+      ></ef-video-hierarchy-item>`;
+    }
+    if (child instanceof EFCaptions) {
+      return html`<ef-captions-hierarchy-item
+        .element=${child}
+      ></ef-captions-hierarchy-item>`;
+    }
+    if (child instanceof EFCaptionsActiveWord) {
+      return html`<ef-captions-active-word-hierarchy-item
+        .element=${child}
+      ></ef-captions-active-word-hierarchy-item>`;
+    }
+    if (child instanceof EFWaveform) {
+      return html`<ef-waveform-hierarchy-item
+        .element=${child}
+      ></ef-waveform-hierarchy-item>`;
+    }
+    return html`<ef-html-hierarchy-item
+      .element=${child}
+    ></ef-html-hierarchy-item>`;
+  });
+};
+
+const renderFilmStripChildren = (
+  children: Element[],
+  pixelsPerMs: number,
+): Array<TemplateResult<1> | typeof nothing> => {
+  return children.map((child) => {
+    if (child instanceof EFTimegroup) {
+      return html`<ef-timegroup-filmstrip
+        .element=${child}
+        .pixelsPerMs=${pixelsPerMs}
+      >
+      </ef-timegroup-filmstrip>`;
+    }
+    if (child instanceof EFImage) {
+      return html`<ef-image-filmstrip
+        .element=${child}
+        .pixelsPerMs=${pixelsPerMs}
+      ></ef-image-filmstrip>`;
+    }
+    if (child instanceof EFAudio) {
+      return html`<ef-audio-filmstrip
+        .element=${child}
+        .pixelsPerMs=${pixelsPerMs}
+      ></ef-audio-filmstrip>`;
+    }
+    if (child instanceof EFVideo) {
+      return html`<ef-video-filmstrip
+        .element=${child}
+        .pixelsPerMs=${pixelsPerMs}
+      ></ef-video-filmstrip>`;
+    }
+    if (child instanceof EFCaptions) {
+      return html`<ef-captions-filmstrip
+        .element=${child}
+        .pixelsPerMs=${pixelsPerMs}
+      ></ef-captions-filmstrip>`;
+    }
+    if (child instanceof EFWaveform) {
+      return html`<ef-waveform-filmstrip
+        .element=${child}
+        .pixelsPerMs=${pixelsPerMs}
+      ></ef-waveform-filmstrip>`;
+    }
+    return html`<ef-html-filmstrip
+      .element=${child}
+      .pixelsPerMs=${pixelsPerMs}
+    ></ef-html-filmstrip>`;
+  });
+};
+
+@customElement("ef-filmstrip")
+export class EFFilmStrip extends TWMixin(LitElement) {
+  @property({ type: Number })
+  pixelsPerMs = 0.04;
+
+  @property({ type: Number })
+  currentTimeMs = 0;
+
+  @property({ type: String, attribute: "target" })
+  targetSelector = "";
+
+  @state()
+  scrubbing = false;
+
+  @state()
+  playing = false;
+
+  @state()
+  timelineScrolltop = 0;
+
+  timegroupController?: TimegroupController;
+
+  connectedCallback(): void {
+    super.connectedCallback();
+    const target = this.target;
+    if (target) {
+      this.timegroupController = new TimegroupController(target, this);
+      // Set the current time to the last saved time to avoid a cycle
+      // where the filmstrip clobbers the time loaded from localStorage
+      this.currentTimeMs = target.currentTimeMs;
+    }
+  }
+
+  @eventOptions({ passive: false })
+  syncGutterScroll() {
+    if (this.gutter && this.hierarchyRef.value) {
+      this.hierarchyRef.value.scrollTop = this.gutter.scrollTop;
+      this.timelineScrolltop = this.gutter.scrollTop;
+    }
+  }
+
+  @eventOptions({ passive: false })
+  syncHierarchyScroll() {
+    if (this.gutter && this.hierarchyRef.value) {
+      this.gutter.scrollTop = this.hierarchyRef.value.scrollTop;
+      this.timelineScrolltop = this.hierarchyRef.value.scrollTop;
+    }
+  }
+
+  #lastTick?: DOMHighResTimeStamp;
+
+  advancePlayhead = (tick?: DOMHighResTimeStamp) => {
+    if (this.#lastTick && tick && this.target) {
+      this.target.currentTimeMs += tick - this.#lastTick;
+      if (this.target.currentTimeMs >= this.target.durationMs) {
+        this.playing = false;
+      }
+    }
+    this.#lastTick = tick;
+
+    if (this.playing) {
+      requestAnimationFrame(this.advancePlayhead);
+    }
+  };
+
+  @eventOptions({ capture: false })
+  scrub(e: MouseEvent) {
+    if (this.playing) {
+      return;
+    }
+    if (!this.scrubbing) {
+      return;
+    }
+    const gutter = this.shadowRoot?.querySelector("#gutter");
+    if (!gutter) {
+      return;
+    }
+    const rect = gutter.getBoundingClientRect();
+    if (this.target) {
+      const layerX = e.pageX - rect.left + gutter.scrollLeft;
+      this.target.currentTimeMs = layerX / this.pixelsPerMs;
+    }
+  }
+
+  @eventOptions({ capture: false })
+  startScrub(e: MouseEvent) {
+    e.preventDefault();
+    this.scrubbing = true;
+    // Running scrub in the current microtask doesn't
+    // result in an actual update. Not sure why.
+    queueMicrotask(() => {
+      const gutter = this.shadowRoot?.querySelector("#gutter");
+      if (!gutter) {
+        return;
+      }
+      const rect = gutter.getBoundingClientRect();
+      if (this.target) {
+        const layerX = e.pageX - rect.left + gutter.scrollLeft;
+        this.target.currentTimeMs = layerX / this.pixelsPerMs;
+      }
+    });
+    addEventListener(
+      "mouseup",
+      () => {
+        this.scrubbing = false;
+      },
+      { once: true },
+    );
+  }
+
+  @eventOptions({ passive: false })
+  scrollScrub(e: WheelEvent) {
+    if (this.target && this.gutter && !this.playing) {
+      e.preventDefault();
+      // Avoid over-scrolling to the left
+      if (
+        this.gutterRef.value &&
+        this.gutterRef.value.scrollLeft === 0 &&
+        e.deltaX < 0
+      ) {
+        this.gutter.scrollBy(0, e.deltaY);
+        return;
+      }
+
+      // Avoid over-scrolling to the right
+      if (
+        this.gutter.scrollWidth - this.gutter.scrollLeft ===
+          this.gutter.clientWidth &&
+        e.deltaX > 0
+      ) {
+        this.gutter.scrollBy(0, e.deltaY);
+        return;
+      }
+
+      if (this) {
+        this.gutter.scrollBy(e.deltaX, e.deltaY);
+        this.target.currentTimeMs += e.deltaX / this.pixelsPerMs;
+      }
+    }
+  }
+
+  gutterRef = createRef<HTMLDivElement>();
+  hierarchyRef = createRef<HTMLDivElement>();
+  playheadRef = createRef<HTMLDivElement>();
+
+  get gutter() {
+    return this.gutterRef.value;
+  }
+
+  render() {
+    const target = this.target;
+
+    return html` <div
+      class="grid h-full bg-slate-100"
+      style=${styleMap({
+        gridTemplateColumns: "200px 1fr",
+        gridTemplateRows: "1.5rem 1fr",
+      })}
+    >
+      <div
+        class="z-20 col-span-2 border-b-slate-600 bg-slate-100 shadow shadow-slate-300"
+      >
+        <input
+          type="range"
+          .value=${this.pixelsPerMs}
+          min="0.01"
+          max="0.1"
+          step="0.001"
+          @input=${(e: Event) => {
+            const target = e.target as HTMLInputElement;
+            this.pixelsPerMs = Number.parseFloat(target.value);
+          }}
+        />
+        <code>${msToTimeCode(this.currentTimeMs, true)} </code> /
+        <code>${msToTimeCode(target?.durationMs ?? 0, true)}</code>
+        ${
+          this.playing
+            ? html`<button
+              @click=${() => {
+                this.playing = false;
+              }}
+            >
+              ⏸️
+            </button>`
+            : html`<button
+              @click=${() => {
+                this.playing = true;
+              }}
+            >
+              ▶️
+            </button>`
+        }
+      </div>
+      <div
+        class="z-10 pl-1 pr-1 pt-2 shadow shadow-slate-600 overflow-auto"
+        ${ref(this.hierarchyRef)}
+        @scroll=${this.syncHierarchyScroll}
+      >
+        ${renderHierarchyChildren(Array.from(target?.children ?? []))}
+      </div>
+      <div
+        class="h-full w-full cursor-crosshair overflow-auto bg-slate-200 pt-2"
+        id="gutter"
+        ${ref(this.gutterRef)}
+        @scroll=${this.syncGutterScroll}
+        @wheel=${this.scrollScrub}
+      >
+        <div
+          class="relative h-full w-full"
+          style="width: ${this.pixelsPerMs * (target?.durationMs ?? 0)}px;"
+          @mousemove=${this.scrub}
+          @mousedown=${this.startScrub}
+        >
+          <div
+            class="border-red pointer-events-none absolute z-10 h-full w-[2px] border-r-2 border-red-700"
+            style=${styleMap({
+              left: `${this.pixelsPerMs * this.currentTimeMs}px`,
+              top: `${this.timelineScrolltop}px`,
+            })}
+            ${ref(this.playheadRef)}
+          ></div>
+
+          ${renderFilmStripChildren(
+            Array.from(target?.children || []),
+            this.pixelsPerMs,
+          )}
+        </div>
+      </div>
+    </div>`;
+  }
+
+  update(changedProperties: Map<string | number | symbol, unknown>) {
+    if (changedProperties.has("playing")) {
+      if (this.playing) {
+        this.advancePlayhead(0);
+      }
+    }
+    super.update(changedProperties);
+  }
+
+  updated(changes: PropertyValueMap<any> | Map<PropertyKey, unknown>) {
+    if (!this.target) {
+      return;
+    }
+    if (changes.has("currentTimeMs")) {
+      if (this.target.currentTimeMs !== this.currentTimeMs) {
+        this.target.currentTimeMs = this.currentTimeMs;
+      }
+    }
+  }
+
+  get target() {
+    if (this.getAttribute("target")) {
+      const target = document.querySelector(this.getAttribute("target") ?? "");
+      if (target instanceof EFTimegroup) {
+        return target;
+      }
+
+      throw new Error("Invalid target, must be an EFTimegroup element");
+    }
+    return undefined;
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "ef-filmstrip": EFFilmStrip;
+    "ef-timegroup-hierarchy-item": EFTimegroupHierarchyItem;
+    "ef-audio-hierarchy-item": EFAudioHierarchyItem;
+    "ef-video-hierarchy-item": EFVideoHierarchyItem;
+    "ef-captions-hierarchy-item": EFCaptionsHierarchyItem;
+    "ef-captions-active-word-hierarchy-item": EFCaptionsActiveWordHierarchyItem;
+    "ef-waveform-hierarchy-item": EFWaveformHierarchyItem;
+    "ef-image-hierarchy-item": EFImageHierarchyItem;
+    "ef-html-hierarchy-item": EFHTMLHierarchyItem;
+    "ef-timegroup-filmstrip": EFTimegroupFilmstrip;
+    "ef-audio-filmstrip": EFAudioFilmstrip;
+    "ef-video-filmstrip": EFVideoFilmstrip;
+    "ef-captions-filmstrip": EFCaptionsFilmstrip;
+    "ef-waveform-filmstrip": EFWaveformFilmstrip;
+    "ef-image-filmstrip": EFImageFilmstrip;
+    "ef-html-filmstrip": EFHTMLFilmstrip;
+  }
+}