@editframe/elements 0.7.0-beta.9 → 0.8.0-beta.1

package/package.json

@@ -1,16 +1,12 @@
 {
   "name": "@editframe/elements",
-  "version": "0.7.0-beta.9",
+  "version": "0.8.0-beta.1",
   "description": "",
   "exports": {
     ".": {
       "import": {
-        "default": "./dist/packages/elements/src/index.js",
-        "types": "./dist/packages/elements/src/index.d.ts"
-      },
-      "require": {
-        "default": "./dist/packages/elements/src/index.cjs",
-        "types": "./dist/packages/elements/src/index.d.ts"
+        "types": "./dist/index.d.ts",
+        "default": "./dist/elements/src/index.js"
       }
     },
     "./styles.css": "./dist/style.css"
@@ -24,7 +20,7 @@
   "author": "",
   "license": "UNLICENSED",
   "dependencies": {
-    "@editframe/assets": "0.7.0-beta.8",
+    "@editframe/assets": "0.8.0-beta.1",
     "@lit/context": "^1.1.2",
     "@lit/task": "^1.0.1",
     "d3": "^7.9.0",
@@ -35,9 +31,10 @@
   "devDependencies": {
     "@types/d3": "^7.4.3",
     "@types/dom-webcodecs": "^0.1.11",
-    "@types/node": "^20.14.9",
+    "@types/node": "^20.14.13",
     "autoprefixer": "^10.4.19",
     "rollup-plugin-tsconfig-paths": "^1.5.2",
+    "typescript": "^5.5.4",
     "vite-plugin-dts": "^3.9.1",
     "vite-tsconfig-paths": "^4.3.2"
   }

package/src/elements/EFAudio.ts

@@ -1,7 +1,7 @@
 import { html } from "lit";
 import { createRef, ref } from "lit/directives/ref.js";
 import { customElement, property } from "lit/decorators.js";
-import { EFMedia } from "./EFMedia";
+import { EFMedia } from "./EFMedia.ts";
 import { Task } from "@lit/task";
 
 @customElement("ef-audio")

package/src/elements/EFCaptions.ts

@@ -1,13 +1,13 @@
-import { EFAudio } from "./EFAudio";
 import { LitElement, type PropertyValueMap, html, css } from "lit";
 import { Task } from "@lit/task";
 import { customElement, property } from "lit/decorators.js";
-import { EFVideo } from "./EFVideo";
-import { EFTemporal } from "./EFTemporal";
-import { CrossUpdateController } from "./CrossUpdateController";
-import { FetchMixin } from "./FetchMixin";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+import { EFVideo } from "./EFVideo.ts";
+import { EFAudio } from "./EFAudio.ts";
+import { EFTemporal } from "./EFTemporal.ts";
+import { CrossUpdateController } from "./CrossUpdateController.ts";
+import { FetchMixin } from "./FetchMixin.ts";
+import { EFSourceMixin } from "./EFSourceMixin.ts";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
 
 interface Word {
   text: string;
@@ -93,8 +93,8 @@ export class EFCaptions extends EFSourceMixin(
 
   protected md5SumLoader = new Task(this, {
     autoRun: false,
-    args: () => [this.target] as const,
-    task: async ([], { signal }) => {
+    args: () => [this.target, this.fetch] as const,
+    task: async ([_target, fetch], { signal }) => {
       const md5Path = `/@ef-asset/${this.targetElement.src ?? ""}`;
       const response = await fetch(md5Path, { method: "HEAD", signal });
       return response.headers.get("etag") ?? undefined;

package/src/elements/EFImage.ts

@@ -2,9 +2,9 @@ import { Task } from "@lit/task";
 import { LitElement, html, css } from "lit";
 import { customElement } from "lit/decorators.js";
 import { createRef, ref } from "lit/directives/ref.js";
-import { FetchMixin } from "./FetchMixin";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+import { FetchMixin } from "./FetchMixin.ts";
+import { EFSourceMixin } from "./EFSourceMixin.ts";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
 
 @customElement("ef-image")
 export class EFImage extends EFSourceMixin(FetchMixin(LitElement), {

package/src/elements/EFMedia.ts

@@ -8,14 +8,14 @@ import debug from "debug";
 
 import type { TrackFragmentIndex, TrackSegment } from "@editframe/assets";
 
-import { MP4File } from "@/av/MP4File";
-import { EFTemporal } from "./EFTemporal";
-import { VideoAsset } from "@/av/EncodedAsset";
-import { FetchMixin } from "./FetchMixin";
-import { apiHostContext } from "../gui/EFWorkbench";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { getStartTimeMs } from "./util";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+import { MP4File } from "@editframe/assets/MP4File.js";
+import { VideoAsset } from "@editframe/assets/EncodedAsset.js";
+import { EFTemporal } from "./EFTemporal.ts";
+import { FetchMixin } from "./FetchMixin.ts";
+import { EFSourceMixin } from "./EFSourceMixin.ts";
+import { getStartTimeMs } from "./util.ts";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
+import { apiHostContext } from "../gui/apiHostContext.ts";
 
 const log = debug("ef:elements:EFMedia");
 
@@ -70,6 +70,9 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
   public trackFragmentIndexLoader = new Task(this, {
     args: () => [this.fragmentIndexPath(), this.fetch] as const,
     task: async ([fragmentIndexPath, fetch], { signal }) => {
+      if (this.src === "") {
+        return;
+      }
       const response = await fetch(fragmentIndexPath, { signal });
       return (await response.json()) as Record<number, TrackFragmentIndex>;
     },

package/src/elements/EFSourceMixin.ts

@@ -1,9 +1,9 @@
 import { consume } from "@lit/context";
 import type { LitElement } from "lit";
-import { apiHostContext } from "../gui/EFWorkbench";
 import { state } from "lit/decorators/state.js";
 import { Task } from "@lit/task";
 import { property } from "lit/decorators/property.js";
+import { apiHostContext } from "../gui/apiHostContext.ts";
 
 export declare class EFSourceMixinInterface {
   productionSrc(): string;

package/src/elements/EFTemporal.ts

@@ -1,11 +1,11 @@
 import type { LitElement, ReactiveController } from "lit";
 import { consume, createContext } from "@lit/context";
 import { property, state } from "lit/decorators.js";
-import type { EFTimegroup } from "./EFTimegroup";
+import type { EFTimegroup } from "./EFTimegroup.ts";
 
-import { durationConverter } from "./durationConverter";
+import { durationConverter } from "./durationConverter.ts";
 import { Task } from "@lit/task";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
 
 export const timegroupContext = createContext<EFTimegroup>(
   Symbol("timeGroupContext"),
@@ -59,10 +59,23 @@ export const deepGetElementsWithFrameTasks = (
   return elements;
 };
 
+let temporalCache: Map<Element, TemporalMixinInterface[]>;
+const resetTemporalCache = () => {
+  temporalCache = new Map();
+  if (typeof requestAnimationFrame !== "undefined") {
+    requestAnimationFrame(resetTemporalCache);
+  }
+};
+resetTemporalCache();
+
 export const shallowGetTemporalElements = (
   element: Element,
   temporals: TemporalMixinInterface[] = [],
 ) => {
+  const cachedResult = temporalCache.get(element);
+  if (cachedResult) {
+    return cachedResult;
+  }
   for (const child of element.children) {
     if (isEFTemporal(child)) {
       temporals.push(child);
@@ -70,6 +83,7 @@ export const shallowGetTemporalElements = (
       shallowGetTemporalElements(child, temporals);
     }
   }
+  temporalCache.set(element, temporals);
   return temporals;
 };
 
@@ -92,6 +106,15 @@ export class OwnCurrentTimeController implements ReactiveController {
 
 type Constructor<T = {}> = new (...args: any[]) => T;
 
+let startTimeMsCache = new WeakMap<Element, number>();
+const resetStartTimeMsCache = () => {
+  startTimeMsCache = new WeakMap();
+  if (typeof requestAnimationFrame !== "undefined") {
+    requestAnimationFrame(resetStartTimeMsCache);
+  }
+};
+resetStartTimeMsCache();
+
 export const EFTemporal = <T extends Constructor<LitElement>>(
   superClass: T,
 ) => {
@@ -172,27 +195,41 @@ export const EFTemporal = <T extends Constructor<LitElement>>(
     }
 
     get startTimeMs(): number {
+      const cachedStartTime = startTimeMsCache.get(this);
+      if (cachedStartTime !== undefined) {
+        return cachedStartTime;
+      }
       const parentTimegroup = this.parentTimegroup;
       if (!parentTimegroup) {
+        startTimeMsCache.set(this, 0);
         return 0;
       }
       switch (parentTimegroup.mode) {
         case "sequence": {
           const siblingTemorals = shallowGetTemporalElements(parentTimegroup);
-          const ownIndex = siblingTemorals.indexOf(
+          const ownIndex = siblingTemorals?.indexOf(
             this as InstanceType<Constructor<TemporalMixinInterface> & T>,
           );
           if (ownIndex === 0) {
+            startTimeMsCache.set(this, parentTimegroup.startTimeMs);
             return parentTimegroup.startTimeMs;
           }
-          const previous = siblingTemorals[ownIndex - 1];
+          const previous = siblingTemorals?.[(ownIndex ?? 0) - 1];
           if (!previous) {
             throw new Error("Previous temporal element not found");
           }
+          startTimeMsCache.set(
+            this,
+            previous.startTimeMs + previous.durationMs,
+          );
           return previous.startTimeMs + previous.durationMs;
         }
         case "contain":
         case "fixed":
+          startTimeMsCache.set(
+            this,
+            parentTimegroup.startTimeMs + this.offsetMs,
+          );
           return parentTimegroup.startTimeMs + this.offsetMs;
         default:
           throw new Error(`Invalid time mode: ${parentTimegroup.mode}`);

package/src/elements/EFTimegroup.browsertest.ts

@@ -5,10 +5,10 @@ import {
   html,
   render as litRender,
 } from "lit";
-import { EFTimegroup } from "./EFTimegroup";
-import "./EFTimegroup";
+import { EFTimegroup } from "./EFTimegroup.ts";
+import "./EFTimegroup.ts";
 import { customElement } from "lit/decorators/custom-element.js";
-import { EFTemporal } from "./EFTemporal";
+import { EFTemporal } from "./EFTemporal.ts";
 
 beforeEach(() => {
   for (let i = 0; i < localStorage.length; i++) {

package/src/elements/EFTimegroup.ts

@@ -9,10 +9,10 @@ import {
   isEFTemporal,
   shallowGetTemporalElements,
   timegroupContext,
-} from "./EFTemporal";
-import { TimegroupController } from "./TimegroupController";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
-import { deepGetMediaElements } from "./EFMedia";
+} from "./EFTemporal.ts";
+import { TimegroupController } from "./TimegroupController.ts";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
+import { deepGetMediaElements } from "./EFMedia.ts";
 
 const log = debug("ef:elements:EFTimegroup");
 
@@ -113,7 +113,9 @@ export class EFTimegroup extends EFTemporal(LitElement) {
   connectedCallback() {
     super.connectedCallback();
     if (this.id) {
-      this.#currentTime = this.maybeLoadTimeFromLocalStorage();
+      this.waitForMediaDurations().then(() => {
+        this.#currentTime = this.maybeLoadTimeFromLocalStorage();
+      });
     }
 
     if (this.parentTimegroup) {
@@ -265,7 +267,8 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     return (
       EF_INTERACTIVE &&
       this.closest("ef-timegroup") === this &&
-      this.closest("ef-workbench") === null
+      this.closest("ef-workbench") === null &&
+      this.closest("ef-preview") === null
     );
   }
 

package/src/elements/EFVideo.ts

@@ -3,8 +3,8 @@ import { Task } from "@lit/task";
 import { createRef, ref } from "lit/directives/ref.js";
 import { customElement } from "lit/decorators.js";
 
-import { EFMedia } from "./EFMedia";
-import { TWMixin } from "../gui/TWMixin";
+import { EFMedia } from "./EFMedia.ts";
+import { TWMixin } from "../gui/TWMixin.ts";
 
 @customElement("ef-video")
 export class EFVideo extends TWMixin(EFMedia) {

package/src/elements/EFWaveform.ts

@@ -1,15 +1,15 @@
-import { EFAudio } from "./EFAudio";
+import { EFAudio } from "./EFAudio.ts";
 
 import { LitElement, html } from "lit";
 import { customElement, property } from "lit/decorators.js";
-import { EFVideo } from "./EFVideo";
-import { EFTemporal } from "./EFTemporal";
-import { CrossUpdateController } from "./CrossUpdateController";
-import { TWMixin } from "../gui/TWMixin";
+import { EFVideo } from "./EFVideo.ts";
+import { EFTemporal } from "./EFTemporal.ts";
+import { CrossUpdateController } from "./CrossUpdateController.ts";
+import { TWMixin } from "../gui/TWMixin.ts";
 import { Task } from "@lit/task";
 import * as d3 from "d3";
 import { type Ref, createRef, ref } from "lit/directives/ref.js";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
 
 @customElement("ef-waveform")
 export class EFWaveform extends EFTemporal(TWMixin(LitElement)) {

package/src/elements/FetchMixin.ts

@@ -1,8 +1,9 @@
-import { consume } from "@lit/context";
 import type { LitElement } from "lit";
-import { fetchContext } from "../gui/EFWorkbench";
+import { consume } from "@lit/context";
 import { state } from "lit/decorators/state.js";
 
+import { fetchContext } from "../gui/fetchContext.ts";
+
 export declare class FetchMixinInterface {
   fetch: typeof fetch;
 }
@@ -12,7 +13,8 @@ export function FetchMixin<T extends Constructor<LitElement>>(superClass: T) {
   class FetchElement extends superClass {
     @consume({ context: fetchContext, subscribe: true })
     @state()
-    fetch = fetch.bind(window);
+    fetch: (url: string, init?: RequestInit) => Promise<Response> =
+      fetch.bind(window);
   }
 
   return FetchElement as Constructor<FetchMixinInterface> & T;

package/src/elements/TimegroupController.ts

@@ -1,5 +1,5 @@
 import type { ReactiveController, LitElement } from "lit";
-import type { EFTimegroup } from "./EFTimegroup";
+import type { EFTimegroup } from "./EFTimegroup.ts";
 
 export class TimegroupController implements ReactiveController {
   constructor(

package/src/elements/durationConverter.ts

@@ -1,4 +1,4 @@
-import { parseTimeToMs } from "./parseTimeToMs";
+import { parseTimeToMs } from "./parseTimeToMs.ts";
 
 export const durationConverter = {
   fromAttribute: (value: string): number => parseTimeToMs(value),

package/src/elements/util.ts

@@ -1,4 +1,4 @@
-import { EFTimegroup } from "./EFTimegroup";
+import { EFTimegroup } from "./EFTimegroup.ts";
 
 export const getRootTimeGroup = (element: Element): EFTimegroup | null => {
   let bestCandidate: EFTimegroup | null = null;

package/src/gui/ContextMixin.ts

@@ -0,0 +1,254 @@
+import type { LitElement } from "lit";
+import { provide } from "@lit/context";
+import { property, state } from "lit/decorators.js";
+
+import { focusContext, type FocusContext } from "./focusContext.ts";
+import { focusedElementContext } from "./focusedElementContext.ts";
+import { fetchContext } from "./fetchContext.ts";
+import { apiHostContext } from "./apiHostContext.ts";
+import { createRef } from "lit/directives/ref.js";
+import { playingContext } from "./playingContext.ts";
+import type { EFTimegroup } from "../elements/EFTimegroup.ts";
+
+declare class ContextMixinInterface {
+  focusContext: FocusContext;
+  focusedElement?: HTMLElement;
+  fetch: typeof fetch;
+  signingURL?: string;
+  apiToken?: string;
+  apiHost: string;
+  stageScale: number;
+  rendering: boolean;
+  stageRef: ReturnType<typeof createRef<HTMLDivElement>>;
+  canvasRef: ReturnType<typeof createRef<HTMLElement>>;
+  playing: boolean;
+  targetTimegroup?: EFTimegroup;
+  currentTimeMs: number;
+}
+
+type Constructor<T = {}> = new (...args: any[]) => T;
+export function ContextMixin<T extends Constructor<LitElement>>(superClass: T) {
+  class ContextElement extends superClass {
+    @provide({ context: focusContext })
+    focusContext = this as FocusContext;
+
+    @provide({ context: focusedElementContext })
+    @state()
+    focusedElement?: HTMLElement;
+
+    @provide({ context: fetchContext })
+    fetch = async (url: string, init: RequestInit = {}) => {
+      init.headers ||= {};
+      Object.assign(init.headers, {
+        "Content-Type": "application/json",
+      });
+
+      const bearerToken = this.apiToken;
+      if (bearerToken) {
+        Object.assign(init.headers, {
+          Authorization: `Bearer ${bearerToken}`,
+        });
+      }
+
+      if (this.signingURL) {
+        if (!this.#signedURLs[url]) {
+          this.#signedURLs[url] = fetch(this.signingURL, {
+            method: "POST",
+            body: JSON.stringify({ url }),
+          }).then(async (response) => {
+            if (response.ok) {
+              return (await response.json()).url;
+            }
+            throw new Error(
+              `Failed to sign URL: ${url}. SigningURL: ${this.signingURL} ${response.status} ${response.statusText}`,
+            );
+          });
+        }
+
+        const signedURL = await this.#signedURLs[url];
+
+        return fetch(signedURL, init);
+      }
+
+      return fetch(url, init);
+    };
+
+    #signedURLs: Record<string, Promise<string>> = {};
+
+    @property({ type: String })
+    signingURL?: string;
+
+    @property({ type: String })
+    apiToken?: string;
+
+    @provide({ context: apiHostContext })
+    @property({ type: String })
+    apiHost = "";
+
+    @provide({ context: playingContext })
+    @property({ type: Boolean, reflect: true })
+    playing = false;
+
+    @property({ type: Boolean, reflect: true })
+    loop = false;
+
+    @state()
+    stageScale = 1;
+
+    @property({ type: Boolean })
+    rendering = false;
+
+    @state()
+    currentTimeMs = 0;
+
+    stageRef = createRef<HTMLDivElement>();
+    canvasRef = createRef<HTMLSlotElement>();
+
+    setStageScale = () => {
+      if (this.isConnected && !this.rendering) {
+        const canvasElement = this.canvasRef.value;
+        const stageElement = this.stageRef.value;
+        if (stageElement && canvasElement) {
+          // Determine the appropriate scale factor to make the canvas fit into
+          // it's parent element.
+          const stageWidth = stageElement.clientWidth;
+          const stageHeight = stageElement.clientHeight;
+          const canvasWidth = canvasElement.clientWidth;
+          const canvasHeight = canvasElement.clientHeight;
+          const stageRatio = stageWidth / stageHeight;
+          const canvasRatio = canvasWidth / canvasHeight;
+          if (stageRatio > canvasRatio) {
+            const scale = stageHeight / canvasHeight;
+            if (this.stageScale !== scale) {
+              canvasElement.style.transform = `scale(${scale})`;
+            }
+            this.stageScale = scale;
+          } else {
+            const scale = stageWidth / canvasWidth;
+            if (this.stageScale !== scale) {
+              canvasElement.style.transform = `scale(${scale})`;
+            }
+            this.stageScale = scale;
+          }
+        }
+      }
+      if (this.isConnected) {
+        requestAnimationFrame(this.setStageScale);
+      }
+    };
+
+    connectedCallback(): void {
+      super.connectedCallback();
+      // Preferrably we would use a resizeObserver, but it is difficult to get the first resize
+      // timed correctly. So we use requestAnimationFrame as a stop-gap.
+      requestAnimationFrame(this.setStageScale);
+    }
+
+    update(changedProperties: Map<string | number | symbol, unknown>) {
+      if (changedProperties.has("playing")) {
+        if (this.playing) {
+          this.#startPlayback();
+        } else {
+          this.#stopPlayback();
+        }
+      }
+
+      if (changedProperties.has("currentTimeMs") && this.targetTimegroup) {
+        if (this.targetTimegroup.currentTimeMs !== this.currentTimeMs) {
+          this.targetTimegroup.currentTimeMs = this.currentTimeMs;
+        }
+      }
+      super.update(changedProperties);
+    }
+
+    get targetTimegroup() {
+      return this.querySelector("ef-timegroup");
+    }
+
+    #playbackAudioContext: AudioContext | null = null;
+    #playbackAnimationFrameRequest: number | null = null;
+    #AUDIO_PLAYBACK_SLICE_MS = 1000;
+
+    #syncPlayheadToAudioContext(target: EFTimegroup, startMs: number) {
+      this.currentTimeMs =
+        startMs + (this.#playbackAudioContext?.currentTime ?? 0) * 1000;
+      this.#playbackAnimationFrameRequest = requestAnimationFrame(() => {
+        this.#syncPlayheadToAudioContext(target, startMs);
+      });
+    }
+
+    async #stopPlayback() {
+      if (this.#playbackAudioContext) {
+        if (this.#playbackAudioContext.state !== "closed") {
+          await this.#playbackAudioContext.close();
+        }
+      }
+      if (this.#playbackAnimationFrameRequest) {
+        cancelAnimationFrame(this.#playbackAnimationFrameRequest);
+      }
+      this.#playbackAudioContext = null;
+    }
+
+    async #startPlayback() {
+      await this.#stopPlayback();
+      const timegroup = this.targetTimegroup;
+      if (!timegroup) {
+        return;
+      }
+
+      let currentMs = timegroup.currentTimeMs;
+      let bufferCount = 0;
+      this.#playbackAudioContext = new AudioContext({
+        latencyHint: "playback",
+      });
+      if (this.#playbackAnimationFrameRequest) {
+        cancelAnimationFrame(this.#playbackAnimationFrameRequest);
+      }
+      this.#syncPlayheadToAudioContext(timegroup, currentMs);
+      const playbackContext = this.#playbackAudioContext;
+      await playbackContext.suspend();
+
+      const fillBuffer = async () => {
+        if (bufferCount > 1) {
+          return;
+        }
+        const canFillBuffer = await queueBufferSource();
+        if (canFillBuffer) {
+          fillBuffer();
+        }
+      };
+
+      const fromMs = currentMs;
+      const toMs = timegroup.endTimeMs;
+
+      const queueBufferSource = async () => {
+        if (currentMs >= toMs) {
+          return false;
+        }
+        const startMs = currentMs;
+        const endMs = currentMs + this.#AUDIO_PLAYBACK_SLICE_MS;
+        currentMs += this.#AUDIO_PLAYBACK_SLICE_MS;
+        const audioBuffer = await timegroup.renderAudio(startMs, endMs);
+        bufferCount++;
+        const source = playbackContext.createBufferSource();
+        source.buffer = audioBuffer;
+        source.connect(playbackContext.destination);
+        source.start((startMs - fromMs) / 1000);
+        source.onended = () => {
+          bufferCount--;
+          if (endMs >= toMs) {
+            this.playing = false;
+          } else {
+            fillBuffer();
+          }
+        };
+        return true;
+      };
+
+      await fillBuffer();
+      await playbackContext.resume();
+    }
+  }
+
+  return ContextElement as Constructor<ContextMixinInterface> & T;
+}