@editframe/elements 0.7.0-beta.9 → 0.8.0-beta.1

package/{src/EF_FRAMEGEN.ts → dist/elements/src/EF_FRAMEGEN.js}

@@ -1,49 +1,9 @@
 import debug from "debug";
 import { TaskStatus } from "@lit/task";
-
-import type { VideoRenderOptions } from "@editframe/assets";
-import { awaitMicrotask } from "@/util/awaitMicrotask";
-
-import { deepGetElementsWithFrameTasks } from "./elements/EFTemporal";
-import { shallowGetTimegroups } from "./elements/EFTimegroup";
-
+import { deepGetElementsWithFrameTasks } from "./elements/EFTemporal.js";
+import { shallowGetTimegroups } from "./elements/EFTimegroup.js";
 const log = debug("ef:elements:EF_FRAMEGEN");
-
-declare global {
-  interface Window {
-    EF_FRAMEGEN?: EfFramegen;
-    FRAMEGEN_BRIDGE?: {
-      onInitialize: (
-        callback: (renderId: string, renderOptions: VideoRenderOptions) => void,
-      ) => void;
-
-      initialized(renderId: string): void;
-
-      onBeginFrame(
-        callback: (
-          renderId: string,
-          frameNumber: number,
-          isLast: boolean,
-        ) => void,
-      ): void;
-
-      onTriggerCanvas(callback: () => void): void;
-
-      frameReady(
-        renderId: string,
-        frameNumber: number,
-        audioSamples: ArrayBuffer,
-      ): void;
-
-      error(renderId: string, error: Error): void;
-    };
-  }
-}
-
 class TriggerCanvas {
-  private canvas: HTMLCanvasElement;
-  private ctx: CanvasRenderingContext2D;
-
   constructor() {
     this.canvas = document.createElement("canvas");
     this.canvas.width = 1;
@@ -54,7 +14,7 @@ class TriggerCanvas {
       left: "0px",
       width: "1px",
       height: "1px",
-      zIndex: "100000",
+      zIndex: "100000"
     });
     document.body.prepend(this.canvas);
     const ctx = this.canvas.getContext("2d", { willReadFrequently: true });
@@ -62,70 +22,54 @@ class TriggerCanvas {
     this.ctx = ctx;
     this.ctx.fillStyle = "black";
   }
-
   trigger() {
     log("TRIGGERING CANVAS");
     this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
   }
 }
-
-export class EfFramegen {
-  time = 0;
-  frameDurationMs = 0;
-  initialBusyTasks: Promise<unknown[]> = Promise.resolve([]);
-  audioBufferPromise?: Promise<AudioBuffer>;
-  renderOptions?: VideoRenderOptions;
-  frameBox = document.createElement("div");
-  BRIDGE = window.FRAMEGEN_BRIDGE;
-  triggerCanvas = new TriggerCanvas();
-
-  trace(...args: any[]) {
-    console.trace("[EF_FRAMEGEN]", ...args);
-  }
-
+class EfFramegen {
   constructor() {
+    this.time = 0;
+    this.frameDurationMs = 0;
+    this.initialBusyTasks = Promise.resolve([]);
+    this.frameBox = document.createElement("div");
+    this.BRIDGE = window.FRAMEGEN_BRIDGE;
+    this.triggerCanvas = new TriggerCanvas();
     if (this.BRIDGE) {
       this.connectToBridge();
     }
   }
-
+  trace(...args) {
+    console.trace("[EF_FRAMEGEN]", ...args);
+  }
   connectToBridge() {
     const BRIDGE = this.BRIDGE;
     if (!BRIDGE) {
       throw new Error("No BRIDGE when attempting to connect to bridge");
     }
-
     BRIDGE.onInitialize(async (renderId, renderOptions) => {
       log("BRIDGE.onInitialize", renderId, renderOptions);
       await this.initialize(renderId, renderOptions);
       BRIDGE.initialized(renderId);
     });
-
     BRIDGE.onBeginFrame((renderId, frameNumber, isLast) => {
       log("BRIDGE.onBeginFrame", renderId, frameNumber, isLast);
       this.beginFrame(renderId, frameNumber, isLast);
     });
-
-    // BRIDGE.onTriggerCanvas(() => {
-    //   // this.triggerCanvas.trigger();
-    // });
   }
-
-  async initialize(renderId: string, renderOptions: VideoRenderOptions) {
+  async initialize(renderId, renderOptions) {
     addEventListener("unhandledrejection", (event) => {
       this.trace("Unhandled rejection:", event.reason);
       if (this.BRIDGE) {
         this.BRIDGE.error(renderId, event.reason);
       }
     });
-
     addEventListener("error", (event) => {
       this.trace("Uncaught error", event.error);
       if (this.BRIDGE) {
         this.BRIDGE.error(renderId, event.error);
       }
     });
-
     this.renderOptions = renderOptions;
     const workbench = document.querySelector("ef-workbench");
     if (!workbench) {
@@ -135,21 +79,14 @@ export class EfFramegen {
     const timegroups = shallowGetTimegroups(workbench);
     const temporals = deepGetElementsWithFrameTasks(workbench);
     const firstGroup = timegroups[0];
-
     if (!firstGroup) {
       throw new Error("No temporal elements found");
     }
     firstGroup.currentTimeMs = renderOptions.encoderOptions.fromMs;
-
-    this.frameDurationMs = 1000 / renderOptions.encoderOptions.video.framerate;
-
+    this.frameDurationMs = 1e3 / renderOptions.encoderOptions.video.framerate;
     this.initialBusyTasks = Promise.all(
-      temporals
-        .filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE)
-        .map((temporal) => temporal.frameTask)
-        .map((task) => task.taskComplete),
+      temporals.filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE).map((temporal) => temporal.frameTask).map((task) => task.taskComplete)
     );
-
     this.time = 0;
     if (renderOptions.showFrameBox) {
       Object.assign(this.frameBox.style, {
@@ -160,21 +97,20 @@ export class EfFramegen {
         position: "absolute",
         top: "0px",
         left: "0px",
-        zIndex: "100000",
+        zIndex: "100000"
       });
       document.body.prepend(this.frameBox);
     }
-
     this.audioBufferPromise = firstGroup.renderAudio(
-      renderOptions.encoderOptions.alignedFromUs / 1000,
-      renderOptions.encoderOptions.alignedToUs / 1000,
+      renderOptions.encoderOptions.alignedFromUs / 1e3,
+      renderOptions.encoderOptions.alignedToUs / 1e3
       // renderOptions.encoderOptions.fromMs,
       // renderOptions.encoderOptions.toMs,
     );
     log("Initialized");
   }
-  async beginFrame(renderId: string, frameNumber: number, isLast: boolean) {
-    if (this.renderOptions === undefined) {
+  async beginFrame(renderId, frameNumber, isLast) {
+    if (this.renderOptions === void 0) {
       throw new Error("No renderOptions");
     }
     if (this.renderOptions.showFrameBox) {
@@ -194,60 +130,40 @@ export class EfFramegen {
     if (!firstGroup) {
       throw new Error("No temporal elements found");
     }
-
-    this.time =
-      this.renderOptions.encoderOptions.fromMs +
-      frameNumber * this.frameDurationMs;
-
+    this.time = this.renderOptions.encoderOptions.fromMs + frameNumber * this.frameDurationMs;
     firstGroup.currentTimeMs = this.time;
-
     log("Awaiting initialBusyTasks");
     await this.initialBusyTasks;
-
     log("Awaiting microtask");
-    await awaitMicrotask();
-
+    await new Promise(queueMicrotask);
     log("Awaiting frame tasks");
     const now = performance.now();
     await Promise.all(
-      temporals
-        .filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE)
-        .map((temporal) => {
-          return temporal.frameTask;
-        })
-        .map((task) => task.taskComplete),
+      temporals.filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE).map((temporal) => {
+        return temporal.frameTask;
+      }).map((task) => task.taskComplete)
     );
-
     log(`frame:${frameNumber} All tasks complete ${performance.now() - now}ms`);
-
     if (isLast && this.audioBufferPromise) {
-      // Currently we emit the audio in one belch at the end of the render.
-      // This is not ideal, but it's the simplest thing that could possibly work.
-      // We could either emit it slices, or in parallel with the video.
-      // But in any case, it's fine for now.
       const renderedAudio = await this.audioBufferPromise;
-
       const channelCount = renderedAudio.numberOfChannels;
-
       const interleavedSamples = new Float32Array(
-        channelCount * renderedAudio.length,
+        channelCount * renderedAudio.length
       );
-
       for (let i = 0; i < renderedAudio.length; i++) {
         for (let j = 0; j < channelCount; j++) {
           interleavedSamples.set(
             renderedAudio.getChannelData(j).slice(i, i + 1),
-            i * channelCount + j,
+            i * channelCount + j
           );
         }
       }
-
       if (this.BRIDGE) {
         this.triggerCanvas.trigger();
         this.BRIDGE.frameReady(
           renderId,
           frameNumber,
-          interleavedSamples.buffer,
+          interleavedSamples.buffer
         );
       } else {
         const fileReader = new FileReader();
@@ -274,5 +190,9 @@ export class EfFramegen {
     }
   }
 }
-
-window.EF_FRAMEGEN = new EfFramegen();
+if (typeof window !== "undefined") {
+  window.EF_FRAMEGEN = new EfFramegen();
+}
+export {
+  EfFramegen
+};

package/dist/elements/src/EF_INTERACTIVE.js

@@ -0,0 +1,7 @@
+let EF_INTERACTIVE = false;
+if (typeof window !== "undefined") {
+  EF_INTERACTIVE = !window.location?.search.includes("EF_NONINTERACTIVE");
+}
+export {
+  EF_INTERACTIVE
+};

package/dist/elements/src/elements/CrossUpdateController.js

@@ -0,0 +1,16 @@
+class CrossUpdateController {
+  constructor(host, target) {
+    this.host = host;
+    this.target = target;
+    this.host.addController(this);
+  }
+  hostUpdate() {
+    this.target.requestUpdate();
+  }
+  remove() {
+    this.host.removeController(this);
+  }
+}
+export {
+  CrossUpdateController
+};

package/dist/elements/src/elements/EFAudio.js

@@ -0,0 +1,54 @@
+import { html } from "lit";
+import { createRef, ref } from "lit/directives/ref.js";
+import { property, customElement } from "lit/decorators.js";
+import { EFMedia } from "./EFMedia.js";
+import { Task } from "@lit/task";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp(target, key, result);
+  return result;
+};
+let EFAudio = class extends EFMedia {
+  constructor() {
+    super(...arguments);
+    this.audioElementRef = createRef();
+    this.src = "";
+    this.frameTask = new Task(this, {
+      args: () => [
+        this.trackFragmentIndexLoader.status,
+        this.initSegmentsLoader.status,
+        this.seekTask.status,
+        this.fetchSeekTask.status,
+        this.videoAssetTask.status
+      ],
+      task: async () => {
+        await this.trackFragmentIndexLoader.taskComplete;
+        await this.initSegmentsLoader.taskComplete;
+        await this.seekTask.taskComplete;
+        await this.fetchSeekTask.taskComplete;
+        await this.videoAssetTask.taskComplete;
+        this.rootTimegroup?.requestUpdate();
+      }
+    });
+  }
+  render() {
+    return html`<audio ${ref(this.audioElementRef)}></audio>`;
+  }
+  get audioElement() {
+    return this.audioElementRef.value;
+  }
+};
+__decorateClass([
+  property({ type: String })
+], EFAudio.prototype, "src", 2);
+EFAudio = __decorateClass([
+  customElement("ef-audio")
+], EFAudio);
+export {
+  EFAudio
+};

package/dist/elements/src/elements/EFCaptions.js

@@ -0,0 +1,166 @@
+import { html, css, LitElement } from "lit";
+import { Task } from "@lit/task";
+import { property, customElement } from "lit/decorators.js";
+import { EFVideo } from "./EFVideo.js";
+import { EFAudio } from "./EFAudio.js";
+import { EFTemporal } from "./EFTemporal.js";
+import { CrossUpdateController } from "./CrossUpdateController.js";
+import { FetchMixin } from "./FetchMixin.js";
+import { EFSourceMixin } from "./EFSourceMixin.js";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp(target, key, result);
+  return result;
+};
+let EFCaptionsActiveWord = class extends EFTemporal(LitElement) {
+  constructor() {
+    super(...arguments);
+    this.wordStartMs = 0;
+    this.wordEndMs = 0;
+    this.wordText = "";
+  }
+  render() {
+    return html`${this.wordText}`;
+  }
+  get startTimeMs() {
+    return this.wordStartMs || 0;
+  }
+  get durationMs() {
+    return this.wordEndMs - this.wordStartMs;
+  }
+};
+EFCaptionsActiveWord.styles = [
+  css`
+      :host {
+        display: inline-block;
+      }
+    `
+];
+__decorateClass([
+  property({ type: Number, attribute: false })
+], EFCaptionsActiveWord.prototype, "wordStartMs", 2);
+__decorateClass([
+  property({ type: Number, attribute: false })
+], EFCaptionsActiveWord.prototype, "wordEndMs", 2);
+__decorateClass([
+  property({ type: String, attribute: false })
+], EFCaptionsActiveWord.prototype, "wordText", 2);
+EFCaptionsActiveWord = __decorateClass([
+  customElement("ef-captions-active-word")
+], EFCaptionsActiveWord);
+let EFCaptions = class extends EFSourceMixin(
+  EFTemporal(FetchMixin(LitElement)),
+  { assetType: "caption_files" }
+) {
+  constructor() {
+    super(...arguments);
+    this.target = null;
+    this.wordStyle = "";
+    this.activeWordContainers = this.getElementsByTagName("ef-captions-active-word");
+    this.md5SumLoader = new Task(this, {
+      autoRun: false,
+      args: () => [this.target, this.fetch],
+      task: async ([_target, fetch], { signal }) => {
+        const md5Path = `/@ef-asset/${this.targetElement.src ?? ""}`;
+        const response = await fetch(md5Path, { method: "HEAD", signal });
+        return response.headers.get("etag") ?? void 0;
+      }
+    });
+    this.captionsDataTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.captionsPath(), this.fetch],
+      task: async ([captionsPath, fetch], { signal }) => {
+        const response = await fetch(captionsPath, { signal });
+        return response.json();
+      }
+    });
+    this.frameTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.captionsDataTask.status],
+      task: async () => {
+        await this.captionsDataTask.taskComplete;
+      }
+    });
+  }
+  captionsPath() {
+    const targetSrc = this.targetElement.src;
+    if (targetSrc.startsWith("editframe://") || targetSrc.startsWith("http")) {
+      return targetSrc.replace("isobmff", "caption");
+    }
+    return `/@ef-captions/${targetSrc}`;
+  }
+  connectedCallback() {
+    super.connectedCallback();
+    if (this.targetElement) {
+      new CrossUpdateController(this.targetElement, this);
+    }
+  }
+  render() {
+    return this.captionsDataTask.render({
+      pending: () => html`<div>Generating captions data...</div>`,
+      error: () => html`<div>🚫 Error generating captions data</div>`,
+      complete: () => html`<slot></slot>`
+    });
+  }
+  updated(_changedProperties) {
+    this.updateActiveWord();
+  }
+  updateActiveWord() {
+    const caption = this.captionsDataTask.value;
+    if (!caption) {
+      return;
+    }
+    const words = [];
+    let startMs = 0;
+    let endMs = 0;
+    for (const segment of caption.segments) {
+      if (this.targetElement.ownCurrentTimeMs >= segment.start * 1e3 && this.targetElement.ownCurrentTimeMs <= segment.end * 1e3) {
+        for (const word of segment.words) {
+          if (this.targetElement.ownCurrentTimeMs >= word.start * 1e3 && this.targetElement.ownCurrentTimeMs <= word.end * 1e3) {
+            words.push(word.text);
+            startMs = word.start * 1e3;
+            endMs = word.end * 1e3;
+          }
+        }
+      }
+    }
+    for (const container of Array.from(this.activeWordContainers)) {
+      container.wordText = words.join(" ");
+      container.wordStartMs = startMs;
+      container.wordEndMs = endMs;
+    }
+  }
+  get targetElement() {
+    const target = document.getElementById(this.getAttribute("target") ?? "");
+    if (target instanceof EFAudio || target instanceof EFVideo) {
+      return target;
+    }
+    throw new Error("Invalid target, must be an EFAudio  or EFVideo element");
+  }
+};
+EFCaptions.styles = [
+  css`
+      :host {
+        display: block;
+      }
+    `
+];
+__decorateClass([
+  property({ type: String, attribute: "target" })
+], EFCaptions.prototype, "target", 2);
+__decorateClass([
+  property({ attribute: "word-style" })
+], EFCaptions.prototype, "wordStyle", 2);
+EFCaptions = __decorateClass([
+  customElement("ef-captions")
+], EFCaptions);
+export {
+  EFCaptions,
+  EFCaptionsActiveWord
+};

package/dist/elements/src/elements/EFImage.js

@@ -0,0 +1,80 @@
+import { Task } from "@lit/task";
+import { html, css, LitElement } from "lit";
+import { customElement } from "lit/decorators.js";
+import { createRef, ref } from "lit/directives/ref.js";
+import { FetchMixin } from "./FetchMixin.js";
+import { EFSourceMixin } from "./EFSourceMixin.js";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp(target, key, result);
+  return result;
+};
+let EFImage = class extends EFSourceMixin(FetchMixin(LitElement), {
+  assetType: "image_files"
+}) {
+  constructor() {
+    super(...arguments);
+    this.imageRef = createRef();
+    this.canvasRef = createRef();
+    this.fetchImage = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.assetPath(), this.fetch],
+      task: async ([assetPath, fetch], { signal }) => {
+        const response = await fetch(assetPath, { signal });
+        const image = new Image();
+        image.src = URL.createObjectURL(await response.blob());
+        await new Promise((resolve) => {
+          image.onload = resolve;
+        });
+        if (!this.canvasRef.value) throw new Error("Canvas not ready");
+        const ctx = this.canvasRef.value.getContext("2d");
+        if (!ctx) throw new Error("Canvas 2d context not ready");
+        this.canvasRef.value.width = image.width;
+        this.canvasRef.value.height = image.height;
+        ctx.drawImage(image, 0, 0);
+      }
+    });
+    this.frameTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.fetchImage.status],
+      task: async () => {
+        await this.fetchImage.taskComplete;
+      }
+    });
+  }
+  render() {
+    return html`<canvas ${ref(this.canvasRef)}></canvas>`;
+  }
+  assetPath() {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return this.src;
+    }
+    return `/@ef-image/${this.src}`;
+  }
+};
+EFImage.styles = [
+  css`
+      :host {
+        display: block;
+      }
+      canvas {
+        display: block;
+        width: 100%;
+        height: 100%;
+        object-fit: fill;
+        object-position: center;
+      }
+    `
+];
+EFImage = __decorateClass([
+  customElement("ef-image")
+], EFImage);
+export {
+  EFImage
+};