@editframe/elements 0.5.0-beta.6 → 0.5.0-beta.8

package/src/EF_FRAMEGEN.ts

@@ -0,0 +1,208 @@
+import type { VideoRenderOptions } from "services/render/app/RenderOptions";
+import {
+  deepGetElementsWithFrameTasks,
+  shallowGetTimegroups,
+} from "./elements/EFTemporal";
+import { awaitAnimationFrame } from "@/util/awaitAnimationFrame";
+import { awaitMicrotask } from "@/util/awaitMicrotask";
+
+declare global {
+  interface Window {
+    EF_FRAMEGEN?: {
+      onRender: (
+        callback: (
+          renderId: string,
+          traceCarrier: unknown,
+          renderOptions: VideoRenderOptions,
+        ) => void,
+      ) => void;
+      onBegin: (
+        callback: (
+          traceCarrier: unknown,
+          frame: number,
+          isLast: boolean,
+        ) => void,
+      ) => void;
+      frameReady: (
+        renderId: string,
+        frameNumber: number,
+        samples: ArrayBufferLike,
+      ) => void;
+      onPaint: (callback: () => void) => void;
+      didPaint: (renderId: string) => void;
+      end: (renderId: string) => void;
+      error: (renderId: string, error: string) => void;
+    };
+  }
+}
+
+class TriggerCanvas {
+  private canvas: HTMLCanvasElement;
+  private ctx: CanvasRenderingContext2D;
+
+  constructor() {
+    this.canvas = document.createElement("canvas");
+    this.canvas.width = 1;
+    this.canvas.height = 1;
+    Object.assign(this.canvas.style, {
+      position: "absolute",
+      top: "0px",
+      left: "0px",
+      width: `1px`,
+      height: `1px`,
+      zIndex: "100000",
+    });
+    document.body.prepend(this.canvas);
+    this.ctx = this.canvas.getContext("2d")!;
+  }
+
+  trigger() {
+    console.log("Triggering");
+    this.ctx.fillStyle = "rgba(50, 0, 0, .8)";
+    this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
+    this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
+  }
+}
+
+if (window.EF_FRAMEGEN !== undefined) {
+  const EF_FRAMEGEN = window.EF_FRAMEGEN;
+  EF_FRAMEGEN.onRender((renderId, traceCarrier, renderOptions) => {
+    const crashOnUnhandledError = (error: string) => {
+      EF_FRAMEGEN.error(renderId, error);
+    };
+    window.addEventListener("error", (error) => {
+      console.warn("Crashing due to unhandled error", error);
+      crashOnUnhandledError(error.message);
+    });
+    window.addEventListener("unhandledrejection", (error) => {
+      console.warn("Crashing due to unhandled rejection", error);
+      crashOnUnhandledError(error.reason);
+    });
+
+    const triggerCanvas = new TriggerCanvas();
+    EF_FRAMEGEN.onPaint(async () => {
+      // Calls clearRect on the canvas to force the compositor to emit a frame.
+      triggerCanvas.trigger();
+
+      // We must wait for 2 frames to ensure offscreen render capture doesn't capture
+      // a stale frame. This is terrible. Fixing this requires contribiting to electron.js
+      // On the positive side, were' running at 240fps so it only burns 8ms max.
+      // (electron can't go faster thon 240fps)
+      // In practice it's less because we're already some ms into the next frame budget.
+      await awaitAnimationFrame();
+      await awaitAnimationFrame();
+
+      EF_FRAMEGEN.didPaint(renderId);
+    });
+    const workbench = document.querySelector("ef-workbench")!;
+    workbench.rendering = true;
+    const timegroups = shallowGetTimegroups(workbench);
+    const temporals = deepGetElementsWithFrameTasks(workbench);
+    const firstGroup = timegroups[0];
+    if (!firstGroup) {
+      throw new Error("No temporal elements found");
+    }
+    firstGroup.currentTimeMs = renderOptions.encoderOptions.fromMs;
+
+    const frameDurationMs = 1000 / renderOptions.encoderOptions.video.framerate;
+
+    const initialBusyTasks = Promise.all(
+      temporals
+        // .filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE)
+        .map((temporal) => temporal.frameTask)
+        .map((task) => task.taskComplete),
+    );
+
+    const frameBox = document.createElement("div");
+    Object.assign(frameBox.style, {
+      width: "200px",
+      height: "100px",
+      font: "30px Arial",
+      backgroundColor: "white",
+      position: "absolute",
+      top: "0px",
+      left: "0px",
+      zIndex: "100000",
+    });
+
+    let time = 0;
+    // document.body.prepend(frameBox);
+    // TODO: terminate if no workbench found
+    console.log("onRender", renderId, traceCarrier, renderOptions);
+
+    const audioBufferPromise = firstGroup.renderAudio(
+      renderOptions.encoderOptions.alignedFromUs / 1000,
+      renderOptions.encoderOptions.alignedToUs / 1000,
+      // renderOptions.encoderOptions.fromMs,
+      // renderOptions.encoderOptions.toMs,
+    );
+    EF_FRAMEGEN.onBegin(async (traceCarrier, frame, isLast) => {
+      time = firstGroup.currentTimeMs =
+        renderOptions.encoderOptions.fromMs + frame * frameDurationMs;
+      console.log("FRAME #", frame);
+      frameBox.innerHTML = `
+      <div>Frame #${frame}</div>
+      <div>${time.toFixed(4)}</div>
+      `;
+
+      await initialBusyTasks;
+
+      console.log("TIME", time.toFixed(4));
+      await awaitMicrotask();
+      console.log("After microtask");
+
+      const now = performance.now();
+      console.log(`frame:${frame} Awaiting busyTasks`);
+      await Promise.all(
+        temporals
+          // .filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE)
+          .map((temporal) => {
+            console.log(
+              "Awaiting",
+              temporal.tagName,
+              temporal.frameTask.status,
+              temporal.frameTask.taskComplete,
+            );
+            return temporal.frameTask;
+          })
+          .map((task) => task.taskComplete),
+      );
+      console.log(
+        `frame:${frame} All tasks complete ${performance.now() - now}ms`,
+      );
+
+      await awaitAnimationFrame();
+      await awaitAnimationFrame();
+
+      // Trigger a small canvas paint to force the compositor to emit a frame.
+      triggerCanvas.trigger();
+
+      if (isLast) {
+        // Currently we emit the audio in one belch at the end of the render.
+        // This is not ideal, but it's the simplest thing that could possibly work.
+        // We could either emit it slices, or in parallel with the video.
+        // But in any case, it's fine for now.
+        const renderedAudio = await audioBufferPromise;
+
+        const channelCount = renderedAudio.numberOfChannels;
+
+        const interleavedSamples = new Float32Array(
+          channelCount * renderedAudio.length,
+        );
+
+        for (let i = 0; i < renderedAudio.length; i++) {
+          for (let j = 0; j < channelCount; j++) {
+            interleavedSamples.set(
+              renderedAudio.getChannelData(j).slice(i, i + 1),
+              i * channelCount + j,
+            );
+          }
+        }
+
+        EF_FRAMEGEN.frameReady(renderId, frame, interleavedSamples.buffer);
+      } else {
+        EF_FRAMEGEN.frameReady(renderId, frame, new Float32Array(0).buffer);
+      }
+    });
+  });
+}

package/src/EF_INTERACTIVE.ts

@@ -0,0 +1,2 @@
+export const EF_INTERACTIVE =
+  !window.location.search.includes("EF_NONINTERACTIVE");

package/src/elements/CrossUpdateController.ts

@@ -0,0 +1,18 @@
+import { LitElement, ReactiveController, ReactiveControllerHost } from "lit";
+
+export class CrossUpdateController implements ReactiveController {
+  constructor(
+    private host: ReactiveControllerHost,
+    private target: LitElement,
+  ) {
+    this.host.addController(this);
+  }
+
+  hostUpdate(): void {
+    this.target.requestUpdate();
+  }
+
+  remove(): void {
+    this.host.removeController(this);
+  }
+}

package/src/elements/EFAudio.ts

@@ -0,0 +1,42 @@
+import { html } from "lit";
+import { createRef, ref } from "lit/directives/ref.js";
+import { customElement, property } from "lit/decorators.js";
+import { EFMedia } from "./EFMedia";
+import { Task } from "@lit/task";
+
+@customElement("ef-audio")
+export class EFAudio extends EFMedia {
+  audioElementRef = createRef<HTMLAudioElement>();
+
+  @property({ type: String })
+  src = "";
+
+  render() {
+    return html`<audio ${ref(this.audioElementRef)}></audio>`;
+  }
+
+  get audioElement() {
+    return this.audioElementRef.value;
+  }
+
+  frameTask = new Task(this, {
+    args: () =>
+      [
+        this.trackFragmentIndexLoader.status,
+        this.initSegmentsLoader.status,
+        this.seekTask.status,
+        this.fetchSeekTask.status,
+        this.videoAssetTask.status,
+      ] as const,
+    task: async () => {
+      console.log("EFAudio frameTask", this.ownCurrentTimeMs);
+      await this.trackFragmentIndexLoader.taskComplete;
+      await this.initSegmentsLoader.taskComplete;
+      await this.seekTask.taskComplete;
+      await this.fetchSeekTask.taskComplete;
+      await this.videoAssetTask.taskComplete;
+      console.log("EFAudio frameTask complete", this.ownCurrentTimeMs);
+      this.rootTimegroup?.requestUpdate();
+    },
+  });
+}

package/src/elements/EFCaptions.ts

@@ -0,0 +1,202 @@
+import { EFAudio } from "./EFAudio";
+import { LitElement, PropertyValueMap, html, css } from "lit";
+import { Task } from "@lit/task";
+import { customElement, property } from "lit/decorators.js";
+import { EFVideo } from "./EFVideo";
+import { EFTemporal } from "./EFTemporal";
+import { CrossUpdateController } from "./CrossUpdateController";
+import { FetchMixin } from "./FetchMixin";
+import { EFSourceMixin } from "./EFSourceMixin";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+
+interface Word {
+  text: string;
+  start: number;
+  end: number;
+  confidence: number;
+}
+
+interface Segment {
+  start: number;
+  end: number;
+  text: string;
+  confidence: number;
+  words: Word[];
+}
+
+interface Caption {
+  text: string;
+  segments: Segment[];
+  language: string;
+}
+
+@customElement("ef-captions-active-word")
+export class EFCaptionsActiveWord extends EFTemporal(LitElement) {
+  static styles = [
+    css`
+      :host {
+        display: inline-block;
+      }
+    `,
+  ];
+
+  render() {
+    return html`${this.wordText}`;
+  }
+
+  @property({ type: Number, attribute: false })
+  wordStartMs = 0;
+
+  @property({ type: Number, attribute: false })
+  wordEndMs = 0;
+
+  @property({ type: String, attribute: false })
+  wordText = "";
+
+  get startTimeMs() {
+    return this.wordStartMs || 0;
+  }
+
+  get durationMs(): number {
+    return this.wordEndMs - this.wordStartMs;
+  }
+}
+
+@customElement("ef-captions")
+export class EFCaptions extends EFSourceMixin(
+  EFTemporal(FetchMixin(LitElement)),
+  { assetType: "caption_files" },
+) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+    `,
+  ];
+
+  @property({ type: String, attribute: "target" })
+  target = null;
+
+  @property({ attribute: "word-style" })
+  wordStyle = "";
+
+  activeWordContainers = this.getElementsByTagName("ef-captions-active-word");
+
+  captionsPath() {
+    const src = this.targetElement.getAttribute("src");
+    if (src?.startsWith("http")) {
+      return src.replace("isobmff", "caption");
+    }
+    return `/@ef-captions/${this.targetElement.getAttribute("src") ?? ""}`;
+  }
+
+  protected md5SumLoader = new Task(this, {
+    autoRun: false,
+    args: () => [this.target] as const,
+    task: async ([], { signal }) => {
+      const md5Path = `/@ef-asset/${this.targetElement.getAttribute("src") ?? ""}`;
+      const response = await fetch(md5Path, { method: "HEAD", signal });
+      return response.headers.get("etag") ?? undefined;
+    },
+  });
+
+  productionSrc() {
+    if (!this.md5SumLoader.value) {
+      throw new Error(
+        `MD5 sum not available for ${this}. Cannot generate production URL`,
+      );
+    }
+    return `http://localhost:3000/api/video2/caption_files/${this.md5SumLoader.value}`;
+  }
+
+  // get requiredAssets() {
+  //   return { [this.md5SumLoader.value]: [this.captionsPath()] };
+  // }
+
+  private captionsDataTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.captionsPath(), this.fetch] as const,
+    task: async ([captionsPath, fetch], { signal }) => {
+      const response = await fetch(captionsPath, { signal });
+      return response.json() as any as Caption;
+    },
+  });
+
+  frameTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.captionsDataTask.status],
+    task: async () => {
+      await this.captionsDataTask.taskComplete;
+    },
+  });
+
+  connectedCallback() {
+    super.connectedCallback();
+    if (this.targetElement) {
+      new CrossUpdateController(this.targetElement, this);
+    }
+  }
+
+  render() {
+    return this.captionsDataTask.render({
+      pending: () => html`<div>Generating captions data...</div>`,
+      error: () => html`<div>🚫 Error generating captions data</div>`,
+      complete: () => html`<slot></slot>`,
+    });
+  }
+
+  protected updated(
+    _changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+  ): void {
+    this.updateActiveWord();
+  }
+
+  updateActiveWord() {
+    const caption = this.captionsDataTask.value;
+    if (!caption) {
+      return;
+    }
+    const words: string[] = [];
+    let startMs: number;
+    let endMs: number;
+    caption.segments.forEach((segment) => {
+      if (
+        this.targetElement.ownCurrentTimeMs >= segment.start * 1000 &&
+        this.targetElement.ownCurrentTimeMs <= segment.end * 1000
+      ) {
+        return segment.words.map((word) => {
+          if (
+            this.targetElement.ownCurrentTimeMs >= word.start * 1000 &&
+            this.targetElement.ownCurrentTimeMs <= word.end * 1000
+          ) {
+            words.push(word.text);
+            startMs = word.start * 1000;
+            endMs = word.end * 1000;
+          } else {
+          }
+        });
+      }
+    });
+    Array.from(this.activeWordContainers).forEach((container) => {
+      container.wordText = words.join(" ");
+      container.wordStartMs = startMs;
+      container.wordEndMs = endMs;
+    });
+  }
+
+  get targetElement() {
+    const target = document.querySelector(this.getAttribute("target") ?? "");
+    if (target instanceof EFAudio || target instanceof EFVideo) {
+      return target;
+    }
+    throw new Error("Invalid target, must be an EFAudio  or EFVideo element");
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "ef-captions": EFCaptions;
+    "ef-captions-active-word": EFCaptionsActiveWord;
+  }
+}

package/src/elements/EFImage.ts

@@ -0,0 +1,70 @@
+import { Task } from "@lit/task";
+import { LitElement, html, css } from "lit";
+import { customElement } from "lit/decorators.js";
+import { createRef, ref } from "lit/directives/ref.js";
+import { FetchMixin } from "./FetchMixin";
+import { EFSourceMixin } from "./EFSourceMixin";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+
+@customElement("ef-image")
+export class EFImage extends EFSourceMixin(FetchMixin(LitElement), {
+  assetType: "image_files",
+}) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+      canvas {
+        display: block;
+        width: 100%;
+        height: 100%;
+        object-fit: fill;
+        object-position: center;
+      }
+    `,
+  ];
+
+  imageRef = createRef<HTMLImageElement>();
+  canvasRef = createRef<HTMLCanvasElement>();
+
+  render() {
+    return html`<canvas ${ref(this.canvasRef)}></canvas>`;
+  }
+
+  assetPath() {
+    if (this.src.startsWith("http")) {
+      return this.src;
+    }
+    return `/@ef-image/${this.src}`;
+  }
+
+  // get requiredAssets() {
+  //   return { [this.md5SumLoader.value]: [this.assetPath()] };
+  // }
+
+  fetchImage = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.assetPath(), this.fetch] as const,
+    task: async ([assetPath, fetch], { signal }) => {
+      const response = await fetch(assetPath, { signal });
+      const image = new Image();
+      image.src = URL.createObjectURL(await response.blob());
+      await new Promise((resolve) => {
+        image.onload = resolve;
+      });
+      this.canvasRef.value!.width = image.width;
+      this.canvasRef.value!.height = image.height;
+      const ctx = this.canvasRef.value!.getContext("2d")!;
+      ctx.drawImage(image, 0, 0);
+    },
+  });
+
+  frameTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.fetchImage.status] as const,
+    task: async () => {
+      await this.fetchImage.taskComplete;
+    },
+  });
+}