@editframe/elements 0.5.0-beta.8 → 0.6.0-beta.1

package/docker-compose.yaml

@@ -1,17 +0,0 @@
-
-services:
-  build-elements:
-    image: telecine-runner
-    depends_on:
-      - runner
-    working_dir: /app
-    env_file:
-      - ${PWD}/.env
-    volumes_from:
-      - runner
-    stop_grace_period: 1s
-    init: true
-    scale: 1
-    environment:
-      - SHELL=/bin/sh
-    command: npx vite build --config lib/elements/vite.config.ts lib/elements/src/elements.ts --watch

package/src/EF_FRAMEGEN.ts

@@ -1,208 +0,0 @@
-import type { VideoRenderOptions } from "services/render/app/RenderOptions";
-import {
-  deepGetElementsWithFrameTasks,
-  shallowGetTimegroups,
-} from "./elements/EFTemporal";
-import { awaitAnimationFrame } from "@/util/awaitAnimationFrame";
-import { awaitMicrotask } from "@/util/awaitMicrotask";
-
-declare global {
-  interface Window {
-    EF_FRAMEGEN?: {
-      onRender: (
-        callback: (
-          renderId: string,
-          traceCarrier: unknown,
-          renderOptions: VideoRenderOptions,
-        ) => void,
-      ) => void;
-      onBegin: (
-        callback: (
-          traceCarrier: unknown,
-          frame: number,
-          isLast: boolean,
-        ) => void,
-      ) => void;
-      frameReady: (
-        renderId: string,
-        frameNumber: number,
-        samples: ArrayBufferLike,
-      ) => void;
-      onPaint: (callback: () => void) => void;
-      didPaint: (renderId: string) => void;
-      end: (renderId: string) => void;
-      error: (renderId: string, error: string) => void;
-    };
-  }
-}
-
-class TriggerCanvas {
-  private canvas: HTMLCanvasElement;
-  private ctx: CanvasRenderingContext2D;
-
-  constructor() {
-    this.canvas = document.createElement("canvas");
-    this.canvas.width = 1;
-    this.canvas.height = 1;
-    Object.assign(this.canvas.style, {
-      position: "absolute",
-      top: "0px",
-      left: "0px",
-      width: `1px`,
-      height: `1px`,
-      zIndex: "100000",
-    });
-    document.body.prepend(this.canvas);
-    this.ctx = this.canvas.getContext("2d")!;
-  }
-
-  trigger() {
-    console.log("Triggering");
-    this.ctx.fillStyle = "rgba(50, 0, 0, .8)";
-    this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
-    this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
-  }
-}
-
-if (window.EF_FRAMEGEN !== undefined) {
-  const EF_FRAMEGEN = window.EF_FRAMEGEN;
-  EF_FRAMEGEN.onRender((renderId, traceCarrier, renderOptions) => {
-    const crashOnUnhandledError = (error: string) => {
-      EF_FRAMEGEN.error(renderId, error);
-    };
-    window.addEventListener("error", (error) => {
-      console.warn("Crashing due to unhandled error", error);
-      crashOnUnhandledError(error.message);
-    });
-    window.addEventListener("unhandledrejection", (error) => {
-      console.warn("Crashing due to unhandled rejection", error);
-      crashOnUnhandledError(error.reason);
-    });
-
-    const triggerCanvas = new TriggerCanvas();
-    EF_FRAMEGEN.onPaint(async () => {
-      // Calls clearRect on the canvas to force the compositor to emit a frame.
-      triggerCanvas.trigger();
-
-      // We must wait for 2 frames to ensure offscreen render capture doesn't capture
-      // a stale frame. This is terrible. Fixing this requires contribiting to electron.js
-      // On the positive side, were' running at 240fps so it only burns 8ms max.
-      // (electron can't go faster thon 240fps)
-      // In practice it's less because we're already some ms into the next frame budget.
-      await awaitAnimationFrame();
-      await awaitAnimationFrame();
-
-      EF_FRAMEGEN.didPaint(renderId);
-    });
-    const workbench = document.querySelector("ef-workbench")!;
-    workbench.rendering = true;
-    const timegroups = shallowGetTimegroups(workbench);
-    const temporals = deepGetElementsWithFrameTasks(workbench);
-    const firstGroup = timegroups[0];
-    if (!firstGroup) {
-      throw new Error("No temporal elements found");
-    }
-    firstGroup.currentTimeMs = renderOptions.encoderOptions.fromMs;
-
-    const frameDurationMs = 1000 / renderOptions.encoderOptions.video.framerate;
-
-    const initialBusyTasks = Promise.all(
-      temporals
-        // .filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE)
-        .map((temporal) => temporal.frameTask)
-        .map((task) => task.taskComplete),
-    );
-
-    const frameBox = document.createElement("div");
-    Object.assign(frameBox.style, {
-      width: "200px",
-      height: "100px",
-      font: "30px Arial",
-      backgroundColor: "white",
-      position: "absolute",
-      top: "0px",
-      left: "0px",
-      zIndex: "100000",
-    });
-
-    let time = 0;
-    // document.body.prepend(frameBox);
-    // TODO: terminate if no workbench found
-    console.log("onRender", renderId, traceCarrier, renderOptions);
-
-    const audioBufferPromise = firstGroup.renderAudio(
-      renderOptions.encoderOptions.alignedFromUs / 1000,
-      renderOptions.encoderOptions.alignedToUs / 1000,
-      // renderOptions.encoderOptions.fromMs,
-      // renderOptions.encoderOptions.toMs,
-    );
-    EF_FRAMEGEN.onBegin(async (traceCarrier, frame, isLast) => {
-      time = firstGroup.currentTimeMs =
-        renderOptions.encoderOptions.fromMs + frame * frameDurationMs;
-      console.log("FRAME #", frame);
-      frameBox.innerHTML = `
-      <div>Frame #${frame}</div>
-      <div>${time.toFixed(4)}</div>
-      `;
-
-      await initialBusyTasks;
-
-      console.log("TIME", time.toFixed(4));
-      await awaitMicrotask();
-      console.log("After microtask");
-
-      const now = performance.now();
-      console.log(`frame:${frame} Awaiting busyTasks`);
-      await Promise.all(
-        temporals
-          // .filter((temporal) => temporal.frameTask.status < TaskStatus.COMPLETE)
-          .map((temporal) => {
-            console.log(
-              "Awaiting",
-              temporal.tagName,
-              temporal.frameTask.status,
-              temporal.frameTask.taskComplete,
-            );
-            return temporal.frameTask;
-          })
-          .map((task) => task.taskComplete),
-      );
-      console.log(
-        `frame:${frame} All tasks complete ${performance.now() - now}ms`,
-      );
-
-      await awaitAnimationFrame();
-      await awaitAnimationFrame();
-
-      // Trigger a small canvas paint to force the compositor to emit a frame.
-      triggerCanvas.trigger();
-
-      if (isLast) {
-        // Currently we emit the audio in one belch at the end of the render.
-        // This is not ideal, but it's the simplest thing that could possibly work.
-        // We could either emit it slices, or in parallel with the video.
-        // But in any case, it's fine for now.
-        const renderedAudio = await audioBufferPromise;
-
-        const channelCount = renderedAudio.numberOfChannels;
-
-        const interleavedSamples = new Float32Array(
-          channelCount * renderedAudio.length,
-        );
-
-        for (let i = 0; i < renderedAudio.length; i++) {
-          for (let j = 0; j < channelCount; j++) {
-            interleavedSamples.set(
-              renderedAudio.getChannelData(j).slice(i, i + 1),
-              i * channelCount + j,
-            );
-          }
-        }
-
-        EF_FRAMEGEN.frameReady(renderId, frame, interleavedSamples.buffer);
-      } else {
-        EF_FRAMEGEN.frameReady(renderId, frame, new Float32Array(0).buffer);
-      }
-    });
-  });
-}

package/src/EF_INTERACTIVE.ts

@@ -1,2 +0,0 @@
-export const EF_INTERACTIVE =
-  !window.location.search.includes("EF_NONINTERACTIVE");

package/src/elements/CrossUpdateController.ts

@@ -1,18 +0,0 @@
-import { LitElement, ReactiveController, ReactiveControllerHost } from "lit";
-
-export class CrossUpdateController implements ReactiveController {
-  constructor(
-    private host: ReactiveControllerHost,
-    private target: LitElement,
-  ) {
-    this.host.addController(this);
-  }
-
-  hostUpdate(): void {
-    this.target.requestUpdate();
-  }
-
-  remove(): void {
-    this.host.removeController(this);
-  }
-}

package/src/elements/EFAudio.ts

@@ -1,42 +0,0 @@
-import { html } from "lit";
-import { createRef, ref } from "lit/directives/ref.js";
-import { customElement, property } from "lit/decorators.js";
-import { EFMedia } from "./EFMedia";
-import { Task } from "@lit/task";
-
-@customElement("ef-audio")
-export class EFAudio extends EFMedia {
-  audioElementRef = createRef<HTMLAudioElement>();
-
-  @property({ type: String })
-  src = "";
-
-  render() {
-    return html`<audio ${ref(this.audioElementRef)}></audio>`;
-  }
-
-  get audioElement() {
-    return this.audioElementRef.value;
-  }
-
-  frameTask = new Task(this, {
-    args: () =>
-      [
-        this.trackFragmentIndexLoader.status,
-        this.initSegmentsLoader.status,
-        this.seekTask.status,
-        this.fetchSeekTask.status,
-        this.videoAssetTask.status,
-      ] as const,
-    task: async () => {
-      console.log("EFAudio frameTask", this.ownCurrentTimeMs);
-      await this.trackFragmentIndexLoader.taskComplete;
-      await this.initSegmentsLoader.taskComplete;
-      await this.seekTask.taskComplete;
-      await this.fetchSeekTask.taskComplete;
-      await this.videoAssetTask.taskComplete;
-      console.log("EFAudio frameTask complete", this.ownCurrentTimeMs);
-      this.rootTimegroup?.requestUpdate();
-    },
-  });
-}

package/src/elements/EFCaptions.ts

@@ -1,202 +0,0 @@
-import { EFAudio } from "./EFAudio";
-import { LitElement, PropertyValueMap, html, css } from "lit";
-import { Task } from "@lit/task";
-import { customElement, property } from "lit/decorators.js";
-import { EFVideo } from "./EFVideo";
-import { EFTemporal } from "./EFTemporal";
-import { CrossUpdateController } from "./CrossUpdateController";
-import { FetchMixin } from "./FetchMixin";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
-
-interface Word {
-  text: string;
-  start: number;
-  end: number;
-  confidence: number;
-}
-
-interface Segment {
-  start: number;
-  end: number;
-  text: string;
-  confidence: number;
-  words: Word[];
-}
-
-interface Caption {
-  text: string;
-  segments: Segment[];
-  language: string;
-}
-
-@customElement("ef-captions-active-word")
-export class EFCaptionsActiveWord extends EFTemporal(LitElement) {
-  static styles = [
-    css`
-      :host {
-        display: inline-block;
-      }
-    `,
-  ];
-
-  render() {
-    return html`${this.wordText}`;
-  }
-
-  @property({ type: Number, attribute: false })
-  wordStartMs = 0;
-
-  @property({ type: Number, attribute: false })
-  wordEndMs = 0;
-
-  @property({ type: String, attribute: false })
-  wordText = "";
-
-  get startTimeMs() {
-    return this.wordStartMs || 0;
-  }
-
-  get durationMs(): number {
-    return this.wordEndMs - this.wordStartMs;
-  }
-}
-
-@customElement("ef-captions")
-export class EFCaptions extends EFSourceMixin(
-  EFTemporal(FetchMixin(LitElement)),
-  { assetType: "caption_files" },
-) {
-  static styles = [
-    css`
-      :host {
-        display: block;
-      }
-    `,
-  ];
-
-  @property({ type: String, attribute: "target" })
-  target = null;
-
-  @property({ attribute: "word-style" })
-  wordStyle = "";
-
-  activeWordContainers = this.getElementsByTagName("ef-captions-active-word");
-
-  captionsPath() {
-    const src = this.targetElement.getAttribute("src");
-    if (src?.startsWith("http")) {
-      return src.replace("isobmff", "caption");
-    }
-    return `/@ef-captions/${this.targetElement.getAttribute("src") ?? ""}`;
-  }
-
-  protected md5SumLoader = new Task(this, {
-    autoRun: false,
-    args: () => [this.target] as const,
-    task: async ([], { signal }) => {
-      const md5Path = `/@ef-asset/${this.targetElement.getAttribute("src") ?? ""}`;
-      const response = await fetch(md5Path, { method: "HEAD", signal });
-      return response.headers.get("etag") ?? undefined;
-    },
-  });
-
-  productionSrc() {
-    if (!this.md5SumLoader.value) {
-      throw new Error(
-        `MD5 sum not available for ${this}. Cannot generate production URL`,
-      );
-    }
-    return `http://localhost:3000/api/video2/caption_files/${this.md5SumLoader.value}`;
-  }
-
-  // get requiredAssets() {
-  //   return { [this.md5SumLoader.value]: [this.captionsPath()] };
-  // }
-
-  private captionsDataTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.captionsPath(), this.fetch] as const,
-    task: async ([captionsPath, fetch], { signal }) => {
-      const response = await fetch(captionsPath, { signal });
-      return response.json() as any as Caption;
-    },
-  });
-
-  frameTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.captionsDataTask.status],
-    task: async () => {
-      await this.captionsDataTask.taskComplete;
-    },
-  });
-
-  connectedCallback() {
-    super.connectedCallback();
-    if (this.targetElement) {
-      new CrossUpdateController(this.targetElement, this);
-    }
-  }
-
-  render() {
-    return this.captionsDataTask.render({
-      pending: () => html`<div>Generating captions data...</div>`,
-      error: () => html`<div>🚫 Error generating captions data</div>`,
-      complete: () => html`<slot></slot>`,
-    });
-  }
-
-  protected updated(
-    _changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
-  ): void {
-    this.updateActiveWord();
-  }
-
-  updateActiveWord() {
-    const caption = this.captionsDataTask.value;
-    if (!caption) {
-      return;
-    }
-    const words: string[] = [];
-    let startMs: number;
-    let endMs: number;
-    caption.segments.forEach((segment) => {
-      if (
-        this.targetElement.ownCurrentTimeMs >= segment.start * 1000 &&
-        this.targetElement.ownCurrentTimeMs <= segment.end * 1000
-      ) {
-        return segment.words.map((word) => {
-          if (
-            this.targetElement.ownCurrentTimeMs >= word.start * 1000 &&
-            this.targetElement.ownCurrentTimeMs <= word.end * 1000
-          ) {
-            words.push(word.text);
-            startMs = word.start * 1000;
-            endMs = word.end * 1000;
-          } else {
-          }
-        });
-      }
-    });
-    Array.from(this.activeWordContainers).forEach((container) => {
-      container.wordText = words.join(" ");
-      container.wordStartMs = startMs;
-      container.wordEndMs = endMs;
-    });
-  }
-
-  get targetElement() {
-    const target = document.querySelector(this.getAttribute("target") ?? "");
-    if (target instanceof EFAudio || target instanceof EFVideo) {
-      return target;
-    }
-    throw new Error("Invalid target, must be an EFAudio  or EFVideo element");
-  }
-}
-
-declare global {
-  interface HTMLElementTagNameMap {
-    "ef-captions": EFCaptions;
-    "ef-captions-active-word": EFCaptionsActiveWord;
-  }
-}

package/src/elements/EFImage.ts

@@ -1,70 +0,0 @@
-import { Task } from "@lit/task";
-import { LitElement, html, css } from "lit";
-import { customElement } from "lit/decorators.js";
-import { createRef, ref } from "lit/directives/ref.js";
-import { FetchMixin } from "./FetchMixin";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
-
-@customElement("ef-image")
-export class EFImage extends EFSourceMixin(FetchMixin(LitElement), {
-  assetType: "image_files",
-}) {
-  static styles = [
-    css`
-      :host {
-        display: block;
-      }
-      canvas {
-        display: block;
-        width: 100%;
-        height: 100%;
-        object-fit: fill;
-        object-position: center;
-      }
-    `,
-  ];
-
-  imageRef = createRef<HTMLImageElement>();
-  canvasRef = createRef<HTMLCanvasElement>();
-
-  render() {
-    return html`<canvas ${ref(this.canvasRef)}></canvas>`;
-  }
-
-  assetPath() {
-    if (this.src.startsWith("http")) {
-      return this.src;
-    }
-    return `/@ef-image/${this.src}`;
-  }
-
-  // get requiredAssets() {
-  //   return { [this.md5SumLoader.value]: [this.assetPath()] };
-  // }
-
-  fetchImage = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.assetPath(), this.fetch] as const,
-    task: async ([assetPath, fetch], { signal }) => {
-      const response = await fetch(assetPath, { signal });
-      const image = new Image();
-      image.src = URL.createObjectURL(await response.blob());
-      await new Promise((resolve) => {
-        image.onload = resolve;
-      });
-      this.canvasRef.value!.width = image.width;
-      this.canvasRef.value!.height = image.height;
-      const ctx = this.canvasRef.value!.getContext("2d")!;
-      ctx.drawImage(image, 0, 0);
-    },
-  });
-
-  frameTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.fetchImage.status] as const,
-    task: async () => {
-      await this.fetchImage.taskComplete;
-    },
-  });
-}