@editframe/elements 0.6.0-beta.9 → 0.7.0-beta.4

package/package.json

@@ -1,15 +1,20 @@
 {
   "name": "@editframe/elements",
-  "version": "0.6.0-beta.9",
+  "version": "0.7.0-beta.4",
   "description": "",
   "exports": {
     ".": {
-      "import": "./dist/packages/elements/src/index.js",
-      "require": "./dist/packages/elements/src/index.cjs"
+      "import": {
+        "default": "./dist/packages/elements/src/index.js",
+        "types": "./dist/packages/elements/src/index.d.ts"
+      },
+      "require": {
+        "default": "./dist/packages/elements/src/index.cjs",
+        "types": "./dist/packages/elements/src/index.d.ts"
+      }
     },
     "./styles.css": "./dist/style.css"
   },
-  "types": "./dist/packages/elements/src/index.d.ts",
   "type": "module",
   "scripts": {
     "typecheck": "tsc --noEmit --emitDeclarationOnly false",
@@ -19,11 +24,12 @@
   "author": "",
   "license": "UNLICENSED",
   "dependencies": {
-    "@editframe/assets": "0.6.0-beta.9",
-    "@lit/context": "^1.1.1",
-    "@lit/task": "^1.0.0",
+    "@editframe/assets": "0.7.0-beta.4",
+    "@lit/context": "^1.1.2",
+    "@lit/task": "^1.0.1",
     "d3": "^7.9.0",
-    "lit": "^3.1.3",
+    "debug": "^4.3.5",
+    "lit": "^3.1.4",
     "mp4box": "^0.5.2"
   },
   "devDependencies": {

package/src/elements/CrossUpdateController.ts

@@ -0,0 +1,22 @@
+import type {
+  LitElement,
+  ReactiveController,
+  ReactiveControllerHost,
+} from "lit";
+
+export class CrossUpdateController implements ReactiveController {
+  constructor(
+    private host: ReactiveControllerHost,
+    private target: LitElement,
+  ) {
+    this.host.addController(this);
+  }
+
+  hostUpdate(): void {
+    this.target.requestUpdate();
+  }
+
+  remove(): void {
+    this.host.removeController(this);
+  }
+}

package/src/elements/EFAudio.ts

@@ -0,0 +1,40 @@
+import { html } from "lit";
+import { createRef, ref } from "lit/directives/ref.js";
+import { customElement, property } from "lit/decorators.js";
+import { EFMedia } from "./EFMedia";
+import { Task } from "@lit/task";
+
+@customElement("ef-audio")
+export class EFAudio extends EFMedia {
+  audioElementRef = createRef<HTMLAudioElement>();
+
+  @property({ type: String })
+  src = "";
+
+  render() {
+    return html`<audio ${ref(this.audioElementRef)}></audio>`;
+  }
+
+  get audioElement() {
+    return this.audioElementRef.value;
+  }
+
+  frameTask = new Task(this, {
+    args: () =>
+      [
+        this.trackFragmentIndexLoader.status,
+        this.initSegmentsLoader.status,
+        this.seekTask.status,
+        this.fetchSeekTask.status,
+        this.videoAssetTask.status,
+      ] as const,
+    task: async () => {
+      await this.trackFragmentIndexLoader.taskComplete;
+      await this.initSegmentsLoader.taskComplete;
+      await this.seekTask.taskComplete;
+      await this.fetchSeekTask.taskComplete;
+      await this.videoAssetTask.taskComplete;
+      this.rootTimegroup?.requestUpdate();
+    },
+  });
+}

package/src/elements/EFCaptions.ts

@@ -0,0 +1,188 @@
+import { EFAudio } from "./EFAudio";
+import { LitElement, type PropertyValueMap, html, css } from "lit";
+import { Task } from "@lit/task";
+import { customElement, property } from "lit/decorators.js";
+import { EFVideo } from "./EFVideo";
+import { EFTemporal } from "./EFTemporal";
+import { CrossUpdateController } from "./CrossUpdateController";
+import { FetchMixin } from "./FetchMixin";
+import { EFSourceMixin } from "./EFSourceMixin";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+
+interface Word {
+  text: string;
+  start: number;
+  end: number;
+  confidence: number;
+}
+
+interface Segment {
+  start: number;
+  end: number;
+  text: string;
+  confidence: number;
+  words: Word[];
+}
+
+interface Caption {
+  text: string;
+  segments: Segment[];
+  language: string;
+}
+
+@customElement("ef-captions-active-word")
+export class EFCaptionsActiveWord extends EFTemporal(LitElement) {
+  static styles = [
+    css`
+      :host {
+        display: inline-block;
+      }
+    `,
+  ];
+
+  render() {
+    return html`${this.wordText}`;
+  }
+
+  @property({ type: Number, attribute: false })
+  wordStartMs = 0;
+
+  @property({ type: Number, attribute: false })
+  wordEndMs = 0;
+
+  @property({ type: String, attribute: false })
+  wordText = "";
+
+  get startTimeMs() {
+    return this.wordStartMs || 0;
+  }
+
+  get durationMs(): number {
+    return this.wordEndMs - this.wordStartMs;
+  }
+}
+
+@customElement("ef-captions")
+export class EFCaptions extends EFSourceMixin(
+  EFTemporal(FetchMixin(LitElement)),
+  { assetType: "caption_files" },
+) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+    `,
+  ];
+
+  @property({ type: String, attribute: "target" })
+  target = null;
+
+  @property({ attribute: "word-style" })
+  wordStyle = "";
+
+  activeWordContainers = this.getElementsByTagName("ef-captions-active-word");
+
+  captionsPath() {
+    const targetSrc = this.targetElement.src;
+    if (targetSrc.startsWith("editframe://") || targetSrc.startsWith("http")) {
+      return targetSrc.replace("isobmff", "caption");
+    }
+    return `/@ef-captions/${targetSrc}`;
+  }
+
+  protected md5SumLoader = new Task(this, {
+    autoRun: false,
+    args: () => [this.target] as const,
+    task: async ([], { signal }) => {
+      const md5Path = `/@ef-asset/${this.targetElement.src ?? ""}`;
+      const response = await fetch(md5Path, { method: "HEAD", signal });
+      return response.headers.get("etag") ?? undefined;
+    },
+  });
+
+  private captionsDataTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.captionsPath(), this.fetch] as const,
+    task: async ([captionsPath, fetch], { signal }) => {
+      const response = await fetch(captionsPath, { signal });
+      return response.json() as any as Caption;
+    },
+  });
+
+  frameTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.captionsDataTask.status],
+    task: async () => {
+      await this.captionsDataTask.taskComplete;
+    },
+  });
+
+  connectedCallback() {
+    super.connectedCallback();
+    if (this.targetElement) {
+      new CrossUpdateController(this.targetElement, this);
+    }
+  }
+
+  render() {
+    return this.captionsDataTask.render({
+      pending: () => html`<div>Generating captions data...</div>`,
+      error: () => html`<div>🚫 Error generating captions data</div>`,
+      complete: () => html`<slot></slot>`,
+    });
+  }
+
+  protected updated(
+    _changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+  ): void {
+    this.updateActiveWord();
+  }
+
+  updateActiveWord() {
+    const caption = this.captionsDataTask.value;
+    if (!caption) {
+      return;
+    }
+    const words: string[] = [];
+    let startMs = 0;
+    let endMs = 0;
+    for (const segment of caption.segments) {
+      if (
+        this.targetElement.ownCurrentTimeMs >= segment.start * 1000 &&
+        this.targetElement.ownCurrentTimeMs <= segment.end * 1000
+      ) {
+        for (const word of segment.words) {
+          if (
+            this.targetElement.ownCurrentTimeMs >= word.start * 1000 &&
+            this.targetElement.ownCurrentTimeMs <= word.end * 1000
+          ) {
+            words.push(word.text);
+            startMs = word.start * 1000;
+            endMs = word.end * 1000;
+          }
+        }
+      }
+    }
+    for (const container of Array.from(this.activeWordContainers)) {
+      container.wordText = words.join(" ");
+      container.wordStartMs = startMs;
+      container.wordEndMs = endMs;
+    }
+  }
+
+  get targetElement() {
+    const target = document.getElementById(this.getAttribute("target") ?? "");
+    if (target instanceof EFAudio || target instanceof EFVideo) {
+      return target;
+    }
+    throw new Error("Invalid target, must be an EFAudio  or EFVideo element");
+  }
+}
+
+declare global {
+  interface HTMLElementTagNameMap {
+    "ef-captions": EFCaptions;
+    "ef-captions-active-word": EFCaptionsActiveWord;
+  }
+}

package/src/elements/EFImage.ts

@@ -0,0 +1,68 @@
+import { Task } from "@lit/task";
+import { LitElement, html, css } from "lit";
+import { customElement } from "lit/decorators.js";
+import { createRef, ref } from "lit/directives/ref.js";
+import { FetchMixin } from "./FetchMixin";
+import { EFSourceMixin } from "./EFSourceMixin";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+
+@customElement("ef-image")
+export class EFImage extends EFSourceMixin(FetchMixin(LitElement), {
+  assetType: "image_files",
+}) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+      canvas {
+        display: block;
+        width: 100%;
+        height: 100%;
+        object-fit: fill;
+        object-position: center;
+      }
+    `,
+  ];
+
+  imageRef = createRef<HTMLImageElement>();
+  canvasRef = createRef<HTMLCanvasElement>();
+
+  render() {
+    return html`<canvas ${ref(this.canvasRef)}></canvas>`;
+  }
+
+  assetPath() {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return this.src;
+    }
+    return `/@ef-image/${this.src}`;
+  }
+
+  fetchImage = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.assetPath(), this.fetch] as const,
+    task: async ([assetPath, fetch], { signal }) => {
+      const response = await fetch(assetPath, { signal });
+      const image = new Image();
+      image.src = URL.createObjectURL(await response.blob());
+      await new Promise((resolve) => {
+        image.onload = resolve;
+      });
+      if (!this.canvasRef.value) throw new Error("Canvas not ready");
+      const ctx = this.canvasRef.value.getContext("2d");
+      if (!ctx) throw new Error("Canvas 2d context not ready");
+      this.canvasRef.value.width = image.width;
+      this.canvasRef.value.height = image.height;
+      ctx.drawImage(image, 0, 0);
+    },
+  });
+
+  frameTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.fetchImage.status] as const,
+    task: async () => {
+      await this.fetchImage.taskComplete;
+    },
+  });
+}