@editframe/elements 0.11.0-beta.9 → 0.12.0-beta.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@editframe/elements",
-  "version": "0.11.0-beta.9",
+  "version": "0.12.0-beta.10",
   "description": "",
   "exports": {
     ".": {
@@ -20,7 +20,7 @@
   "author": "",
   "license": "UNLICENSED",
   "dependencies": {
-    "@editframe/assets": "0.11.0-beta.9",
+    "@editframe/assets": "0.12.0-beta.10",
     "@lit/context": "^1.1.2",
     "@lit/task": "^1.0.1",
     "d3": "^7.9.0",

package/src/elements/EFCaptions.browsertest.ts

@@ -21,14 +21,14 @@ describe("EFCaptions", () => {
 
       const target = document.createElement("ef-video");
       target.setAttribute("id", id);
-      target.assetId = "550e8400-e29b-41d4-a716-446655440000:example.mp4";
+      target.assetId = "550e8400-e29b-41d4-a716-446655440000";
       document.body.appendChild(target);
       const captions = document.createElement("ef-captions");
       captions.setAttribute("target", id);
       document.body.appendChild(captions);
       workbench.appendChild(captions);
       expect(captions.captionsPath()).toBe(
-        "editframe://api/v1/caption_files/550e8400-e29b-41d4-a716-446655440000:example.mp4",
+        "editframe://api/v1/caption_files/550e8400-e29b-41d4-a716-446655440000",
       );
     });
   });
@@ -38,13 +38,13 @@ describe("EFCaptions", () => {
       const id = v4();
       const target = document.createElement("ef-video");
       target.setAttribute("id", id);
-      target.assetId = `${id}:example.mp4`;
+      target.assetId = id;
       document.body.appendChild(target);
       const captions = document.createElement("ef-captions");
       captions.setAttribute("target", id);
       document.body.appendChild(captions);
       expect(captions.captionsPath()).toBe(
-        `https://editframe.dev/api/v1/caption_files/${id}:example.mp4`,
+        `https://editframe.dev/api/v1/caption_files/${id}`,
       );
     });
 
@@ -54,7 +54,7 @@ describe("EFCaptions", () => {
       const id = v4();
       const target = document.createElement("ef-video");
       target.setAttribute("id", id);
-      target.assetId = `${id}:example.mp4`;
+      target.assetId = id;
       document.body.appendChild(target);
       const captions = document.createElement("ef-captions");
       captions.setAttribute("target", id);
@@ -62,7 +62,7 @@ describe("EFCaptions", () => {
       document.body.appendChild(preview);
       preview.apiHost = "test://";
       expect(captions.captionsPath()).toBe(
-        `test:///api/v1/caption_files/${id}:example.mp4`,
+        `test:///api/v1/caption_files/${id}`,
       );
     });
   });

package/src/elements/EFCaptions.ts

@@ -1,48 +1,56 @@
-import { LitElement, type PropertyValueMap, html, css } from "lit";
 import { Task } from "@lit/task";
+import { LitElement, type PropertyValueMap, css, html } from "lit";
 import { customElement, property } from "lit/decorators.js";
-import { EFVideo } from "./EFVideo.ts";
+import type { GetISOBMFFFileTranscriptionResult } from "../../../api/src/index.ts";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
+import { EF_RENDERING } from "../EF_RENDERING.ts";
+import { CrossUpdateController } from "./CrossUpdateController.ts";
 import { EFAudio } from "./EFAudio.ts";
+import { EFSourceMixin } from "./EFSourceMixin.ts";
 import { EFTemporal } from "./EFTemporal.ts";
-import { CrossUpdateController } from "./CrossUpdateController.ts";
+import { EFVideo } from "./EFVideo.ts";
 import { FetchMixin } from "./FetchMixin.ts";
-import { EFSourceMixin } from "./EFSourceMixin.ts";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
-import { EF_RENDERING } from "../EF_RENDERING.ts";
 
-interface Word {
+interface WordSegment {
   text: string;
   start: number;
   end: number;
-  confidence: number;
 }
 
 interface Segment {
   start: number;
   end: number;
   text: string;
-  confidence: number;
-  words: Word[];
 }
 
 interface Caption {
-  text: string;
   segments: Segment[];
-  language: string;
+  word_segments: WordSegment[];
 }
 
+const stopWords = new Set(["", ".", "!", "?", ","]);
+
 @customElement("ef-captions-active-word")
 export class EFCaptionsActiveWord extends EFTemporal(LitElement) {
   static styles = [
     css`
       :host {
         display: inline-block;
+        white-space: pre;
+      }
+      :host([hidden]) {
+        display: none;
       }
     `,
   ];
 
   render() {
-    return html`${this.wordText}`;
+    if (stopWords.has(this.wordText)) {
+      this.hidden = true;
+      return undefined;
+    }
+    this.hidden = false;
+    return html` ${this.wordText.trim()} `;
   }
 
   @property({ type: Number, attribute: false })
@@ -54,6 +62,9 @@ export class EFCaptionsActiveWord extends EFTemporal(LitElement) {
   @property({ type: String, attribute: false })
   wordText = "";
 
+  @property({ type: Boolean, reflect: true })
+  hidden = false;
+
   get startTimeMs() {
     return this.wordStartMs || 0;
   }
@@ -63,6 +74,137 @@ export class EFCaptionsActiveWord extends EFTemporal(LitElement) {
   }
 }
 
+@customElement("ef-captions-segment")
+export class EFCaptionsSegment extends EFTemporal(LitElement) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+      }
+      :host([hidden]) {
+        display: none;
+      }
+    `,
+  ];
+
+  render() {
+    if (stopWords.has(this.segmentText)) {
+      this.hidden = true;
+      return undefined;
+    }
+    this.hidden = false;
+    return html`${this.segmentText}`;
+  }
+
+  @property({ type: Number, attribute: false })
+  segmentStartMs = 0;
+
+  @property({ type: Number, attribute: false })
+  segmentEndMs = 0;
+
+  @property({ type: String, attribute: false })
+  segmentText = "";
+
+  @property({ type: Boolean, reflect: true })
+  hidden = false;
+
+  get startTimeMs() {
+    return this.segmentStartMs || 0;
+  }
+
+  get durationMs(): number {
+    return this.segmentEndMs - this.segmentStartMs;
+  }
+}
+
+@customElement("ef-captions-before-active-word")
+export class EFCaptionsBeforeActiveWord extends EFCaptionsSegment {
+  static styles = [
+    css`
+      :host {
+        display: inline-block;
+        white-space: pre;
+      }
+      :host([hidden]) {
+        display: none;
+      }
+    `,
+  ];
+
+  render() {
+    if (stopWords.has(this.segmentText)) {
+      this.hidden = true;
+      return undefined;
+    }
+    this.hidden = false;
+    return html` ${this.segmentText}`;
+  }
+
+  @property({ type: Boolean, reflect: true })
+  hidden = false;
+
+  @property({ type: String, attribute: false })
+  segmentText = "";
+
+  @property({ type: Number, attribute: false })
+  segmentStartMs = 0;
+
+  @property({ type: Number, attribute: false })
+  segmentEndMs = 0;
+
+  get startTimeMs() {
+    return this.segmentStartMs || 0;
+  }
+
+  get durationMs(): number {
+    return this.segmentEndMs - this.segmentStartMs;
+  }
+}
+
+@customElement("ef-captions-after-active-word")
+export class EFCaptionsAfterActiveWord extends EFCaptionsSegment {
+  static styles = [
+    css`
+      :host {
+        display: inline-block;
+        white-space: pre;
+      }
+      :host([hidden]) {
+        display: none;
+      }
+    `,
+  ];
+
+  render() {
+    if (stopWords.has(this.segmentText)) {
+      this.hidden = true;
+      return undefined;
+    }
+    this.hidden = false;
+    return html`${this.segmentText} `;
+  }
+
+  @property({ type: Boolean, reflect: true })
+  hidden = false;
+
+  @property({ type: String, attribute: false })
+  segmentText = "";
+
+  @property({ type: Number, attribute: false })
+  segmentStartMs = 0;
+
+  @property({ type: Number, attribute: false })
+  segmentEndMs = 0;
+
+  get startTimeMs() {
+    return this.segmentStartMs || 0;
+  }
+
+  get durationMs(): number {
+    return this.segmentEndMs - this.segmentStartMs;
+  }
+}
+
 @customElement("ef-captions")
 export class EFCaptions extends EFSourceMixin(
   EFTemporal(FetchMixin(LitElement)),
@@ -71,11 +213,24 @@ export class EFCaptions extends EFSourceMixin(
   static styles = [
     css`
       :host {
-        display: block;
+        display: flex;
+        flex-wrap: wrap;
+        align-items: baseline;
+        width: fit-content;
+      }
+      ::slotted(*) {
+        margin: 0;
+        padding: 0;
       }
     `,
   ];
 
+  @property({ type: String, attribute: "display-mode", reflect: true })
+  displayMode: "word" | "segment" | "context" = "segment";
+
+  @property({ type: Number, attribute: "context-words", reflect: true })
+  contextWords = 3;
+
   @property({ type: String, attribute: "target", reflect: true })
   targetSelector = "";
 
@@ -87,6 +242,27 @@ export class EFCaptions extends EFSourceMixin(
   wordStyle = "";
 
   activeWordContainers = this.getElementsByTagName("ef-captions-active-word");
+  segmentContainers = this.getElementsByTagName("ef-captions-segment");
+  beforeActiveWordContainers = this.getElementsByTagName(
+    "ef-captions-before-active-word",
+  );
+  afterActiveWordContainers = this.getElementsByTagName(
+    "ef-captions-after-active-word",
+  );
+
+  render() {
+    return html`<slot></slot>`;
+  }
+
+  transcriptionsPath() {
+    if (this.targetElement.assetId) {
+      if (EF_RENDERING()) {
+        return `editframe://api/v1/isobmff_files/${this.targetElement.assetId}/transcription`;
+      }
+      return `${this.apiHost}/api/v1/isobmff_files/${this.targetElement.assetId}/transcription`;
+    }
+    return null;
+  }
 
   captionsPath() {
     if (this.targetElement.assetId) {
@@ -109,20 +285,71 @@ export class EFCaptions extends EFSourceMixin(
     },
   });
 
-  private captionsDataTask = new Task(this, {
+  private transcriptionDataTask = new Task(this, {
     autoRun: EF_INTERACTIVE,
-    args: () => [this.captionsPath(), this.fetch] as const,
-    task: async ([captionsPath, fetch], { signal }) => {
-      const response = await fetch(captionsPath, { signal });
+    args: () => [this.transcriptionsPath(), this.fetch] as const,
+    task: async ([transcriptionsPath, fetch], { signal }) => {
+      if (!transcriptionsPath) {
+        return null;
+      }
+      const response = await fetch(transcriptionsPath, { signal });
+      return response.json() as any as GetISOBMFFFileTranscriptionResult;
+    },
+  });
+
+  private transcriptionFragmentPath(
+    transcriptionId: string,
+    fragmentIndex: number,
+  ) {
+    return `${this.apiHost}/api/v1/transcriptions/${transcriptionId}/fragments/${fragmentIndex}`;
+  }
+
+  private fragmentIndexTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [this.transcriptionDataTask.value, this.ownCurrentTimeMs] as const,
+    task: async ([transcription, ownCurrentTimeMs]) => {
+      if (!transcription) {
+        return null;
+      }
+      const fragmentIndex = Math.floor(
+        ownCurrentTimeMs / transcription.work_slice_ms,
+      );
+      return fragmentIndex;
+    },
+  });
+
+  private transcriptionFragmentDataTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [
+        this.transcriptionDataTask.value,
+        this.fragmentIndexTask.value,
+        this.fetch,
+      ] as const,
+    task: async ([transcription, fragmentIndex, fetch], { signal }) => {
+      if (
+        transcription === null ||
+        transcription === undefined ||
+        fragmentIndex === null ||
+        fragmentIndex === undefined
+      ) {
+        return null;
+      }
+      const fragmentPath = this.transcriptionFragmentPath(
+        transcription.id,
+        fragmentIndex,
+      );
+      const response = await fetch(fragmentPath, { signal });
       return response.json() as any as Caption;
     },
   });
 
   frameTask = new Task(this, {
     autoRun: EF_INTERACTIVE,
-    args: () => [this.captionsDataTask.status],
+    args: () => [this.transcriptionFragmentDataTask.status],
     task: async () => {
-      await this.captionsDataTask.taskComplete;
+      await this.transcriptionFragmentDataTask.taskComplete;
     },
   });
 
@@ -133,51 +360,90 @@ export class EFCaptions extends EFSourceMixin(
     }
   }
 
-  render() {
-    return this.captionsDataTask.render({
-      pending: () => html`<div>Generating captions data...</div>`,
-      error: () => html`<div>🚫 Error generating captions data</div>`,
-      complete: () => html`<slot></slot>`,
-    });
-  }
-
   protected updated(
     _changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
   ): void {
-    this.updateActiveWord();
+    this.updateTextContainers();
   }
 
-  updateActiveWord() {
-    const caption = this.captionsDataTask.value;
-    if (!caption) {
+  updateTextContainers() {
+    const transcriptionFragment = this.transcriptionFragmentDataTask
+      .value as Caption;
+    if (!transcriptionFragment) {
       return;
     }
-    const words: string[] = [];
-    let startMs = 0;
-    let endMs = 0;
-    for (const segment of caption.segments) {
-      if (
-        this.targetElement.trimAdjustedOwnCurrentTimeMs >=
-          segment.start * 1000 &&
-        this.targetElement.trimAdjustedOwnCurrentTimeMs <= segment.end * 1000
-      ) {
-        for (const word of segment.words) {
-          if (
-            this.targetElement.trimAdjustedOwnCurrentTimeMs >=
-              word.start * 1000 &&
-            this.targetElement.trimAdjustedOwnCurrentTimeMs <= word.end * 1000
-          ) {
-            words.push(word.text);
-            startMs = word.start * 1000;
-            endMs = word.end * 1000;
-          }
-        }
+
+    const currentTimeMs = this.targetElement.trimAdjustedOwnCurrentTimeMs;
+    const currentTimeSec = currentTimeMs / 1000;
+
+    // Find the current word from word_segments
+    const currentWord = transcriptionFragment.word_segments.find(
+      (word) => currentTimeSec >= word.start && currentTimeSec <= word.end,
+    );
+
+    // Find the current segment
+    const currentSegment = transcriptionFragment.segments.find(
+      (segment) =>
+        currentTimeSec >= segment.start && currentTimeSec <= segment.end,
+    );
+
+    for (const wordContainer of this.activeWordContainers) {
+      if (currentWord) {
+        wordContainer.wordText = currentWord.text;
+        wordContainer.wordStartMs = currentWord.start * 1000;
+        wordContainer.wordEndMs = currentWord.end * 1000;
+      }
+    }
+
+    for (const segmentContainer of this.segmentContainers) {
+      if (currentSegment) {
+        segmentContainer.segmentText = currentSegment.text;
+        segmentContainer.segmentStartMs = currentSegment.start * 1000;
+        segmentContainer.segmentEndMs = currentSegment.end * 1000;
       }
     }
-    for (const container of Array.from(this.activeWordContainers)) {
-      container.wordText = words.join(" ");
-      container.wordStartMs = startMs;
-      container.wordEndMs = endMs;
+
+    // Only process context if we have both a current word and segment
+    if (currentWord && currentSegment) {
+      // Find all word segments that fall within the current segment's time range
+      const segmentWords = transcriptionFragment.word_segments.filter(
+        (word) =>
+          word.start >= currentSegment.start && word.end <= currentSegment.end,
+      );
+
+      // Find the index of the current word within the segment's word segments
+      const currentWordIndex = segmentWords.findIndex(
+        (word) =>
+          word.start === currentWord.start && word.end === currentWord.end,
+      );
+
+      if (currentWordIndex !== -1) {
+        // Get words before the current word
+        const beforeWords = segmentWords
+          .slice(0, currentWordIndex)
+          .map((w) => w.text.trim())
+          .join(" ");
+
+        // Get words after the current word
+        const afterWords = segmentWords
+          .slice(currentWordIndex + 1)
+          .map((w) => w.text.trim())
+          .join(" ");
+
+        // Update before containers
+        for (const container of this.beforeActiveWordContainers) {
+          container.segmentText = beforeWords;
+          container.segmentStartMs = currentSegment.start * 1000;
+          container.segmentEndMs = currentWord.start * 1000;
+        }
+
+        // Update after containers
+        for (const container of this.afterActiveWordContainers) {
+          container.segmentText = afterWords;
+          container.segmentStartMs = currentWord.end * 1000;
+          container.segmentEndMs = currentSegment.end * 1000;
+        }
+      }
     }
   }
 
@@ -194,5 +460,8 @@ declare global {
   interface HTMLElementTagNameMap {
     "ef-captions": EFCaptions;
     "ef-captions-active-word": EFCaptionsActiveWord;
+    "ef-captions-segment": EFCaptionsSegment;
+    "ef-captions-before-active-word": EFCaptionsBeforeActiveWord;
+    "ef-captions-after-active-word": EFCaptionsAfterActiveWord;
   }
 }

package/src/elements/EFImage.browsertest.ts

@@ -26,23 +26,12 @@ describe("EFImage", () => {
   });
 
   describe("attribute: asset-id", () => {
-    test("must match :id/basename", () => {
-      const image = document.createElement("ef-image");
-      expect(() => {
-        image.assetId = "1234:example.mp4";
-      }).toThrowError(
-        new Error(
-          "EFMedia: asset-id must match <uuid>:<basename>. (like: 550e8400-e29b-41d4-a716-446655440000:example.mp4)",
-        ),
-      );
-    });
-
     test("determines assetPath", () => {
       const id = v4();
       const image = document.createElement("ef-image");
-      image.setAttribute("asset-id", `${id}:example.jpg`);
+      image.setAttribute("asset-id", id);
       expect(image.assetPath()).toBe(
-        `https://editframe.dev/api/v1/image_files/${id}:example.jpg`,
+        `https://editframe.dev/api/v1/image_files/${id}`,
       );
     });
 
@@ -50,13 +39,11 @@ describe("EFImage", () => {
       const id = v4();
       const image = document.createElement("ef-image");
       const preview = document.createElement("ef-preview");
-      image.setAttribute("asset-id", `${id}:example.jpg`);
+      image.setAttribute("asset-id", id);
       preview.appendChild(image);
       preview.apiHost = "test://";
       document.body.appendChild(preview);
-      expect(image.assetPath()).toBe(
-        `test:///api/v1/image_files/${id}:example.jpg`,
-      );
+      expect(image.assetPath()).toBe(`test:///api/v1/image_files/${id}`);
     });
   });
 });

package/src/elements/EFImage.ts

@@ -28,12 +28,6 @@ export class EFImage extends EFSourceMixin(FetchMixin(LitElement), {
   #assetId: string | null = null;
   @property({ type: String, attribute: "asset-id", reflect: true })
   set assetId(value: string | null) {
-    if (!value?.match(/^.{8}-.{4}-.{4}-.{4}-.{12}:.*$/)) {
-      console.error(`EFMedia ${value} is not valid asset-id`);
-      throw new Error(
-        "EFMedia: asset-id must match <uuid>:<basename>. (like: 550e8400-e29b-41d4-a716-446655440000:example.mp4)",
-      );
-    }
     this.#assetId = value;
   }
 

package/src/elements/EFMedia.browsertest.ts

@@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, test } from "vitest";
 import { EFMedia } from "./EFMedia.ts";
 import "../gui/EFWorkbench.ts";
 import "../gui/EFPreview.ts";
+import "./EFTimegroup.ts";
 import { createTestFragmentIndex } from "TEST/createTestFragmentIndex.ts";
 import { useMockWorker } from "TEST/useMockWorker.ts";
 import { http, HttpResponse } from "msw";
@@ -57,58 +58,47 @@ describe("EFMedia", () => {
   });
 
   describe("attribute: asset-id", () => {
-    test("must match :id/basename", () => {
-      const element = document.createElement("test-media");
-      expect(() => {
-        element.assetId = "1234:example.mp4";
-      }).toThrowError(
-        new Error(
-          "EFMedia: asset-id must match <uuid>:<basename>. (like: 550e8400-e29b-41d4-a716-446655440000:example.mp4)",
-        ),
-      );
-    });
-
     test("determines fragmentIndexPath", () => {
       const id = v4();
       const element = document.createElement("test-media");
-      element.setAttribute("asset-id", `${id}:example.mp4`);
+      element.setAttribute("asset-id", id);
       expect(element.fragmentIndexPath()).toBe(
-        `https://editframe.dev/api/v1/isobmff_files/${id}:example.mp4/index`,
+        `https://editframe.dev/api/v1/isobmff_files/${id}/index`,
       );
     });
 
     test("determines fragmentTrackPath", () => {
       const id = v4();
       const element = document.createElement("test-media");
-      element.setAttribute("asset-id", `${id}:example.mp4`);
+      element.setAttribute("asset-id", id);
       expect(element.fragmentTrackPath("1")).toBe(
-        `https://editframe.dev/api/v1/isobmff_tracks/${id}:example.mp4/1`,
+        `https://editframe.dev/api/v1/isobmff_tracks/${id}/1`,
       );
     });
 
     test("honors apiHost in fragmentIndexPath", () => {
       const id = v4();
       const element = document.createElement("test-media");
-      element.setAttribute("asset-id", `${id}:example.mp4`);
+      element.setAttribute("asset-id", id);
       const preview = document.createElement("ef-preview");
       preview.appendChild(element);
       preview.apiHost = "test://";
       document.body.appendChild(preview);
       expect(element.fragmentIndexPath()).toBe(
-        `test:///api/v1/isobmff_files/${id}:example.mp4/index`,
+        `test:///api/v1/isobmff_files/${id}/index`,
       );
     });
 
     test("honors apiHost in fragmentTrackPath", () => {
       const id = v4();
       const element = document.createElement("test-media");
-      element.setAttribute("asset-id", `${id}:example.mp4`);
+      element.setAttribute("asset-id", id);
       const preview = document.createElement("ef-preview");
       preview.appendChild(element);
       preview.apiHost = "test://";
       document.body.appendChild(preview);
       expect(element.fragmentTrackPath("1")).toBe(
-        `test:///api/v1/isobmff_tracks/${id}:example.mp4/1`,
+        `test:///api/v1/isobmff_tracks/${id}/1`,
       );
     });
   });
@@ -302,6 +292,7 @@ describe("EFMedia", () => {
       expect(element.durationMs).toBe(4_000);
       expect(timegroup.durationMs).toBe(4_000);
     });
+
     test("Computes duration from track fragment index sourceout and sourcein", async () => {
       // Mock the request for the track fragment index, responds with a 10 second duration
       worker.use(