@editframe/elements 0.6.0-beta.17 → 0.6.0-beta.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@editframe/elements",
-  "version": "0.6.0-beta.17",
+  "version": "0.6.0-beta.19",
   "description": "",
   "exports": {
     ".": {
@@ -19,7 +19,7 @@
   "author": "",
   "license": "UNLICENSED",
   "dependencies": {
-    "@editframe/assets": "0.6.0-beta.17",
+    "@editframe/assets": "0.6.0-beta.19",
     "@lit/context": "^1.1.1",
     "@lit/task": "^1.0.0",
     "d3": "^7.9.0",

package/src/elements/EFMedia.ts

@@ -52,7 +52,7 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
       return `${this.src}/index`;
     }
-    return `/@ef-track-fragment-index/${this.getAttribute("src") ?? ""}`;
+    return `/@ef-track-fragment-index/${this.src ?? ""}`;
   }
 
   fragmentTrackPath(trackId: string) {

package/src/elements/EFTimegroup.ts

@@ -292,15 +292,14 @@ export class EFTimegroup extends EFTemporal(LitElement) {
     );
   }
 
-  async renderAudio(fromMs: number, toMs: number) {
+  async #addAudioToContext(
+    audioContext: AudioContext | OfflineAudioContext,
+    fromMs: number,
+    toMs: number,
+  ) {
     await this.waitForMediaDurations();
 
     const durationMs = toMs - fromMs;
-    const audioContext = new OfflineAudioContext(
-      2,
-      Math.round((48000 * durationMs) / 1000),
-      48000,
-    );
 
     await Promise.all(
       deepGetMediaElements(this).map(async (mediaElement) => {
@@ -338,7 +337,16 @@ export class EFTimegroup extends EFTemporal(LitElement) {
         );
       }),
     );
+  }
 
+  async renderAudio(fromMs: number, toMs: number) {
+    const durationMs = toMs - fromMs;
+    const audioContext = new OfflineAudioContext(
+      2,
+      Math.round((48000 * durationMs) / 1000),
+      48000,
+    );
+    await this.#addAudioToContext(audioContext, fromMs, toMs);
     return await audioContext.startRendering();
   }
 

package/src/gui/EFFilmstrip.ts

@@ -103,7 +103,7 @@ class FilmstripItem extends TWMixin(LitElement) {
           }
         }}
         ?data-focused=${this.isFocused}
-        class="border-outset relative mb-[1px] block h-[1.25rem] text-nowrap border border-slate-500 bg-blue-200 text-sm data-[focused]:bg-slate-400"
+        class="border-outset relative mb-[1px] block h-[1.1rem] text-nowrap border border-slate-500 bg-blue-200 text-sm data-[focused]:bg-slate-400"
       >
         ${this.animations()}
       </div>
@@ -112,7 +112,7 @@ class FilmstripItem extends TWMixin(LitElement) {
   }
 
   renderChildren(): Array<TemplateResult<1> | typeof nothing> | typeof nothing {
-    return renderFilmStripChildren(
+    return renderFilmstripChildren(
       Array.from(this.element.children),
       this.pixelsPerMs,
     );
@@ -228,7 +228,7 @@ export class EFTimegroupFilmstrip extends FilmstripItem {
   contents() {
     return html`
       <span>TIME GROUP</span>
-      ${renderFilmStripChildren(
+      ${renderFilmstripChildren(
         Array.from(this.element.children || []),
         this.pixelsPerMs,
       )}
@@ -242,7 +242,7 @@ export class EFHTMLFilmstrip extends FilmstripItem {
   contents() {
     return html`
       <span>${this.element.tagName}</span>
-      ${renderFilmStripChildren(
+      ${renderFilmstripChildren(
         Array.from(this.element.children || []),
         this.pixelsPerMs,
       )}
@@ -282,8 +282,8 @@ class EFHierarchyItem<
         <div
           ?data-focused=${this.isFocused}
           class="peer 
-            flex h-[1.25rem] items-center overflow-hidden text-nowrap  border border-slate-500
-          bg-slate-200 pl-2 text-sm hover:bg-slate-400 data-[focused]:bg-slate-400"
+            flex h-[1.1rem] items-center overflow-hidden text-nowrap  border border-slate-500
+          bg-slate-200 pl-2 text-xs font-mono hover:bg-slate-400 data-[focused]:bg-slate-400"
           @mouseenter=${() => {
             if (this.focusContext) {
               this.focusContext.focusedElement = this.element;
@@ -328,7 +328,7 @@ class EFAudioHierarchyItem extends EFHierarchyItem {
   }
 
   displayLabel() {
-    return this.element.getAttribute("src") ?? "(no src)";
+    return this.element.src ?? "(no src)";
   }
 }
 
@@ -339,7 +339,7 @@ class EFVideoHierarchyItem extends EFHierarchyItem {
   }
 
   displayLabel() {
-    return this.element.getAttribute("src") ?? "(no src)";
+    return this.element.src ?? "(no src)";
   }
 }
 
@@ -375,7 +375,7 @@ class EFImageHierarchyItem extends EFHierarchyItem {
   }
 
   displayLabel() {
-    return this.element.getAttribute("src") ?? "(no src)";
+    return this.element.src ?? "(no src)";
   }
 }
 
@@ -431,7 +431,7 @@ const renderHierarchyChildren = (
   });
 };
 
-const renderFilmStripChildren = (
+const renderFilmstripChildren = (
   children: Element[],
   pixelsPerMs: number,
 ): Array<TemplateResult<1> | typeof nothing> => {
@@ -481,14 +481,14 @@ const renderFilmStripChildren = (
 };
 
 @customElement("ef-filmstrip")
-export class EFFilmStrip extends TWMixin(LitElement) {
+export class EFFilmstrip extends TWMixin(LitElement) {
   @property({ type: Number })
   pixelsPerMs = 0.04;
 
   @property({ type: Number })
   currentTimeMs = 0;
 
-  @property({ type: String, attribute: "target" })
+  @property({ type: String, attribute: "target", reflect: true })
   targetSelector = "";
 
   @state()
@@ -504,15 +504,41 @@ export class EFFilmStrip extends TWMixin(LitElement) {
 
   connectedCallback(): void {
     super.connectedCallback();
-    const target = this.target;
+    const target = this.targetTimegroup;
     if (target) {
       this.timegroupController = new TimegroupController(target, this);
       // Set the current time to the last saved time to avoid a cycle
       // where the filmstrip clobbers the time loaded from localStorage
       this.currentTimeMs = target.currentTimeMs;
     }
+
+    window.addEventListener("keypress", this.#handleKeyPress);
+  }
+
+  disconnectedCallback(): void {
+    super.disconnectedCallback();
+    window.removeEventListener("keypress", this.#handleKeyPress);
   }
 
+  #handleKeyPress = (event: KeyboardEvent) => {
+    // On spacebar, toggle playback
+    if (event.key === " ") {
+      // CSS selector to match all interactive elements
+      const interactiveSelector =
+        "input, textarea, button, select, a, [contenteditable]";
+
+      // Check if the event target or its ancestor matches an interactive element
+      const closestInteractive = (event.target as HTMLElement | null)?.closest(
+        interactiveSelector,
+      );
+      if (closestInteractive) {
+        return;
+      }
+      event.preventDefault();
+      this.playing = !this.playing;
+    }
+  };
+
   @eventOptions({ passive: false })
   syncGutterScroll() {
     if (this.gutter && this.hierarchyRef.value) {
@@ -531,10 +557,96 @@ export class EFFilmStrip extends TWMixin(LitElement) {
 
   #lastTick?: DOMHighResTimeStamp;
 
+  #playbackAudioContext: AudioContext | null = null;
+  #playbackAnimationFrameRequest: number | null = null;
+  #AUDIO_PLAYBACK_SLICE_MS = 1000;
+
+  #syncPlayheadToAudioContext(target: EFTimegroup, startMs: number) {
+    target.currentTimeMs =
+      startMs + (this.#playbackAudioContext?.currentTime ?? 0) * 1000;
+    this.#playbackAnimationFrameRequest = requestAnimationFrame(() => {
+      this.#syncPlayheadToAudioContext(target, startMs);
+    });
+  }
+
+  async #stopPlayback() {
+    if (this.#playbackAudioContext) {
+      if (this.#playbackAudioContext.state !== "closed") {
+        await this.#playbackAudioContext.close();
+      }
+    }
+    if (this.#playbackAnimationFrameRequest) {
+      cancelAnimationFrame(this.#playbackAnimationFrameRequest);
+    }
+    this.#playbackAudioContext = null;
+  }
+
+  async #startPlayback() {
+    await this.#stopPlayback();
+    const timegroup = this.targetTimegroup;
+    if (!timegroup) {
+      return;
+    }
+
+    let currentMs = timegroup.currentTimeMs;
+    let bufferCount = 0;
+    this.#playbackAudioContext = new AudioContext({
+      latencyHint: "playback",
+    });
+    if (this.#playbackAnimationFrameRequest) {
+      cancelAnimationFrame(this.#playbackAnimationFrameRequest);
+    }
+    this.#syncPlayheadToAudioContext(timegroup, currentMs);
+    const playbackContext = this.#playbackAudioContext;
+    await playbackContext.suspend();
+
+    const fillBuffer = async () => {
+      if (bufferCount > 1) {
+        return;
+      }
+      const canFillBuffer = await queueBufferSource();
+      if (canFillBuffer) {
+        fillBuffer();
+      }
+    };
+
+    const fromMs = currentMs;
+    const toMs = timegroup.endTimeMs;
+
+    const queueBufferSource = async () => {
+      if (currentMs >= toMs) {
+        return false;
+      }
+      const startMs = currentMs;
+      const endMs = currentMs + this.#AUDIO_PLAYBACK_SLICE_MS;
+      currentMs += this.#AUDIO_PLAYBACK_SLICE_MS;
+      const audioBuffer = await timegroup.renderAudio(startMs, endMs);
+      bufferCount++;
+      const source = playbackContext.createBufferSource();
+      source.buffer = audioBuffer;
+      source.connect(playbackContext.destination);
+      source.start((startMs - fromMs) / 1000);
+      source.onended = () => {
+        bufferCount--;
+        if (endMs >= toMs) {
+          this.playing = false;
+        } else {
+          fillBuffer();
+        }
+      };
+      return true;
+    };
+
+    await fillBuffer();
+    await playbackContext.resume();
+  }
+
   advancePlayhead = (tick?: DOMHighResTimeStamp) => {
-    if (this.#lastTick && tick && this.target) {
-      this.target.currentTimeMs += tick - this.#lastTick;
-      if (this.target.currentTimeMs >= this.target.durationMs) {
+    if (this.#lastTick && tick && this.targetTimegroup) {
+      this.targetTimegroup.currentTimeMs += tick - this.#lastTick;
+      if (
+        this.targetTimegroup.currentTimeMs >= this.targetTimegroup.durationMs
+      ) {
         this.playing = false;
       }
     }
@@ -558,9 +670,9 @@ export class EFFilmStrip extends TWMixin(LitElement) {
       return;
     }
     const rect = gutter.getBoundingClientRect();
-    if (this.target) {
+    if (this.targetTimegroup) {
       const layerX = e.pageX - rect.left + gutter.scrollLeft;
-      this.target.currentTimeMs = layerX / this.pixelsPerMs;
+      this.targetTimegroup.currentTimeMs = layerX / this.pixelsPerMs;
     }
   }
 
@@ -576,9 +688,9 @@ export class EFFilmStrip extends TWMixin(LitElement) {
         return;
       }
       const rect = gutter.getBoundingClientRect();
-      if (this.target) {
+      if (this.targetTimegroup) {
         const layerX = e.pageX - rect.left + gutter.scrollLeft;
-        this.target.currentTimeMs = layerX / this.pixelsPerMs;
+        this.targetTimegroup.currentTimeMs = layerX / this.pixelsPerMs;
       }
     });
     addEventListener(
@@ -592,7 +704,7 @@ export class EFFilmStrip extends TWMixin(LitElement) {
 
   @eventOptions({ passive: false })
   scrollScrub(e: WheelEvent) {
-    if (this.target && this.gutter && !this.playing) {
+    if (this.targetTimegroup && this.gutter && !this.playing) {
       e.preventDefault();
       // Avoid over-scrolling to the left
       if (
@@ -616,7 +728,7 @@ export class EFFilmStrip extends TWMixin(LitElement) {
 
       if (this) {
         this.gutter.scrollBy(e.deltaX, e.deltaY);
-        this.target.currentTimeMs += e.deltaX / this.pixelsPerMs;
+        this.targetTimegroup.currentTimeMs += e.deltaX / this.pixelsPerMs;
       }
     }
   }
@@ -630,7 +742,7 @@ export class EFFilmStrip extends TWMixin(LitElement) {
   }
 
   render() {
-    const target = this.target;
+    const target = this.targetTimegroup;
 
     return html` <div
       class="grid h-full bg-slate-100"
@@ -702,7 +814,7 @@ export class EFFilmStrip extends TWMixin(LitElement) {
             ${ref(this.playheadRef)}
           ></div>
 
-          ${renderFilmStripChildren(
+          ${renderFilmstripChildren(
             Array.from(target?.children || []),
             this.pixelsPerMs,
           )}
@@ -714,24 +826,26 @@ export class EFFilmStrip extends TWMixin(LitElement) {
   update(changedProperties: Map<string | number | symbol, unknown>) {
     if (changedProperties.has("playing")) {
       if (this.playing) {
-        this.advancePlayhead(0);
+        this.#startPlayback();
+      } else {
+        this.#stopPlayback();
       }
     }
     super.update(changedProperties);
   }
 
   updated(changes: PropertyValueMap<any> | Map<PropertyKey, unknown>) {
-    if (!this.target) {
+    if (!this.targetTimegroup) {
       return;
     }
     if (changes.has("currentTimeMs")) {
-      if (this.target.currentTimeMs !== this.currentTimeMs) {
-        this.target.currentTimeMs = this.currentTimeMs;
+      if (this.targetTimegroup.currentTimeMs !== this.currentTimeMs) {
+        this.targetTimegroup.currentTimeMs = this.currentTimeMs;
       }
     }
   }
 
-  get target() {
+  get targetTimegroup() {
     if (this.getAttribute("target")) {
       const target = document.querySelector(this.getAttribute("target") ?? "");
       if (target instanceof EFTimegroup) {
@@ -746,7 +860,7 @@ export class EFFilmStrip extends TWMixin(LitElement) {
 
 declare global {
   interface HTMLElementTagNameMap {
-    "ef-filmstrip": EFFilmStrip;
+    "ef-filmstrip": EFFilmstrip;
     "ef-timegroup-hierarchy-item": EFTimegroupHierarchyItem;
     "ef-audio-hierarchy-item": EFAudioHierarchyItem;
     "ef-video-hierarchy-item": EFVideoHierarchyItem;