@editframe/elements 0.12.0-beta.2 → 0.12.0-beta.6

package/dist/assets/src/MP4File.js

@@ -40,13 +40,13 @@ class MP4File extends MP4Box.ISOFile {
     await this.readyPromise;
     const trackInfo = {};
     for (const videoTrack of this.getInfo().videoTracks) {
-      trackInfo[videoTrack.id] = { index: 0, complete: false };
+      trackInfo[videoTrack.id] = { index: 0 };
       this.setSegmentOptions(videoTrack.id, null, {
         rapAlignement: true
       });
     }
     for (const audioTrack of this.getInfo().audioTracks) {
-      trackInfo[audioTrack.id] = { index: 0, complete: false };
+      trackInfo[audioTrack.id] = { index: 0 };
       const sampleRate = audioTrack.audio.sample_rate;
       const probablePacketSize = 1024;
       const probableFourSecondsOfSamples = Math.ceil(
@@ -61,21 +61,12 @@ class MP4File extends MP4Box.ISOFile {
       yield {
         track: initSegment.id,
         segment: "init",
-        data: initSegment.buffer,
-        complete: false
+        data: initSegment.buffer
       };
     }
     const fragmentStartSamples = {};
     let finishedReading = false;
-    const allTracksFinished = () => {
-      for (const fragmentedTrack of this.fragmentedTracks) {
-        if (!trackInfo[fragmentedTrack.id]?.complete) {
-          return false;
-        }
-      }
-      return true;
-    };
-    while (!(finishedReading && allTracksFinished())) {
+    do {
       for (const fragTrak of this.fragmentedTracks) {
         const trak = fragTrak.trak;
         if (trak.nextSample === void 0) {
@@ -84,6 +75,8 @@ class MP4File extends MP4Box.ISOFile {
         if (trak.samples === void 0) {
           throw new Error("trak.samples is undefined");
         }
+        log("trak.nextSample", fragTrak.id, trak.nextSample);
+        log("trak.samples.length", fragTrak.id, trak.samples.length);
         while (trak.nextSample < trak.samples.length) {
           let result = void 0;
           const fragTrakNextSample = trak.samples[trak.nextSample];
@@ -120,9 +113,6 @@ class MP4File extends MP4Box.ISOFile {
             if (!trackInfoForFrag) {
               throw new Error("trackInfoForFrag is undefined");
             }
-            if (trak.nextSample >= trak.samples.length) {
-              trackInfoForFrag.complete = true;
-            }
             const startSample = fragmentStartSamples[fragTrak.id];
             const endSample = trak.samples[trak.nextSample - 1];
             if (!startSample || !endSample) {
@@ -137,7 +127,6 @@ class MP4File extends MP4Box.ISOFile {
               track: fragTrak.id,
               segment: trackInfoForFrag.index,
               data: fragTrak.segmentStream.buffer,
-              complete: trackInfoForFrag.complete,
               cts: startSample.cts,
               dts: startSample.dts,
               duration: endSample.cts - startSample.cts + endSample.duration
@@ -149,6 +138,68 @@ class MP4File extends MP4Box.ISOFile {
         }
       }
       finishedReading = await this.waitForMoreSamples();
+    } while (!finishedReading);
+    for (const fragTrak of this.fragmentedTracks) {
+      const trak = fragTrak.trak;
+      if (trak.nextSample === void 0) {
+        throw new Error("trak.nextSample is undefined");
+      }
+      if (trak.samples === void 0) {
+        throw new Error("trak.samples is undefined");
+      }
+      while (trak.nextSample < trak.samples.length) {
+        let result = void 0;
+        try {
+          result = this.createFragment(
+            fragTrak.id,
+            trak.nextSample,
+            fragTrak.segmentStream
+          );
+        } catch (error) {
+          console.error("Failed to createFragment", error);
+        }
+        if (result) {
+          fragTrak.segmentStream = result;
+          trak.nextSample++;
+        } else {
+          finishedReading = await this.waitForMoreSamples();
+          break;
+        }
+        const nextSample = trak.samples[trak.nextSample];
+        const emitSegment = (
+          // if rapAlignement is true, we emit a fragment when we have a rap sample coming up next
+          fragTrak.rapAlignement === true && nextSample?.is_sync || // if rapAlignement is false, we emit a fragment when we have the required number of samples
+          !fragTrak.rapAlignement && trak.nextSample % fragTrak.nb_samples === 0 || // if we have more samples than the number of samples requested, we emit the fragment
+          trak.nextSample >= trak.samples.length
+        );
+        if (emitSegment) {
+          const trackInfoForFrag = trackInfo[fragTrak.id];
+          if (!trackInfoForFrag) {
+            throw new Error("trackInfoForFrag is undefined");
+          }
+          const startSample = fragmentStartSamples[fragTrak.id];
+          const endSample = trak.samples[trak.nextSample - 1];
+          if (!startSample || !endSample) {
+            throw new Error("startSample or endSample is undefined");
+          }
+          log(
+            `Yielding fragment #${trackInfoForFrag.index} for track=${fragTrak.id}`,
+            `startTime=${startSample.cts}`,
+            `endTime=${endSample.cts + endSample.duration}`
+          );
+          yield {
+            track: fragTrak.id,
+            segment: trackInfoForFrag.index,
+            data: fragTrak.segmentStream.buffer,
+            cts: startSample.cts,
+            dts: startSample.dts,
+            duration: endSample.cts - startSample.cts + endSample.duration
+          };
+          trackInfoForFrag.index += 1;
+          fragTrak.segmentStream = null;
+          delete fragmentStartSamples[fragTrak.id];
+        }
+      }
     }
   }
   waitForMoreSamples() {

package/dist/elements/EFMedia.d.ts

@@ -47,7 +47,6 @@ export declare class EFMedia extends EFMedia_base {
     protected updated(changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>): void;
     get hasOwnDuration(): boolean;
     get durationMs(): number;
-    get startTimeMs(): number;
     audioBufferTask: Task<readonly [Record<string, File> | undefined, Record<string, {
         segment: TrackSegment;
         track: MP4Box.TrackInfo;

package/dist/elements/src/elements/EFMedia.js

@@ -10,9 +10,8 @@ import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
 import { EF_RENDERING } from "../EF_RENDERING.js";
 import { apiHostContext } from "../gui/apiHostContext.js";
 import { EFSourceMixin } from "./EFSourceMixin.js";
-import { EFTemporal } from "./EFTemporal.js";
+import { EFTemporal, isEFTemporal } from "./EFTemporal.js";
 import { FetchMixin } from "./FetchMixin.js";
-import { getStartTimeMs } from "./util.js";
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __decorateClass = (decorators, target, key, kind) => {
@@ -62,10 +61,10 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
         return await Promise.all(
           Object.entries(fragmentIndex).map(async ([trackId, track]) => {
             const start = track.initSegment.offset;
-            const end = track.initSegment.offset + track.initSegment.size - 1;
+            const end = track.initSegment.offset + track.initSegment.size;
             const response = await fetch(this.fragmentTrackPath(trackId), {
               signal,
-              headers: { Range: `bytes=${start}-${end}` }
+              headers: { Range: `bytes=${start}-${end - 1}` }
             });
             const buffer = await response.arrayBuffer();
             buffer.fileStart = 0;
@@ -131,14 +130,14 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
           const end = segment.offset + segment.size;
           const response = await fetch(this.fragmentTrackPath(trackId), {
             signal,
-            headers: { Range: `bytes=${start}-${end}` }
+            headers: { Range: `bytes=${start}-${end - 1}` }
           });
           if (nextSegment) {
             const nextStart = nextSegment.offset;
             const nextEnd = nextSegment.offset + nextSegment.size;
             fetch(this.fragmentTrackPath(trackId), {
               signal,
-              headers: { Range: `bytes=${nextStart}-${nextEnd}` }
+              headers: { Range: `bytes=${nextStart}-${nextEnd - 1}` }
             }).then(() => {
               log("Prefetched next segment");
             }).catch((error) => {
@@ -270,6 +269,67 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     if (changedProperties.has("ownCurrentTimeMs")) {
       this.executeSeek(this.trimAdjustedOwnCurrentTimeMs);
     }
+    if (changedProperties.has("currentTime") || changedProperties.has("ownCurrentTimeMs")) {
+      const timelineTimeMs = (this.rootTimegroup ?? this).currentTimeMs;
+      if (this.startTimeMs > timelineTimeMs || this.endTimeMs < timelineTimeMs) {
+        this.style.display = "none";
+        return;
+      }
+      this.style.display = "";
+      const animations = this.getAnimations({ subtree: true });
+      this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
+      this.style.setProperty(
+        "--ef-transition--duration",
+        `${this.parentTimegroup?.overlapMs ?? 0}ms`
+      );
+      this.style.setProperty(
+        "--ef-transition-out-start",
+        `${this.durationMs - (this.parentTimegroup?.overlapMs ?? 0)}ms`
+      );
+      for (const animation of animations) {
+        if (animation.playState === "running") {
+          animation.pause();
+        }
+        const effect = animation.effect;
+        if (!(effect && effect instanceof KeyframeEffect)) {
+          return;
+        }
+        const target = effect.target;
+        if (!target) {
+          return;
+        }
+        if (target.closest("ef-video, ef-audio") !== this) {
+          return;
+        }
+        if (isEFTemporal(target)) {
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(target.ownCurrentTimeMs, duration - 1 + delay)
+          );
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        } else if (target) {
+          const nearestTimegroup = target.closest("ef-timegroup");
+          if (!nearestTimegroup) {
+            return;
+          }
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(nearestTimegroup.ownCurrentTimeMs, duration - 1 + delay)
+          );
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        }
+      }
+    }
   }
   get hasOwnDuration() {
     return true;
@@ -300,9 +360,6 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     }
     return Math.max(...durations) - this.trimStartMs - this.trimEndMs;
   }
-  get startTimeMs() {
-    return getStartTimeMs(this);
-  }
   #audioContext;
   async fetchAudioSpanningTime(fromMs, toMs) {
     if (this.sourceInMs) {
@@ -325,11 +382,11 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
       return;
     }
     const start = audioTrackIndex.initSegment.offset;
-    const end = audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size - 1;
+    const end = audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size;
     const audioInitFragmentRequest = this.fetch(
       this.fragmentTrackPath(String(audioTrackId)),
       {
-        headers: { Range: `bytes=${start}-${end}` }
+        headers: { Range: `bytes=${start}-${end - 1}` }
       }
     );
     const fragments = Object.values(audioTrackIndex.segments).filter(
@@ -350,11 +407,11 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
       return;
     }
     const fragmentStart = firstFragment.offset;
-    const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
+    const fragmentEnd = lastFragment.offset + lastFragment.size;
     const audioFragmentRequest = this.fetch(
       this.fragmentTrackPath(String(audioTrackId)),
       {
-        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd}` }
+        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd - 1}` }
       }
     );
     const initResponse = await audioInitFragmentRequest;

package/dist/elements/src/gui/ContextMixin.js

@@ -83,14 +83,14 @@ function ContextMixin(superClass) {
               const scale = stageHeight / canvasHeight;
               if (this.stageScale !== scale) {
                 canvasElement.style.transform = `scale(${scale})`;
-                canvasElement.style.transformOrigin = "top";
+                canvasElement.style.transformOrigin = "center left";
               }
               this.stageScale = scale;
             } else {
               const scale = stageWidth / canvasWidth;
               if (this.stageScale !== scale) {
                 canvasElement.style.transform = `scale(${scale})`;
-                canvasElement.style.transformOrigin = "top";
+                canvasElement.style.transformOrigin = "center left";
               }
               this.stageScale = scale;
             }

package/dist/elements/src/gui/EFPreview.js

@@ -1,8 +1,8 @@
 import { html, css, LitElement } from "lit";
 import { customElement } from "lit/decorators.js";
 import { ref } from "lit/directives/ref.js";
-import { TWMixin } from "./TWMixin.js";
 import { ContextMixin } from "./ContextMixin.js";
+import { TWMixin } from "./TWMixin.js";
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __decorateClass = (decorators, target, key, kind) => {
@@ -23,7 +23,9 @@ let EFPreview = class extends ContextMixin(TWMixin(LitElement)) {
         <slot
           ${ref(this.canvasRef)}
           class="inline-block"
+          name="canvas"
         ></slot>
+        <slot name="controls"></slot>
       </div>
     `;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@editframe/elements",
-  "version": "0.12.0-beta.2",
+  "version": "0.12.0-beta.6",
   "description": "",
   "exports": {
     ".": {
@@ -20,7 +20,7 @@
   "author": "",
   "license": "UNLICENSED",
   "dependencies": {
-    "@editframe/assets": "0.12.0-beta.2",
+    "@editframe/assets": "0.12.0-beta.6",
     "@lit/context": "^1.1.2",
     "@lit/task": "^1.0.1",
     "d3": "^7.9.0",

package/src/elements/EFMedia.browsertest.ts

@@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, test } from "vitest";
 import { EFMedia } from "./EFMedia.ts";
 import "../gui/EFWorkbench.ts";
 import "../gui/EFPreview.ts";
+import "./EFTimegroup.ts";
 import { createTestFragmentIndex } from "TEST/createTestFragmentIndex.ts";
 import { useMockWorker } from "TEST/useMockWorker.ts";
 import { http, HttpResponse } from "msw";
@@ -291,6 +292,7 @@ describe("EFMedia", () => {
       expect(element.durationMs).toBe(4_000);
       expect(timegroup.durationMs).toBe(4_000);
     });
+
     test("Computes duration from track fragment index sourceout and sourcein", async () => {
       // Mock the request for the track fragment index, responds with a 10 second duration
       worker.use(

package/src/elements/EFMedia.ts

@@ -14,9 +14,8 @@ import { EF_INTERACTIVE } from "../EF_INTERACTIVE.ts";
 import { EF_RENDERING } from "../EF_RENDERING.ts";
 import { apiHostContext } from "../gui/apiHostContext.ts";
 import { EFSourceMixin } from "./EFSourceMixin.ts";
-import { EFTemporal } from "./EFTemporal.ts";
+import { EFTemporal, isEFTemporal } from "./EFTemporal.ts";
 import { FetchMixin } from "./FetchMixin.ts";
-import { getStartTimeMs } from "./util.ts";
 
 const log = debug("ef:elements:EFMedia");
 
@@ -110,10 +109,10 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
       return await Promise.all(
         Object.entries(fragmentIndex).map(async ([trackId, track]) => {
           const start = track.initSegment.offset;
-          const end = track.initSegment.offset + track.initSegment.size - 1;
+          const end = track.initSegment.offset + track.initSegment.size;
           const response = await fetch(this.fragmentTrackPath(trackId), {
             signal,
-            headers: { Range: `bytes=${start}-${end}` },
+            headers: { Range: `bytes=${start}-${end - 1}` },
           });
           const buffer =
             (await response.arrayBuffer()) as MP4Box.MP4ArrayBuffer;
@@ -220,7 +219,7 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
 
         const response = await fetch(this.fragmentTrackPath(trackId), {
           signal,
-          headers: { Range: `bytes=${start}-${end}` },
+          headers: { Range: `bytes=${start}-${end - 1}` },
         });
 
         if (nextSegment) {
@@ -228,7 +227,7 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
           const nextEnd = nextSegment.offset + nextSegment.size;
           fetch(this.fragmentTrackPath(trackId), {
             signal,
-            headers: { Range: `bytes=${nextStart}-${nextEnd}` },
+            headers: { Range: `bytes=${nextStart}-${nextEnd - 1}` },
           })
             .then(() => {
               log("Prefetched next segment");
@@ -297,6 +296,83 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     if (changedProperties.has("ownCurrentTimeMs")) {
       this.executeSeek(this.trimAdjustedOwnCurrentTimeMs);
     }
+    // TODO: this is copied straight from EFTimegroup.ts
+    // and should be refactored to be shared/reduce bad duplication of
+    // critical logic.
+    if (
+      changedProperties.has("currentTime") ||
+      changedProperties.has("ownCurrentTimeMs")
+    ) {
+      const timelineTimeMs = (this.rootTimegroup ?? this).currentTimeMs;
+      if (
+        this.startTimeMs > timelineTimeMs ||
+        this.endTimeMs < timelineTimeMs
+      ) {
+        this.style.display = "none";
+        return;
+      }
+      this.style.display = "";
+
+      const animations = this.getAnimations({ subtree: true });
+      this.style.setProperty("--ef-duration", `${this.durationMs}ms`);
+      this.style.setProperty(
+        "--ef-transition--duration",
+        `${this.parentTimegroup?.overlapMs ?? 0}ms`,
+      );
+      this.style.setProperty(
+        "--ef-transition-out-start",
+        `${this.durationMs - (this.parentTimegroup?.overlapMs ?? 0)}ms`,
+      );
+
+      for (const animation of animations) {
+        if (animation.playState === "running") {
+          animation.pause();
+        }
+        const effect = animation.effect;
+        if (!(effect && effect instanceof KeyframeEffect)) {
+          return;
+        }
+        const target = effect.target;
+        // TODO: better generalize work avoidance for temporal elements
+        if (!target) {
+          return;
+        }
+        if (target.closest("ef-video, ef-audio") !== this) {
+          return;
+        }
+
+        // Important to avoid going to the end of the animation
+        // or it will reset awkwardly.
+        if (isEFTemporal(target)) {
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(target.ownCurrentTimeMs, duration - 1 + delay),
+          );
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        } else if (target) {
+          const nearestTimegroup = target.closest("ef-timegroup");
+          if (!nearestTimegroup) {
+            return;
+          }
+          const timing = effect.getTiming();
+          const duration = Number(timing.duration) ?? 0;
+          const delay = Number(timing.delay);
+          const newTime = Math.floor(
+            Math.min(nearestTimegroup.ownCurrentTimeMs, duration - 1 + delay),
+          );
+
+          if (Number.isNaN(newTime)) {
+            return;
+          }
+          animation.currentTime = newTime;
+        }
+      }
+    }
   }
 
   get hasOwnDuration() {
@@ -351,10 +427,6 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     return Math.max(...durations) - this.trimStartMs - this.trimEndMs;
   }
 
-  get startTimeMs() {
-    return getStartTimeMs(this);
-  }
-
   #audioContext = new OfflineAudioContext(2, 48000 / 30, 48000);
 
   audioBufferTask = new Task(this, {
@@ -413,11 +485,11 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
 
     const start = audioTrackIndex.initSegment.offset;
     const end =
-      audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size - 1;
+      audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size;
     const audioInitFragmentRequest = this.fetch(
       this.fragmentTrackPath(String(audioTrackId)),
       {
-        headers: { Range: `bytes=${start}-${end}` },
+        headers: { Range: `bytes=${start}-${end - 1}` },
       },
     );
 
@@ -443,12 +515,12 @@ export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
       return;
     }
     const fragmentStart = firstFragment.offset;
-    const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
+    const fragmentEnd = lastFragment.offset + lastFragment.size;
 
     const audioFragmentRequest = this.fetch(
       this.fragmentTrackPath(String(audioTrackId)),
       {
-        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd}` },
+        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd - 1}` },
       },
     );
 

package/src/gui/ContextMixin.ts

@@ -136,18 +136,19 @@ export function ContextMixin<T extends Constructor<LitElement>>(superClass: T) {
           const canvasHeight = canvasElement.clientHeight;
           const stageRatio = stageWidth / stageHeight;
           const canvasRatio = canvasWidth / canvasHeight;
+
           if (stageRatio > canvasRatio) {
             const scale = stageHeight / canvasHeight;
             if (this.stageScale !== scale) {
               canvasElement.style.transform = `scale(${scale})`;
-              canvasElement.style.transformOrigin = "top";
+              canvasElement.style.transformOrigin = "center left";
             }
             this.stageScale = scale;
           } else {
             const scale = stageWidth / canvasWidth;
             if (this.stageScale !== scale) {
               canvasElement.style.transform = `scale(${scale})`;
-              canvasElement.style.transformOrigin = "top";
+              canvasElement.style.transformOrigin = "center left";
             }
             this.stageScale = scale;
           }

package/src/gui/EFPreview.ts

@@ -1,9 +1,9 @@
-import { LitElement, html, css } from "lit";
+import { LitElement, css, html } from "lit";
 import { customElement } from "lit/decorators.js";
 import { ref } from "lit/directives/ref.js";
 
-import { TWMixin } from "./TWMixin.ts";
 import { ContextMixin } from "./ContextMixin.ts";
+import { TWMixin } from "./TWMixin.ts";
 
 @customElement("ef-preview")
 export class EFPreview extends ContextMixin(TWMixin(LitElement)) {
@@ -26,7 +26,9 @@ export class EFPreview extends ContextMixin(TWMixin(LitElement)) {
         <slot
           ${ref(this.canvasRef)}
           class="inline-block"
+          name="canvas"
         ></slot>
+        <slot name="controls"></slot>
       </div>
     `;
   }

package/dist/elements/src/elements/util.js

@@ -1,11 +0,0 @@
-import { EFTimegroup } from "./EFTimegroup.js";
-const getStartTimeMs = (element) => {
-  const nearestTimeGroup = element.closest("ef-timegroup");
-  if (!(nearestTimeGroup instanceof EFTimegroup)) {
-    return 0;
-  }
-  return nearestTimeGroup.startTimeMs;
-};
-export {
-  getStartTimeMs
-};