@editframe/elements 0.7.0-beta.9 → 0.8.0-beta.10

package/dist/elements/src/elements/EFMedia.js

@@ -0,0 +1,356 @@
+import { css, LitElement } from "lit";
+import { property, state } from "lit/decorators.js";
+import { deepArrayEquals } from "@lit/task/deep-equals.js";
+import { Task } from "@lit/task";
+import { consume } from "@lit/context";
+import debug from "debug";
+import { MP4File } from "../../../assets/dist/MP4File.js";
+import { VideoAsset } from "../../../assets/dist/EncodedAsset.js";
+import { EFTemporal } from "./EFTemporal.js";
+import { FetchMixin } from "./FetchMixin.js";
+import { EFSourceMixin } from "./EFSourceMixin.js";
+import { getStartTimeMs } from "./util.js";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
+import { apiHostContext } from "../gui/apiHostContext.js";
+var __defProp = Object.defineProperty;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = void 0;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = decorator(target, key, result) || result;
+  if (result) __defProp(target, key, result);
+  return result;
+};
+const log = debug("ef:elements:EFMedia");
+const deepGetMediaElements = (element, medias = []) => {
+  for (const child of Array.from(element.children)) {
+    if (child instanceof EFMedia) {
+      medias.push(child);
+    } else {
+      deepGetMediaElements(child, medias);
+    }
+  }
+  return medias;
+};
+class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
+  assetType: "isobmff_files"
+}) {
+  constructor() {
+    super(...arguments);
+    this.currentTimeMs = 0;
+    this.trackFragmentIndexLoader = new Task(this, {
+      args: () => [this.fragmentIndexPath(), this.fetch],
+      task: async ([fragmentIndexPath, fetch], { signal }) => {
+        if (this.src === "") {
+          return;
+        }
+        const response = await fetch(fragmentIndexPath, { signal });
+        return await response.json();
+      },
+      onComplete: () => {
+        this.requestUpdate("ownCurrentTimeMs");
+        this.rootTimegroup?.requestUpdate("ownCurrentTimeMs");
+      }
+    });
+    this.initSegmentsLoader = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.trackFragmentIndexLoader.value, this.src, this.fetch],
+      task: async ([fragmentIndex, _src, fetch], { signal }) => {
+        if (!fragmentIndex) {
+          return;
+        }
+        return await Promise.all(
+          Object.entries(fragmentIndex).map(async ([trackId, track]) => {
+            const start = track.initSegment.offset;
+            const end = track.initSegment.offset + track.initSegment.size - 1;
+            const response = await fetch(this.fragmentTrackPath(trackId), {
+              signal,
+              headers: { Range: `bytes=${start}-${end}` }
+            });
+            const buffer = await response.arrayBuffer();
+            buffer.fileStart = 0;
+            const mp4File = new MP4File();
+            mp4File.appendBuffer(buffer, true);
+            mp4File.flush();
+            await mp4File.readyPromise;
+            return { trackId, buffer, mp4File };
+          })
+        );
+      }
+    });
+    this.seekTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [
+        this.desiredSeekTimeMs,
+        this.trackFragmentIndexLoader.value,
+        this.initSegmentsLoader.value
+      ],
+      task: async ([seekToMs, fragmentIndex, initSegments], { signal: _signal }) => {
+        if (fragmentIndex === void 0) {
+          return;
+        }
+        if (initSegments === void 0) {
+          return;
+        }
+        const result = {};
+        for (const index of Object.values(fragmentIndex)) {
+          const track = initSegments.find((segment2) => segment2.trackId === String(index.track))?.mp4File.getInfo().tracks[0];
+          if (!track) {
+            throw new Error("Could not finding matching track");
+          }
+          const segment = index.segments.toReversed().find((segment2) => {
+            return segment2.dts / track.timescale * 1e3 <= seekToMs;
+          });
+          const nextSegment = index.segments.find((segment2) => {
+            return segment2.dts / track.timescale * 1e3 > seekToMs;
+          });
+          if (!segment) {
+            return;
+          }
+          result[index.track] = { segment, track, nextSegment };
+        }
+        return result;
+      }
+    });
+    this.fetchSeekTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      argsEqual: deepArrayEquals,
+      args: () => [this.initSegmentsLoader.value, this.seekTask.value, this.fetch],
+      task: async ([initSegments, seekResult, fetch], { signal }) => {
+        if (!initSegments) {
+          return;
+        }
+        if (!seekResult) {
+          return;
+        }
+        const files = {};
+        for (const [trackId, { segment, track, nextSegment }] of Object.entries(
+          seekResult
+        )) {
+          const start = segment.offset;
+          const end = segment.offset + segment.size;
+          const response = await fetch(this.fragmentTrackPath(trackId), {
+            signal,
+            headers: { Range: `bytes=${start}-${end}` }
+          });
+          if (nextSegment) {
+            const nextStart = nextSegment.offset;
+            const nextEnd = nextSegment.offset + nextSegment.size;
+            fetch(this.fragmentTrackPath(trackId), {
+              signal,
+              headers: { Range: `bytes=${nextStart}-${nextEnd}` }
+            }).then(() => {
+              log("Prefetched next segment");
+            }).catch((error) => {
+              log("Failed to prefetch next segment", error);
+            });
+          }
+          const initSegment = Object.values(initSegments).find(
+            (initSegment2) => initSegment2.trackId === String(track.id)
+          );
+          if (!initSegment) {
+            throw new Error("Could not find matching init segment");
+          }
+          const initBuffer = initSegment.buffer;
+          const mediaBuffer = await response.arrayBuffer();
+          files[trackId] = new File([initBuffer, mediaBuffer], "video.mp4", {
+            type: "video/mp4"
+          });
+        }
+        return files;
+      }
+    });
+    this.videoAssetTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.fetchSeekTask.value],
+      task: async ([files], { signal: _signal }) => {
+        if (!files) {
+          return;
+        }
+        if (!this.defaultVideoTrackId) {
+          return;
+        }
+        const videoFile = files[this.defaultVideoTrackId];
+        if (!videoFile) {
+          return;
+        }
+        for (const frame of this.videoAssetTask.value?.decodedFrames || []) {
+          frame.close();
+        }
+        this.videoAssetTask.value?.videoDecoder?.close();
+        return await VideoAsset.createFromReadableStream(
+          "video.mp4",
+          videoFile.stream(),
+          videoFile
+        );
+      }
+    });
+    this.desiredSeekTimeMs = 0;
+    this.#audioContext = new OfflineAudioContext(2, 48e3 / 30, 48e3);
+    this.audioBufferTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.fetchSeekTask.value, this.seekTask.value],
+      task: async ([files, segments], { signal: _signal }) => {
+        if (!files) {
+          return;
+        }
+        if (!segments) {
+          return;
+        }
+        if (!this.defaultAudioTrackId) {
+          return;
+        }
+        const segment = segments[this.defaultAudioTrackId];
+        if (!segment) {
+          return;
+        }
+        const audioFile = files[this.defaultAudioTrackId];
+        if (!audioFile) {
+          return;
+        }
+        return {
+          buffer: await this.#audioContext.decodeAudioData(
+            await audioFile.arrayBuffer()
+          ),
+          startOffsetMs: segment.segment.cts / segment.track.timescale * 1e3
+        };
+      }
+    });
+  }
+  static {
+    this.styles = [
+      css`
+      :host {
+        display: block;
+        position: relative;
+        overflow: hidden;
+      }
+    `
+    ];
+  }
+  fragmentIndexPath() {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return `${this.src}/index`;
+    }
+    return `/@ef-track-fragment-index/${this.src ?? ""}`;
+  }
+  fragmentTrackPath(trackId) {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return `${this.src.replace("files", "tracks")}/${trackId}`;
+    }
+    return `/@ef-track/${this.src ?? ""}?trackId=${trackId}`;
+  }
+  get defaultVideoTrackId() {
+    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
+      (track) => track.type === "video"
+    )?.track;
+  }
+  get defaultAudioTrackId() {
+    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
+      (track) => track.type === "audio"
+    )?.track;
+  }
+  async executeSeek(seekToMs) {
+    this.desiredSeekTimeMs = seekToMs;
+  }
+  updated(changedProperties) {
+    if (changedProperties.has("ownCurrentTimeMs")) {
+      this.executeSeek(this.trimAdjustedOwnCurrentTimeMs);
+    }
+  }
+  get hasOwnDuration() {
+    return true;
+  }
+  get durationMs() {
+    if (!this.trackFragmentIndexLoader.value) {
+      return 0;
+    }
+    const durations = Object.values(this.trackFragmentIndexLoader.value).map(
+      (track) => {
+        return track.duration / track.timescale * 1e3;
+      }
+    );
+    if (durations.length === 0) {
+      return 0;
+    }
+    return Math.max(...durations) - this.trimStartMs - this.trimEndMs;
+  }
+  get startTimeMs() {
+    return getStartTimeMs(this);
+  }
+  #audioContext;
+  async fetchAudioSpanningTime(fromMs, toMs) {
+    fromMs -= this.startTimeMs - this.trimStartMs;
+    toMs -= this.startTimeMs - this.trimStartMs;
+    await this.trackFragmentIndexLoader.taskComplete;
+    const audioTrackId = this.defaultAudioTrackId;
+    if (!audioTrackId) {
+      log("No audio track found");
+      return;
+    }
+    const audioTrackIndex = this.trackFragmentIndexLoader.value?.[audioTrackId];
+    if (!audioTrackIndex) {
+      log("No audio track found");
+      return;
+    }
+    const start = audioTrackIndex.initSegment.offset;
+    const end = audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size - 1;
+    const audioInitFragmentRequest = this.fetch(
+      this.fragmentTrackPath(String(audioTrackId)),
+      {
+        headers: { Range: `bytes=${start}-${end}` }
+      }
+    );
+    const fragments = Object.values(audioTrackIndex.segments).filter(
+      (segment) => {
+        const segmentStartsBeforeEnd = segment.dts <= toMs * audioTrackIndex.timescale / 1e3;
+        const segmentEndsAfterStart = segment.dts + segment.duration >= fromMs * audioTrackIndex.timescale / 1e3;
+        return segmentStartsBeforeEnd && segmentEndsAfterStart;
+      }
+    );
+    const firstFragment = fragments[0];
+    if (!firstFragment) {
+      log("No audio fragments found");
+      return;
+    }
+    const lastFragment = fragments[fragments.length - 1];
+    if (!lastFragment) {
+      log("No audio fragments found");
+      return;
+    }
+    const fragmentStart = firstFragment.offset;
+    const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
+    const audioFragmentRequest = this.fetch(
+      this.fragmentTrackPath(String(audioTrackId)),
+      {
+        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd}` }
+      }
+    );
+    const initResponse = await audioInitFragmentRequest;
+    const dataResponse = await audioFragmentRequest;
+    const initBuffer = await initResponse.arrayBuffer();
+    const dataBuffer = await dataResponse.arrayBuffer();
+    const audioBlob = new Blob([initBuffer, dataBuffer], {
+      type: "audio/mp4"
+    });
+    return {
+      blob: audioBlob,
+      startMs: firstFragment.dts / audioTrackIndex.timescale * 1e3 - this.trimStartMs,
+      endMs: lastFragment.dts / audioTrackIndex.timescale * 1e3 + lastFragment.duration / audioTrackIndex.timescale * 1e3 - this.trimEndMs
+    };
+  }
+}
+__decorateClass([
+  property({ type: Number })
+], EFMedia.prototype, "currentTimeMs");
+__decorateClass([
+  consume({ context: apiHostContext, subscribe: true }),
+  state()
+], EFMedia.prototype, "efHost");
+__decorateClass([
+  state()
+], EFMedia.prototype, "desiredSeekTimeMs");
+export {
+  EFMedia,
+  deepGetMediaElements
+};

package/dist/elements/src/elements/EFSourceMixin.js

@@ -0,0 +1,55 @@
+import { consume } from "@lit/context";
+import { state } from "lit/decorators/state.js";
+import { Task } from "@lit/task";
+import { property } from "lit/decorators/property.js";
+import { apiHostContext } from "../gui/apiHostContext.js";
+var __defProp = Object.defineProperty;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = void 0;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = decorator(target, key, result) || result;
+  if (result) __defProp(target, key, result);
+  return result;
+};
+function EFSourceMixin(superClass, options) {
+  class EFSourceElement extends superClass {
+    constructor() {
+      super(...arguments);
+      this.src = "";
+      this.md5SumLoader = new Task(this, {
+        autoRun: false,
+        args: () => [this.src],
+        task: async ([src], { signal }) => {
+          const md5Path = `/@ef-asset/${src}`;
+          const response = await fetch(md5Path, { method: "HEAD", signal });
+          return response.headers.get("etag") ?? void 0;
+        }
+      });
+    }
+    productionSrc() {
+      if (!this.md5SumLoader.value) {
+        throw new Error(
+          `MD5 sum not available for ${this}. Cannot generate production URL`
+        );
+      }
+      if (!this.efHost) {
+        throw new Error(
+          `efHost not available for ${this}. Cannot generate production URL`
+        );
+      }
+      return `${this.efHost}/api/video2/${options.assetType}/${this.md5SumLoader.value}`;
+    }
+  }
+  __decorateClass([
+    consume({ context: apiHostContext, subscribe: true }),
+    state()
+  ], EFSourceElement.prototype, "efHost");
+  __decorateClass([
+    property({ type: String })
+  ], EFSourceElement.prototype, "src");
+  return EFSourceElement;
+}
+export {
+  EFSourceMixin
+};

package/dist/elements/src/elements/EFTemporal.js

@@ -0,0 +1,283 @@
+import { createContext, consume } from "@lit/context";
+import { property, state } from "lit/decorators.js";
+import { durationConverter } from "./durationConverter.js";
+import { Task } from "@lit/task";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result) __defProp(target, key, result);
+  return result;
+};
+const timegroupContext = createContext(
+  Symbol("timeGroupContext")
+);
+const isEFTemporal = (obj) => obj[EF_TEMPORAL];
+const EF_TEMPORAL = Symbol("EF_TEMPORAL");
+const deepGetTemporalElements = (element, temporals = []) => {
+  for (const child of element.children) {
+    if (isEFTemporal(child)) {
+      temporals.push(child);
+    }
+    deepGetTemporalElements(child, temporals);
+  }
+  return temporals;
+};
+const deepGetElementsWithFrameTasks = (element, elements = []) => {
+  for (const child of element.children) {
+    if ("frameTask" in child && child.frameTask instanceof Task) {
+      elements.push(child);
+    }
+    deepGetElementsWithFrameTasks(child, elements);
+  }
+  return elements;
+};
+let temporalCache;
+const resetTemporalCache = () => {
+  temporalCache = /* @__PURE__ */ new Map();
+  if (typeof requestAnimationFrame !== "undefined") {
+    requestAnimationFrame(resetTemporalCache);
+  }
+};
+resetTemporalCache();
+const shallowGetTemporalElements = (element, temporals = []) => {
+  const cachedResult = temporalCache.get(element);
+  if (cachedResult) {
+    return cachedResult;
+  }
+  for (const child of element.children) {
+    if (isEFTemporal(child)) {
+      temporals.push(child);
+    } else {
+      shallowGetTemporalElements(child, temporals);
+    }
+  }
+  temporalCache.set(element, temporals);
+  return temporals;
+};
+class OwnCurrentTimeController {
+  constructor(host, temporal) {
+    this.host = host;
+    this.temporal = temporal;
+    host.addController(this);
+  }
+  hostUpdated() {
+    this.temporal.requestUpdate("ownCurrentTimeMs");
+  }
+  remove() {
+    this.host.removeController(this);
+  }
+}
+let startTimeMsCache = /* @__PURE__ */ new WeakMap();
+const resetStartTimeMsCache = () => {
+  startTimeMsCache = /* @__PURE__ */ new WeakMap();
+  if (typeof requestAnimationFrame !== "undefined") {
+    requestAnimationFrame(resetStartTimeMsCache);
+  }
+};
+resetStartTimeMsCache();
+const EFTemporal = (superClass) => {
+  class TemporalMixinClass extends superClass {
+    constructor() {
+      super(...arguments);
+      this._offsetMs = 0;
+      this._trimStartMs = 0;
+      this._trimEndMs = 0;
+      this._startOffsetMs = 0;
+      this.rootTimegroup = this.getRootTimegroup();
+      this.frameTask = new Task(this, {
+        autoRun: EF_INTERACTIVE,
+        args: () => [this.ownCurrentTimeMs],
+        task: async ([], { signal: _signal }) => {
+          let fullyUpdated = await this.updateComplete;
+          while (!fullyUpdated) {
+            fullyUpdated = await this.updateComplete;
+          }
+        }
+      });
+    }
+    #parentTimegroup;
+    set parentTimegroup(value) {
+      this.#parentTimegroup = value;
+      this.ownCurrentTimeController?.remove();
+      this.rootTimegroup = this.getRootTimegroup();
+      if (this.rootTimegroup) {
+        this.ownCurrentTimeController = new OwnCurrentTimeController(
+          this.rootTimegroup,
+          this
+        );
+      }
+    }
+    get parentTimegroup() {
+      return this.#parentTimegroup;
+    }
+    get trimStartMs() {
+      return this._trimStartMs;
+    }
+    get trimEndMs() {
+      return this._trimEndMs;
+    }
+    get startOffsetMs() {
+      return this._startOffsetMs;
+    }
+    getRootTimegroup() {
+      let parent = this.tagName === "EF-TIMEGROUP" ? this : this.parentTimegroup;
+      while (parent?.parentTimegroup) {
+        parent = parent.parentTimegroup;
+      }
+      return parent;
+    }
+    get hasOwnDuration() {
+      return false;
+    }
+    // Defining this as a getter to a private property allows us to
+    // override it classes that include this mixin.
+    get durationMs() {
+      return this._durationMs || this.parentTimegroup?.durationMs || 0;
+    }
+    get offsetMs() {
+      return this._offsetMs || 0;
+    }
+    get parentTemporal() {
+      let parent = this.parentElement;
+      while (parent && !isEFTemporal(parent)) {
+        parent = parent.parentElement;
+      }
+      return parent;
+    }
+    get startTimeWithinParentMs() {
+      if (!this.parentTemporal) {
+        return 0;
+      }
+      return this.startTimeMs - this.parentTemporal.startTimeMs;
+    }
+    get startTimeMs() {
+      const cachedStartTime = startTimeMsCache.get(this);
+      if (cachedStartTime !== void 0) {
+        return cachedStartTime;
+      }
+      const parentTimegroup = this.parentTimegroup;
+      if (!parentTimegroup) {
+        startTimeMsCache.set(this, 0);
+        return 0;
+      }
+      switch (parentTimegroup.mode) {
+        case "sequence": {
+          const siblingTemorals = shallowGetTemporalElements(parentTimegroup);
+          const ownIndex = siblingTemorals?.indexOf(
+            this
+          );
+          if (ownIndex === 0) {
+            startTimeMsCache.set(this, parentTimegroup.startTimeMs);
+            return parentTimegroup.startTimeMs;
+          }
+          const previous = siblingTemorals?.[(ownIndex ?? 0) - 1];
+          if (!previous) {
+            throw new Error("Previous temporal element not found");
+          }
+          startTimeMsCache.set(
+            this,
+            previous.startTimeMs + previous.durationMs - parentTimegroup.overlapMs
+          );
+          return previous.startTimeMs + previous.durationMs - parentTimegroup.overlapMs;
+        }
+        case "contain":
+        case "fixed":
+          startTimeMsCache.set(
+            this,
+            parentTimegroup.startTimeMs + this.offsetMs
+          );
+          return parentTimegroup.startTimeMs + this.offsetMs;
+        default:
+          throw new Error(`Invalid time mode: ${parentTimegroup.mode}`);
+      }
+    }
+    get endTimeMs() {
+      return this.startTimeMs + this.durationMs;
+    }
+    get ownCurrentTimeMs() {
+      if (this.rootTimegroup) {
+        return Math.min(
+          Math.max(0, this.rootTimegroup.currentTimeMs - this.startTimeMs),
+          this.durationMs
+        );
+      }
+      return 0;
+    }
+    /**
+     * Used to calculate the internal currentTimeMs of the element. This is useful
+     * for mapping to internal media time codes for audio/video elements.
+     */
+    get trimAdjustedOwnCurrentTimeMs() {
+      if (this.rootTimegroup) {
+        return Math.min(
+          Math.max(
+            0,
+            this.rootTimegroup.currentTimeMs - this.startTimeMs + this.trimStartMs
+          ),
+          this.durationMs + Math.abs(this.startOffsetMs) + this.trimStartMs
+        );
+      }
+      return 0;
+    }
+  }
+  __decorateClass([
+    consume({ context: timegroupContext, subscribe: true }),
+    property({ attribute: false })
+  ], TemporalMixinClass.prototype, "parentTimegroup", 1);
+  __decorateClass([
+    property({
+      type: String,
+      attribute: "offset",
+      converter: durationConverter
+    })
+  ], TemporalMixinClass.prototype, "_offsetMs", 2);
+  __decorateClass([
+    property({
+      type: Number,
+      attribute: "duration",
+      converter: durationConverter
+    })
+  ], TemporalMixinClass.prototype, "_durationMs", 2);
+  __decorateClass([
+    property({
+      type: Number,
+      attribute: "trimstart",
+      converter: durationConverter
+    })
+  ], TemporalMixinClass.prototype, "_trimStartMs", 2);
+  __decorateClass([
+    property({
+      type: Number,
+      attribute: "trimend",
+      converter: durationConverter
+    })
+  ], TemporalMixinClass.prototype, "_trimEndMs", 2);
+  __decorateClass([
+    property({
+      type: Number,
+      attribute: "startoffset",
+      converter: durationConverter
+    })
+  ], TemporalMixinClass.prototype, "_startOffsetMs", 2);
+  __decorateClass([
+    state()
+  ], TemporalMixinClass.prototype, "rootTimegroup", 2);
+  Object.defineProperty(TemporalMixinClass.prototype, EF_TEMPORAL, {
+    value: true
+  });
+  return TemporalMixinClass;
+};
+export {
+  EFTemporal,
+  OwnCurrentTimeController,
+  deepGetElementsWithFrameTasks,
+  deepGetTemporalElements,
+  isEFTemporal,
+  shallowGetTemporalElements,
+  timegroupContext
+};