@editframe/elements 0.6.0-beta.16 → 0.6.0-beta.17

package/src/elements/EFMedia.ts

@@ -0,0 +1,384 @@
+import { LitElement, type PropertyValueMap, css } from "lit";
+import { EFTemporal } from "./EFTemporal";
+import { property, state } from "lit/decorators.js";
+import { deepArrayEquals } from "@lit/task/deep-equals.js";
+import { Task } from "@lit/task";
+import type * as MP4Box from "mp4box";
+import { MP4File } from "@/av/MP4File";
+import type { TrackFragmentIndex, TrackSegment } from "@editframe/assets";
+import { getStartTimeMs } from "./util";
+import { VideoAsset } from "@/av/EncodedAsset";
+import { FetchMixin } from "./FetchMixin";
+import { apiHostContext } from "../gui/EFWorkbench";
+import { consume } from "@lit/context";
+import { EFSourceMixin } from "./EFSourceMixin";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+
+export const deepGetMediaElements = (
+  element: Element,
+  medias: EFMedia[] = [],
+) => {
+  for (const child of Array.from(element.children)) {
+    if (child instanceof EFMedia) {
+      medias.push(child);
+    } else {
+      deepGetMediaElements(child, medias);
+    }
+  }
+  return medias;
+};
+
+export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
+  assetType: "isobmff_files",
+}) {
+  static styles = [
+    css`
+      :host {
+        display: block;
+        position: relative;
+        overflow: hidden;
+      }
+    `,
+  ];
+
+  @property({ type: Number })
+  currentTimeMs = 0;
+
+  @consume({ context: apiHostContext, subscribe: true })
+  @state()
+  efHost?: string;
+
+  fragmentIndexPath() {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return `${this.src}/index`;
+    }
+    return `/@ef-track-fragment-index/${this.getAttribute("src") ?? ""}`;
+  }
+
+  fragmentTrackPath(trackId: string) {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return `${this.src.replace("files", "tracks")}/${trackId}`;
+    }
+    return `/@ef-track/${this.src ?? ""}?trackId=${trackId}`;
+  }
+
+  public trackFragmentIndexLoader = new Task(this, {
+    args: () => [this.fragmentIndexPath(), this.fetch] as const,
+    task: async ([fragmentIndexPath, fetch], { signal }) => {
+      const response = await fetch(fragmentIndexPath, { signal });
+      return (await response.json()) as Record<number, TrackFragmentIndex>;
+    },
+    onComplete: () => {
+      this.requestUpdate("ownCurrentTimeMs");
+      this.rootTimegroup?.requestUpdate("ownCurrentTimeMs");
+    },
+  });
+
+  protected initSegmentsLoader = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [this.trackFragmentIndexLoader.value, this.src, this.fetch] as const,
+    task: async ([fragmentIndex, _src, fetch], { signal }) => {
+      if (!fragmentIndex) {
+        return;
+      }
+      return await Promise.all(
+        Object.entries(fragmentIndex).map(async ([trackId, track]) => {
+          const start = track.initSegment.offset;
+          const end = track.initSegment.offset + track.initSegment.size - 1;
+          const response = await fetch(this.fragmentTrackPath(trackId), {
+            signal,
+            headers: { Range: `bytes=${start}-${end}` },
+          });
+          const buffer =
+            (await response.arrayBuffer()) as MP4Box.MP4ArrayBuffer;
+          buffer.fileStart = 0;
+          const mp4File = new MP4File();
+          mp4File.appendBuffer(buffer, true);
+          mp4File.flush();
+          await mp4File.readyPromise;
+
+          return { trackId, buffer, mp4File };
+        }),
+      );
+    },
+  });
+
+  get defaultVideoTrackId() {
+    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
+      (track) => track.type === "video",
+    )?.track;
+  }
+
+  get defaultAudioTrackId() {
+    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
+      (track) => track.type === "audio",
+    )?.track;
+  }
+
+  seekTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () =>
+      [
+        this.desiredSeekTimeMs,
+        this.trackFragmentIndexLoader.value,
+        this.initSegmentsLoader.value,
+      ] as const,
+    task: async (
+      [seekToMs, fragmentIndex, initSegments],
+      { signal: _signal },
+    ) => {
+      if (fragmentIndex === undefined) {
+        return;
+      }
+      if (initSegments === undefined) {
+        return;
+      }
+
+      const result: Record<
+        string,
+        { segment: TrackSegment; track: MP4Box.TrackInfo }
+      > = {};
+
+      for (const index of Object.values(fragmentIndex)) {
+        const track = initSegments
+          .find((segment) => segment.trackId === String(index.track))
+          ?.mp4File.getInfo().tracks[0];
+
+        if (!track) {
+          throw new Error("Could not finding matching track");
+        }
+
+        const segment = index.segments.toReversed().find((segment) => {
+          return (segment.dts / track.timescale) * 1000 <= seekToMs;
+        });
+
+        if (!segment) {
+          return;
+        }
+
+        result[index.track] = { segment, track };
+      }
+
+      return result;
+    },
+  });
+
+  fetchSeekTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    argsEqual: deepArrayEquals,
+    args: () =>
+      [this.initSegmentsLoader.value, this.seekTask.value, this.fetch] as const,
+    task: async ([initSegments, seekResult, fetch], { signal }) => {
+      if (!initSegments) {
+        return;
+      }
+      if (!seekResult) {
+        return;
+      }
+
+      const files: Record<string, File> = {};
+
+      for (const [trackId, { segment, track }] of Object.entries(seekResult)) {
+        const start = segment.offset;
+        const end = segment.offset + segment.size;
+
+        const response = await fetch(this.fragmentTrackPath(trackId), {
+          signal,
+          headers: { Range: `bytes=${start}-${end}` },
+        });
+
+        const initSegment = Object.values(initSegments).find(
+          (initSegment) => initSegment.trackId === String(track.id),
+        );
+        if (!initSegment) {
+          throw new Error("Could not find matching init segment");
+        }
+        const initBuffer = initSegment.buffer;
+
+        const mediaBuffer =
+          (await response.arrayBuffer()) as unknown as MP4Box.MP4ArrayBuffer;
+
+        files[trackId] = new File([initBuffer, mediaBuffer], "video.mp4", {
+          type: "video/mp4",
+        });
+      }
+
+      return files;
+    },
+  });
+
+  videoAssetTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.fetchSeekTask.value] as const,
+    task: async ([files], { signal: _signal }) => {
+      if (!files) {
+        return;
+      }
+      if (!this.defaultVideoTrackId) {
+        return;
+      }
+      const videoFile = files[this.defaultVideoTrackId];
+      if (!videoFile) {
+        return;
+      }
+      // TODO: Extract to general cleanup function
+      for (const frame of this.videoAssetTask.value?.decodedFrames || []) {
+        frame.close();
+      }
+      this.videoAssetTask.value?.videoDecoder?.close();
+      return await VideoAsset.createFromReadableStream(
+        "video.mp4",
+        videoFile.stream(),
+        videoFile,
+      );
+    },
+  });
+
+  @state() desiredSeekTimeMs = 0;
+
+  protected async executeSeek(seekToMs: number) {
+    this.desiredSeekTimeMs = seekToMs;
+  }
+
+  protected updated(
+    changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
+  ): void {
+    if (changedProperties.has("ownCurrentTimeMs")) {
+      this.executeSeek(this.ownCurrentTimeMs);
+    }
+  }
+
+  get hasOwnDuration() {
+    return true;
+  }
+
+  get durationMs() {
+    if (!this.trackFragmentIndexLoader.value) {
+      return 0;
+    }
+
+    const durations = Object.values(this.trackFragmentIndexLoader.value).map(
+      (track) => {
+        return (track.duration / track.timescale) * 1000;
+      },
+    );
+    if (durations.length === 0) {
+      return 0;
+    }
+    return Math.max(...durations);
+  }
+
+  get startTimeMs() {
+    return getStartTimeMs(this);
+  }
+
+  #audioContext = new OfflineAudioContext(2, 48000 / 30, 48000);
+
+  audioBufferTask = new Task(this, {
+    autoRun: EF_INTERACTIVE,
+    args: () => [this.fetchSeekTask.value, this.seekTask.value] as const,
+    task: async ([files, segments], { signal: _signal }) => {
+      if (!files) {
+        return;
+      }
+      if (!segments) {
+        return;
+      }
+      if (!this.defaultAudioTrackId) {
+        return;
+      }
+      const segment = segments[this.defaultAudioTrackId];
+      if (!segment) {
+        return;
+      }
+      const audioFile = files[this.defaultAudioTrackId];
+      if (!audioFile) {
+        return;
+      }
+      return {
+        buffer: await this.#audioContext.decodeAudioData(
+          await audioFile.arrayBuffer(),
+        ),
+        startOffsetMs: (segment.segment.cts / segment.track.timescale) * 1000,
+      };
+    },
+  });
+
+  async fetchAudioSpanningTime(fromMs: number, toMs: number) {
+    // Adjust range for track's own time
+    fromMs -= this.startTimeMs;
+    toMs -= this.startTimeMs;
+
+    await this.trackFragmentIndexLoader.taskComplete;
+    const audioTrackId = this.defaultAudioTrackId;
+    if (!audioTrackId) {
+      console.warn("No audio track found");
+      return;
+    }
+    const audioTrackIndex = this.trackFragmentIndexLoader.value?.[audioTrackId];
+    if (!audioTrackIndex) {
+      console.warn("No audio track found");
+      return;
+    }
+
+    const start = audioTrackIndex.initSegment.offset;
+    const end =
+      audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size - 1;
+    const audioInitFragmentRequest = this.fetch(
+      this.fragmentTrackPath(String(audioTrackId)),
+      {
+        headers: { Range: `bytes=${start}-${end}` },
+      },
+    );
+
+    const fragments = Object.values(audioTrackIndex.segments).filter(
+      (segment) => {
+        const segmentStartsBeforeEnd =
+          segment.dts <= (toMs * audioTrackIndex.timescale) / 1000;
+        const segmentEndsAfterStart =
+          segment.dts + segment.duration >=
+          (fromMs * audioTrackIndex.timescale) / 1000;
+        return segmentStartsBeforeEnd && segmentEndsAfterStart;
+      },
+    );
+
+    const firstFragment = fragments[0];
+    if (!firstFragment) {
+      console.warn("No audio fragments found");
+      return;
+    }
+    const lastFragment = fragments[fragments.length - 1];
+    if (!lastFragment) {
+      console.warn("No audio fragments found");
+      return;
+    }
+    const fragmentStart = firstFragment.offset;
+    const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
+
+    const audioFragmentRequest = this.fetch(
+      this.fragmentTrackPath(String(audioTrackId)),
+      {
+        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd}` },
+      },
+    );
+
+    const initResponse = await audioInitFragmentRequest;
+    const dataResponse = await audioFragmentRequest;
+
+    const initBuffer = await initResponse.arrayBuffer();
+    const dataBuffer = await dataResponse.arrayBuffer();
+
+    const audioBlob = new Blob([initBuffer, dataBuffer], {
+      type: "audio/mp4",
+    });
+
+    return {
+      blob: audioBlob,
+      startMs: (firstFragment.dts / audioTrackIndex.timescale) * 1000,
+      endMs:
+        (lastFragment.dts / audioTrackIndex.timescale) * 1000 +
+        (lastFragment.duration / audioTrackIndex.timescale) * 1000,
+    };
+  }
+}

package/src/elements/EFSourceMixin.ts

@@ -0,0 +1,57 @@
+import { consume } from "@lit/context";
+import type { LitElement } from "lit";
+import { apiHostContext } from "../gui/EFWorkbench";
+import { state } from "lit/decorators/state.js";
+import { Task } from "@lit/task";
+import { property } from "lit/decorators/property.js";
+
+export declare class EFSourceMixinInterface {
+  productionSrc(): string;
+  src: string;
+}
+
+interface EFSourceMixinOptions {
+  assetType: string;
+}
+type Constructor<T = {}> = new (...args: any[]) => T;
+export function EFSourceMixin<T extends Constructor<LitElement>>(
+  superClass: T,
+  options: EFSourceMixinOptions,
+) {
+  class EFSourceElement extends superClass {
+    @consume({ context: apiHostContext, subscribe: true })
+    @state()
+    private efHost?: string;
+
+    @property({ type: String })
+    src = "";
+
+    productionSrc() {
+      if (!this.md5SumLoader.value) {
+        throw new Error(
+          `MD5 sum not available for ${this}. Cannot generate production URL`,
+        );
+      }
+
+      if (!this.efHost) {
+        throw new Error(
+          `efHost not available for ${this}. Cannot generate production URL`,
+        );
+      }
+
+      return `${this.efHost}/api/video2/${options.assetType}/${this.md5SumLoader.value}`;
+    }
+
+    md5SumLoader = new Task(this, {
+      autoRun: false,
+      args: () => [this.src] as const,
+      task: async ([src], { signal }) => {
+        const md5Path = `/@ef-asset/${src}`;
+        const response = await fetch(md5Path, { method: "HEAD", signal });
+        return response.headers.get("etag") ?? undefined;
+      },
+    });
+  }
+
+  return EFSourceElement as Constructor<EFSourceMixinInterface> & T;
+}

package/src/elements/EFTemporal.ts

@@ -0,0 +1,231 @@
+import type { LitElement, ReactiveController } from "lit";
+import { consume, createContext } from "@lit/context";
+import { property, state } from "lit/decorators.js";
+import type { EFTimegroup } from "./EFTimegroup";
+
+import { durationConverter } from "./durationConverter";
+import { Task } from "@lit/task";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
+
+export const timegroupContext = createContext<EFTimegroup>(
+  Symbol("timeGroupContext"),
+);
+
+export declare class TemporalMixinInterface {
+  get hasOwnDuration(): boolean;
+  get durationMs(): number;
+  get startTimeMs(): number;
+  get startTimeWithinParentMs(): number;
+  get endTimeMs(): number;
+  get ownCurrentTimeMs(): number;
+
+  parentTimegroup?: EFTimegroup;
+  rootTimegroup?: EFTimegroup;
+
+  frameTask: Task<readonly unknown[], unknown>;
+}
+
+export const isEFTemporal = (obj: any): obj is TemporalMixinInterface =>
+  obj[EF_TEMPORAL];
+
+const EF_TEMPORAL = Symbol("EF_TEMPORAL");
+
+export const deepGetTemporalElements = (
+  element: Element,
+  temporals: TemporalMixinInterface[] = [],
+) => {
+  for (const child of element.children) {
+    if (isEFTemporal(child)) {
+      temporals.push(child);
+    }
+    deepGetTemporalElements(child, temporals);
+  }
+  return temporals;
+};
+
+export const deepGetElementsWithFrameTasks = (
+  element: Element,
+  elements: Array<HTMLElement & { frameTask: Task }> = [],
+) => {
+  for (const child of element.children) {
+    if ("frameTask" in child && child.frameTask instanceof Task) {
+      elements.push(child as HTMLElement & { frameTask: Task });
+    }
+    deepGetElementsWithFrameTasks(child, elements);
+  }
+  return elements;
+};
+
+export const shallowGetTemporalElements = (
+  element: Element,
+  temporals: TemporalMixinInterface[] = [],
+) => {
+  for (const child of element.children) {
+    if (isEFTemporal(child)) {
+      temporals.push(child);
+    } else {
+      shallowGetTemporalElements(child, temporals);
+    }
+  }
+  return temporals;
+};
+
+export class OwnCurrentTimeController implements ReactiveController {
+  constructor(
+    private host: EFTimegroup,
+    private temporal: TemporalMixinInterface & LitElement,
+  ) {
+    host.addController(this);
+  }
+
+  hostUpdated() {
+    this.temporal.requestUpdate("ownCurrentTimeMs");
+  }
+
+  remove() {
+    this.host.removeController(this);
+  }
+}
+
+type Constructor<T = {}> = new (...args: any[]) => T;
+
+export const EFTemporal = <T extends Constructor<LitElement>>(
+  superClass: T,
+) => {
+  class TemporalMixinClass extends superClass {
+    ownCurrentTimeController?: OwnCurrentTimeController;
+
+    #parentTimegroup?: EFTimegroup;
+    @consume({ context: timegroupContext, subscribe: true })
+    @property({ attribute: false })
+    set parentTimegroup(value: EFTimegroup | undefined) {
+      this.#parentTimegroup = value;
+      this.ownCurrentTimeController?.remove();
+      this.rootTimegroup = this.getRootTimegroup();
+      if (this.rootTimegroup) {
+        this.ownCurrentTimeController = new OwnCurrentTimeController(
+          this.rootTimegroup,
+          this,
+        );
+      }
+    }
+    get parentTimegroup() {
+      return this.#parentTimegroup;
+    }
+
+    @property({
+      type: String,
+      attribute: "offset",
+      converter: durationConverter,
+    })
+    private _offsetMs = 0;
+
+    @property({
+      type: Number,
+      attribute: "duration",
+      converter: durationConverter,
+    })
+    private _durationMs?: number;
+
+    @state()
+    rootTimegroup?: EFTimegroup = this.getRootTimegroup();
+
+    private getRootTimegroup(): EFTimegroup | undefined {
+      let parent =
+        this.tagName === "EF-TIMEGROUP" ? this : this.parentTimegroup;
+      while (parent?.parentTimegroup) {
+        parent = parent.parentTimegroup;
+      }
+      return parent as EFTimegroup | undefined;
+    }
+
+    get hasOwnDuration() {
+      return false;
+    }
+
+    // Defining this as a getter to a private property allows us to
+    // override it classes that include this mixin.
+    get durationMs() {
+      const durationMs =
+        this._durationMs || this.parentTimegroup?.durationMs || 0;
+      return durationMs || 0;
+    }
+
+    get offsetMs() {
+      return this._offsetMs || 0;
+    }
+
+    get parentTemporal() {
+      let parent = this.parentElement;
+      while (parent && !isEFTemporal(parent)) {
+        parent = parent.parentElement;
+      }
+      return parent;
+    }
+
+    get startTimeWithinParentMs() {
+      if (!this.parentTemporal) {
+        return 0;
+      }
+      return this.startTimeMs - this.parentTemporal.startTimeMs;
+    }
+
+    get startTimeMs(): number {
+      const parentTimegroup = this.parentTimegroup;
+      if (!parentTimegroup) {
+        return 0;
+      }
+      switch (parentTimegroup.mode) {
+        case "sequence": {
+          const siblingTemorals = shallowGetTemporalElements(parentTimegroup);
+          const ownIndex = siblingTemorals.indexOf(this);
+          if (ownIndex === 0) {
+            return parentTimegroup.startTimeMs;
+          }
+          const previous = siblingTemorals[ownIndex - 1];
+          if (!previous) {
+            throw new Error("Previous temporal element not found");
+          }
+          return previous.startTimeMs + previous.durationMs;
+        }
+        case "contain":
+        case "fixed":
+          return parentTimegroup.startTimeMs + this.offsetMs;
+        default:
+          throw new Error(`Invalid time mode: ${parentTimegroup.mode}`);
+      }
+    }
+
+    get endTimeMs(): number {
+      return this.startTimeMs + this.durationMs;
+    }
+
+    get ownCurrentTimeMs() {
+      if (this.rootTimegroup) {
+        return Math.min(
+          Math.max(0, this.rootTimegroup.currentTimeMs - this.startTimeMs),
+          this.durationMs,
+        );
+      }
+      return 0;
+    }
+
+    frameTask = new Task(this, {
+      autoRun: EF_INTERACTIVE,
+      args: () => [this.ownCurrentTimeMs] as const,
+      task: async ([], { signal: _signal }) => {
+        let fullyUpdated = await this.updateComplete;
+        while (!fullyUpdated) {
+          fullyUpdated = await this.updateComplete;
+        }
+      },
+    });
+  }
+
+  Object.defineProperty(TemporalMixinClass.prototype, EF_TEMPORAL, {
+    value: true,
+  });
+
+  return TemporalMixinClass as unknown as Constructor<TemporalMixinInterface> &
+    T;
+};