@editframe/elements 0.5.0-beta.8 → 0.6.0-beta.1

package/src/elements/EFMedia.ts

@@ -1,395 +0,0 @@
-import { LitElement, PropertyValueMap, css } from "lit";
-import { EFTemporal } from "./EFTemporal";
-import { property, state } from "lit/decorators.js";
-import { deepArrayEquals } from "@lit/task/deep-equals.js";
-import { Task } from "@lit/task";
-import * as MP4Box from "mp4box";
-import { MP4File } from "@/editor/util/MP4File";
-import { TrackFragmentIndex, TrackSegment } from "@/assets/Probe";
-import { getStartTimeMs } from "./util";
-import { VideoAsset } from "@/editor/util/EncodedAsset/EncodedAsset";
-import { FetchMixin } from "./FetchMixin";
-import { apiHostContext } from "../gui/EFWorkbench";
-import { consume } from "@lit/context";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
-
-export const deepGetMediaElements = (
-  element: Element,
-  medias: EFMedia[] = [],
-) => {
-  for (const child of element.children) {
-    if (child instanceof EFMedia) {
-      medias.push(child);
-    } else {
-      deepGetMediaElements(child, medias);
-    }
-  }
-  return medias;
-};
-
-export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
-  assetType: "isobmff_files",
-}) {
-  static styles = [
-    css`
-      :host {
-        display: block;
-        position: relative;
-        overflow: hidden;
-      }
-    `,
-  ];
-
-  @property({ type: Number })
-  currentTimeMs = 0;
-
-  @consume({ context: apiHostContext, subscribe: true })
-  @state()
-  efHost?: string;
-
-  // get requiredAssets() {
-  //   return { [this.md5SumLoader.value]: this.requiredAssetsPath.value ?? [] };
-  // }
-
-  fragmentIndexPath() {
-    if (this.getAttribute("src")?.startsWith("http")) {
-      return this.getAttribute("src")! + "/index";
-    }
-    return `/@ef-track-fragment-index/${this.getAttribute("src") ?? ""}`;
-  }
-
-  fragmentTrackPath(trackId: string) {
-    if (this.getAttribute("src")?.startsWith("http")) {
-      return (
-        this.getAttribute("src")!.replace("files", "tracks") + `/${trackId}`
-      );
-    }
-    return `/@ef-track/${this.getAttribute("src") ?? ""}?trackId=${trackId}`;
-  }
-
-  // requiredAssetsPath = new Task(this, {
-  //   args: () => [this.trackFragmentIndexLoader.value] as const,
-  //   task: ([trackFragmentIndex]) => {
-  //     if (!trackFragmentIndex) {
-  //       return [];
-  //     }
-  //     const paths = [this.fragmentIndexPath()];
-
-  //     Object.values(trackFragmentIndex).forEach((track) => {
-  //       paths.push(this.fragmentTrackPath(String(track.track)));
-  //     });
-
-  //     return paths;
-  //   },
-  // });
-
-  public trackFragmentIndexLoader = new Task(this, {
-    args: () => [this.fragmentIndexPath(), this.fetch] as const,
-    task: async ([fragmentIndexPath, fetch], { signal }) => {
-      console.log("EFMedia trackFragmentIndexLoader");
-      const response = await fetch(fragmentIndexPath, { signal });
-      return (await response.json()) as Record<number, TrackFragmentIndex>;
-    },
-    onComplete: () => {
-      this.requestUpdate("ownCurrentTimeMs");
-      this.parentTimegroup?.requestUpdate("ownCurrentTimeMs");
-    },
-  });
-
-  protected initSegmentsLoader = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () =>
-      [this.trackFragmentIndexLoader.value, this.src, this.fetch] as const,
-    task: async ([fragmentIndex, _src, fetch], { signal }) => {
-      console.log("EFMedia initSegmentsLoader");
-      if (!fragmentIndex) {
-        return;
-      }
-      return await Promise.all(
-        Object.entries(fragmentIndex).map(async ([trackId, track]) => {
-          const start = track.initSegment.offset;
-          const end = track.initSegment.offset + track.initSegment.size - 1;
-          const response = await fetch(this.fragmentTrackPath(trackId), {
-            signal,
-            headers: { Range: `bytes=${start}-${end}` },
-          });
-          const buffer =
-            (await response.arrayBuffer()) as MP4Box.MP4ArrayBuffer;
-          buffer.fileStart = 0;
-          const mp4File = new MP4File();
-          mp4File.appendBuffer(buffer, true);
-          mp4File.flush();
-          await mp4File.readyPromise;
-
-          return { trackId, buffer, mp4File };
-        }),
-      );
-    },
-  });
-
-  get defaultVideoTrackId() {
-    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
-      (track) => track.type === "video",
-    )?.track;
-  }
-
-  get defaultAudioTrackId() {
-    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
-      (track) => track.type === "audio",
-    )?.track;
-  }
-
-  seekTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () =>
-      [
-        this.desiredSeekTimeMs,
-        this.trackFragmentIndexLoader.value,
-        this.initSegmentsLoader.value,
-      ] as const,
-    task: async ([seekToMs, fragmentIndex, initSegments], { signal }) => {
-      console.log("EFMedia seekTask");
-      if (fragmentIndex === undefined) {
-        return;
-      }
-      if (initSegments === undefined) {
-        return;
-      }
-
-      const result: Record<
-        string,
-        { segment: TrackSegment; track: MP4Box.TrackInfo }
-      > = {};
-
-      Object.values(fragmentIndex).forEach((index) => {
-        const track = initSegments
-          .find((segment) => segment.trackId === String(index.track))
-          ?.mp4File.getInfo().tracks[0];
-
-        if (!track) {
-          throw new Error("Could not finding matching track");
-        }
-
-        const segment = index.segments.toReversed().find((segment) => {
-          return (segment.dts / track.timescale) * 1000 <= seekToMs;
-        });
-
-        if (!segment) {
-          return;
-        }
-
-        result[index.track] = { segment, track };
-      });
-
-      return result;
-    },
-  });
-
-  fetchSeekTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    argsEqual: deepArrayEquals,
-    args: () =>
-      [this.initSegmentsLoader.value, this.seekTask.value, this.fetch] as const,
-    task: async ([initSegments, seekResult, fetch], { signal }) => {
-      console.log("EFMedia fetchSeekTask");
-      if (!initSegments) {
-        return;
-      }
-      if (!seekResult) {
-        return;
-      }
-
-      const files: Record<string, File> = {};
-
-      for (const [trackId, { segment, track }] of Object.entries(seekResult)) {
-        const start = segment.offset;
-        const end = segment.offset + segment.size;
-
-        const response = await fetch(this.fragmentTrackPath(trackId), {
-          signal,
-          headers: { Range: `bytes=${start}-${end}` },
-        });
-
-        const initSegment = Object.values(initSegments).find(
-          (initSegment) => initSegment.trackId === String(track.id),
-        );
-        const initBuffer = initSegment!.buffer;
-
-        const mediaBuffer =
-          (await response.arrayBuffer()) as unknown as MP4Box.MP4ArrayBuffer;
-
-        files[trackId] = new File([initBuffer, mediaBuffer], "video.mp4", {
-          type: "video/mp4",
-        });
-      }
-
-      return files;
-    },
-  });
-
-  videoAssetTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.fetchSeekTask.value] as const,
-    task: async ([files], { signal }) => {
-      console.log("EFMedia videoAssetTask");
-      if (!files) {
-        return;
-      }
-      if (!this.defaultVideoTrackId) {
-        return;
-      }
-      const videoFile = files[this.defaultVideoTrackId];
-      if (!videoFile) {
-        return;
-      }
-      return await VideoAsset.createFromReadableStream(
-        "video.mp4",
-        videoFile.stream(),
-        videoFile,
-      );
-    },
-  });
-
-  @state() desiredSeekTimeMs = 0;
-
-  protected async executeSeek(seekToMs: number) {
-    this.desiredSeekTimeMs = seekToMs;
-  }
-
-  protected updated(
-    changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
-  ): void {
-    if (changedProperties.has("ownCurrentTimeMs")) {
-      this.executeSeek(this.ownCurrentTimeMs);
-    }
-  }
-
-  get hasOwnDuration() {
-    return true;
-  }
-
-  get durationMs() {
-    if (!this.trackFragmentIndexLoader.value) {
-      return 0;
-    }
-
-    const durations = Object.values(this.trackFragmentIndexLoader.value).map(
-      (track) => {
-        return (track.duration / track.timescale) * 1000;
-      },
-    );
-    if (durations.length === 0) {
-      return 0;
-    }
-    return Math.max(...durations);
-  }
-
-  get startTimeMs() {
-    return getStartTimeMs(this);
-  }
-
-  #audioContext = new OfflineAudioContext(2, 48000 / 30, 48000);
-
-  audioBufferTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.fetchSeekTask.value, this.seekTask.value] as const,
-    task: async ([files, segments], { signal }) => {
-      console.log("EFMedia audioBufferTask", this.outerHTML);
-      if (!files) {
-        return;
-      }
-      if (!segments) {
-        return;
-      }
-      if (!this.defaultAudioTrackId) {
-        return;
-      }
-      const segment = segments[this.defaultAudioTrackId];
-      if (!segment) {
-        return;
-      }
-      const audioFile = files[this.defaultAudioTrackId];
-      if (!audioFile) {
-        return;
-      }
-      return {
-        buffer: await this.#audioContext.decodeAudioData(
-          await audioFile.arrayBuffer(),
-        ),
-        startOffsetMs: (segment.segment.cts / segment.track.timescale) * 1000,
-      };
-    },
-  });
-
-  async fetchAudioSpanningTime(fromMs: number, toMs: number) {
-    // Adjust range for track's own time
-    fromMs -= this.startTimeMs;
-    toMs -= this.startTimeMs;
-
-    await this.trackFragmentIndexLoader.taskComplete;
-    const audioTrackId = this.defaultAudioTrackId;
-    if (!audioTrackId) {
-      console.warn("No audio track found");
-      return;
-    }
-    const audioTrackIndex = this.trackFragmentIndexLoader.value?.[audioTrackId];
-    if (!audioTrackIndex) {
-      console.warn("No audio track found");
-      return;
-    }
-
-    const start = audioTrackIndex.initSegment.offset;
-    const end =
-      audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size - 1;
-    const audioInitFragmentRequest = this.fetch(
-      this.fragmentTrackPath(String(audioTrackId)),
-      {
-        headers: { Range: `bytes=${start}-${end}` },
-      },
-    );
-
-    const fragments = Object.values(audioTrackIndex.segments).filter(
-      (segment) => {
-        const segmentStartsBeforeEnd =
-          segment.dts <= (toMs * audioTrackIndex.timescale) / 1000;
-        const segmentEndsAfterStart =
-          segment.dts + segment.duration >=
-          (fromMs * audioTrackIndex.timescale) / 1000;
-        return segmentStartsBeforeEnd && segmentEndsAfterStart;
-      },
-    );
-
-    console.log("FRAGMENTS SPANNING TIME", JSON.stringify(fragments));
-    const firstFragment = fragments[0]!;
-    const lastFragment = fragments[fragments.length - 1]!;
-    const fragmentStart = firstFragment.offset;
-    const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
-
-    console.log("FETCHING BYTES", `bytes=${fragmentStart}-${fragmentEnd}`);
-    const audioFragmentRequest = this.fetch(
-      this.fragmentTrackPath(String(audioTrackId)),
-      {
-        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd}` },
-      },
-    );
-
-    const initResponse = await audioInitFragmentRequest;
-    const dataResponse = await audioFragmentRequest;
-
-    const initBuffer = await initResponse.arrayBuffer();
-    const dataBuffer = await dataResponse.arrayBuffer();
-
-    const audioBlob = new Blob([initBuffer, dataBuffer], {
-      type: "audio/mp4",
-    });
-
-    return {
-      blob: audioBlob,
-      startMs: (firstFragment.dts / audioTrackIndex.timescale) * 1000,
-      endMs:
-        (lastFragment.dts / audioTrackIndex.timescale) * 1000 +
-        (lastFragment.duration / audioTrackIndex.timescale) * 1000,
-    };
-  }
-}

package/src/elements/EFSourceMixin.ts

@@ -1,57 +0,0 @@
-import { consume } from "@lit/context";
-import { LitElement } from "lit";
-import { apiHostContext } from "../gui/EFWorkbench";
-import { state } from "lit/decorators/state.js";
-import { Task } from "@lit/task";
-import { property } from "lit/decorators/property.js";
-
-export declare class EFSourceMixinInterface {
-  productionSrc(): string;
-  src: string;
-}
-
-interface EFSourceMixinOptions {
-  assetType: string;
-}
-
-export function EFSourceMixin<T extends Constructor<LitElement>>(
-  superClass: T,
-  options: EFSourceMixinOptions,
-) {
-  class EFSourceElement extends superClass {
-    @consume({ context: apiHostContext, subscribe: true })
-    @state()
-    private efHost?: string;
-
-    @property({ type: String })
-    src = "";
-
-    productionSrc() {
-      if (!this.md5SumLoader.value) {
-        throw new Error(
-          `MD5 sum not available for ${this}. Cannot generate production URL`,
-        );
-      }
-
-      if (!this.efHost) {
-        throw new Error(
-          `efHost not available for ${this}. Cannot generate production URL`,
-        );
-      }
-
-      return `${this.efHost}/api/video2/${options.assetType}/${this.md5SumLoader.value}`;
-    }
-
-    md5SumLoader = new Task(this, {
-      autoRun: false,
-      args: () => [this.src] as const,
-      task: async ([src], { signal }) => {
-        const md5Path = `/@ef-asset/${src}`;
-        const response = await fetch(md5Path, { method: "HEAD", signal });
-        return response.headers.get("etag") ?? undefined;
-      },
-    });
-  }
-
-  return EFSourceElement as Constructor<EFSourceMixinInterface> & T;
-}

package/src/elements/EFTemporal.ts

@@ -1,246 +0,0 @@
-import { LitElement, ReactiveController } from "lit";
-import { consume, createContext } from "@lit/context";
-import { property, state } from "lit/decorators.js";
-import { EFTimegroup } from "./EFTimegroup";
-
-import { durationConverter } from "./durationConverter";
-import { Task } from "@lit/task";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
-
-export const timegroupContext = createContext<EFTimegroup>(
-  Symbol("timeGroupContext"),
-);
-
-export declare class TemporalMixinInterface {
-  get hasOwnDuration(): boolean;
-  get durationMs(): number;
-  get startTimeMs(): number;
-  get startTimeWithinParentMs(): number;
-  get endTimeMs(): number;
-  get ownCurrentTimeMs(): number;
-
-  parentTimegroup?: EFTimegroup;
-  rootTimegroup?: EFTimegroup;
-
-  frameTask: Task<readonly unknown[], unknown>;
-}
-
-export const isEFTemporal = (obj: any): obj is TemporalMixinInterface =>
-  obj[EF_TEMPORAL];
-
-const EF_TEMPORAL = Symbol("EF_TEMPORAL");
-
-export const deepGetTemporalElements = (
-  element: Element,
-  temporals: TemporalMixinInterface[] = [],
-) => {
-  for (const child of element.children) {
-    if (isEFTemporal(child)) {
-      temporals.push(child);
-    }
-    deepGetTemporalElements(child, temporals);
-  }
-  return temporals;
-};
-
-export const deepGetElementsWithFrameTasks = (
-  element: Element,
-  elements: { frameTask: Task }[] = [],
-) => {
-  for (const child of element.children) {
-    if ("frameTask" in child && child.frameTask instanceof Task) {
-      elements.push(child as { frameTask: Task });
-    }
-    deepGetElementsWithFrameTasks(child, elements);
-  }
-  return elements;
-};
-
-export const shallowGetTemporalElements = (
-  element: Element,
-  temporals: TemporalMixinInterface[] = [],
-) => {
-  for (const child of element.children) {
-    if (isEFTemporal(child)) {
-      temporals.push(child);
-    } else {
-      shallowGetTemporalElements(child, temporals);
-    }
-  }
-  return temporals;
-};
-
-export const shallowGetTimegroups = (
-  element: Element,
-  groups: EFTimegroup[] = [],
-) => {
-  for (const child of element.children) {
-    if (child instanceof EFTimegroup) {
-      groups.push(child);
-    } else {
-      shallowGetTimegroups(child, groups);
-    }
-  }
-  return groups;
-};
-
-export class OwnCurrentTimeController implements ReactiveController {
-  constructor(
-    private host: EFTimegroup,
-    private temporal: TemporalMixinInterface & LitElement,
-  ) {
-    host.addController(this);
-  }
-
-  hostUpdated() {
-    this.temporal.requestUpdate("ownCurrentTimeMs");
-  }
-
-  remove() {
-    this.host.removeController(this);
-  }
-}
-
-export const EFTemporal = <T extends Constructor<LitElement>>(
-  superClass: T,
-) => {
-  class TemporalMixinClass extends superClass {
-    ownCurrentTimeController?: OwnCurrentTimeController;
-
-    #parentTimegroup?: EFTimegroup;
-    @consume({ context: timegroupContext, subscribe: true })
-    @property({ attribute: false })
-    set parentTimegroup(value: EFTimegroup | undefined) {
-      this.#parentTimegroup = value;
-      this.ownCurrentTimeController?.remove();
-      this.rootTimegroup = this.getRootTimegroup();
-      if (this.rootTimegroup) {
-        this.ownCurrentTimeController = new OwnCurrentTimeController(
-          this.rootTimegroup,
-          this,
-        );
-      }
-    }
-    get parentTimegroup() {
-      return this.#parentTimegroup;
-    }
-
-    @property({
-      type: String,
-      attribute: "offset",
-      converter: durationConverter,
-    })
-    private _offsetMs = 0;
-
-    @property({
-      type: Number,
-      attribute: "duration",
-      converter: durationConverter,
-    })
-    private _durationMs?: number;
-
-    private get offsetAppliesToDuration() {
-      return !(this.parentTimegroup?.mode === "sequence");
-    }
-
-    @state()
-    rootTimegroup?: EFTimegroup = this.getRootTimegroup();
-
-    private getRootTimegroup(): EFTimegroup | undefined {
-      let parent = this instanceof EFTimegroup ? this : this.parentTimegroup;
-      while (parent?.parentTimegroup) {
-        parent = parent.parentTimegroup;
-      }
-      return parent;
-    }
-
-    get hasOwnDuration() {
-      return false;
-    }
-
-    // Defining this as a getter to a private property allows us to
-    // override it classes that include this mixin.
-    get durationMs() {
-      let durationMs =
-        this._durationMs || this.parentTimegroup?.durationMs || 0;
-      return durationMs || 0;
-    }
-
-    get offsetMs() {
-      return this._offsetMs || 0;
-    }
-
-    get parentTemporal() {
-      let parent = this.parentElement;
-      while (parent && !isEFTemporal(parent)) {
-        parent = parent.parentElement;
-      }
-      return parent;
-    }
-
-    get startTimeWithinParentMs() {
-      if (!this.parentTemporal) {
-        return 0;
-      }
-      return this.startTimeMs - this.parentTemporal.startTimeMs;
-    }
-
-    get startTimeMs(): number {
-      const parentTimegroup = this.parentTimegroup;
-      if (!parentTimegroup) {
-        return 0;
-      }
-      switch (parentTimegroup.mode) {
-        case "sequence": {
-          const siblingTemorals = shallowGetTemporalElements(parentTimegroup);
-          const ownIndex = siblingTemorals.indexOf(this);
-          if (ownIndex === 0) {
-            return parentTimegroup.startTimeMs;
-          }
-          const previous = siblingTemorals[ownIndex - 1];
-          if (!previous) {
-            throw new Error("Previous temporal element not found");
-          }
-          return previous.startTimeMs + previous.durationMs;
-        }
-        case "contain":
-        case "fixed":
-          return parentTimegroup.startTimeMs + this.offsetMs;
-        default:
-          throw new Error(`Invalid time mode: ${parentTimegroup.mode}`);
-      }
-    }
-
-    get endTimeMs(): number {
-      return this.startTimeMs + this.durationMs;
-    }
-
-    get ownCurrentTimeMs() {
-      if (this.rootTimegroup) {
-        return Math.min(
-          Math.max(0, this.rootTimegroup.currentTimeMs - this.startTimeMs),
-          this.durationMs,
-        );
-      }
-      return 0;
-    }
-
-    frameTask = new Task(this, {
-      autoRun: EF_INTERACTIVE,
-      args: () => [this.ownCurrentTimeMs] as const,
-      task: async ([], { signal }) => {
-        let fullyUpdated = await this.updateComplete;
-        while (!fullyUpdated) {
-          fullyUpdated = await this.updateComplete;
-        }
-      },
-    });
-  }
-
-  Object.defineProperty(TemporalMixinClass.prototype, EF_TEMPORAL, {
-    value: true,
-  });
-
-  return TemporalMixinClass as unknown as Constructor<TemporalMixinInterface> &
-    T;
-};