@editframe/elements 0.5.0-beta.8 → 0.5.0-beta.9

package/src/elements/EFImage.ts

@@ -1,70 +0,0 @@
-import { Task } from "@lit/task";
-import { LitElement, html, css } from "lit";
-import { customElement } from "lit/decorators.js";
-import { createRef, ref } from "lit/directives/ref.js";
-import { FetchMixin } from "./FetchMixin";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
-
-@customElement("ef-image")
-export class EFImage extends EFSourceMixin(FetchMixin(LitElement), {
-  assetType: "image_files",
-}) {
-  static styles = [
-    css`
-      :host {
-        display: block;
-      }
-      canvas {
-        display: block;
-        width: 100%;
-        height: 100%;
-        object-fit: fill;
-        object-position: center;
-      }
-    `,
-  ];
-
-  imageRef = createRef<HTMLImageElement>();
-  canvasRef = createRef<HTMLCanvasElement>();
-
-  render() {
-    return html`<canvas ${ref(this.canvasRef)}></canvas>`;
-  }
-
-  assetPath() {
-    if (this.src.startsWith("http")) {
-      return this.src;
-    }
-    return `/@ef-image/${this.src}`;
-  }
-
-  // get requiredAssets() {
-  //   return { [this.md5SumLoader.value]: [this.assetPath()] };
-  // }
-
-  fetchImage = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.assetPath(), this.fetch] as const,
-    task: async ([assetPath, fetch], { signal }) => {
-      const response = await fetch(assetPath, { signal });
-      const image = new Image();
-      image.src = URL.createObjectURL(await response.blob());
-      await new Promise((resolve) => {
-        image.onload = resolve;
-      });
-      this.canvasRef.value!.width = image.width;
-      this.canvasRef.value!.height = image.height;
-      const ctx = this.canvasRef.value!.getContext("2d")!;
-      ctx.drawImage(image, 0, 0);
-    },
-  });
-
-  frameTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.fetchImage.status] as const,
-    task: async () => {
-      await this.fetchImage.taskComplete;
-    },
-  });
-}

package/src/elements/EFMedia.ts

@@ -1,395 +0,0 @@
-import { LitElement, PropertyValueMap, css } from "lit";
-import { EFTemporal } from "./EFTemporal";
-import { property, state } from "lit/decorators.js";
-import { deepArrayEquals } from "@lit/task/deep-equals.js";
-import { Task } from "@lit/task";
-import * as MP4Box from "mp4box";
-import { MP4File } from "@/editor/util/MP4File";
-import { TrackFragmentIndex, TrackSegment } from "@/assets/Probe";
-import { getStartTimeMs } from "./util";
-import { VideoAsset } from "@/editor/util/EncodedAsset/EncodedAsset";
-import { FetchMixin } from "./FetchMixin";
-import { apiHostContext } from "../gui/EFWorkbench";
-import { consume } from "@lit/context";
-import { EFSourceMixin } from "./EFSourceMixin";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE";
-
-export const deepGetMediaElements = (
-  element: Element,
-  medias: EFMedia[] = [],
-) => {
-  for (const child of element.children) {
-    if (child instanceof EFMedia) {
-      medias.push(child);
-    } else {
-      deepGetMediaElements(child, medias);
-    }
-  }
-  return medias;
-};
-
-export class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
-  assetType: "isobmff_files",
-}) {
-  static styles = [
-    css`
-      :host {
-        display: block;
-        position: relative;
-        overflow: hidden;
-      }
-    `,
-  ];
-
-  @property({ type: Number })
-  currentTimeMs = 0;
-
-  @consume({ context: apiHostContext, subscribe: true })
-  @state()
-  efHost?: string;
-
-  // get requiredAssets() {
-  //   return { [this.md5SumLoader.value]: this.requiredAssetsPath.value ?? [] };
-  // }
-
-  fragmentIndexPath() {
-    if (this.getAttribute("src")?.startsWith("http")) {
-      return this.getAttribute("src")! + "/index";
-    }
-    return `/@ef-track-fragment-index/${this.getAttribute("src") ?? ""}`;
-  }
-
-  fragmentTrackPath(trackId: string) {
-    if (this.getAttribute("src")?.startsWith("http")) {
-      return (
-        this.getAttribute("src")!.replace("files", "tracks") + `/${trackId}`
-      );
-    }
-    return `/@ef-track/${this.getAttribute("src") ?? ""}?trackId=${trackId}`;
-  }
-
-  // requiredAssetsPath = new Task(this, {
-  //   args: () => [this.trackFragmentIndexLoader.value] as const,
-  //   task: ([trackFragmentIndex]) => {
-  //     if (!trackFragmentIndex) {
-  //       return [];
-  //     }
-  //     const paths = [this.fragmentIndexPath()];
-
-  //     Object.values(trackFragmentIndex).forEach((track) => {
-  //       paths.push(this.fragmentTrackPath(String(track.track)));
-  //     });
-
-  //     return paths;
-  //   },
-  // });
-
-  public trackFragmentIndexLoader = new Task(this, {
-    args: () => [this.fragmentIndexPath(), this.fetch] as const,
-    task: async ([fragmentIndexPath, fetch], { signal }) => {
-      console.log("EFMedia trackFragmentIndexLoader");
-      const response = await fetch(fragmentIndexPath, { signal });
-      return (await response.json()) as Record<number, TrackFragmentIndex>;
-    },
-    onComplete: () => {
-      this.requestUpdate("ownCurrentTimeMs");
-      this.parentTimegroup?.requestUpdate("ownCurrentTimeMs");
-    },
-  });
-
-  protected initSegmentsLoader = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () =>
-      [this.trackFragmentIndexLoader.value, this.src, this.fetch] as const,
-    task: async ([fragmentIndex, _src, fetch], { signal }) => {
-      console.log("EFMedia initSegmentsLoader");
-      if (!fragmentIndex) {
-        return;
-      }
-      return await Promise.all(
-        Object.entries(fragmentIndex).map(async ([trackId, track]) => {
-          const start = track.initSegment.offset;
-          const end = track.initSegment.offset + track.initSegment.size - 1;
-          const response = await fetch(this.fragmentTrackPath(trackId), {
-            signal,
-            headers: { Range: `bytes=${start}-${end}` },
-          });
-          const buffer =
-            (await response.arrayBuffer()) as MP4Box.MP4ArrayBuffer;
-          buffer.fileStart = 0;
-          const mp4File = new MP4File();
-          mp4File.appendBuffer(buffer, true);
-          mp4File.flush();
-          await mp4File.readyPromise;
-
-          return { trackId, buffer, mp4File };
-        }),
-      );
-    },
-  });
-
-  get defaultVideoTrackId() {
-    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
-      (track) => track.type === "video",
-    )?.track;
-  }
-
-  get defaultAudioTrackId() {
-    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
-      (track) => track.type === "audio",
-    )?.track;
-  }
-
-  seekTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () =>
-      [
-        this.desiredSeekTimeMs,
-        this.trackFragmentIndexLoader.value,
-        this.initSegmentsLoader.value,
-      ] as const,
-    task: async ([seekToMs, fragmentIndex, initSegments], { signal }) => {
-      console.log("EFMedia seekTask");
-      if (fragmentIndex === undefined) {
-        return;
-      }
-      if (initSegments === undefined) {
-        return;
-      }
-
-      const result: Record<
-        string,
-        { segment: TrackSegment; track: MP4Box.TrackInfo }
-      > = {};
-
-      Object.values(fragmentIndex).forEach((index) => {
-        const track = initSegments
-          .find((segment) => segment.trackId === String(index.track))
-          ?.mp4File.getInfo().tracks[0];
-
-        if (!track) {
-          throw new Error("Could not finding matching track");
-        }
-
-        const segment = index.segments.toReversed().find((segment) => {
-          return (segment.dts / track.timescale) * 1000 <= seekToMs;
-        });
-
-        if (!segment) {
-          return;
-        }
-
-        result[index.track] = { segment, track };
-      });
-
-      return result;
-    },
-  });
-
-  fetchSeekTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    argsEqual: deepArrayEquals,
-    args: () =>
-      [this.initSegmentsLoader.value, this.seekTask.value, this.fetch] as const,
-    task: async ([initSegments, seekResult, fetch], { signal }) => {
-      console.log("EFMedia fetchSeekTask");
-      if (!initSegments) {
-        return;
-      }
-      if (!seekResult) {
-        return;
-      }
-
-      const files: Record<string, File> = {};
-
-      for (const [trackId, { segment, track }] of Object.entries(seekResult)) {
-        const start = segment.offset;
-        const end = segment.offset + segment.size;
-
-        const response = await fetch(this.fragmentTrackPath(trackId), {
-          signal,
-          headers: { Range: `bytes=${start}-${end}` },
-        });
-
-        const initSegment = Object.values(initSegments).find(
-          (initSegment) => initSegment.trackId === String(track.id),
-        );
-        const initBuffer = initSegment!.buffer;
-
-        const mediaBuffer =
-          (await response.arrayBuffer()) as unknown as MP4Box.MP4ArrayBuffer;
-
-        files[trackId] = new File([initBuffer, mediaBuffer], "video.mp4", {
-          type: "video/mp4",
-        });
-      }
-
-      return files;
-    },
-  });
-
-  videoAssetTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.fetchSeekTask.value] as const,
-    task: async ([files], { signal }) => {
-      console.log("EFMedia videoAssetTask");
-      if (!files) {
-        return;
-      }
-      if (!this.defaultVideoTrackId) {
-        return;
-      }
-      const videoFile = files[this.defaultVideoTrackId];
-      if (!videoFile) {
-        return;
-      }
-      return await VideoAsset.createFromReadableStream(
-        "video.mp4",
-        videoFile.stream(),
-        videoFile,
-      );
-    },
-  });
-
-  @state() desiredSeekTimeMs = 0;
-
-  protected async executeSeek(seekToMs: number) {
-    this.desiredSeekTimeMs = seekToMs;
-  }
-
-  protected updated(
-    changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>,
-  ): void {
-    if (changedProperties.has("ownCurrentTimeMs")) {
-      this.executeSeek(this.ownCurrentTimeMs);
-    }
-  }
-
-  get hasOwnDuration() {
-    return true;
-  }
-
-  get durationMs() {
-    if (!this.trackFragmentIndexLoader.value) {
-      return 0;
-    }
-
-    const durations = Object.values(this.trackFragmentIndexLoader.value).map(
-      (track) => {
-        return (track.duration / track.timescale) * 1000;
-      },
-    );
-    if (durations.length === 0) {
-      return 0;
-    }
-    return Math.max(...durations);
-  }
-
-  get startTimeMs() {
-    return getStartTimeMs(this);
-  }
-
-  #audioContext = new OfflineAudioContext(2, 48000 / 30, 48000);
-
-  audioBufferTask = new Task(this, {
-    autoRun: EF_INTERACTIVE,
-    args: () => [this.fetchSeekTask.value, this.seekTask.value] as const,
-    task: async ([files, segments], { signal }) => {
-      console.log("EFMedia audioBufferTask", this.outerHTML);
-      if (!files) {
-        return;
-      }
-      if (!segments) {
-        return;
-      }
-      if (!this.defaultAudioTrackId) {
-        return;
-      }
-      const segment = segments[this.defaultAudioTrackId];
-      if (!segment) {
-        return;
-      }
-      const audioFile = files[this.defaultAudioTrackId];
-      if (!audioFile) {
-        return;
-      }
-      return {
-        buffer: await this.#audioContext.decodeAudioData(
-          await audioFile.arrayBuffer(),
-        ),
-        startOffsetMs: (segment.segment.cts / segment.track.timescale) * 1000,
-      };
-    },
-  });
-
-  async fetchAudioSpanningTime(fromMs: number, toMs: number) {
-    // Adjust range for track's own time
-    fromMs -= this.startTimeMs;
-    toMs -= this.startTimeMs;
-
-    await this.trackFragmentIndexLoader.taskComplete;
-    const audioTrackId = this.defaultAudioTrackId;
-    if (!audioTrackId) {
-      console.warn("No audio track found");
-      return;
-    }
-    const audioTrackIndex = this.trackFragmentIndexLoader.value?.[audioTrackId];
-    if (!audioTrackIndex) {
-      console.warn("No audio track found");
-      return;
-    }
-
-    const start = audioTrackIndex.initSegment.offset;
-    const end =
-      audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size - 1;
-    const audioInitFragmentRequest = this.fetch(
-      this.fragmentTrackPath(String(audioTrackId)),
-      {
-        headers: { Range: `bytes=${start}-${end}` },
-      },
-    );
-
-    const fragments = Object.values(audioTrackIndex.segments).filter(
-      (segment) => {
-        const segmentStartsBeforeEnd =
-          segment.dts <= (toMs * audioTrackIndex.timescale) / 1000;
-        const segmentEndsAfterStart =
-          segment.dts + segment.duration >=
-          (fromMs * audioTrackIndex.timescale) / 1000;
-        return segmentStartsBeforeEnd && segmentEndsAfterStart;
-      },
-    );
-
-    console.log("FRAGMENTS SPANNING TIME", JSON.stringify(fragments));
-    const firstFragment = fragments[0]!;
-    const lastFragment = fragments[fragments.length - 1]!;
-    const fragmentStart = firstFragment.offset;
-    const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
-
-    console.log("FETCHING BYTES", `bytes=${fragmentStart}-${fragmentEnd}`);
-    const audioFragmentRequest = this.fetch(
-      this.fragmentTrackPath(String(audioTrackId)),
-      {
-        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd}` },
-      },
-    );
-
-    const initResponse = await audioInitFragmentRequest;
-    const dataResponse = await audioFragmentRequest;
-
-    const initBuffer = await initResponse.arrayBuffer();
-    const dataBuffer = await dataResponse.arrayBuffer();
-
-    const audioBlob = new Blob([initBuffer, dataBuffer], {
-      type: "audio/mp4",
-    });
-
-    return {
-      blob: audioBlob,
-      startMs: (firstFragment.dts / audioTrackIndex.timescale) * 1000,
-      endMs:
-        (lastFragment.dts / audioTrackIndex.timescale) * 1000 +
-        (lastFragment.duration / audioTrackIndex.timescale) * 1000,
-    };
-  }
-}

package/src/elements/EFSourceMixin.ts

@@ -1,57 +0,0 @@
-import { consume } from "@lit/context";
-import { LitElement } from "lit";
-import { apiHostContext } from "../gui/EFWorkbench";
-import { state } from "lit/decorators/state.js";
-import { Task } from "@lit/task";
-import { property } from "lit/decorators/property.js";
-
-export declare class EFSourceMixinInterface {
-  productionSrc(): string;
-  src: string;
-}
-
-interface EFSourceMixinOptions {
-  assetType: string;
-}
-
-export function EFSourceMixin<T extends Constructor<LitElement>>(
-  superClass: T,
-  options: EFSourceMixinOptions,
-) {
-  class EFSourceElement extends superClass {
-    @consume({ context: apiHostContext, subscribe: true })
-    @state()
-    private efHost?: string;
-
-    @property({ type: String })
-    src = "";
-
-    productionSrc() {
-      if (!this.md5SumLoader.value) {
-        throw new Error(
-          `MD5 sum not available for ${this}. Cannot generate production URL`,
-        );
-      }
-
-      if (!this.efHost) {
-        throw new Error(
-          `efHost not available for ${this}. Cannot generate production URL`,
-        );
-      }
-
-      return `${this.efHost}/api/video2/${options.assetType}/${this.md5SumLoader.value}`;
-    }
-
-    md5SumLoader = new Task(this, {
-      autoRun: false,
-      args: () => [this.src] as const,
-      task: async ([src], { signal }) => {
-        const md5Path = `/@ef-asset/${src}`;
-        const response = await fetch(md5Path, { method: "HEAD", signal });
-        return response.headers.get("etag") ?? undefined;
-      },
-    });
-  }
-
-  return EFSourceElement as Constructor<EFSourceMixinInterface> & T;
-}