@editframe/elements 0.5.0-beta.9 → 0.6.0-beta.1

package/dist/packages/elements/src/elements/EFMedia.cjs

@@ -0,0 +1,334 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const lit = require("lit");
+const EFTemporal = require("./EFTemporal.cjs");
+const decorators_js = require("lit/decorators.js");
+const deepEquals_js = require("@lit/task/deep-equals.js");
+const task = require("@lit/task");
+const MP4File = require("../../../../lib/av/MP4File.cjs");
+const util = require("./util.cjs");
+const EncodedAsset = require("../../../../lib/av/EncodedAsset.cjs");
+const FetchMixin = require("./FetchMixin.cjs");
+const EFWorkbench = require("../gui/EFWorkbench.cjs");
+const context = require("@lit/context");
+const EFSourceMixin = require("./EFSourceMixin.cjs");
+const EF_INTERACTIVE = require("../EF_INTERACTIVE.cjs");
+var __defProp = Object.defineProperty;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = void 0;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = decorator(target, key, result) || result;
+  if (result) __defProp(target, key, result);
+  return result;
+};
+const deepGetMediaElements = (element, medias = []) => {
+  for (const child of Array.from(element.children)) {
+    if (child instanceof EFMedia) {
+      medias.push(child);
+    } else {
+      deepGetMediaElements(child, medias);
+    }
+  }
+  return medias;
+};
+class EFMedia extends EFSourceMixin.EFSourceMixin(EFTemporal.EFTemporal(FetchMixin.FetchMixin(lit.LitElement)), {
+  assetType: "isobmff_files"
+}) {
+  constructor() {
+    super(...arguments);
+    this.currentTimeMs = 0;
+    this.trackFragmentIndexLoader = new task.Task(this, {
+      args: () => [this.fragmentIndexPath(), this.fetch],
+      task: async ([fragmentIndexPath, fetch], { signal }) => {
+        const response = await fetch(fragmentIndexPath, { signal });
+        return await response.json();
+      },
+      onComplete: () => {
+        this.requestUpdate("ownCurrentTimeMs");
+        this.parentTimegroup?.requestUpdate("ownCurrentTimeMs");
+      }
+    });
+    this.initSegmentsLoader = new task.Task(this, {
+      autoRun: EF_INTERACTIVE.EF_INTERACTIVE,
+      args: () => [this.trackFragmentIndexLoader.value, this.src, this.fetch],
+      task: async ([fragmentIndex, _src, fetch], { signal }) => {
+        if (!fragmentIndex) {
+          return;
+        }
+        return await Promise.all(
+          Object.entries(fragmentIndex).map(async ([trackId, track]) => {
+            const start = track.initSegment.offset;
+            const end = track.initSegment.offset + track.initSegment.size - 1;
+            const response = await fetch(this.fragmentTrackPath(trackId), {
+              signal,
+              headers: { Range: `bytes=${start}-${end}` }
+            });
+            const buffer = await response.arrayBuffer();
+            buffer.fileStart = 0;
+            const mp4File = new MP4File.MP4File();
+            mp4File.appendBuffer(buffer, true);
+            mp4File.flush();
+            await mp4File.readyPromise;
+            return { trackId, buffer, mp4File };
+          })
+        );
+      }
+    });
+    this.seekTask = new task.Task(this, {
+      autoRun: EF_INTERACTIVE.EF_INTERACTIVE,
+      args: () => [
+        this.desiredSeekTimeMs,
+        this.trackFragmentIndexLoader.value,
+        this.initSegmentsLoader.value
+      ],
+      task: async ([seekToMs, fragmentIndex, initSegments], { signal: _signal }) => {
+        if (fragmentIndex === void 0) {
+          return;
+        }
+        if (initSegments === void 0) {
+          return;
+        }
+        const result = {};
+        for (const index of Object.values(fragmentIndex)) {
+          const track = initSegments.find((segment2) => segment2.trackId === String(index.track))?.mp4File.getInfo().tracks[0];
+          if (!track) {
+            throw new Error("Could not finding matching track");
+          }
+          const segment = index.segments.toReversed().find((segment2) => {
+            return segment2.dts / track.timescale * 1e3 <= seekToMs;
+          });
+          if (!segment) {
+            return;
+          }
+          result[index.track] = { segment, track };
+        }
+        return result;
+      }
+    });
+    this.fetchSeekTask = new task.Task(this, {
+      autoRun: EF_INTERACTIVE.EF_INTERACTIVE,
+      argsEqual: deepEquals_js.deepArrayEquals,
+      args: () => [this.initSegmentsLoader.value, this.seekTask.value, this.fetch],
+      task: async ([initSegments, seekResult, fetch], { signal }) => {
+        if (!initSegments) {
+          return;
+        }
+        if (!seekResult) {
+          return;
+        }
+        const files = {};
+        for (const [trackId, { segment, track }] of Object.entries(seekResult)) {
+          const start = segment.offset;
+          const end = segment.offset + segment.size;
+          const response = await fetch(this.fragmentTrackPath(trackId), {
+            signal,
+            headers: { Range: `bytes=${start}-${end}` }
+          });
+          const initSegment = Object.values(initSegments).find(
+            (initSegment2) => initSegment2.trackId === String(track.id)
+          );
+          if (!initSegment) {
+            throw new Error("Could not find matching init segment");
+          }
+          const initBuffer = initSegment.buffer;
+          const mediaBuffer = await response.arrayBuffer();
+          files[trackId] = new File([initBuffer, mediaBuffer], "video.mp4", {
+            type: "video/mp4"
+          });
+        }
+        return files;
+      }
+    });
+    this.videoAssetTask = new task.Task(this, {
+      autoRun: EF_INTERACTIVE.EF_INTERACTIVE,
+      args: () => [this.fetchSeekTask.value],
+      task: async ([files], { signal: _signal }) => {
+        if (!files) {
+          return;
+        }
+        if (!this.defaultVideoTrackId) {
+          return;
+        }
+        const videoFile = files[this.defaultVideoTrackId];
+        if (!videoFile) {
+          return;
+        }
+        for (const frame of this.videoAssetTask.value?.decodedFrames || []) {
+          frame.close();
+        }
+        this.videoAssetTask.value?.videoDecoder?.close();
+        return await EncodedAsset.VideoAsset.createFromReadableStream(
+          "video.mp4",
+          videoFile.stream(),
+          videoFile
+        );
+      }
+    });
+    this.desiredSeekTimeMs = 0;
+    this.#audioContext = new OfflineAudioContext(2, 48e3 / 30, 48e3);
+    this.audioBufferTask = new task.Task(this, {
+      autoRun: EF_INTERACTIVE.EF_INTERACTIVE,
+      args: () => [this.fetchSeekTask.value, this.seekTask.value],
+      task: async ([files, segments], { signal: _signal }) => {
+        if (!files) {
+          return;
+        }
+        if (!segments) {
+          return;
+        }
+        if (!this.defaultAudioTrackId) {
+          return;
+        }
+        const segment = segments[this.defaultAudioTrackId];
+        if (!segment) {
+          return;
+        }
+        const audioFile = files[this.defaultAudioTrackId];
+        if (!audioFile) {
+          return;
+        }
+        return {
+          buffer: await this.#audioContext.decodeAudioData(
+            await audioFile.arrayBuffer()
+          ),
+          startOffsetMs: segment.segment.cts / segment.track.timescale * 1e3
+        };
+      }
+    });
+  }
+  static {
+    this.styles = [
+      lit.css`
+      :host {
+        display: block;
+        position: relative;
+        overflow: hidden;
+      }
+    `
+    ];
+  }
+  fragmentIndexPath() {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return `${this.src}/index`;
+    }
+    return `/@ef-track-fragment-index/${this.getAttribute("src") ?? ""}`;
+  }
+  fragmentTrackPath(trackId) {
+    if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
+      return `${this.src.replace("files", "tracks")}/${trackId}`;
+    }
+    return `/@ef-track/${this.src ?? ""}?trackId=${trackId}`;
+  }
+  get defaultVideoTrackId() {
+    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
+      (track) => track.type === "video"
+    )?.track;
+  }
+  get defaultAudioTrackId() {
+    return Object.values(this.trackFragmentIndexLoader.value ?? {}).find(
+      (track) => track.type === "audio"
+    )?.track;
+  }
+  async executeSeek(seekToMs) {
+    this.desiredSeekTimeMs = seekToMs;
+  }
+  updated(changedProperties) {
+    if (changedProperties.has("ownCurrentTimeMs")) {
+      this.executeSeek(this.ownCurrentTimeMs);
+    }
+  }
+  get hasOwnDuration() {
+    return true;
+  }
+  get durationMs() {
+    if (!this.trackFragmentIndexLoader.value) {
+      return 0;
+    }
+    const durations = Object.values(this.trackFragmentIndexLoader.value).map(
+      (track) => {
+        return track.duration / track.timescale * 1e3;
+      }
+    );
+    if (durations.length === 0) {
+      return 0;
+    }
+    return Math.max(...durations);
+  }
+  get startTimeMs() {
+    return util.getStartTimeMs(this);
+  }
+  #audioContext;
+  async fetchAudioSpanningTime(fromMs, toMs) {
+    fromMs -= this.startTimeMs;
+    toMs -= this.startTimeMs;
+    await this.trackFragmentIndexLoader.taskComplete;
+    const audioTrackId = this.defaultAudioTrackId;
+    if (!audioTrackId) {
+      console.warn("No audio track found");
+      return;
+    }
+    const audioTrackIndex = this.trackFragmentIndexLoader.value?.[audioTrackId];
+    if (!audioTrackIndex) {
+      console.warn("No audio track found");
+      return;
+    }
+    const start = audioTrackIndex.initSegment.offset;
+    const end = audioTrackIndex.initSegment.offset + audioTrackIndex.initSegment.size - 1;
+    const audioInitFragmentRequest = this.fetch(
+      this.fragmentTrackPath(String(audioTrackId)),
+      {
+        headers: { Range: `bytes=${start}-${end}` }
+      }
+    );
+    const fragments = Object.values(audioTrackIndex.segments).filter(
+      (segment) => {
+        const segmentStartsBeforeEnd = segment.dts <= toMs * audioTrackIndex.timescale / 1e3;
+        const segmentEndsAfterStart = segment.dts + segment.duration >= fromMs * audioTrackIndex.timescale / 1e3;
+        return segmentStartsBeforeEnd && segmentEndsAfterStart;
+      }
+    );
+    const firstFragment = fragments[0];
+    if (!firstFragment) {
+      console.warn("No audio fragments found");
+      return;
+    }
+    const lastFragment = fragments[fragments.length - 1];
+    if (!lastFragment) {
+      console.warn("No audio fragments found");
+      return;
+    }
+    const fragmentStart = firstFragment.offset;
+    const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
+    const audioFragmentRequest = this.fetch(
+      this.fragmentTrackPath(String(audioTrackId)),
+      {
+        headers: { Range: `bytes=${fragmentStart}-${fragmentEnd}` }
+      }
+    );
+    const initResponse = await audioInitFragmentRequest;
+    const dataResponse = await audioFragmentRequest;
+    const initBuffer = await initResponse.arrayBuffer();
+    const dataBuffer = await dataResponse.arrayBuffer();
+    const audioBlob = new Blob([initBuffer, dataBuffer], {
+      type: "audio/mp4"
+    });
+    return {
+      blob: audioBlob,
+      startMs: firstFragment.dts / audioTrackIndex.timescale * 1e3,
+      endMs: lastFragment.dts / audioTrackIndex.timescale * 1e3 + lastFragment.duration / audioTrackIndex.timescale * 1e3
+    };
+  }
+}
+__decorateClass([
+  decorators_js.property({ type: Number })
+], EFMedia.prototype, "currentTimeMs");
+__decorateClass([
+  context.consume({ context: EFWorkbench.apiHostContext, subscribe: true }),
+  decorators_js.state()
+], EFMedia.prototype, "efHost");
+__decorateClass([
+  decorators_js.state()
+], EFMedia.prototype, "desiredSeekTimeMs");
+exports.EFMedia = EFMedia;
+exports.deepGetMediaElements = deepGetMediaElements;

package/dist/packages/elements/src/elements/EFMedia.d.ts

@@ -0,0 +1,61 @@
+import { LitElement, PropertyValueMap } from 'lit';
+import { Task } from '@lit/task';
+import { MP4File } from '../../../../lib/av/MP4File';
+import { TrackFragmentIndex, TrackSegment } from '../../../assets';
+import { VideoAsset } from '../../../../lib/av/EncodedAsset';
+
+import type * as MP4Box from "mp4box";
+export declare const deepGetMediaElements: (element: Element, medias?: EFMedia[]) => EFMedia[];
+declare const EFMedia_base: (new (...args: any[]) => import('./EFSourceMixin').EFSourceMixinInterface) & (new (...args: any[]) => import('./EFTemporal').TemporalMixinInterface) & (new (...args: any[]) => import('./FetchMixin').FetchMixinInterface) & typeof LitElement;
+export declare class EFMedia extends EFMedia_base {
+    #private;
+    static styles: import('lit').CSSResult[];
+    currentTimeMs: number;
+    efHost?: string;
+    fragmentIndexPath(): string;
+    fragmentTrackPath(trackId: string): string;
+    trackFragmentIndexLoader: Task<readonly [string, typeof fetch], Record<number, TrackFragmentIndex>>;
+    protected initSegmentsLoader: Task<readonly [Record<number, TrackFragmentIndex> | undefined, string, typeof fetch], {
+        trackId: string;
+        buffer: MP4Box.MP4ArrayBuffer;
+        mp4File: MP4File;
+    }[] | undefined>;
+    get defaultVideoTrackId(): number | undefined;
+    get defaultAudioTrackId(): number | undefined;
+    seekTask: Task<readonly [number, Record<number, TrackFragmentIndex> | undefined, {
+        trackId: string;
+        buffer: MP4Box.MP4ArrayBuffer;
+        mp4File: MP4File;
+    }[] | undefined], Record<string, {
+        segment: TrackSegment;
+        track: MP4Box.TrackInfo;
+    }> | undefined>;
+    fetchSeekTask: Task<readonly [{
+        trackId: string;
+        buffer: MP4Box.MP4ArrayBuffer;
+        mp4File: MP4File;
+    }[] | undefined, Record<string, {
+        segment: TrackSegment;
+        track: MP4Box.TrackInfo;
+    }> | undefined, typeof fetch], Record<string, File> | undefined>;
+    videoAssetTask: Task<readonly [Record<string, File> | undefined], VideoAsset | undefined>;
+    desiredSeekTimeMs: number;
+    protected executeSeek(seekToMs: number): Promise<void>;
+    protected updated(changedProperties: PropertyValueMap<any> | Map<PropertyKey, unknown>): void;
+    get hasOwnDuration(): boolean;
+    get durationMs(): number;
+    get startTimeMs(): number;
+    audioBufferTask: Task<readonly [Record<string, File> | undefined, Record<string, {
+        segment: TrackSegment;
+        track: MP4Box.TrackInfo;
+    }> | undefined], {
+        buffer: AudioBuffer;
+        startOffsetMs: number;
+    } | undefined>;
+    fetchAudioSpanningTime(fromMs: number, toMs: number): Promise<{
+        blob: Blob;
+        startMs: number;
+        endMs: number;
+    } | undefined>;
+}
+export {};

package/dist/packages/elements/src/elements/{EFMedia.mjs → EFMedia.js}

@@ -1,16 +1,16 @@
 import { css, LitElement } from "lit";
-import { EFTemporal } from "./EFTemporal.mjs";
+import { EFTemporal } from "./EFTemporal.js";
 import { property, state } from "lit/decorators.js";
 import { deepArrayEquals } from "@lit/task/deep-equals.js";
 import { Task } from "@lit/task";
-import { MP4File } from "../../../../lib/av/MP4File.mjs";
-import { getStartTimeMs } from "./util.mjs";
-import { VideoAsset } from "../../../../lib/av/EncodedAsset.mjs";
-import { FetchMixin } from "./FetchMixin.mjs";
-import { apiHostContext } from "../gui/EFWorkbench.mjs";
+import { MP4File } from "../../../../lib/av/MP4File.js";
+import { getStartTimeMs } from "./util.js";
+import { VideoAsset } from "../../../../lib/av/EncodedAsset.js";
+import { FetchMixin } from "./FetchMixin.js";
+import { apiHostContext } from "../gui/EFWorkbench.js";
 import { consume } from "@lit/context";
-import { EFSourceMixin } from "./EFSourceMixin.mjs";
-import { EF_INTERACTIVE } from "../EF_INTERACTIVE.mjs";
+import { EFSourceMixin } from "./EFSourceMixin.js";
+import { EF_INTERACTIVE } from "../EF_INTERACTIVE.js";
 var __defProp = Object.defineProperty;
 var __decorateClass = (decorators, target, key, kind) => {
   var result = void 0;
@@ -80,7 +80,7 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
         this.trackFragmentIndexLoader.value,
         this.initSegmentsLoader.value
       ],
-      task: async ([seekToMs, fragmentIndex, initSegments], { signal }) => {
+      task: async ([seekToMs, fragmentIndex, initSegments], { signal: _signal }) => {
         if (fragmentIndex === void 0) {
           return;
         }
@@ -88,7 +88,7 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
           return;
         }
         const result = {};
-        Object.values(fragmentIndex).forEach((index) => {
+        for (const index of Object.values(fragmentIndex)) {
           const track = initSegments.find((segment2) => segment2.trackId === String(index.track))?.mp4File.getInfo().tracks[0];
           if (!track) {
             throw new Error("Could not finding matching track");
@@ -100,7 +100,7 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
             return;
           }
           result[index.track] = { segment, track };
-        });
+        }
         return result;
       }
     });
@@ -126,6 +126,9 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
           const initSegment = Object.values(initSegments).find(
             (initSegment2) => initSegment2.trackId === String(track.id)
           );
+          if (!initSegment) {
+            throw new Error("Could not find matching init segment");
+          }
           const initBuffer = initSegment.buffer;
           const mediaBuffer = await response.arrayBuffer();
           files[trackId] = new File([initBuffer, mediaBuffer], "video.mp4", {
@@ -138,7 +141,7 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     this.videoAssetTask = new Task(this, {
       autoRun: EF_INTERACTIVE,
       args: () => [this.fetchSeekTask.value],
-      task: async ([files], { signal }) => {
+      task: async ([files], { signal: _signal }) => {
         if (!files) {
           return;
         }
@@ -149,9 +152,9 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
         if (!videoFile) {
           return;
         }
-        this.videoAssetTask.value?.decodedFrames.forEach(
-          (frame) => frame.close()
-        );
+        for (const frame of this.videoAssetTask.value?.decodedFrames || []) {
+          frame.close();
+        }
         this.videoAssetTask.value?.videoDecoder?.close();
         return await VideoAsset.createFromReadableStream(
           "video.mp4",
@@ -165,7 +168,7 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
     this.audioBufferTask = new Task(this, {
       autoRun: EF_INTERACTIVE,
       args: () => [this.fetchSeekTask.value, this.seekTask.value],
-      task: async ([files, segments], { signal }) => {
+      task: async ([files, segments], { signal: _signal }) => {
         if (!files) {
           return;
         }
@@ -205,13 +208,13 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
   }
   fragmentIndexPath() {
     if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
-      return this.src + "/index";
+      return `${this.src}/index`;
     }
     return `/@ef-track-fragment-index/${this.getAttribute("src") ?? ""}`;
   }
   fragmentTrackPath(trackId) {
     if (this.src.startsWith("editframe://") || this.src.startsWith("http")) {
-      return this.src.replace("files", "tracks") + `/${trackId}`;
+      return `${this.src.replace("files", "tracks")}/${trackId}`;
     }
     return `/@ef-track/${this.src ?? ""}?trackId=${trackId}`;
   }
@@ -284,7 +287,15 @@ class EFMedia extends EFSourceMixin(EFTemporal(FetchMixin(LitElement)), {
       }
     );
     const firstFragment = fragments[0];
+    if (!firstFragment) {
+      console.warn("No audio fragments found");
+      return;
+    }
     const lastFragment = fragments[fragments.length - 1];
+    if (!lastFragment) {
+      console.warn("No audio fragments found");
+      return;
+    }
     const fragmentStart = firstFragment.offset;
     const fragmentEnd = lastFragment.offset + lastFragment.size - 1;
     const audioFragmentRequest = this.fetch(

package/dist/packages/elements/src/elements/EFSourceMixin.cjs

@@ -0,0 +1,55 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const context = require("@lit/context");
+const EFWorkbench = require("../gui/EFWorkbench.cjs");
+const state_js = require("lit/decorators/state.js");
+const task = require("@lit/task");
+const property_js = require("lit/decorators/property.js");
+var __defProp = Object.defineProperty;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = void 0;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = decorator(target, key, result) || result;
+  if (result) __defProp(target, key, result);
+  return result;
+};
+function EFSourceMixin(superClass, options) {
+  class EFSourceElement extends superClass {
+    constructor() {
+      super(...arguments);
+      this.src = "";
+      this.md5SumLoader = new task.Task(this, {
+        autoRun: false,
+        args: () => [this.src],
+        task: async ([src], { signal }) => {
+          const md5Path = `/@ef-asset/${src}`;
+          const response = await fetch(md5Path, { method: "HEAD", signal });
+          return response.headers.get("etag") ?? void 0;
+        }
+      });
+    }
+    productionSrc() {
+      if (!this.md5SumLoader.value) {
+        throw new Error(
+          `MD5 sum not available for ${this}. Cannot generate production URL`
+        );
+      }
+      if (!this.efHost) {
+        throw new Error(
+          `efHost not available for ${this}. Cannot generate production URL`
+        );
+      }
+      return `${this.efHost}/api/video2/${options.assetType}/${this.md5SumLoader.value}`;
+    }
+  }
+  __decorateClass([
+    context.consume({ context: EFWorkbench.apiHostContext, subscribe: true }),
+    state_js.state()
+  ], EFSourceElement.prototype, "efHost");
+  __decorateClass([
+    property_js.property({ type: String })
+  ], EFSourceElement.prototype, "src");
+  return EFSourceElement;
+}
+exports.EFSourceMixin = EFSourceMixin;

package/dist/packages/elements/src/elements/EFSourceMixin.d.ts

@@ -0,0 +1,12 @@
+import { LitElement } from 'lit';
+
+export declare class EFSourceMixinInterface {
+    productionSrc(): string;
+    src: string;
+}
+interface EFSourceMixinOptions {
+    assetType: string;
+}
+type Constructor<T = {}> = new (...args: any[]) => T;
+export declare function EFSourceMixin<T extends Constructor<LitElement>>(superClass: T, options: EFSourceMixinOptions): Constructor<EFSourceMixinInterface> & T;
+export {};

package/dist/packages/elements/src/elements/{EFSourceMixin.mjs → EFSourceMixin.js}

@@ -1,5 +1,5 @@
 import { consume } from "@lit/context";
-import { apiHostContext } from "../gui/EFWorkbench.mjs";
+import { apiHostContext } from "../gui/EFWorkbench.js";
 import { state } from "lit/decorators/state.js";
 import { Task } from "@lit/task";
 import { property } from "lit/decorators/property.js";