@editframe/assets 0.7.0-beta.8 → 0.8.0-beta.2

package/dist/EncodedAsset.d.ts

@@ -0,0 +1,102 @@
+import { ReadableStream as ReadableStreamNode } from 'node:stream/web';
+
+import * as MP4Box from "mp4box";
+type AnyReadableStream = ReadableStream<Uint8Array> | ReadableStreamNode<Uint8Array>;
+type FrameCallback = (frame: VideoFrame) => void;
+export interface FetchContext {
+    fetchConfig: RequestInit;
+    origin: string;
+}
+export declare class AssetNotAvailableLocally extends Error {
+}
+export declare class FileAsset {
+    localName: string;
+    protected readonly file: File;
+    constructor(localName: string, file: File);
+    arrayBuffer(): Promise<ArrayBuffer>;
+    get byteSize(): number;
+    get fileExtension(): string | undefined;
+    slice(start: number, end: number): Blob;
+}
+export declare class ISOFileAsset extends FileAsset {
+    localName: string;
+    protected readonly file: File;
+    readonly mp4boxFile: MP4Box.ISOFile;
+    constructor(localName: string, file: File, mp4boxFile: MP4Box.ISOFile);
+    get fileInfo(): MP4Box.Info;
+    get containerFormat(): "mp4";
+}
+export declare class VideoAsset extends ISOFileAsset {
+    static createFromReadableStream(id: string, stream: AnyReadableStream, file: File): Promise<VideoAsset>;
+    videoDecoder: VideoDecoder;
+    lastDecodedSample?: MP4Box.Sample;
+    lastSoughtFrame?: VideoFrame;
+    decodedFrames: VideoFrame[];
+    requestedSampleNumber: number;
+    outCursor: number;
+    sampleCursor: number;
+    /**
+     * **Only use this function in tests to reset a VideoAsset to its initial state.**
+     *
+     * @deprecated
+     */
+    TEST_ONLY_RESET(): Promise<void>;
+    private readonly eventListeners;
+    addEventListener(type: "frame", callback: FrameCallback): void;
+    removeEventListener(type: "frame", callback: FrameCallback): void;
+    emit(type: "frame", frame: VideoFrame): void;
+    constructor(localName: string, mp4boxFile: MP4Box.ISOFile, file: File);
+    get videoCodec(): string;
+    get fragmentInfo(): {
+        offset: number;
+        size: number;
+        start_ms: number;
+        duration_ms: number;
+    }[];
+    pruneBuffer(): void;
+    get editsOffset(): number;
+    waitUntilVideoQueueDrained(): Promise<void>;
+    get canDecodeNextSample(): boolean;
+    decodeNextSample(): Promise<void>;
+    decodeSlice(start: number, end: number): Promise<void>;
+    get decoderConfiguration(): {
+        codec: string;
+        codedWidth: number;
+        codedHeight: number;
+        optimizeForLatency: true;
+        description: Uint8Array;
+    };
+    /**
+     * Configures the video decoder with the appropriate codec, dimensions, and hardware acceleration settings.
+     * If the decoder is already configured, it will be reset before being reconfigured.
+     */
+    configureDecoder(): void;
+    getSample(index?: number): MP4Box.Sample;
+    get timescale(): number;
+    get samples(): MP4Box.Sample[];
+    get displayOrderedSamples(): MP4Box.Sample[];
+    getSampleClosetToTime(seconds: number): MP4Box.Sample;
+    seekingWillEmitNewFrame(seconds: number): boolean;
+    seekingWillSkipPictureGroup(seconds: number): boolean;
+    seekingWillGoBackwards(seconds: number): boolean;
+    latestSeekCts: number;
+    seekToTime(seconds: number): Promise<VideoFrame | undefined>;
+    get defaultVideoTrack(): MP4Box.VideoTrackInfo | undefined;
+    get defaultVideoTrak(): MP4Box.TrakBox;
+    get duration(): number;
+}
+export declare class AudioAsset extends ISOFileAsset {
+    static createFromReadableStream(id: string, stream: AnyReadableStream, file: File): Promise<AudioAsset>;
+    get defaultAudioTrack(): MP4Box.AudioTrackInfo | undefined;
+    get defaultAudioTrak(): MP4Box.TrakBox;
+    get audioCodec(): string;
+    get samplerate(): number;
+    get channelCount(): number;
+}
+export declare class ImageAsset extends FileAsset {
+    static createFromReadableStream(id: string, file: File): Promise<ImageAsset>;
+    get objectUrl(): string;
+    get format(): string | undefined;
+    get type(): string;
+}
+export {};

package/dist/EncodedAsset.js

@@ -0,0 +1,564 @@
+import * as MP4Box from "mp4box";
+import debug from "debug";
+import { memoize } from "./memoize.js";
+import { MP4File } from "./MP4File.js";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = __getOwnPropDesc(target, key);
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = decorator(target, key, result) || result;
+  if (result) __defProp(target, key, result);
+  return result;
+};
+const log = debug("ef:av");
+const BUFFER_SIZE = 10;
+class AssetNotAvailableLocally extends Error {
+}
+class FileAsset {
+  constructor(localName, file) {
+    this.localName = localName;
+    this.file = file;
+  }
+  async arrayBuffer() {
+    return this.file.arrayBuffer();
+  }
+  get byteSize() {
+    return this.file.size;
+  }
+  get fileExtension() {
+    return this.file.name.split(".").pop();
+  }
+  slice(start, end) {
+    return this.file.slice(start, end);
+  }
+}
+class ISOFileAsset extends FileAsset {
+  constructor(localName, file, mp4boxFile) {
+    super(localName, file);
+    this.localName = localName;
+    this.file = file;
+    this.mp4boxFile = mp4boxFile;
+  }
+  get fileInfo() {
+    return this.mp4boxFile.getInfo();
+  }
+  get containerFormat() {
+    return "mp4";
+  }
+}
+__decorateClass([
+  memoize
+], ISOFileAsset.prototype, "fileInfo");
+const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
+  constructor(localName, mp4boxFile, file) {
+    super(localName, file, mp4boxFile);
+    this.decodedFrames = [];
+    this.requestedSampleNumber = 0;
+    this.outCursor = 0;
+    this.sampleCursor = 0;
+    this.eventListeners = {};
+    this.latestSeekCts = 0;
+    this.videoDecoder = new VideoDecoder({
+      error: (e) => {
+        console.error("Video Decoder Error", e);
+      },
+      // eslint-disable-next-line @typescript-eslint/no-misused-promises
+      output: async (decodedFrame) => {
+        const clone = decodedFrame.clone();
+        log("🖼️ frame cts=", decodedFrame.timestamp);
+        this.decodedFrames.push(clone);
+        this.pruneBuffer();
+        decodedFrame.close();
+        this.outCursor = this.samples.findIndex(
+          (sample) => sample.cts === decodedFrame.timestamp
+        );
+        log(`cursor=${this.sampleCursor} outCursor=${this.outCursor}`);
+        this.emit("frame", clone);
+      }
+    });
+    this.configureDecoder();
+  }
+  static async createFromReadableStream(id, stream, file) {
+    let fileStart = 0;
+    const inputFile = new MP4File();
+    const reader = stream.getReader();
+    const processChunk = ({
+      done,
+      value
+    }) => {
+      if (done) {
+        return;
+      }
+      if (!value) {
+        return;
+      }
+      const mp4buffer = value.buffer;
+      mp4buffer.fileStart = fileStart;
+      const isLast = file.size === fileStart + value.byteLength;
+      inputFile.appendBuffer(mp4buffer, isLast);
+      fileStart += value.byteLength;
+      return reader.read().then(processChunk);
+    };
+    await reader.read().then(processChunk);
+    return new _VideoAsset2(id, inputFile, file);
+  }
+  /**
+   * **Only use this function in tests to reset a VideoAsset to its initial state.**
+   *
+   * @deprecated
+   */
+  async TEST_ONLY_RESET() {
+    await this.videoDecoder.flush();
+    this.configureDecoder();
+    this.requestedSampleNumber = 0;
+    this.outCursor = 0;
+    this.sampleCursor = 0;
+    for (const frame of this.decodedFrames) {
+      frame.close();
+    }
+    this.decodedFrames = [];
+    this.lastDecodedSample = void 0;
+    this.lastSoughtFrame?.close();
+    this.lastSoughtFrame = void 0;
+  }
+  addEventListener(type, callback) {
+    this.eventListeners[type] ||= /* @__PURE__ */ new Set();
+    this.eventListeners[type]?.add(callback);
+  }
+  removeEventListener(type, callback) {
+    this.eventListeners[type]?.delete(callback);
+  }
+  emit(type, ...args) {
+    for (const listener of this.eventListeners[type] ?? []) {
+      listener(...args);
+    }
+  }
+  get videoCodec() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    return this.defaultVideoTrack?.codec;
+  }
+  get fragmentInfo() {
+    const fragments = [];
+    const [first, ...samples] = this.samples;
+    if (!first) {
+      return fragments;
+    }
+    let currentFragment = {
+      offset: first.offset,
+      size: first.size,
+      start_ms: first.cts,
+      duration_ms: 0
+    };
+    for (const sample of samples) {
+      if (sample.is_sync) {
+        if (currentFragment) {
+          currentFragment.duration_ms = sample.cts - currentFragment.start_ms;
+          fragments.push(currentFragment);
+        }
+        currentFragment = {
+          offset: sample.offset,
+          size: sample.size,
+          start_ms: sample.cts,
+          duration_ms: 0
+        };
+      } else {
+        currentFragment.size += sample.size;
+      }
+    }
+    return fragments;
+  }
+  pruneBuffer() {
+    if (this.decodedFrames.length > BUFFER_SIZE) {
+      this.decodedFrames.shift()?.close();
+    }
+  }
+  get editsOffset() {
+    if (!this.defaultVideoTrack?.edits) {
+      return 0;
+    }
+    return this.defaultVideoTrack.edits.reduce((acc, edit) => {
+      return acc + edit.media_time;
+    }, 0);
+  }
+  async waitUntilVideoQueueDrained() {
+    if (this.videoDecoder.decodeQueueSize === 0) {
+      return;
+    }
+    await new Promise((resolve) => {
+      this.videoDecoder.addEventListener(
+        "dequeue",
+        () => {
+          resolve();
+        },
+        { once: true }
+      );
+    });
+    await this.waitUntilVideoQueueDrained();
+  }
+  get canDecodeNextSample() {
+    return this.sampleCursor < this.samples.length;
+  }
+  async decodeNextSample() {
+    if (!this.canDecodeNextSample) {
+      throw new Error("No more samples to decode");
+    }
+    await this.decodeSlice(this.sampleCursor, this.sampleCursor);
+    this.sampleCursor++;
+  }
+  async decodeSlice(start, end) {
+    const samples = this.samples.slice(start, end + 1);
+    const firstSample = samples[0];
+    const lastSample = samples[samples.length - 1];
+    if (!firstSample || !lastSample) {
+      throw new Error("Samples not found");
+    }
+    const sliceStart = firstSample.offset;
+    const sliceEnd = lastSample.offset + lastSample.size;
+    const buffer = await this.file.slice(sliceStart, sliceEnd).arrayBuffer();
+    const firstSampleOffset = firstSample.offset;
+    for (let i = start; i <= end; i++) {
+      await this.waitUntilVideoQueueDrained();
+      const sample = this.getSample(i);
+      log("Decoding sample #", i, `cts=${sample.cts}`);
+      const sampleStart = sample.offset - firstSampleOffset;
+      const sampleEnd = sample.offset + sample.size - firstSampleOffset;
+      const chunk = new EncodedVideoChunk({
+        data: buffer.slice(sampleStart, sampleEnd),
+        timestamp: sample.cts,
+        duration: sample.duration,
+        type: sample.is_sync ? "key" : "delta"
+      });
+      this.videoDecoder.decode(chunk);
+      const nextSample = this.defaultVideoTrak?.samples?.[i + 1];
+      if (nextSample === void 0) {
+        log("ENDFLUSH");
+        await this.videoDecoder.flush();
+      }
+    }
+  }
+  get decoderConfiguration() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    let description = new Uint8Array();
+    const trak = this.mp4boxFile.getTrackById(this.defaultVideoTrack.id);
+    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+      if (entry.avcC ?? entry.hvcC) {
+        const stream = new MP4Box.DataStream(
+          void 0,
+          0,
+          MP4Box.DataStream.BIG_ENDIAN
+        );
+        if (entry.avcC) {
+          entry.avcC.write(stream);
+        } else {
+          entry.hvcC.write(stream);
+        }
+        description = new Uint8Array(stream.buffer, 8);
+        break;
+      }
+    }
+    return {
+      codec: this.defaultVideoTrack.codec,
+      codedWidth: this.defaultVideoTrack.track_width,
+      codedHeight: this.defaultVideoTrack.track_height,
+      optimizeForLatency: true,
+      description
+    };
+  }
+  /**
+   * Configures the video decoder with the appropriate codec, dimensions, and hardware acceleration settings.
+   * If the decoder is already configured, it will be reset before being reconfigured.
+   */
+  configureDecoder() {
+    if (this.videoDecoder.state === "configured") {
+      this.videoDecoder.reset();
+    }
+    log("Attempting to configure decoder", this.decoderConfiguration);
+    this.videoDecoder.configure(this.decoderConfiguration);
+  }
+  // Default to -1 to throw error if called without an index
+  getSample(index = -1) {
+    const sample = this.samples?.[index];
+    if (!sample) {
+      throw new Error(`Sample not found at index ${index}`);
+    }
+    return sample;
+  }
+  get timescale() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    return this.defaultVideoTrack.timescale;
+  }
+  get samples() {
+    if (!this.defaultVideoTrak.samples) {
+      throw new Error("No video samples found");
+    }
+    return this.defaultVideoTrak.samples;
+  }
+  get displayOrderedSamples() {
+    return Array.from(this.samples).sort((a, b) => {
+      return a.cts - b.cts;
+    });
+  }
+  getSampleClosetToTime(seconds) {
+    const targetTime = Math.round(seconds * this.timescale + this.editsOffset);
+    const sampleIndex = this.displayOrderedSamples.findIndex(
+      (sample) => sample.cts >= targetTime
+    );
+    if (sampleIndex === -1) {
+      return this.displayOrderedSamples[this.displayOrderedSamples.length - 1];
+    }
+    return this.displayOrderedSamples[sampleIndex];
+  }
+  seekingWillEmitNewFrame(seconds) {
+    if (!this.lastSoughtFrame) {
+      return true;
+    }
+    if (this.seekingWillGoBackwards(seconds)) {
+      return true;
+    }
+    const nextCts = this.getSampleClosetToTime(seconds).cts;
+    return nextCts > this.lastSoughtFrame.timestamp;
+  }
+  seekingWillSkipPictureGroup(seconds) {
+    let start = this.sampleCursor;
+    const end = this.getSampleClosetToTime(seconds).number;
+    let syncFrameCrossings = 0;
+    while (start <= end) {
+      const sample = this.getSample(start);
+      if (sample.is_sync) {
+        if (syncFrameCrossings > 1) {
+          return true;
+        }
+        syncFrameCrossings++;
+      }
+      start++;
+    }
+    return false;
+  }
+  seekingWillGoBackwards(seconds) {
+    const targetSample = this.getSampleClosetToTime(seconds);
+    const targetIndex = this.displayOrderedSamples.indexOf(targetSample);
+    const targetInCache = this.decodedFrames.find(
+      (frame) => frame.timestamp === targetSample.cts
+    );
+    const atEnd = this.sampleCursor === this.samples.length - 1;
+    log(
+      "this.outCursor <= targetSample.number",
+      this.outCursor <= targetSample.number
+    );
+    log("this.outCursor <= targetIndex", this.outCursor <= targetIndex);
+    if (atEnd) {
+      return false;
+    }
+    if (targetInCache) {
+      return false;
+    }
+    log("========");
+    log("sampleCursor", this.sampleCursor);
+    log("   outCursor", this.outCursor);
+    log("      target", targetSample.number);
+    log(" targetIndex", targetIndex);
+    log("     inCache", !!targetInCache);
+    log("       atEnd", atEnd);
+    return this.outCursor > targetIndex;
+  }
+  async seekToTime(seconds) {
+    const sample = this.getSampleClosetToTime(seconds);
+    const cts = sample.cts;
+    this.latestSeekCts = cts;
+    const alreadyDecodedFrame = this.decodedFrames.find(
+      (f) => f.timestamp === cts
+    );
+    if (alreadyDecodedFrame) {
+      return alreadyDecodedFrame;
+    }
+    if (this.seekingWillSkipPictureGroup(seconds)) {
+      await this.videoDecoder.flush();
+      let syncSampleNumber = sample.number;
+      while (!this.getSample(syncSampleNumber).is_sync) {
+        syncSampleNumber--;
+        if (syncSampleNumber < 0) {
+          throw new Error("No sync sample found when traversing backwards");
+        }
+      }
+      this.sampleCursor = syncSampleNumber;
+    }
+    if (this.seekingWillGoBackwards(seconds)) {
+      log("BACKWARDS FLUSH");
+      await this.videoDecoder.flush();
+      for (const frame2 of this.decodedFrames) {
+        frame2.close();
+      }
+      this.decodedFrames = [];
+      let syncSampleNumber = sample.number;
+      while (!this.getSample(syncSampleNumber).is_sync) {
+        syncSampleNumber--;
+        if (syncSampleNumber < 0) {
+          throw new Error("No sync sample found when traversing backwards");
+        }
+      }
+      this.sampleCursor = syncSampleNumber;
+    }
+    let frame;
+    const maybeFrame = (_frame) => {
+      if (frame) {
+        return;
+      }
+      log("Maybe frame", _frame.timestamp, cts);
+      if (_frame.timestamp === cts) {
+        this.removeEventListener("frame", maybeFrame);
+        frame = _frame;
+      }
+    };
+    this.addEventListener("frame", maybeFrame);
+    while (frame === void 0 && this.canDecodeNextSample) {
+      await this.decodeNextSample();
+    }
+    this.removeEventListener("frame", maybeFrame);
+    if (frame) {
+      if (this.lastSoughtFrame && !this.decodedFrames.includes(this.lastSoughtFrame)) {
+        try {
+          this.lastSoughtFrame.close();
+        } catch (error) {
+        }
+      }
+      this.lastSoughtFrame = frame;
+    }
+    return frame;
+  }
+  get defaultVideoTrack() {
+    return this.fileInfo.videoTracks[0];
+  }
+  get defaultVideoTrak() {
+    return this.mp4boxFile.getTrackById(this.defaultVideoTrack?.id ?? -1);
+  }
+  get duration() {
+    return this.fileInfo.duration / this.fileInfo.timescale;
+  }
+};
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "editsOffset");
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "timescale");
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "samples");
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "displayOrderedSamples");
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "defaultVideoTrack");
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "defaultVideoTrak");
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "duration");
+let VideoAsset = _VideoAsset;
+const _AudioAsset = class _AudioAsset2 extends ISOFileAsset {
+  static async createFromReadableStream(id, stream, file) {
+    let fileStart = 0;
+    const inputFile = new MP4File();
+    const reader = stream.getReader();
+    const processChunk = ({
+      done,
+      value
+    }) => {
+      if (done) {
+        return;
+      }
+      if (!value) {
+        return;
+      }
+      const mp4buffer = value.buffer;
+      mp4buffer.fileStart = fileStart;
+      fileStart += value.byteLength;
+      inputFile.appendBuffer(mp4buffer);
+      return reader.read().then(processChunk);
+    };
+    await reader.read().then(processChunk);
+    return new _AudioAsset2(id, file, inputFile);
+  }
+  get defaultAudioTrack() {
+    return this.fileInfo.audioTracks[0];
+  }
+  get defaultAudioTrak() {
+    return this.mp4boxFile.getTrackById(this.defaultAudioTrack?.id ?? -1);
+  }
+  get audioCodec() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.codec;
+  }
+  get samplerate() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.audio.sample_rate;
+  }
+  get channelCount() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.audio.channel_count;
+  }
+};
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "defaultAudioTrack");
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "defaultAudioTrak");
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "audioCodec");
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "samplerate");
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "channelCount");
+let AudioAsset = _AudioAsset;
+const _ImageAsset = class _ImageAsset2 extends FileAsset {
+  static async createFromReadableStream(id, file) {
+    if (file.size === 0) {
+      throw new AssetNotAvailableLocally();
+    }
+    return new _ImageAsset2(id, file);
+  }
+  get objectUrl() {
+    return URL.createObjectURL(this.file);
+  }
+  get format() {
+    return this.fileExtension;
+  }
+  get type() {
+    return `image/${this.format}`;
+  }
+};
+__decorateClass([
+  memoize
+], _ImageAsset.prototype, "objectUrl");
+__decorateClass([
+  memoize
+], _ImageAsset.prototype, "format");
+let ImageAsset = _ImageAsset;
+export {
+  AssetNotAvailableLocally,
+  AudioAsset,
+  FileAsset,
+  ISOFileAsset,
+  ImageAsset,
+  VideoAsset
+};

package/dist/MP4File.d.ts

@@ -0,0 +1,31 @@
+import * as MP4Box from "mp4box";
+export declare class MP4File extends MP4Box.ISOFile {
+    readyPromise: Promise<void>;
+    setSegmentOptions(id: number, user: any, options: MP4Box.SegmentOptions): void;
+    /**
+     * Fragments all tracks in a file into separate array buffers.
+     */
+    fragmentAllTracks(): Promise<Record<number, ArrayBuffer[]>>;
+    fragmentIterator(): AsyncGenerator<{
+        track: number;
+        segment: "init";
+        data: ArrayBuffer;
+        complete: false;
+        cts?: undefined;
+        dts?: undefined;
+        duration?: undefined;
+    } | {
+        track: number;
+        segment: number;
+        data: ArrayBuffer;
+        complete: boolean;
+        cts: number;
+        dts: number;
+        duration: number;
+    }, void, unknown>;
+    waitingForSamples: Array<(last: boolean) => void>;
+    _hasSeenLastSamples: boolean;
+    waitForMoreSamples(): Promise<boolean>;
+    processSamples(last: boolean): void;
+    _arrayBufferFileStart: number;
+}