@editframe/elements 0.5.0-beta.9 → 0.6.0-beta.1

package/dist/lib/av/EncodedAsset.cjs

@@ -0,0 +1,561 @@
+"use strict";
+Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
+const MP4Box = require("mp4box");
+const memoize = require("../util/memoize.cjs");
+const MP4File = require("./MP4File.cjs");
+function _interopNamespaceDefault(e) {
+  const n = Object.create(null, { [Symbol.toStringTag]: { value: "Module" } });
+  if (e) {
+    for (const k in e) {
+      if (k !== "default") {
+        const d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: () => e[k]
+        });
+      }
+    }
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+const MP4Box__namespace = /* @__PURE__ */ _interopNamespaceDefault(MP4Box);
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = __getOwnPropDesc(target, key);
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = decorator(target, key, result) || result;
+  if (result) __defProp(target, key, result);
+  return result;
+};
+const BUFFER_SIZE = 10;
+class AssetNotAvailableLocally extends Error {
+}
+class FileAsset {
+  constructor(localName, file) {
+    this.localName = localName;
+    this.file = file;
+  }
+  async arrayBuffer() {
+    return this.file.arrayBuffer();
+  }
+  get byteSize() {
+    return this.file.size;
+  }
+  get fileExtension() {
+    return this.file.name.split(".").pop();
+  }
+  slice(start, end) {
+    return this.file.slice(start, end);
+  }
+}
+class ISOFileAsset extends FileAsset {
+  constructor(localName, file, mp4boxFile) {
+    super(localName, file);
+    this.localName = localName;
+    this.file = file;
+    this.mp4boxFile = mp4boxFile;
+  }
+  get fileInfo() {
+    return this.mp4boxFile.getInfo();
+  }
+  get containerFormat() {
+    return "mp4";
+  }
+}
+__decorateClass([
+  memoize.memoize
+], ISOFileAsset.prototype, "fileInfo");
+const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
+  constructor(localName, mp4boxFile, file) {
+    super(localName, file, mp4boxFile);
+    this.decodedFrames = [];
+    this.requestedSampleNumber = 0;
+    this.outCursor = 0;
+    this.sampleCursor = 0;
+    this.eventListeners = {};
+    this.latestSeekCts = 0;
+    this.videoDecoder = new VideoDecoder({
+      error: (e) => {
+        console.error(e);
+      },
+      // eslint-disable-next-line @typescript-eslint/no-misused-promises
+      output: async (decodedFrame) => {
+        const clone = decodedFrame.clone();
+        this.decodedFrames.push(clone);
+        this.pruneBuffer();
+        decodedFrame.close();
+        this.outCursor = this.samples.findIndex(
+          (sample) => sample.cts === decodedFrame.timestamp
+        );
+        this.emit("frame", clone);
+      }
+    });
+    this.configureDecoder();
+  }
+  static async createFromReadableStream(id, stream, file) {
+    let fileStart = 0;
+    const inputFile = new MP4File.MP4File();
+    const reader = stream.getReader();
+    const processChunk = ({
+      done,
+      value
+    }) => {
+      if (done) {
+        return;
+      }
+      if (!value) {
+        return;
+      }
+      const mp4buffer = value.buffer;
+      mp4buffer.fileStart = fileStart;
+      const isLast = file.size === fileStart + value.byteLength;
+      inputFile.appendBuffer(mp4buffer, isLast);
+      fileStart += value.byteLength;
+      return reader.read().then(processChunk);
+    };
+    await reader.read().then(processChunk);
+    return new _VideoAsset2(id, inputFile, file);
+  }
+  /**
+   * **Only use this function in tests to reset a VideoAsset to its initial state.**
+   *
+   * @deprecated
+   */
+  async TEST_ONLY_RESET() {
+    await this.videoDecoder.flush();
+    this.configureDecoder();
+    this.requestedSampleNumber = 0;
+    this.outCursor = 0;
+    this.sampleCursor = 0;
+    for (const frame of this.decodedFrames) {
+      frame.close();
+    }
+    this.decodedFrames = [];
+    this.lastDecodedSample = void 0;
+    this.lastSoughtFrame?.close();
+    this.lastSoughtFrame = void 0;
+  }
+  addEventListener(type, callback) {
+    this.eventListeners[type] ||= /* @__PURE__ */ new Set();
+    this.eventListeners[type]?.add(callback);
+  }
+  removeEventListener(type, callback) {
+    this.eventListeners[type]?.delete(callback);
+  }
+  emit(type, ...args) {
+    for (const listener of this.eventListeners[type] ?? []) {
+      listener(...args);
+    }
+  }
+  get videoCodec() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    return this.defaultVideoTrack?.codec;
+  }
+  get fragmentInfo() {
+    const fragments = [];
+    const [first, ...samples] = this.samples;
+    if (!first) {
+      return fragments;
+    }
+    let currentFragment = {
+      offset: first.offset,
+      size: first.size,
+      start_ms: first.cts,
+      duration_ms: 0
+    };
+    for (const sample of samples) {
+      if (sample.is_sync) {
+        if (currentFragment) {
+          currentFragment.duration_ms = sample.cts - currentFragment.start_ms;
+          fragments.push(currentFragment);
+        }
+        currentFragment = {
+          offset: sample.offset,
+          size: sample.size,
+          start_ms: sample.cts,
+          duration_ms: 0
+        };
+      } else {
+        currentFragment.size += sample.size;
+      }
+    }
+    return fragments;
+  }
+  pruneBuffer() {
+    if (this.decodedFrames.length > BUFFER_SIZE) {
+      this.decodedFrames.shift()?.close();
+    }
+  }
+  prettyPrint() {
+    const samplesInEncoder = this.sampleCursor - this.outCursor;
+    const cachedSamples = this.decodedFrames.length;
+    console.log(
+      Array.from({
+        length: this.sampleCursor - samplesInEncoder - cachedSamples
+      }).fill("⬜️").join(" "),
+      Array.from({ length: cachedSamples }).fill("🟩").join(" "),
+      Array.from({
+        length: samplesInEncoder
+      }).fill("🟨").join(" ")
+    );
+  }
+  get editsOffset() {
+    if (!this.defaultVideoTrack?.edits) {
+      return 0;
+    }
+    return this.defaultVideoTrack.edits.reduce((acc, edit) => {
+      return acc + edit.media_time;
+    }, 0);
+  }
+  async waitUntilVideoQueueDrained() {
+    if (this.videoDecoder.decodeQueueSize === 0) {
+      return;
+    }
+    await new Promise((resolve) => {
+      this.videoDecoder.addEventListener(
+        "dequeue",
+        () => {
+          resolve();
+        },
+        { once: true }
+      );
+    });
+    await this.waitUntilVideoQueueDrained();
+  }
+  get canDecodeNextSample() {
+    return this.sampleCursor < this.samples.length;
+  }
+  async decodeNextSample() {
+    if (!this.canDecodeNextSample) {
+      throw new Error("No more samples to decode");
+    }
+    await this.decodeSlice(this.sampleCursor, this.sampleCursor);
+    this.sampleCursor++;
+  }
+  async decodeSlice(start, end) {
+    const samples = this.samples.slice(start, end + 1);
+    const firstSample = samples[0];
+    const lastSample = samples[samples.length - 1];
+    if (!firstSample || !lastSample) {
+      throw new Error("Samples not found");
+    }
+    const sliceStart = firstSample.offset;
+    const sliceEnd = lastSample.offset + lastSample.size;
+    const buffer = await this.file.slice(sliceStart, sliceEnd).arrayBuffer();
+    const firstSampleOffset = firstSample.offset;
+    for (let i = start; i <= end; i++) {
+      await this.waitUntilVideoQueueDrained();
+      const sample = this.getSample(i);
+      const sampleStart = sample.offset - firstSampleOffset;
+      const sampleEnd = sample.offset + sample.size - firstSampleOffset;
+      const chunk = new EncodedVideoChunk({
+        data: buffer.slice(sampleStart, sampleEnd),
+        timestamp: sample.cts,
+        duration: sample.duration,
+        type: sample.is_sync ? "key" : "delta"
+      });
+      this.videoDecoder.decode(chunk);
+      const nextSample = this.defaultVideoTrak?.samples?.[i + 1];
+      if (nextSample === void 0) {
+        await this.videoDecoder.flush();
+      }
+    }
+  }
+  get decoderConfiguration() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    let description = new Uint8Array();
+    const trak = this.mp4boxFile.getTrackById(this.defaultVideoTrack.id);
+    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+      if (entry.avcC ?? entry.hvcC) {
+        const stream = new MP4Box__namespace.DataStream(
+          void 0,
+          0,
+          MP4Box__namespace.DataStream.BIG_ENDIAN
+        );
+        if (entry.avcC) {
+          entry.avcC.write(stream);
+        } else {
+          entry.hvcC.write(stream);
+        }
+        description = new Uint8Array(stream.buffer, 8);
+        break;
+      }
+    }
+    return {
+      codec: this.defaultVideoTrack.codec,
+      codedWidth: this.defaultVideoTrack.track_width,
+      codedHeight: this.defaultVideoTrack.track_height,
+      optimizeForLatency: true,
+      description
+    };
+  }
+  /**
+   * Configures the video decoder with the appropriate codec, dimensions, and hardware acceleration settings.
+   * If the decoder is already configured, it will be reset before being reconfigured.
+   */
+  configureDecoder() {
+    if (this.videoDecoder.state === "configured") {
+      this.videoDecoder.reset();
+    }
+    this.videoDecoder.configure(this.decoderConfiguration);
+  }
+  // Default to -1 to throw error if called without an index
+  getSample(index = -1) {
+    const sample = this.samples?.[index];
+    if (!sample) {
+      throw new Error(`Sample not found at index ${index}`);
+    }
+    return sample;
+  }
+  get timescale() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    return this.defaultVideoTrack.timescale;
+  }
+  get samples() {
+    if (!this.defaultVideoTrak.samples) {
+      throw new Error("No video samples found");
+    }
+    return this.defaultVideoTrak.samples;
+  }
+  get displayOrderedSamples() {
+    return Array.from(this.samples).sort((a, b) => {
+      return a.cts - b.cts;
+    });
+  }
+  getSampleClosetToTime(seconds) {
+    const targetTime = Math.round(seconds * this.timescale + this.editsOffset);
+    const sampleIndex = this.displayOrderedSamples.findIndex(
+      (sample) => sample.cts >= targetTime
+    );
+    if (sampleIndex === -1) {
+      return this.displayOrderedSamples[this.displayOrderedSamples.length - 1];
+    }
+    return this.displayOrderedSamples[sampleIndex];
+  }
+  seekingWillEmitNewFrame(seconds) {
+    if (!this.lastSoughtFrame) {
+      return true;
+    }
+    if (this.seekingWillGoBackwards(seconds)) {
+      return true;
+    }
+    const nextCts = this.getSampleClosetToTime(seconds).cts;
+    return nextCts > this.lastSoughtFrame.timestamp;
+  }
+  seekingWillSkipPictureGroup(seconds) {
+    let start = this.sampleCursor;
+    const end = this.getSampleClosetToTime(seconds).number;
+    let syncFrameCrossings = 0;
+    while (start <= end) {
+      const sample = this.getSample(start);
+      if (sample.is_sync) {
+        if (syncFrameCrossings > 1) {
+          return true;
+        }
+        syncFrameCrossings++;
+      }
+      start++;
+    }
+    return false;
+  }
+  seekingWillGoBackwards(seconds) {
+    const targetSample = this.getSampleClosetToTime(seconds);
+    const targetIndex = this.displayOrderedSamples.indexOf(targetSample);
+    const targetInCache = this.decodedFrames.find(
+      (frame) => frame.timestamp === targetSample.cts
+    );
+    const atEnd = this.sampleCursor === this.samples.length - 1;
+    if (atEnd) {
+      return false;
+    }
+    if (targetInCache) {
+      return false;
+    }
+    return this.outCursor > targetIndex;
+  }
+  async seekToTime(seconds) {
+    const sample = this.getSampleClosetToTime(seconds);
+    const cts = sample.cts;
+    this.latestSeekCts = cts;
+    const alreadyDecodedFrame = this.decodedFrames.find(
+      (f) => f.timestamp === cts
+    );
+    if (alreadyDecodedFrame) {
+      return alreadyDecodedFrame;
+    }
+    if (this.seekingWillSkipPictureGroup(seconds)) {
+      await this.videoDecoder.flush();
+      let syncSampleNumber = sample.number;
+      while (!this.getSample(syncSampleNumber).is_sync) {
+        syncSampleNumber--;
+      }
+      this.sampleCursor = syncSampleNumber;
+    }
+    if (this.seekingWillGoBackwards(seconds)) {
+      console.log("BACKWARDS FLUSH");
+      await this.videoDecoder.flush();
+      for (const frame2 of this.decodedFrames) {
+        frame2.close();
+      }
+      this.decodedFrames = [];
+      let syncSampleNumber = sample.number;
+      while (!this.getSample(syncSampleNumber).is_sync) {
+        syncSampleNumber--;
+      }
+      this.sampleCursor = syncSampleNumber;
+    }
+    let frame;
+    const maybeFrame = (_frame) => {
+      if (frame) {
+        return;
+      }
+      if (_frame.timestamp === cts) {
+        this.removeEventListener("frame", maybeFrame);
+        frame = _frame;
+      }
+    };
+    this.addEventListener("frame", maybeFrame);
+    while (frame === void 0 && this.canDecodeNextSample) {
+      await this.decodeNextSample();
+    }
+    this.removeEventListener("frame", maybeFrame);
+    if (frame) {
+      if (this.lastSoughtFrame && !this.decodedFrames.includes(this.lastSoughtFrame)) {
+        this.lastSoughtFrame.close();
+      }
+      this.lastSoughtFrame = frame;
+    }
+    return frame;
+  }
+  get defaultVideoTrack() {
+    return this.fileInfo.videoTracks[0];
+  }
+  get defaultVideoTrak() {
+    return this.mp4boxFile.getTrackById(this.defaultVideoTrack?.id ?? -1);
+  }
+  get duration() {
+    return this.fileInfo.duration / this.fileInfo.timescale;
+  }
+};
+__decorateClass([
+  memoize.memoize
+], _VideoAsset.prototype, "editsOffset");
+__decorateClass([
+  memoize.memoize
+], _VideoAsset.prototype, "timescale");
+__decorateClass([
+  memoize.memoize
+], _VideoAsset.prototype, "samples");
+__decorateClass([
+  memoize.memoize
+], _VideoAsset.prototype, "displayOrderedSamples");
+__decorateClass([
+  memoize.memoize
+], _VideoAsset.prototype, "defaultVideoTrack");
+__decorateClass([
+  memoize.memoize
+], _VideoAsset.prototype, "defaultVideoTrak");
+__decorateClass([
+  memoize.memoize
+], _VideoAsset.prototype, "duration");
+let VideoAsset = _VideoAsset;
+const _AudioAsset = class _AudioAsset2 extends ISOFileAsset {
+  static async createFromReadableStream(id, stream, file) {
+    let fileStart = 0;
+    const inputFile = new MP4File.MP4File();
+    const reader = stream.getReader();
+    const processChunk = ({
+      done,
+      value
+    }) => {
+      if (done) {
+        return;
+      }
+      if (!value) {
+        return;
+      }
+      const mp4buffer = value.buffer;
+      mp4buffer.fileStart = fileStart;
+      fileStart += value.byteLength;
+      inputFile.appendBuffer(mp4buffer);
+      return reader.read().then(processChunk);
+    };
+    await reader.read().then(processChunk);
+    return new _AudioAsset2(id, file, inputFile);
+  }
+  get defaultAudioTrack() {
+    return this.fileInfo.audioTracks[0];
+  }
+  get defaultAudioTrak() {
+    return this.mp4boxFile.getTrackById(this.defaultAudioTrack?.id ?? -1);
+  }
+  get audioCodec() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.codec;
+  }
+  get samplerate() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.audio.sample_rate;
+  }
+  get channelCount() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.audio.channel_count;
+  }
+};
+__decorateClass([
+  memoize.memoize
+], _AudioAsset.prototype, "defaultAudioTrack");
+__decorateClass([
+  memoize.memoize
+], _AudioAsset.prototype, "defaultAudioTrak");
+__decorateClass([
+  memoize.memoize
+], _AudioAsset.prototype, "audioCodec");
+__decorateClass([
+  memoize.memoize
+], _AudioAsset.prototype, "samplerate");
+__decorateClass([
+  memoize.memoize
+], _AudioAsset.prototype, "channelCount");
+const _ImageAsset = class _ImageAsset2 extends FileAsset {
+  static async createFromReadableStream(id, file) {
+    if (file.size === 0) {
+      throw new AssetNotAvailableLocally();
+    }
+    return new _ImageAsset2(id, file);
+  }
+  get objectUrl() {
+    return URL.createObjectURL(this.file);
+  }
+  get format() {
+    return this.fileExtension;
+  }
+  get type() {
+    return `image/${this.format}`;
+  }
+};
+__decorateClass([
+  memoize.memoize
+], _ImageAsset.prototype, "objectUrl");
+__decorateClass([
+  memoize.memoize
+], _ImageAsset.prototype, "format");
+exports.AssetNotAvailableLocally = AssetNotAvailableLocally;
+exports.FileAsset = FileAsset;
+exports.ISOFileAsset = ISOFileAsset;
+exports.VideoAsset = VideoAsset;

package/dist/lib/av/{EncodedAsset.mjs → EncodedAsset.js}

@@ -1,6 +1,6 @@
 import * as MP4Box from "mp4box";
-import { memoize } from "../util/memoize.mjs";
-import { MP4File } from "./MP4File.mjs";
+import { memoize } from "../util/memoize.js";
+import { MP4File } from "./MP4File.js";
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __decorateClass = (decorators, target, key, kind) => {
@@ -111,9 +111,9 @@ const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
     this.requestedSampleNumber = 0;
     this.outCursor = 0;
     this.sampleCursor = 0;
-    this.decodedFrames.forEach((frame) => {
+    for (const frame of this.decodedFrames) {
       frame.close();
-    });
+    }
     this.decodedFrames = [];
     this.lastDecodedSample = void 0;
     this.lastSoughtFrame?.close();
@@ -127,9 +127,9 @@ const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
     this.eventListeners[type]?.delete(callback);
   }
   emit(type, ...args) {
-    this.eventListeners[type]?.forEach((listener) => {
+    for (const listener of this.eventListeners[type] ?? []) {
       listener(...args);
-    });
+    }
   }
   get videoCodec() {
     if (!this.defaultVideoTrack) {
@@ -220,10 +220,15 @@ const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
   }
   async decodeSlice(start, end) {
     const samples = this.samples.slice(start, end + 1);
-    const sliceStart = samples[0].offset;
-    const sliceEnd = samples[samples.length - 1].offset + samples[samples.length - 1].size;
+    const firstSample = samples[0];
+    const lastSample = samples[samples.length - 1];
+    if (!firstSample || !lastSample) {
+      throw new Error("Samples not found");
+    }
+    const sliceStart = firstSample.offset;
+    const sliceEnd = lastSample.offset + lastSample.size;
     const buffer = await this.file.slice(sliceStart, sliceEnd).arrayBuffer();
-    const firstSampleOffset = samples[0].offset;
+    const firstSampleOffset = firstSample.offset;
     for (let i = start; i <= end; i++) {
       await this.waitUntilVideoQueueDrained();
       const sample = this.getSample(i);
@@ -246,7 +251,7 @@ const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
     if (!this.defaultVideoTrack) {
       throw new Error("No default video track found");
     }
-    let description;
+    let description = new Uint8Array();
     const trak = this.mp4boxFile.getTrackById(this.defaultVideoTrack.id);
     for (const entry of trak.mdia.minf.stbl.stsd.entries) {
       if (entry.avcC ?? entry.hvcC) {
@@ -286,7 +291,7 @@ const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
   getSample(index = -1) {
     const sample = this.samples?.[index];
     if (!sample) {
-      throw new Error("Sample not found at index " + index);
+      throw new Error(`Sample not found at index ${index}`);
     }
     return sample;
   }
@@ -379,9 +384,9 @@ const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
     if (this.seekingWillGoBackwards(seconds)) {
       console.log("BACKWARDS FLUSH");
       await this.videoDecoder.flush();
-      this.decodedFrames.forEach((frame2) => {
+      for (const frame2 of this.decodedFrames) {
         frame2.close();
-      });
+      }
       this.decodedFrames = [];
       let syncSampleNumber = sample.number;
       while (!this.getSample(syncSampleNumber).is_sync) {