@editframe/elements 0.5.0-beta.3 → 0.5.0-beta.5

package/dist/editor/msToTimeCode.mjs

@@ -0,0 +1,15 @@
+const msToTimeCode = (ms, subSecond = false) => {
+  const seconds = Math.floor(ms / 1e3);
+  const minutes = Math.floor(seconds / 60);
+  const hours = Math.floor(minutes / 60);
+  const pad = (num) => num.toString().padStart(2, "0");
+  let timecode = `${pad(hours)}:${pad(minutes % 60)}:${pad(seconds % 60)}`;
+  if (subSecond) {
+    const subSeconds = Math.floor(ms % 1e3 / 10);
+    timecode += `.${subSeconds.toString().padStart(2, "0")}`;
+  }
+  return timecode;
+};
+export {
+  msToTimeCode
+};

package/dist/editor/util/EncodedAsset/EncodedAsset.mjs

@@ -0,0 +1,537 @@
+import { memoize } from "../../../util/memoize.mjs";
+import * as MP4Box from "mp4box";
+import { MP4File } from "../MP4File.mjs";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __decorateClass = (decorators, target, key, kind) => {
+  var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
+  for (var i = decorators.length - 1, decorator; i >= 0; i--)
+    if (decorator = decorators[i])
+      result = (kind ? decorator(target, key, result) : decorator(result)) || result;
+  if (kind && result)
+    __defProp(target, key, result);
+  return result;
+};
+const BUFFER_SIZE = 10;
+class AssetNotAvailableLocally extends Error {
+}
+class FileAsset {
+  constructor(localName, file) {
+    this.localName = localName;
+    this.file = file;
+  }
+  async arrayBuffer() {
+    return this.file.arrayBuffer();
+  }
+  get byteSize() {
+    return this.file.size;
+  }
+  get fileExtension() {
+    return this.file.name.split(".").pop();
+  }
+  slice(start, end) {
+    return this.file.slice(start, end);
+  }
+}
+class ISOFileAsset extends FileAsset {
+  constructor(localName, file, mp4boxFile) {
+    super(localName, file);
+    this.localName = localName;
+    this.file = file;
+    this.mp4boxFile = mp4boxFile;
+  }
+  get fileInfo() {
+    return this.mp4boxFile.getInfo();
+  }
+  get containerFormat() {
+    return "mp4";
+  }
+}
+__decorateClass([
+  memoize
+], ISOFileAsset.prototype, "fileInfo", 1);
+const _VideoAsset = class _VideoAsset2 extends ISOFileAsset {
+  constructor(localName, mp4boxFile, file) {
+    super(localName, file, mp4boxFile);
+    this.decodedFrames = [];
+    this.requestedSampleNumber = 0;
+    this.outCursor = 0;
+    this.sampleCursor = 0;
+    this.eventListeners = {};
+    this.latestSeekCts = 0;
+    this.videoDecoder = new VideoDecoder({
+      error: (e) => {
+        console.error(e);
+      },
+      // eslint-disable-next-line @typescript-eslint/no-misused-promises
+      output: async (decodedFrame) => {
+        const clone = decodedFrame.clone();
+        this.decodedFrames.push(clone);
+        this.pruneBuffer();
+        decodedFrame.close();
+        this.outCursor = this.samples.findIndex(
+          (sample) => sample.cts === decodedFrame.timestamp
+        );
+        this.emit("frame", clone);
+      }
+    });
+    this.configureDecoder();
+  }
+  static async createFromReadableStream(id, stream, file) {
+    let fileStart = 0;
+    const inputFile = new MP4File();
+    const reader = stream.getReader();
+    const processChunk = ({
+      done,
+      value
+    }) => {
+      if (done) {
+        return;
+      }
+      if (!value) {
+        return;
+      }
+      const mp4buffer = value.buffer;
+      mp4buffer.fileStart = fileStart;
+      const isLast = file.size === fileStart + value.byteLength;
+      inputFile.appendBuffer(mp4buffer, isLast);
+      fileStart += value.byteLength;
+      return reader.read().then(processChunk);
+    };
+    await reader.read().then(processChunk);
+    return new _VideoAsset2(id, inputFile, file);
+  }
+  /**
+   * **Only use this function in tests to reset a VideoAsset to its initial state.**
+   *
+   * @deprecated
+   */
+  async TEST_ONLY_RESET() {
+    await this.videoDecoder.flush();
+    this.configureDecoder();
+    this.requestedSampleNumber = 0;
+    this.outCursor = 0;
+    this.sampleCursor = 0;
+    this.decodedFrames.forEach((frame) => {
+      frame.close();
+    });
+    this.decodedFrames = [];
+    this.lastDecodedSample = void 0;
+    this.lastSoughtFrame = void 0;
+  }
+  addEventListener(type, callback) {
+    this.eventListeners[type] ||= /* @__PURE__ */ new Set();
+    this.eventListeners[type]?.add(callback);
+  }
+  removeEventListener(type, callback) {
+    this.eventListeners[type]?.delete(callback);
+  }
+  emit(type, ...args) {
+    this.eventListeners[type]?.forEach((listener) => {
+      listener(...args);
+    });
+  }
+  get videoCodec() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    return this.defaultVideoTrack?.codec;
+  }
+  get fragmentInfo() {
+    const fragments = [];
+    const [first, ...samples] = this.samples;
+    if (!first) {
+      return fragments;
+    }
+    let currentFragment = {
+      offset: first.offset,
+      size: first.size,
+      start_ms: first.cts,
+      duration_ms: 0
+    };
+    for (const sample of samples) {
+      if (sample.is_sync) {
+        if (currentFragment) {
+          currentFragment.duration_ms = sample.cts - currentFragment.start_ms;
+          fragments.push(currentFragment);
+        }
+        currentFragment = {
+          offset: sample.offset,
+          size: sample.size,
+          start_ms: sample.cts,
+          duration_ms: 0
+        };
+      } else {
+        currentFragment.size += sample.size;
+      }
+    }
+    return fragments;
+  }
+  pruneBuffer() {
+    if (this.decodedFrames.length > BUFFER_SIZE) {
+      this.decodedFrames.shift()?.close();
+    }
+  }
+  prettyPrint() {
+    const samplesInEncoder = this.sampleCursor - this.outCursor;
+    const cachedSamples = this.decodedFrames.length;
+    console.log(
+      Array.from({
+        length: this.sampleCursor - samplesInEncoder - cachedSamples
+      }).fill("⬜️").join(" "),
+      Array.from({ length: cachedSamples }).fill("🟩").join(" "),
+      Array.from({
+        length: samplesInEncoder
+      }).fill("🟨").join(" ")
+    );
+  }
+  get editsOffset() {
+    if (!this.defaultVideoTrack?.edits) {
+      return 0;
+    }
+    return this.defaultVideoTrack.edits.reduce((acc, edit) => {
+      return acc + edit.media_time;
+    }, 0);
+  }
+  async waitUntilVideoQueueDrained() {
+    if (this.videoDecoder.decodeQueueSize === 0) {
+      return;
+    }
+    await new Promise((resolve) => {
+      this.videoDecoder.addEventListener(
+        "dequeue",
+        () => {
+          resolve();
+        },
+        { once: true }
+      );
+    });
+    await this.waitUntilVideoQueueDrained();
+  }
+  get canDecodeNextSample() {
+    return this.sampleCursor < this.samples.length;
+  }
+  async decodeNextSample() {
+    if (!this.canDecodeNextSample) {
+      throw new Error("No more samples to decode");
+    }
+    await this.decodeSlice(this.sampleCursor, this.sampleCursor);
+    this.sampleCursor++;
+  }
+  async decodeSlice(start, end) {
+    const samples = this.samples.slice(start, end + 1);
+    const sliceStart = samples[0].offset;
+    const sliceEnd = samples[samples.length - 1].offset + samples[samples.length - 1].size;
+    const buffer = await this.file.slice(sliceStart, sliceEnd).arrayBuffer();
+    const firstSampleOffset = samples[0].offset;
+    for (let i = start; i <= end; i++) {
+      await this.waitUntilVideoQueueDrained();
+      const sample = this.getSample(i);
+      const sampleStart = sample.offset - firstSampleOffset;
+      const sampleEnd = sample.offset + sample.size - firstSampleOffset;
+      const chunk = new EncodedVideoChunk({
+        data: buffer.slice(sampleStart, sampleEnd),
+        timestamp: sample.cts,
+        duration: sample.duration,
+        type: sample.is_sync ? "key" : "delta"
+      });
+      this.videoDecoder.decode(chunk);
+      const nextSample = this.defaultVideoTrak?.samples?.[i + 1];
+      if (nextSample === void 0) {
+        await this.videoDecoder.flush();
+      }
+    }
+  }
+  get decoderConfiguration() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    let description;
+    const trak = this.mp4boxFile.getTrackById(this.defaultVideoTrack.id);
+    for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+      if (entry.avcC ?? entry.hvcC) {
+        const stream = new MP4Box.DataStream(
+          void 0,
+          0,
+          MP4Box.DataStream.BIG_ENDIAN
+        );
+        if (entry.avcC) {
+          entry.avcC.write(stream);
+        } else {
+          entry.hvcC.write(stream);
+        }
+        description = new Uint8Array(stream.buffer, 8);
+        break;
+      }
+    }
+    return {
+      codec: this.defaultVideoTrack.codec,
+      codedWidth: this.defaultVideoTrack.track_width,
+      codedHeight: this.defaultVideoTrack.track_height,
+      // hardwareAcceleration: "prefer-hardware",
+      optimizeForLatency: true,
+      description
+    };
+  }
+  /**
+   * Configures the video decoder with the appropriate codec, dimensions, and hardware acceleration settings.
+   * If the decoder is already configured, it will be reset before being reconfigured.
+   */
+  configureDecoder() {
+    if (this.videoDecoder.state === "configured") {
+      this.videoDecoder.reset();
+    }
+    this.videoDecoder.configure(this.decoderConfiguration);
+  }
+  // Default to -1 to throw error if called without an index
+  getSample(index = -1) {
+    const sample = this.samples?.[index];
+    if (!sample) {
+      throw new Error("Sample not found at index " + index);
+    }
+    return sample;
+  }
+  get timescale() {
+    if (!this.defaultVideoTrack) {
+      throw new Error("No default video track found");
+    }
+    return this.defaultVideoTrack.timescale;
+  }
+  get samples() {
+    if (!this.defaultVideoTrak.samples) {
+      throw new Error("No video samples found");
+    }
+    return this.defaultVideoTrak.samples;
+  }
+  get displayOrderedSamples() {
+    return Array.from(this.samples).sort((a, b) => {
+      return a.cts - b.cts;
+    });
+  }
+  getSampleClosetToTime(seconds) {
+    const targetTime = Math.round(seconds * this.timescale + this.editsOffset);
+    const sampleIndex = this.displayOrderedSamples.findIndex(
+      (sample) => sample.cts >= targetTime
+    );
+    if (sampleIndex === -1) {
+      return this.displayOrderedSamples[this.displayOrderedSamples.length - 1];
+    }
+    return this.displayOrderedSamples[sampleIndex];
+  }
+  seekingWillEmitNewFrame(seconds) {
+    if (!this.lastSoughtFrame) {
+      return true;
+    }
+    if (this.seekingWillGoBackwards(seconds)) {
+      return true;
+    }
+    const nextCts = this.getSampleClosetToTime(seconds).cts;
+    return nextCts > this.lastSoughtFrame.timestamp;
+  }
+  seekingWillSkipPictureGroup(seconds) {
+    let start = this.sampleCursor;
+    const end = this.getSampleClosetToTime(seconds).number;
+    let syncFrameCrossings = 0;
+    while (start <= end) {
+      const sample = this.getSample(start);
+      if (sample.is_sync) {
+        if (syncFrameCrossings > 1) {
+          return true;
+        }
+        syncFrameCrossings++;
+      }
+      start++;
+    }
+    return false;
+  }
+  seekingWillGoBackwards(seconds) {
+    const targetSample = this.getSampleClosetToTime(seconds);
+    const targetIndex = this.displayOrderedSamples.indexOf(targetSample);
+    const targetInCache = this.decodedFrames.find(
+      (frame) => frame.timestamp === targetSample.cts
+    );
+    const atEnd = this.sampleCursor === this.samples.length - 1;
+    if (atEnd) {
+      return false;
+    }
+    if (targetInCache) {
+      return false;
+    }
+    return this.outCursor > targetIndex;
+  }
+  async seekToTime(seconds) {
+    const sample = this.getSampleClosetToTime(seconds);
+    const cts = sample.cts;
+    this.latestSeekCts = cts;
+    const alreadyDecodedFrame = this.decodedFrames.find(
+      (f) => f.timestamp === cts
+    );
+    if (alreadyDecodedFrame) {
+      return alreadyDecodedFrame;
+    }
+    if (this.seekingWillSkipPictureGroup(seconds)) {
+      await this.videoDecoder.flush();
+      let syncSampleNumber = sample.number;
+      while (!this.getSample(syncSampleNumber).is_sync) {
+        syncSampleNumber--;
+      }
+      this.sampleCursor = syncSampleNumber;
+    }
+    if (this.seekingWillGoBackwards(seconds)) {
+      console.log("BACKWARDS FLUSH");
+      await this.videoDecoder.flush();
+      this.decodedFrames.forEach((frame2) => {
+        frame2.close();
+      });
+      this.decodedFrames = [];
+      let syncSampleNumber = sample.number;
+      while (!this.getSample(syncSampleNumber).is_sync) {
+        syncSampleNumber--;
+      }
+      this.sampleCursor = syncSampleNumber;
+    }
+    let frame;
+    const maybeFrame = (_frame) => {
+      if (frame) {
+        return;
+      }
+      if (_frame.timestamp === cts) {
+        this.removeEventListener("frame", maybeFrame);
+        frame = _frame;
+      }
+    };
+    this.addEventListener("frame", maybeFrame);
+    while (frame === void 0 && this.canDecodeNextSample) {
+      await this.decodeNextSample();
+    }
+    this.removeEventListener("frame", maybeFrame);
+    if (frame) {
+      this.lastSoughtFrame = frame;
+    }
+    return frame;
+  }
+  get defaultVideoTrack() {
+    return this.fileInfo.videoTracks[0];
+  }
+  get defaultVideoTrak() {
+    return this.mp4boxFile.getTrackById(this.defaultVideoTrack?.id ?? -1);
+  }
+  get duration() {
+    return this.fileInfo.duration / this.fileInfo.timescale;
+  }
+};
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "editsOffset", 1);
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "timescale", 1);
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "samples", 1);
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "displayOrderedSamples", 1);
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "defaultVideoTrack", 1);
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "defaultVideoTrak", 1);
+__decorateClass([
+  memoize
+], _VideoAsset.prototype, "duration", 1);
+let VideoAsset = _VideoAsset;
+const _AudioAsset = class _AudioAsset2 extends ISOFileAsset {
+  static async createFromReadableStream(id, stream, file) {
+    let fileStart = 0;
+    const inputFile = new MP4File();
+    const reader = stream.getReader();
+    const processChunk = ({
+      done,
+      value
+    }) => {
+      if (done) {
+        return;
+      }
+      if (!value) {
+        return;
+      }
+      const mp4buffer = value.buffer;
+      mp4buffer.fileStart = fileStart;
+      fileStart += value.byteLength;
+      inputFile.appendBuffer(mp4buffer);
+      return reader.read().then(processChunk);
+    };
+    await reader.read().then(processChunk);
+    return new _AudioAsset2(id, file, inputFile);
+  }
+  get defaultAudioTrack() {
+    return this.fileInfo.audioTracks[0];
+  }
+  get defaultAudioTrak() {
+    return this.mp4boxFile.getTrackById(this.defaultAudioTrack?.id ?? -1);
+  }
+  get audioCodec() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.codec;
+  }
+  get samplerate() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.audio.sample_rate;
+  }
+  get channelCount() {
+    if (!this.defaultAudioTrack) {
+      throw new Error("No default audio track found");
+    }
+    return this.defaultAudioTrack.audio.channel_count;
+  }
+};
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "defaultAudioTrack", 1);
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "defaultAudioTrak", 1);
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "audioCodec", 1);
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "samplerate", 1);
+__decorateClass([
+  memoize
+], _AudioAsset.prototype, "channelCount", 1);
+const _ImageAsset = class _ImageAsset2 extends FileAsset {
+  static async createFromReadableStream(id, file) {
+    if (file.size === 0) {
+      throw new AssetNotAvailableLocally();
+    }
+    return new _ImageAsset2(id, file);
+  }
+  get objectUrl() {
+    return URL.createObjectURL(this.file);
+  }
+  get format() {
+    return this.fileExtension;
+  }
+  get type() {
+    return `image/${this.format}`;
+  }
+};
+__decorateClass([
+  memoize
+], _ImageAsset.prototype, "objectUrl", 1);
+__decorateClass([
+  memoize
+], _ImageAsset.prototype, "format", 1);
+export {
+  AssetNotAvailableLocally,
+  FileAsset,
+  ISOFileAsset,
+  VideoAsset
+};

package/dist/editor/util/MP4File.mjs

@@ -0,0 +1,161 @@
+import * as MP4Box from "mp4box";
+class MP4File extends MP4Box.ISOFile {
+  constructor() {
+    super(...arguments);
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.onReady = () => resolve();
+      this.onError = reject;
+    });
+    this.waitingForSamples = [];
+    this._hasSeenLastSamples = false;
+    this._arrayBufferFileStart = 0;
+  }
+  setSegmentOptions(id, user, options) {
+    const trak = this.getTrackById(id);
+    if (trak) {
+      trak.nextSample = 0;
+      this.fragmentedTracks.push({
+        id,
+        user,
+        trak,
+        segmentStream: null,
+        nb_samples: "nbSamples" in options && options.nbSamples || 1e3,
+        rapAlignement: ("rapAlignement" in options && options.rapAlignement) ?? true
+      });
+    }
+  }
+  /**
+   * Fragments all tracks in a file into separate array buffers.
+   */
+  async fragmentAllTracks() {
+    let trackBuffers = {};
+    for await (const segment of this.fragmentIterator()) {
+      (trackBuffers[segment.track] ??= []).push(segment.data);
+    }
+    return trackBuffers;
+  }
+  async *fragmentIterator() {
+    await this.readyPromise;
+    const trackInfo = {};
+    for (const videoTrack of this.getInfo().videoTracks) {
+      trackInfo[videoTrack.id] = { index: 0, complete: false };
+      this.setSegmentOptions(videoTrack.id, null, {
+        rapAlignement: true
+      });
+    }
+    for (const audioTrack of this.getInfo().audioTracks) {
+      trackInfo[audioTrack.id] = { index: 0, complete: false };
+      const sampleRate = audioTrack.audio.sample_rate;
+      const probablePacketSize = 1024;
+      const probableFourSecondsOfSamples = Math.ceil(
+        sampleRate / probablePacketSize * 4
+      );
+      this.setSegmentOptions(audioTrack.id, null, {
+        nbSamples: probableFourSecondsOfSamples
+      });
+    }
+    const initSegments = this.initializeSegmentation();
+    for (const initSegment of initSegments) {
+      yield {
+        track: initSegment.id,
+        segment: "init",
+        data: initSegment.buffer,
+        complete: false
+      };
+    }
+    const fragmentStartSamples = {};
+    let finishedReading = false;
+    const allTracksFinished = () => {
+      for (const fragmentedTrack of this.fragmentedTracks) {
+        if (!trackInfo[fragmentedTrack.id]?.complete) {
+          return false;
+        }
+      }
+      return true;
+    };
+    while (!(finishedReading && allTracksFinished())) {
+      for (const fragTrak of this.fragmentedTracks) {
+        const trak = fragTrak.trak;
+        if (trak.nextSample === void 0) {
+          throw new Error("trak.nextSample is undefined");
+        }
+        if (trak.samples === void 0) {
+          throw new Error("trak.samples is undefined");
+        }
+        eachSample:
+          while (trak.nextSample < trak.samples.length) {
+            let result = void 0;
+            if (trak?.samples[trak.nextSample]) {
+              fragmentStartSamples[fragTrak.id] ||= trak.samples[trak.nextSample];
+            }
+            try {
+              result = this.createFragment(
+                fragTrak.id,
+                trak.nextSample,
+                fragTrak.segmentStream
+              );
+            } catch (error) {
+              console.log("Failed to createFragment", error);
+            }
+            if (result) {
+              fragTrak.segmentStream = result;
+              trak.nextSample++;
+            } else {
+              finishedReading = await this.waitForMoreSamples();
+              break eachSample;
+            }
+            const nextSample = trak.samples[trak.nextSample];
+            const emitSegment = (
+              // if rapAlignement is true, we emit a fragment when we have a rap sample coming up next
+              fragTrak.rapAlignement === true && nextSample?.is_sync || // if rapAlignement is false, we emit a fragment when we have the required number of samples
+              !fragTrak.rapAlignement && trak.nextSample % fragTrak.nb_samples === 0 || // // if this is the last sample, we emit the fragment
+              // finished ||
+              // if we have more samples than the number of samples requested, we emit the fragment
+              trak.nextSample >= trak.samples.length
+            );
+            if (emitSegment) {
+              if (trak.nextSample >= trak.samples.length) {
+                trackInfo[fragTrak.id].complete = true;
+              }
+              const startSample = fragmentStartSamples[fragTrak.id];
+              const endSample = trak.samples[trak.nextSample - 1];
+              if (!startSample || !endSample) {
+                throw new Error("startSample or endSample is undefined");
+              }
+              yield {
+                track: fragTrak.id,
+                segment: trackInfo[fragTrak.id].index,
+                data: fragTrak.segmentStream.buffer,
+                complete: trackInfo[fragTrak.id].complete,
+                cts: startSample.cts,
+                dts: startSample.dts,
+                duration: endSample.dts - startSample.dts + endSample.duration
+              };
+              trackInfo[fragTrak.id].index += 1;
+              fragTrak.segmentStream = null;
+              delete fragmentStartSamples[fragTrak.id];
+            }
+          }
+      }
+      finishedReading = await this.waitForMoreSamples();
+    }
+  }
+  waitForMoreSamples() {
+    if (this._hasSeenLastSamples) {
+      return Promise.resolve(true);
+    }
+    return new Promise((resolve) => {
+      this.waitingForSamples.push(resolve);
+    });
+  }
+  processSamples(last) {
+    this._hasSeenLastSamples = last;
+    for (const observer of this.waitingForSamples) {
+      observer(last);
+    }
+    this.waitingForSamples = [];
+  }
+}
+export {
+  MP4File
+};