@remotion/studio 4.0.333 → 4.0.335

package/dist/esm/internals.mjs

@@ -29882,7 +29882,8 @@ var performSeek = async ({
   src,
   discardReadBytes,
   fields,
-  prefetchCache
+  prefetchCache,
+  isoState
 }) => {
   const byteInMediaSection = isByteInMediaSection({
     position: seekTo,
@@ -29946,6 +29947,9 @@ var performSeek = async ({
       prefetchCache
     });
   }
+  if (userInitiated) {
+    isoState.flatSamples.updateAfterSeek(seekTo);
+  }
   await controller._internals.checkForAbortAndPause();
 };
 var turnSeekIntoByte = async ({
@@ -30122,7 +30126,8 @@ var workOnSeekRequest = async (options) => {
       src,
       discardReadBytes,
       fields,
-      prefetchCache
+      prefetchCache,
+      isoState
     });
     return;
   }
@@ -30142,7 +30147,8 @@ var workOnSeekRequest = async (options) => {
       src,
       discardReadBytes,
       fields,
-      prefetchCache
+      prefetchCache,
+      isoState
     });
     const { hasChanged } = controller._internals.seekSignal.clearSeekIfStillSame(seek2);
     if (hasChanged) {
@@ -31206,9 +31212,10 @@ var parseFlac = ({
     state
   });
 };
-var calculateFlatSamples = ({
+var calculateSamplePositions = ({
   state,
-  mediaSectionStart
+  mediaSectionStart,
+  trackIds
 }) => {
   const tracks2 = getTracks(state, true);
   const moofBoxes = getMoofBoxes(state.structure.getIsoStructure().boxes);
@@ -31230,11 +31237,13 @@ var calculateFlatSamples = ({
   if (!moov) {
     throw new Error("No moov box found");
   }
-  const offsets = [];
-  const trackIds = [];
-  const map = new Map;
+  const trackIdAndSamplePositions = [];
   for (const track of tracks2) {
     const trakBox = getTrakBoxByTrackId(moov, track.trackId);
+    if (!trackIds.includes(track.trackId)) {
+      Log.verbose(state.logLevel, "Skipping calculating sample positions for track", track.trackId);
+      continue;
+    }
     if (!trakBox) {
       throw new Error("No trak box found");
     }
@@ -31244,36 +31253,88 @@ var calculateFlatSamples = ({
       moofComplete,
       trexBoxes: getTrexBoxes(moov)
     });
-    trackIds.push(track.trackId);
-    for (const samplePosition of samplePositions) {
-      offsets.push(samplePosition.offset);
-      map.set(samplePosition.offset, {
-        track,
-        samplePosition
-      });
+    trackIdAndSamplePositions.push({
+      trackId: track.trackId,
+      samplePositions
+    });
+  }
+  return trackIdAndSamplePositions;
+};
+var updateSampleIndicesAfterSeek = ({
+  samplePositionsForMdatStart,
+  seekedByte
+}) => {
+  const currentSampleIndices = {};
+  const keys = Object.keys(samplePositionsForMdatStart).map(Number).sort();
+  const mdat = keys.find((key4) => seekedByte >= key4);
+  if (!mdat) {
+    return currentSampleIndices;
+  }
+  const samplePositions = samplePositionsForMdatStart[mdat];
+  if (!samplePositions) {
+    return currentSampleIndices;
+  }
+  for (const track of samplePositions) {
+    const currentSampleIndex = track.samplePositions.findIndex((sample) => sample.offset >= seekedByte);
+    if (!currentSampleIndices[mdat]) {
+      currentSampleIndices[mdat] = {};
+    }
+    if (!currentSampleIndices[mdat][track.trackId]) {
+      currentSampleIndices[mdat][track.trackId] = 0;
+    }
+    if (currentSampleIndex === -1) {
+      currentSampleIndices[mdat][track.trackId] = track.samplePositions.length;
+    } else {
+      currentSampleIndices[mdat][track.trackId] = currentSampleIndex;
     }
   }
-  offsets.sort((a, b) => a - b);
-  return { flatSamples: map, offsets, trackIds };
+  return currentSampleIndices;
 };
 var cachedSamplePositionsState = () => {
-  const cachedForMdatStart = {};
-  const jumpMarksForMdatStart = {};
+  const samplePositionsForMdatStart = {};
+  let currentSampleIndex = {};
   return {
     getSamples: (mdatStart) => {
-      return cachedForMdatStart[mdatStart] ?? null;
+      return samplePositionsForMdatStart[mdatStart] ?? null;
     },
     setSamples: (mdatStart, samples) => {
-      cachedForMdatStart[mdatStart] = samples;
+      samplePositionsForMdatStart[mdatStart] = samples;
     },
-    setJumpMarks: (mdatStart, marks) => {
-      jumpMarksForMdatStart[mdatStart] = marks;
+    setCurrentSampleIndex: (mdatStart, trackId, index) => {
+      if (!currentSampleIndex[mdatStart]) {
+        currentSampleIndex[mdatStart] = {};
+      }
+      if (!currentSampleIndex[mdatStart][trackId]) {
+        currentSampleIndex[mdatStart][trackId] = 0;
+      }
+      currentSampleIndex[mdatStart][trackId] = index;
     },
-    getJumpMarks: (mdatStart) => {
-      return jumpMarksForMdatStart[mdatStart];
+    getCurrentSampleIndices: (mdatStart) => {
+      return currentSampleIndex[mdatStart] ?? {};
+    },
+    updateAfterSeek: (seekedByte) => {
+      currentSampleIndex = updateSampleIndicesAfterSeek({
+        samplePositionsForMdatStart,
+        seekedByte
+      });
     }
   };
 };
+var getSampleWithLowestDts = (samplePositions, currentSampleIndexMap) => {
+  const lowestDts = [];
+  for (const track of samplePositions) {
+    const currentSampleIndex = currentSampleIndexMap[track.trackId] ?? 0;
+    const currentSample = track.samplePositions[currentSampleIndex];
+    if (currentSample && (lowestDts.length === 0 || currentSample.decodingTimestamp <= lowestDts[0].samplePosition.decodingTimestamp)) {
+      lowestDts.push({
+        samplePosition: currentSample,
+        trackId: track.trackId,
+        index: currentSampleIndex
+      });
+    }
+  }
+  return lowestDts;
+};
 var getLastMoofBox = (boxes) => {
   if (boxes) {
     const tfras = boxes.filter((b) => b.type === "tfra-box");
@@ -31543,116 +31604,6 @@ var getMoovAtom = async ({
   Log.verbose(state.logLevel, `Finished fetching moov atom in ${Date.now() - start}ms`);
   return moov;
 };
-var MAX_SPREAD_IN_SECONDS = 8;
-var getKey = (samplePositionTrack) => {
-  return `${samplePositionTrack.track.trackId}-${samplePositionTrack.samplePosition.decodingTimestamp}.${samplePositionTrack.samplePosition.offset}`;
-};
-var findBestJump = ({
-  sampleMap,
-  offsetsSorted,
-  visited,
-  progresses
-}) => {
-  const minProgress = Math.min(...Object.values(progresses));
-  const trackNumberWithLowestProgress = Object.entries(progresses).find(([, progress]) => progress === minProgress)?.[0];
-  const firstSampleAboveMinProgress = offsetsSorted.findIndex((offset) => sampleMap.get(offset).track.trackId === Number(trackNumberWithLowestProgress) && !visited.has(getKey(sampleMap.get(offset))));
-  if (firstSampleAboveMinProgress === -1) {
-    const backup = offsetsSorted.findIndex((offset) => !visited.has(getKey(sampleMap.get(offset))));
-    if (backup === -1) {
-      throw new Error("this should not happen");
-    }
-    return backup;
-  }
-  return firstSampleAboveMinProgress;
-};
-var calculateJumpMarks = ({
-  sampleMap,
-  offsetsSorted,
-  trackIds,
-  endOfMdat
-}) => {
-  const progresses = {};
-  for (const trackId of trackIds) {
-    progresses[trackId] = 0;
-  }
-  const jumpMarks = [];
-  let indexToVisit = 0;
-  const visited = new Set;
-  const increaseIndex = () => {
-    indexToVisit++;
-    if (indexToVisit >= offsetsSorted.length) {
-      throw new Error("should not roll over, should jump");
-    }
-  };
-  let lastVisitedSample = null;
-  const addJumpMark = ({
-    firstSampleAboveMinProgress
-  }) => {
-    if (!lastVisitedSample) {
-      throw new Error("no last visited sample");
-    }
-    const jumpMark = {
-      afterSampleWithOffset: lastVisitedSample.samplePosition.offset,
-      jumpToOffset: offsetsSorted[firstSampleAboveMinProgress]
-    };
-    if (firstSampleAboveMinProgress === offsetsSorted.indexOf(lastVisitedSample.samplePosition.offset) + 1) {
-      indexToVisit = firstSampleAboveMinProgress;
-      return;
-    }
-    indexToVisit = firstSampleAboveMinProgress;
-    jumpMarks.push(jumpMark);
-  };
-  const addFinalJumpIfNecessary = () => {
-    if (indexToVisit === offsetsSorted.length - 1) {
-      return;
-    }
-    jumpMarks.push({
-      afterSampleWithOffset: offsetsSorted[indexToVisit],
-      jumpToOffset: endOfMdat
-    });
-  };
-  const considerJump = () => {
-    const firstSampleAboveMinProgress = findBestJump({
-      sampleMap,
-      offsetsSorted,
-      visited,
-      progresses
-    });
-    addJumpMark({ firstSampleAboveMinProgress });
-  };
-  while (true) {
-    const currentSamplePosition = sampleMap.get(offsetsSorted[indexToVisit]);
-    const sampleKey = getKey(currentSamplePosition);
-    if (visited.has(sampleKey)) {
-      considerJump();
-      continue;
-    }
-    visited.add(sampleKey);
-    lastVisitedSample = currentSamplePosition;
-    if (visited.size === offsetsSorted.length) {
-      addFinalJumpIfNecessary();
-      break;
-    }
-    const timestamp = currentSamplePosition.samplePosition.decodingTimestamp / currentSamplePosition.track.originalTimescale;
-    progresses[currentSamplePosition.track.trackId] = timestamp;
-    const progressValues = Object.values(progresses);
-    const maxProgress = Math.max(...progressValues);
-    const minProgress = Math.min(...progressValues);
-    const spread = maxProgress - minProgress;
-    if (visited.size === offsetsSorted.length) {
-      addFinalJumpIfNecessary();
-      break;
-    }
-    if (spread > MAX_SPREAD_IN_SECONDS) {
-      considerJump();
-    } else if (indexToVisit === offsetsSorted.length - 1) {
-      considerJump();
-    } else {
-      increaseIndex();
-    }
-  }
-  return jumpMarks;
-};
 var postprocessBytes = ({
   bytes,
   bigEndian,
@@ -31710,52 +31661,53 @@ var parseMdatSection = async (state) => {
       endOfMdat,
       state
     });
+    const tracksFromMoov = getTracksFromMoovBox(moov);
     state.iso.moov.setMoovBox({
       moovBox: moov,
       precomputed: false
     });
+    const existingTracks = state.callbacks.tracks.getTracks();
+    for (const trackFromMoov of tracksFromMoov) {
+      if (existingTracks.find((t) => t.trackId === trackFromMoov.trackId)) {
+        continue;
+      }
+      if (trackFromMoov.type === "other") {
+        continue;
+      }
+      state.callbacks.tracks.addTrack(trackFromMoov);
+    }
     state.callbacks.tracks.setIsDone(state.logLevel);
     state.structure.getIsoStructure().boxes.push(moov);
     return parseMdatSection(state);
   }
+  const tracks2 = state.callbacks.tracks.getTracks();
   if (!state.iso.flatSamples.getSamples(mediaSection.start)) {
-    const {
-      flatSamples: flatSamplesMap,
-      offsets,
-      trackIds
-    } = calculateFlatSamples({
+    const samplePosition = calculateSamplePositions({
       state,
-      mediaSectionStart: mediaSection.start
-    });
-    const calcedJumpMarks = calculateJumpMarks({
-      sampleMap: flatSamplesMap,
-      offsetsSorted: offsets,
-      trackIds,
-      endOfMdat
+      mediaSectionStart: mediaSection.start,
+      trackIds: tracks2.map((t) => t.trackId)
     });
-    state.iso.flatSamples.setJumpMarks(mediaSection.start, calcedJumpMarks);
-    state.iso.flatSamples.setSamples(mediaSection.start, flatSamplesMap);
+    state.iso.flatSamples.setSamples(mediaSection.start, samplePosition);
   }
-  const flatSamples = state.iso.flatSamples.getSamples(mediaSection.start);
-  const jumpMarks = state.iso.flatSamples.getJumpMarks(mediaSection.start);
-  const { iterator } = state;
-  const samplesWithIndex = flatSamples.get(iterator.counter.getOffset());
-  if (!samplesWithIndex) {
-    const offsets = Array.from(flatSamples.keys());
-    const nextSample_ = offsets.filter((s) => s > iterator.counter.getOffset()).sort((a, b) => a - b)[0];
-    if (nextSample_) {
-      iterator.discard(nextSample_ - iterator.counter.getOffset());
-      return null;
-    }
-    Log.verbose(state.logLevel, "Could not find sample at offset", iterator.counter.getOffset(), "skipping to end of mdat");
+  const samplePositions = state.iso.flatSamples.getSamples(mediaSection.start);
+  const sampleIndices = state.iso.flatSamples.getCurrentSampleIndices(mediaSection.start);
+  const nextSampleArray = getSampleWithLowestDts(samplePositions, sampleIndices);
+  if (nextSampleArray.length === 0) {
+    Log.verbose(state.logLevel, "Iterated over all samples.", endOfMdat);
     return makeSkip(endOfMdat);
   }
-  if (samplesWithIndex.samplePosition.offset + samplesWithIndex.samplePosition.size > state.contentLength) {
-    Log.verbose(state.logLevel, "Sample is beyond the end of the file. Don't process it.", samplesWithIndex.samplePosition.offset + samplesWithIndex.samplePosition.size, endOfMdat);
+  const exactMatch = nextSampleArray.find((s) => s.samplePosition.offset === state.iterator.counter.getOffset());
+  const nextSample = exactMatch ?? nextSampleArray[0];
+  if (nextSample.samplePosition.offset !== state.iterator.counter.getOffset()) {
+    return makeSkip(nextSample.samplePosition.offset);
+  }
+  if (nextSample.samplePosition.offset + nextSample.samplePosition.size > state.contentLength) {
+    Log.verbose(state.logLevel, "Sample is beyond the end of the file. Don't process it.", nextSample.samplePosition.offset + nextSample.samplePosition.size, endOfMdat);
     return makeSkip(endOfMdat);
   }
-  if (iterator.bytesRemaining() < samplesWithIndex.samplePosition.size) {
-    return makeFetchMoreData(samplesWithIndex.samplePosition.size - iterator.bytesRemaining());
+  const { iterator } = state;
+  if (iterator.bytesRemaining() < nextSample.samplePosition.size) {
+    return makeFetchMoreData(nextSample.samplePosition.size - iterator.bytesRemaining());
   }
   const {
     timestamp: rawCts,
@@ -31765,21 +31717,22 @@ var parseMdatSection = async (state) => {
     offset,
     bigEndian,
     chunkSize
-  } = samplesWithIndex.samplePosition;
+  } = nextSample.samplePosition;
+  const track = tracks2.find((t) => t.trackId === nextSample.trackId);
   const {
     originalTimescale,
     startInSeconds,
     trackMediaTimeOffsetInTrackTimescale,
     timescale: trackTimescale
-  } = samplesWithIndex.track;
+  } = track;
   const cts = rawCts + startInSeconds * originalTimescale - trackMediaTimeOffsetInTrackTimescale / trackTimescale * WEBCODECS_TIMESCALE;
   const dts = rawDts + startInSeconds * originalTimescale - trackMediaTimeOffsetInTrackTimescale / trackTimescale * WEBCODECS_TIMESCALE;
   const bytes = postprocessBytes({
-    bytes: iterator.getSlice(samplesWithIndex.samplePosition.size),
+    bytes: iterator.getSlice(nextSample.samplePosition.size),
     bigEndian,
     chunkSize
   });
-  if (samplesWithIndex.track.type === "audio") {
+  if (track.type === "audio") {
     const audioSample = convertAudioOrVideoSampleToWebCodecsTimestamps({
       sample: {
         data: bytes,
@@ -31793,10 +31746,10 @@ var parseMdatSection = async (state) => {
     });
     await state.callbacks.onAudioSample({
       audioSample,
-      trackId: samplesWithIndex.track.trackId
+      trackId: track.trackId
     });
   }
-  if (samplesWithIndex.track.type === "video") {
+  if (track.type === "video") {
     const nalUnitType = bytes[4] & 31;
     let isRecoveryPoint = false;
     if (nalUnitType === 6) {
@@ -31816,14 +31769,10 @@ var parseMdatSection = async (state) => {
     });
     await state.callbacks.onVideoSample({
       videoSample,
-      trackId: samplesWithIndex.track.trackId
+      trackId: track.trackId
     });
   }
-  const jump = jumpMarks.find((j) => j.afterSampleWithOffset === offset);
-  if (jump) {
-    Log.verbose(state.logLevel, "Found jump mark", jump.jumpToOffset, "skipping to jump mark");
-    return makeSkip(jump.jumpToOffset);
-  }
+  state.iso.flatSamples.setCurrentSampleIndex(mediaSection.start, nextSample.trackId, nextSample.index + 1);
   return null;
 };
 var parseIsoBaseMedia = async (state) => {
@@ -35715,7 +35664,8 @@ var parseLoop = async ({
             fields: state.fields,
             src: state.src,
             discardReadBytes: state.discardReadBytes,
-            prefetchCache: state.prefetchCache
+            prefetchCache: state.prefetchCache,
+            isoState: state.iso
           });
           state.timings.timeSeeking += Date.now() - seekStart;
         }
@@ -38036,7 +37986,8 @@ var TimelineSequenceFrame = ({ roundedFrame, premounted }) => {
 };
 
 // src/components/Timeline/TimelineVideoInfo.tsx
-import { extractFrames, rotateAndResizeVideoFrame } from "@remotion/webcodecs";
+import { rotateAndResizeVideoFrame } from "@remotion/webcodecs";
+import { extractFramesOnWebWorker } from "@remotion/webcodecs/worker";
 import { useEffect as useEffect66, useRef as useRef36, useState as useState66 } from "react";
 import { useVideoConfig as useVideoConfig5 } from "remotion";
 
@@ -38278,7 +38229,7 @@ var TimelineVideoInfo = ({ src, visualizationWidth, startFrom, durationInFrames
       }
     }
     clearOldFrames();
-    extractFrames({
+    extractFramesOnWebWorker({
       acknowledgeRemotionLicense: true,
       timestampsInSeconds: ({ track }) => {
         aspectRatio.current = track.width / track.height;