nodejs-audio-visualizer 5.3.0 → 5.4.1

package/src/highlight.ts

@@ -20,6 +20,12 @@ const BEATS_PER_BAR = 4;
 const OFF_BAR_PENALTY = 3;
 /** Grid beats past the analyzed tempo region snap to a detected onset within this share of a beat. */
 const GRID_ONSET_SNAP_SHARE = 0.2;
+/** Seconds of audio compared before and after a frame to score it as a drop. */
+export const DROP_WINDOW_SEC = 4;
+/** Share of the track's biggest energy rise a peak needs to count as a drop. */
+const MIN_DROP_SCORE_SHARE = 0.5;
+/** With a tempo grid, a drop moves onto a beat within this share of a beat, so the hook ends on the beat. */
+const DROP_BEAT_SNAP_SHARE = 0.2;
 
 /** Tempo grid used to snap highlight cuts to beats (subset of `TempoEstimate`). */
 export interface HighlightBeatGrid {
@@ -32,6 +38,8 @@ export interface HighlightBeatGrid {
 export interface HighlightAudioSegment {
   seekSeconds: number;
   durationSeconds: number;
+  /** Silence played before the audio, for a lead-in reaching back past the track start. */
+  delaySeconds?: number;
 }
 
 /** One auto-highlight segment before concatenation (for separate output files). */
@@ -42,16 +50,27 @@ export interface HighlightRun {
   beatFrameIndices: number[];
   beatIntensities: number[];
   audioSegment: HighlightAudioSegment;
+  /** Frames from the run start to the detected drop (or window start); set when a lead-in was requested. */
+  leadInFrames?: number;
 }
 
+/**
+ * `seg.startFrame` may be negative (a lead-in reaching back past the track start): those
+ * frames get silent spectrums and are played as silence before the audio.
+ */
 function buildHighlightRun(
   fps: number,
-  seg: { startFrame: number; highlightFrames: number },
+  seg: { startFrame: number; highlightFrames: number; leadInFrames?: number },
   spectrums: number[][],
   beatEvents: BeatFrameEvent[],
 ): HighlightRun {
-  const { startFrame, highlightFrames } = seg;
-  const spectrumsSlice = spectrums.slice(startFrame, startFrame + highlightFrames);
+  const { startFrame, highlightFrames, leadInFrames } = seg;
+  const silentFrames = Math.max(0, -startFrame);
+  const silentSpectrum = new Array<number>(spectrums[0]?.length ?? 0).fill(0);
+  const spectrumsSlice = [
+    ...Array.from({ length: silentFrames }, () => silentSpectrum.slice()),
+    ...spectrums.slice(startFrame + silentFrames, startFrame + highlightFrames),
+  ];
   const beatsInWindow = beatEvents
     .filter(b => b.frameIndex >= startFrame && b.frameIndex < startFrame + highlightFrames)
     .sort((a, b) => a.frameIndex - b.frameIndex);
@@ -64,9 +83,11 @@ function buildHighlightRun(
     beatFrameIndices,
     beatIntensities,
     audioSegment: {
-      seekSeconds: startFrame / fps,
-      durationSeconds: highlightFrames / fps,
+      seekSeconds: (startFrame + silentFrames) / fps,
+      durationSeconds: (highlightFrames - silentFrames) / fps,
+      ...(silentFrames > 0 && { delaySeconds: silentFrames / fps }),
     },
+    ...(leadInFrames !== undefined && { leadInFrames }),
   };
 }
 
@@ -184,6 +205,107 @@ async function findBestHighlightStart(
   return bestStart;
 }
 
+/**
+ * Scores each frame by the rise in mean energy from the `dropWindowFrames` before it to the
+ * `dropWindowFrames` after it, and picks the best drop, i.e. a frame whose rise is the biggest
+ * within `dropWindowFrames` on either side, such that the lead-in before it plus `mainFrames`
+ * after it fit in the track and miss the excluded ranges. Frames rather than beat candidates are
+ * scored so the drop lands on the actual energy onset (a beat grid can be off by a fraction of a
+ * beat), and only peaks count so a frame just past a drop is never taken for one. Peaks rising
+ * less than `MIN_DROP_SCORE_SHARE` of the biggest one are not drops. The window
+ * starts exactly `leadInFrames` before the drop, so the whole lead-in plays. Drops too close to
+ * the track start for the full lead-in are only used (with the window starting before the track,
+ * i.e. at a negative frame) when no other drop fits.
+ */
+function findBestDropSegment(
+  energies: Float64Array,
+  totalFrames: number,
+  mainFrames: number,
+  leadInFrames: number,
+  dropWindowFrames: number,
+  excludeRanges: Array<{ start: number; endExclusive: number }>,
+): { startFrame: number; dropFrame: number } | null {
+  const prefix = new Float64Array(totalFrames + 1);
+  for (let i = 0; i < totalFrames; i++) {
+    prefix[i + 1] = prefix[i] + energies[i];
+  }
+  const meanEnergy = (from: number, to: number) => {
+    const a = Math.max(0, from);
+    const b = Math.min(totalFrames, to);
+    return b > a ? (prefix[b] - prefix[a]) / (b - a) : 0;
+  };
+  const scores = new Float64Array(totalFrames);
+  for (let f = 0; f < totalFrames; f++) {
+    scores[f] = meanEnergy(f, f + dropWindowFrames) - meanEnergy(f - dropWindowFrames, f);
+  }
+  const dropFrames: number[] = [];
+  for (let f = 1; f < totalFrames; f++) {
+    if (!(scores[f] > 0)) {
+      continue;
+    }
+    let isPeak = true;
+    const from = Math.max(0, f - dropWindowFrames);
+    const to = Math.min(totalFrames - 1, f + dropWindowFrames);
+    for (let g = from; g <= to && isPeak; g++) {
+      // Ties go to the earliest frame, where the rise starts.
+      isPeak = scores[g] < scores[f] || (scores[g] === scores[f] && g >= f);
+    }
+    if (isPeak) {
+      dropFrames.push(f);
+    }
+  }
+  const maxScore = dropFrames.reduce((max, f) => Math.max(max, scores[f]), 0);
+  const strongDropFrames = dropFrames.filter(f => scores[f] >= maxScore * MIN_DROP_SCORE_SHARE);
+
+  const findBest = (allowBeforeTrackStart: boolean) => {
+    let best: { startFrame: number; dropFrame: number } | null = null;
+    let bestScore = -Infinity;
+    for (const dropFrame of strongDropFrames) {
+      const startFrame = dropFrame - leadInFrames;
+      if (startFrame < 0 && !allowBeforeTrackStart) {
+        continue;
+      }
+      if (
+        dropFrame + mainFrames > totalFrames ||
+        rangesOverlap(startFrame, dropFrame + mainFrames, excludeRanges)
+      ) {
+        continue;
+      }
+      if (scores[dropFrame] > bestScore) {
+        bestScore = scores[dropFrame];
+        best = { startFrame, dropFrame };
+      }
+    }
+    return best;
+  };
+  return findBest(false) ?? findBest(true);
+}
+
+/**
+ * Moves a drop (and its window start with it) onto the nearest beat within `DROP_BEAT_SNAP_SHARE`
+ * of a beat. Only done with a tempo grid: raw onsets alone are too unreliable to override the
+ * energy onset. A lead-in that fit in the track is never pushed back past its start.
+ */
+export function snapDropToBeat(
+  drop: { startFrame: number; dropFrame: number },
+  beatFrames: number[],
+  periodFrames?: number,
+): void {
+  if (!periodFrames || !(periodFrames > 0) || !isFinite(periodFrames)) {
+    return;
+  }
+  const beat = nearestOnset(beatFrames, drop.dropFrame);
+  if (beat === null || Math.abs(beat - drop.dropFrame) > periodFrames * DROP_BEAT_SNAP_SHARE) {
+    return;
+  }
+  const startFrame = drop.startFrame + (beat - drop.dropFrame);
+  if (startFrame < 0 && drop.startFrame >= 0) {
+    return;
+  }
+  drop.startFrame = startFrame;
+  drop.dropFrame = beat;
+}
+
 function nearestOnset(sortedOnsets: number[], frame: number): number | null {
   let lo = 0;
   let hi = sortedOnsets.length;
@@ -269,12 +391,14 @@ export function snapSegmentToBeats(
   beatFrames: number[],
   maxShiftFrames: number,
   periodFrames?: number,
+  /** Keep the start where it is and only move the end. */
+  lockStart = false,
 ): { startFrame: number; highlightFrames: number } {
   const rawStart = seg.startFrame;
   const rawEnd = seg.startFrame + seg.highlightFrames;
   const near = (target: number, max: number) =>
     beatFrames.filter(f => Math.abs(f - target) <= maxShiftFrames && f >= 0 && f <= max);
-  const startCands = near(rawStart, totalFrames - 1);
+  const startCands = lockStart ? [rawStart] : near(rawStart, totalFrames - 1);
   const endCands = near(rawEnd, totalFrames);
   if (startCands.length === 0) {
     startCands.push(rawStart);
@@ -314,6 +438,14 @@ export function snapSegmentToBeats(
  * Picks up to `segmentCount` non-overlapping ~15s windows with highest summed spectral energy each,
  * moves their boundaries onto beats (see `snapSegmentToBeats`), sorts them chronologically,
  * concatenates spectrums, and remaps beat indices.
+ *
+ * With `leadInFrames` (e.g. a hook video's length), each window is instead built around a detected
+ * drop: it starts `leadInFrames` before the drop, so the whole lead-in plays and ends on the drop
+ * (see `HighlightRun.leadInFrames`), and runs ~15s past the drop, so the lead-in is added on top of
+ * the ~15s rather than taken out of it. With a tempo grid, the drop is moved onto a nearby beat
+ * (see `snapDropToBeat`), so the hook ends on the beat. When no drop fits, the lead-in is put before the energy
+ * window instead. The lead-in is never shortened: where it reaches back past the track start,
+ * the run starts at a negative frame and that part is silent (see `buildHighlightRun`).
  */
 export async function computeHighlightSlice(
   fps: number,
@@ -322,6 +454,7 @@ export async function computeHighlightSlice(
   beatEvents: BeatFrameEvent[],
   segmentCount = 1,
   beatGrid?: HighlightBeatGrid | null,
+  leadInFrames?: number,
 ): Promise<{
   startFrame: number;
   highlightFrames: number;
@@ -350,23 +483,21 @@ export async function computeHighlightSlice(
   }
 
   if (totalFrames <= highlightFramesTarget) {
-    const audioSegments: HighlightAudioSegment[] = [
-      {
-        seekSeconds: 0,
-        durationSeconds: totalFrames / fps,
-      },
-    ];
-    const fullSeg = { startFrame: 0, highlightFrames: totalFrames };
+    // The whole track follows the lead-in, which is silent as it all comes before the track start.
+    const leadIn = leadInFrames !== undefined && leadInFrames > 0 ? leadInFrames : 0;
+    const fullSeg = leadIn > 0
+      ? { startFrame: -leadIn, highlightFrames: leadIn + totalFrames, leadInFrames: leadIn }
+      : { startFrame: 0, highlightFrames: totalFrames };
     const fullRun = buildHighlightRun(fps, fullSeg, spectrums, beatEvents);
     return {
-      startFrame: 0,
-      highlightFrames: totalFrames,
-      spectrums: spectrums.slice(),
+      startFrame: fullRun.startFrame,
+      highlightFrames: fullRun.highlightFrames,
+      spectrums: fullRun.spectrums,
       beatFrameIndices: fullRun.beatFrameIndices,
       beatIntensities: fullRun.beatIntensities,
-      audioSeekSeconds: 0,
-      audioDurationSeconds: totalFrames / fps,
-      audioSegments,
+      audioSeekSeconds: fullRun.audioSegment.seekSeconds,
+      audioDurationSeconds: fullRun.audioSegment.durationSeconds,
+      audioSegments: [fullRun.audioSegment],
       runs: [fullRun],
     };
   }
@@ -374,12 +505,48 @@ export async function computeHighlightSlice(
   const highlightFrames = highlightFramesTarget;
   const energies = await buildFrameEnergies(spectrums, totalFrames);
   const excludeRanges: Array<{ start: number; endExclusive: number }> = [];
-  const rawSegments: Array<{ startFrame: number; highlightFrames: number }> = [];
+  const rawSegments: Array<{ startFrame: number; highlightFrames: number; leadInFrames?: number }> = [];
   const beatFrames = buildBeatCandidates(fps, totalFrames, beatEvents, beatGrid);
   const maxShiftFrames = Math.round(MAX_BEAT_SNAP_SEC * fps);
+  const useDrops = leadInFrames !== undefined && leadInFrames > 0;
+  const dropWindowFrames = Math.round(DROP_WINDOW_SEC * fps);
 
   const n = Math.max(1, Math.floor(segmentCount));
   for (let k = 0; k < n; k++) {
+    const drop = useDrops
+      ? findBestDropSegment(
+          energies,
+          totalFrames,
+          highlightFrames,
+          leadInFrames as number,
+          dropWindowFrames,
+          excludeRanges,
+        )
+      : null;
+    if (drop) {
+      snapDropToBeat(drop, beatFrames, beatGrid?.periodFrames);
+      // Only the part from the drop on is ~15s; its end is snapped to a beat.
+      const main = snapSegmentToBeats(
+        { startFrame: drop.dropFrame, highlightFrames },
+        totalFrames,
+        beatFrames,
+        maxShiftFrames,
+        beatGrid?.periodFrames,
+        true,
+      );
+      const dropLeadIn = drop.dropFrame - drop.startFrame;
+      rawSegments.push({
+        startFrame: drop.startFrame,
+        highlightFrames: dropLeadIn + main.highlightFrames,
+        leadInFrames: dropLeadIn,
+      });
+      excludeRanges.push({
+        start: drop.startFrame - maxShiftFrames,
+        endExclusive: main.startFrame + main.highlightFrames + maxShiftFrames,
+      });
+      await waitForEventLoop();
+      continue;
+    }
     const startFrame = await findBestHighlightStart(
       energies,
       totalFrames,
@@ -396,7 +563,16 @@ export async function computeHighlightSlice(
       maxShiftFrames,
       beatGrid?.periodFrames,
     );
-    rawSegments.push(snapped);
+    // The whole lead-in plays before the window, even over an earlier window (each highlight with
+    // a lead-in is its own output) or before the track start (as silence).
+    const windowLeadIn = useDrops ? leadInFrames as number : 0;
+    rawSegments.push(windowLeadIn > 0
+      ? {
+          startFrame: snapped.startFrame - windowLeadIn,
+          highlightFrames: windowLeadIn + snapped.highlightFrames,
+          leadInFrames: windowLeadIn,
+        }
+      : snapped);
     // Padded so the next raw window, once snapped, cannot reach back into this one.
     excludeRanges.push({
       start: snapped.startFrame - maxShiftFrames,
@@ -415,25 +591,15 @@ export async function computeHighlightSlice(
   const beatIntensities: number[] = [];
   let outOffset = 0;
 
-  for (const seg of rawSegments) {
-    const chunk = spectrums.slice(seg.startFrame, seg.startFrame + seg.highlightFrames);
-    slicedSpectrums.push(...chunk);
-
-    const beatsInSeg = beatEvents
-      .filter(b => b.frameIndex >= seg.startFrame && b.frameIndex < seg.startFrame + seg.highlightFrames)
-      .sort((a, b) => a.frameIndex - b.frameIndex);
-    for (const b of beatsInSeg) {
-      beatFrameIndices.push(b.frameIndex - seg.startFrame + outOffset);
-      beatIntensities.push(b.intensity);
-    }
-    outOffset += seg.highlightFrames;
+  for (const run of runs) {
+    slicedSpectrums.push(...run.spectrums);
+    beatFrameIndices.push(...run.beatFrameIndices.map(frameIndex => frameIndex + outOffset));
+    beatIntensities.push(...run.beatIntensities);
+    outOffset += run.highlightFrames;
   }
 
   const totalHighlightFrames = slicedSpectrums.length;
-  const audioSegments: HighlightAudioSegment[] = rawSegments.map(seg => ({
-    seekSeconds: seg.startFrame / fps,
-    durationSeconds: seg.highlightFrames / fps,
-  }));
+  const audioSegments: HighlightAudioSegment[] = runs.map(run => run.audioSegment);
 
   const first = audioSegments[0] ?? { seekSeconds: 0, durationSeconds: 0 };
 

package/src/index.ts

@@ -4,6 +4,7 @@ import {
   getAudioFilePath,
   getBackgroundImagePath,
   getBackgroundVideoPaths,
+  getHookVideoPath,
   getOutVideoPath,
   getSubtitleRenderSpec,
   subtitleAlignmentToAss,
@@ -40,9 +41,9 @@ import {
 import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor, pcmU8ToFloatSamples } from './audio';
 import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps, applyCameraShake, getCutShakeOffset, buildCutShakeAmplitudes, CAMERA_SHAKE_DELAY_SECONDS, CAMERA_SHAKE_EVERY_BEATS } from './image';
 import { normalizeInlineSubtitlesToSrt, lrcToSrt } from './subtitleConvert';
-import { spawnFfmpegVideoWriter, waitDrain, waitForProcessExit, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, getCutFrameIndices, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile, VideoSegment } from './video';
+import { spawnFfmpegVideoWriter, waitDrain, waitForProcessExit, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, computeHookFrameCount, getVideoFrameCount, getCutFrameIndices, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile, VideoSegment } from './video';
 import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
-import { createBeatDetector, estimateTempo, TempoEstimate, beatGridFrameIndices, tempoForWindow, MAX_TEMPO_ANALYSIS_SECONDS } from './beats';
+import { createBeatDetector, estimateTempo, TempoEstimate, beatGridFrameIndices, tempoForWindow, shiftTempoPhase, MAX_TEMPO_ANALYSIS_SECONDS } from './beats';
 export { BeatInfo, BeatDetectorOptions, TempoEstimate, estimateTempo, beatGridFrameIndices, shiftTempoPhase, tempoForWindow } from './beats';
 import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
 import { waitForEventLoop } from './waitForEventLoop';
@@ -74,6 +75,14 @@ export interface Config {
      * videos instead of auto-detecting scene changes within a single video.
      */
     path: string | string[];
+    /**
+     * Video played once, always in full, before the `path` video(s). When `audio.autoHighlight`
+     * is true, each highlight is chosen so the hook ends on a detected drop and `path` starts with it,
+     * playing ~15s from the drop (the hook comes on top of the ~15s rather than being counted in it).
+     * If the drop is closer to the track start than the hook's length, the track is preceded by silence.
+     * The hook's sound, if any, is mixed over the track, which fades in from silence to full volume while the hook plays.
+     */
+    hookPath?: string;
     autoEdit?: boolean;
     /** When `autoEdit` is true, disables the brief camera shake applied on cuts, or every 2 beats when there are no cuts (default true). */
     cameraShake?: boolean;
@@ -284,6 +293,10 @@ async function prepareBackgroundForRender(params: {
   tempo?: TempoEstimate | null;
   /** First video to play for `video.path` arrays without `autoEdit`. */
   startVideoIndex?: number;
+  /** Video played once before the background videos. */
+  hookVideoPath?: string;
+  /** Hook length in frames (the lead-in up to a detected drop); the full hook when unset. */
+  hookFrameCount?: number;
 }): Promise<{
   backgroundWidth: number;
   backgroundHeight: number;
@@ -294,6 +307,8 @@ async function prepareBackgroundForRender(params: {
   encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
   concatFilePath?: string;
   cutFrameIndices: number[];
+  /** Hook video whose sound plays over the first `frameCount` output frames. */
+  hookAudio?: { path: string; frameCount: number };
 }> {
   const {
     useVideoBackground,
@@ -308,6 +323,8 @@ async function prepareBackgroundForRender(params: {
     readVideoFrameTimeout,
     tempo,
     startVideoIndex = 0,
+    hookVideoPath,
+    hookFrameCount,
   } = params;
 
   if (useVideoBackground && backgroundVideoPaths && backgroundVideoPaths.length > 0) {
@@ -319,21 +336,43 @@ async function prepareBackgroundForRender(params: {
     const videoFrameSize = backgroundWidth * backgroundHeight * 3;
     const encodeVideoFrame = createBgrFrameEncoder({ width: backgroundWidth, height: backgroundHeight });
 
+    const hookVideoInfo = hookVideoPath ? await getVideoInfo(hookVideoPath) : undefined;
+    // Real frame count, so the hook never has to wrap back to its start.
+    const hookAvailableFrames = hookVideoPath ? await getVideoFrameCount(hookVideoPath, fps) : 0;
+    const hookFrames = !hookVideoInfo
+      ? 0
+      : hookFrameCount !== undefined
+        ? Math.min(hookFrameCount, hookAvailableFrames, framesCount)
+        : computeHookFrameCount(hookAvailableFrames / fps, fps, framesCount);
+    // The background videos fill the frames after the hook, on a timeline starting at the hook's end.
+    const mainFramesCount = framesCount - hookFrames;
+    const mainBeatFrameIndices: number[] = [];
+    const mainBeatIntensities: number[] | undefined = beatIntensities ? [] : undefined;
+    beatFrameIndices.forEach((frameIndex, i) => {
+      if (frameIndex >= hookFrames) {
+        mainBeatFrameIndices.push(frameIndex - hookFrames);
+        mainBeatIntensities?.push(beatIntensities![i]);
+      }
+    });
+    const mainTempo = tempo ? shiftTempoPhase(tempo, hookFrames) : tempo;
+
     let segments: VideoSegment[];
-    if (backgroundVideoPaths.length > 1) {
+    if (mainFramesCount <= 0) {
+      segments = [];
+    } else if (backgroundVideoPaths.length > 1) {
       segments = autoEditVideo
         ? buildShuffledVideoSegments(
-            beatFrameIndices,
-            framesCount,
+            mainBeatFrameIndices,
+            mainFramesCount,
             backgroundVideoPaths.length,
             fps,
             {
-              beatIntensities,
-              ...(tempo ? { tempo } : {}),
+              beatIntensities: mainBeatIntensities,
+              ...(mainTempo ? { tempo: mainTempo } : {}),
             },
           )
         : buildSequentialVideoSegments(
-            framesCount,
+            mainFramesCount,
             videoInfos.map(info => info.duration),
             fps,
             startVideoIndex % backgroundVideoPaths.length,
@@ -343,19 +382,29 @@ async function prepareBackgroundForRender(params: {
         ? await detectSceneChanges(backgroundVideoPaths[0])
         : [];
       segments = buildBeatSyncedSegments(
-        beatFrameIndices,
-        framesCount,
+        mainBeatFrameIndices,
+        mainFramesCount,
         sceneChanges,
         primaryVideoInfo.duration,
         fps,
         {
-          beatIntensities,
-          ...(autoEditVideo && tempo ? { tempo } : {}),
+          beatIntensities: mainBeatIntensities,
+          ...(autoEditVideo && mainTempo ? { tempo: mainTempo } : {}),
         },
       );
     }
 
     const videoSources = videoInfos.map((info, i) => ({ path: backgroundVideoPaths[i], duration: info.duration }));
+    if (hookVideoPath && hookVideoInfo && hookFrames > 0) {
+      segments = [
+        { outputStartFrame: 0, videoSeekSeconds: 0, frameCount: hookFrames, videoIndex: videoSources.length },
+        ...segments.map(segment => ({
+          ...segment,
+          outputStartFrame: segment.outputStartFrame + hookFrames,
+        })),
+      ];
+      videoSources.push({ path: hookVideoPath, duration: hookAvailableFrames / fps });
+    }
     const concatFilePath = writeConcatFile(segments, videoSources, fps);
     const cutFrameIndices = autoEditVideo ? getCutFrameIndices(segments) : [];
 
@@ -377,7 +426,7 @@ async function prepareBackgroundForRender(params: {
       readVideoFrameTimeout,
     );
     if (!firstFrame) {
-      throw new Error(`Could not read frames from video: ${backgroundVideoPaths.join(', ')}`);
+      throw new Error(`Could not read frames from video: ${videoSources.map(source => source.path).join(', ')}`);
     }
     const defaultColor = getVideoFrameColor(firstFrame, backgroundWidth, backgroundHeight);
     const staticBackgroundBuffer = encodeVideoFrame(firstFrame);
@@ -391,6 +440,9 @@ async function prepareBackgroundForRender(params: {
       encodeVideoFrame,
       concatFilePath,
       cutFrameIndices,
+      ...(hookVideoPath && hookVideoInfo?.hasAudio && hookFrames > 0 && {
+        hookAudio: { path: hookVideoPath, frameCount: hookFrames },
+      }),
     };
   }
 
@@ -501,6 +553,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
       }
     }
     const backgroundVideoPaths = getBackgroundVideoPaths(config);
+    const hookVideoPath = getHookVideoPath(config);
     const backgroundImagePath = getBackgroundImagePath(config);
     const useVideoBackground = !!backgroundVideoPaths && backgroundVideoPaths.length > 0;
 
@@ -572,6 +625,10 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
 
     if (autoHighlight) {
       reportProgress(PRE_PROCESS_PROGRESS_SHARE + 1);
+      // With a hook, highlights are built so the hook ends on a detected drop.
+      const hookLeadInFrames = hookVideoPath
+        ? await getVideoFrameCount(hookVideoPath, FPS)
+        : undefined;
       highlightSlice = await computeHighlightSlice(
         FPS,
         framesCount,
@@ -579,6 +636,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
         preprocessed.beatEvents,
         getAudioAutoHighlightCount(config),
         trackTempo,
+        hookLeadInFrames,
       );
       spectrumsForRender = highlightSlice.spectrums;
       beatIndicesForRender = highlightSlice.beatFrameIndices;
@@ -605,6 +663,8 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
       frameCount: number;
       startFrame: number;
       audioSegment: import('./highlight').HighlightAudioSegment | undefined;
+      /** Frames until the detected drop, where the hook should end. */
+      leadInFrames?: number;
     };
 
     const passes: VideoRenderPass[] =
@@ -622,6 +682,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
             frameCount: run.highlightFrames,
             startFrame: run.startFrame,
             audioSegment: run.audioSegment,
+            leadInFrames: run.leadInFrames,
           };
         })
       : [
@@ -638,6 +699,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
               highlightSlice.audioSegments.length > 0
                 ? highlightSlice.audioSegments[0]
                 : undefined,
+            leadInFrames: highlightSlice?.runs.length === 1 ? highlightSlice.runs[0].leadInFrames : undefined,
           },
         ];
 
@@ -659,9 +721,10 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
     try {
       passLoop: for (let passIndex = 0; passIndex < passes.length; passIndex++) {
         const pass = passes[passIndex];
-        const tempo = autoEditVideo && trackTempo
+        const windowTempo = trackTempo
           ? tempoForWindow(trackTempo, pass.startFrame, FPS, pass.frameCount, pass.beatIndices)
           : null;
+        const tempo = autoEditVideo ? windowTempo : null;
         const shakeAmplitudes = tempo
           ? buildCutShakeAmplitudes(tempo.periodFrames)
           : undefined;
@@ -675,6 +738,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
           encodeVideoFrame,
           concatFilePath,
           cutFrameIndices,
+          hookAudio,
         } = await prepareBackgroundForRender({
           useVideoBackground,
           backgroundVideoPaths,
@@ -689,6 +753,8 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
           ...(tempo ? { tempo } : {}),
           // Each separate highlight output opens with a different video from the array.
           startVideoIndex: passIndex,
+          hookVideoPath,
+          ...(pass.leadInFrames !== undefined ? { hookFrameCount: pass.leadInFrames } : {}),
         });
         reportRenderProgress();
 
@@ -713,6 +779,9 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
           }),
           fps: FPS,
           ...(pass.audioSegment && { audioSegment: pass.audioSegment }),
+          ...(hookAudio && {
+            overlayAudio: { filename: hookAudio.path, durationSeconds: hookAudio.frameCount / FPS },
+          }),
           ...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
           ...(ffmpeg_preset && { preset: ffmpeg_preset }),
         });

package/src/tests/config.test.ts

@@ -8,6 +8,7 @@ import {
   getAudioAutoHighlightCount,
   getBackgroundImagePath,
   getBackgroundVideoPaths,
+  getHookVideoPath,
   getCameraShakeEnabled,
   getOutVideoPath,
   getSubtitleRenderSpec,
@@ -93,6 +94,24 @@ describe('config', function() {
     ).to.throw('Invalid video.path');
   });
 
+  it('getHookVideoPath', function() {
+    expect(
+      getHookVideoPath({ audio: { path: 'a.wav' } } as Config)
+    ).equal(undefined);
+
+    expect(
+      getHookVideoPath({ video: { path: 'a.mp4' } } as Config)
+    ).equal(undefined);
+
+    expect(
+      getHookVideoPath({ video: { path: 'a.mp4', hookPath: 'hook.mp4' } } as Config)
+    ).equal(path.resolve('hook.mp4'));
+
+    expect(
+      getHookVideoPath.bind(undefined, { video: { path: 'a.mp4', hookPath: '' } } as Config)
+    ).to.throw('Invalid video.hookPath');
+  });
+
   it('getCameraShakeEnabled', function() {
     expect(
       getCameraShakeEnabled({ audio: { path: 'a.wav' } } as Config)