nodejs-audio-visualizer 5.3.0 → 5.4.0

package/src/index.ts

@@ -4,6 +4,7 @@ import {
   getAudioFilePath,
   getBackgroundImagePath,
   getBackgroundVideoPaths,
+  getHookVideoPath,
   getOutVideoPath,
   getSubtitleRenderSpec,
   subtitleAlignmentToAss,
@@ -40,9 +41,9 @@ import {
 import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor, pcmU8ToFloatSamples } from './audio';
 import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps, applyCameraShake, getCutShakeOffset, buildCutShakeAmplitudes, CAMERA_SHAKE_DELAY_SECONDS, CAMERA_SHAKE_EVERY_BEATS } from './image';
 import { normalizeInlineSubtitlesToSrt, lrcToSrt } from './subtitleConvert';
-import { spawnFfmpegVideoWriter, waitDrain, waitForProcessExit, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, getCutFrameIndices, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile, VideoSegment } from './video';
+import { spawnFfmpegVideoWriter, waitDrain, waitForProcessExit, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, computeHookFrameCount, getVideoFrameCount, getCutFrameIndices, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile, VideoSegment } from './video';
 import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
-import { createBeatDetector, estimateTempo, TempoEstimate, beatGridFrameIndices, tempoForWindow, MAX_TEMPO_ANALYSIS_SECONDS } from './beats';
+import { createBeatDetector, estimateTempo, TempoEstimate, beatGridFrameIndices, tempoForWindow, shiftTempoPhase, MAX_TEMPO_ANALYSIS_SECONDS } from './beats';
 export { BeatInfo, BeatDetectorOptions, TempoEstimate, estimateTempo, beatGridFrameIndices, shiftTempoPhase, tempoForWindow } from './beats';
 import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
 import { waitForEventLoop } from './waitForEventLoop';
@@ -74,6 +75,14 @@ export interface Config {
      * videos instead of auto-detecting scene changes within a single video.
      */
     path: string | string[];
+    /**
+     * Video played once, always in full, before the `path` video(s). When `audio.autoHighlight`
+     * is true, each highlight is chosen so the hook ends on a detected drop and `path` starts with it,
+     * playing ~15s from the drop (the hook comes on top of the ~15s rather than being counted in it).
+     * If the drop is closer to the track start than the hook's length, the track is preceded by silence.
+     * The hook's sound, if any, is mixed over the track, which fades in from silence to full volume while the hook plays.
+     */
+    hookPath?: string;
     autoEdit?: boolean;
     /** When `autoEdit` is true, disables the brief camera shake applied on cuts, or every 2 beats when there are no cuts (default true). */
     cameraShake?: boolean;
@@ -284,6 +293,10 @@ async function prepareBackgroundForRender(params: {
   tempo?: TempoEstimate | null;
   /** First video to play for `video.path` arrays without `autoEdit`. */
   startVideoIndex?: number;
+  /** Video played once before the background videos. */
+  hookVideoPath?: string;
+  /** Hook length in frames (the lead-in up to a detected drop); the full hook when unset. */
+  hookFrameCount?: number;
 }): Promise<{
   backgroundWidth: number;
   backgroundHeight: number;
@@ -294,6 +307,8 @@ async function prepareBackgroundForRender(params: {
   encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
   concatFilePath?: string;
   cutFrameIndices: number[];
+  /** Hook video whose sound plays over the first `frameCount` output frames. */
+  hookAudio?: { path: string; frameCount: number };
 }> {
   const {
     useVideoBackground,
@@ -308,6 +323,8 @@ async function prepareBackgroundForRender(params: {
     readVideoFrameTimeout,
     tempo,
     startVideoIndex = 0,
+    hookVideoPath,
+    hookFrameCount,
   } = params;
 
   if (useVideoBackground && backgroundVideoPaths && backgroundVideoPaths.length > 0) {
@@ -319,21 +336,43 @@ async function prepareBackgroundForRender(params: {
     const videoFrameSize = backgroundWidth * backgroundHeight * 3;
     const encodeVideoFrame = createBgrFrameEncoder({ width: backgroundWidth, height: backgroundHeight });
 
+    const hookVideoInfo = hookVideoPath ? await getVideoInfo(hookVideoPath) : undefined;
+    // Real frame count, so the hook never has to wrap back to its start.
+    const hookAvailableFrames = hookVideoPath ? await getVideoFrameCount(hookVideoPath, fps) : 0;
+    const hookFrames = !hookVideoInfo
+      ? 0
+      : hookFrameCount !== undefined
+        ? Math.min(hookFrameCount, hookAvailableFrames, framesCount)
+        : computeHookFrameCount(hookAvailableFrames / fps, fps, framesCount);
+    // The background videos fill the frames after the hook, on a timeline starting at the hook's end.
+    const mainFramesCount = framesCount - hookFrames;
+    const mainBeatFrameIndices: number[] = [];
+    const mainBeatIntensities: number[] | undefined = beatIntensities ? [] : undefined;
+    beatFrameIndices.forEach((frameIndex, i) => {
+      if (frameIndex >= hookFrames) {
+        mainBeatFrameIndices.push(frameIndex - hookFrames);
+        mainBeatIntensities?.push(beatIntensities![i]);
+      }
+    });
+    const mainTempo = tempo ? shiftTempoPhase(tempo, hookFrames) : tempo;
+
     let segments: VideoSegment[];
-    if (backgroundVideoPaths.length > 1) {
+    if (mainFramesCount <= 0) {
+      segments = [];
+    } else if (backgroundVideoPaths.length > 1) {
       segments = autoEditVideo
         ? buildShuffledVideoSegments(
-            beatFrameIndices,
-            framesCount,
+            mainBeatFrameIndices,
+            mainFramesCount,
             backgroundVideoPaths.length,
             fps,
             {
-              beatIntensities,
-              ...(tempo ? { tempo } : {}),
+              beatIntensities: mainBeatIntensities,
+              ...(mainTempo ? { tempo: mainTempo } : {}),
             },
           )
         : buildSequentialVideoSegments(
-            framesCount,
+            mainFramesCount,
             videoInfos.map(info => info.duration),
             fps,
             startVideoIndex % backgroundVideoPaths.length,
@@ -343,19 +382,29 @@ async function prepareBackgroundForRender(params: {
         ? await detectSceneChanges(backgroundVideoPaths[0])
         : [];
       segments = buildBeatSyncedSegments(
-        beatFrameIndices,
-        framesCount,
+        mainBeatFrameIndices,
+        mainFramesCount,
         sceneChanges,
         primaryVideoInfo.duration,
         fps,
         {
-          beatIntensities,
-          ...(autoEditVideo && tempo ? { tempo } : {}),
+          beatIntensities: mainBeatIntensities,
+          ...(autoEditVideo && mainTempo ? { tempo: mainTempo } : {}),
         },
       );
     }
 
     const videoSources = videoInfos.map((info, i) => ({ path: backgroundVideoPaths[i], duration: info.duration }));
+    if (hookVideoPath && hookVideoInfo && hookFrames > 0) {
+      segments = [
+        { outputStartFrame: 0, videoSeekSeconds: 0, frameCount: hookFrames, videoIndex: videoSources.length },
+        ...segments.map(segment => ({
+          ...segment,
+          outputStartFrame: segment.outputStartFrame + hookFrames,
+        })),
+      ];
+      videoSources.push({ path: hookVideoPath, duration: hookAvailableFrames / fps });
+    }
     const concatFilePath = writeConcatFile(segments, videoSources, fps);
     const cutFrameIndices = autoEditVideo ? getCutFrameIndices(segments) : [];
 
@@ -377,7 +426,7 @@ async function prepareBackgroundForRender(params: {
       readVideoFrameTimeout,
     );
     if (!firstFrame) {
-      throw new Error(`Could not read frames from video: ${backgroundVideoPaths.join(', ')}`);
+      throw new Error(`Could not read frames from video: ${videoSources.map(source => source.path).join(', ')}`);
     }
     const defaultColor = getVideoFrameColor(firstFrame, backgroundWidth, backgroundHeight);
     const staticBackgroundBuffer = encodeVideoFrame(firstFrame);
@@ -391,6 +440,9 @@ async function prepareBackgroundForRender(params: {
       encodeVideoFrame,
       concatFilePath,
       cutFrameIndices,
+      ...(hookVideoPath && hookVideoInfo?.hasAudio && hookFrames > 0 && {
+        hookAudio: { path: hookVideoPath, frameCount: hookFrames },
+      }),
     };
   }
 
@@ -501,6 +553,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
       }
     }
     const backgroundVideoPaths = getBackgroundVideoPaths(config);
+    const hookVideoPath = getHookVideoPath(config);
     const backgroundImagePath = getBackgroundImagePath(config);
     const useVideoBackground = !!backgroundVideoPaths && backgroundVideoPaths.length > 0;
 
@@ -572,6 +625,10 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
 
     if (autoHighlight) {
       reportProgress(PRE_PROCESS_PROGRESS_SHARE + 1);
+      // With a hook, highlights are built so the hook ends on a detected drop.
+      const hookLeadInFrames = hookVideoPath
+        ? await getVideoFrameCount(hookVideoPath, FPS)
+        : undefined;
       highlightSlice = await computeHighlightSlice(
         FPS,
         framesCount,
@@ -579,6 +636,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
         preprocessed.beatEvents,
         getAudioAutoHighlightCount(config),
         trackTempo,
+        hookLeadInFrames,
       );
       spectrumsForRender = highlightSlice.spectrums;
       beatIndicesForRender = highlightSlice.beatFrameIndices;
@@ -605,6 +663,8 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
       frameCount: number;
       startFrame: number;
       audioSegment: import('./highlight').HighlightAudioSegment | undefined;
+      /** Frames until the detected drop, where the hook should end. */
+      leadInFrames?: number;
     };
 
     const passes: VideoRenderPass[] =
@@ -622,6 +682,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
             frameCount: run.highlightFrames,
             startFrame: run.startFrame,
             audioSegment: run.audioSegment,
+            leadInFrames: run.leadInFrames,
           };
         })
       : [
@@ -638,6 +699,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
               highlightSlice.audioSegments.length > 0
                 ? highlightSlice.audioSegments[0]
                 : undefined,
+            leadInFrames: highlightSlice?.runs.length === 1 ? highlightSlice.runs[0].leadInFrames : undefined,
           },
         ];
 
@@ -659,9 +721,10 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
     try {
       passLoop: for (let passIndex = 0; passIndex < passes.length; passIndex++) {
         const pass = passes[passIndex];
-        const tempo = autoEditVideo && trackTempo
+        const windowTempo = trackTempo
           ? tempoForWindow(trackTempo, pass.startFrame, FPS, pass.frameCount, pass.beatIndices)
           : null;
+        const tempo = autoEditVideo ? windowTempo : null;
         const shakeAmplitudes = tempo
           ? buildCutShakeAmplitudes(tempo.periodFrames)
           : undefined;
@@ -675,6 +738,7 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
           encodeVideoFrame,
           concatFilePath,
           cutFrameIndices,
+          hookAudio,
         } = await prepareBackgroundForRender({
           useVideoBackground,
           backgroundVideoPaths,
@@ -689,6 +753,8 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
           ...(tempo ? { tempo } : {}),
           // Each separate highlight output opens with a different video from the array.
           startVideoIndex: passIndex,
+          hookVideoPath,
+          ...(pass.leadInFrames !== undefined ? { hookFrameCount: pass.leadInFrames } : {}),
         });
         reportRenderProgress();
 
@@ -713,6 +779,9 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
           }),
           fps: FPS,
           ...(pass.audioSegment && { audioSegment: pass.audioSegment }),
+          ...(hookAudio && {
+            overlayAudio: { filename: hookAudio.path, durationSeconds: hookAudio.frameCount / FPS },
+          }),
           ...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
           ...(ffmpeg_preset && { preset: ffmpeg_preset }),
         });

package/src/tests/config.test.ts

@@ -8,6 +8,7 @@ import {
   getAudioAutoHighlightCount,
   getBackgroundImagePath,
   getBackgroundVideoPaths,
+  getHookVideoPath,
   getCameraShakeEnabled,
   getOutVideoPath,
   getSubtitleRenderSpec,
@@ -93,6 +94,24 @@ describe('config', function() {
     ).to.throw('Invalid video.path');
   });
 
+  it('getHookVideoPath', function() {
+    expect(
+      getHookVideoPath({ audio: { path: 'a.wav' } } as Config)
+    ).equal(undefined);
+
+    expect(
+      getHookVideoPath({ video: { path: 'a.mp4' } } as Config)
+    ).equal(undefined);
+
+    expect(
+      getHookVideoPath({ video: { path: 'a.mp4', hookPath: 'hook.mp4' } } as Config)
+    ).equal(path.resolve('hook.mp4'));
+
+    expect(
+      getHookVideoPath.bind(undefined, { video: { path: 'a.mp4', hookPath: '' } } as Config)
+    ).to.throw('Invalid video.hookPath');
+  });
+
   it('getCameraShakeEnabled', function() {
     expect(
       getCameraShakeEnabled({ audio: { path: 'a.wav' } } as Config)

package/src/tests/highlight.test.ts

@@ -5,6 +5,7 @@ import {
   MAX_BEAT_SNAP_SEC,
   buildBeatCandidates,
   snapSegmentToBeats,
+  snapDropToBeat,
 } from '../highlight';
 
 const dummySpectrums = (n: number) => Array.from({ length: n }, () => [0]);
@@ -183,6 +184,115 @@ describe('computeHighlightSlice', function() {
     expect(result.runs).have.length(1);
   });
 
+  it('starts the whole lead-in before the drop when leadInFrames is given', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    // Quiet until frame 600, loud after: the drop is at 600.
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 600, totalFrames);
+    const beats = Array.from({ length: 200 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 43);
+
+    expect(result.runs).have.length(1);
+    expect(result.runs[0].startFrame).equal(557);
+    expect(result.runs[0].leadInFrames).equal(43);
+    // The lead-in comes on top of the ~15s that follow the drop.
+    expect(result.runs[0].highlightFrames).equal(43 + Math.ceil(HIGHLIGHT_DURATION_SEC * fps));
+  });
+
+  it('pads the lead-in with silence before the track start when only an early drop fits', async function() {
+    const fps = 10;
+    // A full 130-frame lead-in needs a drop at 130+, but ~15s must follow it: only 100 fits.
+    const totalFrames = 250;
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 100, totalFrames);
+    const beats = Array.from({ length: 50 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 130);
+    const run = result.runs[0];
+
+    expect(run.startFrame).equal(-30);
+    expect(run.leadInFrames).equal(130);
+    expect(run.spectrums).have.length(run.highlightFrames);
+    expect(run.spectrums.slice(0, 30).every(s => s.every(v => v === 0))).equal(true);
+    expect(run.beatFrameIndices[0]).equal(30);
+    expect(run.audioSegment).deep.equal({
+      seekSeconds: 0,
+      durationSeconds: (run.highlightFrames - 30) / fps,
+      delaySeconds: 3,
+    });
+  });
+
+  it('picks a separate drop for each highlight', async function() {
+    const fps = 10;
+    const totalFrames = 2000;
+    const spectrums = Array.from({ length: totalFrames }, (_, i) =>
+      (i >= 500 && i < 800) || (i >= 1400 && i < 1700) ? [1] : [0],
+    );
+    const beats = Array.from({ length: 400 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 2, null, 30);
+
+    expect(result.runs.map(run => run.startFrame + (run.leadInFrames as number))).deep.equal([500, 1400]);
+    expect(result.runs.map(run => run.leadInFrames)).deep.equal([30, 30]);
+  });
+
+  it('ends the lead-in on the energy onset even when beats are off it', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 602, totalFrames);
+    // Beats every 5 frames, none on the onset at 602.
+    const beats = Array.from({ length: 200 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 43);
+
+    expect(result.runs[0].startFrame + (result.runs[0].leadInFrames as number)).equal(602);
+    expect(result.runs[0].leadInFrames).equal(43);
+  });
+
+  it('ends the lead-in on a nearby grid beat when a tempo grid is given', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 602, totalFrames);
+    const beats = Array.from({ length: 200 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+    // Grid beats every 10 frames on multiples of 10: 600 is 2 frames (0.2 beat) from the onset.
+    const grid = { periodFrames: 10, phaseFrame: 0 };
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, grid, 43);
+
+    expect(result.runs[0].startFrame).equal(557);
+    expect(result.runs[0].leadInFrames).equal(43);
+  });
+
+  it('keeps the full lead-in rather than ending it past the drop or on a weak rise', async function() {
+    const fps = 10;
+    const totalFrames = 600;
+    // Big drop at 140, then a small bump at 300; a 200-frame lead-in doesn't fit before 140.
+    const spectrums = Array.from({ length: totalFrames }, (_, i) =>
+      i < 140 ? [0.1] : i < 300 ? [1] : [1.1],
+    );
+    const beats = Array.from({ length: 120 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 200);
+
+    expect(result.runs[0].startFrame).equal(-60);
+    expect(result.runs[0].leadInFrames).equal(200);
+  });
+
+  it('puts the lead-in before the energy window when no drop is found', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    const highlightFrames = Math.ceil(HIGHLIGHT_DURATION_SEC * fps);
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 600, totalFrames);
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, [], 1, null, 900);
+
+    // The window starts at 600; the rest of the lead-in is silence before the track start.
+    expect(result.runs[0].startFrame).equal(-300);
+    expect(result.runs[0].leadInFrames).equal(900);
+    expect(result.runs[0].highlightFrames).equal(900 + highlightFrames);
+    expect(result.runs[0].audioSegment.delaySeconds).equal(30);
+  });
+
   it('snaps highlight start and end onto detected beats', async function() {
     const fps = 30;
     const totalFrames = 1000;
@@ -259,3 +369,29 @@ describe('snapSegmentToBeats', function() {
     expect(snapSegmentToBeats(seg, 1000, [10, 900], 20)).deep.equal(seg);
   });
 });
+
+describe('snapDropToBeat', function() {
+  it('moves the drop and its start together onto a beat within a fifth of a beat', function() {
+    const drop = { startFrame: 60, dropFrame: 103 };
+    snapDropToBeat(drop, [80, 100, 120], 20);
+    expect(drop).deep.equal({ startFrame: 57, dropFrame: 100 });
+  });
+
+  it('keeps the drop when the nearest beat is too far', function() {
+    const drop = { startFrame: 60, dropFrame: 105 };
+    snapDropToBeat(drop, [80, 100, 120], 20);
+    expect(drop).deep.equal({ startFrame: 60, dropFrame: 105 });
+  });
+
+  it('keeps the drop without a tempo period', function() {
+    const drop = { startFrame: 60, dropFrame: 101 };
+    snapDropToBeat(drop, [80, 100, 120]);
+    expect(drop).deep.equal({ startFrame: 60, dropFrame: 101 });
+  });
+
+  it('does not push a lead-in that fit in the track back past its start', function() {
+    const drop = { startFrame: 1, dropFrame: 43 };
+    snapDropToBeat(drop, [40], 20);
+    expect(drop).deep.equal({ startFrame: 1, dropFrame: 43 });
+  });
+});

package/src/tests/video.test.ts

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 import { Writable, Readable, Pipe } from 'stream';
 import { EventEmitter } from 'events';
-import { spawnFfmpegVideoWriter, waitDrain, readVideoFrame, waitForProcessExit, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, getCutFrameIndices, selectAutoEditCutFrames, snapBeatsToTempoGrid, projectBeatsOntoTempoGrid, spawnConcatVideoFrameReader } from '../video';
+import { spawnFfmpegVideoWriter, waitDrain, readVideoFrame, waitForProcessExit, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, computeHookFrameCount, getCutFrameIndices, selectAutoEditCutFrames, snapBeatsToTempoGrid, projectBeatsOntoTempoGrid, spawnConcatVideoFrameReader } from '../video';
 import { createSandbox, SinonStub } from 'sinon';
 import child_process, { ChildProcessWithoutNullStreams } from 'child_process';
 
@@ -70,6 +70,80 @@ describe('video', function () {
     expect(spawnArgs[tIdx + 1]).to.equal('15');
   });
 
+  it('spawnFfmpegVideoWriter mixes overlay audio while the track fades in', function () {
+    const childProcessReadableStream = new Readable();
+    childProcessReadableStream._read = () => { };
+    const childProcessWritableStream = new Writable();
+    (<Pipe>childProcessWritableStream.pipe) = () => childProcessWritableStream;
+
+    childProcessStream.stdin = childProcessWritableStream;
+    childProcessStream.stderr = childProcessReadableStream;
+
+    let spawnArgs: string[] = [];
+    const spawnStub = child_process.spawn as SinonStub;
+    spawnStub.callsFake((_cmd: string, args: string[]) => {
+      spawnArgs = args;
+      return childProcessStream as ChildProcessWithoutNullStreams;
+    });
+
+    spawnFfmpegVideoWriter({
+      audioFilename: 'audio.mp3',
+      videoFileName: 'out.mp4',
+      fps: 25,
+      overlayAudio: { filename: 'hook.mp4', durationSeconds: 3.2 },
+    });
+
+    spawnStub.resetBehavior();
+    spawnStub.returns(childProcessStream as ChildProcessWithoutNullStreams);
+
+    const inputs = spawnArgs.filter((_, i) => spawnArgs[i - 1] === '-i');
+    expect(inputs).deep.equal(['audio.mp3', '-', 'hook.mp4']);
+    const hookIdx = spawnArgs.indexOf('hook.mp4');
+    expect(spawnArgs.slice(hookIdx - 3, hookIdx - 1)).deep.equal(['-t', '3.2']);
+    const filter = spawnArgs[spawnArgs.indexOf('-filter_complex') + 1];
+    expect(filter).to.include("[0:a]aformat=channel_layouts=stereo,volume='if(gte(t,3.2),1,pow(t/3.2,5))':eval=frame[track]");
+    expect(filter).to.include('[2:a]aformat=channel_layouts=stereo[overlay]');
+    expect(filter).to.include('[track][overlay]amix=inputs=2:duration=first');
+    expect(spawnArgs.join(' ')).to.include('-map 1:v -map [aout]');
+  });
+
+  it('spawnFfmpegVideoWriter delays the track by the audio segment delay', function () {
+    const childProcessReadableStream = new Readable();
+    childProcessReadableStream._read = () => { };
+    const childProcessWritableStream = new Writable();
+    (<Pipe>childProcessWritableStream.pipe) = () => childProcessWritableStream;
+
+    childProcessStream.stdin = childProcessWritableStream;
+    childProcessStream.stderr = childProcessReadableStream;
+
+    const spawnStub = child_process.spawn as SinonStub;
+    const run = (overlay: boolean) => {
+      let spawnArgs: string[] = [];
+      spawnStub.callsFake((_cmd: string, args: string[]) => {
+        spawnArgs = args;
+        return childProcessStream as ChildProcessWithoutNullStreams;
+      });
+      spawnFfmpegVideoWriter({
+        audioFilename: 'audio.mp3',
+        videoFileName: 'out.mp4',
+        fps: 25,
+        audioSegment: { seekSeconds: 0, durationSeconds: 12, delaySeconds: 1.5 },
+        ...(overlay && { overlayAudio: { filename: 'hook.mp4', durationSeconds: 3.2 } }),
+      });
+      return spawnArgs;
+    };
+
+    const plainArgs = run(false);
+    expect(plainArgs[plainArgs.indexOf('-af') + 1]).equal('adelay=1500:all=1');
+    const overlayArgs = run(true);
+    spawnStub.resetBehavior();
+    spawnStub.returns(childProcessStream as ChildProcessWithoutNullStreams);
+
+    expect(overlayArgs).to.not.include('-af');
+    const filter = overlayArgs[overlayArgs.indexOf('-filter_complex') + 1];
+    expect(filter).to.include("[0:a]aformat=channel_layouts=stereo,adelay=1500:all=1,volume='if(gte(t,3.2),1,pow(t/3.2,5))':eval=frame[track]");
+  });
+
   it('spawnFfmpegVideoWriter adds subtitles filter when subtitle file is provided', function () {
     const childProcessReadableStream = new Readable();
     childProcessReadableStream._read = () => { };
@@ -283,6 +357,11 @@ describe('video', function () {
     expect(reason).equal('waitForProcessExit timeout (5ms)');
   });
 
+  it('computeHookFrameCount plays the full hook', function () {
+    expect(computeHookFrameCount(2, 30, 300)).equal(60);
+    expect(computeHookFrameCount(20, 30, 300)).equal(300);
+  });
+
   it('getCutFrameIndices skips the opening segment', function () {
     const segments = buildBeatSyncedSegments(
       [12, 24, 36],

package/src/video.ts

@@ -10,6 +10,8 @@ import { TempoEstimate, beatGridFrameIndices } from './beats';
 export interface AudioMuxSegment {
   seekSeconds: number;
   durationSeconds: number;
+  /** Silence played before the audio. */
+  delaySeconds?: number;
 }
 
 interface FfmpegVideoWriterConfig {
@@ -27,6 +29,14 @@ interface FfmpegVideoWriterConfig {
   audioDurationSeconds?: number;
   /** One contiguous trim of the same audio file (muxed highlight). */
   audioSegment?: AudioMuxSegment;
+  /**
+   * Audio mixed over the start of the track (e.g. the hook video's sound), from its beginning.
+   * The track fades in from silence to full volume while it plays, along an easeInQuint curve.
+   */
+  overlayAudio?: {
+    filename: string;
+    durationSeconds: number;
+  };
 }
 
 const escapeSubtitleFilterPath = (subtitlePath: string) =>
@@ -38,6 +48,13 @@ const escapeSubtitleFilterPath = (subtitlePath: string) =>
     .replace(/\[/g, '\\[')
     .replace(/\]/g, '\\]');
 
+/**
+ * Fade-in from silence to full volume over `durationSeconds`, following easeInQuint:
+ * gain(x) = x^5, where x is the fraction of the fade elapsed.
+ */
+const easeInQuintFadeFilter = (durationSeconds: number) =>
+  `volume='if(gte(t,${durationSeconds}),1,pow(t/${durationSeconds},5))':eval=frame`;
+
 export const spawnFfmpegVideoWriter = (config: FfmpegVideoWriterConfig) => {
   if (!ffmpegPath) {
     throw new Error('ffmpeg path not found');
@@ -58,6 +75,30 @@ export const spawnFfmpegVideoWriter = (config: FfmpegVideoWriterConfig) => {
     '-vcodec', 'bmp',
     '-framerate', `${config.fps}`,
     '-i', '-',
+  );
+  const delaySeconds = config.audioSegment?.delaySeconds ?? 0;
+  const trackDelayFilter = delaySeconds > 0
+    ? `adelay=${Math.round(delaySeconds * 1000)}:all=1`
+    : '';
+  if (config.overlayAudio) {
+    const overlaySeconds = config.overlayAudio.durationSeconds;
+    args.push(
+      '-t', String(overlaySeconds),
+      '-i', config.overlayAudio.filename,
+      // Both inputs go to stereo first: mixing in e.g. a 5.1 hook otherwise yields a channel
+      // layout the AAC encoder rejects. `duration=first` keeps the output as long as the track.
+      '-filter_complex',
+      `[0:a]aformat=channel_layouts=stereo,${trackDelayFilter ? `${trackDelayFilter},` : ''}`
+        + `${easeInQuintFadeFilter(overlaySeconds)}[track];`
+        + '[2:a]aformat=channel_layouts=stereo[overlay];'
+        + '[track][overlay]amix=inputs=2:duration=first:dropout_transition=0:normalize=0[aout]',
+      '-map', '1:v',
+      '-map', '[aout]',
+    );
+  } else if (trackDelayFilter) {
+    args.push('-af', trackDelayFilter);
+  }
+  args.push(
     '-crf', crf,
     '-c:a', 'aac', '-b:a', '384k', '-profile:a', 'aac_low',
     '-c:v', 'libx264',
@@ -86,6 +127,7 @@ export interface VideoInfo {
   width: number;
   height: number;
   duration: number;
+  hasAudio: boolean;
 }
 
 export interface SceneChange {
@@ -128,6 +170,37 @@ export const detectSceneChanges = (videoPath: string, threshold = 0.4): Promise<
     ffmpeg.on('error', reject);
   });
 
+/**
+ * Number of whole frames the video stream yields at `fps` (partial last frames dropped).
+ * Unlike the container duration, this never runs past the last video frame (e.g. when the
+ * audio stream is longer), so a clip read for this many frames never wraps back to its start.
+ */
+export const getVideoFrameCount = (videoPath: string, fps: number): Promise<number> =>
+  new Promise((resolvePromise, reject) => {
+    if (!ffmpegPath) {
+      reject(new Error('ffmpeg path not found'));
+      return;
+    }
+    const ffmpeg = spawn(ffmpegPath, [
+      '-i', videoPath,
+      '-map', '0:v:0',
+      '-vf', `fps=${fps}:round=down`,
+      '-f', 'null',
+      '-',
+    ]);
+    let stderr = '';
+    ffmpeg.stderr.on('data', (data: Buffer) => { stderr += data.toString(); });
+    ffmpeg.on('close', () => {
+      const matches = stderr.match(/frame=\s*\d+/g);
+      if (!matches) {
+        reject(new Error(`Could not count video frames for: ${videoPath}`));
+        return;
+      }
+      resolvePromise(parseInt(matches[matches.length - 1].replace(/\D/g, '')));
+    });
+    ffmpeg.on('error', reject);
+  });
+
 export const getVideoInfo = (videoPath: string): Promise<VideoInfo> =>
   new Promise((resolvePromise, reject) => {
     if (!ffmpegPath) {
@@ -151,6 +224,7 @@ export const getVideoInfo = (videoPath: string): Promise<VideoInfo> =>
         width: parseInt(dimMatch[1]),
         height: parseInt(dimMatch[2]),
         duration,
+        hasAudio: /Stream.*Audio:/.test(stderr),
       });
     });
   });
@@ -762,6 +836,13 @@ export const buildBeatSyncedSegments = (
   return segments;
 };
 
+/** Output frames given to the hook video: it plays in full, but never exceeds `totalFrames`. */
+export const computeHookFrameCount = (
+  hookDurationSeconds: number,
+  fps: number,
+  totalFrames: number,
+): number => Math.max(0, Math.min(totalFrames, Math.round(hookDurationSeconds * fps)));
+
 export const getCutFrameIndices = (segments: VideoSegment[]): number[] =>
   segments
     .slice(1)