nodejs-audio-visualizer 5.2.0 → 5.4.0

package/src/tests/config.test.ts

@@ -8,6 +8,7 @@ import {
   getAudioAutoHighlightCount,
   getBackgroundImagePath,
   getBackgroundVideoPaths,
+  getHookVideoPath,
   getCameraShakeEnabled,
   getOutVideoPath,
   getSubtitleRenderSpec,
@@ -93,6 +94,24 @@ describe('config', function() {
     ).to.throw('Invalid video.path');
   });
 
+  it('getHookVideoPath', function() {
+    expect(
+      getHookVideoPath({ audio: { path: 'a.wav' } } as Config)
+    ).equal(undefined);
+
+    expect(
+      getHookVideoPath({ video: { path: 'a.mp4' } } as Config)
+    ).equal(undefined);
+
+    expect(
+      getHookVideoPath({ video: { path: 'a.mp4', hookPath: 'hook.mp4' } } as Config)
+    ).equal(path.resolve('hook.mp4'));
+
+    expect(
+      getHookVideoPath.bind(undefined, { video: { path: 'a.mp4', hookPath: '' } } as Config)
+    ).to.throw('Invalid video.hookPath');
+  });
+
   it('getCameraShakeEnabled', function() {
     expect(
       getCameraShakeEnabled({ audio: { path: 'a.wav' } } as Config)

package/src/tests/highlight.test.ts

@@ -1,5 +1,12 @@
 import { expect } from 'chai';
-import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from '../highlight';
+import {
+  computeHighlightSlice,
+  HIGHLIGHT_DURATION_SEC,
+  MAX_BEAT_SNAP_SEC,
+  buildBeatCandidates,
+  snapSegmentToBeats,
+  snapDropToBeat,
+} from '../highlight';
 
 const dummySpectrums = (n: number) => Array.from({ length: n }, () => [0]);
 
@@ -113,9 +120,10 @@ describe('computeHighlightSlice', function() {
     ]);
 
     expect(result.startFrame).equal(10);
-    expect(result.highlightFrames).equal(highlightFrames);
+    // The end moves from frame 460 onto the nearby beat at 500.
+    expect(result.highlightFrames).equal(490);
     expect(result.audioSegments).deep.equal([
-      { seekSeconds: 10 / fps, durationSeconds: highlightFrames / fps },
+      { seekSeconds: 10 / fps, durationSeconds: 490 / fps },
     ]);
     expect(result.runs).have.length(1);
   });
@@ -175,4 +183,215 @@ describe('computeHighlightSlice', function() {
     expect(result.audioSegments).have.length(1);
     expect(result.runs).have.length(1);
   });
+
+  it('starts the whole lead-in before the drop when leadInFrames is given', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    // Quiet until frame 600, loud after: the drop is at 600.
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 600, totalFrames);
+    const beats = Array.from({ length: 200 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 43);
+
+    expect(result.runs).have.length(1);
+    expect(result.runs[0].startFrame).equal(557);
+    expect(result.runs[0].leadInFrames).equal(43);
+    // The lead-in comes on top of the ~15s that follow the drop.
+    expect(result.runs[0].highlightFrames).equal(43 + Math.ceil(HIGHLIGHT_DURATION_SEC * fps));
+  });
+
+  it('pads the lead-in with silence before the track start when only an early drop fits', async function() {
+    const fps = 10;
+    // A full 130-frame lead-in needs a drop at 130+, but ~15s must follow it: only 100 fits.
+    const totalFrames = 250;
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 100, totalFrames);
+    const beats = Array.from({ length: 50 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 130);
+    const run = result.runs[0];
+
+    expect(run.startFrame).equal(-30);
+    expect(run.leadInFrames).equal(130);
+    expect(run.spectrums).have.length(run.highlightFrames);
+    expect(run.spectrums.slice(0, 30).every(s => s.every(v => v === 0))).equal(true);
+    expect(run.beatFrameIndices[0]).equal(30);
+    expect(run.audioSegment).deep.equal({
+      seekSeconds: 0,
+      durationSeconds: (run.highlightFrames - 30) / fps,
+      delaySeconds: 3,
+    });
+  });
+
+  it('picks a separate drop for each highlight', async function() {
+    const fps = 10;
+    const totalFrames = 2000;
+    const spectrums = Array.from({ length: totalFrames }, (_, i) =>
+      (i >= 500 && i < 800) || (i >= 1400 && i < 1700) ? [1] : [0],
+    );
+    const beats = Array.from({ length: 400 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 2, null, 30);
+
+    expect(result.runs.map(run => run.startFrame + (run.leadInFrames as number))).deep.equal([500, 1400]);
+    expect(result.runs.map(run => run.leadInFrames)).deep.equal([30, 30]);
+  });
+
+  it('ends the lead-in on the energy onset even when beats are off it', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 602, totalFrames);
+    // Beats every 5 frames, none on the onset at 602.
+    const beats = Array.from({ length: 200 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 43);
+
+    expect(result.runs[0].startFrame + (result.runs[0].leadInFrames as number)).equal(602);
+    expect(result.runs[0].leadInFrames).equal(43);
+  });
+
+  it('ends the lead-in on a nearby grid beat when a tempo grid is given', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 602, totalFrames);
+    const beats = Array.from({ length: 200 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+    // Grid beats every 10 frames on multiples of 10: 600 is 2 frames (0.2 beat) from the onset.
+    const grid = { periodFrames: 10, phaseFrame: 0 };
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, grid, 43);
+
+    expect(result.runs[0].startFrame).equal(557);
+    expect(result.runs[0].leadInFrames).equal(43);
+  });
+
+  it('keeps the full lead-in rather than ending it past the drop or on a weak rise', async function() {
+    const fps = 10;
+    const totalFrames = 600;
+    // Big drop at 140, then a small bump at 300; a 200-frame lead-in doesn't fit before 140.
+    const spectrums = Array.from({ length: totalFrames }, (_, i) =>
+      i < 140 ? [0.1] : i < 300 ? [1] : [1.1],
+    );
+    const beats = Array.from({ length: 120 }, (_, i) => ({ frameIndex: i * 5, intensity: 1 }));
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, beats, 1, null, 200);
+
+    expect(result.runs[0].startFrame).equal(-60);
+    expect(result.runs[0].leadInFrames).equal(200);
+  });
+
+  it('puts the lead-in before the energy window when no drop is found', async function() {
+    const fps = 10;
+    const totalFrames = 1000;
+    const highlightFrames = Math.ceil(HIGHLIGHT_DURATION_SEC * fps);
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 600, totalFrames);
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, [], 1, null, 900);
+
+    // The window starts at 600; the rest of the lead-in is silence before the track start.
+    expect(result.runs[0].startFrame).equal(-300);
+    expect(result.runs[0].leadInFrames).equal(900);
+    expect(result.runs[0].highlightFrames).equal(900 + highlightFrames);
+    expect(result.runs[0].audioSegment.delaySeconds).equal(30);
+  });
+
+  it('snaps highlight start and end onto detected beats', async function() {
+    const fps = 30;
+    const totalFrames = 1000;
+    const highlightFrames = Math.ceil(HIGHLIGHT_DURATION_SEC * fps);
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 300, 300 + highlightFrames);
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, [
+      { frameIndex: 290, intensity: 1 },
+      { frameIndex: 600, intensity: 1 },
+      { frameIndex: 760, intensity: 1 },
+    ]);
+
+    expect(result.startFrame).equal(290);
+    expect(result.highlightFrames).equal(760 - 290);
+    expect(result.audioSegments).deep.equal([
+      { seekSeconds: 290 / fps, durationSeconds: (760 - 290) / fps },
+    ]);
+    expect(result.beatFrameIndices).deep.equal([0, 310]);
+  });
+
+  it('snaps to a tempo grid and prefers a whole number of bars', async function() {
+    const fps = 30;
+    const totalFrames = 2000;
+    const highlightFrames = Math.ceil(HIGHLIGHT_DURATION_SEC * fps);
+    const spectrums = spectrumsWithEnergyInRange(totalFrames, 505, 505 + highlightFrames);
+    // 120 BPM at 30 fps: a beat every 15 frames, a bar every 60.
+    const beatGrid = { periodFrames: 15, phaseFrame: 5 };
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, [], 1, beatGrid);
+
+    expect((result.startFrame - 5) % 15).equal(0);
+    expect(result.highlightFrames % 60).equal(0);
+    expect(Math.abs(result.startFrame - 505)).lte(MAX_BEAT_SNAP_SEC * fps);
+    expect(Math.abs(result.highlightFrames - highlightFrames)).lte(2 * MAX_BEAT_SNAP_SEC * fps);
+  });
+
+  it('keeps several snapped highlights from overlapping', async function() {
+    const fps = 30;
+    const totalFrames = 3000;
+    const spectrums = Array.from({ length: totalFrames }, (_, i) => [1 + Math.sin(i / 50)]);
+    const beatGrid = { periodFrames: 14, phaseFrame: 3 };
+
+    const result = await computeHighlightSlice(fps, totalFrames, spectrums, [], 4, beatGrid);
+
+    for (let i = 1; i < result.runs.length; i++) {
+      const prev = result.runs[i - 1];
+      expect(result.runs[i].startFrame).gte(prev.startFrame + prev.highlightFrames);
+    }
+  });
+});
+
+describe('buildBeatCandidates', function() {
+  it('uses onsets when no tempo grid is given', function() {
+    expect(buildBeatCandidates(30, 100, [
+      { frameIndex: 40, intensity: 1 },
+      { frameIndex: 10, intensity: 1 },
+      { frameIndex: 10, intensity: 1 },
+    ])).deep.equal([10, 40]);
+  });
+
+  it('uses tracked beats, then the grid nudged onto nearby onsets', function() {
+    const frames = buildBeatCandidates(10, 60, [{ frameIndex: 41, intensity: 1 }], {
+      periodFrames: 10,
+      phaseFrame: 0,
+      beatsSec: [0.1, 1.1, 2.1],
+    });
+    expect(frames).deep.equal([1, 11, 21, 30, 41, 50, 60]);
+  });
+});
+
+describe('snapSegmentToBeats', function() {
+  it('leaves a segment unchanged when no beat is close enough', function() {
+    const seg = { startFrame: 100, highlightFrames: 200 };
+    expect(snapSegmentToBeats(seg, 1000, [10, 900], 20)).deep.equal(seg);
+  });
+});
+
+describe('snapDropToBeat', function() {
+  it('moves the drop and its start together onto a beat within a fifth of a beat', function() {
+    const drop = { startFrame: 60, dropFrame: 103 };
+    snapDropToBeat(drop, [80, 100, 120], 20);
+    expect(drop).deep.equal({ startFrame: 57, dropFrame: 100 });
+  });
+
+  it('keeps the drop when the nearest beat is too far', function() {
+    const drop = { startFrame: 60, dropFrame: 105 };
+    snapDropToBeat(drop, [80, 100, 120], 20);
+    expect(drop).deep.equal({ startFrame: 60, dropFrame: 105 });
+  });
+
+  it('keeps the drop without a tempo period', function() {
+    const drop = { startFrame: 60, dropFrame: 101 };
+    snapDropToBeat(drop, [80, 100, 120]);
+    expect(drop).deep.equal({ startFrame: 60, dropFrame: 101 });
+  });
+
+  it('does not push a lead-in that fit in the track back past its start', function() {
+    const drop = { startFrame: 1, dropFrame: 43 };
+    snapDropToBeat(drop, [40], 20);
+    expect(drop).deep.equal({ startFrame: 1, dropFrame: 43 });
+  });
 });

package/src/tests/video.test.ts

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 import { Writable, Readable, Pipe } from 'stream';
 import { EventEmitter } from 'events';
-import { spawnFfmpegVideoWriter, waitDrain, readVideoFrame, waitForProcessExit, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, getCutFrameIndices, selectAutoEditCutFrames, snapBeatsToTempoGrid, projectBeatsOntoTempoGrid, spawnConcatVideoFrameReader } from '../video';
+import { spawnFfmpegVideoWriter, waitDrain, readVideoFrame, waitForProcessExit, buildBeatSyncedSegments, buildSequentialVideoSegments, buildShuffledVideoSegments, computeHookFrameCount, getCutFrameIndices, selectAutoEditCutFrames, snapBeatsToTempoGrid, projectBeatsOntoTempoGrid, spawnConcatVideoFrameReader } from '../video';
 import { createSandbox, SinonStub } from 'sinon';
 import child_process, { ChildProcessWithoutNullStreams } from 'child_process';
 
@@ -70,6 +70,80 @@ describe('video', function () {
     expect(spawnArgs[tIdx + 1]).to.equal('15');
   });
 
+  it('spawnFfmpegVideoWriter mixes overlay audio while the track fades in', function () {
+    const childProcessReadableStream = new Readable();
+    childProcessReadableStream._read = () => { };
+    const childProcessWritableStream = new Writable();
+    (<Pipe>childProcessWritableStream.pipe) = () => childProcessWritableStream;
+
+    childProcessStream.stdin = childProcessWritableStream;
+    childProcessStream.stderr = childProcessReadableStream;
+
+    let spawnArgs: string[] = [];
+    const spawnStub = child_process.spawn as SinonStub;
+    spawnStub.callsFake((_cmd: string, args: string[]) => {
+      spawnArgs = args;
+      return childProcessStream as ChildProcessWithoutNullStreams;
+    });
+
+    spawnFfmpegVideoWriter({
+      audioFilename: 'audio.mp3',
+      videoFileName: 'out.mp4',
+      fps: 25,
+      overlayAudio: { filename: 'hook.mp4', durationSeconds: 3.2 },
+    });
+
+    spawnStub.resetBehavior();
+    spawnStub.returns(childProcessStream as ChildProcessWithoutNullStreams);
+
+    const inputs = spawnArgs.filter((_, i) => spawnArgs[i - 1] === '-i');
+    expect(inputs).deep.equal(['audio.mp3', '-', 'hook.mp4']);
+    const hookIdx = spawnArgs.indexOf('hook.mp4');
+    expect(spawnArgs.slice(hookIdx - 3, hookIdx - 1)).deep.equal(['-t', '3.2']);
+    const filter = spawnArgs[spawnArgs.indexOf('-filter_complex') + 1];
+    expect(filter).to.include("[0:a]aformat=channel_layouts=stereo,volume='if(gte(t,3.2),1,pow(t/3.2,5))':eval=frame[track]");
+    expect(filter).to.include('[2:a]aformat=channel_layouts=stereo[overlay]');
+    expect(filter).to.include('[track][overlay]amix=inputs=2:duration=first');
+    expect(spawnArgs.join(' ')).to.include('-map 1:v -map [aout]');
+  });
+
+  it('spawnFfmpegVideoWriter delays the track by the audio segment delay', function () {
+    const childProcessReadableStream = new Readable();
+    childProcessReadableStream._read = () => { };
+    const childProcessWritableStream = new Writable();
+    (<Pipe>childProcessWritableStream.pipe) = () => childProcessWritableStream;
+
+    childProcessStream.stdin = childProcessWritableStream;
+    childProcessStream.stderr = childProcessReadableStream;
+
+    const spawnStub = child_process.spawn as SinonStub;
+    const run = (overlay: boolean) => {
+      let spawnArgs: string[] = [];
+      spawnStub.callsFake((_cmd: string, args: string[]) => {
+        spawnArgs = args;
+        return childProcessStream as ChildProcessWithoutNullStreams;
+      });
+      spawnFfmpegVideoWriter({
+        audioFilename: 'audio.mp3',
+        videoFileName: 'out.mp4',
+        fps: 25,
+        audioSegment: { seekSeconds: 0, durationSeconds: 12, delaySeconds: 1.5 },
+        ...(overlay && { overlayAudio: { filename: 'hook.mp4', durationSeconds: 3.2 } }),
+      });
+      return spawnArgs;
+    };
+
+    const plainArgs = run(false);
+    expect(plainArgs[plainArgs.indexOf('-af') + 1]).equal('adelay=1500:all=1');
+    const overlayArgs = run(true);
+    spawnStub.resetBehavior();
+    spawnStub.returns(childProcessStream as ChildProcessWithoutNullStreams);
+
+    expect(overlayArgs).to.not.include('-af');
+    const filter = overlayArgs[overlayArgs.indexOf('-filter_complex') + 1];
+    expect(filter).to.include("[0:a]aformat=channel_layouts=stereo,adelay=1500:all=1,volume='if(gte(t,3.2),1,pow(t/3.2,5))':eval=frame[track]");
+  });
+
   it('spawnFfmpegVideoWriter adds subtitles filter when subtitle file is provided', function () {
     const childProcessReadableStream = new Readable();
     childProcessReadableStream._read = () => { };
@@ -283,6 +357,11 @@ describe('video', function () {
     expect(reason).equal('waitForProcessExit timeout (5ms)');
   });
 
+  it('computeHookFrameCount plays the full hook', function () {
+    expect(computeHookFrameCount(2, 30, 300)).equal(60);
+    expect(computeHookFrameCount(20, 30, 300)).equal(300);
+  });
+
   it('getCutFrameIndices skips the opening segment', function () {
     const segments = buildBeatSyncedSegments(
       [12, 24, 36],

package/src/video.ts

@@ -10,6 +10,8 @@ import { TempoEstimate, beatGridFrameIndices } from './beats';
 export interface AudioMuxSegment {
   seekSeconds: number;
   durationSeconds: number;
+  /** Silence played before the audio. */
+  delaySeconds?: number;
 }
 
 interface FfmpegVideoWriterConfig {
@@ -27,6 +29,14 @@ interface FfmpegVideoWriterConfig {
   audioDurationSeconds?: number;
   /** One contiguous trim of the same audio file (muxed highlight). */
   audioSegment?: AudioMuxSegment;
+  /**
+   * Audio mixed over the start of the track (e.g. the hook video's sound), from its beginning.
+   * The track fades in from silence to full volume while it plays, along an easeInQuint curve.
+   */
+  overlayAudio?: {
+    filename: string;
+    durationSeconds: number;
+  };
 }
 
 const escapeSubtitleFilterPath = (subtitlePath: string) =>
@@ -38,6 +48,13 @@ const escapeSubtitleFilterPath = (subtitlePath: string) =>
     .replace(/\[/g, '\\[')
     .replace(/\]/g, '\\]');
 
+/**
+ * Fade-in from silence to full volume over `durationSeconds`, following easeInQuint:
+ * gain(x) = x^5, where x is the fraction of the fade elapsed.
+ */
+const easeInQuintFadeFilter = (durationSeconds: number) =>
+  `volume='if(gte(t,${durationSeconds}),1,pow(t/${durationSeconds},5))':eval=frame`;
+
 export const spawnFfmpegVideoWriter = (config: FfmpegVideoWriterConfig) => {
   if (!ffmpegPath) {
     throw new Error('ffmpeg path not found');
@@ -58,6 +75,30 @@ export const spawnFfmpegVideoWriter = (config: FfmpegVideoWriterConfig) => {
     '-vcodec', 'bmp',
     '-framerate', `${config.fps}`,
     '-i', '-',
+  );
+  const delaySeconds = config.audioSegment?.delaySeconds ?? 0;
+  const trackDelayFilter = delaySeconds > 0
+    ? `adelay=${Math.round(delaySeconds * 1000)}:all=1`
+    : '';
+  if (config.overlayAudio) {
+    const overlaySeconds = config.overlayAudio.durationSeconds;
+    args.push(
+      '-t', String(overlaySeconds),
+      '-i', config.overlayAudio.filename,
+      // Both inputs go to stereo first: mixing in e.g. a 5.1 hook otherwise yields a channel
+      // layout the AAC encoder rejects. `duration=first` keeps the output as long as the track.
+      '-filter_complex',
+      `[0:a]aformat=channel_layouts=stereo,${trackDelayFilter ? `${trackDelayFilter},` : ''}`
+        + `${easeInQuintFadeFilter(overlaySeconds)}[track];`
+        + '[2:a]aformat=channel_layouts=stereo[overlay];'
+        + '[track][overlay]amix=inputs=2:duration=first:dropout_transition=0:normalize=0[aout]',
+      '-map', '1:v',
+      '-map', '[aout]',
+    );
+  } else if (trackDelayFilter) {
+    args.push('-af', trackDelayFilter);
+  }
+  args.push(
     '-crf', crf,
     '-c:a', 'aac', '-b:a', '384k', '-profile:a', 'aac_low',
     '-c:v', 'libx264',
@@ -86,6 +127,7 @@ export interface VideoInfo {
   width: number;
   height: number;
   duration: number;
+  hasAudio: boolean;
 }
 
 export interface SceneChange {
@@ -128,6 +170,37 @@ export const detectSceneChanges = (videoPath: string, threshold = 0.4): Promise<
     ffmpeg.on('error', reject);
   });
 
+/**
+ * Number of whole frames the video stream yields at `fps` (partial last frames dropped).
+ * Unlike the container duration, this never runs past the last video frame (e.g. when the
+ * audio stream is longer), so a clip read for this many frames never wraps back to its start.
+ */
+export const getVideoFrameCount = (videoPath: string, fps: number): Promise<number> =>
+  new Promise((resolvePromise, reject) => {
+    if (!ffmpegPath) {
+      reject(new Error('ffmpeg path not found'));
+      return;
+    }
+    const ffmpeg = spawn(ffmpegPath, [
+      '-i', videoPath,
+      '-map', '0:v:0',
+      '-vf', `fps=${fps}:round=down`,
+      '-f', 'null',
+      '-',
+    ]);
+    let stderr = '';
+    ffmpeg.stderr.on('data', (data: Buffer) => { stderr += data.toString(); });
+    ffmpeg.on('close', () => {
+      const matches = stderr.match(/frame=\s*\d+/g);
+      if (!matches) {
+        reject(new Error(`Could not count video frames for: ${videoPath}`));
+        return;
+      }
+      resolvePromise(parseInt(matches[matches.length - 1].replace(/\D/g, '')));
+    });
+    ffmpeg.on('error', reject);
+  });
+
 export const getVideoInfo = (videoPath: string): Promise<VideoInfo> =>
   new Promise((resolvePromise, reject) => {
     if (!ffmpegPath) {
@@ -151,6 +224,7 @@ export const getVideoInfo = (videoPath: string): Promise<VideoInfo> =>
         width: parseInt(dimMatch[1]),
         height: parseInt(dimMatch[2]),
         duration,
+        hasAudio: /Stream.*Audio:/.test(stderr),
       });
     });
   });
@@ -762,6 +836,13 @@ export const buildBeatSyncedSegments = (
   return segments;
 };
 
+/** Output frames given to the hook video: it plays in full, but never exceeds `totalFrames`. */
+export const computeHookFrameCount = (
+  hookDurationSeconds: number,
+  fps: number,
+  totalFrames: number,
+): number => Math.max(0, Math.min(totalFrames, Math.round(hookDurationSeconds * fps)));
+
 export const getCutFrameIndices = (segments: VideoSegment[]): number[] =>
   segments
     .slice(1)