nodejs-audio-visualizer 5.0.0 → 5.1.0

package/src/tests/video.test.ts

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 import { Writable, Readable, Pipe } from 'stream';
 import { EventEmitter } from 'events';
-import { spawnFfmpegVideoWriter, waitDrain, readVideoFrame, waitForProcessExit, spawnConcatVideoFrameReader } from '../video';
+import { spawnFfmpegVideoWriter, waitDrain, readVideoFrame, waitForProcessExit, buildBeatSyncedSegments, getCutFrameIndices, selectAutoEditCutFrames, snapBeatsToTempoGrid, projectBeatsOntoTempoGrid, spawnConcatVideoFrameReader } from '../video';
 import { createSandbox, SinonStub } from 'sinon';
 import child_process, { ChildProcessWithoutNullStreams } from 'child_process';
 
@@ -282,4 +282,166 @@ describe('video', function () {
     expect(exitCode).equal(1);
     expect(reason).equal('waitForProcessExit timeout (5ms)');
   });
+
+  it('getCutFrameIndices skips the opening segment', function () {
+    const segments = buildBeatSyncedSegments(
+      [12, 24, 36],
+      48,
+      [
+        { frameNumber: 0, pts: 0, ptsTime: 1 },
+        { frameNumber: 1, pts: 0, ptsTime: 4 },
+        { frameNumber: 2, pts: 0, ptsTime: 8 },
+      ],
+      12,
+      30,
+      { minCutIntervalSeconds: 0 },
+    );
+    expect(getCutFrameIndices(segments)).deep.equal([12, 24, 36]);
+  });
+
+  it('getCutFrameIndices is empty when auto-edit has no scene changes', function () {
+    const segments = buildBeatSyncedSegments([10, 20], 30, [], 8, 30);
+    expect(segments).deep.equal([
+      { outputStartFrame: 0, videoSeekSeconds: 0, frameCount: 30 },
+    ]);
+    expect(getCutFrameIndices(segments)).deep.equal([]);
+  });
+
+  it('selectAutoEditCutFrames keeps at least 2 seconds between cuts', function () {
+    const fps = 30;
+    const beats = [15, 30, 45, 60, 75, 90, 105, 120].map(frameIndex => ({ frameIndex }));
+    expect(selectAutoEditCutFrames(beats, fps, 150)).deep.equal([60, 120]);
+  });
+
+  it('selectAutoEditCutFrames prefers the strongest beat in the allowed window', function () {
+    const fps = 30;
+    const beats = [
+      { frameIndex: 60, intensity: 0.2 },
+      { frameIndex: 90, intensity: 0.9 },
+      { frameIndex: 100, intensity: 0.4 },
+    ];
+    expect(selectAutoEditCutFrames(beats, fps, 180)).deep.equal([90]);
+  });
+
+  it('buildBeatSyncedSegments spaces auto-edit cuts instead of cutting every beat', function () {
+    const segments = buildBeatSyncedSegments(
+      [15, 30, 45, 60, 75, 90, 105, 120],
+      150,
+      [
+        { frameNumber: 0, pts: 0, ptsTime: 1 },
+        { frameNumber: 1, pts: 0, ptsTime: 4 },
+      ],
+      12,
+      30,
+    );
+    expect(getCutFrameIndices(segments)).deep.equal([60, 120]);
+  });
+
+  it('snapBeatsToTempoGrid snaps onsets within a quarter-beat to the grid', function () {
+    const snapped = snapBeatsToTempoGrid(
+      [
+        { frameIndex: 29, intensity: 0.8 },
+        { frameIndex: 50, intensity: 0.4 },
+      ],
+      { periodFrames: 14, phaseFrame: 0 },
+      200,
+    );
+    expect(snapped.map(beat => beat.frameIndex)).deep.equal([28, 50]);
+    expect(snapped[0].intensity).equal(0.8);
+  });
+
+  it('projectBeatsOntoTempoGrid fills the grid and copies nearby onset intensity', function () {
+    const projected = projectBeatsOntoTempoGrid(
+      [
+        { frameIndex: 29, intensity: 0.8 },
+        { frameIndex: 50, intensity: 0.4 },
+      ],
+      { periodFrames: 14, phaseFrame: 0 },
+      50,
+    );
+    expect(projected.map(beat => beat.frameIndex)).deep.equal([14, 28, 42]);
+    expect(projected.map(beat => beat.intensity)).deep.equal([0, 0.8, 0]);
+  });
+
+  it('buildBeatSyncedSegments uses 2-4 beat spacing when tempo is set', function () {
+    const fps = 30;
+    const bpm = 90;
+    const periodFrames = fps * 60 / bpm;
+    const segments = buildBeatSyncedSegments(
+      [20, 40, 60, 80, 100, 120, 140, 160],
+      200,
+      [
+        { frameNumber: 0, pts: 0, ptsTime: 1 },
+        { frameNumber: 1, pts: 0, ptsTime: 4 },
+      ],
+      12,
+      fps,
+      { tempo: { bpm, periodFrames, phaseFrame: 0 } },
+    );
+    expect(getCutFrameIndices(segments)).deep.equal([40, 80, 120, 160]);
+  });
+
+  it('buildBeatSyncedSegments places scene cuts on the BPM grid', function () {
+    const fps = 30;
+    const bpm = 128;
+    const periodFrames = 14;
+    const totalFrames = 180;
+    const segments = buildBeatSyncedSegments(
+      [57, 113],
+      totalFrames,
+      [
+        { frameNumber: 0, pts: 0, ptsTime: 1 },
+        { frameNumber: 1, pts: 0, ptsTime: 4 },
+      ],
+      12,
+      fps,
+      { tempo: { bpm, periodFrames, phaseFrame: 0 } },
+    );
+    const cuts = getCutFrameIndices(segments);
+    expect(cuts.length).greaterThan(0);
+    for (const cut of cuts) {
+      expect(cut % periodFrames).equal(0);
+    }
+  });
+
+  it('buildBeatSyncedSegments cuts on the BPM grid even without detected onsets', function () {
+    const fps = 30;
+    const bpm = 90;
+    const periodFrames = fps * 60 / bpm;
+    const segments = buildBeatSyncedSegments(
+      [],
+      200,
+      [
+        { frameNumber: 0, pts: 0, ptsTime: 1 },
+        { frameNumber: 1, pts: 0, ptsTime: 4 },
+      ],
+      12,
+      fps,
+      { tempo: { bpm, periodFrames, phaseFrame: 0 } },
+    );
+    expect(getCutFrameIndices(segments)).deep.equal([40, 80, 120, 160]);
+  });
+
+  it('buildBeatSyncedSegments prefers the strongest on-grid beat in the cut window', function () {
+    const fps = 30;
+    const bpm = 90;
+    const periodFrames = fps * 60 / bpm;
+    const beats = [20, 40, 60, 80, 100, 120, 140, 160];
+    const intensities = [0.1, 0.1, 0.9, 0.1, 0.1, 0.8, 0.1, 0.1];
+    const segments = buildBeatSyncedSegments(
+      beats,
+      160,
+      [
+        { frameNumber: 0, pts: 0, ptsTime: 1 },
+        { frameNumber: 1, pts: 0, ptsTime: 4 },
+      ],
+      12,
+      fps,
+      {
+        beatIntensities: intensities,
+        tempo: { bpm, periodFrames, phaseFrame: 0 },
+      },
+    );
+    expect(getCutFrameIndices(segments)).deep.equal([60, 120]);
+  });
 });

package/src/typings.d.ts

@@ -4,4 +4,26 @@ declare module 'fft-js' {
     fftFreq(phasors: [number, number][], sampleRate: number): number[];
     fftMag(phasors: [number, number][]): number[];
   };
+}
+
+declare module 'music-tempo' {
+  export interface MusicTempoParams {
+    bufferSize?: number;
+    hopSize?: number;
+    timeStep?: number;
+    decayRate?: number;
+    peakFindingWindow?: number;
+    meanWndMultiplier?: number;
+    peakThreshold?: number;
+    minBeatInterval?: number;
+    maxBeatInterval?: number;
+    expiryTime?: number;
+  }
+
+  export default class MusicTempo {
+    constructor(audioData: number[] | Float32Array, params?: MusicTempoParams);
+    tempo: string | number;
+    beats: number[];
+    beatInterval: number;
+  }
 }

package/src/video.ts

@@ -5,6 +5,7 @@ import { resolve as resolvePath, join as joinPath } from 'path';
 import { tmpdir } from 'os';
 import ffmpegPath from 'ffmpeg-static';
 import { defaults } from './config';
+import { TempoEstimate, beatGridFrameIndices } from './beats';
 
 export interface AudioMuxSegment {
   seekSeconds: number;
@@ -426,11 +427,156 @@ const mergeSmallScenes = (sceneChanges: SceneChange[], videoDuration: number, mi
   return merged;
 };
 
+/** Minimum time between auto-edit cuts; cuts still land on beats. */
+export const DEFAULT_AUTO_EDIT_MIN_CUT_INTERVAL_SECONDS = 2;
+/** Window after the minimum gap in which the strongest beat is chosen. */
+export const DEFAULT_AUTO_EDIT_MAX_CUT_INTERVAL_SECONDS = 4;
+/** Half a bar of 4/4, used as the minimum gap when tempo is known. */
+export const AUTO_EDIT_MIN_CUT_BEATS = 2;
+/** One bar of 4/4, used as the maximum gap when tempo is known. */
+export const AUTO_EDIT_MAX_CUT_BEATS = 4;
+/** Snap an onset to the tempo grid when it is within this fraction of a beat. */
+export const BEAT_SNAP_TOLERANCE = 0.25;
+
+export const cutIntervalSecondsFromTempo = (bpm: number) => ({
+  minCutIntervalSeconds: AUTO_EDIT_MIN_CUT_BEATS * (60 / bpm),
+  maxCutIntervalSeconds: AUTO_EDIT_MAX_CUT_BEATS * (60 / bpm),
+});
+
+export const snapBeatsToTempoGrid = (
+  beats: Array<{ frameIndex: number; intensity?: number }>,
+  tempo: Pick<TempoEstimate, 'periodFrames' | 'phaseFrame'>,
+  totalFrames: number,
+): Array<{ frameIndex: number; intensity?: number }> => {
+  const period = tempo.periodFrames;
+  if (!(period > 0)) {
+    return beats.slice();
+  }
+  const maxSnap = Math.max(1, period * BEAT_SNAP_TOLERANCE);
+  const merged: Array<{ frameIndex: number; intensity?: number }> = [];
+  const push = (beat: { frameIndex: number; intensity?: number }) => {
+    for (let i = 0; i < merged.length; i++) {
+      if (merged[i].frameIndex === beat.frameIndex) {
+        if ((beat.intensity ?? 0) > (merged[i].intensity ?? 0)) {
+          merged[i] = beat;
+        }
+        return;
+      }
+    }
+    merged.push(beat);
+  };
+  for (const beat of beats) {
+    const n = Math.round((beat.frameIndex - tempo.phaseFrame) / period);
+    const grid = Math.round(tempo.phaseFrame + n * period);
+    const dist = Math.abs(beat.frameIndex - grid);
+    if (dist <= maxSnap && grid > 0 && grid < totalFrames) {
+      push({ frameIndex: grid, intensity: beat.intensity });
+    } else {
+      push(beat);
+    }
+  }
+  return merged.sort((a, b) => a.frameIndex - b.frameIndex);
+};
+
+/** Every BPM-grid beat, with onset intensity copied onto the nearest grid frame. */
+export const projectBeatsOntoTempoGrid = (
+  beats: Array<{ frameIndex: number; intensity?: number }>,
+  tempo: Pick<TempoEstimate, 'periodFrames' | 'phaseFrame'>,
+  totalFrames: number,
+): Array<{ frameIndex: number; intensity?: number }> => {
+  const grid = beatGridFrameIndices(tempo, totalFrames);
+  if (grid.length === 0) {
+    return [];
+  }
+  const period = tempo.periodFrames;
+  const maxSnap = Math.max(1, period * BEAT_SNAP_TOLERANCE);
+  const intensityByFrame = new Map<number, number>();
+  for (const beat of beats) {
+    const n = Math.round((beat.frameIndex - tempo.phaseFrame) / period);
+    const gridFrame = Math.round(tempo.phaseFrame + n * period);
+    if (Math.abs(beat.frameIndex - gridFrame) > maxSnap) {
+      continue;
+    }
+    if (gridFrame <= 0 || gridFrame >= totalFrames) {
+      continue;
+    }
+    const intensity = beat.intensity ?? 0;
+    if (intensity >= (intensityByFrame.get(gridFrame) ?? 0)) {
+      intensityByFrame.set(gridFrame, intensity);
+    }
+  }
+  return grid.map(frameIndex => ({
+    frameIndex,
+    intensity: intensityByFrame.get(frameIndex) ?? 0,
+  }));
+};
+
+export const selectAutoEditCutFrames = (
+  beats: Array<{ frameIndex: number; intensity?: number }>,
+  fps: number,
+  totalFrames: number,
+  minIntervalSeconds = DEFAULT_AUTO_EDIT_MIN_CUT_INTERVAL_SECONDS,
+  maxIntervalSeconds = DEFAULT_AUTO_EDIT_MAX_CUT_INTERVAL_SECONDS,
+): number[] => {
+  const sorted = beats
+    .filter(beat => beat.frameIndex > 0 && beat.frameIndex < totalFrames)
+    .sort((a, b) => a.frameIndex - b.frameIndex);
+  if (sorted.length === 0 || fps <= 0) {
+    return [];
+  }
+  if (minIntervalSeconds <= 0) {
+    return sorted.map(beat => beat.frameIndex);
+  }
+
+  const minGap = Math.max(1, Math.round(fps * minIntervalSeconds));
+  const maxGap = Math.max(minGap, Math.round(fps * maxIntervalSeconds));
+  const selected: number[] = [];
+  let lastCut = 0;
+
+  while (lastCut + minGap < totalFrames) {
+    const windowStart = lastCut + minGap;
+    const windowEnd = Math.min(lastCut + maxGap, totalFrames);
+
+    let best: { frameIndex: number; intensity?: number } | undefined;
+    let nextAfterWindow: { frameIndex: number; intensity?: number } | undefined;
+
+    for (const beat of sorted) {
+      if (beat.frameIndex < windowStart) {
+        continue;
+      }
+      if (beat.frameIndex >= windowEnd) {
+        nextAfterWindow = beat;
+        break;
+      }
+      const intensity = beat.intensity ?? 0;
+      if (!best || intensity > (best.intensity ?? 0)) {
+        best = beat;
+      }
+    }
+
+    const chosen = best ?? nextAfterWindow;
+    if (!chosen) {
+      break;
+    }
+    selected.push(chosen.frameIndex);
+    lastCut = chosen.frameIndex;
+  }
+
+  return selected;
+};
+
 export const buildBeatSyncedSegments = (
   beatFrameIndices: number[],
   totalFrames: number,
   sceneChanges: SceneChange[],
   videoDuration: number,
+  fps: number,
+  options?: {
+    beatIntensities?: number[];
+    minCutIntervalSeconds?: number;
+    maxCutIntervalSeconds?: number;
+    tempo?: TempoEstimate;
+  },
 ): VideoSegment[] => {
   if (sceneChanges.length === 0) {
     return [
@@ -451,7 +597,27 @@ export const buildBeatSyncedSegments = (
     seekPositions = Array.from({ length: count }, (_, i) => (i * videoDuration) / count);
   }
 
-  const boundaries = [0, ...beatFrameIndices];
+  const tempo = options?.tempo;
+  const tempoIntervals = tempo && tempo.bpm > 0
+    ? cutIntervalSecondsFromTempo(tempo.bpm)
+    : undefined;
+  let beats: Array<{ frameIndex: number; intensity?: number }> = beatFrameIndices.map((frameIndex, i) => ({
+    frameIndex,
+    intensity: options?.beatIntensities?.[i],
+  }));
+  if (tempo) {
+    beats = projectBeatsOntoTempoGrid(beats, tempo, totalFrames);
+  }
+
+  const cutBeats = selectAutoEditCutFrames(
+    beats,
+    fps,
+    totalFrames,
+    options?.minCutIntervalSeconds ?? tempoIntervals?.minCutIntervalSeconds,
+    options?.maxCutIntervalSeconds ?? tempoIntervals?.maxCutIntervalSeconds,
+  );
+
+  const boundaries = [0, ...cutBeats];
   if (boundaries[boundaries.length - 1] !== totalFrames) {
     boundaries.push(totalFrames);
   }
@@ -468,6 +634,12 @@ export const buildBeatSyncedSegments = (
   return segments;
 };
 
+export const getCutFrameIndices = (segments: VideoSegment[]): number[] =>
+  segments
+    .slice(1)
+    .map(segment => segment.outputStartFrame)
+    .filter(frameIndex => frameIndex > 0);
+
 export const writeConcatFile = (
   segments: VideoSegment[],
   videoPath: string,