nodejs-audio-visualizer 4.1.1 → 4.2.2

package/src/highlight.ts

@@ -0,0 +1,284 @@
+/** Output length per segment when `audio.autoHighlight` is true (seconds). */
+
+import { waitForEventLoop } from "./waitForEventLoop";
+
+export const HIGHLIGHT_DURATION_SEC = 15;
+
+export interface BeatFrameEvent {
+  frameIndex: number;
+  intensity: number;
+}
+
+export interface HighlightAudioSegment {
+  seekSeconds: number;
+  durationSeconds: number;
+}
+
+/** One auto-highlight segment before concatenation (for separate output files). */
+export interface HighlightRun {
+  startFrame: number;
+  highlightFrames: number;
+  spectrums: number[][];
+  beatFrameIndices: number[];
+  audioSegment: HighlightAudioSegment;
+}
+
+function buildHighlightRun(
+  fps: number,
+  seg: { startFrame: number; highlightFrames: number },
+  spectrums: number[][],
+  beatEvents: BeatFrameEvent[],
+): HighlightRun {
+  const { startFrame, highlightFrames } = seg;
+  const spectrumsSlice = spectrums.slice(startFrame, startFrame + highlightFrames);
+  const beatFrameIndices = beatEvents
+    .map(b => b.frameIndex)
+    .filter(i => i >= startFrame && i < startFrame + highlightFrames)
+    .map(i => i - startFrame)
+    .sort((a, b) => a - b);
+  return {
+    startFrame,
+    highlightFrames,
+    spectrums: spectrumsSlice,
+    beatFrameIndices,
+    audioSegment: {
+      seekSeconds: startFrame / fps,
+      durationSeconds: highlightFrames / fps,
+    },
+  };
+}
+
+function frameEnergy(spectrum: number[]): number {
+  let s = 0;
+  for (let i = 0; i < spectrum.length; i++) {
+    const v = spectrum[i];
+    s += v * v;
+  }
+  return s;
+}
+
+function rangesOverlap(
+  aStart: number,
+  aEndExclusive: number,
+  ranges: Array<{ start: number; endExclusive: number }>,
+): boolean {
+  for (const r of ranges) {
+    if (aStart < r.endExclusive && aEndExclusive > r.start) {
+      return true;
+    }
+  }
+  return false;
+}
+
+async function buildFrameEnergies(
+  spectrums: number[][],
+  totalFrames: number,
+): Promise<Float64Array> {
+  const energies = new Float64Array(totalFrames);
+  for (let i = 0; i < totalFrames; i++) {
+    energies[i] = frameEnergy(spectrums[i] ?? []);
+    await waitForEventLoop();
+  }
+  return energies;
+}
+
+/**
+ * Chooses the fixed-length window whose frames have the highest summed spectral energy
+ * (typical chorus / drop vs a single loud intro transient).
+ * Skips windows that overlap any excluded range.
+ */
+async function findBestHighlightStart(
+  energies: Float64Array,
+  totalFrames: number,
+  windowFrames: number,
+  excludeRanges: Array<{ start: number; endExclusive: number }>,
+): Promise<number | null> {
+  if (totalFrames <= windowFrames) {
+    if (!rangesOverlap(0, totalFrames, excludeRanges)) {
+      return 0;
+    }
+    return null;
+  }
+
+  let minE = Infinity;
+  let maxE = -Infinity;
+  for (let i = 0; i < totalFrames; i++) {
+    const e = energies[i];
+    if (e < minE) {
+      minE = e;
+    }
+    if (e > maxE) {
+      maxE = e;
+    }
+    await waitForEventLoop();
+  }
+
+  const prefix = new Float64Array(totalFrames + 1);
+  for (let i = 0; i < totalFrames; i++) {
+    prefix[i + 1] = prefix[i] + energies[i];
+    await waitForEventLoop();
+  }
+
+  const maxStart = totalFrames - windowFrames;
+  let bestStart: number | null = null;
+  let bestScore = -Infinity;
+
+  for (let start = 0; start <= maxStart; start++) {
+    if (rangesOverlap(start, start + windowFrames, excludeRanges)) {
+      await waitForEventLoop();
+      continue;
+    }
+    const score = prefix[start + windowFrames] - prefix[start];
+    if (score > bestScore) {
+      bestScore = score;
+      bestStart = start;
+    }
+    await waitForEventLoop();
+  }
+
+  if (bestStart === null) {
+    return null;
+  }
+
+  if (maxE <= minE) {
+    const center = Math.floor((totalFrames - windowFrames) / 2);
+    let bestDist = Infinity;
+    let tieBreak: number | null = null;
+    for (let start = 0; start <= maxStart; start++) {
+      if (rangesOverlap(start, start + windowFrames, excludeRanges)) {
+        await waitForEventLoop();
+        continue;
+      }
+      const dist = Math.abs(start - center);
+      if (dist < bestDist) {
+        bestDist = dist;
+        tieBreak = start;
+      }
+      await waitForEventLoop();
+    }
+    return tieBreak;
+  }
+
+  return bestStart;
+}
+
+
+/**
+ * Picks up to `segmentCount` non-overlapping 15s windows with highest summed spectral energy each,
+ * sorts them chronologically, concatenates spectrums, and remaps beat indices.
+ */
+export async function computeHighlightSlice(
+  fps: number,
+  totalFrames: number,
+  spectrums: number[][],
+  beatEvents: BeatFrameEvent[],
+  segmentCount = 1,
+): Promise<{
+  startFrame: number;
+  highlightFrames: number;
+  spectrums: number[][];
+  beatFrameIndices: number[];
+  audioSeekSeconds: number;
+  audioDurationSeconds: number;
+  audioSegments: HighlightAudioSegment[];
+  runs: HighlightRun[];
+}> {
+  const highlightFramesTarget = Math.ceil(HIGHLIGHT_DURATION_SEC * fps);
+
+  if (totalFrames <= 0) {
+    return {
+      startFrame: 0,
+      highlightFrames: 0,
+      spectrums: [],
+      beatFrameIndices: [],
+      audioSeekSeconds: 0,
+      audioDurationSeconds: 0,
+      audioSegments: [],
+      runs: [],
+    };
+  }
+
+  if (totalFrames <= highlightFramesTarget) {
+    const audioSegments: HighlightAudioSegment[] = [
+      {
+        seekSeconds: 0,
+        durationSeconds: totalFrames / fps,
+      },
+    ];
+    const fullSeg = { startFrame: 0, highlightFrames: totalFrames };
+    return {
+      startFrame: 0,
+      highlightFrames: totalFrames,
+      spectrums: spectrums.slice(),
+      beatFrameIndices: beatEvents.map(b => b.frameIndex).sort((a, b) => a - b),
+      audioSeekSeconds: 0,
+      audioDurationSeconds: totalFrames / fps,
+      audioSegments,
+      runs: [buildHighlightRun(fps, fullSeg, spectrums, beatEvents)],
+    };
+  }
+
+  const highlightFrames = highlightFramesTarget;
+  const energies = await buildFrameEnergies(spectrums, totalFrames);
+  const excludeRanges: Array<{ start: number; endExclusive: number }> = [];
+  const rawSegments: Array<{ startFrame: number; highlightFrames: number }> = [];
+
+  const n = Math.max(1, Math.floor(segmentCount));
+  for (let k = 0; k < n; k++) {
+    const startFrame = await findBestHighlightStart(
+      energies,
+      totalFrames,
+      highlightFrames,
+      excludeRanges,
+    );
+    if (startFrame === null) {
+      break;
+    }
+    rawSegments.push({ startFrame, highlightFrames });
+    excludeRanges.push({ start: startFrame, endExclusive: startFrame + highlightFrames });
+  }
+
+  rawSegments.sort((a, b) => a.startFrame - b.startFrame);
+
+  const runs = rawSegments.map(seg =>
+    buildHighlightRun(fps, seg, spectrums, beatEvents),
+  );
+
+  const slicedSpectrums: number[][] = [];
+  const beatFrameIndices: number[] = [];
+  let outOffset = 0;
+
+  for (const seg of rawSegments) {
+    const chunk = spectrums.slice(seg.startFrame, seg.startFrame + seg.highlightFrames);
+    slicedSpectrums.push(...chunk);
+
+    for (const b of beatEvents) {
+      const i = b.frameIndex;
+      if (i >= seg.startFrame && i < seg.startFrame + seg.highlightFrames) {
+        beatFrameIndices.push(i - seg.startFrame + outOffset);
+      }
+    }
+    outOffset += seg.highlightFrames;
+  }
+
+  beatFrameIndices.sort((a, b) => a - b);
+
+  const totalHighlightFrames = slicedSpectrums.length;
+  const audioSegments: HighlightAudioSegment[] = rawSegments.map(seg => ({
+    seekSeconds: seg.startFrame / fps,
+    durationSeconds: seg.highlightFrames / fps,
+  }));
+
+  const first = audioSegments[0] ?? { seekSeconds: 0, durationSeconds: 0 };
+
+  return {
+    startFrame: rawSegments[0]?.startFrame ?? 0,
+    highlightFrames: totalHighlightFrames,
+    spectrums: slicedSpectrums,
+    beatFrameIndices,
+    audioSeekSeconds: first.seekSeconds,
+    audioDurationSeconds: first.durationSeconds,
+    audioSegments,
+    runs,
+  };
+}

package/src/index.ts

@@ -1,3 +1,4 @@
+import path from 'path';
 import {
   getAudioFilePath,
   getBackgroundImagePath,
@@ -28,13 +29,18 @@ import {
   getPolarColor,
   getPolarOpacityParsed,
   getAutoEditVideo,
+  getAudioAutoHighlight,
+  getAudioAutoHighlightCount,
 } from './config';
 import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor } from './audio';
 import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps } from './image';
-import { spawnFfmpegVideoWriter, getProgress, calculateProgress, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, spawnConcatVideoFrameReader, cleanupConcatFile } from './video';
+import { spawnFfmpegVideoWriter, getProgress, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, spawnConcatVideoFrameReader, cleanupConcatFile } from './video';
 import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
 import { createBeatDetector } from './beats';
 export { BeatInfo, BeatDetectorOptions } from './beats';
+import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
+import { waitForEventLoop } from './waitForEventLoop';
+export { computeHighlightSlice, HIGHLIGHT_DURATION_SEC, BeatFrameEvent, HighlightAudioSegment, HighlightRun } from './highlight';
 
 export const PCM_FORMAT = {
   bit: 8,
@@ -46,7 +52,10 @@ const PROCESSING_BUFFER_SIZE = Math.pow(2, 12);
 
 export interface Config {
   audio: {
-    path: string
+    path: string;
+    autoHighlight?: boolean;
+    /** When `autoHighlight` is true, number of non-overlapping 15s windows to stitch (default 1). */
+    autoHighlightCount?: number;
   };
   image?: {
     path: string;
@@ -164,9 +173,10 @@ const createVisualizerFrameGenerator = (
 interface PreProcessedAudio {
   spectrums: number[][];
   beatFrameIndices: number[];
+  beatEvents: { frameIndex: number; intensity: number }[];
 }
 
-const PREPROCESS_YIELD_EVERY = 256;
+const PREPROCESS_YIELD_EVERY = 1;
 
 const preProcessAudio = async (
   audioBuffer: Buffer,
@@ -182,6 +192,7 @@ const preProcessAudio = async (
 
   const spectrums: number[][] = [];
   const beatFrameIndices: number[] = [];
+  const beatEvents: { frameIndex: number; intensity: number }[] = [];
 
   for (let i = 0; i < framesCount; i++) {
     const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
@@ -195,14 +206,13 @@ const preProcessAudio = async (
     spectrums.push(spectrum);
     if (beat.isBeat) {
       beatFrameIndices.push(i);
+      beatEvents.push({ frameIndex: i, intensity: beat.intensity });
     }
 
-    if ((i + 1) % PREPROCESS_YIELD_EVERY === 0) {
-      await new Promise<void>(resolve => setImmediate(resolve));
-    }
+    await waitForEventLoop();
   }
 
-  return { spectrums, beatFrameIndices };
+  return { spectrums, beatFrameIndices, beatEvents };
 };
 
 async function prepareBackgroundForRender(params: {
@@ -338,8 +348,14 @@ async function resolveBackgroundFrameBuffer(params: {
   return staticBackgroundBuffer;
 }
 
+export interface RenderAudioVisualizerResult {
+  exitCode: number;
+  /** Absolute paths of video files written successfully (exit code 0, full pass, no early stop). */
+  outputVideoFiles: string[];
+}
+
 export const renderAudioVisualizer = (config: Config, onProgress?: (progress: number) => any, shouldStop?: () => boolean) =>
-  new Promise<number>(async (resolve) => {
+  new Promise<RenderAudioVisualizerResult>(async (resolve) => {
     if (config.outVideo.spectrum && config.outVideo.polar) {
       throw new Error('Cannot use both "spectrum" and "polar" options. Please specify only one visualizer type.');
     }
@@ -378,74 +394,183 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
     }
     const preprocessed = await preProcessAudio(audioBuffer, sampleRate, FPS, framesCount);
 
-    const {
-      backgroundWidth,
-      backgroundHeight,
-      defaultColor,
-      staticBackgroundBuffer,
-      videoFrameReader,
-      videoFrameSize,
-      encodeVideoFrame,
-      concatFilePath,
-    } = await prepareBackgroundForRender({
-      useVideoBackground,
-      backgroundVideoPath,
-      backgroundImagePath,
-      beatFrameIndices: preprocessed.beatFrameIndices,
-      framesCount,
-      outputResolution,
-      fps: FPS,
-      autoEditVideo: getAutoEditVideo(config),
-    });
-
-    const createVisualizerFrame = createVisualizerFrameGenerator(
-      config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
-    );
-
-    const ffmpegVideoWriter = spawnFfmpegVideoWriter({
-      audioFilename: audioFilePath,
-      videoFileName: outVideoPath,
-      fps: FPS,
-      ...(!!onProgress && { onStderr: getProgress(calculateProgress(framesCount + 1, onProgress)) }),
-      ...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
-      ...(ffmpeg_preset && { preset: ffmpeg_preset }),
-    });
-    ffmpegVideoWriter.on('exit', (code: number) => resolve(code));
+    const autoHighlight = getAudioAutoHighlight(config);
+    let spectrumsForRender = preprocessed.spectrums;
+    let beatIndicesForRender = preprocessed.beatFrameIndices;
+    let framesCountForRender = framesCount;
+    type HighlightSliceResult = ReturnType<typeof computeHighlightSlice> extends Promise<
+      infer R
+    >
+      ? R
+      : never;
+    let highlightSlice: HighlightSliceResult | undefined;
+
+    if (autoHighlight) {
+      highlightSlice = await computeHighlightSlice(
+        FPS,
+        framesCount,
+        preprocessed.spectrums,
+        preprocessed.beatEvents,
+        getAudioAutoHighlightCount(config),
+      );
+      spectrumsForRender = highlightSlice.spectrums;
+      beatIndicesForRender = highlightSlice.beatFrameIndices;
+      framesCountForRender = highlightSlice.highlightFrames;
+    }
 
-    for (let i = 0; i < framesCount; i++) {
-      const spectrum = preprocessed.spectrums[i];
+    const separateHighlightFiles =
+      autoHighlight &&
+      highlightSlice &&
+      highlightSlice.runs.length > 1;
+
+    type VideoRenderPass = {
+      outPath: string;
+      spectrums: number[][];
+      beatIndices: number[];
+      frameCount: number;
+      audioSegment: import('./highlight').HighlightAudioSegment | undefined;
+    };
 
-      const backgroundImageBuffer = await resolveBackgroundFrameBuffer({
-        frameIndex: i,
-        useVideoBackground,
+    const passes: VideoRenderPass[] =
+      separateHighlightFiles && highlightSlice
+      ? highlightSlice.runs.map((run, i) => {
+          const dir = path.dirname(outVideoPath);
+          const ext = path.extname(outVideoPath);
+          const base = path.basename(outVideoPath, ext);
+          const numberedPath = path.resolve(dir, `${base}-${i + 1}${ext}`);
+          return {
+            outPath: numberedPath,
+            spectrums: run.spectrums,
+            beatIndices: run.beatFrameIndices,
+            frameCount: run.highlightFrames,
+            audioSegment: run.audioSegment,
+          };
+        })
+      : [
+          {
+            outPath: outVideoPath,
+            spectrums: spectrumsForRender,
+            beatIndices: beatIndicesForRender,
+            frameCount: framesCountForRender,
+            audioSegment:
+              autoHighlight &&
+              highlightSlice &&
+              highlightSlice.audioSegments.length > 0
+                ? highlightSlice.audioSegments[0]
+                : undefined,
+          },
+        ];
+
+    const progressDenominator =
+      passes.reduce((s, p) => s + p.frameCount, 0) + passes.length;
+    let progressFrameBase = 0;
+    let lastExitCode = 0;
+    const outputVideoFiles: string[] = [];
+
+    passLoop: for (const pass of passes) {
+      const {
+        backgroundWidth,
+        backgroundHeight,
+        defaultColor,
+        staticBackgroundBuffer,
         videoFrameReader,
         videoFrameSize,
         encodeVideoFrame,
-        staticBackgroundBuffer,
+        concatFilePath,
+      } = await prepareBackgroundForRender({
+        useVideoBackground,
+        backgroundVideoPath,
+        backgroundImagePath,
+        beatFrameIndices: pass.beatIndices,
+        framesCount: pass.frameCount,
+        outputResolution,
+        fps: FPS,
+        autoEditVideo: getAutoEditVideo(config),
+      });
+
+      const createVisualizerFrame = createVisualizerFrameGenerator(
+        config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
+      );
+
+      const ffmpegVideoWriter = spawnFfmpegVideoWriter({
+        audioFilename: audioFilePath,
+        videoFileName: pass.outPath,
+        fps: FPS,
+        ...(pass.audioSegment && { audioSegment: pass.audioSegment }),
+        ...(!!onProgress && {
+          onStderr: getProgress((currentFrame: number) => {
+            const g = progressFrameBase + currentFrame;
+            onProgress(
+              +(Math.min(100, (g / progressDenominator) * 100)).toFixed(2),
+            );
+          }),
+        }),
+        ...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
+        ...(ffmpeg_preset && { preset: ffmpeg_preset }),
+      });
+
+      const exitPromise = new Promise<number>(res => {
+        let settled = false;
+        ffmpegVideoWriter.on('exit', (code: number | null) => {
+          if (settled) {
+            return;
+          }
+          settled = true;
+          res(code ?? 0);
+        });
       });
 
-      const commonVisualizerFrameProps: CommonVisualizerFrameProps = {
-        backgroundImageBuffer,
-        spectrum,
-      };
-      const frameImage = createVisualizerFrame(commonVisualizerFrameProps);
-      const isFrameProcessed = ffmpegVideoWriter.stdin.write(frameImage.data);
-      if (!isFrameProcessed) {
-        await waitDrain(ffmpegVideoWriter.stdin);
+      let stoppedEarly = false;
+      for (let i = 0; i < pass.frameCount; i++) {
+        const spectrum = pass.spectrums[i];
+
+        const backgroundImageBuffer = await resolveBackgroundFrameBuffer({
+          frameIndex: i,
+          useVideoBackground,
+          videoFrameReader,
+          videoFrameSize,
+          encodeVideoFrame,
+          staticBackgroundBuffer,
+        });
+
+        const commonVisualizerFrameProps: CommonVisualizerFrameProps = {
+          backgroundImageBuffer,
+          spectrum,
+        };
+        const frameImage = createVisualizerFrame(commonVisualizerFrameProps);
+        const isFrameProcessed = ffmpegVideoWriter.stdin.write(frameImage.data);
+        if (!isFrameProcessed) {
+          await waitDrain(ffmpegVideoWriter.stdin);
+        }
+        if (shouldStop && shouldStop()) {
+          stoppedEarly = true;
+          break;
+        }
+        if (frame_processing_delay) {
+          await sleep(frame_processing_delay);
+        }
+        await waitForEventLoop();
       }
-      if (shouldStop && shouldStop()) {
-        break;
+
+      if (videoFrameReader) {
+        videoFrameReader.kill();
       }
-      if (frame_processing_delay) {
-        await sleep(frame_processing_delay);
+      if (concatFilePath) {
+        cleanupConcatFile(concatFilePath);
       }
-    }
+      ffmpegVideoWriter.stdin.end();
 
-    if (videoFrameReader) {
-      videoFrameReader.kill();
-    }
-    if (concatFilePath) {
-      cleanupConcatFile(concatFilePath);
+      lastExitCode = await exitPromise;
+      progressFrameBase += pass.frameCount + 1;
+
+      if (lastExitCode === 0 && !stoppedEarly) {
+        outputVideoFiles.push(pass.outPath);
+      }
+
+      if (stoppedEarly || lastExitCode !== 0) {
+        break passLoop;
+      }
     }
-    ffmpegVideoWriter.stdin.end();
+
+    resolve({ exitCode: lastExitCode, outputVideoFiles });
   });

package/src/tests/config.test.ts

@@ -4,6 +4,8 @@ import { Config } from '../index';
 import {
   defaults,
   getAudioFilePath,
+  getAudioAutoHighlight,
+  getAudioAutoHighlightCount,
   getBackgroundImagePath,
   getOutVideoPath,
   getFPS,
@@ -30,6 +32,36 @@ describe('config', function() {
     expect(result).equal(expected);
   });
 
+  it('getAudioAutoHighlight', function() {
+    expect(
+      getAudioAutoHighlight({ audio: { path: 'a.wav' } } as Config)
+    ).equal(false);
+    expect(
+      getAudioAutoHighlight({ audio: { path: 'a.wav', autoHighlight: false } } as Config)
+    ).equal(false);
+    expect(
+      getAudioAutoHighlight({ audio: { path: 'a.wav', autoHighlight: true } } as Config)
+    ).equal(true);
+  });
+
+  it('getAudioAutoHighlightCount', function() {
+    expect(
+      getAudioAutoHighlightCount({ audio: { path: 'a.wav' } } as Config)
+    ).equal(1);
+    expect(
+      getAudioAutoHighlightCount({ audio: { path: 'a.wav', autoHighlightCount: 3 } } as Config)
+    ).equal(3);
+    expect(
+      getAudioAutoHighlightCount.bind(undefined, { audio: { path: 'a.wav', autoHighlightCount: 0 } } as Config)
+    ).to.throw('Invalid audio.autoHighlightCount');
+    expect(
+      getAudioAutoHighlightCount.bind(undefined, { audio: { path: 'a.wav', autoHighlightCount: 1.5 } } as Config)
+    ).to.throw('Invalid audio.autoHighlightCount');
+    expect(
+      getAudioAutoHighlightCount.bind(undefined, { audio: { path: 'a.wav', autoHighlightCount: NaN } } as Config)
+    ).to.throw('Invalid audio.autoHighlightCount');
+  });
+
   it('getBackgroundImagePath', function() {
     const result = getBackgroundImagePath({
       image: { path: 'test/path' }