nodejs-audio-visualizer 3.0.0 → 4.1.0

package/src/index.ts

@@ -1,6 +1,7 @@
 import {
   getAudioFilePath,
   getBackgroundImagePath,
+  getBackgroundVideoPath,
   getOutVideoPath,
   getFPS,
   getSpectrumBusMargin,
@@ -13,15 +14,27 @@ import {
   getFfmpeg_cfr,
   getFfmpeg_preset,
   getFrame_processing_delay,
+  getOutputResolution,
   rotationAliasValues,
   getSpectrumRotation,
   getSpectrumEffect,
   SpectrumEffect,
+  getPolarXAbsolute,
+  getPolarYAbsolute,
+  getPolarInnerRadius,
+  getPolarMaxBarLength,
+  getPolarBarWidth,
+  getPolarEffect,
+  getPolarColor,
+  getPolarOpacityParsed,
+  getAutoEditVideo,
 } from './config';
 import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor } from './audio';
-import { parseImage, getImageColor, invertColor, Color, convertToBmp, createVisualizerFrameGenerator } from './image';
-import { spawnFfmpegVideoWriter, getProgress, calculateProgress, waitDrain } from './video';
-import { createBpmEncoder } from './bpmEncoder';
+import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps } from './image';
+import { spawnFfmpegVideoWriter, getProgress, calculateProgress, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, spawnConcatVideoFrameReader, cleanupConcatFile } from './video';
+import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
+import { createBeatDetector } from './beats';
+export { BeatInfo, BeatDetectorOptions } from './beats';
 
 export const PCM_FORMAT = {
   bit: 8,
@@ -35,12 +48,20 @@ export interface Config {
   audio: {
     path: string
   };
-  image: {
+  image?: {
     path: string;
   };
+  video?: {
+    path: string;
+    autoEdit?: boolean;
+  };
   outVideo: {
     path: string;
     fps?: number;
+    resolution?: {
+      width: number;
+      height: number;
+    };
     spectrum?: {
       width?: SpectrumSizeValue;
       height?: SpectrumSizeValue;
@@ -50,6 +71,16 @@ export interface Config {
       effect?: SpectrumEffect;
       color?: Color | string;
       opacity?: string;
+    };
+    polar?: {
+      x?: number | PositionAliasName;
+      y?: number | PositionAliasName;
+      innerRadius?: number;
+      maxBarLength?: number;
+      barWidth?: number;
+      effect?: SpectrumEffect;
+      color?: Color | string;
+      opacity?: string;
     }
   };
   tweaks?: {
@@ -74,14 +105,251 @@ export type RotationAliasName = typeof rotationAliasValues[number];
 const sleep = (timeout: number) =>
   new Promise(resolve => setTimeout(resolve, timeout));
 
+const createVisualizerFrameGenerator = (
+  config: Config,
+  backgroundWidth: number,
+  backgroundHeight: number,
+  defaultColor: Color,
+  spectrumBusMargin: number
+): (params: CommonVisualizerFrameProps) => EncodedBmp => {
+  if (!config.outVideo.spectrum) {
+    const createPolarVisualizerFrame = createPolarVisualizerFrameGenerator();
+    const polarX = getPolarXAbsolute(config, backgroundWidth);
+    const polarY = getPolarYAbsolute(config, backgroundHeight);
+    const polarInnerRadius = getPolarInnerRadius(config);
+    const polarMaxBarLength = getPolarMaxBarLength(config);
+    const polarBarWidth = getPolarBarWidth(config);
+    const polarColor = getPolarColor(config) || invertColor(defaultColor);
+    const polarEffect = getPolarEffect(config);
+    const polarOpacity = getPolarOpacityParsed(config);
+
+    return (params: CommonVisualizerFrameProps) => {
+      return createPolarVisualizerFrame({
+        ...params,
+        centerX: polarX,
+        centerY: polarY,
+        innerRadius: polarInnerRadius,
+        maxBarLength: polarMaxBarLength,
+        barWidth: polarBarWidth,
+        color: polarColor,
+        opacity: polarOpacity,
+        spectrumEffect: polarEffect,
+      });
+    };
+  }
+  const createSpectrumVisualizerFrame = createSpectrumVisualizerFrameGenerator();
+  const spectrumWidth = getSpectrumWidthAbsolute(config, backgroundWidth);
+  const spectrumHeight = getSpectrumHeightAbsolute(config, backgroundHeight);
+  const spectrumX = getSpectrumXAbsolute(config, spectrumWidth, backgroundWidth);
+  const spectrumY = getSpectrumYAbsolute(config, spectrumHeight, backgroundHeight);
+  const spectrumRotation = getSpectrumRotation(config);
+  const spectrumColor = getSpectrumColor(config) || invertColor(defaultColor);
+  const spectrumEffect = getSpectrumEffect(config);
+  const spectrumOpacity = getSpectrumOpacityParsed(config);
+
+  return (params: CommonVisualizerFrameProps) => {
+    return createSpectrumVisualizerFrame({
+      ...params,
+      size: { width: spectrumWidth, height: spectrumHeight },
+      position: { x: spectrumX, y: spectrumY },
+      rotation: spectrumRotation,
+      margin: spectrumBusMargin,
+      color: spectrumColor,
+      opacity: spectrumOpacity,
+      spectrumEffect,
+    });
+  };
+};
+
+interface PreProcessedAudio {
+  spectrums: number[][];
+  beatFrameIndices: number[];
+}
+
+const preProcessAudio = (
+  audioBuffer: Buffer,
+  sampleRate: number,
+  fps: number,
+  framesCount: number,
+): PreProcessedAudio => {
+  const audioDataStep = Math.trunc(audioBuffer.length / framesCount);
+  const processingBuffer = new Float32Array(PROCESSING_BUFFER_SIZE).fill(0);
+  const skipFramesCount = fps < 45 ? 1 : 2;
+  const processSpectrum = createSpectrumsProcessor(sampleRate, skipFramesCount);
+  const detectBeat = createBeatDetector(fps);
+
+  const spectrums: number[][] = [];
+  const beatFrameIndices: number[] = [];
+
+  for (let i = 0; i < framesCount; i++) {
+    const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
+    processingBuffer.copyWithin(0, currentFrameData.length);
+    processingBuffer.set(currentFrameData, PROCESSING_BUFFER_SIZE - currentFrameData.length);
+
+    const audioDataParser = () => Array.from(processingBuffer);
+    const spectrum = processSpectrum(audioDataParser);
+    const beat = detectBeat(spectrum);
+
+    spectrums.push(spectrum);
+    if (beat.isBeat) {
+      beatFrameIndices.push(i);
+    }
+  }
+
+  return { spectrums, beatFrameIndices };
+};
+
+async function prepareBackgroundForRender(params: {
+  useVideoBackground: boolean;
+  backgroundVideoPath: string | undefined;
+  backgroundImagePath: string | undefined;
+  beatFrameIndices: number[];
+  framesCount: number;
+  outputResolution: ReturnType<typeof getOutputResolution>;
+  fps: number;
+  autoEditVideo: boolean;
+}): Promise<{
+  backgroundWidth: number;
+  backgroundHeight: number;
+  defaultColor: Color;
+  staticBackgroundBuffer: EncodedBmp;
+  videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
+  videoFrameSize: number;
+  encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
+  concatFilePath?: string;
+}> {
+  const {
+    useVideoBackground,
+    backgroundVideoPath,
+    backgroundImagePath,
+    beatFrameIndices,
+    framesCount,
+    outputResolution,
+    fps,
+    autoEditVideo,
+  } = params;
+
+  if (useVideoBackground && backgroundVideoPath) {
+    const videoInfo = await getVideoInfo(backgroundVideoPath);
+    const sceneChanges = autoEditVideo
+      ? await detectSceneChanges(backgroundVideoPath)
+      : [];
+    const backgroundWidth = outputResolution?.width ?? videoInfo.width;
+    const backgroundHeight = outputResolution?.height ?? videoInfo.height;
+
+    const videoFrameSize = backgroundWidth * backgroundHeight * 3;
+    const encodeVideoFrame = createBgrFrameEncoder({ width: backgroundWidth, height: backgroundHeight });
+
+    const segments = buildBeatSyncedSegments(
+      beatFrameIndices,
+      framesCount,
+      sceneChanges,
+      videoInfo.duration,
+    );
+
+    const concatFilePath = writeConcatFile(segments, backgroundVideoPath, videoInfo.duration, fps);
+
+    const videoFrameReader = spawnConcatVideoFrameReader({
+      concatFilePath,
+      fps,
+      totalFrames: framesCount,
+      ...(outputResolution && {
+        width: backgroundWidth,
+        height: backgroundHeight,
+        sourceWidth: videoInfo.width,
+        sourceHeight: videoInfo.height,
+      }),
+    });
+
+    const firstFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
+    if (!firstFrame) {
+      throw new Error(`Could not read frames from video: ${backgroundVideoPath}`);
+    }
+    const defaultColor = getVideoFrameColor(firstFrame, backgroundWidth, backgroundHeight);
+    const staticBackgroundBuffer = encodeVideoFrame(firstFrame);
+    return {
+      backgroundWidth,
+      backgroundHeight,
+      defaultColor,
+      staticBackgroundBuffer,
+      videoFrameReader,
+      videoFrameSize,
+      encodeVideoFrame,
+      concatFilePath,
+    };
+  }
+
+  if (!backgroundImagePath) {
+    throw new Error('Background image path is required when not using video background.');
+  }
+  const backgroundImageBmpBuffer = await convertToBmp(
+    backgroundImagePath,
+    outputResolution?.width,
+    outputResolution?.height,
+  );
+  const backgroundImage = parseImage(backgroundImageBmpBuffer);
+  const backgroundWidth = backgroundImage.width;
+  const backgroundHeight = backgroundImage.height;
+  const defaultColor = getImageColor(backgroundImage);
+
+  const bpmEncoder = createBpmEncoder({ width: backgroundWidth, height: backgroundHeight });
+  const staticBackgroundBuffer = bpmEncoder(backgroundImage.data);
+  return {
+    backgroundWidth,
+    backgroundHeight,
+    defaultColor,
+    staticBackgroundBuffer,
+    videoFrameSize: 0,
+  };
+}
+
+async function resolveBackgroundFrameBuffer(params: {
+  frameIndex: number;
+  useVideoBackground: boolean;
+  videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
+  videoFrameSize: number;
+  encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
+  staticBackgroundBuffer: EncodedBmp;
+}): Promise<EncodedBmp> {
+  const {
+    frameIndex,
+    useVideoBackground,
+    videoFrameReader,
+    videoFrameSize,
+    encodeVideoFrame,
+    staticBackgroundBuffer,
+  } = params;
+
+  if (useVideoBackground && videoFrameReader && encodeVideoFrame) {
+    if (frameIndex === 0) {
+      return staticBackgroundBuffer;
+    }
+    const videoFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
+    return videoFrame
+      ? encodeVideoFrame(videoFrame)
+      : staticBackgroundBuffer;
+  }
+  return staticBackgroundBuffer;
+}
+
 export const renderAudioVisualizer = (config: Config, onProgress?: (progress: number) => any, shouldStop?: () => boolean) =>
   new Promise<number>(async (resolve) => {
+    if (config.outVideo.spectrum && config.outVideo.polar) {
+      throw new Error('Cannot use both "spectrum" and "polar" options. Please specify only one visualizer type.');
+    }
+    if (!config.image && !config.video) {
+      throw new Error('Either "image" or "video" must be specified as the background source.');
+    }
+    if (config.image && config.video) {
+      throw new Error('Cannot use both "image" and "video" options. Please specify only one background source.');
+    }
+
     const audioFilePath = getAudioFilePath(config);
-    const backgroundImagePath = getBackgroundImagePath(config);
     const outVideoPath = getOutVideoPath(config);
+    const backgroundVideoPath = getBackgroundVideoPath(config);
+    const backgroundImagePath = getBackgroundImagePath(config);
+    const useVideoBackground = !!backgroundVideoPath;
 
-    const backgroundImageBmpBuffer = await convertToBmp(backgroundImagePath);
-    const backgroundImage = parseImage(backgroundImageBmpBuffer);
     const audioReader = await createAudioBuffer(audioFilePath, FFMPEG_FORMAT);
     const audioBuffer = audioReader.audioBuffer;
     const sampleRate = audioReader.sampleRate;
@@ -91,29 +359,39 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
 
     const spectrumBusMargin = getSpectrumBusMargin();
     const FPS = getFPS(config);
-    const spectrumWidth =
-      getSpectrumWidthAbsolute(config, backgroundImage.width);
-    const spectrumHeight =
-      getSpectrumHeightAbsolute(config, backgroundImage.height);
-    const spectrumX =
-      getSpectrumXAbsolute(config, spectrumWidth, backgroundImage.width);
-    const spectrumY =
-      getSpectrumYAbsolute(config, spectrumHeight, backgroundImage.height);
-    const spectrumRotation =
-      getSpectrumRotation(config);
-    const spectrumColor =
-      getSpectrumColor(config) ||
-      invertColor(getImageColor(backgroundImage));
-    const spectrumEffect = getSpectrumEffect(config);
-    const spectrumOpacity = getSpectrumOpacityParsed(config);
     const ffmpeg_cfr = getFfmpeg_cfr(config);
     const ffmpeg_preset = getFfmpeg_preset(config);
     const frame_processing_delay = getFrame_processing_delay(config);
 
     const audioDuration = audioBuffer.length / sampleRate;
     const framesCount = Math.trunc(audioDuration * FPS);
-    const audioDataStep = Math.trunc(audioBuffer.length / framesCount);
-    const processingBuffer = new Float32Array(PROCESSING_BUFFER_SIZE).fill(0);
+    const outputResolution = getOutputResolution(config);
+
+    const preprocessed = preProcessAudio(audioBuffer, sampleRate, FPS, framesCount);
+
+    const {
+      backgroundWidth,
+      backgroundHeight,
+      defaultColor,
+      staticBackgroundBuffer,
+      videoFrameReader,
+      videoFrameSize,
+      encodeVideoFrame,
+      concatFilePath,
+    } = await prepareBackgroundForRender({
+      useVideoBackground,
+      backgroundVideoPath,
+      backgroundImagePath,
+      beatFrameIndices: preprocessed.beatFrameIndices,
+      framesCount,
+      outputResolution,
+      fps: FPS,
+      autoEditVideo: getAutoEditVideo(config),
+    });
+
+    const createVisualizerFrame = createVisualizerFrameGenerator(
+      config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
+    );
 
     const ffmpegVideoWriter = spawnFfmpegVideoWriter({
       audioFilename: audioFilePath,
@@ -125,29 +403,23 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
     });
     ffmpegVideoWriter.on('exit', (code: number) => resolve(code));
 
-    const bpmEncoder = createBpmEncoder({ width: backgroundImage.width, height: backgroundImage.height });
-    const backgroundImageBuffer = bpmEncoder(backgroundImage.data);
-    const skipFramesCount = FPS < 45 ? 1 : 2;
-    const processSpectrum = createSpectrumsProcessor(sampleRate, skipFramesCount);
-    const createVisualizerFrame = createVisualizerFrameGenerator();
     for (let i = 0; i < framesCount; i++) {
-      const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
-      processingBuffer.copyWithin(0, currentFrameData.length);
-      processingBuffer.set(currentFrameData, PROCESSING_BUFFER_SIZE - currentFrameData.length);
+      const spectrum = preprocessed.spectrums[i];
+
+      const backgroundImageBuffer = await resolveBackgroundFrameBuffer({
+        frameIndex: i,
+        useVideoBackground,
+        videoFrameReader,
+        videoFrameSize,
+        encodeVideoFrame,
+        staticBackgroundBuffer,
+      });
 
-      const audioDataParser = () => Array.from(processingBuffer);
-      const spectrum = processSpectrum(audioDataParser);
-      const frameImage = createVisualizerFrame({
+      const commonVisualizerFrameProps: CommonVisualizerFrameProps = {
         backgroundImageBuffer,
         spectrum,
-        size: { width: spectrumWidth, height: spectrumHeight },
-        position: { x: spectrumX, y: spectrumY },
-        rotation: spectrumRotation,
-        margin: spectrumBusMargin,
-        color: spectrumColor,
-        opacity: spectrumOpacity,
-        spectrumEffect,
-      });
+      };
+      const frameImage = createVisualizerFrame(commonVisualizerFrameProps);
       const isFrameProcessed = ffmpegVideoWriter.stdin.write(frameImage.data);
       if (!isFrameProcessed) {
         await waitDrain(ffmpegVideoWriter.stdin);
@@ -160,5 +432,11 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
       }
     }
 
+    if (videoFrameReader) {
+      videoFrameReader.kill();
+    }
+    if (concatFilePath) {
+      cleanupConcatFile(concatFilePath);
+    }
     ffmpegVideoWriter.stdin.end();
   });

package/src/tests/image.test.ts

@@ -3,7 +3,8 @@ import path from 'path';
 import {
   drawRect,
   Color,
-  createVisualizerFrameGenerator,
+  createSpectrumVisualizerFrameGenerator,
+  createPolarVisualizerFrameGenerator,
   parseImage,
   convertToBmp,
   mixValues,
@@ -76,14 +77,14 @@ describe('image', function () {
     expect(imageDstBuffer.data).deep.equal(expectedImageData);
   });
 
-  it('createVisualizerFrame', async function () {
+  it('createSpectrumVisualizerFrameGenerator', async function () {
     const expectedImageLength = 6220854;
     const backgroundImagePath = path.resolve('example/media/horses.png');
     const backgroundImageBmpBuffer = await convertToBmp(backgroundImagePath);
     const backgroundImage = parseImage(backgroundImageBmpBuffer);
     const bpmEncoder = createBpmEncoder({ width: backgroundImage.width, height: backgroundImage.height });
     const backgroundImageBuffer = bpmEncoder(backgroundImage.data);
-    const createVisualizerFrame = createVisualizerFrameGenerator();
+    const createVisualizerFrame = createSpectrumVisualizerFrameGenerator();
     const frameSpectrumDown = createVisualizerFrame({
       backgroundImageBuffer,
       spectrum: [0.5, 0, 1],
@@ -136,4 +137,86 @@ describe('image', function () {
     const resultSpectrumUpOpacity50 = frameSpectrumUpOpacity50.data.toJSON().data;
     expect(resultSpectrumUpOpacity50).to.have.length(expectedImageLength);
   });
+
+  it('createPolarVisualizerFrameGenerator', async function () {
+    const expectedImageLength = 6220854;
+    const backgroundImagePath = path.resolve('example/media/horses.png');
+    const backgroundImageBmpBuffer = await convertToBmp(backgroundImagePath);
+    const backgroundImage = parseImage(backgroundImageBmpBuffer);
+    const bpmEncoder = createBpmEncoder({ width: backgroundImage.width, height: backgroundImage.height });
+    const backgroundImageBuffer = bpmEncoder(backgroundImage.data);
+    const createPolarVisualizerFrame = createPolarVisualizerFrameGenerator();
+
+    const framePolarBasic = createPolarVisualizerFrame({
+      backgroundImageBuffer,
+      spectrum: [0.5, 0.8, 0.3, 0.9, 0.6, 0.4, 0.7, 0.2],
+      centerX: 500,
+      centerY: 400,
+      innerRadius: 100,
+      maxBarLength: 150,
+      barWidth: 5,
+      color: { red: 255, green: 100, blue: 50 },
+      opacity: 1,
+    });
+    const resultPolarBasic = framePolarBasic.data.toJSON().data;
+    expect(resultPolarBasic).to.have.length(expectedImageLength);
+
+    const framePolarVolume = createPolarVisualizerFrame({
+      backgroundImageBuffer,
+      spectrum: [0.2, 0.5, 0.8, 0.3, 0.6, 0.9, 0.4, 0.7],
+      centerX: 500,
+      centerY: 400,
+      innerRadius: 80,
+      maxBarLength: 120,
+      barWidth: 4,
+      color: { red: 100, green: 200, blue: 255 },
+      opacity: 1,
+      spectrumEffect: 'volume',
+    });
+    const resultPolarVolume = framePolarVolume.data.toJSON().data;
+    expect(resultPolarVolume).to.have.length(expectedImageLength);
+
+    const framePolarSmooth = createPolarVisualizerFrame({
+      backgroundImageBuffer,
+      spectrum: [0.7, 0.4, 0.9, 0.2, 0.5, 0.8, 0.3, 0.6],
+      centerX: 500,
+      centerY: 400,
+      innerRadius: 90,
+      maxBarLength: 130,
+      barWidth: 6,
+      color: { red: 255, green: 255, blue: 0 },
+      opacity: 1,
+      spectrumEffect: 'smooth',
+    });
+    const resultPolarSmooth = framePolarSmooth.data.toJSON().data;
+    expect(resultPolarSmooth).to.have.length(expectedImageLength);
+
+    const framePolarOpacity50 = createPolarVisualizerFrame({
+      backgroundImageBuffer,
+      spectrum: [0.6, 0.3, 0.8, 0.5, 0.4, 0.7, 0.2, 0.9],
+      centerX: 500,
+      centerY: 400,
+      innerRadius: 100,
+      maxBarLength: 140,
+      barWidth: 5,
+      color: { red: 0, green: 255, blue: 128 },
+      opacity: 0.5,
+    });
+    const resultPolarOpacity50 = framePolarOpacity50.data.toJSON().data;
+    expect(resultPolarOpacity50).to.have.length(expectedImageLength);
+
+    const framePolarHexColor = createPolarVisualizerFrame({
+      backgroundImageBuffer,
+      spectrum: [0.5, 0.6, 0.7, 0.8, 0.9, 0.4, 0.3, 0.2],
+      centerX: 500,
+      centerY: 400,
+      innerRadius: 110,
+      maxBarLength: 160,
+      barWidth: 7,
+      color: '#FF5733',
+      opacity: 1,
+    });
+    const resultPolarHexColor = framePolarHexColor.data.toJSON().data;
+    expect(resultPolarHexColor).to.have.length(expectedImageLength);
+  });
 });