nodejs-audio-visualizer 4.3.6 → 4.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +75 -75
  3. package/dist/audio.d.ts +12 -12
  4. package/dist/audio.js +120 -120
  5. package/dist/beats.d.ts +19 -19
  6. package/dist/beats.js +43 -43
  7. package/dist/bpmEncoder.d.ts +14 -14
  8. package/dist/bpmEncoder.js +97 -97
  9. package/dist/config.d.ts +67 -67
  10. package/dist/config.js +356 -356
  11. package/dist/dsp.d.ts +16 -16
  12. package/dist/dsp.js +76 -76
  13. package/dist/highlight.d.ts +32 -32
  14. package/dist/highlight.js +325 -325
  15. package/dist/image.d.ts +79 -79
  16. package/dist/image.js +458 -458
  17. package/dist/index.d.ts +72 -72
  18. package/dist/index.js +536 -533
  19. package/dist/index.js.map +1 -1
  20. package/dist/subtitleConvert.d.ts +3 -3
  21. package/dist/subtitleConvert.js +61 -61
  22. package/dist/video.d.ts +72 -72
  23. package/dist/video.js +338 -338
  24. package/dist/waitForEventLoop.d.ts +1 -1
  25. package/dist/waitForEventLoop.js +5 -5
  26. package/package.json +61 -61
  27. package/src/audio.ts +120 -120
  28. package/src/beats.ts +68 -68
  29. package/src/bpmEncoder.ts +97 -97
  30. package/src/config.ts +462 -462
  31. package/src/dsp.ts +95 -95
  32. package/src/highlight.ts +284 -284
  33. package/src/image.ts +602 -602
  34. package/src/index.ts +678 -674
  35. package/src/subtitleConvert.ts +63 -63
  36. package/src/tests/audio.test.ts +96 -96
  37. package/src/tests/config.test.ts +475 -475
  38. package/src/tests/dsp.test.ts +24 -24
  39. package/src/tests/highlight.test.ts +178 -178
  40. package/src/tests/image.test.ts +222 -222
  41. package/src/tests/index.test.ts +63 -63
  42. package/src/tests/subtitleConvert.test.ts +23 -23
  43. package/src/tests/video.test.ts +138 -138
  44. package/src/typings.d.ts +6 -6
  45. package/src/video.ts +437 -437
  46. package/src/waitForEventLoop.ts +2 -2
package/src/index.ts CHANGED
@@ -1,674 +1,678 @@
1
- import path from 'path';
2
- import { readFileSync } from 'fs';
3
- import {
4
- getAudioFilePath,
5
- getBackgroundImagePath,
6
- getBackgroundVideoPath,
7
- getOutVideoPath,
8
- getSubtitleRenderSpec,
9
- subtitleAlignmentToAss,
10
- getFPS,
11
- getSpectrumBusMargin,
12
- getSpectrumWidthAbsolute,
13
- getSpectrumHeightAbsolute,
14
- getSpectrumXAbsolute,
15
- getSpectrumYAbsolute,
16
- getSpectrumColor,
17
- getSpectrumOpacityParsed,
18
- getFfmpeg_cfr,
19
- getFfmpeg_preset,
20
- getFrame_processing_delay,
21
- getOutputResolution,
22
- rotationAliasValues,
23
- getSpectrumRotation,
24
- getSpectrumEffect,
25
- SpectrumEffect,
26
- getPolarXAbsolute,
27
- getPolarYAbsolute,
28
- getPolarInnerRadius,
29
- getPolarMaxBarLength,
30
- getPolarBarWidth,
31
- getPolarEffect,
32
- getPolarColor,
33
- getPolarOpacityParsed,
34
- getAutoEditVideo,
35
- getAudioAutoHighlight,
36
- getAudioAutoHighlightCount,
37
- } from './config';
38
- import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor } from './audio';
39
- import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps } from './image';
40
- import { normalizeInlineSubtitlesToSrt, lrcToSrt } from './subtitleConvert';
41
- import { spawnFfmpegVideoWriter, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile } from './video';
42
- import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
43
- import { createBeatDetector } from './beats';
44
- export { BeatInfo, BeatDetectorOptions } from './beats';
45
- import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
46
- import { waitForEventLoop } from './waitForEventLoop';
47
- export { computeHighlightSlice, HIGHLIGHT_DURATION_SEC, BeatFrameEvent, HighlightAudioSegment, HighlightRun } from './highlight';
48
-
49
- export const PCM_FORMAT = {
50
- bit: 8,
51
- sign: 'u',
52
- parseFunction: bufferToUInt8
53
- };
54
- const FFMPEG_FORMAT = `${PCM_FORMAT.sign}${PCM_FORMAT.bit}`;
55
- const PROCESSING_BUFFER_SIZE = Math.pow(2, 12);
56
-
57
- export interface Config {
58
- audio: {
59
- path: string;
60
- autoHighlight?: boolean;
61
- /** When `autoHighlight` is true, number of non-overlapping 15s windows to stitch (default 1). */
62
- autoHighlightCount?: number;
63
- };
64
- image?: {
65
- path: string;
66
- };
67
- video?: {
68
- path: string;
69
- autoEdit?: boolean;
70
- };
71
- outVideo: {
72
- path: string;
73
- subtitles?:
74
- | string
75
- | {
76
- path?: string;
77
- rawContent?: string;
78
- alignment?: 'top' | 'middle' | 'bottom';
79
- };
80
- fps?: number;
81
- resolution?: {
82
- width: number;
83
- height: number;
84
- };
85
- spectrum?: {
86
- width?: SpectrumSizeValue;
87
- height?: SpectrumSizeValue;
88
- x?: number | PositionAliasName;
89
- y?: number | PositionAliasName;
90
- rotation?: RotationAliasName;
91
- effect?: SpectrumEffect;
92
- color?: Color | string;
93
- opacity?: string;
94
- };
95
- polar?: {
96
- x?: number | PositionAliasName;
97
- y?: number | PositionAliasName;
98
- innerRadius?: number;
99
- maxBarLength?: number;
100
- barWidth?: number;
101
- effect?: SpectrumEffect;
102
- color?: Color | string;
103
- opacity?: string;
104
- }
105
- };
106
- tweaks?: {
107
- ffmpeg_cfr?: string;
108
- ffmpeg_preset?: string;
109
- frame_processing_delay?: number;
110
- };
111
- }
112
-
113
- export type SpectrumSizeValue = number | string;
114
-
115
- export type PositionAliasName =
116
- 'left' |
117
- 'center' |
118
- 'right' |
119
- 'top' |
120
- 'middle' |
121
- 'bottom';
122
-
123
- export type RotationAliasName = typeof rotationAliasValues[number];
124
-
125
- const sleep = (timeout: number) =>
126
- new Promise(resolve => setTimeout(resolve, timeout));
127
-
128
- const createVisualizerFrameGenerator = (
129
- config: Config,
130
- backgroundWidth: number,
131
- backgroundHeight: number,
132
- defaultColor: Color,
133
- spectrumBusMargin: number
134
- ): (params: CommonVisualizerFrameProps) => EncodedBmp => {
135
- if (!config.outVideo.spectrum) {
136
- const createPolarVisualizerFrame = createPolarVisualizerFrameGenerator();
137
- const polarX = getPolarXAbsolute(config, backgroundWidth);
138
- const polarY = getPolarYAbsolute(config, backgroundHeight);
139
- const polarInnerRadius = getPolarInnerRadius(config);
140
- const polarMaxBarLength = getPolarMaxBarLength(config);
141
- const polarBarWidth = getPolarBarWidth(config);
142
- const polarColor = getPolarColor(config) || invertColor(defaultColor);
143
- const polarEffect = getPolarEffect(config);
144
- const polarOpacity = getPolarOpacityParsed(config);
145
-
146
- return (params: CommonVisualizerFrameProps) => {
147
- return createPolarVisualizerFrame({
148
- ...params,
149
- centerX: polarX,
150
- centerY: polarY,
151
- innerRadius: polarInnerRadius,
152
- maxBarLength: polarMaxBarLength,
153
- barWidth: polarBarWidth,
154
- color: polarColor,
155
- opacity: polarOpacity,
156
- spectrumEffect: polarEffect,
157
- });
158
- };
159
- }
160
- const createSpectrumVisualizerFrame = createSpectrumVisualizerFrameGenerator();
161
- const spectrumWidth = getSpectrumWidthAbsolute(config, backgroundWidth);
162
- const spectrumHeight = getSpectrumHeightAbsolute(config, backgroundHeight);
163
- const spectrumX = getSpectrumXAbsolute(config, spectrumWidth, backgroundWidth);
164
- const spectrumY = getSpectrumYAbsolute(config, spectrumHeight, backgroundHeight);
165
- const spectrumRotation = getSpectrumRotation(config);
166
- const spectrumColor = getSpectrumColor(config) || invertColor(defaultColor);
167
- const spectrumEffect = getSpectrumEffect(config);
168
- const spectrumOpacity = getSpectrumOpacityParsed(config);
169
-
170
- return (params: CommonVisualizerFrameProps) => {
171
- return createSpectrumVisualizerFrame({
172
- ...params,
173
- size: { width: spectrumWidth, height: spectrumHeight },
174
- position: { x: spectrumX, y: spectrumY },
175
- rotation: spectrumRotation,
176
- margin: spectrumBusMargin,
177
- color: spectrumColor,
178
- opacity: spectrumOpacity,
179
- spectrumEffect,
180
- });
181
- };
182
- };
183
-
184
- interface PreProcessedAudio {
185
- spectrums: number[][];
186
- beatFrameIndices: number[];
187
- beatEvents: { frameIndex: number; intensity: number }[];
188
- }
189
-
190
- const PRE_PROCESS_PROGRESS_SHARE = 60;
191
- const POST_AUDIO_PROGRESS_SHARE = 10;
192
- const RENDER_PROGRESS_SHARE = 30;
193
-
194
- const preProcessAudio = async (
195
- audioBuffer: Buffer,
196
- sampleRate: number,
197
- fps: number,
198
- framesCount: number,
199
- onPreProcessProgress?: (totalPercent: number) => void,
200
- ): Promise<PreProcessedAudio> => {
201
- if (framesCount === 0) {
202
- for (let t = 1; t <= PRE_PROCESS_PROGRESS_SHARE; t += 1) {
203
- onPreProcessProgress?.(t);
204
- }
205
- return { spectrums: [], beatFrameIndices: [], beatEvents: [] };
206
- }
207
- const audioDataStep = Math.trunc(audioBuffer.length / framesCount);
208
- const processingBuffer = new Float32Array(PROCESSING_BUFFER_SIZE).fill(0);
209
- const skipFramesCount = fps < 45 ? 1 : 2;
210
- const processSpectrum = createSpectrumsProcessor(sampleRate, skipFramesCount);
211
- const detectBeat = createBeatDetector(fps);
212
-
213
- const spectrums: number[][] = [];
214
- const beatFrameIndices: number[] = [];
215
- const beatEvents: { frameIndex: number; intensity: number }[] = [];
216
- let nextPreProcessMilestone = 1;
217
-
218
- for (let i = 0; i < framesCount; i++) {
219
- const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
220
- processingBuffer.copyWithin(0, currentFrameData.length);
221
- processingBuffer.set(currentFrameData, PROCESSING_BUFFER_SIZE - currentFrameData.length);
222
-
223
- const audioDataParser = () => Array.from(processingBuffer);
224
- const spectrum = processSpectrum(audioDataParser);
225
- const beat = detectBeat(spectrum);
226
-
227
- spectrums.push(spectrum);
228
- if (beat.isBeat) {
229
- beatFrameIndices.push(i);
230
- beatEvents.push({ frameIndex: i, intensity: beat.intensity });
231
- }
232
-
233
- const p =
234
- PRE_PROCESS_PROGRESS_SHARE * (i + 1) / framesCount;
235
- while (
236
- nextPreProcessMilestone <= PRE_PROCESS_PROGRESS_SHARE
237
- && p >= nextPreProcessMilestone
238
- ) {
239
- onPreProcessProgress?.(nextPreProcessMilestone);
240
- nextPreProcessMilestone += 1;
241
- }
242
-
243
- await waitForEventLoop();
244
- }
245
-
246
- return { spectrums, beatFrameIndices, beatEvents };
247
- };
248
-
249
- async function prepareBackgroundForRender(params: {
250
- useVideoBackground: boolean;
251
- backgroundVideoPath: string | undefined;
252
- backgroundImagePath: string | undefined;
253
- beatFrameIndices: number[];
254
- framesCount: number;
255
- outputResolution: ReturnType<typeof getOutputResolution>;
256
- fps: number;
257
- autoEditVideo: boolean;
258
- }): Promise<{
259
- backgroundWidth: number;
260
- backgroundHeight: number;
261
- defaultColor: Color;
262
- staticBackgroundBuffer: EncodedBmp;
263
- videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
264
- videoFrameSize: number;
265
- encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
266
- concatFilePath?: string;
267
- }> {
268
- const {
269
- useVideoBackground,
270
- backgroundVideoPath,
271
- backgroundImagePath,
272
- beatFrameIndices,
273
- framesCount,
274
- outputResolution,
275
- fps,
276
- autoEditVideo,
277
- } = params;
278
-
279
- if (useVideoBackground && backgroundVideoPath) {
280
- const videoInfo = await getVideoInfo(backgroundVideoPath);
281
- const sceneChanges = autoEditVideo
282
- ? await detectSceneChanges(backgroundVideoPath)
283
- : [];
284
- const backgroundWidth = outputResolution?.width ?? videoInfo.width;
285
- const backgroundHeight = outputResolution?.height ?? videoInfo.height;
286
-
287
- const videoFrameSize = backgroundWidth * backgroundHeight * 3;
288
- const encodeVideoFrame = createBgrFrameEncoder({ width: backgroundWidth, height: backgroundHeight });
289
-
290
- const segments = buildBeatSyncedSegments(
291
- beatFrameIndices,
292
- framesCount,
293
- sceneChanges,
294
- videoInfo.duration,
295
- );
296
-
297
- const concatFilePath = writeConcatFile(segments, backgroundVideoPath, videoInfo.duration, fps);
298
-
299
- const videoFrameReader = spawnConcatVideoFrameReader({
300
- concatFilePath,
301
- fps,
302
- totalFrames: framesCount,
303
- ...(outputResolution && {
304
- width: backgroundWidth,
305
- height: backgroundHeight,
306
- sourceWidth: videoInfo.width,
307
- sourceHeight: videoInfo.height,
308
- }),
309
- });
310
-
311
- const firstFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
312
- if (!firstFrame) {
313
- throw new Error(`Could not read frames from video: ${backgroundVideoPath}`);
314
- }
315
- const defaultColor = getVideoFrameColor(firstFrame, backgroundWidth, backgroundHeight);
316
- const staticBackgroundBuffer = encodeVideoFrame(firstFrame);
317
- return {
318
- backgroundWidth,
319
- backgroundHeight,
320
- defaultColor,
321
- staticBackgroundBuffer,
322
- videoFrameReader,
323
- videoFrameSize,
324
- encodeVideoFrame,
325
- concatFilePath,
326
- };
327
- }
328
-
329
- if (!backgroundImagePath) {
330
- throw new Error('Background image path is required when not using video background.');
331
- }
332
- const backgroundImageBmpBuffer = await convertToBmp(
333
- backgroundImagePath,
334
- outputResolution?.width,
335
- outputResolution?.height,
336
- );
337
- const backgroundImage = parseImage(backgroundImageBmpBuffer);
338
- const backgroundWidth = backgroundImage.width;
339
- const backgroundHeight = backgroundImage.height;
340
- const defaultColor = getImageColor(backgroundImage);
341
-
342
- const bpmEncoder = createBpmEncoder({ width: backgroundWidth, height: backgroundHeight });
343
- const staticBackgroundBuffer = bpmEncoder(backgroundImage.data);
344
- return {
345
- backgroundWidth,
346
- backgroundHeight,
347
- defaultColor,
348
- staticBackgroundBuffer,
349
- videoFrameSize: 0,
350
- };
351
- }
352
-
353
- async function resolveBackgroundFrameBuffer(params: {
354
- frameIndex: number;
355
- useVideoBackground: boolean;
356
- videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
357
- videoFrameSize: number;
358
- encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
359
- staticBackgroundBuffer: EncodedBmp;
360
- }): Promise<EncodedBmp> {
361
- const {
362
- frameIndex,
363
- useVideoBackground,
364
- videoFrameReader,
365
- videoFrameSize,
366
- encodeVideoFrame,
367
- staticBackgroundBuffer,
368
- } = params;
369
-
370
- if (useVideoBackground && videoFrameReader && encodeVideoFrame) {
371
- if (frameIndex === 0) {
372
- return staticBackgroundBuffer;
373
- }
374
- const videoFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
375
- return videoFrame
376
- ? encodeVideoFrame(videoFrame)
377
- : staticBackgroundBuffer;
378
- }
379
- return staticBackgroundBuffer;
380
- }
381
-
382
- export interface RenderAudioVisualizerResult {
383
- exitCode: number;
384
- /** Absolute paths of video files written successfully (exit code 0, full pass, no early stop). */
385
- outputVideoFiles: string[];
386
- }
387
-
388
- export const renderAudioVisualizer = (config: Config, onProgress?: (progress: number) => any, shouldStop?: () => boolean) =>
389
- new Promise<RenderAudioVisualizerResult>(async (resolve) => {
390
- if (config.outVideo.spectrum && config.outVideo.polar) {
391
- throw new Error('Cannot use both "spectrum" and "polar" options. Please specify only one visualizer type.');
392
- }
393
- if (!config.image && !config.video) {
394
- throw new Error('Either "image" or "video" must be specified as the background source.');
395
- }
396
- if (config.image && config.video) {
397
- throw new Error('Cannot use both "image" and "video" options. Please specify only one background source.');
398
- }
399
-
400
- const audioFilePath = getAudioFilePath(config);
401
- const outVideoPath = getOutVideoPath(config);
402
- const subtitleSpec = getSubtitleRenderSpec(config);
403
- let subtitleFilePath: string | undefined;
404
- let subtitleFileIsTemporary = false;
405
- let subtitleAlignmentAss = 2;
406
- if (subtitleSpec) {
407
- subtitleAlignmentAss = subtitleAlignmentToAss(subtitleSpec.alignment);
408
- if (subtitleSpec.source.kind === 'file') {
409
- const absPath = subtitleSpec.source.path;
410
- const ext = path.extname(absPath).toLowerCase();
411
- if (ext === '.lrc') {
412
- const raw = readFileSync(absPath, 'utf-8');
413
- subtitleFilePath = writeSubtitlesFile(lrcToSrt(raw));
414
- subtitleFileIsTemporary = true;
415
- } else {
416
- subtitleFilePath = absPath;
417
- }
418
- } else {
419
- subtitleFilePath = writeSubtitlesFile(
420
- normalizeInlineSubtitlesToSrt(subtitleSpec.source.text),
421
- );
422
- subtitleFileIsTemporary = true;
423
- }
424
- }
425
- const backgroundVideoPath = getBackgroundVideoPath(config);
426
- const backgroundImagePath = getBackgroundImagePath(config);
427
- const useVideoBackground = !!backgroundVideoPath;
428
-
429
- const audioReader = await createAudioBuffer(audioFilePath, FFMPEG_FORMAT);
430
- const audioBuffer = audioReader.audioBuffer;
431
- const sampleRate = audioReader.sampleRate;
432
- if (!sampleRate) {
433
- throw new Error('ffmpeg didn\'t show audio sample rate');
434
- }
435
-
436
- const spectrumBusMargin = getSpectrumBusMargin();
437
- const FPS = getFPS(config);
438
- const ffmpeg_cfr = getFfmpeg_cfr(config);
439
- const ffmpeg_preset = getFfmpeg_preset(config);
440
- const frame_processing_delay = getFrame_processing_delay(config);
441
-
442
- const audioDuration = audioBuffer.length / sampleRate;
443
- const framesCount = Math.trunc(audioDuration * FPS);
444
- const outputResolution = getOutputResolution(config);
445
-
446
- let maxProgressReported = -1;
447
- const reportProgress = (progress: number) => {
448
- if (!onProgress) {
449
- return;
450
- }
451
- const normalized = +Math.min(100, Math.max(0, progress)).toFixed(2);
452
- if (normalized <= maxProgressReported) {
453
- return;
454
- }
455
- maxProgressReported = normalized;
456
- onProgress(normalized);
457
- };
458
- reportProgress(0);
459
-
460
- const preprocessed = await preProcessAudio(
461
- audioBuffer,
462
- sampleRate,
463
- FPS,
464
- framesCount,
465
- (progress: number) => {
466
- reportProgress(progress);
467
- },
468
- );
469
- reportProgress(PRE_PROCESS_PROGRESS_SHARE);
470
-
471
- const autoHighlight = getAudioAutoHighlight(config);
472
- let spectrumsForRender = preprocessed.spectrums;
473
- let beatIndicesForRender = preprocessed.beatFrameIndices;
474
- let framesCountForRender = framesCount;
475
- type HighlightSliceResult = ReturnType<typeof computeHighlightSlice> extends Promise<
476
- infer R
477
- >
478
- ? R
479
- : never;
480
- let highlightSlice: HighlightSliceResult | undefined;
481
-
482
- if (autoHighlight) {
483
- reportProgress(PRE_PROCESS_PROGRESS_SHARE + 1);
484
- highlightSlice = await computeHighlightSlice(
485
- FPS,
486
- framesCount,
487
- preprocessed.spectrums,
488
- preprocessed.beatEvents,
489
- getAudioAutoHighlightCount(config),
490
- );
491
- spectrumsForRender = highlightSlice.spectrums;
492
- beatIndicesForRender = highlightSlice.beatFrameIndices;
493
- framesCountForRender = highlightSlice.highlightFrames;
494
- reportProgress(PRE_PROCESS_PROGRESS_SHARE + POST_AUDIO_PROGRESS_SHARE);
495
- } else {
496
- reportProgress(PRE_PROCESS_PROGRESS_SHARE + POST_AUDIO_PROGRESS_SHARE);
497
- }
498
-
499
- const separateHighlightFiles =
500
- autoHighlight &&
501
- highlightSlice &&
502
- highlightSlice.runs.length > 1;
503
-
504
- type VideoRenderPass = {
505
- outPath: string;
506
- spectrums: number[][];
507
- beatIndices: number[];
508
- frameCount: number;
509
- audioSegment: import('./highlight').HighlightAudioSegment | undefined;
510
- };
511
-
512
- const passes: VideoRenderPass[] =
513
- separateHighlightFiles && highlightSlice
514
- ? highlightSlice.runs.map((run, i) => {
515
- const dir = path.dirname(outVideoPath);
516
- const ext = path.extname(outVideoPath);
517
- const base = path.basename(outVideoPath, ext);
518
- const numberedPath = path.resolve(dir, `${base}-${i + 1}${ext}`);
519
- return {
520
- outPath: numberedPath,
521
- spectrums: run.spectrums,
522
- beatIndices: run.beatFrameIndices,
523
- frameCount: run.highlightFrames,
524
- audioSegment: run.audioSegment,
525
- };
526
- })
527
- : [
528
- {
529
- outPath: outVideoPath,
530
- spectrums: spectrumsForRender,
531
- beatIndices: beatIndicesForRender,
532
- frameCount: framesCountForRender,
533
- audioSegment:
534
- autoHighlight &&
535
- highlightSlice &&
536
- highlightSlice.audioSegments.length > 0
537
- ? highlightSlice.audioSegments[0]
538
- : undefined,
539
- },
540
- ];
541
-
542
- const totalPassFrames =
543
- passes.reduce((sum, pass) => sum + pass.frameCount, 0);
544
- const totalRenderMilestones = totalPassFrames + (passes.length * 2);
545
- let renderMilestonesCompleted = 0;
546
- const reportRenderProgress = (milestoneIncrement = 1) => {
547
- renderMilestonesCompleted += milestoneIncrement;
548
- reportProgress(
549
- PRE_PROCESS_PROGRESS_SHARE
550
- + POST_AUDIO_PROGRESS_SHARE
551
- + (RENDER_PROGRESS_SHARE * renderMilestonesCompleted / Math.max(1, totalRenderMilestones)),
552
- );
553
- };
554
- let lastExitCode = 0;
555
- const outputVideoFiles: string[] = [];
556
- try {
557
- passLoop: for (const pass of passes) {
558
- const {
559
- backgroundWidth,
560
- backgroundHeight,
561
- defaultColor,
562
- staticBackgroundBuffer,
563
- videoFrameReader,
564
- videoFrameSize,
565
- encodeVideoFrame,
566
- concatFilePath,
567
- } = await prepareBackgroundForRender({
568
- useVideoBackground,
569
- backgroundVideoPath,
570
- backgroundImagePath,
571
- beatFrameIndices: pass.beatIndices,
572
- framesCount: pass.frameCount,
573
- outputResolution,
574
- fps: FPS,
575
- autoEditVideo: getAutoEditVideo(config),
576
- });
577
- reportRenderProgress();
578
-
579
- const createVisualizerFrame = createVisualizerFrameGenerator(
580
- config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
581
- );
582
-
583
- const ffmpegVideoWriter = spawnFfmpegVideoWriter({
584
- audioFilename: audioFilePath,
585
- videoFileName: pass.outPath,
586
- ...(subtitleFilePath && {
587
- subtitleFilename: subtitleFilePath,
588
- subtitleAlignmentAss,
589
- }),
590
- fps: FPS,
591
- ...(pass.audioSegment && { audioSegment: pass.audioSegment }),
592
- ...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
593
- ...(ffmpeg_preset && { preset: ffmpeg_preset }),
594
- });
595
-
596
- const exitPromise = new Promise<number>(res => {
597
- let settled = false;
598
- ffmpegVideoWriter.on('exit', (code: number | null) => {
599
- if (settled) {
600
- return;
601
- }
602
- settled = true;
603
- res(code ?? 0);
604
- });
605
- });
606
-
607
- let stoppedEarly = false;
608
- for (let i = 0; i < pass.frameCount; i++) {
609
- const spectrum = pass.spectrums[i];
610
-
611
- const backgroundImageBuffer = await resolveBackgroundFrameBuffer({
612
- frameIndex: i,
613
- useVideoBackground,
614
- videoFrameReader,
615
- videoFrameSize,
616
- encodeVideoFrame,
617
- staticBackgroundBuffer,
618
- });
619
-
620
- const commonVisualizerFrameProps: CommonVisualizerFrameProps = {
621
- backgroundImageBuffer,
622
- spectrum,
623
- };
624
- const frameImage = createVisualizerFrame(commonVisualizerFrameProps);
625
- const isFrameProcessed = ffmpegVideoWriter.stdin.write(frameImage.data);
626
- if (!isFrameProcessed) {
627
- const isDrained = await waitDrain(ffmpegVideoWriter.stdin, ffmpegVideoWriter);
628
- if (!isDrained) {
629
- stoppedEarly = true;
630
- break;
631
- }
632
- }
633
- if (shouldStop && shouldStop()) {
634
- stoppedEarly = true;
635
- break;
636
- }
637
- if (frame_processing_delay) {
638
- await sleep(frame_processing_delay);
639
- }
640
- await waitForEventLoop();
641
- reportRenderProgress();
642
- }
643
-
644
- if (videoFrameReader) {
645
- videoFrameReader.kill();
646
- }
647
- if (concatFilePath) {
648
- cleanupConcatFile(concatFilePath);
649
- }
650
- ffmpegVideoWriter.stdin.end();
651
-
652
- lastExitCode = await exitPromise;
653
- reportRenderProgress();
654
-
655
- if (lastExitCode === 0 && !stoppedEarly) {
656
- outputVideoFiles.push(pass.outPath);
657
- }
658
-
659
- if (stoppedEarly || lastExitCode !== 0) {
660
- break passLoop;
661
- }
662
- }
663
-
664
- if (lastExitCode === 0 && outputVideoFiles.length === passes.length) {
665
- reportProgress(100);
666
- }
667
-
668
- resolve({ exitCode: lastExitCode, outputVideoFiles });
669
- } finally {
670
- if (subtitleFilePath && subtitleFileIsTemporary) {
671
- cleanupTempFile(subtitleFilePath);
672
- }
673
- }
674
- });
1
+ import path from 'path';
2
+ import { readFileSync } from 'fs';
3
+ import {
4
+ getAudioFilePath,
5
+ getBackgroundImagePath,
6
+ getBackgroundVideoPath,
7
+ getOutVideoPath,
8
+ getSubtitleRenderSpec,
9
+ subtitleAlignmentToAss,
10
+ getFPS,
11
+ getSpectrumBusMargin,
12
+ getSpectrumWidthAbsolute,
13
+ getSpectrumHeightAbsolute,
14
+ getSpectrumXAbsolute,
15
+ getSpectrumYAbsolute,
16
+ getSpectrumColor,
17
+ getSpectrumOpacityParsed,
18
+ getFfmpeg_cfr,
19
+ getFfmpeg_preset,
20
+ getFrame_processing_delay,
21
+ getOutputResolution,
22
+ rotationAliasValues,
23
+ getSpectrumRotation,
24
+ getSpectrumEffect,
25
+ SpectrumEffect,
26
+ getPolarXAbsolute,
27
+ getPolarYAbsolute,
28
+ getPolarInnerRadius,
29
+ getPolarMaxBarLength,
30
+ getPolarBarWidth,
31
+ getPolarEffect,
32
+ getPolarColor,
33
+ getPolarOpacityParsed,
34
+ getAutoEditVideo,
35
+ getAudioAutoHighlight,
36
+ getAudioAutoHighlightCount,
37
+ } from './config';
38
+ import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor } from './audio';
39
+ import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps } from './image';
40
+ import { normalizeInlineSubtitlesToSrt, lrcToSrt } from './subtitleConvert';
41
+ import { spawnFfmpegVideoWriter, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile } from './video';
42
+ import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
43
+ import { createBeatDetector } from './beats';
44
+ export { BeatInfo, BeatDetectorOptions } from './beats';
45
+ import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
46
+ import { waitForEventLoop } from './waitForEventLoop';
47
+ export { computeHighlightSlice, HIGHLIGHT_DURATION_SEC, BeatFrameEvent, HighlightAudioSegment, HighlightRun } from './highlight';
48
+
49
+ export const PCM_FORMAT = {
50
+ bit: 8,
51
+ sign: 'u',
52
+ parseFunction: bufferToUInt8
53
+ };
54
+ const FFMPEG_FORMAT = `${PCM_FORMAT.sign}${PCM_FORMAT.bit}`;
55
+ const PROCESSING_BUFFER_SIZE = Math.pow(2, 12);
56
+
57
+ export interface Config {
58
+ audio: {
59
+ path: string;
60
+ autoHighlight?: boolean;
61
+ /** When `autoHighlight` is true, number of non-overlapping 15s windows to stitch (default 1). */
62
+ autoHighlightCount?: number;
63
+ };
64
+ image?: {
65
+ path: string;
66
+ };
67
+ video?: {
68
+ path: string;
69
+ autoEdit?: boolean;
70
+ };
71
+ outVideo: {
72
+ path: string;
73
+ subtitles?:
74
+ | string
75
+ | {
76
+ path?: string;
77
+ rawContent?: string;
78
+ alignment?: 'top' | 'middle' | 'bottom';
79
+ };
80
+ fps?: number;
81
+ resolution?: {
82
+ width: number;
83
+ height: number;
84
+ };
85
+ spectrum?: {
86
+ width?: SpectrumSizeValue;
87
+ height?: SpectrumSizeValue;
88
+ x?: number | PositionAliasName;
89
+ y?: number | PositionAliasName;
90
+ rotation?: RotationAliasName;
91
+ effect?: SpectrumEffect;
92
+ color?: Color | string;
93
+ opacity?: string;
94
+ };
95
+ polar?: {
96
+ x?: number | PositionAliasName;
97
+ y?: number | PositionAliasName;
98
+ innerRadius?: number;
99
+ maxBarLength?: number;
100
+ barWidth?: number;
101
+ effect?: SpectrumEffect;
102
+ color?: Color | string;
103
+ opacity?: string;
104
+ }
105
+ };
106
+ tweaks?: {
107
+ ffmpeg_cfr?: string;
108
+ ffmpeg_preset?: string;
109
+ frame_processing_delay?: number;
110
+ };
111
+ }
112
+
113
+ export type SpectrumSizeValue = number | string;
114
+
115
+ export type PositionAliasName =
116
+ 'left' |
117
+ 'center' |
118
+ 'right' |
119
+ 'top' |
120
+ 'middle' |
121
+ 'bottom';
122
+
123
+ export type RotationAliasName = typeof rotationAliasValues[number];
124
+
125
+ const sleep = (timeout: number) =>
126
+ new Promise(resolve => setTimeout(resolve, timeout));
127
+
128
+ const createVisualizerFrameGenerator = (
129
+ config: Config,
130
+ backgroundWidth: number,
131
+ backgroundHeight: number,
132
+ defaultColor: Color,
133
+ spectrumBusMargin: number
134
+ ): (params: CommonVisualizerFrameProps) => EncodedBmp => {
135
+ if (!config.outVideo.spectrum) {
136
+ const createPolarVisualizerFrame = createPolarVisualizerFrameGenerator();
137
+ const polarX = getPolarXAbsolute(config, backgroundWidth);
138
+ const polarY = getPolarYAbsolute(config, backgroundHeight);
139
+ const polarInnerRadius = getPolarInnerRadius(config);
140
+ const polarMaxBarLength = getPolarMaxBarLength(config);
141
+ const polarBarWidth = getPolarBarWidth(config);
142
+ const polarColor = getPolarColor(config) || invertColor(defaultColor);
143
+ const polarEffect = getPolarEffect(config);
144
+ const polarOpacity = getPolarOpacityParsed(config);
145
+
146
+ return (params: CommonVisualizerFrameProps) => {
147
+ return createPolarVisualizerFrame({
148
+ ...params,
149
+ centerX: polarX,
150
+ centerY: polarY,
151
+ innerRadius: polarInnerRadius,
152
+ maxBarLength: polarMaxBarLength,
153
+ barWidth: polarBarWidth,
154
+ color: polarColor,
155
+ opacity: polarOpacity,
156
+ spectrumEffect: polarEffect,
157
+ });
158
+ };
159
+ }
160
+ const createSpectrumVisualizerFrame = createSpectrumVisualizerFrameGenerator();
161
+ const spectrumWidth = getSpectrumWidthAbsolute(config, backgroundWidth);
162
+ const spectrumHeight = getSpectrumHeightAbsolute(config, backgroundHeight);
163
+ const spectrumX = getSpectrumXAbsolute(config, spectrumWidth, backgroundWidth);
164
+ const spectrumY = getSpectrumYAbsolute(config, spectrumHeight, backgroundHeight);
165
+ const spectrumRotation = getSpectrumRotation(config);
166
+ const spectrumColor = getSpectrumColor(config) || invertColor(defaultColor);
167
+ const spectrumEffect = getSpectrumEffect(config);
168
+ const spectrumOpacity = getSpectrumOpacityParsed(config);
169
+
170
+ return (params: CommonVisualizerFrameProps) => {
171
+ return createSpectrumVisualizerFrame({
172
+ ...params,
173
+ size: { width: spectrumWidth, height: spectrumHeight },
174
+ position: { x: spectrumX, y: spectrumY },
175
+ rotation: spectrumRotation,
176
+ margin: spectrumBusMargin,
177
+ color: spectrumColor,
178
+ opacity: spectrumOpacity,
179
+ spectrumEffect,
180
+ });
181
+ };
182
+ };
183
+
184
+ interface PreProcessedAudio {
185
+ spectrums: number[][];
186
+ beatFrameIndices: number[];
187
+ beatEvents: { frameIndex: number; intensity: number }[];
188
+ }
189
+
190
+ const PRE_PROCESS_PROGRESS_SHARE = 60;
191
+ const POST_AUDIO_PROGRESS_SHARE = 10;
192
+ const RENDER_PROGRESS_SHARE = 30;
193
+
194
+ const preProcessAudio = async (
195
+ audioBuffer: Buffer,
196
+ sampleRate: number,
197
+ fps: number,
198
+ framesCount: number,
199
+ onPreProcessProgress?: (totalPercent: number) => void,
200
+ ): Promise<PreProcessedAudio> => {
201
+ if (framesCount === 0) {
202
+ for (let t = 1; t <= PRE_PROCESS_PROGRESS_SHARE; t += 1) {
203
+ onPreProcessProgress?.(t);
204
+ }
205
+ return { spectrums: [], beatFrameIndices: [], beatEvents: [] };
206
+ }
207
+ const audioDataStep = Math.trunc(audioBuffer.length / framesCount);
208
+ const processingBuffer = new Float32Array(PROCESSING_BUFFER_SIZE).fill(0);
209
+ const skipFramesCount = fps < 45 ? 1 : 2;
210
+ const processSpectrum = createSpectrumsProcessor(sampleRate, skipFramesCount);
211
+ const detectBeat = createBeatDetector(fps);
212
+
213
+ const spectrums: number[][] = [];
214
+ const beatFrameIndices: number[] = [];
215
+ const beatEvents: { frameIndex: number; intensity: number }[] = [];
216
+ let nextPreProcessMilestone = 1;
217
+
218
+ for (let i = 0; i < framesCount; i++) {
219
+ const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
220
+ const frameDataLength = Math.min(currentFrameData.length, PROCESSING_BUFFER_SIZE);
221
+ const frameDataTailStart = currentFrameData.length - frameDataLength;
222
+ const frameDataToProcess = currentFrameData.slice(frameDataTailStart);
223
+
224
+ processingBuffer.copyWithin(0, frameDataLength);
225
+ processingBuffer.set(frameDataToProcess, PROCESSING_BUFFER_SIZE - frameDataLength);
226
+
227
+ const audioDataParser = () => Array.from(processingBuffer);
228
+ const spectrum = processSpectrum(audioDataParser);
229
+ const beat = detectBeat(spectrum);
230
+
231
+ spectrums.push(spectrum);
232
+ if (beat.isBeat) {
233
+ beatFrameIndices.push(i);
234
+ beatEvents.push({ frameIndex: i, intensity: beat.intensity });
235
+ }
236
+
237
+ const p =
238
+ PRE_PROCESS_PROGRESS_SHARE * (i + 1) / framesCount;
239
+ while (
240
+ nextPreProcessMilestone <= PRE_PROCESS_PROGRESS_SHARE
241
+ && p >= nextPreProcessMilestone
242
+ ) {
243
+ onPreProcessProgress?.(nextPreProcessMilestone);
244
+ nextPreProcessMilestone += 1;
245
+ }
246
+
247
+ await waitForEventLoop();
248
+ }
249
+
250
+ return { spectrums, beatFrameIndices, beatEvents };
251
+ };
252
+
253
+ async function prepareBackgroundForRender(params: {
254
+ useVideoBackground: boolean;
255
+ backgroundVideoPath: string | undefined;
256
+ backgroundImagePath: string | undefined;
257
+ beatFrameIndices: number[];
258
+ framesCount: number;
259
+ outputResolution: ReturnType<typeof getOutputResolution>;
260
+ fps: number;
261
+ autoEditVideo: boolean;
262
+ }): Promise<{
263
+ backgroundWidth: number;
264
+ backgroundHeight: number;
265
+ defaultColor: Color;
266
+ staticBackgroundBuffer: EncodedBmp;
267
+ videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
268
+ videoFrameSize: number;
269
+ encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
270
+ concatFilePath?: string;
271
+ }> {
272
+ const {
273
+ useVideoBackground,
274
+ backgroundVideoPath,
275
+ backgroundImagePath,
276
+ beatFrameIndices,
277
+ framesCount,
278
+ outputResolution,
279
+ fps,
280
+ autoEditVideo,
281
+ } = params;
282
+
283
+ if (useVideoBackground && backgroundVideoPath) {
284
+ const videoInfo = await getVideoInfo(backgroundVideoPath);
285
+ const sceneChanges = autoEditVideo
286
+ ? await detectSceneChanges(backgroundVideoPath)
287
+ : [];
288
+ const backgroundWidth = outputResolution?.width ?? videoInfo.width;
289
+ const backgroundHeight = outputResolution?.height ?? videoInfo.height;
290
+
291
+ const videoFrameSize = backgroundWidth * backgroundHeight * 3;
292
+ const encodeVideoFrame = createBgrFrameEncoder({ width: backgroundWidth, height: backgroundHeight });
293
+
294
+ const segments = buildBeatSyncedSegments(
295
+ beatFrameIndices,
296
+ framesCount,
297
+ sceneChanges,
298
+ videoInfo.duration,
299
+ );
300
+
301
+ const concatFilePath = writeConcatFile(segments, backgroundVideoPath, videoInfo.duration, fps);
302
+
303
+ const videoFrameReader = spawnConcatVideoFrameReader({
304
+ concatFilePath,
305
+ fps,
306
+ totalFrames: framesCount,
307
+ ...(outputResolution && {
308
+ width: backgroundWidth,
309
+ height: backgroundHeight,
310
+ sourceWidth: videoInfo.width,
311
+ sourceHeight: videoInfo.height,
312
+ }),
313
+ });
314
+
315
+ const firstFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
316
+ if (!firstFrame) {
317
+ throw new Error(`Could not read frames from video: ${backgroundVideoPath}`);
318
+ }
319
+ const defaultColor = getVideoFrameColor(firstFrame, backgroundWidth, backgroundHeight);
320
+ const staticBackgroundBuffer = encodeVideoFrame(firstFrame);
321
+ return {
322
+ backgroundWidth,
323
+ backgroundHeight,
324
+ defaultColor,
325
+ staticBackgroundBuffer,
326
+ videoFrameReader,
327
+ videoFrameSize,
328
+ encodeVideoFrame,
329
+ concatFilePath,
330
+ };
331
+ }
332
+
333
+ if (!backgroundImagePath) {
334
+ throw new Error('Background image path is required when not using video background.');
335
+ }
336
+ const backgroundImageBmpBuffer = await convertToBmp(
337
+ backgroundImagePath,
338
+ outputResolution?.width,
339
+ outputResolution?.height,
340
+ );
341
+ const backgroundImage = parseImage(backgroundImageBmpBuffer);
342
+ const backgroundWidth = backgroundImage.width;
343
+ const backgroundHeight = backgroundImage.height;
344
+ const defaultColor = getImageColor(backgroundImage);
345
+
346
+ const bpmEncoder = createBpmEncoder({ width: backgroundWidth, height: backgroundHeight });
347
+ const staticBackgroundBuffer = bpmEncoder(backgroundImage.data);
348
+ return {
349
+ backgroundWidth,
350
+ backgroundHeight,
351
+ defaultColor,
352
+ staticBackgroundBuffer,
353
+ videoFrameSize: 0,
354
+ };
355
+ }
356
+
357
+ async function resolveBackgroundFrameBuffer(params: {
358
+ frameIndex: number;
359
+ useVideoBackground: boolean;
360
+ videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
361
+ videoFrameSize: number;
362
+ encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
363
+ staticBackgroundBuffer: EncodedBmp;
364
+ }): Promise<EncodedBmp> {
365
+ const {
366
+ frameIndex,
367
+ useVideoBackground,
368
+ videoFrameReader,
369
+ videoFrameSize,
370
+ encodeVideoFrame,
371
+ staticBackgroundBuffer,
372
+ } = params;
373
+
374
+ if (useVideoBackground && videoFrameReader && encodeVideoFrame) {
375
+ if (frameIndex === 0) {
376
+ return staticBackgroundBuffer;
377
+ }
378
+ const videoFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
379
+ return videoFrame
380
+ ? encodeVideoFrame(videoFrame)
381
+ : staticBackgroundBuffer;
382
+ }
383
+ return staticBackgroundBuffer;
384
+ }
385
+
386
+ export interface RenderAudioVisualizerResult {
387
+ exitCode: number;
388
+ /** Absolute paths of video files written successfully (exit code 0, full pass, no early stop). */
389
+ outputVideoFiles: string[];
390
+ }
391
+
392
+ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: number) => any, shouldStop?: () => boolean) =>
393
+ new Promise<RenderAudioVisualizerResult>(async (resolve) => {
394
+ if (config.outVideo.spectrum && config.outVideo.polar) {
395
+ throw new Error('Cannot use both "spectrum" and "polar" options. Please specify only one visualizer type.');
396
+ }
397
+ if (!config.image && !config.video) {
398
+ throw new Error('Either "image" or "video" must be specified as the background source.');
399
+ }
400
+ if (config.image && config.video) {
401
+ throw new Error('Cannot use both "image" and "video" options. Please specify only one background source.');
402
+ }
403
+
404
+ const audioFilePath = getAudioFilePath(config);
405
+ const outVideoPath = getOutVideoPath(config);
406
+ const subtitleSpec = getSubtitleRenderSpec(config);
407
+ let subtitleFilePath: string | undefined;
408
+ let subtitleFileIsTemporary = false;
409
+ let subtitleAlignmentAss = 2;
410
+ if (subtitleSpec) {
411
+ subtitleAlignmentAss = subtitleAlignmentToAss(subtitleSpec.alignment);
412
+ if (subtitleSpec.source.kind === 'file') {
413
+ const absPath = subtitleSpec.source.path;
414
+ const ext = path.extname(absPath).toLowerCase();
415
+ if (ext === '.lrc') {
416
+ const raw = readFileSync(absPath, 'utf-8');
417
+ subtitleFilePath = writeSubtitlesFile(lrcToSrt(raw));
418
+ subtitleFileIsTemporary = true;
419
+ } else {
420
+ subtitleFilePath = absPath;
421
+ }
422
+ } else {
423
+ subtitleFilePath = writeSubtitlesFile(
424
+ normalizeInlineSubtitlesToSrt(subtitleSpec.source.text),
425
+ );
426
+ subtitleFileIsTemporary = true;
427
+ }
428
+ }
429
+ const backgroundVideoPath = getBackgroundVideoPath(config);
430
+ const backgroundImagePath = getBackgroundImagePath(config);
431
+ const useVideoBackground = !!backgroundVideoPath;
432
+
433
+ const audioReader = await createAudioBuffer(audioFilePath, FFMPEG_FORMAT);
434
+ const audioBuffer = audioReader.audioBuffer;
435
+ const sampleRate = audioReader.sampleRate;
436
+ if (!sampleRate) {
437
+ throw new Error('ffmpeg didn\'t show audio sample rate');
438
+ }
439
+
440
+ const spectrumBusMargin = getSpectrumBusMargin();
441
+ const FPS = getFPS(config);
442
+ const ffmpeg_cfr = getFfmpeg_cfr(config);
443
+ const ffmpeg_preset = getFfmpeg_preset(config);
444
+ const frame_processing_delay = getFrame_processing_delay(config);
445
+
446
+ const audioDuration = audioBuffer.length / sampleRate;
447
+ const framesCount = Math.trunc(audioDuration * FPS);
448
+ const outputResolution = getOutputResolution(config);
449
+
450
+ let maxProgressReported = -1;
451
+ const reportProgress = (progress: number) => {
452
+ if (!onProgress) {
453
+ return;
454
+ }
455
+ const normalized = +Math.min(100, Math.max(0, progress)).toFixed(2);
456
+ if (normalized <= maxProgressReported) {
457
+ return;
458
+ }
459
+ maxProgressReported = normalized;
460
+ onProgress(normalized);
461
+ };
462
+ reportProgress(0);
463
+
464
+ const preprocessed = await preProcessAudio(
465
+ audioBuffer,
466
+ sampleRate,
467
+ FPS,
468
+ framesCount,
469
+ (progress: number) => {
470
+ reportProgress(progress);
471
+ },
472
+ );
473
+ reportProgress(PRE_PROCESS_PROGRESS_SHARE);
474
+
475
+ const autoHighlight = getAudioAutoHighlight(config);
476
+ let spectrumsForRender = preprocessed.spectrums;
477
+ let beatIndicesForRender = preprocessed.beatFrameIndices;
478
+ let framesCountForRender = framesCount;
479
+ type HighlightSliceResult = ReturnType<typeof computeHighlightSlice> extends Promise<
480
+ infer R
481
+ >
482
+ ? R
483
+ : never;
484
+ let highlightSlice: HighlightSliceResult | undefined;
485
+
486
+ if (autoHighlight) {
487
+ reportProgress(PRE_PROCESS_PROGRESS_SHARE + 1);
488
+ highlightSlice = await computeHighlightSlice(
489
+ FPS,
490
+ framesCount,
491
+ preprocessed.spectrums,
492
+ preprocessed.beatEvents,
493
+ getAudioAutoHighlightCount(config),
494
+ );
495
+ spectrumsForRender = highlightSlice.spectrums;
496
+ beatIndicesForRender = highlightSlice.beatFrameIndices;
497
+ framesCountForRender = highlightSlice.highlightFrames;
498
+ reportProgress(PRE_PROCESS_PROGRESS_SHARE + POST_AUDIO_PROGRESS_SHARE);
499
+ } else {
500
+ reportProgress(PRE_PROCESS_PROGRESS_SHARE + POST_AUDIO_PROGRESS_SHARE);
501
+ }
502
+
503
+ const separateHighlightFiles =
504
+ autoHighlight &&
505
+ highlightSlice &&
506
+ highlightSlice.runs.length > 1;
507
+
508
+ type VideoRenderPass = {
509
+ outPath: string;
510
+ spectrums: number[][];
511
+ beatIndices: number[];
512
+ frameCount: number;
513
+ audioSegment: import('./highlight').HighlightAudioSegment | undefined;
514
+ };
515
+
516
+ const passes: VideoRenderPass[] =
517
+ separateHighlightFiles && highlightSlice
518
+ ? highlightSlice.runs.map((run, i) => {
519
+ const dir = path.dirname(outVideoPath);
520
+ const ext = path.extname(outVideoPath);
521
+ const base = path.basename(outVideoPath, ext);
522
+ const numberedPath = path.resolve(dir, `${base}-${i + 1}${ext}`);
523
+ return {
524
+ outPath: numberedPath,
525
+ spectrums: run.spectrums,
526
+ beatIndices: run.beatFrameIndices,
527
+ frameCount: run.highlightFrames,
528
+ audioSegment: run.audioSegment,
529
+ };
530
+ })
531
+ : [
532
+ {
533
+ outPath: outVideoPath,
534
+ spectrums: spectrumsForRender,
535
+ beatIndices: beatIndicesForRender,
536
+ frameCount: framesCountForRender,
537
+ audioSegment:
538
+ autoHighlight &&
539
+ highlightSlice &&
540
+ highlightSlice.audioSegments.length > 0
541
+ ? highlightSlice.audioSegments[0]
542
+ : undefined,
543
+ },
544
+ ];
545
+
546
+ const totalPassFrames =
547
+ passes.reduce((sum, pass) => sum + pass.frameCount, 0);
548
+ const totalRenderMilestones = totalPassFrames + (passes.length * 2);
549
+ let renderMilestonesCompleted = 0;
550
+ const reportRenderProgress = (milestoneIncrement = 1) => {
551
+ renderMilestonesCompleted += milestoneIncrement;
552
+ reportProgress(
553
+ PRE_PROCESS_PROGRESS_SHARE
554
+ + POST_AUDIO_PROGRESS_SHARE
555
+ + (RENDER_PROGRESS_SHARE * renderMilestonesCompleted / Math.max(1, totalRenderMilestones)),
556
+ );
557
+ };
558
+ let lastExitCode = 0;
559
+ const outputVideoFiles: string[] = [];
560
+ try {
561
+ passLoop: for (const pass of passes) {
562
+ const {
563
+ backgroundWidth,
564
+ backgroundHeight,
565
+ defaultColor,
566
+ staticBackgroundBuffer,
567
+ videoFrameReader,
568
+ videoFrameSize,
569
+ encodeVideoFrame,
570
+ concatFilePath,
571
+ } = await prepareBackgroundForRender({
572
+ useVideoBackground,
573
+ backgroundVideoPath,
574
+ backgroundImagePath,
575
+ beatFrameIndices: pass.beatIndices,
576
+ framesCount: pass.frameCount,
577
+ outputResolution,
578
+ fps: FPS,
579
+ autoEditVideo: getAutoEditVideo(config),
580
+ });
581
+ reportRenderProgress();
582
+
583
+ const createVisualizerFrame = createVisualizerFrameGenerator(
584
+ config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
585
+ );
586
+
587
+ const ffmpegVideoWriter = spawnFfmpegVideoWriter({
588
+ audioFilename: audioFilePath,
589
+ videoFileName: pass.outPath,
590
+ ...(subtitleFilePath && {
591
+ subtitleFilename: subtitleFilePath,
592
+ subtitleAlignmentAss,
593
+ }),
594
+ fps: FPS,
595
+ ...(pass.audioSegment && { audioSegment: pass.audioSegment }),
596
+ ...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
597
+ ...(ffmpeg_preset && { preset: ffmpeg_preset }),
598
+ });
599
+
600
+ const exitPromise = new Promise<number>(res => {
601
+ let settled = false;
602
+ ffmpegVideoWriter.on('exit', (code: number | null) => {
603
+ if (settled) {
604
+ return;
605
+ }
606
+ settled = true;
607
+ res(code ?? 0);
608
+ });
609
+ });
610
+
611
+ let stoppedEarly = false;
612
+ for (let i = 0; i < pass.frameCount; i++) {
613
+ const spectrum = pass.spectrums[i];
614
+
615
+ const backgroundImageBuffer = await resolveBackgroundFrameBuffer({
616
+ frameIndex: i,
617
+ useVideoBackground,
618
+ videoFrameReader,
619
+ videoFrameSize,
620
+ encodeVideoFrame,
621
+ staticBackgroundBuffer,
622
+ });
623
+
624
+ const commonVisualizerFrameProps: CommonVisualizerFrameProps = {
625
+ backgroundImageBuffer,
626
+ spectrum,
627
+ };
628
+ const frameImage = createVisualizerFrame(commonVisualizerFrameProps);
629
+ const isFrameProcessed = ffmpegVideoWriter.stdin.write(frameImage.data);
630
+ if (!isFrameProcessed) {
631
+ const isDrained = await waitDrain(ffmpegVideoWriter.stdin, ffmpegVideoWriter);
632
+ if (!isDrained) {
633
+ stoppedEarly = true;
634
+ break;
635
+ }
636
+ }
637
+ if (shouldStop && shouldStop()) {
638
+ stoppedEarly = true;
639
+ break;
640
+ }
641
+ if (frame_processing_delay) {
642
+ await sleep(frame_processing_delay);
643
+ }
644
+ await waitForEventLoop();
645
+ reportRenderProgress();
646
+ }
647
+
648
+ if (videoFrameReader) {
649
+ videoFrameReader.kill();
650
+ }
651
+ if (concatFilePath) {
652
+ cleanupConcatFile(concatFilePath);
653
+ }
654
+ ffmpegVideoWriter.stdin.end();
655
+
656
+ lastExitCode = await exitPromise;
657
+ reportRenderProgress();
658
+
659
+ if (lastExitCode === 0 && !stoppedEarly) {
660
+ outputVideoFiles.push(pass.outPath);
661
+ }
662
+
663
+ if (stoppedEarly || lastExitCode !== 0) {
664
+ break passLoop;
665
+ }
666
+ }
667
+
668
+ if (lastExitCode === 0 && outputVideoFiles.length === passes.length) {
669
+ reportProgress(100);
670
+ }
671
+
672
+ resolve({ exitCode: lastExitCode, outputVideoFiles });
673
+ } finally {
674
+ if (subtitleFilePath && subtitleFileIsTemporary) {
675
+ cleanupTempFile(subtitleFilePath);
676
+ }
677
+ }
678
+ });