nodejs-audio-visualizer 4.3.6 → 4.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +75 -75
- package/dist/audio.d.ts +12 -12
- package/dist/audio.js +120 -120
- package/dist/beats.d.ts +19 -19
- package/dist/beats.js +43 -43
- package/dist/bpmEncoder.d.ts +14 -14
- package/dist/bpmEncoder.js +97 -97
- package/dist/config.d.ts +67 -67
- package/dist/config.js +356 -356
- package/dist/dsp.d.ts +16 -16
- package/dist/dsp.js +76 -76
- package/dist/highlight.d.ts +32 -32
- package/dist/highlight.js +325 -325
- package/dist/image.d.ts +79 -79
- package/dist/image.js +458 -458
- package/dist/index.d.ts +72 -72
- package/dist/index.js +536 -533
- package/dist/index.js.map +1 -1
- package/dist/subtitleConvert.d.ts +3 -3
- package/dist/subtitleConvert.js +61 -61
- package/dist/video.d.ts +72 -72
- package/dist/video.js +338 -338
- package/dist/waitForEventLoop.d.ts +1 -1
- package/dist/waitForEventLoop.js +5 -5
- package/package.json +61 -61
- package/src/audio.ts +120 -120
- package/src/beats.ts +68 -68
- package/src/bpmEncoder.ts +97 -97
- package/src/config.ts +462 -462
- package/src/dsp.ts +95 -95
- package/src/highlight.ts +284 -284
- package/src/image.ts +602 -602
- package/src/index.ts +678 -674
- package/src/subtitleConvert.ts +63 -63
- package/src/tests/audio.test.ts +96 -96
- package/src/tests/config.test.ts +475 -475
- package/src/tests/dsp.test.ts +24 -24
- package/src/tests/highlight.test.ts +178 -178
- package/src/tests/image.test.ts +222 -222
- package/src/tests/index.test.ts +63 -63
- package/src/tests/subtitleConvert.test.ts +23 -23
- package/src/tests/video.test.ts +138 -138
- package/src/typings.d.ts +6 -6
- package/src/video.ts +437 -437
- package/src/waitForEventLoop.ts +2 -2
package/src/index.ts
CHANGED
|
@@ -1,674 +1,678 @@
|
|
|
1
|
-
import path from 'path';
|
|
2
|
-
import { readFileSync } from 'fs';
|
|
3
|
-
import {
|
|
4
|
-
getAudioFilePath,
|
|
5
|
-
getBackgroundImagePath,
|
|
6
|
-
getBackgroundVideoPath,
|
|
7
|
-
getOutVideoPath,
|
|
8
|
-
getSubtitleRenderSpec,
|
|
9
|
-
subtitleAlignmentToAss,
|
|
10
|
-
getFPS,
|
|
11
|
-
getSpectrumBusMargin,
|
|
12
|
-
getSpectrumWidthAbsolute,
|
|
13
|
-
getSpectrumHeightAbsolute,
|
|
14
|
-
getSpectrumXAbsolute,
|
|
15
|
-
getSpectrumYAbsolute,
|
|
16
|
-
getSpectrumColor,
|
|
17
|
-
getSpectrumOpacityParsed,
|
|
18
|
-
getFfmpeg_cfr,
|
|
19
|
-
getFfmpeg_preset,
|
|
20
|
-
getFrame_processing_delay,
|
|
21
|
-
getOutputResolution,
|
|
22
|
-
rotationAliasValues,
|
|
23
|
-
getSpectrumRotation,
|
|
24
|
-
getSpectrumEffect,
|
|
25
|
-
SpectrumEffect,
|
|
26
|
-
getPolarXAbsolute,
|
|
27
|
-
getPolarYAbsolute,
|
|
28
|
-
getPolarInnerRadius,
|
|
29
|
-
getPolarMaxBarLength,
|
|
30
|
-
getPolarBarWidth,
|
|
31
|
-
getPolarEffect,
|
|
32
|
-
getPolarColor,
|
|
33
|
-
getPolarOpacityParsed,
|
|
34
|
-
getAutoEditVideo,
|
|
35
|
-
getAudioAutoHighlight,
|
|
36
|
-
getAudioAutoHighlightCount,
|
|
37
|
-
} from './config';
|
|
38
|
-
import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor } from './audio';
|
|
39
|
-
import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps } from './image';
|
|
40
|
-
import { normalizeInlineSubtitlesToSrt, lrcToSrt } from './subtitleConvert';
|
|
41
|
-
import { spawnFfmpegVideoWriter, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile } from './video';
|
|
42
|
-
import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
|
|
43
|
-
import { createBeatDetector } from './beats';
|
|
44
|
-
export { BeatInfo, BeatDetectorOptions } from './beats';
|
|
45
|
-
import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
|
|
46
|
-
import { waitForEventLoop } from './waitForEventLoop';
|
|
47
|
-
export { computeHighlightSlice, HIGHLIGHT_DURATION_SEC, BeatFrameEvent, HighlightAudioSegment, HighlightRun } from './highlight';
|
|
48
|
-
|
|
49
|
-
export const PCM_FORMAT = {
|
|
50
|
-
bit: 8,
|
|
51
|
-
sign: 'u',
|
|
52
|
-
parseFunction: bufferToUInt8
|
|
53
|
-
};
|
|
54
|
-
const FFMPEG_FORMAT = `${PCM_FORMAT.sign}${PCM_FORMAT.bit}`;
|
|
55
|
-
const PROCESSING_BUFFER_SIZE = Math.pow(2, 12);
|
|
56
|
-
|
|
57
|
-
export interface Config {
|
|
58
|
-
audio: {
|
|
59
|
-
path: string;
|
|
60
|
-
autoHighlight?: boolean;
|
|
61
|
-
/** When `autoHighlight` is true, number of non-overlapping 15s windows to stitch (default 1). */
|
|
62
|
-
autoHighlightCount?: number;
|
|
63
|
-
};
|
|
64
|
-
image?: {
|
|
65
|
-
path: string;
|
|
66
|
-
};
|
|
67
|
-
video?: {
|
|
68
|
-
path: string;
|
|
69
|
-
autoEdit?: boolean;
|
|
70
|
-
};
|
|
71
|
-
outVideo: {
|
|
72
|
-
path: string;
|
|
73
|
-
subtitles?:
|
|
74
|
-
| string
|
|
75
|
-
| {
|
|
76
|
-
path?: string;
|
|
77
|
-
rawContent?: string;
|
|
78
|
-
alignment?: 'top' | 'middle' | 'bottom';
|
|
79
|
-
};
|
|
80
|
-
fps?: number;
|
|
81
|
-
resolution?: {
|
|
82
|
-
width: number;
|
|
83
|
-
height: number;
|
|
84
|
-
};
|
|
85
|
-
spectrum?: {
|
|
86
|
-
width?: SpectrumSizeValue;
|
|
87
|
-
height?: SpectrumSizeValue;
|
|
88
|
-
x?: number | PositionAliasName;
|
|
89
|
-
y?: number | PositionAliasName;
|
|
90
|
-
rotation?: RotationAliasName;
|
|
91
|
-
effect?: SpectrumEffect;
|
|
92
|
-
color?: Color | string;
|
|
93
|
-
opacity?: string;
|
|
94
|
-
};
|
|
95
|
-
polar?: {
|
|
96
|
-
x?: number | PositionAliasName;
|
|
97
|
-
y?: number | PositionAliasName;
|
|
98
|
-
innerRadius?: number;
|
|
99
|
-
maxBarLength?: number;
|
|
100
|
-
barWidth?: number;
|
|
101
|
-
effect?: SpectrumEffect;
|
|
102
|
-
color?: Color | string;
|
|
103
|
-
opacity?: string;
|
|
104
|
-
}
|
|
105
|
-
};
|
|
106
|
-
tweaks?: {
|
|
107
|
-
ffmpeg_cfr?: string;
|
|
108
|
-
ffmpeg_preset?: string;
|
|
109
|
-
frame_processing_delay?: number;
|
|
110
|
-
};
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
export type SpectrumSizeValue = number | string;
|
|
114
|
-
|
|
115
|
-
export type PositionAliasName =
|
|
116
|
-
'left' |
|
|
117
|
-
'center' |
|
|
118
|
-
'right' |
|
|
119
|
-
'top' |
|
|
120
|
-
'middle' |
|
|
121
|
-
'bottom';
|
|
122
|
-
|
|
123
|
-
export type RotationAliasName = typeof rotationAliasValues[number];
|
|
124
|
-
|
|
125
|
-
const sleep = (timeout: number) =>
|
|
126
|
-
new Promise(resolve => setTimeout(resolve, timeout));
|
|
127
|
-
|
|
128
|
-
const createVisualizerFrameGenerator = (
|
|
129
|
-
config: Config,
|
|
130
|
-
backgroundWidth: number,
|
|
131
|
-
backgroundHeight: number,
|
|
132
|
-
defaultColor: Color,
|
|
133
|
-
spectrumBusMargin: number
|
|
134
|
-
): (params: CommonVisualizerFrameProps) => EncodedBmp => {
|
|
135
|
-
if (!config.outVideo.spectrum) {
|
|
136
|
-
const createPolarVisualizerFrame = createPolarVisualizerFrameGenerator();
|
|
137
|
-
const polarX = getPolarXAbsolute(config, backgroundWidth);
|
|
138
|
-
const polarY = getPolarYAbsolute(config, backgroundHeight);
|
|
139
|
-
const polarInnerRadius = getPolarInnerRadius(config);
|
|
140
|
-
const polarMaxBarLength = getPolarMaxBarLength(config);
|
|
141
|
-
const polarBarWidth = getPolarBarWidth(config);
|
|
142
|
-
const polarColor = getPolarColor(config) || invertColor(defaultColor);
|
|
143
|
-
const polarEffect = getPolarEffect(config);
|
|
144
|
-
const polarOpacity = getPolarOpacityParsed(config);
|
|
145
|
-
|
|
146
|
-
return (params: CommonVisualizerFrameProps) => {
|
|
147
|
-
return createPolarVisualizerFrame({
|
|
148
|
-
...params,
|
|
149
|
-
centerX: polarX,
|
|
150
|
-
centerY: polarY,
|
|
151
|
-
innerRadius: polarInnerRadius,
|
|
152
|
-
maxBarLength: polarMaxBarLength,
|
|
153
|
-
barWidth: polarBarWidth,
|
|
154
|
-
color: polarColor,
|
|
155
|
-
opacity: polarOpacity,
|
|
156
|
-
spectrumEffect: polarEffect,
|
|
157
|
-
});
|
|
158
|
-
};
|
|
159
|
-
}
|
|
160
|
-
const createSpectrumVisualizerFrame = createSpectrumVisualizerFrameGenerator();
|
|
161
|
-
const spectrumWidth = getSpectrumWidthAbsolute(config, backgroundWidth);
|
|
162
|
-
const spectrumHeight = getSpectrumHeightAbsolute(config, backgroundHeight);
|
|
163
|
-
const spectrumX = getSpectrumXAbsolute(config, spectrumWidth, backgroundWidth);
|
|
164
|
-
const spectrumY = getSpectrumYAbsolute(config, spectrumHeight, backgroundHeight);
|
|
165
|
-
const spectrumRotation = getSpectrumRotation(config);
|
|
166
|
-
const spectrumColor = getSpectrumColor(config) || invertColor(defaultColor);
|
|
167
|
-
const spectrumEffect = getSpectrumEffect(config);
|
|
168
|
-
const spectrumOpacity = getSpectrumOpacityParsed(config);
|
|
169
|
-
|
|
170
|
-
return (params: CommonVisualizerFrameProps) => {
|
|
171
|
-
return createSpectrumVisualizerFrame({
|
|
172
|
-
...params,
|
|
173
|
-
size: { width: spectrumWidth, height: spectrumHeight },
|
|
174
|
-
position: { x: spectrumX, y: spectrumY },
|
|
175
|
-
rotation: spectrumRotation,
|
|
176
|
-
margin: spectrumBusMargin,
|
|
177
|
-
color: spectrumColor,
|
|
178
|
-
opacity: spectrumOpacity,
|
|
179
|
-
spectrumEffect,
|
|
180
|
-
});
|
|
181
|
-
};
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
interface PreProcessedAudio {
|
|
185
|
-
spectrums: number[][];
|
|
186
|
-
beatFrameIndices: number[];
|
|
187
|
-
beatEvents: { frameIndex: number; intensity: number }[];
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
const PRE_PROCESS_PROGRESS_SHARE = 60;
|
|
191
|
-
const POST_AUDIO_PROGRESS_SHARE = 10;
|
|
192
|
-
const RENDER_PROGRESS_SHARE = 30;
|
|
193
|
-
|
|
194
|
-
const preProcessAudio = async (
|
|
195
|
-
audioBuffer: Buffer,
|
|
196
|
-
sampleRate: number,
|
|
197
|
-
fps: number,
|
|
198
|
-
framesCount: number,
|
|
199
|
-
onPreProcessProgress?: (totalPercent: number) => void,
|
|
200
|
-
): Promise<PreProcessedAudio> => {
|
|
201
|
-
if (framesCount === 0) {
|
|
202
|
-
for (let t = 1; t <= PRE_PROCESS_PROGRESS_SHARE; t += 1) {
|
|
203
|
-
onPreProcessProgress?.(t);
|
|
204
|
-
}
|
|
205
|
-
return { spectrums: [], beatFrameIndices: [], beatEvents: [] };
|
|
206
|
-
}
|
|
207
|
-
const audioDataStep = Math.trunc(audioBuffer.length / framesCount);
|
|
208
|
-
const processingBuffer = new Float32Array(PROCESSING_BUFFER_SIZE).fill(0);
|
|
209
|
-
const skipFramesCount = fps < 45 ? 1 : 2;
|
|
210
|
-
const processSpectrum = createSpectrumsProcessor(sampleRate, skipFramesCount);
|
|
211
|
-
const detectBeat = createBeatDetector(fps);
|
|
212
|
-
|
|
213
|
-
const spectrums: number[][] = [];
|
|
214
|
-
const beatFrameIndices: number[] = [];
|
|
215
|
-
const beatEvents: { frameIndex: number; intensity: number }[] = [];
|
|
216
|
-
let nextPreProcessMilestone = 1;
|
|
217
|
-
|
|
218
|
-
for (let i = 0; i < framesCount; i++) {
|
|
219
|
-
const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
nextPreProcessMilestone
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
const
|
|
285
|
-
const
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
const
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
const
|
|
343
|
-
const
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
const
|
|
430
|
-
const
|
|
431
|
-
const
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
const
|
|
441
|
-
|
|
442
|
-
const
|
|
443
|
-
const
|
|
444
|
-
const
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
const
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
if (
|
|
453
|
-
return;
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
const
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
);
|
|
582
|
-
|
|
583
|
-
const
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
const
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
}
|
|
637
|
-
if (
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
}
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
if (
|
|
660
|
-
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
}
|
|
673
|
-
}
|
|
674
|
-
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import { readFileSync } from 'fs';
|
|
3
|
+
import {
|
|
4
|
+
getAudioFilePath,
|
|
5
|
+
getBackgroundImagePath,
|
|
6
|
+
getBackgroundVideoPath,
|
|
7
|
+
getOutVideoPath,
|
|
8
|
+
getSubtitleRenderSpec,
|
|
9
|
+
subtitleAlignmentToAss,
|
|
10
|
+
getFPS,
|
|
11
|
+
getSpectrumBusMargin,
|
|
12
|
+
getSpectrumWidthAbsolute,
|
|
13
|
+
getSpectrumHeightAbsolute,
|
|
14
|
+
getSpectrumXAbsolute,
|
|
15
|
+
getSpectrumYAbsolute,
|
|
16
|
+
getSpectrumColor,
|
|
17
|
+
getSpectrumOpacityParsed,
|
|
18
|
+
getFfmpeg_cfr,
|
|
19
|
+
getFfmpeg_preset,
|
|
20
|
+
getFrame_processing_delay,
|
|
21
|
+
getOutputResolution,
|
|
22
|
+
rotationAliasValues,
|
|
23
|
+
getSpectrumRotation,
|
|
24
|
+
getSpectrumEffect,
|
|
25
|
+
SpectrumEffect,
|
|
26
|
+
getPolarXAbsolute,
|
|
27
|
+
getPolarYAbsolute,
|
|
28
|
+
getPolarInnerRadius,
|
|
29
|
+
getPolarMaxBarLength,
|
|
30
|
+
getPolarBarWidth,
|
|
31
|
+
getPolarEffect,
|
|
32
|
+
getPolarColor,
|
|
33
|
+
getPolarOpacityParsed,
|
|
34
|
+
getAutoEditVideo,
|
|
35
|
+
getAudioAutoHighlight,
|
|
36
|
+
getAudioAutoHighlightCount,
|
|
37
|
+
} from './config';
|
|
38
|
+
import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor } from './audio';
|
|
39
|
+
import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps } from './image';
|
|
40
|
+
import { normalizeInlineSubtitlesToSrt, lrcToSrt } from './subtitleConvert';
|
|
41
|
+
import { spawnFfmpegVideoWriter, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, writeSubtitlesFile, spawnConcatVideoFrameReader, cleanupConcatFile, cleanupTempFile } from './video';
|
|
42
|
+
import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
|
|
43
|
+
import { createBeatDetector } from './beats';
|
|
44
|
+
export { BeatInfo, BeatDetectorOptions } from './beats';
|
|
45
|
+
import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
|
|
46
|
+
import { waitForEventLoop } from './waitForEventLoop';
|
|
47
|
+
export { computeHighlightSlice, HIGHLIGHT_DURATION_SEC, BeatFrameEvent, HighlightAudioSegment, HighlightRun } from './highlight';
|
|
48
|
+
|
|
49
|
+
export const PCM_FORMAT = {
|
|
50
|
+
bit: 8,
|
|
51
|
+
sign: 'u',
|
|
52
|
+
parseFunction: bufferToUInt8
|
|
53
|
+
};
|
|
54
|
+
const FFMPEG_FORMAT = `${PCM_FORMAT.sign}${PCM_FORMAT.bit}`;
|
|
55
|
+
const PROCESSING_BUFFER_SIZE = Math.pow(2, 12);
|
|
56
|
+
|
|
57
|
+
export interface Config {
|
|
58
|
+
audio: {
|
|
59
|
+
path: string;
|
|
60
|
+
autoHighlight?: boolean;
|
|
61
|
+
/** When `autoHighlight` is true, number of non-overlapping 15s windows to stitch (default 1). */
|
|
62
|
+
autoHighlightCount?: number;
|
|
63
|
+
};
|
|
64
|
+
image?: {
|
|
65
|
+
path: string;
|
|
66
|
+
};
|
|
67
|
+
video?: {
|
|
68
|
+
path: string;
|
|
69
|
+
autoEdit?: boolean;
|
|
70
|
+
};
|
|
71
|
+
outVideo: {
|
|
72
|
+
path: string;
|
|
73
|
+
subtitles?:
|
|
74
|
+
| string
|
|
75
|
+
| {
|
|
76
|
+
path?: string;
|
|
77
|
+
rawContent?: string;
|
|
78
|
+
alignment?: 'top' | 'middle' | 'bottom';
|
|
79
|
+
};
|
|
80
|
+
fps?: number;
|
|
81
|
+
resolution?: {
|
|
82
|
+
width: number;
|
|
83
|
+
height: number;
|
|
84
|
+
};
|
|
85
|
+
spectrum?: {
|
|
86
|
+
width?: SpectrumSizeValue;
|
|
87
|
+
height?: SpectrumSizeValue;
|
|
88
|
+
x?: number | PositionAliasName;
|
|
89
|
+
y?: number | PositionAliasName;
|
|
90
|
+
rotation?: RotationAliasName;
|
|
91
|
+
effect?: SpectrumEffect;
|
|
92
|
+
color?: Color | string;
|
|
93
|
+
opacity?: string;
|
|
94
|
+
};
|
|
95
|
+
polar?: {
|
|
96
|
+
x?: number | PositionAliasName;
|
|
97
|
+
y?: number | PositionAliasName;
|
|
98
|
+
innerRadius?: number;
|
|
99
|
+
maxBarLength?: number;
|
|
100
|
+
barWidth?: number;
|
|
101
|
+
effect?: SpectrumEffect;
|
|
102
|
+
color?: Color | string;
|
|
103
|
+
opacity?: string;
|
|
104
|
+
}
|
|
105
|
+
};
|
|
106
|
+
tweaks?: {
|
|
107
|
+
ffmpeg_cfr?: string;
|
|
108
|
+
ffmpeg_preset?: string;
|
|
109
|
+
frame_processing_delay?: number;
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export type SpectrumSizeValue = number | string;
|
|
114
|
+
|
|
115
|
+
export type PositionAliasName =
|
|
116
|
+
'left' |
|
|
117
|
+
'center' |
|
|
118
|
+
'right' |
|
|
119
|
+
'top' |
|
|
120
|
+
'middle' |
|
|
121
|
+
'bottom';
|
|
122
|
+
|
|
123
|
+
export type RotationAliasName = typeof rotationAliasValues[number];
|
|
124
|
+
|
|
125
|
+
const sleep = (timeout: number) =>
|
|
126
|
+
new Promise(resolve => setTimeout(resolve, timeout));
|
|
127
|
+
|
|
128
|
+
const createVisualizerFrameGenerator = (
|
|
129
|
+
config: Config,
|
|
130
|
+
backgroundWidth: number,
|
|
131
|
+
backgroundHeight: number,
|
|
132
|
+
defaultColor: Color,
|
|
133
|
+
spectrumBusMargin: number
|
|
134
|
+
): (params: CommonVisualizerFrameProps) => EncodedBmp => {
|
|
135
|
+
if (!config.outVideo.spectrum) {
|
|
136
|
+
const createPolarVisualizerFrame = createPolarVisualizerFrameGenerator();
|
|
137
|
+
const polarX = getPolarXAbsolute(config, backgroundWidth);
|
|
138
|
+
const polarY = getPolarYAbsolute(config, backgroundHeight);
|
|
139
|
+
const polarInnerRadius = getPolarInnerRadius(config);
|
|
140
|
+
const polarMaxBarLength = getPolarMaxBarLength(config);
|
|
141
|
+
const polarBarWidth = getPolarBarWidth(config);
|
|
142
|
+
const polarColor = getPolarColor(config) || invertColor(defaultColor);
|
|
143
|
+
const polarEffect = getPolarEffect(config);
|
|
144
|
+
const polarOpacity = getPolarOpacityParsed(config);
|
|
145
|
+
|
|
146
|
+
return (params: CommonVisualizerFrameProps) => {
|
|
147
|
+
return createPolarVisualizerFrame({
|
|
148
|
+
...params,
|
|
149
|
+
centerX: polarX,
|
|
150
|
+
centerY: polarY,
|
|
151
|
+
innerRadius: polarInnerRadius,
|
|
152
|
+
maxBarLength: polarMaxBarLength,
|
|
153
|
+
barWidth: polarBarWidth,
|
|
154
|
+
color: polarColor,
|
|
155
|
+
opacity: polarOpacity,
|
|
156
|
+
spectrumEffect: polarEffect,
|
|
157
|
+
});
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
const createSpectrumVisualizerFrame = createSpectrumVisualizerFrameGenerator();
|
|
161
|
+
const spectrumWidth = getSpectrumWidthAbsolute(config, backgroundWidth);
|
|
162
|
+
const spectrumHeight = getSpectrumHeightAbsolute(config, backgroundHeight);
|
|
163
|
+
const spectrumX = getSpectrumXAbsolute(config, spectrumWidth, backgroundWidth);
|
|
164
|
+
const spectrumY = getSpectrumYAbsolute(config, spectrumHeight, backgroundHeight);
|
|
165
|
+
const spectrumRotation = getSpectrumRotation(config);
|
|
166
|
+
const spectrumColor = getSpectrumColor(config) || invertColor(defaultColor);
|
|
167
|
+
const spectrumEffect = getSpectrumEffect(config);
|
|
168
|
+
const spectrumOpacity = getSpectrumOpacityParsed(config);
|
|
169
|
+
|
|
170
|
+
return (params: CommonVisualizerFrameProps) => {
|
|
171
|
+
return createSpectrumVisualizerFrame({
|
|
172
|
+
...params,
|
|
173
|
+
size: { width: spectrumWidth, height: spectrumHeight },
|
|
174
|
+
position: { x: spectrumX, y: spectrumY },
|
|
175
|
+
rotation: spectrumRotation,
|
|
176
|
+
margin: spectrumBusMargin,
|
|
177
|
+
color: spectrumColor,
|
|
178
|
+
opacity: spectrumOpacity,
|
|
179
|
+
spectrumEffect,
|
|
180
|
+
});
|
|
181
|
+
};
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
interface PreProcessedAudio {
|
|
185
|
+
spectrums: number[][];
|
|
186
|
+
beatFrameIndices: number[];
|
|
187
|
+
beatEvents: { frameIndex: number; intensity: number }[];
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
const PRE_PROCESS_PROGRESS_SHARE = 60;
|
|
191
|
+
const POST_AUDIO_PROGRESS_SHARE = 10;
|
|
192
|
+
const RENDER_PROGRESS_SHARE = 30;
|
|
193
|
+
|
|
194
|
+
const preProcessAudio = async (
|
|
195
|
+
audioBuffer: Buffer,
|
|
196
|
+
sampleRate: number,
|
|
197
|
+
fps: number,
|
|
198
|
+
framesCount: number,
|
|
199
|
+
onPreProcessProgress?: (totalPercent: number) => void,
|
|
200
|
+
): Promise<PreProcessedAudio> => {
|
|
201
|
+
if (framesCount === 0) {
|
|
202
|
+
for (let t = 1; t <= PRE_PROCESS_PROGRESS_SHARE; t += 1) {
|
|
203
|
+
onPreProcessProgress?.(t);
|
|
204
|
+
}
|
|
205
|
+
return { spectrums: [], beatFrameIndices: [], beatEvents: [] };
|
|
206
|
+
}
|
|
207
|
+
const audioDataStep = Math.trunc(audioBuffer.length / framesCount);
|
|
208
|
+
const processingBuffer = new Float32Array(PROCESSING_BUFFER_SIZE).fill(0);
|
|
209
|
+
const skipFramesCount = fps < 45 ? 1 : 2;
|
|
210
|
+
const processSpectrum = createSpectrumsProcessor(sampleRate, skipFramesCount);
|
|
211
|
+
const detectBeat = createBeatDetector(fps);
|
|
212
|
+
|
|
213
|
+
const spectrums: number[][] = [];
|
|
214
|
+
const beatFrameIndices: number[] = [];
|
|
215
|
+
const beatEvents: { frameIndex: number; intensity: number }[] = [];
|
|
216
|
+
let nextPreProcessMilestone = 1;
|
|
217
|
+
|
|
218
|
+
for (let i = 0; i < framesCount; i++) {
|
|
219
|
+
const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
|
|
220
|
+
const frameDataLength = Math.min(currentFrameData.length, PROCESSING_BUFFER_SIZE);
|
|
221
|
+
const frameDataTailStart = currentFrameData.length - frameDataLength;
|
|
222
|
+
const frameDataToProcess = currentFrameData.slice(frameDataTailStart);
|
|
223
|
+
|
|
224
|
+
processingBuffer.copyWithin(0, frameDataLength);
|
|
225
|
+
processingBuffer.set(frameDataToProcess, PROCESSING_BUFFER_SIZE - frameDataLength);
|
|
226
|
+
|
|
227
|
+
const audioDataParser = () => Array.from(processingBuffer);
|
|
228
|
+
const spectrum = processSpectrum(audioDataParser);
|
|
229
|
+
const beat = detectBeat(spectrum);
|
|
230
|
+
|
|
231
|
+
spectrums.push(spectrum);
|
|
232
|
+
if (beat.isBeat) {
|
|
233
|
+
beatFrameIndices.push(i);
|
|
234
|
+
beatEvents.push({ frameIndex: i, intensity: beat.intensity });
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const p =
|
|
238
|
+
PRE_PROCESS_PROGRESS_SHARE * (i + 1) / framesCount;
|
|
239
|
+
while (
|
|
240
|
+
nextPreProcessMilestone <= PRE_PROCESS_PROGRESS_SHARE
|
|
241
|
+
&& p >= nextPreProcessMilestone
|
|
242
|
+
) {
|
|
243
|
+
onPreProcessProgress?.(nextPreProcessMilestone);
|
|
244
|
+
nextPreProcessMilestone += 1;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
await waitForEventLoop();
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
return { spectrums, beatFrameIndices, beatEvents };
|
|
251
|
+
};
|
|
252
|
+
|
|
253
|
+
async function prepareBackgroundForRender(params: {
|
|
254
|
+
useVideoBackground: boolean;
|
|
255
|
+
backgroundVideoPath: string | undefined;
|
|
256
|
+
backgroundImagePath: string | undefined;
|
|
257
|
+
beatFrameIndices: number[];
|
|
258
|
+
framesCount: number;
|
|
259
|
+
outputResolution: ReturnType<typeof getOutputResolution>;
|
|
260
|
+
fps: number;
|
|
261
|
+
autoEditVideo: boolean;
|
|
262
|
+
}): Promise<{
|
|
263
|
+
backgroundWidth: number;
|
|
264
|
+
backgroundHeight: number;
|
|
265
|
+
defaultColor: Color;
|
|
266
|
+
staticBackgroundBuffer: EncodedBmp;
|
|
267
|
+
videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
|
|
268
|
+
videoFrameSize: number;
|
|
269
|
+
encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
|
|
270
|
+
concatFilePath?: string;
|
|
271
|
+
}> {
|
|
272
|
+
const {
|
|
273
|
+
useVideoBackground,
|
|
274
|
+
backgroundVideoPath,
|
|
275
|
+
backgroundImagePath,
|
|
276
|
+
beatFrameIndices,
|
|
277
|
+
framesCount,
|
|
278
|
+
outputResolution,
|
|
279
|
+
fps,
|
|
280
|
+
autoEditVideo,
|
|
281
|
+
} = params;
|
|
282
|
+
|
|
283
|
+
if (useVideoBackground && backgroundVideoPath) {
|
|
284
|
+
const videoInfo = await getVideoInfo(backgroundVideoPath);
|
|
285
|
+
const sceneChanges = autoEditVideo
|
|
286
|
+
? await detectSceneChanges(backgroundVideoPath)
|
|
287
|
+
: [];
|
|
288
|
+
const backgroundWidth = outputResolution?.width ?? videoInfo.width;
|
|
289
|
+
const backgroundHeight = outputResolution?.height ?? videoInfo.height;
|
|
290
|
+
|
|
291
|
+
const videoFrameSize = backgroundWidth * backgroundHeight * 3;
|
|
292
|
+
const encodeVideoFrame = createBgrFrameEncoder({ width: backgroundWidth, height: backgroundHeight });
|
|
293
|
+
|
|
294
|
+
const segments = buildBeatSyncedSegments(
|
|
295
|
+
beatFrameIndices,
|
|
296
|
+
framesCount,
|
|
297
|
+
sceneChanges,
|
|
298
|
+
videoInfo.duration,
|
|
299
|
+
);
|
|
300
|
+
|
|
301
|
+
const concatFilePath = writeConcatFile(segments, backgroundVideoPath, videoInfo.duration, fps);
|
|
302
|
+
|
|
303
|
+
const videoFrameReader = spawnConcatVideoFrameReader({
|
|
304
|
+
concatFilePath,
|
|
305
|
+
fps,
|
|
306
|
+
totalFrames: framesCount,
|
|
307
|
+
...(outputResolution && {
|
|
308
|
+
width: backgroundWidth,
|
|
309
|
+
height: backgroundHeight,
|
|
310
|
+
sourceWidth: videoInfo.width,
|
|
311
|
+
sourceHeight: videoInfo.height,
|
|
312
|
+
}),
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
const firstFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
|
|
316
|
+
if (!firstFrame) {
|
|
317
|
+
throw new Error(`Could not read frames from video: ${backgroundVideoPath}`);
|
|
318
|
+
}
|
|
319
|
+
const defaultColor = getVideoFrameColor(firstFrame, backgroundWidth, backgroundHeight);
|
|
320
|
+
const staticBackgroundBuffer = encodeVideoFrame(firstFrame);
|
|
321
|
+
return {
|
|
322
|
+
backgroundWidth,
|
|
323
|
+
backgroundHeight,
|
|
324
|
+
defaultColor,
|
|
325
|
+
staticBackgroundBuffer,
|
|
326
|
+
videoFrameReader,
|
|
327
|
+
videoFrameSize,
|
|
328
|
+
encodeVideoFrame,
|
|
329
|
+
concatFilePath,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
if (!backgroundImagePath) {
|
|
334
|
+
throw new Error('Background image path is required when not using video background.');
|
|
335
|
+
}
|
|
336
|
+
const backgroundImageBmpBuffer = await convertToBmp(
|
|
337
|
+
backgroundImagePath,
|
|
338
|
+
outputResolution?.width,
|
|
339
|
+
outputResolution?.height,
|
|
340
|
+
);
|
|
341
|
+
const backgroundImage = parseImage(backgroundImageBmpBuffer);
|
|
342
|
+
const backgroundWidth = backgroundImage.width;
|
|
343
|
+
const backgroundHeight = backgroundImage.height;
|
|
344
|
+
const defaultColor = getImageColor(backgroundImage);
|
|
345
|
+
|
|
346
|
+
const bpmEncoder = createBpmEncoder({ width: backgroundWidth, height: backgroundHeight });
|
|
347
|
+
const staticBackgroundBuffer = bpmEncoder(backgroundImage.data);
|
|
348
|
+
return {
|
|
349
|
+
backgroundWidth,
|
|
350
|
+
backgroundHeight,
|
|
351
|
+
defaultColor,
|
|
352
|
+
staticBackgroundBuffer,
|
|
353
|
+
videoFrameSize: 0,
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
async function resolveBackgroundFrameBuffer(params: {
|
|
358
|
+
frameIndex: number;
|
|
359
|
+
useVideoBackground: boolean;
|
|
360
|
+
videoFrameReader?: ReturnType<typeof spawnVideoFrameReader>;
|
|
361
|
+
videoFrameSize: number;
|
|
362
|
+
encodeVideoFrame?: (bgrBuffer: Buffer) => EncodedBmp;
|
|
363
|
+
staticBackgroundBuffer: EncodedBmp;
|
|
364
|
+
}): Promise<EncodedBmp> {
|
|
365
|
+
const {
|
|
366
|
+
frameIndex,
|
|
367
|
+
useVideoBackground,
|
|
368
|
+
videoFrameReader,
|
|
369
|
+
videoFrameSize,
|
|
370
|
+
encodeVideoFrame,
|
|
371
|
+
staticBackgroundBuffer,
|
|
372
|
+
} = params;
|
|
373
|
+
|
|
374
|
+
if (useVideoBackground && videoFrameReader && encodeVideoFrame) {
|
|
375
|
+
if (frameIndex === 0) {
|
|
376
|
+
return staticBackgroundBuffer;
|
|
377
|
+
}
|
|
378
|
+
const videoFrame = await readVideoFrame(videoFrameReader.stdout, videoFrameSize);
|
|
379
|
+
return videoFrame
|
|
380
|
+
? encodeVideoFrame(videoFrame)
|
|
381
|
+
: staticBackgroundBuffer;
|
|
382
|
+
}
|
|
383
|
+
return staticBackgroundBuffer;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
export interface RenderAudioVisualizerResult {
|
|
387
|
+
exitCode: number;
|
|
388
|
+
/** Absolute paths of video files written successfully (exit code 0, full pass, no early stop). */
|
|
389
|
+
outputVideoFiles: string[];
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
export const renderAudioVisualizer = (config: Config, onProgress?: (progress: number) => any, shouldStop?: () => boolean) =>
|
|
393
|
+
new Promise<RenderAudioVisualizerResult>(async (resolve) => {
|
|
394
|
+
if (config.outVideo.spectrum && config.outVideo.polar) {
|
|
395
|
+
throw new Error('Cannot use both "spectrum" and "polar" options. Please specify only one visualizer type.');
|
|
396
|
+
}
|
|
397
|
+
if (!config.image && !config.video) {
|
|
398
|
+
throw new Error('Either "image" or "video" must be specified as the background source.');
|
|
399
|
+
}
|
|
400
|
+
if (config.image && config.video) {
|
|
401
|
+
throw new Error('Cannot use both "image" and "video" options. Please specify only one background source.');
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const audioFilePath = getAudioFilePath(config);
|
|
405
|
+
const outVideoPath = getOutVideoPath(config);
|
|
406
|
+
const subtitleSpec = getSubtitleRenderSpec(config);
|
|
407
|
+
let subtitleFilePath: string | undefined;
|
|
408
|
+
let subtitleFileIsTemporary = false;
|
|
409
|
+
let subtitleAlignmentAss = 2;
|
|
410
|
+
if (subtitleSpec) {
|
|
411
|
+
subtitleAlignmentAss = subtitleAlignmentToAss(subtitleSpec.alignment);
|
|
412
|
+
if (subtitleSpec.source.kind === 'file') {
|
|
413
|
+
const absPath = subtitleSpec.source.path;
|
|
414
|
+
const ext = path.extname(absPath).toLowerCase();
|
|
415
|
+
if (ext === '.lrc') {
|
|
416
|
+
const raw = readFileSync(absPath, 'utf-8');
|
|
417
|
+
subtitleFilePath = writeSubtitlesFile(lrcToSrt(raw));
|
|
418
|
+
subtitleFileIsTemporary = true;
|
|
419
|
+
} else {
|
|
420
|
+
subtitleFilePath = absPath;
|
|
421
|
+
}
|
|
422
|
+
} else {
|
|
423
|
+
subtitleFilePath = writeSubtitlesFile(
|
|
424
|
+
normalizeInlineSubtitlesToSrt(subtitleSpec.source.text),
|
|
425
|
+
);
|
|
426
|
+
subtitleFileIsTemporary = true;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
const backgroundVideoPath = getBackgroundVideoPath(config);
|
|
430
|
+
const backgroundImagePath = getBackgroundImagePath(config);
|
|
431
|
+
const useVideoBackground = !!backgroundVideoPath;
|
|
432
|
+
|
|
433
|
+
const audioReader = await createAudioBuffer(audioFilePath, FFMPEG_FORMAT);
|
|
434
|
+
const audioBuffer = audioReader.audioBuffer;
|
|
435
|
+
const sampleRate = audioReader.sampleRate;
|
|
436
|
+
if (!sampleRate) {
|
|
437
|
+
throw new Error('ffmpeg didn\'t show audio sample rate');
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
const spectrumBusMargin = getSpectrumBusMargin();
|
|
441
|
+
const FPS = getFPS(config);
|
|
442
|
+
const ffmpeg_cfr = getFfmpeg_cfr(config);
|
|
443
|
+
const ffmpeg_preset = getFfmpeg_preset(config);
|
|
444
|
+
const frame_processing_delay = getFrame_processing_delay(config);
|
|
445
|
+
|
|
446
|
+
const audioDuration = audioBuffer.length / sampleRate;
|
|
447
|
+
const framesCount = Math.trunc(audioDuration * FPS);
|
|
448
|
+
const outputResolution = getOutputResolution(config);
|
|
449
|
+
|
|
450
|
+
let maxProgressReported = -1;
|
|
451
|
+
const reportProgress = (progress: number) => {
|
|
452
|
+
if (!onProgress) {
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
const normalized = +Math.min(100, Math.max(0, progress)).toFixed(2);
|
|
456
|
+
if (normalized <= maxProgressReported) {
|
|
457
|
+
return;
|
|
458
|
+
}
|
|
459
|
+
maxProgressReported = normalized;
|
|
460
|
+
onProgress(normalized);
|
|
461
|
+
};
|
|
462
|
+
reportProgress(0);
|
|
463
|
+
|
|
464
|
+
const preprocessed = await preProcessAudio(
|
|
465
|
+
audioBuffer,
|
|
466
|
+
sampleRate,
|
|
467
|
+
FPS,
|
|
468
|
+
framesCount,
|
|
469
|
+
(progress: number) => {
|
|
470
|
+
reportProgress(progress);
|
|
471
|
+
},
|
|
472
|
+
);
|
|
473
|
+
reportProgress(PRE_PROCESS_PROGRESS_SHARE);
|
|
474
|
+
|
|
475
|
+
const autoHighlight = getAudioAutoHighlight(config);
|
|
476
|
+
let spectrumsForRender = preprocessed.spectrums;
|
|
477
|
+
let beatIndicesForRender = preprocessed.beatFrameIndices;
|
|
478
|
+
let framesCountForRender = framesCount;
|
|
479
|
+
type HighlightSliceResult = ReturnType<typeof computeHighlightSlice> extends Promise<
|
|
480
|
+
infer R
|
|
481
|
+
>
|
|
482
|
+
? R
|
|
483
|
+
: never;
|
|
484
|
+
let highlightSlice: HighlightSliceResult | undefined;
|
|
485
|
+
|
|
486
|
+
if (autoHighlight) {
|
|
487
|
+
reportProgress(PRE_PROCESS_PROGRESS_SHARE + 1);
|
|
488
|
+
highlightSlice = await computeHighlightSlice(
|
|
489
|
+
FPS,
|
|
490
|
+
framesCount,
|
|
491
|
+
preprocessed.spectrums,
|
|
492
|
+
preprocessed.beatEvents,
|
|
493
|
+
getAudioAutoHighlightCount(config),
|
|
494
|
+
);
|
|
495
|
+
spectrumsForRender = highlightSlice.spectrums;
|
|
496
|
+
beatIndicesForRender = highlightSlice.beatFrameIndices;
|
|
497
|
+
framesCountForRender = highlightSlice.highlightFrames;
|
|
498
|
+
reportProgress(PRE_PROCESS_PROGRESS_SHARE + POST_AUDIO_PROGRESS_SHARE);
|
|
499
|
+
} else {
|
|
500
|
+
reportProgress(PRE_PROCESS_PROGRESS_SHARE + POST_AUDIO_PROGRESS_SHARE);
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const separateHighlightFiles =
|
|
504
|
+
autoHighlight &&
|
|
505
|
+
highlightSlice &&
|
|
506
|
+
highlightSlice.runs.length > 1;
|
|
507
|
+
|
|
508
|
+
type VideoRenderPass = {
|
|
509
|
+
outPath: string;
|
|
510
|
+
spectrums: number[][];
|
|
511
|
+
beatIndices: number[];
|
|
512
|
+
frameCount: number;
|
|
513
|
+
audioSegment: import('./highlight').HighlightAudioSegment | undefined;
|
|
514
|
+
};
|
|
515
|
+
|
|
516
|
+
const passes: VideoRenderPass[] =
|
|
517
|
+
separateHighlightFiles && highlightSlice
|
|
518
|
+
? highlightSlice.runs.map((run, i) => {
|
|
519
|
+
const dir = path.dirname(outVideoPath);
|
|
520
|
+
const ext = path.extname(outVideoPath);
|
|
521
|
+
const base = path.basename(outVideoPath, ext);
|
|
522
|
+
const numberedPath = path.resolve(dir, `${base}-${i + 1}${ext}`);
|
|
523
|
+
return {
|
|
524
|
+
outPath: numberedPath,
|
|
525
|
+
spectrums: run.spectrums,
|
|
526
|
+
beatIndices: run.beatFrameIndices,
|
|
527
|
+
frameCount: run.highlightFrames,
|
|
528
|
+
audioSegment: run.audioSegment,
|
|
529
|
+
};
|
|
530
|
+
})
|
|
531
|
+
: [
|
|
532
|
+
{
|
|
533
|
+
outPath: outVideoPath,
|
|
534
|
+
spectrums: spectrumsForRender,
|
|
535
|
+
beatIndices: beatIndicesForRender,
|
|
536
|
+
frameCount: framesCountForRender,
|
|
537
|
+
audioSegment:
|
|
538
|
+
autoHighlight &&
|
|
539
|
+
highlightSlice &&
|
|
540
|
+
highlightSlice.audioSegments.length > 0
|
|
541
|
+
? highlightSlice.audioSegments[0]
|
|
542
|
+
: undefined,
|
|
543
|
+
},
|
|
544
|
+
];
|
|
545
|
+
|
|
546
|
+
const totalPassFrames =
|
|
547
|
+
passes.reduce((sum, pass) => sum + pass.frameCount, 0);
|
|
548
|
+
const totalRenderMilestones = totalPassFrames + (passes.length * 2);
|
|
549
|
+
let renderMilestonesCompleted = 0;
|
|
550
|
+
const reportRenderProgress = (milestoneIncrement = 1) => {
|
|
551
|
+
renderMilestonesCompleted += milestoneIncrement;
|
|
552
|
+
reportProgress(
|
|
553
|
+
PRE_PROCESS_PROGRESS_SHARE
|
|
554
|
+
+ POST_AUDIO_PROGRESS_SHARE
|
|
555
|
+
+ (RENDER_PROGRESS_SHARE * renderMilestonesCompleted / Math.max(1, totalRenderMilestones)),
|
|
556
|
+
);
|
|
557
|
+
};
|
|
558
|
+
let lastExitCode = 0;
|
|
559
|
+
const outputVideoFiles: string[] = [];
|
|
560
|
+
try {
|
|
561
|
+
passLoop: for (const pass of passes) {
|
|
562
|
+
const {
|
|
563
|
+
backgroundWidth,
|
|
564
|
+
backgroundHeight,
|
|
565
|
+
defaultColor,
|
|
566
|
+
staticBackgroundBuffer,
|
|
567
|
+
videoFrameReader,
|
|
568
|
+
videoFrameSize,
|
|
569
|
+
encodeVideoFrame,
|
|
570
|
+
concatFilePath,
|
|
571
|
+
} = await prepareBackgroundForRender({
|
|
572
|
+
useVideoBackground,
|
|
573
|
+
backgroundVideoPath,
|
|
574
|
+
backgroundImagePath,
|
|
575
|
+
beatFrameIndices: pass.beatIndices,
|
|
576
|
+
framesCount: pass.frameCount,
|
|
577
|
+
outputResolution,
|
|
578
|
+
fps: FPS,
|
|
579
|
+
autoEditVideo: getAutoEditVideo(config),
|
|
580
|
+
});
|
|
581
|
+
reportRenderProgress();
|
|
582
|
+
|
|
583
|
+
const createVisualizerFrame = createVisualizerFrameGenerator(
|
|
584
|
+
config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
|
|
585
|
+
);
|
|
586
|
+
|
|
587
|
+
const ffmpegVideoWriter = spawnFfmpegVideoWriter({
|
|
588
|
+
audioFilename: audioFilePath,
|
|
589
|
+
videoFileName: pass.outPath,
|
|
590
|
+
...(subtitleFilePath && {
|
|
591
|
+
subtitleFilename: subtitleFilePath,
|
|
592
|
+
subtitleAlignmentAss,
|
|
593
|
+
}),
|
|
594
|
+
fps: FPS,
|
|
595
|
+
...(pass.audioSegment && { audioSegment: pass.audioSegment }),
|
|
596
|
+
...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
|
|
597
|
+
...(ffmpeg_preset && { preset: ffmpeg_preset }),
|
|
598
|
+
});
|
|
599
|
+
|
|
600
|
+
const exitPromise = new Promise<number>(res => {
|
|
601
|
+
let settled = false;
|
|
602
|
+
ffmpegVideoWriter.on('exit', (code: number | null) => {
|
|
603
|
+
if (settled) {
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
settled = true;
|
|
607
|
+
res(code ?? 0);
|
|
608
|
+
});
|
|
609
|
+
});
|
|
610
|
+
|
|
611
|
+
let stoppedEarly = false;
|
|
612
|
+
for (let i = 0; i < pass.frameCount; i++) {
|
|
613
|
+
const spectrum = pass.spectrums[i];
|
|
614
|
+
|
|
615
|
+
const backgroundImageBuffer = await resolveBackgroundFrameBuffer({
|
|
616
|
+
frameIndex: i,
|
|
617
|
+
useVideoBackground,
|
|
618
|
+
videoFrameReader,
|
|
619
|
+
videoFrameSize,
|
|
620
|
+
encodeVideoFrame,
|
|
621
|
+
staticBackgroundBuffer,
|
|
622
|
+
});
|
|
623
|
+
|
|
624
|
+
const commonVisualizerFrameProps: CommonVisualizerFrameProps = {
|
|
625
|
+
backgroundImageBuffer,
|
|
626
|
+
spectrum,
|
|
627
|
+
};
|
|
628
|
+
const frameImage = createVisualizerFrame(commonVisualizerFrameProps);
|
|
629
|
+
const isFrameProcessed = ffmpegVideoWriter.stdin.write(frameImage.data);
|
|
630
|
+
if (!isFrameProcessed) {
|
|
631
|
+
const isDrained = await waitDrain(ffmpegVideoWriter.stdin, ffmpegVideoWriter);
|
|
632
|
+
if (!isDrained) {
|
|
633
|
+
stoppedEarly = true;
|
|
634
|
+
break;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
if (shouldStop && shouldStop()) {
|
|
638
|
+
stoppedEarly = true;
|
|
639
|
+
break;
|
|
640
|
+
}
|
|
641
|
+
if (frame_processing_delay) {
|
|
642
|
+
await sleep(frame_processing_delay);
|
|
643
|
+
}
|
|
644
|
+
await waitForEventLoop();
|
|
645
|
+
reportRenderProgress();
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
if (videoFrameReader) {
|
|
649
|
+
videoFrameReader.kill();
|
|
650
|
+
}
|
|
651
|
+
if (concatFilePath) {
|
|
652
|
+
cleanupConcatFile(concatFilePath);
|
|
653
|
+
}
|
|
654
|
+
ffmpegVideoWriter.stdin.end();
|
|
655
|
+
|
|
656
|
+
lastExitCode = await exitPromise;
|
|
657
|
+
reportRenderProgress();
|
|
658
|
+
|
|
659
|
+
if (lastExitCode === 0 && !stoppedEarly) {
|
|
660
|
+
outputVideoFiles.push(pass.outPath);
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
if (stoppedEarly || lastExitCode !== 0) {
|
|
664
|
+
break passLoop;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
if (lastExitCode === 0 && outputVideoFiles.length === passes.length) {
|
|
669
|
+
reportProgress(100);
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
resolve({ exitCode: lastExitCode, outputVideoFiles });
|
|
673
|
+
} finally {
|
|
674
|
+
if (subtitleFilePath && subtitleFileIsTemporary) {
|
|
675
|
+
cleanupTempFile(subtitleFilePath);
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
});
|