nodejs-audio-visualizer 4.1.2 → 4.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/config.d.ts +2 -0
- package/dist/config.js +16 -0
- package/dist/config.js.map +1 -1
- package/dist/highlight.d.ts +32 -0
- package/dist/highlight.js +326 -0
- package/dist/highlight.js.map +1 -0
- package/dist/index.d.ts +10 -1
- package/dist/index.js +199 -72
- package/dist/index.js.map +1 -1
- package/dist/video.d.ts +9 -0
- package/dist/video.js +8 -9
- package/dist/video.js.map +1 -1
- package/dist/waitForEventLoop.d.ts +1 -0
- package/dist/waitForEventLoop.js +6 -0
- package/dist/waitForEventLoop.js.map +1 -0
- package/package.json +1 -1
- package/src/config.ts +21 -0
- package/src/highlight.ts +284 -0
- package/src/index.ts +219 -64
- package/src/tests/config.test.ts +32 -0
- package/src/tests/highlight.test.ts +178 -0
- package/src/tests/index.test.ts +11 -4
- package/src/tests/video.test.ts +36 -1
- package/src/video.ts +20 -3
- package/src/waitForEventLoop.ts +2 -0
package/src/highlight.ts
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/** Output length per segment when `audio.autoHighlight` is true (seconds). */
|
|
2
|
+
|
|
3
|
+
import { waitForEventLoop } from "./waitForEventLoop";
|
|
4
|
+
|
|
5
|
+
export const HIGHLIGHT_DURATION_SEC = 15;
|
|
6
|
+
|
|
7
|
+
export interface BeatFrameEvent {
|
|
8
|
+
frameIndex: number;
|
|
9
|
+
intensity: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface HighlightAudioSegment {
|
|
13
|
+
seekSeconds: number;
|
|
14
|
+
durationSeconds: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** One auto-highlight segment before concatenation (for separate output files). */
|
|
18
|
+
export interface HighlightRun {
|
|
19
|
+
startFrame: number;
|
|
20
|
+
highlightFrames: number;
|
|
21
|
+
spectrums: number[][];
|
|
22
|
+
beatFrameIndices: number[];
|
|
23
|
+
audioSegment: HighlightAudioSegment;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function buildHighlightRun(
|
|
27
|
+
fps: number,
|
|
28
|
+
seg: { startFrame: number; highlightFrames: number },
|
|
29
|
+
spectrums: number[][],
|
|
30
|
+
beatEvents: BeatFrameEvent[],
|
|
31
|
+
): HighlightRun {
|
|
32
|
+
const { startFrame, highlightFrames } = seg;
|
|
33
|
+
const spectrumsSlice = spectrums.slice(startFrame, startFrame + highlightFrames);
|
|
34
|
+
const beatFrameIndices = beatEvents
|
|
35
|
+
.map(b => b.frameIndex)
|
|
36
|
+
.filter(i => i >= startFrame && i < startFrame + highlightFrames)
|
|
37
|
+
.map(i => i - startFrame)
|
|
38
|
+
.sort((a, b) => a - b);
|
|
39
|
+
return {
|
|
40
|
+
startFrame,
|
|
41
|
+
highlightFrames,
|
|
42
|
+
spectrums: spectrumsSlice,
|
|
43
|
+
beatFrameIndices,
|
|
44
|
+
audioSegment: {
|
|
45
|
+
seekSeconds: startFrame / fps,
|
|
46
|
+
durationSeconds: highlightFrames / fps,
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function frameEnergy(spectrum: number[]): number {
|
|
52
|
+
let s = 0;
|
|
53
|
+
for (let i = 0; i < spectrum.length; i++) {
|
|
54
|
+
const v = spectrum[i];
|
|
55
|
+
s += v * v;
|
|
56
|
+
}
|
|
57
|
+
return s;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function rangesOverlap(
|
|
61
|
+
aStart: number,
|
|
62
|
+
aEndExclusive: number,
|
|
63
|
+
ranges: Array<{ start: number; endExclusive: number }>,
|
|
64
|
+
): boolean {
|
|
65
|
+
for (const r of ranges) {
|
|
66
|
+
if (aStart < r.endExclusive && aEndExclusive > r.start) {
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async function buildFrameEnergies(
|
|
74
|
+
spectrums: number[][],
|
|
75
|
+
totalFrames: number,
|
|
76
|
+
): Promise<Float64Array> {
|
|
77
|
+
const energies = new Float64Array(totalFrames);
|
|
78
|
+
for (let i = 0; i < totalFrames; i++) {
|
|
79
|
+
energies[i] = frameEnergy(spectrums[i] ?? []);
|
|
80
|
+
await waitForEventLoop();
|
|
81
|
+
}
|
|
82
|
+
return energies;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Chooses the fixed-length window whose frames have the highest summed spectral energy
|
|
87
|
+
* (typical chorus / drop vs a single loud intro transient).
|
|
88
|
+
* Skips windows that overlap any excluded range.
|
|
89
|
+
*/
|
|
90
|
+
async function findBestHighlightStart(
|
|
91
|
+
energies: Float64Array,
|
|
92
|
+
totalFrames: number,
|
|
93
|
+
windowFrames: number,
|
|
94
|
+
excludeRanges: Array<{ start: number; endExclusive: number }>,
|
|
95
|
+
): Promise<number | null> {
|
|
96
|
+
if (totalFrames <= windowFrames) {
|
|
97
|
+
if (!rangesOverlap(0, totalFrames, excludeRanges)) {
|
|
98
|
+
return 0;
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
let minE = Infinity;
|
|
104
|
+
let maxE = -Infinity;
|
|
105
|
+
for (let i = 0; i < totalFrames; i++) {
|
|
106
|
+
const e = energies[i];
|
|
107
|
+
if (e < minE) {
|
|
108
|
+
minE = e;
|
|
109
|
+
}
|
|
110
|
+
if (e > maxE) {
|
|
111
|
+
maxE = e;
|
|
112
|
+
}
|
|
113
|
+
await waitForEventLoop();
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const prefix = new Float64Array(totalFrames + 1);
|
|
117
|
+
for (let i = 0; i < totalFrames; i++) {
|
|
118
|
+
prefix[i + 1] = prefix[i] + energies[i];
|
|
119
|
+
await waitForEventLoop();
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const maxStart = totalFrames - windowFrames;
|
|
123
|
+
let bestStart: number | null = null;
|
|
124
|
+
let bestScore = -Infinity;
|
|
125
|
+
|
|
126
|
+
for (let start = 0; start <= maxStart; start++) {
|
|
127
|
+
if (rangesOverlap(start, start + windowFrames, excludeRanges)) {
|
|
128
|
+
await waitForEventLoop();
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const score = prefix[start + windowFrames] - prefix[start];
|
|
132
|
+
if (score > bestScore) {
|
|
133
|
+
bestScore = score;
|
|
134
|
+
bestStart = start;
|
|
135
|
+
}
|
|
136
|
+
await waitForEventLoop();
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (bestStart === null) {
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
if (maxE <= minE) {
|
|
144
|
+
const center = Math.floor((totalFrames - windowFrames) / 2);
|
|
145
|
+
let bestDist = Infinity;
|
|
146
|
+
let tieBreak: number | null = null;
|
|
147
|
+
for (let start = 0; start <= maxStart; start++) {
|
|
148
|
+
if (rangesOverlap(start, start + windowFrames, excludeRanges)) {
|
|
149
|
+
await waitForEventLoop();
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const dist = Math.abs(start - center);
|
|
153
|
+
if (dist < bestDist) {
|
|
154
|
+
bestDist = dist;
|
|
155
|
+
tieBreak = start;
|
|
156
|
+
}
|
|
157
|
+
await waitForEventLoop();
|
|
158
|
+
}
|
|
159
|
+
return tieBreak;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
return bestStart;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Picks up to `segmentCount` non-overlapping 15s windows with highest summed spectral energy each,
|
|
168
|
+
* sorts them chronologically, concatenates spectrums, and remaps beat indices.
|
|
169
|
+
*/
|
|
170
|
+
export async function computeHighlightSlice(
|
|
171
|
+
fps: number,
|
|
172
|
+
totalFrames: number,
|
|
173
|
+
spectrums: number[][],
|
|
174
|
+
beatEvents: BeatFrameEvent[],
|
|
175
|
+
segmentCount = 1,
|
|
176
|
+
): Promise<{
|
|
177
|
+
startFrame: number;
|
|
178
|
+
highlightFrames: number;
|
|
179
|
+
spectrums: number[][];
|
|
180
|
+
beatFrameIndices: number[];
|
|
181
|
+
audioSeekSeconds: number;
|
|
182
|
+
audioDurationSeconds: number;
|
|
183
|
+
audioSegments: HighlightAudioSegment[];
|
|
184
|
+
runs: HighlightRun[];
|
|
185
|
+
}> {
|
|
186
|
+
const highlightFramesTarget = Math.ceil(HIGHLIGHT_DURATION_SEC * fps);
|
|
187
|
+
|
|
188
|
+
if (totalFrames <= 0) {
|
|
189
|
+
return {
|
|
190
|
+
startFrame: 0,
|
|
191
|
+
highlightFrames: 0,
|
|
192
|
+
spectrums: [],
|
|
193
|
+
beatFrameIndices: [],
|
|
194
|
+
audioSeekSeconds: 0,
|
|
195
|
+
audioDurationSeconds: 0,
|
|
196
|
+
audioSegments: [],
|
|
197
|
+
runs: [],
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (totalFrames <= highlightFramesTarget) {
|
|
202
|
+
const audioSegments: HighlightAudioSegment[] = [
|
|
203
|
+
{
|
|
204
|
+
seekSeconds: 0,
|
|
205
|
+
durationSeconds: totalFrames / fps,
|
|
206
|
+
},
|
|
207
|
+
];
|
|
208
|
+
const fullSeg = { startFrame: 0, highlightFrames: totalFrames };
|
|
209
|
+
return {
|
|
210
|
+
startFrame: 0,
|
|
211
|
+
highlightFrames: totalFrames,
|
|
212
|
+
spectrums: spectrums.slice(),
|
|
213
|
+
beatFrameIndices: beatEvents.map(b => b.frameIndex).sort((a, b) => a - b),
|
|
214
|
+
audioSeekSeconds: 0,
|
|
215
|
+
audioDurationSeconds: totalFrames / fps,
|
|
216
|
+
audioSegments,
|
|
217
|
+
runs: [buildHighlightRun(fps, fullSeg, spectrums, beatEvents)],
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
const highlightFrames = highlightFramesTarget;
|
|
222
|
+
const energies = await buildFrameEnergies(spectrums, totalFrames);
|
|
223
|
+
const excludeRanges: Array<{ start: number; endExclusive: number }> = [];
|
|
224
|
+
const rawSegments: Array<{ startFrame: number; highlightFrames: number }> = [];
|
|
225
|
+
|
|
226
|
+
const n = Math.max(1, Math.floor(segmentCount));
|
|
227
|
+
for (let k = 0; k < n; k++) {
|
|
228
|
+
const startFrame = await findBestHighlightStart(
|
|
229
|
+
energies,
|
|
230
|
+
totalFrames,
|
|
231
|
+
highlightFrames,
|
|
232
|
+
excludeRanges,
|
|
233
|
+
);
|
|
234
|
+
if (startFrame === null) {
|
|
235
|
+
break;
|
|
236
|
+
}
|
|
237
|
+
rawSegments.push({ startFrame, highlightFrames });
|
|
238
|
+
excludeRanges.push({ start: startFrame, endExclusive: startFrame + highlightFrames });
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
rawSegments.sort((a, b) => a.startFrame - b.startFrame);
|
|
242
|
+
|
|
243
|
+
const runs = rawSegments.map(seg =>
|
|
244
|
+
buildHighlightRun(fps, seg, spectrums, beatEvents),
|
|
245
|
+
);
|
|
246
|
+
|
|
247
|
+
const slicedSpectrums: number[][] = [];
|
|
248
|
+
const beatFrameIndices: number[] = [];
|
|
249
|
+
let outOffset = 0;
|
|
250
|
+
|
|
251
|
+
for (const seg of rawSegments) {
|
|
252
|
+
const chunk = spectrums.slice(seg.startFrame, seg.startFrame + seg.highlightFrames);
|
|
253
|
+
slicedSpectrums.push(...chunk);
|
|
254
|
+
|
|
255
|
+
for (const b of beatEvents) {
|
|
256
|
+
const i = b.frameIndex;
|
|
257
|
+
if (i >= seg.startFrame && i < seg.startFrame + seg.highlightFrames) {
|
|
258
|
+
beatFrameIndices.push(i - seg.startFrame + outOffset);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
outOffset += seg.highlightFrames;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
beatFrameIndices.sort((a, b) => a - b);
|
|
265
|
+
|
|
266
|
+
const totalHighlightFrames = slicedSpectrums.length;
|
|
267
|
+
const audioSegments: HighlightAudioSegment[] = rawSegments.map(seg => ({
|
|
268
|
+
seekSeconds: seg.startFrame / fps,
|
|
269
|
+
durationSeconds: seg.highlightFrames / fps,
|
|
270
|
+
}));
|
|
271
|
+
|
|
272
|
+
const first = audioSegments[0] ?? { seekSeconds: 0, durationSeconds: 0 };
|
|
273
|
+
|
|
274
|
+
return {
|
|
275
|
+
startFrame: rawSegments[0]?.startFrame ?? 0,
|
|
276
|
+
highlightFrames: totalHighlightFrames,
|
|
277
|
+
spectrums: slicedSpectrums,
|
|
278
|
+
beatFrameIndices,
|
|
279
|
+
audioSeekSeconds: first.seekSeconds,
|
|
280
|
+
audioDurationSeconds: first.durationSeconds,
|
|
281
|
+
audioSegments,
|
|
282
|
+
runs,
|
|
283
|
+
};
|
|
284
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import path from 'path';
|
|
1
2
|
import {
|
|
2
3
|
getAudioFilePath,
|
|
3
4
|
getBackgroundImagePath,
|
|
@@ -28,13 +29,18 @@ import {
|
|
|
28
29
|
getPolarColor,
|
|
29
30
|
getPolarOpacityParsed,
|
|
30
31
|
getAutoEditVideo,
|
|
32
|
+
getAudioAutoHighlight,
|
|
33
|
+
getAudioAutoHighlightCount,
|
|
31
34
|
} from './config';
|
|
32
35
|
import { createAudioBuffer, bufferToUInt8, createSpectrumsProcessor } from './audio';
|
|
33
36
|
import { parseImage, getImageColor, getVideoFrameColor, invertColor, Color, convertToBmp, createSpectrumVisualizerFrameGenerator, createPolarVisualizerFrameGenerator, CreatePolarVisualizerFrameProps, CreateVisualizerFrameProps, CommonVisualizerFrameProps } from './image';
|
|
34
|
-
import { spawnFfmpegVideoWriter, getProgress,
|
|
37
|
+
import { spawnFfmpegVideoWriter, getProgress, waitDrain, getVideoInfo, spawnVideoFrameReader, readVideoFrame, detectSceneChanges, buildBeatSyncedSegments, writeConcatFile, spawnConcatVideoFrameReader, cleanupConcatFile } from './video';
|
|
35
38
|
import { createBpmEncoder, createBgrFrameEncoder, EncodedBmp } from './bpmEncoder';
|
|
36
39
|
import { createBeatDetector } from './beats';
|
|
37
40
|
export { BeatInfo, BeatDetectorOptions } from './beats';
|
|
41
|
+
import { computeHighlightSlice, HIGHLIGHT_DURATION_SEC } from './highlight';
|
|
42
|
+
import { waitForEventLoop } from './waitForEventLoop';
|
|
43
|
+
export { computeHighlightSlice, HIGHLIGHT_DURATION_SEC, BeatFrameEvent, HighlightAudioSegment, HighlightRun } from './highlight';
|
|
38
44
|
|
|
39
45
|
export const PCM_FORMAT = {
|
|
40
46
|
bit: 8,
|
|
@@ -46,7 +52,10 @@ const PROCESSING_BUFFER_SIZE = Math.pow(2, 12);
|
|
|
46
52
|
|
|
47
53
|
export interface Config {
|
|
48
54
|
audio: {
|
|
49
|
-
path: string
|
|
55
|
+
path: string;
|
|
56
|
+
autoHighlight?: boolean;
|
|
57
|
+
/** When `autoHighlight` is true, number of non-overlapping 15s windows to stitch (default 1). */
|
|
58
|
+
autoHighlightCount?: number;
|
|
50
59
|
};
|
|
51
60
|
image?: {
|
|
52
61
|
path: string;
|
|
@@ -164,16 +173,24 @@ const createVisualizerFrameGenerator = (
|
|
|
164
173
|
interface PreProcessedAudio {
|
|
165
174
|
spectrums: number[][];
|
|
166
175
|
beatFrameIndices: number[];
|
|
176
|
+
beatEvents: { frameIndex: number; intensity: number }[];
|
|
167
177
|
}
|
|
168
178
|
|
|
169
|
-
const
|
|
179
|
+
const PRE_PROCESS_PROGRESS_SHARE = 10;
|
|
170
180
|
|
|
171
181
|
const preProcessAudio = async (
|
|
172
182
|
audioBuffer: Buffer,
|
|
173
183
|
sampleRate: number,
|
|
174
184
|
fps: number,
|
|
175
185
|
framesCount: number,
|
|
186
|
+
onPreProcessProgress?: (totalPercent: number) => void,
|
|
176
187
|
): Promise<PreProcessedAudio> => {
|
|
188
|
+
if (framesCount === 0) {
|
|
189
|
+
for (let t = 1; t <= PRE_PROCESS_PROGRESS_SHARE; t += 1) {
|
|
190
|
+
onPreProcessProgress?.(t);
|
|
191
|
+
}
|
|
192
|
+
return { spectrums: [], beatFrameIndices: [], beatEvents: [] };
|
|
193
|
+
}
|
|
177
194
|
const audioDataStep = Math.trunc(audioBuffer.length / framesCount);
|
|
178
195
|
const processingBuffer = new Float32Array(PROCESSING_BUFFER_SIZE).fill(0);
|
|
179
196
|
const skipFramesCount = fps < 45 ? 1 : 2;
|
|
@@ -182,6 +199,8 @@ const preProcessAudio = async (
|
|
|
182
199
|
|
|
183
200
|
const spectrums: number[][] = [];
|
|
184
201
|
const beatFrameIndices: number[] = [];
|
|
202
|
+
const beatEvents: { frameIndex: number; intensity: number }[] = [];
|
|
203
|
+
let nextPreProcessMilestone = 1;
|
|
185
204
|
|
|
186
205
|
for (let i = 0; i < framesCount; i++) {
|
|
187
206
|
const currentFrameData = PCM_FORMAT.parseFunction(audioBuffer, i * audioDataStep, i * audioDataStep + audioDataStep);
|
|
@@ -195,14 +214,23 @@ const preProcessAudio = async (
|
|
|
195
214
|
spectrums.push(spectrum);
|
|
196
215
|
if (beat.isBeat) {
|
|
197
216
|
beatFrameIndices.push(i);
|
|
217
|
+
beatEvents.push({ frameIndex: i, intensity: beat.intensity });
|
|
198
218
|
}
|
|
199
219
|
|
|
200
|
-
|
|
201
|
-
|
|
220
|
+
const p =
|
|
221
|
+
PRE_PROCESS_PROGRESS_SHARE * (i + 1) / framesCount;
|
|
222
|
+
while (
|
|
223
|
+
nextPreProcessMilestone <= PRE_PROCESS_PROGRESS_SHARE
|
|
224
|
+
&& p >= nextPreProcessMilestone
|
|
225
|
+
) {
|
|
226
|
+
onPreProcessProgress?.(nextPreProcessMilestone);
|
|
227
|
+
nextPreProcessMilestone += 1;
|
|
202
228
|
}
|
|
229
|
+
|
|
230
|
+
await waitForEventLoop();
|
|
203
231
|
}
|
|
204
232
|
|
|
205
|
-
return { spectrums, beatFrameIndices };
|
|
233
|
+
return { spectrums, beatFrameIndices, beatEvents };
|
|
206
234
|
};
|
|
207
235
|
|
|
208
236
|
async function prepareBackgroundForRender(params: {
|
|
@@ -338,8 +366,14 @@ async function resolveBackgroundFrameBuffer(params: {
|
|
|
338
366
|
return staticBackgroundBuffer;
|
|
339
367
|
}
|
|
340
368
|
|
|
369
|
+
export interface RenderAudioVisualizerResult {
|
|
370
|
+
exitCode: number;
|
|
371
|
+
/** Absolute paths of video files written successfully (exit code 0, full pass, no early stop). */
|
|
372
|
+
outputVideoFiles: string[];
|
|
373
|
+
}
|
|
374
|
+
|
|
341
375
|
export const renderAudioVisualizer = (config: Config, onProgress?: (progress: number) => any, shouldStop?: () => boolean) =>
|
|
342
|
-
new Promise<
|
|
376
|
+
new Promise<RenderAudioVisualizerResult>(async (resolve) => {
|
|
343
377
|
if (config.outVideo.spectrum && config.outVideo.polar) {
|
|
344
378
|
throw new Error('Cannot use both "spectrum" and "polar" options. Please specify only one visualizer type.');
|
|
345
379
|
}
|
|
@@ -376,76 +410,197 @@ export const renderAudioVisualizer = (config: Config, onProgress?: (progress: nu
|
|
|
376
410
|
if (onProgress) {
|
|
377
411
|
onProgress(0);
|
|
378
412
|
}
|
|
379
|
-
const preprocessed = await preProcessAudio(
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
backgroundHeight,
|
|
384
|
-
defaultColor,
|
|
385
|
-
staticBackgroundBuffer,
|
|
386
|
-
videoFrameReader,
|
|
387
|
-
videoFrameSize,
|
|
388
|
-
encodeVideoFrame,
|
|
389
|
-
concatFilePath,
|
|
390
|
-
} = await prepareBackgroundForRender({
|
|
391
|
-
useVideoBackground,
|
|
392
|
-
backgroundVideoPath,
|
|
393
|
-
backgroundImagePath,
|
|
394
|
-
beatFrameIndices: preprocessed.beatFrameIndices,
|
|
413
|
+
const preprocessed = await preProcessAudio(
|
|
414
|
+
audioBuffer,
|
|
415
|
+
sampleRate,
|
|
416
|
+
FPS,
|
|
395
417
|
framesCount,
|
|
396
|
-
|
|
397
|
-
fps: FPS,
|
|
398
|
-
autoEditVideo: getAutoEditVideo(config),
|
|
399
|
-
});
|
|
400
|
-
|
|
401
|
-
const createVisualizerFrame = createVisualizerFrameGenerator(
|
|
402
|
-
config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
|
|
418
|
+
onProgress,
|
|
403
419
|
);
|
|
404
420
|
|
|
405
|
-
const
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
421
|
+
const autoHighlight = getAudioAutoHighlight(config);
|
|
422
|
+
let spectrumsForRender = preprocessed.spectrums;
|
|
423
|
+
let beatIndicesForRender = preprocessed.beatFrameIndices;
|
|
424
|
+
let framesCountForRender = framesCount;
|
|
425
|
+
type HighlightSliceResult = ReturnType<typeof computeHighlightSlice> extends Promise<
|
|
426
|
+
infer R
|
|
427
|
+
>
|
|
428
|
+
? R
|
|
429
|
+
: never;
|
|
430
|
+
let highlightSlice: HighlightSliceResult | undefined;
|
|
431
|
+
|
|
432
|
+
if (autoHighlight) {
|
|
433
|
+
highlightSlice = await computeHighlightSlice(
|
|
434
|
+
FPS,
|
|
435
|
+
framesCount,
|
|
436
|
+
preprocessed.spectrums,
|
|
437
|
+
preprocessed.beatEvents,
|
|
438
|
+
getAudioAutoHighlightCount(config),
|
|
439
|
+
);
|
|
440
|
+
spectrumsForRender = highlightSlice.spectrums;
|
|
441
|
+
beatIndicesForRender = highlightSlice.beatFrameIndices;
|
|
442
|
+
framesCountForRender = highlightSlice.highlightFrames;
|
|
443
|
+
}
|
|
414
444
|
|
|
415
|
-
|
|
416
|
-
|
|
445
|
+
const separateHighlightFiles =
|
|
446
|
+
autoHighlight &&
|
|
447
|
+
highlightSlice &&
|
|
448
|
+
highlightSlice.runs.length > 1;
|
|
449
|
+
|
|
450
|
+
type VideoRenderPass = {
|
|
451
|
+
outPath: string;
|
|
452
|
+
spectrums: number[][];
|
|
453
|
+
beatIndices: number[];
|
|
454
|
+
frameCount: number;
|
|
455
|
+
audioSegment: import('./highlight').HighlightAudioSegment | undefined;
|
|
456
|
+
};
|
|
417
457
|
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
458
|
+
const passes: VideoRenderPass[] =
|
|
459
|
+
separateHighlightFiles && highlightSlice
|
|
460
|
+
? highlightSlice.runs.map((run, i) => {
|
|
461
|
+
const dir = path.dirname(outVideoPath);
|
|
462
|
+
const ext = path.extname(outVideoPath);
|
|
463
|
+
const base = path.basename(outVideoPath, ext);
|
|
464
|
+
const numberedPath = path.resolve(dir, `${base}-${i + 1}${ext}`);
|
|
465
|
+
return {
|
|
466
|
+
outPath: numberedPath,
|
|
467
|
+
spectrums: run.spectrums,
|
|
468
|
+
beatIndices: run.beatFrameIndices,
|
|
469
|
+
frameCount: run.highlightFrames,
|
|
470
|
+
audioSegment: run.audioSegment,
|
|
471
|
+
};
|
|
472
|
+
})
|
|
473
|
+
: [
|
|
474
|
+
{
|
|
475
|
+
outPath: outVideoPath,
|
|
476
|
+
spectrums: spectrumsForRender,
|
|
477
|
+
beatIndices: beatIndicesForRender,
|
|
478
|
+
frameCount: framesCountForRender,
|
|
479
|
+
audioSegment:
|
|
480
|
+
autoHighlight &&
|
|
481
|
+
highlightSlice &&
|
|
482
|
+
highlightSlice.audioSegments.length > 0
|
|
483
|
+
? highlightSlice.audioSegments[0]
|
|
484
|
+
: undefined,
|
|
485
|
+
},
|
|
486
|
+
];
|
|
487
|
+
|
|
488
|
+
const progressDenominator =
|
|
489
|
+
passes.reduce((s, p) => s + p.frameCount, 0) + passes.length;
|
|
490
|
+
let progressFrameBase = 0;
|
|
491
|
+
let lastExitCode = 0;
|
|
492
|
+
const outputVideoFiles: string[] = [];
|
|
493
|
+
|
|
494
|
+
passLoop: for (const pass of passes) {
|
|
495
|
+
const {
|
|
496
|
+
backgroundWidth,
|
|
497
|
+
backgroundHeight,
|
|
498
|
+
defaultColor,
|
|
499
|
+
staticBackgroundBuffer,
|
|
421
500
|
videoFrameReader,
|
|
422
501
|
videoFrameSize,
|
|
423
502
|
encodeVideoFrame,
|
|
424
|
-
|
|
503
|
+
concatFilePath,
|
|
504
|
+
} = await prepareBackgroundForRender({
|
|
505
|
+
useVideoBackground,
|
|
506
|
+
backgroundVideoPath,
|
|
507
|
+
backgroundImagePath,
|
|
508
|
+
beatFrameIndices: pass.beatIndices,
|
|
509
|
+
framesCount: pass.frameCount,
|
|
510
|
+
outputResolution,
|
|
511
|
+
fps: FPS,
|
|
512
|
+
autoEditVideo: getAutoEditVideo(config),
|
|
513
|
+
});
|
|
514
|
+
|
|
515
|
+
const createVisualizerFrame = createVisualizerFrameGenerator(
|
|
516
|
+
config, backgroundWidth, backgroundHeight, defaultColor, spectrumBusMargin
|
|
517
|
+
);
|
|
518
|
+
|
|
519
|
+
const ffmpegVideoWriter = spawnFfmpegVideoWriter({
|
|
520
|
+
audioFilename: audioFilePath,
|
|
521
|
+
videoFileName: pass.outPath,
|
|
522
|
+
fps: FPS,
|
|
523
|
+
...(pass.audioSegment && { audioSegment: pass.audioSegment }),
|
|
524
|
+
...(!!onProgress && {
|
|
525
|
+
onStderr: getProgress((currentFrame: number) => {
|
|
526
|
+
const g = progressFrameBase + currentFrame;
|
|
527
|
+
onProgress(
|
|
528
|
+
+(
|
|
529
|
+
Math.min(
|
|
530
|
+
100,
|
|
531
|
+
PRE_PROCESS_PROGRESS_SHARE
|
|
532
|
+
+ (90 * g) / progressDenominator,
|
|
533
|
+
)
|
|
534
|
+
).toFixed(2),
|
|
535
|
+
);
|
|
536
|
+
}),
|
|
537
|
+
}),
|
|
538
|
+
...(ffmpeg_cfr && { crf: ffmpeg_cfr }),
|
|
539
|
+
...(ffmpeg_preset && { preset: ffmpeg_preset }),
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
const exitPromise = new Promise<number>(res => {
|
|
543
|
+
let settled = false;
|
|
544
|
+
ffmpegVideoWriter.on('exit', (code: number | null) => {
|
|
545
|
+
if (settled) {
|
|
546
|
+
return;
|
|
547
|
+
}
|
|
548
|
+
settled = true;
|
|
549
|
+
res(code ?? 0);
|
|
550
|
+
});
|
|
425
551
|
});
|
|
426
552
|
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
spectrum
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
553
|
+
let stoppedEarly = false;
|
|
554
|
+
for (let i = 0; i < pass.frameCount; i++) {
|
|
555
|
+
const spectrum = pass.spectrums[i];
|
|
556
|
+
|
|
557
|
+
const backgroundImageBuffer = await resolveBackgroundFrameBuffer({
|
|
558
|
+
frameIndex: i,
|
|
559
|
+
useVideoBackground,
|
|
560
|
+
videoFrameReader,
|
|
561
|
+
videoFrameSize,
|
|
562
|
+
encodeVideoFrame,
|
|
563
|
+
staticBackgroundBuffer,
|
|
564
|
+
});
|
|
565
|
+
|
|
566
|
+
const commonVisualizerFrameProps: CommonVisualizerFrameProps = {
|
|
567
|
+
backgroundImageBuffer,
|
|
568
|
+
spectrum,
|
|
569
|
+
};
|
|
570
|
+
const frameImage = createVisualizerFrame(commonVisualizerFrameProps);
|
|
571
|
+
const isFrameProcessed = ffmpegVideoWriter.stdin.write(frameImage.data);
|
|
572
|
+
if (!isFrameProcessed) {
|
|
573
|
+
await waitDrain(ffmpegVideoWriter.stdin);
|
|
574
|
+
}
|
|
575
|
+
if (shouldStop && shouldStop()) {
|
|
576
|
+
stoppedEarly = true;
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
579
|
+
if (frame_processing_delay) {
|
|
580
|
+
await sleep(frame_processing_delay);
|
|
581
|
+
}
|
|
582
|
+
await waitForEventLoop();
|
|
435
583
|
}
|
|
436
|
-
|
|
437
|
-
|
|
584
|
+
|
|
585
|
+
if (videoFrameReader) {
|
|
586
|
+
videoFrameReader.kill();
|
|
438
587
|
}
|
|
439
|
-
if (
|
|
440
|
-
|
|
588
|
+
if (concatFilePath) {
|
|
589
|
+
cleanupConcatFile(concatFilePath);
|
|
441
590
|
}
|
|
442
|
-
|
|
591
|
+
ffmpegVideoWriter.stdin.end();
|
|
443
592
|
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
593
|
+
lastExitCode = await exitPromise;
|
|
594
|
+
progressFrameBase += pass.frameCount + 1;
|
|
595
|
+
|
|
596
|
+
if (lastExitCode === 0 && !stoppedEarly) {
|
|
597
|
+
outputVideoFiles.push(pass.outPath);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
if (stoppedEarly || lastExitCode !== 0) {
|
|
601
|
+
break passLoop;
|
|
602
|
+
}
|
|
449
603
|
}
|
|
450
|
-
|
|
604
|
+
|
|
605
|
+
resolve({ exitCode: lastExitCode, outputVideoFiles });
|
|
451
606
|
});
|
package/src/tests/config.test.ts
CHANGED
|
@@ -4,6 +4,8 @@ import { Config } from '../index';
|
|
|
4
4
|
import {
|
|
5
5
|
defaults,
|
|
6
6
|
getAudioFilePath,
|
|
7
|
+
getAudioAutoHighlight,
|
|
8
|
+
getAudioAutoHighlightCount,
|
|
7
9
|
getBackgroundImagePath,
|
|
8
10
|
getOutVideoPath,
|
|
9
11
|
getFPS,
|
|
@@ -30,6 +32,36 @@ describe('config', function() {
|
|
|
30
32
|
expect(result).equal(expected);
|
|
31
33
|
});
|
|
32
34
|
|
|
35
|
+
it('getAudioAutoHighlight', function() {
|
|
36
|
+
expect(
|
|
37
|
+
getAudioAutoHighlight({ audio: { path: 'a.wav' } } as Config)
|
|
38
|
+
).equal(false);
|
|
39
|
+
expect(
|
|
40
|
+
getAudioAutoHighlight({ audio: { path: 'a.wav', autoHighlight: false } } as Config)
|
|
41
|
+
).equal(false);
|
|
42
|
+
expect(
|
|
43
|
+
getAudioAutoHighlight({ audio: { path: 'a.wav', autoHighlight: true } } as Config)
|
|
44
|
+
).equal(true);
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
it('getAudioAutoHighlightCount', function() {
|
|
48
|
+
expect(
|
|
49
|
+
getAudioAutoHighlightCount({ audio: { path: 'a.wav' } } as Config)
|
|
50
|
+
).equal(1);
|
|
51
|
+
expect(
|
|
52
|
+
getAudioAutoHighlightCount({ audio: { path: 'a.wav', autoHighlightCount: 3 } } as Config)
|
|
53
|
+
).equal(3);
|
|
54
|
+
expect(
|
|
55
|
+
getAudioAutoHighlightCount.bind(undefined, { audio: { path: 'a.wav', autoHighlightCount: 0 } } as Config)
|
|
56
|
+
).to.throw('Invalid audio.autoHighlightCount');
|
|
57
|
+
expect(
|
|
58
|
+
getAudioAutoHighlightCount.bind(undefined, { audio: { path: 'a.wav', autoHighlightCount: 1.5 } } as Config)
|
|
59
|
+
).to.throw('Invalid audio.autoHighlightCount');
|
|
60
|
+
expect(
|
|
61
|
+
getAudioAutoHighlightCount.bind(undefined, { audio: { path: 'a.wav', autoHighlightCount: NaN } } as Config)
|
|
62
|
+
).to.throw('Invalid audio.autoHighlightCount');
|
|
63
|
+
});
|
|
64
|
+
|
|
33
65
|
it('getBackgroundImagePath', function() {
|
|
34
66
|
const result = getBackgroundImagePath({
|
|
35
67
|
image: { path: 'test/path' }
|