nodejs-audio-visualizer 5.5.3 → 5.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +84 -83
- package/dist/audio.d.ts +13 -13
- package/dist/audio.js +128 -128
- package/dist/beats.d.ts +39 -39
- package/dist/beats.js +220 -220
- package/dist/bpmEncoder.d.ts +14 -14
- package/dist/bpmEncoder.js +97 -97
- package/dist/config.d.ts +81 -80
- package/dist/config.js +392 -391
- package/dist/config.js.map +1 -1
- package/dist/dsp.d.ts +16 -16
- package/dist/dsp.js +76 -76
- package/dist/highlight.d.ts +91 -91
- package/dist/highlight.js +606 -606
- package/dist/image.d.ts +107 -97
- package/dist/image.js +614 -545
- package/dist/image.js.map +1 -1
- package/dist/index.d.ts +98 -96
- package/dist/index.js +673 -667
- package/dist/index.js.map +1 -1
- package/dist/subtitleConvert.d.ts +3 -3
- package/dist/subtitleConvert.js +61 -61
- package/dist/video.d.ts +171 -171
- package/dist/video.js +743 -743
- package/dist/waitForEventLoop.d.ts +1 -1
- package/dist/waitForEventLoop.js +5 -5
- package/package.json +62 -62
- package/src/audio.ts +129 -129
- package/src/beats.ts +279 -279
- package/src/bpmEncoder.ts +97 -97
- package/src/config.ts +504 -501
- package/src/dsp.ts +95 -95
- package/src/highlight.ts +617 -617
- package/src/image.ts +791 -707
- package/src/index.ts +896 -885
- package/src/subtitleConvert.ts +63 -63
- package/src/tests/audio.test.ts +102 -102
- package/src/tests/beats.test.ts +134 -134
- package/src/tests/config.test.ts +568 -553
- package/src/tests/dsp.test.ts +24 -24
- package/src/tests/highlight.test.ts +397 -397
- package/src/tests/image.test.ts +371 -319
- package/src/tests/index.test.ts +180 -180
- package/src/tests/subtitleConvert.test.ts +23 -23
- package/src/tests/video.test.ts +652 -652
- package/src/typings.d.ts +28 -28
- package/src/video.ts +947 -947
- package/src/waitForEventLoop.ts +2 -2
package/dist/highlight.js
CHANGED
|
@@ -1,607 +1,607 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/** Output length per segment when `audio.autoHighlight` is true (seconds). */
|
|
3
|
-
var __assign = (this && this.__assign) || function () {
|
|
4
|
-
__assign = Object.assign || function(t) {
|
|
5
|
-
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
6
|
-
s = arguments[i];
|
|
7
|
-
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
8
|
-
t[p] = s[p];
|
|
9
|
-
}
|
|
10
|
-
return t;
|
|
11
|
-
};
|
|
12
|
-
return __assign.apply(this, arguments);
|
|
13
|
-
};
|
|
14
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
15
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
16
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
17
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
18
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
19
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
20
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
21
|
-
});
|
|
22
|
-
};
|
|
23
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
24
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
25
|
-
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
26
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
27
|
-
function step(op) {
|
|
28
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
29
|
-
while (_) try {
|
|
30
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
31
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
32
|
-
switch (op[0]) {
|
|
33
|
-
case 0: case 1: t = op; break;
|
|
34
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
35
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
36
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
37
|
-
default:
|
|
38
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
39
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
40
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
41
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
42
|
-
if (t[2]) _.ops.pop();
|
|
43
|
-
_.trys.pop(); continue;
|
|
44
|
-
}
|
|
45
|
-
op = body.call(thisArg, _);
|
|
46
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
47
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
48
|
-
}
|
|
49
|
-
};
|
|
50
|
-
var __spreadArrays = (this && this.__spreadArrays) || function () {
|
|
51
|
-
for (var s = 0, i = 0, il = arguments.length; i < il; i++) s += arguments[i].length;
|
|
52
|
-
for (var r = Array(s), k = 0, i = 0; i < il; i++)
|
|
53
|
-
for (var a = arguments[i], j = 0, jl = a.length; j < jl; j++, k++)
|
|
54
|
-
r[k] = a[j];
|
|
55
|
-
return r;
|
|
56
|
-
};
|
|
57
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
58
|
-
var waitForEventLoop_1 = require("./waitForEventLoop");
|
|
59
|
-
exports.HIGHLIGHT_DURATION_SEC = 15;
|
|
60
|
-
/**
|
|
61
|
-
* How far (seconds) a highlight boundary may move to land on a beat. The final length can
|
|
62
|
-
* therefore differ from `HIGHLIGHT_DURATION_SEC` by up to twice this value.
|
|
63
|
-
*/
|
|
64
|
-
exports.MAX_BEAT_SNAP_SEC = 1.5;
|
|
65
|
-
/** Beats per bar used when preferring highlight lengths made of whole bars. */
|
|
66
|
-
var BEATS_PER_BAR = 4;
|
|
67
|
-
/** Cost (in beats of boundary shift) for each beat the length is away from a whole bar. */
|
|
68
|
-
var OFF_BAR_PENALTY = 3;
|
|
69
|
-
/** Grid beats past the analyzed tempo region snap to a detected onset within this share of a beat. */
|
|
70
|
-
var GRID_ONSET_SNAP_SHARE = 0.2;
|
|
71
|
-
/** Seconds of audio compared before and after a frame to score it as a drop. */
|
|
72
|
-
exports.DROP_WINDOW_SEC = 4;
|
|
73
|
-
/** Share of the track's biggest energy rise a peak needs to count as a drop. */
|
|
74
|
-
var MIN_DROP_SCORE_SHARE = 0.5;
|
|
75
|
-
/** With a tempo grid, a drop moves onto a beat within this share of a beat, so the hook ends on the beat. */
|
|
76
|
-
var DROP_BEAT_SNAP_SHARE = 0.2;
|
|
77
|
-
/**
|
|
78
|
-
* `seg.startFrame` may be negative (a lead-in reaching back past the track start): those
|
|
79
|
-
* frames get silent spectrums and are played as silence before the audio.
|
|
80
|
-
*/
|
|
81
|
-
function buildHighlightRun(fps, seg, spectrums, beatEvents) {
|
|
82
|
-
var _a, _b;
|
|
83
|
-
var startFrame = seg.startFrame, highlightFrames = seg.highlightFrames, leadInFrames = seg.leadInFrames;
|
|
84
|
-
var silentFrames = Math.max(0, -startFrame);
|
|
85
|
-
var silentSpectrum = new Array((_b = (_a = spectrums[0]) === null || _a === void 0 ? void 0 : _a.length, (_b !== null && _b !== void 0 ? _b : 0))).fill(0);
|
|
86
|
-
var spectrumsSlice = __spreadArrays(Array.from({ length: silentFrames }, function () { return silentSpectrum.slice(); }), spectrums.slice(startFrame + silentFrames, startFrame + highlightFrames));
|
|
87
|
-
var beatsInWindow = beatEvents
|
|
88
|
-
.filter(function (b) { return b.frameIndex >= startFrame && b.frameIndex < startFrame + highlightFrames; })
|
|
89
|
-
.sort(function (a, b) { return a.frameIndex - b.frameIndex; });
|
|
90
|
-
var beatFrameIndices = beatsInWindow.map(function (b) { return b.frameIndex - startFrame; });
|
|
91
|
-
var beatIntensities = beatsInWindow.map(function (b) { return b.intensity; });
|
|
92
|
-
return __assign({ startFrame: startFrame,
|
|
93
|
-
highlightFrames: highlightFrames, spectrums: spectrumsSlice, beatFrameIndices: beatFrameIndices,
|
|
94
|
-
beatIntensities: beatIntensities, audioSegment: __assign({ seekSeconds: (startFrame + silentFrames) / fps, durationSeconds: (highlightFrames - silentFrames) / fps }, (silentFrames > 0 && { delaySeconds: silentFrames / fps })) }, (leadInFrames !== undefined && { leadInFrames: leadInFrames }));
|
|
95
|
-
}
|
|
96
|
-
function frameEnergy(spectrum) {
|
|
97
|
-
var s = 0;
|
|
98
|
-
for (var i = 0; i < spectrum.length; i++) {
|
|
99
|
-
var v = spectrum[i];
|
|
100
|
-
s += v * v;
|
|
101
|
-
}
|
|
102
|
-
return s;
|
|
103
|
-
}
|
|
104
|
-
function rangesOverlap(aStart, aEndExclusive, ranges) {
|
|
105
|
-
for (var _i = 0, ranges_1 = ranges; _i < ranges_1.length; _i++) {
|
|
106
|
-
var r = ranges_1[_i];
|
|
107
|
-
if (aStart < r.endExclusive && aEndExclusive > r.start) {
|
|
108
|
-
return true;
|
|
109
|
-
}
|
|
110
|
-
}
|
|
111
|
-
return false;
|
|
112
|
-
}
|
|
113
|
-
function buildFrameEnergies(spectrums, totalFrames) {
|
|
114
|
-
var _a;
|
|
115
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
116
|
-
var energies, i;
|
|
117
|
-
return __generator(this, function (_b) {
|
|
118
|
-
switch (_b.label) {
|
|
119
|
-
case 0:
|
|
120
|
-
energies = new Float64Array(totalFrames);
|
|
121
|
-
i = 0;
|
|
122
|
-
_b.label = 1;
|
|
123
|
-
case 1:
|
|
124
|
-
if (!(i < totalFrames)) return [3 /*break*/, 4];
|
|
125
|
-
energies[i] = frameEnergy((_a = spectrums[i], (_a !== null && _a !== void 0 ? _a : [])));
|
|
126
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
127
|
-
case 2:
|
|
128
|
-
_b.sent();
|
|
129
|
-
_b.label = 3;
|
|
130
|
-
case 3:
|
|
131
|
-
i++;
|
|
132
|
-
return [3 /*break*/, 1];
|
|
133
|
-
case 4: return [2 /*return*/, energies];
|
|
134
|
-
}
|
|
135
|
-
});
|
|
136
|
-
});
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Chooses the fixed-length window whose frames have the highest summed spectral energy
|
|
140
|
-
* (typical chorus / drop vs a single loud intro transient).
|
|
141
|
-
* Skips windows that overlap any excluded range.
|
|
142
|
-
*/
|
|
143
|
-
function findBestHighlightStart(energies, totalFrames, windowFrames, excludeRanges) {
|
|
144
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
145
|
-
var minE, maxE, i, e, prefix, i, maxStart, bestStart, bestScore, start, score, center, bestDist, tieBreak, start, dist;
|
|
146
|
-
return __generator(this, function (_a) {
|
|
147
|
-
switch (_a.label) {
|
|
148
|
-
case 0:
|
|
149
|
-
if (totalFrames <= windowFrames) {
|
|
150
|
-
if (!rangesOverlap(0, totalFrames, excludeRanges)) {
|
|
151
|
-
return [2 /*return*/, 0];
|
|
152
|
-
}
|
|
153
|
-
return [2 /*return*/, null];
|
|
154
|
-
}
|
|
155
|
-
minE = Infinity;
|
|
156
|
-
maxE = -Infinity;
|
|
157
|
-
i = 0;
|
|
158
|
-
_a.label = 1;
|
|
159
|
-
case 1:
|
|
160
|
-
if (!(i < totalFrames)) return [3 /*break*/, 4];
|
|
161
|
-
e = energies[i];
|
|
162
|
-
if (e < minE) {
|
|
163
|
-
minE = e;
|
|
164
|
-
}
|
|
165
|
-
if (e > maxE) {
|
|
166
|
-
maxE = e;
|
|
167
|
-
}
|
|
168
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
169
|
-
case 2:
|
|
170
|
-
_a.sent();
|
|
171
|
-
_a.label = 3;
|
|
172
|
-
case 3:
|
|
173
|
-
i++;
|
|
174
|
-
return [3 /*break*/, 1];
|
|
175
|
-
case 4:
|
|
176
|
-
prefix = new Float64Array(totalFrames + 1);
|
|
177
|
-
i = 0;
|
|
178
|
-
_a.label = 5;
|
|
179
|
-
case 5:
|
|
180
|
-
if (!(i < totalFrames)) return [3 /*break*/, 8];
|
|
181
|
-
prefix[i + 1] = prefix[i] + energies[i];
|
|
182
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
183
|
-
case 6:
|
|
184
|
-
_a.sent();
|
|
185
|
-
_a.label = 7;
|
|
186
|
-
case 7:
|
|
187
|
-
i++;
|
|
188
|
-
return [3 /*break*/, 5];
|
|
189
|
-
case 8:
|
|
190
|
-
maxStart = totalFrames - windowFrames;
|
|
191
|
-
bestStart = null;
|
|
192
|
-
bestScore = -Infinity;
|
|
193
|
-
start = 0;
|
|
194
|
-
_a.label = 9;
|
|
195
|
-
case 9:
|
|
196
|
-
if (!(start <= maxStart)) return [3 /*break*/, 14];
|
|
197
|
-
if (!rangesOverlap(start, start + windowFrames, excludeRanges)) return [3 /*break*/, 11];
|
|
198
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
199
|
-
case 10:
|
|
200
|
-
_a.sent();
|
|
201
|
-
return [3 /*break*/, 13];
|
|
202
|
-
case 11:
|
|
203
|
-
score = prefix[start + windowFrames] - prefix[start];
|
|
204
|
-
if (score > bestScore) {
|
|
205
|
-
bestScore = score;
|
|
206
|
-
bestStart = start;
|
|
207
|
-
}
|
|
208
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
209
|
-
case 12:
|
|
210
|
-
_a.sent();
|
|
211
|
-
_a.label = 13;
|
|
212
|
-
case 13:
|
|
213
|
-
start++;
|
|
214
|
-
return [3 /*break*/, 9];
|
|
215
|
-
case 14:
|
|
216
|
-
if (bestStart === null) {
|
|
217
|
-
return [2 /*return*/, null];
|
|
218
|
-
}
|
|
219
|
-
if (!(maxE <= minE)) return [3 /*break*/, 21];
|
|
220
|
-
center = Math.floor((totalFrames - windowFrames) / 2);
|
|
221
|
-
bestDist = Infinity;
|
|
222
|
-
tieBreak = null;
|
|
223
|
-
start = 0;
|
|
224
|
-
_a.label = 15;
|
|
225
|
-
case 15:
|
|
226
|
-
if (!(start <= maxStart)) return [3 /*break*/, 20];
|
|
227
|
-
if (!rangesOverlap(start, start + windowFrames, excludeRanges)) return [3 /*break*/, 17];
|
|
228
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
229
|
-
case 16:
|
|
230
|
-
_a.sent();
|
|
231
|
-
return [3 /*break*/, 19];
|
|
232
|
-
case 17:
|
|
233
|
-
dist = Math.abs(start - center);
|
|
234
|
-
if (dist < bestDist) {
|
|
235
|
-
bestDist = dist;
|
|
236
|
-
tieBreak = start;
|
|
237
|
-
}
|
|
238
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
239
|
-
case 18:
|
|
240
|
-
_a.sent();
|
|
241
|
-
_a.label = 19;
|
|
242
|
-
case 19:
|
|
243
|
-
start++;
|
|
244
|
-
return [3 /*break*/, 15];
|
|
245
|
-
case 20: return [2 /*return*/, tieBreak];
|
|
246
|
-
case 21: return [2 /*return*/, bestStart];
|
|
247
|
-
}
|
|
248
|
-
});
|
|
249
|
-
});
|
|
250
|
-
}
|
|
251
|
-
/**
|
|
252
|
-
* Scores each frame by the rise in mean energy from the `dropWindowFrames` before it to the
|
|
253
|
-
* `dropWindowFrames` after it, and picks the best drop, i.e. a frame whose rise is the biggest
|
|
254
|
-
* within `dropWindowFrames` on either side, such that the lead-in before it plus `mainFrames`
|
|
255
|
-
* after it fit in the track and miss the excluded ranges. Frames rather than beat candidates are
|
|
256
|
-
* scored so the drop lands on the actual energy onset (a beat grid can be off by a fraction of a
|
|
257
|
-
* beat), and only peaks count so a frame just past a drop is never taken for one. Peaks rising
|
|
258
|
-
* less than `MIN_DROP_SCORE_SHARE` of the biggest one are not drops. The window
|
|
259
|
-
* starts exactly `leadInFrames` before the drop, so the whole lead-in plays. Drops too close to
|
|
260
|
-
* the track start for the full lead-in are only used (with the window starting before the track,
|
|
261
|
-
* i.e. at a negative frame) when no other drop fits.
|
|
262
|
-
*/
|
|
263
|
-
function findBestDropSegment(energies, totalFrames, mainFrames, leadInFrames, dropWindowFrames, excludeRanges) {
|
|
264
|
-
var _a;
|
|
265
|
-
var prefix = new Float64Array(totalFrames + 1);
|
|
266
|
-
for (var i = 0; i < totalFrames; i++) {
|
|
267
|
-
prefix[i + 1] = prefix[i] + energies[i];
|
|
268
|
-
}
|
|
269
|
-
var meanEnergy = function (from, to) {
|
|
270
|
-
var a = Math.max(0, from);
|
|
271
|
-
var b = Math.min(totalFrames, to);
|
|
272
|
-
return b > a ? (prefix[b] - prefix[a]) / (b - a) : 0;
|
|
273
|
-
};
|
|
274
|
-
var scores = new Float64Array(totalFrames);
|
|
275
|
-
for (var f = 0; f < totalFrames; f++) {
|
|
276
|
-
scores[f] = meanEnergy(f, f + dropWindowFrames) - meanEnergy(f - dropWindowFrames, f);
|
|
277
|
-
}
|
|
278
|
-
var dropFrames = [];
|
|
279
|
-
for (var f = 1; f < totalFrames; f++) {
|
|
280
|
-
if (!(scores[f] > 0)) {
|
|
281
|
-
continue;
|
|
282
|
-
}
|
|
283
|
-
var isPeak = true;
|
|
284
|
-
var from = Math.max(0, f - dropWindowFrames);
|
|
285
|
-
var to = Math.min(totalFrames - 1, f + dropWindowFrames);
|
|
286
|
-
for (var g = from; g <= to && isPeak; g++) {
|
|
287
|
-
// Ties go to the earliest frame, where the rise starts.
|
|
288
|
-
isPeak = scores[g] < scores[f] || (scores[g] === scores[f] && g >= f);
|
|
289
|
-
}
|
|
290
|
-
if (isPeak) {
|
|
291
|
-
dropFrames.push(f);
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
var maxScore = dropFrames.reduce(function (max, f) { return Math.max(max, scores[f]); }, 0);
|
|
295
|
-
var strongDropFrames = dropFrames.filter(function (f) { return scores[f] >= maxScore * MIN_DROP_SCORE_SHARE; });
|
|
296
|
-
var findBest = function (allowBeforeTrackStart) {
|
|
297
|
-
var best = null;
|
|
298
|
-
var bestScore = -Infinity;
|
|
299
|
-
for (var _i = 0, strongDropFrames_1 = strongDropFrames; _i < strongDropFrames_1.length; _i++) {
|
|
300
|
-
var dropFrame = strongDropFrames_1[_i];
|
|
301
|
-
var startFrame = dropFrame - leadInFrames;
|
|
302
|
-
if (startFrame < 0 && !allowBeforeTrackStart) {
|
|
303
|
-
continue;
|
|
304
|
-
}
|
|
305
|
-
if (dropFrame + mainFrames > totalFrames ||
|
|
306
|
-
rangesOverlap(startFrame, dropFrame + mainFrames, excludeRanges)) {
|
|
307
|
-
continue;
|
|
308
|
-
}
|
|
309
|
-
if (scores[dropFrame] > bestScore) {
|
|
310
|
-
bestScore = scores[dropFrame];
|
|
311
|
-
best = { startFrame: startFrame, dropFrame: dropFrame };
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
return best;
|
|
315
|
-
};
|
|
316
|
-
return _a = findBest(false), (_a !== null && _a !== void 0 ? _a : findBest(true));
|
|
317
|
-
}
|
|
318
|
-
/**
|
|
319
|
-
* Moves a drop (and its window start with it) onto the nearest beat within `DROP_BEAT_SNAP_SHARE`
|
|
320
|
-
* of a beat. Only done with a tempo grid: raw onsets alone are too unreliable to override the
|
|
321
|
-
* energy onset. A lead-in that fit in the track is never pushed back past its start.
|
|
322
|
-
*/
|
|
323
|
-
function snapDropToBeat(drop, beatFrames, periodFrames) {
|
|
324
|
-
if (!periodFrames || !(periodFrames > 0) || !isFinite(periodFrames)) {
|
|
325
|
-
return;
|
|
326
|
-
}
|
|
327
|
-
var beat = nearestOnset(beatFrames, drop.dropFrame);
|
|
328
|
-
if (beat === null || Math.abs(beat - drop.dropFrame) > periodFrames * DROP_BEAT_SNAP_SHARE) {
|
|
329
|
-
return;
|
|
330
|
-
}
|
|
331
|
-
var startFrame = drop.startFrame + (beat - drop.dropFrame);
|
|
332
|
-
if (startFrame < 0 && drop.startFrame >= 0) {
|
|
333
|
-
return;
|
|
334
|
-
}
|
|
335
|
-
drop.startFrame = startFrame;
|
|
336
|
-
drop.dropFrame = beat;
|
|
337
|
-
}
|
|
338
|
-
exports.snapDropToBeat = snapDropToBeat;
|
|
339
|
-
function nearestOnset(sortedOnsets, frame) {
|
|
340
|
-
var lo = 0;
|
|
341
|
-
var hi = sortedOnsets.length;
|
|
342
|
-
while (lo < hi) {
|
|
343
|
-
var mid = (lo + hi) >> 1;
|
|
344
|
-
if (sortedOnsets[mid] < frame) {
|
|
345
|
-
lo = mid + 1;
|
|
346
|
-
}
|
|
347
|
-
else {
|
|
348
|
-
hi = mid;
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
var best = null;
|
|
352
|
-
for (var _i = 0, _a = [lo - 1, lo]; _i < _a.length; _i++) {
|
|
353
|
-
var i = _a[_i];
|
|
354
|
-
if (i >= 0 && i < sortedOnsets.length) {
|
|
355
|
-
var v = sortedOnsets[i];
|
|
356
|
-
if (best === null || Math.abs(v - frame) < Math.abs(best - frame)) {
|
|
357
|
-
best = v;
|
|
358
|
-
}
|
|
359
|
-
}
|
|
360
|
-
}
|
|
361
|
-
return best;
|
|
362
|
-
}
|
|
363
|
-
/**
|
|
364
|
-
* Beat frames (sorted, unique, within [0, totalFrames]) where a highlight may start or end.
|
|
365
|
-
* With a tempo grid: tracked beats from the analyzed region, then the regular grid beyond it
|
|
366
|
-
* (each grid beat nudged onto a nearby detected onset to follow slight tempo drift).
|
|
367
|
-
* Without one: the detected onset beats.
|
|
368
|
-
*/
|
|
369
|
-
function buildBeatCandidates(fps, totalFrames, beatEvents, beatGrid) {
|
|
370
|
-
var _a, _b, _c;
|
|
371
|
-
var onsets = Array.from(new Set(beatEvents.map(function (b) { return b.frameIndex; })))
|
|
372
|
-
.filter(function (f) { return f >= 0 && f <= totalFrames; })
|
|
373
|
-
.sort(function (a, b) { return a - b; });
|
|
374
|
-
var period = (_b = (_a = beatGrid) === null || _a === void 0 ? void 0 : _a.periodFrames, (_b !== null && _b !== void 0 ? _b : 0));
|
|
375
|
-
if (!beatGrid || !(period > 0) || !isFinite(period)) {
|
|
376
|
-
return onsets;
|
|
377
|
-
}
|
|
378
|
-
var frames = new Set();
|
|
379
|
-
var lastTracked = -Infinity;
|
|
380
|
-
for (var _i = 0, _d = (_c = beatGrid.beatsSec, (_c !== null && _c !== void 0 ? _c : [])); _i < _d.length; _i++) {
|
|
381
|
-
var t = _d[_i];
|
|
382
|
-
var f = Math.round(t * fps);
|
|
383
|
-
if (f >= 0 && f <= totalFrames) {
|
|
384
|
-
frames.add(f);
|
|
385
|
-
lastTracked = Math.max(lastTracked, f);
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
var maxOnsetShift = period * GRID_ONSET_SNAP_SHARE;
|
|
389
|
-
var firstN = Math.ceil(-beatGrid.phaseFrame / period);
|
|
390
|
-
for (var n = firstN;; n++) {
|
|
391
|
-
var gridFrame = beatGrid.phaseFrame + n * period;
|
|
392
|
-
if (gridFrame > totalFrames) {
|
|
393
|
-
break;
|
|
394
|
-
}
|
|
395
|
-
if (gridFrame <= lastTracked + period / 2) {
|
|
396
|
-
continue;
|
|
397
|
-
}
|
|
398
|
-
var onset = nearestOnset(onsets, gridFrame);
|
|
399
|
-
var f = onset !== null && Math.abs(onset - gridFrame) <= maxOnsetShift
|
|
400
|
-
? onset
|
|
401
|
-
: Math.round(gridFrame);
|
|
402
|
-
if (f >= 0 && f <= totalFrames) {
|
|
403
|
-
frames.add(f);
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
return Array.from(frames).sort(function (a, b) { return a - b; });
|
|
407
|
-
}
|
|
408
|
-
exports.buildBeatCandidates = buildBeatCandidates;
|
|
409
|
-
/**
|
|
410
|
-
* Moves a fixed-length window's start and end onto beats (each by at most `maxShiftFrames`),
|
|
411
|
-
* preferring lengths that span whole bars when the beat period is known.
|
|
412
|
-
* Boundaries without a nearby beat stay where they are.
|
|
413
|
-
*/
|
|
414
|
-
function snapSegmentToBeats(seg, totalFrames, beatFrames, maxShiftFrames, periodFrames,
|
|
415
|
-
/** Keep the start where it is and only move the end. */
|
|
416
|
-
lockStart) {
|
|
417
|
-
if (lockStart === void 0) { lockStart = false; }
|
|
418
|
-
var rawStart = seg.startFrame;
|
|
419
|
-
var rawEnd = seg.startFrame + seg.highlightFrames;
|
|
420
|
-
var near = function (target, max) {
|
|
421
|
-
return beatFrames.filter(function (f) { return Math.abs(f - target) <= maxShiftFrames && f >= 0 && f <= max; });
|
|
422
|
-
};
|
|
423
|
-
var startCands = lockStart ? [rawStart] : near(rawStart, totalFrames - 1);
|
|
424
|
-
var endCands = near(rawEnd, totalFrames);
|
|
425
|
-
if (startCands.length === 0) {
|
|
426
|
-
startCands.push(rawStart);
|
|
427
|
-
}
|
|
428
|
-
if (endCands.length === 0) {
|
|
429
|
-
endCands.push(Math.min(rawEnd, totalFrames));
|
|
430
|
-
}
|
|
431
|
-
var period = periodFrames && periodFrames > 0 && isFinite(periodFrames) ? periodFrames : 0;
|
|
432
|
-
var shiftUnit = period > 0 ? period : Math.max(1, maxShiftFrames);
|
|
433
|
-
var best = null;
|
|
434
|
-
var bestCost = Infinity;
|
|
435
|
-
for (var _i = 0, startCands_1 = startCands; _i < startCands_1.length; _i++) {
|
|
436
|
-
var s = startCands_1[_i];
|
|
437
|
-
for (var _a = 0, endCands_1 = endCands; _a < endCands_1.length; _a++) {
|
|
438
|
-
var e = endCands_1[_a];
|
|
439
|
-
if (e - s < seg.highlightFrames / 2) {
|
|
440
|
-
continue;
|
|
441
|
-
}
|
|
442
|
-
var cost = (Math.abs(s - rawStart) + Math.abs(e - rawEnd)) / shiftUnit;
|
|
443
|
-
if (period > 0) {
|
|
444
|
-
var beats = Math.round((e - s) / period);
|
|
445
|
-
var offBar = Math.abs(beats - BEATS_PER_BAR * Math.round(beats / BEATS_PER_BAR));
|
|
446
|
-
cost += offBar * OFF_BAR_PENALTY;
|
|
447
|
-
}
|
|
448
|
-
if (cost < bestCost) {
|
|
449
|
-
bestCost = cost;
|
|
450
|
-
best = { startFrame: s, endFrame: e };
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
}
|
|
454
|
-
if (!best) {
|
|
455
|
-
return seg;
|
|
456
|
-
}
|
|
457
|
-
return { startFrame: best.startFrame, highlightFrames: best.endFrame - best.startFrame };
|
|
458
|
-
}
|
|
459
|
-
exports.snapSegmentToBeats = snapSegmentToBeats;
|
|
460
|
-
/**
|
|
461
|
-
* Picks up to `segmentCount` non-overlapping ~15s windows with highest summed spectral energy each,
|
|
462
|
-
* moves their boundaries onto beats (see `snapSegmentToBeats`), sorts them chronologically,
|
|
463
|
-
* concatenates spectrums, and remaps beat indices.
|
|
464
|
-
*
|
|
465
|
-
* With `leadInFrames` (e.g. a hook video's length), each window is instead built around a detected
|
|
466
|
-
* drop: it starts `leadInFrames` before the drop, so the whole lead-in plays and ends on the drop
|
|
467
|
-
* (see `HighlightRun.leadInFrames`), and runs ~15s past the drop, so the lead-in is added on top of
|
|
468
|
-
* the ~15s rather than taken out of it. With a tempo grid, the drop is moved onto a nearby beat
|
|
469
|
-
* (see `snapDropToBeat`), so the hook ends on the beat. When no drop fits, the lead-in is put before the energy
|
|
470
|
-
* window instead. The lead-in is never shortened: where it reaches back past the track start,
|
|
471
|
-
* the run starts at a negative frame and that part is silent (see `buildHighlightRun`).
|
|
472
|
-
*/
|
|
473
|
-
function computeHighlightSlice(fps, totalFrames, spectrums, beatEvents, segmentCount, beatGrid, leadInFrames) {
|
|
474
|
-
if (segmentCount === void 0) { segmentCount = 1; }
|
|
475
|
-
var _a, _b, _c, _d, _e, _f;
|
|
476
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
477
|
-
var highlightFramesTarget, leadIn, fullSeg, fullRun, highlightFrames, energies, excludeRanges, rawSegments, beatFrames, maxShiftFrames, useDrops, dropWindowFrames, n, k, drop, main, dropLeadIn, startFrame, snapped, windowLeadIn, runs, slicedSpectrums, beatFrameIndices, beatIntensities, outOffset, _i, runs_1, run_1, totalHighlightFrames, audioSegments, first;
|
|
478
|
-
return __generator(this, function (_g) {
|
|
479
|
-
switch (_g.label) {
|
|
480
|
-
case 0:
|
|
481
|
-
highlightFramesTarget = Math.ceil(exports.HIGHLIGHT_DURATION_SEC * fps);
|
|
482
|
-
if (totalFrames <= 0) {
|
|
483
|
-
return [2 /*return*/, {
|
|
484
|
-
startFrame: 0,
|
|
485
|
-
highlightFrames: 0,
|
|
486
|
-
spectrums: [],
|
|
487
|
-
beatFrameIndices: [],
|
|
488
|
-
beatIntensities: [],
|
|
489
|
-
audioSeekSeconds: 0,
|
|
490
|
-
audioDurationSeconds: 0,
|
|
491
|
-
audioSegments: [],
|
|
492
|
-
runs: [],
|
|
493
|
-
}];
|
|
494
|
-
}
|
|
495
|
-
if (totalFrames <= highlightFramesTarget) {
|
|
496
|
-
leadIn = leadInFrames !== undefined && leadInFrames > 0 ? leadInFrames : 0;
|
|
497
|
-
fullSeg = leadIn > 0
|
|
498
|
-
? { startFrame: -leadIn, highlightFrames: leadIn + totalFrames, leadInFrames: leadIn }
|
|
499
|
-
: { startFrame: 0, highlightFrames: totalFrames };
|
|
500
|
-
fullRun = buildHighlightRun(fps, fullSeg, spectrums, beatEvents);
|
|
501
|
-
return [2 /*return*/, {
|
|
502
|
-
startFrame: fullRun.startFrame,
|
|
503
|
-
highlightFrames: fullRun.highlightFrames,
|
|
504
|
-
spectrums: fullRun.spectrums,
|
|
505
|
-
beatFrameIndices: fullRun.beatFrameIndices,
|
|
506
|
-
beatIntensities: fullRun.beatIntensities,
|
|
507
|
-
audioSeekSeconds: fullRun.audioSegment.seekSeconds,
|
|
508
|
-
audioDurationSeconds: fullRun.audioSegment.durationSeconds,
|
|
509
|
-
audioSegments: [fullRun.audioSegment],
|
|
510
|
-
runs: [fullRun],
|
|
511
|
-
}];
|
|
512
|
-
}
|
|
513
|
-
highlightFrames = highlightFramesTarget;
|
|
514
|
-
return [4 /*yield*/, buildFrameEnergies(spectrums, totalFrames)];
|
|
515
|
-
case 1:
|
|
516
|
-
energies = _g.sent();
|
|
517
|
-
excludeRanges = [];
|
|
518
|
-
rawSegments = [];
|
|
519
|
-
beatFrames = buildBeatCandidates(fps, totalFrames, beatEvents, beatGrid);
|
|
520
|
-
maxShiftFrames = Math.round(exports.MAX_BEAT_SNAP_SEC * fps);
|
|
521
|
-
useDrops = leadInFrames !== undefined && leadInFrames > 0;
|
|
522
|
-
dropWindowFrames = Math.round(exports.DROP_WINDOW_SEC * fps);
|
|
523
|
-
n = Math.max(1, Math.floor(segmentCount));
|
|
524
|
-
k = 0;
|
|
525
|
-
_g.label = 2;
|
|
526
|
-
case 2:
|
|
527
|
-
if (!(k < n)) return [3 /*break*/, 7];
|
|
528
|
-
drop = useDrops
|
|
529
|
-
? findBestDropSegment(energies, totalFrames, highlightFrames, leadInFrames, dropWindowFrames, excludeRanges)
|
|
530
|
-
: null;
|
|
531
|
-
if (!drop) return [3 /*break*/, 4];
|
|
532
|
-
snapDropToBeat(drop, beatFrames, (_a = beatGrid) === null || _a === void 0 ? void 0 : _a.periodFrames);
|
|
533
|
-
main = snapSegmentToBeats({ startFrame: drop.dropFrame, highlightFrames: highlightFrames }, totalFrames, beatFrames, maxShiftFrames, (_b = beatGrid) === null || _b === void 0 ? void 0 : _b.periodFrames, true);
|
|
534
|
-
dropLeadIn = drop.dropFrame - drop.startFrame;
|
|
535
|
-
rawSegments.push({
|
|
536
|
-
startFrame: drop.startFrame,
|
|
537
|
-
highlightFrames: dropLeadIn + main.highlightFrames,
|
|
538
|
-
leadInFrames: dropLeadIn,
|
|
539
|
-
});
|
|
540
|
-
excludeRanges.push({
|
|
541
|
-
start: drop.startFrame - maxShiftFrames,
|
|
542
|
-
endExclusive: main.startFrame + main.highlightFrames + maxShiftFrames,
|
|
543
|
-
});
|
|
544
|
-
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
545
|
-
case 3:
|
|
546
|
-
_g.sent();
|
|
547
|
-
return [3 /*break*/, 6];
|
|
548
|
-
case 4: return [4 /*yield*/, findBestHighlightStart(energies, totalFrames, highlightFrames, excludeRanges)];
|
|
549
|
-
case 5:
|
|
550
|
-
startFrame = _g.sent();
|
|
551
|
-
if (startFrame === null) {
|
|
552
|
-
return [3 /*break*/, 7];
|
|
553
|
-
}
|
|
554
|
-
snapped = snapSegmentToBeats({ startFrame: startFrame, highlightFrames: highlightFrames }, totalFrames, beatFrames, maxShiftFrames, (_c = beatGrid) === null || _c === void 0 ? void 0 : _c.periodFrames);
|
|
555
|
-
windowLeadIn = useDrops ? leadInFrames : 0;
|
|
556
|
-
rawSegments.push(windowLeadIn > 0
|
|
557
|
-
? {
|
|
558
|
-
startFrame: snapped.startFrame - windowLeadIn,
|
|
559
|
-
highlightFrames: windowLeadIn + snapped.highlightFrames,
|
|
560
|
-
leadInFrames: windowLeadIn,
|
|
561
|
-
}
|
|
562
|
-
: snapped);
|
|
563
|
-
// Padded so the next raw window, once snapped, cannot reach back into this one.
|
|
564
|
-
excludeRanges.push({
|
|
565
|
-
start: snapped.startFrame - maxShiftFrames,
|
|
566
|
-
endExclusive: snapped.startFrame + snapped.highlightFrames + maxShiftFrames,
|
|
567
|
-
});
|
|
568
|
-
_g.label = 6;
|
|
569
|
-
case 6:
|
|
570
|
-
k++;
|
|
571
|
-
return [3 /*break*/, 2];
|
|
572
|
-
case 7:
|
|
573
|
-
rawSegments.sort(function (a, b) { return a.startFrame - b.startFrame; });
|
|
574
|
-
runs = rawSegments.map(function (seg) {
|
|
575
|
-
return buildHighlightRun(fps, seg, spectrums, beatEvents);
|
|
576
|
-
});
|
|
577
|
-
slicedSpectrums = [];
|
|
578
|
-
beatFrameIndices = [];
|
|
579
|
-
beatIntensities = [];
|
|
580
|
-
outOffset = 0;
|
|
581
|
-
for (_i = 0, runs_1 = runs; _i < runs_1.length; _i++) {
|
|
582
|
-
run_1 = runs_1[_i];
|
|
583
|
-
slicedSpectrums.push.apply(slicedSpectrums, run_1.spectrums);
|
|
584
|
-
beatFrameIndices.push.apply(beatFrameIndices, run_1.beatFrameIndices.map(function (frameIndex) { return frameIndex + outOffset; }));
|
|
585
|
-
beatIntensities.push.apply(beatIntensities, run_1.beatIntensities);
|
|
586
|
-
outOffset += run_1.highlightFrames;
|
|
587
|
-
}
|
|
588
|
-
totalHighlightFrames = slicedSpectrums.length;
|
|
589
|
-
audioSegments = runs.map(function (run) { return run.audioSegment; });
|
|
590
|
-
first = (_d = audioSegments[0], (_d !== null && _d !== void 0 ? _d : { seekSeconds: 0, durationSeconds: 0 }));
|
|
591
|
-
return [2 /*return*/, {
|
|
592
|
-
startFrame: (_f = (_e = rawSegments[0]) === null || _e === void 0 ? void 0 : _e.startFrame, (_f !== null && _f !== void 0 ? _f : 0)),
|
|
593
|
-
highlightFrames: totalHighlightFrames,
|
|
594
|
-
spectrums: slicedSpectrums,
|
|
595
|
-
beatFrameIndices: beatFrameIndices,
|
|
596
|
-
beatIntensities: beatIntensities,
|
|
597
|
-
audioSeekSeconds: first.seekSeconds,
|
|
598
|
-
audioDurationSeconds: first.durationSeconds,
|
|
599
|
-
audioSegments: audioSegments,
|
|
600
|
-
runs: runs,
|
|
601
|
-
}];
|
|
602
|
-
}
|
|
603
|
-
});
|
|
604
|
-
});
|
|
605
|
-
}
|
|
606
|
-
exports.computeHighlightSlice = computeHighlightSlice;
|
|
1
|
+
"use strict";
|
|
2
|
+
/** Output length per segment when `audio.autoHighlight` is true (seconds). */
|
|
3
|
+
var __assign = (this && this.__assign) || function () {
|
|
4
|
+
__assign = Object.assign || function(t) {
|
|
5
|
+
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
6
|
+
s = arguments[i];
|
|
7
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
8
|
+
t[p] = s[p];
|
|
9
|
+
}
|
|
10
|
+
return t;
|
|
11
|
+
};
|
|
12
|
+
return __assign.apply(this, arguments);
|
|
13
|
+
};
|
|
14
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
15
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
16
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
17
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
18
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
19
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
20
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
21
|
+
});
|
|
22
|
+
};
|
|
23
|
+
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
24
|
+
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
|
|
25
|
+
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
26
|
+
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
27
|
+
function step(op) {
|
|
28
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
29
|
+
while (_) try {
|
|
30
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
31
|
+
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
32
|
+
switch (op[0]) {
|
|
33
|
+
case 0: case 1: t = op; break;
|
|
34
|
+
case 4: _.label++; return { value: op[1], done: false };
|
|
35
|
+
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
36
|
+
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
37
|
+
default:
|
|
38
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
39
|
+
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
40
|
+
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
41
|
+
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
42
|
+
if (t[2]) _.ops.pop();
|
|
43
|
+
_.trys.pop(); continue;
|
|
44
|
+
}
|
|
45
|
+
op = body.call(thisArg, _);
|
|
46
|
+
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
47
|
+
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
var __spreadArrays = (this && this.__spreadArrays) || function () {
|
|
51
|
+
for (var s = 0, i = 0, il = arguments.length; i < il; i++) s += arguments[i].length;
|
|
52
|
+
for (var r = Array(s), k = 0, i = 0; i < il; i++)
|
|
53
|
+
for (var a = arguments[i], j = 0, jl = a.length; j < jl; j++, k++)
|
|
54
|
+
r[k] = a[j];
|
|
55
|
+
return r;
|
|
56
|
+
};
|
|
57
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
58
|
+
var waitForEventLoop_1 = require("./waitForEventLoop");
|
|
59
|
+
exports.HIGHLIGHT_DURATION_SEC = 15;
|
|
60
|
+
/**
|
|
61
|
+
* How far (seconds) a highlight boundary may move to land on a beat. The final length can
|
|
62
|
+
* therefore differ from `HIGHLIGHT_DURATION_SEC` by up to twice this value.
|
|
63
|
+
*/
|
|
64
|
+
exports.MAX_BEAT_SNAP_SEC = 1.5;
|
|
65
|
+
/** Beats per bar used when preferring highlight lengths made of whole bars. */
|
|
66
|
+
var BEATS_PER_BAR = 4;
|
|
67
|
+
/** Cost (in beats of boundary shift) for each beat the length is away from a whole bar. */
|
|
68
|
+
var OFF_BAR_PENALTY = 3;
|
|
69
|
+
/** Grid beats past the analyzed tempo region snap to a detected onset within this share of a beat. */
|
|
70
|
+
var GRID_ONSET_SNAP_SHARE = 0.2;
|
|
71
|
+
/** Seconds of audio compared before and after a frame to score it as a drop. */
|
|
72
|
+
exports.DROP_WINDOW_SEC = 4;
|
|
73
|
+
/** Share of the track's biggest energy rise a peak needs to count as a drop. */
|
|
74
|
+
var MIN_DROP_SCORE_SHARE = 0.5;
|
|
75
|
+
/** With a tempo grid, a drop moves onto a beat within this share of a beat, so the hook ends on the beat. */
|
|
76
|
+
var DROP_BEAT_SNAP_SHARE = 0.2;
|
|
77
|
+
/**
|
|
78
|
+
* `seg.startFrame` may be negative (a lead-in reaching back past the track start): those
|
|
79
|
+
* frames get silent spectrums and are played as silence before the audio.
|
|
80
|
+
*/
|
|
81
|
+
function buildHighlightRun(fps, seg, spectrums, beatEvents) {
|
|
82
|
+
var _a, _b;
|
|
83
|
+
var startFrame = seg.startFrame, highlightFrames = seg.highlightFrames, leadInFrames = seg.leadInFrames;
|
|
84
|
+
var silentFrames = Math.max(0, -startFrame);
|
|
85
|
+
var silentSpectrum = new Array((_b = (_a = spectrums[0]) === null || _a === void 0 ? void 0 : _a.length, (_b !== null && _b !== void 0 ? _b : 0))).fill(0);
|
|
86
|
+
var spectrumsSlice = __spreadArrays(Array.from({ length: silentFrames }, function () { return silentSpectrum.slice(); }), spectrums.slice(startFrame + silentFrames, startFrame + highlightFrames));
|
|
87
|
+
var beatsInWindow = beatEvents
|
|
88
|
+
.filter(function (b) { return b.frameIndex >= startFrame && b.frameIndex < startFrame + highlightFrames; })
|
|
89
|
+
.sort(function (a, b) { return a.frameIndex - b.frameIndex; });
|
|
90
|
+
var beatFrameIndices = beatsInWindow.map(function (b) { return b.frameIndex - startFrame; });
|
|
91
|
+
var beatIntensities = beatsInWindow.map(function (b) { return b.intensity; });
|
|
92
|
+
return __assign({ startFrame: startFrame,
|
|
93
|
+
highlightFrames: highlightFrames, spectrums: spectrumsSlice, beatFrameIndices: beatFrameIndices,
|
|
94
|
+
beatIntensities: beatIntensities, audioSegment: __assign({ seekSeconds: (startFrame + silentFrames) / fps, durationSeconds: (highlightFrames - silentFrames) / fps }, (silentFrames > 0 && { delaySeconds: silentFrames / fps })) }, (leadInFrames !== undefined && { leadInFrames: leadInFrames }));
|
|
95
|
+
}
|
|
96
|
+
function frameEnergy(spectrum) {
|
|
97
|
+
var s = 0;
|
|
98
|
+
for (var i = 0; i < spectrum.length; i++) {
|
|
99
|
+
var v = spectrum[i];
|
|
100
|
+
s += v * v;
|
|
101
|
+
}
|
|
102
|
+
return s;
|
|
103
|
+
}
|
|
104
|
+
function rangesOverlap(aStart, aEndExclusive, ranges) {
|
|
105
|
+
for (var _i = 0, ranges_1 = ranges; _i < ranges_1.length; _i++) {
|
|
106
|
+
var r = ranges_1[_i];
|
|
107
|
+
if (aStart < r.endExclusive && aEndExclusive > r.start) {
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
function buildFrameEnergies(spectrums, totalFrames) {
|
|
114
|
+
var _a;
|
|
115
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
116
|
+
var energies, i;
|
|
117
|
+
return __generator(this, function (_b) {
|
|
118
|
+
switch (_b.label) {
|
|
119
|
+
case 0:
|
|
120
|
+
energies = new Float64Array(totalFrames);
|
|
121
|
+
i = 0;
|
|
122
|
+
_b.label = 1;
|
|
123
|
+
case 1:
|
|
124
|
+
if (!(i < totalFrames)) return [3 /*break*/, 4];
|
|
125
|
+
energies[i] = frameEnergy((_a = spectrums[i], (_a !== null && _a !== void 0 ? _a : [])));
|
|
126
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
127
|
+
case 2:
|
|
128
|
+
_b.sent();
|
|
129
|
+
_b.label = 3;
|
|
130
|
+
case 3:
|
|
131
|
+
i++;
|
|
132
|
+
return [3 /*break*/, 1];
|
|
133
|
+
case 4: return [2 /*return*/, energies];
|
|
134
|
+
}
|
|
135
|
+
});
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Chooses the fixed-length window whose frames have the highest summed spectral energy
|
|
140
|
+
* (typical chorus / drop vs a single loud intro transient).
|
|
141
|
+
* Skips windows that overlap any excluded range.
|
|
142
|
+
*/
|
|
143
|
+
function findBestHighlightStart(energies, totalFrames, windowFrames, excludeRanges) {
|
|
144
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
145
|
+
var minE, maxE, i, e, prefix, i, maxStart, bestStart, bestScore, start, score, center, bestDist, tieBreak, start, dist;
|
|
146
|
+
return __generator(this, function (_a) {
|
|
147
|
+
switch (_a.label) {
|
|
148
|
+
case 0:
|
|
149
|
+
if (totalFrames <= windowFrames) {
|
|
150
|
+
if (!rangesOverlap(0, totalFrames, excludeRanges)) {
|
|
151
|
+
return [2 /*return*/, 0];
|
|
152
|
+
}
|
|
153
|
+
return [2 /*return*/, null];
|
|
154
|
+
}
|
|
155
|
+
minE = Infinity;
|
|
156
|
+
maxE = -Infinity;
|
|
157
|
+
i = 0;
|
|
158
|
+
_a.label = 1;
|
|
159
|
+
case 1:
|
|
160
|
+
if (!(i < totalFrames)) return [3 /*break*/, 4];
|
|
161
|
+
e = energies[i];
|
|
162
|
+
if (e < minE) {
|
|
163
|
+
minE = e;
|
|
164
|
+
}
|
|
165
|
+
if (e > maxE) {
|
|
166
|
+
maxE = e;
|
|
167
|
+
}
|
|
168
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
169
|
+
case 2:
|
|
170
|
+
_a.sent();
|
|
171
|
+
_a.label = 3;
|
|
172
|
+
case 3:
|
|
173
|
+
i++;
|
|
174
|
+
return [3 /*break*/, 1];
|
|
175
|
+
case 4:
|
|
176
|
+
prefix = new Float64Array(totalFrames + 1);
|
|
177
|
+
i = 0;
|
|
178
|
+
_a.label = 5;
|
|
179
|
+
case 5:
|
|
180
|
+
if (!(i < totalFrames)) return [3 /*break*/, 8];
|
|
181
|
+
prefix[i + 1] = prefix[i] + energies[i];
|
|
182
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
183
|
+
case 6:
|
|
184
|
+
_a.sent();
|
|
185
|
+
_a.label = 7;
|
|
186
|
+
case 7:
|
|
187
|
+
i++;
|
|
188
|
+
return [3 /*break*/, 5];
|
|
189
|
+
case 8:
|
|
190
|
+
maxStart = totalFrames - windowFrames;
|
|
191
|
+
bestStart = null;
|
|
192
|
+
bestScore = -Infinity;
|
|
193
|
+
start = 0;
|
|
194
|
+
_a.label = 9;
|
|
195
|
+
case 9:
|
|
196
|
+
if (!(start <= maxStart)) return [3 /*break*/, 14];
|
|
197
|
+
if (!rangesOverlap(start, start + windowFrames, excludeRanges)) return [3 /*break*/, 11];
|
|
198
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
199
|
+
case 10:
|
|
200
|
+
_a.sent();
|
|
201
|
+
return [3 /*break*/, 13];
|
|
202
|
+
case 11:
|
|
203
|
+
score = prefix[start + windowFrames] - prefix[start];
|
|
204
|
+
if (score > bestScore) {
|
|
205
|
+
bestScore = score;
|
|
206
|
+
bestStart = start;
|
|
207
|
+
}
|
|
208
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
209
|
+
case 12:
|
|
210
|
+
_a.sent();
|
|
211
|
+
_a.label = 13;
|
|
212
|
+
case 13:
|
|
213
|
+
start++;
|
|
214
|
+
return [3 /*break*/, 9];
|
|
215
|
+
case 14:
|
|
216
|
+
if (bestStart === null) {
|
|
217
|
+
return [2 /*return*/, null];
|
|
218
|
+
}
|
|
219
|
+
if (!(maxE <= minE)) return [3 /*break*/, 21];
|
|
220
|
+
center = Math.floor((totalFrames - windowFrames) / 2);
|
|
221
|
+
bestDist = Infinity;
|
|
222
|
+
tieBreak = null;
|
|
223
|
+
start = 0;
|
|
224
|
+
_a.label = 15;
|
|
225
|
+
case 15:
|
|
226
|
+
if (!(start <= maxStart)) return [3 /*break*/, 20];
|
|
227
|
+
if (!rangesOverlap(start, start + windowFrames, excludeRanges)) return [3 /*break*/, 17];
|
|
228
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
229
|
+
case 16:
|
|
230
|
+
_a.sent();
|
|
231
|
+
return [3 /*break*/, 19];
|
|
232
|
+
case 17:
|
|
233
|
+
dist = Math.abs(start - center);
|
|
234
|
+
if (dist < bestDist) {
|
|
235
|
+
bestDist = dist;
|
|
236
|
+
tieBreak = start;
|
|
237
|
+
}
|
|
238
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
239
|
+
case 18:
|
|
240
|
+
_a.sent();
|
|
241
|
+
_a.label = 19;
|
|
242
|
+
case 19:
|
|
243
|
+
start++;
|
|
244
|
+
return [3 /*break*/, 15];
|
|
245
|
+
case 20: return [2 /*return*/, tieBreak];
|
|
246
|
+
case 21: return [2 /*return*/, bestStart];
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Scores each frame by the rise in mean energy from the `dropWindowFrames` before it to the
|
|
253
|
+
* `dropWindowFrames` after it, and picks the best drop, i.e. a frame whose rise is the biggest
|
|
254
|
+
* within `dropWindowFrames` on either side, such that the lead-in before it plus `mainFrames`
|
|
255
|
+
* after it fit in the track and miss the excluded ranges. Frames rather than beat candidates are
|
|
256
|
+
* scored so the drop lands on the actual energy onset (a beat grid can be off by a fraction of a
|
|
257
|
+
* beat), and only peaks count so a frame just past a drop is never taken for one. Peaks rising
|
|
258
|
+
* less than `MIN_DROP_SCORE_SHARE` of the biggest one are not drops. The window
|
|
259
|
+
* starts exactly `leadInFrames` before the drop, so the whole lead-in plays. Drops too close to
|
|
260
|
+
* the track start for the full lead-in are only used (with the window starting before the track,
|
|
261
|
+
* i.e. at a negative frame) when no other drop fits.
|
|
262
|
+
*/
|
|
263
|
+
function findBestDropSegment(energies, totalFrames, mainFrames, leadInFrames, dropWindowFrames, excludeRanges) {
|
|
264
|
+
var _a;
|
|
265
|
+
var prefix = new Float64Array(totalFrames + 1);
|
|
266
|
+
for (var i = 0; i < totalFrames; i++) {
|
|
267
|
+
prefix[i + 1] = prefix[i] + energies[i];
|
|
268
|
+
}
|
|
269
|
+
var meanEnergy = function (from, to) {
|
|
270
|
+
var a = Math.max(0, from);
|
|
271
|
+
var b = Math.min(totalFrames, to);
|
|
272
|
+
return b > a ? (prefix[b] - prefix[a]) / (b - a) : 0;
|
|
273
|
+
};
|
|
274
|
+
var scores = new Float64Array(totalFrames);
|
|
275
|
+
for (var f = 0; f < totalFrames; f++) {
|
|
276
|
+
scores[f] = meanEnergy(f, f + dropWindowFrames) - meanEnergy(f - dropWindowFrames, f);
|
|
277
|
+
}
|
|
278
|
+
var dropFrames = [];
|
|
279
|
+
for (var f = 1; f < totalFrames; f++) {
|
|
280
|
+
if (!(scores[f] > 0)) {
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
var isPeak = true;
|
|
284
|
+
var from = Math.max(0, f - dropWindowFrames);
|
|
285
|
+
var to = Math.min(totalFrames - 1, f + dropWindowFrames);
|
|
286
|
+
for (var g = from; g <= to && isPeak; g++) {
|
|
287
|
+
// Ties go to the earliest frame, where the rise starts.
|
|
288
|
+
isPeak = scores[g] < scores[f] || (scores[g] === scores[f] && g >= f);
|
|
289
|
+
}
|
|
290
|
+
if (isPeak) {
|
|
291
|
+
dropFrames.push(f);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
var maxScore = dropFrames.reduce(function (max, f) { return Math.max(max, scores[f]); }, 0);
|
|
295
|
+
var strongDropFrames = dropFrames.filter(function (f) { return scores[f] >= maxScore * MIN_DROP_SCORE_SHARE; });
|
|
296
|
+
var findBest = function (allowBeforeTrackStart) {
|
|
297
|
+
var best = null;
|
|
298
|
+
var bestScore = -Infinity;
|
|
299
|
+
for (var _i = 0, strongDropFrames_1 = strongDropFrames; _i < strongDropFrames_1.length; _i++) {
|
|
300
|
+
var dropFrame = strongDropFrames_1[_i];
|
|
301
|
+
var startFrame = dropFrame - leadInFrames;
|
|
302
|
+
if (startFrame < 0 && !allowBeforeTrackStart) {
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
if (dropFrame + mainFrames > totalFrames ||
|
|
306
|
+
rangesOverlap(startFrame, dropFrame + mainFrames, excludeRanges)) {
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
if (scores[dropFrame] > bestScore) {
|
|
310
|
+
bestScore = scores[dropFrame];
|
|
311
|
+
best = { startFrame: startFrame, dropFrame: dropFrame };
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
return best;
|
|
315
|
+
};
|
|
316
|
+
return _a = findBest(false), (_a !== null && _a !== void 0 ? _a : findBest(true));
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Moves a drop (and its window start with it) onto the nearest beat within `DROP_BEAT_SNAP_SHARE`
|
|
320
|
+
* of a beat. Only done with a tempo grid: raw onsets alone are too unreliable to override the
|
|
321
|
+
* energy onset. A lead-in that fit in the track is never pushed back past its start.
|
|
322
|
+
*/
|
|
323
|
+
function snapDropToBeat(drop, beatFrames, periodFrames) {
|
|
324
|
+
if (!periodFrames || !(periodFrames > 0) || !isFinite(periodFrames)) {
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
var beat = nearestOnset(beatFrames, drop.dropFrame);
|
|
328
|
+
if (beat === null || Math.abs(beat - drop.dropFrame) > periodFrames * DROP_BEAT_SNAP_SHARE) {
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
var startFrame = drop.startFrame + (beat - drop.dropFrame);
|
|
332
|
+
if (startFrame < 0 && drop.startFrame >= 0) {
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
drop.startFrame = startFrame;
|
|
336
|
+
drop.dropFrame = beat;
|
|
337
|
+
}
|
|
338
|
+
exports.snapDropToBeat = snapDropToBeat;
|
|
339
|
+
function nearestOnset(sortedOnsets, frame) {
|
|
340
|
+
var lo = 0;
|
|
341
|
+
var hi = sortedOnsets.length;
|
|
342
|
+
while (lo < hi) {
|
|
343
|
+
var mid = (lo + hi) >> 1;
|
|
344
|
+
if (sortedOnsets[mid] < frame) {
|
|
345
|
+
lo = mid + 1;
|
|
346
|
+
}
|
|
347
|
+
else {
|
|
348
|
+
hi = mid;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
var best = null;
|
|
352
|
+
for (var _i = 0, _a = [lo - 1, lo]; _i < _a.length; _i++) {
|
|
353
|
+
var i = _a[_i];
|
|
354
|
+
if (i >= 0 && i < sortedOnsets.length) {
|
|
355
|
+
var v = sortedOnsets[i];
|
|
356
|
+
if (best === null || Math.abs(v - frame) < Math.abs(best - frame)) {
|
|
357
|
+
best = v;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
return best;
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Beat frames (sorted, unique, within [0, totalFrames]) where a highlight may start or end.
|
|
365
|
+
* With a tempo grid: tracked beats from the analyzed region, then the regular grid beyond it
|
|
366
|
+
* (each grid beat nudged onto a nearby detected onset to follow slight tempo drift).
|
|
367
|
+
* Without one: the detected onset beats.
|
|
368
|
+
*/
|
|
369
|
+
function buildBeatCandidates(fps, totalFrames, beatEvents, beatGrid) {
|
|
370
|
+
var _a, _b, _c;
|
|
371
|
+
var onsets = Array.from(new Set(beatEvents.map(function (b) { return b.frameIndex; })))
|
|
372
|
+
.filter(function (f) { return f >= 0 && f <= totalFrames; })
|
|
373
|
+
.sort(function (a, b) { return a - b; });
|
|
374
|
+
var period = (_b = (_a = beatGrid) === null || _a === void 0 ? void 0 : _a.periodFrames, (_b !== null && _b !== void 0 ? _b : 0));
|
|
375
|
+
if (!beatGrid || !(period > 0) || !isFinite(period)) {
|
|
376
|
+
return onsets;
|
|
377
|
+
}
|
|
378
|
+
var frames = new Set();
|
|
379
|
+
var lastTracked = -Infinity;
|
|
380
|
+
for (var _i = 0, _d = (_c = beatGrid.beatsSec, (_c !== null && _c !== void 0 ? _c : [])); _i < _d.length; _i++) {
|
|
381
|
+
var t = _d[_i];
|
|
382
|
+
var f = Math.round(t * fps);
|
|
383
|
+
if (f >= 0 && f <= totalFrames) {
|
|
384
|
+
frames.add(f);
|
|
385
|
+
lastTracked = Math.max(lastTracked, f);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
var maxOnsetShift = period * GRID_ONSET_SNAP_SHARE;
|
|
389
|
+
var firstN = Math.ceil(-beatGrid.phaseFrame / period);
|
|
390
|
+
for (var n = firstN;; n++) {
|
|
391
|
+
var gridFrame = beatGrid.phaseFrame + n * period;
|
|
392
|
+
if (gridFrame > totalFrames) {
|
|
393
|
+
break;
|
|
394
|
+
}
|
|
395
|
+
if (gridFrame <= lastTracked + period / 2) {
|
|
396
|
+
continue;
|
|
397
|
+
}
|
|
398
|
+
var onset = nearestOnset(onsets, gridFrame);
|
|
399
|
+
var f = onset !== null && Math.abs(onset - gridFrame) <= maxOnsetShift
|
|
400
|
+
? onset
|
|
401
|
+
: Math.round(gridFrame);
|
|
402
|
+
if (f >= 0 && f <= totalFrames) {
|
|
403
|
+
frames.add(f);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
return Array.from(frames).sort(function (a, b) { return a - b; });
|
|
407
|
+
}
|
|
408
|
+
exports.buildBeatCandidates = buildBeatCandidates;
|
|
409
|
+
/**
|
|
410
|
+
* Moves a fixed-length window's start and end onto beats (each by at most `maxShiftFrames`),
|
|
411
|
+
* preferring lengths that span whole bars when the beat period is known.
|
|
412
|
+
* Boundaries without a nearby beat stay where they are.
|
|
413
|
+
*/
|
|
414
|
+
function snapSegmentToBeats(seg, totalFrames, beatFrames, maxShiftFrames, periodFrames,
|
|
415
|
+
/** Keep the start where it is and only move the end. */
|
|
416
|
+
lockStart) {
|
|
417
|
+
if (lockStart === void 0) { lockStart = false; }
|
|
418
|
+
var rawStart = seg.startFrame;
|
|
419
|
+
var rawEnd = seg.startFrame + seg.highlightFrames;
|
|
420
|
+
var near = function (target, max) {
|
|
421
|
+
return beatFrames.filter(function (f) { return Math.abs(f - target) <= maxShiftFrames && f >= 0 && f <= max; });
|
|
422
|
+
};
|
|
423
|
+
var startCands = lockStart ? [rawStart] : near(rawStart, totalFrames - 1);
|
|
424
|
+
var endCands = near(rawEnd, totalFrames);
|
|
425
|
+
if (startCands.length === 0) {
|
|
426
|
+
startCands.push(rawStart);
|
|
427
|
+
}
|
|
428
|
+
if (endCands.length === 0) {
|
|
429
|
+
endCands.push(Math.min(rawEnd, totalFrames));
|
|
430
|
+
}
|
|
431
|
+
var period = periodFrames && periodFrames > 0 && isFinite(periodFrames) ? periodFrames : 0;
|
|
432
|
+
var shiftUnit = period > 0 ? period : Math.max(1, maxShiftFrames);
|
|
433
|
+
var best = null;
|
|
434
|
+
var bestCost = Infinity;
|
|
435
|
+
for (var _i = 0, startCands_1 = startCands; _i < startCands_1.length; _i++) {
|
|
436
|
+
var s = startCands_1[_i];
|
|
437
|
+
for (var _a = 0, endCands_1 = endCands; _a < endCands_1.length; _a++) {
|
|
438
|
+
var e = endCands_1[_a];
|
|
439
|
+
if (e - s < seg.highlightFrames / 2) {
|
|
440
|
+
continue;
|
|
441
|
+
}
|
|
442
|
+
var cost = (Math.abs(s - rawStart) + Math.abs(e - rawEnd)) / shiftUnit;
|
|
443
|
+
if (period > 0) {
|
|
444
|
+
var beats = Math.round((e - s) / period);
|
|
445
|
+
var offBar = Math.abs(beats - BEATS_PER_BAR * Math.round(beats / BEATS_PER_BAR));
|
|
446
|
+
cost += offBar * OFF_BAR_PENALTY;
|
|
447
|
+
}
|
|
448
|
+
if (cost < bestCost) {
|
|
449
|
+
bestCost = cost;
|
|
450
|
+
best = { startFrame: s, endFrame: e };
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
if (!best) {
|
|
455
|
+
return seg;
|
|
456
|
+
}
|
|
457
|
+
return { startFrame: best.startFrame, highlightFrames: best.endFrame - best.startFrame };
|
|
458
|
+
}
|
|
459
|
+
exports.snapSegmentToBeats = snapSegmentToBeats;
|
|
460
|
+
/**
|
|
461
|
+
* Picks up to `segmentCount` non-overlapping ~15s windows with highest summed spectral energy each,
|
|
462
|
+
* moves their boundaries onto beats (see `snapSegmentToBeats`), sorts them chronologically,
|
|
463
|
+
* concatenates spectrums, and remaps beat indices.
|
|
464
|
+
*
|
|
465
|
+
* With `leadInFrames` (e.g. a hook video's length), each window is instead built around a detected
|
|
466
|
+
* drop: it starts `leadInFrames` before the drop, so the whole lead-in plays and ends on the drop
|
|
467
|
+
* (see `HighlightRun.leadInFrames`), and runs ~15s past the drop, so the lead-in is added on top of
|
|
468
|
+
* the ~15s rather than taken out of it. With a tempo grid, the drop is moved onto a nearby beat
|
|
469
|
+
* (see `snapDropToBeat`), so the hook ends on the beat. When no drop fits, the lead-in is put before the energy
|
|
470
|
+
* window instead. The lead-in is never shortened: where it reaches back past the track start,
|
|
471
|
+
* the run starts at a negative frame and that part is silent (see `buildHighlightRun`).
|
|
472
|
+
*/
|
|
473
|
+
function computeHighlightSlice(fps, totalFrames, spectrums, beatEvents, segmentCount, beatGrid, leadInFrames) {
|
|
474
|
+
if (segmentCount === void 0) { segmentCount = 1; }
|
|
475
|
+
var _a, _b, _c, _d, _e, _f;
|
|
476
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
477
|
+
var highlightFramesTarget, leadIn, fullSeg, fullRun, highlightFrames, energies, excludeRanges, rawSegments, beatFrames, maxShiftFrames, useDrops, dropWindowFrames, n, k, drop, main, dropLeadIn, startFrame, snapped, windowLeadIn, runs, slicedSpectrums, beatFrameIndices, beatIntensities, outOffset, _i, runs_1, run_1, totalHighlightFrames, audioSegments, first;
|
|
478
|
+
return __generator(this, function (_g) {
|
|
479
|
+
switch (_g.label) {
|
|
480
|
+
case 0:
|
|
481
|
+
highlightFramesTarget = Math.ceil(exports.HIGHLIGHT_DURATION_SEC * fps);
|
|
482
|
+
if (totalFrames <= 0) {
|
|
483
|
+
return [2 /*return*/, {
|
|
484
|
+
startFrame: 0,
|
|
485
|
+
highlightFrames: 0,
|
|
486
|
+
spectrums: [],
|
|
487
|
+
beatFrameIndices: [],
|
|
488
|
+
beatIntensities: [],
|
|
489
|
+
audioSeekSeconds: 0,
|
|
490
|
+
audioDurationSeconds: 0,
|
|
491
|
+
audioSegments: [],
|
|
492
|
+
runs: [],
|
|
493
|
+
}];
|
|
494
|
+
}
|
|
495
|
+
if (totalFrames <= highlightFramesTarget) {
|
|
496
|
+
leadIn = leadInFrames !== undefined && leadInFrames > 0 ? leadInFrames : 0;
|
|
497
|
+
fullSeg = leadIn > 0
|
|
498
|
+
? { startFrame: -leadIn, highlightFrames: leadIn + totalFrames, leadInFrames: leadIn }
|
|
499
|
+
: { startFrame: 0, highlightFrames: totalFrames };
|
|
500
|
+
fullRun = buildHighlightRun(fps, fullSeg, spectrums, beatEvents);
|
|
501
|
+
return [2 /*return*/, {
|
|
502
|
+
startFrame: fullRun.startFrame,
|
|
503
|
+
highlightFrames: fullRun.highlightFrames,
|
|
504
|
+
spectrums: fullRun.spectrums,
|
|
505
|
+
beatFrameIndices: fullRun.beatFrameIndices,
|
|
506
|
+
beatIntensities: fullRun.beatIntensities,
|
|
507
|
+
audioSeekSeconds: fullRun.audioSegment.seekSeconds,
|
|
508
|
+
audioDurationSeconds: fullRun.audioSegment.durationSeconds,
|
|
509
|
+
audioSegments: [fullRun.audioSegment],
|
|
510
|
+
runs: [fullRun],
|
|
511
|
+
}];
|
|
512
|
+
}
|
|
513
|
+
highlightFrames = highlightFramesTarget;
|
|
514
|
+
return [4 /*yield*/, buildFrameEnergies(spectrums, totalFrames)];
|
|
515
|
+
case 1:
|
|
516
|
+
energies = _g.sent();
|
|
517
|
+
excludeRanges = [];
|
|
518
|
+
rawSegments = [];
|
|
519
|
+
beatFrames = buildBeatCandidates(fps, totalFrames, beatEvents, beatGrid);
|
|
520
|
+
maxShiftFrames = Math.round(exports.MAX_BEAT_SNAP_SEC * fps);
|
|
521
|
+
useDrops = leadInFrames !== undefined && leadInFrames > 0;
|
|
522
|
+
dropWindowFrames = Math.round(exports.DROP_WINDOW_SEC * fps);
|
|
523
|
+
n = Math.max(1, Math.floor(segmentCount));
|
|
524
|
+
k = 0;
|
|
525
|
+
_g.label = 2;
|
|
526
|
+
case 2:
|
|
527
|
+
if (!(k < n)) return [3 /*break*/, 7];
|
|
528
|
+
drop = useDrops
|
|
529
|
+
? findBestDropSegment(energies, totalFrames, highlightFrames, leadInFrames, dropWindowFrames, excludeRanges)
|
|
530
|
+
: null;
|
|
531
|
+
if (!drop) return [3 /*break*/, 4];
|
|
532
|
+
snapDropToBeat(drop, beatFrames, (_a = beatGrid) === null || _a === void 0 ? void 0 : _a.periodFrames);
|
|
533
|
+
main = snapSegmentToBeats({ startFrame: drop.dropFrame, highlightFrames: highlightFrames }, totalFrames, beatFrames, maxShiftFrames, (_b = beatGrid) === null || _b === void 0 ? void 0 : _b.periodFrames, true);
|
|
534
|
+
dropLeadIn = drop.dropFrame - drop.startFrame;
|
|
535
|
+
rawSegments.push({
|
|
536
|
+
startFrame: drop.startFrame,
|
|
537
|
+
highlightFrames: dropLeadIn + main.highlightFrames,
|
|
538
|
+
leadInFrames: dropLeadIn,
|
|
539
|
+
});
|
|
540
|
+
excludeRanges.push({
|
|
541
|
+
start: drop.startFrame - maxShiftFrames,
|
|
542
|
+
endExclusive: main.startFrame + main.highlightFrames + maxShiftFrames,
|
|
543
|
+
});
|
|
544
|
+
return [4 /*yield*/, waitForEventLoop_1.waitForEventLoop()];
|
|
545
|
+
case 3:
|
|
546
|
+
_g.sent();
|
|
547
|
+
return [3 /*break*/, 6];
|
|
548
|
+
case 4: return [4 /*yield*/, findBestHighlightStart(energies, totalFrames, highlightFrames, excludeRanges)];
|
|
549
|
+
case 5:
|
|
550
|
+
startFrame = _g.sent();
|
|
551
|
+
if (startFrame === null) {
|
|
552
|
+
return [3 /*break*/, 7];
|
|
553
|
+
}
|
|
554
|
+
snapped = snapSegmentToBeats({ startFrame: startFrame, highlightFrames: highlightFrames }, totalFrames, beatFrames, maxShiftFrames, (_c = beatGrid) === null || _c === void 0 ? void 0 : _c.periodFrames);
|
|
555
|
+
windowLeadIn = useDrops ? leadInFrames : 0;
|
|
556
|
+
rawSegments.push(windowLeadIn > 0
|
|
557
|
+
? {
|
|
558
|
+
startFrame: snapped.startFrame - windowLeadIn,
|
|
559
|
+
highlightFrames: windowLeadIn + snapped.highlightFrames,
|
|
560
|
+
leadInFrames: windowLeadIn,
|
|
561
|
+
}
|
|
562
|
+
: snapped);
|
|
563
|
+
// Padded so the next raw window, once snapped, cannot reach back into this one.
|
|
564
|
+
excludeRanges.push({
|
|
565
|
+
start: snapped.startFrame - maxShiftFrames,
|
|
566
|
+
endExclusive: snapped.startFrame + snapped.highlightFrames + maxShiftFrames,
|
|
567
|
+
});
|
|
568
|
+
_g.label = 6;
|
|
569
|
+
case 6:
|
|
570
|
+
k++;
|
|
571
|
+
return [3 /*break*/, 2];
|
|
572
|
+
case 7:
|
|
573
|
+
rawSegments.sort(function (a, b) { return a.startFrame - b.startFrame; });
|
|
574
|
+
runs = rawSegments.map(function (seg) {
|
|
575
|
+
return buildHighlightRun(fps, seg, spectrums, beatEvents);
|
|
576
|
+
});
|
|
577
|
+
slicedSpectrums = [];
|
|
578
|
+
beatFrameIndices = [];
|
|
579
|
+
beatIntensities = [];
|
|
580
|
+
outOffset = 0;
|
|
581
|
+
for (_i = 0, runs_1 = runs; _i < runs_1.length; _i++) {
|
|
582
|
+
run_1 = runs_1[_i];
|
|
583
|
+
slicedSpectrums.push.apply(slicedSpectrums, run_1.spectrums);
|
|
584
|
+
beatFrameIndices.push.apply(beatFrameIndices, run_1.beatFrameIndices.map(function (frameIndex) { return frameIndex + outOffset; }));
|
|
585
|
+
beatIntensities.push.apply(beatIntensities, run_1.beatIntensities);
|
|
586
|
+
outOffset += run_1.highlightFrames;
|
|
587
|
+
}
|
|
588
|
+
totalHighlightFrames = slicedSpectrums.length;
|
|
589
|
+
audioSegments = runs.map(function (run) { return run.audioSegment; });
|
|
590
|
+
first = (_d = audioSegments[0], (_d !== null && _d !== void 0 ? _d : { seekSeconds: 0, durationSeconds: 0 }));
|
|
591
|
+
return [2 /*return*/, {
|
|
592
|
+
startFrame: (_f = (_e = rawSegments[0]) === null || _e === void 0 ? void 0 : _e.startFrame, (_f !== null && _f !== void 0 ? _f : 0)),
|
|
593
|
+
highlightFrames: totalHighlightFrames,
|
|
594
|
+
spectrums: slicedSpectrums,
|
|
595
|
+
beatFrameIndices: beatFrameIndices,
|
|
596
|
+
beatIntensities: beatIntensities,
|
|
597
|
+
audioSeekSeconds: first.seekSeconds,
|
|
598
|
+
audioDurationSeconds: first.durationSeconds,
|
|
599
|
+
audioSegments: audioSegments,
|
|
600
|
+
runs: runs,
|
|
601
|
+
}];
|
|
602
|
+
}
|
|
603
|
+
});
|
|
604
|
+
});
|
|
605
|
+
}
|
|
606
|
+
exports.computeHighlightSlice = computeHighlightSlice;
|
|
607
607
|
//# sourceMappingURL=highlight.js.map
|