@octoseq/mir 0.1.0-main.f22b738 → 0.1.0-main.f485112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-5WZ7TTDW.js +3234 -0
- package/dist/chunk-5WZ7TTDW.js.map +1 -0
- package/dist/index.d.ts +2122 -47
- package/dist/index.js +3325 -20
- package/dist/index.js.map +1 -1
- package/dist/{runMir-CSIBwNZ3.d.ts → runMir-Dmu75LP0.d.ts} +2 -2
- package/dist/runner/runMir.d.ts +2 -2
- package/dist/runner/runMir.js +1 -1
- package/dist/runner/workerProtocol.d.ts +8 -1
- package/dist/runner/workerProtocol.js.map +1 -1
- package/dist/types-DqH4umN8.d.ts +761 -0
- package/package.json +2 -2
- package/src/dsp/activity.ts +544 -0
- package/src/dsp/bandCqt.ts +662 -0
- package/src/dsp/bandEvents.ts +351 -0
- package/src/dsp/bandMask.ts +225 -0
- package/src/dsp/bandMir.ts +524 -0
- package/src/dsp/bandProposal.ts +552 -0
- package/src/dsp/beatCandidates.ts +299 -0
- package/src/dsp/cqt.ts +386 -0
- package/src/dsp/cqtSignals.ts +462 -0
- package/src/dsp/customSignalReduction.ts +841 -0
- package/src/dsp/eventToSignal.ts +531 -0
- package/src/dsp/frequencyBand.ts +956 -0
- package/src/dsp/mel.ts +56 -3
- package/src/dsp/musicalTime.ts +240 -0
- package/src/dsp/onset.ts +296 -30
- package/src/dsp/peakPicking.ts +519 -0
- package/src/dsp/phaseAlignment.ts +153 -0
- package/src/dsp/pitch.ts +289 -0
- package/src/dsp/resample.ts +44 -0
- package/src/dsp/signalTransforms.ts +660 -0
- package/src/dsp/silenceGating.ts +511 -0
- package/src/dsp/spectral.ts +124 -4
- package/src/dsp/tempoHypotheses.ts +395 -0
- package/src/index.ts +366 -5
- package/src/runner/runMir.ts +234 -2
- package/src/runner/workerProtocol.ts +9 -1
- package/src/types.ts +768 -3
- package/dist/chunk-DUWYCAVG.js +0 -1525
- package/dist/chunk-DUWYCAVG.js.map +0 -1
- package/dist/types-BE3py4fZ.d.ts +0 -83
|
@@ -0,0 +1,3234 @@
|
|
|
1
|
+
import FFT from 'fft.js';
|
|
2
|
+
|
|
3
|
+
// src/gpu/helpers.ts
|
|
4
|
+
function nowMs() {
|
|
5
|
+
return typeof performance !== "undefined" ? performance.now() : Date.now();
|
|
6
|
+
}
|
|
7
|
+
function byteSizeF32(n) {
|
|
8
|
+
return n * 4;
|
|
9
|
+
}
|
|
10
|
+
function createAndWriteStorageBuffer(gpu, data) {
|
|
11
|
+
const buf = gpu.device.createBuffer({
|
|
12
|
+
size: byteSizeF32(data.length),
|
|
13
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST
|
|
14
|
+
});
|
|
15
|
+
gpu.queue.writeBuffer(buf, 0, data);
|
|
16
|
+
return buf;
|
|
17
|
+
}
|
|
18
|
+
function createUniformBufferU32x4(gpu, u32x4) {
|
|
19
|
+
if (u32x4.length !== 4) throw new Error("@octoseq/mir: uniform buffer must be 4 u32 values");
|
|
20
|
+
const buf = gpu.device.createBuffer({
|
|
21
|
+
size: 16,
|
|
22
|
+
usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST
|
|
23
|
+
});
|
|
24
|
+
gpu.queue.writeBuffer(buf, 0, u32x4);
|
|
25
|
+
return buf;
|
|
26
|
+
}
|
|
27
|
+
function createStorageOutBuffer(gpu, byteLength) {
|
|
28
|
+
return gpu.device.createBuffer({
|
|
29
|
+
size: byteLength,
|
|
30
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC | GPUBufferUsage.COPY_DST
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
function createReadbackBuffer(gpu, byteLength) {
|
|
34
|
+
return gpu.device.createBuffer({
|
|
35
|
+
size: byteLength,
|
|
36
|
+
usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
async function submitAndReadback(gpu, encoder, outBuffer, readback, byteLength) {
|
|
40
|
+
encoder.copyBufferToBuffer(outBuffer, 0, readback, 0, byteLength);
|
|
41
|
+
const tSubmit = nowMs();
|
|
42
|
+
gpu.queue.submit([encoder.finish()]);
|
|
43
|
+
await readback.mapAsync(GPUMapMode.READ);
|
|
44
|
+
const tDone = nowMs();
|
|
45
|
+
const mapped = readback.getMappedRange();
|
|
46
|
+
const copy = mapped.slice(0);
|
|
47
|
+
readback.unmap();
|
|
48
|
+
return {
|
|
49
|
+
value: copy,
|
|
50
|
+
timing: {
|
|
51
|
+
gpuSubmitToReadbackMs: tDone - tSubmit
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// src/gpu/kernels/onsetEnvelope.wgsl.ts
|
|
57
|
+
var onsetEnvelopeWGSL = (
|
|
58
|
+
/* wgsl */
|
|
59
|
+
`
|
|
60
|
+
// Compute onset strength envelope from a (log) mel spectrogram.
|
|
61
|
+
//
|
|
62
|
+
// Input layout: melFlat[t*nMels + m]
|
|
63
|
+
// Output layout: out[t]
|
|
64
|
+
//
|
|
65
|
+
// We compute novelty per frame:
|
|
66
|
+
// novelty[t] = sum_m max(0, mel[t,m] - mel[t-1,m]) (rectified)
|
|
67
|
+
// or sum_m abs(...)
|
|
68
|
+
//
|
|
69
|
+
// One invocation computes one frame index (t). This is memory-bound but reduces a full
|
|
70
|
+
// (frames*mels) loop to the GPU and provides an end-to-end submit->readback timing.
|
|
71
|
+
|
|
72
|
+
struct Params {
|
|
73
|
+
nMels: u32,
|
|
74
|
+
nFrames: u32,
|
|
75
|
+
diffMethod: u32, // 0=rectified, 1=abs
|
|
76
|
+
_pad: u32,
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
@group(0) @binding(0) var<storage, read> melFlat: array<f32>;
|
|
80
|
+
@group(0) @binding(1) var<storage, read_write> out: array<f32>;
|
|
81
|
+
@group(0) @binding(2) var<uniform> params: Params;
|
|
82
|
+
|
|
83
|
+
@compute @workgroup_size(256)
|
|
84
|
+
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
85
|
+
let t = gid.x;
|
|
86
|
+
if (t >= params.nFrames) { return; }
|
|
87
|
+
|
|
88
|
+
if (t == 0u) {
|
|
89
|
+
out[t] = 0.0;
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let nMels = params.nMels;
|
|
94
|
+
var sum: f32 = 0.0;
|
|
95
|
+
|
|
96
|
+
// Linear loop: nMels is small (e.g. 64). Keeping it serial per-frame is fine.
|
|
97
|
+
// (Future optimisation: parallelise reduction within workgroup.)
|
|
98
|
+
for (var m: u32 = 0u; m < nMels; m = m + 1u) {
|
|
99
|
+
let a = melFlat[t * nMels + m];
|
|
100
|
+
let b = melFlat[(t - 1u) * nMels + m];
|
|
101
|
+
let d = a - b;
|
|
102
|
+
|
|
103
|
+
if (params.diffMethod == 1u) {
|
|
104
|
+
// abs
|
|
105
|
+
sum = sum + abs(d);
|
|
106
|
+
} else {
|
|
107
|
+
// rectified
|
|
108
|
+
sum = sum + max(0.0, d);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
out[t] = sum / max(1.0, f32(nMels));
|
|
113
|
+
}
|
|
114
|
+
`
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
// src/gpu/onsetEnvelope.ts
|
|
118
|
+
async function gpuOnsetEnvelopeFromMelFlat(gpu, input) {
|
|
119
|
+
const { device } = gpu;
|
|
120
|
+
const { nFrames, nMels, melFlat, diffMethod } = input;
|
|
121
|
+
if (melFlat.length !== nFrames * nMels) {
|
|
122
|
+
throw new Error("@octoseq/mir: melFlat length mismatch");
|
|
123
|
+
}
|
|
124
|
+
const melBuffer = createAndWriteStorageBuffer(gpu, melFlat);
|
|
125
|
+
const outByteLen = byteSizeF32(nFrames);
|
|
126
|
+
const outBuffer = createStorageOutBuffer(gpu, outByteLen);
|
|
127
|
+
const readback = createReadbackBuffer(gpu, outByteLen);
|
|
128
|
+
const shader = device.createShaderModule({ code: onsetEnvelopeWGSL });
|
|
129
|
+
const pipeline = device.createComputePipeline({
|
|
130
|
+
layout: "auto",
|
|
131
|
+
compute: { module: shader, entryPoint: "main" }
|
|
132
|
+
});
|
|
133
|
+
const diffU32 = diffMethod === "abs" ? 1 : 0;
|
|
134
|
+
const params = createUniformBufferU32x4(gpu, new Uint32Array([nMels, nFrames, diffU32, 0]));
|
|
135
|
+
const bindGroup = device.createBindGroup({
|
|
136
|
+
layout: pipeline.getBindGroupLayout(0),
|
|
137
|
+
entries: [
|
|
138
|
+
{ binding: 0, resource: { buffer: melBuffer } },
|
|
139
|
+
{ binding: 1, resource: { buffer: outBuffer } },
|
|
140
|
+
{ binding: 2, resource: { buffer: params } }
|
|
141
|
+
]
|
|
142
|
+
});
|
|
143
|
+
const encoder = device.createCommandEncoder();
|
|
144
|
+
const pass = encoder.beginComputePass();
|
|
145
|
+
pass.setPipeline(pipeline);
|
|
146
|
+
pass.setBindGroup(0, bindGroup);
|
|
147
|
+
const wg = Math.ceil(nFrames / 256);
|
|
148
|
+
pass.dispatchWorkgroups(wg);
|
|
149
|
+
pass.end();
|
|
150
|
+
const { value: bytes, timing } = await submitAndReadback(gpu, encoder, outBuffer, readback, outByteLen);
|
|
151
|
+
melBuffer.destroy();
|
|
152
|
+
outBuffer.destroy();
|
|
153
|
+
params.destroy();
|
|
154
|
+
readback.destroy();
|
|
155
|
+
return {
|
|
156
|
+
value: { out: new Float32Array(bytes) },
|
|
157
|
+
timing
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// src/dsp/silenceGating.ts
|
|
162
|
+
var DEFAULT_SILENCE_GATE_CONFIG = {
|
|
163
|
+
enabled: false,
|
|
164
|
+
// Disabled by default to avoid breaking existing behavior
|
|
165
|
+
energyPercentile: 10,
|
|
166
|
+
enterMargin: 6,
|
|
167
|
+
exitMargin: 3,
|
|
168
|
+
hangoverMs: 50,
|
|
169
|
+
minActiveMs: 0,
|
|
170
|
+
postSilenceSuppressMs: 50
|
|
171
|
+
};
|
|
172
|
+
var DEFAULT_BIN_GATE_CONFIG = {
|
|
173
|
+
enabled: true,
|
|
174
|
+
binFloorRel: 0.05
|
|
175
|
+
};
|
|
176
|
+
function withSilenceGateDefaults(config) {
|
|
177
|
+
return {
|
|
178
|
+
enabled: config?.enabled ?? DEFAULT_SILENCE_GATE_CONFIG.enabled,
|
|
179
|
+
energyPercentile: config?.energyPercentile ?? DEFAULT_SILENCE_GATE_CONFIG.energyPercentile,
|
|
180
|
+
enterMargin: config?.enterMargin ?? DEFAULT_SILENCE_GATE_CONFIG.enterMargin,
|
|
181
|
+
exitMargin: config?.exitMargin ?? DEFAULT_SILENCE_GATE_CONFIG.exitMargin,
|
|
182
|
+
hangoverMs: config?.hangoverMs ?? DEFAULT_SILENCE_GATE_CONFIG.hangoverMs,
|
|
183
|
+
minActiveMs: config?.minActiveMs ?? DEFAULT_SILENCE_GATE_CONFIG.minActiveMs,
|
|
184
|
+
postSilenceSuppressMs: config?.postSilenceSuppressMs ?? DEFAULT_SILENCE_GATE_CONFIG.postSilenceSuppressMs
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
function withBinGateDefaults(config) {
|
|
188
|
+
return {
|
|
189
|
+
enabled: config?.enabled ?? DEFAULT_BIN_GATE_CONFIG.enabled,
|
|
190
|
+
binFloorRel: config?.binFloorRel ?? DEFAULT_BIN_GATE_CONFIG.binFloorRel
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
function computeFrameEnergyFromMel(melBands) {
|
|
194
|
+
const nFrames = melBands.length;
|
|
195
|
+
const energy = new Float32Array(nFrames);
|
|
196
|
+
for (let t = 0; t < nFrames; t++) {
|
|
197
|
+
const bands = melBands[t];
|
|
198
|
+
if (!bands || bands.length === 0) {
|
|
199
|
+
energy[t] = -100;
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
let sum = 0;
|
|
203
|
+
for (let m = 0; m < bands.length; m++) {
|
|
204
|
+
sum += bands[m] ?? -100;
|
|
205
|
+
}
|
|
206
|
+
energy[t] = sum / bands.length;
|
|
207
|
+
}
|
|
208
|
+
return energy;
|
|
209
|
+
}
|
|
210
|
+
function computeFrameEnergyFromSpectrogram(magnitudes, usePower = false) {
|
|
211
|
+
const nFrames = magnitudes.length;
|
|
212
|
+
const energy = new Float32Array(nFrames);
|
|
213
|
+
for (let t = 0; t < nFrames; t++) {
|
|
214
|
+
const mags = magnitudes[t];
|
|
215
|
+
if (!mags || mags.length === 0) {
|
|
216
|
+
energy[t] = 0;
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
let sum = 0;
|
|
220
|
+
for (let k = 0; k < mags.length; k++) {
|
|
221
|
+
const m = mags[k] ?? 0;
|
|
222
|
+
sum += usePower ? m * m : m;
|
|
223
|
+
}
|
|
224
|
+
energy[t] = sum / mags.length;
|
|
225
|
+
}
|
|
226
|
+
return energy;
|
|
227
|
+
}
|
|
228
|
+
function computePercentile(values, percentile) {
|
|
229
|
+
if (values.length === 0) return 0;
|
|
230
|
+
const sorted = Float32Array.from(values).sort((a, b) => a - b);
|
|
231
|
+
const p = Math.max(0, Math.min(100, percentile));
|
|
232
|
+
const idx = p / 100 * (sorted.length - 1);
|
|
233
|
+
const lower = Math.floor(idx);
|
|
234
|
+
const upper = Math.ceil(idx);
|
|
235
|
+
if (lower === upper) {
|
|
236
|
+
return sorted[lower] ?? 0;
|
|
237
|
+
}
|
|
238
|
+
const frac = idx - lower;
|
|
239
|
+
const lowVal = sorted[lower] ?? 0;
|
|
240
|
+
const highVal = sorted[upper] ?? 0;
|
|
241
|
+
return lowVal + frac * (highVal - lowVal);
|
|
242
|
+
}
|
|
243
|
+
function estimateNoiseFloor(frameEnergy, percentile) {
|
|
244
|
+
return computePercentile(frameEnergy, percentile);
|
|
245
|
+
}
|
|
246
|
+
function buildActivityMask(frameEnergy, enterThreshold, exitThreshold, hangoverFrames) {
|
|
247
|
+
const nFrames = frameEnergy.length;
|
|
248
|
+
const mask = new Uint8Array(nFrames);
|
|
249
|
+
let isActive = false;
|
|
250
|
+
let hangoverRemaining = 0;
|
|
251
|
+
for (let t = 0; t < nFrames; t++) {
|
|
252
|
+
const e = frameEnergy[t] ?? 0;
|
|
253
|
+
if (!isActive) {
|
|
254
|
+
if (e >= enterThreshold) {
|
|
255
|
+
isActive = true;
|
|
256
|
+
hangoverRemaining = hangoverFrames;
|
|
257
|
+
mask[t] = 1;
|
|
258
|
+
} else {
|
|
259
|
+
mask[t] = 0;
|
|
260
|
+
}
|
|
261
|
+
} else {
|
|
262
|
+
if (e >= exitThreshold) {
|
|
263
|
+
hangoverRemaining = hangoverFrames;
|
|
264
|
+
mask[t] = 1;
|
|
265
|
+
} else if (hangoverRemaining > 0) {
|
|
266
|
+
hangoverRemaining--;
|
|
267
|
+
mask[t] = 1;
|
|
268
|
+
} else {
|
|
269
|
+
isActive = false;
|
|
270
|
+
mask[t] = 0;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
return mask;
|
|
275
|
+
}
|
|
276
|
+
function buildSuppressionMask(activityMask, suppressFrames) {
|
|
277
|
+
const nFrames = activityMask.length;
|
|
278
|
+
const suppress = new Uint8Array(nFrames);
|
|
279
|
+
let suppressRemaining = 0;
|
|
280
|
+
let wasActive = false;
|
|
281
|
+
for (let t = 0; t < nFrames; t++) {
|
|
282
|
+
const isActive = (activityMask[t] ?? 0) === 1;
|
|
283
|
+
if (isActive && !wasActive) {
|
|
284
|
+
suppressRemaining = suppressFrames;
|
|
285
|
+
}
|
|
286
|
+
if (isActive && suppressRemaining > 0) {
|
|
287
|
+
suppress[t] = 1;
|
|
288
|
+
suppressRemaining--;
|
|
289
|
+
} else {
|
|
290
|
+
suppress[t] = 0;
|
|
291
|
+
}
|
|
292
|
+
wasActive = isActive;
|
|
293
|
+
}
|
|
294
|
+
return suppress;
|
|
295
|
+
}
|
|
296
|
+
function applyMinActiveDuration(activityMask, minActiveFrames) {
|
|
297
|
+
if (minActiveFrames <= 1) return;
|
|
298
|
+
const nFrames = activityMask.length;
|
|
299
|
+
const regions = [];
|
|
300
|
+
let regionStart = -1;
|
|
301
|
+
for (let t = 0; t <= nFrames; t++) {
|
|
302
|
+
const isActive = t < nFrames && (activityMask[t] ?? 0) === 1;
|
|
303
|
+
if (isActive && regionStart < 0) {
|
|
304
|
+
regionStart = t;
|
|
305
|
+
} else if (!isActive && regionStart >= 0) {
|
|
306
|
+
regions.push({ start: regionStart, end: t });
|
|
307
|
+
regionStart = -1;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
for (const region of regions) {
|
|
311
|
+
const length = region.end - region.start;
|
|
312
|
+
if (length < minActiveFrames) {
|
|
313
|
+
for (let t = region.start; t < region.end; t++) {
|
|
314
|
+
activityMask[t] = 0;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
function computeSilenceGating(frameEnergy, frameDurationSec, config) {
|
|
320
|
+
const cfg = withSilenceGateDefaults(config);
|
|
321
|
+
const nFrames = frameEnergy.length;
|
|
322
|
+
if (!cfg.enabled) {
|
|
323
|
+
return {
|
|
324
|
+
frameEnergy,
|
|
325
|
+
noiseFloor: -Infinity,
|
|
326
|
+
enterThreshold: -Infinity,
|
|
327
|
+
exitThreshold: -Infinity,
|
|
328
|
+
activityMask: new Uint8Array(nFrames).fill(1),
|
|
329
|
+
suppressionMask: new Uint8Array(nFrames)
|
|
330
|
+
// All zeros = no suppression
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
const noiseFloor = estimateNoiseFloor(frameEnergy, cfg.energyPercentile);
|
|
334
|
+
const enterThreshold = noiseFloor + cfg.enterMargin;
|
|
335
|
+
const exitThreshold = noiseFloor + cfg.exitMargin;
|
|
336
|
+
const hangoverFrames = Math.max(0, Math.round(cfg.hangoverMs / 1e3 / frameDurationSec));
|
|
337
|
+
const suppressFrames = Math.max(0, Math.round(cfg.postSilenceSuppressMs / 1e3 / frameDurationSec));
|
|
338
|
+
const minActiveFrames = Math.max(0, Math.round(cfg.minActiveMs / 1e3 / frameDurationSec));
|
|
339
|
+
const activityMask = buildActivityMask(frameEnergy, enterThreshold, exitThreshold, hangoverFrames);
|
|
340
|
+
if (minActiveFrames > 1) {
|
|
341
|
+
applyMinActiveDuration(activityMask, minActiveFrames);
|
|
342
|
+
}
|
|
343
|
+
const suppressionMask = buildSuppressionMask(activityMask, suppressFrames);
|
|
344
|
+
return {
|
|
345
|
+
frameEnergy,
|
|
346
|
+
noiseFloor,
|
|
347
|
+
enterThreshold,
|
|
348
|
+
exitThreshold,
|
|
349
|
+
activityMask,
|
|
350
|
+
suppressionMask
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
function applySilenceGating(novelty, activityMask, suppressionMask) {
|
|
354
|
+
const nFrames = novelty.length;
|
|
355
|
+
for (let t = 0; t < nFrames; t++) {
|
|
356
|
+
const isActive = (activityMask[t] ?? 0) === 1;
|
|
357
|
+
const isSuppressed = (suppressionMask[t] ?? 0) === 1;
|
|
358
|
+
if (!isActive || isSuppressed) {
|
|
359
|
+
novelty[t] = 0;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
function computeBinFloor(frameEnergy, binFloorRel) {
|
|
364
|
+
return frameEnergy * binFloorRel;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// src/dsp/onset.ts
|
|
368
|
+
function movingAverage(values, windowFrames) {
|
|
369
|
+
if (windowFrames <= 1) return values;
|
|
370
|
+
const n = values.length;
|
|
371
|
+
const out = new Float32Array(n);
|
|
372
|
+
const half = Math.floor(windowFrames / 2);
|
|
373
|
+
const prefix = new Float64Array(n + 1);
|
|
374
|
+
prefix[0] = 0;
|
|
375
|
+
for (let i = 0; i < n; i++) {
|
|
376
|
+
prefix[i + 1] = (prefix[i] ?? 0) + (values[i] ?? 0);
|
|
377
|
+
}
|
|
378
|
+
for (let i = 0; i < n; i++) {
|
|
379
|
+
const start = Math.max(0, i - half);
|
|
380
|
+
const end = Math.min(n, i + half + 1);
|
|
381
|
+
const sum = (prefix[end] ?? 0) - (prefix[start] ?? 0);
|
|
382
|
+
const count = Math.max(1, end - start);
|
|
383
|
+
out[i] = sum / count;
|
|
384
|
+
}
|
|
385
|
+
return out;
|
|
386
|
+
}
|
|
387
|
+
function resolveOptions(opts) {
|
|
388
|
+
return {
|
|
389
|
+
useLog: opts?.useLog ?? false,
|
|
390
|
+
smoothMs: opts?.smoothMs ?? 30,
|
|
391
|
+
diffMethod: opts?.diffMethod ?? "rectified",
|
|
392
|
+
silenceGate: withSilenceGateDefaults(opts?.silenceGate),
|
|
393
|
+
binGate: withBinGateDefaults(opts?.binGate),
|
|
394
|
+
returnDiagnostics: opts?.returnDiagnostics ?? false
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
function logCompress(x) {
|
|
398
|
+
return Math.log1p(Math.max(0, x));
|
|
399
|
+
}
|
|
400
|
+
function onsetEnvelopeFromSpectrogram(spec, options) {
|
|
401
|
+
const opts = resolveOptions(options);
|
|
402
|
+
const nFrames = spec.times.length;
|
|
403
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
404
|
+
const out = new Float32Array(nFrames);
|
|
405
|
+
out[0] = 0;
|
|
406
|
+
const frameEnergies = opts.binGate.enabled ? computeFrameEnergyFromSpectrogram(spec.magnitudes, false) : null;
|
|
407
|
+
for (let t = 1; t < nFrames; t++) {
|
|
408
|
+
const cur = spec.magnitudes[t];
|
|
409
|
+
const prev = spec.magnitudes[t - 1];
|
|
410
|
+
if (!cur || !prev) {
|
|
411
|
+
out[t] = 0;
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
const binFloor = frameEnergies && opts.binGate.enabled ? computeBinFloor(frameEnergies[t] ?? 0, opts.binGate.binFloorRel) : 0;
|
|
415
|
+
let sum = 0;
|
|
416
|
+
let validBins = 0;
|
|
417
|
+
for (let k = 0; k < nBins; k++) {
|
|
418
|
+
let a = cur[k] ?? 0;
|
|
419
|
+
let b = prev[k] ?? 0;
|
|
420
|
+
if (opts.binGate.enabled && a < binFloor && b < binFloor) {
|
|
421
|
+
continue;
|
|
422
|
+
}
|
|
423
|
+
if (opts.useLog) {
|
|
424
|
+
a = logCompress(a);
|
|
425
|
+
b = logCompress(b);
|
|
426
|
+
}
|
|
427
|
+
const d = a - b;
|
|
428
|
+
sum += opts.diffMethod === "abs" ? Math.abs(d) : Math.max(0, d);
|
|
429
|
+
validBins++;
|
|
430
|
+
}
|
|
431
|
+
out[t] = validBins > 0 ? sum / validBins : 0;
|
|
432
|
+
}
|
|
433
|
+
const linearEnergy = computeFrameEnergyFromSpectrogram(spec.magnitudes, false);
|
|
434
|
+
const frameEnergy = new Float32Array(nFrames);
|
|
435
|
+
const eps = 1e-12;
|
|
436
|
+
for (let t = 0; t < nFrames; t++) {
|
|
437
|
+
frameEnergy[t] = Math.log10(eps + (linearEnergy[t] ?? 0));
|
|
438
|
+
}
|
|
439
|
+
const frameDurationSec = nFrames >= 2 ? (spec.times[1] ?? 0) - (spec.times[0] ?? 0) : 0.01;
|
|
440
|
+
const gating = computeSilenceGating(frameEnergy, frameDurationSec, opts.silenceGate);
|
|
441
|
+
const rawNovelty = opts.returnDiagnostics ? Float32Array.from(out) : void 0;
|
|
442
|
+
if (opts.silenceGate.enabled) {
|
|
443
|
+
applySilenceGating(out, gating.activityMask, gating.suppressionMask);
|
|
444
|
+
}
|
|
445
|
+
let values = out;
|
|
446
|
+
const smoothMs = opts.smoothMs;
|
|
447
|
+
if (smoothMs > 0 && nFrames >= 2) {
|
|
448
|
+
const dt = frameDurationSec;
|
|
449
|
+
const windowFrames = Math.max(1, Math.round(smoothMs / 1e3 / Math.max(1e-9, dt)));
|
|
450
|
+
const smoothed = movingAverage(out, windowFrames | 1);
|
|
451
|
+
values = new Float32Array(smoothed);
|
|
452
|
+
}
|
|
453
|
+
const result = { times: spec.times, values };
|
|
454
|
+
if (opts.returnDiagnostics && rawNovelty) {
|
|
455
|
+
result.diagnostics = {
|
|
456
|
+
frameEnergy,
|
|
457
|
+
noiseFloor: gating.noiseFloor,
|
|
458
|
+
enterThreshold: gating.enterThreshold,
|
|
459
|
+
exitThreshold: gating.exitThreshold,
|
|
460
|
+
activityMask: gating.activityMask,
|
|
461
|
+
suppressionMask: gating.suppressionMask,
|
|
462
|
+
rawNovelty
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
return result;
|
|
466
|
+
}
|
|
467
|
+
function onsetEnvelopeFromMel(mel, options) {
|
|
468
|
+
const opts = resolveOptions(options);
|
|
469
|
+
const nFrames = mel.times.length;
|
|
470
|
+
const out = new Float32Array(nFrames);
|
|
471
|
+
out[0] = 0;
|
|
472
|
+
const melFrameEnergies = opts.binGate.enabled ? computeFrameEnergyFromMel(mel.melBands) : null;
|
|
473
|
+
for (let t = 1; t < nFrames; t++) {
|
|
474
|
+
const cur = mel.melBands[t];
|
|
475
|
+
const prev = mel.melBands[t - 1];
|
|
476
|
+
if (!cur || !prev) {
|
|
477
|
+
out[t] = 0;
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
const nBands = cur.length;
|
|
481
|
+
const frameEnergyLinear = melFrameEnergies ? Math.pow(10, melFrameEnergies[t] ?? -100) : 0;
|
|
482
|
+
const binFloorLinear = opts.binGate.enabled ? computeBinFloor(frameEnergyLinear, opts.binGate.binFloorRel) : 0;
|
|
483
|
+
let sum = 0;
|
|
484
|
+
let validBands = 0;
|
|
485
|
+
for (let m = 0; m < nBands; m++) {
|
|
486
|
+
let a = cur[m] ?? 0;
|
|
487
|
+
let b = prev[m] ?? 0;
|
|
488
|
+
if (opts.binGate.enabled) {
|
|
489
|
+
const aLinear = Math.pow(10, a);
|
|
490
|
+
const bLinear = Math.pow(10, b);
|
|
491
|
+
if (aLinear < binFloorLinear && bLinear < binFloorLinear) {
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
if (opts.useLog) {
|
|
496
|
+
a = logCompress(a);
|
|
497
|
+
b = logCompress(b);
|
|
498
|
+
}
|
|
499
|
+
const d = a - b;
|
|
500
|
+
sum += opts.diffMethod === "abs" ? Math.abs(d) : Math.max(0, d);
|
|
501
|
+
validBands++;
|
|
502
|
+
}
|
|
503
|
+
out[t] = validBands > 0 ? sum / validBands : 0;
|
|
504
|
+
}
|
|
505
|
+
const frameEnergy = computeFrameEnergyFromMel(mel.melBands);
|
|
506
|
+
const frameDurationSec = nFrames >= 2 ? (mel.times[1] ?? 0) - (mel.times[0] ?? 0) : 0.01;
|
|
507
|
+
const gating = computeSilenceGating(frameEnergy, frameDurationSec, opts.silenceGate);
|
|
508
|
+
const rawNovelty = opts.returnDiagnostics ? Float32Array.from(out) : void 0;
|
|
509
|
+
if (opts.silenceGate.enabled) {
|
|
510
|
+
applySilenceGating(out, gating.activityMask, gating.suppressionMask);
|
|
511
|
+
}
|
|
512
|
+
let values = out;
|
|
513
|
+
const smoothMs = opts.smoothMs;
|
|
514
|
+
if (smoothMs > 0 && nFrames >= 2) {
|
|
515
|
+
const dt = frameDurationSec;
|
|
516
|
+
const windowFrames = Math.max(1, Math.round(smoothMs / 1e3 / Math.max(1e-9, dt)));
|
|
517
|
+
const smoothed = movingAverage(out, windowFrames | 1);
|
|
518
|
+
values = new Float32Array(smoothed);
|
|
519
|
+
}
|
|
520
|
+
const result = { times: mel.times, values };
|
|
521
|
+
if (opts.returnDiagnostics && rawNovelty) {
|
|
522
|
+
result.diagnostics = {
|
|
523
|
+
frameEnergy,
|
|
524
|
+
noiseFloor: gating.noiseFloor,
|
|
525
|
+
enterThreshold: gating.enterThreshold,
|
|
526
|
+
exitThreshold: gating.exitThreshold,
|
|
527
|
+
activityMask: gating.activityMask,
|
|
528
|
+
suppressionMask: gating.suppressionMask,
|
|
529
|
+
rawNovelty
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
return result;
|
|
533
|
+
}
|
|
534
|
+
async function onsetEnvelopeFromMelGpu(mel, gpu, options) {
|
|
535
|
+
const nFrames = mel.times.length;
|
|
536
|
+
const nMels = mel.melBands[0]?.length ?? 0;
|
|
537
|
+
const melFlat = new Float32Array(nFrames * nMels);
|
|
538
|
+
for (let t = 0; t < nFrames; t++) {
|
|
539
|
+
const row = mel.melBands[t];
|
|
540
|
+
if (!row) continue;
|
|
541
|
+
melFlat.set(row, t * nMels);
|
|
542
|
+
}
|
|
543
|
+
const diffMethod = options?.diffMethod ?? "rectified";
|
|
544
|
+
const silenceGateConfig = withSilenceGateDefaults(options?.silenceGate);
|
|
545
|
+
const returnDiagnostics = options?.returnDiagnostics ?? false;
|
|
546
|
+
const smoothMs = options?.smoothMs ?? 30;
|
|
547
|
+
const { value, timing } = await gpuOnsetEnvelopeFromMelFlat(gpu, {
|
|
548
|
+
nFrames,
|
|
549
|
+
nMels,
|
|
550
|
+
melFlat,
|
|
551
|
+
diffMethod
|
|
552
|
+
});
|
|
553
|
+
const out = Float32Array.from(value.out);
|
|
554
|
+
const rawNovelty = returnDiagnostics ? Float32Array.from(out) : void 0;
|
|
555
|
+
const frameEnergy = computeFrameEnergyFromMel(mel.melBands);
|
|
556
|
+
const frameDurationSec = nFrames >= 2 ? (mel.times[1] ?? 0) - (mel.times[0] ?? 0) : 0.01;
|
|
557
|
+
const gating = computeSilenceGating(frameEnergy, frameDurationSec, silenceGateConfig);
|
|
558
|
+
if (silenceGateConfig.enabled) {
|
|
559
|
+
applySilenceGating(out, gating.activityMask, gating.suppressionMask);
|
|
560
|
+
}
|
|
561
|
+
let values = out;
|
|
562
|
+
if (smoothMs > 0 && nFrames >= 2) {
|
|
563
|
+
const dt = frameDurationSec;
|
|
564
|
+
const windowFrames = Math.max(1, Math.round(smoothMs / 1e3 / Math.max(1e-9, dt)));
|
|
565
|
+
const smoothed = movingAverage(out, windowFrames | 1);
|
|
566
|
+
values = new Float32Array(smoothed);
|
|
567
|
+
}
|
|
568
|
+
const result = {
|
|
569
|
+
times: mel.times,
|
|
570
|
+
values,
|
|
571
|
+
gpuTimings: { gpuSubmitToReadbackMs: timing.gpuSubmitToReadbackMs }
|
|
572
|
+
};
|
|
573
|
+
if (returnDiagnostics && rawNovelty) {
|
|
574
|
+
result.diagnostics = {
|
|
575
|
+
frameEnergy,
|
|
576
|
+
noiseFloor: gating.noiseFloor,
|
|
577
|
+
enterThreshold: gating.enterThreshold,
|
|
578
|
+
exitThreshold: gating.exitThreshold,
|
|
579
|
+
activityMask: gating.activityMask,
|
|
580
|
+
suppressionMask: gating.suppressionMask,
|
|
581
|
+
rawNovelty
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
return result;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// src/dsp/spectral.ts
|
|
588
|
+
function amplitudeEnvelope(samples, sampleRate, config) {
|
|
589
|
+
const hopSize = config?.hopSize ?? 512;
|
|
590
|
+
const windowSize = config?.windowSize ?? hopSize;
|
|
591
|
+
const nFrames = Math.floor((samples.length - windowSize) / hopSize) + 1;
|
|
592
|
+
const times = new Float32Array(nFrames);
|
|
593
|
+
const values = new Float32Array(nFrames);
|
|
594
|
+
for (let t = 0; t < nFrames; t++) {
|
|
595
|
+
const start = t * hopSize;
|
|
596
|
+
const end = Math.min(start + windowSize, samples.length);
|
|
597
|
+
let sumSq = 0;
|
|
598
|
+
for (let i = start; i < end; i++) {
|
|
599
|
+
const s = samples[i] ?? 0;
|
|
600
|
+
sumSq += s * s;
|
|
601
|
+
}
|
|
602
|
+
const rms = Math.sqrt(sumSq / (end - start));
|
|
603
|
+
times[t] = (start + windowSize / 2) / sampleRate;
|
|
604
|
+
values[t] = rms;
|
|
605
|
+
}
|
|
606
|
+
return { times, values };
|
|
607
|
+
}
|
|
608
|
+
function spectralCentroid(spec, options) {
|
|
609
|
+
const nFrames = spec.times.length;
|
|
610
|
+
const out = new Float32Array(nFrames);
|
|
611
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
612
|
+
const binHz = spec.sampleRate / spec.fftSize;
|
|
613
|
+
const activity = options?.activity;
|
|
614
|
+
const holdBehavior = options?.inactiveBehavior === "hold";
|
|
615
|
+
let lastActiveValue = 0;
|
|
616
|
+
for (let t = 0; t < nFrames; t++) {
|
|
617
|
+
const isActive = activity ? (activity.isActive[t] ?? 0) === 1 : true;
|
|
618
|
+
if (!isActive) {
|
|
619
|
+
out[t] = holdBehavior ? lastActiveValue : 0;
|
|
620
|
+
continue;
|
|
621
|
+
}
|
|
622
|
+
const mags = spec.magnitudes[t];
|
|
623
|
+
if (!mags) {
|
|
624
|
+
out[t] = holdBehavior ? lastActiveValue : 0;
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
let num = 0;
|
|
628
|
+
let den = 0;
|
|
629
|
+
for (let k = 0; k < nBins; k++) {
|
|
630
|
+
const m = mags[k] ?? 0;
|
|
631
|
+
const f = k * binHz;
|
|
632
|
+
num += f * m;
|
|
633
|
+
den += m;
|
|
634
|
+
}
|
|
635
|
+
const centroid = den > 0 ? num / den : 0;
|
|
636
|
+
out[t] = centroid;
|
|
637
|
+
lastActiveValue = centroid;
|
|
638
|
+
}
|
|
639
|
+
return out;
|
|
640
|
+
}
|
|
641
|
+
function spectralFlux(spec, options) {
|
|
642
|
+
const nFrames = spec.times.length;
|
|
643
|
+
const out = new Float32Array(nFrames);
|
|
644
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
645
|
+
const activity = options?.activity;
|
|
646
|
+
let prev = null;
|
|
647
|
+
for (let t = 0; t < nFrames; t++) {
|
|
648
|
+
const isActive = activity ? (activity.isActive[t] ?? 0) === 1 : true;
|
|
649
|
+
const isSuppressed = activity ? (activity.suppressMask[t] ?? 0) === 1 : false;
|
|
650
|
+
if (!isActive || isSuppressed) {
|
|
651
|
+
out[t] = 0;
|
|
652
|
+
prev = null;
|
|
653
|
+
continue;
|
|
654
|
+
}
|
|
655
|
+
const mags = spec.magnitudes[t];
|
|
656
|
+
if (!mags) {
|
|
657
|
+
out[t] = 0;
|
|
658
|
+
prev = null;
|
|
659
|
+
continue;
|
|
660
|
+
}
|
|
661
|
+
let sum = 0;
|
|
662
|
+
for (let k = 0; k < nBins; k++) sum += mags[k] ?? 0;
|
|
663
|
+
if (sum <= 0) {
|
|
664
|
+
out[t] = 0;
|
|
665
|
+
prev = null;
|
|
666
|
+
continue;
|
|
667
|
+
}
|
|
668
|
+
const cur = new Float32Array(nBins);
|
|
669
|
+
const inv = 1 / sum;
|
|
670
|
+
for (let k = 0; k < nBins; k++) cur[k] = (mags[k] ?? 0) * inv;
|
|
671
|
+
if (!prev) {
|
|
672
|
+
out[t] = 0;
|
|
673
|
+
prev = cur;
|
|
674
|
+
continue;
|
|
675
|
+
}
|
|
676
|
+
let flux = 0;
|
|
677
|
+
for (let k = 0; k < nBins; k++) {
|
|
678
|
+
const d = (cur[k] ?? 0) - (prev[k] ?? 0);
|
|
679
|
+
flux += Math.abs(d);
|
|
680
|
+
}
|
|
681
|
+
out[t] = flux;
|
|
682
|
+
prev = cur;
|
|
683
|
+
}
|
|
684
|
+
return out;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// src/dsp/beatCandidates.ts
|
|
688
|
+
function movingAverage2(values, windowFrames) {
|
|
689
|
+
if (windowFrames <= 1) return values;
|
|
690
|
+
const n = values.length;
|
|
691
|
+
const out = new Float32Array(n);
|
|
692
|
+
const half = Math.floor(windowFrames / 2);
|
|
693
|
+
const prefix = new Float64Array(n + 1);
|
|
694
|
+
prefix[0] = 0;
|
|
695
|
+
for (let i = 0; i < n; i++) {
|
|
696
|
+
prefix[i + 1] = (prefix[i] ?? 0) + (values[i] ?? 0);
|
|
697
|
+
}
|
|
698
|
+
for (let i = 0; i < n; i++) {
|
|
699
|
+
const start = Math.max(0, i - half);
|
|
700
|
+
const end = Math.min(n, i + half + 1);
|
|
701
|
+
const sum = (prefix[end] ?? 0) - (prefix[start] ?? 0);
|
|
702
|
+
const count = Math.max(1, end - start);
|
|
703
|
+
out[i] = sum / count;
|
|
704
|
+
}
|
|
705
|
+
return out;
|
|
706
|
+
}
|
|
707
|
+
function meanStd(values) {
|
|
708
|
+
const n = values.length;
|
|
709
|
+
if (n <= 0) return { mean: 0, std: 0 };
|
|
710
|
+
let mean = 0;
|
|
711
|
+
for (let i = 0; i < n; i++) mean += values[i] ?? 0;
|
|
712
|
+
mean /= n;
|
|
713
|
+
let varSum = 0;
|
|
714
|
+
for (let i = 0; i < n; i++) {
|
|
715
|
+
const d = (values[i] ?? 0) - mean;
|
|
716
|
+
varSum += d * d;
|
|
717
|
+
}
|
|
718
|
+
const std = Math.sqrt(varSum / n);
|
|
719
|
+
return { mean, std };
|
|
720
|
+
}
|
|
721
|
+
function zScoreNormalize(values) {
|
|
722
|
+
const { mean, std } = meanStd(values);
|
|
723
|
+
const n = values.length;
|
|
724
|
+
const out = new Float32Array(n);
|
|
725
|
+
if (std === 0 || !Number.isFinite(std)) {
|
|
726
|
+
out.fill(0);
|
|
727
|
+
return out;
|
|
728
|
+
}
|
|
729
|
+
for (let i = 0; i < n; i++) {
|
|
730
|
+
out[i] = ((values[i] ?? 0) - mean) / std;
|
|
731
|
+
}
|
|
732
|
+
return out;
|
|
733
|
+
}
|
|
734
|
+
function minMaxNormalize(values) {
|
|
735
|
+
const n = values.length;
|
|
736
|
+
if (n === 0) return new Float32Array(0);
|
|
737
|
+
let min = Infinity;
|
|
738
|
+
let max = -Infinity;
|
|
739
|
+
for (let i = 0; i < n; i++) {
|
|
740
|
+
const v = values[i] ?? 0;
|
|
741
|
+
if (v < min) min = v;
|
|
742
|
+
if (v > max) max = v;
|
|
743
|
+
}
|
|
744
|
+
const out = new Float32Array(n);
|
|
745
|
+
const range = max - min;
|
|
746
|
+
if (range === 0 || !Number.isFinite(range)) {
|
|
747
|
+
out.fill(0.5);
|
|
748
|
+
return out;
|
|
749
|
+
}
|
|
750
|
+
for (let i = 0; i < n; i++) {
|
|
751
|
+
out[i] = ((values[i] ?? 0) - min) / range;
|
|
752
|
+
}
|
|
753
|
+
return out;
|
|
754
|
+
}
|
|
755
|
+
function beatSalienceFromMel(mel, spec, options) {
|
|
756
|
+
const smoothMs = options?.smoothMs ?? 50;
|
|
757
|
+
const onset = onsetEnvelopeFromMel(mel, {
|
|
758
|
+
smoothMs,
|
|
759
|
+
diffMethod: "rectified",
|
|
760
|
+
useLog: false
|
|
761
|
+
});
|
|
762
|
+
const flux = spectralFlux(spec);
|
|
763
|
+
const n = Math.min(onset.times.length, flux.length);
|
|
764
|
+
const onsetNorm = zScoreNormalize(onset.values.subarray(0, n));
|
|
765
|
+
const fluxNorm = zScoreNormalize(flux.subarray(0, n));
|
|
766
|
+
const combined = new Float32Array(n);
|
|
767
|
+
const onsetWeight = 0.7;
|
|
768
|
+
const fluxWeight = 0.3;
|
|
769
|
+
for (let i = 0; i < n; i++) {
|
|
770
|
+
combined[i] = onsetWeight * (onsetNorm[i] ?? 0) + fluxWeight * (fluxNorm[i] ?? 0);
|
|
771
|
+
}
|
|
772
|
+
const dt = n >= 2 ? (onset.times[1] ?? 0) - (onset.times[0] ?? 0) : 0.01;
|
|
773
|
+
const windowFrames = Math.max(1, Math.round(smoothMs / 1e3 / Math.max(1e-9, dt)));
|
|
774
|
+
const smoothed = movingAverage2(combined, windowFrames | 1);
|
|
775
|
+
const normalized = minMaxNormalize(smoothed);
|
|
776
|
+
return {
|
|
777
|
+
times: onset.times.subarray(0, n),
|
|
778
|
+
values: normalized
|
|
779
|
+
};
|
|
780
|
+
}
|
|
781
|
+
function pickBeatCandidates(salience, options, source) {
|
|
782
|
+
const minIntervalSec = options.minIntervalSec ?? 0.1;
|
|
783
|
+
const thresholdFactor = options.thresholdFactor ?? 0.5;
|
|
784
|
+
const { times, values } = salience;
|
|
785
|
+
const n = values.length;
|
|
786
|
+
if (n < 3) return [];
|
|
787
|
+
const { mean, std } = meanStd(values);
|
|
788
|
+
const threshold = mean + thresholdFactor * std;
|
|
789
|
+
const candidates = [];
|
|
790
|
+
let lastPeakTime = -Infinity;
|
|
791
|
+
for (let i = 1; i < n - 1; i++) {
|
|
792
|
+
const v = values[i] ?? 0;
|
|
793
|
+
if (v < threshold) continue;
|
|
794
|
+
const prev = values[i - 1] ?? 0;
|
|
795
|
+
const next = values[i + 1] ?? 0;
|
|
796
|
+
if (!(v > prev && v > next)) continue;
|
|
797
|
+
const t = times[i] ?? 0;
|
|
798
|
+
if (t - lastPeakTime < minIntervalSec) {
|
|
799
|
+
const last = candidates[candidates.length - 1];
|
|
800
|
+
if (last && v > last.strength) {
|
|
801
|
+
last.time = t;
|
|
802
|
+
last.strength = v;
|
|
803
|
+
}
|
|
804
|
+
continue;
|
|
805
|
+
}
|
|
806
|
+
candidates.push({
|
|
807
|
+
time: t,
|
|
808
|
+
strength: v,
|
|
809
|
+
source
|
|
810
|
+
});
|
|
811
|
+
lastPeakTime = t;
|
|
812
|
+
}
|
|
813
|
+
return candidates;
|
|
814
|
+
}
|
|
815
|
+
function detectBeatCandidates(mel, spec, options) {
|
|
816
|
+
const opts = {
|
|
817
|
+
minIntervalSec: options?.minIntervalSec ?? 0.1,
|
|
818
|
+
thresholdFactor: options?.thresholdFactor ?? 0.5,
|
|
819
|
+
smoothMs: options?.smoothMs ?? 50
|
|
820
|
+
};
|
|
821
|
+
const salience = beatSalienceFromMel(mel, spec, { smoothMs: opts.smoothMs });
|
|
822
|
+
const candidates = pickBeatCandidates(salience, opts, "combined");
|
|
823
|
+
return {
|
|
824
|
+
candidates,
|
|
825
|
+
salience
|
|
826
|
+
};
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
// src/dsp/tempoHypotheses.ts
|
|
830
|
+
function intervalToBpm(intervalSec) {
|
|
831
|
+
return 60 / intervalSec;
|
|
832
|
+
}
|
|
833
|
+
function bpmToInterval(bpm) {
|
|
834
|
+
return 60 / bpm;
|
|
835
|
+
}
|
|
836
|
+
function computeIOIs(candidates, weightByStrength) {
|
|
837
|
+
if (candidates.length < 2) return [];
|
|
838
|
+
const iois = [];
|
|
839
|
+
const sorted = [...candidates].sort((a, b) => a.time - b.time);
|
|
840
|
+
for (let i = 1; i < sorted.length; i++) {
|
|
841
|
+
const prev = sorted[i - 1];
|
|
842
|
+
const curr = sorted[i];
|
|
843
|
+
const interval = curr.time - prev.time;
|
|
844
|
+
if (interval <= 0) continue;
|
|
845
|
+
const weight = weightByStrength ? Math.sqrt(prev.strength * curr.strength) : 1;
|
|
846
|
+
iois.push({ intervalSec: interval, weight });
|
|
847
|
+
}
|
|
848
|
+
return iois;
|
|
849
|
+
}
|
|
850
|
+
function buildBpmHistogram(iois, minBpm, maxBpm, binSizeBpm) {
|
|
851
|
+
const numBins = Math.ceil((maxBpm - minBpm) / binSizeBpm);
|
|
852
|
+
const counts = new Float32Array(numBins);
|
|
853
|
+
const bpmBins = new Float32Array(numBins);
|
|
854
|
+
for (let i = 0; i < numBins; i++) {
|
|
855
|
+
bpmBins[i] = minBpm + (i + 0.5) * binSizeBpm;
|
|
856
|
+
}
|
|
857
|
+
const minInterval = bpmToInterval(maxBpm);
|
|
858
|
+
const maxInterval = bpmToInterval(minBpm);
|
|
859
|
+
for (const { intervalSec, weight } of iois) {
|
|
860
|
+
if (intervalSec < minInterval || intervalSec > maxInterval) continue;
|
|
861
|
+
const bpm = intervalToBpm(intervalSec);
|
|
862
|
+
const binIndex = Math.floor((bpm - minBpm) / binSizeBpm);
|
|
863
|
+
if (binIndex >= 0 && binIndex < numBins) {
|
|
864
|
+
counts[binIndex] = (counts[binIndex] ?? 0) + weight;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
return { bpmBins, counts };
|
|
868
|
+
}
|
|
869
|
+
function findHistogramPeaks(counts, minHeight) {
|
|
870
|
+
const peaks = [];
|
|
871
|
+
for (let i = 1; i < counts.length - 1; i++) {
|
|
872
|
+
const curr = counts[i];
|
|
873
|
+
const prev = counts[i - 1];
|
|
874
|
+
const next = counts[i + 1];
|
|
875
|
+
if (curr > prev && curr > next && curr >= minHeight) {
|
|
876
|
+
peaks.push({ index: i, height: curr });
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
if (counts.length > 0 && counts[0] >= minHeight && counts[0] > (counts[1] ?? 0)) {
|
|
880
|
+
peaks.push({ index: 0, height: counts[0] });
|
|
881
|
+
}
|
|
882
|
+
if (counts.length > 1) {
|
|
883
|
+
const last = counts.length - 1;
|
|
884
|
+
if (counts[last] >= minHeight && counts[last] > (counts[last - 1] ?? 0)) {
|
|
885
|
+
peaks.push({ index: last, height: counts[last] });
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
peaks.sort((a, b) => b.height - a.height);
|
|
889
|
+
return peaks.map((p) => p.index);
|
|
890
|
+
}
|
|
891
|
+
function refinePeakBpm(peakIndex, bpmBins, counts, binSizeBpm) {
|
|
892
|
+
let totalWeight = 0;
|
|
893
|
+
let weightedBpm = 0;
|
|
894
|
+
let minBinBpm = bpmBins[peakIndex] - binSizeBpm / 2;
|
|
895
|
+
let maxBinBpm = bpmBins[peakIndex] + binSizeBpm / 2;
|
|
896
|
+
for (let offset = -1; offset <= 1; offset++) {
|
|
897
|
+
const idx = peakIndex + offset;
|
|
898
|
+
if (idx < 0 || idx >= bpmBins.length) continue;
|
|
899
|
+
const w = counts[idx];
|
|
900
|
+
const bpm = bpmBins[idx];
|
|
901
|
+
totalWeight += w;
|
|
902
|
+
weightedBpm += w * bpm;
|
|
903
|
+
if (w > 0) {
|
|
904
|
+
minBinBpm = Math.min(minBinBpm, bpm - binSizeBpm / 2);
|
|
905
|
+
maxBinBpm = Math.max(maxBinBpm, bpm + binSizeBpm / 2);
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
const refinedBpm = totalWeight > 0 ? weightedBpm / totalWeight : bpmBins[peakIndex];
|
|
909
|
+
return {
|
|
910
|
+
bpm: refinedBpm,
|
|
911
|
+
peakHeight: counts[peakIndex],
|
|
912
|
+
binRange: [minBinBpm, maxBinBpm],
|
|
913
|
+
totalWeight
|
|
914
|
+
};
|
|
915
|
+
}
|
|
916
|
+
function getHarmonicRatio(bpm1, bpm2, tolerance = 0.03) {
|
|
917
|
+
const ratios = [0.5, 1 / 3, 2 / 3, 1, 1.5, 2, 3];
|
|
918
|
+
for (const ratio of ratios) {
|
|
919
|
+
const expected = bpm1 * ratio;
|
|
920
|
+
const relativeError = Math.abs(bpm2 - expected) / expected;
|
|
921
|
+
if (relativeError <= tolerance) {
|
|
922
|
+
return ratio;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
return null;
|
|
926
|
+
}
|
|
927
|
+
function assignHarmonicFamilies(hypotheses) {
|
|
928
|
+
if (hypotheses.length === 0) return;
|
|
929
|
+
const families = /* @__PURE__ */ new Map();
|
|
930
|
+
for (const hyp of hypotheses) {
|
|
931
|
+
let foundFamily = false;
|
|
932
|
+
for (const [familyId, family] of families) {
|
|
933
|
+
const ratio = getHarmonicRatio(family.rootBpm, hyp.bpm);
|
|
934
|
+
if (ratio !== null) {
|
|
935
|
+
hyp.familyId = familyId;
|
|
936
|
+
hyp.harmonicRatio = ratio;
|
|
937
|
+
family.members.push(hyp);
|
|
938
|
+
foundFamily = true;
|
|
939
|
+
break;
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
if (!foundFamily) {
|
|
943
|
+
const familyId = `fam-${Math.round(hyp.bpm)}`;
|
|
944
|
+
hyp.familyId = familyId;
|
|
945
|
+
hyp.harmonicRatio = 1;
|
|
946
|
+
families.set(familyId, { rootBpm: hyp.bpm, members: [hyp] });
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
function normalizeConfidence(hypotheses) {
|
|
951
|
+
if (hypotheses.length === 0) return;
|
|
952
|
+
const maxHeight = Math.max(...hypotheses.map((h) => h.evidence.peakHeight));
|
|
953
|
+
if (maxHeight <= 0) return;
|
|
954
|
+
for (const hyp of hypotheses) {
|
|
955
|
+
hyp.confidence = hyp.evidence.peakHeight / maxHeight;
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
function generateTempoHypotheses(candidates, options) {
|
|
959
|
+
const minBpm = options?.minBpm ?? 24;
|
|
960
|
+
const maxBpm = options?.maxBpm ?? 300;
|
|
961
|
+
const binSizeBpm = options?.binSizeBpm ?? 1;
|
|
962
|
+
const maxHypotheses = options?.maxHypotheses ?? 10;
|
|
963
|
+
const minConfidence = options?.minConfidence ?? 0.05;
|
|
964
|
+
const weightByStrength = options?.weightByStrength ?? true;
|
|
965
|
+
const includeHistogram = options?.includeHistogram ?? false;
|
|
966
|
+
if (candidates.length < 2) {
|
|
967
|
+
return {
|
|
968
|
+
hypotheses: [],
|
|
969
|
+
inputCandidateCount: candidates.length,
|
|
970
|
+
histogram: includeHistogram ? {
|
|
971
|
+
bpmBins: new Float32Array(0),
|
|
972
|
+
counts: new Float32Array(0)
|
|
973
|
+
} : void 0
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
const iois = computeIOIs(candidates, weightByStrength);
|
|
977
|
+
if (iois.length === 0) {
|
|
978
|
+
return {
|
|
979
|
+
hypotheses: [],
|
|
980
|
+
inputCandidateCount: candidates.length,
|
|
981
|
+
histogram: includeHistogram ? {
|
|
982
|
+
bpmBins: new Float32Array(0),
|
|
983
|
+
counts: new Float32Array(0)
|
|
984
|
+
} : void 0
|
|
985
|
+
};
|
|
986
|
+
}
|
|
987
|
+
const { bpmBins, counts } = buildBpmHistogram(iois, minBpm, maxBpm, binSizeBpm);
|
|
988
|
+
const maxCount = Math.max(...counts);
|
|
989
|
+
const minHeight = maxCount * minConfidence;
|
|
990
|
+
const peakIndices = findHistogramPeaks(counts, minHeight);
|
|
991
|
+
const hypotheses = [];
|
|
992
|
+
for (const peakIndex of peakIndices.slice(0, maxHypotheses * 2)) {
|
|
993
|
+
const { bpm, peakHeight, binRange, totalWeight } = refinePeakBpm(
|
|
994
|
+
peakIndex,
|
|
995
|
+
bpmBins,
|
|
996
|
+
counts,
|
|
997
|
+
binSizeBpm
|
|
998
|
+
);
|
|
999
|
+
if (maxCount > 0 && peakHeight / maxCount < minConfidence) continue;
|
|
1000
|
+
const evidence = {
|
|
1001
|
+
supportingIntervalCount: Math.round(totalWeight),
|
|
1002
|
+
weightedSupport: totalWeight,
|
|
1003
|
+
peakHeight,
|
|
1004
|
+
binRange
|
|
1005
|
+
};
|
|
1006
|
+
hypotheses.push({
|
|
1007
|
+
id: "",
|
|
1008
|
+
// Will be assigned after sorting
|
|
1009
|
+
bpm: Math.round(bpm * 10) / 10,
|
|
1010
|
+
// Round to 0.1 BPM precision
|
|
1011
|
+
confidence: 0,
|
|
1012
|
+
// Will be normalized
|
|
1013
|
+
evidence,
|
|
1014
|
+
familyId: "",
|
|
1015
|
+
// Will be assigned
|
|
1016
|
+
harmonicRatio: 1
|
|
1017
|
+
// Will be assigned
|
|
1018
|
+
});
|
|
1019
|
+
}
|
|
1020
|
+
assignHarmonicFamilies(hypotheses);
|
|
1021
|
+
normalizeConfidence(hypotheses);
|
|
1022
|
+
const filtered = hypotheses.filter((h) => h.confidence >= minConfidence).sort((a, b) => b.confidence - a.confidence).slice(0, maxHypotheses);
|
|
1023
|
+
for (let i = 0; i < filtered.length; i++) {
|
|
1024
|
+
filtered[i].id = `hyp-${i}`;
|
|
1025
|
+
}
|
|
1026
|
+
return {
|
|
1027
|
+
hypotheses: filtered,
|
|
1028
|
+
inputCandidateCount: candidates.length,
|
|
1029
|
+
histogram: includeHistogram ? { bpmBins, counts } : void 0
|
|
1030
|
+
};
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
// src/gpu/kernels/melProject.wgsl.ts
|
|
1034
|
+
var melProjectWGSL = (
|
|
1035
|
+
/* wgsl */
|
|
1036
|
+
`
|
|
1037
|
+
struct Params {
|
|
1038
|
+
nBins: u32,
|
|
1039
|
+
nMels: u32,
|
|
1040
|
+
nFrames: u32,
|
|
1041
|
+
_pad: u32,
|
|
1042
|
+
};
|
|
1043
|
+
|
|
1044
|
+
@group(0) @binding(0) var<storage, read> mags : array<f32>;
|
|
1045
|
+
@group(0) @binding(1) var<storage, read> filters : array<f32>;
|
|
1046
|
+
@group(0) @binding(2) var<storage, read_write> out : array<f32>;
|
|
1047
|
+
@group(0) @binding(3) var<uniform> params : Params;
|
|
1048
|
+
|
|
1049
|
+
fn log10(x: f32) -> f32 {
|
|
1050
|
+
return log(x) / log(10.0);
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
@compute @workgroup_size(16, 16)
|
|
1054
|
+
fn main(@builtin(global_invocation_id) gid : vec3<u32>) {
|
|
1055
|
+
let frame = gid.x;
|
|
1056
|
+
let mel = gid.y;
|
|
1057
|
+
if (frame >= params.nFrames || mel >= params.nMels) {
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
var sum: f32 = 0.0;
|
|
1062
|
+
let bins = params.nBins;
|
|
1063
|
+
let magBase = frame * bins;
|
|
1064
|
+
let filBase = mel * bins;
|
|
1065
|
+
|
|
1066
|
+
for (var k: u32 = 0u; k < bins; k = k + 1u) {
|
|
1067
|
+
sum = sum + mags[magBase + k] * filters[filBase + k];
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
let eps: f32 = 1e-12;
|
|
1071
|
+
out[frame * params.nMels + mel] = log10(eps + sum);
|
|
1072
|
+
}
|
|
1073
|
+
`
|
|
1074
|
+
);
|
|
1075
|
+
|
|
1076
|
+
// src/gpu/melProject.ts
|
|
1077
|
+
async function gpuMelProjectFlat(gpu, input) {
|
|
1078
|
+
const { device } = gpu;
|
|
1079
|
+
const { nFrames, nBins, nMels, magsFlat, filterFlat } = input;
|
|
1080
|
+
if (magsFlat.length !== nFrames * nBins) {
|
|
1081
|
+
throw new Error("@octoseq/mir: magsFlat length mismatch");
|
|
1082
|
+
}
|
|
1083
|
+
if (filterFlat.length !== nMels * nBins) {
|
|
1084
|
+
throw new Error("@octoseq/mir: filterFlat length mismatch");
|
|
1085
|
+
}
|
|
1086
|
+
const magsBuffer = createAndWriteStorageBuffer(gpu, magsFlat);
|
|
1087
|
+
const filterBuffer = createAndWriteStorageBuffer(gpu, filterFlat);
|
|
1088
|
+
const outByteLen = byteSizeF32(nFrames * nMels);
|
|
1089
|
+
const outBuffer = createStorageOutBuffer(gpu, outByteLen);
|
|
1090
|
+
const readback = createReadbackBuffer(gpu, outByteLen);
|
|
1091
|
+
const shader = device.createShaderModule({ code: melProjectWGSL });
|
|
1092
|
+
const pipeline = device.createComputePipeline({
|
|
1093
|
+
layout: "auto",
|
|
1094
|
+
compute: {
|
|
1095
|
+
module: shader,
|
|
1096
|
+
entryPoint: "main"
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
const params = createUniformBufferU32x4(gpu, new Uint32Array([nBins, nMels, nFrames, 0]));
|
|
1100
|
+
const bindGroup = device.createBindGroup({
|
|
1101
|
+
layout: pipeline.getBindGroupLayout(0),
|
|
1102
|
+
entries: [
|
|
1103
|
+
{ binding: 0, resource: { buffer: magsBuffer } },
|
|
1104
|
+
{ binding: 1, resource: { buffer: filterBuffer } },
|
|
1105
|
+
{ binding: 2, resource: { buffer: outBuffer } },
|
|
1106
|
+
{ binding: 3, resource: { buffer: params } }
|
|
1107
|
+
]
|
|
1108
|
+
});
|
|
1109
|
+
const encoder = device.createCommandEncoder();
|
|
1110
|
+
const pass = encoder.beginComputePass();
|
|
1111
|
+
pass.setPipeline(pipeline);
|
|
1112
|
+
pass.setBindGroup(0, bindGroup);
|
|
1113
|
+
const wgX = Math.ceil(nFrames / 16);
|
|
1114
|
+
const wgY = Math.ceil(nMels / 16);
|
|
1115
|
+
pass.dispatchWorkgroups(wgX, wgY);
|
|
1116
|
+
pass.end();
|
|
1117
|
+
const { value: bytes, timing } = await submitAndReadback(gpu, encoder, outBuffer, readback, outByteLen);
|
|
1118
|
+
magsBuffer.destroy();
|
|
1119
|
+
filterBuffer.destroy();
|
|
1120
|
+
outBuffer.destroy();
|
|
1121
|
+
params.destroy();
|
|
1122
|
+
readback.destroy();
|
|
1123
|
+
const outFlat = new Float32Array(bytes);
|
|
1124
|
+
return {
|
|
1125
|
+
value: { outFlat },
|
|
1126
|
+
timing
|
|
1127
|
+
};
|
|
1128
|
+
}
|
|
1129
|
+
|
|
1130
|
+
// src/dsp/mel.ts
|
|
1131
|
+
function assertPositiveInt(name, value) {
|
|
1132
|
+
if (!Number.isFinite(value) || value <= 0 || (value | 0) !== value) {
|
|
1133
|
+
throw new Error(`@octoseq/mir: ${name} must be a positive integer`);
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
function hzToMel(hz) {
|
|
1137
|
+
return 2595 * Math.log10(1 + hz / 700);
|
|
1138
|
+
}
|
|
1139
|
+
function melToHz(mel) {
|
|
1140
|
+
return 700 * (Math.pow(10, mel / 2595) - 1);
|
|
1141
|
+
}
|
|
1142
|
+
function hzToFeatureIndex(hz, config) {
|
|
1143
|
+
const melMin = hzToMel(config.fMin);
|
|
1144
|
+
const melMax = hzToMel(config.fMax);
|
|
1145
|
+
const melHz = hzToMel(hz);
|
|
1146
|
+
const normalized = (melHz - melMin) / (melMax - melMin);
|
|
1147
|
+
return normalized * (config.nMels - 1);
|
|
1148
|
+
}
|
|
1149
|
+
function featureIndexToHz(index, config) {
|
|
1150
|
+
const melMin = hzToMel(config.fMin);
|
|
1151
|
+
const melMax = hzToMel(config.fMax);
|
|
1152
|
+
const normalized = index / (config.nMels - 1);
|
|
1153
|
+
const mel = melMin + normalized * (melMax - melMin);
|
|
1154
|
+
return melToHz(mel);
|
|
1155
|
+
}
|
|
1156
|
+
function buildMelFilterBank(sampleRate, fftSize, nMels, fMin, fMax) {
|
|
1157
|
+
const nBins = (fftSize >>> 1) + 1;
|
|
1158
|
+
const nyquist = sampleRate / 2;
|
|
1159
|
+
const fMinClamped = Math.max(0, Math.min(fMin, nyquist));
|
|
1160
|
+
const fMaxClamped = Math.max(0, Math.min(fMax, nyquist));
|
|
1161
|
+
if (fMaxClamped <= fMinClamped) {
|
|
1162
|
+
throw new Error("@octoseq/mir: mel fMax must be > fMin");
|
|
1163
|
+
}
|
|
1164
|
+
const melMin = hzToMel(fMinClamped);
|
|
1165
|
+
const melMax = hzToMel(fMaxClamped);
|
|
1166
|
+
const melPoints = new Float32Array(nMels + 2);
|
|
1167
|
+
for (let i = 0; i < melPoints.length; i++) {
|
|
1168
|
+
melPoints[i] = melMin + i * (melMax - melMin) / (nMels + 1);
|
|
1169
|
+
}
|
|
1170
|
+
const hzPoints = new Float32Array(melPoints.length);
|
|
1171
|
+
for (let i = 0; i < hzPoints.length; i++) hzPoints[i] = melToHz(melPoints[i] ?? 0);
|
|
1172
|
+
const binHz = sampleRate / fftSize;
|
|
1173
|
+
const binPoints = new Int32Array(hzPoints.length);
|
|
1174
|
+
for (let i = 0; i < binPoints.length; i++) {
|
|
1175
|
+
binPoints[i] = Math.max(0, Math.min(nBins - 1, Math.round((hzPoints[i] ?? 0) / binHz)));
|
|
1176
|
+
}
|
|
1177
|
+
const filters = new Array(nMels);
|
|
1178
|
+
for (let m = 0; m < nMels; m++) {
|
|
1179
|
+
const left = binPoints[m] ?? 0;
|
|
1180
|
+
const center = binPoints[m + 1] ?? 0;
|
|
1181
|
+
const right = binPoints[m + 2] ?? 0;
|
|
1182
|
+
const w = new Float32Array(nBins);
|
|
1183
|
+
if (center === left || right === center) {
|
|
1184
|
+
filters[m] = w;
|
|
1185
|
+
continue;
|
|
1186
|
+
}
|
|
1187
|
+
for (let k = left; k < center; k++) {
|
|
1188
|
+
w[k] = (k - left) / (center - left);
|
|
1189
|
+
}
|
|
1190
|
+
for (let k = center; k < right; k++) {
|
|
1191
|
+
w[k] = (right - k) / (right - center);
|
|
1192
|
+
}
|
|
1193
|
+
filters[m] = w;
|
|
1194
|
+
}
|
|
1195
|
+
return filters;
|
|
1196
|
+
}
|
|
1197
|
+
function cpuMelProject(spec, filters) {
|
|
1198
|
+
const nFrames = spec.times.length;
|
|
1199
|
+
const nMels = filters.length;
|
|
1200
|
+
const out = new Array(nFrames);
|
|
1201
|
+
const eps = 1e-12;
|
|
1202
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1203
|
+
const mags = spec.magnitudes[t];
|
|
1204
|
+
if (!mags) {
|
|
1205
|
+
out[t] = new Float32Array(nMels);
|
|
1206
|
+
continue;
|
|
1207
|
+
}
|
|
1208
|
+
const bands = new Float32Array(nMels);
|
|
1209
|
+
for (let m = 0; m < nMels; m++) {
|
|
1210
|
+
const w = filters[m];
|
|
1211
|
+
if (!w) continue;
|
|
1212
|
+
let sum = 0;
|
|
1213
|
+
for (let k = 0; k < mags.length; k++) {
|
|
1214
|
+
sum += (mags[k] ?? 0) * (w[k] ?? 0);
|
|
1215
|
+
}
|
|
1216
|
+
bands[m] = Math.log10(eps + sum);
|
|
1217
|
+
}
|
|
1218
|
+
out[t] = bands;
|
|
1219
|
+
}
|
|
1220
|
+
return {
|
|
1221
|
+
times: spec.times,
|
|
1222
|
+
melBands: out
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1225
|
+
async function gpuMelProject(spec, filters, gpu) {
|
|
1226
|
+
const nFrames = spec.times.length;
|
|
1227
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
1228
|
+
const nMels = filters.length;
|
|
1229
|
+
const magsFlat = new Float32Array(nFrames * nBins);
|
|
1230
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1231
|
+
const mags = spec.magnitudes[t];
|
|
1232
|
+
if (!mags) continue;
|
|
1233
|
+
magsFlat.set(mags, t * nBins);
|
|
1234
|
+
}
|
|
1235
|
+
const filterFlat = new Float32Array(nMels * nBins);
|
|
1236
|
+
for (let m = 0; m < nMels; m++) {
|
|
1237
|
+
filterFlat.set(filters[m] ?? new Float32Array(nBins), m * nBins);
|
|
1238
|
+
}
|
|
1239
|
+
const { value, timing } = await gpuMelProjectFlat(gpu, {
|
|
1240
|
+
nFrames,
|
|
1241
|
+
nBins,
|
|
1242
|
+
nMels,
|
|
1243
|
+
magsFlat,
|
|
1244
|
+
filterFlat
|
|
1245
|
+
});
|
|
1246
|
+
const outFlat = value.outFlat;
|
|
1247
|
+
const melBands = new Array(nFrames);
|
|
1248
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1249
|
+
melBands[t] = outFlat.subarray(t * nMels, (t + 1) * nMels);
|
|
1250
|
+
}
|
|
1251
|
+
return {
|
|
1252
|
+
times: spec.times,
|
|
1253
|
+
melBands,
|
|
1254
|
+
gpuTimings: {
|
|
1255
|
+
gpuSubmitToReadbackMs: timing.gpuSubmitToReadbackMs
|
|
1256
|
+
}
|
|
1257
|
+
};
|
|
1258
|
+
}
|
|
1259
|
+
async function melSpectrogram(spec, config, gpu) {
|
|
1260
|
+
assertPositiveInt("config.nMels", config.nMels);
|
|
1261
|
+
const fMin = config.fMin ?? 0;
|
|
1262
|
+
const fMax = config.fMax ?? spec.sampleRate / 2;
|
|
1263
|
+
const filters = buildMelFilterBank(
|
|
1264
|
+
spec.sampleRate,
|
|
1265
|
+
spec.fftSize,
|
|
1266
|
+
config.nMels,
|
|
1267
|
+
fMin,
|
|
1268
|
+
fMax
|
|
1269
|
+
);
|
|
1270
|
+
if (gpu) {
|
|
1271
|
+
try {
|
|
1272
|
+
return await gpuMelProject(spec, filters, gpu);
|
|
1273
|
+
} catch {
|
|
1274
|
+
return cpuMelProject(spec, filters);
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
return cpuMelProject(spec, filters);
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1280
|
+
// src/dsp/mfcc.ts
|
|
1281
|
+
function assertPositiveInt2(name, v) {
|
|
1282
|
+
if (!Number.isFinite(v) || v <= 0 || (v | 0) !== v) {
|
|
1283
|
+
throw new Error(`@octoseq/mir: ${name} must be a positive integer`);
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
function buildDctMatrix(nCoeffs, nMels) {
|
|
1287
|
+
const out = new Float32Array(nCoeffs * nMels);
|
|
1288
|
+
const scale0 = Math.sqrt(1 / nMels);
|
|
1289
|
+
const scale = Math.sqrt(2 / nMels);
|
|
1290
|
+
for (let i = 0; i < nCoeffs; i++) {
|
|
1291
|
+
for (let j = 0; j < nMels; j++) {
|
|
1292
|
+
const c = Math.cos(Math.PI / nMels * (j + 0.5) * i);
|
|
1293
|
+
out[i * nMels + j] = (i === 0 ? scale0 : scale) * c;
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
return out;
|
|
1297
|
+
}
|
|
1298
|
+
function mfcc(mel, options = {}) {
|
|
1299
|
+
const nFrames = mel.times.length;
|
|
1300
|
+
const nMels = mel.melBands[0]?.length ?? 0;
|
|
1301
|
+
const nCoeffs = options.nCoeffs ?? 13;
|
|
1302
|
+
assertPositiveInt2("options.nCoeffs", nCoeffs);
|
|
1303
|
+
if (nMels <= 0) {
|
|
1304
|
+
return { times: mel.times, coeffs: new Array(nFrames).fill(0).map(() => new Float32Array(nCoeffs)) };
|
|
1305
|
+
}
|
|
1306
|
+
const dct = buildDctMatrix(nCoeffs, nMels);
|
|
1307
|
+
const out = new Array(nFrames);
|
|
1308
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1309
|
+
const x = mel.melBands[t] ?? new Float32Array(nMels);
|
|
1310
|
+
const c = new Float32Array(nCoeffs);
|
|
1311
|
+
for (let i = 0; i < nCoeffs; i++) {
|
|
1312
|
+
let sum = 0;
|
|
1313
|
+
const rowOff = i * nMels;
|
|
1314
|
+
for (let j = 0; j < nMels; j++) {
|
|
1315
|
+
sum += (dct[rowOff + j] ?? 0) * (x[j] ?? 0);
|
|
1316
|
+
}
|
|
1317
|
+
c[i] = sum;
|
|
1318
|
+
}
|
|
1319
|
+
out[t] = c;
|
|
1320
|
+
}
|
|
1321
|
+
return { times: mel.times, coeffs: out };
|
|
1322
|
+
}
|
|
1323
|
+
function delta(features, options = {}) {
|
|
1324
|
+
const N = options.window ?? 2;
|
|
1325
|
+
assertPositiveInt2("options.window", N);
|
|
1326
|
+
const nFrames = features.times.length;
|
|
1327
|
+
const nFeat = features.values[0]?.length ?? 0;
|
|
1328
|
+
const out = new Array(nFrames);
|
|
1329
|
+
let denom = 0;
|
|
1330
|
+
for (let n = 1; n <= N; n++) denom += n * n;
|
|
1331
|
+
denom *= 2;
|
|
1332
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1333
|
+
const d = new Float32Array(nFeat);
|
|
1334
|
+
for (let f = 0; f < nFeat; f++) {
|
|
1335
|
+
let num = 0;
|
|
1336
|
+
for (let n = 1; n <= N; n++) {
|
|
1337
|
+
const tPlus = Math.min(nFrames - 1, t + n);
|
|
1338
|
+
const tMinus = Math.max(0, t - n);
|
|
1339
|
+
const a = features.values[tPlus]?.[f] ?? 0;
|
|
1340
|
+
const b = features.values[tMinus]?.[f] ?? 0;
|
|
1341
|
+
num += n * (a - b);
|
|
1342
|
+
}
|
|
1343
|
+
d[f] = denom > 0 ? num / denom : 0;
|
|
1344
|
+
}
|
|
1345
|
+
out[t] = d;
|
|
1346
|
+
}
|
|
1347
|
+
return { times: features.times, values: out };
|
|
1348
|
+
}
|
|
1349
|
+
function deltaDelta(features, options = {}) {
|
|
1350
|
+
return delta(delta(features, options), options);
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
// src/dsp/peakPick.ts
|
|
1354
|
+
function meanStd2(values) {
|
|
1355
|
+
const n = values.length;
|
|
1356
|
+
if (n <= 0) return { mean: 0, std: 0 };
|
|
1357
|
+
let mean = 0;
|
|
1358
|
+
for (let i = 0; i < n; i++) mean += values[i] ?? 0;
|
|
1359
|
+
mean /= n;
|
|
1360
|
+
let varSum = 0;
|
|
1361
|
+
for (let i = 0; i < n; i++) {
|
|
1362
|
+
const d = (values[i] ?? 0) - mean;
|
|
1363
|
+
varSum += d * d;
|
|
1364
|
+
}
|
|
1365
|
+
const std = Math.sqrt(varSum / n);
|
|
1366
|
+
return { mean, std };
|
|
1367
|
+
}
|
|
1368
|
+
function median(values) {
|
|
1369
|
+
const arr = Array.from(values);
|
|
1370
|
+
arr.sort((a, b) => a - b);
|
|
1371
|
+
const n = arr.length;
|
|
1372
|
+
if (n === 0) return 0;
|
|
1373
|
+
const mid = n >>> 1;
|
|
1374
|
+
if (n % 2 === 1) return arr[mid] ?? 0;
|
|
1375
|
+
return ((arr[mid - 1] ?? 0) + (arr[mid] ?? 0)) / 2;
|
|
1376
|
+
}
|
|
1377
|
+
function peakPick(times, values, options = {}) {
|
|
1378
|
+
if (times.length !== values.length) {
|
|
1379
|
+
throw new Error("@octoseq/mir: peakPick times/values length mismatch");
|
|
1380
|
+
}
|
|
1381
|
+
const n = values.length;
|
|
1382
|
+
if (n === 0) return [];
|
|
1383
|
+
const strict = options.strict ?? true;
|
|
1384
|
+
let thr = options.threshold ?? 0;
|
|
1385
|
+
if (options.adaptive) {
|
|
1386
|
+
const method = options.adaptive.method ?? "meanStd";
|
|
1387
|
+
const factor = options.adaptive.factor ?? 1;
|
|
1388
|
+
if (method === "median") {
|
|
1389
|
+
thr = median(values) * factor;
|
|
1390
|
+
} else {
|
|
1391
|
+
const { mean, std } = meanStd2(values);
|
|
1392
|
+
thr = mean + factor * std;
|
|
1393
|
+
}
|
|
1394
|
+
}
|
|
1395
|
+
const minIntervalSec = options.minIntervalSec ?? 0;
|
|
1396
|
+
const out = [];
|
|
1397
|
+
let lastPeakTime = -Infinity;
|
|
1398
|
+
for (let i = 1; i < n - 1; i++) {
|
|
1399
|
+
const v = values[i] ?? 0;
|
|
1400
|
+
if (!(v >= thr)) continue;
|
|
1401
|
+
const prev = values[i - 1] ?? 0;
|
|
1402
|
+
const next = values[i + 1] ?? 0;
|
|
1403
|
+
const isMax = strict ? v > prev && v > next : v >= prev && v >= next;
|
|
1404
|
+
if (!isMax) continue;
|
|
1405
|
+
const t = times[i] ?? 0;
|
|
1406
|
+
if (t - lastPeakTime < minIntervalSec) {
|
|
1407
|
+
const last = out[out.length - 1];
|
|
1408
|
+
if (last && v > last.strength) {
|
|
1409
|
+
last.time = t;
|
|
1410
|
+
last.strength = v;
|
|
1411
|
+
last.index = i;
|
|
1412
|
+
lastPeakTime = t;
|
|
1413
|
+
}
|
|
1414
|
+
continue;
|
|
1415
|
+
}
|
|
1416
|
+
out.push({ time: t, strength: v, index: i });
|
|
1417
|
+
lastPeakTime = t;
|
|
1418
|
+
}
|
|
1419
|
+
return out;
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
// src/dsp/hpss.ts
|
|
1423
|
+
function assertOddPositiveInt(name, v) {
|
|
1424
|
+
if (!Number.isFinite(v) || v <= 0 || (v | 0) !== v) {
|
|
1425
|
+
throw new Error(`@octoseq/mir: ${name} must be a positive integer`);
|
|
1426
|
+
}
|
|
1427
|
+
if (v % 2 !== 1) {
|
|
1428
|
+
throw new Error(`@octoseq/mir: ${name} must be odd`);
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
function medianOfWindow(values) {
|
|
1432
|
+
const arr = Array.from(values);
|
|
1433
|
+
arr.sort((a, b) => a - b);
|
|
1434
|
+
const mid = arr.length >>> 1;
|
|
1435
|
+
return arr[mid] ?? 0;
|
|
1436
|
+
}
|
|
1437
|
+
function medianFilterTime(spec, kTime, options) {
|
|
1438
|
+
const nFrames = spec.times.length;
|
|
1439
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
1440
|
+
const half = kTime >>> 1;
|
|
1441
|
+
const out = new Array(nFrames);
|
|
1442
|
+
const window = new Float32Array(kTime);
|
|
1443
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1444
|
+
if (options.isCancelled?.()) throw new Error("@octoseq/mir: cancelled");
|
|
1445
|
+
const row = new Float32Array(nBins);
|
|
1446
|
+
for (let k = 0; k < nBins; k++) {
|
|
1447
|
+
for (let i = -half, wi = 0; i <= half; i++, wi++) {
|
|
1448
|
+
const tt = Math.max(0, Math.min(nFrames - 1, t + i));
|
|
1449
|
+
const mags = spec.magnitudes[tt];
|
|
1450
|
+
window[wi] = mags ? mags[k] ?? 0 : 0;
|
|
1451
|
+
}
|
|
1452
|
+
row[k] = medianOfWindow(window);
|
|
1453
|
+
}
|
|
1454
|
+
out[t] = row;
|
|
1455
|
+
}
|
|
1456
|
+
return out;
|
|
1457
|
+
}
|
|
1458
|
+
function medianFilterFreq(spec, kFreq, options) {
|
|
1459
|
+
const nFrames = spec.times.length;
|
|
1460
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
1461
|
+
const half = kFreq >>> 1;
|
|
1462
|
+
const out = new Array(nFrames);
|
|
1463
|
+
const window = new Float32Array(kFreq);
|
|
1464
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1465
|
+
if (options.isCancelled?.()) throw new Error("@octoseq/mir: cancelled");
|
|
1466
|
+
const mags = spec.magnitudes[t] ?? new Float32Array(nBins);
|
|
1467
|
+
const row = new Float32Array(nBins);
|
|
1468
|
+
for (let k = 0; k < nBins; k++) {
|
|
1469
|
+
for (let i = -half, wi = 0; i <= half; i++, wi++) {
|
|
1470
|
+
const kk = Math.max(0, Math.min(nBins - 1, k + i));
|
|
1471
|
+
window[wi] = mags[kk] ?? 0;
|
|
1472
|
+
}
|
|
1473
|
+
row[k] = medianOfWindow(window);
|
|
1474
|
+
}
|
|
1475
|
+
out[t] = row;
|
|
1476
|
+
}
|
|
1477
|
+
return out;
|
|
1478
|
+
}
|
|
1479
|
+
function hpss(spec, options = {}) {
|
|
1480
|
+
const timeMedian = options.timeMedian ?? 17;
|
|
1481
|
+
const freqMedian = options.freqMedian ?? 17;
|
|
1482
|
+
assertOddPositiveInt("options.timeMedian", timeMedian);
|
|
1483
|
+
assertOddPositiveInt("options.freqMedian", freqMedian);
|
|
1484
|
+
const nFrames = spec.times.length;
|
|
1485
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
1486
|
+
const H = medianFilterTime(spec, timeMedian, options);
|
|
1487
|
+
const P = medianFilterFreq(spec, freqMedian, options);
|
|
1488
|
+
const harmonic = new Array(nFrames);
|
|
1489
|
+
const percussive = new Array(nFrames);
|
|
1490
|
+
const soft = options.softMask ?? true;
|
|
1491
|
+
const eps = 1e-12;
|
|
1492
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1493
|
+
if (options.isCancelled?.()) throw new Error("@octoseq/mir: cancelled");
|
|
1494
|
+
const mags = spec.magnitudes[t] ?? new Float32Array(nBins);
|
|
1495
|
+
const hRow = H[t] ?? new Float32Array(nBins);
|
|
1496
|
+
const pRow = P[t] ?? new Float32Array(nBins);
|
|
1497
|
+
const outH = new Float32Array(nBins);
|
|
1498
|
+
const outP = new Float32Array(nBins);
|
|
1499
|
+
for (let k = 0; k < nBins; k++) {
|
|
1500
|
+
const x = mags[k] ?? 0;
|
|
1501
|
+
const h = hRow[k] ?? 0;
|
|
1502
|
+
const p = pRow[k] ?? 0;
|
|
1503
|
+
if (soft) {
|
|
1504
|
+
const denom = Math.max(eps, h + p);
|
|
1505
|
+
const mh = h / denom;
|
|
1506
|
+
const mp = p / denom;
|
|
1507
|
+
outH[k] = x * mh;
|
|
1508
|
+
outP[k] = x * mp;
|
|
1509
|
+
} else {
|
|
1510
|
+
const isH = h >= p;
|
|
1511
|
+
outH[k] = isH ? x : 0;
|
|
1512
|
+
outP[k] = isH ? 0 : x;
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
harmonic[t] = outH;
|
|
1516
|
+
percussive[t] = outP;
|
|
1517
|
+
}
|
|
1518
|
+
return {
|
|
1519
|
+
harmonic: { times: spec.times, bins: nBins, frames: nFrames, magnitudes: harmonic },
|
|
1520
|
+
percussive: { times: spec.times, bins: nBins, frames: nFrames, magnitudes: percussive }
|
|
1521
|
+
};
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
// src/gpu/kernels/hpssMasks.wgsl.ts
|
|
1525
|
+
var hpssMasksWGSL = (
|
|
1526
|
+
/* wgsl */
|
|
1527
|
+
`
|
|
1528
|
+
struct Params {
|
|
1529
|
+
nBins: u32,
|
|
1530
|
+
nFrames: u32,
|
|
1531
|
+
softMask: u32, // 1 => soft, 0 => hard
|
|
1532
|
+
_pad: u32,
|
|
1533
|
+
};
|
|
1534
|
+
|
|
1535
|
+
@group(0) @binding(0) var<storage, read> mags : array<f32>;
|
|
1536
|
+
@group(0) @binding(1) var<storage, read_write> harmonicMask : array<f32>;
|
|
1537
|
+
@group(0) @binding(2) var<storage, read_write> percussiveMask : array<f32>;
|
|
1538
|
+
@group(0) @binding(3) var<uniform> params : Params;
|
|
1539
|
+
|
|
1540
|
+
fn clamp_i32(x: i32, lo: i32, hi: i32) -> i32 {
|
|
1541
|
+
return max(lo, min(hi, x));
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
fn swap_if_greater(a: ptr<function, f32>, b: ptr<function, f32>) {
|
|
1545
|
+
// Branchless compare\u2013swap.
|
|
1546
|
+
let av = *a;
|
|
1547
|
+
let bv = *b;
|
|
1548
|
+
*a = min(av, bv);
|
|
1549
|
+
*b = max(av, bv);
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
// Sorting network for 9 values; returns the 5th smallest (median).
|
|
1553
|
+
//
|
|
1554
|
+
// Notes:
|
|
1555
|
+
// - This is fixed-cost and data-independent.
|
|
1556
|
+
// - For our HPSS approximation we only need a robust center value, and exact median-of-9
|
|
1557
|
+
// is a good tradeoff vs kernel size.
|
|
1558
|
+
fn median9(v0: f32, v1: f32, v2: f32, v3: f32, v4: f32, v5: f32, v6: f32, v7: f32, v8: f32) -> f32 {
|
|
1559
|
+
var a0 = v0; var a1 = v1; var a2 = v2;
|
|
1560
|
+
var a3 = v3; var a4 = v4; var a5 = v5;
|
|
1561
|
+
var a6 = v6; var a7 = v7; var a8 = v8;
|
|
1562
|
+
|
|
1563
|
+
// 9-input sorting network (compare\u2013swap stages). This is a known minimal-ish network.
|
|
1564
|
+
// We fully sort then take middle; cost is acceptable for 9.
|
|
1565
|
+
// Stage 1
|
|
1566
|
+
swap_if_greater(&a0,&a1); swap_if_greater(&a3,&a4); swap_if_greater(&a6,&a7);
|
|
1567
|
+
// Stage 2
|
|
1568
|
+
swap_if_greater(&a1,&a2); swap_if_greater(&a4,&a5); swap_if_greater(&a7,&a8);
|
|
1569
|
+
// Stage 3
|
|
1570
|
+
swap_if_greater(&a0,&a1); swap_if_greater(&a3,&a4); swap_if_greater(&a6,&a7);
|
|
1571
|
+
// Stage 4
|
|
1572
|
+
swap_if_greater(&a0,&a3); swap_if_greater(&a3,&a6); swap_if_greater(&a0,&a3);
|
|
1573
|
+
// Stage 5
|
|
1574
|
+
swap_if_greater(&a1,&a4); swap_if_greater(&a4,&a7); swap_if_greater(&a1,&a4);
|
|
1575
|
+
// Stage 6
|
|
1576
|
+
swap_if_greater(&a2,&a5); swap_if_greater(&a5,&a8); swap_if_greater(&a2,&a5);
|
|
1577
|
+
// Stage 7
|
|
1578
|
+
swap_if_greater(&a1,&a3); swap_if_greater(&a5,&a7);
|
|
1579
|
+
// Stage 8
|
|
1580
|
+
swap_if_greater(&a2,&a6);
|
|
1581
|
+
// Stage 9
|
|
1582
|
+
swap_if_greater(&a2,&a3); swap_if_greater(&a4,&a6);
|
|
1583
|
+
// Stage 10
|
|
1584
|
+
swap_if_greater(&a2,&a4); swap_if_greater(&a4,&a6);
|
|
1585
|
+
// Stage 11
|
|
1586
|
+
swap_if_greater(&a3,&a5); swap_if_greater(&a5,&a7);
|
|
1587
|
+
// Stage 12
|
|
1588
|
+
swap_if_greater(&a3,&a4); swap_if_greater(&a5,&a6);
|
|
1589
|
+
// Stage 13
|
|
1590
|
+
swap_if_greater(&a4,&a5);
|
|
1591
|
+
|
|
1592
|
+
return a4;
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
fn mag_at(frame: i32, bin: i32) -> f32 {
|
|
1596
|
+
let f = clamp_i32(frame, 0, i32(params.nFrames) - 1);
|
|
1597
|
+
let b = clamp_i32(bin, 0, i32(params.nBins) - 1);
|
|
1598
|
+
let idx = u32(f) * params.nBins + u32(b);
|
|
1599
|
+
return mags[idx];
|
|
1600
|
+
}
|
|
1601
|
+
|
|
1602
|
+
@compute @workgroup_size(16, 16)
|
|
1603
|
+
fn main(@builtin(global_invocation_id) gid : vec3<u32>) {
|
|
1604
|
+
let frame = gid.x;
|
|
1605
|
+
let bin = gid.y;
|
|
1606
|
+
|
|
1607
|
+
if (frame >= params.nFrames || bin >= params.nBins) {
|
|
1608
|
+
return;
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
let f = i32(frame);
|
|
1612
|
+
let b = i32(bin);
|
|
1613
|
+
|
|
1614
|
+
// Harmonic estimate: median in time over 9 taps.
|
|
1615
|
+
let h = median9(
|
|
1616
|
+
mag_at(f-4,b), mag_at(f-3,b), mag_at(f-2,b), mag_at(f-1,b), mag_at(f,b),
|
|
1617
|
+
mag_at(f+1,b), mag_at(f+2,b), mag_at(f+3,b), mag_at(f+4,b)
|
|
1618
|
+
);
|
|
1619
|
+
|
|
1620
|
+
// Percussive estimate: median in frequency over 9 taps.
|
|
1621
|
+
let p = median9(
|
|
1622
|
+
mag_at(f,b-4), mag_at(f,b-3), mag_at(f,b-2), mag_at(f,b-1), mag_at(f,b),
|
|
1623
|
+
mag_at(f,b+1), mag_at(f,b+2), mag_at(f,b+3), mag_at(f,b+4)
|
|
1624
|
+
);
|
|
1625
|
+
|
|
1626
|
+
let eps: f32 = 1e-12;
|
|
1627
|
+
let denom = max(eps, h + p);
|
|
1628
|
+
|
|
1629
|
+
var mh = h / denom;
|
|
1630
|
+
var mp = p / denom;
|
|
1631
|
+
|
|
1632
|
+
// Optional hard mask (kept for compatibility with CPU options).
|
|
1633
|
+
if (params.softMask == 0u) {
|
|
1634
|
+
let isH = h >= p;
|
|
1635
|
+
mh = select(0.0, 1.0, isH);
|
|
1636
|
+
mp = select(1.0, 0.0, isH);
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
let idx = frame * params.nBins + bin;
|
|
1640
|
+
harmonicMask[idx] = mh;
|
|
1641
|
+
percussiveMask[idx] = mp;
|
|
1642
|
+
}
|
|
1643
|
+
`
|
|
1644
|
+
);
|
|
1645
|
+
|
|
1646
|
+
// src/gpu/hpssMasks.ts
|
|
1647
|
+
async function gpuHpssMasks(gpu, input) {
|
|
1648
|
+
const { device } = gpu;
|
|
1649
|
+
const { nFrames, nBins, magsFlat, softMask } = input;
|
|
1650
|
+
if (magsFlat.length !== nFrames * nBins) {
|
|
1651
|
+
throw new Error("@octoseq/mir: magsFlat length mismatch");
|
|
1652
|
+
}
|
|
1653
|
+
const magsBuffer = createAndWriteStorageBuffer(gpu, magsFlat);
|
|
1654
|
+
const outByteLen = byteSizeF32(nFrames * nBins);
|
|
1655
|
+
const harmonicOutBuffer = createStorageOutBuffer(gpu, outByteLen);
|
|
1656
|
+
const percussiveOutBuffer = createStorageOutBuffer(gpu, outByteLen);
|
|
1657
|
+
const harmonicReadback = createReadbackBuffer(gpu, outByteLen);
|
|
1658
|
+
const percussiveReadback = createReadbackBuffer(gpu, outByteLen);
|
|
1659
|
+
const shader = device.createShaderModule({ code: hpssMasksWGSL });
|
|
1660
|
+
const pipeline = device.createComputePipeline({
|
|
1661
|
+
layout: "auto",
|
|
1662
|
+
compute: { module: shader, entryPoint: "main" }
|
|
1663
|
+
});
|
|
1664
|
+
const params = createUniformBufferU32x4(gpu, new Uint32Array([nBins, nFrames, softMask ? 1 : 0, 0]));
|
|
1665
|
+
const bindGroup = device.createBindGroup({
|
|
1666
|
+
layout: pipeline.getBindGroupLayout(0),
|
|
1667
|
+
entries: [
|
|
1668
|
+
{ binding: 0, resource: { buffer: magsBuffer } },
|
|
1669
|
+
{ binding: 1, resource: { buffer: harmonicOutBuffer } },
|
|
1670
|
+
{ binding: 2, resource: { buffer: percussiveOutBuffer } },
|
|
1671
|
+
{ binding: 3, resource: { buffer: params } }
|
|
1672
|
+
]
|
|
1673
|
+
});
|
|
1674
|
+
const encoder = device.createCommandEncoder();
|
|
1675
|
+
const pass = encoder.beginComputePass();
|
|
1676
|
+
pass.setPipeline(pipeline);
|
|
1677
|
+
pass.setBindGroup(0, bindGroup);
|
|
1678
|
+
const wgX = Math.ceil(nFrames / 16);
|
|
1679
|
+
const wgY = Math.ceil(nBins / 16);
|
|
1680
|
+
pass.dispatchWorkgroups(wgX, wgY);
|
|
1681
|
+
pass.end();
|
|
1682
|
+
encoder.copyBufferToBuffer(harmonicOutBuffer, 0, harmonicReadback, 0, outByteLen);
|
|
1683
|
+
encoder.copyBufferToBuffer(percussiveOutBuffer, 0, percussiveReadback, 0, outByteLen);
|
|
1684
|
+
const tSubmit = nowMs();
|
|
1685
|
+
gpu.queue.submit([encoder.finish()]);
|
|
1686
|
+
await Promise.all([harmonicReadback.mapAsync(GPUMapMode.READ), percussiveReadback.mapAsync(GPUMapMode.READ)]);
|
|
1687
|
+
const tDone = nowMs();
|
|
1688
|
+
const hBytes = harmonicReadback.getMappedRange().slice(0);
|
|
1689
|
+
const pBytes = percussiveReadback.getMappedRange().slice(0);
|
|
1690
|
+
harmonicReadback.unmap();
|
|
1691
|
+
percussiveReadback.unmap();
|
|
1692
|
+
magsBuffer.destroy();
|
|
1693
|
+
harmonicOutBuffer.destroy();
|
|
1694
|
+
percussiveOutBuffer.destroy();
|
|
1695
|
+
params.destroy();
|
|
1696
|
+
harmonicReadback.destroy();
|
|
1697
|
+
percussiveReadback.destroy();
|
|
1698
|
+
return {
|
|
1699
|
+
value: {
|
|
1700
|
+
harmonicMaskFlat: new Float32Array(hBytes),
|
|
1701
|
+
percussiveMaskFlat: new Float32Array(pBytes)
|
|
1702
|
+
},
|
|
1703
|
+
timing: {
|
|
1704
|
+
gpuSubmitToReadbackMs: tDone - tSubmit
|
|
1705
|
+
}
|
|
1706
|
+
};
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
// src/dsp/hpssGpu.ts
|
|
1710
|
+
function flattenMagnitudes(mags, nFrames, nBins) {
|
|
1711
|
+
const flat = new Float32Array(nFrames * nBins);
|
|
1712
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1713
|
+
const row = mags[t] ?? new Float32Array(nBins);
|
|
1714
|
+
flat.set(row, t * nBins);
|
|
1715
|
+
}
|
|
1716
|
+
return flat;
|
|
1717
|
+
}
|
|
1718
|
+
function assertFiniteMask(name, v) {
|
|
1719
|
+
if (!Number.isFinite(v)) {
|
|
1720
|
+
throw new Error(`@octoseq/mir: GPU HPSS produced non-finite ${name}`);
|
|
1721
|
+
}
|
|
1722
|
+
}
|
|
1723
|
+
async function hpssGpu(spec, gpu, options = {}) {
|
|
1724
|
+
const nFrames = spec.times.length;
|
|
1725
|
+
const nBins = (spec.fftSize >>> 1) + 1;
|
|
1726
|
+
if (options.isCancelled?.()) throw new Error("@octoseq/mir: cancelled");
|
|
1727
|
+
const magsFlat = flattenMagnitudes(spec.magnitudes, nFrames, nBins);
|
|
1728
|
+
const soft = options.softMask ?? true;
|
|
1729
|
+
const masks = await gpuHpssMasks(gpu, {
|
|
1730
|
+
nFrames,
|
|
1731
|
+
nBins,
|
|
1732
|
+
magsFlat,
|
|
1733
|
+
softMask: soft
|
|
1734
|
+
});
|
|
1735
|
+
if (options.isCancelled?.()) throw new Error("@octoseq/mir: cancelled");
|
|
1736
|
+
const hMask = masks.value.harmonicMaskFlat;
|
|
1737
|
+
const pMask = masks.value.percussiveMaskFlat;
|
|
1738
|
+
const harmonic = new Array(nFrames);
|
|
1739
|
+
const percussive = new Array(nFrames);
|
|
1740
|
+
for (let t = 0; t < nFrames; t++) {
|
|
1741
|
+
if (options.isCancelled?.()) throw new Error("@octoseq/mir: cancelled");
|
|
1742
|
+
const mags = spec.magnitudes[t] ?? new Float32Array(nBins);
|
|
1743
|
+
const outH = new Float32Array(nBins);
|
|
1744
|
+
const outP = new Float32Array(nBins);
|
|
1745
|
+
const base = t * nBins;
|
|
1746
|
+
for (let k = 0; k < nBins; k++) {
|
|
1747
|
+
const x = mags[k] ?? 0;
|
|
1748
|
+
const mh = hMask[base + k] ?? 0;
|
|
1749
|
+
const mp = pMask[base + k] ?? 0;
|
|
1750
|
+
assertFiniteMask("mask", mh);
|
|
1751
|
+
assertFiniteMask("mask", mp);
|
|
1752
|
+
const ch = Math.max(0, Math.min(1, mh));
|
|
1753
|
+
const cp = Math.max(0, Math.min(1, mp));
|
|
1754
|
+
outH[k] = x * ch;
|
|
1755
|
+
outP[k] = x * cp;
|
|
1756
|
+
}
|
|
1757
|
+
harmonic[t] = outH;
|
|
1758
|
+
percussive[t] = outP;
|
|
1759
|
+
}
|
|
1760
|
+
return {
|
|
1761
|
+
harmonic: { times: spec.times, bins: nBins, frames: nFrames, magnitudes: harmonic },
|
|
1762
|
+
percussive: { times: spec.times, bins: nBins, frames: nFrames, magnitudes: percussive },
|
|
1763
|
+
gpuMs: masks.timing.gpuSubmitToReadbackMs
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
// src/dsp/fft.ts
|
|
1768
|
+
function hannWindow(size) {
|
|
1769
|
+
const w = new Float32Array(size);
|
|
1770
|
+
for (let n = 0; n < size; n++) {
|
|
1771
|
+
w[n] = 0.5 - 0.5 * Math.cos(2 * Math.PI * n / size);
|
|
1772
|
+
}
|
|
1773
|
+
return w;
|
|
1774
|
+
}
|
|
1775
|
+
function createFftJsBackend(fftSize) {
|
|
1776
|
+
if (!Number.isFinite(fftSize) || fftSize <= 0 || (fftSize | 0) !== fftSize) {
|
|
1777
|
+
throw new Error("@octoseq/mir: fftSize must be a positive integer");
|
|
1778
|
+
}
|
|
1779
|
+
const fft = new FFT(fftSize);
|
|
1780
|
+
const inReal = new Float32Array(fftSize);
|
|
1781
|
+
const outComplexInterleaved = fft.createComplexArray();
|
|
1782
|
+
const outReal = new Float32Array(fftSize);
|
|
1783
|
+
const outImag = new Float32Array(fftSize);
|
|
1784
|
+
return {
|
|
1785
|
+
fftSize,
|
|
1786
|
+
forwardReal(frame) {
|
|
1787
|
+
if (frame.length !== fftSize) {
|
|
1788
|
+
throw new Error(
|
|
1789
|
+
`@octoseq/mir: FFT input length (${frame.length}) must equal fftSize (${fftSize})`
|
|
1790
|
+
);
|
|
1791
|
+
}
|
|
1792
|
+
inReal.set(frame);
|
|
1793
|
+
fft.realTransform(outComplexInterleaved, inReal);
|
|
1794
|
+
fft.completeSpectrum(outComplexInterleaved);
|
|
1795
|
+
for (let k = 0; k < fftSize; k++) {
|
|
1796
|
+
const re = outComplexInterleaved[2 * k] ?? 0;
|
|
1797
|
+
const im = outComplexInterleaved[2 * k + 1] ?? 0;
|
|
1798
|
+
outReal[k] = re === 0 ? 0 : re;
|
|
1799
|
+
outImag[k] = im === 0 ? 0 : im;
|
|
1800
|
+
}
|
|
1801
|
+
return { real: outReal, imag: outImag };
|
|
1802
|
+
}
|
|
1803
|
+
};
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1806
|
+
// src/dsp/fftBackend.ts
|
|
1807
|
+
var backendCache = /* @__PURE__ */ new Map();
|
|
1808
|
+
function getFftBackend(fftSize) {
|
|
1809
|
+
const existing = backendCache.get(fftSize);
|
|
1810
|
+
if (existing) return existing;
|
|
1811
|
+
const created = createFftJsBackend(fftSize);
|
|
1812
|
+
backendCache.set(fftSize, created);
|
|
1813
|
+
return created;
|
|
1814
|
+
}
|
|
1815
|
+
|
|
1816
|
+
// src/dsp/spectrogram.ts
|
|
1817
|
+
function assertPositiveInt3(name, value) {
|
|
1818
|
+
if (!Number.isFinite(value) || value <= 0 || (value | 0) !== value) {
|
|
1819
|
+
throw new Error(`@octoseq/mir: ${name} must be a positive integer`);
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
function mixToMono(audio) {
|
|
1823
|
+
const nCh = audio.numberOfChannels;
|
|
1824
|
+
if (nCh <= 0) {
|
|
1825
|
+
throw new Error("@octoseq/mir: audio.numberOfChannels must be >= 1");
|
|
1826
|
+
}
|
|
1827
|
+
if (nCh === 1) {
|
|
1828
|
+
return audio.getChannelData(0);
|
|
1829
|
+
}
|
|
1830
|
+
const length = audio.getChannelData(0).length;
|
|
1831
|
+
const out = new Float32Array(length);
|
|
1832
|
+
for (let ch = 0; ch < nCh; ch++) {
|
|
1833
|
+
const data = audio.getChannelData(ch);
|
|
1834
|
+
if (data.length !== length) {
|
|
1835
|
+
throw new Error(
|
|
1836
|
+
"@octoseq/mir: all channels must have equal length (AudioBuffer-like invariant)"
|
|
1837
|
+
);
|
|
1838
|
+
}
|
|
1839
|
+
for (let i = 0; i < length; i++) {
|
|
1840
|
+
out[i] = (out[i] ?? 0) + (data[i] ?? 0);
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
const inv = 1 / nCh;
|
|
1844
|
+
for (let i = 0; i < length; i++) out[i] = (out[i] ?? 0) * inv;
|
|
1845
|
+
return out;
|
|
1846
|
+
}
|
|
1847
|
+
async function spectrogram(audio, config, gpu, options = {}) {
|
|
1848
|
+
assertPositiveInt3("config.fftSize", config.fftSize);
|
|
1849
|
+
assertPositiveInt3("config.hopSize", config.hopSize);
|
|
1850
|
+
if (config.window !== "hann") {
|
|
1851
|
+
throw new Error(
|
|
1852
|
+
`@octoseq/mir: unsupported window '${config.window}'. v0.1 supports only 'hann'.`
|
|
1853
|
+
);
|
|
1854
|
+
}
|
|
1855
|
+
const fftSize = config.fftSize;
|
|
1856
|
+
if ((fftSize & fftSize - 1) !== 0) {
|
|
1857
|
+
throw new Error("@octoseq/mir: config.fftSize must be a power of two");
|
|
1858
|
+
}
|
|
1859
|
+
const hopSize = config.hopSize;
|
|
1860
|
+
if (hopSize > fftSize) {
|
|
1861
|
+
throw new Error(
|
|
1862
|
+
"@octoseq/mir: config.hopSize must be <= config.fftSize"
|
|
1863
|
+
);
|
|
1864
|
+
}
|
|
1865
|
+
const sr = audio.sampleRate;
|
|
1866
|
+
const mono = mixToMono(audio);
|
|
1867
|
+
const nFrames = Math.max(0, 1 + Math.floor((mono.length - fftSize) / hopSize));
|
|
1868
|
+
const times = new Float32Array(nFrames);
|
|
1869
|
+
const mags = new Array(nFrames);
|
|
1870
|
+
const window = hannWindow(fftSize);
|
|
1871
|
+
const fft = getFftBackend(fftSize);
|
|
1872
|
+
const windowedFrame = new Float32Array(fftSize);
|
|
1873
|
+
let totalFftMs = 0;
|
|
1874
|
+
const nowMs3 = () => typeof performance !== "undefined" ? performance.now() : Date.now();
|
|
1875
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
1876
|
+
if (options.isCancelled?.()) {
|
|
1877
|
+
throw new Error("@octoseq/mir: cancelled");
|
|
1878
|
+
}
|
|
1879
|
+
const start = frame * hopSize;
|
|
1880
|
+
times[frame] = (start + fftSize / 2) / sr;
|
|
1881
|
+
for (let i = 0; i < fftSize; i++) {
|
|
1882
|
+
const s = mono[start + i] ?? 0;
|
|
1883
|
+
windowedFrame[i] = s * (window[i] ?? 0);
|
|
1884
|
+
}
|
|
1885
|
+
const t0 = nowMs3();
|
|
1886
|
+
const { real, imag } = fft.forwardReal(windowedFrame);
|
|
1887
|
+
totalFftMs += nowMs3() - t0;
|
|
1888
|
+
const nBins = (fftSize >>> 1) + 1;
|
|
1889
|
+
const out = new Float32Array(nBins);
|
|
1890
|
+
for (let k = 0; k < nBins; k++) {
|
|
1891
|
+
const re = real[k] ?? 0;
|
|
1892
|
+
const im = imag[k] ?? 0;
|
|
1893
|
+
out[k] = Math.hypot(re, im);
|
|
1894
|
+
}
|
|
1895
|
+
mags[frame] = out;
|
|
1896
|
+
}
|
|
1897
|
+
mags.cpuFftTotalMs = totalFftMs;
|
|
1898
|
+
return {
|
|
1899
|
+
sampleRate: sr,
|
|
1900
|
+
fftSize,
|
|
1901
|
+
hopSize,
|
|
1902
|
+
times,
|
|
1903
|
+
magnitudes: mags
|
|
1904
|
+
};
|
|
1905
|
+
}
|
|
1906
|
+
|
|
1907
|
+
// src/dsp/cqt.ts
|
|
1908
|
+
var CQT_DEFAULTS = {
|
|
1909
|
+
/** Quarter-tone resolution (24 bins per octave) */
|
|
1910
|
+
binsPerOctave: 24,
|
|
1911
|
+
/** C1 (lowest note on a standard piano) */
|
|
1912
|
+
fMin: 32.7,
|
|
1913
|
+
/** C9 (well above audible range for most content) */
|
|
1914
|
+
fMax: 8372
|
|
1915
|
+
};
|
|
1916
|
+
function cqtBinToHz(bin, config) {
|
|
1917
|
+
return config.fMin * Math.pow(2, bin / config.binsPerOctave);
|
|
1918
|
+
}
|
|
1919
|
+
function hzToCqtBin(hz, config) {
|
|
1920
|
+
if (hz <= 0) return -Infinity;
|
|
1921
|
+
return config.binsPerOctave * Math.log2(hz / config.fMin);
|
|
1922
|
+
}
|
|
1923
|
+
function getNumOctaves(config) {
|
|
1924
|
+
return Math.log2(config.fMax / config.fMin);
|
|
1925
|
+
}
|
|
1926
|
+
function getNumBins(config) {
|
|
1927
|
+
const nOctaves = getNumOctaves(config);
|
|
1928
|
+
return Math.ceil(nOctaves * config.binsPerOctave);
|
|
1929
|
+
}
|
|
1930
|
+
function getCqtBinFrequencies(config) {
|
|
1931
|
+
const nBins = getNumBins(config);
|
|
1932
|
+
const freqs = new Float32Array(nBins);
|
|
1933
|
+
for (let k = 0; k < nBins; k++) {
|
|
1934
|
+
freqs[k] = cqtBinToHz(k, config);
|
|
1935
|
+
}
|
|
1936
|
+
return freqs;
|
|
1937
|
+
}
|
|
1938
|
+
var kernelBankCache = /* @__PURE__ */ new Map();
|
|
1939
|
+
function kernelCacheKey(config, fftSize, sampleRate) {
|
|
1940
|
+
return `${config.binsPerOctave}:${config.fMin}:${config.fMax}:${fftSize}:${sampleRate}`;
|
|
1941
|
+
}
|
|
1942
|
+
function createCqtKernel(binIndex, config, fftSize, sampleRate) {
|
|
1943
|
+
const centerFreq = cqtBinToHz(binIndex, config);
|
|
1944
|
+
const freqResolution = sampleRate / fftSize;
|
|
1945
|
+
const Q = 1 / (Math.pow(2, 1 / config.binsPerOctave) - 1);
|
|
1946
|
+
const bandwidth = centerFreq / Q;
|
|
1947
|
+
const fLow = centerFreq - bandwidth / 2;
|
|
1948
|
+
const fHigh = centerFreq + bandwidth / 2;
|
|
1949
|
+
const startBin = Math.max(0, Math.floor(fLow / freqResolution));
|
|
1950
|
+
const endBin = Math.min(
|
|
1951
|
+
Math.floor(fftSize / 2) + 1,
|
|
1952
|
+
Math.ceil(fHigh / freqResolution) + 1
|
|
1953
|
+
);
|
|
1954
|
+
const numBins = Math.max(1, endBin - startBin);
|
|
1955
|
+
const weights = new Float32Array(numBins);
|
|
1956
|
+
for (let i = 0; i < numBins; i++) {
|
|
1957
|
+
const binFreq = (startBin + i) * freqResolution;
|
|
1958
|
+
if (binFreq <= centerFreq) {
|
|
1959
|
+
if (centerFreq > fLow) {
|
|
1960
|
+
weights[i] = (binFreq - fLow) / (centerFreq - fLow);
|
|
1961
|
+
} else {
|
|
1962
|
+
weights[i] = 1;
|
|
1963
|
+
}
|
|
1964
|
+
} else {
|
|
1965
|
+
if (fHigh > centerFreq) {
|
|
1966
|
+
weights[i] = (fHigh - binFreq) / (fHigh - centerFreq);
|
|
1967
|
+
} else {
|
|
1968
|
+
weights[i] = 1;
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
weights[i] = Math.max(0, Math.min(1, weights[i] ?? 0));
|
|
1972
|
+
}
|
|
1973
|
+
let sum = 0;
|
|
1974
|
+
for (let i = 0; i < numBins; i++) {
|
|
1975
|
+
sum += weights[i] ?? 0;
|
|
1976
|
+
}
|
|
1977
|
+
if (sum > 0) {
|
|
1978
|
+
for (let i = 0; i < numBins; i++) {
|
|
1979
|
+
weights[i] = (weights[i] ?? 0) / sum;
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
return {
|
|
1983
|
+
centerFreq,
|
|
1984
|
+
startBin,
|
|
1985
|
+
endBin,
|
|
1986
|
+
weights
|
|
1987
|
+
};
|
|
1988
|
+
}
|
|
1989
|
+
function getCqtKernelBank(config, fftSize, sampleRate) {
|
|
1990
|
+
const key = kernelCacheKey(config, fftSize, sampleRate);
|
|
1991
|
+
const cached = kernelBankCache.get(key);
|
|
1992
|
+
if (cached) return cached;
|
|
1993
|
+
const nBins = getNumBins(config);
|
|
1994
|
+
const kernels = new Array(nBins);
|
|
1995
|
+
for (let k = 0; k < nBins; k++) {
|
|
1996
|
+
kernels[k] = createCqtKernel(k, config, fftSize, sampleRate);
|
|
1997
|
+
}
|
|
1998
|
+
const bank = {
|
|
1999
|
+
config,
|
|
2000
|
+
fftSize,
|
|
2001
|
+
sampleRate,
|
|
2002
|
+
kernels
|
|
2003
|
+
};
|
|
2004
|
+
kernelBankCache.set(key, bank);
|
|
2005
|
+
return bank;
|
|
2006
|
+
}
|
|
2007
|
+
function applyCqtKernels(stftMagnitudes, kernelBank) {
|
|
2008
|
+
const nCqtBins = kernelBank.kernels.length;
|
|
2009
|
+
const cqtMagnitudes = new Float32Array(nCqtBins);
|
|
2010
|
+
for (let k = 0; k < nCqtBins; k++) {
|
|
2011
|
+
const kernel = kernelBank.kernels[k];
|
|
2012
|
+
if (!kernel) continue;
|
|
2013
|
+
let sum = 0;
|
|
2014
|
+
for (let i = 0; i < kernel.weights.length; i++) {
|
|
2015
|
+
const stftBin = kernel.startBin + i;
|
|
2016
|
+
const stftMag = stftMagnitudes[stftBin] ?? 0;
|
|
2017
|
+
const weight = kernel.weights[i] ?? 0;
|
|
2018
|
+
sum += stftMag * weight;
|
|
2019
|
+
}
|
|
2020
|
+
cqtMagnitudes[k] = sum;
|
|
2021
|
+
}
|
|
2022
|
+
return cqtMagnitudes;
|
|
2023
|
+
}
|
|
2024
|
+
function withCqtDefaults(partial) {
|
|
2025
|
+
return {
|
|
2026
|
+
binsPerOctave: partial?.binsPerOctave ?? CQT_DEFAULTS.binsPerOctave,
|
|
2027
|
+
fMin: partial?.fMin ?? CQT_DEFAULTS.fMin,
|
|
2028
|
+
fMax: partial?.fMax ?? CQT_DEFAULTS.fMax,
|
|
2029
|
+
hopSize: partial?.hopSize
|
|
2030
|
+
};
|
|
2031
|
+
}
|
|
2032
|
+
async function cqtSpectrogram(audio, config, options = {}) {
|
|
2033
|
+
const sampleRate = audio.sampleRate;
|
|
2034
|
+
if (config.fMin <= 0) {
|
|
2035
|
+
throw new Error("@octoseq/mir: CQT fMin must be positive");
|
|
2036
|
+
}
|
|
2037
|
+
if (config.fMax <= config.fMin) {
|
|
2038
|
+
throw new Error("@octoseq/mir: CQT fMax must be greater than fMin");
|
|
2039
|
+
}
|
|
2040
|
+
if (config.binsPerOctave <= 0) {
|
|
2041
|
+
throw new Error("@octoseq/mir: CQT binsPerOctave must be positive");
|
|
2042
|
+
}
|
|
2043
|
+
const Q = 1 / (Math.pow(2, 1 / config.binsPerOctave) - 1);
|
|
2044
|
+
const minFreqResolution = config.fMin / Q / 2;
|
|
2045
|
+
const minFftSize = Math.ceil(sampleRate / minFreqResolution);
|
|
2046
|
+
let fftSize = 1;
|
|
2047
|
+
while (fftSize < minFftSize) {
|
|
2048
|
+
fftSize *= 2;
|
|
2049
|
+
}
|
|
2050
|
+
fftSize = Math.min(fftSize, 16384);
|
|
2051
|
+
const hopSize = config.hopSize ?? Math.floor(fftSize / 4);
|
|
2052
|
+
const stft = await spectrogram(
|
|
2053
|
+
audio,
|
|
2054
|
+
{ fftSize, hopSize, window: "hann" },
|
|
2055
|
+
void 0,
|
|
2056
|
+
{ isCancelled: options.isCancelled }
|
|
2057
|
+
);
|
|
2058
|
+
const kernelBank = getCqtKernelBank(config, fftSize, sampleRate);
|
|
2059
|
+
const nFrames = stft.magnitudes.length;
|
|
2060
|
+
const cqtMagnitudes = new Array(nFrames);
|
|
2061
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2062
|
+
if (options.isCancelled?.()) {
|
|
2063
|
+
throw new Error("@octoseq/mir: cancelled");
|
|
2064
|
+
}
|
|
2065
|
+
const stftFrame = stft.magnitudes[frame];
|
|
2066
|
+
if (!stftFrame) continue;
|
|
2067
|
+
cqtMagnitudes[frame] = applyCqtKernels(stftFrame, kernelBank);
|
|
2068
|
+
}
|
|
2069
|
+
const nOctaves = getNumOctaves(config);
|
|
2070
|
+
getNumBins(config);
|
|
2071
|
+
return {
|
|
2072
|
+
sampleRate,
|
|
2073
|
+
config,
|
|
2074
|
+
times: stft.times,
|
|
2075
|
+
magnitudes: cqtMagnitudes,
|
|
2076
|
+
nOctaves,
|
|
2077
|
+
binsPerOctave: config.binsPerOctave,
|
|
2078
|
+
binFrequencies: getCqtBinFrequencies(config)
|
|
2079
|
+
};
|
|
2080
|
+
}
|
|
2081
|
+
async function computeCqt(audio, config, options = {}) {
|
|
2082
|
+
const startTime = performance.now();
|
|
2083
|
+
const fullConfig = withCqtDefaults(config);
|
|
2084
|
+
const cqt = await cqtSpectrogram(audio, fullConfig, options);
|
|
2085
|
+
const endTime = performance.now();
|
|
2086
|
+
return {
|
|
2087
|
+
cqt,
|
|
2088
|
+
meta: {
|
|
2089
|
+
backend: "cpu",
|
|
2090
|
+
usedGpu: false,
|
|
2091
|
+
timings: {
|
|
2092
|
+
totalMs: endTime - startTime,
|
|
2093
|
+
cpuMs: endTime - startTime
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
};
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
// src/dsp/cqtSignals.ts
|
|
2100
|
+
var BASS_MIN_HZ = 20;
|
|
2101
|
+
var BASS_MAX_HZ = 300;
|
|
2102
|
+
var TONAL_STABILITY_WINDOW_FRAMES = 20;
|
|
2103
|
+
var CHROMA_BINS = 12;
|
|
2104
|
+
function normalizeMinMax(values) {
|
|
2105
|
+
let min = Infinity;
|
|
2106
|
+
let max = -Infinity;
|
|
2107
|
+
for (let i = 0; i < values.length; i++) {
|
|
2108
|
+
const v = values[i] ?? 0;
|
|
2109
|
+
if (v < min) min = v;
|
|
2110
|
+
if (v > max) max = v;
|
|
2111
|
+
}
|
|
2112
|
+
const range = max - min;
|
|
2113
|
+
const result = new Float32Array(values.length);
|
|
2114
|
+
if (range > 0) {
|
|
2115
|
+
for (let i = 0; i < values.length; i++) {
|
|
2116
|
+
result[i] = ((values[i] ?? 0) - min) / range;
|
|
2117
|
+
}
|
|
2118
|
+
} else {
|
|
2119
|
+
result.fill(0.5);
|
|
2120
|
+
}
|
|
2121
|
+
return result;
|
|
2122
|
+
}
|
|
2123
|
+
function weightedCentroid(values, startIndex = 0) {
|
|
2124
|
+
let sumWeighted = 0;
|
|
2125
|
+
let sumWeights = 0;
|
|
2126
|
+
for (let i = 0; i < values.length; i++) {
|
|
2127
|
+
const weight = values[i] ?? 0;
|
|
2128
|
+
sumWeighted += (startIndex + i) * weight;
|
|
2129
|
+
sumWeights += weight;
|
|
2130
|
+
}
|
|
2131
|
+
return sumWeights > 0 ? sumWeighted / sumWeights : startIndex + values.length / 2;
|
|
2132
|
+
}
|
|
2133
|
+
function computeHarmonicEnergyFrame(frame, cqt) {
|
|
2134
|
+
if (frame.length === 0) return 0;
|
|
2135
|
+
let totalEnergy = 0;
|
|
2136
|
+
for (let i = 0; i < frame.length; i++) {
|
|
2137
|
+
const mag = frame[i] ?? 0;
|
|
2138
|
+
totalEnergy += mag * mag;
|
|
2139
|
+
}
|
|
2140
|
+
if (totalEnergy === 0) return 0;
|
|
2141
|
+
let maxMag = 0;
|
|
2142
|
+
let fundamentalBin = 0;
|
|
2143
|
+
for (let i = 0; i < frame.length; i++) {
|
|
2144
|
+
const mag = frame[i] ?? 0;
|
|
2145
|
+
if (mag > maxMag) {
|
|
2146
|
+
maxMag = mag;
|
|
2147
|
+
fundamentalBin = i;
|
|
2148
|
+
}
|
|
2149
|
+
}
|
|
2150
|
+
const fundamentalFreq = cqtBinToHz(fundamentalBin, cqt.config);
|
|
2151
|
+
let harmonicEnergy2 = 0;
|
|
2152
|
+
const numHarmonics = 6;
|
|
2153
|
+
for (let h = 1; h <= numHarmonics; h++) {
|
|
2154
|
+
const harmonicFreq = fundamentalFreq * h;
|
|
2155
|
+
const harmonicBin = Math.round(hzToCqtBin(harmonicFreq, cqt.config));
|
|
2156
|
+
if (harmonicBin >= 0 && harmonicBin < frame.length) {
|
|
2157
|
+
const mag = frame[harmonicBin] ?? 0;
|
|
2158
|
+
const weight = 1 / h;
|
|
2159
|
+
harmonicEnergy2 += mag * mag * weight;
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
let weightSum = 0;
|
|
2163
|
+
for (let h = 1; h <= numHarmonics; h++) {
|
|
2164
|
+
weightSum += 1 / h;
|
|
2165
|
+
}
|
|
2166
|
+
harmonicEnergy2 /= weightSum;
|
|
2167
|
+
return Math.min(1, harmonicEnergy2 / totalEnergy);
|
|
2168
|
+
}
|
|
2169
|
+
function harmonicEnergy(cqt) {
|
|
2170
|
+
const startTime = performance.now();
|
|
2171
|
+
const nFrames = cqt.magnitudes.length;
|
|
2172
|
+
const values = new Float32Array(nFrames);
|
|
2173
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2174
|
+
const cqtFrame = cqt.magnitudes[frame];
|
|
2175
|
+
if (cqtFrame) {
|
|
2176
|
+
values[frame] = computeHarmonicEnergyFrame(cqtFrame, cqt);
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
const normalized = normalizeMinMax(values);
|
|
2180
|
+
const endTime = performance.now();
|
|
2181
|
+
return {
|
|
2182
|
+
kind: "cqt1d",
|
|
2183
|
+
signalId: "harmonicEnergy",
|
|
2184
|
+
times: cqt.times,
|
|
2185
|
+
values: normalized,
|
|
2186
|
+
meta: {
|
|
2187
|
+
backend: "cpu",
|
|
2188
|
+
usedGpu: false,
|
|
2189
|
+
timings: {
|
|
2190
|
+
totalMs: endTime - startTime,
|
|
2191
|
+
cpuMs: endTime - startTime
|
|
2192
|
+
}
|
|
2193
|
+
}
|
|
2194
|
+
};
|
|
2195
|
+
}
|
|
2196
|
+
function bassPitchMotion(cqt) {
|
|
2197
|
+
const startTime = performance.now();
|
|
2198
|
+
const nFrames = cqt.magnitudes.length;
|
|
2199
|
+
const bassStartBin = Math.max(0, Math.floor(hzToCqtBin(BASS_MIN_HZ, cqt.config)));
|
|
2200
|
+
const bassEndBin = Math.min(
|
|
2201
|
+
cqt.magnitudes[0]?.length ?? 0,
|
|
2202
|
+
Math.ceil(hzToCqtBin(BASS_MAX_HZ, cqt.config))
|
|
2203
|
+
);
|
|
2204
|
+
const bassNumBins = bassEndBin - bassStartBin;
|
|
2205
|
+
if (bassNumBins <= 0) {
|
|
2206
|
+
return {
|
|
2207
|
+
kind: "cqt1d",
|
|
2208
|
+
signalId: "bassPitchMotion",
|
|
2209
|
+
times: cqt.times,
|
|
2210
|
+
values: new Float32Array(nFrames),
|
|
2211
|
+
meta: {
|
|
2212
|
+
backend: "cpu",
|
|
2213
|
+
usedGpu: false,
|
|
2214
|
+
timings: { totalMs: 0, cpuMs: 0 }
|
|
2215
|
+
}
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
const centroids = new Float32Array(nFrames);
|
|
2219
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2220
|
+
const cqtFrame = cqt.magnitudes[frame];
|
|
2221
|
+
if (!cqtFrame) continue;
|
|
2222
|
+
const bassBins = new Float32Array(bassNumBins);
|
|
2223
|
+
for (let i = 0; i < bassNumBins; i++) {
|
|
2224
|
+
bassBins[i] = cqtFrame[bassStartBin + i] ?? 0;
|
|
2225
|
+
}
|
|
2226
|
+
centroids[frame] = weightedCentroid(bassBins, bassStartBin);
|
|
2227
|
+
}
|
|
2228
|
+
const motion = new Float32Array(nFrames);
|
|
2229
|
+
for (let frame = 1; frame < nFrames; frame++) {
|
|
2230
|
+
motion[frame] = Math.abs((centroids[frame] ?? 0) - (centroids[frame - 1] ?? 0));
|
|
2231
|
+
}
|
|
2232
|
+
motion[0] = motion[1] ?? 0;
|
|
2233
|
+
const normalized = normalizeMinMax(motion);
|
|
2234
|
+
const endTime = performance.now();
|
|
2235
|
+
return {
|
|
2236
|
+
kind: "cqt1d",
|
|
2237
|
+
signalId: "bassPitchMotion",
|
|
2238
|
+
times: cqt.times,
|
|
2239
|
+
values: normalized,
|
|
2240
|
+
meta: {
|
|
2241
|
+
backend: "cpu",
|
|
2242
|
+
usedGpu: false,
|
|
2243
|
+
timings: {
|
|
2244
|
+
totalMs: endTime - startTime,
|
|
2245
|
+
cpuMs: endTime - startTime
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
};
|
|
2249
|
+
}
|
|
2250
|
+
function computeChroma(frame, binsPerOctave) {
|
|
2251
|
+
const chroma = new Float32Array(CHROMA_BINS);
|
|
2252
|
+
const binsPerSemitone = binsPerOctave / CHROMA_BINS;
|
|
2253
|
+
for (let i = 0; i < frame.length; i++) {
|
|
2254
|
+
const chromaBin = Math.floor(i % binsPerOctave / binsPerSemitone) % CHROMA_BINS;
|
|
2255
|
+
const mag = frame[i] ?? 0;
|
|
2256
|
+
chroma[chromaBin] = (chroma[chromaBin] ?? 0) + mag * mag;
|
|
2257
|
+
}
|
|
2258
|
+
let sum = 0;
|
|
2259
|
+
for (let i = 0; i < CHROMA_BINS; i++) {
|
|
2260
|
+
sum += chroma[i] ?? 0;
|
|
2261
|
+
}
|
|
2262
|
+
if (sum > 0) {
|
|
2263
|
+
for (let i = 0; i < CHROMA_BINS; i++) {
|
|
2264
|
+
chroma[i] = (chroma[i] ?? 0) / sum;
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
return chroma;
|
|
2268
|
+
}
|
|
2269
|
+
function tonalStability(cqt) {
|
|
2270
|
+
const startTime = performance.now();
|
|
2271
|
+
const nFrames = cqt.magnitudes.length;
|
|
2272
|
+
const chromas = new Array(nFrames);
|
|
2273
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2274
|
+
const cqtFrame = cqt.magnitudes[frame];
|
|
2275
|
+
if (cqtFrame) {
|
|
2276
|
+
chromas[frame] = computeChroma(cqtFrame, cqt.binsPerOctave);
|
|
2277
|
+
} else {
|
|
2278
|
+
chromas[frame] = new Float32Array(CHROMA_BINS);
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
const halfWindow = Math.floor(TONAL_STABILITY_WINDOW_FRAMES / 2);
|
|
2282
|
+
const instability = new Float32Array(nFrames);
|
|
2283
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2284
|
+
const windowStart = Math.max(0, frame - halfWindow);
|
|
2285
|
+
const windowEnd = Math.min(nFrames, frame + halfWindow + 1);
|
|
2286
|
+
const windowSize = windowEnd - windowStart;
|
|
2287
|
+
const avgChroma = new Float32Array(CHROMA_BINS);
|
|
2288
|
+
for (let w = windowStart; w < windowEnd; w++) {
|
|
2289
|
+
const chroma = chromas[w];
|
|
2290
|
+
if (chroma) {
|
|
2291
|
+
for (let c = 0; c < CHROMA_BINS; c++) {
|
|
2292
|
+
avgChroma[c] = (avgChroma[c] ?? 0) + (chroma[c] ?? 0);
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
}
|
|
2296
|
+
for (let c = 0; c < CHROMA_BINS; c++) {
|
|
2297
|
+
avgChroma[c] = (avgChroma[c] ?? 0) / windowSize;
|
|
2298
|
+
}
|
|
2299
|
+
let totalVariance = 0;
|
|
2300
|
+
for (let w = windowStart; w < windowEnd; w++) {
|
|
2301
|
+
const chroma = chromas[w];
|
|
2302
|
+
if (chroma) {
|
|
2303
|
+
for (let c = 0; c < CHROMA_BINS; c++) {
|
|
2304
|
+
const diff = (chroma[c] ?? 0) - (avgChroma[c] ?? 0);
|
|
2305
|
+
totalVariance += diff * diff;
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
totalVariance /= windowSize * CHROMA_BINS;
|
|
2310
|
+
instability[frame] = totalVariance;
|
|
2311
|
+
}
|
|
2312
|
+
const normalizedInstability = normalizeMinMax(instability);
|
|
2313
|
+
const stability = new Float32Array(nFrames);
|
|
2314
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2315
|
+
stability[frame] = 1 - (normalizedInstability[frame] ?? 0);
|
|
2316
|
+
}
|
|
2317
|
+
const endTime = performance.now();
|
|
2318
|
+
return {
|
|
2319
|
+
kind: "cqt1d",
|
|
2320
|
+
signalId: "tonalStability",
|
|
2321
|
+
times: cqt.times,
|
|
2322
|
+
values: stability,
|
|
2323
|
+
meta: {
|
|
2324
|
+
backend: "cpu",
|
|
2325
|
+
usedGpu: false,
|
|
2326
|
+
timings: {
|
|
2327
|
+
totalMs: endTime - startTime,
|
|
2328
|
+
cpuMs: endTime - startTime
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
};
|
|
2332
|
+
}
|
|
2333
|
+
function computeCqtSignal(cqt, signalId) {
|
|
2334
|
+
switch (signalId) {
|
|
2335
|
+
case "harmonicEnergy":
|
|
2336
|
+
return harmonicEnergy(cqt);
|
|
2337
|
+
case "bassPitchMotion":
|
|
2338
|
+
return bassPitchMotion(cqt);
|
|
2339
|
+
case "tonalStability":
|
|
2340
|
+
return tonalStability(cqt);
|
|
2341
|
+
default:
|
|
2342
|
+
throw new Error(`@octoseq/mir: unknown CQT signal ID: ${signalId}`);
|
|
2343
|
+
}
|
|
2344
|
+
}
|
|
2345
|
+
function computeAllCqtSignals(cqt) {
|
|
2346
|
+
const results = /* @__PURE__ */ new Map();
|
|
2347
|
+
results.set("harmonicEnergy", harmonicEnergy(cqt));
|
|
2348
|
+
results.set("bassPitchMotion", bassPitchMotion(cqt));
|
|
2349
|
+
results.set("tonalStability", tonalStability(cqt));
|
|
2350
|
+
return results;
|
|
2351
|
+
}
|
|
2352
|
+
|
|
2353
|
+
// src/dsp/activity.ts
|
|
2354
|
+
var DEFAULT_ACTIVITY_CONFIG = {
|
|
2355
|
+
energyPercentile: 10,
|
|
2356
|
+
enterMargin: 6,
|
|
2357
|
+
exitMargin: 3,
|
|
2358
|
+
hangoverMs: 50,
|
|
2359
|
+
minActiveMs: 30,
|
|
2360
|
+
smoothMs: 20
|
|
2361
|
+
};
|
|
2362
|
+
function withActivityDefaults(config) {
|
|
2363
|
+
return {
|
|
2364
|
+
energyPercentile: config?.energyPercentile ?? DEFAULT_ACTIVITY_CONFIG.energyPercentile,
|
|
2365
|
+
enterMargin: config?.enterMargin ?? DEFAULT_ACTIVITY_CONFIG.enterMargin,
|
|
2366
|
+
exitMargin: config?.exitMargin ?? DEFAULT_ACTIVITY_CONFIG.exitMargin,
|
|
2367
|
+
hangoverMs: config?.hangoverMs ?? DEFAULT_ACTIVITY_CONFIG.hangoverMs,
|
|
2368
|
+
minActiveMs: config?.minActiveMs ?? DEFAULT_ACTIVITY_CONFIG.minActiveMs,
|
|
2369
|
+
smoothMs: config?.smoothMs ?? DEFAULT_ACTIVITY_CONFIG.smoothMs
|
|
2370
|
+
};
|
|
2371
|
+
}
|
|
2372
|
+
function smoothSignal(values, windowFrames) {
|
|
2373
|
+
if (windowFrames <= 1) return values;
|
|
2374
|
+
const n = values.length;
|
|
2375
|
+
const out = new Float32Array(n);
|
|
2376
|
+
const half = Math.floor(windowFrames / 2);
|
|
2377
|
+
const prefix = new Float64Array(n + 1);
|
|
2378
|
+
prefix[0] = 0;
|
|
2379
|
+
for (let i = 0; i < n; i++) {
|
|
2380
|
+
prefix[i + 1] = (prefix[i] ?? 0) + (values[i] ?? 0);
|
|
2381
|
+
}
|
|
2382
|
+
for (let i = 0; i < n; i++) {
|
|
2383
|
+
const start = Math.max(0, i - half);
|
|
2384
|
+
const end = Math.min(n, i + half + 1);
|
|
2385
|
+
const sum = (prefix[end] ?? 0) - (prefix[start] ?? 0);
|
|
2386
|
+
const count = Math.max(1, end - start);
|
|
2387
|
+
out[i] = sum / count;
|
|
2388
|
+
}
|
|
2389
|
+
return out;
|
|
2390
|
+
}
|
|
2391
|
+
function maskToActivityLevel(mask, smoothFrames) {
|
|
2392
|
+
const n = mask.length;
|
|
2393
|
+
const level = new Float32Array(n);
|
|
2394
|
+
for (let i = 0; i < n; i++) {
|
|
2395
|
+
level[i] = mask[i] ?? 0;
|
|
2396
|
+
}
|
|
2397
|
+
if (smoothFrames > 1) {
|
|
2398
|
+
return smoothSignal(level, smoothFrames);
|
|
2399
|
+
}
|
|
2400
|
+
return level;
|
|
2401
|
+
}
|
|
2402
|
+
function computeActivityFromMel(mel, config) {
|
|
2403
|
+
const cfg = withActivityDefaults(config);
|
|
2404
|
+
const nFrames = mel.times.length;
|
|
2405
|
+
const frameEnergy = computeFrameEnergyFromMel(mel.melBands);
|
|
2406
|
+
const noiseFloor = estimateNoiseFloor(frameEnergy, cfg.energyPercentile);
|
|
2407
|
+
const enterThreshold = noiseFloor + cfg.enterMargin;
|
|
2408
|
+
const exitThreshold = noiseFloor + cfg.exitMargin;
|
|
2409
|
+
const frameDurationSec = nFrames >= 2 ? (mel.times[1] ?? 0) - (mel.times[0] ?? 0) : 0.01;
|
|
2410
|
+
const hangoverFrames = Math.max(0, Math.round(cfg.hangoverMs / 1e3 / frameDurationSec));
|
|
2411
|
+
const minActiveFrames = Math.max(0, Math.round(cfg.minActiveMs / 1e3 / frameDurationSec));
|
|
2412
|
+
const smoothFrames = Math.max(1, Math.round(cfg.smoothMs / 1e3 / frameDurationSec)) | 1;
|
|
2413
|
+
const isActive = buildActivityMask(frameEnergy, enterThreshold, exitThreshold, hangoverFrames);
|
|
2414
|
+
if (minActiveFrames > 1) {
|
|
2415
|
+
applyMinActiveDuration(isActive, minActiveFrames);
|
|
2416
|
+
}
|
|
2417
|
+
const suppressFrames = Math.max(0, Math.round(50 / 1e3 / frameDurationSec));
|
|
2418
|
+
const suppressMask = buildSuppressionMask(isActive, suppressFrames);
|
|
2419
|
+
const activityLevel = maskToActivityLevel(isActive, smoothFrames);
|
|
2420
|
+
return {
|
|
2421
|
+
times: mel.times,
|
|
2422
|
+
activityLevel,
|
|
2423
|
+
isActive,
|
|
2424
|
+
suppressMask,
|
|
2425
|
+
diagnostics: {
|
|
2426
|
+
frameEnergy,
|
|
2427
|
+
noiseFloor,
|
|
2428
|
+
enterThreshold,
|
|
2429
|
+
exitThreshold
|
|
2430
|
+
}
|
|
2431
|
+
};
|
|
2432
|
+
}
|
|
2433
|
+
function computeActivityFromSpectrogram(spec, config) {
|
|
2434
|
+
const cfg = withActivityDefaults(config);
|
|
2435
|
+
const nFrames = spec.times.length;
|
|
2436
|
+
const linearEnergy = computeFrameEnergyFromSpectrogram(spec.magnitudes, false);
|
|
2437
|
+
const frameEnergy = new Float32Array(nFrames);
|
|
2438
|
+
const eps = 1e-12;
|
|
2439
|
+
for (let t = 0; t < nFrames; t++) {
|
|
2440
|
+
frameEnergy[t] = Math.log10(eps + (linearEnergy[t] ?? 0));
|
|
2441
|
+
}
|
|
2442
|
+
const noiseFloor = estimateNoiseFloor(frameEnergy, cfg.energyPercentile);
|
|
2443
|
+
const enterThreshold = noiseFloor + cfg.enterMargin;
|
|
2444
|
+
const exitThreshold = noiseFloor + cfg.exitMargin;
|
|
2445
|
+
const frameDurationSec = nFrames >= 2 ? (spec.times[1] ?? 0) - (spec.times[0] ?? 0) : 0.01;
|
|
2446
|
+
const hangoverFrames = Math.max(0, Math.round(cfg.hangoverMs / 1e3 / frameDurationSec));
|
|
2447
|
+
const minActiveFrames = Math.max(0, Math.round(cfg.minActiveMs / 1e3 / frameDurationSec));
|
|
2448
|
+
const smoothFrames = Math.max(1, Math.round(cfg.smoothMs / 1e3 / frameDurationSec)) | 1;
|
|
2449
|
+
const isActive = buildActivityMask(frameEnergy, enterThreshold, exitThreshold, hangoverFrames);
|
|
2450
|
+
if (minActiveFrames > 1) {
|
|
2451
|
+
applyMinActiveDuration(isActive, minActiveFrames);
|
|
2452
|
+
}
|
|
2453
|
+
const suppressFrames = Math.max(0, Math.round(50 / 1e3 / frameDurationSec));
|
|
2454
|
+
const suppressMask = buildSuppressionMask(isActive, suppressFrames);
|
|
2455
|
+
const activityLevel = maskToActivityLevel(isActive, smoothFrames);
|
|
2456
|
+
return {
|
|
2457
|
+
times: spec.times,
|
|
2458
|
+
activityLevel,
|
|
2459
|
+
isActive,
|
|
2460
|
+
suppressMask,
|
|
2461
|
+
diagnostics: {
|
|
2462
|
+
frameEnergy,
|
|
2463
|
+
noiseFloor,
|
|
2464
|
+
enterThreshold,
|
|
2465
|
+
exitThreshold
|
|
2466
|
+
}
|
|
2467
|
+
};
|
|
2468
|
+
}
|
|
2469
|
+
function computeActivityFromAudio(samples, sampleRate, hopSize, windowSize, config) {
|
|
2470
|
+
const cfg = withActivityDefaults(config);
|
|
2471
|
+
const nFrames = Math.max(0, Math.floor((samples.length - windowSize) / hopSize) + 1);
|
|
2472
|
+
const times = new Float32Array(nFrames);
|
|
2473
|
+
const frameEnergy = new Float32Array(nFrames);
|
|
2474
|
+
const eps = 1e-12;
|
|
2475
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2476
|
+
const start = frame * hopSize;
|
|
2477
|
+
const end = Math.min(start + windowSize, samples.length);
|
|
2478
|
+
let sumSq = 0;
|
|
2479
|
+
for (let i = start; i < end; i++) {
|
|
2480
|
+
const s = samples[i] ?? 0;
|
|
2481
|
+
sumSq += s * s;
|
|
2482
|
+
}
|
|
2483
|
+
const rms = Math.sqrt(sumSq / (end - start));
|
|
2484
|
+
times[frame] = (start + windowSize / 2) / sampleRate;
|
|
2485
|
+
frameEnergy[frame] = Math.log10(eps + rms);
|
|
2486
|
+
}
|
|
2487
|
+
const noiseFloor = estimateNoiseFloor(frameEnergy, cfg.energyPercentile);
|
|
2488
|
+
const enterThreshold = noiseFloor + cfg.enterMargin;
|
|
2489
|
+
const exitThreshold = noiseFloor + cfg.exitMargin;
|
|
2490
|
+
const frameDurationSec = hopSize / sampleRate;
|
|
2491
|
+
const hangoverFrames = Math.max(0, Math.round(cfg.hangoverMs / 1e3 / frameDurationSec));
|
|
2492
|
+
const minActiveFrames = Math.max(0, Math.round(cfg.minActiveMs / 1e3 / frameDurationSec));
|
|
2493
|
+
const smoothFrames = Math.max(1, Math.round(cfg.smoothMs / 1e3 / frameDurationSec)) | 1;
|
|
2494
|
+
const isActive = buildActivityMask(frameEnergy, enterThreshold, exitThreshold, hangoverFrames);
|
|
2495
|
+
if (minActiveFrames > 1) {
|
|
2496
|
+
applyMinActiveDuration(isActive, minActiveFrames);
|
|
2497
|
+
}
|
|
2498
|
+
const suppressFrames = Math.max(0, Math.round(50 / 1e3 / frameDurationSec));
|
|
2499
|
+
const suppressMask = buildSuppressionMask(isActive, suppressFrames);
|
|
2500
|
+
const activityLevel = maskToActivityLevel(isActive, smoothFrames);
|
|
2501
|
+
return {
|
|
2502
|
+
times,
|
|
2503
|
+
activityLevel,
|
|
2504
|
+
isActive,
|
|
2505
|
+
suppressMask,
|
|
2506
|
+
diagnostics: {
|
|
2507
|
+
frameEnergy,
|
|
2508
|
+
noiseFloor,
|
|
2509
|
+
enterThreshold,
|
|
2510
|
+
exitThreshold
|
|
2511
|
+
}
|
|
2512
|
+
};
|
|
2513
|
+
}
|
|
2514
|
+
function applyActivityGating(values, activity, options) {
|
|
2515
|
+
const useBinary = options?.useBinaryMask ?? true;
|
|
2516
|
+
const suppressPostSilence = options?.suppressPostSilence ?? true;
|
|
2517
|
+
const levelThreshold = options?.levelThreshold ?? 0.5;
|
|
2518
|
+
const n = Math.min(values.length, activity.activityLevel.length);
|
|
2519
|
+
for (let i = 0; i < n; i++) {
|
|
2520
|
+
let shouldGate = false;
|
|
2521
|
+
if (useBinary) {
|
|
2522
|
+
shouldGate = (activity.isActive[i] ?? 0) === 0;
|
|
2523
|
+
} else {
|
|
2524
|
+
shouldGate = (activity.activityLevel[i] ?? 0) < levelThreshold;
|
|
2525
|
+
}
|
|
2526
|
+
if (suppressPostSilence && (activity.suppressMask[i] ?? 0) === 1) {
|
|
2527
|
+
shouldGate = true;
|
|
2528
|
+
}
|
|
2529
|
+
if (shouldGate) {
|
|
2530
|
+
values[i] = 0;
|
|
2531
|
+
}
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
function interpolateActivity(activity, targetTimes) {
|
|
2535
|
+
const n = targetTimes.length;
|
|
2536
|
+
const activityLevel = new Float32Array(n);
|
|
2537
|
+
const isActive = new Uint8Array(n);
|
|
2538
|
+
const suppressMask = new Uint8Array(n);
|
|
2539
|
+
const frameEnergy = new Float32Array(n);
|
|
2540
|
+
const srcTimes = activity.times;
|
|
2541
|
+
const srcN = srcTimes.length;
|
|
2542
|
+
if (srcN === 0) {
|
|
2543
|
+
return {
|
|
2544
|
+
times: targetTimes,
|
|
2545
|
+
activityLevel,
|
|
2546
|
+
isActive,
|
|
2547
|
+
suppressMask,
|
|
2548
|
+
diagnostics: {
|
|
2549
|
+
frameEnergy,
|
|
2550
|
+
noiseFloor: activity.diagnostics.noiseFloor,
|
|
2551
|
+
enterThreshold: activity.diagnostics.enterThreshold,
|
|
2552
|
+
exitThreshold: activity.diagnostics.exitThreshold
|
|
2553
|
+
}
|
|
2554
|
+
};
|
|
2555
|
+
}
|
|
2556
|
+
for (let i = 0; i < n; i++) {
|
|
2557
|
+
const t = targetTimes[i] ?? 0;
|
|
2558
|
+
let lo = 0;
|
|
2559
|
+
let hi = srcN - 1;
|
|
2560
|
+
while (lo < hi) {
|
|
2561
|
+
const mid = lo + hi >> 1;
|
|
2562
|
+
if ((srcTimes[mid] ?? 0) < t) {
|
|
2563
|
+
lo = mid + 1;
|
|
2564
|
+
} else {
|
|
2565
|
+
hi = mid;
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
let nearest = lo;
|
|
2569
|
+
if (lo > 0 && Math.abs((srcTimes[lo - 1] ?? 0) - t) < Math.abs((srcTimes[lo] ?? 0) - t)) {
|
|
2570
|
+
nearest = lo - 1;
|
|
2571
|
+
}
|
|
2572
|
+
activityLevel[i] = activity.activityLevel[nearest] ?? 0;
|
|
2573
|
+
isActive[i] = activity.isActive[nearest] ?? 0;
|
|
2574
|
+
suppressMask[i] = activity.suppressMask[nearest] ?? 0;
|
|
2575
|
+
frameEnergy[i] = activity.diagnostics.frameEnergy[nearest] ?? 0;
|
|
2576
|
+
}
|
|
2577
|
+
return {
|
|
2578
|
+
times: targetTimes,
|
|
2579
|
+
activityLevel,
|
|
2580
|
+
isActive,
|
|
2581
|
+
suppressMask,
|
|
2582
|
+
diagnostics: {
|
|
2583
|
+
frameEnergy,
|
|
2584
|
+
noiseFloor: activity.diagnostics.noiseFloor,
|
|
2585
|
+
enterThreshold: activity.diagnostics.enterThreshold,
|
|
2586
|
+
exitThreshold: activity.diagnostics.exitThreshold
|
|
2587
|
+
}
|
|
2588
|
+
};
|
|
2589
|
+
}
|
|
2590
|
+
|
|
2591
|
+
// src/dsp/pitch.ts
|
|
2592
|
+
function pitchF0(samples, sampleRate, config) {
|
|
2593
|
+
const result = yinPitchDetection(samples, sampleRate, config);
|
|
2594
|
+
const f0 = result.f0;
|
|
2595
|
+
const activityOpts = config?.activityOptions;
|
|
2596
|
+
if (activityOpts?.activity) {
|
|
2597
|
+
const activity = interpolateActivity(activityOpts.activity, result.times);
|
|
2598
|
+
const behavior = activityOpts.inactiveBehavior ?? "zero";
|
|
2599
|
+
let lastActiveValue = 0;
|
|
2600
|
+
for (let i = 0; i < f0.length; i++) {
|
|
2601
|
+
if (!activity.isActive[i]) {
|
|
2602
|
+
if (behavior === "zero") {
|
|
2603
|
+
f0[i] = 0;
|
|
2604
|
+
} else {
|
|
2605
|
+
f0[i] = lastActiveValue;
|
|
2606
|
+
}
|
|
2607
|
+
} else {
|
|
2608
|
+
lastActiveValue = f0[i] ?? 0;
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
}
|
|
2612
|
+
return {
|
|
2613
|
+
times: result.times,
|
|
2614
|
+
values: f0
|
|
2615
|
+
};
|
|
2616
|
+
}
|
|
2617
|
+
function pitchConfidence(samples, sampleRate, config) {
|
|
2618
|
+
const result = yinPitchDetection(samples, sampleRate, config);
|
|
2619
|
+
const confidence = result.confidence;
|
|
2620
|
+
const activityOpts = config?.activityOptions;
|
|
2621
|
+
if (activityOpts?.activity) {
|
|
2622
|
+
const activity = interpolateActivity(activityOpts.activity, result.times);
|
|
2623
|
+
const behavior = activityOpts.inactiveBehavior ?? "zero";
|
|
2624
|
+
let lastActiveValue = 0;
|
|
2625
|
+
for (let i = 0; i < confidence.length; i++) {
|
|
2626
|
+
if (!activity.isActive[i]) {
|
|
2627
|
+
if (behavior === "zero") {
|
|
2628
|
+
confidence[i] = 0;
|
|
2629
|
+
} else {
|
|
2630
|
+
confidence[i] = lastActiveValue;
|
|
2631
|
+
}
|
|
2632
|
+
} else {
|
|
2633
|
+
lastActiveValue = confidence[i] ?? 0;
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2636
|
+
}
|
|
2637
|
+
return {
|
|
2638
|
+
times: result.times,
|
|
2639
|
+
values: confidence
|
|
2640
|
+
};
|
|
2641
|
+
}
|
|
2642
|
+
function yinPitchDetection(samples, sampleRate, config) {
|
|
2643
|
+
const fMinHz = config?.fMinHz ?? 50;
|
|
2644
|
+
const fMaxHz = config?.fMaxHz ?? 1e3;
|
|
2645
|
+
const hopSize = config?.hopSize ?? 512;
|
|
2646
|
+
const windowSize = config?.windowSize ?? 2048;
|
|
2647
|
+
const threshold = config?.threshold ?? 0.15;
|
|
2648
|
+
const tauMin = Math.max(2, Math.floor(sampleRate / fMaxHz));
|
|
2649
|
+
const tauMax = Math.min(windowSize / 2, Math.ceil(sampleRate / fMinHz));
|
|
2650
|
+
const nFrames = Math.max(0, Math.floor((samples.length - windowSize) / hopSize) + 1);
|
|
2651
|
+
const times = new Float32Array(nFrames);
|
|
2652
|
+
const f0 = new Float32Array(nFrames);
|
|
2653
|
+
const confidence = new Float32Array(nFrames);
|
|
2654
|
+
const d = new Float32Array(tauMax + 1);
|
|
2655
|
+
const dPrime = new Float32Array(tauMax + 1);
|
|
2656
|
+
for (let frame = 0; frame < nFrames; frame++) {
|
|
2657
|
+
const start = frame * hopSize;
|
|
2658
|
+
const frameCenter = start + windowSize / 2;
|
|
2659
|
+
times[frame] = frameCenter / sampleRate;
|
|
2660
|
+
const frameEnd = Math.min(start + windowSize, samples.length);
|
|
2661
|
+
const frameLen = frameEnd - start;
|
|
2662
|
+
let energy = 0;
|
|
2663
|
+
for (let i = 0; i < frameLen; i++) {
|
|
2664
|
+
const s = samples[start + i] ?? 0;
|
|
2665
|
+
energy += s * s;
|
|
2666
|
+
}
|
|
2667
|
+
const rms = Math.sqrt(energy / frameLen);
|
|
2668
|
+
if (rms < 1e-6) {
|
|
2669
|
+
f0[frame] = 0;
|
|
2670
|
+
confidence[frame] = 0;
|
|
2671
|
+
continue;
|
|
2672
|
+
}
|
|
2673
|
+
d[0] = 0;
|
|
2674
|
+
for (let tau = 1; tau <= tauMax && tau < frameLen; tau++) {
|
|
2675
|
+
let sum = 0;
|
|
2676
|
+
const limit = Math.min(frameLen - tau, windowSize - tau);
|
|
2677
|
+
for (let j = 0; j < limit; j++) {
|
|
2678
|
+
const diff = (samples[start + j] ?? 0) - (samples[start + j + tau] ?? 0);
|
|
2679
|
+
sum += diff * diff;
|
|
2680
|
+
}
|
|
2681
|
+
d[tau] = sum;
|
|
2682
|
+
}
|
|
2683
|
+
dPrime[0] = 1;
|
|
2684
|
+
let runningSum = 0;
|
|
2685
|
+
for (let tau = 1; tau <= tauMax && tau < frameLen; tau++) {
|
|
2686
|
+
const dVal = d[tau] ?? 0;
|
|
2687
|
+
runningSum += dVal;
|
|
2688
|
+
if (runningSum > 0) {
|
|
2689
|
+
dPrime[tau] = dVal * tau / runningSum;
|
|
2690
|
+
} else {
|
|
2691
|
+
dPrime[tau] = 1;
|
|
2692
|
+
}
|
|
2693
|
+
}
|
|
2694
|
+
let bestTau = -1;
|
|
2695
|
+
let bestDPrime = 1;
|
|
2696
|
+
for (let tau = tauMin; tau <= tauMax && tau < frameLen - 1; tau++) {
|
|
2697
|
+
const dPrimeVal = dPrime[tau] ?? 1;
|
|
2698
|
+
if (dPrimeVal < threshold) {
|
|
2699
|
+
while (tau + 1 <= tauMax && tau + 1 < frameLen - 1 && (dPrime[tau + 1] ?? 1) < (dPrime[tau] ?? 1)) {
|
|
2700
|
+
tau++;
|
|
2701
|
+
}
|
|
2702
|
+
bestTau = tau;
|
|
2703
|
+
bestDPrime = dPrime[tau] ?? 1;
|
|
2704
|
+
break;
|
|
2705
|
+
}
|
|
2706
|
+
}
|
|
2707
|
+
if (bestTau < 0) {
|
|
2708
|
+
for (let tau = tauMin; tau <= tauMax && tau < frameLen; tau++) {
|
|
2709
|
+
const dPrimeVal = dPrime[tau] ?? 1;
|
|
2710
|
+
if (dPrimeVal < bestDPrime) {
|
|
2711
|
+
bestDPrime = dPrimeVal;
|
|
2712
|
+
bestTau = tau;
|
|
2713
|
+
}
|
|
2714
|
+
}
|
|
2715
|
+
}
|
|
2716
|
+
if (bestTau > tauMin && bestTau < tauMax - 1 && bestTau < frameLen - 1) {
|
|
2717
|
+
const y0 = dPrime[bestTau - 1] ?? 1;
|
|
2718
|
+
const y1 = dPrime[bestTau] ?? 1;
|
|
2719
|
+
const y2 = dPrime[bestTau + 1] ?? 1;
|
|
2720
|
+
const denom = 2 * (2 * y1 - y0 - y2);
|
|
2721
|
+
if (Math.abs(denom) > 1e-10) {
|
|
2722
|
+
const delta2 = (y0 - y2) / denom;
|
|
2723
|
+
const interpolatedTau = bestTau + delta2;
|
|
2724
|
+
if (interpolatedTau >= tauMin && interpolatedTau <= tauMax) {
|
|
2725
|
+
f0[frame] = sampleRate / interpolatedTau;
|
|
2726
|
+
confidence[frame] = Math.max(0, Math.min(1, 1 - bestDPrime));
|
|
2727
|
+
continue;
|
|
2728
|
+
}
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2731
|
+
if (bestTau >= tauMin && bestTau <= tauMax) {
|
|
2732
|
+
f0[frame] = sampleRate / bestTau;
|
|
2733
|
+
confidence[frame] = Math.max(0, Math.min(1, 1 - bestDPrime));
|
|
2734
|
+
} else {
|
|
2735
|
+
f0[frame] = 0;
|
|
2736
|
+
confidence[frame] = 0;
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
return { times, f0, confidence };
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
// src/runner/runMir.ts
|
|
2743
|
+
function nowMs2() {
|
|
2744
|
+
return typeof performance !== "undefined" ? performance.now() : Date.now();
|
|
2745
|
+
}
|
|
2746
|
+
function asAudioBufferLike(audio) {
|
|
2747
|
+
return {
|
|
2748
|
+
sampleRate: audio.sampleRate,
|
|
2749
|
+
numberOfChannels: 1,
|
|
2750
|
+
getChannelData: () => audio.mono
|
|
2751
|
+
};
|
|
2752
|
+
}
|
|
2753
|
+
async function runMir(audio, request, options = {}) {
|
|
2754
|
+
options = {
|
|
2755
|
+
...options,
|
|
2756
|
+
onset: { ...request.onset, ...options.onset },
|
|
2757
|
+
peakPick: { ...request.peakPick, ...options.peakPick },
|
|
2758
|
+
hpss: { ...request.hpss, ...options.hpss },
|
|
2759
|
+
mfcc: { ...request.mfcc, ...options.mfcc }
|
|
2760
|
+
};
|
|
2761
|
+
const t0 = nowMs2();
|
|
2762
|
+
const backend = request.backend ?? "cpu";
|
|
2763
|
+
const specConfig = request.spectrogram ?? {
|
|
2764
|
+
fftSize: 2048,
|
|
2765
|
+
hopSize: 512,
|
|
2766
|
+
window: "hann"
|
|
2767
|
+
};
|
|
2768
|
+
if (request.fn === "amplitudeEnvelope") {
|
|
2769
|
+
const cpuStart2 = nowMs2();
|
|
2770
|
+
const result = amplitudeEnvelope(audio.mono, audio.sampleRate, {
|
|
2771
|
+
hopSize: specConfig.hopSize,
|
|
2772
|
+
windowSize: specConfig.fftSize
|
|
2773
|
+
});
|
|
2774
|
+
const cpuEnd = nowMs2();
|
|
2775
|
+
return {
|
|
2776
|
+
kind: "1d",
|
|
2777
|
+
times: result.times,
|
|
2778
|
+
values: result.values,
|
|
2779
|
+
meta: {
|
|
2780
|
+
backend: "cpu",
|
|
2781
|
+
usedGpu: false,
|
|
2782
|
+
timings: {
|
|
2783
|
+
totalMs: cpuEnd - t0,
|
|
2784
|
+
cpuMs: cpuEnd - cpuStart2
|
|
2785
|
+
}
|
|
2786
|
+
}
|
|
2787
|
+
};
|
|
2788
|
+
}
|
|
2789
|
+
const cpuStart = nowMs2();
|
|
2790
|
+
const spec = await spectrogram(asAudioBufferLike(audio), specConfig, void 0, {
|
|
2791
|
+
isCancelled: options.isCancelled
|
|
2792
|
+
});
|
|
2793
|
+
const cpuAfterSpec = nowMs2();
|
|
2794
|
+
if (options.isCancelled?.()) {
|
|
2795
|
+
throw new Error("@octoseq/mir: cancelled");
|
|
2796
|
+
}
|
|
2797
|
+
if (request.fn === "spectralCentroid") {
|
|
2798
|
+
const values = spectralCentroid(spec);
|
|
2799
|
+
const cpuEnd = nowMs2();
|
|
2800
|
+
return {
|
|
2801
|
+
kind: "1d",
|
|
2802
|
+
times: spec.times,
|
|
2803
|
+
values,
|
|
2804
|
+
meta: {
|
|
2805
|
+
backend: "cpu",
|
|
2806
|
+
usedGpu: false,
|
|
2807
|
+
timings: {
|
|
2808
|
+
totalMs: cpuEnd - t0,
|
|
2809
|
+
cpuMs: cpuEnd - cpuStart
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
};
|
|
2813
|
+
}
|
|
2814
|
+
if (request.fn === "spectralFlux") {
|
|
2815
|
+
const values = spectralFlux(spec);
|
|
2816
|
+
const cpuEnd = nowMs2();
|
|
2817
|
+
return {
|
|
2818
|
+
kind: "1d",
|
|
2819
|
+
times: spec.times,
|
|
2820
|
+
values,
|
|
2821
|
+
meta: {
|
|
2822
|
+
backend: "cpu",
|
|
2823
|
+
usedGpu: false,
|
|
2824
|
+
timings: {
|
|
2825
|
+
totalMs: cpuEnd - t0,
|
|
2826
|
+
cpuMs: cpuEnd - cpuStart
|
|
2827
|
+
}
|
|
2828
|
+
}
|
|
2829
|
+
};
|
|
2830
|
+
}
|
|
2831
|
+
const melConfig = request.mel ?? { nMels: 64 };
|
|
2832
|
+
const computeMel = async (useGpu) => {
|
|
2833
|
+
const melCpuStart = nowMs2();
|
|
2834
|
+
if (useGpu) {
|
|
2835
|
+
if (!options.gpu) {
|
|
2836
|
+
throw new Error("@octoseq/mir: backend='gpu' requested but no MirGPU provided");
|
|
2837
|
+
}
|
|
2838
|
+
const gpuStart = nowMs2();
|
|
2839
|
+
try {
|
|
2840
|
+
const mel3 = await melSpectrogram(spec, melConfig, options.gpu);
|
|
2841
|
+
const gpuEnd = nowMs2();
|
|
2842
|
+
const gpuKernelMs = mel3.gpuTimings?.gpuSubmitToReadbackMs;
|
|
2843
|
+
return {
|
|
2844
|
+
mel: mel3,
|
|
2845
|
+
usedGpu: true,
|
|
2846
|
+
gpuMs: gpuKernelMs ?? gpuEnd - gpuStart,
|
|
2847
|
+
cpuExtraMs: nowMs2() - melCpuStart - (gpuEnd - gpuStart)
|
|
2848
|
+
};
|
|
2849
|
+
} catch (e) {
|
|
2850
|
+
if (options.strictGpu) throw e;
|
|
2851
|
+
}
|
|
2852
|
+
}
|
|
2853
|
+
const mel2 = await melSpectrogram(spec, melConfig, void 0);
|
|
2854
|
+
const melCpuEnd = nowMs2();
|
|
2855
|
+
return {
|
|
2856
|
+
mel: mel2,
|
|
2857
|
+
usedGpu: false,
|
|
2858
|
+
cpuExtraMs: melCpuEnd - melCpuStart
|
|
2859
|
+
};
|
|
2860
|
+
};
|
|
2861
|
+
if (request.fn === "melSpectrogram") {
|
|
2862
|
+
const { mel: mel2, usedGpu: usedGpu2, gpuMs: gpuMs2, cpuExtraMs: cpuExtraMs2 } = await computeMel(backend === "gpu");
|
|
2863
|
+
const end2 = nowMs2();
|
|
2864
|
+
return {
|
|
2865
|
+
kind: "2d",
|
|
2866
|
+
times: mel2.times,
|
|
2867
|
+
data: mel2.melBands,
|
|
2868
|
+
meta: {
|
|
2869
|
+
backend: usedGpu2 ? "gpu" : "cpu",
|
|
2870
|
+
usedGpu: usedGpu2,
|
|
2871
|
+
timings: {
|
|
2872
|
+
totalMs: end2 - t0,
|
|
2873
|
+
cpuMs: cpuAfterSpec - cpuStart + cpuExtraMs2,
|
|
2874
|
+
gpuMs: gpuMs2
|
|
2875
|
+
}
|
|
2876
|
+
}
|
|
2877
|
+
};
|
|
2878
|
+
}
|
|
2879
|
+
if (request.fn === "onsetEnvelope") {
|
|
2880
|
+
if (backend === "gpu") {
|
|
2881
|
+
if (!options.gpu) throw new Error("@octoseq/mir: backend='gpu' requested but no MirGPU provided");
|
|
2882
|
+
const { mel: mel3, usedGpu: usedGpuForMel, gpuMs: melGpuMs, cpuExtraMs: melCpuMs2 } = await computeMel(true);
|
|
2883
|
+
nowMs2();
|
|
2884
|
+
try {
|
|
2885
|
+
const onsetGpu = await onsetEnvelopeFromMelGpu(mel3, options.gpu, {
|
|
2886
|
+
diffMethod: options.onset?.diffMethod
|
|
2887
|
+
});
|
|
2888
|
+
const end3 = nowMs2();
|
|
2889
|
+
return {
|
|
2890
|
+
kind: "1d",
|
|
2891
|
+
times: onsetGpu.times,
|
|
2892
|
+
values: onsetGpu.values,
|
|
2893
|
+
meta: {
|
|
2894
|
+
backend: "gpu",
|
|
2895
|
+
usedGpu: true,
|
|
2896
|
+
timings: {
|
|
2897
|
+
totalMs: end3 - t0,
|
|
2898
|
+
cpuMs: cpuAfterSpec - cpuStart + melCpuMs2,
|
|
2899
|
+
gpuMs: (melGpuMs ?? 0) + onsetGpu.gpuTimings.gpuSubmitToReadbackMs
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
};
|
|
2903
|
+
} catch (e) {
|
|
2904
|
+
if (options.strictGpu) throw e;
|
|
2905
|
+
} finally {
|
|
2906
|
+
}
|
|
2907
|
+
}
|
|
2908
|
+
const { mel: mel2, cpuExtraMs: melCpuMs } = await computeMel(false);
|
|
2909
|
+
const onset = onsetEnvelopeFromMel(mel2, {
|
|
2910
|
+
smoothMs: options.onset?.smoothMs,
|
|
2911
|
+
diffMethod: options.onset?.diffMethod,
|
|
2912
|
+
useLog: options.onset?.useLog
|
|
2913
|
+
});
|
|
2914
|
+
const end2 = nowMs2();
|
|
2915
|
+
return {
|
|
2916
|
+
kind: "1d",
|
|
2917
|
+
times: onset.times,
|
|
2918
|
+
values: onset.values,
|
|
2919
|
+
meta: {
|
|
2920
|
+
backend: "cpu",
|
|
2921
|
+
usedGpu: false,
|
|
2922
|
+
timings: {
|
|
2923
|
+
totalMs: end2 - t0,
|
|
2924
|
+
cpuMs: cpuAfterSpec - cpuStart + melCpuMs
|
|
2925
|
+
}
|
|
2926
|
+
}
|
|
2927
|
+
};
|
|
2928
|
+
}
|
|
2929
|
+
if (request.fn === "onsetPeaks") {
|
|
2930
|
+
const { mel: mel2, cpuExtraMs: melCpuMs } = await computeMel(false);
|
|
2931
|
+
const onset = onsetEnvelopeFromMel(mel2, {
|
|
2932
|
+
smoothMs: options.onset?.smoothMs,
|
|
2933
|
+
diffMethod: options.onset?.diffMethod,
|
|
2934
|
+
useLog: options.onset?.useLog
|
|
2935
|
+
});
|
|
2936
|
+
const events = peakPick(onset.times, onset.values, {
|
|
2937
|
+
minIntervalSec: options.peakPick?.minIntervalSec,
|
|
2938
|
+
threshold: options.peakPick?.threshold,
|
|
2939
|
+
adaptive: options.peakPick?.adaptiveFactor ? { method: "meanStd", factor: options.peakPick.adaptiveFactor } : void 0
|
|
2940
|
+
});
|
|
2941
|
+
const end2 = nowMs2();
|
|
2942
|
+
return {
|
|
2943
|
+
kind: "events",
|
|
2944
|
+
times: onset.times,
|
|
2945
|
+
events,
|
|
2946
|
+
meta: {
|
|
2947
|
+
backend: "cpu",
|
|
2948
|
+
usedGpu: false,
|
|
2949
|
+
timings: {
|
|
2950
|
+
totalMs: end2 - t0,
|
|
2951
|
+
cpuMs: cpuAfterSpec - cpuStart + melCpuMs
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2954
|
+
};
|
|
2955
|
+
}
|
|
2956
|
+
if (request.fn === "beatCandidates") {
|
|
2957
|
+
const { mel: mel2, cpuExtraMs: melCpuMs } = await computeMel(false);
|
|
2958
|
+
const beatOpts = request.beatCandidates ?? {};
|
|
2959
|
+
const result = detectBeatCandidates(mel2, spec, {
|
|
2960
|
+
minIntervalSec: beatOpts.minIntervalSec,
|
|
2961
|
+
thresholdFactor: beatOpts.thresholdFactor,
|
|
2962
|
+
smoothMs: beatOpts.smoothMs
|
|
2963
|
+
});
|
|
2964
|
+
const end2 = nowMs2();
|
|
2965
|
+
return {
|
|
2966
|
+
kind: "beatCandidates",
|
|
2967
|
+
times: result.salience.times,
|
|
2968
|
+
candidates: result.candidates,
|
|
2969
|
+
salience: beatOpts.includeSalience ? result.salience : void 0,
|
|
2970
|
+
meta: {
|
|
2971
|
+
backend: "cpu",
|
|
2972
|
+
usedGpu: false,
|
|
2973
|
+
timings: {
|
|
2974
|
+
totalMs: end2 - t0,
|
|
2975
|
+
cpuMs: cpuAfterSpec - cpuStart + melCpuMs
|
|
2976
|
+
}
|
|
2977
|
+
}
|
|
2978
|
+
};
|
|
2979
|
+
}
|
|
2980
|
+
if (request.fn === "tempoHypotheses") {
|
|
2981
|
+
const { mel: mel2, cpuExtraMs: melCpuMs } = await computeMel(false);
|
|
2982
|
+
const beatOpts = request.beatCandidates ?? {};
|
|
2983
|
+
const beatResult = detectBeatCandidates(mel2, spec, {
|
|
2984
|
+
minIntervalSec: beatOpts.minIntervalSec,
|
|
2985
|
+
thresholdFactor: beatOpts.thresholdFactor,
|
|
2986
|
+
smoothMs: beatOpts.smoothMs
|
|
2987
|
+
});
|
|
2988
|
+
const tempoStart = nowMs2();
|
|
2989
|
+
const tempoOpts = request.tempoHypotheses ?? {};
|
|
2990
|
+
const result = generateTempoHypotheses(beatResult.candidates, {
|
|
2991
|
+
minBpm: tempoOpts.minBpm,
|
|
2992
|
+
maxBpm: tempoOpts.maxBpm,
|
|
2993
|
+
binSizeBpm: tempoOpts.binSizeBpm,
|
|
2994
|
+
maxHypotheses: tempoOpts.maxHypotheses,
|
|
2995
|
+
minConfidence: tempoOpts.minConfidence,
|
|
2996
|
+
weightByStrength: tempoOpts.weightByStrength,
|
|
2997
|
+
includeHistogram: tempoOpts.includeHistogram
|
|
2998
|
+
});
|
|
2999
|
+
const end2 = nowMs2();
|
|
3000
|
+
return {
|
|
3001
|
+
kind: "tempoHypotheses",
|
|
3002
|
+
times: spec.times,
|
|
3003
|
+
hypotheses: result.hypotheses,
|
|
3004
|
+
inputCandidateCount: result.inputCandidateCount,
|
|
3005
|
+
histogram: result.histogram,
|
|
3006
|
+
meta: {
|
|
3007
|
+
backend: "cpu",
|
|
3008
|
+
usedGpu: false,
|
|
3009
|
+
timings: {
|
|
3010
|
+
totalMs: end2 - t0,
|
|
3011
|
+
cpuMs: cpuAfterSpec - cpuStart + melCpuMs + (end2 - tempoStart)
|
|
3012
|
+
}
|
|
3013
|
+
}
|
|
3014
|
+
};
|
|
3015
|
+
}
|
|
3016
|
+
if (request.fn === "hpssHarmonic" || request.fn === "hpssPercussive") {
|
|
3017
|
+
const hpssSpecConfig = options.hpss?.spectrogram ?? specConfig;
|
|
3018
|
+
const needsHpssSpec = hpssSpecConfig.fftSize !== specConfig.fftSize || hpssSpecConfig.hopSize !== specConfig.hopSize;
|
|
3019
|
+
let hpssSpec;
|
|
3020
|
+
let hpssCpuStart = cpuAfterSpec;
|
|
3021
|
+
if (needsHpssSpec) {
|
|
3022
|
+
hpssCpuStart = nowMs2();
|
|
3023
|
+
hpssSpec = await spectrogram(asAudioBufferLike(audio), hpssSpecConfig, void 0, {
|
|
3024
|
+
isCancelled: options.isCancelled
|
|
3025
|
+
});
|
|
3026
|
+
} else {
|
|
3027
|
+
hpssSpec = spec;
|
|
3028
|
+
}
|
|
3029
|
+
const hpssAfterSpec = nowMs2();
|
|
3030
|
+
if (backend === "gpu") {
|
|
3031
|
+
if (!options.gpu) throw new Error("@octoseq/mir: backend='gpu' requested but no MirGPU provided");
|
|
3032
|
+
const hpssStart2 = nowMs2();
|
|
3033
|
+
try {
|
|
3034
|
+
const out = await hpssGpu(hpssSpec, options.gpu, {
|
|
3035
|
+
timeMedian: options.hpss?.timeMedian,
|
|
3036
|
+
freqMedian: options.hpss?.freqMedian,
|
|
3037
|
+
softMask: true,
|
|
3038
|
+
// preserve CPU default
|
|
3039
|
+
isCancelled: options.isCancelled
|
|
3040
|
+
});
|
|
3041
|
+
const end3 = nowMs2();
|
|
3042
|
+
const chosen2 = request.fn === "hpssHarmonic" ? out.harmonic : out.percussive;
|
|
3043
|
+
return {
|
|
3044
|
+
kind: "2d",
|
|
3045
|
+
times: chosen2.times,
|
|
3046
|
+
data: chosen2.magnitudes,
|
|
3047
|
+
meta: {
|
|
3048
|
+
backend: "gpu",
|
|
3049
|
+
usedGpu: true,
|
|
3050
|
+
timings: {
|
|
3051
|
+
totalMs: end3 - t0,
|
|
3052
|
+
cpuMs: (needsHpssSpec ? hpssAfterSpec - hpssCpuStart : cpuAfterSpec - cpuStart) + (end3 - hpssStart2 - out.gpuMs),
|
|
3053
|
+
gpuMs: out.gpuMs
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
};
|
|
3057
|
+
} catch (e) {
|
|
3058
|
+
if (options.strictGpu) throw e;
|
|
3059
|
+
}
|
|
3060
|
+
}
|
|
3061
|
+
const hpssStart = nowMs2();
|
|
3062
|
+
const { harmonic, percussive } = hpss(hpssSpec, {
|
|
3063
|
+
timeMedian: options.hpss?.timeMedian,
|
|
3064
|
+
freqMedian: options.hpss?.freqMedian,
|
|
3065
|
+
isCancelled: options.isCancelled
|
|
3066
|
+
});
|
|
3067
|
+
const end2 = nowMs2();
|
|
3068
|
+
const cpuMs = (needsHpssSpec ? hpssAfterSpec - hpssCpuStart : cpuAfterSpec - cpuStart) + (end2 - hpssStart);
|
|
3069
|
+
const chosen = request.fn === "hpssHarmonic" ? harmonic : percussive;
|
|
3070
|
+
return {
|
|
3071
|
+
kind: "2d",
|
|
3072
|
+
times: chosen.times,
|
|
3073
|
+
data: chosen.magnitudes,
|
|
3074
|
+
meta: {
|
|
3075
|
+
backend: "cpu",
|
|
3076
|
+
usedGpu: false,
|
|
3077
|
+
timings: { totalMs: end2 - t0, cpuMs }
|
|
3078
|
+
}
|
|
3079
|
+
};
|
|
3080
|
+
}
|
|
3081
|
+
if (request.fn === "mfcc" || request.fn === "mfccDelta" || request.fn === "mfccDeltaDelta") {
|
|
3082
|
+
const mfccSpecConfig = options.mfcc?.spectrogram ?? specConfig;
|
|
3083
|
+
const needsMfccSpec = mfccSpecConfig.fftSize !== specConfig.fftSize || mfccSpecConfig.hopSize !== specConfig.hopSize;
|
|
3084
|
+
let mfccMel;
|
|
3085
|
+
let mfccCpuMs;
|
|
3086
|
+
if (needsMfccSpec) {
|
|
3087
|
+
const mfccCpuStart = nowMs2();
|
|
3088
|
+
const mfccSpec = await spectrogram(asAudioBufferLike(audio), mfccSpecConfig, void 0, {
|
|
3089
|
+
isCancelled: options.isCancelled
|
|
3090
|
+
});
|
|
3091
|
+
const mfccMelResult = await melSpectrogram(mfccSpec, melConfig, void 0);
|
|
3092
|
+
mfccMel = mfccMelResult;
|
|
3093
|
+
mfccCpuMs = nowMs2() - mfccCpuStart;
|
|
3094
|
+
} else {
|
|
3095
|
+
const { mel: mel2, cpuExtraMs: cpuExtraMs2 } = await computeMel(false);
|
|
3096
|
+
mfccMel = mel2;
|
|
3097
|
+
mfccCpuMs = cpuAfterSpec - cpuStart + cpuExtraMs2;
|
|
3098
|
+
}
|
|
3099
|
+
const mfccStart = nowMs2();
|
|
3100
|
+
const base = mfcc(mfccMel, { nCoeffs: options.mfcc?.nCoeffs });
|
|
3101
|
+
const features = { times: base.times, values: base.coeffs };
|
|
3102
|
+
const chosen = request.fn === "mfcc" ? features : request.fn === "mfccDelta" ? delta(features) : deltaDelta(features);
|
|
3103
|
+
const end2 = nowMs2();
|
|
3104
|
+
return {
|
|
3105
|
+
kind: "2d",
|
|
3106
|
+
times: chosen.times,
|
|
3107
|
+
data: chosen.values,
|
|
3108
|
+
meta: {
|
|
3109
|
+
backend: "cpu",
|
|
3110
|
+
usedGpu: false,
|
|
3111
|
+
timings: {
|
|
3112
|
+
totalMs: end2 - t0,
|
|
3113
|
+
cpuMs: mfccCpuMs + (end2 - mfccStart)
|
|
3114
|
+
}
|
|
3115
|
+
}
|
|
3116
|
+
};
|
|
3117
|
+
}
|
|
3118
|
+
if (request.fn === "cqtHarmonicEnergy" || request.fn === "cqtBassPitchMotion" || request.fn === "cqtTonalStability") {
|
|
3119
|
+
const cqtStart = nowMs2();
|
|
3120
|
+
const cqtConfig = withCqtDefaults(request.cqt);
|
|
3121
|
+
const cqt = await cqtSpectrogram(asAudioBufferLike(audio), cqtConfig, {
|
|
3122
|
+
isCancelled: options.isCancelled
|
|
3123
|
+
});
|
|
3124
|
+
const cqtEnd = nowMs2();
|
|
3125
|
+
let signal;
|
|
3126
|
+
if (request.fn === "cqtHarmonicEnergy") {
|
|
3127
|
+
signal = harmonicEnergy(cqt);
|
|
3128
|
+
} else if (request.fn === "cqtBassPitchMotion") {
|
|
3129
|
+
signal = bassPitchMotion(cqt);
|
|
3130
|
+
} else {
|
|
3131
|
+
signal = tonalStability(cqt);
|
|
3132
|
+
}
|
|
3133
|
+
const end2 = nowMs2();
|
|
3134
|
+
return {
|
|
3135
|
+
kind: "1d",
|
|
3136
|
+
times: signal.times,
|
|
3137
|
+
values: signal.values,
|
|
3138
|
+
meta: {
|
|
3139
|
+
backend: "cpu",
|
|
3140
|
+
usedGpu: false,
|
|
3141
|
+
timings: {
|
|
3142
|
+
totalMs: end2 - t0,
|
|
3143
|
+
cpuMs: cqtEnd - cqtStart + (end2 - cqtEnd)
|
|
3144
|
+
}
|
|
3145
|
+
}
|
|
3146
|
+
};
|
|
3147
|
+
}
|
|
3148
|
+
if (request.fn === "pitchF0" || request.fn === "pitchConfidence") {
|
|
3149
|
+
const pitchStart = nowMs2();
|
|
3150
|
+
const pitchConfig = {
|
|
3151
|
+
fMinHz: request.pitch?.fMinHz,
|
|
3152
|
+
fMaxHz: request.pitch?.fMaxHz,
|
|
3153
|
+
threshold: request.pitch?.threshold,
|
|
3154
|
+
hopSize: specConfig.hopSize,
|
|
3155
|
+
windowSize: specConfig.fftSize
|
|
3156
|
+
};
|
|
3157
|
+
const result = request.fn === "pitchF0" ? pitchF0(audio.mono, audio.sampleRate, pitchConfig) : pitchConfidence(audio.mono, audio.sampleRate, pitchConfig);
|
|
3158
|
+
const end2 = nowMs2();
|
|
3159
|
+
return {
|
|
3160
|
+
kind: "1d",
|
|
3161
|
+
times: result.times,
|
|
3162
|
+
values: result.values,
|
|
3163
|
+
meta: {
|
|
3164
|
+
backend: "cpu",
|
|
3165
|
+
usedGpu: false,
|
|
3166
|
+
timings: {
|
|
3167
|
+
totalMs: end2 - t0,
|
|
3168
|
+
cpuMs: end2 - pitchStart
|
|
3169
|
+
}
|
|
3170
|
+
}
|
|
3171
|
+
};
|
|
3172
|
+
}
|
|
3173
|
+
if (request.fn === "activity") {
|
|
3174
|
+
const { mel: mel2, cpuExtraMs: melCpuMs } = await computeMel(false);
|
|
3175
|
+
const activityStart = nowMs2();
|
|
3176
|
+
const activityConfig = {
|
|
3177
|
+
energyPercentile: request.activity?.energyPercentile,
|
|
3178
|
+
enterMargin: request.activity?.enterMargin,
|
|
3179
|
+
exitMargin: request.activity?.exitMargin,
|
|
3180
|
+
hangoverMs: request.activity?.hangoverMs,
|
|
3181
|
+
minActiveMs: request.activity?.minActiveMs,
|
|
3182
|
+
smoothMs: request.activity?.smoothMs
|
|
3183
|
+
};
|
|
3184
|
+
const activity = computeActivityFromMel(mel2, activityConfig);
|
|
3185
|
+
const end2 = nowMs2();
|
|
3186
|
+
let activeCount = 0;
|
|
3187
|
+
for (let i = 0; i < activity.isActive.length; i++) {
|
|
3188
|
+
if (activity.isActive[i]) activeCount++;
|
|
3189
|
+
}
|
|
3190
|
+
return {
|
|
3191
|
+
kind: "activity",
|
|
3192
|
+
times: activity.times,
|
|
3193
|
+
activityLevel: activity.activityLevel,
|
|
3194
|
+
isActive: activity.isActive,
|
|
3195
|
+
suppressMask: activity.suppressMask,
|
|
3196
|
+
meta: {
|
|
3197
|
+
backend: "cpu",
|
|
3198
|
+
usedGpu: false,
|
|
3199
|
+
timings: {
|
|
3200
|
+
totalMs: end2 - t0,
|
|
3201
|
+
cpuMs: cpuAfterSpec - cpuStart + melCpuMs + (end2 - activityStart)
|
|
3202
|
+
}
|
|
3203
|
+
},
|
|
3204
|
+
diagnostics: {
|
|
3205
|
+
noiseFloor: activity.diagnostics.noiseFloor,
|
|
3206
|
+
enterThreshold: activity.diagnostics.enterThreshold,
|
|
3207
|
+
exitThreshold: activity.diagnostics.exitThreshold,
|
|
3208
|
+
totalFrames: activity.times.length,
|
|
3209
|
+
activeFrames: activeCount,
|
|
3210
|
+
activeFraction: activity.times.length > 0 ? activeCount / activity.times.length : 0
|
|
3211
|
+
}
|
|
3212
|
+
};
|
|
3213
|
+
}
|
|
3214
|
+
const { mel, usedGpu, gpuMs, cpuExtraMs } = await computeMel(backend === "gpu");
|
|
3215
|
+
const end = nowMs2();
|
|
3216
|
+
return {
|
|
3217
|
+
kind: "2d",
|
|
3218
|
+
times: mel.times,
|
|
3219
|
+
data: mel.melBands,
|
|
3220
|
+
meta: {
|
|
3221
|
+
backend: usedGpu ? "gpu" : "cpu",
|
|
3222
|
+
usedGpu,
|
|
3223
|
+
timings: {
|
|
3224
|
+
totalMs: end - t0,
|
|
3225
|
+
cpuMs: cpuAfterSpec - cpuStart + cpuExtraMs,
|
|
3226
|
+
gpuMs
|
|
3227
|
+
}
|
|
3228
|
+
}
|
|
3229
|
+
};
|
|
3230
|
+
}
|
|
3231
|
+
|
|
3232
|
+
export { CQT_DEFAULTS, amplitudeEnvelope, applyActivityGating, bassPitchMotion, beatSalienceFromMel, buildActivityMask, computeActivityFromAudio, computeActivityFromMel, computeActivityFromSpectrogram, computeAllCqtSignals, computeCqt, computeCqtSignal, computeFrameEnergyFromMel, computeFrameEnergyFromSpectrogram, computeSilenceGating, cqtBinToHz, cqtSpectrogram, delta, deltaDelta, detectBeatCandidates, estimateNoiseFloor, featureIndexToHz, generateTempoHypotheses, getCqtBinFrequencies, getNumBins, getNumOctaves, harmonicEnergy, hpss, hzToCqtBin, hzToFeatureIndex, hzToMel, interpolateActivity, melSpectrogram, melToHz, mfcc, onsetEnvelopeFromMel, onsetEnvelopeFromMelGpu, onsetEnvelopeFromSpectrogram, peakPick, pitchConfidence, pitchF0, runMir, spectralCentroid, spectralFlux, spectrogram, tonalStability, withBinGateDefaults, withCqtDefaults, withSilenceGateDefaults };
|
|
3233
|
+
//# sourceMappingURL=chunk-5WZ7TTDW.js.map
|
|
3234
|
+
//# sourceMappingURL=chunk-5WZ7TTDW.js.map
|