@williamzujkowski/live-coding-music-mcp 4.0.0 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +202 -705
- package/dist/AudioAnalyzer.js +1294 -91
- package/dist/PatternStore.js +276 -23
- package/dist/StrudelController.js +388 -56
- package/dist/server/resources.js +27 -20
- package/dist/server/server.js +384 -76
- package/dist/server/tools/ai.js +124 -91
- package/dist/server/tools/analysis.js +168 -59
- package/dist/server/tools/capture.js +135 -24
- package/dist/server/tools/compose.js +26 -26
- package/dist/server/tools/diagnostics.js +20 -3
- package/dist/server/tools/editor.js +60 -6
- package/dist/server/tools/generate.js +38 -15
- package/dist/server/tools/history.js +30 -8
- package/dist/server/tools/playback.js +2 -1
- package/dist/server/tools/session.js +38 -6
- package/dist/server/tools/storage.js +179 -8
- package/dist/server/tools/transform.js +127 -43
- package/dist/server/tools/types.js +209 -14
- package/dist/services/AudioCaptureService.js +302 -33
- package/dist/services/AudioExportService.js +318 -0
- package/dist/services/BrowserOnlyFunctions.js +149 -0
- package/dist/services/EventDensityProbe.js +324 -0
- package/dist/services/GeminiService.js +221 -36
- package/dist/services/IsolatedEngineRunner.js +308 -0
- package/dist/services/IsolatedStrudelEngine.js +135 -0
- package/dist/services/LocalPatternEngine.js +16 -0
- package/dist/services/MIDIExportService.js +765 -71
- package/dist/services/MIDIImportService.js +474 -0
- package/dist/services/MusicTheory.js +85 -13
- package/dist/services/PatternGenerator.js +168 -85
- package/dist/services/PatternSandbox.js +359 -0
- package/dist/services/SessionManager.js +338 -65
- package/dist/services/StrudelEngine.js +218 -12
- package/dist/services/StrudelEngineHelpers.js +24 -8
- package/dist/services/StyleRegistry.js +204 -0
- package/dist/services/Transferable.js +114 -0
- package/dist/services/ai/AiTransport.js +56 -0
- package/dist/services/ai/AudioMeasurements.js +155 -0
- package/dist/services/ai/CliTransport.js +256 -0
- package/dist/services/engineChild.js +83 -0
- package/dist/services/engineChildPath.js +28 -0
- package/dist/utils/CategorisedError.js +59 -0
- package/dist/utils/ErrorRecovery.js +130 -110
- package/dist/utils/InputValidator.js +55 -7
- package/dist/utils/PatternValidator.js +136 -17
- package/dist/utils/SafePath.js +112 -0
- package/dist/utils/ServerConfig.js +248 -0
- package/dist/utils/TableLookup.js +41 -0
- package/dist/utils/Tempo.js +143 -0
- package/package.json +27 -16
- package/dist/services/__mocks__/StrudelEngine.js +0 -157
- package/dist/types/index.js +0 -1
package/dist/AudioAnalyzer.js
CHANGED
|
@@ -1,14 +1,232 @@
|
|
|
1
|
+
import { Logger } from './utils/Logger.js';
|
|
2
|
+
const DEFAULT_FFT_SIZE = 1024;
|
|
3
|
+
const DEFAULT_SMOOTHING = 0.8;
|
|
4
|
+
const MIN_FFT_SIZE = 32;
|
|
5
|
+
const MAX_FFT_SIZE = 32768;
|
|
6
|
+
/**
|
|
7
|
+
* True if `n` is a power of two within the Web Audio AnalyserNode range
|
|
8
|
+
* (32-32768). Web Audio rejects anything else with an InvalidStateError.
|
|
9
|
+
*/
|
|
10
|
+
function isValidFftSize(n) {
|
|
11
|
+
return (typeof n === 'number' &&
|
|
12
|
+
Number.isInteger(n) &&
|
|
13
|
+
n >= MIN_FFT_SIZE &&
|
|
14
|
+
n <= MAX_FFT_SIZE &&
|
|
15
|
+
(n & (n - 1)) === 0);
|
|
16
|
+
}
|
|
17
|
+
/** Timestamps, whichever shape was passed. */
|
|
18
|
+
export function onsetTimes(onsets) {
|
|
19
|
+
return onsets.map(onset => (typeof onset === 'number' ? onset : onset.t));
|
|
20
|
+
}
|
|
21
|
+
/** Strengths, or an empty array when the caller had none to give. */
|
|
22
|
+
function onsetStrengths(onsets) {
|
|
23
|
+
if (onsets.length === 0 || typeof onsets[0] === 'number')
|
|
24
|
+
return [];
|
|
25
|
+
return onsets.map(onset => onset.strength);
|
|
26
|
+
}
|
|
1
27
|
export class AudioAnalyzer {
|
|
2
28
|
_analysisCache = null;
|
|
3
29
|
_cacheTimestamp = 0;
|
|
4
30
|
ANALYSIS_CACHE_TTL = 50; // milliseconds
|
|
5
31
|
// Advanced analysis tracking
|
|
32
|
+
/**
|
|
33
|
+
* Detected onsets, with the flux that produced each one (#352). Bare
|
|
34
|
+
* timestamps would throw away the only signal that distinguishes a
|
|
35
|
+
* kick from a hi-hat.
|
|
36
|
+
*/
|
|
6
37
|
_onsetHistory = [];
|
|
38
|
+
/**
|
|
39
|
+
* The tempo we have committed to, and the one still on probation.
|
|
40
|
+
*
|
|
41
|
+
* `detectTempo` used to answer from scratch every poll, so the answer
|
|
42
|
+
* moved while the onset window filled. Measured on unchanging audio:
|
|
43
|
+
* the amen break read 167, 83, 83, 83 across four polls, and
|
|
44
|
+
* `gen/intelligent_dnb` read 130, 130, 86, 130 (#374, #501).
|
|
45
|
+
*
|
|
46
|
+
* At most one of 167 and 83 is a measurement. The other is an
|
|
47
|
+
* artifact of how much history had accumulated — and a caller has no
|
|
48
|
+
* way to tell which poll it is holding.
|
|
49
|
+
*
|
|
50
|
+
* So a reading has to be seen twice, from two different windows,
|
|
51
|
+
* before it is reported; and once reported it is not displaced until
|
|
52
|
+
* some other reading has cleared the same bar. Cleared by
|
|
53
|
+
* `resetTempoHistory`, which is the pattern-change boundary.
|
|
54
|
+
*/
|
|
55
|
+
_confirmedTempo = null;
|
|
56
|
+
_unconfirmedBpm = null;
|
|
57
|
+
/**
|
|
58
|
+
* What the onsets looked like at the previous poll.
|
|
59
|
+
*
|
|
60
|
+
* Confirmation has to mean two readings from DIFFERENT evidence.
|
|
61
|
+
* `detectTempo` drains the page's flux buffer, so calling it twice in
|
|
62
|
+
* a row sees the identical onset history and trivially agrees with
|
|
63
|
+
* itself — which would confirm a first reading rather than test it.
|
|
64
|
+
*
|
|
65
|
+
* Count alone is not enough: the history is capped at
|
|
66
|
+
* MAX_HISTORY_LENGTH, so a full window slides without the count
|
|
67
|
+
* changing. The newest onset's timestamp catches that.
|
|
68
|
+
*
|
|
69
|
+
* `null` means no poll has happened yet.
|
|
70
|
+
*/
|
|
71
|
+
_lastOnsetSignature = null;
|
|
7
72
|
_spectralFluxHistory = [];
|
|
8
73
|
_previousMagnitudes = null;
|
|
9
74
|
_chromaHistory = [];
|
|
75
|
+
/**
|
|
76
|
+
* Fixed onset threshold. Retained only as a floor — see
|
|
77
|
+
* `isOnset`, which decides adaptively.
|
|
78
|
+
*
|
|
79
|
+
* As an absolute threshold this was unreachable. Flux is normalized
|
|
80
|
+
* by bin count AND by 255, so 0.3 demands an average jump of +76/255
|
|
81
|
+
* across EVERY bin. A realistic kick transient (low bins 30->240,
|
|
82
|
+
* mids 90->120) measures 0.057 — five times under. Only a
|
|
83
|
+
* silence-to-full-scale transition fired it (#322).
|
|
84
|
+
*/
|
|
10
85
|
ONSET_THRESHOLD = 0.3;
|
|
86
|
+
/**
|
|
87
|
+
* Recent flux values, for the adaptive threshold.
|
|
88
|
+
*
|
|
89
|
+
* Picking a smaller constant would just be a different guess: what
|
|
90
|
+
* counts as a transient depends on the material, and a dense mix has
|
|
91
|
+
* a higher flux floor than a sparse one. So an onset is a local
|
|
92
|
+
* OUTLIER — median plus a multiple of the deviation — which is how
|
|
93
|
+
* onset detection is normally done and which needs no magic number
|
|
94
|
+
* tuned to one corpus.
|
|
95
|
+
*/
|
|
96
|
+
_fluxHistory = [];
|
|
97
|
+
FLUX_WINDOW = 32;
|
|
98
|
+
/**
|
|
99
|
+
* Multiplier applied to the RAW median absolute deviation, not to sigma
|
|
100
|
+
* (#414 item 1).
|
|
101
|
+
*
|
|
102
|
+
* The distinction matters because the two are not the same unit. For
|
|
103
|
+
* Gaussian noise MAD is about 0.6745 sigma, so this 2.5 is roughly
|
|
104
|
+
* **1.69 sigma**, not two and a half. The previous name and doc comment
|
|
105
|
+
* ("deviations above the median") read as sigmas and were wrong by a
|
|
106
|
+
* factor of 1.4826.
|
|
107
|
+
*
|
|
108
|
+
* The VALUE is not known to be wrong. It was arrived at empirically in
|
|
109
|
+
* #350 and the resulting onset behaviour was measured against real
|
|
110
|
+
* playback in #366, so it is tuned for this signal whatever unit it is
|
|
111
|
+
* expressed in. Do not "correct" it to 2.5 sigma (i.e. 3.71 MAD) on the
|
|
112
|
+
* strength of the arithmetic alone — that is a real sensitivity change
|
|
113
|
+
* and needs re-measuring against the #366 corpus first.
|
|
114
|
+
*/
|
|
115
|
+
ONSET_MAD_MULTIPLIER = 2.5;
|
|
116
|
+
/** Flux below this is silence, whatever the local statistics say. */
|
|
117
|
+
FLUX_NOISE_FLOOR = 0.004;
|
|
11
118
|
MAX_HISTORY_LENGTH = 100;
|
|
119
|
+
/**
|
|
120
|
+
* How long after a detected transient further frames are treated as
|
|
121
|
+
* part of it rather than as new onsets.
|
|
122
|
+
*
|
|
123
|
+
* The reasoning for the original 50ms was right and the number was
|
|
124
|
+
* too small to deliver it. "Not shattering every kick into four" is
|
|
125
|
+
* exactly the goal; 50ms did not reach it, because the AnalyserNode
|
|
126
|
+
* runs with `smoothingTimeConstant = 0.8`, which spreads one
|
|
127
|
+
* transient over three or four frames of positive flux — 60 to 100ms.
|
|
128
|
+
* A refractory shorter than the tail of the hit it is suppressing
|
|
129
|
+
* suppresses nothing, so one kick became two onsets ~60ms apart.
|
|
130
|
+
*
|
|
131
|
+
* Measured: ambient had 7 of 33 inter-onset intervals at 100ms or
|
|
132
|
+
* less, where aubio on the same audio had none, and only 59% of our
|
|
133
|
+
* onsets landed on the beat grid against aubio's 89%. Those doubles
|
|
134
|
+
* inflate the inter-onset CV, which is what `tempoFromOnsets` turns
|
|
135
|
+
* into confidence — so a correct 130.4 BPM estimate was scored below
|
|
136
|
+
* MIN_TEMPO_CONFIDENCE and thrown away (#502).
|
|
137
|
+
*
|
|
138
|
+
* Swept against captured flux from six real patterns. The knee is
|
|
139
|
+
* sharp and it is at 65ms:
|
|
140
|
+
*
|
|
141
|
+
* refractory ambient techno dnb
|
|
142
|
+
* 50-60ms 65 @ 0.12 130 @ 0.49 65 @ 0.15
|
|
143
|
+
* 65-80ms 65 @ 0.19 130 @ 0.94 65 @ 0.42
|
|
144
|
+
* 90-100ms 65 @ 0.37 130 @ 0.94 65 @ 0.50
|
|
145
|
+
* 125ms 65 @ 0.48 130 @ 0.94 65 @ 0.52
|
|
146
|
+
*
|
|
147
|
+
* Not a trade: the material that already worked improves most, since
|
|
148
|
+
* techno was carrying the same doubles.
|
|
149
|
+
*
|
|
150
|
+
* 70 rather than 65, to sit past the knee rather than on it, and
|
|
151
|
+
* rather than the 90ms that scores better still: 16th notes at
|
|
152
|
+
* 200 BPM are 75ms apart, and the original comment was right to want
|
|
153
|
+
* headroom there. 125ms takes `intelligent_dnb` from 130 to 65, which
|
|
154
|
+
* is the far edge of this making things worse.
|
|
155
|
+
*/
|
|
156
|
+
static ONSET_REFRACTORY_MS = 70;
|
|
157
|
+
/**
|
|
158
|
+
* How much a candidate's octave relatives count toward its score,
|
|
159
|
+
* relative to the candidate itself.
|
|
160
|
+
*
|
|
161
|
+
* 0.25, measured, not chosen. At 0.5 a 90 BPM shuffle (667ms beat with
|
|
162
|
+
* a 2:1 swing putting a hit at 444ms) read 135, because the swung
|
|
163
|
+
* offbeat gives a spurious relative real correlation. Cross-model
|
|
164
|
+
* review predicted that exact number before I measured it.
|
|
165
|
+
*
|
|
166
|
+
* Enough to break a near-tie the prior would otherwise decide the
|
|
167
|
+
* wrong way; not enough to let a relative carry a candidate.
|
|
168
|
+
*/
|
|
169
|
+
static FAMILY_CORROBORATION = 0.25;
|
|
170
|
+
/**
|
|
171
|
+
* Below this, `detectTempo` reports no tempo instead of a number.
|
|
172
|
+
*
|
|
173
|
+
* A BPM with confidence 0.00 is not a weak measurement, it is the
|
|
174
|
+
* tempo prior's centre with a measurement's face on. Reporting zero
|
|
175
|
+
* lets a caller tell "I could not hear a pulse" from "the pulse is
|
|
176
|
+
* 120", which is the distinction #366 turned on.
|
|
177
|
+
*/
|
|
178
|
+
static MIN_TEMPO_CONFIDENCE = 0.1;
|
|
179
|
+
/**
|
|
180
|
+
* How far two readings may differ and still count as the same answer.
|
|
181
|
+
*
|
|
182
|
+
* 2%, with a one-BPM floor applied at the comparison. Wide enough for
|
|
183
|
+
* the 83/82 wobble adjacent polls produce on identical audio, narrow
|
|
184
|
+
* enough that 130 and 86 — a 2:3 cross-rhythm, and the pair that
|
|
185
|
+
* flipped in #501 — are never mistaken for each other.
|
|
186
|
+
*/
|
|
187
|
+
static TEMPO_AGREEMENT_RATIO = 0.02;
|
|
188
|
+
/**
|
|
189
|
+
* Floor for the autocorrelation branch, on its own scale.
|
|
190
|
+
*
|
|
191
|
+
* `corr(beat)/corr(0)` is not the interval-spread number
|
|
192
|
+
* MIN_TEMPO_CONFIDENCE was calibrated against, so it does not
|
|
193
|
+
* inherit that threshold. Measured: deliberate negatives — intervals
|
|
194
|
+
* drawn flat, onsets scattered through the window — score 0.088 to
|
|
195
|
+
* 0.160, and the weakest genuine reading scores 0.303. 0.25 sits in
|
|
196
|
+
* the gap (#506).
|
|
197
|
+
*/
|
|
198
|
+
static MIN_PERIODICITY_CONFIDENCE = 0.25;
|
|
199
|
+
/**
|
|
200
|
+
* How much of the past a tempo reading may be built from.
|
|
201
|
+
*
|
|
202
|
+
* `MAX_HISTORY_LENGTH` bounds the COUNT, which is not the same thing:
|
|
203
|
+
* 100 onsets can span ten seconds of one pattern or five minutes
|
|
204
|
+
* across four. Age is what matters to a tempo.
|
|
205
|
+
*
|
|
206
|
+
* 12s rather than 6s, measured. A 6s window reads dnb correctly about
|
|
207
|
+
* half the time and then flips (174, 115, 115, 174) and drags trap
|
|
208
|
+
* down to 70 on later polls; 12s gives the same answer every poll for
|
|
209
|
+
* every pattern. The reported failure in #352 was a SPREAD, so
|
|
210
|
+
* stability is the property worth buying — even though it costs the
|
|
211
|
+
* occasional lucky-correct reading a short window produces.
|
|
212
|
+
*/
|
|
213
|
+
static MAX_ONSET_AGE_MS = 12_000;
|
|
214
|
+
/**
|
|
215
|
+
* A gap this large between the newest kept onset and the oldest new
|
|
216
|
+
* one means the audio stopped — a different pattern, a stop/play, or
|
|
217
|
+
* an agent that went away and came back.
|
|
218
|
+
*
|
|
219
|
+
* Measured, this is not theoretical: playing dnb at 174 and then house
|
|
220
|
+
* at 125, the FIRST house reading came back 174. The previous
|
|
221
|
+
* pattern's onsets were still in the history and outnumbered the new
|
|
222
|
+
* ones. Every wrong first reading in that run was the pattern before
|
|
223
|
+
* it (#366).
|
|
224
|
+
*/
|
|
225
|
+
static ONSET_CONTINUITY_GAP_MS = 1_500;
|
|
226
|
+
// Per-instance analyser config (wired through from config.audio_analysis
|
|
227
|
+
// in config.json). #195.
|
|
228
|
+
fftSize;
|
|
229
|
+
smoothing;
|
|
12
230
|
// Pitch classes for key detection
|
|
13
231
|
PITCH_CLASSES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'];
|
|
14
232
|
// Krumhansl-Schmuckler scale profiles
|
|
@@ -21,19 +239,135 @@ export class AudioAnalyzer {
|
|
|
21
239
|
mixolydian: [6.0, 2.5, 3.5, 2.5, 4.5, 4.0, 2.5, 5.0, 3.5, 2.5, 3.5, 3.0],
|
|
22
240
|
locrian: [6.0, 3.0, 2.5, 4.0, 3.0, 3.0, 3.5, 4.0, 3.0, 3.5, 4.0, 2.5]
|
|
23
241
|
};
|
|
242
|
+
/**
|
|
243
|
+
* @param options.fftSize - power of 2 in [32, 32768]. Default 1024.
|
|
244
|
+
* @param options.smoothing - value in [0, 1]. Default 0.8.
|
|
245
|
+
*
|
|
246
|
+
* Invalid values fall back to defaults with a warning rather than
|
|
247
|
+
* throwing; bad config in `config.json` shouldn't crash the server.
|
|
248
|
+
*/
|
|
249
|
+
constructor(options) {
|
|
250
|
+
const logger = new Logger();
|
|
251
|
+
if (options?.fftSize !== undefined && !isValidFftSize(options.fftSize)) {
|
|
252
|
+
logger.warn(`AudioAnalyzer: invalid fftSize ${String(options.fftSize)} — must be a power of 2 in [${MIN_FFT_SIZE}, ${MAX_FFT_SIZE}]. Falling back to ${DEFAULT_FFT_SIZE}.`);
|
|
253
|
+
this.fftSize = DEFAULT_FFT_SIZE;
|
|
254
|
+
}
|
|
255
|
+
else {
|
|
256
|
+
this.fftSize = options?.fftSize ?? DEFAULT_FFT_SIZE;
|
|
257
|
+
}
|
|
258
|
+
if (options?.smoothing !== undefined &&
|
|
259
|
+
(typeof options.smoothing !== 'number' ||
|
|
260
|
+
Number.isNaN(options.smoothing) ||
|
|
261
|
+
options.smoothing < 0 ||
|
|
262
|
+
options.smoothing > 1)) {
|
|
263
|
+
logger.warn(`AudioAnalyzer: invalid smoothing ${String(options.smoothing)} — must be in [0, 1]. Falling back to ${DEFAULT_SMOOTHING}.`);
|
|
264
|
+
this.smoothing = DEFAULT_SMOOTHING;
|
|
265
|
+
}
|
|
266
|
+
else {
|
|
267
|
+
this.smoothing = options?.smoothing ?? DEFAULT_SMOOTHING;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
/** Configured FFT size (read-only). Test/debug helper. */
|
|
271
|
+
getFftSize() {
|
|
272
|
+
return this.fftSize;
|
|
273
|
+
}
|
|
274
|
+
/** Configured smoothing constant (read-only). Test/debug helper. */
|
|
275
|
+
getSmoothing() {
|
|
276
|
+
return this.smoothing;
|
|
277
|
+
}
|
|
24
278
|
/**
|
|
25
279
|
* Injects audio analysis code into the Strudel page
|
|
26
280
|
* @param page - Playwright page instance to inject into
|
|
27
281
|
*/
|
|
28
282
|
async inject(page) {
|
|
283
|
+
const cfg = { fftSize: this.fftSize, smoothing: this.smoothing };
|
|
29
284
|
/* istanbul ignore next -- browser-injected IIFE, covered by integration tests */
|
|
30
|
-
await page.evaluate(() => {
|
|
285
|
+
await page.evaluate(/* istanbul ignore next */ (cfg) => {
|
|
286
|
+
// Stop the previous collector before replacing the object it
|
|
287
|
+
// belongs to.
|
|
288
|
+
//
|
|
289
|
+
// This assignment overwrites `strudelAudioAnalyzer` outright, and
|
|
290
|
+
// the old object's 20ms interval does not stop just because
|
|
291
|
+
// nothing points at it any more. It kept sampling into an
|
|
292
|
+
// orphaned array and held the old AnalyserNode — and through it
|
|
293
|
+
// the old AudioContext — reachable for the life of the page.
|
|
294
|
+
// Measured in real Chromium: 16 samples in 300ms after a second
|
|
295
|
+
// inject (#479).
|
|
296
|
+
//
|
|
297
|
+
// This is also `stopFluxTracking`'s first production caller. It
|
|
298
|
+
// was written in #322 and never invoked, so the collector had no
|
|
299
|
+
// way to stop at all.
|
|
300
|
+
window.strudelAudioAnalyzer?.stopFluxTracking?.();
|
|
31
301
|
window.strudelAudioAnalyzer = {
|
|
32
302
|
analyser: null,
|
|
33
303
|
dataArray: null,
|
|
34
304
|
isConnected: false,
|
|
35
305
|
lastAnalysis: null,
|
|
36
306
|
lastAnalysisTime: 0,
|
|
307
|
+
/**
|
|
308
|
+
* Real hardware sample rate, read from the AudioContext.
|
|
309
|
+
*
|
|
310
|
+
* 44100 was assumed throughout, and 22050 was hardcoded as the
|
|
311
|
+
* Nyquist. The AudioContext rate follows the hardware and is
|
|
312
|
+
* commonly 48000 (typical on Linux/Chrome), where every reported
|
|
313
|
+
* frequency came out 8.8% low — 1.47 semitones, enough to label
|
|
314
|
+
* a real C as B or A# (#321/#323).
|
|
315
|
+
*/
|
|
316
|
+
sampleRate: 44100,
|
|
317
|
+
/**
|
|
318
|
+
* Rolling flux samples, timestamped, collected continuously.
|
|
319
|
+
*
|
|
320
|
+
* detectTempo used to sample the spectrum ONCE per call and push
|
|
321
|
+
* at most one onset, and four are needed before a BPM is
|
|
322
|
+
* reported — so the inter-onset intervals were the gaps between
|
|
323
|
+
* TOOL CALLS and the answer was a function of how often the
|
|
324
|
+
* agent polled (#322).
|
|
325
|
+
*
|
|
326
|
+
* The page records raw flux here; the ONSET DECISION stays on
|
|
327
|
+
* the server, so the adaptive detector is not duplicated into a
|
|
328
|
+
* copy that would drift.
|
|
329
|
+
*/
|
|
330
|
+
fluxSamples: [],
|
|
331
|
+
fluxTimer: null,
|
|
332
|
+
previousMagnitudes: null,
|
|
333
|
+
MAX_FLUX_SAMPLES: 512,
|
|
334
|
+
/** Starts continuous flux sampling. Idempotent. */
|
|
335
|
+
startFluxTracking(intervalMs) {
|
|
336
|
+
if (this.fluxTimer !== null)
|
|
337
|
+
return;
|
|
338
|
+
this.fluxTimer = setInterval(() => {
|
|
339
|
+
if (!this.analyser || !this.dataArray)
|
|
340
|
+
return;
|
|
341
|
+
this.analyser.getByteFrequencyData(this.dataArray);
|
|
342
|
+
const current = Array.from(this.dataArray);
|
|
343
|
+
if (this.previousMagnitudes) {
|
|
344
|
+
let flux = 0;
|
|
345
|
+
for (let i = 0; i < current.length; i++) {
|
|
346
|
+
const diff = current[i] - this.previousMagnitudes[i];
|
|
347
|
+
if (diff > 0)
|
|
348
|
+
flux += diff;
|
|
349
|
+
}
|
|
350
|
+
this.fluxSamples.push({
|
|
351
|
+
t: Date.now(),
|
|
352
|
+
flux: flux / current.length / 255,
|
|
353
|
+
});
|
|
354
|
+
if (this.fluxSamples.length > this.MAX_FLUX_SAMPLES)
|
|
355
|
+
this.fluxSamples.shift();
|
|
356
|
+
}
|
|
357
|
+
this.previousMagnitudes = current;
|
|
358
|
+
}, intervalMs);
|
|
359
|
+
},
|
|
360
|
+
stopFluxTracking() {
|
|
361
|
+
if (this.fluxTimer !== null) {
|
|
362
|
+
clearInterval(this.fluxTimer);
|
|
363
|
+
this.fluxTimer = null;
|
|
364
|
+
}
|
|
365
|
+
},
|
|
366
|
+
takeFluxSamples() {
|
|
367
|
+
const samples = this.fluxSamples;
|
|
368
|
+
this.fluxSamples = [];
|
|
369
|
+
return samples;
|
|
370
|
+
},
|
|
37
371
|
connect() {
|
|
38
372
|
const originalGainConnect = GainNode.prototype.connect;
|
|
39
373
|
let intercepted = false;
|
|
@@ -41,14 +375,18 @@ export class AudioAnalyzer {
|
|
|
41
375
|
if (!intercepted && args[0] && args[0].context) {
|
|
42
376
|
intercepted = true;
|
|
43
377
|
const ctx = args[0].context;
|
|
378
|
+
window.strudelAudioAnalyzer.sampleRate = ctx.sampleRate;
|
|
44
379
|
window.strudelAudioAnalyzer.analyser = ctx.createAnalyser();
|
|
45
|
-
//
|
|
46
|
-
window.strudelAudioAnalyzer.analyser.fftSize =
|
|
47
|
-
window.strudelAudioAnalyzer.analyser.smoothingTimeConstant =
|
|
380
|
+
// Configurable via config.audio_analysis in config.json (#195).
|
|
381
|
+
window.strudelAudioAnalyzer.analyser.fftSize = cfg.fftSize;
|
|
382
|
+
window.strudelAudioAnalyzer.analyser.smoothingTimeConstant = cfg.smoothing;
|
|
48
383
|
window.strudelAudioAnalyzer.dataArray = new Uint8Array(window.strudelAudioAnalyzer.analyser.frequencyBinCount);
|
|
49
384
|
const result = originalGainConnect.apply(this, args);
|
|
50
385
|
originalGainConnect.call(this, window.strudelAudioAnalyzer.analyser);
|
|
51
386
|
window.strudelAudioAnalyzer.isConnected = true;
|
|
387
|
+
// Sample every 20ms — fine enough to resolve 16ths at
|
|
388
|
+
// 200 BPM (75ms apart) without flooding the buffer.
|
|
389
|
+
window.strudelAudioAnalyzer.startFluxTracking(20);
|
|
52
390
|
return result;
|
|
53
391
|
}
|
|
54
392
|
return originalGainConnect.apply(this, args);
|
|
@@ -70,6 +408,17 @@ export class AudioAnalyzer {
|
|
|
70
408
|
// Optimized analysis using typed array operations
|
|
71
409
|
const dataArray = this.dataArray;
|
|
72
410
|
const length = dataArray.length;
|
|
411
|
+
// Frequency-band boundaries scaled to the actual `length`.
|
|
412
|
+
// Reference values are bin indices that worked at fftSize=1024
|
|
413
|
+
// (length=512): bass<4, lowMid<16, mid<64, highMid<128,
|
|
414
|
+
// treble<256. Scaling by `length / 512` keeps each band over
|
|
415
|
+
// the same Hz range when fftSize changes (#195).
|
|
416
|
+
const scale = length / 512;
|
|
417
|
+
const bassEnd = Math.max(1, Math.round(4 * scale));
|
|
418
|
+
const lowMidEnd = Math.max(bassEnd + 1, Math.round(16 * scale));
|
|
419
|
+
const midEnd = Math.max(lowMidEnd + 1, Math.round(64 * scale));
|
|
420
|
+
const highMidEnd = Math.max(midEnd + 1, Math.round(128 * scale));
|
|
421
|
+
const trebleEnd = Math.max(highMidEnd + 1, Math.round(256 * scale));
|
|
73
422
|
// Single-pass computation for better performance
|
|
74
423
|
let sum = 0;
|
|
75
424
|
let peak = 0;
|
|
@@ -85,26 +434,37 @@ export class AudioAnalyzer {
|
|
|
85
434
|
peak = value;
|
|
86
435
|
peakIndex = i;
|
|
87
436
|
}
|
|
88
|
-
// Frequency
|
|
89
|
-
|
|
437
|
+
// Frequency-band boundaries are derived from the bin indices
|
|
438
|
+
// that worked well at fftSize=1024 (length=512): 4, 16, 64,
|
|
439
|
+
// 128, 256. Scaling by `length / 512` preserves the Hz-range
|
|
440
|
+
// each band covers when fftSize changes (#195).
|
|
441
|
+
if (i < bassEnd)
|
|
90
442
|
bassSum += value;
|
|
91
|
-
else if (i <
|
|
443
|
+
else if (i < lowMidEnd)
|
|
92
444
|
lowMidSum += value;
|
|
93
|
-
else if (i <
|
|
445
|
+
else if (i < midEnd)
|
|
94
446
|
midSum += value;
|
|
95
|
-
else if (i <
|
|
447
|
+
else if (i < highMidEnd)
|
|
96
448
|
highMidSum += value;
|
|
97
|
-
else if (i <
|
|
449
|
+
else if (i < trebleEnd)
|
|
98
450
|
trebleSum += value;
|
|
99
451
|
}
|
|
100
452
|
const average = sum / length;
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
const
|
|
107
|
-
const
|
|
453
|
+
// Hz per bin. `weightedSum` accumulates `i * value`, so the
|
|
454
|
+
// raw centroid is a BIN INDEX; comparing it against Hz-scale
|
|
455
|
+
// thresholds made "bright" need bin 500 of 512, i.e. 21.5 kHz
|
|
456
|
+
// out of a 22 kHz maximum — unreachable — and made the answer
|
|
457
|
+
// depend on fftSize (#323).
|
|
458
|
+
const nyquist = this.sampleRate / 2;
|
|
459
|
+
const hzPerBin = nyquist / length;
|
|
460
|
+
const centroidBin = sum > 0 ? weightedSum / sum : 0;
|
|
461
|
+
const centroid = centroidBin * hzPerBin;
|
|
462
|
+
const peakFreq = peakIndex * hzPerBin;
|
|
463
|
+
const bass = bassSum / Math.max(1, bassEnd);
|
|
464
|
+
const lowMid = lowMidSum / Math.max(1, lowMidEnd - bassEnd);
|
|
465
|
+
const mid = midSum / Math.max(1, midEnd - lowMidEnd);
|
|
466
|
+
const highMid = highMidSum / Math.max(1, highMidEnd - midEnd);
|
|
467
|
+
const treble = trebleSum / Math.max(1, trebleEnd - highMidEnd);
|
|
108
468
|
const result = {
|
|
109
469
|
connected: true,
|
|
110
470
|
timestamp: now,
|
|
@@ -121,7 +481,10 @@ export class AudioAnalyzer {
|
|
|
121
481
|
isPlaying: average > 5,
|
|
122
482
|
isSilent: average < 1,
|
|
123
483
|
bassToTrebleRatio: treble > 0 ? (bass / treble).toFixed(2) : 'N/A',
|
|
124
|
-
|
|
484
|
+
// Hz now, and reachable. Roughly: mostly-bass content
|
|
485
|
+
// sits under 1.5 kHz, a full mix lands 1.5-4 kHz, and
|
|
486
|
+
// hat/cymbal-dominated material runs above.
|
|
487
|
+
brightness: centroid > 4000 ? 'bright' : centroid > 1500 ? 'balanced' : 'dark'
|
|
125
488
|
}
|
|
126
489
|
};
|
|
127
490
|
// Cache result
|
|
@@ -131,7 +494,7 @@ export class AudioAnalyzer {
|
|
|
131
494
|
}
|
|
132
495
|
};
|
|
133
496
|
window.strudelAudioAnalyzer.connect();
|
|
134
|
-
});
|
|
497
|
+
}, cfg);
|
|
135
498
|
}
|
|
136
499
|
/**
|
|
137
500
|
* Retrieves audio analysis data from the page
|
|
@@ -185,6 +548,217 @@ export class AudioAnalyzer {
|
|
|
185
548
|
/**
|
|
186
549
|
* Calculate spectral flux (rate of change in frequency spectrum)
|
|
187
550
|
*/
|
|
551
|
+
/**
|
|
552
|
+
* Whether a flux value is an onset, judged against recent history.
|
|
553
|
+
*
|
|
554
|
+
* @param flux - Current spectral flux, 0-1
|
|
555
|
+
* @returns True if this is a local outlier and above the noise floor
|
|
556
|
+
* @example
|
|
557
|
+
* // A steady 0.05 background with a 0.2 spike: only the spike fires.
|
|
558
|
+
*/
|
|
559
|
+
/**
|
|
560
|
+
* Turns a timestamped flux series into onset timestamps.
|
|
561
|
+
*
|
|
562
|
+
* The page samples flux continuously; this applies the same adaptive
|
|
563
|
+
* decision to the whole series at once. Keeping the decision here
|
|
564
|
+
* rather than in the injected script means the detector is not
|
|
565
|
+
* duplicated into a browser copy that would drift from this one — the
|
|
566
|
+
* failure mode #341 found in the style resource (#322).
|
|
567
|
+
*
|
|
568
|
+
* Returns the flux value alongside the timestamp, because a kick and a
|
|
569
|
+
* hi-hat are not equally good evidence of where the beat is, and the
|
|
570
|
+
* autocorrelation needs to be able to tell them apart (#352).
|
|
571
|
+
*
|
|
572
|
+
* @param samples - Timestamped flux values, oldest first
|
|
573
|
+
* @returns Each detected onset, with the flux that triggered it
|
|
574
|
+
* @example
|
|
575
|
+
* analyzer.onsetsFromFlux([{ t: 0, flux: 0.01 }, { t: 20, flux: 0.4 }]);
|
|
576
|
+
*/
|
|
577
|
+
onsetsFromFlux(samples, continueStream = false) {
|
|
578
|
+
return AudioAnalyzer.collapseToPeaks(this.onsetCandidatesFromFlux(samples, continueStream));
|
|
579
|
+
}
|
|
580
|
+
/**
|
|
581
|
+
* Every frame that crossed the threshold, before collapsing.
|
|
582
|
+
*
|
|
583
|
+
* The buffered path needs these rather than the collapsed peaks,
|
|
584
|
+
* because collapsing per buffer and then collapsing the boundary
|
|
585
|
+
* again is not the same operation as collapsing the whole stream
|
|
586
|
+
* once. The first candidates of a buffer get merged among themselves
|
|
587
|
+
* before they can be compared with what the previous buffer kept, and
|
|
588
|
+
* a frame the whole-series pass would have chosen as the peak can be
|
|
589
|
+
* discarded before the boundary merge ever sees it.
|
|
590
|
+
*
|
|
591
|
+
* That was the residual left open in #370 — one onset in 45 at an
|
|
592
|
+
* adversarial 1745ms boundary — and it grew to two as soon as
|
|
593
|
+
* ONSET_REFRACTORY_MS was widened to cover the smoothing tail (#502),
|
|
594
|
+
* because a wider window puts more frames in each run.
|
|
595
|
+
*
|
|
596
|
+
* @param samples - Timestamped flux values, oldest first
|
|
597
|
+
* @param continueStream - True when this buffer continues one already in progress
|
|
598
|
+
* @returns Every frame judged an onset, uncollapsed, oldest first
|
|
599
|
+
*/
|
|
600
|
+
onsetCandidatesFromFlux(samples, continueStream = false) {
|
|
601
|
+
// Resetting between buffers of ONE performance makes every boundary
|
|
602
|
+
// a discontinuity: the first samples of each buffer fall back to the
|
|
603
|
+
// fixed threshold instead of the adaptive one the stream had already
|
|
604
|
+
// learned. Detecting the same audio whole and in 5s pieces then
|
|
605
|
+
// gives different onsets, which is #370. A genuinely new capture
|
|
606
|
+
// still resets — via resetTempoHistory, on pattern change and stop.
|
|
607
|
+
if (!continueStream)
|
|
608
|
+
this.resetOnsetDetection();
|
|
609
|
+
const candidates = [];
|
|
610
|
+
for (const sample of samples) {
|
|
611
|
+
if (this.isOnset(sample.flux))
|
|
612
|
+
candidates.push({ t: sample.t, strength: sample.flux });
|
|
613
|
+
}
|
|
614
|
+
return candidates;
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Collapses a run of adjacent detections into the one transient it is.
|
|
618
|
+
*
|
|
619
|
+
* A drum hit is not instantaneous. Its flux stays above the adaptive
|
|
620
|
+
* threshold for several consecutive 20ms frames, and without this every
|
|
621
|
+
* frame of it became a separate "onset" — measured on real playback,
|
|
622
|
+
* the median inter-onset interval was 20ms, exactly one sampling step,
|
|
623
|
+
* for dnb, techno and house alike. An envelope of solid blocks has no
|
|
624
|
+
* periodicity to find, the autocorrelation came out flat, and the
|
|
625
|
+
* tempo prior answered on its own: 120 BPM for every pattern, at
|
|
626
|
+
* confidence 0.00 (#366).
|
|
627
|
+
*
|
|
628
|
+
* Keeps the loudest frame of each run, which is also the one whose
|
|
629
|
+
* strength should weight the correlation.
|
|
630
|
+
*
|
|
631
|
+
* @param candidates - Every frame that crossed the threshold, in order
|
|
632
|
+
* @param refractoryMs - Frames closer than this belong to one transient
|
|
633
|
+
* @returns One observation per transient
|
|
634
|
+
*/
|
|
635
|
+
static collapseToPeaks(candidates, refractoryMs = AudioAnalyzer.ONSET_REFRACTORY_MS) {
|
|
636
|
+
const peaks = [];
|
|
637
|
+
for (const candidate of candidates) {
|
|
638
|
+
const previous = peaks[peaks.length - 1];
|
|
639
|
+
if (previous !== undefined && candidate.t - previous.t < refractoryMs) {
|
|
640
|
+
// Same transient. Keep the louder frame, and its time with it —
|
|
641
|
+
// the peak is a better estimate of when the hit landed than the
|
|
642
|
+
// first frame that happened to cross.
|
|
643
|
+
if (candidate.strength > previous.strength)
|
|
644
|
+
peaks[peaks.length - 1] = candidate;
|
|
645
|
+
continue;
|
|
646
|
+
}
|
|
647
|
+
peaks.push(candidate);
|
|
648
|
+
}
|
|
649
|
+
return peaks;
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* Folds newly detected onsets into the history, dropping what no
|
|
653
|
+
* longer belongs to the same performance.
|
|
654
|
+
*
|
|
655
|
+
* @param detected - Onsets from the latest buffer, oldest first
|
|
656
|
+
*/
|
|
657
|
+
mergeOnsetHistory(detected) {
|
|
658
|
+
const newest = this._onsetHistory[this._onsetHistory.length - 1];
|
|
659
|
+
const first = detected[0];
|
|
660
|
+
// A silence longer than a bar is a different performance. Keeping
|
|
661
|
+
// the old onsets across it does not add data, it mixes two tempos
|
|
662
|
+
// and puts a multi-second interval between them.
|
|
663
|
+
if (newest !== undefined && first !== undefined &&
|
|
664
|
+
first.t - newest.t > AudioAnalyzer.ONSET_CONTINUITY_GAP_MS) {
|
|
665
|
+
this._onsetHistory = [];
|
|
666
|
+
}
|
|
667
|
+
// A transient that straddles a buffer boundary arrives as the tail of
|
|
668
|
+
// one detection and the head of the next. `collapseToPeaks` merges
|
|
669
|
+
// within a single call and cannot see across one, so one kick counts
|
|
670
|
+
// twice, every buffer.
|
|
671
|
+
//
|
|
672
|
+
// Collapsing `[last kept onset, ...incoming]` rather than special-
|
|
673
|
+
// casing the first element is what makes this EXACTLY the same
|
|
674
|
+
// operation the whole-series path performs. A hand-rolled boundary
|
|
675
|
+
// merge got it nearly right and dropped one onset in 45: it decided
|
|
676
|
+
// the first incoming onset, then pushed the rest unchanged, so an
|
|
677
|
+
// onset that became adjacent to the retained peak was never
|
|
678
|
+
// reconsidered.
|
|
679
|
+
const previous = this._onsetHistory.pop();
|
|
680
|
+
const joined = previous === undefined ? detected : [previous, ...detected];
|
|
681
|
+
this._onsetHistory.push(...AudioAnalyzer.collapseToPeaks(joined));
|
|
682
|
+
const latest = this._onsetHistory[this._onsetHistory.length - 1];
|
|
683
|
+
if (latest !== undefined) {
|
|
684
|
+
const cutoff = latest.t - AudioAnalyzer.MAX_ONSET_AGE_MS;
|
|
685
|
+
// Bound by age first: a count bound alone let a reading span
|
|
686
|
+
// minutes of unrelated audio.
|
|
687
|
+
while (this._onsetHistory.length > 0 && this._onsetHistory[0].t < cutoff) {
|
|
688
|
+
this._onsetHistory.shift();
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
while (this._onsetHistory.length > this.MAX_HISTORY_LENGTH) {
|
|
692
|
+
this._onsetHistory.shift();
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
/**
|
|
696
|
+
* Forgets every onset collected so far.
|
|
697
|
+
*
|
|
698
|
+
* Called when the pattern changes or the transport stops: the next
|
|
699
|
+
* reading must describe the next performance, not average it with the
|
|
700
|
+
* last one.
|
|
701
|
+
*/
|
|
702
|
+
async resetTempoHistory(page) {
|
|
703
|
+
this._onsetHistory = [];
|
|
704
|
+
// The stable reading belongs to the pattern that produced it (#374).
|
|
705
|
+
this._confirmedTempo = null;
|
|
706
|
+
this._unconfirmedBpm = null;
|
|
707
|
+
this._lastOnsetSignature = null;
|
|
708
|
+
this.resetOnsetDetection();
|
|
709
|
+
// The PAGE keeps its own flux buffer, and clearing only the Node
|
|
710
|
+
// side left up to ten seconds of the previous pattern's samples
|
|
711
|
+
// waiting to be drained into the fresh history on the next call.
|
|
712
|
+
// Cross-model review found this: the reset looked complete because
|
|
713
|
+
// everything it could see was cleared, and the contamination lived
|
|
714
|
+
// one process away (#374).
|
|
715
|
+
//
|
|
716
|
+
// Draining and discarding reuses the existing accessor rather than
|
|
717
|
+
// adding a second way to empty the same array.
|
|
718
|
+
if (page !== undefined) {
|
|
719
|
+
try {
|
|
720
|
+
await page.evaluate(/* istanbul ignore next */ () => {
|
|
721
|
+
window.strudelAudioAnalyzer?.takeFluxSamples?.();
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
catch {
|
|
725
|
+
// The page may be gone — a reset that cannot reach it has
|
|
726
|
+
// nothing to contaminate anyway.
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
isOnset(flux) {
|
|
731
|
+
const history = this._fluxHistory;
|
|
732
|
+
history.push(flux);
|
|
733
|
+
if (history.length > this.FLUX_WINDOW)
|
|
734
|
+
history.shift();
|
|
735
|
+
// Below the noise floor is silence regardless of what the window
|
|
736
|
+
// says — otherwise a completely quiet passage generates onsets from
|
|
737
|
+
// its own rounding.
|
|
738
|
+
if (flux < this.FLUX_NOISE_FLOOR)
|
|
739
|
+
return false;
|
|
740
|
+
// Not enough history yet: fall back to the fixed threshold rather
|
|
741
|
+
// than firing on the first sample of anything.
|
|
742
|
+
if (history.length < 8)
|
|
743
|
+
return flux > this.ONSET_THRESHOLD;
|
|
744
|
+
const sorted = [...history].sort((a, b) => a - b);
|
|
745
|
+
const median = sorted[Math.floor(sorted.length / 2)];
|
|
746
|
+
// Median absolute deviation — robust to the very spikes we are
|
|
747
|
+
// looking for, unlike a standard deviation.
|
|
748
|
+
const deviations = sorted.map(v => Math.abs(v - median)).sort((a, b) => a - b);
|
|
749
|
+
const mad = deviations[Math.floor(deviations.length / 2)];
|
|
750
|
+
// A perfectly steady signal has mad 0; require a real rise over the
|
|
751
|
+
// median rather than dividing by zero.
|
|
752
|
+
const threshold = mad > 0
|
|
753
|
+
? median + this.ONSET_MAD_MULTIPLIER * mad
|
|
754
|
+
: median * 1.5 + this.FLUX_NOISE_FLOOR;
|
|
755
|
+
return flux > threshold;
|
|
756
|
+
}
|
|
757
|
+
/** Clears the flux window, so a new capture does not inherit one. */
|
|
758
|
+
resetOnsetDetection() {
|
|
759
|
+
this._fluxHistory = [];
|
|
760
|
+
this._previousMagnitudes = null;
|
|
761
|
+
}
|
|
188
762
|
calculateSpectralFlux(currentMagnitudes) {
|
|
189
763
|
if (!this._previousMagnitudes) {
|
|
190
764
|
this._previousMagnitudes = Array.from(currentMagnitudes);
|
|
@@ -216,16 +790,77 @@ export class AudioAnalyzer {
|
|
|
216
790
|
/**
|
|
217
791
|
* Extract chroma features (12-dimensional pitch class profile) from FFT data
|
|
218
792
|
*/
|
|
219
|
-
|
|
793
|
+
/**
|
|
794
|
+
* Mean in-range bin magnitude below which a spectrum carries no key.
|
|
795
|
+
*
|
|
796
|
+
* On the 0-255 byte scale `getByteFrequencyData` produces, so this is
|
|
797
|
+
* roughly one part in 255 of full scale, averaged over the 20-4000 Hz
|
|
798
|
+
* bins — far below anything audible.
|
|
799
|
+
*
|
|
800
|
+
* Measured against live playback (#412), mean in-range magnitude:
|
|
801
|
+
*
|
|
802
|
+
* stopped 0
|
|
803
|
+
* note("<c3 e3 g3>").s("sine") 14.7 <- quietest content tried
|
|
804
|
+
* s("bd*4") 18.9
|
|
805
|
+
* s("bd*4, ~ cp ~ cp, hh*8") 151.1
|
|
806
|
+
*
|
|
807
|
+
* Silence reads exactly 0 and the quietest musical content read 14.7,
|
|
808
|
+
* so 1 sits in an empty gap an order of magnitude wide on either side.
|
|
809
|
+
*
|
|
810
|
+
* Three patterns on one machine in one browser, which is not a survey
|
|
811
|
+
* — but the gap is wide enough that the exact figure does not matter,
|
|
812
|
+
* and the guard's job is to separate "nothing" from "something".
|
|
813
|
+
*/
|
|
814
|
+
static CHROMA_NOISE_FLOOR = 1;
|
|
815
|
+
/**
|
|
816
|
+
* Mean magnitude of the bins `extractChroma` will actually use.
|
|
817
|
+
*
|
|
818
|
+
* Read before normalization, which is the whole point: the chroma
|
|
819
|
+
* vector sums to 1 for a single non-zero bin exactly as it does for a
|
|
820
|
+
* full mix, so nothing downstream of `extractChroma` can tell silence
|
|
821
|
+
* from music.
|
|
822
|
+
*
|
|
823
|
+
* @param fftData - Byte frequency data
|
|
824
|
+
* @param sampleRate - The rate the AudioContext reported
|
|
825
|
+
* @returns Mean magnitude over the 20-4000 Hz bins, 0 if none
|
|
826
|
+
*/
|
|
827
|
+
meanInRangeMagnitude(fftData, sampleRate) {
|
|
828
|
+
let sum = 0;
|
|
829
|
+
let count = 0;
|
|
830
|
+
for (let i = 0; i < fftData.length; i++) {
|
|
831
|
+
const freq = (i / fftData.length) * (sampleRate / 2);
|
|
832
|
+
if (freq < 20 || freq > 4000)
|
|
833
|
+
continue;
|
|
834
|
+
sum += fftData[i];
|
|
835
|
+
count++;
|
|
836
|
+
}
|
|
837
|
+
return count > 0 ? sum / count : 0;
|
|
838
|
+
}
|
|
839
|
+
extractChroma(fftData, sampleRate = 44100) {
|
|
220
840
|
const chroma = new Array(12).fill(0);
|
|
841
|
+
// How many FFT bins landed in each pitch class. Linear-frequency
|
|
842
|
+
// bins do not divide evenly among twelve logarithmic pitch classes:
|
|
843
|
+
// at fftSize=1024 only 92 of 512 bins fall inside 20-4000 Hz, and
|
|
844
|
+
// they are distributed 4 to 12 per class — A gets 12, C# and D# get
|
|
845
|
+
// 4. Summing raw magnitudes therefore made the classes with more
|
|
846
|
+
// bins louder no matter what the audio was, and flat white noise
|
|
847
|
+
// detected as F major with confidence 0.849 (#321).
|
|
848
|
+
const binCounts = new Array(12).fill(0);
|
|
221
849
|
const fftSize = fftData.length;
|
|
222
|
-
const sampleRate = 44100;
|
|
223
850
|
for (let i = 0; i < fftSize; i++) {
|
|
224
851
|
const freq = (i / fftSize) * (sampleRate / 2);
|
|
225
852
|
if (freq < 20 || freq > 4000)
|
|
226
853
|
continue; // Focus on musical range
|
|
227
854
|
const pitchClass = this.frequencyToPitchClass(freq);
|
|
228
855
|
chroma[pitchClass] += fftData[i];
|
|
856
|
+
binCounts[pitchClass]++;
|
|
857
|
+
}
|
|
858
|
+
// Mean magnitude per class, not total. This is the whole fix: a
|
|
859
|
+
// class with 12 bins is now compared on equal terms with one that
|
|
860
|
+
// has 4.
|
|
861
|
+
for (let pc = 0; pc < 12; pc++) {
|
|
862
|
+
if (binCounts[pc] > 0)
|
|
863
|
+
chroma[pc] /= binCounts[pc];
|
|
229
864
|
}
|
|
230
865
|
// Normalize
|
|
231
866
|
const sum = chroma.reduce((a, b) => a + b, 0);
|
|
@@ -273,17 +908,6 @@ export class AudioAnalyzer {
|
|
|
273
908
|
const magnitude = Math.sqrt(magX * magY);
|
|
274
909
|
return magnitude === 0 ? 0 : dotProduct / magnitude;
|
|
275
910
|
}
|
|
276
|
-
/**
|
|
277
|
-
* Rotate a profile array by a given number of steps
|
|
278
|
-
* For tonic N, rotate the profile so that position N gets the tonic weight
|
|
279
|
-
*/
|
|
280
|
-
rotateProfile(profile, steps) {
|
|
281
|
-
const rotated = new Array(12);
|
|
282
|
-
for (let i = 0; i < 12; i++) {
|
|
283
|
-
rotated[i] = profile[(i - steps + 12) % 12];
|
|
284
|
-
}
|
|
285
|
-
return rotated;
|
|
286
|
-
}
|
|
287
911
|
/**
|
|
288
912
|
* Calculate intervals between consecutive values
|
|
289
913
|
*/
|
|
@@ -346,8 +970,8 @@ export class AudioAnalyzer {
|
|
|
346
970
|
}
|
|
347
971
|
const fftData = new Uint8Array(analyzer.dataArray);
|
|
348
972
|
const flux = this.calculateSpectralFlux(fftData);
|
|
349
|
-
if (
|
|
350
|
-
this._onsetHistory.push(Date.now());
|
|
973
|
+
if (this.isOnset(flux)) {
|
|
974
|
+
this._onsetHistory.push({ t: Date.now(), strength: flux });
|
|
351
975
|
if (this._onsetHistory.length > this.MAX_HISTORY_LENGTH) {
|
|
352
976
|
this._onsetHistory.shift();
|
|
353
977
|
}
|
|
@@ -356,47 +980,540 @@ export class AudioAnalyzer {
|
|
|
356
980
|
}
|
|
357
981
|
}
|
|
358
982
|
else {
|
|
359
|
-
//
|
|
983
|
+
// Prefer the page's continuous buffer.
|
|
984
|
+
//
|
|
985
|
+
// Sampling once per call gave at most one onset per tool call, so
|
|
986
|
+
// the intervals measured were the gaps between CALLS. The page
|
|
987
|
+
// samples every 20ms; this reads what it collected and applies the
|
|
988
|
+
// onset decision to the whole series (#322).
|
|
989
|
+
const buffered = await this.takeBufferedFlux(page);
|
|
990
|
+
if (buffered.length >= 8) {
|
|
991
|
+
// Continuing the same stream: the previous buffer's threshold
|
|
992
|
+
// state is what makes this buffer's first samples comparable.
|
|
993
|
+
// Candidates, not peaks: `mergeOnsetHistory` collapses them
|
|
994
|
+
// together with what the previous buffer kept, so the collapse
|
|
995
|
+
// happens exactly once over the concatenated stream — which is
|
|
996
|
+
// what makes the buffered path equal the whole-series one
|
|
997
|
+
// (#370, #502).
|
|
998
|
+
const detected = this.onsetCandidatesFromFlux(buffered, this._onsetHistory.length > 0);
|
|
999
|
+
this.mergeOnsetHistory(detected);
|
|
1000
|
+
const history = [...this._onsetHistory];
|
|
1001
|
+
return this.stabilise(this.tempoFromOnsets(history), history);
|
|
1002
|
+
}
|
|
1003
|
+
// Fall back to the single-sample path when the page has no buffer
|
|
1004
|
+
// — an older injected script, or a capture that just started.
|
|
360
1005
|
if (!analyzer.dataArray) {
|
|
361
1006
|
throw new Error('Invalid audio data');
|
|
362
1007
|
}
|
|
363
1008
|
const fftData = new Uint8Array(analyzer.dataArray);
|
|
364
1009
|
const flux = this.calculateSpectralFlux(fftData);
|
|
365
|
-
if (
|
|
366
|
-
this._onsetHistory.push(Date.now());
|
|
1010
|
+
if (this.isOnset(flux)) {
|
|
1011
|
+
this._onsetHistory.push({ t: Date.now(), strength: flux });
|
|
367
1012
|
if (this._onsetHistory.length > this.MAX_HISTORY_LENGTH) {
|
|
368
1013
|
this._onsetHistory.shift();
|
|
369
1014
|
}
|
|
370
1015
|
}
|
|
371
1016
|
onsets = [...this._onsetHistory];
|
|
372
1017
|
}
|
|
1018
|
+
return this.stabilise(this.tempoFromOnsets(onsets), onsets);
|
|
1019
|
+
}
|
|
1020
|
+
/**
|
|
1021
|
+
* Whether two readings describe the same pulse.
|
|
1022
|
+
*
|
|
1023
|
+
* Not equality: consecutive readings of the same audio wobble by a
|
|
1024
|
+
* BPM, and the amen break read 83 then 82 on adjacent polls. A
|
|
1025
|
+
* proportional tolerance with a one-BPM floor treats that as the same
|
|
1026
|
+
* answer and still separates 130 from 86.
|
|
1027
|
+
*
|
|
1028
|
+
* @param a - One reading, in BPM
|
|
1029
|
+
* @param b - The other
|
|
1030
|
+
* @returns True when the two agree within tolerance
|
|
1031
|
+
*/
|
|
1032
|
+
static readingsAgree(a, b) {
|
|
1033
|
+
if (a <= 0 || b <= 0)
|
|
1034
|
+
return false;
|
|
1035
|
+
return Math.abs(a - b) <= Math.max(1, a * AudioAnalyzer.TEMPO_AGREEMENT_RATIO);
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* A fingerprint of the onsets a reading was taken from.
|
|
1039
|
+
*
|
|
1040
|
+
* @param onsets - The series the reading came from
|
|
1041
|
+
* @returns A string that changes when the window does
|
|
1042
|
+
*/
|
|
1043
|
+
static onsetSignature(onsets) {
|
|
1044
|
+
const times = onsetTimes(onsets);
|
|
1045
|
+
const newest = times.length > 0 ? times[times.length - 1] : 0;
|
|
1046
|
+
return `${String(times.length)}:${String(newest)}`;
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Holds a reading back until a second window agrees with it.
|
|
1050
|
+
*
|
|
1051
|
+
* The rule, in full:
|
|
1052
|
+
*
|
|
1053
|
+
* - A window that has not changed since the last poll cannot produce
|
|
1054
|
+
* a different answer, so its reading is final and reported at
|
|
1055
|
+
* once. This is the mock and fixture case, and the real case after
|
|
1056
|
+
* the audio stops.
|
|
1057
|
+
* - A reading that agrees with the confirmed one refreshes it. The
|
|
1058
|
+
* confirmed BPM is reported rather than the new one, so the number
|
|
1059
|
+
* does not wobble by a BPM between polls; confidence stays live,
|
|
1060
|
+
* because that is a live property of the evidence.
|
|
1061
|
+
* - A reading that agrees with the previous unconfirmed one is
|
|
1062
|
+
* promoted and reported.
|
|
1063
|
+
* - Anything else is held. If something is already confirmed it
|
|
1064
|
+
* keeps being reported, so one deviant poll cannot flip the
|
|
1065
|
+
* answer — the deviant has to be seen twice, like every other
|
|
1066
|
+
* reading. That is what stops 130, 130, 86, 130.
|
|
1067
|
+
* - `bpm: 0` is not a competing reading, it is the absence of one.
|
|
1068
|
+
* It never displaces a confirmed answer and never counts as
|
|
1069
|
+
* corroboration.
|
|
1070
|
+
*
|
|
1071
|
+
* A held reading returns `bpm: 0` with its measured confidence and
|
|
1072
|
+
* method, matching what the MIN_TEMPO_CONFIDENCE gate already does —
|
|
1073
|
+
* the caller reads `bpm === 0` as "no answer yet" either way.
|
|
1074
|
+
*
|
|
1075
|
+
* @param reading - What this poll measured
|
|
1076
|
+
* @param onsets - The onsets it measured from
|
|
1077
|
+
* @returns The reading to report
|
|
1078
|
+
*/
|
|
1079
|
+
stabilise(reading, onsets) {
|
|
1080
|
+
const signature = AudioAnalyzer.onsetSignature(onsets);
|
|
1081
|
+
const windowMoved = this._lastOnsetSignature !== signature;
|
|
1082
|
+
const firstPoll = this._lastOnsetSignature === null;
|
|
1083
|
+
this._lastOnsetSignature = signature;
|
|
1084
|
+
if (reading.bpm === 0) {
|
|
1085
|
+
// No reading to corroborate, and none to contradict. Whatever was
|
|
1086
|
+
// confirmed still stands until the pattern changes.
|
|
1087
|
+
return this._confirmedTempo === null
|
|
1088
|
+
? reading
|
|
1089
|
+
: { ...this._confirmedTempo, confidence: reading.confidence };
|
|
1090
|
+
}
|
|
1091
|
+
if (!windowMoved && !firstPoll) {
|
|
1092
|
+
// Nothing new can arrive, so waiting cannot improve this answer.
|
|
1093
|
+
this._confirmedTempo = reading;
|
|
1094
|
+
this._unconfirmedBpm = null;
|
|
1095
|
+
return reading;
|
|
1096
|
+
}
|
|
1097
|
+
if (this._confirmedTempo !== null
|
|
1098
|
+
&& AudioAnalyzer.readingsAgree(this._confirmedTempo.bpm, reading.bpm)) {
|
|
1099
|
+
this._unconfirmedBpm = null;
|
|
1100
|
+
return { ...this._confirmedTempo, confidence: reading.confidence };
|
|
1101
|
+
}
|
|
1102
|
+
if (this._unconfirmedBpm !== null
|
|
1103
|
+
&& AudioAnalyzer.readingsAgree(this._unconfirmedBpm, reading.bpm)) {
|
|
1104
|
+
this._confirmedTempo = reading;
|
|
1105
|
+
this._unconfirmedBpm = null;
|
|
1106
|
+
return reading;
|
|
1107
|
+
}
|
|
1108
|
+
this._unconfirmedBpm = reading.bpm;
|
|
1109
|
+
// `settling`, so the caller is not told to check whether audio is
|
|
1110
|
+
// playing while it is playing and a candidate is already in hand.
|
|
1111
|
+
return this._confirmedTempo === null
|
|
1112
|
+
? { ...reading, bpm: 0, alternatives: undefined, settling: true }
|
|
1113
|
+
: { ...this._confirmedTempo, confidence: reading.confidence };
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Halves or doubles a tempo until it lands in the musical range.
|
|
1117
|
+
*
|
|
1118
|
+
* Out-of-range readings used to be discarded outright, which threw
|
|
1119
|
+
* away recoverable answers: 174 BPM drum & bass with onsets on 8ths
|
|
1120
|
+
* reads as 348 and returned bpm 0, on 16ths as 696 and returned 0,
|
|
1121
|
+
* and a 70 BPM tune whose onsets land on half notes reads as 35 and
|
|
1122
|
+
* returned 0. Onset detection lands on whatever subdivision is
|
|
1123
|
+
* loudest, so this is the common case, not the edge case (#322).
|
|
1124
|
+
*
|
|
1125
|
+
* @param bpm - Raw tempo from the median inter-onset interval
|
|
1126
|
+
* @returns A tempo within [40, 200], or null if it cannot be folded there
|
|
1127
|
+
* @example
|
|
1128
|
+
* AudioAnalyzer.foldIntoTempoRange(348); // 174
|
|
1129
|
+
* AudioAnalyzer.foldIntoTempoRange(35); // 70
|
|
1130
|
+
*/
|
|
1131
|
+
static foldIntoTempoRange(bpm) {
|
|
1132
|
+
if (!Number.isFinite(bpm) || bpm <= 0)
|
|
1133
|
+
return null;
|
|
1134
|
+
const MIN = 40;
|
|
1135
|
+
const MAX = 200;
|
|
1136
|
+
let folded = bpm;
|
|
1137
|
+
// Three octaves each way. That covers every real case — onsets on
|
|
1138
|
+
// 8ths is one octave, on 16ths is two — while refusing to invent a
|
|
1139
|
+
// tempo from noise: 5000 BPM means onsets 12ms apart, and folding
|
|
1140
|
+
// that five times to a plausible-looking 156 would be a fabrication,
|
|
1141
|
+
// not a measurement.
|
|
1142
|
+
const MAX_OCTAVES = 3;
|
|
1143
|
+
for (let i = 0; i < MAX_OCTAVES && folded > MAX; i++)
|
|
1144
|
+
folded /= 2;
|
|
1145
|
+
for (let i = 0; i < MAX_OCTAVES && folded < MIN; i++)
|
|
1146
|
+
folded *= 2;
|
|
1147
|
+
return folded >= MIN && folded <= MAX ? folded : null;
|
|
1148
|
+
}
|
|
1149
|
+
// ============================================================================
|
|
1150
|
+
// KEY DETECTION
|
|
1151
|
+
// ============================================================================
|
|
1152
|
+
/**
|
|
1153
|
+
* Reads and clears the page's continuous flux buffer.
|
|
1154
|
+
*
|
|
1155
|
+
* @param page - Playwright page with the analyser injected
|
|
1156
|
+
* @returns Timestamped flux samples, oldest first
|
|
1157
|
+
*/
|
|
1158
|
+
async takeBufferedFlux(page) {
|
|
1159
|
+
try {
|
|
1160
|
+
const samples = await page.evaluate(/* istanbul ignore next */ () => {
|
|
1161
|
+
const a = window.strudelAudioAnalyzer;
|
|
1162
|
+
return typeof a?.takeFluxSamples === 'function' ? a.takeFluxSamples() : [];
|
|
1163
|
+
});
|
|
1164
|
+
return Array.isArray(samples) ? samples : [];
|
|
1165
|
+
}
|
|
1166
|
+
catch {
|
|
1167
|
+
// An older injected script, or a page that went away mid-call.
|
|
1168
|
+
// The caller falls back to single-sample detection.
|
|
1169
|
+
return [];
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
/**
|
|
1173
|
+
* Derives a tempo from onset timestamps.
|
|
1174
|
+
*
|
|
1175
|
+
* @param onsets - Onset times in milliseconds, oldest first
|
|
1176
|
+
* @returns Tempo analysis, or bpm 0 when there is not enough to go on
|
|
1177
|
+
*/
|
|
1178
|
+
/**
|
|
1179
|
+
* Finds the beat period by autocorrelating the onset series.
|
|
1180
|
+
*
|
|
1181
|
+
* A central inter-onset interval measures whatever subdivision the
|
|
1182
|
+
* onsets happen to land on, not the pulse. A 174 BPM track with hits
|
|
1183
|
+
* on 8ths has a 172ms median interval, and the fold-into-range step
|
|
1184
|
+
* then picks whichever octave it lands in first — which is why the
|
|
1185
|
+
* liquid-dnb example read 108, 117 and 190 across runs (#352).
|
|
1186
|
+
*
|
|
1187
|
+
* Autocorrelation asks a different question: at what lag does the
|
|
1188
|
+
* whole series repeat? Hits on 8ths correlate at 172ms AND at 345ms,
|
|
1189
|
+
* and a preference weight centred on 120 BPM breaks the octave tie
|
|
1190
|
+
* the way a listener does.
|
|
1191
|
+
*
|
|
1192
|
+
* @param onsets - Onset times in milliseconds, oldest first
|
|
1193
|
+
* @returns Beat period in ms, or null when no periodicity is clear
|
|
1194
|
+
* @example
|
|
1195
|
+
* // onsets every 172ms from a 174 BPM track -> ~345
|
|
1196
|
+
*/
|
|
1197
|
+
/**
|
|
1198
|
+
* How strongly a candidate's whole octave family correlates, not just
|
|
1199
|
+
* the candidate.
|
|
1200
|
+
*
|
|
1201
|
+
* Measured on a real 174 BPM reading, the raw correlation favoured 174
|
|
1202
|
+
* over 115 (.0041 against .0039) and the 120-centred prior overturned
|
|
1203
|
+
* it: 174 weighs 0.866, 115 weighs 0.998, and 13% was enough. 115 is
|
|
1204
|
+
* then a trap — it sits near the prior's centre AND its double, 230
|
|
1205
|
+
* BPM, is outside the 40-200 window, so the half-time walk has nowhere
|
|
1206
|
+
* to go and the reading can never be corrected.
|
|
1207
|
+
*
|
|
1208
|
+
* 115 is not an octave of 174 at all. It is a cross-rhythm: six
|
|
1209
|
+
* sixteenths where the beat is four.
|
|
1210
|
+
*
|
|
1211
|
+
* Families separate them on evidence rather than on where the prior's
|
|
1212
|
+
* centre was guessed. 43.5 / 87 / 174 all correlate strongly; 115's
|
|
1213
|
+
* family is 57.5 / 115 / 230, one weak and one out of range. Whether a
|
|
1214
|
+
* tempo's halves and doubles also correlate is a property of the
|
|
1215
|
+
* music.
|
|
1216
|
+
*
|
|
1217
|
+
* Members outside the reportable BPM range still count as evidence —
|
|
1218
|
+
* 230 BPM is not an answer this returns, but its correlation says
|
|
1219
|
+
* something about whether 115 is real.
|
|
1220
|
+
*
|
|
1221
|
+
* @param correlation - Autocorrelation of the onset envelope
|
|
1222
|
+
* @param lag - Candidate lag, in bins
|
|
1223
|
+
* @returns Summed correlation across the candidate's octave family
|
|
1224
|
+
*/
|
|
1225
|
+
static familyStrength(correlation, lag) {
|
|
1226
|
+
const own = lag > 0 && lag < correlation.length ? correlation[lag] : 0;
|
|
1227
|
+
let relatives = 0;
|
|
1228
|
+
for (const multiple of [0.25, 0.5, 2, 4]) {
|
|
1229
|
+
const member = Math.round(lag * multiple);
|
|
1230
|
+
if (member > 0 && member < correlation.length)
|
|
1231
|
+
relatives += correlation[member];
|
|
1232
|
+
}
|
|
1233
|
+
// The candidate leads; its family corroborates. A plain sum makes
|
|
1234
|
+
// every member of a family score alike and lets a strong relative
|
|
1235
|
+
// carry a weak candidate, which broke a pure 90 BPM train (read as
|
|
1236
|
+
// 120) and a triplet fixture.
|
|
1237
|
+
return own + AudioAnalyzer.FAMILY_CORROBORATION * relatives;
|
|
1238
|
+
}
|
|
1239
|
+
beatPeriodFromOnsets(onsets) {
|
|
1240
|
+
const times = onsetTimes(onsets);
|
|
1241
|
+
const strengths = onsetStrengths(onsets);
|
|
1242
|
+
// Autocorrelation needs a series long enough to show periodicity.
|
|
1243
|
+
// With 8 onsets over 1.2 seconds it reported 117 for a 174 BPM
|
|
1244
|
+
// train — the correlation simply has too few overlapping terms at
|
|
1245
|
+
// the lags that matter. Below this the median path is better, and
|
|
1246
|
+
// the caller falls through to it (#352).
|
|
1247
|
+
const MIN_ONSETS_FOR_AUTOCORRELATION = 12;
|
|
1248
|
+
if (times.length < MIN_ONSETS_FOR_AUTOCORRELATION)
|
|
1249
|
+
return null;
|
|
1250
|
+
// Impulse train at 10ms resolution — finer than the 20ms flux
|
|
1251
|
+
// sampling, so quantization here adds nothing.
|
|
1252
|
+
const RESOLUTION_MS = 5;
|
|
1253
|
+
const span = times[times.length - 1] - times[0];
|
|
1254
|
+
if (span <= 0)
|
|
1255
|
+
return null;
|
|
1256
|
+
const length = Math.floor(span / RESOLUTION_MS) + 1;
|
|
1257
|
+
if (length < 8)
|
|
1258
|
+
return null;
|
|
1259
|
+
// Weighted by flux, not a binary impulse train.
|
|
1260
|
+
//
|
|
1261
|
+
// A dnb pattern fires onsets on kicks, snares AND hats, and an
|
|
1262
|
+
// unweighted train says they are equally likely to be the beat. They
|
|
1263
|
+
// are not — measured on a synthetic 174 BPM mix with 16th hats, the
|
|
1264
|
+
// unweighted correlation reads 115 BPM, because the 120 BPM prior
|
|
1265
|
+
// tips a near-tie between the beat lag and a six-sixteenth lag. With
|
|
1266
|
+
// the kick contributing ~2.5x what a hat does, the tie disappears.
|
|
1267
|
+
//
|
|
1268
|
+
// Normalized so the loudest onset is 1: flux magnitudes are relative
|
|
1269
|
+
// to whatever the mix is doing, and only the ratios matter here.
|
|
1270
|
+
const loudest = strengths.length > 0 ? Math.max(...strengths) : 1;
|
|
1271
|
+
const scale = loudest > 0 ? 1 / loudest : 1;
|
|
1272
|
+
const envelope = new Array(length).fill(0);
|
|
1273
|
+
// Each onset is spread over a small triangular kernel rather than
|
|
1274
|
+
// written to one bin.
|
|
1275
|
+
//
|
|
1276
|
+
// The flux series is sampled every 20ms, so every onset time is a
|
|
1277
|
+
// multiple of 20ms — and most tempos do not divide into it. A 345ms
|
|
1278
|
+
// beat (174 BPM) lands alternately on 340 and 360, and a bare
|
|
1279
|
+
// impulse train has NO lag that matches both. Measured, tempos whose
|
|
1280
|
+
// period is an exact multiple of the sampling step read correctly
|
|
1281
|
+
// (120 from 500ms, 100 from 600ms) and tempos that are not read
|
|
1282
|
+
// wrong (174, 140) — which is the quantization, not the music.
|
|
1283
|
+
//
|
|
1284
|
+
// A kernel one sampling step wide either side lets a jittered onset
|
|
1285
|
+
// still correlate with the lag it belongs to.
|
|
1286
|
+
// Only for flux-derived onsets. The jitter this corrects is an
|
|
1287
|
+
// artifact of the 20ms sampling, so an input that was not sampled —
|
|
1288
|
+
// exact timestamps from a test, or the mock path — has none, and
|
|
1289
|
+
// smearing it only blurs a peak that was already sharp. Measured:
|
|
1290
|
+
// applying the kernel to an exact 174 BPM 8th-note train moved it
|
|
1291
|
+
// to 117.
|
|
1292
|
+
const KERNEL_BINS = strengths.length > 0 ? Math.max(1, Math.round(20 / RESOLUTION_MS)) : 0;
|
|
1293
|
+
for (let i = 0; i < times.length; i++) {
|
|
1294
|
+
const centre = Math.round((times[i] - times[0]) / RESOLUTION_MS);
|
|
1295
|
+
const weight = strengths.length > 0 ? strengths[i] * scale : 1;
|
|
1296
|
+
for (let d = -KERNEL_BINS; d <= KERNEL_BINS; d++) {
|
|
1297
|
+
const index = centre + d;
|
|
1298
|
+
if (index < 0 || index >= length)
|
|
1299
|
+
continue;
|
|
1300
|
+
const taper = 1 - Math.abs(d) / (KERNEL_BINS + 1);
|
|
1301
|
+
// Keep the strongest contribution rather than summing: two hats
|
|
1302
|
+
// quantized together must not outweigh one kick.
|
|
1303
|
+
envelope[index] = Math.max(envelope[index], weight * taper);
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
const correlation = this.autocorrelate(envelope);
|
|
1307
|
+
// Lags corresponding to 40-200 BPM.
|
|
1308
|
+
const minLag = Math.floor((60000 / 200) / RESOLUTION_MS);
|
|
1309
|
+
const maxLag = Math.ceil((60000 / 40) / RESOLUTION_MS);
|
|
1310
|
+
let bestLag = 0;
|
|
1311
|
+
let bestScore = 0;
|
|
1312
|
+
for (let lag = minLag; lag <= maxLag && lag < correlation.length; lag++) {
|
|
1313
|
+
const bpm = 60000 / (lag * RESOLUTION_MS);
|
|
1314
|
+
// Preference weight in log-tempo space, centred on 120 BPM. This
|
|
1315
|
+
// is what resolves the octave ambiguity: 172ms and 345ms both
|
|
1316
|
+
// correlate, and a listener hears the slower one as the beat.
|
|
1317
|
+
const octaves = Math.log2(bpm / 120);
|
|
1318
|
+
const weight = Math.exp(-(octaves * octaves) / 2);
|
|
1319
|
+
const score = AudioAnalyzer.familyStrength(correlation, lag) * weight;
|
|
1320
|
+
if (score > bestScore) {
|
|
1321
|
+
bestScore = score;
|
|
1322
|
+
bestLag = lag;
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
if (bestLag === 0 || bestScore <= 0)
|
|
1326
|
+
return null;
|
|
1327
|
+
// Prefer the faster reading when it also correlates.
|
|
1328
|
+
//
|
|
1329
|
+
// The autocorrelation of an impulse train peaks at the period AND
|
|
1330
|
+
// at every multiple, and the normalization by (n - lag) slightly
|
|
1331
|
+
// favours the longer one — so a 174 BPM series scored highest at
|
|
1332
|
+
// 690ms and reported 87. Half-time is a real reading of dnb, but
|
|
1333
|
+
// 174 is the one its producer would give, and five existing tests
|
|
1334
|
+
// asserted 174.
|
|
1335
|
+
//
|
|
1336
|
+
// So: if halving the winning lag still correlates substantially,
|
|
1337
|
+
// take the halved one. A 120 BPM series on the beat has no
|
|
1338
|
+
// correlation at 250ms and keeps 500ms; one on 8ths does, and
|
|
1339
|
+
// halves to 250ms — which folds back to 120 anyway.
|
|
1340
|
+
// How strongly the halved lag must correlate before it is preferred.
|
|
1341
|
+
//
|
|
1342
|
+
// Two regimes, because the two inputs carry different information.
|
|
1343
|
+
// With flux strengths the loudest onsets mark the beat, the peak is
|
|
1344
|
+
// trustworthy, and halving should need a near-tie to win — at 0.4 a
|
|
1345
|
+
// 90 BPM pattern with 8th-note hats reads 179. Without strengths
|
|
1346
|
+
// every onset looks alike and a train on 8ths is genuinely
|
|
1347
|
+
// ambiguous: 172.5ms and 345ms are both perfectly periodic, and 174
|
|
1348
|
+
// is a convention rather than a measurement. There, leaning on the
|
|
1349
|
+
// halving is what recovers the number a producer would give.
|
|
1350
|
+
const HALF_TIME_RATIO = strengths.length > 0 ? 0.8 : 0.4;
|
|
1351
|
+
let lag = bestLag;
|
|
1352
|
+
for (let i = 0; i < 3; i++) {
|
|
1353
|
+
const half = Math.round(lag / 2);
|
|
1354
|
+
if (half < minLag || half >= correlation.length)
|
|
1355
|
+
break;
|
|
1356
|
+
if (correlation[half] < correlation[lag] * HALF_TIME_RATIO)
|
|
1357
|
+
break;
|
|
1358
|
+
lag = half;
|
|
1359
|
+
}
|
|
1360
|
+
// The share of the envelope's self-similarity that survives at the
|
|
1361
|
+
// reported period — `corr(beat) / corr(0)`. This is what the
|
|
1362
|
+
// caller uses for confidence, and it is the quantity the answer
|
|
1363
|
+
// was actually derived from (#506).
|
|
1364
|
+
const periodicity = correlation[0] > 0
|
|
1365
|
+
? Math.min(1, correlation[lag] / correlation[0])
|
|
1366
|
+
: 0;
|
|
1367
|
+
return { periodMs: lag * RESOLUTION_MS, periodicity };
|
|
1368
|
+
}
|
|
1369
|
+
/**
|
|
1370
|
+
* Tempos an octave either side of a detected one, within range.
|
|
1371
|
+
*
|
|
1372
|
+
* Half and double time are the SAME onset series — 345ms and 690ms
|
|
1373
|
+
* impulse trains are indistinguishable from timing alone, and
|
|
1374
|
+
* separating them needs onset strength or spectral cues this detector
|
|
1375
|
+
* does not have. Measured at four envelope resolutions from 1ms to
|
|
1376
|
+
* 10ms: the ranking never changed, so it is not a quantization
|
|
1377
|
+
* artefact.
|
|
1378
|
+
*
|
|
1379
|
+
* Rather than pick silently, the alternatives are reported. A dnb
|
|
1380
|
+
* track detected at 87 lists 174, which is the reading its producer
|
|
1381
|
+
* would give (#352).
|
|
1382
|
+
*
|
|
1383
|
+
* @param bpm - The detected tempo
|
|
1384
|
+
* @returns Octave-related tempos inside 40-200, nearest first
|
|
1385
|
+
*/
|
|
1386
|
+
static tempoOctaves(bpm) {
|
|
1387
|
+
const out = [];
|
|
1388
|
+
for (const factor of [2, 0.5, 4, 0.25]) {
|
|
1389
|
+
const candidate = Math.round(bpm * factor);
|
|
1390
|
+
if (candidate >= 40 && candidate <= 200)
|
|
1391
|
+
out.push(candidate);
|
|
1392
|
+
}
|
|
1393
|
+
return out;
|
|
1394
|
+
}
|
|
1395
|
+
tempoFromOnsets(onsets) {
|
|
1396
|
+
const times = onsetTimes(onsets);
|
|
373
1397
|
// Need at least 4 onsets for reliable tempo detection
|
|
374
|
-
if (
|
|
1398
|
+
if (times.length < 4) {
|
|
375
1399
|
return { bpm: 0, confidence: 0, method: 'onset' };
|
|
376
1400
|
}
|
|
377
1401
|
// Calculate inter-onset intervals (IOIs)
|
|
378
|
-
const intervals = this.calculateIntervals(
|
|
379
|
-
//
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
//
|
|
383
|
-
|
|
1402
|
+
const intervals = this.calculateIntervals(times);
|
|
1403
|
+
// Autocorrelation first: it asks at what lag the whole series
|
|
1404
|
+
// repeats, rather than measuring whatever subdivision the onsets
|
|
1405
|
+
// happen to land on. A central inter-onset interval read the
|
|
1406
|
+
// liquid-dnb example as 108, 117 and 190 across runs, because the
|
|
1407
|
+
// fold-into-range step picked whichever octave the subdivision
|
|
1408
|
+
// landed in first (#352).
|
|
1409
|
+
const beatPeriod = this.beatPeriodFromOnsets(onsets);
|
|
1410
|
+
if (beatPeriod !== null && beatPeriod.periodMs > 0) {
|
|
1411
|
+
const rawBpm = 60000 / beatPeriod.periodMs;
|
|
1412
|
+
const folded = AudioAnalyzer.foldIntoTempoRange(rawBpm);
|
|
1413
|
+
if (folded !== null) {
|
|
1414
|
+
const bpm = Math.round(folded);
|
|
1415
|
+
// Confidence from the correlation that produced this number,
|
|
1416
|
+
// not from the spread of the inter-onset intervals.
|
|
1417
|
+
//
|
|
1418
|
+
// The old measure asked "were the detected onsets evenly
|
|
1419
|
+
// spaced?" and was read as "how sure are you about this
|
|
1420
|
+
// tempo?". Those come apart exactly where it matters: the
|
|
1421
|
+
// autocorrelation returned 130.4 BPM in seven of seven
|
|
1422
|
+
// six-second ambient windows while interval CV drifted across
|
|
1423
|
+
// the floor, so a correct answer was reported in four of them
|
|
1424
|
+
// and discarded in three — on a statistic the correlation had
|
|
1425
|
+
// never consulted (#506).
|
|
1426
|
+
//
|
|
1427
|
+
// Measured across real captures and deliberate negatives, the
|
|
1428
|
+
// two quantities rank the cases differently and only one of
|
|
1429
|
+
// them ranks them correctly:
|
|
1430
|
+
//
|
|
1431
|
+
// interval CV corr(beat)/corr(0)
|
|
1432
|
+
// random intervals 0.45 0.11
|
|
1433
|
+
// scattered onsets 0.00 0.09
|
|
1434
|
+
// ambient (correct) 0.19 0.71
|
|
1435
|
+
// amen break 0.39 0.56
|
|
1436
|
+
// techno 0.94 0.95
|
|
1437
|
+
//
|
|
1438
|
+
// The old measure puts a pulseless series ABOVE real ambient.
|
|
1439
|
+
// The new one separates every negative from every positive
|
|
1440
|
+
// with no overlap — 0.16 against 0.30 at the boundary.
|
|
1441
|
+
const confidence = beatPeriod.periodicity;
|
|
1442
|
+
// Below the floor, say nothing rather than say 120.
|
|
1443
|
+
//
|
|
1444
|
+
// The correlation always has a highest lag, and the tempo prior
|
|
1445
|
+
// is centred on 120 — so when the onsets carry no periodicity
|
|
1446
|
+
// the answer is the prior's centre, dressed as a measurement.
|
|
1447
|
+
// Measured on real playback before #366's onset fix: 120 BPM for
|
|
1448
|
+
// dnb, techno and house alike, every run, at confidence 0.00.
|
|
1449
|
+
// The code already knew; nothing was reading it.
|
|
1450
|
+
// Its own floor, because it is its own scale. The negatives
|
|
1451
|
+
// measured top out at 0.16 and the weakest true reading is
|
|
1452
|
+
// 0.30, so 0.25 sits in the gap rather than on either side of
|
|
1453
|
+
// it. MIN_TEMPO_CONFIDENCE still governs the median branch,
|
|
1454
|
+
// where confidence is still interval-based.
|
|
1455
|
+
if (confidence < AudioAnalyzer.MIN_PERIODICITY_CONFIDENCE) {
|
|
1456
|
+
return { bpm: 0, confidence, method: 'autocorrelation' };
|
|
1457
|
+
}
|
|
1458
|
+
return {
|
|
1459
|
+
bpm,
|
|
1460
|
+
confidence,
|
|
1461
|
+
method: 'autocorrelation',
|
|
1462
|
+
alternatives: AudioAnalyzer.tempoOctaves(bpm),
|
|
1463
|
+
};
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
// Median, not mean.
|
|
1467
|
+
//
|
|
1468
|
+
// A single dropped or ghosted onset moved the answer badly: a steady
|
|
1469
|
+
// 120 BPM grid with one missed onset reported 105, and with one
|
|
1470
|
+
// extra reported 135. The median is unmoved by either, and onset
|
|
1471
|
+
// detection misses and doubles constantly (#322).
|
|
1472
|
+
const sorted = [...intervals].sort((a, b) => a - b);
|
|
1473
|
+
const mid = Math.floor(sorted.length / 2);
|
|
1474
|
+
const medianInterval = sorted.length % 2 === 0
|
|
1475
|
+
? (sorted[mid - 1] + sorted[mid]) / 2
|
|
1476
|
+
: sorted[mid];
|
|
1477
|
+
if (medianInterval <= 0) {
|
|
384
1478
|
return { bpm: 0, confidence: 0, method: 'onset' };
|
|
385
1479
|
}
|
|
386
|
-
|
|
387
|
-
const
|
|
388
|
-
|
|
1480
|
+
const rawBpm = 60000 / medianInterval;
|
|
1481
|
+
const bpm = AudioAnalyzer.foldIntoTempoRange(rawBpm);
|
|
1482
|
+
// Unfoldable — more than a few octaves outside the range, so the
|
|
1483
|
+
// onsets are not a tempo at all.
|
|
1484
|
+
if (bpm === null) {
|
|
1485
|
+
return { bpm: 0, confidence: 0, method: 'onset' };
|
|
1486
|
+
}
|
|
1487
|
+
// Confidence still comes from interval consistency, but measured
|
|
1488
|
+
// against the median so one outlier does not inflate the spread it
|
|
1489
|
+
// is being compared to.
|
|
1490
|
+
const variance = this.calculateVariance(intervals, medianInterval);
|
|
1491
|
+
const coefficientOfVariation = Math.sqrt(variance) / medianInterval;
|
|
389
1492
|
// More aggressive penalty for variation
|
|
390
|
-
|
|
1493
|
+
let confidence = Math.max(0, 1 - coefficientOfVariation * 1.5);
|
|
1494
|
+
// A folded reading is a weaker claim than a direct one: we inferred
|
|
1495
|
+
// the beat is half or double what the onsets literally say.
|
|
1496
|
+
if (Math.abs(rawBpm - bpm) > 1) {
|
|
1497
|
+
confidence *= 0.8;
|
|
1498
|
+
}
|
|
1499
|
+
// The same floor the autocorrelation branch applies, which this
|
|
1500
|
+
// branch never had.
|
|
1501
|
+
//
|
|
1502
|
+
// It is the one that let `gen/ambient` report 184 BPM at confidence
|
|
1503
|
+
// 0.000 and `gen/jungle` report 184 at 0.000 — a number with nothing
|
|
1504
|
+
// behind it, in the shape of a measurement. The reasoning on
|
|
1505
|
+
// MIN_TEMPO_CONFIDENCE applies here unchanged; it was simply only
|
|
1506
|
+
// written on one of the two exits (#419).
|
|
1507
|
+
const finalConfidence = Math.min(1, confidence);
|
|
1508
|
+
if (finalConfidence < AudioAnalyzer.MIN_TEMPO_CONFIDENCE) {
|
|
1509
|
+
return { bpm: 0, confidence: finalConfidence, method: 'onset' };
|
|
1510
|
+
}
|
|
391
1511
|
return {
|
|
392
1512
|
bpm: Math.round(bpm),
|
|
393
|
-
confidence:
|
|
1513
|
+
confidence: finalConfidence,
|
|
394
1514
|
method: 'onset'
|
|
395
1515
|
};
|
|
396
1516
|
}
|
|
397
|
-
// ============================================================================
|
|
398
|
-
// KEY DETECTION
|
|
399
|
-
// ============================================================================
|
|
400
1517
|
/**
|
|
401
1518
|
* Detect musical key using Krumhansl-Schmuckler algorithm
|
|
402
1519
|
*/
|
|
@@ -409,6 +1526,9 @@ export class AudioAnalyzer {
|
|
|
409
1526
|
throw new Error('Audio analyzer not connected');
|
|
410
1527
|
}
|
|
411
1528
|
let chroma;
|
|
1529
|
+
// Mean in-range bin magnitude before normalization, or null when the
|
|
1530
|
+
// chroma came from a mock and there is no spectrum behind it.
|
|
1531
|
+
let rawMagnitude = null;
|
|
412
1532
|
// Check if this is a mock with pre-calculated chroma vector (for testing)
|
|
413
1533
|
if (typeof analyzer.analyze === 'function') {
|
|
414
1534
|
const analysis = analyzer.analyze();
|
|
@@ -421,7 +1541,8 @@ export class AudioAnalyzer {
|
|
|
421
1541
|
throw new Error('Invalid audio data');
|
|
422
1542
|
}
|
|
423
1543
|
const fftData = new Uint8Array(analyzer.dataArray);
|
|
424
|
-
|
|
1544
|
+
rawMagnitude = this.meanInRangeMagnitude(fftData, analyzer.sampleRate ?? 44100);
|
|
1545
|
+
chroma = this.extractChroma(fftData, analyzer.sampleRate ?? 44100);
|
|
425
1546
|
}
|
|
426
1547
|
}
|
|
427
1548
|
else {
|
|
@@ -430,18 +1551,61 @@ export class AudioAnalyzer {
|
|
|
430
1551
|
throw new Error('Invalid audio data');
|
|
431
1552
|
}
|
|
432
1553
|
const fftData = new Uint8Array(analyzer.dataArray);
|
|
433
|
-
|
|
1554
|
+
// The rate the AudioContext actually reported, captured at
|
|
1555
|
+
// connect time. 44100 was assumed; the real rate is commonly
|
|
1556
|
+
// 48000, where every frequency came out 8.8% low — 1.47
|
|
1557
|
+
// semitones, enough to put a note in the wrong pitch class (#321).
|
|
1558
|
+
rawMagnitude = this.meanInRangeMagnitude(fftData, analyzer.sampleRate ?? 44100);
|
|
1559
|
+
chroma = this.extractChroma(fftData, analyzer.sampleRate ?? 44100);
|
|
1560
|
+
}
|
|
1561
|
+
// Check for sufficient energy.
|
|
1562
|
+
//
|
|
1563
|
+
// This used to sum the chroma vector and compare against 0.1 — a
|
|
1564
|
+
// test that could never fail, because `extractChroma` normalizes to
|
|
1565
|
+
// sum 1 whenever any bin is non-zero. Measured: an FFT with a single
|
|
1566
|
+
// bin at magnitude 1 out of 1024, which is silence, normalized to
|
|
1567
|
+
// sum 1.000 and came back "C locrian, confidence 0.793". Only an
|
|
1568
|
+
// exactly all-zero spectrum could trip the guard.
|
|
1569
|
+
//
|
|
1570
|
+
// The magnitude has to be read before normalization throws it away.
|
|
1571
|
+
if (rawMagnitude !== null && rawMagnitude < AudioAnalyzer.CHROMA_NOISE_FLOOR) {
|
|
1572
|
+
return { key: 'C', scale: 'major', confidence: 0 };
|
|
434
1573
|
}
|
|
435
|
-
//
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
1574
|
+
// The mock path supplies a chroma directly and has no spectrum to
|
|
1575
|
+
// measure, so it keeps a chroma-side check — meaningful there
|
|
1576
|
+
// because a mock is not required to be normalized.
|
|
1577
|
+
if (rawMagnitude === null && chroma.reduce((sum, val) => sum + val, 0) <= 0) {
|
|
1578
|
+
return { key: 'C', scale: 'major', confidence: 0 };
|
|
439
1579
|
}
|
|
1580
|
+
return this.detectKeyFromChroma(chroma);
|
|
1581
|
+
}
|
|
1582
|
+
/**
|
|
1583
|
+
* Scores a chroma vector against every key/scale profile.
|
|
1584
|
+
*
|
|
1585
|
+
* Extracted from `detectKey` so it can be tested at all. The scoring
|
|
1586
|
+
* used to be inline in a method that needs a live Playwright page, so
|
|
1587
|
+
* nothing could feed it a known chroma and check the answer — which
|
|
1588
|
+
* is how it came to return "dorian" for all twelve canonical minor
|
|
1589
|
+
* profiles without anyone noticing (#320).
|
|
1590
|
+
*
|
|
1591
|
+
* @param chroma - 12-element pitch class profile, any scale
|
|
1592
|
+
* @returns Best key/scale with confidence and alternatives
|
|
1593
|
+
* @example
|
|
1594
|
+
* analyzer.detectKeyFromChroma(KRUMHANSL_MINOR); // C minor
|
|
1595
|
+
*/
|
|
1596
|
+
detectKeyFromChroma(chroma) {
|
|
440
1597
|
// Correlate with all key/scale combinations
|
|
441
1598
|
const scores = [];
|
|
442
1599
|
for (const scale of Object.keys(this.SCALE_PROFILES)) {
|
|
443
1600
|
// Normalize profile to sum to 1
|
|
444
|
-
|
|
1601
|
+
// Key comes from Object.keys(this.SCALE_PROFILES) directly above,
|
|
1602
|
+
// so it is always an own property; hasOwn keeps it that way if the
|
|
1603
|
+
// loop is ever rewritten (#318).
|
|
1604
|
+
const rawProfile = Object.hasOwn(this.SCALE_PROFILES, scale)
|
|
1605
|
+
? this.SCALE_PROFILES[scale]
|
|
1606
|
+
: undefined;
|
|
1607
|
+
if (!rawProfile)
|
|
1608
|
+
continue;
|
|
445
1609
|
const profileSum = rawProfile.reduce((a, b) => a + b, 0);
|
|
446
1610
|
const profile = rawProfile.map(v => v / profileSum);
|
|
447
1611
|
for (let tonic = 0; tonic < 12; tonic++) {
|
|
@@ -451,8 +1615,17 @@ export class AudioAnalyzer {
|
|
|
451
1615
|
for (let i = 0; i < 12; i++) {
|
|
452
1616
|
rotatedChroma[i] = chroma[(i + tonic) % 12];
|
|
453
1617
|
}
|
|
454
|
-
//
|
|
455
|
-
|
|
1618
|
+
// Pearson, which is what Krumhansl-Schmuckler is defined with.
|
|
1619
|
+
//
|
|
1620
|
+
// Cosine was used instead, and `pearsonCorrelation` sat right
|
|
1621
|
+
// beside it, implemented and unit-tested and never called.
|
|
1622
|
+
// Cosine is not mean-centered, so all 24 scores bunch near 1:
|
|
1623
|
+
// for a C-major input the full range was 0.804-1.000, a spread
|
|
1624
|
+
// of 0.196, where Pearson gives -0.683-1.000. The top-1-vs-top-2
|
|
1625
|
+
// gap under cosine was 0.0048 — smaller than the mode boosts
|
|
1626
|
+
// below, which is how those boosts came to decide every
|
|
1627
|
+
// answer (#320).
|
|
1628
|
+
const correlation = this.pearsonCorrelation(rotatedChroma, profile);
|
|
456
1629
|
scores.push({
|
|
457
1630
|
key: this.PITCH_CLASSES[tonic],
|
|
458
1631
|
scale,
|
|
@@ -464,42 +1637,64 @@ export class AudioAnalyzer {
|
|
|
464
1637
|
const chromaWithIndices = chroma.map((v, i) => ({ value: v, index: i }));
|
|
465
1638
|
chromaWithIndices.sort((a, b) => b.value - a.value);
|
|
466
1639
|
const topPitches = chromaWithIndices.slice(0, 3).map(x => this.PITCH_CLASSES[x.index]);
|
|
467
|
-
//
|
|
1640
|
+
// The mode boosts are gone.
|
|
1641
|
+
//
|
|
1642
|
+
// `dorian *= 1.015` was larger than the margin cosine left between
|
|
1643
|
+
// minor and dorian (0.0078), so it flipped EVERY minor key: fed the
|
|
1644
|
+
// exact canonical K-S minor profile for all 12 minor keys, detection
|
|
1645
|
+
// answered "dorian" 12 out of 12 — right tonic, wrong mode, every
|
|
1646
|
+
// time. A thumb on the scale heavier than the scale itself is not a
|
|
1647
|
+
// tie-breaker, it is the decision (#320).
|
|
1648
|
+
//
|
|
1649
|
+
// The tonic boost stays, reduced and applied only to the single
|
|
1650
|
+
// loudest pitch class rather than the top three (boosting all three
|
|
1651
|
+
// by the same 1.075 meant it discriminated nothing between them).
|
|
1652
|
+
// Pearson's spread is wide enough that a nudge this size can only
|
|
1653
|
+
// break a genuine near-tie.
|
|
1654
|
+
//
|
|
1655
|
+
// Applied only to a positive score. Multiplying is a boost above
|
|
1656
|
+
// zero and a PENALTY below it: a candidate at -0.4 becomes -0.408,
|
|
1657
|
+
// pushed down by the nudge that exists to lift it. That only bites
|
|
1658
|
+
// when nothing correlates well, which is exactly where a tonic hint
|
|
1659
|
+
// was supposed to help.
|
|
1660
|
+
const TONIC_NUDGE = 1.02;
|
|
468
1661
|
for (const s of scores) {
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
const pitchBoost = topPitches.indexOf(s.key);
|
|
472
|
-
if (pitchBoost >= 0) {
|
|
473
|
-
// Slightly favor 2nd pitch to handle dominant/mediant being louder than tonic
|
|
474
|
-
const boosts = [1.075, 1.075, 1.075];
|
|
475
|
-
s.score *= boosts[pitchBoost];
|
|
476
|
-
}
|
|
477
|
-
// Boost for common scales
|
|
478
|
-
if (s.scale === 'major') {
|
|
479
|
-
s.score *= 1.03; // 3% boost for major scales (most common)
|
|
480
|
-
}
|
|
481
|
-
else if (s.scale === 'dorian') {
|
|
482
|
-
s.score *= 1.015; // 1.5% boost for dorian (common modal scale)
|
|
1662
|
+
if (s.key === topPitches[0] && s.score > 0) {
|
|
1663
|
+
s.score *= TONIC_NUDGE;
|
|
483
1664
|
}
|
|
484
1665
|
}
|
|
485
1666
|
// Sort by score (after applying biases)
|
|
486
1667
|
scores.sort((a, b) => b.score - a.score);
|
|
487
|
-
// Calculate confidence
|
|
488
1668
|
const best = scores[0];
|
|
489
1669
|
const secondBest = scores[1];
|
|
490
|
-
// Confidence
|
|
491
|
-
//
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
1670
|
+
// Confidence is about discrimination, not similarity.
|
|
1671
|
+
//
|
|
1672
|
+
// It used to be 0.75 * the raw cosine score, which is >= 0.9 for
|
|
1673
|
+
// essentially any non-negative chroma — a flat, zero-information
|
|
1674
|
+
// input returned confidence 0.787. Pearson on a flat chroma is 0
|
|
1675
|
+
// (no correlation with anything), so `strength` now collapses on
|
|
1676
|
+
// exactly the inputs that carry no information, and `separation`
|
|
1677
|
+
// asks whether the winner actually beat the runner-up.
|
|
1678
|
+
const STRENGTH_WEIGHT = 0.6;
|
|
1679
|
+
const SEPARATION_WEIGHT = 0.4;
|
|
1680
|
+
const strength = Math.max(0, best.score);
|
|
1681
|
+
const separation = Math.min(1, Math.max(0, (best.score - secondBest.score) * 5));
|
|
1682
|
+
const confidence = Math.min(1, Math.max(0, strength * STRENGTH_WEIGHT + separation * SEPARATION_WEIGHT));
|
|
495
1683
|
return {
|
|
496
1684
|
key: best.key,
|
|
497
1685
|
scale: best.scale,
|
|
498
1686
|
confidence,
|
|
1687
|
+
// Alternatives are scored the same WAY as the headline, not just
|
|
1688
|
+
// clamped to the same range. Reporting their raw correlation put
|
|
1689
|
+
// them on a different scale entirely — an A-minor input answered
|
|
1690
|
+
// with confidence 0.801 while listing "A dorian 0.926" beneath
|
|
1691
|
+
// it, an alternative apparently more confident than the answer.
|
|
1692
|
+
// A non-winner earns no separation term, by definition: it did
|
|
1693
|
+
// not separate from anything (#320).
|
|
499
1694
|
alternatives: scores.slice(1, 4).map(s => ({
|
|
500
1695
|
key: s.key,
|
|
501
1696
|
scale: s.scale,
|
|
502
|
-
confidence: Math.max(0, s.score)
|
|
1697
|
+
confidence: Math.min(1, Math.max(0, s.score) * STRENGTH_WEIGHT)
|
|
503
1698
|
}))
|
|
504
1699
|
};
|
|
505
1700
|
}
|
|
@@ -538,26 +1733,26 @@ export class AudioAnalyzer {
|
|
|
538
1733
|
// No mock data, use real-time detection
|
|
539
1734
|
const fftData = new Uint8Array(analyzer.dataArray);
|
|
540
1735
|
const flux = this.calculateSpectralFlux(fftData);
|
|
541
|
-
if (
|
|
542
|
-
this._onsetHistory.push(Date.now());
|
|
1736
|
+
if (this.isOnset(flux)) {
|
|
1737
|
+
this._onsetHistory.push({ t: Date.now(), strength: flux });
|
|
543
1738
|
if (this._onsetHistory.length > this.MAX_HISTORY_LENGTH) {
|
|
544
1739
|
this._onsetHistory.shift();
|
|
545
1740
|
}
|
|
546
1741
|
}
|
|
547
|
-
onsets =
|
|
1742
|
+
onsets = onsetTimes(this._onsetHistory);
|
|
548
1743
|
}
|
|
549
1744
|
}
|
|
550
1745
|
else {
|
|
551
1746
|
// No analyze function, use real-time detection
|
|
552
1747
|
const fftData = new Uint8Array(analyzer.dataArray);
|
|
553
1748
|
const flux = this.calculateSpectralFlux(fftData);
|
|
554
|
-
if (
|
|
555
|
-
this._onsetHistory.push(Date.now());
|
|
1749
|
+
if (this.isOnset(flux)) {
|
|
1750
|
+
this._onsetHistory.push({ t: Date.now(), strength: flux });
|
|
556
1751
|
if (this._onsetHistory.length > this.MAX_HISTORY_LENGTH) {
|
|
557
1752
|
this._onsetHistory.shift();
|
|
558
1753
|
}
|
|
559
1754
|
}
|
|
560
|
-
onsets =
|
|
1755
|
+
onsets = onsetTimes(this._onsetHistory);
|
|
561
1756
|
}
|
|
562
1757
|
// Need at least 2 onsets for rhythm analysis
|
|
563
1758
|
if (onsets.length < 2) {
|
|
@@ -625,7 +1820,14 @@ export class AudioAnalyzer {
|
|
|
625
1820
|
return 0;
|
|
626
1821
|
let syncopationScore = 0;
|
|
627
1822
|
for (let i = 1; i < onsets.length; i++) {
|
|
628
|
-
|
|
1823
|
+
// Relative to the first onset, not to the Unix epoch.
|
|
1824
|
+
// `onsets[i]` is a Date.now() value, so this modulo used to depend
|
|
1825
|
+
// on what time of day it was: the same nine onsets 500ms apart
|
|
1826
|
+
// scored 0.000 at one epoch offset and 1.000 at another, and a
|
|
1827
|
+
// perfectly regular grid reported maximum syncopation roughly
|
|
1828
|
+
// three-quarters of the time (#323).
|
|
1829
|
+
const elapsed = onsets[i] - onsets[0];
|
|
1830
|
+
const phase = (elapsed % (meanInterval * 4)) / meanInterval;
|
|
629
1831
|
// Check if onset is on an off-beat (not on 0, 1, 2, 3)
|
|
630
1832
|
const nearestBeat = Math.round(phase);
|
|
631
1833
|
const beatDistance = Math.abs(phase - nearestBeat);
|
|
@@ -646,7 +1848,8 @@ export class AudioAnalyzer {
|
|
|
646
1848
|
const patternLength = 16;
|
|
647
1849
|
const pattern = new Array(patternLength).fill('.');
|
|
648
1850
|
for (const onset of onsets) {
|
|
649
|
-
|
|
1851
|
+
// Same epoch bug as detectSyncopation: relative to the first onset.
|
|
1852
|
+
const position = Math.round(((onset - onsets[0]) % (meanInterval * patternLength)) / meanInterval);
|
|
650
1853
|
if (position < patternLength) {
|
|
651
1854
|
pattern[position] = 'X';
|
|
652
1855
|
}
|