@viji-dev/core 0.3.26 → 0.3.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global.d.ts +4 -1
- package/dist/artist-jsdoc.d.ts +3 -1
- package/dist/assets/{viji.worker-PAf0oIec.js → viji.worker-CdHkRwxI.js} +756 -88
- package/dist/assets/viji.worker-CdHkRwxI.js.map +1 -0
- package/dist/docs-api.js +104 -50
- package/dist/{essentia-wasm.web-1yXYrWJ5.js → essentia-wasm.web-CJ3YX7ue.js} +2 -2
- package/dist/{essentia-wasm.web-1yXYrWJ5.js.map → essentia-wasm.web-CJ3YX7ue.js.map} +1 -1
- package/dist/{index-B8BYfP9z.js → index-i7P2rHW8.js} +1475 -1202
- package/dist/index-i7P2rHW8.js.map +1 -0
- package/dist/index.d.ts +162 -91
- package/dist/index.js +1 -1
- package/dist/shader-uniforms.js +414 -49
- package/package.json +1 -1
- package/dist/assets/viji.worker-PAf0oIec.js.map +0 -1
- package/dist/index-B8BYfP9z.js.map +0 -1
|
@@ -199,6 +199,18 @@ class IFrameManager {
|
|
|
199
199
|
getScale() {
|
|
200
200
|
return this.scale;
|
|
201
201
|
}
|
|
202
|
+
/**
|
|
203
|
+
* Gets the actual ratio of canvas pixels to display (CSS) pixels.
|
|
204
|
+
* Works correctly for both scale-based and explicit resolution modes.
|
|
205
|
+
*/
|
|
206
|
+
getDisplayScale() {
|
|
207
|
+
if (this.isHeadless) return 1;
|
|
208
|
+
if (this.explicitResolution && this.hostContainer) {
|
|
209
|
+
const rect = this.hostContainer.getBoundingClientRect();
|
|
210
|
+
return rect.width > 0 ? this.explicitResolution.width / rect.width : 1;
|
|
211
|
+
}
|
|
212
|
+
return Math.max(0.1, Math.min(1, this.scale));
|
|
213
|
+
}
|
|
202
214
|
// ========================================
|
|
203
215
|
// Device Sensor Support
|
|
204
216
|
// ========================================
|
|
@@ -578,7 +590,7 @@ class IFrameManager {
|
|
|
578
590
|
}
|
|
579
591
|
function WorkerWrapper(options) {
|
|
580
592
|
return new Worker(
|
|
581
|
-
"" + new URL("assets/viji.worker-
|
|
593
|
+
"" + new URL("assets/viji.worker-CdHkRwxI.js", import.meta.url).href,
|
|
582
594
|
{
|
|
583
595
|
type: "module",
|
|
584
596
|
name: options?.name
|
|
@@ -586,9 +598,10 @@ function WorkerWrapper(options) {
|
|
|
586
598
|
);
|
|
587
599
|
}
|
|
588
600
|
class WorkerManager {
|
|
589
|
-
constructor(sceneCode, offscreenCanvas) {
|
|
601
|
+
constructor(sceneCode, offscreenCanvas, isHeadless = false) {
|
|
590
602
|
this.sceneCode = sceneCode;
|
|
591
603
|
this.offscreenCanvas = offscreenCanvas;
|
|
604
|
+
this.isHeadless = isHeadless;
|
|
592
605
|
}
|
|
593
606
|
worker = null;
|
|
594
607
|
messageId = 0;
|
|
@@ -755,7 +768,8 @@ class WorkerManager {
|
|
|
755
768
|
id,
|
|
756
769
|
timestamp: Date.now(),
|
|
757
770
|
data: {
|
|
758
|
-
canvas: this.offscreenCanvas
|
|
771
|
+
canvas: this.offscreenCanvas,
|
|
772
|
+
isHeadless: this.isHeadless
|
|
759
773
|
}
|
|
760
774
|
};
|
|
761
775
|
return new Promise((resolve, reject) => {
|
|
@@ -1076,75 +1090,6 @@ class EnvelopeFollower {
|
|
|
1076
1090
|
};
|
|
1077
1091
|
}
|
|
1078
1092
|
}
|
|
1079
|
-
class AutoGain {
|
|
1080
|
-
recentPeaks = [];
|
|
1081
|
-
gain = 1;
|
|
1082
|
-
targetLevel = 0.7;
|
|
1083
|
-
noiseFloor = 0.01;
|
|
1084
|
-
// -40dB noise floor
|
|
1085
|
-
windowSize;
|
|
1086
|
-
minGain = 0.5;
|
|
1087
|
-
maxGain = 4;
|
|
1088
|
-
gainSmoothingFactor = 0.01;
|
|
1089
|
-
/**
|
|
1090
|
-
* Create auto-gain processor
|
|
1091
|
-
* @param windowSizeMs - Window size in milliseconds for peak tracking (default 3000ms)
|
|
1092
|
-
* @param sampleRate - Sample rate in Hz (default 60 for 60fps)
|
|
1093
|
-
*/
|
|
1094
|
-
constructor(windowSizeMs = 3e3, sampleRate = 60) {
|
|
1095
|
-
this.windowSize = Math.floor(windowSizeMs / 1e3 * sampleRate);
|
|
1096
|
-
}
|
|
1097
|
-
/**
|
|
1098
|
-
* Process input value and return normalized output
|
|
1099
|
-
* @param input - Input value (0-1)
|
|
1100
|
-
* @returns Normalized value (0-1)
|
|
1101
|
-
*/
|
|
1102
|
-
process(input) {
|
|
1103
|
-
if (input < this.noiseFloor) {
|
|
1104
|
-
return 0;
|
|
1105
|
-
}
|
|
1106
|
-
this.recentPeaks.push(input);
|
|
1107
|
-
if (this.recentPeaks.length > this.windowSize) {
|
|
1108
|
-
this.recentPeaks.shift();
|
|
1109
|
-
}
|
|
1110
|
-
if (this.recentPeaks.length > 10) {
|
|
1111
|
-
const sorted = [...this.recentPeaks].sort((a, b) => b - a);
|
|
1112
|
-
const percentileIndex = Math.floor(sorted.length * 0.05);
|
|
1113
|
-
const currentPeak = sorted[percentileIndex];
|
|
1114
|
-
if (currentPeak > this.noiseFloor) {
|
|
1115
|
-
const targetGain = this.targetLevel / currentPeak;
|
|
1116
|
-
const clampedTarget = Math.max(this.minGain, Math.min(this.maxGain, targetGain));
|
|
1117
|
-
this.gain += (clampedTarget - this.gain) * this.gainSmoothingFactor;
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
return Math.max(0, Math.min(1, input * this.gain));
|
|
1121
|
-
}
|
|
1122
|
-
/**
|
|
1123
|
-
* Get current gain value
|
|
1124
|
-
*/
|
|
1125
|
-
getGain() {
|
|
1126
|
-
return this.gain;
|
|
1127
|
-
}
|
|
1128
|
-
/**
|
|
1129
|
-
* Set target level (0-1)
|
|
1130
|
-
*/
|
|
1131
|
-
setTargetLevel(level) {
|
|
1132
|
-
this.targetLevel = Math.max(0.1, Math.min(1, level));
|
|
1133
|
-
}
|
|
1134
|
-
/**
|
|
1135
|
-
* Set noise floor threshold (0-1)
|
|
1136
|
-
*/
|
|
1137
|
-
setNoiseFloor(threshold) {
|
|
1138
|
-
this.noiseFloor = Math.max(0, Math.min(0.1, threshold));
|
|
1139
|
-
}
|
|
1140
|
-
/**
|
|
1141
|
-
* Reset auto-gain state
|
|
1142
|
-
*/
|
|
1143
|
-
reset() {
|
|
1144
|
-
this.recentPeaks = [];
|
|
1145
|
-
this.gain = 1;
|
|
1146
|
-
}
|
|
1147
|
-
}
|
|
1148
1093
|
const BAND_CONFIGS = {
|
|
1149
1094
|
low: {
|
|
1150
1095
|
minThreshold: 2e-3,
|
|
@@ -1801,7 +1746,7 @@ class EssentiaOnsetDetection {
|
|
|
1801
1746
|
this.initPromise = (async () => {
|
|
1802
1747
|
try {
|
|
1803
1748
|
const essentiaModule = await import("./essentia.js-core.es-DnrJE0uR.js");
|
|
1804
|
-
const wasmModule = await import("./essentia-wasm.web-
|
|
1749
|
+
const wasmModule = await import("./essentia-wasm.web-CJ3YX7ue.js").then((n) => n.e);
|
|
1805
1750
|
const EssentiaClass = essentiaModule.Essentia || essentiaModule.default?.Essentia || essentiaModule.default;
|
|
1806
1751
|
let WASMModule = wasmModule.default || wasmModule.EssentiaWASM || wasmModule.default?.EssentiaWASM;
|
|
1807
1752
|
if (!WASMModule) {
|
|
@@ -11883,125 +11828,570 @@ Rejection rate: ${(rejections / duration * 60).toFixed(1)} per minute
|
|
|
11883
11828
|
return this.events.length;
|
|
11884
11829
|
}
|
|
11885
11830
|
}
|
|
11886
|
-
|
|
11887
|
-
|
|
11888
|
-
|
|
11889
|
-
|
|
11890
|
-
|
|
11891
|
-
|
|
11892
|
-
|
|
11893
|
-
|
|
11894
|
-
|
|
11895
|
-
|
|
11896
|
-
|
|
11897
|
-
|
|
11898
|
-
|
|
11899
|
-
|
|
11900
|
-
|
|
11901
|
-
|
|
11902
|
-
|
|
11903
|
-
|
|
11904
|
-
|
|
11905
|
-
|
|
11906
|
-
|
|
11907
|
-
|
|
11908
|
-
|
|
11909
|
-
|
|
11910
|
-
|
|
11911
|
-
|
|
11912
|
-
|
|
11831
|
+
const INSTRUMENTS = ["kick", "snare", "hat"];
|
|
11832
|
+
const SAMPLE_RATE = 60;
|
|
11833
|
+
const TAP_TIMEOUT_MS = 5e3;
|
|
11834
|
+
const MAX_CYCLE_LENGTH = 8;
|
|
11835
|
+
const FUZZY_TOLERANCE = 0.18;
|
|
11836
|
+
const MIN_REPETITIONS = 3;
|
|
11837
|
+
const MAX_TAP_INTERVAL_MS = 3e3;
|
|
11838
|
+
const MIN_TAP_INTERVAL_MS = 100;
|
|
11839
|
+
const MAX_TAP_HISTORY = 64;
|
|
11840
|
+
const MIN_EMA_ALPHA = 0.05;
|
|
11841
|
+
const PATTERN_SAME_TOLERANCE = 0.15;
|
|
11842
|
+
function createInstrumentState() {
|
|
11843
|
+
return {
|
|
11844
|
+
mode: "auto",
|
|
11845
|
+
muted: false,
|
|
11846
|
+
mutedAt: 0,
|
|
11847
|
+
tapIOIs: [],
|
|
11848
|
+
lastTapTime: 0,
|
|
11849
|
+
pattern: null,
|
|
11850
|
+
refinementIndex: 0,
|
|
11851
|
+
refinementCounts: [],
|
|
11852
|
+
replayLastEventTime: 0,
|
|
11853
|
+
replayIndex: 0,
|
|
11854
|
+
pendingTapEvents: [],
|
|
11855
|
+
envelope: new EnvelopeFollower(0, 300, SAMPLE_RATE),
|
|
11856
|
+
envelopeSmoothed: new EnvelopeFollower(5, 500, SAMPLE_RATE)
|
|
11857
|
+
};
|
|
11913
11858
|
}
|
|
11914
|
-
|
|
11915
|
-
|
|
11916
|
-
|
|
11917
|
-
|
|
11918
|
-
|
|
11919
|
-
|
|
11920
|
-
|
|
11921
|
-
|
|
11922
|
-
|
|
11923
|
-
|
|
11924
|
-
|
|
11925
|
-
|
|
11926
|
-
};
|
|
11927
|
-
FFT.prototype.toComplexArray = function toComplexArray(input, storage) {
|
|
11928
|
-
var res = storage || this.createComplexArray();
|
|
11929
|
-
for (var i = 0; i < res.length; i += 2) {
|
|
11930
|
-
res[i] = input[i >>> 1];
|
|
11931
|
-
res[i + 1] = 0;
|
|
11932
|
-
}
|
|
11933
|
-
return res;
|
|
11934
|
-
};
|
|
11935
|
-
FFT.prototype.completeSpectrum = function completeSpectrum(spectrum) {
|
|
11936
|
-
var size = this._csize;
|
|
11937
|
-
var half = size >>> 1;
|
|
11938
|
-
for (var i = 2; i < half; i += 2) {
|
|
11939
|
-
spectrum[size - i] = spectrum[i];
|
|
11940
|
-
spectrum[size - i + 1] = -spectrum[i + 1];
|
|
11941
|
-
}
|
|
11942
|
-
};
|
|
11943
|
-
FFT.prototype.transform = function transform(out, data) {
|
|
11944
|
-
if (out === data)
|
|
11945
|
-
throw new Error("Input and output buffers must be different");
|
|
11946
|
-
this._out = out;
|
|
11947
|
-
this._data = data;
|
|
11948
|
-
this._inv = 0;
|
|
11949
|
-
this._transform4();
|
|
11950
|
-
this._out = null;
|
|
11951
|
-
this._data = null;
|
|
11952
|
-
};
|
|
11953
|
-
FFT.prototype.realTransform = function realTransform(out, data) {
|
|
11954
|
-
if (out === data)
|
|
11955
|
-
throw new Error("Input and output buffers must be different");
|
|
11956
|
-
this._out = out;
|
|
11957
|
-
this._data = data;
|
|
11958
|
-
this._inv = 0;
|
|
11959
|
-
this._realTransform4();
|
|
11960
|
-
this._out = null;
|
|
11961
|
-
this._data = null;
|
|
11962
|
-
};
|
|
11963
|
-
FFT.prototype.inverseTransform = function inverseTransform(out, data) {
|
|
11964
|
-
if (out === data)
|
|
11965
|
-
throw new Error("Input and output buffers must be different");
|
|
11966
|
-
this._out = out;
|
|
11967
|
-
this._data = data;
|
|
11968
|
-
this._inv = 1;
|
|
11969
|
-
this._transform4();
|
|
11970
|
-
for (var i = 0; i < out.length; i++)
|
|
11971
|
-
out[i] /= this.size;
|
|
11972
|
-
this._out = null;
|
|
11973
|
-
this._data = null;
|
|
11974
|
-
};
|
|
11975
|
-
FFT.prototype._transform4 = function _transform4() {
|
|
11976
|
-
var out = this._out;
|
|
11977
|
-
var size = this._csize;
|
|
11978
|
-
var width = this._width;
|
|
11979
|
-
var step2 = 1 << width;
|
|
11980
|
-
var len = size / step2 << 1;
|
|
11981
|
-
var outOff;
|
|
11982
|
-
var t;
|
|
11983
|
-
var bitrev = this._bitrev;
|
|
11984
|
-
if (len === 4) {
|
|
11985
|
-
for (outOff = 0, t = 0; outOff < size; outOff += len, t++) {
|
|
11986
|
-
const off = bitrev[t];
|
|
11987
|
-
this._singleTransform2(outOff, off, step2);
|
|
11859
|
+
class OnsetTapManager {
|
|
11860
|
+
state = {
|
|
11861
|
+
kick: createInstrumentState(),
|
|
11862
|
+
snare: createInstrumentState(),
|
|
11863
|
+
hat: createInstrumentState()
|
|
11864
|
+
};
|
|
11865
|
+
tap(instrument) {
|
|
11866
|
+
const s = this.state[instrument];
|
|
11867
|
+
const now = performance.now();
|
|
11868
|
+
if (s.muted) {
|
|
11869
|
+
s.muted = false;
|
|
11870
|
+
s.mutedAt = 0;
|
|
11988
11871
|
}
|
|
11989
|
-
|
|
11990
|
-
|
|
11991
|
-
|
|
11992
|
-
|
|
11872
|
+
let ioi = -1;
|
|
11873
|
+
if (s.lastTapTime > 0) {
|
|
11874
|
+
ioi = now - s.lastTapTime;
|
|
11875
|
+
if (ioi < MIN_TAP_INTERVAL_MS) return;
|
|
11876
|
+
if (ioi > MAX_TAP_INTERVAL_MS) {
|
|
11877
|
+
s.tapIOIs = [];
|
|
11878
|
+
ioi = -1;
|
|
11879
|
+
} else {
|
|
11880
|
+
s.tapIOIs.push(ioi);
|
|
11881
|
+
if (s.tapIOIs.length > MAX_TAP_HISTORY) s.tapIOIs.shift();
|
|
11882
|
+
}
|
|
11883
|
+
}
|
|
11884
|
+
s.lastTapTime = now;
|
|
11885
|
+
s.pendingTapEvents.push(now);
|
|
11886
|
+
if (s.mode === "auto") {
|
|
11887
|
+
s.mode = "tapping";
|
|
11888
|
+
if (ioi > 0) {
|
|
11889
|
+
const pattern = this.tryRecognizePattern(instrument);
|
|
11890
|
+
if (pattern) this.applyPattern(instrument, pattern);
|
|
11891
|
+
}
|
|
11892
|
+
} else if (s.mode === "tapping") {
|
|
11893
|
+
if (ioi > 0) {
|
|
11894
|
+
const pattern = this.tryRecognizePattern(instrument);
|
|
11895
|
+
if (pattern) this.applyPattern(instrument, pattern);
|
|
11896
|
+
}
|
|
11897
|
+
} else if (s.mode === "pattern") {
|
|
11898
|
+
if (ioi > 0) {
|
|
11899
|
+
this.handlePatternTap(instrument, ioi, now);
|
|
11900
|
+
}
|
|
11993
11901
|
}
|
|
11994
11902
|
}
|
|
11995
|
-
|
|
11996
|
-
|
|
11997
|
-
|
|
11998
|
-
|
|
11999
|
-
|
|
12000
|
-
|
|
12001
|
-
|
|
12002
|
-
|
|
12003
|
-
|
|
12004
|
-
|
|
11903
|
+
clear(instrument) {
|
|
11904
|
+
const s = this.state[instrument];
|
|
11905
|
+
s.mode = "auto";
|
|
11906
|
+
s.muted = false;
|
|
11907
|
+
s.mutedAt = 0;
|
|
11908
|
+
s.tapIOIs = [];
|
|
11909
|
+
s.lastTapTime = 0;
|
|
11910
|
+
s.pattern = null;
|
|
11911
|
+
s.refinementIndex = 0;
|
|
11912
|
+
s.refinementCounts = [];
|
|
11913
|
+
s.replayLastEventTime = 0;
|
|
11914
|
+
s.replayIndex = 0;
|
|
11915
|
+
s.pendingTapEvents = [];
|
|
11916
|
+
s.envelope.reset();
|
|
11917
|
+
s.envelopeSmoothed.reset();
|
|
11918
|
+
}
|
|
11919
|
+
getMode(instrument) {
|
|
11920
|
+
return this.state[instrument].mode;
|
|
11921
|
+
}
|
|
11922
|
+
getPatternInfo(instrument) {
|
|
11923
|
+
const s = this.state[instrument];
|
|
11924
|
+
if (!s.pattern) return null;
|
|
11925
|
+
return {
|
|
11926
|
+
length: s.pattern.length,
|
|
11927
|
+
intervals: [...s.pattern],
|
|
11928
|
+
confidence: 1
|
|
11929
|
+
};
|
|
11930
|
+
}
|
|
11931
|
+
setMuted(instrument, muted) {
|
|
11932
|
+
const s = this.state[instrument];
|
|
11933
|
+
if (s.muted === muted) return;
|
|
11934
|
+
const now = performance.now();
|
|
11935
|
+
if (muted) {
|
|
11936
|
+
s.muted = true;
|
|
11937
|
+
s.mutedAt = now;
|
|
11938
|
+
} else {
|
|
11939
|
+
const pauseDuration = now - s.mutedAt;
|
|
11940
|
+
s.muted = false;
|
|
11941
|
+
s.mutedAt = 0;
|
|
11942
|
+
if (s.mode === "tapping" && s.lastTapTime > 0) {
|
|
11943
|
+
s.lastTapTime += pauseDuration;
|
|
11944
|
+
}
|
|
11945
|
+
}
|
|
11946
|
+
}
|
|
11947
|
+
isMuted(instrument) {
|
|
11948
|
+
return this.state[instrument].muted;
|
|
11949
|
+
}
|
|
11950
|
+
/**
|
|
11951
|
+
* Post-process a beat state produced by BeatStateManager.
|
|
11952
|
+
* For instruments in auto mode, values pass through unchanged.
|
|
11953
|
+
* For tapping/pattern instruments, auto events are suppressed and
|
|
11954
|
+
* tap/pattern events + envelopes are injected instead.
|
|
11955
|
+
*/
|
|
11956
|
+
processFrame(beatState, now, dtMs) {
|
|
11957
|
+
const result = { ...beatState, events: [...beatState.events] };
|
|
11958
|
+
const newEvents = [];
|
|
11959
|
+
for (const inst of INSTRUMENTS) {
|
|
11960
|
+
const s = this.state[inst];
|
|
11961
|
+
if (s.muted) {
|
|
11962
|
+
result.events = result.events.filter((e) => e.type !== inst);
|
|
11963
|
+
s.pendingTapEvents = [];
|
|
11964
|
+
if (s.mode === "pattern" && s.pattern && s.replayLastEventTime > 0) {
|
|
11965
|
+
for (let i = 0; i < 3; i++) {
|
|
11966
|
+
const interval = s.pattern[s.replayIndex % s.pattern.length];
|
|
11967
|
+
if (now - s.replayLastEventTime >= interval) {
|
|
11968
|
+
s.replayLastEventTime += interval;
|
|
11969
|
+
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
11970
|
+
} else {
|
|
11971
|
+
break;
|
|
11972
|
+
}
|
|
11973
|
+
}
|
|
11974
|
+
}
|
|
11975
|
+
s.envelope.process(0, dtMs);
|
|
11976
|
+
s.envelopeSmoothed.process(0, dtMs);
|
|
11977
|
+
const envKey2 = inst;
|
|
11978
|
+
const smoothKey2 = `${inst}Smoothed`;
|
|
11979
|
+
result[envKey2] = s.envelope.getValue();
|
|
11980
|
+
result[smoothKey2] = s.envelopeSmoothed.getValue();
|
|
11981
|
+
continue;
|
|
11982
|
+
}
|
|
11983
|
+
if (s.mode === "auto") {
|
|
11984
|
+
s.envelope.process(0, dtMs);
|
|
11985
|
+
s.envelopeSmoothed.process(0, dtMs);
|
|
11986
|
+
continue;
|
|
11987
|
+
}
|
|
11988
|
+
result.events = result.events.filter((e) => e.type !== inst);
|
|
11989
|
+
if (s.mode === "tapping") {
|
|
11990
|
+
if (now - s.lastTapTime > TAP_TIMEOUT_MS) {
|
|
11991
|
+
s.tapIOIs = [];
|
|
11992
|
+
s.pendingTapEvents = [];
|
|
11993
|
+
if (s.pattern) {
|
|
11994
|
+
s.mode = "pattern";
|
|
11995
|
+
s.replayLastEventTime = now;
|
|
11996
|
+
s.replayIndex = 0;
|
|
11997
|
+
} else {
|
|
11998
|
+
s.mode = "auto";
|
|
11999
|
+
}
|
|
12000
|
+
continue;
|
|
12001
|
+
}
|
|
12002
|
+
while (s.pendingTapEvents.length > 0) {
|
|
12003
|
+
const tapTime = s.pendingTapEvents.shift();
|
|
12004
|
+
newEvents.push(this.createTapEvent(inst, tapTime, result.bpm));
|
|
12005
|
+
s.envelope.trigger(1);
|
|
12006
|
+
s.envelopeSmoothed.trigger(1);
|
|
12007
|
+
s.replayLastEventTime = tapTime;
|
|
12008
|
+
s.replayIndex = 0;
|
|
12009
|
+
}
|
|
12010
|
+
}
|
|
12011
|
+
if (s.mode === "pattern") {
|
|
12012
|
+
while (s.pendingTapEvents.length > 0) {
|
|
12013
|
+
const tapTime = s.pendingTapEvents.shift();
|
|
12014
|
+
newEvents.push(this.createTapEvent(inst, tapTime, result.bpm));
|
|
12015
|
+
s.envelope.trigger(1);
|
|
12016
|
+
s.envelopeSmoothed.trigger(1);
|
|
12017
|
+
}
|
|
12018
|
+
const isUserActive = now - s.lastTapTime < 500;
|
|
12019
|
+
if (!isUserActive && s.pattern) {
|
|
12020
|
+
const scheduled = this.checkPatternReplay(s, inst, now, result.bpm);
|
|
12021
|
+
for (const ev of scheduled) {
|
|
12022
|
+
newEvents.push(ev);
|
|
12023
|
+
s.envelope.trigger(1);
|
|
12024
|
+
s.envelopeSmoothed.trigger(1);
|
|
12025
|
+
}
|
|
12026
|
+
}
|
|
12027
|
+
}
|
|
12028
|
+
s.envelope.process(0, dtMs);
|
|
12029
|
+
s.envelopeSmoothed.process(0, dtMs);
|
|
12030
|
+
const envKey = inst;
|
|
12031
|
+
const smoothKey = `${inst}Smoothed`;
|
|
12032
|
+
result[envKey] = s.envelope.getValue();
|
|
12033
|
+
result[smoothKey] = s.envelopeSmoothed.getValue();
|
|
12034
|
+
}
|
|
12035
|
+
result.events.push(...newEvents);
|
|
12036
|
+
const anyMax = Math.max(
|
|
12037
|
+
result.kick,
|
|
12038
|
+
result.snare,
|
|
12039
|
+
result.hat
|
|
12040
|
+
);
|
|
12041
|
+
const anySmoothedMax = Math.max(
|
|
12042
|
+
result.kickSmoothed,
|
|
12043
|
+
result.snareSmoothed,
|
|
12044
|
+
result.hatSmoothed
|
|
12045
|
+
);
|
|
12046
|
+
if (anyMax > result.any) result.any = anyMax;
|
|
12047
|
+
if (anySmoothedMax > result.anySmoothed) result.anySmoothed = anySmoothedMax;
|
|
12048
|
+
return result;
|
|
12049
|
+
}
|
|
12050
|
+
// ---------------------------------------------------------------------------
|
|
12051
|
+
// Private helpers
|
|
12052
|
+
// ---------------------------------------------------------------------------
|
|
12053
|
+
/**
|
|
12054
|
+
* Handle a tap that arrives while already in pattern mode.
|
|
12055
|
+
* Matching taps refine the pattern via EMA and re-anchor phase.
|
|
12056
|
+
* Non-matching taps trigger new pattern detection on the full IOI history.
|
|
12057
|
+
*/
|
|
12058
|
+
handlePatternTap(instrument, ioi, now) {
|
|
12059
|
+
const s = this.state[instrument];
|
|
12060
|
+
if (!s.pattern) return;
|
|
12061
|
+
const matchedPos = this.findMatchingPosition(s, ioi);
|
|
12062
|
+
if (matchedPos >= 0) {
|
|
12063
|
+
s.refinementCounts[matchedPos] = (s.refinementCounts[matchedPos] || 0) + 1;
|
|
12064
|
+
const alpha = Math.max(MIN_EMA_ALPHA, 1 / s.refinementCounts[matchedPos]);
|
|
12065
|
+
s.pattern[matchedPos] = (1 - alpha) * s.pattern[matchedPos] + alpha * ioi;
|
|
12066
|
+
s.refinementIndex = (matchedPos + 1) % s.pattern.length;
|
|
12067
|
+
s.replayLastEventTime = now;
|
|
12068
|
+
s.replayIndex = s.refinementIndex;
|
|
12069
|
+
} else {
|
|
12070
|
+
const currentPattern = [...s.pattern];
|
|
12071
|
+
const newPattern = this.tryRecognizePattern(instrument);
|
|
12072
|
+
if (newPattern && !this.isPatternSame(newPattern, currentPattern)) {
|
|
12073
|
+
this.applyPattern(instrument, newPattern);
|
|
12074
|
+
}
|
|
12075
|
+
}
|
|
12076
|
+
}
|
|
12077
|
+
/**
|
|
12078
|
+
* Find which pattern position the given IOI matches.
|
|
12079
|
+
* Tries the expected refinementIndex first, then scans all positions.
|
|
12080
|
+
* Returns the matched position index, or -1 if no match.
|
|
12081
|
+
*/
|
|
12082
|
+
findMatchingPosition(s, ioi) {
|
|
12083
|
+
if (!s.pattern) return -1;
|
|
12084
|
+
const expectedPos = s.refinementIndex % s.pattern.length;
|
|
12085
|
+
const expectedInterval = s.pattern[expectedPos];
|
|
12086
|
+
if (Math.abs(ioi - expectedInterval) / expectedInterval <= FUZZY_TOLERANCE) {
|
|
12087
|
+
return expectedPos;
|
|
12088
|
+
}
|
|
12089
|
+
let bestPos = -1;
|
|
12090
|
+
let bestDeviation = Infinity;
|
|
12091
|
+
for (let i = 0; i < s.pattern.length; i++) {
|
|
12092
|
+
if (i === expectedPos) continue;
|
|
12093
|
+
const deviation = Math.abs(ioi - s.pattern[i]) / s.pattern[i];
|
|
12094
|
+
if (deviation <= FUZZY_TOLERANCE && deviation < bestDeviation) {
|
|
12095
|
+
bestDeviation = deviation;
|
|
12096
|
+
bestPos = i;
|
|
12097
|
+
}
|
|
12098
|
+
}
|
|
12099
|
+
return bestPos;
|
|
12100
|
+
}
|
|
12101
|
+
/**
|
|
12102
|
+
* Compare two patterns for equivalence.
|
|
12103
|
+
* Same length and each interval within PATTERN_SAME_TOLERANCE → same.
|
|
12104
|
+
*/
|
|
12105
|
+
isPatternSame(a, b) {
|
|
12106
|
+
if (a.length !== b.length) return false;
|
|
12107
|
+
for (let i = 0; i < a.length; i++) {
|
|
12108
|
+
if (Math.abs(a[i] - b[i]) / Math.max(a[i], b[i]) > PATTERN_SAME_TOLERANCE) return false;
|
|
12109
|
+
}
|
|
12110
|
+
return true;
|
|
12111
|
+
}
|
|
12112
|
+
/**
|
|
12113
|
+
* Pattern recognition: find the shortest repeating IOI cycle
|
|
12114
|
+
* that has been tapped at least MIN_REPETITIONS times.
|
|
12115
|
+
* Returns the recognized pattern as float intervals, or null.
|
|
12116
|
+
* Does NOT mutate state — caller decides what to do with the result.
|
|
12117
|
+
*/
|
|
12118
|
+
tryRecognizePattern(instrument) {
|
|
12119
|
+
const s = this.state[instrument];
|
|
12120
|
+
const iois = s.tapIOIs;
|
|
12121
|
+
if (iois.length < MIN_REPETITIONS) return null;
|
|
12122
|
+
const maxL = Math.min(MAX_CYCLE_LENGTH, Math.floor(iois.length / MIN_REPETITIONS));
|
|
12123
|
+
for (let L = 1; L <= maxL; L++) {
|
|
12124
|
+
const needed = MIN_REPETITIONS * L;
|
|
12125
|
+
if (iois.length < needed) continue;
|
|
12126
|
+
const tolerance = L <= 2 ? FUZZY_TOLERANCE : FUZZY_TOLERANCE * (2 / L);
|
|
12127
|
+
const recent = iois.slice(-needed);
|
|
12128
|
+
const groups = [];
|
|
12129
|
+
for (let g = 0; g < MIN_REPETITIONS; g++) {
|
|
12130
|
+
groups.push(recent.slice(g * L, (g + 1) * L));
|
|
12131
|
+
}
|
|
12132
|
+
const avgGroup = groups[0].map((_, i) => {
|
|
12133
|
+
let sum = 0;
|
|
12134
|
+
for (const grp of groups) sum += grp[i];
|
|
12135
|
+
return sum / groups.length;
|
|
12136
|
+
});
|
|
12137
|
+
let allMatch = true;
|
|
12138
|
+
for (const grp of groups) {
|
|
12139
|
+
for (let i = 0; i < L; i++) {
|
|
12140
|
+
const deviation = Math.abs(grp[i] - avgGroup[i]) / avgGroup[i];
|
|
12141
|
+
if (deviation > tolerance) {
|
|
12142
|
+
allMatch = false;
|
|
12143
|
+
break;
|
|
12144
|
+
}
|
|
12145
|
+
}
|
|
12146
|
+
if (!allMatch) break;
|
|
12147
|
+
}
|
|
12148
|
+
if (allMatch) {
|
|
12149
|
+
return avgGroup;
|
|
12150
|
+
}
|
|
12151
|
+
}
|
|
12152
|
+
return null;
|
|
12153
|
+
}
|
|
12154
|
+
/**
|
|
12155
|
+
* Apply a recognized pattern to the instrument state.
|
|
12156
|
+
* Sets pattern, mode, and initializes replay anchor.
|
|
12157
|
+
*/
|
|
12158
|
+
applyPattern(instrument, pattern) {
|
|
12159
|
+
const s = this.state[instrument];
|
|
12160
|
+
s.pattern = pattern;
|
|
12161
|
+
s.mode = "pattern";
|
|
12162
|
+
s.refinementIndex = 0;
|
|
12163
|
+
s.refinementCounts = new Array(pattern.length).fill(MIN_REPETITIONS);
|
|
12164
|
+
if (s.replayLastEventTime === 0 || s.replayLastEventTime < s.lastTapTime) {
|
|
12165
|
+
s.replayLastEventTime = s.lastTapTime;
|
|
12166
|
+
s.replayIndex = 0;
|
|
12167
|
+
}
|
|
12168
|
+
}
|
|
12169
|
+
createTapEvent(instrument, time, bpm) {
|
|
12170
|
+
return {
|
|
12171
|
+
type: instrument,
|
|
12172
|
+
time,
|
|
12173
|
+
strength: 0.85,
|
|
12174
|
+
isPredicted: false,
|
|
12175
|
+
bpm
|
|
12176
|
+
};
|
|
12177
|
+
}
|
|
12178
|
+
checkPatternReplay(s, instrument, now, bpm) {
|
|
12179
|
+
if (!s.pattern || s.pattern.length === 0) return [];
|
|
12180
|
+
const events = [];
|
|
12181
|
+
const maxEventsPerFrame = 3;
|
|
12182
|
+
for (let safety = 0; safety < maxEventsPerFrame; safety++) {
|
|
12183
|
+
const expectedInterval = s.pattern[s.replayIndex % s.pattern.length];
|
|
12184
|
+
const elapsed = now - s.replayLastEventTime;
|
|
12185
|
+
if (elapsed >= expectedInterval) {
|
|
12186
|
+
s.replayLastEventTime += expectedInterval;
|
|
12187
|
+
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
12188
|
+
events.push({
|
|
12189
|
+
type: instrument,
|
|
12190
|
+
time: s.replayLastEventTime,
|
|
12191
|
+
strength: 0.8,
|
|
12192
|
+
isPredicted: true,
|
|
12193
|
+
bpm
|
|
12194
|
+
});
|
|
12195
|
+
} else {
|
|
12196
|
+
break;
|
|
12197
|
+
}
|
|
12198
|
+
}
|
|
12199
|
+
return events;
|
|
12200
|
+
}
|
|
12201
|
+
reset() {
|
|
12202
|
+
for (const inst of INSTRUMENTS) {
|
|
12203
|
+
this.clear(inst);
|
|
12204
|
+
}
|
|
12205
|
+
}
|
|
12206
|
+
}
|
|
12207
|
+
class AutoGain {
|
|
12208
|
+
recentPeaks = [];
|
|
12209
|
+
gain = 1;
|
|
12210
|
+
targetLevel = 0.7;
|
|
12211
|
+
noiseFloor = 0.01;
|
|
12212
|
+
// -40dB noise floor
|
|
12213
|
+
windowSize;
|
|
12214
|
+
minGain = 0.5;
|
|
12215
|
+
maxGain = 4;
|
|
12216
|
+
gainSmoothingFactor = 0.01;
|
|
12217
|
+
/**
|
|
12218
|
+
* Create auto-gain processor
|
|
12219
|
+
* @param windowSizeMs - Window size in milliseconds for peak tracking (default 3000ms)
|
|
12220
|
+
* @param sampleRate - Sample rate in Hz (default 60 for 60fps)
|
|
12221
|
+
*/
|
|
12222
|
+
constructor(windowSizeMs = 3e3, sampleRate = 60) {
|
|
12223
|
+
this.windowSize = Math.floor(windowSizeMs / 1e3 * sampleRate);
|
|
12224
|
+
}
|
|
12225
|
+
/**
|
|
12226
|
+
* Process input value and return normalized output
|
|
12227
|
+
* @param input - Input value (0-1)
|
|
12228
|
+
* @returns Normalized value (0-1)
|
|
12229
|
+
*/
|
|
12230
|
+
process(input) {
|
|
12231
|
+
if (input < this.noiseFloor) {
|
|
12232
|
+
return 0;
|
|
12233
|
+
}
|
|
12234
|
+
this.recentPeaks.push(input);
|
|
12235
|
+
if (this.recentPeaks.length > this.windowSize) {
|
|
12236
|
+
this.recentPeaks.shift();
|
|
12237
|
+
}
|
|
12238
|
+
if (this.recentPeaks.length > 10) {
|
|
12239
|
+
const sorted = [...this.recentPeaks].sort((a, b) => b - a);
|
|
12240
|
+
const percentileIndex = Math.floor(sorted.length * 0.05);
|
|
12241
|
+
const currentPeak = sorted[percentileIndex];
|
|
12242
|
+
if (currentPeak > this.noiseFloor) {
|
|
12243
|
+
const targetGain = this.targetLevel / currentPeak;
|
|
12244
|
+
const clampedTarget = Math.max(this.minGain, Math.min(this.maxGain, targetGain));
|
|
12245
|
+
this.gain += (clampedTarget - this.gain) * this.gainSmoothingFactor;
|
|
12246
|
+
}
|
|
12247
|
+
}
|
|
12248
|
+
return Math.max(0, Math.min(1, input * this.gain));
|
|
12249
|
+
}
|
|
12250
|
+
/**
|
|
12251
|
+
* Get current gain value
|
|
12252
|
+
*/
|
|
12253
|
+
getGain() {
|
|
12254
|
+
return this.gain;
|
|
12255
|
+
}
|
|
12256
|
+
/**
|
|
12257
|
+
* Set target level (0-1)
|
|
12258
|
+
*/
|
|
12259
|
+
setTargetLevel(level) {
|
|
12260
|
+
this.targetLevel = Math.max(0.1, Math.min(1, level));
|
|
12261
|
+
}
|
|
12262
|
+
/**
|
|
12263
|
+
* Set noise floor threshold (0-1)
|
|
12264
|
+
*/
|
|
12265
|
+
setNoiseFloor(threshold) {
|
|
12266
|
+
this.noiseFloor = Math.max(0, Math.min(0.1, threshold));
|
|
12267
|
+
}
|
|
12268
|
+
/**
|
|
12269
|
+
* Reset auto-gain state
|
|
12270
|
+
*/
|
|
12271
|
+
reset() {
|
|
12272
|
+
this.recentPeaks = [];
|
|
12273
|
+
this.gain = 1;
|
|
12274
|
+
}
|
|
12275
|
+
}
|
|
12276
|
+
function FFT(size) {
|
|
12277
|
+
this.size = size | 0;
|
|
12278
|
+
if (this.size <= 1 || (this.size & this.size - 1) !== 0)
|
|
12279
|
+
throw new Error("FFT size must be a power of two and bigger than 1");
|
|
12280
|
+
this._csize = size << 1;
|
|
12281
|
+
var table = new Array(this.size * 2);
|
|
12282
|
+
for (var i = 0; i < table.length; i += 2) {
|
|
12283
|
+
const angle = Math.PI * i / this.size;
|
|
12284
|
+
table[i] = Math.cos(angle);
|
|
12285
|
+
table[i + 1] = -Math.sin(angle);
|
|
12286
|
+
}
|
|
12287
|
+
this.table = table;
|
|
12288
|
+
var power = 0;
|
|
12289
|
+
for (var t = 1; this.size > t; t <<= 1)
|
|
12290
|
+
power++;
|
|
12291
|
+
this._width = power % 2 === 0 ? power - 1 : power;
|
|
12292
|
+
this._bitrev = new Array(1 << this._width);
|
|
12293
|
+
for (var j = 0; j < this._bitrev.length; j++) {
|
|
12294
|
+
this._bitrev[j] = 0;
|
|
12295
|
+
for (var shift = 0; shift < this._width; shift += 2) {
|
|
12296
|
+
var revShift = this._width - shift - 2;
|
|
12297
|
+
this._bitrev[j] |= (j >>> shift & 3) << revShift;
|
|
12298
|
+
}
|
|
12299
|
+
}
|
|
12300
|
+
this._out = null;
|
|
12301
|
+
this._data = null;
|
|
12302
|
+
this._inv = 0;
|
|
12303
|
+
}
|
|
12304
|
+
var fft = FFT;
|
|
12305
|
+
FFT.prototype.fromComplexArray = function fromComplexArray(complex, storage) {
|
|
12306
|
+
var res = storage || new Array(complex.length >>> 1);
|
|
12307
|
+
for (var i = 0; i < complex.length; i += 2)
|
|
12308
|
+
res[i >>> 1] = complex[i];
|
|
12309
|
+
return res;
|
|
12310
|
+
};
|
|
12311
|
+
FFT.prototype.createComplexArray = function createComplexArray() {
|
|
12312
|
+
const res = new Array(this._csize);
|
|
12313
|
+
for (var i = 0; i < res.length; i++)
|
|
12314
|
+
res[i] = 0;
|
|
12315
|
+
return res;
|
|
12316
|
+
};
|
|
12317
|
+
FFT.prototype.toComplexArray = function toComplexArray(input, storage) {
|
|
12318
|
+
var res = storage || this.createComplexArray();
|
|
12319
|
+
for (var i = 0; i < res.length; i += 2) {
|
|
12320
|
+
res[i] = input[i >>> 1];
|
|
12321
|
+
res[i + 1] = 0;
|
|
12322
|
+
}
|
|
12323
|
+
return res;
|
|
12324
|
+
};
|
|
12325
|
+
FFT.prototype.completeSpectrum = function completeSpectrum(spectrum) {
|
|
12326
|
+
var size = this._csize;
|
|
12327
|
+
var half = size >>> 1;
|
|
12328
|
+
for (var i = 2; i < half; i += 2) {
|
|
12329
|
+
spectrum[size - i] = spectrum[i];
|
|
12330
|
+
spectrum[size - i + 1] = -spectrum[i + 1];
|
|
12331
|
+
}
|
|
12332
|
+
};
|
|
12333
|
+
FFT.prototype.transform = function transform(out, data) {
|
|
12334
|
+
if (out === data)
|
|
12335
|
+
throw new Error("Input and output buffers must be different");
|
|
12336
|
+
this._out = out;
|
|
12337
|
+
this._data = data;
|
|
12338
|
+
this._inv = 0;
|
|
12339
|
+
this._transform4();
|
|
12340
|
+
this._out = null;
|
|
12341
|
+
this._data = null;
|
|
12342
|
+
};
|
|
12343
|
+
FFT.prototype.realTransform = function realTransform(out, data) {
|
|
12344
|
+
if (out === data)
|
|
12345
|
+
throw new Error("Input and output buffers must be different");
|
|
12346
|
+
this._out = out;
|
|
12347
|
+
this._data = data;
|
|
12348
|
+
this._inv = 0;
|
|
12349
|
+
this._realTransform4();
|
|
12350
|
+
this._out = null;
|
|
12351
|
+
this._data = null;
|
|
12352
|
+
};
|
|
12353
|
+
FFT.prototype.inverseTransform = function inverseTransform(out, data) {
|
|
12354
|
+
if (out === data)
|
|
12355
|
+
throw new Error("Input and output buffers must be different");
|
|
12356
|
+
this._out = out;
|
|
12357
|
+
this._data = data;
|
|
12358
|
+
this._inv = 1;
|
|
12359
|
+
this._transform4();
|
|
12360
|
+
for (var i = 0; i < out.length; i++)
|
|
12361
|
+
out[i] /= this.size;
|
|
12362
|
+
this._out = null;
|
|
12363
|
+
this._data = null;
|
|
12364
|
+
};
|
|
12365
|
+
FFT.prototype._transform4 = function _transform4() {
|
|
12366
|
+
var out = this._out;
|
|
12367
|
+
var size = this._csize;
|
|
12368
|
+
var width = this._width;
|
|
12369
|
+
var step2 = 1 << width;
|
|
12370
|
+
var len = size / step2 << 1;
|
|
12371
|
+
var outOff;
|
|
12372
|
+
var t;
|
|
12373
|
+
var bitrev = this._bitrev;
|
|
12374
|
+
if (len === 4) {
|
|
12375
|
+
for (outOff = 0, t = 0; outOff < size; outOff += len, t++) {
|
|
12376
|
+
const off = bitrev[t];
|
|
12377
|
+
this._singleTransform2(outOff, off, step2);
|
|
12378
|
+
}
|
|
12379
|
+
} else {
|
|
12380
|
+
for (outOff = 0, t = 0; outOff < size; outOff += len, t++) {
|
|
12381
|
+
const off = bitrev[t];
|
|
12382
|
+
this._singleTransform4(outOff, off, step2);
|
|
12383
|
+
}
|
|
12384
|
+
}
|
|
12385
|
+
var inv = this._inv ? -1 : 1;
|
|
12386
|
+
var table = this.table;
|
|
12387
|
+
for (step2 >>= 2; step2 >= 2; step2 >>= 2) {
|
|
12388
|
+
len = size / step2 << 1;
|
|
12389
|
+
var quarterLen = len >>> 2;
|
|
12390
|
+
for (outOff = 0; outOff < size; outOff += len) {
|
|
12391
|
+
var limit = outOff + quarterLen;
|
|
12392
|
+
for (var i = outOff, k = 0; i < limit; i += 2, k += step2) {
|
|
12393
|
+
const A = i;
|
|
12394
|
+
const B = A + quarterLen;
|
|
12005
12395
|
const C = B + quarterLen;
|
|
12006
12396
|
const D = C + quarterLen;
|
|
12007
12397
|
const Ar = out[A];
|
|
@@ -12167,495 +12557,271 @@ FFT.prototype._realTransform4 = function _realTransform4() {
|
|
|
12167
12557
|
var T0r = MAr + MCr;
|
|
12168
12558
|
var T0i = MAi + MCi;
|
|
12169
12559
|
var T1r = MAr - MCr;
|
|
12170
|
-
var T1i = MAi - MCi;
|
|
12171
|
-
var T2r = MBr + MDr;
|
|
12172
|
-
var T2i = MBi + MDi;
|
|
12173
|
-
var T3r = inv * (MBr - MDr);
|
|
12174
|
-
var T3i = inv * (MBi - MDi);
|
|
12175
|
-
var FAr = T0r + T2r;
|
|
12176
|
-
var FAi = T0i + T2i;
|
|
12177
|
-
var FBr = T1r + T3i;
|
|
12178
|
-
var FBi = T1i - T3r;
|
|
12179
|
-
out[A] = FAr;
|
|
12180
|
-
out[A + 1] = FAi;
|
|
12181
|
-
out[B] = FBr;
|
|
12182
|
-
out[B + 1] = FBi;
|
|
12183
|
-
if (i === 0) {
|
|
12184
|
-
var FCr = T0r - T2r;
|
|
12185
|
-
var FCi = T0i - T2i;
|
|
12186
|
-
out[C] = FCr;
|
|
12187
|
-
out[C + 1] = FCi;
|
|
12188
|
-
continue;
|
|
12189
|
-
}
|
|
12190
|
-
if (i === hquarterLen)
|
|
12191
|
-
continue;
|
|
12192
|
-
var ST0r = T1r;
|
|
12193
|
-
var ST0i = -T1i;
|
|
12194
|
-
var ST1r = T0r;
|
|
12195
|
-
var ST1i = -T0i;
|
|
12196
|
-
var ST2r = -inv * T3i;
|
|
12197
|
-
var ST2i = -inv * T3r;
|
|
12198
|
-
var ST3r = -inv * T2i;
|
|
12199
|
-
var ST3i = -inv * T2r;
|
|
12200
|
-
var SFAr = ST0r + ST2r;
|
|
12201
|
-
var SFAi = ST0i + ST2i;
|
|
12202
|
-
var SFBr = ST1r + ST3i;
|
|
12203
|
-
var SFBi = ST1i - ST3r;
|
|
12204
|
-
var SA = outOff + quarterLen - i;
|
|
12205
|
-
var SB = outOff + halfLen - i;
|
|
12206
|
-
out[SA] = SFAr;
|
|
12207
|
-
out[SA + 1] = SFAi;
|
|
12208
|
-
out[SB] = SFBr;
|
|
12209
|
-
out[SB + 1] = SFBi;
|
|
12210
|
-
}
|
|
12211
|
-
}
|
|
12212
|
-
}
|
|
12213
|
-
};
|
|
12214
|
-
FFT.prototype._singleRealTransform2 = function _singleRealTransform2(outOff, off, step2) {
|
|
12215
|
-
const out = this._out;
|
|
12216
|
-
const data = this._data;
|
|
12217
|
-
const evenR = data[off];
|
|
12218
|
-
const oddR = data[off + step2];
|
|
12219
|
-
const leftR = evenR + oddR;
|
|
12220
|
-
const rightR = evenR - oddR;
|
|
12221
|
-
out[outOff] = leftR;
|
|
12222
|
-
out[outOff + 1] = 0;
|
|
12223
|
-
out[outOff + 2] = rightR;
|
|
12224
|
-
out[outOff + 3] = 0;
|
|
12225
|
-
};
|
|
12226
|
-
FFT.prototype._singleRealTransform4 = function _singleRealTransform4(outOff, off, step2) {
|
|
12227
|
-
const out = this._out;
|
|
12228
|
-
const data = this._data;
|
|
12229
|
-
const inv = this._inv ? -1 : 1;
|
|
12230
|
-
const step22 = step2 * 2;
|
|
12231
|
-
const step3 = step2 * 3;
|
|
12232
|
-
const Ar = data[off];
|
|
12233
|
-
const Br = data[off + step2];
|
|
12234
|
-
const Cr = data[off + step22];
|
|
12235
|
-
const Dr = data[off + step3];
|
|
12236
|
-
const T0r = Ar + Cr;
|
|
12237
|
-
const T1r = Ar - Cr;
|
|
12238
|
-
const T2r = Br + Dr;
|
|
12239
|
-
const T3r = inv * (Br - Dr);
|
|
12240
|
-
const FAr = T0r + T2r;
|
|
12241
|
-
const FBr = T1r;
|
|
12242
|
-
const FBi = -T3r;
|
|
12243
|
-
const FCr = T0r - T2r;
|
|
12244
|
-
const FDr = T1r;
|
|
12245
|
-
const FDi = T3r;
|
|
12246
|
-
out[outOff] = FAr;
|
|
12247
|
-
out[outOff + 1] = 0;
|
|
12248
|
-
out[outOff + 2] = FBr;
|
|
12249
|
-
out[outOff + 3] = FBi;
|
|
12250
|
-
out[outOff + 4] = FCr;
|
|
12251
|
-
out[outOff + 5] = 0;
|
|
12252
|
-
out[outOff + 6] = FDr;
|
|
12253
|
-
out[outOff + 7] = FDi;
|
|
12254
|
-
};
|
|
12255
|
-
const FFT$1 = /* @__PURE__ */ getDefaultExportFromCjs(fft);
|
|
12256
|
-
const INSTRUMENTS = ["kick", "snare", "hat"];
|
|
12257
|
-
const SAMPLE_RATE = 60;
|
|
12258
|
-
const TAP_TIMEOUT_MS = 5e3;
|
|
12259
|
-
const MAX_CYCLE_LENGTH = 8;
|
|
12260
|
-
const FUZZY_TOLERANCE = 0.18;
|
|
12261
|
-
const MIN_REPETITIONS = 3;
|
|
12262
|
-
const MAX_TAP_INTERVAL_MS = 3e3;
|
|
12263
|
-
const MIN_TAP_INTERVAL_MS = 100;
|
|
12264
|
-
const MAX_TAP_HISTORY = 64;
|
|
12265
|
-
const MIN_EMA_ALPHA = 0.05;
|
|
12266
|
-
const PATTERN_SAME_TOLERANCE = 0.15;
|
|
12267
|
-
function createInstrumentState() {
|
|
12268
|
-
return {
|
|
12269
|
-
mode: "auto",
|
|
12270
|
-
muted: false,
|
|
12271
|
-
mutedAt: 0,
|
|
12272
|
-
tapIOIs: [],
|
|
12273
|
-
lastTapTime: 0,
|
|
12274
|
-
pattern: null,
|
|
12275
|
-
refinementIndex: 0,
|
|
12276
|
-
refinementCounts: [],
|
|
12277
|
-
replayLastEventTime: 0,
|
|
12278
|
-
replayIndex: 0,
|
|
12279
|
-
pendingTapEvents: [],
|
|
12280
|
-
envelope: new EnvelopeFollower(0, 300, SAMPLE_RATE),
|
|
12281
|
-
envelopeSmoothed: new EnvelopeFollower(5, 500, SAMPLE_RATE)
|
|
12282
|
-
};
|
|
12283
|
-
}
|
|
12284
|
-
class OnsetTapManager {
|
|
12285
|
-
state = {
|
|
12286
|
-
kick: createInstrumentState(),
|
|
12287
|
-
snare: createInstrumentState(),
|
|
12288
|
-
hat: createInstrumentState()
|
|
12289
|
-
};
|
|
12290
|
-
tap(instrument) {
|
|
12291
|
-
const s = this.state[instrument];
|
|
12292
|
-
const now = performance.now();
|
|
12293
|
-
if (s.muted) {
|
|
12294
|
-
s.muted = false;
|
|
12295
|
-
s.mutedAt = 0;
|
|
12296
|
-
}
|
|
12297
|
-
let ioi = -1;
|
|
12298
|
-
if (s.lastTapTime > 0) {
|
|
12299
|
-
ioi = now - s.lastTapTime;
|
|
12300
|
-
if (ioi < MIN_TAP_INTERVAL_MS) return;
|
|
12301
|
-
if (ioi > MAX_TAP_INTERVAL_MS) {
|
|
12302
|
-
s.tapIOIs = [];
|
|
12303
|
-
ioi = -1;
|
|
12304
|
-
} else {
|
|
12305
|
-
s.tapIOIs.push(ioi);
|
|
12306
|
-
if (s.tapIOIs.length > MAX_TAP_HISTORY) s.tapIOIs.shift();
|
|
12307
|
-
}
|
|
12308
|
-
}
|
|
12309
|
-
s.lastTapTime = now;
|
|
12310
|
-
s.pendingTapEvents.push(now);
|
|
12311
|
-
if (s.mode === "auto") {
|
|
12312
|
-
s.mode = "tapping";
|
|
12313
|
-
if (ioi > 0) {
|
|
12314
|
-
const pattern = this.tryRecognizePattern(instrument);
|
|
12315
|
-
if (pattern) this.applyPattern(instrument, pattern);
|
|
12316
|
-
}
|
|
12317
|
-
} else if (s.mode === "tapping") {
|
|
12318
|
-
if (ioi > 0) {
|
|
12319
|
-
const pattern = this.tryRecognizePattern(instrument);
|
|
12320
|
-
if (pattern) this.applyPattern(instrument, pattern);
|
|
12321
|
-
}
|
|
12322
|
-
} else if (s.mode === "pattern") {
|
|
12323
|
-
if (ioi > 0) {
|
|
12324
|
-
this.handlePatternTap(instrument, ioi, now);
|
|
12325
|
-
}
|
|
12326
|
-
}
|
|
12327
|
-
}
|
|
12328
|
-
clear(instrument) {
|
|
12329
|
-
const s = this.state[instrument];
|
|
12330
|
-
s.mode = "auto";
|
|
12331
|
-
s.muted = false;
|
|
12332
|
-
s.mutedAt = 0;
|
|
12333
|
-
s.tapIOIs = [];
|
|
12334
|
-
s.lastTapTime = 0;
|
|
12335
|
-
s.pattern = null;
|
|
12336
|
-
s.refinementIndex = 0;
|
|
12337
|
-
s.refinementCounts = [];
|
|
12338
|
-
s.replayLastEventTime = 0;
|
|
12339
|
-
s.replayIndex = 0;
|
|
12340
|
-
s.pendingTapEvents = [];
|
|
12341
|
-
s.envelope.reset();
|
|
12342
|
-
s.envelopeSmoothed.reset();
|
|
12343
|
-
}
|
|
12344
|
-
getMode(instrument) {
|
|
12345
|
-
return this.state[instrument].mode;
|
|
12346
|
-
}
|
|
12347
|
-
getPatternInfo(instrument) {
|
|
12348
|
-
const s = this.state[instrument];
|
|
12349
|
-
if (!s.pattern) return null;
|
|
12350
|
-
return {
|
|
12351
|
-
length: s.pattern.length,
|
|
12352
|
-
intervals: [...s.pattern],
|
|
12353
|
-
confidence: 1
|
|
12354
|
-
};
|
|
12355
|
-
}
|
|
12356
|
-
setMuted(instrument, muted) {
|
|
12357
|
-
const s = this.state[instrument];
|
|
12358
|
-
if (s.muted === muted) return;
|
|
12359
|
-
const now = performance.now();
|
|
12360
|
-
if (muted) {
|
|
12361
|
-
s.muted = true;
|
|
12362
|
-
s.mutedAt = now;
|
|
12363
|
-
} else {
|
|
12364
|
-
const pauseDuration = now - s.mutedAt;
|
|
12365
|
-
s.muted = false;
|
|
12366
|
-
s.mutedAt = 0;
|
|
12367
|
-
if (s.mode === "tapping" && s.lastTapTime > 0) {
|
|
12368
|
-
s.lastTapTime += pauseDuration;
|
|
12369
|
-
}
|
|
12370
|
-
}
|
|
12371
|
-
}
|
|
12372
|
-
isMuted(instrument) {
|
|
12373
|
-
return this.state[instrument].muted;
|
|
12374
|
-
}
|
|
12375
|
-
/**
|
|
12376
|
-
* Post-process a beat state produced by BeatStateManager.
|
|
12377
|
-
* For instruments in auto mode, values pass through unchanged.
|
|
12378
|
-
* For tapping/pattern instruments, auto events are suppressed and
|
|
12379
|
-
* tap/pattern events + envelopes are injected instead.
|
|
12380
|
-
*/
|
|
12381
|
-
processFrame(beatState, now, dtMs) {
|
|
12382
|
-
const result = { ...beatState, events: [...beatState.events] };
|
|
12383
|
-
const newEvents = [];
|
|
12384
|
-
for (const inst of INSTRUMENTS) {
|
|
12385
|
-
const s = this.state[inst];
|
|
12386
|
-
if (s.muted) {
|
|
12387
|
-
result.events = result.events.filter((e) => e.type !== inst);
|
|
12388
|
-
s.pendingTapEvents = [];
|
|
12389
|
-
if (s.mode === "pattern" && s.pattern && s.replayLastEventTime > 0) {
|
|
12390
|
-
for (let i = 0; i < 3; i++) {
|
|
12391
|
-
const interval = s.pattern[s.replayIndex % s.pattern.length];
|
|
12392
|
-
if (now - s.replayLastEventTime >= interval) {
|
|
12393
|
-
s.replayLastEventTime += interval;
|
|
12394
|
-
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
12395
|
-
} else {
|
|
12396
|
-
break;
|
|
12397
|
-
}
|
|
12398
|
-
}
|
|
12399
|
-
}
|
|
12400
|
-
s.envelope.process(0, dtMs);
|
|
12401
|
-
s.envelopeSmoothed.process(0, dtMs);
|
|
12402
|
-
const envKey2 = inst;
|
|
12403
|
-
const smoothKey2 = `${inst}Smoothed`;
|
|
12404
|
-
result[envKey2] = s.envelope.getValue();
|
|
12405
|
-
result[smoothKey2] = s.envelopeSmoothed.getValue();
|
|
12406
|
-
continue;
|
|
12407
|
-
}
|
|
12408
|
-
if (s.mode === "auto") {
|
|
12409
|
-
s.envelope.process(0, dtMs);
|
|
12410
|
-
s.envelopeSmoothed.process(0, dtMs);
|
|
12411
|
-
continue;
|
|
12412
|
-
}
|
|
12413
|
-
result.events = result.events.filter((e) => e.type !== inst);
|
|
12414
|
-
if (s.mode === "tapping") {
|
|
12415
|
-
if (now - s.lastTapTime > TAP_TIMEOUT_MS) {
|
|
12416
|
-
s.tapIOIs = [];
|
|
12417
|
-
s.pendingTapEvents = [];
|
|
12418
|
-
if (s.pattern) {
|
|
12419
|
-
s.mode = "pattern";
|
|
12420
|
-
s.replayLastEventTime = now;
|
|
12421
|
-
s.replayIndex = 0;
|
|
12422
|
-
} else {
|
|
12423
|
-
s.mode = "auto";
|
|
12424
|
-
}
|
|
12560
|
+
var T1i = MAi - MCi;
|
|
12561
|
+
var T2r = MBr + MDr;
|
|
12562
|
+
var T2i = MBi + MDi;
|
|
12563
|
+
var T3r = inv * (MBr - MDr);
|
|
12564
|
+
var T3i = inv * (MBi - MDi);
|
|
12565
|
+
var FAr = T0r + T2r;
|
|
12566
|
+
var FAi = T0i + T2i;
|
|
12567
|
+
var FBr = T1r + T3i;
|
|
12568
|
+
var FBi = T1i - T3r;
|
|
12569
|
+
out[A] = FAr;
|
|
12570
|
+
out[A + 1] = FAi;
|
|
12571
|
+
out[B] = FBr;
|
|
12572
|
+
out[B + 1] = FBi;
|
|
12573
|
+
if (i === 0) {
|
|
12574
|
+
var FCr = T0r - T2r;
|
|
12575
|
+
var FCi = T0i - T2i;
|
|
12576
|
+
out[C] = FCr;
|
|
12577
|
+
out[C + 1] = FCi;
|
|
12425
12578
|
continue;
|
|
12426
12579
|
}
|
|
12427
|
-
|
|
12428
|
-
|
|
12429
|
-
|
|
12430
|
-
|
|
12431
|
-
|
|
12432
|
-
|
|
12433
|
-
|
|
12434
|
-
|
|
12435
|
-
|
|
12436
|
-
|
|
12437
|
-
|
|
12438
|
-
|
|
12439
|
-
|
|
12440
|
-
|
|
12441
|
-
|
|
12442
|
-
|
|
12443
|
-
|
|
12444
|
-
|
|
12445
|
-
|
|
12446
|
-
|
|
12447
|
-
newEvents.push(ev);
|
|
12448
|
-
s.envelope.trigger(1);
|
|
12449
|
-
s.envelopeSmoothed.trigger(1);
|
|
12450
|
-
}
|
|
12451
|
-
}
|
|
12580
|
+
if (i === hquarterLen)
|
|
12581
|
+
continue;
|
|
12582
|
+
var ST0r = T1r;
|
|
12583
|
+
var ST0i = -T1i;
|
|
12584
|
+
var ST1r = T0r;
|
|
12585
|
+
var ST1i = -T0i;
|
|
12586
|
+
var ST2r = -inv * T3i;
|
|
12587
|
+
var ST2i = -inv * T3r;
|
|
12588
|
+
var ST3r = -inv * T2i;
|
|
12589
|
+
var ST3i = -inv * T2r;
|
|
12590
|
+
var SFAr = ST0r + ST2r;
|
|
12591
|
+
var SFAi = ST0i + ST2i;
|
|
12592
|
+
var SFBr = ST1r + ST3i;
|
|
12593
|
+
var SFBi = ST1i - ST3r;
|
|
12594
|
+
var SA = outOff + quarterLen - i;
|
|
12595
|
+
var SB = outOff + halfLen - i;
|
|
12596
|
+
out[SA] = SFAr;
|
|
12597
|
+
out[SA + 1] = SFAi;
|
|
12598
|
+
out[SB] = SFBr;
|
|
12599
|
+
out[SB + 1] = SFBi;
|
|
12452
12600
|
}
|
|
12453
|
-
s.envelope.process(0, dtMs);
|
|
12454
|
-
s.envelopeSmoothed.process(0, dtMs);
|
|
12455
|
-
const envKey = inst;
|
|
12456
|
-
const smoothKey = `${inst}Smoothed`;
|
|
12457
|
-
result[envKey] = s.envelope.getValue();
|
|
12458
|
-
result[smoothKey] = s.envelopeSmoothed.getValue();
|
|
12459
12601
|
}
|
|
12460
|
-
result.events.push(...newEvents);
|
|
12461
|
-
const anyMax = Math.max(
|
|
12462
|
-
result.kick,
|
|
12463
|
-
result.snare,
|
|
12464
|
-
result.hat
|
|
12465
|
-
);
|
|
12466
|
-
const anySmoothedMax = Math.max(
|
|
12467
|
-
result.kickSmoothed,
|
|
12468
|
-
result.snareSmoothed,
|
|
12469
|
-
result.hatSmoothed
|
|
12470
|
-
);
|
|
12471
|
-
if (anyMax > result.any) result.any = anyMax;
|
|
12472
|
-
if (anySmoothedMax > result.anySmoothed) result.anySmoothed = anySmoothedMax;
|
|
12473
|
-
return result;
|
|
12474
12602
|
}
|
|
12475
|
-
|
|
12476
|
-
|
|
12477
|
-
|
|
12478
|
-
|
|
12479
|
-
|
|
12480
|
-
|
|
12481
|
-
|
|
12482
|
-
|
|
12483
|
-
|
|
12484
|
-
|
|
12485
|
-
|
|
12486
|
-
|
|
12487
|
-
|
|
12488
|
-
|
|
12489
|
-
|
|
12490
|
-
|
|
12491
|
-
|
|
12492
|
-
|
|
12493
|
-
|
|
12494
|
-
|
|
12495
|
-
|
|
12496
|
-
|
|
12497
|
-
|
|
12498
|
-
|
|
12499
|
-
|
|
12500
|
-
|
|
12603
|
+
};
|
|
12604
|
+
FFT.prototype._singleRealTransform2 = function _singleRealTransform2(outOff, off, step2) {
|
|
12605
|
+
const out = this._out;
|
|
12606
|
+
const data = this._data;
|
|
12607
|
+
const evenR = data[off];
|
|
12608
|
+
const oddR = data[off + step2];
|
|
12609
|
+
const leftR = evenR + oddR;
|
|
12610
|
+
const rightR = evenR - oddR;
|
|
12611
|
+
out[outOff] = leftR;
|
|
12612
|
+
out[outOff + 1] = 0;
|
|
12613
|
+
out[outOff + 2] = rightR;
|
|
12614
|
+
out[outOff + 3] = 0;
|
|
12615
|
+
};
|
|
12616
|
+
FFT.prototype._singleRealTransform4 = function _singleRealTransform4(outOff, off, step2) {
|
|
12617
|
+
const out = this._out;
|
|
12618
|
+
const data = this._data;
|
|
12619
|
+
const inv = this._inv ? -1 : 1;
|
|
12620
|
+
const step22 = step2 * 2;
|
|
12621
|
+
const step3 = step2 * 3;
|
|
12622
|
+
const Ar = data[off];
|
|
12623
|
+
const Br = data[off + step2];
|
|
12624
|
+
const Cr = data[off + step22];
|
|
12625
|
+
const Dr = data[off + step3];
|
|
12626
|
+
const T0r = Ar + Cr;
|
|
12627
|
+
const T1r = Ar - Cr;
|
|
12628
|
+
const T2r = Br + Dr;
|
|
12629
|
+
const T3r = inv * (Br - Dr);
|
|
12630
|
+
const FAr = T0r + T2r;
|
|
12631
|
+
const FBr = T1r;
|
|
12632
|
+
const FBi = -T3r;
|
|
12633
|
+
const FCr = T0r - T2r;
|
|
12634
|
+
const FDr = T1r;
|
|
12635
|
+
const FDi = T3r;
|
|
12636
|
+
out[outOff] = FAr;
|
|
12637
|
+
out[outOff + 1] = 0;
|
|
12638
|
+
out[outOff + 2] = FBr;
|
|
12639
|
+
out[outOff + 3] = FBi;
|
|
12640
|
+
out[outOff + 4] = FCr;
|
|
12641
|
+
out[outOff + 5] = 0;
|
|
12642
|
+
out[outOff + 6] = FDr;
|
|
12643
|
+
out[outOff + 7] = FDi;
|
|
12644
|
+
};
|
|
12645
|
+
const FFT$1 = /* @__PURE__ */ getDefaultExportFromCjs(fft);
|
|
12646
|
+
class AudioChannel {
|
|
12647
|
+
// Identity
|
|
12648
|
+
streamIndex;
|
|
12649
|
+
// Web Audio nodes (owned per-channel, connected to shared AudioContext)
|
|
12650
|
+
mediaStreamSource = null;
|
|
12651
|
+
analyser = null;
|
|
12652
|
+
workletNode = null;
|
|
12653
|
+
analysisMode = "analyser";
|
|
12654
|
+
workletReady = false;
|
|
12655
|
+
currentStream = null;
|
|
12656
|
+
// FFT resources
|
|
12657
|
+
fftSize;
|
|
12658
|
+
fftEngine = null;
|
|
12659
|
+
fftInput = null;
|
|
12660
|
+
fftOutput = null;
|
|
12661
|
+
hannWindow = null;
|
|
12662
|
+
frequencyData = null;
|
|
12663
|
+
fftMagnitude = null;
|
|
12664
|
+
fftMagnitudeDb = null;
|
|
12665
|
+
fftPhase = null;
|
|
12666
|
+
timeDomainData = null;
|
|
12667
|
+
// Auto-gain normalization
|
|
12668
|
+
volumeAutoGain;
|
|
12669
|
+
bandAutoGain;
|
|
12670
|
+
// Envelope followers for band/volume smoothing
|
|
12671
|
+
envelopeFollowers;
|
|
12672
|
+
// Per-channel noise floor tracking
|
|
12673
|
+
bandNoiseFloor = {
|
|
12674
|
+
low: 1e-4,
|
|
12675
|
+
lowMid: 1e-4,
|
|
12676
|
+
mid: 1e-4,
|
|
12677
|
+
highMid: 1e-4,
|
|
12678
|
+
high: 1e-4
|
|
12679
|
+
};
|
|
12680
|
+
// Analysis state
|
|
12681
|
+
audioState = {
|
|
12682
|
+
isConnected: false,
|
|
12683
|
+
volume: { current: 0, peak: 0, smoothed: 0 },
|
|
12684
|
+
bands: {
|
|
12685
|
+
low: 0,
|
|
12686
|
+
lowMid: 0,
|
|
12687
|
+
mid: 0,
|
|
12688
|
+
highMid: 0,
|
|
12689
|
+
high: 0,
|
|
12690
|
+
lowSmoothed: 0,
|
|
12691
|
+
lowMidSmoothed: 0,
|
|
12692
|
+
midSmoothed: 0,
|
|
12693
|
+
highMidSmoothed: 0,
|
|
12694
|
+
highSmoothed: 0
|
|
12695
|
+
},
|
|
12696
|
+
spectral: { brightness: 0, flatness: 0 }
|
|
12697
|
+
};
|
|
12698
|
+
// Waveform data for transfer to worker
|
|
12699
|
+
lastWaveformFrame = null;
|
|
12700
|
+
// Analysis loop state
|
|
12701
|
+
isAnalysisRunning = false;
|
|
12702
|
+
workletFrameCount = 0;
|
|
12703
|
+
lastFrameTime = 0;
|
|
12704
|
+
lastAnalysisTimestamp = 0;
|
|
12705
|
+
lastDtMs = 0;
|
|
12706
|
+
analysisTicks = 0;
|
|
12707
|
+
currentSampleRate = 44100;
|
|
12708
|
+
constructor(streamIndex, fftSize) {
|
|
12709
|
+
this.streamIndex = streamIndex;
|
|
12710
|
+
this.fftSize = fftSize;
|
|
12711
|
+
this.volumeAutoGain = new AutoGain(3e3, 60);
|
|
12712
|
+
this.bandAutoGain = {
|
|
12713
|
+
low: new AutoGain(3e3, 60),
|
|
12714
|
+
lowMid: new AutoGain(3e3, 60),
|
|
12715
|
+
mid: new AutoGain(3e3, 60),
|
|
12716
|
+
highMid: new AutoGain(3e3, 60),
|
|
12717
|
+
high: new AutoGain(3e3, 60)
|
|
12718
|
+
};
|
|
12719
|
+
const sampleRate = 60;
|
|
12720
|
+
this.envelopeFollowers = {
|
|
12721
|
+
volumeSmoothed: new EnvelopeFollower(50, 200, sampleRate),
|
|
12722
|
+
lowSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12723
|
+
lowMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12724
|
+
midSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12725
|
+
highMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12726
|
+
highSmoothed: new EnvelopeFollower(20, 150, sampleRate)
|
|
12727
|
+
};
|
|
12728
|
+
this.refreshFFTResources();
|
|
12501
12729
|
}
|
|
12502
|
-
|
|
12503
|
-
|
|
12504
|
-
* Tries the expected refinementIndex first, then scans all positions.
|
|
12505
|
-
* Returns the matched position index, or -1 if no match.
|
|
12506
|
-
*/
|
|
12507
|
-
findMatchingPosition(s, ioi) {
|
|
12508
|
-
if (!s.pattern) return -1;
|
|
12509
|
-
const expectedPos = s.refinementIndex % s.pattern.length;
|
|
12510
|
-
const expectedInterval = s.pattern[expectedPos];
|
|
12511
|
-
if (Math.abs(ioi - expectedInterval) / expectedInterval <= FUZZY_TOLERANCE) {
|
|
12512
|
-
return expectedPos;
|
|
12513
|
-
}
|
|
12514
|
-
let bestPos = -1;
|
|
12515
|
-
let bestDeviation = Infinity;
|
|
12516
|
-
for (let i = 0; i < s.pattern.length; i++) {
|
|
12517
|
-
if (i === expectedPos) continue;
|
|
12518
|
-
const deviation = Math.abs(ioi - s.pattern[i]) / s.pattern[i];
|
|
12519
|
-
if (deviation <= FUZZY_TOLERANCE && deviation < bestDeviation) {
|
|
12520
|
-
bestDeviation = deviation;
|
|
12521
|
-
bestPos = i;
|
|
12522
|
-
}
|
|
12523
|
-
}
|
|
12524
|
-
return bestPos;
|
|
12730
|
+
static create(streamIndex, fftSize = 2048) {
|
|
12731
|
+
return new AudioChannel(streamIndex, fftSize);
|
|
12525
12732
|
}
|
|
12526
|
-
|
|
12527
|
-
|
|
12528
|
-
|
|
12529
|
-
|
|
12530
|
-
|
|
12531
|
-
|
|
12532
|
-
|
|
12533
|
-
|
|
12733
|
+
refreshFFTResources() {
|
|
12734
|
+
const binCount = this.fftSize / 2;
|
|
12735
|
+
this.frequencyData = new Uint8Array(binCount);
|
|
12736
|
+
this.fftMagnitude = new Float32Array(binCount);
|
|
12737
|
+
this.fftMagnitudeDb = new Float32Array(binCount);
|
|
12738
|
+
this.fftPhase = new Float32Array(binCount);
|
|
12739
|
+
this.timeDomainData = new Float32Array(this.fftSize);
|
|
12740
|
+
this.fftEngine = new FFT$1(this.fftSize);
|
|
12741
|
+
this.fftInput = new Float32Array(this.fftSize * 2);
|
|
12742
|
+
this.fftOutput = new Float32Array(this.fftSize * 2);
|
|
12743
|
+
this.hannWindow = new Float32Array(this.fftSize);
|
|
12744
|
+
const twoPi = Math.PI * 2;
|
|
12745
|
+
for (let i = 0; i < this.fftSize; i++) {
|
|
12746
|
+
this.hannWindow[i] = 0.5 * (1 - Math.cos(twoPi * i / (this.fftSize - 1)));
|
|
12534
12747
|
}
|
|
12535
|
-
|
|
12748
|
+
this.analysisTicks = 0;
|
|
12749
|
+
this.workletFrameCount = 0;
|
|
12536
12750
|
}
|
|
12537
|
-
|
|
12538
|
-
|
|
12539
|
-
|
|
12540
|
-
|
|
12541
|
-
|
|
12542
|
-
|
|
12543
|
-
|
|
12544
|
-
|
|
12545
|
-
|
|
12546
|
-
|
|
12547
|
-
|
|
12548
|
-
|
|
12549
|
-
|
|
12550
|
-
|
|
12551
|
-
|
|
12552
|
-
|
|
12553
|
-
|
|
12554
|
-
|
|
12555
|
-
|
|
12556
|
-
|
|
12557
|
-
|
|
12558
|
-
let sum = 0;
|
|
12559
|
-
for (const grp of groups) sum += grp[i];
|
|
12560
|
-
return sum / groups.length;
|
|
12561
|
-
});
|
|
12562
|
-
let allMatch = true;
|
|
12563
|
-
for (const grp of groups) {
|
|
12564
|
-
for (let i = 0; i < L; i++) {
|
|
12565
|
-
const deviation = Math.abs(grp[i] - avgGroup[i]) / avgGroup[i];
|
|
12566
|
-
if (deviation > tolerance) {
|
|
12567
|
-
allMatch = false;
|
|
12568
|
-
break;
|
|
12569
|
-
}
|
|
12570
|
-
}
|
|
12571
|
-
if (!allMatch) break;
|
|
12572
|
-
}
|
|
12573
|
-
if (allMatch) {
|
|
12574
|
-
return avgGroup;
|
|
12575
|
-
}
|
|
12751
|
+
resetValues() {
|
|
12752
|
+
this.audioState.volume.current = 0;
|
|
12753
|
+
this.audioState.volume.peak = 0;
|
|
12754
|
+
this.audioState.volume.smoothed = 0;
|
|
12755
|
+
this.audioState.bands.low = 0;
|
|
12756
|
+
this.audioState.bands.lowMid = 0;
|
|
12757
|
+
this.audioState.bands.mid = 0;
|
|
12758
|
+
this.audioState.bands.highMid = 0;
|
|
12759
|
+
this.audioState.bands.high = 0;
|
|
12760
|
+
this.audioState.bands.lowSmoothed = 0;
|
|
12761
|
+
this.audioState.bands.lowMidSmoothed = 0;
|
|
12762
|
+
this.audioState.bands.midSmoothed = 0;
|
|
12763
|
+
this.audioState.bands.highMidSmoothed = 0;
|
|
12764
|
+
this.audioState.bands.highSmoothed = 0;
|
|
12765
|
+
this.audioState.spectral.brightness = 0;
|
|
12766
|
+
this.audioState.spectral.flatness = 0;
|
|
12767
|
+
}
|
|
12768
|
+
disconnectNodes() {
|
|
12769
|
+
if (this.mediaStreamSource) {
|
|
12770
|
+
this.mediaStreamSource.disconnect();
|
|
12771
|
+
this.mediaStreamSource = null;
|
|
12576
12772
|
}
|
|
12577
|
-
|
|
12578
|
-
|
|
12579
|
-
|
|
12580
|
-
* Apply a recognized pattern to the instrument state.
|
|
12581
|
-
* Sets pattern, mode, and initializes replay anchor.
|
|
12582
|
-
*/
|
|
12583
|
-
applyPattern(instrument, pattern) {
|
|
12584
|
-
const s = this.state[instrument];
|
|
12585
|
-
s.pattern = pattern;
|
|
12586
|
-
s.mode = "pattern";
|
|
12587
|
-
s.refinementIndex = 0;
|
|
12588
|
-
s.refinementCounts = new Array(pattern.length).fill(MIN_REPETITIONS);
|
|
12589
|
-
if (s.replayLastEventTime === 0 || s.replayLastEventTime < s.lastTapTime) {
|
|
12590
|
-
s.replayLastEventTime = s.lastTapTime;
|
|
12591
|
-
s.replayIndex = 0;
|
|
12773
|
+
if (this.analyser) {
|
|
12774
|
+
this.analyser.disconnect();
|
|
12775
|
+
this.analyser = null;
|
|
12592
12776
|
}
|
|
12593
|
-
|
|
12594
|
-
|
|
12595
|
-
|
|
12596
|
-
|
|
12597
|
-
|
|
12598
|
-
strength: 0.85,
|
|
12599
|
-
isPredicted: false,
|
|
12600
|
-
bpm
|
|
12601
|
-
};
|
|
12602
|
-
}
|
|
12603
|
-
checkPatternReplay(s, instrument, now, bpm) {
|
|
12604
|
-
if (!s.pattern || s.pattern.length === 0) return [];
|
|
12605
|
-
const events = [];
|
|
12606
|
-
const maxEventsPerFrame = 3;
|
|
12607
|
-
for (let safety = 0; safety < maxEventsPerFrame; safety++) {
|
|
12608
|
-
const expectedInterval = s.pattern[s.replayIndex % s.pattern.length];
|
|
12609
|
-
const elapsed = now - s.replayLastEventTime;
|
|
12610
|
-
if (elapsed >= expectedInterval) {
|
|
12611
|
-
s.replayLastEventTime += expectedInterval;
|
|
12612
|
-
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
12613
|
-
events.push({
|
|
12614
|
-
type: instrument,
|
|
12615
|
-
time: s.replayLastEventTime,
|
|
12616
|
-
strength: 0.8,
|
|
12617
|
-
isPredicted: true,
|
|
12618
|
-
bpm
|
|
12619
|
-
});
|
|
12620
|
-
} else {
|
|
12621
|
-
break;
|
|
12777
|
+
if (this.workletNode) {
|
|
12778
|
+
try {
|
|
12779
|
+
this.workletNode.port.onmessage = null;
|
|
12780
|
+
this.workletNode.disconnect();
|
|
12781
|
+
} catch (_) {
|
|
12622
12782
|
}
|
|
12783
|
+
this.workletNode = null;
|
|
12623
12784
|
}
|
|
12624
|
-
|
|
12785
|
+
this.workletReady = false;
|
|
12786
|
+
this.analysisMode = "analyser";
|
|
12625
12787
|
}
|
|
12626
|
-
|
|
12627
|
-
|
|
12628
|
-
|
|
12629
|
-
|
|
12788
|
+
destroy() {
|
|
12789
|
+
this.disconnectNodes();
|
|
12790
|
+
this.frequencyData = null;
|
|
12791
|
+
this.timeDomainData = null;
|
|
12792
|
+
this.fftMagnitude = null;
|
|
12793
|
+
this.fftMagnitudeDb = null;
|
|
12794
|
+
this.fftPhase = null;
|
|
12795
|
+
this.fftEngine = null;
|
|
12796
|
+
this.fftInput = null;
|
|
12797
|
+
this.fftOutput = null;
|
|
12798
|
+
this.hannWindow = null;
|
|
12799
|
+
this.lastWaveformFrame = null;
|
|
12800
|
+
this.currentStream = null;
|
|
12801
|
+
this.isAnalysisRunning = false;
|
|
12802
|
+
this.audioState.isConnected = false;
|
|
12803
|
+
this.resetValues();
|
|
12630
12804
|
}
|
|
12631
12805
|
}
|
|
12632
12806
|
class AudioSystem {
|
|
12633
|
-
//
|
|
12807
|
+
// Shared AudioContext (one for all channels)
|
|
12634
12808
|
audioContext = null;
|
|
12635
|
-
analyser = null;
|
|
12636
|
-
mediaStreamSource = null;
|
|
12637
|
-
currentStream = null;
|
|
12638
|
-
analysisMode = "analyser";
|
|
12639
|
-
workletNode = null;
|
|
12640
|
-
workletReady = false;
|
|
12641
12809
|
workletRegistered = false;
|
|
12642
|
-
|
|
12643
|
-
|
|
12644
|
-
|
|
12645
|
-
|
|
12646
|
-
|
|
12647
|
-
|
|
12648
|
-
|
|
12649
|
-
|
|
12810
|
+
// Main audio channel (index 0, full analysis with beat detection)
|
|
12811
|
+
mainChannel;
|
|
12812
|
+
// Additional audio channels (lightweight analysis, no beat detection)
|
|
12813
|
+
additionalChannels = /* @__PURE__ */ new Map();
|
|
12814
|
+
// Shared analysis timer (one interval for all analyser-mode channels)
|
|
12815
|
+
analysisInterval = null;
|
|
12816
|
+
analysisIntervalMs = 8;
|
|
12817
|
+
// Staleness timer (shared across all channels)
|
|
12818
|
+
stalenessTimer = null;
|
|
12819
|
+
static STALENESS_THRESHOLD_MS = 500;
|
|
12820
|
+
// Beat-specific state (main channel only)
|
|
12650
12821
|
/** Tracks the last non-zero BPM chosen for output to avoid dropping to 0 */
|
|
12651
12822
|
lastNonZeroBpm = 120;
|
|
12652
12823
|
/** Tracks which source provided the current BPM (pll | tempo | carry | default) */
|
|
12653
12824
|
lastBpmSource = "default";
|
|
12654
|
-
workletFrameCount = 0;
|
|
12655
|
-
lastFrameTime = 0;
|
|
12656
|
-
stalenessTimer = null;
|
|
12657
|
-
static STALENESS_THRESHOLD_MS = 500;
|
|
12658
|
-
analysisTicks = 0;
|
|
12659
12825
|
lastPhaseLogTime = 0;
|
|
12660
12826
|
onsetLogBuffer = [];
|
|
12661
12827
|
// Debug logging control
|
|
@@ -12671,10 +12837,7 @@ class AudioSystem {
|
|
|
12671
12837
|
stateManager;
|
|
12672
12838
|
// Previous frame's kick detection (for PLL sync with 1-frame lag)
|
|
12673
12839
|
lastKickDetected = false;
|
|
12674
|
-
|
|
12675
|
-
volumeAutoGain;
|
|
12676
|
-
bandAutoGain;
|
|
12677
|
-
/** Raw band snapshot before auto-gain (for debug) */
|
|
12840
|
+
/** Raw band snapshot before auto-gain (for debug, main channel only) */
|
|
12678
12841
|
rawBandsPreGain = {
|
|
12679
12842
|
low: 0,
|
|
12680
12843
|
lowMid: 0,
|
|
@@ -12682,16 +12845,13 @@ class AudioSystem {
|
|
|
12682
12845
|
highMid: 0,
|
|
12683
12846
|
high: 0
|
|
12684
12847
|
};
|
|
12685
|
-
/** Last dt used in analysis loop (ms) */
|
|
12686
|
-
lastDtMs = 0;
|
|
12687
|
-
/** Flag to track if we've logged detection method (to avoid spam) */
|
|
12688
12848
|
bandNames = ["low", "lowMid", "mid", "highMid", "high"];
|
|
12689
12849
|
essentiaBandHistories = /* @__PURE__ */ new Map();
|
|
12690
12850
|
essentiaHistoryWindowMs = 5e3;
|
|
12691
12851
|
// Per-instrument onset tap manager
|
|
12692
12852
|
onsetTapManager;
|
|
12693
|
-
//
|
|
12694
|
-
|
|
12853
|
+
// Beat-specific envelope followers (main channel only, managed by BeatStateManager)
|
|
12854
|
+
beatEnvelopeFollowers;
|
|
12695
12855
|
// Feature enable flags
|
|
12696
12856
|
beatDetectionEnabled = true;
|
|
12697
12857
|
onsetDetectionEnabled = true;
|
|
@@ -12747,80 +12907,94 @@ class AudioSystem {
|
|
|
12747
12907
|
return clone;
|
|
12748
12908
|
}
|
|
12749
12909
|
/**
|
|
12750
|
-
* Handle frames pushed from AudioWorklet
|
|
12910
|
+
* Handle frames pushed from AudioWorklet for main channel
|
|
12911
|
+
*/
|
|
12912
|
+
handleMainWorkletFrame(frame, sampleRate, timestampMs) {
|
|
12913
|
+
if (!this.mainChannel.isAnalysisRunning) return;
|
|
12914
|
+
this.mainChannel.workletFrameCount++;
|
|
12915
|
+
this.analyzeMainFrame(frame, sampleRate || this.mainChannel.currentSampleRate, timestampMs || performance.now());
|
|
12916
|
+
}
|
|
12917
|
+
/**
|
|
12918
|
+
* Handle frames pushed from AudioWorklet for an additional channel
|
|
12751
12919
|
*/
|
|
12752
|
-
|
|
12753
|
-
if (!
|
|
12754
|
-
|
|
12755
|
-
this.
|
|
12920
|
+
handleAdditionalWorkletFrame(channel, frame, sampleRate, timestampMs) {
|
|
12921
|
+
if (!channel.isAnalysisRunning) return;
|
|
12922
|
+
channel.workletFrameCount++;
|
|
12923
|
+
this.analyzeAdditionalFrame(channel, frame, sampleRate || channel.currentSampleRate, timestampMs || performance.now());
|
|
12756
12924
|
}
|
|
12757
12925
|
/**
|
|
12758
|
-
*
|
|
12926
|
+
* Common analysis pipeline for any channel (FFT, volume, bands, auto-gain, spectral, smoothing).
|
|
12927
|
+
* Returns raw band energies (pre-auto-gain) for optional beat processing by the caller.
|
|
12759
12928
|
*/
|
|
12760
|
-
|
|
12761
|
-
if (!frame || frame.length === 0) return;
|
|
12762
|
-
|
|
12763
|
-
|
|
12764
|
-
|
|
12929
|
+
analyzeChannelFrame(ch, frame, sampleRate, timestampMs) {
|
|
12930
|
+
if (!frame || frame.length === 0) return null;
|
|
12931
|
+
ch.lastFrameTime = performance.now();
|
|
12932
|
+
ch.analysisTicks++;
|
|
12933
|
+
if (ch.workletFrameCount === 1) {
|
|
12934
|
+
this.debugLog(`[AudioSystem] First frame received for channel ${ch.streamIndex}`, { sampleRate, mode: ch.analysisMode });
|
|
12765
12935
|
}
|
|
12766
|
-
const dtMs =
|
|
12767
|
-
|
|
12768
|
-
|
|
12769
|
-
|
|
12936
|
+
const dtMs = ch.lastAnalysisTimestamp > 0 ? timestampMs - ch.lastAnalysisTimestamp : 1e3 / 60;
|
|
12937
|
+
ch.lastAnalysisTimestamp = timestampMs;
|
|
12938
|
+
ch.lastDtMs = dtMs;
|
|
12939
|
+
ch.currentSampleRate = sampleRate;
|
|
12770
12940
|
const { rms, peak } = this.calculateVolumeMetrics(frame);
|
|
12771
|
-
|
|
12772
|
-
|
|
12773
|
-
|
|
12774
|
-
const fftResult = this.
|
|
12775
|
-
if (!fftResult)
|
|
12776
|
-
|
|
12777
|
-
|
|
12778
|
-
|
|
12779
|
-
return;
|
|
12780
|
-
}
|
|
12781
|
-
const { magnitudes, magnitudesDb, phases, frequencyData, maxMagnitude } = fftResult;
|
|
12782
|
-
this.frequencyData = frequencyData;
|
|
12783
|
-
this.fftMagnitude = magnitudes;
|
|
12784
|
-
this.fftMagnitudeDb = magnitudesDb;
|
|
12785
|
-
this.fftPhase = phases;
|
|
12786
|
-
const bandEnergies = this.calculateFrequencyBandsFromMagnitude(magnitudes, sampleRate, maxMagnitude);
|
|
12787
|
-
const isSilent = maxMagnitude <= 1e-7 || this.audioState.volume.current <= 1e-6;
|
|
12941
|
+
ch.audioState.volume.current = rms;
|
|
12942
|
+
ch.audioState.volume.peak = peak;
|
|
12943
|
+
ch.lastWaveformFrame = frame;
|
|
12944
|
+
const fftResult = this.computeChannelFFT(ch, frame);
|
|
12945
|
+
if (!fftResult) return null;
|
|
12946
|
+
const { magnitudes, maxMagnitude } = fftResult;
|
|
12947
|
+
const bandEnergies = this.calculateChannelBands(ch, magnitudes, sampleRate, maxMagnitude);
|
|
12948
|
+
const isSilent = maxMagnitude <= 1e-7 || ch.audioState.volume.current <= 1e-6;
|
|
12788
12949
|
if (isSilent) {
|
|
12789
|
-
|
|
12790
|
-
...
|
|
12950
|
+
ch.audioState.bands = {
|
|
12951
|
+
...ch.audioState.bands,
|
|
12791
12952
|
low: 0,
|
|
12792
12953
|
lowMid: 0,
|
|
12793
12954
|
mid: 0,
|
|
12794
12955
|
highMid: 0,
|
|
12795
12956
|
high: 0
|
|
12796
12957
|
};
|
|
12797
|
-
|
|
12798
|
-
this.
|
|
12799
|
-
|
|
12800
|
-
this.audioState.beat.kick = decayed.kick;
|
|
12801
|
-
this.audioState.beat.snare = decayed.snare;
|
|
12802
|
-
this.audioState.beat.hat = decayed.hat;
|
|
12803
|
-
this.audioState.beat.any = decayed.any;
|
|
12804
|
-
this.audioState.beat.kickSmoothed = decayed.kickSmoothed;
|
|
12805
|
-
this.audioState.beat.snareSmoothed = decayed.snareSmoothed;
|
|
12806
|
-
this.audioState.beat.hatSmoothed = decayed.hatSmoothed;
|
|
12807
|
-
this.audioState.beat.anySmoothed = decayed.anySmoothed;
|
|
12808
|
-
this.audioState.beat.events = [];
|
|
12809
|
-
this.updateSmoothBands(dtMs);
|
|
12810
|
-
this.sendAnalysisResultsToWorker();
|
|
12811
|
-
return;
|
|
12958
|
+
ch.audioState.spectral = { brightness: 0, flatness: 0 };
|
|
12959
|
+
this.updateChannelSmoothBands(ch, dtMs);
|
|
12960
|
+
return null;
|
|
12812
12961
|
}
|
|
12813
|
-
this.rawBandsPreGain = { ...bandEnergies };
|
|
12814
12962
|
if (this.autoGainEnabled) {
|
|
12815
|
-
this.
|
|
12963
|
+
this.applyChannelAutoGain(ch);
|
|
12964
|
+
}
|
|
12965
|
+
this.calculateChannelSpectral(ch, magnitudes, sampleRate, maxMagnitude);
|
|
12966
|
+
this.updateChannelSmoothBands(ch, dtMs);
|
|
12967
|
+
return bandEnergies;
|
|
12968
|
+
}
|
|
12969
|
+
/**
|
|
12970
|
+
* Full analysis for the main channel (common analysis + beat pipeline).
|
|
12971
|
+
*/
|
|
12972
|
+
analyzeMainFrame(frame, sampleRate, timestampMs) {
|
|
12973
|
+
const ch = this.mainChannel;
|
|
12974
|
+
const rawBands = this.analyzeChannelFrame(ch, frame, sampleRate, timestampMs);
|
|
12975
|
+
const dtMs = ch.lastDtMs;
|
|
12976
|
+
if (!rawBands) {
|
|
12977
|
+
const decayed = this.stateManager.processEnvelopeDecay(dtMs);
|
|
12978
|
+
this.audioStateBeat.kick = decayed.kick;
|
|
12979
|
+
this.audioStateBeat.snare = decayed.snare;
|
|
12980
|
+
this.audioStateBeat.hat = decayed.hat;
|
|
12981
|
+
this.audioStateBeat.any = decayed.any;
|
|
12982
|
+
this.audioStateBeat.kickSmoothed = decayed.kickSmoothed;
|
|
12983
|
+
this.audioStateBeat.snareSmoothed = decayed.snareSmoothed;
|
|
12984
|
+
this.audioStateBeat.hatSmoothed = decayed.hatSmoothed;
|
|
12985
|
+
this.audioStateBeat.anySmoothed = decayed.anySmoothed;
|
|
12986
|
+
this.audioStateBeat.events = [];
|
|
12987
|
+
this.rawBandsPreGain = { low: 0, lowMid: 0, mid: 0, highMid: 0, high: 0 };
|
|
12988
|
+
this.sendChannelResults(ch, true);
|
|
12989
|
+
return;
|
|
12816
12990
|
}
|
|
12817
|
-
this.
|
|
12991
|
+
this.rawBandsPreGain = { ...rawBands };
|
|
12818
12992
|
if (this.onsetDetectionEnabled && this.beatDetectionEnabled) {
|
|
12819
|
-
const beatState = this.runBeatPipeline(
|
|
12993
|
+
const beatState = this.runBeatPipeline(rawBands, dtMs, timestampMs, sampleRate);
|
|
12820
12994
|
if (beatState) {
|
|
12821
12995
|
const { phase: _phase, bar: _bar, debug: _debug, ...beatForState } = beatState;
|
|
12822
12996
|
const now = performance.now();
|
|
12823
|
-
this.
|
|
12997
|
+
this.audioStateBeat = this.onsetTapManager.processFrame(beatForState, now, dtMs);
|
|
12824
12998
|
if (this.debugMode) {
|
|
12825
12999
|
const pllState = this.pll.getState();
|
|
12826
13000
|
const currentState = this.stateManager.getState();
|
|
@@ -12835,9 +13009,7 @@ class AudioSystem {
|
|
|
12835
13009
|
if (bpmVariance !== void 0) trackingData.bpmVariance = bpmVariance;
|
|
12836
13010
|
const compressed = this.stateManager.isCompressed();
|
|
12837
13011
|
if (compressed) trackingData.compressed = compressed;
|
|
12838
|
-
if (beatState.events.some((e) => e.type === "kick"))
|
|
12839
|
-
trackingData.kickPhase = pllState.phase;
|
|
12840
|
-
}
|
|
13012
|
+
if (beatState.events.some((e) => e.type === "kick")) trackingData.kickPhase = pllState.phase;
|
|
12841
13013
|
if (beatState.debug?.pllGain !== void 0) trackingData.pllGain = beatState.debug.pllGain;
|
|
12842
13014
|
if (beatState.debug?.tempoConfidence !== void 0) trackingData.tempoConf = beatState.debug.tempoConfidence;
|
|
12843
13015
|
if (beatState.debug?.trackingConfidence !== void 0) trackingData.trackingConf = beatState.debug.trackingConfidence;
|
|
@@ -12855,11 +13027,11 @@ class AudioSystem {
|
|
|
12855
13027
|
state: currentState
|
|
12856
13028
|
},
|
|
12857
13029
|
{
|
|
12858
|
-
low:
|
|
12859
|
-
lowMid:
|
|
12860
|
-
mid:
|
|
12861
|
-
highMid:
|
|
12862
|
-
high:
|
|
13030
|
+
low: ch.audioState.bands.low,
|
|
13031
|
+
lowMid: ch.audioState.bands.lowMid,
|
|
13032
|
+
mid: ch.audioState.bands.mid,
|
|
13033
|
+
highMid: ch.audioState.bands.highMid,
|
|
13034
|
+
high: ch.audioState.bands.high
|
|
12863
13035
|
},
|
|
12864
13036
|
this.stateManager.getLastRejections(),
|
|
12865
13037
|
trackingData
|
|
@@ -12867,70 +13039,62 @@ class AudioSystem {
|
|
|
12867
13039
|
}
|
|
12868
13040
|
}
|
|
12869
13041
|
}
|
|
12870
|
-
this.
|
|
12871
|
-
|
|
13042
|
+
this.sendChannelResults(ch, true);
|
|
13043
|
+
}
|
|
13044
|
+
/**
|
|
13045
|
+
* Lightweight analysis for additional/device channels (no beat detection).
|
|
13046
|
+
*/
|
|
13047
|
+
analyzeAdditionalFrame(channel, frame, sampleRate, timestampMs) {
|
|
13048
|
+
this.analyzeChannelFrame(channel, frame, sampleRate, timestampMs);
|
|
13049
|
+
this.sendChannelResults(channel, false);
|
|
12872
13050
|
}
|
|
12873
13051
|
/**
|
|
12874
|
-
* Compute FFT
|
|
13052
|
+
* Compute FFT on a specific channel's buffers
|
|
12875
13053
|
*/
|
|
12876
|
-
|
|
12877
|
-
if (!
|
|
12878
|
-
if (this.debugMode && (
|
|
12879
|
-
this.debugLog(
|
|
13054
|
+
computeChannelFFT(ch, frame) {
|
|
13055
|
+
if (!ch.fftEngine || !ch.fftInput || !ch.fftOutput || !ch.hannWindow || !ch.frequencyData || !ch.fftMagnitude || !ch.fftMagnitudeDb || !ch.fftPhase) {
|
|
13056
|
+
if (this.debugMode && (ch.analysisTicks === 1 || ch.analysisTicks % 240 === 0)) {
|
|
13057
|
+
this.debugLog(`[AudioSystem][computeFFT] RETURNING NULL - resources missing at tick ${ch.analysisTicks}, channel ${ch.streamIndex}`);
|
|
12880
13058
|
}
|
|
12881
13059
|
return null;
|
|
12882
13060
|
}
|
|
12883
|
-
const binCount =
|
|
12884
|
-
const input =
|
|
12885
|
-
const output =
|
|
13061
|
+
const binCount = ch.fftSize / 2;
|
|
13062
|
+
const input = ch.fftInput;
|
|
13063
|
+
const output = ch.fftOutput;
|
|
12886
13064
|
input.fill(0);
|
|
12887
|
-
const frameLength = Math.min(frame.length,
|
|
12888
|
-
let frameMax = 0;
|
|
12889
|
-
for (let i = 0; i < Math.min(32, frame.length); i++) {
|
|
12890
|
-
if (Math.abs(frame[i]) > frameMax) frameMax = Math.abs(frame[i]);
|
|
12891
|
-
}
|
|
13065
|
+
const frameLength = Math.min(frame.length, ch.fftSize);
|
|
12892
13066
|
for (let i = 0; i < frameLength; i++) {
|
|
12893
|
-
input[2 * i] = frame[i] *
|
|
13067
|
+
input[2 * i] = frame[i] * ch.hannWindow[i];
|
|
12894
13068
|
input[2 * i + 1] = 0;
|
|
12895
13069
|
}
|
|
12896
|
-
|
|
12897
|
-
for (let i = 0; i < Math.min(64, input.length); i += 2) {
|
|
12898
|
-
if (Math.abs(input[i]) > inputMax) inputMax = Math.abs(input[i]);
|
|
12899
|
-
}
|
|
12900
|
-
this.fftEngine.transform(output, input);
|
|
12901
|
-
let outputMax = 0;
|
|
12902
|
-
for (let i = 0; i < Math.min(64, output.length); i++) {
|
|
12903
|
-
if (Math.abs(output[i]) > outputMax) outputMax = Math.abs(output[i]);
|
|
12904
|
-
}
|
|
13070
|
+
ch.fftEngine.transform(output, input);
|
|
12905
13071
|
let maxMagnitude = 0;
|
|
12906
13072
|
for (let i = 0; i < binCount; i++) {
|
|
12907
13073
|
const re = output[2 * i];
|
|
12908
13074
|
const im = output[2 * i + 1];
|
|
12909
|
-
const mag = Math.sqrt(re * re + im * im) /
|
|
12910
|
-
|
|
12911
|
-
if (mag > maxMagnitude)
|
|
12912
|
-
|
|
12913
|
-
}
|
|
12914
|
-
this.fftPhase[i] = Math.atan2(im, re);
|
|
13075
|
+
const mag = Math.sqrt(re * re + im * im) / ch.fftSize;
|
|
13076
|
+
ch.fftMagnitude[i] = mag;
|
|
13077
|
+
if (mag > maxMagnitude) maxMagnitude = mag;
|
|
13078
|
+
ch.fftPhase[i] = Math.atan2(im, re);
|
|
12915
13079
|
}
|
|
12916
13080
|
if (maxMagnitude <= 0) {
|
|
12917
|
-
|
|
13081
|
+
ch.frequencyData.fill(0);
|
|
12918
13082
|
for (let i = 0; i < binCount; i++) {
|
|
12919
|
-
|
|
13083
|
+
ch.fftMagnitudeDb[i] = 20 * Math.log10(ch.fftMagnitude[i] + 1e-12);
|
|
12920
13084
|
}
|
|
12921
13085
|
} else {
|
|
12922
13086
|
for (let i = 0; i < binCount; i++) {
|
|
12923
|
-
const mag =
|
|
13087
|
+
const mag = ch.fftMagnitude[i];
|
|
12924
13088
|
const norm = mag / maxMagnitude;
|
|
12925
|
-
|
|
12926
|
-
|
|
13089
|
+
ch.frequencyData[i] = Math.min(255, Math.max(0, Math.round(norm * 255)));
|
|
13090
|
+
ch.fftMagnitudeDb[i] = 20 * Math.log10(mag + 1e-12);
|
|
12927
13091
|
}
|
|
12928
13092
|
}
|
|
12929
13093
|
return {
|
|
12930
|
-
magnitudes:
|
|
12931
|
-
magnitudesDb:
|
|
12932
|
-
phases:
|
|
12933
|
-
frequencyData:
|
|
13094
|
+
magnitudes: ch.fftMagnitude,
|
|
13095
|
+
magnitudesDb: ch.fftMagnitudeDb,
|
|
13096
|
+
phases: ch.fftPhase,
|
|
13097
|
+
frequencyData: ch.frequencyData,
|
|
12934
13098
|
maxMagnitude
|
|
12935
13099
|
};
|
|
12936
13100
|
}
|
|
@@ -12950,9 +13114,9 @@ class AudioSystem {
|
|
|
12950
13114
|
return { rms, peak };
|
|
12951
13115
|
}
|
|
12952
13116
|
/**
|
|
12953
|
-
* Calculate perceptual/log-ish bands
|
|
13117
|
+
* Calculate perceptual/log-ish bands on a specific channel
|
|
12954
13118
|
*/
|
|
12955
|
-
|
|
13119
|
+
calculateChannelBands(ch, magnitudes, sampleRate, maxMagnitude) {
|
|
12956
13120
|
const nyquist = sampleRate / 2;
|
|
12957
13121
|
const binCount = magnitudes.length;
|
|
12958
13122
|
const bands = {
|
|
@@ -12976,33 +13140,32 @@ class AudioSystem {
|
|
|
12976
13140
|
const average = count > 0 ? sum / count : 0;
|
|
12977
13141
|
const rawNormalized = maxMagnitude > 0 ? Math.min(1, average / maxMagnitude) : 0;
|
|
12978
13142
|
rawBands[bandName] = rawNormalized;
|
|
12979
|
-
const prevFloor =
|
|
13143
|
+
const prevFloor = ch.bandNoiseFloor[bandName] ?? 1e-4;
|
|
12980
13144
|
const newFloor = prevFloor * 0.995 + average * 5e-3;
|
|
12981
|
-
|
|
13145
|
+
ch.bandNoiseFloor[bandName] = newFloor;
|
|
12982
13146
|
const floorAdjusted = Math.max(0, average - newFloor * 1.5);
|
|
12983
13147
|
const normalized = maxMagnitude > 0 ? Math.min(1, floorAdjusted / maxMagnitude) : 0;
|
|
12984
13148
|
normalizedBands[bandName] = normalized;
|
|
12985
13149
|
}
|
|
12986
|
-
|
|
12987
|
-
...
|
|
13150
|
+
ch.audioState.bands = {
|
|
13151
|
+
...ch.audioState.bands,
|
|
12988
13152
|
low: rawBands.low,
|
|
12989
13153
|
lowMid: rawBands.lowMid,
|
|
12990
13154
|
mid: rawBands.mid,
|
|
12991
13155
|
highMid: rawBands.highMid,
|
|
12992
13156
|
high: rawBands.high,
|
|
12993
|
-
lowSmoothed:
|
|
12994
|
-
lowMidSmoothed:
|
|
12995
|
-
midSmoothed:
|
|
12996
|
-
highMidSmoothed:
|
|
12997
|
-
highSmoothed:
|
|
13157
|
+
lowSmoothed: ch.audioState.bands.lowSmoothed,
|
|
13158
|
+
lowMidSmoothed: ch.audioState.bands.lowMidSmoothed,
|
|
13159
|
+
midSmoothed: ch.audioState.bands.midSmoothed,
|
|
13160
|
+
highMidSmoothed: ch.audioState.bands.highMidSmoothed,
|
|
13161
|
+
highSmoothed: ch.audioState.bands.highSmoothed
|
|
12998
13162
|
};
|
|
12999
|
-
this.rawBandsPreGain = { ...rawBands };
|
|
13000
13163
|
return normalizedBands;
|
|
13001
13164
|
}
|
|
13002
13165
|
/**
|
|
13003
|
-
* Spectral features
|
|
13166
|
+
* Spectral features on a specific channel
|
|
13004
13167
|
*/
|
|
13005
|
-
|
|
13168
|
+
calculateChannelSpectral(ch, magnitudes, sampleRate, maxMagnitude) {
|
|
13006
13169
|
const nyquist = sampleRate / 2;
|
|
13007
13170
|
const binCount = magnitudes.length;
|
|
13008
13171
|
let sumMagnitude = 0;
|
|
@@ -13014,7 +13177,7 @@ class AudioSystem {
|
|
|
13014
13177
|
sumWeightedFreq += freq * magnitude;
|
|
13015
13178
|
}
|
|
13016
13179
|
const centroid = sumMagnitude > 0 ? sumWeightedFreq / sumMagnitude : 0;
|
|
13017
|
-
|
|
13180
|
+
ch.audioState.spectral.brightness = Math.min(1, centroid / nyquist);
|
|
13018
13181
|
let geometricMeanLog = 0;
|
|
13019
13182
|
let arithmeticMeanSum = 0;
|
|
13020
13183
|
let nonZeroCount = 0;
|
|
@@ -13029,9 +13192,9 @@ class AudioSystem {
|
|
|
13029
13192
|
if (nonZeroCount > 0) {
|
|
13030
13193
|
const arithmeticMean = arithmeticMeanSum / nonZeroCount;
|
|
13031
13194
|
const geometricMean = Math.exp(geometricMeanLog / nonZeroCount);
|
|
13032
|
-
|
|
13195
|
+
ch.audioState.spectral.flatness = Math.min(1, geometricMean / arithmeticMean);
|
|
13033
13196
|
} else {
|
|
13034
|
-
|
|
13197
|
+
ch.audioState.spectral.flatness = 0;
|
|
13035
13198
|
}
|
|
13036
13199
|
}
|
|
13037
13200
|
/**
|
|
@@ -13039,12 +13202,12 @@ class AudioSystem {
|
|
|
13039
13202
|
*/
|
|
13040
13203
|
runBeatPipeline(rawBandsForOnsets, dtMs, timestampMs, sampleRate) {
|
|
13041
13204
|
let onsets;
|
|
13042
|
-
const essentiaReady = (this.essentiaOnsetDetection?.isReady() || false) && !!this.fftMagnitudeDb && !!this.fftPhase;
|
|
13043
|
-
const hasFftData = !!(this.fftMagnitudeDb && this.fftPhase);
|
|
13205
|
+
const essentiaReady = (this.essentiaOnsetDetection?.isReady() || false) && !!this.mainChannel.fftMagnitudeDb && !!this.mainChannel.fftPhase;
|
|
13206
|
+
const hasFftData = !!(this.mainChannel.fftMagnitudeDb && this.mainChannel.fftPhase);
|
|
13044
13207
|
if (essentiaReady && hasFftData) {
|
|
13045
13208
|
const essentiaResult = this.essentiaOnsetDetection.detectFromSpectrum(
|
|
13046
|
-
this.fftMagnitudeDb,
|
|
13047
|
-
this.fftPhase,
|
|
13209
|
+
this.mainChannel.fftMagnitudeDb,
|
|
13210
|
+
this.mainChannel.fftPhase,
|
|
13048
13211
|
sampleRate
|
|
13049
13212
|
// DO NOT pass normalized bands - causes false snare detections on kicks
|
|
13050
13213
|
// Essentia should compute its own bands from spectrum for accurate ratios
|
|
@@ -13122,108 +13285,63 @@ class AudioSystem {
|
|
|
13122
13285
|
const barStr = `bar=${onset.bar + 1}`;
|
|
13123
13286
|
console.log(` ${idx + 1}. t=${relTime}s | ${onset.type.padEnd(5)} | ${phaseStr} | ${barStr} | [${onset.bands.join(",")}] | str=${onset.strength.toFixed(2)}`);
|
|
13124
13287
|
});
|
|
13125
|
-
} else {
|
|
13126
|
-
console.log(`🎵 Recent Onsets: none detected`);
|
|
13127
|
-
}
|
|
13128
|
-
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
13129
|
-
`);
|
|
13130
|
-
this.lastPhaseLogTime = timestampMs;
|
|
13131
|
-
}
|
|
13132
|
-
const candidates = [
|
|
13133
|
-
{ value: beatState.bpm, source: "pll" },
|
|
13134
|
-
{ value: tempo.bpm, source: "tempo" },
|
|
13135
|
-
{ value: this.lastNonZeroBpm, source: "carry" },
|
|
13136
|
-
{ value: 120, source: "default" }
|
|
13137
|
-
];
|
|
13138
|
-
const selected = candidates.find((c) => Number.isFinite(c.value) && c.value > 0) ?? { value: 120, source: "default" };
|
|
13139
|
-
const resolvedBpm = selected.value;
|
|
13140
|
-
const prevSource = this.lastBpmSource;
|
|
13141
|
-
this.lastNonZeroBpm = resolvedBpm;
|
|
13142
|
-
this.lastBpmSource = selected.source;
|
|
13143
|
-
if (this.debugMode && prevSource !== this.lastBpmSource) {
|
|
13144
|
-
console.log(`[AudioSystem][bpm] Source changed: ${prevSource} -> ${this.lastBpmSource} (resolved=${resolvedBpm.toFixed(2)}, pll=${beatState.bpm?.toFixed?.(2) ?? beatState.bpm}, tempo=${tempo.bpm?.toFixed?.(2) ?? tempo.bpm})`);
|
|
13145
|
-
}
|
|
13146
|
-
const mergedBeatState = beatState.bpm === resolvedBpm ? beatState : { ...beatState, bpm: resolvedBpm };
|
|
13147
|
-
this.lastKickDetected = mergedBeatState.events.some((e) => e.type === "kick") || pllKickCue;
|
|
13148
|
-
return mergedBeatState;
|
|
13149
|
-
}
|
|
13150
|
-
/**
|
|
13151
|
-
* Debug logging helper
|
|
13152
|
-
*/
|
|
13153
|
-
debugLog(message, ...args) {
|
|
13154
|
-
if (this.debugMode) {
|
|
13155
|
-
console.log(message, ...args);
|
|
13156
|
-
}
|
|
13157
|
-
}
|
|
13158
|
-
// Analysis configuration (optimized for onset detection)
|
|
13159
|
-
fftSize = 2048;
|
|
13160
|
-
// Good balance for quality vs performance
|
|
13161
|
-
// High-speed analysis for onset detection (separate from RAF)
|
|
13162
|
-
analysisInterval = null;
|
|
13163
|
-
analysisIntervalMs = 8;
|
|
13164
|
-
// 125Hz sampling for 5-8ms transient capture
|
|
13165
|
-
// Analysis data arrays
|
|
13166
|
-
frequencyData = null;
|
|
13167
|
-
timeDomainData = null;
|
|
13168
|
-
// FFT data (computed locally or fed to Essentia)
|
|
13169
|
-
fftEngine = null;
|
|
13170
|
-
fftInput = null;
|
|
13171
|
-
fftOutput = null;
|
|
13172
|
-
fftMagnitude = null;
|
|
13173
|
-
fftMagnitudeDb = null;
|
|
13174
|
-
fftPhase = null;
|
|
13175
|
-
hannWindow = null;
|
|
13176
|
-
// Audio analysis state (host-side state)
|
|
13177
|
-
audioState = {
|
|
13178
|
-
isConnected: false,
|
|
13179
|
-
volume: {
|
|
13180
|
-
current: 0,
|
|
13181
|
-
peak: 0,
|
|
13182
|
-
smoothed: 0
|
|
13183
|
-
},
|
|
13184
|
-
bands: {
|
|
13185
|
-
low: 0,
|
|
13186
|
-
lowMid: 0,
|
|
13187
|
-
mid: 0,
|
|
13188
|
-
highMid: 0,
|
|
13189
|
-
high: 0,
|
|
13190
|
-
lowSmoothed: 0,
|
|
13191
|
-
lowMidSmoothed: 0,
|
|
13192
|
-
midSmoothed: 0,
|
|
13193
|
-
highMidSmoothed: 0,
|
|
13194
|
-
highSmoothed: 0
|
|
13195
|
-
},
|
|
13196
|
-
beat: {
|
|
13197
|
-
kick: 0,
|
|
13198
|
-
snare: 0,
|
|
13199
|
-
hat: 0,
|
|
13200
|
-
any: 0,
|
|
13201
|
-
kickSmoothed: 0,
|
|
13202
|
-
snareSmoothed: 0,
|
|
13203
|
-
hatSmoothed: 0,
|
|
13204
|
-
anySmoothed: 0,
|
|
13205
|
-
events: [],
|
|
13206
|
-
bpm: 120,
|
|
13207
|
-
confidence: 0,
|
|
13208
|
-
isLocked: false
|
|
13209
|
-
},
|
|
13210
|
-
spectral: {
|
|
13211
|
-
brightness: 0,
|
|
13212
|
-
flatness: 0
|
|
13288
|
+
} else {
|
|
13289
|
+
console.log(`🎵 Recent Onsets: none detected`);
|
|
13290
|
+
}
|
|
13291
|
+
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
13292
|
+
`);
|
|
13293
|
+
this.lastPhaseLogTime = timestampMs;
|
|
13294
|
+
}
|
|
13295
|
+
const candidates = [
|
|
13296
|
+
{ value: beatState.bpm, source: "pll" },
|
|
13297
|
+
{ value: tempo.bpm, source: "tempo" },
|
|
13298
|
+
{ value: this.lastNonZeroBpm, source: "carry" },
|
|
13299
|
+
{ value: 120, source: "default" }
|
|
13300
|
+
];
|
|
13301
|
+
const selected = candidates.find((c) => Number.isFinite(c.value) && c.value > 0) ?? { value: 120, source: "default" };
|
|
13302
|
+
const resolvedBpm = selected.value;
|
|
13303
|
+
const prevSource = this.lastBpmSource;
|
|
13304
|
+
this.lastNonZeroBpm = resolvedBpm;
|
|
13305
|
+
this.lastBpmSource = selected.source;
|
|
13306
|
+
if (this.debugMode && prevSource !== this.lastBpmSource) {
|
|
13307
|
+
console.log(`[AudioSystem][bpm] Source changed: ${prevSource} -> ${this.lastBpmSource} (resolved=${resolvedBpm.toFixed(2)}, pll=${beatState.bpm?.toFixed?.(2) ?? beatState.bpm}, tempo=${tempo.bpm?.toFixed?.(2) ?? tempo.bpm})`);
|
|
13308
|
+
}
|
|
13309
|
+
const mergedBeatState = beatState.bpm === resolvedBpm ? beatState : { ...beatState, bpm: resolvedBpm };
|
|
13310
|
+
this.lastKickDetected = mergedBeatState.events.some((e) => e.type === "kick") || pllKickCue;
|
|
13311
|
+
return mergedBeatState;
|
|
13312
|
+
}
|
|
13313
|
+
/**
|
|
13314
|
+
* Debug logging helper
|
|
13315
|
+
*/
|
|
13316
|
+
debugLog(message, ...args) {
|
|
13317
|
+
if (this.debugMode) {
|
|
13318
|
+
console.log(message, ...args);
|
|
13213
13319
|
}
|
|
13320
|
+
}
|
|
13321
|
+
// Analysis configuration
|
|
13322
|
+
fftSize = 2048;
|
|
13323
|
+
// Beat state for main channel (not on AudioChannel because it's main-only)
|
|
13324
|
+
audioStateBeat = {
|
|
13325
|
+
kick: 0,
|
|
13326
|
+
snare: 0,
|
|
13327
|
+
hat: 0,
|
|
13328
|
+
any: 0,
|
|
13329
|
+
kickSmoothed: 0,
|
|
13330
|
+
snareSmoothed: 0,
|
|
13331
|
+
hatSmoothed: 0,
|
|
13332
|
+
anySmoothed: 0,
|
|
13333
|
+
events: [],
|
|
13334
|
+
bpm: 120,
|
|
13335
|
+
confidence: 0,
|
|
13336
|
+
isLocked: false
|
|
13214
13337
|
};
|
|
13215
|
-
// Waveform data for transfer to worker
|
|
13216
|
-
lastWaveformFrame = null;
|
|
13217
|
-
// Analysis loop
|
|
13218
|
-
analysisLoopId = null;
|
|
13219
|
-
isAnalysisRunning = false;
|
|
13220
|
-
lastAnalysisTimestamp = 0;
|
|
13221
13338
|
// Callback to send results to worker
|
|
13222
13339
|
sendAnalysisResults = null;
|
|
13223
13340
|
constructor(sendAnalysisResultsCallback) {
|
|
13224
13341
|
this.handleAudioStreamUpdate = this.handleAudioStreamUpdate.bind(this);
|
|
13225
13342
|
this.performAnalysis = this.performAnalysis.bind(this);
|
|
13226
13343
|
this.sendAnalysisResults = sendAnalysisResultsCallback || null;
|
|
13344
|
+
this.mainChannel = AudioChannel.create(0, this.fftSize);
|
|
13227
13345
|
this.onsetDetection = new MultiOnsetDetection();
|
|
13228
13346
|
this.essentiaOnsetDetection = new EssentiaOnsetDetection();
|
|
13229
13347
|
this.initializeEssentia();
|
|
@@ -13231,17 +13349,8 @@ class AudioSystem {
|
|
|
13231
13349
|
this.pll = new PhaseLockedLoop();
|
|
13232
13350
|
this.stateManager = new BeatStateManager();
|
|
13233
13351
|
this.onsetTapManager = new OnsetTapManager();
|
|
13234
|
-
this.volumeAutoGain = new AutoGain(3e3, 60);
|
|
13235
|
-
this.bandAutoGain = {
|
|
13236
|
-
low: new AutoGain(3e3, 60),
|
|
13237
|
-
lowMid: new AutoGain(3e3, 60),
|
|
13238
|
-
mid: new AutoGain(3e3, 60),
|
|
13239
|
-
highMid: new AutoGain(3e3, 60),
|
|
13240
|
-
high: new AutoGain(3e3, 60)
|
|
13241
|
-
};
|
|
13242
13352
|
const sampleRate = 60;
|
|
13243
|
-
this.
|
|
13244
|
-
// Beat energy curves - kept for API compatibility, but values come from stateManager
|
|
13353
|
+
this.beatEnvelopeFollowers = {
|
|
13245
13354
|
kick: new EnvelopeFollower(0, 300, sampleRate),
|
|
13246
13355
|
snare: new EnvelopeFollower(0, 300, sampleRate),
|
|
13247
13356
|
hat: new EnvelopeFollower(0, 300, sampleRate),
|
|
@@ -13249,45 +13358,15 @@ class AudioSystem {
|
|
|
13249
13358
|
kickSmoothed: new EnvelopeFollower(5, 500, sampleRate),
|
|
13250
13359
|
snareSmoothed: new EnvelopeFollower(5, 500, sampleRate),
|
|
13251
13360
|
hatSmoothed: new EnvelopeFollower(5, 500, sampleRate),
|
|
13252
|
-
anySmoothed: new EnvelopeFollower(5, 500, sampleRate)
|
|
13253
|
-
// Volume smoothing
|
|
13254
|
-
volumeSmoothed: new EnvelopeFollower(50, 200, sampleRate),
|
|
13255
|
-
// Band smoothing
|
|
13256
|
-
lowSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13257
|
-
lowMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13258
|
-
midSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13259
|
-
highMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13260
|
-
highSmoothed: new EnvelopeFollower(20, 150, sampleRate)
|
|
13361
|
+
anySmoothed: new EnvelopeFollower(5, 500, sampleRate)
|
|
13261
13362
|
};
|
|
13262
|
-
this.refreshFFTResources();
|
|
13263
13363
|
this.resetEssentiaBandHistories();
|
|
13264
13364
|
}
|
|
13265
|
-
/**
|
|
13266
|
-
* Prepare FFT buffers and windowing for the selected fftSize
|
|
13267
|
-
*/
|
|
13268
|
-
refreshFFTResources() {
|
|
13269
|
-
const binCount = this.fftSize / 2;
|
|
13270
|
-
this.frequencyData = new Uint8Array(binCount);
|
|
13271
|
-
this.fftMagnitude = new Float32Array(binCount);
|
|
13272
|
-
this.fftMagnitudeDb = new Float32Array(binCount);
|
|
13273
|
-
this.fftPhase = new Float32Array(binCount);
|
|
13274
|
-
this.timeDomainData = new Float32Array(this.fftSize);
|
|
13275
|
-
this.fftEngine = new FFT$1(this.fftSize);
|
|
13276
|
-
this.fftInput = new Float32Array(this.fftSize * 2);
|
|
13277
|
-
this.fftOutput = new Float32Array(this.fftSize * 2);
|
|
13278
|
-
this.hannWindow = new Float32Array(this.fftSize);
|
|
13279
|
-
const twoPi = Math.PI * 2;
|
|
13280
|
-
for (let i = 0; i < this.fftSize; i++) {
|
|
13281
|
-
this.hannWindow[i] = 0.5 * (1 - Math.cos(twoPi * i / (this.fftSize - 1)));
|
|
13282
|
-
}
|
|
13283
|
-
this.analysisTicks = 0;
|
|
13284
|
-
this.workletFrameCount = 0;
|
|
13285
|
-
}
|
|
13286
13365
|
/**
|
|
13287
13366
|
* Get the current audio analysis state (for host-side usage)
|
|
13288
13367
|
*/
|
|
13289
13368
|
getAudioState() {
|
|
13290
|
-
return { ...this.audioState };
|
|
13369
|
+
return { ...this.mainChannel.audioState };
|
|
13291
13370
|
}
|
|
13292
13371
|
/**
|
|
13293
13372
|
* Initialize Essentia.js (async WASM loading)
|
|
@@ -13317,161 +13396,140 @@ class AudioSystem {
|
|
|
13317
13396
|
return false;
|
|
13318
13397
|
}
|
|
13319
13398
|
/**
|
|
13320
|
-
* Handle audio stream update (called from VijiCore)
|
|
13399
|
+
* Handle audio stream update for main stream (called from VijiCore)
|
|
13321
13400
|
*/
|
|
13322
13401
|
handleAudioStreamUpdate(data) {
|
|
13323
13402
|
try {
|
|
13324
13403
|
if (data.audioStream) {
|
|
13325
13404
|
this.setAudioStream(data.audioStream);
|
|
13326
13405
|
} else {
|
|
13327
|
-
this.
|
|
13406
|
+
this.disconnectMainStream();
|
|
13328
13407
|
}
|
|
13329
13408
|
} catch (error) {
|
|
13330
13409
|
console.error("Error handling audio stream update:", error);
|
|
13331
|
-
this.audioState.isConnected = false;
|
|
13332
|
-
this.
|
|
13410
|
+
this.mainChannel.audioState.isConnected = false;
|
|
13411
|
+
this.sendChannelResults(this.mainChannel, true);
|
|
13333
13412
|
}
|
|
13334
13413
|
}
|
|
13335
13414
|
/**
|
|
13336
|
-
*
|
|
13415
|
+
* Ensure the shared AudioContext is created and resumed.
|
|
13337
13416
|
*/
|
|
13338
|
-
async
|
|
13339
|
-
this.
|
|
13340
|
-
|
|
13341
|
-
|
|
13342
|
-
|
|
13343
|
-
|
|
13417
|
+
async ensureAudioContext() {
|
|
13418
|
+
if (!this.audioContext) {
|
|
13419
|
+
this.audioContext = new AudioContext();
|
|
13420
|
+
}
|
|
13421
|
+
if (this.audioContext.state === "suspended") {
|
|
13422
|
+
await this.audioContext.resume();
|
|
13423
|
+
}
|
|
13424
|
+
return this.audioContext;
|
|
13425
|
+
}
|
|
13426
|
+
/**
|
|
13427
|
+
* Connect a channel to Web Audio nodes (source, worklet/analyser).
|
|
13428
|
+
* Used for both main and additional channels.
|
|
13429
|
+
*/
|
|
13430
|
+
async connectChannel(ch, audioStream, isMain) {
|
|
13431
|
+
ch.disconnectNodes();
|
|
13432
|
+
ch.refreshFFTResources();
|
|
13433
|
+
ch.workletFrameCount = 0;
|
|
13344
13434
|
const audioTracks = audioStream.getAudioTracks();
|
|
13345
13435
|
if (audioTracks.length === 0) {
|
|
13346
|
-
console.warn(
|
|
13347
|
-
|
|
13348
|
-
this.
|
|
13436
|
+
console.warn(`No audio tracks in stream for channel ${ch.streamIndex}`);
|
|
13437
|
+
ch.audioState.isConnected = false;
|
|
13438
|
+
this.sendChannelResults(ch, isMain);
|
|
13349
13439
|
return;
|
|
13350
13440
|
}
|
|
13351
13441
|
try {
|
|
13352
|
-
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
|
|
13356
|
-
|
|
13357
|
-
|
|
13358
|
-
|
|
13359
|
-
|
|
13360
|
-
|
|
13361
|
-
this.analysisMode = "worklet";
|
|
13362
|
-
this.currentSampleRate = this.audioContext.sampleRate;
|
|
13363
|
-
this.mediaStreamSource.connect(this.workletNode);
|
|
13364
|
-
this.workletNode.connect(this.audioContext.destination);
|
|
13365
|
-
this.debugLog("Audio worklet analysis enabled");
|
|
13442
|
+
const ctx = await this.ensureAudioContext();
|
|
13443
|
+
ch.mediaStreamSource = ctx.createMediaStreamSource(audioStream);
|
|
13444
|
+
const workletOk = await this.setupChannelWorklet(ch, isMain);
|
|
13445
|
+
if (workletOk && ch.workletNode) {
|
|
13446
|
+
ch.analysisMode = "worklet";
|
|
13447
|
+
ch.currentSampleRate = ctx.sampleRate;
|
|
13448
|
+
ch.mediaStreamSource.connect(ch.workletNode);
|
|
13449
|
+
ch.workletNode.connect(ctx.destination);
|
|
13450
|
+
this.debugLog(`Audio worklet enabled for channel ${ch.streamIndex}`);
|
|
13366
13451
|
setTimeout(() => {
|
|
13367
|
-
if (
|
|
13368
|
-
this.debugLog(
|
|
13369
|
-
|
|
13370
|
-
if (
|
|
13452
|
+
if (ch.analysisMode === "worklet" && ch.workletFrameCount === 0) {
|
|
13453
|
+
this.debugLog(`[AudioSystem] Worklet silent for channel ${ch.streamIndex}, falling back to analyser.`);
|
|
13454
|
+
ch.analysisMode = "analyser";
|
|
13455
|
+
if (ch.workletNode) {
|
|
13371
13456
|
try {
|
|
13372
|
-
|
|
13373
|
-
|
|
13457
|
+
ch.workletNode.port.onmessage = null;
|
|
13458
|
+
ch.workletNode.disconnect();
|
|
13374
13459
|
} catch (_) {
|
|
13375
13460
|
}
|
|
13376
|
-
|
|
13461
|
+
ch.workletNode = null;
|
|
13377
13462
|
}
|
|
13378
|
-
|
|
13379
|
-
|
|
13380
|
-
|
|
13381
|
-
|
|
13382
|
-
|
|
13383
|
-
|
|
13384
|
-
|
|
13385
|
-
this.
|
|
13386
|
-
this.startAnalysisLoop();
|
|
13463
|
+
ch.workletReady = false;
|
|
13464
|
+
ch.analyser = ctx.createAnalyser();
|
|
13465
|
+
ch.analyser.fftSize = ch.fftSize;
|
|
13466
|
+
ch.analyser.smoothingTimeConstant = 0;
|
|
13467
|
+
ch.mediaStreamSource.connect(ch.analyser);
|
|
13468
|
+
ch.frequencyData = new Uint8Array(ch.analyser.frequencyBinCount);
|
|
13469
|
+
ch.timeDomainData = new Float32Array(ch.fftSize);
|
|
13470
|
+
this.ensureAnalysisLoop();
|
|
13387
13471
|
}
|
|
13388
13472
|
}, 600);
|
|
13389
13473
|
} else {
|
|
13390
|
-
|
|
13391
|
-
|
|
13392
|
-
|
|
13393
|
-
|
|
13394
|
-
|
|
13395
|
-
|
|
13396
|
-
|
|
13397
|
-
this.
|
|
13398
|
-
|
|
13399
|
-
|
|
13400
|
-
|
|
13401
|
-
|
|
13402
|
-
|
|
13403
|
-
|
|
13404
|
-
|
|
13405
|
-
|
|
13406
|
-
this.startAnalysisLoop();
|
|
13407
|
-
}
|
|
13408
|
-
if (this.analysisMode === "analyser" && !this.isAnalysisRunning) {
|
|
13409
|
-
this.debugLog("[AudioSystem] Analysis loop not running after setup, forcing start.");
|
|
13410
|
-
this.startAnalysisLoop();
|
|
13411
|
-
}
|
|
13412
|
-
const tracks = audioStream.getAudioTracks();
|
|
13413
|
-
this.debugLog("[AudioSystem] Stream info", {
|
|
13414
|
-
trackCount: tracks.length,
|
|
13415
|
-
trackSettings: tracks.map((t) => t.getSettings?.() || {}),
|
|
13416
|
-
trackMuted: tracks.map((t) => t.muted),
|
|
13417
|
-
mode: this.analysisMode
|
|
13418
|
-
});
|
|
13474
|
+
ch.analysisMode = "analyser";
|
|
13475
|
+
ch.analyser = ctx.createAnalyser();
|
|
13476
|
+
ch.analyser.fftSize = ch.fftSize;
|
|
13477
|
+
ch.analyser.smoothingTimeConstant = 0;
|
|
13478
|
+
ch.mediaStreamSource.connect(ch.analyser);
|
|
13479
|
+
ch.frequencyData = new Uint8Array(ch.analyser.frequencyBinCount);
|
|
13480
|
+
ch.timeDomainData = new Float32Array(ch.fftSize);
|
|
13481
|
+
this.debugLog(`Analyser fallback for channel ${ch.streamIndex}`);
|
|
13482
|
+
}
|
|
13483
|
+
ch.currentStream = audioStream;
|
|
13484
|
+
ch.audioState.isConnected = true;
|
|
13485
|
+
if (ch.analysisMode === "worklet") {
|
|
13486
|
+
ch.isAnalysisRunning = true;
|
|
13487
|
+
} else {
|
|
13488
|
+
this.ensureAnalysisLoop();
|
|
13489
|
+
}
|
|
13419
13490
|
this.startStalenessTimer();
|
|
13420
|
-
this.debugLog(
|
|
13421
|
-
sampleRate:
|
|
13422
|
-
fftSize:
|
|
13423
|
-
|
|
13491
|
+
this.debugLog(`Audio stream connected for channel ${ch.streamIndex}`, {
|
|
13492
|
+
sampleRate: ctx.sampleRate,
|
|
13493
|
+
fftSize: ch.fftSize,
|
|
13494
|
+
mode: ch.analysisMode
|
|
13424
13495
|
});
|
|
13425
13496
|
} catch (error) {
|
|
13426
|
-
console.error(
|
|
13427
|
-
|
|
13428
|
-
|
|
13497
|
+
console.error(`Failed to set up audio for channel ${ch.streamIndex}:`, error);
|
|
13498
|
+
ch.audioState.isConnected = false;
|
|
13499
|
+
ch.disconnectNodes();
|
|
13429
13500
|
}
|
|
13430
|
-
this.
|
|
13501
|
+
this.sendChannelResults(ch, isMain);
|
|
13431
13502
|
}
|
|
13432
13503
|
/**
|
|
13433
|
-
*
|
|
13504
|
+
* Set the main audio stream for analysis (preserves original public API surface).
|
|
13434
13505
|
*/
|
|
13435
|
-
|
|
13436
|
-
this.
|
|
13437
|
-
this.stopStalenessTimer();
|
|
13506
|
+
async setAudioStream(audioStream) {
|
|
13507
|
+
this.disconnectMainStream();
|
|
13438
13508
|
this.resetEssentiaBandHistories();
|
|
13439
|
-
|
|
13440
|
-
this.mediaStreamSource.disconnect();
|
|
13441
|
-
this.mediaStreamSource = null;
|
|
13442
|
-
}
|
|
13443
|
-
if (this.analyser) {
|
|
13444
|
-
this.analyser.disconnect();
|
|
13445
|
-
this.analyser = null;
|
|
13446
|
-
}
|
|
13447
|
-
if (this.workletNode) {
|
|
13448
|
-
try {
|
|
13449
|
-
this.workletNode.port.onmessage = null;
|
|
13450
|
-
this.workletNode.disconnect();
|
|
13451
|
-
} catch (_) {
|
|
13452
|
-
}
|
|
13453
|
-
this.workletNode = null;
|
|
13454
|
-
}
|
|
13455
|
-
this.workletReady = false;
|
|
13456
|
-
this.analysisMode = "analyser";
|
|
13457
|
-
this.frequencyData = null;
|
|
13458
|
-
this.timeDomainData = null;
|
|
13459
|
-
this.fftMagnitude = null;
|
|
13460
|
-
this.fftMagnitudeDb = null;
|
|
13461
|
-
this.fftPhase = null;
|
|
13462
|
-
this.currentStream = null;
|
|
13463
|
-
this.audioState.isConnected = false;
|
|
13464
|
-
this.resetAudioValues();
|
|
13465
|
-
this.sendAnalysisResultsToWorker();
|
|
13466
|
-
this.debugLog("Audio stream disconnected (host-side)");
|
|
13509
|
+
await this.connectChannel(this.mainChannel, audioStream, true);
|
|
13467
13510
|
}
|
|
13468
13511
|
/**
|
|
13469
|
-
*
|
|
13512
|
+
* Disconnect the main audio stream (does NOT close AudioContext -- additional channels may still be active).
|
|
13470
13513
|
*/
|
|
13471
|
-
|
|
13472
|
-
|
|
13473
|
-
|
|
13514
|
+
disconnectMainStream() {
|
|
13515
|
+
this.mainChannel.disconnectNodes();
|
|
13516
|
+
this.mainChannel.audioState.isConnected = false;
|
|
13517
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13518
|
+
this.mainChannel.currentStream = null;
|
|
13519
|
+
this.resetAudioValues();
|
|
13520
|
+
if (this.additionalChannels.size === 0) {
|
|
13521
|
+
this.stopAnalysisLoop();
|
|
13522
|
+
this.stopStalenessTimer();
|
|
13474
13523
|
}
|
|
13524
|
+
this.sendChannelResults(this.mainChannel, true);
|
|
13525
|
+
this.debugLog("Main audio stream disconnected (host-side)");
|
|
13526
|
+
}
|
|
13527
|
+
/**
|
|
13528
|
+
* Set up an AudioWorklet node for a specific channel.
|
|
13529
|
+
* Registers the processor module once (shared), creates a new WorkletNode per channel.
|
|
13530
|
+
*/
|
|
13531
|
+
async setupChannelWorklet(ch, isMain) {
|
|
13532
|
+
if (!this.audioContext?.audioWorklet) return false;
|
|
13475
13533
|
try {
|
|
13476
13534
|
if (!this.workletRegistered) {
|
|
13477
13535
|
const workletSource = `
|
|
@@ -13521,65 +13579,71 @@ class AudioSystem {
|
|
|
13521
13579
|
URL.revokeObjectURL(workletUrl);
|
|
13522
13580
|
this.workletRegistered = true;
|
|
13523
13581
|
}
|
|
13524
|
-
|
|
13582
|
+
ch.workletNode = new AudioWorkletNode(this.audioContext, "audio-analysis-processor", {
|
|
13525
13583
|
numberOfOutputs: 1,
|
|
13526
13584
|
outputChannelCount: [1],
|
|
13527
|
-
processorOptions: {
|
|
13528
|
-
fftSize: this.fftSize,
|
|
13529
|
-
hopSize: this.fftSize / 2
|
|
13530
|
-
}
|
|
13585
|
+
processorOptions: { fftSize: ch.fftSize, hopSize: ch.fftSize / 2 }
|
|
13531
13586
|
});
|
|
13532
|
-
|
|
13587
|
+
ch.workletNode.port.onmessage = (event) => {
|
|
13533
13588
|
const data = event.data || {};
|
|
13534
13589
|
if (data.type === "audio-frame" && data.samples) {
|
|
13535
|
-
|
|
13590
|
+
if (isMain) {
|
|
13591
|
+
this.handleMainWorkletFrame(data.samples, data.sampleRate, data.timestamp);
|
|
13592
|
+
} else {
|
|
13593
|
+
this.handleAdditionalWorkletFrame(ch, data.samples, data.sampleRate, data.timestamp);
|
|
13594
|
+
}
|
|
13536
13595
|
}
|
|
13537
13596
|
};
|
|
13538
13597
|
return true;
|
|
13539
13598
|
} catch (error) {
|
|
13540
|
-
console.warn(
|
|
13541
|
-
|
|
13599
|
+
console.warn(`Audio worklet init failed for channel ${ch.streamIndex}:`, error);
|
|
13600
|
+
ch.workletNode = null;
|
|
13542
13601
|
return false;
|
|
13543
13602
|
}
|
|
13544
13603
|
}
|
|
13545
13604
|
/**
|
|
13546
|
-
*
|
|
13605
|
+
* Ensure the shared analysis interval is running (for analyser-mode channels).
|
|
13547
13606
|
*/
|
|
13548
|
-
|
|
13549
|
-
if (this.
|
|
13550
|
-
return;
|
|
13551
|
-
}
|
|
13552
|
-
this.isAnalysisRunning = true;
|
|
13607
|
+
ensureAnalysisLoop() {
|
|
13608
|
+
if (this.analysisInterval !== null) return;
|
|
13553
13609
|
this.analysisInterval = window.setInterval(() => {
|
|
13554
13610
|
this.performAnalysis();
|
|
13555
13611
|
}, this.analysisIntervalMs);
|
|
13556
|
-
this.debugLog("[AudioSystem]
|
|
13557
|
-
intervalMs: this.analysisIntervalMs
|
|
13558
|
-
});
|
|
13612
|
+
this.debugLog("[AudioSystem] Shared analysis loop started", { intervalMs: this.analysisIntervalMs });
|
|
13559
13613
|
}
|
|
13560
13614
|
/**
|
|
13561
|
-
* Stop the
|
|
13615
|
+
* Stop the shared analysis loop.
|
|
13562
13616
|
*/
|
|
13563
13617
|
stopAnalysisLoop() {
|
|
13564
|
-
this.isAnalysisRunning = false;
|
|
13565
13618
|
if (this.analysisInterval !== null) {
|
|
13566
13619
|
clearInterval(this.analysisInterval);
|
|
13567
13620
|
this.analysisInterval = null;
|
|
13568
13621
|
}
|
|
13569
|
-
|
|
13570
|
-
|
|
13571
|
-
|
|
13622
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13623
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13624
|
+
ch.isAnalysisRunning = false;
|
|
13572
13625
|
}
|
|
13573
13626
|
}
|
|
13627
|
+
/**
|
|
13628
|
+
* Shared staleness timer: checks all channels for stale data.
|
|
13629
|
+
*/
|
|
13574
13630
|
startStalenessTimer() {
|
|
13575
|
-
this.
|
|
13576
|
-
this.lastFrameTime = performance.now();
|
|
13631
|
+
if (this.stalenessTimer !== null) return;
|
|
13577
13632
|
this.stalenessTimer = setInterval(() => {
|
|
13578
|
-
|
|
13579
|
-
|
|
13580
|
-
|
|
13581
|
-
|
|
13582
|
-
|
|
13633
|
+
const now = performance.now();
|
|
13634
|
+
if (this.mainChannel.audioState.isConnected && this.mainChannel.lastFrameTime > 0) {
|
|
13635
|
+
if (now - this.mainChannel.lastFrameTime > AudioSystem.STALENESS_THRESHOLD_MS) {
|
|
13636
|
+
this.resetAudioValues();
|
|
13637
|
+
this.sendChannelResults(this.mainChannel, true);
|
|
13638
|
+
}
|
|
13639
|
+
}
|
|
13640
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13641
|
+
if (ch.audioState.isConnected && ch.lastFrameTime > 0) {
|
|
13642
|
+
if (now - ch.lastFrameTime > AudioSystem.STALENESS_THRESHOLD_MS) {
|
|
13643
|
+
ch.resetValues();
|
|
13644
|
+
this.sendChannelResults(ch, false);
|
|
13645
|
+
}
|
|
13646
|
+
}
|
|
13583
13647
|
}
|
|
13584
13648
|
}, 250);
|
|
13585
13649
|
}
|
|
@@ -13591,114 +13655,96 @@ class AudioSystem {
|
|
|
13591
13655
|
}
|
|
13592
13656
|
/**
|
|
13593
13657
|
* Pause audio analysis (for tests or temporary suspension)
|
|
13594
|
-
* The setInterval continues but performAnalysis() exits early
|
|
13595
13658
|
*/
|
|
13596
13659
|
pauseAnalysis() {
|
|
13597
|
-
this.isAnalysisRunning = false;
|
|
13660
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13661
|
+
for (const ch of this.additionalChannels.values()) ch.isAnalysisRunning = false;
|
|
13598
13662
|
}
|
|
13599
13663
|
/**
|
|
13600
13664
|
* Resume audio analysis after pause
|
|
13601
13665
|
*/
|
|
13602
13666
|
resumeAnalysis() {
|
|
13603
|
-
this.isAnalysisRunning = true;
|
|
13667
|
+
this.mainChannel.isAnalysisRunning = true;
|
|
13668
|
+
for (const ch of this.additionalChannels.values()) ch.isAnalysisRunning = true;
|
|
13604
13669
|
}
|
|
13605
13670
|
/**
|
|
13606
|
-
*
|
|
13607
|
-
* Uses industry-grade 4-layer architecture:
|
|
13608
|
-
* Layer 1: MultiOnsetDetection (per-band onset detection)
|
|
13609
|
-
* Layer 2: TempoInduction (dual-method BPM detection)
|
|
13610
|
-
* Layer 3: PhaseLockedLoop (stable phase tracking)
|
|
13611
|
-
* Layer 4: BeatStateManager (state machine + confidence)
|
|
13671
|
+
* Shared analysis loop callback: iterates main + additional channels in analyser mode.
|
|
13612
13672
|
*/
|
|
13613
13673
|
performAnalysis() {
|
|
13614
|
-
if (!this.
|
|
13615
|
-
|
|
13616
|
-
}
|
|
13617
|
-
if (!this.timeDomainData || this.timeDomainData.length !== this.fftSize) {
|
|
13618
|
-
this.timeDomainData = new Float32Array(this.fftSize);
|
|
13619
|
-
}
|
|
13620
|
-
this.analyser.getFloatTimeDomainData(this.timeDomainData);
|
|
13621
|
-
this.analysisTicks++;
|
|
13622
|
-
for (let i = 0; i < 32 && i < this.timeDomainData.length; i++) {
|
|
13623
|
-
Math.abs(this.timeDomainData[i]);
|
|
13624
|
-
}
|
|
13674
|
+
if (!this.audioContext) return;
|
|
13675
|
+
const sampleRate = this.audioContext.sampleRate;
|
|
13625
13676
|
const detectionTimeMs = performance.now();
|
|
13626
|
-
|
|
13627
|
-
|
|
13628
|
-
|
|
13629
|
-
|
|
13630
|
-
|
|
13631
|
-
|
|
13632
|
-
|
|
13633
|
-
|
|
13634
|
-
|
|
13635
|
-
|
|
13636
|
-
|
|
13677
|
+
const mc = this.mainChannel;
|
|
13678
|
+
if (mc.isAnalysisRunning && mc.analysisMode === "analyser" && mc.analyser) {
|
|
13679
|
+
if (!mc.timeDomainData || mc.timeDomainData.length !== mc.fftSize) {
|
|
13680
|
+
mc.timeDomainData = new Float32Array(mc.fftSize);
|
|
13681
|
+
}
|
|
13682
|
+
mc.analyser.getFloatTimeDomainData(mc.timeDomainData);
|
|
13683
|
+
this.analyzeMainFrame(mc.timeDomainData, sampleRate, detectionTimeMs);
|
|
13684
|
+
}
|
|
13685
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13686
|
+
if (ch.isAnalysisRunning && ch.analysisMode === "analyser" && ch.analyser) {
|
|
13687
|
+
if (!ch.timeDomainData || ch.timeDomainData.length !== ch.fftSize) {
|
|
13688
|
+
ch.timeDomainData = new Float32Array(ch.fftSize);
|
|
13689
|
+
}
|
|
13690
|
+
ch.analyser.getFloatTimeDomainData(ch.timeDomainData);
|
|
13691
|
+
this.analyzeAdditionalFrame(ch, ch.timeDomainData, sampleRate, detectionTimeMs);
|
|
13692
|
+
}
|
|
13693
|
+
}
|
|
13694
|
+
}
|
|
13695
|
+
applyChannelAutoGain(ch) {
|
|
13696
|
+
ch.audioState.volume.current = ch.volumeAutoGain.process(ch.audioState.volume.current);
|
|
13697
|
+
ch.audioState.volume.peak = ch.volumeAutoGain.process(ch.audioState.volume.peak);
|
|
13698
|
+
const bands = ch.audioState.bands;
|
|
13699
|
+
bands.low = ch.bandAutoGain.low.process(bands.low);
|
|
13700
|
+
bands.lowMid = ch.bandAutoGain.lowMid.process(bands.lowMid);
|
|
13701
|
+
bands.mid = ch.bandAutoGain.mid.process(bands.mid);
|
|
13702
|
+
bands.highMid = ch.bandAutoGain.highMid.process(bands.highMid);
|
|
13703
|
+
bands.high = ch.bandAutoGain.high.process(bands.high);
|
|
13637
13704
|
}
|
|
13638
13705
|
// Note: updateBeatState is now handled by BeatStateManager
|
|
13639
13706
|
/**
|
|
13640
|
-
* Update smooth versions of frequency bands
|
|
13707
|
+
* Update smooth versions of frequency bands on a specific channel
|
|
13641
13708
|
*/
|
|
13642
|
-
|
|
13643
|
-
|
|
13644
|
-
|
|
13645
|
-
|
|
13646
|
-
);
|
|
13647
|
-
|
|
13648
|
-
|
|
13649
|
-
|
|
13650
|
-
);
|
|
13651
|
-
this.audioState.bands.lowMidSmoothed = this.envelopeFollowers.lowMidSmoothed.process(
|
|
13652
|
-
this.audioState.bands.lowMid,
|
|
13653
|
-
dtMs
|
|
13654
|
-
);
|
|
13655
|
-
this.audioState.bands.midSmoothed = this.envelopeFollowers.midSmoothed.process(
|
|
13656
|
-
this.audioState.bands.mid,
|
|
13657
|
-
dtMs
|
|
13658
|
-
);
|
|
13659
|
-
this.audioState.bands.highMidSmoothed = this.envelopeFollowers.highMidSmoothed.process(
|
|
13660
|
-
this.audioState.bands.highMid,
|
|
13661
|
-
dtMs
|
|
13662
|
-
);
|
|
13663
|
-
this.audioState.bands.highSmoothed = this.envelopeFollowers.highSmoothed.process(
|
|
13664
|
-
this.audioState.bands.high,
|
|
13665
|
-
dtMs
|
|
13666
|
-
);
|
|
13709
|
+
updateChannelSmoothBands(ch, dtMs) {
|
|
13710
|
+
const ef = ch.envelopeFollowers;
|
|
13711
|
+
ch.audioState.volume.smoothed = ef.volumeSmoothed.process(ch.audioState.volume.current, dtMs);
|
|
13712
|
+
ch.audioState.bands.lowSmoothed = ef.lowSmoothed.process(ch.audioState.bands.low, dtMs);
|
|
13713
|
+
ch.audioState.bands.lowMidSmoothed = ef.lowMidSmoothed.process(ch.audioState.bands.lowMid, dtMs);
|
|
13714
|
+
ch.audioState.bands.midSmoothed = ef.midSmoothed.process(ch.audioState.bands.mid, dtMs);
|
|
13715
|
+
ch.audioState.bands.highMidSmoothed = ef.highMidSmoothed.process(ch.audioState.bands.highMid, dtMs);
|
|
13716
|
+
ch.audioState.bands.highSmoothed = ef.highSmoothed.process(ch.audioState.bands.high, dtMs);
|
|
13667
13717
|
}
|
|
13668
13718
|
/**
|
|
13669
|
-
* Send analysis results to worker
|
|
13719
|
+
* Send analysis results for a specific channel to the worker.
|
|
13720
|
+
* Tags with streamIndex and conditionally includes beat data (main channel only).
|
|
13670
13721
|
*/
|
|
13671
|
-
|
|
13672
|
-
if (this.sendAnalysisResults)
|
|
13673
|
-
|
|
13674
|
-
|
|
13675
|
-
|
|
13676
|
-
|
|
13677
|
-
|
|
13678
|
-
|
|
13679
|
-
|
|
13680
|
-
|
|
13681
|
-
|
|
13682
|
-
|
|
13683
|
-
|
|
13684
|
-
|
|
13685
|
-
|
|
13686
|
-
|
|
13687
|
-
});
|
|
13722
|
+
sendChannelResults(ch, includeBeat) {
|
|
13723
|
+
if (!this.sendAnalysisResults) return;
|
|
13724
|
+
const frequencyData = ch.frequencyData ? new Uint8Array(ch.frequencyData) : new Uint8Array(0);
|
|
13725
|
+
const waveformData = ch.lastWaveformFrame ? new Float32Array(ch.lastWaveformFrame) : new Float32Array(0);
|
|
13726
|
+
const data = {
|
|
13727
|
+
streamIndex: ch.streamIndex,
|
|
13728
|
+
isConnected: ch.audioState.isConnected,
|
|
13729
|
+
volume: ch.audioState.volume,
|
|
13730
|
+
bands: ch.audioState.bands,
|
|
13731
|
+
spectral: ch.audioState.spectral,
|
|
13732
|
+
frequencyData,
|
|
13733
|
+
waveformData,
|
|
13734
|
+
timestamp: performance.now()
|
|
13735
|
+
};
|
|
13736
|
+
if (includeBeat) {
|
|
13737
|
+
data.beat = this.audioStateBeat;
|
|
13688
13738
|
}
|
|
13739
|
+
this.sendAnalysisResults({ type: "audio-analysis-update", data });
|
|
13689
13740
|
}
|
|
13690
13741
|
/**
|
|
13691
|
-
* Reset audio values to defaults
|
|
13742
|
+
* Reset audio values to defaults (main channel + beat state)
|
|
13692
13743
|
*/
|
|
13693
13744
|
resetAudioValues() {
|
|
13694
|
-
this.
|
|
13695
|
-
this.audioState.volume.peak = 0;
|
|
13696
|
-
this.audioState.volume.smoothed = 0;
|
|
13745
|
+
this.mainChannel.resetValues();
|
|
13697
13746
|
this.lastKickDetected = false;
|
|
13698
|
-
|
|
13699
|
-
this.audioState.bands[band] = 0;
|
|
13700
|
-
}
|
|
13701
|
-
this.audioState.beat = {
|
|
13747
|
+
this.audioStateBeat = {
|
|
13702
13748
|
kick: 0,
|
|
13703
13749
|
snare: 0,
|
|
13704
13750
|
hat: 0,
|
|
@@ -13712,36 +13758,98 @@ class AudioSystem {
|
|
|
13712
13758
|
confidence: 0,
|
|
13713
13759
|
isLocked: false
|
|
13714
13760
|
};
|
|
13715
|
-
this.audioState.spectral = {
|
|
13716
|
-
brightness: 0,
|
|
13717
|
-
flatness: 0
|
|
13718
|
-
};
|
|
13719
13761
|
}
|
|
13720
13762
|
/**
|
|
13721
|
-
* Reset all audio state (called when destroying)
|
|
13763
|
+
* Reset all audio state (called when destroying).
|
|
13764
|
+
* Disconnects all channels, closes AudioContext, resets all modules.
|
|
13722
13765
|
*/
|
|
13723
13766
|
resetAudioState() {
|
|
13724
|
-
this.
|
|
13767
|
+
this.stopAnalysisLoop();
|
|
13768
|
+
this.stopStalenessTimer();
|
|
13725
13769
|
this.resetEssentiaBandHistories();
|
|
13770
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13771
|
+
ch.destroy();
|
|
13772
|
+
}
|
|
13773
|
+
this.additionalChannels.clear();
|
|
13774
|
+
this.mainChannel.disconnectNodes();
|
|
13775
|
+
this.mainChannel.audioState.isConnected = false;
|
|
13776
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13777
|
+
this.mainChannel.currentStream = null;
|
|
13726
13778
|
if (this.audioContext && this.audioContext.state !== "closed") {
|
|
13727
13779
|
this.audioContext.close();
|
|
13728
13780
|
this.audioContext = null;
|
|
13729
13781
|
}
|
|
13730
|
-
this.workletNode = null;
|
|
13731
|
-
this.workletReady = false;
|
|
13732
13782
|
this.workletRegistered = false;
|
|
13733
|
-
this.analysisMode = "analyser";
|
|
13734
13783
|
this.onsetDetection.reset();
|
|
13735
13784
|
this.tempoInduction.reset();
|
|
13736
13785
|
this.pll.reset();
|
|
13737
13786
|
this.stateManager.reset();
|
|
13738
|
-
this.volumeAutoGain.reset();
|
|
13739
|
-
Object.values(this.bandAutoGain).forEach((g) => g.reset());
|
|
13740
13787
|
this.onsetTapManager.reset();
|
|
13741
|
-
Object.values(this.
|
|
13788
|
+
Object.values(this.beatEnvelopeFollowers).forEach((env) => env.reset());
|
|
13742
13789
|
this.resetAudioValues();
|
|
13743
13790
|
}
|
|
13744
13791
|
// ═══════════════════════════════════════════════════════════
|
|
13792
|
+
// Multi-Channel Stream Management
|
|
13793
|
+
// ═══════════════════════════════════════════════════════════
|
|
13794
|
+
/**
|
|
13795
|
+
* Add an additional audio stream (lightweight analysis, no beat detection).
|
|
13796
|
+
* @param streamIndex Global stream index (from VijiCore: AUDIO_ADDITIONAL_BASE + n or AUDIO_DEVICE_BASE + n)
|
|
13797
|
+
* @param stream The MediaStream to analyze
|
|
13798
|
+
*/
|
|
13799
|
+
async addStream(streamIndex, stream) {
|
|
13800
|
+
if (this.additionalChannels.has(streamIndex)) {
|
|
13801
|
+
this.removeStream(streamIndex);
|
|
13802
|
+
}
|
|
13803
|
+
const ch = AudioChannel.create(streamIndex, this.fftSize);
|
|
13804
|
+
this.additionalChannels.set(streamIndex, ch);
|
|
13805
|
+
await this.connectChannel(ch, stream, false);
|
|
13806
|
+
}
|
|
13807
|
+
/**
|
|
13808
|
+
* Remove an additional audio stream.
|
|
13809
|
+
*/
|
|
13810
|
+
removeStream(streamIndex) {
|
|
13811
|
+
const ch = this.additionalChannels.get(streamIndex);
|
|
13812
|
+
if (!ch) return;
|
|
13813
|
+
ch.destroy();
|
|
13814
|
+
this.additionalChannels.delete(streamIndex);
|
|
13815
|
+
const disconnected = AudioChannel.create(streamIndex, this.fftSize);
|
|
13816
|
+
disconnected.audioState.isConnected = false;
|
|
13817
|
+
this.sendChannelResults(disconnected, false);
|
|
13818
|
+
disconnected.destroy();
|
|
13819
|
+
if (this.additionalChannels.size === 0 && !this.mainChannel.audioState.isConnected) {
|
|
13820
|
+
this.stopAnalysisLoop();
|
|
13821
|
+
this.stopStalenessTimer();
|
|
13822
|
+
}
|
|
13823
|
+
}
|
|
13824
|
+
/**
|
|
13825
|
+
* Tear down all additional channels and rebuild with corrected indices.
|
|
13826
|
+
* Mirrors video's reinitializeAdditionalCoordinators pattern.
|
|
13827
|
+
*/
|
|
13828
|
+
async reinitializeAdditionalChannels(streams, baseIndex) {
|
|
13829
|
+
const toRemove = [];
|
|
13830
|
+
for (const [idx, ch] of this.additionalChannels) {
|
|
13831
|
+
if (idx >= baseIndex && idx < baseIndex + 100) {
|
|
13832
|
+
ch.destroy();
|
|
13833
|
+
toRemove.push(idx);
|
|
13834
|
+
}
|
|
13835
|
+
}
|
|
13836
|
+
for (const idx of toRemove) {
|
|
13837
|
+
this.additionalChannels.delete(idx);
|
|
13838
|
+
}
|
|
13839
|
+
for (let i = 0; i < streams.length; i++) {
|
|
13840
|
+
const newIndex = baseIndex + i;
|
|
13841
|
+
const ch = AudioChannel.create(newIndex, this.fftSize);
|
|
13842
|
+
this.additionalChannels.set(newIndex, ch);
|
|
13843
|
+
await this.connectChannel(ch, streams[i], false);
|
|
13844
|
+
}
|
|
13845
|
+
}
|
|
13846
|
+
/**
|
|
13847
|
+
* Get the number of additional audio channels.
|
|
13848
|
+
*/
|
|
13849
|
+
getChannelCount() {
|
|
13850
|
+
return this.additionalChannels.size;
|
|
13851
|
+
}
|
|
13852
|
+
// ═══════════════════════════════════════════════════════════
|
|
13745
13853
|
// Public API Methods for Audio Analysis Configuration
|
|
13746
13854
|
// ═══════════════════════════════════════════════════════════
|
|
13747
13855
|
/**
|
|
@@ -13804,10 +13912,10 @@ class AudioSystem {
|
|
|
13804
13912
|
setFFTSize(size) {
|
|
13805
13913
|
if (this.fftSize === size) return;
|
|
13806
13914
|
this.fftSize = size;
|
|
13807
|
-
this.
|
|
13808
|
-
|
|
13809
|
-
|
|
13810
|
-
this.setAudioStream(
|
|
13915
|
+
this.mainChannel.fftSize = size;
|
|
13916
|
+
this.mainChannel.refreshFFTResources();
|
|
13917
|
+
if (this.mainChannel.currentStream) {
|
|
13918
|
+
this.setAudioStream(this.mainChannel.currentStream);
|
|
13811
13919
|
}
|
|
13812
13920
|
}
|
|
13813
13921
|
/**
|
|
@@ -13816,8 +13924,12 @@ class AudioSystem {
|
|
|
13816
13924
|
setAutoGain(enabled) {
|
|
13817
13925
|
this.autoGainEnabled = enabled;
|
|
13818
13926
|
if (!enabled) {
|
|
13819
|
-
this.volumeAutoGain.reset();
|
|
13820
|
-
Object.values(this.bandAutoGain).forEach((g) => g.reset());
|
|
13927
|
+
this.mainChannel.volumeAutoGain.reset();
|
|
13928
|
+
Object.values(this.mainChannel.bandAutoGain).forEach((g) => g.reset());
|
|
13929
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13930
|
+
ch.volumeAutoGain.reset();
|
|
13931
|
+
Object.values(ch.bandAutoGain).forEach((g) => g.reset());
|
|
13932
|
+
}
|
|
13821
13933
|
}
|
|
13822
13934
|
}
|
|
13823
13935
|
/**
|
|
@@ -13845,7 +13957,7 @@ class AudioSystem {
|
|
|
13845
13957
|
*/
|
|
13846
13958
|
getState() {
|
|
13847
13959
|
return {
|
|
13848
|
-
isConnected: this.audioState.isConnected,
|
|
13960
|
+
isConnected: this.mainChannel.audioState.isConnected,
|
|
13849
13961
|
currentBPM: this.pll.getBPM(),
|
|
13850
13962
|
confidence: this.tempoInduction.getConfidence(),
|
|
13851
13963
|
isLocked: this.stateManager.isLocked(),
|
|
@@ -13863,11 +13975,11 @@ class AudioSystem {
|
|
|
13863
13975
|
*/
|
|
13864
13976
|
getCurrentAudioData() {
|
|
13865
13977
|
return {
|
|
13866
|
-
isConnected: this.audioState.isConnected,
|
|
13867
|
-
volume: { ...this.audioState.volume },
|
|
13868
|
-
bands: { ...this.audioState.bands },
|
|
13869
|
-
beat: { ...this.
|
|
13870
|
-
spectral: { ...this.audioState.spectral }
|
|
13978
|
+
isConnected: this.mainChannel.audioState.isConnected,
|
|
13979
|
+
volume: { ...this.mainChannel.audioState.volume },
|
|
13980
|
+
bands: { ...this.mainChannel.audioState.bands },
|
|
13981
|
+
beat: { ...this.audioStateBeat },
|
|
13982
|
+
spectral: { ...this.mainChannel.audioState.spectral }
|
|
13871
13983
|
};
|
|
13872
13984
|
}
|
|
13873
13985
|
/**
|
|
@@ -13913,29 +14025,25 @@ class AudioSystem {
|
|
|
13913
14025
|
},
|
|
13914
14026
|
// Raw audio levels (AFTER normalization for visual output)
|
|
13915
14027
|
levels: {
|
|
13916
|
-
low: this.audioState.bands.low,
|
|
13917
|
-
lowMid: this.audioState.bands.lowMid,
|
|
13918
|
-
mid: this.audioState.bands.mid,
|
|
13919
|
-
highMid: this.audioState.bands.highMid,
|
|
13920
|
-
high: this.audioState.bands.high,
|
|
13921
|
-
volume: this.audioState.volume.current
|
|
14028
|
+
low: this.mainChannel.audioState.bands.low,
|
|
14029
|
+
lowMid: this.mainChannel.audioState.bands.lowMid,
|
|
14030
|
+
mid: this.mainChannel.audioState.bands.mid,
|
|
14031
|
+
highMid: this.mainChannel.audioState.bands.highMid,
|
|
14032
|
+
high: this.mainChannel.audioState.bands.high,
|
|
14033
|
+
volume: this.mainChannel.audioState.volume.current
|
|
13922
14034
|
},
|
|
13923
|
-
// Pre-gain band levels (BEFORE auto-gain normalization)
|
|
13924
14035
|
rawBands: { ...this.rawBandsPreGain },
|
|
13925
|
-
// Auto-gain values
|
|
13926
14036
|
bandGains: {
|
|
13927
|
-
low: this.bandAutoGain.low.getGain(),
|
|
13928
|
-
lowMid: this.bandAutoGain.lowMid.getGain(),
|
|
13929
|
-
mid: this.bandAutoGain.mid.getGain(),
|
|
13930
|
-
highMid: this.bandAutoGain.highMid.getGain(),
|
|
13931
|
-
high: this.bandAutoGain.high.getGain()
|
|
14037
|
+
low: this.mainChannel.bandAutoGain.low.getGain(),
|
|
14038
|
+
lowMid: this.mainChannel.bandAutoGain.lowMid.getGain(),
|
|
14039
|
+
mid: this.mainChannel.bandAutoGain.mid.getGain(),
|
|
14040
|
+
highMid: this.mainChannel.bandAutoGain.highMid.getGain(),
|
|
14041
|
+
high: this.mainChannel.bandAutoGain.high.getGain()
|
|
13932
14042
|
},
|
|
13933
|
-
volumeGain: this.volumeAutoGain.getGain(),
|
|
13934
|
-
|
|
13935
|
-
dtMs: this.lastDtMs,
|
|
14043
|
+
volumeGain: this.mainChannel.volumeAutoGain.getGain(),
|
|
14044
|
+
dtMs: this.mainChannel.lastDtMs,
|
|
13936
14045
|
analysisIntervalMs: this.analysisIntervalMs,
|
|
13937
|
-
|
|
13938
|
-
events: [...this.audioState.beat.events]
|
|
14046
|
+
events: [...this.audioStateBeat.events]
|
|
13939
14047
|
};
|
|
13940
14048
|
}
|
|
13941
14049
|
}
|
|
@@ -14360,7 +14468,8 @@ class DeviceSensorManager {
|
|
|
14360
14468
|
name: device.name,
|
|
14361
14469
|
motion: null,
|
|
14362
14470
|
orientation: null,
|
|
14363
|
-
video: null
|
|
14471
|
+
video: null,
|
|
14472
|
+
audio: null
|
|
14364
14473
|
};
|
|
14365
14474
|
this.externalDevices.set(device.id, newDevice);
|
|
14366
14475
|
this.externalDeviceOrder.push(device.id);
|
|
@@ -14460,6 +14569,21 @@ class SceneAnalyzer {
|
|
|
14460
14569
|
}
|
|
14461
14570
|
return "native";
|
|
14462
14571
|
}
|
|
14572
|
+
/**
|
|
14573
|
+
* P5 main-canvas mode from the renderer directive (only meaningful when
|
|
14574
|
+
* {@link detectRendererType} is `'p5'`).
|
|
14575
|
+
*
|
|
14576
|
+
* - `// @renderer p5` → 2D (default)
|
|
14577
|
+
* - `// @renderer p5 webgl` → WEBGL
|
|
14578
|
+
*/
|
|
14579
|
+
static detectP5CanvasMode(sceneCode) {
|
|
14580
|
+
if (/\/\/\s*@renderer\s+p5\s+webgl\b|\/\*\s*@renderer\s+p5\s+webgl\s*\*\//i.test(
|
|
14581
|
+
sceneCode
|
|
14582
|
+
)) {
|
|
14583
|
+
return "webgl";
|
|
14584
|
+
}
|
|
14585
|
+
return "2d";
|
|
14586
|
+
}
|
|
14463
14587
|
}
|
|
14464
14588
|
class VijiCore {
|
|
14465
14589
|
iframeManager = null;
|
|
@@ -14500,16 +14624,25 @@ class VijiCore {
|
|
|
14500
14624
|
// Index 200..299: Direct frame injection (compositor pipeline)
|
|
14501
14625
|
//
|
|
14502
14626
|
// The worker filters by streamType when building artist-facing arrays
|
|
14503
|
-
// (viji.
|
|
14627
|
+
// (viji.videoStreams[], viji.devices[].video), so absolute indices are internal.
|
|
14504
14628
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
14505
14629
|
static ADDITIONAL_STREAM_BASE = 1;
|
|
14506
14630
|
static DEVICE_VIDEO_BASE = 100;
|
|
14507
14631
|
static DIRECT_FRAME_BASE = 200;
|
|
14632
|
+
// Audio stream index ranges (mirroring video)
|
|
14633
|
+
static AUDIO_ADDITIONAL_BASE = 1;
|
|
14634
|
+
static AUDIO_DEVICE_BASE = 100;
|
|
14635
|
+
static MAX_ADDITIONAL_AUDIO_STREAMS = 8;
|
|
14636
|
+
static MAX_ADDITIONAL_VIDEO_STREAMS = 8;
|
|
14508
14637
|
// Separated video stream management
|
|
14509
14638
|
videoStream = null;
|
|
14510
14639
|
// Main stream (CV enabled) - always index 0
|
|
14511
14640
|
videoStreams = [];
|
|
14512
14641
|
// Additional streams (no CV)
|
|
14642
|
+
// Audio stream management
|
|
14643
|
+
audioStreams = [];
|
|
14644
|
+
// Additional audio streams (lightweight analysis)
|
|
14645
|
+
deviceAudioStreamIndices = /* @__PURE__ */ new Map();
|
|
14513
14646
|
// Video coordinators
|
|
14514
14647
|
mainVideoCoordinator = null;
|
|
14515
14648
|
additionalCoordinators = [];
|
|
@@ -14565,6 +14698,7 @@ class VijiCore {
|
|
|
14565
14698
|
this.stats.rendererType = SceneAnalyzer.detectRendererType(config.sceneCode);
|
|
14566
14699
|
this.videoStream = config.videoStream || null;
|
|
14567
14700
|
this.videoStreams = config.videoStreams || [];
|
|
14701
|
+
this.audioStreams = config.audioStreams ? [...config.audioStreams] : [];
|
|
14568
14702
|
this.currentInteractionEnabled = this.config.allowUserInteraction;
|
|
14569
14703
|
if (config.allowDeviceInteraction) {
|
|
14570
14704
|
this.deviceSensorManager = new DeviceSensorManager();
|
|
@@ -14668,6 +14802,19 @@ class VijiCore {
|
|
|
14668
14802
|
getDebugMode() {
|
|
14669
14803
|
return this.debugMode;
|
|
14670
14804
|
}
|
|
14805
|
+
/**
|
|
14806
|
+
* Update the session capability token at runtime.
|
|
14807
|
+
* Forwards the signed JWT to the worker for verification.
|
|
14808
|
+
*/
|
|
14809
|
+
updateSessionConfig(token) {
|
|
14810
|
+
this.workerManager?.postMessage("sc-update", { t: token });
|
|
14811
|
+
}
|
|
14812
|
+
/**
|
|
14813
|
+
* Set a cosmetic render hint for the branding overlay (logo variant / position).
|
|
14814
|
+
*/
|
|
14815
|
+
setRenderHint(hint) {
|
|
14816
|
+
this.workerManager?.postMessage("rh-update", { h: hint });
|
|
14817
|
+
}
|
|
14671
14818
|
/**
|
|
14672
14819
|
* Show the core's iframe (make it visible in the host container).
|
|
14673
14820
|
* No-op for headless cores.
|
|
@@ -14713,10 +14860,14 @@ class VijiCore {
|
|
|
14713
14860
|
}
|
|
14714
14861
|
this.workerManager = new WorkerManager(
|
|
14715
14862
|
this.config.sceneCode,
|
|
14716
|
-
offscreenCanvas
|
|
14863
|
+
offscreenCanvas,
|
|
14864
|
+
this.isHeadless
|
|
14717
14865
|
);
|
|
14718
14866
|
this.setupCommunication();
|
|
14719
14867
|
await this.workerManager.createWorker();
|
|
14868
|
+
if (this.config._sc) {
|
|
14869
|
+
this.workerManager.postMessage("sc-update", { t: this.config._sc });
|
|
14870
|
+
}
|
|
14720
14871
|
this.audioSystem = new AudioSystem((message) => {
|
|
14721
14872
|
if (this.workerManager) {
|
|
14722
14873
|
this.workerManager.postMessage(message.type, message.data);
|
|
@@ -14774,7 +14925,8 @@ class VijiCore {
|
|
|
14774
14925
|
const effectiveResolution = this.iframeManager.getEffectiveResolution();
|
|
14775
14926
|
this.workerManager.postMessage("resolution-update", {
|
|
14776
14927
|
effectiveWidth: effectiveResolution.width,
|
|
14777
|
-
effectiveHeight: effectiveResolution.height
|
|
14928
|
+
effectiveHeight: effectiveResolution.height,
|
|
14929
|
+
displayScale: this.iframeManager.getDisplayScale()
|
|
14778
14930
|
});
|
|
14779
14931
|
await this.detectScreenRefreshRate();
|
|
14780
14932
|
this.workerManager.postMessage("refresh-rate-update", {
|
|
@@ -14783,6 +14935,14 @@ class VijiCore {
|
|
|
14783
14935
|
if (this.config.audioStream) {
|
|
14784
14936
|
await this.setAudioStream(this.config.audioStream);
|
|
14785
14937
|
}
|
|
14938
|
+
for (let i = 0; i < this.audioStreams.length; i++) {
|
|
14939
|
+
const streamIndex = VijiCore.AUDIO_ADDITIONAL_BASE + i;
|
|
14940
|
+
await this.audioSystem.addStream(streamIndex, this.audioStreams[i]);
|
|
14941
|
+
this.workerManager.postMessage("audio-stream-setup", {
|
|
14942
|
+
streamIndex,
|
|
14943
|
+
streamType: "additional"
|
|
14944
|
+
});
|
|
14945
|
+
}
|
|
14786
14946
|
this.stats.resolution = effectiveResolution;
|
|
14787
14947
|
this.stats.scale = this.iframeManager.getScale();
|
|
14788
14948
|
this.updateFrameRateStats();
|
|
@@ -15505,7 +15665,8 @@ class VijiCore {
|
|
|
15505
15665
|
const effectiveResolution = this.iframeManager.getEffectiveResolution();
|
|
15506
15666
|
this.workerManager.postMessage("resolution-update", {
|
|
15507
15667
|
effectiveWidth: effectiveResolution.width,
|
|
15508
|
-
effectiveHeight: effectiveResolution.height
|
|
15668
|
+
effectiveHeight: effectiveResolution.height,
|
|
15669
|
+
displayScale: this.iframeManager.getDisplayScale()
|
|
15509
15670
|
});
|
|
15510
15671
|
this.stats.resolution = effectiveResolution;
|
|
15511
15672
|
this.stats.scale = this.iframeManager.getScale();
|
|
@@ -15586,12 +15747,18 @@ class VijiCore {
|
|
|
15586
15747
|
// ADDITIONAL VIDEO STREAMS API
|
|
15587
15748
|
// ═══════════════════════════════════════════════════════
|
|
15588
15749
|
/**
|
|
15589
|
-
* Adds an additional video stream (no CV). Returns its index in viji.
|
|
15750
|
+
* Adds an additional video stream (no CV). Returns its index in viji.videoStreams[].
|
|
15590
15751
|
*/
|
|
15591
15752
|
async addVideoStream(stream) {
|
|
15592
15753
|
this.validateReady();
|
|
15593
15754
|
const existingIndex = this.videoStreams.indexOf(stream);
|
|
15594
15755
|
if (existingIndex !== -1) return existingIndex;
|
|
15756
|
+
if (this.videoStreams.length >= VijiCore.MAX_ADDITIONAL_VIDEO_STREAMS) {
|
|
15757
|
+
throw new VijiCoreError(
|
|
15758
|
+
`Maximum additional video streams reached (${VijiCore.MAX_ADDITIONAL_VIDEO_STREAMS})`,
|
|
15759
|
+
"STREAM_LIMIT_REACHED"
|
|
15760
|
+
);
|
|
15761
|
+
}
|
|
15595
15762
|
this.videoStreams.push(stream);
|
|
15596
15763
|
const newIndex = this.videoStreams.length - 1;
|
|
15597
15764
|
const streamIndex = VijiCore.ADDITIONAL_STREAM_BASE + newIndex;
|
|
@@ -15636,6 +15803,103 @@ class VijiCore {
|
|
|
15636
15803
|
getVideoStreamCount() {
|
|
15637
15804
|
return this.videoStreams.length;
|
|
15638
15805
|
}
|
|
15806
|
+
// ═══════════════════════════════════════════════════════════
|
|
15807
|
+
// Audio Stream Management (mirroring video stream API)
|
|
15808
|
+
// ═══════════════════════════════════════════════════════════
|
|
15809
|
+
/**
|
|
15810
|
+
* Add an additional audio stream for lightweight analysis (no beat detection).
|
|
15811
|
+
* @param stream MediaStream with audio tracks
|
|
15812
|
+
* @returns Index of the new stream (0-based within additional streams)
|
|
15813
|
+
* @throws VijiCoreError if limit reached (max 8)
|
|
15814
|
+
*/
|
|
15815
|
+
async addAudioStream(stream) {
|
|
15816
|
+
this.validateReady();
|
|
15817
|
+
const existingIndex = this.audioStreams.indexOf(stream);
|
|
15818
|
+
if (existingIndex !== -1) return existingIndex;
|
|
15819
|
+
if (this.audioStreams.length >= VijiCore.MAX_ADDITIONAL_AUDIO_STREAMS) {
|
|
15820
|
+
throw new VijiCoreError(
|
|
15821
|
+
`Maximum additional audio streams reached (${VijiCore.MAX_ADDITIONAL_AUDIO_STREAMS})`,
|
|
15822
|
+
"STREAM_LIMIT_REACHED"
|
|
15823
|
+
);
|
|
15824
|
+
}
|
|
15825
|
+
this.audioStreams.push(stream);
|
|
15826
|
+
const newIndex = this.audioStreams.length - 1;
|
|
15827
|
+
const streamIndex = VijiCore.AUDIO_ADDITIONAL_BASE + newIndex;
|
|
15828
|
+
if (this.audioSystem) {
|
|
15829
|
+
await this.audioSystem.addStream(streamIndex, stream);
|
|
15830
|
+
}
|
|
15831
|
+
this.workerManager?.postMessage("audio-stream-setup", {
|
|
15832
|
+
streamIndex,
|
|
15833
|
+
streamType: "additional"
|
|
15834
|
+
});
|
|
15835
|
+
return newIndex;
|
|
15836
|
+
}
|
|
15837
|
+
/**
|
|
15838
|
+
* Remove an additional audio stream by index.
|
|
15839
|
+
* Triggers re-indexing of remaining streams.
|
|
15840
|
+
*/
|
|
15841
|
+
async removeAudioStream(index) {
|
|
15842
|
+
this.validateReady();
|
|
15843
|
+
if (index < 0 || index >= this.audioStreams.length) {
|
|
15844
|
+
throw new VijiCoreError(`Invalid audio stream index: ${index}`, "INVALID_INDEX");
|
|
15845
|
+
}
|
|
15846
|
+
this.audioStreams[index].getTracks().forEach((track) => track.stop());
|
|
15847
|
+
this.audioStreams.splice(index, 1);
|
|
15848
|
+
if (this.audioSystem) {
|
|
15849
|
+
await this.audioSystem.reinitializeAdditionalChannels(
|
|
15850
|
+
this.audioStreams,
|
|
15851
|
+
VijiCore.AUDIO_ADDITIONAL_BASE
|
|
15852
|
+
);
|
|
15853
|
+
}
|
|
15854
|
+
for (let i = 0; i < this.audioStreams.length; i++) {
|
|
15855
|
+
this.workerManager?.postMessage("audio-stream-setup", {
|
|
15856
|
+
streamIndex: VijiCore.AUDIO_ADDITIONAL_BASE + i,
|
|
15857
|
+
streamType: "additional"
|
|
15858
|
+
});
|
|
15859
|
+
}
|
|
15860
|
+
}
|
|
15861
|
+
/**
|
|
15862
|
+
* Gets the number of additional audio streams.
|
|
15863
|
+
*/
|
|
15864
|
+
getAudioStreamCount() {
|
|
15865
|
+
return this.audioStreams.length;
|
|
15866
|
+
}
|
|
15867
|
+
/**
|
|
15868
|
+
* Set audio stream from an external device.
|
|
15869
|
+
* @param deviceId Device identifier (must be registered via addExternalDevice)
|
|
15870
|
+
* @param stream MediaStream with audio tracks from the device
|
|
15871
|
+
*/
|
|
15872
|
+
async setDeviceAudio(deviceId, stream) {
|
|
15873
|
+
this.validateReady();
|
|
15874
|
+
await this.clearDeviceAudio(deviceId);
|
|
15875
|
+
const usedIndices = new Set(this.deviceAudioStreamIndices.values());
|
|
15876
|
+
let streamIndex = VijiCore.AUDIO_DEVICE_BASE;
|
|
15877
|
+
while (usedIndices.has(streamIndex)) {
|
|
15878
|
+
streamIndex++;
|
|
15879
|
+
}
|
|
15880
|
+
this.deviceAudioStreamIndices.set(deviceId, streamIndex);
|
|
15881
|
+
if (this.audioSystem) {
|
|
15882
|
+
await this.audioSystem.addStream(streamIndex, stream);
|
|
15883
|
+
}
|
|
15884
|
+
this.workerManager?.postMessage("audio-stream-setup", {
|
|
15885
|
+
streamIndex,
|
|
15886
|
+
streamType: "device",
|
|
15887
|
+
deviceId
|
|
15888
|
+
});
|
|
15889
|
+
this.debugLog(`Device audio set for ${deviceId} at index ${streamIndex}`);
|
|
15890
|
+
}
|
|
15891
|
+
/**
|
|
15892
|
+
* Clear audio stream from a device.
|
|
15893
|
+
*/
|
|
15894
|
+
async clearDeviceAudio(deviceId) {
|
|
15895
|
+
const streamIndex = this.deviceAudioStreamIndices.get(deviceId);
|
|
15896
|
+
if (streamIndex === void 0) return;
|
|
15897
|
+
this.deviceAudioStreamIndices.delete(deviceId);
|
|
15898
|
+
if (this.audioSystem) {
|
|
15899
|
+
this.audioSystem.removeStream(streamIndex);
|
|
15900
|
+
}
|
|
15901
|
+
this.debugLog(`Device audio cleared for ${deviceId}`);
|
|
15902
|
+
}
|
|
15639
15903
|
/**
|
|
15640
15904
|
* Reinitializes all additional coordinators after array mutation.
|
|
15641
15905
|
*/
|
|
@@ -15880,7 +16144,8 @@ class VijiCore {
|
|
|
15880
16144
|
}
|
|
15881
16145
|
this.workerManager.postMessage("resolution-update", {
|
|
15882
16146
|
effectiveWidth: effectiveResolution.width,
|
|
15883
|
-
effectiveHeight: effectiveResolution.height
|
|
16147
|
+
effectiveHeight: effectiveResolution.height,
|
|
16148
|
+
displayScale: this.iframeManager.getDisplayScale()
|
|
15884
16149
|
});
|
|
15885
16150
|
this.stats.resolution = effectiveResolution;
|
|
15886
16151
|
this.debugLog(`Resolution: ${effectiveResolution.width}x${effectiveResolution.height}`);
|
|
@@ -15970,6 +16235,9 @@ class VijiCore {
|
|
|
15970
16235
|
if (this.deviceVideoCoordinators.has(deviceId)) {
|
|
15971
16236
|
this.clearDeviceVideo(deviceId);
|
|
15972
16237
|
}
|
|
16238
|
+
if (this.deviceAudioStreamIndices.has(deviceId)) {
|
|
16239
|
+
this.clearDeviceAudio(deviceId);
|
|
16240
|
+
}
|
|
15973
16241
|
this.deviceSensorManager.removeExternalDevice(deviceId);
|
|
15974
16242
|
this.syncDeviceStateToWorker();
|
|
15975
16243
|
this.debugLog(`External device removed: ${deviceId}`);
|
|
@@ -16075,11 +16343,16 @@ class VijiCore {
|
|
|
16075
16343
|
this.deviceSensorManager.destroy();
|
|
16076
16344
|
this.deviceSensorManager = null;
|
|
16077
16345
|
}
|
|
16346
|
+
for (const [deviceId] of this.deviceAudioStreamIndices) {
|
|
16347
|
+
await this.clearDeviceAudio(deviceId);
|
|
16348
|
+
}
|
|
16349
|
+
this.deviceAudioStreamIndices.clear();
|
|
16078
16350
|
if (this.audioSystem) {
|
|
16079
16351
|
this.audioSystem.resetAudioState();
|
|
16080
16352
|
this.audioSystem = null;
|
|
16081
16353
|
}
|
|
16082
16354
|
this.currentAudioStream = null;
|
|
16355
|
+
this.audioStreams = [];
|
|
16083
16356
|
if (this.mainVideoCoordinator) {
|
|
16084
16357
|
this.mainVideoCoordinator.resetVideoState();
|
|
16085
16358
|
this.mainVideoCoordinator = null;
|
|
@@ -16158,4 +16431,4 @@ export {
|
|
|
16158
16431
|
VijiCoreError as b,
|
|
16159
16432
|
getDefaultExportFromCjs as g
|
|
16160
16433
|
};
|
|
16161
|
-
//# sourceMappingURL=index-
|
|
16434
|
+
//# sourceMappingURL=index-i7P2rHW8.js.map
|