@viji-dev/core 0.3.25 → 0.3.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global.d.ts +6 -43
- package/dist/artist-jsdoc.d.ts +3 -1
- package/dist/assets/cv-tasks.worker.js +108 -32
- package/dist/assets/{viji.worker-Zg128woJ.js → viji.worker-jTmB7qoQ.js} +546 -209
- package/dist/assets/viji.worker-jTmB7qoQ.js.map +1 -0
- package/dist/docs-api.js +2145 -513
- package/dist/{essentia-wasm.web-Dq7KVAx1.js → essentia-wasm.web-DtvIO6D8.js} +2 -2
- package/dist/{essentia-wasm.web-Dq7KVAx1.js.map → essentia-wasm.web-DtvIO6D8.js.map} +1 -1
- package/dist/{index-C6fMmKQj.js → index-9YDi9UBP.js} +1439 -1200
- package/dist/index-9YDi9UBP.js.map +1 -0
- package/dist/index.d.ts +153 -133
- package/dist/index.js +1 -1
- package/dist/shader-uniforms.js +540 -60
- package/package.json +1 -1
- package/dist/assets/viji.worker-Zg128woJ.js.map +0 -1
- package/dist/index-C6fMmKQj.js.map +0 -1
|
@@ -578,7 +578,7 @@ class IFrameManager {
|
|
|
578
578
|
}
|
|
579
579
|
function WorkerWrapper(options) {
|
|
580
580
|
return new Worker(
|
|
581
|
-
"" + new URL("assets/viji.worker-
|
|
581
|
+
"" + new URL("assets/viji.worker-jTmB7qoQ.js", import.meta.url).href,
|
|
582
582
|
{
|
|
583
583
|
type: "module",
|
|
584
584
|
name: options?.name
|
|
@@ -1076,75 +1076,6 @@ class EnvelopeFollower {
|
|
|
1076
1076
|
};
|
|
1077
1077
|
}
|
|
1078
1078
|
}
|
|
1079
|
-
class AutoGain {
|
|
1080
|
-
recentPeaks = [];
|
|
1081
|
-
gain = 1;
|
|
1082
|
-
targetLevel = 0.7;
|
|
1083
|
-
noiseFloor = 0.01;
|
|
1084
|
-
// -40dB noise floor
|
|
1085
|
-
windowSize;
|
|
1086
|
-
minGain = 0.5;
|
|
1087
|
-
maxGain = 4;
|
|
1088
|
-
gainSmoothingFactor = 0.01;
|
|
1089
|
-
/**
|
|
1090
|
-
* Create auto-gain processor
|
|
1091
|
-
* @param windowSizeMs - Window size in milliseconds for peak tracking (default 3000ms)
|
|
1092
|
-
* @param sampleRate - Sample rate in Hz (default 60 for 60fps)
|
|
1093
|
-
*/
|
|
1094
|
-
constructor(windowSizeMs = 3e3, sampleRate = 60) {
|
|
1095
|
-
this.windowSize = Math.floor(windowSizeMs / 1e3 * sampleRate);
|
|
1096
|
-
}
|
|
1097
|
-
/**
|
|
1098
|
-
* Process input value and return normalized output
|
|
1099
|
-
* @param input - Input value (0-1)
|
|
1100
|
-
* @returns Normalized value (0-1)
|
|
1101
|
-
*/
|
|
1102
|
-
process(input) {
|
|
1103
|
-
if (input < this.noiseFloor) {
|
|
1104
|
-
return 0;
|
|
1105
|
-
}
|
|
1106
|
-
this.recentPeaks.push(input);
|
|
1107
|
-
if (this.recentPeaks.length > this.windowSize) {
|
|
1108
|
-
this.recentPeaks.shift();
|
|
1109
|
-
}
|
|
1110
|
-
if (this.recentPeaks.length > 10) {
|
|
1111
|
-
const sorted = [...this.recentPeaks].sort((a, b) => b - a);
|
|
1112
|
-
const percentileIndex = Math.floor(sorted.length * 0.05);
|
|
1113
|
-
const currentPeak = sorted[percentileIndex];
|
|
1114
|
-
if (currentPeak > this.noiseFloor) {
|
|
1115
|
-
const targetGain = this.targetLevel / currentPeak;
|
|
1116
|
-
const clampedTarget = Math.max(this.minGain, Math.min(this.maxGain, targetGain));
|
|
1117
|
-
this.gain += (clampedTarget - this.gain) * this.gainSmoothingFactor;
|
|
1118
|
-
}
|
|
1119
|
-
}
|
|
1120
|
-
return Math.max(0, Math.min(1, input * this.gain));
|
|
1121
|
-
}
|
|
1122
|
-
/**
|
|
1123
|
-
* Get current gain value
|
|
1124
|
-
*/
|
|
1125
|
-
getGain() {
|
|
1126
|
-
return this.gain;
|
|
1127
|
-
}
|
|
1128
|
-
/**
|
|
1129
|
-
* Set target level (0-1)
|
|
1130
|
-
*/
|
|
1131
|
-
setTargetLevel(level) {
|
|
1132
|
-
this.targetLevel = Math.max(0.1, Math.min(1, level));
|
|
1133
|
-
}
|
|
1134
|
-
/**
|
|
1135
|
-
* Set noise floor threshold (0-1)
|
|
1136
|
-
*/
|
|
1137
|
-
setNoiseFloor(threshold) {
|
|
1138
|
-
this.noiseFloor = Math.max(0, Math.min(0.1, threshold));
|
|
1139
|
-
}
|
|
1140
|
-
/**
|
|
1141
|
-
* Reset auto-gain state
|
|
1142
|
-
*/
|
|
1143
|
-
reset() {
|
|
1144
|
-
this.recentPeaks = [];
|
|
1145
|
-
this.gain = 1;
|
|
1146
|
-
}
|
|
1147
|
-
}
|
|
1148
1079
|
const BAND_CONFIGS = {
|
|
1149
1080
|
low: {
|
|
1150
1081
|
minThreshold: 2e-3,
|
|
@@ -1801,7 +1732,7 @@ class EssentiaOnsetDetection {
|
|
|
1801
1732
|
this.initPromise = (async () => {
|
|
1802
1733
|
try {
|
|
1803
1734
|
const essentiaModule = await import("./essentia.js-core.es-DnrJE0uR.js");
|
|
1804
|
-
const wasmModule = await import("./essentia-wasm.web-
|
|
1735
|
+
const wasmModule = await import("./essentia-wasm.web-DtvIO6D8.js").then((n) => n.e);
|
|
1805
1736
|
const EssentiaClass = essentiaModule.Essentia || essentiaModule.default?.Essentia || essentiaModule.default;
|
|
1806
1737
|
let WASMModule = wasmModule.default || wasmModule.EssentiaWASM || wasmModule.default?.EssentiaWASM;
|
|
1807
1738
|
if (!WASMModule) {
|
|
@@ -11883,129 +11814,574 @@ Rejection rate: ${(rejections / duration * 60).toFixed(1)} per minute
|
|
|
11883
11814
|
return this.events.length;
|
|
11884
11815
|
}
|
|
11885
11816
|
}
|
|
11886
|
-
|
|
11887
|
-
|
|
11888
|
-
|
|
11889
|
-
|
|
11890
|
-
|
|
11891
|
-
|
|
11892
|
-
|
|
11893
|
-
|
|
11894
|
-
|
|
11895
|
-
|
|
11896
|
-
|
|
11897
|
-
|
|
11898
|
-
|
|
11899
|
-
|
|
11900
|
-
|
|
11901
|
-
|
|
11902
|
-
|
|
11903
|
-
|
|
11904
|
-
|
|
11905
|
-
|
|
11906
|
-
|
|
11907
|
-
|
|
11908
|
-
|
|
11909
|
-
|
|
11910
|
-
|
|
11911
|
-
|
|
11912
|
-
|
|
11817
|
+
const INSTRUMENTS = ["kick", "snare", "hat"];
|
|
11818
|
+
const SAMPLE_RATE = 60;
|
|
11819
|
+
const TAP_TIMEOUT_MS = 5e3;
|
|
11820
|
+
const MAX_CYCLE_LENGTH = 8;
|
|
11821
|
+
const FUZZY_TOLERANCE = 0.18;
|
|
11822
|
+
const MIN_REPETITIONS = 3;
|
|
11823
|
+
const MAX_TAP_INTERVAL_MS = 3e3;
|
|
11824
|
+
const MIN_TAP_INTERVAL_MS = 100;
|
|
11825
|
+
const MAX_TAP_HISTORY = 64;
|
|
11826
|
+
const MIN_EMA_ALPHA = 0.05;
|
|
11827
|
+
const PATTERN_SAME_TOLERANCE = 0.15;
|
|
11828
|
+
function createInstrumentState() {
|
|
11829
|
+
return {
|
|
11830
|
+
mode: "auto",
|
|
11831
|
+
muted: false,
|
|
11832
|
+
mutedAt: 0,
|
|
11833
|
+
tapIOIs: [],
|
|
11834
|
+
lastTapTime: 0,
|
|
11835
|
+
pattern: null,
|
|
11836
|
+
refinementIndex: 0,
|
|
11837
|
+
refinementCounts: [],
|
|
11838
|
+
replayLastEventTime: 0,
|
|
11839
|
+
replayIndex: 0,
|
|
11840
|
+
pendingTapEvents: [],
|
|
11841
|
+
envelope: new EnvelopeFollower(0, 300, SAMPLE_RATE),
|
|
11842
|
+
envelopeSmoothed: new EnvelopeFollower(5, 500, SAMPLE_RATE)
|
|
11843
|
+
};
|
|
11913
11844
|
}
|
|
11914
|
-
|
|
11915
|
-
|
|
11916
|
-
|
|
11917
|
-
|
|
11918
|
-
|
|
11919
|
-
|
|
11920
|
-
|
|
11921
|
-
|
|
11922
|
-
|
|
11923
|
-
|
|
11924
|
-
|
|
11925
|
-
|
|
11926
|
-
};
|
|
11927
|
-
FFT.prototype.toComplexArray = function toComplexArray(input, storage) {
|
|
11928
|
-
var res = storage || this.createComplexArray();
|
|
11929
|
-
for (var i = 0; i < res.length; i += 2) {
|
|
11930
|
-
res[i] = input[i >>> 1];
|
|
11931
|
-
res[i + 1] = 0;
|
|
11932
|
-
}
|
|
11933
|
-
return res;
|
|
11934
|
-
};
|
|
11935
|
-
FFT.prototype.completeSpectrum = function completeSpectrum(spectrum) {
|
|
11936
|
-
var size = this._csize;
|
|
11937
|
-
var half = size >>> 1;
|
|
11938
|
-
for (var i = 2; i < half; i += 2) {
|
|
11939
|
-
spectrum[size - i] = spectrum[i];
|
|
11940
|
-
spectrum[size - i + 1] = -spectrum[i + 1];
|
|
11941
|
-
}
|
|
11942
|
-
};
|
|
11943
|
-
FFT.prototype.transform = function transform(out, data) {
|
|
11944
|
-
if (out === data)
|
|
11945
|
-
throw new Error("Input and output buffers must be different");
|
|
11946
|
-
this._out = out;
|
|
11947
|
-
this._data = data;
|
|
11948
|
-
this._inv = 0;
|
|
11949
|
-
this._transform4();
|
|
11950
|
-
this._out = null;
|
|
11951
|
-
this._data = null;
|
|
11952
|
-
};
|
|
11953
|
-
FFT.prototype.realTransform = function realTransform(out, data) {
|
|
11954
|
-
if (out === data)
|
|
11955
|
-
throw new Error("Input and output buffers must be different");
|
|
11956
|
-
this._out = out;
|
|
11957
|
-
this._data = data;
|
|
11958
|
-
this._inv = 0;
|
|
11959
|
-
this._realTransform4();
|
|
11960
|
-
this._out = null;
|
|
11961
|
-
this._data = null;
|
|
11962
|
-
};
|
|
11963
|
-
FFT.prototype.inverseTransform = function inverseTransform(out, data) {
|
|
11964
|
-
if (out === data)
|
|
11965
|
-
throw new Error("Input and output buffers must be different");
|
|
11966
|
-
this._out = out;
|
|
11967
|
-
this._data = data;
|
|
11968
|
-
this._inv = 1;
|
|
11969
|
-
this._transform4();
|
|
11970
|
-
for (var i = 0; i < out.length; i++)
|
|
11971
|
-
out[i] /= this.size;
|
|
11972
|
-
this._out = null;
|
|
11973
|
-
this._data = null;
|
|
11974
|
-
};
|
|
11975
|
-
FFT.prototype._transform4 = function _transform4() {
|
|
11976
|
-
var out = this._out;
|
|
11977
|
-
var size = this._csize;
|
|
11978
|
-
var width = this._width;
|
|
11979
|
-
var step2 = 1 << width;
|
|
11980
|
-
var len = size / step2 << 1;
|
|
11981
|
-
var outOff;
|
|
11982
|
-
var t;
|
|
11983
|
-
var bitrev = this._bitrev;
|
|
11984
|
-
if (len === 4) {
|
|
11985
|
-
for (outOff = 0, t = 0; outOff < size; outOff += len, t++) {
|
|
11986
|
-
const off = bitrev[t];
|
|
11987
|
-
this._singleTransform2(outOff, off, step2);
|
|
11845
|
+
class OnsetTapManager {
|
|
11846
|
+
state = {
|
|
11847
|
+
kick: createInstrumentState(),
|
|
11848
|
+
snare: createInstrumentState(),
|
|
11849
|
+
hat: createInstrumentState()
|
|
11850
|
+
};
|
|
11851
|
+
tap(instrument) {
|
|
11852
|
+
const s = this.state[instrument];
|
|
11853
|
+
const now = performance.now();
|
|
11854
|
+
if (s.muted) {
|
|
11855
|
+
s.muted = false;
|
|
11856
|
+
s.mutedAt = 0;
|
|
11988
11857
|
}
|
|
11989
|
-
|
|
11990
|
-
|
|
11991
|
-
|
|
11992
|
-
|
|
11858
|
+
let ioi = -1;
|
|
11859
|
+
if (s.lastTapTime > 0) {
|
|
11860
|
+
ioi = now - s.lastTapTime;
|
|
11861
|
+
if (ioi < MIN_TAP_INTERVAL_MS) return;
|
|
11862
|
+
if (ioi > MAX_TAP_INTERVAL_MS) {
|
|
11863
|
+
s.tapIOIs = [];
|
|
11864
|
+
ioi = -1;
|
|
11865
|
+
} else {
|
|
11866
|
+
s.tapIOIs.push(ioi);
|
|
11867
|
+
if (s.tapIOIs.length > MAX_TAP_HISTORY) s.tapIOIs.shift();
|
|
11868
|
+
}
|
|
11869
|
+
}
|
|
11870
|
+
s.lastTapTime = now;
|
|
11871
|
+
s.pendingTapEvents.push(now);
|
|
11872
|
+
if (s.mode === "auto") {
|
|
11873
|
+
s.mode = "tapping";
|
|
11874
|
+
if (ioi > 0) {
|
|
11875
|
+
const pattern = this.tryRecognizePattern(instrument);
|
|
11876
|
+
if (pattern) this.applyPattern(instrument, pattern);
|
|
11877
|
+
}
|
|
11878
|
+
} else if (s.mode === "tapping") {
|
|
11879
|
+
if (ioi > 0) {
|
|
11880
|
+
const pattern = this.tryRecognizePattern(instrument);
|
|
11881
|
+
if (pattern) this.applyPattern(instrument, pattern);
|
|
11882
|
+
}
|
|
11883
|
+
} else if (s.mode === "pattern") {
|
|
11884
|
+
if (ioi > 0) {
|
|
11885
|
+
this.handlePatternTap(instrument, ioi, now);
|
|
11886
|
+
}
|
|
11993
11887
|
}
|
|
11994
11888
|
}
|
|
11995
|
-
|
|
11996
|
-
|
|
11997
|
-
|
|
11998
|
-
|
|
11999
|
-
|
|
12000
|
-
|
|
12001
|
-
|
|
12002
|
-
|
|
12003
|
-
|
|
12004
|
-
|
|
12005
|
-
|
|
12006
|
-
|
|
12007
|
-
|
|
12008
|
-
|
|
11889
|
+
clear(instrument) {
|
|
11890
|
+
const s = this.state[instrument];
|
|
11891
|
+
s.mode = "auto";
|
|
11892
|
+
s.muted = false;
|
|
11893
|
+
s.mutedAt = 0;
|
|
11894
|
+
s.tapIOIs = [];
|
|
11895
|
+
s.lastTapTime = 0;
|
|
11896
|
+
s.pattern = null;
|
|
11897
|
+
s.refinementIndex = 0;
|
|
11898
|
+
s.refinementCounts = [];
|
|
11899
|
+
s.replayLastEventTime = 0;
|
|
11900
|
+
s.replayIndex = 0;
|
|
11901
|
+
s.pendingTapEvents = [];
|
|
11902
|
+
s.envelope.reset();
|
|
11903
|
+
s.envelopeSmoothed.reset();
|
|
11904
|
+
}
|
|
11905
|
+
getMode(instrument) {
|
|
11906
|
+
return this.state[instrument].mode;
|
|
11907
|
+
}
|
|
11908
|
+
getPatternInfo(instrument) {
|
|
11909
|
+
const s = this.state[instrument];
|
|
11910
|
+
if (!s.pattern) return null;
|
|
11911
|
+
return {
|
|
11912
|
+
length: s.pattern.length,
|
|
11913
|
+
intervals: [...s.pattern],
|
|
11914
|
+
confidence: 1
|
|
11915
|
+
};
|
|
11916
|
+
}
|
|
11917
|
+
setMuted(instrument, muted) {
|
|
11918
|
+
const s = this.state[instrument];
|
|
11919
|
+
if (s.muted === muted) return;
|
|
11920
|
+
const now = performance.now();
|
|
11921
|
+
if (muted) {
|
|
11922
|
+
s.muted = true;
|
|
11923
|
+
s.mutedAt = now;
|
|
11924
|
+
} else {
|
|
11925
|
+
const pauseDuration = now - s.mutedAt;
|
|
11926
|
+
s.muted = false;
|
|
11927
|
+
s.mutedAt = 0;
|
|
11928
|
+
if (s.mode === "tapping" && s.lastTapTime > 0) {
|
|
11929
|
+
s.lastTapTime += pauseDuration;
|
|
11930
|
+
}
|
|
11931
|
+
}
|
|
11932
|
+
}
|
|
11933
|
+
isMuted(instrument) {
|
|
11934
|
+
return this.state[instrument].muted;
|
|
11935
|
+
}
|
|
11936
|
+
/**
|
|
11937
|
+
* Post-process a beat state produced by BeatStateManager.
|
|
11938
|
+
* For instruments in auto mode, values pass through unchanged.
|
|
11939
|
+
* For tapping/pattern instruments, auto events are suppressed and
|
|
11940
|
+
* tap/pattern events + envelopes are injected instead.
|
|
11941
|
+
*/
|
|
11942
|
+
processFrame(beatState, now, dtMs) {
|
|
11943
|
+
const result = { ...beatState, events: [...beatState.events] };
|
|
11944
|
+
const newEvents = [];
|
|
11945
|
+
for (const inst of INSTRUMENTS) {
|
|
11946
|
+
const s = this.state[inst];
|
|
11947
|
+
if (s.muted) {
|
|
11948
|
+
result.events = result.events.filter((e) => e.type !== inst);
|
|
11949
|
+
s.pendingTapEvents = [];
|
|
11950
|
+
if (s.mode === "pattern" && s.pattern && s.replayLastEventTime > 0) {
|
|
11951
|
+
for (let i = 0; i < 3; i++) {
|
|
11952
|
+
const interval = s.pattern[s.replayIndex % s.pattern.length];
|
|
11953
|
+
if (now - s.replayLastEventTime >= interval) {
|
|
11954
|
+
s.replayLastEventTime += interval;
|
|
11955
|
+
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
11956
|
+
} else {
|
|
11957
|
+
break;
|
|
11958
|
+
}
|
|
11959
|
+
}
|
|
11960
|
+
}
|
|
11961
|
+
s.envelope.process(0, dtMs);
|
|
11962
|
+
s.envelopeSmoothed.process(0, dtMs);
|
|
11963
|
+
const envKey2 = inst;
|
|
11964
|
+
const smoothKey2 = `${inst}Smoothed`;
|
|
11965
|
+
result[envKey2] = s.envelope.getValue();
|
|
11966
|
+
result[smoothKey2] = s.envelopeSmoothed.getValue();
|
|
11967
|
+
continue;
|
|
11968
|
+
}
|
|
11969
|
+
if (s.mode === "auto") {
|
|
11970
|
+
s.envelope.process(0, dtMs);
|
|
11971
|
+
s.envelopeSmoothed.process(0, dtMs);
|
|
11972
|
+
continue;
|
|
11973
|
+
}
|
|
11974
|
+
result.events = result.events.filter((e) => e.type !== inst);
|
|
11975
|
+
if (s.mode === "tapping") {
|
|
11976
|
+
if (now - s.lastTapTime > TAP_TIMEOUT_MS) {
|
|
11977
|
+
s.tapIOIs = [];
|
|
11978
|
+
s.pendingTapEvents = [];
|
|
11979
|
+
if (s.pattern) {
|
|
11980
|
+
s.mode = "pattern";
|
|
11981
|
+
s.replayLastEventTime = now;
|
|
11982
|
+
s.replayIndex = 0;
|
|
11983
|
+
} else {
|
|
11984
|
+
s.mode = "auto";
|
|
11985
|
+
}
|
|
11986
|
+
continue;
|
|
11987
|
+
}
|
|
11988
|
+
while (s.pendingTapEvents.length > 0) {
|
|
11989
|
+
const tapTime = s.pendingTapEvents.shift();
|
|
11990
|
+
newEvents.push(this.createTapEvent(inst, tapTime, result.bpm));
|
|
11991
|
+
s.envelope.trigger(1);
|
|
11992
|
+
s.envelopeSmoothed.trigger(1);
|
|
11993
|
+
s.replayLastEventTime = tapTime;
|
|
11994
|
+
s.replayIndex = 0;
|
|
11995
|
+
}
|
|
11996
|
+
}
|
|
11997
|
+
if (s.mode === "pattern") {
|
|
11998
|
+
while (s.pendingTapEvents.length > 0) {
|
|
11999
|
+
const tapTime = s.pendingTapEvents.shift();
|
|
12000
|
+
newEvents.push(this.createTapEvent(inst, tapTime, result.bpm));
|
|
12001
|
+
s.envelope.trigger(1);
|
|
12002
|
+
s.envelopeSmoothed.trigger(1);
|
|
12003
|
+
}
|
|
12004
|
+
const isUserActive = now - s.lastTapTime < 500;
|
|
12005
|
+
if (!isUserActive && s.pattern) {
|
|
12006
|
+
const scheduled = this.checkPatternReplay(s, inst, now, result.bpm);
|
|
12007
|
+
for (const ev of scheduled) {
|
|
12008
|
+
newEvents.push(ev);
|
|
12009
|
+
s.envelope.trigger(1);
|
|
12010
|
+
s.envelopeSmoothed.trigger(1);
|
|
12011
|
+
}
|
|
12012
|
+
}
|
|
12013
|
+
}
|
|
12014
|
+
s.envelope.process(0, dtMs);
|
|
12015
|
+
s.envelopeSmoothed.process(0, dtMs);
|
|
12016
|
+
const envKey = inst;
|
|
12017
|
+
const smoothKey = `${inst}Smoothed`;
|
|
12018
|
+
result[envKey] = s.envelope.getValue();
|
|
12019
|
+
result[smoothKey] = s.envelopeSmoothed.getValue();
|
|
12020
|
+
}
|
|
12021
|
+
result.events.push(...newEvents);
|
|
12022
|
+
const anyMax = Math.max(
|
|
12023
|
+
result.kick,
|
|
12024
|
+
result.snare,
|
|
12025
|
+
result.hat
|
|
12026
|
+
);
|
|
12027
|
+
const anySmoothedMax = Math.max(
|
|
12028
|
+
result.kickSmoothed,
|
|
12029
|
+
result.snareSmoothed,
|
|
12030
|
+
result.hatSmoothed
|
|
12031
|
+
);
|
|
12032
|
+
if (anyMax > result.any) result.any = anyMax;
|
|
12033
|
+
if (anySmoothedMax > result.anySmoothed) result.anySmoothed = anySmoothedMax;
|
|
12034
|
+
return result;
|
|
12035
|
+
}
|
|
12036
|
+
// ---------------------------------------------------------------------------
|
|
12037
|
+
// Private helpers
|
|
12038
|
+
// ---------------------------------------------------------------------------
|
|
12039
|
+
/**
|
|
12040
|
+
* Handle a tap that arrives while already in pattern mode.
|
|
12041
|
+
* Matching taps refine the pattern via EMA and re-anchor phase.
|
|
12042
|
+
* Non-matching taps trigger new pattern detection on the full IOI history.
|
|
12043
|
+
*/
|
|
12044
|
+
handlePatternTap(instrument, ioi, now) {
|
|
12045
|
+
const s = this.state[instrument];
|
|
12046
|
+
if (!s.pattern) return;
|
|
12047
|
+
const matchedPos = this.findMatchingPosition(s, ioi);
|
|
12048
|
+
if (matchedPos >= 0) {
|
|
12049
|
+
s.refinementCounts[matchedPos] = (s.refinementCounts[matchedPos] || 0) + 1;
|
|
12050
|
+
const alpha = Math.max(MIN_EMA_ALPHA, 1 / s.refinementCounts[matchedPos]);
|
|
12051
|
+
s.pattern[matchedPos] = (1 - alpha) * s.pattern[matchedPos] + alpha * ioi;
|
|
12052
|
+
s.refinementIndex = (matchedPos + 1) % s.pattern.length;
|
|
12053
|
+
s.replayLastEventTime = now;
|
|
12054
|
+
s.replayIndex = s.refinementIndex;
|
|
12055
|
+
} else {
|
|
12056
|
+
const currentPattern = [...s.pattern];
|
|
12057
|
+
const newPattern = this.tryRecognizePattern(instrument);
|
|
12058
|
+
if (newPattern && !this.isPatternSame(newPattern, currentPattern)) {
|
|
12059
|
+
this.applyPattern(instrument, newPattern);
|
|
12060
|
+
}
|
|
12061
|
+
}
|
|
12062
|
+
}
|
|
12063
|
+
/**
|
|
12064
|
+
* Find which pattern position the given IOI matches.
|
|
12065
|
+
* Tries the expected refinementIndex first, then scans all positions.
|
|
12066
|
+
* Returns the matched position index, or -1 if no match.
|
|
12067
|
+
*/
|
|
12068
|
+
findMatchingPosition(s, ioi) {
|
|
12069
|
+
if (!s.pattern) return -1;
|
|
12070
|
+
const expectedPos = s.refinementIndex % s.pattern.length;
|
|
12071
|
+
const expectedInterval = s.pattern[expectedPos];
|
|
12072
|
+
if (Math.abs(ioi - expectedInterval) / expectedInterval <= FUZZY_TOLERANCE) {
|
|
12073
|
+
return expectedPos;
|
|
12074
|
+
}
|
|
12075
|
+
let bestPos = -1;
|
|
12076
|
+
let bestDeviation = Infinity;
|
|
12077
|
+
for (let i = 0; i < s.pattern.length; i++) {
|
|
12078
|
+
if (i === expectedPos) continue;
|
|
12079
|
+
const deviation = Math.abs(ioi - s.pattern[i]) / s.pattern[i];
|
|
12080
|
+
if (deviation <= FUZZY_TOLERANCE && deviation < bestDeviation) {
|
|
12081
|
+
bestDeviation = deviation;
|
|
12082
|
+
bestPos = i;
|
|
12083
|
+
}
|
|
12084
|
+
}
|
|
12085
|
+
return bestPos;
|
|
12086
|
+
}
|
|
12087
|
+
/**
|
|
12088
|
+
* Compare two patterns for equivalence.
|
|
12089
|
+
* Same length and each interval within PATTERN_SAME_TOLERANCE → same.
|
|
12090
|
+
*/
|
|
12091
|
+
isPatternSame(a, b) {
|
|
12092
|
+
if (a.length !== b.length) return false;
|
|
12093
|
+
for (let i = 0; i < a.length; i++) {
|
|
12094
|
+
if (Math.abs(a[i] - b[i]) / Math.max(a[i], b[i]) > PATTERN_SAME_TOLERANCE) return false;
|
|
12095
|
+
}
|
|
12096
|
+
return true;
|
|
12097
|
+
}
|
|
12098
|
+
/**
|
|
12099
|
+
* Pattern recognition: find the shortest repeating IOI cycle
|
|
12100
|
+
* that has been tapped at least MIN_REPETITIONS times.
|
|
12101
|
+
* Returns the recognized pattern as float intervals, or null.
|
|
12102
|
+
* Does NOT mutate state — caller decides what to do with the result.
|
|
12103
|
+
*/
|
|
12104
|
+
tryRecognizePattern(instrument) {
|
|
12105
|
+
const s = this.state[instrument];
|
|
12106
|
+
const iois = s.tapIOIs;
|
|
12107
|
+
if (iois.length < MIN_REPETITIONS) return null;
|
|
12108
|
+
const maxL = Math.min(MAX_CYCLE_LENGTH, Math.floor(iois.length / MIN_REPETITIONS));
|
|
12109
|
+
for (let L = 1; L <= maxL; L++) {
|
|
12110
|
+
const needed = MIN_REPETITIONS * L;
|
|
12111
|
+
if (iois.length < needed) continue;
|
|
12112
|
+
const tolerance = L <= 2 ? FUZZY_TOLERANCE : FUZZY_TOLERANCE * (2 / L);
|
|
12113
|
+
const recent = iois.slice(-needed);
|
|
12114
|
+
const groups = [];
|
|
12115
|
+
for (let g = 0; g < MIN_REPETITIONS; g++) {
|
|
12116
|
+
groups.push(recent.slice(g * L, (g + 1) * L));
|
|
12117
|
+
}
|
|
12118
|
+
const avgGroup = groups[0].map((_, i) => {
|
|
12119
|
+
let sum = 0;
|
|
12120
|
+
for (const grp of groups) sum += grp[i];
|
|
12121
|
+
return sum / groups.length;
|
|
12122
|
+
});
|
|
12123
|
+
let allMatch = true;
|
|
12124
|
+
for (const grp of groups) {
|
|
12125
|
+
for (let i = 0; i < L; i++) {
|
|
12126
|
+
const deviation = Math.abs(grp[i] - avgGroup[i]) / avgGroup[i];
|
|
12127
|
+
if (deviation > tolerance) {
|
|
12128
|
+
allMatch = false;
|
|
12129
|
+
break;
|
|
12130
|
+
}
|
|
12131
|
+
}
|
|
12132
|
+
if (!allMatch) break;
|
|
12133
|
+
}
|
|
12134
|
+
if (allMatch) {
|
|
12135
|
+
return avgGroup;
|
|
12136
|
+
}
|
|
12137
|
+
}
|
|
12138
|
+
return null;
|
|
12139
|
+
}
|
|
12140
|
+
/**
|
|
12141
|
+
* Apply a recognized pattern to the instrument state.
|
|
12142
|
+
* Sets pattern, mode, and initializes replay anchor.
|
|
12143
|
+
*/
|
|
12144
|
+
applyPattern(instrument, pattern) {
|
|
12145
|
+
const s = this.state[instrument];
|
|
12146
|
+
s.pattern = pattern;
|
|
12147
|
+
s.mode = "pattern";
|
|
12148
|
+
s.refinementIndex = 0;
|
|
12149
|
+
s.refinementCounts = new Array(pattern.length).fill(MIN_REPETITIONS);
|
|
12150
|
+
if (s.replayLastEventTime === 0 || s.replayLastEventTime < s.lastTapTime) {
|
|
12151
|
+
s.replayLastEventTime = s.lastTapTime;
|
|
12152
|
+
s.replayIndex = 0;
|
|
12153
|
+
}
|
|
12154
|
+
}
|
|
12155
|
+
createTapEvent(instrument, time, bpm) {
|
|
12156
|
+
return {
|
|
12157
|
+
type: instrument,
|
|
12158
|
+
time,
|
|
12159
|
+
strength: 0.85,
|
|
12160
|
+
isPredicted: false,
|
|
12161
|
+
bpm
|
|
12162
|
+
};
|
|
12163
|
+
}
|
|
12164
|
+
checkPatternReplay(s, instrument, now, bpm) {
|
|
12165
|
+
if (!s.pattern || s.pattern.length === 0) return [];
|
|
12166
|
+
const events = [];
|
|
12167
|
+
const maxEventsPerFrame = 3;
|
|
12168
|
+
for (let safety = 0; safety < maxEventsPerFrame; safety++) {
|
|
12169
|
+
const expectedInterval = s.pattern[s.replayIndex % s.pattern.length];
|
|
12170
|
+
const elapsed = now - s.replayLastEventTime;
|
|
12171
|
+
if (elapsed >= expectedInterval) {
|
|
12172
|
+
s.replayLastEventTime += expectedInterval;
|
|
12173
|
+
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
12174
|
+
events.push({
|
|
12175
|
+
type: instrument,
|
|
12176
|
+
time: s.replayLastEventTime,
|
|
12177
|
+
strength: 0.8,
|
|
12178
|
+
isPredicted: true,
|
|
12179
|
+
bpm
|
|
12180
|
+
});
|
|
12181
|
+
} else {
|
|
12182
|
+
break;
|
|
12183
|
+
}
|
|
12184
|
+
}
|
|
12185
|
+
return events;
|
|
12186
|
+
}
|
|
12187
|
+
reset() {
|
|
12188
|
+
for (const inst of INSTRUMENTS) {
|
|
12189
|
+
this.clear(inst);
|
|
12190
|
+
}
|
|
12191
|
+
}
|
|
12192
|
+
}
|
|
12193
|
+
class AutoGain {
|
|
12194
|
+
recentPeaks = [];
|
|
12195
|
+
gain = 1;
|
|
12196
|
+
targetLevel = 0.7;
|
|
12197
|
+
noiseFloor = 0.01;
|
|
12198
|
+
// -40dB noise floor
|
|
12199
|
+
windowSize;
|
|
12200
|
+
minGain = 0.5;
|
|
12201
|
+
maxGain = 4;
|
|
12202
|
+
gainSmoothingFactor = 0.01;
|
|
12203
|
+
/**
|
|
12204
|
+
* Create auto-gain processor
|
|
12205
|
+
* @param windowSizeMs - Window size in milliseconds for peak tracking (default 3000ms)
|
|
12206
|
+
* @param sampleRate - Sample rate in Hz (default 60 for 60fps)
|
|
12207
|
+
*/
|
|
12208
|
+
constructor(windowSizeMs = 3e3, sampleRate = 60) {
|
|
12209
|
+
this.windowSize = Math.floor(windowSizeMs / 1e3 * sampleRate);
|
|
12210
|
+
}
|
|
12211
|
+
/**
|
|
12212
|
+
* Process input value and return normalized output
|
|
12213
|
+
* @param input - Input value (0-1)
|
|
12214
|
+
* @returns Normalized value (0-1)
|
|
12215
|
+
*/
|
|
12216
|
+
process(input) {
|
|
12217
|
+
if (input < this.noiseFloor) {
|
|
12218
|
+
return 0;
|
|
12219
|
+
}
|
|
12220
|
+
this.recentPeaks.push(input);
|
|
12221
|
+
if (this.recentPeaks.length > this.windowSize) {
|
|
12222
|
+
this.recentPeaks.shift();
|
|
12223
|
+
}
|
|
12224
|
+
if (this.recentPeaks.length > 10) {
|
|
12225
|
+
const sorted = [...this.recentPeaks].sort((a, b) => b - a);
|
|
12226
|
+
const percentileIndex = Math.floor(sorted.length * 0.05);
|
|
12227
|
+
const currentPeak = sorted[percentileIndex];
|
|
12228
|
+
if (currentPeak > this.noiseFloor) {
|
|
12229
|
+
const targetGain = this.targetLevel / currentPeak;
|
|
12230
|
+
const clampedTarget = Math.max(this.minGain, Math.min(this.maxGain, targetGain));
|
|
12231
|
+
this.gain += (clampedTarget - this.gain) * this.gainSmoothingFactor;
|
|
12232
|
+
}
|
|
12233
|
+
}
|
|
12234
|
+
return Math.max(0, Math.min(1, input * this.gain));
|
|
12235
|
+
}
|
|
12236
|
+
/**
|
|
12237
|
+
* Get current gain value
|
|
12238
|
+
*/
|
|
12239
|
+
getGain() {
|
|
12240
|
+
return this.gain;
|
|
12241
|
+
}
|
|
12242
|
+
/**
|
|
12243
|
+
* Set target level (0-1)
|
|
12244
|
+
*/
|
|
12245
|
+
setTargetLevel(level) {
|
|
12246
|
+
this.targetLevel = Math.max(0.1, Math.min(1, level));
|
|
12247
|
+
}
|
|
12248
|
+
/**
|
|
12249
|
+
* Set noise floor threshold (0-1)
|
|
12250
|
+
*/
|
|
12251
|
+
setNoiseFloor(threshold) {
|
|
12252
|
+
this.noiseFloor = Math.max(0, Math.min(0.1, threshold));
|
|
12253
|
+
}
|
|
12254
|
+
/**
|
|
12255
|
+
* Reset auto-gain state
|
|
12256
|
+
*/
|
|
12257
|
+
reset() {
|
|
12258
|
+
this.recentPeaks = [];
|
|
12259
|
+
this.gain = 1;
|
|
12260
|
+
}
|
|
12261
|
+
}
|
|
12262
|
+
function FFT(size) {
|
|
12263
|
+
this.size = size | 0;
|
|
12264
|
+
if (this.size <= 1 || (this.size & this.size - 1) !== 0)
|
|
12265
|
+
throw new Error("FFT size must be a power of two and bigger than 1");
|
|
12266
|
+
this._csize = size << 1;
|
|
12267
|
+
var table = new Array(this.size * 2);
|
|
12268
|
+
for (var i = 0; i < table.length; i += 2) {
|
|
12269
|
+
const angle = Math.PI * i / this.size;
|
|
12270
|
+
table[i] = Math.cos(angle);
|
|
12271
|
+
table[i + 1] = -Math.sin(angle);
|
|
12272
|
+
}
|
|
12273
|
+
this.table = table;
|
|
12274
|
+
var power = 0;
|
|
12275
|
+
for (var t = 1; this.size > t; t <<= 1)
|
|
12276
|
+
power++;
|
|
12277
|
+
this._width = power % 2 === 0 ? power - 1 : power;
|
|
12278
|
+
this._bitrev = new Array(1 << this._width);
|
|
12279
|
+
for (var j = 0; j < this._bitrev.length; j++) {
|
|
12280
|
+
this._bitrev[j] = 0;
|
|
12281
|
+
for (var shift = 0; shift < this._width; shift += 2) {
|
|
12282
|
+
var revShift = this._width - shift - 2;
|
|
12283
|
+
this._bitrev[j] |= (j >>> shift & 3) << revShift;
|
|
12284
|
+
}
|
|
12285
|
+
}
|
|
12286
|
+
this._out = null;
|
|
12287
|
+
this._data = null;
|
|
12288
|
+
this._inv = 0;
|
|
12289
|
+
}
|
|
12290
|
+
var fft = FFT;
|
|
12291
|
+
FFT.prototype.fromComplexArray = function fromComplexArray(complex, storage) {
|
|
12292
|
+
var res = storage || new Array(complex.length >>> 1);
|
|
12293
|
+
for (var i = 0; i < complex.length; i += 2)
|
|
12294
|
+
res[i >>> 1] = complex[i];
|
|
12295
|
+
return res;
|
|
12296
|
+
};
|
|
12297
|
+
FFT.prototype.createComplexArray = function createComplexArray() {
|
|
12298
|
+
const res = new Array(this._csize);
|
|
12299
|
+
for (var i = 0; i < res.length; i++)
|
|
12300
|
+
res[i] = 0;
|
|
12301
|
+
return res;
|
|
12302
|
+
};
|
|
12303
|
+
FFT.prototype.toComplexArray = function toComplexArray(input, storage) {
|
|
12304
|
+
var res = storage || this.createComplexArray();
|
|
12305
|
+
for (var i = 0; i < res.length; i += 2) {
|
|
12306
|
+
res[i] = input[i >>> 1];
|
|
12307
|
+
res[i + 1] = 0;
|
|
12308
|
+
}
|
|
12309
|
+
return res;
|
|
12310
|
+
};
|
|
12311
|
+
FFT.prototype.completeSpectrum = function completeSpectrum(spectrum) {
|
|
12312
|
+
var size = this._csize;
|
|
12313
|
+
var half = size >>> 1;
|
|
12314
|
+
for (var i = 2; i < half; i += 2) {
|
|
12315
|
+
spectrum[size - i] = spectrum[i];
|
|
12316
|
+
spectrum[size - i + 1] = -spectrum[i + 1];
|
|
12317
|
+
}
|
|
12318
|
+
};
|
|
12319
|
+
FFT.prototype.transform = function transform(out, data) {
|
|
12320
|
+
if (out === data)
|
|
12321
|
+
throw new Error("Input and output buffers must be different");
|
|
12322
|
+
this._out = out;
|
|
12323
|
+
this._data = data;
|
|
12324
|
+
this._inv = 0;
|
|
12325
|
+
this._transform4();
|
|
12326
|
+
this._out = null;
|
|
12327
|
+
this._data = null;
|
|
12328
|
+
};
|
|
12329
|
+
FFT.prototype.realTransform = function realTransform(out, data) {
|
|
12330
|
+
if (out === data)
|
|
12331
|
+
throw new Error("Input and output buffers must be different");
|
|
12332
|
+
this._out = out;
|
|
12333
|
+
this._data = data;
|
|
12334
|
+
this._inv = 0;
|
|
12335
|
+
this._realTransform4();
|
|
12336
|
+
this._out = null;
|
|
12337
|
+
this._data = null;
|
|
12338
|
+
};
|
|
12339
|
+
FFT.prototype.inverseTransform = function inverseTransform(out, data) {
|
|
12340
|
+
if (out === data)
|
|
12341
|
+
throw new Error("Input and output buffers must be different");
|
|
12342
|
+
this._out = out;
|
|
12343
|
+
this._data = data;
|
|
12344
|
+
this._inv = 1;
|
|
12345
|
+
this._transform4();
|
|
12346
|
+
for (var i = 0; i < out.length; i++)
|
|
12347
|
+
out[i] /= this.size;
|
|
12348
|
+
this._out = null;
|
|
12349
|
+
this._data = null;
|
|
12350
|
+
};
|
|
12351
|
+
FFT.prototype._transform4 = function _transform4() {
|
|
12352
|
+
var out = this._out;
|
|
12353
|
+
var size = this._csize;
|
|
12354
|
+
var width = this._width;
|
|
12355
|
+
var step2 = 1 << width;
|
|
12356
|
+
var len = size / step2 << 1;
|
|
12357
|
+
var outOff;
|
|
12358
|
+
var t;
|
|
12359
|
+
var bitrev = this._bitrev;
|
|
12360
|
+
if (len === 4) {
|
|
12361
|
+
for (outOff = 0, t = 0; outOff < size; outOff += len, t++) {
|
|
12362
|
+
const off = bitrev[t];
|
|
12363
|
+
this._singleTransform2(outOff, off, step2);
|
|
12364
|
+
}
|
|
12365
|
+
} else {
|
|
12366
|
+
for (outOff = 0, t = 0; outOff < size; outOff += len, t++) {
|
|
12367
|
+
const off = bitrev[t];
|
|
12368
|
+
this._singleTransform4(outOff, off, step2);
|
|
12369
|
+
}
|
|
12370
|
+
}
|
|
12371
|
+
var inv = this._inv ? -1 : 1;
|
|
12372
|
+
var table = this.table;
|
|
12373
|
+
for (step2 >>= 2; step2 >= 2; step2 >>= 2) {
|
|
12374
|
+
len = size / step2 << 1;
|
|
12375
|
+
var quarterLen = len >>> 2;
|
|
12376
|
+
for (outOff = 0; outOff < size; outOff += len) {
|
|
12377
|
+
var limit = outOff + quarterLen;
|
|
12378
|
+
for (var i = outOff, k = 0; i < limit; i += 2, k += step2) {
|
|
12379
|
+
const A = i;
|
|
12380
|
+
const B = A + quarterLen;
|
|
12381
|
+
const C = B + quarterLen;
|
|
12382
|
+
const D = C + quarterLen;
|
|
12383
|
+
const Ar = out[A];
|
|
12384
|
+
const Ai = out[A + 1];
|
|
12009
12385
|
const Br = out[B];
|
|
12010
12386
|
const Bi = out[B + 1];
|
|
12011
12387
|
const Cr = out[C];
|
|
@@ -12174,488 +12550,264 @@ FFT.prototype._realTransform4 = function _realTransform4() {
|
|
|
12174
12550
|
var T3i = inv * (MBi - MDi);
|
|
12175
12551
|
var FAr = T0r + T2r;
|
|
12176
12552
|
var FAi = T0i + T2i;
|
|
12177
|
-
var FBr = T1r + T3i;
|
|
12178
|
-
var FBi = T1i - T3r;
|
|
12179
|
-
out[A] = FAr;
|
|
12180
|
-
out[A + 1] = FAi;
|
|
12181
|
-
out[B] = FBr;
|
|
12182
|
-
out[B + 1] = FBi;
|
|
12183
|
-
if (i === 0) {
|
|
12184
|
-
var FCr = T0r - T2r;
|
|
12185
|
-
var FCi = T0i - T2i;
|
|
12186
|
-
out[C] = FCr;
|
|
12187
|
-
out[C + 1] = FCi;
|
|
12188
|
-
continue;
|
|
12189
|
-
}
|
|
12190
|
-
if (i === hquarterLen)
|
|
12191
|
-
continue;
|
|
12192
|
-
var ST0r = T1r;
|
|
12193
|
-
var ST0i = -T1i;
|
|
12194
|
-
var ST1r = T0r;
|
|
12195
|
-
var ST1i = -T0i;
|
|
12196
|
-
var ST2r = -inv * T3i;
|
|
12197
|
-
var ST2i = -inv * T3r;
|
|
12198
|
-
var ST3r = -inv * T2i;
|
|
12199
|
-
var ST3i = -inv * T2r;
|
|
12200
|
-
var SFAr = ST0r + ST2r;
|
|
12201
|
-
var SFAi = ST0i + ST2i;
|
|
12202
|
-
var SFBr = ST1r + ST3i;
|
|
12203
|
-
var SFBi = ST1i - ST3r;
|
|
12204
|
-
var SA = outOff + quarterLen - i;
|
|
12205
|
-
var SB = outOff + halfLen - i;
|
|
12206
|
-
out[SA] = SFAr;
|
|
12207
|
-
out[SA + 1] = SFAi;
|
|
12208
|
-
out[SB] = SFBr;
|
|
12209
|
-
out[SB + 1] = SFBi;
|
|
12210
|
-
}
|
|
12211
|
-
}
|
|
12212
|
-
}
|
|
12213
|
-
};
|
|
12214
|
-
FFT.prototype._singleRealTransform2 = function _singleRealTransform2(outOff, off, step2) {
|
|
12215
|
-
const out = this._out;
|
|
12216
|
-
const data = this._data;
|
|
12217
|
-
const evenR = data[off];
|
|
12218
|
-
const oddR = data[off + step2];
|
|
12219
|
-
const leftR = evenR + oddR;
|
|
12220
|
-
const rightR = evenR - oddR;
|
|
12221
|
-
out[outOff] = leftR;
|
|
12222
|
-
out[outOff + 1] = 0;
|
|
12223
|
-
out[outOff + 2] = rightR;
|
|
12224
|
-
out[outOff + 3] = 0;
|
|
12225
|
-
};
|
|
12226
|
-
FFT.prototype._singleRealTransform4 = function _singleRealTransform4(outOff, off, step2) {
|
|
12227
|
-
const out = this._out;
|
|
12228
|
-
const data = this._data;
|
|
12229
|
-
const inv = this._inv ? -1 : 1;
|
|
12230
|
-
const step22 = step2 * 2;
|
|
12231
|
-
const step3 = step2 * 3;
|
|
12232
|
-
const Ar = data[off];
|
|
12233
|
-
const Br = data[off + step2];
|
|
12234
|
-
const Cr = data[off + step22];
|
|
12235
|
-
const Dr = data[off + step3];
|
|
12236
|
-
const T0r = Ar + Cr;
|
|
12237
|
-
const T1r = Ar - Cr;
|
|
12238
|
-
const T2r = Br + Dr;
|
|
12239
|
-
const T3r = inv * (Br - Dr);
|
|
12240
|
-
const FAr = T0r + T2r;
|
|
12241
|
-
const FBr = T1r;
|
|
12242
|
-
const FBi = -T3r;
|
|
12243
|
-
const FCr = T0r - T2r;
|
|
12244
|
-
const FDr = T1r;
|
|
12245
|
-
const FDi = T3r;
|
|
12246
|
-
out[outOff] = FAr;
|
|
12247
|
-
out[outOff + 1] = 0;
|
|
12248
|
-
out[outOff + 2] = FBr;
|
|
12249
|
-
out[outOff + 3] = FBi;
|
|
12250
|
-
out[outOff + 4] = FCr;
|
|
12251
|
-
out[outOff + 5] = 0;
|
|
12252
|
-
out[outOff + 6] = FDr;
|
|
12253
|
-
out[outOff + 7] = FDi;
|
|
12254
|
-
};
|
|
12255
|
-
const FFT$1 = /* @__PURE__ */ getDefaultExportFromCjs(fft);
|
|
12256
|
-
const INSTRUMENTS = ["kick", "snare", "hat"];
|
|
12257
|
-
const SAMPLE_RATE = 60;
|
|
12258
|
-
const TAP_TIMEOUT_MS = 5e3;
|
|
12259
|
-
const MAX_CYCLE_LENGTH = 8;
|
|
12260
|
-
const FUZZY_TOLERANCE = 0.18;
|
|
12261
|
-
const MIN_REPETITIONS = 3;
|
|
12262
|
-
const MAX_TAP_INTERVAL_MS = 3e3;
|
|
12263
|
-
const MIN_TAP_INTERVAL_MS = 100;
|
|
12264
|
-
const MAX_TAP_HISTORY = 64;
|
|
12265
|
-
const MIN_EMA_ALPHA = 0.05;
|
|
12266
|
-
const PATTERN_SAME_TOLERANCE = 0.15;
|
|
12267
|
-
function createInstrumentState() {
|
|
12268
|
-
return {
|
|
12269
|
-
mode: "auto",
|
|
12270
|
-
muted: false,
|
|
12271
|
-
mutedAt: 0,
|
|
12272
|
-
tapIOIs: [],
|
|
12273
|
-
lastTapTime: 0,
|
|
12274
|
-
pattern: null,
|
|
12275
|
-
refinementIndex: 0,
|
|
12276
|
-
refinementCounts: [],
|
|
12277
|
-
replayLastEventTime: 0,
|
|
12278
|
-
replayIndex: 0,
|
|
12279
|
-
pendingTapEvents: [],
|
|
12280
|
-
envelope: new EnvelopeFollower(0, 300, SAMPLE_RATE),
|
|
12281
|
-
envelopeSmoothed: new EnvelopeFollower(5, 500, SAMPLE_RATE)
|
|
12282
|
-
};
|
|
12283
|
-
}
|
|
12284
|
-
class OnsetTapManager {
|
|
12285
|
-
state = {
|
|
12286
|
-
kick: createInstrumentState(),
|
|
12287
|
-
snare: createInstrumentState(),
|
|
12288
|
-
hat: createInstrumentState()
|
|
12289
|
-
};
|
|
12290
|
-
tap(instrument) {
|
|
12291
|
-
const s = this.state[instrument];
|
|
12292
|
-
const now = performance.now();
|
|
12293
|
-
if (s.muted) {
|
|
12294
|
-
s.muted = false;
|
|
12295
|
-
s.mutedAt = 0;
|
|
12296
|
-
}
|
|
12297
|
-
let ioi = -1;
|
|
12298
|
-
if (s.lastTapTime > 0) {
|
|
12299
|
-
ioi = now - s.lastTapTime;
|
|
12300
|
-
if (ioi < MIN_TAP_INTERVAL_MS) return;
|
|
12301
|
-
if (ioi > MAX_TAP_INTERVAL_MS) {
|
|
12302
|
-
s.tapIOIs = [];
|
|
12303
|
-
ioi = -1;
|
|
12304
|
-
} else {
|
|
12305
|
-
s.tapIOIs.push(ioi);
|
|
12306
|
-
if (s.tapIOIs.length > MAX_TAP_HISTORY) s.tapIOIs.shift();
|
|
12307
|
-
}
|
|
12308
|
-
}
|
|
12309
|
-
s.lastTapTime = now;
|
|
12310
|
-
s.pendingTapEvents.push(now);
|
|
12311
|
-
if (s.mode === "auto") {
|
|
12312
|
-
s.mode = "tapping";
|
|
12313
|
-
if (ioi > 0) {
|
|
12314
|
-
const pattern = this.tryRecognizePattern(instrument);
|
|
12315
|
-
if (pattern) this.applyPattern(instrument, pattern);
|
|
12316
|
-
}
|
|
12317
|
-
} else if (s.mode === "tapping") {
|
|
12318
|
-
if (ioi > 0) {
|
|
12319
|
-
const pattern = this.tryRecognizePattern(instrument);
|
|
12320
|
-
if (pattern) this.applyPattern(instrument, pattern);
|
|
12321
|
-
}
|
|
12322
|
-
} else if (s.mode === "pattern") {
|
|
12323
|
-
if (ioi > 0) {
|
|
12324
|
-
this.handlePatternTap(instrument, ioi, now);
|
|
12325
|
-
}
|
|
12326
|
-
}
|
|
12327
|
-
}
|
|
12328
|
-
clear(instrument) {
|
|
12329
|
-
const s = this.state[instrument];
|
|
12330
|
-
s.mode = "auto";
|
|
12331
|
-
s.muted = false;
|
|
12332
|
-
s.mutedAt = 0;
|
|
12333
|
-
s.tapIOIs = [];
|
|
12334
|
-
s.lastTapTime = 0;
|
|
12335
|
-
s.pattern = null;
|
|
12336
|
-
s.refinementIndex = 0;
|
|
12337
|
-
s.refinementCounts = [];
|
|
12338
|
-
s.replayLastEventTime = 0;
|
|
12339
|
-
s.replayIndex = 0;
|
|
12340
|
-
s.pendingTapEvents = [];
|
|
12341
|
-
s.envelope.reset();
|
|
12342
|
-
s.envelopeSmoothed.reset();
|
|
12343
|
-
}
|
|
12344
|
-
getMode(instrument) {
|
|
12345
|
-
return this.state[instrument].mode;
|
|
12346
|
-
}
|
|
12347
|
-
getPatternInfo(instrument) {
|
|
12348
|
-
const s = this.state[instrument];
|
|
12349
|
-
if (!s.pattern) return null;
|
|
12350
|
-
return {
|
|
12351
|
-
length: s.pattern.length,
|
|
12352
|
-
intervals: [...s.pattern],
|
|
12353
|
-
confidence: 1
|
|
12354
|
-
};
|
|
12355
|
-
}
|
|
12356
|
-
setMuted(instrument, muted) {
|
|
12357
|
-
const s = this.state[instrument];
|
|
12358
|
-
if (s.muted === muted) return;
|
|
12359
|
-
const now = performance.now();
|
|
12360
|
-
if (muted) {
|
|
12361
|
-
s.muted = true;
|
|
12362
|
-
s.mutedAt = now;
|
|
12363
|
-
} else {
|
|
12364
|
-
const pauseDuration = now - s.mutedAt;
|
|
12365
|
-
s.muted = false;
|
|
12366
|
-
s.mutedAt = 0;
|
|
12367
|
-
if (s.mode === "tapping" && s.lastTapTime > 0) {
|
|
12368
|
-
s.lastTapTime += pauseDuration;
|
|
12369
|
-
}
|
|
12370
|
-
}
|
|
12371
|
-
}
|
|
12372
|
-
isMuted(instrument) {
|
|
12373
|
-
return this.state[instrument].muted;
|
|
12374
|
-
}
|
|
12375
|
-
/**
|
|
12376
|
-
* Post-process a beat state produced by BeatStateManager.
|
|
12377
|
-
* For instruments in auto mode, values pass through unchanged.
|
|
12378
|
-
* For tapping/pattern instruments, auto events are suppressed and
|
|
12379
|
-
* tap/pattern events + envelopes are injected instead.
|
|
12380
|
-
*/
|
|
12381
|
-
processFrame(beatState, now, dtMs) {
|
|
12382
|
-
const result = { ...beatState, events: [...beatState.events] };
|
|
12383
|
-
const newEvents = [];
|
|
12384
|
-
for (const inst of INSTRUMENTS) {
|
|
12385
|
-
const s = this.state[inst];
|
|
12386
|
-
if (s.muted) {
|
|
12387
|
-
result.events = result.events.filter((e) => e.type !== inst);
|
|
12388
|
-
s.pendingTapEvents = [];
|
|
12389
|
-
if (s.mode === "pattern" && s.pattern && s.replayLastEventTime > 0) {
|
|
12390
|
-
for (let i = 0; i < 3; i++) {
|
|
12391
|
-
const interval = s.pattern[s.replayIndex % s.pattern.length];
|
|
12392
|
-
if (now - s.replayLastEventTime >= interval) {
|
|
12393
|
-
s.replayLastEventTime += interval;
|
|
12394
|
-
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
12395
|
-
} else {
|
|
12396
|
-
break;
|
|
12397
|
-
}
|
|
12398
|
-
}
|
|
12399
|
-
}
|
|
12400
|
-
s.envelope.process(0, dtMs);
|
|
12401
|
-
s.envelopeSmoothed.process(0, dtMs);
|
|
12402
|
-
const envKey2 = inst;
|
|
12403
|
-
const smoothKey2 = `${inst}Smoothed`;
|
|
12404
|
-
result[envKey2] = s.envelope.getValue();
|
|
12405
|
-
result[smoothKey2] = s.envelopeSmoothed.getValue();
|
|
12406
|
-
continue;
|
|
12407
|
-
}
|
|
12408
|
-
if (s.mode === "auto") {
|
|
12409
|
-
s.envelope.process(0, dtMs);
|
|
12410
|
-
s.envelopeSmoothed.process(0, dtMs);
|
|
12411
|
-
continue;
|
|
12412
|
-
}
|
|
12413
|
-
result.events = result.events.filter((e) => e.type !== inst);
|
|
12414
|
-
if (s.mode === "tapping") {
|
|
12415
|
-
if (now - s.lastTapTime > TAP_TIMEOUT_MS) {
|
|
12416
|
-
s.tapIOIs = [];
|
|
12417
|
-
s.pendingTapEvents = [];
|
|
12418
|
-
if (s.pattern) {
|
|
12419
|
-
s.mode = "pattern";
|
|
12420
|
-
s.replayLastEventTime = now;
|
|
12421
|
-
s.replayIndex = 0;
|
|
12422
|
-
} else {
|
|
12423
|
-
s.mode = "auto";
|
|
12424
|
-
}
|
|
12553
|
+
var FBr = T1r + T3i;
|
|
12554
|
+
var FBi = T1i - T3r;
|
|
12555
|
+
out[A] = FAr;
|
|
12556
|
+
out[A + 1] = FAi;
|
|
12557
|
+
out[B] = FBr;
|
|
12558
|
+
out[B + 1] = FBi;
|
|
12559
|
+
if (i === 0) {
|
|
12560
|
+
var FCr = T0r - T2r;
|
|
12561
|
+
var FCi = T0i - T2i;
|
|
12562
|
+
out[C] = FCr;
|
|
12563
|
+
out[C + 1] = FCi;
|
|
12425
12564
|
continue;
|
|
12426
12565
|
}
|
|
12427
|
-
|
|
12428
|
-
|
|
12429
|
-
|
|
12430
|
-
|
|
12431
|
-
|
|
12432
|
-
|
|
12433
|
-
|
|
12434
|
-
|
|
12435
|
-
|
|
12436
|
-
|
|
12437
|
-
|
|
12438
|
-
|
|
12439
|
-
|
|
12440
|
-
|
|
12441
|
-
|
|
12442
|
-
|
|
12443
|
-
|
|
12444
|
-
|
|
12445
|
-
|
|
12446
|
-
|
|
12447
|
-
newEvents.push(ev);
|
|
12448
|
-
s.envelope.trigger(1);
|
|
12449
|
-
s.envelopeSmoothed.trigger(1);
|
|
12450
|
-
}
|
|
12451
|
-
}
|
|
12566
|
+
if (i === hquarterLen)
|
|
12567
|
+
continue;
|
|
12568
|
+
var ST0r = T1r;
|
|
12569
|
+
var ST0i = -T1i;
|
|
12570
|
+
var ST1r = T0r;
|
|
12571
|
+
var ST1i = -T0i;
|
|
12572
|
+
var ST2r = -inv * T3i;
|
|
12573
|
+
var ST2i = -inv * T3r;
|
|
12574
|
+
var ST3r = -inv * T2i;
|
|
12575
|
+
var ST3i = -inv * T2r;
|
|
12576
|
+
var SFAr = ST0r + ST2r;
|
|
12577
|
+
var SFAi = ST0i + ST2i;
|
|
12578
|
+
var SFBr = ST1r + ST3i;
|
|
12579
|
+
var SFBi = ST1i - ST3r;
|
|
12580
|
+
var SA = outOff + quarterLen - i;
|
|
12581
|
+
var SB = outOff + halfLen - i;
|
|
12582
|
+
out[SA] = SFAr;
|
|
12583
|
+
out[SA + 1] = SFAi;
|
|
12584
|
+
out[SB] = SFBr;
|
|
12585
|
+
out[SB + 1] = SFBi;
|
|
12452
12586
|
}
|
|
12453
|
-
s.envelope.process(0, dtMs);
|
|
12454
|
-
s.envelopeSmoothed.process(0, dtMs);
|
|
12455
|
-
const envKey = inst;
|
|
12456
|
-
const smoothKey = `${inst}Smoothed`;
|
|
12457
|
-
result[envKey] = s.envelope.getValue();
|
|
12458
|
-
result[smoothKey] = s.envelopeSmoothed.getValue();
|
|
12459
12587
|
}
|
|
12460
|
-
result.events.push(...newEvents);
|
|
12461
|
-
const anyMax = Math.max(
|
|
12462
|
-
result.kick,
|
|
12463
|
-
result.snare,
|
|
12464
|
-
result.hat
|
|
12465
|
-
);
|
|
12466
|
-
const anySmoothedMax = Math.max(
|
|
12467
|
-
result.kickSmoothed,
|
|
12468
|
-
result.snareSmoothed,
|
|
12469
|
-
result.hatSmoothed
|
|
12470
|
-
);
|
|
12471
|
-
if (anyMax > result.any) result.any = anyMax;
|
|
12472
|
-
if (anySmoothedMax > result.anySmoothed) result.anySmoothed = anySmoothedMax;
|
|
12473
|
-
return result;
|
|
12474
12588
|
}
|
|
12475
|
-
|
|
12476
|
-
|
|
12477
|
-
|
|
12478
|
-
|
|
12479
|
-
|
|
12480
|
-
|
|
12481
|
-
|
|
12482
|
-
|
|
12483
|
-
|
|
12484
|
-
|
|
12485
|
-
|
|
12486
|
-
|
|
12487
|
-
|
|
12488
|
-
|
|
12489
|
-
|
|
12490
|
-
|
|
12491
|
-
|
|
12492
|
-
|
|
12493
|
-
|
|
12494
|
-
|
|
12495
|
-
|
|
12496
|
-
|
|
12497
|
-
|
|
12498
|
-
|
|
12499
|
-
|
|
12500
|
-
|
|
12589
|
+
};
|
|
12590
|
+
FFT.prototype._singleRealTransform2 = function _singleRealTransform2(outOff, off, step2) {
|
|
12591
|
+
const out = this._out;
|
|
12592
|
+
const data = this._data;
|
|
12593
|
+
const evenR = data[off];
|
|
12594
|
+
const oddR = data[off + step2];
|
|
12595
|
+
const leftR = evenR + oddR;
|
|
12596
|
+
const rightR = evenR - oddR;
|
|
12597
|
+
out[outOff] = leftR;
|
|
12598
|
+
out[outOff + 1] = 0;
|
|
12599
|
+
out[outOff + 2] = rightR;
|
|
12600
|
+
out[outOff + 3] = 0;
|
|
12601
|
+
};
|
|
12602
|
+
FFT.prototype._singleRealTransform4 = function _singleRealTransform4(outOff, off, step2) {
|
|
12603
|
+
const out = this._out;
|
|
12604
|
+
const data = this._data;
|
|
12605
|
+
const inv = this._inv ? -1 : 1;
|
|
12606
|
+
const step22 = step2 * 2;
|
|
12607
|
+
const step3 = step2 * 3;
|
|
12608
|
+
const Ar = data[off];
|
|
12609
|
+
const Br = data[off + step2];
|
|
12610
|
+
const Cr = data[off + step22];
|
|
12611
|
+
const Dr = data[off + step3];
|
|
12612
|
+
const T0r = Ar + Cr;
|
|
12613
|
+
const T1r = Ar - Cr;
|
|
12614
|
+
const T2r = Br + Dr;
|
|
12615
|
+
const T3r = inv * (Br - Dr);
|
|
12616
|
+
const FAr = T0r + T2r;
|
|
12617
|
+
const FBr = T1r;
|
|
12618
|
+
const FBi = -T3r;
|
|
12619
|
+
const FCr = T0r - T2r;
|
|
12620
|
+
const FDr = T1r;
|
|
12621
|
+
const FDi = T3r;
|
|
12622
|
+
out[outOff] = FAr;
|
|
12623
|
+
out[outOff + 1] = 0;
|
|
12624
|
+
out[outOff + 2] = FBr;
|
|
12625
|
+
out[outOff + 3] = FBi;
|
|
12626
|
+
out[outOff + 4] = FCr;
|
|
12627
|
+
out[outOff + 5] = 0;
|
|
12628
|
+
out[outOff + 6] = FDr;
|
|
12629
|
+
out[outOff + 7] = FDi;
|
|
12630
|
+
};
|
|
12631
|
+
const FFT$1 = /* @__PURE__ */ getDefaultExportFromCjs(fft);
|
|
12632
|
+
class AudioChannel {
|
|
12633
|
+
// Identity
|
|
12634
|
+
streamIndex;
|
|
12635
|
+
// Web Audio nodes (owned per-channel, connected to shared AudioContext)
|
|
12636
|
+
mediaStreamSource = null;
|
|
12637
|
+
analyser = null;
|
|
12638
|
+
workletNode = null;
|
|
12639
|
+
analysisMode = "analyser";
|
|
12640
|
+
workletReady = false;
|
|
12641
|
+
currentStream = null;
|
|
12642
|
+
// FFT resources
|
|
12643
|
+
fftSize;
|
|
12644
|
+
fftEngine = null;
|
|
12645
|
+
fftInput = null;
|
|
12646
|
+
fftOutput = null;
|
|
12647
|
+
hannWindow = null;
|
|
12648
|
+
frequencyData = null;
|
|
12649
|
+
fftMagnitude = null;
|
|
12650
|
+
fftMagnitudeDb = null;
|
|
12651
|
+
fftPhase = null;
|
|
12652
|
+
timeDomainData = null;
|
|
12653
|
+
// Auto-gain normalization
|
|
12654
|
+
volumeAutoGain;
|
|
12655
|
+
bandAutoGain;
|
|
12656
|
+
// Envelope followers for band/volume smoothing
|
|
12657
|
+
envelopeFollowers;
|
|
12658
|
+
// Per-channel noise floor tracking
|
|
12659
|
+
bandNoiseFloor = {
|
|
12660
|
+
low: 1e-4,
|
|
12661
|
+
lowMid: 1e-4,
|
|
12662
|
+
mid: 1e-4,
|
|
12663
|
+
highMid: 1e-4,
|
|
12664
|
+
high: 1e-4
|
|
12665
|
+
};
|
|
12666
|
+
// Analysis state
|
|
12667
|
+
audioState = {
|
|
12668
|
+
isConnected: false,
|
|
12669
|
+
volume: { current: 0, peak: 0, smoothed: 0 },
|
|
12670
|
+
bands: {
|
|
12671
|
+
low: 0,
|
|
12672
|
+
lowMid: 0,
|
|
12673
|
+
mid: 0,
|
|
12674
|
+
highMid: 0,
|
|
12675
|
+
high: 0,
|
|
12676
|
+
lowSmoothed: 0,
|
|
12677
|
+
lowMidSmoothed: 0,
|
|
12678
|
+
midSmoothed: 0,
|
|
12679
|
+
highMidSmoothed: 0,
|
|
12680
|
+
highSmoothed: 0
|
|
12681
|
+
},
|
|
12682
|
+
spectral: { brightness: 0, flatness: 0 }
|
|
12683
|
+
};
|
|
12684
|
+
// Waveform data for transfer to worker
|
|
12685
|
+
lastWaveformFrame = null;
|
|
12686
|
+
// Analysis loop state
|
|
12687
|
+
isAnalysisRunning = false;
|
|
12688
|
+
workletFrameCount = 0;
|
|
12689
|
+
lastFrameTime = 0;
|
|
12690
|
+
lastAnalysisTimestamp = 0;
|
|
12691
|
+
lastDtMs = 0;
|
|
12692
|
+
analysisTicks = 0;
|
|
12693
|
+
currentSampleRate = 44100;
|
|
12694
|
+
constructor(streamIndex, fftSize) {
|
|
12695
|
+
this.streamIndex = streamIndex;
|
|
12696
|
+
this.fftSize = fftSize;
|
|
12697
|
+
this.volumeAutoGain = new AutoGain(3e3, 60);
|
|
12698
|
+
this.bandAutoGain = {
|
|
12699
|
+
low: new AutoGain(3e3, 60),
|
|
12700
|
+
lowMid: new AutoGain(3e3, 60),
|
|
12701
|
+
mid: new AutoGain(3e3, 60),
|
|
12702
|
+
highMid: new AutoGain(3e3, 60),
|
|
12703
|
+
high: new AutoGain(3e3, 60)
|
|
12704
|
+
};
|
|
12705
|
+
const sampleRate = 60;
|
|
12706
|
+
this.envelopeFollowers = {
|
|
12707
|
+
volumeSmoothed: new EnvelopeFollower(50, 200, sampleRate),
|
|
12708
|
+
lowSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12709
|
+
lowMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12710
|
+
midSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12711
|
+
highMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
12712
|
+
highSmoothed: new EnvelopeFollower(20, 150, sampleRate)
|
|
12713
|
+
};
|
|
12714
|
+
this.refreshFFTResources();
|
|
12501
12715
|
}
|
|
12502
|
-
|
|
12503
|
-
|
|
12504
|
-
* Tries the expected refinementIndex first, then scans all positions.
|
|
12505
|
-
* Returns the matched position index, or -1 if no match.
|
|
12506
|
-
*/
|
|
12507
|
-
findMatchingPosition(s, ioi) {
|
|
12508
|
-
if (!s.pattern) return -1;
|
|
12509
|
-
const expectedPos = s.refinementIndex % s.pattern.length;
|
|
12510
|
-
const expectedInterval = s.pattern[expectedPos];
|
|
12511
|
-
if (Math.abs(ioi - expectedInterval) / expectedInterval <= FUZZY_TOLERANCE) {
|
|
12512
|
-
return expectedPos;
|
|
12513
|
-
}
|
|
12514
|
-
let bestPos = -1;
|
|
12515
|
-
let bestDeviation = Infinity;
|
|
12516
|
-
for (let i = 0; i < s.pattern.length; i++) {
|
|
12517
|
-
if (i === expectedPos) continue;
|
|
12518
|
-
const deviation = Math.abs(ioi - s.pattern[i]) / s.pattern[i];
|
|
12519
|
-
if (deviation <= FUZZY_TOLERANCE && deviation < bestDeviation) {
|
|
12520
|
-
bestDeviation = deviation;
|
|
12521
|
-
bestPos = i;
|
|
12522
|
-
}
|
|
12523
|
-
}
|
|
12524
|
-
return bestPos;
|
|
12716
|
+
static create(streamIndex, fftSize = 2048) {
|
|
12717
|
+
return new AudioChannel(streamIndex, fftSize);
|
|
12525
12718
|
}
|
|
12526
|
-
|
|
12527
|
-
|
|
12528
|
-
|
|
12529
|
-
|
|
12530
|
-
|
|
12531
|
-
|
|
12532
|
-
|
|
12533
|
-
|
|
12719
|
+
refreshFFTResources() {
|
|
12720
|
+
const binCount = this.fftSize / 2;
|
|
12721
|
+
this.frequencyData = new Uint8Array(binCount);
|
|
12722
|
+
this.fftMagnitude = new Float32Array(binCount);
|
|
12723
|
+
this.fftMagnitudeDb = new Float32Array(binCount);
|
|
12724
|
+
this.fftPhase = new Float32Array(binCount);
|
|
12725
|
+
this.timeDomainData = new Float32Array(this.fftSize);
|
|
12726
|
+
this.fftEngine = new FFT$1(this.fftSize);
|
|
12727
|
+
this.fftInput = new Float32Array(this.fftSize * 2);
|
|
12728
|
+
this.fftOutput = new Float32Array(this.fftSize * 2);
|
|
12729
|
+
this.hannWindow = new Float32Array(this.fftSize);
|
|
12730
|
+
const twoPi = Math.PI * 2;
|
|
12731
|
+
for (let i = 0; i < this.fftSize; i++) {
|
|
12732
|
+
this.hannWindow[i] = 0.5 * (1 - Math.cos(twoPi * i / (this.fftSize - 1)));
|
|
12534
12733
|
}
|
|
12535
|
-
|
|
12734
|
+
this.analysisTicks = 0;
|
|
12735
|
+
this.workletFrameCount = 0;
|
|
12536
12736
|
}
|
|
12537
|
-
|
|
12538
|
-
|
|
12539
|
-
|
|
12540
|
-
|
|
12541
|
-
|
|
12542
|
-
|
|
12543
|
-
|
|
12544
|
-
|
|
12545
|
-
|
|
12546
|
-
|
|
12547
|
-
|
|
12548
|
-
|
|
12549
|
-
|
|
12550
|
-
|
|
12551
|
-
|
|
12552
|
-
|
|
12553
|
-
|
|
12554
|
-
|
|
12555
|
-
|
|
12556
|
-
|
|
12557
|
-
|
|
12558
|
-
let sum = 0;
|
|
12559
|
-
for (const grp of groups) sum += grp[i];
|
|
12560
|
-
return sum / groups.length;
|
|
12561
|
-
});
|
|
12562
|
-
let allMatch = true;
|
|
12563
|
-
for (const grp of groups) {
|
|
12564
|
-
for (let i = 0; i < L; i++) {
|
|
12565
|
-
const deviation = Math.abs(grp[i] - avgGroup[i]) / avgGroup[i];
|
|
12566
|
-
if (deviation > tolerance) {
|
|
12567
|
-
allMatch = false;
|
|
12568
|
-
break;
|
|
12569
|
-
}
|
|
12570
|
-
}
|
|
12571
|
-
if (!allMatch) break;
|
|
12572
|
-
}
|
|
12573
|
-
if (allMatch) {
|
|
12574
|
-
return avgGroup;
|
|
12575
|
-
}
|
|
12737
|
+
resetValues() {
|
|
12738
|
+
this.audioState.volume.current = 0;
|
|
12739
|
+
this.audioState.volume.peak = 0;
|
|
12740
|
+
this.audioState.volume.smoothed = 0;
|
|
12741
|
+
this.audioState.bands.low = 0;
|
|
12742
|
+
this.audioState.bands.lowMid = 0;
|
|
12743
|
+
this.audioState.bands.mid = 0;
|
|
12744
|
+
this.audioState.bands.highMid = 0;
|
|
12745
|
+
this.audioState.bands.high = 0;
|
|
12746
|
+
this.audioState.bands.lowSmoothed = 0;
|
|
12747
|
+
this.audioState.bands.lowMidSmoothed = 0;
|
|
12748
|
+
this.audioState.bands.midSmoothed = 0;
|
|
12749
|
+
this.audioState.bands.highMidSmoothed = 0;
|
|
12750
|
+
this.audioState.bands.highSmoothed = 0;
|
|
12751
|
+
this.audioState.spectral.brightness = 0;
|
|
12752
|
+
this.audioState.spectral.flatness = 0;
|
|
12753
|
+
}
|
|
12754
|
+
disconnectNodes() {
|
|
12755
|
+
if (this.mediaStreamSource) {
|
|
12756
|
+
this.mediaStreamSource.disconnect();
|
|
12757
|
+
this.mediaStreamSource = null;
|
|
12576
12758
|
}
|
|
12577
|
-
|
|
12578
|
-
|
|
12579
|
-
|
|
12580
|
-
* Apply a recognized pattern to the instrument state.
|
|
12581
|
-
* Sets pattern, mode, and initializes replay anchor.
|
|
12582
|
-
*/
|
|
12583
|
-
applyPattern(instrument, pattern) {
|
|
12584
|
-
const s = this.state[instrument];
|
|
12585
|
-
s.pattern = pattern;
|
|
12586
|
-
s.mode = "pattern";
|
|
12587
|
-
s.refinementIndex = 0;
|
|
12588
|
-
s.refinementCounts = new Array(pattern.length).fill(MIN_REPETITIONS);
|
|
12589
|
-
if (s.replayLastEventTime === 0 || s.replayLastEventTime < s.lastTapTime) {
|
|
12590
|
-
s.replayLastEventTime = s.lastTapTime;
|
|
12591
|
-
s.replayIndex = 0;
|
|
12759
|
+
if (this.analyser) {
|
|
12760
|
+
this.analyser.disconnect();
|
|
12761
|
+
this.analyser = null;
|
|
12592
12762
|
}
|
|
12593
|
-
|
|
12594
|
-
|
|
12595
|
-
|
|
12596
|
-
|
|
12597
|
-
|
|
12598
|
-
strength: 0.85,
|
|
12599
|
-
isPredicted: false,
|
|
12600
|
-
bpm
|
|
12601
|
-
};
|
|
12602
|
-
}
|
|
12603
|
-
checkPatternReplay(s, instrument, now, bpm) {
|
|
12604
|
-
if (!s.pattern || s.pattern.length === 0) return [];
|
|
12605
|
-
const events = [];
|
|
12606
|
-
const maxEventsPerFrame = 3;
|
|
12607
|
-
for (let safety = 0; safety < maxEventsPerFrame; safety++) {
|
|
12608
|
-
const expectedInterval = s.pattern[s.replayIndex % s.pattern.length];
|
|
12609
|
-
const elapsed = now - s.replayLastEventTime;
|
|
12610
|
-
if (elapsed >= expectedInterval) {
|
|
12611
|
-
s.replayLastEventTime += expectedInterval;
|
|
12612
|
-
s.replayIndex = (s.replayIndex + 1) % s.pattern.length;
|
|
12613
|
-
events.push({
|
|
12614
|
-
type: instrument,
|
|
12615
|
-
time: s.replayLastEventTime,
|
|
12616
|
-
strength: 0.8,
|
|
12617
|
-
isPredicted: true,
|
|
12618
|
-
bpm
|
|
12619
|
-
});
|
|
12620
|
-
} else {
|
|
12621
|
-
break;
|
|
12763
|
+
if (this.workletNode) {
|
|
12764
|
+
try {
|
|
12765
|
+
this.workletNode.port.onmessage = null;
|
|
12766
|
+
this.workletNode.disconnect();
|
|
12767
|
+
} catch (_) {
|
|
12622
12768
|
}
|
|
12769
|
+
this.workletNode = null;
|
|
12623
12770
|
}
|
|
12624
|
-
|
|
12771
|
+
this.workletReady = false;
|
|
12772
|
+
this.analysisMode = "analyser";
|
|
12625
12773
|
}
|
|
12626
|
-
|
|
12627
|
-
|
|
12628
|
-
|
|
12629
|
-
|
|
12774
|
+
destroy() {
|
|
12775
|
+
this.disconnectNodes();
|
|
12776
|
+
this.frequencyData = null;
|
|
12777
|
+
this.timeDomainData = null;
|
|
12778
|
+
this.fftMagnitude = null;
|
|
12779
|
+
this.fftMagnitudeDb = null;
|
|
12780
|
+
this.fftPhase = null;
|
|
12781
|
+
this.fftEngine = null;
|
|
12782
|
+
this.fftInput = null;
|
|
12783
|
+
this.fftOutput = null;
|
|
12784
|
+
this.hannWindow = null;
|
|
12785
|
+
this.lastWaveformFrame = null;
|
|
12786
|
+
this.currentStream = null;
|
|
12787
|
+
this.isAnalysisRunning = false;
|
|
12788
|
+
this.audioState.isConnected = false;
|
|
12789
|
+
this.resetValues();
|
|
12630
12790
|
}
|
|
12631
12791
|
}
|
|
12632
12792
|
class AudioSystem {
|
|
12633
|
-
//
|
|
12793
|
+
// Shared AudioContext (one for all channels)
|
|
12634
12794
|
audioContext = null;
|
|
12635
|
-
analyser = null;
|
|
12636
|
-
mediaStreamSource = null;
|
|
12637
|
-
currentStream = null;
|
|
12638
|
-
analysisMode = "analyser";
|
|
12639
|
-
workletNode = null;
|
|
12640
|
-
workletReady = false;
|
|
12641
12795
|
workletRegistered = false;
|
|
12642
|
-
|
|
12643
|
-
|
|
12644
|
-
|
|
12645
|
-
|
|
12646
|
-
|
|
12647
|
-
|
|
12648
|
-
|
|
12649
|
-
|
|
12796
|
+
// Main audio channel (index 0, full analysis with beat detection)
|
|
12797
|
+
mainChannel;
|
|
12798
|
+
// Additional audio channels (lightweight analysis, no beat detection)
|
|
12799
|
+
additionalChannels = /* @__PURE__ */ new Map();
|
|
12800
|
+
// Shared analysis timer (one interval for all analyser-mode channels)
|
|
12801
|
+
analysisInterval = null;
|
|
12802
|
+
analysisIntervalMs = 8;
|
|
12803
|
+
// Staleness timer (shared across all channels)
|
|
12804
|
+
stalenessTimer = null;
|
|
12805
|
+
static STALENESS_THRESHOLD_MS = 500;
|
|
12806
|
+
// Beat-specific state (main channel only)
|
|
12650
12807
|
/** Tracks the last non-zero BPM chosen for output to avoid dropping to 0 */
|
|
12651
12808
|
lastNonZeroBpm = 120;
|
|
12652
12809
|
/** Tracks which source provided the current BPM (pll | tempo | carry | default) */
|
|
12653
12810
|
lastBpmSource = "default";
|
|
12654
|
-
workletFrameCount = 0;
|
|
12655
|
-
lastFrameTime = 0;
|
|
12656
|
-
stalenessTimer = null;
|
|
12657
|
-
static STALENESS_THRESHOLD_MS = 500;
|
|
12658
|
-
analysisTicks = 0;
|
|
12659
12811
|
lastPhaseLogTime = 0;
|
|
12660
12812
|
onsetLogBuffer = [];
|
|
12661
12813
|
// Debug logging control
|
|
@@ -12671,10 +12823,7 @@ class AudioSystem {
|
|
|
12671
12823
|
stateManager;
|
|
12672
12824
|
// Previous frame's kick detection (for PLL sync with 1-frame lag)
|
|
12673
12825
|
lastKickDetected = false;
|
|
12674
|
-
|
|
12675
|
-
volumeAutoGain;
|
|
12676
|
-
bandAutoGain;
|
|
12677
|
-
/** Raw band snapshot before auto-gain (for debug) */
|
|
12826
|
+
/** Raw band snapshot before auto-gain (for debug, main channel only) */
|
|
12678
12827
|
rawBandsPreGain = {
|
|
12679
12828
|
low: 0,
|
|
12680
12829
|
lowMid: 0,
|
|
@@ -12682,16 +12831,13 @@ class AudioSystem {
|
|
|
12682
12831
|
highMid: 0,
|
|
12683
12832
|
high: 0
|
|
12684
12833
|
};
|
|
12685
|
-
/** Last dt used in analysis loop (ms) */
|
|
12686
|
-
lastDtMs = 0;
|
|
12687
|
-
/** Flag to track if we've logged detection method (to avoid spam) */
|
|
12688
12834
|
bandNames = ["low", "lowMid", "mid", "highMid", "high"];
|
|
12689
12835
|
essentiaBandHistories = /* @__PURE__ */ new Map();
|
|
12690
12836
|
essentiaHistoryWindowMs = 5e3;
|
|
12691
12837
|
// Per-instrument onset tap manager
|
|
12692
12838
|
onsetTapManager;
|
|
12693
|
-
//
|
|
12694
|
-
|
|
12839
|
+
// Beat-specific envelope followers (main channel only, managed by BeatStateManager)
|
|
12840
|
+
beatEnvelopeFollowers;
|
|
12695
12841
|
// Feature enable flags
|
|
12696
12842
|
beatDetectionEnabled = true;
|
|
12697
12843
|
onsetDetectionEnabled = true;
|
|
@@ -12747,80 +12893,94 @@ class AudioSystem {
|
|
|
12747
12893
|
return clone;
|
|
12748
12894
|
}
|
|
12749
12895
|
/**
|
|
12750
|
-
* Handle frames pushed from AudioWorklet
|
|
12896
|
+
* Handle frames pushed from AudioWorklet for main channel
|
|
12897
|
+
*/
|
|
12898
|
+
handleMainWorkletFrame(frame, sampleRate, timestampMs) {
|
|
12899
|
+
if (!this.mainChannel.isAnalysisRunning) return;
|
|
12900
|
+
this.mainChannel.workletFrameCount++;
|
|
12901
|
+
this.analyzeMainFrame(frame, sampleRate || this.mainChannel.currentSampleRate, timestampMs || performance.now());
|
|
12902
|
+
}
|
|
12903
|
+
/**
|
|
12904
|
+
* Handle frames pushed from AudioWorklet for an additional channel
|
|
12751
12905
|
*/
|
|
12752
|
-
|
|
12753
|
-
if (!
|
|
12754
|
-
|
|
12755
|
-
this.
|
|
12906
|
+
handleAdditionalWorkletFrame(channel, frame, sampleRate, timestampMs) {
|
|
12907
|
+
if (!channel.isAnalysisRunning) return;
|
|
12908
|
+
channel.workletFrameCount++;
|
|
12909
|
+
this.analyzeAdditionalFrame(channel, frame, sampleRate || channel.currentSampleRate, timestampMs || performance.now());
|
|
12756
12910
|
}
|
|
12757
12911
|
/**
|
|
12758
|
-
*
|
|
12912
|
+
* Common analysis pipeline for any channel (FFT, volume, bands, auto-gain, spectral, smoothing).
|
|
12913
|
+
* Returns raw band energies (pre-auto-gain) for optional beat processing by the caller.
|
|
12759
12914
|
*/
|
|
12760
|
-
|
|
12761
|
-
if (!frame || frame.length === 0) return;
|
|
12762
|
-
|
|
12763
|
-
|
|
12764
|
-
|
|
12915
|
+
analyzeChannelFrame(ch, frame, sampleRate, timestampMs) {
|
|
12916
|
+
if (!frame || frame.length === 0) return null;
|
|
12917
|
+
ch.lastFrameTime = performance.now();
|
|
12918
|
+
ch.analysisTicks++;
|
|
12919
|
+
if (ch.workletFrameCount === 1) {
|
|
12920
|
+
this.debugLog(`[AudioSystem] First frame received for channel ${ch.streamIndex}`, { sampleRate, mode: ch.analysisMode });
|
|
12765
12921
|
}
|
|
12766
|
-
const dtMs =
|
|
12767
|
-
|
|
12768
|
-
|
|
12769
|
-
|
|
12922
|
+
const dtMs = ch.lastAnalysisTimestamp > 0 ? timestampMs - ch.lastAnalysisTimestamp : 1e3 / 60;
|
|
12923
|
+
ch.lastAnalysisTimestamp = timestampMs;
|
|
12924
|
+
ch.lastDtMs = dtMs;
|
|
12925
|
+
ch.currentSampleRate = sampleRate;
|
|
12770
12926
|
const { rms, peak } = this.calculateVolumeMetrics(frame);
|
|
12771
|
-
|
|
12772
|
-
|
|
12773
|
-
|
|
12774
|
-
const fftResult = this.
|
|
12775
|
-
if (!fftResult)
|
|
12776
|
-
|
|
12777
|
-
|
|
12778
|
-
|
|
12779
|
-
return;
|
|
12780
|
-
}
|
|
12781
|
-
const { magnitudes, magnitudesDb, phases, frequencyData, maxMagnitude } = fftResult;
|
|
12782
|
-
this.frequencyData = frequencyData;
|
|
12783
|
-
this.fftMagnitude = magnitudes;
|
|
12784
|
-
this.fftMagnitudeDb = magnitudesDb;
|
|
12785
|
-
this.fftPhase = phases;
|
|
12786
|
-
const bandEnergies = this.calculateFrequencyBandsFromMagnitude(magnitudes, sampleRate, maxMagnitude);
|
|
12787
|
-
const isSilent = maxMagnitude <= 1e-7 || this.audioState.volume.current <= 1e-6;
|
|
12927
|
+
ch.audioState.volume.current = rms;
|
|
12928
|
+
ch.audioState.volume.peak = peak;
|
|
12929
|
+
ch.lastWaveformFrame = frame;
|
|
12930
|
+
const fftResult = this.computeChannelFFT(ch, frame);
|
|
12931
|
+
if (!fftResult) return null;
|
|
12932
|
+
const { magnitudes, maxMagnitude } = fftResult;
|
|
12933
|
+
const bandEnergies = this.calculateChannelBands(ch, magnitudes, sampleRate, maxMagnitude);
|
|
12934
|
+
const isSilent = maxMagnitude <= 1e-7 || ch.audioState.volume.current <= 1e-6;
|
|
12788
12935
|
if (isSilent) {
|
|
12789
|
-
|
|
12790
|
-
...
|
|
12936
|
+
ch.audioState.bands = {
|
|
12937
|
+
...ch.audioState.bands,
|
|
12791
12938
|
low: 0,
|
|
12792
12939
|
lowMid: 0,
|
|
12793
12940
|
mid: 0,
|
|
12794
12941
|
highMid: 0,
|
|
12795
12942
|
high: 0
|
|
12796
12943
|
};
|
|
12797
|
-
|
|
12798
|
-
this.
|
|
12799
|
-
|
|
12800
|
-
this.audioState.beat.kick = decayed.kick;
|
|
12801
|
-
this.audioState.beat.snare = decayed.snare;
|
|
12802
|
-
this.audioState.beat.hat = decayed.hat;
|
|
12803
|
-
this.audioState.beat.any = decayed.any;
|
|
12804
|
-
this.audioState.beat.kickSmoothed = decayed.kickSmoothed;
|
|
12805
|
-
this.audioState.beat.snareSmoothed = decayed.snareSmoothed;
|
|
12806
|
-
this.audioState.beat.hatSmoothed = decayed.hatSmoothed;
|
|
12807
|
-
this.audioState.beat.anySmoothed = decayed.anySmoothed;
|
|
12808
|
-
this.audioState.beat.events = [];
|
|
12809
|
-
this.updateSmoothBands(dtMs);
|
|
12810
|
-
this.sendAnalysisResultsToWorker();
|
|
12811
|
-
return;
|
|
12944
|
+
ch.audioState.spectral = { brightness: 0, flatness: 0 };
|
|
12945
|
+
this.updateChannelSmoothBands(ch, dtMs);
|
|
12946
|
+
return null;
|
|
12812
12947
|
}
|
|
12813
|
-
this.rawBandsPreGain = { ...bandEnergies };
|
|
12814
12948
|
if (this.autoGainEnabled) {
|
|
12815
|
-
this.
|
|
12949
|
+
this.applyChannelAutoGain(ch);
|
|
12950
|
+
}
|
|
12951
|
+
this.calculateChannelSpectral(ch, magnitudes, sampleRate, maxMagnitude);
|
|
12952
|
+
this.updateChannelSmoothBands(ch, dtMs);
|
|
12953
|
+
return bandEnergies;
|
|
12954
|
+
}
|
|
12955
|
+
/**
|
|
12956
|
+
* Full analysis for the main channel (common analysis + beat pipeline).
|
|
12957
|
+
*/
|
|
12958
|
+
analyzeMainFrame(frame, sampleRate, timestampMs) {
|
|
12959
|
+
const ch = this.mainChannel;
|
|
12960
|
+
const rawBands = this.analyzeChannelFrame(ch, frame, sampleRate, timestampMs);
|
|
12961
|
+
const dtMs = ch.lastDtMs;
|
|
12962
|
+
if (!rawBands) {
|
|
12963
|
+
const decayed = this.stateManager.processEnvelopeDecay(dtMs);
|
|
12964
|
+
this.audioStateBeat.kick = decayed.kick;
|
|
12965
|
+
this.audioStateBeat.snare = decayed.snare;
|
|
12966
|
+
this.audioStateBeat.hat = decayed.hat;
|
|
12967
|
+
this.audioStateBeat.any = decayed.any;
|
|
12968
|
+
this.audioStateBeat.kickSmoothed = decayed.kickSmoothed;
|
|
12969
|
+
this.audioStateBeat.snareSmoothed = decayed.snareSmoothed;
|
|
12970
|
+
this.audioStateBeat.hatSmoothed = decayed.hatSmoothed;
|
|
12971
|
+
this.audioStateBeat.anySmoothed = decayed.anySmoothed;
|
|
12972
|
+
this.audioStateBeat.events = [];
|
|
12973
|
+
this.rawBandsPreGain = { low: 0, lowMid: 0, mid: 0, highMid: 0, high: 0 };
|
|
12974
|
+
this.sendChannelResults(ch, true);
|
|
12975
|
+
return;
|
|
12816
12976
|
}
|
|
12817
|
-
this.
|
|
12977
|
+
this.rawBandsPreGain = { ...rawBands };
|
|
12818
12978
|
if (this.onsetDetectionEnabled && this.beatDetectionEnabled) {
|
|
12819
|
-
const beatState = this.runBeatPipeline(
|
|
12979
|
+
const beatState = this.runBeatPipeline(rawBands, dtMs, timestampMs, sampleRate);
|
|
12820
12980
|
if (beatState) {
|
|
12821
12981
|
const { phase: _phase, bar: _bar, debug: _debug, ...beatForState } = beatState;
|
|
12822
12982
|
const now = performance.now();
|
|
12823
|
-
this.
|
|
12983
|
+
this.audioStateBeat = this.onsetTapManager.processFrame(beatForState, now, dtMs);
|
|
12824
12984
|
if (this.debugMode) {
|
|
12825
12985
|
const pllState = this.pll.getState();
|
|
12826
12986
|
const currentState = this.stateManager.getState();
|
|
@@ -12835,9 +12995,7 @@ class AudioSystem {
|
|
|
12835
12995
|
if (bpmVariance !== void 0) trackingData.bpmVariance = bpmVariance;
|
|
12836
12996
|
const compressed = this.stateManager.isCompressed();
|
|
12837
12997
|
if (compressed) trackingData.compressed = compressed;
|
|
12838
|
-
if (beatState.events.some((e) => e.type === "kick"))
|
|
12839
|
-
trackingData.kickPhase = pllState.phase;
|
|
12840
|
-
}
|
|
12998
|
+
if (beatState.events.some((e) => e.type === "kick")) trackingData.kickPhase = pllState.phase;
|
|
12841
12999
|
if (beatState.debug?.pllGain !== void 0) trackingData.pllGain = beatState.debug.pllGain;
|
|
12842
13000
|
if (beatState.debug?.tempoConfidence !== void 0) trackingData.tempoConf = beatState.debug.tempoConfidence;
|
|
12843
13001
|
if (beatState.debug?.trackingConfidence !== void 0) trackingData.trackingConf = beatState.debug.trackingConfidence;
|
|
@@ -12855,11 +13013,11 @@ class AudioSystem {
|
|
|
12855
13013
|
state: currentState
|
|
12856
13014
|
},
|
|
12857
13015
|
{
|
|
12858
|
-
low:
|
|
12859
|
-
lowMid:
|
|
12860
|
-
mid:
|
|
12861
|
-
highMid:
|
|
12862
|
-
high:
|
|
13016
|
+
low: ch.audioState.bands.low,
|
|
13017
|
+
lowMid: ch.audioState.bands.lowMid,
|
|
13018
|
+
mid: ch.audioState.bands.mid,
|
|
13019
|
+
highMid: ch.audioState.bands.highMid,
|
|
13020
|
+
high: ch.audioState.bands.high
|
|
12863
13021
|
},
|
|
12864
13022
|
this.stateManager.getLastRejections(),
|
|
12865
13023
|
trackingData
|
|
@@ -12867,70 +13025,62 @@ class AudioSystem {
|
|
|
12867
13025
|
}
|
|
12868
13026
|
}
|
|
12869
13027
|
}
|
|
12870
|
-
this.
|
|
12871
|
-
|
|
13028
|
+
this.sendChannelResults(ch, true);
|
|
13029
|
+
}
|
|
13030
|
+
/**
|
|
13031
|
+
* Lightweight analysis for additional/device channels (no beat detection).
|
|
13032
|
+
*/
|
|
13033
|
+
analyzeAdditionalFrame(channel, frame, sampleRate, timestampMs) {
|
|
13034
|
+
this.analyzeChannelFrame(channel, frame, sampleRate, timestampMs);
|
|
13035
|
+
this.sendChannelResults(channel, false);
|
|
12872
13036
|
}
|
|
12873
13037
|
/**
|
|
12874
|
-
* Compute FFT
|
|
13038
|
+
* Compute FFT on a specific channel's buffers
|
|
12875
13039
|
*/
|
|
12876
|
-
|
|
12877
|
-
if (!
|
|
12878
|
-
if (this.debugMode && (
|
|
12879
|
-
this.debugLog(
|
|
13040
|
+
computeChannelFFT(ch, frame) {
|
|
13041
|
+
if (!ch.fftEngine || !ch.fftInput || !ch.fftOutput || !ch.hannWindow || !ch.frequencyData || !ch.fftMagnitude || !ch.fftMagnitudeDb || !ch.fftPhase) {
|
|
13042
|
+
if (this.debugMode && (ch.analysisTicks === 1 || ch.analysisTicks % 240 === 0)) {
|
|
13043
|
+
this.debugLog(`[AudioSystem][computeFFT] RETURNING NULL - resources missing at tick ${ch.analysisTicks}, channel ${ch.streamIndex}`);
|
|
12880
13044
|
}
|
|
12881
13045
|
return null;
|
|
12882
13046
|
}
|
|
12883
|
-
const binCount =
|
|
12884
|
-
const input =
|
|
12885
|
-
const output =
|
|
13047
|
+
const binCount = ch.fftSize / 2;
|
|
13048
|
+
const input = ch.fftInput;
|
|
13049
|
+
const output = ch.fftOutput;
|
|
12886
13050
|
input.fill(0);
|
|
12887
|
-
const frameLength = Math.min(frame.length,
|
|
12888
|
-
let frameMax = 0;
|
|
12889
|
-
for (let i = 0; i < Math.min(32, frame.length); i++) {
|
|
12890
|
-
if (Math.abs(frame[i]) > frameMax) frameMax = Math.abs(frame[i]);
|
|
12891
|
-
}
|
|
13051
|
+
const frameLength = Math.min(frame.length, ch.fftSize);
|
|
12892
13052
|
for (let i = 0; i < frameLength; i++) {
|
|
12893
|
-
input[2 * i] = frame[i] *
|
|
13053
|
+
input[2 * i] = frame[i] * ch.hannWindow[i];
|
|
12894
13054
|
input[2 * i + 1] = 0;
|
|
12895
13055
|
}
|
|
12896
|
-
|
|
12897
|
-
for (let i = 0; i < Math.min(64, input.length); i += 2) {
|
|
12898
|
-
if (Math.abs(input[i]) > inputMax) inputMax = Math.abs(input[i]);
|
|
12899
|
-
}
|
|
12900
|
-
this.fftEngine.transform(output, input);
|
|
12901
|
-
let outputMax = 0;
|
|
12902
|
-
for (let i = 0; i < Math.min(64, output.length); i++) {
|
|
12903
|
-
if (Math.abs(output[i]) > outputMax) outputMax = Math.abs(output[i]);
|
|
12904
|
-
}
|
|
13056
|
+
ch.fftEngine.transform(output, input);
|
|
12905
13057
|
let maxMagnitude = 0;
|
|
12906
13058
|
for (let i = 0; i < binCount; i++) {
|
|
12907
13059
|
const re = output[2 * i];
|
|
12908
13060
|
const im = output[2 * i + 1];
|
|
12909
|
-
const mag = Math.sqrt(re * re + im * im) /
|
|
12910
|
-
|
|
12911
|
-
if (mag > maxMagnitude)
|
|
12912
|
-
|
|
12913
|
-
}
|
|
12914
|
-
this.fftPhase[i] = Math.atan2(im, re);
|
|
13061
|
+
const mag = Math.sqrt(re * re + im * im) / ch.fftSize;
|
|
13062
|
+
ch.fftMagnitude[i] = mag;
|
|
13063
|
+
if (mag > maxMagnitude) maxMagnitude = mag;
|
|
13064
|
+
ch.fftPhase[i] = Math.atan2(im, re);
|
|
12915
13065
|
}
|
|
12916
13066
|
if (maxMagnitude <= 0) {
|
|
12917
|
-
|
|
13067
|
+
ch.frequencyData.fill(0);
|
|
12918
13068
|
for (let i = 0; i < binCount; i++) {
|
|
12919
|
-
|
|
13069
|
+
ch.fftMagnitudeDb[i] = 20 * Math.log10(ch.fftMagnitude[i] + 1e-12);
|
|
12920
13070
|
}
|
|
12921
13071
|
} else {
|
|
12922
13072
|
for (let i = 0; i < binCount; i++) {
|
|
12923
|
-
const mag =
|
|
13073
|
+
const mag = ch.fftMagnitude[i];
|
|
12924
13074
|
const norm = mag / maxMagnitude;
|
|
12925
|
-
|
|
12926
|
-
|
|
13075
|
+
ch.frequencyData[i] = Math.min(255, Math.max(0, Math.round(norm * 255)));
|
|
13076
|
+
ch.fftMagnitudeDb[i] = 20 * Math.log10(mag + 1e-12);
|
|
12927
13077
|
}
|
|
12928
13078
|
}
|
|
12929
13079
|
return {
|
|
12930
|
-
magnitudes:
|
|
12931
|
-
magnitudesDb:
|
|
12932
|
-
phases:
|
|
12933
|
-
frequencyData:
|
|
13080
|
+
magnitudes: ch.fftMagnitude,
|
|
13081
|
+
magnitudesDb: ch.fftMagnitudeDb,
|
|
13082
|
+
phases: ch.fftPhase,
|
|
13083
|
+
frequencyData: ch.frequencyData,
|
|
12934
13084
|
maxMagnitude
|
|
12935
13085
|
};
|
|
12936
13086
|
}
|
|
@@ -12950,9 +13100,9 @@ class AudioSystem {
|
|
|
12950
13100
|
return { rms, peak };
|
|
12951
13101
|
}
|
|
12952
13102
|
/**
|
|
12953
|
-
* Calculate perceptual/log-ish bands
|
|
13103
|
+
* Calculate perceptual/log-ish bands on a specific channel
|
|
12954
13104
|
*/
|
|
12955
|
-
|
|
13105
|
+
calculateChannelBands(ch, magnitudes, sampleRate, maxMagnitude) {
|
|
12956
13106
|
const nyquist = sampleRate / 2;
|
|
12957
13107
|
const binCount = magnitudes.length;
|
|
12958
13108
|
const bands = {
|
|
@@ -12976,33 +13126,32 @@ class AudioSystem {
|
|
|
12976
13126
|
const average = count > 0 ? sum / count : 0;
|
|
12977
13127
|
const rawNormalized = maxMagnitude > 0 ? Math.min(1, average / maxMagnitude) : 0;
|
|
12978
13128
|
rawBands[bandName] = rawNormalized;
|
|
12979
|
-
const prevFloor =
|
|
13129
|
+
const prevFloor = ch.bandNoiseFloor[bandName] ?? 1e-4;
|
|
12980
13130
|
const newFloor = prevFloor * 0.995 + average * 5e-3;
|
|
12981
|
-
|
|
13131
|
+
ch.bandNoiseFloor[bandName] = newFloor;
|
|
12982
13132
|
const floorAdjusted = Math.max(0, average - newFloor * 1.5);
|
|
12983
13133
|
const normalized = maxMagnitude > 0 ? Math.min(1, floorAdjusted / maxMagnitude) : 0;
|
|
12984
13134
|
normalizedBands[bandName] = normalized;
|
|
12985
13135
|
}
|
|
12986
|
-
|
|
12987
|
-
...
|
|
13136
|
+
ch.audioState.bands = {
|
|
13137
|
+
...ch.audioState.bands,
|
|
12988
13138
|
low: rawBands.low,
|
|
12989
13139
|
lowMid: rawBands.lowMid,
|
|
12990
13140
|
mid: rawBands.mid,
|
|
12991
13141
|
highMid: rawBands.highMid,
|
|
12992
13142
|
high: rawBands.high,
|
|
12993
|
-
lowSmoothed:
|
|
12994
|
-
lowMidSmoothed:
|
|
12995
|
-
midSmoothed:
|
|
12996
|
-
highMidSmoothed:
|
|
12997
|
-
highSmoothed:
|
|
13143
|
+
lowSmoothed: ch.audioState.bands.lowSmoothed,
|
|
13144
|
+
lowMidSmoothed: ch.audioState.bands.lowMidSmoothed,
|
|
13145
|
+
midSmoothed: ch.audioState.bands.midSmoothed,
|
|
13146
|
+
highMidSmoothed: ch.audioState.bands.highMidSmoothed,
|
|
13147
|
+
highSmoothed: ch.audioState.bands.highSmoothed
|
|
12998
13148
|
};
|
|
12999
|
-
this.rawBandsPreGain = { ...rawBands };
|
|
13000
13149
|
return normalizedBands;
|
|
13001
13150
|
}
|
|
13002
13151
|
/**
|
|
13003
|
-
* Spectral features
|
|
13152
|
+
* Spectral features on a specific channel
|
|
13004
13153
|
*/
|
|
13005
|
-
|
|
13154
|
+
calculateChannelSpectral(ch, magnitudes, sampleRate, maxMagnitude) {
|
|
13006
13155
|
const nyquist = sampleRate / 2;
|
|
13007
13156
|
const binCount = magnitudes.length;
|
|
13008
13157
|
let sumMagnitude = 0;
|
|
@@ -13014,7 +13163,7 @@ class AudioSystem {
|
|
|
13014
13163
|
sumWeightedFreq += freq * magnitude;
|
|
13015
13164
|
}
|
|
13016
13165
|
const centroid = sumMagnitude > 0 ? sumWeightedFreq / sumMagnitude : 0;
|
|
13017
|
-
|
|
13166
|
+
ch.audioState.spectral.brightness = Math.min(1, centroid / nyquist);
|
|
13018
13167
|
let geometricMeanLog = 0;
|
|
13019
13168
|
let arithmeticMeanSum = 0;
|
|
13020
13169
|
let nonZeroCount = 0;
|
|
@@ -13029,9 +13178,9 @@ class AudioSystem {
|
|
|
13029
13178
|
if (nonZeroCount > 0) {
|
|
13030
13179
|
const arithmeticMean = arithmeticMeanSum / nonZeroCount;
|
|
13031
13180
|
const geometricMean = Math.exp(geometricMeanLog / nonZeroCount);
|
|
13032
|
-
|
|
13181
|
+
ch.audioState.spectral.flatness = Math.min(1, geometricMean / arithmeticMean);
|
|
13033
13182
|
} else {
|
|
13034
|
-
|
|
13183
|
+
ch.audioState.spectral.flatness = 0;
|
|
13035
13184
|
}
|
|
13036
13185
|
}
|
|
13037
13186
|
/**
|
|
@@ -13039,12 +13188,12 @@ class AudioSystem {
|
|
|
13039
13188
|
*/
|
|
13040
13189
|
runBeatPipeline(rawBandsForOnsets, dtMs, timestampMs, sampleRate) {
|
|
13041
13190
|
let onsets;
|
|
13042
|
-
const essentiaReady = (this.essentiaOnsetDetection?.isReady() || false) && !!this.fftMagnitudeDb && !!this.fftPhase;
|
|
13043
|
-
const hasFftData = !!(this.fftMagnitudeDb && this.fftPhase);
|
|
13191
|
+
const essentiaReady = (this.essentiaOnsetDetection?.isReady() || false) && !!this.mainChannel.fftMagnitudeDb && !!this.mainChannel.fftPhase;
|
|
13192
|
+
const hasFftData = !!(this.mainChannel.fftMagnitudeDb && this.mainChannel.fftPhase);
|
|
13044
13193
|
if (essentiaReady && hasFftData) {
|
|
13045
13194
|
const essentiaResult = this.essentiaOnsetDetection.detectFromSpectrum(
|
|
13046
|
-
this.fftMagnitudeDb,
|
|
13047
|
-
this.fftPhase,
|
|
13195
|
+
this.mainChannel.fftMagnitudeDb,
|
|
13196
|
+
this.mainChannel.fftPhase,
|
|
13048
13197
|
sampleRate
|
|
13049
13198
|
// DO NOT pass normalized bands - causes false snare detections on kicks
|
|
13050
13199
|
// Essentia should compute its own bands from spectrum for accurate ratios
|
|
@@ -13115,115 +13264,70 @@ class AudioSystem {
|
|
|
13115
13264
|
console.log(` Conf: ${beatState.confidence.toFixed(2)}`);
|
|
13116
13265
|
if (this.onsetLogBuffer.length > 0) {
|
|
13117
13266
|
console.log(`🎵 Recent Onsets (last ${Math.min(10, this.onsetLogBuffer.length)}):`);
|
|
13118
|
-
const recentOnsets = this.onsetLogBuffer.slice(-10);
|
|
13119
|
-
recentOnsets.forEach((onset, idx) => {
|
|
13120
|
-
const relTime = ((onset.time - timestampMs) / 1e3).toFixed(2);
|
|
13121
|
-
const phaseStr = `φ=${onset.phase.toFixed(2)}`;
|
|
13122
|
-
const barStr = `bar=${onset.bar + 1}`;
|
|
13123
|
-
console.log(` ${idx + 1}. t=${relTime}s | ${onset.type.padEnd(5)} | ${phaseStr} | ${barStr} | [${onset.bands.join(",")}] | str=${onset.strength.toFixed(2)}`);
|
|
13124
|
-
});
|
|
13125
|
-
} else {
|
|
13126
|
-
console.log(`🎵 Recent Onsets: none detected`);
|
|
13127
|
-
}
|
|
13128
|
-
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
13129
|
-
`);
|
|
13130
|
-
this.lastPhaseLogTime = timestampMs;
|
|
13131
|
-
}
|
|
13132
|
-
const candidates = [
|
|
13133
|
-
{ value: beatState.bpm, source: "pll" },
|
|
13134
|
-
{ value: tempo.bpm, source: "tempo" },
|
|
13135
|
-
{ value: this.lastNonZeroBpm, source: "carry" },
|
|
13136
|
-
{ value: 120, source: "default" }
|
|
13137
|
-
];
|
|
13138
|
-
const selected = candidates.find((c) => Number.isFinite(c.value) && c.value > 0) ?? { value: 120, source: "default" };
|
|
13139
|
-
const resolvedBpm = selected.value;
|
|
13140
|
-
const prevSource = this.lastBpmSource;
|
|
13141
|
-
this.lastNonZeroBpm = resolvedBpm;
|
|
13142
|
-
this.lastBpmSource = selected.source;
|
|
13143
|
-
if (this.debugMode && prevSource !== this.lastBpmSource) {
|
|
13144
|
-
console.log(`[AudioSystem][bpm] Source changed: ${prevSource} -> ${this.lastBpmSource} (resolved=${resolvedBpm.toFixed(2)}, pll=${beatState.bpm?.toFixed?.(2) ?? beatState.bpm}, tempo=${tempo.bpm?.toFixed?.(2) ?? tempo.bpm})`);
|
|
13145
|
-
}
|
|
13146
|
-
const mergedBeatState = beatState.bpm === resolvedBpm ? beatState : { ...beatState, bpm: resolvedBpm };
|
|
13147
|
-
this.lastKickDetected = mergedBeatState.events.some((e) => e.type === "kick") || pllKickCue;
|
|
13148
|
-
return mergedBeatState;
|
|
13149
|
-
}
|
|
13150
|
-
/**
|
|
13151
|
-
* Debug logging helper
|
|
13152
|
-
*/
|
|
13153
|
-
debugLog(message, ...args) {
|
|
13154
|
-
if (this.debugMode) {
|
|
13155
|
-
console.log(message, ...args);
|
|
13156
|
-
}
|
|
13157
|
-
}
|
|
13158
|
-
// Analysis configuration (optimized for onset detection)
|
|
13159
|
-
fftSize = 2048;
|
|
13160
|
-
// Good balance for quality vs performance
|
|
13161
|
-
// High-speed analysis for onset detection (separate from RAF)
|
|
13162
|
-
analysisInterval = null;
|
|
13163
|
-
analysisIntervalMs = 8;
|
|
13164
|
-
// 125Hz sampling for 5-8ms transient capture
|
|
13165
|
-
// Analysis data arrays
|
|
13166
|
-
frequencyData = null;
|
|
13167
|
-
timeDomainData = null;
|
|
13168
|
-
// FFT data (computed locally or fed to Essentia)
|
|
13169
|
-
fftEngine = null;
|
|
13170
|
-
fftInput = null;
|
|
13171
|
-
fftOutput = null;
|
|
13172
|
-
fftMagnitude = null;
|
|
13173
|
-
fftMagnitudeDb = null;
|
|
13174
|
-
fftPhase = null;
|
|
13175
|
-
hannWindow = null;
|
|
13176
|
-
// Audio analysis state (host-side state)
|
|
13177
|
-
audioState = {
|
|
13178
|
-
isConnected: false,
|
|
13179
|
-
volume: {
|
|
13180
|
-
current: 0,
|
|
13181
|
-
peak: 0,
|
|
13182
|
-
smoothed: 0
|
|
13183
|
-
},
|
|
13184
|
-
bands: {
|
|
13185
|
-
low: 0,
|
|
13186
|
-
lowMid: 0,
|
|
13187
|
-
mid: 0,
|
|
13188
|
-
highMid: 0,
|
|
13189
|
-
high: 0,
|
|
13190
|
-
lowSmoothed: 0,
|
|
13191
|
-
lowMidSmoothed: 0,
|
|
13192
|
-
midSmoothed: 0,
|
|
13193
|
-
highMidSmoothed: 0,
|
|
13194
|
-
highSmoothed: 0
|
|
13195
|
-
},
|
|
13196
|
-
beat: {
|
|
13197
|
-
kick: 0,
|
|
13198
|
-
snare: 0,
|
|
13199
|
-
hat: 0,
|
|
13200
|
-
any: 0,
|
|
13201
|
-
kickSmoothed: 0,
|
|
13202
|
-
snareSmoothed: 0,
|
|
13203
|
-
hatSmoothed: 0,
|
|
13204
|
-
anySmoothed: 0,
|
|
13205
|
-
events: [],
|
|
13206
|
-
bpm: 120,
|
|
13207
|
-
confidence: 0,
|
|
13208
|
-
isLocked: false
|
|
13209
|
-
},
|
|
13210
|
-
spectral: {
|
|
13211
|
-
brightness: 0,
|
|
13212
|
-
flatness: 0
|
|
13267
|
+
const recentOnsets = this.onsetLogBuffer.slice(-10);
|
|
13268
|
+
recentOnsets.forEach((onset, idx) => {
|
|
13269
|
+
const relTime = ((onset.time - timestampMs) / 1e3).toFixed(2);
|
|
13270
|
+
const phaseStr = `φ=${onset.phase.toFixed(2)}`;
|
|
13271
|
+
const barStr = `bar=${onset.bar + 1}`;
|
|
13272
|
+
console.log(` ${idx + 1}. t=${relTime}s | ${onset.type.padEnd(5)} | ${phaseStr} | ${barStr} | [${onset.bands.join(",")}] | str=${onset.strength.toFixed(2)}`);
|
|
13273
|
+
});
|
|
13274
|
+
} else {
|
|
13275
|
+
console.log(`🎵 Recent Onsets: none detected`);
|
|
13276
|
+
}
|
|
13277
|
+
console.log(`━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
|
13278
|
+
`);
|
|
13279
|
+
this.lastPhaseLogTime = timestampMs;
|
|
13280
|
+
}
|
|
13281
|
+
const candidates = [
|
|
13282
|
+
{ value: beatState.bpm, source: "pll" },
|
|
13283
|
+
{ value: tempo.bpm, source: "tempo" },
|
|
13284
|
+
{ value: this.lastNonZeroBpm, source: "carry" },
|
|
13285
|
+
{ value: 120, source: "default" }
|
|
13286
|
+
];
|
|
13287
|
+
const selected = candidates.find((c) => Number.isFinite(c.value) && c.value > 0) ?? { value: 120, source: "default" };
|
|
13288
|
+
const resolvedBpm = selected.value;
|
|
13289
|
+
const prevSource = this.lastBpmSource;
|
|
13290
|
+
this.lastNonZeroBpm = resolvedBpm;
|
|
13291
|
+
this.lastBpmSource = selected.source;
|
|
13292
|
+
if (this.debugMode && prevSource !== this.lastBpmSource) {
|
|
13293
|
+
console.log(`[AudioSystem][bpm] Source changed: ${prevSource} -> ${this.lastBpmSource} (resolved=${resolvedBpm.toFixed(2)}, pll=${beatState.bpm?.toFixed?.(2) ?? beatState.bpm}, tempo=${tempo.bpm?.toFixed?.(2) ?? tempo.bpm})`);
|
|
13294
|
+
}
|
|
13295
|
+
const mergedBeatState = beatState.bpm === resolvedBpm ? beatState : { ...beatState, bpm: resolvedBpm };
|
|
13296
|
+
this.lastKickDetected = mergedBeatState.events.some((e) => e.type === "kick") || pllKickCue;
|
|
13297
|
+
return mergedBeatState;
|
|
13298
|
+
}
|
|
13299
|
+
/**
|
|
13300
|
+
* Debug logging helper
|
|
13301
|
+
*/
|
|
13302
|
+
debugLog(message, ...args) {
|
|
13303
|
+
if (this.debugMode) {
|
|
13304
|
+
console.log(message, ...args);
|
|
13213
13305
|
}
|
|
13306
|
+
}
|
|
13307
|
+
// Analysis configuration
|
|
13308
|
+
fftSize = 2048;
|
|
13309
|
+
// Beat state for main channel (not on AudioChannel because it's main-only)
|
|
13310
|
+
audioStateBeat = {
|
|
13311
|
+
kick: 0,
|
|
13312
|
+
snare: 0,
|
|
13313
|
+
hat: 0,
|
|
13314
|
+
any: 0,
|
|
13315
|
+
kickSmoothed: 0,
|
|
13316
|
+
snareSmoothed: 0,
|
|
13317
|
+
hatSmoothed: 0,
|
|
13318
|
+
anySmoothed: 0,
|
|
13319
|
+
events: [],
|
|
13320
|
+
bpm: 120,
|
|
13321
|
+
confidence: 0,
|
|
13322
|
+
isLocked: false
|
|
13214
13323
|
};
|
|
13215
|
-
// Waveform data for transfer to worker
|
|
13216
|
-
lastWaveformFrame = null;
|
|
13217
|
-
// Analysis loop
|
|
13218
|
-
analysisLoopId = null;
|
|
13219
|
-
isAnalysisRunning = false;
|
|
13220
|
-
lastAnalysisTimestamp = 0;
|
|
13221
13324
|
// Callback to send results to worker
|
|
13222
13325
|
sendAnalysisResults = null;
|
|
13223
13326
|
constructor(sendAnalysisResultsCallback) {
|
|
13224
13327
|
this.handleAudioStreamUpdate = this.handleAudioStreamUpdate.bind(this);
|
|
13225
13328
|
this.performAnalysis = this.performAnalysis.bind(this);
|
|
13226
13329
|
this.sendAnalysisResults = sendAnalysisResultsCallback || null;
|
|
13330
|
+
this.mainChannel = AudioChannel.create(0, this.fftSize);
|
|
13227
13331
|
this.onsetDetection = new MultiOnsetDetection();
|
|
13228
13332
|
this.essentiaOnsetDetection = new EssentiaOnsetDetection();
|
|
13229
13333
|
this.initializeEssentia();
|
|
@@ -13231,17 +13335,8 @@ class AudioSystem {
|
|
|
13231
13335
|
this.pll = new PhaseLockedLoop();
|
|
13232
13336
|
this.stateManager = new BeatStateManager();
|
|
13233
13337
|
this.onsetTapManager = new OnsetTapManager();
|
|
13234
|
-
this.volumeAutoGain = new AutoGain(3e3, 60);
|
|
13235
|
-
this.bandAutoGain = {
|
|
13236
|
-
low: new AutoGain(3e3, 60),
|
|
13237
|
-
lowMid: new AutoGain(3e3, 60),
|
|
13238
|
-
mid: new AutoGain(3e3, 60),
|
|
13239
|
-
highMid: new AutoGain(3e3, 60),
|
|
13240
|
-
high: new AutoGain(3e3, 60)
|
|
13241
|
-
};
|
|
13242
13338
|
const sampleRate = 60;
|
|
13243
|
-
this.
|
|
13244
|
-
// Beat energy curves - kept for API compatibility, but values come from stateManager
|
|
13339
|
+
this.beatEnvelopeFollowers = {
|
|
13245
13340
|
kick: new EnvelopeFollower(0, 300, sampleRate),
|
|
13246
13341
|
snare: new EnvelopeFollower(0, 300, sampleRate),
|
|
13247
13342
|
hat: new EnvelopeFollower(0, 300, sampleRate),
|
|
@@ -13249,45 +13344,15 @@ class AudioSystem {
|
|
|
13249
13344
|
kickSmoothed: new EnvelopeFollower(5, 500, sampleRate),
|
|
13250
13345
|
snareSmoothed: new EnvelopeFollower(5, 500, sampleRate),
|
|
13251
13346
|
hatSmoothed: new EnvelopeFollower(5, 500, sampleRate),
|
|
13252
|
-
anySmoothed: new EnvelopeFollower(5, 500, sampleRate)
|
|
13253
|
-
// Volume smoothing
|
|
13254
|
-
volumeSmoothed: new EnvelopeFollower(50, 200, sampleRate),
|
|
13255
|
-
// Band smoothing
|
|
13256
|
-
lowSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13257
|
-
lowMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13258
|
-
midSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13259
|
-
highMidSmoothed: new EnvelopeFollower(20, 150, sampleRate),
|
|
13260
|
-
highSmoothed: new EnvelopeFollower(20, 150, sampleRate)
|
|
13347
|
+
anySmoothed: new EnvelopeFollower(5, 500, sampleRate)
|
|
13261
13348
|
};
|
|
13262
|
-
this.refreshFFTResources();
|
|
13263
13349
|
this.resetEssentiaBandHistories();
|
|
13264
13350
|
}
|
|
13265
|
-
/**
|
|
13266
|
-
* Prepare FFT buffers and windowing for the selected fftSize
|
|
13267
|
-
*/
|
|
13268
|
-
refreshFFTResources() {
|
|
13269
|
-
const binCount = this.fftSize / 2;
|
|
13270
|
-
this.frequencyData = new Uint8Array(binCount);
|
|
13271
|
-
this.fftMagnitude = new Float32Array(binCount);
|
|
13272
|
-
this.fftMagnitudeDb = new Float32Array(binCount);
|
|
13273
|
-
this.fftPhase = new Float32Array(binCount);
|
|
13274
|
-
this.timeDomainData = new Float32Array(this.fftSize);
|
|
13275
|
-
this.fftEngine = new FFT$1(this.fftSize);
|
|
13276
|
-
this.fftInput = new Float32Array(this.fftSize * 2);
|
|
13277
|
-
this.fftOutput = new Float32Array(this.fftSize * 2);
|
|
13278
|
-
this.hannWindow = new Float32Array(this.fftSize);
|
|
13279
|
-
const twoPi = Math.PI * 2;
|
|
13280
|
-
for (let i = 0; i < this.fftSize; i++) {
|
|
13281
|
-
this.hannWindow[i] = 0.5 * (1 - Math.cos(twoPi * i / (this.fftSize - 1)));
|
|
13282
|
-
}
|
|
13283
|
-
this.analysisTicks = 0;
|
|
13284
|
-
this.workletFrameCount = 0;
|
|
13285
|
-
}
|
|
13286
13351
|
/**
|
|
13287
13352
|
* Get the current audio analysis state (for host-side usage)
|
|
13288
13353
|
*/
|
|
13289
13354
|
getAudioState() {
|
|
13290
|
-
return { ...this.audioState };
|
|
13355
|
+
return { ...this.mainChannel.audioState };
|
|
13291
13356
|
}
|
|
13292
13357
|
/**
|
|
13293
13358
|
* Initialize Essentia.js (async WASM loading)
|
|
@@ -13317,161 +13382,140 @@ class AudioSystem {
|
|
|
13317
13382
|
return false;
|
|
13318
13383
|
}
|
|
13319
13384
|
/**
|
|
13320
|
-
* Handle audio stream update (called from VijiCore)
|
|
13385
|
+
* Handle audio stream update for main stream (called from VijiCore)
|
|
13321
13386
|
*/
|
|
13322
13387
|
handleAudioStreamUpdate(data) {
|
|
13323
13388
|
try {
|
|
13324
13389
|
if (data.audioStream) {
|
|
13325
13390
|
this.setAudioStream(data.audioStream);
|
|
13326
13391
|
} else {
|
|
13327
|
-
this.
|
|
13392
|
+
this.disconnectMainStream();
|
|
13328
13393
|
}
|
|
13329
13394
|
} catch (error) {
|
|
13330
13395
|
console.error("Error handling audio stream update:", error);
|
|
13331
|
-
this.audioState.isConnected = false;
|
|
13332
|
-
this.
|
|
13396
|
+
this.mainChannel.audioState.isConnected = false;
|
|
13397
|
+
this.sendChannelResults(this.mainChannel, true);
|
|
13333
13398
|
}
|
|
13334
13399
|
}
|
|
13335
13400
|
/**
|
|
13336
|
-
*
|
|
13401
|
+
* Ensure the shared AudioContext is created and resumed.
|
|
13337
13402
|
*/
|
|
13338
|
-
async
|
|
13339
|
-
this.
|
|
13340
|
-
|
|
13341
|
-
|
|
13342
|
-
|
|
13343
|
-
|
|
13403
|
+
async ensureAudioContext() {
|
|
13404
|
+
if (!this.audioContext) {
|
|
13405
|
+
this.audioContext = new AudioContext();
|
|
13406
|
+
}
|
|
13407
|
+
if (this.audioContext.state === "suspended") {
|
|
13408
|
+
await this.audioContext.resume();
|
|
13409
|
+
}
|
|
13410
|
+
return this.audioContext;
|
|
13411
|
+
}
|
|
13412
|
+
/**
|
|
13413
|
+
* Connect a channel to Web Audio nodes (source, worklet/analyser).
|
|
13414
|
+
* Used for both main and additional channels.
|
|
13415
|
+
*/
|
|
13416
|
+
async connectChannel(ch, audioStream, isMain) {
|
|
13417
|
+
ch.disconnectNodes();
|
|
13418
|
+
ch.refreshFFTResources();
|
|
13419
|
+
ch.workletFrameCount = 0;
|
|
13344
13420
|
const audioTracks = audioStream.getAudioTracks();
|
|
13345
13421
|
if (audioTracks.length === 0) {
|
|
13346
|
-
console.warn(
|
|
13347
|
-
|
|
13348
|
-
this.
|
|
13422
|
+
console.warn(`No audio tracks in stream for channel ${ch.streamIndex}`);
|
|
13423
|
+
ch.audioState.isConnected = false;
|
|
13424
|
+
this.sendChannelResults(ch, isMain);
|
|
13349
13425
|
return;
|
|
13350
13426
|
}
|
|
13351
13427
|
try {
|
|
13352
|
-
|
|
13353
|
-
|
|
13354
|
-
|
|
13355
|
-
|
|
13356
|
-
|
|
13357
|
-
|
|
13358
|
-
|
|
13359
|
-
|
|
13360
|
-
|
|
13361
|
-
this.analysisMode = "worklet";
|
|
13362
|
-
this.currentSampleRate = this.audioContext.sampleRate;
|
|
13363
|
-
this.mediaStreamSource.connect(this.workletNode);
|
|
13364
|
-
this.workletNode.connect(this.audioContext.destination);
|
|
13365
|
-
this.debugLog("Audio worklet analysis enabled");
|
|
13428
|
+
const ctx = await this.ensureAudioContext();
|
|
13429
|
+
ch.mediaStreamSource = ctx.createMediaStreamSource(audioStream);
|
|
13430
|
+
const workletOk = await this.setupChannelWorklet(ch, isMain);
|
|
13431
|
+
if (workletOk && ch.workletNode) {
|
|
13432
|
+
ch.analysisMode = "worklet";
|
|
13433
|
+
ch.currentSampleRate = ctx.sampleRate;
|
|
13434
|
+
ch.mediaStreamSource.connect(ch.workletNode);
|
|
13435
|
+
ch.workletNode.connect(ctx.destination);
|
|
13436
|
+
this.debugLog(`Audio worklet enabled for channel ${ch.streamIndex}`);
|
|
13366
13437
|
setTimeout(() => {
|
|
13367
|
-
if (
|
|
13368
|
-
this.debugLog(
|
|
13369
|
-
|
|
13370
|
-
if (
|
|
13438
|
+
if (ch.analysisMode === "worklet" && ch.workletFrameCount === 0) {
|
|
13439
|
+
this.debugLog(`[AudioSystem] Worklet silent for channel ${ch.streamIndex}, falling back to analyser.`);
|
|
13440
|
+
ch.analysisMode = "analyser";
|
|
13441
|
+
if (ch.workletNode) {
|
|
13371
13442
|
try {
|
|
13372
|
-
|
|
13373
|
-
|
|
13443
|
+
ch.workletNode.port.onmessage = null;
|
|
13444
|
+
ch.workletNode.disconnect();
|
|
13374
13445
|
} catch (_) {
|
|
13375
13446
|
}
|
|
13376
|
-
|
|
13447
|
+
ch.workletNode = null;
|
|
13377
13448
|
}
|
|
13378
|
-
|
|
13379
|
-
|
|
13380
|
-
|
|
13381
|
-
|
|
13382
|
-
|
|
13383
|
-
|
|
13384
|
-
|
|
13385
|
-
this.
|
|
13386
|
-
this.startAnalysisLoop();
|
|
13449
|
+
ch.workletReady = false;
|
|
13450
|
+
ch.analyser = ctx.createAnalyser();
|
|
13451
|
+
ch.analyser.fftSize = ch.fftSize;
|
|
13452
|
+
ch.analyser.smoothingTimeConstant = 0;
|
|
13453
|
+
ch.mediaStreamSource.connect(ch.analyser);
|
|
13454
|
+
ch.frequencyData = new Uint8Array(ch.analyser.frequencyBinCount);
|
|
13455
|
+
ch.timeDomainData = new Float32Array(ch.fftSize);
|
|
13456
|
+
this.ensureAnalysisLoop();
|
|
13387
13457
|
}
|
|
13388
13458
|
}, 600);
|
|
13389
13459
|
} else {
|
|
13390
|
-
|
|
13391
|
-
|
|
13392
|
-
|
|
13393
|
-
|
|
13394
|
-
|
|
13395
|
-
|
|
13396
|
-
|
|
13397
|
-
this.
|
|
13398
|
-
|
|
13399
|
-
|
|
13400
|
-
|
|
13401
|
-
|
|
13402
|
-
|
|
13403
|
-
|
|
13404
|
-
|
|
13405
|
-
|
|
13406
|
-
this.startAnalysisLoop();
|
|
13407
|
-
}
|
|
13408
|
-
if (this.analysisMode === "analyser" && !this.isAnalysisRunning) {
|
|
13409
|
-
this.debugLog("[AudioSystem] Analysis loop not running after setup, forcing start.");
|
|
13410
|
-
this.startAnalysisLoop();
|
|
13411
|
-
}
|
|
13412
|
-
const tracks = audioStream.getAudioTracks();
|
|
13413
|
-
this.debugLog("[AudioSystem] Stream info", {
|
|
13414
|
-
trackCount: tracks.length,
|
|
13415
|
-
trackSettings: tracks.map((t) => t.getSettings?.() || {}),
|
|
13416
|
-
trackMuted: tracks.map((t) => t.muted),
|
|
13417
|
-
mode: this.analysisMode
|
|
13418
|
-
});
|
|
13460
|
+
ch.analysisMode = "analyser";
|
|
13461
|
+
ch.analyser = ctx.createAnalyser();
|
|
13462
|
+
ch.analyser.fftSize = ch.fftSize;
|
|
13463
|
+
ch.analyser.smoothingTimeConstant = 0;
|
|
13464
|
+
ch.mediaStreamSource.connect(ch.analyser);
|
|
13465
|
+
ch.frequencyData = new Uint8Array(ch.analyser.frequencyBinCount);
|
|
13466
|
+
ch.timeDomainData = new Float32Array(ch.fftSize);
|
|
13467
|
+
this.debugLog(`Analyser fallback for channel ${ch.streamIndex}`);
|
|
13468
|
+
}
|
|
13469
|
+
ch.currentStream = audioStream;
|
|
13470
|
+
ch.audioState.isConnected = true;
|
|
13471
|
+
if (ch.analysisMode === "worklet") {
|
|
13472
|
+
ch.isAnalysisRunning = true;
|
|
13473
|
+
} else {
|
|
13474
|
+
this.ensureAnalysisLoop();
|
|
13475
|
+
}
|
|
13419
13476
|
this.startStalenessTimer();
|
|
13420
|
-
this.debugLog(
|
|
13421
|
-
sampleRate:
|
|
13422
|
-
fftSize:
|
|
13423
|
-
|
|
13477
|
+
this.debugLog(`Audio stream connected for channel ${ch.streamIndex}`, {
|
|
13478
|
+
sampleRate: ctx.sampleRate,
|
|
13479
|
+
fftSize: ch.fftSize,
|
|
13480
|
+
mode: ch.analysisMode
|
|
13424
13481
|
});
|
|
13425
13482
|
} catch (error) {
|
|
13426
|
-
console.error(
|
|
13427
|
-
|
|
13428
|
-
|
|
13483
|
+
console.error(`Failed to set up audio for channel ${ch.streamIndex}:`, error);
|
|
13484
|
+
ch.audioState.isConnected = false;
|
|
13485
|
+
ch.disconnectNodes();
|
|
13429
13486
|
}
|
|
13430
|
-
this.
|
|
13487
|
+
this.sendChannelResults(ch, isMain);
|
|
13431
13488
|
}
|
|
13432
13489
|
/**
|
|
13433
|
-
*
|
|
13490
|
+
* Set the main audio stream for analysis (preserves original public API surface).
|
|
13434
13491
|
*/
|
|
13435
|
-
|
|
13436
|
-
this.
|
|
13437
|
-
this.stopStalenessTimer();
|
|
13492
|
+
async setAudioStream(audioStream) {
|
|
13493
|
+
this.disconnectMainStream();
|
|
13438
13494
|
this.resetEssentiaBandHistories();
|
|
13439
|
-
|
|
13440
|
-
this.mediaStreamSource.disconnect();
|
|
13441
|
-
this.mediaStreamSource = null;
|
|
13442
|
-
}
|
|
13443
|
-
if (this.analyser) {
|
|
13444
|
-
this.analyser.disconnect();
|
|
13445
|
-
this.analyser = null;
|
|
13446
|
-
}
|
|
13447
|
-
if (this.workletNode) {
|
|
13448
|
-
try {
|
|
13449
|
-
this.workletNode.port.onmessage = null;
|
|
13450
|
-
this.workletNode.disconnect();
|
|
13451
|
-
} catch (_) {
|
|
13452
|
-
}
|
|
13453
|
-
this.workletNode = null;
|
|
13454
|
-
}
|
|
13455
|
-
this.workletReady = false;
|
|
13456
|
-
this.analysisMode = "analyser";
|
|
13457
|
-
this.frequencyData = null;
|
|
13458
|
-
this.timeDomainData = null;
|
|
13459
|
-
this.fftMagnitude = null;
|
|
13460
|
-
this.fftMagnitudeDb = null;
|
|
13461
|
-
this.fftPhase = null;
|
|
13462
|
-
this.currentStream = null;
|
|
13463
|
-
this.audioState.isConnected = false;
|
|
13464
|
-
this.resetAudioValues();
|
|
13465
|
-
this.sendAnalysisResultsToWorker();
|
|
13466
|
-
this.debugLog("Audio stream disconnected (host-side)");
|
|
13495
|
+
await this.connectChannel(this.mainChannel, audioStream, true);
|
|
13467
13496
|
}
|
|
13468
13497
|
/**
|
|
13469
|
-
*
|
|
13498
|
+
* Disconnect the main audio stream (does NOT close AudioContext -- additional channels may still be active).
|
|
13470
13499
|
*/
|
|
13471
|
-
|
|
13472
|
-
|
|
13473
|
-
|
|
13500
|
+
disconnectMainStream() {
|
|
13501
|
+
this.mainChannel.disconnectNodes();
|
|
13502
|
+
this.mainChannel.audioState.isConnected = false;
|
|
13503
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13504
|
+
this.mainChannel.currentStream = null;
|
|
13505
|
+
this.resetAudioValues();
|
|
13506
|
+
if (this.additionalChannels.size === 0) {
|
|
13507
|
+
this.stopAnalysisLoop();
|
|
13508
|
+
this.stopStalenessTimer();
|
|
13474
13509
|
}
|
|
13510
|
+
this.sendChannelResults(this.mainChannel, true);
|
|
13511
|
+
this.debugLog("Main audio stream disconnected (host-side)");
|
|
13512
|
+
}
|
|
13513
|
+
/**
|
|
13514
|
+
* Set up an AudioWorklet node for a specific channel.
|
|
13515
|
+
* Registers the processor module once (shared), creates a new WorkletNode per channel.
|
|
13516
|
+
*/
|
|
13517
|
+
async setupChannelWorklet(ch, isMain) {
|
|
13518
|
+
if (!this.audioContext?.audioWorklet) return false;
|
|
13475
13519
|
try {
|
|
13476
13520
|
if (!this.workletRegistered) {
|
|
13477
13521
|
const workletSource = `
|
|
@@ -13521,65 +13565,71 @@ class AudioSystem {
|
|
|
13521
13565
|
URL.revokeObjectURL(workletUrl);
|
|
13522
13566
|
this.workletRegistered = true;
|
|
13523
13567
|
}
|
|
13524
|
-
|
|
13568
|
+
ch.workletNode = new AudioWorkletNode(this.audioContext, "audio-analysis-processor", {
|
|
13525
13569
|
numberOfOutputs: 1,
|
|
13526
13570
|
outputChannelCount: [1],
|
|
13527
|
-
processorOptions: {
|
|
13528
|
-
fftSize: this.fftSize,
|
|
13529
|
-
hopSize: this.fftSize / 2
|
|
13530
|
-
}
|
|
13571
|
+
processorOptions: { fftSize: ch.fftSize, hopSize: ch.fftSize / 2 }
|
|
13531
13572
|
});
|
|
13532
|
-
|
|
13573
|
+
ch.workletNode.port.onmessage = (event) => {
|
|
13533
13574
|
const data = event.data || {};
|
|
13534
13575
|
if (data.type === "audio-frame" && data.samples) {
|
|
13535
|
-
|
|
13576
|
+
if (isMain) {
|
|
13577
|
+
this.handleMainWorkletFrame(data.samples, data.sampleRate, data.timestamp);
|
|
13578
|
+
} else {
|
|
13579
|
+
this.handleAdditionalWorkletFrame(ch, data.samples, data.sampleRate, data.timestamp);
|
|
13580
|
+
}
|
|
13536
13581
|
}
|
|
13537
13582
|
};
|
|
13538
13583
|
return true;
|
|
13539
13584
|
} catch (error) {
|
|
13540
|
-
console.warn(
|
|
13541
|
-
|
|
13585
|
+
console.warn(`Audio worklet init failed for channel ${ch.streamIndex}:`, error);
|
|
13586
|
+
ch.workletNode = null;
|
|
13542
13587
|
return false;
|
|
13543
13588
|
}
|
|
13544
13589
|
}
|
|
13545
13590
|
/**
|
|
13546
|
-
*
|
|
13591
|
+
* Ensure the shared analysis interval is running (for analyser-mode channels).
|
|
13547
13592
|
*/
|
|
13548
|
-
|
|
13549
|
-
if (this.
|
|
13550
|
-
return;
|
|
13551
|
-
}
|
|
13552
|
-
this.isAnalysisRunning = true;
|
|
13593
|
+
ensureAnalysisLoop() {
|
|
13594
|
+
if (this.analysisInterval !== null) return;
|
|
13553
13595
|
this.analysisInterval = window.setInterval(() => {
|
|
13554
13596
|
this.performAnalysis();
|
|
13555
13597
|
}, this.analysisIntervalMs);
|
|
13556
|
-
this.debugLog("[AudioSystem]
|
|
13557
|
-
intervalMs: this.analysisIntervalMs
|
|
13558
|
-
});
|
|
13598
|
+
this.debugLog("[AudioSystem] Shared analysis loop started", { intervalMs: this.analysisIntervalMs });
|
|
13559
13599
|
}
|
|
13560
13600
|
/**
|
|
13561
|
-
* Stop the
|
|
13601
|
+
* Stop the shared analysis loop.
|
|
13562
13602
|
*/
|
|
13563
13603
|
stopAnalysisLoop() {
|
|
13564
|
-
this.isAnalysisRunning = false;
|
|
13565
13604
|
if (this.analysisInterval !== null) {
|
|
13566
13605
|
clearInterval(this.analysisInterval);
|
|
13567
13606
|
this.analysisInterval = null;
|
|
13568
13607
|
}
|
|
13569
|
-
|
|
13570
|
-
|
|
13571
|
-
|
|
13608
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13609
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13610
|
+
ch.isAnalysisRunning = false;
|
|
13572
13611
|
}
|
|
13573
13612
|
}
|
|
13613
|
+
/**
|
|
13614
|
+
* Shared staleness timer: checks all channels for stale data.
|
|
13615
|
+
*/
|
|
13574
13616
|
startStalenessTimer() {
|
|
13575
|
-
this.
|
|
13576
|
-
this.lastFrameTime = performance.now();
|
|
13617
|
+
if (this.stalenessTimer !== null) return;
|
|
13577
13618
|
this.stalenessTimer = setInterval(() => {
|
|
13578
|
-
|
|
13579
|
-
|
|
13580
|
-
|
|
13581
|
-
|
|
13582
|
-
|
|
13619
|
+
const now = performance.now();
|
|
13620
|
+
if (this.mainChannel.audioState.isConnected && this.mainChannel.lastFrameTime > 0) {
|
|
13621
|
+
if (now - this.mainChannel.lastFrameTime > AudioSystem.STALENESS_THRESHOLD_MS) {
|
|
13622
|
+
this.resetAudioValues();
|
|
13623
|
+
this.sendChannelResults(this.mainChannel, true);
|
|
13624
|
+
}
|
|
13625
|
+
}
|
|
13626
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13627
|
+
if (ch.audioState.isConnected && ch.lastFrameTime > 0) {
|
|
13628
|
+
if (now - ch.lastFrameTime > AudioSystem.STALENESS_THRESHOLD_MS) {
|
|
13629
|
+
ch.resetValues();
|
|
13630
|
+
this.sendChannelResults(ch, false);
|
|
13631
|
+
}
|
|
13632
|
+
}
|
|
13583
13633
|
}
|
|
13584
13634
|
}, 250);
|
|
13585
13635
|
}
|
|
@@ -13591,114 +13641,96 @@ class AudioSystem {
|
|
|
13591
13641
|
}
|
|
13592
13642
|
/**
|
|
13593
13643
|
* Pause audio analysis (for tests or temporary suspension)
|
|
13594
|
-
* The setInterval continues but performAnalysis() exits early
|
|
13595
13644
|
*/
|
|
13596
13645
|
pauseAnalysis() {
|
|
13597
|
-
this.isAnalysisRunning = false;
|
|
13646
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13647
|
+
for (const ch of this.additionalChannels.values()) ch.isAnalysisRunning = false;
|
|
13598
13648
|
}
|
|
13599
13649
|
/**
|
|
13600
13650
|
* Resume audio analysis after pause
|
|
13601
13651
|
*/
|
|
13602
13652
|
resumeAnalysis() {
|
|
13603
|
-
this.isAnalysisRunning = true;
|
|
13653
|
+
this.mainChannel.isAnalysisRunning = true;
|
|
13654
|
+
for (const ch of this.additionalChannels.values()) ch.isAnalysisRunning = true;
|
|
13604
13655
|
}
|
|
13605
13656
|
/**
|
|
13606
|
-
*
|
|
13607
|
-
* Uses industry-grade 4-layer architecture:
|
|
13608
|
-
* Layer 1: MultiOnsetDetection (per-band onset detection)
|
|
13609
|
-
* Layer 2: TempoInduction (dual-method BPM detection)
|
|
13610
|
-
* Layer 3: PhaseLockedLoop (stable phase tracking)
|
|
13611
|
-
* Layer 4: BeatStateManager (state machine + confidence)
|
|
13657
|
+
* Shared analysis loop callback: iterates main + additional channels in analyser mode.
|
|
13612
13658
|
*/
|
|
13613
13659
|
performAnalysis() {
|
|
13614
|
-
if (!this.
|
|
13615
|
-
|
|
13616
|
-
}
|
|
13617
|
-
if (!this.timeDomainData || this.timeDomainData.length !== this.fftSize) {
|
|
13618
|
-
this.timeDomainData = new Float32Array(this.fftSize);
|
|
13619
|
-
}
|
|
13620
|
-
this.analyser.getFloatTimeDomainData(this.timeDomainData);
|
|
13621
|
-
this.analysisTicks++;
|
|
13622
|
-
for (let i = 0; i < 32 && i < this.timeDomainData.length; i++) {
|
|
13623
|
-
Math.abs(this.timeDomainData[i]);
|
|
13624
|
-
}
|
|
13660
|
+
if (!this.audioContext) return;
|
|
13661
|
+
const sampleRate = this.audioContext.sampleRate;
|
|
13625
13662
|
const detectionTimeMs = performance.now();
|
|
13626
|
-
|
|
13627
|
-
|
|
13628
|
-
|
|
13629
|
-
|
|
13630
|
-
|
|
13631
|
-
|
|
13632
|
-
|
|
13633
|
-
|
|
13634
|
-
|
|
13635
|
-
|
|
13636
|
-
|
|
13663
|
+
const mc = this.mainChannel;
|
|
13664
|
+
if (mc.isAnalysisRunning && mc.analysisMode === "analyser" && mc.analyser) {
|
|
13665
|
+
if (!mc.timeDomainData || mc.timeDomainData.length !== mc.fftSize) {
|
|
13666
|
+
mc.timeDomainData = new Float32Array(mc.fftSize);
|
|
13667
|
+
}
|
|
13668
|
+
mc.analyser.getFloatTimeDomainData(mc.timeDomainData);
|
|
13669
|
+
this.analyzeMainFrame(mc.timeDomainData, sampleRate, detectionTimeMs);
|
|
13670
|
+
}
|
|
13671
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13672
|
+
if (ch.isAnalysisRunning && ch.analysisMode === "analyser" && ch.analyser) {
|
|
13673
|
+
if (!ch.timeDomainData || ch.timeDomainData.length !== ch.fftSize) {
|
|
13674
|
+
ch.timeDomainData = new Float32Array(ch.fftSize);
|
|
13675
|
+
}
|
|
13676
|
+
ch.analyser.getFloatTimeDomainData(ch.timeDomainData);
|
|
13677
|
+
this.analyzeAdditionalFrame(ch, ch.timeDomainData, sampleRate, detectionTimeMs);
|
|
13678
|
+
}
|
|
13679
|
+
}
|
|
13680
|
+
}
|
|
13681
|
+
applyChannelAutoGain(ch) {
|
|
13682
|
+
ch.audioState.volume.current = ch.volumeAutoGain.process(ch.audioState.volume.current);
|
|
13683
|
+
ch.audioState.volume.peak = ch.volumeAutoGain.process(ch.audioState.volume.peak);
|
|
13684
|
+
const bands = ch.audioState.bands;
|
|
13685
|
+
bands.low = ch.bandAutoGain.low.process(bands.low);
|
|
13686
|
+
bands.lowMid = ch.bandAutoGain.lowMid.process(bands.lowMid);
|
|
13687
|
+
bands.mid = ch.bandAutoGain.mid.process(bands.mid);
|
|
13688
|
+
bands.highMid = ch.bandAutoGain.highMid.process(bands.highMid);
|
|
13689
|
+
bands.high = ch.bandAutoGain.high.process(bands.high);
|
|
13637
13690
|
}
|
|
13638
13691
|
// Note: updateBeatState is now handled by BeatStateManager
|
|
13639
13692
|
/**
|
|
13640
|
-
* Update smooth versions of frequency bands
|
|
13693
|
+
* Update smooth versions of frequency bands on a specific channel
|
|
13641
13694
|
*/
|
|
13642
|
-
|
|
13643
|
-
|
|
13644
|
-
|
|
13645
|
-
|
|
13646
|
-
);
|
|
13647
|
-
|
|
13648
|
-
|
|
13649
|
-
|
|
13650
|
-
);
|
|
13651
|
-
this.audioState.bands.lowMidSmoothed = this.envelopeFollowers.lowMidSmoothed.process(
|
|
13652
|
-
this.audioState.bands.lowMid,
|
|
13653
|
-
dtMs
|
|
13654
|
-
);
|
|
13655
|
-
this.audioState.bands.midSmoothed = this.envelopeFollowers.midSmoothed.process(
|
|
13656
|
-
this.audioState.bands.mid,
|
|
13657
|
-
dtMs
|
|
13658
|
-
);
|
|
13659
|
-
this.audioState.bands.highMidSmoothed = this.envelopeFollowers.highMidSmoothed.process(
|
|
13660
|
-
this.audioState.bands.highMid,
|
|
13661
|
-
dtMs
|
|
13662
|
-
);
|
|
13663
|
-
this.audioState.bands.highSmoothed = this.envelopeFollowers.highSmoothed.process(
|
|
13664
|
-
this.audioState.bands.high,
|
|
13665
|
-
dtMs
|
|
13666
|
-
);
|
|
13695
|
+
updateChannelSmoothBands(ch, dtMs) {
|
|
13696
|
+
const ef = ch.envelopeFollowers;
|
|
13697
|
+
ch.audioState.volume.smoothed = ef.volumeSmoothed.process(ch.audioState.volume.current, dtMs);
|
|
13698
|
+
ch.audioState.bands.lowSmoothed = ef.lowSmoothed.process(ch.audioState.bands.low, dtMs);
|
|
13699
|
+
ch.audioState.bands.lowMidSmoothed = ef.lowMidSmoothed.process(ch.audioState.bands.lowMid, dtMs);
|
|
13700
|
+
ch.audioState.bands.midSmoothed = ef.midSmoothed.process(ch.audioState.bands.mid, dtMs);
|
|
13701
|
+
ch.audioState.bands.highMidSmoothed = ef.highMidSmoothed.process(ch.audioState.bands.highMid, dtMs);
|
|
13702
|
+
ch.audioState.bands.highSmoothed = ef.highSmoothed.process(ch.audioState.bands.high, dtMs);
|
|
13667
13703
|
}
|
|
13668
13704
|
/**
|
|
13669
|
-
* Send analysis results to worker
|
|
13705
|
+
* Send analysis results for a specific channel to the worker.
|
|
13706
|
+
* Tags with streamIndex and conditionally includes beat data (main channel only).
|
|
13670
13707
|
*/
|
|
13671
|
-
|
|
13672
|
-
if (this.sendAnalysisResults)
|
|
13673
|
-
|
|
13674
|
-
|
|
13675
|
-
|
|
13676
|
-
|
|
13677
|
-
|
|
13678
|
-
|
|
13679
|
-
|
|
13680
|
-
|
|
13681
|
-
|
|
13682
|
-
|
|
13683
|
-
|
|
13684
|
-
|
|
13685
|
-
|
|
13686
|
-
|
|
13687
|
-
});
|
|
13708
|
+
sendChannelResults(ch, includeBeat) {
|
|
13709
|
+
if (!this.sendAnalysisResults) return;
|
|
13710
|
+
const frequencyData = ch.frequencyData ? new Uint8Array(ch.frequencyData) : new Uint8Array(0);
|
|
13711
|
+
const waveformData = ch.lastWaveformFrame ? new Float32Array(ch.lastWaveformFrame) : new Float32Array(0);
|
|
13712
|
+
const data = {
|
|
13713
|
+
streamIndex: ch.streamIndex,
|
|
13714
|
+
isConnected: ch.audioState.isConnected,
|
|
13715
|
+
volume: ch.audioState.volume,
|
|
13716
|
+
bands: ch.audioState.bands,
|
|
13717
|
+
spectral: ch.audioState.spectral,
|
|
13718
|
+
frequencyData,
|
|
13719
|
+
waveformData,
|
|
13720
|
+
timestamp: performance.now()
|
|
13721
|
+
};
|
|
13722
|
+
if (includeBeat) {
|
|
13723
|
+
data.beat = this.audioStateBeat;
|
|
13688
13724
|
}
|
|
13725
|
+
this.sendAnalysisResults({ type: "audio-analysis-update", data });
|
|
13689
13726
|
}
|
|
13690
13727
|
/**
|
|
13691
|
-
* Reset audio values to defaults
|
|
13728
|
+
* Reset audio values to defaults (main channel + beat state)
|
|
13692
13729
|
*/
|
|
13693
13730
|
resetAudioValues() {
|
|
13694
|
-
this.
|
|
13695
|
-
this.audioState.volume.peak = 0;
|
|
13696
|
-
this.audioState.volume.smoothed = 0;
|
|
13731
|
+
this.mainChannel.resetValues();
|
|
13697
13732
|
this.lastKickDetected = false;
|
|
13698
|
-
|
|
13699
|
-
this.audioState.bands[band] = 0;
|
|
13700
|
-
}
|
|
13701
|
-
this.audioState.beat = {
|
|
13733
|
+
this.audioStateBeat = {
|
|
13702
13734
|
kick: 0,
|
|
13703
13735
|
snare: 0,
|
|
13704
13736
|
hat: 0,
|
|
@@ -13712,36 +13744,98 @@ class AudioSystem {
|
|
|
13712
13744
|
confidence: 0,
|
|
13713
13745
|
isLocked: false
|
|
13714
13746
|
};
|
|
13715
|
-
this.audioState.spectral = {
|
|
13716
|
-
brightness: 0,
|
|
13717
|
-
flatness: 0
|
|
13718
|
-
};
|
|
13719
13747
|
}
|
|
13720
13748
|
/**
|
|
13721
|
-
* Reset all audio state (called when destroying)
|
|
13749
|
+
* Reset all audio state (called when destroying).
|
|
13750
|
+
* Disconnects all channels, closes AudioContext, resets all modules.
|
|
13722
13751
|
*/
|
|
13723
13752
|
resetAudioState() {
|
|
13724
|
-
this.
|
|
13753
|
+
this.stopAnalysisLoop();
|
|
13754
|
+
this.stopStalenessTimer();
|
|
13725
13755
|
this.resetEssentiaBandHistories();
|
|
13756
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13757
|
+
ch.destroy();
|
|
13758
|
+
}
|
|
13759
|
+
this.additionalChannels.clear();
|
|
13760
|
+
this.mainChannel.disconnectNodes();
|
|
13761
|
+
this.mainChannel.audioState.isConnected = false;
|
|
13762
|
+
this.mainChannel.isAnalysisRunning = false;
|
|
13763
|
+
this.mainChannel.currentStream = null;
|
|
13726
13764
|
if (this.audioContext && this.audioContext.state !== "closed") {
|
|
13727
13765
|
this.audioContext.close();
|
|
13728
13766
|
this.audioContext = null;
|
|
13729
13767
|
}
|
|
13730
|
-
this.workletNode = null;
|
|
13731
|
-
this.workletReady = false;
|
|
13732
13768
|
this.workletRegistered = false;
|
|
13733
|
-
this.analysisMode = "analyser";
|
|
13734
13769
|
this.onsetDetection.reset();
|
|
13735
13770
|
this.tempoInduction.reset();
|
|
13736
13771
|
this.pll.reset();
|
|
13737
13772
|
this.stateManager.reset();
|
|
13738
|
-
this.volumeAutoGain.reset();
|
|
13739
|
-
Object.values(this.bandAutoGain).forEach((g) => g.reset());
|
|
13740
13773
|
this.onsetTapManager.reset();
|
|
13741
|
-
Object.values(this.
|
|
13774
|
+
Object.values(this.beatEnvelopeFollowers).forEach((env) => env.reset());
|
|
13742
13775
|
this.resetAudioValues();
|
|
13743
13776
|
}
|
|
13744
13777
|
// ═══════════════════════════════════════════════════════════
|
|
13778
|
+
// Multi-Channel Stream Management
|
|
13779
|
+
// ═══════════════════════════════════════════════════════════
|
|
13780
|
+
/**
|
|
13781
|
+
* Add an additional audio stream (lightweight analysis, no beat detection).
|
|
13782
|
+
* @param streamIndex Global stream index (from VijiCore: AUDIO_ADDITIONAL_BASE + n or AUDIO_DEVICE_BASE + n)
|
|
13783
|
+
* @param stream The MediaStream to analyze
|
|
13784
|
+
*/
|
|
13785
|
+
async addStream(streamIndex, stream) {
|
|
13786
|
+
if (this.additionalChannels.has(streamIndex)) {
|
|
13787
|
+
this.removeStream(streamIndex);
|
|
13788
|
+
}
|
|
13789
|
+
const ch = AudioChannel.create(streamIndex, this.fftSize);
|
|
13790
|
+
this.additionalChannels.set(streamIndex, ch);
|
|
13791
|
+
await this.connectChannel(ch, stream, false);
|
|
13792
|
+
}
|
|
13793
|
+
/**
|
|
13794
|
+
* Remove an additional audio stream.
|
|
13795
|
+
*/
|
|
13796
|
+
removeStream(streamIndex) {
|
|
13797
|
+
const ch = this.additionalChannels.get(streamIndex);
|
|
13798
|
+
if (!ch) return;
|
|
13799
|
+
ch.destroy();
|
|
13800
|
+
this.additionalChannels.delete(streamIndex);
|
|
13801
|
+
const disconnected = AudioChannel.create(streamIndex, this.fftSize);
|
|
13802
|
+
disconnected.audioState.isConnected = false;
|
|
13803
|
+
this.sendChannelResults(disconnected, false);
|
|
13804
|
+
disconnected.destroy();
|
|
13805
|
+
if (this.additionalChannels.size === 0 && !this.mainChannel.audioState.isConnected) {
|
|
13806
|
+
this.stopAnalysisLoop();
|
|
13807
|
+
this.stopStalenessTimer();
|
|
13808
|
+
}
|
|
13809
|
+
}
|
|
13810
|
+
/**
|
|
13811
|
+
* Tear down all additional channels and rebuild with corrected indices.
|
|
13812
|
+
* Mirrors video's reinitializeAdditionalCoordinators pattern.
|
|
13813
|
+
*/
|
|
13814
|
+
async reinitializeAdditionalChannels(streams, baseIndex) {
|
|
13815
|
+
const toRemove = [];
|
|
13816
|
+
for (const [idx, ch] of this.additionalChannels) {
|
|
13817
|
+
if (idx >= baseIndex && idx < baseIndex + 100) {
|
|
13818
|
+
ch.destroy();
|
|
13819
|
+
toRemove.push(idx);
|
|
13820
|
+
}
|
|
13821
|
+
}
|
|
13822
|
+
for (const idx of toRemove) {
|
|
13823
|
+
this.additionalChannels.delete(idx);
|
|
13824
|
+
}
|
|
13825
|
+
for (let i = 0; i < streams.length; i++) {
|
|
13826
|
+
const newIndex = baseIndex + i;
|
|
13827
|
+
const ch = AudioChannel.create(newIndex, this.fftSize);
|
|
13828
|
+
this.additionalChannels.set(newIndex, ch);
|
|
13829
|
+
await this.connectChannel(ch, streams[i], false);
|
|
13830
|
+
}
|
|
13831
|
+
}
|
|
13832
|
+
/**
|
|
13833
|
+
* Get the number of additional audio channels.
|
|
13834
|
+
*/
|
|
13835
|
+
getChannelCount() {
|
|
13836
|
+
return this.additionalChannels.size;
|
|
13837
|
+
}
|
|
13838
|
+
// ═══════════════════════════════════════════════════════════
|
|
13745
13839
|
// Public API Methods for Audio Analysis Configuration
|
|
13746
13840
|
// ═══════════════════════════════════════════════════════════
|
|
13747
13841
|
/**
|
|
@@ -13804,10 +13898,10 @@ class AudioSystem {
|
|
|
13804
13898
|
setFFTSize(size) {
|
|
13805
13899
|
if (this.fftSize === size) return;
|
|
13806
13900
|
this.fftSize = size;
|
|
13807
|
-
this.
|
|
13808
|
-
|
|
13809
|
-
|
|
13810
|
-
this.setAudioStream(
|
|
13901
|
+
this.mainChannel.fftSize = size;
|
|
13902
|
+
this.mainChannel.refreshFFTResources();
|
|
13903
|
+
if (this.mainChannel.currentStream) {
|
|
13904
|
+
this.setAudioStream(this.mainChannel.currentStream);
|
|
13811
13905
|
}
|
|
13812
13906
|
}
|
|
13813
13907
|
/**
|
|
@@ -13816,8 +13910,12 @@ class AudioSystem {
|
|
|
13816
13910
|
setAutoGain(enabled) {
|
|
13817
13911
|
this.autoGainEnabled = enabled;
|
|
13818
13912
|
if (!enabled) {
|
|
13819
|
-
this.volumeAutoGain.reset();
|
|
13820
|
-
Object.values(this.bandAutoGain).forEach((g) => g.reset());
|
|
13913
|
+
this.mainChannel.volumeAutoGain.reset();
|
|
13914
|
+
Object.values(this.mainChannel.bandAutoGain).forEach((g) => g.reset());
|
|
13915
|
+
for (const ch of this.additionalChannels.values()) {
|
|
13916
|
+
ch.volumeAutoGain.reset();
|
|
13917
|
+
Object.values(ch.bandAutoGain).forEach((g) => g.reset());
|
|
13918
|
+
}
|
|
13821
13919
|
}
|
|
13822
13920
|
}
|
|
13823
13921
|
/**
|
|
@@ -13845,7 +13943,7 @@ class AudioSystem {
|
|
|
13845
13943
|
*/
|
|
13846
13944
|
getState() {
|
|
13847
13945
|
return {
|
|
13848
|
-
isConnected: this.audioState.isConnected,
|
|
13946
|
+
isConnected: this.mainChannel.audioState.isConnected,
|
|
13849
13947
|
currentBPM: this.pll.getBPM(),
|
|
13850
13948
|
confidence: this.tempoInduction.getConfidence(),
|
|
13851
13949
|
isLocked: this.stateManager.isLocked(),
|
|
@@ -13863,11 +13961,11 @@ class AudioSystem {
|
|
|
13863
13961
|
*/
|
|
13864
13962
|
getCurrentAudioData() {
|
|
13865
13963
|
return {
|
|
13866
|
-
isConnected: this.audioState.isConnected,
|
|
13867
|
-
volume: { ...this.audioState.volume },
|
|
13868
|
-
bands: { ...this.audioState.bands },
|
|
13869
|
-
beat: { ...this.
|
|
13870
|
-
spectral: { ...this.audioState.spectral }
|
|
13964
|
+
isConnected: this.mainChannel.audioState.isConnected,
|
|
13965
|
+
volume: { ...this.mainChannel.audioState.volume },
|
|
13966
|
+
bands: { ...this.mainChannel.audioState.bands },
|
|
13967
|
+
beat: { ...this.audioStateBeat },
|
|
13968
|
+
spectral: { ...this.mainChannel.audioState.spectral }
|
|
13871
13969
|
};
|
|
13872
13970
|
}
|
|
13873
13971
|
/**
|
|
@@ -13913,29 +14011,25 @@ class AudioSystem {
|
|
|
13913
14011
|
},
|
|
13914
14012
|
// Raw audio levels (AFTER normalization for visual output)
|
|
13915
14013
|
levels: {
|
|
13916
|
-
low: this.audioState.bands.low,
|
|
13917
|
-
lowMid: this.audioState.bands.lowMid,
|
|
13918
|
-
mid: this.audioState.bands.mid,
|
|
13919
|
-
highMid: this.audioState.bands.highMid,
|
|
13920
|
-
high: this.audioState.bands.high,
|
|
13921
|
-
volume: this.audioState.volume.current
|
|
14014
|
+
low: this.mainChannel.audioState.bands.low,
|
|
14015
|
+
lowMid: this.mainChannel.audioState.bands.lowMid,
|
|
14016
|
+
mid: this.mainChannel.audioState.bands.mid,
|
|
14017
|
+
highMid: this.mainChannel.audioState.bands.highMid,
|
|
14018
|
+
high: this.mainChannel.audioState.bands.high,
|
|
14019
|
+
volume: this.mainChannel.audioState.volume.current
|
|
13922
14020
|
},
|
|
13923
|
-
// Pre-gain band levels (BEFORE auto-gain normalization)
|
|
13924
14021
|
rawBands: { ...this.rawBandsPreGain },
|
|
13925
|
-
// Auto-gain values
|
|
13926
14022
|
bandGains: {
|
|
13927
|
-
low: this.bandAutoGain.low.getGain(),
|
|
13928
|
-
lowMid: this.bandAutoGain.lowMid.getGain(),
|
|
13929
|
-
mid: this.bandAutoGain.mid.getGain(),
|
|
13930
|
-
highMid: this.bandAutoGain.highMid.getGain(),
|
|
13931
|
-
high: this.bandAutoGain.high.getGain()
|
|
14023
|
+
low: this.mainChannel.bandAutoGain.low.getGain(),
|
|
14024
|
+
lowMid: this.mainChannel.bandAutoGain.lowMid.getGain(),
|
|
14025
|
+
mid: this.mainChannel.bandAutoGain.mid.getGain(),
|
|
14026
|
+
highMid: this.mainChannel.bandAutoGain.highMid.getGain(),
|
|
14027
|
+
high: this.mainChannel.bandAutoGain.high.getGain()
|
|
13932
14028
|
},
|
|
13933
|
-
volumeGain: this.volumeAutoGain.getGain(),
|
|
13934
|
-
|
|
13935
|
-
dtMs: this.lastDtMs,
|
|
14029
|
+
volumeGain: this.mainChannel.volumeAutoGain.getGain(),
|
|
14030
|
+
dtMs: this.mainChannel.lastDtMs,
|
|
13936
14031
|
analysisIntervalMs: this.analysisIntervalMs,
|
|
13937
|
-
|
|
13938
|
-
events: [...this.audioState.beat.events]
|
|
14032
|
+
events: [...this.audioStateBeat.events]
|
|
13939
14033
|
};
|
|
13940
14034
|
}
|
|
13941
14035
|
}
|
|
@@ -14360,7 +14454,8 @@ class DeviceSensorManager {
|
|
|
14360
14454
|
name: device.name,
|
|
14361
14455
|
motion: null,
|
|
14362
14456
|
orientation: null,
|
|
14363
|
-
video: null
|
|
14457
|
+
video: null,
|
|
14458
|
+
audio: null
|
|
14364
14459
|
};
|
|
14365
14460
|
this.externalDevices.set(device.id, newDevice);
|
|
14366
14461
|
this.externalDeviceOrder.push(device.id);
|
|
@@ -14460,6 +14555,21 @@ class SceneAnalyzer {
|
|
|
14460
14555
|
}
|
|
14461
14556
|
return "native";
|
|
14462
14557
|
}
|
|
14558
|
+
/**
|
|
14559
|
+
* P5 main-canvas mode from the renderer directive (only meaningful when
|
|
14560
|
+
* {@link detectRendererType} is `'p5'`).
|
|
14561
|
+
*
|
|
14562
|
+
* - `// @renderer p5` → 2D (default)
|
|
14563
|
+
* - `// @renderer p5 webgl` → WEBGL
|
|
14564
|
+
*/
|
|
14565
|
+
static detectP5CanvasMode(sceneCode) {
|
|
14566
|
+
if (/\/\/\s*@renderer\s+p5\s+webgl\b|\/\*\s*@renderer\s+p5\s+webgl\s*\*\//i.test(
|
|
14567
|
+
sceneCode
|
|
14568
|
+
)) {
|
|
14569
|
+
return "webgl";
|
|
14570
|
+
}
|
|
14571
|
+
return "2d";
|
|
14572
|
+
}
|
|
14463
14573
|
}
|
|
14464
14574
|
class VijiCore {
|
|
14465
14575
|
iframeManager = null;
|
|
@@ -14500,16 +14610,25 @@ class VijiCore {
|
|
|
14500
14610
|
// Index 200..299: Direct frame injection (compositor pipeline)
|
|
14501
14611
|
//
|
|
14502
14612
|
// The worker filters by streamType when building artist-facing arrays
|
|
14503
|
-
// (viji.
|
|
14613
|
+
// (viji.videoStreams[], viji.devices[].video), so absolute indices are internal.
|
|
14504
14614
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
14505
14615
|
static ADDITIONAL_STREAM_BASE = 1;
|
|
14506
14616
|
static DEVICE_VIDEO_BASE = 100;
|
|
14507
14617
|
static DIRECT_FRAME_BASE = 200;
|
|
14618
|
+
// Audio stream index ranges (mirroring video)
|
|
14619
|
+
static AUDIO_ADDITIONAL_BASE = 1;
|
|
14620
|
+
static AUDIO_DEVICE_BASE = 100;
|
|
14621
|
+
static MAX_ADDITIONAL_AUDIO_STREAMS = 8;
|
|
14622
|
+
static MAX_ADDITIONAL_VIDEO_STREAMS = 8;
|
|
14508
14623
|
// Separated video stream management
|
|
14509
14624
|
videoStream = null;
|
|
14510
14625
|
// Main stream (CV enabled) - always index 0
|
|
14511
14626
|
videoStreams = [];
|
|
14512
14627
|
// Additional streams (no CV)
|
|
14628
|
+
// Audio stream management
|
|
14629
|
+
audioStreams = [];
|
|
14630
|
+
// Additional audio streams (lightweight analysis)
|
|
14631
|
+
deviceAudioStreamIndices = /* @__PURE__ */ new Map();
|
|
14513
14632
|
// Video coordinators
|
|
14514
14633
|
mainVideoCoordinator = null;
|
|
14515
14634
|
additionalCoordinators = [];
|
|
@@ -14565,6 +14684,7 @@ class VijiCore {
|
|
|
14565
14684
|
this.stats.rendererType = SceneAnalyzer.detectRendererType(config.sceneCode);
|
|
14566
14685
|
this.videoStream = config.videoStream || null;
|
|
14567
14686
|
this.videoStreams = config.videoStreams || [];
|
|
14687
|
+
this.audioStreams = config.audioStreams ? [...config.audioStreams] : [];
|
|
14568
14688
|
this.currentInteractionEnabled = this.config.allowUserInteraction;
|
|
14569
14689
|
if (config.allowDeviceInteraction) {
|
|
14570
14690
|
this.deviceSensorManager = new DeviceSensorManager();
|
|
@@ -14783,6 +14903,14 @@ class VijiCore {
|
|
|
14783
14903
|
if (this.config.audioStream) {
|
|
14784
14904
|
await this.setAudioStream(this.config.audioStream);
|
|
14785
14905
|
}
|
|
14906
|
+
for (let i = 0; i < this.audioStreams.length; i++) {
|
|
14907
|
+
const streamIndex = VijiCore.AUDIO_ADDITIONAL_BASE + i;
|
|
14908
|
+
await this.audioSystem.addStream(streamIndex, this.audioStreams[i]);
|
|
14909
|
+
this.workerManager.postMessage("audio-stream-setup", {
|
|
14910
|
+
streamIndex,
|
|
14911
|
+
streamType: "additional"
|
|
14912
|
+
});
|
|
14913
|
+
}
|
|
14786
14914
|
this.stats.resolution = effectiveResolution;
|
|
14787
14915
|
this.stats.scale = this.iframeManager.getScale();
|
|
14788
14916
|
this.updateFrameRateStats();
|
|
@@ -15586,12 +15714,18 @@ class VijiCore {
|
|
|
15586
15714
|
// ADDITIONAL VIDEO STREAMS API
|
|
15587
15715
|
// ═══════════════════════════════════════════════════════
|
|
15588
15716
|
/**
|
|
15589
|
-
* Adds an additional video stream (no CV). Returns its index in viji.
|
|
15717
|
+
* Adds an additional video stream (no CV). Returns its index in viji.videoStreams[].
|
|
15590
15718
|
*/
|
|
15591
15719
|
async addVideoStream(stream) {
|
|
15592
15720
|
this.validateReady();
|
|
15593
15721
|
const existingIndex = this.videoStreams.indexOf(stream);
|
|
15594
15722
|
if (existingIndex !== -1) return existingIndex;
|
|
15723
|
+
if (this.videoStreams.length >= VijiCore.MAX_ADDITIONAL_VIDEO_STREAMS) {
|
|
15724
|
+
throw new VijiCoreError(
|
|
15725
|
+
`Maximum additional video streams reached (${VijiCore.MAX_ADDITIONAL_VIDEO_STREAMS})`,
|
|
15726
|
+
"STREAM_LIMIT_REACHED"
|
|
15727
|
+
);
|
|
15728
|
+
}
|
|
15595
15729
|
this.videoStreams.push(stream);
|
|
15596
15730
|
const newIndex = this.videoStreams.length - 1;
|
|
15597
15731
|
const streamIndex = VijiCore.ADDITIONAL_STREAM_BASE + newIndex;
|
|
@@ -15636,6 +15770,103 @@ class VijiCore {
|
|
|
15636
15770
|
getVideoStreamCount() {
|
|
15637
15771
|
return this.videoStreams.length;
|
|
15638
15772
|
}
|
|
15773
|
+
// ═══════════════════════════════════════════════════════════
|
|
15774
|
+
// Audio Stream Management (mirroring video stream API)
|
|
15775
|
+
// ═══════════════════════════════════════════════════════════
|
|
15776
|
+
/**
|
|
15777
|
+
* Add an additional audio stream for lightweight analysis (no beat detection).
|
|
15778
|
+
* @param stream MediaStream with audio tracks
|
|
15779
|
+
* @returns Index of the new stream (0-based within additional streams)
|
|
15780
|
+
* @throws VijiCoreError if limit reached (max 8)
|
|
15781
|
+
*/
|
|
15782
|
+
async addAudioStream(stream) {
|
|
15783
|
+
this.validateReady();
|
|
15784
|
+
const existingIndex = this.audioStreams.indexOf(stream);
|
|
15785
|
+
if (existingIndex !== -1) return existingIndex;
|
|
15786
|
+
if (this.audioStreams.length >= VijiCore.MAX_ADDITIONAL_AUDIO_STREAMS) {
|
|
15787
|
+
throw new VijiCoreError(
|
|
15788
|
+
`Maximum additional audio streams reached (${VijiCore.MAX_ADDITIONAL_AUDIO_STREAMS})`,
|
|
15789
|
+
"STREAM_LIMIT_REACHED"
|
|
15790
|
+
);
|
|
15791
|
+
}
|
|
15792
|
+
this.audioStreams.push(stream);
|
|
15793
|
+
const newIndex = this.audioStreams.length - 1;
|
|
15794
|
+
const streamIndex = VijiCore.AUDIO_ADDITIONAL_BASE + newIndex;
|
|
15795
|
+
if (this.audioSystem) {
|
|
15796
|
+
await this.audioSystem.addStream(streamIndex, stream);
|
|
15797
|
+
}
|
|
15798
|
+
this.workerManager?.postMessage("audio-stream-setup", {
|
|
15799
|
+
streamIndex,
|
|
15800
|
+
streamType: "additional"
|
|
15801
|
+
});
|
|
15802
|
+
return newIndex;
|
|
15803
|
+
}
|
|
15804
|
+
/**
|
|
15805
|
+
* Remove an additional audio stream by index.
|
|
15806
|
+
* Triggers re-indexing of remaining streams.
|
|
15807
|
+
*/
|
|
15808
|
+
async removeAudioStream(index) {
|
|
15809
|
+
this.validateReady();
|
|
15810
|
+
if (index < 0 || index >= this.audioStreams.length) {
|
|
15811
|
+
throw new VijiCoreError(`Invalid audio stream index: ${index}`, "INVALID_INDEX");
|
|
15812
|
+
}
|
|
15813
|
+
this.audioStreams[index].getTracks().forEach((track) => track.stop());
|
|
15814
|
+
this.audioStreams.splice(index, 1);
|
|
15815
|
+
if (this.audioSystem) {
|
|
15816
|
+
await this.audioSystem.reinitializeAdditionalChannels(
|
|
15817
|
+
this.audioStreams,
|
|
15818
|
+
VijiCore.AUDIO_ADDITIONAL_BASE
|
|
15819
|
+
);
|
|
15820
|
+
}
|
|
15821
|
+
for (let i = 0; i < this.audioStreams.length; i++) {
|
|
15822
|
+
this.workerManager?.postMessage("audio-stream-setup", {
|
|
15823
|
+
streamIndex: VijiCore.AUDIO_ADDITIONAL_BASE + i,
|
|
15824
|
+
streamType: "additional"
|
|
15825
|
+
});
|
|
15826
|
+
}
|
|
15827
|
+
}
|
|
15828
|
+
/**
|
|
15829
|
+
* Gets the number of additional audio streams.
|
|
15830
|
+
*/
|
|
15831
|
+
getAudioStreamCount() {
|
|
15832
|
+
return this.audioStreams.length;
|
|
15833
|
+
}
|
|
15834
|
+
/**
|
|
15835
|
+
* Set audio stream from an external device.
|
|
15836
|
+
* @param deviceId Device identifier (must be registered via addExternalDevice)
|
|
15837
|
+
* @param stream MediaStream with audio tracks from the device
|
|
15838
|
+
*/
|
|
15839
|
+
async setDeviceAudio(deviceId, stream) {
|
|
15840
|
+
this.validateReady();
|
|
15841
|
+
await this.clearDeviceAudio(deviceId);
|
|
15842
|
+
const usedIndices = new Set(this.deviceAudioStreamIndices.values());
|
|
15843
|
+
let streamIndex = VijiCore.AUDIO_DEVICE_BASE;
|
|
15844
|
+
while (usedIndices.has(streamIndex)) {
|
|
15845
|
+
streamIndex++;
|
|
15846
|
+
}
|
|
15847
|
+
this.deviceAudioStreamIndices.set(deviceId, streamIndex);
|
|
15848
|
+
if (this.audioSystem) {
|
|
15849
|
+
await this.audioSystem.addStream(streamIndex, stream);
|
|
15850
|
+
}
|
|
15851
|
+
this.workerManager?.postMessage("audio-stream-setup", {
|
|
15852
|
+
streamIndex,
|
|
15853
|
+
streamType: "device",
|
|
15854
|
+
deviceId
|
|
15855
|
+
});
|
|
15856
|
+
this.debugLog(`Device audio set for ${deviceId} at index ${streamIndex}`);
|
|
15857
|
+
}
|
|
15858
|
+
/**
|
|
15859
|
+
* Clear audio stream from a device.
|
|
15860
|
+
*/
|
|
15861
|
+
async clearDeviceAudio(deviceId) {
|
|
15862
|
+
const streamIndex = this.deviceAudioStreamIndices.get(deviceId);
|
|
15863
|
+
if (streamIndex === void 0) return;
|
|
15864
|
+
this.deviceAudioStreamIndices.delete(deviceId);
|
|
15865
|
+
if (this.audioSystem) {
|
|
15866
|
+
this.audioSystem.removeStream(streamIndex);
|
|
15867
|
+
}
|
|
15868
|
+
this.debugLog(`Device audio cleared for ${deviceId}`);
|
|
15869
|
+
}
|
|
15639
15870
|
/**
|
|
15640
15871
|
* Reinitializes all additional coordinators after array mutation.
|
|
15641
15872
|
*/
|
|
@@ -15970,6 +16201,9 @@ class VijiCore {
|
|
|
15970
16201
|
if (this.deviceVideoCoordinators.has(deviceId)) {
|
|
15971
16202
|
this.clearDeviceVideo(deviceId);
|
|
15972
16203
|
}
|
|
16204
|
+
if (this.deviceAudioStreamIndices.has(deviceId)) {
|
|
16205
|
+
this.clearDeviceAudio(deviceId);
|
|
16206
|
+
}
|
|
15973
16207
|
this.deviceSensorManager.removeExternalDevice(deviceId);
|
|
15974
16208
|
this.syncDeviceStateToWorker();
|
|
15975
16209
|
this.debugLog(`External device removed: ${deviceId}`);
|
|
@@ -16075,11 +16309,16 @@ class VijiCore {
|
|
|
16075
16309
|
this.deviceSensorManager.destroy();
|
|
16076
16310
|
this.deviceSensorManager = null;
|
|
16077
16311
|
}
|
|
16312
|
+
for (const [deviceId] of this.deviceAudioStreamIndices) {
|
|
16313
|
+
await this.clearDeviceAudio(deviceId);
|
|
16314
|
+
}
|
|
16315
|
+
this.deviceAudioStreamIndices.clear();
|
|
16078
16316
|
if (this.audioSystem) {
|
|
16079
16317
|
this.audioSystem.resetAudioState();
|
|
16080
16318
|
this.audioSystem = null;
|
|
16081
16319
|
}
|
|
16082
16320
|
this.currentAudioStream = null;
|
|
16321
|
+
this.audioStreams = [];
|
|
16083
16322
|
if (this.mainVideoCoordinator) {
|
|
16084
16323
|
this.mainVideoCoordinator.resetVideoState();
|
|
16085
16324
|
this.mainVideoCoordinator = null;
|
|
@@ -16158,4 +16397,4 @@ export {
|
|
|
16158
16397
|
VijiCoreError as b,
|
|
16159
16398
|
getDefaultExportFromCjs as g
|
|
16160
16399
|
};
|
|
16161
|
-
//# sourceMappingURL=index-
|
|
16400
|
+
//# sourceMappingURL=index-9YDi9UBP.js.map
|