@entros/pulse-sdk 1.5.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.mts +34 -8
- package/dist/index.d.ts +34 -8
- package/dist/index.js +788 -45
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +788 -45
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -582,43 +582,445 @@ function findRoots(coefficients, maxIterations = 50) {
|
|
|
582
582
|
}
|
|
583
583
|
return roots;
|
|
584
584
|
}
|
|
585
|
-
function
|
|
585
|
+
function extractFrameAnalysis(frame, sampleRate, lpcOrder = 12) {
|
|
586
586
|
const r = autocorrelate(frame, lpcOrder);
|
|
587
587
|
const coeffs = levinsonDurbin(r, lpcOrder);
|
|
588
588
|
const roots = findRoots(coeffs);
|
|
589
|
-
const
|
|
589
|
+
const candidates = [];
|
|
590
590
|
for (const [real, imag] of roots) {
|
|
591
591
|
if (imag <= 0) continue;
|
|
592
592
|
const freq = Math.atan2(imag, real) / (2 * Math.PI) * sampleRate;
|
|
593
593
|
const bandwidth = -sampleRate / (2 * Math.PI) * Math.log(Math.sqrt(real * real + imag * imag));
|
|
594
594
|
if (freq > 200 && freq < 5e3 && bandwidth < 500) {
|
|
595
|
-
|
|
595
|
+
candidates.push({ freq, bandwidth });
|
|
596
596
|
}
|
|
597
597
|
}
|
|
598
|
-
|
|
599
|
-
if (
|
|
600
|
-
|
|
598
|
+
candidates.sort((a, b) => a.freq - b.freq);
|
|
599
|
+
if (candidates.length < 3) {
|
|
600
|
+
return { lpcCoefficients: coeffs, formants: null, bandwidths: null };
|
|
601
|
+
}
|
|
602
|
+
const formants = [
|
|
603
|
+
candidates[0].freq,
|
|
604
|
+
candidates[1].freq,
|
|
605
|
+
candidates[2].freq
|
|
606
|
+
];
|
|
607
|
+
const bandwidths = [
|
|
608
|
+
candidates[0].bandwidth,
|
|
609
|
+
candidates[1].bandwidth,
|
|
610
|
+
candidates[2].bandwidth
|
|
611
|
+
];
|
|
612
|
+
return { lpcCoefficients: coeffs, formants, bandwidths };
|
|
601
613
|
}
|
|
602
|
-
function
|
|
614
|
+
function extractLpcAnalysis(samples, sampleRate, frameSize, hopSize, lpcOrder = 12) {
|
|
615
|
+
const lpcCoefficients = Array.from({ length: lpcOrder }, () => []);
|
|
616
|
+
const f1 = [];
|
|
617
|
+
const f2 = [];
|
|
618
|
+
const f3 = [];
|
|
619
|
+
const b1 = [];
|
|
620
|
+
const b2 = [];
|
|
621
|
+
const b3 = [];
|
|
603
622
|
const f1f2 = [];
|
|
604
623
|
const f2f3 = [];
|
|
605
624
|
const numFrames = Math.floor((samples.length - frameSize) / hopSize) + 1;
|
|
625
|
+
let numFramesAnalyzed = 0;
|
|
626
|
+
if (numFrames < 1) {
|
|
627
|
+
return {
|
|
628
|
+
lpcCoefficients,
|
|
629
|
+
f1,
|
|
630
|
+
f2,
|
|
631
|
+
f3,
|
|
632
|
+
b1,
|
|
633
|
+
b2,
|
|
634
|
+
b3,
|
|
635
|
+
f1f2,
|
|
636
|
+
f2f3,
|
|
637
|
+
numFramesAnalyzed: 0
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
const windowed = new Float32Array(frameSize);
|
|
606
641
|
for (let i = 0; i < numFrames; i++) {
|
|
607
642
|
const start = i * hopSize;
|
|
608
643
|
const frame = samples.subarray(start, start + frameSize);
|
|
609
|
-
const windowed = new Float32Array(frameSize);
|
|
610
644
|
for (let j = 0; j < frameSize; j++) {
|
|
611
645
|
windowed[j] = (frame[j] ?? 0) * (0.54 - 0.46 * Math.cos(2 * Math.PI * j / (frameSize - 1)));
|
|
612
646
|
}
|
|
613
|
-
const
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
if (
|
|
647
|
+
const analysis = extractFrameAnalysis(windowed, sampleRate, lpcOrder);
|
|
648
|
+
numFramesAnalyzed++;
|
|
649
|
+
for (let c = 0; c < lpcOrder; c++) {
|
|
650
|
+
const coeff = analysis.lpcCoefficients[c];
|
|
651
|
+
if (Number.isFinite(coeff)) {
|
|
652
|
+
lpcCoefficients[c].push(coeff);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
if (analysis.formants && analysis.bandwidths) {
|
|
656
|
+
const [F1, F2, F3] = analysis.formants;
|
|
657
|
+
const [B1, B2, B3] = analysis.bandwidths;
|
|
658
|
+
f1.push(F1);
|
|
659
|
+
f2.push(F2);
|
|
660
|
+
f3.push(F3);
|
|
661
|
+
b1.push(B1);
|
|
662
|
+
b2.push(B2);
|
|
663
|
+
b3.push(B3);
|
|
664
|
+
if (F2 > 0) f1f2.push(F1 / F2);
|
|
665
|
+
if (F3 > 0) f2f3.push(F2 / F3);
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
return {
|
|
669
|
+
lpcCoefficients,
|
|
670
|
+
f1,
|
|
671
|
+
f2,
|
|
672
|
+
f3,
|
|
673
|
+
b1,
|
|
674
|
+
b2,
|
|
675
|
+
b3,
|
|
676
|
+
f1f2,
|
|
677
|
+
f2f3,
|
|
678
|
+
numFramesAnalyzed
|
|
679
|
+
};
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// src/extraction/mfcc.ts
|
|
683
|
+
var NUM_MFCC_COEFFICIENTS = 13;
|
|
684
|
+
var DELTA_REGRESSION_HALF_WIDTH = 2;
|
|
685
|
+
var MFCC_FEATURE_COUNT = NUM_MFCC_COEFFICIENTS * 4 + // mean, var, skew, kurt per coefficient
|
|
686
|
+
NUM_MFCC_COEFFICIENTS * 2;
|
|
687
|
+
function computeDelta(series, halfWidth) {
|
|
688
|
+
const n = series.length;
|
|
689
|
+
const out = new Array(n);
|
|
690
|
+
const fullDenom = halfWidth * (halfWidth + 1) * (2 * halfWidth + 1) / 3;
|
|
691
|
+
for (let t = 0; t < n; t++) {
|
|
692
|
+
let num = 0;
|
|
693
|
+
let denom = fullDenom;
|
|
694
|
+
for (let k = 1; k <= halfWidth; k++) {
|
|
695
|
+
const tPlus = t + k;
|
|
696
|
+
const tMinus = t - k;
|
|
697
|
+
if (tPlus >= n || tMinus < 0) {
|
|
698
|
+
denom -= 2 * k * k;
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
num += k * (series[tPlus] - series[tMinus]);
|
|
702
|
+
}
|
|
703
|
+
if (denom <= 0) {
|
|
704
|
+
out[t] = 0;
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
out[t] = num / denom;
|
|
708
|
+
}
|
|
709
|
+
return out;
|
|
710
|
+
}
|
|
711
|
+
var meydaModule = null;
|
|
712
|
+
async function getMeyda() {
|
|
713
|
+
if (!meydaModule) {
|
|
714
|
+
try {
|
|
715
|
+
meydaModule = await import("meyda");
|
|
716
|
+
} catch {
|
|
717
|
+
return null;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
return meydaModule.default ?? meydaModule;
|
|
721
|
+
}
|
|
722
|
+
async function extractMfccFeatures(samples, sampleRate, frameSize, hopSize) {
|
|
723
|
+
if (!Number.isFinite(sampleRate) || sampleRate <= 0 || samples.length === 0 || frameSize <= 0 || hopSize <= 0) {
|
|
724
|
+
return new Array(MFCC_FEATURE_COUNT).fill(0);
|
|
725
|
+
}
|
|
726
|
+
const Meyda = await getMeyda();
|
|
727
|
+
if (!Meyda) {
|
|
728
|
+
sdkWarn("[Entros SDK] Meyda unavailable; MFCC features will be zeros.");
|
|
729
|
+
return new Array(MFCC_FEATURE_COUNT).fill(0);
|
|
730
|
+
}
|
|
731
|
+
const numFrames = Math.floor((samples.length - frameSize) / hopSize) + 1;
|
|
732
|
+
if (numFrames < 5) {
|
|
733
|
+
return new Array(MFCC_FEATURE_COUNT).fill(0);
|
|
734
|
+
}
|
|
735
|
+
const mfccTracks = Array.from(
|
|
736
|
+
{ length: NUM_MFCC_COEFFICIENTS },
|
|
737
|
+
() => []
|
|
738
|
+
);
|
|
739
|
+
const frame = new Float32Array(frameSize);
|
|
740
|
+
Meyda.bufferSize = frameSize;
|
|
741
|
+
Meyda.sampleRate = sampleRate;
|
|
742
|
+
for (let i = 0; i < numFrames; i++) {
|
|
743
|
+
const start = i * hopSize;
|
|
744
|
+
frame.set(samples.subarray(start, start + frameSize), 0);
|
|
745
|
+
const result = Meyda.extract("mfcc", frame);
|
|
746
|
+
if (!Array.isArray(result) || result.length !== NUM_MFCC_COEFFICIENTS) {
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
let allFinite = true;
|
|
750
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
751
|
+
if (!Number.isFinite(result[c])) {
|
|
752
|
+
allFinite = false;
|
|
753
|
+
break;
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
if (!allFinite) continue;
|
|
757
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
758
|
+
mfccTracks[c].push(result[c]);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
const out = [];
|
|
762
|
+
out.length = MFCC_FEATURE_COUNT;
|
|
763
|
+
let writeIdx = 0;
|
|
764
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
765
|
+
const stats = condense(mfccTracks[c]);
|
|
766
|
+
out[writeIdx++] = stats.mean;
|
|
767
|
+
out[writeIdx++] = stats.variance;
|
|
768
|
+
out[writeIdx++] = stats.skewness;
|
|
769
|
+
out[writeIdx++] = stats.kurtosis;
|
|
770
|
+
}
|
|
771
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
772
|
+
const delta = computeDelta(mfccTracks[c], DELTA_REGRESSION_HALF_WIDTH);
|
|
773
|
+
const muDelta = mean(delta);
|
|
774
|
+
out[writeIdx++] = muDelta;
|
|
775
|
+
out[writeIdx++] = variance(delta, muDelta);
|
|
776
|
+
}
|
|
777
|
+
return out;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
// src/extraction/voice-quality.ts
|
|
781
|
+
var VOICE_QUALITY_FEATURE_COUNT = 9;
|
|
782
|
+
var LOW_BAND_HZ = 1e3;
|
|
783
|
+
var MID_BAND_HZ = 3e3;
|
|
784
|
+
var HIGH_BAND_HZ = 8e3;
|
|
785
|
+
function cppQuefrencyRange(sampleRate) {
|
|
786
|
+
return {
|
|
787
|
+
qMin: Math.max(2, Math.floor(sampleRate / 400)),
|
|
788
|
+
qMax: Math.floor(sampleRate / 60)
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
var meydaModule2 = null;
|
|
792
|
+
async function getMeyda2() {
|
|
793
|
+
if (!meydaModule2) {
|
|
794
|
+
try {
|
|
795
|
+
meydaModule2 = await import("meyda");
|
|
796
|
+
} catch {
|
|
797
|
+
return null;
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
return meydaModule2.default ?? meydaModule2;
|
|
801
|
+
}
|
|
802
|
+
function cepstralPeakProminence(powerSpectrum, sampleRate) {
|
|
803
|
+
const N = powerSpectrum.length;
|
|
804
|
+
if (N < 8) return 0;
|
|
805
|
+
const { qMin, qMax } = cppQuefrencyRange(sampleRate);
|
|
806
|
+
if (qMax >= N || qMax <= qMin) return 0;
|
|
807
|
+
const FLOOR = 1e-12;
|
|
808
|
+
const logPower = new Array(N);
|
|
809
|
+
for (let i = 0; i < N; i++) {
|
|
810
|
+
const p = Math.max(powerSpectrum[i], FLOOR);
|
|
811
|
+
const l = Math.log(p);
|
|
812
|
+
if (!Number.isFinite(l)) return 0;
|
|
813
|
+
logPower[i] = l;
|
|
814
|
+
}
|
|
815
|
+
const bandLen = qMax - qMin + 1;
|
|
816
|
+
const cepstrumBand = new Array(bandLen);
|
|
817
|
+
const piOverN = Math.PI / N;
|
|
818
|
+
for (let bIdx = 0; bIdx < bandLen; bIdx++) {
|
|
819
|
+
const k = qMin + bIdx;
|
|
820
|
+
let sum = 0;
|
|
821
|
+
for (let n = 0; n < N; n++) {
|
|
822
|
+
sum += logPower[n] * Math.cos(piOverN * (n + 0.5) * k);
|
|
823
|
+
}
|
|
824
|
+
cepstrumBand[bIdx] = sum;
|
|
825
|
+
}
|
|
826
|
+
let peakBIdx = 0;
|
|
827
|
+
let peakVal = cepstrumBand[0];
|
|
828
|
+
for (let bIdx = 1; bIdx < bandLen; bIdx++) {
|
|
829
|
+
if (cepstrumBand[bIdx] > peakVal) {
|
|
830
|
+
peakVal = cepstrumBand[bIdx];
|
|
831
|
+
peakBIdx = bIdx;
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
const peakQuefrency = qMin + peakBIdx;
|
|
835
|
+
const M = bandLen;
|
|
836
|
+
let sx = 0;
|
|
837
|
+
let sy = 0;
|
|
838
|
+
let sxx = 0;
|
|
839
|
+
let sxy = 0;
|
|
840
|
+
for (let bIdx = 0; bIdx < bandLen; bIdx++) {
|
|
841
|
+
const x = qMin + bIdx;
|
|
842
|
+
const y = cepstrumBand[bIdx];
|
|
843
|
+
sx += x;
|
|
844
|
+
sy += y;
|
|
845
|
+
sxx += x * x;
|
|
846
|
+
sxy += x * y;
|
|
847
|
+
}
|
|
848
|
+
const denom = M * sxx - sx * sx;
|
|
849
|
+
if (Math.abs(denom) < 1e-12) return 0;
|
|
850
|
+
const slope = (M * sxy - sx * sy) / denom;
|
|
851
|
+
const intercept = (sy - slope * sx) / M;
|
|
852
|
+
const baselineAtPeak = intercept + slope * peakQuefrency;
|
|
853
|
+
return peakVal - baselineAtPeak;
|
|
854
|
+
}
|
|
855
|
+
function spectralTilt(powerSpectrum, sampleRate) {
|
|
856
|
+
const N = powerSpectrum.length;
|
|
857
|
+
if (N < 8) return 0;
|
|
858
|
+
const FLOOR = 1e-12;
|
|
859
|
+
let sx = 0;
|
|
860
|
+
let sy = 0;
|
|
861
|
+
let sxx = 0;
|
|
862
|
+
let sxy = 0;
|
|
863
|
+
let count = 0;
|
|
864
|
+
const minBin = Math.max(1, Math.floor(100 * 2 * (N - 1) / sampleRate));
|
|
865
|
+
for (let k = minBin; k < N; k++) {
|
|
866
|
+
const p = powerSpectrum[k];
|
|
867
|
+
if (p < FLOOR) continue;
|
|
868
|
+
const x = Math.log(k);
|
|
869
|
+
const y = Math.log(p);
|
|
870
|
+
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
|
|
871
|
+
sx += x;
|
|
872
|
+
sy += y;
|
|
873
|
+
sxx += x * x;
|
|
874
|
+
sxy += x * y;
|
|
875
|
+
count++;
|
|
876
|
+
}
|
|
877
|
+
if (count < 4) return 0;
|
|
878
|
+
const denom = count * sxx - sx * sx;
|
|
879
|
+
if (Math.abs(denom) < 1e-12) return 0;
|
|
880
|
+
return (count * sxy - sx * sy) / denom;
|
|
881
|
+
}
|
|
882
|
+
function h1MinusH2(powerSpectrum, sampleRate, f0) {
|
|
883
|
+
if (!Number.isFinite(f0) || f0 <= 0) return 0;
|
|
884
|
+
const N = powerSpectrum.length;
|
|
885
|
+
if (N < 8) return 0;
|
|
886
|
+
const binPerHz = 2 * (N - 1) / sampleRate;
|
|
887
|
+
const k1 = Math.round(f0 * binPerHz);
|
|
888
|
+
const k2 = Math.round(2 * f0 * binPerHz);
|
|
889
|
+
const window2 = 2;
|
|
890
|
+
function peakNear(k) {
|
|
891
|
+
let best = -Infinity;
|
|
892
|
+
for (let i = k - window2; i <= k + window2; i++) {
|
|
893
|
+
if (i <= 0 || i >= N) continue;
|
|
894
|
+
const p = powerSpectrum[i];
|
|
895
|
+
if (p > best) best = p;
|
|
896
|
+
}
|
|
897
|
+
return best;
|
|
898
|
+
}
|
|
899
|
+
const h1 = peakNear(k1);
|
|
900
|
+
const h2 = peakNear(k2);
|
|
901
|
+
if (!Number.isFinite(h1) || !Number.isFinite(h2) || h1 <= 0 || h2 <= 0) return 0;
|
|
902
|
+
return 10 * Math.log10(h1 / h2);
|
|
903
|
+
}
|
|
904
|
+
function subbandRatios(powerSpectrum, sampleRate) {
|
|
905
|
+
const N = powerSpectrum.length;
|
|
906
|
+
if (N < 4) return [0, 0, 0];
|
|
907
|
+
const binPerHz = 2 * (N - 1) / sampleRate;
|
|
908
|
+
const lowBin = Math.min(N - 1, Math.round(LOW_BAND_HZ * binPerHz));
|
|
909
|
+
const midBin = Math.min(N - 1, Math.round(MID_BAND_HZ * binPerHz));
|
|
910
|
+
const highBin = Math.min(N - 1, Math.round(HIGH_BAND_HZ * binPerHz));
|
|
911
|
+
let total = 0;
|
|
912
|
+
let low = 0;
|
|
913
|
+
let mid = 0;
|
|
914
|
+
let high = 0;
|
|
915
|
+
for (let k = 1; k < N; k++) {
|
|
916
|
+
const p = powerSpectrum[k];
|
|
917
|
+
if (!Number.isFinite(p) || p < 0) continue;
|
|
918
|
+
total += p;
|
|
919
|
+
if (k <= lowBin) low += p;
|
|
920
|
+
else if (k <= midBin) mid += p;
|
|
921
|
+
else if (k <= highBin) high += p;
|
|
922
|
+
}
|
|
923
|
+
if (total < 1e-12) return [0, 0, 0];
|
|
924
|
+
return [low / total, mid / total, high / total];
|
|
925
|
+
}
|
|
926
|
+
async function extractVoiceQualityFeatures(samples, sampleRate, frameSize, hopSize, f0PerFrame) {
|
|
927
|
+
if (!Number.isFinite(sampleRate) || sampleRate <= 0 || samples.length === 0 || frameSize <= 0 || hopSize <= 0) {
|
|
928
|
+
return new Array(VOICE_QUALITY_FEATURE_COUNT).fill(0);
|
|
929
|
+
}
|
|
930
|
+
const Meyda = await getMeyda2();
|
|
931
|
+
if (!Meyda) {
|
|
932
|
+
sdkWarn("[Entros SDK] Meyda unavailable; voice quality features will be zeros.");
|
|
933
|
+
return new Array(VOICE_QUALITY_FEATURE_COUNT).fill(0);
|
|
934
|
+
}
|
|
935
|
+
const numFrames = Math.floor((samples.length - frameSize) / hopSize) + 1;
|
|
936
|
+
if (numFrames < 5) {
|
|
937
|
+
return new Array(VOICE_QUALITY_FEATURE_COUNT).fill(0);
|
|
938
|
+
}
|
|
939
|
+
const cppValues = [];
|
|
940
|
+
const tiltValues = [];
|
|
941
|
+
const h1h2Values = [];
|
|
942
|
+
const lowRatios = [];
|
|
943
|
+
const midRatios = [];
|
|
944
|
+
const highRatios = [];
|
|
945
|
+
const frame = new Float32Array(frameSize);
|
|
946
|
+
Meyda.bufferSize = frameSize;
|
|
947
|
+
Meyda.sampleRate = sampleRate;
|
|
948
|
+
for (let i = 0; i < numFrames; i++) {
|
|
949
|
+
const start = i * hopSize;
|
|
950
|
+
frame.set(samples.subarray(start, start + frameSize), 0);
|
|
951
|
+
const features = Meyda.extract("powerSpectrum", frame);
|
|
952
|
+
const power = features;
|
|
953
|
+
if (!power || power.length === 0) continue;
|
|
954
|
+
const cpp = cepstralPeakProminence(power, sampleRate);
|
|
955
|
+
if (Number.isFinite(cpp)) cppValues.push(cpp);
|
|
956
|
+
const tilt = spectralTilt(power, sampleRate);
|
|
957
|
+
if (Number.isFinite(tilt)) tiltValues.push(tilt);
|
|
958
|
+
const f0 = f0PerFrame[i] ?? 0;
|
|
959
|
+
if (f0 > 0) {
|
|
960
|
+
const h1h2 = h1MinusH2(power, sampleRate, f0);
|
|
961
|
+
if (Number.isFinite(h1h2)) h1h2Values.push(h1h2);
|
|
962
|
+
}
|
|
963
|
+
const [low, mid, high] = subbandRatios(power, sampleRate);
|
|
964
|
+
lowRatios.push(low);
|
|
965
|
+
midRatios.push(mid);
|
|
966
|
+
highRatios.push(high);
|
|
967
|
+
}
|
|
968
|
+
const cppMean = mean(cppValues);
|
|
969
|
+
const cppVar = variance(cppValues, cppMean);
|
|
970
|
+
const tiltMean = mean(tiltValues);
|
|
971
|
+
const tiltVar = variance(tiltValues, tiltMean);
|
|
972
|
+
const h1h2Mean = mean(h1h2Values);
|
|
973
|
+
const h1h2Var = variance(h1h2Values, h1h2Mean);
|
|
974
|
+
const lowMean = mean(lowRatios);
|
|
975
|
+
const midMean = mean(midRatios);
|
|
976
|
+
const highMean = mean(highRatios);
|
|
977
|
+
return [
|
|
978
|
+
cppMean,
|
|
979
|
+
cppVar,
|
|
980
|
+
tiltMean,
|
|
981
|
+
tiltVar,
|
|
982
|
+
h1h2Mean,
|
|
983
|
+
h1h2Var,
|
|
984
|
+
lowMean,
|
|
985
|
+
midMean,
|
|
986
|
+
highMean
|
|
987
|
+
];
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
// src/extraction/dct.ts
|
|
991
|
+
function dctII(input, numCoefficients) {
|
|
992
|
+
const N = input.length;
|
|
993
|
+
const K = Math.max(0, numCoefficients);
|
|
994
|
+
const output = new Array(K).fill(0);
|
|
995
|
+
if (N === 0 || K === 0) return output;
|
|
996
|
+
const upper = Math.min(K, N);
|
|
997
|
+
const piOverN = Math.PI / N;
|
|
998
|
+
for (let k = 0; k < upper; k++) {
|
|
999
|
+
let sum = 0;
|
|
1000
|
+
for (let n = 0; n < N; n++) {
|
|
1001
|
+
sum += input[n] * Math.cos(piOverN * (n + 0.5) * k);
|
|
618
1002
|
}
|
|
1003
|
+
output[k] = sum;
|
|
619
1004
|
}
|
|
620
|
-
return
|
|
1005
|
+
return output;
|
|
1006
|
+
}
|
|
1007
|
+
function pitchContourShape(contour, numCoefficients = 5) {
|
|
1008
|
+
if (numCoefficients <= 0) return [];
|
|
1009
|
+
const zero = () => new Array(numCoefficients).fill(0);
|
|
1010
|
+
const voiced = [];
|
|
1011
|
+
for (const v of contour) {
|
|
1012
|
+
if (Number.isFinite(v) && v > 0) voiced.push(v);
|
|
1013
|
+
}
|
|
1014
|
+
if (voiced.length < numCoefficients * 2) return zero();
|
|
1015
|
+
let sum = 0;
|
|
1016
|
+
for (const v of voiced) sum += v;
|
|
1017
|
+
const mu = sum / voiced.length;
|
|
1018
|
+
const centered = voiced.map((v) => v - mu);
|
|
1019
|
+
const N = centered.length;
|
|
1020
|
+
const norm = 1 / Math.sqrt(N);
|
|
1021
|
+
return dctII(centered, numCoefficients).map((c) => c * norm);
|
|
621
1022
|
}
|
|
1023
|
+
var PITCH_CONTOUR_SHAPE_FEATURE_COUNT = 5;
|
|
622
1024
|
|
|
623
1025
|
// src/yield.ts
|
|
624
1026
|
function yieldToMainThread() {
|
|
@@ -651,10 +1053,13 @@ function getFrameSize(sampleRate) {
|
|
|
651
1053
|
function getHopSize(sampleRate) {
|
|
652
1054
|
return Math.max(1, Math.round(sampleRate * 0.01));
|
|
653
1055
|
}
|
|
654
|
-
var
|
|
1056
|
+
var LEGACY_SPEAKER_FEATURE_COUNT = 44;
|
|
1057
|
+
var LPC_COEFFICIENT_STATS = 12 * 2;
|
|
1058
|
+
var FORMANT_TRAJECTORY_FEATURE_COUNT = 16;
|
|
1059
|
+
var SPEAKER_FEATURE_COUNT = LEGACY_SPEAKER_FEATURE_COUNT + MFCC_FEATURE_COUNT + LPC_COEFFICIENT_STATS + FORMANT_TRAJECTORY_FEATURE_COUNT + VOICE_QUALITY_FEATURE_COUNT + PITCH_CONTOUR_SHAPE_FEATURE_COUNT;
|
|
655
1060
|
var pitchDetector = null;
|
|
656
1061
|
var pitchDetectorRate = 0;
|
|
657
|
-
var
|
|
1062
|
+
var meydaModule3 = null;
|
|
658
1063
|
async function getPitchDetector(sampleRate) {
|
|
659
1064
|
if (!pitchDetector || pitchDetectorRate !== sampleRate) {
|
|
660
1065
|
const PitchFinder = await import("pitchfinder");
|
|
@@ -663,15 +1068,15 @@ async function getPitchDetector(sampleRate) {
|
|
|
663
1068
|
}
|
|
664
1069
|
return pitchDetector;
|
|
665
1070
|
}
|
|
666
|
-
async function
|
|
667
|
-
if (!
|
|
1071
|
+
async function getMeyda3() {
|
|
1072
|
+
if (!meydaModule3) {
|
|
668
1073
|
try {
|
|
669
|
-
|
|
1074
|
+
meydaModule3 = await import("meyda");
|
|
670
1075
|
} catch {
|
|
671
1076
|
return null;
|
|
672
1077
|
}
|
|
673
1078
|
}
|
|
674
|
-
return
|
|
1079
|
+
return meydaModule3.default ?? meydaModule3;
|
|
675
1080
|
}
|
|
676
1081
|
var F0_YIELD_EVERY_N_FRAMES = 16;
|
|
677
1082
|
async function detectF0Contour(samples, sampleRate) {
|
|
@@ -801,8 +1206,10 @@ function computeHNR(samples, sampleRate, f0Contour) {
|
|
|
801
1206
|
async function computeLTAS(samples, sampleRate) {
|
|
802
1207
|
const frameSize = getFrameSize(sampleRate);
|
|
803
1208
|
const hopSize = getHopSize(sampleRate);
|
|
804
|
-
const Meyda = await
|
|
1209
|
+
const Meyda = await getMeyda3();
|
|
805
1210
|
if (!Meyda) return new Array(8).fill(0);
|
|
1211
|
+
Meyda.bufferSize = frameSize;
|
|
1212
|
+
Meyda.sampleRate = sampleRate;
|
|
806
1213
|
const centroids = [];
|
|
807
1214
|
const rolloffs = [];
|
|
808
1215
|
const flatnesses = [];
|
|
@@ -814,8 +1221,7 @@ async function computeLTAS(samples, sampleRate) {
|
|
|
814
1221
|
paddedFrame.set(samples.subarray(start, start + frameSize), 0);
|
|
815
1222
|
const features = Meyda.extract(
|
|
816
1223
|
["spectralCentroid", "spectralRolloff", "spectralFlatness", "spectralSpread"],
|
|
817
|
-
paddedFrame
|
|
818
|
-
{ sampleRate, bufferSize: frameSize }
|
|
1224
|
+
paddedFrame
|
|
819
1225
|
);
|
|
820
1226
|
if (features) {
|
|
821
1227
|
if (Number.isFinite(features.spectralCentroid)) centroids.push(features.spectralCentroid);
|
|
@@ -902,9 +1308,9 @@ async function extractSpeakerFeaturesDetailed(audio) {
|
|
|
902
1308
|
const hnrEntropy = entropy(hnrValues);
|
|
903
1309
|
const hnrFeatures = [hnrStats.mean, hnrStats.variance, hnrStats.skewness, hnrStats.kurtosis, hnrEntropy];
|
|
904
1310
|
await yieldToMainThread();
|
|
905
|
-
const
|
|
906
|
-
const f1f2Stats = condense(f1f2);
|
|
907
|
-
const f2f3Stats = condense(f2f3);
|
|
1311
|
+
const lpc = extractLpcAnalysis(normalizedSamples, sampleRate, frameSize, hopSize);
|
|
1312
|
+
const f1f2Stats = condense(lpc.f1f2);
|
|
1313
|
+
const f2f3Stats = condense(lpc.f2f3);
|
|
908
1314
|
const formantFeatures = [
|
|
909
1315
|
f1f2Stats.mean,
|
|
910
1316
|
f1f2Stats.variance,
|
|
@@ -921,25 +1327,86 @@ async function extractSpeakerFeaturesDetailed(audio) {
|
|
|
921
1327
|
const ampStats = condense(amplitudes);
|
|
922
1328
|
const ampEntropy = entropy(amplitudes);
|
|
923
1329
|
const ampFeatures = [ampStats.mean, ampStats.variance, ampStats.skewness, ampStats.kurtosis, ampEntropy];
|
|
1330
|
+
await yieldToMainThread();
|
|
1331
|
+
const mfccFeatures = await extractMfccFeatures(
|
|
1332
|
+
normalizedSamples,
|
|
1333
|
+
sampleRate,
|
|
1334
|
+
frameSize,
|
|
1335
|
+
hopSize
|
|
1336
|
+
);
|
|
1337
|
+
const lpcStats = [];
|
|
1338
|
+
for (let c = 0; c < 12; c++) {
|
|
1339
|
+
const track = lpc.lpcCoefficients[c] ?? [];
|
|
1340
|
+
const mu = mean(track);
|
|
1341
|
+
lpcStats.push(mu, variance(track, mu));
|
|
1342
|
+
}
|
|
1343
|
+
const f1Stats = { mean: mean(lpc.f1), var: variance(lpc.f1) };
|
|
1344
|
+
const f2Stats = { mean: mean(lpc.f2), var: variance(lpc.f2) };
|
|
1345
|
+
const f3Stats = { mean: mean(lpc.f3), var: variance(lpc.f3) };
|
|
1346
|
+
const f1Delta = derivative(lpc.f1);
|
|
1347
|
+
const f2Delta = derivative(lpc.f2);
|
|
1348
|
+
const f3Delta = derivative(lpc.f3);
|
|
1349
|
+
const f1DeltaMu = mean(f1Delta);
|
|
1350
|
+
const f2DeltaMu = mean(f2Delta);
|
|
1351
|
+
const f3DeltaMu = mean(f3Delta);
|
|
1352
|
+
const b1Mu = mean(lpc.b1);
|
|
1353
|
+
const b2Mu = mean(lpc.b2);
|
|
1354
|
+
const formantTrajectoryFeatures = [
|
|
1355
|
+
f1Stats.mean,
|
|
1356
|
+
f1Stats.var,
|
|
1357
|
+
f2Stats.mean,
|
|
1358
|
+
f2Stats.var,
|
|
1359
|
+
f3Stats.mean,
|
|
1360
|
+
f3Stats.var,
|
|
1361
|
+
f1DeltaMu,
|
|
1362
|
+
variance(f1Delta, f1DeltaMu),
|
|
1363
|
+
f2DeltaMu,
|
|
1364
|
+
variance(f2Delta, f2DeltaMu),
|
|
1365
|
+
f3DeltaMu,
|
|
1366
|
+
variance(f3Delta, f3DeltaMu),
|
|
1367
|
+
b1Mu,
|
|
1368
|
+
variance(lpc.b1, b1Mu),
|
|
1369
|
+
b2Mu,
|
|
1370
|
+
variance(lpc.b2, b2Mu)
|
|
1371
|
+
];
|
|
1372
|
+
await yieldToMainThread();
|
|
1373
|
+
const voiceQualityFeatures = await extractVoiceQualityFeatures(
|
|
1374
|
+
normalizedSamples,
|
|
1375
|
+
sampleRate,
|
|
1376
|
+
frameSize,
|
|
1377
|
+
hopSize,
|
|
1378
|
+
f0
|
|
1379
|
+
);
|
|
1380
|
+
const pitchShapeFeatures = pitchContourShape(f0, PITCH_CONTOUR_SHAPE_FEATURE_COUNT);
|
|
924
1381
|
const features = [
|
|
925
1382
|
...f0Features,
|
|
926
|
-
// 5
|
|
1383
|
+
// 5 [0..5] F0_STATS
|
|
927
1384
|
...f0DeltaFeatures,
|
|
928
|
-
// 4
|
|
1385
|
+
// 4 [5..9] F0_DELTA
|
|
929
1386
|
...jitterFeatures,
|
|
930
|
-
// 4
|
|
1387
|
+
// 4 [9..13] JITTER
|
|
931
1388
|
...shimmerFeatures,
|
|
932
|
-
// 4
|
|
1389
|
+
// 4 [13..17] SHIMMER
|
|
933
1390
|
...hnrFeatures,
|
|
934
|
-
// 5
|
|
1391
|
+
// 5 [17..22] HNR
|
|
935
1392
|
...formantFeatures,
|
|
936
|
-
// 8
|
|
1393
|
+
// 8 [22..30] FORMANT_RATIOS
|
|
937
1394
|
...ltasFeatures,
|
|
938
|
-
// 8
|
|
1395
|
+
// 8 [30..38] LTAS
|
|
939
1396
|
...voicingFeatures,
|
|
940
|
-
// 1
|
|
941
|
-
...ampFeatures
|
|
942
|
-
// 5
|
|
1397
|
+
// 1 [38] VOICING_RATIO
|
|
1398
|
+
...ampFeatures,
|
|
1399
|
+
// 5 [39..44] AMPLITUDE
|
|
1400
|
+
...mfccFeatures,
|
|
1401
|
+
// 78 [44..122] MFCC + delta-MFCC
|
|
1402
|
+
...lpcStats,
|
|
1403
|
+
// 24 [122..146] LPC coefficient stats
|
|
1404
|
+
...formantTrajectoryFeatures,
|
|
1405
|
+
// 16 [146..162] Formant absolutes + dynamics + bandwidths
|
|
1406
|
+
...voiceQualityFeatures,
|
|
1407
|
+
// 9 [162..171] Voice quality
|
|
1408
|
+
...pitchShapeFeatures
|
|
1409
|
+
// 5 [171..176] Pitch contour shape DCT
|
|
943
1410
|
];
|
|
944
1411
|
return { features, f0Contour: f0 };
|
|
945
1412
|
}
|
|
@@ -948,7 +1415,102 @@ async function extractSpeakerFeatures(audio) {
|
|
|
948
1415
|
return features;
|
|
949
1416
|
}
|
|
950
1417
|
|
|
1418
|
+
// src/extraction/fft.ts
|
|
1419
|
+
function nextPow2(n) {
|
|
1420
|
+
if (n <= 2) return 2;
|
|
1421
|
+
let p = 2;
|
|
1422
|
+
while (p < n) p <<= 1;
|
|
1423
|
+
return p;
|
|
1424
|
+
}
|
|
1425
|
+
function realFFT(input, size) {
|
|
1426
|
+
if (size <= 0 || (size & size - 1) !== 0) {
|
|
1427
|
+
throw new Error(`FFT size must be a positive power of two, got ${size}`);
|
|
1428
|
+
}
|
|
1429
|
+
const real = new Array(size);
|
|
1430
|
+
const imag = new Array(size).fill(0);
|
|
1431
|
+
for (let i = 0; i < size; i++) {
|
|
1432
|
+
real[i] = i < input.length ? input[i] ?? 0 : 0;
|
|
1433
|
+
}
|
|
1434
|
+
for (let i = 1, j = 0; i < size; i++) {
|
|
1435
|
+
let bit = size >> 1;
|
|
1436
|
+
for (; j & bit; bit >>= 1) j ^= bit;
|
|
1437
|
+
j ^= bit;
|
|
1438
|
+
if (i < j) {
|
|
1439
|
+
const tr = real[i];
|
|
1440
|
+
real[i] = real[j];
|
|
1441
|
+
real[j] = tr;
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
for (let halfSize = 1; halfSize < size; halfSize <<= 1) {
|
|
1445
|
+
const fullSize = halfSize << 1;
|
|
1446
|
+
const phaseStep = -Math.PI / halfSize;
|
|
1447
|
+
for (let chunkStart = 0; chunkStart < size; chunkStart += fullSize) {
|
|
1448
|
+
for (let k = 0; k < halfSize; k++) {
|
|
1449
|
+
const phase = phaseStep * k;
|
|
1450
|
+
const wr = Math.cos(phase);
|
|
1451
|
+
const wi = Math.sin(phase);
|
|
1452
|
+
const ar = real[chunkStart + k];
|
|
1453
|
+
const ai = imag[chunkStart + k];
|
|
1454
|
+
const br = real[chunkStart + k + halfSize];
|
|
1455
|
+
const bi = imag[chunkStart + k + halfSize];
|
|
1456
|
+
const tr = wr * br - wi * bi;
|
|
1457
|
+
const ti = wr * bi + wi * br;
|
|
1458
|
+
real[chunkStart + k] = ar + tr;
|
|
1459
|
+
imag[chunkStart + k] = ai + ti;
|
|
1460
|
+
real[chunkStart + k + halfSize] = ar - tr;
|
|
1461
|
+
imag[chunkStart + k + halfSize] = ai - ti;
|
|
1462
|
+
}
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
return { real, imag };
|
|
1466
|
+
}
|
|
1467
|
+
function bandEnergy(real, imag, sampleRate, fLow, fHigh) {
|
|
1468
|
+
const N = real.length;
|
|
1469
|
+
if (N === 0 || !Number.isFinite(sampleRate) || sampleRate <= 0 || fLow >= fHigh || fLow < 0) {
|
|
1470
|
+
return 0;
|
|
1471
|
+
}
|
|
1472
|
+
const binHz = sampleRate / N;
|
|
1473
|
+
const kLow = Math.max(0, Math.ceil(fLow / binHz));
|
|
1474
|
+
const kHigh = Math.min(Math.floor(N / 2), Math.floor((fHigh - 1e-9) / binHz));
|
|
1475
|
+
let energy = 0;
|
|
1476
|
+
for (let k = kLow; k <= kHigh; k++) {
|
|
1477
|
+
const re = real[k] ?? 0;
|
|
1478
|
+
const im = imag[k] ?? 0;
|
|
1479
|
+
energy += re * re + im * im;
|
|
1480
|
+
}
|
|
1481
|
+
return energy / (N * N);
|
|
1482
|
+
}
|
|
1483
|
+
function peakInBand(real, imag, sampleRate, fLow, fHigh) {
|
|
1484
|
+
const N = real.length;
|
|
1485
|
+
if (N === 0 || !Number.isFinite(sampleRate) || sampleRate <= 0 || fLow >= fHigh || fLow < 0) {
|
|
1486
|
+
return { freq: 0, amplitude: 0 };
|
|
1487
|
+
}
|
|
1488
|
+
const binHz = sampleRate / N;
|
|
1489
|
+
const kLow = Math.max(0, Math.ceil(fLow / binHz));
|
|
1490
|
+
const kHigh = Math.min(Math.floor(N / 2), Math.floor((fHigh - 1e-9) / binHz));
|
|
1491
|
+
let bestK = -1;
|
|
1492
|
+
let bestAmp = -Infinity;
|
|
1493
|
+
for (let k = kLow; k <= kHigh; k++) {
|
|
1494
|
+
const re = real[k] ?? 0;
|
|
1495
|
+
const im = imag[k] ?? 0;
|
|
1496
|
+
const amp = re * re + im * im;
|
|
1497
|
+
if (amp > bestAmp) {
|
|
1498
|
+
bestAmp = amp;
|
|
1499
|
+
bestK = k;
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
if (bestK < 0) return { freq: 0, amplitude: 0 };
|
|
1503
|
+
return { freq: bestK * binHz, amplitude: bestAmp / (N * N) };
|
|
1504
|
+
}
|
|
1505
|
+
|
|
951
1506
|
// src/extraction/kinematic.ts
|
|
1507
|
+
var MOTION_LEGACY_COUNT = 54;
|
|
1508
|
+
var MOTION_V2_ADDITIONS = 27;
|
|
1509
|
+
var MOTION_FEATURE_COUNT = MOTION_LEGACY_COUNT + MOTION_V2_ADDITIONS;
|
|
1510
|
+
var TOUCH_LEGACY_COUNT = 36;
|
|
1511
|
+
var TOUCH_V2_ADDITIONS = 21;
|
|
1512
|
+
var TOUCH_FEATURE_COUNT = TOUCH_LEGACY_COUNT + TOUCH_V2_ADDITIONS;
|
|
1513
|
+
var MOUSE_DYNAMICS_FEATURE_COUNT = MOTION_FEATURE_COUNT;
|
|
952
1514
|
function extractAccelerationMagnitude(samples, targetFrameCount) {
|
|
953
1515
|
if (samples.length < 2 || targetFrameCount < 2) return [];
|
|
954
1516
|
const magnitudes = samples.map((s) => Math.sqrt(s.ax * s.ax + s.ay * s.ay + s.az * s.az));
|
|
@@ -966,7 +1528,7 @@ function extractAccelerationMagnitude(samples, targetFrameCount) {
|
|
|
966
1528
|
return out;
|
|
967
1529
|
}
|
|
968
1530
|
function extractMotionFeatures(samples) {
|
|
969
|
-
if (samples.length < 5) return new Array(
|
|
1531
|
+
if (samples.length < 5) return new Array(MOTION_FEATURE_COUNT).fill(0);
|
|
970
1532
|
const axes = {
|
|
971
1533
|
ax: samples.map((s) => s.ax),
|
|
972
1534
|
ay: samples.map((s) => s.ay),
|
|
@@ -1001,10 +1563,68 @@ function extractMotionFeatures(samples) {
|
|
|
1001
1563
|
}
|
|
1002
1564
|
features.push(windowVariances.length >= 2 ? variance(windowVariances) : 0);
|
|
1003
1565
|
}
|
|
1566
|
+
features.push(...computeMotionV2(axes, samples));
|
|
1004
1567
|
return features;
|
|
1005
1568
|
}
|
|
1569
|
+
function computeMotionV2(axes, samples) {
|
|
1570
|
+
const out = [];
|
|
1571
|
+
const covPairs = [
|
|
1572
|
+
[axes.ax, axes.gy],
|
|
1573
|
+
[axes.ay, axes.gx],
|
|
1574
|
+
[axes.az, axes.gz],
|
|
1575
|
+
[axes.ax, axes.az],
|
|
1576
|
+
[axes.ay, axes.az],
|
|
1577
|
+
[axes.gx, axes.gy]
|
|
1578
|
+
];
|
|
1579
|
+
for (const [a, b] of covPairs) out.push(covariance(a, b));
|
|
1580
|
+
const sampleRate = sampleRateFromTimestamps(samples.map((s) => s.timestamp));
|
|
1581
|
+
const fftSize = nextPow2(Math.max(64, axes.ax.length));
|
|
1582
|
+
const bands = [
|
|
1583
|
+
[0, 2],
|
|
1584
|
+
[2, 6],
|
|
1585
|
+
[6, 12],
|
|
1586
|
+
[12, 30]
|
|
1587
|
+
];
|
|
1588
|
+
const accelSpectra = [axes.ax, axes.ay, axes.az].map(
|
|
1589
|
+
(axis) => realFFT(meanCenter(axis), fftSize)
|
|
1590
|
+
);
|
|
1591
|
+
for (const spectrum of accelSpectra) {
|
|
1592
|
+
for (const [lo, hi] of bands) {
|
|
1593
|
+
out.push(bandEnergy(spectrum.real, spectrum.imag, sampleRate, lo, hi));
|
|
1594
|
+
}
|
|
1595
|
+
}
|
|
1596
|
+
const magnitude = samples.map(
|
|
1597
|
+
(s) => Math.sqrt(s.ax * s.ax + s.ay * s.ay + s.az * s.az)
|
|
1598
|
+
);
|
|
1599
|
+
const magSpectrum = realFFT(meanCenter(magnitude), fftSize);
|
|
1600
|
+
const tremor = peakInBand(
|
|
1601
|
+
magSpectrum.real,
|
|
1602
|
+
magSpectrum.imag,
|
|
1603
|
+
sampleRate,
|
|
1604
|
+
4,
|
|
1605
|
+
12
|
|
1606
|
+
);
|
|
1607
|
+
out.push(tremor.freq, tremor.amplitude);
|
|
1608
|
+
const duration = captureDurationSec(samples);
|
|
1609
|
+
const reversalRates = [axes.ax, axes.ay, axes.az].map(
|
|
1610
|
+
(axis) => duration > 0 ? signChangeCount(derivative2(axis)) / duration : 0
|
|
1611
|
+
);
|
|
1612
|
+
out.push(mean(reversalRates), variance(reversalRates));
|
|
1613
|
+
let gyroSum = 0;
|
|
1614
|
+
for (let i = 0; i < samples.length; i++) {
|
|
1615
|
+
const gx = samples[i].gx;
|
|
1616
|
+
const gy = samples[i].gy;
|
|
1617
|
+
const gz = samples[i].gz;
|
|
1618
|
+
gyroSum += Math.sqrt(gx * gx + gy * gy + gz * gz);
|
|
1619
|
+
}
|
|
1620
|
+
out.push(samples.length > 0 ? gyroSum / samples.length : 0);
|
|
1621
|
+
for (const lag of [1, 5, 10, 25]) {
|
|
1622
|
+
out.push(autocorrelation(magnitude, lag));
|
|
1623
|
+
}
|
|
1624
|
+
return out;
|
|
1625
|
+
}
|
|
1006
1626
|
function extractTouchFeatures(samples) {
|
|
1007
|
-
if (samples.length < 5) return new Array(
|
|
1627
|
+
if (samples.length < 5) return new Array(TOUCH_FEATURE_COUNT).fill(0);
|
|
1008
1628
|
const x = samples.map((s) => s.x);
|
|
1009
1629
|
const y = samples.map((s) => s.y);
|
|
1010
1630
|
const pressure = samples.map((s) => s.pressure);
|
|
@@ -1032,8 +1652,78 @@ function extractTouchFeatures(samples) {
|
|
|
1032
1652
|
}
|
|
1033
1653
|
features.push(windowVariances.length >= 2 ? variance(windowVariances) : 0);
|
|
1034
1654
|
}
|
|
1655
|
+
features.push(...computeTouchV2(samples, vx, vy));
|
|
1035
1656
|
return features;
|
|
1036
1657
|
}
|
|
1658
|
+
function computeTouchV2(samples, vx, vy) {
|
|
1659
|
+
const out = [];
|
|
1660
|
+
const pressure = samples.map((s) => s.pressure);
|
|
1661
|
+
const dPressure = derivative2(pressure);
|
|
1662
|
+
out.push(...Object.values(condense(dPressure)));
|
|
1663
|
+
const aspect = samples.map((s) => {
|
|
1664
|
+
const h = s.height;
|
|
1665
|
+
return h > 0 ? s.width / h : 0;
|
|
1666
|
+
});
|
|
1667
|
+
out.push(mean(aspect), variance(aspect));
|
|
1668
|
+
const area = samples.map((s) => s.width * s.height);
|
|
1669
|
+
const dArea = derivative2(area);
|
|
1670
|
+
out.push(mean(dArea), variance(dArea));
|
|
1671
|
+
const CURVATURE_REST_EPS = 1e-3;
|
|
1672
|
+
const curvatures = [];
|
|
1673
|
+
for (let i = 1; i < vx.length; i++) {
|
|
1674
|
+
const v1x = vx[i - 1] ?? 0;
|
|
1675
|
+
const v1y = vy[i - 1] ?? 0;
|
|
1676
|
+
const v2x = vx[i] ?? 0;
|
|
1677
|
+
const v2y = vy[i] ?? 0;
|
|
1678
|
+
if (Math.hypot(v1x, v1y) < CURVATURE_REST_EPS || Math.hypot(v2x, v2y) < CURVATURE_REST_EPS) {
|
|
1679
|
+
continue;
|
|
1680
|
+
}
|
|
1681
|
+
const a1 = Math.atan2(v1y, v1x);
|
|
1682
|
+
const a2 = Math.atan2(v2y, v2x);
|
|
1683
|
+
let d = a2 - a1;
|
|
1684
|
+
while (d > Math.PI) d -= 2 * Math.PI;
|
|
1685
|
+
while (d < -Math.PI) d += 2 * Math.PI;
|
|
1686
|
+
curvatures.push(Math.abs(d));
|
|
1687
|
+
}
|
|
1688
|
+
const curvStats = condense(curvatures);
|
|
1689
|
+
out.push(curvStats.mean, curvStats.variance, curvStats.skewness);
|
|
1690
|
+
const speed = vx.map((dx2, i) => {
|
|
1691
|
+
const dy2 = vy[i] ?? 0;
|
|
1692
|
+
return Math.sqrt(dx2 * dx2 + dy2 * dy2);
|
|
1693
|
+
});
|
|
1694
|
+
for (const lag of [1, 3, 5]) out.push(autocorrelation(speed, lag));
|
|
1695
|
+
const gaps = [];
|
|
1696
|
+
for (let i = 1; i < samples.length; i++) {
|
|
1697
|
+
gaps.push((samples[i]?.timestamp ?? 0) - (samples[i - 1]?.timestamp ?? 0));
|
|
1698
|
+
}
|
|
1699
|
+
out.push(...Object.values(condense(gaps)));
|
|
1700
|
+
const totalPath = speed.reduce((a, b) => a + b, 0);
|
|
1701
|
+
const dx = (samples[samples.length - 1]?.x ?? 0) - (samples[0]?.x ?? 0);
|
|
1702
|
+
const dy = (samples[samples.length - 1]?.y ?? 0) - (samples[0]?.y ?? 0);
|
|
1703
|
+
const straight = Math.sqrt(dx * dx + dy * dy);
|
|
1704
|
+
out.push(totalPath > 0 ? straight / totalPath : 0);
|
|
1705
|
+
const strokeLengths = perStrokePathLengths(speed);
|
|
1706
|
+
out.push(mean(strokeLengths), variance(strokeLengths));
|
|
1707
|
+
return out;
|
|
1708
|
+
}
|
|
1709
|
+
function perStrokePathLengths(speed) {
|
|
1710
|
+
const PAUSE_THRESHOLD = 0.5;
|
|
1711
|
+
const lengths = [];
|
|
1712
|
+
let acc = 0;
|
|
1713
|
+
let inStroke = false;
|
|
1714
|
+
for (const s of speed) {
|
|
1715
|
+
if (s >= PAUSE_THRESHOLD) {
|
|
1716
|
+
acc += s;
|
|
1717
|
+
inStroke = true;
|
|
1718
|
+
} else if (inStroke) {
|
|
1719
|
+
lengths.push(acc);
|
|
1720
|
+
acc = 0;
|
|
1721
|
+
inStroke = false;
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
if (inStroke && acc > 0) lengths.push(acc);
|
|
1725
|
+
return lengths;
|
|
1726
|
+
}
|
|
1037
1727
|
function derivative2(values) {
|
|
1038
1728
|
const d = [];
|
|
1039
1729
|
for (let i = 1; i < values.length; i++) {
|
|
@@ -1041,8 +1731,53 @@ function derivative2(values) {
|
|
|
1041
1731
|
}
|
|
1042
1732
|
return d;
|
|
1043
1733
|
}
|
|
1734
|
+
function meanCenter(values) {
|
|
1735
|
+
if (values.length === 0) return [];
|
|
1736
|
+
let sum = 0;
|
|
1737
|
+
for (const v of values) sum += v;
|
|
1738
|
+
const m = sum / values.length;
|
|
1739
|
+
return values.map((v) => v - m);
|
|
1740
|
+
}
|
|
1741
|
+
function covariance(a, b) {
|
|
1742
|
+
const n = Math.min(a.length, b.length);
|
|
1743
|
+
if (n < 2) return 0;
|
|
1744
|
+
let sumA = 0;
|
|
1745
|
+
let sumB = 0;
|
|
1746
|
+
for (let i = 0; i < n; i++) {
|
|
1747
|
+
sumA += a[i] ?? 0;
|
|
1748
|
+
sumB += b[i] ?? 0;
|
|
1749
|
+
}
|
|
1750
|
+
const meanA = sumA / n;
|
|
1751
|
+
const meanB = sumB / n;
|
|
1752
|
+
let cov = 0;
|
|
1753
|
+
for (let i = 0; i < n; i++) {
|
|
1754
|
+
cov += ((a[i] ?? 0) - meanA) * ((b[i] ?? 0) - meanB);
|
|
1755
|
+
}
|
|
1756
|
+
return cov / (n - 1);
|
|
1757
|
+
}
|
|
1758
|
+
function signChangeCount(values) {
|
|
1759
|
+
let count = 0;
|
|
1760
|
+
let last = 0;
|
|
1761
|
+
for (const v of values) {
|
|
1762
|
+
if (v > 0 && last < 0) count++;
|
|
1763
|
+
else if (v < 0 && last > 0) count++;
|
|
1764
|
+
if (v !== 0) last = v;
|
|
1765
|
+
}
|
|
1766
|
+
return count;
|
|
1767
|
+
}
|
|
1768
|
+
function sampleRateFromTimestamps(timestampsMs) {
|
|
1769
|
+
if (timestampsMs.length < 2) return 0;
|
|
1770
|
+
const span = (timestampsMs[timestampsMs.length - 1] ?? 0) - (timestampsMs[0] ?? 0);
|
|
1771
|
+
if (!Number.isFinite(span) || span <= 0) return 0;
|
|
1772
|
+
return (timestampsMs.length - 1) * 1e3 / span;
|
|
1773
|
+
}
|
|
1774
|
+
function captureDurationSec(samples) {
|
|
1775
|
+
if (samples.length < 2) return 0;
|
|
1776
|
+
const span = (samples[samples.length - 1]?.timestamp ?? 0) - (samples[0]?.timestamp ?? 0);
|
|
1777
|
+
return Number.isFinite(span) && span > 0 ? span / 1e3 : 0;
|
|
1778
|
+
}
|
|
1044
1779
|
function extractMouseDynamics(samples) {
|
|
1045
|
-
if (samples.length < 10) return new Array(
|
|
1780
|
+
if (samples.length < 10) return new Array(MOUSE_DYNAMICS_FEATURE_COUNT).fill(0);
|
|
1046
1781
|
const x = samples.map((s) => s.x);
|
|
1047
1782
|
const y = samples.map((s) => s.y);
|
|
1048
1783
|
const pressure = samples.map((s) => s.pressure);
|
|
@@ -1141,7 +1876,7 @@ function extractMouseDynamics(samples) {
|
|
|
1141
1876
|
const pressureStats = condense(pressure);
|
|
1142
1877
|
const moveDurStats = condense(movementDurations);
|
|
1143
1878
|
const segLenStats = condense(segmentLengths);
|
|
1144
|
-
|
|
1879
|
+
const legacyMouseDynamics = [
|
|
1145
1880
|
curvatureStats.mean,
|
|
1146
1881
|
curvatureStats.variance,
|
|
1147
1882
|
curvatureStats.skewness,
|
|
@@ -1197,6 +1932,8 @@ function extractMouseDynamics(samples) {
|
|
|
1197
1932
|
angleAutoCorr[2] ?? 0,
|
|
1198
1933
|
normalizedPathLength
|
|
1199
1934
|
];
|
|
1935
|
+
const padding = MOUSE_DYNAMICS_FEATURE_COUNT - legacyMouseDynamics.length;
|
|
1936
|
+
return padding > 0 ? [...legacyMouseDynamics, ...new Array(padding).fill(0)] : legacyMouseDynamics;
|
|
1200
1937
|
}
|
|
1201
1938
|
|
|
1202
1939
|
// src/hashing/simhash.ts
|
|
@@ -1236,7 +1973,7 @@ function getHyperplanes(dimension) {
|
|
|
1236
1973
|
cachedDimension = dimension;
|
|
1237
1974
|
return planes;
|
|
1238
1975
|
}
|
|
1239
|
-
var EXPECTED_FEATURE_DIMENSION =
|
|
1976
|
+
var EXPECTED_FEATURE_DIMENSION = SPEAKER_FEATURE_COUNT + MOTION_FEATURE_COUNT + TOUCH_FEATURE_COUNT;
|
|
1240
1977
|
function simhash(features) {
|
|
1241
1978
|
if (features.length === 0) {
|
|
1242
1979
|
return new Array(FINGERPRINT_BITS).fill(0);
|
|
@@ -4377,9 +5114,12 @@ async function extractFingerprintAndValidate(sensorData, config, walletAddress,
|
|
|
4377
5114
|
f0Contour,
|
|
4378
5115
|
accelMagnitude
|
|
4379
5116
|
} = await extractFeatures(sensorData);
|
|
5117
|
+
const AUDIO_END = SPEAKER_FEATURE_COUNT;
|
|
5118
|
+
const MOTION_END = AUDIO_END + MOTION_FEATURE_COUNT;
|
|
5119
|
+
const TOUCH_END = MOTION_END + TOUCH_FEATURE_COUNT;
|
|
4380
5120
|
const nonZero = features.filter((v) => v !== 0).length;
|
|
4381
5121
|
sdkLog(
|
|
4382
|
-
`[Entros SDK] Feature vector: ${features.length} dimensions, ${nonZero} non-zero. Audio[0
|
|
5122
|
+
`[Entros SDK] Feature vector: ${features.length} dimensions, ${nonZero} non-zero. Audio[0..${AUDIO_END - 1}]: ${features.slice(0, AUDIO_END).filter((v) => v !== 0).length} non-zero. Motion/Mouse[${AUDIO_END}..${MOTION_END - 1}]: ${features.slice(AUDIO_END, MOTION_END).filter((v) => v !== 0).length} non-zero. Touch[${MOTION_END}..${TOUCH_END - 1}]: ${features.slice(MOTION_END, TOUCH_END).filter((v) => v !== 0).length} non-zero.`
|
|
4383
5123
|
);
|
|
4384
5124
|
const fingerprint = simhash(normalizedFeatures);
|
|
4385
5125
|
const tbh = await generateTBH(fingerprint);
|
|
@@ -4583,9 +5323,12 @@ async function processSensorData(sensorData, config, wallet, connection, onProgr
|
|
|
4583
5323
|
);
|
|
4584
5324
|
solanaProof = serializeProof(proof, publicSignals);
|
|
4585
5325
|
} catch (proofErr) {
|
|
4586
|
-
const
|
|
4587
|
-
const
|
|
4588
|
-
const
|
|
5326
|
+
const motionStart = SPEAKER_FEATURE_COUNT;
|
|
5327
|
+
const touchStart = motionStart + MOTION_FEATURE_COUNT;
|
|
5328
|
+
const touchEnd = touchStart + TOUCH_FEATURE_COUNT;
|
|
5329
|
+
const audioNZ = features.slice(0, motionStart).filter((v) => v !== 0).length;
|
|
5330
|
+
const motionNZ = features.slice(motionStart, touchStart).filter((v) => v !== 0).length;
|
|
5331
|
+
const touchNZ = features.slice(touchStart, touchEnd).filter((v) => v !== 0).length;
|
|
4589
5332
|
const rawAudio = sensorData.audio?.samples.length ?? 0;
|
|
4590
5333
|
const rawMotion = sensorData.motion.length;
|
|
4591
5334
|
const rawTouch = sensorData.touch.length;
|