@entros/pulse-sdk 1.5.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.mts +34 -8
- package/dist/index.d.ts +34 -8
- package/dist/index.js +788 -45
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +788 -45
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -491,43 +491,445 @@ function findRoots(coefficients, maxIterations = 50) {
|
|
|
491
491
|
}
|
|
492
492
|
return roots;
|
|
493
493
|
}
|
|
494
|
-
function
|
|
494
|
+
function extractFrameAnalysis(frame, sampleRate, lpcOrder = 12) {
|
|
495
495
|
const r = autocorrelate(frame, lpcOrder);
|
|
496
496
|
const coeffs = levinsonDurbin(r, lpcOrder);
|
|
497
497
|
const roots = findRoots(coeffs);
|
|
498
|
-
const
|
|
498
|
+
const candidates = [];
|
|
499
499
|
for (const [real, imag] of roots) {
|
|
500
500
|
if (imag <= 0) continue;
|
|
501
501
|
const freq = Math.atan2(imag, real) / (2 * Math.PI) * sampleRate;
|
|
502
502
|
const bandwidth = -sampleRate / (2 * Math.PI) * Math.log(Math.sqrt(real * real + imag * imag));
|
|
503
503
|
if (freq > 200 && freq < 5e3 && bandwidth < 500) {
|
|
504
|
-
|
|
504
|
+
candidates.push({ freq, bandwidth });
|
|
505
505
|
}
|
|
506
506
|
}
|
|
507
|
-
|
|
508
|
-
if (
|
|
509
|
-
|
|
507
|
+
candidates.sort((a, b) => a.freq - b.freq);
|
|
508
|
+
if (candidates.length < 3) {
|
|
509
|
+
return { lpcCoefficients: coeffs, formants: null, bandwidths: null };
|
|
510
|
+
}
|
|
511
|
+
const formants = [
|
|
512
|
+
candidates[0].freq,
|
|
513
|
+
candidates[1].freq,
|
|
514
|
+
candidates[2].freq
|
|
515
|
+
];
|
|
516
|
+
const bandwidths = [
|
|
517
|
+
candidates[0].bandwidth,
|
|
518
|
+
candidates[1].bandwidth,
|
|
519
|
+
candidates[2].bandwidth
|
|
520
|
+
];
|
|
521
|
+
return { lpcCoefficients: coeffs, formants, bandwidths };
|
|
510
522
|
}
|
|
511
|
-
function
|
|
523
|
+
function extractLpcAnalysis(samples, sampleRate, frameSize, hopSize, lpcOrder = 12) {
|
|
524
|
+
const lpcCoefficients = Array.from({ length: lpcOrder }, () => []);
|
|
525
|
+
const f1 = [];
|
|
526
|
+
const f2 = [];
|
|
527
|
+
const f3 = [];
|
|
528
|
+
const b1 = [];
|
|
529
|
+
const b2 = [];
|
|
530
|
+
const b3 = [];
|
|
512
531
|
const f1f2 = [];
|
|
513
532
|
const f2f3 = [];
|
|
514
533
|
const numFrames = Math.floor((samples.length - frameSize) / hopSize) + 1;
|
|
534
|
+
let numFramesAnalyzed = 0;
|
|
535
|
+
if (numFrames < 1) {
|
|
536
|
+
return {
|
|
537
|
+
lpcCoefficients,
|
|
538
|
+
f1,
|
|
539
|
+
f2,
|
|
540
|
+
f3,
|
|
541
|
+
b1,
|
|
542
|
+
b2,
|
|
543
|
+
b3,
|
|
544
|
+
f1f2,
|
|
545
|
+
f2f3,
|
|
546
|
+
numFramesAnalyzed: 0
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
const windowed = new Float32Array(frameSize);
|
|
515
550
|
for (let i = 0; i < numFrames; i++) {
|
|
516
551
|
const start = i * hopSize;
|
|
517
552
|
const frame = samples.subarray(start, start + frameSize);
|
|
518
|
-
const windowed = new Float32Array(frameSize);
|
|
519
553
|
for (let j = 0; j < frameSize; j++) {
|
|
520
554
|
windowed[j] = (frame[j] ?? 0) * (0.54 - 0.46 * Math.cos(2 * Math.PI * j / (frameSize - 1)));
|
|
521
555
|
}
|
|
522
|
-
const
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
if (
|
|
556
|
+
const analysis = extractFrameAnalysis(windowed, sampleRate, lpcOrder);
|
|
557
|
+
numFramesAnalyzed++;
|
|
558
|
+
for (let c = 0; c < lpcOrder; c++) {
|
|
559
|
+
const coeff = analysis.lpcCoefficients[c];
|
|
560
|
+
if (Number.isFinite(coeff)) {
|
|
561
|
+
lpcCoefficients[c].push(coeff);
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
if (analysis.formants && analysis.bandwidths) {
|
|
565
|
+
const [F1, F2, F3] = analysis.formants;
|
|
566
|
+
const [B1, B2, B3] = analysis.bandwidths;
|
|
567
|
+
f1.push(F1);
|
|
568
|
+
f2.push(F2);
|
|
569
|
+
f3.push(F3);
|
|
570
|
+
b1.push(B1);
|
|
571
|
+
b2.push(B2);
|
|
572
|
+
b3.push(B3);
|
|
573
|
+
if (F2 > 0) f1f2.push(F1 / F2);
|
|
574
|
+
if (F3 > 0) f2f3.push(F2 / F3);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
return {
|
|
578
|
+
lpcCoefficients,
|
|
579
|
+
f1,
|
|
580
|
+
f2,
|
|
581
|
+
f3,
|
|
582
|
+
b1,
|
|
583
|
+
b2,
|
|
584
|
+
b3,
|
|
585
|
+
f1f2,
|
|
586
|
+
f2f3,
|
|
587
|
+
numFramesAnalyzed
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// src/extraction/mfcc.ts
|
|
592
|
+
var NUM_MFCC_COEFFICIENTS = 13;
|
|
593
|
+
var DELTA_REGRESSION_HALF_WIDTH = 2;
|
|
594
|
+
var MFCC_FEATURE_COUNT = NUM_MFCC_COEFFICIENTS * 4 + // mean, var, skew, kurt per coefficient
|
|
595
|
+
NUM_MFCC_COEFFICIENTS * 2;
|
|
596
|
+
function computeDelta(series, halfWidth) {
|
|
597
|
+
const n = series.length;
|
|
598
|
+
const out = new Array(n);
|
|
599
|
+
const fullDenom = halfWidth * (halfWidth + 1) * (2 * halfWidth + 1) / 3;
|
|
600
|
+
for (let t = 0; t < n; t++) {
|
|
601
|
+
let num = 0;
|
|
602
|
+
let denom = fullDenom;
|
|
603
|
+
for (let k = 1; k <= halfWidth; k++) {
|
|
604
|
+
const tPlus = t + k;
|
|
605
|
+
const tMinus = t - k;
|
|
606
|
+
if (tPlus >= n || tMinus < 0) {
|
|
607
|
+
denom -= 2 * k * k;
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
num += k * (series[tPlus] - series[tMinus]);
|
|
611
|
+
}
|
|
612
|
+
if (denom <= 0) {
|
|
613
|
+
out[t] = 0;
|
|
614
|
+
continue;
|
|
615
|
+
}
|
|
616
|
+
out[t] = num / denom;
|
|
617
|
+
}
|
|
618
|
+
return out;
|
|
619
|
+
}
|
|
620
|
+
var meydaModule = null;
|
|
621
|
+
async function getMeyda() {
|
|
622
|
+
if (!meydaModule) {
|
|
623
|
+
try {
|
|
624
|
+
meydaModule = await import("meyda");
|
|
625
|
+
} catch {
|
|
626
|
+
return null;
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
return meydaModule.default ?? meydaModule;
|
|
630
|
+
}
|
|
631
|
+
async function extractMfccFeatures(samples, sampleRate, frameSize, hopSize) {
|
|
632
|
+
if (!Number.isFinite(sampleRate) || sampleRate <= 0 || samples.length === 0 || frameSize <= 0 || hopSize <= 0) {
|
|
633
|
+
return new Array(MFCC_FEATURE_COUNT).fill(0);
|
|
634
|
+
}
|
|
635
|
+
const Meyda = await getMeyda();
|
|
636
|
+
if (!Meyda) {
|
|
637
|
+
sdkWarn("[Entros SDK] Meyda unavailable; MFCC features will be zeros.");
|
|
638
|
+
return new Array(MFCC_FEATURE_COUNT).fill(0);
|
|
639
|
+
}
|
|
640
|
+
const numFrames = Math.floor((samples.length - frameSize) / hopSize) + 1;
|
|
641
|
+
if (numFrames < 5) {
|
|
642
|
+
return new Array(MFCC_FEATURE_COUNT).fill(0);
|
|
643
|
+
}
|
|
644
|
+
const mfccTracks = Array.from(
|
|
645
|
+
{ length: NUM_MFCC_COEFFICIENTS },
|
|
646
|
+
() => []
|
|
647
|
+
);
|
|
648
|
+
const frame = new Float32Array(frameSize);
|
|
649
|
+
Meyda.bufferSize = frameSize;
|
|
650
|
+
Meyda.sampleRate = sampleRate;
|
|
651
|
+
for (let i = 0; i < numFrames; i++) {
|
|
652
|
+
const start = i * hopSize;
|
|
653
|
+
frame.set(samples.subarray(start, start + frameSize), 0);
|
|
654
|
+
const result = Meyda.extract("mfcc", frame);
|
|
655
|
+
if (!Array.isArray(result) || result.length !== NUM_MFCC_COEFFICIENTS) {
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
let allFinite = true;
|
|
659
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
660
|
+
if (!Number.isFinite(result[c])) {
|
|
661
|
+
allFinite = false;
|
|
662
|
+
break;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
if (!allFinite) continue;
|
|
666
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
667
|
+
mfccTracks[c].push(result[c]);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
const out = [];
|
|
671
|
+
out.length = MFCC_FEATURE_COUNT;
|
|
672
|
+
let writeIdx = 0;
|
|
673
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
674
|
+
const stats = condense(mfccTracks[c]);
|
|
675
|
+
out[writeIdx++] = stats.mean;
|
|
676
|
+
out[writeIdx++] = stats.variance;
|
|
677
|
+
out[writeIdx++] = stats.skewness;
|
|
678
|
+
out[writeIdx++] = stats.kurtosis;
|
|
679
|
+
}
|
|
680
|
+
for (let c = 0; c < NUM_MFCC_COEFFICIENTS; c++) {
|
|
681
|
+
const delta = computeDelta(mfccTracks[c], DELTA_REGRESSION_HALF_WIDTH);
|
|
682
|
+
const muDelta = mean(delta);
|
|
683
|
+
out[writeIdx++] = muDelta;
|
|
684
|
+
out[writeIdx++] = variance(delta, muDelta);
|
|
685
|
+
}
|
|
686
|
+
return out;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// src/extraction/voice-quality.ts
|
|
690
|
+
var VOICE_QUALITY_FEATURE_COUNT = 9;
|
|
691
|
+
var LOW_BAND_HZ = 1e3;
|
|
692
|
+
var MID_BAND_HZ = 3e3;
|
|
693
|
+
var HIGH_BAND_HZ = 8e3;
|
|
694
|
+
function cppQuefrencyRange(sampleRate) {
|
|
695
|
+
return {
|
|
696
|
+
qMin: Math.max(2, Math.floor(sampleRate / 400)),
|
|
697
|
+
qMax: Math.floor(sampleRate / 60)
|
|
698
|
+
};
|
|
699
|
+
}
|
|
700
|
+
var meydaModule2 = null;
|
|
701
|
+
async function getMeyda2() {
|
|
702
|
+
if (!meydaModule2) {
|
|
703
|
+
try {
|
|
704
|
+
meydaModule2 = await import("meyda");
|
|
705
|
+
} catch {
|
|
706
|
+
return null;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
return meydaModule2.default ?? meydaModule2;
|
|
710
|
+
}
|
|
711
|
+
function cepstralPeakProminence(powerSpectrum, sampleRate) {
|
|
712
|
+
const N = powerSpectrum.length;
|
|
713
|
+
if (N < 8) return 0;
|
|
714
|
+
const { qMin, qMax } = cppQuefrencyRange(sampleRate);
|
|
715
|
+
if (qMax >= N || qMax <= qMin) return 0;
|
|
716
|
+
const FLOOR = 1e-12;
|
|
717
|
+
const logPower = new Array(N);
|
|
718
|
+
for (let i = 0; i < N; i++) {
|
|
719
|
+
const p = Math.max(powerSpectrum[i], FLOOR);
|
|
720
|
+
const l = Math.log(p);
|
|
721
|
+
if (!Number.isFinite(l)) return 0;
|
|
722
|
+
logPower[i] = l;
|
|
723
|
+
}
|
|
724
|
+
const bandLen = qMax - qMin + 1;
|
|
725
|
+
const cepstrumBand = new Array(bandLen);
|
|
726
|
+
const piOverN = Math.PI / N;
|
|
727
|
+
for (let bIdx = 0; bIdx < bandLen; bIdx++) {
|
|
728
|
+
const k = qMin + bIdx;
|
|
729
|
+
let sum = 0;
|
|
730
|
+
for (let n = 0; n < N; n++) {
|
|
731
|
+
sum += logPower[n] * Math.cos(piOverN * (n + 0.5) * k);
|
|
732
|
+
}
|
|
733
|
+
cepstrumBand[bIdx] = sum;
|
|
734
|
+
}
|
|
735
|
+
let peakBIdx = 0;
|
|
736
|
+
let peakVal = cepstrumBand[0];
|
|
737
|
+
for (let bIdx = 1; bIdx < bandLen; bIdx++) {
|
|
738
|
+
if (cepstrumBand[bIdx] > peakVal) {
|
|
739
|
+
peakVal = cepstrumBand[bIdx];
|
|
740
|
+
peakBIdx = bIdx;
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
const peakQuefrency = qMin + peakBIdx;
|
|
744
|
+
const M = bandLen;
|
|
745
|
+
let sx = 0;
|
|
746
|
+
let sy = 0;
|
|
747
|
+
let sxx = 0;
|
|
748
|
+
let sxy = 0;
|
|
749
|
+
for (let bIdx = 0; bIdx < bandLen; bIdx++) {
|
|
750
|
+
const x = qMin + bIdx;
|
|
751
|
+
const y = cepstrumBand[bIdx];
|
|
752
|
+
sx += x;
|
|
753
|
+
sy += y;
|
|
754
|
+
sxx += x * x;
|
|
755
|
+
sxy += x * y;
|
|
756
|
+
}
|
|
757
|
+
const denom = M * sxx - sx * sx;
|
|
758
|
+
if (Math.abs(denom) < 1e-12) return 0;
|
|
759
|
+
const slope = (M * sxy - sx * sy) / denom;
|
|
760
|
+
const intercept = (sy - slope * sx) / M;
|
|
761
|
+
const baselineAtPeak = intercept + slope * peakQuefrency;
|
|
762
|
+
return peakVal - baselineAtPeak;
|
|
763
|
+
}
|
|
764
|
+
function spectralTilt(powerSpectrum, sampleRate) {
|
|
765
|
+
const N = powerSpectrum.length;
|
|
766
|
+
if (N < 8) return 0;
|
|
767
|
+
const FLOOR = 1e-12;
|
|
768
|
+
let sx = 0;
|
|
769
|
+
let sy = 0;
|
|
770
|
+
let sxx = 0;
|
|
771
|
+
let sxy = 0;
|
|
772
|
+
let count = 0;
|
|
773
|
+
const minBin = Math.max(1, Math.floor(100 * 2 * (N - 1) / sampleRate));
|
|
774
|
+
for (let k = minBin; k < N; k++) {
|
|
775
|
+
const p = powerSpectrum[k];
|
|
776
|
+
if (p < FLOOR) continue;
|
|
777
|
+
const x = Math.log(k);
|
|
778
|
+
const y = Math.log(p);
|
|
779
|
+
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
|
|
780
|
+
sx += x;
|
|
781
|
+
sy += y;
|
|
782
|
+
sxx += x * x;
|
|
783
|
+
sxy += x * y;
|
|
784
|
+
count++;
|
|
785
|
+
}
|
|
786
|
+
if (count < 4) return 0;
|
|
787
|
+
const denom = count * sxx - sx * sx;
|
|
788
|
+
if (Math.abs(denom) < 1e-12) return 0;
|
|
789
|
+
return (count * sxy - sx * sy) / denom;
|
|
790
|
+
}
|
|
791
|
+
function h1MinusH2(powerSpectrum, sampleRate, f0) {
|
|
792
|
+
if (!Number.isFinite(f0) || f0 <= 0) return 0;
|
|
793
|
+
const N = powerSpectrum.length;
|
|
794
|
+
if (N < 8) return 0;
|
|
795
|
+
const binPerHz = 2 * (N - 1) / sampleRate;
|
|
796
|
+
const k1 = Math.round(f0 * binPerHz);
|
|
797
|
+
const k2 = Math.round(2 * f0 * binPerHz);
|
|
798
|
+
const window2 = 2;
|
|
799
|
+
function peakNear(k) {
|
|
800
|
+
let best = -Infinity;
|
|
801
|
+
for (let i = k - window2; i <= k + window2; i++) {
|
|
802
|
+
if (i <= 0 || i >= N) continue;
|
|
803
|
+
const p = powerSpectrum[i];
|
|
804
|
+
if (p > best) best = p;
|
|
805
|
+
}
|
|
806
|
+
return best;
|
|
807
|
+
}
|
|
808
|
+
const h1 = peakNear(k1);
|
|
809
|
+
const h2 = peakNear(k2);
|
|
810
|
+
if (!Number.isFinite(h1) || !Number.isFinite(h2) || h1 <= 0 || h2 <= 0) return 0;
|
|
811
|
+
return 10 * Math.log10(h1 / h2);
|
|
812
|
+
}
|
|
813
|
+
function subbandRatios(powerSpectrum, sampleRate) {
|
|
814
|
+
const N = powerSpectrum.length;
|
|
815
|
+
if (N < 4) return [0, 0, 0];
|
|
816
|
+
const binPerHz = 2 * (N - 1) / sampleRate;
|
|
817
|
+
const lowBin = Math.min(N - 1, Math.round(LOW_BAND_HZ * binPerHz));
|
|
818
|
+
const midBin = Math.min(N - 1, Math.round(MID_BAND_HZ * binPerHz));
|
|
819
|
+
const highBin = Math.min(N - 1, Math.round(HIGH_BAND_HZ * binPerHz));
|
|
820
|
+
let total = 0;
|
|
821
|
+
let low = 0;
|
|
822
|
+
let mid = 0;
|
|
823
|
+
let high = 0;
|
|
824
|
+
for (let k = 1; k < N; k++) {
|
|
825
|
+
const p = powerSpectrum[k];
|
|
826
|
+
if (!Number.isFinite(p) || p < 0) continue;
|
|
827
|
+
total += p;
|
|
828
|
+
if (k <= lowBin) low += p;
|
|
829
|
+
else if (k <= midBin) mid += p;
|
|
830
|
+
else if (k <= highBin) high += p;
|
|
831
|
+
}
|
|
832
|
+
if (total < 1e-12) return [0, 0, 0];
|
|
833
|
+
return [low / total, mid / total, high / total];
|
|
834
|
+
}
|
|
835
|
+
async function extractVoiceQualityFeatures(samples, sampleRate, frameSize, hopSize, f0PerFrame) {
|
|
836
|
+
if (!Number.isFinite(sampleRate) || sampleRate <= 0 || samples.length === 0 || frameSize <= 0 || hopSize <= 0) {
|
|
837
|
+
return new Array(VOICE_QUALITY_FEATURE_COUNT).fill(0);
|
|
838
|
+
}
|
|
839
|
+
const Meyda = await getMeyda2();
|
|
840
|
+
if (!Meyda) {
|
|
841
|
+
sdkWarn("[Entros SDK] Meyda unavailable; voice quality features will be zeros.");
|
|
842
|
+
return new Array(VOICE_QUALITY_FEATURE_COUNT).fill(0);
|
|
843
|
+
}
|
|
844
|
+
const numFrames = Math.floor((samples.length - frameSize) / hopSize) + 1;
|
|
845
|
+
if (numFrames < 5) {
|
|
846
|
+
return new Array(VOICE_QUALITY_FEATURE_COUNT).fill(0);
|
|
847
|
+
}
|
|
848
|
+
const cppValues = [];
|
|
849
|
+
const tiltValues = [];
|
|
850
|
+
const h1h2Values = [];
|
|
851
|
+
const lowRatios = [];
|
|
852
|
+
const midRatios = [];
|
|
853
|
+
const highRatios = [];
|
|
854
|
+
const frame = new Float32Array(frameSize);
|
|
855
|
+
Meyda.bufferSize = frameSize;
|
|
856
|
+
Meyda.sampleRate = sampleRate;
|
|
857
|
+
for (let i = 0; i < numFrames; i++) {
|
|
858
|
+
const start = i * hopSize;
|
|
859
|
+
frame.set(samples.subarray(start, start + frameSize), 0);
|
|
860
|
+
const features = Meyda.extract("powerSpectrum", frame);
|
|
861
|
+
const power = features;
|
|
862
|
+
if (!power || power.length === 0) continue;
|
|
863
|
+
const cpp = cepstralPeakProminence(power, sampleRate);
|
|
864
|
+
if (Number.isFinite(cpp)) cppValues.push(cpp);
|
|
865
|
+
const tilt = spectralTilt(power, sampleRate);
|
|
866
|
+
if (Number.isFinite(tilt)) tiltValues.push(tilt);
|
|
867
|
+
const f0 = f0PerFrame[i] ?? 0;
|
|
868
|
+
if (f0 > 0) {
|
|
869
|
+
const h1h2 = h1MinusH2(power, sampleRate, f0);
|
|
870
|
+
if (Number.isFinite(h1h2)) h1h2Values.push(h1h2);
|
|
871
|
+
}
|
|
872
|
+
const [low, mid, high] = subbandRatios(power, sampleRate);
|
|
873
|
+
lowRatios.push(low);
|
|
874
|
+
midRatios.push(mid);
|
|
875
|
+
highRatios.push(high);
|
|
876
|
+
}
|
|
877
|
+
const cppMean = mean(cppValues);
|
|
878
|
+
const cppVar = variance(cppValues, cppMean);
|
|
879
|
+
const tiltMean = mean(tiltValues);
|
|
880
|
+
const tiltVar = variance(tiltValues, tiltMean);
|
|
881
|
+
const h1h2Mean = mean(h1h2Values);
|
|
882
|
+
const h1h2Var = variance(h1h2Values, h1h2Mean);
|
|
883
|
+
const lowMean = mean(lowRatios);
|
|
884
|
+
const midMean = mean(midRatios);
|
|
885
|
+
const highMean = mean(highRatios);
|
|
886
|
+
return [
|
|
887
|
+
cppMean,
|
|
888
|
+
cppVar,
|
|
889
|
+
tiltMean,
|
|
890
|
+
tiltVar,
|
|
891
|
+
h1h2Mean,
|
|
892
|
+
h1h2Var,
|
|
893
|
+
lowMean,
|
|
894
|
+
midMean,
|
|
895
|
+
highMean
|
|
896
|
+
];
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
// src/extraction/dct.ts
|
|
900
|
+
function dctII(input, numCoefficients) {
|
|
901
|
+
const N = input.length;
|
|
902
|
+
const K = Math.max(0, numCoefficients);
|
|
903
|
+
const output = new Array(K).fill(0);
|
|
904
|
+
if (N === 0 || K === 0) return output;
|
|
905
|
+
const upper = Math.min(K, N);
|
|
906
|
+
const piOverN = Math.PI / N;
|
|
907
|
+
for (let k = 0; k < upper; k++) {
|
|
908
|
+
let sum = 0;
|
|
909
|
+
for (let n = 0; n < N; n++) {
|
|
910
|
+
sum += input[n] * Math.cos(piOverN * (n + 0.5) * k);
|
|
527
911
|
}
|
|
912
|
+
output[k] = sum;
|
|
528
913
|
}
|
|
529
|
-
return
|
|
914
|
+
return output;
|
|
915
|
+
}
|
|
916
|
+
function pitchContourShape(contour, numCoefficients = 5) {
|
|
917
|
+
if (numCoefficients <= 0) return [];
|
|
918
|
+
const zero = () => new Array(numCoefficients).fill(0);
|
|
919
|
+
const voiced = [];
|
|
920
|
+
for (const v of contour) {
|
|
921
|
+
if (Number.isFinite(v) && v > 0) voiced.push(v);
|
|
922
|
+
}
|
|
923
|
+
if (voiced.length < numCoefficients * 2) return zero();
|
|
924
|
+
let sum = 0;
|
|
925
|
+
for (const v of voiced) sum += v;
|
|
926
|
+
const mu = sum / voiced.length;
|
|
927
|
+
const centered = voiced.map((v) => v - mu);
|
|
928
|
+
const N = centered.length;
|
|
929
|
+
const norm = 1 / Math.sqrt(N);
|
|
930
|
+
return dctII(centered, numCoefficients).map((c) => c * norm);
|
|
530
931
|
}
|
|
932
|
+
var PITCH_CONTOUR_SHAPE_FEATURE_COUNT = 5;
|
|
531
933
|
|
|
532
934
|
// src/yield.ts
|
|
533
935
|
function yieldToMainThread() {
|
|
@@ -560,10 +962,13 @@ function getFrameSize(sampleRate) {
|
|
|
560
962
|
function getHopSize(sampleRate) {
|
|
561
963
|
return Math.max(1, Math.round(sampleRate * 0.01));
|
|
562
964
|
}
|
|
563
|
-
var
|
|
965
|
+
var LEGACY_SPEAKER_FEATURE_COUNT = 44;
|
|
966
|
+
var LPC_COEFFICIENT_STATS = 12 * 2;
|
|
967
|
+
var FORMANT_TRAJECTORY_FEATURE_COUNT = 16;
|
|
968
|
+
var SPEAKER_FEATURE_COUNT = LEGACY_SPEAKER_FEATURE_COUNT + MFCC_FEATURE_COUNT + LPC_COEFFICIENT_STATS + FORMANT_TRAJECTORY_FEATURE_COUNT + VOICE_QUALITY_FEATURE_COUNT + PITCH_CONTOUR_SHAPE_FEATURE_COUNT;
|
|
564
969
|
var pitchDetector = null;
|
|
565
970
|
var pitchDetectorRate = 0;
|
|
566
|
-
var
|
|
971
|
+
var meydaModule3 = null;
|
|
567
972
|
async function getPitchDetector(sampleRate) {
|
|
568
973
|
if (!pitchDetector || pitchDetectorRate !== sampleRate) {
|
|
569
974
|
const PitchFinder = await import("pitchfinder");
|
|
@@ -572,15 +977,15 @@ async function getPitchDetector(sampleRate) {
|
|
|
572
977
|
}
|
|
573
978
|
return pitchDetector;
|
|
574
979
|
}
|
|
575
|
-
async function
|
|
576
|
-
if (!
|
|
980
|
+
async function getMeyda3() {
|
|
981
|
+
if (!meydaModule3) {
|
|
577
982
|
try {
|
|
578
|
-
|
|
983
|
+
meydaModule3 = await import("meyda");
|
|
579
984
|
} catch {
|
|
580
985
|
return null;
|
|
581
986
|
}
|
|
582
987
|
}
|
|
583
|
-
return
|
|
988
|
+
return meydaModule3.default ?? meydaModule3;
|
|
584
989
|
}
|
|
585
990
|
var F0_YIELD_EVERY_N_FRAMES = 16;
|
|
586
991
|
async function detectF0Contour(samples, sampleRate) {
|
|
@@ -710,8 +1115,10 @@ function computeHNR(samples, sampleRate, f0Contour) {
|
|
|
710
1115
|
async function computeLTAS(samples, sampleRate) {
|
|
711
1116
|
const frameSize = getFrameSize(sampleRate);
|
|
712
1117
|
const hopSize = getHopSize(sampleRate);
|
|
713
|
-
const Meyda = await
|
|
1118
|
+
const Meyda = await getMeyda3();
|
|
714
1119
|
if (!Meyda) return new Array(8).fill(0);
|
|
1120
|
+
Meyda.bufferSize = frameSize;
|
|
1121
|
+
Meyda.sampleRate = sampleRate;
|
|
715
1122
|
const centroids = [];
|
|
716
1123
|
const rolloffs = [];
|
|
717
1124
|
const flatnesses = [];
|
|
@@ -723,8 +1130,7 @@ async function computeLTAS(samples, sampleRate) {
|
|
|
723
1130
|
paddedFrame.set(samples.subarray(start, start + frameSize), 0);
|
|
724
1131
|
const features = Meyda.extract(
|
|
725
1132
|
["spectralCentroid", "spectralRolloff", "spectralFlatness", "spectralSpread"],
|
|
726
|
-
paddedFrame
|
|
727
|
-
{ sampleRate, bufferSize: frameSize }
|
|
1133
|
+
paddedFrame
|
|
728
1134
|
);
|
|
729
1135
|
if (features) {
|
|
730
1136
|
if (Number.isFinite(features.spectralCentroid)) centroids.push(features.spectralCentroid);
|
|
@@ -811,9 +1217,9 @@ async function extractSpeakerFeaturesDetailed(audio) {
|
|
|
811
1217
|
const hnrEntropy = entropy(hnrValues);
|
|
812
1218
|
const hnrFeatures = [hnrStats.mean, hnrStats.variance, hnrStats.skewness, hnrStats.kurtosis, hnrEntropy];
|
|
813
1219
|
await yieldToMainThread();
|
|
814
|
-
const
|
|
815
|
-
const f1f2Stats = condense(f1f2);
|
|
816
|
-
const f2f3Stats = condense(f2f3);
|
|
1220
|
+
const lpc = extractLpcAnalysis(normalizedSamples, sampleRate, frameSize, hopSize);
|
|
1221
|
+
const f1f2Stats = condense(lpc.f1f2);
|
|
1222
|
+
const f2f3Stats = condense(lpc.f2f3);
|
|
817
1223
|
const formantFeatures = [
|
|
818
1224
|
f1f2Stats.mean,
|
|
819
1225
|
f1f2Stats.variance,
|
|
@@ -830,25 +1236,86 @@ async function extractSpeakerFeaturesDetailed(audio) {
|
|
|
830
1236
|
const ampStats = condense(amplitudes);
|
|
831
1237
|
const ampEntropy = entropy(amplitudes);
|
|
832
1238
|
const ampFeatures = [ampStats.mean, ampStats.variance, ampStats.skewness, ampStats.kurtosis, ampEntropy];
|
|
1239
|
+
await yieldToMainThread();
|
|
1240
|
+
const mfccFeatures = await extractMfccFeatures(
|
|
1241
|
+
normalizedSamples,
|
|
1242
|
+
sampleRate,
|
|
1243
|
+
frameSize,
|
|
1244
|
+
hopSize
|
|
1245
|
+
);
|
|
1246
|
+
const lpcStats = [];
|
|
1247
|
+
for (let c = 0; c < 12; c++) {
|
|
1248
|
+
const track = lpc.lpcCoefficients[c] ?? [];
|
|
1249
|
+
const mu = mean(track);
|
|
1250
|
+
lpcStats.push(mu, variance(track, mu));
|
|
1251
|
+
}
|
|
1252
|
+
const f1Stats = { mean: mean(lpc.f1), var: variance(lpc.f1) };
|
|
1253
|
+
const f2Stats = { mean: mean(lpc.f2), var: variance(lpc.f2) };
|
|
1254
|
+
const f3Stats = { mean: mean(lpc.f3), var: variance(lpc.f3) };
|
|
1255
|
+
const f1Delta = derivative(lpc.f1);
|
|
1256
|
+
const f2Delta = derivative(lpc.f2);
|
|
1257
|
+
const f3Delta = derivative(lpc.f3);
|
|
1258
|
+
const f1DeltaMu = mean(f1Delta);
|
|
1259
|
+
const f2DeltaMu = mean(f2Delta);
|
|
1260
|
+
const f3DeltaMu = mean(f3Delta);
|
|
1261
|
+
const b1Mu = mean(lpc.b1);
|
|
1262
|
+
const b2Mu = mean(lpc.b2);
|
|
1263
|
+
const formantTrajectoryFeatures = [
|
|
1264
|
+
f1Stats.mean,
|
|
1265
|
+
f1Stats.var,
|
|
1266
|
+
f2Stats.mean,
|
|
1267
|
+
f2Stats.var,
|
|
1268
|
+
f3Stats.mean,
|
|
1269
|
+
f3Stats.var,
|
|
1270
|
+
f1DeltaMu,
|
|
1271
|
+
variance(f1Delta, f1DeltaMu),
|
|
1272
|
+
f2DeltaMu,
|
|
1273
|
+
variance(f2Delta, f2DeltaMu),
|
|
1274
|
+
f3DeltaMu,
|
|
1275
|
+
variance(f3Delta, f3DeltaMu),
|
|
1276
|
+
b1Mu,
|
|
1277
|
+
variance(lpc.b1, b1Mu),
|
|
1278
|
+
b2Mu,
|
|
1279
|
+
variance(lpc.b2, b2Mu)
|
|
1280
|
+
];
|
|
1281
|
+
await yieldToMainThread();
|
|
1282
|
+
const voiceQualityFeatures = await extractVoiceQualityFeatures(
|
|
1283
|
+
normalizedSamples,
|
|
1284
|
+
sampleRate,
|
|
1285
|
+
frameSize,
|
|
1286
|
+
hopSize,
|
|
1287
|
+
f0
|
|
1288
|
+
);
|
|
1289
|
+
const pitchShapeFeatures = pitchContourShape(f0, PITCH_CONTOUR_SHAPE_FEATURE_COUNT);
|
|
833
1290
|
const features = [
|
|
834
1291
|
...f0Features,
|
|
835
|
-
// 5
|
|
1292
|
+
// 5 [0..5] F0_STATS
|
|
836
1293
|
...f0DeltaFeatures,
|
|
837
|
-
// 4
|
|
1294
|
+
// 4 [5..9] F0_DELTA
|
|
838
1295
|
...jitterFeatures,
|
|
839
|
-
// 4
|
|
1296
|
+
// 4 [9..13] JITTER
|
|
840
1297
|
...shimmerFeatures,
|
|
841
|
-
// 4
|
|
1298
|
+
// 4 [13..17] SHIMMER
|
|
842
1299
|
...hnrFeatures,
|
|
843
|
-
// 5
|
|
1300
|
+
// 5 [17..22] HNR
|
|
844
1301
|
...formantFeatures,
|
|
845
|
-
// 8
|
|
1302
|
+
// 8 [22..30] FORMANT_RATIOS
|
|
846
1303
|
...ltasFeatures,
|
|
847
|
-
// 8
|
|
1304
|
+
// 8 [30..38] LTAS
|
|
848
1305
|
...voicingFeatures,
|
|
849
|
-
// 1
|
|
850
|
-
...ampFeatures
|
|
851
|
-
// 5
|
|
1306
|
+
// 1 [38] VOICING_RATIO
|
|
1307
|
+
...ampFeatures,
|
|
1308
|
+
// 5 [39..44] AMPLITUDE
|
|
1309
|
+
...mfccFeatures,
|
|
1310
|
+
// 78 [44..122] MFCC + delta-MFCC
|
|
1311
|
+
...lpcStats,
|
|
1312
|
+
// 24 [122..146] LPC coefficient stats
|
|
1313
|
+
...formantTrajectoryFeatures,
|
|
1314
|
+
// 16 [146..162] Formant absolutes + dynamics + bandwidths
|
|
1315
|
+
...voiceQualityFeatures,
|
|
1316
|
+
// 9 [162..171] Voice quality
|
|
1317
|
+
...pitchShapeFeatures
|
|
1318
|
+
// 5 [171..176] Pitch contour shape DCT
|
|
852
1319
|
];
|
|
853
1320
|
return { features, f0Contour: f0 };
|
|
854
1321
|
}
|
|
@@ -857,7 +1324,102 @@ async function extractSpeakerFeatures(audio) {
|
|
|
857
1324
|
return features;
|
|
858
1325
|
}
|
|
859
1326
|
|
|
1327
|
+
// src/extraction/fft.ts
|
|
1328
|
+
function nextPow2(n) {
|
|
1329
|
+
if (n <= 2) return 2;
|
|
1330
|
+
let p = 2;
|
|
1331
|
+
while (p < n) p <<= 1;
|
|
1332
|
+
return p;
|
|
1333
|
+
}
|
|
1334
|
+
function realFFT(input, size) {
|
|
1335
|
+
if (size <= 0 || (size & size - 1) !== 0) {
|
|
1336
|
+
throw new Error(`FFT size must be a positive power of two, got ${size}`);
|
|
1337
|
+
}
|
|
1338
|
+
const real = new Array(size);
|
|
1339
|
+
const imag = new Array(size).fill(0);
|
|
1340
|
+
for (let i = 0; i < size; i++) {
|
|
1341
|
+
real[i] = i < input.length ? input[i] ?? 0 : 0;
|
|
1342
|
+
}
|
|
1343
|
+
for (let i = 1, j = 0; i < size; i++) {
|
|
1344
|
+
let bit = size >> 1;
|
|
1345
|
+
for (; j & bit; bit >>= 1) j ^= bit;
|
|
1346
|
+
j ^= bit;
|
|
1347
|
+
if (i < j) {
|
|
1348
|
+
const tr = real[i];
|
|
1349
|
+
real[i] = real[j];
|
|
1350
|
+
real[j] = tr;
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
for (let halfSize = 1; halfSize < size; halfSize <<= 1) {
|
|
1354
|
+
const fullSize = halfSize << 1;
|
|
1355
|
+
const phaseStep = -Math.PI / halfSize;
|
|
1356
|
+
for (let chunkStart = 0; chunkStart < size; chunkStart += fullSize) {
|
|
1357
|
+
for (let k = 0; k < halfSize; k++) {
|
|
1358
|
+
const phase = phaseStep * k;
|
|
1359
|
+
const wr = Math.cos(phase);
|
|
1360
|
+
const wi = Math.sin(phase);
|
|
1361
|
+
const ar = real[chunkStart + k];
|
|
1362
|
+
const ai = imag[chunkStart + k];
|
|
1363
|
+
const br = real[chunkStart + k + halfSize];
|
|
1364
|
+
const bi = imag[chunkStart + k + halfSize];
|
|
1365
|
+
const tr = wr * br - wi * bi;
|
|
1366
|
+
const ti = wr * bi + wi * br;
|
|
1367
|
+
real[chunkStart + k] = ar + tr;
|
|
1368
|
+
imag[chunkStart + k] = ai + ti;
|
|
1369
|
+
real[chunkStart + k + halfSize] = ar - tr;
|
|
1370
|
+
imag[chunkStart + k + halfSize] = ai - ti;
|
|
1371
|
+
}
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
return { real, imag };
|
|
1375
|
+
}
|
|
1376
|
+
function bandEnergy(real, imag, sampleRate, fLow, fHigh) {
|
|
1377
|
+
const N = real.length;
|
|
1378
|
+
if (N === 0 || !Number.isFinite(sampleRate) || sampleRate <= 0 || fLow >= fHigh || fLow < 0) {
|
|
1379
|
+
return 0;
|
|
1380
|
+
}
|
|
1381
|
+
const binHz = sampleRate / N;
|
|
1382
|
+
const kLow = Math.max(0, Math.ceil(fLow / binHz));
|
|
1383
|
+
const kHigh = Math.min(Math.floor(N / 2), Math.floor((fHigh - 1e-9) / binHz));
|
|
1384
|
+
let energy = 0;
|
|
1385
|
+
for (let k = kLow; k <= kHigh; k++) {
|
|
1386
|
+
const re = real[k] ?? 0;
|
|
1387
|
+
const im = imag[k] ?? 0;
|
|
1388
|
+
energy += re * re + im * im;
|
|
1389
|
+
}
|
|
1390
|
+
return energy / (N * N);
|
|
1391
|
+
}
|
|
1392
|
+
function peakInBand(real, imag, sampleRate, fLow, fHigh) {
|
|
1393
|
+
const N = real.length;
|
|
1394
|
+
if (N === 0 || !Number.isFinite(sampleRate) || sampleRate <= 0 || fLow >= fHigh || fLow < 0) {
|
|
1395
|
+
return { freq: 0, amplitude: 0 };
|
|
1396
|
+
}
|
|
1397
|
+
const binHz = sampleRate / N;
|
|
1398
|
+
const kLow = Math.max(0, Math.ceil(fLow / binHz));
|
|
1399
|
+
const kHigh = Math.min(Math.floor(N / 2), Math.floor((fHigh - 1e-9) / binHz));
|
|
1400
|
+
let bestK = -1;
|
|
1401
|
+
let bestAmp = -Infinity;
|
|
1402
|
+
for (let k = kLow; k <= kHigh; k++) {
|
|
1403
|
+
const re = real[k] ?? 0;
|
|
1404
|
+
const im = imag[k] ?? 0;
|
|
1405
|
+
const amp = re * re + im * im;
|
|
1406
|
+
if (amp > bestAmp) {
|
|
1407
|
+
bestAmp = amp;
|
|
1408
|
+
bestK = k;
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
if (bestK < 0) return { freq: 0, amplitude: 0 };
|
|
1412
|
+
return { freq: bestK * binHz, amplitude: bestAmp / (N * N) };
|
|
1413
|
+
}
|
|
1414
|
+
|
|
860
1415
|
// src/extraction/kinematic.ts
|
|
1416
|
+
var MOTION_LEGACY_COUNT = 54;
|
|
1417
|
+
var MOTION_V2_ADDITIONS = 27;
|
|
1418
|
+
var MOTION_FEATURE_COUNT = MOTION_LEGACY_COUNT + MOTION_V2_ADDITIONS;
|
|
1419
|
+
var TOUCH_LEGACY_COUNT = 36;
|
|
1420
|
+
var TOUCH_V2_ADDITIONS = 21;
|
|
1421
|
+
var TOUCH_FEATURE_COUNT = TOUCH_LEGACY_COUNT + TOUCH_V2_ADDITIONS;
|
|
1422
|
+
var MOUSE_DYNAMICS_FEATURE_COUNT = MOTION_FEATURE_COUNT;
|
|
861
1423
|
function extractAccelerationMagnitude(samples, targetFrameCount) {
|
|
862
1424
|
if (samples.length < 2 || targetFrameCount < 2) return [];
|
|
863
1425
|
const magnitudes = samples.map((s) => Math.sqrt(s.ax * s.ax + s.ay * s.ay + s.az * s.az));
|
|
@@ -875,7 +1437,7 @@ function extractAccelerationMagnitude(samples, targetFrameCount) {
|
|
|
875
1437
|
return out;
|
|
876
1438
|
}
|
|
877
1439
|
function extractMotionFeatures(samples) {
|
|
878
|
-
if (samples.length < 5) return new Array(
|
|
1440
|
+
if (samples.length < 5) return new Array(MOTION_FEATURE_COUNT).fill(0);
|
|
879
1441
|
const axes = {
|
|
880
1442
|
ax: samples.map((s) => s.ax),
|
|
881
1443
|
ay: samples.map((s) => s.ay),
|
|
@@ -910,10 +1472,68 @@ function extractMotionFeatures(samples) {
|
|
|
910
1472
|
}
|
|
911
1473
|
features.push(windowVariances.length >= 2 ? variance(windowVariances) : 0);
|
|
912
1474
|
}
|
|
1475
|
+
features.push(...computeMotionV2(axes, samples));
|
|
913
1476
|
return features;
|
|
914
1477
|
}
|
|
1478
|
+
function computeMotionV2(axes, samples) {
|
|
1479
|
+
const out = [];
|
|
1480
|
+
const covPairs = [
|
|
1481
|
+
[axes.ax, axes.gy],
|
|
1482
|
+
[axes.ay, axes.gx],
|
|
1483
|
+
[axes.az, axes.gz],
|
|
1484
|
+
[axes.ax, axes.az],
|
|
1485
|
+
[axes.ay, axes.az],
|
|
1486
|
+
[axes.gx, axes.gy]
|
|
1487
|
+
];
|
|
1488
|
+
for (const [a, b] of covPairs) out.push(covariance(a, b));
|
|
1489
|
+
const sampleRate = sampleRateFromTimestamps(samples.map((s) => s.timestamp));
|
|
1490
|
+
const fftSize = nextPow2(Math.max(64, axes.ax.length));
|
|
1491
|
+
const bands = [
|
|
1492
|
+
[0, 2],
|
|
1493
|
+
[2, 6],
|
|
1494
|
+
[6, 12],
|
|
1495
|
+
[12, 30]
|
|
1496
|
+
];
|
|
1497
|
+
const accelSpectra = [axes.ax, axes.ay, axes.az].map(
|
|
1498
|
+
(axis) => realFFT(meanCenter(axis), fftSize)
|
|
1499
|
+
);
|
|
1500
|
+
for (const spectrum of accelSpectra) {
|
|
1501
|
+
for (const [lo, hi] of bands) {
|
|
1502
|
+
out.push(bandEnergy(spectrum.real, spectrum.imag, sampleRate, lo, hi));
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
const magnitude = samples.map(
|
|
1506
|
+
(s) => Math.sqrt(s.ax * s.ax + s.ay * s.ay + s.az * s.az)
|
|
1507
|
+
);
|
|
1508
|
+
const magSpectrum = realFFT(meanCenter(magnitude), fftSize);
|
|
1509
|
+
const tremor = peakInBand(
|
|
1510
|
+
magSpectrum.real,
|
|
1511
|
+
magSpectrum.imag,
|
|
1512
|
+
sampleRate,
|
|
1513
|
+
4,
|
|
1514
|
+
12
|
|
1515
|
+
);
|
|
1516
|
+
out.push(tremor.freq, tremor.amplitude);
|
|
1517
|
+
const duration = captureDurationSec(samples);
|
|
1518
|
+
const reversalRates = [axes.ax, axes.ay, axes.az].map(
|
|
1519
|
+
(axis) => duration > 0 ? signChangeCount(derivative2(axis)) / duration : 0
|
|
1520
|
+
);
|
|
1521
|
+
out.push(mean(reversalRates), variance(reversalRates));
|
|
1522
|
+
let gyroSum = 0;
|
|
1523
|
+
for (let i = 0; i < samples.length; i++) {
|
|
1524
|
+
const gx = samples[i].gx;
|
|
1525
|
+
const gy = samples[i].gy;
|
|
1526
|
+
const gz = samples[i].gz;
|
|
1527
|
+
gyroSum += Math.sqrt(gx * gx + gy * gy + gz * gz);
|
|
1528
|
+
}
|
|
1529
|
+
out.push(samples.length > 0 ? gyroSum / samples.length : 0);
|
|
1530
|
+
for (const lag of [1, 5, 10, 25]) {
|
|
1531
|
+
out.push(autocorrelation(magnitude, lag));
|
|
1532
|
+
}
|
|
1533
|
+
return out;
|
|
1534
|
+
}
|
|
915
1535
|
function extractTouchFeatures(samples) {
|
|
916
|
-
if (samples.length < 5) return new Array(
|
|
1536
|
+
if (samples.length < 5) return new Array(TOUCH_FEATURE_COUNT).fill(0);
|
|
917
1537
|
const x = samples.map((s) => s.x);
|
|
918
1538
|
const y = samples.map((s) => s.y);
|
|
919
1539
|
const pressure = samples.map((s) => s.pressure);
|
|
@@ -941,8 +1561,78 @@ function extractTouchFeatures(samples) {
|
|
|
941
1561
|
}
|
|
942
1562
|
features.push(windowVariances.length >= 2 ? variance(windowVariances) : 0);
|
|
943
1563
|
}
|
|
1564
|
+
features.push(...computeTouchV2(samples, vx, vy));
|
|
944
1565
|
return features;
|
|
945
1566
|
}
|
|
1567
|
+
function computeTouchV2(samples, vx, vy) {
|
|
1568
|
+
const out = [];
|
|
1569
|
+
const pressure = samples.map((s) => s.pressure);
|
|
1570
|
+
const dPressure = derivative2(pressure);
|
|
1571
|
+
out.push(...Object.values(condense(dPressure)));
|
|
1572
|
+
const aspect = samples.map((s) => {
|
|
1573
|
+
const h = s.height;
|
|
1574
|
+
return h > 0 ? s.width / h : 0;
|
|
1575
|
+
});
|
|
1576
|
+
out.push(mean(aspect), variance(aspect));
|
|
1577
|
+
const area = samples.map((s) => s.width * s.height);
|
|
1578
|
+
const dArea = derivative2(area);
|
|
1579
|
+
out.push(mean(dArea), variance(dArea));
|
|
1580
|
+
const CURVATURE_REST_EPS = 1e-3;
|
|
1581
|
+
const curvatures = [];
|
|
1582
|
+
for (let i = 1; i < vx.length; i++) {
|
|
1583
|
+
const v1x = vx[i - 1] ?? 0;
|
|
1584
|
+
const v1y = vy[i - 1] ?? 0;
|
|
1585
|
+
const v2x = vx[i] ?? 0;
|
|
1586
|
+
const v2y = vy[i] ?? 0;
|
|
1587
|
+
if (Math.hypot(v1x, v1y) < CURVATURE_REST_EPS || Math.hypot(v2x, v2y) < CURVATURE_REST_EPS) {
|
|
1588
|
+
continue;
|
|
1589
|
+
}
|
|
1590
|
+
const a1 = Math.atan2(v1y, v1x);
|
|
1591
|
+
const a2 = Math.atan2(v2y, v2x);
|
|
1592
|
+
let d = a2 - a1;
|
|
1593
|
+
while (d > Math.PI) d -= 2 * Math.PI;
|
|
1594
|
+
while (d < -Math.PI) d += 2 * Math.PI;
|
|
1595
|
+
curvatures.push(Math.abs(d));
|
|
1596
|
+
}
|
|
1597
|
+
const curvStats = condense(curvatures);
|
|
1598
|
+
out.push(curvStats.mean, curvStats.variance, curvStats.skewness);
|
|
1599
|
+
const speed = vx.map((dx2, i) => {
|
|
1600
|
+
const dy2 = vy[i] ?? 0;
|
|
1601
|
+
return Math.sqrt(dx2 * dx2 + dy2 * dy2);
|
|
1602
|
+
});
|
|
1603
|
+
for (const lag of [1, 3, 5]) out.push(autocorrelation(speed, lag));
|
|
1604
|
+
const gaps = [];
|
|
1605
|
+
for (let i = 1; i < samples.length; i++) {
|
|
1606
|
+
gaps.push((samples[i]?.timestamp ?? 0) - (samples[i - 1]?.timestamp ?? 0));
|
|
1607
|
+
}
|
|
1608
|
+
out.push(...Object.values(condense(gaps)));
|
|
1609
|
+
const totalPath = speed.reduce((a, b) => a + b, 0);
|
|
1610
|
+
const dx = (samples[samples.length - 1]?.x ?? 0) - (samples[0]?.x ?? 0);
|
|
1611
|
+
const dy = (samples[samples.length - 1]?.y ?? 0) - (samples[0]?.y ?? 0);
|
|
1612
|
+
const straight = Math.sqrt(dx * dx + dy * dy);
|
|
1613
|
+
out.push(totalPath > 0 ? straight / totalPath : 0);
|
|
1614
|
+
const strokeLengths = perStrokePathLengths(speed);
|
|
1615
|
+
out.push(mean(strokeLengths), variance(strokeLengths));
|
|
1616
|
+
return out;
|
|
1617
|
+
}
|
|
1618
|
+
function perStrokePathLengths(speed) {
|
|
1619
|
+
const PAUSE_THRESHOLD = 0.5;
|
|
1620
|
+
const lengths = [];
|
|
1621
|
+
let acc = 0;
|
|
1622
|
+
let inStroke = false;
|
|
1623
|
+
for (const s of speed) {
|
|
1624
|
+
if (s >= PAUSE_THRESHOLD) {
|
|
1625
|
+
acc += s;
|
|
1626
|
+
inStroke = true;
|
|
1627
|
+
} else if (inStroke) {
|
|
1628
|
+
lengths.push(acc);
|
|
1629
|
+
acc = 0;
|
|
1630
|
+
inStroke = false;
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
if (inStroke && acc > 0) lengths.push(acc);
|
|
1634
|
+
return lengths;
|
|
1635
|
+
}
|
|
946
1636
|
function derivative2(values) {
|
|
947
1637
|
const d = [];
|
|
948
1638
|
for (let i = 1; i < values.length; i++) {
|
|
@@ -950,8 +1640,53 @@ function derivative2(values) {
|
|
|
950
1640
|
}
|
|
951
1641
|
return d;
|
|
952
1642
|
}
|
|
1643
|
+
function meanCenter(values) {
|
|
1644
|
+
if (values.length === 0) return [];
|
|
1645
|
+
let sum = 0;
|
|
1646
|
+
for (const v of values) sum += v;
|
|
1647
|
+
const m = sum / values.length;
|
|
1648
|
+
return values.map((v) => v - m);
|
|
1649
|
+
}
|
|
1650
|
+
function covariance(a, b) {
|
|
1651
|
+
const n = Math.min(a.length, b.length);
|
|
1652
|
+
if (n < 2) return 0;
|
|
1653
|
+
let sumA = 0;
|
|
1654
|
+
let sumB = 0;
|
|
1655
|
+
for (let i = 0; i < n; i++) {
|
|
1656
|
+
sumA += a[i] ?? 0;
|
|
1657
|
+
sumB += b[i] ?? 0;
|
|
1658
|
+
}
|
|
1659
|
+
const meanA = sumA / n;
|
|
1660
|
+
const meanB = sumB / n;
|
|
1661
|
+
let cov = 0;
|
|
1662
|
+
for (let i = 0; i < n; i++) {
|
|
1663
|
+
cov += ((a[i] ?? 0) - meanA) * ((b[i] ?? 0) - meanB);
|
|
1664
|
+
}
|
|
1665
|
+
return cov / (n - 1);
|
|
1666
|
+
}
|
|
1667
|
+
function signChangeCount(values) {
|
|
1668
|
+
let count = 0;
|
|
1669
|
+
let last = 0;
|
|
1670
|
+
for (const v of values) {
|
|
1671
|
+
if (v > 0 && last < 0) count++;
|
|
1672
|
+
else if (v < 0 && last > 0) count++;
|
|
1673
|
+
if (v !== 0) last = v;
|
|
1674
|
+
}
|
|
1675
|
+
return count;
|
|
1676
|
+
}
|
|
1677
|
+
function sampleRateFromTimestamps(timestampsMs) {
|
|
1678
|
+
if (timestampsMs.length < 2) return 0;
|
|
1679
|
+
const span = (timestampsMs[timestampsMs.length - 1] ?? 0) - (timestampsMs[0] ?? 0);
|
|
1680
|
+
if (!Number.isFinite(span) || span <= 0) return 0;
|
|
1681
|
+
return (timestampsMs.length - 1) * 1e3 / span;
|
|
1682
|
+
}
|
|
1683
|
+
function captureDurationSec(samples) {
|
|
1684
|
+
if (samples.length < 2) return 0;
|
|
1685
|
+
const span = (samples[samples.length - 1]?.timestamp ?? 0) - (samples[0]?.timestamp ?? 0);
|
|
1686
|
+
return Number.isFinite(span) && span > 0 ? span / 1e3 : 0;
|
|
1687
|
+
}
|
|
953
1688
|
function extractMouseDynamics(samples) {
|
|
954
|
-
if (samples.length < 10) return new Array(
|
|
1689
|
+
if (samples.length < 10) return new Array(MOUSE_DYNAMICS_FEATURE_COUNT).fill(0);
|
|
955
1690
|
const x = samples.map((s) => s.x);
|
|
956
1691
|
const y = samples.map((s) => s.y);
|
|
957
1692
|
const pressure = samples.map((s) => s.pressure);
|
|
@@ -1050,7 +1785,7 @@ function extractMouseDynamics(samples) {
|
|
|
1050
1785
|
const pressureStats = condense(pressure);
|
|
1051
1786
|
const moveDurStats = condense(movementDurations);
|
|
1052
1787
|
const segLenStats = condense(segmentLengths);
|
|
1053
|
-
|
|
1788
|
+
const legacyMouseDynamics = [
|
|
1054
1789
|
curvatureStats.mean,
|
|
1055
1790
|
curvatureStats.variance,
|
|
1056
1791
|
curvatureStats.skewness,
|
|
@@ -1106,6 +1841,8 @@ function extractMouseDynamics(samples) {
|
|
|
1106
1841
|
angleAutoCorr[2] ?? 0,
|
|
1107
1842
|
normalizedPathLength
|
|
1108
1843
|
];
|
|
1844
|
+
const padding = MOUSE_DYNAMICS_FEATURE_COUNT - legacyMouseDynamics.length;
|
|
1845
|
+
return padding > 0 ? [...legacyMouseDynamics, ...new Array(padding).fill(0)] : legacyMouseDynamics;
|
|
1109
1846
|
}
|
|
1110
1847
|
|
|
1111
1848
|
// src/hashing/simhash.ts
|
|
@@ -1145,7 +1882,7 @@ function getHyperplanes(dimension) {
|
|
|
1145
1882
|
cachedDimension = dimension;
|
|
1146
1883
|
return planes;
|
|
1147
1884
|
}
|
|
1148
|
-
var EXPECTED_FEATURE_DIMENSION =
|
|
1885
|
+
var EXPECTED_FEATURE_DIMENSION = SPEAKER_FEATURE_COUNT + MOTION_FEATURE_COUNT + TOUCH_FEATURE_COUNT;
|
|
1149
1886
|
function simhash(features) {
|
|
1150
1887
|
if (features.length === 0) {
|
|
1151
1888
|
return new Array(FINGERPRINT_BITS).fill(0);
|
|
@@ -4286,9 +5023,12 @@ async function extractFingerprintAndValidate(sensorData, config, walletAddress,
|
|
|
4286
5023
|
f0Contour,
|
|
4287
5024
|
accelMagnitude
|
|
4288
5025
|
} = await extractFeatures(sensorData);
|
|
5026
|
+
const AUDIO_END = SPEAKER_FEATURE_COUNT;
|
|
5027
|
+
const MOTION_END = AUDIO_END + MOTION_FEATURE_COUNT;
|
|
5028
|
+
const TOUCH_END = MOTION_END + TOUCH_FEATURE_COUNT;
|
|
4289
5029
|
const nonZero = features.filter((v) => v !== 0).length;
|
|
4290
5030
|
sdkLog(
|
|
4291
|
-
`[Entros SDK] Feature vector: ${features.length} dimensions, ${nonZero} non-zero. Audio[0
|
|
5031
|
+
`[Entros SDK] Feature vector: ${features.length} dimensions, ${nonZero} non-zero. Audio[0..${AUDIO_END - 1}]: ${features.slice(0, AUDIO_END).filter((v) => v !== 0).length} non-zero. Motion/Mouse[${AUDIO_END}..${MOTION_END - 1}]: ${features.slice(AUDIO_END, MOTION_END).filter((v) => v !== 0).length} non-zero. Touch[${MOTION_END}..${TOUCH_END - 1}]: ${features.slice(MOTION_END, TOUCH_END).filter((v) => v !== 0).length} non-zero.`
|
|
4292
5032
|
);
|
|
4293
5033
|
const fingerprint = simhash(normalizedFeatures);
|
|
4294
5034
|
const tbh = await generateTBH(fingerprint);
|
|
@@ -4492,9 +5232,12 @@ async function processSensorData(sensorData, config, wallet, connection, onProgr
|
|
|
4492
5232
|
);
|
|
4493
5233
|
solanaProof = serializeProof(proof, publicSignals);
|
|
4494
5234
|
} catch (proofErr) {
|
|
4495
|
-
const
|
|
4496
|
-
const
|
|
4497
|
-
const
|
|
5235
|
+
const motionStart = SPEAKER_FEATURE_COUNT;
|
|
5236
|
+
const touchStart = motionStart + MOTION_FEATURE_COUNT;
|
|
5237
|
+
const touchEnd = touchStart + TOUCH_FEATURE_COUNT;
|
|
5238
|
+
const audioNZ = features.slice(0, motionStart).filter((v) => v !== 0).length;
|
|
5239
|
+
const motionNZ = features.slice(motionStart, touchStart).filter((v) => v !== 0).length;
|
|
5240
|
+
const touchNZ = features.slice(touchStart, touchEnd).filter((v) => v !== 0).length;
|
|
4498
5241
|
const rawAudio = sensorData.audio?.samples.length ?? 0;
|
|
4499
5242
|
const rawMotion = sensorData.motion.length;
|
|
4500
5243
|
const rawTouch = sensorData.touch.length;
|