echogarden 1.8.2 → 1.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/alignment/DTWMfccSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWMfccSequenceAlignment.js +6 -5
- package/dist/alignment/DTWMfccSequenceAlignment.js.map +1 -1
- package/dist/alignment/DTWSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignment.js.map +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.js.map +1 -1
- package/dist/alignment/SpeechAlignment.d.ts +2 -0
- package/dist/alignment/SpeechAlignment.js +1 -1
- package/dist/alignment/SpeechAlignment.js.map +1 -1
- package/dist/api/SpeechSearch.d.ts +6 -0
- package/dist/api/SpeechSearch.js +4 -0
- package/dist/api/SpeechSearch.js.map +1 -0
- package/dist/dsp/MFCC.d.ts +5 -6
- package/dist/dsp/MFCC.js +4 -19
- package/dist/dsp/MFCC.js.map +1 -1
- package/dist/math/MedianFilter.d.ts +2 -2
- package/dist/math/MedianFilter.js +3 -4
- package/dist/math/MedianFilter.js.map +1 -1
- package/dist/math/VectorMath.d.ts +34 -34
- package/dist/math/VectorMath.js +63 -44
- package/dist/math/VectorMath.js.map +1 -1
- package/dist/nlp/IPA.d.ts +3 -3
- package/dist/recognition/WhisperSTT.d.ts +1 -0
- package/dist/recognition/WhisperSTT.js +36 -20
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/speech-search/DTWSpeechSearch.d.ts +2 -0
- package/dist/speech-search/DTWSpeechSearch.js +18 -0
- package/dist/speech-search/DTWSpeechSearch.js.map +1 -0
- package/dist/utilities/RandomGenerator.d.ts +2 -2
- package/dist/utilities/RandomGenerator.js.map +1 -1
- package/dist/utilities/Utilities.js +7 -1
- package/dist/utilities/Utilities.js.map +1 -1
- package/dist/utilities/WebReader.js +2 -2
- package/dist/utilities/WebReader.js.map +1 -1
- package/docs/Options.md +8 -6
- package/package.json +2 -2
- package/src/alignment/DTWMfccSequenceAlignment.ts +9 -6
- package/src/alignment/DTWSequenceAlignment.ts +2 -2
- package/src/alignment/DTWSequenceAlignmentWindowed.ts +3 -3
- package/src/alignment/SpeechAlignment.ts +1 -1
- package/src/api/SpeechSearch.ts +12 -0
- package/src/dsp/MFCC.ts +11 -31
- package/src/math/MedianFilter.ts +5 -5
- package/src/math/VectorMath.ts +109 -75
- package/src/recognition/WhisperSTT.ts +45 -27
- package/src/speech-search/DTWSpeechSearch.ts +26 -0
- package/src/utilities/RandomGenerator.ts +2 -2
- package/src/utilities/Utilities.ts +11 -3
- package/src/utilities/WebReader.ts +2 -2
package/src/math/VectorMath.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { clip } from "../utilities/Utilities.js"
|
|
2
2
|
|
|
3
|
-
export function covarianceMatrixOfSamples(samples: number[]
|
|
3
|
+
export function covarianceMatrixOfSamples(samples: ArrayLike<number>[], weights?: ArrayLike<number>, biased = false) {
|
|
4
4
|
if (samples.length == 0) {
|
|
5
5
|
throw new Error('No vectors given')
|
|
6
6
|
}
|
|
@@ -18,7 +18,7 @@ export function covarianceMatrixOfSamples(samples: number[][], weights?: number[
|
|
|
18
18
|
return { covarianceMatrix, mean }
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
export function covarianceMatrixOfCenteredSamples(centeredSamples: number[]
|
|
21
|
+
export function covarianceMatrixOfCenteredSamples(centeredSamples: ArrayLike<number>[], biased = false, diagonalRegularizationAmount = 1e-6) {
|
|
22
22
|
const sampleCount = centeredSamples.length
|
|
23
23
|
|
|
24
24
|
if (sampleCount == 0) {
|
|
@@ -60,7 +60,7 @@ export function covarianceMatrixOfCenteredSamples(centeredSamples: number[][], b
|
|
|
60
60
|
return covarianceMatrix
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
export function weightedCovarianceMatrixOfCenteredSamples(centeredSamples: number[]
|
|
63
|
+
export function weightedCovarianceMatrixOfCenteredSamples(centeredSamples: ArrayLike<number>[], weights: ArrayLike<number>, diagonalRegularizationAmount = 1e-6) {
|
|
64
64
|
const sampleCount = centeredSamples.length
|
|
65
65
|
|
|
66
66
|
if (sampleCount == 0) {
|
|
@@ -102,33 +102,36 @@ export function weightedCovarianceMatrixOfCenteredSamples(centeredSamples: numbe
|
|
|
102
102
|
return covarianceMatrix
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
-
export function centerVectors(vectors: number[]
|
|
105
|
+
export function centerVectors(vectors: ArrayLike<number>[], weights?: ArrayLike<number>) {
|
|
106
106
|
const vectorCount = vectors.length
|
|
107
107
|
|
|
108
108
|
if (vectorCount == 0) {
|
|
109
|
-
return { centeredVectors: [], mean:
|
|
109
|
+
return { centeredVectors: [] as Float32Array[], mean: new Float32Array(0) }
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
let mean:
|
|
112
|
+
let mean: Float32Array
|
|
113
|
+
|
|
113
114
|
if (weights) {
|
|
114
115
|
mean = weightedMeanOfVectors(vectors, weights)
|
|
115
116
|
} else {
|
|
116
117
|
mean = meanOfVectors(vectors)
|
|
117
118
|
}
|
|
118
119
|
|
|
119
|
-
const centeredVectors:
|
|
120
|
+
const centeredVectors: Float32Array[] = []
|
|
120
121
|
|
|
121
122
|
for (let i = 0; i < vectorCount; i++) {
|
|
122
|
-
|
|
123
|
+
const centeredVector = subtractVectors(vectors[i], mean)
|
|
124
|
+
|
|
125
|
+
centeredVectors.push(centeredVector)
|
|
123
126
|
}
|
|
124
127
|
|
|
125
128
|
return { centeredVectors, mean }
|
|
126
129
|
}
|
|
127
130
|
|
|
128
|
-
export function centerVector(vector: number
|
|
131
|
+
export function centerVector(vector: ArrayLike<number>) {
|
|
129
132
|
const mean = meanOfVector(vector)
|
|
130
133
|
|
|
131
|
-
const centeredVector
|
|
134
|
+
const centeredVector = new Float32Array(vector.length)
|
|
132
135
|
|
|
133
136
|
for (let i = 0; i < vector.length; i++) {
|
|
134
137
|
centeredVector[i] = vector[i] - mean
|
|
@@ -137,18 +140,18 @@ export function centerVector(vector: number[]) {
|
|
|
137
140
|
return centeredVector
|
|
138
141
|
}
|
|
139
142
|
|
|
140
|
-
export function scaleToSumTo1(vector: number
|
|
143
|
+
export function scaleToSumTo1(vector: ArrayLike<number>) {
|
|
141
144
|
if (vector.length == 0) {
|
|
142
|
-
return
|
|
145
|
+
return new Float32Array(0)
|
|
143
146
|
}
|
|
144
147
|
|
|
145
148
|
if (vector.length == 1) {
|
|
146
|
-
return [1]
|
|
149
|
+
return Float32Array.from([1])
|
|
147
150
|
}
|
|
148
151
|
|
|
149
152
|
const minValue = vector[indexOfMin(vector)]
|
|
150
153
|
|
|
151
|
-
const scaledVector =
|
|
154
|
+
const scaledVector = Float32Array.from(vector)
|
|
152
155
|
|
|
153
156
|
if (minValue < 0) {
|
|
154
157
|
const addedOffset = -minValue * 2
|
|
@@ -196,11 +199,11 @@ export function normalizeVector(vector: ArrayLike<number>, kind: 'population' |
|
|
|
196
199
|
return { normalizedVector, mean, stdDeviation }
|
|
197
200
|
}
|
|
198
201
|
|
|
199
|
-
export function normalizeVectors(vectors: number[]
|
|
202
|
+
export function normalizeVectors(vectors: ArrayLike<number>[], kind: 'population' | 'sample' = 'population') {
|
|
200
203
|
const vectorCount = vectors.length
|
|
201
204
|
|
|
202
205
|
if (vectorCount == 0) {
|
|
203
|
-
return { normalizedVectors: [], mean:
|
|
206
|
+
return { normalizedVectors: [] as Float32Array[], mean: new Float32Array(0), stdDeviation: new Float32Array(0) }
|
|
204
207
|
}
|
|
205
208
|
|
|
206
209
|
const featureCount = vectors[0].length
|
|
@@ -208,7 +211,7 @@ export function normalizeVectors(vectors: number[][], kind: 'population' | 'samp
|
|
|
208
211
|
const mean = meanOfVectors(vectors)
|
|
209
212
|
const stdDeviation = stdDeviationOfVectors(vectors, kind, mean)
|
|
210
213
|
|
|
211
|
-
const normalizedVectors:
|
|
214
|
+
const normalizedVectors: Float32Array[] = []
|
|
212
215
|
|
|
213
216
|
for (const vector of vectors) {
|
|
214
217
|
const normalizedVector = createVector(featureCount)
|
|
@@ -225,16 +228,16 @@ export function normalizeVectors(vectors: number[][], kind: 'population' | 'samp
|
|
|
225
228
|
return { normalizedVectors, mean, stdDeviation }
|
|
226
229
|
}
|
|
227
230
|
|
|
228
|
-
export function deNormalizeVectors(normalizedVectors: number[]
|
|
231
|
+
export function deNormalizeVectors(normalizedVectors: ArrayLike<number>[], originalMean: ArrayLike<number>, originalStdDeviation: ArrayLike<number>) {
|
|
229
232
|
const vectorCount = normalizeVectors.length
|
|
230
233
|
|
|
231
234
|
if (vectorCount == 0) {
|
|
232
|
-
return []
|
|
235
|
+
return [] as Float32Array[]
|
|
233
236
|
}
|
|
234
237
|
|
|
235
238
|
const featureCount = normalizedVectors[0].length
|
|
236
239
|
|
|
237
|
-
const deNormalizedVectors:
|
|
240
|
+
const deNormalizedVectors: Float32Array[] = []
|
|
238
241
|
|
|
239
242
|
for (const normalizedVector of normalizedVectors) {
|
|
240
243
|
const deNormalizedVector = createVector(featureCount)
|
|
@@ -249,11 +252,11 @@ export function deNormalizeVectors(normalizedVectors: number[][], originalMean:
|
|
|
249
252
|
return deNormalizedVectors
|
|
250
253
|
}
|
|
251
254
|
|
|
252
|
-
export function meanOfVectors(vectors: number[]
|
|
255
|
+
export function meanOfVectors(vectors: ArrayLike<number>[]) {
|
|
253
256
|
const vectorCount = vectors.length
|
|
254
257
|
|
|
255
258
|
if (vectorCount == 0) {
|
|
256
|
-
return
|
|
259
|
+
return new Float32Array(0)
|
|
257
260
|
}
|
|
258
261
|
|
|
259
262
|
const featureCount = vectors[0].length
|
|
@@ -273,11 +276,11 @@ export function meanOfVectors(vectors: number[][]) {
|
|
|
273
276
|
return result
|
|
274
277
|
}
|
|
275
278
|
|
|
276
|
-
export function weightedMeanOfVectors(vectors: number[]
|
|
279
|
+
export function weightedMeanOfVectors(vectors: ArrayLike<number>[], weights: ArrayLike<number>) {
|
|
277
280
|
const vectorCount = vectors.length
|
|
278
281
|
|
|
279
282
|
if (vectorCount == 0) {
|
|
280
|
-
return
|
|
283
|
+
return new Float32Array(0)
|
|
281
284
|
}
|
|
282
285
|
|
|
283
286
|
const featureCount = vectors[0].length
|
|
@@ -296,15 +299,15 @@ export function weightedMeanOfVectors(vectors: number[][], weights: number[]) {
|
|
|
296
299
|
return result
|
|
297
300
|
}
|
|
298
301
|
|
|
299
|
-
export function stdDeviationOfVectors(vectors: number[]
|
|
302
|
+
export function stdDeviationOfVectors(vectors: ArrayLike<number>[], kind: 'population' | 'sample' = 'population', mean?: ArrayLike<number>) {
|
|
300
303
|
return varianceOfVectors(vectors, kind, mean).map(v => Math.sqrt(v))
|
|
301
304
|
}
|
|
302
305
|
|
|
303
|
-
export function varianceOfVectors(vectors: number[]
|
|
306
|
+
export function varianceOfVectors(vectors: ArrayLike<number>[], kind: 'population' | 'sample' = 'population', mean?: ArrayLike<number>) {
|
|
304
307
|
const vectorCount = vectors.length
|
|
305
308
|
|
|
306
309
|
if (vectorCount == 0) {
|
|
307
|
-
return
|
|
310
|
+
return new Float32Array(0)
|
|
308
311
|
}
|
|
309
312
|
|
|
310
313
|
const sampleSizeMetric = kind == 'population' || vectorCount == 1 ? vectorCount : vectorCount - 1
|
|
@@ -369,15 +372,31 @@ export function varianceOfVector(vector: ArrayLike<number>, kind: 'population' |
|
|
|
369
372
|
return result / sampleSizeMetric
|
|
370
373
|
}
|
|
371
374
|
|
|
372
|
-
export function logOfVector(vector: number
|
|
373
|
-
|
|
375
|
+
export function logOfVector(vector: ArrayLike<number>, minVal = 1e-40) {
|
|
376
|
+
const result = new Float32Array(vector.length)
|
|
377
|
+
|
|
378
|
+
for (let i = 0; i < vector.length; i++) {
|
|
379
|
+
const value = vector[i]
|
|
380
|
+
|
|
381
|
+
result[i] = Math.log(value + minVal)
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
return result
|
|
374
385
|
}
|
|
375
386
|
|
|
376
|
-
export function expOfVector(vector: number
|
|
377
|
-
|
|
387
|
+
export function expOfVector(vector: ArrayLike<number>) {
|
|
388
|
+
const result = new Float32Array(vector.length)
|
|
389
|
+
|
|
390
|
+
for (let i = 0; i < vector.length; i++) {
|
|
391
|
+
const value = vector[i]
|
|
392
|
+
|
|
393
|
+
result[i] = Math.exp(value)
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
return result
|
|
378
397
|
}
|
|
379
398
|
|
|
380
|
-
export function transpose(matrix: number[]
|
|
399
|
+
export function transpose(matrix: ArrayLike<number>[]) {
|
|
381
400
|
const vectorCount = matrix.length
|
|
382
401
|
const featureCount = matrix[0].length
|
|
383
402
|
|
|
@@ -392,31 +411,31 @@ export function transpose(matrix: number[][]) {
|
|
|
392
411
|
return transposedMatrix
|
|
393
412
|
}
|
|
394
413
|
|
|
395
|
-
export function movingAverageOfWindow3(vector: number
|
|
414
|
+
export function movingAverageOfWindow3(vector: ArrayLike<number>) {
|
|
396
415
|
const elementCount = vector.length
|
|
397
416
|
|
|
398
417
|
if (elementCount == 0) {
|
|
399
|
-
return
|
|
418
|
+
return new Float32Array(0)
|
|
400
419
|
}
|
|
401
420
|
|
|
402
421
|
if (elementCount == 1) {
|
|
403
|
-
return
|
|
422
|
+
return Float32Array.from(vector)
|
|
404
423
|
}
|
|
405
424
|
|
|
406
|
-
const result
|
|
425
|
+
const result = new Float32Array(elementCount)
|
|
407
426
|
|
|
408
|
-
result
|
|
427
|
+
result[0] = (vector[0] + vector[0] + vector[1]) / 3
|
|
409
428
|
|
|
410
429
|
for (let i = 1; i < elementCount - 1; i++) {
|
|
411
|
-
result
|
|
430
|
+
result[i] = (vector[i - 1] + vector[i] + vector[i + 1]) / 3
|
|
412
431
|
}
|
|
413
432
|
|
|
414
|
-
result
|
|
433
|
+
result[elementCount - 1] = (vector[elementCount - 2] + vector[elementCount - 1] + vector[elementCount - 1]) / 3
|
|
415
434
|
|
|
416
435
|
return result
|
|
417
436
|
}
|
|
418
437
|
|
|
419
|
-
export function averageMeanSquaredError(actual: number[]
|
|
438
|
+
export function averageMeanSquaredError(actual: ArrayLike<number>[], expected: ArrayLike<number>[]) {
|
|
420
439
|
if (actual.length != expected.length) {
|
|
421
440
|
throw new Error('Vectors are not the same length')
|
|
422
441
|
}
|
|
@@ -592,14 +611,16 @@ export function minkowskiDistance(vector1: ArrayLike<number>, vector2: ArrayLike
|
|
|
592
611
|
return sum ** (1 / power)
|
|
593
612
|
}
|
|
594
613
|
|
|
595
|
-
export function subtractVectors(vector1: number
|
|
614
|
+
export function subtractVectors(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
|
|
615
|
+
const elementCount = vector1.length
|
|
616
|
+
|
|
596
617
|
if (vector1.length != vector2.length) {
|
|
597
618
|
throw new Error('Vectors are not the same length')
|
|
598
619
|
}
|
|
599
620
|
|
|
600
621
|
const result = createVector(vector1.length)
|
|
601
622
|
|
|
602
|
-
for (let i = 0; i <
|
|
623
|
+
for (let i = 0; i < elementCount; i++) {
|
|
603
624
|
result[i] = vector1[i] - vector2[i]
|
|
604
625
|
}
|
|
605
626
|
|
|
@@ -607,15 +628,29 @@ export function subtractVectors(vector1: number[], vector2: number[]) {
|
|
|
607
628
|
}
|
|
608
629
|
|
|
609
630
|
export function sumVector(vector: ArrayLike<number>) {
|
|
631
|
+
const elementCount = vector.length
|
|
632
|
+
|
|
610
633
|
let result = 0.0
|
|
611
634
|
|
|
612
|
-
for (let i = 0; i <
|
|
635
|
+
for (let i = 0; i < elementCount; i++) {
|
|
613
636
|
result += vector[i]
|
|
614
637
|
}
|
|
615
638
|
|
|
616
639
|
return result
|
|
617
640
|
}
|
|
618
641
|
|
|
642
|
+
export function sumOfSquaresForVector(vector: ArrayLike<number>) {
|
|
643
|
+
const elementCount = vector.length
|
|
644
|
+
|
|
645
|
+
let result = 0.0
|
|
646
|
+
|
|
647
|
+
for (let i = 0; i < elementCount; i++) {
|
|
648
|
+
result += vector[i] ** 2
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
return result
|
|
652
|
+
}
|
|
653
|
+
|
|
619
654
|
export function dotProduct(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
|
|
620
655
|
if (vector1.length != vector2.length) {
|
|
621
656
|
throw new Error('Vectors are not the same length')
|
|
@@ -671,10 +706,11 @@ export function minValue(vector: ArrayLike<number>) {
|
|
|
671
706
|
}
|
|
672
707
|
|
|
673
708
|
export function indexOfMin(vector: ArrayLike<number>) {
|
|
709
|
+
const elementCount = vector.length
|
|
674
710
|
let minValue = Infinity
|
|
675
711
|
let result = -1
|
|
676
712
|
|
|
677
|
-
for (let i = 0; i <
|
|
713
|
+
for (let i = 0; i < elementCount; i++) {
|
|
678
714
|
if (vector[i] < minValue) {
|
|
679
715
|
minValue = vector[i]
|
|
680
716
|
result = i
|
|
@@ -690,40 +726,46 @@ export function sigmoid(x: number) {
|
|
|
690
726
|
return zeroIfNaN(result)
|
|
691
727
|
}
|
|
692
728
|
|
|
693
|
-
export function softmax(logits: number
|
|
694
|
-
|
|
695
|
-
|
|
729
|
+
export function softmax(logits: ArrayLike<number>, temperature = 1.0) {
|
|
730
|
+
const logitCount = logits.length
|
|
731
|
+
|
|
732
|
+
if (logitCount === 0) {
|
|
733
|
+
return new Float32Array(0)
|
|
696
734
|
}
|
|
697
735
|
|
|
698
736
|
let maxValue = -Infinity
|
|
699
737
|
|
|
700
|
-
for (
|
|
701
|
-
|
|
702
|
-
|
|
738
|
+
for (let i = 0; i < logitCount; i++) {
|
|
739
|
+
const value = logits[i]
|
|
740
|
+
|
|
741
|
+
if (value > maxValue) {
|
|
742
|
+
maxValue = value
|
|
703
743
|
}
|
|
704
744
|
}
|
|
705
745
|
|
|
706
746
|
const temperatureReciprocal = 1 / (temperature + 1e-40)
|
|
707
747
|
|
|
708
|
-
const
|
|
748
|
+
const results = new Float32Array(logitCount)
|
|
709
749
|
|
|
710
750
|
let sumOfExponentiatedValues = 0.0
|
|
711
751
|
|
|
712
|
-
for (
|
|
752
|
+
for (let i = 0; i < logitCount; i++) {
|
|
753
|
+
const value = logits[i]
|
|
754
|
+
|
|
713
755
|
const eToValue = Math.exp((value - maxValue) * temperatureReciprocal)
|
|
714
756
|
|
|
715
757
|
sumOfExponentiatedValues += eToValue
|
|
716
758
|
|
|
717
|
-
|
|
759
|
+
results[i] = eToValue
|
|
718
760
|
}
|
|
719
761
|
|
|
720
762
|
const sumOfExponentiatedValuesReciprocal = 1 / (sumOfExponentiatedValues + 1e-40)
|
|
721
763
|
|
|
722
|
-
for (let i = 0; i <
|
|
723
|
-
|
|
764
|
+
for (let i = 0; i < logitCount; i++) {
|
|
765
|
+
results[i] *= sumOfExponentiatedValuesReciprocal
|
|
724
766
|
}
|
|
725
767
|
|
|
726
|
-
return
|
|
768
|
+
return results
|
|
727
769
|
}
|
|
728
770
|
|
|
729
771
|
export function hammingDistance(value1: number, value2: number, bitLength = 32) {
|
|
@@ -740,7 +782,7 @@ export function hammingDistance(value1: number, value2: number, bitLength = 32)
|
|
|
740
782
|
}
|
|
741
783
|
|
|
742
784
|
export function createVectorArray(vectorCount: number, featureCount: number, initialValue = 0.0) {
|
|
743
|
-
const result:
|
|
785
|
+
const result: Float32Array[] = new Array(vectorCount)
|
|
744
786
|
|
|
745
787
|
for (let i = 0; i < vectorCount; i++) {
|
|
746
788
|
result[i] = createVector(featureCount, initialValue)
|
|
@@ -750,25 +792,15 @@ export function createVectorArray(vectorCount: number, featureCount: number, ini
|
|
|
750
792
|
}
|
|
751
793
|
|
|
752
794
|
export function createVector(elementCount: number, initialValue = 0.0) {
|
|
753
|
-
const result
|
|
795
|
+
const result = new Float32Array(elementCount)
|
|
754
796
|
|
|
755
|
-
|
|
756
|
-
result
|
|
797
|
+
if (initialValue !== 0) {
|
|
798
|
+
result.fill(initialValue)
|
|
757
799
|
}
|
|
758
800
|
|
|
759
801
|
return result
|
|
760
802
|
}
|
|
761
803
|
|
|
762
|
-
export function createVectorForIntegerRange(start: number, end: number) {
|
|
763
|
-
const newVector: number[] = []
|
|
764
|
-
|
|
765
|
-
for (let i = start; i < end; i++) {
|
|
766
|
-
newVector.push(i)
|
|
767
|
-
}
|
|
768
|
-
|
|
769
|
-
return newVector
|
|
770
|
-
}
|
|
771
|
-
|
|
772
804
|
export function zeroIfNaN(val: number) {
|
|
773
805
|
if (isNaN(val)) {
|
|
774
806
|
return 0
|
|
@@ -777,21 +809,23 @@ export function zeroIfNaN(val: number) {
|
|
|
777
809
|
}
|
|
778
810
|
}
|
|
779
811
|
|
|
780
|
-
export function logSumExp(values: number
|
|
812
|
+
export function logSumExp(values: ArrayLike<number>, minVal = 1e-40) {
|
|
781
813
|
return Math.log(minVal + sumExp(values))
|
|
782
814
|
}
|
|
783
815
|
|
|
784
|
-
export function sumExp(values: number
|
|
816
|
+
export function sumExp(values: ArrayLike<number>) {
|
|
785
817
|
let sumOfExp = 0
|
|
786
818
|
|
|
787
|
-
for (
|
|
819
|
+
for (let i = 0; i < values.length; i++) {
|
|
820
|
+
const value = values[i]
|
|
821
|
+
|
|
788
822
|
sumOfExp += Math.exp(value)
|
|
789
823
|
}
|
|
790
824
|
|
|
791
825
|
return sumOfExp
|
|
792
826
|
}
|
|
793
827
|
|
|
794
|
-
export function logSoftmax(values: number
|
|
828
|
+
export function logSoftmax(values: ArrayLike<number>, minVal = 1e-40) {
|
|
795
829
|
const softMaxOfValues = softmax(values)
|
|
796
830
|
|
|
797
831
|
return logOfVector(softMaxOfValues, minVal)
|
|
@@ -814,7 +848,7 @@ export class IncrementalMean {
|
|
|
814
848
|
// 3.81
|
|
815
849
|
}
|
|
816
850
|
|
|
817
|
-
export type DistanceFunction = (a: number
|
|
851
|
+
export type DistanceFunction = (a: ArrayLike<number>, b: ArrayLike<number>) => number
|
|
818
852
|
|
|
819
853
|
export interface ComplexNumber {
|
|
820
854
|
real: number
|
|
@@ -2,8 +2,8 @@ import type * as Onnx from 'onnxruntime-node'
|
|
|
2
2
|
|
|
3
3
|
import { Logger } from '../utilities/Logger.js'
|
|
4
4
|
import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
|
|
5
|
-
import { clip, getIntegerRange, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
|
|
6
|
-
import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, stdDeviationOfVector } from '../math/VectorMath.js'
|
|
5
|
+
import { clip, concatFloat32Arrays, getIntegerRange, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
|
|
6
|
+
import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, stdDeviationOfVector, sumOfSquaresForVector, sumVector } from '../math/VectorMath.js'
|
|
7
7
|
|
|
8
8
|
import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
|
|
9
9
|
import { extendDeep } from '../utilities/ObjectUtilities.js'
|
|
@@ -373,6 +373,10 @@ export class Whisper {
|
|
|
373
373
|
options = extendDeep(defaultWhisperOptions, options)
|
|
374
374
|
options.model = this.modelName
|
|
375
375
|
|
|
376
|
+
if (!options.timestampAccuracy) {
|
|
377
|
+
options.timestampAccuracy = this.defaultTimestampAccuracy
|
|
378
|
+
}
|
|
379
|
+
|
|
376
380
|
const audioSamples = rawAudio.audioChannels[0]
|
|
377
381
|
const sampleRate = rawAudio.sampleRate
|
|
378
382
|
const prompt = options.prompt
|
|
@@ -467,7 +471,7 @@ export class Whisper {
|
|
|
467
471
|
// Find alignment path
|
|
468
472
|
let alignmentHeads: number[] | undefined
|
|
469
473
|
|
|
470
|
-
if (options.timestampAccuracy === 'medium') {
|
|
474
|
+
if (options.timestampAccuracy === 'medium' || options.model == 'large-v3-turbo') {
|
|
471
475
|
alignmentHeads = this.alignmentHeadIndexes
|
|
472
476
|
} else if (options.timestampAccuracy === 'high') {
|
|
473
477
|
alignmentHeads = undefined
|
|
@@ -517,6 +521,10 @@ export class Whisper {
|
|
|
517
521
|
|
|
518
522
|
whisperAlignmentOptions = extendDeep(defaultWhisperAlignmentOptions, whisperAlignmentOptions)
|
|
519
523
|
|
|
524
|
+
if (!whisperAlignmentOptions.timestampAccuracy) {
|
|
525
|
+
whisperAlignmentOptions.timestampAccuracy = this.defaultTimestampAccuracy
|
|
526
|
+
}
|
|
527
|
+
|
|
520
528
|
const targetLanguage = task === 'transcribe' ? sourceLanguage : 'en'
|
|
521
529
|
|
|
522
530
|
const shouldSplitToSentences = false
|
|
@@ -1389,19 +1397,14 @@ export class Whisper {
|
|
|
1389
1397
|
const frameCount = qksTensors[0].dims[4]
|
|
1390
1398
|
|
|
1391
1399
|
if (!headIndexes) {
|
|
1392
|
-
headIndexes =
|
|
1393
|
-
|
|
1394
|
-
for (let i = 0; i < layerCount * headCount; i++) {
|
|
1395
|
-
//for (let i = Math.floor(layerCount * headCount / 2); i < layerCount * headCount; i++) {
|
|
1396
|
-
headIndexes.push(i)
|
|
1397
|
-
}
|
|
1400
|
+
headIndexes = getIntegerRange(0, layerCount * headCount)
|
|
1398
1401
|
}
|
|
1399
1402
|
|
|
1400
1403
|
// Load attention head weights from tensors
|
|
1401
|
-
const attentionHeads:
|
|
1404
|
+
const attentionHeads: Float32Array[][] = [] // structure: [heads, tokens, frames]
|
|
1402
1405
|
|
|
1403
1406
|
for (const headIndex of headIndexes) {
|
|
1404
|
-
const attentionHead:
|
|
1407
|
+
const attentionHead: Float32Array[] = [] // structure: [tokens, frames]
|
|
1405
1408
|
|
|
1406
1409
|
for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) {
|
|
1407
1410
|
const bufferOffset = headIndex * frameCount
|
|
@@ -1410,7 +1413,7 @@ export class Whisper {
|
|
|
1410
1413
|
|
|
1411
1414
|
const framesForHead = qksTensors[tokenIndex].data.slice(startIndexInBuffer, endIndexInBuffer)
|
|
1412
1415
|
|
|
1413
|
-
attentionHead.push(
|
|
1416
|
+
attentionHead.push(framesForHead)
|
|
1414
1417
|
}
|
|
1415
1418
|
|
|
1416
1419
|
attentionHeads.push(attentionHead)
|
|
@@ -1435,10 +1438,19 @@ export class Whisper {
|
|
|
1435
1438
|
// Normalize all weights in each individual head, if enabled
|
|
1436
1439
|
if (normalize) {
|
|
1437
1440
|
for (const head of attentionHeads) {
|
|
1438
|
-
|
|
1441
|
+
let sumOfAllWeightsForHead = 0
|
|
1442
|
+
let sumOfAllSquaredWeightsForHead = 0
|
|
1443
|
+
let countOfAllWeightsForHead = 0
|
|
1444
|
+
|
|
1445
|
+
for (const tokenFrames of head) {
|
|
1446
|
+
sumOfAllWeightsForHead += sumVector(tokenFrames)
|
|
1447
|
+
sumOfAllSquaredWeightsForHead += sumOfSquaresForVector(tokenFrames)
|
|
1448
|
+
countOfAllWeightsForHead += tokenFrames.length
|
|
1449
|
+
}
|
|
1439
1450
|
|
|
1440
|
-
const meanOfAllWeightsForHead =
|
|
1441
|
-
const
|
|
1451
|
+
const meanOfAllWeightsForHead = sumOfAllWeightsForHead / countOfAllWeightsForHead
|
|
1452
|
+
const varianceOfAllWeightsForHead = sumOfAllSquaredWeightsForHead / countOfAllWeightsForHead
|
|
1453
|
+
const stdDeviationOfAllWeightsForHead = Math.sqrt(varianceOfAllWeightsForHead)
|
|
1442
1454
|
|
|
1443
1455
|
const stdDeviationReciprocal = 1.0 / (stdDeviationOfAllWeightsForHead + 1e-10)
|
|
1444
1456
|
|
|
@@ -1459,16 +1471,12 @@ export class Whisper {
|
|
|
1459
1471
|
}
|
|
1460
1472
|
}
|
|
1461
1473
|
|
|
1462
|
-
// Compute the mean for all layers
|
|
1463
|
-
const frameMeansForToken:
|
|
1464
|
-
|
|
1465
|
-
for (let i = 0; i < tokenCount; i++) {
|
|
1466
|
-
const frameMeans = new Array(segmentFrameCount)
|
|
1467
|
-
|
|
1468
|
-
frameMeansForToken.push(frameMeans)
|
|
1469
|
-
}
|
|
1474
|
+
// Compute the mean of the selected attention heads for all layers
|
|
1475
|
+
const frameMeansForToken: Float32Array[] = []
|
|
1470
1476
|
|
|
1471
1477
|
for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) {
|
|
1478
|
+
const meansForFrames = new Float32Array(segmentFrameCount)
|
|
1479
|
+
|
|
1472
1480
|
for (let frameIndex = 0; frameIndex < segmentFrameCount; frameIndex++) {
|
|
1473
1481
|
let sum = 0
|
|
1474
1482
|
|
|
@@ -1478,8 +1486,10 @@ export class Whisper {
|
|
|
1478
1486
|
|
|
1479
1487
|
const frameMean = sum / attentionHeads.length
|
|
1480
1488
|
|
|
1481
|
-
|
|
1489
|
+
meansForFrames[frameIndex] = frameMean
|
|
1482
1490
|
}
|
|
1491
|
+
|
|
1492
|
+
frameMeansForToken.push(meansForFrames)
|
|
1483
1493
|
}
|
|
1484
1494
|
|
|
1485
1495
|
// Anchor timestamp tokens timestamps to their original values, if enabled
|
|
@@ -1499,7 +1509,7 @@ export class Whisper {
|
|
|
1499
1509
|
}
|
|
1500
1510
|
}
|
|
1501
1511
|
|
|
1502
|
-
// Perform DTW
|
|
1512
|
+
// Perform DTW to align tokens indexes to frame indexes
|
|
1503
1513
|
const tokenIndexes = getIntegerRange(0, tokenCount)
|
|
1504
1514
|
const frameIndexes = getIntegerRange(0, segmentFrameCount)
|
|
1505
1515
|
|
|
@@ -1773,6 +1783,14 @@ export class Whisper {
|
|
|
1773
1783
|
return alignmentHeadsIndexes[this.modelName]
|
|
1774
1784
|
}
|
|
1775
1785
|
|
|
1786
|
+
get defaultTimestampAccuracy() {
|
|
1787
|
+
if (this.modelName.startsWith('tiny') || this.modelName.startsWith('base')) {
|
|
1788
|
+
return 'high'
|
|
1789
|
+
} else {
|
|
1790
|
+
return 'medium'
|
|
1791
|
+
}
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1776
1794
|
getSuppressedTokens() {
|
|
1777
1795
|
return [
|
|
1778
1796
|
...this.getSuppressedTextTokens(),
|
|
@@ -2521,7 +2539,7 @@ export const defaultWhisperOptions: WhisperOptions = {
|
|
|
2521
2539
|
decodeTimestampTokens: true,
|
|
2522
2540
|
endTokenThreshold: 0.9,
|
|
2523
2541
|
includeEndTokenInCandidates: true,
|
|
2524
|
-
timestampAccuracy:
|
|
2542
|
+
timestampAccuracy: undefined,
|
|
2525
2543
|
encoderProvider: undefined,
|
|
2526
2544
|
decoderProvider: undefined,
|
|
2527
2545
|
seed: undefined,
|
|
@@ -2542,7 +2560,7 @@ export const defaultWhisperAlignmentOptions: WhisperAlignmentOptions = {
|
|
|
2542
2560
|
model: undefined,
|
|
2543
2561
|
endTokenThreshold: 0.9,
|
|
2544
2562
|
maxTokensPerPart: 250,
|
|
2545
|
-
timestampAccuracy:
|
|
2563
|
+
timestampAccuracy: undefined,
|
|
2546
2564
|
|
|
2547
2565
|
encoderProvider: undefined,
|
|
2548
2566
|
decoderProvider: undefined,
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { DtwGranularity, getMfccOptionsForGranularity } from "../alignment/SpeechAlignment.js";
|
|
2
|
+
import { RawAudio } from "../audio/AudioUtilities.js";
|
|
3
|
+
import { computeMFCCs, extendDefaultMfccOptions, MfccOptions } from "../dsp/MFCC.js";
|
|
4
|
+
import { Logger } from "../utilities/Logger.js";
|
|
5
|
+
|
|
6
|
+
export async function searchSpeech(sourceRawAudio: RawAudio, referenceRawAudio: RawAudio) {
|
|
7
|
+
const logger = new Logger()
|
|
8
|
+
|
|
9
|
+
const granularity: DtwGranularity = 'low'
|
|
10
|
+
|
|
11
|
+
const mfccOptions = extendDefaultMfccOptions({ ...getMfccOptionsForGranularity(granularity), zeroFirstCoefficient: true }) as MfccOptions
|
|
12
|
+
|
|
13
|
+
// Compute reference MFCCs
|
|
14
|
+
logger.start('Compute reference MFCC features')
|
|
15
|
+
const referenceMfccs = await computeMFCCs(referenceRawAudio, mfccOptions)
|
|
16
|
+
|
|
17
|
+
// Compute source MFCCs
|
|
18
|
+
logger.start('Compute source MFCC features')
|
|
19
|
+
const sourceMfccs = await computeMFCCs(sourceRawAudio, mfccOptions)
|
|
20
|
+
|
|
21
|
+
logger.start('Compute source MFCC features')
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function computeCostMatrix<T, U>(sequence1: T[], sequence2: U[]) {
|
|
25
|
+
|
|
26
|
+
}
|
|
@@ -29,7 +29,7 @@ export abstract class RandomGenerator {
|
|
|
29
29
|
return result
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
getNormallyDistributedVector(elementCount: number, meanVector: number
|
|
32
|
+
getNormallyDistributedVector(elementCount: number, meanVector: ArrayLike<number>, standardDeviationVector: ArrayLike<number>) {
|
|
33
33
|
const features = this.getNormallyDistributedValues(elementCount)
|
|
34
34
|
|
|
35
35
|
for (let i = 0; i < features.length; i++) {
|
|
@@ -78,7 +78,7 @@ export abstract class RandomGenerator {
|
|
|
78
78
|
return [n1, n2]
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
-
selectRandomIndexFromDistribution(distribution: number
|
|
81
|
+
selectRandomIndexFromDistribution(distribution: ArrayLike<number>) {
|
|
82
82
|
const sum = sumVector(distribution)
|
|
83
83
|
|
|
84
84
|
const randomTarget = this.getFloatInRange(0, sum)
|