echogarden 1.8.2 → 1.8.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/alignment/DTWMfccSequenceAlignment.d.ts +1 -1
  2. package/dist/alignment/DTWMfccSequenceAlignment.js +6 -5
  3. package/dist/alignment/DTWMfccSequenceAlignment.js.map +1 -1
  4. package/dist/alignment/DTWSequenceAlignment.d.ts +1 -1
  5. package/dist/alignment/DTWSequenceAlignment.js.map +1 -1
  6. package/dist/alignment/DTWSequenceAlignmentWindowed.d.ts +1 -1
  7. package/dist/alignment/DTWSequenceAlignmentWindowed.js.map +1 -1
  8. package/dist/alignment/SpeechAlignment.d.ts +2 -0
  9. package/dist/alignment/SpeechAlignment.js +1 -1
  10. package/dist/alignment/SpeechAlignment.js.map +1 -1
  11. package/dist/api/SpeechSearch.d.ts +6 -0
  12. package/dist/api/SpeechSearch.js +4 -0
  13. package/dist/api/SpeechSearch.js.map +1 -0
  14. package/dist/dsp/MFCC.d.ts +5 -6
  15. package/dist/dsp/MFCC.js +4 -19
  16. package/dist/dsp/MFCC.js.map +1 -1
  17. package/dist/math/MedianFilter.d.ts +2 -2
  18. package/dist/math/MedianFilter.js +3 -4
  19. package/dist/math/MedianFilter.js.map +1 -1
  20. package/dist/math/VectorMath.d.ts +34 -34
  21. package/dist/math/VectorMath.js +63 -44
  22. package/dist/math/VectorMath.js.map +1 -1
  23. package/dist/nlp/IPA.d.ts +3 -3
  24. package/dist/recognition/WhisperSTT.d.ts +1 -0
  25. package/dist/recognition/WhisperSTT.js +36 -20
  26. package/dist/recognition/WhisperSTT.js.map +1 -1
  27. package/dist/speech-search/DTWSpeechSearch.d.ts +2 -0
  28. package/dist/speech-search/DTWSpeechSearch.js +18 -0
  29. package/dist/speech-search/DTWSpeechSearch.js.map +1 -0
  30. package/dist/utilities/RandomGenerator.d.ts +2 -2
  31. package/dist/utilities/RandomGenerator.js.map +1 -1
  32. package/dist/utilities/Utilities.js +7 -1
  33. package/dist/utilities/Utilities.js.map +1 -1
  34. package/dist/utilities/WebReader.js +2 -2
  35. package/dist/utilities/WebReader.js.map +1 -1
  36. package/docs/Options.md +8 -6
  37. package/package.json +2 -2
  38. package/src/alignment/DTWMfccSequenceAlignment.ts +9 -6
  39. package/src/alignment/DTWSequenceAlignment.ts +2 -2
  40. package/src/alignment/DTWSequenceAlignmentWindowed.ts +3 -3
  41. package/src/alignment/SpeechAlignment.ts +1 -1
  42. package/src/api/SpeechSearch.ts +12 -0
  43. package/src/dsp/MFCC.ts +11 -31
  44. package/src/math/MedianFilter.ts +5 -5
  45. package/src/math/VectorMath.ts +109 -75
  46. package/src/recognition/WhisperSTT.ts +45 -27
  47. package/src/speech-search/DTWSpeechSearch.ts +26 -0
  48. package/src/utilities/RandomGenerator.ts +2 -2
  49. package/src/utilities/Utilities.ts +11 -3
  50. package/src/utilities/WebReader.ts +2 -2
@@ -1,6 +1,6 @@
1
1
  import { clip } from "../utilities/Utilities.js"
2
2
 
3
- export function covarianceMatrixOfSamples(samples: number[][], weights?: number[], biased = false) {
3
+ export function covarianceMatrixOfSamples(samples: ArrayLike<number>[], weights?: ArrayLike<number>, biased = false) {
4
4
  if (samples.length == 0) {
5
5
  throw new Error('No vectors given')
6
6
  }
@@ -18,7 +18,7 @@ export function covarianceMatrixOfSamples(samples: number[][], weights?: number[
18
18
  return { covarianceMatrix, mean }
19
19
  }
20
20
 
21
- export function covarianceMatrixOfCenteredSamples(centeredSamples: number[][], biased = false, diagonalRegularizationAmount = 1e-6) {
21
+ export function covarianceMatrixOfCenteredSamples(centeredSamples: ArrayLike<number>[], biased = false, diagonalRegularizationAmount = 1e-6) {
22
22
  const sampleCount = centeredSamples.length
23
23
 
24
24
  if (sampleCount == 0) {
@@ -60,7 +60,7 @@ export function covarianceMatrixOfCenteredSamples(centeredSamples: number[][], b
60
60
  return covarianceMatrix
61
61
  }
62
62
 
63
- export function weightedCovarianceMatrixOfCenteredSamples(centeredSamples: number[][], weights: number[], diagonalRegularizationAmount = 1e-6) {
63
+ export function weightedCovarianceMatrixOfCenteredSamples(centeredSamples: ArrayLike<number>[], weights: ArrayLike<number>, diagonalRegularizationAmount = 1e-6) {
64
64
  const sampleCount = centeredSamples.length
65
65
 
66
66
  if (sampleCount == 0) {
@@ -102,33 +102,36 @@ export function weightedCovarianceMatrixOfCenteredSamples(centeredSamples: numbe
102
102
  return covarianceMatrix
103
103
  }
104
104
 
105
- export function centerVectors(vectors: number[][], weights?: number[]) {
105
+ export function centerVectors(vectors: ArrayLike<number>[], weights?: ArrayLike<number>) {
106
106
  const vectorCount = vectors.length
107
107
 
108
108
  if (vectorCount == 0) {
109
- return { centeredVectors: [], mean: [] }
109
+ return { centeredVectors: [] as Float32Array[], mean: new Float32Array(0) }
110
110
  }
111
111
 
112
- let mean: number[]
112
+ let mean: Float32Array
113
+
113
114
  if (weights) {
114
115
  mean = weightedMeanOfVectors(vectors, weights)
115
116
  } else {
116
117
  mean = meanOfVectors(vectors)
117
118
  }
118
119
 
119
- const centeredVectors: number[][] = new Array(vectorCount)
120
+ const centeredVectors: Float32Array[] = []
120
121
 
121
122
  for (let i = 0; i < vectorCount; i++) {
122
- centeredVectors[i] = subtractVectors(vectors[i], mean)
123
+ const centeredVector = subtractVectors(vectors[i], mean)
124
+
125
+ centeredVectors.push(centeredVector)
123
126
  }
124
127
 
125
128
  return { centeredVectors, mean }
126
129
  }
127
130
 
128
- export function centerVector(vector: number[]) {
131
+ export function centerVector(vector: ArrayLike<number>) {
129
132
  const mean = meanOfVector(vector)
130
133
 
131
- const centeredVector: number[] = new Array(vector.length)
134
+ const centeredVector = new Float32Array(vector.length)
132
135
 
133
136
  for (let i = 0; i < vector.length; i++) {
134
137
  centeredVector[i] = vector[i] - mean
@@ -137,18 +140,18 @@ export function centerVector(vector: number[]) {
137
140
  return centeredVector
138
141
  }
139
142
 
140
- export function scaleToSumTo1(vector: number[]) {
143
+ export function scaleToSumTo1(vector: ArrayLike<number>) {
141
144
  if (vector.length == 0) {
142
- return []
145
+ return new Float32Array(0)
143
146
  }
144
147
 
145
148
  if (vector.length == 1) {
146
- return [1]
149
+ return Float32Array.from([1])
147
150
  }
148
151
 
149
152
  const minValue = vector[indexOfMin(vector)]
150
153
 
151
- const scaledVector = vector.slice()
154
+ const scaledVector = Float32Array.from(vector)
152
155
 
153
156
  if (minValue < 0) {
154
157
  const addedOffset = -minValue * 2
@@ -196,11 +199,11 @@ export function normalizeVector(vector: ArrayLike<number>, kind: 'population' |
196
199
  return { normalizedVector, mean, stdDeviation }
197
200
  }
198
201
 
199
- export function normalizeVectors(vectors: number[][], kind: 'population' | 'sample' = 'population') {
202
+ export function normalizeVectors(vectors: ArrayLike<number>[], kind: 'population' | 'sample' = 'population') {
200
203
  const vectorCount = vectors.length
201
204
 
202
205
  if (vectorCount == 0) {
203
- return { normalizedVectors: [], mean: [], stdDeviation: [] }
206
+ return { normalizedVectors: [] as Float32Array[], mean: new Float32Array(0), stdDeviation: new Float32Array(0) }
204
207
  }
205
208
 
206
209
  const featureCount = vectors[0].length
@@ -208,7 +211,7 @@ export function normalizeVectors(vectors: number[][], kind: 'population' | 'samp
208
211
  const mean = meanOfVectors(vectors)
209
212
  const stdDeviation = stdDeviationOfVectors(vectors, kind, mean)
210
213
 
211
- const normalizedVectors: number[][] = []
214
+ const normalizedVectors: Float32Array[] = []
212
215
 
213
216
  for (const vector of vectors) {
214
217
  const normalizedVector = createVector(featureCount)
@@ -225,16 +228,16 @@ export function normalizeVectors(vectors: number[][], kind: 'population' | 'samp
225
228
  return { normalizedVectors, mean, stdDeviation }
226
229
  }
227
230
 
228
- export function deNormalizeVectors(normalizedVectors: number[][], originalMean: number[], originalStdDeviation: number[]) {
231
+ export function deNormalizeVectors(normalizedVectors: ArrayLike<number>[], originalMean: ArrayLike<number>, originalStdDeviation: ArrayLike<number>) {
229
232
  const vectorCount = normalizeVectors.length
230
233
 
231
234
  if (vectorCount == 0) {
232
- return []
235
+ return [] as Float32Array[]
233
236
  }
234
237
 
235
238
  const featureCount = normalizedVectors[0].length
236
239
 
237
- const deNormalizedVectors: number[][] = []
240
+ const deNormalizedVectors: Float32Array[] = []
238
241
 
239
242
  for (const normalizedVector of normalizedVectors) {
240
243
  const deNormalizedVector = createVector(featureCount)
@@ -249,11 +252,11 @@ export function deNormalizeVectors(normalizedVectors: number[][], originalMean:
249
252
  return deNormalizedVectors
250
253
  }
251
254
 
252
- export function meanOfVectors(vectors: number[][]) {
255
+ export function meanOfVectors(vectors: ArrayLike<number>[]) {
253
256
  const vectorCount = vectors.length
254
257
 
255
258
  if (vectorCount == 0) {
256
- return []
259
+ return new Float32Array(0)
257
260
  }
258
261
 
259
262
  const featureCount = vectors[0].length
@@ -273,11 +276,11 @@ export function meanOfVectors(vectors: number[][]) {
273
276
  return result
274
277
  }
275
278
 
276
- export function weightedMeanOfVectors(vectors: number[][], weights: number[]) {
279
+ export function weightedMeanOfVectors(vectors: ArrayLike<number>[], weights: ArrayLike<number>) {
277
280
  const vectorCount = vectors.length
278
281
 
279
282
  if (vectorCount == 0) {
280
- return []
283
+ return new Float32Array(0)
281
284
  }
282
285
 
283
286
  const featureCount = vectors[0].length
@@ -296,15 +299,15 @@ export function weightedMeanOfVectors(vectors: number[][], weights: number[]) {
296
299
  return result
297
300
  }
298
301
 
299
- export function stdDeviationOfVectors(vectors: number[][], kind: 'population' | 'sample' = 'population', mean?: number[]) {
302
+ export function stdDeviationOfVectors(vectors: ArrayLike<number>[], kind: 'population' | 'sample' = 'population', mean?: ArrayLike<number>) {
300
303
  return varianceOfVectors(vectors, kind, mean).map(v => Math.sqrt(v))
301
304
  }
302
305
 
303
- export function varianceOfVectors(vectors: number[][], kind: 'population' | 'sample' = 'population', mean?: number[]) {
306
+ export function varianceOfVectors(vectors: ArrayLike<number>[], kind: 'population' | 'sample' = 'population', mean?: ArrayLike<number>) {
304
307
  const vectorCount = vectors.length
305
308
 
306
309
  if (vectorCount == 0) {
307
- return []
310
+ return new Float32Array(0)
308
311
  }
309
312
 
310
313
  const sampleSizeMetric = kind == 'population' || vectorCount == 1 ? vectorCount : vectorCount - 1
@@ -369,15 +372,31 @@ export function varianceOfVector(vector: ArrayLike<number>, kind: 'population' |
369
372
  return result / sampleSizeMetric
370
373
  }
371
374
 
372
- export function logOfVector(vector: number[], minVal = 1e-40) {
373
- return vector.map(value => Math.log(value + minVal))
375
+ export function logOfVector(vector: ArrayLike<number>, minVal = 1e-40) {
376
+ const result = new Float32Array(vector.length)
377
+
378
+ for (let i = 0; i < vector.length; i++) {
379
+ const value = vector[i]
380
+
381
+ result[i] = Math.log(value + minVal)
382
+ }
383
+
384
+ return result
374
385
  }
375
386
 
376
- export function expOfVector(vector: number[]) {
377
- return vector.map(value => Math.exp(value))
387
+ export function expOfVector(vector: ArrayLike<number>) {
388
+ const result = new Float32Array(vector.length)
389
+
390
+ for (let i = 0; i < vector.length; i++) {
391
+ const value = vector[i]
392
+
393
+ result[i] = Math.exp(value)
394
+ }
395
+
396
+ return result
378
397
  }
379
398
 
380
- export function transpose(matrix: number[][]) {
399
+ export function transpose(matrix: ArrayLike<number>[]) {
381
400
  const vectorCount = matrix.length
382
401
  const featureCount = matrix[0].length
383
402
 
@@ -392,31 +411,31 @@ export function transpose(matrix: number[][]) {
392
411
  return transposedMatrix
393
412
  }
394
413
 
395
- export function movingAverageOfWindow3(vector: number[]) {
414
+ export function movingAverageOfWindow3(vector: ArrayLike<number>) {
396
415
  const elementCount = vector.length
397
416
 
398
417
  if (elementCount == 0) {
399
- return []
418
+ return new Float32Array(0)
400
419
  }
401
420
 
402
421
  if (elementCount == 1) {
403
- return vector.slice()
422
+ return Float32Array.from(vector)
404
423
  }
405
424
 
406
- const result: number[] = []
425
+ const result = new Float32Array(elementCount)
407
426
 
408
- result.push((vector[0] + vector[0] + vector[1]) / 3)
427
+ result[0] = (vector[0] + vector[0] + vector[1]) / 3
409
428
 
410
429
  for (let i = 1; i < elementCount - 1; i++) {
411
- result.push((vector[i - 1] + vector[i] + vector[i + 1]) / 3)
430
+ result[i] = (vector[i - 1] + vector[i] + vector[i + 1]) / 3
412
431
  }
413
432
 
414
- result.push((vector[elementCount - 2] + vector[elementCount - 1] + vector[elementCount - 1]) / 3)
433
+ result[elementCount - 1] = (vector[elementCount - 2] + vector[elementCount - 1] + vector[elementCount - 1]) / 3
415
434
 
416
435
  return result
417
436
  }
418
437
 
419
- export function averageMeanSquaredError(actual: number[][], expected: number[][]) {
438
+ export function averageMeanSquaredError(actual: ArrayLike<number>[], expected: ArrayLike<number>[]) {
420
439
  if (actual.length != expected.length) {
421
440
  throw new Error('Vectors are not the same length')
422
441
  }
@@ -592,14 +611,16 @@ export function minkowskiDistance(vector1: ArrayLike<number>, vector2: ArrayLike
592
611
  return sum ** (1 / power)
593
612
  }
594
613
 
595
- export function subtractVectors(vector1: number[], vector2: number[]) {
614
+ export function subtractVectors(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
615
+ const elementCount = vector1.length
616
+
596
617
  if (vector1.length != vector2.length) {
597
618
  throw new Error('Vectors are not the same length')
598
619
  }
599
620
 
600
621
  const result = createVector(vector1.length)
601
622
 
602
- for (let i = 0; i < vector1.length; i++) {
623
+ for (let i = 0; i < elementCount; i++) {
603
624
  result[i] = vector1[i] - vector2[i]
604
625
  }
605
626
 
@@ -607,15 +628,29 @@ export function subtractVectors(vector1: number[], vector2: number[]) {
607
628
  }
608
629
 
609
630
  export function sumVector(vector: ArrayLike<number>) {
631
+ const elementCount = vector.length
632
+
610
633
  let result = 0.0
611
634
 
612
- for (let i = 0; i < vector.length; i++) {
635
+ for (let i = 0; i < elementCount; i++) {
613
636
  result += vector[i]
614
637
  }
615
638
 
616
639
  return result
617
640
  }
618
641
 
642
+ export function sumOfSquaresForVector(vector: ArrayLike<number>) {
643
+ const elementCount = vector.length
644
+
645
+ let result = 0.0
646
+
647
+ for (let i = 0; i < elementCount; i++) {
648
+ result += vector[i] ** 2
649
+ }
650
+
651
+ return result
652
+ }
653
+
619
654
  export function dotProduct(vector1: ArrayLike<number>, vector2: ArrayLike<number>) {
620
655
  if (vector1.length != vector2.length) {
621
656
  throw new Error('Vectors are not the same length')
@@ -671,10 +706,11 @@ export function minValue(vector: ArrayLike<number>) {
671
706
  }
672
707
 
673
708
  export function indexOfMin(vector: ArrayLike<number>) {
709
+ const elementCount = vector.length
674
710
  let minValue = Infinity
675
711
  let result = -1
676
712
 
677
- for (let i = 0; i < vector.length; i++) {
713
+ for (let i = 0; i < elementCount; i++) {
678
714
  if (vector[i] < minValue) {
679
715
  minValue = vector[i]
680
716
  result = i
@@ -690,40 +726,46 @@ export function sigmoid(x: number) {
690
726
  return zeroIfNaN(result)
691
727
  }
692
728
 
693
- export function softmax(logits: number[], temperature = 1.0) {
694
- if (logits.length === 0) {
695
- return []
729
+ export function softmax(logits: ArrayLike<number>, temperature = 1.0) {
730
+ const logitCount = logits.length
731
+
732
+ if (logitCount === 0) {
733
+ return new Float32Array(0)
696
734
  }
697
735
 
698
736
  let maxValue = -Infinity
699
737
 
700
- for (const val of logits) {
701
- if (val > maxValue) {
702
- maxValue = val
738
+ for (let i = 0; i < logitCount; i++) {
739
+ const value = logits[i]
740
+
741
+ if (value > maxValue) {
742
+ maxValue = value
703
743
  }
704
744
  }
705
745
 
706
746
  const temperatureReciprocal = 1 / (temperature + 1e-40)
707
747
 
708
- const result: number[] = []
748
+ const results = new Float32Array(logitCount)
709
749
 
710
750
  let sumOfExponentiatedValues = 0.0
711
751
 
712
- for (const value of logits) {
752
+ for (let i = 0; i < logitCount; i++) {
753
+ const value = logits[i]
754
+
713
755
  const eToValue = Math.exp((value - maxValue) * temperatureReciprocal)
714
756
 
715
757
  sumOfExponentiatedValues += eToValue
716
758
 
717
- result.push(eToValue)
759
+ results[i] = eToValue
718
760
  }
719
761
 
720
762
  const sumOfExponentiatedValuesReciprocal = 1 / (sumOfExponentiatedValues + 1e-40)
721
763
 
722
- for (let i = 0; i < result.length; i++) {
723
- result[i] *= sumOfExponentiatedValuesReciprocal
764
+ for (let i = 0; i < logitCount; i++) {
765
+ results[i] *= sumOfExponentiatedValuesReciprocal
724
766
  }
725
767
 
726
- return result
768
+ return results
727
769
  }
728
770
 
729
771
  export function hammingDistance(value1: number, value2: number, bitLength = 32) {
@@ -740,7 +782,7 @@ export function hammingDistance(value1: number, value2: number, bitLength = 32)
740
782
  }
741
783
 
742
784
  export function createVectorArray(vectorCount: number, featureCount: number, initialValue = 0.0) {
743
- const result: number[][] = new Array(vectorCount)
785
+ const result: Float32Array[] = new Array(vectorCount)
744
786
 
745
787
  for (let i = 0; i < vectorCount; i++) {
746
788
  result[i] = createVector(featureCount, initialValue)
@@ -750,25 +792,15 @@ export function createVectorArray(vectorCount: number, featureCount: number, ini
750
792
  }
751
793
 
752
794
  export function createVector(elementCount: number, initialValue = 0.0) {
753
- const result: number[] = new Array(elementCount)
795
+ const result = new Float32Array(elementCount)
754
796
 
755
- for (let i = 0; i < elementCount; i++) {
756
- result[i] = initialValue
797
+ if (initialValue !== 0) {
798
+ result.fill(initialValue)
757
799
  }
758
800
 
759
801
  return result
760
802
  }
761
803
 
762
- export function createVectorForIntegerRange(start: number, end: number) {
763
- const newVector: number[] = []
764
-
765
- for (let i = start; i < end; i++) {
766
- newVector.push(i)
767
- }
768
-
769
- return newVector
770
- }
771
-
772
804
  export function zeroIfNaN(val: number) {
773
805
  if (isNaN(val)) {
774
806
  return 0
@@ -777,21 +809,23 @@ export function zeroIfNaN(val: number) {
777
809
  }
778
810
  }
779
811
 
780
- export function logSumExp(values: number[], minVal = 1e-40) {
812
+ export function logSumExp(values: ArrayLike<number>, minVal = 1e-40) {
781
813
  return Math.log(minVal + sumExp(values))
782
814
  }
783
815
 
784
- export function sumExp(values: number[]) {
816
+ export function sumExp(values: ArrayLike<number>) {
785
817
  let sumOfExp = 0
786
818
 
787
- for (const value of values) {
819
+ for (let i = 0; i < values.length; i++) {
820
+ const value = values[i]
821
+
788
822
  sumOfExp += Math.exp(value)
789
823
  }
790
824
 
791
825
  return sumOfExp
792
826
  }
793
827
 
794
- export function logSoftmax(values: number[], minVal = 1e-40) {
828
+ export function logSoftmax(values: ArrayLike<number>, minVal = 1e-40) {
795
829
  const softMaxOfValues = softmax(values)
796
830
 
797
831
  return logOfVector(softMaxOfValues, minVal)
@@ -814,7 +848,7 @@ export class IncrementalMean {
814
848
  // 3.81
815
849
  }
816
850
 
817
- export type DistanceFunction = (a: number[], b: number[]) => number
851
+ export type DistanceFunction = (a: ArrayLike<number>, b: ArrayLike<number>) => number
818
852
 
819
853
  export interface ComplexNumber {
820
854
  real: number
@@ -2,8 +2,8 @@ import type * as Onnx from 'onnxruntime-node'
2
2
 
3
3
  import { Logger } from '../utilities/Logger.js'
4
4
  import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
5
- import { clip, getIntegerRange, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
6
- import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, stdDeviationOfVector } from '../math/VectorMath.js'
5
+ import { clip, concatFloat32Arrays, getIntegerRange, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
6
+ import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, stdDeviationOfVector, sumOfSquaresForVector, sumVector } from '../math/VectorMath.js'
7
7
 
8
8
  import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
9
9
  import { extendDeep } from '../utilities/ObjectUtilities.js'
@@ -373,6 +373,10 @@ export class Whisper {
373
373
  options = extendDeep(defaultWhisperOptions, options)
374
374
  options.model = this.modelName
375
375
 
376
+ if (!options.timestampAccuracy) {
377
+ options.timestampAccuracy = this.defaultTimestampAccuracy
378
+ }
379
+
376
380
  const audioSamples = rawAudio.audioChannels[0]
377
381
  const sampleRate = rawAudio.sampleRate
378
382
  const prompt = options.prompt
@@ -467,7 +471,7 @@ export class Whisper {
467
471
  // Find alignment path
468
472
  let alignmentHeads: number[] | undefined
469
473
 
470
- if (options.timestampAccuracy === 'medium') {
474
+ if (options.timestampAccuracy === 'medium' || options.model == 'large-v3-turbo') {
471
475
  alignmentHeads = this.alignmentHeadIndexes
472
476
  } else if (options.timestampAccuracy === 'high') {
473
477
  alignmentHeads = undefined
@@ -517,6 +521,10 @@ export class Whisper {
517
521
 
518
522
  whisperAlignmentOptions = extendDeep(defaultWhisperAlignmentOptions, whisperAlignmentOptions)
519
523
 
524
+ if (!whisperAlignmentOptions.timestampAccuracy) {
525
+ whisperAlignmentOptions.timestampAccuracy = this.defaultTimestampAccuracy
526
+ }
527
+
520
528
  const targetLanguage = task === 'transcribe' ? sourceLanguage : 'en'
521
529
 
522
530
  const shouldSplitToSentences = false
@@ -1389,19 +1397,14 @@ export class Whisper {
1389
1397
  const frameCount = qksTensors[0].dims[4]
1390
1398
 
1391
1399
  if (!headIndexes) {
1392
- headIndexes = []
1393
-
1394
- for (let i = 0; i < layerCount * headCount; i++) {
1395
- //for (let i = Math.floor(layerCount * headCount / 2); i < layerCount * headCount; i++) {
1396
- headIndexes.push(i)
1397
- }
1400
+ headIndexes = getIntegerRange(0, layerCount * headCount)
1398
1401
  }
1399
1402
 
1400
1403
  // Load attention head weights from tensors
1401
- const attentionHeads: number[][][] = [] // structure: [heads, tokens, frames]
1404
+ const attentionHeads: Float32Array[][] = [] // structure: [heads, tokens, frames]
1402
1405
 
1403
1406
  for (const headIndex of headIndexes) {
1404
- const attentionHead: number[][] = [] // structure: [tokens, frames]
1407
+ const attentionHead: Float32Array[] = [] // structure: [tokens, frames]
1405
1408
 
1406
1409
  for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) {
1407
1410
  const bufferOffset = headIndex * frameCount
@@ -1410,7 +1413,7 @@ export class Whisper {
1410
1413
 
1411
1414
  const framesForHead = qksTensors[tokenIndex].data.slice(startIndexInBuffer, endIndexInBuffer)
1412
1415
 
1413
- attentionHead.push(Array.from(framesForHead as any))
1416
+ attentionHead.push(framesForHead)
1414
1417
  }
1415
1418
 
1416
1419
  attentionHeads.push(attentionHead)
@@ -1435,10 +1438,19 @@ export class Whisper {
1435
1438
  // Normalize all weights in each individual head, if enabled
1436
1439
  if (normalize) {
1437
1440
  for (const head of attentionHeads) {
1438
- const allWeightsForHead = head.flatMap(tokenFrames => tokenFrames)
1441
+ let sumOfAllWeightsForHead = 0
1442
+ let sumOfAllSquaredWeightsForHead = 0
1443
+ let countOfAllWeightsForHead = 0
1444
+
1445
+ for (const tokenFrames of head) {
1446
+ sumOfAllWeightsForHead += sumVector(tokenFrames)
1447
+ sumOfAllSquaredWeightsForHead += sumOfSquaresForVector(tokenFrames)
1448
+ countOfAllWeightsForHead += tokenFrames.length
1449
+ }
1439
1450
 
1440
- const meanOfAllWeightsForHead = meanOfVector(allWeightsForHead)
1441
- const stdDeviationOfAllWeightsForHead = stdDeviationOfVector(allWeightsForHead, 'population', meanOfAllWeightsForHead) + 1e-10
1451
+ const meanOfAllWeightsForHead = sumOfAllWeightsForHead / countOfAllWeightsForHead
1452
+ const varianceOfAllWeightsForHead = sumOfAllSquaredWeightsForHead / countOfAllWeightsForHead
1453
+ const stdDeviationOfAllWeightsForHead = Math.sqrt(varianceOfAllWeightsForHead)
1442
1454
 
1443
1455
  const stdDeviationReciprocal = 1.0 / (stdDeviationOfAllWeightsForHead + 1e-10)
1444
1456
 
@@ -1459,16 +1471,12 @@ export class Whisper {
1459
1471
  }
1460
1472
  }
1461
1473
 
1462
- // Compute the mean for all layers and heads
1463
- const frameMeansForToken: number[][] = []
1464
-
1465
- for (let i = 0; i < tokenCount; i++) {
1466
- const frameMeans = new Array(segmentFrameCount)
1467
-
1468
- frameMeansForToken.push(frameMeans)
1469
- }
1474
+ // Compute the mean of the selected attention heads for all layers
1475
+ const frameMeansForToken: Float32Array[] = []
1470
1476
 
1471
1477
  for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) {
1478
+ const meansForFrames = new Float32Array(segmentFrameCount)
1479
+
1472
1480
  for (let frameIndex = 0; frameIndex < segmentFrameCount; frameIndex++) {
1473
1481
  let sum = 0
1474
1482
 
@@ -1478,8 +1486,10 @@ export class Whisper {
1478
1486
 
1479
1487
  const frameMean = sum / attentionHeads.length
1480
1488
 
1481
- frameMeansForToken[tokenIndex][frameIndex] = frameMean
1489
+ meansForFrames[frameIndex] = frameMean
1482
1490
  }
1491
+
1492
+ frameMeansForToken.push(meansForFrames)
1483
1493
  }
1484
1494
 
1485
1495
  // Anchor timestamp tokens timestamps to their original values, if enabled
@@ -1499,7 +1509,7 @@ export class Whisper {
1499
1509
  }
1500
1510
  }
1501
1511
 
1502
- // Perform DTW
1512
+ // Perform DTW to align tokens indexes to frame indexes
1503
1513
  const tokenIndexes = getIntegerRange(0, tokenCount)
1504
1514
  const frameIndexes = getIntegerRange(0, segmentFrameCount)
1505
1515
 
@@ -1773,6 +1783,14 @@ export class Whisper {
1773
1783
  return alignmentHeadsIndexes[this.modelName]
1774
1784
  }
1775
1785
 
1786
+ get defaultTimestampAccuracy() {
1787
+ if (this.modelName.startsWith('tiny') || this.modelName.startsWith('base')) {
1788
+ return 'high'
1789
+ } else {
1790
+ return 'medium'
1791
+ }
1792
+ }
1793
+
1776
1794
  getSuppressedTokens() {
1777
1795
  return [
1778
1796
  ...this.getSuppressedTextTokens(),
@@ -2521,7 +2539,7 @@ export const defaultWhisperOptions: WhisperOptions = {
2521
2539
  decodeTimestampTokens: true,
2522
2540
  endTokenThreshold: 0.9,
2523
2541
  includeEndTokenInCandidates: true,
2524
- timestampAccuracy: 'medium',
2542
+ timestampAccuracy: undefined,
2525
2543
  encoderProvider: undefined,
2526
2544
  decoderProvider: undefined,
2527
2545
  seed: undefined,
@@ -2542,7 +2560,7 @@ export const defaultWhisperAlignmentOptions: WhisperAlignmentOptions = {
2542
2560
  model: undefined,
2543
2561
  endTokenThreshold: 0.9,
2544
2562
  maxTokensPerPart: 250,
2545
- timestampAccuracy: 'medium',
2563
+ timestampAccuracy: undefined,
2546
2564
 
2547
2565
  encoderProvider: undefined,
2548
2566
  decoderProvider: undefined,
@@ -0,0 +1,26 @@
1
+ import { DtwGranularity, getMfccOptionsForGranularity } from "../alignment/SpeechAlignment.js";
2
+ import { RawAudio } from "../audio/AudioUtilities.js";
3
+ import { computeMFCCs, extendDefaultMfccOptions, MfccOptions } from "../dsp/MFCC.js";
4
+ import { Logger } from "../utilities/Logger.js";
5
+
6
+ export async function searchSpeech(sourceRawAudio: RawAudio, referenceRawAudio: RawAudio) {
7
+ const logger = new Logger()
8
+
9
+ const granularity: DtwGranularity = 'low'
10
+
11
+ const mfccOptions = extendDefaultMfccOptions({ ...getMfccOptionsForGranularity(granularity), zeroFirstCoefficient: true }) as MfccOptions
12
+
13
+ // Compute reference MFCCs
14
+ logger.start('Compute reference MFCC features')
15
+ const referenceMfccs = await computeMFCCs(referenceRawAudio, mfccOptions)
16
+
17
+ // Compute source MFCCs
18
+ logger.start('Compute source MFCC features')
19
+ const sourceMfccs = await computeMFCCs(sourceRawAudio, mfccOptions)
20
+
21
+ logger.start('Compute source MFCC features')
22
+ }
23
+
24
+ function computeCostMatrix<T, U>(sequence1: T[], sequence2: U[]) {
25
+
26
+ }
@@ -29,7 +29,7 @@ export abstract class RandomGenerator {
29
29
  return result
30
30
  }
31
31
 
32
- getNormallyDistributedVector(elementCount: number, meanVector: number[], standardDeviationVector: number[]) {
32
+ getNormallyDistributedVector(elementCount: number, meanVector: ArrayLike<number>, standardDeviationVector: ArrayLike<number>) {
33
33
  const features = this.getNormallyDistributedValues(elementCount)
34
34
 
35
35
  for (let i = 0; i < features.length; i++) {
@@ -78,7 +78,7 @@ export abstract class RandomGenerator {
78
78
  return [n1, n2]
79
79
  }
80
80
 
81
- selectRandomIndexFromDistribution(distribution: number[]) {
81
+ selectRandomIndexFromDistribution(distribution: ArrayLike<number>) {
82
82
  const sum = sumVector(distribution)
83
83
 
84
84
  const randomTarget = this.getFloatInRange(0, sum)