echogarden 2.8.5 → 2.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/math/VectorMath.js +3 -1
- package/dist/math/VectorMath.js.map +1 -1
- package/dist/nlp/Segmentation.d.ts +1 -1
- package/dist/nlp/Segmentation.js +4 -3
- package/dist/nlp/Segmentation.js.map +1 -1
- package/dist/recognition/WhisperSTT.d.ts +4 -1
- package/dist/recognition/WhisperSTT.js +55 -29
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/utilities/Timer.d.ts +2 -2
- package/dist/utilities/Timer.js +12 -21
- package/dist/utilities/Timer.js.map +1 -1
- package/dist/utilities/Utilities.d.ts +1 -1
- package/dist/utilities/Utilities.js +15 -3
- package/dist/utilities/Utilities.js.map +1 -1
- package/docs/Licenses.md +0 -2
- package/docs/Tasklist.md +10 -5
- package/package.json +7 -7
- package/src/math/VectorMath.ts +5 -1
- package/src/nlp/Segmentation.ts +6 -3
- package/src/recognition/WhisperSTT.ts +71 -31
- package/src/utilities/Timer.ts +14 -30
- package/src/utilities/Utilities.ts +17 -4
package/docs/Tasklist.md
CHANGED
|
@@ -76,7 +76,8 @@
|
|
|
76
76
|
* `speak-youtube-subtitles`: To speak the subtitles of a YouTube video
|
|
77
77
|
|
|
78
78
|
### API
|
|
79
|
-
*
|
|
79
|
+
* Allow callers from API to cancel a task via `AbortController` and `AbortSignal`
|
|
80
|
+
* Validate timelines to ensure timestamps are always increasing: no undefined timestamps, no negative timestamps, out-of-order timestamps, or timestamps over the duration of the audio. No sentences without words, etc. Missing or incorrect word offsets, etc.
|
|
80
81
|
* Add support for phrases in timelines
|
|
81
82
|
* Accept voice list caching options in `SynthesisOptions`
|
|
82
83
|
|
|
@@ -100,7 +101,7 @@
|
|
|
100
101
|
### Synthesis
|
|
101
102
|
* Option to disable alignment (only for some engines). Alternative: use a low granularity DTW setting that is very fast to compute
|
|
102
103
|
* Find places to add commas (",") to improve speech fluency. VITS voices don't normally add speech breaks if there is no punctuation
|
|
103
|
-
* An isolated dash " - " can be converted to a " , " to ensure there's a break in the speech
|
|
104
|
+
* An isolated dash " - " can maybe be converted to a " , " to ensure there's a break in the speech
|
|
104
105
|
* Find a way to manually reset voice list cache
|
|
105
106
|
* When synthesized text isn't pre-split to sentences, apply sentence splits by using the existing method to convert the output of word timelines to sentence/segment timelines
|
|
106
107
|
* Some `sapi` voices and `msspeech` languages output phones that are converted to Microsoft alphabet, not IPA symbols. Try to see if these can be translated to IPA
|
|
@@ -126,11 +127,15 @@
|
|
|
126
127
|
* Option to add POS tags to timeline, if available
|
|
127
128
|
|
|
128
129
|
### Synthesis / VITS
|
|
129
|
-
* Allow limiting how many models are cached in memory
|
|
130
|
+
* Allow limiting how many models are cached in memory.
|
|
131
|
+
* Ensure that caching behaves correctly when the same model is used with different execution providers
|
|
130
132
|
* Custom model paths (decide how to implement)
|
|
131
133
|
* Pull voice list from JSON file, or based on URL? Is that a good idea?
|
|
132
134
|
* Add speaker names to voice list somehow
|
|
133
135
|
|
|
136
|
+
### Synthesis / Kokoro
|
|
137
|
+
* Ensure that caching behaves correctly when the same model is used with different execution providers
|
|
138
|
+
|
|
134
139
|
### Synthesis / Azure Cognitive Services
|
|
135
140
|
* Currently, when input is set to be SSML, it is wrapped in a `<speak>` tag. Handle the case where the user made their own SSML document wrapped with a `<speak>` tag as well. Currently, it may send invalid input to Azure
|
|
136
141
|
|
|
@@ -139,8 +144,8 @@
|
|
|
139
144
|
|
|
140
145
|
### Recognition / Whisper
|
|
141
146
|
* Whisper's Chinese and Japanese output can be split into words in a more accurate way. Consider using a dedicated segmentation library to perform the segmentation in character sequences that have no punctuation characters to aid on guessing word boundaries
|
|
142
|
-
* Cache last model (if enough memory is available)
|
|
143
|
-
*
|
|
147
|
+
* Cache last model (if enough memory is available). Ensure that caching works when switching between different execution providers
|
|
148
|
+
* Whisper timestamp tokens can be used to split into segments, otherwise it is possible to try to guess using pause lengths or voice activity detection
|
|
144
149
|
* Bring back the option to use eSpeak DTW based alignment on segments, as an alternative approach
|
|
145
150
|
|
|
146
151
|
### Alignment
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "echogarden",
|
|
3
|
-
"version": "2.8.
|
|
3
|
+
"version": "2.8.7",
|
|
4
4
|
"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "GPL-3.0",
|
|
@@ -54,20 +54,20 @@
|
|
|
54
54
|
"dependencies": {
|
|
55
55
|
"@aws-sdk/client-polly": "~3.812.0",
|
|
56
56
|
"@aws-sdk/client-transcribe-streaming": "~3.812.0",
|
|
57
|
-
"@echogarden/audio-io": "~0.4.
|
|
57
|
+
"@echogarden/audio-io": "~0.4.2",
|
|
58
58
|
"@echogarden/espeak-ng-emscripten": "~0.3.3",
|
|
59
59
|
"@echogarden/fasttext-wasm": "~0.1.0",
|
|
60
60
|
"@echogarden/flite-wasi": "~0.1.1",
|
|
61
61
|
"@echogarden/fvad-wasm": "~0.2.0",
|
|
62
62
|
"@echogarden/gnuspeech-wasm": "~0.2.1",
|
|
63
|
-
"@echogarden/icu-segmentation-wasm": "~0.2.
|
|
63
|
+
"@echogarden/icu-segmentation-wasm": "~0.2.2",
|
|
64
64
|
"@echogarden/pffft-wasm": "~0.4.2",
|
|
65
65
|
"@echogarden/rnnoise-wasm": "~0.2.0",
|
|
66
66
|
"@echogarden/rubberband-wasm": "~0.2.0",
|
|
67
67
|
"@echogarden/sonic-wasm": "~0.2.0",
|
|
68
68
|
"@echogarden/speex-resampler-wasm": "~0.3.0",
|
|
69
69
|
"@echogarden/svoxpico-wasm": "~0.2.0",
|
|
70
|
-
"@echogarden/text-segmentation": "~0.3.
|
|
70
|
+
"@echogarden/text-segmentation": "~0.3.3",
|
|
71
71
|
"@echogarden/transformers-nodejs-lite": "~2.17.1-lite.4",
|
|
72
72
|
"@echogarden/wave-codec": "~0.3.3",
|
|
73
73
|
"@mozilla/readability": "~0.6.0",
|
|
@@ -82,10 +82,10 @@
|
|
|
82
82
|
"jsdom": "~26.1.0",
|
|
83
83
|
"json5": "~2.2.3",
|
|
84
84
|
"kuromoji": "~0.1.2",
|
|
85
|
-
"microsoft-cognitiveservices-speech-sdk": "~1.
|
|
85
|
+
"microsoft-cognitiveservices-speech-sdk": "~1.44.0",
|
|
86
86
|
"msgpack-lite": "~0.1.26",
|
|
87
87
|
"onnxruntime-node": "~1.21.1",
|
|
88
|
-
"openai": "~4.
|
|
88
|
+
"openai": "~4.102.0",
|
|
89
89
|
"regexp-composer": "~0.3.0",
|
|
90
90
|
"sam-js": "~0.3.1",
|
|
91
91
|
"strip-ansi": "~7.1.0",
|
|
@@ -114,7 +114,7 @@
|
|
|
114
114
|
"@types/graceful-fs": "~4.1.9",
|
|
115
115
|
"@types/jsdom": "~21.1.7",
|
|
116
116
|
"@types/msgpack-lite": "~0.1.11",
|
|
117
|
-
"@types/node": "~22.15.
|
|
117
|
+
"@types/node": "~22.15.21",
|
|
118
118
|
"@types/tar": "~6.1.13",
|
|
119
119
|
"@types/ws": "~8.18.1",
|
|
120
120
|
"ts-json-schema-generator": "~2.4.0",
|
package/src/math/VectorMath.ts
CHANGED
|
@@ -345,7 +345,11 @@ export function medianOfVector(vector: ArrayLike<number>) {
|
|
|
345
345
|
throw new Error('Vector is empty')
|
|
346
346
|
}
|
|
347
347
|
|
|
348
|
-
|
|
348
|
+
const sortedArray = Array.from(vector).sort((a, b) => a - b)
|
|
349
|
+
|
|
350
|
+
const median = sortedArray[Math.floor(sortedArray.length / 2)]
|
|
351
|
+
|
|
352
|
+
return median
|
|
349
353
|
}
|
|
350
354
|
|
|
351
355
|
export function stdDeviationOfVector(vector: ArrayLike<number>, kind: 'population' | 'sample' = 'population', mean?: number) {
|
package/src/nlp/Segmentation.ts
CHANGED
|
@@ -10,11 +10,14 @@ import { splitJapaneseTextToWords_Kuromoji } from './JapaneseSegmentation.js'
|
|
|
10
10
|
const log = logToStderr
|
|
11
11
|
|
|
12
12
|
export const wordCharacterRegExp = /[\p{Letter}\p{Number}]/u
|
|
13
|
-
|
|
13
|
+
|
|
14
|
+
// See: https://mathiasbynens.be/notes/es-unicode-property-escapes
|
|
15
|
+
export const emojiSequenceRegExp = /\p{Emoji_Modifier_Base}\p{Emoji_Modifier}?|\p{Emoji_Presentation}|\p{Emoji}\uFE0F/u
|
|
16
|
+
|
|
14
17
|
export const punctuationRegExp = /[\p{Punctuation}]/u
|
|
15
18
|
|
|
16
19
|
export const phraseSeparators = [',', '、', ',', '،', ';', ';', ':', ':', '—']
|
|
17
|
-
export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '*', '×', '=', '¼', '½', '¾']
|
|
20
|
+
export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '\\', '^', '*', '×', '=', '¼', '½', '¾']
|
|
18
21
|
|
|
19
22
|
///////////////////////////////////////////////////////////////////////////////////////////////
|
|
20
23
|
// Predicates
|
|
@@ -36,7 +39,7 @@ export function includesPunctuation(str: string) {
|
|
|
36
39
|
}
|
|
37
40
|
|
|
38
41
|
export function includesEmoji(str: string) {
|
|
39
|
-
return
|
|
42
|
+
return emojiSequenceRegExp.test(str?.trim())
|
|
40
43
|
}
|
|
41
44
|
|
|
42
45
|
export function isWhitespace(str: string) {
|
|
@@ -3,7 +3,7 @@ import type * as Onnx from 'onnxruntime-node'
|
|
|
3
3
|
import { Logger } from '../utilities/Logger.js'
|
|
4
4
|
import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
|
|
5
5
|
import { clip, getIntegerRange, getTopKIndexes, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
|
|
6
|
-
import { indexOfMax, logOfVector, logSumExp, meanOfVector, softmax, sumOfSquaresForVector, sumVector } from '../math/VectorMath.js'
|
|
6
|
+
import { indexOfMax, logOfVector, logSumExp, meanOfVector, medianOfVector, softmax, sumOfSquaresForVector, sumVector } from '../math/VectorMath.js'
|
|
7
7
|
|
|
8
8
|
import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
|
|
9
9
|
import { extendDeep } from '../utilities/ObjectUtilities.js'
|
|
@@ -11,7 +11,7 @@ import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
|
|
|
11
11
|
import { AlignmentPath } from '../alignment/SpeechAlignment.js'
|
|
12
12
|
import { getRawAudioDuration, RawAudio, sliceRawAudio } from '../audio/AudioUtilities.js'
|
|
13
13
|
import { readFileAsUtf8 } from '../utilities/FileSystem.js'
|
|
14
|
-
import type
|
|
14
|
+
import { logLevelGreaterOrEqualTo, type LanguageDetectionResults } from '../api/API.js'
|
|
15
15
|
import { formatLanguageCodeWithName, getShortLanguageCode, languageCodeToName } from '../utilities/Locale.js'
|
|
16
16
|
import { loadPackage } from '../utilities/PackageManager.js'
|
|
17
17
|
import chalk from 'chalk'
|
|
@@ -25,6 +25,7 @@ import { dmlProviderAvailable, getOnnxSessionOptions, makeOnnxLikeFloat32Tensor,
|
|
|
25
25
|
import { murmurHash3_int32Input } from '../utilities/Hashing.js'
|
|
26
26
|
import { containsInvalidCodepoint, getTokenRepetitionScore } from '../utilities/StringUtilities.js'
|
|
27
27
|
import { joinPath } from '../utilities/PathUtilities.js'
|
|
28
|
+
import { Timer } from '../utilities/Timer.js'
|
|
28
29
|
|
|
29
30
|
export async function recognize(
|
|
30
31
|
sourceRawAudio: RawAudio,
|
|
@@ -36,7 +37,7 @@ export async function recognize(
|
|
|
36
37
|
|
|
37
38
|
options = extendDeep(defaultWhisperOptions, options)
|
|
38
39
|
|
|
39
|
-
if (sourceRawAudio.sampleRate
|
|
40
|
+
if (sourceRawAudio.sampleRate !== 16000) {
|
|
40
41
|
throw new Error('Source audio must have a sample rate of 16000 Hz')
|
|
41
42
|
}
|
|
42
43
|
|
|
@@ -46,7 +47,7 @@ export async function recognize(
|
|
|
46
47
|
throw new Error(`The language ${formatLanguageCodeWithName(sourceLanguage)} is not supported by the Whisper engine.`)
|
|
47
48
|
}
|
|
48
49
|
|
|
49
|
-
if (isEnglishOnlyModel(modelName) && sourceLanguage
|
|
50
|
+
if (isEnglishOnlyModel(modelName) && sourceLanguage !== 'en') {
|
|
50
51
|
throw new Error(`The model '${modelName}' can only be used with English inputs. However, the given source language was ${languageCodeToName(sourceLanguage)}.`)
|
|
51
52
|
}
|
|
52
53
|
|
|
@@ -60,7 +61,7 @@ export async function recognize(
|
|
|
60
61
|
|
|
61
62
|
// Workaround issue with large-v3-turbo that produces invalid results when a prompt is passed to it.
|
|
62
63
|
// Always disable autoprompting for that model.
|
|
63
|
-
if (options.autoPromptParts && modelName
|
|
64
|
+
if (options.autoPromptParts && modelName === 'large-v3-turbo') {
|
|
64
65
|
options.autoPromptParts = false
|
|
65
66
|
}
|
|
66
67
|
|
|
@@ -96,7 +97,7 @@ export async function align(
|
|
|
96
97
|
|
|
97
98
|
options = extendDeep(defaultWhisperAlignmentOptions, options)
|
|
98
99
|
|
|
99
|
-
if (sourceRawAudio.sampleRate
|
|
100
|
+
if (sourceRawAudio.sampleRate !== 16000) {
|
|
100
101
|
throw new Error('Source audio must have a sample rate of 16000 Hz')
|
|
101
102
|
}
|
|
102
103
|
|
|
@@ -106,7 +107,7 @@ export async function align(
|
|
|
106
107
|
throw new Error(`The language ${formatLanguageCodeWithName(sourceLanguage)} is not supported by the Whisper engine.`)
|
|
107
108
|
}
|
|
108
109
|
|
|
109
|
-
if (isEnglishOnlyModel(modelName) && sourceLanguage
|
|
110
|
+
if (isEnglishOnlyModel(modelName) && sourceLanguage !== 'en') {
|
|
110
111
|
throw new Error(`The model '${modelName}' can only be used with English inputs. However, the given source language was ${languageCodeToName(sourceLanguage)}.`)
|
|
111
112
|
}
|
|
112
113
|
|
|
@@ -139,7 +140,7 @@ export async function alignEnglishTranslation(
|
|
|
139
140
|
|
|
140
141
|
options = extendDeep(defaultWhisperAlignmentOptions, options)
|
|
141
142
|
|
|
142
|
-
if (sourceRawAudio.sampleRate
|
|
143
|
+
if (sourceRawAudio.sampleRate !== 16000) {
|
|
143
144
|
throw new Error('Source audio must have a sample rate of 16000 Hz')
|
|
144
145
|
}
|
|
145
146
|
|
|
@@ -184,7 +185,7 @@ export async function detectLanguage(
|
|
|
184
185
|
|
|
185
186
|
options = extendDeep(defaultWhisperLanguageDetectionOptions, options)
|
|
186
187
|
|
|
187
|
-
if (sourceRawAudio.sampleRate
|
|
188
|
+
if (sourceRawAudio.sampleRate !== 16000) {
|
|
188
189
|
throw new Error('Source audio must have a sample rate of 16000 Hz')
|
|
189
190
|
}
|
|
190
191
|
|
|
@@ -232,7 +233,7 @@ export async function detectVoiceActivity(
|
|
|
232
233
|
|
|
233
234
|
options = extendDeep(defaultWhisperVADOptions, options)
|
|
234
235
|
|
|
235
|
-
if (sourceRawAudio.sampleRate
|
|
236
|
+
if (sourceRawAudio.sampleRate !== 16000) {
|
|
236
237
|
throw new Error('Source audio must have a sample rate of 16000 Hz')
|
|
237
238
|
}
|
|
238
239
|
|
|
@@ -441,7 +442,11 @@ export class Whisper {
|
|
|
441
442
|
let {
|
|
442
443
|
decodedTokens: partTokens,
|
|
443
444
|
decodedTokensConfidence: partTokensConfidence,
|
|
444
|
-
decodedTokensCrossAttentionQKs:
|
|
445
|
+
decodedTokensCrossAttentionQKs: partTokensCrossAttentionQKs,
|
|
446
|
+
|
|
447
|
+
decodedTokensDecodingTime: partTokensDecodingTime,
|
|
448
|
+
decodedTokensInferenceTime: partTokensInferenceTime,
|
|
449
|
+
decodedTokensOverheadTime: partTokensOverheadTime,
|
|
445
450
|
} = await this.decodeTokens(
|
|
446
451
|
audioPartFeatures,
|
|
447
452
|
initialTokens,
|
|
@@ -470,19 +475,19 @@ export class Whisper {
|
|
|
470
475
|
|
|
471
476
|
await logger.startAsync(`Extract timeline for part (timestamp accuracy: ${options.timestampAccuracy!})`)
|
|
472
477
|
|
|
473
|
-
if (partTokens.length
|
|
474
|
-
throw new Error('Unexpected: partTokens.length
|
|
478
|
+
if (partTokens.length !== partTokensCrossAttentionQKs.length) {
|
|
479
|
+
throw new Error('Unexpected: partTokens.length !== partCrossAttentionQKs.length')
|
|
475
480
|
}
|
|
476
481
|
|
|
477
482
|
// Prepare tokens
|
|
478
483
|
partTokens = partTokens.slice(initialTokens.length)
|
|
479
484
|
partTokensConfidence = partTokensConfidence.slice(initialTokens.length)
|
|
480
|
-
|
|
485
|
+
partTokensCrossAttentionQKs = partTokensCrossAttentionQKs.slice(initialTokens.length)
|
|
481
486
|
|
|
482
487
|
// Find alignment path
|
|
483
488
|
let alignmentHeads: number[] | undefined
|
|
484
489
|
|
|
485
|
-
if (options.timestampAccuracy === 'medium' || options.model
|
|
490
|
+
if (options.timestampAccuracy === 'medium' || options.model === 'large-v3-turbo') {
|
|
486
491
|
alignmentHeads = this.alignmentHeadIndexes
|
|
487
492
|
} else if (options.timestampAccuracy === 'high') {
|
|
488
493
|
alignmentHeads = undefined
|
|
@@ -490,7 +495,7 @@ export class Whisper {
|
|
|
490
495
|
throw new Error(`Unsupported timestamp accuracy '${options.timestampAccuracy}', can only be 'medium' or 'high'.`)
|
|
491
496
|
}
|
|
492
497
|
|
|
493
|
-
const alignmentPath = await this.findAlignmentPathFromQKs(
|
|
498
|
+
const alignmentPath = await this.findAlignmentPathFromQKs(partTokensCrossAttentionQKs, partTokens, 0, segmentFrameCount, alignmentHeads)
|
|
494
499
|
|
|
495
500
|
// Generate timeline from alignment path
|
|
496
501
|
const partTimeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, partTokens, segmentStartTime, segmentEndTime, partTokensConfidence)
|
|
@@ -514,6 +519,15 @@ export class Whisper {
|
|
|
514
519
|
audioOffset = audioEndOffset
|
|
515
520
|
|
|
516
521
|
logger.end()
|
|
522
|
+
|
|
523
|
+
if (logLevelGreaterOrEqualTo('trace')) {
|
|
524
|
+
const promptDecodingTime = partTokensDecodingTime[0]
|
|
525
|
+
const medianTokenDecodingTime = medianOfVector(partTokensDecodingTime.slice(1))
|
|
526
|
+
const medianTokenInferenceTime = medianOfVector(partTokensInferenceTime.slice(1))
|
|
527
|
+
const medianOverheadTime = medianOfVector(partTokensOverheadTime.slice(1))
|
|
528
|
+
|
|
529
|
+
logger.log(`${chalk.blueBright('Context')}: ${initialTokens.length + partTokens.length} tokens (${initialTokens.length} prompt, ${partTokens.length} decoded)\n${chalk.blueBright('Prompt decode time')}: ${promptDecodingTime.toFixed(1)}ms\n${chalk.blueBright('Median token decode time')}: ${medianTokenDecodingTime.toFixed(1)}ms (${medianTokenInferenceTime.toFixed(1)}ms inference, ${medianOverheadTime.toFixed(2)}ms overhead)`, 'trace')
|
|
530
|
+
}
|
|
517
531
|
}
|
|
518
532
|
|
|
519
533
|
// Convert token timeline to word timeline
|
|
@@ -812,8 +826,17 @@ export class Whisper {
|
|
|
812
826
|
|
|
813
827
|
const maxTokensPerPart = Math.min(options.maxTokensPerPart!, largestMaximumTokensPerPart)
|
|
814
828
|
|
|
829
|
+
let decodedTokensInferenceTime: number[] = []
|
|
830
|
+
let decodedTokensDecodingTime: number[] = []
|
|
831
|
+
|
|
832
|
+
const tokenDecodingTimeTimer = new Timer()
|
|
833
|
+
|
|
815
834
|
// Start decoding loop
|
|
816
835
|
for (let decodedTokenCount = 0; decodedTokenCount < maxTokensPerPart; decodedTokenCount++) {
|
|
836
|
+
if (decodedTokenCount > 0) {
|
|
837
|
+
decodedTokensDecodingTime.push(tokenDecodingTimeTimer.getElapsedTimeAndRestart())
|
|
838
|
+
}
|
|
839
|
+
|
|
817
840
|
const isInitialState = decodedTokens.length === initialTokens.length
|
|
818
841
|
const atLeastOneTextTokenDecoded = decodedTokens.slice(initialTokens.length).some(token => this.isTextToken(token))
|
|
819
842
|
|
|
@@ -847,9 +870,13 @@ export class Whisper {
|
|
|
847
870
|
offset: offsetTensor
|
|
848
871
|
}
|
|
849
872
|
|
|
850
|
-
|
|
873
|
+
//// Infer with ONNX decoder model
|
|
874
|
+
const tokenInferenceTimeTimer = new Timer()
|
|
875
|
+
|
|
851
876
|
const decoderOutputs = await this.textDecoder!.run(decoderInputs)
|
|
852
877
|
|
|
878
|
+
decodedTokensInferenceTime.push(tokenInferenceTimeTimer.elapsedTime)
|
|
879
|
+
|
|
853
880
|
// Extract decoder model results
|
|
854
881
|
const logitsBuffer = decoderOutputs['logits'].data as Float32Array
|
|
855
882
|
kvCacheTensor = decoderOutputs['output_kv_cache'] as any
|
|
@@ -1142,6 +1169,10 @@ export class Whisper {
|
|
|
1142
1169
|
await yieldToEventLoop()
|
|
1143
1170
|
}
|
|
1144
1171
|
|
|
1172
|
+
if (decodedTokensDecodingTime.length === decodedTokensInferenceTime.length - 1) {
|
|
1173
|
+
decodedTokensDecodingTime.push(tokenDecodingTimeTimer.getElapsedTimeAndRestart())
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1145
1176
|
// If at least two timestamp tokens were decoded and it's not the final part,
|
|
1146
1177
|
// truncate up to the last timestamp token
|
|
1147
1178
|
if (timestampTokenSeenCount >= 2 && !isFinalPart) {
|
|
@@ -1151,8 +1182,13 @@ export class Whisper {
|
|
|
1151
1182
|
decodedTokensTimestampLogits = decodedTokensTimestampLogits.slice(0, sliceEndTokenIndex)
|
|
1152
1183
|
decodedTokensCrossAttentionQKs = decodedTokensCrossAttentionQKs.slice(0, sliceEndTokenIndex)
|
|
1153
1184
|
decodedTokensConfidence = decodedTokensConfidence.slice(0, sliceEndTokenIndex)
|
|
1185
|
+
|
|
1186
|
+
decodedTokensDecodingTime = decodedTokensDecodingTime.slice(0, sliceEndTokenIndex)
|
|
1187
|
+
decodedTokensInferenceTime = decodedTokensInferenceTime.slice(0, sliceEndTokenIndex)
|
|
1154
1188
|
}
|
|
1155
1189
|
|
|
1190
|
+
const decodedTokensOverheadTime = decodedTokensDecodingTime.map((time, index) => time - decodedTokensInferenceTime[index])
|
|
1191
|
+
|
|
1156
1192
|
logger.write('\n')
|
|
1157
1193
|
logger.end()
|
|
1158
1194
|
|
|
@@ -1162,6 +1198,10 @@ export class Whisper {
|
|
|
1162
1198
|
decodedTokensTimestampLogits,
|
|
1163
1199
|
decodedTokensConfidence,
|
|
1164
1200
|
decodedTokensCrossAttentionQKs,
|
|
1201
|
+
|
|
1202
|
+
decodedTokensDecodingTime,
|
|
1203
|
+
decodedTokensInferenceTime,
|
|
1204
|
+
decodedTokensOverheadTime,
|
|
1165
1205
|
}
|
|
1166
1206
|
}
|
|
1167
1207
|
|
|
@@ -1186,7 +1226,7 @@ export class Whisper {
|
|
|
1186
1226
|
const maxAudioSamples = sampleRate * 30
|
|
1187
1227
|
const maxAudioFrames = 3000
|
|
1188
1228
|
|
|
1189
|
-
if (sampleRate
|
|
1229
|
+
if (sampleRate !== 16000) {
|
|
1190
1230
|
throw new Error('Audio must have a sample rate of 16000 Hz')
|
|
1191
1231
|
}
|
|
1192
1232
|
|
|
@@ -1267,7 +1307,7 @@ export class Whisper {
|
|
|
1267
1307
|
return isSeparatorCharacter(text[text.length - 1])
|
|
1268
1308
|
}
|
|
1269
1309
|
|
|
1270
|
-
if (language
|
|
1310
|
+
if (language !== 'zh' && language !== 'ja') {
|
|
1271
1311
|
tokenTimeline = tokenTimeline.filter(entry => this.isTextToken(entry.id!))
|
|
1272
1312
|
}
|
|
1273
1313
|
|
|
@@ -1282,7 +1322,7 @@ export class Whisper {
|
|
|
1282
1322
|
const text = entry.text
|
|
1283
1323
|
const previousEntryText = previousEntry?.text
|
|
1284
1324
|
|
|
1285
|
-
if (groups.length
|
|
1325
|
+
if (groups.length === 0 ||
|
|
1286
1326
|
text === '' ||
|
|
1287
1327
|
startsWithSeparatorCharacter(text) ||
|
|
1288
1328
|
(previousEntryText != null && endsWithSeparatorCharacter(previousEntryText))) {
|
|
@@ -1346,14 +1386,14 @@ export class Whisper {
|
|
|
1346
1386
|
}
|
|
1347
1387
|
|
|
1348
1388
|
async getTokenTimelineFromAlignmentPath(alignmentPath: AlignmentPath, tokens: number[], startTimeOffset: number, endTimeOffset: number, tokensConfidence?: number[], correctionAmount = 0.0) {
|
|
1349
|
-
if (alignmentPath.length
|
|
1389
|
+
if (alignmentPath.length === 0) {
|
|
1350
1390
|
return []
|
|
1351
1391
|
}
|
|
1352
1392
|
|
|
1353
1393
|
const tokenTimeline: Timeline = []
|
|
1354
1394
|
|
|
1355
1395
|
for (let pathIndex = 0; pathIndex < alignmentPath.length; pathIndex++) {
|
|
1356
|
-
if (pathIndex
|
|
1396
|
+
if (pathIndex !== 0 && alignmentPath[pathIndex].source === alignmentPath[pathIndex - 1].source) {
|
|
1357
1397
|
continue
|
|
1358
1398
|
}
|
|
1359
1399
|
|
|
@@ -1604,7 +1644,7 @@ export class Whisper {
|
|
|
1604
1644
|
|
|
1605
1645
|
const onnxSessionOptions = getOnnxSessionOptions({ executionProviders: this.encoderExecutionProviders })
|
|
1606
1646
|
|
|
1607
|
-
const onnxProvidersString = onnxSessionOptions.executionProviders!.length > 0 ? `${
|
|
1647
|
+
const onnxProvidersString = onnxSessionOptions.executionProviders!.length > 0 ? `${onnxSessionOptions.executionProviders!.join(', ')}` : `default`
|
|
1608
1648
|
|
|
1609
1649
|
await logger.startAsync(`Create encoder inference session for model '${this.modelName}' (ONNX provider: ${onnxProvidersString})`)
|
|
1610
1650
|
|
|
@@ -1642,15 +1682,15 @@ export class Whisper {
|
|
|
1642
1682
|
getKvDimensions(groupCount: number, length: number) {
|
|
1643
1683
|
const modelName = this.modelName
|
|
1644
1684
|
|
|
1645
|
-
if (modelName
|
|
1685
|
+
if (modelName === 'tiny' || modelName === 'tiny.en') {
|
|
1646
1686
|
return [8, groupCount, length, 384]
|
|
1647
|
-
} else if (modelName
|
|
1687
|
+
} else if (modelName === 'base' || modelName === 'base.en') {
|
|
1648
1688
|
return [12, groupCount, length, 512]
|
|
1649
|
-
} else if (modelName
|
|
1689
|
+
} else if (modelName === 'small' || modelName === 'small.en') {
|
|
1650
1690
|
return [24, groupCount, length, 768]
|
|
1651
|
-
} else if (modelName
|
|
1691
|
+
} else if (modelName === 'medium' || modelName === 'medium.en') {
|
|
1652
1692
|
return [48, groupCount, length, 1024]
|
|
1653
|
-
} else if (modelName
|
|
1693
|
+
} else if (modelName === 'large-v1' || modelName === 'large-v2' || modelName === 'large-v3' || modelName === 'large-v3-turbo') {
|
|
1654
1694
|
return [64, groupCount, length, 1280]
|
|
1655
1695
|
} else {
|
|
1656
1696
|
throw new Error(`Unsupported model: ${modelName}`)
|
|
@@ -1664,7 +1704,7 @@ export class Whisper {
|
|
|
1664
1704
|
|
|
1665
1705
|
if (this.isMultiligualModel) {
|
|
1666
1706
|
const languageToken = this.tokenConfig.languageTokensStart + languageIdLookup[language]
|
|
1667
|
-
const taskToken = task
|
|
1707
|
+
const taskToken = task === 'translate' ? this.tokenConfig.translateTaskToken : this.tokenConfig.transcribeTaskToken
|
|
1668
1708
|
|
|
1669
1709
|
tokens = [startOfTextToken, languageToken, taskToken]
|
|
1670
1710
|
} else {
|
|
@@ -1900,7 +1940,7 @@ export async function loadPackagesAndGetPaths(modelName: WhisperModelName | unde
|
|
|
1900
1940
|
if (languageCode) {
|
|
1901
1941
|
const shortLanguageCode = getShortLanguageCode(languageCode)
|
|
1902
1942
|
|
|
1903
|
-
modelName = shortLanguageCode
|
|
1943
|
+
modelName = shortLanguageCode === 'en' ? 'tiny.en' : 'tiny'
|
|
1904
1944
|
} else {
|
|
1905
1945
|
modelName = 'tiny'
|
|
1906
1946
|
}
|
|
@@ -1918,7 +1958,7 @@ export async function loadPackagesAndGetPaths(modelName: WhisperModelName | unde
|
|
|
1918
1958
|
}
|
|
1919
1959
|
|
|
1920
1960
|
export function normalizeWhisperModelName(modelName: WhisperModelName, languageCode: string | undefined): WhisperModelName {
|
|
1921
|
-
if (languageCode
|
|
1961
|
+
if (languageCode !== 'en' && modelName.endsWith('.en')) {
|
|
1922
1962
|
const originalModelName = modelName
|
|
1923
1963
|
modelName = modelName.slice(0, modelName.length - 3) as WhisperModelName
|
|
1924
1964
|
|
package/src/utilities/Timer.ts
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
import { logToStderr, roundToDigits } from './Utilities.js'
|
|
2
2
|
|
|
3
|
-
declare const chrome: any
|
|
4
|
-
declare const process: any
|
|
5
|
-
|
|
6
3
|
export class Timer {
|
|
7
4
|
startTime = 0
|
|
8
5
|
|
|
@@ -46,50 +43,37 @@ export class Timer {
|
|
|
46
43
|
}
|
|
47
44
|
|
|
48
45
|
static get currentTime(): number {
|
|
49
|
-
if (!this.
|
|
50
|
-
this.
|
|
46
|
+
if (!this.getTimestamp) {
|
|
47
|
+
this.createTimestampFunction()
|
|
51
48
|
}
|
|
52
49
|
|
|
53
|
-
return this.
|
|
50
|
+
return this.getTimestamp()
|
|
54
51
|
}
|
|
55
52
|
|
|
56
53
|
static get microsecondTimestamp(): number {
|
|
57
54
|
return Math.floor(Timer.currentTime * 1000)
|
|
58
55
|
}
|
|
59
56
|
|
|
60
|
-
private static
|
|
57
|
+
private static createTimestampFunction() {
|
|
61
58
|
if (typeof process === 'object' && typeof process.hrtime === 'function') {
|
|
62
59
|
let baseTimestamp = 0
|
|
63
60
|
|
|
64
|
-
this.
|
|
65
|
-
const
|
|
66
|
-
const
|
|
61
|
+
this.getTimestamp = () => {
|
|
62
|
+
const nodeTimeNanoSeconds = process.hrtime.bigint()
|
|
63
|
+
const nodeTimeMilliseconds = Number(nodeTimeNanoSeconds) / 1_000_000
|
|
67
64
|
|
|
68
|
-
return baseTimestamp +
|
|
65
|
+
return baseTimestamp + nodeTimeMilliseconds
|
|
69
66
|
}
|
|
70
67
|
|
|
71
|
-
baseTimestamp = Date.now() - this.
|
|
72
|
-
}
|
|
73
|
-
else if (typeof chrome === 'object' && chrome.Interval) {
|
|
74
|
-
const baseTimestamp = Date.now()
|
|
75
|
-
|
|
76
|
-
const chromeIntervalObject = new chrome.Interval()
|
|
77
|
-
chromeIntervalObject.start()
|
|
78
|
-
|
|
79
|
-
this.timestampFunc = () => baseTimestamp + chromeIntervalObject.microseconds() / 1000
|
|
80
|
-
}
|
|
81
|
-
else if (typeof performance === 'object' && performance.now) {
|
|
68
|
+
baseTimestamp = Date.now() - this.getTimestamp()
|
|
69
|
+
} else if (typeof performance === 'object' && performance.now) {
|
|
82
70
|
const baseTimestamp = Date.now() - performance.now()
|
|
83
71
|
|
|
84
|
-
this.
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
this.timestampFunc = () => Date.now()
|
|
88
|
-
}
|
|
89
|
-
else {
|
|
90
|
-
this.timestampFunc = () => (new Date()).getTime()
|
|
72
|
+
this.getTimestamp = () => baseTimestamp + performance.now()
|
|
73
|
+
} else {
|
|
74
|
+
this.getTimestamp = () => Date.now()
|
|
91
75
|
}
|
|
92
76
|
}
|
|
93
77
|
|
|
94
|
-
private static
|
|
78
|
+
private static getTimestamp: () => number
|
|
95
79
|
}
|
|
@@ -8,6 +8,7 @@ import { ChildProcessWithoutNullStreams } from 'node:child_process'
|
|
|
8
8
|
import { inspect } from 'node:util'
|
|
9
9
|
import { TypedArray, TypedArrayConstructor } from '../typings/TypedArray.js'
|
|
10
10
|
import { encodeHex } from '../encodings/Hex.js'
|
|
11
|
+
import { Timer } from './Timer.js'
|
|
11
12
|
|
|
12
13
|
const log = logToStderr
|
|
13
14
|
|
|
@@ -123,9 +124,21 @@ export function roundToDigits(val: number, digits = 3) {
|
|
|
123
124
|
return Math.round(val * multiplier) / multiplier
|
|
124
125
|
}
|
|
125
126
|
|
|
126
|
-
export function
|
|
127
|
-
|
|
128
|
-
|
|
127
|
+
export function sleep(timeMs: number) {
|
|
128
|
+
const timer = new Timer()
|
|
129
|
+
|
|
130
|
+
return new Promise<void>((resolve) => {
|
|
131
|
+
const tickCallback = () => {
|
|
132
|
+
if (timer.elapsedTime < timeMs) {
|
|
133
|
+
//setImmediate(tickCallback)
|
|
134
|
+
setTimeout(tickCallback, 0)
|
|
135
|
+
} else {
|
|
136
|
+
resolve()
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
//setImmediate(tickCallback)
|
|
141
|
+
setTimeout(tickCallback, 0)
|
|
129
142
|
})
|
|
130
143
|
}
|
|
131
144
|
|
|
@@ -440,7 +453,7 @@ export async function runOperationWithRetries<R>(
|
|
|
440
453
|
logger.log('', 'error')
|
|
441
454
|
logger.logTitledMessage(`${operationName} failed`, `Trying again in ${delayBetweenRetries}ms..`, chalk.redBright, 'error')
|
|
442
455
|
|
|
443
|
-
await
|
|
456
|
+
await sleep(delayBetweenRetries)
|
|
444
457
|
|
|
445
458
|
logger.log(``, 'warning')
|
|
446
459
|
logger.logTitledMessage(`Starting retry attempt`, `${retryIndex} / ${maxRetries}`, chalk.yellowBright, 'warning')
|