echogarden 1.3.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/schemas/options.json +3 -0
- package/dist/api/LanguageDetection.js +1 -1
- package/dist/api/LanguageDetection.js.map +1 -1
- package/dist/api/Recognition.js +1 -1
- package/dist/api/Recognition.js.map +1 -1
- package/dist/api/Translation.js +1 -1
- package/dist/api/Translation.js.map +1 -1
- package/dist/api/TranslationAlignment.js +1 -1
- package/dist/api/TranslationAlignment.js.map +1 -1
- package/dist/audio/AudioBufferConversion.d.ts +2 -2
- package/dist/audio/AudioBufferConversion.js +46 -36
- package/dist/audio/AudioBufferConversion.js.map +1 -1
- package/dist/cli/CLI.js +1 -0
- package/dist/cli/CLI.js.map +1 -1
- package/dist/codecs/FFMpegTranscoder.js +2 -1
- package/dist/codecs/FFMpegTranscoder.js.map +1 -1
- package/dist/codecs/WaveCodec.js +8 -3
- package/dist/codecs/WaveCodec.js.map +1 -1
- package/dist/recognition/WhisperSTT.d.ts +1 -0
- package/dist/recognition/WhisperSTT.js +13 -6
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/utilities/BinaryArrayConversion.d.ts +18 -8
- package/dist/utilities/BinaryArrayConversion.js +63 -36
- package/dist/utilities/BinaryArrayConversion.js.map +1 -1
- package/dist/utilities/Compression.js +1 -1
- package/dist/utilities/Compression.js.map +1 -1
- package/docs/Options.md +1 -0
- package/docs/Tasklist.md +4 -3
- package/package.json +6 -6
- package/src/api/LanguageDetection.ts +1 -1
- package/src/api/Recognition.ts +1 -1
- package/src/api/Translation.ts +1 -1
- package/src/api/TranslationAlignment.ts +1 -1
- package/src/audio/AudioBufferConversion.ts +46 -36
- package/src/cli/CLI.ts +1 -0
- package/src/codecs/FFMpegTranscoder.ts +3 -1
- package/src/codecs/WaveCodec.ts +11 -3
- package/src/recognition/WhisperSTT.ts +14 -7
- package/src/utilities/BinaryArrayConversion.ts +73 -41
- package/src/utilities/Compression.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "echogarden",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.3",
|
|
4
4
|
"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
|
|
5
5
|
"author": "Rotem Dan",
|
|
6
6
|
"license": "GPL-3.0",
|
|
@@ -55,8 +55,8 @@
|
|
|
55
55
|
"echogarden": "./dist/cli/CLILauncher.js"
|
|
56
56
|
},
|
|
57
57
|
"dependencies": {
|
|
58
|
-
"@aws-sdk/client-polly": "^3.
|
|
59
|
-
"@aws-sdk/client-transcribe-streaming": "^3.
|
|
58
|
+
"@aws-sdk/client-polly": "^3.572.0",
|
|
59
|
+
"@aws-sdk/client-transcribe-streaming": "^3.572.0",
|
|
60
60
|
"@echogarden/espeak-ng-emscripten": "^0.1.2",
|
|
61
61
|
"@echogarden/fasttext-wasm": "^0.1.0",
|
|
62
62
|
"@echogarden/flite-wasi": "^0.1.1",
|
|
@@ -89,7 +89,7 @@
|
|
|
89
89
|
"moving-median": "^1.0.0",
|
|
90
90
|
"msgpack-lite": "^0.1.26",
|
|
91
91
|
"onnxruntime-node": "^1.17.3",
|
|
92
|
-
"openai": "^4.
|
|
92
|
+
"openai": "^4.43.0",
|
|
93
93
|
"sam-js": "^0.2.1",
|
|
94
94
|
"strip-ansi": "^7.1.0",
|
|
95
95
|
"tar": "^7.1.0",
|
|
@@ -120,11 +120,11 @@
|
|
|
120
120
|
"@types/graceful-fs": "^4.1.9",
|
|
121
121
|
"@types/jsdom": "^21.1.6",
|
|
122
122
|
"@types/msgpack-lite": "^0.1.11",
|
|
123
|
-
"@types/node": "^20.12.
|
|
123
|
+
"@types/node": "^20.12.11",
|
|
124
124
|
"@types/recursive-readdir": "^2.2.4",
|
|
125
125
|
"@types/tar": "^6.1.13",
|
|
126
126
|
"@types/ws": "^8.5.10",
|
|
127
|
-
"ts-json-schema-generator": "^2.1.1",
|
|
127
|
+
"ts-json-schema-generator": "^2.1.2-next.1",
|
|
128
128
|
"typescript": "^5.4.5"
|
|
129
129
|
}
|
|
130
130
|
}
|
|
@@ -158,7 +158,7 @@ export async function detectSpeechLanguageByParts(sourceRawAudio: RawAudio, getR
|
|
|
158
158
|
const endOffset = Math.min(audioTimeOffset + audioPartDuration, audioDuration)
|
|
159
159
|
const audioPartLength = endOffset - startOffset
|
|
160
160
|
|
|
161
|
-
logger.logTitledMessage(`\
|
|
161
|
+
logger.logTitledMessage(`\nDetect speech language starting at audio offset`, `${startOffset.toFixed(1)}`, chalk.magentaBright)
|
|
162
162
|
const audioPart = sliceRawAudioByTime(sourceRawAudio, startOffset, endOffset)
|
|
163
163
|
|
|
164
164
|
const resultsForPart = await getResultsForAudioPart(audioPart)
|
package/src/api/Recognition.ts
CHANGED
|
@@ -72,7 +72,7 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
|
|
|
72
72
|
logger.end()
|
|
73
73
|
logger.logTitledMessage('Language specified', formatLanguageCodeWithName(options.language))
|
|
74
74
|
} else {
|
|
75
|
-
logger.start('No language specified.
|
|
75
|
+
logger.start('No language specified. Detect speech language')
|
|
76
76
|
const { detectedLanguage } = await API.detectSpeechLanguage(sourceRawAudio, options.languageDetection!)
|
|
77
77
|
|
|
78
78
|
options.language = detectedLanguage
|
package/src/api/Translation.ts
CHANGED
|
@@ -72,7 +72,7 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
|
|
|
72
72
|
logger.end()
|
|
73
73
|
logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(options.sourceLanguage))
|
|
74
74
|
} else {
|
|
75
|
-
logger.start('No source language specified.
|
|
75
|
+
logger.start('No source language specified. Detect speech language')
|
|
76
76
|
const { detectedLanguage } = await detectSpeechLanguage(sourceRawAudio, options.languageDetection || {})
|
|
77
77
|
|
|
78
78
|
options.sourceLanguage = detectedLanguage
|
|
@@ -68,7 +68,7 @@ export async function alignTranslation(input: AudioSourceParam, transcript: stri
|
|
|
68
68
|
logger.end()
|
|
69
69
|
logger.logTitledMessage('Source language specified', formatLanguageCodeWithName(sourceLanguage))
|
|
70
70
|
} else {
|
|
71
|
-
logger.start('No source language specified.
|
|
71
|
+
logger.start('No source language specified. Detect speech language')
|
|
72
72
|
const { detectedLanguage } = await API.detectSpeechLanguage(sourceRawAudio, options.languageDetection || {})
|
|
73
73
|
|
|
74
74
|
sourceLanguage = detectedLanguage
|
|
@@ -8,34 +8,36 @@ import { BitDepth, SampleFormat } from '../codecs/WaveCodec.js'
|
|
|
8
8
|
export function encodeToAudioBuffer(audioChannels: Float32Array[], targetBitDepth: BitDepth = 16, targetSampleFormat: SampleFormat = SampleFormat.PCM) {
|
|
9
9
|
const interleavedChannels = interleaveChannels(audioChannels)
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
audioChannels = [] // Zero the array references to allow the GC to free up memory, if possible
|
|
12
|
+
|
|
13
|
+
if (targetSampleFormat === SampleFormat.PCM) {
|
|
14
|
+
if (targetBitDepth === 8) {
|
|
15
|
+
return BinaryArrayConversion.int8ToBuffer(float32ToInt8Pcm(interleavedChannels))
|
|
16
|
+
} else if (targetBitDepth === 16) {
|
|
15
17
|
return BinaryArrayConversion.int16ToBufferLE(float32ToInt16Pcm(interleavedChannels))
|
|
16
|
-
} else if (targetBitDepth
|
|
18
|
+
} else if (targetBitDepth === 24) {
|
|
17
19
|
return BinaryArrayConversion.int24ToBufferLE(float32ToInt24Pcm(interleavedChannels))
|
|
18
|
-
} else if (targetBitDepth
|
|
20
|
+
} else if (targetBitDepth === 32) {
|
|
19
21
|
return BinaryArrayConversion.int32ToBufferLE(float32ToInt32Pcm(interleavedChannels))
|
|
20
22
|
} else {
|
|
21
23
|
throw new Error(`Unsupported PCM bit depth: ${targetBitDepth}`)
|
|
22
24
|
}
|
|
23
|
-
} else if (targetSampleFormat
|
|
24
|
-
if (targetBitDepth
|
|
25
|
+
} else if (targetSampleFormat === SampleFormat.Float) {
|
|
26
|
+
if (targetBitDepth === 32) {
|
|
25
27
|
return BinaryArrayConversion.float32ToBufferLE(interleavedChannels)
|
|
26
|
-
} else if (targetBitDepth
|
|
28
|
+
} else if (targetBitDepth === 64) {
|
|
27
29
|
return BinaryArrayConversion.float64ToBufferLE(BinaryArrayConversion.float32Tofloat64(interleavedChannels))
|
|
28
30
|
} else {
|
|
29
31
|
throw new Error(`Unsupported float bit depth: ${targetBitDepth}`)
|
|
30
32
|
}
|
|
31
|
-
} else if (targetSampleFormat
|
|
32
|
-
if (targetBitDepth
|
|
33
|
+
} else if (targetSampleFormat === SampleFormat.Alaw) {
|
|
34
|
+
if (targetBitDepth === 8) {
|
|
33
35
|
return Buffer.from(AlawMulaw.alaw.encode(float32ToInt16Pcm(interleavedChannels)))
|
|
34
36
|
} else {
|
|
35
37
|
throw new Error(`Unsupported alaw bit depth: ${targetBitDepth}`)
|
|
36
38
|
}
|
|
37
|
-
} else if (targetSampleFormat
|
|
38
|
-
if (targetBitDepth
|
|
39
|
+
} else if (targetSampleFormat === SampleFormat.Mulaw) {
|
|
40
|
+
if (targetBitDepth === 8) {
|
|
39
41
|
return Buffer.from(AlawMulaw.mulaw.encode(float32ToInt16Pcm(interleavedChannels)))
|
|
40
42
|
} else {
|
|
41
43
|
throw new Error(`Unsupported mulaw bit depth: ${targetBitDepth}`)
|
|
@@ -48,34 +50,34 @@ export function encodeToAudioBuffer(audioChannels: Float32Array[], targetBitDept
|
|
|
48
50
|
export function decodeToChannels(audioBuffer: Buffer, channelCount: number, sourceBitDepth: number, sourceSampleFormat: SampleFormat) {
|
|
49
51
|
let interleavedChannels: Float32Array
|
|
50
52
|
|
|
51
|
-
if (sourceSampleFormat
|
|
52
|
-
if (sourceBitDepth
|
|
53
|
-
interleavedChannels =
|
|
54
|
-
} else if (sourceBitDepth
|
|
53
|
+
if (sourceSampleFormat === SampleFormat.PCM) {
|
|
54
|
+
if (sourceBitDepth === 8) {
|
|
55
|
+
interleavedChannels = int8PcmToFloat32(BinaryArrayConversion.bufferToInt8(audioBuffer))
|
|
56
|
+
} else if (sourceBitDepth === 16) {
|
|
55
57
|
interleavedChannels = int16PcmToFloat32(BinaryArrayConversion.bufferLEToInt16(audioBuffer))
|
|
56
|
-
} else if (sourceBitDepth
|
|
58
|
+
} else if (sourceBitDepth === 24) {
|
|
57
59
|
interleavedChannels = int24PcmToFloat32(BinaryArrayConversion.bufferLEToInt24(audioBuffer))
|
|
58
|
-
} else if (sourceBitDepth
|
|
60
|
+
} else if (sourceBitDepth === 32) {
|
|
59
61
|
interleavedChannels = int32PcmToFloat32(BinaryArrayConversion.bufferLEToInt32(audioBuffer))
|
|
60
62
|
} else {
|
|
61
63
|
throw new Error(`Unsupported PCM bit depth: ${sourceBitDepth}`)
|
|
62
64
|
}
|
|
63
|
-
} else if (sourceSampleFormat
|
|
64
|
-
if (sourceBitDepth
|
|
65
|
+
} else if (sourceSampleFormat === SampleFormat.Float) {
|
|
66
|
+
if (sourceBitDepth === 32) {
|
|
65
67
|
interleavedChannels = BinaryArrayConversion.bufferLEToFloat32(audioBuffer)
|
|
66
|
-
} else if (sourceBitDepth
|
|
68
|
+
} else if (sourceBitDepth === 64) {
|
|
67
69
|
interleavedChannels = BinaryArrayConversion.float64Tofloat32(BinaryArrayConversion.bufferLEToFloat64(audioBuffer))
|
|
68
70
|
} else {
|
|
69
71
|
throw new Error(`Unsupported float bit depth: ${sourceBitDepth}`)
|
|
70
72
|
}
|
|
71
|
-
} else if (sourceSampleFormat
|
|
72
|
-
if (sourceBitDepth
|
|
73
|
+
} else if (sourceSampleFormat === SampleFormat.Alaw) {
|
|
74
|
+
if (sourceBitDepth === 8) {
|
|
73
75
|
interleavedChannels = int16PcmToFloat32(AlawMulaw.alaw.decode(audioBuffer))
|
|
74
76
|
} else {
|
|
75
77
|
throw new Error(`Unsupported alaw bit depth: ${sourceBitDepth}`)
|
|
76
78
|
}
|
|
77
|
-
} else if (sourceSampleFormat
|
|
78
|
-
if (sourceBitDepth
|
|
79
|
+
} else if (sourceSampleFormat === SampleFormat.Mulaw) {
|
|
80
|
+
if (sourceBitDepth === 8) {
|
|
79
81
|
interleavedChannels = int16PcmToFloat32(AlawMulaw.mulaw.decode(audioBuffer))
|
|
80
82
|
} else {
|
|
81
83
|
throw new Error(`Unsupported mulaw bit depth: ${sourceBitDepth}`)
|
|
@@ -84,27 +86,29 @@ export function decodeToChannels(audioBuffer: Buffer, channelCount: number, sour
|
|
|
84
86
|
throw new Error(`Unsupported audio format: ${sourceSampleFormat}`)
|
|
85
87
|
}
|
|
86
88
|
|
|
89
|
+
audioBuffer = Buffer.from([]) // Zero the buffer reference to allow the GC to free up memory, if possible
|
|
90
|
+
|
|
87
91
|
return deInterleaveChannels(interleavedChannels, channelCount)
|
|
88
92
|
}
|
|
89
93
|
|
|
90
94
|
// Int8 PCM <-> Float32 conversion
|
|
91
|
-
export function
|
|
95
|
+
export function int8PcmToFloat32(input: Int8Array) {
|
|
92
96
|
const output = new Float32Array(input.length)
|
|
93
97
|
|
|
94
98
|
for (let i = 0; i < input.length; i++) {
|
|
95
|
-
const sample = input[i]
|
|
99
|
+
const sample = input[i]
|
|
96
100
|
output[i] = sample < 0 ? sample / 128 : sample / 127
|
|
97
101
|
}
|
|
98
102
|
|
|
99
103
|
return output
|
|
100
104
|
}
|
|
101
105
|
|
|
102
|
-
export function
|
|
103
|
-
const output = new
|
|
106
|
+
export function float32ToInt8Pcm(input: Float32Array) {
|
|
107
|
+
const output = new Int8Array(input.length)
|
|
104
108
|
|
|
105
109
|
for (let i = 0; i < input.length; i++) {
|
|
106
110
|
const sample = clampFloatSample(input[i])
|
|
107
|
-
output[i] = (
|
|
111
|
+
output[i] = (sample < 0 ? sample * 128 : sample * 127) | 0
|
|
108
112
|
}
|
|
109
113
|
|
|
110
114
|
return output
|
|
@@ -185,11 +189,11 @@ export function float32ToInt32Pcm(input: Float32Array) {
|
|
|
185
189
|
export function interleaveChannels(channels: Float32Array[]) {
|
|
186
190
|
const channelCount = channels.length
|
|
187
191
|
|
|
188
|
-
if (channelCount
|
|
192
|
+
if (channelCount === 0) {
|
|
189
193
|
throw new Error('Empty channel array received')
|
|
190
194
|
}
|
|
191
195
|
|
|
192
|
-
if (channelCount
|
|
196
|
+
if (channelCount === 1) {
|
|
193
197
|
return channels[0]
|
|
194
198
|
}
|
|
195
199
|
|
|
@@ -209,11 +213,11 @@ export function interleaveChannels(channels: Float32Array[]) {
|
|
|
209
213
|
}
|
|
210
214
|
|
|
211
215
|
export function deInterleaveChannels(interleavedChannels: Float32Array, channelCount: number) {
|
|
212
|
-
if (channelCount
|
|
216
|
+
if (channelCount === 0) {
|
|
213
217
|
throw new Error('0 channel count received')
|
|
214
218
|
}
|
|
215
219
|
|
|
216
|
-
if (channelCount
|
|
220
|
+
if (channelCount === 1) {
|
|
217
221
|
return [interleavedChannels]
|
|
218
222
|
}
|
|
219
223
|
|
|
@@ -244,5 +248,11 @@ export function deInterleaveChannels(interleavedChannels: Float32Array, channelC
|
|
|
244
248
|
// Utilities
|
|
245
249
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
246
250
|
export function clampFloatSample(floatSample: number) {
|
|
247
|
-
|
|
251
|
+
if (floatSample < -1.0) {
|
|
252
|
+
return -1.0
|
|
253
|
+
} else if (floatSample > 1.0) {
|
|
254
|
+
return 1.0
|
|
255
|
+
} else {
|
|
256
|
+
return floatSample
|
|
257
|
+
}
|
|
248
258
|
}
|
package/src/cli/CLI.ts
CHANGED
|
@@ -425,6 +425,7 @@ async function speak(operationData: CLIOperationData) {
|
|
|
425
425
|
} else if (sourceFileExtension == 'srt' || sourceFileExtension == 'vtt') {
|
|
426
426
|
const fileContent = await readFile(sourceFile, { encoding: 'utf-8' })
|
|
427
427
|
textSegments = subtitlesToTimeline(fileContent).map(entry => entry.text)
|
|
428
|
+
//textSegments = [subtitlesToText(fileContent)]
|
|
428
429
|
} else if (sourceFileExtension == 'xml' || sourceFileExtension == 'ssml') {
|
|
429
430
|
options.ssml = true
|
|
430
431
|
textSegments = [fileContent]
|
|
@@ -88,7 +88,9 @@ async function transcode_CLI(ffmpegCommand: string, input: string | Buffer, outp
|
|
|
88
88
|
|
|
89
89
|
process.on('close', (exitCode) => {
|
|
90
90
|
if (exitCode == 0) {
|
|
91
|
-
|
|
91
|
+
const concatenatedChunks = Buffer.concat(stdoutChunks)
|
|
92
|
+
|
|
93
|
+
resolve(concatenatedChunks)
|
|
92
94
|
} else {
|
|
93
95
|
reject(`ffmpeg exited with code ${exitCode}`)
|
|
94
96
|
log(stderrOutput)
|
package/src/codecs/WaveCodec.ts
CHANGED
|
@@ -87,13 +87,15 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = true, ignor
|
|
|
87
87
|
throw new Error('A data subchunk was encountered before a format subchunk')
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
+
// If the data chunk is truncated or extended beyond 4 GiB,
|
|
91
|
+
// the data would be read up to the end of the buffer
|
|
90
92
|
if (ignoreOverflowingDataChunks && subChunkSize === 4294967295) {
|
|
91
93
|
subChunkSize = waveData.length - readOffset
|
|
92
94
|
}
|
|
93
95
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
dataBuffers.push(
|
|
96
|
+
const subChunkData = waveData.subarray(readOffset, readOffset + subChunkSize)
|
|
97
|
+
|
|
98
|
+
dataBuffers.push(subChunkData)
|
|
97
99
|
}
|
|
98
100
|
// All sub chunks other than 'data' (e.g. 'LIST', 'fact', 'plst', 'junk' etc.) are ignored
|
|
99
101
|
|
|
@@ -111,6 +113,10 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = true, ignor
|
|
|
111
113
|
throw new Error('No format subchunk was found in the wave file')
|
|
112
114
|
}
|
|
113
115
|
|
|
116
|
+
if (dataBuffers.length === 0) {
|
|
117
|
+
throw new Error('No data subchunks were found in the wave file')
|
|
118
|
+
}
|
|
119
|
+
|
|
114
120
|
const waveFormat = WaveFormat.deserializeFrom(formatSubChunkBodyBuffer)
|
|
115
121
|
|
|
116
122
|
const sampleFormat = waveFormat.sampleFormat
|
|
@@ -120,6 +126,8 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = true, ignor
|
|
|
120
126
|
const speakerPositionMask = waveFormat.speakerPositionMask
|
|
121
127
|
|
|
122
128
|
const concatenatedDataBuffers = Buffer.concat(dataBuffers)
|
|
129
|
+
dataBuffers.length = 0 // Allow the garbage collector to free up memory held by the data buffers
|
|
130
|
+
|
|
123
131
|
const audioChannels = AudioBufferConversion.decodeToChannels(concatenatedDataBuffers, channelCount, bitDepth, sampleFormat)
|
|
124
132
|
|
|
125
133
|
return {
|
|
@@ -442,11 +442,6 @@ export class Whisper {
|
|
|
442
442
|
partTokensConfidence = partTokensConfidence.slice(initialTokens.length)
|
|
443
443
|
partCrossAttentionQKs = partCrossAttentionQKs.slice(initialTokens.length)
|
|
444
444
|
|
|
445
|
-
// Compute compression ratio for part (disabled for now)
|
|
446
|
-
if (false) {
|
|
447
|
-
const compressionRatioForPart = (await getDeflateCompressionMetricsForString(this.tokensToText(partTokens))).ratio
|
|
448
|
-
}
|
|
449
|
-
|
|
450
445
|
// Find alignment path
|
|
451
446
|
const alignmentPath = await this.findAlignmentPathFromQKs(partCrossAttentionQKs, partTokens, 0, segmentFrameCount) //, alignmentHeadsIndexes[this.modelName])
|
|
452
447
|
|
|
@@ -457,8 +452,17 @@ export class Whisper {
|
|
|
457
452
|
allDecodedTokens.push(...partTokens)
|
|
458
453
|
timeline.push(...partTimeline)
|
|
459
454
|
|
|
460
|
-
//
|
|
461
|
-
|
|
455
|
+
// Determine compression ratio for recognized text (normalized to lowercase) of this part
|
|
456
|
+
const compressionRatioForPart = (await getDeflateCompressionMetricsForString(this.tokensToText(partTokens).toLocaleLowerCase())).ratio
|
|
457
|
+
|
|
458
|
+
// If the recognized text isn't too repetitive
|
|
459
|
+
if (compressionRatioForPart < options.repetitionThreshold!) {
|
|
460
|
+
// Set current part tokens as the previous part text tokens
|
|
461
|
+
previousPartTextTokens = partTokens.filter(token => this.isTextToken(token))
|
|
462
|
+
} else {
|
|
463
|
+
// Otherwise, set previous part tokens to an empty array
|
|
464
|
+
previousPartTextTokens = []
|
|
465
|
+
}
|
|
462
466
|
|
|
463
467
|
audioOffset = audioEndOffset
|
|
464
468
|
|
|
@@ -544,6 +548,7 @@ export class Whisper {
|
|
|
544
548
|
autoPromptParts: false,
|
|
545
549
|
maxTokensPerPart: Infinity,
|
|
546
550
|
suppressRepetition: false,
|
|
551
|
+
repetitionThreshold: Infinity,
|
|
547
552
|
decodeTimestampTokens: true,
|
|
548
553
|
endTokenThreshold: whisperAlignmentOptions!.endTokenThreshold!,
|
|
549
554
|
includeEndTokenInCandidates: false,
|
|
@@ -2111,6 +2116,7 @@ export interface WhisperOptions {
|
|
|
2111
2116
|
autoPromptParts?: boolean
|
|
2112
2117
|
maxTokensPerPart?: number
|
|
2113
2118
|
suppressRepetition?: boolean
|
|
2119
|
+
repetitionThreshold?: number
|
|
2114
2120
|
decodeTimestampTokens?: boolean
|
|
2115
2121
|
endTokenThreshold?: number
|
|
2116
2122
|
includeEndTokenInCandidates?: boolean
|
|
@@ -2128,6 +2134,7 @@ export const defaultWhisperOptions: WhisperOptions = {
|
|
|
2128
2134
|
autoPromptParts: true,
|
|
2129
2135
|
maxTokensPerPart: 250,
|
|
2130
2136
|
suppressRepetition: true,
|
|
2137
|
+
repetitionThreshold: 2.4,
|
|
2131
2138
|
decodeTimestampTokens: true,
|
|
2132
2139
|
endTokenThreshold: 0.9,
|
|
2133
2140
|
includeEndTokenInCandidates: true,
|
|
@@ -1,17 +1,30 @@
|
|
|
1
1
|
// Typed arrays to Buffer (little endian) conversions
|
|
2
|
+
//
|
|
3
|
+
// The faster conversion methods (other than the methods for int8) would only work correctly
|
|
4
|
+
// on little-endian architectures, since they assume the byte order of the underlying architecture.
|
|
5
|
+
//
|
|
6
|
+
// Since Echogarden only supports little-endian architectures, this shouldn't matter.
|
|
2
7
|
|
|
3
8
|
// int8 <-> bufferLE
|
|
4
|
-
export function int8ToBuffer(
|
|
5
|
-
|
|
9
|
+
export function int8ToBuffer(int8s: Int8Array) {
|
|
10
|
+
return Buffer.copyBytesFrom(int8s)
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function int8ToBuffer_Slow(int8s: Int8Array) {
|
|
14
|
+
const buffer = Buffer.alloc(int8s.length)
|
|
6
15
|
|
|
7
|
-
for (let i = 0; i <
|
|
8
|
-
buffer[i] =
|
|
16
|
+
for (let i = 0; i < int8s.length; i++) {
|
|
17
|
+
buffer[i] = int8s[i] + 128
|
|
9
18
|
}
|
|
10
19
|
|
|
11
20
|
return buffer
|
|
12
21
|
}
|
|
13
22
|
|
|
14
23
|
export function bufferToInt8(buffer: Buffer) {
|
|
24
|
+
return new Int8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function bufferToInt8_Slow(buffer: Buffer) {
|
|
15
28
|
const result = new Int8Array(buffer.length)
|
|
16
29
|
|
|
17
30
|
for (let i = 0; i < result.length; i++) {
|
|
@@ -22,17 +35,25 @@ export function bufferToInt8(buffer: Buffer) {
|
|
|
22
35
|
}
|
|
23
36
|
|
|
24
37
|
// int16 <-> bufferLE
|
|
25
|
-
export function int16ToBufferLE(
|
|
26
|
-
|
|
38
|
+
export function int16ToBufferLE(int16s: Int16Array) {
|
|
39
|
+
return Buffer.copyBytesFrom(int16s)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function int16ToBufferLE_Slow(int16s: Int16Array) {
|
|
43
|
+
const buffer = Buffer.alloc(int16s.length * 2)
|
|
27
44
|
|
|
28
|
-
for (let i = 0; i <
|
|
29
|
-
buffer.writeInt16LE(
|
|
45
|
+
for (let i = 0; i < int16s.length; i++) {
|
|
46
|
+
buffer.writeInt16LE(int16s[i], i * 2)
|
|
30
47
|
}
|
|
31
48
|
|
|
32
49
|
return buffer
|
|
33
50
|
}
|
|
34
51
|
|
|
35
52
|
export function bufferLEToInt16(buffer: Buffer) {
|
|
53
|
+
return new Int16Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 2)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function bufferLEToInt16_Slow(buffer: Buffer) {
|
|
36
57
|
const result = new Int16Array(buffer.length / 2)
|
|
37
58
|
|
|
38
59
|
for (let i = 0; i < result.length; i++) {
|
|
@@ -43,11 +64,11 @@ export function bufferLEToInt16(buffer: Buffer) {
|
|
|
43
64
|
}
|
|
44
65
|
|
|
45
66
|
// int24 <-> bufferLE (uses int32 for storage)
|
|
46
|
-
export function int24ToBufferLE(
|
|
47
|
-
const buffer = Buffer.alloc(
|
|
67
|
+
export function int24ToBufferLE(int24s: Int32Array) {
|
|
68
|
+
const buffer = Buffer.alloc(int24s.length * 3)
|
|
48
69
|
|
|
49
|
-
for (let i = 0; i <
|
|
50
|
-
const val =
|
|
70
|
+
for (let i = 0; i < int24s.length; i++) {
|
|
71
|
+
const val = int24s[i]
|
|
51
72
|
const encodedVal = val < 0 ? val + 0x1000000 : val
|
|
52
73
|
|
|
53
74
|
buffer[(i * 3) + 0] = (encodedVal >> 0) & 0xff
|
|
@@ -74,17 +95,25 @@ export function bufferLEToInt24(buffer: Buffer) {
|
|
|
74
95
|
}
|
|
75
96
|
|
|
76
97
|
// int32 <-> bufferLE
|
|
77
|
-
export function int32ToBufferLE(
|
|
78
|
-
|
|
98
|
+
export function int32ToBufferLE(int32s: Int32Array) {
|
|
99
|
+
return Buffer.copyBytesFrom(int32s)
|
|
100
|
+
}
|
|
79
101
|
|
|
80
|
-
|
|
81
|
-
|
|
102
|
+
export function int32ToBufferLE_Slow(int32s: Int32Array) {
|
|
103
|
+
const buffer = Buffer.alloc(int32s.length * 4)
|
|
104
|
+
|
|
105
|
+
for (let i = 0; i < int32s.length; i++) {
|
|
106
|
+
buffer.writeInt32LE(int32s[i], i * 4)
|
|
82
107
|
}
|
|
83
108
|
|
|
84
109
|
return buffer
|
|
85
110
|
}
|
|
86
111
|
|
|
87
112
|
export function bufferLEToInt32(buffer: Buffer) {
|
|
113
|
+
return new Int32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function bufferLEToInt32_Slow(buffer: Buffer) {
|
|
88
117
|
const result = new Int32Array(buffer.length / 4)
|
|
89
118
|
|
|
90
119
|
for (let i = 0; i < result.length; i++) {
|
|
@@ -95,17 +124,25 @@ export function bufferLEToInt32(buffer: Buffer) {
|
|
|
95
124
|
}
|
|
96
125
|
|
|
97
126
|
// float32 <-> bufferLE
|
|
98
|
-
export function float32ToBufferLE(
|
|
99
|
-
|
|
127
|
+
export function float32ToBufferLE(float32s: Float32Array) {
|
|
128
|
+
return Buffer.copyBytesFrom(float32s)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function float32ToBufferLE_Slow(float32s: Float32Array) {
|
|
132
|
+
const buffer = Buffer.alloc(float32s.length * 4)
|
|
100
133
|
|
|
101
|
-
for (let i = 0; i <
|
|
102
|
-
buffer.writeFloatLE(
|
|
134
|
+
for (let i = 0; i < float32s.length; i++) {
|
|
135
|
+
buffer.writeFloatLE(float32s[i], i * 4)
|
|
103
136
|
}
|
|
104
137
|
|
|
105
138
|
return buffer
|
|
106
139
|
}
|
|
107
140
|
|
|
108
141
|
export function bufferLEToFloat32(buffer: Buffer) {
|
|
142
|
+
return new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function bufferLEToFloat32_Slow(buffer: Buffer) {
|
|
109
146
|
const result = new Float32Array(buffer.length / 4)
|
|
110
147
|
|
|
111
148
|
for (let i = 0; i < result.length; i++) {
|
|
@@ -116,17 +153,25 @@ export function bufferLEToFloat32(buffer: Buffer) {
|
|
|
116
153
|
}
|
|
117
154
|
|
|
118
155
|
// float64 <-> bufferLE
|
|
119
|
-
export function float64ToBufferLE(
|
|
120
|
-
|
|
156
|
+
export function float64ToBufferLE(float64s: Float64Array) {
|
|
157
|
+
return Buffer.copyBytesFrom(float64s)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export function float64ToBufferLE_Slow(float64s: Float64Array) {
|
|
161
|
+
const buffer = Buffer.alloc(float64s.length * 8)
|
|
121
162
|
|
|
122
|
-
for (let i = 0; i <
|
|
123
|
-
buffer.writeDoubleLE(
|
|
163
|
+
for (let i = 0; i < float64s.length; i++) {
|
|
164
|
+
buffer.writeDoubleLE(float64s[i], i * 8)
|
|
124
165
|
}
|
|
125
166
|
|
|
126
167
|
return buffer
|
|
127
168
|
}
|
|
128
169
|
|
|
129
170
|
export function bufferLEToFloat64(buffer: Buffer) {
|
|
171
|
+
return new Float64Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 8)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function bufferLEToFloat64_Slow(buffer: Buffer) {
|
|
130
175
|
const result = new Float64Array(buffer.length / 8)
|
|
131
176
|
|
|
132
177
|
for (let i = 0; i < result.length; i++) {
|
|
@@ -137,23 +182,10 @@ export function bufferLEToFloat64(buffer: Buffer) {
|
|
|
137
182
|
}
|
|
138
183
|
|
|
139
184
|
// float64 <-> float32
|
|
140
|
-
export function float64Tofloat32(
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
for (let i = 0; i < doubles.length; i++) {
|
|
144
|
-
floats[i] = doubles[i]
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
return floats
|
|
185
|
+
export function float64Tofloat32(float64s: Float64Array) {
|
|
186
|
+
return Float32Array.from(float64s)
|
|
148
187
|
}
|
|
149
188
|
|
|
150
|
-
export function float32Tofloat64(
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
for (let i = 0; i < floats.length; i++) {
|
|
154
|
-
doubles[i] = floats[i]
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
return doubles
|
|
189
|
+
export function float32Tofloat64(float32s: Float32Array) {
|
|
190
|
+
return Float64Array.from(float32s)
|
|
158
191
|
}
|
|
159
|
-
|
|
@@ -107,6 +107,6 @@ export async function getDeflateCompressionMetricsForString(str: string) {
|
|
|
107
107
|
return {
|
|
108
108
|
originalSize: originalStringBytes.length,
|
|
109
109
|
compressedSize: compressedStringBytes.length,
|
|
110
|
-
ratio:
|
|
110
|
+
ratio: originalStringBytes.length / compressedStringBytes.length
|
|
111
111
|
}
|
|
112
112
|
}
|