echogarden 1.3.2 → 1.3.3

package/src/audio/AudioBufferConversion.ts

@@ -8,34 +8,36 @@ import { BitDepth, SampleFormat } from '../codecs/WaveCodec.js'
 export function encodeToAudioBuffer(audioChannels: Float32Array[], targetBitDepth: BitDepth = 16, targetSampleFormat: SampleFormat = SampleFormat.PCM) {
 	const interleavedChannels = interleaveChannels(audioChannels)
 
-	if (targetSampleFormat == SampleFormat.PCM) {
-		if (targetBitDepth == 8) {
-			return Buffer.from(float32ToUint8Pcm(interleavedChannels).buffer)
-		} else if (targetBitDepth == 16) {
+	audioChannels = [] // Zero the array references to allow the GC to free up memory, if possible
+
+	if (targetSampleFormat === SampleFormat.PCM) {
+		if (targetBitDepth === 8) {
+			return BinaryArrayConversion.int8ToBuffer(float32ToInt8Pcm(interleavedChannels))
+		} else if (targetBitDepth === 16) {
 			return BinaryArrayConversion.int16ToBufferLE(float32ToInt16Pcm(interleavedChannels))
-		} else if (targetBitDepth == 24) {
+		} else if (targetBitDepth === 24) {
 			return BinaryArrayConversion.int24ToBufferLE(float32ToInt24Pcm(interleavedChannels))
-		} else if (targetBitDepth == 32) {
+		} else if (targetBitDepth === 32) {
 			return BinaryArrayConversion.int32ToBufferLE(float32ToInt32Pcm(interleavedChannels))
 		} else {
 			throw new Error(`Unsupported PCM bit depth: ${targetBitDepth}`)
 		}
-	} else if (targetSampleFormat == SampleFormat.Float) {
-		if (targetBitDepth == 32) {
+	} else if (targetSampleFormat === SampleFormat.Float) {
+		if (targetBitDepth === 32) {
 			return BinaryArrayConversion.float32ToBufferLE(interleavedChannels)
-		} else if (targetBitDepth == 64) {
+		} else if (targetBitDepth === 64) {
 			return BinaryArrayConversion.float64ToBufferLE(BinaryArrayConversion.float32Tofloat64(interleavedChannels))
 		} else {
 			throw new Error(`Unsupported float bit depth: ${targetBitDepth}`)
 		}
-	} else if (targetSampleFormat == SampleFormat.Alaw) {
-		if (targetBitDepth == 8) {
+	} else if (targetSampleFormat === SampleFormat.Alaw) {
+		if (targetBitDepth === 8) {
 			return Buffer.from(AlawMulaw.alaw.encode(float32ToInt16Pcm(interleavedChannels)))
 		} else {
 			throw new Error(`Unsupported alaw bit depth: ${targetBitDepth}`)
 		}
-	} else if (targetSampleFormat == SampleFormat.Mulaw) {
-		if (targetBitDepth == 8) {
+	} else if (targetSampleFormat === SampleFormat.Mulaw) {
+		if (targetBitDepth === 8) {
 			return Buffer.from(AlawMulaw.mulaw.encode(float32ToInt16Pcm(interleavedChannels)))
 		} else {
 			throw new Error(`Unsupported mulaw bit depth: ${targetBitDepth}`)
@@ -48,34 +50,34 @@ export function encodeToAudioBuffer(audioChannels: Float32Array[], targetBitDept
 export function decodeToChannels(audioBuffer: Buffer, channelCount: number, sourceBitDepth: number, sourceSampleFormat: SampleFormat) {
 	let interleavedChannels: Float32Array
 
-	if (sourceSampleFormat == SampleFormat.PCM) {
-		if (sourceBitDepth == 8) {
-			interleavedChannels = uint8PcmToFloat32(audioBuffer)
-		} else if (sourceBitDepth == 16) {
+	if (sourceSampleFormat === SampleFormat.PCM) {
+		if (sourceBitDepth === 8) {
+			interleavedChannels = int8PcmToFloat32(BinaryArrayConversion.bufferToInt8(audioBuffer))
+		} else if (sourceBitDepth === 16) {
 			interleavedChannels = int16PcmToFloat32(BinaryArrayConversion.bufferLEToInt16(audioBuffer))
-		} else if (sourceBitDepth == 24) {
+		} else if (sourceBitDepth === 24) {
 			interleavedChannels = int24PcmToFloat32(BinaryArrayConversion.bufferLEToInt24(audioBuffer))
-		} else if (sourceBitDepth == 32) {
+		} else if (sourceBitDepth === 32) {
 			interleavedChannels = int32PcmToFloat32(BinaryArrayConversion.bufferLEToInt32(audioBuffer))
 		} else {
 			throw new Error(`Unsupported PCM bit depth: ${sourceBitDepth}`)
 		}
-	} else if (sourceSampleFormat == SampleFormat.Float) {
-		if (sourceBitDepth == 32) {
+	} else if (sourceSampleFormat === SampleFormat.Float) {
+		if (sourceBitDepth === 32) {
 			interleavedChannels = BinaryArrayConversion.bufferLEToFloat32(audioBuffer)
-		} else if (sourceBitDepth == 64) {
+		} else if (sourceBitDepth === 64) {
 			interleavedChannels = BinaryArrayConversion.float64Tofloat32(BinaryArrayConversion.bufferLEToFloat64(audioBuffer))
 		} else {
 			throw new Error(`Unsupported float bit depth: ${sourceBitDepth}`)
 		}
-	} else if (sourceSampleFormat == SampleFormat.Alaw) {
-		if (sourceBitDepth == 8) {
+	} else if (sourceSampleFormat === SampleFormat.Alaw) {
+		if (sourceBitDepth === 8) {
 			interleavedChannels = int16PcmToFloat32(AlawMulaw.alaw.decode(audioBuffer))
 		} else {
 			throw new Error(`Unsupported alaw bit depth: ${sourceBitDepth}`)
 		}
-	} else if (sourceSampleFormat == SampleFormat.Mulaw) {
-		if (sourceBitDepth == 8) {
+	} else if (sourceSampleFormat === SampleFormat.Mulaw) {
+		if (sourceBitDepth === 8) {
 			interleavedChannels = int16PcmToFloat32(AlawMulaw.mulaw.decode(audioBuffer))
 		} else {
 			throw new Error(`Unsupported mulaw bit depth: ${sourceBitDepth}`)
@@ -84,27 +86,29 @@ export function decodeToChannels(audioBuffer: Buffer, channelCount: number, sour
 		throw new Error(`Unsupported audio format: ${sourceSampleFormat}`)
 	}
 
+	audioBuffer = Buffer.from([]) // Zero the buffer reference to allow the GC to free up memory, if possible
+
 	return deInterleaveChannels(interleavedChannels, channelCount)
 }
 
 // Int8 PCM <-> Float32 conversion
-export function uint8PcmToFloat32(input: Uint8Array) {
+export function int8PcmToFloat32(input: Int8Array) {
 	const output = new Float32Array(input.length)
 
 	for (let i = 0; i < input.length; i++) {
-		const sample = input[i] - 128
+		const sample = input[i]
 		output[i] = sample < 0 ? sample / 128 : sample / 127
 	}
 
 	return output
 }
 
-export function float32ToUint8Pcm(input: Float32Array) {
-	const output = new Uint8Array(input.length)
+export function float32ToInt8Pcm(input: Float32Array) {
+	const output = new Int8Array(input.length)
 
 	for (let i = 0; i < input.length; i++) {
 		const sample = clampFloatSample(input[i])
-		output[i] = ((sample < 0 ? sample * 128 : sample * 127) | 0) + 128
+		output[i] = (sample < 0 ? sample * 128 : sample * 127) | 0
 	}
 
 	return output
@@ -185,11 +189,11 @@ export function float32ToInt32Pcm(input: Float32Array) {
 export function interleaveChannels(channels: Float32Array[]) {
 	const channelCount = channels.length
 
-	if (channelCount == 0) {
+	if (channelCount === 0) {
 		throw new Error('Empty channel array received')
 	}
 
-	if (channelCount == 1) {
+	if (channelCount === 1) {
 		return channels[0]
 	}
 
@@ -209,11 +213,11 @@ export function interleaveChannels(channels: Float32Array[]) {
 }
 
 export function deInterleaveChannels(interleavedChannels: Float32Array, channelCount: number) {
-	if (channelCount == 0) {
+	if (channelCount === 0) {
 		throw new Error('0 channel count received')
 	}
 
-	if (channelCount == 1) {
+	if (channelCount === 1) {
 		return [interleavedChannels]
 	}
 
@@ -244,5 +248,11 @@ export function deInterleaveChannels(interleavedChannels: Float32Array, channelC
 // Utilities
 /////////////////////////////////////////////////////////////////////////////////////////////
 export function clampFloatSample(floatSample: number) {
-	return Math.max(-1, Math.min(floatSample, 1))
+	if (floatSample < -1.0) {
+		return -1.0
+	} else if (floatSample > 1.0) {
+		return 1.0
+	} else {
+		return floatSample
+	}
 }

package/src/cli/CLI.ts

@@ -425,6 +425,7 @@ async function speak(operationData: CLIOperationData) {
 		} else if (sourceFileExtension == 'srt' || sourceFileExtension == 'vtt') {
 			const fileContent = await readFile(sourceFile, { encoding: 'utf-8' })
 			textSegments = subtitlesToTimeline(fileContent).map(entry => entry.text)
+			//textSegments = [subtitlesToText(fileContent)]
 		} else if (sourceFileExtension == 'xml' || sourceFileExtension == 'ssml') {
 			options.ssml = true
 			textSegments = [fileContent]

package/src/codecs/FFMpegTranscoder.ts

@@ -88,7 +88,9 @@ async function transcode_CLI(ffmpegCommand: string, input: string | Buffer, outp
 
 		process.on('close', (exitCode) => {
 			if (exitCode == 0) {
-				resolve(Buffer.concat(stdoutChunks))
+				const concatenatedChunks = Buffer.concat(stdoutChunks)
+
+				resolve(concatenatedChunks)
 			} else {
 				reject(`ffmpeg exited with code ${exitCode}`)
 				log(stderrOutput)

package/src/codecs/WaveCodec.ts

@@ -87,13 +87,15 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = true, ignor
 				throw new Error('A data subchunk was encountered before a format subchunk')
 			}
 
+			// If the data chunk is truncated or extended beyond 4 GiB,
+			// the data would be read up to the end of the buffer
 			if (ignoreOverflowingDataChunks && subChunkSize === 4294967295) {
 				subChunkSize = waveData.length - readOffset
 			}
 
-			// If the data chunk is truncated, but truncations are ignored,
-			// it would be read up to the end of the buffer
-			dataBuffers.push(waveData.subarray(readOffset, readOffset + subChunkSize))
+			const subChunkData = waveData.subarray(readOffset, readOffset + subChunkSize)
+
+			dataBuffers.push(subChunkData)
 		}
 		// All sub chunks other than 'data' (e.g. 'LIST', 'fact', 'plst', 'junk' etc.) are ignored
 
@@ -111,6 +113,10 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = true, ignor
 		throw new Error('No format subchunk was found in the wave file')
 	}
 
+	if (dataBuffers.length === 0) {
+		throw new Error('No data subchunks were found in the wave file')
+	}
+
 	const waveFormat = WaveFormat.deserializeFrom(formatSubChunkBodyBuffer)
 
 	const sampleFormat = waveFormat.sampleFormat
@@ -120,6 +126,8 @@ export function decodeWave(waveData: Buffer, ignoreTruncatedChunks = true, ignor
 	const speakerPositionMask = waveFormat.speakerPositionMask
 
 	const concatenatedDataBuffers = Buffer.concat(dataBuffers)
+	dataBuffers.length = 0 // Allow the garbage collector to free up memory held by the data buffers
+
 	const audioChannels = AudioBufferConversion.decodeToChannels(concatenatedDataBuffers, channelCount, bitDepth, sampleFormat)
 
 	return {

package/src/recognition/WhisperSTT.ts

@@ -442,11 +442,6 @@ export class Whisper {
 			partTokensConfidence = partTokensConfidence.slice(initialTokens.length)
 			partCrossAttentionQKs = partCrossAttentionQKs.slice(initialTokens.length)
 
-			// Compute compression ratio for part (disabled for now)
-			if (false) {
-				const compressionRatioForPart = (await getDeflateCompressionMetricsForString(this.tokensToText(partTokens))).ratio
-			}
-
 			// Find alignment path
 			const alignmentPath = await this.findAlignmentPathFromQKs(partCrossAttentionQKs, partTokens, 0, segmentFrameCount) //, alignmentHeadsIndexes[this.modelName])
 
@@ -457,8 +452,17 @@ export class Whisper {
 			allDecodedTokens.push(...partTokens)
 			timeline.push(...partTimeline)
 
-			// Update previous text tokens
-			previousPartTextTokens = partTokens.filter(token => this.isTextToken(token))
+			// Determine compression ratio for recognized text (normalized to lowercase) of this part
+			const compressionRatioForPart = (await getDeflateCompressionMetricsForString(this.tokensToText(partTokens).toLocaleLowerCase())).ratio
+
+			// If the recognized text isn't too repetitive
+			if (compressionRatioForPart < options.repetitionThreshold!) {
+				// Set current part tokens as the previous part text tokens
+				previousPartTextTokens = partTokens.filter(token => this.isTextToken(token))
+			} else {
+				// Otherwise, set previous part tokens to an empty array
+				previousPartTextTokens = []
+			}
 
 			audioOffset = audioEndOffset
 
@@ -544,6 +548,7 @@ export class Whisper {
 			autoPromptParts: false,
 			maxTokensPerPart: Infinity,
 			suppressRepetition: false,
+			repetitionThreshold: Infinity,
 			decodeTimestampTokens: true,
 			endTokenThreshold: whisperAlignmentOptions!.endTokenThreshold!,
 			includeEndTokenInCandidates: false,
@@ -2111,6 +2116,7 @@ export interface WhisperOptions {
 	autoPromptParts?: boolean
 	maxTokensPerPart?: number
 	suppressRepetition?: boolean
+	repetitionThreshold?: number
 	decodeTimestampTokens?: boolean
 	endTokenThreshold?: number
 	includeEndTokenInCandidates?: boolean
@@ -2128,6 +2134,7 @@ export const defaultWhisperOptions: WhisperOptions = {
 	autoPromptParts: true,
 	maxTokensPerPart: 250,
 	suppressRepetition: true,
+	repetitionThreshold: 2.4,
 	decodeTimestampTokens: true,
 	endTokenThreshold: 0.9,
 	includeEndTokenInCandidates: true,

package/src/utilities/Compression.ts

@@ -107,6 +107,6 @@ export async function getDeflateCompressionMetricsForString(str: string) {
 	return {
 		originalSize: originalStringBytes.length,
 		compressedSize: compressedStringBytes.length,
-		ratio: compressedStringBytes.length / originalStringBytes.length
+		ratio: originalStringBytes.length / compressedStringBytes.length
 	}
 }