echogarden 2.10.2 → 3.0.0

package/src/codecs/FFMpegTranscoder.ts

@@ -4,10 +4,11 @@ import { encodeRawAudioToWave, decodeWaveToRawAudio, RawAudio } from '../audio/A
 
 import { Logger } from '../utilities/Logger.js'
 import { commandExists, concatUint8Arrays, isUint8Array, logToStderr } from '../utilities/Utilities.js'
-import { loadPackage } from '../utilities/PackageManager.js'
+import { loadPackage, LoadPackageCallbacks } from '../utilities/PackageManager.js'
 import { getGlobalOption } from '../api/GlobalOptions.js'
 import { existsSync } from '../utilities/FileSystem.js'
 import { joinPath } from '../utilities/PathUtilities.js'
+import { OperationCallbacks } from '../api/Common.js'
 
 const log = logToStderr
 
@@ -23,11 +24,11 @@ export type FFMpegOutputOptions = {
 	customOptions?: string[]
 }
 
-export async function encodeFromChannels(rawAudio: RawAudio, outputOptions: FFMpegOutputOptions) {
-	return transcode(encodeRawAudioToWave(rawAudio), outputOptions)
+export async function encodeFromChannels(rawAudio: RawAudio, outputOptions: FFMpegOutputOptions, callbacks: OperationCallbacks) {
+	return transcode(encodeRawAudioToWave(rawAudio), outputOptions, callbacks)
 }
 
-export async function decodeToChannels(input: string | Uint8Array, outSampleRate?: number, outChannelCount?: number) {
+export async function decodeToChannels(input: string | Uint8Array, outSampleRate: number | undefined, outChannelCount: number | undefined, callbacks: OperationCallbacks) {
 	const outputOptions: FFMpegOutputOptions = {
 		codec: 'pcm_f32le',
 		format: 'wav',
@@ -36,9 +37,10 @@ export async function decodeToChannels(input: string | Uint8Array, outSampleRate
 		audioOnly: true
 	}
 
-	const waveAudio = await transcode(input, outputOptions)
+	const logger = new Logger(callbacks.logLevel)
 
-	const logger = new Logger()
+	logger.start('Transcode with command-line ffmpeg')
+	const waveAudio = await transcode(input, outputOptions, callbacks)
 
 	logger.start(`Convert wave buffer to raw audio`)
 	const { rawAudio } = decodeWaveToRawAudio(waveAudio)
@@ -47,21 +49,18 @@ export async function decodeToChannels(input: string | Uint8Array, outSampleRate
 	return rawAudio
 }
 
-export async function transcode(input: string | Uint8Array, outputOptions: FFMpegOutputOptions) {
-	const executablePath = await getFFMpegExecutablePath()
+export async function transcode(input: string | Uint8Array, outputOptions: FFMpegOutputOptions, callbacks: OperationCallbacks) {
+	const executablePath = await resolveFFMpegExecutable(callbacks)
 
 	if (!executablePath) {
 		throw new Error(`The ffmpeg utility wasn't found. Please ensure it is available on the system path.`)
 	}
 
-	return transcode_CLI(executablePath, input, outputOptions)
+	return transcode_CLI(executablePath, input, outputOptions, callbacks)
 }
 
-async function transcode_CLI(ffmpegCommand: string, input: string | Uint8Array, outputOptions: FFMpegOutputOptions) {
+async function transcode_CLI(ffmpegCommand: string, input: string | Uint8Array, outputOptions: FFMpegOutputOptions, callbacks: OperationCallbacks) {
 	return new Promise<Uint8Array>((resolve, reject) => {
-		const logger = new Logger()
-		logger.start('Transcode with command-line ffmpeg')
-
 		const args = buildCommandLineArguments(isUint8Array(input) ? '-' : input, outputOptions)
 
 		const process = spawn(ffmpegCommand, args)
@@ -101,8 +100,11 @@ async function transcode_CLI(ffmpegCommand: string, input: string | Uint8Array,
 				reject(`ffmpeg exited with code ${exitCode}`)
 				log(stderrOutput)
 			}
+		})
 
-			logger.end()
+		callbacks?.abortSignal?.addEventListener('abort', () => {
+			reject(new DOMException('FFMpeg CLI was aborted.', 'AbortError'))
+			process.kill()
 		})
 	})
 }
@@ -159,7 +161,7 @@ function buildCommandLineArguments(inputFilename: string, outputOptions: FFMpegO
 	return args
 }
 
-async function getFFMpegExecutablePath() {
+async function resolveFFMpegExecutable(callbacks: LoadPackageCallbacks) {
 	// If a global option set for the path, use it
 	if (getGlobalOption('ffmpegPath')) {
 		return getGlobalOption('ffmpegPath')
@@ -200,7 +202,7 @@ async function getFFMpegExecutablePath() {
 		return undefined
 	}
 
-	const ffmpegPackagePath = await loadPackage(packageName)
+	const ffmpegPackagePath = await loadPackage(packageName, callbacks)
 
 	let filename = packageName
 

package/src/denoising/NSNet2.ts

@@ -7,13 +7,14 @@ import { joinPath } from '../utilities/PathUtilities.js'
 import { stftr, stiftr } from '../dsp/FFT.js'
 import { clip, concatFloat32Arrays } from '../utilities/Utilities.js'
 import { Logger } from '../utilities/Logger.js'
+import { DenoisingCallbacks } from '../api/Denoising.js'
 
-export async function denoiseAudio(rawAudio: RawAudio, options: NSNet2Options) {
+export async function denoiseAudio(rawAudio: RawAudio, options: NSNet2Options, callbacks: DenoisingCallbacks) {
 	const onnxExecutionProviders: OnnxExecutionProvider[] = options.provider ? [options.provider] : []//['dml', 'cpu']
 
 	const denoiser = new NSNet2(options.model!, options.modelDirectoryPath!, onnxExecutionProviders, options.maxAttenuation!)
 
-	const result = await denoiser.denoiseAudio(rawAudio)
+	const result = await denoiser.denoiseAudio(rawAudio, callbacks)
 
 	return result
 }
@@ -28,8 +29,8 @@ export class NSNet2 {
 		public readonly maxAttenuation: number) {
 	}
 
-	async denoiseAudio(rawAudio: RawAudio) {
-		const logger = new Logger()
+	async denoiseAudio(rawAudio: RawAudio, callbacks: DenoisingCallbacks) {
+		const logger = new Logger(callbacks.logLevel)
 
 		logger.start(`Initialize ONNX model ${this.modelName}`)
 		await this.initializeIfNeeded()

package/src/denoising/RNNoise.ts

@@ -1,13 +1,14 @@
 import { float32ToInt16Pcm } from '../audio/AudioBufferConversion.js'
-import { concatFloat32Arrays } from '../utilities/Utilities.js'
-import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
+import { concatFloat32Arrays, throwIfAborted } from '../utilities/Utilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { RawAudio, cloneRawAudio } from '../audio/AudioUtilities.js'
+import { DenoisingCallbacks } from '../api/Denoising.js'
 
 let rnnoiseInstance: any
 
-export async function denoiseAudio(rawAudio: RawAudio) {
-	const logger = new Logger()
+export async function denoiseAudio(rawAudio: RawAudio, callbacks: DenoisingCallbacks) {
+	const logger = new Logger(callbacks.logLevel)
+
 	if (rawAudio.sampleRate != 48000) {
 		throw new Error(`RNNoise requires a 48000 Hz sample rate (${rawAudio.sampleRate} Hz given)`)
 	}
@@ -24,6 +25,7 @@ export async function denoiseAudio(rawAudio: RawAudio) {
 	const m = await getRnnoiseInstance()
 
 	logger.start('Process with RNNoise')
+	const { wrapEmscriptenModuleHeap } = await import('wasm-heap-manager')
 	const wasmHeap = wrapEmscriptenModuleHeap(m)
 
 	const stateSize = m._rnnoise_get_size()
@@ -46,7 +48,14 @@ export async function denoiseAudio(rawAudio: RawAudio) {
 		frameVadProbabilities.push(vadProbability)
 	}
 
+	function destroyState() {
+		m._rnnoise_destroy(denoiseState)
+		wasmHeap.freeAll()
+	}
+
 	for (let readOffset = 0; readOffset < int16Samples.length; readOffset += frameSize) {
+		throwIfAborted(callbacks.abortSignal, destroyState)
+
 		let frame: Float32Array<ArrayBufferLike> = int16SamplesAsFloats.subarray(readOffset, readOffset + frameSize)
 
 		if (frame.length < frameSize) {
@@ -69,8 +78,7 @@ export async function denoiseAudio(rawAudio: RawAudio) {
 	const lastFrameVadProbability = m._rnnoise_process_frame(denoiseState, outputRef.address, inputRef.address)
 	outputNewFrame(outputRef.view.slice(), lastFrameVadProbability)
 
-	m._rnnoise_destroy(denoiseState)
-	wasmHeap.freeAll()
+	destroyState()
 
 	const int16DenoisedSamplesAsFloats = concatFloat32Arrays(processedFrames)
 

package/src/dsp/FFT.ts

@@ -1,6 +1,5 @@
 import { ComplexNumber } from '../math/VectorMath.js'
 import { concatFloat32Arrays, isWasmSimdSupported } from '../utilities/Utilities.js'
-import { createWasmHeapManager } from 'wasm-heap-manager'
 
 // Compute short-term Fourier transform (real-valued)
 export async function stftr(samples: Float32Array, fftOrder: number, windowSize: number, hopSize: number, windowType: WindowType) {
@@ -38,6 +37,7 @@ export async function createStftrGenerator(samples: Float32Array, fftOrder: numb
 
 	const m = await getPFFFTInstance(await isPffftSimdSupportedForFFTOrder(fftOrder))
 
+	const { createWasmHeapManager } = await import('wasm-heap-manager')
 	const wasmHeap = createWasmHeapManager(
 		() => m.HEAPU8.buffer,
 		m._pffft_aligned_malloc,
@@ -109,6 +109,7 @@ export async function stiftr(binsForFrames: Float32Array[], fftOrder: number, wi
 
 	const m = await getPFFFTInstance(await isPffftSimdSupportedForFFTOrder(fftOrder))
 
+	const { createWasmHeapManager } = await import('wasm-heap-manager')
 	const wasmHeap = createWasmHeapManager(
 		() => m.HEAPU8.buffer,
 		m._pffft_aligned_malloc,

package/src/dsp/MFCC.ts

@@ -4,16 +4,18 @@ import { resampleAudioSpeex } from './SpeexResampler.js'
 import { computeMelSpectrogram } from './MelSpectrogram.js'
 import { RawAudio, powerToDecibels } from '../audio/AudioUtilities.js'
 import { normalizeVectors } from '../math/VectorMath.js'
+import { OperationCallbacks } from '../api/Common.js'
+
+export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions, callbacks: OperationCallbacks) {
+	const logger = new Logger(callbacks.logLevel)
 
-export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {}) {
-	const logger = new Logger()
 	logger.start('Initialize options')
 
 	if (monoAudio.audioChannels.length != 1) {
 		throw new Error('Audio must be mono')
 	}
 
-	options = extendDefaultMfccOptions(options)
+	options = extendDefaultMfccOptions(options ?? {})
 
 	const analysisSampleRate = options.analysisSampleRate!
 	const featureCount = options.featureCount!
@@ -42,7 +44,7 @@ export async function computeMFCCs(monoAudio: RawAudio, options: MfccOptions = {
 	}
 
 	logger.start('Compute Mel spectrogram')
-	const { melSpectrogram } = await computeMelSpectrogram(resampledAudio, fftOrder, windowSize, hopLength, filterbankCount, lowerFrequencyHz, upperFrequencyHz)
+	const { melSpectrogram } = await computeMelSpectrogram(resampledAudio, fftOrder, windowSize, hopLength, filterbankCount, lowerFrequencyHz, upperFrequencyHz, undefined, callbacks)
 
 	logger.start('Extract MFCCs from Mel spectrogram')
 	let mfccs: Float32Array<ArrayBufferLike>[] = melSpectrogramToMFCCs(melSpectrogram, featureCount)
@@ -202,6 +204,6 @@ export const defaultMfccOptions: MfccOptions = {
 	zeroFirstCoefficient: false,
 }
 
-export function extendDefaultMfccOptions(options: MfccOptions) {
+export function extendDefaultMfccOptions(options: MfccOptions): MfccOptions {
 	return extendDeep(defaultMfccOptions, options)
 }

package/src/dsp/MelSpectrogram.ts

@@ -1,11 +1,19 @@
+import { OperationCallbacks } from '../api/Common.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import * as FFT from './FFT.js'
 
-export async function computeMelSpectrogram(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbankCount: number, lowerFrequencyHz: number, upperFrequencyHz: number, windowType: FFT.WindowType = 'hann') {
-	const logger = new Logger()
+export async function computeMelSpectrogram(
+	rawAudio: RawAudio,
+	fftOrder: number,
+	windowSize: number,
+	hopLength: number,
+	melBandCount: number,
+	lowerFrequencyHz: number,
+	upperFrequencyHz: number,
+	windowType: FFT.WindowType = 'hann',
+	callbacks: OperationCallbacks) {
 
-	logger.start('Compute Mel filterbank')
 	const binCount = (fftOrder / 2) + 2
 	const nyquistFrequency = rawAudio.sampleRate / 2
 	const binFrequencies = FFT.getBinFrequencies(binCount, nyquistFrequency)
@@ -13,60 +21,66 @@ export async function computeMelSpectrogram(rawAudio: RawAudio, fftOrder: number
 	const lowerFrequencyMel = hertzToMel(lowerFrequencyHz)
 	const upperFrequencyMel = hertzToMel(upperFrequencyHz)
 
-	const filterbanksCenterFrequencies = getMelFilterbanksCenterFrequencies(filterbankCount, lowerFrequencyMel, upperFrequencyMel)
-	const melFilterbanks = getMelFilterbanks(binFrequencies, filterbanksCenterFrequencies, lowerFrequencyMel, upperFrequencyMel)
+	const filterbankCenterFrequencies = getMelFilterbankCenterFrequencies(melBandCount, lowerFrequencyMel, upperFrequencyMel)
+	const melFilterbank = getMelFilterbank(binFrequencies, filterbankCenterFrequencies, lowerFrequencyMel, upperFrequencyMel)
 
-	logger.end()
-
-	return computeMelSpectrogramUsingFilterbanks(rawAudio, fftOrder, windowSize, hopLength, melFilterbanks, windowType)
+	return computeMelSpectrogramUsingFilterbank(rawAudio, fftOrder, windowSize, hopLength, melFilterbank, windowType, callbacks)
 }
 
-export async function computeMelSpectrogramUsingFilterbanks(rawAudio: RawAudio, fftOrder: number, windowSize: number, hopLength: number, filterbanks: Filterbank[], windowType: FFT.WindowType = 'hann') {
-	const logger = new Logger()
+export async function computeMelSpectrogramUsingFilterbank(
+	rawAudio: RawAudio,
+	fftOrder: number,
+	windowSize: number,
+	hopLength: number,
+	filterbank: Filterbank,
+	windowType: FFT.WindowType = 'hann',
+	callbacks: OperationCallbacks) {
+
+	const logger = new Logger(callbacks.logLevel)
 
 	logger.start('Compute short-time FFTs')
 	const audioSamples = rawAudio.audioChannels[0]
 	const fftFrames = await FFT.stftr(audioSamples, fftOrder, windowSize, hopLength, windowType)
 
 	logger.start('Convert FFT frames to Mel spectrogram')
-	const melSpectrogram = fftFramesToMelSpectrogram(fftFrames, filterbanks)
+	const melSpectrogram = fftFramesToMelSpectrogram(fftFrames, filterbank)
 
 	logger.end()
 
 	return { melSpectrogram, fftFrames }
 }
 
-export function fftFramesToMelSpectrogram(fftFrames: Float32Array[], melFilterbanks: Filterbank[]) {
+export function fftFramesToMelSpectrogram(fftFrames: Float32Array[], melFilterbank: Filterbank) {
 	return fftFrames.map(fftFrame => {
 		const powerSpectrum = FFT.fftFrameToPowerSpectrum(fftFrame)
 
-		return powerSpectrumToMelSpectrum(powerSpectrum, melFilterbanks)
+		return powerSpectrumToMelSpectrum(powerSpectrum, melFilterbank)
 	})
 }
 
-export function powerSpectrumToMelSpectrum(powerSpectrum: Float32Array, filterbanks: Filterbank[]) {
-	const filterbankCount = filterbanks.length
-	const melSpectrum = new Float32Array(filterbankCount)
+export function powerSpectrumToMelSpectrum(powerSpectrum: Float32Array, filterbank: Filterbank) {
+	const melBandCount = filterbank.length
+	const melSpectrum = new Float32Array(melBandCount)
 
-	for (let melBandIndex = 0; melBandIndex < filterbankCount; melBandIndex++) {
-		const filterbank = filterbanks[melBandIndex]
-		const filterbankStartIndex = filterbank.startIndex
-		const filterbankWeights = filterbank.weights
+	for (let melBandIndex = 0; melBandIndex < melBandCount; melBandIndex++) {
+		const melFilter = filterbank[melBandIndex]
+		const melFilterStartIndex = melFilter.startIndex
+		const melFilterWeights = melFilter.weights
 
-		if (filterbankStartIndex === -1) {
+		if (melFilterStartIndex === -1) {
 			continue
 		}
 
 		let melBandValue = 0
 
-		for (let i = 0; i < filterbankWeights.length; i++) {
-			const powerSpectrumIndex = filterbankStartIndex + i
+		for (let i = 0; i < melFilterWeights.length; i++) {
+			const powerSpectrumIndex = melFilterStartIndex + i
 
 			if (powerSpectrumIndex >= powerSpectrum.length) {
 				break
 			}
 
-			const weight = filterbankWeights[i]
+			const weight = melFilterWeights[i]
 			const powerSpectrumValue = powerSpectrum[powerSpectrumIndex]
 
 			melBandValue += weight * powerSpectrumValue
@@ -78,17 +92,20 @@ export function powerSpectrumToMelSpectrum(powerSpectrum: Float32Array, filterba
 	return melSpectrum
 }
 
-export function getMelFilterbanks(powerSpectrumFrequenciesHz: Float32Array, centerFrequenciesMel: Float32Array, lowerFrequencyMel: number, upperFrequencyMel: number) {
-	const filterbankCount = centerFrequenciesMel.length
-	const powerSpectrumFrequenciesMel = powerSpectrumFrequenciesHz.map(frequencyHz => hertzToMel(frequencyHz))
+export function getMelFilterbank(
+	powerSpectrumFrequenciesHz: Float32Array,
+	centerFrequenciesMel: Float32Array,
+	lowerFrequencyMel: number,
+	upperFrequencyMel: number) {
 
-	const filterbanks: Filterbank[] = []
+	const melBandCount = centerFrequenciesMel.length
+	const filterbank: Filterbank = []
 
-	for (let filterbankIndex = 0; filterbankIndex < filterbankCount; filterbankIndex++) {
+	for (let filterbankIndex = 0; filterbankIndex < melBandCount; filterbankIndex++) {
 		const centerFrequency = centerFrequenciesMel[filterbankIndex]
 
 		const leftFrequency = filterbankIndex > 0 ? centerFrequenciesMel[filterbankIndex - 1] : lowerFrequencyMel
-		const rightFrequency = filterbankIndex < filterbankCount - 1 ? centerFrequenciesMel[filterbankIndex + 1] : upperFrequencyMel
+		const rightFrequency = filterbankIndex < melBandCount - 1 ? centerFrequenciesMel[filterbankIndex + 1] : upperFrequencyMel
 
 		const width = rightFrequency - leftFrequency
 		const halfWidth = width / 2
@@ -98,8 +115,9 @@ export function getMelFilterbanks(powerSpectrumFrequenciesHz: Float32Array, cent
 
 		let weightSum = 0
 
-		for (let powerSpectrumBandIndex = 0; powerSpectrumBandIndex < powerSpectrumFrequenciesMel.length; powerSpectrumBandIndex++) {
-			const powerSpectrumBandFrequencyMel = powerSpectrumFrequenciesMel[powerSpectrumBandIndex]
+		for (let powerSpectrumBandIndex = 0; powerSpectrumBandIndex < powerSpectrumFrequenciesHz.length; powerSpectrumBandIndex++) {
+			const powerSpectrumBandFrequencyHz = powerSpectrumFrequenciesHz[powerSpectrumBandIndex]
+			const powerSpectrumBandFrequencyMel = hertzToMel(powerSpectrumBandFrequencyHz)
 
 			let weight = 0
 
@@ -123,13 +141,13 @@ export function getMelFilterbanks(powerSpectrumFrequenciesHz: Float32Array, cent
 
 		weights = weights.map(weight => weight / weightSum)
 
-		filterbanks.push({ startIndex, weights })
+		filterbank.push({ startIndex, weights })
 	}
 
-	return filterbanks
+	return filterbank
 }
 
-export function getMelFilterbanksCenterFrequencies(melBandCount: number, lowerFrequencyMel: number, upperFrequencyMel: number) {
+export function getMelFilterbankCenterFrequencies(melBandCount: number, lowerFrequencyMel: number, upperFrequencyMel: number) {
 	const stepSizeMel = (upperFrequencyMel - lowerFrequencyMel) / (melBandCount + 1)
 
 	const centerFrequencies = new Float32Array(melBandCount)
@@ -149,7 +167,9 @@ export function melToHertz(mel: number) {
 	return 700.0 * (Math.pow(10.0, mel / 2595.0) - 1.0)
 }
 
-export type Filterbank = {
+export type Filterbank = FilterbankElement[]
+
+export interface FilterbankElement {
 	startIndex: number
 	weights: number[]
 }

package/src/dsp/Rubberband.ts

@@ -1,7 +1,8 @@
+import { getGlobalOption } from '../api/GlobalOptions.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { concatFloat32Arrays } from '../utilities/Utilities.js'
-import { Float32ArrayRef, wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
+import type { Float32ArrayRef } from 'wasm-heap-manager'
 
 let rubberbandInstance: any
 
@@ -14,6 +15,8 @@ export async function stretchTimePitch(rawAudio: RawAudio, speed: number, pitchS
 	const sampleRate = rawAudio.sampleRate
 
 	const m = await getRubberbandInstance()
+
+	const { wrapEmscriptenModuleHeap } = await import('wasm-heap-manager')
 	const wasmHeap = wrapEmscriptenModuleHeap(m)
 
 	const optionFlags = rubberBandOptionsToFlags(options)
@@ -112,9 +115,13 @@ export async function stretchTimePitch(rawAudio: RawAudio, speed: number, pitchS
 
 export async function getRubberbandInstance() {
 	if (!rubberbandInstance) {
-		const { default: RubberbandInitializer } = await import('@echogarden/rubberband-wasm')
+		try {
+			const { default: RubberbandInitializer } = await import('@echogarden/rubberband-wasm')
 
-		rubberbandInstance = await RubberbandInitializer()
+			rubberbandInstance = await RubberbandInitializer()
+		} catch {
+			throw new Error(`Couldn't load the '@echogarden/rubberband-wasm' module. Since Echogarden v3.0.0 it is an optional dependency, due to its GPL-2 license. To enable it, you'll need to install it manually via 'npm install [-g] @echogarden/rubberband-wasm'.`)
+		}
 	}
 
 	return rubberbandInstance

package/src/dsp/Sonic.ts

@@ -1,5 +1,4 @@
 import { RawAudio } from '../audio/AudioUtilities.js'
-import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
 
 let sonicInstance: any
 
@@ -10,6 +9,8 @@ export async function stretchTimePitch(rawAudio: RawAudio, speed: number, pitchS
 	const inputSampleCount = rawAudio.audioChannels[0].length
 
 	const m = await getSonicInstance()
+
+	const { wrapEmscriptenModuleHeap } = await import('wasm-heap-manager')
 	const wasmHeap = wrapEmscriptenModuleHeap(m)
 
 	const streamPtr = m._sonicCreateStream(sampleRate, channelCount)

package/src/dsp/SpeexResampler.ts

@@ -1,6 +1,5 @@
 import { RawAudio, cloneRawAudio } from '../audio/AudioUtilities.js'
 import { concatFloat32Arrays, isWasmSimdSupported } from '../utilities/Utilities.js'
-import { wrapEmscriptenModuleHeap } from 'wasm-heap-manager'
 
 let speexResamplerInstance: any
 
@@ -23,6 +22,8 @@ export async function resampleAudioSpeex(rawAudio: RawAudio, outSampleRate: numb
 	}
 
 	const m = await getSpeexResamplerInstance()
+
+	const { wrapEmscriptenModuleHeap } = await import('wasm-heap-manager')
 	const wasmHeap = wrapEmscriptenModuleHeap(m)
 
 	function speexResultCodeToString(resultCode: number) {

package/src/encodings/Ascii.ts

@@ -1,4 +1,4 @@
-import { EncodeIntoResult } from "./TextEncodingsCommon.js"
+import { EncodeIntoResult } from './TextEncodingsCommon.js'
 
 export function encodeAscii(asciiString: string) {
 	const charCount = asciiString.length

package/src/encodings/Base64.ts

@@ -1,4 +1,4 @@
-import { decodeAscii } from './Ascii.js';
+import { decodeAscii } from './Ascii.js'
 
 export function encodeBase64(inputBytes: Uint8Array,
 	paddingCharacter: string | undefined = '=',

package/src/math/VectorMath.ts

@@ -747,24 +747,23 @@ export function sigmoid(x: number) {
 }
 
 export function softmax(logits: ArrayLike<number>, temperature = 1.0) {
-	temperature = Math.max(temperature, 0.00001)
-
 	const logitCount = logits.length
 
 	if (logitCount === 0) {
 		return new Float32Array(0)
 	}
 
-	let maxValue = -Infinity
+	let maxLogit = -Infinity
 
 	for (let i = 0; i < logitCount; i++) {
 		const value = logits[i]
 
-		if (value > maxValue) {
-			maxValue = value
+		if (value > maxLogit) {
+			maxLogit = value
 		}
 	}
 
+	temperature = Math.max(temperature, 1e-5)
 	const temperatureReciprocal = 1 / temperature
 
 	const results = new Float32Array(logitCount)
@@ -774,7 +773,8 @@ export function softmax(logits: ArrayLike<number>, temperature = 1.0) {
 	for (let i = 0; i < logitCount; i++) {
 		const logit = logits[i]
 
-		const eToLogit = Math.exp((logit - maxValue) * temperatureReciprocal)
+		const normalizedLogit = (logit - maxLogit) * temperatureReciprocal
+		const eToLogit = Math.exp(normalizedLogit)
 
 		sumOfExponentiatedLogits += eToLogit
 
@@ -831,6 +831,10 @@ export function zeroIfNaN(val: number) {
 	}
 }
 
+export function logSum(values: ArrayLike<number>, minVal = 1e-40) {
+	return Math.log(minVal + sumVector(values))
+}
+
 export function logSumExp(values: ArrayLike<number>, minVal = 1e-40) {
 	return Math.log(minVal + sumExp(values))
 }

package/src/nlp/EspeakPhonemizer.ts

@@ -4,8 +4,8 @@ import * as Segmentation from './Segmentation.js'
 
 const log = logToStderr
 
-export async function phonemizeSentence(sentence: string, espeakVoice: string, substitutionMap?: Map<string, string[]>, useIpa = true) {
-	const ipaString = await EspeakTTS.textToPhonemes(sentence, espeakVoice, useIpa)
+export function phonemizeSentence(sentence: string, espeakVoice: string, substitutionMap?: Map<string, string[]>, useIpa = true) {
+	const ipaString = EspeakTTS.textToPhonemes(sentence, espeakVoice, useIpa)
 
 	const phraseStrings = ipaString.split(' | ')
 

package/src/nlp/Segmentation.ts

@@ -17,7 +17,7 @@ export const emojiSequenceRegExp = /\p{Emoji_Modifier_Base}\p{Emoji_Modifier}?|\
 export const punctuationRegExp = /[\p{Punctuation}]/u
 
 export const phraseSeparators = [',', '、', '，', '،', ';', '；', ':', '：', '—']
-export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '\\', '^', '*', '×', '=', '¼', '½', '¾']
+export const symbolWords = ['$', '€', '¢', '£', '¥', '©', '®', '™', '%', '&', '#', '~', '@', '+', '±', '÷', '/', '\\', '^', '*', '×', '=', '≈', '¼', '½', '¾']
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
 // Predicates
@@ -42,8 +42,8 @@ export function includesEmoji(str: string) {
 	return emojiSequenceRegExp.test(str?.trim())
 }
 
-export function isWhitespace(str: string) {
-	return str?.trim().length === 0
+export function isAllWhitespace(str: string) {
+	return str && /^\s+$/.test(str)
 }
 
 ///////////////////////////////////////////////////////////////////////////////////////////////
@@ -60,6 +60,11 @@ export function splitToParagraphs(text: string, paragraphBreakType: ParagraphBre
 		throw new Error(`Invalid paragraph break type: '${paragraphBreakType}'`)
 	}
 
+	if (paragraphs.length >= 2 && isAllWhitespace(paragraphs[0])) {
+		paragraphs[1] = paragraphs[0] + paragraphs[1]
+		paragraphs.shift()
+	}
+
 	paragraphs = paragraphs.map(p => applyWhitespaceProcessing(p, whitespaceProcessingMethod))
 	paragraphs = paragraphs.filter(p => p.length > 0)
 
@@ -108,7 +113,7 @@ export async function splitToWords(text: string, langCode: string): Promise<Text
 
 		return wordSequence
 	} else {
-		return TextSegmentation.splitToWords(text, { language: langCode })
+		return TextSegmentation.splitToWords(text, { language: shortLangCode })
 	}
 }
 

package/src/recognition/AmazonTranscribeSTT.ts

@@ -4,11 +4,12 @@ import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
+import { RecognitionCallbacks } from '../api/Recognition.js'
 
-export async function recgonize(rawAudio: RawAudio, languageCode: string, region: string, accessKeyId: string, secretAccessKey: string) {
-	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: 'flac', sampleRate: 16000, sampleFormat: 's16', channelCount: 1 })
+export async function recgonize(rawAudio: RawAudio, languageCode: string, region: string, accessKeyId: string, secretAccessKey: string, callbacks: RecognitionCallbacks) {
+	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: 'flac', sampleRate: 16000, sampleFormat: 's16', channelCount: 1 }, callbacks)
 
-	const logger = new Logger()
+	const logger = new Logger(callbacks.logLevel)
 
 	if (!(languageCode in LanguageCode)) {
 		let matchingLanguageCode: string | undefined = undefined

package/src/recognition/AzureCognitiveServicesSTT.ts

@@ -3,13 +3,14 @@ import * as SpeechSDK from 'microsoft-cognitiveservices-speech-sdk'
 import { RawAudio, encodeRawAudioToWave } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
+import { RecognitionCallbacks } from '../api/Recognition.js'
 
-export async function recognize(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw) {
-	const logger = new Logger()
+export async function recognize(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw, callbacks: RecognitionCallbacks) {
+	const logger = new Logger(callbacks.logLevel)
 
 	logger.start('Request recognition from Azure Cognitive Services')
 
-	const result = await requestRecognition(rawAudio, subscriptionKey, serviceRegion, languageCode)
+	const result = await requestRecognition(rawAudio, subscriptionKey, serviceRegion, languageCode, undefined, callbacks)
 
 	logger.start('Process result')
 
@@ -38,7 +39,7 @@ export async function recognize(rawAudio: RawAudio, subscriptionKey: string, ser
 	return { transcript, timeline }
 }
 
-async function requestRecognition(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw) {
+async function requestRecognition(rawAudio: RawAudio, subscriptionKey: string, serviceRegion: string, languageCode: string, profanity: SpeechSDK.ProfanityOption = SpeechSDK.ProfanityOption.Raw, callbacks: RecognitionCallbacks) {
 	const encodedAudio = encodeRawAudioToWave(rawAudio)
 
 	return new Promise<SpeechSDK.SpeechRecognitionResult>((resolve, reject) => {