echogarden 2.10.2 → 3.0.0

package/src/api/VoiceActivityDetection.ts

@@ -1,34 +1,33 @@
-import { extendDeep } from '../utilities/ObjectUtilities.js'
+import chalk from 'chalk'
 
-import { logToStderr } from '../utilities/Utilities.js'
+import * as API from './API.js'
+
+import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { AudioSourceParam, RawAudio, cropToTimeline, ensureRawAudio, } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 
 import { Timeline } from '../utilities/Timeline.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import { EngineMetadata } from './Common.js'
-import chalk from 'chalk'
 import { type AdaptiveGateVADOptions } from '../voice-activity-detection/AdaptiveGateVAD.js'
-import { type WhisperVADOptions } from '../recognition/WhisperSTT.js'
 import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 import { joinPath } from '../utilities/PathUtilities.js'
 
-const log = logToStderr
+export async function detectVoiceActivity(input: AudioSourceParam, options: VoiceActivityDetectionOptions, callbacks?: VoiceActivityDetectionCallbacks): Promise<VoiceActivityDetectionResult> {
+	options = extendDeep(defaultVoiceActivityDetectionOptions, options)
+	callbacks = { logLevel: API.getGlobalLogLevel(), ...callbacks }
 
-export async function detectVoiceActivity(input: AudioSourceParam, options: VADOptions): Promise<VADResult> {
-	const logger = new Logger()
+	const logger = new Logger(callbacks.logLevel)
 
 	const startTimestamp = logger.getTimestamp()
 
-	const inputRawAudio = await ensureRawAudio(input)
+	const inputRawAudio = await ensureRawAudio(input, undefined, undefined, callbacks)
 
 	logger.start(`Resample audio to 16kHz mono`)
-	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1, callbacks)
 
 	logger.start(`Detect voice activity with ${options.engine}`)
 
-	options = extendDeep(defaultVADOptions, options)
-
 	const activityThreshold = options.activityThreshold!
 
 	let verboseTimeline: Timeline
@@ -52,7 +51,7 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 
 			const sileroOptions = options.silero!
 
-			const modelDir = await loadPackage('silero-vad')
+			const modelDir = await loadPackage('silero-vad', callbacks)
 
 			const modelPath = joinPath(modelDir, 'silero-vad.onnx')
 			const frameDuration = sileroOptions.frameDuration!
@@ -75,11 +74,11 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 		case 'rnnoise': {
 			const RNNoise = await import('../denoising/RNNoise.js')
 
-			const audio48k = await ensureRawAudio(sourceRawAudio, 48000, 1)
+			const audio48k = await ensureRawAudio(sourceRawAudio, 48000, 1, callbacks)
 
 			const rnnoiseOptions = options.rnnoise!
 
-			const { denoisedRawAudio, frameVadProbabilities } = await RNNoise.denoiseAudio(audio48k)
+			const { denoisedRawAudio, frameVadProbabilities } = await RNNoise.denoiseAudio(audio48k, callbacks)
 
 			const frameDurationSeconds = 0.01
 			const frameProbabilities = frameVadProbabilities
@@ -89,52 +88,16 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 			break
 		}
 
-		case 'whisper': {
-			const WhisperSTT = await import('../recognition/WhisperSTT.js')
-
-			const whisperVADOptions = options.whisper!
-
-			logger.end()
-
-			const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperVADOptions.model, 'de')
-
-			logger.end();
-
-			const { partProbabilities } = await WhisperSTT.detectVoiceActivity(
-				sourceRawAudio,
-				modelName,
-				modelDir,
-				whisperVADOptions,
-			)
-
-			verboseTimeline = []
-
-			for (const entry of partProbabilities) {
-				const hasSpeech = entry.confidence! >= activityThreshold
-
-				const text = hasSpeech ? 'active' : 'inactive'
-
-				if (verboseTimeline.length === 0 || verboseTimeline[verboseTimeline.length - 1].text != text) {
-					verboseTimeline.push({
-						type: 'segment',
-						text,
-						startTime: entry.startTime,
-						endTime: entry.endTime
-					})
-				} else {
-					verboseTimeline[verboseTimeline.length - 1].endTime = entry.endTime
-				}
-			}
-
-			break
-		}
-
 		case 'adaptive-gate': {
 			const AdaptiveGateVAD = await import('../voice-activity-detection/AdaptiveGateVAD.js')
 
 			const adaptiveGateOptions = options.adaptiveGate!
 
-			verboseTimeline = await AdaptiveGateVAD.detectVoiceActivity(sourceRawAudio, adaptiveGateOptions)
+			verboseTimeline = await AdaptiveGateVAD.detectVoiceActivity(
+				sourceRawAudio,
+				adaptiveGateOptions,
+				callbacks
+			)
 
 			break
 		}
@@ -150,7 +113,7 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 
 	logger.end()
 	logger.log('')
-	logger.logDuration(`Total voice activity detection time`, startTimestamp, chalk.magentaBright)
+	logger.logDuration(`Total voice activity detection time`, startTimestamp, 'info', chalk.magentaBright)
 
 	return {
 		timeline,
@@ -280,7 +243,7 @@ interface UncropTimelineMapResult {
 	mappedEndTime: number
 }
 
-export interface VADResult {
+export interface VoiceActivityDetectionResult {
 	timeline: Timeline
 	verboseTimeline: Timeline
 
@@ -288,10 +251,10 @@ export interface VADResult {
 	croppedRawAudio: RawAudio
 }
 
-export type VADEngine = 'webrtc' | 'silero' | 'rnnoise' | 'whisper' | 'adaptive-gate'
+export type VoiceActivityDetectionEngine = 'webrtc' | 'silero' | 'rnnoise' | 'whisper' | 'adaptive-gate'
 
-export interface VADOptions {
-	engine?: VADEngine
+export interface VoiceActivityDetectionOptions extends API.OperationOptions {
+	engine?: VoiceActivityDetectionEngine
 
 	activityThreshold?: number
 
@@ -308,12 +271,10 @@ export interface VADOptions {
 	rnnoise?: {
 	}
 
-	whisper?: WhisperVADOptions
-
 	adaptiveGate?: AdaptiveGateVADOptions
 }
 
-export const defaultVADOptions: VADOptions = {
+export const defaultVoiceActivityDetectionOptions: VoiceActivityDetectionOptions = {
 	engine: 'silero',
 
 	activityThreshold: 0.5,
@@ -331,15 +292,13 @@ export const defaultVADOptions: VADOptions = {
 	rnnoise: {
 	},
 
-	whisper: {
-		model: 'tiny',
-		temperature: 1.0,
-	},
-
 	adaptiveGate: {
 	}
 }
 
+export interface VoiceActivityDetectionCallbacks extends API.OperationCallbacks {
+}
+
 export const vadEngines: EngineMetadata[] = [
 	{
 		id: 'webrtc',
@@ -358,5 +317,11 @@ export const vadEngines: EngineMetadata[] = [
 		name: 'RNNoise',
 		description: `Uses RNNoise's internal speech probabilities as VAD metrics.`,
 		type: 'local'
+	},
+	{
+		id: 'adaptive-gate',
+		name: 'Adaptive Gate',
+		description: `Band-limited adaptive gate.`,
+		type: 'local'
 	}
 ]

package/src/audio/AudioPlayer.ts

@@ -1,36 +1,15 @@
 import { parentPort } from 'node:worker_threads'
 
-import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
+import { RawAudio, getRawAudioDuration, sliceAudioChannels } from './AudioUtilities.js'
 
-import { RawAudio, encodeRawAudioToWave, fadeAudioInOut, getRawAudioDuration, sliceAudioChannels } from './AudioUtilities.js'
-import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
-
-import { Timer } from '../utilities/Timer.js'
-import { clip, getRandomHexString, waitTimeout, writeToStderr } from '../utilities/Utilities.js'
-import { encodeToAudioBuffer, float32ToInt16Pcm, interleaveChannels } from './AudioBufferConversion.js'
+import { clip, writeToStderr } from '../utilities/Utilities.js'
+import { float32ToInt16Pcm, interleaveChannels } from './AudioBufferConversion.js'
 import { OpenPromise } from '../utilities/OpenPromise.js'
 import { Timeline, addWordTextOffsetsToTimelineInPlace } from '../utilities/Timeline.js'
-import { readAndParseJsonFile, readFileAsUtf8, remove, writeFile } from '../utilities/FileSystem.js'
-import { tryResolvingSoxPath } from './SoxPath.js'
 import { SignalChannel } from '../utilities/SignalChannel.js'
 import { deepClone } from '../utilities/ObjectUtilities.js'
-import { appName } from '../api/Common.js'
-import { getAppTempDir, joinPath } from '../utilities/PathUtilities.js'
 import { type AudioOutput } from '@echogarden/audio-io'
 
-export async function playAudioFileWithTimelineFile(audioFilename: string, timelineFileName: string, transcriptFileName?: string, player?: AudioPlayerID) {
-	const rawAudio = await FFMpegTranscoder.decodeToChannels(audioFilename, 48000, 1)
-
-	const timeline = await readAndParseJsonFile(timelineFileName)
-
-	let transcript: string | undefined
-	if (transcriptFileName) {
-		transcript = await readFileAsUtf8(transcriptFileName)
-	}
-
-	await playAudioWithWordTimeline(rawAudio, timeline, transcript, player)
-}
-
 export async function playAudioWithWordTimeline(rawAudio: RawAudio, wordTimeline: Timeline, transcript?: string, player?: AudioPlayerID) {
 	if (!transcript) {
 		transcript = wordTimeline.map(entry => entry.text).join(' ')
@@ -163,8 +142,6 @@ export function playAudioSamples(rawAudio: RawAudio, onTimePosition?: (timePosit
 
 	if (player === 'audio-io') {
 		return playAudioSamples_AudioIO(rawAudio, onTimePosition, signalChannel)
-	} else if (player === 'sox') {
-		return playAudioSamples_Sox(rawAudio, onTimePosition, signalChannel)
 	} else {
 		throw new Error(`Unsupported audio player ID: ${player}`)
 	}
@@ -252,211 +229,6 @@ export async function playAudioSamples_AudioIO(rawAudio: RawAudio, onTimePositio
 	return openPromise.promise
 }
 
-export function playAudioSamples_Sox(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, signalChannel?: SignalChannel, microFadeInOut = true) {
-	return new Promise<void>(async (resolve, reject) => {
-		if (microFadeInOut) {
-			rawAudio = fadeAudioInOut(rawAudio, 0.0025)
-		}
-
-		let playerProcessClosed = false
-
-		const channelCount = rawAudio.audioChannels.length
-		const audioDuration = getRawAudioDuration(rawAudio)
-
-		const playerSpawnedOpenPromise = new OpenPromise<null>()
-
-		const soxPath = await tryResolvingSoxPath()
-
-		if (!soxPath) {
-			throw new Error(`Couldn't find or install the SoX utility. Please install the SoX utility on your system path to enable audio playback.`)
-		}
-
-		let aborted = false
-
-		let streamToStdin = true
-
-		if (process.platform == 'darwin') {
-			streamToStdin = false
-		}
-
-		let tempFilePath: string | undefined
-		let audioBuffer: Uint8Array | undefined
-
-		async function cleanup() {
-			if (tempFilePath) {
-				await remove(tempFilePath)
-			}
-		}
-
-		let playerProcess: ChildProcessWithoutNullStreams
-
-		if (streamToStdin) {
-			audioBuffer = encodeToAudioBuffer(rawAudio.audioChannels)
-
-			playerProcess = spawn(
-				soxPath,
-				['-t', 'raw', '-r', `${rawAudio.sampleRate}`, '-e', 'signed', '-b', '16', '-c', channelCount.toString(), '-', '-d'],
-				{}
-			)
-		} else {
-			tempFilePath = joinPath(getAppTempDir(appName), `${getRandomHexString(16)}.wav`)
-			const waveFileBuffer = encodeRawAudioToWave(rawAudio)
-			await writeFile(tempFilePath, waveFileBuffer)
-
-			playerProcess = spawn(
-				soxPath,
-				[tempFilePath, '-d'],
-				{}
-			)
-		}
-
-		if (signalChannel) {
-			signalChannel.on('abort', () => {
-				aborted = true
-				playerProcess.kill('SIGKILL')
-			})
-		}
-
-		// Required to work around SoX bug:
-		playerProcess.stderr.on('data', (data) => {
-			//writeToStderr(data.toString('utf-8'))
-		})
-
-		playerProcess.stdout.on('data', (data) => {
-			//writeToStderr(data.toString('utf-8'))
-		})
-
-		playerProcess.once('spawn', () => {
-			if (audioBuffer != undefined) {
-				playerProcess.stdin!.write(audioBuffer)
-				playerProcess.stdin!.end()
-				playerProcess.stdin!.on('error', () => { })
-			}
-
-			playerSpawnedOpenPromise.resolve(null)
-		})
-
-		playerProcess.once('error', async (e) => {
-			await cleanup()
-			playerProcessClosed = true
-
-			reject(e)
-		})
-
-		playerProcess.once('close', async () => {
-			await cleanup()
-			playerProcessClosed = true
-
-			resolve()
-		})
-
-		await playerSpawnedOpenPromise.promise
-
-		const timer = new Timer()
-
-		while (!playerProcessClosed && !aborted) {
-			const elapsedTime = timer.elapsedTimeSeconds
-
-			if (onTimePosition) {
-				onTimePosition(elapsedTime)
-			}
-
-			if (playerProcessClosed || elapsedTime >= audioDuration) {
-				if (onTimePosition) {
-					onTimePosition(audioDuration)
-				}
-
-				return
-			}
-
-			await waitTimeout(20)
-		}
-	})
-}
-
-/*
-export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (timePosition: number) => void, microFadeInOut = true) {
-	return new Promise<void>(async (resolve, reject) => {
-		if (microFadeInOut) {
-			rawAudio = fadeAudioInOut(rawAudio, 0.0025)
-		}
-
-		const channelCount = rawAudio.audioChannels.length
-		let audioData = encodeToAudioBuffer(rawAudio.audioChannels)
-
-		const { default: Speaker } = await import('speaker')
-
-		const speaker = new Speaker({
-			channels: rawAudio.audioChannels.length,
-			bitDepth: 16,
-			sampleRate: rawAudio.sampleRate,
-		})
-
-		speaker.on('error', (e: any) => {
-			reject(e)
-		})
-
-		const bytesPerSecond = rawAudio.sampleRate * 2 * channelCount
-
-		const byteCountToDuration = (byteCount: number) => {
-			return byteCount / bytesPerSecond
-		}
-
-		const audioDuration = byteCountToDuration(audioData.length)
-
-		let mpg123AudioBufferSize: number
-		let mpg123AudioBufferDuration: number
-
-		if (process.platform == 'win32') {
-			mpg123AudioBufferSize = 65536
-			mpg123AudioBufferDuration = byteCountToDuration(mpg123AudioBufferSize)
-		} else {
-			mpg123AudioBufferDuration = 0.5
-			mpg123AudioBufferSize = bytesPerSecond * mpg123AudioBufferDuration
-		}
-
-		audioData = concatUint8Arrays([audioData, new Uint8Array(mpg123AudioBufferSize)])
-
-		const maxChunkSize = mpg123AudioBufferSize
-
-		const writeAheadDuration = 0.5
-
-		const timer = new Timer()
-		let readOffset = 0
-		let targetTimePosition = 0
-
-		while (true) {
-			const elapsedTime = timer.elapsedTimeSeconds
-
-			if (onTimePosition) {
-				onTimePosition(elapsedTime)
-			}
-
-			if (readOffset < audioData.length) {
-				const targetWriteTime = targetTimePosition - writeAheadDuration
-
-				if (elapsedTime >= targetWriteTime) {
-					const chunk = audioData.subarray(readOffset, readOffset + maxChunkSize)
-
-					speaker.write(chunk)
-
-					readOffset += chunk.length
-					targetTimePosition += byteCountToDuration(chunk.length)
-				}
-			}
-
-			if (elapsedTime >= audioDuration) {
-				//speaker.close(false)
-				resolve()
-				return
-			}
-
-			await waitTimeout(20)
-		}
-	})
-}
-*/
-
 export const charactersToWriteAhead = [
 	',', '.', '，', '、', '：', '；',
 	'。', ':', ';', '?', '？', '!', '！',
@@ -464,4 +236,4 @@ export const charactersToWriteAhead = [
 	'-', '—', '»', '،', '؟'
 ]
 
-export type AudioPlayerID = 'audio-io' | 'sox'
+export type AudioPlayerID = 'audio-io'

package/src/audio/AudioUtilities.ts

@@ -1,8 +1,10 @@
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+
 import { SampleFormat, BitDepth, encodeWaveFromFloat32Channels, decodeWaveToFloat32Channels } from '@echogarden/wave-codec'
 import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { concatFloat32Arrays } from '../utilities/Utilities.js'
+import { OperationCallbacks } from '../api/Common.js'
 
 ////////////////////////////////////////////////////////////////////////////////////////////////
 // Wave encoding and decoding
@@ -400,7 +402,7 @@ export function getRawAudioDuration(rawAudio: RawAudio) {
 	return rawAudio.audioChannels[0].length / rawAudio.sampleRate
 }
 
-export async function ensureRawAudio(input: AudioSourceParam, outSampleRate?: number, outChannelCount?: number) {
+export async function ensureRawAudio(input: AudioSourceParam, outSampleRate: number | undefined, outChannelCount: number | undefined, callbacks: OperationCallbacks) {
 	let inputAsRawAudio: RawAudio = input as RawAudio
 
 	if (isRawAudio(input)) {
@@ -421,7 +423,12 @@ export async function ensureRawAudio(input: AudioSourceParam, outSampleRate?: nu
 	} else if (typeof input == 'string' || input instanceof Uint8Array) {
 		const inputAsStringOrUint8Array = input as string | Uint8Array
 
-		inputAsRawAudio = await FFMpegTranscoder.decodeToChannels(inputAsStringOrUint8Array, outSampleRate, outChannelCount)
+		inputAsRawAudio = await FFMpegTranscoder.decodeToChannels(
+			inputAsStringOrUint8Array,
+			outSampleRate,
+			outChannelCount,
+			callbacks,
+		)
 	} else {
 		throw new Error('Received an invalid input audio data type.')
 	}

package/src/build-tools/MakeTarballsForInstalledPackages.ts

@@ -1,10 +1,11 @@
+import { OperationCallbacks } from '../api/Common.js'
 import { existsSync, readdir } from '../utilities/FileSystem.js'
 import { ensureAndGetPackagesDir } from '../utilities/PackageManager.js'
 import { joinPath } from '../utilities/PathUtilities.js'
 import { createNamedTarball } from '../utilities/TarballMaker.js'
 import { setupProgramTerminationListeners } from '../utilities/Utilities.js'
 
-async function makeTarballsForInstalledPackages(tarballDir: string, skipIfExists = false) {
+async function makeTarballsForInstalledPackages(tarballDir: string, skipIfExists = false, callbacks: OperationCallbacks) {
 	const packagesDir = await ensureAndGetPackagesDir()
 	const packageList = await readdir(packagesDir)
 
@@ -14,12 +15,12 @@ async function makeTarballsForInstalledPackages(tarballDir: string, skipIfExists
 		}
 
 		const packagePath = joinPath(packagesDir, packageName)
-		await createNamedTarball(packagePath, packageName, tarballDir)
+		await createNamedTarball(packagePath, packageName, tarballDir, callbacks)
 	}
 }
 
 setupProgramTerminationListeners()
 
-await makeTarballsForInstalledPackages('../resources/tarballs', true)
+await makeTarballsForInstalledPackages('../resources/tarballs', true, { logLevel: 'info' })
 
 process.exit(0)