echogarden 0.11.12 → 0.11.13

package/src/api/Translation.ts

@@ -0,0 +1,143 @@
+import { extendDeep } from "../utilities/ObjectUtilities.js"
+
+import { logToStderr } from "../utilities/Utilities.js"
+import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevel, trimAudioEnd } from "../audio/AudioUtilities.js"
+import { Logger } from "../utilities/Logger.js"
+
+import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from "../utilities/Timeline.js"
+import { whisperOptionsDefaults, type WhisperOptions } from "../recognition/WhisperSTT.js"
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from "../utilities/Locale.js"
+import { EngineMetadata } from "./Common.js"
+import { SpeechLanguageDetectionOptions, detectSpeechLanguage } from "./API.js"
+import chalk from "chalk"
+import { SubtitlesConfig, defaultSubtitlesBaseConfig } from "../subtitles/Subtitles.js"
+
+const log = logToStderr
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Speech translation
+/////////////////////////////////////////////////////////////////////////////////////////////
+export async function translateSpeech(input: AudioSourceParam, options: SpeechTranslationOptions): Promise<SpeechTranslationResult> {
+	const logger = new Logger()
+	const startTimestamp = logger.getTimestamp()
+
+	logger.start("Prepare for speech translation")
+
+	const inputRawAudio = await ensureRawAudio(input)
+
+	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
+	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
+
+	options = extendDeep(defaultSpeechTranslationOptions, options)
+
+	if (!options.sourceLanguage) {
+		logger.start("No language specified. Detecting speech language")
+		const { detectedLanguage } = await detectSpeechLanguage(inputRawAudio, options.languageDetection || {})
+
+		logger.end()
+		logger.logTitledMessage('Language detected', formatLanguageCodeWithName(detectedLanguage))
+
+		options.sourceLanguage = detectedLanguage
+	}
+
+	logger.start("Preprocess audio for translation")
+
+	const engine = options.engine!
+	const sourceLanguage = normalizeLanguageCode(options.sourceLanguage!)
+	const targetLanguage = options.targetLanguage!
+
+	let transcript: string
+	let timeline: Timeline | undefined
+
+	logger.start(`Load ${engine} module`)
+
+	switch (engine) {
+		case "whisper": {
+			const WhisperSTT = await import("../recognition/WhisperSTT.js")
+
+			const whisperOptions = options.whisper!
+
+			const shortSourceLanguageCode = getShortLanguageCode(sourceLanguage)
+			const shortTargetLanguageCode = getShortLanguageCode(targetLanguage)
+
+			const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, shortSourceLanguageCode)
+
+			if (shortTargetLanguageCode != "en") {
+				throw new Error("Whisper translation only supports English as target language")
+			}
+
+			if (modelName.endsWith(".en")) {
+				throw new Error("Whisper translation tasks are only possible with a multilingual model")
+			}
+
+			if (shortSourceLanguageCode == "en" && shortTargetLanguageCode == "en") {
+				throw new Error("Both translation source and target language are English")
+			}
+
+			logger.end();
+
+			({ transcript, timeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, tokenizerDir, "translate", sourceLanguage, whisperOptions))
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	addWordTextOffsetsToTimeline(timeline, transcript)
+
+	const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(timeline, transcript, targetLanguage, 'single', 'preserve')
+
+	logger.end()
+	logger.log('')
+	logger.logDuration(`Total speech translation time`, startTimestamp, chalk.magentaBright)
+
+	return { transcript, timeline: segmentTimeline, wordTimeline: timeline, sourceLanguage, targetLanguage, inputRawAudio }
+}
+
+export interface SpeechTranslationResult {
+	transcript: string
+	timeline: Timeline
+	wordTimeline: Timeline
+	sourceLanguage: string
+	targetLanguage: string
+	inputRawAudio: RawAudio
+}
+
+export type SpeechTranslationEngine = "whisper"
+
+export interface SpeechTranslationOptions {
+	engine?: SpeechTranslationEngine
+
+	sourceLanguage?: string
+	targetLanguage?: string
+	languageDetection?: SpeechLanguageDetectionOptions
+	subtitles?: SubtitlesConfig
+
+	whisper?: WhisperOptions
+}
+
+export const defaultSpeechTranslationOptions: SpeechTranslationOptions = {
+	engine: "whisper",
+
+	sourceLanguage: undefined,
+	targetLanguage: "en",
+
+	languageDetection: undefined,
+
+	subtitles: defaultSubtitlesBaseConfig,
+
+	whisper: whisperOptionsDefaults,
+}
+
+export const speechTranslationEngines: EngineMetadata[] = [
+	{
+		id: 'whisper',
+		name: 'OpenAI Whisper',
+		description: "Uses Whisper's speech translation capability to produce an English transcript from speech in a different language.",
+		type: 'local'
+	}
+]

package/src/api/Vad.ts

@@ -0,0 +1,172 @@
+import { extendDeep } from "../utilities/ObjectUtilities.js"
+
+import { logToStderr } from "../utilities/Utilities.js"
+import { AudioSourceParam, RawAudio, ensureRawAudio, } from "../audio/AudioUtilities.js"
+import { Logger } from "../utilities/Logger.js"
+
+import { Timeline } from "../utilities/Timeline.js"
+import path from "path"
+import { loadPackage } from "../utilities/PackageManager.js"
+import { EngineMetadata } from "./Common.js"
+import chalk from "chalk"
+
+const log = logToStderr
+
+export async function detectVoiceActivity(input: AudioSourceParam, options: VADOptions): Promise<VADResult> {
+	const logger = new Logger()
+	const startTimestamp = logger.getTimestamp()
+
+	logger.start("Prepare for voice activity detection")
+
+	const inputRawAudio = await ensureRawAudio(input)
+
+	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+
+	options = extendDeep(defaultVADOptions, options)
+
+	logger.start(`Detect voice activity with ${options.engine}`)
+
+	let frameDurationSeconds: number
+	let frameProbabilities: number[]
+
+	switch (options.engine) {
+		case "webrtc": {
+			const WebRtcVAD = await import("../voice-activity-detection/WebRtcVAD.js")
+
+			const webrtcOptions = options.webrtc!
+
+			frameProbabilities = await WebRtcVAD.detectVoiceActivity(sourceRawAudio, webrtcOptions.frameDuration!)
+			frameDurationSeconds = webrtcOptions.frameDuration! / 1000
+
+			break
+		}
+
+		case "silero": {
+			const SileroVAD = await import("../voice-activity-detection/SileroVAD.js")
+
+			const sileroOptions = options.silero!
+
+			const modelDir = await loadPackage("silero-vad")
+
+			const modelPath = path.join(modelDir, "silero-vad.onnx")
+			const frameDuration = sileroOptions.frameDuration!
+
+			frameProbabilities = await SileroVAD.detectVoiceActivity(sourceRawAudio, modelPath, frameDuration)
+			frameDurationSeconds = sileroOptions.frameDuration! / 1000
+
+			break
+		}
+
+		case "rnnoise": {
+			const RNNoise = await import("../denoising/RNNoise.js")
+
+			const rnnoiseOptions = options.rnnoise!
+
+			const { denoisedRawAudio, frameVadProbabilities } = await RNNoise.denoiseAudio(sourceRawAudio)
+
+			frameDurationSeconds = 0.01
+			frameProbabilities = frameVadProbabilities
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	const timeline: Timeline = []
+
+	for (let i = 0; i < frameProbabilities.length; i++) {
+		const frameProbability = frameProbabilities[i]
+
+		const startTime = i * frameDurationSeconds
+		const endTime = (i + 1) * frameDurationSeconds
+
+		if (frameProbability >= options.activityThreshold!) {
+			if (timeline.length == 0 || timeline[timeline.length - 1].text == "nonspeech") {
+				timeline.push({ type: "segment", text: "speech", startTime, endTime })
+				continue
+			}
+		} else {
+			if (timeline.length == 0 || timeline[timeline.length - 1].text == "speech") {
+				timeline.push({ type: "segment", text: "nonspeech", startTime, endTime })
+				continue
+			}
+		}
+
+		timeline[timeline.length - 1].endTime = endTime
+	}
+
+	logger.end()
+	logger.log('')
+	logger.logDuration(`Total voice activity detection time`, startTimestamp, chalk.magentaBright)
+
+	return { timeline, inputRawAudio }
+}
+
+export interface VADResult {
+	timeline: Timeline
+	inputRawAudio: RawAudio
+}
+
+export type VADEngine = "webrtc" | "silero" | "rnnoise"
+
+export interface VADOptions {
+	engine?: VADEngine
+
+	activityThreshold?: number
+
+	webrtc?: {
+		frameDuration?: 10 | 20 | 30
+		mode?: 0 | 1 | 2 | 3
+	}
+
+	silero?: {
+		modelPath?: string
+		frameDuration?: 30 | 60 | 90
+	}
+
+	rnnoise?: {
+	}
+}
+
+export const defaultVADOptions: VADOptions = {
+	engine: "webrtc",
+
+	activityThreshold: 0.5,
+
+	webrtc: {
+		frameDuration: 30,
+		mode: 1
+	},
+
+	silero: {
+		modelPath: undefined,
+		frameDuration: 90,
+	},
+
+	rnnoise: {
+	}
+}
+
+export const vadEngines: EngineMetadata[] = [
+	{
+		id: 'webrtc',
+		name: 'WebRTC VAD',
+		description: 'A voice activity detector from the Chromium browser sources.',
+		type: 'local'
+	},
+	{
+		id: 'silero',
+		name: 'Silero VAD',
+		description: 'A voice activity detection model by Silero.',
+		type: 'local'
+	},
+	{
+		id: 'rnnoise',
+		name: 'RNNoise',
+		description: "Uses RNNoise's speech probabilities as VAD metrics.",
+		type: 'local'
+	}
+]

package/src/audio/AudioBufferConversion.ts

@@ -0,0 +1,248 @@
+import AlawMulaw from "alawmulaw"
+import * as BinaryArrayConversion from '../utilities/BinaryArrayConversion.js'
+import { BitDepth, SampleFormat } from "../codecs/WaveCodec.js"
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Low level audio sample conversions
+/////////////////////////////////////////////////////////////////////////////////////////////
+export function encodeToAudioBuffer(audioChannels: Float32Array[], targetBitDepth: BitDepth = 16, targetSampleFormat: SampleFormat = SampleFormat.PCM) {
+	const interleavedChannels = interleaveChannels(audioChannels)
+
+	if (targetSampleFormat == SampleFormat.PCM) {
+		if (targetBitDepth == 8) {
+			return Buffer.from(float32ToUint8Pcm(interleavedChannels).buffer)
+		} else if (targetBitDepth == 16) {
+			return BinaryArrayConversion.int16ToBufferLE(float32ToInt16Pcm(interleavedChannels))
+		} else if (targetBitDepth == 24) {
+			return BinaryArrayConversion.int24ToBufferLE(float32ToInt24Pcm(interleavedChannels))
+		} else if (targetBitDepth == 32) {
+			return BinaryArrayConversion.int32ToBufferLE(float32ToInt32Pcm(interleavedChannels))
+		} else {
+			throw new Error(`Unsupported PCM bit depth: ${targetBitDepth}`)
+		}
+	} else if (targetSampleFormat == SampleFormat.Float) {
+		if (targetBitDepth == 32) {
+			return BinaryArrayConversion.float32ToBufferLE(interleavedChannels)
+		} else if (targetBitDepth == 64) {
+			return BinaryArrayConversion.float64ToBufferLE(BinaryArrayConversion.float32Tofloat64(interleavedChannels))
+		} else {
+			throw new Error(`Unsupported float bit depth: ${targetBitDepth}`)
+		}
+	} else if (targetSampleFormat == SampleFormat.Alaw) {
+		if (targetBitDepth == 8) {
+			return Buffer.from(AlawMulaw.alaw.encode(float32ToInt16Pcm(interleavedChannels)))
+		} else {
+			throw new Error(`Unsupported alaw bit depth: ${targetBitDepth}`)
+		}
+	} else if (targetSampleFormat == SampleFormat.Mulaw) {
+		if (targetBitDepth == 8) {
+			return Buffer.from(AlawMulaw.mulaw.encode(float32ToInt16Pcm(interleavedChannels)))
+		} else {
+			throw new Error(`Unsupported mulaw bit depth: ${targetBitDepth}`)
+		}
+	} else {
+		throw new Error(`Unsupported audio format: ${targetSampleFormat}`)
+	}
+}
+
+export function decodeToChannels(audioBuffer: Buffer, channelCount: number, sourceBitDepth: number, sourceSampleFormat: SampleFormat) {
+	let interleavedChannels: Float32Array
+
+	if (sourceSampleFormat == SampleFormat.PCM) {
+		if (sourceBitDepth == 8) {
+			interleavedChannels = uint8PcmToFloat32(audioBuffer)
+		} else if (sourceBitDepth == 16) {
+			interleavedChannels = int16PcmToFloat32(BinaryArrayConversion.bufferLEToInt16(audioBuffer))
+		} else if (sourceBitDepth == 24) {
+			interleavedChannels = int24PcmToFloat32(BinaryArrayConversion.bufferLEToInt24(audioBuffer))
+		} else if (sourceBitDepth == 32) {
+			interleavedChannels = int32PcmToFloat32(BinaryArrayConversion.bufferLEToInt32(audioBuffer))
+		} else {
+			throw new Error(`Unsupported PCM bit depth: ${sourceBitDepth}`)
+		}
+	} else if (sourceSampleFormat == SampleFormat.Float) {
+		if (sourceBitDepth == 32) {
+			interleavedChannels = BinaryArrayConversion.bufferLEToFloat32(audioBuffer)
+		} else if (sourceBitDepth == 64) {
+			interleavedChannels = BinaryArrayConversion.float64Tofloat32(BinaryArrayConversion.bufferLEToFloat64(audioBuffer))
+		} else {
+			throw new Error(`Unsupported float bit depth: ${sourceBitDepth}`)
+		}
+	} else if (sourceSampleFormat == SampleFormat.Alaw) {
+		if (sourceBitDepth == 8) {
+			interleavedChannels = int16PcmToFloat32(AlawMulaw.alaw.decode(audioBuffer))
+		} else {
+			throw new Error(`Unsupported alaw bit depth: ${sourceBitDepth}`)
+		}
+	} else if (sourceSampleFormat == SampleFormat.Mulaw) {
+		if (sourceBitDepth == 8) {
+			interleavedChannels = int16PcmToFloat32(AlawMulaw.mulaw.decode(audioBuffer))
+		} else {
+			throw new Error(`Unsupported mulaw bit depth: ${sourceBitDepth}`)
+		}
+	} else {
+		throw new Error(`Unsupported audio format: ${sourceSampleFormat}`)
+	}
+
+	return deInterleaveChannels(interleavedChannels, channelCount)
+}
+
+// Int8 PCM <-> Float32 conversion
+export function uint8PcmToFloat32(input: Uint8Array) {
+	const output = new Float32Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = input[i] - 128
+		output[i] = sample < 0 ? sample / 128 : sample / 127
+	}
+
+	return output
+}
+
+export function float32ToUint8Pcm(input: Float32Array) {
+	const output = new Uint8Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = clampFloatSample(input[i])
+		output[i] = ((sample < 0 ? sample * 128 : sample * 127) | 0) + 128
+	}
+
+	return output
+}
+
+// Int16 PCM <-> Float32 conversion
+export function int16PcmToFloat32(input: Int16Array) {
+	const output = new Float32Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = input[i]
+		output[i] = sample < 0 ? sample / 32768 : sample / 32767
+	}
+
+	return output
+}
+
+export function float32ToInt16Pcm(input: Float32Array) {
+	const output = new Int16Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = clampFloatSample(input[i])
+		output[i] = (sample < 0 ? sample * 32768 : sample * 32767) | 0
+	}
+
+	return output
+}
+
+// Int24 PCM <-> Float32 conversion (uses int32 for storage)
+export function int24PcmToFloat32(input: Int32Array) {
+	const output = new Float32Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = input[i]
+		output[i] = sample < 0 ? sample / 8388608 : sample / 8388607
+	}
+
+	return output
+}
+
+export function float32ToInt24Pcm(input: Float32Array) {
+	const output = new Int32Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = clampFloatSample(input[i])
+		output[i] = (sample < 0 ? sample * 8388608 : sample * 8388607) | 0
+	}
+
+	return output
+}
+
+// Int32 PCM <-> Float32 conversion
+export function int32PcmToFloat32(input: Int32Array) {
+	const output = new Float32Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = input[i]
+		output[i] = sample < 0 ? sample / 2147483648 : sample / 2147483647
+	}
+
+	return output
+}
+
+export function float32ToInt32Pcm(input: Float32Array) {
+	const output = new Int32Array(input.length)
+
+	for (let i = 0; i < input.length; i++) {
+		const sample = clampFloatSample(input[i])
+		output[i] = (sample < 0 ? sample * 2147483648 : sample * 2147483647) | 0
+	}
+
+	return output
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Channel interleaving
+/////////////////////////////////////////////////////////////////////////////////////////////
+export function interleaveChannels(channels: Float32Array[]) {
+	const channelCount = channels.length
+
+	if (channelCount == 0) {
+		throw new Error("Empty channel array received")
+	}
+
+	if (channelCount == 1) {
+		return channels[0]
+	}
+
+	const sampleCount = channels[0].length
+	const result = new Float32Array(sampleCount * channelCount)
+
+	let writeIndex = 0
+
+	for (let i = 0; i < sampleCount; i++) {
+		for (let c = 0; c < channelCount; c++) {
+			result[writeIndex] = channels[c][i]
+			writeIndex += 1
+		}
+	}
+
+	return result
+}
+
+export function deInterleaveChannels(interleavedChannels: Float32Array, channelCount: number) {
+	if (channelCount == 0) {
+		throw new Error("0 channel count received")
+	}
+
+	if (channelCount == 1) {
+		return [interleavedChannels]
+	}
+
+	if (interleavedChannels.length % channelCount != 0) {
+		throw new Error(`Size of interleaved channels (${interleaveChannels.length}) is not a multiple of channel count (${channelCount})`)
+	}
+
+	const sampleCount = interleavedChannels.length / channelCount
+	const channels: Float32Array[] = []
+
+	for (let i = 0; i < channelCount; i++) {
+		channels.push(new Float32Array(sampleCount))
+	}
+
+	let readIndex = 0
+
+	for (let i = 0; i < sampleCount; i++) {
+		for (let c = 0; c < channelCount; c++) {
+			channels[c][i] = interleavedChannels[readIndex]
+			readIndex += 1
+		}
+	}
+
+	return channels
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Utilities
+/////////////////////////////////////////////////////////////////////////////////////////////
+export function clampFloatSample(floatSample: number) {
+	return Math.max(-1, Math.min(floatSample, 1))
+}