echogarden 1.0.4 → 1.1.0

package/src/api/TranslationAlignment.ts

@@ -0,0 +1,213 @@
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+import { logToStderr } from '../utilities/Utilities.js'
+import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevel, trimAudioEnd } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+
+import * as API from './API.js'
+import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
+import { type WhisperAlignmentOptions } from '../recognition/WhisperSTT.js'
+import chalk from 'chalk'
+import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
+
+const log = logToStderr
+
+export async function alignTranslation(input: AudioSourceParam, transcript: string, options: TranslationAlignmentOptions): Promise<TranslationAlignmentResult> {
+	const logger = new Logger()
+
+	const startTimestamp = logger.getTimestamp()
+
+	options = extendDeep(defaultTranslationAlignmentOptions, options)
+
+	const inputRawAudio = await ensureRawAudio(input)
+
+	let sourceRawAudio: RawAudio
+	let isolatedRawAudio: RawAudio | undefined
+	let backgroundRawAudio: RawAudio | undefined
+
+	if (options.isolate) {
+		logger.log(``)
+		logger.end();
+
+		({ isolatedRawAudio, backgroundRawAudio } = await API.isolate(inputRawAudio, options.sourceSeparation!))
+
+		logger.end()
+		logger.log(``)
+
+		sourceRawAudio = await ensureRawAudio(isolatedRawAudio, 16000, 1)
+	} else {
+		sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+	}
+
+	let sourceUncropTimeline: Timeline | undefined
+
+	if (options.crop) {
+		logger.start('Crop using voice activity detection');
+		({ timeline: sourceUncropTimeline, croppedRawAudio: sourceRawAudio } = await API.detectVoiceActivity(sourceRawAudio, options.vad!))
+
+		logger.end()
+	}
+
+	logger.start('Prepare for alignment')
+
+	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
+	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
+
+	let sourceLanguage: string
+
+	if (options.sourceLanguage) {
+		sourceLanguage = normalizeLanguageCode(options.sourceLanguage!)
+	} else {
+		logger.start('No source language specified. Detecting speech language')
+		const { detectedLanguage } = await API.detectSpeechLanguage(sourceRawAudio, options.languageDetection || {})
+
+		logger.end()
+		logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
+
+		sourceLanguage = detectedLanguage
+	}
+
+	const targetLanguage = normalizeLanguageCode(options.targetLanguage!)
+
+	let mappedTimeline: Timeline
+
+	switch (options.engine) {
+		case 'whisper': {
+			const WhisperSTT = await import('../recognition/WhisperSTT.js')
+
+			const shortSourceLanguageCode = getShortLanguageCode(sourceLanguage)
+			const shortTargetLanguageCode = getShortLanguageCode(targetLanguage)
+
+			if (shortTargetLanguageCode != 'en') {
+				throw new Error('Whisper translation only supports English as target language')
+			}
+
+			if (shortSourceLanguageCode == 'en' && shortTargetLanguageCode == 'en') {
+				throw new Error('Both translation source and target languages are English')
+			}
+
+			const whisperAlignmnentOptions = options.whisper!
+
+			const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperAlignmnentOptions.model, shortSourceLanguageCode)
+
+			logger.end()
+
+			if (modelName.endsWith('.en')) {
+				throw new Error('Whisper translation tasks are only possible with a multilingual model')
+			}
+
+			mappedTimeline = await WhisperSTT.alignEnglishTranslation(sourceRawAudio, transcript, modelName, modelDir, shortSourceLanguageCode, whisperAlignmnentOptions)
+
+			break
+		}
+
+		default: {
+			throw new Error(`Engine '${options.engine}' is not supported`)
+		}
+	}
+
+	// If the audio was cropped before recognition, map the timestamps back to the original audio
+	if (sourceUncropTimeline && sourceUncropTimeline.length > 0) {
+		API.convertCroppedToUncroppedTimeline(mappedTimeline, sourceUncropTimeline)
+	}
+
+	// Add text offsets
+	addWordTextOffsetsToTimeline(mappedTimeline, transcript)
+
+	// Make segment timeline
+	const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(mappedTimeline, transcript, sourceLanguage, options.plainText?.paragraphBreaks, options.plainText?.whitespace)
+
+	logger.end()
+	logger.logDuration(`Total translation alignment time`, startTimestamp, chalk.magentaBright)
+
+	return {
+		timeline: segmentTimeline,
+		wordTimeline: mappedTimeline,
+
+		transcript,
+		language: sourceLanguage,
+
+		inputRawAudio,
+		isolatedRawAudio,
+		backgroundRawAudio,
+	}
+}
+
+export interface TranslationAlignmentResult {
+	timeline: Timeline
+	wordTimeline: Timeline
+
+	transcript: string
+	language: string
+
+	inputRawAudio: RawAudio
+	isolatedRawAudio?: RawAudio
+	backgroundRawAudio?: RawAudio
+}
+
+export type TranslationAlignmentEngine = 'whisper'
+
+export interface TranslationAlignmentOptions {
+	engine?: TranslationAlignmentEngine
+
+	sourceLanguage?: string
+	targetLanguage?: string
+
+	isolate?: boolean
+
+	crop?: boolean
+
+	languageDetection?: API.SpeechLanguageDetectionOptions
+
+	vad?: API.VADOptions
+
+	plainText?: API.PlainTextOptions
+
+	subtitles?: SubtitlesConfig
+
+	sourceSeparation?: API.SourceSeparationOptions
+
+	whisper?: WhisperAlignmentOptions
+}
+
+export const defaultTranslationAlignmentOptions: TranslationAlignmentOptions = {
+	engine: 'whisper',
+
+	sourceLanguage: undefined,
+	targetLanguage: 'en',
+
+	isolate: false,
+
+	crop: true,
+
+	languageDetection: {
+	},
+
+	plainText: {
+		paragraphBreaks: 'double',
+		whitespace: 'collapse'
+	},
+
+	subtitles: {
+	},
+
+	vad: {
+		engine: 'adaptive-gate'
+	},
+
+	sourceSeparation: {
+	},
+
+	whisper: {
+	}
+}
+
+export const translationAlignmentEngines: API.EngineMetadata[] = [
+	{
+		id: 'whisper',
+		name: 'OpenAI Whisper',
+		description: 'Extracts timestamps by guiding the Whisper recognition model to recognize the translated transcript tokens.',
+		type: 'local'
+	}
+]

package/src/api/VoiceActivityDetection.ts

@@ -10,6 +10,8 @@ import { loadPackage } from '../utilities/PackageManager.js'
 import { EngineMetadata } from './Common.js'
 import chalk from 'chalk'
 import { type AdaptiveGateVADOptions } from '../voice-activity-detection/AdaptiveGateVAD.js'
+import { type WhisperVADOptions } from '../recognition/WhisperSTT.js'
+import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 
 const log = logToStderr
 
@@ -56,7 +58,14 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 			const modelPath = path.join(modelDir, 'silero-vad.onnx')
 			const frameDuration = sileroOptions.frameDuration!
 
-			const frameProbabilities = await SileroVAD.detectVoiceActivity(sourceRawAudio, modelPath, frameDuration)
+			const onnxExecutionProviders: OnnxExecutionProvider[] = sileroOptions.provider ? [sileroOptions.provider] : []
+
+			const frameProbabilities = await SileroVAD.detectVoiceActivity(
+				sourceRawAudio,
+				modelPath,
+				frameDuration,
+				onnxExecutionProviders)
+
 			const frameDurationSeconds = sileroOptions.frameDuration! / 1000
 
 			verboseTimeline = frameProbabilitiesToTimeline(frameProbabilities, frameDurationSeconds, activityThreshold)
@@ -81,6 +90,46 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 			break
 		}
 
+		case 'whisper': {
+			const WhisperSTT = await import('../recognition/WhisperSTT.js')
+
+			const whisperVADOptions = options.whisper!
+
+			logger.end()
+
+			const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperVADOptions.model, 'de')
+
+			logger.end();
+
+			const { partProbabilities } = await WhisperSTT.detectVoiceActivity(
+				sourceRawAudio,
+				modelName,
+				modelDir,
+				whisperVADOptions,
+			)
+
+			verboseTimeline = []
+
+			for (const entry of partProbabilities) {
+				const hasSpeech = entry.confidence! >= activityThreshold
+
+				const text = hasSpeech ? 'active' : 'inactive'
+
+				if (verboseTimeline.length === 0 || verboseTimeline[verboseTimeline.length - 1].text != text) {
+					verboseTimeline.push({
+						type: 'segment',
+						text,
+						startTime: entry.startTime,
+						endTime: entry.endTime
+					})
+				} else {
+					verboseTimeline[verboseTimeline.length - 1].endTime = entry.endTime
+				}
+			}
+
+			break
+		}
+
 		case 'adaptive-gate': {
 			const AdaptiveGateVAD = await import('../voice-activity-detection/AdaptiveGateVAD.js')
 
@@ -104,7 +153,12 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
 	logger.log('')
 	logger.logDuration(`Total voice activity detection time`, startTimestamp, chalk.magentaBright)
 
-	return { timeline, verboseTimeline, inputRawAudio, croppedRawAudio }
+	return {
+		timeline,
+		verboseTimeline,
+		inputRawAudio,
+		croppedRawAudio
+	}
 }
 
 function frameProbabilitiesToTimeline(frameProbabilities: number[], frameDurationSeconds: number, activityThreshold: number) {
@@ -176,7 +230,7 @@ export interface VADResult {
 	croppedRawAudio: RawAudio
 }
 
-export type VADEngine = 'webrtc' | 'silero' | 'rnnoise' | 'adaptive-gate'
+export type VADEngine = 'webrtc' | 'silero' | 'rnnoise' | 'whisper' | 'adaptive-gate'
 
 export interface VADOptions {
 	engine?: VADEngine
@@ -190,11 +244,14 @@ export interface VADOptions {
 
 	silero?: {
 		frameDuration?: 30 | 60 | 90
+		provider?: OnnxExecutionProvider
 	}
 
 	rnnoise?: {
 	}
 
+	whisper?: WhisperVADOptions
+
 	adaptiveGate?: AdaptiveGateVADOptions
 }
 
@@ -210,11 +267,17 @@ export const defaultVADOptions: VADOptions = {
 
 	silero: {
 		frameDuration: 90,
+		provider: undefined,
 	},
 
 	rnnoise: {
 	},
 
+	whisper: {
+		model: 'tiny',
+		temperature: 1.0,
+	},
+
 	adaptiveGate: {
 	}
 }

package/src/audio/AudioPlayer.ts

@@ -355,5 +355,9 @@ export function playAudioSamples_Speaker(rawAudio: RawAudio, onTimePosition?: (t
 	})
 }
 
-export const charactersToWriteAhead =
-	[',', '.', '，', '、', '：', '；', '。', ':', ';', '?', '!', ')', ']', '}', `"`, `'`, '”', '’', '-', '—', '»']
+export const charactersToWriteAhead = [
+	',', '.', '，', '、', '：', '；',
+	'。', ':', ';', '?', '？', '!', '！',
+	')', ']', '}', `"`, `'`, '”', '’',
+	'-', '—', '»', '،', '؟'
+]

package/src/cli/CLI.ts

@@ -13,8 +13,8 @@ import { encodeFromChannels, getDefaultFFMpegOptionsForSpeech } from '../codecs/
 import path, { parse as parsePath } from 'node:path'
 import { splitToParagraphs, splitToWords, wordCharacterPattern } from '../nlp/Segmentation.js'
 import { playAudioSamples, playAudioWithWordTimeline } from '../audio/AudioPlayer.js'
-import { deepClone, extendDeep } from '../utilities/ObjectUtilities.js'
-import { Timeline, TimelineEntry, addTimeOffsetToTimeline, addWordTextOffsetsToTimeline, roundTimelineProperties, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { Timeline, TimelineEntry, addTimeOffsetToTimeline, addWordTextOffsetsToTimeline, roundTimelineProperties } from '../utilities/Timeline.js'
 import { ensureDir, existsSync, readAndParseJsonFile, readFile, readdir, writeFileSafe } from '../utilities/FileSystem.js'
 import { formatLanguageCodeWithName, getShortLanguageCode } from '../utilities/Locale.js'
 import { APIOptions } from '../api/APIOptions.js'
@@ -167,6 +167,8 @@ const commandHelp = [
 	`    Align audio file to the reference transcript file\n`,
 	`${executableName} ${chalk.magentaBright('translate-speech')} inputFile [output files...] [options...]`,
 	`    Transcribe audio file directly to a different language\n`,
+	`${executableName} ${chalk.magentaBright('align-translation')} audioFile referenceFile [output files...] [options...]`,
+	`    Align audio file to the reference translated transcript file\n`,
 	`${executableName} ${chalk.magentaBright('detect-speech-language')} audioFile [output files...] [options...]`,
 	`    Detect language of audio file\n`,
 	`${executableName} ${chalk.magentaBright('detect-text-language')} inputFile [output files...] [options...]`,
@@ -220,6 +222,11 @@ async function startWithArgs(parsedArgs: CLIArguments) {
 			break
 		}
 
+		case 'align-translation': {
+			await alignTranslation(parsedArgs.commandArgs, parsedArgs.options)
+			break
+		}
+
 		case 'detect-language': {
 			await detectLanguage(parsedArgs.commandArgs, parsedArgs.options, 'auto')
 			break
@@ -607,6 +614,112 @@ async function align(commandArgs: string[], cliOptions: Map<string, string>) {
 	}
 }
 
+async function alignTranslation(commandArgs: string[], cliOptions: Map<string, string>) {
+	const logger = new Logger()
+
+	const audioFilename = commandArgs[0]
+	const outputFilenames = commandArgs.slice(2)
+
+	if (audioFilename == undefined) {
+		throw new Error(`align-translation requires an argument containing the audio file path.`)
+	}
+
+	if (!existsSync(audioFilename)) {
+		throw new Error(`The given source file '${audioFilename}' was not found.`)
+	}
+
+	const alignmentReferenceFile = commandArgs[1]
+
+	if (alignmentReferenceFile == undefined) {
+		throw new Error(`align-translation requires a second argument containing the translated reference file path.`)
+	}
+
+	if (!existsSync(alignmentReferenceFile)) {
+		throw new Error(`The given reference file '${alignmentReferenceFile}' was not found.`)
+	}
+
+	const referenceFileExtension = getLowercaseFileExtension(alignmentReferenceFile)
+	const fileContent = await readFile(alignmentReferenceFile, { encoding: 'utf-8' })
+
+	let text: string
+
+	if (referenceFileExtension == 'txt') {
+		text = fileContent
+	} else if (referenceFileExtension == 'html' || referenceFileExtension == 'htm') {
+		text = await convertHtmlToText(fileContent)
+	} else if (referenceFileExtension == 'srt' || referenceFileExtension == 'vtt') {
+		text = subtitlesToText(fileContent)
+	} else {
+		throw new Error(`align only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`)
+	}
+
+	const additionalOptionsSchema = new Map<string, SchemaTypeDefinition>()
+	additionalOptionsSchema.set('play', { type: 'boolean' })
+	additionalOptionsSchema.set('overwrite', { type: 'boolean' })
+
+	if (!cliOptions.has('play') && !cliOptions.has('no-play')) {
+		cliOptions.set('play', `${outputFilenames.length == 0}`)
+	}
+
+	const options: API.TranslationAlignmentOptions = await cliOptionsMapToOptionsObject(cliOptions, 'TranslationAlignmentOptions', additionalOptionsSchema)
+
+	const allowOverwrite = getWithDefault((options as any).overwrite, overwriteByDefault)
+	const { includesPlaceholderPattern } = await checkOutputFilenames(outputFilenames, true, true, true)
+
+	const {
+		timeline,
+		wordTimeline,
+		transcript,
+		language,
+		inputRawAudio,
+		isolatedRawAudio,
+		backgroundRawAudio } = await API.alignTranslation(audioFilename, text, options)
+
+	if (outputFilenames.length > 0) {
+		logger.start('\nWrite output files')
+	}
+
+	if (includesPlaceholderPattern) {
+		for (let segmentIndex = 0; segmentIndex < timeline.length; segmentIndex++) {
+			const segmentEntry = timeline[segmentIndex]
+			const segmentAudio = sliceRawAudioByTime(inputRawAudio, segmentEntry.startTime, segmentEntry.endTime)
+			const sentenceTimeline = addTimeOffsetToTimeline(segmentEntry.timeline!, -segmentEntry.startTime)
+
+			await writeOutputFilesForSegment(outputFilenames, segmentIndex, timeline.length, segmentAudio, sentenceTimeline, segmentEntry.text, language, allowOverwrite)
+		}
+	}
+
+	for (const outputFilename of outputFilenames) {
+		const partPatternMatch = outputFilename.match(filenamePlaceholderPattern)
+
+		if (partPatternMatch) {
+			continue
+		}
+
+		const fileSaver = getFileSaver(outputFilename, allowOverwrite)
+
+		await fileSaver(inputRawAudio, timeline, transcript, options.subtitles)
+
+		await writeSourceSeparationOutputIfNeeded(outputFilename, isolatedRawAudio, backgroundRawAudio, allowOverwrite, true)
+	}
+
+	logger.end()
+
+	if ((options as any).play) {
+		let audioToPlay: RawAudio
+
+		if (isolatedRawAudio) {
+			audioToPlay = isolatedRawAudio
+		} else {
+			audioToPlay = inputRawAudio
+		}
+
+		const normalizedAudioToPlay = normalizeAudioLevel(audioToPlay)
+
+		await playAudioWithWordTimeline(normalizedAudioToPlay, wordTimeline, transcript)
+	}
+}
+
 async function translateSpeech(commandArgs: string[], cliOptions: Map<string, string>) {
 	const logger = new Logger()
 
@@ -976,6 +1089,12 @@ async function listEngines(commandArgs: string[], cliOptions: Map<string, string
 			break
 		}
 
+		case 'align-translation': {
+			engines = API.translationAlignmentEngines
+
+			break
+		}
+
 		case 'translate-speech': {
 			engines = API.speechTranslationEngines
 

package/src/dsp/FFT.ts

@@ -48,6 +48,7 @@ export async function stftr(samples: Float32Array, fftOrder: number, windowSize:
 		}
 
 		binsBufferRef.clear()
+		
 		m._kiss_fftr(statePtr, frameBufferRef.address, binsBufferRef.address)
 
 		const bins = binsBufferRef.view.slice(0, fftOrder + 2)
@@ -103,7 +104,9 @@ export async function stiftr(binsForFrames: Float32Array[], fftOrder: number, wi
 		binsRef.view.set(binsForFrame)
 
 		frameBufferRef.clear()
+
 		m._kiss_fftri(statePtr, binsRef.address, frameBufferRef.address)
+
 		const frameSamples = frameBufferRef.view
 
 		const frameStartOffset = frameIndex * hopSize

package/src/math/MedianFilter.ts

@@ -0,0 +1,124 @@
+import { createVector } from "./VectorMath.js"
+
+export function medianOf5Filter(points: number[]) {
+	// This function computes the moving median with a window of 5 elements.
+
+	// I initialized the window such that at the edges of the range no median would be computed.
+	// This is a form of optimization assuming that computing a median edge points
+	// is less important.
+
+	const pointCount = points.length
+
+	if (pointCount < 5) {
+		return points
+	}
+
+	const medians = createVector(pointCount)
+
+	medians[0] = points[0]
+	medians[1] = points[1]
+	medians[pointCount - 2] = points[pointCount - 2]
+	medians[pointCount - 1] = points[pointCount - 1]
+
+	for (let i = 2; i < pointCount - 2; i++) {
+		medians[i] = medianOf5(points[i - 2], points[i - 1], points[i], points[i + 1], points[i + 2])
+	}
+
+	return medians
+}
+
+export function medianOf3Filter(points: ArrayLike<number>) {
+	// This function computes the moving median with a window of 3 elements.
+
+	// I initialized the window such that at the edges of the range no median would be computed.
+
+	const pointCount = points.length
+
+	if (pointCount < 3) {
+		return points
+	}
+
+	const medians: number[] = createVector(pointCount)
+
+	medians[0] = points[0]
+	medians[pointCount - 1] = points[pointCount - 1]
+
+	for (let i = 1; i < pointCount - 1; i++) {
+		medians[i] = medianOf3(points[i - 1], points[i], points[i + 1])
+	}
+
+	return medians
+}
+
+export function medianOf5(a: number, b: number, c: number, d: number, e: number) {
+	// These swapping computation should be faster than separately using the minimum and maximum
+	// functions but maybe less readable.
+
+	// Ensure b is greater or equal to a (swap if needed)
+	if (b < a) {
+		[a, b] = [b, a]
+	}
+
+	// Ensure d is greater or equal to c (swap if needed)
+	if (d < c) {
+		[c, d] = [d, c]
+	}
+
+	// What this part does is compute the two middle medians of the first 4 elements
+	// given to the function (a, b, c, d), but it doesn't actually determine their relative order:
+	const firstMedianOfABCD = Math.max(a, c) // First median of a, b, c, d
+	const secondMedianOfABCD = Math.min(b, d) // Second median of a, b, c, d
+
+	// Now in relation to all five numbers, the median can only be either
+	// the first median of ABCD, the second median of ABCD, or E:
+	return medianOf3(firstMedianOfABCD, secondMedianOfABCD, e)
+}
+
+export function medianOf3(a: number, b: number, c: number) {
+	// This function uses a decision tree to find the median of three numbers.
+	//
+	// I tried to ensure that the comparison preserved the natural altering of
+	// a, b and c such that in case that they are given already in order,
+	// then all the initial branches would be directly taken.
+
+	// Possible orderings:
+	//
+	// a, b, c
+	// a, c, b
+	// b, a, c
+	// b, c, a
+	// c, a, b
+	// c, b, a
+
+	if (a <= b) {
+		if (b <= c) {
+			return b // a, b, c
+		} else if (a <= c) {
+			return c // a, c, b
+		} else {
+			return a // c, a, b
+		}
+	} else {
+		if (a <= c) {
+			return a // b, a, c
+		} else if (b <= c) {
+			return c // b, c, a
+		} else {
+			return b // c, b, a
+		}
+	}
+}
+
+// Slower, variable-width median filter using the `moving-median` package
+export async function medianFilter(points: number[], width: number) {
+	const { default: createMedianFilter } = await import('moving-median')
+
+	const filter = createMedianFilter(width)
+	const result = []
+
+	for (let i = 0; i < points.length; i++) {
+		result.push(filter(points[i]))
+	}
+
+	return result
+}