echogarden 0.12.2 → 1.0.0

package/src/api/Alignment.ts

@@ -1,36 +1,61 @@
-import { extendDeep } from "../utilities/ObjectUtilities.js"
-
-import { logToStderr } from "../utilities/Utilities.js"
-import { AudioSourceParam, RawAudio, downmixToMonoAndNormalize, ensureRawAudio, getRawAudioDuration, normalizeAudioLevel, trimAudioEnd } from "../audio/AudioUtilities.js"
-import { Logger } from "../utilities/Logger.js"
-import { resampleAudioSpeex } from "../dsp/SpeexResampler.js"
-
-import * as API from "./API.js"
-import { Timeline, addTimeOffsetToTimeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from "../utilities/Timeline.js"
-import { formatLanguageCodeWithName, getDefaultDialectForLanguageCodeIfPossible, getShortLanguageCode, normalizeLanguageCode } from "../utilities/Locale.js"
-import { WhisperOptions, whisperOptionsDefaults } from "../recognition/WhisperSTT.js"
-import chalk from "chalk"
-import { DtwGranularity } from "../alignment/SpeechAlignment.js"
-import { SubtitlesConfig, defaultSubtitlesBaseConfig } from "../subtitles/Subtitles.js"
-import { synthesize } from "./API.js"
-import { EspeakOptions, defaultEspeakOptions } from "../synthesis/EspeakTTS.js"
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+import { logToStderr } from '../utilities/Utilities.js'
+import { AudioSourceParam, RawAudio, ensureRawAudio, getRawAudioDuration, normalizeAudioLevel, trimAudioEnd } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+
+import * as API from './API.js'
+import { Timeline, addTimeOffsetToTimeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
+import { formatLanguageCodeWithName, getDefaultDialectForLanguageCodeIfPossible, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
+import { WhisperOptions } from '../recognition/WhisperSTT.js'
+import chalk from 'chalk'
+import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
+import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
+
 
 const log = logToStderr
 
 export async function align(input: AudioSourceParam, transcript: string, options: AlignmentOptions): Promise<AlignmentResult> {
 	const logger = new Logger()
+
 	const startTimestamp = logger.getTimestamp()
 
-	logger.start("Prepare for alignment")
+	options = extendDeep(defaultAlignmentOptions, options)
 
 	const inputRawAudio = await ensureRawAudio(input)
 
-	let sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+	let sourceRawAudio: RawAudio
+	let isolatedRawAudio: RawAudio | undefined
+	let backgroundRawAudio: RawAudio | undefined
+
+	if (options.isolate) {
+		logger.log(``)
+		logger.end();
+
+		({ isolatedRawAudio, backgroundRawAudio } = await API.isolate(inputRawAudio, options.sourceSeparation!))
+
+		logger.end()
+		logger.log(``)
+
+		sourceRawAudio = await ensureRawAudio(isolatedRawAudio, 16000, 1)
+	} else {
+		sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
+	}
+
+	let sourceUncropTimeline: Timeline | undefined
+
+	if (options.crop) {
+		logger.start('Crop using voice activity detection');
+		({ timeline: sourceUncropTimeline, croppedRawAudio: sourceRawAudio } = await API.detectVoiceActivity(sourceRawAudio, options.vad!))
+
+		logger.end()
+	}
+
+	logger.start('Prepare for alignment')
+
 	sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
 	sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
 
-	options = extendDeep(defaultAlignmentOptions, options)
-
 	if (options.dtw!.windowDuration == null) {
 		const sourceAudioDuration = getRawAudioDuration(sourceRawAudio)
 
@@ -48,7 +73,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 	if (options.language) {
 		language = normalizeLanguageCode(options.language!)
 	} else {
-		logger.start("No language specified. Detecting language")
+		logger.start('No language specified. Detecting language')
 		const { detectedLanguage } = await API.detectTextLanguage(transcript, options.languageDetection || {})
 		language = detectedLanguage
 
@@ -58,34 +83,9 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 	language = getDefaultDialectForLanguageCodeIfPossible(language)
 
-	logger.start("Load alignment module")
-
-	const { alignUsingDtwWithRecognition, alignUsingDtw } = await import("../alignment/SpeechAlignment.js")
+	logger.start('Load alignment module')
 
-	async function getAlignmentReference() {
-		logger.start("Create alignment reference with eSpeak")
-
-		const synthesisOptions: API.SynthesisOptions = {
-			engine: "espeak",
-			language,
-			plainText: options.plainText,
-			customLexiconPaths: options.customLexiconPaths,
-
-			espeak: {
-				useKlatt: false
-			}
-		}
-
-		let { audio: referenceRawAudio,  timeline: segmentTimeline, voice: espeakVoice } = await synthesize(transcript, synthesisOptions)
-
-		const sentenceTimeline = segmentTimeline.flatMap(entry => entry.timeline!)
-		const wordTimeline = sentenceTimeline.flatMap(entry => entry.timeline!)
-
-		referenceRawAudio = await resampleAudioSpeex(referenceRawAudio as RawAudio, 16000)
-		referenceRawAudio = downmixToMonoAndNormalize(referenceRawAudio)
-
-		return { referenceRawAudio, referenceTimeline: wordTimeline, espeakVoice }
-	}
+	const { alignUsingDtwWithRecognition: alignUsingDtwWithRecognitionReference, alignUsingDtw } = await import('../alignment/SpeechAlignment.js')
 
 	function getDtwWindowDurationsAndGranularities() {
 		let granularities: DtwGranularity[]
@@ -123,8 +123,12 @@ export async function align(input: AudioSourceParam, transcript: string, options
 	let mappedTimeline: Timeline
 
 	switch (options.engine) {
-		case "dtw": {
-			const { referenceRawAudio, referenceTimeline } = await getAlignmentReference()
+		case 'dtw': {
+			const {
+				referenceRawAudio,
+				referenceTimeline
+			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths)
+
 			logger.end()
 
 			const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities()
@@ -134,62 +138,70 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			break
 		}
 
-		case "dtw-ra": {
-			/*
-			const promptWords = (await splitToWords(prompt, language)).filter(word => isWord(word))
+		case 'dtw-ra': {
+			const recognitionOptions: API.RecognitionOptions =
+				extendDeep({ crop: options.crop, language }, options.recognition)
 
-			shuffleArrayInPlace(promptWords, this.randomGen)
-			//promptWords.reverse()
+			logger.end()
 
-			prompt = promptWords.join(" ")
-			*/
+			// Recognize source audio
+			const { transcript: recognizedTranscript, wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
 
-			const recognitionOptionsDefaults: API.RecognitionOptions = {
-				engine: "whisper",
-				language,
-			}
+			// Synthesize the ground-truth transcript and get its timeline
+			logger.start('Synthesize ground-truth transcript with eSpeak')
 
-			const recognitionOptions: API.RecognitionOptions = extendDeep(recognitionOptionsDefaults, options.recognition || {})
+			const {
+				referenceRawAudio,
+				referenceTimeline,
+			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, true)
 
-			logger.end()
-
-			const { wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
+			// Synthesize the recognized transcript and get its timeline
+			logger.start('Synthesize recognized transcript with eSpeak')
 
-			const { referenceRawAudio, referenceTimeline, espeakVoice } = await getAlignmentReference()
+			const {
+				referenceRawAudio: synthesizedRecognizedTranscriptRawAudio,
+				referenceTimeline: synthesizedRecognitionTimeline
+			} = await createAlignmentReferenceUsingEspeak(recognizedTranscript, language, undefined, undefined, true)
 
 			logger.end()
 
 			const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities()
 
-			const espeakOptions: EspeakOptions = { ...defaultEspeakOptions, voice: espeakVoice, useKlatt: false }
-
 			const phoneAlignmentMethod = options.dtw!.phoneAlignmentMethod!
 
-			mappedTimeline = await alignUsingDtwWithRecognition(sourceRawAudio, referenceRawAudio, referenceTimeline, recognitionTimeline, granularities, windowDurations, espeakOptions, phoneAlignmentMethod)
+			// Align the ground-truth transcript and the recognized transcript
+			mappedTimeline = await alignUsingDtwWithRecognitionReference(
+				sourceRawAudio,
+				referenceRawAudio,
+				referenceTimeline,
+
+				recognitionTimeline,
+				synthesizedRecognizedTranscriptRawAudio,
+				synthesizedRecognitionTimeline,
+
+				granularities,
+				windowDurations,
+				phoneAlignmentMethod)
 
 			break
 		}
 
-		case "whisper": {
-			const WhisperSTT = await import("../recognition/WhisperSTT.js")
+		case 'whisper': {
+			const WhisperSTT = await import('../recognition/WhisperSTT.js')
 
 			const whisperOptions = options.whisper!
 
 			const shortLanguageCode = getShortLanguageCode(language)
 
-			const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, language)
-
-			if (modelName.endsWith(".en") && shortLanguageCode != "en") {
-				throw new Error(`The model '${modelName}' is English only and cannot transcribe language '${shortLanguageCode}'`)
-			}
+			const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, language)
 
 			if (getRawAudioDuration(sourceRawAudio) > 30) {
-				throw new Error("Whisper based alignment currently only supports audio inputs that are 30s or less")
+				throw new Error('Whisper based alignment currently only supports audio inputs that are 30s or less')
 			}
 
 			logger.end()
 
-			mappedTimeline = await WhisperSTT.align(sourceRawAudio, transcript, modelName, modelDir, tokenizerDir, shortLanguageCode)
+			mappedTimeline = await WhisperSTT.align(sourceRawAudio, transcript, modelName, modelDir, shortLanguageCode)
 
 			break
 		}
@@ -199,8 +211,15 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		}
 	}
 
+	// If the audio was cropped before recognition, map the timestamps back to the original audio
+	if (sourceUncropTimeline && sourceUncropTimeline.length > 0) {
+		API.convertCroppedToUncroppedTimeline(mappedTimeline, sourceUncropTimeline)
+	}
+
+	// Add text offsets
 	addWordTextOffsetsToTimeline(mappedTimeline, transcript)
 
+	// Make segment timeline
 	const { segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(mappedTimeline, transcript, language, options.plainText?.paragraphBreaks, options.plainText?.whitespace)
 
 	logger.end()
@@ -209,9 +228,13 @@ export async function align(input: AudioSourceParam, transcript: string, options
 	return {
 		timeline: segmentTimeline,
 		wordTimeline: mappedTimeline,
-		inputRawAudio,
+
 		transcript,
-		language
+		language,
+
+		inputRawAudio,
+		isolatedRawAudio,
+		backgroundRawAudio,
 	}
 }
 
@@ -246,23 +269,33 @@ export async function alignSegments(sourceRawAudio: RawAudio, segmentTimeline: T
 export interface AlignmentResult {
 	timeline: Timeline
 	wordTimeline: Timeline
+
 	transcript: string
 	language: string
+
 	inputRawAudio: RawAudio
+	isolatedRawAudio?: RawAudio
+	backgroundRawAudio?: RawAudio
 }
 
-export type AlignmentEngine = "dtw" | "dtw-ra" | "whisper"
-export type PhoneAlignmentMethod = "interpolation" | "dtw"
+export type AlignmentEngine = 'dtw' | 'dtw-ra' | 'whisper'
+export type PhoneAlignmentMethod = 'interpolation' | 'dtw'
 
 export interface AlignmentOptions {
 	engine?: AlignmentEngine
 
 	language?: string
 
-	languageDetection?: API.TextLanguageDetectionOptions
+	isolate?: boolean
+
+	crop?: boolean
 
 	customLexiconPaths?: string[]
 
+	languageDetection?: API.TextLanguageDetectionOptions
+
+	vad?: API.VADOptions
+
 	plainText?: API.PlainTextOptions
 
 	subtitles?: SubtitlesConfig
@@ -275,19 +308,25 @@ export interface AlignmentOptions {
 
 	recognition?: API.RecognitionOptions
 
+	sourceSeparation?: API.SourceSeparationOptions
+
 	whisper?: WhisperOptions
 }
 
 export const defaultAlignmentOptions: AlignmentOptions = {
-	engine: "dtw",
+	engine: 'dtw',
 
 	language: undefined,
 
-	languageDetection: {
-	},
+	isolate: false,
+
+	crop: true,
 
 	customLexiconPaths: undefined,
 
+	languageDetection: {
+	},
+
 	plainText: {
 		paragraphBreaks: 'double',
 		whitespace: 'collapse'
@@ -302,28 +341,46 @@ export const defaultAlignmentOptions: AlignmentOptions = {
 	},
 
 	recognition: {
+		whisper: {
+			temperature: 0.15,
+			topCandidateCount: 5,
+			punctuationThreshold: 0.2,
+			maxTokensPerPart: 250,
+			autoPromptParts: true,
+			suppressRepetition: true,
+			seed: undefined,
+			decodeTimestampTokens: false,
+		}
+	},
+
+	vad: {
+		engine: 'adaptive-gate'
+	},
+
+	sourceSeparation: {
 	},
 
-	whisper: whisperOptionsDefaults
+	whisper: {
+	}
 }
 
 export const alignmentEngines: API.EngineMetadata[] = [
 	{
 		id: 'dtw',
 		name: 'Dynamic Time Warping',
-		description: 'Makes use of synthesis to find the best mapping between the original audio and its transcript.',
+		description: 'Makes use of a synthesized reference to find the best mapping between the spoken audio and its transcript.',
 		type: 'local'
 	},
 	{
 		id: 'dtw-ra',
 		name: 'Dynamic Time Warping with Recognition Assist',
-		description: 'Makes use of both synthesis and recognition to find the best mapping between the original audio and its transcript.',
+		description: 'Makes use of both a synthesized reference and a synthsized recognized transcript to find the best mapping between the spoken audio and its transcript.',
 		type: 'local'
 	},
 	{
 		id: 'whisper',
 		name: 'OpenAI Whisper',
-		description: 'Extracts timestamps from the internal state of the Whisper recognition model (note: currently limited to a maximum of 30s audio length).',
+		description: 'Extracts timestamps from the internal state of the Whisper recognition model (note: currently limited to a maximum audio duration of 30 seconds).',
 		type: 'local'
 	}
 ]

package/src/api/Common.ts

@@ -1,4 +1,4 @@
-export const appName = "echogarden"
+export const appName = 'echogarden'
 
 export interface EngineMetadata {
 	id: string

package/src/api/Denoising.ts

@@ -1,12 +1,12 @@
-import { extendDeep } from "../utilities/ObjectUtilities.js"
+import { extendDeep } from '../utilities/ObjectUtilities.js'
 
-import { AudioSourceParam, RawAudio, applyGainDecibels, ensureRawAudio, getAudioPeakDecibels, mixAudio, normalizeAudioLevel } from "../audio/AudioUtilities.js"
-import { Logger } from "../utilities/Logger.js"
+import { AudioSourceParam, RawAudio, applyGainDecibels, ensureRawAudio, getSamplePeakDecibels, mixAudio, normalizeAudioLevel } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
 
-import { logToStderr } from "../utilities/Utilities.js"
-import { resampleAudioSpeex } from "../dsp/SpeexResampler.js"
-import { EngineMetadata } from "./Common.js"
-import chalk from "chalk"
+import { logToStderr } from '../utilities/Utilities.js'
+import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
+import { EngineMetadata } from './Common.js'
+import chalk from 'chalk'
 
 const log = logToStderr
 
@@ -14,7 +14,7 @@ export async function denoise(input: AudioSourceParam, options: DenoisingOptions
 	const logger = new Logger()
 	const startTime = logger.getTimestamp()
 
-	logger.start("Prepare for denoising")
+	logger.start('Prepare for denoising')
 
 	options = extendDeep(defaultDenoisingOptions, options)
 
@@ -30,8 +30,8 @@ export async function denoise(input: AudioSourceParam, options: DenoisingOptions
 	let denoisedAudio: RawAudio
 
 	switch (options.method) {
-		case "rnnoise": {
-			const RNNoise = await import("../denoising/RNNoise.js")
+		case 'rnnoise': {
+			const RNNoise = await import('../denoising/RNNoise.js')
 			logger.end()
 
 			const denoisedAudioChannels: Float32Array[] = []
@@ -53,27 +53,27 @@ export async function denoise(input: AudioSourceParam, options: DenoisingOptions
 		}
 	}
 
-	logger.start("Postprocess audio")
+	logger.start('Postprocess audio')
 
 	const shouldNormalize = options.postProcessing!.normalizeAudio!
-	const targetPeakDb = options.postProcessing!.targetPeakDb!
-	const maxIncreaseDb = options.postProcessing!.maxIncreaseDb!
-	const dryMixGainDb = options.postProcessing!.dryMixGainDb!
+	const targetPeakDecibels = options.postProcessing!.targetPeak!
+	const maxGainIncreaseDecibels = options.postProcessing!.maxGainIncrease!
+	const dryMixGainDecibels = options.postProcessing!.dryMixGain!
 
-	const preMixPeakDb = getAudioPeakDecibels(denoisedAudio.audioChannels)
-	denoisedAudio = mixAudio(denoisedAudio, applyGainDecibels(resampledRawAudio, dryMixGainDb))
-	const postMixPeakDb = getAudioPeakDecibels(denoisedAudio.audioChannels)
+	const preMixPeakDecibels = getSamplePeakDecibels(denoisedAudio.audioChannels)
+	denoisedAudio = mixAudio(denoisedAudio, applyGainDecibels(resampledRawAudio, dryMixGainDecibels))
+	const postMixPeakDecibels = getSamplePeakDecibels(denoisedAudio.audioChannels)
 
 	if (shouldNormalize) {
-		denoisedAudio = normalizeAudioLevel(denoisedAudio, targetPeakDb, maxIncreaseDb)
+		denoisedAudio = normalizeAudioLevel(denoisedAudio, targetPeakDecibels, maxGainIncreaseDecibels)
 	} else {
-		denoisedAudio = applyGainDecibels(denoisedAudio, preMixPeakDb - postMixPeakDb)
+		denoisedAudio = applyGainDecibels(denoisedAudio, preMixPeakDecibels - postMixPeakDecibels)
 	}
 
 	logger.end()
 
 	logger.log('')
-	logger.logDuration("Total denoising time", startTime, chalk.magentaBright)
+	logger.logDuration('Total denoising time', startTime, chalk.magentaBright)
 
 	return {
 		denoisedAudio,
@@ -86,27 +86,27 @@ export interface DenoisingResult {
 	inputRawAudio: RawAudio
 }
 
-export type DenoisingMethod = "rnnoise"
+export type DenoisingMethod = 'rnnoise'
 
 export interface DenoisingOptions {
 	method?: DenoisingMethod,
 	postProcessing?: {
 		normalizeAudio: boolean
-		targetPeakDb: number
-		maxIncreaseDb: number
+		targetPeak: number
+		maxGainIncrease: number
 
-		dryMixGainDb?: number
+		dryMixGain?: number
 	}
 }
 
 export const defaultDenoisingOptions: DenoisingOptions = {
-	method: "rnnoise",
+	method: 'rnnoise',
 
 	postProcessing: {
 		normalizeAudio: false,
-		targetPeakDb: -3,
-		maxIncreaseDb: 30,
-		dryMixGainDb: -20,
+		targetPeak: -3,
+		maxGainIncrease: 30,
+		dryMixGain: -20,
 	}
 }