echogarden 0.12.2 → 1.0.0

package/src/source-separation/MDXNetSourceSeparation.ts

@@ -0,0 +1,228 @@
+import Onnx from 'onnxruntime-node'
+import { RawAudio } from '../audio/AudioUtilities.js';
+import { binBufferToComplex, complexToBinBuffer, getWindowWeights, stftr, stiftr } from '../dsp/FFT.js';
+import { ComplexNumber } from '../math/VectorMath.js';
+import { logToStderr } from '../utilities/Utilities.js';
+import { Logger } from '../utilities/Logger.js';
+
+const log = logToStderr
+
+export async function isolate(rawAudio: RawAudio, modelFilePath: string) {
+	const model = new MDXNet(modelFilePath)
+
+	return model.processAudio(rawAudio)
+}
+
+export class MDXNet {
+	session?: Onnx.InferenceSession
+
+	constructor(public readonly modelFilePath: string) {
+	}
+
+	async processAudio(rawAudio: RawAudio) {
+		if (rawAudio.audioChannels.length != 2) {
+			throw new Error(`Input audio must be stereo`)
+		}
+
+		if (rawAudio.sampleRate != 44100) {
+			throw new Error(`Input audio must have a 44100 Hz sampling rate`)
+		}
+
+		if (!this.session) {
+			await this.initializeSession(this.modelFilePath)
+		}
+
+		const logger = new Logger()
+
+		const session = this.session!
+
+		const sampleRate = rawAudio.sampleRate
+		const fftSize = 6144
+		const fftCount = 2048
+		const fftWindowSize = fftSize
+		const fftHopSize = 1024
+
+		const segmentSize = 256
+		const segmentHopSize = 240
+
+		const sampleCount = rawAudio.audioChannels[0].length
+
+		logger.start('Compute STFT of full waveform')
+
+		const fftFramesLeft = await stftr(rawAudio.audioChannels[0], fftSize, fftWindowSize, fftHopSize, 'hann')
+		const fftFramesRight = await stftr(rawAudio.audioChannels[1], fftSize, fftWindowSize, fftHopSize, 'hann')
+
+		const fftFramesLeftComplex = fftFramesLeft.map(frame => binBufferToComplex(frame).slice(0, fftCount))
+		const fftFramesRightComplex = fftFramesRight.map(frame => binBufferToComplex(frame).slice(0, fftCount))
+
+		const audioForSegments: Float32Array[][] = []
+
+		for (let segmentOffset = 0; segmentOffset < fftFramesLeft.length; segmentOffset += segmentHopSize) {
+			const timePosition = segmentOffset * (fftHopSize / sampleRate)
+
+			logger.start(`Process segment at time position ${timePosition.toFixed(2)}`)
+
+			const fftFramesLeftComplexForSegment = fftFramesLeftComplex.slice(segmentOffset, segmentOffset + segmentSize)
+			const fftFramesRightComplexForSegment = fftFramesRightComplex.slice(segmentOffset, segmentOffset + segmentSize)
+
+			const segmentLength = fftFramesLeftComplexForSegment.length
+
+			const flattenedInputTensor = new Float32Array(1 * 4 * fftCount * segmentSize)
+
+			{
+				let writePosition = 0
+
+				for (let tensorChannelIndex = 0; tensorChannelIndex < 4; tensorChannelIndex++) {
+					for (let binIndex = 0; binIndex < fftCount; binIndex++) {
+						for (let frameIndex = 0; frameIndex < segmentSize; frameIndex++) {
+							let value = 0
+
+							if (frameIndex < segmentLength && binIndex >= 0) {
+								let frame: ComplexNumber[]
+
+								if (tensorChannelIndex < 2) {
+									frame = fftFramesLeftComplexForSegment[frameIndex]
+								} else {
+									frame = fftFramesRightComplexForSegment[frameIndex]
+								}
+
+								const bin = frame[binIndex]
+
+								if (tensorChannelIndex % 2 === 0) {
+									value = bin.real
+								} else {
+									value = bin.imaginary
+								}
+							}
+
+							flattenedInputTensor[writePosition++] = value
+						}
+					}
+				}
+			}
+
+			const inputTensor = new Onnx.Tensor('float32', flattenedInputTensor, [1, 4, 2048, 256])
+
+			const { output: outputTensor } = await session.run({ input: inputTensor })
+
+			const flattenedOutputTensor = outputTensor.data as Float32Array
+
+			const outputChannelComplexFrames: ComplexNumber[][][] = []
+
+			{
+				for (let outChannelIndex = 0; outChannelIndex < 2; outChannelIndex++) {
+					const framesForChannel: ComplexNumber[][] = []
+
+					for (let frameIndex = 0; frameIndex < 256; frameIndex++) {
+						const frame: ComplexNumber[] = []
+
+						for (let binIndex = 0; binIndex < fftSize; binIndex++) {
+							frame.push({ real: 0, imaginary: 0 })
+						}
+
+						framesForChannel.push(frame)
+					}
+
+					outputChannelComplexFrames.push(framesForChannel)
+				}
+
+				let readPosition = 0
+
+				for (let tensorChannelIndex = 0; tensorChannelIndex < 4; tensorChannelIndex++) {
+					const outChannelIndex = tensorChannelIndex < 2 ? 0 : 1
+
+					for (let binIndex = 0; binIndex < 2048; binIndex++) {
+						for (let frameIndex = 0; frameIndex < 256; frameIndex++) {
+							const bin = outputChannelComplexFrames[outChannelIndex][frameIndex][binIndex]
+
+							if (tensorChannelIndex % 2 === 0) {
+								bin.real = flattenedOutputTensor[readPosition++]
+							} else {
+								bin.imaginary = flattenedOutputTensor[readPosition++]
+							}
+						}
+					}
+				}
+			}
+
+			const outputAudioChannels: Float32Array[] = []
+
+			//logger.start(`Compute inverse STFT for segment`)
+			for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
+				let outputChannelFlattenedFrames = outputChannelComplexFrames[channelIndex]
+					.map(frame => complexToBinBuffer(frame).map(value => value / fftSize))
+
+				const samples = await stiftr(
+					outputChannelFlattenedFrames,
+					fftSize,
+					fftWindowSize,
+					fftHopSize,
+					'hann')
+
+				outputAudioChannels.push(samples)
+			}
+
+			//logger.log(`Reconstructed waveform peak: ${getAudioPeakDecibels(outputAudioChannels).toFixed(3)}dB`)
+			//await playAudioSamples({ audioChannels: outputAudioChannels, sampleRate })
+
+			audioForSegments.push(outputAudioChannels)
+		}
+
+		// Join segments using overlapping Hann windows
+		logger.start(`Join segments`)
+		const concatenatedAudioChannels = [new Float32Array(sampleCount), new Float32Array(sampleCount)]
+
+		{
+			const segmentCount = audioForSegments.length
+
+			const segmentSampleCount = audioForSegments[0][0].length
+
+			const windowWeights = getWindowWeights('hann', segmentSampleCount)
+
+			const sumOfWeightsForSample = new Float32Array(sampleCount)
+
+			for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex++) {
+				const segmentStartFrameIndex = segmentIndex * segmentHopSize
+				const segmentStartSampleIndex = segmentStartFrameIndex * fftHopSize
+
+				const segmentSamples = audioForSegments[segmentIndex]
+
+				for (let segmentSampleOffset = 0; segmentSampleOffset < segmentSampleCount; segmentSampleOffset++) {
+					const sampleIndex = segmentStartSampleIndex + segmentSampleOffset
+
+					if (sampleIndex >= sampleCount) {
+						break
+					}
+
+					const weight = windowWeights[segmentSampleOffset]
+
+					for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
+						concatenatedAudioChannels[channelIndex][sampleIndex] += segmentSamples[channelIndex][segmentSampleOffset] * weight
+					}
+
+					sumOfWeightsForSample[sampleIndex] += weight
+				}
+			}
+
+			for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) {
+				for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
+					concatenatedAudioChannels[channelIndex][sampleIndex] /= sumOfWeightsForSample[sampleIndex] + 1e-8
+				}
+			}
+		}
+
+		const isolatedRawAudio: RawAudio = { audioChannels: concatenatedAudioChannels, sampleRate }
+
+		logger.end()
+
+		return isolatedRawAudio
+	}
+
+	private async initializeSession(modelPath: string) {
+		const onnxOptions: Onnx.InferenceSession.SessionOptions = {
+			logSeverityLevel: 3
+		}
+
+		this.session = await Onnx.InferenceSession.create(modelPath, onnxOptions)
+	}
+}

package/src/speech-language-detection/SileroLanguageDetection.ts

@@ -1,7 +1,7 @@
 import Onnx from 'onnxruntime-node'
 import { softmax } from '../math/VectorMath.js'
-import { Logger } from "../utilities/Logger.js"
-import { RawAudio } from "../audio/AudioUtilities.js"
+import { Logger } from '../utilities/Logger.js'
+import { RawAudio } from '../audio/AudioUtilities.js'
 import { readAndParseJsonFile } from '../utilities/FileSystem.js'
 import { detectSpeechLanguageByParts, type LanguageDetectionResults } from '../api/LanguageDetection.js'
 import { languageCodeToName } from '../utilities/Locale.js'
@@ -41,7 +41,7 @@ export class SileroLanguageDetection {
 
 	async initialize() {
 		const logger = new Logger()
-		logger.start("Initialize ONNX inference session")
+		logger.start('Initialize ONNX inference session')
 
 		this.languageDictionary = await readAndParseJsonFile(this.languageDictionaryPath)
 		this.languageGroupDictionary = await readAndParseJsonFile(this.languageGroupDictionaryPath)
@@ -58,7 +58,7 @@ export class SileroLanguageDetection {
 	async detectLanguage(rawAudio: RawAudio) {
 		const logger = new Logger()
 
-		logger.start("Detect language with Silero")
+		logger.start('Detect language with Silero')
 
 		const audioSamples = rawAudio.audioChannels[0]
 
@@ -68,10 +68,10 @@ export class SileroLanguageDetection {
 
 		const results = await this.session!.run(inputs)
 
-		logger.start("Parse model results")
+		logger.start('Parse model results')
 
-		const languageLogits = results["output"].data
-		const languageGroupLogits = results["2038"].data
+		const languageLogits = results['output'].data
+		const languageGroupLogits = results['2038'].data
 
 		const languageProbabilities = softmax(languageLogits as any)
 		const languageGroupProbabilities = softmax(languageGroupLogits as any)
@@ -80,7 +80,7 @@ export class SileroLanguageDetection {
 
 		for (let i = 0; i < languageProbabilities.length; i++) {
 			const languageString = this.languageDictionary[i]
-			const languageCode = languageString.replace(/,.*$/, "")
+			const languageCode = languageString.replace(/,.*$/, '')
 
 			languageResults.push({
 				language: languageCode,

package/src/subtitles/Subtitles.ts

@@ -428,11 +428,11 @@ function getCuesFromTimeline_IsolateLines(timeline: Timeline, config: SubtitlesC
 
 	addCuesFrom(timeline)
 	addCueFromCurrentWords() // Add any remaining words
-	
+
 	return cues
 }
 
-function tryParseTimeRangePatternWithHours(line: string) {
+export function tryParseTimeRangePatternWithHours(line: string) {
 	const timeRangePatternWithHours = /^(\d+)\:(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)\:(\d+)[\.,](\d+)/
 	const match = timeRangePatternWithHours.exec(line)
 
@@ -456,7 +456,7 @@ function tryParseTimeRangePatternWithHours(line: string) {
 	return { startTime, endTime, succeeded: true }
 }
 
-function tryParseTimeRangePatternWithoutHours(line: string) {
+export function tryParseTimeRangePatternWithoutHours(line: string) {
 	const timeRangePatternWithHours = /^(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)[\.,](\d+)/
 	const match = timeRangePatternWithHours.exec(line)
 

package/src/synthesis/AwsPollyTTS.ts

@@ -1,15 +1,15 @@
-import type { LanguageCode, SynthesizeSpeechCommandInput, VoiceId } from "@aws-sdk/client-polly"
-import { IncomingMessage } from "http"
-import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
-import { Logger } from "../utilities/Logger.js"
+import type { LanguageCode, SynthesizeSpeechCommandInput, VoiceId } from '@aws-sdk/client-polly'
+import { IncomingMessage } from 'http'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Logger } from '../utilities/Logger.js'
 
-import { readBinaryIncomingMessage } from "../utilities/Utilities.js"
+import { readBinaryIncomingMessage } from '../utilities/Utilities.js'
 
-export async function synthesize(text: string, language: string | undefined, voice: string, region: string, accessKeyId: string, secretAccessKey: string, engine: "standard" | "neural" = "standard", ssmlEnabled = false, lexiconNames?: string[]) {
+export async function synthesize(text: string, language: string | undefined, voice: string, region: string, accessKeyId: string, secretAccessKey: string, engine: 'standard' | 'neural' = 'standard', ssmlEnabled = false, lexiconNames?: string[]) {
 	const logger = new Logger()
-	logger.start("Load AWS SDK client module")
+	logger.start('Load AWS SDK client module')
 
-	const polly = await import("@aws-sdk/client-polly")
+	const polly = await import('@aws-sdk/client-polly')
 
 	const pollyClient = new polly.PollyClient({
 		region,
@@ -28,12 +28,12 @@ export async function synthesize(text: string, language: string | undefined, voi
 		Text: text,
 		LexiconNames: lexiconNames,
 
-		TextType: ssmlEnabled ? "ssml" : "text",
+		TextType: ssmlEnabled ? 'ssml' : 'text',
 
-		OutputFormat: "mp3",
+		OutputFormat: 'mp3',
 	}
 
-	logger.start("Request synthesis from AWS Polly")
+	logger.start('Request synthesis from AWS Polly')
 
 	const command = new polly.SynthesizeSpeechCommand(params)
 
@@ -52,11 +52,11 @@ export async function synthesize(text: string, language: string | undefined, voi
 
 export async function getVoiceList(region: string, accessKeyId: string, secretAccessKey: string) {
 	const logger = new Logger()
-	logger.start("Load AWS SDK client module")
+	logger.start('Load AWS SDK client module')
 
-	const polly = await import("@aws-sdk/client-polly")
+	const polly = await import('@aws-sdk/client-polly')
 
-	logger.start("Request voice list from AWS Polly")
+	logger.start('Request voice list from AWS Polly')
 
 	const pollyClient = new polly.PollyClient({
 		region,

package/src/synthesis/AzureCognitiveServicesTTS.ts

@@ -1,6 +1,6 @@
 import * as SpeechSDK from 'microsoft-cognitiveservices-speech-sdk'
 
-import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 
 import { escape } from 'html-escaper'
 
@@ -12,15 +12,15 @@ export async function synthesize(
 	text: string,
 	subscriptionKey: string,
 	serviceRegion: string,
-	languageCode = "en-US",
-	voice = "Microsoft Server Speech Text to Speech Voice (en-US, AvaNeural)",
+	languageCode = 'en-US',
+	voice = 'Microsoft Server Speech Text to Speech Voice (en-US, AvaNeural)',
 	ssmlEnabled = false,
-	ssmlPitchString = "+0Hz",
-	ssmlRateString = "+0%") {
+	ssmlPitchString = '+0Hz',
+	ssmlRateString = '+0%') {
 
 	return new Promise<{ rawAudio: RawAudio, timeline: Timeline }>((resolve, reject) => {
 		const logger = new Logger()
-		logger.start("Request synthesis from Azure Cognitive Services")
+		logger.start('Request synthesis from Azure Cognitive Services')
 
 		const speechConfig = SpeechSDK.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion)
 
@@ -72,7 +72,7 @@ export async function synthesize(
 
 			const rawAudio = await FFMpegTranscoder.decodeToChannels(encodedAudio, 24000, 1)
 
-			logger.start("Convert boundary events to a timeline")
+			logger.start('Convert boundary events to a timeline')
 
 			const timeline = boundaryEventsToTimeline(events, getRawAudioDuration(rawAudio))
 
@@ -85,7 +85,7 @@ export async function synthesize(
 			reject(error)
 		}
 
-		if (!ssmlEnabled && ssmlPitchString != "+0%" || ssmlRateString != "+0Hz") {
+		if (!ssmlEnabled && ssmlPitchString != '+0%' || ssmlRateString != '+0Hz') {
 			ssmlEnabled = true
 			text = escape(text)
 		}
@@ -123,7 +123,7 @@ export function boundaryEventsToTimeline(events: any[], totalDuration: number) {
 	for (const event of events) {
 		const boundaryType = event.boundaryType != null ? event.boundaryType : event.Type
 
-		if (boundaryType != "WordBoundary") {
+		if (boundaryType != 'WordBoundary') {
 			continue
 		}
 
@@ -135,7 +135,7 @@ export function boundaryEventsToTimeline(events: any[], totalDuration: number) {
 		const endTime = (offset + duration) / 10000000
 
 		timeline.push({
-			type: "word",
+			type: 'word',
 			text,
 			startTime,
 			endTime

package/src/synthesis/CoquiServerTTS.ts

@@ -1,27 +1,27 @@
-import { request } from "gaxios"
-import { decodeWaveBuffer } from "../audio/AudioUtilities.js"
-import { Logger } from "../utilities/Logger.js"
-import { logToStderr } from "../utilities/Utilities.js"
+import { request } from 'gaxios'
+import { decodeWaveToRawAudio } from '../audio/AudioUtilities.js'
+import { Logger } from '../utilities/Logger.js'
+import { logToStderr } from '../utilities/Utilities.js'
 const log = logToStderr
 
-export async function synthesize(text: string, speakerId: string | null, serverURL = "http://[::1]:5002") {
+export async function synthesize(text: string, speakerId: string | null, serverURL = 'http://[::1]:5002') {
 	const logger = new Logger()
-	logger.start("Request synthesis from Coqui Server")
+	logger.start('Request synthesis from Coqui Server')
 
 	const response = await request<Buffer>({
 		url: `${serverURL}/api/tts`,
 
 		params: {
-			"text": text,
-			"speaker_id": speakerId
+			'text': text,
+			'speaker_id': speakerId
 		},
 
-		responseType: "arraybuffer"
+		responseType: 'arraybuffer'
 	})
 
 	const waveData = Buffer.from(response.data)
 
-	const rawAudio = decodeWaveBuffer(waveData).rawAudio
+	const rawAudio = decodeWaveToRawAudio(waveData).rawAudio
 
 	logger.end()
 

package/src/synthesis/ElevenlabsTTS.ts

@@ -0,0 +1,137 @@
+import { GaxiosResponse, request } from 'gaxios'
+import { SynthesisVoice, VoiceGender } from '../api/API.js'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Logger } from '../utilities/Logger.js'
+import { logToStderr } from '../utilities/Utilities.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+const log = logToStderr
+
+export async function synthesize(text: string, voiceId: string, modelId: string, options: ElevenlabsTTSOptions) {
+	const logger = new Logger()
+	logger.start('Request synthesis from ElevenLabs')
+
+	options = extendDeep(defaultElevenlabsTTSOptions, options)
+
+	let response: GaxiosResponse<any>
+
+	try {
+		response = await request<any>({
+			url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`,
+
+			method: 'POST',
+
+			headers: {
+				'accept': 'audio/mpeg',
+				'xi-api-key': options.apiKey,
+			},
+
+			data: {
+				text,
+
+				model_id: modelId,
+
+				voice_setting: {
+					stability: options.stability,
+					similarity_boost: options.similarityBoost,
+					style: options.style,
+					use_speaker_boost: options.useSpeakerBoost
+				}
+			},
+
+			responseType: 'arraybuffer'
+		})
+	} catch (e: any) {
+		const response = e.response
+
+		if (response) {
+			logger.log(`Request failed with status code ${response.status}`)
+
+			if (response.data) {
+				logger.log(`Server responded with:`)
+				logger.log(response.data)
+			}
+		}
+
+		throw e
+	}
+
+	logger.start('Decode synthesized audio')
+	const rawAudio = await FFMpegTranscoder.decodeToChannels(Buffer.from(response.data))
+
+	logger.end()
+
+	return { rawAudio }
+}
+
+export async function getVoiceList(apiKey: string) {
+	const response = await request<any>({
+		method: 'GET',
+
+		url: 'https://api.elevenlabs.io/v1/voices',
+
+		headers: {
+			'accept': 'accept: application/json',
+			'xi-api-key': apiKey
+		},
+
+		responseType: 'json'
+	})
+
+	const elevenlabsVoices: any[] = response.data.voices
+
+	const voices: SynthesisVoice[] = elevenlabsVoices.map(elevenlabsVoice => {
+		const modelId: string = elevenlabsVoice?.high_quality_base_model_ids?.[0] ?? 'eleven_monolingual_v1'
+		const accent: string | undefined = elevenlabsVoice?.labels?.accent
+		const gender: VoiceGender = elevenlabsVoice?.labels?.gender ?? 'unknown'
+
+		const supportedLanguages: string[] = []
+
+		if (accent) {
+			if (accent.startsWith('american')) {
+				supportedLanguages.push('en-US')
+			} else if (accent.startsWith('british')) {
+				supportedLanguages.push('en-GB')
+			} else if (accent === 'irish') {
+				supportedLanguages.push('en-IE')
+			} else if (accent == 'australian') {
+				supportedLanguages.push('en-AU')
+			}
+		}
+
+		if (modelId.includes('multilingual')) {
+			supportedLanguages.push('en', ...supporteMultilingualLanguages)
+		} else {
+			supportedLanguages.push('en')
+		}
+
+		return {
+			name: elevenlabsVoice.name,
+			languages: supportedLanguages,
+			gender,
+
+			elevenLabsVoiceId: elevenlabsVoice.voice_id,
+			elevenLabsModelId: modelId
+		}
+	})
+
+	return voices
+}
+
+export interface ElevenlabsTTSOptions {
+	apiKey?: string
+	stability?: number
+	similarityBoost?: number
+	style?: number
+	useSpeakerBoost?: boolean
+}
+
+export const defaultElevenlabsTTSOptions = {
+	apiKey: undefined,
+	stability: 0.5,
+	similarityBoost: 0.5,
+	style: 0,
+	useSpeakerBoost: true
+}
+
+export const supporteMultilingualLanguages = ['zh', 'ko', 'nl', 'tr', 'sv', 'id', 'tl', 'ja', 'uk', 'el', 'cs', 'fi', 'ro', 'ru', 'da', 'bg', 'ms', 'sk', 'hr', 'ar', 'ta', 'pl', 'de', 'es', 'fr', 'it', 'hi', 'pt']