echogarden 1.0.4 → 1.1.0

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "echogarden",
-	"version": "1.0.4",
-	"description": "An integrated speech toolset, providing a variety of synthesis, recognition, alignment, and other processing tools, designed to be directly accessible to end-users.",
+	"version": "1.1.0",
+	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
 	"keywords": [
@@ -14,7 +14,8 @@
 		"forced alignment",
 		"speech translation",
 		"language identification",
-		"language detection"
+		"language detection",
+		"source separation"
 	],
 	"repository": {
 		"type": "git",
@@ -73,7 +74,6 @@
 		"cldr-segmentation": "^2.2.0",
 		"command-exists": "^1.2.9",
 		"compromise": "^14.13.0",
-		"compromise-dates": "^3.5.0",
 		"fs-extra": "^11.2.0",
 		"gaxios": "^6.5.0",
 		"graceful-fs": "^4.2.11",
@@ -87,10 +87,8 @@
 		"microsoft-cognitiveservices-speech-sdk": "^1.36.0",
 		"moving-median": "^1.0.0",
 		"msgpack-lite": "^0.1.26",
-		"ndarray": "^1.0.19",
-		"ndarray-ops": "^1.2.2",
-		"onnxruntime-node": "^1.17.0",
-		"openai": "^4.37.0",
+		"onnxruntime-node": "^1.17.3",
+		"openai": "^4.38.2",
 		"sam-js": "^0.2.1",
 		"strip-ansi": "^7.1.0",
 		"tar": "^7.0.1",
@@ -121,13 +119,11 @@
 		"@types/graceful-fs": "^4.1.9",
 		"@types/jsdom": "^21.1.6",
 		"@types/msgpack-lite": "^0.1.11",
-		"@types/ndarray": "^1.0.14",
-		"@types/ndarray-ops": "^1.2.7",
 		"@types/node": "^20.12.7",
 		"@types/recursive-readdir": "^2.2.4",
-		"@types/tar": "^6.1.12",
+		"@types/tar": "^6.1.13",
 		"@types/ws": "^8.5.10",
-		"ts-json-schema-generator": "^2.0.1",
+		"ts-json-schema-generator": "^2.1.0",
 		"typescript": "^5.4.5"
 	}
 }

package/src/alignment/SpeechAlignment.ts

@@ -452,7 +452,7 @@ export async function alignPhoneTimelines(
 	return alignedWordTimeline
 }
 
-export async function createAlignmentReferenceUsingEspeakForFragments(fragments: string[], espeakOptions: EspeakOptions, insertSeparators = true) {
+export async function createAlignmentReferenceUsingEspeakForFragments(fragments: string[], espeakOptions: EspeakOptions) {
 	const progressLogger = new Logger()
 
 	progressLogger.start("Load espeak module")

package/src/api/API.ts

@@ -6,6 +6,7 @@ export * from './Synthesis.js'
 export * from './Recognition.js'
 export * from './Alignment.js'
 export * from './Translation.js'
+export * from './TranslationAlignment.js'
 export * from './LanguageDetection.js'
 export * from './VoiceActivityDetection.js'
 export * from './Denoising.js'

package/src/api/APIOptions.ts

@@ -6,6 +6,7 @@ export interface APIOptions {
 	SynthesisOptions: API.SynthesisOptions
 	RecognitionOptions: API.RecognitionOptions
 	AlignmentOptions: API.AlignmentOptions
+	TranslationAlignmentOptions: API.TranslationAlignmentOptions
 	SpeechTranslationOptions: API.SpeechTranslationOptions
 	SpeechLanguageDetectionOptions: API.SpeechLanguageDetectionOptions
 	TextLanguageDetectionOptions: API.TextLanguageDetectionOptions

package/src/api/Alignment.ts

@@ -7,11 +7,11 @@ import { Logger } from '../utilities/Logger.js'
 import * as API from './API.js'
 import { Timeline, addTimeOffsetToTimeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
 import { formatLanguageCodeWithName, getDefaultDialectForLanguageCodeIfPossible, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
-import { type WhisperOptions } from '../recognition/WhisperSTT.js'
+import { type WhisperAlignmentOptions } from '../recognition/WhisperSTT.js'
 import chalk from 'chalk'
 import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
-import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
-import { EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
+import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
+import { type EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
 import { isWord } from '../nlp/Segmentation.js'
 
 const log = logToStderr
@@ -194,19 +194,15 @@ export async function align(input: AudioSourceParam, transcript: string, options
 		case 'whisper': {
 			const WhisperSTT = await import('../recognition/WhisperSTT.js')
 
-			const whisperOptions = options.whisper!
+			const whisperAlignmnentOptions = options.whisper!
 
 			const shortLanguageCode = getShortLanguageCode(language)
 
-			const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, language)
-
-			if (getRawAudioDuration(sourceRawAudio) > 30) {
-				throw new Error('Whisper based alignment currently only supports audio inputs that are 30s or less')
-			}
+			const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperAlignmnentOptions.model, shortLanguageCode)
 
 			logger.end()
 
-			mappedTimeline = await WhisperSTT.align(sourceRawAudio, transcript, modelName, modelDir, shortLanguageCode)
+			mappedTimeline = await WhisperSTT.align(sourceRawAudio, transcript, modelName, modelDir, shortLanguageCode, whisperAlignmnentOptions)
 
 			break
 		}
@@ -315,7 +311,7 @@ export interface AlignmentOptions {
 
 	sourceSeparation?: API.SourceSeparationOptions
 
-	whisper?: WhisperOptions
+	whisper?: WhisperAlignmentOptions
 }
 
 export const defaultAlignmentOptions: AlignmentOptions = {
@@ -337,7 +333,8 @@ export const defaultAlignmentOptions: AlignmentOptions = {
 		whitespace: 'collapse'
 	},
 
-	subtitles: defaultSubtitlesBaseConfig,
+	subtitles: {
+	},
 
 	dtw: {
 		granularity: 'auto',
@@ -354,7 +351,6 @@ export const defaultAlignmentOptions: AlignmentOptions = {
 			autoPromptParts: false,
 			suppressRepetition: true,
 			decodeTimestampTokens: true,
-			seed: undefined,
 		}
 	},
 
@@ -385,7 +381,7 @@ export const alignmentEngines: API.EngineMetadata[] = [
 	{
 		id: 'whisper',
 		name: 'OpenAI Whisper',
-		description: 'Extracts timestamps from the internal state of the Whisper recognition model (note: currently limited to a maximum audio duration of 30 seconds).',
+		description: 'Extracts timestamps by guiding the Whisper recognition model to recognize the transcript tokens.',
 		type: 'local'
 	}
 ]

package/src/api/LanguageDetection.ts

@@ -6,11 +6,13 @@ import { Logger } from '../utilities/Logger.js'
 import * as API from './API.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import path from 'path'
-import { type WhisperModelName } from '../recognition/WhisperSTT.js'
+import { type WhisperLanguageDetectionOptions } from '../recognition/WhisperSTT.js'
 import { formatLanguageCodeWithName, languageCodeToName } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import chalk from 'chalk'
-import { WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
+import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
+import { type SileroLanguageDetectionOptions } from '../speech-language-detection/SileroLanguageDetection.js'
+import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 
 const log = logToStderr
 
@@ -59,12 +61,14 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 			const modelPath = path.join(modelDir, 'lang_classifier_95.onnx')
 			const languageDictionaryPath = path.join(modelDir, 'lang_dict_95.json')
 			const languageGroupDictionaryPath = path.join(modelDir, 'lang_group_dict_95.json')
+			const onnxExecutionProviders: OnnxExecutionProvider[] = sileroOptions.provider ? [sileroOptions.provider] : []
 
 			const languageResults = await SileroLanguageDetection.detectLanguage(
 				sourceRawAudio,
 				modelPath,
 				languageDictionaryPath,
-				languageGroupDictionaryPath)
+				languageGroupDictionaryPath,
+				onnxExecutionProviders)
 
 			detectedLanguageProbabilities = languageResults
 
@@ -80,7 +84,11 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 
 			logger.end()
 
-			detectedLanguageProbabilities = await WhisperSTT.detectLanguage(sourceRawAudio, modelName, modelDir, whisperOptions.temperature!)
+			detectedLanguageProbabilities = await WhisperSTT.detectLanguage(
+				sourceRawAudio,
+				modelName,
+				modelDir,
+				whisperOptions)
 
 			break
 		}
@@ -198,13 +206,9 @@ export interface SpeechLanguageDetectionOptions {
 
 	crop?: boolean
 
-	silero?: {
-	}
+	silero?: SileroLanguageDetectionOptions
 
-	whisper?: {
-		model?: WhisperModelName
-		temperature?: number
-	}
+	whisper?: WhisperLanguageDetectionOptions
 
 	whisperCpp?: WhisperCppOptions
 

package/src/api/Recognition.ts

@@ -6,13 +6,16 @@ import { Logger } from '../utilities/Logger.js'
 
 import * as API from './API.js'
 import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
-import { type WhisperOptions } from '../recognition/WhisperSTT.js'
 import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import chalk from 'chalk'
-import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
-import { OpenAICloudSTTOptions, defaultOpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
+
+import { type WhisperOptions } from '../recognition/WhisperSTT.js'
+import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
+import { type OpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
 import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
+import { type SileroRecognitionOptions } from '../recognition/SileroSTT.js'
+import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 
 const log = logToStderr
 
@@ -172,9 +175,14 @@ export async function recognize(input: AudioSourceParam, options: RecognitionOpt
 				modelPath = await loadPackage(packageName)
 			}
 
+			const onnxExecutionProviders: OnnxExecutionProvider[] = sileroOptions.provider ? [sileroOptions.provider] : []
+
 			logger.end();
 
-			({ transcript, timeline } = await SileroSTT.recognize(sourceRawAudio, modelPath))
+			({ transcript, timeline } = await SileroSTT.recognize(
+				sourceRawAudio,
+				modelPath,
+				onnxExecutionProviders))
 
 			break
 		}
@@ -346,9 +354,7 @@ export interface RecognitionOptions {
 		modelPath?: string
 	}
 
-	silero?: {
-		modelPath?: string
-	}
+	silero?: SileroRecognitionOptions
 
 	googleCloud?: {
 		apiKey?: string
@@ -389,7 +395,8 @@ export const defaultRecognitionOptions: RecognitionOptions = {
 	languageDetection: {
 	},
 
-	subtitles: defaultSubtitlesBaseConfig,
+	subtitles: {
+	},
 
 	vad: {
 		engine: 'adaptive-gate'
@@ -406,7 +413,6 @@ export const defaultRecognitionOptions: RecognitionOptions = {
 	},
 
 	silero: {
-		modelPath: undefined
 	},
 
 	googleCloud: {
@@ -428,7 +434,8 @@ export const defaultRecognitionOptions: RecognitionOptions = {
 		secretAccessKey: undefined,
 	},
 
-	openAICloud: defaultOpenAICloudSTTOptions
+	openAICloud: {
+	},
 }
 
 export const recognitionEngines: API.EngineMetadata[] = [

package/src/api/SourceSeparation.ts

@@ -6,6 +6,7 @@ import { EngineMetadata } from './Common.js';
 import chalk from 'chalk';
 import { readdir } from '../utilities/FileSystem.js';
 import path from 'node:path';
+import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js';
 
 export async function isolate(input: AudioSourceParam, options: SourceSeparationOptions): Promise<SourceSeparationResult> {
 	const logger = new Logger()
@@ -26,6 +27,8 @@ export async function isolate(input: AudioSourceParam, options: SourceSeparation
 
 			const mdxNetOptions = options.mdxNet!
 
+			const executionProviders: OnnxExecutionProvider[] = mdxNetOptions.executionProvider ? [mdxNetOptions.executionProvider] : []
+
 			const packageDir = await loadPackage(`mdxnet-${mdxNetOptions.model!}`)
 			const modelFilename = (await readdir(packageDir)).filter(name => name.endsWith('onnx'))[0]
 
@@ -39,7 +42,7 @@ export async function isolate(input: AudioSourceParam, options: SourceSeparation
 
 			const audioStereo44100 = await ensureRawAudio(inputRawAudio, 44100, 2)
 
-			isolatedRawAudio = await MDXNetSourceSeparation.isolate(audioStereo44100, modelPath)
+			isolatedRawAudio = await MDXNetSourceSeparation.isolate(audioStereo44100, modelPath, executionProviders)
 
 			logger.end()
 
@@ -72,6 +75,7 @@ export interface SourceSeparationOptions {
 
 	mdxNet?: {
 		model?: string
+		executionProvider?: OnnxExecutionProvider
 	}
 }
 
@@ -79,7 +83,8 @@ export const defaultSourceSeparationOptions: SourceSeparationOptions = {
 	engine: 'mdx-net',
 
 	mdxNet: {
-		model: 'UVR_MDXNET_1_9703'
+		model: 'UVR_MDXNET_1_9703',
+		executionProvider: undefined,
 	}
 }
 

package/src/api/Synthesis.ts

@@ -4,7 +4,7 @@ import { deepClone, extendDeep } from '../utilities/ObjectUtilities.js'
 
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 
-import { clip, convertHtmlToText, sha256AsHex, simplifyPunctuationCharacters, stringifyAndFormatJson, logToStderr, yieldToEventLoop, delay, runOperationWithRetries } from '../utilities/Utilities.js'
+import { clip, convertHtmlToText, sha256AsHex, simplifyPunctuationCharacters, stringifyAndFormatJson, logToStderr, yieldToEventLoop, runOperationWithRetries } from '../utilities/Utilities.js'
 import { RawAudio, attenuateIfClipping, concatAudioSegments, downmixToMono, encodeRawAudioToWave, getSamplePeakDecibels, getEmptyRawAudio, getRawAudioDuration, normalizeAudioLevel, trimAudioEnd, trimAudioStart } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 
@@ -20,10 +20,11 @@ import { loadPackage } from '../utilities/PackageManager.js'
 import { EngineMetadata, appName } from './Common.js'
 import { shouldCancelCurrentTask } from '../server/Worker.js'
 import chalk from 'chalk'
-import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
+import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 import { type EspeakOptions } from '../synthesis/EspeakTTS.js'
-import { type OpenAICloudTTSOptions, defaultOpenAICloudTTSOptions } from '../synthesis/OpenAICloudTTS.js'
-import { type ElevenlabsTTSOptions, defaultElevenlabsTTSOptions } from '../synthesis/ElevenlabsTTS.js'
+import { type OpenAICloudTTSOptions  } from '../synthesis/OpenAICloudTTS.js'
+import { type ElevenlabsTTSOptions } from '../synthesis/ElevenlabsTTS.js'
+import { OnnxExecutionProvider } from '../utilities/OnnxUtilities.js'
 
 const log = logToStderr
 
@@ -358,9 +359,9 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 			const lengthScale = 1 / speed
 
-			const engineOptions = options.vits!
+			const vitsOptions = options.vits!
 
-			const speakerId = engineOptions.speakerId
+			const speakerId = vitsOptions.speakerId
 
 			if (speakerId != undefined) {
 				if (selectedVoice.speakerCount == undefined) {
@@ -368,7 +369,7 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 						throw new Error('Selected VITS model has only one speaker. Speaker ID must be 0 if specified.')
 					}
 				} else if (speakerId < 0 || speakerId >= selectedVoice.speakerCount) {
-					throw new Error(`Selected VITS model has ${selectedVoice.speakerCount} voices. Speaker ID should be in the range ${0} to ${selectedVoice.speakerCount - 1}`)
+					throw new Error(`Selected VITS model has ${selectedVoice.speakerCount} speaker IDs. Speaker ID should be in the range ${0} to ${selectedVoice.speakerCount - 1}`)
 				}
 			}
 
@@ -376,9 +377,18 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 			const modelPath = voicePackagePath!
 
+			const onnxExecutionProviders: OnnxExecutionProvider[] = vitsOptions.provider ? [vitsOptions.provider] : []
+
 			logger.end()
 
-			const { rawAudio, timeline: outTimeline } = await vitsTTS.synthesizeSentence(text, voice, modelPath, lengthScale, speakerId, lexicons)
+			const { rawAudio, timeline: outTimeline } = await vitsTTS.synthesizeSentence(
+				text,
+				voice,
+				modelPath,
+				lengthScale,
+				speakerId ?? 0,
+				lexicons,
+				onnxExecutionProviders)
 
 			synthesizedAudio = rawAudio
 			timeline = outTimeline
@@ -1024,6 +1034,7 @@ export interface SynthesisOptions {
 
 	vits?: {
 		speakerId?: number
+		provider?: OnnxExecutionProvider
 	}
 
 	pico?: {
@@ -1153,10 +1164,12 @@ export const defaultSynthesisOptions: SynthesisOptions = {
 
 	languageDetection: undefined,
 
-	subtitles: defaultSubtitlesBaseConfig,
+	subtitles: {
+	},
 
 	vits: {
 		speakerId: undefined,
+		provider: undefined,
 	},
 
 	pico: {
@@ -1216,9 +1229,11 @@ export const defaultSynthesisOptions: SynthesisOptions = {
 		lexiconNames: undefined,
 	},
 
-	openAICloud: defaultOpenAICloudTTSOptions,
+	openAICloud: {
+	},
 
-	elevenlabs: defaultElevenlabsTTSOptions,
+	elevenlabs: {
+	},
 
 	googleTranslate: {
 		tld: 'us'

package/src/api/Translation.ts

@@ -5,16 +5,16 @@ import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevel, trimAu
 import { Logger } from '../utilities/Logger.js'
 
 import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
-import { defaultWhisperOptions, type WhisperOptions } from '../recognition/WhisperSTT.js'
+import { type WhisperOptions } from '../recognition/WhisperSTT.js'
 import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
 import { EngineMetadata } from './Common.js'
 import { type SpeechLanguageDetectionOptions, detectSpeechLanguage } from './API.js'
 import chalk from 'chalk'
-import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
+import { type SubtitlesConfig } from '../subtitles/Subtitles.js'
 
 import * as API from './API.js'
-import { type OpenAICloudSTTOptions, defaultOpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
-import { type WhisperCppOptions, defaultWhisperCppOptions } from '../recognition/WhisperCppSTT.js'
+import { type OpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
+import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 
 const log = logToStderr
 
@@ -263,15 +263,21 @@ export const defaultSpeechTranslationOptions: SpeechTranslationOptions = {
 
 	languageDetection: undefined,
 
-	subtitles: defaultSubtitlesBaseConfig,
+	subtitles: {
+	},
 
 	vad: {
 		engine: 'adaptive-gate'
 	},
 
-	whisper: defaultWhisperOptions,
-	whisperCpp: defaultWhisperCppOptions,
-	openAICloud: defaultOpenAICloudSTTOptions,
+	whisper: {
+	},
+
+	whisperCpp: {
+	},
+
+	openAICloud: {
+	},
 }
 
 export const speechTranslationEngines: EngineMetadata[] = [