echogarden 2.3.5 → 2.5.0

package/docs/Options.md

@@ -101,7 +101,8 @@ Applies to CLI operations: `speak`, `speak-file`, `speak-url`, `speak-wikipedia`
 * `openAICloud.apiKey`: API key (required)
 * `openAICloud.organization`: organization identifier. Optional
 * `openAICloud.baseURL`: override the default base URL for the API. Optional
-* `openAICloud.model`: model to use. Can be either `tts-1` or `tts-1-hd`. Defaults to `tts-1`
+* `openAICloud.model`: model to use. Can be either `tts-1`, `tts-1-hd` or `gpt-4o-mini-tts`. Defaults to `tts-1`
+* `openAICloud.instructions`: instructions for model. Only works with `gpt-4o-mini-tts`. Optional
 * `openAICloud.timeout`: request timeout. Optional
 * `openAICloud.maxRetries`: maximum retries on failure. Defaults to 10
 
@@ -213,7 +214,7 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 
 **OpenAI Cloud**:
 * `openAICloud.apiKey`: API key (required)
-* `openAICloud.model`: model to use. When using the default provider (OpenAI), can only be `whisper-1`. For a custom provider, like Groq, see its documentation
+* `openAICloud.model`: model to use. When using the default provider (OpenAI), can be `whisper-1`, `gpt-4o-mini-transcribe` or `gpt-4o-transcribe`. For a custom provider, like Groq, see its documentation
 * `openAICloud.organization`: organization identifier. Optional
 * `openAICloud.baseURL`: override the default endpoint used by the API. For example, set `https://api.groq.com/openai/v1` to use Groq's OpenAI-compatible API instead of the default one. Optional
 * `openAICloud.temperature`: temperature. Choosing `0` uses a dynamic temperature approach. Defaults to `0`

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "2.3.5",
+	"version": "2.5.0",
 	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -52,8 +52,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "^3.758.0",
-		"@aws-sdk/client-transcribe-streaming": "^3.758.0",
+		"@aws-sdk/client-polly": "^3.777.0",
+		"@aws-sdk/client-transcribe-streaming": "^3.777.0",
 		"@echogarden/audio-io": "^0.3.0",
 		"@echogarden/espeak-ng-emscripten": "^0.3.3",
 		"@echogarden/fasttext-wasm": "^0.1.0",
@@ -67,7 +67,7 @@
 		"@echogarden/speex-resampler-wasm": "^0.3.0",
 		"@echogarden/svoxpico-wasm": "^0.2.0",
 		"@echogarden/transformers-nodejs-lite": "^2.17.1-lite.3",
-		"@mozilla/readability": "^0.5.0",
+		"@mozilla/readability": "^0.6.0",
 		"alawmulaw": "^6.0.0",
 		"chalk": "^5.4.1",
 		"cldr-segmentation": "^2.2.1",
@@ -82,10 +82,10 @@
 		"jsdom": "^26.0.0",
 		"json5": "^2.2.3",
 		"kuromoji": "^0.1.2",
-		"microsoft-cognitiveservices-speech-sdk": "^1.42.0",
+		"microsoft-cognitiveservices-speech-sdk": "^1.43.0",
 		"msgpack-lite": "^0.1.26",
-		"onnxruntime-node": "^1.20.1",
-		"openai": "^4.86.1",
+		"onnxruntime-node": "^1.21.0",
+		"openai": "^4.90.0",
 		"sam-js": "^0.3.1",
 		"strip-ansi": "^7.1.0",
 		"tar": "^7.4.3",
@@ -112,9 +112,9 @@
 		"@types/graceful-fs": "^4.1.9",
 		"@types/jsdom": "^21.1.7",
 		"@types/msgpack-lite": "^0.1.11",
-		"@types/node": "^22.13.9",
+		"@types/node": "^22.13.14",
 		"@types/tar": "^6.1.13",
-		"@types/ws": "^8.5.14",
+		"@types/ws": "^8.18.0",
 		"ts-json-schema-generator": "^2.3.0",
 		"typescript": "^5.8.2"
 	},

package/src/api/Synthesis.ts

@@ -701,8 +701,7 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 			logger.end()
 
-			const { audioData, timepoints } = await GoogleCloudTTS.synthesize(text, apiKey, language, voice, speed, pitchDeltaSemitones, 0, inputIsSSML)
-			const rawAudio = await FFMpegTranscoder.decodeToChannels(audioData)
+			const { rawAudio, timepoints } = await GoogleCloudTTS.synthesize(text, apiKey, language, voice, speed, pitchDeltaSemitones, 0, inputIsSSML)
 
 			synthesizedAudio = rawAudio
 

package/src/cli/CLI.ts

@@ -1698,21 +1698,27 @@ async function writeSourceSeparationOutputIfNeeded(outputFilename: string, isola
 		{
 			const isolatedOutputFilePath = prefixIsolated ? `${pathWithoutExtension}.isolated.${fileExtension}` : outputFilename
 
-			const fileSaver = getFileSaver(isolatedOutputFilePath, allowOverwrite)
-
-			await fileSaver(isolatedRawAudio, [], '')
+			await saveAudioWithBitrate(isolatedRawAudio, isolatedOutputFilePath, fileExtension, 64)
 		}
 
 		{
 			const backgroundOutputFilePath = `${pathWithoutExtension}.background.${fileExtension}`
 
-			const fileSaver = getFileSaver(backgroundOutputFilePath, allowOverwrite)
-
-			await fileSaver(backgroundRawAudio, [], '')
+			await saveAudioWithBitrate(backgroundRawAudio, backgroundOutputFilePath, fileExtension, 128)
 		}
 	}
 }
 
+async function saveAudioWithBitrate(rawAudio: RawAudio, filepath: string, fileExtension: string, bitrate: number) {
+	const ffmpegOptions = getDefaultFFMpegOptionsForSpeech(fileExtension, bitrate)
+	ffmpegOptions.filename = filepath
+
+	const fileDir = parsePath(filepath).dir || './'
+
+	await ensureDir(fileDir)
+	await encodeFromChannels(rawAudio, ffmpegOptions)
+}
+
 async function optionsLookupToTypedObject<K extends keyof APIOptions>(cliOptionsMap: Map<string, string>, optionsRoot: K, additionalOptionsSchema?: Map<string, SchemaTypeDefinition>): Promise<APIOptions[K]> {
 	const optionsSchema = await getOptionsSchema()
 	const resultingObj: any = {}

package/src/recognition/OpenAICloudSTT.ts

@@ -35,6 +35,14 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 
 	logger.start(options.baseURL ? `Send request to ${options.baseURL}` : 'Send request to OpenAI Cloud API')
 
+	let responseFormat: 'verbose_json' | 'json'
+
+	if (options.model === 'gpt-4o-mini-transcribe' || options.model === 'gpt-4o-transcribe') {
+		responseFormat = 'json'
+	} else {
+		responseFormat = 'verbose_json'
+	}
+
 	let response: VerboseResponse
 
 	if (task == 'transcribe') {
@@ -46,7 +54,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			model: options.model,
 			language: languageCode,
 			prompt: options.prompt,
-			response_format: 'verbose_json',
+			response_format: responseFormat,
 			temperature: options.temperature,
 			timestamp_granularities,
 		}) as any as VerboseResponse
@@ -55,7 +63,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			file: virtualFileStream,
 			model: options.model,
 			prompt: options.prompt,
-			response_format: 'verbose_json',
+			response_format: responseFormat,
 			temperature: options.temperature,
 		}) as any as VerboseResponse
 	} else {
@@ -64,7 +72,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 
 	const transcript = response.text.trim()
 
-	let timeline: Timeline
+	let timeline: Timeline | undefined
 
 	if (response.words) {
 		timeline = response.words.map<TimelineEntry>(entry => ({
@@ -73,7 +81,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			startTime: entry.start,
 			endTime: entry.end
 		}))
-	} else {
+	} else if (response.segments) {
 		const segmentTimeline = response.segments.map<TimelineEntry>(entry => ({
 			type: 'segment',
 			text: entry.text,
@@ -129,7 +137,7 @@ interface VerboseResponse {
 type Task = 'transcribe' | 'translate'
 
 export interface OpenAICloudSTTOptions {
-	model?: 'whisper-1' | string
+	model?: 'whisper-1' | 'gpt-4o-mini-transcribe' | 'gpt-4o-transcribe' | string
 
 	apiKey?: string
 	organization?: string

package/src/synthesis/GoogleCloudTTS.ts

@@ -2,6 +2,7 @@ import { request } from 'gaxios'
 import { Logger } from '../utilities/Logger.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { decodeBase64 } from '../encodings/Base64.js'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 
 const log = logToStderr
 
@@ -14,7 +15,7 @@ export async function synthesize(
 	pitchDeltaSemitones = 0.0,
 	volumeGainDecibels = 0.0,
 	ssml = false,
-	audioEncoding: AudioEncoding = 'MP3_64_KBPS',
+	audioEncoding: AudioEncoding = 'MP3',
 	sampleRate = 24000) {
 
 	const logger = new Logger()
@@ -70,9 +71,14 @@ export async function synthesize(
 
 	const result = parseResponseBody(response.data)
 
+	logger.start('Decode to raw audio')
+
+	const rawAudio = await FFMpegTranscoder.decodeToChannels(result.audioData)
+	const timepoints = result.timepoints
+
 	logger.end()
 
-	return result
+	return { rawAudio, timepoints }
 }
 
 function parseResponseBody(responseBody: any) {
@@ -116,7 +122,13 @@ export type GoogleCloudVoice = {
 	naturalSampleRateHertz: number
 }
 
-export type AudioEncoding = 'LINEAR16' | 'MP3' | 'MP3_64_KBPS' | 'OGG_OPUS' | 'MULAW' | 'ALAW'
+export type AudioEncoding =
+	'AUDIO_ENCODING_UNSPECIFIED' |
+	'LINEAR16' |
+	'MP3' |
+	'OGG_OPUS' |
+	'MULAW' |
+	'ALAW'
 
 export type timePoint = {
 	markName: string,

package/src/synthesis/OpenAICloudTTS.ts

@@ -22,6 +22,7 @@ export async function synthesize(text: string, voice: string, speed: number, opt
 		input: text,
 		model: options.model!,
 		voice: voice as any,
+		instructions: options.instructions,
 		response_format: 'opus',
 		speed,
 	}, {
@@ -43,7 +44,9 @@ export interface OpenAICloudTTSOptions {
 	organization?: string
 	baseURL?: string
 
-	model?: 'tts-1' | 'tts-1-hd'
+	model?: 'tts-1' | 'tts-1-hd' | 'gpt-4o-mini-tts'
+
+	instructions?: string
 
 	timeout?: number
 	maxRetries?: number
@@ -56,110 +59,72 @@ export const defaultOpenAICloudTTSOptions: OpenAICloudTTSOptions = {
 
 	model: 'tts-1',
 
+	instructions: undefined,
+
 	timeout: undefined,
 	maxRetries: 10,
 }
 
+// Reference: https://platform.openai.com/docs/guides/text-to-speech#supported-languages
 export const supportedLanguages = [
 	'en',
-	'zh',
-	'de',
-	'es',
-	'ru',
-	'ko',
-	'fr',
-	'ja',
-	'pt',
-	'tr',
-	'pl',
+	'af',
+	'ar',
+	'hy',
+	'az',
+	'be',
+	'bs',
+	'bg',
 	'ca',
+	'zh',
+	'hr',
+	'cs',
+	'da',
 	'nl',
-	'ar',
-	'sv',
-	'it',
-	'id',
-	'hi',
+	'en',
+	'et',
 	'fi',
-	'vi',
-	'iw',
-	'uk',
+	'fr',
+	'gl',
+	'de',
 	'el',
-	'ms',
-	'cs',
-	'ro',
-	'da',
+	'he',
+	'hi',
 	'hu',
-	'ta',
-	'no',
-	'th',
-	'ur',
-	'hr',
-	'bg',
+	'is',
+	'id',
+	'it',
+	'ja',
+	'kn',
+	'kk',
+	'ko',
+	'lv',
 	'lt',
-	'la',
+	'mk',
+	'ms',
+	'mr',
 	'mi',
-	'ml',
-	'cy',
-	'sk',
-	'te',
+	'ne',
+	'no',
 	'fa',
-	'lv',
-	'bn',
+	'pl',
+	'pt',
+	'ro',
+	'ru',
 	'sr',
-	'az',
+	'sk',
 	'sl',
-	'kn',
-	'et',
-	'mk',
-	'br',
-	'eu',
-	'is',
-	'hy',
-	'ne',
-	'mn',
-	'bs',
-	'kk',
-	'sq',
+	'es',
 	'sw',
-	'gl',
-	'mr',
-	'pa',
-	'si',
-	'km',
-	'sn',
-	'yo',
-	'so',
-	'af',
-	'oc',
-	'ka',
-	'be',
-	'tg',
-	'sd',
-	'gu',
-	'am',
-	'yi',
-	'lo',
-	'uz',
-	'fo',
-	'ht',
-	'ps',
-	'tk',
-	'nn',
-	'mt',
-	'sa',
-	'lb',
-	'my',
-	'bo',
+	'sv',
 	'tl',
-	'mg',
-	'as',
-	'tt',
-	'haw',
-	'ln',
-	'ha',
-	'ba',
-	'jw',
-	'su',
+	'ta',
+	'th',
+	'tr',
+	'uk',
+	'ur',
+	'vi',
+	'cy',
 ]
 
 export const voiceList: SynthesisVoice[] = [
@@ -173,6 +138,11 @@ export const voiceList: SynthesisVoice[] = [
 		languages: ['en-US', ...supportedLanguages],
 		gender: 'male',
 	},
+	{
+		name: 'ballad',
+		languages: ['en-GB', ...supportedLanguages],
+		gender: 'male',
+	},
 	{
 		name: 'coral',
 		languages: ['en-US', ...supportedLanguages],
@@ -208,4 +178,9 @@ export const voiceList: SynthesisVoice[] = [
 		languages: ['en-US', ...supportedLanguages],
 		gender: 'female',
 	},
+	{
+		name: 'verse',
+		languages: ['en-US', ...supportedLanguages],
+		gender: 'male',
+	},
 ]

package/src/typings/Fillers.d.ts

@@ -55,6 +55,9 @@ declare module '@echogarden/speex-resampler-wasm'
 declare module '@echogarden/speex-resampler-wasm/simd'
 declare module '@echogarden/vosk'
 
+/*
+// Not needed anymore in `onnxruntime-node` 1.21.0:
+
 declare module 'onnxruntime-node' {
 	// From index.d.ts
 	export * from 'onnxruntime-common'
@@ -109,3 +112,4 @@ declare module 'onnxruntime-node' {
 	// From version.d.ts
 	export const version = '1.20.0'
 }
+*/