echogarden 0.12.2 → 1.0.0

package/src/synthesis/OpenAICloudTTS.ts

@@ -0,0 +1,196 @@
+import { ensureRawAudio } from '../audio/AudioUtilities.js';
+import { SynthesisVoice } from '../api/Synthesis.js';
+import { Logger } from '../utilities/Logger.js';
+import { extendDeep } from '../utilities/ObjectUtilities.js';
+
+export async function synthesize(text: string, voice: string, speed: number, options: OpenAICloudTTSOptions) {
+	const logger = new Logger()
+
+	logger.start('Request synthesis from OpenAI Cloud API')
+
+	options = extendDeep(defaultOpenAICloudTTSOptions, options)
+
+	if (!options.apiKey) {
+		throw new Error(`No API key given`)
+	}
+
+	const { default: OpenAI } = await import('openai')
+
+	const openai = new OpenAI(options)
+
+	const result = await openai.audio.speech.create({
+		input: text,
+		model: options.model!,
+		voice: voice as any,
+		response_format: 'opus',
+		speed,
+	}, {
+		maxRetries: 10
+	})
+
+	const resultBuffer = await result.buffer()
+
+	logger.start('Decode returned audio')
+	const resultRawAudio = ensureRawAudio(resultBuffer)
+
+	logger.end()
+
+	return resultRawAudio
+}
+
+export interface OpenAICloudTTSOptions {
+	apiKey?: string
+	organization?: string
+	baseURL?: string
+
+	model?: 'tts-1' | 'tts-1-hd'
+
+	timeout?: number
+	maxRetries?: number
+}
+
+export const defaultOpenAICloudTTSOptions: OpenAICloudTTSOptions = {
+	apiKey: undefined,
+	organization: undefined,
+	baseURL: undefined,
+
+	model: 'tts-1',
+
+	timeout: undefined,
+	maxRetries: 10,
+}
+
+export const supportedLanguages = [
+	'en',
+	'zh',
+	'de',
+	'es',
+	'ru',
+	'ko',
+	'fr',
+	'ja',
+	'pt',
+	'tr',
+	'pl',
+	'ca',
+	'nl',
+	'ar',
+	'sv',
+	'it',
+	'id',
+	'hi',
+	'fi',
+	'vi',
+	'iw',
+	'uk',
+	'el',
+	'ms',
+	'cs',
+	'ro',
+	'da',
+	'hu',
+	'ta',
+	'no',
+	'th',
+	'ur',
+	'hr',
+	'bg',
+	'lt',
+	'la',
+	'mi',
+	'ml',
+	'cy',
+	'sk',
+	'te',
+	'fa',
+	'lv',
+	'bn',
+	'sr',
+	'az',
+	'sl',
+	'kn',
+	'et',
+	'mk',
+	'br',
+	'eu',
+	'is',
+	'hy',
+	'ne',
+	'mn',
+	'bs',
+	'kk',
+	'sq',
+	'sw',
+	'gl',
+	'mr',
+	'pa',
+	'si',
+	'km',
+	'sn',
+	'yo',
+	'so',
+	'af',
+	'oc',
+	'ka',
+	'be',
+	'tg',
+	'sd',
+	'gu',
+	'am',
+	'yi',
+	'lo',
+	'uz',
+	'fo',
+	'ht',
+	'ps',
+	'tk',
+	'nn',
+	'mt',
+	'sa',
+	'lb',
+	'my',
+	'bo',
+	'tl',
+	'mg',
+	'as',
+	'tt',
+	'haw',
+	'ln',
+	'ha',
+	'ba',
+	'jw',
+	'su',
+]
+
+export const voiceList: SynthesisVoice[] = [
+	{
+		name: 'alloy',
+		languages: ['en-US', ...supportedLanguages],
+		gender: 'male',
+	},
+	{
+		name: 'echo',
+		languages: ['en-US', ...supportedLanguages],
+		gender: 'male',
+	},
+	{
+		name: 'fable',
+		languages: ['en-GB', ...supportedLanguages],
+		gender: 'male',
+	},
+	{
+		name: 'onyx',
+		languages: ['en-US', ...supportedLanguages],
+		gender: 'male',
+	},
+	{
+		name: 'nova',
+		languages: ['en-US', ...supportedLanguages],
+		gender: 'female',
+	},
+	{
+		name: 'shimmer',
+		languages: ['en-US', ...supportedLanguages],
+		gender: 'female',
+	},
+]

package/src/synthesis/SamTTS.ts

@@ -9,17 +9,17 @@ import { Logger } from '../utilities/Logger.js'
 
 export async function synthesize(text: string, pitch = 64, speed = 72, mouth = 128, throat = 128) {
 	const logger = new Logger()
-	logger.start("Initialize sam module")
+	logger.start('Initialize sam module')
 
 	const { default: SamJs } = await import('sam-js')
 
 	const sam = new SamJs({ pitch, speed, mouth, throat })
 
-	logger.start("Synthesize with sam")
+	logger.start('Synthesize with sam')
 	const samples: Float32Array = sam.buf32(text) as Float32Array
 
 	if (!samples) {
-		throw new Error("Sam TTS failed")
+		throw new Error('Sam TTS failed')
 	}
 
 	const rawAudio: RawAudio = { audioChannels: [samples], sampleRate: 22050 }

package/src/synthesis/SapiTTS.ts

@@ -1,28 +1,28 @@
-import { SynthesisVoice } from "../api/API.js"
-import { decodeToChannels } from "../audio/AudioBufferConversion.js"
-import { RawAudio } from "../audio/AudioUtilities.js"
-import { SampleFormat } from "../codecs/WaveCodec.js"
-import { getShortLanguageCode, lcidToIsoLanguageCode } from "../utilities/Locale.js"
-import { Logger } from "../utilities/Logger.js"
-import { Timeline, TimelineEntry } from "../utilities/Timeline.js"
-import { logToStderr } from "../utilities/Utilities.js"
+import { SynthesisVoice } from '../api/API.js'
+import { decodeToChannels } from '../audio/AudioBufferConversion.js'
+import { RawAudio } from '../audio/AudioUtilities.js'
+import { SampleFormat } from '../codecs/WaveCodec.js'
+import { getShortLanguageCode, lcidToIsoLanguageCode } from '../utilities/Locale.js'
+import { Logger } from '../utilities/Logger.js'
+import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
+import { logToStderr } from '../utilities/Utilities.js'
 
 const log = logToStderr
 
 export function synthesize(text: string, voiceName: string, rate = 0, useSpeechPlatform = false) {
 	return new Promise<{ rawAudio: RawAudio, timeline: Timeline }>(async (resolve, reject) => {
 		const logger = new Logger()
-		logger.start("Initialize winax module")
+		logger.start('Initialize winax module')
 
-		const { default: WinAX } = await import("winax")
+		const { default: WinAX } = await import('winax')
 
 		const ActiveXObject = (global as any).ActiveXObject
 
-		logger.start("Create SAPI COM object")
-		const sapiVoice = new ActiveXObject(useSpeechPlatform ? "Speech.SPVoice" : "SAPI.SPVoice")
+		logger.start('Create SAPI COM object')
+		const sapiVoice = new ActiveXObject(useSpeechPlatform ? 'Speech.SPVoice' : 'SAPI.SPVoice')
 		sapiVoice.EventInterests = 33790
 
-		logger.start("Get SAPI voice list and select best match")
+		logger.start('Get SAPI voice list and select best match')
 
 		if (voiceName) {
 			const voiceObjects = sapiVoice.GetVoices()
@@ -40,13 +40,13 @@ export function synthesize(text: string, voiceName: string, rate = 0, useSpeechP
 		sapiVoice.Rate = rate
 
 		// Create phone converter for language
-		const sapiPhoneConverter = new ActiveXObject(useSpeechPlatform ? "Speech.SpPhoneConverter" : "SAPI.SpPhoneConverter")
+		const sapiPhoneConverter = new ActiveXObject(useSpeechPlatform ? 'Speech.SpPhoneConverter' : 'SAPI.SpPhoneConverter')
 		const sapiLanguageCodeHex = sapiVoice.Voice.GetAttribute('Language')
 		const sapiLanguageCode = parseInt(sapiLanguageCodeHex, 16)
 
 		sapiPhoneConverter.LanguageId = sapiLanguageCode
 
-		logger.start("Synthesize with SAPI")
+		logger.start('Synthesize with SAPI')
 
 		const sampleRate = 22050
 		const bytesPerSecond = sampleRate * 2
@@ -80,7 +80,7 @@ export function synthesize(text: string, voiceName: string, rate = 0, useSpeechP
 				const wordText = text.substring(charPos, charPos + length)
 				const startTime = streamPos / bytesPerSecond
 
-				const wordEvent = { type: "word", text: wordText, startTime, endTime: -1, timeline: [] } as TimelineEntry
+				const wordEvent = { type: 'word', text: wordText, startTime, endTime: -1, timeline: [] } as TimelineEntry
 				events.push(wordEvent)
 
 				lastWordEvent = wordEvent
@@ -94,14 +94,14 @@ export function synthesize(text: string, voiceName: string, rate = 0, useSpeechP
 
 				const phoneText = sapiPhoneConverter.IdToPhone(currentPhoneId)
 
-				if (phoneText == "," || phoneText == "_") {
+				if (phoneText == ',' || phoneText == '_') {
 					return
 				}
 
 				const startTime = streamPos / bytesPerSecond
 				const endTime = startTime + (duration / 1000)
 
-				events.push({ type: "phone", text: phoneText, startTime, endTime })
+				events.push({ type: 'phone', text: phoneText, startTime, endTime })
 			},
 
 			EndStream: (streamId: number, streamPos: number) => {
@@ -124,11 +124,11 @@ export function synthesize(text: string, voiceName: string, rate = 0, useSpeechP
 }
 
 export async function getVoiceList(useSpeechPlatform = false) {
-	const { default: WinAX } = await import("winax")
+	const { default: WinAX } = await import('winax')
 
 	const ActiveXObject = (global as any).ActiveXObject
 
-	const sapiVoice = new ActiveXObject(useSpeechPlatform ? "Speech.SPVoice" : "SAPI.SPVoice")
+	const sapiVoice = new ActiveXObject(useSpeechPlatform ? 'Speech.SPVoice' : 'SAPI.SPVoice')
 
 	const voiceObjects = sapiVoice.GetVoices()
 
@@ -137,7 +137,7 @@ export async function getVoiceList(useSpeechPlatform = false) {
 	for (let i = 0; i < voiceObjects.Count; i++) {
 		const voiceObject = voiceObjects.Item(i)
 		const voiceName = voiceObject.GetDescription()
-		const voiceGender = voiceObject.GetAttribute("Gender")?.toLowerCase()
+		const voiceGender = voiceObject.GetAttribute('Gender')?.toLowerCase()
 
 		const sapiLanguageCodeHex = voiceObject.GetAttribute('Language')
 		const sapiLanguageCode = parseInt(sapiLanguageCodeHex, 16)
@@ -164,7 +164,7 @@ export async function getVoiceList(useSpeechPlatform = false) {
 		voices.push({
 			name: voiceName,
 			languages: resultLanguageCodes,
-			gender: voiceGender || "unknown"
+			gender: voiceGender || 'unknown'
 		})
 	}
 
@@ -174,12 +174,12 @@ export async function getVoiceList(useSpeechPlatform = false) {
 }
 
 export async function AssertSAPIAvailable(testForSpeechPlatform = false) {
-	if (process.platform != "win32") {
+	if (process.platform != 'win32') {
 		throw new Error(`SAPI is not available on your platform. SAPI is a Microsoft Windows technology that is only runs on a Windows OS.`)
 	}
 
 	try {
-		const { default: WinAX } = await import("winax")
+		const { default: WinAX } = await import('winax')
 	} catch (e) {
 		throw new Error(`winax package, which is required for SAPI support, was not found. You can install it by running 'npm install winax -g'.`)
 	}
@@ -187,13 +187,13 @@ export async function AssertSAPIAvailable(testForSpeechPlatform = false) {
 	const ActiveXObject = (global as any).ActiveXObject
 
 	try {
-		const voice = new ActiveXObject("SAPI.SPVoice")
+		const voice = new ActiveXObject('SAPI.SPVoice')
 	} catch (e) {
 		throw new Error(`Failed creating a SAPI instance: ${e}`)
 	}
 
 	try {
-		const voice = new ActiveXObject("Speech.SPVoice")
+		const voice = new ActiveXObject('Speech.SPVoice')
 	} catch(e) {
 		throw new Error(`Failed creating an msspeech instance. Please ensure you installed the Microsoft Speech Platform runtime correctly.`)
 	}
@@ -203,11 +203,11 @@ function eventsToTimeline(events: Timeline, totalDuration: number): Timeline {
 	const timeline: Timeline = []
 
 	for (const event of events) {
-		if (event.type == "word") {
+		if (event.type == 'word') {
 			timeline.push(event)
-		} else if (event.type == "phone") {
+		} else if (event.type == 'phone') {
 			if (timeline.length == 0) {
-				throw new Error("Unexpected: phone event preceded a word event")
+				throw new Error('Unexpected: phone event preceded a word event')
 			}
 
 			const lastWordEntry = timeline[timeline.length - 1]

package/src/synthesis/StreamlabsPollyTTS.ts

@@ -1,11 +1,11 @@
-import { request } from "gaxios"
-import { SynthesisVoice } from "../api/API.js"
-import { trimAudioEnd } from "../audio/AudioUtilities.js"
-import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js"
-import { Phrase, splitToFragments } from "../nlp/Segmentation.js"
-import { Logger } from "../utilities/Logger.js"
-import { concatFloat32Arrays, logToStderr } from "../utilities/Utilities.js"
-import { Timeline } from "../utilities/Timeline.js"
+import { request } from 'gaxios'
+import { SynthesisVoice } from '../api/API.js'
+import { trimAudioEnd } from '../audio/AudioUtilities.js'
+import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { Phrase, splitToFragments } from '../nlp/Segmentation.js'
+import { Logger } from '../utilities/Logger.js'
+import { concatFloat32Arrays, logToStderr } from '../utilities/Utilities.js'
+import { Timeline } from '../utilities/Timeline.js'
 
 const log = logToStderr
 
@@ -13,11 +13,11 @@ const maxTextLengthPerRequest = 200
 
 export async function synthesizeLongText(text: string, voice: string, languageCode: string, sentenceEndPause = 0.75, segmentEndPause = 1.0) {
 	if (text.length == 0) {
-		throw new Error("Text is empty")
+		throw new Error('Text is empty')
 	}
 
 	const logger = new Logger()
-	logger.start("Prepare and split text")
+	logger.start('Prepare and split text')
 
 	const fragments = await splitToFragments(text, maxTextLengthPerRequest, languageCode)
 
@@ -57,7 +57,7 @@ export async function synthesizeLongText(text: string, voice: string, languageCo
 		const endTime = startTime + (trimmedAudio.length / fragmentsSampleRate)
 
 		timeline.push({
-			type: "segment",
+			type: 'segment',
 			text: fragment.text,
 			startTime,
 			endTime
@@ -78,37 +78,37 @@ export async function synthesizeFragment(text: string, voice: string) {
 	const response = await request<any>({
 		url: `https://streamlabs.com/polly/speak`,
 
-		method: "POST",
+		method: 'POST',
 
 		data: {
 			voice,
 			text,
 		},
 
-		responseType: "json"
+		responseType: 'json'
 	})
 
 	const responseObject = response.data
 	const audioUrl = responseObject.speak_url
-	const audioUrlResponse = await request<ArrayBuffer>({ url: audioUrl, responseType: "arraybuffer" })
+	const audioUrlResponse = await request<ArrayBuffer>({ url: audioUrl, responseType: 'arraybuffer' })
 
 	return Buffer.from(audioUrlResponse.data)
 }
 
 export const voiceList: SynthesisVoice[] = [
-	{ name: "Brian", languages: ["en-GB", "en"], gender: "male" },
-	{ name: "Emma", languages: ["en-GB", "en"], gender: "female" },
-	{ name: "Russell", languages: ["en-AU", "en"], gender: "male" },
-	{ name: "Joey", languages: ["en-US", "en"], gender: "male" },
-	{ name: "Matthew", languages: ["en-US", "en"], gender: "male" },
-	{ name: "Joanna", languages: ["en-US", "en"], gender: "female" },
-	{ name: "Kimberly", languages: ["en-US", "en"], gender: "female" },
-	{ name: "Amy", languages: ["en-GB", "en"], gender: "female" },
-	{ name: "Geraint", languages: ["en-GB-WLS", "en-GB", "en"], gender: "male" },
-	{ name: "Nicole", languages: ["en-AU", "en"], gender: "female" },
-	{ name: "Justin", languages: ["en-US", "en"], gender: "male" },
-	{ name: "Ivy", languages: ["en-US", "en"], gender: "female" },
-	{ name: "Kendra", languages: ["en-US", "en"], gender: "female" },
-	{ name: "Salli", languages: ["en-US", "en"], gender: "female" },
-	{ name: "Raveena", languages: ["en-IN", "en"], gender: "female" },
+	{ name: 'Brian', languages: ['en-GB', 'en'], gender: 'male' },
+	{ name: 'Emma', languages: ['en-GB', 'en'], gender: 'female' },
+	{ name: 'Russell', languages: ['en-AU', 'en'], gender: 'male' },
+	{ name: 'Joey', languages: ['en-US', 'en'], gender: 'male' },
+	{ name: 'Matthew', languages: ['en-US', 'en'], gender: 'male' },
+	{ name: 'Joanna', languages: ['en-US', 'en'], gender: 'female' },
+	{ name: 'Kimberly', languages: ['en-US', 'en'], gender: 'female' },
+	{ name: 'Amy', languages: ['en-GB', 'en'], gender: 'female' },
+	{ name: 'Geraint', languages: ['en-GB-WLS', 'en-GB', 'en'], gender: 'male' },
+	{ name: 'Nicole', languages: ['en-AU', 'en'], gender: 'female' },
+	{ name: 'Justin', languages: ['en-US', 'en'], gender: 'male' },
+	{ name: 'Ivy', languages: ['en-US', 'en'], gender: 'female' },
+	{ name: 'Kendra', languages: ['en-US', 'en'], gender: 'female' },
+	{ name: 'Salli', languages: ['en-US', 'en'], gender: 'female' },
+	{ name: 'Raveena', languages: ['en-IN', 'en'], gender: 'female' },
 ]