echogarden 2.0.13 → 2.1.0

package/src/api/Synthesis.ts

@@ -159,10 +159,10 @@ async function synthesizeSegments(segments: string[], options: SynthesisOptions,
 
 			const sentenceStartTime = timeOffset
 
-			let sentencetSynthesisOptions: SynthesisOptions = { postProcessing: { normalizeAudio: false } }
-			sentencetSynthesisOptions = extendDeep(options, sentencetSynthesisOptions)
+			let sentenceSynthesisOptions: SynthesisOptions = { postProcessing: { normalizeAudio: false } }
+			sentenceSynthesisOptions = extendDeep(options, sentenceSynthesisOptions)
 
-			const { synthesizedAudio: sentenceRawAudio, timeline: sentenceTimeline } = await synthesizeSegment(sentenceText, sentencetSynthesisOptions)
+			const { synthesizedAudio: sentenceRawAudio, timeline: sentenceTimeline } = await synthesizeSegment(sentenceText, sentenceSynthesisOptions)
 
 			const endPause = sentenceIndex == sentences.length - 1 ? options.segmentEndPause! : options.sentenceEndPause!
 			sentenceRawAudio.audioChannels[0] = trimAudioEnd(sentenceRawAudio.audioChannels[0], endPause * sentenceRawAudio.sampleRate)
@@ -414,6 +414,47 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 			break
 		}
 
+		case 'kokoro': {
+			if (inputIsSSML) {
+				throw new Error(`The Kokoro engine doesn't currently support SSML inputs`)
+			}
+
+			const kokoroOptions = options.kokoro!
+
+			const kokoroTTS = await import('../synthesis/KokoroTTS.js')
+
+			const lexicons = await loadLexiconsForLanguage(language, options.customLexiconPaths)
+			const onnxExecutionProviders: OnnxExecutionProvider[] = kokoroOptions.provider ? [kokoroOptions.provider] : []
+			const modelName = kokoroOptions.model!
+			const modelPackageName = `kokoro-${modelName}`
+
+			const modelPath = await loadPackage(modelPackageName)
+			const voicesPath = await loadPackage('kokoro-82m-v1.0-voices')
+
+			logger.end()
+
+			logger.logTitledMessage(`Using model`, modelPackageName)
+
+			const { rawAudio, timeline: outTimeline } = await kokoroTTS.synthesizeSentence(
+				text,
+				selectedVoice,
+				speed,
+				lexicons,
+				modelPath,
+				voicesPath,
+				onnxExecutionProviders
+			)
+
+			synthesizedAudio = rawAudio
+			timeline = outTimeline
+
+			shouldPostprocessPitch = true
+
+			logger.end()
+
+			break
+		}
+
 		case 'pico': {
 			if (inputIsSSML) {
 				throw new Error(`The SVOX Pico engine doesn't currently support SSML inputs`)
@@ -459,6 +500,35 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 			break
 		}
 
+		case 'gnuspeech': {
+			if (inputIsSSML) {
+				throw new Error(`The Gnuspeech engine doesn't currently support SSML inputs`)
+			}
+
+			const engineOptions = options.gnuspeech!
+
+			const GnuSpeech = await import('../synthesis/GnuSpeechTTS.js')
+			const { defaultGnuSpeechOptions } = await import('@echogarden/gnuspeech-wasm')
+
+			const gnuSpeechOptions = extendDeep(defaultGnuSpeechOptions, engineOptions)
+
+			if (!engineOptions.tempo) {
+				gnuSpeechOptions.tempo = speed
+			}
+
+			await logger.startAsync(`Synthesize with Gnuspeech`)
+
+			const { rawAudio } = await GnuSpeech.synthesize(simplifiedText, gnuSpeechOptions)
+
+			synthesizedAudio = rawAudio
+
+			shouldPostprocessPitch = true
+
+			logger.end()
+
+			break
+		}
+
 		case 'espeak': {
 			const EspeakTTS = await import('../synthesis/EspeakTTS.js')
 
@@ -959,7 +1029,7 @@ async function synthesizeSegment(text: string, options: SynthesisOptions) {
 
 	logger.end()
 
-	logger.logDuration('Segment synthesis time', startTimestamp, chalk.magentaBright)
+	logger.logDuration('Part synthesis time', startTimestamp, chalk.magentaBright)
 
 	return { synthesizedAudio, timeline }
 }
@@ -997,7 +1067,12 @@ function convertPitchScaleToSSMLValueString(pitch: number, voiceGender: VoiceGen
 	}
 }
 
-export type SynthesisEngine = 'vits' | 'pico' | 'flite' | 'espeak' | 'sam' | 'sapi' | 'msspeech' | 'coqui-server' | 'google-cloud' | 'microsoft-azure' | 'amazon-polly' | 'openai-cloud' | 'elevenlabs' | 'google-translate' | 'microsoft-edge' | 'streamlabs-polly'
+export type SynthesisEngine =
+	'vits' | 'kokoro' | 'pico' | 'flite' | 'gnuspeech' |
+	'espeak' | 'sam' | 'sapi' | 'msspeech' | 'coqui-server' |
+	'google-cloud' | 'microsoft-azure' | 'amazon-polly' |
+	'openai-cloud' | 'elevenlabs' | 'google-translate' |
+	'microsoft-edge' | 'streamlabs-polly'
 
 export type TimePitchShiftingMethod = 'sonic' | 'rubberband'
 
@@ -1051,12 +1126,23 @@ export interface SynthesisOptions {
 		provider?: OnnxExecutionProvider
 	}
 
+	kokoro?: {
+		provider?: OnnxExecutionProvider
+		model?: '82m-v1.0-fp32' | '82m-v1.0-quantized'
+	}
+
 	pico?: {
 	}
 
 	flite?: {
 	}
 
+	gnuspeech?: {
+		tempo?: number
+		controlRate?: number
+		debug?: boolean
+	}
+
 	espeak?: {
 		rate?: number
 		pitch?: number
@@ -1186,17 +1272,25 @@ export const defaultSynthesisOptions: SynthesisOptions = {
 		provider: undefined,
 	},
 
+	kokoro: {
+		model: '82m-v1.0-fp32'
+	},
+
 	pico: {
 	},
 
 	flite: {
 	},
 
+	gnuspeech: {
+		debug: false,
+	},
+
 	espeak: {
 		rate: undefined,
 		pitch: undefined,
 		pitchRange: undefined,
-		useKlatt: false
+		useKlatt: false,
 	},
 
 	sam: {
@@ -1330,6 +1424,14 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 				break
 			}
 
+			case 'gnuspeech': {
+				const GnuSpeech = await import('../synthesis/GnuSpeechTTS.js')
+
+				voiceList = GnuSpeech.voiceList
+
+				break
+			}
+
 			case 'sam': {
 				voiceList.push({
 					name: 'sam',
@@ -1350,6 +1452,14 @@ export async function requestVoiceList(options: VoiceListRequestOptions): Promis
 				break
 			}
 
+			case 'kokoro': {
+				const KokoroTTS = await import('../synthesis/KokoroTTS.js')
+
+				voiceList = KokoroTTS.voiceList
+
+				break
+			}
+
 			case 'sapi': {
 				const SapiTTS = await import('../synthesis/SapiTTS.js')
 
@@ -1701,28 +1811,40 @@ export const synthesisEngines: EngineMetadata[] = [
 		description: 'A high-quality end-to-end neural speech synthesis architecture.',
 		type: 'local'
 	},
+	{
+		id: 'kokoro',
+		name: 'Kokoro',
+		description: 'A high-quality neural speech synthesis model based on the StyleTTS 2 architecture.',
+		type: 'local'
+	},
 	{
 		id: 'pico',
 		name: 'SVOX Pico',
-		description: 'A legacy diphone-based synthesis engine.',
+		description: 'A legacy diphone-based speech synthesizer.',
 		type: 'local'
 	},
 	{
 		id: 'flite',
 		name: 'Flite',
-		description: 'A legacy diphone-based synthesis engine.',
+		description: 'A legacy diphone-based speech synthesizer.',
+		type: 'local'
+	},
+	{
+		id: 'gnuspeech',
+		name: 'Gnuspeech',
+		description: 'A legacy articulatory speech synthesizer.',
 		type: 'local'
 	},
 	{
 		id: 'espeak',
 		name: 'eSpeak NG',
-		description: `A lightweight 'robot' sounding formant-based synthesizer.`,
+		description: `A lightweight, highly multilingual, 'robot'-like formant-based speech synthesizer.`,
 		type: 'local'
 	},
 	{
 		id: 'sam',
 		name: 'SAM (Software Automatic Mouth)',
-		description: `A classic 'robot' speech synthesizer from 1982.`,
+		description: `A classic 'robot'-like speech synthesizer from 1982.`,
 		type: 'local'
 	},
 	{

package/src/recognition/WhisperSTT.ts

@@ -792,7 +792,7 @@ export class Whisper {
 		let decodedTokensCrossAttentionQKs: OnnxLikeFloat32Tensor[] = []
 
 		for (let i = 0; i < decodedTokens.length; i++) {
-			decodedTokensTimestampLogits.push(new Array(1501))
+			decodedTokensTimestampLogits.push(new Array(1501)) // Should the length be 1500 instead?
 			decodedTokensConfidence.push(1.0)
 			decodedTokensCrossAttentionQKs.push(undefined as any)
 		}
@@ -1569,7 +1569,7 @@ export class Whisper {
 
 		const timestampTokensCount = 1501
 
-		for (let i = 0; i < timestampTokensCount; i++) {
+		for (let i = 0; i <= timestampTokensCount; i++) {
 			const tokenIndex = this.tokenConfig.timestampTokensStart + i
 			const tokenTime = this.timestampTokenToSeconds(tokenIndex)
 
@@ -1727,11 +1727,15 @@ export class Whisper {
 			throw new Error(`Invalid timestamp token: ${timestampToken}`)
 		}
 
-		return (timestampToken - this.tokenConfig.timestampTokensStart) * 0.02
+		let seconds = (timestampToken - this.tokenConfig.timestampTokensStart) * 0.02
+		seconds = clip(seconds, 0.0, 30.0)
+
+		return seconds
 	}
 
 	isValidToken(token: number) {
-		return token < this.tokenConfig.timestampTokensEnd
+		//return token < this.tokenConfig.timestampTokensEnd
+		return token <= this.tokenConfig.timestampTokensEnd
 	}
 
 	assertIsValidToken(token: number) {

package/src/synthesis/GnuSpeechTTS.ts

@@ -0,0 +1,40 @@
+import { defaultGnuSpeechOptions, GnuSpeechOptions, synthesize as gnuSpeechSynthesize } from '@echogarden/gnuspeech-wasm'
+import { SynthesisVoice } from '../api/Synthesis.js'
+import { decodeWaveToRawAudio } from '../audio/AudioUtilities.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+
+export async function synthesize(text: string, options: GnuSpeechOptions) {
+	const gnuSpeechOptions = extendDeep(defaultGnuSpeechOptions, options)
+
+	const { audioData, params } = await gnuSpeechSynthesize(text, gnuSpeechOptions)
+
+	return decodeWaveToRawAudio(audioData)
+}
+
+export const voiceList: SynthesisVoice[] = [
+	{
+		name: 'male',
+		languages: ['en-US', 'en'],
+		gender: 'male',
+	},
+	{
+		name: 'female',
+		languages: ['en-US', 'en'],
+		gender: 'female',
+	},
+	{
+		name: 'large_child',
+		languages: ['en-US', 'en'],
+		gender: 'unknown',
+	},
+	{
+		name: 'small_child',
+		languages: ['en-US', 'en'],
+		gender: 'unknown',
+	},
+	{
+		name: 'baby',
+		languages: ['en-US', 'en'],
+		gender: 'unknown',
+	},
+]