echogarden 2.10.1 → 3.0.0

package/src/recognition/VoskSTT.ts

@@ -1,112 +0,0 @@
-import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
-import * as AudioBufferConversion from '../audio/AudioBufferConversion.js'
-import { Logger } from '../utilities/Logger.js'
-import { concatUint8Arrays, logToStderr } from '../utilities/Utilities.js'
-import { Timeline } from '../utilities/Timeline.js'
-import { RawAudio } from '../audio/AudioUtilities.js'
-
-const log = logToStderr
-
-export async function recognizeFile(filename: string, modelPath: string, verbose = true) {
-	const rawAudio = await FFMpegTranscoder.decodeToChannels(filename, 16000, 1)
-	return recognize(rawAudio, modelPath, verbose)
-}
-
-export async function recognize(rawAudio: RawAudio, modelPath: string, verbose = true) {
-	const logger = new Logger()
-	logger.start('Initialize vosk recognizer')
-
-	const audioChannels = rawAudio.audioChannels
-	const sampleRate = rawAudio.sampleRate
-
-	let Vosk = await import('@echogarden/vosk')
-
-	Vosk.setLogLevel(-1)
-
-	const model = await new Vosk.Model(modelPath)
-
-	const recognizer = new Vosk.Recognizer({ model, sampleRate })
-
-	recognizer.setMaxAlternatives(0)
-	recognizer.setWords(true)
-	recognizer.setPartialWords(true)
-
-	logger.start('Recognize with vosk')
-
-	const recognitionStartTimestamp = logger.getTimestamp()
-
-	const pcmAudio = AudioBufferConversion.encodeToAudioBuffer(audioChannels, 16)
-	const trailingSilence = new Uint8Array(sampleRate * 4)
-	const pcmAudioWithTrailingSilence = concatUint8Arrays([pcmAudio, trailingSilence])
-	const pcmAudioByteCount = pcmAudioWithTrailingSilence.length
-
-	const maxChunkSize = sampleRate * 2.0
-
-	let previousResultText = ''
-
-	for (let readOffset = 0; readOffset < pcmAudioByteCount; readOffset += maxChunkSize) {
-		const chunkSize = Math.min(maxChunkSize, pcmAudioByteCount - readOffset)
-
-		const chunk = pcmAudioWithTrailingSilence.subarray(readOffset, readOffset + chunkSize)
-
-		const speechEnded = await recognizer.acceptWaveformAsync(chunk)
-
-		if (verbose) {
-			const partialResultText = recognizer.partialResult().partial
-
-			if (partialResultText != previousResultText) {
-				//logger.log(partialResultText)
-				//logger.log('')
-
-				previousResultText = partialResultText
-			}
-		}
-	}
-
-	//const speechEnded = await recognizer.acceptWaveformAsync(pcmAudioWithTrailingSilence)
-
-	const result = recognizer.finalResult()
-
-	recognizer.reset()
-	recognizer.free()
-	model.free()
-
-	const transcript: string = result.text
-	const events: VoskRecognitionEvent[] = result.result
-
-	if (events.length == 0) {
-		return { transcript, timeline: [] }
-	}
-
-	const timeline: Timeline = []
-
-	for (let i = 0; i < events.length; i++) {
-		const event = events[i]
-
-		const eventText = event.word
-		const eventStart = event.start
-		const eventEnd = event.end
-		const eventConfidence = event.conf
-
-		timeline.push({
-			type: 'word',
-			text: eventText,
-			startTime: eventStart,
-			endTime: eventEnd,
-			confidence: eventConfidence
-		})
-	}
-
-	//logger.logDuration(`Recognition with vosk`, recognitionStartTimestamp)
-
-	logger.end()
-
-	return { transcript, timeline }
-}
-
-type VoskRecognitionEvent = {
-	word: string
-	start: number
-	end: number
-	conf: number
-}

package/src/synthesis/StreamlabsPollyTTS.ts

@@ -1,114 +0,0 @@
-import { request } from 'gaxios'
-import { SynthesisVoice } from '../api/API.js'
-import { trimAudioEnd } from '../audio/AudioUtilities.js'
-import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
-import { Phrase, splitToFragments } from '../nlp/Segmentation.js'
-import { Logger } from '../utilities/Logger.js'
-import { concatFloat32Arrays, logToStderr } from '../utilities/Utilities.js'
-import { Timeline } from '../utilities/Timeline.js'
-
-const log = logToStderr
-
-const maxTextLengthPerRequest = 200
-
-export async function synthesizeLongText(text: string, voice: string, languageCode: string, sentenceEndPause = 0.75, segmentEndPause = 1.0) {
-	if (text.length == 0) {
-		throw new Error('Text is empty')
-	}
-
-	const logger = new Logger()
-	logger.start('Prepare and split text')
-
-	const fragments = await splitToFragments(text, maxTextLengthPerRequest, languageCode)
-
-	const audioFragments: Float32Array[] = []
-	let fragmentsSampleRate = 0
-
-	const timeline: Timeline = []
-
-	for (let i = 0; i < fragments.length; i++) {
-		const fragment = fragments[i]
-
-		logger.start(`Request synthesis for text fragment ${i + 1}/${fragments.length} from Streamslabs Polly`)
-		const fragmentMp3Stream = await synthesizeFragment(fragment.text, voice)
-
-		if (fragmentMp3Stream.length == 0) {
-			continue
-		}
-
-		const rawAudio = await FFMpegTranscoder.decodeToChannels(fragmentMp3Stream, 24000, 1)
-		fragmentsSampleRate = rawAudio.sampleRate
-
-		let targetEndingSilenceTime: number
-
-		if (fragment.lastSegment?.isSentenceFinalizer) {
-			targetEndingSilenceTime = sentenceEndPause
-		} else if (fragment.lastSegment instanceof Phrase) {
-			targetEndingSilenceTime = segmentEndPause
-		} else {
-			targetEndingSilenceTime = 0
-		}
-
-		const trimmedAudio = trimAudioEnd(rawAudio.audioChannels[0], targetEndingSilenceTime * rawAudio.sampleRate)
-
-		audioFragments.push(trimmedAudio)
-
-		const startTime = timeline.length == 0 ? 0 : timeline[timeline.length - 1].endTime
-		const endTime = startTime + (trimmedAudio.length / fragmentsSampleRate)
-
-		timeline.push({
-			type: 'segment',
-			text: fragment.text,
-			startTime,
-			endTime
-		})
-	}
-
-	const rawAudio = { audioChannels: [concatFloat32Arrays(audioFragments)], sampleRate: fragmentsSampleRate }
-
-	logger.end()
-
-	return {
-		rawAudio,
-		timeline
-	}
-}
-
-export async function synthesizeFragment(text: string, voice: string) {
-	const response = await request<any>({
-		url: `https://streamlabs.com/polly/speak`,
-
-		method: 'POST',
-
-		data: {
-			voice,
-			text,
-		},
-
-		responseType: 'json'
-	})
-
-	const responseObject = response.data
-	const audioUrl = responseObject.speak_url
-	const audioUrlResponse = await request<ArrayBuffer>({ url: audioUrl, responseType: 'arraybuffer' })
-
-	return new Uint8Array(audioUrlResponse.data)
-}
-
-export const voiceList: SynthesisVoice[] = [
-	{ name: 'Brian', languages: ['en-GB', 'en'], gender: 'male' },
-	{ name: 'Emma', languages: ['en-GB', 'en'], gender: 'female' },
-	{ name: 'Russell', languages: ['en-AU', 'en'], gender: 'male' },
-	{ name: 'Joey', languages: ['en-US', 'en'], gender: 'male' },
-	{ name: 'Matthew', languages: ['en-US', 'en'], gender: 'male' },
-	{ name: 'Joanna', languages: ['en-US', 'en'], gender: 'female' },
-	{ name: 'Kimberly', languages: ['en-US', 'en'], gender: 'female' },
-	{ name: 'Amy', languages: ['en-GB', 'en'], gender: 'female' },
-	{ name: 'Geraint', languages: ['en-GB-WLS', 'en-GB', 'en'], gender: 'male' },
-	{ name: 'Nicole', languages: ['en-AU', 'en'], gender: 'female' },
-	{ name: 'Justin', languages: ['en-US', 'en'], gender: 'male' },
-	{ name: 'Ivy', languages: ['en-US', 'en'], gender: 'female' },
-	{ name: 'Kendra', languages: ['en-US', 'en'], gender: 'female' },
-	{ name: 'Salli', languages: ['en-US', 'en'], gender: 'female' },
-	{ name: 'Raveena', languages: ['en-IN', 'en'], gender: 'female' },
-]

package/src/utilities/BinaryUtilities.ts

@@ -1,138 +0,0 @@
-/////////////////////////////////////////////////////////////////////////////////////
-// Unsigned operations
-/////////////////////////////////////////////////////////////////////////////////////
-export function readUint8(buffer: Uint8Array, offset: number) {
-	return buffer[offset]
-}
-
-export function writeUint8(buffer: Uint8Array, value: number, offset: number) {
-	if (value < 0 || value > 255) {
-		throw new Error(`Value ${value} is outside the range of an 8-bit unsigned integer`)
-	}
-
-	buffer[offset] = value
-}
-
-export function readUint16LE(buffer: Uint8Array, offset: number) {
-	return (buffer[offset]) | (buffer[offset + 1] << 8)
-}
-
-export function writeUint16LE(buffer: Uint8Array, value: number, offset: number) {
-	if (value < 0 || value > 65535) {
-		throw new Error(`Value ${value} is outside the range of a 16-bit unsigned integer`)
-	}
-
-	buffer[offset] = value & 0xff
-	buffer[offset + 1] = (value >>> 8) & 0xff
-}
-
-export function readUint32LE(buffer: Uint8Array, offset: number) {
-	return readInt32LE(buffer, offset) >>> 0
-}
-
-export function writeUint32LE(buffer: Uint8Array, value: number, offset: number) {
-	if (value < 0 || value > 4294967295) {
-		throw new Error(`Value ${value} is outside the range of a 32-bit unsigned integer`)
-	}
-
-	buffer[offset] = value & 0xff
-	buffer[offset + 1] = (value >>> 8) & 0xff
-	buffer[offset + 2] = (value >>> 16) & 0xff
-	buffer[offset + 3] = (value >>> 24) & 0xff
-}
-
-/////////////////////////////////////////////////////////////////////////////////////
-// Signed operations
-/////////////////////////////////////////////////////////////////////////////////////
-export function readInt8(buffer: Uint8Array, offset: number) {
-	const unsignedValue = buffer[offset]
-
-	if (unsignedValue < 128) {
-		return unsignedValue
-	} else {
-		return unsignedValue - 256
-	}
-}
-
-export function writeInt8(buffer: Uint8Array, value: number, offset: number) {
-	if (value < -128 || value > 127) {
-		throw new Error(`Value ${value} is outside the range of an 8-bit signed integer`)
-	}
-
-	let unsignedValue: number
-
-	if (value >= 0) {
-		unsignedValue = value
-	} else {
-		unsignedValue = value + 256
-	}
-
-	buffer[offset] = unsignedValue
-}
-
-export function readInt16LE(buffer: Uint8Array, offset: number) {
-	const unsignedValue = readUint16LE(buffer, offset)
-
-	if (unsignedValue < 32768) {
-		return unsignedValue
-	} else {
-		return unsignedValue - 65536
-	}
-}
-
-export function writeInt16LE(buffer: Uint8Array, value: number, offset: number) {
-	if (value < -32768 || value > 32767) {
-		throw new Error(`Value ${value} is outside the range of a 16-bit signed integer`)
-	}
-
-	let unsignedValue: number
-
-	if (value >= 0) {
-		unsignedValue = value
-	} else {
-		unsignedValue = value + 65536
-	}
-
-	writeUint16LE(buffer, unsignedValue, offset)
-}
-
-export function readInt32LE(buffer: Uint8Array, offset: number) {
-	const value =
-		(buffer[offset]) |
-		(buffer[offset + 1] << 8) |
-		(buffer[offset + 2] << 16) |
-		(buffer[offset + 3] << 24)
-
-	return value
-}
-
-export function writeInt32LE(buffer: Uint8Array, value: number, offset: number) {
-	if (value < -2147483648 || value > 2147483647) {
-		throw new Error(`Value ${value} is outside the range of a 32-bit signed integer`)
-	}
-
-	buffer[offset] = value & 0xff
-	buffer[offset + 1] = (value >> 8) & 0xff
-	buffer[offset + 2] = (value >> 16) & 0xff
-	buffer[offset + 3] = (value >> 24) & 0xff
-}
-
-/////////////////////////////////////////////////////////////////////////////////////
-// Misc operations
-/////////////////////////////////////////////////////////////////////////////////////
-export function writeAscii(buffer: Uint8Array, asciiString: string, writeStartOffset: number) {
-	const writeEndOffset = Math.min(writeStartOffset + asciiString.length, buffer.length)
-
-	let readOffset = 0
-	let writeOffset = writeStartOffset
-
-	while (writeOffset < writeEndOffset) {
-		const charCode = asciiString.charCodeAt(readOffset++)
-
-		if (charCode >= 128) {
-			throw new Error(`Character '${asciiString[readOffset]}' (code: ${charCode}) can't be encoded as ASCII`)
-		}
-
-		buffer[writeOffset++] = charCode
-	}
-}