echogarden 2.10.2 → 3.0.0

package/src/recognition/DeepgramSTT.ts

@@ -1,12 +1,13 @@
-import { GaxiosResponse, request } from 'gaxios'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { Logger } from '../utilities/Logger.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
-import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
+import { TimelineEntry } from '../utilities/Timeline.js'
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
+import { RecognitionCallbacks } from '../api/Recognition.js'
+import { EasierHttpRequestError, requestHttp } from 'easier-http-request'
 
-export async function recognize(rawAudio: RawAudio, languageCode: string | undefined, options: DeepgramSTTOptions) {
-	const logger = new Logger()
+export async function recognize(rawAudio: RawAudio, languageCode: string | undefined, options: DeepgramSTTOptions, callbacks: RecognitionCallbacks) {
+	const logger = new Logger(callbacks.logLevel)
 
 	logger.start('Initialize Deepgram recognition')
 
@@ -35,15 +36,16 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 
 	const audioData = await FFMpegTranscoder.encodeFromChannels(
 		rawAudio,
-		FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('opus')
+		FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('opus'),
+		callbacks
 	)
 
 	logger.start('Send request to Deepgram API')
 
-	let response: GaxiosResponse<any>
+	let deepgramResponse: DeepgramResponse
 
 	try {
-		response = await request<any>({
+		const response = await requestHttp({
 			method: 'POST',
 
 			url: 'https://api.deepgram.com/v1/listen',
@@ -58,25 +60,23 @@ export async function recognize(rawAudio: RawAudio, languageCode: string | undef
 
 			body: audioData,
 
-			responseType: 'json',
+			abortSignal: callbacks?.abortSignal,
 		})
-	} catch (e: any) {
-		const response = e.response
 
-		if (response) {
-			logger.log(`Request failed with status code ${response.status}`)
+		deepgramResponse = await response.json()
+	} catch (e: any) {
+		if (e instanceof EasierHttpRequestError) {
+			logger.log(`Request failed with status code ${e.statusCode}: ${e.statusText}.`)
 
-			if (response.data) {
+			if (e.errorBody) {
 				logger.log(`Server responded with:`)
-				logger.log(response.data)
+				logger.log(e.errorBody)
 			}
 		}
 
 		throw e
 	}
 
-	const deepgramResponse: DeepgramResponse = response.data
-
 	const firstAlternative = deepgramResponse.results?.channels[0]?.alternatives[0]
 
 	// Extract transcript and create timeline

package/src/recognition/GoogleCloudSTT.ts

@@ -1,17 +1,17 @@
-import { request } from 'gaxios'
-
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
 import { Logger } from '../utilities/Logger.js'
 import { Timeline } from '../utilities/Timeline.js'
 import { RawAudio } from '../audio/AudioUtilities.js'
 import { encodeBase64 } from '../encodings/Base64.js'
+import { RecognitionCallbacks } from '../api/Recognition.js'
+import { requestHttp } from 'easier-http-request'
 
 export type AudioEncoding = 'LINEAR16' | 'FLAC' | 'MULAW' | 'AMR' | 'AMR' | 'AMR_WB' | 'OGG_OPUS' | 'SPEEX_WITH_HEADER_BYTE' | 'MP3' | 'WEBM_OPUS'
 
-export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode = 'en-US') {
-	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: 'flac', sampleRate: 16000, sampleFormat: 's16', channelCount: 1 })
+export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode = 'en-US', callbacks: RecognitionCallbacks) {
+	const flac16Khz16bitMonoAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, { format: 'flac', sampleRate: 16000, sampleFormat: 's16', channelCount: 1 }, callbacks)
 
-	const logger = new Logger()
+	const logger = new Logger(callbacks.logLevel)
 	logger.start('Request recognition from Google Cloud')
 
 	const requestBody = {
@@ -35,7 +35,7 @@ export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode
 		}
 	}
 
-	const response = await request<any>({
+	const response = await requestHttp({
 		method: 'POST',
 
 		url: `https://speech.googleapis.com/v1p1beta1/speech:recognize`,
@@ -48,22 +48,24 @@ export async function recognize(rawAudio: RawAudio, apiKey: string, languageCode
 			'User-Agent': ''
 		},
 
-		data: requestBody,
+		body: requestBody,
 
-		responseType: 'json'
+		abortSignal: callbacks.abortSignal,
 	})
 
+	const responseObject = await response.json()
+
 	logger.start('Parse response body')
 
-	const result = parseResponseBody(response.data)
+	const result = parseResponseObject(responseObject)
 
 	logger.end()
 
 	return result
 }
 
-function parseResponseBody(responseBody: any) {
-	const results = responseBody.results
+function parseResponseObject(responseObject: any) {
+	const results = responseObject.results
 
 	let transcript = ''
 	const timeline: Timeline = []
@@ -81,7 +83,7 @@ function parseResponseBody(responseBody: any) {
 			timeline.push({
 				type: 'word',
 				text: wordEvent.word,
-				startTime: parseFloat(wordEvent.startTime.replace('s','')),
+				startTime: parseFloat(wordEvent.startTime.replace('s', '')),
 				endTime: parseFloat(wordEvent.endTime.replace('s', '')),
 				confidence: wordEvent.confidence
 			})

package/src/recognition/OpenAICloudSTT.ts

@@ -5,9 +5,10 @@ import { Logger } from '../utilities/Logger.js'
 import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { alignSegments } from '../api/Alignment.js'
+import { RecognitionCallbacks } from '../api/Recognition.js'
 
-export async function recognize(rawAudio: RawAudio, languageCode: string, options: OpenAICloudSTTOptions, task: Task = 'transcribe') {
-	const logger = new Logger()
+export async function recognize(rawAudio: RawAudio, languageCode: string, options: OpenAICloudSTTOptions, task: Task = 'transcribe', callbacks: RecognitionCallbacks) {
+	const logger = new Logger(callbacks.logLevel)
 
 	logger.start('Load OpenAI module')
 
@@ -30,7 +31,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 
 	logger.start('Encode audio to send')
 	const ffmpegOptions = FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('mp3')
-	const encodedAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, ffmpegOptions)
+	const encodedAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, ffmpegOptions, callbacks)
 	const virtualFileStream = createVirtualFileReadStreamForBuffer(encodedAudio, 'audio.mp3')
 
 	logger.start(options.baseURL ? `Send request to ${options.baseURL}` : 'Send request to OpenAI Cloud API')
@@ -57,7 +58,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			response_format: responseFormat,
 			temperature: options.temperature,
 			timestamp_granularities,
-		}) as any as VerboseResponse
+		}, { signal: callbacks.abortSignal }) as any as VerboseResponse
 	} else if (task == 'translate') {
 		response = await openai.audio.translations.create({
 			file: virtualFileStream,
@@ -65,7 +66,7 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			prompt: options.prompt,
 			response_format: responseFormat,
 			temperature: options.temperature,
-		}) as any as VerboseResponse
+		}, { signal: callbacks.abortSignal }) as any as VerboseResponse
 	} else {
 		throw new Error(`Invalid task`)
 	}
@@ -92,7 +93,12 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 		if (task === 'transcribe') {
 			logger.start('Align segments')
 
-			timeline = await alignSegments(rawAudio, segmentTimeline, { language: languageCode })
+			timeline = await alignSegments(
+				rawAudio,
+				segmentTimeline,
+				{ language: languageCode },
+				{ ...callbacks, logLevel: 'warning' },
+			)
 		} else {
 			timeline = segmentTimeline
 		}

package/src/recognition/WhisperCommon.ts

@@ -0,0 +1,130 @@
+import { getShortLanguageCode } from '../utilities/Locale.js'
+import { loadPackage, LoadPackageCallbacks } from '../utilities/PackageManager.js'
+import { joinPath } from '../utilities/PathUtilities.js'
+
+export async function loadModelPackage(quantizedModelId: WhisperQuantizedModelId | undefined, languageCode: string | undefined, callbacks: LoadPackageCallbacks) {
+	if (quantizedModelId === 'large') {
+		quantizedModelId = 'large-v2'
+	}
+
+	if (quantizedModelId) {
+		const modelName = getShortModelIdFromWhisperCppModelId(quantizedModelId)
+
+		if (languageCode != 'en' && modelName.endsWith('.en')) {
+			throw new Error(`The English-only model '${modelName}' cannot be used with a non-English language '${languageCode}'.`)
+		}
+	} else {
+		if (languageCode) {
+			const shortLanguageCode = getShortLanguageCode(languageCode)
+
+			quantizedModelId = shortLanguageCode == 'en' ? 'base.en' : 'base'
+		} else {
+			quantizedModelId = 'base'
+		}
+	}
+
+	const packageName = `whisper.cpp-${quantizedModelId}`
+	const modelDir = await loadPackage(packageName, callbacks)
+	const modelPath = joinPath(modelDir, `ggml-${quantizedModelId}.bin`)
+	const modelId = getShortModelIdFromWhisperCppModelId(quantizedModelId)
+
+	return { modelId, modelPath }
+}
+
+function getShortModelIdFromWhisperCppModelId(quantizedModelId: WhisperQuantizedModelId): WhisperModelId {
+	if (quantizedModelId.startsWith('large-v1')) {
+		return 'large-v1'
+	}
+
+	if (quantizedModelId.startsWith('large-v2')) {
+		return 'large-v2'
+	}
+
+	if (quantizedModelId.startsWith('large-v3-turbo')) {
+		return 'large-v3-turbo'
+	}
+
+	if (quantizedModelId.startsWith('large-v3')) {
+		return 'large-v3'
+	}
+
+	const lastDashIndex = quantizedModelId.lastIndexOf('-')
+
+	let modelName: string
+
+	if (lastDashIndex >= 0) {
+		modelName = quantizedModelId.substring(0, lastDashIndex) as WhisperModelId
+	} else {
+		modelName = quantizedModelId
+	}
+
+	return modelName as WhisperModelId
+}
+
+export type WhisperModelId =
+	'tiny' |
+	'tiny.en' |
+
+	'base' |
+	'base.en' |
+
+	'small' |
+	'small.en' |
+
+	'medium' |
+	'medium.en' |
+
+	'large-v1' |
+	'large-v2' |
+	'large-v3' |
+	'large-v3-turbo'
+
+export type WhisperQuantizedModelId =
+	'tiny' |
+	'tiny-q5_1' |
+	'tiny-q8_0' |
+
+	'tiny.en' |
+	'tiny.en-q5_1' |
+	'tiny.en-q8_0' |
+
+	'base' |
+	'base-q5_1' |
+	'base-q8_0' |
+
+	'base.en' |
+	'base.en-q5_1' |
+	'base.en-q8_0' |
+
+	'small' |
+	'small-q5_1' |
+	'small-q8_0' |
+
+	'small.en' |
+	'small.en-q5_1' |
+	'small.en-q8_0' |
+
+	'medium' |
+	'medium-q5_0' |
+	'medium-q8_0' |
+
+	'medium.en' |
+	'medium.en-q5_0' |
+	'medium.en-q8_0' |
+
+	'large' |
+
+	'large-v1' |
+
+	'large-v2' |
+	'large-v2-q5_0' |
+	'large-v2-q8_0' |
+
+	'large-v3' |
+	'large-v3-q5_0' |
+
+	'large-v3-turbo' |
+	'large-v3-turbo-q5_0' |
+	'large-v3-turbo-q8_0'
+
+export type WhisperTask = 'transcribe' | 'translate' | 'detect-language'