echogarden 1.5.0 → 1.6.0

package/src/recognition/OpenAICloudSTT.ts

@@ -1,8 +1,10 @@
-import { RawAudio } from '../audio/AudioUtilities.js';
 import * as FFMpegTranscoder from '../codecs/FFMpegTranscoder.js'
-import { Logger } from '../utilities/Logger.js';
-import { extendDeep } from '../utilities/ObjectUtilities.js';
-import { Timeline, TimelineEntry } from '../utilities/Timeline.js';
+import { RawAudio } from '../audio/AudioUtilities.js'
+import { createVirtualFileReadStreamForBuffer } from '../utilities/VirtualFileReadStream.js'
+import { Logger } from '../utilities/Logger.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
+import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
+import { alignSegments } from '../api/Alignment.js'
 
 export async function recognize(rawAudio: RawAudio, languageCode: string, options: OpenAICloudSTTOptions, task: Task = 'transcribe') {
 	const logger = new Logger()
@@ -11,41 +13,56 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 
 	options = extendDeep(defaultOpenAICloudSTTOptions, options)
 
+	if (options.requestWordTimestamps === undefined) {
+		options.requestWordTimestamps = options.baseURL === undefined
+	}
+
+	if (options.model === undefined) {
+		if (options.baseURL === undefined) {
+			options.model = 'whisper-1'
+		} else {
+			throw new Error(`A custom provider for the OpenAI Cloud API requires specifying a model name`)
+		}
+	}
+
 	const { default: OpenAI } = await import('openai')
 	const openai = new OpenAI(options)
 
 	logger.start('Encode audio to send')
 	const ffmpegOptions = FFMpegTranscoder.getDefaultFFMpegOptionsForSpeech('mp3')
 	const encodedAudio = await FFMpegTranscoder.encodeFromChannels(rawAudio, ffmpegOptions)
-	const audioAsWaveBlob = new FileLikeBlob([encodedAudio], 'audio', Date.now(), { type: 'audio/mpeg' })
+	const virtualFileStream = createVirtualFileReadStreamForBuffer(encodedAudio, 'audio.mp3')
 
-	logger.start('Request recognition from OpenAI Cloud API')
+	logger.start(options.baseURL ? `Send request to ${options.baseURL}` : 'Send request to OpenAI Cloud API')
 
 	let response: VerboseResponse
 
-	if (task =='transcribe') {
+	if (task == 'transcribe') {
+		const timestamp_granularities: ('word' | 'segment')[] | undefined =
+			options.requestWordTimestamps ? ['word', 'segment'] : undefined
+
 		response = await openai.audio.transcriptions.create({
-			file: audioAsWaveBlob,
-			model: options.model!,
+			file: virtualFileStream,
+			model: options.model,
 			language: languageCode,
 			prompt: options.prompt,
 			response_format: 'verbose_json',
 			temperature: options.temperature,
-			timestamp_granularities: ['word', 'segment']
-		}) as VerboseResponse
+			timestamp_granularities,
+		}) as any as VerboseResponse
 	} else if (task == 'translate') {
 		response = await openai.audio.translations.create({
-			file: audioAsWaveBlob,
-			model: options.model!,
+			file: virtualFileStream,
+			model: options.model,
 			prompt: options.prompt,
 			response_format: 'verbose_json',
 			temperature: options.temperature,
-		}) as VerboseResponse
+		}) as any as VerboseResponse
 	} else {
 		throw new Error(`Invalid task`)
 	}
 
-	const transcript = response.text
+	const transcript = response.text.trim()
 
 	let timeline: Timeline
 
@@ -57,12 +74,20 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 			endTime: entry.end
 		}))
 	} else {
-		timeline = response.segments.map<TimelineEntry>(entry => ({
+		const segmentTimeline = response.segments.map<TimelineEntry>(entry => ({
 			type: 'segment',
 			text: entry.text,
 			startTime: entry.start,
 			endTime: entry.end
 		}))
+
+		if (task === 'transcribe') {
+			logger.start('Align segments')
+
+			timeline = await alignSegments(rawAudio, segmentTimeline, { language: languageCode })
+		} else {
+			timeline = segmentTimeline
+		}
 	}
 
 	logger.end()
@@ -70,17 +95,6 @@ export async function recognize(rawAudio: RawAudio, languageCode: string, option
 	return { transcript, timeline }
 }
 
-class FileLikeBlob extends Blob {
-	constructor(
-		public readonly parts: BlobPart[],
-		public readonly name: string,
-		public readonly lastModified: number,
-		options: BlobPropertyBag,
-	) {
-		super(parts, options)
-	}
-}
-
 interface VerboseResponse {
 	task: string
 	language: string
@@ -115,7 +129,7 @@ interface VerboseResponse {
 type Task = 'transcribe' | 'translate'
 
 export interface OpenAICloudSTTOptions {
-	model?: 'whisper-1'
+	model?: 'whisper-1' | string
 
 	apiKey?: string
 	organization?: string
@@ -126,6 +140,8 @@ export interface OpenAICloudSTTOptions {
 
 	timeout?: number
 	maxRetries?: number
+
+	requestWordTimestamps?: boolean
 }
 
 export const defaultOpenAICloudSTTOptions: OpenAICloudSTTOptions = {
@@ -133,10 +149,12 @@ export const defaultOpenAICloudSTTOptions: OpenAICloudSTTOptions = {
 	organization: undefined,
 	baseURL: undefined,
 
-	model: 'whisper-1',
+	model: undefined,
 	temperature: 0,
 	prompt: undefined,
 
 	timeout: undefined,
 	maxRetries: 10,
+
+	requestWordTimestamps: undefined,
 }

package/src/recognition/WhisperCppSTT.ts

@@ -54,7 +54,7 @@ export async function recognize(
 				}
 			} else {
 				if (options.enableGPU) {
-					buildKind = 'cublas-11.8.0'
+					buildKind = 'cublas-12.4.0'
 				} else {
 					buildKind = 'cpu'
 				}
@@ -240,6 +240,8 @@ async function parseResultObject(resultObject: WhisperCppVerboseResult, modelNam
 
 	let currentCorrectionTimeOffset = 0
 
+	let lastTokenEndOffset = 0
+
 	for (let segmentIndex = 0; segmentIndex < resultObject.transcription.length; segmentIndex++) {
 		const segmentObject = resultObject.transcription[segmentIndex]
 
@@ -248,6 +250,17 @@ async function parseResultObject(resultObject: WhisperCppVerboseResult, modelNam
 		for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex++) {
 			const tokenObject = tokens[tokenIndex]
 
+			// Workaround whisper.cpp issue with missing offsets by falling back to last known end offset
+			// when they are not included
+			if (!tokenObject.offsets) {
+				tokenObject.offsets = {
+					from: lastTokenEndOffset,
+					to: lastTokenEndOffset,
+				}
+			} else {
+				lastTokenEndOffset = tokenObject.offsets.to
+			}
+
 			if (tokenIndex === 0 && tokenObject.text === '[_BEG_]' && tokenObject.offsets.from === 0) {
 				currentCorrectionTimeOffset = segmentObject.offsets.from / 1000
 			}
@@ -370,7 +383,7 @@ export async function loadModelPackage(modelId: WhisperCppModelId | undefined, l
 	return { modelName, modelPath }
 }
 
-export type WhisperCppBuild = 'cpu' | 'cublas-11.8.0' | 'cublas-12.4.0' | 'custom'
+export type WhisperCppBuild = 'cpu' | 'cublas-12.4.0' | 'custom'
 
 export async function loadExecutablePackage(buildKind: WhisperCppBuild) {
 	if (buildKind === 'custom') {
@@ -384,20 +397,20 @@ export async function loadExecutablePackage(buildKind: WhisperCppBuild) {
 
 	if (buildKind.startsWith('cublas-')) {
 		if (platform === 'win32' && arch === 'x64') {
-			packageName = `whisper.cpp-binaries-windows-x64-${buildKind}-latest-patched`
+			packageName = `whisper.cpp-binaries-windows-x64-${buildKind}-latest`
 		} else {
-			throw new Error(`GPU builds (NVIDIA CUDA only) are currently only available as packages for Windows x64. Please specify a custom path to the binary in the 'executablePath' option.`)
+			throw new Error(`whisper.cpp GPU builds (NVIDIA CUDA only) are currently only available as packages for Windows x64. Please specify a custom path to a whisper.cpp 'main' binary in the 'executablePath' option.`)
 		}
 	} else if (buildKind === 'cpu') {
 		if (platform === 'win32' && arch === 'x64') {
-			packageName = `whisper.cpp-binaries-windows-x64-cpu-latest-patched`
+			packageName = `whisper.cpp-binaries-windows-x64-cpu-latest`
 		} else if (platform === 'linux' && arch === 'x64') {
-			packageName = `whisper.cpp-binaries-linux-x64-cpu-latest-patched`
+			packageName = `whisper.cpp-binaries-linux-x64-cpu-latest`
 		} else {
-			throw new Error(`Couldn't find a matching whisper.cpp binary package. Please specify a custom path to the binary in the 'executablePath' option.`)
+			throw new Error(`Couldn't find a matching whisper.cpp binary package. Please specify a custom path to a whisper.cpp 'main' binary in the 'executablePath' option.`)
 		}
 	} else {
-		throw new Error(`Unknown build kind '${buildKind}'`)
+		throw new Error(`Unsupported build kind '${buildKind}'`)
 	}
 
 	const packagePath = await loadPackage(packageName)
@@ -553,4 +566,6 @@ export type WhisperCppModelId =
 	'large-v2' |
 	'large-v2-q5_0' |
 	'large-v3' |
-	'large-v3-q5_0'
+	'large-v3-q5_0' |
+	`large-v3-turbo` |
+	`large-v3-turbo-q5_0`