echogarden 0.12.2 → 1.0.0

package/src/recognition/WhisperSTT.ts

@@ -1,17 +1,16 @@
 import Onnx from 'onnxruntime-node'
 
 import { Logger } from '../utilities/Logger.js'
-import { computeMelSpectogramUsingFilterbanks, Filterbank } from "../dsp/MelSpectogram.js"
-import { clip, splitFloat32Array, writeToStderr, yieldToEventLoop } from '../utilities/Utilities.js'
+import { computeMelSpectogramUsingFilterbanks, Filterbank } from '../dsp/MelSpectogram.js'
+import { clip, getIntegerRange, getRepetitionScoreRelativeToFirstSubstring, splitFloat32Array, yieldToEventLoop } from '../utilities/Utilities.js'
 import { indexOfMax, logOfVector, logSumExp, meanOfVector, medianFilter, softmax, stdDeviationOfVector } from '../math/VectorMath.js'
-import { isWordOrSymbolWord, splitToWords } from '../nlp/Segmentation.js'
 
 import { alignDTWWindowed } from '../alignment/DTWSequenceAlignmentWindowed.js'
-import { deepClone, extendDeep } from '../utilities/ObjectUtilities.js'
+import { extendDeep } from '../utilities/ObjectUtilities.js'
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js'
 import { AlignmentPath } from '../alignment/SpeechAlignment.js'
 import { getRawAudioDuration, RawAudio } from '../audio/AudioUtilities.js'
-import { readAndParseJsonFile, readFile } from '../utilities/FileSystem.js'
+import { readFile } from '../utilities/FileSystem.js'
 import path from 'path'
 import type { LanguageDetectionResults } from '../api/API.js'
 import { getShortLanguageCode, languageCodeToName } from '../utilities/Locale.js'
@@ -19,10 +18,11 @@ import { loadPackage } from '../utilities/PackageManager.js'
 import chalk from 'chalk'
 import { XorShift32RNG } from '../utilities/RandomGenerator.js'
 import { detectSpeechLanguageByParts } from '../api/LanguageDetection.js'
+import { type Tiktoken } from 'tiktoken/lite'
 
-export async function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, tokenizerDir: string, task: WhisperTask, sourceLanguage: string, options: WhisperOptions) {
+export async function recognize(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, task: WhisperTask, sourceLanguage: string, options: WhisperOptions) {
 	if (sourceRawAudio.sampleRate != 16000) {
-		throw new Error("Source audio must have a sampling rate of 16000")
+		throw new Error('Source audio must have a sampling rate of 16000')
 	}
 
 	sourceLanguage = getShortLanguageCode(sourceLanguage)
@@ -31,21 +31,30 @@ export async function recognize(sourceRawAudio: RawAudio, modelName: WhisperMode
 		throw new Error(`The language ${languageCodeToName(sourceLanguage)} is not supported by the Whisper engine.`)
 	}
 
-	if (!isMultiligualModel(modelName) && sourceLanguage != 'en') {
+	if (isEnglishOnlyModel(modelName) && sourceLanguage != 'en') {
 		throw new Error(`The model '${modelName}' can only be used with English inputs. However, the given source language was ${languageCodeToName(sourceLanguage)}.`)
 	}
 
-	const whisper = new Whisper(modelName, modelDir, tokenizerDir)
-	await whisper.initialize()
+	if (options.temperature && options.temperature < 0) {
+		throw new Error(`Temperature can't be negative`)
+	}
+
+	let seed = options.seed
+
+	if (seed) {
+		seed = Math.max(Math.floor(seed), 1) | 0
+	}
+
+	const whisper = new Whisper(modelName, modelDir, seed)
 
 	const result = await whisper.recognize(sourceRawAudio, task, sourceLanguage, options)
 
 	return result
 }
 
-export async function align(sourceRawAudio: RawAudio, referenceText: string, modelName: WhisperModelName, modelDir: string, tokenizerDir: string, sourceLanguage: string) {
+export async function align(sourceRawAudio: RawAudio, referenceText: string, modelName: WhisperModelName, modelDir: string, sourceLanguage: string) {
 	if (sourceRawAudio.sampleRate != 16000) {
-		throw new Error("Source audio must have a sampling rate of 16000")
+		throw new Error('Source audio must have a sampling rate of 16000')
 	}
 
 	sourceLanguage = getShortLanguageCode(sourceLanguage)
@@ -54,29 +63,35 @@ export async function align(sourceRawAudio: RawAudio, referenceText: string, mod
 		throw new Error(`The language ${languageCodeToName(sourceLanguage)} is not supported by the Whisper engine.`)
 	}
 
-	if (!isMultiligualModel(modelName) && sourceLanguage != 'en') {
+	if (isEnglishOnlyModel(modelName) && sourceLanguage != 'en') {
 		throw new Error(`The model '${modelName}' can only be used with English inputs. However, the given source language was ${languageCodeToName(sourceLanguage)}.`)
 	}
 
-	const whisper = new Whisper(modelName, modelDir, tokenizerDir)
-	await whisper.initialize()
+	const whisper = new Whisper(modelName, modelDir)
 
 	const timeline = await whisper.align(sourceRawAudio, referenceText, sourceLanguage)
 
 	return timeline
 }
 
-export async function detectLanguage(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, tokenizerDir: string) {
+export async function detectLanguage(sourceRawAudio: RawAudio, modelName: WhisperModelName, modelDir: string, temperature: number) {
 	if (sourceRawAudio.sampleRate != 16000) {
-		throw new Error("Source audio must have a sampling rate of 16000")
+		throw new Error('Source audio must have a sampling rate of 16000')
+	}
+
+	if (!isMultilingualModel(modelName)) {
+		throw new Error(`Language detection is only supported with multilingual models.`)
 	}
 
-	const whisper = new Whisper(modelName, modelDir, tokenizerDir)
-	await whisper.initialize()
+	if (temperature < 0) {
+		throw new Error(`Temperature cannot be negative`)
+	}
+
+	const whisper = new Whisper(modelName, modelDir)
 
 	async function detectLanguageForPart(partAudio: RawAudio) {
 		const audioFeatures = await whisper.encodeAudio(partAudio)
-		const partResults = await whisper.detectLanguage(audioFeatures)
+		const partResults = await whisper.detectLanguage(audioFeatures, temperature)
 
 		return partResults
 	}
@@ -91,17 +106,13 @@ export async function detectLanguage(sourceRawAudio: RawAudio, modelName: Whispe
 export class Whisper {
 	modelName: WhisperModelName
 	modelDir: string
-	tokenizerDir: string
 
 	isMultiligualModel: boolean
 
 	audioEncoder?: Onnx.InferenceSession
 	textDecoder?: Onnx.InferenceSession
 
-	textToTokenLookup = new Map<string, number>()
-	tokenToTextLookup = new Map<number, string>()
-
-	merges: [string, string][] = []
+	tiktoken?: Tiktoken
 
 	onnxOptions: Onnx.InferenceSession.SessionOptions = {
 		logSeverityLevel: 2,
@@ -109,109 +120,203 @@ export class Whisper {
 	}
 
 	tokenConfig: {
-		suppressedTokens: number[]
-		sotToken: number
-		sotPrevToken: number
-		eotToken: number
+		endOfTextToken: number
+		startOfTextToken: number
+
+		languageTokensStart: number
+		languageTokensEnd: number
+
+		translateTaskToken: number
+		transcribeTaskToken: number
+		startOfPromptToken: number
+		nonSpeechToken: number
 		noTimestampsToken: number
-		noSpeechToken: number
+
 		timestampTokensStart: number
+		timestampTokensEnd: number
+
+		suppressedTokens: Set<number>
 	}
 
-	randomGen = new XorShift32RNG(23948203)
+	randomGen: XorShift32RNG
 
-	constructor(modelName: WhisperModelName, modelDir: string, tokenizerDir: string) {
-		this.modelDir = modelDir
+	constructor(modelName: WhisperModelName, modelDir: string, rngSeed = 461845907) {
 		this.modelName = modelName
-		this.tokenizerDir = tokenizerDir
+		this.modelDir = modelDir
 
-		this.isMultiligualModel = isMultiligualModel(this.modelName)
+		this.isMultiligualModel = isMultilingualModel(this.modelName)
 
 		if (this.isMultiligualModel) {
 			this.tokenConfig = {
-				sotToken: 50258,
-				sotPrevToken: 50361,
-				eotToken: 50257,
-				noSpeechToken: 50362,
+				endOfTextToken: 50257,
+				startOfTextToken: 50258,
+
+				languageTokensStart: 50259,
+				languageTokensEnd: 50358,
+
+				translateTaskToken: 50358,
+				transcribeTaskToken: 50359,
+				startOfPromptToken: 50361,
+				nonSpeechToken: 50362,
 				noTimestampsToken: 50363,
+
 				timestampTokensStart: 50364,
-				suppressedTokens: [1, 2, 6, 7, 8, 9, 10, 12, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362]
+				timestampTokensEnd: 50364 + 1501,
+
+				suppressedTokens: new Set()
 			}
+
+			const suppressedTextTokens = [1, 2, 6, 7, 8, 9, 10, 12, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 359, 503, 522, 542, 873, 893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627, 3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647, 7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793, 14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675, 22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865, 42863, 47425, 49870, 50254]
+			const suppressedMetadataTokens = [50256, ...getIntegerRange(50258, 50364)]
+
+			const suppressedTokens = [...suppressedTextTokens, ...suppressedMetadataTokens]
+
+			this.tokenConfig.suppressedTokens = new Set(suppressedTokens)
 		} else {
 			this.tokenConfig = {
-				sotToken: 50257,
-				sotPrevToken: 50360,
-				eotToken: 50256,
-				noSpeechToken: 50361,
+				endOfTextToken: 50256,
+				startOfTextToken: 50257,
+
+				languageTokensStart: 50258,
+				languageTokensEnd: 50358,
+
+				translateTaskToken: 50358,
+				transcribeTaskToken: 50359,
+				startOfPromptToken: 50360,
+				nonSpeechToken: 50361,
 				noTimestampsToken: 50362,
+
 				timestampTokensStart: 50363,
-				suppressedTokens: [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, 1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211, 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361]
+				timestampTokensEnd: 50363 + 1501,
+
+				suppressedTokens: new Set()
 			}
+
+			const suppressedTextTokens = [1, 2, 7, 8, 9, 10, 14, 25, 26, 27, 28, 29, 31, 58, 59, 60, 61, 62, 63, 90, 91, 92, 93, 357, 366, 438, 532, 685, 705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377, 1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211, 4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786, 11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791, 17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409, 34949, 40283, 40493, 40549, 47282, 49146]
+			const suppressedMetadataTokens = [...getIntegerRange(50257, 50363)]
+
+			const suppressedTokens = [...suppressedTextTokens, ...suppressedMetadataTokens]
+
+			this.tokenConfig.suppressedTokens = new Set(suppressedTokens)
 		}
+
+		this.randomGen = new XorShift32RNG(rngSeed)
+	}
+
+	async initializeIfNeeded() {
+		await this.initializeTokenizerIfNeeded()
+		await this.initializeEncoderSessionIfNeeded()
+		await this.initializeDecoderSessionIfNeeded()
 	}
 
-	async initialize() {
+	async initializeTokenizerIfNeeded() {
+		if (this.tiktoken) {
+			return
+		}
+
 		const logger = new Logger()
-		await logger.startAsync("Load tokenizer data")
+		await logger.startAsync('Load tokenizer data')
+
+		const tiktokenModulePackagePath = await loadPackage('whisper-tiktoken-data')
 
-		const encoderFilePath = path.join(this.modelDir, "encoder.onnx")
-		const decoderFilePath = path.join(this.modelDir, "decoder.onnx")
+		const tiktokenDataFilePath = path.join(tiktokenModulePackagePath, this.isMultiligualModel ? 'multilingual.tiktoken' : 'gpt2.tiktoken')
+		let tiktokenData = await readFile(tiktokenDataFilePath, { encoding: 'utf8' })
 
-		const vocabFilePath = path.join(this.tokenizerDir, "vocab.json")
-		const mergesFilePath = path.join(this.tokenizerDir, "merges.txt")
+		const tokenConfig = this.tokenConfig
 
-		const vocabObject = await readAndParseJsonFile(vocabFilePath)
+		const metadataTokens: Record<number, string> = {
+			[tokenConfig.endOfTextToken]: '[EndOfText]',
+			[tokenConfig.startOfTextToken]: '[StartOfText]',
+			[tokenConfig.translateTaskToken]: '[TranslateTask]',
+			[tokenConfig.transcribeTaskToken]: '[TranscribeTask]',
+			[tokenConfig.startOfPromptToken]: '[StartOfPrompt]',
+			[tokenConfig.nonSpeechToken]: '[NonSpeech]',
+			[tokenConfig.noTimestampsToken]: '[NoTimestamps]',
+		}
 
-		function bpeEncodedStrToString(str: string) {
-			const decodedChars = []
+		if (this.isMultiligualModel) {
+			metadataTokens[50256] = '[Unused_50256]'
+			metadataTokens[50360] = '[Unused_50360]'
+		}
 
-			for (const char of str) {
-				const decodedChar = vocabCharacterSetLookup[char]
+		const languageTokenCount = tokenConfig.languageTokensEnd - tokenConfig.languageTokensStart
 
-				if (decodedChar == undefined) {
-					throw new Error(`Invalid char: '${char}'`)
-				}
+		for (let i = 0; i < languageTokenCount; i++) {
+			const tokenIndex = this.tokenConfig.languageTokensStart + i
 
-				decodedChars.push(decodedChar)
-			}
+			metadataTokens[tokenIndex] = `[Language_${i}]`
+		}
+
+		const timestampTokensCount = 1501
 
-			return Buffer.from(decodedChars).toString("utf-8")
+		for (let i = 0; i < timestampTokensCount; i++) {
+			const tokenIndex = this.tokenConfig.timestampTokensStart + i
+			const tokenTime = this.timestampTokenToSeconds(tokenIndex)
+
+			metadataTokens[tokenIndex] = `[Timestamp_${tokenTime.toFixed(2)}]`
+		}
+
+		const inverseMetadataTokensLookup: Record<string, number> = {}
+
+		for (const [key, value] of Object.entries(metadataTokens)) {
+			inverseMetadataTokensLookup[value] = parseInt(key)
 		}
 
-		for (const key in vocabObject) {
-			const value = vocabObject[key]
+		const patternString = `'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)|\\s+`
 
-			const decodedKey = bpeEncodedStrToString(key)
+		const { Tiktoken } = await import('tiktoken/lite')
+
+		this.tiktoken = new Tiktoken(tiktokenData, inverseMetadataTokensLookup, patternString)
+
+		logger.end()
+	}
 
-			this.textToTokenLookup.set(decodedKey, value)
-			this.tokenToTextLookup.set(value, decodedKey)
+	async initializeEncoderSessionIfNeeded() {
+		if (this.audioEncoder) {
+			return
 		}
 
-		const mergesFileRawLines = (await readFile(mergesFilePath, "utf8")).trim().split(/\r?\n/g)
-		const mergesFileRawEntries = mergesFileRawLines.map(line => line.trim().split(" "))
-		this.merges = mergesFileRawEntries.map(entry => [bpeEncodedStrToString(entry[0]), bpeEncodedStrToString(entry[1])])
+		const logger = new Logger()
+
+		await logger.startAsync(`Create encoder model inference session for model '${this.modelName}'`)
 
-		await logger.startAsync(`Create ONNX inference session for model '${this.modelName}'`)
+		const encoderFilePath = path.join(this.modelDir, 'encoder.onnx')
 
 		this.audioEncoder = await Onnx.InferenceSession.create(encoderFilePath, this.onnxOptions)
+
+		logger.end()
+	}
+
+	async initializeDecoderSessionIfNeeded() {
+		if (this.textDecoder) {
+			return
+		}
+
+		const logger = new Logger()
+
+		await logger.startAsync(`Create decoder model inference session for model '${this.modelName}'`)
+
+		const decoderFilePath = path.join(this.modelDir, 'decoder.onnx')
+
 		this.textDecoder = await Onnx.InferenceSession.create(decoderFilePath, this.onnxOptions)
 
 		logger.end()
 	}
 
 	async recognize(rawAudio: RawAudio, task: WhisperTask, language: string, options: WhisperOptions) {
-		const logger = new Logger()
+		await this.initializeIfNeeded()
 
-		const timestampTokensStart = this.tokenConfig.timestampTokensStart
+		const logger = new Logger()
 
 		const audioSamples = rawAudio.audioChannels[0]
 		const sampleRate = rawAudio.sampleRate
-		const audioDuration = getRawAudioDuration(rawAudio)
 		const prompt = options.prompt
 
-		const maxAudioSamples = sampleRate * 30
+		const maxAudioSamplesPerPart = sampleRate * 30
 
-		let previousPartTokens: number[] = []
+		const decodeTimestampTokens = options.decodeTimestampTokens!
+
+		let previousPartTextTokens: number[] = []
 
 		let timeline: Timeline = []
 		let allDecodedTokens: number[] = []
@@ -221,7 +326,7 @@ export class Whisper {
 
 			await logger.startAsync(`\nPrepare audio part at time position ${segmentStartTime.toFixed(2)}`, undefined, chalk.magentaBright)
 
-			const audioPartSamples = audioSamples.slice(audioOffset, audioOffset + maxAudioSamples)
+			const audioPartSamples = audioSamples.slice(audioOffset, audioOffset + maxAudioSamplesPerPart)
 			const audioPartRawAudio: RawAudio = { audioChannels: [audioPartSamples], sampleRate }
 			const audioPartDuration = getRawAudioDuration(audioPartRawAudio)
 
@@ -229,77 +334,73 @@ export class Whisper {
 
 			const audioPartFeatures = await this.encodeAudio(audioPartRawAudio)
 
-			const isFirstPart = audioOffset == 0
-			const isFinalPart = audioOffset + maxAudioSamples > audioSamples.length
+			const isFirstPart = audioOffset === 0
+			const isFinalPart = audioOffset + maxAudioSamplesPerPart >= audioSamples.length
 
 			let initialTokens: number[] = []
 
 			if (isFirstPart && prompt) {
-				const promptTokens = await this.textToTokens(prompt, language)
+				const promptTokens = this.textToTokens(prompt)
 
-				initialTokens = [this.tokenConfig.sotPrevToken, ...promptTokens]
-			} else if (options.autoPromptParts && previousPartTokens.length > 0) {
-				initialTokens = [this.tokenConfig.sotPrevToken, ...previousPartTokens]
+				initialTokens = [this.tokenConfig.startOfPromptToken, ...promptTokens]
+			} else if (options.autoPromptParts && previousPartTextTokens.length > 0) {
+				initialTokens = [this.tokenConfig.startOfPromptToken, ...previousPartTextTokens]
 			}
 
-			initialTokens = [...initialTokens, ...this.getInitialTokens(language, task)]
+			initialTokens = [...initialTokens, ...this.getTextStartTokens(language, task, !decodeTimestampTokens)]
 
 			logger.end()
 
-			let { decodedTokens: partTokens, crossAttentionQKs: partCrossAttentionQKs, decodedTokensConfidence } = await this.decodeTokens(audioPartFeatures, initialTokens, audioPartDuration, isFirstPart, isFinalPart, options)
+			let {
+				decodedTokens: partTokens,
+				crossAttentionQKs: partCrossAttentionQKs,
+				decodedTokensConfidence
+			} = await this.decodeTokens(audioPartFeatures, initialTokens, audioPartDuration, isFirstPart, isFinalPart, options)
 
 			const lastToken = partTokens[partTokens.length - 1]
-			const lastTokenIsTimestamp = lastToken >= timestampTokensStart
+			const lastTokenIsTimestamp = this.isTimestampToken(lastToken)
 
 			let audioEndOffset: number
 
 			if (!isFinalPart && lastTokenIsTimestamp) {
-				const timePosition = (lastToken - timestampTokensStart) * 0.02
+				const timePosition = this.timestampTokenToSeconds(lastToken)
 
 				audioEndOffset = audioOffset + Math.floor(timePosition * sampleRate)
 			} else {
-				audioEndOffset = Math.min(audioOffset + maxAudioSamples, audioSamples.length)
+				audioEndOffset = Math.min(audioOffset + maxAudioSamplesPerPart, audioSamples.length)
 			}
 
 			const segmentEndTime = audioEndOffset / sampleRate
-			const segmentFrameCount = Math.floor((segmentEndTime - segmentStartTime) / 0.02)
+			const segmentFrameCount = this.secondsRangeToFrameCount(segmentStartTime, segmentEndTime)
 
 			await logger.startAsync(`Extract timeline for part`)
 
 			if (partTokens.length != partCrossAttentionQKs.length) {
-				throw new Error("Unexpected: partTokens.length != partCrossAttentionQKs.length")
+				throw new Error('Unexpected: partTokens.length != partCrossAttentionQKs.length')
 			}
 
-			//partTokens = partTokens.filter(token => token < timestampTokensStart)
-			//partCrossAttentionQKs = await this.inferCrossAttentionQKs(partTokens, audioPartFeatures)
-
 			partTokens = partTokens.slice(initialTokens.length)
-			partCrossAttentionQKs = partCrossAttentionQKs.slice(initialTokens.length)
 
-			//await this.addWordsToTimeline(timeline, partTokens, audioPartRawAudio, partCrossAttentionQKs, initialAudioTimeOffset, audioPartSamples.length / sampleRate)
+			//const compressionRatioForPart = (await getDeflateCompressionMetricsForString(this.tokensToText(partTokens))).ratio
 
-			const alignmentPath = await this.findAlignmentPathFromQKs(partCrossAttentionQKs, partTokens, 0, segmentFrameCount) //, alignmentHeadsIndexes[this.modelName])
-			const partTimeline = await this.getWordTimelineFromAlignmentPath(alignmentPath, partTokens, segmentStartTime, segmentEndTime, decodedTokensConfidence)
+			partCrossAttentionQKs = partCrossAttentionQKs.slice(initialTokens.length)
 
-			timeline.push(...partTimeline)
+			const alignmentPath = await this.findAlignmentPathFromQKs(partCrossAttentionQKs, partTokens, 0, segmentFrameCount) //, alignmentHeadsIndexes[this.modelName])
+			const partTimeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, partTokens, segmentStartTime, segmentEndTime, decodedTokensConfidence)
 
 			audioOffset = audioEndOffset
 
-			previousPartTokens = partTokens.filter(token => token < this.tokenConfig.eotToken)
+			allDecodedTokens.push(...partTokens)
+			timeline.push(...partTimeline)
 
-			allDecodedTokens.push(...previousPartTokens)
+			previousPartTextTokens = partTokens.filter(token => this.isTextToken(token))
 
 			logger.end()
 		}
 
-		if (timeline.length > 0) {
-			timeline[timeline.length - 1].endTime = audioDuration
-		}
+		timeline = this.tokenTimelineToWordTimeline(timeline)
 
-		timeline = this.mergeSuccessiveWordFragmentsInTimeline(timeline)
-		timeline.forEach(entry => { entry.text = entry.text.trim() })
-
-		const transcript = this.tokensToText(allDecodedTokens)
+		const transcript = this.tokensToText(allDecodedTokens).trim()
 
 		logger.end()
 
@@ -307,51 +408,56 @@ export class Whisper {
 	}
 
 	async align(rawAudio: RawAudio, referenceText: string, language: string) {
+		await this.initializeIfNeeded()
+
 		const logger = new Logger()
 
-		await logger.startAsync("Prepare for alignment")
+		await logger.startAsync('Prepare for alignment')
+
+		referenceText = referenceText.replaceAll(/\s+/g, ' ')
+
 		const audioDuration = Math.min(getRawAudioDuration(rawAudio), 30)
-		const audioFrameCount = Math.floor(audioDuration / 0.02)
+		const audioFrameCount = this.secondsToFrame(audioDuration)
 
-		const initialTokens = this.getInitialTokens(language, "transcribe", true)
-		const timestampTokensStart = this.tokenConfig.timestampTokensStart
-		const eotToken = this.tokenConfig.eotToken
+		const initialTokens = this.getTextStartTokens(language, 'transcribe', true)
 
-		let tokens = [...initialTokens, ...await this.textToTokens(referenceText, language), eotToken]
+		const endOfTextToken = this.tokenConfig.endOfTextToken
+
+		let tokens = [...initialTokens, ...this.textToTokens(referenceText), endOfTextToken]
 
 		logger.end()
 		const audioFeatures = await this.encodeAudio(rawAudio)
 
-		await logger.startAsync("Infer cross-attention QKs")
+		await logger.startAsync('Infer cross-attention QKs')
 		let crossAttentionQKs = await this.inferCrossAttentionQKs(tokens, audioFeatures)
 
 		tokens = tokens.slice(initialTokens.length, tokens.length - 1)
 		crossAttentionQKs = crossAttentionQKs.slice(initialTokens.length, crossAttentionQKs.length - 1)
 
-		await logger.startAsync("Extract word timeline")
+		await logger.startAsync('Extract word timeline')
 		const alignmentPath = await this.findAlignmentPathFromQKs(crossAttentionQKs, tokens, 0, audioFrameCount)//, this.getAlignmentHeadIndexes())
-		let timeline = await this.getWordTimelineFromAlignmentPath(alignmentPath, tokens, 0, audioDuration)
+		let timeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, tokens, 0, audioDuration)
 
-		timeline = this.mergeSuccessiveWordFragmentsInTimeline(timeline)
-		timeline.forEach(entry => { entry.text = entry.text.trim() })
-		//timeline = timeline.filter(entry => isWordOrSymbolWord(entry.text))
+		timeline = this.tokenTimelineToWordTimeline(timeline)
 
 		logger.end()
 
 		return timeline
 	}
 
-	async detectLanguage(audioFeatures: Onnx.Tensor): Promise<LanguageDetectionResults> {
-		const logger = new Logger()
-
+	async detectLanguage(audioFeatures: Onnx.Tensor, temperature: number): Promise<LanguageDetectionResults> {
 		if (!this.isMultiligualModel) {
-			throw new Error("Language detection only works for a multilingual model")
+			throw new Error('Language detection is only supported with multilingual models')
 		}
 
+		await this.initializeTokenizerIfNeeded()
+		await this.initializeDecoderSessionIfNeeded()
+
 		// Prepare and run decoder
-		await logger.startAsync("Detect language with Whisper model")
+		const logger = new Logger()
+		await logger.startAsync('Detect language with Whisper model')
 
-		const sotToken = this.tokenConfig.sotToken
+		const sotToken = this.tokenConfig.startOfTextToken
 
 		const initialTokens = [sotToken]
 		const offset = 0
@@ -362,13 +468,20 @@ export class Whisper {
 		const tokensTensor = new Onnx.Tensor('int64', new BigInt64Array(initialTokens.map(token => BigInt(token))), [1, initialTokens.length])
 		const offsetTensor = new Onnx.Tensor('int64', new BigInt64Array([BigInt(offset)]), [])
 
-		const decoderInputs = { tokens: tokensTensor, audio_features: audioFeatures, kv_cache: kvCacheTensor, offset: offsetTensor }
+		const decoderInputs = {
+			tokens: tokensTensor,
+			audio_features: audioFeatures,
+			kv_cache: kvCacheTensor,
+			offset: offsetTensor
+		}
 
 		const decoderOutputs = await this.textDecoder!.run(decoderInputs)
-		const logitsBuffer = decoderOutputs["logits"].data as Float32Array
+		const logitsBuffer = decoderOutputs['logits'].data as Float32Array
 
-		const languageTokensLogits = Array.from(logitsBuffer.slice(sotToken + 1, sotToken + 1 + 99))
-		const languageTokensProbabilities = softmax(languageTokensLogits, 1.0)
+		const tokenConfig = this.tokenConfig
+
+		const languageTokensLogits = Array.from(logitsBuffer.slice(tokenConfig.languageTokensStart, tokenConfig.languageTokensEnd))
+		const languageTokensProbabilities = softmax(languageTokensLogits, temperature)
 
 		const results: LanguageDetectionResults = []
 
@@ -388,24 +501,33 @@ export class Whisper {
 		return results
 	}
 
-	async decodeTokens(audioFeatures: Onnx.Tensor, initialTokens: number[], audioDuration: number, isFirstPart: boolean, isFinalPart: boolean, options: WhisperOptions) {
+	async decodeTokens(
+		audioFeatures: Onnx.Tensor,
+		initialTokens: number[],
+		audioDuration: number,
+		isFirstPart: boolean,
+		isFinalPart: boolean,
+		options: WhisperOptions) {
+
+		await this.initializeTokenizerIfNeeded()
+		await this.initializeDecoderSessionIfNeeded()
+
 		const logger = new Logger()
-		await logger.startAsync("Decode text tokens with Whisper decoder model")
 
-		options = extendDeep(whisperOptionsDefaults, options)
+		await logger.startAsync('Decode text tokens with Whisper decoder model')
 
-		const noSpeechThreshold = 0.6
+		options = extendDeep(defaultWhisperOptions, options)
 
-		const blankToken = this.textToTokenLookup.get(" ")
+		const endOfTextToken = this.tokenConfig.endOfTextToken
 
-		const suppressedTokens = this.tokenConfig.suppressedTokens
-		const sotToken = this.tokenConfig.sotToken
-		const eotToken = this.tokenConfig.eotToken
-		const noTimestampsToken = this.tokenConfig.noTimestampsToken
-		const noSpeechToken = this.tokenConfig.noSpeechToken
 		const timestampTokensStart = this.tokenConfig.timestampTokensStart
+		const suppressedTokens = this.tokenConfig.suppressedTokens
 
-		const maxDecodedTokenCount = 250
+		const spaceToken = this.textToTokens(' ')[0]
+
+		const maxDecodedTokenCount = options.maxTokensPerPart!
+
+		//const suppressedTokensText = Array.from(this.tokenConfig.suppressedTokens).map(token => this.tokenToText(token, true))
 
 		let decodedTokens = initialTokens.slice()
 		const initialKvDimensions = this.getKvDimensions(1, decodedTokens.length)
@@ -455,147 +577,196 @@ export class Whisper {
 
 			const decoderOutputs = await this.textDecoder!.run(decoderInputs)
 
-			const logitsBuffer = decoderOutputs["logits"].data as Float32Array
-			kvCacheTensor = decoderOutputs["output_kv_cache"] as any
+			const logitsBuffer = decoderOutputs['logits'].data as Float32Array
+			kvCacheTensor = decoderOutputs['output_kv_cache'] as any
 
 			// Compute logits
-			const resultLogits = splitFloat32Array(logitsBuffer, logitsBuffer.length / decoderOutputs["logits"].dims[1])
-			const tokenLogits = resultLogits[resultLogits.length - 1]
-			const tokenTimestampLogits = Array.from(tokenLogits.slice(timestampTokensStart))
+			const resultLogits = splitFloat32Array(logitsBuffer, logitsBuffer.length / decoderOutputs['logits'].dims[1])
+			const allTokenLogits = Array.from(resultLogits[resultLogits.length - 1])
+			const timestampTokenLogits = allTokenLogits.slice(timestampTokensStart)
 
 			// Suppress tokens
-			for (let logitIndex = 0; logitIndex < tokenLogits.length; logitIndex++) {
-				const isWrongTokenForInitialState = isInitialState && (logitIndex == blankToken || logitIndex == eotToken)
-				const isInSupressedList = suppressedTokens.includes(logitIndex)
-				const isNoTimestampsToken = logitIndex == noTimestampsToken
+			for (let logitIndex = 0; logitIndex < allTokenLogits.length; logitIndex++) {
+				const isWrongTokenForInitialState =
+					isInitialState &&
+					(logitIndex === spaceToken || logitIndex === endOfTextToken)
 
-				const shouldSupressToken = isWrongTokenForInitialState || isInSupressedList || isNoTimestampsToken
+				const isInSuppressedList = suppressedTokens.has(logitIndex)
 
-				if (shouldSupressToken) {
-					tokenLogits[logitIndex] = -Infinity
+				const shouldSuppressToken = isWrongTokenForInitialState || isInSuppressedList
+
+				if (shouldSuppressToken) {
+					allTokenLogits[logitIndex] = -Infinity
 				}
 			}
 
-			// Find token distributions and best token
-			const probs = softmax(tokenLogits as any)
-			const logProbs = logOfVector(probs)
+			// Derive token probabilities
+			let bufferedTokensToPrint: number[] = []
+
+			// Add best token
+			function addToken(tokenToAdd: number, timestampLogits: number[], confidence: number) {
+				decodedTokens.push(tokenToAdd)
+				decodedTokensTimestampLogits.push(timestampLogits)
+				decodedTokensCrossAttentionQKs.push(decoderOutputs['cross_attention_qks'])
+				decodedTokensConfidence.push(confidence)
+			}
+
+			let shouldDecodeNonTimestampToken = true
 
-			const textTokenLogProbs = logProbs.slice(0, timestampTokensStart)
-			const timestampTokenLogProbs = logProbs.slice(timestampTokensStart)
+			if (options.decodeTimestampTokens) {
+				const probabilities = softmax(allTokenLogits as any, 1.0)
+				const logProbabilities = logOfVector(probabilities)
 
-			const indexOfMaxTextLogProb = indexOfMax(textTokenLogProbs)
-			const valueOfMaxTextLogProb = textTokenLogProbs[indexOfMaxTextLogProb]
+				const nonTimestampTokenLogProbs = logProbabilities.slice(0, timestampTokensStart)
 
-			const indexOfMaxTimestampLogProb = indexOfMax(timestampTokenLogProbs)
+				const indexOfMaxNonTimestampLogProb = indexOfMax(nonTimestampTokenLogProbs)
+				const valueOfMaxNonTimestampLogProb = nonTimestampTokenLogProbs[indexOfMaxNonTimestampLogProb]
 
-			const logSumExpOfTimestampTokenLogProbs = logSumExp(timestampTokenLogProbs)
+				const timestampTokenLogProbs = logProbabilities.slice(timestampTokensStart)
+				const indexOfMaxTimestampLogProb = indexOfMax(timestampTokenLogProbs)
 
-			const isTimestampToken = logSumExpOfTimestampTokenLogProbs > valueOfMaxTextLogProb
-			const previousTokenWasTimestamp = decodedTokens[decodedTokens.length - 1] >= timestampTokensStart
-			const secondPreviousTokenWasTimestamp = decodedTokens.length < 2 || decodedTokens[decodedTokens.length - 2] >= timestampTokensStart
+				const logSumExpOfTimestampTokenLogProbs = logSumExp(timestampTokenLogProbs)
 
-			if (isTimestampToken && !previousTokenWasTimestamp) {
-				timestampsSeenCount += 1
-			}
+				const shouldDecodeTimestampToken = logSumExpOfTimestampTokenLogProbs > valueOfMaxNonTimestampLogProb
 
-			//
-			//const topLogits = [...tokenLogits].map((logit, index) => ({ index, logit, token: this.tokenToTextLookup.get(index) || "", prob: probs[index] }))
-			//topLogits.sort((a, b) => b.logit - a.logit)
-			///
+				const previousTokenWasTimestamp = this.isTimestampToken(decodedTokens[decodedTokens.length - 1])
+				const secondPreviousTokenWasTimestamp = decodedTokens.length < 2 || this.isTimestampToken(decodedTokens[decodedTokens.length - 2])
 
-			// Add best token
-			function addToken(tokenToAdd: number, timestampLogits: number[], confidence: number) {
-				decodedTokens.push(tokenToAdd)
-				decodedTokensTimestampLogits.push(timestampLogits)
-				decodedTokensCrossAttentionQKs.push(decoderOutputs["cross_attention_qks"])
-				decodedTokensConfidence.push(confidence)
-			}
+				if (shouldDecodeTimestampToken && !previousTokenWasTimestamp) {
+					timestampsSeenCount += 1
+				}
 
-			if (isTimestampToken || (previousTokenWasTimestamp && !secondPreviousTokenWasTimestamp)) {
-				if (previousTokenWasTimestamp) {
-					const previousToken = decodedTokens[decodedTokens.length - 1]
-					const previousTokenTimestampLogits = decodedTokensTimestampLogits[decodedTokensTimestampLogits.length - 1]
-					const previousTokenConfidence = decodedTokensConfidence[decodedTokensConfidence.length - 1]
+				if (shouldDecodeTimestampToken || (previousTokenWasTimestamp && !secondPreviousTokenWasTimestamp)) {
+					if (previousTokenWasTimestamp) {
+						const previousToken = decodedTokens[decodedTokens.length - 1]
+						const previousTokenTimestampLogits = decodedTokensTimestampLogits[decodedTokensTimestampLogits.length - 1]
+						const previousTokenConfidence = decodedTokensConfidence[decodedTokensConfidence.length - 1]
+
+						addToken(previousToken, previousTokenTimestampLogits, previousTokenConfidence)
 
-					addToken(previousToken, previousTokenTimestampLogits, previousTokenConfidence)
+						lastTimestampTokenIndex = decodedTokens.length
 
-					lastTimestampTokenIndex = decodedTokens.length
+						const previousTokenTimestamp = this.timestampTokenToSeconds(previousToken)
 
-					const previousTokenTimestamp = (previousToken - timestampTokensStart) * 0.02
+						if (previousTokenTimestamp >= audioDuration) {
+							break
+						}
+					} else {
+						const timestampToken = timestampTokensStart + indexOfMaxTimestampLogProb
+						const confidence = probabilities[timestampToken]
 
-					if (previousTokenTimestamp >= audioDuration) {
-						break
+						addToken(timestampToken, timestampTokenLogits, confidence)
 					}
-				} else {
-					const timestampToken = timestampTokensStart + indexOfMaxTimestampLogProb
-					const confidence = probs[timestampToken]
 
-					addToken(timestampToken, tokenTimestampLogits, confidence)
+					shouldDecodeNonTimestampToken = false
 				}
-			} else if (indexOfMaxTextLogProb == eotToken) {
-				break
-			} else {
-				let chosenTokenIndex: number
+			}
 
-				if (options.temperature == 0.0) {
-					chosenTokenIndex = indexOfMaxTextLogProb
-				} else {
-					const topLogitCount = options.topCandidateCount!
-
-					const textTokenLogits = tokenLogits.slice(0, timestampTokensStart)
-					const sortedTextTokenLogitsWithIndexes = Array.from(textTokenLogits).map((logit, index) => ({ logit, index }))
-					sortedTextTokenLogitsWithIndexes.sort((a, b) => b.logit - a.logit)
-					let topLogitsWithIndexes = sortedTextTokenLogitsWithIndexes.slice(0, topLogitCount)
-
-					////
-					/*
-					topLogitsWithIndexes = topLogitsWithIndexes.filter(entry => {
-						const lastDecodedTextTokens = decodedTokens.filter(token => token < eotToken).reverse().slice(0, 20)
-						const { maxScore } = getRepetitionScoreRelativeToFirstSubstring([entry.index, ...lastDecodedTextTokens])
-
-						if (maxScore < 4) {
-							return true
-						} else {
-							return false
-						}
-					})
-					*/
-					////
+			if (shouldDecodeNonTimestampToken) {
+				const topLogitCount = options.topCandidateCount!
 
-					const topLogits = topLogitsWithIndexes.map(a => a.logit)
-					const textTokenProbs = softmax(topLogits, options.temperature)
+				const nonTimestampTokenLogits = allTokenLogits.slice(0, timestampTokensStart)
 
-					const topIndexOfPromisingPunctuationLogit = topLogitsWithIndexes.findIndex(entry => {
-						const tokenText = (this.tokenToTextLookup.get(entry.index) || "").trim()
-						const tokenProb = probs[entry.index]
+				const sortedNonTimestampTokenLogitsWithIndexes =
+					Array.from(nonTimestampTokenLogits).map((logit, index) => ({ token: index, logit }))
 
-						return tokenProb >= options.punctuationThreshold! && [',', '，', '.', '。', '!', '?'].includes(tokenText)
+				sortedNonTimestampTokenLogitsWithIndexes.sort((a, b) => b.logit - a.logit)
+
+				let topCandidates = sortedNonTimestampTokenLogitsWithIndexes.slice(0, topLogitCount)
+					.map(entry => ({
+						token: entry.token,
+						logit: entry.logit,
+						text: this.tokenToText(entry.token, true)
+					}))
+
+				//// Repetition suppression code
+				if (options.suppressRepetition) {
+					const topCandidatesRepetitionScores = topCandidates.map(entry => {
+						const lastDecodedTextTokens = decodedTokens.filter(token => this.isTextToken(token)).reverse().slice(0, 20)
+						const { maxScore } = getRepetitionScoreRelativeToFirstSubstring([entry.token, ...lastDecodedTextTokens])
+
+						return maxScore
 					})
 
-					let chosenTokenIndexInTopLogits: number
+					const thresholdRepetitionScore = 3
 
-					if (topIndexOfPromisingPunctuationLogit >= 0) {
-						chosenTokenIndexInTopLogits = topIndexOfPromisingPunctuationLogit
+					if (topCandidatesRepetitionScores.every(score => score >= thresholdRepetitionScore)) {
+						const indexOfMaxScore = topCandidatesRepetitionScores.indexOf(Math.max(...topCandidatesRepetitionScores))
+						topCandidates = [topCandidates[indexOfMaxScore]]
 					} else {
-						chosenTokenIndexInTopLogits = this.randomGen.selectRandomIndexFromDistribution(textTokenProbs)
+						topCandidates = topCandidates.filter((candidate, index) => topCandidatesRepetitionScores[index] < thresholdRepetitionScore)
 					}
+				}
+				////
+
+				const topCandidateProbabilities = softmax(topCandidates.map(a => a.logit), options.temperature)
+
+				//// Remove end-of-text token from candidates if its probability isn't high enough
+				if (options.decodeTimestampTokens === false) {
+					topCandidates = topCandidates.filter((candidate, index) => {
+						if (candidate.token === endOfTextToken) {
+							return topCandidateProbabilities[index] >= 0.9
+						}
 
-					chosenTokenIndex = sortedTextTokenLogitsWithIndexes[chosenTokenIndexInTopLogits].index
+						return true
+					})
 				}
+				////
+
+				const rankOfPromisingPunctuationToken = topCandidates.findIndex((entry, index) => {
+					const tokenText = this.tokenToText(entry.token).trim()
 
-				if (chosenTokenIndex < eotToken) {
-					let chosenTokenText = this.tokenToTextLookup.get(chosenTokenIndex) || ""
+					const isPunctuationToken = [',', '，', '、', '.', '。', '!', '?'].includes(tokenText)
 
-					if (isFirstPart && decodedTokens.every(token => token >= eotToken)) {
-						chosenTokenText = chosenTokenText.trimStart()
+					if (!isPunctuationToken) {
+						return false
 					}
 
-					writeToStderr(chosenTokenText)
+					const tokenProb = topCandidateProbabilities[index]
+
+					return tokenProb >= options.punctuationThreshold!
+				})
+
+				let rankOfSpaceToken = topCandidates.findIndex(candidate => candidate.token === spaceToken)
+
+				if (rankOfSpaceToken < 0) {
+					rankOfSpaceToken = Infinity
+				}
+
+				let chosenCandidateRank: number
+
+				if (rankOfPromisingPunctuationToken >= 0 &&
+					rankOfPromisingPunctuationToken < rankOfSpaceToken) {
+					chosenCandidateRank = rankOfPromisingPunctuationToken
+				} else {
+					chosenCandidateRank = this.randomGen.selectRandomIndexFromDistribution(topCandidateProbabilities)
 				}
 
-				const confidence = probs[chosenTokenIndex]
+				const chosenToken = topCandidates[chosenCandidateRank].token
+
+				if (this.isTextToken(chosenToken)) {
+					bufferedTokensToPrint.push(chosenToken)
+
+					let textToPrint = this.tokensToText(bufferedTokensToPrint)
+
+					if (textToPrint.codePointAt(0) !== 65533) {
+						if (isFirstPart && decodedTokens.every(token => this.isMetadataToken(token))) {
+							textToPrint = textToPrint.trimStart()
+						}
 
-				addToken(chosenTokenIndex, tokenTimestampLogits, confidence)
+						logger.write(textToPrint)
+
+						bufferedTokensToPrint = []
+					}
+				}
+
+				const confidence = topCandidateProbabilities[chosenCandidateRank]
+
+				addToken(chosenToken, timestampTokenLogits, confidence)
+
+				if (chosenToken === endOfTextToken) {
+					break
+				}
 			}
 
 			await yieldToEventLoop()
@@ -607,11 +778,16 @@ export class Whisper {
 			decodedTokensCrossAttentionQKs = decodedTokensCrossAttentionQKs.slice(0, lastTimestampTokenIndex)
 		}
 
-		writeToStderr("\n")
+		logger.write('\n')
 		logger.end()
 
 		// Return the tokens
-		return { decodedTokens, decodedTokensTimestampLogits, crossAttentionQKs: decodedTokensCrossAttentionQKs, decodedTokensConfidence }
+		return {
+			decodedTokens,
+			decodedTokensTimestampLogits,
+			crossAttentionQKs: decodedTokensCrossAttentionQKs,
+			decodedTokensConfidence
+		}
 	}
 
 	async inferCrossAttentionQKs(tokens: number[], audioFeatures: Onnx.Tensor) {
@@ -627,7 +803,7 @@ export class Whisper {
 
 		const decoderOutputs = await this.textDecoder!.run(decoderInputs)
 
-		const crossAttentionQKsTensor = decoderOutputs["cross_attention_qks"]
+		const crossAttentionQKsTensor = decoderOutputs['cross_attention_qks']
 
 		const tensorShape = crossAttentionQKsTensor.dims.slice()
 
@@ -663,6 +839,8 @@ export class Whisper {
 	}
 
 	async encodeAudio(rawAudio: RawAudio) {
+		await this.initializeEncoderSessionIfNeeded()
+
 		const logger = new Logger()
 
 		const audioSamples = rawAudio.audioChannels[0]
@@ -675,7 +853,7 @@ export class Whisper {
 		const maxAudioSamples = sampleRate * 30
 		const maxAudioFrames = 3000
 
-		await logger.startAsync("Extract mel spectogram from audio part")
+		await logger.startAsync('Extract mel spectogram from audio part')
 
 		const paddedAudioSamples = new Float32Array(maxAudioSamples)
 		paddedAudioSamples.set(audioSamples.subarray(0, maxAudioSamples), 0)
@@ -684,7 +862,7 @@ export class Whisper {
 
 		const { melSpectogram } = await computeMelSpectogramUsingFilterbanks(rawAudioPart, fftOrder, fftOrder, hopLength, filterbanks)
 
-		await logger.startAsync("Normalize mel spectogram")
+		await logger.startAsync('Normalize mel spectogram')
 
 		const logMelSpectogram = melSpectogram.map(spectrum => spectrum.map(mel => Math.log10(Math.max(mel, 1e-10))))
 		let maxLogMel = -Infinity
@@ -708,14 +886,14 @@ export class Whisper {
 			}
 		}
 
-		await logger.startAsync("Encode mel spectogram with Whisper encoder model")
+		await logger.startAsync('Encode mel spectogram with Whisper encoder model')
 
 		const inputTensor = new Onnx.Tensor('float32', flattenedNormalizedLogMelSpectogram, [1, filterbankCount, maxAudioFrames])
 
 		const encoderInputs = { mel: inputTensor }
 
 		const encoderOutputs = await this.audioEncoder!.run(encoderInputs)
-		const encodedAudioFeatures = encoderOutputs["output"]
+		const encodedAudioFeatures = encoderOutputs['output']
 
 		logger.end()
 
@@ -728,7 +906,7 @@ export class Whisper {
 		for (let i = 0; i < tokens.length; i++) {
 			const token = tokens[i]
 
-			if (token == this.tokenConfig.sotToken || token == this.tokenConfig.eotToken) {
+			if (token == this.tokenConfig.startOfTextToken || token == this.tokenConfig.endOfTextToken) {
 				continue
 			}
 
@@ -740,7 +918,7 @@ export class Whisper {
 					continue
 				}
 
-				let startTime = initialTimeOffset + (token - timestampTokensStart) * 0.02
+				let startTime = initialTimeOffset + this.timestampTokenToSeconds(token)
 
 				startTime = Math.min(startTime, audioDuration)
 
@@ -749,22 +927,22 @@ export class Whisper {
 				}
 
 				timeline.push({
-					type: "segment",
-					text: "",
+					type: 'segment',
+					text: '',
 					startTime,
 					endTime: -1,
 				})
 			} else {
 				if (timeline.length == 0) {
 					timeline.push({
-						type: "segment",
-						text: "",
+						type: 'segment',
+						text: '',
 						startTime: initialTimeOffset,
 						endTime: -1,
 					})
 				}
 
-				const tokenText = this.tokenToTextLookup.get(token) || ""
+				const tokenText = this.tokenToText(token)
 
 				timeline[timeline.length - 1].text += tokenText
 			}
@@ -772,8 +950,6 @@ export class Whisper {
 	}
 
 	async addWordsToTimeline(timeline: Timeline, tokens: number[], rawAudio: RawAudio, crossAttentionQKs: Onnx.Tensor[], initialAudioTimeOffset: number, duration: number) {
-		const timestampTokensStart = this.tokenConfig.timestampTokensStart
-
 		let segmentStartTime = 0
 		let segmentTokens: number[] = []
 		let segmentCrossAttentionQKs: Onnx.Tensor[] = []
@@ -782,15 +958,15 @@ export class Whisper {
 			const token = tokens[tokenIndex]
 			const tokenCrossAttentionQKs = crossAttentionQKs[tokenIndex]
 
-			const segmentTokensWithoutTimestamps = segmentTokens.filter(token => token < this.tokenConfig.timestampTokensStart)
+			const segmentTokensWithoutTimestamps = segmentTokens.filter(token => this.isNonTimestampToken(token))
 
-			const isTimestamp = token >= timestampTokensStart
+			const isTimestamp = this.isTimestampToken(token)
 
 			if (isTimestamp || tokenIndex == tokens.length - 1) {
 				let tokenTime: number
 
 				if (isTimestamp) {
-					tokenTime = (token - timestampTokensStart) * 0.02
+					tokenTime = this.timestampTokenToSeconds(token)
 				} else {
 					tokenTime = duration
 				}
@@ -798,8 +974,8 @@ export class Whisper {
 				if (segmentTokensWithoutTimestamps.length > 0) {
 					const segmentEndTime = tokenTime
 
-					const segmentStartFrame = Math.floor(segmentStartTime / 0.02)
-					let segmentEndFrame = Math.floor(segmentEndTime / 0.02)
+					const segmentStartFrame = this.secondsToFrame(segmentStartTime)
+					let segmentEndFrame = this.secondsToFrame(segmentEndTime)
 
 					if (segmentStartFrame == segmentEndFrame) {
 						segmentEndFrame += 1
@@ -810,7 +986,7 @@ export class Whisper {
 					const reinferCrossAttentionQKs = true
 
 					if (reinferCrossAttentionQKs) {
-						const initialTokens = this.getInitialTokens('en', 'transcribe')
+						const initialTokens = this.getTextStartTokens('en', 'transcribe')
 						const tokensToDecode = [...initialTokens, ...segmentTokensWithoutTimestamps]
 
 						//const segmentAudioFeaturesBuffer = audioFeatures.data.slice(segmentStartFrame * audioFeatures.dims[2], segmentEndFrame * audioFeatures.dims[2])
@@ -825,14 +1001,14 @@ export class Whisper {
 						reinferredCrossAttentionQKs.slice(initialTokens.length)
 
 						const alignmentPath = await this.findAlignmentPathFromQKs(reinferredCrossAttentionQKs, tokensToDecode, 0, segmentFrameCount)//, alignmentHeadsIndexes[modelName])
-						const wordTimeline = await this.getWordTimelineFromAlignmentPath(alignmentPath, segmentTokensWithoutTimestamps, initialAudioTimeOffset + segmentStartTime, initialAudioTimeOffset + segmentEndTime)
+						const tokenTimeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, segmentTokensWithoutTimestamps, initialAudioTimeOffset + segmentStartTime, initialAudioTimeOffset + segmentEndTime)
 
-						timeline.push(...wordTimeline)
+						timeline.push(...tokenTimeline)
 					} else {
 						const alignmentPath = await this.findAlignmentPathFromQKs(segmentCrossAttentionQKs, segmentTokens, segmentStartFrame, segmentEndFrame)//, alignmentHeadsIndexes[modelName])
-						const wordTimeline = await this.getWordTimelineFromAlignmentPath(alignmentPath, segmentTokens, initialAudioTimeOffset, initialAudioTimeOffset + segmentEndTime)
+						const tokenTimeline = await this.getTokenTimelineFromAlignmentPath(alignmentPath, segmentTokens, initialAudioTimeOffset, initialAudioTimeOffset + segmentEndTime)
 
-						timeline.push(...wordTimeline)
+						timeline.push(...tokenTimeline)
 					}
 				}
 
@@ -846,17 +1022,34 @@ export class Whisper {
 		}
 	}
 
-	mergeSuccessiveWordFragmentsInTimeline(timeline: Timeline) {
+	tokenTimelineToWordTimeline(tokenTimeline: Timeline) {
+		const separatorChars =
+			[' ', '–', '一', '，', '、', '|', '/', '\\', ';', '"', '“', '”', '…', '(', ')', '[', ']', '{', '}']
+
+		function startsWithSeparatingPunctuation(text: string) {
+			return separatorChars.some(char => text.startsWith(char))
+		}
+
+		function isSeparatorPunctuation(text: string) {
+			return separatorChars.includes(text)
+		}
+
 		const resultTimeline: Timeline = []
 
 		const groups: TimelineEntry[][] = []
 
-		for (const entry of timeline) {
-			if (entry.type != "word") {
-				continue
-			}
+		for (let i = 0; i < tokenTimeline.length; i++) {
+			const entry = tokenTimeline[i]
+			const previousEntry = i > 0 ? tokenTimeline[i - 1] : undefined
+
+			const text = entry.text
+			const previousEntryText = previousEntry?.text
+
+			if (groups.length == 0 ||
+				text === '' ||
+				startsWithSeparatingPunctuation(text) ||
+				(previousEntryText != null && isSeparatorPunctuation(previousEntryText))) {
 
-			if (groups.length == 0 || entry.text.startsWith(" ")) {
 				groups.push([entry])
 			} else {
 				groups[groups.length - 1].push(entry)
@@ -864,39 +1057,41 @@ export class Whisper {
 		}
 
 		for (const group of groups) {
-			if (group.length == 1) {
-				resultTimeline.push(deepClone(group[0]))
-			} else {
-				const text = group.map(entry => entry.text).join("")
-				const startTime = group[0].startTime
-				const endTime = group[group.length - 1].endTime
-				let confidence: number | undefined = undefined
+			const groupText = this.tokensToText(group.map(entry => entry.id!))
 
-				if (group[0].confidence != null) {
-					confidence = meanOfVector(group.map(entry => entry.confidence!))
-				}
+			if (groupText === '') {
+				continue
+			}
 
-				const newEntry: TimelineEntry = {
-					type: "word",
-					text,
-					startTime,
-					endTime,
-					confidence
-				}
+			const startTime = group[0].startTime
+			const endTime = group[group.length - 1].endTime
+			let confidence: number | undefined = undefined
+
+			if (group[0].confidence != null) {
+				confidence = meanOfVector(group.map(entry => entry.confidence!))
+			}
 
-				resultTimeline.push(newEntry)
+			const newEntry: TimelineEntry = {
+				type: 'word',
+				text: groupText.trim(),
+				startTime,
+				endTime,
+				confidence,
+				timeline: group,
 			}
+
+			resultTimeline.push(newEntry)
 		}
 
 		return resultTimeline
 	}
 
-	async getWordTimelineFromAlignmentPath(alignmentPath: AlignmentPath, tokens: number[], startTimeOffset: number, endTimeOffset: number, tokensConfidence?: number[], correctionAmount = 0.0) {
+	async getTokenTimelineFromAlignmentPath(alignmentPath: AlignmentPath, tokens: number[], startTimeOffset: number, endTimeOffset: number, tokensConfidence?: number[], correctionAmount = 0.0) {
 		if (alignmentPath.length == 0) {
 			return []
 		}
 
-		const wordTimeline: Timeline = []
+		const tokenTimeline: Timeline = []
 
 		for (let pathIndex = 0; pathIndex < alignmentPath.length; pathIndex++) {
 			if (pathIndex != 0 && alignmentPath[pathIndex].source == alignmentPath[pathIndex - 1].source) {
@@ -908,41 +1103,37 @@ export class Whisper {
 			const tokenIndex = tokenMappingEntry.source
 			const token = tokens[tokenIndex]
 			const tokenConfidence = tokensConfidence ? tokensConfidence[tokenIndex] : undefined
-			const tokenText = this.tokenToTextLookup.get(token)
-
-			if (token >= this.tokenConfig.eotToken || !tokenText) {
-				continue
-			}
+			const tokenText = this.tokenToText(token, true)
 
 			let startTime = startTimeOffset + (tokenMappingEntry.dest * 0.02)
 
 			startTime = Math.max(startTime + correctionAmount, startTimeOffset)
 
-			if (wordTimeline.length > 0) {
-				wordTimeline[wordTimeline.length - 1].endTime = startTime
+			if (tokenTimeline.length > 0) {
+				tokenTimeline[tokenTimeline.length - 1].endTime = startTime
 			}
 
-			wordTimeline.push({
-				type: "word",
+			tokenTimeline.push({
+				type: 'token',
 				text: tokenText,
+				id: token,
 				startTime,
 				endTime: -1,
 				confidence: tokenConfidence
 			})
 		}
 
-		if (wordTimeline.length > 0) {
-			wordTimeline[wordTimeline.length - 1].endTime = endTimeOffset
+		if (tokenTimeline.length > 0) {
+			tokenTimeline[tokenTimeline.length - 1].endTime = endTimeOffset
 		}
 
-		return wordTimeline
+		return tokenTimeline
 	}
 
 	async findAlignmentPathFromQKs(qksTensors: Onnx.Tensor[], tokens: number[], segmentStartFrame: number, segmentEndFrame: number, headIndexes?: number[]) {
 		const segmentFrameCount = segmentEndFrame - segmentStartFrame
 
-		if (segmentFrameCount == 0) {
-			//throw new Error("Segment has 0 frames")
+		if (segmentFrameCount === 0 || tokens.length === 0 || qksTensors.length === 0) {
 			return []
 		}
 
@@ -961,10 +1152,10 @@ export class Whisper {
 		}
 
 		// Load attention head weights from tensors
-		const attentionHeads: number[][][] = [] // [heads, tokens, frames]
+		const attentionHeads: number[][][] = [] // structure: [heads, tokens, frames]
 
 		for (const headIndex of headIndexes) {
-			const attentionHead: number[][] = [] // [tokens, frames]
+			const attentionHead: number[][] = [] // structure: [tokens, frames]
 
 			for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) {
 				const bufferOffset = headIndex * frameCount
@@ -1002,7 +1193,7 @@ export class Whisper {
 				const allWeightsForHead = head.flatMap(tokenFrames => tokenFrames)
 
 				const meanOfAllWeights = meanOfVector(allWeightsForHead)
-				const stdDeviationOfAllWeights = stdDeviationOfVector(allWeightsForHead)
+				const stdDeviationOfAllWeights = stdDeviationOfVector(allWeightsForHead) + 1e-10
 
 				for (const tokenFrames of head) {
 					for (let frameIndex = 0; frameIndex < tokenFrames.length; frameIndex++) {
@@ -1045,11 +1236,15 @@ export class Whisper {
 		}
 
 		if (fixateTimestampTokens) {
+			// Fixate timestamp tokens to the original ones detected
 			const timestampTokensStart = this.tokenConfig.timestampTokensStart
 
 			for (let tokenIndex = 0; tokenIndex < tokens.length; tokenIndex++) {
-				if (tokens[tokenIndex] >= timestampTokensStart) {
-					let timestampFrame = tokens[tokenIndex] - timestampTokensStart
+				const token = tokens[tokenIndex]
+
+				if (this.isTimestampToken(token)) {
+					let timestampFrame = token - timestampTokensStart
+
 					timestampFrame = clip(timestampFrame, segmentStartFrame, segmentEndFrame - 1)
 
 					frameMeansForToken[tokenIndex][timestampFrame] = 100
@@ -1061,9 +1256,9 @@ export class Whisper {
 		const tokenIndexes = [...Array(tokenCount).keys()]
 		const frameIndexes = [...Array(segmentFrameCount).keys()]
 
-		let { path } = await alignDTWWindowed(tokenIndexes, frameIndexes, (tokenIndex, frameIndex) => {
+		let { path } = alignDTWWindowed(tokenIndexes, frameIndexes, (tokenIndex, frameIndex) => {
 			return -frameMeansForToken[tokenIndex][frameIndex]
-		}, 1000)
+		}, segmentFrameCount)
 
 		path = path.map(entry => ({ source: entry.source, dest: segmentStartFrame + entry.dest }))
 
@@ -1073,101 +1268,138 @@ export class Whisper {
 	getKvDimensions(groupCount: number, length: number) {
 		const modelName = this.modelName
 
-		if (modelName == "tiny" || modelName == "tiny.en") {
+		if (modelName == 'tiny' || modelName == 'tiny.en') {
 			return [8, groupCount, length, 384]
-		} else if (modelName == "base" || modelName == "base.en") {
+		} else if (modelName == 'base' || modelName == 'base.en') {
 			return [12, groupCount, length, 512]
-		} else if (modelName == "small" || modelName == "small.en") {
+		} else if (modelName == 'small' || modelName == 'small.en') {
 			return [24, groupCount, length, 768]
-		} else if (modelName == "medium" || modelName == "medium.en") {
+		} else if (modelName == 'medium' || modelName == 'medium.en') {
 			return [48, groupCount, length, 1024]
-		} else if (modelName == "large" || modelName == "large-v1" || modelName == "large-v2" || modelName == "large-v3") {
+		} else if (modelName == 'large' || modelName == 'large-v1' || modelName == 'large-v2' || modelName == 'large-v3') {
 			return [64, groupCount, length, 1280]
 		} else {
 			throw new Error(`Unsupported model: ${modelName}`)
 		}
 	}
 
-	getInitialTokens(language: string, task: WhisperTask, disableTimestamps = false) {
-		const sotToken = this.tokenConfig.sotToken
+	getTextStartTokens(language: string, task: WhisperTask, disableTimestamps = false) {
+		const startOfTextToken = this.tokenConfig.startOfTextToken
 
-		let initialTokens: number[]
+		let tokens: number[]
 
 		if (this.isMultiligualModel) {
-			const languageToken = sotToken + 1 + languageIdLookup[language]
-			const translateTaskToken = 50358
-			const transcribeTaskToken = 50359
-			const taskToken = task == "transcribe" ? transcribeTaskToken : translateTaskToken
+			const languageToken = this.tokenConfig.languageTokensStart + languageIdLookup[language]
+			const taskToken = task == 'translate' ? this.tokenConfig.translateTaskToken : this.tokenConfig.transcribeTaskToken
 
-			initialTokens = [sotToken, languageToken, taskToken]
+			tokens = [startOfTextToken, languageToken, taskToken]
 		} else {
-			initialTokens = [sotToken]
+			tokens = [startOfTextToken]
 		}
 
 		if (disableTimestamps) {
-			initialTokens.push(this.tokenConfig.noTimestampsToken)
+			tokens.push(this.tokenConfig.noTimestampsToken)
 		}
 
-		return initialTokens
+		return tokens
 	}
 
-	getAlignmentHeadIndexes() {
-		return alignmentHeadsIndexes[this.modelName]
+	tokenToText(token: number, includeMetadataTokens = false) {
+		return this.tokensToText([token], includeMetadataTokens)
 	}
 
-	tokensToText(tokens: number[]) {
-		return tokens.map(token => this.tokenToTextLookup.get(token) || "").join("").trim()
+	tokensToText(tokens: number[], includeMetadataTokens = false) {
+		tokens.forEach(token => this.assertIsValidToken(token))
+
+		if (includeMetadataTokens === false) {
+			tokens = tokens.filter(token => this.isTextToken(token))
+		}
+
+		if (tokens.length === 0) {
+			return ''
+		}
+
+		try {
+			return Buffer.from(this.tiktoken!.decode(new Uint32Array(tokens))).toString('utf8')
+		} catch {
+			return '[TOKENIZER_FAILED]'
+		}
 	}
 
-	async textToTokens(text: string, language: string) {
-		const resultTokens: number[] = []
+	textToTokens(text: string) {
+		return Array.from(this.tiktoken!.encode(text))
+	}
 
-		const words = (await splitToWords(text, language)).filter(w => w.trim().length > 0)
+	isTextToken(token: number) {
+		return token < this.tokenConfig.endOfTextToken
+	}
 
-		//words = words.filter(word => wordCharacterPattern.test(word))
+	isMetadataToken(token: number) {
+		return token >= this.tokenConfig.endOfTextToken
+	}
 
-		for (let i = 1; i < words.length; i++) {
-			words[i] = ` ${words[i]}`
-		}
+	isLanguageToken(token: number) {
+		return token >= this.tokenConfig.languageTokensStart && token < this.tokenConfig.languageTokensEnd
+	}
 
-		const allResultingSubwords: string[][] = []
+	isTimestampToken(token: number) {
+		return token >= this.tokenConfig.timestampTokensStart
+	}
 
-		for (const word of words) {
-			const tokenForEntireWord = this.textToTokenLookup.get(word)
+	isNonTimestampToken(token: number) {
+		return token < this.tokenConfig.timestampTokensStart
+	}
 
-			if (tokenForEntireWord) {
-				resultTokens.push(tokenForEntireWord)
-				allResultingSubwords.push([word])
-				continue
-			}
+	timestampTokenToSeconds(timestampToken: number) {
+		this.assertIsValidToken(timestampToken)
 
-			const subwords = word.split("")
+		if (this.isNonTimestampToken(timestampToken)) {
+			throw new Error(`Invalid timestamp token: ${timestampToken}`)
+		}
 
-			for (const mergeRule of this.merges) {
-				for (let i = 0; i < subwords.length - 1; i++) {
-					const currentSubword = subwords[i]
-					const nextSubword = subwords[i + 1]
+		return (timestampToken - this.tokenConfig.timestampTokensStart) * 0.02
+	}
 
-					if (currentSubword == mergeRule[0] && nextSubword == mergeRule[1]) {
-						subwords.splice(i, 2, mergeRule[0] + mergeRule[1])
-					}
-				}
-			}
+	isValidToken(token: number) {
+		return token < this.tokenConfig.timestampTokensEnd
+	}
 
-			for (const subword of subwords) {
-				const tokenForSubword = this.textToTokenLookup.get(subword)
+	assertIsValidToken(token: number) {
+		if (!this.isValidToken(token)) {
+			throw new Error(`Invalid token: ${token}`)
+		}
+	}
 
-				if (!tokenForSubword) {
-					throw new Error(`Failed tokenizing the given text. The word '${word}' contains a subword '${subword}' which is not in the vocabulary.`)
-				}
+	secondsToFrame(seconds: number) {
+		return Math.floor(seconds / 0.02)
+	}
 
-				resultTokens.push(tokenForSubword)
-			}
+	secondsRangeToFrameCount(startSeconds: number, endSeconds: number) {
+		if (startSeconds > endSeconds) {
+			throw new Error(`Invalid range: ${startSeconds} > ${endSeconds}`)
+		}
 
-			allResultingSubwords.push(subwords)
+		return this.secondsToFrame(endSeconds - startSeconds)
+	}
+
+	languageTokenToLanguageIndex(languageToken: number) {
+		if (!this.isLanguageToken(languageToken)) {
+			throw new Error(`Invalid language token: ${languageToken}`)
 		}
 
-		return resultTokens
+		let languageIndex = languageToken - this.tokenConfig.languageTokensStart
+
+		if (this.isEnglishOnlyModel) {
+			languageIndex += 1
+		}
+	}
+
+	get isEnglishOnlyModel() {
+		return this.isMultiligualModel === false
+	}
+
+	getAlignmentHeadIndexes() {
+		return alignmentHeadsIndexes[this.modelName]
 	}
 }
 
@@ -1334,171 +1566,184 @@ const filterbanks: Filterbank[] = [
 ]
 
 export async function loadPackagesAndGetPaths(modelName: WhisperModelName | undefined, languageCode: string | undefined) {
-	if (!modelName) {
+	if (modelName) {
+		modelName = normalizeWhisperModelName(modelName, languageCode)
+	} else {
 		if (languageCode) {
 			const shortLanguageCode = getShortLanguageCode(languageCode)
 
-			modelName = shortLanguageCode == "en" ? "tiny.en" : "tiny"
+			modelName = shortLanguageCode == 'en' ? 'tiny.en' : 'tiny'
 		} else {
-			modelName = "tiny"
+			modelName = 'tiny'
 		}
 	}
 
+	if (modelName.startsWith('large')) {
+		throw new Error(`Large models are not currently supported by the integrated Whisper engine due to model size restrictions of onnxruntime-node. To use large models, you can select the whisper.cpp engine instead.`)
+	}
+
 	const packageName = modelNameToPackageName[modelName]
 
 	const modelDir = await loadPackage(packageName)
 
-	const tokenizerPackagePath = await loadPackage(tokenizerPackageName)
-	const tokenizerDir = isMultiligualModel(modelName) ? path.join(tokenizerPackagePath, "multilingual") : path.join(tokenizerPackagePath, "gpt2")
-
-	return { modelName, modelDir, tokenizerDir }
+	return { modelName, modelDir }
 }
 
-export function isMultiligualModel(modelName: WhisperModelName) {
-	return !modelName.endsWith(".en")
+export function normalizeWhisperModelName(modelName: WhisperModelName, languageCode: string | undefined): WhisperModelName {
+	if (languageCode != 'en' && modelName.endsWith('.en')) {
+		const originalModelName = modelName
+		modelName = modelName.slice(0, modelName.length - 3) as WhisperModelName
+
+		const logger = new Logger()
+		logger.logTitledMessage(`Warning`, `The model '${originalModelName}' is English only and cannot be used to transcribe language '${languageCode}'. using '${modelName}' instead.`, chalk.yellowBright)
+	}
+
+	return modelName
 }
 
-export type WhisperModelName = "tiny" | "tiny.en" | "base" | "base.en" | "small" | "small.en" | "medium" | "medium.en" | "large" | "large-v1" | "large-v2" | "large-v3"
-export type WhisperTask = "transcribe" | "translate"
+export function isMultilingualModel(modelName: WhisperModelName) {
+	return !isEnglishOnlyModel(modelName)
+}
 
-export const modelNameToPackageName: { [modelName in WhisperModelName]: string } = {
-	"tiny": "whisper-tiny",
-	"tiny.en": "whisper-tiny.en",
-	"base": "whisper-base",
-	"base.en": "whisper-base.en",
-	"small": "whisper-small",
-	"small.en": "whisper-small.en",
-	"medium": "whisper-medium",
-	"medium.en": "whisper-medium.en",
-	"large": "whisper-large-v3",
-	"large-v1": "whisper-large-v1",
-	"large-v2": "whisper-large-v2",
-	"large-v3": "whisper-large-v3"
+export function isEnglishOnlyModel(modelName: WhisperModelName) {
+	return modelName.endsWith('.en')
 }
 
-export const tokenizerPackageName = "whisper-tokenizer"
+export type WhisperModelName = 'tiny' | 'tiny.en' | 'base' | 'base.en' | 'small' | 'small.en' | 'medium' | 'medium.en' | 'large' | 'large-v1' | 'large-v2' | 'large-v3'
+export type WhisperTask = 'transcribe' | 'translate' | 'detect-language'
 
-const vocabCharacterSetLookup: { [s: string]: number } = {
-	"!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54,
-	"7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ":
-		21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "Ń": 173
+export const modelNameToPackageName: { [modelName in WhisperModelName]: string } = {
+	'tiny': 'whisper-tiny',
+	'tiny.en': 'whisper-tiny.en',
+	'base': 'whisper-base',
+	'base.en': 'whisper-base.en',
+	'small': 'whisper-small',
+	'small.en': 'whisper-small.en',
+	'medium': 'whisper-medium',
+	'medium.en': 'whisper-medium.en',
+	'large': 'whisper-large-v3',
+	'large-v1': 'whisper-large-v1',
+	'large-v2': 'whisper-large-v2',
+	'large-v3': 'whisper-large-v3'
 }
 
+export const tokenizerPackageName = 'whisper-tokenizer'
+
 const languageIdLookup: { [s: string]: number } = {
-	"en": 0,
-	"zh": 1,
-	"de": 2,
-	"es": 3,
-	"ru": 4,
-	"ko": 5,
-	"fr": 6,
-	"ja": 7,
-	"pt": 8,
-	"tr": 9,
-	"pl": 10,
-	"ca": 11,
-	"nl": 12,
-	"ar": 13,
-	"sv": 14,
-	"it": 15,
-	"id": 16,
-	"hi": 17,
-	"fi": 18,
-	"vi": 19,
-	"iw": 20,
-	"uk": 21,
-	"el": 22,
-	"ms": 23,
-	"cs": 24,
-	"ro": 25,
-	"da": 26,
-	"hu": 27,
-	"ta": 28,
-	"no": 29,
-	"th": 30,
-	"ur": 31,
-	"hr": 32,
-	"bg": 33,
-	"lt": 34,
-	"la": 35,
-	"mi": 36,
-	"ml": 37,
-	"cy": 38,
-	"sk": 39,
-	"te": 40,
-	"fa": 41,
-	"lv": 42,
-	"bn": 43,
-	"sr": 44,
-	"az": 45,
-	"sl": 46,
-	"kn": 47,
-	"et": 48,
-	"mk": 49,
-	"br": 50,
-	"eu": 51,
-	"is": 52,
-	"hy": 53,
-	"ne": 54,
-	"mn": 55,
-	"bs": 56,
-	"kk": 57,
-	"sq": 58,
-	"sw": 59,
-	"gl": 60,
-	"mr": 61,
-	"pa": 62,
-	"si": 63,
-	"km": 64,
-	"sn": 65,
-	"yo": 66,
-	"so": 67,
-	"af": 68,
-	"oc": 69,
-	"ka": 70,
-	"be": 71,
-	"tg": 72,
-	"sd": 73,
-	"gu": 74,
-	"am": 75,
-	"yi": 76,
-	"lo": 77,
-	"uz": 78,
-	"fo": 79,
-	"ht": 80,
-	"ps": 81,
-	"tk": 82,
-	"nn": 83,
-	"mt": 84,
-	"sa": 85,
-	"lb": 86,
-	"my": 87,
-	"bo": 88,
-	"tl": 89,
-	"mg": 90,
-	"as": 91,
-	"tt": 92,
-	"haw": 93,
-	"ln": 94,
-	"ha": 95,
-	"ba": 96,
-	"jw": 97,
-	"su": 98,
+	'en': 0,
+	'zh': 1,
+	'de': 2,
+	'es': 3,
+	'ru': 4,
+	'ko': 5,
+	'fr': 6,
+	'ja': 7,
+	'pt': 8,
+	'tr': 9,
+	'pl': 10,
+	'ca': 11,
+	'nl': 12,
+	'ar': 13,
+	'sv': 14,
+	'it': 15,
+	'id': 16,
+	'hi': 17,
+	'fi': 18,
+	'vi': 19,
+	'iw': 20,
+	'uk': 21,
+	'el': 22,
+	'ms': 23,
+	'cs': 24,
+	'ro': 25,
+	'da': 26,
+	'hu': 27,
+	'ta': 28,
+	'no': 29,
+	'th': 30,
+	'ur': 31,
+	'hr': 32,
+	'bg': 33,
+	'lt': 34,
+	'la': 35,
+	'mi': 36,
+	'ml': 37,
+	'cy': 38,
+	'sk': 39,
+	'te': 40,
+	'fa': 41,
+	'lv': 42,
+	'bn': 43,
+	'sr': 44,
+	'az': 45,
+	'sl': 46,
+	'kn': 47,
+	'et': 48,
+	'mk': 49,
+	'br': 50,
+	'eu': 51,
+	'is': 52,
+	'hy': 53,
+	'ne': 54,
+	'mn': 55,
+	'bs': 56,
+	'kk': 57,
+	'sq': 58,
+	'sw': 59,
+	'gl': 60,
+	'mr': 61,
+	'pa': 62,
+	'si': 63,
+	'km': 64,
+	'sn': 65,
+	'yo': 66,
+	'so': 67,
+	'af': 68,
+	'oc': 69,
+	'ka': 70,
+	'be': 71,
+	'tg': 72,
+	'sd': 73,
+	'gu': 74,
+	'am': 75,
+	'yi': 76,
+	'lo': 77,
+	'uz': 78,
+	'fo': 79,
+	'ht': 80,
+	'ps': 81,
+	'tk': 82,
+	'nn': 83,
+	'mt': 84,
+	'sa': 85,
+	'lb': 86,
+	'my': 87,
+	'bo': 88,
+	'tl': 89,
+	'mg': 90,
+	'as': 91,
+	'tt': 92,
+	'haw': 93,
+	'ln': 94,
+	'ha': 95,
+	'ba': 96,
+	'jw': 97,
+	'su': 98,
 }
 
 const alignmentHeadsIndexes: { [name in WhisperModelName]: number[] } = {
-	"tiny.en": [6, 12, 17, 18, 19, 20, 21, 22],
-	"tiny": [14, 18, 20, 21, 22, 23],
-	"base.en": [27, 39, 41, 45, 47],
-	"base": [25, 34, 35, 39, 41, 42, 44, 46],
-	"small.en": [78, 84, 87, 92, 98, 101, 103, 108, 112, 116, 118, 120, 121, 122, 123, 126, 131, 134, 136],
-	"small": [63, 69, 96, 100, 103, 104, 108, 115, 117, 125],
-	"medium.en": [180, 225, 236, 238, 244, 256, 260, 265, 284, 286, 295, 298, 303, 320, 323, 329, 334, 348],
-	"medium": [223, 244, 255, 257, 320, 372],
-	"large-v1": [199, 222, 224, 237, 447, 451, 457, 462, 475],
-	"large-v2": [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555],
-	"large-v3": [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555], // Temporary (may not be correct)
-	"large": [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555],
+	'tiny.en': [6, 12, 17, 18, 19, 20, 21, 22],
+	'tiny': [14, 18, 20, 21, 22, 23],
+	'base.en': [27, 39, 41, 45, 47],
+	'base': [25, 34, 35, 39, 41, 42, 44, 46],
+	'small.en': [78, 84, 87, 92, 98, 101, 103, 108, 112, 116, 118, 120, 121, 122, 123, 126, 131, 134, 136],
+	'small': [63, 69, 96, 100, 103, 104, 108, 115, 117, 125],
+	'medium.en': [180, 225, 236, 238, 244, 256, 260, 265, 284, 286, 295, 298, 303, 320, 323, 329, 334, 348],
+	'medium': [223, 244, 255, 257, 320, 372],
+	'large-v1': [199, 222, 224, 237, 447, 451, 457, 462, 475],
+	'large-v2': [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555],
+	'large-v3': [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555], // Temporary (may not be correct)
+	'large': [212, 277, 331, 332, 333, 355, 356, 364, 371, 379, 391, 422, 423, 443, 449, 452, 465, 467, 473, 505, 521, 532, 555],
 }
 
 export interface WhisperOptions {
@@ -1508,13 +1753,21 @@ export interface WhisperOptions {
 	topCandidateCount?: number
 	punctuationThreshold?: number
 	autoPromptParts?: boolean
+	maxTokensPerPart?: number
+	suppressRepetition?: boolean
+	seed?: number
+	decodeTimestampTokens?: boolean
 }
 
-export const whisperOptionsDefaults: WhisperOptions = {
+export const defaultWhisperOptions: WhisperOptions = {
 	model: undefined,
 	temperature: 0.1,
 	prompt: undefined,
 	topCandidateCount: 5,
 	punctuationThreshold: 0.2,
-	autoPromptParts: true
+	autoPromptParts: true,
+	maxTokensPerPart: 250,
+	suppressRepetition: true,
+	seed: undefined,
+	decodeTimestampTokens: false,
 }