echogarden 1.0.0 → 1.0.2

package/docs/Options.md

@@ -148,14 +148,15 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 * `whisper.autoPromptParts`: use previous part's recognized text as prompt for the next part. Disabling this may help to prevent repetition carrying over between parts, in some cases. Defaults to `true`
 * `whisper.maxTokensPerPart`: maximum number of tokens to decode for each 30 second audio part. Defaults to `250`
 * `whisper.suppressRepetition`: attempt to suppress decoding repeating token patterns. Defaults to `true`
-* `whisper.seed`: use a custom seed for token selection, when temperature is greater than 0. Uses a constant seed by default
+* `whisper.decodeTimestampTokens`: enable/disable decoding of timestamp tokens, since more accurate timing is already extracted via cross-attention weight alignment. For unclear reasons, setting to `false` can significantly reduce the occurrence of hallucinations and token repetition loops, and increases word timestamp accuracy. However, there are cases where this can cause the model to end a part prematurely, especially in singing and less speech-like voice segments, or when there are multiple speakers. Defaults to `true`
+* `whisper.seed`: provide a custom random seed for token selection when temperature is greater than 0. Uses a constant seed by default to ensure reproducibility
 
 **Whisper.cpp**:
 * `whisperCpp.model`: selects which `whisper.cpp` model to use.  Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large` (same as `large-v2`), `large-v1`, `large-v2`, `large-v3`. These quantized models are also supported: `tiny-q5_1`, `tiny.en-q5_1`, `tiny.en-q8_0`,`base-q5_1`, `base.en-q5_1`, `small-q5_1`, `small.en-q5_1`, `medium-q5_0`, `medium.en-q5_0`, `large-v2-q5_0`, `large-v3-q5_0`. Defaults to `base` or `base.en`
 * `whisperCpp.executablePath`: custom `whisper.cpp` executable path (currently required for macOS)
 * `whisperCpp.build`: type of `whisper.cpp` build to use. Can be set `cpu`, `cublas-11.8.0`, `cublas-12.4.0`. By default, builds are auto-selected and downloaded for Windows x64 (`cpu`, `cublas-11.8.0`, `cublas-12.4.0`) and Linux x64 (`cpu`). Using other builds requires providing a custom `executablePath`
 * `whisperCpp.threadCount`: number of threads to use, defaults to `4`
-* `whisperCpp.splitCount`: number of splits of the audio data to process in parallel (called `--processors` in the CLI). A value greater than `1` can increase memory use significantly, reduce timing accuracy, and slow down execution in some cases. Defaults to `1` (highly recommended)
+* `whisperCpp.splitCount`: number of splits of the audio data to process in parallel (called `--processors` in the `whisper.cpp` CLI). A value greater than `1` can increase memory use significantly, reduce timing accuracy, and slow down execution in some cases. Defaults to `1` (highly recommended)
 * `whisperCpp.enableGPU`: enable GPU processing. Defaults to `true` on CUDA-enabled builds, otherwise `false`
 * `whisperCpp.topCandidateCount`: the number of top candidate tokens to consider. Defaults to `5`
 * `whisperCpp.beamCount`: the number of decoding paths to use during beam search. Defaults to `5`
@@ -205,7 +206,7 @@ Applies to CLI operation: `align`, API method: `align`
 * `engine`: alignment algorithm to use, can be `dtw`, `dtw-ra` or `whisper`. Defaults to `dtw`
 * `language`: language code for the audio and transcript ([ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)), like `en`, `fr`, `en-US`, `pt-BR`. Auto-detected from transcript if not set
 * `crop`: crop to active parts using voice activity detection before starting. Defaults to `true`
-* `isolate`: apply source separation to isolate voice before starting recognition. Defaults to `false`
+* `isolate`: apply source separation to isolate voice before starting alignment. Defaults to `false`
 * `customLexiconPaths`: an array of custom lexicon file paths. Optional
 * `subtitles`: prefix to provide options for subtitles. Options detailed in section for subtitles
 * `vad`: prefix to provide options for voice activity detection when `crop` is set to `true`. Options detailed in section for voice activity detection
@@ -221,7 +222,7 @@ Applies to CLI operation: `align`, API method: `align`
 
 **DTW-RA only**:
 * `recognition`: prefix to provide recognition options when using `dtw-ra` method, for example: setting `recognition.engine = whisper` and `recognition.whisper.model = base.en`
-* `dtw.phoneAlignmentMethod`: algorithm to use when aligning phones: can either be set to `dtw` or `interpolate`. Defaults to `dtw`
+* `dtw.phoneAlignmentMethod`: algorithm to use when aligning phones: can either be set to `dtw` or `interpolation`. Defaults to `dtw`
 
 **Whisper alignment only**:
 * `whisper`: prefix to provide Whisper options when the `whisper` alignment engine is used (does not apply to `dtw-ra` when `whisper` engine is used, for that use `recognition.whisper` prefix instead).
@@ -333,10 +334,3 @@ These are shared between text-to-speech, speech-to-text and alignment operations
 * `maxAddedDuration`: maximum extra time (in seconds) that may be added after a cue's end time. This gives the reader additional time to read the cue, and also ensures that very short duration cues aren't shown in a flash. Defaults to `3.0`
 
 **Note**: options `maxLineCount`, `maxLineWidth`, `minWordsInLine`, `separatePhrases`, are only effective when using the `segment` and `sentence` modes, and are ignored in all other modes. `maxAddedDuration` doesn't apply to modes `word`, `phone` and `word+phone` (they always use the exact start and end timestamps).
-
-# General CLI options
-
-* `--play`, `--no-play`: enable/disable audio playback. Defaults to play if there is no output file specified
-* `--overwrite`, `--no-overwrite`: overwrite/keep existing files. Doesn't overwrite by default
-* `--debug`, `--no-debug`: show/hide the full details of JavaScript errors, if they occur. Disabled by default
-* `--config=...`: path to configuration file to use. See the [CLI guide](CLI.md) for more information about configuration files

package/docs/Releases.md

@@ -1,6 +1,6 @@
 # Release notes
 
-## `1.0.0` (April 4, 2024)
+## `1.0.0` (April 12, 2024)
 
 **New features**:
 * Add support for [`whisper.cpp`](https://github.com/ggerganov/whisper.cpp), a C++ port of OpenAI's Whisper speech recognition architecture. It is faster than the integrated `whisper` engine, supports large models, and GPU processing. It can now be used for recognition, speech translation and alignment (via `dtw-ra`), though its word timestamps are less accurate than the integrated `whisper` engine.
@@ -13,6 +13,11 @@
 * Expose several new configuration options for the Whisper engine: a settings for maximum tokens per part, and a setting to enable/disable repetition suppression, set custom random seed, disable/enable decoding of timestamp tokens
 * Expose more options for the Elevenlabs engine
 
+**Enhancements**:
+* `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
+* Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
+* Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
+
 **Behavioral and breaking changes**:
 * Minimal required node version changed to `18.0.0`
 * All recognition, alignment, translation and language recognition operations first apply the new adaptive gate VAD (can be changed to any other VAD engine via the `vad.` option prefix) and remove any sections that are not identified as containing voice, before starting processing. This should improve results in most cases, and reduce processing time
@@ -30,11 +35,6 @@
 * `maxIncreaseDb` post-processing and denoising option renamed to `maxGainIncrease`
 * `dryMixGainDb` denoising option renamed to `dryMixGain`
 
-**Enhancements**:
-* `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
-* Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
-* Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
-
 **Fixes**:
 * Integrated `whisper` engine now uses `tiktoken` to tokenize text, which produces near identical tokenization compared to the official Python implementation. This fixes issues with decoding Chinese characters, correctly encoding prompts, and several others languages with characters that span multiple tokens
 * Fix voice language lists for multilingual voices in Elevenlabs TTS engine
@@ -43,14 +43,15 @@
 * Fix issue with identifying legacy IEEE Float wave format
 * Fix issue with language detection failing with empty inputs
 * Fix issue with Whisper model failing when no tokens are detected in a part
-* Fix issue with phone alignment not getting the right DTW window size, producing warnings like `all cost directions are equal to infinity
+* Fix issue with phone alignment not getting the right DTW window size, producing warnings like `all cost directions are equal to infinity`
 * Fix phone timelines in DTW-RA
 * Add more workarounds for eSpeak tokenization bugs
 * Various other fixes
 
 **Documentation**:
-* Add new 'Releases' page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
-* Options reference page restructured and updated with some missing information
+* Add new [release notes](./Releases.md) page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
+* Add new [development page](./Development.md) with a quick guide for setting up a development environment to make changes to the Echogarden codebase
+* Options reference page reorganized and updated with some missing information
 
 ## `0.12.x` (March 16, 2024)
 
@@ -67,6 +68,7 @@
 **Other**:
 * Include `package-lock.json` in repository
 
+
 ## `0.11.x` (August 17, 2023)
 
 Many features, enhancements, and fixes were incrementally added over the span of 7 months, up to March 2024.
@@ -87,6 +89,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Fix audio playback in macOS
 * Many other fixes
 
+
 ## `0.10.x` (August 2, 2023)
 
 **New features**:
@@ -100,6 +103,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 **Fixes**:
 * Many fixes
 
+
 ## `0.9.x` (July 29, 2023)
 
 **New features**:
@@ -114,6 +118,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Work around several eSpeak bugs
 * Many fixes
 
+
 ## `0.8.x` (July 25, 2023)
 
 **New features**:
@@ -122,26 +127,25 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add punctuation thresholds to Whisper decoder
 * Expose subtitle configuration options to CLI
 
-**Behavioral changes**:
-* Rename `subtitles.minWords` to `subtitles.minWordsInLine`
-
 **Enhancements**:
 * Improve speech language detection to work with arbitrary length audio. Split audio to overlapping parts and detect each part individually. Then average the results.
 * Add more heteronyms
 
+**Behavioral changes**:
+* Rename `subtitles.minWords` to `subtitles.minWordsInLine`
+
 **Fixes**:
 * Improve numerical stability of softmax function
 * Work around eSpeak bug with markers in long inputs
 * Various fixes
 
+
 ## `0.7.x` (July 23, 2023)
 
 **New features**:
 * Alignment: Add option to accept custom lexicons
 * Implement and add decoder temperature option to Whisper model
 
-**Behavioral changes**:
-
 **Enhancements**:
 * Alignment: use preprocessing and lexicons
 * Add language detection to speech translation
@@ -151,10 +155,13 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add colors to log messages
 * Warn when maximum DTW window duration is smaller than source audio duration.
 
+**Behavioral changes**:
+
 **Fixes**:
 * Fix support for SSML input in eSpeak engine
 * Various fixes
 
+
 ## `0.6.x` (July 20, 2023)
 
 **New features**:
@@ -165,16 +172,16 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add arguments to customize paragraph parsing
 * Change whitespace option to include option for collapsing all whitespace
 
-**Behavioral changes**:
-* Set Elevenlabs defaults to mid-values
-* Split plain text to paragraphs using double line breaks by default.
-* Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
-
 **Enhancements**:
 * Change whitespace option to include option for collapsing all whitespace
 * Add awareness of guillemets
 * Log full language of selected voice
 
+**Behavioral changes**:
+* Set Elevenlabs defaults to mid-values
+* Split plain text to paragraphs using double line breaks by default.
+* Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
+
 **Fixes**:
 * Fix and update Elevenlabs engine
 * Fix incorrect gender properties for some VITS voices
@@ -184,26 +191,25 @@ Many features, enhancements, and fixes were incrementally added over the span of
 **Other**:
 * Remove `package-lock.json` from the repository
 
+
 ## `0.5.x` (July, 19 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Move to new package system supporting version tags, and hosted in a Hugging Face repository
-
 **Enhancements**:
 * Add large numbers of VITS voices
 
+**Behavioral changes**:
+* Move to new package system supporting version tags, and hosted in a Hugging Face repository
+
 **Fixes**:
 * Various fixes
 
+
 ## `0.4.x` (July 9, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
-
 **Enhancements**:
 * Improve text normalization and add support for currencies
 * Improve heteronym lexicons
@@ -212,32 +218,34 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add some British English (RP) pronunciations to heteronym lexicon
 * Add large numbers of VITS voices
 
+**Behavioral changes**:
+* Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
+
 **Fixes**:
 * Various fixes
 
+
 ## `0.3.x` (July 1, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-
 **Enhancements**:
 * Upgrade heteronym disambiguation to an improved, rule-based approach, which doesn't use POS tagging
 * Extend decade normalization
 * Improve text normalization and add support to currencies
 * Update the heteronym lexicon
 
+**Behavioral changes**:
+
 **Fixes**:
 * Remove sentences containing only whitespace when synthesizing
 * Many fixes
 
+
 ## `0.2.x` (May 10, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Remove dependency on `xregexp` package. Use Unicode RegExp instead.
-
 **Enhancements**:
 * Show current sentence and segment in synthesis log.
 * Add check for cancellation flag.
@@ -245,6 +253,9 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Improve auto TTS engine selection
 * Many Enhancements
 
+**Behavioral changes**:
+* Remove dependency on `xregexp` package. Use Unicode RegExp instead.
+
 **Fixes**:
 * Many fixes
 

package/docs/Tasklist.md

@@ -2,8 +2,12 @@
 
 ## Bugs
 
-### Synthesis
+### Alignment
 
+* In DTW-RA, recognition transcript including something like "Question 2.What does Juan", where "2.What" has a point in the middle, is breaking playback of the timeline.
+* DTW-RA will not work correctly with Polish language texts, due to issues with the eSpeak engine pronouncing `|` characters, which are intended to be used as separators and ignored by all other eSpeak languages.
+
+### Synthesis
 
 ### Phoneme processing
 * IPA -> Kirshenbaum translation is still not completely similar to what is output by eSpeak. Also, in rare situations, it outputs characters that are not accepted by eSpeak and eSpeak errors. Investigate when that happens and how to improve on this.
@@ -107,8 +111,8 @@
 * See if it's possible to reliably use eSpeak as a segmentation engine
 
 ### Subtitles
+* Split long words if needed. This is especially important for Chinese
 * If a subtitle is too short and at the end of the audio, try to extend it back if possible (for example, if the previous subtitle is already extended, take back from it)
-* Split long words if needed
 * Decide how many punctuation characters to allow before breaking to a new line (currently it's infinite)
 * Add more clause separators, for even more special cases
 * Add option to output usable word or phoneme-level caption files (investigate how it's done on YouTube auto-captions)
@@ -154,10 +158,10 @@
 * Currently, when input is set to be SSML, it is wrapped in a `<speak>` tag. Handle the case where the user made their own SSML document wrapped with a `<speak>` tag as well. Currently, it may send invalid input to Azure
 
 ### Recognition
+* Recognized word entries that span VAD segment boundaries can be split
 * Show alternatives when playing in the CLI. Clear current line and rewrite already printed text for alternatives during the speech recognition process
 * Option to split recognized audio to segments or sentences, as is done with synthesized audio
 * Try to exclude the timing for trailing punctuation tokens in words that contain them. This can help narrow down the end timestamp to cover the word more tightly
-* Recognized word entries that span VAD boundaries may be split
 
 ### Recognition / Whisper
 * May get stuck in a token repeat loop when silence or non-speech segment encountered in audio. Decide what to do

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "1.0.0",
+	"version": "1.0.2",
 	"description": "An integrated speech toolset, providing a variety of synthesis, recognition, alignment, and other processing tools, designed to be directly accessible to end-users.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -90,11 +90,11 @@
 		"ndarray": "^1.0.19",
 		"ndarray-ops": "^1.2.2",
 		"onnxruntime-node": "^1.17.0",
-		"openai": "^4.33.0",
+		"openai": "^4.33.1",
 		"sam-js": "^0.2.1",
 		"strip-ansi": "^7.1.0",
 		"tar": "^7.0.0",
-		"tiktoken": "^1.0.13",
+		"tiktoken": "^1.0.14",
 		"tinyld": "^1.3.4",
 		"ws": "^8.16.0",
 		"wtf_wikipedia": "^10.3.0"
@@ -127,7 +127,7 @@
 		"@types/recursive-readdir": "^2.2.4",
 		"@types/tar": "^6.1.12",
 		"@types/ws": "^8.5.10",
-		"ts-json-schema-generator": "^1.5.1",
+		"ts-json-schema-generator": "^2.0.0",
 		"typescript": "^5.4.5"
 	}
 }

package/src/alignment/SpeechAlignment.ts

@@ -12,6 +12,7 @@ import { synthesize } from '../api/API.js'
 import { resampleAudioSpeex } from '../dsp/SpeexResampler.js'
 import { deepClone } from '../utilities/ObjectUtilities.js'
 import { zeroIfNaN } from '../math/VectorMath.js'
+import { EspeakOptions } from '../synthesis/EspeakTTS.js'
 
 export async function alignUsingDtw(
 	sourceRawAudio: RawAudio,
@@ -155,10 +156,9 @@ export async function alignUsingDtwWithRecognition(
 	referenceRawAudio: RawAudio,
 	referenceTimeline: Timeline,
 	recognitionTimeline: Timeline,
-	synthesizedRecognizedTranscriptRawAudio: RawAudio,
-	synthesizedRecognitionTimeline: Timeline,
 	granularities: DtwGranularity[],
 	windowDurations: number[],
+	espeakOptions: EspeakOptions,
 	phoneAlignmentMethod: API.PhoneAlignmentMethod = 'interpolation') {
 
 	const logger = new Logger()
@@ -182,13 +182,25 @@ export async function alignUsingDtwWithRecognition(
 		return interpolatedTimeline
 	}
 
+	// Synthesize the recognized transcript and get its timeline
+	logger.start("Synthesize recognized transcript with eSpeak")
+	const recognizedWords = recognitionTimeline.map(entry => entry.text)
+
+	const {
+		rawAudio: synthesizedRecognizedTranscriptRawAudio,
+		timeline: synthesizedRecognitionTimeline
+	} = await createAlignmentReferenceUsingEspeakForFragments(recognizedWords, espeakOptions)
+
 	let recognitionTimelineWithPhones: Timeline
 
 	if (phoneAlignmentMethod == 'interpolation') {
 		// Add phone timelines by interpolating from reference words
 		logger.start('Interpolate phone timing')
 
-		recognitionTimelineWithPhones = await interpolatePhoneTimelines(recognitionTimeline, synthesizedRecognitionTimeline)
+		recognitionTimelineWithPhones = await interpolatePhoneTimelines(
+			recognitionTimeline,
+			synthesizedRecognitionTimeline
+		)
 	} else if (phoneAlignmentMethod == 'dtw') {
 		logger.start('Align phone timing')
 
@@ -205,7 +217,7 @@ export async function alignUsingDtwWithRecognition(
 	}
 
 	// Create a mapping from the synthesized recognized timeline to the recognized timeline
-	logger.start('Map from the synthesized recognized timeline to the recognized timeline')
+	logger.start("Map from the synthesized recognized timeline to the recognized timeline")
 
 	type SynthesizedToRecognizedTimeMappingEntry = {
 		synthesized: number
@@ -217,16 +229,18 @@ export async function alignUsingDtwWithRecognition(
 	const synthesizedToRecognizedTimeMapping: SynthesizedToRecognizedTimeMapping = []
 
 	for (let wordEntryIndex = 0; wordEntryIndex < synthesizedRecognitionTimeline.length; wordEntryIndex++) {
-		const synthesizedWordTimelineEntry = synthesizedRecognitionTimeline[wordEntryIndex]
-		const recognitionWordTimelineEntry = recognitionTimelineWithPhones[wordEntryIndex]
+		const synthesizedTimelineEntry = synthesizedRecognitionTimeline[wordEntryIndex]
+		const recognitionTimelineEntry = recognitionTimelineWithPhones[wordEntryIndex]
 
-		for (let tokenEntryIndex = 0; tokenEntryIndex < synthesizedWordTimelineEntry.timeline!.length; tokenEntryIndex++) {
-			const synthesizedTokenTimelineEntry = synthesizedWordTimelineEntry.timeline![tokenEntryIndex]
-			const recognitionTokenTimelineEntry = recognitionWordTimelineEntry.timeline![tokenEntryIndex]
+		synthesizedToRecognizedTimeMapping.push({
+			synthesized: synthesizedTimelineEntry.startTime,
+			recognized: recognitionTimelineEntry.startTime
+		})
 
-			for (let phoneEntryIndex = 0; phoneEntryIndex < synthesizedTokenTimelineEntry.timeline!.length; phoneEntryIndex++) {
-				const synthesizedPhoneTimelineEntry = synthesizedTokenTimelineEntry.timeline![phoneEntryIndex]
-				const recognitionPhoneTimelineEntry = recognitionTokenTimelineEntry.timeline![phoneEntryIndex]
+		if (synthesizedTimelineEntry.timeline) {
+			for (let tokenEntryIndex = 0; tokenEntryIndex < synthesizedTimelineEntry.timeline.length; tokenEntryIndex++) {
+				const synthesizedPhoneTimelineEntry = synthesizedTimelineEntry.timeline[tokenEntryIndex]
+				const recognitionPhoneTimelineEntry = recognitionTimelineEntry.timeline![tokenEntryIndex]
 
 				synthesizedToRecognizedTimeMapping.push({
 					synthesized: synthesizedPhoneTimelineEntry.startTime,
@@ -239,10 +253,16 @@ export async function alignUsingDtwWithRecognition(
 				})
 			}
 		}
+
+		synthesizedToRecognizedTimeMapping.push({
+			synthesized: synthesizedTimelineEntry.endTime,
+			recognized: recognitionTimelineEntry.endTime
+		})
 	}
 
-	logger.start('Align the synthesized recognized transcript with the synthesized ground-truth transcript')
 	// Align the synthesized recognized transcript to the synthesized reference transcript
+	logger.start("Align the synthesized recognized transcript with the synthesized ground-truth transcript")
+
 	const alignedSynthesizedRecognitionTimeline = await alignUsingDtw(
 		synthesizedRecognizedTranscriptRawAudio,
 		referenceRawAudio,
@@ -250,41 +270,45 @@ export async function alignUsingDtwWithRecognition(
 		granularities,
 		windowDurations)
 
-	function mapTimeline(timeline: Timeline) {
-		function mapSynthesizedToRecognizedTime(synthesizedTime: number) {
-			for (let mappingIndex = 0; ; mappingIndex += 1) {
-				const left = synthesizedToRecognizedTimeMapping[mappingIndex].synthesized
+	let currentSynthesizedToRecognizedMappingIndex = 0
+
+	// Map from synthesized reference timestamps to the recognition timestamps
+	function mapSynthesizedToRecognizedTimeAndAdvance(synthesizedTime: number) {
+		for (; ; currentSynthesizedToRecognizedMappingIndex += 1) {
+			const left = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].synthesized
 
-				let right: number
+			let right: number
 
-				if (mappingIndex < synthesizedToRecognizedTimeMapping.length - 1) {
-					right = synthesizedToRecognizedTimeMapping[mappingIndex + 1].synthesized
-				} else {
-					right = Infinity
-				}
+			if (currentSynthesizedToRecognizedMappingIndex < synthesizedToRecognizedTimeMapping.length - 1) {
+				right = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex + 1].synthesized
+			} else {
+				right = Infinity
+			}
 
-				if (left > right) {
-					throw new Error('Left is greater than right!')
-				}
+			if (left > right) {
+				throw new Error("Left is larger than right!")
+			}
 
-				if (Math.abs(synthesizedTime - left) < Math.abs(synthesizedTime - right)) {
-					return synthesizedToRecognizedTimeMapping[mappingIndex].recognized
-				}
+			if (Math.abs(synthesizedTime - left) < Math.abs(synthesizedTime - right)) {
+				return synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].recognized
 			}
 		}
+	}
 
+	function mapTimeline(timeline: Timeline) {
 		const mappedTimeline: Timeline = []
 
 		for (const entry of timeline) {
-			const mappedEntry = deepClone(entry)
+			const mappedEntry = { ...entry }
 
-			mappedEntry.startTime = mapSynthesizedToRecognizedTime(entry.startTime)
-			mappedEntry.endTime = mapSynthesizedToRecognizedTime(entry.endTime)
+			mappedEntry.startTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.startTime)
 
 			if (entry.timeline) {
 				mappedEntry.timeline = mapTimeline(entry.timeline)
 			}
 
+			mappedEntry.endTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.endTime)
+
 			mappedTimeline.push(mappedEntry)
 		}
 
@@ -301,10 +325,10 @@ export async function alignUsingDtwWithRecognition(
 export async function interpolatePhoneTimelines(sourceTimeline: Timeline, referenceTimeline: Timeline) {
 	const interpolatedTimeline: Timeline = []
 
-	for (let i = 0; i < sourceTimeline.length; i++) {
-		const referenceEntry = referenceTimeline[i]
+	for (let wordEntryIndex = 0; wordEntryIndex < sourceTimeline.length; wordEntryIndex++) {
+		const referenceEntry = referenceTimeline[wordEntryIndex]
 
-		const interpolatedEntry = deepClone(sourceTimeline[i])
+		const interpolatedEntry = deepClone(sourceTimeline[wordEntryIndex])
 		interpolatedTimeline.push(interpolatedEntry)
 
 		if (interpolatedEntry.type != 'word') {
@@ -335,23 +359,13 @@ export async function interpolatePhoneTimelines(sourceTimeline: Timeline, refere
 			}
 		}
 
-		const interpolatedTokenEntries: Timeline = []
-
-		for (const tokenEntry of (referenceEntry.timeline || [])!) {
-			const interpolatedTokenEntry = mapEntry(tokenEntry)
-
-			const interpolatedPhoneEntries: Timeline = []
-
-			for (const phoneEntry of (tokenEntry.timeline || [])) {
-				interpolatedPhoneEntries.push(mapEntry(phoneEntry))
-			}
-
-			interpolatedTokenEntry.timeline = interpolatedPhoneEntries
+		const interpolatedPhoneEntries: Timeline = []
 
-			interpolatedTokenEntries.push(interpolatedTokenEntry)
+		for (const phoneEntry of (referenceEntry.timeline || [])) {
+			interpolatedPhoneEntries.push(mapEntry(phoneEntry))
 		}
 
-		interpolatedEntry.timeline = interpolatedTokenEntries
+		interpolatedEntry.timeline = interpolatedPhoneEntries
 	}
 
 	return interpolatedTimeline
@@ -424,23 +438,13 @@ export async function alignPhoneTimelines(
 		}
 
 		// Add phone timeline using the mapped time information
-		const alignedTokenTimeline: Timeline = []
+		const alignedPhoneTimeline: Timeline = []
 
-		for (const referenceTokenEntry of (referenceWordEntry.timeline || [])) {
-			const alignedPhoneTimeline: Timeline = []
-
-			for (const referencePhoneEntry of (referenceTokenEntry.timeline || [])) {
-				alignedPhoneTimeline.push(mapEntry(referencePhoneEntry))
-			}
-
-			alignedTokenTimeline.push({
-				...mapEntry(referenceTokenEntry),
-
-				timeline: alignedPhoneTimeline
-			})
+		for (const referencePhoneEntry of (referenceWordEntry.timeline || [])) {
+			alignedPhoneTimeline.push(mapEntry(referencePhoneEntry))
 		}
 
-		alignedWordEntry.timeline = alignedTokenTimeline
+		alignedWordEntry.timeline = alignedPhoneTimeline
 
 		alignedWordTimeline.push(alignedWordEntry)
 	}
@@ -448,6 +452,27 @@ export async function alignPhoneTimelines(
 	return alignedWordTimeline
 }
 
+export async function createAlignmentReferenceUsingEspeakForFragments(fragments: string[], espeakOptions: EspeakOptions, insertSeparators = true) {
+	const progressLogger = new Logger()
+
+	progressLogger.start("Load espeak module")
+	const Espeak = await import("../synthesis/EspeakTTS.js")
+
+	progressLogger.start("Create alignment reference with eSpeak")
+
+	const result = await Espeak.synthesizeFragments(fragments, espeakOptions)
+
+	result.timeline = result.timeline.flatMap(clause => clause.timeline!)
+
+	for (const wordEntry of result.timeline) {
+		wordEntry.timeline = wordEntry.timeline!.flatMap(tokenEntry => tokenEntry.timeline!)
+	}
+
+	progressLogger.end()
+
+	return result
+}
+
 export async function createAlignmentReferenceUsingEspeak(transcript: string, language: string, plaintextOptions?: API.PlainTextOptions, customLexiconPaths?: string[], insertSeparators?: boolean) {
 	const logger = new Logger()