echogarden 1.4.4 → 1.6.0

package/docs/Options.md

@@ -141,24 +141,24 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 * `sourceSeparation`: prefix to provide options for source separation when `isolate` is set to `true`. Options detailed in section for source separation
 
 **Whisper**:
-* `whisper.model`: selects which Whisper model to use. Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large` (same as `large-v2`), `large-v1`, `large-v2`, `large-v3` (**Note**: large models aren't yet supported by `onnxruntime-node` due to their size). Defaults to `tiny` or `tiny.en`
+* `whisper.model`: selects which Whisper model to use. Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en` or `large-v3-turbo`. Defaults to `tiny` or `tiny.en`
 * `whisper.temperature`: temperature setting for the text decoder. Impacts the amount of randomization for token selection. It is recommended to leave at `0.1` (close to no randomization - almost always chooses the top ranked token) or choose a relatively low value (`0.25` or lower) for best results. Defaults to `0.1`
 * `whisper.prompt`: initial text to give the Whisper model. Can be a vocabulary, or example text of some sort. Note that if the prompt is very similar to the transcript, the model may intentionally avoid producing the transcript tokens as it may assume that they have already been transcribed. Optional
 * `whisper.topCandidateCount`: the number of top candidate tokens to consider. Defaults to `5`
-* `whisper.punctuationThreshold`: the minimal probability for a punctuation token, included in the top candidates, to be chosen unconditionally. A lower threshold encourages the model to output more punctuation symbols. Defaults to `0.2`
-* `whisper.autoPromptParts`: use previous part's recognized text as the prompt for the next part. Disabling this may help to prevent repetition carrying over between parts, in some cases. Defaults to `true`
+* `whisper.punctuationThreshold`: the minimal probability for a punctuation token, included in the top candidates, to be chosen unconditionally. A lower threshold encourages the model to output more punctuation characters. Defaults to `0.2`
+* `whisper.autoPromptParts`: use previous part's recognized text as the prompt for the next part. Disabling this may help to prevent repetition carrying over between parts, in some cases. Defaults to `true` (**Note**: currently always disabled for `large-v3-turbo` model due to an apparent issue with corrupt output when prompted)
 * `whisper.maxTokensPerPart`: maximum number of tokens to decode for each audio part. Defaults to `250`
 * `whisper.suppressRepetition`: attempt to suppress decoding of repeating token patterns. Defaults to `true`
 * `whisper.repetitionThreshold`: minimal repetition / compressibility score to cause a part not to be auto-prompted to the next part. Defaults to `2.4`
 * `whisper.decodeTimestampTokens`: enable/disable decoding of timestamp tokens. Setting to `false` can reduce the occurrence of hallucinations and token repetition loops, possibly due to the overall reduction in the number of tokens decoded. This has no impact on the accuracy of timestamps, since they are derived independently using cross-attention weights. However, there are cases where this can cause the model to end a part prematurely, especially in singing and less speech-like voice segments, or when there are multiple speakers. Defaults to `true`
-* `whisper.encoderProvider`: identifier for the ONNX execution provider to use with the encoder model. Can be `cpu` or `dml` ([DirectML](https://microsoft.github.io/DirectML/)-based GPU acceleration - Windows only). In general, GPU-based encoding should be significantly faster. Defaults to `cpu`, or `dml` if available
-* `whisper.decoderProvider`: identifier for the ONNX execution provider to use with the decoder model. Can be `cpu` or `dml` (Windows only). Using GPU acceleration for the decoder may be faster than CPU, especially for larger models, but that depends on your particular combination of CPU and GPU. Defaults to `cpu`
+* `whisper.encoderProvider`: identifier for the ONNX execution provider to use with the encoder model. Can be `cpu`, `dml` ([DirectML](https://microsoft.github.io/DirectML/)-based GPU acceleration - Windows only) or `cuda` (Linux only). In general, GPU-based encoding should be significantly faster. Defaults to `cpu`, or `dml` if available
+* `whisper.decoderProvider`: identifier for the ONNX execution provider to use with the decoder model. Can be `cpu`, `dml` (Windows only) or `cuda` (Linux only). Using GPU acceleration for the decoder may be faster than CPU, especially for larger models, but that depends on your particular combination of CPU and GPU. Defaults to `cpu`, and on Windows, `dml` if available for larger models (`small`, `medium`, `large`)
 * `whisper.seed`: provide a custom random seed for token selection when temperature is greater than 0. Uses a constant seed by default to ensure reproducibility
 
 **Whisper.cpp**:
-* `whisperCpp.model`: selects which `whisper.cpp` model to use.  Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large` (same as `large-v2`), `large-v1`, `large-v2`, `large-v3`. These quantized models are also supported: `tiny-q5_1`, `tiny.en-q5_1`, `tiny.en-q8_0`,`base-q5_1`, `base.en-q5_1`, `small-q5_1`, `small.en-q5_1`, `medium-q5_0`, `medium.en-q5_0`, `large-v2-q5_0`, `large-v3-q5_0`. Defaults to `base` or `base.en`
-* `whisperCpp.executablePath`: custom `whisper.cpp` executable path (currently required for macOS)
-* `whisperCpp.build`: type of `whisper.cpp` build to use. Can be set `cpu`, `cublas-11.8.0`, `cublas-12.4.0`. By default, builds are auto-selected and downloaded for Windows x64 (`cpu`, `cublas-11.8.0`, `cublas-12.4.0`) and Linux x64 (`cpu`). Using other builds requires providing a custom `executablePath`
+* `whisperCpp.model`: selects which `whisper.cpp` model to use.  Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large` (same as `large-v2`), `large-v1`, `large-v2`, `large-v3`. These quantized models are also supported: `tiny-q5_1`, `tiny.en-q5_1`, `tiny.en-q8_0`,`base-q5_1`, `base.en-q5_1`, `small-q5_1`, `small.en-q5_1`, `medium-q5_0`, `medium.en-q5_0`, `large-v2-q5_0`, `large-v3-q5_0`, `large-v3-turbo`, `large-v3-turbo-q5_0`. Defaults to `base` or `base.en`
+* `whisperCpp.executablePath`: a path to a custom `whisper.cpp` `main` executable (currently required for macOS)
+* `whisperCpp.build`: type of `whisper.cpp` build to use. Can be set to `cpu` or `cublas-12.4.0`. By default, builds are auto-selected and downloaded for Windows x64 (`cpu`, `cublas-12.4.0`) and Linux x64 (`cpu`). Using other builds requires providing a custom `executablePath`
 * `whisperCpp.threadCount`: number of threads to use, defaults to `4`
 * `whisperCpp.splitCount`: number of splits of the audio data to process in parallel (called `--processors` in the `whisper.cpp` CLI). A value greater than `1` can increase memory use significantly, reduce timing accuracy, and slow down execution in some cases. Defaults to `1` (highly recommended)
 * `whisperCpp.enableGPU`: enable GPU processing. Setting to `true` will try to use a CUDA build, if available for your system. Defaults to `true` when a CUDA-enabled build is selected via `whisperCpp.build`, otherwise `false`
@@ -166,7 +166,7 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 * `whisperCpp.beamCount`: the number of decoding paths to use during beam search. Defaults to `5`
 * `whisperCpp.repetitionThreshold`: minimal repetition / compressibility score to cause a decoded segment to be discarded. Defaults to `2.4`
 * `whisperCpp.prompt`: initial text to give the Whisper model. Can be a vocabulary, or example text of some sort. Note that if the prompt is very similar to the transcript, the model may intentionally avoid producing the transcript tokens as it may assume that they have already been transcribed. Optional
-* `whisperCpp.enableDTW`: enable experimental `whisper.cpp` internal DTW-based token alignment to be used to derive timestamps. Defaults to `false` (recommended for now)
+* `whisperCpp.enableDTW`: enable `whisper.cpp`'s own experimental DTW-based token alignment to be used to derive timestamps. Defaults to `false` (highly recommended)
 * `whisperCpp.verbose`: show all CLI messages during execution. Defaults to `false`
 
 **Vosk**:
@@ -194,9 +194,9 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 
 **OpenAI Cloud**:
 * `openAICloud.apiKey`: API key (required)
-* `openAICloud.model`: model to use. Can only be `whisper-1`
+* `openAICloud.model`: model to use. When using the default provider (OpenAI), can only be `whisper-1`. For a custom provider, like Groq, see its documentation
 * `openAICloud.organization`: organization identifier. Optional
-* `openAICloud.baseURL`: override the default base URL used by the API. Optional
+* `openAICloud.baseURL`: override the default base URL used by the API. For example, set `https://api.groq.com/openai/v1` to use Groq's OpenAI compatible Whisper API instead. Optional
 * `openAICloud.temperature`: temperature. Choosing `0` uses a dynamic temperature approach. Defaults to `0`
 * `openAICloud.prompt`: initial prompt for the model. Optional
 * `openAICloud.timeout`: request timeout. Optional
@@ -265,13 +265,23 @@ Applies to CLI operation: `translate-speech`, API method: `translateSpeech`
 
 * `openAICloud`: prefix to provide options for OpenAI cloud. Same options as detailed in the recognition section above
 
+## Text-to-text translation
+
+Applies to CLI operation: `translate-text`, API method: `translateText`
+
+* `engine`: only `google-translate` supported
+* `sourceLanguage`: the source language code for the input text. Auto-detected if not set
+* `targetLanguage`: the target language code for the output text. Required
+* `languageDetection`: language detection options. Optional
+
 ## Speech-to-translated-transcript alignment
 
 Applies to CLI operation: `align-translation`, API method: `alignTranslation`
 
 **General**:
 * `engine`: alignment algorithm to use, can only be `whisper`. Defaults to `whisper`
-* `language`: language code for the source audio ([ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)), like `en`, `fr`, `zh`, etc. Auto-detected from audio if not set
+* `sourceLanguage`: language code for the source audio ([ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)), like `en`, `fr`, `zh`, etc. Auto-detected from audio if not set
+* `targetLanguage`: language code for the translated transcript. Can only be `en` for now. Defaults to `en`
 * `crop`: crop to active parts using voice activity detection before starting. Defaults to `true`
 * `isolate`: apply source separation to isolate voice before starting alignment. Defaults to `false`
 * `subtitles`: prefix to provide options for subtitles. Options detailed in section for subtitles
@@ -284,6 +294,37 @@ Applies to CLI operation: `align-translation`, API method: `alignTranslation`
 * `whisper.encoderProvider`: encoder ONNX execution provider. See details in recognition section above
 * `whisper.decoderProvider`: decoder ONNX execution provider. See details in recognition section above
 
+## Speech-to-transcript-and-translation alignment
+
+Applies to CLI operation: `align-transcript-and-translation`, API method: `alignTranscriptAndTranslation`
+
+**General**:
+* `engine`: can only be `two-stage`. Defaults to `two-stage`
+* `sourceLanguage`: language code for the source audio ([ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)), like `en`, `fr`, `zh`, etc. Auto-detected from audio if not set
+* `targetLanguage`: language code for the translated transcript. Can only be `en` for now. Defaults to `en`
+* `crop`: crop to active parts using voice activity detection before starting. Defaults to `true`
+* `isolate`: apply source separation to isolate voice before starting alignment. Defaults to `false`
+* `alignment`: prefix to provide options for alignment. Options detailed in section for alignment
+* `timelineAlignment`: prefix to provide options for timeline alignment. Options detailed in section for timeline alignment
+* `vad`: prefix to provide options for voice activity detection when `crop` is set to `true`. Options detailed in section for voice activity detection
+* `sourceSeparation`: prefix to provide options for source separation when `isolate` is set to `true`. Options detailed in section for source separation
+* `subtitles`: prefix to provide options for subtitles. Options detailed in section for subtitles
+
+## Timeline-to-translated-text alignment
+
+Applies to CLI operation: `align-timeline-translation`, API method: `alignTimelineTranslation`
+
+**General**:
+* `engine`: alignment engine to use. Can only be `e5`. Defaults to `e5`
+* `sourceLanguage`: language code for the source timeline. Auto-detected from timeline if not set
+* `targetLanguage`: language code for the translated transcript. Auto-detected if not set
+* `audio`: spoken audio to play when previewing the result in the CLI (not required or used by the alignment itself). Optional
+* `languageDetection`: prefix to provide options for language detection. Options detailed in section for text language detection
+* `subtitles`: prefix to provide options for subtitles. Options detailed in section for subtitles
+
+**E5**:
+* `e5.model`: E5 model to use. Defaults to `e5-small-fp16` (support for additional models will be added in the future)
+
 ## Language detection
 
 ### Speech language detection

package/docs/Tasklist.md

@@ -2,11 +2,6 @@
 
 ## Bugs
 
-### Alignment / DTW-RA
-
-* In DTW-RA, a recognized transcript including something like "Question 2.What does Juan", where "2.What" has a point in the middle, is breaking playback of the timeline
-
-### Synthesis
 
 ### eSpeak
 
@@ -24,6 +19,7 @@
 * `espeak-ng`: 'Oh dear!”' is read as "oh dear exclamation mark", because of the special quote character following the exclamation mark
 * `espeak-ng`: [Marker right after sentence end is not reported as an event](https://github.com/espeak-ng/espeak-ng/issues/920)
 * `espeak-ng`: On Japanese text, it says "Chinese character" or "Japanese character" for characters it doesn't know
+* `espeak-ng`: Broken markers on the Korean voice
 * `wtf_wikipedia` Sometimes fails on `getResult.js` without throwing a humanly readable error
 * `wtf_wikipedia` Sometimes captures markup like `.svg` etc.
 * `msspeech`: Initialization fails on Chinese and Japanese voices (but not Korean)
@@ -255,15 +251,7 @@
 * Allow `dtw` mode work with more speech synthesizers to produce its reference
 * Predict timing for individual letters (graphemes) based on phoneme timestamps (especially useful for Chinese and Japanese)
 
-### Translation
-* Add text-to-text translation with cloud translation APIs like Google Translate or DeepL, or offline models like OpenNMT or  NLLB-200
-
-### Translation alignment
-* Perform word-level alignment of text-to-text translations from and to any language (not just English) using methods like multilingual embeddings, or specialized models, and then use the text-based alignment to align speech in any recognized language, to its translated transcript in any language supported by the text-to-text alignment approach
-
-### Speech-to-text translation
 
-* Hybrid approach: recognize speech in its native language using any recognition model, then translate the resulting transcript using a text-to-text translation engine, and then align the translated transcript to the original one using text-to-text alignment, and map back to the original speech using the recognition timestamps, to get word-level alignment for the translated transcript
 
 ## Possible new engines or platforms
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "1.4.4",
+	"version": "1.6.0",
 	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -55,8 +55,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "^3.576.0",
-		"@aws-sdk/client-transcribe-streaming": "^3.576.0",
+		"@aws-sdk/client-polly": "^3.664.0",
+		"@aws-sdk/client-transcribe-streaming": "^3.664.0",
 		"@echogarden/espeak-ng-emscripten": "^0.1.2",
 		"@echogarden/fasttext-wasm": "^0.1.0",
 		"@echogarden/flite-wasi": "^0.1.1",
@@ -74,42 +74,38 @@
 		"chalk": "^5.3.0",
 		"cldr-segmentation": "^2.2.1",
 		"command-exists": "^1.2.9",
-		"compromise": "^14.13.0",
+		"compromise": "^14.14.0",
 		"fs-extra": "^11.2.0",
-		"gaxios": "^6.5.0",
+		"gaxios": "^6.7.1",
 		"graceful-fs": "^4.2.11",
 		"html-escaper": "^3.0.3",
 		"html-to-text": "^9.0.5",
 		"import-meta-resolve": "^4.1.0",
-		"jieba-wasm": "^0.0.2",
-		"jsdom": "^24.0.0",
+		"jieba-wasm": "^2.1.1",
+		"jsdom": "^25.0.1",
 		"json5": "^2.2.3",
 		"kuromoji": "^0.1.2",
-		"microsoft-cognitiveservices-speech-sdk": "^1.36.0",
+		"microsoft-cognitiveservices-speech-sdk": "^1.40.0",
 		"moving-median": "^1.0.0",
 		"msgpack-lite": "^0.1.26",
-		"onnxruntime-node": "^1.17.3",
-		"openai": "^4.47.1",
-		"sam-js": "^0.2.1",
+		"onnxruntime-node": "^1.19.2",
+		"openai": "^4.67.1",
+		"sam-js": "^0.3.1",
 		"strip-ansi": "^7.1.0",
-		"tar": "^7.1.0",
-		"tiktoken": "^1.0.15",
+		"tar": "^7.4.3",
+		"tiktoken": "^1.0.16",
 		"tinyld": "^1.3.4",
-		"ws": "^8.17.0",
-		"wtf_wikipedia": "^10.3.1"
+		"ws": "^8.18.0",
+		"wtf_wikipedia": "^10.3.2"
 	},
 	"peerDependencies": {
 		"@echogarden/vosk": "^0.3.39-patched.1",
-		"speaker": "^0.5.5",
 		"winax": "^3.4.2"
 	},
 	"peerDependenciesMeta": {
 		"@echogarden/vosk": {
 			"optional": true
 		},
-		"speaker": {
-			"optional": true
-		},
 		"winax": {
 			"optional": true
 		}
@@ -118,13 +114,13 @@
 		"@types/buffer-split": "^1.0.2",
 		"@types/fs-extra": "^11.0.4",
 		"@types/graceful-fs": "^4.1.9",
-		"@types/jsdom": "^21.1.6",
+		"@types/jsdom": "^21.1.7",
 		"@types/msgpack-lite": "^0.1.11",
-		"@types/node": "^20.12.12",
+		"@types/node": "^22.7.4",
 		"@types/recursive-readdir": "^2.2.4",
 		"@types/tar": "^6.1.13",
-		"@types/ws": "^8.5.10",
-		"ts-json-schema-generator": "^2.1.2-next.1",
-		"typescript": "^5.4.5"
+		"@types/ws": "^8.5.12",
+		"ts-json-schema-generator": "^2.3.0",
+		"typescript": "^5.6.2"
 	}
 }

package/src/alignment/DTWMfccSequenceAlignment.ts

@@ -1,15 +1,15 @@
-import { cosineDistancePrecomputedMagnitudes, createVectorForIntegerRange, euclidianDistance, euclidianDistance13Dim, magnitude } from '../math/VectorMath.js'
+import { cosineDistancePrecomputedMagnitudes, createVectorForIntegerRange, euclideanDistance, euclideanDistance13Dim, magnitude } from '../math/VectorMath.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js'
 
 const log = logToStderr
 
-export async function alignMFCC_DTW(mfccFrames1: number[][], mfccFrames2: number[][], windowLength: number, distanceFunctionKind: 'euclidian' | 'cosine' = 'euclidian', centerIndexes?: number[]) {
-	if (distanceFunctionKind == 'euclidian') {
-		let distanceFunction = euclidianDistance
+export async function alignMFCC_DTW(mfccFrames1: number[][], mfccFrames2: number[][], windowLength: number, distanceFunctionKind: 'euclidean' | 'cosine' = 'euclidean', centerIndexes?: number[]) {
+	if (distanceFunctionKind == 'euclidean') {
+		let distanceFunction = euclideanDistance
 
 		if (mfccFrames1.length > 0 && mfccFrames1[0].length === 13) {
-			distanceFunction = euclidianDistance13Dim
+			distanceFunction = euclideanDistance13Dim
 		}
 
 		const { path } = alignDTWWindowed(

package/src/alignment/DTWSequenceAlignmentWindowed.ts

@@ -4,9 +4,7 @@ import { AlignmentPath } from './SpeechAlignment.js'
 const log = logToStderr
 
 export function alignDTWWindowed<T, U>(sequence1: T[], sequence2: U[], costFunction: (a: T, b: U) => number, windowMaxLength: number, centerIndexes?: number[]) {
-	if (windowMaxLength < 2) {
-		throw new Error('Window length must be greater or equal to 2')
-	}
+	windowMaxLength = Math.max(windowMaxLength, 2)
 
 	if (sequence1.length == 0 || sequence2.length == 0) {
 		return {