echogarden 2.10.2 → 3.0.0

package/docs/Engines.md

@@ -16,7 +16,6 @@
 **Offline, Windows only**:
 
 * [SAPI](https://en.wikipedia.org/wiki/Microsoft_Speech_API) (`sapi`): Microsoft Speech API. Supports the system's language voices, as well as legacy voices produced by third-party vendors, like Ivona, NeoSpeech, Acapela, Cepstral, CereProc, Nuance, AT&T, Loquendo, ScanSoft and others (note that only 64-bit SAPI voices are supported, which makes it incompatible with a significant portion of older voices)
-
 * [Microsoft Speech Platform](https://www.microsoft.com/en-us/download/details.aspx?id=27225) (`msspeech`): Microsoft Server Speech API. Requires [installing a runtime (2.6MB)](https://www.microsoft.com/en-us/download/details.aspx?id=27225). Supports 28 dialects, which can be individually downloaded via [freely available installers](https://www.microsoft.com/en-us/download/details.aspx?id=27224), or, for convenience, bundled as [a single 358MB zip file](https://drive.google.com/u/0/uc?id=1uQdFNxLzUxpaEwVVKhMawys8cIh3F21T&export=download). Has voices for English (US, UK, AU, CA), Spanish (ES, MX), Portuguese (BR, PT), German, French (FR, CA), Italian, Norwegian, Dutch, Russian, Swedish, Danish, Catalan, Finnish, Japanese, Korean and Chinese (ZH, HK, TW). All voices are female
 
 **Note**: both these engines require manually installing the [`winax` npm package](https://www.npmjs.com/package/winax) by running `npm install winax -g`.
@@ -44,15 +43,13 @@ These cloud-based engines connect to public cloud APIs that are not officially p
 
 * Google Translate (`google-translate`): used by the [Google Translate web UI](https://translate.google.com/) to speak written text in any one of its supported languages. Offers a single voice for each language (usually female)
 * Microsoft Edge (`microsoft-edge`): subset of the Azure Cognitive Services cloud TTS API used by the Microsoft Edge browser as part of its support for the [Web Speech API](https://developer.mozilla.org/en-US/docs/Web/API/Web_Speech_API) and its [Read Aloud](https://www.microsoft.com/en-us/edge/features/read-aloud?form=MT00D8) feature. Using this engine requires a special token, which should be passed via the `microsoftEdge.trustedClientToken` option
-* Streamlabs Polly (`streamlabs-polly`): a public REST API by Streamlabs, primarily intended for generating speech for TTS donations. It includes a few English (US, UK, AU, IN) voices, which are similar to some of the non-neural (Ivona-based) voices offered by Amazon Polly (**Note**: as of April 2024, the public Streamlabs Polly REST API doesn't seem to be accessible anymore)
 
 ## Speech-to-text
 
 **Offline**:
-* [OpenAI Whisper](https://github.com/openai/whisper) (`whisper`): high-accuracy transformer-based speech recognition architecture. TypeScript implementation, with inference done via the [ONNX runtime](https://onnxruntime.ai/). Supports [98 languages](https://platform.openai.com/docs/guides/speech-to-text/supported-languages). There are several models of different sizes, some are multilingual, and some are English only: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large-v3-turbo`. **Note**: Unquantized large models are not currently supported by `onnxruntime-node` due to model size restrictions
-* [Whisper.cpp](https://github.com/ggerganov/whisper.cpp) (`whisper.cpp`): a C++ port of the Whisper architecture by Georgi Gerganov. Supports all Whisper models, including several quantized ones (see full model list in the [options reference](docs/Options.md)). Has various builds, including CUDA and OpenCL for GPU support
-* [Vosk](https://github.com/alphacep/vosk-api) (`vosk`): models available for 25+ languages. **Note**: the Vosk package is not included in the default installation, but you can add support for it using `npm install @echogarden/vosk -g`. Then, you'll need to manually [download a model](https://alphacephei.com/vosk/models) and specify its directory path via the `vosk.modelPath` option
-* [Silero](https://github.com/snakers4/silero-models) (`silero`): models available for English, Spanish, German and Ukrainian. For [non-commercial use only](https://github.com/snakers4/silero-models/blob/master/LICENSE)
+
+* [OpenAI Whisper](https://github.com/openai/whisper) (`whisper`): high-accuracy transformer-based speech recognition architecture. TypeScript implementation. Since `v3.0.0`, it internally uses a minimal NAPI binding to a small set of `whisper.cpp` operations via its C API (before `v3.0.0` it used a much slower ONNX-based implementation)
+* [Whisper.cpp](https://github.com/ggerganov/whisper.cpp) (`whisper.cpp`): uses the CLI (command line interface) of `whisper.cpp`, a C++ port of the Whisper architecture by Georgi Gerganov
 
 **Cloud services**:
 
@@ -75,10 +72,12 @@ These engines' goal is to match (or "align") a given spoken recording with a giv
 ## Speech-to-text translation
 
 **Offline**:
+
 * [Whisper](https://github.com/openai/whisper) (`whisper`): the Whisper model can recognize speech in any one of its supported languages and output a transcript directly translated to English. Other languages are not supported as targets
-* [Whisper.cpp](https://github.com/ggerganov/whisper.cpp) (`whisper.cpp`): supports translation to English
+* [Whisper.cpp](https://github.com/ggerganov/whisper.cpp) (`whisper.cpp`): uses `whisper.cpp` CLI. Supports translation to English only
 
 **Cloud services**:
+
 * [OpenAI Cloud Platform](https://platform.openai.com/) (`openai-cloud`): runs the `large-v2` Whisper model on the cloud. Only supports English as target
 
 ## Text-to-text translation
@@ -121,10 +120,12 @@ Another use case it to take a timeline produced as part of synthesized or recogn
 ## Language detection
 
 **Spoken language detection**:
+
 * [Whisper](https://github.com/openai/whisper) (`whisper`): uses the language token produced by the `whisper` speech recognition model to generate a set of probabilities for the [98 languages](https://platform.openai.com/docs/guides/speech-to-text/supported-languages) it has been trained on
 * [Silero Language Classifier](https://github.com/snakers4/silero-vad/wiki/Other-Models) (`silero`): a speech language classification model by Silero
 
 **Text language detection**:
+
 * [TinyLD](https://www.npmjs.com/package/tinyld) (`tinyld`): a simple language detection library
 * [FastText](https://github.com/facebookresearch/fastText) (`fasttext`): a library for word representations and sentence classification by Facebook research
 

package/docs/Licenses.md

@@ -8,7 +8,6 @@
 * `pico`: [Apache License 2.0](https://github.com/gmn/nanotts/blob/master/LICENSE)
 * `gnuspeech`: [GNU GPL v3](https://github.com/mym-br/gnuspeech_sa/blob/master/COPYING.txt)
 * `sam`: [Abandonware / fair-use / unknown](https://github.com/discordier/sam#license)
-* `vosk` [Apache License 2.0](https://github.com/alphacep/vosk-api/blob/master/COPYING)
 * `tinyld` [MIT License](https://github.com/komodojp/tinyld/blob/develop/license)
 * `fasttext`: [MIT License](https://github.com/loretoparisi/fasttext.js/blob/master/LICENSE)
 * `fvad` [BSD-3-Clause](https://github.com/cpuimage/WebRTC_VAD/blob/master/LICENSE)
@@ -31,7 +30,6 @@ All are freely distributable, with varying licenses:
 * Flite voices (`flite-`): [BSD License](https://github.com/festvox/flite/blob/master/COPYING)
 * SVOX Pico resources (`pico-`): [Apache License 2.0](https://github.com/gmn/nanotts/blob/master/LICENSE)
 * Silero VAD (`silero-vad`) and Silero language classifier (`silero-lang-classifier-95`): [MIT License](https://github.com/snakers4/silero-vad/blob/master/LICENSE)
-* Silero speech recognition models (`silero-en-`, `silero-de-`, `silero-es-`, `silero-ua-`): [BY-NC-SA](https://github.com/snakers4/silero-models/blob/master/LICENSE)
 * VITS pre-trained models (`vits-`): licensed under various creative commons licenses: [CC0](https://creativecommons.org/share-your-work/public-domain/cc0/), [CC-BY](https://creativecommons.org/licenses/by/4.0/) and [BY-NC-SA](https://creativecommons.org/licenses/by-nc-sa/4.0/), and few are public domain. You can view the individual license for each model in the model cards on the [Piper samples page](https://rhasspy.github.io/piper-samples/)
 * Whisper pre-trained models (`whisper-`): [MIT License](https://github.com/openai/whisper/blob/main/LICENSE)
 * MDX-NET source separation models (`mdxnet-`): [MIT License](https://github.com/kuielab/mdx-net/blob/main/LICENSE)
@@ -39,5 +37,4 @@ All are freely distributable, with varying licenses:
 
 Tool binary distributions
 * FFmpeg: [LGPL, GPL v2 and GPL v3 Licenses](https://github.com/FFmpeg/FFmpeg)
-* SoX: [GPL v2 License](https://github.com/chirlu/sox/blob/master/LICENSE.GPL)
 * whisper.cpp: [MIT License](https://github.com/ggerganov/whisper.cpp/blob/master/LICENSE)

package/docs/Options.md

@@ -12,7 +12,7 @@ Here's a detailed reference for all the options accepted by the Echogarden CLI a
 Applies to CLI operations: `speak`, `speak-file`, `speak-url`, `speak-wikipedia`, API method: `synthesize`
 
 **General**:
-* `engine`: identifier of the synthesis engine to use. Can be `kokoro`, `vits`, `pico`, `flite`, `gnuspeech`, `espeak`, `sam`, `sapi`, `msspeech`, `coqui-server`, `google-cloud`, `microsoft-azure`, `amazon-polly`, `openai-cloud`, `elevenlabs`, `google-translate`, `microsoft-edge` or `streamlabs-polly` (see [the engines documentation page](Engines.md) for more detailed information about each engine). Auto-selected if not set
+* `engine`: identifier of the synthesis engine to use. Can be `kokoro`, `vits`, `pico`, `flite`, `gnuspeech`, `espeak`, `sam`, `sapi`, `msspeech`, `coqui-server`, `google-cloud`, `microsoft-azure`, `amazon-polly`, `openai-cloud`, `elevenlabs`, `google-translate` or `microsoft-edge` (see [the engines documentation page](Engines.md) for more detailed information about each engine). Auto-selected if not set
 * `language`: language code, can be ([ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)), like `en`, `fr`, `en-US`, `pt-BR`, etc. [ISO 639-3](https://en.wikipedia.org/wiki/ISO_639-3), or a plain language name like `french`. Auto-detected if not set
 * `voice`: name of the voice to use. Can be a search string. Auto-selected if not set
 * `voiceGender`: gender of the voice to use. Optional
@@ -146,7 +146,7 @@ Also accepted are the following engine-specific options that may be required in
 Applies to CLI operation: `transcribe`, API method: `recognize`
 
 **General**:
-* `engine`: identifier of the recognition engine to use, can be `whisper`, `whisper.cpp`, `vosk` or `silero` (see [the full engine list](Engines.md) for more detailed information)
+* `engine`: identifier of the recognition engine to use, can be `whisper`, `whisper.cpp`, `google-cloud`, `microsoft-azure`, `amazon-transcribe`, `openai-cloud`, `deepgram` (see [the full engine list](Engines.md) for more detailed information)
 * `language`: language code ([ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)) for the audio, like `en`, `fr`, `de`. Auto-detected if not set
 * `crop`: crop to active parts using voice activity detection before starting recognition. Defaults to `true`
 * `isolate`: apply source separation to isolate voice before starting recognition. Defaults to `false`
@@ -157,7 +157,10 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 * `sourceSeparation`: prefix to provide options for source separation when `isolate` is set to `true`. Options detailed in section for source separation
 
 **Whisper**:
-* `whisper.model`: selects which Whisper model to use. Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en` or `large-v3-turbo`. Defaults to `tiny` or `tiny.en`
+* `whisper.model`: selects which Whisper model to use. Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large` (same as `large-v2`), `large-v1`, `large-v2`, `large-v3`, `large-v3-turbo`. The following quantized models are also supported: `tiny-q5_1`,`tiny-q8_0`, `tiny.en-q5_1`, `tiny.en-q8_0`,`base-q5_1`, `base-q8_0`, `base.en-q5_1`, `base.en-q8_0`,`small-q5_1`, `small.en-q5_1`,  `small.en-q8_0`,`medium-q5_0`, `medium.en-q5_0`, `medium.en-q8_0`, `large-v2-q5_0`, `large-v2-q8_0`, `large-v3-q5_0`, `large-v3-turbo-q5_0`, `large-v3-turbo-q8_0`. Defaults to `base` or `base.en` for English
+* `whisper.enableGPU`: enable GPU processing. Setting to `true` will try to use a CUDA build, if available for your architecture (currently CUDA 12.4 builds are available for Windows x64 and Linux x64).
+* `whisper.threadCount`: number of threads to use, defaults to `4`
+* `whisper.gpuDeviceIndex`: GPU device index (if GPU is enabled), defaults to `0`
 * `whisper.temperature`: temperature setting for the text decoder. Impacts the amount of randomization for token selection. It is recommended to leave at `0.1` (close to no randomization - almost always chooses the top ranked token) or choose a relatively low value (`0.25` or lower) for best results. Defaults to `0.1`
 * `whisper.prompt`: initial text to give the Whisper model. Can be a vocabulary, or example text of some sort. Note that if the prompt is very similar to the transcript, the model may intentionally avoid producing the transcript tokens as it may assume that they have already been transcribed. Optional
 * `whisper.topCandidateCount`: the number of top candidate tokens to consider. Defaults to `5`
@@ -168,17 +171,15 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 * `whisper.repetitionThreshold`: minimal repetition / compressibility score to cause a part not to be auto-prompted to the next part. Defaults to `2.4`
 * `whisper.decodeTimestampTokens`: enable/disable decoding of timestamp tokens. Setting to `false` can reduce the occurrence of hallucinations and token repetition loops, possibly due to the overall reduction in the number of tokens decoded. This has no impact on the accuracy of timestamps, since they are derived independently using cross-attention weights. However, there are cases where this can cause the model to end a part prematurely, especially in singing and less speech-like voice segments, or when there are multiple speakers. Defaults to `true`
 * `whisper.timestampAccuracy`: timestamp accuracy. can be `medium` or `high`. `medium` uses a reduced subset of attention heads for alignment, `high` uses all attention heads and is thus more accurate at the word level, but slower for larger models. Defaults to `high` for the `tiny` and `base` models, and `medium` for the larger models
-* `whisper.encoderProvider`: identifier for the ONNX execution provider to use with the encoder model. Can be `cpu`, `dml` (Windows only, uses [DirectML](https://microsoft.github.io/DirectML/)-based GPU acceleration), or `cuda` (Linux only, requires system-wide installation of CUDA and cuDNN SDKs, see [this guide](./CUDA.md) for more detailed information). In general, GPU-based encoding should be significantly faster. Defaults to `cpu`, or `dml` if available
-* `whisper.decoderProvider`: identifier for the ONNX execution provider to use with the decoder model. Can be `cpu`, `dml` (Windows only, uses [DirectML](https://microsoft.github.io/DirectML/)-based GPU acceleration), or `cuda` (Linux only, requires system-wide installation of CUDA and cuDNN SDKs, see [this guide](./CUDA.md) for more detailed information).. Using GPU acceleration for the decoder may be faster than CPU, especially for larger models, but that depends on your particular combination of CPU and GPU. Defaults to `cpu`, and on Windows, `dml` if available, for larger models (`small`, `medium`, `large`)
 * `whisper.seed`: provide a custom random seed for token selection when temperature is greater than 0. Uses a constant seed by default to ensure reproducibility
 
-**Whisper.cpp**:
-* `whisperCpp.model`: selects which `whisper.cpp` model to use.  Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large` (same as `large-v2`), `large-v1`, `large-v2`, `large-v3`, `large-v3-turbo`. The following quantized models are also supported: `tiny-q5_1`, `tiny.en-q5_1`, `tiny.en-q8_0`,`base-q5_1`, `base.en-q5_1`, `small-q5_1`, `small.en-q5_1`, `medium-q5_0`, `medium.en-q5_0`, `large-v2-q5_0`, `large-v3-q5_0`, `large-v3-turbo-q5_0`. Defaults to `base` or `base.en`
+**Whisper.cpp CLI**:
+* `whisperCpp.model`: selects which `whisper.cpp` model to use. Model identifiers and defaults are identical to those in the `whisper` engine
 * `whisperCpp.executablePath`: a path to a custom `whisper.cpp` `whisper-cli` executable (currently required for macOS)
 * `whisperCpp.build`: type of `whisper.cpp` build to use. Can be set to `cpu`, `cublas-12.4.0` or `custom`. By default, builds are auto-selected and downloaded for Windows x64 and Linux x64 (for both `cpu` and `cublas-12.4.0`). The Linux CUDA build requires a [system-wide installation of CUDA Toolkit 12.4+](https://developer.nvidia.com/cuda-downloads), available on path. Using other builds requires providing a custom `executablePath`, which will automatically set this option to `custom`
+* `whisperCpp.enableGPU`: enable GPU processing. Setting to `true` will try to use a CUDA build, if available for your system. Defaults to `true` when a CUDA-enabled build is selected via `whisperCpp.build`, otherwise `false`. If a custom build is used, it will enable or disable GPU for that build
 * `whisperCpp.threadCount`: number of threads to use, defaults to `4`
 * `whisperCpp.splitCount`: number of splits of the audio data to process in parallel (called `--processors` in the `whisper.cpp` CLI). A value greater than `1` can increase memory use significantly, reduce timing accuracy, and slow down execution in some cases. Defaults to `1` (highly recommended)
-* `whisperCpp.enableGPU`: enable GPU processing. Setting to `true` will try to use a CUDA build, if available for your system. Defaults to `true` when a CUDA-enabled build is selected via `whisperCpp.build`, otherwise `false`. If a custom build is used, it will enable or disable GPU for that build
 * `whisperCpp.topCandidateCount`: the number of top candidate tokens to consider. Defaults to `5`
 * `whisperCpp.beamCount`: the number of decoding paths to use during beam search. Defaults to `5`
 * `whisperCpp.temperature`: set temperature. Defaults to `0.0`
@@ -189,13 +190,6 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 * `whisperCpp.enableFlashAttention`: enable flash attention. Can significantly increase performance for some configurations (**Note**: setting this to `true` will cause `enableDTW` to always be set to `false` since it's not compatible with flash attention). Defaults to `false`
 * `whisperCpp.verbose`: show all CLI messages during execution. Defaults to `false`
 
-**Vosk**:
-* `vosk.modelPath`: path to the Vosk model to be used
-
-**Silero**:
-* `silero.modelPath`: path to a Silero model. Note that latest `en`, `de`, `fr` and `uk` models are automatically installed when needed based on the selected language. This should only be used to manually specify a different model, otherwise specify `language` instead
-* `silero.provider`: ONNX execution provider to use. Can be `cpu`, `dml` (Windows only, uses [DirectML](https://microsoft.github.io/DirectML/)-based GPU acceleration), or `cuda` (Linux only, requires system-wide installation of CUDA and cuDNN SDKs, see [this guide](./CUDA.md) for more detailed information). Defaults to `cpu`, or `dml` if available
-
 **Google Cloud**:
 * `googleCloud.apiKey`: Google Cloud API key (required)
 * `googleCloud.alternativeLanguageCodes`: An array of alternative language codes. Optional
@@ -258,19 +252,20 @@ Applies to CLI operation: `align`, API method: `align`
 
 Applies to the `whisper` engine only. To provide Whisper options for `dtw-ra`, use `recognition.whisper` instead.
 
-* `whisper.model`: Whisper model to use. Defaults to `tiny` or `tiny.en`
+* `whisper.model`: Whisper model to use. Defaults to `base` or `base.en`
+* `whisper.enableGPU`: enable whisper GPU build. See details in recognition section above
+* `whisper.threadCount`: number of threads to use, defaults to `4`
+* `whisper.gpuDeviceIndex`: GPU device index (if GPU is enabled), defaults to `0`
 * `whisper.endTokenThreshold`: minimal probability to accept an end token for a recognized part. The probability is measured via the softmax between the end token's logit and the second highest logit. You can try to adjust this threshold in cases the model is ending a part with too few, or many tokens decoded. Defaults to `0.9`. On the last audio part, it is always effectively set to `Infinity`, to ensure the remaining transcript tokens are decoded in full
 * `whisper.maxTokensPerPart`: maximum number of tokens to decode per part. Should help avoid edge cases where the model never reaches an end token for the part, which otherwise may cause the model to decode too many tokens and eventually crash. Defaults to `220` (highest possible)
 * `whisper.timestampAccuracy`: timestamp accuracy. can be `medium` or `high`. `medium` uses a reduced subset of attention heads for alignment, `high` uses all attention heads and is thus more accurate at the word level, but slower for larger models. Defaults to `high` for the `tiny` and `base` models, and `medium` for the larger models. Note: setting `high` for model `large-v3-turbo` reverts to `medium`, due to poor results with the `high` setting for that model (possibly related to the 16-bit quantization).
-* `whisper.encoderProvider`: encoder ONNX provider. See details in recognition section above
-* `whisper.decoderProvider`: decoder ONNX provider. See details in recognition section above
 
 ## Speech-to-text translation
 
 Applies to CLI operation: `translate-speech`, API method: `translateSpeech`
 
 **General**:
-* `engine`: only `whisper` supported
+* `engine`: Can be `whisper`, `whisper.cpp` or `openai-cloud`. Defaults to `whisper`
 * `sourceLanguage`: the source language code for the input speech. Auto-detected if not set
 * `targetLanguage`: the target language code for the output speech. Only `en` (English) supported by the `whisper` engine. Optional
 * `crop`: crop to active parts using voice activity detection before starting. Defaults to `true`
@@ -283,9 +278,9 @@ Applies to CLI operation: `translate-speech`, API method: `translateSpeech`
 
 * `whisper`: prefix to provide options for the Whisper model. Same options as detailed in the recognition section above
 
-**Whisper.cpp**:
+**Whisper.cpp CLI**:
 
-* `whisper.cpp`: prefix to provide options for the Whisper.cpp model. Same options as detailed in the recognition section above
+* `whisperCpp`: prefix to provide options for the Whisper.cpp model. Same options as detailed in the recognition section above
 
 **OpenAI Cloud**:
 
@@ -321,10 +316,11 @@ Applies to CLI operation: `align-translation`, API method: `alignTranslation`
 * `sourceSeparation`: prefix to provide options for source separation when `isolate` is set to `true`. Options detailed in section for source separation
 
 **Whisper**:
-* `whisper.model`: Whisper model to use. Only multilingual models can be used. Defaults to `tiny`
+* `whisper.model`: Whisper model to use. Only multilingual models can be used. Defaults to `base`
+* `whisper.enableGPU`: enable whisper GPU build. See details in recognition section above
+* `whisper.threadCount`: number of threads to use, defaults to `4`
+* `whisper.gpuDeviceIndex`: GPU device index (if GPU is enabled), defaults to `0`
 * `whisper.endTokenThreshold`: see details in the alignment section above
-* `whisper.encoderProvider`: encoder ONNX execution provider. See details in recognition section above
-* `whisper.decoderProvider`: decoder ONNX execution provider. See details in recognition section above
 
 ## Speech-to-transcript-and-translation alignment
 
@@ -371,10 +367,11 @@ Applies to CLI operation: `detect-speech-langauge`, API method: `detectSpeechLan
 * `vad`: prefix to provide options for voice activity detection when `crop` is set to `true`. Options detailed in section for voice activity detection
 
 **Whisper**:
-* `whisper.model`: Whisper model to use. See model list in the recognition section
+* `whisper.model`: Whisper model to use. See model list in the recognition section. Defaults to `tiny`
+* `whisper.enableGPU`: enable whisper GPU build. See details in recognition section above
+* `whisper.threadCount`: number of threads to use, defaults to `4`
+* `whisper.gpuDeviceIndex`: GPU device index (if GPU is enabled), defaults to `0`
 * `whisper.temperature`: impacts the distribution of candidate languages when applying the softmax function to compute language probabilities over the model output. Higher temperature causes the distribution to be more uniform, while lower temperature causes it to be more strongly weighted towards the best scoring candidates. Defaults to `1.0`
-* `whisper.encoderProvider`: encoder ONNX execution provider. See details in recognition section above
-* `whisper.decoderProvider`: decoder ONNX execution provider. See details in recognition section above
 
 **Silero**:
 * `silero.provider`: ONNX execution provider to use. Can be `cpu`, `dml` (Windows only, uses [DirectML](https://microsoft.github.io/DirectML/)-based GPU acceleration), or `cuda` (Linux only, requires system-wide installation of CUDA and cuDNN SDKs, see [this guide](./CUDA.md) for more detailed information). Using GPU may be faster, but the initialization overhead is larger. **Note**: `dml` provider seems to be unstable at the moment for this model. Defaults to `cpu`
@@ -455,9 +452,8 @@ These are shared between text-to-speech, speech-to-text and alignment operations
 On the CLI, global options can be used with any operation. To set global options via the API, use the `setGlobalOption(key, value)` method (see the [API reference](API.md) for more details).
 
 * `ffmpegPath`: sets a custom path for the FFmpeg executable
-* `soxPath`: sets a custom path for the SoX executable
 * `packageBaseURL`: sets a custom base URL for the remote package repository used to download missing packages. Default is `https://huggingface.co/echogarden/echogarden-packages/resolve/main/`. If `huggingface.co` isn't accessible in your location, you can set to use a mirror by changing `huggingface.co` to an alternative domain like `hf-mirror.com`
-* `logLevel`: adjusts the quantity of log messages shown during processing. Possible values: `silent`, `output`, `error`, `warning`, `info`, `trace`. Defaults to `info`
+* `logLevel`: adjusts the default amount of log messages shown during processing. Possible values: `silent`, `output`, `error`, `warning`, `info`, `trace`. Defaults to `info`
 
 
 ## CLI options
@@ -465,10 +461,10 @@ On the CLI, global options can be used with any operation. To set global options
 These options are for the CLI only.
 
 * `--play`, `--no-play`: enable/disable audio playback. Defaults to play if there is no output file specified
-* `--player`: audio player to use. Can be `audio-io` (uses the [`audio-io` package](https://github.com/echogarden-project/audio-io) to directly output to native OS audio buffers) or `sox` (requires `sox` to be available on path on macOS, auto-downloaded on other platforms). Defaults to `audio-io`
+* `--player`: audio player to use. Can only be `audio-io` (uses the [`audio-io` package](https://github.com/echogarden-project/audio-io) to directly output to native OS audio buffers). Defaults to `audio-io`
 * `--overwrite`, `--no-overwrite`: overwrite/keep existing files. Doesn't overwrite by default
 * `--debug`, `--no-debug`: show/hide the full details of JavaScript errors, if they occur. Disabled by default
-* `--config=...`: path to configuration file to use. Defaults to `echogarden.config` or `echogarden.config.json`, if found at the current directory
+* `--config=...`: path to configuration file to use. Defaults to `echogarden.config` or `echogarden.config.json` or `echogarden.config.json5`, if found at the current directory
 
 ## Using a configuration file
 

package/docs/Server.md

@@ -11,6 +11,7 @@ echogarden serve [options]
 ```
 
 **Options** (all optional):
+
 * `port`: Port number. Defaults to `45054`
 * `secure`: Start a secure server? Defaults to `false`
 * `certPath`: Path to a certificate file, required when `secure = true`

package/docs/Tasklist.md

@@ -127,6 +127,7 @@
 * Add SAPI pronunciation to lexicons (you already have the pronunciations for `en_US` and `en_GB`)
 * Try to use entity recognition to detect years, dates, currencies etc., which would disambiguate cases where it is not clear, like "in 1993" in "She was born in 1993" and "It searched in 1993 websites"
 * Option to add POS tags to timeline, if available
+* `≈` symbol as a word character
 
 ### Synthesis / VITS
 * Consider adding `⦁︎` (when surrounded by whitespace) as phrase separator
@@ -144,6 +145,7 @@
 * Currently, when input is set to be SSML, it is wrapped in a `<speak>` tag. Handle the case where the user made their own SSML document wrapped with a `<speak>` tag as well. Currently, it may send invalid input to Azure
 
 ### Recognition
+* Add token UTF-8 bytes in token entries on timeline. Some tokens are only a part of a codepoint, so this will allow to know exactly what they contain - the string representation is would be a Unicode error symbol in that case
 * Show alternatives when playing in the CLI. Clear current line and rewrite already printed text for alternatives during the speech recognition process
 
 ### Recognition / Whisper
@@ -244,7 +246,6 @@
 * Live input / microphone recognition
 * Implement beam search for Whisper decoder
 * Implement beam search for Silero decoder
-* Live Vosk alternatives events
 * Investigate exporting Whisper models to 16-bit quantized ONNX or a mix of 16-bit and 32-bit
 
 ### Alignment

package/docs/Technical.md

@@ -2,7 +2,7 @@
 
 * Echogarden is written in TypeScript and targets the Node.js platform.
 * It uses ESM modules and latest ECMAScript and TypeScript features.
-* It does not depend on essential binary executables. Instead, all of its engines either use pure JavaScript, WebAssembly, WASI, or the ONNX runtime, with some exceptions: the CLI does invoke a command line `ffmpeg` tool, auto-downloaded using its internal package system.
+* It does not depend on essential binary executables. Instead, all of its engines either use pure JavaScript, WebAssembly, WASI, or the ONNX runtime, with some exceptions: the CLI does invoke a command line `ffmpeg` tool, auto-downloaded using its internal package system. Since `v3.0.0` it uses a native NAPI binding to `whisper.cpp` in the `whisper` engine
 * It does not depend on essential native node.js modules requiring install-time compilation with `node-gyp`. This greatly simplifies the installation experience for end-users (the ONNX runtime bundles precompiled NAPI modules for all supported platforms - it doesn't require any compilation during its installation).
 
 ## Package system
@@ -12,6 +12,7 @@ Echogarden uses its own package system to download and install various component
 Packages are downloaded as `.tar.gz` files, and are extracted to `[data-folder]/packages/[package-id-string]`. Each package has its own subdirectory.
 
 `[data-folder]` is located at:
+
 * `%AppData%\Local\echogarden` on Windows
 * `Users/User/Library/Application Support/echogarden` on macOS
 * `/home/user/.local/share/echogarden` on Linux
@@ -33,7 +34,6 @@ Currently, the largest contributors to the size are:
 
 * `onnxruntime-node` (core inference runtime, NAPI): 180MB
 * `kuromoji` (Japanese tokenizer, JavaScript) 40MB
-* `espeak-ng-emscripten` (core synthesis and phonemization engine, WASM): 23MB
 * `tiktoken` (tokenizer library, WASM): 22MB
 * `flite-wasi` (synthesis engine, WASI): 20MB
 * `jieba-wasm` (Chinese tokenizer, WASM): 14MB

package/package.json

@@ -1,9 +1,9 @@
 {
 	"name": "echogarden",
-	"version": "2.10.2",
+	"version": "3.0.0",
 	"description": "An easy-to-use speech toolset. Includes tools for synthesis, recognition, alignment, speech translation, language detection, source separation and more.",
 	"author": "Rotem Dan",
-	"license": "GPL-3.0",
+	"license": "MIT AND GPL-3.0",
 	"keywords": [
 		"speech",
 		"text-to-speech",
@@ -52,57 +52,59 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "~3.817.0",
-		"@aws-sdk/client-transcribe-streaming": "~3.817.0",
+		"@aws-sdk/client-polly": "~3.1095.0",
+		"@aws-sdk/client-transcribe-streaming": "~3.1095.0",
 		"@echogarden/audio-io": "~0.4.2",
-		"@echogarden/espeak-ng-emscripten": "~0.3.5",
 		"@echogarden/fasttext-wasm": "~0.1.0",
 		"@echogarden/flite-wasi": "~0.1.1",
 		"@echogarden/fvad-wasm": "~0.2.0",
-		"@echogarden/gnuspeech-wasm": "~0.2.1",
-		"@echogarden/icu-segmentation-wasm": "~0.2.2",
+		"@echogarden/icu-segmentation-wasm": "~0.4.0",
 		"@echogarden/pffft-wasm": "~0.4.2",
 		"@echogarden/rnnoise-wasm": "~0.2.0",
-		"@echogarden/rubberband-wasm": "~0.2.0",
 		"@echogarden/sonic-wasm": "~0.2.0",
 		"@echogarden/speex-resampler-wasm": "~0.3.0",
 		"@echogarden/svoxpico-wasm": "~0.2.0",
-		"@echogarden/text-segmentation": "~0.4.1",
+		"@echogarden/text-segmentation": "~0.7.0",
 		"@echogarden/transformers-nodejs-lite": "~2.17.1-lite.4",
-		"@echogarden/wave-codec": "~0.3.3",
+		"@echogarden/wave-codec": "~0.4.0",
+		"@echogarden/whisper.cpp-binding": "~0.2.1",
 		"@mozilla/readability": "~0.6.0",
-		"chalk": "~5.4.1",
+		"chalk": "~5.6.2",
 		"command-exists": "~1.2.9",
-		"fs-extra": "~11.3.0",
-		"gaxios": "~6.7.1",
+		"fs-extra": "~11.4.0",
 		"graceful-fs": "~4.2.11",
-		"html-to-text": "~9.0.5",
-		"import-meta-resolve": "~4.1.0",
-		"jieba-wasm": "~2.2.0",
-		"jsdom": "~26.1.0",
-		"json5": "~2.2.3",
+		"html-to-text": "~10.0.0",
+		"import-meta-resolve": "~4.2.0",
+		"jieba-wasm": "~2.4.0",
+		"jsdom": "~29.1.1",
 		"kuromoji": "~0.1.2",
-		"microsoft-cognitiveservices-speech-sdk": "~1.44.0",
+		"microsoft-cognitiveservices-speech-sdk": "~1.51.0",
 		"msgpack-lite": "~0.1.26",
 		"onnxruntime-node": "~1.21.1",
-		"openai": "~4.103.0",
+		"openai": "~6.49.0",
+		"quick-json5": "~0.4.3",
+		"easier-http-request": "~0.1.0",
 		"regexp-composer": "~0.3.0",
 		"sam-js": "~0.3.1",
-		"strip-ansi": "~7.1.0",
-		"tar": "~7.4.3",
-		"tiktoken": "~1.0.21",
+		"strip-ansi": "~7.2.0",
+		"tar": "~7.5.22",
+		"tiktoken": "~1.0.22",
 		"tinyld": "~1.3.4",
 		"wasm-feature-detect": "~1.8.0",
-		"wasm-heap-manager": "~0.2.2",
-		"ws": "~8.18.2",
-		"wtf_wikipedia": "~10.4.0"
+		"wasm-heap-manager": "~0.4.0",
+		"ws": "~8.21.1",
+		"wtf_wikipedia": "~10.4.2"
 	},
 	"peerDependencies": {
-		"@echogarden/vosk": "~0.3.39-patched.1",
+		"@echogarden/gnuspeech-wasm": "~0.2.1",
+		"@echogarden/rubberband-wasm": "~0.2.0",
 		"winax": "~3.4.2"
 	},
 	"peerDependenciesMeta": {
-		"@echogarden/vosk": {
+		"@echogarden/gnuspeech-wasm": {
+			"optional": true
+		},
+		"@echogarden/rubberband-wasm": {
 			"optional": true
 		},
 		"winax": {
@@ -110,17 +112,25 @@
 		}
 	},
 	"devDependencies": {
+		"@echogarden/gnuspeech-wasm": "~0.3.0",
 		"@types/fs-extra": "~11.0.4",
 		"@types/graceful-fs": "~4.1.9",
-		"@types/jsdom": "~21.1.7",
-		"@types/msgpack-lite": "~0.1.11",
-		"@types/node": "~22.15.23",
-		"@types/tar": "~6.1.13",
+		"@types/jsdom": "~28.0.3",
+		"@types/msgpack-lite": "~0.1.12",
+		"@types/node": "~26.1.1",
 		"@types/ws": "~8.18.1",
-		"ts-json-schema-generator": "~2.4.0",
-		"typescript": "~5.8.3"
+		"ts-json-schema-generator": "~2.9.0",
+		"typescript": "~7.0.2"
 	},
 	"overrides": {
-		"whatwg-url": "~14.0.0"
+		"whatwg-url": "~14.0.0",
+		"adm-zip": "~0.6.0",
+		"uuid": "~11.1.1",
+		"brace-expansion": "~5.0.8",
+		"undici": "~7.28.0"
+	},
+	"allowScripts": {
+		"onnxruntime-node": true,
+		"wtf_wikipedia": true
 	}
 }

package/src/alignment/SemanticTextAlignment.ts

@@ -5,9 +5,11 @@ import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js'
 import { cosineDistance } from '../math/VectorMath.js'
 import { includesPunctuation, isWord, parseText } from '../nlp/Segmentation.js'
 import { Timeline, extractEntries } from '../utilities/Timeline.js'
+import { TimelineTranslationAlignmentCallbacks } from '../api/TimelineTranslationAlignment.js'
+import { OperationCallbacks } from '../api/Common.js'
 
-export async function alignTimelineToTextSemantically(timeline: Timeline, text: string, textLangCode: string) {
-	const logger = new Logger()
+export async function alignTimelineToTextSemantically(timeline: Timeline, text: string, textLangCode: string, callbacks: TimelineTranslationAlignmentCallbacks) {
+	const logger = new Logger(callbacks.logLevel)
 
 	logger.start(`Prepare text for semantic alignment`)
 
@@ -40,7 +42,12 @@ export async function alignTimelineToTextSemantically(timeline: Timeline, text:
 
 	logger.end()
 
-	const wordMappingEntries = await alignWordsToWordsSemantically(timelineWordGroups, textWordGroups)
+	const wordMappingEntries = await alignWordsToWordsSemantically(
+		timelineWordGroups,
+		textWordGroups,
+		undefined,
+		callbacks
+	)
 
 	logger.start(`Build timeline for translation`)
 
@@ -117,11 +124,11 @@ export async function alignTimelineToTextSemantically(timeline: Timeline, text:
 	return resultTimeline
 }
 
-export async function alignWordsToWordsSemantically(wordsGroups1: string[][], wordsGroups2: string[][], windowTokenCount = 20000) {
-	const logger = new Logger()
+export async function alignWordsToWordsSemantically(wordsGroups1: string[][], wordsGroups2: string[][], windowTokenCount = 20000, callbacks: OperationCallbacks) {
+	const logger = new Logger(callbacks.logLevel)
 
 	// Load embedding model
-	const modelPath = await loadPackage(`xenova-multilingual-e5-small-fp16`)
+	const modelPath = await loadPackage(`xenova-multilingual-e5-small-fp16`, callbacks)
 
 	const embeddingModel = new E5TextEmbedding(modelPath)
 
@@ -129,7 +136,7 @@ export async function alignWordsToWordsSemantically(wordsGroups1: string[][], wo
 	await embeddingModel.initializeIfNeeded()
 
 	async function extractEmbeddingsFromWordGroups(wordGroups: string[][]) {
-		const logger = new Logger()
+		const logger = new Logger(callbacks.logLevel, 'trace')
 
 		const maxTokensPerFragment = 512
 		const { Tensor } = await import('@echogarden/transformers-nodejs-lite')