echogarden 1.0.1 → 1.0.3

package/dist/tests/Test.js

@@ -5,12 +5,12 @@ setupProgramTerminationListeners();
 //process.env.http_proxy = 'http://localhost:8080'
 //const testText = `After a while, finding that nothing more happened, she decided on going into the garden at once; but, alas for poor Alice! when she got to the door, she found she had forgotten the little golden key, and when she went back to the table for it, she found she could not possibly reach it: she could see it quite plainly through the glass, and she tried her best to climb up one of the legs of the table, but it was too slippery; and when she had tired herself out with trying, the poor little thing sat down and cried.`
 //await testKirshenbaumPhonemization(testText)
-//await makeTarballsForInstalledPackages(true)
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'c', 'a', 'b', 'c'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'd', 'a', 'b', 'a', 'd'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'b', 'c', 'a', 'b', 'a', 'b'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'a', 'a', 'a', 'b'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'b', 'a', 'c', 'a', 'b', 'a', 'c', 'a'])
 //getRepetitionScoreRelativeToFirstSubstring(['a', 'a', 'a', 'b', 'b', 'a', 'a', 'a', 'b'])
+//await makeTarballsForInstalledPackages(true)
 process.exit(0);
 //# sourceMappingURL=Test.js.map

package/dist/tests/Test.js.map

@@ -1 +1 @@
-{"version":3,"file":"Test.js","sourceRoot":"","sources":["../../src/tests/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAA8C,WAAW,EAAE,gCAAgC,EAAE,MAAM,2BAA2B,CAAA;AAIrI,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,gCAAgC,EAAE,CAAA;AAClC,gDAAgD;AAChD,kDAAkD;AAElD,yhBAAyhB;AAEzhB,8CAA8C;AAE9C,8CAA8C;AAE9C,4EAA4E;AAC5E,sFAAsF;AACtF,2FAA2F;AAC3F,sFAAsF;AACtF,2FAA2F;AAC3F,2FAA2F;AAE3F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA"}
+{"version":3,"file":"Test.js","sourceRoot":"","sources":["../../src/tests/Test.ts"],"names":[],"mappings":"AAAA,OAAO,EAA8C,WAAW,EAAE,gCAAgC,EAAiB,MAAM,2BAA2B,CAAA;AAIpJ,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,gCAAgC,EAAE,CAAA;AAElC,gDAAgD;AAChD,kDAAkD;AAElD,yhBAAyhB;AACzhB,8CAA8C;AAE9C,4EAA4E;AAC5E,sFAAsF;AACtF,2FAA2F;AAC3F,sFAAsF;AACtF,2FAA2F;AAC3F,2FAA2F;AAE3F,8CAA8C;AAE9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA"}

package/dist/voice-activity-detection/SileroVAD.d.ts

@@ -1,14 +1,14 @@
 /// <reference path="../../src/typings/Fillers.d.ts" />
-import Onnx from 'onnxruntime-node';
+import type * as Onnx from 'onnxruntime-node';
 import { RawAudio } from '../audio/AudioUtilities.js';
 export declare function detectVoiceActivity(rawAudio: RawAudio, modelPath: string, frameDuration: 30 | 60 | 90): Promise<number[]>;
 export declare class SileroVAD {
     session?: Onnx.InferenceSession;
-    modelStateH: Onnx.Tensor;
-    modelStateC: Onnx.Tensor;
-    modelSampleRate: Onnx.Tensor;
+    modelStateH?: Onnx.Tensor;
+    modelStateC?: Onnx.Tensor;
+    modelSampleRate?: Onnx.Tensor;
     modelPath: string;
     constructor(modelPath: string);
     predictAudioFrame(frame: Float32Array): Promise<number>;
-    private initializeSession;
+    private initializeIfNeeded;
 }

package/dist/voice-activity-detection/SileroVAD.js

@@ -1,4 +1,3 @@
-import Onnx from 'onnxruntime-node';
 import { concatFloat32Arrays } from '../utilities/Utilities.js';
 let sileroVad;
 export async function detectVoiceActivity(rawAudio, modelPath, frameDuration) {
@@ -28,19 +27,12 @@ export class SileroVAD {
     modelSampleRate;
     modelPath;
     constructor(modelPath) {
-        this.modelSampleRate = new Onnx.Tensor('int64', new BigInt64Array([BigInt(16000)]), []);
-        const h = new Float32Array(2 * 1 * 64);
-        const c = new Float32Array(2 * 1 * 64);
-        this.modelStateH = new Onnx.Tensor('float32', h, [2, 1, 64]);
-        this.modelStateC = new Onnx.Tensor('float32', c, [2, 1, 64]);
         this.modelPath = modelPath;
     }
     async predictAudioFrame(frame) {
-        if (!this.session) {
-            await this.initializeSession(this.modelPath);
-        }
+        await this.initializeIfNeeded();
+        const Onnx = await import('onnxruntime-node');
         const inputTensor = new Onnx.Tensor('float32', frame, [1, frame.length]);
-        //const inputs = { input: inputTensor, h0: this.modelStateH, c0: this.modelStateC }
         const inputs = { input: inputTensor, sr: this.modelSampleRate, h: this.modelStateH, c: this.modelStateC };
         const results = await this.session.run(inputs);
         const probability = results['output'].data[0];
@@ -48,11 +40,20 @@ export class SileroVAD {
         this.modelStateC = results['cn'];
         return probability;
     }
-    async initializeSession(modelPath) {
+    async initializeIfNeeded() {
+        if (this.session) {
+            return;
+        }
+        const Onnx = await import('onnxruntime-node');
+        const h = new Float32Array(2 * 1 * 64);
+        const c = new Float32Array(2 * 1 * 64);
+        this.modelStateH = new Onnx.Tensor('float32', h, [2, 1, 64]);
+        this.modelStateC = new Onnx.Tensor('float32', c, [2, 1, 64]);
+        this.modelSampleRate = new Onnx.Tensor('int64', new BigInt64Array([BigInt(16000)]), []);
         const onnxOptions = {
             logSeverityLevel: 3
         };
-        this.session = await Onnx.InferenceSession.create(modelPath, onnxOptions);
+        this.session = await Onnx.InferenceSession.create(this.modelPath, onnxOptions);
     }
 }
 //# sourceMappingURL=SileroVAD.js.map

package/dist/voice-activity-detection/SileroVAD.js.map

@@ -1 +1 @@
-{"version":3,"file":"SileroVAD.js","sourceRoot":"","sources":["../../src/voice-activity-detection/SileroVAD.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,kBAAkB,CAAA;AAEnC,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AAG/D,IAAI,SAAoB,CAAA;AAExB,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAkB,EAAE,SAAiB,EAAE,aAA2B;IAC3G,IAAI,QAAQ,CAAC,UAAU,IAAI,KAAK,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;IACnD,CAAC;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAA;IAE9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAA;IAE9D,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,SAAS,GAAG,IAAI,SAAS,CAAC,SAAS,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,kBAAkB,GAAa,EAAE,CAAA;IAEvC,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC,MAAM,EAAE,QAAQ,IAAI,WAAW,EAAE,CAAC;QAChF,IAAI,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,WAAW,CAAC,CAAA;QAEnE,IAAI,KAAK,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YAChC,KAAK,GAAG,mBAAmB,CAAC,CAAC,KAAK,EAAE,IAAI,YAAY,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QACnF,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAA;QAE5D,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,MAAM,OAAO,SAAS;IACrB,OAAO,CAAwB;IAE/B,WAAW,CAAa;IACxB,WAAW,CAAa;IACxB,eAAe,CAAa;IAE5B,SAAS,CAAQ;IAEjB,YAAY,SAAiB;QAC5B,IAAI,CAAC,eAAe,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAEvF,MAAM,CAAC,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAA;QACtC,MAAM,CAAC,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAA;QAEtC,IAAI,CAAC,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QAC5D,IAAI,CAAC,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QAE5D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,KAAmB;QAC1C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAC7C,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;QAExE,mFAAmF;QACnF,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,EAAE,CAAA;QAEzG,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QAE/C,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAW,CAAA;QAEvD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAEhC,OAAO,WAAW,CAAA;IACnB,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAAiB;QAChD,MAAM,WAAW,GAAyC;YACzD,gBAAgB,EAAE,CAAC;SACnB,CAAA;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IAC1E,CAAC;CACD"}
+{"version":3,"file":"SileroVAD.js","sourceRoot":"","sources":["../../src/voice-activity-detection/SileroVAD.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAA;AAG/D,IAAI,SAAoB,CAAA;AAExB,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAkB,EAAE,SAAiB,EAAE,aAA2B;IAC3G,IAAI,QAAQ,CAAC,UAAU,IAAI,KAAK,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;IACnD,CAAC;IAED,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAA;IAE9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAA;IAE9D,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,SAAS,GAAG,IAAI,SAAS,CAAC,SAAS,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,kBAAkB,GAAa,EAAE,CAAA;IAEvC,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,CAAC,MAAM,EAAE,QAAQ,IAAI,WAAW,EAAE,CAAC;QAChF,IAAI,KAAK,GAAG,YAAY,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,WAAW,CAAC,CAAA;QAEnE,IAAI,KAAK,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YAChC,KAAK,GAAG,mBAAmB,CAAC,CAAC,KAAK,EAAE,IAAI,YAAY,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;QACnF,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAA;QAE5D,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IACrC,CAAC;IAED,OAAO,kBAAkB,CAAA;AAC1B,CAAC;AAED,MAAM,OAAO,SAAS;IACrB,OAAO,CAAwB;IAE/B,WAAW,CAAc;IACzB,WAAW,CAAc;IACzB,eAAe,CAAc;IAE7B,SAAS,CAAQ;IAEjB,YAAY,SAAiB;QAC5B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,KAAmB;QAC1C,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAE/B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAE7C,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;QAExE,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,CAAC,eAAgB,EAAE,CAAC,EAAE,IAAI,CAAC,WAAY,EAAE,CAAC,EAAE,IAAI,CAAC,WAAY,EAAE,CAAA;QAE5G,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;QAE/C,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAW,CAAA;QAEvD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;QAEhC,OAAO,WAAW,CAAA;IACnB,CAAC;IACO,KAAK,CAAC,kBAAkB;QAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,OAAM;QACP,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAE7C,MAAM,CAAC,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAA;QACtC,MAAM,CAAC,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAA;QAEtC,IAAI,CAAC,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QAC5D,IAAI,CAAC,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QAE5D,IAAI,CAAC,eAAe,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QAEvF,MAAM,WAAW,GAAyC;YACzD,gBAAgB,EAAE,CAAC;SACnB,CAAA;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IAC/E,CAAC;CACD"}

package/docs/CLI.md

@@ -311,3 +311,10 @@ Uninstall one or more expansion packages
 ### `list-packages`
 
 Show a list of installed expansion packages
+
+## General CLI options
+
+* `--play`, `--no-play`: enable/disable audio playback. Defaults to play if there is no output file specified
+* `--overwrite`, `--no-overwrite`: overwrite/keep existing files. Doesn't overwrite by default
+* `--debug`, `--no-debug`: show/hide the full details of JavaScript errors, if they occur. Disabled by default
+* `--config=...`: path to configuration file to use. Defaults to `echogarden.config` or `echogarden.config.json`, if found at the current directory

package/docs/Development.md

@@ -12,7 +12,7 @@ Here's a quick guide on how to set up a development environment for making and t
 * Fork or clone the repository (`git clone https://github.com/echogarden-project/echogarden`)
 * Ensure you have [Node.js](https://nodejs.org/en/download) installed
 * Ensure you have TypeScript installed (`npm install typescript -g`)
-* Run `npm install` in the project base directory
+* Run `npm install` at the project base directory
 * Compile using `tsc .` at the project base directory
 
 ## Auto-compiling in Visual Studio Code

package/docs/Engines.md

@@ -48,7 +48,7 @@ These cloud-based engines connect to public cloud APIs that are not officially p
 **Offline**:
 * [OpenAI Whisper](https://github.com/openai/whisper) (`whisper`): high accuracy transformer-based speech recognition architecture. Supports 99 languages. There are several models of different sizes, some are multilingual, and some are English only (`.en`): `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large`, `large-v1` and `large-v2`, `large-v3`. **Note**: large models are not currently supported by `onnxruntime-node` due to model size restrictions.
 * [Whisper.cpp](https://github.com/ggerganov/whisper.cpp) (`whisper.cpp`): a port of the Whisper architecture to C++, by Georgi Gerganov. Supports all Whisper models, including several quantized ones (see full list in the options page). Has various different builds, including CUDA and OpenCL for GPU support.
-* [Vosk](https://github.com/alphacep/vosk-api) (`vosk`): models available for 25+ languages. _Note_: the Vosk package is not included in the default installation, but you can add support for it using `npm install @echogarden/vosk -g`. Then, you'll need to manually [download a model](https://alphacephei.com/vosk/models) and specify its directory path via the `vosk.modelPath` option.
+* [Vosk](https://github.com/alphacep/vosk-api) (`vosk`): models available for 25+ languages. **Note**: the Vosk package is not included in the default installation, but you can add support for it using `npm install @echogarden/vosk -g`. Then, you'll need to manually [download a model](https://alphacephei.com/vosk/models) and specify its directory path via the `vosk.modelPath` option.
 * [Silero](https://github.com/snakers4/silero-models) (`silero`): models available for English, Spanish, German and Ukrainian. For [non-commercial use only](https://github.com/snakers4/silero-models/blob/master/LICENSE).
 
 **Cloud services**:

package/docs/Options.md

@@ -148,14 +148,15 @@ Applies to CLI operation: `transcribe`, API method: `recognize`
 * `whisper.autoPromptParts`: use previous part's recognized text as prompt for the next part. Disabling this may help to prevent repetition carrying over between parts, in some cases. Defaults to `true`
 * `whisper.maxTokensPerPart`: maximum number of tokens to decode for each 30 second audio part. Defaults to `250`
 * `whisper.suppressRepetition`: attempt to suppress decoding repeating token patterns. Defaults to `true`
-* `whisper.seed`: use a custom seed for token selection, when temperature is greater than 0. Uses a constant seed by default
+* `whisper.decodeTimestampTokens`: enable/disable decoding of timestamp tokens, since more accurate timing is already extracted via cross-attention weight alignment. For unclear reasons, setting to `false` can significantly reduce the occurrence of hallucinations and token repetition loops, and increases word timestamp accuracy. However, there are cases where this can cause the model to end a part prematurely, especially in singing and less speech-like voice segments, or when there are multiple speakers. Defaults to `true`
+* `whisper.seed`: provide a custom random seed for token selection when temperature is greater than 0. Uses a constant seed by default to ensure reproducibility
 
 **Whisper.cpp**:
 * `whisperCpp.model`: selects which `whisper.cpp` model to use.  Can be `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large` (same as `large-v2`), `large-v1`, `large-v2`, `large-v3`. These quantized models are also supported: `tiny-q5_1`, `tiny.en-q5_1`, `tiny.en-q8_0`,`base-q5_1`, `base.en-q5_1`, `small-q5_1`, `small.en-q5_1`, `medium-q5_0`, `medium.en-q5_0`, `large-v2-q5_0`, `large-v3-q5_0`. Defaults to `base` or `base.en`
 * `whisperCpp.executablePath`: custom `whisper.cpp` executable path (currently required for macOS)
 * `whisperCpp.build`: type of `whisper.cpp` build to use. Can be set `cpu`, `cublas-11.8.0`, `cublas-12.4.0`. By default, builds are auto-selected and downloaded for Windows x64 (`cpu`, `cublas-11.8.0`, `cublas-12.4.0`) and Linux x64 (`cpu`). Using other builds requires providing a custom `executablePath`
 * `whisperCpp.threadCount`: number of threads to use, defaults to `4`
-* `whisperCpp.splitCount`: number of splits of the audio data to process in parallel (called `--processors` in the CLI). A value greater than `1` can increase memory use significantly, reduce timing accuracy, and slow down execution in some cases. Defaults to `1` (highly recommended)
+* `whisperCpp.splitCount`: number of splits of the audio data to process in parallel (called `--processors` in the `whisper.cpp` CLI). A value greater than `1` can increase memory use significantly, reduce timing accuracy, and slow down execution in some cases. Defaults to `1` (highly recommended)
 * `whisperCpp.enableGPU`: enable GPU processing. Defaults to `true` on CUDA-enabled builds, otherwise `false`
 * `whisperCpp.topCandidateCount`: the number of top candidate tokens to consider. Defaults to `5`
 * `whisperCpp.beamCount`: the number of decoding paths to use during beam search. Defaults to `5`
@@ -205,7 +206,7 @@ Applies to CLI operation: `align`, API method: `align`
 * `engine`: alignment algorithm to use, can be `dtw`, `dtw-ra` or `whisper`. Defaults to `dtw`
 * `language`: language code for the audio and transcript ([ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)), like `en`, `fr`, `en-US`, `pt-BR`. Auto-detected from transcript if not set
 * `crop`: crop to active parts using voice activity detection before starting. Defaults to `true`
-* `isolate`: apply source separation to isolate voice before starting recognition. Defaults to `false`
+* `isolate`: apply source separation to isolate voice before starting alignment. Defaults to `false`
 * `customLexiconPaths`: an array of custom lexicon file paths. Optional
 * `subtitles`: prefix to provide options for subtitles. Options detailed in section for subtitles
 * `vad`: prefix to provide options for voice activity detection when `crop` is set to `true`. Options detailed in section for voice activity detection
@@ -333,10 +334,3 @@ These are shared between text-to-speech, speech-to-text and alignment operations
 * `maxAddedDuration`: maximum extra time (in seconds) that may be added after a cue's end time. This gives the reader additional time to read the cue, and also ensures that very short duration cues aren't shown in a flash. Defaults to `3.0`
 
 **Note**: options `maxLineCount`, `maxLineWidth`, `minWordsInLine`, `separatePhrases`, are only effective when using the `segment` and `sentence` modes, and are ignored in all other modes. `maxAddedDuration` doesn't apply to modes `word`, `phone` and `word+phone` (they always use the exact start and end timestamps).
-
-# General CLI options
-
-* `--play`, `--no-play`: enable/disable audio playback. Defaults to play if there is no output file specified
-* `--overwrite`, `--no-overwrite`: overwrite/keep existing files. Doesn't overwrite by default
-* `--debug`, `--no-debug`: show/hide the full details of JavaScript errors, if they occur. Disabled by default
-* `--config=...`: path to configuration file to use. See the [CLI guide](CLI.md) for more information about configuration files

package/docs/Releases.md

@@ -1,6 +1,6 @@
 # Release notes
 
-## `1.0.0` (April 4, 2024)
+## `1.0.0` (April 12, 2024)
 
 **New features**:
 * Add support for [`whisper.cpp`](https://github.com/ggerganov/whisper.cpp), a C++ port of OpenAI's Whisper speech recognition architecture. It is faster than the integrated `whisper` engine, supports large models, and GPU processing. It can now be used for recognition, speech translation and alignment (via `dtw-ra`), though its word timestamps are less accurate than the integrated `whisper` engine.
@@ -13,6 +13,11 @@
 * Expose several new configuration options for the Whisper engine: a settings for maximum tokens per part, and a setting to enable/disable repetition suppression, set custom random seed, disable/enable decoding of timestamp tokens
 * Expose more options for the Elevenlabs engine
 
+**Enhancements**:
+* `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
+* Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
+* Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
+
 **Behavioral and breaking changes**:
 * Minimal required node version changed to `18.0.0`
 * All recognition, alignment, translation and language recognition operations first apply the new adaptive gate VAD (can be changed to any other VAD engine via the `vad.` option prefix) and remove any sections that are not identified as containing voice, before starting processing. This should improve results in most cases, and reduce processing time
@@ -30,11 +35,6 @@
 * `maxIncreaseDb` post-processing and denoising option renamed to `maxGainIncrease`
 * `dryMixGainDb` denoising option renamed to `dryMixGain`
 
-**Enhancements**:
-* `detect-voice-activity` now also outputs the cropped voice when an audio file is given as output, with `.cropped` suffix added
-* Improved word segmentation of whisper engine. Words spanning multiple recognized time segments are now split. Word timestamp ranges don't overlap non-speech sections.
-* Whisper and Whisper.cpp outputs now include timestamps for individual recognized tokens
-
 **Fixes**:
 * Integrated `whisper` engine now uses `tiktoken` to tokenize text, which produces near identical tokenization compared to the official Python implementation. This fixes issues with decoding Chinese characters, correctly encoding prompts, and several others languages with characters that span multiple tokens
 * Fix voice language lists for multilingual voices in Elevenlabs TTS engine
@@ -43,14 +43,15 @@
 * Fix issue with identifying legacy IEEE Float wave format
 * Fix issue with language detection failing with empty inputs
 * Fix issue with Whisper model failing when no tokens are detected in a part
-* Fix issue with phone alignment not getting the right DTW window size, producing warnings like `all cost directions are equal to infinity
+* Fix issue with phone alignment not getting the right DTW window size, producing warnings like `all cost directions are equal to infinity`
 * Fix phone timelines in DTW-RA
 * Add more workarounds for eSpeak tokenization bugs
 * Various other fixes
 
 **Documentation**:
-* Add new 'Releases' page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
-* Options reference page restructured and updated with some missing information
+* Add new [release notes](./Releases.md) page. Releases before `1.0.x` were retroactively documented based on commit history (may not exactly detail all historical changes)
+* Add new [development page](./Development.md) with a quick guide for setting up a development environment to make changes to the Echogarden codebase
+* Options reference page reorganized and updated with some missing information
 
 ## `0.12.x` (March 16, 2024)
 
@@ -67,6 +68,7 @@
 **Other**:
 * Include `package-lock.json` in repository
 
+
 ## `0.11.x` (August 17, 2023)
 
 Many features, enhancements, and fixes were incrementally added over the span of 7 months, up to March 2024.
@@ -87,6 +89,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Fix audio playback in macOS
 * Many other fixes
 
+
 ## `0.10.x` (August 2, 2023)
 
 **New features**:
@@ -100,6 +103,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 **Fixes**:
 * Many fixes
 
+
 ## `0.9.x` (July 29, 2023)
 
 **New features**:
@@ -114,6 +118,7 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Work around several eSpeak bugs
 * Many fixes
 
+
 ## `0.8.x` (July 25, 2023)
 
 **New features**:
@@ -122,26 +127,25 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add punctuation thresholds to Whisper decoder
 * Expose subtitle configuration options to CLI
 
-**Behavioral changes**:
-* Rename `subtitles.minWords` to `subtitles.minWordsInLine`
-
 **Enhancements**:
 * Improve speech language detection to work with arbitrary length audio. Split audio to overlapping parts and detect each part individually. Then average the results.
 * Add more heteronyms
 
+**Behavioral changes**:
+* Rename `subtitles.minWords` to `subtitles.minWordsInLine`
+
 **Fixes**:
 * Improve numerical stability of softmax function
 * Work around eSpeak bug with markers in long inputs
 * Various fixes
 
+
 ## `0.7.x` (July 23, 2023)
 
 **New features**:
 * Alignment: Add option to accept custom lexicons
 * Implement and add decoder temperature option to Whisper model
 
-**Behavioral changes**:
-
 **Enhancements**:
 * Alignment: use preprocessing and lexicons
 * Add language detection to speech translation
@@ -151,10 +155,13 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add colors to log messages
 * Warn when maximum DTW window duration is smaller than source audio duration.
 
+**Behavioral changes**:
+
 **Fixes**:
 * Fix support for SSML input in eSpeak engine
 * Various fixes
 
+
 ## `0.6.x` (July 20, 2023)
 
 **New features**:
@@ -165,16 +172,16 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add arguments to customize paragraph parsing
 * Change whitespace option to include option for collapsing all whitespace
 
-**Behavioral changes**:
-* Set Elevenlabs defaults to mid-values
-* Split plain text to paragraphs using double line breaks by default.
-* Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
-
 **Enhancements**:
 * Change whitespace option to include option for collapsing all whitespace
 * Add awareness of guillemets
 * Log full language of selected voice
 
+**Behavioral changes**:
+* Set Elevenlabs defaults to mid-values
+* Split plain text to paragraphs using double line breaks by default.
+* Rename `awsPolly` options to `amazonPolly` to be more consistent with documentation
+
 **Fixes**:
 * Fix and update Elevenlabs engine
 * Fix incorrect gender properties for some VITS voices
@@ -184,26 +191,25 @@ Many features, enhancements, and fixes were incrementally added over the span of
 **Other**:
 * Remove `package-lock.json` from the repository
 
+
 ## `0.5.x` (July, 19 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Move to new package system supporting version tags, and hosted in a Hugging Face repository
-
 **Enhancements**:
 * Add large numbers of VITS voices
 
+**Behavioral changes**:
+* Move to new package system supporting version tags, and hosted in a Hugging Face repository
+
 **Fixes**:
 * Various fixes
 
+
 ## `0.4.x` (July 9, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
-
 **Enhancements**:
 * Improve text normalization and add support for currencies
 * Improve heteronym lexicons
@@ -212,32 +218,34 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Add some British English (RP) pronunciations to heteronym lexicon
 * Add large numbers of VITS voices
 
+**Behavioral changes**:
+* Remove support for `afplay` and `aplay` for playback. Only SoX is used now.
+
 **Fixes**:
 * Various fixes
 
+
 ## `0.3.x` (July 1, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-
 **Enhancements**:
 * Upgrade heteronym disambiguation to an improved, rule-based approach, which doesn't use POS tagging
 * Extend decade normalization
 * Improve text normalization and add support to currencies
 * Update the heteronym lexicon
 
+**Behavioral changes**:
+
 **Fixes**:
 * Remove sentences containing only whitespace when synthesizing
 * Many fixes
 
+
 ## `0.2.x` (May 10, 2023)
 
 **New features**:
 
-**Behavioral changes**:
-* Remove dependency on `xregexp` package. Use Unicode RegExp instead.
-
 **Enhancements**:
 * Show current sentence and segment in synthesis log.
 * Add check for cancellation flag.
@@ -245,6 +253,9 @@ Many features, enhancements, and fixes were incrementally added over the span of
 * Improve auto TTS engine selection
 * Many Enhancements
 
+**Behavioral changes**:
+* Remove dependency on `xregexp` package. Use Unicode RegExp instead.
+
 **Fixes**:
 * Many fixes
 

package/docs/Tasklist.md

@@ -2,7 +2,10 @@
 
 ## Bugs
 
+### Alignment
+
 * In DTW-RA, recognition transcript including something like "Question 2.What does Juan", where "2.What" has a point in the middle, is breaking playback of the timeline.
+* DTW-RA will not work correctly with Polish language texts, due to issues with the eSpeak engine pronouncing `|` characters, which are intended to be used as separators and ignored by all other eSpeak languages.
 
 ### Synthesis
 
@@ -108,8 +111,8 @@
 * See if it's possible to reliably use eSpeak as a segmentation engine
 
 ### Subtitles
+* Split long words if needed. This is especially important for Chinese
 * If a subtitle is too short and at the end of the audio, try to extend it back if possible (for example, if the previous subtitle is already extended, take back from it)
-* Split long words if needed
 * Decide how many punctuation characters to allow before breaking to a new line (currently it's infinite)
 * Add more clause separators, for even more special cases
 * Add option to output usable word or phoneme-level caption files (investigate how it's done on YouTube auto-captions)
@@ -155,10 +158,10 @@
 * Currently, when input is set to be SSML, it is wrapped in a `<speak>` tag. Handle the case where the user made their own SSML document wrapped with a `<speak>` tag as well. Currently, it may send invalid input to Azure
 
 ### Recognition
+* Recognized word entries that span VAD segment boundaries can be split
 * Show alternatives when playing in the CLI. Clear current line and rewrite already printed text for alternatives during the speech recognition process
 * Option to split recognized audio to segments or sentences, as is done with synthesized audio
 * Try to exclude the timing for trailing punctuation tokens in words that contain them. This can help narrow down the end timestamp to cover the word more tightly
-* Recognized word entries that span VAD boundaries may be split
 
 ### Recognition / Whisper
 * May get stuck in a token repeat loop when silence or non-speech segment encountered in audio. Decide what to do

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "echogarden",
-	"version": "1.0.1",
+	"version": "1.0.3",
 	"description": "An integrated speech toolset, providing a variety of synthesis, recognition, alignment, and other processing tools, designed to be directly accessible to end-users.",
 	"author": "Rotem Dan",
 	"license": "GPL-3.0",
@@ -54,8 +54,8 @@
 		"echogarden": "./dist/cli/CLILauncher.js"
 	},
 	"dependencies": {
-		"@aws-sdk/client-polly": "^3.554.0",
-		"@aws-sdk/client-transcribe-streaming": "^3.554.0",
+		"@aws-sdk/client-polly": "^3.556.0",
+		"@aws-sdk/client-transcribe-streaming": "^3.556.0",
 		"@echogarden/espeak-ng-emscripten": "^0.1.2",
 		"@echogarden/fasttext-wasm": "^0.1.0",
 		"@echogarden/flite-wasi": "^0.1.1",
@@ -75,7 +75,7 @@
 		"compromise": "^14.13.0",
 		"compromise-dates": "^3.5.0",
 		"fs-extra": "^11.2.0",
-		"gaxios": "^6.4.0",
+		"gaxios": "^6.5.0",
 		"graceful-fs": "^4.2.11",
 		"html-escaper": "^3.0.3",
 		"html-to-text": "^9.0.5",
@@ -90,11 +90,11 @@
 		"ndarray": "^1.0.19",
 		"ndarray-ops": "^1.2.2",
 		"onnxruntime-node": "^1.17.0",
-		"openai": "^4.33.0",
+		"openai": "^4.37.0",
 		"sam-js": "^0.2.1",
 		"strip-ansi": "^7.1.0",
-		"tar": "^7.0.0",
-		"tiktoken": "^1.0.13",
+		"tar": "^7.0.1",
+		"tiktoken": "^1.0.14",
 		"tinyld": "^1.3.4",
 		"ws": "^8.16.0",
 		"wtf_wikipedia": "^10.3.0"
@@ -127,7 +127,7 @@
 		"@types/recursive-readdir": "^2.2.4",
 		"@types/tar": "^6.1.12",
 		"@types/ws": "^8.5.10",
-		"ts-json-schema-generator": "^1.5.1",
+		"ts-json-schema-generator": "^2.0.1",
 		"typescript": "^5.4.5"
 	}
 }

package/src/alignment/SpeechAlignment.ts

@@ -151,7 +151,7 @@ export async function alignUsingDtw(
 	return mappedTimeline
 }
 
-export async function alignUsingDtwWithRecognitionReference(
+export async function alignUsingDtwWithRecognition(
 	sourceRawAudio: RawAudio,
 	referenceRawAudio: RawAudio,
 	referenceTimeline: Timeline,
@@ -272,6 +272,7 @@ export async function alignUsingDtwWithRecognitionReference(
 
 	let currentSynthesizedToRecognizedMappingIndex = 0
 
+	// Map from synthesized reference timestamps to the recognition timestamps
 	function mapSynthesizedToRecognizedTimeAndAdvance(synthesizedTime: number) {
 		for (; ; currentSynthesizedToRecognizedMappingIndex += 1) {
 			const left = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].synthesized

package/src/api/API.ts

@@ -7,7 +7,7 @@ export * from './Recognition.js'
 export * from './Alignment.js'
 export * from './Translation.js'
 export * from './LanguageDetection.js'
-export * from './Vad.js'
+export * from './VoiceActivityDetection.js'
 export * from './Denoising.js'
 export * from './SourceSeparation.js'
 export * from '../server/Server.js'

package/src/api/Alignment.ts

@@ -7,11 +7,12 @@ import { Logger } from '../utilities/Logger.js'
 import * as API from './API.js'
 import { Timeline, addTimeOffsetToTimeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
 import { formatLanguageCodeWithName, getDefaultDialectForLanguageCodeIfPossible, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
-import { WhisperOptions } from '../recognition/WhisperSTT.js'
+import { type WhisperOptions } from '../recognition/WhisperSTT.js'
 import chalk from 'chalk'
 import { DtwGranularity, createAlignmentReferenceUsingEspeak } from '../alignment/SpeechAlignment.js'
 import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
 import { EspeakOptions, defaultEspeakOptions } from '../synthesis/EspeakTTS.js'
+import { isWord } from '../nlp/Segmentation.js'
 
 const log = logToStderr
 
@@ -85,7 +86,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 
 	logger.start('Load alignment module')
 
-	const { alignUsingDtwWithRecognitionReference, alignUsingDtw } = await import('../alignment/SpeechAlignment.js')
+	const { alignUsingDtwWithRecognition, alignUsingDtw } = await import('../alignment/SpeechAlignment.js')
 
 	function getDtwWindowDurationsAndGranularities() {
 		let granularities: DtwGranularity[]
@@ -127,7 +128,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			const {
 				referenceRawAudio,
 				referenceTimeline
-			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths)
+			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, false)
 
 			logger.end()
 
@@ -145,7 +146,12 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			logger.end()
 
 			// Recognize source audio
-			const { transcript: recognizedTranscript, wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
+			let { wordTimeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions)
+
+			logger.log('')
+
+			// Remove non-word entries from recognition timeline
+			recognitionTimeline = recognitionTimeline.filter(entry => isWord(entry.text))
 
 			// Synthesize the ground-truth transcript and get its timeline
 			logger.start('Synthesize ground-truth transcript with eSpeak')
@@ -154,14 +160,14 @@ export async function align(input: AudioSourceParam, transcript: string, options
 				referenceRawAudio,
 				referenceTimeline,
 				espeakVoice,
-			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, true)
+			} = await createAlignmentReferenceUsingEspeak(transcript, language, options.plainText, options.customLexiconPaths, false)
 
 			logger.end()
 
 			const { windowDurations, granularities } = getDtwWindowDurationsAndGranularities()
 
 			const phoneAlignmentMethod = options.dtw!.phoneAlignmentMethod!
-			
+
 			const espeakOptions: EspeakOptions = {
 				...defaultEspeakOptions,
 				voice: espeakVoice,
@@ -170,7 +176,7 @@ export async function align(input: AudioSourceParam, transcript: string, options
 			}
 
 			// Align the ground-truth transcript and the recognized transcript
-			mappedTimeline = await alignUsingDtwWithRecognitionReference(
+			mappedTimeline = await alignUsingDtwWithRecognition(
 				sourceRawAudio,
 				referenceRawAudio,
 				referenceTimeline,
@@ -345,10 +351,10 @@ export const defaultAlignmentOptions: AlignmentOptions = {
 			topCandidateCount: 5,
 			punctuationThreshold: 0.2,
 			maxTokensPerPart: 250,
-			autoPromptParts: true,
+			autoPromptParts: false,
 			suppressRepetition: true,
+			decodeTimestampTokens: true,
 			seed: undefined,
-			decodeTimestampTokens: false,
 		}
 	},
 

package/src/api/LanguageDetection.ts

@@ -6,7 +6,7 @@ import { Logger } from '../utilities/Logger.js'
 import * as API from './API.js'
 import { logToStderr } from '../utilities/Utilities.js'
 import path from 'path'
-import { WhisperModelName } from '../recognition/WhisperSTT.js'
+import { type WhisperModelName } from '../recognition/WhisperSTT.js'
 import { formatLanguageCodeWithName, languageCodeToName } from '../utilities/Locale.js'
 import { loadPackage } from '../utilities/PackageManager.js'
 import chalk from 'chalk'
@@ -117,7 +117,7 @@ export async function detectSpeechLanguage(input: AudioSourceParam, options: Spe
 	}
 
 	logger.end()
-	logger.logDuration('\nTotal detection time', startTime, chalk.magentaBright)
+	logger.logDuration('\nTotal language detection time', startTime, chalk.magentaBright)
 
 	return {
 		detectedLanguage,

package/src/api/Recognition.ts

@@ -12,7 +12,7 @@ import { loadPackage } from '../utilities/PackageManager.js'
 import chalk from 'chalk'
 import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
 import { OpenAICloudSTTOptions, defaultOpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
-import { WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
+import { type WhisperCppOptions } from '../recognition/WhisperCppSTT.js'
 
 const log = logToStderr