echogarden 0.12.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -14
- package/data/schemas/options.json +398 -111
- package/dist/alignment/DTWMfccSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWMfccSequenceAlignment.js +8 -8
- package/dist/alignment/DTWSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignment.js +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.js +2 -2
- package/dist/alignment/LevenshteinSequenceAlignment.d.ts +1 -1
- package/dist/alignment/LevenshteinSequenceAlignment.js +1 -1
- package/dist/alignment/SpeechAlignment.d.ts +9 -10
- package/dist/alignment/SpeechAlignment.js +136 -105
- package/dist/alignment/SpeechAlignment.js.map +1 -1
- package/dist/api/API.d.ts +13 -12
- package/dist/api/API.js +14 -13
- package/dist/api/API.js.map +1 -1
- package/dist/api/APIOptions.d.ts +5 -4
- package/dist/api/Alignment.d.ts +15 -9
- package/dist/api/Alignment.js +88 -74
- package/dist/api/Alignment.js.map +1 -1
- package/dist/api/Common.js +1 -1
- package/dist/api/Denoising.d.ts +6 -6
- package/dist/api/Denoising.js +23 -23
- package/dist/api/Denoising.js.map +1 -1
- package/dist/api/LanguageDetection.d.ts +19 -12
- package/dist/api/LanguageDetection.js +88 -38
- package/dist/api/LanguageDetection.js.map +1 -1
- package/dist/api/Recognition.d.ts +16 -6
- package/dist/api/Recognition.js +129 -55
- package/dist/api/Recognition.js.map +1 -1
- package/dist/api/SourceSeparation.d.ts +17 -0
- package/dist/api/SourceSeparation.js +61 -0
- package/dist/api/SourceSeparation.js.map +1 -0
- package/dist/api/Synthesis.d.ts +18 -18
- package/dist/api/Synthesis.js +191 -164
- package/dist/api/Synthesis.js.map +1 -1
- package/dist/api/Translation.d.ts +19 -8
- package/dist/api/Translation.js +132 -35
- package/dist/api/Translation.js.map +1 -1
- package/dist/api/Vad.d.ts +10 -5
- package/dist/api/Vad.js +76 -38
- package/dist/api/Vad.js.map +1 -1
- package/dist/audio/AudioBufferConversion.d.ts +1 -1
- package/dist/audio/AudioBufferConversion.js +4 -4
- package/dist/audio/AudioPlayer.d.ts +1 -1
- package/dist/audio/AudioPlayer.js +26 -26
- package/dist/audio/AudioPlayer.js.map +1 -1
- package/dist/audio/AudioRecorder.d.ts +1 -1
- package/dist/audio/AudioRecorder.js +5 -5
- package/dist/audio/AudioUtilities.d.ts +13 -9
- package/dist/audio/AudioUtilities.js +86 -24
- package/dist/audio/AudioUtilities.js.map +1 -1
- package/dist/cli/CLI.d.ts +3 -3
- package/dist/cli/CLI.js +271 -162
- package/dist/cli/CLI.js.map +1 -1
- package/dist/cli/CLIConfigFile.js +8 -8
- package/dist/cli/CLILauncher.js +6 -6
- package/dist/cli/CLIOptionsSchema.js +2 -2
- package/dist/cli/CLIParser.js +5 -5
- package/dist/cli/CLIStarter.js +4 -4
- package/dist/codecs/FFMpegTranscoder.d.ts +2 -2
- package/dist/codecs/FFMpegTranscoder.js +37 -37
- package/dist/codecs/FFMpegTranscoder.js.map +1 -1
- package/dist/codecs/TIMITCodec.js +5 -5
- package/dist/codecs/WaveCodec.d.ts +1 -1
- package/dist/codecs/WaveCodec.js +22 -22
- package/dist/denoising/RNNoise.d.ts +1 -1
- package/dist/denoising/RNNoise.js +9 -9
- package/dist/dsp/BiquadFilter.d.ts +3 -2
- package/dist/dsp/BiquadFilter.js +18 -11
- package/dist/dsp/BiquadFilter.js.map +1 -1
- package/dist/dsp/DecayingPeakEstimator.d.ts +16 -0
- package/dist/dsp/DecayingPeakEstimator.js +23 -0
- package/dist/dsp/DecayingPeakEstimator.js.map +1 -0
- package/dist/dsp/FFT.d.ts +8 -4
- package/dist/dsp/FFT.js +76 -30
- package/dist/dsp/FFT.js.map +1 -1
- package/dist/dsp/KWeightingFilter.d.ts +9 -0
- package/dist/dsp/KWeightingFilter.js +40 -0
- package/dist/dsp/KWeightingFilter.js.map +1 -0
- package/dist/dsp/LoudnessEstimator.d.ts +21 -0
- package/dist/dsp/LoudnessEstimator.js +47 -0
- package/dist/dsp/LoudnessEstimator.js.map +1 -0
- package/dist/dsp/MFCC.d.ts +2 -2
- package/dist/dsp/MFCC.js +15 -15
- package/dist/dsp/MelSpectogram.d.ts +1 -1
- package/dist/dsp/MelSpectogram.js +6 -6
- package/dist/dsp/Rubberband.d.ts +11 -11
- package/dist/dsp/Rubberband.js +27 -27
- package/dist/dsp/Sonic.d.ts +1 -1
- package/dist/dsp/Sonic.js +3 -3
- package/dist/dsp/SpeexResampler.d.ts +1 -1
- package/dist/dsp/SpeexResampler.js +2 -2
- package/dist/math/VectorMath.d.ts +12 -8
- package/dist/math/VectorMath.js +35 -32
- package/dist/math/VectorMath.js.map +1 -1
- package/dist/nlp/ChineseSegmentation.js +2 -2
- package/dist/nlp/CompromiseNLP.js +3 -3
- package/dist/nlp/EspeakPhonemizer.js +30 -30
- package/dist/nlp/IPA.js +20 -20
- package/dist/nlp/JapaneseSegmentation.js +6 -6
- package/dist/nlp/Lexicon.d.ts +1 -1
- package/dist/nlp/Lexicon.js +7 -7
- package/dist/nlp/Segmentation.d.ts +3 -0
- package/dist/nlp/Segmentation.js +21 -14
- package/dist/nlp/Segmentation.js.map +1 -1
- package/dist/nlp/TextNormalizer.js +16 -16
- package/dist/recognition/AmazonTranscribeSTT.d.ts +2 -2
- package/dist/recognition/AmazonTranscribeSTT.js +13 -14
- package/dist/recognition/AmazonTranscribeSTT.js.map +1 -1
- package/dist/recognition/AzureCognitiveServicesSTT.js +5 -6
- package/dist/recognition/AzureCognitiveServicesSTT.js.map +1 -1
- package/dist/recognition/GoogleCloudSTT.d.ts +3 -3
- package/dist/recognition/GoogleCloudSTT.js +18 -18
- package/dist/recognition/OpenAICloudSTT.d.ts +19 -0
- package/dist/recognition/OpenAICloudSTT.js +81 -0
- package/dist/recognition/OpenAICloudSTT.js.map +1 -0
- package/dist/recognition/SileroSTT.d.ts +2 -2
- package/dist/recognition/SileroSTT.js +25 -25
- package/dist/recognition/VoskSTT.d.ts +2 -2
- package/dist/recognition/VoskSTT.js +8 -8
- package/dist/recognition/WhisperCppSTT.d.ts +88 -0
- package/dist/recognition/WhisperCppSTT.js +332 -0
- package/dist/recognition/WhisperCppSTT.js.map +1 -0
- package/dist/recognition/WhisperSTT.d.ts +49 -25
- package/dist/recognition/WhisperSTT.js +626 -481
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/server/Client.d.ts +1 -1
- package/dist/server/Client.js +22 -22
- package/dist/server/Server.js +9 -9
- package/dist/server/Server.js.map +1 -1
- package/dist/server/Worker.d.ts +22 -22
- package/dist/server/Worker.js +36 -36
- package/dist/server/Worker.js.map +1 -1
- package/dist/server/WorkerStarter.js +2 -2
- package/dist/source-separation/MDXNetSourceSeparation.d.ts +11 -0
- package/dist/source-separation/MDXNetSourceSeparation.js +161 -0
- package/dist/source-separation/MDXNetSourceSeparation.js.map +1 -0
- package/dist/speech-language-detection/SileroLanguageDetection.d.ts +1 -1
- package/dist/speech-language-detection/SileroLanguageDetection.js +7 -7
- package/dist/subtitles/Subtitles.d.ts +10 -0
- package/dist/subtitles/Subtitles.js +2 -2
- package/dist/subtitles/Subtitles.js.map +1 -1
- package/dist/synthesis/AwsPollyTTS.d.ts +1 -1
- package/dist/synthesis/AwsPollyTTS.js +12 -12
- package/dist/synthesis/AzureCognitiveServicesTTS.js +7 -7
- package/dist/synthesis/CoquiServerTTS.js +10 -10
- package/dist/synthesis/CoquiServerTTS.js.map +1 -1
- package/dist/synthesis/ElevenlabsTTS.d.ts +23 -0
- package/dist/synthesis/ElevenlabsTTS.js +103 -0
- package/dist/synthesis/ElevenlabsTTS.js.map +1 -0
- package/dist/synthesis/EspeakTTS.d.ts +6 -5
- package/dist/synthesis/EspeakTTS.js +81 -69
- package/dist/synthesis/EspeakTTS.js.map +1 -1
- package/dist/synthesis/FliteTTS.d.ts +3 -3
- package/dist/synthesis/FliteTTS.js +154 -154
- package/dist/synthesis/FliteTTS.js.map +1 -1
- package/dist/synthesis/GoogleCloudTTS.d.ts +3 -3
- package/dist/synthesis/GoogleCloudTTS.js +17 -17
- package/dist/synthesis/GoogleCloudTTS.js.map +1 -1
- package/dist/synthesis/GoogleTranslateTTS.d.ts +1 -1
- package/dist/synthesis/GoogleTranslateTTS.js +103 -103
- package/dist/synthesis/MicrosoftEdgeTTS.d.ts +2 -2
- package/dist/synthesis/MicrosoftEdgeTTS.js +74 -74
- package/dist/synthesis/OpenAICloudTTS.d.ts +13 -0
- package/dist/synthesis/OpenAICloudTTS.js +169 -0
- package/dist/synthesis/OpenAICloudTTS.js.map +1 -0
- package/dist/synthesis/SamTTS.js +3 -3
- package/dist/synthesis/SapiTTS.d.ts +3 -3
- package/dist/synthesis/SapiTTS.js +26 -26
- package/dist/synthesis/StreamlabsPollyTTS.d.ts +2 -2
- package/dist/synthesis/StreamlabsPollyTTS.js +27 -27
- package/dist/synthesis/SvoxPicoTTS.d.ts +2 -2
- package/dist/synthesis/SvoxPicoTTS.js +65 -65
- package/dist/synthesis/SvoxPicoTTS.js.map +1 -1
- package/dist/synthesis/VitsTTS.d.ts +3 -3
- package/dist/synthesis/VitsTTS.js +378 -378
- package/dist/synthesis/VitsTTS.js.map +1 -1
- package/dist/tests/Test.js +2 -2
- package/dist/utilities/Compression.d.ts +5 -0
- package/dist/utilities/Compression.js +29 -13
- package/dist/utilities/Compression.js.map +1 -1
- package/dist/utilities/FileDownloader.d.ts +1 -1
- package/dist/utilities/FileDownloader.js +16 -16
- package/dist/utilities/FileSystem.js +7 -7
- package/dist/utilities/Locale.d.ts +7 -7
- package/dist/utilities/Locale.js +15 -15
- package/dist/utilities/Logger.js +3 -3
- package/dist/utilities/ObjectUtilities.js +19 -19
- package/dist/utilities/OpenPromise.js +2 -2
- package/dist/utilities/OpenPromise.js.map +1 -1
- package/dist/utilities/PackageManager.js +31 -0
- package/dist/utilities/PackageManager.js.map +1 -1
- package/dist/utilities/PathUtilities.js +8 -8
- package/dist/utilities/RandomGenerator.js +2 -2
- package/dist/utilities/SmoothEstimator.d.ts +8 -0
- package/dist/utilities/SmoothEstimator.js +25 -0
- package/dist/utilities/SmoothEstimator.js.map +1 -0
- package/dist/utilities/TarballMaker.js +8 -8
- package/dist/utilities/Timeline.d.ts +3 -2
- package/dist/utilities/Timeline.js +11 -11
- package/dist/utilities/Timeline.js.map +1 -1
- package/dist/utilities/Timer.js +4 -4
- package/dist/utilities/Utilities.d.ts +4 -0
- package/dist/utilities/Utilities.js +38 -15
- package/dist/utilities/Utilities.js.map +1 -1
- package/dist/utilities/WasmMemoryManager.js +7 -7
- package/dist/utilities/WebReader.js +23 -23
- package/dist/utilities/WikipediaReader.js +2 -2
- package/dist/voice-activity-detection/AdaptiveGateVAD.d.ts +28 -0
- package/dist/voice-activity-detection/AdaptiveGateVAD.js +138 -0
- package/dist/voice-activity-detection/AdaptiveGateVAD.js.map +1 -0
- package/dist/voice-activity-detection/SileroVAD.d.ts +1 -1
- package/dist/voice-activity-detection/SileroVAD.js +5 -5
- package/dist/voice-activity-detection/SileroVAD.js.map +1 -1
- package/dist/voice-activity-detection/WebRtcVAD.d.ts +1 -1
- package/dist/voice-activity-detection/WebRtcVAD.js +4 -4
- package/docs/API.md +29 -11
- package/docs/CLI.md +31 -7
- package/docs/Contributing.md +38 -0
- package/docs/Development.md +93 -19
- package/docs/Engines.md +28 -16
- package/docs/Licenses.md +4 -1
- package/docs/Options.md +158 -78
- package/docs/Releases.md +262 -0
- package/docs/Server.md +7 -7
- package/docs/Tasklist.md +95 -76
- package/docs/Technical.md +4 -4
- package/package.json +13 -14
- package/src/alignment/DTWMfccSequenceAlignment.ts +9 -9
- package/src/alignment/DTWSequenceAlignment.ts +2 -2
- package/src/alignment/DTWSequenceAlignmentWindowed.ts +3 -3
- package/src/alignment/LevenshteinSequenceAlignment.ts +2 -2
- package/src/alignment/SpeechAlignment.ts +204 -119
- package/src/api/API.ts +14 -13
- package/src/api/APIOptions.ts +12 -11
- package/src/api/Alignment.ts +147 -90
- package/src/api/Common.ts +1 -1
- package/src/api/Denoising.ts +28 -28
- package/src/api/LanguageDetection.ts +135 -48
- package/src/api/Recognition.ts +198 -59
- package/src/api/SourceSeparation.ts +99 -0
- package/src/api/Synthesis.ts +217 -181
- package/src/api/Translation.ts +193 -40
- package/src/api/Vad.ts +110 -41
- package/src/audio/AudioBufferConversion.ts +4 -4
- package/src/audio/AudioPlayer.ts +27 -27
- package/src/audio/AudioRecorder.ts +5 -5
- package/src/audio/AudioUtilities.ts +107 -24
- package/src/cli/CLI.ts +313 -164
- package/src/cli/CLIConfigFile.ts +8 -8
- package/src/cli/CLILauncher.ts +6 -6
- package/src/cli/CLIOptionsSchema.ts +2 -2
- package/src/cli/CLIParser.ts +5 -5
- package/src/cli/CLIStarter.ts +4 -4
- package/src/codecs/FFMpegTranscoder.ts +38 -38
- package/src/codecs/TIMITCodec.ts +5 -5
- package/src/codecs/WaveCodec.ts +22 -22
- package/src/denoising/RNNoise.ts +9 -9
- package/src/dsp/BiquadFilter.ts +19 -11
- package/src/dsp/DecayingPeakEstimator.ts +35 -0
- package/src/dsp/FFT.ts +103 -35
- package/src/dsp/KWeightingFilter.ts +43 -0
- package/src/dsp/LoudnessEstimator.ts +74 -0
- package/src/dsp/MFCC.ts +15 -15
- package/src/dsp/MelSpectogram.ts +7 -7
- package/src/dsp/Rubberband.ts +38 -38
- package/src/dsp/Sonic.ts +4 -4
- package/src/dsp/SpeexResampler.ts +2 -2
- package/src/math/VectorMath.ts +42 -33
- package/src/nlp/ChineseSegmentation.ts +3 -3
- package/src/nlp/CompromiseNLP.ts +3 -3
- package/src/nlp/EspeakPhonemizer.ts +30 -30
- package/src/nlp/IPA.ts +20 -20
- package/src/nlp/JapaneseSegmentation.ts +6 -6
- package/src/nlp/Lexicon.ts +8 -8
- package/src/nlp/Segmentation.ts +23 -14
- package/src/nlp/TextNormalizer.ts +16 -16
- package/src/recognition/AmazonTranscribeSTT.ts +16 -17
- package/src/recognition/AzureCognitiveServicesSTT.ts +8 -6
- package/src/recognition/GoogleCloudSTT.ts +21 -21
- package/src/recognition/OpenAICloudSTT.ts +142 -0
- package/src/recognition/SileroSTT.ts +26 -26
- package/src/recognition/VoskSTT.ts +10 -10
- package/src/recognition/WhisperCppSTT.ts +555 -0
- package/src/recognition/WhisperSTT.ts +760 -507
- package/src/server/Client.ts +23 -23
- package/src/server/Server.ts +9 -9
- package/src/server/Worker.ts +53 -53
- package/src/server/WorkerStarter.ts +2 -2
- package/src/source-separation/MDXNetSourceSeparation.ts +228 -0
- package/src/speech-language-detection/SileroLanguageDetection.ts +8 -8
- package/src/subtitles/Subtitles.ts +3 -3
- package/src/synthesis/AwsPollyTTS.ts +14 -14
- package/src/synthesis/AzureCognitiveServicesTTS.ts +10 -10
- package/src/synthesis/CoquiServerTTS.ts +10 -10
- package/src/synthesis/ElevenlabsTTS.ts +137 -0
- package/src/synthesis/EspeakTTS.ts +90 -71
- package/src/synthesis/FliteTTS.ts +157 -157
- package/src/synthesis/GoogleCloudTTS.ts +19 -19
- package/src/synthesis/GoogleTranslateTTS.ts +104 -104
- package/src/synthesis/MicrosoftEdgeTTS.ts +80 -80
- package/src/synthesis/OpenAICloudTTS.ts +196 -0
- package/src/synthesis/SamTTS.ts +3 -3
- package/src/synthesis/SapiTTS.ts +29 -29
- package/src/synthesis/StreamlabsPollyTTS.ts +29 -29
- package/src/synthesis/SvoxPicoTTS.ts +67 -67
- package/src/synthesis/VitsTTS.ts +380 -380
- package/src/tests/Test.ts +4 -4
- package/src/utilities/Compression.ts +34 -13
- package/src/utilities/FileDownloader.ts +19 -19
- package/src/utilities/FileSystem.ts +7 -7
- package/src/utilities/Locale.ts +22 -22
- package/src/utilities/Logger.ts +4 -4
- package/src/utilities/ObjectUtilities.ts +19 -19
- package/src/utilities/OpenPromise.ts +2 -2
- package/src/utilities/PackageManager.ts +40 -0
- package/src/utilities/PathUtilities.ts +8 -8
- package/src/utilities/RandomGenerator.ts +3 -3
- package/src/utilities/SmoothEstimator.ts +35 -0
- package/src/utilities/TarballMaker.ts +9 -9
- package/src/utilities/Timeline.ts +15 -13
- package/src/utilities/Timer.ts +4 -4
- package/src/utilities/Utilities.ts +49 -15
- package/src/utilities/WasmMemoryManager.ts +7 -7
- package/src/utilities/WebReader.ts +23 -23
- package/src/utilities/WikipediaReader.ts +2 -2
- package/src/voice-activity-detection/AdaptiveGateVAD.ts +202 -0
- package/src/voice-activity-detection/SileroVAD.ts +5 -5
- package/src/voice-activity-detection/WebRtcVAD.ts +5 -5
- package/dist/synthesis/ElevenLabsTTS.d.ts +0 -8
- package/dist/synthesis/ElevenLabsTTS.js +0 -82
- package/dist/synthesis/ElevenLabsTTS.js.map +0 -1
- package/src/synthesis/ElevenLabsTTS.ts +0 -104
package/src/api/Translation.ts
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
|
-
import { extendDeep } from
|
|
1
|
+
import { extendDeep } from '../utilities/ObjectUtilities.js'
|
|
2
2
|
|
|
3
|
-
import { logToStderr } from
|
|
4
|
-
import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevel, trimAudioEnd } from
|
|
5
|
-
import { Logger } from
|
|
3
|
+
import { logToStderr } from '../utilities/Utilities.js'
|
|
4
|
+
import { AudioSourceParam, RawAudio, ensureRawAudio, normalizeAudioLevel, trimAudioEnd } from '../audio/AudioUtilities.js'
|
|
5
|
+
import { Logger } from '../utilities/Logger.js'
|
|
6
6
|
|
|
7
|
-
import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from
|
|
8
|
-
import {
|
|
9
|
-
import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from
|
|
10
|
-
import { EngineMetadata } from
|
|
11
|
-
import { SpeechLanguageDetectionOptions, detectSpeechLanguage } from
|
|
12
|
-
import chalk from
|
|
13
|
-
import { SubtitlesConfig, defaultSubtitlesBaseConfig } from
|
|
7
|
+
import { Timeline, addWordTextOffsetsToTimeline, wordTimelineToSegmentSentenceTimeline } from '../utilities/Timeline.js'
|
|
8
|
+
import { defaultWhisperOptions, type WhisperOptions } from '../recognition/WhisperSTT.js'
|
|
9
|
+
import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from '../utilities/Locale.js'
|
|
10
|
+
import { EngineMetadata } from './Common.js'
|
|
11
|
+
import { SpeechLanguageDetectionOptions, detectSpeechLanguage } from './API.js'
|
|
12
|
+
import chalk from 'chalk'
|
|
13
|
+
import { SubtitlesConfig, defaultSubtitlesBaseConfig } from '../subtitles/Subtitles.js'
|
|
14
|
+
|
|
15
|
+
import * as API from './API.js'
|
|
16
|
+
import { OpenAICloudSTTOptions, defaultOpenAICloudSTTOptions } from '../recognition/OpenAICloudSTT.js'
|
|
17
|
+
import { WhisperCppOptions, defaultWhisperCppOptions } from '../recognition/WhisperCppSTT.js'
|
|
14
18
|
|
|
15
19
|
const log = logToStderr
|
|
16
20
|
|
|
@@ -19,65 +23,164 @@ const log = logToStderr
|
|
|
19
23
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
20
24
|
export async function translateSpeech(input: AudioSourceParam, options: SpeechTranslationOptions): Promise<SpeechTranslationResult> {
|
|
21
25
|
const logger = new Logger()
|
|
26
|
+
|
|
22
27
|
const startTimestamp = logger.getTimestamp()
|
|
23
28
|
|
|
24
|
-
|
|
29
|
+
options = extendDeep(defaultSpeechTranslationOptions, options)
|
|
25
30
|
|
|
26
31
|
const inputRawAudio = await ensureRawAudio(input)
|
|
27
32
|
|
|
28
|
-
let sourceRawAudio
|
|
33
|
+
let sourceRawAudio: RawAudio
|
|
34
|
+
let isolatedRawAudio: RawAudio | undefined
|
|
35
|
+
let backgroundRawAudio: RawAudio | undefined
|
|
36
|
+
|
|
37
|
+
if (options.isolate) {
|
|
38
|
+
logger.log(``)
|
|
39
|
+
logger.end();
|
|
40
|
+
|
|
41
|
+
({ isolatedRawAudio, backgroundRawAudio } = await API.isolate(inputRawAudio, options.sourceSeparation!))
|
|
42
|
+
|
|
43
|
+
logger.end()
|
|
44
|
+
logger.log(``)
|
|
45
|
+
|
|
46
|
+
sourceRawAudio = await ensureRawAudio(isolatedRawAudio, 16000, 1)
|
|
47
|
+
} else {
|
|
48
|
+
sourceRawAudio = await ensureRawAudio(inputRawAudio, 16000, 1)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let sourceUncropTimeline: Timeline | undefined
|
|
52
|
+
|
|
53
|
+
if (options.crop) {
|
|
54
|
+
logger.start('Crop using voice activity detection');
|
|
55
|
+
({ timeline: sourceUncropTimeline, croppedRawAudio: sourceRawAudio } = await API.detectVoiceActivity(sourceRawAudio, options.vad!))
|
|
56
|
+
|
|
57
|
+
logger.end()
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
logger.start('Prepare for speech translation')
|
|
61
|
+
|
|
29
62
|
sourceRawAudio = normalizeAudioLevel(sourceRawAudio)
|
|
30
63
|
sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0])
|
|
31
64
|
|
|
32
|
-
options = extendDeep(defaultSpeechTranslationOptions, options)
|
|
33
|
-
|
|
34
65
|
if (!options.sourceLanguage) {
|
|
35
|
-
logger.start(
|
|
36
|
-
const { detectedLanguage } = await detectSpeechLanguage(
|
|
66
|
+
logger.start('No source language specified. Detecting speech language')
|
|
67
|
+
const { detectedLanguage } = await detectSpeechLanguage(sourceRawAudio, options.languageDetection || {})
|
|
37
68
|
|
|
38
69
|
logger.end()
|
|
39
|
-
logger.logTitledMessage('
|
|
70
|
+
logger.logTitledMessage('Source language detected', formatLanguageCodeWithName(detectedLanguage))
|
|
40
71
|
|
|
41
72
|
options.sourceLanguage = detectedLanguage
|
|
73
|
+
} else {
|
|
74
|
+
logger.end()
|
|
75
|
+
|
|
76
|
+
const specifiedLanguageFormatted = formatLanguageCodeWithName(getShortLanguageCode(normalizeLanguageCode(options.sourceLanguage)))
|
|
77
|
+
|
|
78
|
+
logger.logTitledMessage('Source language', specifiedLanguageFormatted)
|
|
42
79
|
}
|
|
43
80
|
|
|
44
|
-
logger.
|
|
81
|
+
logger.logTitledMessage('Target language', formatLanguageCodeWithName(getShortLanguageCode(normalizeLanguageCode(options.targetLanguage!))))
|
|
82
|
+
|
|
83
|
+
logger.start('Preprocess audio for translation')
|
|
45
84
|
|
|
46
85
|
const engine = options.engine!
|
|
47
86
|
const sourceLanguage = normalizeLanguageCode(options.sourceLanguage!)
|
|
48
87
|
const targetLanguage = options.targetLanguage!
|
|
49
88
|
|
|
50
89
|
let transcript: string
|
|
51
|
-
let
|
|
90
|
+
let wordTimeline: Timeline | undefined
|
|
91
|
+
let segmentTimeline: Timeline | undefined
|
|
52
92
|
|
|
53
93
|
logger.start(`Load ${engine} module`)
|
|
54
94
|
|
|
55
95
|
switch (engine) {
|
|
56
|
-
case
|
|
57
|
-
const WhisperSTT = await import(
|
|
96
|
+
case 'whisper': {
|
|
97
|
+
const WhisperSTT = await import('../recognition/WhisperSTT.js')
|
|
58
98
|
|
|
59
99
|
const whisperOptions = options.whisper!
|
|
60
100
|
|
|
61
101
|
const shortSourceLanguageCode = getShortLanguageCode(sourceLanguage)
|
|
62
102
|
const shortTargetLanguageCode = getShortLanguageCode(targetLanguage)
|
|
63
103
|
|
|
64
|
-
const { modelName, modelDir
|
|
104
|
+
const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, shortSourceLanguageCode)
|
|
105
|
+
|
|
106
|
+
if (shortTargetLanguageCode != 'en') {
|
|
107
|
+
throw new Error('Whisper translation only supports English as target language')
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (modelName.endsWith('.en')) {
|
|
111
|
+
throw new Error('Whisper translation tasks are only possible with a multilingual model')
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (shortSourceLanguageCode == 'en' && shortTargetLanguageCode == 'en') {
|
|
115
|
+
throw new Error('Both translation source and target languages are English')
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
logger.end();
|
|
119
|
+
|
|
120
|
+
({ transcript, timeline: wordTimeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, 'translate', sourceLanguage, whisperOptions))
|
|
121
|
+
|
|
122
|
+
addWordTextOffsetsToTimeline(wordTimeline, transcript);
|
|
123
|
+
|
|
124
|
+
({ segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(wordTimeline, transcript, targetLanguage, 'single', 'preserve'))
|
|
125
|
+
|
|
126
|
+
break
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
case 'whisper.cpp': {
|
|
130
|
+
const WhisperCppSTT = await import('../recognition/WhisperCppSTT.js')
|
|
131
|
+
|
|
132
|
+
const whisperCppOptions = options.whisperCpp!
|
|
133
|
+
|
|
134
|
+
const shortSourceLanguageCode = getShortLanguageCode(sourceLanguage)
|
|
135
|
+
const shortTargetLanguageCode = getShortLanguageCode(targetLanguage)
|
|
136
|
+
|
|
137
|
+
logger.end()
|
|
65
138
|
|
|
66
|
-
|
|
67
|
-
|
|
139
|
+
const { modelName, modelPath } = await WhisperCppSTT.loadModelPackage(whisperCppOptions.model, shortSourceLanguageCode)
|
|
140
|
+
|
|
141
|
+
if (shortTargetLanguageCode != 'en') {
|
|
142
|
+
throw new Error('Whisper.cpp translation only supports English as target language')
|
|
68
143
|
}
|
|
69
144
|
|
|
70
|
-
if (modelName.endsWith(
|
|
71
|
-
throw new Error(
|
|
145
|
+
if (modelName.endsWith('.en')) {
|
|
146
|
+
throw new Error('Whisper.cpp translation tasks are only possible with a multilingual model')
|
|
72
147
|
}
|
|
73
148
|
|
|
74
|
-
|
|
75
|
-
|
|
149
|
+
logger.end();
|
|
150
|
+
|
|
151
|
+
({ transcript, timeline: wordTimeline } = await WhisperCppSTT.recognize(
|
|
152
|
+
sourceRawAudio,
|
|
153
|
+
'translate',
|
|
154
|
+
shortSourceLanguageCode,
|
|
155
|
+
modelName,
|
|
156
|
+
modelPath,
|
|
157
|
+
whisperCppOptions,
|
|
158
|
+
));
|
|
159
|
+
|
|
160
|
+
({ segmentTimeline } = await wordTimelineToSegmentSentenceTimeline(wordTimeline, transcript, targetLanguage, 'single', 'preserve'))
|
|
161
|
+
|
|
162
|
+
break
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
case 'openai-cloud': {
|
|
166
|
+
const OpenAICloudSTT = await import('../recognition/OpenAICloudSTT.js')
|
|
167
|
+
|
|
168
|
+
const openAICloudSTTOptions = options.openAICloud!
|
|
169
|
+
|
|
170
|
+
if (!openAICloudSTTOptions.apiKey) {
|
|
171
|
+
throw new Error(`No OpenAI Cloud API key provided`)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const shortSourceLanguageCode = getShortLanguageCode(sourceLanguage)
|
|
175
|
+
const shortTargetLanguageCode = getShortLanguageCode(targetLanguage)
|
|
176
|
+
|
|
177
|
+
if (shortTargetLanguageCode != 'en') {
|
|
178
|
+
throw new Error('OpenAI cloud speech translation only supports English as target language')
|
|
76
179
|
}
|
|
77
180
|
|
|
78
181
|
logger.end();
|
|
79
182
|
|
|
80
|
-
({ transcript, timeline } = await
|
|
183
|
+
({ transcript, timeline: segmentTimeline } = await OpenAICloudSTT.recognize(sourceRawAudio, shortSourceLanguageCode, openAICloudSTTOptions, 'translate'))
|
|
81
184
|
|
|
82
185
|
break
|
|
83
186
|
}
|
|
@@ -87,57 +190,107 @@ export async function translateSpeech(input: AudioSourceParam, options: SpeechTr
|
|
|
87
190
|
}
|
|
88
191
|
}
|
|
89
192
|
|
|
90
|
-
|
|
193
|
+
logger.end()
|
|
91
194
|
|
|
92
|
-
|
|
195
|
+
// If the audio was cropped before recognition, map the timestamps back to the original audio
|
|
196
|
+
if (sourceUncropTimeline && sourceUncropTimeline.length > 0) {
|
|
197
|
+
API.convertCroppedToUncroppedTimeline(segmentTimeline, sourceUncropTimeline)
|
|
198
|
+
|
|
199
|
+
if (wordTimeline) {
|
|
200
|
+
API.convertCroppedToUncroppedTimeline(wordTimeline, sourceUncropTimeline)
|
|
201
|
+
}
|
|
202
|
+
}
|
|
93
203
|
|
|
94
|
-
logger.end()
|
|
95
204
|
logger.log('')
|
|
96
205
|
logger.logDuration(`Total speech translation time`, startTimestamp, chalk.magentaBright)
|
|
97
206
|
|
|
98
|
-
return {
|
|
207
|
+
return {
|
|
208
|
+
transcript,
|
|
209
|
+
timeline: segmentTimeline,
|
|
210
|
+
wordTimeline,
|
|
211
|
+
|
|
212
|
+
sourceLanguage,
|
|
213
|
+
targetLanguage,
|
|
214
|
+
|
|
215
|
+
inputRawAudio,
|
|
216
|
+
isolatedRawAudio,
|
|
217
|
+
backgroundRawAudio,
|
|
218
|
+
}
|
|
99
219
|
}
|
|
100
220
|
|
|
101
221
|
export interface SpeechTranslationResult {
|
|
102
222
|
transcript: string
|
|
103
223
|
timeline: Timeline
|
|
104
|
-
wordTimeline
|
|
224
|
+
wordTimeline?: Timeline
|
|
225
|
+
|
|
105
226
|
sourceLanguage: string
|
|
106
227
|
targetLanguage: string
|
|
228
|
+
|
|
107
229
|
inputRawAudio: RawAudio
|
|
230
|
+
isolatedRawAudio?: RawAudio
|
|
231
|
+
backgroundRawAudio?: RawAudio
|
|
108
232
|
}
|
|
109
233
|
|
|
110
|
-
export type SpeechTranslationEngine =
|
|
234
|
+
export type SpeechTranslationEngine = 'whisper' | 'whisper.cpp' | 'openai-cloud'
|
|
111
235
|
|
|
112
236
|
export interface SpeechTranslationOptions {
|
|
113
237
|
engine?: SpeechTranslationEngine
|
|
114
238
|
|
|
115
239
|
sourceLanguage?: string
|
|
116
240
|
targetLanguage?: string
|
|
241
|
+
|
|
242
|
+
crop?: boolean
|
|
243
|
+
isolate?: boolean
|
|
244
|
+
|
|
117
245
|
languageDetection?: SpeechLanguageDetectionOptions
|
|
118
246
|
subtitles?: SubtitlesConfig
|
|
247
|
+
vad?: API.VADOptions
|
|
248
|
+
sourceSeparation?: API.SourceSeparationOptions
|
|
119
249
|
|
|
120
250
|
whisper?: WhisperOptions
|
|
251
|
+
whisperCpp?: WhisperCppOptions
|
|
252
|
+
openAICloud?: OpenAICloudSTTOptions
|
|
121
253
|
}
|
|
122
254
|
|
|
123
255
|
export const defaultSpeechTranslationOptions: SpeechTranslationOptions = {
|
|
124
|
-
engine:
|
|
256
|
+
engine: 'whisper',
|
|
125
257
|
|
|
126
258
|
sourceLanguage: undefined,
|
|
127
|
-
targetLanguage:
|
|
259
|
+
targetLanguage: 'en',
|
|
260
|
+
|
|
261
|
+
crop: true,
|
|
262
|
+
isolate: false,
|
|
128
263
|
|
|
129
264
|
languageDetection: undefined,
|
|
130
265
|
|
|
131
266
|
subtitles: defaultSubtitlesBaseConfig,
|
|
132
267
|
|
|
133
|
-
|
|
268
|
+
vad: {
|
|
269
|
+
engine: 'adaptive-gate'
|
|
270
|
+
},
|
|
271
|
+
|
|
272
|
+
whisper: defaultWhisperOptions,
|
|
273
|
+
whisperCpp: defaultWhisperCppOptions,
|
|
274
|
+
openAICloud: defaultOpenAICloudSTTOptions,
|
|
134
275
|
}
|
|
135
276
|
|
|
136
277
|
export const speechTranslationEngines: EngineMetadata[] = [
|
|
137
278
|
{
|
|
138
279
|
id: 'whisper',
|
|
139
280
|
name: 'OpenAI Whisper',
|
|
140
|
-
description:
|
|
281
|
+
description: `Uses Whisper's speech translation capability to produce an English transcript from speech in a different language.`,
|
|
282
|
+
type: 'local'
|
|
283
|
+
},
|
|
284
|
+
{
|
|
285
|
+
id: 'whisper.cpp',
|
|
286
|
+
name: 'OpenAI Whisper (C++ port)',
|
|
287
|
+
description: `Uses Whisper's speech translation capability to produce an English transcript from speech in a different language.`,
|
|
141
288
|
type: 'local'
|
|
289
|
+
},
|
|
290
|
+
{
|
|
291
|
+
id: 'openai-cloud',
|
|
292
|
+
name: 'OpenAI Cloud',
|
|
293
|
+
description: 'Speech translation cloud service provided by OpenAI. Only support English as target language.',
|
|
294
|
+
type: 'cloud'
|
|
142
295
|
}
|
|
143
296
|
]
|
package/src/api/Vad.ts
CHANGED
|
@@ -1,22 +1,24 @@
|
|
|
1
|
-
import { extendDeep } from
|
|
1
|
+
import { extendDeep } from '../utilities/ObjectUtilities.js'
|
|
2
2
|
|
|
3
|
-
import { logToStderr } from
|
|
4
|
-
import { AudioSourceParam, RawAudio, ensureRawAudio, } from
|
|
5
|
-
import { Logger } from
|
|
3
|
+
import { logToStderr } from '../utilities/Utilities.js'
|
|
4
|
+
import { AudioSourceParam, RawAudio, cropToTimeline, ensureRawAudio, } from '../audio/AudioUtilities.js'
|
|
5
|
+
import { Logger } from '../utilities/Logger.js'
|
|
6
6
|
|
|
7
|
-
import { Timeline } from
|
|
8
|
-
import path from
|
|
9
|
-
import { loadPackage } from
|
|
10
|
-
import { EngineMetadata } from
|
|
11
|
-
import chalk from
|
|
7
|
+
import { Timeline } from '../utilities/Timeline.js'
|
|
8
|
+
import path from 'path'
|
|
9
|
+
import { loadPackage } from '../utilities/PackageManager.js'
|
|
10
|
+
import { EngineMetadata } from './Common.js'
|
|
11
|
+
import chalk from 'chalk'
|
|
12
|
+
import { type AdaptiveGateVADOptions } from '../voice-activity-detection/AdaptiveGateVAD.js'
|
|
12
13
|
|
|
13
14
|
const log = logToStderr
|
|
14
15
|
|
|
15
16
|
export async function detectVoiceActivity(input: AudioSourceParam, options: VADOptions): Promise<VADResult> {
|
|
16
17
|
const logger = new Logger()
|
|
18
|
+
|
|
17
19
|
const startTimestamp = logger.getTimestamp()
|
|
18
20
|
|
|
19
|
-
logger.start(
|
|
21
|
+
logger.start('Prepare for voice activity detection')
|
|
20
22
|
|
|
21
23
|
const inputRawAudio = await ensureRawAudio(input)
|
|
22
24
|
|
|
@@ -26,46 +28,65 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
|
|
|
26
28
|
|
|
27
29
|
logger.start(`Detect voice activity with ${options.engine}`)
|
|
28
30
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
+
const activityThreshold = options.activityThreshold!
|
|
32
|
+
|
|
33
|
+
let verboseTimeline: Timeline
|
|
31
34
|
|
|
32
35
|
switch (options.engine) {
|
|
33
|
-
case
|
|
34
|
-
const WebRtcVAD = await import(
|
|
36
|
+
case 'webrtc': {
|
|
37
|
+
const WebRtcVAD = await import('../voice-activity-detection/WebRtcVAD.js')
|
|
35
38
|
|
|
36
39
|
const webrtcOptions = options.webrtc!
|
|
37
40
|
|
|
38
|
-
frameProbabilities = await WebRtcVAD.detectVoiceActivity(sourceRawAudio, webrtcOptions.frameDuration!)
|
|
39
|
-
frameDurationSeconds = webrtcOptions.frameDuration! / 1000
|
|
41
|
+
const frameProbabilities = await WebRtcVAD.detectVoiceActivity(sourceRawAudio, webrtcOptions.frameDuration!)
|
|
42
|
+
const frameDurationSeconds = webrtcOptions.frameDuration! / 1000
|
|
43
|
+
|
|
44
|
+
verboseTimeline = frameProbabilitiesToTimeline(frameProbabilities, frameDurationSeconds, activityThreshold)
|
|
40
45
|
|
|
41
46
|
break
|
|
42
47
|
}
|
|
43
48
|
|
|
44
|
-
case
|
|
45
|
-
const SileroVAD = await import(
|
|
49
|
+
case 'silero': {
|
|
50
|
+
const SileroVAD = await import('../voice-activity-detection/SileroVAD.js')
|
|
46
51
|
|
|
47
52
|
const sileroOptions = options.silero!
|
|
48
53
|
|
|
49
|
-
const modelDir = await loadPackage(
|
|
54
|
+
const modelDir = await loadPackage('silero-vad')
|
|
50
55
|
|
|
51
|
-
const modelPath = path.join(modelDir,
|
|
56
|
+
const modelPath = path.join(modelDir, 'silero-vad.onnx')
|
|
52
57
|
const frameDuration = sileroOptions.frameDuration!
|
|
53
58
|
|
|
54
|
-
frameProbabilities = await SileroVAD.detectVoiceActivity(sourceRawAudio, modelPath, frameDuration)
|
|
55
|
-
frameDurationSeconds = sileroOptions.frameDuration! / 1000
|
|
59
|
+
const frameProbabilities = await SileroVAD.detectVoiceActivity(sourceRawAudio, modelPath, frameDuration)
|
|
60
|
+
const frameDurationSeconds = sileroOptions.frameDuration! / 1000
|
|
61
|
+
|
|
62
|
+
verboseTimeline = frameProbabilitiesToTimeline(frameProbabilities, frameDurationSeconds, activityThreshold)
|
|
56
63
|
|
|
57
64
|
break
|
|
58
65
|
}
|
|
59
66
|
|
|
60
|
-
case
|
|
61
|
-
const RNNoise = await import(
|
|
67
|
+
case 'rnnoise': {
|
|
68
|
+
const RNNoise = await import('../denoising/RNNoise.js')
|
|
69
|
+
|
|
70
|
+
const audio48k = await ensureRawAudio(sourceRawAudio, 48000, 1)
|
|
62
71
|
|
|
63
72
|
const rnnoiseOptions = options.rnnoise!
|
|
64
73
|
|
|
65
|
-
const { denoisedRawAudio, frameVadProbabilities } = await RNNoise.denoiseAudio(
|
|
74
|
+
const { denoisedRawAudio, frameVadProbabilities } = await RNNoise.denoiseAudio(audio48k)
|
|
75
|
+
|
|
76
|
+
const frameDurationSeconds = 0.01
|
|
77
|
+
const frameProbabilities = frameVadProbabilities
|
|
78
|
+
|
|
79
|
+
verboseTimeline = frameProbabilitiesToTimeline(frameProbabilities, frameDurationSeconds, activityThreshold)
|
|
80
|
+
|
|
81
|
+
break
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
case 'adaptive-gate': {
|
|
85
|
+
const AdaptiveGateVAD = await import('../voice-activity-detection/AdaptiveGateVAD.js')
|
|
66
86
|
|
|
67
|
-
|
|
68
|
-
|
|
87
|
+
const adaptiveGateOptions = options.adaptiveGate!
|
|
88
|
+
|
|
89
|
+
verboseTimeline = await AdaptiveGateVAD.detectVoiceActivity(sourceRawAudio, adaptiveGateOptions)
|
|
69
90
|
|
|
70
91
|
break
|
|
71
92
|
}
|
|
@@ -75,6 +96,18 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
|
|
|
75
96
|
}
|
|
76
97
|
}
|
|
77
98
|
|
|
99
|
+
const timeline = verboseTimeline.filter(entry => entry.text === 'active')
|
|
100
|
+
|
|
101
|
+
const croppedRawAudio = cropToTimeline(inputRawAudio, timeline)
|
|
102
|
+
|
|
103
|
+
logger.end()
|
|
104
|
+
logger.log('')
|
|
105
|
+
logger.logDuration(`Total voice activity detection time`, startTimestamp, chalk.magentaBright)
|
|
106
|
+
|
|
107
|
+
return { timeline, verboseTimeline, inputRawAudio, croppedRawAudio }
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function frameProbabilitiesToTimeline(frameProbabilities: number[], frameDurationSeconds: number, activityThreshold: number) {
|
|
78
111
|
const timeline: Timeline = []
|
|
79
112
|
|
|
80
113
|
for (let i = 0; i < frameProbabilities.length; i++) {
|
|
@@ -83,14 +116,16 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
|
|
|
83
116
|
const startTime = i * frameDurationSeconds
|
|
84
117
|
const endTime = (i + 1) * frameDurationSeconds
|
|
85
118
|
|
|
86
|
-
if (frameProbability >=
|
|
87
|
-
if (timeline.length == 0 || timeline[timeline.length - 1].text ==
|
|
88
|
-
timeline.push({ type:
|
|
119
|
+
if (frameProbability >= activityThreshold) {
|
|
120
|
+
if (timeline.length == 0 || timeline[timeline.length - 1].text == 'inactive') {
|
|
121
|
+
timeline.push({ type: 'segment', text: 'active', startTime, endTime })
|
|
122
|
+
|
|
89
123
|
continue
|
|
90
124
|
}
|
|
91
125
|
} else {
|
|
92
|
-
if (timeline.length == 0 || timeline[timeline.length - 1].text ==
|
|
93
|
-
timeline.push({ type:
|
|
126
|
+
if (timeline.length == 0 || timeline[timeline.length - 1].text == 'active') {
|
|
127
|
+
timeline.push({ type: 'segment', text: 'inactive', startTime, endTime })
|
|
128
|
+
|
|
94
129
|
continue
|
|
95
130
|
}
|
|
96
131
|
}
|
|
@@ -98,19 +133,50 @@ export async function detectVoiceActivity(input: AudioSourceParam, options: VADO
|
|
|
98
133
|
timeline[timeline.length - 1].endTime = endTime
|
|
99
134
|
}
|
|
100
135
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
136
|
+
return timeline
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
export function convertCroppedToUncroppedTimeline(timeline: Timeline, uncropTimeline: Timeline) {
|
|
140
|
+
for (const entry of timeline) {
|
|
141
|
+
entry.startTime = mapTimestampUsingUncropTimeline(entry.startTime, uncropTimeline)
|
|
142
|
+
entry.endTime = mapTimestampUsingUncropTimeline(entry.endTime, uncropTimeline)
|
|
104
143
|
|
|
105
|
-
|
|
144
|
+
if (entry.timeline) {
|
|
145
|
+
convertCroppedToUncroppedTimeline(entry.timeline, uncropTimeline)
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function mapTimestampUsingUncropTimeline(timeInCroppedAudio: number, uncropTimeline: Timeline) {
|
|
151
|
+
let offsetInCroppedAudio = 0
|
|
152
|
+
|
|
153
|
+
for (let i = 0; i < uncropTimeline.length; i++) {
|
|
154
|
+
const entry = uncropTimeline[i]
|
|
155
|
+
|
|
156
|
+
const entryDuration = entry.endTime - entry.startTime
|
|
157
|
+
|
|
158
|
+
const endOffset = offsetInCroppedAudio + entryDuration
|
|
159
|
+
|
|
160
|
+
if ((i === uncropTimeline.length - 1) ||
|
|
161
|
+
(timeInCroppedAudio >= offsetInCroppedAudio && timeInCroppedAudio < endOffset)) {
|
|
162
|
+
return entry.startTime + (timeInCroppedAudio - offsetInCroppedAudio)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
offsetInCroppedAudio += entryDuration
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
throw new Error(`Should not be reached`)
|
|
106
169
|
}
|
|
107
170
|
|
|
108
171
|
export interface VADResult {
|
|
109
172
|
timeline: Timeline
|
|
173
|
+
verboseTimeline: Timeline
|
|
174
|
+
|
|
110
175
|
inputRawAudio: RawAudio
|
|
176
|
+
croppedRawAudio: RawAudio
|
|
111
177
|
}
|
|
112
178
|
|
|
113
|
-
export type VADEngine =
|
|
179
|
+
export type VADEngine = 'webrtc' | 'silero' | 'rnnoise' | 'adaptive-gate'
|
|
114
180
|
|
|
115
181
|
export interface VADOptions {
|
|
116
182
|
engine?: VADEngine
|
|
@@ -123,16 +189,17 @@ export interface VADOptions {
|
|
|
123
189
|
}
|
|
124
190
|
|
|
125
191
|
silero?: {
|
|
126
|
-
modelPath?: string
|
|
127
192
|
frameDuration?: 30 | 60 | 90
|
|
128
193
|
}
|
|
129
194
|
|
|
130
195
|
rnnoise?: {
|
|
131
196
|
}
|
|
197
|
+
|
|
198
|
+
adaptiveGate?: AdaptiveGateVADOptions
|
|
132
199
|
}
|
|
133
200
|
|
|
134
201
|
export const defaultVADOptions: VADOptions = {
|
|
135
|
-
engine:
|
|
202
|
+
engine: 'silero',
|
|
136
203
|
|
|
137
204
|
activityThreshold: 0.5,
|
|
138
205
|
|
|
@@ -142,11 +209,13 @@ export const defaultVADOptions: VADOptions = {
|
|
|
142
209
|
},
|
|
143
210
|
|
|
144
211
|
silero: {
|
|
145
|
-
modelPath: undefined,
|
|
146
212
|
frameDuration: 90,
|
|
147
213
|
},
|
|
148
214
|
|
|
149
215
|
rnnoise: {
|
|
216
|
+
},
|
|
217
|
+
|
|
218
|
+
adaptiveGate: {
|
|
150
219
|
}
|
|
151
220
|
}
|
|
152
221
|
|
|
@@ -166,7 +235,7 @@ export const vadEngines: EngineMetadata[] = [
|
|
|
166
235
|
{
|
|
167
236
|
id: 'rnnoise',
|
|
168
237
|
name: 'RNNoise',
|
|
169
|
-
description:
|
|
238
|
+
description: `Uses RNNoise's internal speech probabilities as VAD metrics.`,
|
|
170
239
|
type: 'local'
|
|
171
240
|
}
|
|
172
241
|
]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import AlawMulaw from
|
|
1
|
+
import AlawMulaw from 'alawmulaw'
|
|
2
2
|
import * as BinaryArrayConversion from '../utilities/BinaryArrayConversion.js'
|
|
3
|
-
import { BitDepth, SampleFormat } from
|
|
3
|
+
import { BitDepth, SampleFormat } from '../codecs/WaveCodec.js'
|
|
4
4
|
|
|
5
5
|
/////////////////////////////////////////////////////////////////////////////////////////////
|
|
6
6
|
// Low level audio sample conversions
|
|
@@ -186,7 +186,7 @@ export function interleaveChannels(channels: Float32Array[]) {
|
|
|
186
186
|
const channelCount = channels.length
|
|
187
187
|
|
|
188
188
|
if (channelCount == 0) {
|
|
189
|
-
throw new Error(
|
|
189
|
+
throw new Error('Empty channel array received')
|
|
190
190
|
}
|
|
191
191
|
|
|
192
192
|
if (channelCount == 1) {
|
|
@@ -210,7 +210,7 @@ export function interleaveChannels(channels: Float32Array[]) {
|
|
|
210
210
|
|
|
211
211
|
export function deInterleaveChannels(interleavedChannels: Float32Array, channelCount: number) {
|
|
212
212
|
if (channelCount == 0) {
|
|
213
|
-
throw new Error(
|
|
213
|
+
throw new Error('0 channel count received')
|
|
214
214
|
}
|
|
215
215
|
|
|
216
216
|
if (channelCount == 1) {
|