echogarden 0.12.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -14
- package/data/schemas/options.json +398 -111
- package/dist/alignment/DTWMfccSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWMfccSequenceAlignment.js +8 -8
- package/dist/alignment/DTWSequenceAlignment.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignment.js +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.d.ts +1 -1
- package/dist/alignment/DTWSequenceAlignmentWindowed.js +2 -2
- package/dist/alignment/LevenshteinSequenceAlignment.d.ts +1 -1
- package/dist/alignment/LevenshteinSequenceAlignment.js +1 -1
- package/dist/alignment/SpeechAlignment.d.ts +9 -10
- package/dist/alignment/SpeechAlignment.js +136 -105
- package/dist/alignment/SpeechAlignment.js.map +1 -1
- package/dist/api/API.d.ts +13 -12
- package/dist/api/API.js +14 -13
- package/dist/api/API.js.map +1 -1
- package/dist/api/APIOptions.d.ts +5 -4
- package/dist/api/Alignment.d.ts +15 -9
- package/dist/api/Alignment.js +88 -74
- package/dist/api/Alignment.js.map +1 -1
- package/dist/api/Common.js +1 -1
- package/dist/api/Denoising.d.ts +6 -6
- package/dist/api/Denoising.js +23 -23
- package/dist/api/Denoising.js.map +1 -1
- package/dist/api/LanguageDetection.d.ts +19 -12
- package/dist/api/LanguageDetection.js +88 -38
- package/dist/api/LanguageDetection.js.map +1 -1
- package/dist/api/Recognition.d.ts +16 -6
- package/dist/api/Recognition.js +129 -55
- package/dist/api/Recognition.js.map +1 -1
- package/dist/api/SourceSeparation.d.ts +17 -0
- package/dist/api/SourceSeparation.js +61 -0
- package/dist/api/SourceSeparation.js.map +1 -0
- package/dist/api/Synthesis.d.ts +18 -18
- package/dist/api/Synthesis.js +191 -164
- package/dist/api/Synthesis.js.map +1 -1
- package/dist/api/Translation.d.ts +19 -8
- package/dist/api/Translation.js +132 -35
- package/dist/api/Translation.js.map +1 -1
- package/dist/api/Vad.d.ts +10 -5
- package/dist/api/Vad.js +76 -38
- package/dist/api/Vad.js.map +1 -1
- package/dist/audio/AudioBufferConversion.d.ts +1 -1
- package/dist/audio/AudioBufferConversion.js +4 -4
- package/dist/audio/AudioPlayer.d.ts +1 -1
- package/dist/audio/AudioPlayer.js +26 -26
- package/dist/audio/AudioPlayer.js.map +1 -1
- package/dist/audio/AudioRecorder.d.ts +1 -1
- package/dist/audio/AudioRecorder.js +5 -5
- package/dist/audio/AudioUtilities.d.ts +13 -9
- package/dist/audio/AudioUtilities.js +86 -24
- package/dist/audio/AudioUtilities.js.map +1 -1
- package/dist/cli/CLI.d.ts +3 -3
- package/dist/cli/CLI.js +271 -162
- package/dist/cli/CLI.js.map +1 -1
- package/dist/cli/CLIConfigFile.js +8 -8
- package/dist/cli/CLILauncher.js +6 -6
- package/dist/cli/CLIOptionsSchema.js +2 -2
- package/dist/cli/CLIParser.js +5 -5
- package/dist/cli/CLIStarter.js +4 -4
- package/dist/codecs/FFMpegTranscoder.d.ts +2 -2
- package/dist/codecs/FFMpegTranscoder.js +37 -37
- package/dist/codecs/FFMpegTranscoder.js.map +1 -1
- package/dist/codecs/TIMITCodec.js +5 -5
- package/dist/codecs/WaveCodec.d.ts +1 -1
- package/dist/codecs/WaveCodec.js +22 -22
- package/dist/denoising/RNNoise.d.ts +1 -1
- package/dist/denoising/RNNoise.js +9 -9
- package/dist/dsp/BiquadFilter.d.ts +3 -2
- package/dist/dsp/BiquadFilter.js +18 -11
- package/dist/dsp/BiquadFilter.js.map +1 -1
- package/dist/dsp/DecayingPeakEstimator.d.ts +16 -0
- package/dist/dsp/DecayingPeakEstimator.js +23 -0
- package/dist/dsp/DecayingPeakEstimator.js.map +1 -0
- package/dist/dsp/FFT.d.ts +8 -4
- package/dist/dsp/FFT.js +76 -30
- package/dist/dsp/FFT.js.map +1 -1
- package/dist/dsp/KWeightingFilter.d.ts +9 -0
- package/dist/dsp/KWeightingFilter.js +40 -0
- package/dist/dsp/KWeightingFilter.js.map +1 -0
- package/dist/dsp/LoudnessEstimator.d.ts +21 -0
- package/dist/dsp/LoudnessEstimator.js +47 -0
- package/dist/dsp/LoudnessEstimator.js.map +1 -0
- package/dist/dsp/MFCC.d.ts +2 -2
- package/dist/dsp/MFCC.js +15 -15
- package/dist/dsp/MelSpectogram.d.ts +1 -1
- package/dist/dsp/MelSpectogram.js +6 -6
- package/dist/dsp/Rubberband.d.ts +11 -11
- package/dist/dsp/Rubberband.js +27 -27
- package/dist/dsp/Sonic.d.ts +1 -1
- package/dist/dsp/Sonic.js +3 -3
- package/dist/dsp/SpeexResampler.d.ts +1 -1
- package/dist/dsp/SpeexResampler.js +2 -2
- package/dist/math/VectorMath.d.ts +12 -8
- package/dist/math/VectorMath.js +35 -32
- package/dist/math/VectorMath.js.map +1 -1
- package/dist/nlp/ChineseSegmentation.js +2 -2
- package/dist/nlp/CompromiseNLP.js +3 -3
- package/dist/nlp/EspeakPhonemizer.js +30 -30
- package/dist/nlp/IPA.js +20 -20
- package/dist/nlp/JapaneseSegmentation.js +6 -6
- package/dist/nlp/Lexicon.d.ts +1 -1
- package/dist/nlp/Lexicon.js +7 -7
- package/dist/nlp/Segmentation.d.ts +3 -0
- package/dist/nlp/Segmentation.js +21 -14
- package/dist/nlp/Segmentation.js.map +1 -1
- package/dist/nlp/TextNormalizer.js +16 -16
- package/dist/recognition/AmazonTranscribeSTT.d.ts +2 -2
- package/dist/recognition/AmazonTranscribeSTT.js +13 -14
- package/dist/recognition/AmazonTranscribeSTT.js.map +1 -1
- package/dist/recognition/AzureCognitiveServicesSTT.js +5 -6
- package/dist/recognition/AzureCognitiveServicesSTT.js.map +1 -1
- package/dist/recognition/GoogleCloudSTT.d.ts +3 -3
- package/dist/recognition/GoogleCloudSTT.js +18 -18
- package/dist/recognition/OpenAICloudSTT.d.ts +19 -0
- package/dist/recognition/OpenAICloudSTT.js +81 -0
- package/dist/recognition/OpenAICloudSTT.js.map +1 -0
- package/dist/recognition/SileroSTT.d.ts +2 -2
- package/dist/recognition/SileroSTT.js +25 -25
- package/dist/recognition/VoskSTT.d.ts +2 -2
- package/dist/recognition/VoskSTT.js +8 -8
- package/dist/recognition/WhisperCppSTT.d.ts +88 -0
- package/dist/recognition/WhisperCppSTT.js +332 -0
- package/dist/recognition/WhisperCppSTT.js.map +1 -0
- package/dist/recognition/WhisperSTT.d.ts +49 -25
- package/dist/recognition/WhisperSTT.js +626 -481
- package/dist/recognition/WhisperSTT.js.map +1 -1
- package/dist/server/Client.d.ts +1 -1
- package/dist/server/Client.js +22 -22
- package/dist/server/Server.js +9 -9
- package/dist/server/Server.js.map +1 -1
- package/dist/server/Worker.d.ts +22 -22
- package/dist/server/Worker.js +36 -36
- package/dist/server/Worker.js.map +1 -1
- package/dist/server/WorkerStarter.js +2 -2
- package/dist/source-separation/MDXNetSourceSeparation.d.ts +11 -0
- package/dist/source-separation/MDXNetSourceSeparation.js +161 -0
- package/dist/source-separation/MDXNetSourceSeparation.js.map +1 -0
- package/dist/speech-language-detection/SileroLanguageDetection.d.ts +1 -1
- package/dist/speech-language-detection/SileroLanguageDetection.js +7 -7
- package/dist/subtitles/Subtitles.d.ts +10 -0
- package/dist/subtitles/Subtitles.js +2 -2
- package/dist/subtitles/Subtitles.js.map +1 -1
- package/dist/synthesis/AwsPollyTTS.d.ts +1 -1
- package/dist/synthesis/AwsPollyTTS.js +12 -12
- package/dist/synthesis/AzureCognitiveServicesTTS.js +7 -7
- package/dist/synthesis/CoquiServerTTS.js +10 -10
- package/dist/synthesis/CoquiServerTTS.js.map +1 -1
- package/dist/synthesis/ElevenlabsTTS.d.ts +23 -0
- package/dist/synthesis/ElevenlabsTTS.js +103 -0
- package/dist/synthesis/ElevenlabsTTS.js.map +1 -0
- package/dist/synthesis/EspeakTTS.d.ts +6 -5
- package/dist/synthesis/EspeakTTS.js +81 -69
- package/dist/synthesis/EspeakTTS.js.map +1 -1
- package/dist/synthesis/FliteTTS.d.ts +3 -3
- package/dist/synthesis/FliteTTS.js +154 -154
- package/dist/synthesis/FliteTTS.js.map +1 -1
- package/dist/synthesis/GoogleCloudTTS.d.ts +3 -3
- package/dist/synthesis/GoogleCloudTTS.js +17 -17
- package/dist/synthesis/GoogleCloudTTS.js.map +1 -1
- package/dist/synthesis/GoogleTranslateTTS.d.ts +1 -1
- package/dist/synthesis/GoogleTranslateTTS.js +103 -103
- package/dist/synthesis/MicrosoftEdgeTTS.d.ts +2 -2
- package/dist/synthesis/MicrosoftEdgeTTS.js +74 -74
- package/dist/synthesis/OpenAICloudTTS.d.ts +13 -0
- package/dist/synthesis/OpenAICloudTTS.js +169 -0
- package/dist/synthesis/OpenAICloudTTS.js.map +1 -0
- package/dist/synthesis/SamTTS.js +3 -3
- package/dist/synthesis/SapiTTS.d.ts +3 -3
- package/dist/synthesis/SapiTTS.js +26 -26
- package/dist/synthesis/StreamlabsPollyTTS.d.ts +2 -2
- package/dist/synthesis/StreamlabsPollyTTS.js +27 -27
- package/dist/synthesis/SvoxPicoTTS.d.ts +2 -2
- package/dist/synthesis/SvoxPicoTTS.js +65 -65
- package/dist/synthesis/SvoxPicoTTS.js.map +1 -1
- package/dist/synthesis/VitsTTS.d.ts +3 -3
- package/dist/synthesis/VitsTTS.js +378 -378
- package/dist/synthesis/VitsTTS.js.map +1 -1
- package/dist/tests/Test.js +2 -2
- package/dist/utilities/Compression.d.ts +5 -0
- package/dist/utilities/Compression.js +29 -13
- package/dist/utilities/Compression.js.map +1 -1
- package/dist/utilities/FileDownloader.d.ts +1 -1
- package/dist/utilities/FileDownloader.js +16 -16
- package/dist/utilities/FileSystem.js +7 -7
- package/dist/utilities/Locale.d.ts +7 -7
- package/dist/utilities/Locale.js +15 -15
- package/dist/utilities/Logger.js +3 -3
- package/dist/utilities/ObjectUtilities.js +19 -19
- package/dist/utilities/OpenPromise.js +2 -2
- package/dist/utilities/OpenPromise.js.map +1 -1
- package/dist/utilities/PackageManager.js +31 -0
- package/dist/utilities/PackageManager.js.map +1 -1
- package/dist/utilities/PathUtilities.js +8 -8
- package/dist/utilities/RandomGenerator.js +2 -2
- package/dist/utilities/SmoothEstimator.d.ts +8 -0
- package/dist/utilities/SmoothEstimator.js +25 -0
- package/dist/utilities/SmoothEstimator.js.map +1 -0
- package/dist/utilities/TarballMaker.js +8 -8
- package/dist/utilities/Timeline.d.ts +3 -2
- package/dist/utilities/Timeline.js +11 -11
- package/dist/utilities/Timeline.js.map +1 -1
- package/dist/utilities/Timer.js +4 -4
- package/dist/utilities/Utilities.d.ts +4 -0
- package/dist/utilities/Utilities.js +38 -15
- package/dist/utilities/Utilities.js.map +1 -1
- package/dist/utilities/WasmMemoryManager.js +7 -7
- package/dist/utilities/WebReader.js +23 -23
- package/dist/utilities/WikipediaReader.js +2 -2
- package/dist/voice-activity-detection/AdaptiveGateVAD.d.ts +28 -0
- package/dist/voice-activity-detection/AdaptiveGateVAD.js +138 -0
- package/dist/voice-activity-detection/AdaptiveGateVAD.js.map +1 -0
- package/dist/voice-activity-detection/SileroVAD.d.ts +1 -1
- package/dist/voice-activity-detection/SileroVAD.js +5 -5
- package/dist/voice-activity-detection/SileroVAD.js.map +1 -1
- package/dist/voice-activity-detection/WebRtcVAD.d.ts +1 -1
- package/dist/voice-activity-detection/WebRtcVAD.js +4 -4
- package/docs/API.md +29 -11
- package/docs/CLI.md +31 -7
- package/docs/Contributing.md +38 -0
- package/docs/Development.md +93 -19
- package/docs/Engines.md +28 -16
- package/docs/Licenses.md +4 -1
- package/docs/Options.md +158 -78
- package/docs/Releases.md +262 -0
- package/docs/Server.md +7 -7
- package/docs/Tasklist.md +95 -76
- package/docs/Technical.md +4 -4
- package/package.json +13 -14
- package/src/alignment/DTWMfccSequenceAlignment.ts +9 -9
- package/src/alignment/DTWSequenceAlignment.ts +2 -2
- package/src/alignment/DTWSequenceAlignmentWindowed.ts +3 -3
- package/src/alignment/LevenshteinSequenceAlignment.ts +2 -2
- package/src/alignment/SpeechAlignment.ts +204 -119
- package/src/api/API.ts +14 -13
- package/src/api/APIOptions.ts +12 -11
- package/src/api/Alignment.ts +147 -90
- package/src/api/Common.ts +1 -1
- package/src/api/Denoising.ts +28 -28
- package/src/api/LanguageDetection.ts +135 -48
- package/src/api/Recognition.ts +198 -59
- package/src/api/SourceSeparation.ts +99 -0
- package/src/api/Synthesis.ts +217 -181
- package/src/api/Translation.ts +193 -40
- package/src/api/Vad.ts +110 -41
- package/src/audio/AudioBufferConversion.ts +4 -4
- package/src/audio/AudioPlayer.ts +27 -27
- package/src/audio/AudioRecorder.ts +5 -5
- package/src/audio/AudioUtilities.ts +107 -24
- package/src/cli/CLI.ts +313 -164
- package/src/cli/CLIConfigFile.ts +8 -8
- package/src/cli/CLILauncher.ts +6 -6
- package/src/cli/CLIOptionsSchema.ts +2 -2
- package/src/cli/CLIParser.ts +5 -5
- package/src/cli/CLIStarter.ts +4 -4
- package/src/codecs/FFMpegTranscoder.ts +38 -38
- package/src/codecs/TIMITCodec.ts +5 -5
- package/src/codecs/WaveCodec.ts +22 -22
- package/src/denoising/RNNoise.ts +9 -9
- package/src/dsp/BiquadFilter.ts +19 -11
- package/src/dsp/DecayingPeakEstimator.ts +35 -0
- package/src/dsp/FFT.ts +103 -35
- package/src/dsp/KWeightingFilter.ts +43 -0
- package/src/dsp/LoudnessEstimator.ts +74 -0
- package/src/dsp/MFCC.ts +15 -15
- package/src/dsp/MelSpectogram.ts +7 -7
- package/src/dsp/Rubberband.ts +38 -38
- package/src/dsp/Sonic.ts +4 -4
- package/src/dsp/SpeexResampler.ts +2 -2
- package/src/math/VectorMath.ts +42 -33
- package/src/nlp/ChineseSegmentation.ts +3 -3
- package/src/nlp/CompromiseNLP.ts +3 -3
- package/src/nlp/EspeakPhonemizer.ts +30 -30
- package/src/nlp/IPA.ts +20 -20
- package/src/nlp/JapaneseSegmentation.ts +6 -6
- package/src/nlp/Lexicon.ts +8 -8
- package/src/nlp/Segmentation.ts +23 -14
- package/src/nlp/TextNormalizer.ts +16 -16
- package/src/recognition/AmazonTranscribeSTT.ts +16 -17
- package/src/recognition/AzureCognitiveServicesSTT.ts +8 -6
- package/src/recognition/GoogleCloudSTT.ts +21 -21
- package/src/recognition/OpenAICloudSTT.ts +142 -0
- package/src/recognition/SileroSTT.ts +26 -26
- package/src/recognition/VoskSTT.ts +10 -10
- package/src/recognition/WhisperCppSTT.ts +555 -0
- package/src/recognition/WhisperSTT.ts +760 -507
- package/src/server/Client.ts +23 -23
- package/src/server/Server.ts +9 -9
- package/src/server/Worker.ts +53 -53
- package/src/server/WorkerStarter.ts +2 -2
- package/src/source-separation/MDXNetSourceSeparation.ts +228 -0
- package/src/speech-language-detection/SileroLanguageDetection.ts +8 -8
- package/src/subtitles/Subtitles.ts +3 -3
- package/src/synthesis/AwsPollyTTS.ts +14 -14
- package/src/synthesis/AzureCognitiveServicesTTS.ts +10 -10
- package/src/synthesis/CoquiServerTTS.ts +10 -10
- package/src/synthesis/ElevenlabsTTS.ts +137 -0
- package/src/synthesis/EspeakTTS.ts +90 -71
- package/src/synthesis/FliteTTS.ts +157 -157
- package/src/synthesis/GoogleCloudTTS.ts +19 -19
- package/src/synthesis/GoogleTranslateTTS.ts +104 -104
- package/src/synthesis/MicrosoftEdgeTTS.ts +80 -80
- package/src/synthesis/OpenAICloudTTS.ts +196 -0
- package/src/synthesis/SamTTS.ts +3 -3
- package/src/synthesis/SapiTTS.ts +29 -29
- package/src/synthesis/StreamlabsPollyTTS.ts +29 -29
- package/src/synthesis/SvoxPicoTTS.ts +67 -67
- package/src/synthesis/VitsTTS.ts +380 -380
- package/src/tests/Test.ts +4 -4
- package/src/utilities/Compression.ts +34 -13
- package/src/utilities/FileDownloader.ts +19 -19
- package/src/utilities/FileSystem.ts +7 -7
- package/src/utilities/Locale.ts +22 -22
- package/src/utilities/Logger.ts +4 -4
- package/src/utilities/ObjectUtilities.ts +19 -19
- package/src/utilities/OpenPromise.ts +2 -2
- package/src/utilities/PackageManager.ts +40 -0
- package/src/utilities/PathUtilities.ts +8 -8
- package/src/utilities/RandomGenerator.ts +3 -3
- package/src/utilities/SmoothEstimator.ts +35 -0
- package/src/utilities/TarballMaker.ts +9 -9
- package/src/utilities/Timeline.ts +15 -13
- package/src/utilities/Timer.ts +4 -4
- package/src/utilities/Utilities.ts +49 -15
- package/src/utilities/WasmMemoryManager.ts +7 -7
- package/src/utilities/WebReader.ts +23 -23
- package/src/utilities/WikipediaReader.ts +2 -2
- package/src/voice-activity-detection/AdaptiveGateVAD.ts +202 -0
- package/src/voice-activity-detection/SileroVAD.ts +5 -5
- package/src/voice-activity-detection/WebRtcVAD.ts +5 -5
- package/dist/synthesis/ElevenLabsTTS.d.ts +0 -8
- package/dist/synthesis/ElevenLabsTTS.js +0 -82
- package/dist/synthesis/ElevenLabsTTS.js.map +0 -1
- package/src/synthesis/ElevenLabsTTS.ts +0 -104
package/dist/server/Worker.d.ts
CHANGED
|
@@ -1,75 +1,75 @@
|
|
|
1
1
|
/// <reference types="node" resolution-mode="require"/>
|
|
2
|
-
import { RequestVoiceListResult, SynthesisOptions, SynthesisSegmentEventData, SynthesisResult, VoiceListRequestOptions } from
|
|
3
|
-
import { AudioSourceParam } from
|
|
4
|
-
import { RecognitionOptions, RecognitionResult } from
|
|
5
|
-
import { AlignmentOptions, AlignmentResult } from
|
|
6
|
-
import { SpeechTranslationOptions, SpeechTranslationResult } from
|
|
2
|
+
import { RequestVoiceListResult, SynthesisOptions, SynthesisSegmentEventData, SynthesisResult, VoiceListRequestOptions } from '../api/Synthesis.js';
|
|
3
|
+
import { AudioSourceParam } from '../audio/AudioUtilities.js';
|
|
4
|
+
import { RecognitionOptions, RecognitionResult } from '../api/Recognition.js';
|
|
5
|
+
import { AlignmentOptions, AlignmentResult } from '../api/Alignment.js';
|
|
6
|
+
import { SpeechTranslationOptions, SpeechTranslationResult } from '../api/Translation.js';
|
|
7
7
|
import { Worker } from 'node:worker_threads';
|
|
8
|
-
import { SpeechLanguageDetectionOptions, SpeechLanguageDetectionResult, TextLanguageDetectionOptions, TextLanguageDetectionResult } from
|
|
8
|
+
import { SpeechLanguageDetectionOptions, SpeechLanguageDetectionResult, TextLanguageDetectionOptions, TextLanguageDetectionResult } from '../api/LanguageDetection.js';
|
|
9
9
|
export declare function startMessageChannel(): void;
|
|
10
10
|
export declare function shouldCancelCurrentTask(): boolean;
|
|
11
11
|
export declare function processMessage(message: WorkerRequestMessage, sendMessage: MessageFunc): Promise<void>;
|
|
12
12
|
export interface SynthesisRequestMessage extends WorkerMessageBase {
|
|
13
|
-
messageType:
|
|
13
|
+
messageType: 'SynthesisRequest';
|
|
14
14
|
input: string | string[];
|
|
15
15
|
options: SynthesisOptions;
|
|
16
16
|
}
|
|
17
17
|
export interface SynthesisResponseMessage extends WorkerMessageBase, SynthesisResult {
|
|
18
|
-
messageType:
|
|
18
|
+
messageType: 'SynthesisResponse';
|
|
19
19
|
}
|
|
20
20
|
export interface SynthesisSegmentEventMessage extends WorkerMessageBase, SynthesisSegmentEventData {
|
|
21
|
-
messageType:
|
|
21
|
+
messageType: 'SynthesisSegmentEvent';
|
|
22
22
|
}
|
|
23
23
|
export interface SynthesisSentenceEventMessage extends WorkerMessageBase, SynthesisSegmentEventData {
|
|
24
|
-
messageType:
|
|
24
|
+
messageType: 'SynthesisSentenceEvent';
|
|
25
25
|
}
|
|
26
26
|
export interface VoiceListRequestMessage extends WorkerMessageBase {
|
|
27
|
-
messageType:
|
|
27
|
+
messageType: 'VoiceListRequest';
|
|
28
28
|
options: VoiceListRequestOptions;
|
|
29
29
|
}
|
|
30
30
|
export interface VoiceListResponseMessage extends WorkerMessageBase, RequestVoiceListResult {
|
|
31
|
-
messageType:
|
|
31
|
+
messageType: 'VoiceListResponse';
|
|
32
32
|
}
|
|
33
33
|
export interface RecognitionRequestMessage extends WorkerMessageBase {
|
|
34
|
-
messageType:
|
|
34
|
+
messageType: 'RecognitionRequest';
|
|
35
35
|
input: AudioSourceParam;
|
|
36
36
|
options: RecognitionOptions;
|
|
37
37
|
}
|
|
38
38
|
export interface RecognitionResponseMessage extends WorkerMessageBase, RecognitionResult {
|
|
39
|
-
messageType:
|
|
39
|
+
messageType: 'RecognitionResponse';
|
|
40
40
|
}
|
|
41
41
|
export interface AlignmentRequestMessage extends WorkerMessageBase {
|
|
42
|
-
messageType:
|
|
42
|
+
messageType: 'AlignmentRequest';
|
|
43
43
|
input: AudioSourceParam;
|
|
44
44
|
transcript: string;
|
|
45
45
|
options: AlignmentOptions;
|
|
46
46
|
}
|
|
47
47
|
export interface AlignmentResponseMessage extends WorkerMessageBase, AlignmentResult {
|
|
48
|
-
messageType:
|
|
48
|
+
messageType: 'AlignmentResponse';
|
|
49
49
|
}
|
|
50
50
|
export interface SpeechTranslationRequestMessage extends WorkerMessageBase {
|
|
51
|
-
messageType:
|
|
51
|
+
messageType: 'SpeechTranslationRequest';
|
|
52
52
|
input: AudioSourceParam;
|
|
53
53
|
options: SpeechTranslationOptions;
|
|
54
54
|
}
|
|
55
55
|
export interface SpeechTranslationResponseMessage extends WorkerMessageBase, SpeechTranslationResult {
|
|
56
|
-
messageType:
|
|
56
|
+
messageType: 'SpeechTranslationResponse';
|
|
57
57
|
}
|
|
58
58
|
export interface SpeechLanguageDetectionRequestMessage extends WorkerMessageBase {
|
|
59
|
-
messageType:
|
|
59
|
+
messageType: 'SpeechLanguageDetectionRequest';
|
|
60
60
|
input: AudioSourceParam;
|
|
61
61
|
options: SpeechLanguageDetectionOptions;
|
|
62
62
|
}
|
|
63
63
|
export interface SpeechLanguageDetectionResponseMessage extends WorkerMessageBase, SpeechLanguageDetectionResult {
|
|
64
|
-
messageType:
|
|
64
|
+
messageType: 'SpeechLanguageDetectionResponse';
|
|
65
65
|
}
|
|
66
66
|
export interface TextLanguageDetectionRequestMessage extends WorkerMessageBase {
|
|
67
|
-
messageType:
|
|
67
|
+
messageType: 'TextLanguageDetectionRequest';
|
|
68
68
|
input: string;
|
|
69
69
|
options: TextLanguageDetectionOptions;
|
|
70
70
|
}
|
|
71
71
|
export interface TextLanguageDetectionResponseMessage extends WorkerMessageBase, TextLanguageDetectionResult {
|
|
72
|
-
messageType:
|
|
72
|
+
messageType: 'TextLanguageDetectionResponse';
|
|
73
73
|
}
|
|
74
74
|
export declare function sendMessageToWorker(message: any): void;
|
|
75
75
|
export declare function addListenerToWorkerMessages(handler: MessageFunc): void;
|
package/dist/server/Worker.js
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { requestVoiceList, synthesize } from
|
|
2
|
-
import { Queue } from
|
|
3
|
-
import { logToStderr, yieldToEventLoop } from
|
|
4
|
-
import { recognize } from
|
|
5
|
-
import { align } from
|
|
6
|
-
import { translateSpeech } from
|
|
7
|
-
import { resetActiveLogger } from
|
|
1
|
+
import { requestVoiceList, synthesize } from '../api/Synthesis.js';
|
|
2
|
+
import { Queue } from '../utilities/Queue.js';
|
|
3
|
+
import { logToStderr, yieldToEventLoop } from '../utilities/Utilities.js';
|
|
4
|
+
import { recognize } from '../api/Recognition.js';
|
|
5
|
+
import { align } from '../api/Alignment.js';
|
|
6
|
+
import { translateSpeech } from '../api/Translation.js';
|
|
7
|
+
import { resetActiveLogger } from '../utilities/Logger.js';
|
|
8
8
|
import { writeToStderr } from '../utilities/Utilities.js';
|
|
9
9
|
import { Worker, SHARE_ENV } from 'node:worker_threads';
|
|
10
|
-
import { detectSpeechLanguage, detectTextLanguage } from
|
|
11
|
-
import chalk from
|
|
12
|
-
import { resolveToModuleRootDir } from
|
|
10
|
+
import { detectSpeechLanguage, detectTextLanguage } from '../api/LanguageDetection.js';
|
|
11
|
+
import chalk from 'chalk';
|
|
12
|
+
import { resolveToModuleRootDir } from '../utilities/PathUtilities.js';
|
|
13
13
|
const log = logToStderr;
|
|
14
14
|
let messageChannel = undefined;
|
|
15
15
|
const canceledRequests = new Set();
|
|
@@ -24,7 +24,7 @@ export function startMessageChannel() {
|
|
|
24
24
|
messageChannel.port1.start();
|
|
25
25
|
messageChannel.port2.start();
|
|
26
26
|
addListenerToClientMessages((message) => {
|
|
27
|
-
if (message.messageType ==
|
|
27
|
+
if (message.messageType == 'CancelationRequest' || message.messageType == 'CancellationRequest') {
|
|
28
28
|
//log(`CANCEL REQUESTED FOR ${message.requestId}`)
|
|
29
29
|
canceledRequests.add(message.requestId);
|
|
30
30
|
return;
|
|
@@ -53,33 +53,33 @@ async function processQueueIfIdle() {
|
|
|
53
53
|
...outgoingMessage,
|
|
54
54
|
});
|
|
55
55
|
}
|
|
56
|
-
function
|
|
56
|
+
function setCancellationFlagIfNeeded() {
|
|
57
57
|
if (canceledRequests.has(requestId)) {
|
|
58
58
|
cancelCurrentTask = true;
|
|
59
59
|
canceledRequests.delete(requestId);
|
|
60
60
|
}
|
|
61
61
|
}
|
|
62
62
|
await yieldToEventLoop();
|
|
63
|
-
|
|
64
|
-
const
|
|
63
|
+
setCancellationFlagIfNeeded();
|
|
64
|
+
const cancellationFlagSetterInterval = setInterval(setCancellationFlagIfNeeded, 20);
|
|
65
65
|
try {
|
|
66
66
|
if (cancelCurrentTask) {
|
|
67
67
|
//log(`******* CANCELED BEFORE START: ${requestId} *******`)
|
|
68
|
-
throw new Error(
|
|
68
|
+
throw new Error('Canceled');
|
|
69
69
|
}
|
|
70
70
|
await processMessage(incomingMessage, sendMessage);
|
|
71
71
|
}
|
|
72
72
|
catch (e) {
|
|
73
|
-
log(`${chalk.redBright(
|
|
73
|
+
log(`${chalk.redBright('Error')}: ${e.message}`);
|
|
74
74
|
sendMessageToClient({
|
|
75
75
|
requestId,
|
|
76
|
-
messageType:
|
|
76
|
+
messageType: 'Error',
|
|
77
77
|
error: e
|
|
78
78
|
});
|
|
79
79
|
}
|
|
80
80
|
finally {
|
|
81
81
|
resetActiveLogger();
|
|
82
|
-
clearInterval(
|
|
82
|
+
clearInterval(cancellationFlagSetterInterval);
|
|
83
83
|
cancelCurrentTask = false;
|
|
84
84
|
}
|
|
85
85
|
}
|
|
@@ -87,31 +87,31 @@ async function processQueueIfIdle() {
|
|
|
87
87
|
}
|
|
88
88
|
export async function processMessage(message, sendMessage) {
|
|
89
89
|
switch (message.messageType) {
|
|
90
|
-
case
|
|
90
|
+
case 'SynthesisRequest': {
|
|
91
91
|
await processSynthesisRequest(message, sendMessage);
|
|
92
92
|
break;
|
|
93
93
|
}
|
|
94
|
-
case
|
|
94
|
+
case 'VoiceListRequest': {
|
|
95
95
|
await processVoiceListRequest(message, sendMessage);
|
|
96
96
|
break;
|
|
97
97
|
}
|
|
98
|
-
case
|
|
98
|
+
case 'RecognitionRequest': {
|
|
99
99
|
await processRecognitionRequest(message, sendMessage);
|
|
100
100
|
break;
|
|
101
101
|
}
|
|
102
|
-
case
|
|
102
|
+
case 'AlignmentRequest': {
|
|
103
103
|
await processAlignmentRequest(message, sendMessage);
|
|
104
104
|
break;
|
|
105
105
|
}
|
|
106
|
-
case
|
|
106
|
+
case 'SpeechTranslationRequest': {
|
|
107
107
|
await processSpeechTranslationRequest(message, sendMessage);
|
|
108
108
|
break;
|
|
109
109
|
}
|
|
110
|
-
case
|
|
110
|
+
case 'SpeechLanguageDetectionRequest': {
|
|
111
111
|
await processSpeechLanguageDetectionRequest(message, sendMessage);
|
|
112
112
|
break;
|
|
113
113
|
}
|
|
114
|
-
case
|
|
114
|
+
case 'TextLanguageDetectionRequest': {
|
|
115
115
|
await processTextLanguageDetectionRequest(message, sendMessage);
|
|
116
116
|
break;
|
|
117
117
|
}
|
|
@@ -126,21 +126,21 @@ export async function processMessage(message, sendMessage) {
|
|
|
126
126
|
async function processSynthesisRequest(message, sendMessage) {
|
|
127
127
|
async function onSegment(eventData) {
|
|
128
128
|
const responseMessage = {
|
|
129
|
-
messageType:
|
|
129
|
+
messageType: 'SynthesisSegmentEvent',
|
|
130
130
|
...eventData
|
|
131
131
|
};
|
|
132
132
|
sendMessage(responseMessage);
|
|
133
133
|
}
|
|
134
134
|
async function onSentence(eventData) {
|
|
135
135
|
const responseMessage = {
|
|
136
|
-
messageType:
|
|
136
|
+
messageType: 'SynthesisSentenceEvent',
|
|
137
137
|
...eventData
|
|
138
138
|
};
|
|
139
139
|
sendMessage(responseMessage);
|
|
140
140
|
}
|
|
141
141
|
const result = await synthesize(message.input, message.options, onSegment, onSentence);
|
|
142
142
|
const responseMessage = {
|
|
143
|
-
messageType:
|
|
143
|
+
messageType: 'SynthesisResponse',
|
|
144
144
|
...result
|
|
145
145
|
};
|
|
146
146
|
sendMessage(responseMessage);
|
|
@@ -148,7 +148,7 @@ async function processSynthesisRequest(message, sendMessage) {
|
|
|
148
148
|
async function processVoiceListRequest(message, sendMessage) {
|
|
149
149
|
const result = await requestVoiceList(message.options);
|
|
150
150
|
const responseMessage = {
|
|
151
|
-
messageType:
|
|
151
|
+
messageType: 'VoiceListResponse',
|
|
152
152
|
...result
|
|
153
153
|
};
|
|
154
154
|
sendMessage(responseMessage);
|
|
@@ -159,7 +159,7 @@ async function processVoiceListRequest(message, sendMessage) {
|
|
|
159
159
|
async function processRecognitionRequest(message, sendMessage) {
|
|
160
160
|
const result = await recognize(message.input, message.options);
|
|
161
161
|
const responseMessage = {
|
|
162
|
-
messageType:
|
|
162
|
+
messageType: 'RecognitionResponse',
|
|
163
163
|
...result
|
|
164
164
|
};
|
|
165
165
|
sendMessage(responseMessage);
|
|
@@ -170,7 +170,7 @@ async function processRecognitionRequest(message, sendMessage) {
|
|
|
170
170
|
async function processAlignmentRequest(message, sendMessage) {
|
|
171
171
|
const result = await align(message.input, message.transcript, message.options);
|
|
172
172
|
const responseMessage = {
|
|
173
|
-
messageType:
|
|
173
|
+
messageType: 'AlignmentResponse',
|
|
174
174
|
...result
|
|
175
175
|
};
|
|
176
176
|
sendMessage(responseMessage);
|
|
@@ -181,7 +181,7 @@ async function processAlignmentRequest(message, sendMessage) {
|
|
|
181
181
|
async function processSpeechTranslationRequest(message, sendMessage) {
|
|
182
182
|
const result = await translateSpeech(message.input, message.options);
|
|
183
183
|
const responseMessage = {
|
|
184
|
-
messageType:
|
|
184
|
+
messageType: 'SpeechTranslationResponse',
|
|
185
185
|
...result
|
|
186
186
|
};
|
|
187
187
|
sendMessage(responseMessage);
|
|
@@ -192,7 +192,7 @@ async function processSpeechTranslationRequest(message, sendMessage) {
|
|
|
192
192
|
async function processSpeechLanguageDetectionRequest(message, sendMessage) {
|
|
193
193
|
const result = await detectSpeechLanguage(message.input, message.options);
|
|
194
194
|
const responseMessage = {
|
|
195
|
-
messageType:
|
|
195
|
+
messageType: 'SpeechLanguageDetectionResponse',
|
|
196
196
|
...result
|
|
197
197
|
};
|
|
198
198
|
sendMessage(responseMessage);
|
|
@@ -203,7 +203,7 @@ async function processSpeechLanguageDetectionRequest(message, sendMessage) {
|
|
|
203
203
|
async function processTextLanguageDetectionRequest(message, sendMessage) {
|
|
204
204
|
const result = await detectTextLanguage(message.input, message.options);
|
|
205
205
|
const responseMessage = {
|
|
206
|
-
messageType:
|
|
206
|
+
messageType: 'TextLanguageDetectionResponse',
|
|
207
207
|
...result
|
|
208
208
|
};
|
|
209
209
|
sendMessage(responseMessage);
|
|
@@ -244,8 +244,8 @@ export async function startNewWorkerThread() {
|
|
|
244
244
|
argv: process.argv.slice(2),
|
|
245
245
|
env: SHARE_ENV
|
|
246
246
|
});
|
|
247
|
-
workerThread.on(
|
|
248
|
-
if (message.name ==
|
|
247
|
+
workerThread.on('message', (message) => {
|
|
248
|
+
if (message.name == 'writeToStdErr') {
|
|
249
249
|
writeToStderr(message.text);
|
|
250
250
|
}
|
|
251
251
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Worker.js","sourceRoot":"","sources":["../../src/server/Worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiH,gBAAgB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AACjL,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAEzE,OAAO,EAAyC,SAAS,EAAE,MAAM,uBAAuB,CAAA;AACxF,OAAO,EAAqC,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC9E,OAAO,EAAqD,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAC1G,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AACzD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AACvD,OAAO,EAA4H,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChN,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAEtE,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,IAAI,cAAc,GAA+B,SAAS,CAAA;AAE1D,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;AAE1C,IAAI,iBAAiB,GAAG,KAAK,CAAA;AAE7B,MAAM,YAAY,GAAG,IAAI,KAAK,EAAO,CAAA;AACrC,IAAI,YAAY,GAAG,KAAK,CAAA;AAExB,MAAM,UAAU,mBAAmB;IAClC,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC;QAC5B,OAAM;IACP,CAAC;IAED,cAAc,GAAG,IAAI,cAAc,EAAE,CAAA;IACrC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;IAC5B,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;IAE5B,2BAA2B,CAAC,CAAC,OAAO,EAAE,EAAE;QACvC,IAAI,OAAO,CAAC,WAAW,IAAI,oBAAoB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"Worker.js","sourceRoot":"","sources":["../../src/server/Worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiH,gBAAgB,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AACjL,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAC7C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAA;AAEzE,OAAO,EAAyC,SAAS,EAAE,MAAM,uBAAuB,CAAA;AACxF,OAAO,EAAqC,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAC9E,OAAO,EAAqD,eAAe,EAAE,MAAM,uBAAuB,CAAA;AAC1G,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AACzD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AACvD,OAAO,EAA4H,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChN,OAAO,KAAK,MAAM,OAAO,CAAA;AACzB,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAEtE,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,IAAI,cAAc,GAA+B,SAAS,CAAA;AAE1D,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAU,CAAA;AAE1C,IAAI,iBAAiB,GAAG,KAAK,CAAA;AAE7B,MAAM,YAAY,GAAG,IAAI,KAAK,EAAO,CAAA;AACrC,IAAI,YAAY,GAAG,KAAK,CAAA;AAExB,MAAM,UAAU,mBAAmB;IAClC,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC;QAC5B,OAAM;IACP,CAAC;IAED,cAAc,GAAG,IAAI,cAAc,EAAE,CAAA;IACrC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;IAC5B,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;IAE5B,2BAA2B,CAAC,CAAC,OAAO,EAAE,EAAE;QACvC,IAAI,OAAO,CAAC,WAAW,IAAI,oBAAoB,IAAI,OAAO,CAAC,WAAW,IAAI,qBAAqB,EAAE,CAAC;YACjG,kDAAkD;YAClD,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YACvC,OAAM;QACP,CAAC;QAED,uBAAuB,CAAC,OAAO,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB;IACtC,OAAO,iBAAiB,CAAA;AACzB,CAAC;AAED,SAAS,uBAAuB,CAAC,OAAY;IAC5C,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAC7B,kBAAkB,EAAE,CAAA;AACrB,CAAC;AAED,KAAK,UAAU,kBAAkB;IAChC,IAAI,YAAY,EAAE,CAAC;QAClB,OAAM;IACP,CAAC;IAED,YAAY,GAAG,IAAI,CAAA;IAEnB,OAAO,CAAC,YAAY,CAAC,OAAO,EAAE,CAAC;QAC9B,MAAM,eAAe,GAAG,YAAY,CAAC,OAAO,EAAE,CAAA;QAC9C,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,CAAA;QAE3C,SAAS,WAAW,CAAC,eAAoB;YACxC,mBAAmB,CAAC;gBACnB,SAAS;gBACT,GAAG,eAAe;aAClB,CAAC,CAAA;QACH,CAAC;QAED,SAAS,2BAA2B;YACnC,IAAI,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrC,iBAAiB,GAAG,IAAI,CAAA;gBACxB,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACnC,CAAC;QACF,CAAC;QAED,MAAM,gBAAgB,EAAE,CAAA;QAExB,2BAA2B,EAAE,CAAA;QAC7B,MAAM,8BAA8B,GAAG,WAAW,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAA;QAEnF,IAAI,CAAC;YACJ,IAAI,iBAAiB,EAAE,CAAC;gBACvB,4DAA4D;gBAC5D,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,CAAA;YAC5B,CAAC;YAED,MAAM,cAAc,CAAC,eAAe,EAAE,WAAW,CAAC,CAAA;QACnD,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACjB,GAAG,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAA;YAEhD,mBAAmB,CAAC;gBACnB,SAAS;gBACT,WAAW,EAAE,OAAO;gBACpB,KAAK,EAAE,CAAC;aACR,CAAC,CAAA;QACH,CAAC;gBAAS,CAAC;YACV,iBAAiB,EAAE,CAAA;YAEnB,aAAa,CAAC,8BAA8B,CAAC,CAAA;YAC7C,iBAAiB,GAAG,KAAK,CAAA;QAC1B,CAAC;IACF,CAAC;IAED,YAAY,GAAG,KAAK,CAAA;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA6B,EAAE,WAAwB;IAC3F,QAAQ,OAAO,CAAC,WAAW,EAAE,CAAC;QAC7B,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACzB,MAAM,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YACnD,MAAK;QACN,CAAC;QAED,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACzB,MAAM,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YACnD,MAAK;QACN,CAAC;QAED,KAAK,oBAAoB,CAAC,CAAC,CAAC;YAC3B,MAAM,yBAAyB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YACrD,MAAK;QACN,CAAC;QAED,KAAK,kBAAkB,CAAC,CAAC,CAAC;YACzB,MAAM,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YACnD,MAAK;QACN,CAAC;QAED,KAAK,0BAA0B,CAAC,CAAC,CAAC;YACjC,MAAM,+BAA+B,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YAC3D,MAAK;QACN,CAAC;QAED,KAAK,gCAAgC,CAAC,CAAC,CAAC;YACvC,MAAM,qCAAqC,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YACjE,MAAK;QACN,CAAC;QAED,KAAK,8BAA8B,CAAC,CAAC,CAAC;YACrC,MAAM,mCAAmC,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YAC/D,MAAK;QACN,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,yBAA0B,OAAe,CAAC,WAAW,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;AACF,CAAC;AAED,+FAA+F;AAC/F,uBAAuB;AACvB,+FAA+F;AAC/F,KAAK,UAAU,uBAAuB,CAAC,OAAgC,EAAE,WAAwB;IAChG,KAAK,UAAU,SAAS,CAAC,SAAoC;QAC5D,MAAM,eAAe,GAAiC;YACrD,WAAW,EAAE,uBAAuB;YACpC,GAAG,SAAS;SACZ,CAAA;QAED,WAAW,CAAC,eAAe,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,UAAU,UAAU,CAAC,SAAoC;QAC7D,MAAM,eAAe,GAAkC;YACtD,WAAW,EAAE,wBAAwB;YACrC,GAAG,SAAS;SACZ,CAAA;QAED,WAAW,CAAC,eAAe,CAAC,CAAA;IAC7B,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,CAAA;IAEtF,MAAM,eAAe,GAA6B;QACjD,WAAW,EAAE,mBAAmB;QAChC,GAAG,MAAM;KACT,CAAA;IAED,WAAW,CAAC,eAAe,CAAC,CAAA;AAC7B,CAAC;AAqBD,KAAK,UAAU,uBAAuB,CAAC,OAAgC,EAAE,WAAwB;IAChG,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEtD,MAAM,eAAe,GAA6B;QACjD,WAAW,EAAE,mBAAmB;QAChC,GAAG,MAAM;KACT,CAAA;IAED,WAAW,CAAC,eAAe,CAAC,CAAA;AAC7B,CAAC;AAYD,+FAA+F;AAC/F,yBAAyB;AACzB,+FAA+F;AAC/F,KAAK,UAAU,yBAAyB,CAAC,OAAkC,EAAE,WAAwB;IACpG,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAE9D,MAAM,eAAe,GAA+B;QACnD,WAAW,EAAE,qBAAqB;QAClC,GAAG,MAAM;KACT,CAAA;IAED,WAAW,CAAC,eAAe,CAAC,CAAA;AAC7B,CAAC;AAaD,+FAA+F;AAC/F,uBAAuB;AACvB,+FAA+F;AAC/F,KAAK,UAAU,uBAAuB,CAAC,OAAgC,EAAE,WAAwB;IAChG,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAE9E,MAAM,eAAe,GAA6B;QACjD,WAAW,EAAE,mBAAmB;QAChC,GAAG,MAAM;KACT,CAAA;IAED,WAAW,CAAC,eAAe,CAAC,CAAA;AAC7B,CAAC;AAcD,+FAA+F;AAC/F,gCAAgC;AAChC,+FAA+F;AAC/F,KAAK,UAAU,+BAA+B,CAAC,OAAwC,EAAE,WAAwB;IAChH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAEpE,MAAM,eAAe,GAAqC;QACzD,WAAW,EAAE,2BAA2B;QACxC,GAAG,MAAM;KACT,CAAA;IAED,WAAW,CAAC,eAAe,CAAC,CAAA;AAC7B,CAAC;AAaD,+FAA+F;AAC/F,uCAAuC;AACvC,+FAA+F;AAC/F,KAAK,UAAU,qCAAqC,CAAC,OAA8C,EAAE,WAAwB;IAC5H,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzE,MAAM,eAAe,GAA2C;QAC/D,WAAW,EAAE,iCAAiC;QAC9C,GAAG,MAAM;KACT,CAAA;IAED,WAAW,CAAC,eAAe,CAAC,CAAA;AAC7B,CAAC;AAaD,+FAA+F;AAC/F,qCAAqC;AACrC,+FAA+F;AAC/F,KAAK,UAAU,mCAAmC,CAAC,OAA4C,EAAE,WAAwB;IACxH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAEvE,MAAM,eAAe,GAAyC;QAC7D,WAAW,EAAE,+BAA+B;QAC5C,GAAG,MAAM;KACT,CAAA;IAED,WAAW,CAAC,eAAe,CAAC,CAAA;AAC7B,CAAC;AAaD,+FAA+F;AAC/F,oBAAoB;AACpB,+FAA+F;AAC/F,MAAM,UAAU,mBAAmB,CAAC,OAAY;IAC/C,2BAA2B,EAAE,CAAA;IAE7B,cAAc,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;AAC3C,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,OAAoB;IAC/D,2BAA2B,EAAE,CAAA;IAE7B,cAAc,EAAE,KAAK,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;QAC3D,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACpB,CAAC,CAAC,CAAA;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAY;IACxC,2BAA2B,EAAE,CAAA;IAE7B,cAAc,EAAE,KAAK,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;AAC3C,CAAC;AAED,SAAS,2BAA2B,CAAC,OAAoB;IACxD,2BAA2B,EAAE,CAAA;IAE7B,cAAc,EAAE,KAAK,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;QAC3D,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACpB,CAAC,CAAC,CAAA;AACH,CAAC;AAED,SAAS,2BAA2B;IACnC,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IACxD,CAAC;AACF,CAAC;AAED,+FAA+F;AAC/F,wBAAwB;AACxB,+FAA+F;AAC/F,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACzC,MAAM,YAAY,GAAG,IAAI,MAAM,CAAC,sBAAsB,CAAC,8BAA8B,CAAC,EAAE;QACvF,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3B,GAAG,EAAE,SAAS;KACd,CAAC,CAAA;IAEF,YAAY,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE;QACtC,IAAI,OAAO,CAAC,IAAI,IAAI,eAAe,EAAE,CAAC;YACrC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAC5B,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,YAAY,CAAC,WAAW,CAAC;QACxB,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK;QACjC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,KAAK;KAC9E,CAAC,CAAA;IAEF,OAAO,YAAY,CAAA;AACpB,CAAC"}
|
|
@@ -10,7 +10,7 @@ async function startIfInWorkerThread() {
|
|
|
10
10
|
parentPort?.postMessage(message);
|
|
11
11
|
});
|
|
12
12
|
const initOpenPromise = new OpenPromise();
|
|
13
|
-
parentPort.once(
|
|
13
|
+
parentPort.once('message', (message) => {
|
|
14
14
|
if (message.name == 'init') {
|
|
15
15
|
process.stderr.isTTY = message.stdErrIsTTY;
|
|
16
16
|
process.stderr.hasColors = () => message.hasColors;
|
|
@@ -22,7 +22,7 @@ async function startIfInWorkerThread() {
|
|
|
22
22
|
}
|
|
23
23
|
});
|
|
24
24
|
await initOpenPromise.promise;
|
|
25
|
-
parentPort.on(
|
|
25
|
+
parentPort.on('message', (message) => {
|
|
26
26
|
sendMessageToWorker(message);
|
|
27
27
|
});
|
|
28
28
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/// <reference path="../../src/typings/Fillers.d.ts" />
|
|
2
|
+
import Onnx from 'onnxruntime-node';
|
|
3
|
+
import { RawAudio } from '../audio/AudioUtilities.js';
|
|
4
|
+
export declare function isolate(rawAudio: RawAudio, modelFilePath: string): Promise<RawAudio>;
|
|
5
|
+
export declare class MDXNet {
|
|
6
|
+
readonly modelFilePath: string;
|
|
7
|
+
session?: Onnx.InferenceSession;
|
|
8
|
+
constructor(modelFilePath: string);
|
|
9
|
+
processAudio(rawAudio: RawAudio): Promise<RawAudio>;
|
|
10
|
+
private initializeSession;
|
|
11
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import Onnx from 'onnxruntime-node';
|
|
2
|
+
import { binBufferToComplex, complexToBinBuffer, getWindowWeights, stftr, stiftr } from '../dsp/FFT.js';
|
|
3
|
+
import { logToStderr } from '../utilities/Utilities.js';
|
|
4
|
+
import { Logger } from '../utilities/Logger.js';
|
|
5
|
+
const log = logToStderr;
|
|
6
|
+
export async function isolate(rawAudio, modelFilePath) {
|
|
7
|
+
const model = new MDXNet(modelFilePath);
|
|
8
|
+
return model.processAudio(rawAudio);
|
|
9
|
+
}
|
|
10
|
+
export class MDXNet {
|
|
11
|
+
modelFilePath;
|
|
12
|
+
session;
|
|
13
|
+
constructor(modelFilePath) {
|
|
14
|
+
this.modelFilePath = modelFilePath;
|
|
15
|
+
}
|
|
16
|
+
async processAudio(rawAudio) {
|
|
17
|
+
if (rawAudio.audioChannels.length != 2) {
|
|
18
|
+
throw new Error(`Input audio must be stereo`);
|
|
19
|
+
}
|
|
20
|
+
if (rawAudio.sampleRate != 44100) {
|
|
21
|
+
throw new Error(`Input audio must have a 44100 Hz sampling rate`);
|
|
22
|
+
}
|
|
23
|
+
if (!this.session) {
|
|
24
|
+
await this.initializeSession(this.modelFilePath);
|
|
25
|
+
}
|
|
26
|
+
const logger = new Logger();
|
|
27
|
+
const session = this.session;
|
|
28
|
+
const sampleRate = rawAudio.sampleRate;
|
|
29
|
+
const fftSize = 6144;
|
|
30
|
+
const fftCount = 2048;
|
|
31
|
+
const fftWindowSize = fftSize;
|
|
32
|
+
const fftHopSize = 1024;
|
|
33
|
+
const segmentSize = 256;
|
|
34
|
+
const segmentHopSize = 240;
|
|
35
|
+
const sampleCount = rawAudio.audioChannels[0].length;
|
|
36
|
+
logger.start('Compute STFT of full waveform');
|
|
37
|
+
const fftFramesLeft = await stftr(rawAudio.audioChannels[0], fftSize, fftWindowSize, fftHopSize, 'hann');
|
|
38
|
+
const fftFramesRight = await stftr(rawAudio.audioChannels[1], fftSize, fftWindowSize, fftHopSize, 'hann');
|
|
39
|
+
const fftFramesLeftComplex = fftFramesLeft.map(frame => binBufferToComplex(frame).slice(0, fftCount));
|
|
40
|
+
const fftFramesRightComplex = fftFramesRight.map(frame => binBufferToComplex(frame).slice(0, fftCount));
|
|
41
|
+
const audioForSegments = [];
|
|
42
|
+
for (let segmentOffset = 0; segmentOffset < fftFramesLeft.length; segmentOffset += segmentHopSize) {
|
|
43
|
+
const timePosition = segmentOffset * (fftHopSize / sampleRate);
|
|
44
|
+
logger.start(`Process segment at time position ${timePosition.toFixed(2)}`);
|
|
45
|
+
const fftFramesLeftComplexForSegment = fftFramesLeftComplex.slice(segmentOffset, segmentOffset + segmentSize);
|
|
46
|
+
const fftFramesRightComplexForSegment = fftFramesRightComplex.slice(segmentOffset, segmentOffset + segmentSize);
|
|
47
|
+
const segmentLength = fftFramesLeftComplexForSegment.length;
|
|
48
|
+
const flattenedInputTensor = new Float32Array(1 * 4 * fftCount * segmentSize);
|
|
49
|
+
{
|
|
50
|
+
let writePosition = 0;
|
|
51
|
+
for (let tensorChannelIndex = 0; tensorChannelIndex < 4; tensorChannelIndex++) {
|
|
52
|
+
for (let binIndex = 0; binIndex < fftCount; binIndex++) {
|
|
53
|
+
for (let frameIndex = 0; frameIndex < segmentSize; frameIndex++) {
|
|
54
|
+
let value = 0;
|
|
55
|
+
if (frameIndex < segmentLength && binIndex >= 0) {
|
|
56
|
+
let frame;
|
|
57
|
+
if (tensorChannelIndex < 2) {
|
|
58
|
+
frame = fftFramesLeftComplexForSegment[frameIndex];
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
frame = fftFramesRightComplexForSegment[frameIndex];
|
|
62
|
+
}
|
|
63
|
+
const bin = frame[binIndex];
|
|
64
|
+
if (tensorChannelIndex % 2 === 0) {
|
|
65
|
+
value = bin.real;
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
value = bin.imaginary;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
flattenedInputTensor[writePosition++] = value;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
const inputTensor = new Onnx.Tensor('float32', flattenedInputTensor, [1, 4, 2048, 256]);
|
|
77
|
+
const { output: outputTensor } = await session.run({ input: inputTensor });
|
|
78
|
+
const flattenedOutputTensor = outputTensor.data;
|
|
79
|
+
const outputChannelComplexFrames = [];
|
|
80
|
+
{
|
|
81
|
+
for (let outChannelIndex = 0; outChannelIndex < 2; outChannelIndex++) {
|
|
82
|
+
const framesForChannel = [];
|
|
83
|
+
for (let frameIndex = 0; frameIndex < 256; frameIndex++) {
|
|
84
|
+
const frame = [];
|
|
85
|
+
for (let binIndex = 0; binIndex < fftSize; binIndex++) {
|
|
86
|
+
frame.push({ real: 0, imaginary: 0 });
|
|
87
|
+
}
|
|
88
|
+
framesForChannel.push(frame);
|
|
89
|
+
}
|
|
90
|
+
outputChannelComplexFrames.push(framesForChannel);
|
|
91
|
+
}
|
|
92
|
+
let readPosition = 0;
|
|
93
|
+
for (let tensorChannelIndex = 0; tensorChannelIndex < 4; tensorChannelIndex++) {
|
|
94
|
+
const outChannelIndex = tensorChannelIndex < 2 ? 0 : 1;
|
|
95
|
+
for (let binIndex = 0; binIndex < 2048; binIndex++) {
|
|
96
|
+
for (let frameIndex = 0; frameIndex < 256; frameIndex++) {
|
|
97
|
+
const bin = outputChannelComplexFrames[outChannelIndex][frameIndex][binIndex];
|
|
98
|
+
if (tensorChannelIndex % 2 === 0) {
|
|
99
|
+
bin.real = flattenedOutputTensor[readPosition++];
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
bin.imaginary = flattenedOutputTensor[readPosition++];
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const outputAudioChannels = [];
|
|
109
|
+
//logger.start(`Compute inverse STFT for segment`)
|
|
110
|
+
for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
|
|
111
|
+
let outputChannelFlattenedFrames = outputChannelComplexFrames[channelIndex]
|
|
112
|
+
.map(frame => complexToBinBuffer(frame).map(value => value / fftSize));
|
|
113
|
+
const samples = await stiftr(outputChannelFlattenedFrames, fftSize, fftWindowSize, fftHopSize, 'hann');
|
|
114
|
+
outputAudioChannels.push(samples);
|
|
115
|
+
}
|
|
116
|
+
//logger.log(`Reconstructed waveform peak: ${getAudioPeakDecibels(outputAudioChannels).toFixed(3)}dB`)
|
|
117
|
+
//await playAudioSamples({ audioChannels: outputAudioChannels, sampleRate })
|
|
118
|
+
audioForSegments.push(outputAudioChannels);
|
|
119
|
+
}
|
|
120
|
+
// Join segments using overlapping Hann windows
|
|
121
|
+
logger.start(`Join segments`);
|
|
122
|
+
const concatenatedAudioChannels = [new Float32Array(sampleCount), new Float32Array(sampleCount)];
|
|
123
|
+
{
|
|
124
|
+
const segmentCount = audioForSegments.length;
|
|
125
|
+
const segmentSampleCount = audioForSegments[0][0].length;
|
|
126
|
+
const windowWeights = getWindowWeights('hann', segmentSampleCount);
|
|
127
|
+
const sumOfWeightsForSample = new Float32Array(sampleCount);
|
|
128
|
+
for (let segmentIndex = 0; segmentIndex < segmentCount; segmentIndex++) {
|
|
129
|
+
const segmentStartFrameIndex = segmentIndex * segmentHopSize;
|
|
130
|
+
const segmentStartSampleIndex = segmentStartFrameIndex * fftHopSize;
|
|
131
|
+
const segmentSamples = audioForSegments[segmentIndex];
|
|
132
|
+
for (let segmentSampleOffset = 0; segmentSampleOffset < segmentSampleCount; segmentSampleOffset++) {
|
|
133
|
+
const sampleIndex = segmentStartSampleIndex + segmentSampleOffset;
|
|
134
|
+
if (sampleIndex >= sampleCount) {
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
const weight = windowWeights[segmentSampleOffset];
|
|
138
|
+
for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
|
|
139
|
+
concatenatedAudioChannels[channelIndex][sampleIndex] += segmentSamples[channelIndex][segmentSampleOffset] * weight;
|
|
140
|
+
}
|
|
141
|
+
sumOfWeightsForSample[sampleIndex] += weight;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
for (let sampleIndex = 0; sampleIndex < sampleCount; sampleIndex++) {
|
|
145
|
+
for (let channelIndex = 0; channelIndex < 2; channelIndex++) {
|
|
146
|
+
concatenatedAudioChannels[channelIndex][sampleIndex] /= sumOfWeightsForSample[sampleIndex] + 1e-8;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
const isolatedRawAudio = { audioChannels: concatenatedAudioChannels, sampleRate };
|
|
151
|
+
logger.end();
|
|
152
|
+
return isolatedRawAudio;
|
|
153
|
+
}
|
|
154
|
+
async initializeSession(modelPath) {
|
|
155
|
+
const onnxOptions = {
|
|
156
|
+
logSeverityLevel: 3
|
|
157
|
+
};
|
|
158
|
+
this.session = await Onnx.InferenceSession.create(modelPath, onnxOptions);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=MDXNetSourceSeparation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MDXNetSourceSeparation.js","sourceRoot":"","sources":["../../src/source-separation/MDXNetSourceSeparation.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,kBAAkB,CAAA;AAEnC,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAExG,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAEhD,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAkB,EAAE,aAAqB;IACtE,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,CAAA;IAEvC,OAAO,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;AACpC,CAAC;AAED,MAAM,OAAO,MAAM;IAGU;IAF5B,OAAO,CAAwB;IAE/B,YAA4B,aAAqB;QAArB,kBAAa,GAAb,aAAa,CAAQ;IACjD,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,QAAkB;QACpC,IAAI,QAAQ,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC9C,CAAC;QAED,IAAI,QAAQ,CAAC,UAAU,IAAI,KAAK,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QAClE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;QACjD,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;QAE3B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAQ,CAAA;QAE7B,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAA;QACtC,MAAM,OAAO,GAAG,IAAI,CAAA;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAA;QACrB,MAAM,aAAa,GAAG,OAAO,CAAA;QAC7B,MAAM,UAAU,GAAG,IAAI,CAAA;QAEvB,MAAM,WAAW,GAAG,GAAG,CAAA;QACvB,MAAM,cAAc,GAAG,GAAG,CAAA;QAE1B,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAEpD,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;QAE7C,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QACxG,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;QAEzG,MAAM,oBAAoB,GAAG,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAA;QACrG,MAAM,qBAAqB,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAA;QAEvG,MAAM,gBAAgB,GAAqB,EAAE,CAAA;QAE7C,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,GAAG,aAAa,CAAC,MAAM,EAAE,aAAa,IAAI,cAAc,EAAE,CAAC;YACnG,MAAM,YAAY,GAAG,aAAa,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC,CAAA;YAE9D,MAAM,CAAC,KAAK,CAAC,oCAAoC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAE3E,MAAM,8BAA8B,GAAG,oBAAoB,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,GAAG,WAAW,CAAC,CAAA;YAC7G,MAAM,+BAA+B,GAAG,qBAAqB,CAAC,KAAK,CAAC,aAAa,EAAE,aAAa,GAAG,WAAW,CAAC,CAAA;YAE/G,MAAM,aAAa,GAAG,8BAA8B,CAAC,MAAM,CAAA;YAE3D,MAAM,oBAAoB,GAAG,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,GAAG,WAAW,CAAC,CAAA;YAE7E,CAAC;gBACA,IAAI,aAAa,GAAG,CAAC,CAAA;gBAErB,KAAK,IAAI,kBAAkB,GAAG,CAAC,EAAE,kBAAkB,GAAG,CAAC,EAAE,kBAAkB,EAAE,EAAE,CAAC;oBAC/E,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;wBACxD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,WAAW,EAAE,UAAU,EAAE,EAAE,CAAC;4BACjE,IAAI,KAAK,GAAG,CAAC,CAAA;4BAEb,IAAI,UAAU,GAAG,aAAa,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gCACjD,IAAI,KAAsB,CAAA;gCAE1B,IAAI,kBAAkB,GAAG,CAAC,EAAE,CAAC;oCAC5B,KAAK,GAAG,8BAA8B,CAAC,UAAU,CAAC,CAAA;gCACnD,CAAC;qCAAM,CAAC;oCACP,KAAK,GAAG,+BAA+B,CAAC,UAAU,CAAC,CAAA;gCACpD,CAAC;gCAED,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAA;gCAE3B,IAAI,kBAAkB,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oCAClC,KAAK,GAAG,GAAG,CAAC,IAAI,CAAA;gCACjB,CAAC;qCAAM,CAAC;oCACP,KAAK,GAAG,GAAG,CAAC,SAAS,CAAA;gCACtB,CAAC;4BACF,CAAC;4BAED,oBAAoB,CAAC,aAAa,EAAE,CAAC,GAAG,KAAK,CAAA;wBAC9C,CAAC;oBACF,CAAC;gBACF,CAAC;YACF,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,oBAAoB,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAA;YAEvF,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAA;YAE1E,MAAM,qBAAqB,GAAG,YAAY,CAAC,IAAoB,CAAA;YAE/D,MAAM,0BAA0B,GAAwB,EAAE,CAAA;YAE1D,CAAC;gBACA,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,CAAC,EAAE,eAAe,EAAE,EAAE,CAAC;oBACtE,MAAM,gBAAgB,GAAsB,EAAE,CAAA;oBAE9C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC;wBACzD,MAAM,KAAK,GAAoB,EAAE,CAAA;wBAEjC,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,OAAO,EAAE,QAAQ,EAAE,EAAE,CAAC;4BACvD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAA;wBACtC,CAAC;wBAED,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAC7B,CAAC;oBAED,0BAA0B,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;gBAClD,CAAC;gBAED,IAAI,YAAY,GAAG,CAAC,CAAA;gBAEpB,KAAK,IAAI,kBAAkB,GAAG,CAAC,EAAE,kBAAkB,GAAG,CAAC,EAAE,kBAAkB,EAAE,EAAE,CAAC;oBAC/E,MAAM,eAAe,GAAG,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;oBAEtD,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC;wBACpD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC;4BACzD,MAAM,GAAG,GAAG,0BAA0B,CAAC,eAAe,CAAC,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAA;4BAE7E,IAAI,kBAAkB,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gCAClC,GAAG,CAAC,IAAI,GAAG,qBAAqB,CAAC,YAAY,EAAE,CAAC,CAAA;4BACjD,CAAC;iCAAM,CAAC;gCACP,GAAG,CAAC,SAAS,GAAG,qBAAqB,CAAC,YAAY,EAAE,CAAC,CAAA;4BACtD,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;YACF,CAAC;YAED,MAAM,mBAAmB,GAAmB,EAAE,CAAA;YAE9C,kDAAkD;YAClD,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC;gBAC7D,IAAI,4BAA4B,GAAG,0BAA0B,CAAC,YAAY,CAAC;qBACzE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,CAAA;gBAEvE,MAAM,OAAO,GAAG,MAAM,MAAM,CAC3B,4BAA4B,EAC5B,OAAO,EACP,aAAa,EACb,UAAU,EACV,MAAM,CAAC,CAAA;gBAER,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YAClC,CAAC;YAED,sGAAsG;YACtG,4EAA4E;YAE5E,gBAAgB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAA;QAC3C,CAAC;QAED,+CAA+C;QAC/C,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;QAC7B,MAAM,yBAAyB,GAAG,CAAC,IAAI,YAAY,CAAC,WAAW,CAAC,EAAE,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC,CAAA;QAEhG,CAAC;YACA,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAA;YAE5C,MAAM,kBAAkB,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;YAExD,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;YAElE,MAAM,qBAAqB,GAAG,IAAI,YAAY,CAAC,WAAW,CAAC,CAAA;YAE3D,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,YAAY,EAAE,EAAE,CAAC;gBACxE,MAAM,sBAAsB,GAAG,YAAY,GAAG,cAAc,CAAA;gBAC5D,MAAM,uBAAuB,GAAG,sBAAsB,GAAG,UAAU,CAAA;gBAEnE,MAAM,cAAc,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAA;gBAErD,KAAK,IAAI,mBAAmB,GAAG,CAAC,EAAE,mBAAmB,GAAG,kBAAkB,EAAE,mBAAmB,EAAE,EAAE,CAAC;oBACnG,MAAM,WAAW,GAAG,uBAAuB,GAAG,mBAAmB,CAAA;oBAEjE,IAAI,WAAW,IAAI,WAAW,EAAE,CAAC;wBAChC,MAAK;oBACN,CAAC;oBAED,MAAM,MAAM,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAA;oBAEjD,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC;wBAC7D,yBAAyB,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC,mBAAmB,CAAC,GAAG,MAAM,CAAA;oBACnH,CAAC;oBAED,qBAAqB,CAAC,WAAW,CAAC,IAAI,MAAM,CAAA;gBAC7C,CAAC;YACF,CAAC;YAED,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,WAAW,GAAG,WAAW,EAAE,WAAW,EAAE,EAAE,CAAC;gBACpE,KAAK,IAAI,YAAY,GAAG,CAAC,EAAE,YAAY,GAAG,CAAC,EAAE,YAAY,EAAE,EAAE,CAAC;oBAC7D,yBAAyB,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,IAAI,qBAAqB,CAAC,WAAW,CAAC,GAAG,IAAI,CAAA;gBAClG,CAAC;YACF,CAAC;QACF,CAAC;QAED,MAAM,gBAAgB,GAAa,EAAE,aAAa,EAAE,yBAAyB,EAAE,UAAU,EAAE,CAAA;QAE3F,MAAM,CAAC,GAAG,EAAE,CAAA;QAEZ,OAAO,gBAAgB,CAAA;IACxB,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAAiB;QAChD,MAAM,WAAW,GAAyC;YACzD,gBAAgB,EAAE,CAAC;SACnB,CAAA;QAED,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,CAAA;IAC1E,CAAC;CACD"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/// <reference path="../../src/typings/Fillers.d.ts" />
|
|
2
2
|
import Onnx from 'onnxruntime-node';
|
|
3
|
-
import { RawAudio } from
|
|
3
|
+
import { RawAudio } from '../audio/AudioUtilities.js';
|
|
4
4
|
import { type LanguageDetectionResults } from '../api/LanguageDetection.js';
|
|
5
5
|
export declare function detectLanguage(rawAudio: RawAudio, modelPath: string, languageDictionaryPath: string, languageGroupDictionaryPath: string): Promise<LanguageDetectionResults>;
|
|
6
6
|
export declare class SileroLanguageDetection {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import Onnx from 'onnxruntime-node';
|
|
2
2
|
import { softmax } from '../math/VectorMath.js';
|
|
3
|
-
import { Logger } from
|
|
3
|
+
import { Logger } from '../utilities/Logger.js';
|
|
4
4
|
import { readAndParseJsonFile } from '../utilities/FileSystem.js';
|
|
5
5
|
import { detectSpeechLanguageByParts } from '../api/LanguageDetection.js';
|
|
6
6
|
import { languageCodeToName } from '../utilities/Locale.js';
|
|
@@ -29,7 +29,7 @@ export class SileroLanguageDetection {
|
|
|
29
29
|
}
|
|
30
30
|
async initialize() {
|
|
31
31
|
const logger = new Logger();
|
|
32
|
-
logger.start(
|
|
32
|
+
logger.start('Initialize ONNX inference session');
|
|
33
33
|
this.languageDictionary = await readAndParseJsonFile(this.languageDictionaryPath);
|
|
34
34
|
this.languageGroupDictionary = await readAndParseJsonFile(this.languageGroupDictionaryPath);
|
|
35
35
|
const onnxOptions = {
|
|
@@ -40,20 +40,20 @@ export class SileroLanguageDetection {
|
|
|
40
40
|
}
|
|
41
41
|
async detectLanguage(rawAudio) {
|
|
42
42
|
const logger = new Logger();
|
|
43
|
-
logger.start(
|
|
43
|
+
logger.start('Detect language with Silero');
|
|
44
44
|
const audioSamples = rawAudio.audioChannels[0];
|
|
45
45
|
const inputTensor = new Onnx.Tensor('float32', audioSamples, [1, audioSamples.length]);
|
|
46
46
|
const inputs = { input: inputTensor };
|
|
47
47
|
const results = await this.session.run(inputs);
|
|
48
|
-
logger.start(
|
|
49
|
-
const languageLogits = results[
|
|
50
|
-
const languageGroupLogits = results[
|
|
48
|
+
logger.start('Parse model results');
|
|
49
|
+
const languageLogits = results['output'].data;
|
|
50
|
+
const languageGroupLogits = results['2038'].data;
|
|
51
51
|
const languageProbabilities = softmax(languageLogits);
|
|
52
52
|
const languageGroupProbabilities = softmax(languageGroupLogits);
|
|
53
53
|
const languageResults = [];
|
|
54
54
|
for (let i = 0; i < languageProbabilities.length; i++) {
|
|
55
55
|
const languageString = this.languageDictionary[i];
|
|
56
|
-
const languageCode = languageString.replace(/,.*$/,
|
|
56
|
+
const languageCode = languageString.replace(/,.*$/, '');
|
|
57
57
|
languageResults.push({
|
|
58
58
|
language: languageCode,
|
|
59
59
|
languageName: languageCodeToName(languageCode),
|
|
@@ -3,6 +3,16 @@ export declare function subtitlesFileToText(filename: string): Promise<string>;
|
|
|
3
3
|
export declare function subtitlesToText(subtitles: string): string;
|
|
4
4
|
export declare function subtitlesToTimeline(subtitles: string, removeMarkup?: boolean): Timeline;
|
|
5
5
|
export declare function timelineToSubtitles(timeline: Timeline, subtitlesConfig?: SubtitlesConfig): string;
|
|
6
|
+
export declare function tryParseTimeRangePatternWithHours(line: string): {
|
|
7
|
+
startTime: number;
|
|
8
|
+
endTime: number;
|
|
9
|
+
succeeded: boolean;
|
|
10
|
+
};
|
|
11
|
+
export declare function tryParseTimeRangePatternWithoutHours(line: string): {
|
|
12
|
+
startTime: number;
|
|
13
|
+
endTime: number;
|
|
14
|
+
succeeded: boolean;
|
|
15
|
+
};
|
|
6
16
|
export type Cue = {
|
|
7
17
|
lines: string[];
|
|
8
18
|
startTime: number;
|
|
@@ -328,7 +328,7 @@ function getCuesFromTimeline_IsolateLines(timeline, config) {
|
|
|
328
328
|
addCueFromCurrentWords(); // Add any remaining words
|
|
329
329
|
return cues;
|
|
330
330
|
}
|
|
331
|
-
function tryParseTimeRangePatternWithHours(line) {
|
|
331
|
+
export function tryParseTimeRangePatternWithHours(line) {
|
|
332
332
|
const timeRangePatternWithHours = /^(\d+)\:(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)\:(\d+)[\.,](\d+)/;
|
|
333
333
|
const match = timeRangePatternWithHours.exec(line);
|
|
334
334
|
if (!match) {
|
|
@@ -346,7 +346,7 @@ function tryParseTimeRangePatternWithHours(line) {
|
|
|
346
346
|
const endTime = (endMilliseconds / 1000) + (endSeconds) + (endMinutes * 60) + (endHours * 60 * 60);
|
|
347
347
|
return { startTime, endTime, succeeded: true };
|
|
348
348
|
}
|
|
349
|
-
function tryParseTimeRangePatternWithoutHours(line) {
|
|
349
|
+
export function tryParseTimeRangePatternWithoutHours(line) {
|
|
350
350
|
const timeRangePatternWithHours = /^(\d+)\:(\d+)[\.,](\d+)[ ]*-->[ ]*(\d+)\:(\d+)[\.,](\d+)/;
|
|
351
351
|
const match = timeRangePatternWithHours.exec(line);
|
|
352
352
|
if (!match) {
|