echogarden 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -1
- package/data/lexicons/heteronyms.json +674 -0
- package/data/schemas/options.json +1057 -0
- package/data/tables/lcid-table.json +7427 -0
- package/dist/alignment/DTWMfccSequenceAlignment.d.ts +2 -0
- package/dist/alignment/DTWMfccSequenceAlignment.js +26 -0
- package/dist/alignment/DTWMfccSequenceAlignment.js.map +1 -0
- package/dist/alignment/DTWSequenceAlignment.d.ts +5 -0
- package/dist/alignment/DTWSequenceAlignment.js +99 -0
- package/dist/alignment/DTWSequenceAlignment.js.map +1 -0
- package/dist/alignment/DTWSequenceAlignmentWindowed.d.ts +5 -0
- package/dist/alignment/DTWSequenceAlignmentWindowed.js +161 -0
- package/dist/alignment/DTWSequenceAlignmentWindowed.js.map +1 -0
- package/dist/alignment/LevenshteinSequenceAlignment.d.ts +5 -0
- package/dist/alignment/LevenshteinSequenceAlignment.js +102 -0
- package/dist/alignment/LevenshteinSequenceAlignment.js.map +1 -0
- package/dist/alignment/SpeechAlignment.d.ts +27 -0
- package/dist/alignment/SpeechAlignment.js +266 -0
- package/dist/alignment/SpeechAlignment.js.map +1 -0
- package/dist/api/API.d.ts +8 -0
- package/dist/api/API.js +10 -0
- package/dist/api/API.js.map +1 -0
- package/dist/api/APIOptions.d.ts +12 -0
- package/dist/api/APIOptions.js +2 -0
- package/dist/api/APIOptions.js.map +1 -0
- package/dist/api/Alignment.d.ts +32 -0
- package/dist/api/Alignment.js +134 -0
- package/dist/api/Alignment.js.map +1 -0
- package/dist/api/Denoising.d.ts +14 -0
- package/dist/api/Denoising.js +66 -0
- package/dist/api/Denoising.js.map +1 -0
- package/dist/api/Globals.d.ts +1 -0
- package/dist/api/Globals.js +2 -0
- package/dist/api/Globals.js.map +1 -0
- package/dist/api/LanguageDetection.d.ts +49 -0
- package/dist/api/LanguageDetection.js +118 -0
- package/dist/api/LanguageDetection.js.map +1 -0
- package/dist/api/Recognition.d.ts +49 -0
- package/dist/api/Recognition.js +162 -0
- package/dist/api/Recognition.js.map +1 -0
- package/dist/api/Synthesis.d.ts +125 -0
- package/dist/api/Synthesis.js +892 -0
- package/dist/api/Synthesis.js.map +1 -0
- package/dist/api/Translation.d.ts +25 -0
- package/dist/api/Translation.js +69 -0
- package/dist/api/Translation.js.map +1 -0
- package/dist/api/Vad.d.ts +25 -0
- package/dist/api/Vad.js +88 -0
- package/dist/api/Vad.js.map +1 -0
- package/dist/audio/AudioBufferConversion.d.ts +15 -0
- package/dist/audio/AudioBufferConversion.js +228 -0
- package/dist/audio/AudioBufferConversion.js.map +1 -0
- package/dist/audio/AudioPlayer.d.ts +9 -0
- package/dist/audio/AudioPlayer.js +232 -0
- package/dist/audio/AudioPlayer.js.map +1 -0
- package/dist/audio/AudioRecorder.d.ts +3 -0
- package/dist/audio/AudioRecorder.js +68 -0
- package/dist/audio/AudioRecorder.js.map +1 -0
- package/dist/audio/AudioUtilities.d.ts +48 -0
- package/dist/audio/AudioUtilities.js +209 -0
- package/dist/audio/AudioUtilities.js.map +1 -0
- package/dist/audio/SoxPath.d.ts +1 -0
- package/dist/audio/SoxPath.js +19 -0
- package/dist/audio/SoxPath.js.map +1 -0
- package/dist/cli/CLI.d.ts +8 -0
- package/dist/cli/CLI.js +860 -0
- package/dist/cli/CLI.js.map +1 -0
- package/dist/cli/CLIConfigFile.d.ts +3 -0
- package/dist/cli/CLIConfigFile.js +62 -0
- package/dist/cli/CLIConfigFile.js.map +1 -0
- package/dist/cli/CLILauncher.d.ts +2 -0
- package/dist/cli/CLILauncher.js +22 -0
- package/dist/cli/CLILauncher.js.map +1 -0
- package/dist/cli/CLIOptionsSchema.d.ts +5 -0
- package/dist/cli/CLIOptionsSchema.js +36 -0
- package/dist/cli/CLIOptionsSchema.js.map +1 -0
- package/dist/cli/CLIParser.d.ts +6 -0
- package/dist/cli/CLIParser.js +34 -0
- package/dist/cli/CLIParser.js.map +1 -0
- package/dist/cli/CLIStarter.d.ts +1 -0
- package/dist/cli/CLIStarter.js +3 -0
- package/dist/cli/CLIStarter.js.map +1 -0
- package/dist/codecs/FFMpegTranscoder.d.ts +20 -0
- package/dist/codecs/FFMpegTranscoder.js +169 -0
- package/dist/codecs/FFMpegTranscoder.js.map +1 -0
- package/dist/codecs/TIMITCodec.d.ts +9 -0
- package/dist/codecs/TIMITCodec.js +14 -0
- package/dist/codecs/TIMITCodec.js.map +1 -0
- package/dist/codecs/WaveCodec.d.ts +19 -0
- package/dist/codecs/WaveCodec.js +208 -0
- package/dist/codecs/WaveCodec.js.map +1 -0
- package/dist/denoising/RNNoise.d.ts +6 -0
- package/dist/denoising/RNNoise.js +68 -0
- package/dist/denoising/RNNoise.js.map +1 -0
- package/dist/dsp/BiquadFilter.d.ts +26 -0
- package/dist/dsp/BiquadFilter.js +399 -0
- package/dist/dsp/BiquadFilter.js.map +1 -0
- package/dist/dsp/FFT.d.ts +9 -0
- package/dist/dsp/FFT.js +135 -0
- package/dist/dsp/FFT.js.map +1 -0
- package/dist/dsp/MFCC.d.ts +25 -0
- package/dist/dsp/MFCC.js +162 -0
- package/dist/dsp/MFCC.js.map +1 -0
- package/dist/dsp/MelSpectogram.d.ts +19 -0
- package/dist/dsp/MelSpectogram.js +102 -0
- package/dist/dsp/MelSpectogram.js.map +1 -0
- package/dist/dsp/Rubberband.d.ts +52 -0
- package/dist/dsp/Rubberband.js +186 -0
- package/dist/dsp/Rubberband.js.map +1 -0
- package/dist/dsp/Sonic.d.ts +2 -0
- package/dist/dsp/Sonic.js +39 -0
- package/dist/dsp/Sonic.js.map +1 -0
- package/dist/dsp/SpeexResampler.d.ts +3 -0
- package/dist/dsp/SpeexResampler.js +53 -0
- package/dist/dsp/SpeexResampler.js.map +1 -0
- package/dist/math/VectorMath.d.ts +70 -0
- package/dist/math/VectorMath.js +564 -0
- package/dist/math/VectorMath.js.map +1 -0
- package/dist/nlp/ChineseSegmentation.d.ts +1 -0
- package/dist/nlp/ChineseSegmentation.js +53 -0
- package/dist/nlp/ChineseSegmentation.js.map +1 -0
- package/dist/nlp/CompromiseNLP.d.ts +15 -0
- package/dist/nlp/CompromiseNLP.js +66 -0
- package/dist/nlp/CompromiseNLP.js.map +1 -0
- package/dist/nlp/EspeakPhonemizer.d.ts +4 -0
- package/dist/nlp/EspeakPhonemizer.js +133 -0
- package/dist/nlp/EspeakPhonemizer.js.map +1 -0
- package/dist/nlp/IPA.d.ts +19 -0
- package/dist/nlp/IPA.js +113 -0
- package/dist/nlp/IPA.js.map +1 -0
- package/dist/nlp/JapaneseSegmentation.d.ts +1 -0
- package/dist/nlp/JapaneseSegmentation.js +40 -0
- package/dist/nlp/JapaneseSegmentation.js.map +1 -0
- package/dist/nlp/Lexicon.d.ts +17 -0
- package/dist/nlp/Lexicon.js +6 -0
- package/dist/nlp/Lexicon.js.map +1 -0
- package/dist/nlp/PhoneConversion.d.ts +6 -0
- package/dist/nlp/PhoneConversion.js +467 -0
- package/dist/nlp/PhoneConversion.js.map +1 -0
- package/dist/nlp/Segmentation.d.ts +41 -0
- package/dist/nlp/Segmentation.js +158 -0
- package/dist/nlp/Segmentation.js.map +1 -0
- package/dist/nlp/TextNormalizer.d.ts +4 -0
- package/dist/nlp/TextNormalizer.js +69 -0
- package/dist/nlp/TextNormalizer.js.map +1 -0
- package/dist/recognition/AmazonTranscribeSTT.d.ts +6 -0
- package/dist/recognition/AmazonTranscribeSTT.js +79 -0
- package/dist/recognition/AmazonTranscribeSTT.js.map +1 -0
- package/dist/recognition/AzureCognitiveServicesSTT.d.ts +7 -0
- package/dist/recognition/AzureCognitiveServicesSTT.js +51 -0
- package/dist/recognition/AzureCognitiveServicesSTT.js.map +1 -0
- package/dist/recognition/GoogleCloudSTT.d.ts +7 -0
- package/dist/recognition/GoogleCloudSTT.js +66 -0
- package/dist/recognition/GoogleCloudSTT.js.map +1 -0
- package/dist/recognition/SileroSTT.d.ts +9 -0
- package/dist/recognition/SileroSTT.js +125 -0
- package/dist/recognition/SileroSTT.js.map +1 -0
- package/dist/recognition/VoskSTT.d.ts +10 -0
- package/dist/recognition/VoskSTT.js +78 -0
- package/dist/recognition/VoskSTT.js.map +1 -0
- package/dist/recognition/WhisperSTT.d.ts +69 -0
- package/dist/recognition/WhisperSTT.js +977 -0
- package/dist/recognition/WhisperSTT.js.map +1 -0
- package/dist/server/Server.d.ts +1 -0
- package/dist/server/Server.js +18 -0
- package/dist/server/Server.js.map +1 -0
- package/dist/speech-language-detection/SileroLanguageDetection.d.ts +9 -0
- package/dist/speech-language-detection/SileroLanguageDetection.js +47 -0
- package/dist/speech-language-detection/SileroLanguageDetection.js.map +1 -0
- package/dist/subtitles/Subtitles.d.ts +25 -0
- package/dist/subtitles/Subtitles.js +286 -0
- package/dist/subtitles/Subtitles.js.map +1 -0
- package/dist/synthesis/AwsPollyTTS.d.ts +7 -0
- package/dist/synthesis/AwsPollyTTS.js +51 -0
- package/dist/synthesis/AwsPollyTTS.js.map +1 -0
- package/dist/synthesis/AzureCognitiveServicesTTS.d.ts +9 -0
- package/dist/synthesis/AzureCognitiveServicesTTS.js +103 -0
- package/dist/synthesis/AzureCognitiveServicesTTS.js.map +1 -0
- package/dist/synthesis/CoquiServerTTS.d.ts +6 -0
- package/dist/synthesis/CoquiServerTTS.js +22 -0
- package/dist/synthesis/CoquiServerTTS.js.map +1 -0
- package/dist/synthesis/ElevenLabsTTS.d.ts +8 -0
- package/dist/synthesis/ElevenLabsTTS.js +48 -0
- package/dist/synthesis/ElevenLabsTTS.js.map +1 -0
- package/dist/synthesis/EspeakTTS.d.ts +46 -0
- package/dist/synthesis/EspeakTTS.js +353 -0
- package/dist/synthesis/EspeakTTS.js.map +1 -0
- package/dist/synthesis/FliteTTS.d.ts +17 -0
- package/dist/synthesis/FliteTTS.js +326 -0
- package/dist/synthesis/FliteTTS.js.map +1 -0
- package/dist/synthesis/GoogleCloudTTS.d.ts +16 -0
- package/dist/synthesis/GoogleCloudTTS.js +72 -0
- package/dist/synthesis/GoogleCloudTTS.js.map +1 -0
- package/dist/synthesis/GoogleTranslateTTS.d.ts +13 -0
- package/dist/synthesis/GoogleTranslateTTS.js +177 -0
- package/dist/synthesis/GoogleTranslateTTS.js.map +1 -0
- package/dist/synthesis/MicrosoftEdgeTTS.d.ts +10 -0
- package/dist/synthesis/MicrosoftEdgeTTS.js +216 -0
- package/dist/synthesis/MicrosoftEdgeTTS.js.map +1 -0
- package/dist/synthesis/SamTTS.d.ts +4 -0
- package/dist/synthesis/SamTTS.js +21 -0
- package/dist/synthesis/SamTTS.js.map +1 -0
- package/dist/synthesis/SapiTTS.d.ts +9 -0
- package/dist/synthesis/SapiTTS.js +211 -0
- package/dist/synthesis/SapiTTS.js.map +1 -0
- package/dist/synthesis/StreamlabsPollyTTS.d.ts +12 -0
- package/dist/synthesis/StreamlabsPollyTTS.js +88 -0
- package/dist/synthesis/StreamlabsPollyTTS.js.map +1 -0
- package/dist/synthesis/SvoxPicoTTS.d.ts +11 -0
- package/dist/synthesis/SvoxPicoTTS.js +236 -0
- package/dist/synthesis/SvoxPicoTTS.js.map +1 -0
- package/dist/synthesis/VitsTTS.d.ts +27 -0
- package/dist/synthesis/VitsTTS.js +359 -0
- package/dist/synthesis/VitsTTS.js.map +1 -0
- package/dist/tests/Test.d.ts +1 -0
- package/dist/tests/Test.js +10 -0
- package/dist/tests/Test.js.map +1 -0
- package/dist/text-language-detection/FastTextLanguageDetection.d.ts +2 -0
- package/dist/text-language-detection/FastTextLanguageDetection.js +38 -0
- package/dist/text-language-detection/FastTextLanguageDetection.js.map +1 -0
- package/dist/text-language-detection/TinyLDLanguageDetection.d.ts +2 -0
- package/dist/text-language-detection/TinyLDLanguageDetection.js +12 -0
- package/dist/text-language-detection/TinyLDLanguageDetection.js.map +1 -0
- package/dist/utilities/BinaryArrayConversion.d.ts +15 -0
- package/dist/utilities/BinaryArrayConversion.js +115 -0
- package/dist/utilities/BinaryArrayConversion.js.map +1 -0
- package/dist/utilities/Compression.d.ts +4 -0
- package/dist/utilities/Compression.js +67 -0
- package/dist/utilities/Compression.js.map +1 -0
- package/dist/utilities/FileDownloader.d.ts +3 -0
- package/dist/utilities/FileDownloader.js +147 -0
- package/dist/utilities/FileDownloader.js.map +1 -0
- package/dist/utilities/FileSystem.d.ts +30 -0
- package/dist/utilities/FileSystem.js +141 -0
- package/dist/utilities/FileSystem.js.map +1 -0
- package/dist/utilities/Hashing.d.ts +10 -0
- package/dist/utilities/Hashing.js +169 -0
- package/dist/utilities/Hashing.js.map +1 -0
- package/dist/utilities/Locale.d.ts +9 -0
- package/dist/utilities/Locale.js +65 -0
- package/dist/utilities/Locale.js.map +1 -0
- package/dist/utilities/Logger.d.ts +11 -0
- package/dist/utilities/Logger.js +49 -0
- package/dist/utilities/Logger.js.map +1 -0
- package/dist/utilities/NdArrayUtilities.d.ts +3 -0
- package/dist/utilities/NdArrayUtilities.js +22 -0
- package/dist/utilities/NdArrayUtilities.js.map +1 -0
- package/dist/utilities/ObjectUtilities.d.ts +4 -0
- package/dist/utilities/ObjectUtilities.js +132 -0
- package/dist/utilities/ObjectUtilities.js.map +1 -0
- package/dist/utilities/OpenPromise.d.ts +6 -0
- package/dist/utilities/OpenPromise.js +12 -0
- package/dist/utilities/OpenPromise.js.map +1 -0
- package/dist/utilities/PackageManager.d.ts +4 -0
- package/dist/utilities/PackageManager.js +46 -0
- package/dist/utilities/PackageManager.js.map +1 -0
- package/dist/utilities/RandomGenerator.d.ts +35 -0
- package/dist/utilities/RandomGenerator.js +149 -0
- package/dist/utilities/RandomGenerator.js.map +1 -0
- package/dist/utilities/TarballMaker.d.ts +4 -0
- package/dist/utilities/TarballMaker.js +50 -0
- package/dist/utilities/TarballMaker.js.map +1 -0
- package/dist/utilities/Timeline.d.ts +20 -0
- package/dist/utilities/Timeline.js +110 -0
- package/dist/utilities/Timeline.js.map +1 -0
- package/dist/utilities/Timer.d.ts +13 -0
- package/dist/utilities/Timer.js +70 -0
- package/dist/utilities/Timer.js.map +1 -0
- package/dist/utilities/Utilities.d.ts +68 -0
- package/dist/utilities/Utilities.js +305 -0
- package/dist/utilities/Utilities.js.map +1 -0
- package/dist/utilities/WasmMemoryManager.d.ts +142 -0
- package/dist/utilities/WasmMemoryManager.js +407 -0
- package/dist/utilities/WasmMemoryManager.js.map +1 -0
- package/dist/utilities/WebReader.d.ts +1 -0
- package/dist/utilities/WebReader.js +47 -0
- package/dist/utilities/WebReader.js.map +1 -0
- package/dist/utilities/WikipediaReader.d.ts +1 -0
- package/dist/utilities/WikipediaReader.js +31 -0
- package/dist/utilities/WikipediaReader.js.map +1 -0
- package/dist/voice-activity-detection/SileroVAD.d.ts +13 -0
- package/dist/voice-activity-detection/SileroVAD.js +58 -0
- package/dist/voice-activity-detection/SileroVAD.js.map +1 -0
- package/dist/voice-activity-detection/WebRtcVAD.d.ts +3 -0
- package/dist/voice-activity-detection/WebRtcVAD.js +53 -0
- package/dist/voice-activity-detection/WebRtcVAD.js.map +1 -0
- package/docs/CLI.md +234 -0
- package/docs/Development.md +3 -0
- package/docs/Engines.md +80 -0
- package/docs/Licenses.md +37 -0
- package/docs/Options.md +192 -0
- package/docs/Roadmap.md +16 -0
- package/docs/Technical.md +72 -0
- package/package.json +119 -19
- package/cli.js +0 -3
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { splitToWords } from "../nlp/Segmentation.js";
|
|
2
|
+
import { clip, roundToDigits } from "../utilities/Utilities.js";
|
|
3
|
+
import { computeMFCCs, extendDefaultMfccOptions } from "../dsp/MFCC.js";
|
|
4
|
+
import { alignMFCC_DTW, getCostMatrixMemorySizeMiB } from "./DTWMfccSequenceAlignment.js";
|
|
5
|
+
import { Logger } from "../utilities/Logger.js";
|
|
6
|
+
import { getRawAudioDuration } from "../audio/AudioUtilities.js";
|
|
7
|
+
export async function alignUsingDtw(sourceRawAudio, referenceRawAudio, referenceTimeline, windowDuration = 60) {
|
|
8
|
+
const logger = new Logger();
|
|
9
|
+
const mfccOptions = extendDefaultMfccOptions({ zeroFirstCoefficient: true });
|
|
10
|
+
const framesPerSecond = 1 / mfccOptions.hopDuration;
|
|
11
|
+
// Compute reference MFCCs
|
|
12
|
+
logger.start("Compute reference MFCC features");
|
|
13
|
+
const referenceMfccs = await computeMFCCs(referenceRawAudio, mfccOptions);
|
|
14
|
+
// Compute source MFCCs
|
|
15
|
+
logger.start("Compute source MFCC features");
|
|
16
|
+
const sourceMfccs = await computeMFCCs(sourceRawAudio, mfccOptions);
|
|
17
|
+
logger.end();
|
|
18
|
+
// Compute path
|
|
19
|
+
logger.log(`DTW cost matrix memory size (${windowDuration}s window): ${roundToDigits(getCostMatrixMemorySizeMiB(referenceMfccs.length, sourceMfccs.length, windowDuration * framesPerSecond), 1)}MiB`);
|
|
20
|
+
logger.start("Align MFCC features with DTW");
|
|
21
|
+
const dtwWindowLength = Math.floor(windowDuration * framesPerSecond);
|
|
22
|
+
const rawPath = await alignMFCC_DTW(referenceMfccs, sourceMfccs, dtwWindowLength);
|
|
23
|
+
logger.start("Convert path to timeline");
|
|
24
|
+
const compactedPath = compactPath(rawPath);
|
|
25
|
+
function getMappedTimelineEntry(timelineEntry, recurse = true) {
|
|
26
|
+
const referenceStartFrameIndex = Math.floor(timelineEntry.startTime * framesPerSecond);
|
|
27
|
+
const referenceEndFrameIndex = Math.floor(timelineEntry.endTime * framesPerSecond);
|
|
28
|
+
if (referenceStartFrameIndex < 0 || referenceEndFrameIndex < 0) {
|
|
29
|
+
throw new Error("Unexpected: encountered a negative timestamp in timeline");
|
|
30
|
+
}
|
|
31
|
+
const mappedStartFrameIndex = getMappedFrameIndexForPath(referenceStartFrameIndex, compactedPath, "first");
|
|
32
|
+
const mappedEndFrameIndex = getMappedFrameIndexForPath(referenceEndFrameIndex, compactedPath, "first");
|
|
33
|
+
let innerTimeline;
|
|
34
|
+
if (recurse && timelineEntry.timeline != null) {
|
|
35
|
+
innerTimeline = timelineEntry.timeline.map((entry) => getMappedTimelineEntry(entry));
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
type: timelineEntry.type,
|
|
39
|
+
text: timelineEntry.text,
|
|
40
|
+
startTime: mappedStartFrameIndex / framesPerSecond,
|
|
41
|
+
endTime: mappedEndFrameIndex / framesPerSecond,
|
|
42
|
+
timeline: innerTimeline
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
const mappedTimeline = referenceTimeline.map((timelineEntry) => getMappedTimelineEntry(timelineEntry));
|
|
46
|
+
logger.end();
|
|
47
|
+
return mappedTimeline;
|
|
48
|
+
}
|
|
49
|
+
export async function alignUsingDtwWithRecognition(sourceRawAudio, referenceRawAudio, referenceTimeline, recognitionTimeline, espeakVoice = "gmw/en-US", phoneAlignmentMethod = "interpolation", windowDuration = 60) {
|
|
50
|
+
const logger = new Logger();
|
|
51
|
+
if (recognitionTimeline.length == 0) {
|
|
52
|
+
const sourceDuration = getRawAudioDuration(sourceRawAudio);
|
|
53
|
+
const referenceDuration = getRawAudioDuration(referenceRawAudio);
|
|
54
|
+
const ratio = sourceDuration / referenceDuration;
|
|
55
|
+
const interpolatedTimeline = [];
|
|
56
|
+
for (const entry of referenceTimeline) {
|
|
57
|
+
interpolatedTimeline.push({
|
|
58
|
+
type: entry.type,
|
|
59
|
+
text: entry.text,
|
|
60
|
+
startTime: entry.startTime * ratio,
|
|
61
|
+
endTime: entry.endTime * ratio
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
return interpolatedTimeline;
|
|
65
|
+
}
|
|
66
|
+
// Synthesize the recognized transcript and get its timeline
|
|
67
|
+
logger.start("Synthesize recognized transcript with espeak");
|
|
68
|
+
const recognizedWords = recognitionTimeline.map(entry => entry.text);
|
|
69
|
+
const { rawAudio: synthesizedRecognizedTranscriptRawAudio, timeline: synthesizedRecognitionTimeline } = await createAlignmentReferenceUsingEspeakForFragments(recognizedWords, espeakVoice);
|
|
70
|
+
let recognitionTimelineWithPhones;
|
|
71
|
+
if (phoneAlignmentMethod == "interpolation") {
|
|
72
|
+
// Add phone timelines by interpolating from reference words
|
|
73
|
+
logger.start("Interpolate phone timing");
|
|
74
|
+
recognitionTimelineWithPhones = await interpolatePhoneTimelines(recognitionTimeline, synthesizedRecognitionTimeline);
|
|
75
|
+
}
|
|
76
|
+
else if (phoneAlignmentMethod == "dtw") {
|
|
77
|
+
logger.start("Align phone timing");
|
|
78
|
+
// Add phone timelines by aligning each individual recognized word with the corresponding word
|
|
79
|
+
// in the reference timeline
|
|
80
|
+
recognitionTimelineWithPhones = await alignPhoneTimelines(sourceRawAudio, recognitionTimeline, synthesizedRecognizedTranscriptRawAudio, synthesizedRecognitionTimeline);
|
|
81
|
+
}
|
|
82
|
+
else if (phoneAlignmentMethod == "dtw-knn") {
|
|
83
|
+
logger.start("Align phone timing");
|
|
84
|
+
throw new Error("Not implemented");
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
throw new Error(`Unknown phone alignment method: ${phoneAlignmentMethod}`);
|
|
88
|
+
}
|
|
89
|
+
logger.start("Map from the synthesized recognized timeline to the recognized timeline");
|
|
90
|
+
const synthesizedToRecognizedTimeMapping = [];
|
|
91
|
+
for (let i = 0; i < synthesizedRecognitionTimeline.length; i++) {
|
|
92
|
+
const synthesizedTimelineEntry = synthesizedRecognitionTimeline[i];
|
|
93
|
+
const recognitionTimelineEntry = recognitionTimelineWithPhones[i];
|
|
94
|
+
synthesizedToRecognizedTimeMapping.push({ synthesized: synthesizedTimelineEntry.startTime, recognized: recognitionTimelineEntry.startTime });
|
|
95
|
+
if (synthesizedTimelineEntry.timeline) {
|
|
96
|
+
for (let j = 0; j < synthesizedTimelineEntry.timeline.length; j++) {
|
|
97
|
+
const synthesizedPhoneTimelineEntry = synthesizedTimelineEntry.timeline[j];
|
|
98
|
+
const recognitionPhoneTimelineEntry = recognitionTimelineEntry.timeline[j];
|
|
99
|
+
synthesizedToRecognizedTimeMapping.push({ synthesized: synthesizedPhoneTimelineEntry.startTime, recognized: recognitionPhoneTimelineEntry.startTime });
|
|
100
|
+
synthesizedToRecognizedTimeMapping.push({ synthesized: synthesizedPhoneTimelineEntry.endTime, recognized: recognitionPhoneTimelineEntry.endTime });
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
synthesizedToRecognizedTimeMapping.push({ synthesized: synthesizedTimelineEntry.endTime, recognized: recognitionTimelineEntry.endTime });
|
|
104
|
+
}
|
|
105
|
+
logger.start("Align the synthesized recognized transcript with the synthesized ground-truth transcript");
|
|
106
|
+
// Align the synthesized recognized transcript to the synthesized reference transcript
|
|
107
|
+
const alignedSynthesizedRecognitionTimeline = await alignUsingDtw(synthesizedRecognizedTranscriptRawAudio, referenceRawAudio, referenceTimeline, windowDuration);
|
|
108
|
+
let currentSynthesizedToRecognizedMappingIndex = 0;
|
|
109
|
+
function mapSynthesizedToRecognizedTimeAndAdvance(synthesizedTime) {
|
|
110
|
+
for (;; currentSynthesizedToRecognizedMappingIndex += 1) {
|
|
111
|
+
const left = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].synthesized;
|
|
112
|
+
let right;
|
|
113
|
+
if (currentSynthesizedToRecognizedMappingIndex < synthesizedToRecognizedTimeMapping.length - 1) {
|
|
114
|
+
right = synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex + 1].synthesized;
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
right = Infinity;
|
|
118
|
+
}
|
|
119
|
+
if (left > right) {
|
|
120
|
+
throw new Error("left is larger than right!");
|
|
121
|
+
}
|
|
122
|
+
if (Math.abs(synthesizedTime - left) < Math.abs(synthesizedTime - right)) {
|
|
123
|
+
return synthesizedToRecognizedTimeMapping[currentSynthesizedToRecognizedMappingIndex].recognized;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
function mapTimeline(timeline) {
|
|
128
|
+
const mappedTimeline = [];
|
|
129
|
+
for (const entry of timeline) {
|
|
130
|
+
const mappedEntry = { ...entry };
|
|
131
|
+
mappedEntry.startTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.startTime);
|
|
132
|
+
if (entry.timeline) {
|
|
133
|
+
mappedEntry.timeline = mapTimeline(entry.timeline);
|
|
134
|
+
}
|
|
135
|
+
mappedEntry.endTime = mapSynthesizedToRecognizedTimeAndAdvance(entry.endTime);
|
|
136
|
+
mappedTimeline.push(mappedEntry);
|
|
137
|
+
}
|
|
138
|
+
return mappedTimeline;
|
|
139
|
+
}
|
|
140
|
+
const result = mapTimeline(alignedSynthesizedRecognitionTimeline);
|
|
141
|
+
logger.end();
|
|
142
|
+
return result;
|
|
143
|
+
}
|
|
144
|
+
export async function interpolatePhoneTimelines(sourceTimeline, referenceTimeline) {
|
|
145
|
+
const interpolatedTimeline = [];
|
|
146
|
+
for (let i = 0; i < sourceTimeline.length; i++) {
|
|
147
|
+
const referenceEntry = referenceTimeline[i];
|
|
148
|
+
const interpolatedEntry = { ...sourceTimeline[i] };
|
|
149
|
+
interpolatedTimeline.push(interpolatedEntry);
|
|
150
|
+
if (interpolatedEntry.type != "word") {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
const interpolatedEntryDuration = interpolatedEntry.endTime - interpolatedEntry.startTime;
|
|
154
|
+
const synthesisEntryDuration = referenceEntry.endTime - referenceEntry.startTime;
|
|
155
|
+
interpolatedEntry.timeline = [];
|
|
156
|
+
for (const phoneEntry of referenceEntry.timeline) {
|
|
157
|
+
const phoneStartTimePercentageRelativeToWord = (phoneEntry.startTime - referenceEntry.startTime) / synthesisEntryDuration;
|
|
158
|
+
const phoneEndTimePercentageRelativeToWord = (phoneEntry.endTime - referenceEntry.startTime) / synthesisEntryDuration;
|
|
159
|
+
const interpolatedPhoneStartTime = interpolatedEntry.startTime + (phoneStartTimePercentageRelativeToWord * interpolatedEntryDuration);
|
|
160
|
+
const interpolatedPhoneEndTime = interpolatedEntry.startTime + (phoneEndTimePercentageRelativeToWord * interpolatedEntryDuration);
|
|
161
|
+
interpolatedEntry.timeline.push({
|
|
162
|
+
...phoneEntry,
|
|
163
|
+
startTime: interpolatedPhoneStartTime,
|
|
164
|
+
endTime: interpolatedPhoneEndTime
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return interpolatedTimeline;
|
|
169
|
+
}
|
|
170
|
+
export async function alignPhoneTimelines(sourceRawAudio, sourceWordTimeline, referenceRawAudio, referenceTimeline) {
|
|
171
|
+
const mfccOptions = extendDefaultMfccOptions({ zeroFirstCoefficient: true });
|
|
172
|
+
const framesPerSecond = 1 / mfccOptions.hopDuration;
|
|
173
|
+
const referenceMfccs = await computeMFCCs(referenceRawAudio, mfccOptions);
|
|
174
|
+
const sourceMfccs = await computeMFCCs(sourceRawAudio, mfccOptions);
|
|
175
|
+
const alignedWordTimeline = [];
|
|
176
|
+
for (let i = 0; i < referenceTimeline.length; i++) {
|
|
177
|
+
const referenceWordEntry = referenceTimeline[i];
|
|
178
|
+
const alignedWordEntry = { ...sourceWordTimeline[i] };
|
|
179
|
+
alignedWordTimeline.push(alignedWordEntry);
|
|
180
|
+
if (alignedWordEntry.type != "word") {
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
const referenceWordStartFrameIndex = Math.floor(referenceWordEntry.startTime * framesPerSecond);
|
|
184
|
+
let referenceWordEndFrameIndex = Math.floor(referenceWordEntry.endTime * framesPerSecond);
|
|
185
|
+
// Ensure there is at least one frame in range
|
|
186
|
+
if (referenceWordEndFrameIndex <= referenceWordStartFrameIndex) {
|
|
187
|
+
referenceWordEndFrameIndex = referenceWordEndFrameIndex + 1;
|
|
188
|
+
}
|
|
189
|
+
const referenceWordMfccs = referenceMfccs.slice(referenceWordStartFrameIndex, referenceWordEndFrameIndex);
|
|
190
|
+
const alignedWordStartFrameIndex = Math.floor(alignedWordEntry.startTime * framesPerSecond);
|
|
191
|
+
let alignedWordEndFrameIndex = Math.floor(alignedWordEntry.endTime * framesPerSecond);
|
|
192
|
+
// Ensure there is at least one frame in range
|
|
193
|
+
if (alignedWordEndFrameIndex <= alignedWordStartFrameIndex) {
|
|
194
|
+
alignedWordEndFrameIndex = alignedWordStartFrameIndex + 1;
|
|
195
|
+
}
|
|
196
|
+
const sourceWordMfccs = sourceMfccs.slice(alignedWordStartFrameIndex, alignedWordEndFrameIndex);
|
|
197
|
+
// Compute DTW path
|
|
198
|
+
const rawPath = await alignMFCC_DTW(referenceWordMfccs, sourceWordMfccs, 60);
|
|
199
|
+
const compactedPath = compactPath(rawPath);
|
|
200
|
+
// Add phone timeline using the mapped time information
|
|
201
|
+
alignedWordEntry.timeline = [];
|
|
202
|
+
for (const referencePhoneEntry of referenceWordEntry.timeline) {
|
|
203
|
+
const referencePhoneStartFrameOffset = Math.floor((referencePhoneEntry.startTime - referenceWordEntry.startTime) * framesPerSecond);
|
|
204
|
+
const alignedPhoneStartFrameOffset = getMappedFrameIndexForPath(referencePhoneStartFrameOffset, compactedPath);
|
|
205
|
+
const alignedPhoneStartTime = alignedWordEntry.startTime + (alignedPhoneStartFrameOffset / framesPerSecond);
|
|
206
|
+
const referencePhoneEndFrameOffset = Math.floor((referencePhoneEntry.endTime - referenceWordEntry.startTime) * framesPerSecond);
|
|
207
|
+
const alignedPhoneEndFrameOffset = getMappedFrameIndexForPath(referencePhoneEndFrameOffset, compactedPath);
|
|
208
|
+
const alignedPhoneEndTime = alignedWordEntry.startTime + (alignedPhoneEndFrameOffset / framesPerSecond);
|
|
209
|
+
alignedWordEntry.timeline.push({
|
|
210
|
+
...referencePhoneEntry,
|
|
211
|
+
startTime: alignedPhoneStartTime,
|
|
212
|
+
endTime: alignedPhoneEndTime
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return alignedWordTimeline;
|
|
217
|
+
}
|
|
218
|
+
export async function createAlignmentReferenceUsingEspeak(text, language, voice, insertSeparators = true, rate, pitch, pitchRange) {
|
|
219
|
+
const progressLogger = new Logger();
|
|
220
|
+
progressLogger.start("Segment text to words");
|
|
221
|
+
const fragments = await splitToWords(text, language);
|
|
222
|
+
progressLogger.end();
|
|
223
|
+
return createAlignmentReferenceUsingEspeakForFragments(fragments, voice, insertSeparators, rate, pitch, pitchRange);
|
|
224
|
+
}
|
|
225
|
+
export async function createAlignmentReferenceUsingEspeakForFragments(fragments, voice, insertSeparators = true, rate, pitch, pitchRange) {
|
|
226
|
+
const progressLogger = new Logger();
|
|
227
|
+
progressLogger.start("Load espeak module");
|
|
228
|
+
const Espeak = await import("../synthesis/EspeakTTS.js");
|
|
229
|
+
progressLogger.start("Create alignment reference with Espeak");
|
|
230
|
+
const result = await Espeak.synthesizeFragments(fragments, voice, insertSeparators, rate, pitch, pitchRange);
|
|
231
|
+
result.timeline = result.timeline.flatMap(clause => clause.timeline);
|
|
232
|
+
for (const wordEntry of result.timeline) {
|
|
233
|
+
wordEntry.timeline = wordEntry.timeline.flatMap(tokenEntry => tokenEntry.timeline);
|
|
234
|
+
}
|
|
235
|
+
progressLogger.end();
|
|
236
|
+
return result;
|
|
237
|
+
}
|
|
238
|
+
function compactPath(path) {
|
|
239
|
+
const compactedPath = [];
|
|
240
|
+
for (let i = 0; i < path.length; i++) {
|
|
241
|
+
const pathEntry = path[i];
|
|
242
|
+
if (compactedPath.length <= pathEntry.source) {
|
|
243
|
+
compactedPath.push({ first: pathEntry.dest, last: pathEntry.dest });
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
compactedPath[compactedPath.length - 1].last = pathEntry.dest;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return compactedPath;
|
|
250
|
+
}
|
|
251
|
+
function getMappedFrameIndexForPath(referenceFrameIndex, compactedPath, mappingKind = "first") {
|
|
252
|
+
if (compactedPath.length == 0) {
|
|
253
|
+
return 0;
|
|
254
|
+
}
|
|
255
|
+
referenceFrameIndex = clip(referenceFrameIndex, 0, compactedPath.length - 1);
|
|
256
|
+
const compactedPathEntry = compactedPath[referenceFrameIndex];
|
|
257
|
+
let mappedFrameIndex;
|
|
258
|
+
if (mappingKind == "first") {
|
|
259
|
+
mappedFrameIndex = compactedPathEntry.first;
|
|
260
|
+
}
|
|
261
|
+
else {
|
|
262
|
+
mappedFrameIndex = compactedPathEntry.last;
|
|
263
|
+
}
|
|
264
|
+
return mappedFrameIndex;
|
|
265
|
+
}
|
|
266
|
+
//# sourceMappingURL=SpeechAlignment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SpeechAlignment.js","sourceRoot":"","sources":["../../src/alignment/SpeechAlignment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AACrD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AAI/D,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAe,MAAM,gBAAgB,CAAA;AACpF,OAAO,EAAE,aAAa,EAAE,0BAA0B,EAAE,MAAM,+BAA+B,CAAA;AACzF,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAE,mBAAmB,EAAY,MAAM,4BAA4B,CAAA;AAE1E,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,cAAwB,EAAE,iBAA2B,EAAE,iBAA2B,EAAE,cAAc,GAAG,EAAE;IAC1I,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,WAAW,GAAgB,wBAAwB,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzF,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,CAAC,WAAY,CAAA;IAEpD,0BAA0B;IAC1B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAA;IAC/C,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAA;IAEzE,uBAAuB;IACvB,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;IAC5C,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;IACnE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,eAAe;IACf,MAAM,CAAC,GAAG,CAAC,gCAAgC,cAAc,cAAc,aAAa,CAAC,0BAA0B,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAA;IAEtM,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;IAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,eAAe,CAAC,CAAA;IAEpE,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,WAAW,EAAE,eAAe,CAAC,CAAA;IAEjF,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;IACxC,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;IAE1C,SAAS,sBAAsB,CAAC,aAA4B,EAAE,OAAO,GAAG,IAAI;QAC3E,MAAM,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QACtF,MAAM,sBAAsB,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAElF,IAAI,wBAAwB,GAAG,CAAC,IAAI,sBAAsB,GAAG,CAAC,EAAE;YAC/D,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;SAC3E;QAED,MAAM,qBAAqB,GAAG,0BAA0B,CAAC,wBAAwB,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;QAC1G,MAAM,mBAAmB,GAAG,0BAA0B,CAAC,sBAAsB,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;QAEtG,IAAI,aAAmC,CAAA;QAEvC,IAAI,OAAO,IAAI,aAAa,CAAC,QAAQ,IAAI,IAAI,EAAE;YAC9C,aAAa,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAA;SACpF;QAED,OAAO;YACN,IAAI,EAAE,aAAa,CAAC,IAAI;YACxB,IAAI,EAAE,aAAa,CAAC,IAAI;YAExB,SAAS,EAAE,qBAAqB,GAAG,eAAe;YAClD,OAAO,EAAE,mBAAmB,GAAG,eAAe;YAE9C,QAAQ,EAAE,aAAa;SACvB,CAAA;IACF,CAAC;IAED,MAAM,cAAc,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,sBAAsB,CAAC,aAAa,CAAC,CAAC,CAAA;IAEtG,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,cAAc,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,cAAwB,EAAE,iBAA2B,EAAE,iBAA2B,EAAE,mBAA6B,EAAE,WAAW,GAAG,WAAW,EAAE,uBAAiD,eAAe,EAAE,cAAc,GAAG,EAAE;IACrR,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,IAAI,mBAAmB,CAAC,MAAM,IAAI,CAAC,EAAE;QACpC,MAAM,cAAc,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;QAC1D,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,iBAAiB,CAAC,CAAA;QAChE,MAAM,KAAK,GAAG,cAAc,GAAG,iBAAiB,CAAA;QAEhD,MAAM,oBAAoB,GAAa,EAAE,CAAA;QAEzC,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE;YACtC,oBAAoB,CAAC,IAAI,CAAC;gBACzB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,SAAS,EAAE,KAAK,CAAC,SAAS,GAAG,KAAK;gBAClC,OAAO,EAAE,KAAK,CAAC,OAAO,GAAG,KAAK;aAC9B,CAAC,CAAA;SACF;QAED,OAAO,oBAAoB,CAAA;KAC3B;IAED,4DAA4D;IAC5D,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;IAC5D,MAAM,eAAe,GAAG,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAEpE,MAAM,EAAE,QAAQ,EAAE,uCAAuC,EAAE,QAAQ,EAAE,8BAA8B,EAAE,GAAG,MAAM,+CAA+C,CAAC,eAAe,EAAE,WAAW,CAAC,CAAA;IAE3L,IAAI,6BAAuC,CAAA;IAE3C,IAAI,oBAAoB,IAAI,eAAe,EAAE;QAC5C,4DAA4D;QAC5D,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;QAExC,6BAA6B,GAAG,MAAM,yBAAyB,CAAC,mBAAmB,EAAE,8BAA8B,CAAC,CAAA;KACpH;SAAM,IAAI,oBAAoB,IAAI,KAAK,EAAE;QACzC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAA;QAElC,8FAA8F;QAC9F,4BAA4B;QAC5B,6BAA6B,GAAG,MAAM,mBAAmB,CAAC,cAAc,EAAE,mBAAmB,EAAE,uCAAuC,EAAE,8BAA8B,CAAC,CAAA;KACvK;SAAM,IAAI,oBAAoB,IAAI,SAAS,EAAE;QAC7C,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAA;QAClC,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAA;KAClC;SAAM;QACN,MAAM,IAAI,KAAK,CAAC,mCAAmC,oBAAoB,EAAE,CAAC,CAAA;KAC1E;IAED,MAAM,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAA;IAKvF,MAAM,kCAAkC,GAAuC,EAAE,CAAA;IAEjF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,8BAA8B,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAC/D,MAAM,wBAAwB,GAAG,8BAA8B,CAAC,CAAC,CAAC,CAAA;QAClE,MAAM,wBAAwB,GAAG,6BAA6B,CAAC,CAAC,CAAC,CAAA;QAEjE,kCAAkC,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,wBAAwB,CAAC,SAAS,EAAE,UAAU,EAAE,wBAAwB,CAAC,SAAS,EAAE,CAAC,CAAA;QAE5I,IAAI,wBAAwB,CAAC,QAAQ,EAAE;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,wBAAwB,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAClE,MAAM,6BAA6B,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;gBAC1E,MAAM,6BAA6B,GAAG,wBAAwB,CAAC,QAAS,CAAC,CAAC,CAAC,CAAA;gBAE3E,kCAAkC,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,6BAA6B,CAAC,SAAS,EAAE,UAAU,EAAE,6BAA6B,CAAC,SAAS,EAAE,CAAC,CAAA;gBACtJ,kCAAkC,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,6BAA6B,CAAC,OAAO,EAAE,UAAU,EAAE,6BAA6B,CAAC,OAAO,EAAE,CAAC,CAAA;aAClJ;SACD;QAED,kCAAkC,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,wBAAwB,CAAC,OAAO,EAAE,UAAU,EAAE,wBAAwB,CAAC,OAAO,EAAE,CAAC,CAAA;KACxI;IAED,MAAM,CAAC,KAAK,CAAC,0FAA0F,CAAC,CAAA;IACxG,sFAAsF;IACtF,MAAM,qCAAqC,GAAG,MAAM,aAAa,CAAC,uCAAuC,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,cAAc,CAAC,CAAA;IAEhK,IAAI,0CAA0C,GAAG,CAAC,CAAA;IAElD,SAAS,wCAAwC,CAAC,eAAuB;QACxE,QAAS,0CAA0C,IAAI,CAAC,EAAE;YACzD,MAAM,IAAI,GAAG,kCAAkC,CAAC,0CAA0C,CAAC,CAAC,WAAW,CAAA;YAEvG,IAAI,KAAa,CAAA;YAEjB,IAAI,0CAA0C,GAAG,kCAAkC,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC/F,KAAK,GAAG,kCAAkC,CAAC,0CAA0C,GAAG,CAAC,CAAC,CAAC,WAAW,CAAA;aACtG;iBAAM;gBACN,KAAK,GAAG,QAAQ,CAAA;aAChB;YAED,IAAI,IAAI,GAAG,KAAK,EAAE;gBACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;aAC7C;YAED,IAAI,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,KAAK,CAAC,EAAE;gBACzE,OAAO,kCAAkC,CAAC,0CAA0C,CAAC,CAAC,UAAU,CAAA;aAChG;SACD;IACF,CAAC;IAED,SAAS,WAAW,CAAC,QAAkB;QACtC,MAAM,cAAc,GAAa,EAAE,CAAA;QAEnC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;YAC7B,MAAM,WAAW,GAAG,EAAE,GAAG,KAAK,EAAE,CAAA;YAEhC,WAAW,CAAC,SAAS,GAAG,wCAAwC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YAEjF,IAAI,KAAK,CAAC,QAAQ,EAAE;gBACnB,WAAW,CAAC,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;aAClD;YAED,WAAW,CAAC,OAAO,GAAG,wCAAwC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;YAE7E,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;SAChC;QAED,OAAO,cAAc,CAAA;IACtB,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,CAAC,qCAAqC,CAAC,CAAA;IAEjE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,cAAwB,EAAE,iBAA2B;IACpG,MAAM,oBAAoB,GAAa,EAAE,CAAA;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAC/C,MAAM,cAAc,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;QAE3C,MAAM,iBAAiB,GAAG,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAA;QAClD,oBAAoB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;QAE5C,IAAI,iBAAiB,CAAC,IAAI,IAAI,MAAM,EAAE;YACrC,SAAQ;SACR;QAED,MAAM,yBAAyB,GAAG,iBAAiB,CAAC,OAAO,GAAG,iBAAiB,CAAC,SAAS,CAAA;QACzF,MAAM,sBAAsB,GAAG,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,SAAS,CAAA;QAEhF,iBAAiB,CAAC,QAAQ,GAAG,EAAE,CAAA;QAE/B,KAAK,MAAM,UAAU,IAAI,cAAc,CAAC,QAAS,EAAE;YAClD,MAAM,sCAAsC,GAC3C,CAAC,UAAU,CAAC,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,sBAAsB,CAAA;YAE3E,MAAM,oCAAoC,GACzC,CAAC,UAAU,CAAC,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,sBAAsB,CAAA;YAEzE,MAAM,0BAA0B,GAAG,iBAAiB,CAAC,SAAS,GAAG,CAAC,sCAAsC,GAAG,yBAAyB,CAAC,CAAA;YACrI,MAAM,wBAAwB,GAAG,iBAAiB,CAAC,SAAS,GAAG,CAAC,oCAAoC,GAAG,yBAAyB,CAAC,CAAA;YAEjI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC/B,GAAG,UAAU;gBAEb,SAAS,EAAE,0BAA0B;gBACrC,OAAO,EAAE,wBAAwB;aACjC,CAAC,CAAA;SACF;KACD;IAED,OAAO,oBAAoB,CAAA;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,cAAwB,EAAE,kBAA4B,EAAE,iBAA2B,EAAE,iBAA2B;IACzJ,MAAM,WAAW,GAAgB,wBAAwB,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC,CAAA;IAEzF,MAAM,eAAe,GAAG,CAAC,GAAG,WAAW,CAAC,WAAY,CAAA;IAEpD,MAAM,cAAc,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,WAAW,CAAC,CAAA;IACzE,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;IAEnE,MAAM,mBAAmB,GAAa,EAAE,CAAA;IAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAClD,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;QAE/C,MAAM,gBAAgB,GAAG,EAAE,GAAG,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAA;QACrD,mBAAmB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAA;QAE1C,IAAI,gBAAgB,CAAC,IAAI,IAAI,MAAM,EAAE;YACpC,SAAQ;SACR;QAED,MAAM,4BAA4B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QAC/F,IAAI,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAEzF,8CAA8C;QAC9C,IAAI,0BAA0B,IAAI,4BAA4B,EAAE;YAC/D,0BAA0B,GAAG,0BAA0B,GAAG,CAAC,CAAA;SAC3D;QAED,MAAM,kBAAkB,GAAG,cAAc,CAAC,KAAK,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAA;QAEzG,MAAM,0BAA0B,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,SAAS,GAAG,eAAe,CAAC,CAAA;QAC3F,IAAI,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,GAAG,eAAe,CAAC,CAAA;QAErF,8CAA8C;QAC9C,IAAI,wBAAwB,IAAI,0BAA0B,EAAE;YAC3D,wBAAwB,GAAG,0BAA0B,GAAG,CAAC,CAAA;SACzD;QAED,MAAM,eAAe,GAAG,WAAW,CAAC,KAAK,CAAC,0BAA0B,EAAE,wBAAwB,CAAC,CAAA;QAE/F,mBAAmB;QACnB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,kBAAkB,EAAE,eAAe,EAAE,EAAE,CAAC,CAAA;QAC5E,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,CAAA;QAE1C,uDAAuD;QACvD,gBAAgB,CAAC,QAAQ,GAAG,EAAE,CAAA;QAE9B,KAAK,MAAM,mBAAmB,IAAI,kBAAkB,CAAC,QAAS,EAAE;YAC/D,MAAM,8BAA8B,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,SAAS,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC,CAAA;YACnI,MAAM,4BAA4B,GAAG,0BAA0B,CAAC,8BAA8B,EAAE,aAAa,CAAC,CAAA;YAC9G,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,SAAS,GAAG,CAAC,4BAA4B,GAAG,eAAe,CAAC,CAAA;YAE3G,MAAM,4BAA4B,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC,CAAA;YAC/H,MAAM,0BAA0B,GAAG,0BAA0B,CAAC,4BAA4B,EAAE,aAAa,CAAC,CAAA;YAC1G,MAAM,mBAAmB,GAAG,gBAAgB,CAAC,SAAS,GAAG,CAAC,0BAA0B,GAAG,eAAe,CAAC,CAAA;YAEvG,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC9B,GAAG,mBAAmB;gBACtB,SAAS,EAAE,qBAAqB;gBAChC,OAAO,EAAE,mBAAmB;aAC5B,CAAC,CAAA;SACF;KACD;IAED,OAAO,mBAAmB,CAAA;AAC3B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,IAAY,EAAE,QAAgB,EAAE,KAAa,EAAE,gBAAgB,GAAG,IAAI,EAAE,IAAa,EAAE,KAAc,EAAE,UAAmB;IACnL,MAAM,cAAc,GAAG,IAAI,MAAM,EAAE,CAAA;IAEnC,cAAc,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAA;IAE7C,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IAEpD,cAAc,CAAC,GAAG,EAAE,CAAA;IAEpB,OAAO,+CAA+C,CAAC,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAA;AACpH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+CAA+C,CAAC,SAAmB,EAAE,KAAa,EAAE,gBAAgB,GAAG,IAAI,EAAE,IAAa,EAAE,KAAc,EAAE,UAAmB;IACpL,MAAM,cAAc,GAAG,IAAI,MAAM,EAAE,CAAA;IAEnC,cAAc,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;IAExD,cAAc,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAE9D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAA;IAE5G,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAS,CAAC,CAAA;IAErE,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,QAAQ,EAAE;QACxC,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,QAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,QAAS,CAAC,CAAA;KACpF;IAED,cAAc,CAAC,GAAG,EAAE,CAAA;IAEpB,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAAC,IAAmB;IACvC,MAAM,aAAa,GAAkB,EAAE,CAAA;IAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEzB,IAAI,aAAa,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,EAAE;YAC7C,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;SACnE;aAAM;YACN,aAAa,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAA;SAC7D;KACD;IAED,OAAO,aAAa,CAAA;AACrB,CAAC;AAED,SAAS,0BAA0B,CAAC,mBAA2B,EAAE,aAA4B,EAAE,cAAgC,OAAO;IACrI,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,EAAE;QAC9B,OAAO,CAAC,CAAA;KACR;IAED,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC,EAAE,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IAE5E,MAAM,kBAAkB,GAAG,aAAa,CAAC,mBAAmB,CAAC,CAAA;IAE7D,IAAI,gBAAwB,CAAA;IAE5B,IAAI,WAAW,IAAI,OAAO,EAAE;QAC3B,gBAAgB,GAAG,kBAAkB,CAAC,KAAK,CAAA;KAC3C;SAAM;QACN,gBAAgB,GAAG,kBAAkB,CAAC,IAAI,CAAA;KAC1C;IAED,OAAO,gBAAgB,CAAA;AACxB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/// <reference path="../../src/typings/Fillers.d.ts" />
|
|
2
|
+
export * from "./Synthesis.js";
|
|
3
|
+
export * from "./Recognition.js";
|
|
4
|
+
export * from "./Alignment.js";
|
|
5
|
+
export * from "./Translation.js";
|
|
6
|
+
export * from "./LanguageDetection.js";
|
|
7
|
+
export * from "./Vad.js";
|
|
8
|
+
export * from "./Denoising.js";
|
package/dist/api/API.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/// <reference path="../typings/Fillers.d.ts" />
|
|
2
|
+
//export * from "./Globals.js"
|
|
3
|
+
export * from "./Synthesis.js";
|
|
4
|
+
export * from "./Recognition.js";
|
|
5
|
+
export * from "./Alignment.js";
|
|
6
|
+
export * from "./Translation.js";
|
|
7
|
+
export * from "./LanguageDetection.js";
|
|
8
|
+
export * from "./Vad.js";
|
|
9
|
+
export * from "./Denoising.js";
|
|
10
|
+
//# sourceMappingURL=API.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"API.js","sourceRoot":"","sources":["../../src/api/API.ts"],"names":[],"mappings":"AAAA,gDAAgD;AAEhD,8BAA8B;AAC9B,cAAc,gBAAgB,CAAA;AAC9B,cAAc,kBAAkB,CAAA;AAChC,cAAc,gBAAgB,CAAA;AAC9B,cAAc,kBAAkB,CAAA;AAChC,cAAc,wBAAwB,CAAA;AACtC,cAAc,UAAU,CAAA;AACxB,cAAc,gBAAgB,CAAA"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import * as API from "./API.js";
|
|
2
|
+
export type APIOptions = {
|
|
3
|
+
VoiceListRequestOptions: API.VoiceListRequestOptions;
|
|
4
|
+
SynthesisOptions: API.SynthesisOptions;
|
|
5
|
+
RecognitionOptions: API.RecognitionOptions;
|
|
6
|
+
AlignmentOptions: API.AlignmentOptions;
|
|
7
|
+
SpeechTranslationOptions: API.SpeechTranslationOptions;
|
|
8
|
+
SpeechLanguageDetectionOptions: API.SpeechLanguageDetectionOptions;
|
|
9
|
+
TextLanguageDetectionOptions: API.TextLanguageDetectionOptions;
|
|
10
|
+
VADOptions: API.VADOptions;
|
|
11
|
+
DenoisingOptions: API.DenoisingOptions;
|
|
12
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"APIOptions.js","sourceRoot":"","sources":["../../src/api/APIOptions.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { RawAudio } from "../audio/AudioUtilities.js";
|
|
2
|
+
import * as API from "./API.js";
|
|
3
|
+
import { Timeline } from "../utilities/Timeline.js";
|
|
4
|
+
import { WhisperModelName } from "../recognition/WhisperSTT.js";
|
|
5
|
+
export declare function alignFile(audioFilename: string, text: string, options: AlignmentOptions): Promise<{
|
|
6
|
+
wordTimeline: Timeline;
|
|
7
|
+
rawAudio: RawAudio;
|
|
8
|
+
transcript: string;
|
|
9
|
+
language: string;
|
|
10
|
+
}>;
|
|
11
|
+
export declare function alignSegments(sourceRawAudio: RawAudio, segmentTimeline: Timeline, alignmentOptions: AlignmentOptions): Promise<Timeline>;
|
|
12
|
+
export declare function align(inputRawAudio: RawAudio, transcript: string, options: AlignmentOptions): Promise<{
|
|
13
|
+
wordTimeline: Timeline;
|
|
14
|
+
rawAudio: RawAudio;
|
|
15
|
+
transcript: string;
|
|
16
|
+
language: string;
|
|
17
|
+
}>;
|
|
18
|
+
export type AlignmentEngine = "dtw" | "dtw-ra" | "whisper";
|
|
19
|
+
export type PhoneAlignmentMethod = "interpolation" | "dtw";
|
|
20
|
+
export interface AlignmentOptions {
|
|
21
|
+
engine?: AlignmentEngine;
|
|
22
|
+
language?: string;
|
|
23
|
+
dtw?: {
|
|
24
|
+
windowDuration?: number;
|
|
25
|
+
recognition?: API.RecognitionOptions;
|
|
26
|
+
phoneAlignmentMethod?: PhoneAlignmentMethod;
|
|
27
|
+
};
|
|
28
|
+
whisper?: {
|
|
29
|
+
model?: WhisperModelName;
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export declare const defaultAlignmentOptions: AlignmentOptions;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { extendDeep } from "../utilities/ObjectUtilities.js";
|
|
2
|
+
import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js";
|
|
3
|
+
import { logToStderr } from "../utilities/Utilities.js";
|
|
4
|
+
import { downmixToMonoAndNormalize, getRawAudioDuration } from "../audio/AudioUtilities.js";
|
|
5
|
+
import { Logger } from "../utilities/Logger.js";
|
|
6
|
+
import { resampleAudioSpeex } from "../dsp/SpeexResampler.js";
|
|
7
|
+
import * as API from "./API.js";
|
|
8
|
+
import { addTimeOffsetToTimeline } from "../utilities/Timeline.js";
|
|
9
|
+
import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from "../utilities/Locale.js";
|
|
10
|
+
const log = logToStderr;
|
|
11
|
+
export async function alignFile(audioFilename, text, options) {
|
|
12
|
+
const rawAudio = await FFMpegTranscoder.decodeToChannels(audioFilename);
|
|
13
|
+
return align(rawAudio, text, options);
|
|
14
|
+
}
|
|
15
|
+
export async function alignSegments(sourceRawAudio, segmentTimeline, alignmentOptions) {
|
|
16
|
+
const timeline = [];
|
|
17
|
+
for (const segmentEntry of segmentTimeline) {
|
|
18
|
+
const segmentText = segmentEntry.text;
|
|
19
|
+
const segmentStartTime = segmentEntry.startTime;
|
|
20
|
+
const segmentEndTime = segmentEntry.endTime;
|
|
21
|
+
const segmentStartSampleIndex = Math.floor(segmentStartTime * sourceRawAudio.sampleRate);
|
|
22
|
+
const segmentEndSampleIndex = Math.floor(segmentEndTime * sourceRawAudio.sampleRate);
|
|
23
|
+
const segmentAudioSamples = sourceRawAudio.audioChannels[0].slice(segmentStartSampleIndex, segmentEndSampleIndex);
|
|
24
|
+
const segmentRawAudio = {
|
|
25
|
+
audioChannels: [segmentAudioSamples],
|
|
26
|
+
sampleRate: sourceRawAudio.sampleRate
|
|
27
|
+
};
|
|
28
|
+
const { wordTimeline: mappedTimeline } = await align(segmentRawAudio, segmentText, alignmentOptions);
|
|
29
|
+
const segmentTimelineWithOffset = addTimeOffsetToTimeline(mappedTimeline, segmentStartTime);
|
|
30
|
+
timeline.push(...segmentTimelineWithOffset);
|
|
31
|
+
}
|
|
32
|
+
return timeline;
|
|
33
|
+
}
|
|
34
|
+
export async function align(inputRawAudio, transcript, options) {
|
|
35
|
+
const logger = new Logger();
|
|
36
|
+
const startTimestamp = logger.getTimestamp();
|
|
37
|
+
logger.start("Prepare for alignment");
|
|
38
|
+
options = extendDeep(defaultAlignmentOptions, options);
|
|
39
|
+
let sourceRawAudio = downmixToMonoAndNormalize(inputRawAudio);
|
|
40
|
+
sourceRawAudio = await resampleAudioSpeex(sourceRawAudio, 16000);
|
|
41
|
+
let language;
|
|
42
|
+
if (options.language) {
|
|
43
|
+
language = normalizeLanguageCode(options.language);
|
|
44
|
+
if (language == 'en') {
|
|
45
|
+
language = 'en-us';
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
logger.start("No language specified. Detecting language");
|
|
50
|
+
const { detectedLanguage } = await API.detectTextLanguage(transcript, {});
|
|
51
|
+
language = detectedLanguage;
|
|
52
|
+
logger.end();
|
|
53
|
+
logger.log(`Language detected: ${formatLanguageCodeWithName(detectedLanguage)}`);
|
|
54
|
+
}
|
|
55
|
+
logger.start("Get espeak voice list and select best matching voice");
|
|
56
|
+
const { bestMatchingVoice } = await API.requestVoiceList({ engine: "espeak", language });
|
|
57
|
+
if (!bestMatchingVoice) {
|
|
58
|
+
throw new Error("No matching voice found");
|
|
59
|
+
}
|
|
60
|
+
const espeakVoice = bestMatchingVoice.name;
|
|
61
|
+
const dtwWindowDuration = options.dtw.windowDuration;
|
|
62
|
+
logger.start("Load alignment module");
|
|
63
|
+
const { alignUsingDtwWithRecognition: alignUsingDtwWithRecognitionAssist, createAlignmentReferenceUsingEspeak, alignUsingDtw } = await import("../alignment/SpeechAlignment.js");
|
|
64
|
+
async function getAlignmentReference() {
|
|
65
|
+
logger.start("Create alignment reference with espeak");
|
|
66
|
+
let { rawAudio: referenceRawAudio, timeline: referenceTimeline } = await createAlignmentReferenceUsingEspeak(transcript, language, espeakVoice, true);
|
|
67
|
+
referenceRawAudio = await resampleAudioSpeex(referenceRawAudio, 16000);
|
|
68
|
+
referenceRawAudio = downmixToMonoAndNormalize(referenceRawAudio);
|
|
69
|
+
return { referenceRawAudio, referenceTimeline };
|
|
70
|
+
}
|
|
71
|
+
let mappedTimeline;
|
|
72
|
+
switch (options.engine) {
|
|
73
|
+
case "dtw": {
|
|
74
|
+
const { referenceRawAudio, referenceTimeline } = await getAlignmentReference();
|
|
75
|
+
logger.end();
|
|
76
|
+
mappedTimeline = await alignUsingDtw(sourceRawAudio, referenceRawAudio, referenceTimeline, dtwWindowDuration);
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
case "dtw-ra": {
|
|
80
|
+
const recognitionOptionsDefaults = {
|
|
81
|
+
engine: "whisper",
|
|
82
|
+
language,
|
|
83
|
+
};
|
|
84
|
+
const engineOptions = options.dtw;
|
|
85
|
+
const recognitionOptions = extendDeep(recognitionOptionsDefaults, engineOptions.recognition);
|
|
86
|
+
logger.end();
|
|
87
|
+
const { timeline: recognitionTimeline } = await API.recognize(sourceRawAudio, recognitionOptions);
|
|
88
|
+
const { referenceRawAudio, referenceTimeline } = await getAlignmentReference();
|
|
89
|
+
logger.end();
|
|
90
|
+
const phoneAlignmentMethod = options.dtw.phoneAlignmentMethod;
|
|
91
|
+
mappedTimeline = await alignUsingDtwWithRecognitionAssist(sourceRawAudio, referenceRawAudio, referenceTimeline, recognitionTimeline, espeakVoice, phoneAlignmentMethod, dtwWindowDuration);
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
case "whisper": {
|
|
95
|
+
const WhisperSTT = await import("../recognition/WhisperSTT.js");
|
|
96
|
+
const whisperOptions = options.whisper;
|
|
97
|
+
const shortLanguageCode = getShortLanguageCode(language);
|
|
98
|
+
const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, language);
|
|
99
|
+
if (modelName.endsWith(".en") && shortLanguageCode != "en") {
|
|
100
|
+
throw new Error(`The model '${modelName}' is English only and cannot transcribe language '${shortLanguageCode}'`);
|
|
101
|
+
}
|
|
102
|
+
if (getRawAudioDuration(sourceRawAudio) > 30) {
|
|
103
|
+
throw new Error("Whisper based alignment currently only supports audio inputs that are 30s or less");
|
|
104
|
+
}
|
|
105
|
+
logger.end();
|
|
106
|
+
mappedTimeline = await WhisperSTT.align(sourceRawAudio, transcript, modelName, modelDir, tokenizerDir, shortLanguageCode);
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
default: {
|
|
110
|
+
throw new Error(`Engine '${options.engine}' is not supported`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
logger.end();
|
|
114
|
+
logger.logDuration(`Total alignment time`, startTimestamp);
|
|
115
|
+
return {
|
|
116
|
+
wordTimeline: mappedTimeline,
|
|
117
|
+
rawAudio: inputRawAudio,
|
|
118
|
+
transcript,
|
|
119
|
+
language
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
export const defaultAlignmentOptions = {
|
|
123
|
+
engine: "dtw",
|
|
124
|
+
language: undefined,
|
|
125
|
+
dtw: {
|
|
126
|
+
windowDuration: 120,
|
|
127
|
+
recognition: undefined,
|
|
128
|
+
phoneAlignmentMethod: "dtw",
|
|
129
|
+
},
|
|
130
|
+
whisper: {
|
|
131
|
+
model: undefined
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
//# sourceMappingURL=Alignment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Alignment.js","sourceRoot":"","sources":["../../src/api/Alignment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAY,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACrG,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAE7D,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,EAAY,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AAC5E,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAGhH,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,aAAqB,EAAE,IAAY,EAAE,OAAyB;IAC7F,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAA;IAEvE,OAAO,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,cAAwB,EAAE,eAAyB,EAAE,gBAAkC;IAC1H,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,KAAK,MAAM,YAAY,IAAI,eAAe,EAAE;QAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAA;QAErC,MAAM,gBAAgB,GAAG,YAAY,CAAC,SAAS,CAAA;QAC/C,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAA;QAE3C,MAAM,uBAAuB,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;QACxF,MAAM,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;QAEpF,MAAM,mBAAmB,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,uBAAuB,EAAE,qBAAqB,CAAC,CAAA;QACjH,MAAM,eAAe,GAAa;YACjC,aAAa,EAAE,CAAC,mBAAmB,CAAC;YACpC,UAAU,EAAE,cAAc,CAAC,UAAU;SACrC,CAAA;QAED,MAAM,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,MAAM,KAAK,CAAC,eAAe,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAA;QAEpG,MAAM,yBAAyB,GAAG,uBAAuB,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAA;QAE3F,QAAQ,CAAC,IAAI,CAAC,GAAG,yBAAyB,CAAC,CAAA;KAC3C;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,aAAuB,EAAE,UAAkB,EAAE,OAAyB;IACjG,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IAC5C,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAA;IAErC,OAAO,GAAG,UAAU,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAA;IAEtD,IAAI,cAAc,GAAG,yBAAyB,CAAC,aAAa,CAAC,CAAA;IAC7D,cAAc,GAAG,MAAM,kBAAkB,CAAC,cAAc,EAAE,KAAK,CAAC,CAAA;IAEhE,IAAI,QAAgB,CAAA;IAEpB,IAAI,OAAO,CAAC,QAAQ,EAAE;QACrB,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAS,CAAC,CAAA;QAEnD,IAAI,QAAQ,IAAI,IAAI,EAAE;YACrB,QAAQ,GAAG,OAAO,CAAA;SAClB;KACD;SAAM;QACN,MAAM,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;QACzD,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,GAAG,CAAC,kBAAkB,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;QACzE,QAAQ,GAAG,gBAAgB,CAAA;QAE3B,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,GAAG,CAAC,sBAAsB,0BAA0B,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAA;KAChF;IAED,MAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAA;IACpE,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,GAAG,CAAC,gBAAgB,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAExF,IAAI,CAAC,iBAAiB,EAAE;QACvB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;KAC1C;IAED,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAA;IAC1C,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAI,CAAC,cAAe,CAAA;IAEtD,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAA;IAErC,MAAM,EAAE,4BAA4B,EAAE,kCAAkC,EAAE,mCAAmC,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAA;IAEhL,KAAK,UAAU,qBAAqB;QACnC,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;QACtD,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,MAAM,mCAAmC,CAAC,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,CAAC,CAAA;QAErJ,iBAAiB,GAAG,MAAM,kBAAkB,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAA;QACtE,iBAAiB,GAAG,yBAAyB,CAAC,iBAAiB,CAAC,CAAA;QAEhE,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,CAAA;IAChD,CAAC;IAED,IAAI,cAAwB,CAAA;IAE5B,QAAQ,OAAO,CAAC,MAAM,EAAE;QACvB,KAAK,KAAK,CAAC,CAAC;YACX,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,MAAM,qBAAqB,EAAE,CAAA;YAC9E,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,cAAc,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,iBAAiB,CAAC,CAAA;YAE7G,MAAK;SACL;QAED,KAAK,QAAQ,CAAC,CAAC;YACd,MAAM,0BAA0B,GAA2B;gBAC1D,MAAM,EAAE,SAAS;gBACjB,QAAQ;aACR,CAAA;YAED,MAAM,aAAa,GAAG,OAAO,CAAC,GAAI,CAAA;YAElC,MAAM,kBAAkB,GAA2B,UAAU,CAAC,0BAA0B,EAAE,aAAa,CAAC,WAAW,CAAC,CAAA;YAEpH,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,QAAQ,EAAE,mBAAmB,EAAE,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAA;YAEjG,MAAM,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,GAAG,MAAM,qBAAqB,EAAE,CAAA;YAE9E,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,oBAAoB,GAAG,OAAO,CAAC,GAAI,CAAC,oBAAqB,CAAA;YAE/D,cAAc,GAAG,MAAM,kCAAkC,CAAC,cAAc,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,WAAW,EAAE,oBAAoB,EAAE,iBAAiB,CAAC,CAAA;YAE1L,MAAK;SACL;QAED,KAAK,SAAS,CAAC,CAAC;YACf,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;YAExD,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;YAEtH,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,iBAAiB,IAAI,IAAI,EAAE;gBAC3D,MAAM,IAAI,KAAK,CAAC,cAAc,SAAS,qDAAqD,iBAAiB,GAAG,CAAC,CAAA;aACjH;YAED,IAAI,mBAAmB,CAAC,cAAc,CAAC,GAAG,EAAE,EAAE;gBAC7C,MAAM,IAAI,KAAK,CAAC,mFAAmF,CAAC,CAAA;aACpG;YAED,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,cAAc,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAA;YAEzH,MAAK;SACL;QAED,OAAO,CAAC,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;SAC9D;KACD;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IACZ,MAAM,CAAC,WAAW,CAAC,sBAAsB,EAAE,cAAc,CAAC,CAAA;IAE1D,OAAO;QACN,YAAY,EAAE,cAAc;QAC5B,QAAQ,EAAE,aAAa;QACvB,UAAU;QACV,QAAQ;KACR,CAAA;AACF,CAAC;AAqBD,MAAM,CAAC,MAAM,uBAAuB,GAAqB;IACxD,MAAM,EAAE,KAAK;IAEb,QAAQ,EAAE,SAAS;IAEnB,GAAG,EAAE;QACJ,cAAc,EAAE,GAAG;QACnB,WAAW,EAAE,SAAS;QACtB,oBAAoB,EAAE,KAAK;KAC3B;IAED,OAAO,EAAE;QACR,KAAK,EAAE,SAAS;KAChB;CACD,CAAA"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { RawAudio } from "../audio/AudioUtilities.js";
|
|
2
|
+
export declare function denoiseFile(filename: string, options: DenoisingOptions): Promise<RawAudio>;
|
|
3
|
+
export declare function denoise(rawAudio: RawAudio, options: DenoisingOptions): Promise<RawAudio>;
|
|
4
|
+
export type DenoisingMethod = "rnnoise";
|
|
5
|
+
export interface DenoisingOptions {
|
|
6
|
+
method?: DenoisingMethod;
|
|
7
|
+
postProcessing?: {
|
|
8
|
+
normalizeAudio: boolean;
|
|
9
|
+
targetPeakDb: number;
|
|
10
|
+
maxIncreaseDb: number;
|
|
11
|
+
dryMixGainDb?: number;
|
|
12
|
+
};
|
|
13
|
+
}
|
|
14
|
+
export declare const defaultDenoisingOptions: DenoisingOptions;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { extendDeep } from "../utilities/ObjectUtilities.js";
|
|
2
|
+
import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js";
|
|
3
|
+
import { applyGainDecibels, getAudioPeakDecibels, mixAudio, normalizeAudioLevel } from "../audio/AudioUtilities.js";
|
|
4
|
+
import { Logger } from "../utilities/Logger.js";
|
|
5
|
+
import { logToStderr } from "../utilities/Utilities.js";
|
|
6
|
+
import { resampleAudioSpeex } from "../dsp/SpeexResampler.js";
|
|
7
|
+
const log = logToStderr;
|
|
8
|
+
export async function denoiseFile(filename, options) {
|
|
9
|
+
const rawAudio = await FFMpegTranscoder.decodeToChannels(filename);
|
|
10
|
+
return denoise(rawAudio, options);
|
|
11
|
+
}
|
|
12
|
+
export async function denoise(rawAudio, options) {
|
|
13
|
+
const logger = new Logger();
|
|
14
|
+
const startTime = logger.getTimestamp();
|
|
15
|
+
logger.start("Prepare for denoising");
|
|
16
|
+
options = extendDeep(defaultDenoisingOptions, options);
|
|
17
|
+
const processingSampleRate = 48000;
|
|
18
|
+
logger.start(`Resample audio to ${processingSampleRate} Hz`);
|
|
19
|
+
const resampledRawAudio = await resampleAudioSpeex(rawAudio, processingSampleRate, 3);
|
|
20
|
+
logger.start(`Initialize ${options.method} module`);
|
|
21
|
+
let denoisedAudio;
|
|
22
|
+
switch (options.method) {
|
|
23
|
+
case "rnnoise": {
|
|
24
|
+
const RNNoise = await import("../denoising/RNNoise.js");
|
|
25
|
+
logger.end();
|
|
26
|
+
const denoisedAudioChannels = [];
|
|
27
|
+
for (const audioChannel of resampledRawAudio.audioChannels) {
|
|
28
|
+
const audioChannelRawAudio = { audioChannels: [audioChannel], sampleRate: processingSampleRate };
|
|
29
|
+
const { denoisedRawAudio, frameVadProbabilities } = await RNNoise.denoiseAudio(audioChannelRawAudio);
|
|
30
|
+
denoisedAudioChannels.push(denoisedRawAudio.audioChannels[0]);
|
|
31
|
+
}
|
|
32
|
+
denoisedAudio = { audioChannels: denoisedAudioChannels, sampleRate: processingSampleRate };
|
|
33
|
+
break;
|
|
34
|
+
}
|
|
35
|
+
default: {
|
|
36
|
+
throw new Error(`Method: '${options.method}' is not supported`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
logger.start("Postprocess audio");
|
|
40
|
+
const shouldNormalize = options.postProcessing.normalizeAudio;
|
|
41
|
+
const targetPeakDb = options.postProcessing.targetPeakDb;
|
|
42
|
+
const maxIncreaseDb = options.postProcessing.maxIncreaseDb;
|
|
43
|
+
const dryMixGainDb = options.postProcessing.dryMixGainDb;
|
|
44
|
+
const preMixPeakDb = getAudioPeakDecibels(denoisedAudio.audioChannels);
|
|
45
|
+
denoisedAudio = mixAudio(denoisedAudio, applyGainDecibels(resampledRawAudio, dryMixGainDb));
|
|
46
|
+
const postMixPeakDb = getAudioPeakDecibels(denoisedAudio.audioChannels);
|
|
47
|
+
if (shouldNormalize) {
|
|
48
|
+
denoisedAudio = normalizeAudioLevel(denoisedAudio, targetPeakDb, maxIncreaseDb);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
denoisedAudio = applyGainDecibels(denoisedAudio, preMixPeakDb - postMixPeakDb);
|
|
52
|
+
}
|
|
53
|
+
logger.end();
|
|
54
|
+
logger.logDuration("Total denoising time", startTime);
|
|
55
|
+
return denoisedAudio;
|
|
56
|
+
}
|
|
57
|
+
export const defaultDenoisingOptions = {
|
|
58
|
+
method: "rnnoise",
|
|
59
|
+
postProcessing: {
|
|
60
|
+
normalizeAudio: false,
|
|
61
|
+
targetPeakDb: -3,
|
|
62
|
+
maxIncreaseDb: 30,
|
|
63
|
+
dryMixGainDb: -20,
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
//# sourceMappingURL=Denoising.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Denoising.js","sourceRoot":"","sources":["../../src/api/Denoising.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AAGjE,OAAO,EAAY,iBAAiB,EAA6B,oBAAoB,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AACxJ,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAE7D,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,OAAyB;IAC5E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IAElE,OAAO,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAkB,EAAE,OAAyB;IAC1E,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAC3B,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IACvC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAA;IAErC,OAAO,GAAG,UAAU,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAA;IAEtD,MAAM,oBAAoB,GAAG,KAAK,CAAA;IAElC,MAAM,CAAC,KAAK,CAAC,qBAAqB,oBAAoB,KAAK,CAAC,CAAA;IAC5D,MAAM,iBAAiB,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,CAAC,CAAC,CAAA;IAErF,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,CAAC,MAAM,SAAS,CAAC,CAAA;IAEnD,IAAI,aAAuB,CAAA;IAE3B,QAAQ,OAAO,CAAC,MAAM,EAAE;QACvB,KAAK,SAAS,CAAC,CAAC;YACf,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAA;YACvD,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,qBAAqB,GAAmB,EAAE,CAAA;YAEhD,KAAK,MAAM,YAAY,IAAI,iBAAiB,CAAC,aAAa,EAAE;gBAC3D,MAAM,oBAAoB,GAAa,EAAE,aAAa,EAAE,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAA;gBAE1G,MAAM,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,oBAAoB,CAAC,CAAA;gBACpG,qBAAqB,CAAC,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;aAC7D;YAED,aAAa,GAAG,EAAE,aAAa,EAAE,qBAAqB,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAA;YAE1F,MAAK;SACL;QAED,OAAO,CAAC,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;SAC/D;KACD;IAED,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAA;IAEjC,MAAM,eAAe,GAAG,OAAO,CAAC,cAAe,CAAC,cAAe,CAAA;IAC/D,MAAM,YAAY,GAAG,OAAO,CAAC,cAAe,CAAC,YAAa,CAAA;IAC1D,MAAM,aAAa,GAAG,OAAO,CAAC,cAAe,CAAC,aAAc,CAAA;IAC5D,MAAM,YAAY,GAAG,OAAO,CAAC,cAAe,CAAC,YAAa,CAAA;IAE1D,MAAM,YAAY,GAAG,oBAAoB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;IACtE,aAAa,GAAG,QAAQ,CAAC,aAAa,EAAE,iBAAiB,CAAC,iBAAiB,EAAE,YAAY,CAAC,CAAC,CAAA;IAC3F,MAAM,aAAa,GAAG,oBAAoB,CAAC,aAAa,CAAC,aAAa,CAAC,CAAA;IAEvE,IAAI,eAAe,EAAE;QACpB,aAAa,GAAG,mBAAmB,CAAC,aAAa,EAAE,YAAY,EAAE,aAAa,CAAC,CAAA;KAC/E;SAAM;QACN,aAAa,GAAG,iBAAiB,CAAC,aAAa,EAAE,YAAY,GAAG,aAAa,CAAC,CAAA;KAC9E;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,MAAM,CAAC,WAAW,CAAC,sBAAsB,EAAE,SAAS,CAAC,CAAA;IAErD,OAAO,aAAa,CAAA;AACrB,CAAC;AAeD,MAAM,CAAC,MAAM,uBAAuB,GAAqB;IACxD,MAAM,EAAE,SAAS;IAEjB,cAAc,EAAE;QACf,cAAc,EAAE,KAAK;QACrB,YAAY,EAAE,CAAC,CAAC;QAChB,aAAa,EAAE,EAAE;QACjB,YAAY,EAAE,CAAC,EAAE;KACjB;CACD,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const appName = "echogarden";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Globals.js","sourceRoot":"","sources":["../../src/api/Globals.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,YAAY,CAAA"}
|