echogarden 1.4.3 → 1.5.0

package/dist/cli/CLI.js

@@ -5,7 +5,7 @@ import { getOptionTypeFromSchema } from './CLIOptionsSchema.js';
 import { parseConfigFile, parseJSONConfigFile } from './CLIConfigFile.js';
 import chalk from 'chalk';
 import { applyGainDecibels, encodeRawAudioToWave, getEmptyRawAudio, getRawAudioDuration, normalizeAudioLevel, sliceRawAudioByTime } from '../audio/AudioUtilities.js';
-import { subtitlesToText, subtitlesToTimeline, timelineToSubtitles } from '../subtitles/Subtitles.js';
+import { subtitlesToText, timelineToSubtitles } from '../subtitles/Subtitles.js';
 import { Logger, resetActiveLogger } from '../utilities/Logger.js';
 import { isMainThread, parentPort } from 'node:worker_threads';
 import { encodeFromChannels, getDefaultFFMpegOptionsForSpeech } from '../codecs/FFMpegTranscoder.js';
@@ -170,17 +170,21 @@ const help = [
     `${executableName} ${chalk.magentaBright('speak-url')} url [output files...] [options...]`,
     `    Speak the HTML document on the given URL\n`,
     `${executableName} ${chalk.magentaBright('speak-wikipedia')} articleName [output files...] [options...]`,
-    `    Speak the given wikipedia article, language edition can be specified by --language=<langCode>\n`,
+    `    Speak the given Wikipedia article. Language edition can be specified by --language=<langCode>\n`,
     `${executableName} ${chalk.magentaBright('transcribe')} audioFile [output files...] [options...]`,
-    `    Transcribe audio file\n`,
-    `${executableName} ${chalk.magentaBright('align')} audioFile referenceFile [output files...] [options...]`,
-    `    Align audio file to the reference transcript file\n`,
-    `${executableName} ${chalk.magentaBright('translate-speech')} inputFile [output files...] [options...]`,
+    `    Transcribe a spoken audio file\n`,
+    `${executableName} ${chalk.magentaBright('align')} audioFile transcriptFile [output files...] [options...]`,
+    `    Align spoken audio file to its transcript\n`,
+    `${executableName} ${chalk.magentaBright('translate-speech')} audioFile [output files...] [options...]`,
     `    Transcribe audio file directly to a different language\n`,
-    `${executableName} ${chalk.magentaBright('align-translation')} audioFile referenceFile [output files...] [options...]`,
-    `    Align audio file to the reference translated transcript file\n`,
+    `${executableName} ${chalk.magentaBright('align-translation')} audioFile translatedTranscriptFile [output files...] [options...]`,
+    `    Align spoken audio file to its translated transcript\n`,
+    `${executableName} ${chalk.magentaBright('align-transcript-and-translation')} audioFile transcriptFile translatedTranscriptFile [output files...] [options...]`,
+    `    Align spoken audio file to both its transcript and its translated transcript using a two-stage approach.\n`,
+    `${executableName} ${chalk.magentaBright('align-timeline-translation')} timelineFile translatedFile [output files...] [options...]`,
+    `    Align a given timeline file to its translated text\n`,
     `${executableName} ${chalk.magentaBright('detect-speech-language')} audioFile [output files...] [options...]`,
-    `    Detect language of audio file\n`,
+    `    Detect language of spoken audio file\n`,
     `${executableName} ${chalk.magentaBright('detect-text-language')} inputFile [output files...] [options...]`,
     `    Detect language of textual file\n`,
     `${executableName} ${chalk.magentaBright('detect-voice-activity')} audioFile [output files...] [options...]`,
@@ -229,6 +233,14 @@ async function startWithArgs(operationData) {
             await alignTranslation(operationData);
             break;
         }
+        case 'align-transcript-and-translation': {
+            await alignTranscriptAndTranslation(operationData);
+            break;
+        }
+        case 'align-timeline-translation': {
+            await alignTimelineTranslation(operationData);
+            break;
+        }
         case 'detect-language': {
             await detectLanguage(operationData, 'auto');
             break;
@@ -283,7 +295,7 @@ async function startWithArgs(operationData) {
         }
     }
 }
-async function speak(operationData) {
+export async function speak(operationData) {
     const logger = new Logger();
     const { operationArgs, operation, operationOptionsLookup, cliOptions } = operationData;
     const mainArg = operationArgs[0];
@@ -345,8 +357,8 @@ async function speak(operationData) {
         }
         else if (sourceFileExtension == 'srt' || sourceFileExtension == 'vtt') {
             const fileContent = await readFile(sourceFile, { encoding: 'utf-8' });
-            textSegments = subtitlesToTimeline(fileContent).map(entry => entry.text);
-            //textSegments = [subtitlesToText(fileContent)]
+            //textSegments = subtitlesToTimeline(fileContent).map(entry => entry.text)
+            textSegments = [subtitlesToText(fileContent)];
         }
         else if (sourceFileExtension == 'xml' || sourceFileExtension == 'ssml') {
             options.ssml = true;
@@ -415,7 +427,7 @@ async function speak(operationData) {
     }
     logger.end();
 }
-async function transcribe(operationData) {
+export async function transcribe(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const sourceFilename = operationArgs[0];
@@ -458,7 +470,7 @@ async function transcribe(operationData) {
         await playAudioWithWordTimeline(normalizedAudioToPlay, wordTimeline, transcript);
     }
 }
-async function align(operationData) {
+export async function align(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const audioFilename = operationArgs[0];
@@ -531,13 +543,13 @@ async function align(operationData) {
         await playAudioWithWordTimeline(normalizedAudioToPlay, wordTimeline, transcript);
     }
 }
-async function alignTranslation(operationData) {
+export async function alignTranslation(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const audioFilename = operationArgs[0];
     const outputFilenames = operationArgs.slice(2);
     if (audioFilename == undefined) {
-        throw new Error(`align-translation requires an argument containing the audio file path.`);
+        throw new Error(`align-translation requires a first argument containing the audio file path.`);
     }
     if (!existsSync(audioFilename)) {
         throw new Error(`The given source file '${audioFilename}' was not found.`);
@@ -562,7 +574,7 @@ async function alignTranslation(operationData) {
         text = subtitlesToText(fileContent);
     }
     else {
-        throw new Error(`align only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`);
+        throw new Error(`align-translation only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`);
     }
     if (cliOptions.play == null) {
         cliOptions.play = outputFilenames.length === 0;
@@ -570,7 +582,7 @@ async function alignTranslation(operationData) {
     const options = await optionsLookupToTypedObject(operationOptionsLookup, 'TranslationAlignmentOptions');
     const allowOverwrite = getWithDefault(cliOptions.overwrite, overwriteByDefault);
     const { includesPlaceholderPattern } = await checkOutputFilenames(outputFilenames, true, true, true);
-    const { timeline, wordTimeline, transcript, language, inputRawAudio, isolatedRawAudio, backgroundRawAudio } = await API.alignTranslation(audioFilename, text, options);
+    const { timeline, wordTimeline, translatedTranscript, sourceLanguage, targetLanguage, inputRawAudio, isolatedRawAudio, backgroundRawAudio } = await API.alignTranslation(audioFilename, text, options);
     if (outputFilenames.length > 0) {
         logger.start('\nWrite output files');
     }
@@ -579,9 +591,100 @@ async function alignTranslation(operationData) {
             const segmentEntry = timeline[segmentIndex];
             const segmentAudio = sliceRawAudioByTime(inputRawAudio, segmentEntry.startTime, segmentEntry.endTime);
             const sentenceTimeline = addTimeOffsetToTimeline(segmentEntry.timeline, -segmentEntry.startTime);
-            await writeOutputFilesForSegment(outputFilenames, segmentIndex, timeline.length, segmentAudio, sentenceTimeline, segmentEntry.text, language, allowOverwrite);
+            await writeOutputFilesForSegment(outputFilenames, segmentIndex, timeline.length, segmentAudio, sentenceTimeline, segmentEntry.text, targetLanguage, allowOverwrite);
         }
     }
+    for (const outputFilename of outputFilenames) {
+        const partPatternMatch = outputFilename.match(filenamePlaceholderPattern);
+        if (partPatternMatch) {
+            continue;
+        }
+        const fileSaver = getFileSaver(outputFilename, allowOverwrite);
+        await fileSaver(inputRawAudio, timeline, translatedTranscript, options.subtitles);
+        await writeSourceSeparationOutputIfNeeded(outputFilename, isolatedRawAudio, backgroundRawAudio, allowOverwrite, true);
+    }
+    logger.end();
+    if (cliOptions.play) {
+        let audioToPlay;
+        if (isolatedRawAudio) {
+            audioToPlay = isolatedRawAudio;
+        }
+        else {
+            audioToPlay = inputRawAudio;
+        }
+        const normalizedAudioToPlay = normalizeAudioLevel(audioToPlay);
+        await playAudioWithWordTimeline(normalizedAudioToPlay, wordTimeline, translatedTranscript);
+    }
+}
+export async function alignTranscriptAndTranslation(operationData) {
+    const logger = new Logger();
+    const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
+    const audioFilename = operationArgs[0];
+    const outputFilenames = operationArgs.slice(3);
+    if (audioFilename == undefined) {
+        throw new Error(`align-transcript-and-translation requires a first argument containing the audio file path.`);
+    }
+    if (!existsSync(audioFilename)) {
+        throw new Error(`The given source file '${audioFilename}' was not found.`);
+    }
+    const nativeTranscriptFilePath = operationArgs[1];
+    if (nativeTranscriptFilePath == undefined) {
+        throw new Error(`align-transcript-and-translation requires a second argument containing the native language transcript file path.`);
+    }
+    if (!existsSync(nativeTranscriptFilePath)) {
+        throw new Error(`The given transcript file '${nativeTranscriptFilePath}' was not found.`);
+    }
+    const translatedTranscriptFilePath = operationArgs[2];
+    if (translatedTranscriptFilePath == undefined) {
+        throw new Error(`align-transcript-and-translation requires a third argument containing the translated language transcript file path.`);
+    }
+    if (!existsSync(translatedTranscriptFilePath)) {
+        throw new Error(`The given translated transcript file '${nativeTranscriptFilePath}' was not found.`);
+    }
+    let transcript;
+    {
+        const nativeTranscriptFileExtension = getLowercaseFileExtension(nativeTranscriptFilePath);
+        const fileContent = await readFile(nativeTranscriptFilePath, { encoding: 'utf-8' });
+        if (nativeTranscriptFileExtension == 'txt') {
+            transcript = fileContent;
+        }
+        else if (nativeTranscriptFileExtension == 'html' || nativeTranscriptFileExtension == 'htm') {
+            transcript = await convertHtmlToText(fileContent);
+        }
+        else if (nativeTranscriptFileExtension == 'srt' || nativeTranscriptFileExtension == 'vtt') {
+            transcript = subtitlesToText(fileContent);
+        }
+        else {
+            throw new Error(`align-transcript-and-translation only supports transcript files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`);
+        }
+    }
+    let translatedTranscript;
+    {
+        const translatedTranscriptFileExtension = getLowercaseFileExtension(translatedTranscriptFilePath);
+        const fileContent = await readFile(translatedTranscriptFilePath, { encoding: 'utf-8' });
+        if (translatedTranscriptFileExtension == 'txt') {
+            translatedTranscript = fileContent;
+        }
+        else if (translatedTranscriptFileExtension == 'html' || translatedTranscriptFileExtension == 'htm') {
+            translatedTranscript = await convertHtmlToText(fileContent);
+        }
+        else if (translatedTranscriptFileExtension == 'srt' || translatedTranscriptFileExtension == 'vtt') {
+            translatedTranscript = subtitlesToText(fileContent);
+        }
+        else {
+            throw new Error(`align-transcript-and-translation only supports transcript files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`);
+        }
+    }
+    if (cliOptions.play == null) {
+        cliOptions.play = outputFilenames.length === 0;
+    }
+    const options = await optionsLookupToTypedObject(operationOptionsLookup, 'TranscriptAndTranslationAlignmentOptions');
+    const allowOverwrite = getWithDefault(cliOptions.overwrite, overwriteByDefault);
+    const { includesPlaceholderPattern } = await checkOutputFilenames(outputFilenames, true, true, true);
+    const { timeline, wordTimeline, translatedTimeline, translatedWordTimeline, sourceLanguage, targetLanguage, inputRawAudio, isolatedRawAudio, backgroundRawAudio } = await API.alignTranscriptAndTranslation(audioFilename, transcript, translatedTranscript, options);
+    if (outputFilenames.length > 0) {
+        logger.start('\nWrite output files');
+    }
     for (const outputFilename of outputFilenames) {
         const partPatternMatch = outputFilename.match(filenamePlaceholderPattern);
         if (partPatternMatch) {
@@ -590,6 +693,13 @@ async function alignTranslation(operationData) {
         const fileSaver = getFileSaver(outputFilename, allowOverwrite);
         await fileSaver(inputRawAudio, timeline, transcript, options.subtitles);
         await writeSourceSeparationOutputIfNeeded(outputFilename, isolatedRawAudio, backgroundRawAudio, allowOverwrite, true);
+        const fileExtension = getLowercaseFileExtension(outputFilename);
+        if (['json', 'txt', 'srt', 'vtt'].includes(fileExtension)) {
+            const pathWithoutExtension = outputFilename.substring(0, outputFilename.lastIndexOf('.'));
+            const translatedOutputPath = `${pathWithoutExtension}.translated.${fileExtension}`;
+            const translatedFileSaver = getFileSaver(translatedOutputPath, allowOverwrite);
+            await translatedFileSaver(inputRawAudio, translatedTimeline, translatedTranscript, options.subtitles);
+        }
     }
     logger.end();
     if (cliOptions.play) {
@@ -601,10 +711,71 @@ async function alignTranslation(operationData) {
             audioToPlay = inputRawAudio;
         }
         const normalizedAudioToPlay = normalizeAudioLevel(audioToPlay);
-        await playAudioWithWordTimeline(normalizedAudioToPlay, wordTimeline, transcript);
+        await playAudioWithWordTimeline(normalizedAudioToPlay, translatedWordTimeline, translatedTranscript);
+    }
+}
+export async function alignTimelineTranslation(operationData) {
+    const logger = new Logger();
+    const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
+    const timelineFilename = operationArgs[0];
+    const outputFilenames = operationArgs.slice(2);
+    if (timelineFilename == undefined) {
+        throw new Error(`align-timeline-translation requires a first argument containing the timeline file path.`);
+    }
+    if (getLowercaseFileExtension(timelineFilename) != 'json') {
+        throw new Error(`align-timeline-translation only supports timeline files with extension 'json'`);
+    }
+    if (!existsSync(timelineFilename)) {
+        throw new Error(`The given timeline file '${timelineFilename}' was not found.`);
+    }
+    const timeline = await readAndParseJsonFile(timelineFilename);
+    const translationFilePath = operationArgs[1];
+    if (translationFilePath == undefined) {
+        throw new Error(`align-timeline-translation requires a second argument containing the translated reference file path.`);
+    }
+    if (!existsSync(translationFilePath)) {
+        throw new Error(`The given reference file '${translationFilePath}' was not found.`);
+    }
+    const translationFileExtension = getLowercaseFileExtension(translationFilePath);
+    const translationFileContent = await readFile(translationFilePath, { encoding: 'utf-8' });
+    let translationText;
+    if (translationFileExtension == 'txt') {
+        translationText = translationFileContent;
+    }
+    else if (translationFileExtension == 'html' || translationFileExtension == 'htm') {
+        translationText = await convertHtmlToText(translationFileContent);
+    }
+    else if (translationFileExtension == 'srt' || translationFileExtension == 'vtt') {
+        translationText = subtitlesToText(translationFileContent);
+    }
+    else {
+        throw new Error(`align only supports reference files with extensions 'txt', 'html', 'htm', 'srt' or 'vtt'`);
+    }
+    const options = await optionsLookupToTypedObject(operationOptionsLookup, 'TimelineTranslationAlignmentOptions');
+    const { timeline: translationTimeline, wordTimeline: translationWordTimeline, rawAudio } = await API.alignTimelineTranslation(timeline, translationText, options);
+    if (outputFilenames.length > 0) {
+        logger.start('\nWrite output files');
+    }
+    const allowOverwrite = getWithDefault(cliOptions.overwrite, overwriteByDefault);
+    for (const outputFilename of outputFilenames) {
+        const partPatternMatch = outputFilename.match(filenamePlaceholderPattern);
+        if (partPatternMatch) {
+            continue;
+        }
+        const fileSaver = getFileSaver(outputFilename, allowOverwrite);
+        await fileSaver(getEmptyRawAudio(1, 16000), translationTimeline, translationText, options.subtitles);
+    }
+    logger.end();
+    if (cliOptions.play && rawAudio) {
+        const normalizedAudioToPlay = normalizeAudioLevel(rawAudio);
+        let transcriptToPlay;
+        let timelineToPlay;
+        transcriptToPlay = translationText;
+        timelineToPlay = translationWordTimeline;
+        await playAudioWithWordTimeline(normalizedAudioToPlay, timelineToPlay, transcriptToPlay);
     }
 }
-async function translateSpeech(operationData) {
+export async function translateSpeech(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const inputFilename = operationArgs[0];
@@ -667,7 +838,7 @@ async function translateSpeech(operationData) {
         await playAudioWithWordTimeline(normalizedAudioToPlay, timelineToPlay, transcriptToPlay);
     }
 }
-async function detectLanguage(operationData, mode) {
+export async function detectLanguage(operationData, mode) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const inputFilePath = operationArgs[0];
@@ -721,7 +892,7 @@ async function detectLanguage(operationData, mode) {
     }
     logger.end();
 }
-async function detectVoiceActivity(operationData) {
+export async function detectVoiceActivity(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const audioFilename = operationArgs[0];
@@ -766,7 +937,7 @@ async function detectVoiceActivity(operationData) {
         await playAudioWithWordTimeline(normalizedAudio, timelineToPlay);
     }
 }
-async function denoise(operationData) {
+export async function denoise(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const audioFilename = operationArgs[0];
@@ -796,7 +967,7 @@ async function denoise(operationData) {
         await playAudioSamples(denoisedAudio);
     }
 }
-async function isolate(operationData) {
+export async function isolate(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const audioFilename = operationArgs[0];
@@ -825,7 +996,7 @@ async function isolate(operationData) {
         await playAudioSamples(isolatedRawAudio);
     }
 }
-async function listEngines(operationData) {
+export async function listEngines(operationData) {
     const logger = new Logger();
     const { operationArgs } = operationData;
     const targetOperation = operationArgs[0];
@@ -905,7 +1076,7 @@ async function listEngines(operationData) {
         }
     }
 }
-async function listTTSVoices(operationData) {
+export async function listTTSVoices(operationData) {
     const logger = new Logger();
     const { operationArgs, operationOptionsLookup, cliOptions } = operationData;
     const targetEngine = operationArgs[0];
@@ -947,7 +1118,7 @@ async function listTTSVoices(operationData) {
     }
     logger.end();
 }
-async function installPackages(operationData) {
+export async function installPackages(operationData) {
     const logger = new Logger();
     const { operationArgs } = operationData;
     if (operationArgs.length == 0) {
@@ -973,7 +1144,7 @@ async function installPackages(operationData) {
         }
     }
 }
-async function uninstallPackages(operationData) {
+export async function uninstallPackages(operationData) {
     const logger = new Logger();
     const { operationArgs } = operationData;
     if (operationArgs.length == 0) {
@@ -994,7 +1165,7 @@ async function uninstallPackages(operationData) {
         logger.log(`The packages ${failedPackageNames.join(', ')} failed to uninstall`, 'error');
     }
 }
-async function listPackages(operationData) {
+export async function listPackages(operationData) {
     const logger = new Logger();
     const packagesDir = await ensureAndGetPackagesDir();
     const installedPackageNames = await readdir(packagesDir);
@@ -1017,7 +1188,7 @@ async function listPackages(operationData) {
     logger.log(``);
     logger.log(installedPackageNamesFormatted.join('\n'));
 }
-async function serve(operationData) {
+export async function serve(operationData) {
     const { operationOptionsLookup } = operationData;
     const options = await optionsLookupToTypedObject(operationOptionsLookup, 'ServerOptions');
     async function onServerStarted(serverOptions) {
@@ -1143,13 +1314,13 @@ async function optionsLookupToTypedObject(cliOptionsMap, optionsRoot, additional
     return resultingObj;
 }
 let cachedOptionsSchema;
-export async function getOptionsSchema() {
+async function getOptionsSchema() {
     if (!cachedOptionsSchema) {
         cachedOptionsSchema = await readAndParseJsonFile(resolveToModuleRootDir('data/schemas/options.json'));
     }
     return cachedOptionsSchema;
 }
-export async function checkOutputFilenames(outputFilenames, acceptMediaOutputs, acceptMetadataOutputs, acceptSubtitleOutputs) {
+async function checkOutputFilenames(outputFilenames, acceptMediaOutputs, acceptMetadataOutputs, acceptSubtitleOutputs) {
     const supportedFileExtensions = [];
     if (acceptMediaOutputs) {
         supportedFileExtensions.push(...supportedOutputMediaFileExtensions);
@@ -1188,7 +1359,7 @@ export async function checkOutputFilenames(outputFilenames, acceptMediaOutputs,
     }
     return { includesPlaceholderPattern };
 }
-export async function writeOutputFilesForSegment(outputFilenames, index, total, audio, timeline, text, language, allowOverwrite) {
+async function writeOutputFilesForSegment(outputFilenames, index, total, audio, timeline, text, language, allowOverwrite) {
     const digitCount = Math.max((total + 1).toString().length, 2);
     const segmentWords = (await splitToWords(text, language)).filter(text => wordCharacterPattern.test(text));
     const segmentJoinedWords = segmentWords.join(' ').trim();