echogarden 0.0.1 → 0.1.0

package/dist/api/LanguageDetection.d.ts

@@ -0,0 +1,49 @@
+import { RawAudio } from "../audio/AudioUtilities.js";
+import * as API from "./API.js";
+import type { WhisperModelName } from "../recognition/WhisperSTT.js";
+export declare function detectSpeechFileLanguage(filename: string, options: SpeechLanguageDetectionOptions): Promise<{
+    detectedLanguage: string;
+    detectedLanguageName: string;
+    detectedLanguageProbabilities: API.LanguageDetectionResults;
+    detectedLanguageGroupProbabilities: API.LanguageDetectionGroupResults;
+}>;
+export declare function detectSpeechLanguage(rawAudio: RawAudio, options: SpeechLanguageDetectionOptions): Promise<{
+    detectedLanguage: string;
+    detectedLanguageName: string;
+    detectedLanguageProbabilities: API.LanguageDetectionResults;
+    detectedLanguageGroupProbabilities: API.LanguageDetectionGroupResults;
+}>;
+export type SpeechLanguageDetectionEngine = "silero" | "whisper";
+export interface SpeechLanguageDetectionOptions {
+    engine?: SpeechLanguageDetectionEngine;
+    defaultLanguage?: string;
+    fallbackThresholdProbability?: number;
+    silero?: {};
+    whisper?: {
+        model: WhisperModelName;
+    };
+}
+export declare const defaultSpeechLanguageDetectionOptions: SpeechLanguageDetectionOptions;
+export declare function detectTextLanguage(text: string, options: TextLanguageDetectionOptions): Promise<{
+    detectedLanguage: string;
+    detectedLanguageName: string;
+    detectedLanguageProbabilities: API.LanguageDetectionResults;
+}>;
+export type LanguageDetectionResults = LanguageDetectionResultsEntry[];
+export type LanguageDetectionResultsEntry = {
+    language: string;
+    languageName: string;
+    probability: number;
+};
+export type LanguageDetectionGroupResults = LanguageDetectionGroupResultsEntry[];
+export type LanguageDetectionGroupResultsEntry = {
+    languageGroup: string;
+    probability: number;
+};
+export type TextLanguageDetectionEngine = "tinyld" | "fasttext";
+export type TextLanguageDetectionOptions = {
+    engine?: TextLanguageDetectionEngine;
+    defaultLanguage?: string;
+    fallbackThresholdProbability?: number;
+};
+export declare const defaultTextLanguageDetectionOptions: TextLanguageDetectionOptions;

package/dist/api/LanguageDetection.js

@@ -0,0 +1,118 @@
+import { extendDeep } from "../utilities/ObjectUtilities.js";
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js";
+import { downmixToMonoAndNormalize } from "../audio/AudioUtilities.js";
+import { Logger } from "../utilities/Logger.js";
+import { resampleAudioSpeex } from "../dsp/SpeexResampler.js";
+import { logToStderr } from "../utilities/Utilities.js";
+import path from "path";
+import { languageCodeToName } from "../utilities/Locale.js";
+import { loadPackage } from "../utilities/PackageManager.js";
+const log = logToStderr;
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Speech language detection
+/////////////////////////////////////////////////////////////////////////////////////////////
+export async function detectSpeechFileLanguage(filename, options) {
+    const rawAudio = await FFMpegTranscoder.decodeToChannels(filename);
+    return detectSpeechLanguage(rawAudio, options);
+}
+export async function detectSpeechLanguage(rawAudio, options) {
+    const logger = new Logger();
+    logger.start("Prepare for speech language detection");
+    options = extendDeep(defaultSpeechLanguageDetectionOptions, options);
+    const defaultLanguage = options.defaultLanguage;
+    const fallbackThresholdProbability = options.fallbackThresholdProbability;
+    rawAudio = await resampleAudioSpeex(rawAudio, 16000);
+    rawAudio = downmixToMonoAndNormalize(rawAudio);
+    logger.start(`Initialize ${options.engine} module`);
+    let detectedLanguageProbabilities;
+    let detectedLanguageGroupProbabilities;
+    switch (options.engine) {
+        case "silero": {
+            const SileroLanguageDetection = await import("../speech-language-detection/SileroLanguageDetection.js");
+            logger.end();
+            const sileroOptions = options.silero;
+            const modelDir = await loadPackage("silero-lang-classifier-95");
+            const modelPath = path.join(modelDir, "lang_classifier_95.onnx");
+            const languageDictionaryPath = path.join(modelDir, "lang_dict_95.json");
+            const languageGroupDictionaryPath = path.join(modelDir, "lang_group_dict_95.json");
+            const { languageResults, languageGroupResults } = await SileroLanguageDetection.detectLanguage(rawAudio, modelPath, languageDictionaryPath, languageGroupDictionaryPath);
+            detectedLanguageProbabilities = languageResults;
+            detectedLanguageGroupProbabilities = languageGroupResults;
+            break;
+        }
+        case "whisper": {
+            const WhisperSTT = await import("../recognition/WhisperSTT.js");
+            const whisperOptions = options.whisper;
+            const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, undefined);
+            logger.end();
+            detectedLanguageProbabilities = await WhisperSTT.detectLanguage(rawAudio, modelName, modelDir, tokenizerDir);
+            detectedLanguageGroupProbabilities = [];
+            break;
+        }
+        default: {
+            throw new Error(`Engine '${options.engine}' is not supported`);
+        }
+    }
+    let detectedLanguage;
+    let detectedLanguageName;
+    if (detectedLanguageProbabilities.length == 0 ||
+        detectedLanguageProbabilities[0].probability < fallbackThresholdProbability) {
+        detectedLanguage = defaultLanguage;
+    }
+    else {
+        detectedLanguage = detectedLanguageProbabilities[0].language;
+    }
+    logger.end();
+    return { detectedLanguage, detectedLanguageName: languageCodeToName(detectedLanguage), detectedLanguageProbabilities, detectedLanguageGroupProbabilities };
+}
+export const defaultSpeechLanguageDetectionOptions = {
+    engine: "silero",
+    silero: {},
+    whisper: {
+        model: "tiny",
+    }
+};
+/////////////////////////////////////////////////////////////////////////////////////////////
+// Text language detection
+/////////////////////////////////////////////////////////////////////////////////////////////
+export async function detectTextLanguage(text, options) {
+    const logger = new Logger();
+    options = extendDeep(defaultTextLanguageDetectionOptions, options);
+    const defaultLanguage = options.defaultLanguage;
+    const fallbackThresholdProbability = options.fallbackThresholdProbability;
+    let detectedLanguageProbabilities;
+    logger.start(`Initialize ${options.engine} module`);
+    switch (options.engine) {
+        case "tinyld": {
+            const { detectLanguage } = await import("../text-language-detection/TinyLDLanguageDetection.js");
+            logger.start("Detecting text language using tinyld");
+            detectedLanguageProbabilities = await detectLanguage(text);
+            break;
+        }
+        case "fasttext": {
+            const { detectLanguage } = await import("../text-language-detection/FastTextLanguageDetection.js");
+            logger.start("Detecting text language using FastText");
+            detectedLanguageProbabilities = await detectLanguage(text);
+            break;
+        }
+        default: {
+            throw new Error(`Engine '${options.engine}' is not supported`);
+        }
+    }
+    let detectedLanguage;
+    if (detectedLanguageProbabilities.length == 0 ||
+        detectedLanguageProbabilities[0].probability < fallbackThresholdProbability) {
+        detectedLanguage = defaultLanguage;
+    }
+    else {
+        detectedLanguage = detectedLanguageProbabilities[0].language;
+    }
+    logger.end();
+    return { detectedLanguage, detectedLanguageName: languageCodeToName(detectedLanguage), detectedLanguageProbabilities };
+}
+export const defaultTextLanguageDetectionOptions = {
+    engine: "tinyld",
+    defaultLanguage: "en",
+    fallbackThresholdProbability: 0.05
+};
+//# sourceMappingURL=LanguageDetection.js.map

package/dist/api/LanguageDetection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LanguageDetection.js","sourceRoot":"","sources":["../../src/api/LanguageDetection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AAEjE,OAAO,EAAY,yBAAyB,EAAE,MAAM,4BAA4B,CAAA;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAG7D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,IAAI,MAAM,MAAM,CAAA;AAEvB,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AAC3D,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAE5D,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,6FAA6F;AAC7F,4BAA4B;AAC5B,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,QAAgB,EAAE,OAAuC;IACvG,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IAElE,OAAO,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,QAAkB,EAAE,OAAuC;IACrG,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAC3B,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAA;IAErD,OAAO,GAAG,UAAU,CAAC,qCAAqC,EAAE,OAAO,CAAC,CAAA;IAEpE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAgB,CAAA;IAChD,MAAM,4BAA4B,GAAG,OAAO,CAAC,4BAA6B,CAAA;IAE1E,QAAQ,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;IACpD,QAAQ,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAA;IAE9C,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,CAAC,MAAM,SAAS,CAAC,CAAA;IAEnD,IAAI,6BAAuD,CAAA;IAC3D,IAAI,kCAAiE,CAAA;IAErE,QAAQ,OAAO,CAAC,MAAM,EAAE;QACvB,KAAK,QAAQ,CAAC,CAAC;YACd,MAAM,uBAAuB,GAAG,MAAM,MAAM,CAAC,yDAAyD,CAAC,CAAA;YAEvG,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,aAAa,GAAG,OAAO,CAAC,MAAO,CAAA;YAErC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAA;YAE/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAA;YAChE,MAAM,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAA;YACvE,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAA;YAElF,MAAM,EAAE,eAAe,EAAE,oBAAoB,EAAE,GAAG,MAAM,uBAAuB,CAAC,cAAc,CAC7F,QAAQ,EACR,SAAS,EACT,sBAAsB,EACtB,2BAA2B,CAAC,CAAA;YAE7B,6BAA6B,GAAG,eAAe,CAAA;YAC/C,kCAAkC,GAAG,oBAAoB,CAAA;YAEzD,MAAK;SACL;QAED,KAAK,SAAS,CAAC,CAAC;YACf,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;YAEvH,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,6BAA6B,GAAG,MAAM,UAAU,CAAC,cAAc,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAA;YAC5G,kCAAkC,GAAG,EAAE,CAAA;YAEvC,MAAK;SACL;QAED,OAAO,CAAC,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;SAC9D;KACD;IAED,IAAI,gBAAwB,CAAA;IAC5B,IAAI,oBAA4B,CAAA;IAEhC,IAAI,6BAA6B,CAAC,MAAM,IAAI,CAAC;QAC5C,6BAA6B,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,4BAA4B,EAAE;QAE7E,gBAAgB,GAAG,eAAe,CAAA;KAClC;SAAM;QACN,gBAAgB,GAAG,6BAA6B,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;KAC5D;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,kBAAkB,CAAC,gBAAgB,CAAC,EAAE,6BAA6B,EAAE,kCAAkC,EAAE,CAAA;AAC3J,CAAC;AAiBD,MAAM,CAAC,MAAM,qCAAqC,GAAmC;IACpF,MAAM,EAAE,QAAQ;IAEhB,MAAM,EAAE,EACP;IAED,OAAO,EAAE;QACR,KAAK,EAAE,MAAM;KACb;CACD,CAAA;AAED,6FAA6F;AAC7F,0BAA0B;AAC1B,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY,EAAE,OAAqC;IAC3F,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,OAAO,GAAG,UAAU,CAAC,mCAAmC,EAAE,OAAO,CAAC,CAAA;IAElE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAgB,CAAA;IAChD,MAAM,4BAA4B,GAAG,OAAO,CAAC,4BAA6B,CAAA;IAE1E,IAAI,6BAAuD,CAAA;IAE3D,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,CAAC,MAAM,SAAS,CAAC,CAAA;IAEnD,QAAQ,OAAO,CAAC,MAAM,EAAE;QACvB,KAAK,QAAQ,CAAC,CAAC;YACd,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uDAAuD,CAAC,CAAA;YAEhG,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAA;YAEpD,6BAA6B,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAA;YAE1D,MAAK;SACL;QAED,KAAK,UAAU,CAAC,CAAC;YAChB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,yDAAyD,CAAC,CAAA;YAElG,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;YAEtD,6BAA6B,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAA;YAE1D,MAAK;SACL;QAED,OAAO,CAAC,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;SAC9D;KACD;IAED,IAAI,gBAAwB,CAAA;IAE5B,IAAI,6BAA6B,CAAC,MAAM,IAAI,CAAC;QAC5C,6BAA6B,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,4BAA4B,EAAE;QAE7E,gBAAgB,GAAG,eAAe,CAAA;KAClC;SAAM;QACN,gBAAgB,GAAG,6BAA6B,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;KAC5D;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,kBAAkB,CAAC,gBAAgB,CAAC,EAAE,6BAA6B,EAAE,CAAA;AACvH,CAAC;AAgBD,MAAM,CAAC,MAAM,mCAAmC,GAAiC;IAChF,MAAM,EAAE,QAAQ;IAChB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,IAAI;CAClC,CAAA"}

package/dist/api/Recognition.d.ts

@@ -0,0 +1,49 @@
+import { RawAudio } from "../audio/AudioUtilities.js";
+import * as API from "./API.js";
+import { Timeline } from "../utilities/Timeline.js";
+import type { WhisperModelName } from "../recognition/WhisperSTT.js";
+export declare function recognizeFile(filename: string, options: RecognitionOptions): Promise<{
+    transcript: string;
+    timeline: Timeline;
+    rawAudio: RawAudio;
+    language: string;
+}>;
+export declare function recognize(inputRawAudio: RawAudio, options: RecognitionOptions): Promise<{
+    transcript: string;
+    timeline: Timeline;
+    rawAudio: RawAudio;
+    language: string;
+}>;
+export type RecognitionEngine = "whisper" | "vosk" | "silero" | "google-cloud" | "microsoft-azure" | "amazon-transcribe";
+export interface RecognitionOptions {
+    engine?: RecognitionEngine;
+    language?: string;
+    maxAlternatives?: number;
+    alignment?: API.AlignmentOptions;
+    whisper?: {
+        model?: WhisperModelName;
+    };
+    vosk?: {
+        modelPath?: string;
+    };
+    silero?: {
+        modelPath?: string;
+    };
+    googleCloud?: {
+        apiKey?: string;
+        alternativeLanguageCodes?: string[];
+        profanityFilter?: boolean;
+        autoPunctuation?: boolean;
+        useEnhancedModel?: boolean;
+    };
+    microsoftAzure?: {
+        subscriptionKey?: string;
+        serviceRegion?: string;
+    };
+    amazonTranscribe?: {
+        region?: string;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+    };
+}
+export declare const defaultRecognitionOptions: RecognitionOptions;

package/dist/api/Recognition.js

@@ -0,0 +1,162 @@
+import { extendDeep } from "../utilities/ObjectUtilities.js";
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js";
+import { logToStderr } from "../utilities/Utilities.js";
+import { downmixToMonoAndNormalize, trimAudioEnd } from "../audio/AudioUtilities.js";
+import { Logger } from "../utilities/Logger.js";
+import { resampleAudioSpeex } from "../dsp/SpeexResampler.js";
+import * as API from "./API.js";
+import { formatLanguageCodeWithName, getShortLanguageCode, normalizeLanguageCode } from "../utilities/Locale.js";
+import { loadPackage } from "../utilities/PackageManager.js";
+const log = logToStderr;
+export async function recognizeFile(filename, options) {
+    const rawAudio = await FFMpegTranscoder.decodeToChannels(filename);
+    return recognize(rawAudio, options);
+}
+export async function recognize(inputRawAudio, options) {
+    const logger = new Logger();
+    const startTimestamp = logger.getTimestamp();
+    logger.start("Prepare for recognition");
+    options = extendDeep(defaultRecognitionOptions, options);
+    const engine = options.engine;
+    let sourceRawAudio = await resampleAudioSpeex(inputRawAudio, 16000);
+    sourceRawAudio = downmixToMonoAndNormalize(sourceRawAudio);
+    sourceRawAudio.audioChannels[0] = trimAudioEnd(sourceRawAudio.audioChannels[0], 0, -40);
+    if (!options.language) { // && options.engine != "whisper") {
+        logger.start("No language provided. Detecting audio language");
+        const { detectedLanguage } = await API.detectSpeechLanguage(inputRawAudio, { engine: "silero" });
+        logger.end();
+        logger.log(`Language detected: ${formatLanguageCodeWithName(detectedLanguage)}`);
+        options.language = detectedLanguage;
+    }
+    let language = normalizeLanguageCode(options.language);
+    let transcript;
+    let timeline;
+    logger.start(`Load ${engine} module`);
+    switch (engine) {
+        case "whisper": {
+            const WhisperSTT = await import("../recognition/WhisperSTT.js");
+            const whisperOptions = options.whisper;
+            const shortLanguageCode = getShortLanguageCode(language);
+            const { modelName, modelDir, tokenizerDir } = await WhisperSTT.loadPackagesAndGetPaths(whisperOptions.model, shortLanguageCode);
+            if (shortLanguageCode != "en" && modelName.endsWith(".en")) {
+                throw new Error(`The model '${modelName}' is English only and cannot transcribe language '${shortLanguageCode}'`);
+            }
+            logger.end();
+            ({ transcript, timeline } = await WhisperSTT.recognize(sourceRawAudio, modelName, modelDir, tokenizerDir, "transcribe", language));
+            break;
+        }
+        case "vosk": {
+            const VoskSTT = await import("../recognition/VoskSTT.js");
+            const voskOptions = options.vosk;
+            const modelPath = voskOptions.modelPath;
+            if (!modelPath) {
+                throw new Error("Vosk models are not currently auto-downloaded. You'll need to download a model manually and set a model path in 'vosk.modelPath'.");
+            }
+            logger.end();
+            ({ transcript, timeline } = await VoskSTT.recognize(sourceRawAudio, modelPath, true));
+            break;
+        }
+        case "silero": {
+            const SileroSTT = await import("../recognition/SileroSTT.js");
+            const sileroOptions = options.silero;
+            let modelPath = sileroOptions.modelPath;
+            if (!modelPath) {
+                const shortLanguageCode = getShortLanguageCode(language);
+                const packageName = SileroSTT.languageCodeToPackageName[shortLanguageCode];
+                if (!packageName) {
+                    throw new Error(`Language '${shortLanguageCode}' is not supported by Silero`);
+                }
+                modelPath = await loadPackage(packageName);
+            }
+            logger.end();
+            ({ transcript, timeline } = await SileroSTT.recognize(sourceRawAudio, modelPath));
+            break;
+        }
+        case "google-cloud": {
+            const GoogleCloudSTT = await import("../recognition/GoogleCloudSTT.js");
+            const apiKey = options.googleCloud.apiKey;
+            if (!apiKey) {
+                throw new Error(`No API key given`);
+            }
+            logger.end();
+            ({ transcript, timeline } = await GoogleCloudSTT.recognize(sourceRawAudio, apiKey, language));
+            break;
+        }
+        case "microsoft-azure": {
+            const AzureCognitiveServicesSTT = await import("../recognition/AzureCognitiveServicesSTT.js");
+            const subscriptionKey = options.microsoftAzure.subscriptionKey;
+            if (!subscriptionKey) {
+                throw new Error(`No subscription key given`);
+            }
+            const serviceRegion = options.microsoftAzure.serviceRegion;
+            if (!serviceRegion) {
+                throw new Error(`No service region given`);
+            }
+            logger.end();
+            ({ transcript, timeline } = await AzureCognitiveServicesSTT.recognize(sourceRawAudio, subscriptionKey, serviceRegion, language));
+            break;
+        }
+        case "amazon-transcribe": {
+            const AmazonTranscribeSTT = await import("../recognition/AmazonTranscribeSTT.js");
+            const region = options.amazonTranscribe.region;
+            if (!region) {
+                throw new Error(`No region given`);
+            }
+            const accessKeyId = options.amazonTranscribe.accessKeyId;
+            if (!accessKeyId) {
+                throw new Error(`No access key id given`);
+            }
+            const secretAccessKey = options.amazonTranscribe.secretAccessKey;
+            if (!secretAccessKey) {
+                throw new Error(`No secret access key given`);
+            }
+            logger.end();
+            ({ transcript, timeline } = await AmazonTranscribeSTT.recgonize(sourceRawAudio, language, region, accessKeyId, secretAccessKey));
+            break;
+        }
+        default: {
+            throw new Error(`Engine '${options.engine}' is not supported`);
+        }
+    }
+    if (!timeline) {
+        logger.start(`Align audio to transcript`);
+        const alignmentOptions = extendDeep(options.alignment, { language: language });
+        const { wordTimeline } = await API.align(sourceRawAudio, transcript, alignmentOptions);
+        timeline = wordTimeline;
+    }
+    logger.end();
+    logger.logDuration(`Total recognition time`, startTimestamp);
+    return { transcript, timeline, rawAudio: inputRawAudio, language };
+}
+export const defaultRecognitionOptions = {
+    engine: "whisper",
+    language: undefined,
+    maxAlternatives: 1,
+    alignment: undefined,
+    whisper: {
+        model: undefined,
+    },
+    vosk: {
+        modelPath: undefined
+    },
+    silero: {
+        modelPath: undefined
+    },
+    googleCloud: {
+        apiKey: undefined,
+        alternativeLanguageCodes: [],
+        profanityFilter: false,
+        autoPunctuation: true,
+        useEnhancedModel: true,
+    },
+    microsoftAzure: {
+        subscriptionKey: undefined,
+        serviceRegion: undefined
+    },
+    amazonTranscribe: {
+        region: undefined,
+        accessKeyId: undefined,
+        secretAccessKey: undefined,
+    }
+};
+//# sourceMappingURL=Recognition.js.map

package/dist/api/Recognition.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"Recognition.js","sourceRoot":"","sources":["../../src/api/Recognition.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAY,yBAAyB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC9F,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAE7D,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAG/B,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAChH,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAE5D,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,OAA2B;IAChF,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IAClE,OAAO,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,aAAuB,EAAE,OAA2B;IACnF,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAC3B,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IAE5C,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAA;IAEvC,OAAO,GAAG,UAAU,CAAC,yBAAyB,EAAE,OAAO,CAAC,CAAA;IAExD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAO,CAAA;IAE9B,IAAI,cAAc,GAAG,MAAM,kBAAkB,CAAC,aAAa,EAAE,KAAK,CAAC,CAAA;IACnE,cAAc,GAAG,yBAAyB,CAAC,cAAc,CAAC,CAAA;IAC1D,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;IAEvF,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,oCAAoC;QAC5D,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;QAC9D,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,GAAG,CAAC,oBAAoB,CAAC,aAAa,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;QAEhG,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,GAAG,CAAC,sBAAsB,0BAA0B,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAA;QAEhF,OAAO,CAAC,QAAQ,GAAG,gBAAgB,CAAA;KACnC;IAED,IAAI,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAEtD,IAAI,UAAkB,CAAA;IACtB,IAAI,QAA8B,CAAA;IAElC,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,SAAS,CAAC,CAAA;IAErC,QAAQ,MAAM,EAAE;QACf,KAAK,SAAS,CAAC,CAAC;YACf,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;YAExD,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAA;YAE/H,IAAI,iBAAiB,IAAI,IAAI,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;gBAC3D,MAAM,IAAI,KAAK,CAAC,cAAc,SAAS,qDAAqD,iBAAiB,GAAG,CAAC,CAAA;aACjH;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAA;YAElI,MAAK;SACL;QAED,KAAK,MAAM,CAAC,CAAC;YACZ,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;YAEzD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAK,CAAA;YAEjC,MAAM,SAAS,GAAG,WAAW,CAAC,SAAS,CAAA;YAEvC,IAAI,CAAC,SAAS,EAAE;gBACf,MAAM,IAAI,KAAK,CAAC,mIAAmI,CAAC,CAAA;aACpJ;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;YAErF,MAAK;SACL;QAED,KAAK,QAAQ,CAAC,CAAC;YACd,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAA;YAE7D,MAAM,aAAa,GAAG,OAAO,CAAC,MAAO,CAAA;YAErC,IAAI,SAAS,GAAG,aAAa,CAAC,SAAS,CAAA;YAEvC,IAAI,CAAC,SAAS,EAAE;gBACf,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;gBACxD,MAAM,WAAW,GAAG,SAAS,CAAC,yBAAyB,CAAC,iBAAiB,CAAC,CAAA;gBAE1E,IAAI,CAAC,WAAW,EAAE;oBACjB,MAAM,IAAI,KAAK,CAAC,aAAa,iBAAiB,8BAA8B,CAAC,CAAA;iBAC7E;gBAED,SAAS,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,CAAA;aAC1C;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC,CAAA;YAEjF,MAAK;SACL;QAED,KAAK,cAAc,CAAC,CAAC;YACpB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;YAEvE,MAAM,MAAM,GAAG,OAAO,CAAC,WAAY,CAAC,MAAM,CAAA;YAE1C,IAAI,CAAC,MAAM,EAAE;gBACZ,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAA;aACnC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAA;YAE7F,MAAK;SACL;QAED,KAAK,iBAAiB,CAAC,CAAC;YACvB,MAAM,yBAAyB,GAAG,MAAM,MAAM,CAAC,6CAA6C,CAAC,CAAA;YAE7F,MAAM,eAAe,GAAG,OAAO,CAAC,cAAe,CAAC,eAAe,CAAA;YAE/D,IAAI,CAAC,eAAe,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;aAC5C;YAED,MAAM,aAAa,GAAG,OAAO,CAAC,cAAe,CAAC,aAAa,CAAA;YAE3D,IAAI,CAAC,aAAa,EAAE;gBACnB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAA;aAC1C;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,yBAAyB,CAAC,SAAS,CAAC,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC,CAAA;YAEhI,MAAK;SACL;QAED,KAAK,mBAAmB,CAAC,CAAC;YACzB,MAAM,mBAAmB,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAA;YAEjF,MAAM,MAAM,GAAG,OAAO,CAAC,gBAAiB,CAAC,MAAM,CAAA;YAE/C,IAAI,CAAC,MAAM,EAAE;gBACZ,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAA;aAClC;YAED,MAAM,WAAW,GAAG,OAAO,CAAC,gBAAiB,CAAC,WAAW,CAAA;YAEzD,IAAI,CAAC,WAAW,EAAE;gBACjB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;aACzC;YAED,MAAM,eAAe,GAAG,OAAO,CAAC,gBAAiB,CAAC,eAAe,CAAA;YAEjE,IAAI,CAAC,eAAe,EAAE;gBACrB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;aAC7C;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,eAAe,CAAC,CAAC,CAAA;YAEhI,MAAK;SACL;QAED,OAAO,CAAC,CAAC;YACR,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;SAC9D;KACD;IAED,IAAI,CAAC,QAAQ,EAAE;QACd,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAA;QACzC,MAAM,gBAAgB,GAAyB,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;QAEpG,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAA;QAEtF,QAAQ,GAAG,YAAY,CAAA;KACvB;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IACZ,MAAM,CAAC,WAAW,CAAC,wBAAwB,EAAE,cAAc,CAAC,CAAA;IAE5D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAA;AACnE,CAAC;AA6CD,MAAM,CAAC,MAAM,yBAAyB,GAAuB;IAC5D,MAAM,EAAE,SAAS;IAEjB,QAAQ,EAAE,SAAS;IAEnB,eAAe,EAAE,CAAC;IAElB,SAAS,EAAE,SAAS;IAEpB,OAAO,EAAE;QACR,KAAK,EAAE,SAAS;KAChB;IAED,IAAI,EAAE;QACL,SAAS,EAAE,SAAS;KACpB;IAED,MAAM,EAAE;QACP,SAAS,EAAE,SAAS;KACpB;IAED,WAAW,EAAE;QACZ,MAAM,EAAE,SAAS;QACjB,wBAAwB,EAAE,EAAE;QAC5B,eAAe,EAAE,KAAK;QACtB,eAAe,EAAE,IAAI;QACrB,gBAAgB,EAAE,IAAI;KACtB;IAED,cAAc,EAAE;QACf,eAAe,EAAE,SAAS;QAC1B,aAAa,EAAE,SAAS;KACxB;IAED,gBAAgB,EAAE;QACjB,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,SAAS;QACtB,eAAe,EAAE,SAAS;KAC1B;CACD,CAAA"}

package/dist/api/Synthesis.d.ts

@@ -0,0 +1,125 @@
+import { RawAudio } from "../audio/AudioUtilities.js";
+import { type RubberbandOptions } from "../dsp/Rubberband.js";
+import * as API from "./API.js";
+import { Timeline } from "../utilities/Timeline.js";
+export declare function synthesizeSegments(segments: string[], options: SynthesisOptions, onSegment?: SynthesisSegmentEvent, onSentence?: SynthesisSegmentEvent): Promise<{
+    synthesizedAudio: RawAudio;
+    timeline: Timeline;
+    segmentsAudio: RawAudio[];
+    segmentsTimelines: Timeline[];
+}>;
+export type SynthesisEngine = "vits" | "pico" | "flite" | "espeak" | "sam" | "sapi" | "msspeech" | "coqui-server" | "google-cloud" | "microsoft-azure" | "amazon-polly" | "elevenlabs" | "google-translate" | "microsoft-edge" | "streamlabs-polly";
+export type TimePitchShiftingMethod = "sonic" | "rubberband";
+export interface SynthesisOptions {
+    engine?: SynthesisEngine;
+    language?: string;
+    voice?: string;
+    voiceGender?: VoiceGender;
+    speed?: number;
+    pitch?: number;
+    pitchVariation?: number;
+    ssml?: boolean;
+    splitToSentences?: boolean;
+    segmentEndPause?: number;
+    sentenceEndPause?: number;
+    alignment?: API.AlignmentOptions;
+    postProcessing?: {
+        normalizeAudio?: boolean;
+        targetPeakDb?: number;
+        maxIncreaseDb?: number;
+        speed?: number;
+        pitch?: number;
+        timePitchShiftingMethod?: TimePitchShiftingMethod;
+        rubberband?: RubberbandOptions;
+    };
+    vits?: {
+        speakerId?: number;
+    };
+    pico?: {};
+    flite?: {};
+    espeak?: {
+        rate?: number;
+        pitch?: number;
+        pitchRange?: number;
+    };
+    sam?: {
+        pitch?: number;
+        speed?: number;
+        mouth?: number;
+        throat?: number;
+    };
+    sapi?: {
+        rate?: number;
+    };
+    msspeech?: {
+        rate?: number;
+    };
+    coquiServer?: {
+        serverUrl?: string;
+        speakerId?: string | null;
+    };
+    googleCloud?: {
+        apiKey?: string;
+        pitchDeltaSemitones?: number;
+        customVoice?: {
+            model?: string;
+            reportedUsage?: string;
+        };
+    };
+    microsoftAzure?: {
+        subscriptionKey?: string;
+        serviceRegion?: string;
+        pitchDeltaHz?: number;
+    };
+    awsPolly?: {
+        region?: string;
+        accessKeyId?: string;
+        secretAccessKey?: string;
+        pollyEngine?: "standard" | "neural";
+        lexiconNames?: string[];
+    };
+    elevenlabs?: {
+        apiKey?: string;
+        stability?: number;
+        similarityBoost?: number;
+    };
+    googleTranslate?: {
+        tld?: string;
+    };
+    microsoftEdge?: {
+        trustedClientToken?: string;
+        pitchDeltaHz?: number;
+    };
+    streamlabsPolly?: {};
+}
+export declare const defaultSynthesisOptions: SynthesisOptions;
+export declare function requestVoiceList(options: VoiceListRequestOptions): Promise<{
+    voiceList: API.SynthesisVoice[];
+    bestMatchingVoice: API.SynthesisVoice;
+}>;
+export declare function selectBestOfflineEngineForLanguage(language: string): SynthesisEngine;
+export interface VoiceListRequestOptions extends SynthesisOptions {
+    cache?: {
+        path?: string;
+        duration?: number;
+    };
+}
+export declare const defaultVoiceListRequestOptions: VoiceListRequestOptions;
+export interface SynthesisSegmentEventData {
+    index: number;
+    total: number;
+    audio: RawAudio;
+    timeline: Timeline;
+    transcript: string;
+    language: string;
+    peakDecibelsSoFar: number;
+}
+export type SynthesisSegmentEvent = (data: SynthesisSegmentEventData) => Promise<void>;
+export interface SynthesisVoice {
+    name: string;
+    languages: string[];
+    gender: VoiceGender;
+    speakerCount?: number;
+    packageName?: string;
+}
+export type VoiceGender = "male" | "female" | "unknown";