echogarden 0.0.1 → 0.1.0

package/dist/synthesis/AzureCognitiveServicesTTS.js

@@ -0,0 +1,103 @@
+import SpeechSDK from 'microsoft-cognitiveservices-speech-sdk';
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js";
+import { escape } from 'html-escaper';
+import { Logger } from '../utilities/Logger.js';
+import { getRawAudioDuration } from '../audio/AudioUtilities.js';
+export async function synthesize(text, subscriptionKey, serviceRegion, languageCode = "en-US", voice = "Microsoft Server Speech Text to Speech Voice (en-US, AriaNeural)", ssmlEnabled = false, ssmlPitchString = "+0Hz", ssmlRateString = "+0%") {
+    return new Promise((resolve, reject) => {
+        const logger = new Logger();
+        logger.start("Request synthesis from Azure Cognitive Services");
+        const speechConfig = SpeechSDK.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion);
+        speechConfig.speechSynthesisLanguage = languageCode;
+        speechConfig.speechSynthesisVoiceName = voice;
+        speechConfig.speechSynthesisOutputFormat = SpeechSDK.SpeechSynthesisOutputFormat.Ogg24Khz16BitMonoOpus;
+        const audioOutputStream = SpeechSDK.AudioOutputStream.createPullStream();
+        const audioConfig = SpeechSDK.AudioConfig.fromStreamOutput(audioOutputStream);
+        const synthesis = new SpeechSDK.SpeechSynthesizer(speechConfig, audioConfig);
+        const events = [];
+        synthesis.wordBoundary = (sender, event) => {
+            events.push(event);
+        };
+        const onResult = async (result) => {
+            if (result.errorDetails != null) {
+                reject(result.errorDetails);
+                return;
+            }
+            /*
+            const bufferSize = 2 ** 16
+            const buffers: Buffer[] = []
+
+            while (true) {
+
+                const buffer = Buffer.alloc(bufferSize)
+                const amountRead = await audioOutputStream.read(buffer)
+
+                if (amountRead == 0) {
+                    audioOutputStream.close()
+                    break
+                }
+
+                buffers.push(buffer.subarray(0, amountRead))
+            }
+
+            const encodedAudio = Buffer.concat(buffers)
+            */
+            const encodedAudio = Buffer.from(result.audioData);
+            logger.end();
+            const rawAudio = await FFMpegTranscoder.decodeToChannels(encodedAudio, 24000, 1);
+            logger.start("Convert boundary events to a timeline");
+            const timeline = boundaryEventsToTimeline(events, getRawAudioDuration(rawAudio));
+            logger.end();
+            resolve({ rawAudio, timeline: timeline });
+        };
+        const onError = (error) => {
+            reject(error);
+        };
+        if (!ssmlEnabled && ssmlPitchString != "+0%" || ssmlRateString != "+0Hz") {
+            ssmlEnabled = true;
+            text = escape(text);
+        }
+        if (ssmlEnabled) {
+            text =
+                `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xml:lang="en-US">` +
+                    `<voice name="${voice}">` +
+                    `<prosody pitch="${ssmlPitchString}" rate="${ssmlRateString}">` +
+                    text +
+                    `</prosody>` +
+                    `</voice>` +
+                    `</speak>`;
+            synthesis.speakSsmlAsync(text, onResult, onError);
+        }
+        else {
+            synthesis.speakTextAsync(text, onResult, onError);
+        }
+    });
+}
+export async function getVoiceList(subscriptionKey, serviceRegion) {
+    const speechConfig = SpeechSDK.SpeechConfig.fromSubscription(subscriptionKey, serviceRegion);
+    const synthesis = new SpeechSDK.SpeechSynthesizer(speechConfig, undefined);
+    const result = await synthesis.getVoicesAsync();
+    return result.voices;
+}
+export function boundaryEventsToTimeline(events, totalDuration) {
+    const timeline = [];
+    for (const event of events) {
+        const boundaryType = event.boundaryType != null ? event.boundaryType : event.Type;
+        if (boundaryType != "WordBoundary") {
+            continue;
+        }
+        const text = event.text != null ? event.text : event.Data.text.Text;
+        const offset = event.audioOffset != null ? event.audioOffset : event.Data.Offset;
+        const duration = event.duration != null ? event.duration : event.Data.Duration;
+        const startTime = offset / 10000000;
+        const endTime = (offset + duration) / 10000000;
+        timeline.push({
+            type: "word",
+            text,
+            startTime,
+            endTime
+        });
+    }
+    return timeline;
+}
+//# sourceMappingURL=AzureCognitiveServicesTTS.js.map

package/dist/synthesis/AzureCognitiveServicesTTS.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AzureCognitiveServicesTTS.js","sourceRoot":"","sources":["../../src/synthesis/AzureCognitiveServicesTTS.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,wCAAwC,CAAA;AAE9D,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AAEjE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAA;AAErC,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAY,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAE1E,MAAM,CAAC,KAAK,UAAU,UAAU,CAC/B,IAAY,EACZ,eAAuB,EACvB,aAAqB,EACrB,YAAY,GAAG,OAAO,EACtB,KAAK,GAAG,kEAAkE,EAC1E,WAAW,GAAG,KAAK,EACnB,eAAe,GAAG,MAAM,EACxB,cAAc,GAAG,KAAK;IAEtB,OAAO,IAAI,OAAO,CAA6C,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAClF,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;QAC3B,MAAM,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAA;QAE/D,MAAM,YAAY,GAAG,SAAS,CAAC,YAAY,CAAC,gBAAgB,CAAC,eAAe,EAAE,aAAa,CAAC,CAAA;QAE5F,YAAY,CAAC,uBAAuB,GAAG,YAAY,CAAA;QACnD,YAAY,CAAC,wBAAwB,GAAG,KAAK,CAAA;QAC7C,YAAY,CAAC,2BAA2B,GAAG,SAAS,CAAC,2BAA2B,CAAC,qBAAqB,CAAA;QAEtG,MAAM,iBAAiB,GAAG,SAAS,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,CAAA;QAExE,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAA;QAE7E,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,iBAAiB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;QAE5E,MAAM,MAAM,GAAqD,EAAE,CAAA;QAEnE,SAAS,CAAC,YAAY,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YAC1C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACnB,CAAC,CAAA;QAED,MAAM,QAAQ,GAAG,KAAK,EAAE,MAAuC,EAAE,EAAE;YAClE,IAAI,MAAM,CAAC,YAAY,IAAI,IAAI,EAAE;gBAChC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;gBAC3B,OAAM;aACN;YAED;;;;;;;;;;;;;;;;;;cAkBE;YAEF,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YAElD,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;YAEhF,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAA;YAErD,MAAM,QAAQ,GAAG,wBAAwB,CAAC,MAAM,EAAE,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAA;YAEhF,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,OAAO,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;QAC1C,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,EAAE;YACjC,MAAM,CAAC,KAAK,CAAC,CAAA;QACd,CAAC,CAAA;QAED,IAAI,CAAC,WAAW,IAAI,eAAe,IAAI,KAAK,IAAI,cAAc,IAAI,MAAM,EAAE;YACzE,WAAW,GAAG,IAAI,CAAA;YAClB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;SACnB;QAED,IAAI,WAAW,EAAE;YAChB,IAAI;gBACH,oFAAoF;oBACpF,gBAAgB,KAAK,IAAI;oBACzB,mBAAmB,eAAe,WAAW,cAAc,IAAI;oBAC/D,IAAI;oBACJ,YAAY;oBACZ,UAAU;oBACV,UAAU,CAAA;YAEX,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;SACjD;aAAM;YACN,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;SACjD;IACF,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,eAAuB,EAAE,aAAqB;IAChF,MAAM,YAAY,GAAG,SAAS,CAAC,YAAY,CAAC,gBAAgB,CAAC,eAAe,EAAE,aAAa,CAAC,CAAA;IAE5F,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,iBAAiB,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;IAE1E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,cAAc,EAAE,CAAA;IAE/C,OAAO,MAAM,CAAC,MAAM,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAa,EAAE,aAAqB;IAC5E,MAAM,QAAQ,GAAa,EAAE,CAAA;IAE7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAA;QAEjF,IAAI,YAAY,IAAI,cAAc,EAAE;YACnC,SAAQ;SACR;QAED,MAAM,IAAI,GAAW,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAA;QAC3E,MAAM,MAAM,GAAW,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAA;QACxF,MAAM,QAAQ,GAAW,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAA;QAEtF,MAAM,SAAS,GAAG,MAAM,GAAG,QAAQ,CAAA;QACnC,MAAM,OAAO,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,QAAQ,CAAA;QAE9C,QAAQ,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,MAAM;YACZ,IAAI;YACJ,SAAS;YACT,OAAO;SACP,CAAC,CAAA;KACF;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/dist/synthesis/CoquiServerTTS.d.ts

@@ -0,0 +1,6 @@
+export declare function synthesize(text: string, speakerId: string | null, serverURL?: string): Promise<{
+    rawAudio: {
+        audioChannels: Float32Array[];
+        sampleRate: number;
+    };
+}>;

package/dist/synthesis/CoquiServerTTS.js

@@ -0,0 +1,22 @@
+import { request } from "gaxios";
+import { decodeWaveBuffer } from "../audio/AudioUtilities.js";
+import { Logger } from "../utilities/Logger.js";
+import { logToStderr } from "../utilities/Utilities.js";
+const log = logToStderr;
+export async function synthesize(text, speakerId, serverURL = "http://[::1]:5002") {
+    const logger = new Logger();
+    logger.start("Request synthesis from Coqui Server");
+    const response = await request({
+        url: `${serverURL}/api/tts`,
+        params: {
+            "text": text,
+            "speaker_id": speakerId
+        },
+        responseType: "arraybuffer"
+    });
+    const waveData = Buffer.from(response.data);
+    const rawAudio = decodeWaveBuffer(waveData).rawAudio;
+    logger.end();
+    return { rawAudio };
+}
+//# sourceMappingURL=CoquiServerTTS.js.map

package/dist/synthesis/CoquiServerTTS.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CoquiServerTTS.js","sourceRoot":"","sources":["../../src/synthesis/CoquiServerTTS.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AAC7D,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,SAAwB,EAAE,SAAS,GAAG,mBAAmB;IACvG,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAC3B,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;IAEnD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAS;QACtC,GAAG,EAAE,GAAG,SAAS,UAAU;QAE3B,MAAM,EAAE;YACP,MAAM,EAAE,IAAI;YACZ,YAAY,EAAE,SAAS;SACvB;QAED,YAAY,EAAE,aAAa;KAC3B,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IAE3C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAA;IAEpD,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,QAAQ,EAAE,CAAA;AACpB,CAAC"}

package/dist/synthesis/ElevenLabsTTS.d.ts

@@ -0,0 +1,8 @@
+import { SynthesisVoice } from "../api/API.js";
+export declare function synthesize(text: string, voiceId: string, apiKey: string, stability?: number, similarityBoost?: number): Promise<{
+    rawAudio: {
+        audioChannels: Float32Array[];
+        sampleRate: number;
+    };
+}>;
+export declare function getVoiceList(): Promise<SynthesisVoice[]>;

package/dist/synthesis/ElevenLabsTTS.js

@@ -0,0 +1,48 @@
+import { request } from "gaxios";
+import * as FFMpegTranscoder from "../codecs/FFMpegTranscoder.js";
+import { Logger } from "../utilities/Logger.js";
+import { logToStderr } from "../utilities/Utilities.js";
+const log = logToStderr;
+export async function synthesize(text, voiceId, apiKey, stability = 0, similarityBoost = 0) {
+    const logger = new Logger();
+    logger.start("Request synthesis from ElevenLabs");
+    const response = await request({
+        url: `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`,
+        method: "POST",
+        data: {
+            text,
+            voice_setting: {
+                stability,
+                similarity_boost: similarityBoost
+            }
+        },
+        headers: {
+            "dnt": "1",
+            "xi-api-key": apiKey,
+        },
+        responseType: "arraybuffer"
+    });
+    logger.start("Decode synthesized audio");
+    const rawAudio = await FFMpegTranscoder.decodeToChannels(Buffer.from(response.data));
+    logger.end();
+    return { rawAudio };
+}
+export async function getVoiceList() {
+    const response = await request({
+        method: "GET",
+        url: "https://api.elevenlabs.io/v1/voices",
+        headers: {
+            "dnt": "1",
+        },
+        responseType: "json"
+    });
+    const elevenLabsVoices = response.data.voices;
+    const voices = elevenLabsVoices.map(elevenLabsVoice => ({
+        name: elevenLabsVoice.name,
+        languages: ['en-US', 'en'],
+        gender: "unknown",
+        elevenLabsVoiceId: elevenLabsVoice.voice_id
+    }));
+    return voices;
+}
+//# sourceMappingURL=ElevenLabsTTS.js.map

package/dist/synthesis/ElevenLabsTTS.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ElevenLabsTTS.js","sourceRoot":"","sources":["../../src/synthesis/ElevenLabsTTS.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAA;AAEhC,OAAO,KAAK,gBAAgB,MAAM,+BAA+B,CAAA;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AAEvD,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,OAAe,EAAE,MAAc,EAAE,SAAS,GAAG,CAAC,EAAE,eAAe,GAAG,CAAC;IACjH,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAC3B,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAM;QACnC,GAAG,EAAE,+CAA+C,OAAO,EAAE;QAE7D,MAAM,EAAE,MAAM;QAEd,IAAI,EAAE;YACL,IAAI;YAEJ,aAAa,EAAE;gBACd,SAAS;gBACT,gBAAgB,EAAE,eAAe;aACjC;SACD;QAED,OAAO,EAAE;YACR,KAAK,EAAE,GAAG;YACV,YAAY,EAAE,MAAM;SACpB;QAED,YAAY,EAAE,aAAa;KAC3B,CAAC,CAAA;IAEF,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;IACxC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IAEpF,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,QAAQ,EAAE,CAAA;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY;IACjC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAM;QACnC,MAAM,EAAE,KAAK;QAEb,GAAG,EAAE,qCAAqC;QAC1C,OAAO,EAAE;YACR,KAAK,EAAE,GAAG;SACV;QAED,YAAY,EAAE,MAAM;KACpB,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAU,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAA;IAEpD,MAAM,MAAM,GAAqB,gBAAgB,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QACzE,IAAI,EAAE,eAAe,CAAC,IAAI;QAC1B,SAAS,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC;QAC1B,MAAM,EAAE,SAAS;QAEjB,iBAAiB,EAAE,eAAe,CAAC,QAAQ;KAC3C,CAAC,CAAC,CAAA;IAEH,OAAO,MAAM,CAAA;AACd,CAAC"}

package/dist/synthesis/EspeakTTS.d.ts

@@ -0,0 +1,46 @@
+import { RawAudio } from "../audio/AudioUtilities.js";
+import { Lexicon } from "../nlp/Lexicon.js";
+import { Timeline } from "../utilities/Timeline.js";
+export declare function synthesize(text: string, ssmlEnabled: boolean): Promise<{
+    rawAudio: RawAudio;
+    events: EspeakEvent[];
+}>;
+export declare function preprocessAndSynthesizeSentence(sentence: string, espeakVoice: string, lexicons?: Lexicon[], normalize?: boolean, rate?: number, pitch?: number, pitchRange?: number): Promise<{
+    referenceSynthesizedAudio: RawAudio;
+    referenceTimeline: Timeline;
+    fragments: string[];
+    preprocessedFragments: string[];
+    phonemizedFragmentsSubstitutions: Map<number, string[]>;
+    phonemizedSentence: string[][][];
+}>;
+export declare function synthesizeFragments(fragments: string[], voice: string, insertSeparators?: boolean, rate?: number, pitch?: number, pitchRange?: number): Promise<{
+    rawAudio: RawAudio;
+    timeline: Timeline;
+    events: EspeakEvent[];
+}>;
+export declare function textToIPA(text: string, voice: string): Promise<string>;
+export declare function textToPhonemes(text: string, voice: string, useIPA?: boolean): Promise<string>;
+export declare function setVoice(voiceId: string): Promise<void>;
+export declare function setVolume(volume: number): Promise<any>;
+export declare function setRate(rate: number): Promise<any>;
+export declare function setPitch(pitch: number): Promise<any>;
+export declare function setPitchRange(pitchRange: number): Promise<any>;
+export declare function getSampleRate(): Promise<22050>;
+export declare function listVoices(): Promise<{
+    identifier: string;
+    name: string;
+    languages: {
+        priority: number;
+        name: string;
+    }[];
+}[]>;
+export type EspeakEventType = "sentence" | "word" | "phoneme" | "end" | "mark" | "play" | "msg_terminated" | "list_terminated" | "samplerate";
+export interface EspeakEvent {
+    audio_position: number;
+    type: EspeakEventType;
+    text_position: number;
+    word_length: number;
+    id?: string | number;
+}
+export declare function testEspeakSynthesisWithPrePhonemizedInputs(text: string): Promise<void>;
+export declare function testKirshenbaumPhonemization(text: string): Promise<void>;

package/dist/synthesis/EspeakTTS.js

@@ -0,0 +1,353 @@
+import { concatFloat32Arrays, logToStderr, objToString } from "../utilities/Utilities.js";
+import { int16PcmToFloat32 } from "../audio/AudioBufferConversion.js";
+import { Logger } from '../utilities/Logger.js';
+import { WasmMemoryManager } from "../utilities/WasmMemoryManager.js";
+import { getEmptyRawAudio } from "../audio/AudioUtilities.js";
+import { playAudioWithTimelinePhones } from "../audio/AudioPlayer.js";
+import * as CompromiseNLP from "../nlp/CompromiseNLP.js";
+import { getNormalizationMapForSpeech } from "../nlp/TextNormalizer.js";
+import { ipaPhoneToKirshenbaum } from "../nlp/PhoneConversion.js";
+import { splitToWords } from "../nlp/Segmentation.js";
+import { phonemizeSentence } from "../nlp/EspeakPhonemizer.js";
+const log = logToStderr;
+let espeakInstance;
+let espeakModule;
+export async function synthesize(text, ssmlEnabled) {
+    const logger = new Logger();
+    logger.start("Get espeak WASM instance");
+    if (!ssmlEnabled) {
+        const { escape } = await import('html-escaper');
+        text = escape(text);
+    }
+    const { instance } = await getEspeakInstance();
+    const sampleChunks = [];
+    const allEvents = [];
+    logger.start("Synthesize with espeak");
+    instance.synthesize(text, (samples, events) => {
+        if (samples && samples.length > 0) {
+            sampleChunks.push(int16PcmToFloat32(samples));
+        }
+        for (const event of events) {
+            if (event.type == "word") {
+                const textPosition = event.text_position - 1;
+                event["text"] = text.substring(textPosition, textPosition + event.word_length);
+            }
+        }
+        allEvents.push(...events);
+    });
+    const concatenatedSamples = concatFloat32Arrays(sampleChunks);
+    const rawAudio = { audioChannels: [concatenatedSamples], sampleRate: 22050 };
+    logger.end();
+    return { rawAudio, events: allEvents };
+}
+export async function preprocessAndSynthesizeSentence(sentence, espeakVoice, lexicons = [], normalize = true, rate = 150, pitch = 50, pitchRange = 50) {
+    const logger = new Logger();
+    await logger.startAsync("Tokenize and analyze text");
+    let fragments;
+    let preprocessedFragments;
+    const phonemizedFragmentsSubstitutions = new Map();
+    if (espeakVoice == "en-us") {
+        fragments = [];
+        preprocessedFragments = [];
+        const parsedSentences = await CompromiseNLP.parse(sentence);
+        const terms = parsedSentences.flatMap(s => s);
+        const normalizationMap = getNormalizationMapForSpeech(terms, espeakVoice);
+        for (let termIndex = 0; termIndex < terms.length; termIndex++) {
+            const term = terms[termIndex];
+            const trimmedPreText = term.preText.trim();
+            if (trimmedPreText != "") {
+                fragments.push(trimmedPreText);
+                preprocessedFragments.push(trimmedPreText);
+            }
+            const termText = term.text;
+            if (termText != "") {
+                const substitutionPhonemes = CompromiseNLP.tryMatchInLexicons(term, lexicons, espeakVoice);
+                if (substitutionPhonemes) {
+                    phonemizedFragmentsSubstitutions.set(fragments.length, substitutionPhonemes);
+                    const referenceIPA = (await textToPhonemes(term.text, espeakVoice, true)).replaceAll("_", " ");
+                    const referenceKirshenbaum = (await textToPhonemes(term.text, espeakVoice, false)).replaceAll("_", "");
+                    logger.log(`\nSubstitution references: Target IPA: ${substitutionPhonemes.join(" ")}, IPA: ${referenceIPA}, Kirshenbaum: ${referenceKirshenbaum}`);
+                    const kirshenbaumPhonemes = substitutionPhonemes.map(phone => ipaPhoneToKirshenbaum(phone)).join("");
+                    logger.log(`Kirshenbaum conversion result: ${kirshenbaumPhonemes}`);
+                    const substitutionPhonemesFragment = ` [[${kirshenbaumPhonemes}]] `;
+                    preprocessedFragments.push(substitutionPhonemesFragment);
+                }
+                else {
+                    let normalizedFragment = undefined;
+                    if (normalize) {
+                        normalizedFragment = normalizationMap.get(termIndex);
+                    }
+                    if (normalizedFragment) {
+                        preprocessedFragments.push(normalizedFragment);
+                    }
+                    else {
+                        preprocessedFragments.push(termText);
+                    }
+                }
+                fragments.push(termText);
+            }
+            const trimmedPostText = term.postText.trim();
+            if (trimmedPostText != "") {
+                fragments.push(trimmedPostText);
+                preprocessedFragments.push(trimmedPostText);
+            }
+        }
+    }
+    else {
+        fragments = (await splitToWords(sentence, espeakVoice)).filter(word => word.trim() != "");
+        preprocessedFragments = fragments;
+    }
+    logger.start("Synthesize preprocessed fragments with Espeak");
+    const { rawAudio: referenceSynthesizedAudio, timeline: referenceTimeline } = await synthesizeFragments(preprocessedFragments, espeakVoice, false);
+    await logger.startAsync("Build phonemized tokens");
+    const phonemizedSentence = [];
+    let wordIndex = 0;
+    for (const phraseEntry of referenceTimeline) {
+        const phrase = [];
+        for (const wordEntry of phraseEntry.timeline) {
+            wordEntry.text = fragments[wordIndex];
+            if (phonemizedFragmentsSubstitutions.has(wordIndex)) {
+                phrase.push(phonemizedFragmentsSubstitutions.get(wordIndex));
+            }
+            else {
+                for (const tokenEntry of wordEntry.timeline) {
+                    const tokenPhonemes = [];
+                    for (const phoneme of tokenEntry.timeline) {
+                        if (phoneme.text) {
+                            tokenPhonemes.push(phoneme.text);
+                        }
+                    }
+                    if (tokenPhonemes.length > 0) {
+                        phrase.push(tokenPhonemes);
+                    }
+                }
+            }
+            wordIndex += 1;
+        }
+        if (phrase.length > 0) {
+            phonemizedSentence.push(phrase);
+        }
+    }
+    logger.log(phonemizedSentence.map(phrase => phrase.map(word => word.join(" ")).join(" | ")).join(" || "));
+    logger.end();
+    return { referenceSynthesizedAudio, referenceTimeline, fragments, preprocessedFragments, phonemizedFragmentsSubstitutions, phonemizedSentence };
+}
+export async function synthesizeFragments(fragments, voice, insertSeparators = false, rate = 150, pitch = 50, pitchRange = 50) {
+    const logger = new Logger();
+    await setVoice(voice);
+    await setRate(rate);
+    await setPitch(pitch);
+    await setPitchRange(pitchRange);
+    const sampleRate = await getSampleRate();
+    //fragments = fragments.filter(fragment => fragment.trim() != "")
+    if (fragments.length == 0) {
+        return {
+            rawAudio: getEmptyRawAudio(1, sampleRate),
+            timeline: [],
+            events: []
+        };
+    }
+    const { escape } = await import('html-escaper');
+    let textWithMarkers = '';
+    for (let i = 0; i < fragments.length; i++) {
+        //let fragment = escape(fragments[i])
+        let fragment = fragments[i];
+        if (insertSeparators) {
+            textWithMarkers += `<mark name="wordstart-${i}"/> | ${fragment} | <mark name="wordend-${i}"/>`;
+        }
+        else {
+            if (fragment.endsWith(".")) {
+                fragment += " ()";
+            }
+            textWithMarkers += `<mark name="wordstart-${i}"/>${fragment}<mark name="wordend-${i}"/> `;
+        }
+    }
+    //log(textWithMarkers)
+    const { rawAudio, events } = await synthesize(textWithMarkers, true);
+    const timeline = fragments.map(word => ({
+        type: "word",
+        text: word,
+        startTime: -1,
+        endTime: -1,
+        timeline: [{
+                type: "token",
+                text: "",
+                startTime: -1,
+                endTime: -1,
+                timeline: []
+            }]
+    }));
+    let wordIndex = 0;
+    const clauseEndIndexes = [];
+    for (const event of events) {
+        const eventTime = event.audio_position / 1000;
+        const currentWordEntry = timeline[wordIndex];
+        const currentSubwordTimeline = currentWordEntry.timeline;
+        const currentSubwordEntry = currentSubwordTimeline[currentSubwordTimeline.length - 1];
+        const currentPhoneTimeline = currentSubwordEntry.timeline;
+        const lastPhoneEntry = currentPhoneTimeline[currentPhoneTimeline.length - 1];
+        if (lastPhoneEntry && lastPhoneEntry.endTime == -1) {
+            lastPhoneEntry.endTime = eventTime;
+        }
+        if (event.type == "word") {
+            if (!event.id || currentPhoneTimeline.length == 0) {
+                continue;
+            }
+            if (currentSubwordEntry.endTime == -1) {
+                currentSubwordEntry.endTime = eventTime;
+            }
+            currentSubwordTimeline.push({
+                type: "token",
+                text: "",
+                startTime: eventTime,
+                endTime: -1,
+                timeline: []
+            });
+        }
+        else if (event.type == "phoneme") {
+            const phoneText = event.id;
+            if (!phoneText || phoneText.startsWith("(")) {
+                continue;
+            }
+            currentPhoneTimeline.push({
+                type: "phone",
+                text: phoneText,
+                startTime: eventTime,
+                endTime: -1
+            });
+            currentSubwordEntry.text += phoneText;
+            currentSubwordEntry.startTime = currentPhoneTimeline[0].startTime;
+        }
+        else if (event.type == "mark") {
+            const markerName = event.id;
+            if (markerName.startsWith("wordstart-")) {
+                const markerIndex = parseInt(markerName.substring(10));
+                if (markerIndex != wordIndex) {
+                    throw new Error(`Word start marker for index ${wordIndex} is not consistent with word index. The words were: ${objToString(fragments)}`);
+                }
+                if (currentPhoneTimeline.length > 0) {
+                    throw new Error(`Word entry ${wordIndex} already has phones before its start marker was seen. The words were: ${objToString(fragments)}`);
+                }
+                currentWordEntry.startTime = eventTime;
+                currentSubwordEntry.startTime = eventTime;
+            }
+            else if (markerName.startsWith("wordend-")) {
+                const markerIndex = parseInt(markerName.substring(8));
+                if (markerIndex != wordIndex) {
+                    throw new Error(`Word end marker for index ${wordIndex} is not consistent with word index. The words were: ${objToString(fragments)}`);
+                }
+                currentWordEntry.startTime = currentSubwordTimeline[0].startTime;
+                currentWordEntry.endTime = eventTime;
+                currentSubwordEntry.endTime = eventTime;
+                wordIndex += 1;
+                if (wordIndex == timeline.length) {
+                    break;
+                }
+            }
+            else {
+                continue;
+            }
+        }
+        else if (event.type == "end") {
+            clauseEndIndexes.push(wordIndex);
+        }
+    }
+    clauseEndIndexes.push(timeline.length);
+    const timelineWithClauses = [];
+    let clauseStartIndex = 0;
+    for (const clauseEndIndex of clauseEndIndexes) {
+        const newClause = {
+            type: "clause",
+            text: "",
+            startTime: -1,
+            endTime: -1,
+            timeline: []
+        };
+        for (let entryIndex = clauseStartIndex; entryIndex <= clauseEndIndex && entryIndex < timeline.length; entryIndex++) {
+            const wordEntry = timeline[entryIndex];
+            if (newClause.startTime == -1) {
+                newClause.startTime = wordEntry.startTime;
+            }
+            newClause.endTime = wordEntry.endTime;
+            newClause.text += `${wordEntry.text} `;
+            newClause.timeline.push(wordEntry);
+        }
+        if (newClause.timeline.length > 0) {
+            timelineWithClauses.push(newClause);
+            clauseStartIndex = clauseEndIndex + 1;
+        }
+    }
+    return { rawAudio, timeline: timelineWithClauses, events };
+}
+export async function textToIPA(text, voice) {
+    await setVoice(voice);
+    const { instance } = await getEspeakInstance();
+    const ipa = instance.synthesize_ipa(text).ipa.trim();
+    return ipa;
+}
+export async function textToPhonemes(text, voice, useIPA = true) {
+    await setVoice(voice);
+    const { instance, module } = await getEspeakInstance();
+    const textPtr = instance.convert_to_phonemes(text, useIPA);
+    const wasmMemory = new WasmMemoryManager(module);
+    const resultRef = wasmMemory.wrapNullTerminatedUtf8String(textPtr.ptr);
+    const result = resultRef.getValue();
+    wasmMemory.freeAll();
+    return result;
+}
+export async function setVoice(voiceId) {
+    const { instance } = await getEspeakInstance();
+    instance.set_voice(voiceId);
+}
+export async function setVolume(volume) {
+    const { instance } = await getEspeakInstance();
+    return instance.setVolume(volume);
+}
+export async function setRate(rate) {
+    const { instance } = await getEspeakInstance();
+    return instance.set_rate(rate);
+}
+export async function setPitch(pitch) {
+    const { instance } = await getEspeakInstance();
+    return instance.set_pitch(pitch);
+}
+export async function setPitchRange(pitchRange) {
+    const { instance } = await getEspeakInstance();
+    return instance.set_range(pitchRange);
+}
+export async function getSampleRate() {
+    return 22050;
+}
+export async function listVoices() {
+    const { instance } = await getEspeakInstance();
+    const voiceList = instance.list_voices();
+    return voiceList;
+}
+async function getEspeakInstance() {
+    if (!espeakInstance) {
+        const { default: EspeakInitializer } = await import('@echogarden/espeak-ng-emscripten');
+        const m = await EspeakInitializer();
+        espeakInstance = await (new m.eSpeakNGWorker());
+        espeakModule = m;
+    }
+    return { instance: espeakInstance, module: espeakModule };
+}
+export async function testEspeakSynthesisWithPrePhonemizedInputs(text) {
+    const ipaPhonemizedSentence = (await phonemizeSentence(text, "en-us")).flatMap(clause => clause);
+    const kirshenbaumPhonemizedSentence = (await phonemizeSentence(text, "en-us", undefined, false)).flatMap(clause => clause);
+    log(kirshenbaumPhonemizedSentence);
+    const fragments = ipaPhonemizedSentence.map(word => word.map(phoneme => ipaPhoneToKirshenbaum(phoneme)).join("")).map(word => ` [[${word}]] `);
+    const { rawAudio, timeline } = await synthesizeFragments(fragments, 'en-us');
+    await playAudioWithTimelinePhones(rawAudio, timeline);
+}
+export async function testKirshenbaumPhonemization(text) {
+    const ipaPhonemizedSentence = (await phonemizeSentence(text, "en-us")).flatMap(clause => clause);
+    const kirshenbaumPhonemizedSentence = (await phonemizeSentence(text, "en-us", undefined, false)).flatMap(clause => clause);
+    const ipaFragments = ipaPhonemizedSentence.map(word => word.join(""));
+    const kirshenbaumFragments = kirshenbaumPhonemizedSentence.map(word => word.join(""));
+    const fragments = ipaPhonemizedSentence.map(word => word.map(phoneme => ipaPhoneToKirshenbaum(phoneme)).join(""));
+    for (let i = 0; i < fragments.length; i++) {
+        log(`IPA: ${ipaFragments[i]} | converted: ${fragments[i]} | ground truth: ${kirshenbaumFragments[i]}`);
+    }
+}
+//# sourceMappingURL=EspeakTTS.js.map

package/dist/synthesis/EspeakTTS.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EspeakTTS.js","sourceRoot":"","sources":["../../src/synthesis/EspeakTTS.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAA;AACrE,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mCAAmC,CAAA;AACrE,OAAO,EAAY,gBAAgB,EAAE,MAAM,4BAA4B,CAAA;AACvE,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAA;AACrE,OAAO,KAAK,aAAa,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,4BAA4B,EAAE,MAAM,0BAA0B,CAAA;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAA;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAErD,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAI9D,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,IAAI,cAAmB,CAAA;AACvB,IAAI,YAAiB,CAAA;AAErB,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,WAAoB;IAClE,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAC3B,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAExC,IAAI,CAAC,WAAW,EAAE;QACjB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAA;QAE/C,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;KACnB;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE9C,MAAM,YAAY,GAAmB,EAAE,CAAA;IACvC,MAAM,SAAS,GAAkB,EAAE,CAAA;IAEnC,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAA;IAEtC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,OAAmB,EAAE,MAAqB,EAAE,EAAE;QACxE,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE;YAClC,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAA;SAC7C;QAED,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;YAC3B,IAAI,KAAK,CAAC,IAAI,IAAI,MAAM,EAAE;gBACzB,MAAM,YAAY,GAAG,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC;gBAC5C,KAAa,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,YAAY,GAAG,KAAK,CAAC,WAAW,CAAC,CAAA;aACvF;SACD;QAED,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;IAEF,MAAM,mBAAmB,GAAG,mBAAmB,CAAC,YAAY,CAAC,CAAA;IAE7D,MAAM,QAAQ,GAAa,EAAE,aAAa,EAAE,CAAC,mBAAmB,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAA;IAEtF,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,QAAgB,EAAE,WAAmB,EAAE,WAAsB,EAAE,EAAE,SAAS,GAAG,IAAI,EAAE,IAAI,GAAG,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,UAAU,GAAG,EAAE;IAC/K,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAA;IAEpD,IAAI,SAAmB,CAAA;IACvB,IAAI,qBAA+B,CAAA;IACnC,MAAM,gCAAgC,GAAG,IAAI,GAAG,EAAoB,CAAA;IAEpE,IAAI,WAAW,IAAI,OAAO,EAAE;QAC3B,SAAS,GAAG,EAAE,CAAA;QACd,qBAAqB,GAAG,EAAE,CAAA;QAE1B,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;QAC3D,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAE7C,MAAM,gBAAgB,GAAG,4BAA4B,CAAC,KAAK,EAAE,WAAW,CAAC,CAAA;QAEzE,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE;YAC9D,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAA;YAE7B,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;YAC1C,IAAI,cAAc,IAAI,EAAE,EAAE;gBACzB,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;gBAC9B,qBAAqB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;aAC1C;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAA;YAE1B,IAAI,QAAQ,IAAI,EAAE,EAAE;gBACnB,MAAM,oBAAoB,GAAG,aAAa,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAA;gBAE1F,IAAI,oBAAoB,EAAE;oBACzB,gCAAgC,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAA;oBAC5E,MAAM,YAAY,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;oBAC9F,MAAM,oBAAoB,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;oBAEtG,MAAM,CAAC,GAAG,CAAC,0CAA0C,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,YAAY,kBAAkB,oBAAoB,EAAE,CAAC,CAAA;oBAElJ,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;oBAEpG,MAAM,CAAC,GAAG,CAAC,kCAAkC,mBAAmB,EAAE,CAAC,CAAA;oBAEnE,MAAM,4BAA4B,GAAG,MAAM,mBAAmB,KAAK,CAAA;oBAEnE,qBAAqB,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAA;iBACxD;qBAAM;oBACN,IAAI,kBAAkB,GAAuB,SAAS,CAAA;oBAEtD,IAAI,SAAS,EAAE;wBACd,kBAAkB,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;qBACpD;oBAED,IAAI,kBAAkB,EAAE;wBACvB,qBAAqB,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAA;qBAC9C;yBAAM;wBACN,qBAAqB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;qBACpC;iBACD;gBAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;aACxB;YAED,MAAM,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;YAC5C,IAAI,eAAe,IAAI,EAAE,EAAE;gBAC1B,SAAS,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;gBAC/B,qBAAqB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;aAC3C;SACD;KACD;SAAM;QACN,SAAS,GAAG,CAAC,MAAM,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACzF,qBAAqB,GAAG,SAAS,CAAA;KACjC;IAED,MAAM,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAA;IAE7D,MAAM,EAAE,QAAQ,EAAE,yBAAyB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,MAAM,mBAAmB,CAAC,qBAAqB,EAAE,WAAW,EAAE,KAAK,CAAC,CAAA;IAEjJ,MAAM,MAAM,CAAC,UAAU,CAAC,yBAAyB,CAAC,CAAA;IAElD,MAAM,kBAAkB,GAAiB,EAAE,CAAA;IAE3C,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,KAAK,MAAM,WAAW,IAAI,iBAAiB,EAAE;QAC5C,MAAM,MAAM,GAAe,EAAE,CAAA;QAE7B,KAAK,MAAM,SAAS,IAAI,WAAW,CAAC,QAAS,EAAE;YAC9C,SAAS,CAAC,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;YAErC,IAAI,gCAAgC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBACpD,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC,CAAA;aAC7D;iBAAM;gBACN,KAAK,MAAM,UAAU,IAAI,SAAS,CAAC,QAAS,EAAE;oBAC7C,MAAM,aAAa,GAAa,EAAE,CAAA;oBAElC,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAS,EAAE;wBAC3C,IAAI,OAAO,CAAC,IAAI,EAAE;4BACjB,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;yBAChC;qBACD;oBAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE;wBAC7B,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;qBAC1B;iBACD;aACD;YAED,SAAS,IAAI,CAAC,CAAA;SACd;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACtB,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;SAC/B;KACD;IAED,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAEzG,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,EAAE,yBAAyB,EAAE,iBAAiB,EAAE,SAAS,EAAE,qBAAqB,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,CAAA;AAChJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,SAAmB,EAAE,KAAa,EAAE,gBAAgB,GAAG,KAAK,EAAE,IAAI,GAAG,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,UAAU,GAAG,EAAE;IAC9I,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrB,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;IACnB,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrB,MAAM,aAAa,CAAC,UAAU,CAAC,CAAA;IAE/B,MAAM,UAAU,GAAG,MAAM,aAAa,EAAE,CAAA;IAExC,iEAAiE;IAEjE,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE;QAC1B,OAAO;YACN,QAAQ,EAAE,gBAAgB,CAAC,CAAC,EAAE,UAAU,CAAC;YACzC,QAAQ,EAAE,EAAc;YACxB,MAAM,EAAG,EAAmB;SAC5B,CAAA;KACD;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAA;IAE/C,IAAI,eAAe,GAAG,EAAE,CAAA;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAC1C,qCAAqC;QACrC,IAAI,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAA;QAE3B,IAAI,gBAAgB,EAAE;YACrB,eAAe,IAAI,yBAAyB,CAAC,SAAS,QAAQ,0BAA0B,CAAC,KAAK,CAAA;SAC9F;aAAM;YACN,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;gBAC3B,QAAQ,IAAI,KAAK,CAAA;aACjB;YAED,eAAe,IAAI,yBAAyB,CAAC,MAAM,QAAQ,uBAAuB,CAAC,MAAM,CAAA;SACzF;KACD;IAED,sBAAsB;IAEtB,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,IAAI,CAAC,CAAA;IAEpE,MAAM,QAAQ,GAAa,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjD,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,IAAI;QACV,SAAS,EAAE,CAAC,CAAC;QACb,OAAO,EAAE,CAAC,CAAC;QACX,QAAQ,EAAE,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,EAAE;gBACR,SAAS,EAAE,CAAC,CAAC;gBACb,OAAO,EAAE,CAAC,CAAC;gBACX,QAAQ,EAAE,EAAE;aACZ,CAAC;KACF,CAAC,CAAC,CAAA;IAEH,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,MAAM,gBAAgB,GAAa,EAAE,CAAA;IAErC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;QAC3B,MAAM,SAAS,GAAG,KAAK,CAAC,cAAc,GAAG,IAAI,CAAA;QAE7C,MAAM,gBAAgB,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAE5C,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,QAAS,CAAA;QACzD,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAErF,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,QAAS,CAAA;QAC1D,MAAM,cAAc,GAAG,oBAAoB,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE5E,IAAI,cAAc,IAAI,cAAc,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE;YACnD,cAAc,CAAC,OAAO,GAAG,SAAS,CAAA;SAClC;QAED,IAAI,KAAK,CAAC,IAAI,IAAI,MAAM,EAAE;YACzB,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,oBAAoB,CAAC,MAAM,IAAI,CAAC,EAAE;gBAClD,SAAQ;aACR;YAED,IAAI,mBAAmB,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE;gBACtC,mBAAmB,CAAC,OAAO,GAAG,SAAS,CAAA;aACvC;YAED,sBAAsB,CAAC,IAAI,CAAC;gBAC3B,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,EAAE;gBACR,SAAS,EAAE,SAAS;gBACpB,OAAO,EAAE,CAAC,CAAC;gBACX,QAAQ,EAAE,EAAE;aACZ,CAAC,CAAA;SACF;aAAM,IAAI,KAAK,CAAC,IAAI,IAAI,SAAS,EAAE;YACnC,MAAM,SAAS,GAAG,KAAK,CAAC,EAAY,CAAA;YAEpC,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBAC5C,SAAQ;aACR;YAED,oBAAoB,CAAC,IAAI,CAAC;gBACzB,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE,SAAS;gBACpB,OAAO,EAAE,CAAC,CAAC;aACX,CAAC,CAAA;YAEF,mBAAmB,CAAC,IAAI,IAAI,SAAS,CAAA;YACrC,mBAAmB,CAAC,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;SACjE;aAAM,IAAI,KAAK,CAAC,IAAI,IAAI,MAAM,EAAE;YAChC,MAAM,UAAU,GAAG,KAAK,CAAC,EAAa,CAAA;YAEtC,IAAI,UAAU,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;gBACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAA;gBAEtD,IAAI,WAAW,IAAI,SAAS,EAAE;oBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,SAAS,uDAAuD,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;iBACxI;gBAED,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE;oBACpC,MAAM,IAAI,KAAK,CAAC,cAAc,SAAS,yEAAyE,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;iBACzI;gBAED,gBAAgB,CAAC,SAAS,GAAG,SAAS,CAAA;gBACtC,mBAAmB,CAAC,SAAS,GAAG,SAAS,CAAA;aACzC;iBAAM,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBAC7C,MAAM,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAA;gBAErD,IAAI,WAAW,IAAI,SAAS,EAAE;oBAC7B,MAAM,IAAI,KAAK,CAAC,6BAA6B,SAAS,uDAAuD,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;iBACtI;gBAED,gBAAgB,CAAC,SAAS,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;gBAEhE,gBAAgB,CAAC,OAAO,GAAG,SAAS,CAAA;gBACpC,mBAAmB,CAAC,OAAO,GAAG,SAAS,CAAA;gBAEvC,SAAS,IAAI,CAAC,CAAA;gBAEd,IAAI,SAAS,IAAI,QAAQ,CAAC,MAAM,EAAE;oBACjC,MAAK;iBACL;aACD;iBAAM;gBACN,SAAQ;aACR;SACD;aAAM,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,EAAE;YAC/B,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;SAChC;KACD;IAED,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAEtC,MAAM,mBAAmB,GAAa,EAAE,CAAA;IAExC,IAAI,gBAAgB,GAAG,CAAC,CAAA;IAExB,KAAK,MAAM,cAAc,IAAI,gBAAgB,EAAE;QAC9C,MAAM,SAAS,GAAkB;YAChC,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,EAAE;YACR,SAAS,EAAE,CAAC,CAAC;YACb,OAAO,EAAE,CAAC,CAAC;YACX,QAAQ,EAAE,EAAE;SACZ,CAAA;QAED,KAAK,IAAI,UAAU,GAAG,gBAAgB,EAAE,UAAU,IAAI,cAAc,IAAI,UAAU,GAAG,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,EAAE;YACnH,MAAM,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAA;YACtC,IAAI,SAAS,CAAC,SAAS,IAAI,CAAC,CAAC,EAAE;gBAC9B,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAA;aACzC;YAED,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,OAAO,CAAA;YAErC,SAAS,CAAC,IAAI,IAAI,GAAG,SAAS,CAAC,IAAI,GAAG,CAAA;YAEtC,SAAS,CAAC,QAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;SACnC;QAED,IAAI,SAAS,CAAC,QAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACnC,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACnC,gBAAgB,GAAG,cAAc,GAAG,CAAC,CAAA;SACrC;KACD;IAED,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,EAAE,CAAA;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrB,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAC9C,MAAM,GAAG,GAAY,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,GAAc,CAAC,IAAI,EAAE,CAAA;IAExE,OAAO,GAAG,CAAA;AACX,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,KAAa,EAAE,MAAM,GAAG,IAAI;IAC9E,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;IACrB,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IACtD,MAAM,OAAO,GAAG,QAAQ,CAAC,mBAAmB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE1D,MAAM,UAAU,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAA;IAEhD,MAAM,SAAS,GAAG,UAAU,CAAC,4BAA4B,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IACtE,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,CAAA;IAEnC,UAAU,CAAC,OAAO,EAAE,CAAA;IAEpB,OAAO,MAAM,CAAA;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAe;IAC7C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE9C,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc;IAC7C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE9C,OAAO,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAY;IACzC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE9C,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAa;IAC3C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE9C,OAAO,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,UAAkB;IACrD,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE9C,OAAO,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAA;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa;IAClC,OAAO,KAAK,CAAA;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU;IAC/B,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,iBAAiB,EAAE,CAAA;IAE9C,MAAM,SAAS,GAA4F,QAAQ,CAAC,WAAW,EAAE,CAAA;IAEjI,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,KAAK,UAAU,iBAAiB;IAC/B,IAAI,CAAC,cAAc,EAAE;QACpB,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;QAEvF,MAAM,CAAC,GAAG,MAAM,iBAAiB,EAAE,CAAA;QACnC,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC,CAAA;QAC/C,YAAY,GAAG,CAAC,CAAA;KAChB;IAED,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,CAAA;AAC1D,CAAC;AAoBD,MAAM,CAAC,KAAK,UAAU,0CAA0C,CAAC,IAAY;IAC5E,MAAM,qBAAqB,GAAG,CAAC,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAA;IAChG,MAAM,6BAA6B,GAAG,CAAC,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAA;IAC1H,GAAG,CAAC,6BAA6B,CAAC,CAAA;IAElC,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAClB,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,IAAI,KAAK,CAAC,CAAA;IAExE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,mBAAmB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAE5E,MAAM,2BAA2B,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,IAAY;IAC9D,MAAM,qBAAqB,GAAG,CAAC,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAA;IAChG,MAAM,6BAA6B,GAAG,CAAC,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAA;IAE1H,MAAM,YAAY,GAAG,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAErE,MAAM,oBAAoB,GAAG,6BAA6B,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAErF,MAAM,SAAS,GAAG,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAClD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAClB,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAC1C,GAAG,CAAC,QAAQ,YAAY,CAAC,CAAC,CAAC,iBAAiB,SAAS,CAAC,CAAC,CAAC,oBAAoB,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;KACtG;AACF,CAAC"}

package/dist/synthesis/FliteTTS.d.ts

@@ -0,0 +1,17 @@
+import { SynthesisVoice } from "../api/API.js";
+export type FliteVoiceName = "kal" | "kal16" | "awb" | "rms" | "slt" | string;
+export declare function synthesize(text: string, voice: FliteVoiceName, voiceDir: string | undefined, rate: number, pitchMeanHz?: number, pitchStdDev?: number): Promise<{
+    rawAudio: {
+        audioChannels: Float32Array[];
+        sampleRate: number;
+    };
+    events: FliteEvent[];
+}>;
+export type FliteEventType = "phone" | "pause" | "phrasePause";
+export type FliteEvent = {
+    type: FliteEventType;
+    id: string;
+    startTime: number;
+    endTime: number;
+};
+export declare const voices: SynthesisVoice[];