echogarden 1.4.4 → 1.6.0

package/dist/alignment/SemanticTextAlignment.js

@@ -0,0 +1,336 @@
+import { Logger } from '../utilities/Logger.js';
+import { loadPackage } from '../utilities/PackageManager.js';
+import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js';
+import { cosineDistance } from '../math/VectorMath.js';
+import { isPunctuation, isWord, splitToSentences, splitToWords } from '../nlp/Segmentation.js';
+import { extractEntries } from '../utilities/Timeline.js';
+export async function alignTimelineToTextSemantically(timeline, text, textLangCode) {
+    const logger = new Logger();
+    logger.start(`Prepare text for semantic alignment`);
+    const timelineSentenceEntries = extractEntries(timeline, entry => entry.type === 'sentence');
+    const timelineWordEntryGroups = [];
+    const timelineWordGroups = [];
+    for (const sentenceEntry of timelineSentenceEntries) {
+        const wordEntryGroup = sentenceEntry.timeline
+            .filter(wordEntry => isWord(wordEntry.text));
+        timelineWordEntryGroups.push(wordEntryGroup);
+        timelineWordGroups.push(wordEntryGroup.map(wordEntry => wordEntry.text));
+    }
+    const timelineWordEntriesFiltered = timelineWordEntryGroups.flat();
+    const textSentences = splitToSentences(text, textLangCode);
+    const textWordGroups = [];
+    for (const sentenceText of textSentences) {
+        let wordGroup = await splitToWords(sentenceText, textLangCode);
+        wordGroup = wordGroup.filter(word => isWord(word));
+        textWordGroups.push(wordGroup);
+    }
+    const textWords = textWordGroups.flat();
+    logger.end();
+    const wordMappingEntries = await alignWordsToWordsSemantically(timelineWordGroups, textWordGroups);
+    logger.start(`Build timeline for translation`);
+    const mappingGroups = new Map();
+    for (const wordMappingEntry of wordMappingEntries) {
+        const wordIndex1 = wordMappingEntry.wordIndex1;
+        const wordIndex2 = wordMappingEntry.wordIndex2;
+        let group = mappingGroups.get(wordIndex1);
+        if (!group) {
+            group = [];
+            mappingGroups.set(wordIndex1, group);
+        }
+        if (!group.includes(wordIndex2)) {
+            group.push(wordIndex2);
+        }
+    }
+    const timeSlicesLookup = new Map();
+    for (const [wordIndex1, mappedWordIndexes] of mappingGroups) {
+        if (mappedWordIndexes.length === 0) {
+            continue;
+        }
+        const startTime = timelineWordEntriesFiltered[wordIndex1].startTime;
+        const endTime = timelineWordEntriesFiltered[wordIndex1].endTime;
+        const splitCount = mappedWordIndexes.length;
+        const sliceDuration = (endTime - startTime) / splitCount;
+        let timeOffset = 0;
+        for (let i = 0; i < splitCount; i++) {
+            const timeSlice = {
+                startTime: startTime + timeOffset,
+                endTime: startTime + timeOffset + sliceDuration
+            };
+            const wordIndex2 = mappedWordIndexes[i];
+            let timeSlicesForTargetWord = timeSlicesLookup.get(wordIndex2);
+            if (!timeSlicesForTargetWord) {
+                timeSlicesForTargetWord = [];
+                timeSlicesLookup.set(wordIndex2, timeSlicesForTargetWord);
+            }
+            timeSlicesForTargetWord.push(timeSlice);
+            timeOffset += sliceDuration;
+        }
+    }
+    const resultTimeline = [];
+    for (const [key, value] of timeSlicesLookup) {
+        resultTimeline.push({
+            type: 'word',
+            text: textWords[key],
+            startTime: value[0].startTime,
+            endTime: value[value.length - 1].endTime
+        });
+    }
+    logger.end();
+    return resultTimeline;
+}
+export async function alignWordsToWordsSemantically(wordsGroups1, wordsGroups2, windowTokenCount = 20000) {
+    const logger = new Logger();
+    // Load embedding model
+    const modelPath = await loadPackage(`xenova-multilingual-e5-small-fp16`);
+    const embeddingModel = new E5TextEmbedding(modelPath);
+    logger.start(`Initialize E5 embedding model`);
+    await embeddingModel.initializeIfNeeded();
+    async function extractEmbeddingsFromWordGroups(wordGroups) {
+        const logger = new Logger();
+        const maxTokensPerFragment = 512;
+        const { Tensor } = await import('@echogarden/transformers-nodejs-lite');
+        const words = [];
+        const embeddings = [];
+        const tokenToWordIndexMapping = [];
+        for (const wordGroup of wordGroups) {
+            const { joinedText: joinedTextForGroup, offsets: offsetsForGroup } = joinAndGetOffsets(wordGroup);
+            logger.start(`Tokenize text`);
+            const inputsForGroup = await embeddingModel.tokenizeToModelInputs(joinedTextForGroup);
+            logger.start(`Infer embeddings for text`);
+            const allTokenIds = inputsForGroup['input_ids'].data;
+            const allAttentionMask = inputsForGroup['attention_mask'].data;
+            let embeddingsForGroup = [];
+            for (let tokenStart = 0; tokenStart < allTokenIds.length; tokenStart += maxTokensPerFragment) {
+                const tokenEnd = Math.min(tokenStart + maxTokensPerFragment, allTokenIds.length);
+                const fragmentTokenCount = tokenEnd - tokenStart;
+                const fragmentInputIdsTensor = new Tensor('int64', allTokenIds.slice(tokenStart, tokenEnd), [1, fragmentTokenCount]);
+                const fragmentAttentionMaskTensor = new Tensor('int64', allAttentionMask.slice(tokenStart, tokenEnd), [1, fragmentTokenCount]);
+                const inputsForFragment = { input_ids: fragmentInputIdsTensor, attention_mask: fragmentAttentionMaskTensor };
+                const embeddingsForFragment = await embeddingModel.inferTokenEmbeddings(inputsForFragment);
+                embeddingsForGroup.push(...embeddingsForFragment);
+            }
+            logger.start(`Compute token to word mapping for text`);
+            const filteredEmbeddingsForGroup = embeddingsForGroup.filter((embedding) => embedding.text !== '▁' && embedding.text !== '<s>' && embedding.text !== '</s>');
+            const tokenToWordIndexMappingForGroup = mapTokenEmbeddingsToWordIndexes(filteredEmbeddingsForGroup, joinedTextForGroup, offsetsForGroup);
+            const tokenToWordIndexMappingForGroupWithOffset = tokenToWordIndexMappingForGroup.map(value => words.length + value);
+            embeddings.push(...filteredEmbeddingsForGroup);
+            tokenToWordIndexMapping.push(...tokenToWordIndexMappingForGroupWithOffset);
+            words.push(...wordGroup);
+        }
+        return { words, embeddings, tokenToWordIndexMapping };
+    }
+    logger.start(`Extract embeddings from source 1`);
+    const { words: words1, embeddings: embeddings1, tokenToWordIndexMapping: tokenToWordIndexMapping1 } = await extractEmbeddingsFromWordGroups(wordsGroups1);
+    logger.start(`Extract embeddings from source 2`);
+    const { words: words2, embeddings: embeddings2, tokenToWordIndexMapping: tokenToWordIndexMapping2 } = await extractEmbeddingsFromWordGroups(wordsGroups2);
+    // Align
+    function costFunction(a, b) {
+        const aIsPunctuation = isPunctuation(a.text);
+        const bIsPunctuation = isPunctuation(b.text);
+        if (aIsPunctuation === bIsPunctuation) {
+            return cosineDistance(a.embeddingVector, b.embeddingVector);
+        }
+        else {
+            return 1.0;
+        }
+    }
+    logger.start(`Align token embedding vectors using DTW`);
+    const { path } = alignDTWWindowed(embeddings1, embeddings2, costFunction, windowTokenCount);
+    // Use alignment path to words to words
+    logger.start(`Map tokens to words`);
+    const wordMapping = [];
+    for (let i = 0; i < path.length; i++) {
+        const pathEntry = path[i];
+        const sourceTokenIndex = pathEntry.source;
+        const destTokenIndex = pathEntry.dest;
+        const mappedWordIndex1 = tokenToWordIndexMapping1[sourceTokenIndex];
+        const mappedWordIndex2 = tokenToWordIndexMapping2[destTokenIndex];
+        wordMapping.push({
+            wordIndex1: mappedWordIndex1,
+            word1: words1[mappedWordIndex1],
+            wordIndex2: mappedWordIndex2,
+            word2: words2[mappedWordIndex2],
+        });
+    }
+    logger.end();
+    return wordMapping;
+}
+function mapTokenEmbeddingsToWordIndexes(embeddings, text, textWordOffsets) {
+    const tokenToWordIndex = [];
+    let currentTextOffset = 0;
+    for (let i = 0; i < embeddings.length; i++) {
+        const embedding = embeddings[i];
+        let tokenText = embedding.text;
+        if (tokenText === '<s>' || tokenText === '</s>') {
+            tokenToWordIndex.push(-1);
+            continue;
+        }
+        if (tokenText.startsWith('▁')) {
+            tokenText = tokenText.substring(1);
+        }
+        const matchPosition = text.indexOf(tokenText, currentTextOffset);
+        if (matchPosition === -1) {
+            throw new Error(`Token '${tokenText}' not found in text`);
+        }
+        currentTextOffset = matchPosition + tokenText.length;
+        let tokenMatchingWordIndex = textWordOffsets.findIndex((index) => index > matchPosition);
+        if (tokenMatchingWordIndex === -1) {
+            throw new Error(`Token '${tokenText}' not found in text`);
+        }
+        else {
+            tokenMatchingWordIndex = Math.max(tokenMatchingWordIndex - 1, 0);
+        }
+        tokenToWordIndex.push(tokenMatchingWordIndex);
+    }
+    return tokenToWordIndex;
+}
+function joinAndGetOffsets(words) {
+    let joinedText = '';
+    const offsets = [];
+    let offset = 0;
+    for (const word of words) {
+        const extendedWord = `${word} `;
+        joinedText += extendedWord;
+        offsets.push(offset);
+        offset += extendedWord.length;
+    }
+    offsets.push(joinedText.length);
+    return { joinedText, offsets };
+}
+export class E5TextEmbedding {
+    modelPath;
+    tokenizer;
+    model;
+    constructor(modelPath) {
+        this.modelPath = modelPath;
+    }
+    async tokenizeToModelInputs(text) {
+        await this.initializeIfNeeded();
+        const inputs = await this.tokenizer(text);
+        return inputs;
+    }
+    async inferTokenEmbeddings(inputs) {
+        await this.initializeIfNeeded();
+        const tokensText = this.tokenizer.model.convert_ids_to_tokens(Array.from(inputs.input_ids.data));
+        const result = await this.model(inputs);
+        const lastHiddenState = result.last_hidden_state;
+        const tokenCount = lastHiddenState.dims[1];
+        const embeddingSize = lastHiddenState.dims[2];
+        const tokenEmbeddings = [];
+        for (let i = 0; i < tokenCount; i++) {
+            const tokenEmbeddingVector = lastHiddenState.data.slice(i * embeddingSize, (i + 1) * embeddingSize);
+            const tokenId = Number(inputs.input_ids.data[i]);
+            const tokenText = tokensText[i];
+            tokenEmbeddings.push({
+                id: tokenId,
+                text: tokenText,
+                embeddingVector: tokenEmbeddingVector
+            });
+        }
+        return tokenEmbeddings;
+    }
+    async initializeIfNeeded() {
+        if (this.tokenizer && this.model) {
+            return;
+        }
+        const { AutoTokenizer, AutoModel } = await import('@echogarden/transformers-nodejs-lite');
+        this.tokenizer = await AutoTokenizer.from_pretrained(this.modelPath);
+        this.model = await AutoModel.from_pretrained(this.modelPath);
+    }
+}
+export const e5SupportedLanguages = [
+    'af', // Afrikaans
+    'am', // Amharic
+    'ar', // Arabic
+    'as', // Assamese
+    'az', // Azerbaijani
+    'be', // Belarusian
+    'bg', // Bulgarian
+    'bn', // Bengali
+    'br', // Breton
+    'bs', // Bosnian
+    'ca', // Catalan
+    'cs', // Czech
+    'cy', // Welsh
+    'da', // Danish
+    'de', // German
+    'el', // Greek
+    'en', // English
+    'eo', // Esperanto
+    'es', // Spanish
+    'et', // Estonian
+    'eu', // Basque
+    'fa', // Persian
+    'fi', // Finnish
+    'fr', // French
+    'fy', // Western Frisian
+    'ga', // Irish
+    'gd', // Scottish Gaelic
+    'gl', // Galician
+    'gu', // Gujarati
+    'ha', // Hausa
+    'he', // Hebrew
+    'hi', // Hindi
+    'hr', // Croatian
+    'hu', // Hungarian
+    'hy', // Armenian
+    'id', // Indonesian
+    'is', // Icelandic
+    'it', // Italian
+    'ja', // Japanese
+    'jv', // Javanese
+    'ka', // Georgian
+    'kk', // Kazakh
+    'km', // Khmer
+    'kn', // Kannada
+    'ko', // Korean
+    'ku', // Kurdish
+    'ky', // Kyrgyz
+    'la', // Latin
+    'lo', // Lao
+    'lt', // Lithuanian
+    'lv', // Latvian
+    'mg', // Malagasy
+    'mk', // Macedonian
+    'ml', // Malayalam
+    'mn', // Mongolian
+    'mr', // Marathi
+    'ms', // Malay
+    'my', // Burmese
+    'ne', // Nepali
+    'nl', // Dutch
+    'no', // Norwegian
+    'om', // Oromo
+    'or', // Oriya
+    'pa', // Panjabi
+    'pl', // Polish
+    'ps', // Pashto
+    'pt', // Portuguese
+    'ro', // Romanian
+    'ru', // Russian
+    'sa', // Sanskrit
+    'sd', // Sindhi
+    'si', // Sinhala
+    'sk', // Slovak
+    'sl', // Slovenian
+    'so', // Somali
+    'sq', // Albanian
+    'sr', // Serbian
+    'su', // Sundanese
+    'sv', // Swedish
+    'sw', // Swahili
+    'ta', // Tamil
+    'te', // Telugu
+    'th', // Thai
+    'tl', // Tagalog
+    'tr', // Turkish
+    'ug', // Uyghur
+    'uk', // Ukrainian
+    'ur', // Urdu
+    'uz', // Uzbek
+    'vi', // Vietnamese
+    'xh', // Xhosa
+    'yi', // Yiddish
+    'zh', // Chinese
+];
+//# sourceMappingURL=SemanticTextAlignment.js.map

package/dist/alignment/SemanticTextAlignment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SemanticTextAlignment.js","sourceRoot":"","sources":["../../src/alignment/SemanticTextAlignment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAA;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAC9F,OAAO,EAAY,cAAc,EAAE,MAAM,0BAA0B,CAAA;AAEnE,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,QAAkB,EAAE,IAAY,EAAE,YAAoB;IAC3G,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;IAEnD,MAAM,uBAAuB,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,CAAC,CAAA;IAE5F,MAAM,uBAAuB,GAAe,EAAE,CAAA;IAC9C,MAAM,kBAAkB,GAAe,EAAE,CAAA;IAEzC,KAAK,MAAM,aAAa,IAAI,uBAAuB,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,aAAa,CAAC,QAAS;aAC5C,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;QAE7C,uBAAuB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QAC5C,kBAAkB,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;IACzE,CAAC;IAED,MAAM,2BAA2B,GAAG,uBAAuB,CAAC,IAAI,EAAE,CAAA;IAElE,MAAM,aAAa,GAAG,gBAAgB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IAE1D,MAAM,cAAc,GAAe,EAAE,CAAA;IAErC,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;QAC1C,IAAI,SAAS,GAAG,MAAM,YAAY,CAAC,YAAY,EAAE,YAAY,CAAC,CAAA;QAC9D,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QAElD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAC/B,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,EAAE,CAAA;IAEvC,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,MAAM,kBAAkB,GAAG,MAAM,6BAA6B,CAAC,kBAAkB,EAAE,cAAc,CAAC,CAAA;IAElG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAA;IAE9C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAA;IAEjD,KAAK,MAAM,gBAAgB,IAAI,kBAAkB,EAAE,CAAC;QACnD,MAAM,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAA;QAC9C,MAAM,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAA;QAE9C,IAAI,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAEzC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,CAAA;YACV,aAAa,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;QACrC,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACvB,CAAC;IACF,CAAC;IAID,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAuB,CAAA;IAEvD,KAAK,MAAM,CAAC,UAAU,EAAE,iBAAiB,CAAC,IAAI,aAAa,EAAE,CAAC;QAC7D,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,SAAQ;QACT,CAAC;QAED,MAAM,SAAS,GAAG,2BAA2B,CAAC,UAAU,CAAC,CAAC,SAAS,CAAA;QACnE,MAAM,OAAO,GAAG,2BAA2B,CAAC,UAAU,CAAC,CAAC,OAAO,CAAA;QAE/D,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAA;QAE3C,MAAM,aAAa,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,GAAG,UAAU,CAAA;QAExD,IAAI,UAAU,GAAG,CAAC,CAAA;QAElB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,SAAS,GAAc;gBAC5B,SAAS,EAAE,SAAS,GAAG,UAAU;gBACjC,OAAO,EAAE,SAAS,GAAG,UAAU,GAAG,aAAa;aAC/C,CAAA;YAED,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAA;YAEvC,IAAI,uBAAuB,GAAG,gBAAgB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;YAE9D,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAC9B,uBAAuB,GAAG,EAAE,CAAA;gBAC5B,gBAAgB,CAAC,GAAG,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAA;YAC1D,CAAC;YAED,uBAAuB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAEvC,UAAU,IAAI,aAAa,CAAA;QAC5B,CAAC;IACF,CAAC;IAED,MAAM,cAAc,GAAa,EAAE,CAAA;IAEnC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,gBAAgB,EAAE,CAAC;QAC7C,cAAc,CAAC,IAAI,CAAC;YACnB,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC;YACpB,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;YAC7B,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO;SACxC,CAAC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,cAAc,CAAA;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,YAAwB,EAAE,YAAwB,EAAE,gBAAgB,GAAG,KAAK;IAC/H,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,uBAAuB;IACvB,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,mCAAmC,CAAC,CAAA;IAExE,MAAM,cAAc,GAAG,IAAI,eAAe,CAAC,SAAS,CAAC,CAAA;IAErD,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;IAC7C,MAAM,cAAc,CAAC,kBAAkB,EAAE,CAAA;IAEzC,KAAK,UAAU,+BAA+B,CAAC,UAAsB;QACpE,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;QAE3B,MAAM,oBAAoB,GAAG,GAAG,CAAA;QAChC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAA;QAEvE,MAAM,KAAK,GAAa,EAAE,CAAA;QAE1B,MAAM,UAAU,GAAyB,EAAE,CAAA;QAC3C,MAAM,uBAAuB,GAAa,EAAE,CAAA;QAE5C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACpC,MAAM,EAAE,UAAU,EAAE,kBAAkB,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAA;YAEjG,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;YAC7B,MAAM,cAAc,GAAG,MAAM,cAAc,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,CAAA;YAErF,MAAM,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAA;YAEzC,MAAM,WAAW,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC,IAAI,CAAA;YACpD,MAAM,gBAAgB,GAAG,cAAc,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAA;YAE9D,IAAI,kBAAkB,GAAyB,EAAE,CAAA;YAEjD,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,WAAW,CAAC,MAAM,EAAE,UAAU,IAAI,oBAAoB,EAAE,CAAC;gBAC9F,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,oBAAoB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;gBAChF,MAAM,kBAAkB,GAAG,QAAQ,GAAG,UAAU,CAAA;gBAEhD,MAAM,sBAAsB,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAA;gBACpH,MAAM,2BAA2B,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,CAAA;gBAE9H,MAAM,iBAAiB,GAAG,EAAE,SAAS,EAAE,sBAAsB,EAAE,cAAc,EAAE,2BAA2B,EAAE,CAAA;gBAE5G,MAAM,qBAAqB,GAAG,MAAM,cAAc,CAAC,oBAAoB,CAAC,iBAAiB,CAAC,CAAA;gBAE1F,kBAAkB,CAAC,IAAI,CAAC,GAAG,qBAAqB,CAAC,CAAA;YAClD,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;YACtD,MAAM,0BAA0B,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,GAAG,IAAI,SAAS,CAAC,IAAI,KAAK,KAAK,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;YAC5J,MAAM,+BAA+B,GAAG,+BAA+B,CAAC,0BAA0B,EAAE,kBAAkB,EAAE,eAAe,CAAC,CAAA;YACxI,MAAM,yCAAyC,GAAG,+BAA+B,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,CAAA;YAEpH,UAAU,CAAC,IAAI,CAAC,GAAG,0BAA0B,CAAC,CAAA;YAC9C,uBAAuB,CAAC,IAAI,CAAC,GAAG,yCAAyC,CAAC,CAAA;YAE1E,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAA;QACzB,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,uBAAuB,EAAE,CAAA;IACtD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;IAChD,MAAM,EACL,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,WAAW,EACvB,uBAAuB,EAAE,wBAAwB,EACjD,GAAG,MAAM,+BAA+B,CAAC,YAAY,CAAC,CAAA;IAEvD,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;IAChD,MAAM,EACL,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,WAAW,EACvB,uBAAuB,EAAE,wBAAwB,EACjD,GAAG,MAAM,+BAA+B,CAAC,YAAY,CAAC,CAAA;IAEvD,QAAQ;IACR,SAAS,YAAY,CAAC,CAAqB,EAAE,CAAqB;QACjE,MAAM,cAAc,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC5C,MAAM,cAAc,GAAG,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAE5C,IAAI,cAAc,KAAK,cAAc,EAAE,CAAC;YACvC,OAAO,cAAc,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,CAAC,CAAA;QAC5D,CAAC;aAAM,CAAC;YACP,OAAO,GAAG,CAAA;QACX,CAAC;IACF,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAA;IAEvD,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,gBAAgB,CAAC,CAAA;IAE3F,uCAAuC;IACvC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAEnC,MAAM,WAAW,GAAkB,EAAE,CAAA;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEzB,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAA;QACzC,MAAM,cAAc,GAAG,SAAS,CAAC,IAAI,CAAA;QAErC,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,gBAAgB,CAAC,CAAA;QACnE,MAAM,gBAAgB,GAAG,wBAAwB,CAAC,cAAc,CAAC,CAAA;QAEjE,WAAW,CAAC,IAAI,CAAC;YAChB,UAAU,EAAE,gBAAgB;YAC5B,KAAK,EAAE,MAAM,CAAC,gBAAgB,CAAC;YAC/B,UAAU,EAAE,gBAAgB;YAC5B,KAAK,EAAE,MAAM,CAAC,gBAAgB,CAAC;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO,WAAW,CAAA;AACnB,CAAC;AAED,SAAS,+BAA+B,CAAC,UAAgC,EAAE,IAAY,EAAE,eAAyB;IACjH,MAAM,gBAAgB,GAAa,EAAE,CAAA;IAErC,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;QAC/B,IAAI,SAAS,GAAG,SAAS,CAAC,IAAI,CAAA;QAE9B,IAAI,SAAS,KAAK,KAAK,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;YACjD,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;QACnC,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAA;QAEhE,IAAI,aAAa,KAAK,CAAC,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,UAAU,SAAS,qBAAqB,CAAC,CAAA;QAC1D,CAAC;QAED,iBAAiB,GAAG,aAAa,GAAG,SAAS,CAAC,MAAM,CAAA;QAEpD,IAAI,sBAAsB,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,aAAa,CAAC,CAAA;QAExF,IAAI,sBAAsB,KAAK,CAAC,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,UAAU,SAAS,qBAAqB,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACP,sBAAsB,GAAG,IAAI,CAAC,GAAG,CAAC,sBAAsB,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QACjE,CAAC;QAED,gBAAgB,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAA;IAC9C,CAAC;IAED,OAAO,gBAAgB,CAAA;AACxB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAe;IACzC,IAAI,UAAU,GAAG,EAAE,CAAA;IACnB,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,YAAY,GAAG,GAAG,IAAI,GAAG,CAAA;QAC/B,UAAU,IAAI,YAAY,CAAA;QAE1B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEpB,MAAM,IAAI,YAAY,CAAC,MAAM,CAAA;IAC9B,CAAC;IAED,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;IAE/B,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAA;AAC/B,CAAC;AAED,MAAM,OAAO,eAAe;IAIC;IAH5B,SAAS,CAAsB;IAC/B,KAAK,CAAkB;IAEvB,YAA4B,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;IAC7C,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,IAAY;QACvC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAE/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,CAAA;QAE1C,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,MAAW;QACrC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAE/B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAU,CAAC,KAAK,CAAC,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;QAEjG,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAM,CAAC,MAAM,CAAC,CAAA;QAExC,MAAM,eAAe,GAAG,MAAM,CAAC,iBAAiB,CAAA;QAEhD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7C,MAAM,eAAe,GAAyB,EAAE,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,oBAAoB,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,aAAa,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CAAA;YAEnG,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAChD,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAE/B,eAAe,CAAC,IAAI,CAAC;gBACpB,EAAE,EAAE,OAAO;gBACX,IAAI,EAAE,SAAS;gBACf,eAAe,EAAE,oBAAoB;aACrC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,eAAe,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,kBAAkB;QACvB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAClC,OAAM;QACP,CAAC;QAED,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAA;QAEzF,IAAI,CAAC,SAAS,GAAG,MAAM,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACpE,IAAI,CAAC,KAAK,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAC7D,CAAC;CACD;AAgBD,MAAM,CAAC,MAAM,oBAAoB,GAAa;IAC7C,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,cAAc;IACpB,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,kBAAkB;IACxB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,MAAM;IACZ,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,OAAO;IACb,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,SAAS;IACf,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,OAAO;IACb,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,aAAa;IACnB,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,UAAU;CAChB,CAAA"}

package/dist/alignment/SpeechAlignment.d.ts

@@ -1,17 +1,18 @@
 import * as API from '../api/API.js';
 import { Timeline, TimelineEntry } from '../utilities/Timeline.js';
 import { RawAudio } from '../audio/AudioUtilities.js';
-import { EspeakOptions } from '../synthesis/EspeakTTS.js';
+import { EspeakEvent, EspeakOptions } from '../synthesis/EspeakTTS.js';
 export declare function alignUsingDtw(sourceRawAudio: RawAudio, referenceRawAudio: RawAudio, referenceTimeline: Timeline, granularities: DtwGranularity[], windowDurations: number[]): Promise<TimelineEntry[]>;
 export declare function alignUsingDtwWithRecognition(sourceRawAudio: RawAudio, referenceRawAudio: RawAudio, referenceTimeline: Timeline, recognitionTimeline: Timeline, granularities: DtwGranularity[], windowDurations: number[], espeakOptions: EspeakOptions, phoneAlignmentMethod?: API.PhoneAlignmentMethod): Promise<Timeline>;
+export declare function alignUsingDtwWithEmbeddings(sourceRawAudio: RawAudio, referenceRawAudio: RawAudio, referenceTimeline: Timeline, language: string, granularities: DtwGranularity[], windowDurations: number[]): Promise<TimelineEntry[]>;
 export declare function interpolatePhoneTimelines(sourceTimeline: Timeline, referenceTimeline: Timeline): Promise<Timeline>;
 export declare function alignPhoneTimelines(sourceRawAudio: RawAudio, sourceWordTimeline: Timeline, referenceRawAudio: RawAudio, referenceTimeline: Timeline, windowDuration: number): Promise<Timeline>;
 export declare function createAlignmentReferenceUsingEspeakForFragments(fragments: string[], espeakOptions: EspeakOptions): Promise<{
     rawAudio: RawAudio;
     timeline: Timeline;
-    events: import("../synthesis/EspeakTTS.js").EspeakEvent[];
+    events: EspeakEvent[];
 }>;
-export declare function createAlignmentReferenceUsingEspeak(transcript: string, language: string, plaintextOptions?: API.PlainTextOptions, customLexiconPaths?: string[], insertSeparators?: boolean): Promise<{
+export declare function createAlignmentReferenceUsingEspeak(transcript: string, language: string, plaintextOptions?: API.PlainTextOptions, customLexiconPaths?: string[], insertSeparators?: boolean, useKlatt?: boolean): Promise<{
     referenceRawAudio: RawAudio;
     referenceTimeline: TimelineEntry[];
     espeakVoice: string;

package/dist/alignment/SpeechAlignment.js

@@ -1,13 +1,17 @@
-import { clip } from '../utilities/Utilities.js';
+import { clip, splitFloat32Array } from '../utilities/Utilities.js';
 import { computeMFCCs, extendDefaultMfccOptions } from '../dsp/MFCC.js';
 import { alignMFCC_DTW, getCostMatrixMemorySizeMB } from './DTWMfccSequenceAlignment.js';
 import { Logger } from '../utilities/Logger.js';
-import { downmixToMonoAndNormalize, getEndingSilentSampleCount, getRawAudioDuration, getStartingSilentSampleCount } from '../audio/AudioUtilities.js';
+import { addTimeOffsetToTimeline } from '../utilities/Timeline.js';
+import { concatAudioSegments, downmixToMonoAndNormalize, getEmptyRawAudio, getEndingSilentSampleCount, getRawAudioDuration, getStartingSilentSampleCount } from '../audio/AudioUtilities.js';
 import chalk from 'chalk';
 import { synthesize } from '../api/API.js';
 import { resampleAudioSpeex } from '../dsp/SpeexResampler.js';
 import { deepClone } from '../utilities/ObjectUtilities.js';
-import { zeroIfNaN } from '../math/VectorMath.js';
+import { cosineDistance, zeroIfNaN } from '../math/VectorMath.js';
+import { alignDTWWindowed } from './DTWSequenceAlignmentWindowed.js';
+import { loadPackage } from '../utilities/PackageManager.js';
+import path from 'path';
 export async function alignUsingDtw(sourceRawAudio, referenceRawAudio, referenceTimeline, granularities, windowDurations) {
     const logger = new Logger();
     if (windowDurations.length == 0) {
@@ -60,39 +64,7 @@ export async function alignUsingDtw(sourceRawAudio, referenceRawAudio, reference
         logger.end();
     }
     logger.start('\nConvert path to timeline');
-    function getMappedTimelineEntry(timelineEntry, recurse = true) {
-        const referenceStartFrameIndex = Math.floor(timelineEntry.startTime * framesPerSecond);
-        const referenceEndFrameIndex = Math.floor(timelineEntry.endTime * framesPerSecond);
-        if (referenceStartFrameIndex < 0 || referenceEndFrameIndex < 0) {
-            throw new Error('Unexpected: encountered a negative timestamp in timeline');
-        }
-        const mappedStartFrameIndex = getMappedFrameIndexForPath(referenceStartFrameIndex, compactedPath, 'first');
-        const mappedEndFrameIndex = getMappedFrameIndexForPath(referenceEndFrameIndex, compactedPath, 'first');
-        let innerTimeline;
-        if (recurse && timelineEntry.timeline != null) {
-            innerTimeline = timelineEntry.timeline.map((entry) => getMappedTimelineEntry(entry));
-        }
-        // Trim silent samples from start and end of mapped entry range
-        const sourceSamplesPerFrame = Math.floor(sourceRawAudio.sampleRate / framesPerSecond);
-        let startSampleIndex = mappedStartFrameIndex * sourceSamplesPerFrame;
-        let endSampleIndex = mappedEndFrameIndex * sourceSamplesPerFrame;
-        const frameSamples = sourceRawAudio.audioChannels[0].subarray(startSampleIndex, endSampleIndex);
-        const silenceThresholdDecibels = -40;
-        startSampleIndex += getStartingSilentSampleCount(frameSamples, silenceThresholdDecibels);
-        endSampleIndex -= getEndingSilentSampleCount(frameSamples, silenceThresholdDecibels);
-        endSampleIndex = Math.max(endSampleIndex, startSampleIndex);
-        // Build mapped timeline entry
-        const startTime = startSampleIndex / sourceRawAudio.sampleRate;
-        const endTime = endSampleIndex / sourceRawAudio.sampleRate;
-        return {
-            type: timelineEntry.type,
-            text: timelineEntry.text,
-            startTime,
-            endTime,
-            timeline: innerTimeline
-        };
-    }
-    const mappedTimeline = referenceTimeline.map((timelineEntry) => getMappedTimelineEntry(timelineEntry));
+    const mappedTimeline = referenceTimeline.map(entry => getMappedTimelineEntry(entry, sourceRawAudio, framesPerSecond, compactedPath));
     logger.end();
     return mappedTimeline;
 }
@@ -201,6 +173,103 @@ export async function alignUsingDtwWithRecognition(sourceRawAudio, referenceRawA
     logger.end();
     return result;
 }
+// This is experimental code. It doesn't work well enough to be usable for anything.
+// Just testing some alternative approaches.
+export async function alignUsingDtwWithEmbeddings(sourceRawAudio, referenceRawAudio, referenceTimeline, language, granularities, windowDurations) {
+    const logger = new Logger();
+    if (sourceRawAudio.sampleRate != 16000) {
+        throw new Error('Source audio must have a sample rate of 16000 Hz');
+    }
+    if (referenceRawAudio.sampleRate != 16000) {
+        throw new Error('Reference audio must have a sample rate of 16000 Hz');
+    }
+    const embeddingType = 'w2v-bert-2.0';
+    let sourceEmbeddings;
+    let referenceEmbeddings;
+    let framesPerSecond;
+    if (embeddingType === 'w2v-bert-2.0') {
+        const packageName = 'w2v-bert-2.0-uint8';
+        const modelDir = await loadPackage(packageName);
+        const modelFilePath = path.join(modelDir, `${packageName}.onnx`);
+        const { Wav2Vec2BertFeatureEmbeddings } = await import('../speech-embeddings/WavToVec2BertFeatureEmbeddings.js');
+        const wav2vecBert = new Wav2Vec2BertFeatureEmbeddings(modelFilePath, ['cpu']);
+        logger.start(`Extract source audio embeddings using the W2V-BERT-2.0 model`);
+        sourceEmbeddings = await wav2vecBert.computeEmbeddings(sourceRawAudio);
+        logger.start(`Extract reference audio embeddings using the W2V-BERT-2.0 model`);
+        referenceEmbeddings = await wav2vecBert.computeEmbeddings(referenceRawAudio);
+        framesPerSecond = 1000 / 10 / 2;
+    }
+    else if (embeddingType === 'whisper') {
+        const sourceSamples = sourceRawAudio.audioChannels[0];
+        const referenceSamples = referenceRawAudio.audioChannels[0];
+        const WhisperSTT = await import(`../recognition/WhisperSTT.js`);
+        const { modelName, modelDir } = await WhisperSTT.loadPackagesAndGetPaths('base.en', language);
+        const whisper = new WhisperSTT.Whisper(modelName, modelDir, ['dml', 'cpu'], ['cpu']);
+        async function encodeToAudioFeatures(samples) {
+            const featureVectors = [];
+            for (let i = 0; i < samples.length; i += 16000 * 30) {
+                const startSampleIndex = i;
+                const endSampleIndex = Math.min(samples.length, i + 16000 * 30);
+                const partSampleCount = endSampleIndex - startSampleIndex;
+                const audioPart = samples.subarray(startSampleIndex, endSampleIndex);
+                const rawAudioForPart = { audioChannels: [audioPart], sampleRate: 16000 };
+                const resultTensor = await whisper.encodeAudio(rawAudioForPart);
+                const vectorLength = resultTensor.dims[2];
+                let featureVectorsForPart = splitFloat32Array(resultTensor.data, vectorLength);
+                featureVectorsForPart = featureVectorsForPart.slice(0, Math.floor((partSampleCount / (16000 * 30)) * 1500));
+                featureVectors.push(...featureVectorsForPart);
+            }
+            return featureVectors;
+        }
+        logger.start(`Extract source audio embeddings using the Whisper encoder model`);
+        sourceEmbeddings = await encodeToAudioFeatures(sourceSamples);
+        logger.start(`Extract reference audio embeddings using the Whisper encoder model`);
+        referenceEmbeddings = await encodeToAudioFeatures(referenceSamples);
+        framesPerSecond = 1500 / 30;
+    }
+    else {
+        throw new Error(`Unknown embedding type: ${embeddingType}`);
+    }
+    logger.start(`Align source and reference audio embeddings using DTW`);
+    const { path: alignmentPath } = alignDTWWindowed(referenceEmbeddings, sourceEmbeddings, cosineDistance, 1000 * 1000);
+    const compactedPath = compactPath(alignmentPath);
+    logger.start('\nConvert path to timeline');
+    const mappedTimeline = referenceTimeline.map(entry => getMappedTimelineEntry(entry, sourceRawAudio, framesPerSecond, compactedPath));
+    logger.end();
+    return mappedTimeline;
+}
+function getMappedTimelineEntry(timelineEntry, sourceRawAudio, framesPerSecond, compactedPath, recurse = true) {
+    const referenceStartFrameIndex = Math.floor(timelineEntry.startTime * framesPerSecond);
+    const referenceEndFrameIndex = Math.floor(timelineEntry.endTime * framesPerSecond);
+    if (referenceStartFrameIndex < 0 || referenceEndFrameIndex < 0) {
+        throw new Error('Unexpected: encountered a negative timestamp in timeline');
+    }
+    const mappedStartFrameIndex = getMappedFrameIndexForPath(referenceStartFrameIndex, compactedPath, 'first');
+    const mappedEndFrameIndex = getMappedFrameIndexForPath(referenceEndFrameIndex, compactedPath, 'first');
+    let innerTimeline;
+    if (recurse && timelineEntry.timeline != null) {
+        innerTimeline = timelineEntry.timeline.map((entry) => getMappedTimelineEntry(entry, sourceRawAudio, framesPerSecond, compactedPath, recurse));
+    }
+    // Trim silent samples from start and end of mapped entry range
+    const sourceSamplesPerFrame = Math.floor(sourceRawAudio.sampleRate / framesPerSecond);
+    let startSampleIndex = mappedStartFrameIndex * sourceSamplesPerFrame;
+    let endSampleIndex = mappedEndFrameIndex * sourceSamplesPerFrame;
+    const frameSamples = sourceRawAudio.audioChannels[0].subarray(startSampleIndex, endSampleIndex);
+    const silenceThresholdDecibels = -40;
+    startSampleIndex += getStartingSilentSampleCount(frameSamples, silenceThresholdDecibels);
+    endSampleIndex -= getEndingSilentSampleCount(frameSamples, silenceThresholdDecibels);
+    endSampleIndex = Math.max(endSampleIndex, startSampleIndex);
+    // Build mapped timeline entry
+    const startTime = startSampleIndex / sourceRawAudio.sampleRate;
+    const endTime = endSampleIndex / sourceRawAudio.sampleRate;
+    return {
+        type: timelineEntry.type,
+        text: timelineEntry.text,
+        startTime,
+        endTime,
+        timeline: innerTimeline
+    };
+}
 export async function interpolatePhoneTimelines(sourceTimeline, referenceTimeline) {
     const interpolatedTimeline = [];
     for (let wordEntryIndex = 0; wordEntryIndex < sourceTimeline.length; wordEntryIndex++) {
@@ -288,7 +357,29 @@ export async function createAlignmentReferenceUsingEspeakForFragments(fragments,
     progressLogger.start("Load espeak module");
     const Espeak = await import("../synthesis/EspeakTTS.js");
     progressLogger.start("Synthesize alignment reference with eSpeak");
-    const result = await Espeak.synthesizeFragments(fragments, espeakOptions);
+    const result = {
+        rawAudio: getEmptyRawAudio(1, await Espeak.getSampleRate()),
+        timeline: [],
+        events: [],
+    };
+    // Split fragments to chunks and process each chunk individually,
+    // and incrementally merge the chunks to the final result.
+    {
+        const maxFragmentsInChunk = 1000;
+        let timeOffset = 0;
+        for (let startOffset = 0; startOffset < fragments.length; startOffset += maxFragmentsInChunk) {
+            const chunk = fragments.slice(startOffset, startOffset + maxFragmentsInChunk);
+            const chunkResult = await Espeak.synthesizeFragments(chunk, espeakOptions);
+            result.rawAudio = {
+                sampleRate: result.rawAudio.sampleRate,
+                audioChannels: concatAudioSegments([result.rawAudio.audioChannels, chunkResult.rawAudio.audioChannels])
+            };
+            const chunkTimeline = addTimeOffsetToTimeline(chunkResult.timeline, timeOffset);
+            result.timeline = [...result.timeline, ...chunkTimeline];
+            result.events = [...result.events, ...chunkResult.events];
+            timeOffset += getRawAudioDuration(chunkResult.rawAudio);
+        }
+    }
     result.timeline = result.timeline.flatMap(clause => clause.timeline);
     for (const wordEntry of result.timeline) {
         wordEntry.timeline = wordEntry.timeline.flatMap(tokenEntry => tokenEntry.timeline);
@@ -296,7 +387,7 @@ export async function createAlignmentReferenceUsingEspeakForFragments(fragments,
     progressLogger.end();
     return result;
 }
-export async function createAlignmentReferenceUsingEspeak(transcript, language, plaintextOptions, customLexiconPaths, insertSeparators) {
+export async function createAlignmentReferenceUsingEspeak(transcript, language, plaintextOptions, customLexiconPaths, insertSeparators, useKlatt) {
     const logger = new Logger();
     logger.start('Synthesize alignment reference with eSpeak');
     const synthesisOptions = {
@@ -305,7 +396,7 @@ export async function createAlignmentReferenceUsingEspeak(transcript, language,
         plainText: plaintextOptions,
         customLexiconPaths: customLexiconPaths,
         espeak: {
-            useKlatt: false,
+            useKlatt,
             insertSeparators,
         }
     };