echogarden 1.4.3 → 1.5.0

package/src/utilities/Timeline.ts

@@ -207,6 +207,20 @@ export function addWordTextOffsetsToTimeline(timeline: Timeline, text: string, c
 	return currentOffset
 }
 
+export function extractEntries(timeline: Timeline, predicate: (entry: TimelineEntry) => boolean): TimelineEntry[] {
+	const timelineWordEntries: TimelineEntry[] = []
+
+	for (const entry of timeline) {
+		if (predicate(entry)) {
+			timelineWordEntries.push(entry)
+		} else if (entry.timeline) {
+			timelineWordEntries.push(...extractEntries(entry.timeline, predicate))
+		}
+	}
+
+	return timelineWordEntries
+}
+
 export type TimelineEntryType = 'segment' | 'paragraph' | 'sentence' | 'clause' | 'phrase' | 'word' | 'token' | 'letter' | 'phone' | 'subphone'
 
 export type TimelineEntry = {

package/dist/alignment/TextAlignment.js

@@ -1,63 +0,0 @@
-import { Logger } from '../utilities/Logger.js';
-import { loadPackage } from '../utilities/PackageManager.js';
-export async function alignText(text1, text2) {
-    const logger = new Logger();
-    text1 = text1.replaceAll(/(\r?\n)+/g, ' ');
-    text2 = text2.replaceAll(/(\r?\n)+/g, ' ');
-    const modelPath = await loadPackage(`xenova-multilingual-e5-small-quantized`);
-    const embeddingModel = new E5TextEmbedding(modelPath);
-    logger.start(`Initialize E5 embedding model`);
-    await embeddingModel.initializeIfNeeded();
-    logger.start(`Tokenize text 1`);
-    const inputs1 = await embeddingModel.tokenizeToModelInputs(text1);
-    logger.start(`Infer embeddings for text 1`);
-    const embeddings1 = await embeddingModel.inferTokenEmbeddings(inputs1);
-    logger.start(`Tokenize text 2`);
-    const inputs2 = await embeddingModel.tokenizeToModelInputs(text2);
-    logger.start(`Infer embeddings for text 2`);
-    const embeddings2 = await embeddingModel.inferTokenEmbeddings(inputs1);
-    logger.end();
-    logger.log(embeddings1);
-}
-export class E5TextEmbedding {
-    modelPath;
-    tokenizer;
-    model;
-    constructor(modelPath) {
-        this.modelPath = modelPath;
-    }
-    async tokenizeToModelInputs(text) {
-        await this.initializeIfNeeded();
-        const inputs = await this.tokenizer(text);
-        return inputs;
-    }
-    async inferTokenEmbeddings(inputs) {
-        await this.initializeIfNeeded();
-        const tokensText = this.tokenizer.model.convert_ids_to_tokens(Array.from(inputs.input_ids.data));
-        const result = await this.model(inputs);
-        const lastHiddenState = result.last_hidden_state;
-        const tokenCount = lastHiddenState.dims[1];
-        const embeddingSize = lastHiddenState.dims[2];
-        const tokenEmbeddings = [];
-        for (let i = 0; i < tokenCount; i++) {
-            const tokenEmbeddingVector = lastHiddenState.data.slice(i * embeddingSize, (i + 1) * embeddingSize);
-            const tokenId = Number(inputs.input_ids.data[i]);
-            const tokenText = tokensText[i];
-            tokenEmbeddings.push({
-                id: tokenId,
-                text: tokenText,
-                embeddingVector: tokenEmbeddingVector
-            });
-        }
-        return tokenEmbeddings;
-    }
-    async initializeIfNeeded() {
-        if (this.tokenizer && this.model) {
-            return;
-        }
-        const { AutoTokenizer, AutoModel } = await import('@echogarden/transformers-nodejs-lite');
-        this.tokenizer = await AutoTokenizer.from_pretrained(this.modelPath);
-        this.model = await AutoModel.from_pretrained(this.modelPath);
-    }
-}
-//# sourceMappingURL=TextAlignment.js.map

package/dist/alignment/TextAlignment.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"TextAlignment.js","sourceRoot":"","sources":["../../src/alignment/TextAlignment.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAE5D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IAC3D,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAC1C,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,WAAW,EAAE,GAAG,CAAC,CAAA;IAE1C,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,wCAAwC,CAAC,CAAA;IAE7E,MAAM,cAAc,GAAG,IAAI,eAAe,CAAC,SAAS,CAAC,CAAA;IAErD,MAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAA;IAC7C,MAAM,cAAc,CAAC,kBAAkB,EAAE,CAAA;IAEzC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;IAC/B,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAA;IAEjE,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAA;IAEtE,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;IAC/B,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAA;IAEjE,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAA;IAEtE,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;AACxB,CAAC;AAED,MAAM,OAAO,eAAe;IAIC;IAH5B,SAAS,CAAsB;IAC/B,KAAK,CAAkB;IAEvB,YAA4B,SAAiB;QAAjB,cAAS,GAAT,SAAS,CAAQ;IAC7C,CAAC;IAED,KAAK,CAAC,qBAAqB,CAAC,IAAY;QACvC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAE/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAU,CAAC,IAAI,CAAC,CAAA;QAE1C,OAAO,MAAM,CAAA;IACd,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,MAAW;QACrC,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAA;QAE/B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAU,CAAC,KAAK,CAAC,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAA;QAEjG,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAM,CAAC,MAAM,CAAC,CAAA;QAExC,MAAM,eAAe,GAAG,MAAM,CAAC,iBAAiB,CAAA;QAEhD,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAE7C,MAAM,eAAe,GAAyB,EAAE,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,oBAAoB,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,aAAa,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CAAA;YAEnG,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;YAChD,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YAE/B,eAAe,CAAC,IAAI,CAAC;gBACpB,EAAE,EAAE,OAAO;gBACX,IAAI,EAAE,SAAS;gBACf,eAAe,EAAE,oBAAoB;aACrC,CAAC,CAAA;QACH,CAAC;QAED,OAAO,eAAe,CAAA;IACvB,CAAC;IAED,KAAK,CAAC,kBAAkB;QACvB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAClC,OAAM;QACP,CAAC;QAED,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,sCAAsC,CAAC,CAAA;QAEzF,IAAI,CAAC,SAAS,GAAG,MAAM,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACpE,IAAI,CAAC,KAAK,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAC7D,CAAC;CACD"}

package/dist/api/LanguageDetection.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"LanguageDetection.js","sourceRoot":"","sources":["../../src/api/LanguageDetection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAEvE,OAAO,EAA8B,cAAc,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AACpK,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,IAAI,MAAM,MAAM,CAAA;AAEvB,OAAO,EAAE,0BAA0B,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAA;AACvF,OAAO,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAC5D,OAAO,KAAK,MAAM,OAAO,CAAA;AAKzB,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,6FAA6F;AAC7F,4BAA4B;AAC5B,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,KAAuB,EAAE,OAAuC;IAC1G,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IAEvC,OAAO,GAAG,UAAU,CAAC,qCAAqC,EAAE,OAAO,CAAC,CAAA;IAEpE,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;IAEjD,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;IAC5C,IAAI,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAClE,cAAc,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;IACpD,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IAE/E,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,cAAc,EAAE,OAAO,CAAC,GAAI,CAAC,CAAC,CAAA;QAEnG,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,CAAC,MAAM,SAAS,CAAC,CAAA;IAEnD,MAAM,eAAe,GAAG,OAAO,CAAC,eAAgB,CAAA;IAChD,MAAM,4BAA4B,GAAG,OAAO,CAAC,4BAA6B,CAAA;IAE1E,IAAI,6BAAuD,CAAA;IAE3D,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACf,MAAM,uBAAuB,GAAG,MAAM,MAAM,CAAC,yDAAyD,CAAC,CAAA;YAEvG,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,aAAa,GAAG,OAAO,CAAC,MAAO,CAAA;YAErC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,2BAA2B,CAAC,CAAA;YAE/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAA;YAChE,MAAM,sBAAsB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAA;YACvE,MAAM,2BAA2B,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAA;YAClF,MAAM,sBAAsB,GAA4B,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YAE9G,MAAM,eAAe,GAAG,MAAM,uBAAuB,CAAC,cAAc,CACnE,cAAc,EACd,SAAS,EACT,sBAAsB,EACtB,2BAA2B,EAC3B,sBAAsB,CAAC,CAAA;YAExB,6BAA6B,GAAG,eAAe,CAAA;YAE/C,MAAK;QACN,CAAC;QAED,KAAK,SAAS,CAAC,CAAC,CAAC;YAChB,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;YAEzG,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,6BAA6B,GAAG,MAAM,UAAU,CAAC,cAAc,CAC9D,cAAc,EACd,SAAS,EACT,QAAQ,EACR,cAAc,CAAC,CAAA;YAEhB,MAAK;QACN,CAAC;QAED,KAAK,aAAa,CAAC,CAAC,CAAC;YACpB,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAA;YAErE,MAAM,iBAAiB,GAAG,OAAO,CAAC,UAAW,CAAA;YAE7C,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;YAEzG,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,6BAA6B,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,cAAc,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YAExG,MAAK;QACN,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,IAAI,gBAAwB,CAAA;IAE5B,IAAI,6BAA6B,CAAC,MAAM,IAAI,CAAC;QAC5C,6BAA6B,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,4BAA4B,EAAE,CAAC;QAE9E,gBAAgB,GAAG,eAAe,CAAA;IACnC,CAAC;SAAM,CAAC;QACP,gBAAgB,GAAG,6BAA6B,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;IAC7D,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IACZ,MAAM,CAAC,WAAW,CAAC,iCAAiC,EAAE,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;IAErF,OAAO;QACN,gBAAgB;QAChB,oBAAoB,EAAE,kBAAkB,CAAC,gBAAgB,CAAC;QAC1D,6BAA6B;QAE7B,aAAa;KACb,CAAA;AACF,CAAC;AASD,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,cAAwB,EAAE,sBAAkF,EAAE,iBAAiB,GAAG,EAAE,EAAE,WAAW,GAAG,EAAE;IACvM,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,aAAa,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;IAEzD,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,eAAe,GAA+B,EAAE,CAAA;IAEtD,KAAK,IAAI,eAAe,GAAG,CAAC,EAAE,eAAe,GAAG,aAAa,EAAE,eAAe,IAAI,WAAW,EAAE,CAAC;QAC/F,MAAM,WAAW,GAAG,eAAe,CAAA;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,iBAAiB,EAAE,aAAa,CAAC,CAAA;QAC9E,MAAM,eAAe,GAAG,SAAS,GAAG,WAAW,CAAA;QAE/C,MAAM,CAAC,gBAAgB,CAAC,mDAAmD,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;QAC9H,MAAM,SAAS,GAAG,mBAAmB,CAAC,cAAc,EAAE,WAAW,EAAE,SAAS,CAAC,CAAA;QAE7E,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,SAAS,CAAC,CAAA;QAE9D,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QAEpC,MAAM,oBAAoB,GAAG,SAAS,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAA;QAEpG,IAAI,oBAAoB,GAAa,EAAE,CAAA;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnE,oBAAoB,CAAC,IAAI,CAAC,GAAG,0BAA0B,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,oBAAoB,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAChJ,CAAC;QAED,MAAM,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAE1E,IAAI,eAAe,GAAG,iBAAiB,EAAE,CAAC;YACzC,MAAK;QACN,CAAC;IACF,CAAC;IAED,MAAM,eAAe,GAA6B,SAAS,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/E,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,WAAW,GAAG,GAAG,CAAA,CAAC,CAAC,CAAC,CAAA;IAE7D,KAAK,MAAM,WAAW,IAAI,eAAe,EAAE,CAAC;QAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,eAAe,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAA;QAC7D,CAAC;IACF,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;QACtC,MAAM,CAAC,WAAW,IAAI,eAAe,CAAC,MAAM,CAAA;IAC7C,CAAC;IAED,OAAO,eAAe,CAAA;AACvB,CAAC;AAoBD,MAAM,CAAC,MAAM,qCAAqC,GAAmC;IACpF,MAAM,EAAE,SAAS;IACjB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,IAAI;IAElC,IAAI,EAAE,IAAI;IAEV,MAAM,EAAE,EACP;IAED,OAAO,EAAE;QACR,KAAK,EAAE,MAAM;QACb,WAAW,EAAE,GAAG;KAChB;IAED,UAAU,EAAE;QACX,KAAK,EAAE,MAAM;KACb;IAED,GAAG,EAAE;QACJ,MAAM,EAAE,eAAe;KACvB;CACD,CAAA;AAED,6FAA6F;AAC7F,0BAA0B;AAC1B,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAa,EAAE,OAAqC;IAC5F,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,OAAO,GAAG,UAAU,CAAC,mCAAmC,EAAE,OAAO,CAAC,CAAA;IAElE,MAAM,eAAe,GAAG,OAAO,CAAC,eAAgB,CAAA;IAChD,MAAM,4BAA4B,GAAG,OAAO,CAAC,4BAA6B,CAAA;IAE1E,IAAI,6BAAuD,CAAA;IAE3D,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,CAAC,MAAM,SAAS,CAAC,CAAA;IAEnD,QAAQ,OAAO,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uDAAuD,CAAC,CAAA;YAEhG,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAA;YAEjD,6BAA6B,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;YAE3D,MAAK;QACN,CAAC;QAED,KAAK,UAAU,CAAC,CAAC,CAAC;YACjB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,yDAAyD,CAAC,CAAA;YAElG,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;YAEnD,6BAA6B,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;YAE3D,MAAK;QACN,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,IAAI,gBAAwB,CAAA;IAE5B,IAAI,6BAA6B,CAAC,MAAM,IAAI,CAAC;QAC5C,6BAA6B,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,4BAA4B,EAAE,CAAC;QAE9E,gBAAgB,GAAG,eAAe,CAAA;IACnC,CAAC;SAAM,CAAC;QACP,gBAAgB,GAAG,6BAA6B,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAA;IAC7D,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,OAAO;QACN,gBAAgB;QAChB,oBAAoB,EAAE,kBAAkB,CAAC,gBAAgB,CAAC;QAC1D,6BAA6B;KAC7B,CAAA;AACF,CAAC;AAiCD,6FAA6F;AAC7F,YAAY;AACZ,6FAA6F;AAE7F,MAAM,CAAC,MAAM,mCAAmC,GAAiC;IAChF,MAAM,EAAE,QAAQ;IAChB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,IAAI;CAClC,CAAA;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAyB;IACnE;QACC,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,mDAAmD;QAChE,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,SAAS;QACb,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,yFAAyF;QACtG,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,aAAa;QACjB,IAAI,EAAE,2BAA2B;QACjC,WAAW,EAAE,mFAAmF;QAChG,IAAI,EAAE,OAAO;KACb;CACD,CAAA;AAED,MAAM,CAAC,MAAM,4BAA4B,GAAyB;IACjE;QACC,EAAE,EAAE,QAAQ;QACZ,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE,sCAAsC;QACnD,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,UAAU;QAChB,WAAW,EAAE,sFAAsF;QACnG,IAAI,EAAE,OAAO;KACb;CACD,CAAA"}

package/dist/api/Translation.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"Translation.js","sourceRoot":"","sources":["../../src/api/Translation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAA;AAE5D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAA8B,cAAc,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAA;AAC1H,OAAO,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAA;AAE/C,OAAO,EAAY,4BAA4B,EAAE,qCAAqC,EAAE,MAAM,0BAA0B,CAAA;AAExH,OAAO,EAAE,0BAA0B,EAAE,oBAAoB,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAA;AAEjJ,OAAO,EAAuC,oBAAoB,EAAE,MAAM,UAAU,CAAA;AACpF,OAAO,KAAK,MAAM,OAAO,CAAA;AAGzB,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAI/B,MAAM,GAAG,GAAG,WAAW,CAAA;AAEvB,6FAA6F;AAC7F,qBAAqB;AACrB,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAuB,EAAE,OAAiC;IAC/F,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAA;IAE3B,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,EAAE,CAAA;IAE5C,OAAO,GAAG,UAAU,CAAC,+BAA+B,EAAE,OAAO,CAAC,CAAA;IAE9D,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAA;IAEjD,IAAI,cAAwB,CAAA;IAC5B,IAAI,gBAAsC,CAAA;IAC1C,IAAI,kBAAwC,CAAA;IAE5C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACd,MAAM,CAAC,GAAG,EAAE,CAAC;QAEb,CAAC,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,gBAAiB,CAAC,CAAC,CAAA;QAExG,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEd,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,cAAc,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAClE,CAAC;SAAM,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAC5C,cAAc,GAAG,MAAM,cAAc,CAAC,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAC/D,CAAC;IAED,IAAI,oBAA0C,CAAA;IAE9C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAClB,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACpD,CAAC,EAAE,QAAQ,EAAE,oBAAoB,EAAE,eAAe,EAAE,cAAc,EAAE,GAAG,MAAM,GAAG,CAAC,mBAAmB,CAAC,cAAc,EAAE,OAAO,CAAC,GAAI,CAAC,CAAC,CAAA;QAEnI,MAAM,CAAC,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAA;IAExC,cAAc,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAA;IACpD,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAA;IAE/E,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC5B,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,cAAc,CAAC,CAAA;QAEtE,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC,IAAI,CAAA;QAE1C,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,gBAAgB,CAAC,2BAA2B,EAAE,0BAA0B,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IACzG,CAAC;SAAM,CAAC;QACP,MAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAA;QACpE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,oBAAoB,CAAC,cAAc,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAA;QAExG,OAAO,CAAC,cAAc,GAAG,gBAAgB,CAAA;QAEzC,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,CAAC,gBAAgB,CAAC,0BAA0B,EAAE,0BAA0B,CAAC,gBAAgB,CAAC,CAAC,CAAA;IAClG,CAAC;IAED,OAAO,CAAC,cAAc,GAAG,MAAM,iCAAiC,CAAC,OAAO,CAAC,cAAe,CAAC,CAAA;IAEzF,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,0BAA0B,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAA;IAE9F,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAA;IAEhD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAO,CAAA;IAC9B,MAAM,cAAc,GAAG,OAAO,CAAC,cAAe,CAAA;IAC9C,MAAM,cAAc,GAAG,OAAO,CAAC,cAAe,CAAA;IAE9C,IAAI,UAAkB,CAAA;IACtB,IAAI,YAAkC,CAAA;IACtC,IAAI,eAAqC,CAAA;IAEzC,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,SAAS,CAAC,CAAA;IAErC,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,SAAS,CAAC,CAAC,CAAC;YAChB,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;YAE/D,MAAM,cAAc,GAAG,OAAO,CAAC,OAAQ,CAAA;YAEvC,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YACpE,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YAEpE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,uBAAuB,CAAC,cAAc,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAA;YAEvH,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;YAChF,CAAC;YAED,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;YACzF,CAAC;YAED,IAAI,uBAAuB,IAAI,IAAI,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACxE,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAA;YAC5E,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,CAAC,CAAC,CAAA;YAEvJ,MAAK;QACN,CAAC;QAED,KAAK,aAAa,CAAC,CAAC,CAAC;YACpB,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAA;YAErE,MAAM,iBAAiB,GAAG,OAAO,CAAC,UAAW,CAAA;YAE7C,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YACpE,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YAEpE,MAAM,CAAC,GAAG,EAAE,CAAA;YAEZ,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,EAAE,uBAAuB,CAAC,CAAA;YAEvH,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAA;YACpF,CAAC;YAED,IAAI,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAA;YAC7F,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,GAAG,MAAM,aAAa,CAAC,SAAS,CACtE,cAAc,EACd,WAAW,EACX,uBAAuB,EACvB,SAAS,EACT,SAAS,EACT,iBAAiB,CACjB,CAAC,CAAA;YAEF,MAAK;QACN,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACrB,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,kCAAkC,CAAC,CAAA;YAEvE,MAAM,qBAAqB,GAAG,OAAO,CAAC,WAAY,CAAA;YAElD,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;YACpD,CAAC;YAED,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YACpE,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;YAEpE,IAAI,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAA;YAC5F,CAAC;YAED,MAAM,CAAC,GAAG,EAAE,CAAC;YAEb,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,cAAc,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,WAAW,CAAC,CAAC,CAAA;YAEzJ,MAAK;QACN,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAA;QAC/D,CAAC;IACF,CAAC;IAED,MAAM,CAAC,GAAG,EAAE,CAAA;IAEZ,6FAA6F;IAC7F,IAAI,oBAAoB,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,IAAI,YAAY,EAAE,CAAC;YAClB,GAAG,CAAC,iCAAiC,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAA;QAC1E,CAAC;aAAM,IAAI,eAAe,EAAE,CAAC;YAC5B,GAAG,CAAC,iCAAiC,CAAC,eAAe,EAAE,oBAAoB,CAAC,CAAA;QAC7E,CAAC;IACF,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QAClB,4BAA4B,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;IACvD,CAAC;IAED,IAAI,CAAC,eAAe,EAAE,CAAC;QACtB,CAAC,EAAE,eAAe,EAAE,GAAG,MAAM,qCAAqC,CAAC,YAAa,EAAE,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAA;IACrI,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACd,MAAM,CAAC,WAAW,CAAC,+BAA+B,EAAE,cAAc,EAAE,KAAK,CAAC,aAAa,CAAC,CAAA;IAExF,OAAO;QACN,UAAU;QACV,QAAQ,EAAE,eAAe;QACzB,YAAY;QAEZ,cAAc;QACd,cAAc;QAEd,aAAa;QACb,gBAAgB;QAChB,kBAAkB;KAClB,CAAA;AACF,CAAC;AAoCD,MAAM,CAAC,MAAM,+BAA+B,GAA6B;IACxE,MAAM,EAAE,SAAS;IAEjB,cAAc,EAAE,SAAS;IACzB,cAAc,EAAE,IAAI;IAEpB,IAAI,EAAE,IAAI;IACV,OAAO,EAAE,KAAK;IAEd,iBAAiB,EAAE,SAAS;IAE5B,SAAS,EAAE,EACV;IAED,GAAG,EAAE;QACJ,MAAM,EAAE,eAAe;KACvB;IAED,OAAO,EAAE,EACR;IAED,UAAU,EAAE,EACX;IAED,WAAW,EAAE,EACZ;CACD,CAAA;AAED,MAAM,CAAC,MAAM,wBAAwB,GAAqB;IACzD;QACC,EAAE,EAAE,SAAS;QACb,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE,oHAAoH;QACjI,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,aAAa;QACjB,IAAI,EAAE,2BAA2B;QACjC,WAAW,EAAE,oHAAoH;QACjI,IAAI,EAAE,OAAO;KACb;IACD;QACC,EAAE,EAAE,cAAc;QAClB,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,+FAA+F;QAC5G,IAAI,EAAE,OAAO;KACb;CACD,CAAA"}

package/src/alignment/TextAlignment.ts

@@ -1,96 +0,0 @@
-import { type PreTrainedModel, type PreTrainedTokenizer } from '@echogarden/transformers-nodejs-lite'
-import { Logger } from '../utilities/Logger.js'
-import { loadPackage } from '../utilities/PackageManager.js'
-
-export async function alignText(text1: string, text2: string) {
-	const logger = new Logger()
-
-	text1 = text1.replaceAll(/(\r?\n)+/g, ' ')
-	text2 = text2.replaceAll(/(\r?\n)+/g, ' ')
-
-	const modelPath = await loadPackage(`xenova-multilingual-e5-small-quantized`)
-
-	const embeddingModel = new E5TextEmbedding(modelPath)
-
-	logger.start(`Initialize E5 embedding model`)
-	await embeddingModel.initializeIfNeeded()
-
-	logger.start(`Tokenize text 1`)
-	const inputs1 = await embeddingModel.tokenizeToModelInputs(text1)
-
-	logger.start(`Infer embeddings for text 1`)
-	const embeddings1 = await embeddingModel.inferTokenEmbeddings(inputs1)
-
-	logger.start(`Tokenize text 2`)
-	const inputs2 = await embeddingModel.tokenizeToModelInputs(text2)
-
-	logger.start(`Infer embeddings for text 2`)
-	const embeddings2 = await embeddingModel.inferTokenEmbeddings(inputs1)
-
-	logger.end()
-
-	logger.log(embeddings1)
-}
-
-export class E5TextEmbedding {
-	tokenizer?: PreTrainedTokenizer
-	model?: PreTrainedModel
-
-	constructor(public readonly modelPath: string) {
-	}
-
-	async tokenizeToModelInputs(text: string) {
-		await this.initializeIfNeeded()
-
-		const inputs = await this.tokenizer!(text)
-
-		return inputs
-	}
-
-	async inferTokenEmbeddings(inputs: any) {
-		await this.initializeIfNeeded()
-
-		const tokensText = this.tokenizer!.model.convert_ids_to_tokens(Array.from(inputs.input_ids.data))
-
-		const result = await this.model!(inputs)
-
-		const lastHiddenState = result.last_hidden_state
-
-		const tokenCount = lastHiddenState.dims[1]
-		const embeddingSize = lastHiddenState.dims[2]
-
-		const tokenEmbeddings: TokenEmbeddingData[] = []
-
-		for (let i = 0; i < tokenCount; i++) {
-			const tokenEmbeddingVector = lastHiddenState.data.slice(i * embeddingSize, (i + 1) * embeddingSize)
-
-			const tokenId = Number(inputs.input_ids.data[i])
-			const tokenText = tokensText[i]
-
-			tokenEmbeddings.push({
-				id: tokenId,
-				text: tokenText,
-				embeddingVector: tokenEmbeddingVector
-			})
-		}
-
-		return tokenEmbeddings
-	}
-
-	async initializeIfNeeded() {
-		if (this.tokenizer && this.model) {
-			return
-		}
-
-		const { AutoTokenizer, AutoModel } = await import('@echogarden/transformers-nodejs-lite')
-
-		this.tokenizer = await AutoTokenizer.from_pretrained(this.modelPath)
-		this.model = await AutoModel.from_pretrained(this.modelPath)
-	}
-}
-
-export interface TokenEmbeddingData {
-	id: number
-	text: string
-	embeddingVector: Float32Array
-}