hybridtm 0.1.0

package/dist/vectorSimilarityTester.js

@@ -0,0 +1,164 @@
+"use strict";
+/*******************************************************************************
+ * Copyright (c) 2025 Maxprograms.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 1.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/org/documents/epl-v10.html
+ *
+ * Contributors:
+ *     Maxprograms - initial API and implementation
+ *******************************************************************************/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VectorSimilarityTester = void 0;
+const transformers_1 = require("@xenova/transformers");
+class VectorSimilarityTester {
+    static DEFAULT_MODEL = 'Xenova/LaBSE';
+    embedder = null;
+    modelName;
+    constructor(modelName = VectorSimilarityTester.DEFAULT_MODEL) {
+        this.modelName = modelName;
+    }
+    static manhattanSimilarity(vecA, vecB) {
+        if (vecA.length !== vecB.length) {
+            throw new Error('Vectors must have the same length');
+        }
+        let sumAbsDiff = 0;
+        for (let i = 0; i < vecA.length; i++) {
+            sumAbsDiff += Math.abs(vecA[i] - vecB[i]);
+        }
+        return Math.exp(-sumAbsDiff / vecA.length);
+    }
+    static cosineSimilarity(vecA, vecB) {
+        if (vecA.length !== vecB.length) {
+            throw new Error('Vectors must have the same length');
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < vecA.length; i++) {
+            const a = vecA[i];
+            const b = vecB[i];
+            dotProduct += a * b;
+            normA += a * a;
+            normB += b * b;
+        }
+        if (normA === 0 || normB === 0) {
+            return 0;
+        }
+        return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+    }
+    static buildResult(label, manhattanScore, cosineScore) {
+        const cosineMinusManhattan = cosineScore - manhattanScore;
+        const absoluteDifference = Math.abs(cosineMinusManhattan);
+        return {
+            label,
+            manhattanScore,
+            cosineScore,
+            cosineMinusManhattan,
+            absoluteDifference,
+            manhattanPercent: Math.round(manhattanScore * 100),
+            cosinePercent: Math.round(cosineScore * 100)
+        };
+    }
+    async ensureEmbedder() {
+        if (this.embedder) {
+            return;
+        }
+        this.embedder = await (0, transformers_1.pipeline)('feature-extraction', this.modelName);
+    }
+    async embed(text) {
+        await this.ensureEmbedder();
+        if (!this.embedder) {
+            throw new Error('Failed to initialize embedder');
+        }
+        const result = await this.embedder(text, {
+            pooling: 'mean',
+            normalize: true
+        });
+        return Array.from(result.data);
+    }
+    static compareVectors(input) {
+        const manhattanScore = VectorSimilarityTester.manhattanSimilarity(input.sourceVector, input.targetVector);
+        const cosineScore = VectorSimilarityTester.cosineSimilarity(input.sourceVector, input.targetVector);
+        return VectorSimilarityTester.buildResult(input.label, manhattanScore, cosineScore);
+    }
+    async compareTexts(inputs) {
+        if (inputs.length === 0) {
+            return [];
+        }
+        const cache = new Map();
+        const getEmbedding = async (text) => {
+            const normalized = text || '';
+            const cached = cache.get(normalized);
+            if (cached) {
+                return cached;
+            }
+            const embedding = await this.embed(normalized);
+            cache.set(normalized, embedding);
+            return embedding;
+        };
+        const results = [];
+        for (const input of inputs) {
+            const sourceEmbedding = await getEmbedding(input.sourceText);
+            const targetEmbedding = await getEmbedding(input.targetText);
+            const comparison = VectorSimilarityTester.buildResult(input.label, VectorSimilarityTester.manhattanSimilarity(sourceEmbedding, targetEmbedding), VectorSimilarityTester.cosineSimilarity(sourceEmbedding, targetEmbedding));
+            results.push({
+                ...comparison,
+                sourceText: input.sourceText,
+                targetText: input.targetText
+            });
+        }
+        return results;
+    }
+    static summarize(results) {
+        if (results.length === 0) {
+            return {
+                averageManhattan: 0,
+                averageCosine: 0,
+                averageAbsoluteDifference: 0,
+                maxAbsoluteDifference: 0,
+                minAbsoluteDifference: 0
+            };
+        }
+        let sumManhattan = 0;
+        let sumCosine = 0;
+        let sumAbs = 0;
+        let maxAbs = Number.NEGATIVE_INFINITY;
+        let minAbs = Number.POSITIVE_INFINITY;
+        for (const result of results) {
+            sumManhattan += result.manhattanScore;
+            sumCosine += result.cosineScore;
+            sumAbs += result.absoluteDifference;
+            if (result.absoluteDifference > maxAbs) {
+                maxAbs = result.absoluteDifference;
+            }
+            if (result.absoluteDifference < minAbs) {
+                minAbs = result.absoluteDifference;
+            }
+        }
+        return {
+            averageManhattan: sumManhattan / results.length,
+            averageCosine: sumCosine / results.length,
+            averageAbsoluteDifference: sumAbs / results.length,
+            maxAbsoluteDifference: maxAbs,
+            minAbsoluteDifference: minAbs
+        };
+    }
+    static formatResults(results) {
+        if (results.length === 0) {
+            return 'No comparison data available.';
+        }
+        const lines = results.map(function (result, index) {
+            const label = result.label || 'pair-' + (index + 1);
+            const manhattanPercent = result.manhattanPercent.toString().padStart(3, ' ');
+            const cosinePercent = result.cosinePercent.toString().padStart(3, ' ');
+            const delta = result.cosineMinusManhattan.toFixed(4);
+            return label + ': manhattan=' + manhattanPercent + '% cosine=' + cosinePercent + '% delta=' + delta;
+        });
+        return lines.join('\n');
+    }
+}
+exports.VectorSimilarityTester = VectorSimilarityTester;
+//# sourceMappingURL=vectorSimilarityTester.js.map

package/dist/vectorSimilarityTester.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectorSimilarityTester.js","sourceRoot":"","sources":["../ts/vectorSimilarityTester.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;;AAEjF,uDAAmF;AAqCnF,MAAa,sBAAsB;IAE/B,MAAM,CAAU,aAAa,GAAW,cAAc,CAAC;IAE/C,QAAQ,GAAqC,IAAI,CAAC;IACzC,SAAS,CAAS;IAEnC,YAAY,YAAoB,sBAAsB,CAAC,aAAa;QAChE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;IAEO,MAAM,CAAC,mBAAmB,CAAC,IAAc,EAAE,IAAc;QAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,UAAU,GAAW,CAAC,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAC/C,CAAC;IAEO,MAAM,CAAC,gBAAgB,CAAC,IAAc,EAAE,IAAc;QAC1D,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,UAAU,GAAW,CAAC,CAAC;QAC3B,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QAEtB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,GAAW,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,CAAC,GAAW,IAAI,CAAC,CAAC,CAAC,CAAC;YAC1B,UAAU,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,CAAC;QACb,CAAC;QAED,OAAO,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9D,CAAC;IAEO,MAAM,CAAC,WAAW,CAAC,KAAyB,EAAE,cAAsB,EAAE,WAAmB;QAC7F,MAAM,oBAAoB,GAAW,WAAW,GAAG,cAAc,CAAC;QAClE,MAAM,kBAAkB,GAAW,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClE,OAAO;YACH,KAAK;YACL,cAAc;YACd,WAAW;YACX,oBAAoB;YACpB,kBAAkB;YAClB,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,GAAG,CAAC;YAClD,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC;SAC/C,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,cAAc;QACxB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,MAAM,IAAA,uBAAQ,EAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzE,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,IAAY;QAC5B,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC5B,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACrD,CAAC;QACD,MAAM,MAAM,GAAW,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;YAC7C,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAEM,MAAM,CAAC,cAAc,CAAC,KAA4B;QACrD,MAAM,cAAc,GAAW,sBAAsB,CAAC,mBAAmB,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;QAClH,MAAM,WAAW,GAAW,sBAAsB,CAAC,gBAAgB,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;QAC5G,OAAO,sBAAsB,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC;IACxF,CAAC;IAEM,KAAK,CAAC,YAAY,CAAC,MAA6B;QACnD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAA0B,IAAI,GAAG,EAAoB,CAAC;QAEjE,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAAqB,EAAE;YAC3D,MAAM,UAAU,GAAW,IAAI,IAAI,EAAE,CAAC;YACtC,MAAM,MAAM,GAAyB,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC3D,IAAI,MAAM,EAAE,CAAC;gBACT,OAAO,MAAM,CAAC;YAClB,CAAC;YACD,MAAM,SAAS,GAAa,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACzD,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YACjC,OAAO,SAAS,CAAC;QACrB,CAAC,CAAC;QAEF,MAAM,OAAO,GAA2B,EAAE,CAAC;QAE3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,eAAe,GAAa,MAAM,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACvE,MAAM,eAAe,GAAa,MAAM,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACvE,MAAM,UAAU,GAA2B,sBAAsB,CAAC,WAAW,CACzE,KAAK,CAAC,KAAK,EACX,sBAAsB,CAAC,mBAAmB,CAAC,eAAe,EAAE,eAAe,CAAC,EAC5E,sBAAsB,CAAC,gBAAgB,CAAC,eAAe,EAAE,eAAe,CAAC,CAC5E,CAAC;YACF,OAAO,CAAC,IAAI,CAAC;gBACT,GAAG,UAAU;gBACb,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;aAC/B,CAAC,CAAC;QACP,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,OAAiC;QACrD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;gBACH,gBAAgB,EAAE,CAAC;gBACnB,aAAa,EAAE,CAAC;gBAChB,yBAAyB,EAAE,CAAC;gBAC5B,qBAAqB,EAAE,CAAC;gBACxB,qBAAqB,EAAE,CAAC;aAC3B,CAAC;QACN,CAAC;QAED,IAAI,YAAY,GAAW,CAAC,CAAC;QAC7B,IAAI,SAAS,GAAW,CAAC,CAAC;QAC1B,IAAI,MAAM,GAAW,CAAC,CAAC;QACvB,IAAI,MAAM,GAAW,MAAM,CAAC,iBAAiB,CAAC;QAC9C,IAAI,MAAM,GAAW,MAAM,CAAC,iBAAiB,CAAC;QAE9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,YAAY,IAAI,MAAM,CAAC,cAAc,CAAC;YACtC,SAAS,IAAI,MAAM,CAAC,WAAW,CAAC;YAChC,MAAM,IAAI,MAAM,CAAC,kBAAkB,CAAC;YACpC,IAAI,MAAM,CAAC,kBAAkB,GAAG,MAAM,EAAE,CAAC;gBACrC,MAAM,GAAG,MAAM,CAAC,kBAAkB,CAAC;YACvC,CAAC;YACD,IAAI,MAAM,CAAC,kBAAkB,GAAG,MAAM,EAAE,CAAC;gBACrC,MAAM,GAAG,MAAM,CAAC,kBAAkB,CAAC;YACvC,CAAC;QACL,CAAC;QAED,OAAO;YACH,gBAAgB,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM;YAC/C,aAAa,EAAE,SAAS,GAAG,OAAO,CAAC,MAAM;YACzC,yBAAyB,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;YAClD,qBAAqB,EAAE,MAAM;YAC7B,qBAAqB,EAAE,MAAM;SAChC,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,OAAiC;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,+BAA+B,CAAC;QAC3C,CAAC;QAED,MAAM,KAAK,GAAa,OAAO,CAAC,GAAG,CAAC,UAAS,MAA8B,EAAE,KAAa;YACtF,MAAM,KAAK,GAAW,MAAM,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC5D,MAAM,gBAAgB,GAAW,MAAM,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACrF,MAAM,aAAa,GAAW,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC/E,MAAM,KAAK,GAAW,MAAM,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAC7D,OAAO,KAAK,GAAG,cAAc,GAAG,gBAAgB,GAAG,WAAW,GAAG,aAAa,GAAG,UAAU,GAAG,KAAK,CAAC;QACxG,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;;AAjLL,wDAkLC"}

package/dist/xliffhandler.d.ts

@@ -0,0 +1,67 @@
+/*******************************************************************************
+ * Copyright (c) 2025-2026 Maxprograms.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse   License 1.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/org/documents/epl-v10.html
+ *
+ * Contributors:
+ *     Maxprograms - initial API and implementation
+ *******************************************************************************/
+import { Catalog, CData, ContentHandler, Grammar, XMLAttribute, XMLElement } from "typesxml";
+import { ImportOptions } from './importOptions.js';
+export declare class XLIFFHandler implements ContentHandler {
+    inCdData: boolean;
+    currentCData: CData;
+    stack: Array<XMLElement>;
+    srcLang: string;
+    tgtLang: string;
+    original: string;
+    fileId: string;
+    private writeStream;
+    private readonly completionPromise;
+    private entryCount;
+    private readonly options;
+    constructor(tempFilePath: string, options?: ImportOptions);
+    setGrammar(grammar: Grammar | undefined): void;
+    getEntryCount(): number;
+    waitForCompletion(): Promise<void>;
+    initialize(): void;
+    setCatalog(catalog: Catalog): void;
+    startDocument(): void;
+    endDocument(): void;
+    xmlDeclaration(version: string, encoding: string, standalone: string | undefined): void;
+    startElement(name: string, atts: Array<XMLAttribute>): void;
+    endElement(name: string): void;
+    internalSubset(declaration: string): void;
+    characters(ch: string): void;
+    ignorableWhitespace(ch: string): void;
+    comment(ch: string): void;
+    processingInstruction(target: string, data: string): void;
+    startCDATA(): void;
+    endCDATA(): void;
+    startDTD(name: string, publicId: string, systemId: string): void;
+    endDTD(): void;
+    skippedEntity(name: string): void;
+    processUnit(unit: XMLElement): void;
+    private buildSegments;
+    private buildSegmentData;
+    private buildSingleSegment;
+    private writeSegmentEntries;
+    private applySegmentPosition;
+    private writeMergedEntry;
+    private writeEntry;
+    private hasMetadata;
+    private appendElementContent;
+    private extractMetadata;
+    private assignMetadataString;
+    private readAttribute;
+    private readSubState;
+    private collectNotes;
+    private collectMetadataProperties;
+    private getStateRank;
+    private getStateRankFromOption;
+    private isTranslationState;
+    getGrammar(): Grammar | undefined;
+}