hybridtm 0.3.0 → 0.5.0

package/dist/ngramIndex.js

@@ -1,132 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NGramIndex = void 0;
-const apache_arrow_1 = require("apache-arrow");
-const utils_1 = require("./utils");
-class NGramIndex {
-    db;
-    static TABLE_NAME = "ngramIndex";
-    table = null;
-    initializing = null;
-    constructor(db) {
-        this.db = db;
-        // Expect external initialization
-    }
-    async initialize() {
-        if (this.initializing) {
-            await this.initializing;
-            return;
-        }
-        this.initializing = this.ensureTable();
-        try {
-            await this.initializing;
-        }
-        finally {
-            this.initializing = null;
-        }
-    }
-    async upsert(entries) {
-        if (!entries || entries.length === 0) {
-            return;
-        }
-        const table = await this.ensureTable();
-        const uniqueIds = new Set();
-        const rows = [];
-        for (const entry of entries) {
-            uniqueIds.add(entry.entryId);
-            const uniqueNgrams = Array.from(new Set(entry.ngrams));
-            for (const ngram of uniqueNgrams) {
-                rows.push({
-                    ngram: ngram,
-                    language: entry.language,
-                    entryid: entry.entryId
-                });
-            }
-        }
-        await this.deleteByEntryIds(Array.from(uniqueIds), table);
-        if (rows.length > 0) {
-            await table.add(rows);
-        }
-    }
-    async remove(entryId) {
-        if (!entryId) {
-            return;
-        }
-        const table = await this.ensureTable();
-        await table.delete("entryid = '" + utils_1.Utils.escapeSqlLiteral(entryId) + "'");
-    }
-    async collect(language, ngrams) {
-        const result = new Map();
-        if (!language || !ngrams || ngrams.length === 0) {
-            return result;
-        }
-        const table = await this.ensureTable();
-        const unique = Array.from(new Set(ngrams));
-        const chunkSize = 512;
-        for (let i = 0; i < unique.length; i += chunkSize) {
-            const chunk = unique.slice(i, i + chunkSize);
-            if (chunk.length === 0) {
-                continue;
-            }
-            const ngramList = chunk.join(",");
-            const filter = "language = '" + language + "' AND ngram IN (" + ngramList + ")";
-            const rows = await table.query().where(filter).toArray();
-            for (const row of rows) {
-                const existing = result.get(row.entryid);
-                if (typeof existing === "number") {
-                    result.set(row.entryid, existing + 1);
-                }
-                else {
-                    result.set(row.entryid, 1);
-                }
-            }
-        }
-        return result;
-    }
-    async ensureTable() {
-        if (this.table) {
-            return this.table;
-        }
-        const names = await this.db.tableNames();
-        if (!names.includes(NGramIndex.TABLE_NAME)) {
-            const schema = new apache_arrow_1.Schema([
-                apache_arrow_1.Field.new("ngram", new apache_arrow_1.Int32(), false),
-                apache_arrow_1.Field.new("language", new apache_arrow_1.Utf8(), false),
-                apache_arrow_1.Field.new("entryid", new apache_arrow_1.Utf8(), false)
-            ]);
-            this.table = await this.db.createEmptyTable(NGramIndex.TABLE_NAME, schema);
-        }
-        else {
-            this.table = await this.db.openTable(NGramIndex.TABLE_NAME);
-        }
-        return this.table;
-    }
-    async deleteByEntryIds(entryIds, table) {
-        if (!entryIds || entryIds.length === 0) {
-            return;
-        }
-        const target = table ? table : await this.ensureTable();
-        const uniqueIds = Array.from(new Set(entryIds));
-        const chunkSize = 256;
-        for (let i = 0; i < uniqueIds.length; i += chunkSize) {
-            const chunk = uniqueIds.slice(i, i + chunkSize).map(function (id) {
-                return "'" + utils_1.Utils.escapeSqlLiteral(id) + "'";
-            });
-            const filter = "entryid IN (" + chunk.join(",") + ")";
-            await target.delete(filter);
-        }
-    }
-}
-exports.NGramIndex = NGramIndex;
-//# sourceMappingURL=ngramIndex.js.map

package/dist/ngramIndex.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ngramIndex.js","sourceRoot":"","sources":["../ts/ngramIndex.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;;AAGjF,+CAA0D;AAC1D,mCAAgC;AAehC,MAAa,UAAU;IAOU;IALrB,MAAM,CAAU,UAAU,GAAW,YAAY,CAAC;IAElD,KAAK,GAAiB,IAAI,CAAC;IAC3B,YAAY,GAA0B,IAAI,CAAC;IAEnD,YAA6B,EAAc;QAAd,OAAE,GAAF,EAAE,CAAY;QACvC,iCAAiC;IACrC,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,YAAY,CAAC;YACxB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACvC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO;QACX,CAAC;QACD,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,SAAS,GAAgB,IAAI,GAAG,EAAU,CAAC;QACjD,MAAM,IAAI,GAAoB,EAAE,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC7B,MAAM,YAAY,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACjE,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC;oBACN,KAAK,EAAE,KAAK;oBACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;iBACzB,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAE1D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAe;QACxB,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,aAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAgB,EAAE,MAAgB;QAC5C,MAAM,MAAM,GAAwB,IAAI,GAAG,EAAkB,CAAC;QAC9D,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9C,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAW,GAAG,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACxD,MAAM,KAAK,GAAa,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YACvD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,SAAS;YACb,CAAC;YACD,MAAM,SAAS,GAAW,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAW,cAAc,GAAG,QAAQ,GAAG,kBAAkB,GAAG,SAAS,GAAG,GAAG,CAAC;YACxF,MAAM,IAAI,GAAoB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,EAAqB,CAAC;YAC7F,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAuB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC7D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAC/B,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,WAAW;QACrB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,MAAM,KAAK,GAAa,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,MAAM,MAAM,GAAW,IAAI,qBAAM,CAAC;gBAC9B,oBAAK,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,oBAAK,EAAE,EAAE,KAAK,CAAC;gBACtC,oBAAK,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,mBAAI,EAAE,EAAE,KAAK,CAAC;gBACxC,oBAAK,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,mBAAI,EAAE,EAAE,KAAK,CAAC;aAC1C,CAAC,CAAC;YACH,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,UAAU,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,QAAkB,EAAE,KAAa;QAC5D,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAU,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAW,GAAG,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAa,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,UAAS,EAAU;gBAC7E,OAAO,GAAG,GAAG,aAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAW,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YAC9D,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;;AA3HL,gCA4HC"}

package/dist/ngrams.d.ts

@@ -1,21 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export declare class NGrams {
-    private static readonly NGRAM_SIZE;
-    static readonly SEPARATORS: string;
-    static readonly TERM_SEPARATORS: string;
-    private constructor();
-    static getNGrams(source: string): number[];
-    static buildWordList(source: string, separators: string): string[];
-    private static escapeForRegex;
-    private static javaHashCode;
-}

package/dist/ngrams.js

@@ -1,73 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NGrams = void 0;
-class NGrams {
-    static NGRAM_SIZE = 3;
-    static SEPARATORS = " \r\n\f\t\u2028\u2029,.;\":<>¿?¡!()[]{}=+-/*\u00AB\u00BB\u201C\u201D\u201E\uFF00";
-    static TERM_SEPARATORS = " \u00A0\r\n\f\t\u2028\u2029,.;\":<>¿?¡!()[]{}=+/*\u00AB\u00BB\u201C\u201D\u201E\uFF00";
-    constructor() {
-        // Utility class
-    }
-    static getNGrams(source) {
-        if (!source) {
-            return [];
-        }
-        const src = source.toLowerCase();
-        const words = NGrams.buildWordList(src, NGrams.SEPARATORS);
-        const unique = new Set();
-        for (const word of words) {
-            if (!word) {
-                continue;
-            }
-            const length = word.length;
-            const chars = Array.from(word);
-            const segments = Math.ceil(length / NGrams.NGRAM_SIZE);
-            for (let i = 0; i < segments; i++) {
-                let gram = "";
-                for (let j = 0; j < NGrams.NGRAM_SIZE; j++) {
-                    const idx = i * NGrams.NGRAM_SIZE + j;
-                    if (idx < length) {
-                        gram += chars[idx];
-                    }
-                }
-                if (gram.length > 0) {
-                    unique.add(NGrams.javaHashCode(gram));
-                }
-            }
-        }
-        return Array.from(unique);
-    }
-    static buildWordList(source, separators) {
-        if (!source) {
-            return [];
-        }
-        const escaped = NGrams.escapeForRegex(separators);
-        const regex = new RegExp("[" + escaped + "]+", "g");
-        return source.split(regex).filter((token) => token.length > 0);
-    }
-    static escapeForRegex(value) {
-        return value.replace(/[\\\-\[\]{}()*+?.,^$|#\s]/g, function (match) {
-            return "\\" + match;
-        });
-    }
-    static javaHashCode(value) {
-        let hash = 0;
-        for (let i = 0; i < value.length; i++) {
-            hash = ((hash << 5) - hash + value.charCodeAt(i)) | 0;
-        }
-        return hash;
-    }
-}
-exports.NGrams = NGrams;
-//# sourceMappingURL=ngrams.js.map

package/dist/ngrams.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ngrams.js","sourceRoot":"","sources":["../ts/ngrams.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;;AAEjF,MAAa,MAAM;IAEP,MAAM,CAAU,UAAU,GAAW,CAAC,CAAC;IACxC,MAAM,CAAU,UAAU,GAAW,kFAAkF,CAAC;IACxH,MAAM,CAAU,eAAe,GAAW,uFAAuF,CAAC;IAEzI;QACI,gBAAgB;IACpB,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,MAAc;QAClC,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,GAAG,GAAW,MAAM,CAAC,WAAW,EAAE,CAAC;QACzC,MAAM,KAAK,GAAa,MAAM,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACrE,MAAM,MAAM,GAAgB,IAAI,GAAG,EAAU,CAAC;QAE9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,SAAS;YACb,CAAC;YACD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,CAAC;YACnC,MAAM,KAAK,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;YAC/D,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,IAAI,IAAI,GAAW,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACjD,MAAM,GAAG,GAAW,CAAC,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC;oBAC9C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;wBACf,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;oBACvB,CAAC;gBACL,CAAC;gBACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC1C,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,MAAc,EAAE,UAAkB;QAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAW,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,GAAG,GAAG,OAAO,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3E,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAa;QACvC,OAAO,KAAK,CAAC,OAAO,CAAC,4BAA4B,EAAE,UAAS,KAAa;YACrE,OAAO,IAAI,GAAG,KAAK,CAAC;QACxB,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,KAAa;QACrC,IAAI,IAAI,GAAW,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;;AA9DL,wBA+DC"}

package/dist/package.json

@@ -1,35 +0,0 @@
-{
-    "name": "hybridtm",
-    "productName": "HybridTM",
-    "version": "0.2.0",
-    "description": "HybridTM is a Translation Memory (TM) engine that combines semantic embeddings with fuzzy string matching.",
-    "main": "dist/index.js",
-    "type": "module",
-    "types": "dist/index.d.ts",
-    "files": [
-        "/dist"
-    ],
-    "scripts": {
-        "build": "tsc"
-    },
-    "author": {
-        "name": "Rodolfo M. Raya",
-        "url": "https://maxprograms.com",
-        "email": "rmraya@maxprograms.com"
-    },
-    "keywords": [],
-    "license": "EPL-1.0",
-    "dependencies": {
-        "@lancedb/lancedb": "^0.23.0",
-        "@xenova/transformers": "^2.17.2",
-        "apache-arrow": "^18.1.0",
-        "sdltm": "^2.0.0",
-        "typesbcp47": "^1.17.0",
-        "typesxml": "^1.17.0"
-    },
-    "devDependencies": {
-        "@types/node": "^24.10.4",
-        "ts-node": "^10.9.2",
-        "typescript": "^5.9.3"
-    }
-}

package/dist/progressProvider.d.ts

@@ -1,17 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export interface ProgressProvider {
-    getProgress(): {
-        processed: number;
-        pending: number;
-    };
-}

package/dist/progressProvider.js

@@ -1,14 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=progressProvider.js.map

package/dist/progressProvider.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"progressProvider.js","sourceRoot":"","sources":["../ts/progressProvider.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF"}

package/dist/simplexlifftest.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/simplexlifftest.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const xliffreader_1 = require("./xliffreader");
-async function test() {
-    const reader = new xliffreader_1.XLIFFReader('/tmp/test.xlf');
-    try {
-        console.log('Starting parse...');
-        await reader.parse();
-        console.log('Parse completed!');
-        console.log('Entry count:', reader.getEntryCount());
-    }
-    catch (error) {
-        console.error('Error:', error);
-    }
-}
-test().then(() => {
-    console.log('Test finished');
-    process.exit(0);
-}).catch((error) => {
-    console.error('Test failed:', error);
-    process.exit(1);
-});
-//# sourceMappingURL=simplexlifftest.js.map

package/dist/simplexlifftest.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"simplexlifftest.js","sourceRoot":"","sources":["../ts/simplexlifftest.ts"],"names":[],"mappings":";;AAAA,+CAA4C;AAE5C,KAAK,UAAU,IAAI;IACf,MAAM,MAAM,GAAG,IAAI,yBAAW,CAAC,eAAe,CAAC,CAAC;IAChD,IAAI,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,aAAa,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;IACb,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;IACf,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC"}

package/dist/test-import.d.ts

@@ -1,12 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export {};

package/dist/test-import.js

@@ -1,101 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-const hybridtm_1 = require("./hybridtm");
-async function testImportValidation() {
-    console.log('Testing import validation...\n');
-    const tm = new hybridtm_1.HybridTM('/tmp/test-validation.db', hybridtm_1.HybridTM.SPEED_MODEL);
-    // Test 1: Valid file path
-    console.log('Test 1: Valid file path');
-    try {
-        const validPath = '/Users/rmraya/Documents/GitHub/HybridTM/samples/sample.tmx';
-        console.log(`Testing with valid path: ${validPath}`);
-        // This should work (even if file doesn't exist, the filename extraction should succeed)
-        const fileName = require('path').basename(validPath);
-        console.log(`✓ Filename extracted successfully: '${fileName}'`);
-    }
-    catch (error) {
-        console.log(`✗ Unexpected error: ${error}`);
-    }
-    console.log();
-    // Test 2: Empty string path
-    console.log('Test 2: Empty string path');
-    try {
-        await tm.importTMX('');
-        console.log('✗ Should have thrown an error for empty path');
-    }
-    catch (error) {
-        console.log(`✓ Correctly threw error: ${error instanceof Error ? error.message : String(error)}`);
-    }
-    console.log();
-    // Test 3: Root path only
-    console.log('Test 3: Root path only');
-    try {
-        await tm.importTMX('/');
-        console.log('✗ Should have thrown an error for root path');
-    }
-    catch (error) {
-        console.log(`✓ Correctly threw error: ${error instanceof Error ? error.message : String(error)}`);
-    }
-    console.log();
-    // Test 4: Path with only separators
-    console.log('Test 4: Path with only separators');
-    try {
-        await tm.importXLIFF('///');
-        console.log('✗ Should have thrown an error for separator-only path');
-    }
-    catch (error) {
-        console.log(`✓ Correctly threw error: ${error instanceof Error ? error.message : String(error)}`);
-    }
-    console.log();
-    // Test 5: Valid filename (this should work even if file doesn't exist)
-    console.log('Test 5: Valid filename extraction');
-    try {
-        const testPath = '/some/valid/path/test.xliff';
-        const fileName = require('path').basename(testPath);
-        if (fileName) {
-            console.log(`✓ Valid filename extraction works: '${fileName}'`);
-        }
-        else {
-            console.log('✗ Filename extraction failed unexpectedly');
-        }
-    }
-    catch (error) {
-        console.log(`✗ Unexpected error: ${error}`);
-    }
-    console.log();
-    // Test 6: Test path.basename() edge cases directly
-    console.log('Test 6: Direct path.basename() edge cases');
-    const path = require('path');
-    const testCases = [
-        { input: '', expected: '' },
-        { input: '/', expected: '' },
-        { input: '//', expected: '' },
-        { input: '///', expected: '' },
-        { input: '/file.tmx', expected: 'file.tmx' },
-        { input: '/path/to/file.tmx', expected: 'file.tmx' },
-        { input: 'file.tmx', expected: 'file.tmx' },
-    ];
-    testCases.forEach(({ input, expected }) => {
-        const result = path.basename(input);
-        const matches = result === expected;
-        console.log(`  '${input}' -> '${result}' ${matches ? '✓' : '✗ (expected: \'' + expected + '\')'}`);
-    });
-    await tm.close();
-    console.log('\nValidation tests completed!');
-}
-testImportValidation().catch(error => {
-    console.error('Test failed:', error);
-    process.exit(1);
-});
-//# sourceMappingURL=test-import.js.map

package/dist/test-import.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"test-import.js","sourceRoot":"","sources":["../ts/test-import.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;AAEjF,yCAAsC;AAEtC,KAAK,UAAU,oBAAoB;IAC/B,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAE9C,MAAM,EAAE,GAAG,IAAI,mBAAQ,CAAC,yBAAyB,EAAE,mBAAQ,CAAC,WAAW,CAAC,CAAC;IAEzE,0BAA0B;IAC1B,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;IACvC,IAAI,CAAC;QACD,MAAM,SAAS,GAAG,4DAA4D,CAAC;QAC/E,OAAO,CAAC,GAAG,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;QAErD,wFAAwF;QACxF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,uCAAuC,QAAQ,GAAG,CAAC,CAAC;IACpE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,4BAA4B;IAC5B,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACzC,IAAI,CAAC;QACD,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtG,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,yBAAyB;IACzB,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,IAAI,CAAC;QACD,MAAM,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtG,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,oCAAoC;IACpC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,IAAI,CAAC;QACD,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;IACzE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,4BAA4B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtG,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,uEAAuE;IACvE,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,IAAI,CAAC;QACD,MAAM,QAAQ,GAAG,6BAA6B,CAAC;QAC/C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,QAAQ,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,uCAAuC,QAAQ,GAAG,CAAC,CAAC;QACpE,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;QAC7D,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,mDAAmD;IACnD,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7B,MAAM,SAAS,GAAG;QACd,EAAE,KAAK,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC3B,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC5B,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC7B,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE;QAC9B,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE;QAC5C,EAAE,KAAK,EAAE,mBAAmB,EAAE,QAAQ,EAAE,UAAU,EAAE;QACpD,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE;KAC9C,CAAC;IAEF,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,KAAK,QAAQ,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,SAAS,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,GAAG,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC;IACvG,CAAC,CAAC,CAAC;IAEH,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;IAEjB,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;AACjD,CAAC;AAED,oBAAoB,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;IACjC,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC"}

package/dist/test-progress-fix.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/test-progress-fix.js

@@ -1,32 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const hybridtm_1 = require("./hybridtm");
-async function testProgressTracking() {
-    console.log('Testing updated progress tracking...\n');
-    const tm = new hybridtm_1.HybridTM('/tmp/test-progress.lancedb', hybridtm_1.HybridTM.SPEED_MODEL);
-    let pollCount = 0;
-    const progressInterval = setInterval(() => {
-        const progress = tm.getImportProgress();
-        pollCount++;
-        console.log(`Poll ${pollCount}: processed=${progress.entriesProcessed} TUs, pending=${progress.entriesPending} TUs, importing=${progress.isImporting}`);
-        if (!progress.isImporting && pollCount > 3) {
-            clearInterval(progressInterval);
-        }
-    }, 2000);
-    try {
-        await tm.importTMX('/Users/rmraya/Desktop/Norden/NordenTM_2023_rev2.tmx');
-    }
-    catch (error) {
-        console.error('Import failed:', error);
-    }
-    finally {
-        clearInterval(progressInterval);
-    }
-    await tm.close();
-    console.log('Test completed');
-}
-testProgressTracking().catch(error => {
-    console.error('Test failed:', error);
-    process.exit(1);
-});
-//# sourceMappingURL=test-progress-fix.js.map

package/dist/test-progress-fix.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"test-progress-fix.js","sourceRoot":"","sources":["../ts/test-progress-fix.ts"],"names":[],"mappings":";;AAAA,yCAAsC;AAEtC,KAAK,UAAU,oBAAoB;IAC/B,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IAEtD,MAAM,EAAE,GAAG,IAAI,mBAAQ,CAAC,4BAA4B,EAAE,mBAAQ,CAAC,WAAW,CAAC,CAAC;IAE5E,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,EAAE;QACtC,MAAM,QAAQ,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC;QACxC,SAAS,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,QAAQ,SAAS,eAAe,QAAQ,CAAC,gBAAgB,iBAAiB,QAAQ,CAAC,cAAc,mBAAmB,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAExJ,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACzC,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACpC,CAAC;IACL,CAAC,EAAE,IAAI,CAAC,CAAC;IAET,IAAI,CAAC;QACD,MAAM,EAAE,CAAC,SAAS,CAAC,qDAAqD,CAAC,CAAC;IAC9E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;YAAS,CAAC;QACP,aAAa,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;IACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;AAClC,CAAC;AAED,oBAAoB,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;IACjC,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC"}

package/dist/test-progress.d.ts

@@ -1,12 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export {};

package/dist/test-progress.js

@@ -1,32 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-const hybridtm_1 = require("./hybridtm");
-async function testProgressTracking() {
-    console.log('Testing progress tracking...\n');
-    const tm = new hybridtm_1.HybridTM('/tmp/test-progress.lancedb', hybridtm_1.HybridTM.SPEED_MODEL);
-    // Start import (this would start parsing)
-    console.log('Starting simulated import...');
-    // In a real scenario, the progress would be polled during actual import
-    // Here we'll just verify that the progress method structure is correct
-    console.log('✓ Progress tracking structure verified');
-    console.log('✓ Expected: processed should be multiples of 1000, pending should be 0-999');
-    await tm.close();
-    console.log('\nProgress tracking test completed!');
-    console.log('\nNow run the actual TMX import to see the fixed progress tracking.');
-}
-testProgressTracking().catch(error => {
-    console.error('Test failed:', error);
-    process.exit(1);
-});
-//# sourceMappingURL=test-progress.js.map

package/dist/test-progress.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"test-progress.js","sourceRoot":"","sources":["../ts/test-progress.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;AAEjF,yCAAsC;AAEtC,KAAK,UAAU,oBAAoB;IAC/B,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAE9C,MAAM,EAAE,GAAG,IAAI,mBAAQ,CAAC,4BAA4B,EAAE,mBAAQ,CAAC,WAAW,CAAC,CAAC;IAE5E,0CAA0C;IAC1C,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;IAE5C,wEAAwE;IACxE,uEAAuE;IAEvE,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,4EAA4E,CAAC,CAAC;IAE1F,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;IAEjB,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,qEAAqE,CAAC,CAAC;AACvF,CAAC;AAED,oBAAoB,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;IACjC,OAAO,CAAC,KAAK,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC"}

package/dist/tests/vectorSimilarityComparison.d.ts

@@ -1,12 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export {};