hybridtm 0.4.0 → 0.5.0

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "hybridtm",
     "productName": "HybridTM",
-    "version": "0.4.0",
+    "version": "0.5.0",
     "description": "HybridTM is a Translation Memory (TM) engine that combines semantic embeddings with fuzzy string matching.",
     "main": "dist/index.js",
     "type": "module",
@@ -10,7 +10,8 @@
         "/dist"
     ],
     "scripts": {
-        "build": "tsc"
+        "clean": "node -e \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"",
+        "build": "npm run clean && tsc"
     },
     "author": {
         "name": "Rodolfo M. Raya",
@@ -27,12 +28,12 @@
         "@lancedb/lancedb": "^0.27.2",
         "@xenova/transformers": "^2.17.2",
         "apache-arrow": "^18.1.0",
-        "sdltm": "^2.2.0",
-        "typesbcp47": "^1.21.0",
-        "typesxml": "^2.0.0"
+        "sdltm": "^2.5.0",
+        "typesbcp47": "^1.24.0",
+        "typesxml": "^2.2.0"
     },
     "devDependencies": {
-        "@types/node": "^24.12.2",
-        "typescript": "^6.0.2"
+        "@types/node": "^24.13.0",
+        "typescript": "^6.0.3"
     }
-}
+}

package/dist/batchimporter.d.ts

@@ -1,22 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025-2026 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-import { HybridTM } from './hybridtm.js';
-export declare class BatchImporter {
-    private tm;
-    private tempFilePath;
-    private batchSize;
-    private totalEntries;
-    constructor(tm: HybridTM, tempFilePath: string, totalEntries?: number, batchSize?: number);
-    import(): Promise<void>;
-    private parseMetadata;
-    private cleanup;
-}

package/dist/batchimporter.js

@@ -1,203 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025-2026 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-import { createReadStream, existsSync, unlinkSync } from "node:fs";
-import { createInterface } from 'node:readline';
-import { Utils } from './utils.js';
-export class BatchImporter {
-    tm;
-    tempFilePath;
-    batchSize;
-    totalEntries;
-    constructor(tm, tempFilePath, totalEntries = 0, batchSize = 1000) {
-        this.tm = tm;
-        this.tempFilePath = tempFilePath;
-        this.totalEntries = totalEntries;
-        this.batchSize = batchSize;
-    }
-    async import() {
-        try {
-            console.log('Starting batch import from ' + this.tempFilePath + '...');
-            if (this.totalEntries > 0) {
-                console.log('Total entries to import: ' + this.totalEntries.toLocaleString());
-            }
-            const fileStream = createReadStream(this.tempFilePath, { encoding: 'utf8' });
-            const rl = createInterface({
-                input: fileStream,
-                crlfDelay: Infinity
-            });
-            let batch = [];
-            let totalProcessed = 0;
-            let batchCount = 0;
-            const startTime = Date.now();
-            for await (const line of rl) {
-                if (!line.trim()) {
-                    continue; // Skip empty lines
-                }
-                try {
-                    const jsonEntry = JSON.parse(line);
-                    const segmentIndex = typeof jsonEntry.segmentIndex === 'number' ? jsonEntry.segmentIndex : 0;
-                    const segmentCount = typeof jsonEntry.segmentCount === 'number' ? jsonEntry.segmentCount : 1;
-                    const metadata = this.parseMetadata(jsonEntry.metadata);
-                    // Convert JSON entry to PendingEntry
-                    const entry = {
-                        language: jsonEntry.language,
-                        fileId: jsonEntry.fileId,
-                        original: jsonEntry.original,
-                        unitId: jsonEntry.unitId,
-                        pureText: jsonEntry.pureText,
-                        element: Utils.buildXMLElement(jsonEntry.element),
-                        segmentIndex,
-                        segmentCount,
-                        metadata
-                    };
-                    batch.push(entry);
-                    // Process batch when it reaches the batch size
-                    if (batch.length >= this.batchSize) {
-                        batchCount++;
-                        await this.tm.storeBatchEntries(batch);
-                        totalProcessed += batch.length;
-                        // Calculate progress and ETA
-                        const elapsed = Date.now() - startTime;
-                        const rate = totalProcessed / (elapsed / 1000); // entries per second
-                        if (this.totalEntries > 0) {
-                            const progress = (totalProcessed / this.totalEntries) * 100;
-                            const remaining = this.totalEntries - totalProcessed;
-                            const etaSeconds = remaining / rate;
-                            const etaMinutes = Math.floor(etaSeconds / 60);
-                            const etaSecondsRemainder = Math.floor(etaSeconds % 60);
-                            console.log('Progress: ' + totalProcessed.toLocaleString() + '/' + this.totalEntries.toLocaleString() + ' (' + progress.toFixed(1) + '%) - ETA: ' + etaMinutes + 'm ' + etaSecondsRemainder + 's - ' + rate.toFixed(1) + ' entries/sec');
-                        }
-                        else {
-                            console.log('Processed: ' + totalProcessed.toLocaleString() + ' entries - ' + rate.toFixed(1) + ' entries/sec');
-                        }
-                        batch = [];
-                    }
-                }
-                catch (parseErr) {
-                    console.error('Error parsing JSONL line: ' + parseErr);
-                    // Continue with next line
-                }
-            }
-            // Process any remaining entries in the final batch
-            if (batch.length > 0) {
-                batchCount++;
-                await this.tm.storeBatchEntries(batch);
-                totalProcessed += batch.length;
-            }
-            const totalTime = (Date.now() - startTime) / 1000;
-            const minutes = Math.floor(totalTime / 60);
-            const seconds = Math.floor(totalTime % 60);
-            const avgRate = totalProcessed / totalTime;
-            console.log('\nBatch import complete!');
-            console.log('Total entries imported: ' + totalProcessed.toLocaleString());
-            console.log('Total time: ' + minutes + 'm ' + seconds + 's');
-            console.log('Average rate: ' + avgRate.toFixed(1) + ' entries/sec');
-            // Clean up the temporary file
-            await this.cleanup();
-        }
-        catch (err) {
-            console.error('Error during batch import:', err);
-            throw err;
-        }
-    }
-    parseMetadata(metadata) {
-        if (!metadata || typeof metadata !== 'object') {
-            return {};
-        }
-        const result = {};
-        const source = metadata;
-        const copyString = (key) => {
-            const value = source[key];
-            if (typeof value === 'string' && value.length > 0) {
-                result[key] = value;
-            }
-        };
-        const copyNumber = (key) => {
-            const value = source[key];
-            if (typeof value === 'number' && !Number.isNaN(value)) {
-                result[key] = value;
-            }
-        };
-        copyString('state');
-        copyString('subState');
-        copyNumber('quality');
-        copyString('creationDate');
-        copyString('creationId');
-        copyString('changeDate');
-        copyString('changeId');
-        copyString('creationTool');
-        copyString('creationToolVersion');
-        copyString('context');
-        copyNumber('usageCount');
-        copyString('lastUsageDate');
-        const notes = source.notes;
-        if (Array.isArray(notes)) {
-            const filtered = notes.filter((item) => typeof item === 'string' && item.length > 0);
-            if (filtered.length > 0) {
-                result.notes = filtered;
-            }
-        }
-        const properties = source.properties;
-        if (properties && typeof properties === 'object') {
-            const props = {};
-            Object.entries(properties).forEach(([key, value]) => {
-                if (typeof value === 'string') {
-                    props[key] = value;
-                }
-            });
-            if (Object.keys(props).length > 0) {
-                result.properties = props;
-            }
-        }
-        const segment = source.segment;
-        if (segment && typeof segment === 'object') {
-            const segmentSource = segment;
-            const segmentMetadata = {};
-            const copySegmentString = (key) => {
-                const value = segmentSource[key];
-                if (typeof value === 'string' && value.length > 0) {
-                    segmentMetadata[key] = value;
-                }
-            };
-            const copySegmentNumber = (key) => {
-                const value = segmentSource[key];
-                if (typeof value === 'number' && !Number.isNaN(value)) {
-                    segmentMetadata[key] = value;
-                }
-            };
-            copySegmentString('provider');
-            copySegmentString('fileHash');
-            copySegmentString('fileId');
-            copySegmentString('unitId');
-            copySegmentString('segmentId');
-            copySegmentString('segmentKey');
-            copySegmentNumber('segmentIndex');
-            copySegmentNumber('segmentCount');
-            if (Object.keys(segmentMetadata).length > 0) {
-                result.segment = segmentMetadata;
-            }
-        }
-        return Object.keys(result).length > 0 ? result : {};
-    }
-    async cleanup() {
-        try {
-            if (existsSync(this.tempFilePath)) {
-                unlinkSync(this.tempFilePath);
-            }
-        }
-        catch (err) {
-            console.error('Error cleaning up temporary file ' + this.tempFilePath + ':', err);
-            // Don't throw - cleanup failure shouldn't fail the import
-        }
-    }
-}
-//# sourceMappingURL=batchImporter.js.map

package/dist/constants.d.ts

@@ -1,15 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025-2026 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export declare class Constants {
-    static readonly VERSION: string;
-    static readonly BUILD: string;
-}

package/dist/constants.js

@@ -1,16 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025-2026 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export class Constants {
-    static VERSION = "1.0.0";
-    static BUILD = "20260104_0539";
-}
-//# sourceMappingURL=constants.js.map

package/dist/constants.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.js","sourceRoot":"","sources":["../ts/constants.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;iFAUiF;AAEjF,MAAM,OAAO,SAAS;IACX,MAAM,CAAU,OAAO,GAAW,OAAO,CAAC;IAC1C,MAAM,CAAU,KAAK,GAAW,eAAe,CAAC"}

package/dist/debug-progress.d.ts

@@ -1,12 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export {};

package/dist/debug-progress.js

@@ -1,49 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-const hybridtm_1 = require("./hybridtm");
-async function debugProgress() {
-    console.log('Debugging progress tracking during actual TMX import...\n');
-    const tm = new hybridtm_1.HybridTM('/tmp/debug-progress.lancedb', hybridtm_1.HybridTM.SPEED_MODEL);
-    // Start progress monitoring with debugging
-    let pollCount = 0;
-    const progressInterval = setInterval(() => {
-        const progress = tm.getImportProgress();
-        pollCount++;
-        console.log(`Poll ${pollCount}: processed=${progress.entriesProcessed}, pending=${progress.entriesPending}, importing=${progress.isImporting}, fileName='${progress.fileName}'`);
-        if (!progress.isImporting && pollCount > 5) {
-            console.log('Import appears to have finished');
-            clearInterval(progressInterval);
-        }
-    }, 1000); // Check every 1 second for debugging
-    try {
-        console.log('Starting TMX import...');
-        await tm.importTMX('/Users/rmraya/Desktop/Norden/NordenTM_2023_rev2.tmx');
-        console.log('TMX import completed');
-    }
-    catch (error) {
-        console.error('Import failed:', error);
-    }
-    finally {
-        clearInterval(progressInterval);
-    }
-    const finalProgress = tm.getImportProgress();
-    console.log(`Final: processed=${finalProgress.entriesProcessed}, pending=${finalProgress.entriesPending}`);
-    await tm.close();
-    console.log('✓ Progress debug completed');
-}
-debugProgress().catch(error => {
-    console.error('Debug failed:', error);
-    process.exit(1);
-});
-//# sourceMappingURL=debug-progress.js.map

package/dist/debug-progress.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"debug-progress.js","sourceRoot":"","sources":["../ts/debug-progress.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;AAEjF,yCAAsC;AAEtC,KAAK,UAAU,aAAa;IACxB,OAAO,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;IAEzE,MAAM,EAAE,GAAG,IAAI,mBAAQ,CAAC,6BAA6B,EAAE,mBAAQ,CAAC,WAAW,CAAC,CAAC;IAE7E,2CAA2C;IAC3C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,gBAAgB,GAAG,WAAW,CAAC,GAAG,EAAE;QACtC,MAAM,QAAQ,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC;QACxC,SAAS,EAAE,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,QAAQ,SAAS,eAAe,QAAQ,CAAC,gBAAgB,aAAa,QAAQ,CAAC,cAAc,eAAe,QAAQ,CAAC,WAAW,eAAe,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC;QAEjL,IAAI,CAAC,QAAQ,CAAC,WAAW,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACzC,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;YAC/C,aAAa,CAAC,gBAAgB,CAAC,CAAC;QACpC,CAAC;IACL,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,qCAAqC;IAE/C,IAAI,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,MAAM,EAAE,CAAC,SAAS,CAAC,qDAAqD,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;IAC3C,CAAC;YAAS,CAAC;QACP,aAAa,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAED,MAAM,aAAa,GAAG,EAAE,CAAC,iBAAiB,EAAE,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,oBAAoB,aAAa,CAAC,gBAAgB,aAAa,aAAa,CAAC,cAAc,EAAE,CAAC,CAAC;IAE3G,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;IACjB,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;AAC9C,CAAC;AAED,aAAa,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;IAC1B,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;IACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC"}

package/dist/ngramIndex.d.ts

@@ -1,31 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-import { Connection } from "@lancedb/lancedb";
-interface NGramUpsert {
-    language: string;
-    entryId: string;
-    ngrams: number[];
-}
-export declare class NGramIndex {
-    private readonly db;
-    private static readonly TABLE_NAME;
-    private table;
-    private initializing;
-    constructor(db: Connection);
-    initialize(): Promise<void>;
-    upsert(entries: NGramUpsert[]): Promise<void>;
-    remove(entryId: string): Promise<void>;
-    collect(language: string, ngrams: number[]): Promise<Map<string, number>>;
-    private ensureTable;
-    private deleteByEntryIds;
-}
-export {};

package/dist/ngramIndex.js

@@ -1,132 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NGramIndex = void 0;
-const apache_arrow_1 = require("apache-arrow");
-const utils_1 = require("./utils");
-class NGramIndex {
-    db;
-    static TABLE_NAME = "ngramIndex";
-    table = null;
-    initializing = null;
-    constructor(db) {
-        this.db = db;
-        // Expect external initialization
-    }
-    async initialize() {
-        if (this.initializing) {
-            await this.initializing;
-            return;
-        }
-        this.initializing = this.ensureTable();
-        try {
-            await this.initializing;
-        }
-        finally {
-            this.initializing = null;
-        }
-    }
-    async upsert(entries) {
-        if (!entries || entries.length === 0) {
-            return;
-        }
-        const table = await this.ensureTable();
-        const uniqueIds = new Set();
-        const rows = [];
-        for (const entry of entries) {
-            uniqueIds.add(entry.entryId);
-            const uniqueNgrams = Array.from(new Set(entry.ngrams));
-            for (const ngram of uniqueNgrams) {
-                rows.push({
-                    ngram: ngram,
-                    language: entry.language,
-                    entryid: entry.entryId
-                });
-            }
-        }
-        await this.deleteByEntryIds(Array.from(uniqueIds), table);
-        if (rows.length > 0) {
-            await table.add(rows);
-        }
-    }
-    async remove(entryId) {
-        if (!entryId) {
-            return;
-        }
-        const table = await this.ensureTable();
-        await table.delete("entryid = '" + utils_1.Utils.escapeSqlLiteral(entryId) + "'");
-    }
-    async collect(language, ngrams) {
-        const result = new Map();
-        if (!language || !ngrams || ngrams.length === 0) {
-            return result;
-        }
-        const table = await this.ensureTable();
-        const unique = Array.from(new Set(ngrams));
-        const chunkSize = 512;
-        for (let i = 0; i < unique.length; i += chunkSize) {
-            const chunk = unique.slice(i, i + chunkSize);
-            if (chunk.length === 0) {
-                continue;
-            }
-            const ngramList = chunk.join(",");
-            const filter = "language = '" + language + "' AND ngram IN (" + ngramList + ")";
-            const rows = await table.query().where(filter).toArray();
-            for (const row of rows) {
-                const existing = result.get(row.entryid);
-                if (typeof existing === "number") {
-                    result.set(row.entryid, existing + 1);
-                }
-                else {
-                    result.set(row.entryid, 1);
-                }
-            }
-        }
-        return result;
-    }
-    async ensureTable() {
-        if (this.table) {
-            return this.table;
-        }
-        const names = await this.db.tableNames();
-        if (!names.includes(NGramIndex.TABLE_NAME)) {
-            const schema = new apache_arrow_1.Schema([
-                apache_arrow_1.Field.new("ngram", new apache_arrow_1.Int32(), false),
-                apache_arrow_1.Field.new("language", new apache_arrow_1.Utf8(), false),
-                apache_arrow_1.Field.new("entryid", new apache_arrow_1.Utf8(), false)
-            ]);
-            this.table = await this.db.createEmptyTable(NGramIndex.TABLE_NAME, schema);
-        }
-        else {
-            this.table = await this.db.openTable(NGramIndex.TABLE_NAME);
-        }
-        return this.table;
-    }
-    async deleteByEntryIds(entryIds, table) {
-        if (!entryIds || entryIds.length === 0) {
-            return;
-        }
-        const target = table ? table : await this.ensureTable();
-        const uniqueIds = Array.from(new Set(entryIds));
-        const chunkSize = 256;
-        for (let i = 0; i < uniqueIds.length; i += chunkSize) {
-            const chunk = uniqueIds.slice(i, i + chunkSize).map(function (id) {
-                return "'" + utils_1.Utils.escapeSqlLiteral(id) + "'";
-            });
-            const filter = "entryid IN (" + chunk.join(",") + ")";
-            await target.delete(filter);
-        }
-    }
-}
-exports.NGramIndex = NGramIndex;
-//# sourceMappingURL=ngramIndex.js.map

package/dist/ngramIndex.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ngramIndex.js","sourceRoot":"","sources":["../ts/ngramIndex.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;;AAGjF,+CAA0D;AAC1D,mCAAgC;AAehC,MAAa,UAAU;IAOU;IALrB,MAAM,CAAU,UAAU,GAAW,YAAY,CAAC;IAElD,KAAK,GAAiB,IAAI,CAAC;IAC3B,YAAY,GAA0B,IAAI,CAAC;IAEnD,YAA6B,EAAc;QAAd,OAAE,GAAF,EAAE,CAAY;QACvC,iCAAiC;IACrC,CAAC;IAED,KAAK,CAAC,UAAU;QACZ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,YAAY,CAAC;YACxB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QACvC,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC;QAC5B,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO;QACX,CAAC;QACD,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,SAAS,GAAgB,IAAI,GAAG,EAAU,CAAC;QACjD,MAAM,IAAI,GAAoB,EAAE,CAAC;QAEjC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC7B,MAAM,YAAY,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YACjE,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC;oBACN,KAAK,EAAE,KAAK;oBACZ,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;iBACzB,CAAC,CAAC;YACP,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC;QAE1D,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAe;QACxB,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,aAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAgB,EAAE,MAAgB;QAC5C,MAAM,MAAM,GAAwB,IAAI,GAAG,EAAkB,CAAC;QAC9D,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9C,OAAO,MAAM,CAAC;QAClB,CAAC;QAED,MAAM,KAAK,GAAU,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAW,GAAG,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACxD,MAAM,KAAK,GAAa,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YACvD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,SAAS;YACb,CAAC;YACD,MAAM,SAAS,GAAW,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,MAAM,GAAW,cAAc,GAAG,QAAQ,GAAG,kBAAkB,GAAG,SAAS,GAAG,GAAG,CAAC;YACxF,MAAM,IAAI,GAAoB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,EAAqB,CAAC;YAC7F,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAuB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC7D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBAC/B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACJ,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAC/B,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,WAAW;QACrB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,MAAM,KAAK,GAAa,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC;QACnD,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACzC,MAAM,MAAM,GAAW,IAAI,qBAAM,CAAC;gBAC9B,oBAAK,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,oBAAK,EAAE,EAAE,KAAK,CAAC;gBACtC,oBAAK,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,mBAAI,EAAE,EAAE,KAAK,CAAC;gBACxC,oBAAK,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,mBAAI,EAAE,EAAE,KAAK,CAAC;aAC1C,CAAC,CAAC;YACH,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,UAAU,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,QAAkB,EAAE,KAAa;QAC5D,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAU,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAW,GAAG,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAa,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,UAAS,EAAU;gBAC7E,OAAO,GAAG,GAAG,aAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC;YAClD,CAAC,CAAC,CAAC;YACH,MAAM,MAAM,GAAW,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YAC9D,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;;AA3HL,gCA4HC"}

package/dist/ngrams.d.ts

@@ -1,21 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-export declare class NGrams {
-    private static readonly NGRAM_SIZE;
-    static readonly SEPARATORS: string;
-    static readonly TERM_SEPARATORS: string;
-    private constructor();
-    static getNGrams(source: string): number[];
-    static buildWordList(source: string, separators: string): string[];
-    private static escapeForRegex;
-    private static javaHashCode;
-}

package/dist/ngrams.js

@@ -1,73 +0,0 @@
-"use strict";
-/*******************************************************************************
- * Copyright (c) 2025 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NGrams = void 0;
-class NGrams {
-    static NGRAM_SIZE = 3;
-    static SEPARATORS = " \r\n\f\t\u2028\u2029,.;\":<>¿?¡!()[]{}=+-/*\u00AB\u00BB\u201C\u201D\u201E\uFF00";
-    static TERM_SEPARATORS = " \u00A0\r\n\f\t\u2028\u2029,.;\":<>¿?¡!()[]{}=+/*\u00AB\u00BB\u201C\u201D\u201E\uFF00";
-    constructor() {
-        // Utility class
-    }
-    static getNGrams(source) {
-        if (!source) {
-            return [];
-        }
-        const src = source.toLowerCase();
-        const words = NGrams.buildWordList(src, NGrams.SEPARATORS);
-        const unique = new Set();
-        for (const word of words) {
-            if (!word) {
-                continue;
-            }
-            const length = word.length;
-            const chars = Array.from(word);
-            const segments = Math.ceil(length / NGrams.NGRAM_SIZE);
-            for (let i = 0; i < segments; i++) {
-                let gram = "";
-                for (let j = 0; j < NGrams.NGRAM_SIZE; j++) {
-                    const idx = i * NGrams.NGRAM_SIZE + j;
-                    if (idx < length) {
-                        gram += chars[idx];
-                    }
-                }
-                if (gram.length > 0) {
-                    unique.add(NGrams.javaHashCode(gram));
-                }
-            }
-        }
-        return Array.from(unique);
-    }
-    static buildWordList(source, separators) {
-        if (!source) {
-            return [];
-        }
-        const escaped = NGrams.escapeForRegex(separators);
-        const regex = new RegExp("[" + escaped + "]+", "g");
-        return source.split(regex).filter((token) => token.length > 0);
-    }
-    static escapeForRegex(value) {
-        return value.replace(/[\\\-\[\]{}()*+?.,^$|#\s]/g, function (match) {
-            return "\\" + match;
-        });
-    }
-    static javaHashCode(value) {
-        let hash = 0;
-        for (let i = 0; i < value.length; i++) {
-            hash = ((hash << 5) - hash + value.charCodeAt(i)) | 0;
-        }
-        return hash;
-    }
-}
-exports.NGrams = NGrams;
-//# sourceMappingURL=ngrams.js.map

package/dist/ngrams.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ngrams.js","sourceRoot":"","sources":["../ts/ngrams.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;iFAUiF;;;AAEjF,MAAa,MAAM;IAEP,MAAM,CAAU,UAAU,GAAW,CAAC,CAAC;IACxC,MAAM,CAAU,UAAU,GAAW,kFAAkF,CAAC;IACxH,MAAM,CAAU,eAAe,GAAW,uFAAuF,CAAC;IAEzI;QACI,gBAAgB;IACpB,CAAC;IAEM,MAAM,CAAC,SAAS,CAAC,MAAc;QAClC,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,GAAG,GAAW,MAAM,CAAC,WAAW,EAAE,CAAC;QACzC,MAAM,KAAK,GAAa,MAAM,CAAC,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACrE,MAAM,MAAM,GAAgB,IAAI,GAAG,EAAU,CAAC;QAE9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACR,SAAS;YACb,CAAC;YACD,MAAM,MAAM,GAAW,IAAI,CAAC,MAAM,CAAC;YACnC,MAAM,KAAK,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;YAC/D,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,IAAI,IAAI,GAAW,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;oBACjD,MAAM,GAAG,GAAW,CAAC,GAAG,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC;oBAC9C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;wBACf,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;oBACvB,CAAC;gBACL,CAAC;gBACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAClB,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC1C,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,MAAc,EAAE,UAAkB;QAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAAW,MAAM,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAW,IAAI,MAAM,CAAC,GAAG,GAAG,OAAO,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3E,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,KAAa;QACvC,OAAO,KAAK,CAAC,OAAO,CAAC,4BAA4B,EAAE,UAAS,KAAa;YACrE,OAAO,IAAI,GAAG,KAAK,CAAC;QACxB,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,MAAM,CAAC,YAAY,CAAC,KAAa;QACrC,IAAI,IAAI,GAAW,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;;AA9DL,wBA+DC"}