hybridtm 0.1.0 → 0.4.0

package/dist/ts/hybridtm.js

@@ -0,0 +1,932 @@
+/*******************************************************************************
+ * Copyright (c) 2025-2026 Maxprograms.
+ *
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse   License 1.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/org/documents/epl-v10.html
+ *
+ * Contributors:
+ *     Maxprograms - initial API and implementation
+ *******************************************************************************/
+import { connect } from '@lancedb/lancedb';
+import { pipeline } from '@xenova/transformers';
+import { Field, FixedSizeList, Float32, Int32, Schema, Utf8 } from 'apache-arrow';
+import { unlinkSync } from 'node:fs';
+import { tmpdir } from "node:os";
+import { join } from 'node:path';
+import { TMReader } from 'sdltm';
+import { BatchImporter } from './batchImporter.js';
+import { resolveImportOptions } from './importOptions.js';
+import { Match } from './match.js';
+import { MatchQuality } from './matchQuality.js';
+import { TMXReader } from './tmxReader.js';
+import { Utils } from './utils.js';
+import { XLIFFReader } from './xliffReader.js';
+export class HybridTM {
+    // OPTIMIZED MODELS
+    static SPEED_MODEL = 'Xenova/bge-small-en-v1.5'; // 384-dim, optimized for real-time
+    static QUALITY_MODEL = 'Xenova/LaBSE'; // 768-dim, optimized for accuracy
+    static RESOURCE_MODEL = 'Xenova/multilingual-e5-small'; // 384-dim, optimized for modest hardware
+    name;
+    db = null;
+    table = null;
+    dbPath = '';
+    embedder = null;
+    modelName = '';
+    initialized = false;
+    initializationPromise = null;
+    constructor(name, filePath, modelName = HybridTM.QUALITY_MODEL) {
+        this.name = name;
+        this.dbPath = filePath;
+        this.modelName = modelName;
+    }
+    // ============================
+    // INITIALIZATION METHODS
+    // ============================
+    async detectModelDimensions() {
+        try {
+            // Ensure embedder is initialized
+            if (!this.embedder) {
+                await this.initializeEmbedder();
+            }
+            if (!this.embedder) {
+                throw new Error('Failed to initialize embedder for dimension detection');
+            }
+            // Generate a small test embedding
+            const testResult = await this.embedder('test', {
+                pooling: 'mean',
+                normalize: true
+            });
+            return Array.from(testResult.data).length;
+        }
+        catch (err) {
+            console.error('Error detecting model dimensions:', err);
+            throw new Error('Unable to detect model dimensions for ' + this.modelName + ': ' + (err instanceof Error ? err.message : String(err)));
+        }
+    }
+    async initialize() {
+        try {
+            // Initialize database and embedder in parallel
+            await Promise.all([
+                this.initializeDatabase(),
+                this.initializeEmbedder()
+            ]);
+        }
+        catch (err) {
+            console.error('Failed to initialize HybridTM:', err);
+            throw new Error('HybridTM initialization failed: ' + (err instanceof Error ? err.message : String(err)));
+        }
+    }
+    async initializeDatabase() {
+        try {
+            // Connect to LanceDB
+            this.db = await connect(this.dbPath);
+            // Check if table exists, if not create it
+            const tableNames = await this.db.tableNames();
+            if (!tableNames.includes('langEntry')) {
+                // Detect model dimensions first
+                const dimensions = await this.detectModelDimensions();
+                // Create Arrow schema for the langEntry table
+                const schema = new Schema([
+                    Field.new('id', new Utf8(), false),
+                    Field.new('language', new Utf8(), false),
+                    Field.new('pureText', new Utf8(), false),
+                    Field.new('element', new Utf8(), false),
+                    Field.new('fileId', new Utf8(), false),
+                    Field.new('original', new Utf8(), false),
+                    Field.new('unitId', new Utf8(), false),
+                    Field.new('segmentIndex', new Int32(), false),
+                    Field.new('segmentCount', new Int32(), false),
+                    Field.new('metadataState', new Utf8(), true),
+                    Field.new('metadataSubState', new Utf8(), true),
+                    Field.new('metadataQuality', new Int32(), true),
+                    Field.new('metadataCreationDate', new Utf8(), true),
+                    Field.new('metadataCreationId', new Utf8(), true),
+                    Field.new('metadataChangeDate', new Utf8(), true),
+                    Field.new('metadataChangeId', new Utf8(), true),
+                    Field.new('metadataCreationTool', new Utf8(), true),
+                    Field.new('metadataCreationToolVersion', new Utf8(), true),
+                    Field.new('metadataContext', new Utf8(), true),
+                    Field.new('metadataNotes', new Utf8(), true),
+                    Field.new('metadataUsageCount', new Int32(), true),
+                    Field.new('metadataLastUsageDate', new Utf8(), true),
+                    Field.new('metadataProperties', new Utf8(), true),
+                    Field.new('metadataSegment', new Utf8(), true),
+                    Field.new('vector', new FixedSizeList(dimensions, Field.new('item', new Float32(), false)), false),
+                ]);
+                // Create table with the schema
+                this.table = await this.db.createEmptyTable('langEntry', schema);
+            }
+            else {
+                this.table = await this.db.openTable('langEntry');
+            }
+        }
+        catch (err) {
+            console.error('Error initializing LanceDB:', err);
+            throw err;
+        }
+    }
+    async initializeEmbedder() {
+        try {
+            this.embedder = await pipeline('feature-extraction', this.modelName);
+        }
+        catch (err) {
+            console.error('Error initializing embedder:', err);
+            throw err;
+        }
+    }
+    async generateEmbedding(text) {
+        try {
+            // Ensure embedder is initialized
+            if (!this.embedder) {
+                await this.initializeEmbedder();
+            }
+            if (!this.embedder) {
+                throw new Error('Failed to initialize embedder');
+            }
+            // Generate embeddings using the transformer model
+            const result = await this.embedder(text, {
+                pooling: 'mean',
+                normalize: true
+            });
+            // Convert tensor to array
+            const embedding = Array.from(result.data);
+            return embedding;
+        }
+        catch (err) {
+            console.error('Error generating embedding:', err);
+            throw err;
+        }
+    }
+    async ensureInitialized() {
+        if (this.initialized) {
+            return;
+        }
+        if (this.initializationPromise) {
+            await this.initializationPromise;
+            return;
+        }
+        this.initializationPromise = this.initialize();
+        await this.initializationPromise;
+        this.initialized = true;
+    }
+    async ensureTable() {
+        await this.ensureInitialized();
+        if (!this.table) {
+            throw new Error('Database table is not available');
+        }
+        return this.table;
+    }
+    hydrateEntries(rows) {
+        if (!Array.isArray(rows)) {
+            return [];
+        }
+        return rows.map((row) => this.hydrateEntry((row ?? {})));
+    }
+    hydrateEntry(row) {
+        const metadata = this.unflattenMetadata(row);
+        const vectorData = row.vector;
+        const vector = Array.isArray(vectorData)
+            ? vectorData
+            : ArrayBuffer.isView(vectorData)
+                ? Array.from(vectorData)
+                : typeof vectorData === 'object' && vectorData !== null && typeof vectorData.toArray === 'function'
+                    ? Array.from(vectorData.toArray())
+                    : [];
+        const hydrated = { ...row };
+        hydrated.id = typeof row.id === 'string' ? row.id : '';
+        hydrated.language = typeof row.language === 'string' ? row.language : '';
+        hydrated.pureText = typeof row.pureText === 'string' ? row.pureText : '';
+        hydrated.element = typeof row.element === 'string' ? row.element : '';
+        hydrated.fileId = typeof row.fileId === 'string' ? row.fileId : '';
+        hydrated.original = typeof row.original === 'string' ? row.original : '';
+        hydrated.unitId = typeof row.unitId === 'string' ? row.unitId : '';
+        hydrated.vector = vector;
+        hydrated.segmentIndex = this.parseNumber(row.segmentIndex, 0);
+        hydrated.segmentCount = this.parseNumber(row.segmentCount, 1);
+        hydrated.metadata = metadata;
+        return hydrated;
+    }
+    flattenEntry(entry) {
+        const metadataFields = this.flattenMetadata(entry.metadata);
+        return {
+            id: entry.id,
+            language: entry.language,
+            pureText: entry.pureText,
+            element: entry.element,
+            fileId: entry.fileId,
+            original: entry.original,
+            unitId: entry.unitId,
+            segmentIndex: entry.segmentIndex,
+            segmentCount: entry.segmentCount,
+            vector: entry.vector,
+            ...metadataFields,
+        };
+    }
+    mapToSearchResult(entry) {
+        if (!entry.metadata) {
+            entry.metadata = {};
+        }
+        return {
+            id: entry.id,
+            language: entry.language,
+            pureText: entry.pureText,
+            element: entry.element,
+            fileId: entry.fileId,
+            original: entry.original,
+            unitId: entry.unitId,
+            segmentIndex: entry.segmentIndex,
+            segmentCount: entry.segmentCount,
+            metadata: entry.metadata,
+        };
+    }
+    buildEntryId(fileId, unitId, segmentIndex, lang) {
+        return fileId + ':' + unitId + ':' + segmentIndex + ':' + lang;
+    }
+    flattenMetadata(metadata) {
+        const safeMetadata = metadata ? metadata : {};
+        return {
+            metadataState: safeMetadata.state ?? null,
+            metadataSubState: safeMetadata.subState ?? null,
+            metadataQuality: typeof safeMetadata.quality === 'number' ? safeMetadata.quality : null,
+            metadataCreationDate: safeMetadata.creationDate ?? null,
+            metadataCreationId: safeMetadata.creationId ?? null,
+            metadataChangeDate: safeMetadata.changeDate ?? null,
+            metadataChangeId: safeMetadata.changeId ?? null,
+            metadataCreationTool: safeMetadata.creationTool ?? null,
+            metadataCreationToolVersion: safeMetadata.creationToolVersion ?? null,
+            metadataContext: safeMetadata.context ?? null,
+            metadataNotes: safeMetadata.notes ? JSON.stringify(safeMetadata.notes) : null,
+            metadataUsageCount: typeof safeMetadata.usageCount === 'number' ? safeMetadata.usageCount : null,
+            metadataLastUsageDate: safeMetadata.lastUsageDate ?? null,
+            metadataProperties: safeMetadata.properties ? JSON.stringify(safeMetadata.properties) : null,
+            metadataSegment: safeMetadata.segment ? JSON.stringify(safeMetadata.segment) : null,
+        };
+    }
+    unflattenMetadata(row) {
+        const metadata = {};
+        const str = (value) => {
+            if (typeof value === 'string' && value.length > 0) {
+                return value;
+            }
+            return undefined;
+        };
+        const num = (value) => {
+            if (typeof value === 'number' && !Number.isNaN(value)) {
+                return value;
+            }
+            if (typeof value === 'string' && value.length > 0) {
+                const parsed = Number(value);
+                if (!Number.isNaN(parsed)) {
+                    return parsed;
+                }
+            }
+            return undefined;
+        };
+        const stateValue = str(row.metadataState);
+        const validStatuses = ['initial', 'translated', 'reviewed', 'final'];
+        if (stateValue && validStatuses.includes(stateValue)) {
+            metadata.state = stateValue;
+        }
+        const subStateValue = str(row.metadataSubState);
+        if (subStateValue) {
+            metadata.subState = subStateValue;
+        }
+        const qualityValue = num(row.metadataQuality);
+        if (qualityValue !== undefined) {
+            metadata.quality = qualityValue;
+        }
+        const creationDateValue = str(row.metadataCreationDate);
+        if (creationDateValue) {
+            metadata.creationDate = creationDateValue;
+        }
+        const creationIdValue = str(row.metadataCreationId);
+        if (creationIdValue) {
+            metadata.creationId = creationIdValue;
+        }
+        const changeDateValue = str(row.metadataChangeDate);
+        if (changeDateValue) {
+            metadata.changeDate = changeDateValue;
+        }
+        const changeIdValue = str(row.metadataChangeId);
+        if (changeIdValue) {
+            metadata.changeId = changeIdValue;
+        }
+        const creationToolValue = str(row.metadataCreationTool);
+        if (creationToolValue) {
+            metadata.creationTool = creationToolValue;
+        }
+        const creationToolVersionValue = str(row.metadataCreationToolVersion);
+        if (creationToolVersionValue) {
+            metadata.creationToolVersion = creationToolVersionValue;
+        }
+        const contextValue = str(row.metadataContext);
+        if (contextValue) {
+            metadata.context = contextValue;
+        }
+        const notesValue = this.parseNotes(row.metadataNotes);
+        if (notesValue && notesValue.length > 0) {
+            metadata.notes = notesValue;
+        }
+        const usageCountValue = num(row.metadataUsageCount);
+        if (usageCountValue !== undefined) {
+            metadata.usageCount = usageCountValue;
+        }
+        const lastUsageDateValue = str(row.metadataLastUsageDate);
+        if (lastUsageDateValue) {
+            metadata.lastUsageDate = lastUsageDateValue;
+        }
+        const properties = this.parseProperties(row.metadataProperties);
+        if (properties && Object.keys(properties).length > 0) {
+            metadata.properties = properties;
+        }
+        const segment = this.parseSegment(row.metadataSegment);
+        if (segment) {
+            metadata.segment = segment;
+        }
+        return metadata;
+    }
+    parseNotes(value) {
+        if (typeof value !== 'string' || value.length === 0) {
+            return undefined;
+        }
+        try {
+            const parsed = JSON.parse(value);
+            if (Array.isArray(parsed)) {
+                return parsed.filter((item) => typeof item === 'string');
+            }
+        }
+        catch {
+            return undefined;
+        }
+        return undefined;
+    }
+    parseSegment(value) {
+        if (typeof value !== 'string' || value.length === 0) {
+            return undefined;
+        }
+        try {
+            const parsed = JSON.parse(value);
+            if (parsed && typeof parsed === 'object') {
+                return parsed;
+            }
+        }
+        catch (err) {
+            console.warn('Failed to parse metadata.segment:', err);
+        }
+        return undefined;
+    }
+    parseProperties(value) {
+        if (typeof value !== 'string' || value.length === 0) {
+            return undefined;
+        }
+        try {
+            const parsed = JSON.parse(value);
+            if (parsed && typeof parsed === 'object') {
+                const result = {};
+                Object.entries(parsed).forEach(([key, val]) => {
+                    if (typeof val === 'string') {
+                        result[key] = val;
+                    }
+                });
+                return result;
+            }
+        }
+        catch {
+            return undefined;
+        }
+        return undefined;
+    }
+    parseNumber(value, fallback) {
+        const parsed = typeof value === 'number' && !Number.isNaN(value)
+            ? value
+            : typeof value === 'string' && value.length > 0 && !Number.isNaN(Number(value))
+                ? Number(value)
+                : undefined;
+        return typeof parsed === 'number' ? parsed : fallback;
+    }
+    async close() {
+        try {
+            if (this.db) {
+                // LanceDB connections are automatically managed
+            }
+        }
+        catch (err) {
+            console.error('Error closing database:', err);
+        }
+    }
+    // ============================
+    // SEARCH METHODS
+    // ============================
+    async concordanceSearch(textFragment, language, limit = 100, filters) {
+        // Enhanced concordance search: finds text fragments and returns all language variants for matching units
+        try {
+            const table = await this.ensureTable();
+            const escapeLiteral = (value) => value.replaceAll("'", "''");
+            const escapedFragment = escapeLiteral(textFragment);
+            const whereFragment = 'language = ' + '\'' + language + '\'' + ' AND contains(pureText, ' + '\'' + escapedFragment + '\')';
+            const fragmentEntries = this.hydrateEntries(await table
+                .query()
+                .where(whereFragment)
+                .limit(limit)
+                .toArray());
+            const filteredEntries = filters
+                ? fragmentEntries.filter((entry) => this.metadataMatches(entry.metadata, filters))
+                : fragmentEntries;
+            if (filteredEntries.length === 0) {
+                return [];
+            }
+            const segmentDescriptors = new Map();
+            filteredEntries.forEach((entry) => {
+                const segmentIndex = typeof entry.segmentIndex === 'number' ? entry.segmentIndex : 0;
+                const descriptorKey = entry.fileId + '|' + entry.unitId + '|' + segmentIndex;
+                if (!segmentDescriptors.has(descriptorKey)) {
+                    segmentDescriptors.set(descriptorKey, {
+                        fileId: entry.fileId,
+                        unitId: entry.unitId,
+                        segmentIndex
+                    });
+                }
+            });
+            const result = [];
+            for (const descriptor of segmentDescriptors.values()) {
+                const segmentPrefix = descriptor.fileId + ':' + descriptor.unitId + ':' + descriptor.segmentIndex + ':';
+                const sanitizedPrefix = escapeLiteral(segmentPrefix);
+                const segmentVariants = this.hydrateEntries(await table
+                    .query()
+                    .where('starts_with(id, ' + '\'' + sanitizedPrefix + '\')')
+                    .toArray());
+                if (segmentVariants.length === 0) {
+                    continue;
+                }
+                const languageMap = new Map();
+                for (const variant of segmentVariants) {
+                    try {
+                        const xmlElement = Utils.buildXMLElement(variant.element);
+                        languageMap.set(variant.language, xmlElement);
+                    }
+                    catch (parseErr) {
+                        const errorMessage = parseErr instanceof Error ? parseErr.message : String(parseErr);
+                        throw new Error('Failed to create XML element for ' + variant.id + ': ' + errorMessage);
+                    }
+                }
+                if (languageMap.size > 0) {
+                    result.push(languageMap);
+                    if (result.length >= limit) {
+                        break;
+                    }
+                }
+            }
+            return result;
+        }
+        catch (err) {
+            console.error('Error performing concordance search:', err);
+            throw err;
+        }
+    }
+    async semanticSearch(queryText, language, limit = 10, filters) {
+        try {
+            const table = await this.ensureTable();
+            const queryEmbedding = await this.generateEmbedding(queryText);
+            const results = this.hydrateEntries(await table
+                .vectorSearch(queryEmbedding)
+                .where('language = ' + '\'' + language + '\'')
+                .limit(limit)
+                .toArray());
+            const filtered = filters
+                ? results.filter((entry) => this.metadataMatches(entry.metadata, filters))
+                : results;
+            return filtered.map((entry) => this.mapToSearchResult(entry));
+        }
+        catch (err) {
+            console.error('Error performing semantic search:', err);
+            throw err;
+        }
+    }
+    async semanticTranslationSearch(searchStr, srcLang, tgtLang, similarity, limit = 100, filters) {
+        try {
+            const table = await this.ensureTable();
+            // Generate embedding for the search string
+            const queryEmbedding = await this.generateEmbedding(searchStr);
+            // Get all entries for the source language
+            const sourceEntries = this.hydrateEntries(await table
+                .vectorSearch(queryEmbedding)
+                .where('language = ' + '\'' + srcLang + '\'')
+                .toArray());
+            const rankedMatches = [];
+            const sourceCriteria = filters?.source;
+            const targetCriteria = filters?.target ?? filters?.source;
+            for (const sourceEntry of sourceEntries) {
+                if (!sourceEntry.pureText || !sourceEntry.vector) {
+                    continue;
+                }
+                if (sourceCriteria && !this.metadataMatches(sourceEntry.metadata, sourceCriteria)) {
+                    continue;
+                }
+                // Linear mapping (more intuitive)
+                const rawDistance = sourceEntry._distance;
+                const l2Distance = typeof rawDistance === 'number'
+                    ? rawDistance
+                    : typeof rawDistance === 'string' && rawDistance.length > 0 && !Number.isNaN(Number(rawDistance))
+                        ? Number(rawDistance)
+                        : 0;
+                const semanticScore = Math.max(0, Math.round((2 - l2Distance) / 2 * 100));
+                const fuzzyScore = MatchQuality.similarity(searchStr, sourceEntry.pureText);
+                const hybridScore = Math.round((semanticScore + fuzzyScore) / 2);
+                // Only include matches that meet the minimum similarity threshold
+                if (hybridScore >= similarity) {
+                    const targetEntry = await this.findTargetEntry(table, sourceEntry, tgtLang, targetCriteria);
+                    if (!targetEntry) {
+                        continue;
+                    }
+                    if (targetCriteria && !this.metadataMatches(targetEntry.metadata, targetCriteria)) {
+                        continue;
+                    }
+                    try {
+                        const sourceElement = Utils.buildXMLElement(sourceEntry.element);
+                        const targetElement = Utils.buildXMLElement(targetEntry.element);
+                        const match = new Match(sourceElement, targetElement, this.name, semanticScore, fuzzyScore);
+                        const rankingScore = this.computeRankingScore(match.hybridScore(), sourceEntry, targetEntry);
+                        rankedMatches.push({ match, score: rankingScore });
+                    }
+                    catch (parseErr) {
+                        console.error('Error creating Match for semantic translation search: ' + (parseErr instanceof Error ? parseErr.message : String(parseErr)));
+                    }
+                }
+            }
+            rankedMatches.sort((a, b) => b.score - a.score);
+            return rankedMatches.slice(0, limit).map((item) => item.match);
+        }
+        catch (err) {
+            console.error('Error performing semantic search with quality:', err);
+            return [];
+        }
+    }
+    async findTargetEntry(table, sourceEntry, tgtLang, filters) {
+        if (!sourceEntry.fileId || !sourceEntry.unitId) {
+            return null;
+        }
+        const exactId = this.buildEntryId(sourceEntry.fileId, sourceEntry.unitId, sourceEntry.segmentIndex, tgtLang);
+        const exactMatches = this.hydrateEntries(await table
+            .query()
+            .where('id = ' + '\'' + Utils.replaceQuotes(exactId) + '\'')
+            .toArray());
+        const exactMatch = this.selectPreferredCandidate(exactMatches, sourceEntry.segmentIndex, filters);
+        if (exactMatch) {
+            return exactMatch;
+        }
+        const unitPrefix = sourceEntry.fileId + ':' + sourceEntry.unitId + ':';
+        const sanitizedPrefix = Utils.replaceQuotes(unitPrefix);
+        const sanitizedLang = Utils.replaceQuotes(tgtLang);
+        const unitMatches = this.hydrateEntries(await table
+            .query()
+            .where('starts_with(id, ' + '\'' + sanitizedPrefix + '\') AND language = ' + '\'' + sanitizedLang + '\'')
+            .limit(50)
+            .toArray());
+        if (unitMatches.length === 0) {
+            return null;
+        }
+        const preferred = this.selectPreferredCandidate(unitMatches, sourceEntry.segmentIndex, filters);
+        return preferred ?? null;
+    }
+    computeRankingScore(baseScore, sourceEntry, targetEntry) {
+        let score = baseScore;
+        if (sourceEntry.segmentIndex > 0 && targetEntry.segmentIndex > 0) {
+            score += sourceEntry.segmentIndex === targetEntry.segmentIndex ? 10 : 5;
+        }
+        const quality = targetEntry.metadata && typeof targetEntry.metadata.quality === 'number'
+            ? targetEntry.metadata.quality
+            : undefined;
+        if (quality !== undefined && !Number.isNaN(quality)) {
+            score += Math.min(Math.max(quality, 0), 100) / 20;
+        }
+        const recencyTimestamp = this.parseMetadataTimestamp(targetEntry.metadata?.changeDate || targetEntry.metadata?.creationDate);
+        if (recencyTimestamp !== undefined) {
+            const ageDays = (Date.now() - recencyTimestamp) / 86400000;
+            if (Number.isFinite(ageDays) && ageDays >= 0) {
+                const normalized = Math.max(0, 1 - Math.min(ageDays, 365) / 365);
+                score += normalized * 5;
+            }
+        }
+        if (targetEntry.metadata?.state === 'final') {
+            score += 3;
+        }
+        else if (targetEntry.metadata?.state === 'reviewed') {
+            score += 2;
+        }
+        else if (targetEntry.metadata?.state === 'translated') {
+            score += 1;
+        }
+        return score;
+    }
+    parseMetadataTimestamp(value) {
+        if (!value) {
+            return undefined;
+        }
+        const parsed = Date.parse(value);
+        return Number.isNaN(parsed) ? undefined : parsed;
+    }
+    selectPreferredCandidate(entries, desiredSegmentIndex, filters) {
+        if (!entries || entries.length === 0) {
+            return undefined;
+        }
+        const filtered = filters
+            ? entries.filter((entry) => this.metadataMatches(entry.metadata, filters))
+            : entries.slice();
+        if (filtered.length === 0) {
+            return undefined;
+        }
+        if (desiredSegmentIndex > 0) {
+            const exact = filtered.find((entry) => entry.segmentIndex === desiredSegmentIndex);
+            if (exact) {
+                return exact;
+            }
+        }
+        const segmentEntry = filtered.find((entry) => entry.segmentIndex > 0);
+        if (segmentEntry) {
+            return segmentEntry;
+        }
+        return filtered[0];
+    }
+    metadataMatches(metadata, filter) {
+        if (!filter) {
+            return true;
+        }
+        if (!metadata) {
+            return false;
+        }
+        if (filter.states && filter.states.length > 0) {
+            if (!metadata.state || !filter.states.includes(metadata.state)) {
+                return false;
+            }
+        }
+        if (filter.minState) {
+            const metadataRank = this.getStateRank(metadata.state);
+            if (metadataRank === 0 || metadataRank < this.getStateRank(filter.minState)) {
+                return false;
+            }
+        }
+        if (typeof filter.minQuality === 'number') {
+            if (typeof metadata.quality !== 'number' || metadata.quality < filter.minQuality) {
+                return false;
+            }
+        }
+        if (filter.contextIncludes && filter.contextIncludes.length > 0) {
+            const contextValue = (metadata.context || '').toLowerCase();
+            const matchesAll = filter.contextIncludes.every((needle) => contextValue.includes(needle.toLowerCase()));
+            if (!matchesAll) {
+                return false;
+            }
+        }
+        if (filter.requiredProperties) {
+            if (!metadata.properties) {
+                return false;
+            }
+            for (const [key, expected] of Object.entries(filter.requiredProperties)) {
+                if (metadata.properties[key] !== expected) {
+                    return false;
+                }
+            }
+        }
+        if (filter.provider) {
+            const provider = metadata.segment?.provider;
+            if (!provider || provider !== filter.provider) {
+                return false;
+            }
+        }
+        return true;
+    }
+    getStateRank(value) {
+        if (!value) {
+            return 0;
+        }
+        switch (value) {
+            case 'initial':
+                return 1;
+            case 'translated':
+                return 2;
+            case 'reviewed':
+                return 3;
+            case 'final':
+                return 4;
+            default:
+                return 0;
+        }
+    }
+    // ============================
+    // DATA MANAGEMENT METHODS
+    // ============================
+    async storeLangEntry(fileId, original, unitId, lang, pureText, element, embeddings, segmentIndex = 0, segmentCount = 1, metadata) {
+        try {
+            const table = await this.ensureTable();
+            const sanitizedFileId = Utils.replaceQuotes(fileId);
+            const sanitizedUnitId = Utils.replaceQuotes(unitId);
+            const safeSegmentIndex = typeof segmentIndex === 'number' ? segmentIndex : 0;
+            const safeSegmentCount = typeof segmentCount === 'number' && segmentCount > 0 ? segmentCount : 1;
+            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, safeSegmentIndex, lang);
+            const existingEntries = this.hydrateEntries(await table
+                .query()
+                .where('id = ' + '\'' + entryId + '\'')
+                .toArray());
+            if (existingEntries.length > 0) {
+                const existingEntry = existingEntries[0];
+                const contentChanged = (existingEntry.pureText !== pureText ||
+                    existingEntry.element !== element.toString() ||
+                    existingEntry.original !== original);
+                if (!contentChanged) {
+                    return;
+                }
+            }
+            // Generate embeddings if not provided
+            let vectorEmbeddings;
+            if (embeddings && embeddings.length > 0) {
+                vectorEmbeddings = embeddings;
+            }
+            else {
+                vectorEmbeddings = await this.generateEmbedding(pureText);
+            }
+            const entry = {
+                id: entryId,
+                language: lang,
+                pureText: pureText,
+                element: element.toString(),
+                fileId: sanitizedFileId,
+                original: original,
+                unitId: sanitizedUnitId,
+                vector: vectorEmbeddings,
+                segmentIndex: safeSegmentIndex,
+                segmentCount: safeSegmentCount,
+                metadata: metadata ?? {}
+            };
+            // Delete existing entry first (LanceDB upsert approach)
+            try {
+                await table.delete('id = ' + '\'' + entryId + '\'');
+            }
+            catch (deleteErr) {
+                // Entry might not exist, which is fine
+            }
+            await table.add([this.flattenEntry(entry)]);
+        }
+        catch (err) {
+            console.error('Error storing language entry:', err);
+            throw err;
+        }
+    }
+    async storeBatchEntries(entries) {
+        try {
+            const table = await this.ensureTable();
+            const langEntries = [];
+            // Collect all entry IDs for bulk deletion
+            const entryIds = [];
+            // Generate embeddings one by one and build LangEntry objects
+            for (const entry of entries) {
+                const sanitizedFileId = Utils.replaceQuotes(entry.fileId);
+                const sanitizedUnitId = Utils.replaceQuotes(entry.unitId);
+                const vectorEmbeddings = await this.generateEmbedding(entry.pureText);
+                const safeSegmentIndex = typeof entry.segmentIndex === 'number' ? entry.segmentIndex : 0;
+                const safeSegmentCount = typeof entry.segmentCount === 'number' && entry.segmentCount > 0 ? entry.segmentCount : 1;
+                const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, safeSegmentIndex, entry.language);
+                const langEntry = {
+                    id: entryId,
+                    language: entry.language,
+                    pureText: entry.pureText,
+                    element: entry.element.toString(),
+                    fileId: sanitizedFileId,
+                    original: entry.original,
+                    unitId: sanitizedUnitId,
+                    vector: vectorEmbeddings,
+                    segmentIndex: safeSegmentIndex,
+                    segmentCount: safeSegmentCount,
+                    metadata: entry.metadata ?? {}
+                };
+                langEntries.push(langEntry);
+                entryIds.push(entryId);
+            }
+            // Delete any existing entries with these IDs to prevent duplicates
+            // This ensures that if the same file is imported twice, entries are overwritten
+            if (entryIds.length > 0) {
+                const idsFilter = entryIds.map(id => '\'' + id + '\'').join(',');
+                try {
+                    await table.delete('id IN (' + idsFilter + ')');
+                }
+                catch (deleteErr) {
+                    // Entries might not exist, which is fine for first import
+                }
+            }
+            // Bulk insert all entries at once
+            const flattenedEntries = langEntries.map((item) => this.flattenEntry(item));
+            await table.add(flattenedEntries);
+        }
+        catch (err) {
+            console.error('Error storing batch entries:', err);
+            throw err;
+        }
+    }
+    async entryExists(fileId, unitId, lang, segmentIndex = 0) {
+        try {
+            if (!this.table) {
+                await this.initializeDatabase();
+            }
+            if (!this.table) {
+                throw new Error('Failed to initialize database table');
+            }
+            const sanitizedFileId = Utils.replaceQuotes(fileId);
+            const sanitizedUnitId = Utils.replaceQuotes(unitId);
+            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
+            const existingEntries = this.hydrateEntries(await this.table
+                .query()
+                .where('id = ' + '\'' + entryId + '\'')
+                .toArray());
+            return existingEntries.length > 0;
+        }
+        catch (err) {
+            console.error('Error checking entry existence:', err);
+            return false;
+        }
+    }
+    async getLangEntry(fileId, unitId, lang, segmentIndex = 0) {
+        try {
+            if (!this.table) {
+                await this.initializeDatabase();
+            }
+            if (!this.table) {
+                throw new Error('Failed to initialize database table');
+            }
+            const sanitizedFileId = Utils.replaceQuotes(fileId);
+            const sanitizedUnitId = Utils.replaceQuotes(unitId);
+            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
+            const existingEntries = this.hydrateEntries(await this.table
+                .query()
+                .where('id = ' + '\'' + entryId + '\'')
+                .toArray());
+            return existingEntries.length > 0 ? existingEntries[0] : null;
+        }
+        catch (err) {
+            console.error('Error retrieving language entry:', err);
+            return null;
+        }
+    }
+    async deleteLangEntry(fileId, unitId, lang, segmentIndex = 0) {
+        try {
+            if (!this.table) {
+                await this.initializeDatabase();
+            }
+            if (!this.table) {
+                throw new Error('Failed to initialize database table');
+            }
+            const sanitizedFileId = Utils.replaceQuotes(fileId);
+            const sanitizedUnitId = Utils.replaceQuotes(unitId);
+            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
+            // Check if entry exists first
+            const exists = await this.entryExists(fileId, unitId, lang, segmentIndex);
+            if (!exists) {
+                return false;
+            }
+            // Delete the entry
+            await this.table.delete('id = \'' + entryId + '\'');
+            return true;
+        }
+        catch (err) {
+            console.error('Error deleting language entry:', err);
+            return false;
+        }
+    }
+    // ============================
+    // DATA IMPORT METHODS
+    // ============================
+    async importXLIFF(filePath, options) {
+        // Phase 1: Parse XLIFF and write to temporary JSONL file
+        const resolvedOptions = resolveImportOptions(options);
+        const reader = new XLIFFReader(filePath, resolvedOptions);
+        await reader.parse();
+        // Phase 2: Batch import from JSONL file (asynchronous)
+        const importer = new BatchImporter(this, reader.getTempFilePath(), reader.getEntryCount());
+        await importer.import();
+    }
+    async importTMX(filePath, options) {
+        // Phase 1: Parse TMX and write to temporary JSONL file
+        const resolvedOptions = resolveImportOptions(options);
+        const reader = new TMXReader(filePath, resolvedOptions);
+        await reader.parse();
+        // Phase 2: Batch import from JSONL file (asynchronous)
+        const importer = new BatchImporter(this, reader.getTempFilePath(), reader.getEntryCount());
+        await importer.import();
+    }
+    async importSDLTM(filePath, options) {
+        const tempDir = tmpdir();
+        const tempFileName = 'tmx_' + Date.now() + '_' + Math.random().toString(36).substring(7) + '.tmx';
+        const tempFilePath = join(tempDir, tempFileName);
+        const packageJson = await import('../package.json', { assert: { type: 'json' } });
+        const productName = packageJson.default.productName;
+        const version = packageJson.default.version;
+        const tmReader = new TMReader({ 'productName': productName, 'version': version });
+        const data = await tmReader.convert(filePath, tempFilePath);
+        if (data.status !== 'Success') {
+            throw new Error('SDL TM conversion failed for file ' + filePath);
+        }
+        await this.importTMX(tempFilePath, options);
+        unlinkSync(tempFilePath);
+    }
+}
+//# sourceMappingURL=hybridtm.js.map