hybridtm 0.4.0 → 0.6.0

package/dist/ts/hybridtm.js

@@ -1,932 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2025-2026 Maxprograms.
- *
- * This program and the accompanying materials
- * are made available under the terms of the Eclipse   License 1.0
- * which accompanies this distribution, and is available at
- * https://www.eclipse.org/org/documents/epl-v10.html
- *
- * Contributors:
- *     Maxprograms - initial API and implementation
- *******************************************************************************/
-import { connect } from '@lancedb/lancedb';
-import { pipeline } from '@xenova/transformers';
-import { Field, FixedSizeList, Float32, Int32, Schema, Utf8 } from 'apache-arrow';
-import { unlinkSync } from 'node:fs';
-import { tmpdir } from "node:os";
-import { join } from 'node:path';
-import { TMReader } from 'sdltm';
-import { BatchImporter } from './batchImporter.js';
-import { resolveImportOptions } from './importOptions.js';
-import { Match } from './match.js';
-import { MatchQuality } from './matchQuality.js';
-import { TMXReader } from './tmxReader.js';
-import { Utils } from './utils.js';
-import { XLIFFReader } from './xliffReader.js';
-export class HybridTM {
-    // OPTIMIZED MODELS
-    static SPEED_MODEL = 'Xenova/bge-small-en-v1.5'; // 384-dim, optimized for real-time
-    static QUALITY_MODEL = 'Xenova/LaBSE'; // 768-dim, optimized for accuracy
-    static RESOURCE_MODEL = 'Xenova/multilingual-e5-small'; // 384-dim, optimized for modest hardware
-    name;
-    db = null;
-    table = null;
-    dbPath = '';
-    embedder = null;
-    modelName = '';
-    initialized = false;
-    initializationPromise = null;
-    constructor(name, filePath, modelName = HybridTM.QUALITY_MODEL) {
-        this.name = name;
-        this.dbPath = filePath;
-        this.modelName = modelName;
-    }
-    // ============================
-    // INITIALIZATION METHODS
-    // ============================
-    async detectModelDimensions() {
-        try {
-            // Ensure embedder is initialized
-            if (!this.embedder) {
-                await this.initializeEmbedder();
-            }
-            if (!this.embedder) {
-                throw new Error('Failed to initialize embedder for dimension detection');
-            }
-            // Generate a small test embedding
-            const testResult = await this.embedder('test', {
-                pooling: 'mean',
-                normalize: true
-            });
-            return Array.from(testResult.data).length;
-        }
-        catch (err) {
-            console.error('Error detecting model dimensions:', err);
-            throw new Error('Unable to detect model dimensions for ' + this.modelName + ': ' + (err instanceof Error ? err.message : String(err)));
-        }
-    }
-    async initialize() {
-        try {
-            // Initialize database and embedder in parallel
-            await Promise.all([
-                this.initializeDatabase(),
-                this.initializeEmbedder()
-            ]);
-        }
-        catch (err) {
-            console.error('Failed to initialize HybridTM:', err);
-            throw new Error('HybridTM initialization failed: ' + (err instanceof Error ? err.message : String(err)));
-        }
-    }
-    async initializeDatabase() {
-        try {
-            // Connect to LanceDB
-            this.db = await connect(this.dbPath);
-            // Check if table exists, if not create it
-            const tableNames = await this.db.tableNames();
-            if (!tableNames.includes('langEntry')) {
-                // Detect model dimensions first
-                const dimensions = await this.detectModelDimensions();
-                // Create Arrow schema for the langEntry table
-                const schema = new Schema([
-                    Field.new('id', new Utf8(), false),
-                    Field.new('language', new Utf8(), false),
-                    Field.new('pureText', new Utf8(), false),
-                    Field.new('element', new Utf8(), false),
-                    Field.new('fileId', new Utf8(), false),
-                    Field.new('original', new Utf8(), false),
-                    Field.new('unitId', new Utf8(), false),
-                    Field.new('segmentIndex', new Int32(), false),
-                    Field.new('segmentCount', new Int32(), false),
-                    Field.new('metadataState', new Utf8(), true),
-                    Field.new('metadataSubState', new Utf8(), true),
-                    Field.new('metadataQuality', new Int32(), true),
-                    Field.new('metadataCreationDate', new Utf8(), true),
-                    Field.new('metadataCreationId', new Utf8(), true),
-                    Field.new('metadataChangeDate', new Utf8(), true),
-                    Field.new('metadataChangeId', new Utf8(), true),
-                    Field.new('metadataCreationTool', new Utf8(), true),
-                    Field.new('metadataCreationToolVersion', new Utf8(), true),
-                    Field.new('metadataContext', new Utf8(), true),
-                    Field.new('metadataNotes', new Utf8(), true),
-                    Field.new('metadataUsageCount', new Int32(), true),
-                    Field.new('metadataLastUsageDate', new Utf8(), true),
-                    Field.new('metadataProperties', new Utf8(), true),
-                    Field.new('metadataSegment', new Utf8(), true),
-                    Field.new('vector', new FixedSizeList(dimensions, Field.new('item', new Float32(), false)), false),
-                ]);
-                // Create table with the schema
-                this.table = await this.db.createEmptyTable('langEntry', schema);
-            }
-            else {
-                this.table = await this.db.openTable('langEntry');
-            }
-        }
-        catch (err) {
-            console.error('Error initializing LanceDB:', err);
-            throw err;
-        }
-    }
-    async initializeEmbedder() {
-        try {
-            this.embedder = await pipeline('feature-extraction', this.modelName);
-        }
-        catch (err) {
-            console.error('Error initializing embedder:', err);
-            throw err;
-        }
-    }
-    async generateEmbedding(text) {
-        try {
-            // Ensure embedder is initialized
-            if (!this.embedder) {
-                await this.initializeEmbedder();
-            }
-            if (!this.embedder) {
-                throw new Error('Failed to initialize embedder');
-            }
-            // Generate embeddings using the transformer model
-            const result = await this.embedder(text, {
-                pooling: 'mean',
-                normalize: true
-            });
-            // Convert tensor to array
-            const embedding = Array.from(result.data);
-            return embedding;
-        }
-        catch (err) {
-            console.error('Error generating embedding:', err);
-            throw err;
-        }
-    }
-    async ensureInitialized() {
-        if (this.initialized) {
-            return;
-        }
-        if (this.initializationPromise) {
-            await this.initializationPromise;
-            return;
-        }
-        this.initializationPromise = this.initialize();
-        await this.initializationPromise;
-        this.initialized = true;
-    }
-    async ensureTable() {
-        await this.ensureInitialized();
-        if (!this.table) {
-            throw new Error('Database table is not available');
-        }
-        return this.table;
-    }
-    hydrateEntries(rows) {
-        if (!Array.isArray(rows)) {
-            return [];
-        }
-        return rows.map((row) => this.hydrateEntry((row ?? {})));
-    }
-    hydrateEntry(row) {
-        const metadata = this.unflattenMetadata(row);
-        const vectorData = row.vector;
-        const vector = Array.isArray(vectorData)
-            ? vectorData
-            : ArrayBuffer.isView(vectorData)
-                ? Array.from(vectorData)
-                : typeof vectorData === 'object' && vectorData !== null && typeof vectorData.toArray === 'function'
-                    ? Array.from(vectorData.toArray())
-                    : [];
-        const hydrated = { ...row };
-        hydrated.id = typeof row.id === 'string' ? row.id : '';
-        hydrated.language = typeof row.language === 'string' ? row.language : '';
-        hydrated.pureText = typeof row.pureText === 'string' ? row.pureText : '';
-        hydrated.element = typeof row.element === 'string' ? row.element : '';
-        hydrated.fileId = typeof row.fileId === 'string' ? row.fileId : '';
-        hydrated.original = typeof row.original === 'string' ? row.original : '';
-        hydrated.unitId = typeof row.unitId === 'string' ? row.unitId : '';
-        hydrated.vector = vector;
-        hydrated.segmentIndex = this.parseNumber(row.segmentIndex, 0);
-        hydrated.segmentCount = this.parseNumber(row.segmentCount, 1);
-        hydrated.metadata = metadata;
-        return hydrated;
-    }
-    flattenEntry(entry) {
-        const metadataFields = this.flattenMetadata(entry.metadata);
-        return {
-            id: entry.id,
-            language: entry.language,
-            pureText: entry.pureText,
-            element: entry.element,
-            fileId: entry.fileId,
-            original: entry.original,
-            unitId: entry.unitId,
-            segmentIndex: entry.segmentIndex,
-            segmentCount: entry.segmentCount,
-            vector: entry.vector,
-            ...metadataFields,
-        };
-    }
-    mapToSearchResult(entry) {
-        if (!entry.metadata) {
-            entry.metadata = {};
-        }
-        return {
-            id: entry.id,
-            language: entry.language,
-            pureText: entry.pureText,
-            element: entry.element,
-            fileId: entry.fileId,
-            original: entry.original,
-            unitId: entry.unitId,
-            segmentIndex: entry.segmentIndex,
-            segmentCount: entry.segmentCount,
-            metadata: entry.metadata,
-        };
-    }
-    buildEntryId(fileId, unitId, segmentIndex, lang) {
-        return fileId + ':' + unitId + ':' + segmentIndex + ':' + lang;
-    }
-    flattenMetadata(metadata) {
-        const safeMetadata = metadata ? metadata : {};
-        return {
-            metadataState: safeMetadata.state ?? null,
-            metadataSubState: safeMetadata.subState ?? null,
-            metadataQuality: typeof safeMetadata.quality === 'number' ? safeMetadata.quality : null,
-            metadataCreationDate: safeMetadata.creationDate ?? null,
-            metadataCreationId: safeMetadata.creationId ?? null,
-            metadataChangeDate: safeMetadata.changeDate ?? null,
-            metadataChangeId: safeMetadata.changeId ?? null,
-            metadataCreationTool: safeMetadata.creationTool ?? null,
-            metadataCreationToolVersion: safeMetadata.creationToolVersion ?? null,
-            metadataContext: safeMetadata.context ?? null,
-            metadataNotes: safeMetadata.notes ? JSON.stringify(safeMetadata.notes) : null,
-            metadataUsageCount: typeof safeMetadata.usageCount === 'number' ? safeMetadata.usageCount : null,
-            metadataLastUsageDate: safeMetadata.lastUsageDate ?? null,
-            metadataProperties: safeMetadata.properties ? JSON.stringify(safeMetadata.properties) : null,
-            metadataSegment: safeMetadata.segment ? JSON.stringify(safeMetadata.segment) : null,
-        };
-    }
-    unflattenMetadata(row) {
-        const metadata = {};
-        const str = (value) => {
-            if (typeof value === 'string' && value.length > 0) {
-                return value;
-            }
-            return undefined;
-        };
-        const num = (value) => {
-            if (typeof value === 'number' && !Number.isNaN(value)) {
-                return value;
-            }
-            if (typeof value === 'string' && value.length > 0) {
-                const parsed = Number(value);
-                if (!Number.isNaN(parsed)) {
-                    return parsed;
-                }
-            }
-            return undefined;
-        };
-        const stateValue = str(row.metadataState);
-        const validStatuses = ['initial', 'translated', 'reviewed', 'final'];
-        if (stateValue && validStatuses.includes(stateValue)) {
-            metadata.state = stateValue;
-        }
-        const subStateValue = str(row.metadataSubState);
-        if (subStateValue) {
-            metadata.subState = subStateValue;
-        }
-        const qualityValue = num(row.metadataQuality);
-        if (qualityValue !== undefined) {
-            metadata.quality = qualityValue;
-        }
-        const creationDateValue = str(row.metadataCreationDate);
-        if (creationDateValue) {
-            metadata.creationDate = creationDateValue;
-        }
-        const creationIdValue = str(row.metadataCreationId);
-        if (creationIdValue) {
-            metadata.creationId = creationIdValue;
-        }
-        const changeDateValue = str(row.metadataChangeDate);
-        if (changeDateValue) {
-            metadata.changeDate = changeDateValue;
-        }
-        const changeIdValue = str(row.metadataChangeId);
-        if (changeIdValue) {
-            metadata.changeId = changeIdValue;
-        }
-        const creationToolValue = str(row.metadataCreationTool);
-        if (creationToolValue) {
-            metadata.creationTool = creationToolValue;
-        }
-        const creationToolVersionValue = str(row.metadataCreationToolVersion);
-        if (creationToolVersionValue) {
-            metadata.creationToolVersion = creationToolVersionValue;
-        }
-        const contextValue = str(row.metadataContext);
-        if (contextValue) {
-            metadata.context = contextValue;
-        }
-        const notesValue = this.parseNotes(row.metadataNotes);
-        if (notesValue && notesValue.length > 0) {
-            metadata.notes = notesValue;
-        }
-        const usageCountValue = num(row.metadataUsageCount);
-        if (usageCountValue !== undefined) {
-            metadata.usageCount = usageCountValue;
-        }
-        const lastUsageDateValue = str(row.metadataLastUsageDate);
-        if (lastUsageDateValue) {
-            metadata.lastUsageDate = lastUsageDateValue;
-        }
-        const properties = this.parseProperties(row.metadataProperties);
-        if (properties && Object.keys(properties).length > 0) {
-            metadata.properties = properties;
-        }
-        const segment = this.parseSegment(row.metadataSegment);
-        if (segment) {
-            metadata.segment = segment;
-        }
-        return metadata;
-    }
-    parseNotes(value) {
-        if (typeof value !== 'string' || value.length === 0) {
-            return undefined;
-        }
-        try {
-            const parsed = JSON.parse(value);
-            if (Array.isArray(parsed)) {
-                return parsed.filter((item) => typeof item === 'string');
-            }
-        }
-        catch {
-            return undefined;
-        }
-        return undefined;
-    }
-    parseSegment(value) {
-        if (typeof value !== 'string' || value.length === 0) {
-            return undefined;
-        }
-        try {
-            const parsed = JSON.parse(value);
-            if (parsed && typeof parsed === 'object') {
-                return parsed;
-            }
-        }
-        catch (err) {
-            console.warn('Failed to parse metadata.segment:', err);
-        }
-        return undefined;
-    }
-    parseProperties(value) {
-        if (typeof value !== 'string' || value.length === 0) {
-            return undefined;
-        }
-        try {
-            const parsed = JSON.parse(value);
-            if (parsed && typeof parsed === 'object') {
-                const result = {};
-                Object.entries(parsed).forEach(([key, val]) => {
-                    if (typeof val === 'string') {
-                        result[key] = val;
-                    }
-                });
-                return result;
-            }
-        }
-        catch {
-            return undefined;
-        }
-        return undefined;
-    }
-    parseNumber(value, fallback) {
-        const parsed = typeof value === 'number' && !Number.isNaN(value)
-            ? value
-            : typeof value === 'string' && value.length > 0 && !Number.isNaN(Number(value))
-                ? Number(value)
-                : undefined;
-        return typeof parsed === 'number' ? parsed : fallback;
-    }
-    async close() {
-        try {
-            if (this.db) {
-                // LanceDB connections are automatically managed
-            }
-        }
-        catch (err) {
-            console.error('Error closing database:', err);
-        }
-    }
-    // ============================
-    // SEARCH METHODS
-    // ============================
-    async concordanceSearch(textFragment, language, limit = 100, filters) {
-        // Enhanced concordance search: finds text fragments and returns all language variants for matching units
-        try {
-            const table = await this.ensureTable();
-            const escapeLiteral = (value) => value.replaceAll("'", "''");
-            const escapedFragment = escapeLiteral(textFragment);
-            const whereFragment = 'language = ' + '\'' + language + '\'' + ' AND contains(pureText, ' + '\'' + escapedFragment + '\')';
-            const fragmentEntries = this.hydrateEntries(await table
-                .query()
-                .where(whereFragment)
-                .limit(limit)
-                .toArray());
-            const filteredEntries = filters
-                ? fragmentEntries.filter((entry) => this.metadataMatches(entry.metadata, filters))
-                : fragmentEntries;
-            if (filteredEntries.length === 0) {
-                return [];
-            }
-            const segmentDescriptors = new Map();
-            filteredEntries.forEach((entry) => {
-                const segmentIndex = typeof entry.segmentIndex === 'number' ? entry.segmentIndex : 0;
-                const descriptorKey = entry.fileId + '|' + entry.unitId + '|' + segmentIndex;
-                if (!segmentDescriptors.has(descriptorKey)) {
-                    segmentDescriptors.set(descriptorKey, {
-                        fileId: entry.fileId,
-                        unitId: entry.unitId,
-                        segmentIndex
-                    });
-                }
-            });
-            const result = [];
-            for (const descriptor of segmentDescriptors.values()) {
-                const segmentPrefix = descriptor.fileId + ':' + descriptor.unitId + ':' + descriptor.segmentIndex + ':';
-                const sanitizedPrefix = escapeLiteral(segmentPrefix);
-                const segmentVariants = this.hydrateEntries(await table
-                    .query()
-                    .where('starts_with(id, ' + '\'' + sanitizedPrefix + '\')')
-                    .toArray());
-                if (segmentVariants.length === 0) {
-                    continue;
-                }
-                const languageMap = new Map();
-                for (const variant of segmentVariants) {
-                    try {
-                        const xmlElement = Utils.buildXMLElement(variant.element);
-                        languageMap.set(variant.language, xmlElement);
-                    }
-                    catch (parseErr) {
-                        const errorMessage = parseErr instanceof Error ? parseErr.message : String(parseErr);
-                        throw new Error('Failed to create XML element for ' + variant.id + ': ' + errorMessage);
-                    }
-                }
-                if (languageMap.size > 0) {
-                    result.push(languageMap);
-                    if (result.length >= limit) {
-                        break;
-                    }
-                }
-            }
-            return result;
-        }
-        catch (err) {
-            console.error('Error performing concordance search:', err);
-            throw err;
-        }
-    }
-    async semanticSearch(queryText, language, limit = 10, filters) {
-        try {
-            const table = await this.ensureTable();
-            const queryEmbedding = await this.generateEmbedding(queryText);
-            const results = this.hydrateEntries(await table
-                .vectorSearch(queryEmbedding)
-                .where('language = ' + '\'' + language + '\'')
-                .limit(limit)
-                .toArray());
-            const filtered = filters
-                ? results.filter((entry) => this.metadataMatches(entry.metadata, filters))
-                : results;
-            return filtered.map((entry) => this.mapToSearchResult(entry));
-        }
-        catch (err) {
-            console.error('Error performing semantic search:', err);
-            throw err;
-        }
-    }
-    async semanticTranslationSearch(searchStr, srcLang, tgtLang, similarity, limit = 100, filters) {
-        try {
-            const table = await this.ensureTable();
-            // Generate embedding for the search string
-            const queryEmbedding = await this.generateEmbedding(searchStr);
-            // Get all entries for the source language
-            const sourceEntries = this.hydrateEntries(await table
-                .vectorSearch(queryEmbedding)
-                .where('language = ' + '\'' + srcLang + '\'')
-                .toArray());
-            const rankedMatches = [];
-            const sourceCriteria = filters?.source;
-            const targetCriteria = filters?.target ?? filters?.source;
-            for (const sourceEntry of sourceEntries) {
-                if (!sourceEntry.pureText || !sourceEntry.vector) {
-                    continue;
-                }
-                if (sourceCriteria && !this.metadataMatches(sourceEntry.metadata, sourceCriteria)) {
-                    continue;
-                }
-                // Linear mapping (more intuitive)
-                const rawDistance = sourceEntry._distance;
-                const l2Distance = typeof rawDistance === 'number'
-                    ? rawDistance
-                    : typeof rawDistance === 'string' && rawDistance.length > 0 && !Number.isNaN(Number(rawDistance))
-                        ? Number(rawDistance)
-                        : 0;
-                const semanticScore = Math.max(0, Math.round((2 - l2Distance) / 2 * 100));
-                const fuzzyScore = MatchQuality.similarity(searchStr, sourceEntry.pureText);
-                const hybridScore = Math.round((semanticScore + fuzzyScore) / 2);
-                // Only include matches that meet the minimum similarity threshold
-                if (hybridScore >= similarity) {
-                    const targetEntry = await this.findTargetEntry(table, sourceEntry, tgtLang, targetCriteria);
-                    if (!targetEntry) {
-                        continue;
-                    }
-                    if (targetCriteria && !this.metadataMatches(targetEntry.metadata, targetCriteria)) {
-                        continue;
-                    }
-                    try {
-                        const sourceElement = Utils.buildXMLElement(sourceEntry.element);
-                        const targetElement = Utils.buildXMLElement(targetEntry.element);
-                        const match = new Match(sourceElement, targetElement, this.name, semanticScore, fuzzyScore);
-                        const rankingScore = this.computeRankingScore(match.hybridScore(), sourceEntry, targetEntry);
-                        rankedMatches.push({ match, score: rankingScore });
-                    }
-                    catch (parseErr) {
-                        console.error('Error creating Match for semantic translation search: ' + (parseErr instanceof Error ? parseErr.message : String(parseErr)));
-                    }
-                }
-            }
-            rankedMatches.sort((a, b) => b.score - a.score);
-            return rankedMatches.slice(0, limit).map((item) => item.match);
-        }
-        catch (err) {
-            console.error('Error performing semantic search with quality:', err);
-            return [];
-        }
-    }
-    async findTargetEntry(table, sourceEntry, tgtLang, filters) {
-        if (!sourceEntry.fileId || !sourceEntry.unitId) {
-            return null;
-        }
-        const exactId = this.buildEntryId(sourceEntry.fileId, sourceEntry.unitId, sourceEntry.segmentIndex, tgtLang);
-        const exactMatches = this.hydrateEntries(await table
-            .query()
-            .where('id = ' + '\'' + Utils.replaceQuotes(exactId) + '\'')
-            .toArray());
-        const exactMatch = this.selectPreferredCandidate(exactMatches, sourceEntry.segmentIndex, filters);
-        if (exactMatch) {
-            return exactMatch;
-        }
-        const unitPrefix = sourceEntry.fileId + ':' + sourceEntry.unitId + ':';
-        const sanitizedPrefix = Utils.replaceQuotes(unitPrefix);
-        const sanitizedLang = Utils.replaceQuotes(tgtLang);
-        const unitMatches = this.hydrateEntries(await table
-            .query()
-            .where('starts_with(id, ' + '\'' + sanitizedPrefix + '\') AND language = ' + '\'' + sanitizedLang + '\'')
-            .limit(50)
-            .toArray());
-        if (unitMatches.length === 0) {
-            return null;
-        }
-        const preferred = this.selectPreferredCandidate(unitMatches, sourceEntry.segmentIndex, filters);
-        return preferred ?? null;
-    }
-    computeRankingScore(baseScore, sourceEntry, targetEntry) {
-        let score = baseScore;
-        if (sourceEntry.segmentIndex > 0 && targetEntry.segmentIndex > 0) {
-            score += sourceEntry.segmentIndex === targetEntry.segmentIndex ? 10 : 5;
-        }
-        const quality = targetEntry.metadata && typeof targetEntry.metadata.quality === 'number'
-            ? targetEntry.metadata.quality
-            : undefined;
-        if (quality !== undefined && !Number.isNaN(quality)) {
-            score += Math.min(Math.max(quality, 0), 100) / 20;
-        }
-        const recencyTimestamp = this.parseMetadataTimestamp(targetEntry.metadata?.changeDate || targetEntry.metadata?.creationDate);
-        if (recencyTimestamp !== undefined) {
-            const ageDays = (Date.now() - recencyTimestamp) / 86400000;
-            if (Number.isFinite(ageDays) && ageDays >= 0) {
-                const normalized = Math.max(0, 1 - Math.min(ageDays, 365) / 365);
-                score += normalized * 5;
-            }
-        }
-        if (targetEntry.metadata?.state === 'final') {
-            score += 3;
-        }
-        else if (targetEntry.metadata?.state === 'reviewed') {
-            score += 2;
-        }
-        else if (targetEntry.metadata?.state === 'translated') {
-            score += 1;
-        }
-        return score;
-    }
-    parseMetadataTimestamp(value) {
-        if (!value) {
-            return undefined;
-        }
-        const parsed = Date.parse(value);
-        return Number.isNaN(parsed) ? undefined : parsed;
-    }
-    selectPreferredCandidate(entries, desiredSegmentIndex, filters) {
-        if (!entries || entries.length === 0) {
-            return undefined;
-        }
-        const filtered = filters
-            ? entries.filter((entry) => this.metadataMatches(entry.metadata, filters))
-            : entries.slice();
-        if (filtered.length === 0) {
-            return undefined;
-        }
-        if (desiredSegmentIndex > 0) {
-            const exact = filtered.find((entry) => entry.segmentIndex === desiredSegmentIndex);
-            if (exact) {
-                return exact;
-            }
-        }
-        const segmentEntry = filtered.find((entry) => entry.segmentIndex > 0);
-        if (segmentEntry) {
-            return segmentEntry;
-        }
-        return filtered[0];
-    }
-    metadataMatches(metadata, filter) {
-        if (!filter) {
-            return true;
-        }
-        if (!metadata) {
-            return false;
-        }
-        if (filter.states && filter.states.length > 0) {
-            if (!metadata.state || !filter.states.includes(metadata.state)) {
-                return false;
-            }
-        }
-        if (filter.minState) {
-            const metadataRank = this.getStateRank(metadata.state);
-            if (metadataRank === 0 || metadataRank < this.getStateRank(filter.minState)) {
-                return false;
-            }
-        }
-        if (typeof filter.minQuality === 'number') {
-            if (typeof metadata.quality !== 'number' || metadata.quality < filter.minQuality) {
-                return false;
-            }
-        }
-        if (filter.contextIncludes && filter.contextIncludes.length > 0) {
-            const contextValue = (metadata.context || '').toLowerCase();
-            const matchesAll = filter.contextIncludes.every((needle) => contextValue.includes(needle.toLowerCase()));
-            if (!matchesAll) {
-                return false;
-            }
-        }
-        if (filter.requiredProperties) {
-            if (!metadata.properties) {
-                return false;
-            }
-            for (const [key, expected] of Object.entries(filter.requiredProperties)) {
-                if (metadata.properties[key] !== expected) {
-                    return false;
-                }
-            }
-        }
-        if (filter.provider) {
-            const provider = metadata.segment?.provider;
-            if (!provider || provider !== filter.provider) {
-                return false;
-            }
-        }
-        return true;
-    }
-    getStateRank(value) {
-        if (!value) {
-            return 0;
-        }
-        switch (value) {
-            case 'initial':
-                return 1;
-            case 'translated':
-                return 2;
-            case 'reviewed':
-                return 3;
-            case 'final':
-                return 4;
-            default:
-                return 0;
-        }
-    }
-    // ============================
-    // DATA MANAGEMENT METHODS
-    // ============================
-    async storeLangEntry(fileId, original, unitId, lang, pureText, element, embeddings, segmentIndex = 0, segmentCount = 1, metadata) {
-        try {
-            const table = await this.ensureTable();
-            const sanitizedFileId = Utils.replaceQuotes(fileId);
-            const sanitizedUnitId = Utils.replaceQuotes(unitId);
-            const safeSegmentIndex = typeof segmentIndex === 'number' ? segmentIndex : 0;
-            const safeSegmentCount = typeof segmentCount === 'number' && segmentCount > 0 ? segmentCount : 1;
-            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, safeSegmentIndex, lang);
-            const existingEntries = this.hydrateEntries(await table
-                .query()
-                .where('id = ' + '\'' + entryId + '\'')
-                .toArray());
-            if (existingEntries.length > 0) {
-                const existingEntry = existingEntries[0];
-                const contentChanged = (existingEntry.pureText !== pureText ||
-                    existingEntry.element !== element.toString() ||
-                    existingEntry.original !== original);
-                if (!contentChanged) {
-                    return;
-                }
-            }
-            // Generate embeddings if not provided
-            let vectorEmbeddings;
-            if (embeddings && embeddings.length > 0) {
-                vectorEmbeddings = embeddings;
-            }
-            else {
-                vectorEmbeddings = await this.generateEmbedding(pureText);
-            }
-            const entry = {
-                id: entryId,
-                language: lang,
-                pureText: pureText,
-                element: element.toString(),
-                fileId: sanitizedFileId,
-                original: original,
-                unitId: sanitizedUnitId,
-                vector: vectorEmbeddings,
-                segmentIndex: safeSegmentIndex,
-                segmentCount: safeSegmentCount,
-                metadata: metadata ?? {}
-            };
-            // Delete existing entry first (LanceDB upsert approach)
-            try {
-                await table.delete('id = ' + '\'' + entryId + '\'');
-            }
-            catch (deleteErr) {
-                // Entry might not exist, which is fine
-            }
-            await table.add([this.flattenEntry(entry)]);
-        }
-        catch (err) {
-            console.error('Error storing language entry:', err);
-            throw err;
-        }
-    }
-    async storeBatchEntries(entries) {
-        try {
-            const table = await this.ensureTable();
-            const langEntries = [];
-            // Collect all entry IDs for bulk deletion
-            const entryIds = [];
-            // Generate embeddings one by one and build LangEntry objects
-            for (const entry of entries) {
-                const sanitizedFileId = Utils.replaceQuotes(entry.fileId);
-                const sanitizedUnitId = Utils.replaceQuotes(entry.unitId);
-                const vectorEmbeddings = await this.generateEmbedding(entry.pureText);
-                const safeSegmentIndex = typeof entry.segmentIndex === 'number' ? entry.segmentIndex : 0;
-                const safeSegmentCount = typeof entry.segmentCount === 'number' && entry.segmentCount > 0 ? entry.segmentCount : 1;
-                const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, safeSegmentIndex, entry.language);
-                const langEntry = {
-                    id: entryId,
-                    language: entry.language,
-                    pureText: entry.pureText,
-                    element: entry.element.toString(),
-                    fileId: sanitizedFileId,
-                    original: entry.original,
-                    unitId: sanitizedUnitId,
-                    vector: vectorEmbeddings,
-                    segmentIndex: safeSegmentIndex,
-                    segmentCount: safeSegmentCount,
-                    metadata: entry.metadata ?? {}
-                };
-                langEntries.push(langEntry);
-                entryIds.push(entryId);
-            }
-            // Delete any existing entries with these IDs to prevent duplicates
-            // This ensures that if the same file is imported twice, entries are overwritten
-            if (entryIds.length > 0) {
-                const idsFilter = entryIds.map(id => '\'' + id + '\'').join(',');
-                try {
-                    await table.delete('id IN (' + idsFilter + ')');
-                }
-                catch (deleteErr) {
-                    // Entries might not exist, which is fine for first import
-                }
-            }
-            // Bulk insert all entries at once
-            const flattenedEntries = langEntries.map((item) => this.flattenEntry(item));
-            await table.add(flattenedEntries);
-        }
-        catch (err) {
-            console.error('Error storing batch entries:', err);
-            throw err;
-        }
-    }
-    async entryExists(fileId, unitId, lang, segmentIndex = 0) {
-        try {
-            if (!this.table) {
-                await this.initializeDatabase();
-            }
-            if (!this.table) {
-                throw new Error('Failed to initialize database table');
-            }
-            const sanitizedFileId = Utils.replaceQuotes(fileId);
-            const sanitizedUnitId = Utils.replaceQuotes(unitId);
-            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
-            const existingEntries = this.hydrateEntries(await this.table
-                .query()
-                .where('id = ' + '\'' + entryId + '\'')
-                .toArray());
-            return existingEntries.length > 0;
-        }
-        catch (err) {
-            console.error('Error checking entry existence:', err);
-            return false;
-        }
-    }
-    async getLangEntry(fileId, unitId, lang, segmentIndex = 0) {
-        try {
-            if (!this.table) {
-                await this.initializeDatabase();
-            }
-            if (!this.table) {
-                throw new Error('Failed to initialize database table');
-            }
-            const sanitizedFileId = Utils.replaceQuotes(fileId);
-            const sanitizedUnitId = Utils.replaceQuotes(unitId);
-            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
-            const existingEntries = this.hydrateEntries(await this.table
-                .query()
-                .where('id = ' + '\'' + entryId + '\'')
-                .toArray());
-            return existingEntries.length > 0 ? existingEntries[0] : null;
-        }
-        catch (err) {
-            console.error('Error retrieving language entry:', err);
-            return null;
-        }
-    }
-    async deleteLangEntry(fileId, unitId, lang, segmentIndex = 0) {
-        try {
-            if (!this.table) {
-                await this.initializeDatabase();
-            }
-            if (!this.table) {
-                throw new Error('Failed to initialize database table');
-            }
-            const sanitizedFileId = Utils.replaceQuotes(fileId);
-            const sanitizedUnitId = Utils.replaceQuotes(unitId);
-            const entryId = this.buildEntryId(sanitizedFileId, sanitizedUnitId, segmentIndex, lang);
-            // Check if entry exists first
-            const exists = await this.entryExists(fileId, unitId, lang, segmentIndex);
-            if (!exists) {
-                return false;
-            }
-            // Delete the entry
-            await this.table.delete('id = \'' + entryId + '\'');
-            return true;
-        }
-        catch (err) {
-            console.error('Error deleting language entry:', err);
-            return false;
-        }
-    }
-    // ============================
-    // DATA IMPORT METHODS
-    // ============================
-    async importXLIFF(filePath, options) {
-        // Phase 1: Parse XLIFF and write to temporary JSONL file
-        const resolvedOptions = resolveImportOptions(options);
-        const reader = new XLIFFReader(filePath, resolvedOptions);
-        await reader.parse();
-        // Phase 2: Batch import from JSONL file (asynchronous)
-        const importer = new BatchImporter(this, reader.getTempFilePath(), reader.getEntryCount());
-        await importer.import();
-    }
-    async importTMX(filePath, options) {
-        // Phase 1: Parse TMX and write to temporary JSONL file
-        const resolvedOptions = resolveImportOptions(options);
-        const reader = new TMXReader(filePath, resolvedOptions);
-        await reader.parse();
-        // Phase 2: Batch import from JSONL file (asynchronous)
-        const importer = new BatchImporter(this, reader.getTempFilePath(), reader.getEntryCount());
-        await importer.import();
-    }
-    async importSDLTM(filePath, options) {
-        const tempDir = tmpdir();
-        const tempFileName = 'tmx_' + Date.now() + '_' + Math.random().toString(36).substring(7) + '.tmx';
-        const tempFilePath = join(tempDir, tempFileName);
-        const packageJson = await import('../package.json', { assert: { type: 'json' } });
-        const productName = packageJson.default.productName;
-        const version = packageJson.default.version;
-        const tmReader = new TMReader({ 'productName': productName, 'version': version });
-        const data = await tmReader.convert(filePath, tempFilePath);
-        if (data.status !== 'Success') {
-            throw new Error('SDL TM conversion failed for file ' + filePath);
-        }
-        await this.importTMX(tempFilePath, options);
-        unlinkSync(tempFilePath);
-    }
-}
-//# sourceMappingURL=hybridtm.js.map