@framers/agentos 0.6.30 → 0.7.1

package/dist/rag/vector_stores/SqlVectorStore.js

@@ -1,1210 +0,0 @@
-/**
- * @fileoverview SQL-backed Vector Store Implementation
- *
- * Implements `IVectorStore` using `@framers/sql-storage-adapter` for persistence.
- * This allows vector storage to work across all platforms supported by the storage
- * adapter (SQLite, PostgreSQL, IndexedDB, Capacitor, etc.).
- *
- * **Key Features:**
- * - Cross-platform persistence using sql-storage-adapter
- * - Storage-feature-aware SQL, FTS, and blob encoding
- * - Hybrid search (vector similarity + keyword matching)
- * - Automatic schema management
- * - Portable embedding storage across SQLite, PostgreSQL, IndexedDB, etc.
- *
- * **Architecture:**
- * ```
- * AgentOS RAG System (RetrievalAugmentor)
- *          ↓
- *   VectorStoreManager
- *          ↓
- *   SqlVectorStore (this file)
- *          ↓
- *   @framers/sql-storage-adapter
- *          ↓
- *   Database (SQLite/PostgreSQL/IndexedDB/etc.)
- * ```
- *
- * @module @framers/agentos/rag/vector_stores/SqlVectorStore
- * @see ../../IVectorStore.ts for the interface definition.
- * @see @framers/sql-storage-adapter for storage abstraction.
- */
-import { resolveStorageAdapter, createStorageFeatures, } from '@framers/sql-storage-adapter';
-import { cosineSimilarity as vecCosineSimilarity, dotProduct as vecDotProduct, euclideanDistance as vecEuclideanDistance, } from '../utils/vectorMath.js';
-import { GMIError, GMIErrorCode } from '../../core/utils/errors.js';
-import { uuidv4 } from '../../core/utils/uuid.js';
-import { getNaturalStopWords } from '../../nlp/filters/StopWordFilter.js';
-// ============================================================================
-// SQL Vector Store Implementation
-// ============================================================================
-/**
- * SQL-backed vector store implementation.
- *
- * Uses `@framers/sql-storage-adapter` for cross-platform persistence.
- * Stores embeddings as base64-encoded Float32 payloads and computes similarity
- * in application code.
- *
- * @class SqlVectorStore
- * @implements {IVectorStore}
- *
- * @example
- * ```typescript
- * const store = new SqlVectorStore();
- *
- * await store.initialize({
- *   id: 'sql-vector-store',
- *   type: 'sql',
- *   storage: {
- *     filePath: './vectors.db',
- *     priority: ['better-sqlite3', 'sqljs']
- *   },
- *   enableFullTextSearch: true
- * });
- *
- * // Create a collection
- * await store.createCollection('documents', 1536);
- *
- * // Upsert documents
- * await store.upsert('documents', [{
- *   id: 'doc-1',
- *   embedding: [...], // 1536-dim vector
- *   textContent: 'Example document content',
- *   metadata: { author: 'Alice', category: 'tech' }
- * }]);
- *
- * // Query by similarity
- * const results = await store.query('documents', queryEmbedding, { topK: 5 });
- * ```
- */
-export class SqlVectorStore {
-    /**
-     * Constructs a SqlVectorStore instance.
-     * The store is not operational until `initialize()` is called.
-     */
-    constructor() {
-        this.ownsAdapter = false; // Whether we created the adapter
-        this.isInitialized = false;
-        this.tablePrefix = 'agentos_rag_';
-        /** Per-collection HNSW sidecars for accelerated vector search. */
-        this.sidecars = new Map();
-        this.providerId = `sql-vector-store-${uuidv4()}`;
-    }
-    /**
-     * Initializes the vector store with the provided configuration.
-     *
-     * Creates necessary tables and indexes if they don't exist.
-     *
-     * @param {VectorStoreProviderConfig} config - Configuration object
-     * @throws {GMIError} If configuration is invalid or initialization fails
-     */
-    async initialize(config) {
-        if (this.isInitialized) {
-            console.warn(`SqlVectorStore (ID: ${this.providerId}) already initialized. Re-initializing.`);
-            await this.shutdown();
-        }
-        if (config.type !== 'sql') {
-            throw new GMIError(`Invalid configuration type for SqlVectorStore: ${config.type}. Expected 'sql'.`, GMIErrorCode.CONFIG_ERROR, { providedType: config.type });
-        }
-        this.config = config;
-        this.tablePrefix = this.config.tablePrefix ?? 'agentos_rag_';
-        // Initialize storage adapter
-        if (this.config.adapter) {
-            this.adapter = this.config.adapter;
-            this.ownsAdapter = false;
-        }
-        else if (this.config.storage) {
-            this.adapter = await resolveStorageAdapter(this.config.storage);
-            this.ownsAdapter = true;
-        }
-        else {
-            // Default to sql.js (in-memory when no file path provided)
-            this.adapter = await resolveStorageAdapter({ priority: ['sqljs'] });
-            this.ownsAdapter = true;
-            console.warn(`SqlVectorStore (ID: ${this.providerId}): No storage config provided, using sql.js (in-memory).`);
-        }
-        this.features = createStorageFeatures(this.adapter);
-        // Create schema
-        await this.createSchema();
-        // Store pipeline reference
-        this.pipeline = this.config.pipeline;
-        this.sidecars.clear();
-        this.hnswSidecarClass = undefined;
-        this.isInitialized = true;
-        console.log(`SqlVectorStore (ID: ${this.providerId}, Config ID: ${this.config.id}) initialized successfully.`);
-    }
-    /**
-     * Creates the database schema for vector storage.
-     * @private
-     */
-    async createSchema() {
-        // Collections metadata table
-        await this.adapter.exec(`
-      CREATE TABLE IF NOT EXISTS ${this.tablePrefix}collections (
-        name TEXT PRIMARY KEY,
-        dimension INTEGER NOT NULL,
-        similarity_metric TEXT NOT NULL DEFAULT 'cosine',
-        document_count INTEGER NOT NULL DEFAULT 0,
-        created_at INTEGER NOT NULL,
-        updated_at INTEGER NOT NULL
-      );
-    `);
-        // Documents table - stores vectors as base64-encoded binary blobs
-        await this.adapter.exec(`
-      CREATE TABLE IF NOT EXISTS ${this.tablePrefix}documents (
-        id TEXT NOT NULL,
-        collection_name TEXT NOT NULL,
-        embedding_blob TEXT NOT NULL,
-        text_content TEXT,
-        metadata_json TEXT,
-        created_at INTEGER NOT NULL,
-        updated_at INTEGER NOT NULL,
-        PRIMARY KEY (collection_name, id),
-        FOREIGN KEY (collection_name) REFERENCES ${this.tablePrefix}collections(name) ON DELETE CASCADE
-      );
-      
-      CREATE INDEX IF NOT EXISTS idx_${this.tablePrefix}docs_collection 
-        ON ${this.tablePrefix}documents(collection_name);
-      
-      CREATE INDEX IF NOT EXISTS idx_${this.tablePrefix}docs_updated 
-        ON ${this.tablePrefix}documents(updated_at);
-    `);
-        // Full-text search index
-        if (this.config.enableFullTextSearch !== false) {
-            try {
-                await this.adapter.exec(this.features.fts.createIndex({
-                    table: `${this.tablePrefix}documents_fts`,
-                    columns: ['id', 'collection_name', 'text_content'],
-                    contentTable: `${this.tablePrefix}documents`,
-                    tokenizer: 'porter ascii',
-                }));
-                console.log(`SqlVectorStore (ID: ${this.providerId}): search index created.`);
-            }
-            catch (error) {
-                console.warn(`SqlVectorStore (ID: ${this.providerId}): search index not available: ${error.message}`);
-            }
-        }
-    }
-    /**
-     * Ensures the store is initialized before operations.
-     * @private
-     */
-    ensureInitialized() {
-        if (!this.isInitialized) {
-            throw new GMIError(`SqlVectorStore (ID: ${this.providerId}) is not initialized. Call initialize() first.`, GMIErrorCode.NOT_INITIALIZED);
-        }
-    }
-    /**
-     * Creates a new collection for storing vectors.
-     *
-     * @param {string} collectionName - Unique name for the collection
-     * @param {number} dimension - Vector embedding dimension
-     * @param {CreateCollectionOptions} [options] - Creation options
-     */
-    async createCollection(collectionName, dimension, options) {
-        this.ensureInitialized();
-        if (dimension <= 0) {
-            throw new GMIError(`Invalid dimension for collection '${collectionName}': ${dimension}. Must be positive.`, GMIErrorCode.VALIDATION_ERROR, { dimension });
-        }
-        const exists = await this.collectionExists(collectionName);
-        if (exists) {
-            if (options?.overwriteIfExists) {
-                await this.deleteCollection(collectionName);
-            }
-            else {
-                throw new GMIError(`Collection '${collectionName}' already exists.`, GMIErrorCode.ALREADY_EXISTS, { collectionName });
-            }
-        }
-        const now = Date.now();
-        const metric = options?.similarityMetric ?? this.config.similarityMetric ?? 'cosine';
-        await this.adapter.run(`INSERT INTO ${this.tablePrefix}collections 
-       (name, dimension, similarity_metric, document_count, created_at, updated_at)
-       VALUES (?, ?, ?, 0, ?, ?)`, [collectionName, dimension, metric, now, now]);
-        console.log(`SqlVectorStore (ID: ${this.providerId}): Collection '${collectionName}' created (dim=${dimension}, metric=${metric}).`);
-    }
-    /**
-     * Checks if a collection exists.
-     *
-     * @param {string} collectionName - Collection name to check
-     * @returns {Promise<boolean>} True if collection exists
-     */
-    async collectionExists(collectionName) {
-        this.ensureInitialized();
-        const row = await this.adapter.get(`SELECT COUNT(*) as count FROM ${this.tablePrefix}collections WHERE name = ?`, [collectionName]);
-        return (row?.count ?? 0) > 0;
-    }
-    /**
-     * Deletes a collection and all its documents.
-     *
-     * @param {string} collectionName - Collection to delete
-     */
-    async deleteCollection(collectionName) {
-        this.ensureInitialized();
-        // Delete documents first (due to foreign key)
-        await this.adapter.run(`DELETE FROM ${this.tablePrefix}documents WHERE collection_name = ?`, [collectionName]);
-        // Delete collection metadata
-        await this.adapter.run(`DELETE FROM ${this.tablePrefix}collections WHERE name = ?`, [collectionName]);
-        const sidecar = this.sidecars.get(collectionName);
-        if (sidecar) {
-            await sidecar.shutdown();
-            this.sidecars.delete(collectionName);
-        }
-        console.log(`SqlVectorStore (ID: ${this.providerId}): Collection '${collectionName}' deleted.`);
-    }
-    /**
-     * Gets collection metadata.
-     * @private
-     */
-    async getCollectionMetadata(collectionName) {
-        const row = await this.adapter.get(`SELECT * FROM ${this.tablePrefix}collections WHERE name = ?`, [collectionName]);
-        if (!row) {
-            throw new GMIError(`Collection '${collectionName}' not found.`, GMIErrorCode.NOT_FOUND, { collectionName });
-        }
-        return {
-            name: row.name,
-            dimension: row.dimension,
-            similarityMetric: row.similarity_metric,
-            documentCount: row.document_count,
-            createdAt: row.created_at,
-            updatedAt: row.updated_at,
-        };
-    }
-    /**
-     * Upserts documents into a collection.
-     *
-     * @param {string} collectionName - Target collection
-     * @param {VectorDocument[]} documents - Documents to upsert
-     * @param {UpsertOptions} [options] - Upsert options
-     * @returns {Promise<UpsertResult>} Result of the upsert operation
-     */
-    async upsert(collectionName, documents, options) {
-        this.ensureInitialized();
-        const collection = await this.getCollectionMetadata(collectionName);
-        const upsertedIds = [];
-        const errors = [];
-        const now = Date.now();
-        for (const doc of documents) {
-            // Validate dimension
-            if (doc.embedding.length !== collection.dimension) {
-                errors.push({
-                    id: doc.id,
-                    message: `Embedding dimension ${doc.embedding.length} does not match collection dimension ${collection.dimension}.`,
-                    details: { expected: collection.dimension, got: doc.embedding.length }
-                });
-                continue;
-            }
-            try {
-                // Check if document exists
-                const existing = await this.adapter.get(`SELECT id FROM ${this.tablePrefix}documents WHERE collection_name = ? AND id = ?`, [collectionName, doc.id]);
-                const embeddingBlob = this.encodeEmbedding(doc.embedding);
-                const metadataJson = doc.metadata ? JSON.stringify(doc.metadata) : null;
-                if (existing && options?.overwrite === false) {
-                    errors.push({
-                        id: doc.id,
-                        message: `Document '${doc.id}' already exists and overwrite is disabled.`,
-                        details: { reason: 'NO_OVERWRITE' }
-                    });
-                    continue;
-                }
-                if (existing) {
-                    // Update existing document
-                    await this.adapter.run(`UPDATE ${this.tablePrefix}documents 
-             SET embedding_blob = ?, text_content = ?, metadata_json = ?, updated_at = ?
-             WHERE collection_name = ? AND id = ?`, [embeddingBlob, doc.textContent ?? null, metadataJson, now, collectionName, doc.id]);
-                }
-                else {
-                    // Insert new document
-                    await this.adapter.run(`INSERT INTO ${this.tablePrefix}documents 
-             (id, collection_name, embedding_blob, text_content, metadata_json, created_at, updated_at)
-             VALUES (?, ?, ?, ?, ?, ?, ?)`, [doc.id, collectionName, embeddingBlob, doc.textContent ?? null, metadataJson, now, now]);
-                }
-                upsertedIds.push(doc.id);
-            }
-            catch (error) {
-                errors.push({
-                    id: doc.id,
-                    message: `Failed to upsert document: ${error.message}`,
-                    details: error
-                });
-            }
-        }
-        // Update collection document count
-        const countResult = await this.adapter.get(`SELECT COUNT(*) as count FROM ${this.tablePrefix}documents WHERE collection_name = ?`, [collectionName]);
-        await this.adapter.run(`UPDATE ${this.tablePrefix}collections SET document_count = ?, updated_at = ? WHERE name = ?`, [countResult?.count ?? 0, now, collectionName]);
-        // ── HNSW sidecar: add upserted vectors + check threshold ──────────
-        const sidecar = await this.getOrCreateSidecar(collection);
-        if (sidecar?.isAvailable() && upsertedIds.length > 0) {
-            const docsWithEmbeddings = documents
-                .filter(d => upsertedIds.includes(d.id) && d.embedding?.length > 0)
-                .map(d => ({ id: d.id, embedding: d.embedding }));
-            if (sidecar.isActive()) {
-                await sidecar.upsertBatch(docsWithEmbeddings);
-            }
-            else {
-                // Check if we just crossed the activation threshold
-                const docCount = countResult?.count ?? 0;
-                const threshold = this.config.hnswThreshold ?? 1000;
-                if (docCount >= threshold) {
-                    // Load ALL embeddings from SQLite and rebuild the HNSW index
-                    const allRows = await this.adapter.all(`SELECT id, embedding_blob FROM ${this.tablePrefix}documents WHERE collection_name = ?`, [collectionName]);
-                    const allItems = allRows
-                        .map(row => ({
-                        id: row.id,
-                        embedding: this.decodeStoredEmbedding(row.embedding_blob),
-                    }))
-                        .filter(item => item.embedding.length > 0);
-                    await sidecar.rebuildFromData(allItems);
-                }
-            }
-        }
-        return {
-            upsertedCount: upsertedIds.length,
-            upsertedIds,
-            failedCount: errors.length,
-            errors: errors.length > 0 ? errors : undefined,
-        };
-    }
-    /**
-     * Queries a collection for similar documents.
-     *
-     * @param {string} collectionName - Collection to query
-     * @param {number[]} queryEmbedding - Query vector
-     * @param {QueryOptions} [options] - Query options
-     * @returns {Promise<QueryResult>} Query results sorted by similarity
-     */
-    async query(collectionName, queryEmbedding, options) {
-        this.ensureInitialized();
-        const collection = await this.getCollectionMetadata(collectionName);
-        const topK = options?.topK ?? 10;
-        // Validate query embedding dimension
-        if (queryEmbedding.length !== collection.dimension) {
-            throw new GMIError(`Query embedding dimension ${queryEmbedding.length} does not match collection dimension ${collection.dimension}.`, GMIErrorCode.VALIDATION_ERROR, { expected: collection.dimension, got: queryEmbedding.length });
-        }
-        // ── HNSW fast path ─────────────────────────────────────────────────
-        // When the sidecar is active, use O(log n) ANN search to get top
-        // candidates by ID, then fetch full documents from SQLite. Falls through
-        // to brute-force when the sidecar is inactive or unavailable.
-        const sidecar = await this.getOrCreateSidecar(collection);
-        if (sidecar?.isActive()) {
-            const hnswCandidates = await sidecar.search(queryEmbedding, topK * 3);
-            if (hnswCandidates.length > 0) {
-                const candidateIds = hnswCandidates.map(c => c.id);
-                const placeholders = candidateIds.map(() => '?').join(',');
-                let hnswQuery = `SELECT * FROM ${this.tablePrefix}documents WHERE collection_name = ? AND id IN (${placeholders})`;
-                const hnswParams = [collectionName, ...candidateIds];
-                if (options?.filter) {
-                    const filterSQL = this.buildMetadataFilterSQL(options.filter);
-                    hnswQuery += filterSQL.clause;
-                    hnswParams.push(...filterSQL.params);
-                }
-                const rows = await this.adapter.all(hnswQuery, hnswParams);
-                const scoreMap = new Map(hnswCandidates.map(c => [c.id, c.score]));
-                const candidates = rows.map(row => {
-                    const metadata = row.metadata_json ? JSON.parse(row.metadata_json) : undefined;
-                    if (options?.filter && !this.matchesFilter(metadata, options.filter)) {
-                        return null;
-                    }
-                    const embedding = options?.includeEmbedding
-                        ? this.decodeStoredEmbedding(row.embedding_blob)
-                        : [];
-                    const doc = {
-                        id: row.id,
-                        embedding,
-                        similarityScore: scoreMap.get(row.id) ?? 0,
-                    };
-                    if (options?.includeMetadata !== false && metadata)
-                        doc.metadata = metadata;
-                    if (options?.includeTextContent && row.text_content)
-                        doc.textContent = row.text_content;
-                    return doc;
-                }).filter((d) => {
-                    if (!d)
-                        return false;
-                    return options?.minSimilarityScore === undefined || d.similarityScore >= options.minSimilarityScore;
-                });
-                candidates.sort((a, b) => b.similarityScore - a.similarityScore);
-                const results = candidates.slice(0, topK);
-                const requiresExactFallback = (options?.filter !== undefined || options?.minSimilarityScore !== undefined)
-                    && results.length < topK;
-                if (results.length > 0 && !requiresExactFallback) {
-                    return {
-                        documents: results,
-                        queryId: `sql-hnsw-query-${uuidv4()}`,
-                        stats: { totalCandidates: hnswCandidates.length, filteredCandidates: candidates.length, returnedCount: results.length },
-                    };
-                }
-            }
-        }
-        // ── Brute-force fallback ──────────────────────────────────────────
-        // Pre-filter in SQL, then apply exact JS filter semantics for any
-        // operators the SQL fragment cannot fully express.
-        let query = `SELECT * FROM ${this.tablePrefix}documents WHERE collection_name = ?`;
-        const params = [collectionName];
-        // Push metadata filters into SQL WHERE clauses — avoids loading unmatched rows
-        if (options?.filter) {
-            const filterSQL = this.buildMetadataFilterSQL(options.filter);
-            query += filterSQL.clause;
-            params.push(...filterSQL.params);
-        }
-        const rows = await this.adapter.all(query, params);
-        // Compute similarities on the (now pre-filtered) result set
-        const candidates = [];
-        for (const row of rows) {
-            const embedding = this.decodeStoredEmbedding(row.embedding_blob);
-            const metadata = row.metadata_json ? JSON.parse(row.metadata_json) : undefined;
-            if (options?.filter && !this.matchesFilter(metadata, options.filter)) {
-                continue;
-            }
-            // Compute similarity
-            let similarityScore;
-            switch (collection.similarityMetric) {
-                case 'euclidean':
-                    similarityScore = -vecEuclideanDistance(queryEmbedding, embedding); // Negate for "higher is better"
-                    break;
-                case 'dotproduct':
-                    similarityScore = vecDotProduct(queryEmbedding, embedding);
-                    break;
-                case 'cosine':
-                default:
-                    similarityScore = vecCosineSimilarity(queryEmbedding, embedding);
-                    break;
-            }
-            // Apply minimum similarity threshold
-            if (options?.minSimilarityScore !== undefined && similarityScore < options.minSimilarityScore) {
-                continue;
-            }
-            // Build result document
-            const retrievedDoc = {
-                id: row.id,
-                embedding: options?.includeEmbedding ? embedding : [],
-                similarityScore,
-            };
-            if (options?.includeMetadata !== false && metadata) {
-                retrievedDoc.metadata = metadata;
-            }
-            if (options?.includeTextContent && row.text_content) {
-                retrievedDoc.textContent = row.text_content;
-            }
-            candidates.push(retrievedDoc);
-        }
-        // Sort by similarity (descending) and take topK
-        candidates.sort((a, b) => b.similarityScore - a.similarityScore);
-        const results = candidates.slice(0, topK);
-        return {
-            documents: results,
-            queryId: `sql-query-${uuidv4()}`,
-            stats: {
-                totalCandidates: rows.length,
-                filteredCandidates: candidates.length,
-                returnedCount: results.length,
-            },
-        };
-    }
-    async scanByMetadata(collectionName, options) {
-        this.ensureInitialized();
-        await this.getCollectionMetadata(collectionName);
-        const limit = Math.max(1, options?.limit ?? 100);
-        const rows = await this.adapter.all(`SELECT id, embedding_blob, text_content, metadata_json
-       FROM ${this.tablePrefix}documents
-       WHERE collection_name = ?
-       ORDER BY updated_at ASC, id ASC`, [collectionName]);
-        const documents = [];
-        for (const row of rows) {
-            const metadata = row.metadata_json ? JSON.parse(row.metadata_json) : undefined;
-            if (options?.filter && !this.matchesFilter(metadata, options.filter)) {
-                continue;
-            }
-            const scannedDoc = {
-                id: row.id,
-                embedding: options?.includeEmbedding ? this.decodeStoredEmbedding(row.embedding_blob) : [],
-                similarityScore: 1,
-            };
-            if (options?.includeMetadata !== false && metadata) {
-                scannedDoc.metadata = metadata;
-            }
-            if (options?.includeTextContent && row.text_content) {
-                scannedDoc.textContent = row.text_content;
-            }
-            documents.push(scannedDoc);
-            if (documents.length >= limit)
-                break;
-        }
-        return {
-            documents,
-            stats: {
-                totalCandidates: rows.length,
-                returnedCount: documents.length,
-            },
-        };
-    }
-    /**
-     * Performs hybrid search combining vector similarity with keyword matching.
-     *
-     * @param {string} collectionName - Collection to search
-     * @param {number[]} queryEmbedding - Query vector for semantic search
-     * @param {string} queryText - Text query for keyword search
-     * @param {QueryOptions & { alpha?: number }} [options] - Search options
-     * @returns {Promise<QueryResult>} Combined search results
-     *
-     * @example
-     * ```typescript
-     * const results = await store.hybridSearch(
-     *   'documents',
-     *   queryEmbedding,
-     *   'machine learning tutorial',
-     *   { topK: 10, alpha: 0.7 } // 70% vector, 30% keyword
-     * );
-     * ```
-     */
-    async hybridSearch(collectionName, queryEmbedding, queryText, options) {
-        this.ensureInitialized();
-        const alphaRaw = options?.alpha ?? 0.7;
-        const alpha = Number.isFinite(alphaRaw) ? Math.max(0, Math.min(1, alphaRaw)) : 0.7;
-        const topK = options?.topK ?? 10;
-        const fusion = options?.fusion === 'weighted' ? 'weighted' : 'rrf';
-        const rrfK = Number.isFinite(options?.rrfK) ? Math.max(1, options.rrfK) : 60;
-        const lexicalTopK = Number.isFinite(options?.lexicalTopK) ? Math.max(1, options.lexicalTopK) : topK * 3;
-        const denseTopK = topK * 3;
-        const collection = await this.getCollectionMetadata(collectionName);
-        if (queryEmbedding.length !== collection.dimension) {
-            throw new GMIError(`Query embedding dimension ${queryEmbedding.length} does not match collection dimension ${collection.dimension}.`, GMIErrorCode.VALIDATION_ERROR, { expected: collection.dimension, got: queryEmbedding.length });
-        }
-        // Load documents with SQL-level metadata filtering (avoids loading unmatched rows).
-        let hybridQuery = `SELECT * FROM ${this.tablePrefix}documents WHERE collection_name = ?`;
-        const hybridParams = [collectionName];
-        if (options?.filter) {
-            const filterSQL = this.buildMetadataFilterSQL(options.filter);
-            hybridQuery += filterSQL.clause;
-            hybridParams.push(...filterSQL.params);
-        }
-        const rows = await this.adapter.all(hybridQuery, hybridParams);
-        const tokenize = (text) => {
-            /* Use pluggable pipeline when configured */
-            if (this.pipeline)
-                return this.pipeline.processToStrings(text);
-            /* Fallback: built-in regex tokenizer with natural's 170-word stop word list */
-            const stopWords = getNaturalStopWords();
-            return text.toLowerCase().split(/[^a-z0-9_]+/g).filter((t) => t.length > 2 && !stopWords.has(t));
-        };
-        const queryTerms = tokenize(queryText);
-        const queryTermSet = new Set(queryTerms);
-        const scored = [];
-        const termDocFreq = new Map(); // df per term
-        let totalDocLength = 0;
-        // First pass: dense score + collect BM25 stats (doc length + df for query terms)
-        for (const row of rows) {
-            const embedding = this.decodeStoredEmbedding(row.embedding_blob);
-            const metadata = row.metadata_json ? JSON.parse(row.metadata_json) : undefined;
-            // SQL pre-filter narrows the candidate set; JS post-filter keeps the
-            // full MetadataFilter semantics for array and mixed-type operators.
-            if (options?.filter && !this.matchesFilter(metadata, options.filter)) {
-                continue;
-            }
-            let denseScore;
-            switch (collection.similarityMetric) {
-                case 'euclidean':
-                    denseScore = -vecEuclideanDistance(queryEmbedding, embedding);
-                    break;
-                case 'dotproduct':
-                    denseScore = vecDotProduct(queryEmbedding, embedding);
-                    break;
-                case 'cosine':
-                default:
-                    denseScore = vecCosineSimilarity(queryEmbedding, embedding);
-                    break;
-            }
-            // Allow dense thresholding without suppressing lexical-only matches.
-            if (options?.minSimilarityScore !== undefined && denseScore < options.minSimilarityScore) {
-                denseScore = Number.NEGATIVE_INFINITY;
-            }
-            const textContent = row.text_content ?? undefined;
-            let docLength = 0;
-            let uniqueTermsInDoc = null;
-            if (textContent && queryTermSet.size > 0) {
-                const tokens = tokenize(textContent);
-                docLength = tokens.length;
-                totalDocLength += docLength;
-                uniqueTermsInDoc = new Set();
-                for (const token of tokens) {
-                    if (queryTermSet.has(token))
-                        uniqueTermsInDoc.add(token);
-                }
-                for (const term of uniqueTermsInDoc) {
-                    termDocFreq.set(term, (termDocFreq.get(term) ?? 0) + 1);
-                }
-            }
-            scored.push({
-                id: row.id,
-                embedding,
-                textContent,
-                metadata: options?.includeMetadata !== false ? metadata : undefined,
-                denseScore,
-                bm25Score: 0,
-            });
-        }
-        const N = scored.length;
-        const avgdl = N > 0 ? totalDocLength / Math.max(1, N) : 0;
-        const k1 = 1.2;
-        const b = 0.75;
-        // Second pass: compute BM25 for query terms
-        if (queryTerms.length > 0 && N > 0 && avgdl > 0) {
-            const idfCache = new Map();
-            for (const term of queryTermSet) {
-                const df = termDocFreq.get(term) ?? 0;
-                const idf = Math.log(1 + (N - df + 0.5) / (df + 0.5));
-                idfCache.set(term, idf);
-            }
-            for (const doc of scored) {
-                if (!doc.textContent)
-                    continue;
-                const tokens = tokenize(doc.textContent);
-                const dl = tokens.length;
-                if (dl === 0)
-                    continue;
-                const tf = new Map();
-                for (const token of tokens) {
-                    if (queryTermSet.has(token))
-                        tf.set(token, (tf.get(token) ?? 0) + 1);
-                }
-                let score = 0;
-                for (const term of queryTermSet) {
-                    const f = tf.get(term) ?? 0;
-                    if (f === 0)
-                        continue;
-                    const idf = idfCache.get(term) ?? 0;
-                    const denom = f + k1 * (1 - b + b * (dl / avgdl));
-                    score += idf * ((f * (k1 + 1)) / denom);
-                }
-                doc.bm25Score = score;
-            }
-        }
-        // Build ranked lists for fusion.
-        const denseRanked = scored
-            .filter((d) => Number.isFinite(d.denseScore) && d.denseScore !== Number.NEGATIVE_INFINITY)
-            .sort((a, b) => b.denseScore - a.denseScore)
-            .slice(0, denseTopK);
-        const lexicalRanked = scored
-            .filter((d) => d.bm25Score > 0)
-            .sort((a, b) => b.bm25Score - a.bm25Score)
-            .slice(0, lexicalTopK);
-        const denseRank = new Map();
-        denseRanked.forEach((d, idx) => denseRank.set(d.id, idx + 1));
-        const lexRank = new Map();
-        lexicalRanked.forEach((d, idx) => lexRank.set(d.id, idx + 1));
-        const candidateIds = new Set();
-        denseRanked.forEach((d) => candidateIds.add(d.id));
-        lexicalRanked.forEach((d) => candidateIds.add(d.id));
-        const docById = new Map();
-        scored.forEach((d) => docById.set(d.id, d));
-        const fused = [];
-        if (fusion === 'weighted') {
-            const denseScores = denseRanked.map((d) => d.denseScore);
-            const lexScores = lexicalRanked.map((d) => d.bm25Score);
-            const denseMin = denseScores.length ? Math.min(...denseScores) : 0;
-            const denseMax = denseScores.length ? Math.max(...denseScores) : 1;
-            const lexMax = lexScores.length ? Math.max(...lexScores) : 1;
-            for (const id of candidateIds) {
-                const doc = docById.get(id);
-                if (!doc)
-                    continue;
-                const dense = denseRank.has(id)
-                    ? (doc.denseScore - denseMin) / Math.max(1e-9, denseMax - denseMin)
-                    : 0;
-                const lex = lexRank.has(id) ? doc.bm25Score / Math.max(1e-9, lexMax) : 0;
-                fused.push({ doc, fusedScore: alpha * dense + (1 - alpha) * lex });
-            }
-        }
-        else {
-            for (const id of candidateIds) {
-                const doc = docById.get(id);
-                if (!doc)
-                    continue;
-                const dr = denseRank.get(id);
-                const lr = lexRank.get(id);
-                const dense = dr ? alpha * (1 / (rrfK + dr)) : 0;
-                const lex = lr ? (1 - alpha) * (1 / (rrfK + lr)) : 0;
-                fused.push({ doc, fusedScore: dense + lex });
-            }
-        }
-        fused.sort((a, b) => b.fusedScore - a.fusedScore);
-        const documents = fused.slice(0, topK).map(({ doc, fusedScore }) => {
-            const out = {
-                id: doc.id,
-                embedding: options?.includeEmbedding ? doc.embedding : [],
-                similarityScore: fusedScore,
-            };
-            if (options?.includeTextContent && doc.textContent) {
-                out.textContent = doc.textContent;
-            }
-            if (options?.includeMetadata !== false && doc.metadata) {
-                out.metadata = doc.metadata;
-            }
-            return out;
-        });
-        return {
-            documents,
-            queryId: `sql-hybrid-${uuidv4()}`,
-            stats: {
-                fusion,
-                vectorWeight: alpha,
-                lexicalWeight: 1 - alpha,
-                rrfK: fusion === 'rrf' ? rrfK : undefined,
-                queryTerms: queryTerms.length,
-                corpusSize: N,
-                denseCandidates: denseRanked.length,
-                lexicalCandidates: lexicalRanked.length,
-                returnedCount: documents.length,
-            },
-        };
-    }
-    /**
-     * Deletes documents from a collection.
-     *
-     * @param {string} collectionName - Collection to delete from
-     * @param {string[]} [ids] - Specific document IDs to delete
-     * @param {DeleteOptions} [options] - Delete options (filter, deleteAll)
-     * @returns {Promise<DeleteResult>} Deletion result
-     */
-    async delete(collectionName, ids, options) {
-        this.ensureInitialized();
-        const collection = await this.getCollectionMetadata(collectionName);
-        let deletedCount = 0;
-        let deletedIds = [];
-        const errors = [];
-        if (options?.deleteAll && !ids && !options.filter) {
-            // Delete all documents in collection
-            const rows = await this.adapter.all(`SELECT id FROM ${this.tablePrefix}documents WHERE collection_name = ?`, [collectionName]);
-            deletedIds = rows.map(row => row.id);
-            const result = await this.adapter.run(`DELETE FROM ${this.tablePrefix}documents WHERE collection_name = ?`, [collectionName]);
-            deletedCount = result.changes;
-            console.warn(`SqlVectorStore (ID: ${this.providerId}): All ${deletedCount} documents deleted from '${collectionName}'.`);
-        }
-        else if (ids && ids.length > 0) {
-            // Delete specific IDs
-            deletedIds = [...ids];
-            const placeholders = ids.map(() => '?').join(',');
-            const result = await this.adapter.run(`DELETE FROM ${this.tablePrefix}documents WHERE collection_name = ? AND id IN (${placeholders})`, [collectionName, ...ids]);
-            deletedCount = result.changes;
-        }
-        else if (options?.filter) {
-            // Delete by filter (fetch matching docs first, then delete)
-            const rows = await this.adapter.all(`SELECT id, metadata_json FROM ${this.tablePrefix}documents WHERE collection_name = ?`, [collectionName]);
-            const idsToDelete = [];
-            for (const row of rows) {
-                const metadata = row.metadata_json ? JSON.parse(row.metadata_json) : undefined;
-                if (this.matchesFilter(metadata, options.filter)) {
-                    idsToDelete.push(row.id);
-                }
-            }
-            if (idsToDelete.length > 0) {
-                deletedIds = idsToDelete;
-                const placeholders = idsToDelete.map(() => '?').join(',');
-                const result = await this.adapter.run(`DELETE FROM ${this.tablePrefix}documents WHERE collection_name = ? AND id IN (${placeholders})`, [collectionName, ...idsToDelete]);
-                deletedCount = result.changes;
-            }
-        }
-        // Update collection document count
-        const now = Date.now();
-        const countResult = await this.adapter.get(`SELECT COUNT(*) as count FROM ${this.tablePrefix}documents WHERE collection_name = ?`, [collectionName]);
-        await this.adapter.run(`UPDATE ${this.tablePrefix}collections SET document_count = ?, updated_at = ? WHERE name = ?`, [countResult?.count ?? 0, now, collectionName]);
-        const sidecar = await this.getOrCreateSidecar(collection);
-        if (sidecar?.isActive() && deletedIds.length > 0) {
-            await sidecar.removeBatch(deletedIds);
-        }
-        return {
-            deletedCount,
-            failedCount: errors.length > 0 ? errors.length : undefined,
-            errors: errors.length > 0 ? errors : undefined,
-        };
-    }
-    /**
-     * Checks the health of the vector store.
-     *
-     * @returns {Promise<{ isHealthy: boolean; details?: any }>} Health status
-     */
-    async checkHealth() {
-        try {
-            this.ensureInitialized();
-            // Simple health check query
-            const collections = await this.adapter.all(`SELECT name, document_count FROM ${this.tablePrefix}collections`);
-            const totalDocuments = collections.reduce((sum, c) => sum + c.document_count, 0);
-            return {
-                isHealthy: true,
-                details: {
-                    providerId: this.providerId,
-                    configId: this.config.id,
-                    type: 'sql',
-                    collectionCount: collections.length,
-                    totalDocuments,
-                    collections: collections.map(c => ({ name: c.name, documentCount: c.document_count })),
-                },
-            };
-        }
-        catch (error) {
-            return {
-                isHealthy: false,
-                details: {
-                    providerId: this.providerId,
-                    error: error.message,
-                },
-            };
-        }
-    }
-    /**
-     * Gracefully shuts down the vector store.
-     */
-    async shutdown() {
-        if (!this.isInitialized) {
-            return;
-        }
-        for (const sidecar of this.sidecars.values()) {
-            await sidecar.shutdown();
-        }
-        this.sidecars.clear();
-        this.hnswSidecarClass = undefined;
-        if (this.ownsAdapter && this.adapter) {
-            await this.adapter.close();
-        }
-        this.isInitialized = false;
-        console.log(`SqlVectorStore (ID: ${this.providerId}) shut down.`);
-    }
-    /**
-     * Gets statistics for a collection or the entire store.
-     *
-     * @param {string} [collectionName] - Specific collection, or all if omitted
-     * @returns {Promise<Record<string, any>>} Statistics
-     */
-    async getStats(collectionName) {
-        this.ensureInitialized();
-        if (collectionName) {
-            const collection = await this.getCollectionMetadata(collectionName);
-            return {
-                collectionName: collection.name,
-                dimension: collection.dimension,
-                similarityMetric: collection.similarityMetric,
-                documentCount: collection.documentCount,
-                createdAt: collection.createdAt,
-                updatedAt: collection.updatedAt,
-            };
-        }
-        const collections = await this.adapter.all(`SELECT * FROM ${this.tablePrefix}collections`);
-        return {
-            providerId: this.providerId,
-            configId: this.config.id,
-            type: 'sql',
-            collectionCount: collections.length,
-            totalDocuments: collections.reduce((sum, c) => sum + c.document_count, 0),
-            collections: collections.map((c) => ({
-                name: c.name,
-                dimension: c.dimension,
-                documentCount: c.document_count,
-            })),
-        };
-    }
-    // ============================================================================
-    // Private Helper Methods
-    // ============================================================================
-    /**
-     * Lazily load the HNSW sidecar class once for this store instance.
-     */
-    async loadHnswSidecarClass() {
-        if (this.config.hnswThreshold === Infinity) {
-            return null;
-        }
-        if (this.hnswSidecarClass !== undefined) {
-            return this.hnswSidecarClass;
-        }
-        try {
-            const { HnswIndexSidecar } = await import('../vector-search/HnswIndexSidecar.js');
-            this.hnswSidecarClass = HnswIndexSidecar;
-        }
-        catch {
-            this.hnswSidecarClass = null;
-        }
-        return this.hnswSidecarClass;
-    }
-    /**
-     * Get or create the HNSW sidecar for a specific collection.
-     *
-     * Sidecars are collection-scoped so dimensions, metrics, and document IDs
-     * stay isolated between collections.
-     */
-    async getOrCreateSidecar(collection) {
-        const existing = this.sidecars.get(collection.name);
-        if (existing) {
-            return existing;
-        }
-        const sidecar = this.config.hnswSidecarFactory
-            ? this.config.hnswSidecarFactory()
-            : await (async () => {
-                const HnswSidecarClass = await this.loadHnswSidecarClass();
-                return HnswSidecarClass ? new HnswSidecarClass() : null;
-            })();
-        if (!sidecar) {
-            return null;
-        }
-        await sidecar.initialize({
-            indexPath: this.getSidecarIndexPath(collection.name),
-            dimensions: this.config.hnswDimensions ?? collection.dimension,
-            metric: collection.similarityMetric,
-            activationThreshold: this.config.hnswThreshold ?? 1000,
-        });
-        if (!sidecar.isAvailable()) {
-            this.hnswSidecarClass = null;
-            return null;
-        }
-        this.sidecars.set(collection.name, sidecar);
-        return sidecar;
-    }
-    /**
-     * Derive a stable per-collection sidecar path from the configured SQL store.
-     */
-    getSidecarIndexPath(collectionName) {
-        const storageConfig = this.config.storage;
-        const storagePath = storageConfig?.filePath ?? storageConfig?.database;
-        const safeCollectionName = collectionName.replace(/[^a-z0-9._-]+/gi, '_');
-        return storagePath
-            ? `${storagePath}.${safeCollectionName}.hnsw`
-            : `/tmp/agentos-rag-${this.providerId}-${safeCollectionName}.hnsw`;
-    }
-    /**
-     * Translate a MetadataFilter into dialect-aware SQL WHERE clauses.
-     * Pushes the easy scalar predicates into SQL so fewer rows are loaded into JS.
-     * Exact semantics are enforced by `matchesFilter()` after row hydration.
-     *
-     * @param filter   - The metadata filter to translate.
-     * @param column   - The JSON column name (default: 'metadata_json').
-     * @returns Object with `clause` (SQL fragment) and `params` (bind values).
-     */
-    buildMetadataFilterSQL(filter, column = 'metadata_json') {
-        const conditions = [];
-        const params = [];
-        for (const [field, condition] of Object.entries(filter)) {
-            const path = this.features.dialect.jsonExtract(column, `$.${field}`);
-            // Direct scalar match (implicit $eq)
-            if (typeof condition !== 'object' || condition === null) {
-                conditions.push(`${path} = ?`);
-                params.push(condition);
-                continue;
-            }
-            const cond = condition;
-            if (cond.$eq !== undefined) {
-                conditions.push(`${path} = ?`);
-                params.push(cond.$eq);
-            }
-            if (cond.$ne !== undefined) {
-                conditions.push(`${path} != ?`);
-                params.push(cond.$ne);
-            }
-            if (cond.$gt !== undefined) {
-                conditions.push(`${path} > ?`);
-                params.push(cond.$gt);
-            }
-            if (cond.$gte !== undefined) {
-                conditions.push(`${path} >= ?`);
-                params.push(cond.$gte);
-            }
-            if (cond.$lt !== undefined) {
-                conditions.push(`${path} < ?`);
-                params.push(cond.$lt);
-            }
-            if (cond.$lte !== undefined) {
-                conditions.push(`${path} <= ?`);
-                params.push(cond.$lte);
-            }
-            if (cond.$in !== undefined && Array.isArray(cond.$in)) {
-                const placeholders = cond.$in.map(() => '?').join(', ');
-                conditions.push(`${path} IN (${placeholders})`);
-                params.push(...cond.$in);
-            }
-            if (cond.$nin !== undefined && Array.isArray(cond.$nin)) {
-                const placeholders = cond.$nin.map(() => '?').join(', ');
-                conditions.push(`${path} NOT IN (${placeholders})`);
-                params.push(...cond.$nin);
-            }
-            if (cond.$exists !== undefined) {
-                conditions.push(cond.$exists ? `${path} IS NOT NULL` : `${path} IS NULL`);
-            }
-            if (cond.$contains !== undefined) {
-                // For string fields: LIKE %value%. For JSON arrays: json_each check.
-                conditions.push(`${path} LIKE ?`);
-                params.push(`%${cond.$contains}%`);
-            }
-            if (cond.$textSearch !== undefined) {
-                conditions.push(`${path} LIKE ?`);
-                params.push(`%${cond.$textSearch}%`);
-            }
-        }
-        return {
-            clause: conditions.length > 0 ? ` AND ${conditions.join(' AND ')}` : '',
-            params,
-        };
-    }
-    encodeEmbedding(embedding) {
-        return this.bytesToBase64(this.features.blobCodec.encode(embedding));
-    }
-    decodeStoredEmbedding(value) {
-        if (typeof value === 'string') {
-            if (this.isLegacyJsonEmbedding(value)) {
-                return JSON.parse(value);
-            }
-            return this.features.blobCodec.decode(this.base64ToBytes(value));
-        }
-        const bytes = this.asBinaryBytes(value);
-        return bytes ? this.features.blobCodec.decode(bytes) : [];
-    }
-    isLegacyJsonEmbedding(value) {
-        return value.trimStart().startsWith('[');
-    }
-    asBinaryBytes(value) {
-        if (value == null)
-            return null;
-        if (value instanceof Uint8Array)
-            return value;
-        if (value instanceof ArrayBuffer)
-            return new Uint8Array(value);
-        if (ArrayBuffer.isView(value)) {
-            return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
-        }
-        return null;
-    }
-    bytesToBase64(bytes) {
-        if (typeof Buffer !== 'undefined') {
-            return Buffer.from(bytes).toString('base64');
-        }
-        const btoaFn = globalThis.btoa;
-        if (!btoaFn) {
-            throw new Error('No base64 encoder available in this runtime.');
-        }
-        let binary = '';
-        const chunkSize = 0x8000;
-        for (let i = 0; i < bytes.length; i += chunkSize) {
-            binary += String.fromCharCode(...bytes.subarray(i, i + chunkSize));
-        }
-        return btoaFn(binary);
-    }
-    base64ToBytes(encoded) {
-        if (typeof Buffer !== 'undefined') {
-            return new Uint8Array(Buffer.from(encoded, 'base64'));
-        }
-        const atobFn = globalThis.atob;
-        if (!atobFn) {
-            throw new Error('No base64 decoder available in this runtime.');
-        }
-        const binary = atobFn(encoded);
-        const bytes = new Uint8Array(binary.length);
-        for (let i = 0; i < binary.length; i++) {
-            bytes[i] = binary.charCodeAt(i);
-        }
-        return bytes;
-    }
-    // Distance methods removed — now imported from rag/utils/vectorMath.ts
-    /**
-     * Checks if metadata matches a filter.
-     * @private
-     */
-    matchesFilter(metadata, filter) {
-        if (!metadata) {
-            // Check if filter only has $exists: false conditions
-            for (const key in filter) {
-                const condition = filter[key];
-                if (typeof condition === 'object' && condition !== null && condition.$exists === false) {
-                    continue;
-                }
-                return false;
-            }
-            return true;
-        }
-        for (const key in filter) {
-            const docValue = metadata[key];
-            const filterValue = filter[key];
-            if (typeof filterValue === 'object' && filterValue !== null) {
-                if (!this.evaluateCondition(docValue, filterValue)) {
-                    return false;
-                }
-            }
-            else {
-                // Direct equality check
-                if (Array.isArray(docValue)) {
-                    if (!docValue.includes(filterValue)) {
-                        return false;
-                    }
-                }
-                else if (docValue !== filterValue) {
-                    return false;
-                }
-            }
-        }
-        return true;
-    }
-    /**
-     * Evaluates a single filter condition.
-     * @private
-     */
-    evaluateCondition(docValue, condition) {
-        if (condition.$exists !== undefined) {
-            return condition.$exists === (docValue !== undefined);
-        }
-        if (docValue === undefined)
-            return false;
-        if (condition.$eq !== undefined && docValue !== condition.$eq)
-            return false;
-        if (condition.$ne !== undefined && docValue === condition.$ne)
-            return false;
-        if (typeof docValue === 'number') {
-            if (typeof condition.$gt === 'number' && !(docValue > condition.$gt))
-                return false;
-            if (typeof condition.$gte === 'number' && !(docValue >= condition.$gte))
-                return false;
-            if (typeof condition.$lt === 'number' && !(docValue < condition.$lt))
-                return false;
-            if (typeof condition.$lte === 'number' && !(docValue <= condition.$lte))
-                return false;
-        }
-        else if (typeof docValue === 'string') {
-            if (condition.$gt !== undefined && !(docValue > String(condition.$gt)))
-                return false;
-            if (condition.$gte !== undefined && !(docValue >= String(condition.$gte)))
-                return false;
-            if (condition.$lt !== undefined && !(docValue < String(condition.$lt)))
-                return false;
-            if (condition.$lte !== undefined && !(docValue <= String(condition.$lte)))
-                return false;
-        }
-        if (condition.$in !== undefined) {
-            if (Array.isArray(docValue)) {
-                if (!docValue.some(val => condition.$in.includes(val)))
-                    return false;
-            }
-            else {
-                if (!condition.$in.includes(docValue))
-                    return false;
-            }
-        }
-        if (condition.$nin !== undefined) {
-            if (Array.isArray(docValue)) {
-                if (docValue.some(val => condition.$nin.includes(val)))
-                    return false;
-            }
-            else {
-                if (condition.$nin.includes(docValue))
-                    return false;
-            }
-        }
-        if (Array.isArray(docValue)) {
-            if (condition.$contains !== undefined && !docValue.includes(condition.$contains))
-                return false;
-            if (condition.$all !== undefined && !condition.$all.every(item => docValue.includes(item)))
-                return false;
-        }
-        else if (typeof docValue === 'string' && condition.$contains !== undefined) {
-            if (!docValue.includes(String(condition.$contains)))
-                return false;
-        }
-        if (condition.$textSearch !== undefined && typeof docValue === 'string') {
-            if (!docValue.toLowerCase().includes(condition.$textSearch.toLowerCase()))
-                return false;
-        }
-        return true;
-    }
-}
-//# sourceMappingURL=SqlVectorStore.js.map