agentdb 1.0.0

package/dist/core/native-backend.mjs

@@ -0,0 +1,449 @@
+/**
+ * Native backend implementation using better-sqlite3
+ * High-performance backend for Node.js environments
+ */
+import Database from 'better-sqlite3';
+import { HNSWIndex, DEFAULT_HNSW_CONFIG } from '../index/hnsw.mjs';
+/**
+ * Native backend using better-sqlite3
+ * Optimized for Node.js with native SQLite bindings
+ * Features HNSW index for high-performance approximate nearest neighbor search
+ */
+export class NativeBackend {
+    constructor() {
+        this.db = null;
+        this.insertStmt = null;
+        this.initialized = false;
+        this.hnswIndex = null;
+        this.hnswConfig = DEFAULT_HNSW_CONFIG;
+    }
+    /**
+     * Initialize the native backend
+     */
+    initialize(config = {}) {
+        if (this.initialized) {
+            return;
+        }
+        const { path = ':memory:', memoryMode = true, cacheSize = 100 * 1024, // 100MB in KB
+        walMode = true, mmapSize = 256 * 1024 * 1024, // 256MB
+        hnsw = {} } = config;
+        // Create database instance
+        this.db = new Database(memoryMode ? ':memory:' : path);
+        // Initialize schema and prepare statements
+        this.initializeSchema(cacheSize, walMode, mmapSize);
+        this.registerCustomFunctions();
+        this.prepareStatements();
+        // Initialize HNSW index
+        this.hnswConfig = { ...DEFAULT_HNSW_CONFIG, ...hnsw };
+        if (this.hnswConfig.enabled) {
+            this.hnswIndex = new HNSWIndex(this.db, this.hnswConfig);
+        }
+        this.initialized = true;
+    }
+    /**
+     * Initialize database schema
+     */
+    initializeSchema(cacheSize, walMode, mmapSize) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Optimize SQLite for vector operations
+        this.db.pragma('journal_mode = ' + (walMode ? 'WAL' : 'MEMORY'));
+        this.db.pragma('synchronous = NORMAL');
+        this.db.pragma(`cache_size = -${cacheSize}`);
+        this.db.pragma('temp_store = MEMORY');
+        this.db.pragma(`mmap_size = ${mmapSize}`);
+        this.db.pragma('page_size = 4096');
+        // Create vectors table with optimized schema
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS vectors (
+        id TEXT PRIMARY KEY,
+        embedding BLOB NOT NULL,
+        norm REAL NOT NULL,
+        metadata TEXT,
+        timestamp INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_vectors_norm ON vectors(norm);
+      CREATE INDEX IF NOT EXISTS idx_vectors_timestamp ON vectors(timestamp);
+    `);
+    }
+    /**
+     * Register custom SQL functions for similarity calculations
+     */
+    registerCustomFunctions() {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Cosine similarity
+        this.db.function('cosine_similarity', { deterministic: true }, (a, b, normA, normB) => {
+            const arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
+            const arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
+            let dotProduct = 0;
+            for (let i = 0; i < arrA.length; i++) {
+                dotProduct += arrA[i] * arrB[i];
+            }
+            return dotProduct / (normA * normB);
+        });
+        // Euclidean distance
+        this.db.function('euclidean_distance', { deterministic: true }, (a, b) => {
+            const arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
+            const arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
+            let sum = 0;
+            for (let i = 0; i < arrA.length; i++) {
+                const diff = arrA[i] - arrB[i];
+                sum += diff * diff;
+            }
+            return Math.sqrt(sum);
+        });
+        // Dot product
+        this.db.function('dot_product', { deterministic: true }, (a, b) => {
+            const arrA = new Float32Array(a.buffer, a.byteOffset, a.byteLength / 4);
+            const arrB = new Float32Array(b.buffer, b.byteOffset, b.byteLength / 4);
+            let dotProduct = 0;
+            for (let i = 0; i < arrA.length; i++) {
+                dotProduct += arrA[i] * arrB[i];
+            }
+            return dotProduct;
+        });
+    }
+    /**
+     * Prepare frequently used statements
+     */
+    prepareStatements() {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        this.insertStmt = this.db.prepare(`
+      INSERT OR REPLACE INTO vectors (id, embedding, norm, metadata, timestamp)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    }
+    /**
+     * Calculate L2 norm of a vector
+     */
+    calculateNorm(embedding) {
+        let sum = 0;
+        for (const val of embedding) {
+            sum += val * val;
+        }
+        return Math.sqrt(sum);
+    }
+    /**
+     * Serialize embedding to binary format
+     */
+    serializeEmbedding(embedding) {
+        const buffer = Buffer.allocUnsafe(embedding.length * 4);
+        const view = new Float32Array(buffer.buffer, buffer.byteOffset, embedding.length);
+        view.set(embedding);
+        return buffer;
+    }
+    /**
+     * Deserialize embedding from binary format
+     */
+    deserializeEmbedding(buffer) {
+        const view = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
+        return Array.from(view);
+    }
+    /**
+     * Generate unique ID
+     */
+    generateId() {
+        return `vec_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+    }
+    /**
+     * Insert a single vector
+     */
+    insert(vector) {
+        if (!this.db || !this.insertStmt)
+            throw new Error('Database not initialized');
+        const id = vector.id || this.generateId();
+        const norm = this.calculateNorm(vector.embedding);
+        const embedding = this.serializeEmbedding(vector.embedding);
+        const metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
+        const timestamp = vector.timestamp || Date.now();
+        this.insertStmt.run(id, embedding, norm, metadata, timestamp);
+        // Update HNSW index if enabled
+        if (this.hnswIndex && this.shouldUseHNSW()) {
+            this.hnswIndex.insert(id, vector.embedding);
+        }
+        return id;
+    }
+    /**
+     * Insert multiple vectors in a transaction with optimized batch processing
+     * Performance improvements:
+     * - Single transaction for all inserts
+     * - Reused prepared statement
+     * - No wrapper function overhead
+     * - Chunked processing for large batches
+     * - Automatic HNSW index building after batch insert
+     */
+    insertBatch(vectors) {
+        if (!this.db || !this.insertStmt)
+            throw new Error('Database not initialized');
+        const ids = [];
+        const CHUNK_SIZE = 5000; // Process in chunks to avoid memory spikes
+        // Helper function to process a chunk of vectors
+        const processChunk = (chunk) => {
+            const chunkIds = [];
+            const transaction = this.db.transaction((vecs) => {
+                // Reuse the prepared statement for all inserts
+                for (const vector of vecs) {
+                    const id = vector.id || this.generateId();
+                    const norm = this.calculateNorm(vector.embedding);
+                    const embedding = this.serializeEmbedding(vector.embedding);
+                    const metadata = vector.metadata ? JSON.stringify(vector.metadata) : null;
+                    const timestamp = vector.timestamp || Date.now();
+                    this.insertStmt.run(id, embedding, norm, metadata, timestamp);
+                    chunkIds.push(id);
+                }
+            });
+            transaction(chunk);
+            return chunkIds;
+        };
+        // Process vectors in chunks
+        if (vectors.length <= CHUNK_SIZE) {
+            // Small batch - process directly
+            ids.push(...processChunk(vectors));
+        }
+        else {
+            // Large batch - process in chunks to manage memory
+            for (let i = 0; i < vectors.length; i += CHUNK_SIZE) {
+                const chunk = vectors.slice(i, i + CHUNK_SIZE);
+                ids.push(...processChunk(chunk));
+            }
+        }
+        // Build HNSW index if threshold reached and not already built
+        if (this.hnswIndex && this.shouldUseHNSW() && !this.hnswIndex.isReady()) {
+            console.log('Building HNSW index after batch insert...');
+            this.hnswIndex.build();
+        }
+        return ids;
+    }
+    /**
+     * Check if HNSW index should be used
+     */
+    shouldUseHNSW() {
+        if (!this.hnswIndex || !this.hnswConfig.enabled) {
+            return false;
+        }
+        const stats = this.stats();
+        return stats.count >= this.hnswConfig.minVectorsForIndex;
+    }
+    /**
+     * Search for k-nearest neighbors
+     * Automatically uses HNSW index when available and beneficial
+     */
+    search(queryEmbedding, k = 5, metric = 'cosine', threshold = 0.0) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Use HNSW index for euclidean distance if available
+        // Note: HNSW currently optimized for euclidean, can be extended for other metrics
+        if (this.hnswIndex && this.hnswIndex.isReady() && this.shouldUseHNSW() && metric === 'euclidean') {
+            return this.searchWithHNSW(queryEmbedding, k, threshold);
+        }
+        // Fall back to brute-force search
+        return this.bruteForceSearch(queryEmbedding, k, metric, threshold);
+    }
+    /**
+     * Search using HNSW index (fast approximate search)
+     */
+    searchWithHNSW(queryEmbedding, k, threshold) {
+        if (!this.hnswIndex || !this.db) {
+            throw new Error('HNSW index not initialized');
+        }
+        const results = this.hnswIndex.search(queryEmbedding, k);
+        // Filter by threshold and get metadata from main table
+        return results
+            .filter(r => threshold === 0 || r.distance <= threshold)
+            .map(result => {
+            const stmt = this.db.prepare('SELECT metadata FROM vectors WHERE id = ?');
+            const row = stmt.get(result.vectorId);
+            return {
+                id: result.vectorId,
+                score: result.distance,
+                embedding: result.embedding,
+                metadata: row?.metadata ? JSON.parse(row.metadata) : undefined
+            };
+        });
+    }
+    /**
+     * Brute-force search (accurate but slower)
+     */
+    bruteForceSearch(queryEmbedding, k, metric, threshold) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        const queryBuffer = this.serializeEmbedding(queryEmbedding);
+        const queryNorm = this.calculateNorm(queryEmbedding);
+        let sql;
+        let params;
+        switch (metric) {
+            case 'cosine':
+                sql = `
+          SELECT
+            id,
+            embedding,
+            metadata,
+            cosine_similarity(embedding, ?, norm, ?) as score
+          FROM vectors
+          WHERE cosine_similarity(embedding, ?, norm, ?) >= ?
+          ORDER BY score DESC
+          LIMIT ?
+        `;
+                params = [queryBuffer, queryNorm, queryBuffer, queryNorm, threshold, k];
+                break;
+            case 'euclidean':
+                sql = `
+          SELECT
+            id,
+            embedding,
+            metadata,
+            euclidean_distance(embedding, ?) as score
+          FROM vectors
+          WHERE euclidean_distance(embedding, ?) <= ? OR ? = 0
+          ORDER BY score ASC
+          LIMIT ?
+        `;
+                params = [queryBuffer, queryBuffer, threshold, threshold, k];
+                break;
+            case 'dot':
+                sql = `
+          SELECT
+            id,
+            embedding,
+            metadata,
+            dot_product(embedding, ?) as score
+          FROM vectors
+          WHERE dot_product(embedding, ?) >= ?
+          ORDER BY score DESC
+          LIMIT ?
+        `;
+                params = [queryBuffer, queryBuffer, threshold, k];
+                break;
+            default:
+                throw new Error(`Unsupported similarity metric: ${metric}`);
+        }
+        const stmt = this.db.prepare(sql);
+        const rows = stmt.all(...params);
+        return rows.map(row => ({
+            id: row.id,
+            score: row.score,
+            embedding: this.deserializeEmbedding(row.embedding),
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+        }));
+    }
+    /**
+     * Get vector by ID
+     */
+    get(id) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        const stmt = this.db.prepare('SELECT * FROM vectors WHERE id = ?');
+        const row = stmt.get(id);
+        if (!row)
+            return null;
+        return {
+            id: row.id,
+            embedding: this.deserializeEmbedding(row.embedding),
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
+            norm: row.norm,
+            timestamp: row.timestamp
+        };
+    }
+    /**
+     * Delete vector by ID
+     */
+    delete(id) {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        // Delete from HNSW index if enabled
+        if (this.hnswIndex && this.shouldUseHNSW()) {
+            this.hnswIndex.delete(id);
+        }
+        const stmt = this.db.prepare('DELETE FROM vectors WHERE id = ?');
+        const result = stmt.run(id);
+        return result.changes > 0;
+    }
+    /**
+     * Get database statistics
+     */
+    stats() {
+        if (!this.db)
+            throw new Error('Database not initialized');
+        const countStmt = this.db.prepare('SELECT COUNT(*) as count FROM vectors');
+        const sizeStmt = this.db.prepare('SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()');
+        const count = countStmt.get().count;
+        const size = sizeStmt.get().size;
+        return { count, size };
+    }
+    /**
+     * Close database connection
+     */
+    close() {
+        if (this.db) {
+            this.db.close();
+            this.db = null;
+        }
+        this.insertStmt = null;
+        this.initialized = false;
+    }
+    /**
+     * Get raw database instance (for advanced usage)
+     */
+    getDatabase() {
+        return this.db;
+    }
+    /**
+     * Check if backend is initialized
+     */
+    isInitialized() {
+        return this.initialized;
+    }
+    /**
+     * Build HNSW index manually
+     * Useful for pre-building index on existing data
+     */
+    buildHNSWIndex() {
+        if (!this.hnswIndex) {
+            throw new Error('HNSW index not enabled');
+        }
+        if (!this.shouldUseHNSW()) {
+            const stats = this.stats();
+            console.warn(`Vector count (${stats.count}) below threshold (${this.hnswConfig.minVectorsForIndex}). Index build skipped.`);
+            return;
+        }
+        this.hnswIndex.build();
+    }
+    /**
+     * Get HNSW index statistics
+     */
+    getHNSWStats() {
+        if (!this.hnswIndex) {
+            return null;
+        }
+        const stats = this.hnswIndex.getStats();
+        return {
+            enabled: this.hnswConfig.enabled,
+            ready: this.hnswIndex.isReady(),
+            ...stats
+        };
+    }
+    /**
+     * Clear HNSW index
+     */
+    clearHNSWIndex() {
+        if (this.hnswIndex) {
+            this.hnswIndex.clear();
+        }
+    }
+    /**
+     * Update HNSW configuration and rebuild if needed
+     */
+    updateHNSWConfig(config, rebuild = false) {
+        if (!this.hnswIndex) {
+            throw new Error('HNSW index not enabled');
+        }
+        this.hnswIndex.updateConfig(config);
+        this.hnswConfig = { ...this.hnswConfig, ...config };
+        if (rebuild) {
+            this.hnswIndex.build();
+        }
+    }
+}

package/dist/core/vector-db.d.ts

@@ -0,0 +1,127 @@
+/**
+ * Core SQLiteVector database implementation with backend abstraction
+ * Supports both native (better-sqlite3) and WASM (sql.js) backends
+ */
+import { Vector, SearchResult, SimilarityMetric, DatabaseConfig } from '../types';
+import { VectorBackend, BackendType, ExtendedDatabaseConfig } from './backend-interface';
+import { QueryCache } from '../cache/query-cache';
+import { ProductQuantizer } from '../quantization/product-quantization';
+import { VectorQueryBuilder } from '../query/query-builder';
+export declare class SQLiteVectorDB {
+    private backend;
+    private backendType;
+    private queryCache?;
+    private quantizer?;
+    constructor(config?: ExtendedDatabaseConfig);
+    /**
+     * Detect appropriate backend based on environment and configuration
+     */
+    private detectBackend;
+    /**
+     * Create backend instance
+     */
+    private createBackend;
+    /**
+     * Async initialization for WASM backend
+     * Must be called after constructor if using WASM backend
+     */
+    initializeAsync(config?: DatabaseConfig): Promise<void>;
+    /**
+     * Get backend type
+     */
+    getBackendType(): BackendType;
+    /**
+     * Check if backend is initialized
+     */
+    isInitialized(): boolean;
+    /**
+     * Insert a single vector
+     */
+    insert(vector: Vector): string;
+    /**
+     * Insert multiple vectors in a transaction
+     */
+    insertBatch(vectors: Vector[]): string[];
+    /**
+     * Search for k-nearest neighbors with optional caching
+     */
+    search(queryEmbedding: number[], k?: number, metric?: SimilarityMetric, threshold?: number): SearchResult[];
+    /**
+     * Get vector by ID
+     */
+    get(id: string): Vector | null;
+    /**
+     * Delete vector by ID
+     */
+    delete(id: string): boolean;
+    /**
+     * Get database statistics
+     */
+    stats(): {
+        count: number;
+        size: number;
+    };
+    /**
+     * Close database connection
+     */
+    close(): void;
+    /**
+     * Export database to binary format (WASM only)
+     */
+    export(): Uint8Array | undefined;
+    /**
+     * Import database from binary format (WASM only)
+     */
+    importAsync(data: Uint8Array): Promise<void>;
+    /**
+     * Get raw backend instance (for advanced usage)
+     */
+    getBackend(): VectorBackend;
+    /**
+     * Get raw database instance (for advanced usage with native backend)
+     * @deprecated Use getBackend() instead for cross-backend compatibility
+     */
+    getDatabase(): any;
+    /**
+     * Get query cache instance
+     */
+    getQueryCache(): QueryCache<SearchResult[]> | undefined;
+    /**
+     * Get query cache statistics
+     */
+    getCacheStats(): import("../cache/query-cache").CacheStats | undefined;
+    /**
+     * Clear query cache
+     */
+    clearCache(): void;
+    /**
+     * Get quantizer instance
+     */
+    getQuantizer(): ProductQuantizer | undefined;
+    /**
+     * Train quantizer with existing vectors
+     */
+    trainQuantizer(): Promise<void>;
+    /**
+     * Get compression statistics (if quantization enabled)
+     */
+    getCompressionStats(): import("../quantization/product-quantization").CompressionStats | undefined;
+    /**
+     * Create a fluent query builder for complex queries
+     *
+     * @returns VectorQueryBuilder instance
+     *
+     * @example
+     * ```typescript
+     * const results = await db.query()
+     *   .similarTo(queryVector)
+     *   .where('metadata.category', '=', 'tech')
+     *   .whereBetween('metadata.year', 2020, 2024)
+     *   .orderBySimilarity('desc')
+     *   .limit(10)
+     *   .execute();
+     * ```
+     */
+    query<T = any>(): VectorQueryBuilder<T>;
+}
+//# sourceMappingURL=vector-db.d.ts.map

package/dist/core/vector-db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-db.d.ts","sourceRoot":"","sources":["../../src/core/vector-db.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAGzF,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAE5D,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,UAAU,CAAC,CAA6B;IAChD,OAAO,CAAC,SAAS,CAAC,CAAmB;gBAEzB,MAAM,GAAE,sBAA2B;IAiC/C;;OAEG;IACH,OAAO,CAAC,aAAa;IAqBrB;;OAEG;IACH,OAAO,CAAC,aAAa;IAWrB;;;OAGG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAS7D;;OAEG;IACH,cAAc,IAAI,WAAW;IAI7B;;OAEG;IACH,aAAa,IAAI,OAAO;IAOxB;;OAEG;IACH,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAI9B;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAIxC;;OAEG;IACH,MAAM,CACJ,cAAc,EAAE,MAAM,EAAE,EACxB,CAAC,GAAE,MAAU,EACb,MAAM,GAAE,gBAA2B,EACnC,SAAS,GAAE,MAAY,GACtB,YAAY,EAAE;IAmBjB;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAI9B;;OAEG;IACH,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAI3B;;OAEG;IACH,KAAK,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAIxC;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACH,MAAM,IAAI,UAAU,GAAG,SAAS;IAOhC;;OAEG;IACG,WAAW,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAQlD;;OAEG;IACH,UAAU,IAAI,aAAa;IAI3B;;;OAGG;IACH,WAAW,IAAI,GAAG;IAOlB;;OAEG;IACH,aAAa,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC,GAAG,SAAS;IAIvD;;OAEG;IACH,aAAa;IAIb;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,YAAY,IAAI,gBAAgB,GAAG,SAAS;IAI5C;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBrC;;OAEG;IACH,mBAAmB;IAInB;;;;;;;;;;;;;;;OAeG;IACH,KAAK,CAAC,CAAC,GAAG,GAAG,KAAK,kBAAkB,CAAC,CAAC,CAAC;CAGxC"}