crewly 1.3.31 → 1.4.1

package/dist/backend/backend/src/services/knowledge/vector-store.service.js

@@ -1,9 +1,18 @@
 /**
- * Vector Store Service
+ * Vector Store Service — v2
  *
- * Provides local, on-device vector storage using SQLite (better-sqlite3).
- * Stores document embeddings and performs cosine-similarity search entirely
- * offline — no external vector database or cloud API is needed at query time.
+ * Local, on-device vector storage using SQLite (better-sqlite3) with optional
+ * sqlite-vec extension for native KNN search. Falls back gracefully to
+ * JavaScript cosine similarity when the extension is unavailable.
+ *
+ * v2 improvements over v1:
+ * - Binary BLOB storage (Float32Array) instead of JSON TEXT (~4× smaller, faster)
+ * - Optional sqlite-vec integration for native SQL-based KNN search (cosine metric)
+ * - Dimension validation ensures all vectors in a scope share the same width
+ * - Prepared statement caching for repeated operations
+ * - Input sanitization (NaN, Infinity rejection)
+ * - Batch upsert with transaction wrapping
+ * - Automatic schema migration from v1 (JSON TEXT) databases
  *
  * Storage locations:
  * - Global scope: ~/.crewly/vector-store.db
@@ -13,9 +22,60 @@
  */
 import * as path from 'path';
 import { existsSync, mkdirSync } from 'fs';
-import Database from 'better-sqlite3';
 import { LoggerService } from '../core/logger.service.js';
 import { CREWLY_CONSTANTS } from '../../constants.js';
+// ---------------------------------------------------------------------------
+// Lazy module loading
+// ---------------------------------------------------------------------------
+/**
+ * Lazy-loaded better-sqlite3 module reference.
+ * Uses dynamic require so that a native-module load failure does NOT cascade
+ * to every test file that transitively imports this service.
+ *
+ * @see https://github.com/stevehuang0115/crewly/issues/170
+ */
+let _BetterSqlite3 = null;
+/**
+ * Load better-sqlite3 on demand. Throws a clear error if the native module
+ * is not available (e.g. needs `npm rebuild better-sqlite3`).
+ */
+function getBetterSqlite3() {
+    if (!_BetterSqlite3) {
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            _BetterSqlite3 = require('better-sqlite3');
+        }
+        catch (err) {
+            throw new Error('better-sqlite3 native module failed to load. Run `npm rebuild better-sqlite3` to fix. ' +
+                `Original error: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return _BetterSqlite3;
+}
+/** Lazy-loaded sqlite-vec extension loader. */
+let _sqliteVec = null;
+let _sqliteVecLoadAttempted = false;
+/**
+ * Attempt to load the sqlite-vec module. Returns null if unavailable.
+ */
+function getSqliteVec() {
+    if (!_sqliteVecLoadAttempted) {
+        _sqliteVecLoadAttempted = true;
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            _sqliteVec = require('sqlite-vec');
+        }
+        catch {
+            _sqliteVec = null;
+        }
+    }
+    return _sqliteVec;
+}
+/** Reset the sqlite-vec loader state (for testing). */
+export function resetSqliteVecLoader() {
+    _sqliteVec = null;
+    _sqliteVecLoadAttempted = false;
+}
 /** Constants for the vector store */
 export const VECTOR_STORE_CONSTANTS = {
     /** SQLite database filename */
@@ -26,7 +86,36 @@ export const VECTOR_STORE_CONSTANTS = {
     DEFAULT_SIMILARITY_THRESHOLD: 0.1,
     /** Maximum metadata JSON size (bytes) */
     MAX_METADATA_SIZE: 65536,
+    /** Current schema version */
+    SCHEMA_VERSION: 2,
+    /** Maximum supported embedding dimensions */
+    MAX_DIMENSIONS: 4096,
+    /** Maximum items in a single batch upsert */
+    MAX_BATCH_SIZE: 500,
 };
+// ---------------------------------------------------------------------------
+// Utility functions
+// ---------------------------------------------------------------------------
+/**
+ * Convert a number array to a Float32Array Buffer for efficient BLOB storage.
+ *
+ * @param embedding - The embedding vector
+ * @returns Buffer containing the Float32Array binary data
+ */
+export function embeddingToBlob(embedding) {
+    const f32 = new Float32Array(embedding);
+    return Buffer.from(f32.buffer);
+}
+/**
+ * Convert a BLOB Buffer back to a number array.
+ *
+ * @param blob - Buffer containing Float32Array binary data
+ * @returns The embedding as a number array
+ */
+export function blobToEmbedding(blob) {
+    const f32 = new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / Float32Array.BYTES_PER_ELEMENT);
+    return Array.from(f32);
+}
 /**
  * Compute cosine similarity between two vectors.
  *
@@ -50,20 +139,47 @@ export function cosineSimilarity(a, b) {
     return denominator === 0 ? 0 : dotProduct / denominator;
 }
 /**
- * VectorStoreService provides local SQLite-based vector storage.
+ * Validate an embedding vector for storage. Rejects empty arrays, values
+ * that are not finite numbers (NaN, Infinity), and dimensions exceeding
+ * the maximum.
  *
- * Uses better-sqlite3 for synchronous, high-performance SQLite access.
- * Embeddings are stored as JSON arrays and cosine similarity is computed
- * in JavaScript for portability (no native vector extension required).
+ * @param embedding - The embedding to validate
+ * @throws Error if the embedding is invalid
+ */
+export function validateEmbedding(embedding) {
+    if (!embedding || embedding.length === 0) {
+        throw new Error('embedding must be a non-empty array');
+    }
+    if (embedding.length > VECTOR_STORE_CONSTANTS.MAX_DIMENSIONS) {
+        throw new Error(`embedding dimensions (${embedding.length}) exceed maximum (${VECTOR_STORE_CONSTANTS.MAX_DIMENSIONS})`);
+    }
+    for (let i = 0; i < embedding.length; i++) {
+        if (typeof embedding[i] !== 'number' || !isFinite(embedding[i])) {
+            throw new Error(`embedding contains invalid value at index ${i}: ${embedding[i]}`);
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Service
+// ---------------------------------------------------------------------------
+/**
+ * VectorStoreService provides local SQLite-based vector storage with optional
+ * native KNN search via the sqlite-vec extension.
  *
- * Each scope (global / project) gets its own database file so project
- * data stays isolated and portable.
+ * When sqlite-vec is available, search uses a `vec0` virtual table with cosine
+ * distance metric for hardware-accelerated nearest-neighbor queries. When it is
+ * not available, search falls back to JavaScript cosine similarity (fully
+ * portable, no native extension required).
+ *
+ * Each scope (global / project) gets its own database file so project data
+ * stays isolated and portable.
  *
  * @example
  * ```typescript
  * const store = VectorStoreService.getInstance();
  * store.upsert('doc-123', embedding, { title: 'My Doc' }, 'global');
- * const results = store.search(queryEmbedding, 'global', 5);
+ * const results = store.search(queryEmbedding, 'global');
+ * console.log(store.isVecEnabled('global')); // true if sqlite-vec loaded
  * ```
  */
 export class VectorStoreService {
@@ -119,54 +235,246 @@ export class VectorStoreService {
     // Database management
     // ---------------------------------------------------------------------------
     /**
-     * Open or retrieve a cached SQLite database connection.
-     * Creates the parent directory and schema if they don't exist.
+     * Check whether the sqlite-vec native KNN search is available for a scope.
+     *
+     * @param scope - Storage scope
+     * @param projectPath - Required when scope is 'project'
+     * @returns true if vec0 virtual table is active
+     */
+    isVecEnabled(scope, projectPath) {
+        const state = this.getDbState(scope, projectPath);
+        return state.vecEnabled;
+    }
+    /**
+     * Get (or prepare + cache) a statement for the given database state.
+     *
+     * @param state - Database state to prepare on
+     * @param key - Unique cache key
+     * @param sql - SQL to prepare if not yet cached
+     * @returns A prepared Statement
+     */
+    getStmt(state, key, sql) {
+        let stmt = state.stmtCache.get(key);
+        if (!stmt) {
+            stmt = state.db.prepare(sql);
+            state.stmtCache.set(key, stmt);
+        }
+        return stmt;
+    }
+    /**
+     * Open or retrieve a cached database connection. Creates the parent
+     * directory, schema, and vec0 table (if sqlite-vec is available) on first
+     * access. Handles automatic migration from v1 databases.
      *
      * @param scope - Document scope
      * @param projectPath - Project path for project-scoped databases
-     * @returns Open Database instance
+     * @returns Internal DatabaseState
      */
-    getDb(scope, projectPath) {
+    getDbState(scope, projectPath) {
         const dbFile = this.dbPath(scope, projectPath);
         const cached = this.databases.get(dbFile);
-        if (cached) {
+        if (cached)
             return cached;
-        }
         // Ensure parent directory exists
         const dir = path.dirname(dbFile);
         if (!existsSync(dir)) {
             mkdirSync(dir, { recursive: true });
         }
+        const Database = getBetterSqlite3();
         const db = new Database(dbFile);
-        // Enable WAL mode for better concurrent read performance
+        // Performance pragmas
         db.pragma('journal_mode = WAL');
-        // Create schema
+        db.pragma('synchronous = NORMAL');
+        // Try loading sqlite-vec extension
+        let vecEnabled = false;
+        const sqliteVec = getSqliteVec();
+        if (sqliteVec) {
+            try {
+                sqliteVec.load(db);
+                vecEnabled = true;
+                this.logger.info('sqlite-vec extension loaded', { path: dbFile });
+            }
+            catch (err) {
+                this.logger.warn('sqlite-vec extension failed to load, using JS fallback', {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        // Run schema migration
+        this.migrateSchema(db, vecEnabled);
+        const state = {
+            db,
+            vecEnabled,
+            dimensions: this.getStoredDimensions(db),
+            stmtCache: new Map(),
+        };
+        this.databases.set(dbFile, state);
+        this.logger.info('Opened vector store database', { path: dbFile, scope, vecEnabled });
+        return state;
+    }
+    /**
+     * Migrate database schema to v2.
+     * - Creates vec_config table for tracking schema version and dimensions
+     * - Adds dimensions column if missing
+     * - Converts JSON TEXT embeddings to BLOB if needed
+     * - Creates vec0 virtual table if sqlite-vec is available and dimensions are known
+     *
+     * @param db - The database to migrate
+     * @param vecEnabled - Whether sqlite-vec extension is loaded
+     */
+    migrateSchema(db, vecEnabled) {
+        // Config table for schema metadata
         db.exec(`
-      CREATE TABLE IF NOT EXISTS embeddings (
-        id TEXT PRIMARY KEY,
-        embedding TEXT NOT NULL,
-        metadata TEXT NOT NULL DEFAULT '{}',
-        created_at TEXT NOT NULL,
-        updated_at TEXT NOT NULL
+      CREATE TABLE IF NOT EXISTS vec_config (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
       );
-      CREATE INDEX IF NOT EXISTS idx_embeddings_updated_at ON embeddings(updated_at);
     `);
-        this.databases.set(dbFile, db);
-        this.logger.info('Opened vector store database', { path: dbFile, scope });
-        return db;
+        const versionRow = db.prepare('SELECT value FROM vec_config WHERE key = ?').get('schema_version');
+        const currentVersion = versionRow ? parseInt(versionRow.value, 10) : 0;
+        if (currentVersion < VECTOR_STORE_CONSTANTS.SCHEMA_VERSION) {
+            const tableExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='embeddings'").get();
+            if (tableExists) {
+                // Check column types to detect v1 vs v2
+                const colInfo = db.prepare('PRAGMA table_info(embeddings)').all();
+                const embeddingCol = colInfo.find((c) => c.name === 'embedding');
+                const hasDimensionsCol = colInfo.some((c) => c.name === 'dimensions');
+                if (embeddingCol && embeddingCol.type === 'TEXT') {
+                    // v1 → v2: convert JSON TEXT embeddings to BLOB
+                    this.logger.info('Migrating vector store from v1 (JSON) to v2 (BLOB)');
+                    if (!hasDimensionsCol) {
+                        db.exec('ALTER TABLE embeddings ADD COLUMN dimensions INTEGER NOT NULL DEFAULT 0');
+                    }
+                    const rows = db.prepare("SELECT id, embedding FROM embeddings WHERE typeof(embedding) = 'text'").all();
+                    const updateStmt = db.prepare('UPDATE embeddings SET embedding = ?, dimensions = ? WHERE id = ?');
+                    const migrate = db.transaction(() => {
+                        for (const row of rows) {
+                            try {
+                                const arr = JSON.parse(row.embedding);
+                                updateStmt.run(embeddingToBlob(arr), arr.length, row.id);
+                            }
+                            catch (err) {
+                                this.logger.warn('Failed to migrate embedding', {
+                                    id: row.id,
+                                    error: String(err),
+                                });
+                            }
+                        }
+                    });
+                    migrate();
+                }
+                else if (!hasDimensionsCol) {
+                    db.exec('ALTER TABLE embeddings ADD COLUMN dimensions INTEGER NOT NULL DEFAULT 0');
+                }
+            }
+            else {
+                // Fresh database — create v2 schema directly
+                db.exec(`
+          CREATE TABLE embeddings (
+            id TEXT PRIMARY KEY,
+            embedding BLOB NOT NULL,
+            metadata TEXT NOT NULL DEFAULT '{}',
+            dimensions INTEGER NOT NULL DEFAULT 0,
+            created_at TEXT NOT NULL,
+            updated_at TEXT NOT NULL
+          );
+          CREATE INDEX IF NOT EXISTS idx_embeddings_updated_at ON embeddings(updated_at);
+        `);
+            }
+            db.prepare('INSERT OR REPLACE INTO vec_config (key, value) VALUES (?, ?)').run('schema_version', String(VECTOR_STORE_CONSTANTS.SCHEMA_VERSION));
+        }
+        // Create vec0 virtual table if sqlite-vec is available and dimensions are known
+        if (vecEnabled) {
+            this.ensureVecTable(db);
+        }
+    }
+    /**
+     * Create the vec0 virtual table if it doesn't already exist and dimensions
+     * are known. Returns true if the table is ready for use.
+     *
+     * @param db - The database
+     * @param dimensions - Explicit dimensions; if omitted, reads from vec_config
+     * @returns true if vec0 table exists and is ready
+     */
+    ensureVecTable(db, dimensions) {
+        const dim = dimensions || this.getStoredDimensions(db);
+        if (!dim || dim <= 0)
+            return false;
+        const vecTableExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='vec_embeddings'").get();
+        if (!vecTableExists) {
+            try {
+                db.exec(`CREATE VIRTUAL TABLE vec_embeddings USING vec0(embedding float[${dim}] distance_metric=cosine)`);
+                db.prepare('INSERT OR REPLACE INTO vec_config (key, value) VALUES (?, ?)').run('vec_dimensions', String(dim));
+                this.logger.info('Created vec0 virtual table', { dimensions: dim });
+                // Backfill existing embeddings into vec0
+                this.backfillVecTable(db, dim);
+                return true;
+            }
+            catch (err) {
+                this.logger.warn('Failed to create vec0 table', {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+                return false;
+            }
+        }
+        return true;
     }
     /**
-     * Close all open database connections.
+     * Copy all existing embeddings matching the target dimensions into the vec0
+     * virtual table.
+     *
+     * @param db - The database
+     * @param dimensions - Target embedding dimensions
+     */
+    backfillVecTable(db, dimensions) {
+        const rows = db.prepare('SELECT rowid, embedding, dimensions FROM embeddings WHERE dimensions = ?').all(dimensions);
+        if (rows.length === 0)
+            return;
+        const insertVec = db.prepare('INSERT OR REPLACE INTO vec_embeddings(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)');
+        const backfill = db.transaction(() => {
+            for (const row of rows) {
+                try {
+                    insertVec.run(row.rowid, row.embedding);
+                }
+                catch (err) {
+                    this.logger.warn('Failed to backfill vec embedding', {
+                        rowid: row.rowid,
+                        error: String(err),
+                    });
+                }
+            }
+        });
+        backfill();
+        this.logger.info('Backfilled vec0 table', { count: rows.length });
+    }
+    /**
+     * Read the stored embedding dimensions from the vec_config table.
+     *
+     * @param db - The database
+     * @returns Stored dimensions, or null if not yet determined
+     */
+    getStoredDimensions(db) {
+        try {
+            const row = db.prepare('SELECT value FROM vec_config WHERE key = ?').get('vec_dimensions');
+            return row ? parseInt(row.value, 10) : null;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Close all open database connections and clear caches.
      */
     closeAll() {
-        for (const [dbPath, db] of this.databases) {
+        for (const [filePath, state] of this.databases) {
             try {
-                db.close();
-                this.logger.debug('Closed vector store database', { path: dbPath });
+                state.stmtCache.clear();
+                state.db.close();
+                this.logger.debug('Closed vector store database', { path: filePath });
             }
             catch (error) {
                 this.logger.warn('Error closing database', {
-                    path: dbPath,
+                    path: filePath,
                     error: error instanceof Error ? error.message : String(error),
                 });
             }
@@ -184,32 +492,112 @@ export class VectorStoreService {
      * @param metadata - Optional metadata to associate with the embedding
      * @param scope - Storage scope ('global' or 'project')
      * @param projectPath - Required when scope is 'project'
-     * @throws Error if embedding is empty or metadata exceeds size limit
+     * @throws Error if id is empty, embedding is invalid, metadata exceeds size,
+     *         or dimensions don't match previously stored embeddings
      */
     upsert(id, embedding, metadata, scope, projectPath) {
         if (!id) {
             throw new Error('id is required');
         }
-        if (!embedding || embedding.length === 0) {
-            throw new Error('embedding must be a non-empty array');
-        }
+        validateEmbedding(embedding);
         const metadataJson = JSON.stringify(metadata);
         if (metadataJson.length > VECTOR_STORE_CONSTANTS.MAX_METADATA_SIZE) {
             throw new Error(`metadata exceeds maximum size of ${VECTOR_STORE_CONSTANTS.MAX_METADATA_SIZE} bytes`);
         }
-        const db = this.getDb(scope, projectPath);
+        const state = this.getDbState(scope, projectPath);
+        // Dimension consistency: all vectors in a scope must share the same width
+        if (state.dimensions && state.dimensions !== embedding.length) {
+            throw new Error(`Dimension mismatch: expected ${state.dimensions}, got ${embedding.length}. ` +
+                'All embeddings in a scope must have the same dimensions.');
+        }
         const now = new Date().toISOString();
-        const embeddingJson = JSON.stringify(embedding);
-        const stmt = db.prepare(`
-      INSERT INTO embeddings (id, embedding, metadata, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?)
-      ON CONFLICT(id) DO UPDATE SET
-        embedding = excluded.embedding,
-        metadata = excluded.metadata,
-        updated_at = excluded.updated_at
-    `);
-        stmt.run(id, embeddingJson, metadataJson, now, now);
-        this.logger.debug('Upserted embedding', { id, scope, dimensions: embedding.length });
+        const blob = embeddingToBlob(embedding);
+        const upsertStmt = this.getStmt(state, 'upsert', `INSERT INTO embeddings (id, embedding, metadata, dimensions, created_at, updated_at)
+       VALUES (?, ?, ?, ?, ?, ?)
+       ON CONFLICT(id) DO UPDATE SET
+         embedding = excluded.embedding,
+         metadata = excluded.metadata,
+         dimensions = excluded.dimensions,
+         updated_at = excluded.updated_at`);
+        upsertStmt.run(id, blob, metadataJson, embedding.length, now, now);
+        // Track dimensions on first insert
+        if (!state.dimensions) {
+            state.dimensions = embedding.length;
+            state.db.prepare('INSERT OR REPLACE INTO vec_config (key, value) VALUES (?, ?)').run('vec_dimensions', String(embedding.length));
+            // Now we know dimensions — create vec0 table if available
+            if (state.vecEnabled) {
+                this.ensureVecTable(state.db, embedding.length);
+            }
+        }
+        // Sync to vec0 table
+        if (state.vecEnabled && state.dimensions === embedding.length) {
+            this.syncToVec(state, id, blob);
+        }
+        this.logger.debug('Upserted embedding', {
+            id,
+            scope,
+            dimensions: embedding.length,
+            vecEnabled: state.vecEnabled,
+        });
+    }
+    /**
+     * Sync a single embedding to the vec0 virtual table.
+     * Uses CAST(? AS INTEGER) for rowid because sqlite-vec's vec0 virtual
+     * table does not accept JS number params directly for primary keys.
+     *
+     * @param state - Database state
+     * @param id - Document identifier
+     * @param blob - Embedding as a Float32Array buffer
+     */
+    syncToVec(state, id, blob) {
+        try {
+            const rowObj = state.db.prepare('SELECT rowid FROM embeddings WHERE id = ?').get(id);
+            if (rowObj) {
+                state.db.prepare('DELETE FROM vec_embeddings WHERE rowid = CAST(? AS INTEGER)').run(rowObj.rowid);
+                state.db.prepare('INSERT INTO vec_embeddings(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)').run(rowObj.rowid, blob);
+            }
+        }
+        catch (err) {
+            this.logger.warn('Failed to sync embedding to vec0', {
+                id,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    /**
+     * Batch upsert multiple embeddings in a single transaction. Significantly
+     * faster than individual upserts for bulk imports.
+     *
+     * @param items - Array of { id, embedding, metadata } objects
+     * @param scope - Storage scope
+     * @param projectPath - Required when scope is 'project'
+     * @returns Count of succeeded and failed inserts
+     * @throws Error if batch size exceeds MAX_BATCH_SIZE
+     */
+    batchUpsert(items, scope, projectPath) {
+        if (items.length > VECTOR_STORE_CONSTANTS.MAX_BATCH_SIZE) {
+            throw new Error(`Batch size ${items.length} exceeds maximum ${VECTOR_STORE_CONSTANTS.MAX_BATCH_SIZE}`);
+        }
+        let succeeded = 0;
+        let failed = 0;
+        const state = this.getDbState(scope, projectPath);
+        const txn = state.db.transaction(() => {
+            for (const item of items) {
+                try {
+                    this.upsert(item.id, item.embedding, item.metadata, scope, projectPath);
+                    succeeded++;
+                }
+                catch (err) {
+                    this.logger.warn('Batch upsert item failed', {
+                        id: item.id,
+                        error: err instanceof Error ? err.message : String(err),
+                    });
+                    failed++;
+                }
+            }
+        });
+        txn();
+        return { succeeded, failed };
     }
     /**
      * Retrieve a stored embedding by document ID.
@@ -220,14 +608,17 @@ export class VectorStoreService {
      * @returns The vector record, or null if not found
      */
     get(id, scope, projectPath) {
-        const db = this.getDb(scope, projectPath);
-        const row = db.prepare('SELECT * FROM embeddings WHERE id = ?').get(id);
+        const state = this.getDbState(scope, projectPath);
+        const stmt = this.getStmt(state, 'get', 'SELECT * FROM embeddings WHERE id = ?');
+        const row = stmt.get(id);
         if (!row) {
             return null;
         }
         return {
             id: row.id,
-            embedding: JSON.parse(row.embedding),
+            embedding: row.embedding instanceof Buffer
+                ? blobToEmbedding(row.embedding)
+                : JSON.parse(row.embedding),
             metadata: JSON.parse(row.metadata),
             createdAt: row.created_at,
             updatedAt: row.updated_at,
@@ -242,8 +633,21 @@ export class VectorStoreService {
      * @returns True if a record was deleted, false if not found
      */
     delete(id, scope, projectPath) {
-        const db = this.getDb(scope, projectPath);
-        const result = db.prepare('DELETE FROM embeddings WHERE id = ?').run(id);
+        const state = this.getDbState(scope, projectPath);
+        // Remove from vec0 first if available
+        if (state.vecEnabled) {
+            try {
+                const rowObj = state.db.prepare('SELECT rowid FROM embeddings WHERE id = ?').get(id);
+                if (rowObj) {
+                    state.db.prepare('DELETE FROM vec_embeddings WHERE rowid = CAST(? AS INTEGER)').run(rowObj.rowid);
+                }
+            }
+            catch {
+                // vec0 delete failure is non-fatal
+            }
+        }
+        const stmt = this.getStmt(state, 'delete', 'DELETE FROM embeddings WHERE id = ?');
+        const result = stmt.run(id);
         return result.changes > 0;
     }
     /**
@@ -255,9 +659,9 @@ export class VectorStoreService {
      * @returns True if the embedding exists
      */
     has(id, scope, projectPath) {
-        const db = this.getDb(scope, projectPath);
-        const row = db.prepare('SELECT 1 FROM embeddings WHERE id = ?').get(id);
-        return row !== undefined;
+        const state = this.getDbState(scope, projectPath);
+        const stmt = this.getStmt(state, 'has', 'SELECT 1 FROM embeddings WHERE id = ?');
+        return stmt.get(id) !== undefined;
     }
     /**
      * Return the total number of stored embeddings.
@@ -267,16 +671,14 @@ export class VectorStoreService {
      * @returns Count of stored embeddings
      */
     count(scope, projectPath) {
-        const db = this.getDb(scope, projectPath);
-        const row = db.prepare('SELECT COUNT(*) as cnt FROM embeddings').get();
-        return row.cnt;
+        const state = this.getDbState(scope, projectPath);
+        const stmt = this.getStmt(state, 'count', 'SELECT COUNT(*) as cnt FROM embeddings');
+        return stmt.get().cnt;
     }
     /**
-     * Search for the most similar embeddings using cosine similarity.
-     *
-     * Loads all embeddings from the store, computes cosine similarity
-     * against the query vector, and returns the top results above the
-     * similarity threshold.
+     * Search for the most similar embeddings. Uses native vec0 KNN search when
+     * sqlite-vec is available, otherwise falls back to JavaScript cosine
+     * similarity.
      *
      * @param queryEmbedding - The query embedding vector
      * @param scope - Storage scope
@@ -289,11 +691,82 @@ export class VectorStoreService {
         if (!queryEmbedding || queryEmbedding.length === 0) {
             return [];
         }
-        const db = this.getDb(scope, projectPath);
-        const rows = db.prepare('SELECT id, embedding, metadata FROM embeddings').all();
+        const state = this.getDbState(scope, projectPath);
+        // Try native vec0 KNN search
+        if (state.vecEnabled && state.dimensions === queryEmbedding.length) {
+            try {
+                return this.vecSearch(state, queryEmbedding, limit, threshold);
+            }
+            catch (err) {
+                this.logger.warn('vec0 search failed, falling back to JS', {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        // Fallback: JavaScript cosine similarity
+        return this.jsFallbackSearch(state, queryEmbedding, limit, threshold);
+    }
+    /**
+     * Native KNN search using sqlite-vec's vec0 virtual table with cosine
+     * distance metric. Results are converted from cosine distance (0 = identical,
+     * 1 = orthogonal) to cosine similarity (1 = identical, 0 = orthogonal).
+     *
+     * @param state - Database state
+     * @param queryEmbedding - The query vector
+     * @param limit - Max results
+     * @param threshold - Minimum cosine similarity
+     * @returns Sorted search results
+     */
+    vecSearch(state, queryEmbedding, limit, threshold) {
+        const queryBlob = embeddingToBlob(queryEmbedding);
+        // Fetch candidates from vec0 (request extra to account for threshold filtering)
+        const vecRows = state.db.prepare(`SELECT rowid, distance
+       FROM vec_embeddings
+       WHERE embedding MATCH ?
+       ORDER BY distance
+       LIMIT ?`).all(queryBlob, limit * 2);
+        if (vecRows.length === 0)
+            return [];
+        // Fetch metadata for matched rowids
+        const rowids = vecRows.map((r) => r.rowid);
+        const placeholders = rowids.map(() => '?').join(',');
+        const metaRows = state.db.prepare(`SELECT rowid, id, metadata FROM embeddings WHERE rowid IN (${placeholders})`).all(...rowids);
+        const metaMap = new Map(metaRows.map((r) => [r.rowid, r]));
+        // Convert cosine distance to cosine similarity: sim = 1 - distance
+        const results = [];
+        for (const vr of vecRows) {
+            const meta = metaMap.get(vr.rowid);
+            if (!meta)
+                continue;
+            const score = 1 - vr.distance;
+            if (score >= threshold) {
+                results.push({
+                    id: meta.id,
+                    score,
+                    metadata: JSON.parse(meta.metadata),
+                });
+            }
+        }
+        results.sort((a, b) => b.score - a.score);
+        return results.slice(0, limit);
+    }
+    /**
+     * JavaScript fallback search using cosine similarity on BLOB embeddings.
+     * Used when sqlite-vec is not available.
+     *
+     * @param state - Database state
+     * @param queryEmbedding - The query vector
+     * @param limit - Max results
+     * @param threshold - Minimum cosine similarity
+     * @returns Sorted search results
+     */
+    jsFallbackSearch(state, queryEmbedding, limit, threshold) {
+        const rows = state.db.prepare('SELECT id, embedding, metadata FROM embeddings').all();
         const results = [];
         for (const row of rows) {
-            const storedEmbedding = JSON.parse(row.embedding);
+            const storedEmbedding = row.embedding instanceof Buffer
+                ? blobToEmbedding(row.embedding)
+                : JSON.parse(row.embedding);
             const score = cosineSimilarity(queryEmbedding, storedEmbedding);
             if (score >= threshold) {
                 results.push({
@@ -307,15 +780,34 @@ export class VectorStoreService {
         return results.slice(0, limit);
     }
     /**
-     * Delete all embeddings from the store.
+     * Delete all embeddings from the store. Also resets the stored dimension
+     * constraint so that subsequent inserts may use a different embedding width.
      *
      * @param scope - Storage scope
      * @param projectPath - Required when scope is 'project'
      * @returns Number of records deleted
      */
     clear(scope, projectPath) {
-        const db = this.getDb(scope, projectPath);
-        const result = db.prepare('DELETE FROM embeddings').run();
+        const state = this.getDbState(scope, projectPath);
+        // Drop vec0 table so it can be recreated with new dimensions
+        if (state.vecEnabled) {
+            try {
+                state.db.exec('DROP TABLE IF EXISTS vec_embeddings');
+            }
+            catch {
+                // Non-fatal
+            }
+        }
+        const stmt = this.getStmt(state, 'clear', 'DELETE FROM embeddings');
+        const result = stmt.run();
+        // Reset dimension tracking — no data means no dimension constraint
+        state.dimensions = null;
+        try {
+            state.db.prepare('DELETE FROM vec_config WHERE key = ?').run('vec_dimensions');
+        }
+        catch {
+            // Non-fatal
+        }
         return result.changes;
     }
 }