@robthepcguy/rag-vault 1.7.0 → 1.8.0

package/dist/vectordb/index.d.ts

@@ -10,12 +10,6 @@ export { DatabaseError } from '../errors/index.js';
  * @returns true if path is safe for use in queries
  */
 export declare function isValidFilePath(filePath: string): boolean;
-/**
- * Generate a content-based fingerprint for a chunk.
- * Uses SHA-256 hash of normalized text (first 16 hex chars for compactness).
- * This enables stable chunk identification across re-indexing.
- */
-export declare function generateChunkFingerprint(text: string): string;
 /**
  * Grouping mode for quality filtering
  * - 'similar': Only return the most similar group (stops at first distance jump)
@@ -25,7 +19,7 @@ export type GroupingMode = 'similar' | 'related';
 /**
  * VectorStore configuration
  */
-export interface VectorStoreConfig {
+interface VectorStoreConfig {
     /** LanceDB database path */
     dbPath: string;
     /** Table name */
@@ -44,7 +38,7 @@ export interface VectorStoreConfig {
 /**
  * Document metadata
  */
-export interface DocumentMetadata {
+interface DocumentMetadata {
     /** File name */
     fileName: string;
     /** File size in bytes */
@@ -78,7 +72,7 @@ export interface VectorChunk {
 /**
  * Search result
  */
-export interface SearchResult {
+interface SearchResult {
     /** File path */
     filePath: string;
     /** Chunk index */
@@ -183,6 +177,15 @@ export declare class VectorStore {
      * @param filePath - File path (absolute)
      */
     deleteChunks(filePath: string): Promise<void>;
+    /**
+     * Delete chunks for a file, excluding a set of IDs.
+     * Used by insert-then-delete re-ingestion to remove old vectors
+     * while keeping newly inserted ones.
+     *
+     * @param filePath - File path whose old chunks should be removed
+     * @param excludeIds - Set of chunk IDs to keep (the new batch)
+     */
+    deleteChunksExcluding(filePath: string, excludeIds: Set<string>): Promise<void>;
     /**
      * Batch insert vector chunks
      *

package/dist/vectordb/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/vectordb/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AA4FlD;;;;GAIG;AACH;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAOzD;AAgBD;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAK7D;AAMD;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,SAAS,CAAA;AAEhD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAA;IAChB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,mEAAmE;IACnE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAA;IACV,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,oDAAoD;IACpD,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,eAAe;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAA;IACjB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,eAAe;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAuGD;;;;;;;;;;;;GAYG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,eAAe,CAAI;IAC3B,OAAO,CAAC,cAAc,CAAsB;IAC5C,2DAA2D;IAC3D,OAAO,CAAC,kBAAkB,CAAgC;IAC1D,qEAAqE;IACrE,OAAO,CAAC,oBAAoB,CAAsB;IAClD,6EAA6E;IAC7E,OAAO,CAAC,oBAAoB,CAA6B;gBAE7C,MAAM,EAAE,iBAAiB;IAIrC;;OAEG;IACH,eAAe,IAAI,MAAM;IAIzB;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAOrC;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAqCxB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAQxB;;;;;;;OAOG;IACH,OAAO,CAAC,qCAAqC;IAU7C;;;;;;OAMG;IACH,OAAO,CAAC,8BAA8B;IAoDtC;;;;;OAKG;YACW,2BAA2B;IA+CzC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA6CjC;;;;OAIG;IACG,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDnD;;;;OAIG;IACG,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA4DxD;;;;OAIG;YACW,cAAc;IAyC5B;;;;OAIG;YACW,eAAe;IAY7B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IA2CrB;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAqE5F;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IA0CzB;;;;;;;OAOG;IACG,SAAS,CAAC,OAAO,CAAC,EAAE;QACxB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;KAChB,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IA2D1E;;;;;;;;;OASG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuC5B;;;;;OAKG;IACG,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAqClE;;;;;;;;OAQG;IACG,iBAAiB,CACrB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,KAAK,SAAI,EACT,mBAAmB,UAAO,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC;IA6E1B;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC;QACzB,aAAa,EAAE,MAAM,CAAA;QACrB,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,MAAM,EAAE,MAAM,CAAA;QACd,eAAe,EAAE,OAAO,CAAA;QACxB,UAAU,EAAE,QAAQ,GAAG,aAAa,CAAA;KACrC,CAAC;CA2CH"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/vectordb/index.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AA0HlD;;;;GAIG;AACH;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAOzD;AAgCD;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,SAAS,CAAA;AAEhD;;GAEG;AACH,UAAU,iBAAiB;IACzB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,YAAY,CAAA;IACvB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,UAAU,gBAAgB;IACxB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAA;IAChB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,mEAAmE;IACnE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,sBAAsB;IACtB,EAAE,EAAE,MAAM,CAAA;IACV,2BAA2B;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,oDAAoD;IACpD,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,eAAe;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAA;IACjB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,UAAU,YAAY;IACpB,gBAAgB;IAChB,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAA;IACb,eAAe;IACf,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAuGD;;;;;;;;;;;;GAYG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,KAAK,CAAqB;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,eAAe,CAAI;IAC3B,OAAO,CAAC,cAAc,CAAsB;IAC5C,2DAA2D;IAC3D,OAAO,CAAC,kBAAkB,CAAgC;IAC1D,qEAAqE;IACrE,OAAO,CAAC,oBAAoB,CAAsB;IAClD,6EAA6E;IAC7E,OAAO,CAAC,oBAAoB,CAA6B;gBAE7C,MAAM,EAAE,iBAAiB;IAIrC;;OAEG;IACH,eAAe,IAAI,MAAM;IAIzB;;;OAGG;IACH,eAAe,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAOrC;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAqCxB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAQxB;;;;;;;OAOG;IACH,OAAO,CAAC,qCAAqC;IAU7C;;;;;;OAMG;IACH,OAAO,CAAC,8BAA8B;IAoDtC;;;;;OAKG;YACW,2BAA2B;IA+CzC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA6CjC;;;;OAIG;IACG,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDnD;;;;;;;OAOG;IACG,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CrF;;;;OAIG;IACG,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA4DxD;;;;OAIG;YACW,cAAc;IAyC5B;;;;OAIG;YACW,eAAe;IAY7B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IA2CrB;;;;;;;;;;;;OAYG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,KAAK,SAAK,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IA6E5F;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB;IA0CzB;;;;;;;OAOG;IACG,SAAS,CAAC,OAAO,CAAC,EAAE;QACxB,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,MAAM,CAAC,EAAE,MAAM,CAAA;KAChB,GAAG,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IA8D1E;;;;;;;;;OASG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAuC5B;;;;;OAKG;IACG,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAsClE;;;;;;;;OAQG;IACG,iBAAiB,CACrB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,KAAK,SAAI,EACT,mBAAmB,UAAO,GACzB,OAAO,CAAC,YAAY,EAAE,CAAC;IAgF1B;;;;OAIG;IACG,SAAS,IAAI,OAAO,CAAC;QACzB,aAAa,EAAE,MAAM,CAAA;QACrB,UAAU,EAAE,MAAM,CAAA;QAClB,WAAW,EAAE,MAAM,CAAA;QACnB,MAAM,EAAE,MAAM,CAAA;QACd,eAAe,EAAE,OAAO,CAAA;QACxB,UAAU,EAAE,QAAQ,GAAG,aAAa,CAAA;KACrC,CAAC;CA2CH"}

package/dist/vectordb/index.js

@@ -1,6 +1,6 @@
 // VectorStore implementation with LanceDB integration
 import { createHash } from 'node:crypto';
-import { Index, connect } from '@lancedb/lancedb';
+import { connect, Index } from '@lancedb/lancedb';
 import { DatabaseError } from '../errors/index.js';
 // Re-export error class for backwards compatibility
 export { DatabaseError } from '../errors/index.js';
@@ -15,7 +15,9 @@ function parseEnvNumber(envVar, defaultValue) {
     if (!value)
         return defaultValue;
     const parsed = Number.parseFloat(value);
-    return Number.isNaN(parsed) ? defaultValue : parsed;
+    if (Number.isNaN(parsed) || !Number.isFinite(parsed))
+        return defaultValue;
+    return parsed;
 }
 /**
  * Parse an integer environment variable with fallback
@@ -25,7 +27,9 @@ function parseEnvInt(envVar, defaultValue) {
     if (!value)
         return defaultValue;
     const parsed = Number.parseInt(value, 10);
-    return Number.isNaN(parsed) ? defaultValue : parsed;
+    if (Number.isNaN(parsed) || !Number.isFinite(parsed))
+        return defaultValue;
+    return parsed;
 }
 /**
  * Standard deviation multiplier for detecting group boundaries.
@@ -81,6 +85,27 @@ const CUSTOM_METADATA_ALL_FIELDS = '__all__';
  * Rejects paths with SQL injection attempts or path traversal.
  */
 const SAFE_PATH_REGEX = /^[a-zA-Z0-9\\/_.:\- ]+$/;
+/**
+ * Retry a read-only async operation with exponential backoff.
+ * Used for transient disk/IO errors on VectorStore reads.
+ */
+async function withRetry(fn, label, maxAttempts = 3, baseDelayMs = 100) {
+    let lastError;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt < maxAttempts) {
+                const delayMs = baseDelayMs * 2 ** (attempt - 1);
+                console.warn(`${label}: attempt ${attempt}/${maxAttempts} failed (${lastError.message}), retrying in ${delayMs}ms...`);
+                await new Promise((resolve) => setTimeout(resolve, delayMs));
+            }
+        }
+    }
+    throw lastError;
+}
 /**
  * Validate file path to prevent SQL injection and path traversal attacks.
  * @param filePath - The file path to validate
@@ -121,7 +146,7 @@ function normalizeTextForFingerprint(text) {
  * Uses SHA-256 hash of normalized text (first 16 hex chars for compactness).
  * This enables stable chunk identification across re-indexing.
  */
-export function generateChunkFingerprint(text) {
+function generateChunkFingerprint(text) {
     const normalized = normalizeTextForFingerprint(text);
     const hash = createHash('sha256').update(normalized, 'utf8').digest('hex');
     // Use first 16 characters (64 bits) - sufficient for practical uniqueness
@@ -507,6 +532,48 @@ export class VectorStore {
             // Ignorable errors (no matching records) are logged but not thrown
         }
     }
+    /**
+     * Delete chunks for a file, excluding a set of IDs.
+     * Used by insert-then-delete re-ingestion to remove old vectors
+     * while keeping newly inserted ones.
+     *
+     * @param filePath - File path whose old chunks should be removed
+     * @param excludeIds - Set of chunk IDs to keep (the new batch)
+     */
+    async deleteChunksExcluding(filePath, excludeIds) {
+        if (!this.table || excludeIds.size === 0) {
+            return;
+        }
+        if (!isValidFilePath(filePath)) {
+            throw new DatabaseError('Invalid file path: contains disallowed characters or patterns');
+        }
+        const escapedFilePath = filePath.replace(/'/g, "''");
+        try {
+            // Query existing chunks for this file to find old IDs
+            const existing = await this.table
+                .query()
+                .where(`\`filePath\` = '${escapedFilePath}'`)
+                .select(['id'])
+                .toArray();
+            const oldIds = existing.map((row) => row.id).filter((id) => !excludeIds.has(id));
+            if (oldIds.length === 0) {
+                return;
+            }
+            // Delete old chunks by ID
+            const idList = oldIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(', ');
+            await this.table.delete(`\`id\` IN (${idList})`);
+            console.error(`VectorStore: Removed ${oldIds.length} old chunks for "${filePath}"`);
+            await this.rebuildFtsIndex();
+        }
+        catch (error) {
+            // Non-fatal: temporary duplicates are acceptable
+            const errorMessage = error.message.toLowerCase();
+            const isIgnorable = DELETE_IGNORABLE_PATTERNS.some((pattern) => errorMessage.includes(pattern));
+            if (!isIgnorable) {
+                throw new DatabaseError(`Failed to clean up old chunks for file: ${filePath}`, error);
+            }
+        }
+    }
     /**
      * Batch insert vector chunks
      *
@@ -688,55 +755,61 @@ export class VectorStore {
         if (limit < 1 || limit > 20) {
             throw new DatabaseError(`Invalid limit: expected 1-20, got ${limit}`);
         }
-        try {
-            // Step 1: Semantic (vector) search - always the primary search
-            const candidateLimit = limit * HYBRID_SEARCH_CANDIDATE_MULTIPLIER;
-            // Assumes normalized embeddings so dot behaves like cosine distance (lower is better, [0,2]).
-            let query = this.table.vectorSearch(queryVector).distanceType('dot').limit(candidateLimit);
-            // Apply distance threshold at query level
-            if (this.config.maxDistance !== undefined) {
-                query = query.distanceRange(undefined, this.config.maxDistance);
-            }
-            const vectorResults = await query.toArray();
-            // Convert to SearchResult format with type validation
-            let results = vectorResults.map((result) => toSearchResult(result));
-            // Step 2: Apply grouping filter on vector distances (before keyword boost)
-            // Grouping is meaningful only on semantic distances, not after keyword boost
-            if (this.config.grouping && results.length > 1) {
-                results = this.applyGrouping(results, this.config.grouping);
-            }
-            // Step 3: Apply keyword boost if enabled (with circuit breaker)
-            const hybridWeight = this.getHybridWeight();
-            if (this.shouldAttemptFts() && queryText && queryText.trim().length > 0 && hybridWeight > 0) {
-                try {
-                    // Get unique filePaths from vector results to filter FTS search
-                    const uniqueFilePaths = [...new Set(results.map((r) => r.filePath))];
-                    // Build WHERE clause with IN for targeted FTS search
-                    // Use backticks for column name (required for camelCase in LanceDB)
-                    const escapedPaths = uniqueFilePaths.map((p) => `'${p.replace(/'/g, "''")}'`);
-                    const whereClause = `\`filePath\` IN (${escapedPaths.join(', ')})`;
-                    const ftsResults = await this.table
-                        .search(queryText, 'fts', 'text')
-                        .where(whereClause)
-                        .select(['filePath', 'chunkIndex', 'text', 'metadata', '_score'])
-                        .limit(results.length * 2) // Enough to cover all vector results
-                        .toArray();
-                    results = this.applyKeywordBoost(results, ftsResults, hybridWeight);
-                    // FTS succeeded - reset circuit breaker
-                    this.recordFtsSuccess();
+        const table = this.table;
+        return withRetry(async () => {
+            try {
+                // Step 1: Semantic (vector) search - always the primary search
+                const candidateLimit = limit * HYBRID_SEARCH_CANDIDATE_MULTIPLIER;
+                // Assumes normalized embeddings so dot behaves like cosine distance (lower is better, [0,2]).
+                let query = table.vectorSearch(queryVector).distanceType('dot').limit(candidateLimit);
+                // Apply distance threshold at query level
+                if (this.config.maxDistance !== undefined) {
+                    query = query.distanceRange(undefined, this.config.maxDistance);
                 }
-                catch (ftsError) {
-                    // Record failure for circuit breaker (will auto-recover after cooldown)
-                    this.recordFtsFailure(ftsError);
-                    // Continue with vector-only results
+                const vectorResults = await query.toArray();
+                // Convert to SearchResult format with type validation
+                let results = vectorResults.map((result) => toSearchResult(result));
+                // Step 2: Apply grouping filter on vector distances (before keyword boost)
+                // Grouping is meaningful only on semantic distances, not after keyword boost
+                if (this.config.grouping && results.length > 1) {
+                    results = this.applyGrouping(results, this.config.grouping);
                 }
+                // Step 3: Apply keyword boost if enabled (with circuit breaker)
+                const hybridWeight = this.getHybridWeight();
+                if (this.shouldAttemptFts() &&
+                    queryText &&
+                    queryText.trim().length > 0 &&
+                    hybridWeight > 0) {
+                    try {
+                        // Get unique filePaths from vector results to filter FTS search
+                        const uniqueFilePaths = [...new Set(results.map((r) => r.filePath))];
+                        // Build WHERE clause with IN for targeted FTS search
+                        // Use backticks for column name (required for camelCase in LanceDB)
+                        const escapedPaths = uniqueFilePaths.map((p) => `'${p.replace(/'/g, "''")}'`);
+                        const whereClause = `\`filePath\` IN (${escapedPaths.join(', ')})`;
+                        const ftsResults = await table
+                            .search(queryText, 'fts', 'text')
+                            .where(whereClause)
+                            .select(['filePath', 'chunkIndex', 'text', 'metadata', '_score'])
+                            .limit(results.length * 2) // Enough to cover all vector results
+                            .toArray();
+                        results = this.applyKeywordBoost(results, ftsResults, hybridWeight);
+                        // FTS succeeded - reset circuit breaker
+                        this.recordFtsSuccess();
+                    }
+                    catch (ftsError) {
+                        // Record failure for circuit breaker (will auto-recover after cooldown)
+                        this.recordFtsFailure(ftsError);
+                        // Continue with vector-only results
+                    }
+                }
+                // Return top results after all filtering and boosting
+                return results.slice(0, limit);
             }
-            // Return top results after all filtering and boosting
-            return results.slice(0, limit);
-        }
-        catch (error) {
-            throw new DatabaseError('Failed to search vectors', error);
-        }
+            catch (error) {
+                throw new DatabaseError('Failed to search vectors', error);
+            }
+        }, 'VectorStore.search');
     }
     /**
      * Apply keyword boost to rerank vector search results
@@ -798,52 +871,55 @@ export class VectorStore {
         if (!this.table) {
             return []; // Return empty array if table doesn't exist
         }
-        try {
-            // Retrieve all records - LanceDB doesn't support GROUP BY aggregation,
-            // so we must fetch records and group in memory
-            // TODO(perf): Consider caching file list or using incremental updates for very large datasets
-            const allRecords = await this.table.query().toArray();
-            // Group by file path
-            const fileMap = new Map();
-            for (const record of allRecords) {
-                const filePath = record.filePath;
-                const timestamp = record.timestamp;
-                if (fileMap.has(filePath)) {
-                    const fileInfo = fileMap.get(filePath);
-                    if (fileInfo) {
-                        fileInfo.chunkCount += 1;
-                        // Keep most recent timestamp
-                        if (timestamp > fileInfo.timestamp) {
-                            fileInfo.timestamp = timestamp;
+        const table = this.table;
+        return withRetry(async () => {
+            try {
+                // Retrieve all records - LanceDB doesn't support GROUP BY aggregation,
+                // so we must fetch records and group in memory
+                // TODO(perf): Consider caching file list or using incremental updates for very large datasets
+                const allRecords = await table.query().toArray();
+                // Group by file path
+                const fileMap = new Map();
+                for (const record of allRecords) {
+                    const filePath = record.filePath;
+                    const timestamp = record.timestamp;
+                    if (fileMap.has(filePath)) {
+                        const fileInfo = fileMap.get(filePath);
+                        if (fileInfo) {
+                            fileInfo.chunkCount += 1;
+                            // Keep most recent timestamp
+                            if (timestamp > fileInfo.timestamp) {
+                                fileInfo.timestamp = timestamp;
+                            }
                         }
                     }
+                    else {
+                        fileMap.set(filePath, { chunkCount: 1, timestamp });
+                    }
                 }
-                else {
-                    fileMap.set(filePath, { chunkCount: 1, timestamp });
+                // Convert Map to array of objects
+                let results = Array.from(fileMap.entries()).map(([filePath, info]) => ({
+                    filePath,
+                    chunkCount: info.chunkCount,
+                    timestamp: info.timestamp,
+                }));
+                // Sort by timestamp descending (most recent first)
+                results.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+                // Apply pagination if provided
+                const offset = options?.offset ?? 0;
+                const limit = options?.limit;
+                if (offset > 0) {
+                    results = results.slice(offset);
                 }
+                if (limit !== undefined && limit > 0) {
+                    results = results.slice(0, limit);
+                }
+                return results;
             }
-            // Convert Map to array of objects
-            let results = Array.from(fileMap.entries()).map(([filePath, info]) => ({
-                filePath,
-                chunkCount: info.chunkCount,
-                timestamp: info.timestamp,
-            }));
-            // Sort by timestamp descending (most recent first)
-            results.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
-            // Apply pagination if provided
-            const offset = options?.offset ?? 0;
-            const limit = options?.limit;
-            if (offset > 0) {
-                results = results.slice(offset);
-            }
-            if (limit !== undefined && limit > 0) {
-                results = results.slice(0, limit);
+            catch (error) {
+                throw new DatabaseError('Failed to list files', error);
             }
-            return results;
-        }
-        catch (error) {
-            throw new DatabaseError('Failed to list files', error);
-        }
+        }, 'VectorStore.listFiles');
     }
     /**
      * Close the database connection and release resources
@@ -902,30 +978,30 @@ export class VectorStore {
         if (!isValidFilePath(filePath)) {
             throw new DatabaseError(`Invalid file path: contains disallowed characters or patterns`);
         }
-        try {
-            const escapedFilePath = filePath.replace(/'/g, "''");
-            const results = await this.table
-                .query()
-                .where(`\`filePath\` = '${escapedFilePath}'`)
-                .toArray();
-            // Convert to SearchResult format and sort by chunkIndex
-            const chunks = results.map((record) => {
-                const text = record.text;
-                return {
-                    filePath: record.filePath,
-                    chunkIndex: record.chunkIndex,
-                    text,
-                    score: 0, // No distance score for direct retrieval
-                    metadata: record.metadata,
-                    // Include fingerprint - generate if not stored (backwards compatibility)
-                    fingerprint: record.fingerprint || generateChunkFingerprint(text),
-                };
-            });
-            return chunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
-        }
-        catch (error) {
-            throw new DatabaseError(`Failed to get document chunks for: ${filePath}`, error);
-        }
+        const table = this.table;
+        return withRetry(async () => {
+            try {
+                const escapedFilePath = filePath.replace(/'/g, "''");
+                const results = await table.query().where(`\`filePath\` = '${escapedFilePath}'`).toArray();
+                // Convert to SearchResult format and sort by chunkIndex
+                const chunks = results.map((record) => {
+                    const text = record.text;
+                    return {
+                        filePath: record.filePath,
+                        chunkIndex: record.chunkIndex,
+                        text,
+                        score: 0, // No distance score for direct retrieval
+                        metadata: record.metadata,
+                        // Include fingerprint - generate if not stored (backwards compatibility)
+                        fingerprint: record.fingerprint || generateChunkFingerprint(text),
+                    };
+                });
+                return chunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
+            }
+            catch (error) {
+                throw new DatabaseError(`Failed to get document chunks for: ${filePath}`, error);
+            }
+        }, 'VectorStore.getDocumentChunks');
     }
     /**
      * Find related chunks using a chunk's stored embedding
@@ -944,59 +1020,62 @@ export class VectorStore {
         if (!isValidFilePath(filePath)) {
             throw new DatabaseError(`Invalid file path: contains disallowed characters or patterns`);
         }
-        try {
-            // First, fetch the source chunk to get its vector
-            const escapedFilePath = filePath.replace(/'/g, "''");
-            const sourceResults = await this.table
-                .query()
-                .where(`\`filePath\` = '${escapedFilePath}' AND \`chunkIndex\` = ${chunkIndex}`)
-                .toArray();
-            if (sourceResults.length === 0) {
-                return [];
-            }
-            const sourceChunk = sourceResults[0];
-            const rawVector = sourceChunk?.vector;
-            // LanceDB may return vectors as Arrow Vector or Float32Array, not plain Array
-            // Convert to number[] for compatibility
-            let sourceVector;
-            if (rawVector) {
-                if (Array.isArray(rawVector)) {
-                    sourceVector = rawVector;
+        const table = this.table;
+        return withRetry(async () => {
+            try {
+                // First, fetch the source chunk to get its vector
+                const escapedFilePath = filePath.replace(/'/g, "''");
+                const sourceResults = await table
+                    .query()
+                    .where(`\`filePath\` = '${escapedFilePath}' AND \`chunkIndex\` = ${chunkIndex}`)
+                    .toArray();
+                if (sourceResults.length === 0) {
+                    return [];
                 }
-                else if (typeof rawVector === 'object' && 'length' in rawVector) {
-                    // Handle Arrow Vector, Float32Array, or other array-like objects
-                    sourceVector = Array.from(rawVector);
+                const sourceChunk = sourceResults[0];
+                const rawVector = sourceChunk?.vector;
+                // LanceDB may return vectors as Arrow Vector or Float32Array, not plain Array
+                // Convert to number[] for compatibility
+                let sourceVector;
+                if (rawVector) {
+                    if (Array.isArray(rawVector)) {
+                        sourceVector = rawVector;
+                    }
+                    else if (typeof rawVector === 'object' && 'length' in rawVector) {
+                        // Handle Arrow Vector, Float32Array, or other array-like objects
+                        sourceVector = Array.from(rawVector);
+                    }
                 }
+                if (!sourceVector || sourceVector.length === 0) {
+                    // Chunk exists but has no embedding (e.g., upload timed out mid-process)
+                    // Return empty results instead of throwing - allows batch operations to continue
+                    console.warn(`Chunk ${filePath}:${chunkIndex} has no valid vector (possibly corrupted)`);
+                    return [];
+                }
+                // Search for similar chunks using the source vector
+                // Request more candidates to allow for filtering
+                const candidateLimit = excludeSameDocument ? limit * 3 : limit + 1;
+                let query = table.vectorSearch(sourceVector).distanceType('dot').limit(candidateLimit);
+                // Apply distance threshold if configured
+                if (this.config.maxDistance !== undefined) {
+                    query = query.distanceRange(undefined, this.config.maxDistance);
+                }
+                const vectorResults = await query.toArray();
+                // Convert to SearchResult format with type validation
+                let results = vectorResults.map((result) => toSearchResult(result));
+                // Filter out the source chunk itself
+                results = results.filter((r) => !(r.filePath === filePath && r.chunkIndex === chunkIndex));
+                // Optionally filter out same-document chunks
+                if (excludeSameDocument) {
+                    results = results.filter((r) => r.filePath !== filePath);
+                }
+                return results.slice(0, limit);
             }
-            if (!sourceVector || sourceVector.length === 0) {
-                // Chunk exists but has no embedding (e.g., upload timed out mid-process)
-                // Return empty results instead of throwing - allows batch operations to continue
-                console.warn(`Chunk ${filePath}:${chunkIndex} has no valid vector (possibly corrupted)`);
-                return [];
-            }
-            // Search for similar chunks using the source vector
-            // Request more candidates to allow for filtering
-            const candidateLimit = excludeSameDocument ? limit * 3 : limit + 1;
-            let query = this.table.vectorSearch(sourceVector).distanceType('dot').limit(candidateLimit);
-            // Apply distance threshold if configured
-            if (this.config.maxDistance !== undefined) {
-                query = query.distanceRange(undefined, this.config.maxDistance);
-            }
-            const vectorResults = await query.toArray();
-            // Convert to SearchResult format with type validation
-            let results = vectorResults.map((result) => toSearchResult(result));
-            // Filter out the source chunk itself
-            results = results.filter((r) => !(r.filePath === filePath && r.chunkIndex === chunkIndex));
-            // Optionally filter out same-document chunks
-            if (excludeSameDocument) {
-                results = results.filter((r) => r.filePath !== filePath);
+            catch (error) {
+                const cause = error instanceof Error ? error.message : String(error);
+                throw new DatabaseError(`Failed to find related chunks for: ${filePath}:${chunkIndex}: ${cause}`, error);
             }
-            return results.slice(0, limit);
-        }
-        catch (error) {
-            const cause = error instanceof Error ? error.message : String(error);
-            throw new DatabaseError(`Failed to find related chunks for: ${filePath}:${chunkIndex}: ${cause}`, error);
-        }
+        }, 'VectorStore.findRelatedChunks');
     }
     /**
      * Get system status