@massa-ai/core 1.2.1

package/dist/data/vector/hybrid-search.js

@@ -0,0 +1,238 @@
+/**
+ * Hybrid Search
+ *
+ * Combines vector search (semantic) with keyword search (lexical)
+ * using Reciprocal Rank Fusion (RRF) for reranking
+ */
+import { getVectorStore } from './vector-store-factory.js';
+import { KeywordSearchPg } from '../keyword/keyword-search-pg.js';
+import { logger } from '@massa-ai/shared';
+/**
+ * Reciprocal Rank Fusion constant
+ * Higher k = more smoothing (less impact of exact rank position)
+ */
+const RRF_K = 60;
+/**
+ * Hybrid Search implementation
+ */
+export class HybridSearch {
+    vectorStore;
+    keywordSearch;
+    constructor() {
+        this.vectorStore = getVectorStore();
+        this.keywordSearch = new KeywordSearchPg();
+        logger.info('Hybrid Search initialized');
+    }
+    /**
+     * Search using both vector and keyword search
+     */
+    async search(query, options) {
+        const maxResults = options.maxResults || 10;
+        const minScore = options.minScore || 0;
+        try {
+            logger.debug('Starting hybrid search', {
+                query: query.slice(0, 50),
+                maxResults
+            });
+            // Execute searches in parallel (2x results to have buffer for fusion)
+            const [vectorResults, keywordResults] = await Promise.all([
+                (await this.vectorStore).search(query, maxResults * 2),
+                this.keywordSearch.search(query, maxResults * 2)
+            ]);
+            logger.debug('Search results retrieved', {
+                vectorCount: vectorResults.length,
+                keywordCount: keywordResults.length
+            });
+            // Combine and rerank using RRF
+            const fusedResults = this.rerank([vectorResults, keywordResults]);
+            // Filter by minimum score and limit
+            const filtered = fusedResults
+                .filter(result => result.score >= minScore)
+                .slice(0, maxResults);
+            logger.info('Hybrid search completed', {
+                totalResults: filtered.length,
+                avgScore: this.calculateAvgScore(filtered)
+            });
+            return filtered;
+        }
+        catch (error) {
+            logger.error('Hybrid search failed', error, { query });
+            return [];
+        }
+    }
+    /**
+     * Rerank results using Reciprocal Rank Fusion (RRF)
+     *
+     * RRF formula: score(d) = Σ 1 / (k + rank(d))
+     * where k is a constant (typically 60) and rank is the position in each list
+     */
+    rerank(resultSets) {
+        const scoreMap = new Map();
+        // Calculate RRF score for each result
+        for (const results of resultSets) {
+            results.forEach((result, rank) => {
+                const rrfScore = 1 / (RRF_K + rank + 1); // +1 because rank is 0-indexed
+                if (scoreMap.has(result.id)) {
+                    // Result appears in multiple lists - accumulate scores
+                    const existing = scoreMap.get(result.id);
+                    existing.rrfScore += rrfScore;
+                }
+                else {
+                    // First time seeing this result
+                    scoreMap.set(result.id, {
+                        result: { ...result },
+                        rrfScore
+                    });
+                }
+            });
+        }
+        // Sort entries by RRF score descending
+        const sorted = Array.from(scoreMap.values()).sort((a, b) => b.rrfScore - a.rrfScore);
+        // Normalize all scores in a single pass using the observed max
+        const normalizedMap = this.normalizeRRFScores(sorted.map(({ result, rrfScore }) => ({ id: result.id, rrfScore })));
+        const reranked = sorted.map(({ result }) => ({
+            ...result,
+            score: normalizedMap.get(result.id) ?? 0,
+        }));
+        logger.debug('Results reranked using RRF', {
+            uniqueResults: reranked.length,
+            topScore: reranked[0]?.score
+        });
+        return reranked;
+    }
+    /**
+     * Normalize RRF scores to 0-1 range using the actual max in the result set.
+     *
+     * The theoretical max per result per list is 1/(k+1). With N lists the
+     * absolute ceiling is N/(k+1). But actual maxima vary, so we use the
+     * observed max to give the best-ranked result a score of 1 and scale
+     * everything else relative to it. Falls back to the theoretical ceiling
+     * when all scores are equal (e.g. single-result sets).
+     */
+    normalizeRRFScores(values) {
+        if (values.length === 0)
+            return new Map();
+        const maxRRF = Math.max(...values.map((v) => v.rrfScore));
+        // Theoretical max for k=60 with 2 lists
+        const theoreticalMax = 2 / (RRF_K + 1);
+        const divisor = maxRRF > 0 ? maxRRF : theoreticalMax;
+        return new Map(values.map((v) => [v.id, Math.min(1, v.rrfScore / divisor)]));
+    }
+    /**
+     * Calculate average score
+     */
+    calculateAvgScore(results) {
+        if (results.length === 0)
+            return 0;
+        const sum = results.reduce((acc, r) => acc + r.score, 0);
+        return sum / results.length;
+    }
+    /**
+     * Search with custom weights for vector vs keyword
+     */
+    async searchWeighted(query, options, weights = { vector: 0.7, keyword: 0.3 }) {
+        const maxResults = options.maxResults || 10;
+        try {
+            // Get results from both sources
+            const [vectorResults, keywordResults] = await Promise.all([
+                (await this.vectorStore).search(query, maxResults * 2),
+                this.keywordSearch.search(query, maxResults * 2)
+            ]);
+            // Apply weights to scores
+            const weightedVector = vectorResults.map(r => ({
+                ...r,
+                score: r.score * weights.vector
+            }));
+            const weightedKeyword = keywordResults.map(r => ({
+                ...r,
+                score: r.score * weights.keyword
+            }));
+            // Combine by ID, summing weighted scores
+            const combined = new Map();
+            for (const result of [...weightedVector, ...weightedKeyword]) {
+                if (combined.has(result.id)) {
+                    const existing = combined.get(result.id);
+                    existing.score += result.score;
+                }
+                else {
+                    combined.set(result.id, { ...result });
+                }
+            }
+            // Sort and limit
+            const sorted = Array.from(combined.values())
+                .sort((a, b) => b.score - a.score)
+                .slice(0, maxResults);
+            logger.debug('Weighted hybrid search completed', {
+                weights,
+                resultsCount: sorted.length
+            });
+            return sorted;
+        }
+        catch (error) {
+            logger.error('Weighted hybrid search failed', error);
+            return [];
+        }
+    }
+    /**
+     * Search with boost for specific sources
+     */
+    async searchWithBoost(query, options, boosts = {}) {
+        const results = await this.search(query, options);
+        // Apply boosts based on metadata
+        const boosted = results.map(result => {
+            let boostFactor = 1.0;
+            // Recency boost (newer items score higher)
+            if (boosts.recency && result.metadata?.createdAt) {
+                const createdAt = result.metadata.createdAt;
+                const ageInDays = (Date.now() - new Date(createdAt).getTime()) / (1000 * 60 * 60 * 24);
+                boostFactor *= Math.max(0.5, 1 - (ageInDays / 365) * boosts.recency);
+            }
+            // Popularity boost (frequently accessed items score higher)
+            if (boosts.popularity && result.metadata?.accessCount) {
+                const accessCount = result.metadata.accessCount;
+                boostFactor *= 1 + (Math.log(accessCount + 1) / 10) * boosts.popularity;
+            }
+            return {
+                ...result,
+                score: Math.min(1, result.score * boostFactor)
+            };
+        });
+        // Re-sort after boosting
+        return boosted.sort((a, b) => b.score - a.score);
+    }
+    /**
+     * Diversify results to reduce redundancy
+     */
+    diversifyResults(results, maxSimilarityThreshold = 0.85) {
+        if (results.length === 0)
+            return results;
+        const diversified = [results[0]]; // Always include top result
+        for (let i = 1; i < results.length; i++) {
+            const candidate = results[i];
+            // Check if candidate is too similar to any already selected result
+            const isTooSimilar = diversified.some(selected => {
+                return this.calculateContentSimilarity(selected.content, candidate.content)
+                    > maxSimilarityThreshold;
+            });
+            if (!isTooSimilar) {
+                diversified.push(candidate);
+            }
+        }
+        logger.debug('Results diversified', {
+            original: results.length,
+            diversified: diversified.length
+        });
+        return diversified;
+    }
+    /**
+     * Calculate simple content similarity (Jaccard similarity of words)
+     */
+    calculateContentSimilarity(content1, content2) {
+        const words1 = new Set(content1.toLowerCase().split(/\s+/));
+        const words2 = new Set(content2.toLowerCase().split(/\s+/));
+        const intersection = new Set([...words1].filter(w => words2.has(w)));
+        const union = new Set([...words1, ...words2]);
+        return intersection.size / union.size;
+    }
+}
+//# sourceMappingURL=hybrid-search.js.map

package/dist/data/vector/hybrid-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-search.js","sourceRoot":"","sources":["../../../src/data/vector/hybrid-search.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAE1C;;;GAGG;AACH,MAAM,KAAK,GAAG,EAAE,CAAC;AAEjB;;GAEG;AACH,MAAM,OAAO,YAAY;IACf,WAAW,CAAwB;IACnC,aAAa,CAAkB;IAEvC;QACE,IAAI,CAAC,WAAW,GAAG,cAAc,EAAE,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,IAAI,eAAe,EAAE,CAAC;QAE3C,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAyB;QACnD,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;QAEvC,IAAI,CAAC;YACH,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;gBACrC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;gBACzB,UAAU;aACX,CAAC,CAAC;YAEH,sEAAsE;YACtE,MAAM,CAAC,aAAa,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACxD,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC;gBACtD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC;aACjD,CAAC,CAAC;YAEH,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;gBACvC,WAAW,EAAE,aAAa,CAAC,MAAM;gBACjC,YAAY,EAAE,cAAc,CAAC,MAAM;aACpC,CAAC,CAAC;YAEH,+BAA+B;YAC/B,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC;YAElE,oCAAoC;YACpC,MAAM,QAAQ,GAAG,YAAY;iBAC1B,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC;iBAC1C,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YAExB,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;gBACrC,YAAY,EAAE,QAAQ,CAAC,MAAM;gBAC7B,QAAQ,EAAE,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC;aAC3C,CAAC,CAAC;YAEH,OAAO,QAAQ,CAAC;QAElB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAc,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,UAA4B;QACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAsD,CAAC;QAE/E,sCAAsC;QACtC,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;YACjC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE;gBAC/B,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,+BAA+B;gBAExE,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC5B,uDAAuD;oBACvD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC;oBAC1C,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC;gBAChC,CAAC;qBAAM,CAAC;oBACN,gCAAgC;oBAChC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE;wBACtB,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE;wBACrB,QAAQ;qBACT,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,uCAAuC;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAC/C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAClC,CAAC;QAEF,+DAA+D;QAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CACpE,CAAC;QAEF,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3C,GAAG,MAAM;YACT,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC;SACzC,CAAC,CAAC,CAAC;QAEJ,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE;YACzC,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,KAAK;SAC7B,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;;;;;;OAQG;IACK,kBAAkB,CAAC,MAA0C;QACnE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,GAAG,EAAE,CAAC;QAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1D,wCAAwC;QACxC,MAAM,cAAc,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC;QAErD,OAAO,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,OAAuB;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACzD,OAAO,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,KAAa,EACb,OAAyB,EACzB,UAA+C,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE;QAE5E,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QAE5C,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,CAAC,aAAa,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gBACxD,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC;gBACtD,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC;aACjD,CAAC,CAAC;YAEH,0BAA0B;YAC1B,MAAM,cAAc,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7C,GAAG,CAAC;gBACJ,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM;aAChC,CAAC,CAAC,CAAC;YAEJ,MAAM,eAAe,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC/C,GAAG,CAAC;gBACJ,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO;aACjC,CAAC,CAAC,CAAC;YAEJ,yCAAyC;YACzC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;YAEjD,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,cAAc,EAAE,GAAG,eAAe,CAAC,EAAE,CAAC;gBAC7D,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;oBAC5B,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAE,CAAC;oBAC1C,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC;gBACjC,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;YAED,iBAAiB;YACjB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;iBACzC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;iBACjC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;YAExB,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;gBAC/C,OAAO;gBACP,YAAY,EAAE,MAAM,CAAC,MAAM;aAC5B,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAEhB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAc,CAAC,CAAC;YAC9D,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CACnB,KAAa,EACb,OAAyB,EACzB,SAAoD,EAAE;QAEtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAElD,iCAAiC;QACjC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YACnC,IAAI,WAAW,GAAG,GAAG,CAAC;YAEtB,2CAA2C;YAC3C,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;gBACjD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAgB,CAAC;gBACnD,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;gBACvF,WAAW,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;YACvE,CAAC;YAED,4DAA4D;YAC5D,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC;gBACtD,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAqB,CAAC;gBAC1D,WAAW,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC;YAC1E,CAAC;YAED,OAAO;gBACL,GAAG,MAAM;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,GAAG,WAAW,CAAC;aAC/C,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,yBAAyB;QACzB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,gBAAgB,CACd,OAAuB,EACvB,yBAAiC,IAAI;QAErC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,OAAO,CAAC;QAEzC,MAAM,WAAW,GAAmB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,4BAA4B;QAE9E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAE7B,mEAAmE;YACnE,MAAM,YAAY,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAC/C,OAAO,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC;sBACvE,sBAAsB,CAAC;YAC7B,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;YAClC,QAAQ,EAAE,OAAO,CAAC,MAAM;YACxB,WAAW,EAAE,WAAW,CAAC,MAAM;SAChC,CAAC,CAAC;QAEH,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACK,0BAA0B,CAAC,QAAgB,EAAE,QAAgB;QACnE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QAE5D,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;QAE9C,OAAO,YAAY,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxC,CAAC;CACF"}

package/dist/data/vector/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Vector Search Components Export
+ */
+export { HybridSearch } from './hybrid-search.js';
+export { getVectorStore, resetVectorStore } from './vector-store-factory.js';
+export type { VectorStoreConfig } from './vector-store-factory.js';
+export type { PostgresVectorStore, PostgresConfig } from './postgres-vector-store.js';
+export { BaseVectorStore } from './base-vector-store.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/data/vector/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/data/vector/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7E,YAAY,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAEnE,YAAY,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAEtF,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/data/vector/index.js

@@ -0,0 +1,7 @@
+/**
+ * Vector Search Components Export
+ */
+export { HybridSearch } from './hybrid-search.js';
+export { getVectorStore, resetVectorStore } from './vector-store-factory.js';
+export { BaseVectorStore } from './base-vector-store.js';
+//# sourceMappingURL=index.js.map

package/dist/data/vector/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/data/vector/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAK7E,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/data/vector/postgres-vector-store.d.ts

@@ -0,0 +1,118 @@
+/**
+ * PostgreSQL + pgvector Vector Store
+ *
+ * High-performance vector store using PostgreSQL with pgvector extension.
+ *
+ * Index strategy (per embedding dimension):
+ *   ≤ 2000 dims  →  HNSW on vector column (vector_cosine_ops)   O(log n)
+ *   > 2000 dims  →  Two-phase binary quantization search:
+ *                     Phase 1: ANN on bit column (bit_hamming_ops, HNSW)  → candidates
+ *                     Phase 2: exact cosine re-rank on vector column       → top-k
+ *
+ * All write/delete/list operations use raw SQL to avoid the Prisma 7.7.0 + Bun ORM bug
+ * (isObjectEnumValue is not a function).
+ */
+import { BaseVectorStore } from './base-vector-store.js';
+import { IVectorCollection, VectorDocument, VectorStoreStats, ProjectInfo, SearchResult } from '@massa-ai/shared';
+import type { Pool } from 'pg';
+export interface PostgresConfig {
+    connectionString: string;
+    poolSize?: number;
+    indexType?: 'hnsw' | 'ivfflat';
+    indexParams?: {
+        m?: number;
+        efConstruction?: number;
+        lists?: number;
+    };
+}
+export declare class PostgresVectorStore extends BaseVectorStore {
+    private pool;
+    private config;
+    private initialized;
+    private schemaDimensions;
+    private tableName;
+    /** True when the table has embedding_bq column + HNSW index. Set during init. */
+    private bqEnabled;
+    constructor(config: PostgresConfig);
+    private getTableName;
+    /**
+     * Clamp a raw pgvector cosine-similarity (`1 - (embedding <=> query)`) into
+     * the [0,1] relevance contract exposed by SearchResult.score. pgvector
+     * returns NULL — which parseFloats to NaN — for rows whose embedding is
+     * missing or zero-norm, and cosine similarity can dip below 0 for
+     * anti-correlated vectors. Both are normalized to 0 so every search result
+     * always carries a finite, in-range score.
+     */
+    private normalizeScore;
+    /**
+     * Convert float32 embedding to a bit-string for binary quantization.
+     * Positive (≥ 0) → '1', Negative (< 0) → '0'.
+     * The result is passed to PostgreSQL as `$1::bit(N)`.
+     */
+    private floatsToBit;
+    ensureInitialized(): Promise<Pool>;
+    /**
+     * Detect chunks indexed under a different dimension than the current provider.
+     *
+     * When the user changes EMBEDDING_PROVIDER or the embedding model, the new
+     * provider's dims may not match existing rows (which are stored in
+     * vector_documents_<oldDim>d). Search then silently hits an empty table for
+     * affected projects. We warn so the user knows a reindex is required.
+     */
+    private detectOrphanedChunks;
+    private createFallbackTable;
+    private createVectorIndex;
+    /**
+     * Create HNSW index on the binary-quantized bit column.
+     * bit_hamming_ops supports any dimension (no pgvector limit).
+     */
+    private createBqIndex;
+    addDocument(id: string, content: string, metadata?: Record<string, unknown>): Promise<void>;
+    /**
+     * Insert documents, embedding them in sub-batches to avoid overwhelming
+     * the embedding backend. Each sub-batch is its own short transaction
+     * (matches PostgresVectorStore semantics — partial progress survives an
+     * Ollama crash mid-file, instead of rolling back the whole file).
+     * Fails-open per-document when a whole sub-batch's embed call errors.
+     */
+    addDocuments(documents: VectorDocument[]): Promise<void>;
+    /** Insert a pre-embedded sub-batch inside a single short transaction. */
+    private insertSubBatch;
+    search(query: string, limit?: number, projectId?: string): Promise<SearchResult[]>;
+    searchByEmbedding(embedding: number[], limit?: number, projectId?: string): Promise<SearchResult[]>;
+    /**
+     * Direct cosine similarity search (for dims ≤ 2000 with HNSW index).
+     */
+    private searchDirect;
+    /**
+     * Two-phase binary-quantization search (for dims > 2000).
+     *
+     * Phase 1: ANN using Hamming distance on bit column → `candidates` rows
+     * Phase 2: exact cosine re-rank on float vector → top `limit` rows
+     */
+    private searchTwoPhase;
+    delete(id: string): Promise<boolean>;
+    deleteByProject(projectId: string): Promise<number>;
+    update(id: string, content: string, metadata?: Record<string, unknown>): Promise<void>;
+    getStats(projectId?: string): Promise<VectorStoreStats>;
+    listProjects(): Promise<ProjectInfo[]>;
+    /**
+     * Get or create a collection (for IVectorStore interface compatibility).
+     *
+     * Mirrors PostgresVectorStore.getCollection semantics: returns a handle bound
+     * to this store's pool/table that scopes read/write by `name` (the projectId).
+     * IndexManager uses this for _metadata document lookup/persistence.
+     */
+    getCollection(name: string): Promise<IVectorCollection>;
+    healthCheck(): Promise<boolean>;
+    close(): Promise<void>;
+    /** Embedding dimensions declared by the table schema, or null if unknown. */
+    getSchemaDimensions(): number | null;
+    /** True when the table has the embedding_bq column + index. */
+    isBqEnabled(): boolean;
+    /** Encode a float embedding to a pgvector bit-string (binary quantization). */
+    toBitString(embedding: number[]): string;
+    /** Batch-embed texts via the shared provider (exposed for the collection). */
+    embedBatchPublic(contents: string[]): Promise<number[][]>;
+}
+//# sourceMappingURL=postgres-vector-store.d.ts.map

package/dist/data/vector/postgres-vector-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"postgres-vector-store.d.ts","sourceRoot":"","sources":["../../../src/data/vector/postgres-vector-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,YAAY,EAEb,MAAM,kBAAkB,CAAC;AAG1B,OAAO,KAAK,EAAE,IAAI,EAAc,MAAM,IAAI,CAAC;AAE3C,MAAM,WAAW,cAAc;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,CAAC,EAAE;QACZ,CAAC,CAAC,EAAE,MAAM,CAAC;QACX,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,qBAAa,mBAAoB,SAAQ,eAAe;IACtD,OAAO,CAAC,IAAI,CAAqB;IACjC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,SAAS,CAA8B;IAC/C,iFAAiF;IACjF,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,cAAc;IAalC,OAAO,CAAC,YAAY;IAIpB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAQtB;;;;OAIG;IACH,OAAO,CAAC,WAAW;IAMb,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2DxC;;;;;;;OAOG;YACW,oBAAoB;YA2CpB,mBAAmB;YAkCnB,iBAAiB;IA+C/B;;;OAGG;YACW,aAAa;IAoDrB,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAwCjG;;;;;;OAMG;IACG,YAAY,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA6D9D,yEAAyE;YAC3D,cAAc;IA4DtB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAKtF,iBAAiB,CACrB,SAAS,EAAE,MAAM,EAAE,EACnB,KAAK,GAAE,MAAW,EAClB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,YAAY,EAAE,CAAC;IAgB1B;;OAEG;YACW,YAAY;IA6B1B;;;;;OAKG;YACW,cAAc;IAqDtB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IASpC,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IASnD,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAgCvD,YAAY,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAyB5C;;;;;;OAMG;IACG,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAKvD,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAU/B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B,6EAA6E;IAC7E,mBAAmB,IAAI,MAAM,GAAG,IAAI;IAIpC,+DAA+D;IAC/D,WAAW,IAAI,OAAO;IAItB,+EAA+E;IAC/E,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM;IAIxC,8EAA8E;IACxE,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGhE"}