@psiclawops/hypermem 0.1.0

package/dist/hybrid-retrieval.js

@@ -0,0 +1,340 @@
+/**
+ * HyperMem Hybrid Retrieval — FTS5 + KNN Score Fusion
+ *
+ * Merges keyword (FTS5/BM25) and semantic (KNN/vector) results into a
+ * single ranked list using Reciprocal Rank Fusion (RRF). This avoids
+ * vocabulary mismatch (KNN-only misses exact terms) and semantic gap
+ * (FTS5-only misses paraphrases).
+ *
+ * Architecture:
+ *   - FTS5 results from library.db (facts_fts, knowledge_fts, episodes_fts)
+ *   - KNN results from vectors.db via VectorStore
+ *   - RRF merges both ranked lists with configurable k constant
+ *   - Deduplication by (sourceTable, sourceId)
+ *   - Token-budgeted output for compositor consumption
+ */
+// ─── FTS5 Query Building ───────────────────────────────────────
+/** Stop words to exclude from FTS5 queries */
+const STOP_WORDS = new Set([
+    'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+    'should', 'may', 'might', 'shall', 'can', 'need', 'dare', 'ought',
+    'used', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from',
+    'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below',
+    'between', 'out', 'off', 'over', 'under', 'again', 'further', 'then',
+    'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each',
+    'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'no',
+    'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
+    'just', 'don', 'now', 'and', 'but', 'or', 'if', 'it', 'its', 'this',
+    'that', 'these', 'those', 'i', 'me', 'my', 'we', 'our', 'you', 'your',
+    'he', 'him', 'his', 'she', 'her', 'they', 'them', 'their', 'what',
+    'which', 'who', 'whom', 'about', 'up',
+]);
+/**
+ * Build an FTS5 query from a natural language string.
+ * Extracts meaningful words, removes stop words, uses OR conjunction.
+ */
+export function buildFtsQuery(input) {
+    const words = input
+        .toLowerCase()
+        .replace(/[^\w\s-]/g, ' ') // strip punctuation except hyphens
+        .split(/\s+/)
+        .filter(w => w.length >= 3 && !STOP_WORDS.has(w));
+    if (words.length === 0)
+        return '';
+    // Deduplicate, sort by length descending (more specific terms first),
+    // cap at 8 terms to keep queries reasonable
+    const unique = [...new Set(words)]
+        .sort((a, b) => b.length - a.length)
+        .slice(0, 8);
+    // Use prefix matching (*) and OR so any term can match
+    return unique.map(w => `"${w}"*`).join(' OR ');
+}
+/**
+ * Search facts via FTS5.
+ */
+function searchFactsFts(db, query, agentId, limit = 20) {
+    // Two-phase query: FTS runs first in subquery (fast), then filters on
+    // the small result set.  Joining FTS + non-FTS predicates + ORDER BY rank
+    // in one pass forces SQLite to materialise the full FTS match set before
+    // applying LIMIT — O(matches) instead of O(limit).  See data-access-bench.
+    const innerLimit = agentId ? limit * 4 : limit; // over-fetch to survive filter
+    let sql = `
+    SELECT f.id, sub.rank, f.content, f.domain, f.agent_id
+    FROM (
+      SELECT rowid, rank FROM facts_fts WHERE facts_fts MATCH ? ORDER BY rank LIMIT ?
+    ) sub
+    JOIN facts f ON f.id = sub.rowid
+    WHERE f.superseded_by IS NULL
+    AND f.decay_score < 0.8
+  `;
+    const params = [query, innerLimit];
+    if (agentId) {
+        sql += ' AND f.agent_id = ?';
+        params.push(agentId);
+    }
+    sql += ' ORDER BY sub.rank LIMIT ?';
+    params.push(limit);
+    const rows = db.prepare(sql).all(...params);
+    return rows.map(r => ({
+        id: r.id,
+        rank: r.rank,
+        content: r.content,
+        domain: r.domain,
+        agentId: r.agent_id,
+    }));
+}
+/**
+ * Search knowledge via FTS5.
+ */
+function searchKnowledgeFts(db, query, agentId, limit = 20) {
+    const innerLimit = agentId ? limit * 4 : limit;
+    let sql = `
+    SELECT k.id, sub.rank, k.content, k.domain, k.agent_id, k.key
+    FROM (
+      SELECT rowid, rank FROM knowledge_fts WHERE knowledge_fts MATCH ? ORDER BY rank LIMIT ?
+    ) sub
+    JOIN knowledge k ON k.id = sub.rowid
+    WHERE k.superseded_by IS NULL
+  `;
+    const params = [query, innerLimit];
+    if (agentId) {
+        sql += ' AND k.agent_id = ?';
+        params.push(agentId);
+    }
+    sql += ' ORDER BY sub.rank LIMIT ?';
+    params.push(limit);
+    const rows = db.prepare(sql).all(...params);
+    return rows.map(r => ({
+        id: r.id,
+        rank: r.rank,
+        content: r.content,
+        domain: r.domain,
+        agentId: r.agent_id,
+        metadata: r.key,
+    }));
+}
+/**
+ * Search episodes via FTS5.
+ */
+function searchEpisodesFts(db, query, agentId, limit = 20) {
+    let sql;
+    let params;
+    if (agentId) {
+        // Agent-scoped: use WHERE IN (FTS5 subquery) instead of FTS5→JOIN→filter.
+        // SQLite uses the agent_id index to narrow first, then checks FTS5 membership.
+        // Benchmarked: 2.3ms avg vs 8.5ms avg for the post-join approach (13k+ episodes).
+        sql = `
+      SELECT e.id, 0 as rank, e.summary, e.event_type, e.agent_id, e.participants
+      FROM episodes e
+      WHERE e.agent_id = ?
+        AND e.decay_score < 0.8
+        AND e.id IN (SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH ?)
+      ORDER BY e.created_at DESC
+      LIMIT ?
+    `;
+        params = [agentId, query, limit];
+    }
+    else {
+        sql = `
+      SELECT e.id, sub.rank, e.summary, e.event_type, e.agent_id, e.participants
+      FROM (
+        SELECT rowid, rank FROM episodes_fts WHERE episodes_fts MATCH ? ORDER BY rank LIMIT ?
+      ) sub
+      JOIN episodes e ON e.id = sub.rowid
+      WHERE e.decay_score < 0.8
+      ORDER BY sub.rank LIMIT ?
+    `;
+        params = [query, limit, limit];
+    }
+    const rows = db.prepare(sql).all(...params);
+    return rows.map(r => ({
+        id: r.id,
+        rank: r.rank,
+        content: r.summary,
+        domain: r.event_type,
+        agentId: r.agent_id,
+        metadata: r.participants || undefined,
+    }));
+}
+function resultKey(table, id) {
+    return `${table}:${id}`;
+}
+/**
+ * Merge FTS5 and KNN results using Reciprocal Rank Fusion.
+ *
+ * RRF score = Σ (weight / (k + rank)) for each result list the item appears in.
+ * k is a constant (default 60) that dampens the effect of high rank positions.
+ */
+function fuseResults(ftsResults, knnResults, k, ftsWeight, knnWeight) {
+    const merged = new Map();
+    // Add FTS results
+    for (const [key, entry] of ftsResults) {
+        const score = ftsWeight / (k + (entry.ftsRank || 1));
+        merged.set(key, { ...entry, score, sources: ['fts'] });
+    }
+    // Merge KNN results
+    for (const [key, entry] of knnResults) {
+        const knnScore = knnWeight / (k + (entry.knnRank || 1));
+        const existing = merged.get(key);
+        if (existing) {
+            // Item found by both — boost score
+            existing.score += knnScore;
+            existing.knnRank = entry.knnRank;
+            existing.knnDistance = entry.knnDistance;
+            existing.sources = ['fts', 'knn'];
+        }
+        else {
+            merged.set(key, { ...entry, score: knnScore, sources: ['knn'] });
+        }
+    }
+    // Sort by fused score descending
+    return [...merged.values()].sort((a, b) => b.score - a.score);
+}
+// ─── Public API ────────────────────────────────────────────────
+/**
+ * Hybrid search combining FTS5 keyword search and KNN vector search.
+ *
+ * When vectorStore is null, falls back to FTS5-only.
+ * When FTS5 query is empty (all stop words), falls back to KNN-only.
+ */
+export async function hybridSearch(libraryDb, vectorStore, query, opts) {
+    const tables = opts?.tables || ['facts', 'knowledge', 'episodes'];
+    const limit = opts?.limit || 10;
+    const maxKnnDistance = opts?.maxKnnDistance || 1.2;
+    const rrfK = opts?.rrfK || 60;
+    const ftsWeight = opts?.ftsWeight || 1.0;
+    const knnWeight = opts?.knnWeight || 1.0;
+    const minFtsTerms = opts?.minFtsTerms || 1;
+    // ── FTS5 retrieval ──
+    const ftsQuery = buildFtsQuery(query);
+    const ftsMap = new Map();
+    if (ftsQuery && ftsQuery.split(' OR ').length >= minFtsTerms) {
+        try {
+            const perTableLimit = Math.ceil(limit * 1.5); // Over-fetch for fusion
+            if (tables.includes('facts')) {
+                const results = searchFactsFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
+                results.forEach((r, i) => {
+                    const key = resultKey('facts', r.id);
+                    ftsMap.set(key, {
+                        sourceTable: 'facts',
+                        sourceId: r.id,
+                        content: r.content,
+                        domain: r.domain,
+                        agentId: r.agentId,
+                        ftsRank: i + 1,
+                        score: 0,
+                        sources: ['fts'],
+                    });
+                });
+            }
+            if (tables.includes('knowledge')) {
+                const results = searchKnowledgeFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
+                results.forEach((r, i) => {
+                    const key = resultKey('knowledge', r.id);
+                    ftsMap.set(key, {
+                        sourceTable: 'knowledge',
+                        sourceId: r.id,
+                        content: r.content,
+                        domain: r.domain,
+                        agentId: r.agentId,
+                        metadata: r.metadata,
+                        ftsRank: i + 1,
+                        score: 0,
+                        sources: ['fts'],
+                    });
+                });
+            }
+            if (tables.includes('episodes')) {
+                const results = searchEpisodesFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
+                results.forEach((r, i) => {
+                    const key = resultKey('episodes', r.id);
+                    ftsMap.set(key, {
+                        sourceTable: 'episodes',
+                        sourceId: r.id,
+                        content: r.content,
+                        domain: r.domain,
+                        agentId: r.agentId,
+                        metadata: r.metadata,
+                        ftsRank: i + 1,
+                        score: 0,
+                        sources: ['fts'],
+                    });
+                });
+            }
+        }
+        catch {
+            // FTS5 failure is non-fatal — fall through to KNN-only
+        }
+    }
+    // ── KNN retrieval ──
+    const knnMap = new Map();
+    if (vectorStore) {
+        try {
+            const knnResults = await vectorStore.search(query, {
+                tables,
+                limit: Math.ceil(limit * 1.5),
+                maxDistance: maxKnnDistance,
+            });
+            knnResults.forEach((r, i) => {
+                const key = resultKey(r.sourceTable, r.sourceId);
+                knnMap.set(key, {
+                    sourceTable: r.sourceTable,
+                    sourceId: r.sourceId,
+                    content: r.content,
+                    domain: r.domain,
+                    agentId: r.agentId,
+                    metadata: r.metadata,
+                    knnRank: i + 1,
+                    knnDistance: r.distance,
+                    score: 0,
+                    sources: ['knn'],
+                });
+            });
+        }
+        catch {
+            // KNN failure is non-fatal — use FTS-only
+        }
+    }
+    // ── Fusion ──
+    if (ftsMap.size === 0 && knnMap.size === 0) {
+        return [];
+    }
+    // If only one source has results, skip fusion overhead but assign scores
+    if (ftsMap.size === 0) {
+        // KNN-only: score by inverse distance
+        return [...knnMap.values()]
+            .sort((a, b) => (a.knnDistance || 99) - (b.knnDistance || 99))
+            .slice(0, limit)
+            .map((entry, i) => toHybridResult({
+            ...entry,
+            score: knnWeight / (rrfK + i + 1),
+        }));
+    }
+    if (knnMap.size === 0) {
+        // FTS-only: score by rank position
+        return [...ftsMap.values()]
+            .sort((a, b) => (a.ftsRank || 99) - (b.ftsRank || 99))
+            .slice(0, limit)
+            .map((entry, i) => toHybridResult({
+            ...entry,
+            score: ftsWeight / (rrfK + i + 1),
+        }));
+    }
+    // Both sources present — RRF fusion
+    const fused = fuseResults(ftsMap, knnMap, rrfK, ftsWeight, knnWeight);
+    return fused.slice(0, limit).map(toHybridResult);
+}
+function toHybridResult(entry) {
+    return {
+        sourceTable: entry.sourceTable,
+        sourceId: entry.sourceId,
+        content: entry.content,
+        domain: entry.domain,
+        agentId: entry.agentId,
+        metadata: entry.metadata,
+        score: entry.score,
+        sources: entry.sources,
+    };
+}
+//# sourceMappingURL=hybrid-retrieval.js.map