@psiclawops/hypermem 0.1.0 → 0.5.0

package/dist/hybrid-retrieval.js

@@ -1,340 +0,0 @@
-/**
- * HyperMem Hybrid Retrieval — FTS5 + KNN Score Fusion
- *
- * Merges keyword (FTS5/BM25) and semantic (KNN/vector) results into a
- * single ranked list using Reciprocal Rank Fusion (RRF). This avoids
- * vocabulary mismatch (KNN-only misses exact terms) and semantic gap
- * (FTS5-only misses paraphrases).
- *
- * Architecture:
- *   - FTS5 results from library.db (facts_fts, knowledge_fts, episodes_fts)
- *   - KNN results from vectors.db via VectorStore
- *   - RRF merges both ranked lists with configurable k constant
- *   - Deduplication by (sourceTable, sourceId)
- *   - Token-budgeted output for compositor consumption
- */
-// ─── FTS5 Query Building ───────────────────────────────────────
-/** Stop words to exclude from FTS5 queries */
-const STOP_WORDS = new Set([
-    'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
-    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
-    'should', 'may', 'might', 'shall', 'can', 'need', 'dare', 'ought',
-    'used', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from',
-    'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below',
-    'between', 'out', 'off', 'over', 'under', 'again', 'further', 'then',
-    'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each',
-    'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'no',
-    'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
-    'just', 'don', 'now', 'and', 'but', 'or', 'if', 'it', 'its', 'this',
-    'that', 'these', 'those', 'i', 'me', 'my', 'we', 'our', 'you', 'your',
-    'he', 'him', 'his', 'she', 'her', 'they', 'them', 'their', 'what',
-    'which', 'who', 'whom', 'about', 'up',
-]);
-/**
- * Build an FTS5 query from a natural language string.
- * Extracts meaningful words, removes stop words, uses OR conjunction.
- */
-export function buildFtsQuery(input) {
-    const words = input
-        .toLowerCase()
-        .replace(/[^\w\s-]/g, ' ') // strip punctuation except hyphens
-        .split(/\s+/)
-        .filter(w => w.length >= 3 && !STOP_WORDS.has(w));
-    if (words.length === 0)
-        return '';
-    // Deduplicate, sort by length descending (more specific terms first),
-    // cap at 8 terms to keep queries reasonable
-    const unique = [...new Set(words)]
-        .sort((a, b) => b.length - a.length)
-        .slice(0, 8);
-    // Use prefix matching (*) and OR so any term can match
-    return unique.map(w => `"${w}"*`).join(' OR ');
-}
-/**
- * Search facts via FTS5.
- */
-function searchFactsFts(db, query, agentId, limit = 20) {
-    // Two-phase query: FTS runs first in subquery (fast), then filters on
-    // the small result set.  Joining FTS + non-FTS predicates + ORDER BY rank
-    // in one pass forces SQLite to materialise the full FTS match set before
-    // applying LIMIT — O(matches) instead of O(limit).  See data-access-bench.
-    const innerLimit = agentId ? limit * 4 : limit; // over-fetch to survive filter
-    let sql = `
-    SELECT f.id, sub.rank, f.content, f.domain, f.agent_id
-    FROM (
-      SELECT rowid, rank FROM facts_fts WHERE facts_fts MATCH ? ORDER BY rank LIMIT ?
-    ) sub
-    JOIN facts f ON f.id = sub.rowid
-    WHERE f.superseded_by IS NULL
-    AND f.decay_score < 0.8
-  `;
-    const params = [query, innerLimit];
-    if (agentId) {
-        sql += ' AND f.agent_id = ?';
-        params.push(agentId);
-    }
-    sql += ' ORDER BY sub.rank LIMIT ?';
-    params.push(limit);
-    const rows = db.prepare(sql).all(...params);
-    return rows.map(r => ({
-        id: r.id,
-        rank: r.rank,
-        content: r.content,
-        domain: r.domain,
-        agentId: r.agent_id,
-    }));
-}
-/**
- * Search knowledge via FTS5.
- */
-function searchKnowledgeFts(db, query, agentId, limit = 20) {
-    const innerLimit = agentId ? limit * 4 : limit;
-    let sql = `
-    SELECT k.id, sub.rank, k.content, k.domain, k.agent_id, k.key
-    FROM (
-      SELECT rowid, rank FROM knowledge_fts WHERE knowledge_fts MATCH ? ORDER BY rank LIMIT ?
-    ) sub
-    JOIN knowledge k ON k.id = sub.rowid
-    WHERE k.superseded_by IS NULL
-  `;
-    const params = [query, innerLimit];
-    if (agentId) {
-        sql += ' AND k.agent_id = ?';
-        params.push(agentId);
-    }
-    sql += ' ORDER BY sub.rank LIMIT ?';
-    params.push(limit);
-    const rows = db.prepare(sql).all(...params);
-    return rows.map(r => ({
-        id: r.id,
-        rank: r.rank,
-        content: r.content,
-        domain: r.domain,
-        agentId: r.agent_id,
-        metadata: r.key,
-    }));
-}
-/**
- * Search episodes via FTS5.
- */
-function searchEpisodesFts(db, query, agentId, limit = 20) {
-    let sql;
-    let params;
-    if (agentId) {
-        // Agent-scoped: use WHERE IN (FTS5 subquery) instead of FTS5→JOIN→filter.
-        // SQLite uses the agent_id index to narrow first, then checks FTS5 membership.
-        // Benchmarked: 2.3ms avg vs 8.5ms avg for the post-join approach (13k+ episodes).
-        sql = `
-      SELECT e.id, 0 as rank, e.summary, e.event_type, e.agent_id, e.participants
-      FROM episodes e
-      WHERE e.agent_id = ?
-        AND e.decay_score < 0.8
-        AND e.id IN (SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH ?)
-      ORDER BY e.created_at DESC
-      LIMIT ?
-    `;
-        params = [agentId, query, limit];
-    }
-    else {
-        sql = `
-      SELECT e.id, sub.rank, e.summary, e.event_type, e.agent_id, e.participants
-      FROM (
-        SELECT rowid, rank FROM episodes_fts WHERE episodes_fts MATCH ? ORDER BY rank LIMIT ?
-      ) sub
-      JOIN episodes e ON e.id = sub.rowid
-      WHERE e.decay_score < 0.8
-      ORDER BY sub.rank LIMIT ?
-    `;
-        params = [query, limit, limit];
-    }
-    const rows = db.prepare(sql).all(...params);
-    return rows.map(r => ({
-        id: r.id,
-        rank: r.rank,
-        content: r.summary,
-        domain: r.event_type,
-        agentId: r.agent_id,
-        metadata: r.participants || undefined,
-    }));
-}
-function resultKey(table, id) {
-    return `${table}:${id}`;
-}
-/**
- * Merge FTS5 and KNN results using Reciprocal Rank Fusion.
- *
- * RRF score = Σ (weight / (k + rank)) for each result list the item appears in.
- * k is a constant (default 60) that dampens the effect of high rank positions.
- */
-function fuseResults(ftsResults, knnResults, k, ftsWeight, knnWeight) {
-    const merged = new Map();
-    // Add FTS results
-    for (const [key, entry] of ftsResults) {
-        const score = ftsWeight / (k + (entry.ftsRank || 1));
-        merged.set(key, { ...entry, score, sources: ['fts'] });
-    }
-    // Merge KNN results
-    for (const [key, entry] of knnResults) {
-        const knnScore = knnWeight / (k + (entry.knnRank || 1));
-        const existing = merged.get(key);
-        if (existing) {
-            // Item found by both — boost score
-            existing.score += knnScore;
-            existing.knnRank = entry.knnRank;
-            existing.knnDistance = entry.knnDistance;
-            existing.sources = ['fts', 'knn'];
-        }
-        else {
-            merged.set(key, { ...entry, score: knnScore, sources: ['knn'] });
-        }
-    }
-    // Sort by fused score descending
-    return [...merged.values()].sort((a, b) => b.score - a.score);
-}
-// ─── Public API ────────────────────────────────────────────────
-/**
- * Hybrid search combining FTS5 keyword search and KNN vector search.
- *
- * When vectorStore is null, falls back to FTS5-only.
- * When FTS5 query is empty (all stop words), falls back to KNN-only.
- */
-export async function hybridSearch(libraryDb, vectorStore, query, opts) {
-    const tables = opts?.tables || ['facts', 'knowledge', 'episodes'];
-    const limit = opts?.limit || 10;
-    const maxKnnDistance = opts?.maxKnnDistance || 1.2;
-    const rrfK = opts?.rrfK || 60;
-    const ftsWeight = opts?.ftsWeight || 1.0;
-    const knnWeight = opts?.knnWeight || 1.0;
-    const minFtsTerms = opts?.minFtsTerms || 1;
-    // ── FTS5 retrieval ──
-    const ftsQuery = buildFtsQuery(query);
-    const ftsMap = new Map();
-    if (ftsQuery && ftsQuery.split(' OR ').length >= minFtsTerms) {
-        try {
-            const perTableLimit = Math.ceil(limit * 1.5); // Over-fetch for fusion
-            if (tables.includes('facts')) {
-                const results = searchFactsFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
-                results.forEach((r, i) => {
-                    const key = resultKey('facts', r.id);
-                    ftsMap.set(key, {
-                        sourceTable: 'facts',
-                        sourceId: r.id,
-                        content: r.content,
-                        domain: r.domain,
-                        agentId: r.agentId,
-                        ftsRank: i + 1,
-                        score: 0,
-                        sources: ['fts'],
-                    });
-                });
-            }
-            if (tables.includes('knowledge')) {
-                const results = searchKnowledgeFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
-                results.forEach((r, i) => {
-                    const key = resultKey('knowledge', r.id);
-                    ftsMap.set(key, {
-                        sourceTable: 'knowledge',
-                        sourceId: r.id,
-                        content: r.content,
-                        domain: r.domain,
-                        agentId: r.agentId,
-                        metadata: r.metadata,
-                        ftsRank: i + 1,
-                        score: 0,
-                        sources: ['fts'],
-                    });
-                });
-            }
-            if (tables.includes('episodes')) {
-                const results = searchEpisodesFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
-                results.forEach((r, i) => {
-                    const key = resultKey('episodes', r.id);
-                    ftsMap.set(key, {
-                        sourceTable: 'episodes',
-                        sourceId: r.id,
-                        content: r.content,
-                        domain: r.domain,
-                        agentId: r.agentId,
-                        metadata: r.metadata,
-                        ftsRank: i + 1,
-                        score: 0,
-                        sources: ['fts'],
-                    });
-                });
-            }
-        }
-        catch {
-            // FTS5 failure is non-fatal — fall through to KNN-only
-        }
-    }
-    // ── KNN retrieval ──
-    const knnMap = new Map();
-    if (vectorStore) {
-        try {
-            const knnResults = await vectorStore.search(query, {
-                tables,
-                limit: Math.ceil(limit * 1.5),
-                maxDistance: maxKnnDistance,
-            });
-            knnResults.forEach((r, i) => {
-                const key = resultKey(r.sourceTable, r.sourceId);
-                knnMap.set(key, {
-                    sourceTable: r.sourceTable,
-                    sourceId: r.sourceId,
-                    content: r.content,
-                    domain: r.domain,
-                    agentId: r.agentId,
-                    metadata: r.metadata,
-                    knnRank: i + 1,
-                    knnDistance: r.distance,
-                    score: 0,
-                    sources: ['knn'],
-                });
-            });
-        }
-        catch {
-            // KNN failure is non-fatal — use FTS-only
-        }
-    }
-    // ── Fusion ──
-    if (ftsMap.size === 0 && knnMap.size === 0) {
-        return [];
-    }
-    // If only one source has results, skip fusion overhead but assign scores
-    if (ftsMap.size === 0) {
-        // KNN-only: score by inverse distance
-        return [...knnMap.values()]
-            .sort((a, b) => (a.knnDistance || 99) - (b.knnDistance || 99))
-            .slice(0, limit)
-            .map((entry, i) => toHybridResult({
-            ...entry,
-            score: knnWeight / (rrfK + i + 1),
-        }));
-    }
-    if (knnMap.size === 0) {
-        // FTS-only: score by rank position
-        return [...ftsMap.values()]
-            .sort((a, b) => (a.ftsRank || 99) - (b.ftsRank || 99))
-            .slice(0, limit)
-            .map((entry, i) => toHybridResult({
-            ...entry,
-            score: ftsWeight / (rrfK + i + 1),
-        }));
-    }
-    // Both sources present — RRF fusion
-    const fused = fuseResults(ftsMap, knnMap, rrfK, ftsWeight, knnWeight);
-    return fused.slice(0, limit).map(toHybridResult);
-}
-function toHybridResult(entry) {
-    return {
-        sourceTable: entry.sourceTable,
-        sourceId: entry.sourceId,
-        content: entry.content,
-        domain: entry.domain,
-        agentId: entry.agentId,
-        metadata: entry.metadata,
-        score: entry.score,
-        sources: entry.sources,
-    };
-}
-//# sourceMappingURL=hybrid-retrieval.js.map