@psiclawops/hypermem 0.9.0 → 0.9.1

package/dist/hybrid-retrieval.js

@@ -1,429 +0,0 @@
-/**
- * hypermem Hybrid Retrieval — FTS5 + KNN Score Fusion
- *
- * Merges keyword (FTS5/BM25) and semantic (KNN/vector) results into a
- * single ranked list using Reciprocal Rank Fusion (RRF). This avoids
- * vocabulary mismatch (KNN-only misses exact terms) and semantic gap
- * (FTS5-only misses paraphrases).
- *
- * Architecture:
- *   - FTS5 results from library.db (facts_fts, knowledge_fts, episodes_fts)
- *   - KNN results from vectors.db via VectorStore
- *   - RRF merges both ranked lists with configurable k constant
- *   - Deduplication by (sourceTable, sourceId)
- *   - Token-budgeted output for compositor consumption
- */
-function emitRerankerLog(ev) {
-    // Single-line, structured, metadata-only — safe for production logs.
-    // Guarded behind HYPERMEM_RERANKER_LOG=1 to avoid console spam on every query.
-    if (process.env['HYPERMEM_RERANKER_LOG'] === '1') {
-        console.log(`[hypermem] reranker: provider=${ev.provider ?? 'none'} candidates=${ev.candidates} status=${ev.status}`);
-    }
-}
-// ─── FTS5 Query Building ───────────────────────────────────────
-/** Stop words to exclude from FTS5 queries */
-const STOP_WORDS = new Set([
-    'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
-    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
-    'should', 'may', 'might', 'shall', 'can', 'need', 'dare', 'ought',
-    'used', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from',
-    'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below',
-    'between', 'out', 'off', 'over', 'under', 'again', 'further', 'then',
-    'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each',
-    'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'no',
-    'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very',
-    'just', 'don', 'now', 'and', 'but', 'or', 'if', 'it', 'its', 'this',
-    'that', 'these', 'those', 'i', 'me', 'my', 'we', 'our', 'you', 'your',
-    'he', 'him', 'his', 'she', 'her', 'they', 'them', 'their', 'what',
-    'which', 'who', 'whom', 'about', 'up',
-]);
-/**
- * Build an FTS5 query from a natural language string.
- * Extracts meaningful words, removes stop words, uses OR conjunction.
- */
-export function buildFtsQuery(input) {
-    const words = input
-        .toLowerCase()
-        .replace(/[^\w\s-]/g, ' ') // strip punctuation except hyphens
-        .split(/\s+/)
-        .filter(w => w.length >= 3 && !STOP_WORDS.has(w));
-    if (words.length === 0)
-        return '';
-    // Deduplicate, sort by length descending (more specific terms first),
-    // cap at 8 terms to keep queries reasonable
-    const unique = [...new Set(words)]
-        .sort((a, b) => b.length - a.length)
-        .slice(0, 8);
-    // Use prefix matching (*) and OR so any term can match
-    return unique.map(w => `"${w}"*`).join(' OR ');
-}
-/**
- * Search facts via FTS5.
- */
-function searchFactsFts(db, query, agentId, limit = 20) {
-    // Two-phase query: FTS runs first in subquery (fast), then filters on
-    // the small result set.  Joining FTS + non-FTS predicates + ORDER BY rank
-    // in one pass forces SQLite to materialise the full FTS match set before
-    // applying LIMIT — O(matches) instead of O(limit).  See data-access-bench.
-    const innerLimit = agentId ? limit * 4 : limit; // over-fetch to survive filter
-    let sql = `
-    SELECT f.id, sub.rank, f.content, f.domain, f.agent_id
-    FROM (
-      SELECT rowid, rank FROM facts_fts WHERE facts_fts MATCH ? ORDER BY rank LIMIT ?
-    ) sub
-    JOIN facts f ON f.id = sub.rowid
-    WHERE f.superseded_by IS NULL
-    AND f.decay_score < 0.8
-  `;
-    const params = [query, innerLimit];
-    if (agentId) {
-        sql += ' AND f.agent_id = ?';
-        params.push(agentId);
-    }
-    sql += ' ORDER BY sub.rank LIMIT ?';
-    params.push(limit);
-    const rows = db.prepare(sql).all(...params);
-    return rows.map(r => ({
-        id: r.id,
-        rank: r.rank,
-        content: r.content,
-        domain: r.domain,
-        agentId: r.agent_id,
-    }));
-}
-/**
- * Search knowledge via FTS5.
- */
-function searchKnowledgeFts(db, query, agentId, limit = 20) {
-    const innerLimit = agentId ? limit * 4 : limit;
-    let sql = `
-    SELECT k.id, sub.rank, k.content, k.domain, k.agent_id, k.key
-    FROM (
-      SELECT rowid, rank FROM knowledge_fts WHERE knowledge_fts MATCH ? ORDER BY rank LIMIT ?
-    ) sub
-    JOIN knowledge k ON k.id = sub.rowid
-    WHERE k.superseded_by IS NULL
-  `;
-    const params = [query, innerLimit];
-    if (agentId) {
-        sql += ' AND k.agent_id = ?';
-        params.push(agentId);
-    }
-    sql += ' ORDER BY sub.rank LIMIT ?';
-    params.push(limit);
-    const rows = db.prepare(sql).all(...params);
-    return rows.map(r => ({
-        id: r.id,
-        rank: r.rank,
-        content: r.content,
-        domain: r.domain,
-        agentId: r.agent_id,
-        metadata: r.key,
-    }));
-}
-/**
- * Search episodes via FTS5.
- */
-function searchEpisodesFts(db, query, agentId, limit = 20) {
-    let sql;
-    let params;
-    if (agentId) {
-        // Agent-scoped: use WHERE IN (FTS5 subquery) instead of FTS5→JOIN→filter.
-        // SQLite uses the agent_id index to narrow first, then checks FTS5 membership.
-        // Benchmarked: 2.3ms avg vs 8.5ms avg for the post-join approach (13k+ episodes).
-        sql = `
-      SELECT e.id, 0 as rank, e.summary, e.event_type, e.agent_id, e.participants, e.created_at
-      FROM episodes e
-      WHERE e.agent_id = ?
-        AND e.decay_score < 0.8
-        AND e.id IN (SELECT rowid FROM episodes_fts WHERE episodes_fts MATCH ?)
-      ORDER BY e.created_at DESC
-      LIMIT ?
-    `;
-        params = [agentId, query, limit];
-    }
-    else {
-        sql = `
-      SELECT e.id, sub.rank, e.summary, e.event_type, e.agent_id, e.participants, e.created_at
-      FROM (
-        SELECT rowid, rank FROM episodes_fts WHERE episodes_fts MATCH ? ORDER BY rank LIMIT ?
-      ) sub
-      JOIN episodes e ON e.id = sub.rowid
-      WHERE e.decay_score < 0.8
-      ORDER BY sub.rank LIMIT ?
-    `;
-        params = [query, limit, limit];
-    }
-    const rows = db.prepare(sql).all(...params);
-    return rows.map(r => ({
-        id: r.id,
-        rank: r.rank,
-        content: r.summary,
-        domain: r.event_type,
-        agentId: r.agent_id,
-        metadata: r.participants || undefined,
-        createdAt: r.created_at || undefined,
-    }));
-}
-function resultKey(table, id) {
-    return `${table}:${id}`;
-}
-/**
- * Merge FTS5 and KNN results using Reciprocal Rank Fusion.
- *
- * RRF score = Σ (weight / (k + rank)) for each result list the item appears in.
- * k is a constant (default 60) that dampens the effect of high rank positions.
- */
-function fuseResults(ftsResults, knnResults, k, ftsWeight, knnWeight) {
-    const merged = new Map();
-    // Add FTS results
-    for (const [key, entry] of ftsResults) {
-        const score = ftsWeight / (k + (entry.ftsRank || 1));
-        merged.set(key, { ...entry, score, sources: ['fts'] });
-    }
-    // Merge KNN results
-    for (const [key, entry] of knnResults) {
-        const knnScore = knnWeight / (k + (entry.knnRank || 1));
-        const existing = merged.get(key);
-        if (existing) {
-            // Item found by both — boost score
-            existing.score += knnScore;
-            existing.knnRank = entry.knnRank;
-            existing.knnDistance = entry.knnDistance;
-            existing.sources = ['fts', 'knn'];
-        }
-        else {
-            merged.set(key, { ...entry, score: knnScore, sources: ['knn'] });
-        }
-    }
-    // Sort by fused score descending
-    return [...merged.values()].sort((a, b) => b.score - a.score);
-}
-// ─── Public API ────────────────────────────────────────────────
-/**
- * Hybrid search combining FTS5 keyword search and KNN vector search.
- *
- * When vectorStore is null, falls back to FTS5-only.
- * When FTS5 query is empty (all stop words), falls back to KNN-only.
- */
-export async function hybridSearch(libraryDb, vectorStore, query, opts) {
-    const tables = opts?.tables || ['facts', 'knowledge', 'episodes'];
-    const limit = opts?.limit || 10;
-    const maxKnnDistance = opts?.maxKnnDistance || 1.2;
-    const rrfK = opts?.rrfK || 60;
-    const ftsWeight = opts?.ftsWeight || 1.0;
-    const knnWeight = opts?.knnWeight || 1.0;
-    const minFtsTerms = opts?.minFtsTerms || 1;
-    // ── FTS5 retrieval ──
-    const ftsQuery = buildFtsQuery(query);
-    const ftsMap = new Map();
-    if (ftsQuery && ftsQuery.split(' OR ').length >= minFtsTerms) {
-        try {
-            const perTableLimit = Math.ceil(limit * 1.5); // Over-fetch for fusion
-            if (tables.includes('facts')) {
-                const results = searchFactsFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
-                results.forEach((r, i) => {
-                    const key = resultKey('facts', r.id);
-                    ftsMap.set(key, {
-                        sourceTable: 'facts',
-                        sourceId: r.id,
-                        content: r.content,
-                        domain: r.domain,
-                        agentId: r.agentId,
-                        ftsRank: i + 1,
-                        score: 0,
-                        sources: ['fts'],
-                    });
-                });
-            }
-            if (tables.includes('knowledge')) {
-                const results = searchKnowledgeFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
-                results.forEach((r, i) => {
-                    const key = resultKey('knowledge', r.id);
-                    ftsMap.set(key, {
-                        sourceTable: 'knowledge',
-                        sourceId: r.id,
-                        content: r.content,
-                        domain: r.domain,
-                        agentId: r.agentId,
-                        metadata: r.metadata,
-                        ftsRank: i + 1,
-                        score: 0,
-                        sources: ['fts'],
-                    });
-                });
-            }
-            if (tables.includes('episodes')) {
-                const results = searchEpisodesFts(libraryDb, ftsQuery, opts?.agentId, perTableLimit);
-                results.forEach((r, i) => {
-                    const key = resultKey('episodes', r.id);
-                    ftsMap.set(key, {
-                        sourceTable: 'episodes',
-                        sourceId: r.id,
-                        content: r.content,
-                        domain: r.domain,
-                        agentId: r.agentId,
-                        metadata: r.metadata,
-                        createdAt: r.createdAt,
-                        ftsRank: i + 1,
-                        score: 0,
-                        sources: ['fts'],
-                    });
-                });
-            }
-        }
-        catch {
-            // FTS5 failure is non-fatal — fall through to KNN-only
-        }
-    }
-    // ── KNN retrieval ──
-    const knnMap = new Map();
-    if (vectorStore) {
-        try {
-            const knnResults = await vectorStore.search(query, {
-                tables,
-                limit: Math.ceil(limit * 1.5),
-                maxDistance: maxKnnDistance,
-                precomputedEmbedding: opts?.precomputedEmbedding,
-            });
-            knnResults.forEach((r, i) => {
-                const key = resultKey(r.sourceTable, r.sourceId);
-                knnMap.set(key, {
-                    sourceTable: r.sourceTable,
-                    sourceId: r.sourceId,
-                    content: r.content,
-                    domain: r.domain,
-                    agentId: r.agentId,
-                    metadata: r.metadata,
-                    knnRank: i + 1,
-                    knnDistance: r.distance,
-                    score: 0,
-                    sources: ['knn'],
-                });
-            });
-        }
-        catch {
-            // KNN failure is non-fatal — use FTS-only
-        }
-    }
-    // ── Fusion ──
-    if (ftsMap.size === 0 && knnMap.size === 0) {
-        return [];
-    }
-    // If only one source has results, skip fusion overhead but assign scores
-    if (ftsMap.size === 0) {
-        // KNN-only: score by inverse distance
-        return [...knnMap.values()]
-            .sort((a, b) => (a.knnDistance || 99) - (b.knnDistance || 99))
-            .slice(0, limit)
-            .map((entry, i) => toHybridResult({
-            ...entry,
-            score: knnWeight / (rrfK + i + 1),
-        }));
-    }
-    if (knnMap.size === 0) {
-        // FTS-only: score by rank position
-        return [...ftsMap.values()]
-            .sort((a, b) => (a.ftsRank || 99) - (b.ftsRank || 99))
-            .slice(0, limit)
-            .map((entry, i) => toHybridResult({
-            ...entry,
-            score: ftsWeight / (rrfK + i + 1),
-        }));
-    }
-    // Both sources present — RRF fusion
-    const fused = fuseResults(ftsMap, knnMap, rrfK, ftsWeight, knnWeight);
-    // ── Reranker hook ─────────────────────────────────────────────
-    // Reranking runs only on the fused path. FTS-only / KNN-only branches
-    // above return before this point. On any error, timeout, or null result
-    // the original fused ordering is preserved.
-    const reranked = await maybeRerank(fused, query, opts);
-    return reranked.slice(0, limit).map(toHybridResult);
-}
-/**
- * Apply a reranker to the fused candidate list, falling back to the original
- * order on any failure. Emits a single-line telemetry event.
- */
-async function maybeRerank(fused, query, opts) {
-    const reranker = opts?.reranker ?? null;
-    const emit = opts?.onRerankerTelemetry ?? emitRerankerLog;
-    if (!reranker) {
-        emit({ provider: null, candidates: fused.length, status: 'bypass_no_provider' });
-        return fused;
-    }
-    const minCandidates = opts?.rerankerMinCandidates ?? 2;
-    if (fused.length < minCandidates) {
-        emit({ provider: reranker.name, candidates: fused.length, status: 'bypass_below_threshold' });
-        return fused;
-    }
-    const maxDocs = opts?.rerankerMaxDocuments ?? fused.length;
-    const slice = fused.slice(0, Math.max(0, maxDocs));
-    const topK = opts?.rerankerTopK ?? slice.length;
-    const timeoutMs = opts?.rerankerTimeoutMs ?? 3000;
-    let timedOut = false;
-    const timeoutToken = Symbol('rerank-timeout');
-    let timer;
-    const timeoutPromise = new Promise((resolve) => {
-        timer = setTimeout(() => { timedOut = true; resolve(timeoutToken); }, timeoutMs);
-    });
-    try {
-        const rerankCall = reranker.rerank(query, slice.map((e) => e.content), topK);
-        const race = await Promise.race([rerankCall, timeoutPromise]);
-        if (race === timeoutToken) {
-            emit({ provider: reranker.name, candidates: slice.length, status: 'timeout' });
-            return fused;
-        }
-        const results = race;
-        if (!results || results.length === 0) {
-            emit({ provider: reranker.name, candidates: slice.length, status: 'failed' });
-            return fused;
-        }
-        const seen = new Set();
-        const reordered = [];
-        for (const r of results) {
-            if (r.index < 0 || r.index >= slice.length)
-                continue;
-            if (seen.has(r.index))
-                continue;
-            seen.add(r.index);
-            reordered.push(slice[r.index]);
-        }
-        // Preserve any fused entries the reranker did not score so we never shrink
-        // the candidate set on a graceful-degrade path. Keep original RRF order
-        // for the tail.
-        for (let i = 0; i < fused.length; i++) {
-            if (i < slice.length && seen.has(i))
-                continue;
-            reordered.push(fused[i]);
-        }
-        emit({ provider: reranker.name, candidates: slice.length, status: 'applied' });
-        return reordered;
-    }
-    catch {
-        if (timedOut) {
-            emit({ provider: reranker.name, candidates: slice.length, status: 'timeout' });
-        }
-        else {
-            emit({ provider: reranker.name, candidates: slice.length, status: 'failed' });
-        }
-        return fused;
-    }
-    finally {
-        if (timer)
-            clearTimeout(timer);
-    }
-}
-function toHybridResult(entry) {
-    return {
-        sourceTable: entry.sourceTable,
-        sourceId: entry.sourceId,
-        content: entry.content,
-        domain: entry.domain,
-        agentId: entry.agentId,
-        metadata: entry.metadata,
-        createdAt: entry.createdAt,
-        score: entry.score,
-        sources: entry.sources,
-    };
-}
-//# sourceMappingURL=hybrid-retrieval.js.map

package/dist/image-eviction.d.ts

@@ -1,49 +0,0 @@
-/**
- * Image and heavy-content eviction pre-pass.
- *
- * Runs before the LLM call inside assemble(). Scans the live messages array
- * for stale image payloads and large tool results, replaces them with compact
- * text descriptors. This frees tokens before pressure tiers fire, so compaction
- * runs less often.
- *
- * Works on the raw OpenClaw message format (AgentMessage / content arrays),
- * not on hypermem's internal NeutralMessage format.
- *
- * What gets evicted:
- *   - Base64 image blocks (OpenAI image_url / Anthropic image source) older than imageAgeTurns
- *   - Tool results larger than minTokensToEvict, older than toolResultAgeTurns
- *
- * What is never touched:
- *   - User text content
- *   - Assistant text responses
- *   - Recent messages (within staleness thresholds)
- *   - Tool call structure (type, id, name preserved — required for API pairing)
- *   - Small tool results (below minTokensToEvict)
- *   - Thinking blocks
- */
-export interface ImageEvictionConfig {
-    /** Turns after which images are evicted. Default: 1 */
-    imageAgeTurns: number;
-    /** Turns after which large tool results are evicted. Default: 1 */
-    toolResultAgeTurns: number;
-    /** Minimum estimated tokens to bother evicting a tool result. Default: 200 */
-    minTokensToEvict: number;
-    /** Characters of preview to keep from evicted tool result content. Default: 120 */
-    keepPreviewChars: number;
-}
-export interface EvictionStats {
-    imagesEvicted: number;
-    toolResultsEvicted: number;
-    tokensFreed: number;
-}
-export interface EvictionResult {
-    messages: unknown[];
-    stats: EvictionStats;
-}
-export declare const DEFAULT_EVICTION_CONFIG: ImageEvictionConfig;
-/**
- * Evict stale images and large tool results from the messages array.
- * Returns a new array (original is not mutated) plus eviction stats.
- */
-export declare function evictStaleContent(messages: unknown[], config?: Partial<ImageEvictionConfig>): EvictionResult;
-//# sourceMappingURL=image-eviction.d.ts.map

package/dist/image-eviction.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"image-eviction.d.ts","sourceRoot":"","sources":["../src/image-eviction.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,MAAM,WAAW,mBAAmB;IAClC,uDAAuD;IACvD,aAAa,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,8EAA8E;IAC9E,gBAAgB,EAAE,MAAM,CAAC;IACzB,mFAAmF;IACnF,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,eAAO,MAAM,uBAAuB,EAAE,mBAKrC,CAAC;AAqJF;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,OAAO,EAAE,EACnB,MAAM,GAAE,OAAO,CAAC,mBAAmB,CAAM,GACxC,cAAc,CAqFhB"}