hippo-memory 0.36.0 → 0.37.0

package/dist/src/search.js

@@ -0,0 +1,801 @@
+/**
+ * BM25 search + optional embedding hybrid search for Hippo.
+ * Zero external dependencies when embeddings are not available.
+ */
+import { calculateStrength } from './memory.js';
+import { extractPathTags, pathOverlapScore } from './path-context.js';
+import { detectScope, scopeMatch } from './scope.js';
+import { isEmbeddingAvailable, getEmbedding, cosineSimilarity, embeddingModelRequiresReindex, loadEmbeddingIndex, resolveEmbeddingModel, } from './embeddings.js';
+import { physicsScore as computePhysicsScores } from './physics.js';
+import { DEFAULT_PHYSICS_CONFIG } from './physics-config.js';
+import { loadPhysicsState } from './physics-state.js';
+import { openHippoDb, closeHippoDb } from './db.js';
+// ---------------------------------------------------------------------------
+// Tokenizer
+// ---------------------------------------------------------------------------
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\w\s]/g, ' ')
+        .split(/\s+/)
+        .filter((t) => t.length > 1);
+}
+const BM25_K1 = 1.5;
+const BM25_B = 0.75;
+export function buildCorpus(texts) {
+    const docs = texts.map(tokenize);
+    const N = docs.length;
+    const df = new Map();
+    let totalLen = 0;
+    for (const doc of docs) {
+        totalLen += doc.length;
+        const seen = new Set();
+        for (const term of doc) {
+            if (!seen.has(term)) {
+                df.set(term, (df.get(term) ?? 0) + 1);
+                seen.add(term);
+            }
+        }
+    }
+    const avgLen = N > 0 ? totalLen / N : 1;
+    return { docs, avgLen, df, N };
+}
+function bm25Score(corpus, docIdx, queryTerms) {
+    const doc = corpus.docs[docIdx];
+    const docLen = doc.length;
+    let score = 0;
+    // Term frequency map for this doc
+    const tf = new Map();
+    for (const t of doc)
+        tf.set(t, (tf.get(t) ?? 0) + 1);
+    for (const term of queryTerms) {
+        const f = tf.get(term) ?? 0;
+        if (f === 0)
+            continue;
+        const df = corpus.df.get(term) ?? 0;
+        const idf = Math.log((corpus.N - df + 0.5) / (df + 0.5) + 1);
+        const numerator = f * (BM25_K1 + 1);
+        const denominator = f + BM25_K1 * (1 - BM25_B + BM25_B * (docLen / corpus.avgLen));
+        score += idf * (numerator / denominator);
+    }
+    return score;
+}
+// ---------------------------------------------------------------------------
+// Token budget estimation
+// ---------------------------------------------------------------------------
+/**
+ * Rough token estimate: characters / 4 (works well for English text).
+ */
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+// ---------------------------------------------------------------------------
+// Recency boost
+// ---------------------------------------------------------------------------
+function recencyBoost(entry, now) {
+    const created = new Date(entry.created);
+    const ageDays = (now.getTime() - created.getTime()) / (1000 * 60 * 60 * 24);
+    // Exponential decay: memories < 1 day get boost ~1.0, older get less
+    return Math.exp(-ageDays / 30);
+}
+// ---------------------------------------------------------------------------
+// Temporal-aware scoring
+// ---------------------------------------------------------------------------
+const TEMPORAL_RECENT_CUES = new Set(['recently', 'latest', 'last', 'newest', 'current', 'today']);
+const TEMPORAL_OLDEST_CUES = new Set(['first', 'earliest', 'oldest', 'initially', 'originally']);
+export function detectTemporalDirection(query) {
+    const words = query.toLowerCase().split(/\s+/);
+    for (const w of words) {
+        if (TEMPORAL_RECENT_CUES.has(w))
+            return 'recent';
+        if (TEMPORAL_OLDEST_CUES.has(w))
+            return 'oldest';
+    }
+    return null;
+}
+export function computeTemporalRange(entries) {
+    let minTime = Infinity;
+    let maxTime = -Infinity;
+    for (const e of entries) {
+        const t = new Date(e.created).getTime();
+        if (t < minTime)
+            minTime = t;
+        if (t > maxTime)
+            maxTime = t;
+    }
+    return { minTime, maxTime };
+}
+export function temporalBoost(entry, direction, range) {
+    if (!direction)
+        return 1.0;
+    const span = range.maxTime - range.minTime;
+    if (span === 0)
+        return 1.0;
+    const entryTime = new Date(entry.created).getTime();
+    const normalized = (entryTime - range.minTime) / span;
+    if (direction === 'recent') {
+        return 0.8 + 0.4 * normalized;
+    }
+    else {
+        return 0.8 + 0.4 * (1 - normalized);
+    }
+}
+/**
+ * Hybrid search: BM25 + cosine similarity (when embeddings are available).
+ * score = 0.4 * bm25_norm + 0.6 * cosine_sim  (with embeddings)
+ * score = bm25_norm * strength * recency        (BM25-only fallback)
+ *
+ * embeddingWeight: weight for the cosine similarity component (0.0 to 1.0).
+ */
+export async function hybridSearch(query, entries, options = {}) {
+    const now = options.now ?? new Date();
+    const budget = options.budget ?? 4000;
+    const minResults = options.minResults ?? 1;
+    const embeddingWeight = options.embeddingWeight ?? 0.6;
+    const bm25Weight = 1 - embeddingWeight;
+    const scoringMode = options.scoring ?? 'blend';
+    const explain = options.explain ?? false;
+    const mmrEnabled = options.mmr ?? true;
+    const mmrLambda = options.mmrLambda ?? 0.7;
+    // Bi-temporal filtering
+    if (options.asOf) {
+        const asOfDate = new Date(options.asOf);
+        const successorValidFrom = new Map();
+        for (const e of entries) {
+            if (e.superseded_by) {
+                const successor = entries.find(s => s.id === e.superseded_by);
+                if (successor)
+                    successorValidFrom.set(e.id, successor.valid_from);
+            }
+        }
+        entries = entries.filter(e => {
+            if (new Date(e.valid_from) > asOfDate)
+                return false;
+            if (!e.superseded_by)
+                return true;
+            const succVf = successorValidFrom.get(e.id);
+            return succVf ? new Date(succVf) > asOfDate : true;
+        });
+    }
+    else if (!options.includeSuperseded) {
+        entries = entries.filter(e => !e.superseded_by);
+    }
+    if (entries.length === 0)
+        return [];
+    const queryTerms = tokenize(query);
+    if (queryTerms.length === 0)
+        return [];
+    // Build BM25 corpus (or reuse one the caller already built).
+    const corpus = options.preparedCorpus
+        ?? buildCorpus(entries.map((e) => `${e.content} ${e.tags.join(' ')}`));
+    // Score all entries with BM25
+    const bm25Scores = entries.map((_, i) => bm25Score(corpus, i, queryTerms));
+    const maxBm25 = bm25Scores.reduce((a, b) => Math.max(a, b), 1e-9);
+    // Try to get embedding scores if available
+    let useEmbeddings = false;
+    let embeddingIndex = {};
+    let queryVector = [];
+    if (isEmbeddingAvailable() && options.hippoRoot) {
+        try {
+            const model = resolveEmbeddingModel(options.hippoRoot);
+            if (!embeddingModelRequiresReindex(options.hippoRoot, model)) {
+                queryVector = await getEmbedding(query, model);
+                if (queryVector.length > 0) {
+                    embeddingIndex = loadEmbeddingIndex(options.hippoRoot);
+                    useEmbeddings = true;
+                }
+            }
+        }
+        catch {
+            // Fall through to BM25-only
+        }
+    }
+    // Compute cosine similarities for RRF ranking (need all before scoring)
+    const cosineScores = new Array(entries.length).fill(0);
+    const hadCachedVecs = new Array(entries.length).fill(false);
+    if (useEmbeddings) {
+        for (let i = 0; i < entries.length; i++) {
+            const cached = embeddingIndex[entries[i].id];
+            hadCachedVecs[i] = Boolean(cached && queryVector.length > 0);
+            cosineScores[i] = hadCachedVecs[i]
+                ? Math.max(0, cosineSimilarity(queryVector, cached))
+                : 0;
+        }
+    }
+    // For RRF: build rank maps from BM25 and cosine orderings
+    let rrfScores = null;
+    if (useEmbeddings && scoringMode === 'rrf') {
+        const RRF_K = 60;
+        const bm25Ranked = entries.map((_, i) => i).filter(i => bm25Scores[i] > 0 || cosineScores[i] > 0);
+        bm25Ranked.sort((a, b) => bm25Scores[b] - bm25Scores[a]);
+        const cosineRanked = entries.map((_, i) => i).filter(i => bm25Scores[i] > 0 || cosineScores[i] > 0);
+        cosineRanked.sort((a, b) => cosineScores[b] - cosineScores[a]);
+        const bm25RankMap = new Map();
+        bm25Ranked.forEach((idx, rank) => bm25RankMap.set(idx, rank + 1));
+        const cosineRankMap = new Map();
+        cosineRanked.forEach((idx, rank) => cosineRankMap.set(idx, rank + 1));
+        rrfScores = new Map();
+        const allCandidates = new Set([...bm25Ranked, ...cosineRanked]);
+        for (const idx of allCandidates) {
+            const bm25Rank = bm25RankMap.get(idx) ?? (entries.length + 1);
+            const cosineRank = cosineRankMap.get(idx) ?? (entries.length + 1);
+            rrfScores.set(idx, bm25Weight / (RRF_K + bm25Rank) + embeddingWeight / (RRF_K + cosineRank));
+        }
+    }
+    // Score each entry
+    const scored = [];
+    const currentPathTags = extractPathTags(process.cwd());
+    const activeScope = options.scope !== undefined ? options.scope : detectScope();
+    const queryTermSet = new Set(queryTerms);
+    const temporalDirAsync = detectTemporalDirection(query);
+    const temporalRangeAsync = temporalDirAsync ? computeTemporalRange(entries) : { minTime: 0, maxTime: 0 };
+    for (let i = 0; i < entries.length; i++) {
+        const rawBm25 = bm25Scores[i];
+        const cosineScore = cosineScores[i];
+        const hadCachedVec = hadCachedVecs[i];
+        if (!useEmbeddings && rawBm25 <= 0)
+            continue;
+        const normBm25 = rawBm25 / maxBm25;
+        const strength = calculateStrength(entries[i], now);
+        const recency = recencyBoost(entries[i], now);
+        const strengthMultiplier = 0.5 + 0.5 * strength;
+        const recencyMultiplier = 0.8 + 0.2 * recency;
+        let compositeScore;
+        let base;
+        let modeLabel;
+        if (useEmbeddings) {
+            if (rrfScores) {
+                base = rrfScores.get(i) ?? 0;
+            }
+            else {
+                base = bm25Weight * normBm25 + embeddingWeight * cosineScore;
+            }
+            compositeScore = base * strengthMultiplier * recencyMultiplier;
+            modeLabel = hadCachedVec ? 'hybrid' : 'hybrid-no-vec';
+        }
+        else {
+            base = queryTerms.length > 0 ? rawBm25 / queryTerms.length : rawBm25;
+            compositeScore = base * strengthMultiplier * recencyMultiplier;
+            modeLabel = 'bm25-only';
+        }
+        // Decision-tagged memories get a 1.2x recall boost
+        const decisionBoost = entries[i].tags.includes('decision') ? 1.2 : 1.0;
+        compositeScore *= decisionBoost;
+        // Path-based boost: memories tagged with matching path segments get up to 1.3x
+        const memPathTags = entries[i].tags.filter(t => t.startsWith('path:'));
+        const pathScore = pathOverlapScore(memPathTags, currentPathTags);
+        const pathBoost = 1.0 + (pathScore * 0.3);
+        compositeScore *= pathBoost;
+        // Retrieval-time outcome personalization: nudge up/down from user feedback.
+        // Distinct from reward-factor-via-strength (slow); this is immediate.
+        const pos = entries[i].outcome_positive ?? 0;
+        const neg = entries[i].outcome_negative ?? 0;
+        const outcomeBoost = pos === 0 && neg === 0
+            ? 1.0
+            : Math.max(0.85, Math.min(1.15, 1 + 0.15 * Math.tanh((pos - neg) / 2)));
+        compositeScore *= outcomeBoost;
+        // Scope boost: memories tagged with the active scope get 1.5x; mismatching scopes get 0.5x
+        const scopeSignal = scopeMatch(entries[i].tags, activeScope);
+        const scopeBoost = scopeSignal === 1 ? 1.5 : scopeSignal === -1 ? 0.5 : 1.0;
+        compositeScore *= scopeBoost;
+        const extractionBoost = entries[i].tags.includes('extracted') ? 1.3 : 1.0;
+        compositeScore *= extractionBoost;
+        compositeScore *= temporalBoost(entries[i], temporalDirAsync, temporalRangeAsync);
+        if (compositeScore <= 0)
+            continue;
+        const tokens = estimateTokens(entries[i].content);
+        const result = {
+            entry: entries[i],
+            score: compositeScore,
+            bm25: rawBm25,
+            cosine: cosineScore,
+            tokens,
+        };
+        if (explain) {
+            const docTerms = new Set(tokenize(`${entries[i].content} ${entries[i].tags.join(' ')}`));
+            const matchedTerms = [];
+            for (const t of queryTermSet)
+                if (docTerms.has(t))
+                    matchedTerms.push(t);
+            const ageDays = Math.max(0, Math.floor((now.getTime() - new Date(entries[i].created).getTime()) / 86_400_000));
+            result.breakdown = {
+                mode: modeLabel,
+                normBm25,
+                bm25Weight: useEmbeddings ? bm25Weight : 1,
+                embeddingWeight: useEmbeddings ? embeddingWeight : 0,
+                cosine: cosineScore,
+                base,
+                strengthMultiplier,
+                recencyMultiplier,
+                decisionBoost,
+                pathBoost,
+                scopeBoost,
+                sourceBump: 1,
+                outcomeBoost,
+                matchedTerms,
+                final: compositeScore,
+                ageDays,
+            };
+        }
+        scored.push(result);
+    }
+    // Sort by composite score descending
+    scored.sort((a, b) => b.score - a.score);
+    // Deduplicate: when an extracted fact and its source both appear,
+    // keep only the higher-scoring one (typically the fact).
+    const seenExtractedFrom = new Set();
+    const deduped = [];
+    for (const result of scored) {
+        const entry = result.entry;
+        if (entry.extracted_from) {
+            seenExtractedFrom.add(entry.extracted_from);
+            const sourceIdx = deduped.findIndex((d) => d.entry.id === entry.extracted_from);
+            if (sourceIdx >= 0)
+                deduped.splice(sourceIdx, 1);
+            deduped.push(result);
+        }
+        else if (seenExtractedFrom.has(entry.id)) {
+            continue;
+        }
+        else {
+            deduped.push(result);
+        }
+    }
+    const scoredDeduped = deduped;
+    // DAG drill-down: when a summary node matches, inject its children
+    const summaryIdsAsync = scoredDeduped
+        .filter((r) => r.entry.tags.includes('dag-summary'))
+        .map((r) => r.entry.id);
+    if (summaryIdsAsync.length > 0) {
+        const childEntriesAsync = entries.filter((e) => e.dag_parent_id && summaryIdsAsync.includes(e.dag_parent_id));
+        for (const child of childEntriesAsync) {
+            if (!scoredDeduped.some((r) => r.entry.id === child.id)) {
+                const parentResult = scoredDeduped.find((r) => r.entry.id === child.dag_parent_id);
+                const childScore = parentResult ? parentResult.score * 0.9 : 0;
+                scoredDeduped.push({
+                    entry: child,
+                    score: childScore,
+                    bm25: 0,
+                    cosine: 0,
+                    tokens: estimateTokens(child.content),
+                });
+            }
+        }
+        scoredDeduped.sort((a, b) => b.score - a.score);
+    }
+    // MMR re-ranking: de-cluster near-duplicates by trading relevance for
+    // diversity. Only applies when embeddings are loaded (doc-to-doc similarity
+    // is via cosine of cached vectors); otherwise we return the pure-relevance
+    // ordering unchanged.
+    //
+    // MMR is O(K^2) in cosine similarity ops, which on large corpora (1000+
+    // candidates) dominates query time. Cap the re-ranking window to the top
+    // relevance-scored candidates — anything below top-K was never going to
+    // surface anyway after budget filtering.
+    const MMR_CANDIDATE_CAP = 100;
+    const applyMmr = mmrEnabled && useEmbeddings && scoredDeduped.length > 1 && mmrLambda < 1;
+    let ordered;
+    if (applyMmr) {
+        const head = scoredDeduped.slice(0, MMR_CANDIDATE_CAP);
+        const tail = scoredDeduped.slice(MMR_CANDIDATE_CAP);
+        ordered = [...mmrRerank(head, embeddingIndex, mmrLambda, explain), ...tail];
+    }
+    else {
+        ordered = scoredDeduped;
+    }
+    // Apply token budget (guarantee at least minResults items)
+    const results = [];
+    let usedTokens = 0;
+    for (let i = 0; i < ordered.length; i++) {
+        const tokens = ordered[i].tokens;
+        if (results.length >= minResults && usedTokens + tokens > budget)
+            continue;
+        usedTokens += tokens;
+        results.push(ordered[i]);
+    }
+    return results;
+}
+/**
+ * MMR (Maximal Marginal Relevance) re-ranking.
+ *
+ * Iteratively picks the candidate that maximises
+ *   lambda * relevance - (1 - lambda) * max(cos(cand, picked))
+ *
+ * Inputs must already be sorted by relevance descending. When `explain` is
+ * true, attaches `preMmrRank` / `postMmrRank` to each result's breakdown.
+ * Exported for unit tests; production callers go through hybridSearch.
+ */
+export function mmrRerank(scored, embeddingIndex, lambda, explain) {
+    if (scored.length === 0)
+        return scored;
+    const maxScore = scored[0].score || 1;
+    const normScore = scored.map((r) => r.score / maxScore);
+    const vectors = scored.map((r) => embeddingIndex[r.entry.id] ?? null);
+    const picked = [];
+    const remaining = new Set(scored.map((_, i) => i));
+    while (remaining.size > 0) {
+        let bestIdx = -1;
+        let bestMmr = -Infinity;
+        for (const i of remaining) {
+            const rel = normScore[i];
+            let maxSim = 0;
+            const vi = vectors[i];
+            if (vi) {
+                for (const p of picked) {
+                    const vp = embeddingIndex[p.entry.id];
+                    if (!vp || vp.length !== vi.length)
+                        continue;
+                    const sim = Math.max(0, cosineSimilarity(vi, vp));
+                    if (sim > maxSim)
+                        maxSim = sim;
+                }
+            }
+            const mmr = lambda * rel - (1 - lambda) * maxSim;
+            if (mmr > bestMmr) {
+                bestMmr = mmr;
+                bestIdx = i;
+            }
+        }
+        if (bestIdx === -1)
+            break;
+        remaining.delete(bestIdx);
+        picked.push(scored[bestIdx]);
+    }
+    if (explain) {
+        const preRank = new Map();
+        scored.forEach((r, i) => preRank.set(r.entry.id, i + 1));
+        picked.forEach((r, i) => {
+            if (r.breakdown) {
+                r.breakdown.preMmrRank = preRank.get(r.entry.id);
+                r.breakdown.postMmrRank = i + 1;
+            }
+        });
+    }
+    return picked;
+}
+/**
+ * Physics-based search: scores memories using gravitational force, momentum,
+ * and cluster amplification. Falls back to classic hybrid for memories
+ * without physics state.
+ */
+export async function physicsSearch(query, entries, options = {}) {
+    const now = options.now ?? new Date();
+    const budget = options.budget ?? 4000;
+    const minResults = options.minResults ?? 1;
+    const config = options.physicsConfig ?? DEFAULT_PHYSICS_CONFIG;
+    const explain = options.explain ?? false;
+    if (entries.length === 0 || !options.hippoRoot)
+        return [];
+    // Get query embedding (use pre-computed if provided)
+    let queryVector = options.queryEmbedding ?? [];
+    if (queryVector.length === 0) {
+        if (!isEmbeddingAvailable()) {
+            return hybridSearch(query, entries, options);
+        }
+        const model = resolveEmbeddingModel(options.hippoRoot);
+        if (embeddingModelRequiresReindex(options.hippoRoot, model)) {
+            return hybridSearch(query, entries, options);
+        }
+        queryVector = await getEmbedding(query, model);
+        if (queryVector.length === 0) {
+            return hybridSearch(query, entries, options);
+        }
+    }
+    // Load physics state
+    let physicsMap;
+    try {
+        const db = openHippoDb(options.hippoRoot);
+        try {
+            physicsMap = loadPhysicsState(db);
+        }
+        finally {
+            closeHippoDb(db);
+        }
+    }
+    catch {
+        return hybridSearch(query, entries, options);
+    }
+    // Split entries into physics-enabled and classic
+    const physicsEntries = [];
+    const physicsParticles = [];
+    const classicEntries = [];
+    for (const entry of entries) {
+        const particle = physicsMap.get(entry.id);
+        if (particle
+            && particle.position.length > 0
+            && particle.position.length === queryVector.length
+            && particle.velocity.length === queryVector.length) {
+            physicsEntries.push(entry);
+            physicsParticles.push(particle);
+        }
+        else {
+            classicEntries.push(entry);
+        }
+    }
+    // Score physics-enabled memories
+    const physicsResults = [];
+    if (physicsParticles.length > 0) {
+        const scored = computePhysicsScores(physicsParticles, queryVector, config);
+        const entryMap = new Map(physicsEntries.map(e => [e.id, e]));
+        for (const s of scored) {
+            if (s.finalScore <= 0)
+                continue;
+            const entry = entryMap.get(s.memoryId);
+            if (!entry)
+                continue;
+            const result = {
+                entry,
+                score: s.finalScore,
+                bm25: 0,
+                cosine: s.baseScore,
+                tokens: estimateTokens(entry.content),
+            };
+            if (explain) {
+                const ageDays = Math.max(0, Math.floor((now.getTime() - new Date(entry.created).getTime()) / 86_400_000));
+                result.breakdown = {
+                    mode: 'physics',
+                    normBm25: 0,
+                    bm25Weight: 0,
+                    embeddingWeight: 1,
+                    cosine: s.baseScore,
+                    base: s.baseScore,
+                    strengthMultiplier: 1,
+                    recencyMultiplier: 1,
+                    decisionBoost: 1,
+                    pathBoost: 1,
+                    scopeBoost: 1,
+                    sourceBump: 1,
+                    outcomeBoost: 1,
+                    matchedTerms: [],
+                    final: s.finalScore,
+                    ageDays,
+                };
+            }
+            physicsResults.push(result);
+        }
+    }
+    // Score classic memories (no physics state)
+    const classicResults = classicEntries.length > 0
+        ? await hybridSearch(query, classicEntries, { ...options, budget: Infinity, explain })
+        : [];
+    // Normalize both pools to [0, 1] and merge
+    const merged = mergeScorePools(physicsResults, classicResults);
+    // Sort and apply budget
+    merged.sort((a, b) => b.score - a.score);
+    const results = [];
+    let usedTokens = 0;
+    for (let i = 0; i < merged.length; i++) {
+        const tokens = merged[i].tokens;
+        if (results.length >= minResults && usedTokens + tokens > budget)
+            continue;
+        usedTokens += tokens;
+        results.push(merged[i]);
+    }
+    return results;
+}
+/** Normalize two score pools to [0,1] and combine. */
+function mergeScorePools(poolA, poolB) {
+    const maxA = poolA.reduce((m, r) => Math.max(m, r.score), 1e-9);
+    const maxB = poolB.reduce((m, r) => Math.max(m, r.score), 1e-9);
+    const merged = [];
+    for (const r of poolA) {
+        merged.push({ ...r, score: r.score / maxA });
+    }
+    for (const r of poolB) {
+        merged.push({ ...r, score: r.score / maxB });
+    }
+    return merged;
+}
+/**
+ * Search entries using BM25 + strength + recency composite score.
+ * When embeddings are available and hippoRoot is provided, uses hybrid scoring.
+ * Returns results sorted by score, capped at token budget.
+ *
+ * Also updates retrieval metadata on returned entries (side effect: caller
+ * must persist the updated entries).
+ */
+export function search(query, entries, options = {}) {
+    // Synchronous path: BM25 only (no async hybrid)
+    const now = options.now ?? new Date();
+    const budget = options.budget ?? 4000;
+    const minResults = options.minResults ?? 1;
+    // Bi-temporal filtering
+    if (options.asOf) {
+        const asOfDate = new Date(options.asOf);
+        const successorValidFrom = new Map();
+        for (const e of entries) {
+            if (e.superseded_by) {
+                const successor = entries.find(s => s.id === e.superseded_by);
+                if (successor)
+                    successorValidFrom.set(e.id, successor.valid_from);
+            }
+        }
+        entries = entries.filter(e => {
+            if (new Date(e.valid_from) > asOfDate)
+                return false;
+            if (!e.superseded_by)
+                return true;
+            const succVf = successorValidFrom.get(e.id);
+            return succVf ? new Date(succVf) > asOfDate : true;
+        });
+    }
+    else if (!options.includeSuperseded) {
+        entries = entries.filter(e => !e.superseded_by);
+    }
+    if (entries.length === 0)
+        return [];
+    const queryTerms = tokenize(query);
+    if (queryTerms.length === 0)
+        return [];
+    // Build corpus from all entries (content + tags joined)
+    const texts = entries.map((e) => `${e.content} ${e.tags.join(' ')}`);
+    const corpus = buildCorpus(texts);
+    // Score each entry
+    const scored = [];
+    const currentPathTagsSync = extractPathTags(process.cwd());
+    const activeScopeSync = detectScope();
+    const temporalDir = detectTemporalDirection(query);
+    const temporalRangeSync = temporalDir ? computeTemporalRange(entries) : { minTime: 0, maxTime: 0 };
+    for (let i = 0; i < entries.length; i++) {
+        const bm25 = bm25Score(corpus, i, queryTerms);
+        if (bm25 <= 0)
+            continue;
+        const strength = calculateStrength(entries[i], now);
+        const recency = recencyBoost(entries[i], now);
+        // Composite: BM25 relevance * strength * recency
+        // Normalise BM25 against query term count to keep scale consistent
+        const normBm25 = queryTerms.length > 0 ? bm25 / queryTerms.length : bm25;
+        let composite = normBm25 * (0.5 + 0.5 * strength) * (0.8 + 0.2 * recency);
+        // Decision-tagged memories get a 1.2x recall boost
+        const decisionBoost = entries[i].tags.includes('decision') ? 1.2 : 1.0;
+        composite *= decisionBoost;
+        // Path-based boost: memories tagged with matching path segments get up to 1.3x
+        const memPathTagsSync = entries[i].tags.filter(t => t.startsWith('path:'));
+        const pathScoreSync = pathOverlapScore(memPathTagsSync, currentPathTagsSync);
+        const pathBoostSync = 1.0 + (pathScoreSync * 0.3);
+        composite *= pathBoostSync;
+        // Scope boost (sync path)
+        const scopeSignalSync = scopeMatch(entries[i].tags, activeScopeSync);
+        const scopeBoostSync = scopeSignalSync === 1 ? 1.5 : scopeSignalSync === -1 ? 0.5 : 1.0;
+        composite *= scopeBoostSync;
+        const extractionBoostSync = entries[i].tags.includes('extracted') ? 1.3 : 1.0;
+        composite *= extractionBoostSync;
+        composite *= temporalBoost(entries[i], temporalDir, temporalRangeSync);
+        const tokens = estimateTokens(entries[i].content);
+        scored.push({ entry: entries[i], score: composite, bm25, cosine: 0, tokens });
+    }
+    // Sort by composite score descending
+    scored.sort((a, b) => b.score - a.score);
+    const seenExtractedFromSync = new Set();
+    const dedupedSync = [];
+    for (const result of scored) {
+        const entry = result.entry;
+        if (entry.extracted_from) {
+            seenExtractedFromSync.add(entry.extracted_from);
+            const sourceIdx = dedupedSync.findIndex((d) => d.entry.id === entry.extracted_from);
+            if (sourceIdx >= 0)
+                dedupedSync.splice(sourceIdx, 1);
+            dedupedSync.push(result);
+        }
+        else if (seenExtractedFromSync.has(entry.id)) {
+            continue;
+        }
+        else {
+            dedupedSync.push(result);
+        }
+    }
+    // DAG drill-down: when a summary node matches, inject its children
+    const summaryIdsSync = dedupedSync
+        .filter((r) => r.entry.tags.includes('dag-summary'))
+        .map((r) => r.entry.id);
+    if (summaryIdsSync.length > 0) {
+        const childEntries = entries.filter((e) => e.dag_parent_id && summaryIdsSync.includes(e.dag_parent_id));
+        for (const child of childEntries) {
+            if (!dedupedSync.some((r) => r.entry.id === child.id)) {
+                const parentResult = dedupedSync.find((r) => r.entry.id === child.dag_parent_id);
+                const childScore = parentResult ? parentResult.score * 0.9 : 0;
+                dedupedSync.push({
+                    entry: child,
+                    score: childScore,
+                    bm25: 0,
+                    cosine: 0,
+                    tokens: estimateTokens(child.content),
+                });
+            }
+        }
+        dedupedSync.sort((a, b) => b.score - a.score);
+    }
+    // Apply token budget
+    const results = [];
+    let usedTokens = 0;
+    for (let i = 0; i < dedupedSync.length; i++) {
+        const tokens = dedupedSync[i].tokens;
+        if (results.length >= minResults && usedTokens + tokens > budget)
+            continue;
+        usedTokens += tokens;
+        results.push(dedupedSync[i]);
+    }
+    return results;
+}
+/**
+ * Update retrieval metadata on entries that were returned by a search.
+ * Returns the mutated copies (caller must persist to disk).
+ */
+export function markRetrieved(entries, now = new Date()) {
+    return entries.map((e) => {
+        if (e.superseded_by)
+            return e;
+        const updated = {
+            ...e,
+            retrieval_count: e.retrieval_count + 1,
+            last_retrieved: now.toISOString(),
+            // Extend half-life by +2 days per retrieval (PLAN.md)
+            half_life_days: e.half_life_days + 2,
+            // A stale memory that gets used again becomes live context.
+            confidence: e.confidence === 'stale' ? 'observed' : e.confidence,
+        };
+        updated.strength = calculateStrength(updated, now);
+        return updated;
+    });
+}
+/**
+ * Explain why a search result matched a query.
+ * Computes which query terms overlapped with the document and whether
+ * BM25 and/or embedding similarity contributed to the composite score.
+ */
+export function explainMatch(query, result) {
+    const queryTerms = new Set(tokenize(query));
+    const docTerms = new Set(tokenize(`${result.entry.content} ${result.entry.tags.join(' ')}`));
+    const matchedTerms = [];
+    for (const term of queryTerms) {
+        if (docTerms.has(term)) {
+            matchedTerms.push(term);
+        }
+    }
+    const hasBm25 = result.bm25 > 0;
+    const hasEmbedding = result.cosine > 0;
+    const parts = [];
+    if (hasBm25) {
+        parts.push(`BM25: matched terms [${matchedTerms.join(', ')}]`);
+    }
+    if (hasEmbedding) {
+        parts.push(`embedding similarity: ${result.cosine.toFixed(3)}`);
+    }
+    if (parts.length === 0) {
+        parts.push('no direct term or embedding match');
+    }
+    return {
+        reason: parts.join('; '),
+        matchedTerms,
+        hasBm25,
+        hasEmbedding,
+        cosineSimilarity: result.cosine,
+        envelope: {
+            kind: result.entry.kind ?? 'distilled',
+            scope: result.entry.scope ?? null,
+            owner: result.entry.owner ?? null,
+            artifact_ref: result.entry.artifact_ref ?? null,
+            session_id: result.entry.source_session_id ?? null,
+            confidence: result.entry.confidence ?? 'observed',
+        },
+    };
+}
+/**
+ * Compute text overlap ratio between two strings (Jaccard on token sets).
+ */
+export function textOverlap(a, b) {
+    const setA = new Set(tokenize(a));
+    const setB = new Set(tokenize(b));
+    if (setA.size === 0 && setB.size === 0)
+        return 1;
+    if (setA.size === 0 || setB.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const t of setA) {
+        if (setB.has(t))
+            intersection++;
+    }
+    const union = setA.size + setB.size - intersection;
+    return intersection / union;
+}
+//# sourceMappingURL=search.js.map