hippo-memory 1.72.0 → 2.1.0

package/dist/api/{recall-pipeline.js → recall/pipeline.js}

@@ -1,26 +1,32 @@
 // Ranking core shared by `hippo recall` and `hippo explain`: load, search, expand, re-rank and filter, with no
 // writes and no direct output. Callers own flag parsing, printing, budget fitting and persistence.
-import { evalNow } from '../core/ablation.js';
-import { oneCopyPerMemory } from './context-select.js';
-import { compareEntryIdentity, compareScoresDesc } from '../core/compare.js';
-import { isEmbeddingAvailable } from '../embeddings/local.js';
-import { activeGoalsWithPolicies } from '../store/goals.js';
-import { boostByGoals } from '../search/goal-boost.js';
-import { graphExpandRecall } from '../graph/recall.js';
-import { DEFAULT_GRAPH_STREAM_WEIGHT } from '../graph/stream.js';
-import { Layer } from '../core/memory.js';
-import { multihopSearch } from '../search/multihop.js';
-import { passesCliRecallScopeFilter } from '../core/recall-scope.js';
-import { currentEntries } from '../search/as-of.js';
-import { STRENGTH_RANK_FLOOR, STRENGTH_RANK_SPAN } from '../search/boosts.js';
-import { hybridSearch } from '../search/hybrid.js';
-import { physicsSearch } from '../search/physics-search.js';
-import { rankBothStores } from '../sharing/search-both.js';
-import { loadRecallSearchEntries, recallScopeFilter } from '../store/search-rows.js';
-import { isJsonNumber } from '../util/json.js';
-import { textOverlap, tokenize as tokenizeQuery } from '../util/tokenize.js';
-/** Ranks memories for a query as `hippo recall` does, through the `limit` slice. Reads stores, embeddings and physics state; writes and prints nothing
- *  (notes go to `ctx.note`). The caller supplies the cost function and reranker in `opts` and persists the returned goal-log rows. */
+import { evalNow } from '../../core/ablation.js';
+import { oneCopyPerMemory } from '../context-select.js';
+import { compareEntryIdentity, compareScoresDesc } from '../../core/compare.js';
+import { isEmbeddingAvailable } from '../../embeddings/local.js';
+import { activeGoalsWithPolicies } from '../../store/goals.js';
+import { boostByGoals } from '../../search/goal-boost.js';
+import { graphExpandRecall } from '../../graph/recall.js';
+import { DEFAULT_GRAPH_STREAM_WEIGHT } from '../../graph/stream.js';
+import { Layer } from '../../core/memory.js';
+import { multihopSearch } from '../../search/multihop.js';
+import { passesCliRecallScopeFilter } from '../../core/recall-scope.js';
+import { currentEntries } from '../../search/as-of.js';
+import { STRENGTH_RANK_FLOOR, STRENGTH_RANK_SPAN } from '../../search/boosts.js';
+import { hybridSearch, vectorSourceOf } from '../../search/hybrid.js';
+import { searchByEngine } from '../../search/engine.js';
+import { CONFLICT_PEER_MULTIPLIER, SALIENCE_MIN_MULTIPLIER, UTILITY_LENGTH_PENALTY_CAP, UTILITY_LENGTH_TOKENS, VALUE_MULT_MAX, VALUE_MULT_MIN, VALUE_MULT_SLOPE, } from '../../core/search-types.js';
+import { rankBothStores } from '../../sharing/search-both.js';
+import { loadRecallEntries, recallScopeFilter } from '../../store/search-rows.js';
+import { isJsonNumber } from '../../util/json.js';
+import { currentSignal } from '../../util/request-scope.js';
+import { textOverlap, tokenize as tokenizeQuery } from '../../util/tokenize.js';
+/**
+ * Ranks memories for a query as `hippo recall` does, through the `limit` slice. Reads stores, embeddings and physics
+ * state; writes and prints nothing
+ * (notes go to `ctx.note`). The caller supplies the cost function and reranker in `opts` and persists the returned
+ * goal-log rows.
+ */
 export async function rankRecall(ctx, opts) {
     const pool = loadRecallPool(ctx, opts);
     const state = { results: [], droppedPreRank: pool.dropped, graphAdded: 0, goalRecallLog: [] };
@@ -35,6 +41,8 @@ export async function rankRecall(ctx, opts) {
         halted,
     });
     state.results = await searchPool(ctx, opts, pool);
+    // A request past its deadline during the search has no caller left for the later stages.
+    currentSignal()?.throwIfAborted();
     if (opts.haltBefore === 'expand')
         return done(true);
     if (opts.graphHops)
@@ -65,10 +73,14 @@ function loadRecallPool(ctx, opts) {
     // An explicit --scope unlocks that envelope scope; the regex-only `<source>:private:*` deny is the JS half below.
     const requested = opts.explicitScope || undefined;
     const loadSuperseded = opts.includeSuperseded || Boolean(opts.asOf);
-    let local = loadRecallSearchEntries(ctx.hippoRoot, opts.query, undefined, ctx.tenantId, requested, 'additive', loadSuperseded);
-    let global = ctx.globalRoot
-        ? loadRecallSearchEntries(ctx.globalRoot, opts.query, undefined, ctx.tenantId, requested, 'additive', loadSuperseded)
-        : [];
+    const load = {
+        tenantId: ctx.tenantId,
+        requestedScope: requested,
+        explicitScopeMode: 'additive',
+        includeSuperseded: loadSuperseded,
+    };
+    let local = loadRecallEntries(ctx.hippoRoot, opts.query, load);
+    let global = ctx.globalRoot ? loadRecallEntries(ctx.globalRoot, opts.query, load) : [];
     // SQL-excluded rows are pre-candidate, so the total is taken before the JS filters, which normally drop nothing.
     const total = local.length + global.length;
     const passes = (e) => passesCliRecallScopeFilter(e.scope ?? null, requested);
@@ -79,54 +91,103 @@ function loadRecallPool(ctx, opts) {
     global = currentEntries(global, currentness);
     return { local, global, total, dropped: total - (local.length + global.length) };
 }
-async function searchPool(ctx, opts, pool) {
+function graphStreamSearch(ctx, opts, graphStream, pool) {
     const { query, budget, cost, includeSuperseded, asOf, activeScope: scope, search } = opts;
-    const { mmr, mmrLambda, minResults, explain } = search;
+    const { mmr, mmrLambda, minResults } = search;
     const globalRoot = pool.global.length > 0 ? ctx.globalRoot : undefined;
-    if (search.graphStream) {
-        // Without embeddings hybridSearch falls back to BM25-only and the stream is inert; say so rather than no-op.
-        if (!isEmbeddingAvailable()) {
-            ctx.note?.('[note] --graph-stream needs embeddings (rrf fusion); none available, so the graph stream is inert. Run `hippo embed` first.');
-        }
-        if (globalRoot)
-            ctx.note?.('[note] --graph-stream searches the local store only; global graph fusion is a follow-up.');
-        // With seedCount or fewer candidates every one is a seed and the stream degrades to the 2-list fusion.
-        return hybridSearch(query, pool.local, {
-            budget, cost, hippoRoot: ctx.hippoRoot, mmr, mmrLambda, minResults, scope, includeSuperseded, asOf,
-            scoring: 'rrf',
-            graphStream: { weight: DEFAULT_GRAPH_STREAM_WEIGHT, tenantId: ctx.tenantId, hops: search.graphStream.hops, seedCount: search.graphStream.seeds },
-        });
+    // Without embeddings hybridSearch falls back to BM25-only and the stream is inert; say so rather than no-op.
+    if (!isEmbeddingAvailable()) {
+        ctx.note?.('[note] --graph-stream needs embeddings (rrf fusion); none available, so the graph stream is inert. Run ' +
+            '`hippo embed` first.');
     }
-    if (search.multihop) {
-        // Unlike rankBothStores below, multihop ranks one pooled list, so a shared memory's two copies both compete.
-        const allEntries = oneCopyPerMemory(pool.local, pool.global, evalNow()).flat();
-        return multihopSearch(query, allEntries, { budget, cost, hippoRoot: ctx.hippoRoot, minResults, includeSuperseded, asOf });
+    if (globalRoot) {
+        ctx.note?.('[note] --graph-stream searches the local store only; global graph fusion is a follow-up.');
     }
+    // With seedCount or fewer candidates every one is a seed and the stream degrades to the 2-list fusion.
+    return hybridSearch(query, pool.local, {
+        budget, cost, hippoRoot: ctx.hippoRoot, mmr, mmrLambda, minResults, scope, includeSuperseded, asOf,
+        scoring: 'rrf',
+        graphStream: {
+            weight: DEFAULT_GRAPH_STREAM_WEIGHT,
+            tenantId: ctx.tenantId,
+            hops: graphStream.hops,
+            seedCount: graphStream.seeds,
+        },
+    });
+}
+function multihopPoolSearch(ctx, opts, pool) {
+    const { query, budget, cost, includeSuperseded, asOf, search } = opts;
+    const { minResults } = search;
+    // Unlike rankBothStores below, multihop ranks one pooled list, so a shared memory's two copies both compete.
+    const allEntries = oneCopyPerMemory(pool.local, pool.global, evalNow()).flat();
+    return multihopSearch(query, allEntries, {
+        budget,
+        cost,
+        hippoRoot: ctx.hippoRoot,
+        minResults,
+        includeSuperseded,
+        asOf,
+    });
+}
+function poolVectorCandidates(ctx, opts) {
     const requested = opts.explicitScope || undefined;
-    const vectorCandidates = {
+    return {
         tenantId: ctx.tenantId,
         scope: recallScopeFilter(requested, 'additive'),
-        includeSuperseded: includeSuperseded || Boolean(asOf),
+        includeSuperseded: opts.includeSuperseded || Boolean(opts.asOf),
         admit: (e) => passesCliRecallScopeFilter(e.scope ?? null, requested),
     };
-    if (search.usePhysics && !globalRoot) {
-        // Explain leaves minResults and includeSuperseded out to hold its output steady; asOf must reach physics or later rows leak.
-        const temporal = explain ? { asOf } : { minResults, includeSuperseded, asOf };
-        return physicsSearch(query, pool.local, {
-            budget, cost, hippoRoot: ctx.hippoRoot, physicsConfig: search.physicsConfig, scope, explain, vectorCandidates, ...temporal
-        });
-    }
+}
+async function searchPool(ctx, opts, pool) {
+    const { query, budget, cost, includeSuperseded, asOf, activeScope: scope, search } = opts;
+    const { mmr, mmrLambda, minResults, explain } = search;
+    const globalRoot = pool.global.length > 0 ? ctx.globalRoot : undefined;
+    if (search.graphStream)
+        return graphStreamSearch(ctx, opts, search.graphStream, pool);
+    if (search.multihop)
+        return multihopPoolSearch(ctx, opts, pool);
+    const vectorCandidates = poolVectorCandidates(ctx, opts);
     if (globalRoot) {
-        // The pool already holds searchBothHybrid's additive-scope load for each store, so rank it rather than read both stores again.
+        // The pool already holds searchBothHybrid's additive-scope load for each store, so rank it rather than read both
+        // stores again.
         return rankBothStores(query, { local: ctx.hippoRoot, global: globalRoot }, { local: pool.local, global: pool.global }, vectorCandidates, {
-            budget, cost, explain, mmr, mmrLambda, localBump: search.localBump, minResults, scope, tenantId: ctx.tenantId,
-            includeSuperseded, asOf,
+            budget,
+            cost,
+            explain,
+            mmr,
+            mmrLambda,
+            localBump: search.localBump,
+            minResults,
+            scope,
+            tenantId: ctx.tenantId,
+            includeSuperseded,
+            asOf,
         });
     }
-    return hybridSearch(query, pool.local, {
-        budget, cost, hippoRoot: ctx.hippoRoot, explain, mmr, mmrLambda, minResults, scope, includeSuperseded, asOf, vectorCandidates
+    const physics = search.usePhysics ? search.physicsConfig : null;
+    // Physics takes no MMR; explain drops minResults and includeSuperseded to hold its output steady; asOf must reach
+    // physics or later rows leak.
+    const tuning = !physics
+        ? { mmr, mmrLambda, minResults, includeSuperseded, asOf }
+        : explain
+            ? { asOf }
+            : { minResults, includeSuperseded, asOf };
+    return searchByEngine(hippoDbSource(ctx.hippoRoot), query, pool.local, physics, {
+        budget,
+        cost,
+        scope,
+        explain,
+        vectorCandidates,
+        ...tuning,
     });
 }
+/** The CLI core holds no store: it ranks on hippo.db under its root, as its loads above do. */
+function hippoDbSource(hippoRoot) {
+    const source = vectorSourceOf({ hippoRoot });
+    if (!source)
+        throw new Error('The CLI ranking core needs a hippo root to rank on');
+    return source;
+}
 /** Adds memories reached by walking the entity graph out from the lexical seeds. */
 function expandGraph(ctx, opts, graph, state) {
     if (graph.hops <= 0)
@@ -161,57 +222,67 @@ function traced(opts, prev, next, step) {
 function byScore(results) {
     return results.sort((a, b) => compareScoresDesc(a.score, b.score));
 }
+function demoteConflictPeers(opts, state) {
+    // Recorded conflicts only: a lexical-overlap gate once wrecked benchmark recall. Down-rank, never delete.
+    const presentIds = new Set(state.results.map((r) => r.entry.id));
+    state.results = byScore(state.results.map((r) => {
+        const hasPeerInResults = (r.entry.conflicts_with || []).some((peerId) => presentIds.has(peerId));
+        if (!hasPeerInResults)
+            return r;
+        const next = { ...r, score: r.score * CONFLICT_PEER_MULTIPLIER };
+        return traced(opts, r, next, {
+            stage: 'interference',
+            multiplier: CONFLICT_PEER_MULTIPLIER,
+            scoreBefore: r.score,
+            scoreAfter: next.score,
+        });
+    }));
+}
+function applyValueAware(opts, state) {
+    // Wider clamp than the always-on outcome boost, so outcome history can decide the order.
+    state.results = byScore(state.results.map((r) => {
+        const pos = r.entry.outcome_positive ?? 0;
+        const neg = r.entry.outcome_negative ?? 0;
+        if (pos === 0 && neg === 0)
+            return r;
+        const valueMult = Math.max(VALUE_MULT_MIN, Math.min(VALUE_MULT_MAX, 1 + VALUE_MULT_SLOPE * Math.tanh(pos - neg)));
+        const next = { ...r, score: r.score * valueMult };
+        return traced(opts, r, next, {
+            stage: 'value',
+            multiplier: valueMult,
+            scoreBefore: r.score,
+            scoreAfter: next.score,
+        });
+    }));
+}
+function applyUtilityRank(opts, state) {
+    // utility = score * (0.5 + 0.5 * strength) * (1 - min(0.3, tokens / 10000)); long evidence-rich rows pay for
+    // length.
+    state.results = byScore(state.results.map((r) => {
+        const strength = isJsonNumber(r.entry.strength) ? r.entry.strength : 1.0;
+        const utilityMult = (STRENGTH_RANK_FLOOR + STRENGTH_RANK_SPAN * strength) *
+            (1 - Math.min(UTILITY_LENGTH_PENALTY_CAP, (r.tokens || 0) / UTILITY_LENGTH_TOKENS));
+        const utility = r.score * utilityMult;
+        return traced(opts, r, { ...r, score: utility }, { stage: 'utility', multiplier: utilityMult, scoreBefore: r.score, scoreAfter: utility });
+    }));
+}
 function applyPfcRerankers(opts, state) {
     if (opts.evcAdaptive && state.results.length >= 2)
         state.results = evcAdaptive(opts.query, state.results);
     if (opts.filterConflicts) {
         dropUnless(state, (r) => !r.entry.superseded_by);
-        // Recorded conflicts only: a lexical-overlap gate once wrecked benchmark recall. Down-rank, never delete.
-        const presentIds = new Set(state.results.map((r) => r.entry.id));
-        state.results = byScore(state.results.map((r) => {
-            const hasPeerInResults = (r.entry.conflicts_with || []).some((peerId) => presentIds.has(peerId));
-            if (!hasPeerInResults)
-                return r;
-            const next = { ...r, score: r.score * CONFLICT_PEER_MULTIPLIER };
-            return traced(opts, r, next, { stage: 'interference', multiplier: CONFLICT_PEER_MULTIPLIER, scoreBefore: r.score, scoreAfter: next.score });
-        }));
-    }
-    if (opts.valueAware && state.results.length >= 1) {
-        // Wider clamp than the always-on outcome boost, so outcome history can decide the order.
-        state.results = byScore(state.results.map((r) => {
-            const pos = r.entry.outcome_positive ?? 0;
-            const neg = r.entry.outcome_negative ?? 0;
-            if (pos === 0 && neg === 0)
-                return r;
-            const valueMult = Math.max(VALUE_MULT_MIN, Math.min(VALUE_MULT_MAX, 1 + VALUE_MULT_SLOPE * Math.tanh(pos - neg)));
-            const next = { ...r, score: r.score * valueMult };
-            return traced(opts, r, next, { stage: 'value', multiplier: valueMult, scoreBefore: r.score, scoreAfter: next.score });
-        }));
-    }
-    if (opts.rerankUtility) {
-        // utility = score * (0.5 + 0.5 * strength) * (1 - min(0.3, tokens / 10000)); long evidence-rich rows pay for length.
-        state.results = byScore(state.results.map((r) => {
-            const strength = isJsonNumber(r.entry.strength) ? r.entry.strength : 1.0;
-            const utilityMult = (STRENGTH_RANK_FLOOR + STRENGTH_RANK_SPAN * strength) * (1 - Math.min(UTILITY_LENGTH_PENALTY_CAP, (r.tokens || 0) / UTILITY_LENGTH_TOKENS));
-            const utility = r.score * utilityMult;
-            return traced(opts, r, { ...r, score: utility }, { stage: 'utility', multiplier: utilityMult, scoreBefore: r.score, scoreAfter: utility });
-        }));
+        demoteConflictPeers(opts, state);
     }
+    if (opts.valueAware && state.results.length >= 1)
+        applyValueAware(opts, state);
+    if (opts.rerankUtility)
+        applyUtilityRank(opts, state);
 }
-// A recorded conflict peer in the same result list is cut to this share of its score.
-const CONFLICT_PEER_MULTIPLIER = 0.3;
-// The value-aware clamp is wider than the always-on outcome nudge, so outcome history can decide the order.
-const VALUE_MULT_SLOPE = 0.3;
-const VALUE_MULT_MIN = 0.7;
-const VALUE_MULT_MAX = 1.3;
-const UTILITY_LENGTH_PENALTY_CAP = 0.3;
-const UTILITY_LENGTH_TOKENS = 10000;
 // evcAdaptive acts when the top hits overlap this much; it keeps rows near the best score or covering the query.
 const NEAR_DUPLICATE_OVERLAP_MIN = 0.4;
 const SCORE_FLOOR_FRACTION = 0.5;
 const QUERY_COVERAGE_MIN = 0.6;
 const GOAL_TAG_BOOST = 1.5;
-const SALIENCE_MIN_MULTIPLIER = 0.5;
 /** When the top hits are near-duplicates (same topic, different facts), surface the newest on-topic row first. */
 function evcAdaptive(query, results) {
     const slice = results.slice(0, Math.min(3, results.length));
@@ -268,7 +339,11 @@ function applyGoalBoosts(ctx, opts, state) {
                 return r;
             const boosted = { ...r, score: r.score * GOAL_TAG_BOOST };
             return traced(opts, r, boosted, {
-                stage: 'goal', multiplier: GOAL_TAG_BOOST, scoreBefore: r.score, scoreAfter: r.score * GOAL_TAG_BOOST, note: `--goal ${goalTag}`
+                stage: 'goal',
+                multiplier: GOAL_TAG_BOOST,
+                scoreBefore: r.score,
+                scoreAfter: r.score * GOAL_TAG_BOOST,
+                note: `--goal ${goalTag}`,
             });
         }));
         return;
@@ -279,7 +354,11 @@ function applyGoalBoosts(ctx, opts, state) {
     const goalBoostTrace = opts.why ? new Map() : undefined;
     const session = { sessionId: opts.sessionId, tenantId: ctx.tenantId };
     // The log may name global rows; the store drops those when it writes, as it does for api.recall.
-    const boost = boostByGoals(state.results, activeGoalsWithPolicies(ctx.hippoRoot, session), { ...session, limit: opts.limit, trace: goalBoostTrace });
+    const boost = boostByGoals(state.results, activeGoalsWithPolicies(ctx.hippoRoot, session), {
+        ...session,
+        limit: opts.limit,
+        trace: goalBoostTrace,
+    });
     state.results = boost.results;
     state.goalRecallLog = boost.log;
     if (goalBoostTrace && goalBoostTrace.size > 0) {
@@ -297,7 +376,12 @@ function applySalience(opts, threshold, results) {
             return r;
         const mult = Math.max(SALIENCE_MIN_MULTIPLIER, count / threshold);
         const next = { ...r, score: r.score * mult };
-        return traced(opts, r, next, { stage: 'retrieval-count-downweight', multiplier: mult, scoreBefore: r.score, scoreAfter: next.score });
+        return traced(opts, r, next, {
+            stage: 'retrieval-count-downweight',
+            multiplier: mult,
+            scoreBefore: r.score,
+            scoreAfter: next.score,
+        });
     }));
 }
 function dropUnless(state, keep) {
@@ -305,4 +389,4 @@ function dropUnless(state, keep) {
     state.results = state.results.filter(keep);
     state.droppedPreRank += before - state.results.length;
 }
-//# sourceMappingURL=recall-pipeline.js.map
+//# sourceMappingURL=pipeline.js.map

package/dist/api/recall/recall.d.ts

@@ -0,0 +1,6 @@
+import type { RecallOpts, RecallResult } from './types.js';
+import { type StoreContext } from '../types.js';
+/** The one recall entry: ranks with the ranker `opts` names, then writes the recall, and records it in full for a
+ * surface that sets `recordAs`. Only the CLI core ranker writes last_retrieval_ids (contract lock). */
+export declare function retrieve(ctx: StoreContext, opts: RecallOpts): Promise<RecallResult>;
+//# sourceMappingURL=recall.d.ts.map