hippo-memory 1.61.0 → 1.62.0

package/dist/api/context-select.js

@@ -0,0 +1,344 @@
+// getContext's selection stages: the pinned-only branch, the strongest-first branch and the search branch.
+import { openHippoDb, closeHippoDb } from '../db.js';
+import { recallScopeFilter } from '../store/search-rows.js';
+import { loadIndex } from '../store/index-and-stats.js';
+import { calculateStrength } from '../memory.js';
+import { appendAuditEvent, auditQueryFields, isContentWorthStoring } from '../audit.js';
+import { rankBothStores } from '../shared.js';
+import { evalNow } from '../ablation.js';
+import { hybridSearch } from '../search/hybrid.js';
+import { physicsSearch } from '../search/physics-search.js';
+import { compareScoredResults } from '../compare.js';
+import { scopeMatch } from '../scope.js';
+import { loadConfig } from '../config.js';
+import { promptTokens, contentTokens, gatePromptRecall, } from '../prompt-recall.js';
+// Share and promote copy a memory to the global store under a new id, so equal content is the only link.
+// A pinned copy wins, then the stronger one after the ranking's own global discount; a tie keeps the local copy.
+export function oneCopyPerMemory(local, global, now) {
+    const score = (e, isGlobal) => calculateStrength(e, now) * (isGlobal ? 1 / 1.2 : 1);
+    const best = new Map();
+    const offer = (entry, isGlobal) => {
+        const held = best.get(entry.content);
+        const wins = !held || (held.entry.pinned !== entry.pinned
+            ? entry.pinned
+            : score(entry, isGlobal) > score(held.entry, held.isGlobal));
+        if (wins)
+            best.set(entry.content, { entry, isGlobal });
+    };
+    for (const e of local)
+        offer(e, false);
+    for (const e of global)
+        offer(e, true);
+    const kept = new Set([...best.values()].map((b) => b.entry));
+    return [local.filter((e) => kept.has(e)), global.filter((e) => kept.has(e))];
+}
+export const finiteOr = (v, dflt, min) => Number.isFinite(v) && v >= min ? v : dflt;
+/** Pins plus the prompt-recall or recent-N backfill; null means the block is empty. */
+export function selectPinned(ctx, opts, plan, left, pools, admission) {
+    const { obs, primaryIsGlobal } = plan;
+    // loadConfig is safe even when local isn't initialised — returns defaults.
+    const pinnedCfg = loadConfig(ctx.hippoRoot);
+    if (!pinnedCfg.pinnedInject.enabled) {
+        return null;
+    }
+    // Effective budget: explicit opts.budget wins over config, less what the sections took.
+    const effBudget = left;
+    const nowP = evalNow(); // honors HIPPO_FAKE_NOW (eval-only; see ablation.ts)
+    const localEntries = pools.local.entries;
+    const globalEntries = pools.global.entries;
+    obs?.offer(localEntries, primaryIsGlobal);
+    obs?.offer(globalEntries, true);
+    const [localPool, globalPool] = oneCopyPerMemory(localEntries, globalEntries, nowP);
+    obs?.dropMissing([...localEntries, ...globalEntries], [...localPool, ...globalPool], 'load', 'duplicate');
+    const picked = { items: [], ids: new Set(), used: 0 };
+    // Pinned entries are explicit user intent, the recent-N list an automatic
+    // backfill. Both loops share ONE budget and the recent loop runs first, so
+    // pins are ranked here and reserve their share before it can spend.
+    const pinnedLocal = localPool.filter((e) => e.pinned);
+    const pinnedGlobal = globalPool.filter((e) => e.pinned);
+    const rankedPinned = rankPinned(plan, pinnedLocal, pinnedGlobal, nowP);
+    const recentBudget = Math.max(0, effBudget - reservePinned(rankedPinned, effBudget));
+    // Prompt recall gates the backfill on the prompt instead of recency.
+    if (plan.promptRecallPending) {
+        const candidates = () => promptRecallCandidates(plan, pools, admission.admit, rankedPinned, nowP);
+        backfillFromPrompt(opts, plan, pinnedCfg, candidates, picked, recentBudget);
+    }
+    else if (plan.includeRecent > 0) {
+        backfillRecent(plan, localPool, globalPool, picked, recentBudget, nowP);
+    }
+    if (pinnedLocal.length === 0 &&
+        pinnedGlobal.length === 0 &&
+        picked.items.length === 0 &&
+        !admission.digestHidden()) {
+        return null;
+    }
+    admitWithinBudget(rankedPinned, picked, effBudget, obs);
+    return picked.items;
+}
+function rankPinned(plan, pinnedLocal, pinnedGlobal, nowP) {
+    return [
+        ...pinnedLocal.map((e) => ({ entry: e, isGlobal: plan.primaryIsGlobal })),
+        ...pinnedGlobal.map((e) => ({ entry: e, isGlobal: true })),
+    ]
+        .map(({ entry, isGlobal }) => {
+        const scopeSig = scopeMatch(entry.tags, plan.activeScope);
+        const sBst = scopeSig === 1 ? 1.5 : scopeSig === -1 ? 0.5 : 1.0;
+        return {
+            entry,
+            score: calculateStrength(entry, nowP) * (isGlobal ? 1 / 1.2 : 1) * sBst,
+            tokens: plan.price(entry, isGlobal),
+            isGlobal,
+        };
+    })
+        .sort(compareScoredResults);
+}
+// Mirrors the pin loop's continue-not-break so a big pin cannot block smaller ones from reserving, and dedupes by id
+// because a synced pin sits in both stores. A pin also in the recent slice is counted twice: recents under-fill, safely.
+function reservePinned(rankedPinned, effBudget) {
+    let pinnedReserve = 0;
+    const reservedIds = new Set();
+    for (const r of rankedPinned) {
+        if (reservedIds.has(r.entry.id))
+            continue;
+        if (pinnedReserve + r.tokens <= effBudget) {
+            pinnedReserve += r.tokens;
+            reservedIds.add(r.entry.id);
+        }
+    }
+    return pinnedReserve;
+}
+/** Skips ids already picked and rows past the budget, so a large row never blocks smaller ones behind it. */
+function admitWithinBudget(rows, picked, budget, obs) {
+    for (const r of rows) {
+        if (picked.ids.has(r.entry.id))
+            continue;
+        if (picked.used + r.tokens > budget) {
+            obs?.reject(r.entry, 'budget', 'budget', r.score, r.tokens);
+            continue;
+        }
+        picked.items.push(r);
+        picked.ids.add(r.entry.id);
+        picked.used += r.tokens;
+    }
+}
+function backfillFromPrompt(opts, plan, pinnedCfg, candidates, picked, recentBudget) {
+    const rawMetric = pinnedCfg.pinnedInject.promptRecallMetric;
+    const metric = rawMetric === 'cosine' ? 'cosine' : 'jaccard';
+    const gate = {
+        metric,
+        threshold: finiteOr(pinnedCfg.pinnedInject.promptRecallThreshold, 0.04, 0),
+        minShared: finiteOr(pinnedCfg.pinnedInject.promptRecallMinShared, 2, 0),
+        maxItems: finiteOr(pinnedCfg.pinnedInject.promptRecallMaxItems, 5, 1),
+    };
+    const p = promptTokens(opts.prompt ?? '');
+    if (p.size === 0)
+        return;
+    const candidateItems = candidates();
+    const gated = gatePromptRecall(p, candidateItems, gate);
+    plan.obs?.gated(p, candidateItems, gate, gated);
+    for (const g of gated) {
+        if (picked.ids.has(g.item.id))
+            continue;
+        const tokens = plan.price(g.item.entry, g.item.isGlobal, true);
+        if (picked.used + tokens > recentBudget) {
+            plan.obs?.reject(g.item.entry, 'budget', 'budget', g.score, tokens);
+            continue;
+        }
+        picked.items.push({ entry: g.item.entry, score: g.score, tokens, isGlobal: g.item.isGlobal, promptRecall: true });
+        picked.ids.add(g.item.id);
+        picked.used += tokens;
+    }
+}
+// Candidates came off the ambient load's own connection (the recall request), not a fresh open.
+function promptRecallCandidates(plan, pools, admit, rankedPinned, nowP) {
+    const { obs, primaryIsGlobal } = plan;
+    // A candidate carrying a pin's text would inject that memory a second time.
+    const pinnedText = new Set(rankedPinned.map((r) => r.entry.content));
+    const ineligibleReason = (e) => !admit(e) ? 'scope'
+        : e.pinned ? 'pinned'
+            : !isContentWorthStoring(e.content) ? 'quality'
+                : pinnedText.has(e.content) ? 'duplicate'
+                    : null;
+    const eligible = (e) => {
+        const why = ineligibleReason(e);
+        if (why !== null && why !== 'pinned')
+            obs?.reject(e, 'eligible', why);
+        return why === null;
+    };
+    obs?.offer(pools.local.recall ?? [], primaryIsGlobal, 'prompt-recall');
+    obs?.offer(pools.global.recall ?? [], true, 'prompt-recall');
+    const localEligible = (pools.local.recall ?? []).filter(eligible);
+    const globalEligible = (pools.global.recall ?? []).filter(eligible);
+    const [localCandidates, globalCandidates] = oneCopyPerMemory(localEligible, globalEligible, nowP);
+    obs?.dropMissing([...localEligible, ...globalEligible], [...localCandidates, ...globalCandidates], 'eligible', 'duplicate');
+    const seenCandidateIds = new Set();
+    const candidateItems = [];
+    // Local wins the id collision (a global row synced into the local store).
+    for (const e of localCandidates) {
+        if (seenCandidateIds.has(e.id))
+            continue;
+        seenCandidateIds.add(e.id);
+        candidateItems.push({ id: e.id, tokens: contentTokens(e.content), entry: e, isGlobal: primaryIsGlobal });
+    }
+    for (const e of globalCandidates) {
+        if (seenCandidateIds.has(e.id))
+            continue;
+        seenCandidateIds.add(e.id);
+        candidateItems.push({ id: e.id, tokens: contentTokens(e.content), entry: e, isGlobal: true });
+    }
+    return candidateItems;
+}
+function backfillRecent(plan, localPool, globalPool, picked, recentBudget, nowP) {
+    const recent = [
+        ...localPool.map((entry) => ({ entry, isGlobal: plan.primaryIsGlobal })),
+        ...globalPool.map((entry) => ({ entry, isGlobal: true })),
+    ]
+        // Newest first, then id: stable within one store, but same-millisecond rows fall to random ids across ingests.
+        .sort((a, b) => {
+        const byCreated = Date.parse(b.entry.created) - Date.parse(a.entry.created);
+        return byCreated !== 0 ? byCreated : b.entry.id.localeCompare(a.entry.id);
+    })
+        // Filter before slice so a junk row is backfilled past, not counted against N. Pins bypass the floor: a dropped
+        // pin's share of the shared budget would go to a backfilled row, and the pin loop could not win it back.
+        .filter(({ entry }) => entry.pinned || isContentWorthStoring(entry.content))
+        .slice(0, plan.includeRecent)
+        .map(({ entry, isGlobal }) => ({
+        entry,
+        score: calculateStrength(entry, nowP) * (isGlobal ? 1 / 1.2 : 1),
+        tokens: plan.price(entry, isGlobal),
+        isGlobal,
+    }));
+    admitWithinBudget(recent, picked, recentBudget, plan.obs);
+}
+/** No query: the strongest memories by strength, up to budget. */
+export function selectStrongest(plan, left, pools) {
+    const now = evalNow(); // honors HIPPO_FAKE_NOW (eval-only; see ablation.ts)
+    const [localPool, globalPool] = oneCopyPerMemory(pools.local.entries, pools.global.entries, now);
+    const localRanked = localPool
+        .map((e) => ({
+        entry: e,
+        score: calculateStrength(e, now),
+        tokens: plan.price(e, plan.primaryIsGlobal),
+        isGlobal: plan.primaryIsGlobal,
+    }))
+        .sort(compareScoredResults);
+    const globalRanked = globalPool
+        .map((e) => ({
+        entry: e,
+        score: calculateStrength(e, now) * (1 / 1.2),
+        tokens: plan.price(e, true),
+        isGlobal: true,
+    }))
+        .sort(compareScoredResults);
+    const combined = [...localRanked, ...globalRanked].sort(compareScoredResults);
+    const selected = [];
+    let used = 0;
+    for (const r of combined) {
+        if (used + r.tokens > left)
+            continue;
+        selected.push(r);
+        used += r.tokens;
+    }
+    return selected;
+}
+/** Real query: hybrid search over both stores, or physics/hybrid over the local rows; emits the 'recall' audit row. */
+export async function selectBySearch(ctx, plan, left, pools, admission) {
+    const minResults = plan.cost ? 0 : undefined; // a priced block skips an oversize top hit too, so the budget bounds it
+    const results = plan.hasGlobal && !plan.primaryIsGlobal
+        ? await searchBothStores(ctx, plan, left, minResults, pools, admission.bothStoresAdmit)
+        : await searchLocalRows(ctx, plan, left, minResults, pools.local.entries, admission.admit);
+    auditContextRecall(ctx, plan, results.length);
+    return results;
+}
+// The pools were admitted at load, before ranking, dedupe and budget: a post-filter would let an excluded row fill the
+// budget or shadow its admitted duplicate.
+async function searchBothStores(ctx, plan, left, minResults, pools, admit) {
+    const { cost, price } = plan;
+    const localIndex = loadIndex(ctx.hippoRoot);
+    const isGlobalHit = (e) => !localIndex.entries[e.id];
+    const roots = { local: ctx.hippoRoot, global: plan.globalRoot };
+    const merged = await rankBothStores(plan.query, roots, { local: pools.local.entries, global: pools.global.entries }, contextVectorSpec(ctx, plan, admit), {
+        budget: left,
+        minResults,
+        cost: cost && ((r) => price(r.entry, isGlobalHit(r.entry))),
+        scope: plan.activeScope,
+    });
+    return merged.map((r) => ({
+        entry: r.entry,
+        score: r.score,
+        tokens: price(r.entry, isGlobalHit(r.entry)),
+        isGlobal: isGlobalHit(r.entry),
+    }));
+}
+/** The vector arm under the lexical window's own tenant, scope and current-row rules. */
+function contextVectorSpec(ctx, plan, admit) {
+    return { tenantId: ctx.tenantId, scope: recallScopeFilter(plan.exactScope, 'exact'), includeSuperseded: false, admit };
+}
+async function searchLocalRows(ctx, plan, left, minResults, localEntries, admit) {
+    const { cost, price, primaryIsGlobal, query } = plan;
+    const ctxConfig = loadConfig(ctx.hippoRoot);
+    const usePhysicsCtx = ctxConfig.physics?.enabled !== false;
+    const localCost = cost && ((r) => price(r.entry, primaryIsGlobal));
+    const vectorCandidates = contextVectorSpec(ctx, plan, admit);
+    const ctxResults = usePhysicsCtx
+        ? await physicsSearch(query, localEntries, {
+            budget: left,
+            minResults,
+            cost: localCost,
+            hippoRoot: ctx.hippoRoot,
+            physicsConfig: ctxConfig.physics,
+            scope: plan.activeScope,
+            vectorCandidates,
+        })
+        : await hybridSearch(query, localEntries, {
+            budget: left,
+            minResults,
+            cost: localCost,
+            hippoRoot: ctx.hippoRoot,
+            scope: plan.activeScope,
+            vectorCandidates,
+        });
+    return ctxResults.map((r) => ({
+        entry: r.entry,
+        score: r.score,
+        tokens: price(r.entry, primaryIsGlobal),
+        isGlobal: primaryIsGlobal,
+    }));
+}
+// Same 'recall' op api.recall emits; the pinned-only and no-query branches never search, so they never emit.
+function auditContextRecall(ctx, plan, resultCount) {
+    const ctxRecallMetadata = {
+        ...auditQueryFields(plan.query),
+        results: resultCount,
+        mode: 'context',
+    };
+    if (plan.hasLocal) {
+        const localDb = openHippoDb(ctx.hippoRoot);
+        try {
+            appendAuditEvent(localDb, {
+                tenantId: ctx.tenantId,
+                actor: ctx.actor.subject,
+                op: 'recall',
+                metadata: ctxRecallMetadata,
+            });
+        }
+        finally {
+            closeHippoDb(localDb);
+        }
+    }
+    if (plan.hasGlobal && !plan.primaryIsGlobal) {
+        const globalDb = openHippoDb(plan.globalRoot);
+        try {
+            appendAuditEvent(globalDb, {
+                tenantId: ctx.tenantId,
+                actor: ctx.actor.subject,
+                op: 'recall',
+                metadata: ctxRecallMetadata,
+            });
+        }
+        finally {
+            closeHippoDb(globalDb);
+        }
+    }
+}
+//# sourceMappingURL=context-select.js.map

package/dist/api/context.d.ts

@@ -1,6 +1,7 @@
 import { type MemoryEntry } from '../memory.js';
 import type { ContextOpts, ContextResult } from './context-types.js';
 import type { Context } from './types.js';
+export { oneCopyPerMemory } from './context-select.js';
 /**
  * v39 S4: the secret half of the ambient policy on its own, for callers
  * that apply their own scope rule. A flagged row is only admitted inside its owning project;
@@ -9,7 +10,6 @@ import type { Context } from './types.js';
 export declare function ambientSecretAdmit(e: MemoryEntry, currentProjectName: string): boolean;
 /** Most rows per store a no-query context reads; past it, ranking and ambientState see the strongest by decay. */
 export declare const CONTEXT_CANDIDATE_CAP = 2000;
-export declare function oneCopyPerMemory(local: readonly MemoryEntry[], global: readonly MemoryEntry[], now: Date): [MemoryEntry[], MemoryEntry[]];
 /**
  * Assemble a context bundle: recalled memories (pinned-only / strength-sorted
  * fallback / hybrid search) + active task snapshot + session handoff + recent
@@ -23,7 +23,7 @@ export declare function oneCopyPerMemory(local: readonly MemoryEntry[], global:
  * Tenant scope: all `loadAllEntries` / snapshot / handoff / events reads use
  * `ctx.tenantId`. Cross-tenant rows are filtered out.
  *
- * Returns an empty result (`entries: []`, snapshot/handoff/events undefined)
+ * @returns An empty result (`entries: []`, snapshot/handoff/events undefined)
  * when there's nothing to surface (no memories AND no snapshot AND no handoff
  * AND no recent events).
  */