hippo-memory 1.72.0 → 2.1.0

package/dist/api/recall/finish.js

@@ -0,0 +1,301 @@
+// What a recall remembers and records: the anchoring detector and query history, the session rings and audit rows, then
+// the write around one recall (history, ring, recalled counter and token ledger row).
+import { isRecallBoostAblated } from '../../core/ablation.js';
+import { auditQueryFields } from '../../store/audit.js';
+import { envAnchoringOff, envAvailabilityOff } from '../../util/env.js';
+import { recordTokens } from '../tokens.js';
+const DEFAULT_MIN_DOMINANCE = 3;
+const DEFAULT_RECENT_REPEAT_WINDOW = 5;
+const DEFAULT_COOLDOWN = 3;
+/**
+ * Normalizes a query (lowercase, strip non-alphanumerics, drop short tokens, sort, dedup, join) and hashes it with
+ * 32-bit FNV-1a.
+ * Reordered words collide intentionally; deterministic across processes and V8 versions. */
+export function hashQueryText(query) {
+    if (!query)
+        return 0;
+    // Tokens are deduped before join so `foo foo bar` and `foo bar` count as one query for memory_dominance.
+    // Unicode classes (\p{L}, \p{N}, \p{M}; need /u) keep non-Latin queries from collapsing to an empty token set and
+    // colliding at hash 0.
+    const normalized = query
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\p{M}\s]/gu, ' ')
+        .split(/\s+/)
+        .filter((t) => t.length > 0);
+    // Drop tokens under 3 chars (filler words) so `a login bug` and `login bug` hash
+    // alike; matches the >=3 filter in src/learn/forward-claim-detector.ts.
+    const filtered = normalized.filter((t) => t.length >= 3);
+    // Fall back to all tokens when the filter empties the query (CJK 2-char words,
+    // acronyms like `AI`) so distinct short queries do not all hash to fnv1a32('').
+    const tokens = filtered.length > 0 ? filtered : normalized;
+    const deduped = Array.from(new Set(tokens)).sort();
+    return fnv1a32(deduped.join(' '));
+}
+function fnv1a32(text) {
+    // FNV-1a 32-bit. Offset basis 2166136261, prime 16777619.
+    let hash = 2166136261;
+    for (let i = 0; i < text.length; i++) {
+        hash ^= text.charCodeAt(i);
+        // 32-bit multiply via Math.imul — handles overflow correctly.
+        hash = Math.imul(hash, 16777619);
+    }
+    // Coerce to unsigned 32-bit for stable comparison.
+    return hash >>> 0;
+}
+/**
+ * Detects anchoring against the current (queryHash, topMemoryId); memory_dominance wins when both rules fire, since it
+ * spans DIFFERENT queries.
+ * Cooldown is per memory: a hint on the SAME topMemoryId within `cooldown=3` history entries is suppressed. Returns an
+ * AnchoringHint or null.
+ */
+export function detectAnchoring(history, currentQueryHash, currentTopMemoryId, opts = {}) {
+    if (currentTopMemoryId === null)
+        return null;
+    const minDominance = opts.minDominance ?? DEFAULT_MIN_DOMINANCE;
+    const recentRepeatWindow = opts.recentRepeatWindow ?? DEFAULT_RECENT_REPEAT_WINDOW;
+    const cooldown = opts.cooldown ?? DEFAULT_COOLDOWN;
+    // Cooldown gate: was the most-recent hint (across the last `cooldown`
+    // entries) for THIS memory? If yes, suppress regardless of rule.
+    const cooldownSlice = history.slice(-cooldown);
+    for (const entry of cooldownSlice) {
+        if (entry.anchoredOn === currentTopMemoryId) {
+            return null;
+        }
+    }
+    // memory_dominance check FIRST (wins on tie).
+    const queryCount = countDistinctQueries(history, currentQueryHash, currentTopMemoryId);
+    if (queryCount >= minDominance) {
+        return {
+            reason: 'memory_dominance',
+            memoryId: currentTopMemoryId,
+            queryCount,
+            summary: `Memory ${currentTopMemoryId} has been the top result for ${queryCount} distinct queries in this ` +
+                `session and may be anchoring your reasoning.`,
+            source: 'j1-recurrence',
+        };
+    }
+    if (isRecentRepeat(history, recentRepeatWindow, currentQueryHash, currentTopMemoryId)) {
+        return {
+            reason: 'query_repeat',
+            memoryId: currentTopMemoryId,
+            summary: `Same query phrasing as a recent recall returned the same top result (${currentTopMemoryId}); you may ` +
+                `be re-asking the same question.`,
+            source: 'j1-recurrence',
+        };
+    }
+    return null;
+}
+// Count distinct queryHashes in history where topMemoryId === the current top
+// (excluding null tops), plus the current query itself.
+function countDistinctQueries(history, currentQueryHash, currentTopMemoryId) {
+    const matchingQueryHashes = new Set();
+    for (const entry of history) {
+        if (entry.topMemoryId === currentTopMemoryId) {
+            matchingQueryHashes.add(entry.queryHash);
+        }
+    }
+    matchingQueryHashes.add(currentQueryHash);
+    return matchingQueryHashes.size;
+}
+// query_repeat check: is currentQueryHash present in the last `recentRepeatWindow`
+// entries AND was that entry's topMemoryId === currentTopMemoryId?
+function isRecentRepeat(history, recentRepeatWindow, currentQueryHash, currentTopMemoryId) {
+    return history
+        .slice(-recentRepeatWindow)
+        .some((entry) => entry.queryHash === currentQueryHash && entry.topMemoryId === currentTopMemoryId);
+}
+const MAX_HISTORY = 10;
+const DEFAULT_MAX_SESSIONS = 1000;
+/**
+ * Bounded FIFO ring of RecallHistoryEntry (oldest evicted when full); a thin array wrapper so snapshotRing returns a
+ * readonly view without copying.
+ */
+export class RingBuffer {
+    entries = [];
+    append(entry) {
+        this.entries.push(entry);
+        if (this.entries.length > MAX_HISTORY) {
+            this.entries.shift();
+        }
+    }
+    snapshot() {
+        return this.entries;
+    }
+    size() {
+        return this.entries.length;
+    }
+}
+/** Builds the (tenant, session) key with a NUL delimiter: tenant and session ids are validated elsewhere to reject NUL,
+ * so concatenation cannot collide even though API-key-derived subjects can contain `:`. */
+export function buildSessionKey(tenantId, sessionId) {
+    return `${tenantId}\x00${sessionId}`;
+}
+/**
+ * Get-or-create a RingBuffer per session key, capped at `maxSessions` (default 1000); at the cap the oldest-inserted
+ * key (first from Map.keys()) is
+ * evicted. */
+export function getOrCreateRing(map, key, maxSessions = DEFAULT_MAX_SESSIONS) {
+    const existing = map.get(key);
+    if (existing) {
+        // LRU touch: delete + re-insert to move to back of iteration order.
+        map.delete(key);
+        map.set(key, existing);
+        return existing;
+    }
+    if (map.size >= maxSessions) {
+        const oldest = map.keys().next().value;
+        if (oldest !== undefined)
+            map.delete(oldest);
+    }
+    const ring = new RingBuffer();
+    map.set(key, ring);
+    return ring;
+}
+/**
+ * Appends a recall to a ring; `anchoredOn` is the memoryId of the hint that fired on THIS recall (or undefined), read
+ * by detectAnchoring for cooldown.
+ */
+export function appendRecall(ring, queryHash, topMemoryId, anchoredOn) {
+    const entry = {
+        queryHash,
+        topMemoryId,
+        ts: new Date().toISOString(),
+    };
+    if (anchoredOn !== undefined)
+        entry.anchoredOn = anchoredOn;
+    ring.append(entry);
+}
+/** Snapshot a ring as a readonly RecallHistorySnapshot. */
+export function snapshotRing(ring) {
+    return ring.snapshot();
+}
+/**
+ * Whether a recall bias hint is enabled; reads env at call time, so `HIPPO_ANCHORING=off` or `HIPPO_AVAILABILITY=off`
+ * disables only that kind.
+ */
+export function biasHintEnabled(kind) {
+    return kind === 'anchoring'
+        ? !envAnchoringOff()
+        : !envAvailabilityOff();
+}
+// entriesByIds reads at most this many ids a call.
+const ID_READ_CHUNK = 500;
+/** The ids among `ids` that a tenant's rows in `store` hold. */
+export async function heldIds(store, ids, tenantId) {
+    const held = new Set();
+    for (let i = 0; i < ids.length; i += ID_READ_CHUNK) {
+        for (const entry of await store.entriesByIds(ids.slice(i, i + ID_READ_CHUNK), tenantId))
+            held.add(entry.id);
+    }
+    return held;
+}
+// One map per surface, each with its own LRU cap, so a repeat on one surface is a first recall on another.
+const rings = {
+    cli: new Map(),
+    mcp: new Map(),
+    http: new Map(),
+};
+/** The session's ring, created on first use; null when anchoring is off or the call has no session. */
+export function sessionRing(surface, tenantId, sessionId) {
+    if (!biasHintEnabled('anchoring') || !sessionId)
+        return null;
+    return getOrCreateRing(rings[surface], buildSessionKey(tenantId, sessionId));
+}
+/**
+ * The session's history, read without creating or touching its ring, so a request that then fails evicts no live
+ * session.
+ */
+function peekSessionRing(surface, tenantId, sessionId) {
+    const ring = rings[surface].get(buildSessionKey(tenantId, sessionId));
+    return ring ? snapshotRing(ring) : [];
+}
+/** Records this recall's top row and hint, which the next recall's cooldown reads. */
+export function noteRecall(ring, query, topId, anchoredOn) {
+    appendRecall(ring, hashQueryText(query), topId, anchoredOn);
+}
+export function resetSessionRings(surface) {
+    rings[surface].clear();
+}
+export function callerOf(ctx) {
+    return { tenantId: ctx.tenantId, actor: ctx.actor.subject };
+}
+/** The strengthen every ranker of `retrieve` hands to the recall's write: the shown ids, in the caller's tenant. */
+export function strengthenOf(ctx, ids) {
+    return { ids, opts: { tenantId: ctx.tenantId, recallBoostAblated: isRecallBoostAblated() } };
+}
+export function recallAuditRow(who, op, targetId, metadata) {
+    return { tenantId: who.tenantId, actor: who.actor, op, targetId, metadata };
+}
+// The row stores a hash of the query, never its text, so an archived memory's words cannot persist there.
+export function recallAuditMetadata(query, results) {
+    return { ...auditQueryFields(query), results };
+}
+/**
+ * No session means no ring; the row hashes with SHA-256/16, since hashQueryText is FNV-1a and easy to reverse on short
+ * queries.
+ */
+export function anchorSkippedRows(who, query) {
+    if (!biasHintEnabled('anchoring'))
+        return [];
+    return [recallAuditRow(who, 'recall_anchor_skipped_no_session', undefined, auditQueryFields(query))];
+}
+export function anchoringRows(who, hint) {
+    if (hint?.reason === 'memory_dominance') {
+        return [recallAuditRow(who, 'recall_anchor_detected_memory_dominance', hint.memoryId, {
+                memory_id: hint.memoryId,
+                query_count: hint.queryCount ?? null,
+            })];
+    }
+    if (hint?.reason === 'query_repeat') {
+        return [recallAuditRow(who, 'recall_anchor_detected_query_repeat', hint.memoryId, { memory_id: hint.memoryId })];
+    }
+    return [];
+}
+export function availabilityRows(who, hint) {
+    if (!hint)
+        return [];
+    return [recallAuditRow(who, 'recall_availability_detected', undefined, {
+            recent_fraction: hint.recentFraction,
+            older_passed_over: hint.olderCandidatesPassedOver,
+            returned_count: hint.returnedCount,
+        })];
+}
+/**
+ * The hint rows for a list a surface showed, so a surface on the store port can hand them to the recall's one write.
+ */
+export function shownRecallRows(who, shown) {
+    return [
+        ...(shown.ring ? [] : anchorSkippedRows(who, shown.query)),
+        ...anchoringRows(who, shown.anchoring),
+        ...availabilityRows(who, shown.availability),
+    ];
+}
+/** The surfaces `retrieve` records in full. MCP is pending: src/mcp still feeds its own ring and books its own ledger
+ * row, and counts no stats. */
+export const RECALL_RECORDING = {
+    http: { ring: 'http', ledger: 'http_recall', stats: 'every' },
+    cli: { ring: 'cli', ledger: 'recall', stats: 'shown' },
+};
+/** For a surface whose hint `retrieve` judges: the session's history, or the row saying the recall named no session. */
+export function recallLead(ctx, how, query, sessionId) {
+    return {
+        // Peeked, never created: a recall that then fails must not LRU-evict a live session.
+        recallHistory: sessionId && biasHintEnabled('anchoring') ? peekSessionRing(how.ring, ctx.tenantId, sessionId) : undefined,
+        // Written first in the recall's own write, so a recall that fails leaves no row.
+        leadingAudit: sessionId ? [] : anchorSkippedRows(callerOf(ctx), query),
+    };
+}
+/** Runs after the recall's own write, in this order: the ring, then the counter, then the ledger. */
+export async function recordShownRecall(ctx, store, how, shown) {
+    // A surface that only peeked has its ring created here, after the
+    // write, so a recall that fails cannot LRU-evict a live session.
+    const ring = sessionRing(how.ring, ctx.tenantId, shown.sessionId);
+    if (ring)
+        noteRecall(ring, shown.query, shown.topId, shown.anchoredOn);
+    // A recall with no audit row counts nothing, as it strengthens and traces nothing.
+    if (shown.written && (how.stats === 'every' || shown.items > 0))
+        await store.bumpRecallStats(shown.items);
+    // The ledger books text that was sent, so its row does not wait on the audit write.
+    await recordTokens(ctx, how.ledger, { items: shown.items, tokens: shown.tokens, sessionId: shown.ledgerSessionId });
+}
+//# sourceMappingURL=finish.js.map

package/dist/api/{recall-pipeline.d.ts → recall/pipeline.d.ts}

@@ -1,8 +1,8 @@
-import { type GoalRecallLogRow } from '../store/goals.js';
-import { type MemoryEntry } from '../core/memory.js';
-import type { PhysicsConfig } from '../core/physics-config.js';
-import type { RerankerFn } from '../rerankers/types.js';
-import type { ResultCost, SearchResult } from '../core/search-types.js';
+import { type GoalRecallLogRow } from '../../store/goals.js';
+import { type MemoryEntry } from '../../core/memory.js';
+import type { PhysicsConfig } from '../../core/physics-config.js';
+import type { RerankerFn } from '../../rerankers/types.js';
+import { type ResultCost, type SearchResult } from '../../core/search-types.js';
 /** Stores rankRecall reads and where it sends operator notes. */
 export interface RankRecallCtx {
     hippoRoot: string;
@@ -94,7 +94,11 @@ export interface RankRecallResult {
     /** True when ranking stopped at `haltBefore`. */
     halted: boolean;
 }
-/** Ranks memories for a query as `hippo recall` does, through the `limit` slice. Reads stores, embeddings and physics state; writes and prints nothing
- *  (notes go to `ctx.note`). The caller supplies the cost function and reranker in `opts` and persists the returned goal-log rows. */
+/**
+ * Ranks memories for a query as `hippo recall` does, through the `limit` slice. Reads stores, embeddings and physics
+ * state; writes and prints nothing
+ * (notes go to `ctx.note`). The caller supplies the cost function and reranker in `opts` and persists the returned
+ * goal-log rows.
+ */
 export declare function rankRecall(ctx: RankRecallCtx, opts: RankRecallOpts): Promise<RankRecallResult>;
-//# sourceMappingURL=recall-pipeline.d.ts.map
+//# sourceMappingURL=pipeline.d.ts.map