hippo-memory 1.64.0 → 1.65.0

package/README.md

@@ -994,7 +994,7 @@ Three benchmarks testing three different things. Full details in [`benchmarks/`]
 
 [LongMemEval](https://arxiv.org/abs/2410.10813) (ICLR 2025) is the industry-standard benchmark: 500 questions across 5 memory abilities, embedded in 115k+ token chat histories.
 
-**`hippo recall` (measured 2026-09-28 on hippo-memory 1.52.5).** On a default install, `hippo recall` puts an answer session in its top 5 for 85.6% of the 500 `_s` questions (95% CI 82.4 to 88.6), and 87.6% with the optional MiniLM embedder (84.6 to 90.4). Most of the gap to the scripts below is the default 4,000-token budget. Each memory in this test is a whole session, about 2,600 tokens at the median, so recall returns a median of 2 sessions. With the budget lifted, the same rankings score 96.8% (95.2 to 98.2) and 97.4% (96.0 to 98.6). A lifted call returns every candidate, a median of 47 sessions and 123,491 tokens per question, so those two figures measure the ranking, not an amount of text an agent could take in. Result: [`docs/evals/2026-09-28-recall-cli-longmemeval-result.md`](docs/evals/2026-09-28-recall-cli-longmemeval-result.md).
+**`hippo recall` (measured 2026-09-28 on hippo-memory 1.52.5, re-run 2026-10-06 on 1.64.0).** On a default install, `hippo recall` puts an answer session in its top 5 for 85.6% of the 500 `_s` questions (95% CI 82.4 to 88.6), and 87.8% with the optional MiniLM embedder (84.8 to 90.6; 87.6% on 1.52.5). Most of the gap to the scripts below is the default 4,000-token budget. Each memory in this test is a whole session, about 2,600 tokens at the median, so recall returns a median of 2 sessions. With the budget lifted, the same rankings score 96.8% (95.2 to 98.2) and 97.4% (96.0 to 98.6). A lifted call returns every candidate, a median of 47 sessions and 123,491 tokens per question, so those two figures measure the ranking, not an amount of text an agent could take in. Result: [`docs/evals/2026-09-28-recall-cli-longmemeval-result.md`](docs/evals/2026-09-28-recall-cli-longmemeval-result.md); re-run: [`docs/evals/2026-10-06-recall-cli-longmemeval-rerun-1.64.0.md`](docs/evals/2026-10-06-recall-cli-longmemeval-rerun-1.64.0.md).
 
 **The benchmark scripts (`_s` split; MiniLM re-measured 2026-09-23, voyage measured 2026-06-09).** Each question is scored against its own ~48-session haystack, the same way gbrain and other published systems report. The v1.23.0 pluggable embedding provider lets you choose the embedder:
 

package/dist/agent-memories/sync.js

@@ -5,7 +5,7 @@ import os from 'node:os';
 import path from 'node:path';
 import { errorMessage } from '../log.js';
 import { loadConfig } from '../config.js';
-import { closeHippoDb, isSqliteBusy, openHippoDb } from '../db.js';
+import { closeHippoDb, isSqliteBusy, openHippoDb, outsideRequestStores } from '../db.js';
 import { namesFoldedInto } from '../project-merge.js';
 import { isGlobalStoreRoot, originInSql, projectNames, resolveGlobalRootDir, resolveProjectIdentity } from '../project-identity.js';
 import { duplicateKey, heldTextKeys } from '../same-text.js';
@@ -169,7 +169,8 @@ function openTarget(target, hasItems, opts) {
 }
 function emptyStandIn(global) {
     const root = fs.mkdtempSync(path.join(os.tmpdir(), 'hippo-agent-memories-'));
-    const db = openHippoDb(root);
+    // Its folder is deleted at close, so a request scope must not keep the file open past that.
+    const db = outsideRequestStores(() => openHippoDb(root));
     return {
         db, root, global,
         close: () => {

package/dist/api/recall-record.d.ts

@@ -0,0 +1,46 @@
+import { type AppendAuditOpts, type AuditOp } from '../audit.js';
+import type { AvailabilityHint } from '../availability.js';
+import { RingBuffer, type AnchoringHint, type RecallHistorySnapshot } from '../recall-history.js';
+export type RecallSurface = 'cli' | 'mcp' | 'http';
+/** The session's ring, created on first use; null when anchoring is off or the call has no session. */
+export declare function sessionRing(surface: RecallSurface, tenantId: string, sessionId: string | undefined): RingBuffer | null;
+/** The session's history, read without creating or touching its ring, so a request that then fails evicts no live session. */
+export declare function peekSessionRing(surface: RecallSurface, tenantId: string, sessionId: string): RecallHistorySnapshot;
+/** Records this recall's top row and hint, which the next recall's cooldown reads. */
+export declare function noteRecall(ring: RingBuffer, query: string, topId: string | null, anchoredOn: string | undefined): void;
+export declare function resetSessionRings(surface: RecallSurface): void;
+export interface RecallAuditor {
+    readonly hippoRoot: string;
+    readonly tenantId: string;
+    readonly actor: string;
+    /** Log and count a failed write instead of throwing, for a CLI command that has already done its work. */
+    readonly bestEffort?: boolean;
+}
+/** Whom a recall audit row names; a surface that writes through the store port needs no root. */
+export type RecallAuditCaller = Pick<RecallAuditor, 'tenantId' | 'actor'>;
+type RecallAuditMetadata = Readonly<Record<string, string | number | null>>;
+export declare function recallAuditRow(who: RecallAuditCaller, op: AuditOp, targetId?: string, metadata?: RecallAuditMetadata): AppendAuditOpts;
+/** One audit row on its own short-lived handle. */
+export declare function appendRecallAudit(who: RecallAuditor, op: AuditOp, targetId?: string, metadata?: RecallAuditMetadata): void;
+export declare function recallAuditMetadata(query: string, results: number): {
+    results: number;
+    query_hash: string;
+    query_length: number;
+};
+/** No session means no ring; the row hashes with SHA-256/16, since hashQueryText is FNV-1a and easy to reverse on short queries. */
+export declare function anchorSkippedRows(who: RecallAuditCaller, query: string): AppendAuditOpts[];
+export declare function anchoringRows(who: RecallAuditCaller, hint: AnchoringHint | null): AppendAuditOpts[];
+export declare function availabilityRows(who: RecallAuditCaller, hint: AvailabilityHint | null): AppendAuditOpts[];
+export interface ShownRecall {
+    readonly query: string;
+    readonly ring: RingBuffer | null;
+    readonly topId: string | null;
+    readonly anchoring: AnchoringHint | null;
+    readonly availability: AvailabilityHint | null;
+}
+/** The hint rows for a list a surface showed, so a surface on the store port can hand them to the recall's one write. */
+export declare function shownRecallRows(who: RecallAuditCaller, shown: ShownRecall): AppendAuditOpts[];
+/** For a surface that computes its hints over the list it shows: feeds the ring after the final detect, then audits the hints. */
+export declare function recordShownRecall(who: RecallAuditor, shown: ShownRecall): void;
+export {};
+//# sourceMappingURL=recall-record.d.ts.map

package/dist/api/recall-record.js

@@ -0,0 +1,98 @@
+// The recall session rings and the recall audit rows, in one place for the CLI, MCP and HTTP surfaces.
+import { appendAuditEvent, auditQueryFields, reportAuditWriteFailure } from '../audit.js';
+import { closeHippoDb, openHippoDb } from '../db.js';
+import { appendRecall, biasHintEnabled, buildSessionKey, getOrCreateRing, hashQueryText, snapshotRing, } from '../recall-history.js';
+// One map per surface, each with its own LRU cap, so a repeat on one surface is a first recall on another.
+const rings = {
+    cli: new Map(),
+    mcp: new Map(),
+    http: new Map(),
+};
+/** The session's ring, created on first use; null when anchoring is off or the call has no session. */
+export function sessionRing(surface, tenantId, sessionId) {
+    if (!biasHintEnabled('anchoring') || !sessionId)
+        return null;
+    return getOrCreateRing(rings[surface], buildSessionKey(tenantId, sessionId));
+}
+/** The session's history, read without creating or touching its ring, so a request that then fails evicts no live session. */
+export function peekSessionRing(surface, tenantId, sessionId) {
+    const ring = rings[surface].get(buildSessionKey(tenantId, sessionId));
+    return ring ? snapshotRing(ring) : [];
+}
+/** Records this recall's top row and hint, which the next recall's cooldown reads. */
+export function noteRecall(ring, query, topId, anchoredOn) {
+    appendRecall(ring, hashQueryText(query), topId, anchoredOn);
+}
+export function resetSessionRings(surface) {
+    rings[surface].clear();
+}
+export function recallAuditRow(who, op, targetId, metadata) {
+    return { tenantId: who.tenantId, actor: who.actor, op, targetId, metadata };
+}
+function writeRecallAudit(who, row) {
+    try {
+        const db = openHippoDb(who.hippoRoot);
+        try {
+            appendAuditEvent(db, row);
+        }
+        finally {
+            closeHippoDb(db);
+        }
+    }
+    catch (err) {
+        if (!who.bestEffort)
+            throw err;
+        reportAuditWriteFailure(row.op, String(err), row.targetId);
+    }
+}
+/** One audit row on its own short-lived handle. */
+export function appendRecallAudit(who, op, targetId, metadata) {
+    writeRecallAudit(who, recallAuditRow(who, op, targetId, metadata));
+}
+// The row stores a hash of the query, never its text, so an archived memory's words cannot persist there.
+export function recallAuditMetadata(query, results) {
+    return { ...auditQueryFields(query), results };
+}
+/** No session means no ring; the row hashes with SHA-256/16, since hashQueryText is FNV-1a and easy to reverse on short queries. */
+export function anchorSkippedRows(who, query) {
+    if (!biasHintEnabled('anchoring'))
+        return [];
+    return [recallAuditRow(who, 'recall_anchor_skipped_no_session', undefined, auditQueryFields(query))];
+}
+export function anchoringRows(who, hint) {
+    if (hint?.reason === 'memory_dominance') {
+        return [recallAuditRow(who, 'recall_anchor_detected_memory_dominance', hint.memoryId, {
+                memory_id: hint.memoryId,
+                query_count: hint.queryCount ?? null,
+            })];
+    }
+    if (hint?.reason === 'query_repeat') {
+        return [recallAuditRow(who, 'recall_anchor_detected_query_repeat', hint.memoryId, { memory_id: hint.memoryId })];
+    }
+    return [];
+}
+export function availabilityRows(who, hint) {
+    if (!hint)
+        return [];
+    return [recallAuditRow(who, 'recall_availability_detected', undefined, {
+            recent_fraction: hint.recentFraction,
+            older_passed_over: hint.olderCandidatesPassedOver,
+            returned_count: hint.returnedCount,
+        })];
+}
+/** The hint rows for a list a surface showed, so a surface on the store port can hand them to the recall's one write. */
+export function shownRecallRows(who, shown) {
+    return [
+        ...(shown.ring ? [] : anchorSkippedRows(who, shown.query)),
+        ...anchoringRows(who, shown.anchoring),
+        ...availabilityRows(who, shown.availability),
+    ];
+}
+/** For a surface that computes its hints over the list it shows: feeds the ring after the final detect, then audits the hints. */
+export function recordShownRecall(who, shown) {
+    if (shown.ring)
+        noteRecall(shown.ring, shown.query, shown.topId, shown.anchoring?.memoryId);
+    for (const row of shownRecallRows(who, shown))
+        writeRecallAudit(who, row);
+}
+//# sourceMappingURL=recall-record.js.map

package/dist/api/recall-request.d.ts

@@ -0,0 +1,32 @@
+import { BadRequestError } from '../api-errors.js';
+import { type JsonValue } from '../json.js';
+import type { ContextOpts } from './context-types.js';
+import type { RecallOpts } from './recall-types.js';
+/** A recall or context input the caller must fix; MCP answers it with an isError result, HTTP with a 400. */
+export declare class RecallRequestError extends BadRequestError {
+    constructor(message: string);
+}
+type ParamValue = JsonValue | undefined;
+/** A request's named inputs; `queryName` is the surface's name for the query text. */
+export interface RequestParams {
+    readonly get: (name: string) => ParamValue;
+    readonly queryName: 'q' | 'query';
+}
+export declare function httpParams(query: URLSearchParams): RequestParams;
+export declare function toolParams(args: Readonly<Record<string, JsonValue>>): RequestParams;
+/** The recall options every surface passes through to retrieve(). */
+export type SharedRecallOpts = Pick<RecallOpts, 'query' | 'scope' | 'freshTailCount' | 'freshTailSessionId' | 'summarizeOverflow' | 'scorerWindow' | 'sessionId'> & {
+    includeContinuity: boolean;
+};
+export interface ParsedRecallRequest {
+    readonly opts: SharedRecallOpts;
+    readonly limit: number | undefined;
+    readonly mode: RecallOpts['mode'];
+    readonly explain: boolean;
+}
+export type ParsedContextRequest = Pick<ContextOpts, 'q' | 'budget' | 'limit' | 'pinnedOnly' | 'scope' | 'includeRecent' | 'crossProject'>;
+/** Checks in a fixed order, so the first bad input is the one reported. */
+export declare function parseRecallRequest(params: RequestParams): ParsedRecallRequest;
+export declare function parseContextRequest(params: RequestParams): ParsedContextRequest;
+export {};
+//# sourceMappingURL=recall-request.d.ts.map

package/dist/api/recall-request.js

@@ -0,0 +1,121 @@
+// The recall and context input rules in one copy, so MCP and HTTP reject the same inputs with the same messages.
+import { BadRequestError } from '../api-errors.js';
+import { MAX_ID_LEN } from '../http-util.js';
+import { isJsonString } from '../json.js';
+const MAX_RECALL_LIMIT = 1000;
+// recall() owns the shape rule for scorer_window (invalid_scorer_window); this cap bounds remote cost.
+const MAX_SCORER_WINDOW = 1000;
+// Bounds BM25 tokenisation cost while covering pasted error messages.
+const MAX_CONTEXT_QUERY_LEN = 1024;
+const RECALL_MODES = ['bm25', 'hybrid', 'physics'];
+/** A recall or context input the caller must fix; MCP answers it with an isError result, HTTP with a 400. */
+export class RecallRequestError extends BadRequestError {
+    constructor(message) {
+        super(message);
+        this.name = 'RecallRequestError';
+    }
+}
+export function httpParams(query) {
+    return { get: (name) => query.get(name) ?? undefined, queryName: 'q' };
+}
+export function toolParams(args) {
+    return { get: (name) => args[name], queryName: 'query' };
+}
+function numberParam(params, name) {
+    const value = params.get(name);
+    return value === undefined || value === null ? undefined : Number(value);
+}
+function nonNegativeParam(params, name) {
+    const n = numberParam(params, name);
+    if (n !== undefined && (!Number.isFinite(n) || n < 0))
+        throw new RecallRequestError(`${name} must be a non-negative number`);
+    return n;
+}
+// Only `true`, "1" and "true" turn a flag on, so `?summarize_overflow=banana` or an empty value stays off.
+function flagParam(params, name) {
+    const value = params.get(name);
+    return value === true || value === '1' || value === 'true';
+}
+function optionalFlagParam(params, name) {
+    const value = params.get(name);
+    return value === undefined || value === null ? undefined : flagParam(params, name);
+}
+function stringParam(params, name) {
+    const value = params.get(name);
+    return isJsonString(value) && value.length > 0 ? value : undefined;
+}
+/** An id-shaped string capped at MAX_ID_LEN; empty reads as absent. */
+function idParam(params, name) {
+    const value = params.get(name);
+    if (isJsonString(value) && value.length > MAX_ID_LEN)
+        throw new RecallRequestError(`${name} exceeds ${MAX_ID_LEN}-character cap`);
+    return stringParam(params, name);
+}
+function limitParam(params) {
+    const limit = numberParam(params, 'limit');
+    if (limit !== undefined && (!Number.isInteger(limit) || limit <= 0 || limit > MAX_RECALL_LIMIT)) {
+        throw new RecallRequestError(`limit must be a positive integer <= ${MAX_RECALL_LIMIT}`);
+    }
+    return limit;
+}
+function modeParam(params) {
+    const value = params.get('mode');
+    if (value === undefined || value === null)
+        return undefined;
+    const mode = RECALL_MODES.find((m) => m === value);
+    if (!mode)
+        throw new RecallRequestError("mode must be 'bm25', 'hybrid', or 'physics'");
+    return mode;
+}
+function scorerWindowParam(params) {
+    const scorerWindow = numberParam(params, 'scorer_window');
+    if (scorerWindow !== undefined && scorerWindow > MAX_SCORER_WINDOW) {
+        throw new RecallRequestError(`scorer_window must be <= ${MAX_SCORER_WINDOW}`);
+    }
+    return scorerWindow;
+}
+/** Trimmed, and absent when blank, so a whitespace id never keys a goal stack or a session ring. */
+function sessionIdParam(params) {
+    const value = idParam(params, 'session_id');
+    const trimmed = value?.trim() ?? '';
+    return trimmed.length > 0 ? trimmed : undefined;
+}
+/** Checks in a fixed order, so the first bad input is the one reported. */
+export function parseRecallRequest(params) {
+    const query = params.get(params.queryName);
+    if (!isJsonString(query) || query.length === 0)
+        throw new RecallRequestError(`${params.queryName} is required`);
+    const limit = limitParam(params);
+    const mode = modeParam(params);
+    const scope = stringParam(params, 'scope');
+    const includeContinuity = flagParam(params, 'include_continuity');
+    const freshTailCount = nonNegativeParam(params, 'fresh_tail_count');
+    const freshTailSessionId = idParam(params, 'fresh_tail_session_id');
+    const summarizeOverflow = optionalFlagParam(params, 'summarize_overflow');
+    const scorerWindow = scorerWindowParam(params);
+    const sessionId = sessionIdParam(params);
+    const explain = flagParam(params, 'explain');
+    return {
+        opts: { query, scope, includeContinuity, freshTailCount, freshTailSessionId, summarizeOverflow, scorerWindow, sessionId },
+        limit,
+        mode,
+        explain,
+    };
+}
+export function parseContextRequest(params) {
+    const qValue = params.get(params.queryName);
+    const q = isJsonString(qValue) ? qValue : undefined;
+    if (q !== undefined && q.length > MAX_CONTEXT_QUERY_LEN) {
+        throw new RecallRequestError(`${params.queryName} exceeds ${MAX_CONTEXT_QUERY_LEN}-character cap`);
+    }
+    const budget = nonNegativeParam(params, 'budget');
+    const limit = numberParam(params, 'limit');
+    if (limit !== undefined && (!Number.isFinite(limit) || limit <= 0))
+        throw new RecallRequestError('limit must be a positive number');
+    const pinnedOnly = flagParam(params, 'pinned_only');
+    const scope = idParam(params, 'scope');
+    const includeRecent = nonNegativeParam(params, 'include_recent');
+    const crossProject = flagParam(params, 'cross_project');
+    return { q, budget, limit, pinnedOnly, scope, includeRecent, crossProject };
+}
+//# sourceMappingURL=recall-request.js.map

package/dist/api/recall-types.d.ts

@@ -5,6 +5,7 @@ import type { RerankStep, SearchResult } from '../search/types.js';
 import type { PlanningFallacyHint, PlanningFallacyWatching } from '../predictions/planning-fallacy.js';
 import type { AnchoringHint, RecallHistorySnapshot } from '../recall-history.js';
 import type { AvailabilityHint } from '../availability.js';
+import type { AppendAuditOpts } from '../audit.js';
 import type { CallerProject } from '../prompt-hook.js';
 export interface RecallOpts {
     query: string;
@@ -152,7 +153,12 @@ export interface RecallOpts {
      *  then keeps a memory that a merged row in the same result holds word for word. Other callers leave it unset. */
     keepHeldCopies?: boolean;
     /** MCP recall only: `retrieve` ranks the whole scoped store and strengthens and traces (pipeline 'mcp') just the ids this returns; `results` stays the window band. */
-    showRanked?: (ranking: StoreRanking, result: RecallResult) => readonly string[];
+    showRanked?: (ranking: StoreRanking, result: RecallResult) => ShownRanking;
+}
+/** `ids`: what the caller showed; `audit`: its own rows, written after the recall's and in the same transaction, so all land or none. */
+export interface ShownRanking {
+    ids: readonly string[];
+    audit: readonly AppendAuditOpts[];
 }
 /** `ranked`: every scored row, best first, goal boost applied, entries as loaded; `pool`: the store after the scope filter. */
 export interface StoreRanking {

package/dist/api/recall.d.ts

@@ -16,7 +16,8 @@ export declare function buildSuppressionSummary(counts: {
     suppressedByInterference: number;
 }): RecallSuppressionSummary;
 /**
- * Domain-level recall. Loads BM25-ranked candidates from SQLite scoped to
+ * Domain-level recall on hippo.db only, never `ctx.store`, since it is synchronous: under another store its first open throws
+ * `SqliteBlockedError`. Loads BM25-ranked candidates from SQLite scoped to
  * `ctx.tenantId` and keeps that order whatever `mode` says; `retrieve` is the
  * mode-aware, strengthening variant the HTTP route uses.
  *
@@ -31,6 +32,6 @@ export declare function buildSuppressionSummary(counts: {
  * `tests/api-recall-no-side-effects.test.ts`.
  */
 export declare function recall(ctx: Context, opts: RecallOpts): RecallResult;
-/** Mode-aware recall that strengthens each returned row; never writes last_retrieval_ids (contract lock). */
+/** Mode-aware recall through the served store that strengthens each returned row; never writes last_retrieval_ids (contract lock). */
 export declare function retrieve(ctx: Context, opts: RecallOpts): Promise<RecallResult>;
 //# sourceMappingURL=recall.d.ts.map