hippo-memory 1.72.0 → 2.1.0

package/dist/api/recall/request.js

@@ -0,0 +1,261 @@
+// The recall and context input rules in one copy, so MCP and HTTP reject the same inputs with the same messages.
+// retrieveWithCliCore, the CLI ranking core, ranks through retrieve() and records the list the caller shows once.
+import { BadRequestError, ForbiddenError } from '../../core/api-errors.js';
+import { decidePlanningFallacy, detectPlanningClaim, } from '../../predictions/planning-fallacy.js';
+import { reportAuditWriteFailure } from '../../store/audit.js';
+import { isJsonString } from '../../util/json.js';
+import { MAX_ID_LEN } from '../../util/limits.js';
+import { estimateTokens } from '../../util/token-text.js';
+import { narrowContext } from '../types.js';
+import { rankRecall } from './pipeline.js';
+import { callerOf, heldIds, RECALL_RECORDING, recallAuditMetadata, recallAuditRow, recordShownRecall, strengthenOf, } from './finish.js';
+const MAX_RECALL_LIMIT = 1000;
+// recall() owns the shape rule for scorer_window (invalid_scorer_window); this cap bounds remote cost.
+const MAX_SCORER_WINDOW = 1000;
+// Bounds BM25 tokenisation cost while covering pasted error messages.
+const MAX_CONTEXT_QUERY_LEN = 1024;
+const RECALL_MODES = ['bm25', 'hybrid', 'physics'];
+/** A recall or context input the caller must fix; MCP answers it with an isError result, HTTP with a 400. */
+export class RecallRequestError extends BadRequestError {
+    constructor(message) {
+        super(message);
+        this.name = 'RecallRequestError';
+    }
+}
+export function httpParams(query) {
+    return { get: (name) => query.get(name) ?? undefined, queryName: 'q' };
+}
+export function toolParams(args) {
+    return { get: (name) => args[name], queryName: 'query' };
+}
+function numberParam(params, name) {
+    const value = params.get(name);
+    return value === undefined || value === null ? undefined : Number(value);
+}
+function nonNegativeParam(params, name) {
+    const n = numberParam(params, name);
+    if (n !== undefined && (!Number.isFinite(n) || n < 0)) {
+        throw new RecallRequestError(`${name} must be a non-negative number`);
+    }
+    return n;
+}
+// Only `true`, "1" and "true" turn a flag on, so `?summarize_overflow=banana` or an empty value stays off.
+function flagParam(params, name) {
+    const value = params.get(name);
+    return value === true || value === '1' || value === 'true';
+}
+function optionalFlagParam(params, name) {
+    const value = params.get(name);
+    return value === undefined || value === null ? undefined : flagParam(params, name);
+}
+function stringParam(params, name) {
+    const value = params.get(name);
+    return isJsonString(value) && value.length > 0 ? value : undefined;
+}
+/** An id-shaped string capped at MAX_ID_LEN; empty reads as absent. */
+function idParam(params, name) {
+    const value = params.get(name);
+    if (isJsonString(value) && value.length > MAX_ID_LEN) {
+        throw new RecallRequestError(`${name} exceeds ${MAX_ID_LEN}-character cap`);
+    }
+    return stringParam(params, name);
+}
+function limitParam(params) {
+    const limit = numberParam(params, 'limit');
+    if (limit !== undefined && (!Number.isInteger(limit) || limit <= 0 || limit > MAX_RECALL_LIMIT)) {
+        throw new RecallRequestError(`limit must be a positive integer <= ${MAX_RECALL_LIMIT}`);
+    }
+    return limit;
+}
+function modeParam(params) {
+    const value = params.get('mode');
+    if (value === undefined || value === null)
+        return undefined;
+    const mode = RECALL_MODES.find((m) => m === value);
+    if (!mode)
+        throw new RecallRequestError("mode must be 'bm25', 'hybrid', or 'physics'");
+    return mode;
+}
+function scorerWindowParam(params) {
+    const scorerWindow = numberParam(params, 'scorer_window');
+    if (scorerWindow !== undefined && scorerWindow > MAX_SCORER_WINDOW) {
+        throw new RecallRequestError(`scorer_window must be <= ${MAX_SCORER_WINDOW}`);
+    }
+    return scorerWindow;
+}
+/** Trimmed, and absent when blank, so a whitespace id never keys a goal stack or a session ring. */
+function sessionIdParam(params) {
+    const value = idParam(params, 'session_id');
+    const trimmed = value?.trim() ?? '';
+    return trimmed.length > 0 ? trimmed : undefined;
+}
+/** Checks in a fixed order, so the first bad input is the one reported. */
+export function parseRecallRequest(params) {
+    const query = params.get(params.queryName);
+    if (!isJsonString(query) || query.length === 0)
+        throw new RecallRequestError(`${params.queryName} is required`);
+    const limit = limitParam(params);
+    const mode = modeParam(params);
+    const scope = stringParam(params, 'scope');
+    const includeContinuity = flagParam(params, 'include_continuity');
+    const freshTailCount = nonNegativeParam(params, 'fresh_tail_count');
+    const freshTailSessionId = idParam(params, 'fresh_tail_session_id');
+    const summarizeOverflow = optionalFlagParam(params, 'summarize_overflow');
+    const scorerWindow = scorerWindowParam(params);
+    const sessionId = sessionIdParam(params);
+    const explain = flagParam(params, 'explain');
+    return {
+        opts: {
+            query,
+            scope,
+            includeContinuity,
+            freshTailCount,
+            freshTailSessionId,
+            summarizeOverflow,
+            scorerWindow,
+            sessionId,
+        },
+        limit,
+        mode,
+        explain,
+    };
+}
+export function parseContextRequest(params) {
+    const qValue = params.get(params.queryName);
+    const q = isJsonString(qValue) ? qValue : undefined;
+    if (q !== undefined && q.length > MAX_CONTEXT_QUERY_LEN) {
+        throw new RecallRequestError(`${params.queryName} exceeds ${MAX_CONTEXT_QUERY_LEN}-character cap`);
+    }
+    const budget = nonNegativeParam(params, 'budget');
+    const limit = numberParam(params, 'limit');
+    if (limit !== undefined && (!Number.isFinite(limit) || limit <= 0)) {
+        throw new RecallRequestError('limit must be a positive number');
+    }
+    const pinnedOnly = flagParam(params, 'pinned_only');
+    const scope = idParam(params, 'scope');
+    const includeRecent = nonNegativeParam(params, 'include_recent');
+    const crossProject = flagParam(params, 'cross_project');
+    return { q, budget, limit, pinnedOnly, scope, includeRecent, crossProject };
+}
+export async function retrieveWithCliCore(ctx, opts, core) {
+    // This ranker applies the operator's scope rule and may search a second store, so no bearer client may reach it.
+    if (!ctx.actor.hostAdmin)
+        throw new ForbiddenError('The CLI ranking core serves the host admin only');
+    const globalRoot = core.sources?.globalRoot;
+    // The ranking core reads hippo.db under hippoRoot, yet writes through the caller's store, whose finishRecall skips
+    // any id it does not hold.
+    const store = ctx.store;
+    const ranking = await rankRecall({ hippoRoot: ctx.hippoRoot, globalRoot, tenantId: ctx.tenantId, note: core.note }, { ...core.rank, query: opts.query, sessionId: opts.sessionId, goalTag: opts.goalTag });
+    if (ranking.halted) {
+        // The stages that ran earned these rows, so the caller's flag error does not drop them.
+        if (ranking.goalRecallLog.length > 0)
+            await store.finishRecall({ goalLog: ranking.goalRecallLog, audit: [] });
+        return resultOf(ranking, []);
+    }
+    if ('inspect' in core) {
+        core.inspect(ranking);
+        return resultOf(ranking, ranking.results);
+    }
+    const planning = await planningFor(ctx, store, opts.query);
+    const shown = await core.show(ranking, planning?.output ?? {});
+    await recordShown(ctx, store, opts, core, { ranking, shown, planningAudit: planning?.audit });
+    const result = resultOf(ranking, shown.results);
+    if (planning?.output.hint)
+        result.planningFallacyHint = planning.output.hint;
+    if (planning?.output.watching)
+        result.planningFallacyWatching = planning.output.watching;
+    return result;
+}
+async function planningFor(ctx, store, query) {
+    const claim = detectPlanningClaim(query);
+    if (!claim)
+        return null;
+    return decidePlanningFallacy(claim, await store.planningFallacyEvidence(ctx.tenantId, claim.classQueryTokens), ctx.tenantId, ctx.actor.subject);
+}
+function resultOf(ranking, shown) {
+    const results = shown.map((r) => ({
+        id: r.entry.id,
+        content: r.entry.content,
+        score: r.score,
+        layer: r.entry.layer,
+        strength: r.entry.strength,
+    }));
+    return {
+        results,
+        total: ranking.totalCandidates + ranking.graphAdded,
+        tokens: results.reduce((sum, r) => sum + estimateTokens(r.content), 0),
+    };
+}
+/** Audit rows first: a recall they cannot be written for leaves no
+ * strengthen, trace, marker or count, as under the other rankers. */
+async function recordShown(ctx, store, opts, core, done) {
+    const { ranking, shown, planningAudit } = done;
+    const ids = shown.results.map((r) => r.entry.id);
+    const recallRow = recallAuditRow(callerOf(ctx), 'recall', undefined, recallAuditMetadata(opts.query, ids.length));
+    const second = secondStore(ctx, core);
+    const heldElsewhere = second !== undefined ? await idsNotHeld(store, ids, ctx.tenantId) : [];
+    const writes = {
+        goalLog: ranking.goalRecallLog,
+        audit: [...(planningAudit ? [planningAudit] : []), ...shown.audit, recallRow],
+        strengthen: strengthenOf(ctx, ids),
+        trace: traceOf(ctx, opts, core, shown.results),
+    };
+    const audited = await finishOrReport(() => finishShown(ctx, writes, ids.length > 0));
+    if (audited) {
+        // The second store logs that it was searched, and strengthens only the rows the first does not hold.
+        if (second !== undefined) {
+            await finishOrReport(() => second.finishRecall({ goalLog: [], audit: [recallRow], strengthen: strengthenOf(ctx, heldElsewhere) }));
+        }
+    }
+    // The caller prints whether or not the record was written, so the ring and the ledger still take this list; `written`
+    // holds back the count.
+    await recordShownRecall(ctx, store, RECALL_RECORDING.cli, {
+        query: opts.query,
+        sessionId: opts.sessionId,
+        topId: ids[0] ?? null,
+        anchoredOn: shown.anchoredOn,
+        items: ids.length,
+        tokens: shown.tokens,
+        ledgerSessionId: core.hostSessionId ?? null,
+        written: audited,
+    });
+}
+/** The second store takes its own recall rows, unless it is the root being searched already. */
+function secondStore(ctx, core) {
+    const sources = core.sources;
+    return sources !== undefined && sources.globalRoot !== ctx.hippoRoot ? sources.globalStore : undefined;
+}
+// The caller has ranked and is about to print, so a failed audit write is logged and counted, never thrown.
+async function finishOrReport(finish) {
+    try {
+        await finish();
+        return true;
+    }
+    catch (err) {
+        reportAuditWriteFailure('recall', String(err));
+        return false;
+    }
+}
+async function idsNotHeld(store, ids, tenantId) {
+    const held = await heldIds(store, ids, tenantId);
+    return ids.filter((id) => !held.has(id));
+}
+// An empty list is traced too, so a coverage gap still reaches the training corpus.
+function traceOf(ctx, opts, core, shown) {
+    return {
+        tenantId: ctx.tenantId,
+        sessionId: opts.sessionId || core.hostSessionId || null,
+        pipeline: 'cli',
+        query: opts.query,
+        explainMode: core.rank.why === true,
+        results: shown.map((r) => ({ memoryId: r.entry.id, score: r.score, rerankSteps: r.rerankTrace })),
+    };
+}
+// A non-empty list is also the last recall, one write with its trace so `hippo outcome` never pairs these ids with an
+// older trace; an empty one leaves the marker on the previous recall.
+function finishShown(ctx, writes, marks) {
+    if (!marks)
+        return ctx.store.finishRecall(writes);
+    return narrowContext(ctx, 'lastRecall').store.lastRecall.finishRecall(writes, undefined);
+}
+//# sourceMappingURL=request.js.map

package/dist/api/{recall-cli.d.ts → recall/rows.d.ts}

@@ -1,11 +1,11 @@
-import { confidenceFacets } from '../core/memory.js';
-import type { ResultCost, SearchResult } from '../core/search-types.js';
-import type { RerankerFn } from '../rerankers/types.js';
-import { explainMatch } from '../search/explain.js';
-import type { ContinuityBlock } from '../store/port.js';
-import type { Context } from './types.js';
-export { DEFAULT_MAX_NEIGHBORS, MAX_HOPS } from '../graph/recall.js';
-/** The global store beside this one, whether this one is it, and the scope that boosts rows. */
+import { confidenceFacets } from '../../core/memory.js';
+import type { ResultCost, SearchResult } from '../../core/search-types.js';
+import type { RerankerFn } from '../../rerankers/types.js';
+import { explainMatch } from '../../search/explain.js';
+import type { ContinuityBlock } from '../../store/port.js';
+import type { Context, StoreContext } from '../types.js';
+export { DEFAULT_MAX_NEIGHBORS, MAX_HOPS } from '../../graph/recall.js';
+/** The global store beside this one ('' when the call reaches none), whether this one is it, and the boosting scope. */
 export interface CliRecallSetting {
     readonly globalRoot: string;
     readonly primaryIsGlobal: boolean;
@@ -15,7 +15,10 @@ export interface CliRecallSetting {
 export declare function cliRecallSetting(ctx: Context, explicitScope: string | null): CliRecallSetting;
 export interface CliRecallOrigin {
     readonly globalOn: boolean;
-    /** True when a ranked row came from the global store: this store is it, or the global store is on and this one lacks the id. */
+    /**
+     * True when a ranked row came from the global store: this store is it, or the global store is on and this one lacks
+     * the id.
+     */
     readonly isGlobal: (id: string) => boolean;
 }
 export declare function cliRecallOrigin(ctx: Context, setting: CliRecallSetting): CliRecallOrigin;
@@ -23,12 +26,15 @@ export declare function cliRecallOrigin(ctx: Context, setting: CliRecallSetting)
  *  window-capped, so this is a floor on a large store. */
 export declare function scopeHiddenCount(ctx: Context, query: string, globalRoot: string | undefined, requested: string | undefined): number;
 /** The active snapshot, its session's latest handoff and last five events, each kept only when its scope passes. */
-export declare function loadCliRecallContinuity(ctx: Context, setting: CliRecallSetting, wanted: boolean): ContinuityBlock;
+export declare function loadCliRecallContinuity(ctx: StoreContext, setting: CliRecallSetting, wanted: boolean): Promise<ContinuityBlock>;
 export interface CliRecallRerankerPick {
     readonly fn: RerankerFn;
     readonly defaultTopK: number;
 }
-/** The reranker a `--reranker` name picks and the top-k it takes by default; an unknown name throws, a blank one picks none. */
+/**
+ * The reranker a `--reranker` name picks and the top-k it takes by default; an unknown name throws, a blank one picks
+ * none.
+ */
 export declare function cliRecallReranker(name: string): CliRecallRerankerPick | null;
 /** The ranked rows a print budget holds, best first; the first `floor` stay whatever they cost. */
 export declare function fitRecallRows(ranked: SearchResult[], budget: number, floor: number, cost: ResultCost): SearchResult[];
@@ -55,4 +61,4 @@ export interface RecallJsonRow {
     rerankTrace?: SearchResult['rerankTrace'];
 }
 export declare function recallJsonRow(r: SearchResult, query: string, showWhy: boolean, isGlobal: boolean): RecallJsonRow;
-//# sourceMappingURL=recall-cli.d.ts.map
+//# sourceMappingURL=rows.d.ts.map

package/dist/api/{recall-cli.js → recall/rows.js}

@@ -1,26 +1,27 @@
-// What the `hippo recall` verb reads beside its ranking core: the stores it searches, the resume packet and the JSON rows.
-import { confidenceFacets, Layer } from '../core/memory.js';
-import { isGlobalStoreRoot, isInitialized } from '../core/project-identity.js';
-import { getReranker } from '../rerankers/index.js';
-import { isClefModel } from '../rerankers/clef.js';
-import { JEV_DEFAULT_TOP_K } from '../rerankers/jev.js';
-import { explainMatch } from '../search/explain.js';
-import { fitBudget } from '../search/finalize.js';
-import { getGlobalRoot } from '../sharing/global-store.js';
-import { detectScope } from '../core/active-scope.js';
-import { loadLatestHandoff } from '../store/handoffs.js';
-import { loadIndex } from '../store/index-and-stats.js';
-import { passesCliRecallScopeFilter, passesScopeFilterForRecall } from '../core/recall-scope.js';
-import { loadSearchEntries } from '../store/search-rows.js';
-import { listSessionEvents, loadActiveTaskSnapshot } from '../store/sessions.js';
-export { DEFAULT_MAX_NEIGHBORS, MAX_HOPS } from '../graph/recall.js';
+// What the `hippo recall` verb reads beside its ranking core: the stores it searches, the resume packet and the JSON
+// rows.
+import { confidenceFacets, Layer } from '../../core/memory.js';
+import { getReranker } from '../../rerankers/index.js';
+import { isClefModel } from '../../rerankers/clef.js';
+import { JEV_DEFAULT_TOP_K } from '../../rerankers/jev.js';
+import { explainMatch } from '../../search/explain.js';
+import { fitBudget } from '../../search/finalize.js';
+import { detectScope } from '../../core/active-scope.js';
+import { loadIndex } from '../../store/index-and-stats.js';
+import { passesCliRecallScopeFilter, passesScopeFilterForRecall } from '../../core/recall-scope.js';
+import { loadSearchEntries } from '../../store/search-rows.js';
+export { DEFAULT_MAX_NEIGHBORS, MAX_HOPS } from '../../graph/recall.js';
 /** The active scope merges `--scope` with the detected one, which only boosts. */
 export function cliRecallSetting(ctx, explicitScope) {
-    return { globalRoot: getGlobalRoot(), primaryIsGlobal: isGlobalStoreRoot(ctx.hippoRoot), activeScope: explicitScope || detectScope() };
+    return {
+        globalRoot: ctx.reach?.global?.root ?? '',
+        primaryIsGlobal: ctx.reach?.isGlobal ?? false,
+        activeScope: explicitScope || detectScope(),
+    };
 }
 export function cliRecallOrigin(ctx, setting) {
     const localIndex = loadIndex(ctx.hippoRoot);
-    const globalOn = isInitialized(setting.globalRoot);
+    const globalOn = ctx.reach?.global !== undefined;
     return { globalOn, isGlobal: (id) => setting.primaryIsGlobal || (globalOn && !localIndex.entries[id]) };
 }
 /** How many of `query`'s search candidates the scope rule hides from a recall asking for `requested`; the read is
@@ -33,14 +34,11 @@ export function scopeHiddenCount(ctx, query, globalRoot, requested) {
     return probe.filter((e) => !passesCliRecallScopeFilter(e.scope ?? null, requested)).length;
 }
 /** The active snapshot, its session's latest handoff and last five events, each kept only when its scope passes. */
-export function loadCliRecallContinuity(ctx, setting, wanted) {
-    if (!wanted || setting.primaryIsGlobal)
+export async function loadCliRecallContinuity(ctx, setting, wanted) {
+    if (!wanted || setting.primaryIsGlobal) {
         return { activeSnapshot: null, sessionHandoff: null, recentSessionEvents: [] };
-    const { hippoRoot, tenantId } = ctx;
-    const rawSnapshot = loadActiveTaskSnapshot(hippoRoot, tenantId);
-    const sessionId = rawSnapshot?.session_id ?? undefined;
-    const rawHandoff = sessionId ? loadLatestHandoff(hippoRoot, tenantId, sessionId) : null;
-    const rawEvents = sessionId ? listSessionEvents(hippoRoot, tenantId, { session_id: sessionId, limit: 5 }) : [];
+    }
+    const { activeSnapshot: rawSnapshot, sessionHandoff: rawHandoff, recentSessionEvents: rawEvents } = await ctx.store.continuity(ctx.tenantId, 5, null);
     // The same shared scope rule as api.recall; the active scope merges --scope and detectScope().
     const effectiveScope = setting.activeScope || undefined;
     const passes = (r) => passesScopeFilterForRecall(r.scope ?? null, effectiveScope);
@@ -50,7 +48,10 @@ export function loadCliRecallContinuity(ctx, setting, wanted) {
         recentSessionEvents: rawEvents.filter(passes),
     };
 }
-/** The reranker a `--reranker` name picks and the top-k it takes by default; an unknown name throws, a blank one picks none. */
+/**
+ * The reranker a `--reranker` name picks and the top-k it takes by default; an unknown name throws, a blank one picks
+ * none.
+ */
 export function cliRecallReranker(name) {
     const fn = getReranker(name);
     if (!fn)
@@ -100,4 +101,4 @@ export function recallJsonRow(r, query, showWhy, isGlobal) {
     }
     return base;
 }
-//# sourceMappingURL=recall-cli.js.map
+//# sourceMappingURL=rows.js.map