hippo-memory 1.24.1 → 1.26.0

package/dist/src/cli.js

@@ -35,8 +35,9 @@ import { installJsonHooks, uninstallJsonHooks, resolveJsonHookPaths, detectInsta
 import { createMemory, calculateStrength, calculateRewardFactor, deriveHalfLife, resolveConfidence, computeSchemaFit, Layer, } from './memory.js';
 import { resolveProjectIdentity } from './project-identity.js';
 import { detectSecret } from './secret-detect.js';
-import { getHippoRoot, isInitialized, initStore, writeEntry, readEntry, deleteEntry, loadAllEntries, loadSearchEntries, loadIndex, saveIndex, loadStats, updateStats, saveActiveTaskSnapshot, loadActiveTaskSnapshot, clearActiveTaskSnapshot, appendSessionEvent, listSessionEvents, listMemoryConflicts, resolveConflict, saveSessionHandoff, loadLatestHandoff, loadHandoffById, RECALL_DEFAULT_DENY_SCOPES, } from './store.js';
-import { markRetrieved, hybridSearch, physicsSearch, explainMatch, textOverlap } from './search.js';
+import { getHippoRoot, isInitialized, initStore, writeEntry, readEntry, deleteEntry, loadAllEntries, loadSearchEntries, loadRecallSearchEntries, loadIndex, saveIndex, loadStats, updateStats, saveActiveTaskSnapshot, loadActiveTaskSnapshot, clearActiveTaskSnapshot, appendSessionEvent, listSessionEvents, listMemoryConflicts, resolveConflict, saveSessionHandoff, loadLatestHandoff, loadHandoffById, RECALL_DEFAULT_DENY_SCOPES, } from './store.js';
+import { markRetrieved, hybridSearch, physicsSearch, explainMatch, textOverlap, tokenize as tokenizeQuery } from './search.js';
+import { compareEntryIdentity } from './compare.js';
 import { renderTraceContent, parseSteps } from './trace.js';
 import { deduplicateStore } from './dedupe.js';
 import { isEmbeddingAvailable, embedAll, embedMemory, loadEmbeddingIndex, resolveEmbeddingModel, embeddingModelRequiresReindex, } from './embeddings.js';
@@ -722,15 +723,43 @@ async function cmdRecall(hippoRoot, query, flags) {
     // A5 stub auth: resolve the active tenant once and thread it through every
     // recall-time SELECT against `memories`. Cross-tenant rows must never surface.
     const tenantId = resolveTenantId({});
-    let localEntries = loadSearchEntries(hippoRoot, query, undefined, tenantId);
-    let globalEntries = isInitialized(globalRoot) ? loadSearchEntries(globalRoot, query, undefined, tenantId) : [];
+    // v1.25.0 (v39 follow-up #1): the direct CLI recall path applies the recall
+    // scope rule. SQL half via loadRecallSearchEntries in 'additive' mode —
+    // default-deny (`unknown:legacy` quarantine) always, and an explicit
+    // `--scope X` UNLOCKS envelope scope X on top of the default-admitted set
+    // (the CLI flag is historically a tag-boost hint over scope-NULL rows, so
+    // api.recall's narrowing exact-match would empty every tag-scoped recall —
+    // see passesCliRecallScopeFilter in recall-scope.ts). The regex-only
+    // `<source>:private:*` half is the JS post-filter below. Hoisted here
+    // (it previously lived with the boost flags): recallExplicitScope is the
+    // FILTER input; recallActiveScope (which falls back to detectScope()) stays
+    // boost-only — auto-detection must never become a filter input or
+    // detected-project recalls would change shape.
+    const recallExplicitScope = flags['scope'] !== undefined ? String(flags['scope']).trim() : null;
+    const requestedScopeForFilter = recallExplicitScope || undefined;
+    let localEntries = loadRecallSearchEntries(hippoRoot, query, undefined, tenantId, requestedScopeForFilter, 'additive');
+    let globalEntries = isInitialized(globalRoot) ? loadRecallSearchEntries(globalRoot, query, undefined, tenantId, requestedScopeForFilter, 'additive') : [];
     // v1.12.13 / C5 — WYSIATI counters. Track filter activity per the plan v3
     // Task 3 mapping table. dropped_pre_rank is the SUM of all non-budget
     // filter drops (pre-rank AND post-rank). Search-engine internal drops
     // (scored-to-zero rows that hybridSearch/physicsSearch returns fewer of)
     // are NOT counted in v1 — they are part of the rank step, not a filter.
+    // totalCandidates = post-SQL-predicate count (api.recall parity: measured
+    // after loadRecallSearchEntries, before the JS scope filter). NOTE the
+    // v1.12.13 accounting convention: SQL-excluded rows (quarantine + the
+    // v1.25.0 pre-window ':private:' exclusion) are pre-candidate and are NOT
+    // counted as drops; the JS half below normally drops 0 and exists as
+    // defense-in-depth (LIKE/regex divergence, exact-mode mismatch).
     const totalCandidatesCountCmd = localEntries.length + globalEntries.length;
     let droppedPreRankCountCmd = 0;
+    // v1.25.0: JS half of the recall scope rule (private-scope regex deny with
+    // explicit-request unlock), via the canonical helper — do not inline a
+    // fourth copy of this predicate.
+    const passesRecallScope = (e) => api.passesCliRecallScopeFilter(e.scope ?? null, requestedScopeForFilter);
+    const beforeScopeFilterCmd = localEntries.length + globalEntries.length;
+    localEntries = localEntries.filter(passesRecallScope);
+    globalEntries = globalEntries.filter(passesRecallScope);
+    droppedPreRankCountCmd += beforeScopeFilterCmd - (localEntries.length + globalEntries.length);
     // Bi-temporal filtering for physics path (hybridSearch handles it internally)
     if (asOf) {
         const filterAsOf = (entries) => {
@@ -782,7 +811,8 @@ async function cmdRecall(hippoRoot, query, flags) {
     const minResults = flags['min-results'] !== undefined
         ? parseInt(String(flags['min-results']), 10)
         : undefined;
-    const recallExplicitScope = flags['scope'] !== undefined ? String(flags['scope']).trim() : null;
+    // recallExplicitScope hoisted above the candidate loads (v1.25.0) — see the
+    // scope-filter block near the top of cmdRecall.
     const recallActiveScope = recallExplicitScope || detectScope();
     const useMultihop = flags['multihop'] === true || config.multihop.enabled;
     // L1 — graph-retrieval stream. `--graph-stream` forces rrf fusion + the graph stream
@@ -859,10 +889,16 @@ async function cmdRecall(hippoRoot, query, flags) {
         });
     }
     else if (hasGlobal) {
-        // Use searchBothHybrid for merged results with embedding support
+        // Use searchBothHybrid for merged results with embedding support.
+        // recallScope (v1.25.0): searchBothHybrid re-loads candidates internally,
+        // so the scope rule must be plumbed in — the filtered localEntries /
+        // globalEntries above are NOT what this path ranks.
         results = await searchBothHybrid(query, hippoRoot, globalRoot, {
             budget, mmr: mmrEnabled, mmrLambda, localBump, minResults, scope: recallActiveScope, tenantId,
             includeSuperseded, asOf,
+            recallScope: recallExplicitScope
+                ? { requested: recallExplicitScope, additive: true }
+                : {},
         });
     }
     else {
@@ -940,15 +976,39 @@ async function cmdRecall(hippoRoot, query, flags) {
             const tail = results.slice(poolSize);
             const maxScore = pool.reduce((m, r) => Math.max(m, r.score), 0);
             const scoreFloor = maxScore * 0.5;
+            // On-topic test: score floor OR query coverage. The score floor alone
+            // proxied topicality via ranking score, but the disambiguating update
+            // this mechanic exists to surface is BY NATURE phrased differently
+            // (weaker lexical/embedding overlap), so under the #t2 embed-text
+            // format (docs/plans/2026-07-09-recall-determinism.md T1, which
+            // de-compressed similarity gaps) it fell below any sane floor
+            // (measured 0.33x max on the acc-evc micro fixture). Query coverage —
+            // the fraction of query tokens present in the candidate — is the
+            // mechanic's own definition of "same topic, different fact" applied
+            // to the query, and is score-scale-independent. Principled EVC
+            // calibration remains roadmapped as B1 depth.
+            const queryTokens = new Set(tokenizeQuery(query));
             const onTopic = [];
             const offTopic = [];
             for (const r of pool) {
-                (r.score >= scoreFloor ? onTopic : offTopic).push(r);
+                let hits = 0;
+                if (queryTokens.size > 0) {
+                    const candTokens = new Set(tokenizeQuery(r.entry.content));
+                    for (const t of queryTokens)
+                        if (candTokens.has(t))
+                            hits++;
+                }
+                const queryCoverage = queryTokens.size > 0 ? hits / queryTokens.size : 0;
+                (r.score >= scoreFloor || queryCoverage >= 0.6 ? onTopic : offTopic).push(r);
             }
+            // Recency is the true primary key (unchanged) — that's the whole
+            // point of --evc-adaptive. compareEntryIdentity is only a TAIL for
+            // entries created at the exact same timestamp, which previously fell
+            // to array/scan order (T2, deterministic tie keys).
             onTopic.sort((a, b) => {
                 const ta = new Date(a.entry.created).getTime();
                 const tb = new Date(b.entry.created).getTime();
-                return tb - ta;
+                return tb !== ta ? tb - ta : compareEntryIdentity(a.entry, b.entry);
             });
             results = [...onTopic, ...offTopic, ...tail];
         }
@@ -985,6 +1045,11 @@ async function cmdRecall(hippoRoot, query, flags) {
             }
             return next;
         });
+        // T2 note: PLAIN stable score sort on purpose (here and in the rerank
+        // blocks below) -- these are RE-SORTS of an already deterministically-
+        // ordered ranking, so sort stability inherits the upstream content-tail
+        // determinism, and ties preserve the prior rank rather than reordering
+        // by content (a no-signal boost must not shuffle its input).
         results.sort((a, b) => b.score - a.score);
     }
     // vmPFC continuous value attribution (RESEARCH.md §PFC.vmPFC). Continuous
@@ -1013,7 +1078,7 @@ async function cmdRecall(hippoRoot, query, flags) {
             }
             return next;
         });
-        results.sort((a, b) => b.score - a.score);
+        results.sort((a, b) => b.score - a.score); // T2: plain stable re-sort, see --filter-conflicts note
     }
     // OFC option-value re-ranker MVP (RESEARCH.md §PFC.OFC). Combine relevance,
     // strength, and integration cost into a single utility score and re-sort.
@@ -1047,7 +1112,7 @@ async function cmdRecall(hippoRoot, query, flags) {
             }
             return next;
         })
-            .sort((a, b) => b.score - a.score);
+            .sort((a, b) => b.score - a.score); // T2: plain stable re-sort, see --filter-conflicts note
     }
     // F6 reranker pass (docs/plans/2026-05-10-f6-reranker-hardening.md). When
     // --reranker <name> is set, look up the reranker fn from the registry
@@ -1120,7 +1185,7 @@ async function cmdRecall(hippoRoot, query, flags) {
             }
             return boosted;
         })
-            .sort((a, b) => b.score - a.score);
+            .sort((a, b) => b.score - a.score); // T2: plain stable re-sort, see --filter-conflicts note
     }
     // dlPFC depth (B3, v0.38; lifted v1.7.4 into applyGoalStackBoost). When
     // HIPPO_SESSION_ID is set (env or --session-id flag) and the
@@ -1199,7 +1264,7 @@ async function cmdRecall(hippoRoot, query, flags) {
             }
             return next;
         })
-            .sort((a, b) => b.score - a.score);
+            .sort((a, b) => b.score - a.score); // T2: plain stable re-sort, see --filter-conflicts note
     }
     // --outcome filter: drop trace entries whose trace_outcome !== target.
     // Non-trace entries pass through unaffected (traces are the only layer with
@@ -1695,8 +1760,30 @@ async function cmdExplain(hippoRoot, query, flags) {
     const globalRoot = getGlobalRoot();
     // A5: scope explain results to the active tenant.
     const tenantId = resolveTenantId({});
-    let explainLocalEntries = loadSearchEntries(hippoRoot, query, undefined, tenantId);
-    let explainGlobalEntries = isInitialized(globalRoot) ? loadSearchEntries(globalRoot, query, undefined, tenantId) : [];
+    // v1.25.0 (v39 follow-up #1): cmdExplain shares cmdRecall's leak class —
+    // same unscoped loads, same fix, same semantics (explain explains what
+    // recall would see). Flag hoisted above the loads; the note below keeps the
+    // command honest for an operator debugging a hidden row.
+    const explainExplicitScope = flags['scope'] !== undefined ? String(flags['scope']).trim() : null;
+    const explainRequestedScope = explainExplicitScope || undefined;
+    let explainLocalEntries = loadRecallSearchEntries(hippoRoot, query, undefined, tenantId, explainRequestedScope, 'additive');
+    let explainGlobalEntries = isInitialized(globalRoot) ? loadRecallSearchEntries(globalRoot, query, undefined, tenantId, explainRequestedScope, 'additive') : [];
+    const passesExplainScope = (e) => api.passesCliRecallScopeFilter(e.scope ?? null, explainRequestedScope);
+    explainLocalEntries = explainLocalEntries.filter(passesExplainScope);
+    explainGlobalEntries = explainGlobalEntries.filter(passesExplainScope);
+    // Honesty note (grill finding #2): the SQL predicate excludes denied rows
+    // BEFORE the candidate window (codex P2 fix), so the pipeline never sees
+    // them. For the diagnostic note only, probe the unscoped window and count
+    // what the scope policy hides — window-capped, so the count is a floor on
+    // large stores, which is fine for a "why is my row missing" hint.
+    const explainUnscopedProbe = [
+        ...loadSearchEntries(hippoRoot, query, undefined, tenantId),
+        ...(isInitialized(globalRoot) ? loadSearchEntries(globalRoot, query, undefined, tenantId) : []),
+    ];
+    const explainScopeDropped = explainUnscopedProbe.filter((e) => !passesExplainScope(e)).length;
+    if (explainScopeDropped > 0) {
+        console.error(`[note] ${explainScopeDropped} candidate${explainScopeDropped === 1 ? '' : 's'} hidden by recall scope policy (pass an explicit --scope to inspect).`);
+    }
     // Bi-temporal filtering
     if (explainAsOf) {
         const filterAsOfExplain = (entries) => {
@@ -1739,7 +1826,7 @@ async function cmdExplain(hippoRoot, query, flags) {
         : flags['local-bump'] !== undefined
             ? parseFloat(String(flags['local-bump']))
             : config.search.localBump;
-    const explainExplicitScope = flags['scope'] !== undefined ? String(flags['scope']).trim() : null;
+    // explainExplicitScope hoisted above the candidate loads (v1.25.0).
     const explainActiveScope = explainExplicitScope || detectScope();
     let results;
     let modeUsed;
@@ -1757,6 +1844,9 @@ async function cmdExplain(hippoRoot, query, flags) {
         results = await searchBothHybrid(query, hippoRoot, globalRoot, {
             budget, explain: true, mmr: mmrEnabled, mmrLambda, localBump, scope: explainActiveScope,
             includeSuperseded: explainIncludeSuperseded, asOf: explainAsOf, tenantId,
+            recallScope: explainExplicitScope
+                ? { requested: explainExplicitScope, additive: true }
+                : {},
         });
         modeUsed = 'searchBothHybrid';
     }
@@ -2429,6 +2519,10 @@ export function renderSleepResult(result) {
     if (result.shared !== undefined && result.shared > 0) {
         console.log(`\nAuto-shared ${result.shared} high-value memories to global store.`);
     }
+    if (result.secretSkipped !== undefined && result.secretSkipped > 0) {
+        // v1.25.0 (v39 follow-up #2): the secret veto is no longer silent.
+        console.log(`\nAuto-share: withheld ${result.secretSkipped} secret-flagged ${result.secretSkipped === 1 ? 'memory' : 'memories'} (secret veto).`);
+    }
     if (result.ambient) {
         console.log(`\n${renderAmbientSummary(result.ambient)}`);
     }