hippo-memory 1.62.0 → 1.63.1

package/dist/dedupe.js

@@ -22,7 +22,9 @@
  */
 import { textOverlap } from './tokenize.js';
 import { loadAllEntries } from './store/entry-reads.js';
-import { deleteEntry, memoriesBackingObjects } from './store/delete-and-batch.js';
+import { deleteEntryOn, memoriesBackingObjects } from './store/delete-and-batch.js';
+import { openStore } from './store/open.js';
+import { closeHippoDb } from './db.js';
 import { compareEntryIdentity } from './compare.js';
 import { canAutoDelete } from './memory.js';
 import { derivationPartitionKey } from './recall-scope.js';
@@ -56,6 +58,71 @@ export function strengthBucket(strength) {
     const s = strength ?? 0;
     return Number.isFinite(s) ? Math.round(s / STRENGTH_TIE_EPSILON) : 0;
 }
+// finiteCount mirrors strengthBucket's non-finite hardening on the
+// retrieval leg: a NaN retrieval_count would make the comparator return
+// NaN and break the total order the same way a NaN bucket would.
+// Unreachable via the schema (non-nullable INTEGER column), so this is
+// symmetry, not a live bug.
+const finiteCount = (n) => Number.isFinite(n ?? 0) ? (n ?? 0) : 0;
+/** The v1.26.3 survivor total order: strength bucket desc, then retrieval_count desc, then compareEntryIdentity. */
+function survivorOrder(a, b) {
+    const bucketDiff = strengthBucket(b.strength) - strengthBucket(a.strength);
+    if (bucketDiff !== 0)
+        return bucketDiff;
+    const retrievalDiff = finiteCount(b.retrieval_count) - finiteCount(a.retrieval_count);
+    if (retrievalDiff !== 0)
+        return retrievalDiff;
+    return compareEntryIdentity(a, b);
+}
+// Tenant partition (mirrors consolidate.ts mergeCandidatesByTenant and
+// dag.ts unparentedByTenant): group by tenantId BEFORE the sort so a
+// duplicate pair can never form across tenants. Map preserves insertion
+// order, so a single-tenant store (every row 'default') gets exactly one
+// group and the sort plus pair loop below run byte-identical to the
+// pre-fix global pass.
+function entriesByPartition(entries) {
+    const entriesByTenant = new Map();
+    for (const entry of entries) {
+        // EI2: also split by restricted scope, else a private copy can delete the readable one.
+        const key = derivationPartitionKey(entry.tenantId, entry.scope, entry.origin_project);
+        const bucket = entriesByTenant.get(key);
+        if (bucket)
+            bucket.push(entry);
+        else
+            entriesByTenant.set(key, [entry]);
+    }
+    return entriesByTenant;
+}
+function dedupPair(kept, dropped) {
+    return {
+        kept: kept.id,
+        keptContent: kept.content,
+        keptLayer: kept.layer,
+        keptStrength: kept.strength ?? 0,
+        removed: dropped.id,
+        removedContent: dropped.content,
+        removedLayer: dropped.layer,
+        removedStrength: dropped.strength ?? 0,
+        similarity: textOverlap(kept.content, dropped.content),
+    };
+}
+/** Pairs within one partition, already in survivor order, grouped by survivor; marks each loser in `removed`. */
+function partitionPairs(tenantEntries, removed, backing) {
+    const groups = new Map();
+    for (const entry of tenantEntries) {
+        const key = duplicateKey(entry.content);
+        const group = groups.get(key);
+        if (!group) {
+            groups.set(key, { survivor: entry, pairs: [] });
+            continue;
+        }
+        if (!canAutoDelete(entry) || backing.has(entry.id))
+            continue;
+        removed.add(entry.id);
+        group.pairs.push(dedupPair(group.survivor, entry));
+    }
+    return [...groups.values()].flatMap((group) => group.pairs);
+}
 /**
  * Scan the store for duplicates and remove the weaker copy: same text apart
  * from spacing, since a near-duplicate can differ in a value (port, version,
@@ -70,28 +137,7 @@ export function deduplicateStore(hippoRoot, options = {}) {
     // Only current distilled rows compete: raw rows are append-only (the delete
     // trigger would abort sleep mid-loop) and superseded rows are history, as in consolidate.ts.
     const entries = loadAllEntries(hippoRoot).filter((e) => e.kind === 'distilled' && !e.superseded_by);
-    // Tenant partition (mirrors consolidate.ts mergeCandidatesByTenant and
-    // dag.ts unparentedByTenant): group by tenantId BEFORE the sort so a
-    // duplicate pair can never form across tenants. Map preserves insertion
-    // order, so a single-tenant store (every row 'default') gets exactly one
-    // group and the sort plus pair loop below run byte-identical to the
-    // pre-fix global pass.
-    const entriesByTenant = new Map();
-    for (const entry of entries) {
-        // EI2: also split by restricted scope, else a private copy can delete the readable one.
-        const key = derivationPartitionKey(entry.tenantId, entry.scope, entry.origin_project);
-        const bucket = entriesByTenant.get(key);
-        if (bucket)
-            bucket.push(entry);
-        else
-            entriesByTenant.set(key, [entry]);
-    }
-    // finiteCount mirrors strengthBucket's non-finite hardening on the
-    // retrieval leg: a NaN retrieval_count would make the comparator return
-    // NaN and break the total order the same way a NaN bucket would.
-    // Unreachable via the schema (non-nullable INTEGER column), so this is
-    // symmetry, not a live bug.
-    const finiteCount = (n) => Number.isFinite(n ?? 0) ? (n ?? 0) : 0;
+    const entriesByTenant = entriesByPartition(entries);
     // Shared across tenant groups: safe because memory ids are globally
     // unique (crypto.randomUUID at creation), so an id in `removed` can never
     // collide with another tenant's row, and deleteEntry below deletes by
@@ -100,45 +146,24 @@ export function deduplicateStore(hippoRoot, options = {}) {
     const pairs = [];
     const backing = memoriesBackingObjects(hippoRoot);
     for (const tenantEntries of entriesByTenant.values()) {
-        // The v1.26.3 survivor total order (see the file-level docstring:
-        // strength bucket desc -> retrieval_count desc -> compareEntryIdentity),
-        // scoped per tenant group: the total order holds within a tenant only,
-        // matching the partition above.
-        tenantEntries.sort((a, b) => {
-            const bucketDiff = strengthBucket(b.strength) - strengthBucket(a.strength);
-            if (bucketDiff !== 0)
-                return bucketDiff;
-            const retrievalDiff = finiteCount(b.retrieval_count) - finiteCount(a.retrieval_count);
-            if (retrievalDiff !== 0)
-                return retrievalDiff;
-            return compareEntryIdentity(a, b);
-        });
-        const texts = tenantEntries.map((e) => duplicateKey(e.content));
-        for (let i = 0; i < tenantEntries.length; i++) {
-            if (removed.has(tenantEntries[i].id))
-                continue;
-            for (let j = i + 1; j < tenantEntries.length; j++) {
-                if (removed.has(tenantEntries[j].id) || !canAutoDelete(tenantEntries[j]) || backing.has(tenantEntries[j].id))
-                    continue;
-                if (texts[j] !== texts[i])
-                    continue;
-                const similarity = textOverlap(tenantEntries[i].content, tenantEntries[j].content);
-                removed.add(tenantEntries[j].id);
-                pairs.push({
-                    kept: tenantEntries[i].id,
-                    keptContent: tenantEntries[i].content,
-                    keptLayer: tenantEntries[i].layer,
-                    keptStrength: tenantEntries[i].strength ?? 0,
-                    removed: tenantEntries[j].id,
-                    removedContent: tenantEntries[j].content,
-                    removedLayer: tenantEntries[j].layer,
-                    removedStrength: tenantEntries[j].strength ?? 0,
-                    similarity,
-                });
-            }
-        }
+        // The survivor total order (see the file-level docstring), scoped per
+        // tenant group: the total order holds within a tenant only, matching the partition above.
+        tenantEntries.sort(survivorOrder);
+        pairs.push(...partitionPairs(tenantEntries, removed, backing));
     }
-    const done = dryRun ? pairs : pairs.filter((p) => deleteEntry(hippoRoot, p.removed, { actor: options.actor, reason: `dedup: duplicate of ${p.kept}`, automatic: true }));
+    const done = dryRun ? pairs : deletePairs(hippoRoot, pairs, options.actor);
     return { removed: done.length, pairs: done };
 }
+/** The pairs whose loser was deleted; one store handle, one transaction per delete. */
+function deletePairs(hippoRoot, pairs, actor) {
+    if (pairs.length === 0)
+        return [];
+    const db = openStore(hippoRoot);
+    try {
+        return pairs.filter((p) => deleteEntryOn(db, hippoRoot, p.removed, { actor, reason: `dedup: duplicate of ${p.kept}`, automatic: true }));
+    }
+    finally {
+        closeHippoDb(db);
+    }
+}
 //# sourceMappingURL=dedupe.js.map

package/dist/delivery-recorder.js

@@ -24,12 +24,7 @@ function byDepthThenScore(a, b) {
         return score;
     return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
 }
-/** A recorder for one call; every observer method is guarded, and a throw marks it broken instead of escaping. */
-export function createDeliveryRecorder(init) {
-    const startedMs = Date.now();
-    const ts = evalNow().toISOString();
-    // Test-only fault injection, as HIPPO_FAKE_NOW is for time.
-    const fault = envTestDeliveryFault();
+function readPayload(init) {
     const payloadSession = hookPayloadSessionId(init.stdinText);
     const subagent = isSubagentPayload(init.stdinText);
     const envSession = init.envSessionId !== undefined && init.envSessionId !== '' ? init.envSessionId : null;
@@ -40,148 +35,127 @@ export function createDeliveryRecorder(init) {
     const sessionState = subagent
         ? 'subagent'
         : payloadSession !== null ? 'payload' : envSession !== null ? 'env' : 'missing';
-    const candidates = new Map();
-    const picked = new Map();
-    const filtered = new Set();
-    let facts = null;
-    let shown = 0;
-    let dropped = 0;
-    let disabledSeen = false;
-    let outcome = { state: 'empty' };
-    let broken = null;
-    let flushed = false;
-    const guard = (fn) => {
-        if (broken !== null)
-            return;
-        try {
-            if (fault === 'observe')
-                throw new Error('injected observe fault');
-            fn();
-        }
-        catch (error) {
-            broken = error instanceof Error ? error.message : String(error);
-        }
-    };
-    const rejectId = (id, stage, reason, score, tokens) => {
-        const held = candidates.get(id);
-        // First rejection wins; an id never offered is not a candidate of this call.
-        if (!held || held.reason !== null)
-            return;
-        candidates.set(id, { ...held, stage, reason, score, tokens });
-    };
-    const build = () => {
-        if (fault === 'build')
-            throw new Error('injected build fault');
-        const staticReused = outcome.staticReused === true;
-        const rows = [];
-        for (const p of picked.values()) {
-            const held = candidates.get(p.entry.id);
-            rows.push({
-                memoryId: p.entry.id,
-                sourceStore: p.sourceStore,
-                pool: p.promptRecall ? 'prompt-recall' : held?.pool === 'pin' || p.entry.pinned ? 'pin' : 'recent',
-                stage: 'final',
-                outcome: staticReused && !p.promptRecall ? 'reused' : 'emitted',
-                reason: null,
-                rank: p.rank,
-                score: p.score,
-                tokens: p.tokens,
-            });
-        }
-        const rejected = [];
-        let undecided = 0;
-        for (const c of candidates.values()) {
-            if (picked.has(c.id))
-                continue;
-            if (c.reason === null)
-                undecided += 1;
-            else
-                rejected.push(c);
-        }
-        rejected.sort(byDepthThenScore);
-        for (const c of rejected.slice(0, DELIVERY_REJECTED_ROW_CAP)) {
-            rows.push({
-                memoryId: c.id, sourceStore: c.sourceStore, pool: c.pool, stage: c.stage ?? 'load', outcome: 'rejected',
-                reason: c.reason, rank: null, score: c.score, tokens: c.tokens,
-            });
-        }
-        const overflow = Math.max(0, rejected.length - DELIVERY_REJECTED_ROW_CAP);
-        const emitted = outcome.emittedText ?? null;
-        return {
-            ts,
-            tenantId: init.tenantId,
-            runtime: hostTurnId !== null ? 'codex' : hookEvent !== null ? 'claude-code' : 'unknown',
-            eventType: hookEvent === 'UserPromptSubmit' ? 'prompt-submit' : 'pinned-manual',
-            surface: 'hook',
-            storeHash: init.storeHash,
-            writeStore: init.writeStore,
-            projectHash: facts !== null && facts.projectName !== '' ? blockHash(facts.projectName) : null,
-            sessionId: payloadSession ?? envSession,
-            sessionState,
-            hostTurnId,
-            promptHash: prompt !== null ? blockHash(prompt) : null,
-            promptLength: prompt?.length ?? 0,
-            queryHash: null,
-            recallTraceId: null,
-            blockState: disabledSeen ? 'disabled' : outcome.state,
-            promptRecall: facts?.promptRecall === true,
-            consideredCount: new Set([...candidates.keys(), ...picked.keys()]).size,
-            filteredCount: filtered.size,
-            selectedCount: picked.size,
-            emittedCount: rows.filter((r) => r.outcome === 'emitted').length,
-            rejectedCount: rejected.length + undecided,
-            rejectedUnlisted: overflow + undecided,
-            sectionsShown: shown,
-            sectionsDropped: dropped,
-            budgetTokens: facts?.budgetTokens ?? 0,
-            selectedTokens: [...picked.values()].reduce((sum, p) => sum + p.tokens, 0),
-            injectedTokens: emitted !== null ? estimateTokens(emitted) : 0,
-            staticHash: outcome.staticHash ?? null,
-            recallHash: outcome.recallHash ?? null,
-            emittedHash: emitted !== null ? blockHash(emitted) : null,
-            elapsedMs: Math.max(0, Date.now() - startedMs),
-            candidates: rows,
-        };
+    return { payloadSession, envSession, prompt, hostTurnId, hookEvent, sessionState };
+}
+function rejectId(state, id, stage, reason, score, tokens) {
+    const held = state.candidates.get(id);
+    // First rejection wins; an id never offered is not a candidate of this call.
+    if (!held || held.reason !== null)
+        return;
+    state.candidates.set(id, { ...held, stage, reason, score, tokens });
+}
+function pickedRows(state) {
+    const staticReused = state.outcome.staticReused === true;
+    const rows = [];
+    for (const p of state.picked.values()) {
+        const held = state.candidates.get(p.entry.id);
+        rows.push({
+            memoryId: p.entry.id,
+            sourceStore: p.sourceStore,
+            pool: p.promptRecall ? 'prompt-recall' : held?.pool === 'pin' || p.entry.pinned ? 'pin' : 'recent',
+            stage: 'final',
+            outcome: staticReused && !p.promptRecall ? 'reused' : 'emitted',
+            reason: null,
+            rank: p.rank,
+            score: p.score,
+            tokens: p.tokens,
+        });
+    }
+    return rows;
+}
+/** Rejected candidates deepest first, capped at DELIVERY_REJECTED_ROW_CAP rows. */
+function rejectedRows(state) {
+    const rejected = [];
+    let undecided = 0;
+    for (const c of state.candidates.values()) {
+        if (state.picked.has(c.id))
+            continue;
+        if (c.reason === null)
+            undecided += 1;
+        else
+            rejected.push(c);
+    }
+    rejected.sort(byDepthThenScore);
+    const rows = rejected.slice(0, DELIVERY_REJECTED_ROW_CAP).map((c) => ({
+        memoryId: c.id, sourceStore: c.sourceStore, pool: c.pool, stage: c.stage ?? 'load', outcome: 'rejected',
+        reason: c.reason, rank: null, score: c.score, tokens: c.tokens,
+    }));
+    const overflow = Math.max(0, rejected.length - DELIVERY_REJECTED_ROW_CAP);
+    return { rows, rejected: rejected.length, undecided, overflow };
+}
+function buildEvent(init, payload, state, ts, startedMs) {
+    const { facts, outcome } = state;
+    const picked = pickedRows(state);
+    const rejected = rejectedRows(state);
+    const rows = [...picked, ...rejected.rows];
+    const emitted = outcome.emittedText ?? null;
+    return {
+        ts,
+        tenantId: init.tenantId,
+        runtime: payload.hostTurnId !== null ? 'codex' : payload.hookEvent !== null ? 'claude-code' : 'unknown',
+        eventType: payload.hookEvent === 'UserPromptSubmit' ? 'prompt-submit' : 'pinned-manual',
+        surface: 'hook',
+        storeHash: init.storeHash,
+        writeStore: init.writeStore,
+        projectHash: facts !== null && facts.projectName !== '' ? blockHash(facts.projectName) : null,
+        sessionId: payload.payloadSession ?? payload.envSession,
+        sessionState: payload.sessionState,
+        hostTurnId: payload.hostTurnId,
+        promptHash: payload.prompt !== null ? blockHash(payload.prompt) : null,
+        promptLength: payload.prompt?.length ?? 0,
+        queryHash: null,
+        recallTraceId: null,
+        blockState: state.disabledSeen ? 'disabled' : outcome.state,
+        promptRecall: facts?.promptRecall === true,
+        consideredCount: new Set([...state.candidates.keys(), ...state.picked.keys()]).size,
+        filteredCount: state.filtered.size,
+        selectedCount: state.picked.size,
+        emittedCount: rows.filter((r) => r.outcome === 'emitted').length,
+        rejectedCount: rejected.rejected + rejected.undecided,
+        rejectedUnlisted: rejected.overflow + rejected.undecided,
+        sectionsShown: state.shown,
+        sectionsDropped: state.dropped,
+        budgetTokens: facts?.budgetTokens ?? 0,
+        selectedTokens: [...state.picked.values()].reduce((sum, p) => sum + p.tokens, 0),
+        injectedTokens: emitted !== null ? estimateTokens(emitted) : 0,
+        staticHash: outcome.staticHash ?? null,
+        recallHash: outcome.recallHash ?? null,
+        emittedHash: emitted !== null ? blockHash(emitted) : null,
+        elapsedMs: Math.max(0, Date.now() - startedMs),
+        candidates: rows,
     };
+}
+/** The candidate-tracking half of the observer, each method run through `guard`. */
+function candidateMethods(state, guard) {
     return {
-        root: init.root,
-        facts: (f) => guard(() => { facts = { ...f }; }),
-        sections: (s, d) => guard(() => { shown = s; dropped = d; }),
-        watchAdmit: (admit) => (e) => {
-            const ok = admit(e);
-            if (!ok)
-                guard(() => { filtered.add(e.id); });
-            return ok;
-        },
         qualityDropped: (e, isGlobal) => guard(() => {
-            filtered.add(e.id);
-            if (!candidates.has(e.id)) {
-                candidates.set(e.id, {
+            state.filtered.add(e.id);
+            if (!state.candidates.has(e.id)) {
+                state.candidates.set(e.id, {
                     id: e.id, sourceStore: storeOf(isGlobal), pool: 'recent', stage: null, reason: null, score: null, tokens: null,
                 });
             }
-            rejectId(e.id, 'load', 'quality', null, null);
+            rejectId(state, e.id, 'load', 'quality', null, null);
         }),
-        disabled: () => guard(() => { disabledSeen = true; }),
         offer: (entries, isGlobal, pool) => guard(() => {
             for (const e of entries) {
-                const held = candidates.get(e.id);
+                const held = state.candidates.get(e.id);
                 const wanted = pool ?? (e.pinned ? 'pin' : 'recent');
                 // A loaded recent row the prompt-recall gate then judges belongs to that pool.
                 const relabel = held !== undefined && held.reason === null && held.pool === 'recent' && wanted === 'prompt-recall';
                 if (held !== undefined && !relabel)
                     continue;
-                candidates.set(e.id, {
+                state.candidates.set(e.id, {
                     id: e.id, sourceStore: storeOf(isGlobal), pool: wanted, stage: null, reason: null, score: null, tokens: null,
                 });
             }
         }),
-        reject: (e, stage, reason, score, tokens) => guard(() => rejectId(e.id, stage, reason, score ?? null, tokens ?? null)),
+        reject: (e, stage, reason, score, tokens) => guard(() => rejectId(state, e.id, stage, reason, score ?? null, tokens ?? null)),
         dropMissing: (before, after, stage, reason) => guard(() => {
             const kept = new Set(after.map((e) => e.id));
             for (const e of before)
                 if (!kept.has(e.id))
-                    rejectId(e.id, stage, reason, null, null);
+                    rejectId(state, e.id, stage, reason, null, null);
         }),
         gated: (prompt, items, gate, kept) => guard(() => {
             const keptIds = new Set(kept.map((g) => g.item.id));
@@ -190,29 +164,74 @@ export function createDeliveryRecorder(init) {
                     continue;
                 const { score, shared } = scoreOverlap(prompt, c.tokens, gate.metric);
                 const cleared = score >= gate.threshold && shared >= gate.minShared;
-                rejectId(c.id, 'gate', cleared ? 'gate-max-items' : 'gate-below-threshold', score, null);
+                rejectId(state, c.id, 'gate', cleared ? 'gate-max-items' : 'gate-below-threshold', score, null);
             }
         }),
         selected: (items) => guard(() => {
-            picked.clear();
+            state.picked.clear();
             items.forEach((r, i) => {
-                picked.set(r.entry.id, {
+                state.picked.set(r.entry.id, {
                     entry: r.entry, rank: i + 1, score: r.score, tokens: r.tokens,
                     sourceStore: storeOf(r.isGlobal), promptRecall: r.promptRecall === true,
                 });
             });
         }),
-        delivered: (o) => guard(() => { outcome = { ...o }; }),
+    };
+}
+/** A recorder for one call; every observer method is guarded, and a throw marks it broken instead of escaping. */
+export function createDeliveryRecorder(init) {
+    const startedMs = Date.now();
+    const ts = evalNow().toISOString();
+    // Test-only fault injection, as HIPPO_FAKE_NOW is for time.
+    const fault = envTestDeliveryFault();
+    const payload = readPayload(init);
+    const state = {
+        candidates: new Map(), picked: new Map(), filtered: new Set(), facts: null, shown: 0, dropped: 0,
+        disabledSeen: false, outcome: { state: 'empty' }, broken: null, flushed: false,
+    };
+    const guard = (fn) => {
+        if (state.broken !== null)
+            return;
+        try {
+            if (fault === 'observe')
+                throw new Error('injected observe fault');
+            fn();
+        }
+        catch (error) {
+            state.broken = error instanceof Error ? error.message : String(error);
+        }
+    };
+    const { qualityDropped, offer, reject, dropMissing, gated, selected } = candidateMethods(state, guard);
+    return {
+        root: init.root,
+        facts: (f) => guard(() => { state.facts = { ...f }; }),
+        sections: (s, d) => guard(() => { state.shown = s; state.dropped = d; }),
+        watchAdmit: (admit) => (e) => {
+            const ok = admit(e);
+            if (!ok)
+                guard(() => { state.filtered.add(e.id); });
+            return ok;
+        },
+        qualityDropped,
+        disabled: () => guard(() => { state.disabledSeen = true; }),
+        offer,
+        reject,
+        dropMissing,
+        gated,
+        selected,
+        delivered: (o) => guard(() => { state.outcome = { ...o }; }),
         flush: (write) => {
-            if (flushed)
+            if (state.flushed)
                 return;
-            flushed = true;
-            if (broken !== null) {
+            state.flushed = true;
+            if (state.broken !== null) {
                 // Same pinned `[hippo] delivery ledger` hook stderr line as recall-trace.ts's write failure.
-                console.error(`[hippo] delivery ledger skipped: recorder failed: ${broken}`);
+                console.error(`[hippo] delivery ledger skipped: recorder failed: ${state.broken}`);
                 return;
             }
-            const input = build();
+            if (fault === 'build')
+                throw new Error('injected build fault');
+            const input = buildEvent(init, payload, state, ts, startedMs);
             if (fault === 'flush')
                 throw new Error('injected flush fault');
             write(input);