hippo-memory 1.59.0 → 1.61.0

package/dist/consolidate/merge.js

@@ -0,0 +1,251 @@
+import { Layer, calculateStrength, createMemory } from '../memory.js';
+import { tokenize } from '../tokenize.js';
+import { jaccardMinShared, overlapPartners } from '../overlap-index.js';
+import { compareEntryIdentity } from '../compare.js';
+import { duplicateKey, mergedText } from '../same-text.js';
+import { successorAfterRetirement } from '../merged-row.js';
+import { rejectionDigest, findRejectedValue } from '../rejection.js';
+import { appendAuditEvent, reportAuditWriteFailure } from '../audit.js';
+import { derivationScope, derivationPartitionKey } from '../recall-scope.js';
+import { jaccardSets } from './conflicts.js';
+import { keptAsWritten } from './run.js';
+const MERGE_OVERLAP_THRESHOLD = 0.35; // Jaccard similarity for "related"
+const MERGE_MIN_CLUSTER = 2; // minimum cluster size to merge
+const MERGE_MAX_SOURCES = 5; // with MERGE_MAX_CHARS, keeps a merged row near 500 tokens, a third of the 1,500-token context budget
+const MERGE_MAX_CHARS = 2000; // total source text; sources past either cap stay unmerged and keep their half-life
+// Half-life scale for merged source episodics. Demotion must go through
+// half_life_days: calculateStrength() recomputes live strength from
+// last_retrieved/half_life and never reads the stored strength field, so a
+// stored-strength write is inert for ranking and gets overwritten by the
+// next sleep's decay pass anyway.
+const MERGE_SOURCE_HALF_LIFE_FACTOR = 0.3;
+export function retireHeldTexts(run) {
+    const { survivors } = run;
+    const byId = new Map(run.all.map((e) => [e.id, e]));
+    const rejectedIn = (tenantId) => (text) => {
+        const db = run.getConsolidateDb();
+        return db !== null && findRejectedValue(db, tenantId, rejectionDigest(text)) !== null;
+    };
+    for (let i = survivors.length - 1; i >= 0; i--) {
+        const row = survivors[i];
+        const successor = run.retirable(row) ? successorAfterRetirement(row, byId, rejectedIn(row.tenantId)) : undefined;
+        if (successor === undefined)
+            continue;
+        run.result.details.push(`  ✂️  ${row.id} held a retired text${successor ? `, ${successor.id} holds the rest` : ''}`);
+        if (run.dryRun)
+            continue;
+        run.pendingDeletes.push(row.id);
+        if (successor) {
+            run.pendingWrites.push(successor);
+            survivors[i] = successor;
+        }
+        else {
+            survivors.splice(i, 1);
+        }
+    }
+}
+// -------------------------------------------------------------------------
+// 3. Merge pass  - episodic entries only
+// -------------------------------------------------------------------------
+/** Returns how many clusters were skipped because their merged text matches a rejected value. */
+export function mergePass(run) {
+    const alreadyMergedIds = new Set(run.survivors.flatMap((e) => e.parents));
+    const mergeCandidates = run.survivors.filter((e) => e.layer === Layer.Episodic && !e.superseded_by && !keptAsWritten(e) && !alreadyMergedIds.has(e.id)
+        && !e.pinned // a pin merged with a look-alike would read as one of two values
+        && tokenize(e.content).length > 0);
+    const used = new Set();
+    // T1 fix (2026-08-15 hardening pass): partition by tenantId BEFORE the
+    // overlap loop so a cluster can never span tenants. Previously textOverlap
+    // clustered across the whole host-wide `survivors` list with no tenant
+    // boundary, and mergeContents concatenated cross-tenant content into one
+    // row. Map preserves insertion order, so single-tenant stores (every row
+    // 'default') get exactly one partition and iterate in the same order as
+    // before this fix — byte-identical behavior there.
+    const mergeCandidatesByTenant = new Map();
+    for (const entry of mergeCandidates) {
+        const key = derivationPartitionKey(entry.tenantId, entry.scope, entry.origin_project);
+        const bucket = mergeCandidatesByTenant.get(key);
+        if (bucket)
+            bucket.push(entry);
+        else
+            mergeCandidatesByTenant.set(key, [entry]);
+    }
+    // AT1 consolidation-loop fix (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
+    // reuses the single consolidateDb handle opened lazily in consolidate()
+    // for the whole non-dry-run consolidate — see that
+    // declaration's comment. Only needed for real writes — a dry-run preview
+    // never reaches batchWriteAndDelete's guard bypass, so there is nothing
+    // here for it to protect against.
+    let mergesSkippedRejected = 0;
+    for (const [, tenantCandidates] of mergeCandidatesByTenant) {
+        const partition = {
+            tenantId: tenantCandidates[0].tenantId,
+            scope: derivationScope(tenantCandidates[0].scope),
+            origin: tenantCandidates[0].origin_project,
+        };
+        const partnersOf = mergePartners(tenantCandidates.map((e) => e.content));
+        for (let i = 0; i < tenantCandidates.length; i++) {
+            if (used.has(tenantCandidates[i].id) || tenantCandidates[i].content.length > MERGE_MAX_CHARS)
+                continue;
+            const related = [tenantCandidates[i]];
+            for (const j of partnersOf(i)) {
+                if (!used.has(tenantCandidates[j].id))
+                    related.push(tenantCandidates[j]);
+            }
+            const cluster = [];
+            let clusterChars = 0;
+            for (const e of related) {
+                if (cluster.length === MERGE_MAX_SOURCES || clusterChars + e.content.length > MERGE_MAX_CHARS)
+                    continue;
+                cluster.push(e);
+                clusterChars += e.content.length;
+            }
+            if (cluster.length < MERGE_MIN_CLUSTER)
+                continue;
+            if (!mergeCluster(run, partition, cluster, related, used))
+                mergesSkippedRejected++;
+        }
+    }
+    return mergesSkippedRejected;
+}
+/** Merges one cluster into a semantic row; returns false when a tombstone refuses the merged text. */
+function mergeCluster(run, partition, cluster, related, used) {
+    const { result, dryRun } = run;
+    // Create a semantic summary
+    const mergedContent = mergeContents(cluster);
+    const allTags = Array.from(new Set(cluster.flatMap((e) => e.tags))).sort();
+    const maxValence = pickStrongestValence(cluster);
+    // AT1 P2 fix: build the semantic entry FIRST — createMemory is cheap
+    // and pure — so the tombstone check below runs under the tenant the
+    // row will ACTUALLY land in.
+    // T1 fix: createMemory now receives tenantId: mergeTenant (the
+    // partition's tenant — every member of `cluster` shares it by
+    // construction), so the row lands in its source tenant instead of
+    // always 'default'.
+    let semantic = null;
+    if (!dryRun) {
+        semantic = {
+            ...createMemory(mergedContent, {
+                layer: Layer.Semantic,
+                tags: allTags,
+                emotional_valence: maxValence,
+                schema_fit: 0.7,
+                source: 'consolidation',
+                confidence: 'inferred',
+                tenantId: partition.tenantId,
+                scope: partition.scope,
+                baseHalfLifeDays: run.config.defaultHalfLifeDays,
+            }),
+            origin_project: partition.origin,
+            parents: cluster.map((e) => e.id),
+        };
+    }
+    if (semantic && mergeRejected(run, semantic, cluster, related, used))
+        return false;
+    // Mark cluster members as used
+    for (const e of cluster)
+        used.add(e.id);
+    result.merged += cluster.length;
+    result.details.push(`  🔀 merged ${cluster.length} episodic entries into semantic: "${mergedContent.slice(0, 60)}..."`);
+    if (!dryRun && semantic) {
+        run.pendingWrites.push(semantic);
+        result.semanticCreated++;
+        // Demote source episodics (they've been compressed into neocortex):
+        // scale half_life_days so they decay sooner while staying recoverable.
+        // Immediate ranking is deliberately unchanged: the 2026-06-10 DAG
+        // slice-1 eval measured that dropping children below a worse-retrieving
+        // summary regresses budget-bounded QA (docs/evals/). The stored
+        // strength is refreshed to the live value so inspect, replay sampling,
+        // and strength-sorted assembly see the truth instead of a fake 0.3.
+        // Mutate in place (not a copy): `cluster` holds the same object
+        // references as `survivors`, and the later detectConflicts(survivors)
+        // pass in this same run must see the post-demotion half-life, or it
+        // can persist conflicts for entries the just-written state excludes.
+        for (const e of cluster) {
+            e.half_life_days = Math.max(1, Math.floor(e.half_life_days * MERGE_SOURCE_HALF_LIFE_FACTOR));
+            e.strength = calculateStrength(e, run.now, run.decayOpts);
+            run.pendingWrites.push(e);
+        }
+    }
+    return true;
+}
+// mergeContents is DETERMINISTIC CONCATENATION (not an LLM paraphrase)
+// — if a human rejected exactly this byte-identical rollup before, an
+// unguarded sleep would regenerate it every cycle and
+// batchWriteAndDelete's guard bypass (store.ts) would silently
+// re-assert it forever. This producer-side check is what makes that
+// bypass safe. A hit skips the WHOLE cluster: sources stay unmerged —
+// not demoted, not deleted — so a later sleep gets another chance if
+// the tombstone is lifted.
+function mergeRejected(run, semantic, cluster, related, used) {
+    const consolidateDb = run.getConsolidateDb();
+    if (!consolidateDb)
+        return false;
+    const newDigest = rejectionDigest(semantic.content);
+    const oldDigest = rejectionDigest(legacyMergeContents(related)); // tombstones from older releases hold this format's digest
+    const newHit = findRejectedValue(consolidateDb, semantic.tenantId, newDigest);
+    const tombstone = newHit ?? findRejectedValue(consolidateDb, semantic.tenantId, oldDigest);
+    const mergeDigest = newHit ? newDigest : oldDigest;
+    if (!tombstone)
+        return false;
+    // Still mark used — these members are not re-tried against a
+    // DIFFERENT cluster within this same pass; next sleep re-clusters
+    // them fresh.
+    const rejected = newHit ? cluster : related; // the old format digested the uncapped list, so rows past the cap were rejected too
+    for (const e of rejected)
+        used.add(e.id);
+    try {
+        appendAuditEvent(consolidateDb, {
+            tenantId: semantic.tenantId,
+            actor: 'sleep',
+            op: 'reject_refusal',
+            metadata: {
+                digest: mergeDigest,
+                reason: tombstone.reason,
+                sourceIds: rejected.map((e) => e.id),
+            },
+        });
+    }
+    catch (error) {
+        reportAuditWriteFailure('reject_refusal', String(error));
+    }
+    return true;
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function mergeContents(entries) {
+    // Each distinct text goes in once and in full (the merge demotes every source), one bullet with its lines indented, so heldTextKeys can read it back.
+    // Newest first says which version is current; compareEntryIdentity settles ties, so the row and its rejection digest depend only on the sources.
+    const sorted = [...entries].sort((a, b) => (Date.parse(b.created) - Date.parse(a.created)) || compareEntryIdentity(a, b));
+    const texts = new Map();
+    for (const e of sorted) {
+        if (!texts.has(duplicateKey(e.content)))
+            texts.set(duplicateKey(e.content), e.content);
+    }
+    const header = entries.length === 2 ? '[Consolidated from 2 related memories, newest first]' : `[Consolidated pattern from ${entries.length} related memories, newest first]`;
+    return mergedText(header, [...texts.values()]);
+}
+function legacyMergeContents(entries) {
+    // The old format dropped text, so it is only ever digested to match rejections recorded against it, never written.
+    const sorted = [...entries].sort((a, b) => (b.content.length - a.content.length) || compareEntryIdentity(a, b));
+    if (entries.length === 2)
+        return `[Consolidated from ${entries.length} related memories]\n\n${sorted[0].content}`;
+    const bullets = sorted.map((e) => `- ${e.content.split('\n')[0].slice(0, 120)}`).join('\n');
+    return `[Consolidated pattern from ${entries.length} related memories]\n\n${bullets}`;
+}
+function pickStrongestValence(entries) {
+    const order = ['critical', 'negative', 'positive', 'neutral'];
+    for (const v of order) {
+        if (entries.some((e) => e.emotional_valence === v))
+            return v;
+    }
+    return 'neutral';
+}
+/** Maps i to each j > i, ascending, whose text overlap with i reaches the merge threshold; every text needs at least one token. */
+export function mergePartners(contents) {
+    const sets = contents.map((text) => new Set(tokenize(text)));
+    const candidatesOf = overlapPartners(sets, jaccardMinShared(MERGE_OVERLAP_THRESHOLD));
+    return (i) => candidatesOf(i).filter((j) => jaccardSets(sets[i], sets[j]) >= MERGE_OVERLAP_THRESHOLD);
+}
+//# sourceMappingURL=merge.js.map

package/dist/consolidate/physics-pass.d.ts

@@ -0,0 +1,3 @@
+import type { SleepRun } from './run.js';
+export declare function physicsPass(run: SleepRun): void;
+//# sourceMappingURL=physics-pass.d.ts.map

package/dist/consolidate/physics-pass.js

@@ -0,0 +1,60 @@
+import { openHippoDb, closeHippoDb } from '../db.js';
+import { loadPhysicsState, savePhysicsState, refreshParticleProperties } from '../physics-state.js';
+import { simulate } from '../physics.js';
+// -------------------------------------------------------------------------
+// 2. Physics simulation pass
+// -------------------------------------------------------------------------
+export function physicsPass(run) {
+    if (run.dryRun)
+        return;
+    const { config, result, survivors } = run;
+    try {
+        const physicsEnabled = config.physics.enabled === true
+            || (config.physics.enabled === 'auto');
+        if (physicsEnabled) {
+            const db = openHippoDb(run.hippoRoot);
+            try {
+                const physicsMap = loadPhysicsState(db);
+                const particles = Array.from(physicsMap.values());
+                if (particles.length > 0) {
+                    // Build entry lookup for property refresh
+                    const entryMap = new Map(survivors.map(e => [e.id, e]));
+                    refreshParticleProperties(particles, entryMap, run.now);
+                    // Build conflict pairs from survivors
+                    const conflictPairs = new Map();
+                    for (const entry of survivors) {
+                        if (entry.conflicts_with.length > 0) {
+                            const set = conflictPairs.get(entry.id) ?? new Set();
+                            for (const cid of entry.conflicts_with)
+                                set.add(cid);
+                            conflictPairs.set(entry.id, set);
+                        }
+                    }
+                    // Build half-life lookup
+                    const halfLives = new Map();
+                    for (const entry of survivors) {
+                        halfLives.set(entry.id, entry.half_life_days);
+                    }
+                    const ctx = {
+                        conflictPairs,
+                        halfLives,
+                        config: config.physics,
+                    };
+                    const stats = simulate(particles, ctx);
+                    savePhysicsState(db, particles);
+                    result.physicsSimulated = stats.particleCount;
+                    result.details.push(`  ⚛️  physics: ${stats.particleCount} particles, ` +
+                        `avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
+                        `energy ${stats.energy.total.toFixed(4)}`);
+                }
+            }
+            finally {
+                closeHippoDb(db);
+            }
+        }
+    }
+    catch (error) {
+        result.details.push(`  ⚠️ physics simulation skipped: ${error instanceof Error ? error.message : 'unknown error'}`);
+    }
+}
+//# sourceMappingURL=physics-pass.js.map

package/dist/consolidate/run.d.ts

@@ -0,0 +1,69 @@
+import { MemoryEntry, type DecayOptions } from '../memory.js';
+import { type DatabaseSyncLike } from '../db.js';
+import { type DormantMove } from '../dormant.js';
+import { loadConfig } from '../config.js';
+export interface ConsolidationResult {
+    decayed: number;
+    removed: number;
+    /** Faded memories moved to the dormant store instead of deleted (config
+     *  `dormant.enabled`; src/dormant.ts). Always 0 when that is off. */
+    dormant: number;
+    /** Dormant memories deleted for good this sleep because they outlived
+     *  `dormant.retentionDays` without a restore. */
+    dormantExpired: number;
+    merged: number;
+    semanticCreated: number;
+    replayed: number;
+    promotedTraces: number;
+    /** T7: sessions skipped because their events span two derivation scopes. */
+    tracesSkippedMixedScope: number;
+    extractionCandidates: number;
+    extracted: number;
+    dagCandidateClusters: number;
+    dagSummariesCreated: number;
+    summariesRebuilt: number;
+    summariesRebuildFailed: number;
+    summariesZeroChildSkipped: number;
+    summariesRebuildRefused: number;
+    summariesRebuildCapped: boolean;
+    entityProfilesCreated: number;
+    dryRun: boolean;
+    details: string[];
+    physicsSimulated: number;
+    /** Ids the decay pass removes (or would remove, under dryRun). */
+    removedIds?: string[];
+}
+export declare const REPLAY_COUNT_DEFAULT = 5;
+export declare function keptAsWritten(entry: MemoryEntry): boolean;
+/** JSON value shape for a session event's free-form metadata field, cast to
+ *  once at its `Record<string, unknown>` origin so it can be narrowed via
+ *  isJsonString below rather than left as unparsed `unknown`. */
+export type JsonValue = string | number | boolean | null | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+export declare function isJsonString(value: JsonValue): value is string;
+/** The sleep's one tombstone-check handle: opened on first use, never under dryRun, closed once. */
+interface LazyDb {
+    get: () => DatabaseSyncLike | null;
+    close: () => void;
+}
+export declare function lazyConsolidateDb(hippoRoot: string, dryRun: boolean): LazyDb;
+/** State every sleep stage reads or appends to; the pending lists are flushed in one transaction at the end. */
+export interface SleepRun {
+    hippoRoot: string;
+    now: Date;
+    dryRun: boolean;
+    config: ReturnType<typeof loadConfig>;
+    decayOpts: DecayOptions;
+    result: ConsolidationResult;
+    all: MemoryEntry[];
+    retirable: (entry: MemoryEntry) => boolean;
+    getConsolidateDb: () => DatabaseSyncLike | null;
+    survivors: MemoryEntry[];
+    pendingWrites: MemoryEntry[];
+    pendingDeletes: string[];
+    pendingDormant: DormantMove[];
+}
+export declare function newConsolidationResult(dryRun: boolean): ConsolidationResult;
+export {};
+//# sourceMappingURL=run.d.ts.map

package/dist/consolidate/run.js

@@ -0,0 +1,76 @@
+import { openHippoDb, closeHippoDb } from '../db.js';
+import { NO_MERGE_TAGS } from '../shared.js';
+export const REPLAY_COUNT_DEFAULT = 5;
+export function keptAsWritten(entry) {
+    return entry.tags.some((tag) => NO_MERGE_TAGS.has(tag));
+}
+export function isJsonString(value) {
+    return typeof value === 'string';
+}
+// AT1 rejection-guard db handle (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
+// covers BOTH the auto-promote pass (1.4) and the merge
+// pass (3) — both build deterministic content that
+// batchWriteAndDelete writes through the guard's bypass, so both need a
+// producer-side tombstone check before pushing to pendingWrites.
+//
+// T3 fix (2026-08-15 hardening pass, perf hygiene): memoized lazy getter,
+// not an eager open. The handle only serves these two tombstone checks —
+// a sleep with zero promotable sessions and zero merge clusters never
+// reaches either use site, so opening it unconditionally on every
+// non-dry-run sleep paid a db-open cost for nothing. dryRun still never
+// opens (getConsolidateDb short-circuits before touching the handle).
+// consolidateDbOpened (not just a truthy handle check) is the
+// single source of truth for "was this ever opened", so the finally
+// closes it exactly once and never double-opens.
+//
+// AT1 P2 fix (codex, handle-leak restructure): the getter's lifetime must
+// start IMMEDIATELY before the try whose finally closes it, covering every
+// phase that can touch it — not just the merge pass. An exception thrown by
+// auto-promote (1.4), replay (1.5), batch extraction (1.6), the DAG
+// passes (1.7-1.9), or physics (2) would otherwise propagate past an open handle
+// with nothing to close it.
+export function lazyConsolidateDb(hippoRoot, dryRun) {
+    let consolidateDbHandle = null;
+    let consolidateDbOpened = false;
+    const get = () => {
+        if (dryRun)
+            return null;
+        if (!consolidateDbOpened) {
+            consolidateDbHandle = openHippoDb(hippoRoot);
+            consolidateDbOpened = true;
+        }
+        return consolidateDbHandle;
+    };
+    const close = () => {
+        if (consolidateDbHandle)
+            closeHippoDb(consolidateDbHandle);
+    };
+    return { get, close };
+}
+export function newConsolidationResult(dryRun) {
+    return {
+        decayed: 0,
+        removed: 0,
+        dormant: 0,
+        dormantExpired: 0,
+        merged: 0,
+        semanticCreated: 0,
+        replayed: 0,
+        promotedTraces: 0,
+        tracesSkippedMixedScope: 0,
+        extractionCandidates: 0,
+        extracted: 0,
+        dagCandidateClusters: 0,
+        dagSummariesCreated: 0,
+        summariesRebuilt: 0,
+        summariesRebuildFailed: 0,
+        summariesZeroChildSkipped: 0,
+        summariesRebuildRefused: 0,
+        summariesRebuildCapped: false,
+        entityProfilesCreated: 0,
+        dryRun,
+        details: [],
+        physicsSimulated: 0,
+    };
+}
+//# sourceMappingURL=run.js.map

package/dist/consolidate/sleep.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Consolidation engine ("Sleep") for Hippo.
+ *
+ * Steps:
+ * 1. Decay pass  - remove entries below strength threshold
+ * 2. Merge pass  - find episodic entries with high text overlap, create semantic summaries
+ * 3. Stats tracking
+ */
+import { type ConsolidationResult } from './run.js';
+/**
+ * Run a full consolidation pass.
+ */
+export declare function consolidate(hippoRoot: string, options?: {
+    dryRun?: boolean;
+    now?: Date;
+    fetcher?: typeof fetch;
+}): Promise<ConsolidationResult>;
+//# sourceMappingURL=sleep.d.ts.map