hippo-memory 1.29.0 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/api.d.ts +49 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +78 -2
- package/dist/api.js.map +1 -1
- package/dist/audit.d.ts +1 -1
- package/dist/audit.d.ts.map +1 -1
- package/dist/audit.js.map +1 -1
- package/dist/capture.d.ts.map +1 -1
- package/dist/capture.js +79 -23
- package/dist/capture.js.map +1 -1
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +234 -13
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +8 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +21 -0
- package/dist/config.js.map +1 -1
- package/dist/connectors/github/ingest.d.ts.map +1 -1
- package/dist/connectors/github/ingest.js +17 -0
- package/dist/connectors/github/ingest.js.map +1 -1
- package/dist/connectors/slack/ingest.d.ts.map +1 -1
- package/dist/connectors/slack/ingest.js +15 -0
- package/dist/connectors/slack/ingest.js.map +1 -1
- package/dist/consolidate.d.ts.map +1 -1
- package/dist/consolidate.js +623 -312
- package/dist/consolidate.js.map +1 -1
- package/dist/dag.d.ts +10 -0
- package/dist/dag.d.ts.map +1 -1
- package/dist/dag.js +32 -3
- package/dist/dag.js.map +1 -1
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +43 -1
- package/dist/db.js.map +1 -1
- package/dist/extract.d.ts.map +1 -1
- package/dist/extract.js +18 -1
- package/dist/extract.js.map +1 -1
- package/dist/importers.d.ts +10 -0
- package/dist/importers.d.ts.map +1 -1
- package/dist/importers.js +242 -159
- package/dist/importers.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +41 -5
- package/dist/mcp/server.js.map +1 -1
- package/dist/memory-value-weights.d.ts +22 -0
- package/dist/memory-value-weights.d.ts.map +1 -0
- package/dist/memory-value-weights.js +31 -0
- package/dist/memory-value-weights.js.map +1 -0
- package/dist/memory-value.d.ts +123 -0
- package/dist/memory-value.d.ts.map +1 -0
- package/dist/memory-value.js +255 -0
- package/dist/memory-value.js.map +1 -0
- package/dist/reject-flow.d.ts +66 -0
- package/dist/reject-flow.d.ts.map +1 -0
- package/dist/reject-flow.js +207 -0
- package/dist/reject-flow.js.map +1 -0
- package/dist/rejection.d.ts +100 -0
- package/dist/rejection.d.ts.map +1 -0
- package/dist/rejection.js +157 -0
- package/dist/rejection.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +5 -0
- package/dist/server.js.map +1 -1
- package/dist/shared.d.ts +9 -0
- package/dist/shared.d.ts.map +1 -1
- package/dist/shared.js +57 -6
- package/dist/shared.js.map +1 -1
- package/dist/sleep-redact.d.ts +2 -0
- package/dist/sleep-redact.d.ts.map +1 -1
- package/dist/sleep-redact.js +2 -0
- package/dist/sleep-redact.js.map +1 -1
- package/dist/src/api.js +78 -2
- package/dist/src/api.js.map +1 -1
- package/dist/src/audit.js.map +1 -1
- package/dist/src/capture.js +79 -23
- package/dist/src/capture.js.map +1 -1
- package/dist/src/cli.js +234 -13
- package/dist/src/cli.js.map +1 -1
- package/dist/src/config.js +21 -0
- package/dist/src/config.js.map +1 -1
- package/dist/src/connectors/github/ingest.js +17 -0
- package/dist/src/connectors/github/ingest.js.map +1 -1
- package/dist/src/connectors/slack/ingest.js +15 -0
- package/dist/src/connectors/slack/ingest.js.map +1 -1
- package/dist/src/consolidate.js +623 -312
- package/dist/src/consolidate.js.map +1 -1
- package/dist/src/dag.js +32 -3
- package/dist/src/dag.js.map +1 -1
- package/dist/src/db.js +43 -1
- package/dist/src/db.js.map +1 -1
- package/dist/src/extract.js +18 -1
- package/dist/src/extract.js.map +1 -1
- package/dist/src/importers.js +242 -159
- package/dist/src/importers.js.map +1 -1
- package/dist/src/mcp/server.js +41 -5
- package/dist/src/mcp/server.js.map +1 -1
- package/dist/src/memory-value-weights.js +31 -0
- package/dist/src/memory-value-weights.js.map +1 -0
- package/dist/src/memory-value.js +255 -0
- package/dist/src/memory-value.js.map +1 -0
- package/dist/src/reject-flow.js +207 -0
- package/dist/src/reject-flow.js.map +1 -0
- package/dist/src/rejection.js +157 -0
- package/dist/src/rejection.js.map +1 -0
- package/dist/src/server.js +5 -0
- package/dist/src/server.js.map +1 -1
- package/dist/src/shared.js +57 -6
- package/dist/src/shared.js.map +1 -1
- package/dist/src/sleep-redact.js +2 -0
- package/dist/src/sleep-redact.js.map +1 -1
- package/dist/src/store.js +498 -35
- package/dist/src/store.js.map +1 -1
- package/dist/src/version.js +1 -1
- package/dist/store.d.ts +107 -3
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +498 -35
- package/dist/store.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/consolidate.js
CHANGED
|
@@ -12,12 +12,16 @@ import { loadAllEntries, batchWriteAndDelete, appendConsolidationRun, replaceDet
|
|
|
12
12
|
import { textOverlap, markRetrieved } from './search.js';
|
|
13
13
|
import { compareEntryIdentity } from './compare.js';
|
|
14
14
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
15
|
+
import { rejectionDigest, findRejectedValue } from './rejection.js';
|
|
15
16
|
import { loadPhysicsState, savePhysicsState, refreshParticleProperties } from './physics-state.js';
|
|
16
17
|
import { simulate } from './physics.js';
|
|
17
18
|
import { loadConfig } from './config.js';
|
|
18
19
|
import { sampleForReplay } from './replay.js';
|
|
19
20
|
import { renderTraceContent } from './trace.js';
|
|
20
21
|
import { resolveTenantId } from './tenant.js';
|
|
22
|
+
import { rescueSet, rankNonPinnedByTenant, validateWeights } from './memory-value.js';
|
|
23
|
+
import { MEMORY_VALUE_WEIGHTS, SOURCE_ARTIFACT_SHA256 } from './memory-value-weights.js';
|
|
24
|
+
import { appendAuditEvent } from './audit.js';
|
|
21
25
|
const DECAY_THRESHOLD = 0.05;
|
|
22
26
|
const MERGE_OVERLAP_THRESHOLD = 0.35; // Jaccard similarity for "related"
|
|
23
27
|
const MERGE_MIN_CLUSTER = 2; // minimum cluster size to merge
|
|
@@ -97,356 +101,604 @@ export async function consolidate(hippoRoot, options = {}) {
|
|
|
97
101
|
// -------------------------------------------------------------------------
|
|
98
102
|
// 1. Decay pass
|
|
99
103
|
// -------------------------------------------------------------------------
|
|
104
|
+
// LC2-E3 (opt-in, default off; docs/plans/2026-08-10-lc2-e3-mv-wiring.md):
|
|
105
|
+
// flag OFF keeps the single-phase loop below byte-identical to pre-E3
|
|
106
|
+
// behavior (pre-registered gate G2). Flag ON restructures into two phases:
|
|
107
|
+
// phase 1 classifies every entry (condemned vs survivor) with ZERO
|
|
108
|
+
// commits; phase 2 runs rescueSet over the per-tenant candidate groups,
|
|
109
|
+
// then commits — rescued entries get the standard survivor bookkeeping
|
|
110
|
+
// refresh (stored strength + effective confidence; no half-life edits, no
|
|
111
|
+
// rank-derived writes) and are pushed to survivors so they fully
|
|
112
|
+
// participate in this cycle's merge/physics/conflict passes; non-rescued
|
|
113
|
+
// condemned entries follow the existing pendingDeletes/result.removed/
|
|
114
|
+
// details path.
|
|
100
115
|
const survivors = [];
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
116
|
+
let rescuedIds = new Set();
|
|
117
|
+
// Carried forward to the post-flush "4. Log run" section below, where the
|
|
118
|
+
// mv_rescue audit rows are actually written (code-review fix: writing them
|
|
119
|
+
// here, before batchWriteAndDelete, would assert rescues for a cycle whose
|
|
120
|
+
// effects might never land if a later phase throws).
|
|
121
|
+
let rescuedEntries = [];
|
|
122
|
+
let rankById = new Map();
|
|
123
|
+
if (config.memoryValue.enabled) {
|
|
124
|
+
// --- Phase 1: classify (zero commits) ---
|
|
125
|
+
const condemned = [];
|
|
126
|
+
const strengthById = new Map();
|
|
127
|
+
for (const entry of all) {
|
|
128
|
+
const strength = calculateStrength(entry, now, decayOpts);
|
|
129
|
+
strengthById.set(entry.id, strength);
|
|
130
|
+
if (!entry.pinned && strength < DECAY_THRESHOLD) {
|
|
131
|
+
condemned.push(entry);
|
|
108
132
|
}
|
|
109
133
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
134
|
+
// --- Phase 2a: rescue decision (pure compute) ---
|
|
135
|
+
// Runs under --dry-run too (only pendingDeletes/the audit write in
|
|
136
|
+
// "4. Log run" below stay !dryRun-gated), so the preview matches what a
|
|
137
|
+
// real run would decide.
|
|
138
|
+
const condemnedIds = new Set(condemned.map((e) => e.id));
|
|
139
|
+
// Fail-loud must not depend on condemnation traffic (round-2 code-review
|
|
140
|
+
// P2-2): validate the frozen weights constant unconditionally, even on a
|
|
141
|
+
// sleep with nothing condemned.
|
|
142
|
+
validateWeights();
|
|
143
|
+
if (condemnedIds.size > 0) {
|
|
144
|
+
// Compute the per-tenant ranking ONCE (round-2 code-review P2-2):
|
|
145
|
+
// rankById feeds both rescueSet's decision (via precomputedRanks,
|
|
146
|
+
// skipping its own internal rankNonPinnedByTenant call) and the
|
|
147
|
+
// detail/audit rank context below, so the whole-store ranking pass
|
|
148
|
+
// runs a single time per sleep instead of twice, and only when there
|
|
149
|
+
// is actually something condemned to rank against.
|
|
150
|
+
rankById = rankNonPinnedByTenant(all, now);
|
|
151
|
+
rescuedIds = rescueSet(all, condemnedIds, now, MEMORY_VALUE_WEIGHTS, SOURCE_ARTIFACT_SHA256, rankById);
|
|
152
|
+
// Review-round F2: entries with a non-finite computed feature (e.g. a
|
|
153
|
+
// malformed `created` string -> Date.parse NaN) score -Infinity in
|
|
154
|
+
// rankById and rescueSet's explicit finite-score guard means they can
|
|
155
|
+
// never be rescued. Surfaced here as one details warning line naming
|
|
156
|
+
// them, rather than leaving it a silent NaN-driven scoring detail.
|
|
157
|
+
const nonFiniteIds = [...rankById.entries()]
|
|
158
|
+
.filter(([, info]) => !Number.isFinite(info.score))
|
|
159
|
+
.map(([id]) => id);
|
|
160
|
+
if (nonFiniteIds.length > 0) {
|
|
161
|
+
result.details.push(` ⚠️ memory-value: skipped ${nonFiniteIds.length} entr${nonFiniteIds.length === 1 ? 'y' : 'ies'} ` +
|
|
162
|
+
`with non-finite computed features (never rescued): ${nonFiniteIds.join(', ')}`);
|
|
121
163
|
}
|
|
122
|
-
result.decayed++;
|
|
123
164
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
165
|
+
// --- Phase 2b: commit, one pass over `all` in ITS ORIGINAL ORDER ---
|
|
166
|
+
// (review-round F4: rescued entries used to be appended at the tail of
|
|
167
|
+
// survivors, systematically starving them in downstream order-sensitive
|
|
168
|
+
// passes like extraction's slice(0,20) — a single pass over `all`
|
|
169
|
+
// preserves flag-off's ordering semantics exactly.)
|
|
170
|
+
for (const entry of all) {
|
|
171
|
+
const strength = strengthById.get(entry.id);
|
|
172
|
+
if (!entry.pinned && strength < DECAY_THRESHOLD) {
|
|
173
|
+
if (rescuedIds.has(entry.id)) {
|
|
174
|
+
// Rescued (D1): standard survivor bookkeeping refresh — same
|
|
175
|
+
// stored-strength + effective-confidence update every other
|
|
176
|
+
// survivor gets (round-2 code-review P2-1). D1's "kept as-is"
|
|
177
|
+
// protects against half-life edits and rank-derived writes, not
|
|
178
|
+
// against the ordinary decay-pass refresh every survivor
|
|
179
|
+
// receives; leaving a rescued entry's stale strength (e.g. 1.0)
|
|
180
|
+
// on disk would mislead downstream replay/admission reads.
|
|
181
|
+
const effectiveConfidence = resolveConfidence(entry, now);
|
|
182
|
+
const updated = { ...entry, strength, confidence: effectiveConfidence };
|
|
183
|
+
survivors.push(updated);
|
|
184
|
+
if (!dryRun && (strength !== entry.strength || effectiveConfidence !== entry.confidence)) {
|
|
185
|
+
pendingWrites.push(updated);
|
|
186
|
+
}
|
|
187
|
+
result.decayed++;
|
|
188
|
+
rescuedEntries.push(updated);
|
|
189
|
+
const rank = rankById.get(entry.id);
|
|
190
|
+
const rankNote = rank
|
|
191
|
+
? ` - rescued (rank ${rank.rank}/${rank.totalNonPinned} in tenant ${rank.tenantId}, top ${rank.keepN})`
|
|
192
|
+
: ' - rescued';
|
|
193
|
+
result.details.push(` 🛟 ${entry.id} (strength ${strength.toFixed(4)} < ${DECAY_THRESHOLD})${rankNote}`);
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
result.removed++;
|
|
197
|
+
result.details.push(` 🗑 removed ${entry.id} (strength ${strength.toFixed(4)} < ${DECAY_THRESHOLD})`);
|
|
198
|
+
if (!dryRun) {
|
|
199
|
+
pendingDeletes.push(entry.id);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
const effectiveConfidence = resolveConfidence(entry, now);
|
|
205
|
+
const updated = { ...entry, strength, confidence: effectiveConfidence };
|
|
206
|
+
survivors.push(updated);
|
|
207
|
+
if (!dryRun && (strength !== entry.strength || effectiveConfidence !== entry.confidence)) {
|
|
208
|
+
pendingWrites.push(updated);
|
|
209
|
+
}
|
|
210
|
+
result.decayed++;
|
|
157
211
|
}
|
|
158
|
-
const outcome = outcomeRaw;
|
|
159
|
-
const steps = events
|
|
160
|
-
.filter((e) => e.event_type !== 'session_complete')
|
|
161
|
-
.map((e) => ({ action: e.content, observation: '' }));
|
|
162
|
-
const summary = typeof completeEvent.metadata.summary === 'string'
|
|
163
|
-
? completeEvent.metadata.summary
|
|
164
|
-
: '(untitled)';
|
|
165
|
-
const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
|
|
166
|
-
layer: Layer.Trace,
|
|
167
|
-
trace_outcome: outcome,
|
|
168
|
-
source_session_id: session.session_id,
|
|
169
|
-
tags: ['auto-promoted'],
|
|
170
|
-
source: 'auto-promote',
|
|
171
|
-
});
|
|
172
|
-
pendingWrites.push(trace);
|
|
173
|
-
survivors.push(trace);
|
|
174
|
-
result.promotedTraces++;
|
|
175
|
-
result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
|
|
176
|
-
}
|
|
177
|
-
if (result.promotedTraces > 0) {
|
|
178
|
-
result.details.push(` 🧬 promoted ${result.promotedTraces} trace${result.promotedTraces === 1 ? '' : 's'} from completed session${result.promotedTraces === 1 ? '' : 's'}`);
|
|
179
212
|
}
|
|
180
213
|
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
// + idle time, then apply the same retrieval-strengthening `markRetrieved`
|
|
188
|
-
// applies to real queries. Distinct from decay (removal), physics (motion),
|
|
189
|
-
// and merge (compression) — this is the "rehearse the important stuff so
|
|
190
|
-
// it doesn't fade" pass.
|
|
191
|
-
{
|
|
192
|
-
const replayCount = config.replay?.count ?? REPLAY_COUNT_DEFAULT;
|
|
193
|
-
// EVAL-ONLY ablation (see ablation.ts): replay rehearsal IS recall
|
|
194
|
-
// strengthening (same markRetrieved dynamics), so the strengthen-off arm
|
|
195
|
-
// silences the whole pass - markRetrieved would return unmutated entries
|
|
196
|
-
// and persisting them anyway would still refresh updated_at / mirrors.
|
|
197
|
-
if (replayCount > 0 && survivors.length > 0 && !isRecallBoostAblated()) {
|
|
198
|
-
const seed = Math.floor(now.getTime() / 1000) & 0xffffffff;
|
|
199
|
-
const picked = sampleForReplay(survivors, replayCount, now, seed);
|
|
200
|
-
if (picked.length > 0) {
|
|
201
|
-
const rehearsed = markRetrieved(picked, now);
|
|
202
|
-
const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
|
|
203
|
-
// Update survivors in place so downstream passes see rehearsed state.
|
|
204
|
-
for (let i = 0; i < survivors.length; i++) {
|
|
205
|
-
const replacement = rehearsedById.get(survivors[i].id);
|
|
206
|
-
if (replacement)
|
|
207
|
-
survivors[i] = replacement;
|
|
208
|
-
}
|
|
209
|
-
result.replayed = rehearsed.length;
|
|
210
|
-
result.details.push(` 💭 replayed ${rehearsed.length} memor${rehearsed.length === 1 ? 'y' : 'ies'}: ` +
|
|
211
|
-
rehearsed.map((e) => e.id).join(', '));
|
|
214
|
+
else {
|
|
215
|
+
for (const entry of all) {
|
|
216
|
+
const strength = calculateStrength(entry, now, decayOpts);
|
|
217
|
+
if (!entry.pinned && strength < DECAY_THRESHOLD) {
|
|
218
|
+
result.removed++;
|
|
219
|
+
result.details.push(` 🗑 removed ${entry.id} (strength ${strength.toFixed(4)} < ${DECAY_THRESHOLD})`);
|
|
212
220
|
if (!dryRun) {
|
|
213
|
-
|
|
214
|
-
|
|
221
|
+
pendingDeletes.push(entry.id);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
else {
|
|
225
|
+
// Update the stored strength value and persist stale confidence when applicable.
|
|
226
|
+
const effectiveConfidence = resolveConfidence(entry, now);
|
|
227
|
+
const updated = {
|
|
228
|
+
...entry,
|
|
229
|
+
strength,
|
|
230
|
+
confidence: effectiveConfidence,
|
|
231
|
+
};
|
|
232
|
+
survivors.push(updated);
|
|
233
|
+
if (!dryRun && (strength !== entry.strength || effectiveConfidence !== entry.confidence)) {
|
|
234
|
+
pendingWrites.push(updated);
|
|
215
235
|
}
|
|
236
|
+
result.decayed++;
|
|
216
237
|
}
|
|
217
238
|
}
|
|
218
239
|
}
|
|
219
|
-
//
|
|
220
|
-
//
|
|
221
|
-
//
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
240
|
+
// AT1 rejection-guard db handle (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
|
|
241
|
+
// opened ONCE for the whole non-dry-run consolidate, covering BOTH the
|
|
242
|
+
// auto-promote pass (1.4, immediately below) and the merge pass (3,
|
|
243
|
+
// further down) — both build deterministic content that batchWriteAndDelete
|
|
244
|
+
// writes through the guard's bypass, so both need a producer-side
|
|
245
|
+
// tombstone check before pushing to pendingWrites. A single handle here
|
|
246
|
+
// replaces what used to be a per-pass open. Dry-run never reaches the
|
|
247
|
+
// bypass (no batchWriteAndDelete call), so there is nothing for a handle
|
|
248
|
+
// to protect against — stays null and every `if (consolidateDb)` below is
|
|
249
|
+
// a no-op.
|
|
250
|
+
//
|
|
251
|
+
// AT1 P2 fix (codex, handle-leak restructure): the open must be followed
|
|
252
|
+
// IMMEDIATELY by the try whose finally closes it, covering every phase
|
|
253
|
+
// that can touch consolidateDb — not just the merge pass. Previously the
|
|
254
|
+
// try/finally wrapped only section 3 (merge pass); an exception thrown by
|
|
255
|
+
// auto-promote (1.4), replay (1.5), batch extraction (1.6), the DAG
|
|
256
|
+
// passes (1.7-1.9), or physics (2) would propagate past the open handle
|
|
257
|
+
// with nothing to close it. No behavior change on the happy path — each
|
|
258
|
+
// of those phases already best-effort catches its own exceptions today
|
|
259
|
+
// (physics has its own try/catch below; each DAG pass wraps its own
|
|
260
|
+
// dynamic import + call in try/catch) — this only closes the handle on
|
|
261
|
+
// the rare path where one of them throws past its own catch. Deliberately
|
|
262
|
+
// NOT re-indented (mechanical wrap only, kept surgical): every statement
|
|
263
|
+
// between this try and its finally (below, at the end of the merge pass)
|
|
264
|
+
// stays at its original indentation.
|
|
265
|
+
const consolidateDb = dryRun ? null : openHippoDb(hippoRoot);
|
|
266
|
+
// Declared here (not at the merge pass, its point of use) so it survives
|
|
267
|
+
// this try/finally — a `let` declared INSIDE the try would go out of
|
|
268
|
+
// scope before the `if (mergesSkippedRejected > 0)` check that reads it
|
|
269
|
+
// after the finally closes the handle.
|
|
270
|
+
let mergesSkippedRejected = 0;
|
|
271
|
+
try {
|
|
272
|
+
// -------------------------------------------------------------------------
|
|
273
|
+
// 1.4. Auto-promote complete sessions to traces
|
|
274
|
+
// -------------------------------------------------------------------------
|
|
275
|
+
//
|
|
276
|
+
// For each session within the configured window that has a `session_complete`
|
|
277
|
+
// event and no existing trace (idempotency via the source_session_id column),
|
|
278
|
+
// render the action sequence as markdown and persist a Layer.Trace memory.
|
|
279
|
+
// Traces inherit decay, search, replay, and physics from the base MemoryEntry.
|
|
280
|
+
if (!dryRun && config.autoTraceCapture !== false) {
|
|
281
|
+
let tracesSkippedRejected = 0;
|
|
282
|
+
const windowDays = config.autoTraceWindowDays ?? 7;
|
|
283
|
+
const sinceMs = now.getTime() - windowDays * 24 * 60 * 60 * 1000;
|
|
284
|
+
// Auto-trace currently runs in a single-tenant context (the env-resolved
|
|
285
|
+
// tenant for this process). Multi-tenant deployments that want
|
|
286
|
+
// consolidation across all tenants need a per-tenant loop layered on top
|
|
287
|
+
// of this — tracked in docs/plans/2026-05-02-continuity-tables-tenant-scope.md.
|
|
288
|
+
const consolidationTenant = resolveTenantId({});
|
|
289
|
+
const promotable = findPromotableSessions(hippoRoot, consolidationTenant, sinceMs);
|
|
290
|
+
for (const session of promotable) {
|
|
291
|
+
// Idempotency: skip if a trace for this session already exists.
|
|
292
|
+
if (traceExistsForSession(hippoRoot, consolidationTenant, session.session_id))
|
|
293
|
+
continue;
|
|
294
|
+
const events = listSessionEvents(hippoRoot, consolidationTenant, {
|
|
295
|
+
session_id: session.session_id,
|
|
296
|
+
limit: 1000,
|
|
235
297
|
});
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
298
|
+
const completeEvent = events.find((e) => e.event_type === 'session_complete');
|
|
299
|
+
if (!completeEvent)
|
|
300
|
+
continue; // defence-in-depth; findPromotableSessions filters already.
|
|
301
|
+
const outcomeRaw = completeEvent.content;
|
|
302
|
+
if (outcomeRaw !== 'success' && outcomeRaw !== 'failure' && outcomeRaw !== 'partial') {
|
|
303
|
+
// Malformed terminal event — skip rather than crash the whole sleep.
|
|
304
|
+
continue;
|
|
239
305
|
}
|
|
306
|
+
const outcome = outcomeRaw;
|
|
307
|
+
const steps = events
|
|
308
|
+
.filter((e) => e.event_type !== 'session_complete')
|
|
309
|
+
.map((e) => ({ action: e.content, observation: '' }));
|
|
310
|
+
const summary = typeof completeEvent.metadata.summary === 'string'
|
|
311
|
+
? completeEvent.metadata.summary
|
|
312
|
+
: '(untitled)';
|
|
313
|
+
const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
|
|
314
|
+
layer: Layer.Trace,
|
|
315
|
+
trace_outcome: outcome,
|
|
316
|
+
source_session_id: session.session_id,
|
|
317
|
+
tags: ['auto-promoted'],
|
|
318
|
+
source: 'auto-promote',
|
|
319
|
+
});
|
|
320
|
+
// AT1 (same producer-side pattern as the merge pass below): traceExistsForSession
|
|
321
|
+
// only sees rows CURRENTLY in the store — once a rejected trace is
|
|
322
|
+
// removed, that idempotency check no longer blocks regeneration, and
|
|
323
|
+
// this write would otherwise reach batchWriteAndDelete's guard bypass
|
|
324
|
+
// unchecked, resurrecting it every sleep. Check under THE ENTRY'S OWN
|
|
325
|
+
// stamped tenantId (read off `trace` after createMemory — never guess
|
|
326
|
+
// the tenant) + the built content's digest. A hit skips the push
|
|
327
|
+
// entirely: not counted as promoted, not added to survivors.
|
|
328
|
+
if (consolidateDb) {
|
|
329
|
+
const traceDigest = rejectionDigest(trace.content);
|
|
330
|
+
const tombstone = findRejectedValue(consolidateDb, trace.tenantId, traceDigest);
|
|
331
|
+
if (tombstone) {
|
|
332
|
+
tracesSkippedRejected++;
|
|
333
|
+
try {
|
|
334
|
+
appendAuditEvent(consolidateDb, {
|
|
335
|
+
tenantId: trace.tenantId,
|
|
336
|
+
actor: 'sleep',
|
|
337
|
+
op: 'reject_refusal',
|
|
338
|
+
metadata: {
|
|
339
|
+
digest: traceDigest,
|
|
340
|
+
reason: tombstone.reason,
|
|
341
|
+
sourceSessionId: session.session_id,
|
|
342
|
+
},
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
// Best-effort — mirrors store.ts's audit() semantics.
|
|
347
|
+
}
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
pendingWrites.push(trace);
|
|
352
|
+
survivors.push(trace);
|
|
353
|
+
result.promotedTraces++;
|
|
354
|
+
result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
|
|
240
355
|
}
|
|
241
|
-
|
|
242
|
-
|
|
356
|
+
if (result.promotedTraces > 0) {
|
|
357
|
+
result.details.push(` 🧬 promoted ${result.promotedTraces} trace${result.promotedTraces === 1 ? '' : 's'} from completed session${result.promotedTraces === 1 ? '' : 's'}`);
|
|
243
358
|
}
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
}
|
|
247
|
-
// -------------------------------------------------------------------------
|
|
248
|
-
// 1.7. DAG summarization — cluster extracted facts and generate summaries
|
|
249
|
-
// -------------------------------------------------------------------------
|
|
250
|
-
const extractedFacts = survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1);
|
|
251
|
-
if (apiKey && extractedFacts.length >= 3 && !dryRun) {
|
|
252
|
-
try {
|
|
253
|
-
const { buildDag } = await import('./dag.js');
|
|
254
|
-
const dagResult = await buildDag(hippoRoot, extractedFacts, {
|
|
255
|
-
apiKey,
|
|
256
|
-
model: config.extraction.model,
|
|
257
|
-
});
|
|
258
|
-
result.dagCandidateClusters = dagResult.candidateClusters;
|
|
259
|
-
result.dagSummariesCreated = dagResult.summariesCreated;
|
|
260
|
-
if (dagResult.summariesCreated > 0) {
|
|
261
|
-
result.details.push(` 🌳 DAG: ${dagResult.summariesCreated} summaries created, ${dagResult.factsLinked} facts linked`);
|
|
359
|
+
if (tracesSkippedRejected > 0) {
|
|
360
|
+
console.error(`consolidate: skipped ${tracesSkippedRejected} auto-promoted trace(s) whose content matches a rejected value`);
|
|
262
361
|
}
|
|
263
362
|
}
|
|
264
|
-
|
|
265
|
-
|
|
363
|
+
// -------------------------------------------------------------------------
|
|
364
|
+
// 1.5. Replay pass — rehearse high-value survivors
|
|
365
|
+
// -------------------------------------------------------------------------
|
|
366
|
+
//
|
|
367
|
+
// Biologically-inspired counterpart to hippocampal replay during slow-wave
|
|
368
|
+
// sleep: sample N memories weighted by outcome + valence + under-rehearsal
|
|
369
|
+
// + idle time, then apply the same retrieval-strengthening `markRetrieved`
|
|
370
|
+
// applies to real queries. Distinct from decay (removal), physics (motion),
|
|
371
|
+
// and merge (compression) — this is the "rehearse the important stuff so
|
|
372
|
+
// it doesn't fade" pass.
|
|
373
|
+
{
|
|
374
|
+
const replayCount = config.replay?.count ?? REPLAY_COUNT_DEFAULT;
|
|
375
|
+
// EVAL-ONLY ablation (see ablation.ts): replay rehearsal IS recall
|
|
376
|
+
// strengthening (same markRetrieved dynamics), so the strengthen-off arm
|
|
377
|
+
// silences the whole pass - markRetrieved would return unmutated entries
|
|
378
|
+
// and persisting them anyway would still refresh updated_at / mirrors.
|
|
379
|
+
if (replayCount > 0 && survivors.length > 0 && !isRecallBoostAblated()) {
|
|
380
|
+
const seed = Math.floor(now.getTime() / 1000) & 0xffffffff;
|
|
381
|
+
const picked = sampleForReplay(survivors, replayCount, now, seed);
|
|
382
|
+
if (picked.length > 0) {
|
|
383
|
+
const rehearsed = markRetrieved(picked, now);
|
|
384
|
+
const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
|
|
385
|
+
// Update survivors in place so downstream passes see rehearsed state.
|
|
386
|
+
for (let i = 0; i < survivors.length; i++) {
|
|
387
|
+
const replacement = rehearsedById.get(survivors[i].id);
|
|
388
|
+
if (replacement)
|
|
389
|
+
survivors[i] = replacement;
|
|
390
|
+
}
|
|
391
|
+
result.replayed = rehearsed.length;
|
|
392
|
+
result.details.push(` 💭 replayed ${rehearsed.length} memor${rehearsed.length === 1 ? 'y' : 'ies'}: ` +
|
|
393
|
+
rehearsed.map((e) => e.id).join(', '));
|
|
394
|
+
if (!dryRun) {
|
|
395
|
+
for (const r of rehearsed)
|
|
396
|
+
pendingWrites.push(r);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
266
400
|
}
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
|
|
293
|
-
result.summariesRebuildCapped = rebuildResult.capped;
|
|
294
|
-
if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0) {
|
|
295
|
-
const parts = [];
|
|
296
|
-
if (rebuildResult.rebuilt > 0)
|
|
297
|
-
parts.push(`${rebuildResult.rebuilt} rebuilt`);
|
|
298
|
-
if (rebuildResult.zeroChildSkipped > 0)
|
|
299
|
-
parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
|
|
300
|
-
if (rebuildResult.failed > 0)
|
|
301
|
-
parts.push(`${rebuildResult.failed} failed`);
|
|
302
|
-
if (rebuildResult.capped)
|
|
303
|
-
parts.push(`CAPPED@${cap}`);
|
|
304
|
-
result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
|
|
401
|
+
// -------------------------------------------------------------------------
|
|
402
|
+
// 1.6. Batch extraction — extract facts from episodic memories missing them
|
|
403
|
+
// -------------------------------------------------------------------------
|
|
404
|
+
const extractedFromIds = new Set(survivors.filter((e) => e.extracted_from).map((e) => e.extracted_from));
|
|
405
|
+
const extractionCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !extractedFromIds.has(e.id));
|
|
406
|
+
result.extractionCandidates = extractionCandidates.length;
|
|
407
|
+
const apiKey = process.env.ANTHROPIC_API_KEY ?? '';
|
|
408
|
+
if (apiKey && extractionCandidates.length > 0 && !dryRun) {
|
|
409
|
+
const { extractFacts, storeExtractedFacts } = await import('./extract.js');
|
|
410
|
+
const batchLimit = 20;
|
|
411
|
+
let extractedCount = 0;
|
|
412
|
+
for (const candidate of extractionCandidates.slice(0, batchLimit)) {
|
|
413
|
+
try {
|
|
414
|
+
const facts = await extractFacts(candidate.content, {
|
|
415
|
+
apiKey,
|
|
416
|
+
model: config.extraction.model,
|
|
417
|
+
});
|
|
418
|
+
if (facts.length > 0) {
|
|
419
|
+
storeExtractedFacts(hippoRoot, candidate, facts);
|
|
420
|
+
extractedCount += facts.length;
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
catch {
|
|
424
|
+
// Best-effort — continue with next candidate
|
|
425
|
+
}
|
|
305
426
|
}
|
|
427
|
+
result.extracted = extractedCount;
|
|
306
428
|
}
|
|
307
|
-
|
|
308
|
-
|
|
429
|
+
// -------------------------------------------------------------------------
|
|
430
|
+
// 1.7. DAG summarization — cluster extracted facts and generate summaries
|
|
431
|
+
// -------------------------------------------------------------------------
|
|
432
|
+
const extractedFacts = survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1);
|
|
433
|
+
if (apiKey && extractedFacts.length >= 3 && !dryRun) {
|
|
434
|
+
try {
|
|
435
|
+
const { buildDag } = await import('./dag.js');
|
|
436
|
+
const dagResult = await buildDag(hippoRoot, extractedFacts, {
|
|
437
|
+
apiKey,
|
|
438
|
+
model: config.extraction.model,
|
|
439
|
+
});
|
|
440
|
+
result.dagCandidateClusters = dagResult.candidateClusters;
|
|
441
|
+
result.dagSummariesCreated = dagResult.summariesCreated;
|
|
442
|
+
if (dagResult.summariesCreated > 0) {
|
|
443
|
+
result.details.push(` 🌳 DAG: ${dagResult.summariesCreated} summaries created, ${dagResult.factsLinked} facts linked`);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
catch {
|
|
447
|
+
// Best-effort
|
|
448
|
+
}
|
|
309
449
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
450
|
+
// -------------------------------------------------------------------------
|
|
451
|
+
// 1.8. DAG summary rebuild — drain dirty queue from E2's child-write hooks
|
|
452
|
+
// -------------------------------------------------------------------------
|
|
453
|
+
// Consumer of E2's summary_dirty flag. Walks dirty L2 summaries, regenerates
|
|
454
|
+
// each via generateDagSummary, atomically refreshes content + 6 metadata
|
|
455
|
+
// columns + clears summary_dirty (with FTS sync). Same apiKey/dryRun gate
|
|
456
|
+
// as buildDag above. Cap HIPPO_DAG_REBUILD_CAP (default 20, hard ceiling
|
|
457
|
+
// 1000) prevents runaway LLM cost.
|
|
458
|
+
if (apiKey && !dryRun) {
|
|
459
|
+
try {
|
|
460
|
+
const { rebuildDirtySummaries } = await import('./dag.js');
|
|
461
|
+
const rawCap = parseInt(process.env.HIPPO_DAG_REBUILD_CAP ?? '20', 10);
|
|
462
|
+
// R1 MED must-fix: hard ceiling so misconfigured env can't burn
|
|
463
|
+
// unbounded LLM cost.
|
|
464
|
+
const cap = Number.isFinite(rawCap) && rawCap > 0
|
|
465
|
+
? Math.min(rawCap, 1000)
|
|
466
|
+
: 20;
|
|
467
|
+
const rebuildResult = await rebuildDirtySummaries(hippoRoot, {
|
|
327
468
|
apiKey,
|
|
328
469
|
model: config.extraction.model,
|
|
470
|
+
cap,
|
|
329
471
|
});
|
|
330
|
-
result.
|
|
331
|
-
|
|
332
|
-
|
|
472
|
+
result.summariesRebuilt = rebuildResult.rebuilt;
|
|
473
|
+
result.summariesRebuildFailed = rebuildResult.failed;
|
|
474
|
+
result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
|
|
475
|
+
result.summariesRebuildCapped = rebuildResult.capped;
|
|
476
|
+
if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0) {
|
|
477
|
+
const parts = [];
|
|
478
|
+
if (rebuildResult.rebuilt > 0)
|
|
479
|
+
parts.push(`${rebuildResult.rebuilt} rebuilt`);
|
|
480
|
+
if (rebuildResult.zeroChildSkipped > 0)
|
|
481
|
+
parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
|
|
482
|
+
if (rebuildResult.failed > 0)
|
|
483
|
+
parts.push(`${rebuildResult.failed} failed`);
|
|
484
|
+
if (rebuildResult.capped)
|
|
485
|
+
parts.push(`CAPPED@${cap}`);
|
|
486
|
+
result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
|
|
333
487
|
}
|
|
334
488
|
}
|
|
489
|
+
catch {
|
|
490
|
+
// Best-effort — same posture as buildDag block above.
|
|
491
|
+
}
|
|
335
492
|
}
|
|
336
|
-
|
|
337
|
-
|
|
493
|
+
// -------------------------------------------------------------------------
|
|
494
|
+
// 1.9. DAG entity profiles — cluster L2 topic summaries into L3 profiles
|
|
495
|
+
// -------------------------------------------------------------------------
|
|
496
|
+
// E5 phase: aggregate per-entity L2 summaries (e.g. all the speaker:Alice
|
|
497
|
+
// topic summaries) into a single L3 entity profile. Runs even when phase
|
|
498
|
+
// 1.7 buildDag was skipped (re-clusters existing L2s every sleep).
|
|
499
|
+
//
|
|
500
|
+
// Uses loadAllL2Summaries (not `survivors`) because phase 1.7 wrote new
|
|
501
|
+
// L2s directly via writeEntry without pushing back into survivors.
|
|
502
|
+
if (apiKey && !dryRun) {
|
|
503
|
+
try {
|
|
504
|
+
const { buildEntityProfiles } = await import('./dag.js');
|
|
505
|
+
const { loadAllL2Summaries } = await import('./store.js');
|
|
506
|
+
const l2Summaries = loadAllL2Summaries(hippoRoot);
|
|
507
|
+
if (l2Summaries.length >= 2) {
|
|
508
|
+
const profileResult = await buildEntityProfiles(hippoRoot, l2Summaries, {
|
|
509
|
+
apiKey,
|
|
510
|
+
model: config.extraction.model,
|
|
511
|
+
});
|
|
512
|
+
result.entityProfilesCreated = profileResult.profilesCreated;
|
|
513
|
+
if (profileResult.profilesCreated > 0) {
|
|
514
|
+
result.details.push(` 🌲 DAG L3: ${profileResult.profilesCreated} entity profiles, ${profileResult.l2sLinked} L2s linked`);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
catch {
|
|
519
|
+
// Best-effort.
|
|
520
|
+
}
|
|
338
521
|
}
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
set.
|
|
363
|
-
|
|
522
|
+
// -------------------------------------------------------------------------
|
|
523
|
+
// 2. Physics simulation pass
|
|
524
|
+
// -------------------------------------------------------------------------
|
|
525
|
+
if (!dryRun) {
|
|
526
|
+
try {
|
|
527
|
+
const physicsEnabled = config.physics.enabled === true
|
|
528
|
+
|| (config.physics.enabled === 'auto');
|
|
529
|
+
if (physicsEnabled) {
|
|
530
|
+
const db = openHippoDb(hippoRoot);
|
|
531
|
+
try {
|
|
532
|
+
const physicsMap = loadPhysicsState(db);
|
|
533
|
+
const particles = Array.from(physicsMap.values());
|
|
534
|
+
if (particles.length > 0) {
|
|
535
|
+
// Build entry lookup for property refresh
|
|
536
|
+
const entryMap = new Map(survivors.map(e => [e.id, e]));
|
|
537
|
+
refreshParticleProperties(particles, entryMap, now);
|
|
538
|
+
// Build conflict pairs from survivors
|
|
539
|
+
const conflictPairs = new Map();
|
|
540
|
+
for (const entry of survivors) {
|
|
541
|
+
if (entry.conflicts_with.length > 0) {
|
|
542
|
+
const set = conflictPairs.get(entry.id) ?? new Set();
|
|
543
|
+
for (const cid of entry.conflicts_with)
|
|
544
|
+
set.add(cid);
|
|
545
|
+
conflictPairs.set(entry.id, set);
|
|
546
|
+
}
|
|
364
547
|
}
|
|
548
|
+
// Build half-life lookup
|
|
549
|
+
const halfLives = new Map();
|
|
550
|
+
for (const entry of survivors) {
|
|
551
|
+
halfLives.set(entry.id, entry.half_life_days);
|
|
552
|
+
}
|
|
553
|
+
const ctx = {
|
|
554
|
+
conflictPairs,
|
|
555
|
+
halfLives,
|
|
556
|
+
config: config.physics,
|
|
557
|
+
};
|
|
558
|
+
const stats = simulate(particles, ctx);
|
|
559
|
+
savePhysicsState(db, particles);
|
|
560
|
+
result.physicsSimulated = stats.particleCount;
|
|
561
|
+
result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
|
|
562
|
+
`avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
|
|
563
|
+
`energy ${stats.energy.total.toFixed(4)}`);
|
|
365
564
|
}
|
|
366
|
-
// Build half-life lookup
|
|
367
|
-
const halfLives = new Map();
|
|
368
|
-
for (const entry of survivors) {
|
|
369
|
-
halfLives.set(entry.id, entry.half_life_days);
|
|
370
|
-
}
|
|
371
|
-
const ctx = {
|
|
372
|
-
conflictPairs,
|
|
373
|
-
halfLives,
|
|
374
|
-
config: config.physics,
|
|
375
|
-
};
|
|
376
|
-
const stats = simulate(particles, ctx);
|
|
377
|
-
savePhysicsState(db, particles);
|
|
378
|
-
result.physicsSimulated = stats.particleCount;
|
|
379
|
-
result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
|
|
380
|
-
`avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
|
|
381
|
-
`energy ${stats.energy.total.toFixed(4)}`);
|
|
382
565
|
}
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
566
|
+
finally {
|
|
567
|
+
closeHippoDb(db);
|
|
568
|
+
}
|
|
386
569
|
}
|
|
387
570
|
}
|
|
571
|
+
catch (error) {
|
|
572
|
+
result.details.push(` ⚠️ physics simulation skipped: ${error instanceof Error ? error.message : 'unknown error'}`);
|
|
573
|
+
}
|
|
388
574
|
}
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
575
|
+
// -------------------------------------------------------------------------
|
|
576
|
+
// 3. Merge pass - episodic entries only
|
|
577
|
+
// -------------------------------------------------------------------------
|
|
578
|
+
const mergeCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !e.tags.includes('extracted'));
|
|
579
|
+
const used = new Set();
|
|
580
|
+
// AT1 consolidation-loop fix (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
|
|
581
|
+
// reuses the single consolidateDb handle opened once above (before the
|
|
582
|
+
// auto-promote pass, 1.4) for the whole non-dry-run consolidate — see that
|
|
583
|
+
// declaration's comment. Only needed for real writes — a dry-run preview
|
|
584
|
+
// never reaches batchWriteAndDelete's guard bypass, so there is nothing
|
|
585
|
+
// here for it to protect against. (mergesSkippedRejected itself is
|
|
586
|
+
// declared up at the try's opening above 1.4, not here — it has to
|
|
587
|
+
// survive the try/finally that now wraps this whole section; see the
|
|
588
|
+
// handle-leak restructure comment there.)
|
|
589
|
+
for (let i = 0; i < mergeCandidates.length; i++) {
|
|
590
|
+
if (used.has(mergeCandidates[i].id))
|
|
404
591
|
continue;
|
|
405
|
-
const
|
|
406
|
-
|
|
407
|
-
|
|
592
|
+
const cluster = [mergeCandidates[i]];
|
|
593
|
+
for (let j = i + 1; j < mergeCandidates.length; j++) {
|
|
594
|
+
if (used.has(mergeCandidates[j].id))
|
|
595
|
+
continue;
|
|
596
|
+
const overlap = textOverlap(mergeCandidates[i].content, mergeCandidates[j].content);
|
|
597
|
+
if (overlap >= MERGE_OVERLAP_THRESHOLD) {
|
|
598
|
+
cluster.push(mergeCandidates[j]);
|
|
599
|
+
}
|
|
408
600
|
}
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
//
|
|
441
|
-
//
|
|
442
|
-
//
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
601
|
+
if (cluster.length < MERGE_MIN_CLUSTER)
|
|
602
|
+
continue;
|
|
603
|
+
// Create a semantic summary
|
|
604
|
+
const mergedContent = mergeContents(cluster);
|
|
605
|
+
const allTags = Array.from(new Set(cluster.flatMap((e) => e.tags)));
|
|
606
|
+
const maxValence = pickStrongestValence(cluster);
|
|
607
|
+
// AT1 P2 fix: build the semantic entry FIRST — createMemory is cheap
|
|
608
|
+
// and pure — so the tombstone check below runs under the tenant the
|
|
609
|
+
// row will ACTUALLY land in. The prior version checked
|
|
610
|
+
// cluster[0].tenantId, but createMemory (below) is never passed a
|
|
611
|
+
// tenantId option, so it always stamps its own default ('default',
|
|
612
|
+
// see memory.ts) regardless of the cluster's source tenant — the
|
|
613
|
+
// check was consulting a tenant's tombstones that the write was never
|
|
614
|
+
// going to land in (a false negative on the real destination, and a
|
|
615
|
+
// possible false-block on the source tenant's unrelated tombstones).
|
|
616
|
+
// This fix only makes the CHECK match wherever the row actually
|
|
617
|
+
// lands; it deliberately does NOT change that destination — merge
|
|
618
|
+
// rows always landing in 'default' regardless of source tenant is a
|
|
619
|
+
// real, pre-existing cross-tenant question, tracked separately as an
|
|
620
|
+
// AT1 follow-up (not this fix's scope) rather than folded in here.
|
|
621
|
+
let semantic = null;
|
|
622
|
+
if (!dryRun) {
|
|
623
|
+
semantic = createMemory(mergedContent, {
|
|
624
|
+
layer: Layer.Semantic,
|
|
625
|
+
tags: allTags,
|
|
626
|
+
emotional_valence: maxValence,
|
|
627
|
+
schema_fit: 0.7,
|
|
628
|
+
source: 'consolidation',
|
|
629
|
+
confidence: 'inferred',
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
// mergeContents is DETERMINISTIC CONCATENATION (not an LLM paraphrase)
|
|
633
|
+
// — if a human rejected exactly this byte-identical rollup before, an
|
|
634
|
+
// unguarded sleep would regenerate it every cycle and
|
|
635
|
+
// batchWriteAndDelete's guard bypass (store.ts) would silently
|
|
636
|
+
// re-assert it forever. This producer-side check is what makes that
|
|
637
|
+
// bypass safe. A hit skips the WHOLE cluster: sources stay unmerged —
|
|
638
|
+
// not demoted, not deleted — so a later sleep gets another chance if
|
|
639
|
+
// the tombstone is lifted.
|
|
640
|
+
if (consolidateDb && semantic) {
|
|
641
|
+
const mergeDigest = rejectionDigest(semantic.content);
|
|
642
|
+
const tombstone = findRejectedValue(consolidateDb, semantic.tenantId, mergeDigest);
|
|
643
|
+
if (tombstone) {
|
|
644
|
+
// Still mark used — these members are not re-tried against a
|
|
645
|
+
// DIFFERENT cluster within this same pass; next sleep re-clusters
|
|
646
|
+
// them fresh.
|
|
647
|
+
for (const e of cluster)
|
|
648
|
+
used.add(e.id);
|
|
649
|
+
mergesSkippedRejected++;
|
|
650
|
+
try {
|
|
651
|
+
appendAuditEvent(consolidateDb, {
|
|
652
|
+
tenantId: semantic.tenantId,
|
|
653
|
+
actor: 'sleep',
|
|
654
|
+
op: 'reject_refusal',
|
|
655
|
+
metadata: {
|
|
656
|
+
digest: mergeDigest,
|
|
657
|
+
reason: tombstone.reason,
|
|
658
|
+
sourceIds: cluster.map((e) => e.id),
|
|
659
|
+
},
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
catch {
|
|
663
|
+
// Best-effort — mirrors store.ts's audit() semantics.
|
|
664
|
+
}
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
// Mark cluster members as used
|
|
669
|
+
for (const e of cluster)
|
|
670
|
+
used.add(e.id);
|
|
671
|
+
result.merged += cluster.length;
|
|
672
|
+
result.details.push(` 🔀 merged ${cluster.length} episodic entries into semantic: "${mergedContent.slice(0, 60)}..."`);
|
|
673
|
+
if (!dryRun && semantic) {
|
|
674
|
+
pendingWrites.push(semantic);
|
|
675
|
+
result.semanticCreated++;
|
|
676
|
+
// Demote source episodics (they've been compressed into neocortex):
|
|
677
|
+
// scale half_life_days so they decay sooner while staying recoverable.
|
|
678
|
+
// Immediate ranking is deliberately unchanged: the 2026-06-10 DAG
|
|
679
|
+
// slice-1 eval measured that dropping children below a worse-retrieving
|
|
680
|
+
// summary regresses budget-bounded QA (docs/evals/). The stored
|
|
681
|
+
// strength is refreshed to the live value so inspect, replay sampling,
|
|
682
|
+
// and strength-sorted assembly see the truth instead of a fake 0.3.
|
|
683
|
+
// Mutate in place (not a copy): `cluster` holds the same object
|
|
684
|
+
// references as `survivors`, and the later detectConflicts(survivors)
|
|
685
|
+
// pass in this same run must see the post-demotion half-life, or it
|
|
686
|
+
// can persist conflicts for entries the just-written state excludes.
|
|
687
|
+
for (const e of cluster) {
|
|
688
|
+
e.half_life_days = Math.max(1, Math.floor(e.half_life_days * MERGE_SOURCE_HALF_LIFE_FACTOR));
|
|
689
|
+
e.strength = calculateStrength(e, now, decayOpts);
|
|
690
|
+
pendingWrites.push(e);
|
|
691
|
+
}
|
|
447
692
|
}
|
|
448
693
|
}
|
|
449
694
|
}
|
|
695
|
+
finally {
|
|
696
|
+
if (consolidateDb)
|
|
697
|
+
closeHippoDb(consolidateDb);
|
|
698
|
+
}
|
|
699
|
+
if (mergesSkippedRejected > 0) {
|
|
700
|
+
console.error(`consolidate: skipped ${mergesSkippedRejected} merge(s) whose content matches a rejected value`);
|
|
701
|
+
}
|
|
450
702
|
// Flush all writes/deletes in a single transaction
|
|
451
703
|
if (!dryRun) {
|
|
452
704
|
batchWriteAndDelete(hippoRoot, pendingWrites, pendingDeletes);
|
|
@@ -455,7 +707,7 @@ export async function consolidate(hippoRoot, options = {}) {
|
|
|
455
707
|
// 4. Log run
|
|
456
708
|
// -------------------------------------------------------------------------
|
|
457
709
|
if (!dryRun) {
|
|
458
|
-
const detectedConflicts = detectConflicts(survivors, now, decayOpts);
|
|
710
|
+
const detectedConflicts = detectConflicts(survivors, now, decayOpts, rescuedIds);
|
|
459
711
|
replaceDetectedConflicts(hippoRoot, detectedConflicts, now.toISOString());
|
|
460
712
|
if (detectedConflicts.length > 0) {
|
|
461
713
|
result.details.push(` ⚠️ detected ${detectedConflicts.length} memory conflict${detectedConflicts.length === 1 ? '' : 's'}`);
|
|
@@ -467,6 +719,57 @@ export async function consolidate(hippoRoot, options = {}) {
|
|
|
467
719
|
removed: result.removed,
|
|
468
720
|
});
|
|
469
721
|
incrementSleepCount(hippoRoot);
|
|
722
|
+
// One audit row per rescue (attributability, D1). Written here, AFTER
|
|
723
|
+
// batchWriteAndDelete above has committed this cycle's writes/deletes
|
|
724
|
+
// (and after conflict detection + run logging), not inline in the decay
|
|
725
|
+
// pass — same durability posture as api.ts's top-level 'consolidate'
|
|
726
|
+
// summary audit row (written only once the whole sleep has completed).
|
|
727
|
+
// Writing it earlier would assert rescues for a cycle whose effects
|
|
728
|
+
// never landed if a later phase threw. Real writes only — dry-run
|
|
729
|
+
// previews the decision (details line above) but persists nothing.
|
|
730
|
+
if (rescuedEntries.length > 0) {
|
|
731
|
+
try {
|
|
732
|
+
const auditDb = openHippoDb(hippoRoot);
|
|
733
|
+
try {
|
|
734
|
+
// Review-round F5: per-row try/catch, not one try/catch around the
|
|
735
|
+
// whole loop — a single failed appendAuditEvent must not silently
|
|
736
|
+
// drop every remaining row. Mirrors the physics pass's
|
|
737
|
+
// skipped-warning precedent (line ~564 above): count losses, keep
|
|
738
|
+
// the overall fail-soft posture, tell the operator via details.
|
|
739
|
+
let auditFailures = 0;
|
|
740
|
+
for (const entry of rescuedEntries) {
|
|
741
|
+
try {
|
|
742
|
+
const rank = rankById.get(entry.id);
|
|
743
|
+
appendAuditEvent(auditDb, {
|
|
744
|
+
tenantId: entry.tenantId,
|
|
745
|
+
actor: 'sleep',
|
|
746
|
+
op: 'mv_rescue',
|
|
747
|
+
targetId: entry.id,
|
|
748
|
+
metadata: rank
|
|
749
|
+
? { rank: rank.rank, totalNonPinned: rank.totalNonPinned, keepN: rank.keepN, score: rank.score }
|
|
750
|
+
: {},
|
|
751
|
+
});
|
|
752
|
+
}
|
|
753
|
+
catch {
|
|
754
|
+
auditFailures++;
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
if (auditFailures > 0) {
|
|
758
|
+
result.details.push(` ⚠️ memory-value: ${auditFailures} mv_rescue audit row${auditFailures === 1 ? '' : 's'} ` +
|
|
759
|
+
`failed to write (the rescue itself still landed)`);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
finally {
|
|
763
|
+
closeHippoDb(auditDb);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
catch {
|
|
767
|
+
// openHippoDb/closeHippoDb-level failure: audit must never crash a
|
|
768
|
+
// mutation (mirrors store.ts's audit() posture).
|
|
769
|
+
result.details.push(` ⚠️ memory-value: mv_rescue audit unavailable this cycle ` +
|
|
770
|
+
`(${rescuedEntries.length} rescue${rescuedEntries.length === 1 ? '' : 's'} not audited)`);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
470
773
|
}
|
|
471
774
|
return result;
|
|
472
775
|
}
|
|
@@ -495,8 +798,16 @@ function pickStrongestValence(entries) {
|
|
|
495
798
|
}
|
|
496
799
|
return 'neutral';
|
|
497
800
|
}
|
|
498
|
-
function detectConflicts(entries, now, decayOpts = {}
|
|
499
|
-
|
|
801
|
+
function detectConflicts(entries, now, decayOpts = {},
|
|
802
|
+
// LC2-E3 (opt-in, default off): ids rescued by this cycle's decay pass.
|
|
803
|
+
// detectConflicts recomputes its own strength>=DECAY_THRESHOLD survivor
|
|
804
|
+
// filter independently of the decay pass above; without this bypass,
|
|
805
|
+
// rescued entries would be silently re-excluded from conflict detection
|
|
806
|
+
// every cycle even though the decay pass just decided to keep them.
|
|
807
|
+
// Default empty set: flag-off behavior is unchanged.
|
|
808
|
+
rescuedIds = new Set()) {
|
|
809
|
+
const survivors = entries.filter((entry) => entry.layer !== Layer.Semantic
|
|
810
|
+
&& (rescuedIds.has(entry.id) || calculateStrength(entry, now, decayOpts) >= DECAY_THRESHOLD));
|
|
500
811
|
const detected = [];
|
|
501
812
|
for (let i = 0; i < survivors.length; i++) {
|
|
502
813
|
for (let j = i + 1; j < survivors.length; j++) {
|