hippo-memory 1.30.0 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +49 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +78 -2
- package/dist/api.js.map +1 -1
- package/dist/audit.d.ts +1 -1
- package/dist/audit.d.ts.map +1 -1
- package/dist/audit.js.map +1 -1
- package/dist/capture.d.ts.map +1 -1
- package/dist/capture.js +79 -23
- package/dist/capture.js.map +1 -1
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +233 -13
- package/dist/cli.js.map +1 -1
- package/dist/connectors/github/ingest.d.ts.map +1 -1
- package/dist/connectors/github/ingest.js +17 -0
- package/dist/connectors/github/ingest.js.map +1 -1
- package/dist/connectors/slack/ingest.d.ts.map +1 -1
- package/dist/connectors/slack/ingest.js +15 -0
- package/dist/connectors/slack/ingest.js.map +1 -1
- package/dist/consolidate.d.ts +1 -0
- package/dist/consolidate.d.ts.map +1 -1
- package/dist/consolidate.js +471 -291
- package/dist/consolidate.js.map +1 -1
- package/dist/dag.d.ts +17 -1
- package/dist/dag.d.ts.map +1 -1
- package/dist/dag.js +115 -40
- package/dist/dag.js.map +1 -1
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +43 -1
- package/dist/db.js.map +1 -1
- package/dist/extract.d.ts.map +1 -1
- package/dist/extract.js +24 -1
- package/dist/extract.js.map +1 -1
- package/dist/importers.d.ts +10 -0
- package/dist/importers.d.ts.map +1 -1
- package/dist/importers.js +242 -159
- package/dist/importers.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +58 -24
- package/dist/mcp/server.js.map +1 -1
- package/dist/reject-flow.d.ts +66 -0
- package/dist/reject-flow.d.ts.map +1 -0
- package/dist/reject-flow.js +207 -0
- package/dist/reject-flow.js.map +1 -0
- package/dist/rejection.d.ts +100 -0
- package/dist/rejection.d.ts.map +1 -0
- package/dist/rejection.js +157 -0
- package/dist/rejection.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +4 -0
- package/dist/server.js.map +1 -1
- package/dist/shared.d.ts +9 -0
- package/dist/shared.d.ts.map +1 -1
- package/dist/shared.js +57 -6
- package/dist/shared.js.map +1 -1
- package/dist/sleep-redact.d.ts +2 -0
- package/dist/sleep-redact.d.ts.map +1 -1
- package/dist/sleep-redact.js +2 -0
- package/dist/sleep-redact.js.map +1 -1
- package/dist/src/api.js +78 -2
- package/dist/src/api.js.map +1 -1
- package/dist/src/audit.js.map +1 -1
- package/dist/src/capture.js +79 -23
- package/dist/src/capture.js.map +1 -1
- package/dist/src/cli.js +233 -13
- package/dist/src/cli.js.map +1 -1
- package/dist/src/connectors/github/ingest.js +17 -0
- package/dist/src/connectors/github/ingest.js.map +1 -1
- package/dist/src/connectors/slack/ingest.js +15 -0
- package/dist/src/connectors/slack/ingest.js.map +1 -1
- package/dist/src/consolidate.js +471 -291
- package/dist/src/consolidate.js.map +1 -1
- package/dist/src/dag.js +115 -40
- package/dist/src/dag.js.map +1 -1
- package/dist/src/db.js +43 -1
- package/dist/src/db.js.map +1 -1
- package/dist/src/extract.js +24 -1
- package/dist/src/extract.js.map +1 -1
- package/dist/src/importers.js +242 -159
- package/dist/src/importers.js.map +1 -1
- package/dist/src/mcp/server.js +58 -24
- package/dist/src/mcp/server.js.map +1 -1
- package/dist/src/reject-flow.js +207 -0
- package/dist/src/reject-flow.js.map +1 -0
- package/dist/src/rejection.js +157 -0
- package/dist/src/rejection.js.map +1 -0
- package/dist/src/server.js +4 -0
- package/dist/src/server.js.map +1 -1
- package/dist/src/shared.js +57 -6
- package/dist/src/shared.js.map +1 -1
- package/dist/src/sleep-redact.js +2 -0
- package/dist/src/sleep-redact.js.map +1 -1
- package/dist/src/store.js +511 -38
- package/dist/src/store.js.map +1 -1
- package/dist/src/version.js +1 -1
- package/dist/store.d.ts +116 -6
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +511 -38
- package/dist/store.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/consolidate.js
CHANGED
|
@@ -12,6 +12,7 @@ import { loadAllEntries, batchWriteAndDelete, appendConsolidationRun, replaceDet
|
|
|
12
12
|
import { textOverlap, markRetrieved } from './search.js';
|
|
13
13
|
import { compareEntryIdentity } from './compare.js';
|
|
14
14
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
15
|
+
import { rejectionDigest, findRejectedValue } from './rejection.js';
|
|
15
16
|
import { loadPhysicsState, savePhysicsState, refreshParticleProperties } from './physics-state.js';
|
|
16
17
|
import { simulate } from './physics.js';
|
|
17
18
|
import { loadConfig } from './config.js';
|
|
@@ -75,6 +76,7 @@ export async function consolidate(hippoRoot, options = {}) {
|
|
|
75
76
|
summariesRebuilt: 0,
|
|
76
77
|
summariesRebuildFailed: 0,
|
|
77
78
|
summariesZeroChildSkipped: 0,
|
|
79
|
+
summariesRebuildRefused: 0,
|
|
78
80
|
summariesRebuildCapped: false,
|
|
79
81
|
entityProfilesCreated: 0,
|
|
80
82
|
dryRun,
|
|
@@ -236,331 +238,509 @@ export async function consolidate(hippoRoot, options = {}) {
|
|
|
236
238
|
}
|
|
237
239
|
}
|
|
238
240
|
}
|
|
239
|
-
//
|
|
240
|
-
// 1.4
|
|
241
|
-
//
|
|
241
|
+
// AT1 rejection-guard db handle (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
|
|
242
|
+
// covers BOTH the auto-promote pass (1.4, immediately below) and the merge
|
|
243
|
+
// pass (3, further down) — both build deterministic content that
|
|
244
|
+
// batchWriteAndDelete writes through the guard's bypass, so both need a
|
|
245
|
+
// producer-side tombstone check before pushing to pendingWrites.
|
|
242
246
|
//
|
|
243
|
-
//
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
247
|
+
// T3 fix (2026-08-15 hardening pass, perf hygiene): memoized lazy getter,
|
|
248
|
+
// not an eager open. The handle only serves these two tombstone checks —
|
|
249
|
+
// a sleep with zero promotable sessions and zero merge clusters never
|
|
250
|
+
// reaches either use site, so opening it unconditionally on every
|
|
251
|
+
// non-dry-run sleep paid a db-open cost for nothing. dryRun still never
|
|
252
|
+
// opens (getConsolidateDb short-circuits before touching the handle).
|
|
253
|
+
// Every `if (consolidateDb)` guard below becomes `const consolidateDb =
|
|
254
|
+
// getConsolidateDb();` at its use site — same semantics, opened on first
|
|
255
|
+
// real use. consolidateDbOpened (not just a truthy handle check) is the
|
|
256
|
+
// single source of truth for "was this ever opened", so the finally below
|
|
257
|
+
// closes it exactly once and never double-opens.
|
|
258
|
+
//
|
|
259
|
+
// AT1 P2 fix (codex, handle-leak restructure): the getter's lifetime must
|
|
260
|
+
// start IMMEDIATELY before the try whose finally closes it, covering every
|
|
261
|
+
// phase that can touch it — not just the merge pass. Previously the
|
|
262
|
+
// try/finally wrapped only section 3 (merge pass); an exception thrown by
|
|
263
|
+
// auto-promote (1.4), replay (1.5), batch extraction (1.6), the DAG
|
|
264
|
+
// passes (1.7-1.9), or physics (2) would propagate past an open handle
|
|
265
|
+
// with nothing to close it. No behavior change on the happy path — each
|
|
266
|
+
// of those phases already best-effort catches its own exceptions today
|
|
267
|
+
// (physics has its own try/catch below; each DAG pass wraps its own
|
|
268
|
+
// dynamic import + call in try/catch) — this only closes the handle on
|
|
269
|
+
// the rare path where one of them throws past its own catch. Deliberately
|
|
270
|
+
// NOT re-indented (mechanical wrap only, kept surgical): every statement
|
|
271
|
+
// between this try and its finally (below, at the end of the merge pass)
|
|
272
|
+
// stays at its original indentation.
|
|
273
|
+
let consolidateDbHandle = null;
|
|
274
|
+
let consolidateDbOpened = false;
|
|
275
|
+
const getConsolidateDb = () => {
|
|
276
|
+
if (dryRun)
|
|
277
|
+
return null;
|
|
278
|
+
if (!consolidateDbOpened) {
|
|
279
|
+
consolidateDbHandle = openHippoDb(hippoRoot);
|
|
280
|
+
consolidateDbOpened = true;
|
|
281
|
+
}
|
|
282
|
+
return consolidateDbHandle;
|
|
283
|
+
};
|
|
284
|
+
// Declared here (not at the merge pass, its point of use) so it survives
|
|
285
|
+
// this try/finally — a `let` declared INSIDE the try would go out of
|
|
286
|
+
// scope before the `if (mergesSkippedRejected > 0)` check that reads it
|
|
287
|
+
// after the finally closes the handle.
|
|
288
|
+
let mergesSkippedRejected = 0;
|
|
289
|
+
try {
|
|
290
|
+
// -------------------------------------------------------------------------
|
|
291
|
+
// 1.4. Auto-promote complete sessions to traces
|
|
292
|
+
// -------------------------------------------------------------------------
|
|
293
|
+
//
|
|
294
|
+
// For each session within the configured window that has a `session_complete`
|
|
295
|
+
// event and no existing trace (idempotency via the source_session_id column),
|
|
296
|
+
// render the action sequence as markdown and persist a Layer.Trace memory.
|
|
297
|
+
// Traces inherit decay, search, replay, and physics from the base MemoryEntry.
|
|
298
|
+
if (!dryRun && config.autoTraceCapture !== false) {
|
|
299
|
+
let tracesSkippedRejected = 0;
|
|
300
|
+
const windowDays = config.autoTraceWindowDays ?? 7;
|
|
301
|
+
const sinceMs = now.getTime() - windowDays * 24 * 60 * 60 * 1000;
|
|
302
|
+
// Auto-trace currently runs in a single-tenant context (the env-resolved
|
|
303
|
+
// tenant for this process). Multi-tenant deployments that want
|
|
304
|
+
// consolidation across all tenants need a per-tenant loop layered on top
|
|
305
|
+
// of this — tracked in docs/plans/2026-05-02-continuity-tables-tenant-scope.md.
|
|
306
|
+
const consolidationTenant = resolveTenantId({});
|
|
307
|
+
const promotable = findPromotableSessions(hippoRoot, consolidationTenant, sinceMs);
|
|
308
|
+
for (const session of promotable) {
|
|
309
|
+
// Idempotency: skip if a trace for this session already exists.
|
|
310
|
+
if (traceExistsForSession(hippoRoot, consolidationTenant, session.session_id))
|
|
311
|
+
continue;
|
|
312
|
+
const events = listSessionEvents(hippoRoot, consolidationTenant, {
|
|
313
|
+
session_id: session.session_id,
|
|
314
|
+
limit: 1000,
|
|
315
|
+
});
|
|
316
|
+
const completeEvent = events.find((e) => e.event_type === 'session_complete');
|
|
317
|
+
if (!completeEvent)
|
|
318
|
+
continue; // defence-in-depth; findPromotableSessions filters already.
|
|
319
|
+
const outcomeRaw = completeEvent.content;
|
|
320
|
+
if (outcomeRaw !== 'success' && outcomeRaw !== 'failure' && outcomeRaw !== 'partial') {
|
|
321
|
+
// Malformed terminal event — skip rather than crash the whole sleep.
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
const outcome = outcomeRaw;
|
|
325
|
+
const steps = events
|
|
326
|
+
.filter((e) => e.event_type !== 'session_complete')
|
|
327
|
+
.map((e) => ({ action: e.content, observation: '' }));
|
|
328
|
+
const summary = typeof completeEvent.metadata.summary === 'string'
|
|
329
|
+
? completeEvent.metadata.summary
|
|
330
|
+
: '(untitled)';
|
|
331
|
+
const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
|
|
332
|
+
layer: Layer.Trace,
|
|
333
|
+
trace_outcome: outcome,
|
|
334
|
+
source_session_id: session.session_id,
|
|
335
|
+
tags: ['auto-promoted'],
|
|
336
|
+
source: 'auto-promote',
|
|
337
|
+
// T1 fix (2026-08-15 hardening pass): stamp the trace into the SAME
|
|
338
|
+
// tenant the traceExistsForSession idempotency check (above) runs
|
|
339
|
+
// under. Before
|
|
340
|
+
// this, createMemory omitted tenantId and the trace always landed
|
|
341
|
+
// 'default' (memory.ts:535) while the idempotency check ran under
|
|
342
|
+
// consolidationTenant — for any non-default tenant that check never
|
|
343
|
+
// hit, and the trace regenerated every sleep.
|
|
344
|
+
tenantId: consolidationTenant,
|
|
345
|
+
});
|
|
346
|
+
// AT1 (same producer-side pattern as the merge pass below): traceExistsForSession
|
|
347
|
+
// only sees rows CURRENTLY in the store — once a rejected trace is
|
|
348
|
+
// removed, that idempotency check no longer blocks regeneration, and
|
|
349
|
+
// this write would otherwise reach batchWriteAndDelete's guard bypass
|
|
350
|
+
// unchecked, resurrecting it every sleep. Check under THE ENTRY'S OWN
|
|
351
|
+
// stamped tenantId (read off `trace` after createMemory — never guess
|
|
352
|
+
// the tenant) + the built content's digest. A hit skips the push
|
|
353
|
+
// entirely: not counted as promoted, not added to survivors.
|
|
354
|
+
const consolidateDb = getConsolidateDb();
|
|
355
|
+
if (consolidateDb) {
|
|
356
|
+
const traceDigest = rejectionDigest(trace.content);
|
|
357
|
+
const tombstone = findRejectedValue(consolidateDb, trace.tenantId, traceDigest);
|
|
358
|
+
if (tombstone) {
|
|
359
|
+
tracesSkippedRejected++;
|
|
360
|
+
try {
|
|
361
|
+
appendAuditEvent(consolidateDb, {
|
|
362
|
+
tenantId: trace.tenantId,
|
|
363
|
+
actor: 'sleep',
|
|
364
|
+
op: 'reject_refusal',
|
|
365
|
+
metadata: {
|
|
366
|
+
digest: traceDigest,
|
|
367
|
+
reason: tombstone.reason,
|
|
368
|
+
sourceSessionId: session.session_id,
|
|
369
|
+
},
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
catch {
|
|
373
|
+
// Best-effort — mirrors store.ts's audit() semantics.
|
|
374
|
+
}
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
pendingWrites.push(trace);
|
|
379
|
+
survivors.push(trace);
|
|
380
|
+
result.promotedTraces++;
|
|
381
|
+
result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
|
|
382
|
+
}
|
|
383
|
+
if (result.promotedTraces > 0) {
|
|
384
|
+
result.details.push(` 🧬 promoted ${result.promotedTraces} trace${result.promotedTraces === 1 ? '' : 's'} from completed session${result.promotedTraces === 1 ? '' : 's'}`);
|
|
385
|
+
}
|
|
386
|
+
if (tracesSkippedRejected > 0) {
|
|
387
|
+
console.error(`consolidate: skipped ${tracesSkippedRejected} auto-promoted trace(s) whose content matches a rejected value`);
|
|
271
388
|
}
|
|
272
|
-
const outcome = outcomeRaw;
|
|
273
|
-
const steps = events
|
|
274
|
-
.filter((e) => e.event_type !== 'session_complete')
|
|
275
|
-
.map((e) => ({ action: e.content, observation: '' }));
|
|
276
|
-
const summary = typeof completeEvent.metadata.summary === 'string'
|
|
277
|
-
? completeEvent.metadata.summary
|
|
278
|
-
: '(untitled)';
|
|
279
|
-
const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
|
|
280
|
-
layer: Layer.Trace,
|
|
281
|
-
trace_outcome: outcome,
|
|
282
|
-
source_session_id: session.session_id,
|
|
283
|
-
tags: ['auto-promoted'],
|
|
284
|
-
source: 'auto-promote',
|
|
285
|
-
});
|
|
286
|
-
pendingWrites.push(trace);
|
|
287
|
-
survivors.push(trace);
|
|
288
|
-
result.promotedTraces++;
|
|
289
|
-
result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
|
|
290
389
|
}
|
|
291
|
-
|
|
292
|
-
|
|
390
|
+
// -------------------------------------------------------------------------
|
|
391
|
+
// 1.5. Replay pass — rehearse high-value survivors
|
|
392
|
+
// -------------------------------------------------------------------------
|
|
393
|
+
//
|
|
394
|
+
// Biologically-inspired counterpart to hippocampal replay during slow-wave
|
|
395
|
+
// sleep: sample N memories weighted by outcome + valence + under-rehearsal
|
|
396
|
+
// + idle time, then apply the same retrieval-strengthening `markRetrieved`
|
|
397
|
+
// applies to real queries. Distinct from decay (removal), physics (motion),
|
|
398
|
+
// and merge (compression) — this is the "rehearse the important stuff so
|
|
399
|
+
// it doesn't fade" pass.
|
|
400
|
+
{
|
|
401
|
+
const replayCount = config.replay?.count ?? REPLAY_COUNT_DEFAULT;
|
|
402
|
+
// EVAL-ONLY ablation (see ablation.ts): replay rehearsal IS recall
|
|
403
|
+
// strengthening (same markRetrieved dynamics), so the strengthen-off arm
|
|
404
|
+
// silences the whole pass - markRetrieved would return unmutated entries
|
|
405
|
+
// and persisting them anyway would still refresh updated_at / mirrors.
|
|
406
|
+
if (replayCount > 0 && survivors.length > 0 && !isRecallBoostAblated()) {
|
|
407
|
+
const seed = Math.floor(now.getTime() / 1000) & 0xffffffff;
|
|
408
|
+
const picked = sampleForReplay(survivors, replayCount, now, seed);
|
|
409
|
+
if (picked.length > 0) {
|
|
410
|
+
const rehearsed = markRetrieved(picked, now);
|
|
411
|
+
const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
|
|
412
|
+
// Update survivors in place so downstream passes see rehearsed state.
|
|
413
|
+
for (let i = 0; i < survivors.length; i++) {
|
|
414
|
+
const replacement = rehearsedById.get(survivors[i].id);
|
|
415
|
+
if (replacement)
|
|
416
|
+
survivors[i] = replacement;
|
|
417
|
+
}
|
|
418
|
+
result.replayed = rehearsed.length;
|
|
419
|
+
result.details.push(` 💭 replayed ${rehearsed.length} memor${rehearsed.length === 1 ? 'y' : 'ies'}: ` +
|
|
420
|
+
rehearsed.map((e) => e.id).join(', '));
|
|
421
|
+
if (!dryRun) {
|
|
422
|
+
for (const r of rehearsed)
|
|
423
|
+
pendingWrites.push(r);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
293
427
|
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
const rehearsed = markRetrieved(picked, now);
|
|
316
|
-
const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
|
|
317
|
-
// Update survivors in place so downstream passes see rehearsed state.
|
|
318
|
-
for (let i = 0; i < survivors.length; i++) {
|
|
319
|
-
const replacement = rehearsedById.get(survivors[i].id);
|
|
320
|
-
if (replacement)
|
|
321
|
-
survivors[i] = replacement;
|
|
428
|
+
// -------------------------------------------------------------------------
|
|
429
|
+
// 1.6. Batch extraction — extract facts from episodic memories missing them
|
|
430
|
+
// -------------------------------------------------------------------------
|
|
431
|
+
const extractedFromIds = new Set(survivors.filter((e) => e.extracted_from).map((e) => e.extracted_from));
|
|
432
|
+
const extractionCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !extractedFromIds.has(e.id));
|
|
433
|
+
result.extractionCandidates = extractionCandidates.length;
|
|
434
|
+
const apiKey = process.env.ANTHROPIC_API_KEY ?? '';
|
|
435
|
+
if (apiKey && extractionCandidates.length > 0 && !dryRun) {
|
|
436
|
+
const { extractFacts, storeExtractedFacts } = await import('./extract.js');
|
|
437
|
+
const batchLimit = 20;
|
|
438
|
+
let extractedCount = 0;
|
|
439
|
+
for (const candidate of extractionCandidates.slice(0, batchLimit)) {
|
|
440
|
+
try {
|
|
441
|
+
const facts = await extractFacts(candidate.content, {
|
|
442
|
+
apiKey,
|
|
443
|
+
model: config.extraction.model,
|
|
444
|
+
});
|
|
445
|
+
if (facts.length > 0) {
|
|
446
|
+
storeExtractedFacts(hippoRoot, candidate, facts);
|
|
447
|
+
extractedCount += facts.length;
|
|
448
|
+
}
|
|
322
449
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
rehearsed.map((e) => e.id).join(', '));
|
|
326
|
-
if (!dryRun) {
|
|
327
|
-
for (const r of rehearsed)
|
|
328
|
-
pendingWrites.push(r);
|
|
450
|
+
catch {
|
|
451
|
+
// Best-effort — continue with next candidate
|
|
329
452
|
}
|
|
330
453
|
}
|
|
454
|
+
result.extracted = extractedCount;
|
|
331
455
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
const extractionCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !extractedFromIds.has(e.id));
|
|
338
|
-
result.extractionCandidates = extractionCandidates.length;
|
|
339
|
-
const apiKey = process.env.ANTHROPIC_API_KEY ?? '';
|
|
340
|
-
if (apiKey && extractionCandidates.length > 0 && !dryRun) {
|
|
341
|
-
const { extractFacts, storeExtractedFacts } = await import('./extract.js');
|
|
342
|
-
const batchLimit = 20;
|
|
343
|
-
let extractedCount = 0;
|
|
344
|
-
for (const candidate of extractionCandidates.slice(0, batchLimit)) {
|
|
456
|
+
// -------------------------------------------------------------------------
|
|
457
|
+
// 1.7. DAG summarization — cluster extracted facts and generate summaries
|
|
458
|
+
// -------------------------------------------------------------------------
|
|
459
|
+
const extractedFacts = survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1);
|
|
460
|
+
if (apiKey && extractedFacts.length >= 3 && !dryRun) {
|
|
345
461
|
try {
|
|
346
|
-
const
|
|
462
|
+
const { buildDag } = await import('./dag.js');
|
|
463
|
+
const dagResult = await buildDag(hippoRoot, extractedFacts, {
|
|
347
464
|
apiKey,
|
|
348
465
|
model: config.extraction.model,
|
|
349
466
|
});
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
467
|
+
result.dagCandidateClusters = dagResult.candidateClusters;
|
|
468
|
+
result.dagSummariesCreated = dagResult.summariesCreated;
|
|
469
|
+
if (dagResult.summariesCreated > 0) {
|
|
470
|
+
result.details.push(` 🌳 DAG: ${dagResult.summariesCreated} summaries created, ${dagResult.factsLinked} facts linked`);
|
|
353
471
|
}
|
|
354
472
|
}
|
|
355
473
|
catch {
|
|
356
|
-
// Best-effort
|
|
474
|
+
// Best-effort
|
|
357
475
|
}
|
|
358
476
|
}
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
}
|
|
378
|
-
catch {
|
|
379
|
-
// Best-effort
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
// -------------------------------------------------------------------------
|
|
383
|
-
// 1.8. DAG summary rebuild — drain dirty queue from E2's child-write hooks
|
|
384
|
-
// -------------------------------------------------------------------------
|
|
385
|
-
// Consumer of E2's summary_dirty flag. Walks dirty L2 summaries, regenerates
|
|
386
|
-
// each via generateDagSummary, atomically refreshes content + 6 metadata
|
|
387
|
-
// columns + clears summary_dirty (with FTS sync). Same apiKey/dryRun gate
|
|
388
|
-
// as buildDag above. Cap HIPPO_DAG_REBUILD_CAP (default 20, hard ceiling
|
|
389
|
-
// 1000) prevents runaway LLM cost.
|
|
390
|
-
if (apiKey && !dryRun) {
|
|
391
|
-
try {
|
|
392
|
-
const { rebuildDirtySummaries } = await import('./dag.js');
|
|
393
|
-
const rawCap = parseInt(process.env.HIPPO_DAG_REBUILD_CAP ?? '20', 10);
|
|
394
|
-
// R1 MED must-fix: hard ceiling so misconfigured env can't burn
|
|
395
|
-
// unbounded LLM cost.
|
|
396
|
-
const cap = Number.isFinite(rawCap) && rawCap > 0
|
|
397
|
-
? Math.min(rawCap, 1000)
|
|
398
|
-
: 20;
|
|
399
|
-
const rebuildResult = await rebuildDirtySummaries(hippoRoot, {
|
|
400
|
-
apiKey,
|
|
401
|
-
model: config.extraction.model,
|
|
402
|
-
cap,
|
|
403
|
-
});
|
|
404
|
-
result.summariesRebuilt = rebuildResult.rebuilt;
|
|
405
|
-
result.summariesRebuildFailed = rebuildResult.failed;
|
|
406
|
-
result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
|
|
407
|
-
result.summariesRebuildCapped = rebuildResult.capped;
|
|
408
|
-
if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0) {
|
|
409
|
-
const parts = [];
|
|
410
|
-
if (rebuildResult.rebuilt > 0)
|
|
411
|
-
parts.push(`${rebuildResult.rebuilt} rebuilt`);
|
|
412
|
-
if (rebuildResult.zeroChildSkipped > 0)
|
|
413
|
-
parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
|
|
414
|
-
if (rebuildResult.failed > 0)
|
|
415
|
-
parts.push(`${rebuildResult.failed} failed`);
|
|
416
|
-
if (rebuildResult.capped)
|
|
417
|
-
parts.push(`CAPPED@${cap}`);
|
|
418
|
-
result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
|
|
419
|
-
}
|
|
420
|
-
}
|
|
421
|
-
catch {
|
|
422
|
-
// Best-effort — same posture as buildDag block above.
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
// -------------------------------------------------------------------------
|
|
426
|
-
// 1.9. DAG entity profiles — cluster L2 topic summaries into L3 profiles
|
|
427
|
-
// -------------------------------------------------------------------------
|
|
428
|
-
// E5 phase: aggregate per-entity L2 summaries (e.g. all the speaker:Alice
|
|
429
|
-
// topic summaries) into a single L3 entity profile. Runs even when phase
|
|
430
|
-
// 1.7 buildDag was skipped (re-clusters existing L2s every sleep).
|
|
431
|
-
//
|
|
432
|
-
// Uses loadAllL2Summaries (not `survivors`) because phase 1.7 wrote new
|
|
433
|
-
// L2s directly via writeEntry without pushing back into survivors.
|
|
434
|
-
if (apiKey && !dryRun) {
|
|
435
|
-
try {
|
|
436
|
-
const { buildEntityProfiles } = await import('./dag.js');
|
|
437
|
-
const { loadAllL2Summaries } = await import('./store.js');
|
|
438
|
-
const l2Summaries = loadAllL2Summaries(hippoRoot);
|
|
439
|
-
if (l2Summaries.length >= 2) {
|
|
440
|
-
const profileResult = await buildEntityProfiles(hippoRoot, l2Summaries, {
|
|
477
|
+
// -------------------------------------------------------------------------
|
|
478
|
+
// 1.8. DAG summary rebuild — drain dirty queue from E2's child-write hooks
|
|
479
|
+
// -------------------------------------------------------------------------
|
|
480
|
+
// Consumer of E2's summary_dirty flag. Walks dirty L2 summaries, regenerates
|
|
481
|
+
// each via generateDagSummary, atomically refreshes content + 6 metadata
|
|
482
|
+
// columns + clears summary_dirty (with FTS sync). Same apiKey/dryRun gate
|
|
483
|
+
// as buildDag above. Cap HIPPO_DAG_REBUILD_CAP (default 20, hard ceiling
|
|
484
|
+
// 1000) prevents runaway LLM cost.
|
|
485
|
+
if (apiKey && !dryRun) {
|
|
486
|
+
try {
|
|
487
|
+
const { rebuildDirtySummaries } = await import('./dag.js');
|
|
488
|
+
const rawCap = parseInt(process.env.HIPPO_DAG_REBUILD_CAP ?? '20', 10);
|
|
489
|
+
// R1 MED must-fix: hard ceiling so misconfigured env can't burn
|
|
490
|
+
// unbounded LLM cost.
|
|
491
|
+
const cap = Number.isFinite(rawCap) && rawCap > 0
|
|
492
|
+
? Math.min(rawCap, 1000)
|
|
493
|
+
: 20;
|
|
494
|
+
const rebuildResult = await rebuildDirtySummaries(hippoRoot, {
|
|
441
495
|
apiKey,
|
|
442
496
|
model: config.extraction.model,
|
|
497
|
+
cap,
|
|
443
498
|
});
|
|
444
|
-
result.
|
|
445
|
-
|
|
446
|
-
|
|
499
|
+
result.summariesRebuilt = rebuildResult.rebuilt;
|
|
500
|
+
result.summariesRebuildFailed = rebuildResult.failed;
|
|
501
|
+
result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
|
|
502
|
+
result.summariesRebuildRefused = rebuildResult.refused;
|
|
503
|
+
result.summariesRebuildCapped = rebuildResult.capped;
|
|
504
|
+
if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0 || rebuildResult.refused > 0) {
|
|
505
|
+
const parts = [];
|
|
506
|
+
if (rebuildResult.rebuilt > 0)
|
|
507
|
+
parts.push(`${rebuildResult.rebuilt} rebuilt`);
|
|
508
|
+
if (rebuildResult.refused > 0)
|
|
509
|
+
parts.push(`${rebuildResult.refused} refused`);
|
|
510
|
+
if (rebuildResult.zeroChildSkipped > 0)
|
|
511
|
+
parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
|
|
512
|
+
if (rebuildResult.failed > 0)
|
|
513
|
+
parts.push(`${rebuildResult.failed} failed`);
|
|
514
|
+
if (rebuildResult.capped)
|
|
515
|
+
parts.push(`CAPPED@${cap}`);
|
|
516
|
+
result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
|
|
447
517
|
}
|
|
448
518
|
}
|
|
519
|
+
catch {
|
|
520
|
+
// Best-effort — same posture as buildDag block above.
|
|
521
|
+
}
|
|
449
522
|
}
|
|
450
|
-
|
|
451
|
-
|
|
523
|
+
// -------------------------------------------------------------------------
|
|
524
|
+
// 1.9. DAG entity profiles — cluster L2 topic summaries into L3 profiles
|
|
525
|
+
// -------------------------------------------------------------------------
|
|
526
|
+
// E5 phase: aggregate per-entity L2 summaries (e.g. all the speaker:Alice
|
|
527
|
+
// topic summaries) into a single L3 entity profile. Runs even when phase
|
|
528
|
+
// 1.7 buildDag was skipped (re-clusters existing L2s every sleep).
|
|
529
|
+
//
|
|
530
|
+
// Uses loadAllL2Summaries (not `survivors`) because phase 1.7 wrote new
|
|
531
|
+
// L2s directly via writeEntry without pushing back into survivors.
|
|
532
|
+
if (apiKey && !dryRun) {
|
|
533
|
+
try {
|
|
534
|
+
const { buildEntityProfiles } = await import('./dag.js');
|
|
535
|
+
const { loadAllL2Summaries } = await import('./store.js');
|
|
536
|
+
const l2Summaries = loadAllL2Summaries(hippoRoot);
|
|
537
|
+
if (l2Summaries.length >= 2) {
|
|
538
|
+
const profileResult = await buildEntityProfiles(hippoRoot, l2Summaries, {
|
|
539
|
+
apiKey,
|
|
540
|
+
model: config.extraction.model,
|
|
541
|
+
});
|
|
542
|
+
result.entityProfilesCreated = profileResult.profilesCreated;
|
|
543
|
+
if (profileResult.profilesCreated > 0) {
|
|
544
|
+
result.details.push(` 🌲 DAG L3: ${profileResult.profilesCreated} entity profiles, ${profileResult.l2sLinked} L2s linked`);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
catch {
|
|
549
|
+
// Best-effort.
|
|
550
|
+
}
|
|
452
551
|
}
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
set.
|
|
477
|
-
|
|
552
|
+
// -------------------------------------------------------------------------
|
|
553
|
+
// 2. Physics simulation pass
|
|
554
|
+
// -------------------------------------------------------------------------
|
|
555
|
+
if (!dryRun) {
|
|
556
|
+
try {
|
|
557
|
+
const physicsEnabled = config.physics.enabled === true
|
|
558
|
+
|| (config.physics.enabled === 'auto');
|
|
559
|
+
if (physicsEnabled) {
|
|
560
|
+
const db = openHippoDb(hippoRoot);
|
|
561
|
+
try {
|
|
562
|
+
const physicsMap = loadPhysicsState(db);
|
|
563
|
+
const particles = Array.from(physicsMap.values());
|
|
564
|
+
if (particles.length > 0) {
|
|
565
|
+
// Build entry lookup for property refresh
|
|
566
|
+
const entryMap = new Map(survivors.map(e => [e.id, e]));
|
|
567
|
+
refreshParticleProperties(particles, entryMap, now);
|
|
568
|
+
// Build conflict pairs from survivors
|
|
569
|
+
const conflictPairs = new Map();
|
|
570
|
+
for (const entry of survivors) {
|
|
571
|
+
if (entry.conflicts_with.length > 0) {
|
|
572
|
+
const set = conflictPairs.get(entry.id) ?? new Set();
|
|
573
|
+
for (const cid of entry.conflicts_with)
|
|
574
|
+
set.add(cid);
|
|
575
|
+
conflictPairs.set(entry.id, set);
|
|
576
|
+
}
|
|
478
577
|
}
|
|
578
|
+
// Build half-life lookup
|
|
579
|
+
const halfLives = new Map();
|
|
580
|
+
for (const entry of survivors) {
|
|
581
|
+
halfLives.set(entry.id, entry.half_life_days);
|
|
582
|
+
}
|
|
583
|
+
const ctx = {
|
|
584
|
+
conflictPairs,
|
|
585
|
+
halfLives,
|
|
586
|
+
config: config.physics,
|
|
587
|
+
};
|
|
588
|
+
const stats = simulate(particles, ctx);
|
|
589
|
+
savePhysicsState(db, particles);
|
|
590
|
+
result.physicsSimulated = stats.particleCount;
|
|
591
|
+
result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
|
|
592
|
+
`avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
|
|
593
|
+
`energy ${stats.energy.total.toFixed(4)}`);
|
|
479
594
|
}
|
|
480
|
-
// Build half-life lookup
|
|
481
|
-
const halfLives = new Map();
|
|
482
|
-
for (const entry of survivors) {
|
|
483
|
-
halfLives.set(entry.id, entry.half_life_days);
|
|
484
|
-
}
|
|
485
|
-
const ctx = {
|
|
486
|
-
conflictPairs,
|
|
487
|
-
halfLives,
|
|
488
|
-
config: config.physics,
|
|
489
|
-
};
|
|
490
|
-
const stats = simulate(particles, ctx);
|
|
491
|
-
savePhysicsState(db, particles);
|
|
492
|
-
result.physicsSimulated = stats.particleCount;
|
|
493
|
-
result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
|
|
494
|
-
`avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
|
|
495
|
-
`energy ${stats.energy.total.toFixed(4)}`);
|
|
496
595
|
}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
596
|
+
finally {
|
|
597
|
+
closeHippoDb(db);
|
|
598
|
+
}
|
|
500
599
|
}
|
|
501
600
|
}
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
result.details.push(` ⚠️ physics simulation skipped: ${error instanceof Error ? error.message : 'unknown error'}`);
|
|
505
|
-
}
|
|
506
|
-
}
|
|
507
|
-
// -------------------------------------------------------------------------
|
|
508
|
-
// 3. Merge pass - episodic entries only
|
|
509
|
-
// -------------------------------------------------------------------------
|
|
510
|
-
const mergeCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !e.tags.includes('extracted'));
|
|
511
|
-
const used = new Set();
|
|
512
|
-
for (let i = 0; i < mergeCandidates.length; i++) {
|
|
513
|
-
if (used.has(mergeCandidates[i].id))
|
|
514
|
-
continue;
|
|
515
|
-
const cluster = [mergeCandidates[i]];
|
|
516
|
-
for (let j = i + 1; j < mergeCandidates.length; j++) {
|
|
517
|
-
if (used.has(mergeCandidates[j].id))
|
|
518
|
-
continue;
|
|
519
|
-
const overlap = textOverlap(mergeCandidates[i].content, mergeCandidates[j].content);
|
|
520
|
-
if (overlap >= MERGE_OVERLAP_THRESHOLD) {
|
|
521
|
-
cluster.push(mergeCandidates[j]);
|
|
601
|
+
catch (error) {
|
|
602
|
+
result.details.push(` ⚠️ physics simulation skipped: ${error instanceof Error ? error.message : 'unknown error'}`);
|
|
522
603
|
}
|
|
523
604
|
}
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
//
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
//
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
605
|
+
// -------------------------------------------------------------------------
|
|
606
|
+
// 3. Merge pass - episodic entries only
|
|
607
|
+
// -------------------------------------------------------------------------
|
|
608
|
+
const mergeCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !e.tags.includes('extracted'));
|
|
609
|
+
const used = new Set();
|
|
610
|
+
// T1 fix (2026-08-15 hardening pass): partition by tenantId BEFORE the
|
|
611
|
+
// overlap loop so a cluster can never span tenants. Previously textOverlap
|
|
612
|
+
// clustered across the whole host-wide `survivors` list with no tenant
|
|
613
|
+
// boundary, and mergeContents concatenated cross-tenant content into one
|
|
614
|
+
// row. Map preserves insertion order, so single-tenant stores (every row
|
|
615
|
+
// 'default') get exactly one partition and iterate in the same order as
|
|
616
|
+
// before this fix — byte-identical behavior there.
|
|
617
|
+
const mergeCandidatesByTenant = new Map();
|
|
618
|
+
for (const entry of mergeCandidates) {
|
|
619
|
+
const bucket = mergeCandidatesByTenant.get(entry.tenantId);
|
|
620
|
+
if (bucket)
|
|
621
|
+
bucket.push(entry);
|
|
622
|
+
else
|
|
623
|
+
mergeCandidatesByTenant.set(entry.tenantId, [entry]);
|
|
624
|
+
}
|
|
625
|
+
// AT1 consolidation-loop fix (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
|
|
626
|
+
// reuses the single consolidateDb handle opened once above (before the
|
|
627
|
+
// auto-promote pass, 1.4) for the whole non-dry-run consolidate — see that
|
|
628
|
+
// declaration's comment. Only needed for real writes — a dry-run preview
|
|
629
|
+
// never reaches batchWriteAndDelete's guard bypass, so there is nothing
|
|
630
|
+
// here for it to protect against. (mergesSkippedRejected itself is
|
|
631
|
+
// declared up at the try's opening above 1.4, not here — it has to
|
|
632
|
+
// survive the try/finally that now wraps this whole section; see the
|
|
633
|
+
// handle-leak restructure comment there.)
|
|
634
|
+
for (const [mergeTenant, tenantCandidates] of mergeCandidatesByTenant) {
|
|
635
|
+
for (let i = 0; i < tenantCandidates.length; i++) {
|
|
636
|
+
if (used.has(tenantCandidates[i].id))
|
|
637
|
+
continue;
|
|
638
|
+
const cluster = [tenantCandidates[i]];
|
|
639
|
+
for (let j = i + 1; j < tenantCandidates.length; j++) {
|
|
640
|
+
if (used.has(tenantCandidates[j].id))
|
|
641
|
+
continue;
|
|
642
|
+
const overlap = textOverlap(tenantCandidates[i].content, tenantCandidates[j].content);
|
|
643
|
+
if (overlap >= MERGE_OVERLAP_THRESHOLD) {
|
|
644
|
+
cluster.push(tenantCandidates[j]);
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
if (cluster.length < MERGE_MIN_CLUSTER)
|
|
648
|
+
continue;
|
|
649
|
+
// Create a semantic summary
|
|
650
|
+
const mergedContent = mergeContents(cluster);
|
|
651
|
+
const allTags = Array.from(new Set(cluster.flatMap((e) => e.tags)));
|
|
652
|
+
const maxValence = pickStrongestValence(cluster);
|
|
653
|
+
// AT1 P2 fix: build the semantic entry FIRST — createMemory is cheap
|
|
654
|
+
// and pure — so the tombstone check below runs under the tenant the
|
|
655
|
+
// row will ACTUALLY land in.
|
|
656
|
+
// T1 fix: createMemory now receives tenantId: mergeTenant (the
|
|
657
|
+
// partition's tenant — every member of `cluster` shares it by
|
|
658
|
+
// construction), so the row lands in its source tenant instead of
|
|
659
|
+
// always 'default'.
|
|
660
|
+
let semantic = null;
|
|
661
|
+
if (!dryRun) {
|
|
662
|
+
semantic = createMemory(mergedContent, {
|
|
663
|
+
layer: Layer.Semantic,
|
|
664
|
+
tags: allTags,
|
|
665
|
+
emotional_valence: maxValence,
|
|
666
|
+
schema_fit: 0.7,
|
|
667
|
+
source: 'consolidation',
|
|
668
|
+
confidence: 'inferred',
|
|
669
|
+
tenantId: mergeTenant,
|
|
670
|
+
});
|
|
671
|
+
}
|
|
672
|
+
// mergeContents is DETERMINISTIC CONCATENATION (not an LLM paraphrase)
|
|
673
|
+
// — if a human rejected exactly this byte-identical rollup before, an
|
|
674
|
+
// unguarded sleep would regenerate it every cycle and
|
|
675
|
+
// batchWriteAndDelete's guard bypass (store.ts) would silently
|
|
676
|
+
// re-assert it forever. This producer-side check is what makes that
|
|
677
|
+
// bypass safe. A hit skips the WHOLE cluster: sources stay unmerged —
|
|
678
|
+
// not demoted, not deleted — so a later sleep gets another chance if
|
|
679
|
+
// the tombstone is lifted.
|
|
680
|
+
const consolidateDb = getConsolidateDb();
|
|
681
|
+
if (consolidateDb && semantic) {
|
|
682
|
+
const mergeDigest = rejectionDigest(semantic.content);
|
|
683
|
+
const tombstone = findRejectedValue(consolidateDb, semantic.tenantId, mergeDigest);
|
|
684
|
+
if (tombstone) {
|
|
685
|
+
// Still mark used — these members are not re-tried against a
|
|
686
|
+
// DIFFERENT cluster within this same pass; next sleep re-clusters
|
|
687
|
+
// them fresh.
|
|
688
|
+
for (const e of cluster)
|
|
689
|
+
used.add(e.id);
|
|
690
|
+
mergesSkippedRejected++;
|
|
691
|
+
try {
|
|
692
|
+
appendAuditEvent(consolidateDb, {
|
|
693
|
+
tenantId: semantic.tenantId,
|
|
694
|
+
actor: 'sleep',
|
|
695
|
+
op: 'reject_refusal',
|
|
696
|
+
metadata: {
|
|
697
|
+
digest: mergeDigest,
|
|
698
|
+
reason: tombstone.reason,
|
|
699
|
+
sourceIds: cluster.map((e) => e.id),
|
|
700
|
+
},
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
catch {
|
|
704
|
+
// Best-effort — mirrors store.ts's audit() semantics.
|
|
705
|
+
}
|
|
706
|
+
continue;
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
// Mark cluster members as used
|
|
710
|
+
for (const e of cluster)
|
|
711
|
+
used.add(e.id);
|
|
712
|
+
result.merged += cluster.length;
|
|
713
|
+
result.details.push(` 🔀 merged ${cluster.length} episodic entries into semantic: "${mergedContent.slice(0, 60)}..."`);
|
|
714
|
+
if (!dryRun && semantic) {
|
|
715
|
+
pendingWrites.push(semantic);
|
|
716
|
+
result.semanticCreated++;
|
|
717
|
+
// Demote source episodics (they've been compressed into neocortex):
|
|
718
|
+
// scale half_life_days so they decay sooner while staying recoverable.
|
|
719
|
+
// Immediate ranking is deliberately unchanged: the 2026-06-10 DAG
|
|
720
|
+
// slice-1 eval measured that dropping children below a worse-retrieving
|
|
721
|
+
// summary regresses budget-bounded QA (docs/evals/). The stored
|
|
722
|
+
// strength is refreshed to the live value so inspect, replay sampling,
|
|
723
|
+
// and strength-sorted assembly see the truth instead of a fake 0.3.
|
|
724
|
+
// Mutate in place (not a copy): `cluster` holds the same object
|
|
725
|
+
// references as `survivors`, and the later detectConflicts(survivors)
|
|
726
|
+
// pass in this same run must see the post-demotion half-life, or it
|
|
727
|
+
// can persist conflicts for entries the just-written state excludes.
|
|
728
|
+
for (const e of cluster) {
|
|
729
|
+
e.half_life_days = Math.max(1, Math.floor(e.half_life_days * MERGE_SOURCE_HALF_LIFE_FACTOR));
|
|
730
|
+
e.strength = calculateStrength(e, now, decayOpts);
|
|
731
|
+
pendingWrites.push(e);
|
|
732
|
+
}
|
|
733
|
+
}
|
|
561
734
|
}
|
|
562
735
|
}
|
|
563
736
|
}
|
|
737
|
+
finally {
|
|
738
|
+
if (consolidateDbHandle)
|
|
739
|
+
closeHippoDb(consolidateDbHandle);
|
|
740
|
+
}
|
|
741
|
+
if (mergesSkippedRejected > 0) {
|
|
742
|
+
console.error(`consolidate: skipped ${mergesSkippedRejected} merge(s) whose content matches a rejected value`);
|
|
743
|
+
}
|
|
564
744
|
// Flush all writes/deletes in a single transaction
|
|
565
745
|
if (!dryRun) {
|
|
566
746
|
batchWriteAndDelete(hippoRoot, pendingWrites, pendingDeletes);
|