hippo-memory 1.30.0 → 1.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api.d.ts +49 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +78 -2
- package/dist/api.js.map +1 -1
- package/dist/audit.d.ts +1 -1
- package/dist/audit.d.ts.map +1 -1
- package/dist/audit.js.map +1 -1
- package/dist/capture.d.ts.map +1 -1
- package/dist/capture.js +79 -23
- package/dist/capture.js.map +1 -1
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +233 -13
- package/dist/cli.js.map +1 -1
- package/dist/connectors/github/ingest.d.ts.map +1 -1
- package/dist/connectors/github/ingest.js +17 -0
- package/dist/connectors/github/ingest.js.map +1 -1
- package/dist/connectors/slack/ingest.d.ts.map +1 -1
- package/dist/connectors/slack/ingest.js +15 -0
- package/dist/connectors/slack/ingest.js.map +1 -1
- package/dist/consolidate.d.ts.map +1 -1
- package/dist/consolidate.js +428 -290
- package/dist/consolidate.js.map +1 -1
- package/dist/dag.d.ts +10 -0
- package/dist/dag.d.ts.map +1 -1
- package/dist/dag.js +32 -3
- package/dist/dag.js.map +1 -1
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +43 -1
- package/dist/db.js.map +1 -1
- package/dist/extract.d.ts.map +1 -1
- package/dist/extract.js +18 -1
- package/dist/extract.js.map +1 -1
- package/dist/importers.d.ts +10 -0
- package/dist/importers.d.ts.map +1 -1
- package/dist/importers.js +242 -159
- package/dist/importers.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +34 -4
- package/dist/mcp/server.js.map +1 -1
- package/dist/reject-flow.d.ts +66 -0
- package/dist/reject-flow.d.ts.map +1 -0
- package/dist/reject-flow.js +207 -0
- package/dist/reject-flow.js.map +1 -0
- package/dist/rejection.d.ts +100 -0
- package/dist/rejection.d.ts.map +1 -0
- package/dist/rejection.js +157 -0
- package/dist/rejection.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +4 -0
- package/dist/server.js.map +1 -1
- package/dist/shared.d.ts +9 -0
- package/dist/shared.d.ts.map +1 -1
- package/dist/shared.js +57 -6
- package/dist/shared.js.map +1 -1
- package/dist/sleep-redact.d.ts +2 -0
- package/dist/sleep-redact.d.ts.map +1 -1
- package/dist/sleep-redact.js +2 -0
- package/dist/sleep-redact.js.map +1 -1
- package/dist/src/api.js +78 -2
- package/dist/src/api.js.map +1 -1
- package/dist/src/audit.js.map +1 -1
- package/dist/src/capture.js +79 -23
- package/dist/src/capture.js.map +1 -1
- package/dist/src/cli.js +233 -13
- package/dist/src/cli.js.map +1 -1
- package/dist/src/connectors/github/ingest.js +17 -0
- package/dist/src/connectors/github/ingest.js.map +1 -1
- package/dist/src/connectors/slack/ingest.js +15 -0
- package/dist/src/connectors/slack/ingest.js.map +1 -1
- package/dist/src/consolidate.js +428 -290
- package/dist/src/consolidate.js.map +1 -1
- package/dist/src/dag.js +32 -3
- package/dist/src/dag.js.map +1 -1
- package/dist/src/db.js +43 -1
- package/dist/src/db.js.map +1 -1
- package/dist/src/extract.js +18 -1
- package/dist/src/extract.js.map +1 -1
- package/dist/src/importers.js +242 -159
- package/dist/src/importers.js.map +1 -1
- package/dist/src/mcp/server.js +34 -4
- package/dist/src/mcp/server.js.map +1 -1
- package/dist/src/reject-flow.js +207 -0
- package/dist/src/reject-flow.js.map +1 -0
- package/dist/src/rejection.js +157 -0
- package/dist/src/rejection.js.map +1 -0
- package/dist/src/server.js +4 -0
- package/dist/src/server.js.map +1 -1
- package/dist/src/shared.js +57 -6
- package/dist/src/shared.js.map +1 -1
- package/dist/src/sleep-redact.js +2 -0
- package/dist/src/sleep-redact.js.map +1 -1
- package/dist/src/store.js +498 -35
- package/dist/src/store.js.map +1 -1
- package/dist/src/version.js +1 -1
- package/dist/store.d.ts +107 -3
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +498 -35
- package/dist/store.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/src/consolidate.js
CHANGED
|
@@ -12,6 +12,7 @@ import { loadAllEntries, batchWriteAndDelete, appendConsolidationRun, replaceDet
|
|
|
12
12
|
import { textOverlap, markRetrieved } from './search.js';
|
|
13
13
|
import { compareEntryIdentity } from './compare.js';
|
|
14
14
|
import { openHippoDb, closeHippoDb } from './db.js';
|
|
15
|
+
import { rejectionDigest, findRejectedValue } from './rejection.js';
|
|
15
16
|
import { loadPhysicsState, savePhysicsState, refreshParticleProperties } from './physics-state.js';
|
|
16
17
|
import { simulate } from './physics.js';
|
|
17
18
|
import { loadConfig } from './config.js';
|
|
@@ -236,331 +237,468 @@ export async function consolidate(hippoRoot, options = {}) {
|
|
|
236
237
|
}
|
|
237
238
|
}
|
|
238
239
|
}
|
|
239
|
-
//
|
|
240
|
-
//
|
|
241
|
-
//
|
|
240
|
+
// AT1 rejection-guard db handle (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
|
|
241
|
+
// opened ONCE for the whole non-dry-run consolidate, covering BOTH the
|
|
242
|
+
// auto-promote pass (1.4, immediately below) and the merge pass (3,
|
|
243
|
+
// further down) — both build deterministic content that batchWriteAndDelete
|
|
244
|
+
// writes through the guard's bypass, so both need a producer-side
|
|
245
|
+
// tombstone check before pushing to pendingWrites. A single handle here
|
|
246
|
+
// replaces what used to be a per-pass open. Dry-run never reaches the
|
|
247
|
+
// bypass (no batchWriteAndDelete call), so there is nothing for a handle
|
|
248
|
+
// to protect against — stays null and every `if (consolidateDb)` below is
|
|
249
|
+
// a no-op.
|
|
242
250
|
//
|
|
243
|
-
//
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
251
|
+
// AT1 P2 fix (codex, handle-leak restructure): the open must be followed
|
|
252
|
+
// IMMEDIATELY by the try whose finally closes it, covering every phase
|
|
253
|
+
// that can touch consolidateDb — not just the merge pass. Previously the
|
|
254
|
+
// try/finally wrapped only section 3 (merge pass); an exception thrown by
|
|
255
|
+
// auto-promote (1.4), replay (1.5), batch extraction (1.6), the DAG
|
|
256
|
+
// passes (1.7-1.9), or physics (2) would propagate past the open handle
|
|
257
|
+
// with nothing to close it. No behavior change on the happy path — each
|
|
258
|
+
// of those phases already best-effort catches its own exceptions today
|
|
259
|
+
// (physics has its own try/catch below; each DAG pass wraps its own
|
|
260
|
+
// dynamic import + call in try/catch) — this only closes the handle on
|
|
261
|
+
// the rare path where one of them throws past its own catch. Deliberately
|
|
262
|
+
// NOT re-indented (mechanical wrap only, kept surgical): every statement
|
|
263
|
+
// between this try and its finally (below, at the end of the merge pass)
|
|
264
|
+
// stays at its original indentation.
|
|
265
|
+
const consolidateDb = dryRun ? null : openHippoDb(hippoRoot);
|
|
266
|
+
// Declared here (not at the merge pass, its point of use) so it survives
|
|
267
|
+
// this try/finally — a `let` declared INSIDE the try would go out of
|
|
268
|
+
// scope before the `if (mergesSkippedRejected > 0)` check that reads it
|
|
269
|
+
// after the finally closes the handle.
|
|
270
|
+
let mergesSkippedRejected = 0;
|
|
271
|
+
try {
|
|
272
|
+
// -------------------------------------------------------------------------
|
|
273
|
+
// 1.4. Auto-promote complete sessions to traces
|
|
274
|
+
// -------------------------------------------------------------------------
|
|
275
|
+
//
|
|
276
|
+
// For each session within the configured window that has a `session_complete`
|
|
277
|
+
// event and no existing trace (idempotency via the source_session_id column),
|
|
278
|
+
// render the action sequence as markdown and persist a Layer.Trace memory.
|
|
279
|
+
// Traces inherit decay, search, replay, and physics from the base MemoryEntry.
|
|
280
|
+
if (!dryRun && config.autoTraceCapture !== false) {
|
|
281
|
+
let tracesSkippedRejected = 0;
|
|
282
|
+
const windowDays = config.autoTraceWindowDays ?? 7;
|
|
283
|
+
const sinceMs = now.getTime() - windowDays * 24 * 60 * 60 * 1000;
|
|
284
|
+
// Auto-trace currently runs in a single-tenant context (the env-resolved
|
|
285
|
+
// tenant for this process). Multi-tenant deployments that want
|
|
286
|
+
// consolidation across all tenants need a per-tenant loop layered on top
|
|
287
|
+
// of this — tracked in docs/plans/2026-05-02-continuity-tables-tenant-scope.md.
|
|
288
|
+
const consolidationTenant = resolveTenantId({});
|
|
289
|
+
const promotable = findPromotableSessions(hippoRoot, consolidationTenant, sinceMs);
|
|
290
|
+
for (const session of promotable) {
|
|
291
|
+
// Idempotency: skip if a trace for this session already exists.
|
|
292
|
+
if (traceExistsForSession(hippoRoot, consolidationTenant, session.session_id))
|
|
293
|
+
continue;
|
|
294
|
+
const events = listSessionEvents(hippoRoot, consolidationTenant, {
|
|
295
|
+
session_id: session.session_id,
|
|
296
|
+
limit: 1000,
|
|
297
|
+
});
|
|
298
|
+
const completeEvent = events.find((e) => e.event_type === 'session_complete');
|
|
299
|
+
if (!completeEvent)
|
|
300
|
+
continue; // defence-in-depth; findPromotableSessions filters already.
|
|
301
|
+
const outcomeRaw = completeEvent.content;
|
|
302
|
+
if (outcomeRaw !== 'success' && outcomeRaw !== 'failure' && outcomeRaw !== 'partial') {
|
|
303
|
+
// Malformed terminal event — skip rather than crash the whole sleep.
|
|
304
|
+
continue;
|
|
305
|
+
}
|
|
306
|
+
const outcome = outcomeRaw;
|
|
307
|
+
const steps = events
|
|
308
|
+
.filter((e) => e.event_type !== 'session_complete')
|
|
309
|
+
.map((e) => ({ action: e.content, observation: '' }));
|
|
310
|
+
const summary = typeof completeEvent.metadata.summary === 'string'
|
|
311
|
+
? completeEvent.metadata.summary
|
|
312
|
+
: '(untitled)';
|
|
313
|
+
const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
|
|
314
|
+
layer: Layer.Trace,
|
|
315
|
+
trace_outcome: outcome,
|
|
316
|
+
source_session_id: session.session_id,
|
|
317
|
+
tags: ['auto-promoted'],
|
|
318
|
+
source: 'auto-promote',
|
|
319
|
+
});
|
|
320
|
+
// AT1 (same producer-side pattern as the merge pass below): traceExistsForSession
|
|
321
|
+
// only sees rows CURRENTLY in the store — once a rejected trace is
|
|
322
|
+
// removed, that idempotency check no longer blocks regeneration, and
|
|
323
|
+
// this write would otherwise reach batchWriteAndDelete's guard bypass
|
|
324
|
+
// unchecked, resurrecting it every sleep. Check under THE ENTRY'S OWN
|
|
325
|
+
// stamped tenantId (read off `trace` after createMemory — never guess
|
|
326
|
+
// the tenant) + the built content's digest. A hit skips the push
|
|
327
|
+
// entirely: not counted as promoted, not added to survivors.
|
|
328
|
+
if (consolidateDb) {
|
|
329
|
+
const traceDigest = rejectionDigest(trace.content);
|
|
330
|
+
const tombstone = findRejectedValue(consolidateDb, trace.tenantId, traceDigest);
|
|
331
|
+
if (tombstone) {
|
|
332
|
+
tracesSkippedRejected++;
|
|
333
|
+
try {
|
|
334
|
+
appendAuditEvent(consolidateDb, {
|
|
335
|
+
tenantId: trace.tenantId,
|
|
336
|
+
actor: 'sleep',
|
|
337
|
+
op: 'reject_refusal',
|
|
338
|
+
metadata: {
|
|
339
|
+
digest: traceDigest,
|
|
340
|
+
reason: tombstone.reason,
|
|
341
|
+
sourceSessionId: session.session_id,
|
|
342
|
+
},
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
// Best-effort — mirrors store.ts's audit() semantics.
|
|
347
|
+
}
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
pendingWrites.push(trace);
|
|
352
|
+
survivors.push(trace);
|
|
353
|
+
result.promotedTraces++;
|
|
354
|
+
result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
|
|
355
|
+
}
|
|
356
|
+
if (result.promotedTraces > 0) {
|
|
357
|
+
result.details.push(` 🧬 promoted ${result.promotedTraces} trace${result.promotedTraces === 1 ? '' : 's'} from completed session${result.promotedTraces === 1 ? '' : 's'}`);
|
|
358
|
+
}
|
|
359
|
+
if (tracesSkippedRejected > 0) {
|
|
360
|
+
console.error(`consolidate: skipped ${tracesSkippedRejected} auto-promoted trace(s) whose content matches a rejected value`);
|
|
271
361
|
}
|
|
272
|
-
const outcome = outcomeRaw;
|
|
273
|
-
const steps = events
|
|
274
|
-
.filter((e) => e.event_type !== 'session_complete')
|
|
275
|
-
.map((e) => ({ action: e.content, observation: '' }));
|
|
276
|
-
const summary = typeof completeEvent.metadata.summary === 'string'
|
|
277
|
-
? completeEvent.metadata.summary
|
|
278
|
-
: '(untitled)';
|
|
279
|
-
const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
|
|
280
|
-
layer: Layer.Trace,
|
|
281
|
-
trace_outcome: outcome,
|
|
282
|
-
source_session_id: session.session_id,
|
|
283
|
-
tags: ['auto-promoted'],
|
|
284
|
-
source: 'auto-promote',
|
|
285
|
-
});
|
|
286
|
-
pendingWrites.push(trace);
|
|
287
|
-
survivors.push(trace);
|
|
288
|
-
result.promotedTraces++;
|
|
289
|
-
result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
|
|
290
362
|
}
|
|
291
|
-
|
|
292
|
-
|
|
363
|
+
// -------------------------------------------------------------------------
|
|
364
|
+
// 1.5. Replay pass — rehearse high-value survivors
|
|
365
|
+
// -------------------------------------------------------------------------
|
|
366
|
+
//
|
|
367
|
+
// Biologically-inspired counterpart to hippocampal replay during slow-wave
|
|
368
|
+
// sleep: sample N memories weighted by outcome + valence + under-rehearsal
|
|
369
|
+
// + idle time, then apply the same retrieval-strengthening `markRetrieved`
|
|
370
|
+
// applies to real queries. Distinct from decay (removal), physics (motion),
|
|
371
|
+
// and merge (compression) — this is the "rehearse the important stuff so
|
|
372
|
+
// it doesn't fade" pass.
|
|
373
|
+
{
|
|
374
|
+
const replayCount = config.replay?.count ?? REPLAY_COUNT_DEFAULT;
|
|
375
|
+
// EVAL-ONLY ablation (see ablation.ts): replay rehearsal IS recall
|
|
376
|
+
// strengthening (same markRetrieved dynamics), so the strengthen-off arm
|
|
377
|
+
// silences the whole pass - markRetrieved would return unmutated entries
|
|
378
|
+
// and persisting them anyway would still refresh updated_at / mirrors.
|
|
379
|
+
if (replayCount > 0 && survivors.length > 0 && !isRecallBoostAblated()) {
|
|
380
|
+
const seed = Math.floor(now.getTime() / 1000) & 0xffffffff;
|
|
381
|
+
const picked = sampleForReplay(survivors, replayCount, now, seed);
|
|
382
|
+
if (picked.length > 0) {
|
|
383
|
+
const rehearsed = markRetrieved(picked, now);
|
|
384
|
+
const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
|
|
385
|
+
// Update survivors in place so downstream passes see rehearsed state.
|
|
386
|
+
for (let i = 0; i < survivors.length; i++) {
|
|
387
|
+
const replacement = rehearsedById.get(survivors[i].id);
|
|
388
|
+
if (replacement)
|
|
389
|
+
survivors[i] = replacement;
|
|
390
|
+
}
|
|
391
|
+
result.replayed = rehearsed.length;
|
|
392
|
+
result.details.push(` 💭 replayed ${rehearsed.length} memor${rehearsed.length === 1 ? 'y' : 'ies'}: ` +
|
|
393
|
+
rehearsed.map((e) => e.id).join(', '));
|
|
394
|
+
if (!dryRun) {
|
|
395
|
+
for (const r of rehearsed)
|
|
396
|
+
pendingWrites.push(r);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
293
400
|
}
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
const rehearsed = markRetrieved(picked, now);
|
|
316
|
-
const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
|
|
317
|
-
// Update survivors in place so downstream passes see rehearsed state.
|
|
318
|
-
for (let i = 0; i < survivors.length; i++) {
|
|
319
|
-
const replacement = rehearsedById.get(survivors[i].id);
|
|
320
|
-
if (replacement)
|
|
321
|
-
survivors[i] = replacement;
|
|
401
|
+
// -------------------------------------------------------------------------
|
|
402
|
+
// 1.6. Batch extraction — extract facts from episodic memories missing them
|
|
403
|
+
// -------------------------------------------------------------------------
|
|
404
|
+
const extractedFromIds = new Set(survivors.filter((e) => e.extracted_from).map((e) => e.extracted_from));
|
|
405
|
+
const extractionCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !extractedFromIds.has(e.id));
|
|
406
|
+
result.extractionCandidates = extractionCandidates.length;
|
|
407
|
+
const apiKey = process.env.ANTHROPIC_API_KEY ?? '';
|
|
408
|
+
if (apiKey && extractionCandidates.length > 0 && !dryRun) {
|
|
409
|
+
const { extractFacts, storeExtractedFacts } = await import('./extract.js');
|
|
410
|
+
const batchLimit = 20;
|
|
411
|
+
let extractedCount = 0;
|
|
412
|
+
for (const candidate of extractionCandidates.slice(0, batchLimit)) {
|
|
413
|
+
try {
|
|
414
|
+
const facts = await extractFacts(candidate.content, {
|
|
415
|
+
apiKey,
|
|
416
|
+
model: config.extraction.model,
|
|
417
|
+
});
|
|
418
|
+
if (facts.length > 0) {
|
|
419
|
+
storeExtractedFacts(hippoRoot, candidate, facts);
|
|
420
|
+
extractedCount += facts.length;
|
|
421
|
+
}
|
|
322
422
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
rehearsed.map((e) => e.id).join(', '));
|
|
326
|
-
if (!dryRun) {
|
|
327
|
-
for (const r of rehearsed)
|
|
328
|
-
pendingWrites.push(r);
|
|
423
|
+
catch {
|
|
424
|
+
// Best-effort — continue with next candidate
|
|
329
425
|
}
|
|
330
426
|
}
|
|
427
|
+
result.extracted = extractedCount;
|
|
331
428
|
}
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
const extractionCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !extractedFromIds.has(e.id));
|
|
338
|
-
result.extractionCandidates = extractionCandidates.length;
|
|
339
|
-
const apiKey = process.env.ANTHROPIC_API_KEY ?? '';
|
|
340
|
-
if (apiKey && extractionCandidates.length > 0 && !dryRun) {
|
|
341
|
-
const { extractFacts, storeExtractedFacts } = await import('./extract.js');
|
|
342
|
-
const batchLimit = 20;
|
|
343
|
-
let extractedCount = 0;
|
|
344
|
-
for (const candidate of extractionCandidates.slice(0, batchLimit)) {
|
|
429
|
+
// -------------------------------------------------------------------------
|
|
430
|
+
// 1.7. DAG summarization — cluster extracted facts and generate summaries
|
|
431
|
+
// -------------------------------------------------------------------------
|
|
432
|
+
const extractedFacts = survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1);
|
|
433
|
+
if (apiKey && extractedFacts.length >= 3 && !dryRun) {
|
|
345
434
|
try {
|
|
346
|
-
const
|
|
435
|
+
const { buildDag } = await import('./dag.js');
|
|
436
|
+
const dagResult = await buildDag(hippoRoot, extractedFacts, {
|
|
347
437
|
apiKey,
|
|
348
438
|
model: config.extraction.model,
|
|
349
439
|
});
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
440
|
+
result.dagCandidateClusters = dagResult.candidateClusters;
|
|
441
|
+
result.dagSummariesCreated = dagResult.summariesCreated;
|
|
442
|
+
if (dagResult.summariesCreated > 0) {
|
|
443
|
+
result.details.push(` 🌳 DAG: ${dagResult.summariesCreated} summaries created, ${dagResult.factsLinked} facts linked`);
|
|
353
444
|
}
|
|
354
445
|
}
|
|
355
446
|
catch {
|
|
356
|
-
// Best-effort
|
|
357
|
-
}
|
|
358
|
-
}
|
|
359
|
-
result.extracted = extractedCount;
|
|
360
|
-
}
|
|
361
|
-
// -------------------------------------------------------------------------
|
|
362
|
-
// 1.7. DAG summarization — cluster extracted facts and generate summaries
|
|
363
|
-
// -------------------------------------------------------------------------
|
|
364
|
-
const extractedFacts = survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1);
|
|
365
|
-
if (apiKey && extractedFacts.length >= 3 && !dryRun) {
|
|
366
|
-
try {
|
|
367
|
-
const { buildDag } = await import('./dag.js');
|
|
368
|
-
const dagResult = await buildDag(hippoRoot, extractedFacts, {
|
|
369
|
-
apiKey,
|
|
370
|
-
model: config.extraction.model,
|
|
371
|
-
});
|
|
372
|
-
result.dagCandidateClusters = dagResult.candidateClusters;
|
|
373
|
-
result.dagSummariesCreated = dagResult.summariesCreated;
|
|
374
|
-
if (dagResult.summariesCreated > 0) {
|
|
375
|
-
result.details.push(` 🌳 DAG: ${dagResult.summariesCreated} summaries created, ${dagResult.factsLinked} facts linked`);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
catch {
|
|
379
|
-
// Best-effort
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
// -------------------------------------------------------------------------
|
|
383
|
-
// 1.8. DAG summary rebuild — drain dirty queue from E2's child-write hooks
|
|
384
|
-
// -------------------------------------------------------------------------
|
|
385
|
-
// Consumer of E2's summary_dirty flag. Walks dirty L2 summaries, regenerates
|
|
386
|
-
// each via generateDagSummary, atomically refreshes content + 6 metadata
|
|
387
|
-
// columns + clears summary_dirty (with FTS sync). Same apiKey/dryRun gate
|
|
388
|
-
// as buildDag above. Cap HIPPO_DAG_REBUILD_CAP (default 20, hard ceiling
|
|
389
|
-
// 1000) prevents runaway LLM cost.
|
|
390
|
-
if (apiKey && !dryRun) {
|
|
391
|
-
try {
|
|
392
|
-
const { rebuildDirtySummaries } = await import('./dag.js');
|
|
393
|
-
const rawCap = parseInt(process.env.HIPPO_DAG_REBUILD_CAP ?? '20', 10);
|
|
394
|
-
// R1 MED must-fix: hard ceiling so misconfigured env can't burn
|
|
395
|
-
// unbounded LLM cost.
|
|
396
|
-
const cap = Number.isFinite(rawCap) && rawCap > 0
|
|
397
|
-
? Math.min(rawCap, 1000)
|
|
398
|
-
: 20;
|
|
399
|
-
const rebuildResult = await rebuildDirtySummaries(hippoRoot, {
|
|
400
|
-
apiKey,
|
|
401
|
-
model: config.extraction.model,
|
|
402
|
-
cap,
|
|
403
|
-
});
|
|
404
|
-
result.summariesRebuilt = rebuildResult.rebuilt;
|
|
405
|
-
result.summariesRebuildFailed = rebuildResult.failed;
|
|
406
|
-
result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
|
|
407
|
-
result.summariesRebuildCapped = rebuildResult.capped;
|
|
408
|
-
if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0) {
|
|
409
|
-
const parts = [];
|
|
410
|
-
if (rebuildResult.rebuilt > 0)
|
|
411
|
-
parts.push(`${rebuildResult.rebuilt} rebuilt`);
|
|
412
|
-
if (rebuildResult.zeroChildSkipped > 0)
|
|
413
|
-
parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
|
|
414
|
-
if (rebuildResult.failed > 0)
|
|
415
|
-
parts.push(`${rebuildResult.failed} failed`);
|
|
416
|
-
if (rebuildResult.capped)
|
|
417
|
-
parts.push(`CAPPED@${cap}`);
|
|
418
|
-
result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
|
|
447
|
+
// Best-effort
|
|
419
448
|
}
|
|
420
449
|
}
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
if (l2Summaries.length >= 2) {
|
|
440
|
-
const profileResult = await buildEntityProfiles(hippoRoot, l2Summaries, {
|
|
450
|
+
// -------------------------------------------------------------------------
|
|
451
|
+
// 1.8. DAG summary rebuild — drain dirty queue from E2's child-write hooks
|
|
452
|
+
// -------------------------------------------------------------------------
|
|
453
|
+
// Consumer of E2's summary_dirty flag. Walks dirty L2 summaries, regenerates
|
|
454
|
+
// each via generateDagSummary, atomically refreshes content + 6 metadata
|
|
455
|
+
// columns + clears summary_dirty (with FTS sync). Same apiKey/dryRun gate
|
|
456
|
+
// as buildDag above. Cap HIPPO_DAG_REBUILD_CAP (default 20, hard ceiling
|
|
457
|
+
// 1000) prevents runaway LLM cost.
|
|
458
|
+
if (apiKey && !dryRun) {
|
|
459
|
+
try {
|
|
460
|
+
const { rebuildDirtySummaries } = await import('./dag.js');
|
|
461
|
+
const rawCap = parseInt(process.env.HIPPO_DAG_REBUILD_CAP ?? '20', 10);
|
|
462
|
+
// R1 MED must-fix: hard ceiling so misconfigured env can't burn
|
|
463
|
+
// unbounded LLM cost.
|
|
464
|
+
const cap = Number.isFinite(rawCap) && rawCap > 0
|
|
465
|
+
? Math.min(rawCap, 1000)
|
|
466
|
+
: 20;
|
|
467
|
+
const rebuildResult = await rebuildDirtySummaries(hippoRoot, {
|
|
441
468
|
apiKey,
|
|
442
469
|
model: config.extraction.model,
|
|
470
|
+
cap,
|
|
443
471
|
});
|
|
444
|
-
result.
|
|
445
|
-
|
|
446
|
-
|
|
472
|
+
result.summariesRebuilt = rebuildResult.rebuilt;
|
|
473
|
+
result.summariesRebuildFailed = rebuildResult.failed;
|
|
474
|
+
result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
|
|
475
|
+
result.summariesRebuildCapped = rebuildResult.capped;
|
|
476
|
+
if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0) {
|
|
477
|
+
const parts = [];
|
|
478
|
+
if (rebuildResult.rebuilt > 0)
|
|
479
|
+
parts.push(`${rebuildResult.rebuilt} rebuilt`);
|
|
480
|
+
if (rebuildResult.zeroChildSkipped > 0)
|
|
481
|
+
parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
|
|
482
|
+
if (rebuildResult.failed > 0)
|
|
483
|
+
parts.push(`${rebuildResult.failed} failed`);
|
|
484
|
+
if (rebuildResult.capped)
|
|
485
|
+
parts.push(`CAPPED@${cap}`);
|
|
486
|
+
result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
|
|
447
487
|
}
|
|
448
488
|
}
|
|
489
|
+
catch {
|
|
490
|
+
// Best-effort — same posture as buildDag block above.
|
|
491
|
+
}
|
|
449
492
|
}
|
|
450
|
-
|
|
451
|
-
|
|
493
|
+
// -------------------------------------------------------------------------
|
|
494
|
+
// 1.9. DAG entity profiles — cluster L2 topic summaries into L3 profiles
|
|
495
|
+
// -------------------------------------------------------------------------
|
|
496
|
+
// E5 phase: aggregate per-entity L2 summaries (e.g. all the speaker:Alice
|
|
497
|
+
// topic summaries) into a single L3 entity profile. Runs even when phase
|
|
498
|
+
// 1.7 buildDag was skipped (re-clusters existing L2s every sleep).
|
|
499
|
+
//
|
|
500
|
+
// Uses loadAllL2Summaries (not `survivors`) because phase 1.7 wrote new
|
|
501
|
+
// L2s directly via writeEntry without pushing back into survivors.
|
|
502
|
+
if (apiKey && !dryRun) {
|
|
503
|
+
try {
|
|
504
|
+
const { buildEntityProfiles } = await import('./dag.js');
|
|
505
|
+
const { loadAllL2Summaries } = await import('./store.js');
|
|
506
|
+
const l2Summaries = loadAllL2Summaries(hippoRoot);
|
|
507
|
+
if (l2Summaries.length >= 2) {
|
|
508
|
+
const profileResult = await buildEntityProfiles(hippoRoot, l2Summaries, {
|
|
509
|
+
apiKey,
|
|
510
|
+
model: config.extraction.model,
|
|
511
|
+
});
|
|
512
|
+
result.entityProfilesCreated = profileResult.profilesCreated;
|
|
513
|
+
if (profileResult.profilesCreated > 0) {
|
|
514
|
+
result.details.push(` 🌲 DAG L3: ${profileResult.profilesCreated} entity profiles, ${profileResult.l2sLinked} L2s linked`);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
catch {
|
|
519
|
+
// Best-effort.
|
|
520
|
+
}
|
|
452
521
|
}
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
set.
|
|
477
|
-
|
|
522
|
+
// -------------------------------------------------------------------------
|
|
523
|
+
// 2. Physics simulation pass
|
|
524
|
+
// -------------------------------------------------------------------------
|
|
525
|
+
if (!dryRun) {
|
|
526
|
+
try {
|
|
527
|
+
const physicsEnabled = config.physics.enabled === true
|
|
528
|
+
|| (config.physics.enabled === 'auto');
|
|
529
|
+
if (physicsEnabled) {
|
|
530
|
+
const db = openHippoDb(hippoRoot);
|
|
531
|
+
try {
|
|
532
|
+
const physicsMap = loadPhysicsState(db);
|
|
533
|
+
const particles = Array.from(physicsMap.values());
|
|
534
|
+
if (particles.length > 0) {
|
|
535
|
+
// Build entry lookup for property refresh
|
|
536
|
+
const entryMap = new Map(survivors.map(e => [e.id, e]));
|
|
537
|
+
refreshParticleProperties(particles, entryMap, now);
|
|
538
|
+
// Build conflict pairs from survivors
|
|
539
|
+
const conflictPairs = new Map();
|
|
540
|
+
for (const entry of survivors) {
|
|
541
|
+
if (entry.conflicts_with.length > 0) {
|
|
542
|
+
const set = conflictPairs.get(entry.id) ?? new Set();
|
|
543
|
+
for (const cid of entry.conflicts_with)
|
|
544
|
+
set.add(cid);
|
|
545
|
+
conflictPairs.set(entry.id, set);
|
|
546
|
+
}
|
|
478
547
|
}
|
|
548
|
+
// Build half-life lookup
|
|
549
|
+
const halfLives = new Map();
|
|
550
|
+
for (const entry of survivors) {
|
|
551
|
+
halfLives.set(entry.id, entry.half_life_days);
|
|
552
|
+
}
|
|
553
|
+
const ctx = {
|
|
554
|
+
conflictPairs,
|
|
555
|
+
halfLives,
|
|
556
|
+
config: config.physics,
|
|
557
|
+
};
|
|
558
|
+
const stats = simulate(particles, ctx);
|
|
559
|
+
savePhysicsState(db, particles);
|
|
560
|
+
result.physicsSimulated = stats.particleCount;
|
|
561
|
+
result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
|
|
562
|
+
`avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
|
|
563
|
+
`energy ${stats.energy.total.toFixed(4)}`);
|
|
479
564
|
}
|
|
480
|
-
// Build half-life lookup
|
|
481
|
-
const halfLives = new Map();
|
|
482
|
-
for (const entry of survivors) {
|
|
483
|
-
halfLives.set(entry.id, entry.half_life_days);
|
|
484
|
-
}
|
|
485
|
-
const ctx = {
|
|
486
|
-
conflictPairs,
|
|
487
|
-
halfLives,
|
|
488
|
-
config: config.physics,
|
|
489
|
-
};
|
|
490
|
-
const stats = simulate(particles, ctx);
|
|
491
|
-
savePhysicsState(db, particles);
|
|
492
|
-
result.physicsSimulated = stats.particleCount;
|
|
493
|
-
result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
|
|
494
|
-
`avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
|
|
495
|
-
`energy ${stats.energy.total.toFixed(4)}`);
|
|
496
565
|
}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
566
|
+
finally {
|
|
567
|
+
closeHippoDb(db);
|
|
568
|
+
}
|
|
500
569
|
}
|
|
501
570
|
}
|
|
571
|
+
catch (error) {
|
|
572
|
+
result.details.push(` ⚠️ physics simulation skipped: ${error instanceof Error ? error.message : 'unknown error'}`);
|
|
573
|
+
}
|
|
502
574
|
}
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
575
|
+
// -------------------------------------------------------------------------
|
|
576
|
+
// 3. Merge pass - episodic entries only
|
|
577
|
+
// -------------------------------------------------------------------------
|
|
578
|
+
const mergeCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !e.tags.includes('extracted'));
|
|
579
|
+
const used = new Set();
|
|
580
|
+
// AT1 consolidation-loop fix (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
|
|
581
|
+
// reuses the single consolidateDb handle opened once above (before the
|
|
582
|
+
// auto-promote pass, 1.4) for the whole non-dry-run consolidate — see that
|
|
583
|
+
// declaration's comment. Only needed for real writes — a dry-run preview
|
|
584
|
+
// never reaches batchWriteAndDelete's guard bypass, so there is nothing
|
|
585
|
+
// here for it to protect against. (mergesSkippedRejected itself is
|
|
586
|
+
// declared up at the try's opening above 1.4, not here — it has to
|
|
587
|
+
// survive the try/finally that now wraps this whole section; see the
|
|
588
|
+
// handle-leak restructure comment there.)
|
|
589
|
+
for (let i = 0; i < mergeCandidates.length; i++) {
|
|
590
|
+
if (used.has(mergeCandidates[i].id))
|
|
518
591
|
continue;
|
|
519
|
-
const
|
|
520
|
-
|
|
521
|
-
|
|
592
|
+
const cluster = [mergeCandidates[i]];
|
|
593
|
+
for (let j = i + 1; j < mergeCandidates.length; j++) {
|
|
594
|
+
if (used.has(mergeCandidates[j].id))
|
|
595
|
+
continue;
|
|
596
|
+
const overlap = textOverlap(mergeCandidates[i].content, mergeCandidates[j].content);
|
|
597
|
+
if (overlap >= MERGE_OVERLAP_THRESHOLD) {
|
|
598
|
+
cluster.push(mergeCandidates[j]);
|
|
599
|
+
}
|
|
522
600
|
}
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
//
|
|
555
|
-
//
|
|
556
|
-
//
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
601
|
+
if (cluster.length < MERGE_MIN_CLUSTER)
|
|
602
|
+
continue;
|
|
603
|
+
// Create a semantic summary
|
|
604
|
+
const mergedContent = mergeContents(cluster);
|
|
605
|
+
const allTags = Array.from(new Set(cluster.flatMap((e) => e.tags)));
|
|
606
|
+
const maxValence = pickStrongestValence(cluster);
|
|
607
|
+
// AT1 P2 fix: build the semantic entry FIRST — createMemory is cheap
|
|
608
|
+
// and pure — so the tombstone check below runs under the tenant the
|
|
609
|
+
// row will ACTUALLY land in. The prior version checked
|
|
610
|
+
// cluster[0].tenantId, but createMemory (below) is never passed a
|
|
611
|
+
// tenantId option, so it always stamps its own default ('default',
|
|
612
|
+
// see memory.ts) regardless of the cluster's source tenant — the
|
|
613
|
+
// check was consulting a tenant's tombstones that the write was never
|
|
614
|
+
// going to land in (a false negative on the real destination, and a
|
|
615
|
+
// possible false-block on the source tenant's unrelated tombstones).
|
|
616
|
+
// This fix only makes the CHECK match wherever the row actually
|
|
617
|
+
// lands; it deliberately does NOT change that destination — merge
|
|
618
|
+
// rows always landing in 'default' regardless of source tenant is a
|
|
619
|
+
// real, pre-existing cross-tenant question, tracked separately as an
|
|
620
|
+
// AT1 follow-up (not this fix's scope) rather than folded in here.
|
|
621
|
+
let semantic = null;
|
|
622
|
+
if (!dryRun) {
|
|
623
|
+
semantic = createMemory(mergedContent, {
|
|
624
|
+
layer: Layer.Semantic,
|
|
625
|
+
tags: allTags,
|
|
626
|
+
emotional_valence: maxValence,
|
|
627
|
+
schema_fit: 0.7,
|
|
628
|
+
source: 'consolidation',
|
|
629
|
+
confidence: 'inferred',
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
// mergeContents is DETERMINISTIC CONCATENATION (not an LLM paraphrase)
|
|
633
|
+
// — if a human rejected exactly this byte-identical rollup before, an
|
|
634
|
+
// unguarded sleep would regenerate it every cycle and
|
|
635
|
+
// batchWriteAndDelete's guard bypass (store.ts) would silently
|
|
636
|
+
// re-assert it forever. This producer-side check is what makes that
|
|
637
|
+
// bypass safe. A hit skips the WHOLE cluster: sources stay unmerged —
|
|
638
|
+
// not demoted, not deleted — so a later sleep gets another chance if
|
|
639
|
+
// the tombstone is lifted.
|
|
640
|
+
if (consolidateDb && semantic) {
|
|
641
|
+
const mergeDigest = rejectionDigest(semantic.content);
|
|
642
|
+
const tombstone = findRejectedValue(consolidateDb, semantic.tenantId, mergeDigest);
|
|
643
|
+
if (tombstone) {
|
|
644
|
+
// Still mark used — these members are not re-tried against a
|
|
645
|
+
// DIFFERENT cluster within this same pass; next sleep re-clusters
|
|
646
|
+
// them fresh.
|
|
647
|
+
for (const e of cluster)
|
|
648
|
+
used.add(e.id);
|
|
649
|
+
mergesSkippedRejected++;
|
|
650
|
+
try {
|
|
651
|
+
appendAuditEvent(consolidateDb, {
|
|
652
|
+
tenantId: semantic.tenantId,
|
|
653
|
+
actor: 'sleep',
|
|
654
|
+
op: 'reject_refusal',
|
|
655
|
+
metadata: {
|
|
656
|
+
digest: mergeDigest,
|
|
657
|
+
reason: tombstone.reason,
|
|
658
|
+
sourceIds: cluster.map((e) => e.id),
|
|
659
|
+
},
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
catch {
|
|
663
|
+
// Best-effort — mirrors store.ts's audit() semantics.
|
|
664
|
+
}
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
// Mark cluster members as used
|
|
669
|
+
for (const e of cluster)
|
|
670
|
+
used.add(e.id);
|
|
671
|
+
result.merged += cluster.length;
|
|
672
|
+
result.details.push(` 🔀 merged ${cluster.length} episodic entries into semantic: "${mergedContent.slice(0, 60)}..."`);
|
|
673
|
+
if (!dryRun && semantic) {
|
|
674
|
+
pendingWrites.push(semantic);
|
|
675
|
+
result.semanticCreated++;
|
|
676
|
+
// Demote source episodics (they've been compressed into neocortex):
|
|
677
|
+
// scale half_life_days so they decay sooner while staying recoverable.
|
|
678
|
+
// Immediate ranking is deliberately unchanged: the 2026-06-10 DAG
|
|
679
|
+
// slice-1 eval measured that dropping children below a worse-retrieving
|
|
680
|
+
// summary regresses budget-bounded QA (docs/evals/). The stored
|
|
681
|
+
// strength is refreshed to the live value so inspect, replay sampling,
|
|
682
|
+
// and strength-sorted assembly see the truth instead of a fake 0.3.
|
|
683
|
+
// Mutate in place (not a copy): `cluster` holds the same object
|
|
684
|
+
// references as `survivors`, and the later detectConflicts(survivors)
|
|
685
|
+
// pass in this same run must see the post-demotion half-life, or it
|
|
686
|
+
// can persist conflicts for entries the just-written state excludes.
|
|
687
|
+
for (const e of cluster) {
|
|
688
|
+
e.half_life_days = Math.max(1, Math.floor(e.half_life_days * MERGE_SOURCE_HALF_LIFE_FACTOR));
|
|
689
|
+
e.strength = calculateStrength(e, now, decayOpts);
|
|
690
|
+
pendingWrites.push(e);
|
|
691
|
+
}
|
|
561
692
|
}
|
|
562
693
|
}
|
|
563
694
|
}
|
|
695
|
+
finally {
|
|
696
|
+
if (consolidateDb)
|
|
697
|
+
closeHippoDb(consolidateDb);
|
|
698
|
+
}
|
|
699
|
+
if (mergesSkippedRejected > 0) {
|
|
700
|
+
console.error(`consolidate: skipped ${mergesSkippedRejected} merge(s) whose content matches a rejected value`);
|
|
701
|
+
}
|
|
564
702
|
// Flush all writes/deletes in a single transaction
|
|
565
703
|
if (!dryRun) {
|
|
566
704
|
batchWriteAndDelete(hippoRoot, pendingWrites, pendingDeletes);
|