hippo-memory 1.30.0 → 1.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/dist/api.d.ts +49 -0
  2. package/dist/api.d.ts.map +1 -1
  3. package/dist/api.js +78 -2
  4. package/dist/api.js.map +1 -1
  5. package/dist/audit.d.ts +1 -1
  6. package/dist/audit.d.ts.map +1 -1
  7. package/dist/audit.js.map +1 -1
  8. package/dist/capture.d.ts.map +1 -1
  9. package/dist/capture.js +79 -23
  10. package/dist/capture.js.map +1 -1
  11. package/dist/cli.d.ts +3 -0
  12. package/dist/cli.d.ts.map +1 -1
  13. package/dist/cli.js +233 -13
  14. package/dist/cli.js.map +1 -1
  15. package/dist/connectors/github/ingest.d.ts.map +1 -1
  16. package/dist/connectors/github/ingest.js +17 -0
  17. package/dist/connectors/github/ingest.js.map +1 -1
  18. package/dist/connectors/slack/ingest.d.ts.map +1 -1
  19. package/dist/connectors/slack/ingest.js +15 -0
  20. package/dist/connectors/slack/ingest.js.map +1 -1
  21. package/dist/consolidate.d.ts +1 -0
  22. package/dist/consolidate.d.ts.map +1 -1
  23. package/dist/consolidate.js +471 -291
  24. package/dist/consolidate.js.map +1 -1
  25. package/dist/dag.d.ts +17 -1
  26. package/dist/dag.d.ts.map +1 -1
  27. package/dist/dag.js +115 -40
  28. package/dist/dag.js.map +1 -1
  29. package/dist/db.d.ts.map +1 -1
  30. package/dist/db.js +43 -1
  31. package/dist/db.js.map +1 -1
  32. package/dist/extract.d.ts.map +1 -1
  33. package/dist/extract.js +24 -1
  34. package/dist/extract.js.map +1 -1
  35. package/dist/importers.d.ts +10 -0
  36. package/dist/importers.d.ts.map +1 -1
  37. package/dist/importers.js +242 -159
  38. package/dist/importers.js.map +1 -1
  39. package/dist/mcp/server.d.ts.map +1 -1
  40. package/dist/mcp/server.js +58 -24
  41. package/dist/mcp/server.js.map +1 -1
  42. package/dist/reject-flow.d.ts +66 -0
  43. package/dist/reject-flow.d.ts.map +1 -0
  44. package/dist/reject-flow.js +207 -0
  45. package/dist/reject-flow.js.map +1 -0
  46. package/dist/rejection.d.ts +100 -0
  47. package/dist/rejection.d.ts.map +1 -0
  48. package/dist/rejection.js +157 -0
  49. package/dist/rejection.js.map +1 -0
  50. package/dist/server.d.ts.map +1 -1
  51. package/dist/server.js +4 -0
  52. package/dist/server.js.map +1 -1
  53. package/dist/shared.d.ts +9 -0
  54. package/dist/shared.d.ts.map +1 -1
  55. package/dist/shared.js +57 -6
  56. package/dist/shared.js.map +1 -1
  57. package/dist/sleep-redact.d.ts +2 -0
  58. package/dist/sleep-redact.d.ts.map +1 -1
  59. package/dist/sleep-redact.js +2 -0
  60. package/dist/sleep-redact.js.map +1 -1
  61. package/dist/src/api.js +78 -2
  62. package/dist/src/api.js.map +1 -1
  63. package/dist/src/audit.js.map +1 -1
  64. package/dist/src/capture.js +79 -23
  65. package/dist/src/capture.js.map +1 -1
  66. package/dist/src/cli.js +233 -13
  67. package/dist/src/cli.js.map +1 -1
  68. package/dist/src/connectors/github/ingest.js +17 -0
  69. package/dist/src/connectors/github/ingest.js.map +1 -1
  70. package/dist/src/connectors/slack/ingest.js +15 -0
  71. package/dist/src/connectors/slack/ingest.js.map +1 -1
  72. package/dist/src/consolidate.js +471 -291
  73. package/dist/src/consolidate.js.map +1 -1
  74. package/dist/src/dag.js +115 -40
  75. package/dist/src/dag.js.map +1 -1
  76. package/dist/src/db.js +43 -1
  77. package/dist/src/db.js.map +1 -1
  78. package/dist/src/extract.js +24 -1
  79. package/dist/src/extract.js.map +1 -1
  80. package/dist/src/importers.js +242 -159
  81. package/dist/src/importers.js.map +1 -1
  82. package/dist/src/mcp/server.js +58 -24
  83. package/dist/src/mcp/server.js.map +1 -1
  84. package/dist/src/reject-flow.js +207 -0
  85. package/dist/src/reject-flow.js.map +1 -0
  86. package/dist/src/rejection.js +157 -0
  87. package/dist/src/rejection.js.map +1 -0
  88. package/dist/src/server.js +4 -0
  89. package/dist/src/server.js.map +1 -1
  90. package/dist/src/shared.js +57 -6
  91. package/dist/src/shared.js.map +1 -1
  92. package/dist/src/sleep-redact.js +2 -0
  93. package/dist/src/sleep-redact.js.map +1 -1
  94. package/dist/src/store.js +511 -38
  95. package/dist/src/store.js.map +1 -1
  96. package/dist/src/version.js +1 -1
  97. package/dist/store.d.ts +116 -6
  98. package/dist/store.d.ts.map +1 -1
  99. package/dist/store.js +511 -38
  100. package/dist/store.js.map +1 -1
  101. package/dist/version.d.ts +1 -1
  102. package/dist/version.js +1 -1
  103. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  104. package/extensions/openclaw-plugin/package.json +1 -1
  105. package/openclaw.plugin.json +1 -1
  106. package/package.json +1 -1
@@ -12,6 +12,7 @@ import { loadAllEntries, batchWriteAndDelete, appendConsolidationRun, replaceDet
12
12
  import { textOverlap, markRetrieved } from './search.js';
13
13
  import { compareEntryIdentity } from './compare.js';
14
14
  import { openHippoDb, closeHippoDb } from './db.js';
15
+ import { rejectionDigest, findRejectedValue } from './rejection.js';
15
16
  import { loadPhysicsState, savePhysicsState, refreshParticleProperties } from './physics-state.js';
16
17
  import { simulate } from './physics.js';
17
18
  import { loadConfig } from './config.js';
@@ -75,6 +76,7 @@ export async function consolidate(hippoRoot, options = {}) {
75
76
  summariesRebuilt: 0,
76
77
  summariesRebuildFailed: 0,
77
78
  summariesZeroChildSkipped: 0,
79
+ summariesRebuildRefused: 0,
78
80
  summariesRebuildCapped: false,
79
81
  entityProfilesCreated: 0,
80
82
  dryRun,
@@ -236,331 +238,509 @@ export async function consolidate(hippoRoot, options = {}) {
236
238
  }
237
239
  }
238
240
  }
239
- // -------------------------------------------------------------------------
240
- // 1.4. Auto-promote complete sessions to traces
241
- // -------------------------------------------------------------------------
241
+ // AT1 rejection-guard db handle (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
242
+ // covers BOTH the auto-promote pass (1.4, immediately below) and the merge
243
+ // pass (3, further down) — both build deterministic content that
244
+ // batchWriteAndDelete writes through the guard's bypass, so both need a
245
+ // producer-side tombstone check before pushing to pendingWrites.
242
246
  //
243
- // For each session within the configured window that has a `session_complete`
244
- // event and no existing trace (idempotency via the source_session_id column),
245
- // render the action sequence as markdown and persist a Layer.Trace memory.
246
- // Traces inherit decay, search, replay, and physics from the base MemoryEntry.
247
- if (!dryRun && config.autoTraceCapture !== false) {
248
- const windowDays = config.autoTraceWindowDays ?? 7;
249
- const sinceMs = now.getTime() - windowDays * 24 * 60 * 60 * 1000;
250
- // Auto-trace currently runs in a single-tenant context (the env-resolved
251
- // tenant for this process). Multi-tenant deployments that want
252
- // consolidation across all tenants need a per-tenant loop layered on top
253
- // of this — tracked in docs/plans/2026-05-02-continuity-tables-tenant-scope.md.
254
- const consolidationTenant = resolveTenantId({});
255
- const promotable = findPromotableSessions(hippoRoot, consolidationTenant, sinceMs);
256
- for (const session of promotable) {
257
- // Idempotency: skip if a trace for this session already exists.
258
- if (traceExistsForSession(hippoRoot, consolidationTenant, session.session_id))
259
- continue;
260
- const events = listSessionEvents(hippoRoot, consolidationTenant, {
261
- session_id: session.session_id,
262
- limit: 1000,
263
- });
264
- const completeEvent = events.find((e) => e.event_type === 'session_complete');
265
- if (!completeEvent)
266
- continue; // defence-in-depth; findPromotableSessions filters already.
267
- const outcomeRaw = completeEvent.content;
268
- if (outcomeRaw !== 'success' && outcomeRaw !== 'failure' && outcomeRaw !== 'partial') {
269
- // Malformed terminal event — skip rather than crash the whole sleep.
270
- continue;
247
+ // T3 fix (2026-08-15 hardening pass, perf hygiene): memoized lazy getter,
248
+ // not an eager open. The handle only serves these two tombstone checks —
249
+ // a sleep with zero promotable sessions and zero merge clusters never
250
+ // reaches either use site, so opening it unconditionally on every
251
+ // non-dry-run sleep paid a db-open cost for nothing. dryRun still never
252
+ // opens (getConsolidateDb short-circuits before touching the handle).
253
+ // Every `if (consolidateDb)` guard below becomes `const consolidateDb =
254
+ // getConsolidateDb();` at its use site — same semantics, opened on first
255
+ // real use. consolidateDbOpened (not just a truthy handle check) is the
256
+ // single source of truth for "was this ever opened", so the finally below
257
+ // closes it exactly once and never double-opens.
258
+ //
259
+ // AT1 P2 fix (codex, handle-leak restructure): the getter's lifetime must
260
+ // start IMMEDIATELY before the try whose finally closes it, covering every
261
+ // phase that can touch it — not just the merge pass. Previously the
262
+ // try/finally wrapped only section 3 (merge pass); an exception thrown by
263
+ // auto-promote (1.4), replay (1.5), batch extraction (1.6), the DAG
264
+ // passes (1.7-1.9), or physics (2) would propagate past an open handle
265
+ // with nothing to close it. No behavior change on the happy path — each
266
+ // of those phases already best-effort catches its own exceptions today
267
+ // (physics has its own try/catch below; each DAG pass wraps its own
268
+ // dynamic import + call in try/catch) — this only closes the handle on
269
+ // the rare path where one of them throws past its own catch. Deliberately
270
+ // NOT re-indented (mechanical wrap only, kept surgical): every statement
271
+ // between this try and its finally (below, at the end of the merge pass)
272
+ // stays at its original indentation.
273
+ let consolidateDbHandle = null;
274
+ let consolidateDbOpened = false;
275
+ const getConsolidateDb = () => {
276
+ if (dryRun)
277
+ return null;
278
+ if (!consolidateDbOpened) {
279
+ consolidateDbHandle = openHippoDb(hippoRoot);
280
+ consolidateDbOpened = true;
281
+ }
282
+ return consolidateDbHandle;
283
+ };
284
+ // Declared here (not at the merge pass, its point of use) so it survives
285
+ // this try/finally — a `let` declared INSIDE the try would go out of
286
+ // scope before the `if (mergesSkippedRejected > 0)` check that reads it
287
+ // after the finally closes the handle.
288
+ let mergesSkippedRejected = 0;
289
+ try {
290
+ // -------------------------------------------------------------------------
291
+ // 1.4. Auto-promote complete sessions to traces
292
+ // -------------------------------------------------------------------------
293
+ //
294
+ // For each session within the configured window that has a `session_complete`
295
+ // event and no existing trace (idempotency via the source_session_id column),
296
+ // render the action sequence as markdown and persist a Layer.Trace memory.
297
+ // Traces inherit decay, search, replay, and physics from the base MemoryEntry.
298
+ if (!dryRun && config.autoTraceCapture !== false) {
299
+ let tracesSkippedRejected = 0;
300
+ const windowDays = config.autoTraceWindowDays ?? 7;
301
+ const sinceMs = now.getTime() - windowDays * 24 * 60 * 60 * 1000;
302
+ // Auto-trace currently runs in a single-tenant context (the env-resolved
303
+ // tenant for this process). Multi-tenant deployments that want
304
+ // consolidation across all tenants need a per-tenant loop layered on top
305
+ // of this — tracked in docs/plans/2026-05-02-continuity-tables-tenant-scope.md.
306
+ const consolidationTenant = resolveTenantId({});
307
+ const promotable = findPromotableSessions(hippoRoot, consolidationTenant, sinceMs);
308
+ for (const session of promotable) {
309
+ // Idempotency: skip if a trace for this session already exists.
310
+ if (traceExistsForSession(hippoRoot, consolidationTenant, session.session_id))
311
+ continue;
312
+ const events = listSessionEvents(hippoRoot, consolidationTenant, {
313
+ session_id: session.session_id,
314
+ limit: 1000,
315
+ });
316
+ const completeEvent = events.find((e) => e.event_type === 'session_complete');
317
+ if (!completeEvent)
318
+ continue; // defence-in-depth; findPromotableSessions filters already.
319
+ const outcomeRaw = completeEvent.content;
320
+ if (outcomeRaw !== 'success' && outcomeRaw !== 'failure' && outcomeRaw !== 'partial') {
321
+ // Malformed terminal event — skip rather than crash the whole sleep.
322
+ continue;
323
+ }
324
+ const outcome = outcomeRaw;
325
+ const steps = events
326
+ .filter((e) => e.event_type !== 'session_complete')
327
+ .map((e) => ({ action: e.content, observation: '' }));
328
+ const summary = typeof completeEvent.metadata.summary === 'string'
329
+ ? completeEvent.metadata.summary
330
+ : '(untitled)';
331
+ const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
332
+ layer: Layer.Trace,
333
+ trace_outcome: outcome,
334
+ source_session_id: session.session_id,
335
+ tags: ['auto-promoted'],
336
+ source: 'auto-promote',
337
+ // T1 fix (2026-08-15 hardening pass): stamp the trace into the SAME
338
+ // tenant the traceExistsForSession idempotency check (above) runs
339
+ // under. Before
340
+ // this, createMemory omitted tenantId and the trace always landed
341
+ // 'default' (memory.ts:535) while the idempotency check ran under
342
+ // consolidationTenant — for any non-default tenant that check never
343
+ // hit, and the trace regenerated every sleep.
344
+ tenantId: consolidationTenant,
345
+ });
346
+ // AT1 (same producer-side pattern as the merge pass below): traceExistsForSession
347
+ // only sees rows CURRENTLY in the store — once a rejected trace is
348
+ // removed, that idempotency check no longer blocks regeneration, and
349
+ // this write would otherwise reach batchWriteAndDelete's guard bypass
350
+ // unchecked, resurrecting it every sleep. Check under THE ENTRY'S OWN
351
+ // stamped tenantId (read off `trace` after createMemory — never guess
352
+ // the tenant) + the built content's digest. A hit skips the push
353
+ // entirely: not counted as promoted, not added to survivors.
354
+ const consolidateDb = getConsolidateDb();
355
+ if (consolidateDb) {
356
+ const traceDigest = rejectionDigest(trace.content);
357
+ const tombstone = findRejectedValue(consolidateDb, trace.tenantId, traceDigest);
358
+ if (tombstone) {
359
+ tracesSkippedRejected++;
360
+ try {
361
+ appendAuditEvent(consolidateDb, {
362
+ tenantId: trace.tenantId,
363
+ actor: 'sleep',
364
+ op: 'reject_refusal',
365
+ metadata: {
366
+ digest: traceDigest,
367
+ reason: tombstone.reason,
368
+ sourceSessionId: session.session_id,
369
+ },
370
+ });
371
+ }
372
+ catch {
373
+ // Best-effort — mirrors store.ts's audit() semantics.
374
+ }
375
+ continue;
376
+ }
377
+ }
378
+ pendingWrites.push(trace);
379
+ survivors.push(trace);
380
+ result.promotedTraces++;
381
+ result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
382
+ }
383
+ if (result.promotedTraces > 0) {
384
+ result.details.push(` 🧬 promoted ${result.promotedTraces} trace${result.promotedTraces === 1 ? '' : 's'} from completed session${result.promotedTraces === 1 ? '' : 's'}`);
385
+ }
386
+ if (tracesSkippedRejected > 0) {
387
+ console.error(`consolidate: skipped ${tracesSkippedRejected} auto-promoted trace(s) whose content matches a rejected value`);
271
388
  }
272
- const outcome = outcomeRaw;
273
- const steps = events
274
- .filter((e) => e.event_type !== 'session_complete')
275
- .map((e) => ({ action: e.content, observation: '' }));
276
- const summary = typeof completeEvent.metadata.summary === 'string'
277
- ? completeEvent.metadata.summary
278
- : '(untitled)';
279
- const trace = createMemory(renderTraceContent({ task: summary, steps, outcome }), {
280
- layer: Layer.Trace,
281
- trace_outcome: outcome,
282
- source_session_id: session.session_id,
283
- tags: ['auto-promoted'],
284
- source: 'auto-promote',
285
- });
286
- pendingWrites.push(trace);
287
- survivors.push(trace);
288
- result.promotedTraces++;
289
- result.details.push(` 🧬 promoted trace ${trace.id} from session ${session.session_id} (${outcome})`);
290
389
  }
291
- if (result.promotedTraces > 0) {
292
- result.details.push(` 🧬 promoted ${result.promotedTraces} trace${result.promotedTraces === 1 ? '' : 's'} from completed session${result.promotedTraces === 1 ? '' : 's'}`);
390
+ // -------------------------------------------------------------------------
391
+ // 1.5. Replay pass — rehearse high-value survivors
392
+ // -------------------------------------------------------------------------
393
+ //
394
+ // Biologically-inspired counterpart to hippocampal replay during slow-wave
395
+ // sleep: sample N memories weighted by outcome + valence + under-rehearsal
396
+ // + idle time, then apply the same retrieval-strengthening `markRetrieved`
397
+ // applies to real queries. Distinct from decay (removal), physics (motion),
398
+ // and merge (compression) — this is the "rehearse the important stuff so
399
+ // it doesn't fade" pass.
400
+ {
401
+ const replayCount = config.replay?.count ?? REPLAY_COUNT_DEFAULT;
402
+ // EVAL-ONLY ablation (see ablation.ts): replay rehearsal IS recall
403
+ // strengthening (same markRetrieved dynamics), so the strengthen-off arm
404
+ // silences the whole pass - markRetrieved would return unmutated entries
405
+ // and persisting them anyway would still refresh updated_at / mirrors.
406
+ if (replayCount > 0 && survivors.length > 0 && !isRecallBoostAblated()) {
407
+ const seed = Math.floor(now.getTime() / 1000) & 0xffffffff;
408
+ const picked = sampleForReplay(survivors, replayCount, now, seed);
409
+ if (picked.length > 0) {
410
+ const rehearsed = markRetrieved(picked, now);
411
+ const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
412
+ // Update survivors in place so downstream passes see rehearsed state.
413
+ for (let i = 0; i < survivors.length; i++) {
414
+ const replacement = rehearsedById.get(survivors[i].id);
415
+ if (replacement)
416
+ survivors[i] = replacement;
417
+ }
418
+ result.replayed = rehearsed.length;
419
+ result.details.push(` 💭 replayed ${rehearsed.length} memor${rehearsed.length === 1 ? 'y' : 'ies'}: ` +
420
+ rehearsed.map((e) => e.id).join(', '));
421
+ if (!dryRun) {
422
+ for (const r of rehearsed)
423
+ pendingWrites.push(r);
424
+ }
425
+ }
426
+ }
293
427
  }
294
- }
295
- // -------------------------------------------------------------------------
296
- // 1.5. Replay pass — rehearse high-value survivors
297
- // -------------------------------------------------------------------------
298
- //
299
- // Biologically-inspired counterpart to hippocampal replay during slow-wave
300
- // sleep: sample N memories weighted by outcome + valence + under-rehearsal
301
- // + idle time, then apply the same retrieval-strengthening `markRetrieved`
302
- // applies to real queries. Distinct from decay (removal), physics (motion),
303
- // and merge (compression) — this is the "rehearse the important stuff so
304
- // it doesn't fade" pass.
305
- {
306
- const replayCount = config.replay?.count ?? REPLAY_COUNT_DEFAULT;
307
- // EVAL-ONLY ablation (see ablation.ts): replay rehearsal IS recall
308
- // strengthening (same markRetrieved dynamics), so the strengthen-off arm
309
- // silences the whole pass - markRetrieved would return unmutated entries
310
- // and persisting them anyway would still refresh updated_at / mirrors.
311
- if (replayCount > 0 && survivors.length > 0 && !isRecallBoostAblated()) {
312
- const seed = Math.floor(now.getTime() / 1000) & 0xffffffff;
313
- const picked = sampleForReplay(survivors, replayCount, now, seed);
314
- if (picked.length > 0) {
315
- const rehearsed = markRetrieved(picked, now);
316
- const rehearsedById = new Map(rehearsed.map((e) => [e.id, e]));
317
- // Update survivors in place so downstream passes see rehearsed state.
318
- for (let i = 0; i < survivors.length; i++) {
319
- const replacement = rehearsedById.get(survivors[i].id);
320
- if (replacement)
321
- survivors[i] = replacement;
428
+ // -------------------------------------------------------------------------
429
+ // 1.6. Batch extraction — extract facts from episodic memories missing them
430
+ // -------------------------------------------------------------------------
431
+ const extractedFromIds = new Set(survivors.filter((e) => e.extracted_from).map((e) => e.extracted_from));
432
+ const extractionCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !extractedFromIds.has(e.id));
433
+ result.extractionCandidates = extractionCandidates.length;
434
+ const apiKey = process.env.ANTHROPIC_API_KEY ?? '';
435
+ if (apiKey && extractionCandidates.length > 0 && !dryRun) {
436
+ const { extractFacts, storeExtractedFacts } = await import('./extract.js');
437
+ const batchLimit = 20;
438
+ let extractedCount = 0;
439
+ for (const candidate of extractionCandidates.slice(0, batchLimit)) {
440
+ try {
441
+ const facts = await extractFacts(candidate.content, {
442
+ apiKey,
443
+ model: config.extraction.model,
444
+ });
445
+ if (facts.length > 0) {
446
+ storeExtractedFacts(hippoRoot, candidate, facts);
447
+ extractedCount += facts.length;
448
+ }
322
449
  }
323
- result.replayed = rehearsed.length;
324
- result.details.push(` 💭 replayed ${rehearsed.length} memor${rehearsed.length === 1 ? 'y' : 'ies'}: ` +
325
- rehearsed.map((e) => e.id).join(', '));
326
- if (!dryRun) {
327
- for (const r of rehearsed)
328
- pendingWrites.push(r);
450
+ catch {
451
+ // Best-effort — continue with next candidate
329
452
  }
330
453
  }
454
+ result.extracted = extractedCount;
331
455
  }
332
- }
333
- // -------------------------------------------------------------------------
334
- // 1.6. Batch extraction — extract facts from episodic memories missing them
335
- // -------------------------------------------------------------------------
336
- const extractedFromIds = new Set(survivors.filter((e) => e.extracted_from).map((e) => e.extracted_from));
337
- const extractionCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !extractedFromIds.has(e.id));
338
- result.extractionCandidates = extractionCandidates.length;
339
- const apiKey = process.env.ANTHROPIC_API_KEY ?? '';
340
- if (apiKey && extractionCandidates.length > 0 && !dryRun) {
341
- const { extractFacts, storeExtractedFacts } = await import('./extract.js');
342
- const batchLimit = 20;
343
- let extractedCount = 0;
344
- for (const candidate of extractionCandidates.slice(0, batchLimit)) {
456
+ // -------------------------------------------------------------------------
457
+ // 1.7. DAG summarization — cluster extracted facts and generate summaries
458
+ // -------------------------------------------------------------------------
459
+ const extractedFacts = survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1);
460
+ if (apiKey && extractedFacts.length >= 3 && !dryRun) {
345
461
  try {
346
- const facts = await extractFacts(candidate.content, {
462
+ const { buildDag } = await import('./dag.js');
463
+ const dagResult = await buildDag(hippoRoot, extractedFacts, {
347
464
  apiKey,
348
465
  model: config.extraction.model,
349
466
  });
350
- if (facts.length > 0) {
351
- storeExtractedFacts(hippoRoot, candidate, facts);
352
- extractedCount += facts.length;
467
+ result.dagCandidateClusters = dagResult.candidateClusters;
468
+ result.dagSummariesCreated = dagResult.summariesCreated;
469
+ if (dagResult.summariesCreated > 0) {
470
+ result.details.push(` 🌳 DAG: ${dagResult.summariesCreated} summaries created, ${dagResult.factsLinked} facts linked`);
353
471
  }
354
472
  }
355
473
  catch {
356
- // Best-effort — continue with next candidate
474
+ // Best-effort
357
475
  }
358
476
  }
359
- result.extracted = extractedCount;
360
- }
361
- // -------------------------------------------------------------------------
362
- // 1.7. DAG summarization — cluster extracted facts and generate summaries
363
- // -------------------------------------------------------------------------
364
- const extractedFacts = survivors.filter((e) => e.tags.includes('extracted') && e.dag_level === 1);
365
- if (apiKey && extractedFacts.length >= 3 && !dryRun) {
366
- try {
367
- const { buildDag } = await import('./dag.js');
368
- const dagResult = await buildDag(hippoRoot, extractedFacts, {
369
- apiKey,
370
- model: config.extraction.model,
371
- });
372
- result.dagCandidateClusters = dagResult.candidateClusters;
373
- result.dagSummariesCreated = dagResult.summariesCreated;
374
- if (dagResult.summariesCreated > 0) {
375
- result.details.push(` 🌳 DAG: ${dagResult.summariesCreated} summaries created, ${dagResult.factsLinked} facts linked`);
376
- }
377
- }
378
- catch {
379
- // Best-effort
380
- }
381
- }
382
- // -------------------------------------------------------------------------
383
- // 1.8. DAG summary rebuild — drain dirty queue from E2's child-write hooks
384
- // -------------------------------------------------------------------------
385
- // Consumer of E2's summary_dirty flag. Walks dirty L2 summaries, regenerates
386
- // each via generateDagSummary, atomically refreshes content + 6 metadata
387
- // columns + clears summary_dirty (with FTS sync). Same apiKey/dryRun gate
388
- // as buildDag above. Cap HIPPO_DAG_REBUILD_CAP (default 20, hard ceiling
389
- // 1000) prevents runaway LLM cost.
390
- if (apiKey && !dryRun) {
391
- try {
392
- const { rebuildDirtySummaries } = await import('./dag.js');
393
- const rawCap = parseInt(process.env.HIPPO_DAG_REBUILD_CAP ?? '20', 10);
394
- // R1 MED must-fix: hard ceiling so misconfigured env can't burn
395
- // unbounded LLM cost.
396
- const cap = Number.isFinite(rawCap) && rawCap > 0
397
- ? Math.min(rawCap, 1000)
398
- : 20;
399
- const rebuildResult = await rebuildDirtySummaries(hippoRoot, {
400
- apiKey,
401
- model: config.extraction.model,
402
- cap,
403
- });
404
- result.summariesRebuilt = rebuildResult.rebuilt;
405
- result.summariesRebuildFailed = rebuildResult.failed;
406
- result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
407
- result.summariesRebuildCapped = rebuildResult.capped;
408
- if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0) {
409
- const parts = [];
410
- if (rebuildResult.rebuilt > 0)
411
- parts.push(`${rebuildResult.rebuilt} rebuilt`);
412
- if (rebuildResult.zeroChildSkipped > 0)
413
- parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
414
- if (rebuildResult.failed > 0)
415
- parts.push(`${rebuildResult.failed} failed`);
416
- if (rebuildResult.capped)
417
- parts.push(`CAPPED@${cap}`);
418
- result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
419
- }
420
- }
421
- catch {
422
- // Best-effort — same posture as buildDag block above.
423
- }
424
- }
425
- // -------------------------------------------------------------------------
426
- // 1.9. DAG entity profiles — cluster L2 topic summaries into L3 profiles
427
- // -------------------------------------------------------------------------
428
- // E5 phase: aggregate per-entity L2 summaries (e.g. all the speaker:Alice
429
- // topic summaries) into a single L3 entity profile. Runs even when phase
430
- // 1.7 buildDag was skipped (re-clusters existing L2s every sleep).
431
- //
432
- // Uses loadAllL2Summaries (not `survivors`) because phase 1.7 wrote new
433
- // L2s directly via writeEntry without pushing back into survivors.
434
- if (apiKey && !dryRun) {
435
- try {
436
- const { buildEntityProfiles } = await import('./dag.js');
437
- const { loadAllL2Summaries } = await import('./store.js');
438
- const l2Summaries = loadAllL2Summaries(hippoRoot);
439
- if (l2Summaries.length >= 2) {
440
- const profileResult = await buildEntityProfiles(hippoRoot, l2Summaries, {
477
+ // -------------------------------------------------------------------------
478
+ // 1.8. DAG summary rebuild — drain dirty queue from E2's child-write hooks
479
+ // -------------------------------------------------------------------------
480
+ // Consumer of E2's summary_dirty flag. Walks dirty L2 summaries, regenerates
481
+ // each via generateDagSummary, atomically refreshes content + 6 metadata
482
+ // columns + clears summary_dirty (with FTS sync). Same apiKey/dryRun gate
483
+ // as buildDag above. Cap HIPPO_DAG_REBUILD_CAP (default 20, hard ceiling
484
+ // 1000) prevents runaway LLM cost.
485
+ if (apiKey && !dryRun) {
486
+ try {
487
+ const { rebuildDirtySummaries } = await import('./dag.js');
488
+ const rawCap = parseInt(process.env.HIPPO_DAG_REBUILD_CAP ?? '20', 10);
489
+ // R1 MED must-fix: hard ceiling so misconfigured env can't burn
490
+ // unbounded LLM cost.
491
+ const cap = Number.isFinite(rawCap) && rawCap > 0
492
+ ? Math.min(rawCap, 1000)
493
+ : 20;
494
+ const rebuildResult = await rebuildDirtySummaries(hippoRoot, {
441
495
  apiKey,
442
496
  model: config.extraction.model,
497
+ cap,
443
498
  });
444
- result.entityProfilesCreated = profileResult.profilesCreated;
445
- if (profileResult.profilesCreated > 0) {
446
- result.details.push(` 🌲 DAG L3: ${profileResult.profilesCreated} entity profiles, ${profileResult.l2sLinked} L2s linked`);
499
+ result.summariesRebuilt = rebuildResult.rebuilt;
500
+ result.summariesRebuildFailed = rebuildResult.failed;
501
+ result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
502
+ result.summariesRebuildRefused = rebuildResult.refused;
503
+ result.summariesRebuildCapped = rebuildResult.capped;
504
+ if (rebuildResult.rebuilt > 0 || rebuildResult.zeroChildSkipped > 0 || rebuildResult.failed > 0 || rebuildResult.refused > 0) {
505
+ const parts = [];
506
+ if (rebuildResult.rebuilt > 0)
507
+ parts.push(`${rebuildResult.rebuilt} rebuilt`);
508
+ if (rebuildResult.refused > 0)
509
+ parts.push(`${rebuildResult.refused} refused`);
510
+ if (rebuildResult.zeroChildSkipped > 0)
511
+ parts.push(`${rebuildResult.zeroChildSkipped} zero-child-skipped`);
512
+ if (rebuildResult.failed > 0)
513
+ parts.push(`${rebuildResult.failed} failed`);
514
+ if (rebuildResult.capped)
515
+ parts.push(`CAPPED@${cap}`);
516
+ result.details.push(` 🌳 DAG rebuild: ${parts.join(', ')}`);
447
517
  }
448
518
  }
519
+ catch {
520
+ // Best-effort — same posture as buildDag block above.
521
+ }
449
522
  }
450
- catch {
451
- // Best-effort.
523
+ // -------------------------------------------------------------------------
524
+ // 1.9. DAG entity profiles — cluster L2 topic summaries into L3 profiles
525
+ // -------------------------------------------------------------------------
526
+ // E5 phase: aggregate per-entity L2 summaries (e.g. all the speaker:Alice
527
+ // topic summaries) into a single L3 entity profile. Runs even when phase
528
+ // 1.7 buildDag was skipped (re-clusters existing L2s every sleep).
529
+ //
530
+ // Uses loadAllL2Summaries (not `survivors`) because phase 1.7 wrote new
531
+ // L2s directly via writeEntry without pushing back into survivors.
532
+ if (apiKey && !dryRun) {
533
+ try {
534
+ const { buildEntityProfiles } = await import('./dag.js');
535
+ const { loadAllL2Summaries } = await import('./store.js');
536
+ const l2Summaries = loadAllL2Summaries(hippoRoot);
537
+ if (l2Summaries.length >= 2) {
538
+ const profileResult = await buildEntityProfiles(hippoRoot, l2Summaries, {
539
+ apiKey,
540
+ model: config.extraction.model,
541
+ });
542
+ result.entityProfilesCreated = profileResult.profilesCreated;
543
+ if (profileResult.profilesCreated > 0) {
544
+ result.details.push(` 🌲 DAG L3: ${profileResult.profilesCreated} entity profiles, ${profileResult.l2sLinked} L2s linked`);
545
+ }
546
+ }
547
+ }
548
+ catch {
549
+ // Best-effort.
550
+ }
452
551
  }
453
- }
454
- // -------------------------------------------------------------------------
455
- // 2. Physics simulation pass
456
- // -------------------------------------------------------------------------
457
- if (!dryRun) {
458
- try {
459
- const physicsEnabled = config.physics.enabled === true
460
- || (config.physics.enabled === 'auto');
461
- if (physicsEnabled) {
462
- const db = openHippoDb(hippoRoot);
463
- try {
464
- const physicsMap = loadPhysicsState(db);
465
- const particles = Array.from(physicsMap.values());
466
- if (particles.length > 0) {
467
- // Build entry lookup for property refresh
468
- const entryMap = new Map(survivors.map(e => [e.id, e]));
469
- refreshParticleProperties(particles, entryMap, now);
470
- // Build conflict pairs from survivors
471
- const conflictPairs = new Map();
472
- for (const entry of survivors) {
473
- if (entry.conflicts_with.length > 0) {
474
- const set = conflictPairs.get(entry.id) ?? new Set();
475
- for (const cid of entry.conflicts_with)
476
- set.add(cid);
477
- conflictPairs.set(entry.id, set);
552
+ // -------------------------------------------------------------------------
553
+ // 2. Physics simulation pass
554
+ // -------------------------------------------------------------------------
555
+ if (!dryRun) {
556
+ try {
557
+ const physicsEnabled = config.physics.enabled === true
558
+ || (config.physics.enabled === 'auto');
559
+ if (physicsEnabled) {
560
+ const db = openHippoDb(hippoRoot);
561
+ try {
562
+ const physicsMap = loadPhysicsState(db);
563
+ const particles = Array.from(physicsMap.values());
564
+ if (particles.length > 0) {
565
+ // Build entry lookup for property refresh
566
+ const entryMap = new Map(survivors.map(e => [e.id, e]));
567
+ refreshParticleProperties(particles, entryMap, now);
568
+ // Build conflict pairs from survivors
569
+ const conflictPairs = new Map();
570
+ for (const entry of survivors) {
571
+ if (entry.conflicts_with.length > 0) {
572
+ const set = conflictPairs.get(entry.id) ?? new Set();
573
+ for (const cid of entry.conflicts_with)
574
+ set.add(cid);
575
+ conflictPairs.set(entry.id, set);
576
+ }
478
577
  }
578
+ // Build half-life lookup
579
+ const halfLives = new Map();
580
+ for (const entry of survivors) {
581
+ halfLives.set(entry.id, entry.half_life_days);
582
+ }
583
+ const ctx = {
584
+ conflictPairs,
585
+ halfLives,
586
+ config: config.physics,
587
+ };
588
+ const stats = simulate(particles, ctx);
589
+ savePhysicsState(db, particles);
590
+ result.physicsSimulated = stats.particleCount;
591
+ result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
592
+ `avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
593
+ `energy ${stats.energy.total.toFixed(4)}`);
479
594
  }
480
- // Build half-life lookup
481
- const halfLives = new Map();
482
- for (const entry of survivors) {
483
- halfLives.set(entry.id, entry.half_life_days);
484
- }
485
- const ctx = {
486
- conflictPairs,
487
- halfLives,
488
- config: config.physics,
489
- };
490
- const stats = simulate(particles, ctx);
491
- savePhysicsState(db, particles);
492
- result.physicsSimulated = stats.particleCount;
493
- result.details.push(` ⚛️ physics: ${stats.particleCount} particles, ` +
494
- `avg vel ${stats.avgVelocityMagnitude.toFixed(4)}, ` +
495
- `energy ${stats.energy.total.toFixed(4)}`);
496
595
  }
497
- }
498
- finally {
499
- closeHippoDb(db);
596
+ finally {
597
+ closeHippoDb(db);
598
+ }
500
599
  }
501
600
  }
502
- }
503
- catch (error) {
504
- result.details.push(` ⚠️ physics simulation skipped: ${error instanceof Error ? error.message : 'unknown error'}`);
505
- }
506
- }
507
- // -------------------------------------------------------------------------
508
- // 3. Merge pass - episodic entries only
509
- // -------------------------------------------------------------------------
510
- const mergeCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !e.tags.includes('extracted'));
511
- const used = new Set();
512
- for (let i = 0; i < mergeCandidates.length; i++) {
513
- if (used.has(mergeCandidates[i].id))
514
- continue;
515
- const cluster = [mergeCandidates[i]];
516
- for (let j = i + 1; j < mergeCandidates.length; j++) {
517
- if (used.has(mergeCandidates[j].id))
518
- continue;
519
- const overlap = textOverlap(mergeCandidates[i].content, mergeCandidates[j].content);
520
- if (overlap >= MERGE_OVERLAP_THRESHOLD) {
521
- cluster.push(mergeCandidates[j]);
601
+ catch (error) {
602
+ result.details.push(` ⚠️ physics simulation skipped: ${error instanceof Error ? error.message : 'unknown error'}`);
522
603
  }
523
604
  }
524
- if (cluster.length < MERGE_MIN_CLUSTER)
525
- continue;
526
- // Mark cluster members as used
527
- for (const e of cluster)
528
- used.add(e.id);
529
- result.merged += cluster.length;
530
- // Create a semantic summary
531
- const mergedContent = mergeContents(cluster);
532
- const allTags = Array.from(new Set(cluster.flatMap((e) => e.tags)));
533
- const maxValence = pickStrongestValence(cluster);
534
- result.details.push(` 🔀 merged ${cluster.length} episodic entries into semantic: "${mergedContent.slice(0, 60)}..."`);
535
- if (!dryRun) {
536
- const semantic = createMemory(mergedContent, {
537
- layer: Layer.Semantic,
538
- tags: allTags,
539
- emotional_valence: maxValence,
540
- schema_fit: 0.7,
541
- source: 'consolidation',
542
- confidence: 'inferred',
543
- });
544
- pendingWrites.push(semantic);
545
- result.semanticCreated++;
546
- // Demote source episodics (they've been compressed into neocortex):
547
- // scale half_life_days so they decay sooner while staying recoverable.
548
- // Immediate ranking is deliberately unchanged: the 2026-06-10 DAG
549
- // slice-1 eval measured that dropping children below a worse-retrieving
550
- // summary regresses budget-bounded QA (docs/evals/). The stored
551
- // strength is refreshed to the live value so inspect, replay sampling,
552
- // and strength-sorted assembly see the truth instead of a fake 0.3.
553
- // Mutate in place (not a copy): `cluster` holds the same object
554
- // references as `survivors`, and the later detectConflicts(survivors)
555
- // pass in this same run must see the post-demotion half-life, or it
556
- // can persist conflicts for entries the just-written state excludes.
557
- for (const e of cluster) {
558
- e.half_life_days = Math.max(1, Math.floor(e.half_life_days * MERGE_SOURCE_HALF_LIFE_FACTOR));
559
- e.strength = calculateStrength(e, now, decayOpts);
560
- pendingWrites.push(e);
605
+ // -------------------------------------------------------------------------
606
+ // 3. Merge pass - episodic entries only
607
+ // -------------------------------------------------------------------------
608
+ const mergeCandidates = survivors.filter((e) => e.layer === Layer.Episodic && !e.tags.includes('extracted'));
609
+ const used = new Set();
610
+ // T1 fix (2026-08-15 hardening pass): partition by tenantId BEFORE the
611
+ // overlap loop so a cluster can never span tenants. Previously textOverlap
612
+ // clustered across the whole host-wide `survivors` list with no tenant
613
+ // boundary, and mergeContents concatenated cross-tenant content into one
614
+ // row. Map preserves insertion order, so single-tenant stores (every row
615
+ // 'default') get exactly one partition and iterate in the same order as
616
+ // before this fix — byte-identical behavior there.
617
+ const mergeCandidatesByTenant = new Map();
618
+ for (const entry of mergeCandidates) {
619
+ const bucket = mergeCandidatesByTenant.get(entry.tenantId);
620
+ if (bucket)
621
+ bucket.push(entry);
622
+ else
623
+ mergeCandidatesByTenant.set(entry.tenantId, [entry]);
624
+ }
625
+ // AT1 consolidation-loop fix (docs/plans/2026-08-15-at1-rejected-value-tombstone.md):
626
+ // reuses the single consolidateDb handle opened once above (before the
627
+ // auto-promote pass, 1.4) for the whole non-dry-run consolidate — see that
628
+ // declaration's comment. Only needed for real writes — a dry-run preview
629
+ // never reaches batchWriteAndDelete's guard bypass, so there is nothing
630
+ // here for it to protect against. (mergesSkippedRejected itself is
631
+ // declared up at the try's opening above 1.4, not here — it has to
632
+ // survive the try/finally that now wraps this whole section; see the
633
+ // handle-leak restructure comment there.)
634
+ for (const [mergeTenant, tenantCandidates] of mergeCandidatesByTenant) {
635
+ for (let i = 0; i < tenantCandidates.length; i++) {
636
+ if (used.has(tenantCandidates[i].id))
637
+ continue;
638
+ const cluster = [tenantCandidates[i]];
639
+ for (let j = i + 1; j < tenantCandidates.length; j++) {
640
+ if (used.has(tenantCandidates[j].id))
641
+ continue;
642
+ const overlap = textOverlap(tenantCandidates[i].content, tenantCandidates[j].content);
643
+ if (overlap >= MERGE_OVERLAP_THRESHOLD) {
644
+ cluster.push(tenantCandidates[j]);
645
+ }
646
+ }
647
+ if (cluster.length < MERGE_MIN_CLUSTER)
648
+ continue;
649
+ // Create a semantic summary
650
+ const mergedContent = mergeContents(cluster);
651
+ const allTags = Array.from(new Set(cluster.flatMap((e) => e.tags)));
652
+ const maxValence = pickStrongestValence(cluster);
653
+ // AT1 P2 fix: build the semantic entry FIRST — createMemory is cheap
654
+ // and pure — so the tombstone check below runs under the tenant the
655
+ // row will ACTUALLY land in.
656
+ // T1 fix: createMemory now receives tenantId: mergeTenant (the
657
+ // partition's tenant — every member of `cluster` shares it by
658
+ // construction), so the row lands in its source tenant instead of
659
+ // always 'default'.
660
+ let semantic = null;
661
+ if (!dryRun) {
662
+ semantic = createMemory(mergedContent, {
663
+ layer: Layer.Semantic,
664
+ tags: allTags,
665
+ emotional_valence: maxValence,
666
+ schema_fit: 0.7,
667
+ source: 'consolidation',
668
+ confidence: 'inferred',
669
+ tenantId: mergeTenant,
670
+ });
671
+ }
672
+ // mergeContents is DETERMINISTIC CONCATENATION (not an LLM paraphrase)
673
+ // — if a human rejected exactly this byte-identical rollup before, an
674
+ // unguarded sleep would regenerate it every cycle and
675
+ // batchWriteAndDelete's guard bypass (store.ts) would silently
676
+ // re-assert it forever. This producer-side check is what makes that
677
+ // bypass safe. A hit skips the WHOLE cluster: sources stay unmerged —
678
+ // not demoted, not deleted — so a later sleep gets another chance if
679
+ // the tombstone is lifted.
680
+ const consolidateDb = getConsolidateDb();
681
+ if (consolidateDb && semantic) {
682
+ const mergeDigest = rejectionDigest(semantic.content);
683
+ const tombstone = findRejectedValue(consolidateDb, semantic.tenantId, mergeDigest);
684
+ if (tombstone) {
685
+ // Still mark used — these members are not re-tried against a
686
+ // DIFFERENT cluster within this same pass; next sleep re-clusters
687
+ // them fresh.
688
+ for (const e of cluster)
689
+ used.add(e.id);
690
+ mergesSkippedRejected++;
691
+ try {
692
+ appendAuditEvent(consolidateDb, {
693
+ tenantId: semantic.tenantId,
694
+ actor: 'sleep',
695
+ op: 'reject_refusal',
696
+ metadata: {
697
+ digest: mergeDigest,
698
+ reason: tombstone.reason,
699
+ sourceIds: cluster.map((e) => e.id),
700
+ },
701
+ });
702
+ }
703
+ catch {
704
+ // Best-effort — mirrors store.ts's audit() semantics.
705
+ }
706
+ continue;
707
+ }
708
+ }
709
+ // Mark cluster members as used
710
+ for (const e of cluster)
711
+ used.add(e.id);
712
+ result.merged += cluster.length;
713
+ result.details.push(` 🔀 merged ${cluster.length} episodic entries into semantic: "${mergedContent.slice(0, 60)}..."`);
714
+ if (!dryRun && semantic) {
715
+ pendingWrites.push(semantic);
716
+ result.semanticCreated++;
717
+ // Demote source episodics (they've been compressed into neocortex):
718
+ // scale half_life_days so they decay sooner while staying recoverable.
719
+ // Immediate ranking is deliberately unchanged: the 2026-06-10 DAG
720
+ // slice-1 eval measured that dropping children below a worse-retrieving
721
+ // summary regresses budget-bounded QA (docs/evals/). The stored
722
+ // strength is refreshed to the live value so inspect, replay sampling,
723
+ // and strength-sorted assembly see the truth instead of a fake 0.3.
724
+ // Mutate in place (not a copy): `cluster` holds the same object
725
+ // references as `survivors`, and the later detectConflicts(survivors)
726
+ // pass in this same run must see the post-demotion half-life, or it
727
+ // can persist conflicts for entries the just-written state excludes.
728
+ for (const e of cluster) {
729
+ e.half_life_days = Math.max(1, Math.floor(e.half_life_days * MERGE_SOURCE_HALF_LIFE_FACTOR));
730
+ e.strength = calculateStrength(e, now, decayOpts);
731
+ pendingWrites.push(e);
732
+ }
733
+ }
561
734
  }
562
735
  }
563
736
  }
737
+ finally {
738
+ if (consolidateDbHandle)
739
+ closeHippoDb(consolidateDbHandle);
740
+ }
741
+ if (mergesSkippedRejected > 0) {
742
+ console.error(`consolidate: skipped ${mergesSkippedRejected} merge(s) whose content matches a rejected value`);
743
+ }
564
744
  // Flush all writes/deletes in a single transaction
565
745
  if (!dryRun) {
566
746
  batchWriteAndDelete(hippoRoot, pendingWrites, pendingDeletes);