hippo-memory 3.3.3 → 3.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/README.md +4 -4
  2. package/dist/api/callers.d.ts +4 -1
  3. package/dist/api/context/context-plan.js +3 -0
  4. package/dist/api/context/context-pools.js +4 -1
  5. package/dist/api/context/user-rule.js +2 -1
  6. package/dist/api/dashboard/detail.js +2 -1
  7. package/dist/api/dashboard/live.d.ts +0 -4
  8. package/dist/api/dashboard/live.js +0 -8
  9. package/dist/api/memories/conflicts.d.ts +2 -1
  10. package/dist/api/memories/conflicts.js +3 -8
  11. package/dist/api/memories/promote.js +6 -2
  12. package/dist/api/objects/object-kinds.d.ts +1 -4
  13. package/dist/api/objects/object-kinds.js +1 -5
  14. package/dist/api/recall/compose.js +2 -1
  15. package/dist/api/recall/rank-stages.js +2 -1
  16. package/dist/api/recall/rows.js +2 -1
  17. package/dist/api/session-end.d.ts +3 -2
  18. package/dist/api/session-end.js +8 -4
  19. package/dist/api/sharing.d.ts +4 -3
  20. package/dist/api/sharing.js +2 -2
  21. package/dist/api/sleep/auto-sleep.d.ts +3 -2
  22. package/dist/api/sleep/auto-sleep.js +16 -3
  23. package/dist/api/sleep/one-sleep.d.ts +23 -0
  24. package/dist/api/sleep/one-sleep.js +53 -0
  25. package/dist/api/sleep/sleep-run.js +2 -1
  26. package/dist/api/sleep/sleep.d.ts +5 -0
  27. package/dist/api/sleep/sleep.js +2 -1
  28. package/dist/api/summary-parents.js +2 -1
  29. package/dist/base/config/config-keys.js +1 -0
  30. package/dist/base/config/config.d.ts +2 -0
  31. package/dist/base/config/config.js +1 -0
  32. package/dist/base/embeddings/local.d.ts +1 -1
  33. package/dist/base/embeddings/local.js +2 -3
  34. package/dist/base/memory/ambient.js +2 -1
  35. package/dist/base/memory/entry.d.ts +4 -1
  36. package/dist/base/memory/entry.js +7 -1
  37. package/dist/base/memory/half-life-plan.js +6 -4
  38. package/dist/base/memory/keep-rules.js +2 -1
  39. package/dist/base/memory/liveness.d.ts +7 -0
  40. package/dist/base/memory/liveness.js +6 -0
  41. package/dist/base/memory/markdown.js +15 -5
  42. package/dist/base/memory/memory-audit.js +2 -1
  43. package/dist/base/memory/memory-facts.d.ts +0 -2
  44. package/dist/base/memory/memory-facts.js +0 -2
  45. package/dist/base/memory/merged-row.js +3 -2
  46. package/dist/base/memory/rule-replaces.js +2 -1
  47. package/dist/base/memory/stale-mark.d.ts +9 -0
  48. package/dist/base/memory/stale-mark.js +18 -0
  49. package/dist/base/ports/store/capture.d.ts +2 -0
  50. package/dist/base/ports/store/conflicts.d.ts +5 -1
  51. package/dist/base/ports/store/conflicts.js +2 -0
  52. package/dist/base/ports/store/consolidation.d.ts +66 -2
  53. package/dist/base/ports/store/consolidation.js +5 -0
  54. package/dist/base/ports/store/context-reads.d.ts +2 -0
  55. package/dist/base/ports/store/objects.d.ts +15 -4
  56. package/dist/base/ports/store/sessions.d.ts +8 -2
  57. package/dist/base/ports/store/sessions.js +6 -0
  58. package/dist/base/ports/store/store-effects.d.ts +5 -0
  59. package/dist/base/ports/store.d.ts +1 -1
  60. package/dist/base/scope/live-memory.d.ts +7 -0
  61. package/dist/base/scope/live-memory.js +10 -0
  62. package/dist/base/scope/read-admission.d.ts +2 -0
  63. package/dist/base/scope/read-admission.js +9 -2
  64. package/dist/base/util/version.d.ts +1 -1
  65. package/dist/base/util/version.js +1 -1
  66. package/dist/db/continuity-tables.d.ts +4 -1
  67. package/dist/db/continuity-tables.js +21 -12
  68. package/dist/db/migrate.js +2 -2
  69. package/dist/db/migrations/index.d.ts +1 -1
  70. package/dist/db/migrations/index.js +5 -1
  71. package/dist/db/migrations/v61.js +3 -3
  72. package/dist/db/migrations/v62.d.ts +3 -0
  73. package/dist/db/migrations/v62.js +43 -0
  74. package/dist/db/migrations/v63.d.ts +3 -0
  75. package/dist/db/migrations/v63.js +9 -0
  76. package/dist/db/required-objects.js +2 -1
  77. package/dist/db/sql-predicates.d.ts +22 -2
  78. package/dist/db/sql-predicates.js +27 -2
  79. package/dist/domain/agent-memories/apply.js +2 -1
  80. package/dist/domain/agent-memories/container-view.js +2 -1
  81. package/dist/domain/consolidate/conflicts.js +2 -1
  82. package/dist/domain/consolidate/dag-summary.d.ts +2 -0
  83. package/dist/domain/consolidate/dag.d.ts +3 -0
  84. package/dist/domain/consolidate/dag.js +53 -8
  85. package/dist/domain/consolidate/dedupe.js +2 -1
  86. package/dist/domain/consolidate/events.d.ts +2 -0
  87. package/dist/domain/consolidate/llm-passes.js +94 -20
  88. package/dist/domain/consolidate/merge.js +11 -10
  89. package/dist/domain/consolidate/retry-holds.d.ts +14 -0
  90. package/dist/domain/consolidate/retry-holds.js +26 -0
  91. package/dist/domain/consolidate/run.d.ts +8 -0
  92. package/dist/domain/consolidate/run.js +2 -0
  93. package/dist/domain/consolidate/tombstones.d.ts +3 -4
  94. package/dist/domain/consolidate/tombstones.js +3 -3
  95. package/dist/domain/graph/recall.js +3 -2
  96. package/dist/domain/learn/churn-stale.js +2 -1
  97. package/dist/domain/learn/extract.d.ts +10 -0
  98. package/dist/domain/learn/extract.js +10 -5
  99. package/dist/domain/learn/invalidation.d.ts +1 -1
  100. package/dist/domain/learn/invalidation.js +4 -11
  101. package/dist/domain/objects/decisions.d.ts +4 -7
  102. package/dist/domain/objects/decisions.js +1 -14
  103. package/dist/domain/objects/descriptor.d.ts +2 -0
  104. package/dist/domain/objects/lifecycle.js +8 -2
  105. package/dist/domain/search/as-of.js +3 -2
  106. package/dist/domain/sharing/share.d.ts +16 -4
  107. package/dist/domain/sharing/share.js +33 -17
  108. package/dist/entry/store-kit.js +2 -1
  109. package/dist/infra/local-embedder.js +14 -2
  110. package/dist/infra/spool-claims.d.ts +2 -2
  111. package/dist/infra/spool-claims.js +4 -3
  112. package/dist/infra/spool-import.js +1 -1
  113. package/dist/infra/spool-record.d.ts +3 -1
  114. package/dist/store/sqlite/capture/compaction-hooks.js +14 -6
  115. package/dist/store/sqlite/capture/compaction-rows.js +9 -2
  116. package/dist/store/sqlite/capture/spool-request-id.d.ts +3 -0
  117. package/dist/store/sqlite/capture/spool-request-id.js +12 -0
  118. package/dist/store/sqlite/connectors/quarantine.js +13 -4
  119. package/dist/store/sqlite/consolidation/consolidation-summaries.d.ts +4 -5
  120. package/dist/store/sqlite/consolidation/consolidation-summaries.js +46 -26
  121. package/dist/store/sqlite/consolidation/consolidation.js +7 -0
  122. package/dist/store/sqlite/consolidation/empty-extractions.d.ts +14 -0
  123. package/dist/store/sqlite/consolidation/empty-extractions.js +57 -0
  124. package/dist/store/sqlite/consolidation/rebuild-holds.d.ts +21 -0
  125. package/dist/store/sqlite/consolidation/rebuild-holds.js +68 -0
  126. package/dist/store/sqlite/consolidation/sleep-lock.d.ts +4 -0
  127. package/dist/store/sqlite/consolidation/sleep-lock.js +69 -0
  128. package/dist/store/sqlite/entries/entry-filter.js +3 -3
  129. package/dist/store/sqlite/entries/entry-reads.js +2 -1
  130. package/dist/store/sqlite/entries/entry-row.js +5 -1
  131. package/dist/store/sqlite/entries/entry-writes.js +9 -15
  132. package/dist/store/sqlite/entries/raw-archive.d.ts +2 -1
  133. package/dist/store/sqlite/entries/raw-archive.js +12 -3
  134. package/dist/store/sqlite/maintenance/agent-import-steps.js +7 -2
  135. package/dist/store/sqlite/maintenance/project-tags.js +4 -3
  136. package/dist/store/sqlite/recall/candidates.js +12 -11
  137. package/dist/store/sqlite/recall/search-rows.d.ts +3 -0
  138. package/dist/store/sqlite/recall/search-rows.js +26 -9
  139. package/dist/store/sqlite/records/object-rows.d.ts +2 -0
  140. package/dist/store/sqlite/records/object-rows.js +5 -1
  141. package/dist/store/sqlite/records/object-writes.d.ts +3 -0
  142. package/dist/store/sqlite/records/object-writes.js +43 -1
  143. package/dist/store/sqlite/reports/dashboard-lists.js +3 -2
  144. package/dist/store/sqlite/reports/dashboard-reads.js +5 -3
  145. package/dist/store/sqlite/reports/dashboard-sql.d.ts +0 -3
  146. package/dist/store/sqlite/reports/dashboard-sql.js +2 -9
  147. package/dist/store/sqlite/reports/reports.js +2 -1
  148. package/dist/store/sqlite/review/conflicts.js +10 -5
  149. package/dist/store/sqlite/runtime/rule-sql.d.ts +1 -0
  150. package/dist/store/sqlite/runtime/rule-sql.js +4 -1
  151. package/dist/store/sqlite/runtime/status-transitions.d.ts +98 -0
  152. package/dist/store/sqlite/runtime/status-transitions.js +105 -0
  153. package/dist/store/sqlite/sessions/goals.js +11 -12
  154. package/dist/store/sqlite/sessions/sessions.d.ts +1 -1
  155. package/dist/store/sqlite/sessions/sessions.js +20 -8
  156. package/dist/surface/capture/session-worker.d.ts +3 -2
  157. package/dist/surface/capture/session-worker.js +36 -10
  158. package/dist/surface/cli/api-context.js +2 -1
  159. package/dist/surface/cli/client.d.ts +6 -5
  160. package/dist/surface/cli/client.js +50 -19
  161. package/dist/surface/cli/conflicts.js +12 -1
  162. package/dist/surface/cli/continuity.js +4 -0
  163. package/dist/surface/cli/decisions.js +16 -17
  164. package/dist/surface/cli/hooks/session-end-worker.js +3 -2
  165. package/dist/surface/cli/install-steps.js +99 -13
  166. package/dist/surface/cli/memory-writes.js +3 -2
  167. package/dist/surface/cli/print.js +2 -1
  168. package/dist/surface/cli/sleep-render.js +7 -0
  169. package/dist/surface/cli/sleep.d.ts +3 -2
  170. package/dist/surface/cli/sleep.js +10 -4
  171. package/dist/surface/cli/transfer.js +6 -3
  172. package/dist/surface/cli/verbs/objects.js +2 -2
  173. package/dist/surface/dashboard/dashboard-actions.js +1 -1
  174. package/dist/surface/hooks-install/codex-wrapper.js +52 -11
  175. package/dist/surface/hooks-install/rules/hook-blocks.js +9 -2
  176. package/dist/surface/mcp/admin-tools.js +5 -2
  177. package/dist/surface/render/sleep-events.js +2 -0
  178. package/dist/surface/server/routes/decisions.js +2 -2
  179. package/dist/surface/server/sleep-child.js +5 -4
  180. package/dist/surface/server/sleep-offload.js +36 -25
  181. package/dist-ui/assets/index-Cst0ubfs.js +1 -0
  182. package/dist-ui/index.html +1 -1
  183. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  184. package/extensions/openclaw-plugin/package.json +1 -1
  185. package/openclaw.plugin.json +1 -1
  186. package/package.json +1 -1
  187. package/dist-ui/assets/index-Q9SY-1Bj.js +0 -1
@@ -2,10 +2,13 @@ import { DAG_LEVEL, EXTRACTED_TAG, Layer } from '../../base/memory/entry.js';
2
2
  import { errorFields, errorMessage, log } from '../../base/util/log.js';
3
3
  import { keptAsWritten } from './run.js';
4
4
  import { buildDag, buildEntityProfiles, rebuildDirtySummaries } from './dag.js';
5
- import { extractFacts, storeExtractedFacts } from '../learn/extract.js';
6
- import { READ_PAGE_ROWS } from '../../base/util/pages.js';
7
- import { isReusable } from '../../base/memory/memory-quality.js';
5
+ import { emptyMarkStands } from './retry-holds.js';
6
+ import { extractFactsOutcome, storeExtractedFacts } from '../learn/extract.js';
7
+ import { READ_PAGE_ROWS, readAll } from '../../base/util/pages.js';
8
+ import { blockHash } from '../../base/util/token-text.js';
9
+ import { certainDefect, isReusable } from '../../base/memory/memory-quality.js';
8
10
  import { isPermanentFailure, llmFailureFields, } from '../../base/ports/llm.js';
11
+ import { isSuperseded } from '../../base/memory/liveness.js';
9
12
  // Bounds the LLM calls one sleep spends on extraction.
10
13
  const EXTRACTION_BATCH_LIMIT = 20;
11
14
  // The ceiling keeps a misconfigured env from burning unbounded LLM cost.
@@ -28,30 +31,63 @@ function sleepLlm(run, llm) {
28
31
  };
29
32
  return { llmError, llmOpts: { llm, model: config.extraction.model } };
30
33
  }
34
+ /** Every empty extraction mark the store holds, by source id; none when the store keeps none or
35
+ * the read fails, since a failed read only costs the sleep the calls the marks would have saved. */
36
+ async function readEmptyMarks({ store }) {
37
+ const read = store.consolidation.emptyExtractions?.bind(store.consolidation);
38
+ const marks = new Map();
39
+ if (read === undefined)
40
+ return marks;
41
+ try {
42
+ const pages = await readAll(async (after) => {
43
+ const page = await read({ limit: READ_PAGE_ROWS, after });
44
+ return { entries: page.marks, next: page.next };
45
+ });
46
+ for (const { id, digest, at } of pages)
47
+ marks.set(id, { digest, at });
48
+ }
49
+ catch (err) {
50
+ log.warn('consolidate extraction could not read empty marks', errorFields(err));
51
+ marks.clear();
52
+ }
53
+ return marks;
54
+ }
55
+ /** Whether sleep could extract facts from `e`: a live episodic memory the model has not been asked
56
+ * about and nothing keeps as written. */
57
+ function extractable(e, extractedFromIds) {
58
+ return (e.layer === Layer.Episodic &&
59
+ !isSuperseded(e) &&
60
+ !extractedFromIds.has(e.id) &&
61
+ !keptAsWritten(e) &&
62
+ e.traceOutcome === null &&
63
+ e.source !== 'auto-promote' &&
64
+ isReusable(e));
65
+ }
31
66
  /** Counts the survivors extraction could read and returns the first `limit` of them, holding no
32
- * others. */
67
+ * others. A source with a standing empty mark is counted in `extractionHeld` instead, and only
68
+ * when a model may run (`limit` above 0), so a sleep with no model reads no marks. */
33
69
  async function extractionBatch(run, limit) {
34
70
  const extractedFromIds = new Set();
35
71
  for await (const e of run.survivors.rows()) {
36
72
  if (e.extractedFrom)
37
73
  extractedFromIds.add(e.extractedFrom);
38
74
  }
75
+ const marks = limit > 0 ? await readEmptyMarks(run) : new Map();
39
76
  const batch = [];
40
77
  let candidates = 0;
78
+ let held = 0;
41
79
  for await (const e of run.survivors.rows()) {
42
- const candidate = e.layer === Layer.Episodic &&
43
- !e.supersededBy &&
44
- !extractedFromIds.has(e.id) &&
45
- !keptAsWritten(e) &&
46
- e.traceOutcome === null &&
47
- e.source !== 'auto-promote' &&
48
- isReusable(e);
49
- if (!candidate)
80
+ if (!extractable(e, extractedFromIds))
81
+ continue;
82
+ if (emptyMarkStands(marks.get(e.id), e.content, run.now)) {
83
+ held++;
50
84
  continue;
85
+ }
51
86
  if (candidates++ < limit)
52
87
  batch.push(e);
53
88
  }
54
89
  run.result.extractionCandidates = candidates;
90
+ run.result.extractionHeld = held;
55
91
  return batch;
56
92
  }
57
93
  export async function llmPasses(run, model) {
@@ -97,6 +133,37 @@ function keepFacts(run, source, facts) {
97
133
  now: run.now,
98
134
  });
99
135
  }
136
+ /** Whether nothing was stored only because rejected values refused every fact: none has a quality
137
+ * defect, which would be bad model output worth a retry. */
138
+ function refusedByRejectedValue(facts) {
139
+ return facts.length > 0 && facts.every((fact) => certainDefect(fact.content) === null);
140
+ }
141
+ /** A stored child clears the source's mark. An explicit empty array, or facts that all passed the
142
+ * quality check and were all refused by a rejected value, marks it; any other reply leaves it. */
143
+ function noteOutcome(run, note, source, out, storedCount) {
144
+ if (storedCount > 0) {
145
+ note.clear.push(source.id);
146
+ }
147
+ else if (out.emptyArray || refusedByRejectedValue(out.facts)) {
148
+ note.mark.push({
149
+ id: source.id,
150
+ digest: blockHash(source.content),
151
+ at: run.now.toISOString(),
152
+ });
153
+ }
154
+ }
155
+ /** Hands the store the pass's marks in one call; a failure is logged and the sources are offered
156
+ * again next sleep. */
157
+ async function saveEmptyNote(run, note) {
158
+ if (note.mark.length === 0 && note.clear.length === 0)
159
+ return;
160
+ try {
161
+ await run.store.consolidation.noteEmptyExtractions?.(note);
162
+ }
163
+ catch (err) {
164
+ log.warn('consolidate extraction could not note empty results', errorFields(err));
165
+ }
166
+ }
100
167
  /**
101
168
  * Extracts facts from each candidate, up to four model calls at once; every write, count and report
102
169
  * then runs in input order. True when it stopped on a failure every later call would repeat (bad
@@ -107,6 +174,7 @@ async function extractionPass(run, llm, batch) {
107
174
  const stop = { at: Number.POSITIVE_INFINITY };
108
175
  const results = await pooled(batch.length, EXTRACTION_CONCURRENCY, () => stop.at === Number.POSITIVE_INFINITY, (index) => extractOne(llm, batch[index].content, index, stop));
109
176
  let extractedCount = 0;
177
+ const note = { mark: [], clear: [] };
110
178
  for (const [index, candidate] of batch.entries()) {
111
179
  const out = results[index];
112
180
  if (out === undefined)
@@ -116,10 +184,10 @@ async function extractionPass(run, llm, batch) {
116
184
  try {
117
185
  if (out.threw !== undefined)
118
186
  throw out.threw;
119
- if (out.facts.length > 0) {
120
- const stored = await keepFacts(run, candidate, out.facts);
121
- extractedCount += stored.length;
122
- }
187
+ const stored = out.facts.length > 0 ? await keepFacts(run, candidate, out.facts) : [];
188
+ extractedCount += stored.length;
189
+ if (out.errors.length === 0)
190
+ noteOutcome(run, note, candidate, out, stored.length);
123
191
  }
124
192
  catch (err) {
125
193
  report(errorMessage(err));
@@ -131,17 +199,20 @@ async function extractionPass(run, llm, batch) {
131
199
  }
132
200
  }
133
201
  run.result.extracted = extractedCount;
202
+ await saveEmptyNote(run, note);
134
203
  return stop.at !== Number.POSITIVE_INFINITY;
135
204
  }
136
205
  async function extractOne(llm, content, index, stop) {
137
- const out = { facts: [], errors: [] };
206
+ const out = { facts: [], emptyArray: false, errors: [] };
138
207
  const onError = (msg, failure) => {
139
208
  out.errors.push([msg, failure]);
140
209
  if (failure && isPermanentFailure(failure))
141
210
  stop.at = Math.min(stop.at, index);
142
211
  };
143
212
  try {
144
- out.facts = await extractFacts(content, { ...llm.llmOpts, onError });
213
+ const outcome = await extractFactsOutcome(content, { ...llm.llmOpts, onError });
214
+ out.facts = outcome.facts;
215
+ out.emptyArray = outcome.emptyArray;
145
216
  }
146
217
  catch (err) {
147
218
  out.threw = err;
@@ -162,7 +233,7 @@ async function dagBuildPass(run, { llmError, llmOpts }) {
162
233
  for await (const e of run.survivors.rows()) {
163
234
  if (!(e.tags.includes(EXTRACTED_TAG) &&
164
235
  e.dagLevel === DAG_LEVEL.extracted &&
165
- !e.supersededBy &&
236
+ !isSuperseded(e) &&
166
237
  isReusable(e))) {
167
238
  continue;
168
239
  }
@@ -213,16 +284,19 @@ async function dagRebuildPass(run, { llmError, llmOpts }) {
213
284
  result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
214
285
  result.summariesRebuildRefused = rebuildResult.refused;
215
286
  result.summariesRebuildCapped = rebuildResult.capped;
287
+ result.summariesRebuildHeld = rebuildResult.held;
216
288
  if (rebuildResult.rebuilt > 0 ||
217
289
  rebuildResult.zeroChildSkipped > 0 ||
218
290
  rebuildResult.failed > 0 ||
219
- rebuildResult.refused > 0) {
291
+ rebuildResult.refused > 0 ||
292
+ rebuildResult.held > 0) {
220
293
  result.events.push({
221
294
  kind: 'dagRebuild',
222
295
  rebuilt: rebuildResult.rebuilt,
223
296
  refused: rebuildResult.refused,
224
297
  zeroChildSkipped: rebuildResult.zeroChildSkipped,
225
298
  failed: rebuildResult.failed,
299
+ held: rebuildResult.held,
226
300
  cappedAt: rebuildResult.capped ? cap : null,
227
301
  });
228
302
  }
@@ -9,6 +9,7 @@ import { rejectionDigest } from '../../base/records/rejection-digest.js';
9
9
  import { derivationScope, derivationPartitionKey } from '../../base/scope/recall-scope.js';
10
10
  import { keptAsWritten } from './run.js';
11
11
  import { isReusable } from '../../base/memory/memory-quality.js';
12
+ import { isSuperseded } from '../../base/memory/liveness.js';
12
13
  const MERGE_OVERLAP_THRESHOLD = 0.35; // Jaccard similarity for "related"
13
14
  const MERGE_MIN_CLUSTER = 2; // minimum cluster size to merge
14
15
  // with MERGE_MAX_CHARS, keeps a merged row near 500 tokens, a third of the 1,500-token context
@@ -31,7 +32,7 @@ export async function retireHeldTexts(run) {
31
32
  // page past.
32
33
  const held = [];
33
34
  for await (const [index, row] of survivors.indexed()) {
34
- if (row.source === 'consolidation' && !row.supersededBy && run.retirable(row)) {
35
+ if (row.source === 'consolidation' && !isSuperseded(row) && run.retirable(row)) {
35
36
  held.push([index, row.id]);
36
37
  }
37
38
  }
@@ -70,11 +71,11 @@ function retireHeldText({ run, index, row, sources, rejected }) {
70
71
  }
71
72
  /** The texts the retirement check asks about, the row's own and its live sources'. */
72
73
  function askedTexts(row, sources) {
73
- if (row.source !== 'consolidation' || row.supersededBy)
74
+ if (row.source !== 'consolidation' || isSuperseded(row))
74
75
  return [];
75
76
  const live = row.parents.flatMap((id) => {
76
77
  const source = sources.get(id);
77
- return source && !source.supersededBy ? [source.content] : [];
78
+ return source && !isSuperseded(source) ? [source.content] : [];
78
79
  });
79
80
  return [...new Set([...live, ...heldTexts(row)])];
80
81
  }
@@ -106,7 +107,7 @@ async function partitionMergeCandidates(run) {
106
107
  const buckets = new Map();
107
108
  for await (const e of run.survivors.rows()) {
108
109
  const candidate = e.layer === Layer.Episodic &&
109
- !e.supersededBy &&
110
+ !isSuperseded(e) &&
110
111
  !keptAsWritten(e) &&
111
112
  !alreadyMergedIds.has(e.id) &&
112
113
  !e.pinned && // a pin merged with a look-alike would read as one of two values
@@ -134,8 +135,8 @@ async function partitionMergeCandidates(run) {
134
135
  /** Returns how many clusters were skipped because their merged text matches a rejected value. */
135
136
  export async function mergePass(run) {
136
137
  const used = new Set();
137
- // The rejection guard reuses the run's tombstone checks; a dry-run never reaches
138
- // batchWriteAndDelete's guard bypass, so it has nothing to protect there.
138
+ // The rejection guard reuses the run's tombstone checks; a dry run reads them too, so it counts a
139
+ // refused cluster as refused though it never reaches batchWriteAndDelete's guard bypass.
139
140
  let mergesSkippedRejected = 0;
140
141
  for (const bucket of (await partitionMergeCandidates(run)).values()) {
141
142
  const { ids, lengths } = bucket;
@@ -178,9 +179,9 @@ async function mergeCluster({ run, partition, cluster, related, used, }) {
178
179
  const mergedContent = mergeContents(cluster);
179
180
  // Build the semantic entry FIRST (createMemory is cheap and pure) so the tombstone check below
180
181
  // runs under the partition's tenant, the one every cluster member shares and the row will land
181
- // in.
182
- const semantic = dryRun ? null : semanticSummary(mergedContent, cluster, partition, run);
183
- if (semantic && (await mergeRejected({ run, semantic, cluster, related, used })))
182
+ // in. A dry run builds it too, so it refuses the cluster a real run would.
183
+ const semantic = semanticSummary(mergedContent, cluster, partition, run);
184
+ if (await mergeRejected({ run, semantic, cluster, related, used }))
184
185
  return false;
185
186
  for (const e of cluster)
186
187
  used.add(e.id);
@@ -190,7 +191,7 @@ async function mergeCluster({ run, partition, cluster, related, used, }) {
190
191
  count: cluster.length,
191
192
  preview: mergedContent.slice(0, MERGE_DETAIL_PREVIEW_CHARS),
192
193
  });
193
- if (!dryRun && semantic) {
194
+ if (!dryRun) {
194
195
  run.pendingWrites.push(semantic);
195
196
  run.units.push([semantic.id, ...cluster.map((e) => e.id)]);
196
197
  result.semanticCreated++;
@@ -0,0 +1,14 @@
1
+ /** Failed rebuilds in a row after which a dirty summary is held back from the model. */
2
+ export declare const REBUILD_FAILURES_BEFORE_HOLD = 3;
3
+ /** The ISO time a summary with `failures` failed rebuilds in a row is next sent to the model: one
4
+ * day after the third, doubling with each further one up to seven days; null below the third. */
5
+ export declare function rebuildRetryAfter(failures: number, now: Date): string | null;
6
+ /** Days an empty extraction mark keeps its source out of the batch. */
7
+ export declare const EMPTY_EXTRACTION_TTL_DAYS = 30;
8
+ /** Whether `mark` still stands for `content`: its digest is the text's and it is under 30 days old.
9
+ * A mark whose time does not parse stands for nothing. */
10
+ export declare function emptyMarkStands(mark: {
11
+ readonly digest: string;
12
+ readonly at: string;
13
+ } | undefined, content: string, now: Date): boolean;
14
+ //# sourceMappingURL=retry-holds.d.ts.map
@@ -0,0 +1,26 @@
1
+ // When sleep stops sending work to the model that keeps failing; pure, the store keeps only the numbers.
2
+ import { DAY_MS } from '../../base/util/time.js';
3
+ import { blockHash } from '../../base/util/token-text.js';
4
+ /** Failed rebuilds in a row after which a dirty summary is held back from the model. */
5
+ export const REBUILD_FAILURES_BEFORE_HOLD = 3;
6
+ const REBUILD_HOLD_BASE_DAYS = 1;
7
+ const REBUILD_HOLD_MAX_DAYS = 7;
8
+ /** The ISO time a summary with `failures` failed rebuilds in a row is next sent to the model: one
9
+ * day after the third, doubling with each further one up to seven days; null below the third. */
10
+ export function rebuildRetryAfter(failures, now) {
11
+ if (failures < REBUILD_FAILURES_BEFORE_HOLD)
12
+ return null;
13
+ const doublings = failures - REBUILD_FAILURES_BEFORE_HOLD;
14
+ const days = Math.min(REBUILD_HOLD_BASE_DAYS * 2 ** doublings, REBUILD_HOLD_MAX_DAYS);
15
+ return new Date(now.getTime() + days * DAY_MS).toISOString();
16
+ }
17
+ /** Days an empty extraction mark keeps its source out of the batch. */
18
+ export const EMPTY_EXTRACTION_TTL_DAYS = 30;
19
+ /** Whether `mark` still stands for `content`: its digest is the text's and it is under 30 days old.
20
+ * A mark whose time does not parse stands for nothing. */
21
+ export function emptyMarkStands(mark, content, now) {
22
+ if (mark === undefined || mark.digest !== blockHash(content))
23
+ return false;
24
+ return now.getTime() - Date.parse(mark.at) < EMPTY_EXTRACTION_TTL_DAYS * DAY_MS;
25
+ }
26
+ //# sourceMappingURL=retry-holds.js.map
@@ -28,7 +28,12 @@ export interface ConsolidationResult {
28
28
  promotedTraces: number;
29
29
  /** Sessions skipped because their events span two derivation scopes. */
30
30
  tracesSkippedMixedScope: number;
31
+ /** Episodic memories extraction could read. A dry run counts every one, marked sources included;
32
+ * a sleep that may call a model leaves out those held by `extractionHeld`. */
31
33
  extractionCandidates: number;
34
+ /** Sources left out because extraction found nothing in their current text less than 30 days
35
+ * ago. Always 0 when no model may run, as no mark is read then. */
36
+ extractionHeld: number;
32
37
  extracted: number;
33
38
  dagCandidateClusters: number;
34
39
  dagSummariesCreated: number;
@@ -37,6 +42,9 @@ export interface ConsolidationResult {
37
42
  summariesZeroChildSkipped: number;
38
43
  summariesRebuildRefused: number;
39
44
  summariesRebuildCapped: boolean;
45
+ /** Dirty summaries left out because their rebuild failed three times in a row and the hold has
46
+ * not ended; they stay dirty and are not in `summariesRebuildFailed`. */
47
+ summariesRebuildHeld: number;
40
48
  entityProfilesCreated: number;
41
49
  dryRun: boolean;
42
50
  /** What the run did, in order; the CLI and /v1/sleep render each event as a line. */
@@ -16,6 +16,7 @@ export function newConsolidationResult(dryRun) {
16
16
  promotedTraces: 0,
17
17
  tracesSkippedMixedScope: 0,
18
18
  extractionCandidates: 0,
19
+ extractionHeld: 0,
19
20
  extracted: 0,
20
21
  dagCandidateClusters: 0,
21
22
  dagSummariesCreated: 0,
@@ -24,6 +25,7 @@ export function newConsolidationResult(dryRun) {
24
25
  summariesZeroChildSkipped: 0,
25
26
  summariesRebuildRefused: 0,
26
27
  summariesRebuildCapped: false,
28
+ summariesRebuildHeld: 0,
27
29
  entityProfilesCreated: 0,
28
30
  dryRun,
29
31
  events: [],
@@ -2,10 +2,9 @@ import type { StoreWith } from '../../base/ports/store.js';
2
2
  import type { RejectedValueRow } from '../../base/ports/store/rejections.js';
3
3
  import type { AppendAuditOptions } from '../../base/ports/store/types.js';
4
4
  export interface TombstoneChecks {
5
- /** The tenant's tombstone for `digest`; null when there is none, and always under dryRun. */
5
+ /** The tenant's tombstone for `digest`; null when there is none. */
6
6
  find(tenantId: string, digest: string): Promise<RejectedValueRow | null>;
7
- /** The digests among `digests` the tenant holds a tombstone for, in one store call; none under
8
- * dryRun, as `find`. */
7
+ /** The digests among `digests` the tenant holds a tombstone for, in one store call. */
9
8
  rejectedAmong(tenantId: string, digests: readonly string[]): Promise<Set<string>>;
10
9
  /** One audit row; nothing under dryRun. */
11
10
  audit(event: AppendAuditOptions): Promise<void>;
@@ -13,6 +12,6 @@ export interface TombstoneChecks {
13
12
  * stands without its row. */
14
13
  auditOrCount(event: AppendAuditOptions): Promise<void>;
15
14
  }
16
- /** The checks of one sleep on `store`; under dryRun no call reaches the store. */
15
+ /** The checks of one sleep on `store`; under dryRun the lookups read and no audit row is written. */
17
16
  export declare function tombstoneChecks(store: StoreWith<'consolidation'>, dryRun: boolean): TombstoneChecks;
18
17
  //# sourceMappingURL=tombstones.d.ts.map
@@ -1,14 +1,14 @@
1
1
  import { reportAuditWriteFailure } from '../../base/records/audit-report.js';
2
- /** The checks of one sleep on `store`; under dryRun no call reaches the store. */
2
+ /** The checks of one sleep on `store`; under dryRun the lookups read and no audit row is written. */
3
3
  export function tombstoneChecks(store, dryRun) {
4
4
  const audit = async (event) => {
5
5
  if (!dryRun)
6
6
  await store.appendAuditEvents([event]);
7
7
  };
8
8
  return {
9
- find: async (tenantId, digest) => dryRun ? null : ((await store.consolidation.rejected(tenantId, [digest]))[0] ?? null),
9
+ find: async (tenantId, digest) => (await store.consolidation.rejected(tenantId, [digest]))[0] ?? null,
10
10
  async rejectedAmong(tenantId, digests) {
11
- if (dryRun || digests.length === 0)
11
+ if (digests.length === 0)
12
12
  return new Set();
13
13
  return new Set((await store.consolidation.rejected(tenantId, digests)).map((row) => row.digest));
14
14
  },
@@ -2,6 +2,7 @@ import { DEFAULT_MIN_RESULTS, DEFAULT_RECALL_BUDGET, } from '../../base/rank/sea
2
2
  import { estimateTokens } from '../../base/util/token-text.js';
3
3
  import { compareEntryIdentity, compareScoresDesc } from '../../base/rank/compare.js';
4
4
  import { passesCliRecallScopeFilter, passesScopeFilterForRecall, } from '../../base/scope/recall-scope.js';
5
+ import { isSuperseded } from '../../base/memory/liveness.js';
5
6
  /** Hard cap on `--hops` (a higher value just walks more of a finite graph; this bounds worst-case work and keeps the
6
7
  * flag honest). */
7
8
  export const MAX_HOPS = 3;
@@ -82,14 +83,14 @@ function passesRecallFilters(mem, via, successorValidFrom, opts) {
82
83
  if (asOfDate) {
83
84
  if (new Date(mem.validFrom) > asOfDate)
84
85
  return false; // not yet valid at asOf
85
- if (mem.supersededBy) {
86
+ if (isSuperseded(mem)) {
86
87
  const succVf = successorValidFrom.get(mem.supersededBy);
87
88
  // Visible only while its successor was NOT yet valid at asOf (matches recall).
88
89
  if (succVf && new Date(succVf) <= asOfDate)
89
90
  return false;
90
91
  }
91
92
  }
92
- else if (!includeSuperseded && (mem.supersededBy || isSupersededEndpoint)) {
93
+ else if (!includeSuperseded && (isSuperseded(mem) || isSupersededEndpoint)) {
93
94
  return false; // default recall drops superseded
94
95
  }
95
96
  return recallScope.additive
@@ -4,6 +4,7 @@ import { GitReadError } from './churn-git.js';
4
4
  import { READ_PAGE_ROWS, readPages } from '../../base/util/pages.js';
5
5
  import { previewHeadline } from './invalidation.js';
6
6
  import { extractChurnRefs, loadChurnGitView, churnEvidence, } from './churn-evidence.js';
7
+ import { isSuperseded } from '../../base/memory/liveness.js';
7
8
  function emptyChurnResult(dryRun, skippedPinned = [], error) {
8
9
  const result = {
9
10
  checked: 0,
@@ -25,7 +26,7 @@ function collectChurnCandidates(entries, opts) {
25
26
  (entry.originProject !== opts.projectName && entry.originProject !== opts.legacyName)) {
26
27
  continue;
27
28
  }
28
- if (entry.supersededBy)
29
+ if (isSuperseded(entry))
29
30
  continue;
30
31
  if (entry.kind === 'raw' || entry.kind === 'archived')
31
32
  continue;
@@ -18,7 +18,17 @@ export interface ExtractOptions {
18
18
  /** The request's abort signal; the model call stops when it fires. */
19
19
  signal?: AbortSignal;
20
20
  }
21
+ /** What one extraction call produced. */
22
+ export interface ExtractOutcome {
23
+ facts: ExtractedFact[];
24
+ /** True only when the reply parsed as a JSON array with no element: the model found nothing to
25
+ * keep. An empty or unparseable reply, a non-array and an array of unusable items are false. */
26
+ emptyArray: boolean;
27
+ }
28
+ /** The facts the model extracts from `text`; none when the call or the parse fails. */
21
29
  export declare function extractFacts(text: string, opts: ExtractOptions): Promise<ExtractedFact[]>;
30
+ /** The facts the model extracts from `text`, and whether it answered an empty array. */
31
+ export declare function extractFactsOutcome(text: string, opts: ExtractOptions): Promise<ExtractOutcome>;
22
32
  export interface StoreExtractedFactsOptions {
23
33
  readonly source: MemoryEntry;
24
34
  readonly facts: ExtractedFact[];
@@ -23,7 +23,12 @@ Input text:
23
23
  `;
24
24
  // Output budget for one extraction reply.
25
25
  const EXTRACTION_MAX_TOKENS = 1200;
26
+ /** The facts the model extracts from `text`; none when the call or the parse fails. */
26
27
  export async function extractFacts(text, opts) {
28
+ return (await extractFactsOutcome(text, opts)).facts;
29
+ }
30
+ /** The facts the model extracts from `text`, and whether it answered an empty array. */
31
+ export async function extractFactsOutcome(text, opts) {
27
32
  const reply = await opts.llm.send({
28
33
  model: opts.model,
29
34
  maxTokens: EXTRACTION_MAX_TOKENS,
@@ -36,16 +41,16 @@ export async function extractFacts(text, opts) {
36
41
  opts.onError(msg, reply.failure);
37
42
  else
38
43
  log.warn(`fact extraction failed: ${msg}`, llmFailureFields(reply.failure));
39
- return [];
44
+ return { facts: [], emptyArray: false };
40
45
  }
41
46
  try {
42
47
  const raw = reply.text;
43
48
  if (!raw)
44
- return [];
49
+ return { facts: [], emptyArray: false };
45
50
  const parsed = JSON.parse(raw);
46
51
  if (!Array.isArray(parsed))
47
- return [];
48
- return parseExtractedFacts(parsed);
52
+ return { facts: [], emptyArray: false };
53
+ return { facts: parseExtractedFacts(parsed), emptyArray: parsed.length === 0 };
49
54
  }
50
55
  catch (err) {
51
56
  const msg = `unparseable response: ${errorMessage(err)}`;
@@ -53,7 +58,7 @@ export async function extractFacts(text, opts) {
53
58
  opts.onError(msg);
54
59
  else
55
60
  log.warn(`fact extraction failed: ${msg}`, errorFields(err));
56
- return [];
61
+ return { facts: [], emptyArray: false };
57
62
  }
58
63
  }
59
64
  function parseExtractedFacts(parsed) {
@@ -24,7 +24,7 @@ export interface InvalidationOptions {
24
24
  /** Extract what was replaced/removed from a commit message.
25
25
  * Returns null if the commit isn't a breaking/migration change. */
26
26
  export declare function extractInvalidationTarget(message: string): InvalidationTarget | null;
27
- /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days,
27
+ /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days once,
28
28
  * confidence 'stale', add 'invalidated' tag; pinned are skipped). Tag matching is EXACT: the
29
29
  * FULL pattern must equal a tag, so a pattern merely containing a common tag word cannot
30
30
  * mass-weaken every memory with that tag. */
@@ -1,5 +1,6 @@
1
1
  import { INVALIDATION_STOPWORDS } from '../../base/util/stopwords.js';
2
2
  import { CJK_LETTERS, hasCjk, splitCjkRuns } from '../../base/util/tokenize.js';
3
+ import { INVALIDATED_TAG, markStale } from '../../base/memory/stale-mark.js';
3
4
  import { READ_PAGE_ROWS } from '../../base/util/pages.js';
4
5
  const HEADLINE_CHARS = 60;
5
6
  // The share of a row's tokens that must match the target's before the row counts as mentioning it.
@@ -46,7 +47,7 @@ export function extractInvalidationTarget(message) {
46
47
  }
47
48
  return null;
48
49
  }
49
- /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days,
50
+ /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days once,
50
51
  * confidence 'stale', add 'invalidated' tag; pinned are skipped). Tag matching is EXACT: the
51
52
  * FULL pattern must equal a tag, so a pattern merely containing a common tag word cannot
52
53
  * mass-weaken every memory with that tag. */
@@ -118,7 +119,7 @@ function referencesPattern(entry, fromTokens, exactTag) {
118
119
  const tagMatch = tagTokens.includes(exactTag);
119
120
  return tokenMatch >= MENTION_TOKEN_MATCH_MIN || tagMatch;
120
121
  }
121
- /** The match pass: weakens each unpinned match in place and returns them, writing nothing. */
122
+ /** The match pass: returns a weakened copy of each unpinned match and writes nothing. */
122
123
  function weakenMatches(entries, target, options) {
123
124
  const fromTokens = invalidationTokenize(target.from);
124
125
  const exactTag = target.from.toLowerCase().trim();
@@ -147,8 +148,7 @@ function weakenMatches(entries, target, options) {
147
148
  countTarget(result, entry);
148
149
  if (dryRun)
149
150
  continue;
150
- weakenEntry(entry);
151
- weakened.push(entry);
151
+ weakened.push(markStale(entry, INVALIDATED_TAG));
152
152
  }
153
153
  return { result, weakened };
154
154
  }
@@ -164,13 +164,6 @@ function countTarget(result, entry) {
164
164
  headline: previewHeadline(entry),
165
165
  });
166
166
  }
167
- function weakenEntry(entry) {
168
- entry.halfLifeDays = Math.max(1, Math.floor(entry.halfLifeDays / 2));
169
- entry.confidence = 'stale';
170
- if (!entry.tags.includes('invalidated')) {
171
- entry.tags.push('invalidated');
172
- }
173
- }
174
167
  const NOT_INVALIDATION_WORD = new RegExp(String.raw `[^a-z0-9\s_.${CJK_LETTERS}-]`, 'gu');
175
168
  function invalidationTokenize(text) {
176
169
  return text
@@ -3,9 +3,8 @@
3
3
  * `active` despite memory decay. The memory mirror (memory_id NULLABLE, ON DELETE SET NULL) is for
4
4
  * recall only; saveDecision commits both in one `objects` transaction.
5
5
  */
6
- import type { MemoryEntry } from '../../base/memory/entry.js';
7
6
  import type { KeysetPosition } from '../../base/util/keyset.js';
8
- import type { EntryWrites, Objects } from '../../base/ports/store.js';
7
+ import type { Objects } from '../../base/ports/store.js';
9
8
  import type { SavableDescriptor } from './descriptor.js';
10
9
  import type { Decision, DecisionStatus } from '../../base/records/object-types.js';
11
10
  export type { Decision, DecisionStatus } from '../../base/records/object-types.js';
@@ -16,6 +15,9 @@ export interface SaveDecisionOptions {
16
15
  /** Table id of an ACTIVE decision this one supersedes; the CLI resolves a `--supersedes <memory-id>` via
17
16
  * resolveActiveDecisionIdByMemory, HTTP/SDK pass the table id. */
18
17
  supersedesDecisionId?: number;
18
+ /** A memory of the tenant the save also weakens, for a `--supersedes <memory-id>` that names one
19
+ * no active decision backs; the save is refused when the tenant holds none. */
20
+ weakensMemoryId?: string;
19
21
  /** Extra memory tags merged after ['decision'] (the CLI passes path-context tags; HTTP/SDK
20
22
  * pass none). */
21
23
  extraTags?: string[];
@@ -27,11 +29,6 @@ export interface ListDecisionsOptions {
27
29
  after?: KeysetPosition;
28
30
  }
29
31
  export declare const DECISION: SavableDescriptor<'decision', SaveDecisionOptions>;
30
- /**
31
- * Weaken the memory a new decision replaces, once the save has committed: half-life halved to a day
32
- * at least, marked stale, tagged `superseded`.
33
- */
34
- export declare function weakenSupersededMemory(writes: EntryWrites, entry: MemoryEntry, actor?: string): Promise<void>;
35
32
  export declare function loadDecisions(objects: Objects, tenantId: string, opts?: ListDecisionsOptions): Promise<Decision[]>;
36
33
  /** Resolve a `--supersedes <memory-id>` to the table id of the ACTIVE decision backed by that memory, or null.
37
34
  * Exported so the CLI's backward-compat path is testable at the store layer. */
@@ -7,8 +7,6 @@ import { assertTenantId } from '../../base/scope/tenant-id.js';
7
7
  import { MAX_SHORT_FIELD_LEN } from '../../base/util/limits.js';
8
8
  import { requireText } from './fields.js';
9
9
  import { listObjects } from './lifecycle.js';
10
- import { CLI_ACTOR } from '../../base/scope/actor.js';
11
- const SUPERSEDED_TAG = 'superseded';
12
10
  export const VALID_DECISION_STATES = new Set([
13
11
  'active',
14
12
  'superseded',
@@ -27,22 +25,11 @@ export const DECISION = {
27
25
  content: opts.context ? `${decisionText}\n\nContext: ${opts.context}` : decisionText,
28
26
  tags: opts.extraTags ?? [],
29
27
  supersedesId: opts.supersedesDecisionId,
28
+ weakensMemoryId: opts.weakensMemoryId,
30
29
  at: now.toISOString(),
31
30
  };
32
31
  },
33
32
  };
34
- /**
35
- * Weaken the memory a new decision replaces, once the save has committed: half-life halved to a day
36
- * at least, marked stale, tagged `superseded`.
37
- */
38
- export function weakenSupersededMemory(writes, entry, actor = CLI_ACTOR) {
39
- const tags = entry.tags.includes(SUPERSEDED_TAG) ? entry.tags : [...entry.tags, SUPERSEDED_TAG];
40
- const halved = Math.max(1, Math.floor(entry.halfLifeDays / 2));
41
- return writes.writeEntry({
42
- entry: { ...entry, halfLifeDays: halved, confidence: 'stale', tags },
43
- actor,
44
- });
45
- }
46
33
  export function loadDecisions(objects, tenantId, opts = {}) {
47
34
  return listObjects(objects, DECISION, tenantId, opts);
48
35
  }
@@ -22,6 +22,8 @@ export interface ObjectDraft<K extends SavableKind> {
22
22
  readonly supersedesId: number | undefined;
23
23
  /** What changed; stored only on a versioned successor. */
24
24
  readonly changeSummary?: string;
25
+ /** A decision only: a memory the save also weakens, beside the one the replaced row mirrors. */
26
+ readonly weakensMemoryId?: string;
25
27
  /** Read once the checks pass, so one instant stamps everything the save writes. */
26
28
  readonly at: string;
27
29
  }