hippo-memory 3.3.2 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/dist/api/context/context-pools.js +2 -1
  2. package/dist/api/context/user-rule.js +2 -1
  3. package/dist/api/dashboard/detail.js +2 -1
  4. package/dist/api/dashboard/live.d.ts +0 -4
  5. package/dist/api/dashboard/live.js +0 -8
  6. package/dist/api/memories/conflicts.d.ts +2 -1
  7. package/dist/api/memories/conflicts.js +3 -8
  8. package/dist/api/memories/promote.js +6 -2
  9. package/dist/api/recall/compose.js +2 -1
  10. package/dist/api/recall/rank-stages.js +2 -1
  11. package/dist/api/recall/rows.js +2 -1
  12. package/dist/api/sleep/sleep-run.js +2 -1
  13. package/dist/api/summary-parents.js +2 -1
  14. package/dist/base/memory/ambient.js +2 -1
  15. package/dist/base/memory/entry.d.ts +4 -1
  16. package/dist/base/memory/entry.js +7 -1
  17. package/dist/base/memory/half-life-plan.js +6 -4
  18. package/dist/base/memory/keep-rules.js +2 -1
  19. package/dist/base/memory/liveness.d.ts +7 -0
  20. package/dist/base/memory/liveness.js +6 -0
  21. package/dist/base/memory/markdown.js +15 -5
  22. package/dist/base/memory/memory-audit.js +2 -1
  23. package/dist/base/memory/memory-facts.d.ts +0 -2
  24. package/dist/base/memory/memory-facts.js +0 -2
  25. package/dist/base/memory/merged-row.js +3 -2
  26. package/dist/base/memory/rule-replaces.js +2 -1
  27. package/dist/base/memory/stale-mark.d.ts +9 -0
  28. package/dist/base/memory/stale-mark.js +18 -0
  29. package/dist/base/ports/store/conflicts.d.ts +5 -1
  30. package/dist/base/ports/store/conflicts.js +2 -0
  31. package/dist/base/ports/store/consolidation.d.ts +51 -2
  32. package/dist/base/ports/store/consolidation.js +3 -0
  33. package/dist/base/ports/store/sessions.d.ts +8 -2
  34. package/dist/base/ports/store/sessions.js +6 -0
  35. package/dist/base/ports/store/store-effects.d.ts +3 -0
  36. package/dist/base/scope/live-memory.d.ts +7 -0
  37. package/dist/base/scope/live-memory.js +10 -0
  38. package/dist/base/scope/read-admission.js +2 -1
  39. package/dist/base/util/version.d.ts +1 -1
  40. package/dist/base/util/version.js +1 -1
  41. package/dist/db/continuity-tables.d.ts +6 -1
  42. package/dist/db/continuity-tables.js +27 -15
  43. package/dist/db/migrate.js +8 -4
  44. package/dist/db/migrations/index.d.ts +1 -1
  45. package/dist/db/migrations/index.js +3 -1
  46. package/dist/db/migrations/v61.js +3 -3
  47. package/dist/db/migrations/v62.d.ts +3 -0
  48. package/dist/db/migrations/v62.js +43 -0
  49. package/dist/db/required-objects.js +2 -1
  50. package/dist/db/sql-predicates.d.ts +22 -2
  51. package/dist/db/sql-predicates.js +27 -2
  52. package/dist/domain/agent-memories/apply.js +2 -1
  53. package/dist/domain/agent-memories/container-view.js +2 -1
  54. package/dist/domain/consolidate/conflicts.js +2 -1
  55. package/dist/domain/consolidate/dag-summary.d.ts +2 -0
  56. package/dist/domain/consolidate/dag.d.ts +3 -0
  57. package/dist/domain/consolidate/dag.js +53 -8
  58. package/dist/domain/consolidate/dedupe.js +2 -1
  59. package/dist/domain/consolidate/events.d.ts +2 -0
  60. package/dist/domain/consolidate/llm-passes.js +94 -20
  61. package/dist/domain/consolidate/merge.js +5 -4
  62. package/dist/domain/consolidate/retry-holds.d.ts +14 -0
  63. package/dist/domain/consolidate/retry-holds.js +26 -0
  64. package/dist/domain/consolidate/run.d.ts +8 -0
  65. package/dist/domain/consolidate/run.js +2 -0
  66. package/dist/domain/graph/recall.js +3 -2
  67. package/dist/domain/learn/churn-stale.js +2 -1
  68. package/dist/domain/learn/extract.d.ts +10 -0
  69. package/dist/domain/learn/extract.js +10 -5
  70. package/dist/domain/learn/invalidation.d.ts +1 -1
  71. package/dist/domain/learn/invalidation.js +4 -11
  72. package/dist/domain/objects/decisions.d.ts +2 -2
  73. package/dist/domain/objects/decisions.js +4 -9
  74. package/dist/domain/search/as-of.js +3 -2
  75. package/dist/store/sqlite/capture/compaction-rows.js +9 -2
  76. package/dist/store/sqlite/connectors/quarantine.js +13 -4
  77. package/dist/store/sqlite/consolidation/consolidation-summaries.d.ts +4 -5
  78. package/dist/store/sqlite/consolidation/consolidation-summaries.js +46 -26
  79. package/dist/store/sqlite/consolidation/consolidation.js +5 -0
  80. package/dist/store/sqlite/consolidation/empty-extractions.d.ts +14 -0
  81. package/dist/store/sqlite/consolidation/empty-extractions.js +57 -0
  82. package/dist/store/sqlite/consolidation/rebuild-holds.d.ts +21 -0
  83. package/dist/store/sqlite/consolidation/rebuild-holds.js +68 -0
  84. package/dist/store/sqlite/entries/entry-filter.js +2 -1
  85. package/dist/store/sqlite/entries/entry-reads.js +2 -1
  86. package/dist/store/sqlite/entries/entry-writes.js +6 -5
  87. package/dist/store/sqlite/maintenance/agent-import-steps.js +7 -2
  88. package/dist/store/sqlite/maintenance/project-tags.js +4 -3
  89. package/dist/store/sqlite/recall/candidates.js +10 -4
  90. package/dist/store/sqlite/recall/search-rows.js +4 -3
  91. package/dist/store/sqlite/reports/dashboard-lists.js +3 -2
  92. package/dist/store/sqlite/reports/dashboard-reads.js +5 -3
  93. package/dist/store/sqlite/reports/dashboard-sql.d.ts +0 -3
  94. package/dist/store/sqlite/reports/dashboard-sql.js +2 -9
  95. package/dist/store/sqlite/reports/reports.js +2 -1
  96. package/dist/store/sqlite/review/conflicts.js +7 -1
  97. package/dist/store/sqlite/runtime/rule-sql.js +2 -1
  98. package/dist/store/sqlite/runtime/status-transitions.d.ts +98 -0
  99. package/dist/store/sqlite/runtime/status-transitions.js +105 -0
  100. package/dist/store/sqlite/sessions/goals.js +11 -12
  101. package/dist/store/sqlite/sessions/sessions.d.ts +1 -1
  102. package/dist/store/sqlite/sessions/sessions.js +20 -8
  103. package/dist/surface/cli/conflicts.js +12 -1
  104. package/dist/surface/cli/continuity.js +4 -0
  105. package/dist/surface/cli/print.js +2 -1
  106. package/dist/surface/cli/verbs/objects.js +2 -2
  107. package/dist/surface/dashboard/dashboard-actions.js +1 -1
  108. package/dist/surface/render/sleep-events.js +2 -0
  109. package/dist-ui/assets/index-Cst0ubfs.js +1 -0
  110. package/dist-ui/index.html +1 -1
  111. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  112. package/extensions/openclaw-plugin/package.json +1 -1
  113. package/openclaw.plugin.json +1 -1
  114. package/package.json +1 -1
  115. package/dist-ui/assets/index-Q9SY-1Bj.js +0 -1
@@ -10,6 +10,7 @@ import { canAutoDelete } from '../../base/memory/keep-rules.js';
10
10
  import { derivationPartitionKey } from '../../base/scope/recall-scope.js';
11
11
  import { duplicateKey } from '../../base/util/same-text.js';
12
12
  import { CLI_ACTOR } from '../../base/scope/actor.js';
13
+ import { isSuperseded } from '../../base/memory/liveness.js';
13
14
  /**
14
15
  * Quantization step for strength-tie comparisons: the 0.01 epsilon applied by rounding instead of
15
16
  * an abs-diff threshold, so the tiebreak is transitive.
@@ -84,7 +85,7 @@ function partitionPairs(groups, backing) {
84
85
  .flatMap((group) => group.losers.sort(survivorOrder).map((loser) => dedupPair(group.survivor, loser)));
85
86
  }
86
87
  /** The rows duplicateCandidates selects, picked from pages of every row. */
87
- const isCurrentDistilled = (e) => (e.kind ?? 'distilled') === 'distilled' && !e.supersededBy;
88
+ const isCurrentDistilled = (e) => (e.kind ?? 'distilled') === 'distilled' && !isSuperseded(e);
88
89
  /** Groups rows fed a page at a time; between pages only each group's survivor and losers are
89
90
  * held. Map order is first appearance, so partitions keep the order the pages gave them. */
90
91
  async function partitionRows(pages) {
@@ -40,6 +40,8 @@ export type ConsolidateEvent = {
40
40
  refused: number;
41
41
  zeroChildSkipped: number;
42
42
  failed: number;
43
+ /** Dirty summaries left out because their hold had not ended. */
44
+ held: number;
43
45
  /** The rebuild cap when it stopped the pass, else null. */
44
46
  cappedAt: number | null;
45
47
  } | {
@@ -2,10 +2,13 @@ import { DAG_LEVEL, EXTRACTED_TAG, Layer } from '../../base/memory/entry.js';
2
2
  import { errorFields, errorMessage, log } from '../../base/util/log.js';
3
3
  import { keptAsWritten } from './run.js';
4
4
  import { buildDag, buildEntityProfiles, rebuildDirtySummaries } from './dag.js';
5
- import { extractFacts, storeExtractedFacts } from '../learn/extract.js';
6
- import { READ_PAGE_ROWS } from '../../base/util/pages.js';
7
- import { isReusable } from '../../base/memory/memory-quality.js';
5
+ import { emptyMarkStands } from './retry-holds.js';
6
+ import { extractFactsOutcome, storeExtractedFacts } from '../learn/extract.js';
7
+ import { READ_PAGE_ROWS, readAll } from '../../base/util/pages.js';
8
+ import { blockHash } from '../../base/util/token-text.js';
9
+ import { certainDefect, isReusable } from '../../base/memory/memory-quality.js';
8
10
  import { isPermanentFailure, llmFailureFields, } from '../../base/ports/llm.js';
11
+ import { isSuperseded } from '../../base/memory/liveness.js';
9
12
  // Bounds the LLM calls one sleep spends on extraction.
10
13
  const EXTRACTION_BATCH_LIMIT = 20;
11
14
  // The ceiling keeps a misconfigured env from burning unbounded LLM cost.
@@ -28,30 +31,63 @@ function sleepLlm(run, llm) {
28
31
  };
29
32
  return { llmError, llmOpts: { llm, model: config.extraction.model } };
30
33
  }
34
+ /** Every empty extraction mark the store holds, by source id; none when the store keeps none or
35
+ * the read fails, since a failed read only costs the sleep the calls the marks would have saved. */
36
+ async function readEmptyMarks({ store }) {
37
+ const read = store.consolidation.emptyExtractions?.bind(store.consolidation);
38
+ const marks = new Map();
39
+ if (read === undefined)
40
+ return marks;
41
+ try {
42
+ const pages = await readAll(async (after) => {
43
+ const page = await read({ limit: READ_PAGE_ROWS, after });
44
+ return { entries: page.marks, next: page.next };
45
+ });
46
+ for (const { id, digest, at } of pages)
47
+ marks.set(id, { digest, at });
48
+ }
49
+ catch (err) {
50
+ log.warn('consolidate extraction could not read empty marks', errorFields(err));
51
+ marks.clear();
52
+ }
53
+ return marks;
54
+ }
55
+ /** Whether sleep could extract facts from `e`: a live episodic memory the model has not been asked
56
+ * about and nothing keeps as written. */
57
+ function extractable(e, extractedFromIds) {
58
+ return (e.layer === Layer.Episodic &&
59
+ !isSuperseded(e) &&
60
+ !extractedFromIds.has(e.id) &&
61
+ !keptAsWritten(e) &&
62
+ e.traceOutcome === null &&
63
+ e.source !== 'auto-promote' &&
64
+ isReusable(e));
65
+ }
31
66
  /** Counts the survivors extraction could read and returns the first `limit` of them, holding no
32
- * others. */
67
+ * others. A source with a standing empty mark is counted in `extractionHeld` instead, and only
68
+ * when a model may run (`limit` above 0), so a sleep with no model reads no marks. */
33
69
  async function extractionBatch(run, limit) {
34
70
  const extractedFromIds = new Set();
35
71
  for await (const e of run.survivors.rows()) {
36
72
  if (e.extractedFrom)
37
73
  extractedFromIds.add(e.extractedFrom);
38
74
  }
75
+ const marks = limit > 0 ? await readEmptyMarks(run) : new Map();
39
76
  const batch = [];
40
77
  let candidates = 0;
78
+ let held = 0;
41
79
  for await (const e of run.survivors.rows()) {
42
- const candidate = e.layer === Layer.Episodic &&
43
- !e.supersededBy &&
44
- !extractedFromIds.has(e.id) &&
45
- !keptAsWritten(e) &&
46
- e.traceOutcome === null &&
47
- e.source !== 'auto-promote' &&
48
- isReusable(e);
49
- if (!candidate)
80
+ if (!extractable(e, extractedFromIds))
81
+ continue;
82
+ if (emptyMarkStands(marks.get(e.id), e.content, run.now)) {
83
+ held++;
50
84
  continue;
85
+ }
51
86
  if (candidates++ < limit)
52
87
  batch.push(e);
53
88
  }
54
89
  run.result.extractionCandidates = candidates;
90
+ run.result.extractionHeld = held;
55
91
  return batch;
56
92
  }
57
93
  export async function llmPasses(run, model) {
@@ -97,6 +133,37 @@ function keepFacts(run, source, facts) {
97
133
  now: run.now,
98
134
  });
99
135
  }
136
+ /** Whether nothing was stored only because rejected values refused every fact: none has a quality
137
+ * defect, which would be bad model output worth a retry. */
138
+ function refusedByRejectedValue(facts) {
139
+ return facts.length > 0 && facts.every((fact) => certainDefect(fact.content) === null);
140
+ }
141
+ /** A stored child clears the source's mark. An explicit empty array, or facts that all passed the
142
+ * quality check and were all refused by a rejected value, marks it; any other reply leaves it. */
143
+ function noteOutcome(run, note, source, out, storedCount) {
144
+ if (storedCount > 0) {
145
+ note.clear.push(source.id);
146
+ }
147
+ else if (out.emptyArray || refusedByRejectedValue(out.facts)) {
148
+ note.mark.push({
149
+ id: source.id,
150
+ digest: blockHash(source.content),
151
+ at: run.now.toISOString(),
152
+ });
153
+ }
154
+ }
155
+ /** Hands the store the pass's marks in one call; a failure is logged and the sources are offered
156
+ * again next sleep. */
157
+ async function saveEmptyNote(run, note) {
158
+ if (note.mark.length === 0 && note.clear.length === 0)
159
+ return;
160
+ try {
161
+ await run.store.consolidation.noteEmptyExtractions?.(note);
162
+ }
163
+ catch (err) {
164
+ log.warn('consolidate extraction could not note empty results', errorFields(err));
165
+ }
166
+ }
100
167
  /**
101
168
  * Extracts facts from each candidate, up to four model calls at once; every write, count and report
102
169
  * then runs in input order. True when it stopped on a failure every later call would repeat (bad
@@ -107,6 +174,7 @@ async function extractionPass(run, llm, batch) {
107
174
  const stop = { at: Number.POSITIVE_INFINITY };
108
175
  const results = await pooled(batch.length, EXTRACTION_CONCURRENCY, () => stop.at === Number.POSITIVE_INFINITY, (index) => extractOne(llm, batch[index].content, index, stop));
109
176
  let extractedCount = 0;
177
+ const note = { mark: [], clear: [] };
110
178
  for (const [index, candidate] of batch.entries()) {
111
179
  const out = results[index];
112
180
  if (out === undefined)
@@ -116,10 +184,10 @@ async function extractionPass(run, llm, batch) {
116
184
  try {
117
185
  if (out.threw !== undefined)
118
186
  throw out.threw;
119
- if (out.facts.length > 0) {
120
- const stored = await keepFacts(run, candidate, out.facts);
121
- extractedCount += stored.length;
122
- }
187
+ const stored = out.facts.length > 0 ? await keepFacts(run, candidate, out.facts) : [];
188
+ extractedCount += stored.length;
189
+ if (out.errors.length === 0)
190
+ noteOutcome(run, note, candidate, out, stored.length);
123
191
  }
124
192
  catch (err) {
125
193
  report(errorMessage(err));
@@ -131,17 +199,20 @@ async function extractionPass(run, llm, batch) {
131
199
  }
132
200
  }
133
201
  run.result.extracted = extractedCount;
202
+ await saveEmptyNote(run, note);
134
203
  return stop.at !== Number.POSITIVE_INFINITY;
135
204
  }
136
205
  async function extractOne(llm, content, index, stop) {
137
- const out = { facts: [], errors: [] };
206
+ const out = { facts: [], emptyArray: false, errors: [] };
138
207
  const onError = (msg, failure) => {
139
208
  out.errors.push([msg, failure]);
140
209
  if (failure && isPermanentFailure(failure))
141
210
  stop.at = Math.min(stop.at, index);
142
211
  };
143
212
  try {
144
- out.facts = await extractFacts(content, { ...llm.llmOpts, onError });
213
+ const outcome = await extractFactsOutcome(content, { ...llm.llmOpts, onError });
214
+ out.facts = outcome.facts;
215
+ out.emptyArray = outcome.emptyArray;
145
216
  }
146
217
  catch (err) {
147
218
  out.threw = err;
@@ -162,7 +233,7 @@ async function dagBuildPass(run, { llmError, llmOpts }) {
162
233
  for await (const e of run.survivors.rows()) {
163
234
  if (!(e.tags.includes(EXTRACTED_TAG) &&
164
235
  e.dagLevel === DAG_LEVEL.extracted &&
165
- !e.supersededBy &&
236
+ !isSuperseded(e) &&
166
237
  isReusable(e))) {
167
238
  continue;
168
239
  }
@@ -213,16 +284,19 @@ async function dagRebuildPass(run, { llmError, llmOpts }) {
213
284
  result.summariesZeroChildSkipped = rebuildResult.zeroChildSkipped;
214
285
  result.summariesRebuildRefused = rebuildResult.refused;
215
286
  result.summariesRebuildCapped = rebuildResult.capped;
287
+ result.summariesRebuildHeld = rebuildResult.held;
216
288
  if (rebuildResult.rebuilt > 0 ||
217
289
  rebuildResult.zeroChildSkipped > 0 ||
218
290
  rebuildResult.failed > 0 ||
219
- rebuildResult.refused > 0) {
291
+ rebuildResult.refused > 0 ||
292
+ rebuildResult.held > 0) {
220
293
  result.events.push({
221
294
  kind: 'dagRebuild',
222
295
  rebuilt: rebuildResult.rebuilt,
223
296
  refused: rebuildResult.refused,
224
297
  zeroChildSkipped: rebuildResult.zeroChildSkipped,
225
298
  failed: rebuildResult.failed,
299
+ held: rebuildResult.held,
226
300
  cappedAt: rebuildResult.capped ? cap : null,
227
301
  });
228
302
  }
@@ -9,6 +9,7 @@ import { rejectionDigest } from '../../base/records/rejection-digest.js';
9
9
  import { derivationScope, derivationPartitionKey } from '../../base/scope/recall-scope.js';
10
10
  import { keptAsWritten } from './run.js';
11
11
  import { isReusable } from '../../base/memory/memory-quality.js';
12
+ import { isSuperseded } from '../../base/memory/liveness.js';
12
13
  const MERGE_OVERLAP_THRESHOLD = 0.35; // Jaccard similarity for "related"
13
14
  const MERGE_MIN_CLUSTER = 2; // minimum cluster size to merge
14
15
  // with MERGE_MAX_CHARS, keeps a merged row near 500 tokens, a third of the 1,500-token context
@@ -31,7 +32,7 @@ export async function retireHeldTexts(run) {
31
32
  // page past.
32
33
  const held = [];
33
34
  for await (const [index, row] of survivors.indexed()) {
34
- if (row.source === 'consolidation' && !row.supersededBy && run.retirable(row)) {
35
+ if (row.source === 'consolidation' && !isSuperseded(row) && run.retirable(row)) {
35
36
  held.push([index, row.id]);
36
37
  }
37
38
  }
@@ -70,11 +71,11 @@ function retireHeldText({ run, index, row, sources, rejected }) {
70
71
  }
71
72
  /** The texts the retirement check asks about, the row's own and its live sources'. */
72
73
  function askedTexts(row, sources) {
73
- if (row.source !== 'consolidation' || row.supersededBy)
74
+ if (row.source !== 'consolidation' || isSuperseded(row))
74
75
  return [];
75
76
  const live = row.parents.flatMap((id) => {
76
77
  const source = sources.get(id);
77
- return source && !source.supersededBy ? [source.content] : [];
78
+ return source && !isSuperseded(source) ? [source.content] : [];
78
79
  });
79
80
  return [...new Set([...live, ...heldTexts(row)])];
80
81
  }
@@ -106,7 +107,7 @@ async function partitionMergeCandidates(run) {
106
107
  const buckets = new Map();
107
108
  for await (const e of run.survivors.rows()) {
108
109
  const candidate = e.layer === Layer.Episodic &&
109
- !e.supersededBy &&
110
+ !isSuperseded(e) &&
110
111
  !keptAsWritten(e) &&
111
112
  !alreadyMergedIds.has(e.id) &&
112
113
  !e.pinned && // a pin merged with a look-alike would read as one of two values
@@ -0,0 +1,14 @@
1
+ /** Failed rebuilds in a row after which a dirty summary is held back from the model. */
2
+ export declare const REBUILD_FAILURES_BEFORE_HOLD = 3;
3
+ /** The ISO time a summary with `failures` failed rebuilds in a row is next sent to the model: one
4
+ * day after the third, doubling with each further one up to seven days; null below the third. */
5
+ export declare function rebuildRetryAfter(failures: number, now: Date): string | null;
6
+ /** Days an empty extraction mark keeps its source out of the batch. */
7
+ export declare const EMPTY_EXTRACTION_TTL_DAYS = 30;
8
+ /** Whether `mark` still stands for `content`: its digest is the text's and it is under 30 days old.
9
+ * A mark whose time does not parse stands for nothing. */
10
+ export declare function emptyMarkStands(mark: {
11
+ readonly digest: string;
12
+ readonly at: string;
13
+ } | undefined, content: string, now: Date): boolean;
14
+ //# sourceMappingURL=retry-holds.d.ts.map
@@ -0,0 +1,26 @@
1
+ // When sleep stops sending work to the model that keeps failing; pure, the store keeps only the numbers.
2
+ import { DAY_MS } from '../../base/util/time.js';
3
+ import { blockHash } from '../../base/util/token-text.js';
4
+ /** Failed rebuilds in a row after which a dirty summary is held back from the model. */
5
+ export const REBUILD_FAILURES_BEFORE_HOLD = 3;
6
+ const REBUILD_HOLD_BASE_DAYS = 1;
7
+ const REBUILD_HOLD_MAX_DAYS = 7;
8
+ /** The ISO time a summary with `failures` failed rebuilds in a row is next sent to the model: one
9
+ * day after the third, doubling with each further one up to seven days; null below the third. */
10
+ export function rebuildRetryAfter(failures, now) {
11
+ if (failures < REBUILD_FAILURES_BEFORE_HOLD)
12
+ return null;
13
+ const doublings = failures - REBUILD_FAILURES_BEFORE_HOLD;
14
+ const days = Math.min(REBUILD_HOLD_BASE_DAYS * 2 ** doublings, REBUILD_HOLD_MAX_DAYS);
15
+ return new Date(now.getTime() + days * DAY_MS).toISOString();
16
+ }
17
+ /** Days an empty extraction mark keeps its source out of the batch. */
18
+ export const EMPTY_EXTRACTION_TTL_DAYS = 30;
19
+ /** Whether `mark` still stands for `content`: its digest is the text's and it is under 30 days old.
20
+ * A mark whose time does not parse stands for nothing. */
21
+ export function emptyMarkStands(mark, content, now) {
22
+ if (mark === undefined || mark.digest !== blockHash(content))
23
+ return false;
24
+ return now.getTime() - Date.parse(mark.at) < EMPTY_EXTRACTION_TTL_DAYS * DAY_MS;
25
+ }
26
+ //# sourceMappingURL=retry-holds.js.map
@@ -28,7 +28,12 @@ export interface ConsolidationResult {
28
28
  promotedTraces: number;
29
29
  /** Sessions skipped because their events span two derivation scopes. */
30
30
  tracesSkippedMixedScope: number;
31
+ /** Episodic memories extraction could read. A dry run counts every one, marked sources included;
32
+ * a sleep that may call a model leaves out those held by `extractionHeld`. */
31
33
  extractionCandidates: number;
34
+ /** Sources left out because extraction found nothing in their current text less than 30 days
35
+ * ago. Always 0 when no model may run, as no mark is read then. */
36
+ extractionHeld: number;
32
37
  extracted: number;
33
38
  dagCandidateClusters: number;
34
39
  dagSummariesCreated: number;
@@ -37,6 +42,9 @@ export interface ConsolidationResult {
37
42
  summariesZeroChildSkipped: number;
38
43
  summariesRebuildRefused: number;
39
44
  summariesRebuildCapped: boolean;
45
+ /** Dirty summaries left out because their rebuild failed three times in a row and the hold has
46
+ * not ended; they stay dirty and are not in `summariesRebuildFailed`. */
47
+ summariesRebuildHeld: number;
40
48
  entityProfilesCreated: number;
41
49
  dryRun: boolean;
42
50
  /** What the run did, in order; the CLI and /v1/sleep render each event as a line. */
@@ -16,6 +16,7 @@ export function newConsolidationResult(dryRun) {
16
16
  promotedTraces: 0,
17
17
  tracesSkippedMixedScope: 0,
18
18
  extractionCandidates: 0,
19
+ extractionHeld: 0,
19
20
  extracted: 0,
20
21
  dagCandidateClusters: 0,
21
22
  dagSummariesCreated: 0,
@@ -24,6 +25,7 @@ export function newConsolidationResult(dryRun) {
24
25
  summariesZeroChildSkipped: 0,
25
26
  summariesRebuildRefused: 0,
26
27
  summariesRebuildCapped: false,
28
+ summariesRebuildHeld: 0,
27
29
  entityProfilesCreated: 0,
28
30
  dryRun,
29
31
  events: [],
@@ -2,6 +2,7 @@ import { DEFAULT_MIN_RESULTS, DEFAULT_RECALL_BUDGET, } from '../../base/rank/sea
2
2
  import { estimateTokens } from '../../base/util/token-text.js';
3
3
  import { compareEntryIdentity, compareScoresDesc } from '../../base/rank/compare.js';
4
4
  import { passesCliRecallScopeFilter, passesScopeFilterForRecall, } from '../../base/scope/recall-scope.js';
5
+ import { isSuperseded } from '../../base/memory/liveness.js';
5
6
  /** Hard cap on `--hops` (a higher value just walks more of a finite graph; this bounds worst-case work and keeps the
6
7
  * flag honest). */
7
8
  export const MAX_HOPS = 3;
@@ -82,14 +83,14 @@ function passesRecallFilters(mem, via, successorValidFrom, opts) {
82
83
  if (asOfDate) {
83
84
  if (new Date(mem.validFrom) > asOfDate)
84
85
  return false; // not yet valid at asOf
85
- if (mem.supersededBy) {
86
+ if (isSuperseded(mem)) {
86
87
  const succVf = successorValidFrom.get(mem.supersededBy);
87
88
  // Visible only while its successor was NOT yet valid at asOf (matches recall).
88
89
  if (succVf && new Date(succVf) <= asOfDate)
89
90
  return false;
90
91
  }
91
92
  }
92
- else if (!includeSuperseded && (mem.supersededBy || isSupersededEndpoint)) {
93
+ else if (!includeSuperseded && (isSuperseded(mem) || isSupersededEndpoint)) {
93
94
  return false; // default recall drops superseded
94
95
  }
95
96
  return recallScope.additive
@@ -4,6 +4,7 @@ import { GitReadError } from './churn-git.js';
4
4
  import { READ_PAGE_ROWS, readPages } from '../../base/util/pages.js';
5
5
  import { previewHeadline } from './invalidation.js';
6
6
  import { extractChurnRefs, loadChurnGitView, churnEvidence, } from './churn-evidence.js';
7
+ import { isSuperseded } from '../../base/memory/liveness.js';
7
8
  function emptyChurnResult(dryRun, skippedPinned = [], error) {
8
9
  const result = {
9
10
  checked: 0,
@@ -25,7 +26,7 @@ function collectChurnCandidates(entries, opts) {
25
26
  (entry.originProject !== opts.projectName && entry.originProject !== opts.legacyName)) {
26
27
  continue;
27
28
  }
28
- if (entry.supersededBy)
29
+ if (isSuperseded(entry))
29
30
  continue;
30
31
  if (entry.kind === 'raw' || entry.kind === 'archived')
31
32
  continue;
@@ -18,7 +18,17 @@ export interface ExtractOptions {
18
18
  /** The request's abort signal; the model call stops when it fires. */
19
19
  signal?: AbortSignal;
20
20
  }
21
+ /** What one extraction call produced. */
22
+ export interface ExtractOutcome {
23
+ facts: ExtractedFact[];
24
+ /** True only when the reply parsed as a JSON array with no element: the model found nothing to
25
+ * keep. An empty or unparseable reply, a non-array and an array of unusable items are false. */
26
+ emptyArray: boolean;
27
+ }
28
+ /** The facts the model extracts from `text`; none when the call or the parse fails. */
21
29
  export declare function extractFacts(text: string, opts: ExtractOptions): Promise<ExtractedFact[]>;
30
+ /** The facts the model extracts from `text`, and whether it answered an empty array. */
31
+ export declare function extractFactsOutcome(text: string, opts: ExtractOptions): Promise<ExtractOutcome>;
22
32
  export interface StoreExtractedFactsOptions {
23
33
  readonly source: MemoryEntry;
24
34
  readonly facts: ExtractedFact[];
@@ -23,7 +23,12 @@ Input text:
23
23
  `;
24
24
  // Output budget for one extraction reply.
25
25
  const EXTRACTION_MAX_TOKENS = 1200;
26
+ /** The facts the model extracts from `text`; none when the call or the parse fails. */
26
27
  export async function extractFacts(text, opts) {
28
+ return (await extractFactsOutcome(text, opts)).facts;
29
+ }
30
+ /** The facts the model extracts from `text`, and whether it answered an empty array. */
31
+ export async function extractFactsOutcome(text, opts) {
27
32
  const reply = await opts.llm.send({
28
33
  model: opts.model,
29
34
  maxTokens: EXTRACTION_MAX_TOKENS,
@@ -36,16 +41,16 @@ export async function extractFacts(text, opts) {
36
41
  opts.onError(msg, reply.failure);
37
42
  else
38
43
  log.warn(`fact extraction failed: ${msg}`, llmFailureFields(reply.failure));
39
- return [];
44
+ return { facts: [], emptyArray: false };
40
45
  }
41
46
  try {
42
47
  const raw = reply.text;
43
48
  if (!raw)
44
- return [];
49
+ return { facts: [], emptyArray: false };
45
50
  const parsed = JSON.parse(raw);
46
51
  if (!Array.isArray(parsed))
47
- return [];
48
- return parseExtractedFacts(parsed);
52
+ return { facts: [], emptyArray: false };
53
+ return { facts: parseExtractedFacts(parsed), emptyArray: parsed.length === 0 };
49
54
  }
50
55
  catch (err) {
51
56
  const msg = `unparseable response: ${errorMessage(err)}`;
@@ -53,7 +58,7 @@ export async function extractFacts(text, opts) {
53
58
  opts.onError(msg);
54
59
  else
55
60
  log.warn(`fact extraction failed: ${msg}`, errorFields(err));
56
- return [];
61
+ return { facts: [], emptyArray: false };
57
62
  }
58
63
  }
59
64
  function parseExtractedFacts(parsed) {
@@ -24,7 +24,7 @@ export interface InvalidationOptions {
24
24
  /** Extract what was replaced/removed from a commit message.
25
25
  * Returns null if the commit isn't a breaking/migration change. */
26
26
  export declare function extractInvalidationTarget(message: string): InvalidationTarget | null;
27
- /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days,
27
+ /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days once,
28
28
  * confidence 'stale', add 'invalidated' tag; pinned are skipped). Tag matching is EXACT: the
29
29
  * FULL pattern must equal a tag, so a pattern merely containing a common tag word cannot
30
30
  * mass-weaken every memory with that tag. */
@@ -1,5 +1,6 @@
1
1
  import { INVALIDATION_STOPWORDS } from '../../base/util/stopwords.js';
2
2
  import { CJK_LETTERS, hasCjk, splitCjkRuns } from '../../base/util/tokenize.js';
3
+ import { INVALIDATED_TAG, markStale } from '../../base/memory/stale-mark.js';
3
4
  import { READ_PAGE_ROWS } from '../../base/util/pages.js';
4
5
  const HEADLINE_CHARS = 60;
5
6
  // The share of a row's tokens that must match the target's before the row counts as mentioning it.
@@ -46,7 +47,7 @@ export function extractInvalidationTarget(message) {
46
47
  }
47
48
  return null;
48
49
  }
49
- /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days,
50
+ /** Find memories that reference the invalidated pattern and weaken them (halve half_life_days once,
50
51
  * confidence 'stale', add 'invalidated' tag; pinned are skipped). Tag matching is EXACT: the
51
52
  * FULL pattern must equal a tag, so a pattern merely containing a common tag word cannot
52
53
  * mass-weaken every memory with that tag. */
@@ -118,7 +119,7 @@ function referencesPattern(entry, fromTokens, exactTag) {
118
119
  const tagMatch = tagTokens.includes(exactTag);
119
120
  return tokenMatch >= MENTION_TOKEN_MATCH_MIN || tagMatch;
120
121
  }
121
- /** The match pass: weakens each unpinned match in place and returns them, writing nothing. */
122
+ /** The match pass: returns a weakened copy of each unpinned match and writes nothing. */
122
123
  function weakenMatches(entries, target, options) {
123
124
  const fromTokens = invalidationTokenize(target.from);
124
125
  const exactTag = target.from.toLowerCase().trim();
@@ -147,8 +148,7 @@ function weakenMatches(entries, target, options) {
147
148
  countTarget(result, entry);
148
149
  if (dryRun)
149
150
  continue;
150
- weakenEntry(entry);
151
- weakened.push(entry);
151
+ weakened.push(markStale(entry, INVALIDATED_TAG));
152
152
  }
153
153
  return { result, weakened };
154
154
  }
@@ -164,13 +164,6 @@ function countTarget(result, entry) {
164
164
  headline: previewHeadline(entry),
165
165
  });
166
166
  }
167
- function weakenEntry(entry) {
168
- entry.halfLifeDays = Math.max(1, Math.floor(entry.halfLifeDays / 2));
169
- entry.confidence = 'stale';
170
- if (!entry.tags.includes('invalidated')) {
171
- entry.tags.push('invalidated');
172
- }
173
- }
174
167
  const NOT_INVALIDATION_WORD = new RegExp(String.raw `[^a-z0-9\s_.${CJK_LETTERS}-]`, 'gu');
175
168
  function invalidationTokenize(text) {
176
169
  return text
@@ -28,8 +28,8 @@ export interface ListDecisionsOptions {
28
28
  }
29
29
  export declare const DECISION: SavableDescriptor<'decision', SaveDecisionOptions>;
30
30
  /**
31
- * Weaken the memory a new decision replaces, once the save has committed: half-life halved to a day
32
- * at least, marked stale, tagged `superseded`.
31
+ * Weaken the memory a new decision replaces, once the save has committed: marked stale, tagged
32
+ * `superseded`, and its half-life halved to a day at least the first time only.
33
33
  */
34
34
  export declare function weakenSupersededMemory(writes: EntryWrites, entry: MemoryEntry, actor?: string): Promise<void>;
35
35
  export declare function loadDecisions(objects: Objects, tenantId: string, opts?: ListDecisionsOptions): Promise<Decision[]>;
@@ -3,12 +3,12 @@
3
3
  * `active` despite memory decay. The memory mirror (memory_id NULLABLE, ON DELETE SET NULL) is for
4
4
  * recall only; saveDecision commits both in one `objects` transaction.
5
5
  */
6
+ import { markStale, SUPERSEDED_TAG } from '../../base/memory/stale-mark.js';
6
7
  import { assertTenantId } from '../../base/scope/tenant-id.js';
7
8
  import { MAX_SHORT_FIELD_LEN } from '../../base/util/limits.js';
8
9
  import { requireText } from './fields.js';
9
10
  import { listObjects } from './lifecycle.js';
10
11
  import { CLI_ACTOR } from '../../base/scope/actor.js';
11
- const SUPERSEDED_TAG = 'superseded';
12
12
  export const VALID_DECISION_STATES = new Set([
13
13
  'active',
14
14
  'superseded',
@@ -32,16 +32,11 @@ export const DECISION = {
32
32
  },
33
33
  };
34
34
  /**
35
- * Weaken the memory a new decision replaces, once the save has committed: half-life halved to a day
36
- * at least, marked stale, tagged `superseded`.
35
+ * Weaken the memory a new decision replaces, once the save has committed: marked stale, tagged
36
+ * `superseded`, and its half-life halved to a day at least the first time only.
37
37
  */
38
38
  export function weakenSupersededMemory(writes, entry, actor = CLI_ACTOR) {
39
- const tags = entry.tags.includes(SUPERSEDED_TAG) ? entry.tags : [...entry.tags, SUPERSEDED_TAG];
40
- const halved = Math.max(1, Math.floor(entry.halfLifeDays / 2));
41
- return writes.writeEntry({
42
- entry: { ...entry, halfLifeDays: halved, confidence: 'stale', tags },
43
- actor,
44
- });
39
+ return writes.writeEntry({ entry: markStale(entry, SUPERSEDED_TAG), actor });
45
40
  }
46
41
  export function loadDecisions(objects, tenantId, opts = {}) {
47
42
  return listObjects(objects, DECISION, tenantId, opts);
@@ -1,8 +1,9 @@
1
+ import { isSuperseded } from '../../base/memory/liveness.js';
1
2
  /** Rows current at `asOf`, or rows not superseded unless `includeSuperseded`. */
2
3
  export function currentEntries(entries, options) {
3
4
  if (options.asOf)
4
5
  return entriesAsOf(entries, new Date(options.asOf));
5
- return options.includeSuperseded ? entries : entries.filter((e) => !e.supersededBy);
6
+ return options.includeSuperseded ? entries : entries.filter((e) => !isSuperseded(e));
6
7
  }
7
8
  function entriesAsOf(entries, asOfDate) {
8
9
  // A merged pool can repeat an id; the first row wins, as a scan from the front would pick.
@@ -13,7 +14,7 @@ function entriesAsOf(entries, asOfDate) {
13
14
  return entries.filter((e) => {
14
15
  if (new Date(e.validFrom) > asOfDate)
15
16
  return false;
16
- if (!e.supersededBy)
17
+ if (!isSuperseded(e))
17
18
  return true;
18
19
  const successor = byId.get(e.supersededBy);
19
20
  return successor?.validFrom ? new Date(successor.validFrom) > asOfDate : true;
@@ -1,5 +1,7 @@
1
1
  import { originInSql, scopeAdmitSql } from '../runtime/rule-sql.js';
2
2
  import { cachedStatement } from '../../../db/statement-cache.js';
3
+ import { notSupersededIn } from '../../../db/sql-predicates.js';
4
+ import { moveOneStatus } from '../runtime/status-transitions.js';
3
5
  const COLUMNS = 'tenant_id, id, session_id, origin_project, compact_trigger, cwd, transcript_path, snapshot_saved, started_at, ' +
4
6
  'summarised_at, summary, items_json, items_written, status';
5
7
  function listRows(db, where, ...params) {
@@ -55,7 +57,12 @@ export function compactionProgress(db, tenantId, id) {
55
57
  return cachedStatement(db, `SELECT status, items_written FROM compactions WHERE tenant_id = ? AND id = ?`).get(tenantId, id);
56
58
  }
57
59
  export function markCompactionDone(db, tenantId, id, itemsWritten) {
58
- cachedStatement(db, `UPDATE compactions SET items_written = ?, status = 'done' WHERE tenant_id = ? AND id = ?`).run(itemsWritten, tenantId, id);
60
+ moveOneStatus(db, 'compactionDone', {
61
+ tenantId,
62
+ id,
63
+ to: 'done',
64
+ set: { items_written: itemsWritten },
65
+ });
59
66
  }
60
67
  /** Rows changed: 1 when the `started` row was closed, 0 when it had already moved. */
61
68
  export function closeStartedWithoutSummary(db, tenantId, id) {
@@ -70,7 +77,7 @@ export function nextCompactionStart(db, { tenantId, sessionId, after }) {
70
77
  export function heldMemoryRows(db, tenantId, origins) {
71
78
  const deny = scopeAdmitSql('');
72
79
  return cachedStatement(db, `SELECT id, source_session_id, content FROM memories WHERE tenant_id = ? AND ${originInSql(origins)} AND ` +
73
- `superseded_by IS NULL AND kind != 'raw'
80
+ `${notSupersededIn()} AND kind != 'raw'
74
81
  AND ${deny.sql}`).all(tenantId, ...origins, ...deny.params);
75
82
  }
76
83
  export function restampCompactionOrigin(db, { tenantId, from, into }) {
@@ -12,6 +12,7 @@ import { bound } from '../runtime/bound.js';
12
12
  import { writeEntryMirrors } from '../entries/entry-writes.js';
13
13
  import { cachedStatement } from '../../../db/statement-cache.js';
14
14
  import { handleStamp } from '../runtime/handle-clock.js';
15
+ import { moveOneStatus } from '../runtime/status-transitions.js';
15
16
  export { QUARANTINE_SCOPE_PREFIX };
16
17
  const SELECT_COLUMNS = 'tenant_id, memory_id, original_scope, reason, status, quarantined_at, decided_at, decided_by';
17
18
  function fromDbRow(row) {
@@ -63,12 +64,20 @@ export function recordQuarantine(db, opts) {
63
64
  ]);
64
65
  }
65
66
  function approveQuarantineRow(db, { tenantId, id: memoryId, actor: decidedBy }) {
66
- cachedStatement(db, `UPDATE memory_quarantine SET status = 'approved', decided_at = ?, decided_by = ? WHERE tenant_id = ? AND ` +
67
- `memory_id = ?`).run(handleStamp(db), decidedBy, tenantId, memoryId);
67
+ moveOneStatus(db, 'quarantineApprove', {
68
+ id: memoryId,
69
+ to: 'approved',
70
+ tenantId,
71
+ set: { decided_at: handleStamp(db), decided_by: decidedBy },
72
+ });
68
73
  }
69
74
  function rejectQuarantineRow(db, { tenantId, id: memoryId, actor: decidedBy }) {
70
- cachedStatement(db, `UPDATE memory_quarantine SET status = 'rejected', decided_at = ?, decided_by = ? WHERE tenant_id = ? AND ` +
71
- `memory_id = ?`).run(handleStamp(db), decidedBy, tenantId, memoryId);
75
+ moveOneStatus(db, 'quarantineReject', {
76
+ id: memoryId,
77
+ to: 'rejected',
78
+ tenantId,
79
+ set: { decided_at: handleStamp(db), decided_by: decidedBy },
80
+ });
72
81
  }
73
82
  /** A page of the tenant's records, each with its memory's content read in the same query batch. */
74
83
  function listQuarantined(db, tenantId, { status, limit, after }) {