hippo-memory 3.1.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/api/context/context-plan.js +2 -4
  2. package/dist/api/context/context-select.js +12 -7
  3. package/dist/api/recall/rank-stages.js +2 -2
  4. package/dist/base/memory/entry.js +2 -2
  5. package/dist/base/memory/markdown-delimiters.d.ts +15 -0
  6. package/dist/base/memory/markdown-delimiters.js +94 -0
  7. package/dist/base/memory/memory-audit.js +4 -4
  8. package/dist/base/memory/memory-quality.d.ts +12 -0
  9. package/dist/base/memory/memory-quality.js +92 -3
  10. package/dist/base/ports/store/types.d.ts +2 -1
  11. package/dist/base/rank/prompt-recall.d.ts +16 -3
  12. package/dist/base/rank/prompt-recall.js +47 -7
  13. package/dist/base/util/tokenize.d.ts +29 -0
  14. package/dist/base/util/tokenize.js +68 -5
  15. package/dist/base/util/version.d.ts +1 -1
  16. package/dist/base/util/version.js +1 -1
  17. package/dist/db/meta.d.ts +1 -0
  18. package/dist/db/meta.js +1 -0
  19. package/dist/domain/consolidate/conflicts.js +2 -2
  20. package/dist/domain/learn/forward-claim-detector.js +4 -2
  21. package/dist/domain/learn/invalidation.js +5 -2
  22. package/dist/domain/search/bm25.js +3 -3
  23. package/dist/store/rules/windows.d.ts +3 -0
  24. package/dist/store/rules/windows.js +3 -0
  25. package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
  26. package/dist/store/sqlite/recall/candidates.js +1 -0
  27. package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
  28. package/dist/store/sqlite/recall/search-rows.js +62 -26
  29. package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
  30. package/dist/store/sqlite/sharing/copy-rows.js +2 -1
  31. package/dist/surface/capture/command.d.ts +8 -1
  32. package/dist/surface/capture/command.js +38 -8
  33. package/dist/surface/capture/extract.d.ts +10 -0
  34. package/dist/surface/capture/extract.js +199 -26
  35. package/dist/surface/capture/session-texts.js +8 -6
  36. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  37. package/extensions/openclaw-plugin/package.json +1 -1
  38. package/openclaw.plugin.json +1 -1
  39. package/package.json +1 -1
@@ -1,6 +1,7 @@
1
1
  import { duplicateKey } from '../../base/util/same-text.js';
2
- import { assessAutomaticMemory } from '../../base/memory/memory-quality.js';
2
+ import { assessAutomaticMemory, asideCloseOffset, QUOTE_MARK, } from '../../base/memory/memory-quality.js';
3
3
  import { ERROR_TAG } from '../../base/memory/entry.js';
4
+ import { blankCodeSpans, strongMarkers } from '../../base/memory/markdown-delimiters.js';
4
5
  const DECISION_PATTERNS = [
5
6
  /\b(?:we(?:'ve| have)?|i(?:'ve| have)?|let's)\s+decid(?:ed|e)\b/i,
6
7
  /\b(?:let's|we(?:'ll| will| should)?)\s+(?:go with|do|use|try|build|implement|switch to)\b/i,
@@ -85,32 +86,139 @@ function continues(line, previous) {
85
86
  return false;
86
87
  return !SENTENCE_END.test(masked(`${previous} ${line}`).slice(0, previous.length));
87
88
  }
88
- function proseBlocks(text) {
89
+ /** What capture read before it removed bold markers. */
90
+ const keepAll = () => [];
91
+ /** Offsets of each `**` that pairs with the nearest opener before it. */
92
+ function pairedBold(line) {
93
+ const paired = [];
94
+ const openers = [];
95
+ // Blanked code spans keep every offset, so the markers found here are cut from the source.
96
+ for (const marker of strongMarkers(blankCodeSpans(line, '`'))) {
97
+ if (marker.inWord)
98
+ continue;
99
+ const opener = marker.close ? openers.pop() : undefined;
100
+ if (opener !== undefined)
101
+ paired.push(opener, marker.index);
102
+ else if (marker.open)
103
+ openers.push(marker.index);
104
+ }
105
+ return paired;
106
+ }
107
+ /** The segmenter splits after "**Label.**" and leaves the closing `**` on the next rule, so each
108
+ * real pair goes; a glob, `2**10` or `**kwargs` has no partner and stays as written. */
109
+ const boldPairs = (line) => pairedBold(line).sort((a, b) => a - b);
110
+ /** `line` without the two characters at each of the sorted `cuts`, built in one pass. */
111
+ function cutOut(line, cuts) {
112
+ let out = '';
113
+ let kept = 0;
114
+ for (const at of cuts) {
115
+ out += line.slice(kept, at);
116
+ kept = at + 2;
117
+ }
118
+ return out + line.slice(kept);
119
+ }
120
+ /** How many of the sorted `values` are at most `limit`. */
121
+ function countUpTo(values, limit) {
122
+ let low = 0;
123
+ let high = values.length;
124
+ while (low < high) {
125
+ const middle = (low + high) >> 1;
126
+ if (values[middle] <= limit)
127
+ low = middle + 1;
128
+ else
129
+ high = middle;
130
+ }
131
+ return low;
132
+ }
133
+ function blockOf(pieces, cuts) {
134
+ const raw = pieces.map((piece) => piece.text).join(' ');
135
+ const cutAt = cuts(raw);
136
+ // The sentences are those 3.1.0 split from the uncut text, so a removed `**` merges none: ICU
137
+ // breaks "**.Always" but not "n.Always". Masking keeps offsets.
138
+ const cutEnds = cutAt.map((at) => at + 2);
139
+ const sentenceStarts = [
140
+ ...new Intl.Segmenter('en', { granularity: 'sentence' }).segment(masked(raw)),
141
+ ].map(({ index }) => index - 2 * countUpTo(cutEnds, index));
142
+ // Where each cut sits in the cut text, and where each piece starts in the raw text.
143
+ const shifted = cutAt.map((at, k) => at - 2 * k);
144
+ const starts = [];
145
+ let sum = 0;
146
+ for (const piece of pieces) {
147
+ starts.push(sum);
148
+ sum += piece.text.length + 1;
149
+ }
150
+ const toProse = (rawIndex) => {
151
+ const k = Math.max(0, countUpTo(starts, rawIndex) - 1);
152
+ return pieces[k].at + Math.min(rawIndex - starts[k], pieces[k].text.length);
153
+ };
154
+ return {
155
+ text: cutOut(raw, cutAt),
156
+ sentenceStarts,
157
+ sourceOf: (from, to) => [
158
+ toProse(from + 2 * countUpTo(shifted, from)),
159
+ toProse(to + 2 * countUpTo(shifted, to - 1)),
160
+ ],
161
+ };
162
+ }
163
+ function proseBlocks(prose, cuts) {
89
164
  const blocks = [];
90
165
  let pending = [];
166
+ // Cut once the lines are joined, as a spec bullet is, so a bold pair that wraps a line goes whole.
91
167
  const flush = () => {
92
168
  if (pending.length > 0)
93
- blocks.push(pending.join(' '));
169
+ blocks.push(blockOf(pending, cuts));
94
170
  pending = [];
95
171
  };
96
- for (const line of text.split('\n')) {
172
+ let offset = 0;
173
+ for (const line of prose.split('\n')) {
97
174
  const trimmed = line.trim();
175
+ const end = offset + line.trimEnd().length;
176
+ offset += line.length + 1;
98
177
  if (!trimmed || /^#+\s/.test(trimmed)) {
99
178
  flush();
100
179
  continue;
101
180
  }
102
181
  const bullet = trimmed.match(BULLET);
103
- if (bullet || !continues(trimmed, pending.at(-1)))
182
+ if (bullet || !continues(trimmed, pending.at(-1)?.text))
104
183
  flush();
105
- pending.push(bullet?.[1] ?? trimmed);
184
+ const text = bullet?.[1] ?? trimmed;
185
+ pending.push({ text, at: end - text.length });
106
186
  }
107
187
  flush();
108
188
  return blocks;
109
189
  }
110
- function completeSentences(text) {
111
- // Masking keeps offsets, so each sentence is cut from the unchanged source.
112
- const segments = new Intl.Segmenter('en', { granularity: 'sentence' }).segment(masked(text));
113
- return [...segments].map(({ index, segment }) => text.slice(index, index + segment.length).trim());
190
+ /** Each sentence of a block, trimmed, with where it starts in the block text. */
191
+ function completeSentences({ text, sentenceStarts, }) {
192
+ return sentenceStarts.map((start, k) => {
193
+ const piece = text.slice(start, sentenceStarts[k + 1] ?? text.length);
194
+ return { sentence: piece.trim(), at: start + piece.length - piece.trimStart().length };
195
+ });
196
+ }
197
+ // A "(" after ":" or ";" is an emoticon, and one with a quote mark on both sides is quoted text.
198
+ function opensAside(text, at) {
199
+ const before = text[at - 1] ?? '';
200
+ const quoted = QUOTE_MARK.test(before) && QUOTE_MARK.test(text[at + 1] ?? '');
201
+ return text[at] === '(' && !/[:;]/.test(before) && !quoted;
202
+ }
203
+ // A sentence starts inside a cut aside: an unclosed "(" before it, an aside-closing ")" in it, and
204
+ // a word after that ")".
205
+ function asideCutter(block) {
206
+ const blanked = blankCodeSpans(block);
207
+ let depth = 0;
208
+ let seen = 0;
209
+ return (sentence, at) => {
210
+ while (seen < at) {
211
+ if (opensAside(blanked, seen))
212
+ depth++;
213
+ else if (blanked[seen] === ')')
214
+ depth = Math.max(0, depth - 1);
215
+ seen++;
216
+ }
217
+ if (depth === 0)
218
+ return false;
219
+ const close = asideCloseOffset(sentence);
220
+ return close >= 0 && /[\p{L}\p{N}]/u.test(sentence.slice(close + 1));
221
+ };
114
222
  }
115
223
  function insideBrackets(lead) {
116
224
  return (lead.match(/[([{]/g) ?? []).length > (lead.match(/[)\]}]/g) ?? []).length;
@@ -128,12 +236,15 @@ function extractFromPatterns(sentence, patterns, category) {
128
236
  .trim();
129
237
  return content.length <= MAX_CHARS ? { content, category, tags: [category, 'captured'] } : null;
130
238
  }
131
- function extractSpecSections(text) {
239
+ function extractSpecSections(prose, cuts) {
132
240
  const bullets = [];
133
241
  let inSpecSection = false;
134
242
  let open = false;
135
- for (const line of text.split('\n')) {
243
+ let offset = 0;
244
+ for (const line of prose.split('\n')) {
136
245
  const trimmed = line.trim();
246
+ const end = offset + line.trimEnd().length;
247
+ offset += line.length + 1;
137
248
  if (SPEC_HEADING_PATTERNS.some((pattern) => pattern.test(trimmed))) {
138
249
  inSpecSection = true;
139
250
  open = false;
@@ -142,46 +253,62 @@ function extractSpecSections(text) {
142
253
  if (/^#+\s/.test(trimmed) || /^[A-Z][a-z]+:$/.test(trimmed))
143
254
  inSpecSection = false;
144
255
  const bullet = trimmed.match(BULLET);
256
+ const last = bullets.at(-1);
145
257
  const wrapped = inSpecSection &&
146
258
  open &&
147
259
  !bullet &&
148
260
  trimmed !== '' &&
149
- (/^\s/.test(line) || continues(trimmed, bullets.at(-1)));
150
- if (inSpecSection && bullet)
151
- bullets.push(bullet[1].trim());
152
- else if (wrapped)
153
- bullets[bullets.length - 1] += ` ${trimmed}`;
261
+ (/^\s/.test(line) || continues(trimmed, last?.text));
262
+ if (inSpecSection && bullet) {
263
+ const text = bullet[1].trim();
264
+ bullets.push({ text, at: end - text.length, to: end });
265
+ }
266
+ else if (wrapped && last) {
267
+ last.text += ` ${trimmed}`;
268
+ last.to = end;
269
+ }
154
270
  open = inSpecSection && (bullet !== null || wrapped);
155
271
  }
272
+ // Cut once the lines are joined, so a pair that wraps a line goes whole.
156
273
  return bullets
157
- .filter((content) => content.length <= MAX_CHARS && !content.endsWith('?'))
158
- .map((content) => ({ content, category: 'spec', tags: ['spec', 'captured'] }));
274
+ .map(({ text, at, to }) => ({ content: cutOut(text, cuts(text)), from: at, to }))
275
+ .filter(({ content }) => content.length <= MAX_CHARS && !content.endsWith('?'))
276
+ .map(({ content, from, to }) => ({
277
+ item: { content, category: 'spec', tags: ['spec', 'captured'] },
278
+ from,
279
+ to,
280
+ }));
159
281
  }
160
- function extractOne(text) {
282
+ // The items of the spec bullets first, then those of each prose block.
283
+ function extractSourced(text, cuts) {
161
284
  const prose = proseOnly(text);
162
- const items = extractSpecSections(prose);
163
- for (const block of proseBlocks(prose)) {
164
- for (const sentence of completeSentences(block)) {
285
+ const found = extractSpecSections(prose, cuts);
286
+ for (const block of proseBlocks(prose, cuts)) {
287
+ const cutByAside = asideCutter(block.text);
288
+ for (const { sentence, at } of completeSentences(block)) {
165
289
  // A question asks rather than states; far over the bound, skip before any pattern runs.
166
290
  if (sentence.endsWith('?') || sentence.length > 2 * MAX_CHARS)
167
291
  continue;
292
+ if (cutByAside(sentence, at))
293
+ continue;
168
294
  for (const [category, patterns] of CATEGORIES) {
169
295
  const item = extractFromPatterns(sentence, patterns, category);
170
296
  if (item) {
171
- items.push(item);
297
+ const [from, to] = block.sourceOf(at, at + sentence.length);
298
+ found.push({ item, from, to });
172
299
  break;
173
300
  }
174
301
  }
175
302
  }
176
303
  }
177
- return items;
304
+ return found;
178
305
  }
179
306
  /** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
180
307
  * The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`. */
181
308
  export function extractFromTexts(texts, userTexts = 0) {
182
309
  const seen = new Set();
183
310
  return texts
184
- .flatMap((text, i) => extractOne(text).map((item) => (i < userTexts ? { ...item, fromUser: true } : item)))
311
+ .flatMap((text, i) => extractSourced(text, boldPairs).map(({ item }) => i < userTexts ? { ...item, fromUser: true } : item))
185
312
  .filter((item) => {
186
313
  const key = duplicateKey(item.content);
187
314
  if (seen.has(key) || !assessAutomaticMemory(item.content).accepted)
@@ -190,4 +317,50 @@ export function extractFromTexts(texts, userTexts = 0) {
190
317
  return true;
191
318
  });
192
319
  }
320
+ function trackOf(items) {
321
+ return { items, ends: items.map(({ to }) => to) };
322
+ }
323
+ /** The items of `track` whose range overlaps [from, to). */
324
+ function overlapping(track, from, to) {
325
+ const found = [];
326
+ for (let k = countUpTo(track.ends, from); k < track.items.length && track.items[k].from < to; k++) {
327
+ found.push(track.items[k]);
328
+ }
329
+ return found;
330
+ }
331
+ /** Whether each range covers the other, but for markers, spaces and sentence marks at the edges. */
332
+ function sameSpan(prose, a, b) {
333
+ const gap = (x, y) => prose.slice(Math.min(x, y), Math.max(x, y));
334
+ return /^[\s*.!?]*$/.test(gap(a.from, b.from)) && /^[\s*.!?]*$/.test(gap(a.to, b.to));
335
+ }
336
+ /** The twins of `texts`: each old item pairs, one to one, with the new item of the same span, the
337
+ * same category first and then the larger overlap; a pair whose texts differ. A new item that
338
+ * holds more or less than the old one is no twin, or a held row would block a rule it lacks. */
339
+ export function boldTwins(texts, scrub) {
340
+ const twins = [];
341
+ for (const text of texts.filter((t) => t.includes('**'))) {
342
+ const prose = proseOnly(text);
343
+ const fresh = extractSourced(text, boldPairs);
344
+ // Spec bullets and prose sentences overlap each other, so each kind keeps its own order.
345
+ const tracks = [trackOf(fresh.filter(({ item }) => item.category === 'spec'))];
346
+ tracks.push(trackOf(fresh.filter(({ item }) => item.category !== 'spec')));
347
+ const taken = new Set();
348
+ for (const sourced of extractSourced(text, keepAll)) {
349
+ const { item: old, from, to } = sourced;
350
+ const size = (found) => Math.min(found.to, to) - Math.max(found.from, from);
351
+ const same = (found) => Number(found.item.category === old.category);
352
+ const [match] = tracks
353
+ .flatMap((track) => overlapping(track, from, to))
354
+ .filter((found) => !taken.has(found) && sameSpan(prose, found, sourced))
355
+ .sort((a, b) => same(b) - same(a) || size(b) - size(a));
356
+ if (match === undefined)
357
+ continue;
358
+ taken.add(match);
359
+ if (old.content !== match.item.content) {
360
+ twins.push({ old: old.content, clean: scrub(match.item.content) });
361
+ }
362
+ }
363
+ }
364
+ return twins;
365
+ }
193
366
  //# sourceMappingURL=extract.js.map
@@ -3,11 +3,10 @@
3
3
  import { BadRequestError } from '../../base/errors/api-errors.js';
4
4
  import { assertCallerIds } from '../../api/prompt-hook.js';
5
5
  import { assertCallerProject, projectNames } from '../../base/scope/project-identity.js';
6
- import { storedTextKeys } from '../../base/util/same-text.js';
7
6
  import { scrubForSharing } from './share-scrub.js';
8
- import { captureExtractedItems } from './command.js';
7
+ import { captureExtractedItems, heldKeysOf, scrubbedItems } from './command.js';
9
8
  import { detachedEmbedder } from '../../api/embeddings.js';
10
- import { extractFromTexts } from './extract.js';
9
+ import { boldTwins, extractFromTexts } from './extract.js';
11
10
  import { withEntryInfra } from '../entry-infra.js';
12
11
  import { withDetached } from '../../api/detached.js';
13
12
  const MAX_TEXTS = 30;
@@ -40,7 +39,9 @@ function assertSessionCapture(req) {
40
39
  export async function captureSessionTexts(given, req) {
41
40
  const ctx = withEntryInfra(given);
42
41
  assertSessionCapture(req);
43
- const extracted = extractFromTexts(req.texts.map(scrubForSharing));
42
+ const scrubbed = req.texts.map(scrubForSharing);
43
+ // Scrubbed again after extraction: bold removal can rebuild an address or key.
44
+ const extracted = scrubbedItems(extractFromTexts(scrubbed), scrubForSharing);
44
45
  // The newest items win, since the end of a session states where it landed.
45
46
  const items = extracted.slice(-MAX_ITEMS);
46
47
  if (items.length === 0)
@@ -48,7 +49,8 @@ export async function captureSessionTexts(given, req) {
48
49
  // The project filter is classifyOriginProject's in SQL: the caller's rows and user-global rows
49
50
  // block a copy, other projects' rows do not.
50
51
  const dedup = { tenantId: ctx.tenantId, project: projectNames(req.project) };
51
- const held = await ctx.store.captures.textsHolding(dedup.tenantId, items.map((i) => i.content), { project: dedup.project });
52
+ const twins = boldTwins(scrubbed, scrubForSharing);
53
+ const held = await ctx.store.captures.textsHolding(dedup.tenantId, [...items.map((i) => i.content), ...twins.map((twin) => twin.old)], { project: dedup.project });
52
54
  const { store, config } = ctx;
53
55
  const folder = { store, config, embedderFor: ctx.infra.embedderFor };
54
56
  const tally = await withDetached(ctx, (detached) => captureExtractedItems({ store, config, embed: detachedEmbedder(folder, ctx.infra, detached) }, {
@@ -59,7 +61,7 @@ export async function captureSessionTexts(given, req) {
59
61
  lean: true,
60
62
  dedup,
61
63
  now: ctx.infra.clock(),
62
- }, items, storedTextKeys(held)));
64
+ }, items, heldKeysOf(held, twins)));
63
65
  return { ...tally, skipped: tally.skipped + extracted.length - items.length };
64
66
  }
65
67
  //# sourceMappingURL=session-texts.js.map
@@ -2,7 +2,7 @@
2
2
  "id": "hippo-memory",
3
3
  "name": "Hippo Memory",
4
4
  "description": "Make your agent's memory work like a brain. Hippo is long-term memory for coding agents.",
5
- "version": "3.1.0",
5
+ "version": "3.1.1",
6
6
 
7
7
  "configSchema": {
8
8
  "type": "object",
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hippo-memory",
3
- "version": "3.1.0",
3
+ "version": "3.1.1",
4
4
  "type": "module",
5
5
  "description": "Make your agent's memory work like a brain. Hippo is long-term memory for coding agents.",
6
6
  "main": "index.ts",
@@ -2,7 +2,7 @@
2
2
  "id": "hippo-memory",
3
3
  "name": "Hippo Memory",
4
4
  "description": "Make your agent's memory work like a brain. Hippo is long-term memory for coding agents.",
5
- "version": "3.1.0",
5
+ "version": "3.1.1",
6
6
  "configSchema": {
7
7
  "type": "object",
8
8
  "additionalProperties": false,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hippo-memory",
3
- "version": "3.1.0",
3
+ "version": "3.1.1",
4
4
  "description": "Make your agent's memory work like a brain. Hippo is long-term memory for coding agents.",
5
5
  "mcpName": "io.github.kitfunso/hippo-memory",
6
6
  "type": "module",