hippo-memory 3.0.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/README.md +1 -0
  2. package/dist/api/context/context-plan.js +2 -4
  3. package/dist/api/context/context-select.js +12 -7
  4. package/dist/api/detached.d.ts +6 -0
  5. package/dist/api/detached.js +16 -0
  6. package/dist/api/embeddings.d.ts +4 -3
  7. package/dist/api/embeddings.js +4 -4
  8. package/dist/api/memories/promote.js +2 -1
  9. package/dist/api/memories/remember-local.d.ts +5 -4
  10. package/dist/api/memories/remember-local.js +13 -10
  11. package/dist/api/recall/rank-stages.js +2 -2
  12. package/dist/api/session-end.js +1 -0
  13. package/dist/api/sharing.js +9 -7
  14. package/dist/api/sleep/auto-sleep.d.ts +1 -1
  15. package/dist/api/sleep/auto-sleep.js +5 -3
  16. package/dist/api/sleep/learn.js +5 -2
  17. package/dist/api/sleep/sleep-run.js +4 -2
  18. package/dist/api/types.d.ts +4 -0
  19. package/dist/base/memory/entry.js +2 -2
  20. package/dist/base/memory/markdown-delimiters.d.ts +15 -0
  21. package/dist/base/memory/markdown-delimiters.js +94 -0
  22. package/dist/base/memory/memory-audit.js +4 -4
  23. package/dist/base/memory/memory-quality.d.ts +12 -0
  24. package/dist/base/memory/memory-quality.js +92 -3
  25. package/dist/base/ports/store/types.d.ts +2 -1
  26. package/dist/base/rank/prompt-recall.d.ts +16 -3
  27. package/dist/base/rank/prompt-recall.js +47 -7
  28. package/dist/base/util/detached-set.d.ts +8 -0
  29. package/dist/base/util/detached-set.js +35 -0
  30. package/dist/base/util/tokenize.d.ts +29 -0
  31. package/dist/base/util/tokenize.js +68 -5
  32. package/dist/base/util/version.d.ts +1 -1
  33. package/dist/base/util/version.js +1 -1
  34. package/dist/db/meta.d.ts +1 -0
  35. package/dist/db/meta.js +1 -0
  36. package/dist/db/store-runtime.d.ts +10 -3
  37. package/dist/db/store-runtime.js +19 -3
  38. package/dist/domain/consolidate/conflicts.js +2 -2
  39. package/dist/domain/learn/forward-claim-detector.js +4 -2
  40. package/dist/domain/learn/invalidation.js +5 -2
  41. package/dist/domain/search/bm25.js +3 -3
  42. package/dist/domain/search/detached-work.d.ts +6 -0
  43. package/dist/domain/search/detached-work.js +2 -0
  44. package/dist/domain/search/embed-memories.d.ts +4 -4
  45. package/dist/domain/search/embed-memories.js +4 -5
  46. package/dist/domain/sharing/global-sync.d.ts +8 -2
  47. package/dist/domain/sharing/global-sync.js +5 -5
  48. package/dist/domain/sharing/share.d.ts +4 -0
  49. package/dist/domain/sharing/share.js +2 -2
  50. package/dist/entry/eval-scripts.d.ts +1 -0
  51. package/dist/entry/sleep.d.ts +1 -0
  52. package/dist/index.d.ts +1 -1
  53. package/dist/server.d.ts +1 -0
  54. package/dist/store/rules/windows.d.ts +3 -0
  55. package/dist/store/rules/windows.js +3 -0
  56. package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
  57. package/dist/store/sqlite/recall/candidates.js +1 -0
  58. package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
  59. package/dist/store/sqlite/recall/search-rows.js +62 -26
  60. package/dist/store/sqlite/runtime/server-runtime.d.ts +6 -1
  61. package/dist/store/sqlite/runtime/server-runtime.js +15 -3
  62. package/dist/store/sqlite/runtime/worker-jobs.js +2 -2
  63. package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
  64. package/dist/store/sqlite/sharing/copy-rows.js +2 -1
  65. package/dist/surface/capture/command.d.ts +8 -1
  66. package/dist/surface/capture/command.js +38 -8
  67. package/dist/surface/capture/compact.d.ts +3 -3
  68. package/dist/surface/capture/compact.js +4 -4
  69. package/dist/surface/capture/extract.d.ts +10 -0
  70. package/dist/surface/capture/extract.js +199 -26
  71. package/dist/surface/capture/session-texts.js +11 -11
  72. package/dist/surface/cli/api-context.d.ts +5 -0
  73. package/dist/surface/cli/api-context.js +12 -0
  74. package/dist/surface/cli/goals.js +6 -8
  75. package/dist/surface/cli/hook-runtime.js +2 -2
  76. package/dist/surface/cli/memory-import.js +20 -15
  77. package/dist/surface/mcp/protocol.d.ts +6 -0
  78. package/dist/surface/mcp/protocol.js +1 -0
  79. package/dist/surface/mcp/stdio.js +11 -5
  80. package/dist/surface/open-store.d.ts +2 -2
  81. package/dist/surface/open-store.js +11 -7
  82. package/dist/surface/server/auth.d.ts +1 -1
  83. package/dist/surface/server/auth.js +13 -2
  84. package/dist/surface/server/boot.js +3 -2
  85. package/dist/surface/server/dashboard-listener.js +2 -2
  86. package/dist/surface/server/lifecycle.d.ts +2 -0
  87. package/dist/surface/server/lifecycle.js +4 -2
  88. package/dist/surface/server/mcp-http.js +1 -0
  89. package/dist/surface/server/types.d.ts +3 -0
  90. package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
  91. package/extensions/openclaw-plugin/package.json +1 -1
  92. package/openclaw.plugin.json +1 -1
  93. package/package.json +1 -1
@@ -2,13 +2,14 @@ import * as fs from 'node:fs';
2
2
  import * as path from 'node:path';
3
3
  import { createMemory, Layer } from '../../base/memory/entry.js';
4
4
  import { duplicateKey, storedTextKeys } from '../../base/util/same-text.js';
5
+ import { isCertainReason, storedDefect } from '../../base/memory/memory-quality.js';
5
6
  import { maskEmails, redactSecretsStrict } from '../../base/util/secret-detect.js';
6
7
  import { projectId, projectNames } from '../../base/scope/project-identity.js';
7
8
  import { isInitialized } from '../../host/project-identity.js';
8
9
  import { errorMessage, log, errorFields } from '../../base/util/log.js';
9
10
  import { teeStdStreams } from '../../host/stream-tee.js';
10
11
  import { ALL_TENANTS } from '../../base/util/tenant.js';
11
- import { extractFromTexts } from './extract.js';
12
+ import { boldTwins, extractFromTexts } from './extract.js';
12
13
  import { collectSessionTurns, sessionTail, resolveLastSessionTranscript, } from './transcript.js';
13
14
  import { PRIVATE_DIR } from '../../host/private-dir.js';
14
15
  import { evalClock } from '../../infra/eval-switches.js';
@@ -71,14 +72,16 @@ async function runCaptureCore(target, options) {
71
72
  console.log('No text to capture from.');
72
73
  return;
73
74
  }
74
- // Scrub once here, as the snapshot fields are: every source can carry a pasted token (AGENTS.md:
75
- // no secrets in memories).
76
- const extracted = extractFromTexts(texts.map((text) => maskEmails(redactSecretsStrict(text))), userTexts);
75
+ // Scrub the texts and again each item: bold removal can rebuild an address or key the first
76
+ // pass could not see (AGENTS.md: no secrets in memories).
77
+ const scrub = (text) => maskEmails(redactSecretsStrict(text));
78
+ const scrubbed = texts.map(scrub);
79
+ const extracted = scrubbedItems(extractFromTexts(scrubbed, userTexts), scrub);
77
80
  if (extracted.length === 0) {
78
81
  console.log('No actionable items found in the input.');
79
82
  return;
80
83
  }
81
- const keys = await storedKeysInView(target.store, options, extracted);
84
+ const keys = await storedKeysInView(target.store, options, extracted, boldTwins(scrubbed, scrub));
82
85
  printCaptureTally(options, await captureFromCli(target, options, extracted, keys));
83
86
  }
84
87
  async function captureFromCli({ store, config, embed }, options, extracted, keys) {
@@ -94,12 +97,39 @@ async function captureFromCli({ store, config, embed }, options, extracted, keys
94
97
  }
95
98
  return previewCapture({ store, config }, writeOpts, extracted, keys);
96
99
  }
100
+ /** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
101
+ * items one, and the first stays. */
102
+ export function scrubbedItems(items, scrub) {
103
+ const seen = new Set();
104
+ return items
105
+ .map((item) => ({ ...item, content: scrub(item.content) }))
106
+ .filter((item) => {
107
+ const key = duplicateKey(item.content);
108
+ if (seen.has(key))
109
+ return false;
110
+ seen.add(key);
111
+ return true;
112
+ });
113
+ }
114
+ /** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
115
+ * row: a pinned row, or one whose text is no certain defect. */
116
+ export function heldKeysOf(rows, twins) {
117
+ const keys = storedTextKeys(rows);
118
+ const pinned = storedTextKeys(rows.filter((row) => row.pinned));
119
+ const clean = twins
120
+ .filter(({ old }) => (isCertainReason(storedDefect(old)) ? pinned : keys).has(duplicateKey(old)))
121
+ .map((twin) => duplicateKey(twin.clean));
122
+ for (const key of clean)
123
+ keys.add(key);
124
+ return keys;
125
+ }
97
126
  // Dedup only against rows this capture's reader sees: another tenant's or project's rows are
98
127
  // hidden, so they must not stop its own copy.
99
- async function storedKeysInView(store, options, extracted) {
128
+ async function storedKeysInView(store, options, extracted, twins) {
100
129
  const { tenantId, ...filter } = captureDedup(options);
101
- const texts = extracted.map((item) => item.content);
102
- return storedTextKeys(await store.captures.textsHolding(tenantId, texts, filter));
130
+ // The lookup matches stored text exactly, so it also asks for the lines as capture once stored them.
131
+ const texts = [...extracted.map((item) => item.content), ...twins.map((twin) => twin.old)];
132
+ return heldKeysOf(await store.captures.textsHolding(tenantId, texts, filter), twins);
103
133
  }
104
134
  /** The rows a capture's dedup reads, as data, so the store's write can read them again in its
105
135
  * own transaction. */
@@ -1,8 +1,8 @@
1
1
  import { type HookRuntime } from '../../base/session/capture-contract.js';
2
2
  import type { StoreGroups } from '../../base/ports/store.js';
3
- /** Log-forgery guard: strip C0 control chars (newlines above all) from payload-controlled values
4
- * so a crafted value cannot inject fake `[hippo]` log lines. Exported as the one shared guard
5
- * for every `[hippo]`-prefixed log writer that interpolates payload values. */
3
+ /** Log-forgery guard: strip control chars (U+0000-U+001F, DEL, U+0080-U+009F; newlines and NEL
4
+ * above all) so a crafted value cannot inject fake `[hippo]` lines or terminal escapes. The one
5
+ * shared guard for every log line and printed name that interpolates payload values. */
6
6
  export declare function sanitizeLogMessage(message: string): string;
7
7
  /** The folder's store as a compaction hook reaches it, built by the CLI root. */
8
8
  export type CompactionStore = Pick<StoreGroups, 'compactionHooks' | 'sessions'>;
@@ -15,12 +15,12 @@ import { transcriptWorkingState } from './working-state.js';
15
15
  import { PRIVATE_DIR } from '../../host/private-dir.js';
16
16
  // Diagnostic-only log; a long-lived install must not grow it unbounded.
17
17
  const PRE_COMPACT_LOG_MAX_BYTES = 256 * 1024;
18
- /** Log-forgery guard: strip C0 control chars (newlines above all) from payload-controlled values
19
- * so a crafted value cannot inject fake `[hippo]` log lines. Exported as the one shared guard
20
- * for every `[hippo]`-prefixed log writer that interpolates payload values. */
18
+ /** Log-forgery guard: strip control chars (U+0000-U+001F, DEL, U+0080-U+009F; newlines and NEL
19
+ * above all) so a crafted value cannot inject fake `[hippo]` lines or terminal escapes. The one
20
+ * shared guard for every log line and printed name that interpolates payload values. */
21
21
  export function sanitizeLogMessage(message) {
22
22
  // eslint-disable-next-line no-control-regex
23
- return message.replace(/[\x00-\x1f]/g, '');
23
+ return message.replace(/[\x00-\x1f\x7f-\x9f]/g, '');
24
24
  }
25
25
  function appendPreCompactLog(logFile, message, fields) {
26
26
  try {
@@ -8,4 +8,14 @@ export interface ExtractedItem {
8
8
  /** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
9
9
  * The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`. */
10
10
  export declare function extractFromTexts(texts: readonly string[], userTexts?: number): ExtractedItem[];
11
+ /** A text an older capture, which kept every bold marker, stored for a line, and what capture
12
+ * extracts from that line now. */
13
+ export interface BoldTwin {
14
+ readonly old: string;
15
+ readonly clean: string;
16
+ }
17
+ /** The twins of `texts`: each old item pairs, one to one, with the new item of the same span, the
18
+ * same category first and then the larger overlap; a pair whose texts differ. A new item that
19
+ * holds more or less than the old one is no twin, or a held row would block a rule it lacks. */
20
+ export declare function boldTwins(texts: readonly string[], scrub: (text: string) => string): BoldTwin[];
11
21
  //# sourceMappingURL=extract.d.ts.map
@@ -1,6 +1,7 @@
1
1
  import { duplicateKey } from '../../base/util/same-text.js';
2
- import { assessAutomaticMemory } from '../../base/memory/memory-quality.js';
2
+ import { assessAutomaticMemory, asideCloseOffset, QUOTE_MARK, } from '../../base/memory/memory-quality.js';
3
3
  import { ERROR_TAG } from '../../base/memory/entry.js';
4
+ import { blankCodeSpans, strongMarkers } from '../../base/memory/markdown-delimiters.js';
4
5
  const DECISION_PATTERNS = [
5
6
  /\b(?:we(?:'ve| have)?|i(?:'ve| have)?|let's)\s+decid(?:ed|e)\b/i,
6
7
  /\b(?:let's|we(?:'ll| will| should)?)\s+(?:go with|do|use|try|build|implement|switch to)\b/i,
@@ -85,32 +86,139 @@ function continues(line, previous) {
85
86
  return false;
86
87
  return !SENTENCE_END.test(masked(`${previous} ${line}`).slice(0, previous.length));
87
88
  }
88
- function proseBlocks(text) {
89
+ /** What capture read before it removed bold markers. */
90
+ const keepAll = () => [];
91
+ /** Offsets of each `**` that pairs with the nearest opener before it. */
92
+ function pairedBold(line) {
93
+ const paired = [];
94
+ const openers = [];
95
+ // Blanked code spans keep every offset, so the markers found here are cut from the source.
96
+ for (const marker of strongMarkers(blankCodeSpans(line, '`'))) {
97
+ if (marker.inWord)
98
+ continue;
99
+ const opener = marker.close ? openers.pop() : undefined;
100
+ if (opener !== undefined)
101
+ paired.push(opener, marker.index);
102
+ else if (marker.open)
103
+ openers.push(marker.index);
104
+ }
105
+ return paired;
106
+ }
107
+ /** The segmenter splits after "**Label.**" and leaves the closing `**` on the next rule, so each
108
+ * real pair goes; a glob, `2**10` or `**kwargs` has no partner and stays as written. */
109
+ const boldPairs = (line) => pairedBold(line).sort((a, b) => a - b);
110
+ /** `line` without the two characters at each of the sorted `cuts`, built in one pass. */
111
+ function cutOut(line, cuts) {
112
+ let out = '';
113
+ let kept = 0;
114
+ for (const at of cuts) {
115
+ out += line.slice(kept, at);
116
+ kept = at + 2;
117
+ }
118
+ return out + line.slice(kept);
119
+ }
120
+ /** How many of the sorted `values` are at most `limit`. */
121
+ function countUpTo(values, limit) {
122
+ let low = 0;
123
+ let high = values.length;
124
+ while (low < high) {
125
+ const middle = (low + high) >> 1;
126
+ if (values[middle] <= limit)
127
+ low = middle + 1;
128
+ else
129
+ high = middle;
130
+ }
131
+ return low;
132
+ }
133
+ function blockOf(pieces, cuts) {
134
+ const raw = pieces.map((piece) => piece.text).join(' ');
135
+ const cutAt = cuts(raw);
136
+ // The sentences are those 3.1.0 split from the uncut text, so a removed `**` merges none: ICU
137
+ // breaks "**.Always" but not "n.Always". Masking keeps offsets.
138
+ const cutEnds = cutAt.map((at) => at + 2);
139
+ const sentenceStarts = [
140
+ ...new Intl.Segmenter('en', { granularity: 'sentence' }).segment(masked(raw)),
141
+ ].map(({ index }) => index - 2 * countUpTo(cutEnds, index));
142
+ // Where each cut sits in the cut text, and where each piece starts in the raw text.
143
+ const shifted = cutAt.map((at, k) => at - 2 * k);
144
+ const starts = [];
145
+ let sum = 0;
146
+ for (const piece of pieces) {
147
+ starts.push(sum);
148
+ sum += piece.text.length + 1;
149
+ }
150
+ const toProse = (rawIndex) => {
151
+ const k = Math.max(0, countUpTo(starts, rawIndex) - 1);
152
+ return pieces[k].at + Math.min(rawIndex - starts[k], pieces[k].text.length);
153
+ };
154
+ return {
155
+ text: cutOut(raw, cutAt),
156
+ sentenceStarts,
157
+ sourceOf: (from, to) => [
158
+ toProse(from + 2 * countUpTo(shifted, from)),
159
+ toProse(to + 2 * countUpTo(shifted, to - 1)),
160
+ ],
161
+ };
162
+ }
163
+ function proseBlocks(prose, cuts) {
89
164
  const blocks = [];
90
165
  let pending = [];
166
+ // Cut once the lines are joined, as a spec bullet is, so a bold pair that wraps a line goes whole.
91
167
  const flush = () => {
92
168
  if (pending.length > 0)
93
- blocks.push(pending.join(' '));
169
+ blocks.push(blockOf(pending, cuts));
94
170
  pending = [];
95
171
  };
96
- for (const line of text.split('\n')) {
172
+ let offset = 0;
173
+ for (const line of prose.split('\n')) {
97
174
  const trimmed = line.trim();
175
+ const end = offset + line.trimEnd().length;
176
+ offset += line.length + 1;
98
177
  if (!trimmed || /^#+\s/.test(trimmed)) {
99
178
  flush();
100
179
  continue;
101
180
  }
102
181
  const bullet = trimmed.match(BULLET);
103
- if (bullet || !continues(trimmed, pending.at(-1)))
182
+ if (bullet || !continues(trimmed, pending.at(-1)?.text))
104
183
  flush();
105
- pending.push(bullet?.[1] ?? trimmed);
184
+ const text = bullet?.[1] ?? trimmed;
185
+ pending.push({ text, at: end - text.length });
106
186
  }
107
187
  flush();
108
188
  return blocks;
109
189
  }
110
- function completeSentences(text) {
111
- // Masking keeps offsets, so each sentence is cut from the unchanged source.
112
- const segments = new Intl.Segmenter('en', { granularity: 'sentence' }).segment(masked(text));
113
- return [...segments].map(({ index, segment }) => text.slice(index, index + segment.length).trim());
190
+ /** Each sentence of a block, trimmed, with where it starts in the block text. */
191
+ function completeSentences({ text, sentenceStarts, }) {
192
+ return sentenceStarts.map((start, k) => {
193
+ const piece = text.slice(start, sentenceStarts[k + 1] ?? text.length);
194
+ return { sentence: piece.trim(), at: start + piece.length - piece.trimStart().length };
195
+ });
196
+ }
197
+ // A "(" after ":" or ";" is an emoticon, and one with a quote mark on both sides is quoted text.
198
+ function opensAside(text, at) {
199
+ const before = text[at - 1] ?? '';
200
+ const quoted = QUOTE_MARK.test(before) && QUOTE_MARK.test(text[at + 1] ?? '');
201
+ return text[at] === '(' && !/[:;]/.test(before) && !quoted;
202
+ }
203
+ // A sentence starts inside a cut aside: an unclosed "(" before it, an aside-closing ")" in it, and
204
+ // a word after that ")".
205
+ function asideCutter(block) {
206
+ const blanked = blankCodeSpans(block);
207
+ let depth = 0;
208
+ let seen = 0;
209
+ return (sentence, at) => {
210
+ while (seen < at) {
211
+ if (opensAside(blanked, seen))
212
+ depth++;
213
+ else if (blanked[seen] === ')')
214
+ depth = Math.max(0, depth - 1);
215
+ seen++;
216
+ }
217
+ if (depth === 0)
218
+ return false;
219
+ const close = asideCloseOffset(sentence);
220
+ return close >= 0 && /[\p{L}\p{N}]/u.test(sentence.slice(close + 1));
221
+ };
114
222
  }
115
223
  function insideBrackets(lead) {
116
224
  return (lead.match(/[([{]/g) ?? []).length > (lead.match(/[)\]}]/g) ?? []).length;
@@ -128,12 +236,15 @@ function extractFromPatterns(sentence, patterns, category) {
128
236
  .trim();
129
237
  return content.length <= MAX_CHARS ? { content, category, tags: [category, 'captured'] } : null;
130
238
  }
131
- function extractSpecSections(text) {
239
+ function extractSpecSections(prose, cuts) {
132
240
  const bullets = [];
133
241
  let inSpecSection = false;
134
242
  let open = false;
135
- for (const line of text.split('\n')) {
243
+ let offset = 0;
244
+ for (const line of prose.split('\n')) {
136
245
  const trimmed = line.trim();
246
+ const end = offset + line.trimEnd().length;
247
+ offset += line.length + 1;
137
248
  if (SPEC_HEADING_PATTERNS.some((pattern) => pattern.test(trimmed))) {
138
249
  inSpecSection = true;
139
250
  open = false;
@@ -142,46 +253,62 @@ function extractSpecSections(text) {
142
253
  if (/^#+\s/.test(trimmed) || /^[A-Z][a-z]+:$/.test(trimmed))
143
254
  inSpecSection = false;
144
255
  const bullet = trimmed.match(BULLET);
256
+ const last = bullets.at(-1);
145
257
  const wrapped = inSpecSection &&
146
258
  open &&
147
259
  !bullet &&
148
260
  trimmed !== '' &&
149
- (/^\s/.test(line) || continues(trimmed, bullets.at(-1)));
150
- if (inSpecSection && bullet)
151
- bullets.push(bullet[1].trim());
152
- else if (wrapped)
153
- bullets[bullets.length - 1] += ` ${trimmed}`;
261
+ (/^\s/.test(line) || continues(trimmed, last?.text));
262
+ if (inSpecSection && bullet) {
263
+ const text = bullet[1].trim();
264
+ bullets.push({ text, at: end - text.length, to: end });
265
+ }
266
+ else if (wrapped && last) {
267
+ last.text += ` ${trimmed}`;
268
+ last.to = end;
269
+ }
154
270
  open = inSpecSection && (bullet !== null || wrapped);
155
271
  }
272
+ // Cut once the lines are joined, so a pair that wraps a line goes whole.
156
273
  return bullets
157
- .filter((content) => content.length <= MAX_CHARS && !content.endsWith('?'))
158
- .map((content) => ({ content, category: 'spec', tags: ['spec', 'captured'] }));
274
+ .map(({ text, at, to }) => ({ content: cutOut(text, cuts(text)), from: at, to }))
275
+ .filter(({ content }) => content.length <= MAX_CHARS && !content.endsWith('?'))
276
+ .map(({ content, from, to }) => ({
277
+ item: { content, category: 'spec', tags: ['spec', 'captured'] },
278
+ from,
279
+ to,
280
+ }));
159
281
  }
160
- function extractOne(text) {
282
+ // The items of the spec bullets first, then those of each prose block.
283
+ function extractSourced(text, cuts) {
161
284
  const prose = proseOnly(text);
162
- const items = extractSpecSections(prose);
163
- for (const block of proseBlocks(prose)) {
164
- for (const sentence of completeSentences(block)) {
285
+ const found = extractSpecSections(prose, cuts);
286
+ for (const block of proseBlocks(prose, cuts)) {
287
+ const cutByAside = asideCutter(block.text);
288
+ for (const { sentence, at } of completeSentences(block)) {
165
289
  // A question asks rather than states; far over the bound, skip before any pattern runs.
166
290
  if (sentence.endsWith('?') || sentence.length > 2 * MAX_CHARS)
167
291
  continue;
292
+ if (cutByAside(sentence, at))
293
+ continue;
168
294
  for (const [category, patterns] of CATEGORIES) {
169
295
  const item = extractFromPatterns(sentence, patterns, category);
170
296
  if (item) {
171
- items.push(item);
297
+ const [from, to] = block.sourceOf(at, at + sentence.length);
298
+ found.push({ item, from, to });
172
299
  break;
173
300
  }
174
301
  }
175
302
  }
176
303
  }
177
- return items;
304
+ return found;
178
305
  }
179
306
  /** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
180
307
  * The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`. */
181
308
  export function extractFromTexts(texts, userTexts = 0) {
182
309
  const seen = new Set();
183
310
  return texts
184
- .flatMap((text, i) => extractOne(text).map((item) => (i < userTexts ? { ...item, fromUser: true } : item)))
311
+ .flatMap((text, i) => extractSourced(text, boldPairs).map(({ item }) => i < userTexts ? { ...item, fromUser: true } : item))
185
312
  .filter((item) => {
186
313
  const key = duplicateKey(item.content);
187
314
  if (seen.has(key) || !assessAutomaticMemory(item.content).accepted)
@@ -190,4 +317,50 @@ export function extractFromTexts(texts, userTexts = 0) {
190
317
  return true;
191
318
  });
192
319
  }
320
+ function trackOf(items) {
321
+ return { items, ends: items.map(({ to }) => to) };
322
+ }
323
+ /** The items of `track` whose range overlaps [from, to). */
324
+ function overlapping(track, from, to) {
325
+ const found = [];
326
+ for (let k = countUpTo(track.ends, from); k < track.items.length && track.items[k].from < to; k++) {
327
+ found.push(track.items[k]);
328
+ }
329
+ return found;
330
+ }
331
+ /** Whether each range covers the other, but for markers, spaces and sentence marks at the edges. */
332
+ function sameSpan(prose, a, b) {
333
+ const gap = (x, y) => prose.slice(Math.min(x, y), Math.max(x, y));
334
+ return /^[\s*.!?]*$/.test(gap(a.from, b.from)) && /^[\s*.!?]*$/.test(gap(a.to, b.to));
335
+ }
336
+ /** The twins of `texts`: each old item pairs, one to one, with the new item of the same span, the
337
+ * same category first and then the larger overlap; a pair whose texts differ. A new item that
338
+ * holds more or less than the old one is no twin, or a held row would block a rule it lacks. */
339
+ export function boldTwins(texts, scrub) {
340
+ const twins = [];
341
+ for (const text of texts.filter((t) => t.includes('**'))) {
342
+ const prose = proseOnly(text);
343
+ const fresh = extractSourced(text, boldPairs);
344
+ // Spec bullets and prose sentences overlap each other, so each kind keeps its own order.
345
+ const tracks = [trackOf(fresh.filter(({ item }) => item.category === 'spec'))];
346
+ tracks.push(trackOf(fresh.filter(({ item }) => item.category !== 'spec')));
347
+ const taken = new Set();
348
+ for (const sourced of extractSourced(text, keepAll)) {
349
+ const { item: old, from, to } = sourced;
350
+ const size = (found) => Math.min(found.to, to) - Math.max(found.from, from);
351
+ const same = (found) => Number(found.item.category === old.category);
352
+ const [match] = tracks
353
+ .flatMap((track) => overlapping(track, from, to))
354
+ .filter((found) => !taken.has(found) && sameSpan(prose, found, sourced))
355
+ .sort((a, b) => same(b) - same(a) || size(b) - size(a));
356
+ if (match === undefined)
357
+ continue;
358
+ taken.add(match);
359
+ if (old.content !== match.item.content) {
360
+ twins.push({ old: old.content, clean: scrub(match.item.content) });
361
+ }
362
+ }
363
+ }
364
+ return twins;
365
+ }
193
366
  //# sourceMappingURL=extract.js.map
@@ -3,12 +3,12 @@
3
3
  import { BadRequestError } from '../../base/errors/api-errors.js';
4
4
  import { assertCallerIds } from '../../api/prompt-hook.js';
5
5
  import { assertCallerProject, projectNames } from '../../base/scope/project-identity.js';
6
- import { storedTextKeys } from '../../base/util/same-text.js';
7
6
  import { scrubForSharing } from './share-scrub.js';
8
- import { captureExtractedItems } from './command.js';
7
+ import { captureExtractedItems, heldKeysOf, scrubbedItems } from './command.js';
9
8
  import { detachedEmbedder } from '../../api/embeddings.js';
10
- import { extractFromTexts } from './extract.js';
9
+ import { boldTwins, extractFromTexts } from './extract.js';
11
10
  import { withEntryInfra } from '../entry-infra.js';
11
+ import { withDetached } from '../../api/detached.js';
12
12
  const MAX_TEXTS = 30;
13
13
  const MAX_TEXT_BYTES = 32 * 1024;
14
14
  const MAX_TOTAL_BYTES = 256 * 1024;
@@ -39,7 +39,9 @@ function assertSessionCapture(req) {
39
39
  export async function captureSessionTexts(given, req) {
40
40
  const ctx = withEntryInfra(given);
41
41
  assertSessionCapture(req);
42
- const extracted = extractFromTexts(req.texts.map(scrubForSharing));
42
+ const scrubbed = req.texts.map(scrubForSharing);
43
+ // Scrubbed again after extraction: bold removal can rebuild an address or key.
44
+ const extracted = scrubbedItems(extractFromTexts(scrubbed), scrubForSharing);
43
45
  // The newest items win, since the end of a session states where it landed.
44
46
  const items = extracted.slice(-MAX_ITEMS);
45
47
  if (items.length === 0)
@@ -47,13 +49,11 @@ export async function captureSessionTexts(given, req) {
47
49
  // The project filter is classifyOriginProject's in SQL: the caller's rows and user-global rows
48
50
  // block a copy, other projects' rows do not.
49
51
  const dedup = { tenantId: ctx.tenantId, project: projectNames(req.project) };
50
- const held = await ctx.store.captures.textsHolding(dedup.tenantId, items.map((i) => i.content), { project: dedup.project });
52
+ const twins = boldTwins(scrubbed, scrubForSharing);
53
+ const held = await ctx.store.captures.textsHolding(dedup.tenantId, [...items.map((i) => i.content), ...twins.map((twin) => twin.old)], { project: dedup.project });
51
54
  const { store, config } = ctx;
52
- const tally = await captureExtractedItems({
53
- store,
54
- config,
55
- embed: detachedEmbedder({ store, config, embedderFor: ctx.infra.embedderFor }, ctx.infra),
56
- }, {
55
+ const folder = { store, config, embedderFor: ctx.infra.embedderFor };
56
+ const tally = await withDetached(ctx, (detached) => captureExtractedItems({ store, config, embed: detachedEmbedder(folder, ctx.infra, detached) }, {
57
57
  tenantId: ctx.tenantId,
58
58
  originProject: req.project,
59
59
  sessionId: req.sessionId,
@@ -61,7 +61,7 @@ export async function captureSessionTexts(given, req) {
61
61
  lean: true,
62
62
  dedup,
63
63
  now: ctx.infra.clock(),
64
- }, items, storedTextKeys(held));
64
+ }, items, heldKeysOf(held, twins)));
65
65
  return { ...tally, skipped: tally.skipped + extracted.length - items.length };
66
66
  }
67
67
  //# sourceMappingURL=session-texts.js.map
@@ -11,6 +11,7 @@ import type { OptionalGroup } from '../../base/ports/store.js';
11
11
  import { type StoreSession } from '../open-store.js';
12
12
  import type { RerankerFn } from '../../base/ports/reranker.js';
13
13
  import { type CallerState } from '../../api/callers.js';
14
+ import type { DetachedWork } from '../../domain/search/detached-work.js';
14
15
  /** The CLI process's config reader: the stdio MCP server lives as long as its host, so each call
15
16
  * parses an unchanged config.json once. */
16
17
  export declare const cliConfig: ConfigReader;
@@ -49,6 +50,10 @@ export declare function cliStoreBatch(hippoRoot: string): <T>(run: (store: Hippo
49
50
  /** The session a scoped CLI verb runs in, one handle per store; named here so no other surface
50
51
  * opens one. */
51
52
  export declare function cliRequest<T>(run: () => T | Promise<T>): Promise<T>;
53
+ /** Where a verb hands work it does not wait on, such as an embed: the command's runtime, which
54
+ * settles it before the command ends. Outside a command no runtime closes under it, so it runs on
55
+ * unowned, as before. */
56
+ export declare function cliDetached(): DetachedWork;
52
57
  /** The CLI's store is hippo.db, which sets every optional group, so its verbs call the api
53
58
  * unchecked; the folder rides beside the api's Context for the verbs that reopen it. */
54
59
  export type CliContext = ContextWith<OptionalGroup> & {
@@ -15,6 +15,7 @@ import { createCallerState, newRowsOn, } from '../../api/callers.js';
15
15
  import { rootInfra } from '../../infra/ports.js';
16
16
  import { createHost } from '../../host/host.js';
17
17
  import { serverNearestAsk } from './server-nearest.js';
18
+ import { DetachedSet } from '../../base/util/detached-set.js';
18
19
  /** The CLI process's config reader: the stdio MCP server lives as long as its host, so each call
19
20
  * parses an unchanged config.json once. */
20
21
  export const cliConfig = configReader(envRuntimeSettings);
@@ -108,6 +109,12 @@ export function cliStoreBatch(hippoRoot) {
108
109
  export function cliRequest(run) {
109
110
  return runStoreRequest((session) => asCommand(session, run), 'cli', cliRuntime());
110
111
  }
112
+ /** Where a verb hands work it does not wait on, such as an embed: the command's runtime, which
113
+ * settles it before the command ends. Outside a command no runtime closes under it, so it runs on
114
+ * unowned, as before. */
115
+ export function cliDetached() {
116
+ return cliRuntime() ?? new DetachedSet();
117
+ }
111
118
  /** The stores a CLI call on `hippoRoot` reaches, each global one with its config and on the
112
119
  * call's session. */
113
120
  function cliReach(hippoRoot, store, embedderFor, session = commandSession) {
@@ -137,6 +144,7 @@ export function cliApiContext(hippoRoot, tenantId, callers = createCallerState()
137
144
  config,
138
145
  callers,
139
146
  infra,
147
+ detached: cliRuntime(),
140
148
  };
141
149
  }
142
150
  /** A second store a CLI read searches beside its own, with that store's config. */
@@ -167,6 +175,8 @@ export function cliStdioHost() {
167
175
  const hostTenantId = envTenant();
168
176
  const shared = createCallerState();
169
177
  const tools = allMcpTools();
178
+ // The command's runtime under `hippo mcp`; a host built outside a command keeps its own.
179
+ const detached = cliRuntime() ?? new DetachedSet();
170
180
  // The rings span the loop; each folder keeps an auto-sleep and its latch, as a write's context
171
181
  // names no folder, and each frame counts on its own store.
172
182
  const byRoot = new Map();
@@ -196,12 +206,14 @@ export function cliStdioHost() {
196
206
  callers: callersOn(hippoRoot, store),
197
207
  infra,
198
208
  signal,
209
+ detached,
199
210
  };
200
211
  };
201
212
  // Frames interleave, so each takes a session of its own and hands it to every store its context
202
213
  // builds.
203
214
  return {
204
215
  runFrame: (signal, run) => runStoreRequest((session) => run(contextIn(session, signal)), 'cli', cliRuntime()),
216
+ settle: (withinMs) => detached.settle(withinMs),
205
217
  };
206
218
  }
207
219
  //# sourceMappingURL=api-context.js.map
@@ -4,6 +4,7 @@ import { DEFAULT_TENANT_ID } from '../../base/util/tenant.js';
4
4
  import * as api from '../../api/index.js';
5
5
  import { cliApiContext } from './api-context.js';
6
6
  import { printError } from './output.js';
7
+ import { sanitizeLogMessage } from '../capture/compact.js';
7
8
  import { boolFlag, flagValueless, stringFlag, stringFlagOrExit, } from './flag-values.js';
8
9
  import { fail } from './exit.js';
9
10
  // `hippo goal <push|list|complete|suspend|resume>`
@@ -16,10 +17,6 @@ const GOAL_POLICY_TYPES = [
16
17
  function isGoalPolicyType(value) {
17
18
  return GOAL_POLICY_TYPES.some((type) => type === value);
18
19
  }
19
- function sanitizeGoalName(s) {
20
- // Strip C0 control chars + DEL to prevent terminal escape injection.
21
- return s.replace(/[^\x20-\x7e\u0080-\uffff]/g, '?');
22
- }
23
20
  function resolveGoalSession(flags, defaultTenantId) {
24
21
  const sessionId = (stringFlagOrExit(flags, 'session-id') ?? envHippoSessionId() ?? '').trim();
25
22
  if (!sessionId) {
@@ -55,11 +52,12 @@ function readGoalLevel(flags) {
55
52
  }
56
53
  async function pushGoal(hippoRoot, defaultTenantId, args, flags) {
57
54
  const rawName = args.join(' ').trim();
58
- if (!rawName) {
55
+ // Sanitize at WRITE time so corrupt names never enter the DB, and before the empty check so a
56
+ // name of control characters alone is no name.
57
+ const name = sanitizeLogMessage(rawName).trim();
58
+ if (!name) {
59
59
  fail('Usage: hippo goal push <name> [--policy <type>] [--success "<condition>"] [--level N] [--parent <goalId>]');
60
60
  }
61
- // Sanitize at WRITE time so corrupt names never enter the DB.
62
- const name = sanitizeGoalName(rawName);
63
61
  if (name !== rawName) {
64
62
  printError('note: stripped control characters from goal name');
65
63
  }
@@ -97,7 +95,7 @@ async function listGoals(hippoRoot, defaultTenantId, flags) {
97
95
  const rows = goals.map((g) => ({
98
96
  id: g.id,
99
97
  status: g.status,
100
- name: sanitizeGoalName(g.goalName),
98
+ name: sanitizeLogMessage(g.goalName),
101
99
  outcome: g.outcomeScore !== undefined ? g.outcomeScore.toString() : '-',
102
100
  }));
103
101
  const widths = {
@@ -5,7 +5,7 @@ import * as path from 'node:path';
5
5
  import * as fs from 'node:fs';
6
6
  import { bookTokenUse } from '../../api/ledger-db.js';
7
7
  import { sessionPilotArm } from '../../api/pilot-arm.js';
8
- import { cliFolder, cliInfra, cliLedgerReach, cliSyncOptions } from './api-context.js';
8
+ import { cliDetached, cliFolder, cliInfra, cliLedgerReach, cliSyncOptions } from './api-context.js';
9
9
  import { detachedEmbedder } from '../../api/embeddings.js';
10
10
  import { hookPayloadSessionId, hookPayloadString, isSubagentPayload, } from '../../base/util/hook-payload.js';
11
11
  import { importAtSessionEnd } from '../../api/agent-memories.js';
@@ -156,7 +156,7 @@ export function captureTarget(root) {
156
156
  root,
157
157
  store: folder.store,
158
158
  config: folder.config,
159
- embed: detachedEmbedder(folder, cliInfra()),
159
+ embed: detachedEmbedder(folder, cliInfra(), cliDetached()),
160
160
  };
161
161
  }
162
162
  //# sourceMappingURL=hook-runtime.js.map