hippo-memory 3.0.0 → 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/api/context/context-plan.js +2 -4
- package/dist/api/context/context-select.js +12 -7
- package/dist/api/detached.d.ts +6 -0
- package/dist/api/detached.js +16 -0
- package/dist/api/embeddings.d.ts +4 -3
- package/dist/api/embeddings.js +4 -4
- package/dist/api/memories/promote.js +2 -1
- package/dist/api/memories/remember-local.d.ts +5 -4
- package/dist/api/memories/remember-local.js +13 -10
- package/dist/api/recall/rank-stages.js +2 -2
- package/dist/api/session-end.js +1 -0
- package/dist/api/sharing.js +9 -7
- package/dist/api/sleep/auto-sleep.d.ts +1 -1
- package/dist/api/sleep/auto-sleep.js +5 -3
- package/dist/api/sleep/learn.js +5 -2
- package/dist/api/sleep/sleep-run.js +4 -2
- package/dist/api/types.d.ts +4 -0
- package/dist/base/memory/entry.js +2 -2
- package/dist/base/memory/markdown-delimiters.d.ts +15 -0
- package/dist/base/memory/markdown-delimiters.js +94 -0
- package/dist/base/memory/memory-audit.js +4 -4
- package/dist/base/memory/memory-quality.d.ts +12 -0
- package/dist/base/memory/memory-quality.js +92 -3
- package/dist/base/ports/store/types.d.ts +2 -1
- package/dist/base/rank/prompt-recall.d.ts +16 -3
- package/dist/base/rank/prompt-recall.js +47 -7
- package/dist/base/util/detached-set.d.ts +8 -0
- package/dist/base/util/detached-set.js +35 -0
- package/dist/base/util/tokenize.d.ts +29 -0
- package/dist/base/util/tokenize.js +68 -5
- package/dist/base/util/version.d.ts +1 -1
- package/dist/base/util/version.js +1 -1
- package/dist/db/meta.d.ts +1 -0
- package/dist/db/meta.js +1 -0
- package/dist/db/store-runtime.d.ts +10 -3
- package/dist/db/store-runtime.js +19 -3
- package/dist/domain/consolidate/conflicts.js +2 -2
- package/dist/domain/learn/forward-claim-detector.js +4 -2
- package/dist/domain/learn/invalidation.js +5 -2
- package/dist/domain/search/bm25.js +3 -3
- package/dist/domain/search/detached-work.d.ts +6 -0
- package/dist/domain/search/detached-work.js +2 -0
- package/dist/domain/search/embed-memories.d.ts +4 -4
- package/dist/domain/search/embed-memories.js +4 -5
- package/dist/domain/sharing/global-sync.d.ts +8 -2
- package/dist/domain/sharing/global-sync.js +5 -5
- package/dist/domain/sharing/share.d.ts +4 -0
- package/dist/domain/sharing/share.js +2 -2
- package/dist/entry/eval-scripts.d.ts +1 -0
- package/dist/entry/sleep.d.ts +1 -0
- package/dist/index.d.ts +1 -1
- package/dist/server.d.ts +1 -0
- package/dist/store/rules/windows.d.ts +3 -0
- package/dist/store/rules/windows.js +3 -0
- package/dist/store/sqlite/audit/audit-maintenance.js +16 -13
- package/dist/store/sqlite/recall/candidates.js +1 -0
- package/dist/store/sqlite/recall/search-rows.d.ts +2 -2
- package/dist/store/sqlite/recall/search-rows.js +62 -26
- package/dist/store/sqlite/runtime/server-runtime.d.ts +6 -1
- package/dist/store/sqlite/runtime/server-runtime.js +15 -3
- package/dist/store/sqlite/runtime/worker-jobs.js +2 -2
- package/dist/store/sqlite/sharing/copy-rows.d.ts +1 -0
- package/dist/store/sqlite/sharing/copy-rows.js +2 -1
- package/dist/surface/capture/command.d.ts +8 -1
- package/dist/surface/capture/command.js +38 -8
- package/dist/surface/capture/compact.d.ts +3 -3
- package/dist/surface/capture/compact.js +4 -4
- package/dist/surface/capture/extract.d.ts +10 -0
- package/dist/surface/capture/extract.js +199 -26
- package/dist/surface/capture/session-texts.js +11 -11
- package/dist/surface/cli/api-context.d.ts +5 -0
- package/dist/surface/cli/api-context.js +12 -0
- package/dist/surface/cli/goals.js +6 -8
- package/dist/surface/cli/hook-runtime.js +2 -2
- package/dist/surface/cli/memory-import.js +20 -15
- package/dist/surface/mcp/protocol.d.ts +6 -0
- package/dist/surface/mcp/protocol.js +1 -0
- package/dist/surface/mcp/stdio.js +11 -5
- package/dist/surface/open-store.d.ts +2 -2
- package/dist/surface/open-store.js +11 -7
- package/dist/surface/server/auth.d.ts +1 -1
- package/dist/surface/server/auth.js +13 -2
- package/dist/surface/server/boot.js +3 -2
- package/dist/surface/server/dashboard-listener.js +2 -2
- package/dist/surface/server/lifecycle.d.ts +2 -0
- package/dist/surface/server/lifecycle.js +4 -2
- package/dist/surface/server/mcp-http.js +1 -0
- package/dist/surface/server/types.d.ts +3 -0
- package/extensions/openclaw-plugin/openclaw.plugin.json +1 -1
- package/extensions/openclaw-plugin/package.json +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -2,13 +2,14 @@ import * as fs from 'node:fs';
|
|
|
2
2
|
import * as path from 'node:path';
|
|
3
3
|
import { createMemory, Layer } from '../../base/memory/entry.js';
|
|
4
4
|
import { duplicateKey, storedTextKeys } from '../../base/util/same-text.js';
|
|
5
|
+
import { isCertainReason, storedDefect } from '../../base/memory/memory-quality.js';
|
|
5
6
|
import { maskEmails, redactSecretsStrict } from '../../base/util/secret-detect.js';
|
|
6
7
|
import { projectId, projectNames } from '../../base/scope/project-identity.js';
|
|
7
8
|
import { isInitialized } from '../../host/project-identity.js';
|
|
8
9
|
import { errorMessage, log, errorFields } from '../../base/util/log.js';
|
|
9
10
|
import { teeStdStreams } from '../../host/stream-tee.js';
|
|
10
11
|
import { ALL_TENANTS } from '../../base/util/tenant.js';
|
|
11
|
-
import { extractFromTexts } from './extract.js';
|
|
12
|
+
import { boldTwins, extractFromTexts } from './extract.js';
|
|
12
13
|
import { collectSessionTurns, sessionTail, resolveLastSessionTranscript, } from './transcript.js';
|
|
13
14
|
import { PRIVATE_DIR } from '../../host/private-dir.js';
|
|
14
15
|
import { evalClock } from '../../infra/eval-switches.js';
|
|
@@ -71,14 +72,16 @@ async function runCaptureCore(target, options) {
|
|
|
71
72
|
console.log('No text to capture from.');
|
|
72
73
|
return;
|
|
73
74
|
}
|
|
74
|
-
// Scrub
|
|
75
|
-
// no secrets in memories).
|
|
76
|
-
const
|
|
75
|
+
// Scrub the texts and again each item: bold removal can rebuild an address or key the first
|
|
76
|
+
// pass could not see (AGENTS.md: no secrets in memories).
|
|
77
|
+
const scrub = (text) => maskEmails(redactSecretsStrict(text));
|
|
78
|
+
const scrubbed = texts.map(scrub);
|
|
79
|
+
const extracted = scrubbedItems(extractFromTexts(scrubbed, userTexts), scrub);
|
|
77
80
|
if (extracted.length === 0) {
|
|
78
81
|
console.log('No actionable items found in the input.');
|
|
79
82
|
return;
|
|
80
83
|
}
|
|
81
|
-
const keys = await storedKeysInView(target.store, options, extracted);
|
|
84
|
+
const keys = await storedKeysInView(target.store, options, extracted, boldTwins(scrubbed, scrub));
|
|
82
85
|
printCaptureTally(options, await captureFromCli(target, options, extracted, keys));
|
|
83
86
|
}
|
|
84
87
|
async function captureFromCli({ store, config, embed }, options, extracted, keys) {
|
|
@@ -94,12 +97,39 @@ async function captureFromCli({ store, config, embed }, options, extracted, keys
|
|
|
94
97
|
}
|
|
95
98
|
return previewCapture({ store, config }, writeOpts, extracted, keys);
|
|
96
99
|
}
|
|
100
|
+
/** The items with `scrub` applied to each text, as a path stores them; scrubbing can make two
|
|
101
|
+
* items one, and the first stays. */
|
|
102
|
+
export function scrubbedItems(items, scrub) {
|
|
103
|
+
const seen = new Set();
|
|
104
|
+
return items
|
|
105
|
+
.map((item) => ({ ...item, content: scrub(item.content) }))
|
|
106
|
+
.filter((item) => {
|
|
107
|
+
const key = duplicateKey(item.content);
|
|
108
|
+
if (seen.has(key))
|
|
109
|
+
return false;
|
|
110
|
+
seen.add(key);
|
|
111
|
+
return true;
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
/** Held keys, each twin's clean text too when a held row has its old text and repair keeps that
|
|
115
|
+
* row: a pinned row, or one whose text is no certain defect. */
|
|
116
|
+
export function heldKeysOf(rows, twins) {
|
|
117
|
+
const keys = storedTextKeys(rows);
|
|
118
|
+
const pinned = storedTextKeys(rows.filter((row) => row.pinned));
|
|
119
|
+
const clean = twins
|
|
120
|
+
.filter(({ old }) => (isCertainReason(storedDefect(old)) ? pinned : keys).has(duplicateKey(old)))
|
|
121
|
+
.map((twin) => duplicateKey(twin.clean));
|
|
122
|
+
for (const key of clean)
|
|
123
|
+
keys.add(key);
|
|
124
|
+
return keys;
|
|
125
|
+
}
|
|
97
126
|
// Dedup only against rows this capture's reader sees: another tenant's or project's rows are
|
|
98
127
|
// hidden, so they must not stop its own copy.
|
|
99
|
-
async function storedKeysInView(store, options, extracted) {
|
|
128
|
+
async function storedKeysInView(store, options, extracted, twins) {
|
|
100
129
|
const { tenantId, ...filter } = captureDedup(options);
|
|
101
|
-
|
|
102
|
-
|
|
130
|
+
// The lookup matches stored text exactly, so it also asks for the lines as capture once stored them.
|
|
131
|
+
const texts = [...extracted.map((item) => item.content), ...twins.map((twin) => twin.old)];
|
|
132
|
+
return heldKeysOf(await store.captures.textsHolding(tenantId, texts, filter), twins);
|
|
103
133
|
}
|
|
104
134
|
/** The rows a capture's dedup reads, as data, so the store's write can read them again in its
|
|
105
135
|
* own transaction. */
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { type HookRuntime } from '../../base/session/capture-contract.js';
|
|
2
2
|
import type { StoreGroups } from '../../base/ports/store.js';
|
|
3
|
-
/** Log-forgery guard: strip
|
|
4
|
-
* so a crafted value cannot inject fake `[hippo]`
|
|
5
|
-
* for every
|
|
3
|
+
/** Log-forgery guard: strip control chars (U+0000-U+001F, DEL, U+0080-U+009F; newlines and NEL
|
|
4
|
+
* above all) so a crafted value cannot inject fake `[hippo]` lines or terminal escapes. The one
|
|
5
|
+
* shared guard for every log line and printed name that interpolates payload values. */
|
|
6
6
|
export declare function sanitizeLogMessage(message: string): string;
|
|
7
7
|
/** The folder's store as a compaction hook reaches it, built by the CLI root. */
|
|
8
8
|
export type CompactionStore = Pick<StoreGroups, 'compactionHooks' | 'sessions'>;
|
|
@@ -15,12 +15,12 @@ import { transcriptWorkingState } from './working-state.js';
|
|
|
15
15
|
import { PRIVATE_DIR } from '../../host/private-dir.js';
|
|
16
16
|
// Diagnostic-only log; a long-lived install must not grow it unbounded.
|
|
17
17
|
const PRE_COMPACT_LOG_MAX_BYTES = 256 * 1024;
|
|
18
|
-
/** Log-forgery guard: strip
|
|
19
|
-
* so a crafted value cannot inject fake `[hippo]`
|
|
20
|
-
* for every
|
|
18
|
+
/** Log-forgery guard: strip control chars (U+0000-U+001F, DEL, U+0080-U+009F; newlines and NEL
|
|
19
|
+
* above all) so a crafted value cannot inject fake `[hippo]` lines or terminal escapes. The one
|
|
20
|
+
* shared guard for every log line and printed name that interpolates payload values. */
|
|
21
21
|
export function sanitizeLogMessage(message) {
|
|
22
22
|
// eslint-disable-next-line no-control-regex
|
|
23
|
-
return message.replace(/[\x00-\x1f]/g, '');
|
|
23
|
+
return message.replace(/[\x00-\x1f\x7f-\x9f]/g, '');
|
|
24
24
|
}
|
|
25
25
|
function appendPreCompactLog(logFile, message, fields) {
|
|
26
26
|
try {
|
|
@@ -8,4 +8,14 @@ export interface ExtractedItem {
|
|
|
8
8
|
/** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
|
|
9
9
|
* The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`. */
|
|
10
10
|
export declare function extractFromTexts(texts: readonly string[], userTexts?: number): ExtractedItem[];
|
|
11
|
+
/** A text an older capture, which kept every bold marker, stored for a line, and what capture
|
|
12
|
+
* extracts from that line now. */
|
|
13
|
+
export interface BoldTwin {
|
|
14
|
+
readonly old: string;
|
|
15
|
+
readonly clean: string;
|
|
16
|
+
}
|
|
17
|
+
/** The twins of `texts`: each old item pairs, one to one, with the new item of the same span, the
|
|
18
|
+
* same category first and then the larger overlap; a pair whose texts differ. A new item that
|
|
19
|
+
* holds more or less than the old one is no twin, or a held row would block a rule it lacks. */
|
|
20
|
+
export declare function boldTwins(texts: readonly string[], scrub: (text: string) => string): BoldTwin[];
|
|
11
21
|
//# sourceMappingURL=extract.d.ts.map
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { duplicateKey } from '../../base/util/same-text.js';
|
|
2
|
-
import { assessAutomaticMemory } from '../../base/memory/memory-quality.js';
|
|
2
|
+
import { assessAutomaticMemory, asideCloseOffset, QUOTE_MARK, } from '../../base/memory/memory-quality.js';
|
|
3
3
|
import { ERROR_TAG } from '../../base/memory/entry.js';
|
|
4
|
+
import { blankCodeSpans, strongMarkers } from '../../base/memory/markdown-delimiters.js';
|
|
4
5
|
const DECISION_PATTERNS = [
|
|
5
6
|
/\b(?:we(?:'ve| have)?|i(?:'ve| have)?|let's)\s+decid(?:ed|e)\b/i,
|
|
6
7
|
/\b(?:let's|we(?:'ll| will| should)?)\s+(?:go with|do|use|try|build|implement|switch to)\b/i,
|
|
@@ -85,32 +86,139 @@ function continues(line, previous) {
|
|
|
85
86
|
return false;
|
|
86
87
|
return !SENTENCE_END.test(masked(`${previous} ${line}`).slice(0, previous.length));
|
|
87
88
|
}
|
|
88
|
-
|
|
89
|
+
/** What capture read before it removed bold markers. */
|
|
90
|
+
const keepAll = () => [];
|
|
91
|
+
/** Offsets of each `**` that pairs with the nearest opener before it. */
|
|
92
|
+
function pairedBold(line) {
|
|
93
|
+
const paired = [];
|
|
94
|
+
const openers = [];
|
|
95
|
+
// Blanked code spans keep every offset, so the markers found here are cut from the source.
|
|
96
|
+
for (const marker of strongMarkers(blankCodeSpans(line, '`'))) {
|
|
97
|
+
if (marker.inWord)
|
|
98
|
+
continue;
|
|
99
|
+
const opener = marker.close ? openers.pop() : undefined;
|
|
100
|
+
if (opener !== undefined)
|
|
101
|
+
paired.push(opener, marker.index);
|
|
102
|
+
else if (marker.open)
|
|
103
|
+
openers.push(marker.index);
|
|
104
|
+
}
|
|
105
|
+
return paired;
|
|
106
|
+
}
|
|
107
|
+
/** The segmenter splits after "**Label.**" and leaves the closing `**` on the next rule, so each
|
|
108
|
+
* real pair goes; a glob, `2**10` or `**kwargs` has no partner and stays as written. */
|
|
109
|
+
const boldPairs = (line) => pairedBold(line).sort((a, b) => a - b);
|
|
110
|
+
/** `line` without the two characters at each of the sorted `cuts`, built in one pass. */
|
|
111
|
+
function cutOut(line, cuts) {
|
|
112
|
+
let out = '';
|
|
113
|
+
let kept = 0;
|
|
114
|
+
for (const at of cuts) {
|
|
115
|
+
out += line.slice(kept, at);
|
|
116
|
+
kept = at + 2;
|
|
117
|
+
}
|
|
118
|
+
return out + line.slice(kept);
|
|
119
|
+
}
|
|
120
|
+
/** How many of the sorted `values` are at most `limit`. */
|
|
121
|
+
function countUpTo(values, limit) {
|
|
122
|
+
let low = 0;
|
|
123
|
+
let high = values.length;
|
|
124
|
+
while (low < high) {
|
|
125
|
+
const middle = (low + high) >> 1;
|
|
126
|
+
if (values[middle] <= limit)
|
|
127
|
+
low = middle + 1;
|
|
128
|
+
else
|
|
129
|
+
high = middle;
|
|
130
|
+
}
|
|
131
|
+
return low;
|
|
132
|
+
}
|
|
133
|
+
function blockOf(pieces, cuts) {
|
|
134
|
+
const raw = pieces.map((piece) => piece.text).join(' ');
|
|
135
|
+
const cutAt = cuts(raw);
|
|
136
|
+
// The sentences are those 3.1.0 split from the uncut text, so a removed `**` merges none: ICU
|
|
137
|
+
// breaks "**.Always" but not "n.Always". Masking keeps offsets.
|
|
138
|
+
const cutEnds = cutAt.map((at) => at + 2);
|
|
139
|
+
const sentenceStarts = [
|
|
140
|
+
...new Intl.Segmenter('en', { granularity: 'sentence' }).segment(masked(raw)),
|
|
141
|
+
].map(({ index }) => index - 2 * countUpTo(cutEnds, index));
|
|
142
|
+
// Where each cut sits in the cut text, and where each piece starts in the raw text.
|
|
143
|
+
const shifted = cutAt.map((at, k) => at - 2 * k);
|
|
144
|
+
const starts = [];
|
|
145
|
+
let sum = 0;
|
|
146
|
+
for (const piece of pieces) {
|
|
147
|
+
starts.push(sum);
|
|
148
|
+
sum += piece.text.length + 1;
|
|
149
|
+
}
|
|
150
|
+
const toProse = (rawIndex) => {
|
|
151
|
+
const k = Math.max(0, countUpTo(starts, rawIndex) - 1);
|
|
152
|
+
return pieces[k].at + Math.min(rawIndex - starts[k], pieces[k].text.length);
|
|
153
|
+
};
|
|
154
|
+
return {
|
|
155
|
+
text: cutOut(raw, cutAt),
|
|
156
|
+
sentenceStarts,
|
|
157
|
+
sourceOf: (from, to) => [
|
|
158
|
+
toProse(from + 2 * countUpTo(shifted, from)),
|
|
159
|
+
toProse(to + 2 * countUpTo(shifted, to - 1)),
|
|
160
|
+
],
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
function proseBlocks(prose, cuts) {
|
|
89
164
|
const blocks = [];
|
|
90
165
|
let pending = [];
|
|
166
|
+
// Cut once the lines are joined, as a spec bullet is, so a bold pair that wraps a line goes whole.
|
|
91
167
|
const flush = () => {
|
|
92
168
|
if (pending.length > 0)
|
|
93
|
-
blocks.push(pending
|
|
169
|
+
blocks.push(blockOf(pending, cuts));
|
|
94
170
|
pending = [];
|
|
95
171
|
};
|
|
96
|
-
|
|
172
|
+
let offset = 0;
|
|
173
|
+
for (const line of prose.split('\n')) {
|
|
97
174
|
const trimmed = line.trim();
|
|
175
|
+
const end = offset + line.trimEnd().length;
|
|
176
|
+
offset += line.length + 1;
|
|
98
177
|
if (!trimmed || /^#+\s/.test(trimmed)) {
|
|
99
178
|
flush();
|
|
100
179
|
continue;
|
|
101
180
|
}
|
|
102
181
|
const bullet = trimmed.match(BULLET);
|
|
103
|
-
if (bullet || !continues(trimmed, pending.at(-1)))
|
|
182
|
+
if (bullet || !continues(trimmed, pending.at(-1)?.text))
|
|
104
183
|
flush();
|
|
105
|
-
|
|
184
|
+
const text = bullet?.[1] ?? trimmed;
|
|
185
|
+
pending.push({ text, at: end - text.length });
|
|
106
186
|
}
|
|
107
187
|
flush();
|
|
108
188
|
return blocks;
|
|
109
189
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
190
|
+
/** Each sentence of a block, trimmed, with where it starts in the block text. */
|
|
191
|
+
function completeSentences({ text, sentenceStarts, }) {
|
|
192
|
+
return sentenceStarts.map((start, k) => {
|
|
193
|
+
const piece = text.slice(start, sentenceStarts[k + 1] ?? text.length);
|
|
194
|
+
return { sentence: piece.trim(), at: start + piece.length - piece.trimStart().length };
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
// A "(" after ":" or ";" is an emoticon, and one with a quote mark on both sides is quoted text.
|
|
198
|
+
function opensAside(text, at) {
|
|
199
|
+
const before = text[at - 1] ?? '';
|
|
200
|
+
const quoted = QUOTE_MARK.test(before) && QUOTE_MARK.test(text[at + 1] ?? '');
|
|
201
|
+
return text[at] === '(' && !/[:;]/.test(before) && !quoted;
|
|
202
|
+
}
|
|
203
|
+
// A sentence starts inside a cut aside: an unclosed "(" before it, an aside-closing ")" in it, and
|
|
204
|
+
// a word after that ")".
|
|
205
|
+
function asideCutter(block) {
|
|
206
|
+
const blanked = blankCodeSpans(block);
|
|
207
|
+
let depth = 0;
|
|
208
|
+
let seen = 0;
|
|
209
|
+
return (sentence, at) => {
|
|
210
|
+
while (seen < at) {
|
|
211
|
+
if (opensAside(blanked, seen))
|
|
212
|
+
depth++;
|
|
213
|
+
else if (blanked[seen] === ')')
|
|
214
|
+
depth = Math.max(0, depth - 1);
|
|
215
|
+
seen++;
|
|
216
|
+
}
|
|
217
|
+
if (depth === 0)
|
|
218
|
+
return false;
|
|
219
|
+
const close = asideCloseOffset(sentence);
|
|
220
|
+
return close >= 0 && /[\p{L}\p{N}]/u.test(sentence.slice(close + 1));
|
|
221
|
+
};
|
|
114
222
|
}
|
|
115
223
|
function insideBrackets(lead) {
|
|
116
224
|
return (lead.match(/[([{]/g) ?? []).length > (lead.match(/[)\]}]/g) ?? []).length;
|
|
@@ -128,12 +236,15 @@ function extractFromPatterns(sentence, patterns, category) {
|
|
|
128
236
|
.trim();
|
|
129
237
|
return content.length <= MAX_CHARS ? { content, category, tags: [category, 'captured'] } : null;
|
|
130
238
|
}
|
|
131
|
-
function extractSpecSections(
|
|
239
|
+
function extractSpecSections(prose, cuts) {
|
|
132
240
|
const bullets = [];
|
|
133
241
|
let inSpecSection = false;
|
|
134
242
|
let open = false;
|
|
135
|
-
|
|
243
|
+
let offset = 0;
|
|
244
|
+
for (const line of prose.split('\n')) {
|
|
136
245
|
const trimmed = line.trim();
|
|
246
|
+
const end = offset + line.trimEnd().length;
|
|
247
|
+
offset += line.length + 1;
|
|
137
248
|
if (SPEC_HEADING_PATTERNS.some((pattern) => pattern.test(trimmed))) {
|
|
138
249
|
inSpecSection = true;
|
|
139
250
|
open = false;
|
|
@@ -142,46 +253,62 @@ function extractSpecSections(text) {
|
|
|
142
253
|
if (/^#+\s/.test(trimmed) || /^[A-Z][a-z]+:$/.test(trimmed))
|
|
143
254
|
inSpecSection = false;
|
|
144
255
|
const bullet = trimmed.match(BULLET);
|
|
256
|
+
const last = bullets.at(-1);
|
|
145
257
|
const wrapped = inSpecSection &&
|
|
146
258
|
open &&
|
|
147
259
|
!bullet &&
|
|
148
260
|
trimmed !== '' &&
|
|
149
|
-
(/^\s/.test(line) || continues(trimmed,
|
|
150
|
-
if (inSpecSection && bullet)
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
261
|
+
(/^\s/.test(line) || continues(trimmed, last?.text));
|
|
262
|
+
if (inSpecSection && bullet) {
|
|
263
|
+
const text = bullet[1].trim();
|
|
264
|
+
bullets.push({ text, at: end - text.length, to: end });
|
|
265
|
+
}
|
|
266
|
+
else if (wrapped && last) {
|
|
267
|
+
last.text += ` ${trimmed}`;
|
|
268
|
+
last.to = end;
|
|
269
|
+
}
|
|
154
270
|
open = inSpecSection && (bullet !== null || wrapped);
|
|
155
271
|
}
|
|
272
|
+
// Cut once the lines are joined, so a pair that wraps a line goes whole.
|
|
156
273
|
return bullets
|
|
157
|
-
.
|
|
158
|
-
.
|
|
274
|
+
.map(({ text, at, to }) => ({ content: cutOut(text, cuts(text)), from: at, to }))
|
|
275
|
+
.filter(({ content }) => content.length <= MAX_CHARS && !content.endsWith('?'))
|
|
276
|
+
.map(({ content, from, to }) => ({
|
|
277
|
+
item: { content, category: 'spec', tags: ['spec', 'captured'] },
|
|
278
|
+
from,
|
|
279
|
+
to,
|
|
280
|
+
}));
|
|
159
281
|
}
|
|
160
|
-
|
|
282
|
+
// The items of the spec bullets first, then those of each prose block.
|
|
283
|
+
function extractSourced(text, cuts) {
|
|
161
284
|
const prose = proseOnly(text);
|
|
162
|
-
const
|
|
163
|
-
for (const block of proseBlocks(prose)) {
|
|
164
|
-
|
|
285
|
+
const found = extractSpecSections(prose, cuts);
|
|
286
|
+
for (const block of proseBlocks(prose, cuts)) {
|
|
287
|
+
const cutByAside = asideCutter(block.text);
|
|
288
|
+
for (const { sentence, at } of completeSentences(block)) {
|
|
165
289
|
// A question asks rather than states; far over the bound, skip before any pattern runs.
|
|
166
290
|
if (sentence.endsWith('?') || sentence.length > 2 * MAX_CHARS)
|
|
167
291
|
continue;
|
|
292
|
+
if (cutByAside(sentence, at))
|
|
293
|
+
continue;
|
|
168
294
|
for (const [category, patterns] of CATEGORIES) {
|
|
169
295
|
const item = extractFromPatterns(sentence, patterns, category);
|
|
170
296
|
if (item) {
|
|
171
|
-
|
|
297
|
+
const [from, to] = block.sourceOf(at, at + sentence.length);
|
|
298
|
+
found.push({ item, from, to });
|
|
172
299
|
break;
|
|
173
300
|
}
|
|
174
301
|
}
|
|
175
302
|
}
|
|
176
303
|
}
|
|
177
|
-
return
|
|
304
|
+
return found;
|
|
178
305
|
}
|
|
179
306
|
/** Captures complete supported statements, each text parsed alone so a fence or heading in one never reaches the next.
|
|
180
307
|
* The first `userTexts` texts are a person's own words; on a repeat their copy wins, so it keeps `fromUser`. */
|
|
181
308
|
export function extractFromTexts(texts, userTexts = 0) {
|
|
182
309
|
const seen = new Set();
|
|
183
310
|
return texts
|
|
184
|
-
.flatMap((text, i) =>
|
|
311
|
+
.flatMap((text, i) => extractSourced(text, boldPairs).map(({ item }) => i < userTexts ? { ...item, fromUser: true } : item))
|
|
185
312
|
.filter((item) => {
|
|
186
313
|
const key = duplicateKey(item.content);
|
|
187
314
|
if (seen.has(key) || !assessAutomaticMemory(item.content).accepted)
|
|
@@ -190,4 +317,50 @@ export function extractFromTexts(texts, userTexts = 0) {
|
|
|
190
317
|
return true;
|
|
191
318
|
});
|
|
192
319
|
}
|
|
320
|
+
function trackOf(items) {
|
|
321
|
+
return { items, ends: items.map(({ to }) => to) };
|
|
322
|
+
}
|
|
323
|
+
/** The items of `track` whose range overlaps [from, to). */
|
|
324
|
+
function overlapping(track, from, to) {
|
|
325
|
+
const found = [];
|
|
326
|
+
for (let k = countUpTo(track.ends, from); k < track.items.length && track.items[k].from < to; k++) {
|
|
327
|
+
found.push(track.items[k]);
|
|
328
|
+
}
|
|
329
|
+
return found;
|
|
330
|
+
}
|
|
331
|
+
/** Whether each range covers the other, but for markers, spaces and sentence marks at the edges. */
|
|
332
|
+
function sameSpan(prose, a, b) {
|
|
333
|
+
const gap = (x, y) => prose.slice(Math.min(x, y), Math.max(x, y));
|
|
334
|
+
return /^[\s*.!?]*$/.test(gap(a.from, b.from)) && /^[\s*.!?]*$/.test(gap(a.to, b.to));
|
|
335
|
+
}
|
|
336
|
+
/** The twins of `texts`: each old item pairs, one to one, with the new item of the same span, the
|
|
337
|
+
* same category first and then the larger overlap; a pair whose texts differ. A new item that
|
|
338
|
+
* holds more or less than the old one is no twin, or a held row would block a rule it lacks. */
|
|
339
|
+
export function boldTwins(texts, scrub) {
|
|
340
|
+
const twins = [];
|
|
341
|
+
for (const text of texts.filter((t) => t.includes('**'))) {
|
|
342
|
+
const prose = proseOnly(text);
|
|
343
|
+
const fresh = extractSourced(text, boldPairs);
|
|
344
|
+
// Spec bullets and prose sentences overlap each other, so each kind keeps its own order.
|
|
345
|
+
const tracks = [trackOf(fresh.filter(({ item }) => item.category === 'spec'))];
|
|
346
|
+
tracks.push(trackOf(fresh.filter(({ item }) => item.category !== 'spec')));
|
|
347
|
+
const taken = new Set();
|
|
348
|
+
for (const sourced of extractSourced(text, keepAll)) {
|
|
349
|
+
const { item: old, from, to } = sourced;
|
|
350
|
+
const size = (found) => Math.min(found.to, to) - Math.max(found.from, from);
|
|
351
|
+
const same = (found) => Number(found.item.category === old.category);
|
|
352
|
+
const [match] = tracks
|
|
353
|
+
.flatMap((track) => overlapping(track, from, to))
|
|
354
|
+
.filter((found) => !taken.has(found) && sameSpan(prose, found, sourced))
|
|
355
|
+
.sort((a, b) => same(b) - same(a) || size(b) - size(a));
|
|
356
|
+
if (match === undefined)
|
|
357
|
+
continue;
|
|
358
|
+
taken.add(match);
|
|
359
|
+
if (old.content !== match.item.content) {
|
|
360
|
+
twins.push({ old: old.content, clean: scrub(match.item.content) });
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
return twins;
|
|
365
|
+
}
|
|
193
366
|
//# sourceMappingURL=extract.js.map
|
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
import { BadRequestError } from '../../base/errors/api-errors.js';
|
|
4
4
|
import { assertCallerIds } from '../../api/prompt-hook.js';
|
|
5
5
|
import { assertCallerProject, projectNames } from '../../base/scope/project-identity.js';
|
|
6
|
-
import { storedTextKeys } from '../../base/util/same-text.js';
|
|
7
6
|
import { scrubForSharing } from './share-scrub.js';
|
|
8
|
-
import { captureExtractedItems } from './command.js';
|
|
7
|
+
import { captureExtractedItems, heldKeysOf, scrubbedItems } from './command.js';
|
|
9
8
|
import { detachedEmbedder } from '../../api/embeddings.js';
|
|
10
|
-
import { extractFromTexts } from './extract.js';
|
|
9
|
+
import { boldTwins, extractFromTexts } from './extract.js';
|
|
11
10
|
import { withEntryInfra } from '../entry-infra.js';
|
|
11
|
+
import { withDetached } from '../../api/detached.js';
|
|
12
12
|
const MAX_TEXTS = 30;
|
|
13
13
|
const MAX_TEXT_BYTES = 32 * 1024;
|
|
14
14
|
const MAX_TOTAL_BYTES = 256 * 1024;
|
|
@@ -39,7 +39,9 @@ function assertSessionCapture(req) {
|
|
|
39
39
|
export async function captureSessionTexts(given, req) {
|
|
40
40
|
const ctx = withEntryInfra(given);
|
|
41
41
|
assertSessionCapture(req);
|
|
42
|
-
const
|
|
42
|
+
const scrubbed = req.texts.map(scrubForSharing);
|
|
43
|
+
// Scrubbed again after extraction: bold removal can rebuild an address or key.
|
|
44
|
+
const extracted = scrubbedItems(extractFromTexts(scrubbed), scrubForSharing);
|
|
43
45
|
// The newest items win, since the end of a session states where it landed.
|
|
44
46
|
const items = extracted.slice(-MAX_ITEMS);
|
|
45
47
|
if (items.length === 0)
|
|
@@ -47,13 +49,11 @@ export async function captureSessionTexts(given, req) {
|
|
|
47
49
|
// The project filter is classifyOriginProject's in SQL: the caller's rows and user-global rows
|
|
48
50
|
// block a copy, other projects' rows do not.
|
|
49
51
|
const dedup = { tenantId: ctx.tenantId, project: projectNames(req.project) };
|
|
50
|
-
const
|
|
52
|
+
const twins = boldTwins(scrubbed, scrubForSharing);
|
|
53
|
+
const held = await ctx.store.captures.textsHolding(dedup.tenantId, [...items.map((i) => i.content), ...twins.map((twin) => twin.old)], { project: dedup.project });
|
|
51
54
|
const { store, config } = ctx;
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
config,
|
|
55
|
-
embed: detachedEmbedder({ store, config, embedderFor: ctx.infra.embedderFor }, ctx.infra),
|
|
56
|
-
}, {
|
|
55
|
+
const folder = { store, config, embedderFor: ctx.infra.embedderFor };
|
|
56
|
+
const tally = await withDetached(ctx, (detached) => captureExtractedItems({ store, config, embed: detachedEmbedder(folder, ctx.infra, detached) }, {
|
|
57
57
|
tenantId: ctx.tenantId,
|
|
58
58
|
originProject: req.project,
|
|
59
59
|
sessionId: req.sessionId,
|
|
@@ -61,7 +61,7 @@ export async function captureSessionTexts(given, req) {
|
|
|
61
61
|
lean: true,
|
|
62
62
|
dedup,
|
|
63
63
|
now: ctx.infra.clock(),
|
|
64
|
-
}, items,
|
|
64
|
+
}, items, heldKeysOf(held, twins)));
|
|
65
65
|
return { ...tally, skipped: tally.skipped + extracted.length - items.length };
|
|
66
66
|
}
|
|
67
67
|
//# sourceMappingURL=session-texts.js.map
|
|
@@ -11,6 +11,7 @@ import type { OptionalGroup } from '../../base/ports/store.js';
|
|
|
11
11
|
import { type StoreSession } from '../open-store.js';
|
|
12
12
|
import type { RerankerFn } from '../../base/ports/reranker.js';
|
|
13
13
|
import { type CallerState } from '../../api/callers.js';
|
|
14
|
+
import type { DetachedWork } from '../../domain/search/detached-work.js';
|
|
14
15
|
/** The CLI process's config reader: the stdio MCP server lives as long as its host, so each call
|
|
15
16
|
* parses an unchanged config.json once. */
|
|
16
17
|
export declare const cliConfig: ConfigReader;
|
|
@@ -49,6 +50,10 @@ export declare function cliStoreBatch(hippoRoot: string): <T>(run: (store: Hippo
|
|
|
49
50
|
/** The session a scoped CLI verb runs in, one handle per store; named here so no other surface
|
|
50
51
|
* opens one. */
|
|
51
52
|
export declare function cliRequest<T>(run: () => T | Promise<T>): Promise<T>;
|
|
53
|
+
/** Where a verb hands work it does not wait on, such as an embed: the command's runtime, which
|
|
54
|
+
* settles it before the command ends. Outside a command no runtime closes under it, so it runs on
|
|
55
|
+
* unowned, as before. */
|
|
56
|
+
export declare function cliDetached(): DetachedWork;
|
|
52
57
|
/** The CLI's store is hippo.db, which sets every optional group, so its verbs call the api
|
|
53
58
|
* unchecked; the folder rides beside the api's Context for the verbs that reopen it. */
|
|
54
59
|
export type CliContext = ContextWith<OptionalGroup> & {
|
|
@@ -15,6 +15,7 @@ import { createCallerState, newRowsOn, } from '../../api/callers.js';
|
|
|
15
15
|
import { rootInfra } from '../../infra/ports.js';
|
|
16
16
|
import { createHost } from '../../host/host.js';
|
|
17
17
|
import { serverNearestAsk } from './server-nearest.js';
|
|
18
|
+
import { DetachedSet } from '../../base/util/detached-set.js';
|
|
18
19
|
/** The CLI process's config reader: the stdio MCP server lives as long as its host, so each call
|
|
19
20
|
* parses an unchanged config.json once. */
|
|
20
21
|
export const cliConfig = configReader(envRuntimeSettings);
|
|
@@ -108,6 +109,12 @@ export function cliStoreBatch(hippoRoot) {
|
|
|
108
109
|
export function cliRequest(run) {
|
|
109
110
|
return runStoreRequest((session) => asCommand(session, run), 'cli', cliRuntime());
|
|
110
111
|
}
|
|
112
|
+
/** Where a verb hands work it does not wait on, such as an embed: the command's runtime, which
|
|
113
|
+
* settles it before the command ends. Outside a command no runtime closes under it, so it runs on
|
|
114
|
+
* unowned, as before. */
|
|
115
|
+
export function cliDetached() {
|
|
116
|
+
return cliRuntime() ?? new DetachedSet();
|
|
117
|
+
}
|
|
111
118
|
/** The stores a CLI call on `hippoRoot` reaches, each global one with its config and on the
|
|
112
119
|
* call's session. */
|
|
113
120
|
function cliReach(hippoRoot, store, embedderFor, session = commandSession) {
|
|
@@ -137,6 +144,7 @@ export function cliApiContext(hippoRoot, tenantId, callers = createCallerState()
|
|
|
137
144
|
config,
|
|
138
145
|
callers,
|
|
139
146
|
infra,
|
|
147
|
+
detached: cliRuntime(),
|
|
140
148
|
};
|
|
141
149
|
}
|
|
142
150
|
/** A second store a CLI read searches beside its own, with that store's config. */
|
|
@@ -167,6 +175,8 @@ export function cliStdioHost() {
|
|
|
167
175
|
const hostTenantId = envTenant();
|
|
168
176
|
const shared = createCallerState();
|
|
169
177
|
const tools = allMcpTools();
|
|
178
|
+
// The command's runtime under `hippo mcp`; a host built outside a command keeps its own.
|
|
179
|
+
const detached = cliRuntime() ?? new DetachedSet();
|
|
170
180
|
// The rings span the loop; each folder keeps an auto-sleep and its latch, as a write's context
|
|
171
181
|
// names no folder, and each frame counts on its own store.
|
|
172
182
|
const byRoot = new Map();
|
|
@@ -196,12 +206,14 @@ export function cliStdioHost() {
|
|
|
196
206
|
callers: callersOn(hippoRoot, store),
|
|
197
207
|
infra,
|
|
198
208
|
signal,
|
|
209
|
+
detached,
|
|
199
210
|
};
|
|
200
211
|
};
|
|
201
212
|
// Frames interleave, so each takes a session of its own and hands it to every store its context
|
|
202
213
|
// builds.
|
|
203
214
|
return {
|
|
204
215
|
runFrame: (signal, run) => runStoreRequest((session) => run(contextIn(session, signal)), 'cli', cliRuntime()),
|
|
216
|
+
settle: (withinMs) => detached.settle(withinMs),
|
|
205
217
|
};
|
|
206
218
|
}
|
|
207
219
|
//# sourceMappingURL=api-context.js.map
|
|
@@ -4,6 +4,7 @@ import { DEFAULT_TENANT_ID } from '../../base/util/tenant.js';
|
|
|
4
4
|
import * as api from '../../api/index.js';
|
|
5
5
|
import { cliApiContext } from './api-context.js';
|
|
6
6
|
import { printError } from './output.js';
|
|
7
|
+
import { sanitizeLogMessage } from '../capture/compact.js';
|
|
7
8
|
import { boolFlag, flagValueless, stringFlag, stringFlagOrExit, } from './flag-values.js';
|
|
8
9
|
import { fail } from './exit.js';
|
|
9
10
|
// `hippo goal <push|list|complete|suspend|resume>`
|
|
@@ -16,10 +17,6 @@ const GOAL_POLICY_TYPES = [
|
|
|
16
17
|
function isGoalPolicyType(value) {
|
|
17
18
|
return GOAL_POLICY_TYPES.some((type) => type === value);
|
|
18
19
|
}
|
|
19
|
-
function sanitizeGoalName(s) {
|
|
20
|
-
// Strip C0 control chars + DEL to prevent terminal escape injection.
|
|
21
|
-
return s.replace(/[^\x20-\x7e\u0080-\uffff]/g, '?');
|
|
22
|
-
}
|
|
23
20
|
function resolveGoalSession(flags, defaultTenantId) {
|
|
24
21
|
const sessionId = (stringFlagOrExit(flags, 'session-id') ?? envHippoSessionId() ?? '').trim();
|
|
25
22
|
if (!sessionId) {
|
|
@@ -55,11 +52,12 @@ function readGoalLevel(flags) {
|
|
|
55
52
|
}
|
|
56
53
|
async function pushGoal(hippoRoot, defaultTenantId, args, flags) {
|
|
57
54
|
const rawName = args.join(' ').trim();
|
|
58
|
-
|
|
55
|
+
// Sanitize at WRITE time so corrupt names never enter the DB, and before the empty check so a
|
|
56
|
+
// name of control characters alone is no name.
|
|
57
|
+
const name = sanitizeLogMessage(rawName).trim();
|
|
58
|
+
if (!name) {
|
|
59
59
|
fail('Usage: hippo goal push <name> [--policy <type>] [--success "<condition>"] [--level N] [--parent <goalId>]');
|
|
60
60
|
}
|
|
61
|
-
// Sanitize at WRITE time so corrupt names never enter the DB.
|
|
62
|
-
const name = sanitizeGoalName(rawName);
|
|
63
61
|
if (name !== rawName) {
|
|
64
62
|
printError('note: stripped control characters from goal name');
|
|
65
63
|
}
|
|
@@ -97,7 +95,7 @@ async function listGoals(hippoRoot, defaultTenantId, flags) {
|
|
|
97
95
|
const rows = goals.map((g) => ({
|
|
98
96
|
id: g.id,
|
|
99
97
|
status: g.status,
|
|
100
|
-
name:
|
|
98
|
+
name: sanitizeLogMessage(g.goalName),
|
|
101
99
|
outcome: g.outcomeScore !== undefined ? g.outcomeScore.toString() : '-',
|
|
102
100
|
}));
|
|
103
101
|
const widths = {
|
|
@@ -5,7 +5,7 @@ import * as path from 'node:path';
|
|
|
5
5
|
import * as fs from 'node:fs';
|
|
6
6
|
import { bookTokenUse } from '../../api/ledger-db.js';
|
|
7
7
|
import { sessionPilotArm } from '../../api/pilot-arm.js';
|
|
8
|
-
import { cliFolder, cliInfra, cliLedgerReach, cliSyncOptions } from './api-context.js';
|
|
8
|
+
import { cliDetached, cliFolder, cliInfra, cliLedgerReach, cliSyncOptions } from './api-context.js';
|
|
9
9
|
import { detachedEmbedder } from '../../api/embeddings.js';
|
|
10
10
|
import { hookPayloadSessionId, hookPayloadString, isSubagentPayload, } from '../../base/util/hook-payload.js';
|
|
11
11
|
import { importAtSessionEnd } from '../../api/agent-memories.js';
|
|
@@ -156,7 +156,7 @@ export function captureTarget(root) {
|
|
|
156
156
|
root,
|
|
157
157
|
store: folder.store,
|
|
158
158
|
config: folder.config,
|
|
159
|
-
embed: detachedEmbedder(folder, cliInfra()),
|
|
159
|
+
embed: detachedEmbedder(folder, cliInfra(), cliDetached()),
|
|
160
160
|
};
|
|
161
161
|
}
|
|
162
162
|
//# sourceMappingURL=hook-runtime.js.map
|