@openclaw/memory-lancedb 2026.7.2-beta.3 → 2026.7.2-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-metadata.js +3 -1
- package/dist/cli-output-mode.js +13 -0
- package/dist/config.js +2 -18
- package/dist/doctor-contract-api.js +76 -1
- package/dist/embeddings.js +299 -0
- package/dist/index.js +32 -876
- package/dist/lancedb-store.js +32 -16
- package/dist/memory-capture-sanitization.js +346 -0
- package/dist/memory-cli.js +134 -0
- package/dist/memory-policy.js +145 -0
- package/package.json +6 -7
- package/npm-shrinkwrap.json +0 -481
package/dist/lancedb-store.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { loadLanceDbModule } from "./lancedb-runtime.js";
|
|
2
2
|
import { MEMORY_TABLE_NAME, hasAgentScopeColumn, legacyMemorySchemaError, memoryAgentPredicate, quoteLanceSqlString } from "./lancedb-schema.js";
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { setTimeout } from "node:timers/promises";
|
|
5
|
+
import { Field, FixedSizeList, Float32, Float64, Schema, Utf8 } from "apache-arrow";
|
|
4
6
|
//#region extensions/memory-lancedb/lancedb-store.ts
|
|
5
|
-
const SCHEMA_SENTINEL_ID = "__schema__";
|
|
6
7
|
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
8
|
+
const TABLE_INITIALIZATION_ATTEMPTS = 3;
|
|
7
9
|
const MEMORY_QUERY_COLUMNS = [
|
|
8
10
|
"id",
|
|
9
11
|
"text",
|
|
@@ -11,6 +13,33 @@ const MEMORY_QUERY_COLUMNS = [
|
|
|
11
13
|
"category",
|
|
12
14
|
"createdAt"
|
|
13
15
|
];
|
|
16
|
+
function createMemoryTableSchema(vectorDim) {
|
|
17
|
+
return new Schema([
|
|
18
|
+
new Field("id", new Utf8(), true),
|
|
19
|
+
new Field("text", new Utf8(), true),
|
|
20
|
+
new Field("vector", new FixedSizeList(vectorDim, new Field("item", new Float32(), true)), true),
|
|
21
|
+
new Field("importance", new Float64(), true),
|
|
22
|
+
new Field("category", new Utf8(), true),
|
|
23
|
+
new Field("createdAt", new Float64(), true),
|
|
24
|
+
new Field("agentId", new Utf8(), true)
|
|
25
|
+
]);
|
|
26
|
+
}
|
|
27
|
+
async function openOrCreateMemoryTable(db, vectorDim) {
|
|
28
|
+
let lastError;
|
|
29
|
+
for (let attempt = 1; attempt <= TABLE_INITIALIZATION_ATTEMPTS; attempt += 1) {
|
|
30
|
+
let table = null;
|
|
31
|
+
try {
|
|
32
|
+
table = (await db.tableNames()).includes("memories") ? await db.openTable(MEMORY_TABLE_NAME) : await db.createEmptyTable(MEMORY_TABLE_NAME, createMemoryTableSchema(vectorDim), { existOk: true });
|
|
33
|
+
await table.schema();
|
|
34
|
+
return table;
|
|
35
|
+
} catch (error) {
|
|
36
|
+
table?.close();
|
|
37
|
+
lastError = error;
|
|
38
|
+
if (attempt < TABLE_INITIALIZATION_ATTEMPTS) await setTimeout(attempt * 10);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
throw lastError;
|
|
42
|
+
}
|
|
14
43
|
function formatQueryFilter(filter) {
|
|
15
44
|
if (filter.operator === "LIKE" && typeof filter.value !== "string") throw new Error("LIKE requires a string memory filter value");
|
|
16
45
|
if (typeof filter.value === "number" && !Number.isFinite(filter.value)) throw new Error("Memory filter number must be finite");
|
|
@@ -45,21 +74,8 @@ var MemoryDB = class {
|
|
|
45
74
|
const db = await lancedb.connect(this.dbPath, connectionOptions);
|
|
46
75
|
let table = null;
|
|
47
76
|
try {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if (!hasAgentScopeColumn(await table.schema())) throw legacyMemorySchemaError();
|
|
51
|
-
} else {
|
|
52
|
-
table = await db.createTable(MEMORY_TABLE_NAME, [{
|
|
53
|
-
id: SCHEMA_SENTINEL_ID,
|
|
54
|
-
text: "",
|
|
55
|
-
vector: Array.from({ length: this.vectorDim }).fill(0),
|
|
56
|
-
importance: 0,
|
|
57
|
-
category: "other",
|
|
58
|
-
createdAt: 0,
|
|
59
|
-
agentId: SCHEMA_SENTINEL_ID
|
|
60
|
-
}]);
|
|
61
|
-
await table.delete(`id = ${quoteLanceSqlString(SCHEMA_SENTINEL_ID)}`);
|
|
62
|
-
}
|
|
77
|
+
table = await openOrCreateMemoryTable(db, this.vectorDim);
|
|
78
|
+
if (!hasAgentScopeColumn(await table.schema())) throw legacyMemorySchemaError();
|
|
63
79
|
this.db = db;
|
|
64
80
|
this.table = table;
|
|
65
81
|
} catch (error) {
|
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
|
2
|
+
import { BUNDLED_CHAT_CHANNEL_ENVELOPE_PREFIXES } from "openclaw/plugin-sdk/chat-channel-ids";
|
|
3
|
+
import { expectDefined } from "openclaw/plugin-sdk/expect-runtime";
|
|
4
|
+
import { MESSAGE_TOOL_DELIVERY_HINTS } from "openclaw/plugin-sdk/message-tool-delivery-hints";
|
|
5
|
+
//#region extensions/memory-lancedb/memory-capture-sanitization.ts
|
|
6
|
+
const MEDIA_NOTE_HEADER = /^\[media attached(?: \d+\/\d+)?: /;
|
|
7
|
+
function stripMediaNoteLine(line) {
|
|
8
|
+
return MEDIA_NOTE_HEADER.test(line) && line.endsWith("]") ? null : line;
|
|
9
|
+
}
|
|
10
|
+
function dropMediaNoteLines(text) {
|
|
11
|
+
return text.split("\n").map(stripMediaNoteLine).filter((line) => line !== null).join("\n");
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Provenance marker appended to every OpenClaw-injected inbound context header
|
|
15
|
+
* by `buildInboundUserContextPrefix`. `sanitizeForMemoryCapture` and
|
|
16
|
+
* `looksLikeEnvelopeSludge` key on this marker rather than on label text, so
|
|
17
|
+
* detection is label-agnostic (arbitrary plugin `ChannelStructuredContext`
|
|
18
|
+
* labels are covered) and never collides with a user's own `<heading>:` + JSON.
|
|
19
|
+
* The marker glyph is duplicated inline in the regexes below because extensions
|
|
20
|
+
* must not import core internals; keep byte-identical with
|
|
21
|
+
* `src/auto-reply/reply/inbound-context-marker.ts`.
|
|
22
|
+
*/
|
|
23
|
+
const MARKER_HEADER_LINE_RE = /^[^\n]*⟦openclaw:ctx⟧[ \t]*$/m;
|
|
24
|
+
const MARKER_JSON_BLOCK_RE = /^[^\n]*⟦openclaw:ctx⟧[ \t]*\n[ \t]*```json[ \t]*\n[\s\S]*?\n[ \t]*```[ \t]*\n?/gm;
|
|
25
|
+
const LEADING_CHRONOLOGICAL_MARKER_HEADER_RE = /^\s*[^\n]*chronological[^\n]*⟦openclaw:ctx⟧[ \t]*(?:\n|$)/;
|
|
26
|
+
const MESSAGE_TOOL_DELIVERY_HINT_RE = new RegExp(`^\\s*(?:${MESSAGE_TOOL_DELIVERY_HINTS.map((hint) => hint.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})\\s*$`, "m");
|
|
27
|
+
const HISTORY_CONTEXT_MARKER = "[Chat messages since your last reply - for context]";
|
|
28
|
+
const CURRENT_MESSAGE_MARKER = "[Current message - respond to this]";
|
|
29
|
+
const HISTORY_CONTEXT_MARKERS = [
|
|
30
|
+
HISTORY_CONTEXT_MARKER,
|
|
31
|
+
"[Chat messages since your last reply — CONTEXT ONLY]",
|
|
32
|
+
"[Merged earlier messages — CONTEXT ONLY]"
|
|
33
|
+
];
|
|
34
|
+
const CURRENT_MESSAGE_MARKERS = [
|
|
35
|
+
CURRENT_MESSAGE_MARKER,
|
|
36
|
+
"[CURRENT MESSAGE — reply to this]",
|
|
37
|
+
"[CURRENT MESSAGE — reply using the context above]"
|
|
38
|
+
];
|
|
39
|
+
const ACTIVE_TURN_RECOVERY_RE = /active-turn-recovery/i;
|
|
40
|
+
const BRACKETED_PREFIX_RE = /\[[^\]\n]{1,500}\]\s/g;
|
|
41
|
+
const LEADING_CURRENT_MESSAGE_CONTEXT_RE = /^\s*Current message:[ \t]*(?:\n|$)/;
|
|
42
|
+
const LEADING_CURRENT_MESSAGE_REPLY_LINE_RE = /^\s*\[Replying to:[^\n]{0,1000}\]\s*\n/;
|
|
43
|
+
const LEADING_CURRENT_MESSAGE_ID_SENDER_RE = /^#\d+\s+[^\n:]{1,100}:\s*/;
|
|
44
|
+
const CONTEXT_HEADER_RE = /^Context:[ \t]*⟦openclaw:ctx⟧[ \t]*$/m;
|
|
45
|
+
/**
|
|
46
|
+
* Matches JSON blobs that look like OpenClaw transport envelope metadata.
|
|
47
|
+
* Orthogonal to the header marker: it catches a bare envelope payload by its
|
|
48
|
+
* compound keys even when no marker header precedes it (e.g. a fragment that
|
|
49
|
+
* leaked outside its ```json fence). Core's `formatContextJsonBlock` emits
|
|
50
|
+
* compact single-line JSON; the optional-newline branch also catches legacy
|
|
51
|
+
* pretty-printed blocks. Key list mirrors envelope identifiers used by
|
|
52
|
+
* `buildInboundUserContextPrefix` and stays narrow to avoid false-positives on
|
|
53
|
+
* legitimate user JSON with bare keys like "conversation" or "sender".
|
|
54
|
+
*/
|
|
55
|
+
const ENVELOPE_JSON_LINE_RE = /^\s*\{\s*(?:\n\s*)?"(?:chat_id|message_id|reply_to_id|sender_id|conversation_label|conversation_info|sender_name|channel_id|channel_type|group_subject|group_channel|group_space|topic_id|thread_label)"\s*:/m;
|
|
56
|
+
/**
|
|
57
|
+
* Leading bracketed envelope header injected by `formatAgentEnvelope` /
|
|
58
|
+
* `formatInboundEnvelope` (src/auto-reply/envelope.ts). Real shape, with parts
|
|
59
|
+
* joined by spaces inside a single `[...]`:
|
|
60
|
+
*
|
|
61
|
+
* `[<channel> <from> +<elapsed>? <host>? <ip>? <Wkd YYYY-MM-DD HH:MM TZ>?] <body>`
|
|
62
|
+
*
|
|
63
|
+
* Examples:
|
|
64
|
+
* `[Telegram Alice +5m] I prefer dark mode`
|
|
65
|
+
* `[Telegram Group id:123 Alice +5m Mon 2026-05-17 14:30 EDT] Alice: text`
|
|
66
|
+
* `[Discord #general user +0s Mon 2026-05-17T14:30Z] text`
|
|
67
|
+
*
|
|
68
|
+
* Detection keys on the load-bearing parts that mark this header as an
|
|
69
|
+
* envelope (rather than arbitrary user-typed `[brackets]`): an elapsed marker
|
|
70
|
+
* `+<n><unit>` produced by `formatTimeAgo({suffix:false})` (units: s/m/h/d, or
|
|
71
|
+
* the literal `just now` fallback), or a weekday + ISO date pair produced by
|
|
72
|
+
* `formatEnvelopeTimestamp`. Either marker is unique enough that quoting
|
|
73
|
+
* `[5m]` or `[Mon 2026-05-17]` mid-sentence will not look like an envelope
|
|
74
|
+
* prefix because the regex is anchored to start-of-string and requires the
|
|
75
|
+
* marker to live inside the leading bracket followed by `]<space>`.
|
|
76
|
+
*
|
|
77
|
+
* Capture group 1 is the inside-bracket text, used by the sender-prefix
|
|
78
|
+
* gating logic in `sanitizeForMemoryCapture` to scope which body labels we
|
|
79
|
+
* are willing to strip. Header part length is capped at 300 chars to avoid
|
|
80
|
+
* catastrophic backtracking on pathological inputs; real envelopes are well
|
|
81
|
+
* under that.
|
|
82
|
+
*/
|
|
83
|
+
const INBOUND_ENVELOPE_PREFIX_RE = /^\[([^\]\n]{0,300}?(?:\s\+(?:\d+[smhdwy]|just now)\b|\s[A-Za-z]{3}\s\d{4}-\d{2}-\d{2})[^\]\n]{0,200})\]\s/;
|
|
84
|
+
/**
|
|
85
|
+
* Marker-free leading envelope header. The elapsed/date marker regex above
|
|
86
|
+
* misses envelopes where `formatAgentEnvelope` drops every optional marker.
|
|
87
|
+
* Because channel labels can also be ordinary words, callers only accept this
|
|
88
|
+
* match after `matchKnownChannelMarkerFreeEnvelopePrefix` finds a stronger
|
|
89
|
+
* group/thread or body-sender signal.
|
|
90
|
+
*
|
|
91
|
+
* Anchoring on a known bundled/official channel prefix from
|
|
92
|
+
* `BUNDLED_CHAT_CHANNEL_ENVELOPE_PREFIXES` keeps the detector and formatter in
|
|
93
|
+
* sync across callers that pass either ids or display labels like `Google Chat`.
|
|
94
|
+
* Case insensitive because the formatter does not lowercase `params.channel`
|
|
95
|
+
* itself; production paths feed mixed ids and labels.
|
|
96
|
+
*
|
|
97
|
+
* From-label must be at least one non-whitespace token so user prose like
|
|
98
|
+
* `[note]` or `[telegram] ...` (no following label) is not mistaken for an
|
|
99
|
+
* envelope. Capture group 1 is the inside-bracket text (channel + from-label
|
|
100
|
+
* and any remaining header parts), used by the sender-prefix gating logic in
|
|
101
|
+
* `sanitizeForMemoryCapture`. Header part length is capped at 300 chars to
|
|
102
|
+
* match the marker-aware regex above and avoid catastrophic backtracking.
|
|
103
|
+
*
|
|
104
|
+
* Guarded against an empty `BUNDLED_CHAT_CHANNEL_ENVELOPE_PREFIXES` so the
|
|
105
|
+
* alternation never degenerates into `(?:)` (which would match the empty string
|
|
106
|
+
* and flag every `[...]` prefix as an envelope). When the bundled list is empty the
|
|
107
|
+
* known-channel detector is disabled and only the marker-aware regex above
|
|
108
|
+
* applies.
|
|
109
|
+
*/
|
|
110
|
+
const ENVELOPE_KNOWN_CHANNEL_PATTERN = BUNDLED_CHAT_CHANNEL_ENVELOPE_PREFIXES.map((prefix) => prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
|
|
111
|
+
const INBOUND_ENVELOPE_KNOWN_CHANNEL_PREFIX_RE = ENVELOPE_KNOWN_CHANNEL_PATTERN ? new RegExp(`^\\[((?:${ENVELOPE_KNOWN_CHANNEL_PATTERN})\\s+[^\\]\\n\\s][^\\]\\n]{0,299})\\]\\s`, "i") : null;
|
|
112
|
+
/**
|
|
113
|
+
* Group-chat envelope bodies prepend `<Sender>: ` to the raw user text (see
|
|
114
|
+
* `formatInboundEnvelope`). After stripping the leading envelope bracket,
|
|
115
|
+
* this pattern matches that body sender prefix; capture group 1 is the label
|
|
116
|
+
* itself so the gated strip in `sanitizeForMemoryCapture` can compare it
|
|
117
|
+
* against the envelope header before removing it. Sender label is capped at
|
|
118
|
+
* the same length as `sanitizeEnvelopeHeaderPart` would produce in practice
|
|
119
|
+
* (the envelope formatter does not truncate, but a 120-char ceiling keeps the
|
|
120
|
+
* regex bounded and matches realistic display names).
|
|
121
|
+
*/
|
|
122
|
+
const ENVELOPE_BODY_SENDER_PREFIX_RE = /^([^\n:]{1,120}):\s/;
|
|
123
|
+
const ENVELOPE_BODY_DIRECT_PREFIX = "(sender)";
|
|
124
|
+
const ENVELOPE_BODY_SELF_PREFIX = "(self)";
|
|
125
|
+
const SENDER_PREFIXED_ENVELOPE_CHANNEL_RE = /^(?:discord|imessage|line|mattermost|qqbot|signal|slack|telegram|whatsapp)(?:\s|$)/i;
|
|
126
|
+
const NON_DIRECT_ENVELOPE_HEADER_RE = /(?:^|\s)(?:#[^\s]+|group:[^\s]+|group\s+id:[^\s]+|room:[^\s]+|channel\s+id:[^\s]+|id:-[^\s]+|unknown-group|[^\s]+@g\.us)(?:\s|$)/i;
|
|
127
|
+
const USER_AUTHORED_BODY_LABEL_RE = /^(?:action|decision|fixme|note|question|reminder|todo)$/i;
|
|
128
|
+
function matchKnownChannelMarkerFreeEnvelopePrefix(text, options) {
|
|
129
|
+
const match = INBOUND_ENVELOPE_KNOWN_CHANNEL_PREFIX_RE?.exec(text);
|
|
130
|
+
if (!match) return null;
|
|
131
|
+
const headerInside = match[1] ?? "";
|
|
132
|
+
if (NON_DIRECT_ENVELOPE_HEADER_RE.test(headerInside)) return match;
|
|
133
|
+
const body = text.slice(match[0].length);
|
|
134
|
+
if (stripEnvelopeBodySenderPrefix(body, headerInside) !== body) return match;
|
|
135
|
+
return options?.allowAmbiguousDirect ? match : null;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Returns true if `text` looks like it contains OpenClaw-injected envelope or
|
|
139
|
+
* transport metadata that should never be persisted as a long-term memory.
|
|
140
|
+
*/
|
|
141
|
+
function looksLikeEnvelopeSludge(text) {
|
|
142
|
+
if (!text) return false;
|
|
143
|
+
if (MARKER_HEADER_LINE_RE.test(text)) return true;
|
|
144
|
+
if (MESSAGE_TOOL_DELIVERY_HINT_RE.test(text)) return true;
|
|
145
|
+
if (HISTORY_CONTEXT_MARKERS.some((marker) => text.includes(marker)) || CURRENT_MESSAGE_MARKERS.some((marker) => text.includes(marker))) return true;
|
|
146
|
+
if (ACTIVE_TURN_RECOVERY_RE.test(text)) return true;
|
|
147
|
+
if (ENVELOPE_JSON_LINE_RE.test(text)) return true;
|
|
148
|
+
return INBOUND_ENVELOPE_PREFIX_RE.test(text) || matchKnownChannelMarkerFreeEnvelopePrefix(text) !== null;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Timestamp prefix pattern injected by `injectTimestamp`.
|
|
152
|
+
* Canonical source: src/auto-reply/reply/strip-inbound-meta.ts
|
|
153
|
+
*/
|
|
154
|
+
const LEADING_TIMESTAMP_PREFIX_RE = /^\[[A-Za-z]{3} \d{4}-\d{2}-\d{2} \d{2}:\d{2}[^\]]*\] */;
|
|
155
|
+
/**
|
|
156
|
+
* Decide whether a `<X>: ` body prefix that follows a stripped envelope
|
|
157
|
+
* bracket was emitted by the formatter (vs being user-typed prose). The
|
|
158
|
+
* formatter contract in `src/auto-reply/envelope.ts` only ever prepends:
|
|
159
|
+
* - `(self): ` for direct chats with `fromMe`, OR
|
|
160
|
+
* - `<resolvedSender>: ` for non-direct chats with a sender label.
|
|
161
|
+
*
|
|
162
|
+
* Some channel paths call `formatInboundEnvelope` and therefore put the room in
|
|
163
|
+
* the header while keeping the sender as the body label, for example
|
|
164
|
+
* `[Slack #general] Alice: text`. Generic `formatAgentEnvelope` callers and
|
|
165
|
+
* direct `formatInboundEnvelope` bodies do not add that body label, so require
|
|
166
|
+
* structural non-direct markers and preserve common user-authored labels like
|
|
167
|
+
* `TODO:`.
|
|
168
|
+
*/
|
|
169
|
+
function stripEnvelopeBodySenderPrefix(body, headerInside) {
|
|
170
|
+
const match = body.match(ENVELOPE_BODY_SENDER_PREFIX_RE);
|
|
171
|
+
if (!match) return body;
|
|
172
|
+
const label = expectDefined(match[1], "envelope body sender capture");
|
|
173
|
+
if (label === ENVELOPE_BODY_SELF_PREFIX || label === ENVELOPE_BODY_DIRECT_PREFIX) return body.slice(match[0].length);
|
|
174
|
+
if (SENDER_PREFIXED_ENVELOPE_CHANNEL_RE.test(headerInside) && NON_DIRECT_ENVELOPE_HEADER_RE.test(headerInside) && !USER_AUTHORED_BODY_LABEL_RE.test(label)) return body.slice(match[0].length);
|
|
175
|
+
if (headerInside.split(/\s+/).includes(label) || headerInside.includes(label)) return body.slice(match[0].length);
|
|
176
|
+
return body;
|
|
177
|
+
}
|
|
178
|
+
function stripLeadingMessageToolDeliveryHints(text) {
|
|
179
|
+
const lines = text.split("\n");
|
|
180
|
+
let index = 0;
|
|
181
|
+
let stripped = false;
|
|
182
|
+
while (index < lines.length) {
|
|
183
|
+
const trimmed = lines[index]?.trim();
|
|
184
|
+
if (!trimmed) {
|
|
185
|
+
index += 1;
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
if (!MESSAGE_TOOL_DELIVERY_HINTS.some((hint) => hint === trimmed)) break;
|
|
189
|
+
stripped = true;
|
|
190
|
+
index += 1;
|
|
191
|
+
}
|
|
192
|
+
return stripped ? lines.slice(index).join("\n") : text;
|
|
193
|
+
}
|
|
194
|
+
function findFirstInboundEnvelopeIndex(text, options) {
|
|
195
|
+
for (const match of text.matchAll(BRACKETED_PREFIX_RE)) {
|
|
196
|
+
const index = match.index;
|
|
197
|
+
if (options?.skipReplyQuoteLine) {
|
|
198
|
+
const lineStart = text.lastIndexOf("\n", index - 1) + 1;
|
|
199
|
+
if (text.slice(lineStart, index).includes("[Replying to:")) continue;
|
|
200
|
+
}
|
|
201
|
+
const candidate = text.slice(index);
|
|
202
|
+
if (INBOUND_ENVELOPE_PREFIX_RE.test(candidate) || matchKnownChannelMarkerFreeEnvelopePrefix(candidate, { allowAmbiguousDirect: options?.allowAmbiguousMarkerFree })) return index;
|
|
203
|
+
}
|
|
204
|
+
return -1;
|
|
205
|
+
}
|
|
206
|
+
function stripPendingHistoryContextBeforeCurrentMessage(text) {
|
|
207
|
+
const candidateText = text.trimStart();
|
|
208
|
+
if (!HISTORY_CONTEXT_MARKERS.some((marker) => candidateText.startsWith(marker))) return text;
|
|
209
|
+
const currentMarker = findLastContextMarker(candidateText, CURRENT_MESSAGE_MARKERS);
|
|
210
|
+
if (!currentMarker) return text;
|
|
211
|
+
return candidateText.slice(currentMarker.index + currentMarker.marker.length);
|
|
212
|
+
}
|
|
213
|
+
function stripToCurrentMessageMarker(text) {
|
|
214
|
+
const currentMarker = findLastContextMarker(text, CURRENT_MESSAGE_MARKERS);
|
|
215
|
+
if (!currentMarker) return null;
|
|
216
|
+
return text.slice(currentMarker.index + currentMarker.marker.length);
|
|
217
|
+
}
|
|
218
|
+
function findLastContextMarker(text, markers) {
|
|
219
|
+
let result = null;
|
|
220
|
+
for (const marker of markers) {
|
|
221
|
+
const index = text.lastIndexOf(marker);
|
|
222
|
+
if (index !== -1 && (!result || index > result.index)) result = {
|
|
223
|
+
index,
|
|
224
|
+
marker
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
return result;
|
|
228
|
+
}
|
|
229
|
+
function stripLeadingCurrentMessageContextBeforeEnvelope(text) {
|
|
230
|
+
const candidateText = text.trimStart();
|
|
231
|
+
if (!LEADING_CURRENT_MESSAGE_CONTEXT_RE.test(candidateText)) return text;
|
|
232
|
+
const envelopeIndex = findFirstInboundEnvelopeIndex(candidateText, {
|
|
233
|
+
allowAmbiguousMarkerFree: true,
|
|
234
|
+
skipReplyQuoteLine: true
|
|
235
|
+
});
|
|
236
|
+
if (envelopeIndex === -1) {
|
|
237
|
+
let plainBody = candidateText.replace(LEADING_CURRENT_MESSAGE_CONTEXT_RE, "").trimStart();
|
|
238
|
+
for (let pass = 0; pass < 4; pass += 1) {
|
|
239
|
+
const replyLineMatch = plainBody.match(LEADING_CURRENT_MESSAGE_REPLY_LINE_RE);
|
|
240
|
+
if (!replyLineMatch) break;
|
|
241
|
+
plainBody = plainBody.slice(replyLineMatch[0].length).trimStart();
|
|
242
|
+
}
|
|
243
|
+
const currentMessagePrefixMatch = plainBody.match(LEADING_CURRENT_MESSAGE_ID_SENDER_RE);
|
|
244
|
+
return currentMessagePrefixMatch ? plainBody.slice(currentMessagePrefixMatch[0].length) : text;
|
|
245
|
+
}
|
|
246
|
+
return candidateText.slice(envelopeIndex);
|
|
247
|
+
}
|
|
248
|
+
function stripLeadingPlainTextMetadataBody(text) {
|
|
249
|
+
const candidateText = text.trimStart();
|
|
250
|
+
const markerBody = stripToCurrentMessageMarker(candidateText);
|
|
251
|
+
if (markerBody !== null) return markerBody;
|
|
252
|
+
const currentMessageBody = stripLeadingCurrentMessageContextBeforeEnvelope(candidateText);
|
|
253
|
+
return currentMessageBody === candidateText ? "" : currentMessageBody;
|
|
254
|
+
}
|
|
255
|
+
function stripLeadingInboundEnvelope(text, options) {
|
|
256
|
+
const strippedCandidate = stripLeadingCurrentMessageContextBeforeEnvelope(stripPendingHistoryContextBeforeCurrentMessage(stripLeadingMessageToolDeliveryHints(text)));
|
|
257
|
+
const candidateText = strippedCandidate.trimStart();
|
|
258
|
+
const allowAmbiguousMarkerFree = options?.allowAmbiguousMarkerFree || strippedCandidate !== text;
|
|
259
|
+
const envelopePrefixMatch = candidateText.match(INBOUND_ENVELOPE_PREFIX_RE) ?? matchKnownChannelMarkerFreeEnvelopePrefix(candidateText, { allowAmbiguousDirect: allowAmbiguousMarkerFree });
|
|
260
|
+
if (!envelopePrefixMatch) return strippedCandidate === text ? text : candidateText;
|
|
261
|
+
const headerInside = envelopePrefixMatch[1] ?? "";
|
|
262
|
+
return stripEnvelopeBodySenderPrefix(candidateText.slice(envelopePrefixMatch[0].length), headerInside);
|
|
263
|
+
}
|
|
264
|
+
function stripLeadingChronologicalContextBlocks(text) {
|
|
265
|
+
let cleaned = text;
|
|
266
|
+
let remainingPasses = 16;
|
|
267
|
+
while (remainingPasses > 0) {
|
|
268
|
+
remainingPasses -= 1;
|
|
269
|
+
const match = cleaned.match(LEADING_CHRONOLOGICAL_MARKER_HEADER_RE);
|
|
270
|
+
if (!match) return cleaned;
|
|
271
|
+
const afterLabel = cleaned.slice(match[0].length);
|
|
272
|
+
const bodyStart = afterLabel.search(/\S/);
|
|
273
|
+
if (bodyStart === -1) return "";
|
|
274
|
+
const bodyLineEnd = afterLabel.indexOf("\n", bodyStart);
|
|
275
|
+
const firstBodyLine = bodyLineEnd === -1 ? afterLabel.slice(bodyStart) : afterLabel.slice(bodyStart, bodyLineEnd);
|
|
276
|
+
let lineEnvelopeIndex = firstBodyLine.trimStart().startsWith("[") ? findFirstInboundEnvelopeIndex(firstBodyLine, {
|
|
277
|
+
allowAmbiguousMarkerFree: true,
|
|
278
|
+
skipReplyQuoteLine: true
|
|
279
|
+
}) : -1;
|
|
280
|
+
if (lineEnvelopeIndex === -1 && match[0].includes("selected for current message")) {
|
|
281
|
+
const inlineEnvelopeIndex = findFirstInboundEnvelopeIndex(firstBodyLine, {
|
|
282
|
+
allowAmbiguousMarkerFree: true,
|
|
283
|
+
skipReplyQuoteLine: true
|
|
284
|
+
});
|
|
285
|
+
const prefix = inlineEnvelopeIndex === -1 ? "" : firstBodyLine.slice(0, inlineEnvelopeIndex);
|
|
286
|
+
lineEnvelopeIndex = /^#\d+\s/.test(prefix.trimStart()) ? inlineEnvelopeIndex : -1;
|
|
287
|
+
}
|
|
288
|
+
const envelopeIndex = lineEnvelopeIndex === -1 ? -1 : bodyStart + lineEnvelopeIndex;
|
|
289
|
+
if (envelopeIndex === -1) {
|
|
290
|
+
const separatorMatch = /\n[ \t]*\n/.exec(afterLabel);
|
|
291
|
+
cleaned = separatorMatch ? afterLabel.slice(separatorMatch.index + separatorMatch[0].length) : "";
|
|
292
|
+
} else cleaned = afterLabel.slice(envelopeIndex);
|
|
293
|
+
if (!cleaned) return "";
|
|
294
|
+
}
|
|
295
|
+
return cleaned;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Strips OpenClaw-injected envelope metadata from a user message so that only
|
|
299
|
+
* the user's actual intent text remains. Returns empty string if nothing
|
|
300
|
+
* meaningful survives.
|
|
301
|
+
*/
|
|
302
|
+
function sanitizeForMemoryCapture(text) {
|
|
303
|
+
if (!text) return "";
|
|
304
|
+
const MAX_SANITIZE_CHARS = 1e4;
|
|
305
|
+
let cleaned = text.length > MAX_SANITIZE_CHARS ? truncateUtf16Safe(text, MAX_SANITIZE_CHARS) : text;
|
|
306
|
+
let strippedInjectedContext = false;
|
|
307
|
+
cleaned = cleaned.replace(LEADING_TIMESTAMP_PREFIX_RE, "");
|
|
308
|
+
cleaned = dropMediaNoteLines(cleaned);
|
|
309
|
+
const afterDeliveryHints = stripLeadingMessageToolDeliveryHints(cleaned);
|
|
310
|
+
strippedInjectedContext ||= afterDeliveryHints !== cleaned;
|
|
311
|
+
cleaned = afterDeliveryHints;
|
|
312
|
+
const afterJsonMetaBlocks = cleaned.replace(MARKER_JSON_BLOCK_RE, "");
|
|
313
|
+
strippedInjectedContext ||= afterJsonMetaBlocks !== cleaned;
|
|
314
|
+
cleaned = afterJsonMetaBlocks;
|
|
315
|
+
const afterChronologicalContext = stripLeadingChronologicalContextBlocks(cleaned);
|
|
316
|
+
strippedInjectedContext ||= afterChronologicalContext !== cleaned;
|
|
317
|
+
cleaned = afterChronologicalContext;
|
|
318
|
+
for (let pass = 0; pass < 16; pass += 1) {
|
|
319
|
+
const headerMatch = cleaned.match(MARKER_HEADER_LINE_RE);
|
|
320
|
+
if (headerMatch?.index === void 0) break;
|
|
321
|
+
const before = cleaned.slice(0, headerMatch.index);
|
|
322
|
+
if (before.trim().length > 0) {
|
|
323
|
+
cleaned = before;
|
|
324
|
+
break;
|
|
325
|
+
}
|
|
326
|
+
const lineEnd = cleaned.indexOf("\n");
|
|
327
|
+
const afterHeader = lineEnd === -1 ? "" : cleaned.slice(lineEnd + 1);
|
|
328
|
+
const afterPlainTextMetadata = afterHeader.trimStart().startsWith("```json") ? afterHeader : stripLeadingPlainTextMetadataBody(afterHeader);
|
|
329
|
+
strippedInjectedContext ||= afterPlainTextMetadata !== cleaned;
|
|
330
|
+
cleaned = afterPlainTextMetadata;
|
|
331
|
+
}
|
|
332
|
+
const afterActiveMemoryContext = cleaned.replace(/^Context:[ \t]*\n<active_memory_plugin>[\s\S]*?<\/active_memory_plugin>\s*/gm, "");
|
|
333
|
+
strippedInjectedContext ||= afterActiveMemoryContext !== cleaned;
|
|
334
|
+
cleaned = afterActiveMemoryContext;
|
|
335
|
+
const untrustedLineMatch = CONTEXT_HEADER_RE.exec(cleaned);
|
|
336
|
+
if (untrustedLineMatch) {
|
|
337
|
+
strippedInjectedContext = true;
|
|
338
|
+
cleaned = cleaned.slice(0, untrustedLineMatch.index);
|
|
339
|
+
}
|
|
340
|
+
cleaned = stripLeadingInboundEnvelope(cleaned, { allowAmbiguousMarkerFree: strippedInjectedContext });
|
|
341
|
+
cleaned = cleaned.replace(/<active_memory_plugin>[\s\S]*?<\/active_memory_plugin>/g, "");
|
|
342
|
+
cleaned = cleaned.replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ").trim();
|
|
343
|
+
return cleaned;
|
|
344
|
+
}
|
|
345
|
+
//#endregion
|
|
346
|
+
export { dropMediaNoteLines, looksLikeEnvelopeSludge, sanitizeForMemoryCapture };
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { MEMORY_QUERY_COLUMNS } from "./lancedb-store.js";
|
|
2
|
+
import { isMemoryMachineOutput } from "./cli-output-mode.js";
|
|
3
|
+
import { normalizeRecallQuery } from "./memory-policy.js";
|
|
4
|
+
import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime";
|
|
5
|
+
import { defaultRuntime } from "openclaw/plugin-sdk/runtime";
|
|
6
|
+
//#region extensions/memory-lancedb/memory-cli.ts
|
|
7
|
+
function parsePositiveIntegerOption(value, flag) {
|
|
8
|
+
if (value === void 0) return;
|
|
9
|
+
const parsed = parseStrictPositiveInteger(value);
|
|
10
|
+
if (parsed === void 0) throw new Error(`${flag} must be a positive integer`);
|
|
11
|
+
return parsed;
|
|
12
|
+
}
|
|
13
|
+
function parseMemoryCliColumns(value) {
|
|
14
|
+
if (typeof value !== "string") return [...MEMORY_QUERY_COLUMNS];
|
|
15
|
+
const columns = value.split(",").map((column) => column.trim());
|
|
16
|
+
const invalid = columns.filter((column) => !MEMORY_QUERY_COLUMNS.includes(column));
|
|
17
|
+
if (invalid.length > 0) throw new Error(`Unsupported memory columns: ${invalid.join(", ")}`);
|
|
18
|
+
return columns;
|
|
19
|
+
}
|
|
20
|
+
function parseMemoryCliOrder(value) {
|
|
21
|
+
if (typeof value !== "string" || !value.trim()) return null;
|
|
22
|
+
const [column, direction = "asc", extra] = value.split(":");
|
|
23
|
+
if (extra !== void 0 || !MEMORY_QUERY_COLUMNS.includes(column) || !["asc", "desc"].includes(direction.toLowerCase())) throw new Error("--order-by must be <id|text|importance|category|createdAt>:<asc|desc>");
|
|
24
|
+
return {
|
|
25
|
+
column,
|
|
26
|
+
direction: direction.toLowerCase() === "desc" ? -1 : 1
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function parseMemoryCliFilter(rawValue) {
|
|
30
|
+
if (rawValue === void 0) return;
|
|
31
|
+
if (typeof rawValue !== "string") throw new Error("--filter must be a string");
|
|
32
|
+
const filter = rawValue.trim();
|
|
33
|
+
if (filter.length > 200) throw new Error("Filter condition exceeds maximum length of 200 characters");
|
|
34
|
+
const match = /^(id|text|importance|category|createdAt)\s*(=|!=|<>|<=|>=|<|>|LIKE)\s*(?:'((?:''|[^'])*)'|(-?(?:\d+(?:\.\d+)?|\.\d+)))$/i.exec(filter);
|
|
35
|
+
if (!match) throw new Error("--filter must be one comparison using id, text, importance, category, or createdAt");
|
|
36
|
+
const rawColumn = match[1];
|
|
37
|
+
const rawOperator = match[2];
|
|
38
|
+
const rawString = match[3];
|
|
39
|
+
const rawNumber = match[4];
|
|
40
|
+
const column = MEMORY_QUERY_COLUMNS.find((candidate) => candidate.toLowerCase() === rawColumn.toLowerCase());
|
|
41
|
+
if (!column) throw new Error(`Unsupported memory filter column: ${rawColumn}`);
|
|
42
|
+
const operator = rawOperator.toUpperCase();
|
|
43
|
+
const value = rawString !== void 0 ? rawString.replaceAll("''", "'") : Number(rawNumber);
|
|
44
|
+
if (typeof value === "number" && !Number.isFinite(value)) throw new Error("--filter numeric value must be finite");
|
|
45
|
+
const expectsNumber = column === "importance" || column === "createdAt";
|
|
46
|
+
if (expectsNumber !== (typeof value === "number")) throw new Error(`--filter ${column} requires a ${expectsNumber ? "number" : "quoted string"}`);
|
|
47
|
+
if (operator === "LIKE" && typeof value !== "string") throw new Error("--filter LIKE requires a quoted string");
|
|
48
|
+
return {
|
|
49
|
+
column,
|
|
50
|
+
operator,
|
|
51
|
+
value
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
function registerMemoryCli(api, db, embeddings, resolveCliAgentId, recallMaxChars) {
|
|
55
|
+
api.registerCli(({ program }) => {
|
|
56
|
+
const memory = program.command("ltm").description("LanceDB memory plugin commands");
|
|
57
|
+
memory.command("list").description("List memories").option("--agent <id>", "Agent id (default: configured default agent)").option("--limit <n>", "Max results").option("--order-by-created-at", "Order memories by createdAt descending", false).action(async (opts) => {
|
|
58
|
+
const agentId = resolveCliAgentId(opts.agent);
|
|
59
|
+
const limit = parsePositiveIntegerOption(opts.limit, "--limit");
|
|
60
|
+
const entries = await db.list(agentId, limit, { orderByCreatedAt: Boolean(opts.orderByCreatedAt) });
|
|
61
|
+
defaultRuntime.writeJson(entries);
|
|
62
|
+
});
|
|
63
|
+
memory.command("search").description("Search memories").argument("<query>", "Search query").option("--agent <id>", "Agent id (default: configured default agent)").option("--limit <n>", "Max results", "5").action(async (query, opts) => {
|
|
64
|
+
let operationError;
|
|
65
|
+
let operationFailed = false;
|
|
66
|
+
try {
|
|
67
|
+
const agentId = resolveCliAgentId(opts.agent);
|
|
68
|
+
const vector = await embeddings.embed(normalizeRecallQuery(query, recallMaxChars));
|
|
69
|
+
const limit = parsePositiveIntegerOption(opts.limit, "--limit");
|
|
70
|
+
const output = (await db.search(agentId, vector, limit, .3)).map((r) => ({
|
|
71
|
+
id: r.entry.id,
|
|
72
|
+
text: r.entry.text,
|
|
73
|
+
category: r.entry.category,
|
|
74
|
+
importance: r.entry.importance,
|
|
75
|
+
score: r.score
|
|
76
|
+
}));
|
|
77
|
+
defaultRuntime.writeJson(output);
|
|
78
|
+
} catch (err) {
|
|
79
|
+
operationError = err;
|
|
80
|
+
operationFailed = true;
|
|
81
|
+
}
|
|
82
|
+
let closeError;
|
|
83
|
+
let closeFailed = false;
|
|
84
|
+
try {
|
|
85
|
+
await embeddings.close?.();
|
|
86
|
+
} catch (err) {
|
|
87
|
+
closeError = err;
|
|
88
|
+
closeFailed = true;
|
|
89
|
+
}
|
|
90
|
+
if (operationFailed) throw operationError;
|
|
91
|
+
if (closeFailed) throw closeError;
|
|
92
|
+
});
|
|
93
|
+
memory.command("query").description("Query memories (non-vector search)").option("--agent <id>", "Agent id (default: configured default agent)").option("--cols <columns>", "Columns to select, comma-separated").option("--filter <condition>", "Filter condition").option("--limit <n>", "Limit number of results", "10").option("--order-by <order>", "Order by column and direction (e.g., createdAt:desc)").action(async (opts) => {
|
|
94
|
+
const agentId = resolveCliAgentId(opts.agent);
|
|
95
|
+
const outputColumns = parseMemoryCliColumns(opts.cols);
|
|
96
|
+
const order = parseMemoryCliOrder(opts.orderBy);
|
|
97
|
+
const selectedColumns = [...outputColumns];
|
|
98
|
+
if (order && !selectedColumns.includes(order.column)) selectedColumns.push(order.column);
|
|
99
|
+
const limit = parsePositiveIntegerOption(opts.limit, "--limit") ?? 10;
|
|
100
|
+
let rows = await db.query(agentId, {
|
|
101
|
+
columns: selectedColumns,
|
|
102
|
+
filter: parseMemoryCliFilter(opts.filter),
|
|
103
|
+
...order ? {} : { limit }
|
|
104
|
+
});
|
|
105
|
+
if (order) {
|
|
106
|
+
rows.sort((a, b) => {
|
|
107
|
+
const aValue = a[order.column];
|
|
108
|
+
const bValue = b[order.column];
|
|
109
|
+
if (aValue < bValue) return -1 * order.direction;
|
|
110
|
+
if (aValue > bValue) return order.direction;
|
|
111
|
+
return 0;
|
|
112
|
+
});
|
|
113
|
+
rows = rows.slice(0, limit);
|
|
114
|
+
if (!outputColumns.includes(order.column)) for (const row of rows) delete row[order.column];
|
|
115
|
+
}
|
|
116
|
+
defaultRuntime.writeJson(rows);
|
|
117
|
+
});
|
|
118
|
+
memory.command("stats").description("Show memory statistics").option("--agent <id>", "Agent id (default: configured default agent)").action(async (opts) => {
|
|
119
|
+
const agentId = resolveCliAgentId(opts.agent);
|
|
120
|
+
const count = await db.count(agentId);
|
|
121
|
+
console.log(`Total memories: ${count}`);
|
|
122
|
+
});
|
|
123
|
+
}, {
|
|
124
|
+
commands: ["ltm"],
|
|
125
|
+
descriptors: [{
|
|
126
|
+
name: "ltm",
|
|
127
|
+
description: "LanceDB memory plugin commands",
|
|
128
|
+
hasSubcommands: true,
|
|
129
|
+
machineOutput: isMemoryMachineOutput
|
|
130
|
+
}]
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
//#endregion
|
|
134
|
+
export { parseMemoryCliFilter, registerMemoryCli };
|