@xdarkicex/openclaw-memory-libravdb 1.6.27 → 1.6.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context-engine.d.ts +10 -2
- package/dist/context-engine.js +202 -18
- package/dist/index.js +180 -19
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
package/dist/context-engine.d.ts
CHANGED
|
@@ -33,6 +33,9 @@ type OpenClawCompatibleCompactResult = {
|
|
|
33
33
|
details?: unknown;
|
|
34
34
|
};
|
|
35
35
|
};
|
|
36
|
+
type KernelContentNormalizationOptions = {
|
|
37
|
+
retainOpenClawContext?: boolean;
|
|
38
|
+
};
|
|
36
39
|
/**
|
|
37
40
|
* Normalizes a single kernel message into the kernel-compatible format.
|
|
38
41
|
*/
|
|
@@ -40,15 +43,20 @@ export declare function normalizeKernelMessage(message: {
|
|
|
40
43
|
role: string;
|
|
41
44
|
content: unknown;
|
|
42
45
|
id?: string;
|
|
43
|
-
|
|
46
|
+
[key: string]: unknown;
|
|
47
|
+
}, options?: KernelContentNormalizationOptions): KernelCompatibleMessage;
|
|
44
48
|
/**
|
|
45
49
|
* Normalizes an array of kernel messages.
|
|
50
|
+
*
|
|
51
|
+
* Non-user messages whose normalized content is empty or whitespace-only
|
|
52
|
+
* are dropped. This prevents assistant/system turns that consisted entirely
|
|
53
|
+
* of stripped metadata from persisting as empty records.
|
|
46
54
|
*/
|
|
47
55
|
export declare function normalizeKernelMessages(messages: Array<{
|
|
48
56
|
role: string;
|
|
49
57
|
content: unknown;
|
|
50
58
|
id?: string;
|
|
51
|
-
}
|
|
59
|
+
}>, options?: KernelContentNormalizationOptions): KernelCompatibleMessage[];
|
|
52
60
|
/**
|
|
53
61
|
* Normalizes a compact result into the OpenClaw-compatible assemble result format.
|
|
54
62
|
*/
|
package/dist/context-engine.js
CHANGED
|
@@ -14,6 +14,15 @@ const EXACT_RECALL_SEARCH_K = 32;
|
|
|
14
14
|
const EXACT_RECALL_MAX_TOKENS = 4;
|
|
15
15
|
const RESERVED_CURRENT_TURN_TOKENS = 150;
|
|
16
16
|
const AFTER_TURN_INGEST_MAX_TOKENS = 2048;
|
|
17
|
+
const OPENCLAW_LEADING_TIMESTAMP_PREFIX_RE = /^\[[A-Za-z]{3} \d{4}-\d{2}-\d{2} \d{2}:\d{2}[^\]]*\] */;
|
|
18
|
+
const OPENCLAW_METADATA_HEADERS = [
|
|
19
|
+
"Conversation info (untrusted metadata):",
|
|
20
|
+
"Sender (untrusted metadata):",
|
|
21
|
+
"Thread starter (untrusted, for context):",
|
|
22
|
+
"Reply target of current user message (untrusted, for context):",
|
|
23
|
+
"Forwarded message context (untrusted metadata):",
|
|
24
|
+
"Chat history since last reply (untrusted, for context):",
|
|
25
|
+
];
|
|
17
26
|
const COMMON_QUERY_WORDS = new Set([
|
|
18
27
|
"what", "does", "mean", "remember", "recall", "about", "this", "that",
|
|
19
28
|
"the", "and", "for", "with", "from", "your", "have", "been", "were",
|
|
@@ -45,15 +54,22 @@ function normalizeCompactResult(response, options = {}) {
|
|
|
45
54
|
? response.summaryText
|
|
46
55
|
: undefined,
|
|
47
56
|
};
|
|
57
|
+
// When the engine owns compaction but refuses to compact while the session
|
|
58
|
+
// exceeds the threshold, this is not a successful skip — it's a failure.
|
|
59
|
+
// Signal ok:false so OpenClaw falls back to normal transcript compaction
|
|
60
|
+
// instead of accepting a bloated session.
|
|
61
|
+
const threshold = options.threshold;
|
|
62
|
+
const overBudget = threshold != null && tokensBefore >= threshold;
|
|
63
|
+
const engineRefused = !didCompact && overBudget;
|
|
48
64
|
return {
|
|
49
|
-
ok:
|
|
65
|
+
ok: !engineRefused,
|
|
50
66
|
compacted: didCompact,
|
|
51
|
-
...(didCompact ? {} : { reason: "not_compacted" }),
|
|
67
|
+
...(didCompact ? {} : { reason: engineRefused ? "overbudget_not_compacted" : "not_compacted" }),
|
|
52
68
|
result: {
|
|
53
69
|
tokensBefore,
|
|
54
70
|
...(details.summaryMethod ? { summary: details.summaryMethod } : {}),
|
|
55
71
|
...(details.summaryText ? { summaryText: details.summaryText } : {}),
|
|
56
|
-
details,
|
|
72
|
+
details: { ...details, ...(threshold != null ? { threshold } : {}) },
|
|
57
73
|
},
|
|
58
74
|
};
|
|
59
75
|
}
|
|
@@ -96,14 +112,171 @@ function stringifyKernelBlock(block) {
|
|
|
96
112
|
/**
|
|
97
113
|
* Normalizes kernel content (string or block array) to a flat string.
|
|
98
114
|
*/
|
|
99
|
-
function normalizeKernelContent(content) {
|
|
100
|
-
|
|
101
|
-
|
|
115
|
+
function normalizeKernelContent(content, options = {}) {
|
|
116
|
+
const text = typeof content === "string"
|
|
117
|
+
? content
|
|
118
|
+
: Array.isArray(content)
|
|
119
|
+
? content.map(stringifyKernelBlock).filter((part) => part.length > 0).join("\n")
|
|
120
|
+
: "";
|
|
121
|
+
return stripOpenClawUntrustedMetadataEnvelope(text, {
|
|
122
|
+
retainContext: options.retainOpenClawContext === true,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
function stripOpenClawUntrustedMetadataEnvelope(text, options = {}) {
|
|
126
|
+
let remaining = text
|
|
127
|
+
.replace(OPENCLAW_LEADING_TIMESTAMP_PREFIX_RE, "")
|
|
128
|
+
.replace(/\r\n/g, "\n");
|
|
129
|
+
// Capture any preamble that precedes the first metadata header.
|
|
130
|
+
const preambleEnd = findFirstHeaderPosition(remaining);
|
|
131
|
+
let preamble = "";
|
|
132
|
+
if (preambleEnd > 0) {
|
|
133
|
+
const newlineIndex = remaining.lastIndexOf("\n", preambleEnd);
|
|
134
|
+
preamble = newlineIndex >= 0 ? remaining.slice(0, newlineIndex + 1) : remaining.slice(0, preambleEnd);
|
|
135
|
+
remaining = remaining.slice(preamble.length);
|
|
102
136
|
}
|
|
103
|
-
|
|
104
|
-
|
|
137
|
+
const retainedContext = [];
|
|
138
|
+
let stripped = false;
|
|
139
|
+
while (true) {
|
|
140
|
+
const next = stripOneOpenClawMetadataBlock(remaining);
|
|
141
|
+
if (next.text === remaining) {
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
stripped = true;
|
|
145
|
+
if (next.context.length > 0) {
|
|
146
|
+
retainedContext.push(...next.context);
|
|
147
|
+
}
|
|
148
|
+
remaining = next.text;
|
|
149
|
+
}
|
|
150
|
+
if (!stripped) {
|
|
151
|
+
return text;
|
|
152
|
+
}
|
|
153
|
+
const contextLine = options.retainContext === true
|
|
154
|
+
? formatRetainedOpenClawContext(retainedContext)
|
|
155
|
+
: "";
|
|
156
|
+
const strippedText = remaining.trimStart();
|
|
157
|
+
const result = contextLine ? `${contextLine}\n${strippedText}` : strippedText;
|
|
158
|
+
return preamble ? `${preamble}${result}` : result;
|
|
159
|
+
}
|
|
160
|
+
function findFirstHeaderPosition(text) {
|
|
161
|
+
let pos = -1;
|
|
162
|
+
for (const header of OPENCLAW_METADATA_HEADERS) {
|
|
163
|
+
const p = text.indexOf(header);
|
|
164
|
+
if (p >= 0 && (pos < 0 || p < pos)) {
|
|
165
|
+
pos = p;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return pos;
|
|
169
|
+
}
|
|
170
|
+
function stripOneOpenClawMetadataBlock(text) {
|
|
171
|
+
const leadingWhitespaceLength = text.length - text.trimStart().length;
|
|
172
|
+
const offsetText = text.slice(leadingWhitespaceLength);
|
|
173
|
+
const header = OPENCLAW_METADATA_HEADERS.find((candidate) => offsetText.startsWith(candidate)) ?? null;
|
|
174
|
+
if (!header) {
|
|
175
|
+
return { text, context: [] };
|
|
176
|
+
}
|
|
177
|
+
const afterHeader = offsetText.slice(header.length);
|
|
178
|
+
const fenceStartMatch = afterHeader.match(/^\n```(?:json)?\n/i);
|
|
179
|
+
if (!fenceStartMatch) {
|
|
180
|
+
const afterHeaderLines = afterHeader.replace(/^\n?/, "").split("\n");
|
|
181
|
+
const firstBlankIndex = afterHeaderLines.findIndex((line) => line.trim() === "");
|
|
182
|
+
if (firstBlankIndex < 0) {
|
|
183
|
+
// No fence and no blank line — cannot positively identify envelope shape.
|
|
184
|
+
// Return original text unchanged to avoid silently erasing content.
|
|
185
|
+
return { text, context: [] };
|
|
186
|
+
}
|
|
187
|
+
return { text: afterHeaderLines.slice(firstBlankIndex + 1).join("\n"), context: [] };
|
|
188
|
+
}
|
|
189
|
+
const bodyStart = header.length + fenceStartMatch[0].length;
|
|
190
|
+
const fenceEnd = offsetText.indexOf("\n```", bodyStart);
|
|
191
|
+
if (fenceEnd < 0) {
|
|
192
|
+
// Unclosed fence — cannot positively identify envelope shape.
|
|
193
|
+
return { text, context: [] };
|
|
194
|
+
}
|
|
195
|
+
const jsonText = offsetText.slice(bodyStart, fenceEnd);
|
|
196
|
+
const afterFence = fenceEnd + "\n```".length;
|
|
197
|
+
const trailingNewlineLength = offsetText.slice(afterFence).startsWith("\n") ? 1 : 0;
|
|
198
|
+
return {
|
|
199
|
+
text: offsetText.slice(afterFence + trailingNewlineLength),
|
|
200
|
+
context: summarizeOpenClawMetadataBlock(header, jsonText),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
function summarizeOpenClawMetadataBlock(header, jsonText) {
|
|
204
|
+
const parsed = parseJsonRecord(jsonText);
|
|
205
|
+
if (!parsed) {
|
|
206
|
+
return [];
|
|
207
|
+
}
|
|
208
|
+
if (header === "Conversation info (untrusted metadata):") {
|
|
209
|
+
const hasIMessageContext = firstString(parsed.chat_guid, parsed.chatGuid, parsed.chat_identifier, parsed.chatIdentifier, parsed.chat_name, parsed.chatName, parsed.service) != null;
|
|
210
|
+
return [
|
|
211
|
+
labelValue("channel", firstString(parsed.group_channel, parsed.channel, parsed.group_subject)),
|
|
212
|
+
labelValue("channel_id", firstString(parsed.chat_id, parsed.channel_id)),
|
|
213
|
+
labelValue("account_id", firstString(parsed.account_id, parsed.accountId)),
|
|
214
|
+
labelValue("provider", firstString(parsed.provider, parsed.surface)),
|
|
215
|
+
labelValue("chat_id", hasIMessageContext ? firstString(parsed.chat_id, parsed.chatId) : undefined),
|
|
216
|
+
labelValue("chat_guid", firstString(parsed.chat_guid, parsed.chatGuid)),
|
|
217
|
+
labelValue("chat_identifier", firstString(parsed.chat_identifier, parsed.chatIdentifier)),
|
|
218
|
+
labelValue("chat_name", firstString(parsed.chat_name, parsed.chatName)),
|
|
219
|
+
labelValue("is_group", firstString(parsed.is_group, parsed.isGroup, parsed.is_group_chat)),
|
|
220
|
+
labelValue("chat_type", firstString(parsed.chat_type, parsed.chatType)),
|
|
221
|
+
labelValue("service", firstString(parsed.service)),
|
|
222
|
+
labelValue("server_id", firstString(parsed.group_space, parsed.guild_id, parsed.server_id)),
|
|
223
|
+
labelValue("sender_id", firstString(parsed.sender_id, parsed.user_id)),
|
|
224
|
+
labelValue("sender", firstString(parsed.sender)),
|
|
225
|
+
labelValue("emoji_id", firstString(parsed.emoji_id, parsed.server_emoji_id, parsed.guild_emoji_id)),
|
|
226
|
+
labelValue("emoji", firstString(parsed.emoji_name, parsed.emoji)),
|
|
227
|
+
].filter(isNonEmptyString);
|
|
228
|
+
}
|
|
229
|
+
if (header === "Sender (untrusted metadata):") {
|
|
230
|
+
return [
|
|
231
|
+
labelValue("username", firstString(parsed.username, parsed.tag, parsed.name, parsed.label)),
|
|
232
|
+
labelValue("user_id", firstString(parsed.id, parsed.user_id, parsed.sender_id)),
|
|
233
|
+
labelValue("sender", firstString(parsed.sender, parsed.e164)),
|
|
234
|
+
].filter(isNonEmptyString);
|
|
235
|
+
}
|
|
236
|
+
return [];
|
|
237
|
+
}
|
|
238
|
+
function parseJsonRecord(jsonText) {
|
|
239
|
+
try {
|
|
240
|
+
const parsed = JSON.parse(jsonText);
|
|
241
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
|
242
|
+
? parsed
|
|
243
|
+
: null;
|
|
244
|
+
}
|
|
245
|
+
catch {
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function labelValue(label, value) {
|
|
250
|
+
return value ? `${label}=${sanitizeOpenClawContextValue(value)}` : "";
|
|
251
|
+
}
|
|
252
|
+
function firstString(...values) {
|
|
253
|
+
for (const value of values) {
|
|
254
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
255
|
+
return value.trim();
|
|
256
|
+
}
|
|
257
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
258
|
+
return String(value);
|
|
259
|
+
}
|
|
260
|
+
if (typeof value === "boolean") {
|
|
261
|
+
return String(value);
|
|
262
|
+
}
|
|
105
263
|
}
|
|
106
|
-
return
|
|
264
|
+
return undefined;
|
|
265
|
+
}
|
|
266
|
+
function sanitizeOpenClawContextValue(value) {
|
|
267
|
+
// 120 chars is a conservative bound for a single routing field value
|
|
268
|
+
// (channel name, server id, etc.). Any field exceeding this is likely
|
|
269
|
+
// malformed or adversarial input, not useful routing metadata.
|
|
270
|
+
return value.replace(/[\r\n;]+/g, " ").trim().slice(0, 120);
|
|
271
|
+
}
|
|
272
|
+
function formatRetainedOpenClawContext(values) {
|
|
273
|
+
const uniqueValues = [...new Set(values.filter(isNonEmptyString))];
|
|
274
|
+
return uniqueValues.length > 0
|
|
275
|
+
? `[OpenClaw context: ${uniqueValues.join("; ")}]`
|
|
276
|
+
: "";
|
|
277
|
+
}
|
|
278
|
+
function isNonEmptyString(value) {
|
|
279
|
+
return value.trim().length > 0;
|
|
107
280
|
}
|
|
108
281
|
/**
|
|
109
282
|
* Approximates token count for a text string.
|
|
@@ -375,18 +548,24 @@ function resolveAfterTurnPredictiveCompactionTokenCount(args) {
|
|
|
375
548
|
/**
|
|
376
549
|
* Normalizes a single kernel message into the kernel-compatible format.
|
|
377
550
|
*/
|
|
378
|
-
export function normalizeKernelMessage(message) {
|
|
551
|
+
export function normalizeKernelMessage(message, options = {}) {
|
|
379
552
|
return {
|
|
380
553
|
role: message.role,
|
|
381
|
-
content: normalizeKernelContent(message.content),
|
|
382
|
-
id: message.id
|
|
554
|
+
content: normalizeKernelContent(message.content, options),
|
|
555
|
+
id: typeof message.id === "string" ? message.id : randomUUID(),
|
|
383
556
|
};
|
|
384
557
|
}
|
|
385
558
|
/**
|
|
386
559
|
* Normalizes an array of kernel messages.
|
|
560
|
+
*
|
|
561
|
+
* Non-user messages whose normalized content is empty or whitespace-only
|
|
562
|
+
* are dropped. This prevents assistant/system turns that consisted entirely
|
|
563
|
+
* of stripped metadata from persisting as empty records.
|
|
387
564
|
*/
|
|
388
|
-
export function normalizeKernelMessages(messages) {
|
|
389
|
-
return messages
|
|
565
|
+
export function normalizeKernelMessages(messages, options = {}) {
|
|
566
|
+
return messages
|
|
567
|
+
.map((message) => normalizeKernelMessage(message, options))
|
|
568
|
+
.filter((message) => message.role === "user" || message.content.trim().length > 0);
|
|
390
569
|
}
|
|
391
570
|
/**
|
|
392
571
|
* Extracts tokens for exact recall matching from text.
|
|
@@ -912,8 +1091,10 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
912
1091
|
const request = buildCompactSessionRequest(args);
|
|
913
1092
|
try {
|
|
914
1093
|
const client = await runtime.getClient();
|
|
1094
|
+
const threshold = getDynamicCompactThreshold(args.tokenBudget);
|
|
915
1095
|
return normalizeCompactResult(await client.compactSession(request), {
|
|
916
1096
|
tokensBefore: args.currentTokenCount,
|
|
1097
|
+
...(threshold != null ? { threshold } : {}),
|
|
917
1098
|
});
|
|
918
1099
|
}
|
|
919
1100
|
catch (error) {
|
|
@@ -1019,6 +1200,9 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
1019
1200
|
sessionKey: args.sessionKey,
|
|
1020
1201
|
});
|
|
1021
1202
|
const messages = normalizeKernelMessages(args.messages);
|
|
1203
|
+
const strippedPrompt = args.prompt
|
|
1204
|
+
? normalizeKernelContent(args.prompt, { retainOpenClawContext: false })
|
|
1205
|
+
: "";
|
|
1022
1206
|
const lastUserMessage = findLastReplaySafeUserMessage(messages);
|
|
1023
1207
|
const reservedCurrentTurnTokens = lastUserMessage
|
|
1024
1208
|
? approximateMessageTokens(lastUserMessage)
|
|
@@ -1026,7 +1210,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
1026
1210
|
const currentContextTokens = resolvePredictiveCompactionTokenCount({
|
|
1027
1211
|
currentTokenCount: args.currentTokenCount,
|
|
1028
1212
|
messages,
|
|
1029
|
-
prompt:
|
|
1213
|
+
prompt: strippedPrompt,
|
|
1030
1214
|
});
|
|
1031
1215
|
const dynamicCompactThreshold = getDynamicCompactThreshold(args.tokenBudget);
|
|
1032
1216
|
const predictiveTargetSize = resolvePredictiveCompactionTarget({
|
|
@@ -1073,7 +1257,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
1073
1257
|
sessionId,
|
|
1074
1258
|
sessionKey: args.sessionKey,
|
|
1075
1259
|
userId,
|
|
1076
|
-
prompt:
|
|
1260
|
+
prompt: strippedPrompt,
|
|
1077
1261
|
messages,
|
|
1078
1262
|
tokenBudget: args.tokenBudget,
|
|
1079
1263
|
config: buildAssemblyConfig(args.tokenBudget),
|
|
@@ -1081,7 +1265,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
1081
1265
|
});
|
|
1082
1266
|
const assembled = normalizeAssembleResult(resp, args.messages);
|
|
1083
1267
|
let enforced = enforceTokenBudgetInvariant(await augmentWithExactRecall(assembled, {
|
|
1084
|
-
queryText:
|
|
1268
|
+
queryText: strippedPrompt || (messages[messages.length - 1]?.content ?? ""),
|
|
1085
1269
|
userId,
|
|
1086
1270
|
sessionId,
|
|
1087
1271
|
tokenBudget: args.tokenBudget,
|
|
@@ -1163,7 +1347,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
1163
1347
|
// Load manifest and normalize messages in parallel
|
|
1164
1348
|
const manifest = manifestStore.load(sessionId, logger);
|
|
1165
1349
|
const afterTurnMessages = selectAfterTurnMessages(args.messages, args.prePromptMessageCount, logger);
|
|
1166
|
-
const messages = normalizeKernelMessages(afterTurnMessages);
|
|
1350
|
+
const messages = normalizeKernelMessages(afterTurnMessages, { retainOpenClawContext: true });
|
|
1167
1351
|
// Find overlap: messages already in our manifest
|
|
1168
1352
|
const overlapIndex = manifestStore.findOverlapIndex(manifest, messages);
|
|
1169
1353
|
const newMessages = messages.slice(overlapIndex);
|
package/dist/index.js
CHANGED
|
@@ -26667,6 +26667,15 @@ var EXACT_RECALL_SEARCH_K = 32;
|
|
|
26667
26667
|
var EXACT_RECALL_MAX_TOKENS = 4;
|
|
26668
26668
|
var RESERVED_CURRENT_TURN_TOKENS = 150;
|
|
26669
26669
|
var AFTER_TURN_INGEST_MAX_TOKENS = 2048;
|
|
26670
|
+
var OPENCLAW_LEADING_TIMESTAMP_PREFIX_RE = /^\[[A-Za-z]{3} \d{4}-\d{2}-\d{2} \d{2}:\d{2}[^\]]*\] */;
|
|
26671
|
+
var OPENCLAW_METADATA_HEADERS = [
|
|
26672
|
+
"Conversation info (untrusted metadata):",
|
|
26673
|
+
"Sender (untrusted metadata):",
|
|
26674
|
+
"Thread starter (untrusted, for context):",
|
|
26675
|
+
"Reply target of current user message (untrusted, for context):",
|
|
26676
|
+
"Forwarded message context (untrusted metadata):",
|
|
26677
|
+
"Chat history since last reply (untrusted, for context):"
|
|
26678
|
+
];
|
|
26670
26679
|
var COMMON_QUERY_WORDS = /* @__PURE__ */ new Set([
|
|
26671
26680
|
"what",
|
|
26672
26681
|
"does",
|
|
@@ -26714,15 +26723,18 @@ function normalizeCompactResult(response, options = {}) {
|
|
|
26714
26723
|
meanConfidence: typeof response?.meanConfidence === "number" ? response.meanConfidence : void 0,
|
|
26715
26724
|
summaryText: typeof response?.summaryText === "string" && response.summaryText.length > 0 ? response.summaryText : void 0
|
|
26716
26725
|
};
|
|
26726
|
+
const threshold = options.threshold;
|
|
26727
|
+
const overBudget = threshold != null && tokensBefore >= threshold;
|
|
26728
|
+
const engineRefused = !didCompact && overBudget;
|
|
26717
26729
|
return {
|
|
26718
|
-
ok:
|
|
26730
|
+
ok: !engineRefused,
|
|
26719
26731
|
compacted: didCompact,
|
|
26720
|
-
...didCompact ? {} : { reason: "not_compacted" },
|
|
26732
|
+
...didCompact ? {} : { reason: engineRefused ? "overbudget_not_compacted" : "not_compacted" },
|
|
26721
26733
|
result: {
|
|
26722
26734
|
tokensBefore,
|
|
26723
26735
|
...details.summaryMethod ? { summary: details.summaryMethod } : {},
|
|
26724
26736
|
...details.summaryText ? { summaryText: details.summaryText } : {},
|
|
26725
|
-
details
|
|
26737
|
+
details: { ...details, ...threshold != null ? { threshold } : {} }
|
|
26726
26738
|
}
|
|
26727
26739
|
};
|
|
26728
26740
|
}
|
|
@@ -26757,14 +26769,160 @@ function stringifyKernelBlock(block) {
|
|
|
26757
26769
|
return typeof record.text === "string" ? record.text : "";
|
|
26758
26770
|
}
|
|
26759
26771
|
}
|
|
26760
|
-
function normalizeKernelContent(content) {
|
|
26761
|
-
|
|
26762
|
-
|
|
26772
|
+
function normalizeKernelContent(content, options = {}) {
|
|
26773
|
+
const text = typeof content === "string" ? content : Array.isArray(content) ? content.map(stringifyKernelBlock).filter((part) => part.length > 0).join("\n") : "";
|
|
26774
|
+
return stripOpenClawUntrustedMetadataEnvelope(text, {
|
|
26775
|
+
retainContext: options.retainOpenClawContext === true
|
|
26776
|
+
});
|
|
26777
|
+
}
|
|
26778
|
+
function stripOpenClawUntrustedMetadataEnvelope(text, options = {}) {
|
|
26779
|
+
let remaining = text.replace(OPENCLAW_LEADING_TIMESTAMP_PREFIX_RE, "").replace(/\r\n/g, "\n");
|
|
26780
|
+
const preambleEnd = findFirstHeaderPosition(remaining);
|
|
26781
|
+
let preamble = "";
|
|
26782
|
+
if (preambleEnd > 0) {
|
|
26783
|
+
const newlineIndex = remaining.lastIndexOf("\n", preambleEnd);
|
|
26784
|
+
preamble = newlineIndex >= 0 ? remaining.slice(0, newlineIndex + 1) : remaining.slice(0, preambleEnd);
|
|
26785
|
+
remaining = remaining.slice(preamble.length);
|
|
26786
|
+
}
|
|
26787
|
+
const retainedContext = [];
|
|
26788
|
+
let stripped = false;
|
|
26789
|
+
while (true) {
|
|
26790
|
+
const next = stripOneOpenClawMetadataBlock(remaining);
|
|
26791
|
+
if (next.text === remaining) {
|
|
26792
|
+
break;
|
|
26793
|
+
}
|
|
26794
|
+
stripped = true;
|
|
26795
|
+
if (next.context.length > 0) {
|
|
26796
|
+
retainedContext.push(...next.context);
|
|
26797
|
+
}
|
|
26798
|
+
remaining = next.text;
|
|
26763
26799
|
}
|
|
26764
|
-
if (!
|
|
26765
|
-
return
|
|
26800
|
+
if (!stripped) {
|
|
26801
|
+
return text;
|
|
26802
|
+
}
|
|
26803
|
+
const contextLine = options.retainContext === true ? formatRetainedOpenClawContext(retainedContext) : "";
|
|
26804
|
+
const strippedText = remaining.trimStart();
|
|
26805
|
+
const result = contextLine ? `${contextLine}
|
|
26806
|
+
${strippedText}` : strippedText;
|
|
26807
|
+
return preamble ? `${preamble}${result}` : result;
|
|
26808
|
+
}
|
|
26809
|
+
function findFirstHeaderPosition(text) {
|
|
26810
|
+
let pos = -1;
|
|
26811
|
+
for (const header of OPENCLAW_METADATA_HEADERS) {
|
|
26812
|
+
const p = text.indexOf(header);
|
|
26813
|
+
if (p >= 0 && (pos < 0 || p < pos)) {
|
|
26814
|
+
pos = p;
|
|
26815
|
+
}
|
|
26816
|
+
}
|
|
26817
|
+
return pos;
|
|
26818
|
+
}
|
|
26819
|
+
function stripOneOpenClawMetadataBlock(text) {
|
|
26820
|
+
const leadingWhitespaceLength = text.length - text.trimStart().length;
|
|
26821
|
+
const offsetText = text.slice(leadingWhitespaceLength);
|
|
26822
|
+
const header = OPENCLAW_METADATA_HEADERS.find((candidate) => offsetText.startsWith(candidate)) ?? null;
|
|
26823
|
+
if (!header) {
|
|
26824
|
+
return { text, context: [] };
|
|
26825
|
+
}
|
|
26826
|
+
const afterHeader = offsetText.slice(header.length);
|
|
26827
|
+
const fenceStartMatch = afterHeader.match(/^\n```(?:json)?\n/i);
|
|
26828
|
+
if (!fenceStartMatch) {
|
|
26829
|
+
const afterHeaderLines = afterHeader.replace(/^\n?/, "").split("\n");
|
|
26830
|
+
const firstBlankIndex = afterHeaderLines.findIndex((line) => line.trim() === "");
|
|
26831
|
+
if (firstBlankIndex < 0) {
|
|
26832
|
+
return { text, context: [] };
|
|
26833
|
+
}
|
|
26834
|
+
return { text: afterHeaderLines.slice(firstBlankIndex + 1).join("\n"), context: [] };
|
|
26835
|
+
}
|
|
26836
|
+
const bodyStart = header.length + fenceStartMatch[0].length;
|
|
26837
|
+
const fenceEnd = offsetText.indexOf("\n```", bodyStart);
|
|
26838
|
+
if (fenceEnd < 0) {
|
|
26839
|
+
return { text, context: [] };
|
|
26840
|
+
}
|
|
26841
|
+
const jsonText = offsetText.slice(bodyStart, fenceEnd);
|
|
26842
|
+
const afterFence = fenceEnd + "\n```".length;
|
|
26843
|
+
const trailingNewlineLength = offsetText.slice(afterFence).startsWith("\n") ? 1 : 0;
|
|
26844
|
+
return {
|
|
26845
|
+
text: offsetText.slice(afterFence + trailingNewlineLength),
|
|
26846
|
+
context: summarizeOpenClawMetadataBlock(header, jsonText)
|
|
26847
|
+
};
|
|
26848
|
+
}
|
|
26849
|
+
function summarizeOpenClawMetadataBlock(header, jsonText) {
|
|
26850
|
+
const parsed = parseJsonRecord(jsonText);
|
|
26851
|
+
if (!parsed) {
|
|
26852
|
+
return [];
|
|
26853
|
+
}
|
|
26854
|
+
if (header === "Conversation info (untrusted metadata):") {
|
|
26855
|
+
const hasIMessageContext = firstString2(
|
|
26856
|
+
parsed.chat_guid,
|
|
26857
|
+
parsed.chatGuid,
|
|
26858
|
+
parsed.chat_identifier,
|
|
26859
|
+
parsed.chatIdentifier,
|
|
26860
|
+
parsed.chat_name,
|
|
26861
|
+
parsed.chatName,
|
|
26862
|
+
parsed.service
|
|
26863
|
+
) != null;
|
|
26864
|
+
return [
|
|
26865
|
+
labelValue("channel", firstString2(parsed.group_channel, parsed.channel, parsed.group_subject)),
|
|
26866
|
+
labelValue("channel_id", firstString2(parsed.chat_id, parsed.channel_id)),
|
|
26867
|
+
labelValue("account_id", firstString2(parsed.account_id, parsed.accountId)),
|
|
26868
|
+
labelValue("provider", firstString2(parsed.provider, parsed.surface)),
|
|
26869
|
+
labelValue("chat_id", hasIMessageContext ? firstString2(parsed.chat_id, parsed.chatId) : void 0),
|
|
26870
|
+
labelValue("chat_guid", firstString2(parsed.chat_guid, parsed.chatGuid)),
|
|
26871
|
+
labelValue("chat_identifier", firstString2(parsed.chat_identifier, parsed.chatIdentifier)),
|
|
26872
|
+
labelValue("chat_name", firstString2(parsed.chat_name, parsed.chatName)),
|
|
26873
|
+
labelValue("is_group", firstString2(parsed.is_group, parsed.isGroup, parsed.is_group_chat)),
|
|
26874
|
+
labelValue("chat_type", firstString2(parsed.chat_type, parsed.chatType)),
|
|
26875
|
+
labelValue("service", firstString2(parsed.service)),
|
|
26876
|
+
labelValue("server_id", firstString2(parsed.group_space, parsed.guild_id, parsed.server_id)),
|
|
26877
|
+
labelValue("sender_id", firstString2(parsed.sender_id, parsed.user_id)),
|
|
26878
|
+
labelValue("sender", firstString2(parsed.sender)),
|
|
26879
|
+
labelValue("emoji_id", firstString2(parsed.emoji_id, parsed.server_emoji_id, parsed.guild_emoji_id)),
|
|
26880
|
+
labelValue("emoji", firstString2(parsed.emoji_name, parsed.emoji))
|
|
26881
|
+
].filter(isNonEmptyString);
|
|
26882
|
+
}
|
|
26883
|
+
if (header === "Sender (untrusted metadata):") {
|
|
26884
|
+
return [
|
|
26885
|
+
labelValue("username", firstString2(parsed.username, parsed.tag, parsed.name, parsed.label)),
|
|
26886
|
+
labelValue("user_id", firstString2(parsed.id, parsed.user_id, parsed.sender_id)),
|
|
26887
|
+
labelValue("sender", firstString2(parsed.sender, parsed.e164))
|
|
26888
|
+
].filter(isNonEmptyString);
|
|
26889
|
+
}
|
|
26890
|
+
return [];
|
|
26891
|
+
}
|
|
26892
|
+
function parseJsonRecord(jsonText) {
|
|
26893
|
+
try {
|
|
26894
|
+
const parsed = JSON.parse(jsonText);
|
|
26895
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
26896
|
+
} catch {
|
|
26897
|
+
return null;
|
|
26898
|
+
}
|
|
26899
|
+
}
|
|
26900
|
+
function labelValue(label, value) {
|
|
26901
|
+
return value ? `${label}=${sanitizeOpenClawContextValue(value)}` : "";
|
|
26902
|
+
}
|
|
26903
|
+
function firstString2(...values) {
|
|
26904
|
+
for (const value of values) {
|
|
26905
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
26906
|
+
return value.trim();
|
|
26907
|
+
}
|
|
26908
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
26909
|
+
return String(value);
|
|
26910
|
+
}
|
|
26911
|
+
if (typeof value === "boolean") {
|
|
26912
|
+
return String(value);
|
|
26913
|
+
}
|
|
26766
26914
|
}
|
|
26767
|
-
return
|
|
26915
|
+
return void 0;
|
|
26916
|
+
}
|
|
26917
|
+
function sanitizeOpenClawContextValue(value) {
|
|
26918
|
+
return value.replace(/[\r\n;]+/g, " ").trim().slice(0, 120);
|
|
26919
|
+
}
|
|
26920
|
+
function formatRetainedOpenClawContext(values) {
|
|
26921
|
+
const uniqueValues = [...new Set(values.filter(isNonEmptyString))];
|
|
26922
|
+
return uniqueValues.length > 0 ? `[OpenClaw context: ${uniqueValues.join("; ")}]` : "";
|
|
26923
|
+
}
|
|
26924
|
+
function isNonEmptyString(value) {
|
|
26925
|
+
return value.trim().length > 0;
|
|
26768
26926
|
}
|
|
26769
26927
|
function approximateTokenCount(text) {
|
|
26770
26928
|
if (typeof text === "string") {
|
|
@@ -26979,15 +27137,15 @@ function resolveAfterTurnPredictiveCompactionTokenCount(args) {
|
|
|
26979
27137
|
}
|
|
26980
27138
|
return Math.max(currentTokenCount, forwardedMessageTokens);
|
|
26981
27139
|
}
|
|
26982
|
-
function normalizeKernelMessage(message) {
|
|
27140
|
+
function normalizeKernelMessage(message, options = {}) {
|
|
26983
27141
|
return {
|
|
26984
27142
|
role: message.role,
|
|
26985
|
-
content: normalizeKernelContent(message.content),
|
|
26986
|
-
id: message.id
|
|
27143
|
+
content: normalizeKernelContent(message.content, options),
|
|
27144
|
+
id: typeof message.id === "string" ? message.id : randomUUID()
|
|
26987
27145
|
};
|
|
26988
27146
|
}
|
|
26989
|
-
function normalizeKernelMessages(messages) {
|
|
26990
|
-
return messages.map((message) => normalizeKernelMessage(message));
|
|
27147
|
+
function normalizeKernelMessages(messages, options = {}) {
|
|
27148
|
+
return messages.map((message) => normalizeKernelMessage(message, options)).filter((message) => message.role === "user" || message.content.trim().length > 0);
|
|
26991
27149
|
}
|
|
26992
27150
|
function extractExactRecallTokens(text) {
|
|
26993
27151
|
const tokens = /* @__PURE__ */ new Set();
|
|
@@ -27428,8 +27586,10 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
27428
27586
|
const request3 = buildCompactSessionRequest(args);
|
|
27429
27587
|
try {
|
|
27430
27588
|
const client = await runtime.getClient();
|
|
27589
|
+
const threshold = getDynamicCompactThreshold(args.tokenBudget);
|
|
27431
27590
|
return normalizeCompactResult(await client.compactSession(request3), {
|
|
27432
|
-
tokensBefore: args.currentTokenCount
|
|
27591
|
+
tokensBefore: args.currentTokenCount,
|
|
27592
|
+
...threshold != null ? { threshold } : {}
|
|
27433
27593
|
});
|
|
27434
27594
|
} catch (error2) {
|
|
27435
27595
|
return {
|
|
@@ -27533,12 +27693,13 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
27533
27693
|
sessionKey: args.sessionKey
|
|
27534
27694
|
});
|
|
27535
27695
|
const messages = normalizeKernelMessages(args.messages);
|
|
27696
|
+
const strippedPrompt = args.prompt ? normalizeKernelContent(args.prompt, { retainOpenClawContext: false }) : "";
|
|
27536
27697
|
const lastUserMessage = findLastReplaySafeUserMessage(messages);
|
|
27537
27698
|
const reservedCurrentTurnTokens = lastUserMessage ? approximateMessageTokens(lastUserMessage) : RESERVED_CURRENT_TURN_TOKENS;
|
|
27538
27699
|
const currentContextTokens = resolvePredictiveCompactionTokenCount({
|
|
27539
27700
|
currentTokenCount: args.currentTokenCount,
|
|
27540
27701
|
messages,
|
|
27541
|
-
prompt:
|
|
27702
|
+
prompt: strippedPrompt
|
|
27542
27703
|
});
|
|
27543
27704
|
const dynamicCompactThreshold = getDynamicCompactThreshold(args.tokenBudget);
|
|
27544
27705
|
const predictiveTargetSize = resolvePredictiveCompactionTarget({
|
|
@@ -27586,7 +27747,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
27586
27747
|
sessionId,
|
|
27587
27748
|
sessionKey: args.sessionKey,
|
|
27588
27749
|
userId,
|
|
27589
|
-
prompt:
|
|
27750
|
+
prompt: strippedPrompt,
|
|
27590
27751
|
messages,
|
|
27591
27752
|
tokenBudget: args.tokenBudget,
|
|
27592
27753
|
config: buildAssemblyConfig(args.tokenBudget),
|
|
@@ -27595,7 +27756,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
27595
27756
|
const assembled = normalizeAssembleResult(resp, args.messages);
|
|
27596
27757
|
let enforced = enforceTokenBudgetInvariant(
|
|
27597
27758
|
await augmentWithExactRecall(assembled, {
|
|
27598
|
-
queryText:
|
|
27759
|
+
queryText: strippedPrompt || (messages[messages.length - 1]?.content ?? ""),
|
|
27599
27760
|
userId,
|
|
27600
27761
|
sessionId,
|
|
27601
27762
|
tokenBudget: args.tokenBudget,
|
|
@@ -27680,7 +27841,7 @@ function buildContextEngineFactory(runtime, cfg, logger = console) {
|
|
|
27680
27841
|
});
|
|
27681
27842
|
const manifest = manifestStore.load(sessionId, logger);
|
|
27682
27843
|
const afterTurnMessages = selectAfterTurnMessages(args.messages, args.prePromptMessageCount, logger);
|
|
27683
|
-
const messages = normalizeKernelMessages(afterTurnMessages);
|
|
27844
|
+
const messages = normalizeKernelMessages(afterTurnMessages, { retainOpenClawContext: true });
|
|
27684
27845
|
const overlapIndex = manifestStore.findOverlapIndex(manifest, messages);
|
|
27685
27846
|
const newMessages = messages.slice(overlapIndex);
|
|
27686
27847
|
const ingestMessages = boundAfterTurnMessagesForIngest(newMessages, logger, sessionId);
|
package/openclaw.plugin.json
CHANGED