@xdarkicex/openclaw-memory-libravdb 1.6.25 → 1.6.28

package/dist/context-engine.js

@@ -1,5 +1,7 @@
+import { randomUUID } from "node:crypto";
 import { resolveIdentity } from "./identity.js";
 import { resolveUserCollection } from "./memory-scopes.js";
+import { manifestStore } from "./manifest.js";
 const APPROX_CHARS_PER_TOKEN = 4;
 const PROMPT_AUTHORITY_PREASSEMBLY_MAY_OVERFLOW = "preassembly_may_overflow";
 const ASSEMBLE_BUDGET_HEADROOM_TOKENS = 256;
@@ -12,6 +14,15 @@ const EXACT_RECALL_SEARCH_K = 32;
 const EXACT_RECALL_MAX_TOKENS = 4;
 const RESERVED_CURRENT_TURN_TOKENS = 150;
 const AFTER_TURN_INGEST_MAX_TOKENS = 2048;
+const OPENCLAW_LEADING_TIMESTAMP_PREFIX_RE = /^\[[A-Za-z]{3} \d{4}-\d{2}-\d{2} \d{2}:\d{2}[^\]]*\] */;
+const OPENCLAW_METADATA_HEADERS = [
+    "Conversation info (untrusted metadata):",
+    "Sender (untrusted metadata):",
+    "Thread starter (untrusted, for context):",
+    "Reply target of current user message (untrusted, for context):",
+    "Forwarded message context (untrusted metadata):",
+    "Chat history since last reply (untrusted, for context):",
+];
 const COMMON_QUERY_WORDS = new Set([
     "what", "does", "mean", "remember", "recall", "about", "this", "that",
     "the", "and", "for", "with", "from", "your", "have", "been", "were",
@@ -94,14 +105,171 @@ function stringifyKernelBlock(block) {
 /**
  * Normalizes kernel content (string or block array) to a flat string.
  */
-function normalizeKernelContent(content) {
-    if (typeof content === "string") {
-        return content;
+function normalizeKernelContent(content, options = {}) {
+    const text = typeof content === "string"
+        ? content
+        : Array.isArray(content)
+            ? content.map(stringifyKernelBlock).filter((part) => part.length > 0).join("\n")
+            : "";
+    return stripOpenClawUntrustedMetadataEnvelope(text, {
+        retainContext: options.retainOpenClawContext === true,
+    });
+}
+function stripOpenClawUntrustedMetadataEnvelope(text, options = {}) {
+    let remaining = text
+        .replace(OPENCLAW_LEADING_TIMESTAMP_PREFIX_RE, "")
+        .replace(/\r\n/g, "\n");
+    // Capture any preamble that precedes the first metadata header.
+    const preambleEnd = findFirstHeaderPosition(remaining);
+    let preamble = "";
+    if (preambleEnd > 0) {
+        const newlineIndex = remaining.lastIndexOf("\n", preambleEnd);
+        preamble = newlineIndex >= 0 ? remaining.slice(0, newlineIndex + 1) : remaining.slice(0, preambleEnd);
+        remaining = remaining.slice(preamble.length);
     }
-    if (!Array.isArray(content)) {
-        return "";
+    const retainedContext = [];
+    let stripped = false;
+    while (true) {
+        const next = stripOneOpenClawMetadataBlock(remaining);
+        if (next.text === remaining) {
+            break;
+        }
+        stripped = true;
+        if (next.context.length > 0) {
+            retainedContext.push(...next.context);
+        }
+        remaining = next.text;
+    }
+    if (!stripped) {
+        return text;
+    }
+    const contextLine = options.retainContext === true
+        ? formatRetainedOpenClawContext(retainedContext)
+        : "";
+    const strippedText = remaining.trimStart();
+    const result = contextLine ? `${contextLine}\n${strippedText}` : strippedText;
+    return preamble ? `${preamble}${result}` : result;
+}
+function findFirstHeaderPosition(text) {
+    let pos = -1;
+    for (const header of OPENCLAW_METADATA_HEADERS) {
+        const p = text.indexOf(header);
+        if (p >= 0 && (pos < 0 || p < pos)) {
+            pos = p;
+        }
+    }
+    return pos;
+}
+function stripOneOpenClawMetadataBlock(text) {
+    const leadingWhitespaceLength = text.length - text.trimStart().length;
+    const offsetText = text.slice(leadingWhitespaceLength);
+    const header = OPENCLAW_METADATA_HEADERS.find((candidate) => offsetText.startsWith(candidate)) ?? null;
+    if (!header) {
+        return { text, context: [] };
+    }
+    const afterHeader = offsetText.slice(header.length);
+    const fenceStartMatch = afterHeader.match(/^\n```(?:json)?\n/i);
+    if (!fenceStartMatch) {
+        const afterHeaderLines = afterHeader.replace(/^\n?/, "").split("\n");
+        const firstBlankIndex = afterHeaderLines.findIndex((line) => line.trim() === "");
+        if (firstBlankIndex < 0) {
+            // No fence and no blank line — cannot positively identify envelope shape.
+            // Return original text unchanged to avoid silently erasing content.
+            return { text, context: [] };
+        }
+        return { text: afterHeaderLines.slice(firstBlankIndex + 1).join("\n"), context: [] };
+    }
+    const bodyStart = header.length + fenceStartMatch[0].length;
+    const fenceEnd = offsetText.indexOf("\n```", bodyStart);
+    if (fenceEnd < 0) {
+        // Unclosed fence — cannot positively identify envelope shape.
+        return { text, context: [] };
+    }
+    const jsonText = offsetText.slice(bodyStart, fenceEnd);
+    const afterFence = fenceEnd + "\n```".length;
+    const trailingNewlineLength = offsetText.slice(afterFence).startsWith("\n") ? 1 : 0;
+    return {
+        text: offsetText.slice(afterFence + trailingNewlineLength),
+        context: summarizeOpenClawMetadataBlock(header, jsonText),
+    };
+}
+function summarizeOpenClawMetadataBlock(header, jsonText) {
+    const parsed = parseJsonRecord(jsonText);
+    if (!parsed) {
+        return [];
+    }
+    if (header === "Conversation info (untrusted metadata):") {
+        const hasIMessageContext = firstString(parsed.chat_guid, parsed.chatGuid, parsed.chat_identifier, parsed.chatIdentifier, parsed.chat_name, parsed.chatName, parsed.service) != null;
+        return [
+            labelValue("channel", firstString(parsed.group_channel, parsed.channel, parsed.group_subject)),
+            labelValue("channel_id", firstString(parsed.chat_id, parsed.channel_id)),
+            labelValue("account_id", firstString(parsed.account_id, parsed.accountId)),
+            labelValue("provider", firstString(parsed.provider, parsed.surface)),
+            labelValue("chat_id", hasIMessageContext ? firstString(parsed.chat_id, parsed.chatId) : undefined),
+            labelValue("chat_guid", firstString(parsed.chat_guid, parsed.chatGuid)),
+            labelValue("chat_identifier", firstString(parsed.chat_identifier, parsed.chatIdentifier)),
+            labelValue("chat_name", firstString(parsed.chat_name, parsed.chatName)),
+            labelValue("is_group", firstString(parsed.is_group, parsed.isGroup, parsed.is_group_chat)),
+            labelValue("chat_type", firstString(parsed.chat_type, parsed.chatType)),
+            labelValue("service", firstString(parsed.service)),
+            labelValue("server_id", firstString(parsed.group_space, parsed.guild_id, parsed.server_id)),
+            labelValue("sender_id", firstString(parsed.sender_id, parsed.user_id)),
+            labelValue("sender", firstString(parsed.sender)),
+            labelValue("emoji_id", firstString(parsed.emoji_id, parsed.server_emoji_id, parsed.guild_emoji_id)),
+            labelValue("emoji", firstString(parsed.emoji_name, parsed.emoji)),
+        ].filter(isNonEmptyString);
+    }
+    if (header === "Sender (untrusted metadata):") {
+        return [
+            labelValue("username", firstString(parsed.username, parsed.tag, parsed.name, parsed.label)),
+            labelValue("user_id", firstString(parsed.id, parsed.user_id, parsed.sender_id)),
+            labelValue("sender", firstString(parsed.sender, parsed.e164)),
+        ].filter(isNonEmptyString);
     }
-    return content.map(stringifyKernelBlock).filter((part) => part.length > 0).join("\n");
+    return [];
+}
+function parseJsonRecord(jsonText) {
+    try {
+        const parsed = JSON.parse(jsonText);
+        return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+            ? parsed
+            : null;
+    }
+    catch {
+        return null;
+    }
+}
+function labelValue(label, value) {
+    return value ? `${label}=${sanitizeOpenClawContextValue(value)}` : "";
+}
+function firstString(...values) {
+    for (const value of values) {
+        if (typeof value === "string" && value.trim().length > 0) {
+            return value.trim();
+        }
+        if (typeof value === "number" && Number.isFinite(value)) {
+            return String(value);
+        }
+        if (typeof value === "boolean") {
+            return String(value);
+        }
+    }
+    return undefined;
+}
+function sanitizeOpenClawContextValue(value) {
+    // 120 chars is a conservative bound for a single routing field value
+    // (channel name, server id, etc.). Any field exceeding this is likely
+    // malformed or adversarial input, not useful routing metadata.
+    return value.replace(/[\r\n;]+/g, " ").trim().slice(0, 120);
+}
+function formatRetainedOpenClawContext(values) {
+    const uniqueValues = [...new Set(values.filter(isNonEmptyString))];
+    return uniqueValues.length > 0
+        ? `[OpenClaw context: ${uniqueValues.join("; ")}]`
+        : "";
+}
+function isNonEmptyString(value) {
+    return value.trim().length > 0;
 }
 /**
  * Approximates token count for a text string.
@@ -373,18 +541,24 @@ function resolveAfterTurnPredictiveCompactionTokenCount(args) {
 /**
  * Normalizes a single kernel message into the kernel-compatible format.
  */
-export function normalizeKernelMessage(message) {
+export function normalizeKernelMessage(message, options = {}) {
     return {
         role: message.role,
-        content: normalizeKernelContent(message.content),
-        ...(typeof message.id === "string" ? { id: message.id } : {}),
+        content: normalizeKernelContent(message.content, options),
+        id: typeof message.id === "string" ? message.id : randomUUID(),
     };
 }
 /**
  * Normalizes an array of kernel messages.
+ *
+ * Non-user messages whose normalized content is empty or whitespace-only
+ * are dropped. This prevents assistant/system turns that consisted entirely
+ * of stripped metadata from persisting as empty records.
  */
-export function normalizeKernelMessages(messages) {
-    return messages.map((message) => normalizeKernelMessage(message));
+export function normalizeKernelMessages(messages, options = {}) {
+    return messages
+        .map((message) => normalizeKernelMessage(message, options))
+        .filter((message) => message.role === "user" || message.content.trim().length > 0);
 }
 /**
  * Extracts tokens for exact recall matching from text.
@@ -471,9 +645,12 @@ function escapeMemoryFactText(text) {
         .replaceAll("\t", "&#9;");
 }
 // Tool-call pattern detection for sanitization
-const TOOL_CALL_BRACKET_RE = /\[tool:([^\]]+)\]/gi;
-const TOOL_CALL_JSON_RE = /\{\s*"name"\s*:\s*"([^"]+)"[^}]*\}/g;
-const TOOL_RESULT_ANNOTATION_RE = /\[tool:[^\]]+\](?:\s*[^{\[]*)?/g;
+// Matches [tool:name] followed by optional whitespace and any trailing JSON object {...}, array [...], or string "..."
+const TOOL_CALL_BRACKET_RE = /\[tool:([^\]]+)\](?:\s*(?:\{[\s\S]*?\}|\[[\s\S]*?\]|".*?"))?/gi;
+// Matches raw JSON tool-call objects targeting a "name\" field
+const TOOL_CALL_JSON_RE = /\{\s*"name"\s*:\s*"([^"]+)"[\s\S]*?\}/g;
+// Matches older annotations, aggressively consuming trailing characters on the same line
+const TOOL_RESULT_ANNOTATION_RE = /\[tool:[^\]]+\][^\n]*/g;
 /**
  * Sanitizes text that may contain tool-call syntax to prevent loop-priming.
  * Replaces executable-looking patterns with neutral summaries rather than
@@ -697,9 +874,10 @@ export function normalizeAssembleResult(result, sourceMessages) {
                 isRealTranscript = message.role === "user" || message.role === "assistant";
             }
             if (isRealTranscript) {
+                // BUG PATH A SEALED: Sanitize the content before pushing to the trajectory
                 messages.push({
                     role: message.role === "user" ? "user" : "assistant",
-                    content,
+                    content: sanitizeToolCallPatterns(content),
                     ...(typeof message.id === "string" ? { id: message.id } : {}),
                 });
             }
@@ -724,6 +902,20 @@ export function normalizeAssembleResult(result, sourceMessages) {
         ...(result.debug != null ? { debug: result.debug } : {}),
     };
 }
+function extractCursorFromResult(result) {
+    if (result && typeof result === "object" && "cursor" in result) {
+        const cursor = result.cursor;
+        if (cursor && typeof cursor === "object") {
+            const c = cursor;
+            if (typeof c.lastProcessedIndex === "number" &&
+                typeof c.sessionVersion === "number" &&
+                typeof c.manifestTailHash === "string") {
+                return c;
+            }
+        }
+    }
+    return undefined;
+}
 /**
  * Builds the context engine factory with the given client getter.
  */
@@ -999,6 +1191,9 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                 sessionKey: args.sessionKey,
             });
             const messages = normalizeKernelMessages(args.messages);
+            const strippedPrompt = args.prompt
+                ? normalizeKernelContent(args.prompt, { retainOpenClawContext: false })
+                : "";
             const lastUserMessage = findLastReplaySafeUserMessage(messages);
             const reservedCurrentTurnTokens = lastUserMessage
                 ? approximateMessageTokens(lastUserMessage)
@@ -1006,7 +1201,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
             const currentContextTokens = resolvePredictiveCompactionTokenCount({
                 currentTokenCount: args.currentTokenCount,
                 messages,
-                prompt: args.prompt,
+                prompt: strippedPrompt,
             });
             const dynamicCompactThreshold = getDynamicCompactThreshold(args.tokenBudget);
             const predictiveTargetSize = resolvePredictiveCompactionTarget({
@@ -1053,7 +1248,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                     sessionId,
                     sessionKey: args.sessionKey,
                     userId,
-                    prompt: args.prompt ?? "",
+                    prompt: strippedPrompt,
                     messages,
                     tokenBudget: args.tokenBudget,
                     config: buildAssemblyConfig(args.tokenBudget),
@@ -1061,7 +1256,7 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                 });
                 const assembled = normalizeAssembleResult(resp, args.messages);
                 let enforced = enforceTokenBudgetInvariant(await augmentWithExactRecall(assembled, {
-                    queryText: args.prompt ?? messages[messages.length - 1]?.content ?? "",
+                    queryText: strippedPrompt || (messages[messages.length - 1]?.content ?? ""),
                     userId,
                     sessionId,
                     tokenBudget: args.tokenBudget,
@@ -1140,12 +1335,24 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                 userIdOverride: args.userId,
                 sessionKey: args.sessionKey,
             });
+            // Load manifest and normalize messages in parallel
+            const manifest = manifestStore.load(sessionId, logger);
             const afterTurnMessages = selectAfterTurnMessages(args.messages, args.prePromptMessageCount, logger);
-            const messages = normalizeKernelMessages(afterTurnMessages);
-            const ingestMessages = boundAfterTurnMessagesForIngest(messages, logger, sessionId);
-            const msgCount = messages.length;
+            const messages = normalizeKernelMessages(afterTurnMessages, { retainOpenClawContext: true });
+            // Find overlap: messages already in our manifest
+            const overlapIndex = manifestStore.findOverlapIndex(manifest, messages);
+            const newMessages = messages.slice(overlapIndex);
+            // Apply token budget cap only to new messages
+            const ingestMessages = boundAfterTurnMessagesForIngest(newMessages, logger, sessionId);
+            const startIndex = manifestStore.deriveStartingIndex(manifest, args.prePromptMessageCount);
+            const cursor = {
+                lastProcessedIndex: startIndex > 0 ? startIndex - 1 : 0,
+                sessionVersion: manifest.version,
+                manifestTailHash: manifest.tailHash,
+            };
             logger.info?.(`LibraVDB afterTurn sessionId=${sessionId} userId=${userId} ` +
-                `messageCount=${msgCount} totalMessages=${args.messages.length} ` +
+                `messageCount=${messages.length} newMessages=${newMessages.length} ` +
+                `overlapIndex=${overlapIndex} startIndex=${startIndex} ` +
                 `prePromptMessageCount=${args.prePromptMessageCount ?? "unknown"} ` +
                 `heartbeat=${args.isHeartbeat ?? false}`);
             try {
@@ -1158,8 +1365,40 @@ export function buildContextEngineFactory(runtime, cfg, logger = console) {
                     sessionKey: args.sessionKey,
                     userId,
                     messages: ingestMessages,
+                    prePromptMessageCount: args.prePromptMessageCount,
                     isHeartbeat: args.isHeartbeat,
+                    cursor,
                 });
+                // Reconcile manifest with daemon-confirmed cursor.
+                // The daemon returns a cursor even when it ingests zero messages
+                // (e.g. gap detected, all messages deduped). Trust its
+                // lastProcessedIndex over our optimistic startIndex math.
+                const daemonCursor = extractCursorFromResult(result);
+                if (daemonCursor) {
+                    if (!daemonCursor.manifestTailHash) {
+                        // Daemon detected a gap: its DB is behind our manifest.
+                        // It did NOT ingest our messages. Reset the manifest so the
+                        // next turn does a full re-sync.
+                        logger.warn?.(`[LibraVDB] Daemon reported cursor gap for session ${sessionId}. ` +
+                            `Resetting manifest for full re-sync next turn.`);
+                        manifestStore.save(manifestStore.createEmpty(sessionId));
+                    }
+                    else if (ingestMessages.length > 0) {
+                        // Normal path: reconcile to what the daemon actually confirmed.
+                        const confirmedIndex = daemonCursor.lastProcessedIndex;
+                        const ackCount = Math.max(0, confirmedIndex - startIndex + 1);
+                        if (ackCount > 0) {
+                            const ackedMessages = ingestMessages.slice(0, ackCount);
+                            const updatedManifest = manifestStore.appendACKedMessages(manifest, ackedMessages, startIndex);
+                            manifestStore.save(updatedManifest);
+                        }
+                    }
+                }
+                else if (ingestMessages.length > 0) {
+                    // Legacy daemon (no cursor in response): optimistic ACK.
+                    const updatedManifest = manifestStore.appendACKedMessages(manifest, ingestMessages, startIndex);
+                    manifestStore.save(updatedManifest);
+                }
                 await performAfterTurnPredictiveCompaction({
                     sessionId,
                     messages,

package/dist/identity.d.ts

@@ -1,4 +1,4 @@
-import type { LoggerLike } from "./types.js";
+import type { LoggerLike, PluginConfig } from "./types.js";
 export type IdentitySource = "config" | "file" | "auto" | "session-key" | "default";
 export type ResolvedIdentity = {
     userId: string;
@@ -13,3 +13,12 @@ export declare function resolveIdentity(params: {
      *  read-only commands (e.g. status --deep) that should not mutate disk. */
     noAutoPersist?: boolean;
 }): ResolvedIdentity;
+/**
+ * Resolves a stable tenant key for multi-agent DB routing.
+ *
+ * Priority chain:
+ *   1. cfg.tenantId (explicit config, highest priority)
+ *   2. LIBRAVDB_AGENT_ID env var (container/CI override)
+ *   3. Fall back to resolved userId (existing identity system)
+ */
+export declare function resolveTenantKey(cfg: PluginConfig): string;

package/dist/identity.js

@@ -118,3 +118,23 @@ export function resolveIdentity(params) {
     }
     return { userId: autoId, source: "auto" };
 }
+/**
+ * Resolves a stable tenant key for multi-agent DB routing.
+ *
+ * Priority chain:
+ *   1. cfg.tenantId (explicit config, highest priority)
+ *   2. LIBRAVDB_AGENT_ID env var (container/CI override)
+ *   3. Fall back to resolved userId (existing identity system)
+ */
+export function resolveTenantKey(cfg) {
+    const explicit = cfg.tenantId?.trim();
+    if (explicit)
+        return explicit;
+    const envId = process.env.LIBRAVDB_AGENT_ID?.trim();
+    if (envId)
+        return envId;
+    return resolveIdentity({
+        configUserId: cfg.userId,
+        identityPath: cfg.identityPath,
+    }).userId;
+}