memory-braid 0.4.7 → 0.5.0

package/README.md

@@ -9,6 +9,65 @@ Memory Braid is an OpenClaw `kind: "memory"` plugin that augments local memory s
 - Capture pipeline modes: `local`, `hybrid`, `ml`.
 - Optional entity extraction: local multilingual NER or OpenAI NER with canonical `entity://...` URIs in memory metadata.
 - Structured debug logs for troubleshooting and tuning.
+- Debug-only LLM usage observability: per-turn cache usage, rolling windows, and rising/stable/improving trend logs.
+
+## Hardening update
+
+This release hardens capture and remediation for historical installs.
+
+- Bug class: historical prompt or transcript content could be captured as Mem0 memories and later re-injected.
+- Impact: inflated prompt size, noisier recall, and potentially higher Anthropic cache-write costs.
+- Fix: new captures are assembled from the trusted current turn instead of mining the full `agent_end` transcript.
+- Metadata: new captured memories now include additive provenance fields such as `captureOrigin`, `captureMessageHash`, `captureTurnHash`, `capturePath`, and `pluginCaptureVersion`.
+- Historical installs: no startup mutation is performed automatically. Operators should audit first, then explicitly quarantine or delete suspicious captured memories.
+
+## Remediation commands
+
+Memory Braid now exposes read-only audit and explicit remediation commands:
+
+```bash
+/memorybraid audit
+/memorybraid remediate audit
+/memorybraid remediate quarantine
+/memorybraid remediate quarantine --apply
+/memorybraid remediate delete --apply
+/memorybraid remediate purge-all-captured --apply
+```
+
+Notes:
+
+- Dry-run is the default for remediation commands. Nothing mutates until you pass `--apply`.
+- `audit` reports counts by `sourceType`, `captureOrigin`, and `pluginCaptureVersion`, plus suspicious legacy samples.
+- `quarantine --apply` excludes suspicious captured memories from future Mem0 injection. It records quarantine state locally and also tags Mem0 metadata where supported.
+- `delete --apply` deletes suspicious captured memories only.
+- `purge-all-captured --apply` deletes all plugin-captured Mem0 records for the current workspace scope without touching local markdown memory.
+- Optional flags:
+  - `--limit N` controls how many Mem0 records are fetched during audit/remediation.
+  - `--sample N` controls how many suspicious samples are shown in the audit report.
+
+## Debug cost observability
+
+When `debug.enabled` is `true`, Memory Braid also emits debug-only LLM usage observability logs from the `llm_output` hook:
+
+- `memory_braid.cost.turn`: per-turn input/output/cache tokens, cache ratios, and a best-effort estimated USD cost when the provider/model has a known pricing profile.
+- `memory_braid.cost.window`: rolling 5-turn and 20-turn averages plus `rising|stable|improving` trend labels for prompt size, cache-write rate, cache-hit rate, and estimated cost.
+- `memory_braid.cost.alert`: emitted only when recent cache writes, prompt size, or estimated cost rise materially above the previous short window.
+
+Important:
+
+- `estimatedCostUsd` is intentionally labeled as an estimate.
+- Unknown models still log token and cache trends, but the cost basis becomes `token_only`.
+
+## Self-hosted reset option
+
+If you are self-hosting and prefer a full reset instead of selective remediation, you can clear Memory Braid's OSS Mem0 state and restart OpenClaw:
+
+```bash
+rm -rf ~/.openclaw/memory-braid
+openclaw gateway restart
+```
+
+This is intentionally not done by the plugin itself. It is an operator choice.
 
 ## Breaking changes in 0.4.0
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-braid",
-  "version": "0.4.7",
+  "version": "0.5.0",
   "description": "OpenClaw memory plugin that augments local memory with Mem0 capture and recall.",
   "type": "module",
   "main": "./src/index.ts",

package/src/capture.ts

@@ -0,0 +1,251 @@
+import { normalizeForHash, normalizeWhitespace, sha256 } from "./chunking.js";
+import type {
+  AssembledCaptureInput,
+  CaptureInputMessage,
+  PendingInboundTurn,
+} from "./types.js";
+
+type NormalizedHookMessage = {
+  role: string;
+  text: string;
+};
+
+function asRecord(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return {};
+  }
+  return value as Record<string, unknown>;
+}
+
+export function extractHookMessageText(content: unknown): string {
+  if (typeof content === "string") {
+    return normalizeWhitespace(content);
+  }
+  if (!Array.isArray(content)) {
+    return "";
+  }
+
+  const parts: string[] = [];
+  for (const block of content) {
+    if (!block || typeof block !== "object") {
+      continue;
+    }
+    const item = block as { type?: unknown; text?: unknown };
+    if (item.type === "text" && typeof item.text === "string") {
+      const normalized = normalizeWhitespace(item.text);
+      if (normalized) {
+        parts.push(normalized);
+      }
+    }
+  }
+  return parts.join(" ");
+}
+
+export function normalizeHookMessages(messages: unknown[]): NormalizedHookMessage[] {
+  const out: NormalizedHookMessage[] = [];
+  for (const entry of messages) {
+    if (!entry || typeof entry !== "object") {
+      continue;
+    }
+
+    const direct = entry as { role?: unknown; content?: unknown };
+    if (typeof direct.role === "string") {
+      const text = extractHookMessageText(direct.content);
+      if (text) {
+        out.push({ role: direct.role, text });
+      }
+      continue;
+    }
+
+    const wrapped = entry as { message?: { role?: unknown; content?: unknown } };
+    if (wrapped.message && typeof wrapped.message.role === "string") {
+      const text = extractHookMessageText(wrapped.message.content);
+      if (text) {
+        out.push({ role: wrapped.message.role, text });
+      }
+    }
+  }
+  return out;
+}
+
+function normalizeProvenanceKind(value: unknown): string | undefined {
+  const record = asRecord(value);
+  const kind = typeof record.kind === "string" ? record.kind.trim().toLowerCase() : "";
+  return kind || undefined;
+}
+
+export function getPendingInboundTurn(message: unknown): PendingInboundTurn | undefined {
+  const record = asRecord(message);
+  const role = typeof record.role === "string" ? record.role.trim().toLowerCase() : "";
+  const provenanceKind = normalizeProvenanceKind(record.provenance);
+  if (role !== "user" || provenanceKind !== "external_user") {
+    return undefined;
+  }
+
+  const text = extractHookMessageText(record.content);
+  if (!text) {
+    return undefined;
+  }
+
+  return {
+    text,
+    messageHash: sha256(normalizeForHash(text)),
+    receivedAt: Date.now(),
+  };
+}
+
+function buildCaptureInputMessage(
+  role: "user" | "assistant",
+  origin: "external_user" | "assistant_derived",
+  text: string,
+): CaptureInputMessage {
+  return {
+    role,
+    origin,
+    text,
+    messageHash: sha256(normalizeForHash(text)),
+  };
+}
+
+export function assembleCaptureInput(params: {
+  messages: unknown[];
+  includeAssistant: boolean;
+  pendingInboundTurn?: PendingInboundTurn;
+}): AssembledCaptureInput | undefined {
+  const normalized = normalizeHookMessages(params.messages);
+  const lastUserIndex = (() => {
+    for (let i = normalized.length - 1; i >= 0; i -= 1) {
+      if (normalized[i]?.role === "user") {
+        return i;
+      }
+    }
+    return -1;
+  })();
+
+  const userText = params.pendingInboundTurn?.text ?? normalized[lastUserIndex]?.text ?? "";
+  if (!userText) {
+    return undefined;
+  }
+
+  const assembled: CaptureInputMessage[] = [
+    buildCaptureInputMessage("user", "external_user", userText),
+  ];
+
+  if (params.includeAssistant) {
+    const assistantStart = lastUserIndex >= 0 ? lastUserIndex + 1 : normalized.length;
+    for (let i = assistantStart; i < normalized.length; i += 1) {
+      const message = normalized[i];
+      if (!message || message.role !== "assistant" || !message.text) {
+        continue;
+      }
+      assembled.push(buildCaptureInputMessage("assistant", "assistant_derived", message.text));
+    }
+  }
+
+  const hashInput = assembled.map((message) => message.messageHash).join("|");
+  return {
+    messages: assembled,
+    capturePath: params.pendingInboundTurn ? "before_message_write" : "agent_end_last_turn",
+    turnHash: sha256(hashInput),
+    fallbackUsed: !params.pendingInboundTurn,
+  };
+}
+
+function tokenize(text: string): Set<string> {
+  const tokens = text.match(/[\p{L}\p{N}]+/gu) ?? [];
+  const out = new Set<string>();
+  for (const token of tokens) {
+    const normalized = token
+      .toLowerCase()
+      .normalize("NFKD")
+      .replace(/\p{M}+/gu, "");
+    if (normalized.length >= 3) {
+      out.add(normalized);
+    }
+  }
+  return out;
+}
+
+function overlapRatio(left: Set<string>, right: Set<string>): number {
+  if (left.size === 0 || right.size === 0) {
+    return 0;
+  }
+  let shared = 0;
+  for (const token of left) {
+    if (right.has(token)) {
+      shared += 1;
+    }
+  }
+  return shared / Math.max(left.size, right.size);
+}
+
+export function matchCandidateToCaptureInput(
+  candidateText: string,
+  messages: CaptureInputMessage[],
+): CaptureInputMessage | undefined {
+  const candidateHash = sha256(normalizeForHash(candidateText));
+  for (const message of messages) {
+    if (message.messageHash === candidateHash) {
+      return message;
+    }
+  }
+
+  const candidateTokens = tokenize(candidateText);
+  let bestMatch: CaptureInputMessage | undefined;
+  let bestScore = 0;
+
+  for (const message of messages) {
+    const score = overlapRatio(candidateTokens, tokenize(message.text));
+    if (score > bestScore) {
+      bestScore = score;
+      bestMatch = message;
+    }
+  }
+
+  return bestScore >= 0.24 ? bestMatch : undefined;
+}
+
+const ROLE_PREFIX_LINE = /^(?:assistant|system|developer|tool|user|human|bot|ai|agent)\s*:/i;
+const INLINE_ROLE_LABEL = /\b(?:assistant|system|developer|tool|user)\s*:/gi;
+const STRUCTURED_METADATA_KEY =
+  /^\s*["']?(?:message_id|reply_to_id|sender_id|sender|timestamp|thread|conversation|channel|metadata)\b/i;
+
+export function isLikelyTranscriptLikeText(text: string): boolean {
+  const normalized = normalizeWhitespace(text);
+  if (!normalized) {
+    return false;
+  }
+
+  const lines = normalized
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  if (lines.length === 0) {
+    return false;
+  }
+
+  const rolePrefixedLines = lines.filter((line) => ROLE_PREFIX_LINE.test(line)).length;
+  const inlineRoleLabels = normalized.match(INLINE_ROLE_LABEL)?.length ?? 0;
+  const fencedBlocks = normalized.match(/```/g)?.length ?? 0;
+  const metadataLines = lines.filter((line) => STRUCTURED_METADATA_KEY.test(line)).length;
+
+  if (rolePrefixedLines >= 2) {
+    return true;
+  }
+  if (inlineRoleLabels >= 3) {
+    return true;
+  }
+  if (fencedBlocks >= 2 && metadataLines >= 2) {
+    return true;
+  }
+  return metadataLines >= 4 && lines.length >= 6;
+}
+
+export function isOversizedAtomicMemory(text: string): boolean {
+  const normalized = normalizeWhitespace(text);
+  if (!normalized) {
+    return false;
+  }
+  const lines = normalized.split(/\r?\n/).filter((line) => line.trim().length > 0);
+  return normalized.length > 1600 || lines.length > 18;
+}

package/src/extract.ts

@@ -1,5 +1,6 @@
 import { normalizeForHash, normalizeWhitespace, sha256 } from "./chunking.js";
 import type { MemoryBraidConfig } from "./config.js";
+import { isLikelyTranscriptLikeText, isOversizedAtomicMemory } from "./capture.js";
 import { MemoryBraidLogger } from "./logger.js";
 import type { ExtractedCandidate } from "./types.js";
 
@@ -18,6 +19,9 @@ const FEED_TAG_PATTERN = /\[(?:n8n|rss|alert|news|cron|slack|discord|telegram|em
 const ROLE_LABEL_PATTERN = /\b(?:assistant|system|tool|developer)\s*:/gi;
 
 function isLikelyFeedOrImportedText(text: string): boolean {
+  if (isLikelyTranscriptLikeText(text) || isOversizedAtomicMemory(text)) {
+    return true;
+  }
   if (FEED_TAG_PATTERN.test(text)) {
     return true;
   }
@@ -398,6 +402,9 @@ function applyMlExtractionResult(
     if (!text || text.length < 20 || text.length > 3000) {
       continue;
     }
+    if (isLikelyFeedOrImportedText(text)) {
+      continue;
+    }
     const key = sha256(normalizeForHash(text));
     if (seen.has(key)) {
       continue;