memory-braid 0.4.7 → 0.6.0

package/README.md

@@ -9,6 +9,65 @@ Memory Braid is an OpenClaw `kind: "memory"` plugin that augments local memory s
 - Capture pipeline modes: `local`, `hybrid`, `ml`.
 - Optional entity extraction: local multilingual NER or OpenAI NER with canonical `entity://...` URIs in memory metadata.
 - Structured debug logs for troubleshooting and tuning.
+- Debug-only LLM usage observability: per-turn cache usage, rolling windows, and rising/stable/improving trend logs.
+
+## Hardening update
+
+This release hardens capture and remediation for historical installs.
+
+- Bug class: historical prompt or transcript content could be captured as Mem0 memories and later re-injected.
+- Impact: inflated prompt size, noisier recall, and potentially higher Anthropic cache-write costs.
+- Fix: new captures are assembled from the trusted current turn instead of mining the full `agent_end` transcript.
+- Metadata: new captured memories now include additive provenance fields such as `captureOrigin`, `captureMessageHash`, `captureTurnHash`, `capturePath`, and `pluginCaptureVersion`.
+- Historical installs: no startup mutation is performed automatically. Operators should audit first, then explicitly quarantine or delete suspicious captured memories.
+
+## Remediation commands
+
+Memory Braid now exposes read-only audit and explicit remediation commands:
+
+```bash
+/memorybraid audit
+/memorybraid remediate audit
+/memorybraid remediate quarantine
+/memorybraid remediate quarantine --apply
+/memorybraid remediate delete --apply
+/memorybraid remediate purge-all-captured --apply
+```
+
+Notes:
+
+- Dry-run is the default for remediation commands. Nothing mutates until you pass `--apply`.
+- `audit` reports counts by `sourceType`, `captureOrigin`, and `pluginCaptureVersion`, plus suspicious legacy samples.
+- `quarantine --apply` excludes suspicious captured memories from future Mem0 injection. It records quarantine state locally and also tags Mem0 metadata where supported.
+- `delete --apply` deletes suspicious captured memories only.
+- `purge-all-captured --apply` deletes all plugin-captured Mem0 records for the current workspace scope without touching local markdown memory.
+- Optional flags:
+  - `--limit N` controls how many Mem0 records are fetched during audit/remediation.
+  - `--sample N` controls how many suspicious samples are shown in the audit report.
+
+## Debug cost observability
+
+When `debug.enabled` is `true`, Memory Braid also emits debug-only LLM usage observability logs from the `llm_output` hook:
+
+- `memory_braid.cost.turn`: per-turn input/output/cache tokens, cache ratios, and a best-effort estimated USD cost when the provider/model has a known pricing profile.
+- `memory_braid.cost.window`: rolling 5-turn and 20-turn averages plus `rising|stable|improving` trend labels for prompt size, cache-write rate, cache-hit rate, and estimated cost.
+- `memory_braid.cost.alert`: emitted only when recent cache writes, prompt size, or estimated cost rise materially above the previous short window.
+
+Important:
+
+- `estimatedCostUsd` is intentionally labeled as an estimate.
+- Unknown models still log token and cache trends, but the cost basis becomes `token_only`.
+
+## Self-hosted reset option
+
+If you are self-hosting and prefer a full reset instead of selective remediation, you can clear Memory Braid's OSS Mem0 state and restart OpenClaw:
+
+```bash
+rm -rf ~/.openclaw/memory-braid
+openclaw gateway restart
+```
+
+This is intentionally not done by the plugin itself. It is an operator choice.
 
 ## Breaking changes in 0.4.0
 
@@ -429,7 +488,17 @@ Use this preset when:
     "memory-braid": {
       "recall": {
         "maxResults": 8,
-        "injectTopK": 5,
+        "injectTopK": 4,
+        "user": {
+          "enabled": true,
+          "injectTopK": 4
+        },
+        "agent": {
+          "enabled": true,
+          "injectTopK": 2,
+          "minScore": 0.78,
+          "onlyPlanning": true
+        },
         "merge": {
           "rrfK": 60,
           "localWeight": 1,
@@ -441,6 +510,16 @@ Use this preset when:
         "mode": "hybrid",
         "includeAssistant": false,
         "maxItemsPerRun": 6,
+        "assistant": {
+          "enabled": true,
+          "autoCapture": false,
+          "explicitTool": true,
+          "maxItemsPerRun": 2,
+          "minUtilityScore": 0.8,
+          "minNoveltyScore": 0.85,
+          "maxWritesPerSessionWindow": 3,
+          "cooldownMinutes": 5
+        },
         "ml": {
           "provider": "openai",
           "model": "gpt-4o-mini",
@@ -489,8 +568,20 @@ Capture defaults are:
 
 - `capture.enabled`: `true`
 - `capture.mode`: `"local"`
-- `capture.includeAssistant`: `false` (default user-only capture)
+- `capture.includeAssistant`: `false` (legacy alias for `capture.assistant.autoCapture`)
 - `capture.maxItemsPerRun`: `6`
+- `capture.assistant.enabled`: `true`
+- `capture.assistant.autoCapture`: `false`
+- `capture.assistant.explicitTool`: `true`
+- `capture.assistant.maxItemsPerRun`: `2`
+- `capture.assistant.minUtilityScore`: `0.8`
+- `capture.assistant.minNoveltyScore`: `0.85`
+- `capture.assistant.maxWritesPerSessionWindow`: `3`
+- `capture.assistant.cooldownMinutes`: `5`
+- `recall.user.injectTopK`: `5` (legacy `recall.injectTopK` still works)
+- `recall.agent.injectTopK`: `2`
+- `recall.agent.minScore`: `0.78`
+- `recall.agent.onlyPlanning`: `true`
 - `capture.ml.provider`: unset
 - `capture.ml.model`: unset
 - `capture.ml.timeoutMs`: `2500`
@@ -505,14 +596,39 @@ Important behavior:
 - `capture.mode = "local"`: heuristic-only extraction.
 - `capture.mode = "hybrid"`: heuristic extraction + ML enrichment when ML config is set.
 - `capture.mode = "ml"`: ML-first extraction; falls back to heuristic if ML config/call is unavailable.
-- `capture.includeAssistant = false` (default): only `user` messages are considered for capture.
-- `capture.includeAssistant = true`: both `user` and `assistant` messages are considered for capture.
+- New memories are persisted by `workspace + agent`, not by session. `sessionKey` is kept only as metadata and for assistant-learning cooldown/window logic.
+- Recall still performs a legacy dual-read fallback for older session-scoped Mem0 records, without rewriting them.
+- `capture.includeAssistant = false` (default): assistant auto-capture is off.
+- `capture.includeAssistant = true` or `capture.assistant.autoCapture = true`: assistant messages are eligible for strict agent-learning auto-capture.
+- `capture.assistant.explicitTool = true`: exposes the `remember_learning` tool.
+- `recall.user.*` controls injected user memories.
+- `recall.agent.*` controls injected agent learnings.
 - ML calls run only when both `capture.ml.provider` and `capture.ml.model` are set.
 - `timeDecay.enabled = true`: applies temporal decay to Mem0 results using Memory Core's `agents.*.memorySearch.query.hybrid.temporalDecay` settings.
 - If Memory Core temporal decay is disabled, Mem0 decay is skipped even when `timeDecay.enabled = true`.
 - `lifecycle.enabled = true`: tracks captured Mem0 IDs, applies TTL cleanup, and exposes `/memorybraid cleanup`.
 - `lifecycle.reinforceOnRecall = true`: successful recalls refresh lifecycle timestamps, extending TTL survival for frequently used memories.
 
+## Agent learnings
+
+Memory Braid v2 adds explicit and implicit agent learnings.
+
+- `remember_learning` stores compact reusable heuristics, lessons, and strategies for future runs.
+- Use it for operational guidance that helps the agent avoid repeated mistakes or reduce tool cost/noise.
+- Do not use it for long summaries, transient details, or raw reasoning.
+- Assistant auto-capture is still available, but it is stricter than user-memory capture and only persists compact learnings that pass utility, novelty, and cooldown checks.
+
+Recall is now split into two dynamic blocks:
+
+- `<user-memories>`: user facts, preferences, decisions, and tasks.
+- `<agent-learnings>`: reusable agent heuristics, lessons, and strategies.
+
+Cache safety:
+
+- Tool awareness for `remember_learning` is injected through a stable `systemPrompt`.
+- Retrieved memories stay in dynamic `prependContext`, not in the stable prompt body.
+- Agent learnings use low `top-k`, high relevance thresholds, and deterministic formatting to avoid unnecessary prompt churn.
+
 ## Entity extraction defaults
 
 Entity extraction defaults are:

package/openclaw.plugin.json

@@ -30,6 +30,24 @@
         "properties": {
           "maxResults": { "type": "integer", "minimum": 1, "maximum": 50, "default": 8 },
           "injectTopK": { "type": "integer", "minimum": 1, "maximum": 20, "default": 5 },
+          "user": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "enabled": { "type": "boolean", "default": true },
+              "injectTopK": { "type": "integer", "minimum": 1, "maximum": 20, "default": 5 }
+            }
+          },
+          "agent": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "enabled": { "type": "boolean", "default": true },
+              "injectTopK": { "type": "integer", "minimum": 1, "maximum": 20, "default": 2 },
+              "minScore": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.78 },
+              "onlyPlanning": { "type": "boolean", "default": true }
+            }
+          },
           "merge": {
             "type": "object",
             "additionalProperties": false,
@@ -54,6 +72,20 @@
           },
           "includeAssistant": { "type": "boolean", "default": false },
           "maxItemsPerRun": { "type": "integer", "minimum": 1, "maximum": 50, "default": 6 },
+          "assistant": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+              "enabled": { "type": "boolean", "default": true },
+              "autoCapture": { "type": "boolean", "default": false },
+              "explicitTool": { "type": "boolean", "default": true },
+              "maxItemsPerRun": { "type": "integer", "minimum": 1, "maximum": 10, "default": 2 },
+              "minUtilityScore": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.8 },
+              "minNoveltyScore": { "type": "number", "minimum": 0, "maximum": 1, "default": 0.85 },
+              "maxWritesPerSessionWindow": { "type": "integer", "minimum": 1, "maximum": 20, "default": 3 },
+              "cooldownMinutes": { "type": "integer", "minimum": 0, "maximum": 240, "default": 5 }
+            }
+          },
           "ml": {
             "type": "object",
             "additionalProperties": false,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-braid",
-  "version": "0.4.7",
+  "version": "0.6.0",
   "description": "OpenClaw memory plugin that augments local memory with Mem0 capture and recall.",
   "type": "module",
   "main": "./src/index.ts",

package/src/capture.ts

@@ -0,0 +1,315 @@
+import { normalizeForHash, normalizeWhitespace, sha256 } from "./chunking.js";
+import type {
+  AssembledCaptureInput,
+  CaptureInputMessage,
+  PendingInboundTurn,
+} from "./types.js";
+
+type NormalizedHookMessage = {
+  role: string;
+  text: string;
+};
+
+function asRecord(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return {};
+  }
+  return value as Record<string, unknown>;
+}
+
+export function extractHookMessageText(content: unknown): string {
+  if (typeof content === "string") {
+    return normalizeWhitespace(content);
+  }
+  if (!Array.isArray(content)) {
+    return "";
+  }
+
+  const parts: string[] = [];
+  for (const block of content) {
+    if (!block || typeof block !== "object") {
+      continue;
+    }
+    const item = block as { type?: unknown; text?: unknown };
+    if (item.type === "text" && typeof item.text === "string") {
+      const normalized = normalizeWhitespace(item.text);
+      if (normalized) {
+        parts.push(normalized);
+      }
+    }
+  }
+  return parts.join(" ");
+}
+
+export function normalizeHookMessages(messages: unknown[]): NormalizedHookMessage[] {
+  const out: NormalizedHookMessage[] = [];
+  for (const entry of messages) {
+    if (!entry || typeof entry !== "object") {
+      continue;
+    }
+
+    const direct = entry as { role?: unknown; content?: unknown };
+    if (typeof direct.role === "string") {
+      const text = extractHookMessageText(direct.content);
+      if (text) {
+        out.push({ role: direct.role, text });
+      }
+      continue;
+    }
+
+    const wrapped = entry as { message?: { role?: unknown; content?: unknown } };
+    if (wrapped.message && typeof wrapped.message.role === "string") {
+      const text = extractHookMessageText(wrapped.message.content);
+      if (text) {
+        out.push({ role: wrapped.message.role, text });
+      }
+    }
+  }
+  return out;
+}
+
+function normalizeProvenanceKind(value: unknown): string | undefined {
+  const record = asRecord(value);
+  const kind = typeof record.kind === "string" ? record.kind.trim().toLowerCase() : "";
+  return kind || undefined;
+}
+
+export function getPendingInboundTurn(message: unknown): PendingInboundTurn | undefined {
+  const record = asRecord(message);
+  const role = typeof record.role === "string" ? record.role.trim().toLowerCase() : "";
+  const provenanceKind = normalizeProvenanceKind(record.provenance);
+  if (role !== "user" || provenanceKind !== "external_user") {
+    return undefined;
+  }
+
+  const text = extractHookMessageText(record.content);
+  if (!text) {
+    return undefined;
+  }
+
+  return {
+    text,
+    messageHash: sha256(normalizeForHash(text)),
+    receivedAt: Date.now(),
+  };
+}
+
+function buildCaptureInputMessage(
+  role: "user" | "assistant",
+  origin: "external_user" | "assistant_derived",
+  text: string,
+): CaptureInputMessage {
+  return {
+    role,
+    origin,
+    text,
+    messageHash: sha256(normalizeForHash(text)),
+  };
+}
+
+export function assembleCaptureInput(params: {
+  messages: unknown[];
+  includeAssistant: boolean;
+  pendingInboundTurn?: PendingInboundTurn;
+}): AssembledCaptureInput | undefined {
+  const normalized = normalizeHookMessages(params.messages);
+  const lastUserIndex = (() => {
+    for (let i = normalized.length - 1; i >= 0; i -= 1) {
+      if (normalized[i]?.role === "user") {
+        return i;
+      }
+    }
+    return -1;
+  })();
+
+  const userText = params.pendingInboundTurn?.text ?? normalized[lastUserIndex]?.text ?? "";
+  if (!userText) {
+    return undefined;
+  }
+
+  const assembled: CaptureInputMessage[] = [
+    buildCaptureInputMessage("user", "external_user", userText),
+  ];
+
+  if (params.includeAssistant) {
+    const assistantStart = lastUserIndex >= 0 ? lastUserIndex + 1 : normalized.length;
+    for (let i = assistantStart; i < normalized.length; i += 1) {
+      const message = normalized[i];
+      if (!message || message.role !== "assistant" || !message.text) {
+        continue;
+      }
+      assembled.push(buildCaptureInputMessage("assistant", "assistant_derived", message.text));
+    }
+  }
+
+  const hashInput = assembled.map((message) => message.messageHash).join("|");
+  return {
+    messages: assembled,
+    capturePath: params.pendingInboundTurn ? "before_message_write" : "agent_end_last_turn",
+    turnHash: sha256(hashInput),
+    fallbackUsed: !params.pendingInboundTurn,
+  };
+}
+
+function tokenize(text: string): Set<string> {
+  const tokens = text.match(/[\p{L}\p{N}]+/gu) ?? [];
+  const out = new Set<string>();
+  for (const token of tokens) {
+    const normalized = token
+      .toLowerCase()
+      .normalize("NFKD")
+      .replace(/\p{M}+/gu, "");
+    if (normalized.length >= 3) {
+      out.add(normalized);
+    }
+  }
+  return out;
+}
+
+function overlapRatio(left: Set<string>, right: Set<string>): number {
+  if (left.size === 0 || right.size === 0) {
+    return 0;
+  }
+  let shared = 0;
+  for (const token of left) {
+    if (right.has(token)) {
+      shared += 1;
+    }
+  }
+  return shared / Math.max(left.size, right.size);
+}
+
+export function matchCandidateToCaptureInput(
+  candidateText: string,
+  messages: CaptureInputMessage[],
+): CaptureInputMessage | undefined {
+  const candidateHash = sha256(normalizeForHash(candidateText));
+  for (const message of messages) {
+    if (message.messageHash === candidateHash) {
+      return message;
+    }
+  }
+
+  const candidateTokens = tokenize(candidateText);
+  let bestMatch: CaptureInputMessage | undefined;
+  let bestScore = 0;
+
+  for (const message of messages) {
+    const score = overlapRatio(candidateTokens, tokenize(message.text));
+    if (score > bestScore) {
+      bestScore = score;
+      bestMatch = message;
+    }
+  }
+
+  return bestScore >= 0.24 ? bestMatch : undefined;
+}
+
+const ROLE_PREFIX_LINE = /^(?:assistant|system|developer|tool|user|human|bot|ai|agent)\s*:/i;
+const INLINE_ROLE_LABEL = /\b(?:assistant|system|developer|tool|user)\s*:/gi;
+const STRUCTURED_METADATA_KEY =
+  /^\s*["']?(?:message_id|reply_to_id|sender_id|sender|timestamp|thread|conversation|channel|metadata)\b/i;
+
+export function isLikelyTranscriptLikeText(text: string): boolean {
+  const normalized = normalizeWhitespace(text);
+  if (!normalized) {
+    return false;
+  }
+
+  const lines = normalized
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  if (lines.length === 0) {
+    return false;
+  }
+
+  const rolePrefixedLines = lines.filter((line) => ROLE_PREFIX_LINE.test(line)).length;
+  const inlineRoleLabels = normalized.match(INLINE_ROLE_LABEL)?.length ?? 0;
+  const fencedBlocks = normalized.match(/```/g)?.length ?? 0;
+  const metadataLines = lines.filter((line) => STRUCTURED_METADATA_KEY.test(line)).length;
+
+  if (rolePrefixedLines >= 2) {
+    return true;
+  }
+  if (inlineRoleLabels >= 3) {
+    return true;
+  }
+  if (fencedBlocks >= 2 && metadataLines >= 2) {
+    return true;
+  }
+  return metadataLines >= 4 && lines.length >= 6;
+}
+
+export function isOversizedAtomicMemory(text: string): boolean {
+  const normalized = normalizeWhitespace(text);
+  if (!normalized) {
+    return false;
+  }
+  const lines = normalized.split(/\r?\n/).filter((line) => line.trim().length > 0);
+  return normalized.length > 1600 || lines.length > 18;
+}
+
+const RECAP_PREFIXES = [
+  /^the user\b/i,
+  /^user\b/i,
+  /^usuario\b/i,
+  /^in this (?:turn|conversation)\b/i,
+  /^(?:we|i) (?:discussed|talked about|went over|covered)\b/i,
+  /^(?:summary|recap)\b/i,
+];
+
+const TEMPORAL_REFERENCE_PATTERN =
+  /\b(?:today|tomorrow|yesterday|this turn|this session|earlier in this session|just now|in this chat)\b/i;
+
+export function isLikelyTurnRecap(text: string): boolean {
+  const normalized = normalizeWhitespace(text);
+  if (!normalized) {
+    return false;
+  }
+  if (normalized.length > 260 && /\b(?:asked|wanted|needed|said|requested)\b/i.test(normalized)) {
+    return true;
+  }
+  return RECAP_PREFIXES.some((pattern) => pattern.test(normalized));
+}
+
+function splitIntoSentences(text: string): string[] {
+  return text
+    .split(/(?<=[.!?])\s+/)
+    .map((sentence) => normalizeWhitespace(sentence))
+    .filter(Boolean);
+}
+
+function looksReusableLearning(text: string): boolean {
+  if (text.length < 24 || text.length > 220) {
+    return false;
+  }
+  if (TEMPORAL_REFERENCE_PATTERN.test(text)) {
+    return false;
+  }
+  if (isLikelyTranscriptLikeText(text) || isLikelyTurnRecap(text)) {
+    return false;
+  }
+  return /\b(?:prefer|avoid|use|keep|store|remember|dedupe|inject|search|persist|reject|limit|filter|only|always|never|when)\b/i.test(
+    text,
+  );
+}
+
+export function compactAgentLearning(text: string): string | undefined {
+  const normalized = normalizeWhitespace(text);
+  if (!normalized || isOversizedAtomicMemory(normalized) || isLikelyTranscriptLikeText(normalized)) {
+    return undefined;
+  }
+  if (looksReusableLearning(normalized)) {
+    return normalized;
+  }
+
+  const sentences = splitIntoSentences(normalized);
+  for (const sentence of sentences) {
+    if (looksReusableLearning(sentence)) {
+      return sentence;
+    }
+  }
+
+  return undefined;
+}