memory-braid 0.4.3 → 0.4.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-braid",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "description": "OpenClaw memory plugin that augments local memory with Mem0 capture and recall.",
   "type": "module",
   "main": "./src/index.ts",

package/src/entities.ts

@@ -89,6 +89,20 @@ type NormalizedEntityToken = {
   end?: number;
 };
 
+const ENTITY_CONNECTOR_WORDS = new Set([
+  "and",
+  "da",
+  "de",
+  "del",
+  "la",
+  "las",
+  "los",
+  "of",
+  "the",
+  "y",
+]);
+const ENTITY_MAX_MERGED_WORDS = 3;
+
 function asFiniteNumber(value: unknown): number | undefined {
   if (typeof value !== "number" || !Number.isFinite(value)) {
     return undefined;
@@ -96,6 +110,21 @@ function asFiniteNumber(value: unknown): number | undefined {
   return value;
 }
 
+function splitEntityWords(text: string): string[] {
+  return text.match(/[\p{L}\p{N}]+/gu) ?? [];
+}
+
+function isLikelyNoisyShortWord(word: string): boolean {
+  const normalized = word.toLowerCase();
+  if (normalized.length >= 3) {
+    return false;
+  }
+  if (ENTITY_CONNECTOR_WORDS.has(normalized)) {
+    return false;
+  }
+  return !/^[A-Z]\.?$/.test(word);
+}
+
 function joinEntityText(left: NormalizedEntityToken, right: NormalizedEntityToken): string {
   const leftEnd = left.end;
   const rightStart = right.start;
@@ -108,11 +137,32 @@ function joinEntityText(left: NormalizedEntityToken, right: NormalizedEntityToke
   return `${left.text} ${right.text}`;
 }
 
-function shouldMergeEntityTokens(left: NormalizedEntityToken, right: NormalizedEntityToken): boolean {
+function shouldMergeEntityTokens(
+  left: NormalizedEntityToken,
+  right: NormalizedEntityToken,
+  sourceText?: string,
+): boolean {
   if (left.type !== right.type || !left.text || !right.text) {
     return false;
   }
 
+  const leftWords = splitEntityWords(left.text);
+  const rightWords = splitEntityWords(right.text);
+  if (leftWords.length === 0 || rightWords.length === 0) {
+    return false;
+  }
+  if (leftWords.length + rightWords.length > ENTITY_MAX_MERGED_WORDS) {
+    return false;
+  }
+  const leftLastWord = leftWords[leftWords.length - 1];
+  const rightFirstWord = rightWords[0];
+  if (!leftLastWord || !rightFirstWord) {
+    return false;
+  }
+  if (isLikelyNoisyShortWord(leftLastWord) || isLikelyNoisyShortWord(rightFirstWord)) {
+    return false;
+  }
+
   const leftEnd = left.end;
   const rightStart = right.start;
   if (typeof leftEnd === "number" && typeof rightStart === "number") {
@@ -120,7 +170,16 @@ function shouldMergeEntityTokens(left: NormalizedEntityToken, right: NormalizedE
     if (gap < 0) {
       return false;
     }
-    return gap <= 1;
+    if (gap > 1) {
+      return false;
+    }
+    if (sourceText && gap > 0) {
+      const between = sourceText.slice(leftEnd, rightStart);
+      if (between && /[^\s]/u.test(between)) {
+        return false;
+      }
+    }
+    return true;
   }
 
   if (/[.,!?;:]$/.test(left.text) || /^[.,!?;:]/.test(right.text)) {
@@ -129,7 +188,10 @@ function shouldMergeEntityTokens(left: NormalizedEntityToken, right: NormalizedE
   return true;
 }
 
-function collapseAdjacentEntityTokens(tokens: NormalizedEntityToken[]): NormalizedEntityToken[] {
+function collapseAdjacentEntityTokens(
+  tokens: NormalizedEntityToken[],
+  sourceText?: string,
+): NormalizedEntityToken[] {
   if (tokens.length <= 1) {
     return tokens;
   }
@@ -137,7 +199,7 @@ function collapseAdjacentEntityTokens(tokens: NormalizedEntityToken[]): Normaliz
   const collapsed: NormalizedEntityToken[] = [];
   for (const token of tokens) {
     const previous = collapsed[collapsed.length - 1];
-    if (!previous || !shouldMergeEntityTokens(previous, token)) {
+    if (!previous || !shouldMergeEntityTokens(previous, token, sourceText)) {
       collapsed.push({ ...token });
       continue;
     }
@@ -416,7 +478,7 @@ export class EntityExtractionManager {
       });
     }
 
-    const collapsed = collapseAdjacentEntityTokens(normalized);
+    const collapsed = collapseAdjacentEntityTokens(normalized, params.text);
     const deduped = new Map<string, ExtractedEntity>();
     for (const token of collapsed) {
       const canonicalUri = buildCanonicalEntityUri(token.type, token.text);

package/src/extract.ts

@@ -12,6 +12,34 @@ const HEURISTIC_PATTERNS = [
   /my name is|i am|contact me at|email is|phone is/i,
   /deadline|due date|todo|action item|follow up/i,
 ];
+const HEURISTIC_LOOKBACK_MULTIPLIER = 4;
+const HEURISTIC_MIN_LOOKBACK_MESSAGES = 12;
+const FEED_TAG_PATTERN = /\[(?:n8n|rss|alert|news|cron|slack|discord|telegram|email|github|jira)[^[]*]/i;
+const ROLE_LABEL_PATTERN = /\b(?:assistant|system|tool|developer)\s*:/gi;
+
+function isLikelyFeedOrImportedText(text: string): boolean {
+  if (FEED_TAG_PATTERN.test(text)) {
+    return true;
+  }
+
+  const roleLabels = text.match(ROLE_LABEL_PATTERN)?.length ?? 0;
+  if (roleLabels >= 2) {
+    return true;
+  }
+
+  const lines = text
+    .split(/\r?\n+/)
+    .map((line) => line.trim())
+    .filter(Boolean);
+  if (lines.length === 0) {
+    return false;
+  }
+
+  const rolePrefixedLines = lines.filter((line) =>
+    /^(assistant|system|tool|developer|user)\s*:/i.test(line),
+  ).length;
+  return rolePrefixedLines >= 2;
+}
 
 function extractMessageText(content: unknown): string {
   if (typeof content === "string") {
@@ -98,8 +126,10 @@ function pickHeuristicCandidates(
 ): ExtractedCandidate[] {
   const out: ExtractedCandidate[] = [];
   const seen = new Set<string>();
+  const lookback = Math.max(HEURISTIC_MIN_LOOKBACK_MESSAGES, maxItems * HEURISTIC_LOOKBACK_MULTIPLIER);
+  const startIndex = Math.max(0, messages.length - lookback);
 
-  for (let i = messages.length - 1; i >= 0; i -= 1) {
+  for (let i = messages.length - 1; i >= startIndex; i -= 1) {
     const message = messages[i];
     if (!message || (message.role !== "user" && message.role !== "assistant")) {
       continue;
@@ -107,6 +137,9 @@ function pickHeuristicCandidates(
     if (message.text.length < 20 || message.text.length > 3000) {
       continue;
     }
+    if (isLikelyFeedOrImportedText(message.text)) {
+      continue;
+    }
 
     const score = scoreHeuristic(message.text);
     if (score < 0.2) {
@@ -435,6 +468,17 @@ export async function extractCandidates(params: {
 
   try {
     if (params.cfg.capture.mode === "hybrid") {
+      if (heuristic.length === 0) {
+        params.log.debug("memory_braid.capture.ml", {
+          runId: params.runId,
+          mode: params.cfg.capture.mode,
+          provider: params.cfg.capture.ml.provider,
+          model: params.cfg.capture.ml.model,
+          decision: "skip_ml_enrichment_no_heuristic_candidates",
+        });
+        return heuristic;
+      }
+
       const ml = await callMlEnrichment({
         provider: params.cfg.capture.ml.provider,
         model: params.cfg.capture.ml.model,

package/src/index.ts

@@ -24,7 +24,7 @@ import {
   writeStatsState,
 } from "./state.js";
 import type { LifecycleEntry, MemoryBraidResult, ScopeKey } from "./types.js";
-import { normalizeForHash, sha256 } from "./chunking.js";
+import { normalizeForHash, normalizeWhitespace, sha256 } from "./chunking.js";
 
 function jsonToolResult(payload: unknown) {
   return {
@@ -225,6 +225,17 @@ function isGenericUserSummary(text: string): boolean {
   );
 }
 
+function sanitizeRecallQuery(text: string): string {
+  if (!text) {
+    return "";
+  }
+  const withoutInjectedMemories = text.replace(
+    /<relevant-memories>[\s\S]*?<\/relevant-memories>/gi,
+    " ",
+  );
+  return normalizeWhitespace(withoutInjectedMemories);
+}
+
 function applyMem0QualityAdjustments(params: {
   results: MemoryBraidResult[];
   query: string;
@@ -1220,6 +1231,10 @@ const memoryBraidPlugin = {
 
     api.on("before_agent_start", async (event, ctx) => {
       const runId = log.newRunId();
+      const recallQuery = sanitizeRecallQuery(event.prompt);
+      if (!recallQuery) {
+        return;
+      }
       const toolCtx: OpenClawPluginToolContext = {
         config: api.config,
         workspaceDir: ctx.workspaceDir,
@@ -1235,17 +1250,17 @@ const memoryBraidPlugin = {
         log,
         ctx: toolCtx,
         statePaths: runtimeStatePaths,
-        query: event.prompt,
+        query: recallQuery,
         args: {
-          query: event.prompt,
+          query: recallQuery,
           maxResults: cfg.recall.maxResults,
         },
         runId,
       });
 
       const selected = selectMemoriesForInjection({
-        query: event.prompt,
-        results: recall.merged,
+        query: recallQuery,
+        results: recall.mem0,
         limit: cfg.recall.injectTopK,
       });
       if (selected.injected.length === 0) {
@@ -1256,6 +1271,7 @@ const memoryBraidPlugin = {
           sessionKey: scope.sessionKey,
           workspaceHash: scope.workspaceHash,
           count: 0,
+          source: "mem0",
           queryTokens: selected.queryTokens,
           filteredOut: selected.filteredOut,
           genericRejected: selected.genericRejected,
@@ -1272,6 +1288,7 @@ const memoryBraidPlugin = {
         sessionKey: scope.sessionKey,
         workspaceHash: scope.workspaceHash,
         count: selected.injected.length,
+        source: "mem0",
         queryTokens: selected.queryTokens,
         filteredOut: selected.filteredOut,
         genericRejected: selected.genericRejected,
@@ -1328,105 +1345,153 @@ const memoryBraidPlugin = {
         return;
       }
 
-      await withStateLock(runtimeStatePaths.stateLockFile, async () => {
+      const thirtyDays = 30 * 24 * 60 * 60 * 1000;
+      const candidateEntries = candidates.map((candidate) => ({
+        candidate,
+        hash: sha256(normalizeForHash(candidate.text)),
+      }));
+
+      const prepared = await withStateLock(runtimeStatePaths.stateLockFile, async () => {
         const dedupe = await readCaptureDedupeState(runtimeStatePaths);
-        const stats = await readStatsState(runtimeStatePaths);
-        const lifecycle = cfg.lifecycle.enabled
-          ? await readLifecycleState(runtimeStatePaths)
-          : null;
         const now = Date.now();
-        const thirtyDays = 30 * 24 * 60 * 60 * 1000;
+
+        let pruned = 0;
         for (const [key, ts] of Object.entries(dedupe.seen)) {
           if (now - ts > thirtyDays) {
             delete dedupe.seen[key];
+            pruned += 1;
           }
         }
 
-        let persisted = 0;
         let dedupeSkipped = 0;
-        let entityAnnotatedCandidates = 0;
-        let totalEntitiesAttached = 0;
-        let mem0AddAttempts = 0;
-        let mem0AddWithId = 0;
-        let mem0AddWithoutId = 0;
-        for (const candidate of candidates) {
-          const hash = sha256(normalizeForHash(candidate.text));
-          if (dedupe.seen[hash]) {
+        const pending: typeof candidateEntries = [];
+        const seenInBatch = new Set<string>();
+        for (const entry of candidateEntries) {
+          if (dedupe.seen[entry.hash] || seenInBatch.has(entry.hash)) {
             dedupeSkipped += 1;
             continue;
           }
+          seenInBatch.add(entry.hash);
+          pending.push(entry);
+        }
+
+        if (pruned > 0) {
+          await writeCaptureDedupeState(runtimeStatePaths, dedupe);
+        }
+
+        return {
+          dedupeSkipped,
+          pending,
+        };
+      });
+
+      let entityAnnotatedCandidates = 0;
+      let totalEntitiesAttached = 0;
+      let mem0AddAttempts = 0;
+      let mem0AddWithId = 0;
+      let mem0AddWithoutId = 0;
+      const successfulAdds: Array<{
+        memoryId: string;
+        hash: string;
+        category: (typeof candidates)[number]["category"];
+      }> = [];
+
+      for (const entry of prepared.pending) {
+        const { candidate, hash } = entry;
+        const metadata: Record<string, unknown> = {
+          sourceType: "capture",
+          workspaceHash: scope.workspaceHash,
+          agentId: scope.agentId,
+          sessionKey: scope.sessionKey,
+          category: candidate.category,
+          captureScore: candidate.score,
+          extractionSource: candidate.source,
+          contentHash: hash,
+          indexedAt: new Date().toISOString(),
+        };
+
+        if (cfg.entityExtraction.enabled) {
+          const entities = await entityExtraction.extract({
+            text: candidate.text,
+            runId,
+          });
+          if (entities.length > 0) {
+            entityAnnotatedCandidates += 1;
+            totalEntitiesAttached += entities.length;
+            metadata.entityUris = entities.map((entity) => entity.canonicalUri);
+            metadata.entities = entities;
+          }
+        }
 
-          const metadata: Record<string, unknown> = {
-            sourceType: "capture",
+        mem0AddAttempts += 1;
+        const addResult = await mem0.addMemory({
+          text: candidate.text,
+          scope,
+          metadata,
+          runId,
+        });
+        if (addResult.id) {
+          mem0AddWithId += 1;
+          successfulAdds.push({
+            memoryId: addResult.id,
+            hash,
+            category: candidate.category,
+          });
+        } else {
+          mem0AddWithoutId += 1;
+          log.warn("memory_braid.capture.persist", {
+            runId,
+            reason: "mem0_add_missing_id",
             workspaceHash: scope.workspaceHash,
             agentId: scope.agentId,
             sessionKey: scope.sessionKey,
+            contentHashPrefix: hash.slice(0, 12),
             category: candidate.category,
-            captureScore: candidate.score,
-            extractionSource: candidate.source,
-            contentHash: hash,
-            indexedAt: new Date(now).toISOString(),
-          };
+          });
+        }
+      }
 
-          if (cfg.entityExtraction.enabled) {
-            const entities = await entityExtraction.extract({
-              text: candidate.text,
-              runId,
-            });
-            if (entities.length > 0) {
-              entityAnnotatedCandidates += 1;
-              totalEntitiesAttached += entities.length;
-              metadata.entityUris = entities.map((entity) => entity.canonicalUri);
-              metadata.entities = entities;
-            }
+      await withStateLock(runtimeStatePaths.stateLockFile, async () => {
+        const dedupe = await readCaptureDedupeState(runtimeStatePaths);
+        const stats = await readStatsState(runtimeStatePaths);
+        const lifecycle = cfg.lifecycle.enabled
+          ? await readLifecycleState(runtimeStatePaths)
+          : null;
+        const now = Date.now();
+
+        for (const [key, ts] of Object.entries(dedupe.seen)) {
+          if (now - ts > thirtyDays) {
+            delete dedupe.seen[key];
           }
+        }
 
-          mem0AddAttempts += 1;
-          const addResult = await mem0.addMemory({
-            text: candidate.text,
-            scope,
-            metadata,
-            runId,
-          });
-          if (addResult.id) {
-            dedupe.seen[hash] = now;
-            mem0AddWithId += 1;
-            persisted += 1;
-            if (lifecycle) {
-              const memoryId = addResult.id;
-              const existing = lifecycle.entries[memoryId];
-              lifecycle.entries[memoryId] = {
-                memoryId,
-                contentHash: hash,
-                workspaceHash: scope.workspaceHash,
-                agentId: scope.agentId,
-                sessionKey: scope.sessionKey,
-                category: candidate.category,
-                createdAt: existing?.createdAt ?? now,
-                lastCapturedAt: now,
-                lastRecalledAt: existing?.lastRecalledAt,
-                recallCount: existing?.recallCount ?? 0,
-                updatedAt: now,
-              };
-            }
-          } else {
-            mem0AddWithoutId += 1;
-            log.warn("memory_braid.capture.persist", {
-              runId,
-              reason: "mem0_add_missing_id",
+        let persisted = 0;
+        for (const entry of successfulAdds) {
+          dedupe.seen[entry.hash] = now;
+          persisted += 1;
+
+          if (lifecycle) {
+            const existing = lifecycle.entries[entry.memoryId];
+            lifecycle.entries[entry.memoryId] = {
+              memoryId: entry.memoryId,
+              contentHash: entry.hash,
               workspaceHash: scope.workspaceHash,
               agentId: scope.agentId,
               sessionKey: scope.sessionKey,
-              contentHashPrefix: hash.slice(0, 12),
-              category: candidate.category,
-            });
+              category: entry.category,
+              createdAt: existing?.createdAt ?? now,
+              lastCapturedAt: now,
+              lastRecalledAt: existing?.lastRecalledAt,
+              recallCount: existing?.recallCount ?? 0,
+              updatedAt: now,
+            };
           }
         }
 
         stats.capture.runs += 1;
         stats.capture.runsWithCandidates += 1;
         stats.capture.candidates += candidates.length;
-        stats.capture.dedupeSkipped += dedupeSkipped;
+        stats.capture.dedupeSkipped += prepared.dedupeSkipped;
         stats.capture.persisted += persisted;
         stats.capture.mem0AddAttempts += mem0AddAttempts;
         stats.capture.mem0AddWithId += mem0AddWithId;
@@ -1447,7 +1512,8 @@ const memoryBraidPlugin = {
           agentId: scope.agentId,
           sessionKey: scope.sessionKey,
           candidates: candidates.length,
-          dedupeSkipped,
+          pending: prepared.pending.length,
+          dedupeSkipped: prepared.dedupeSkipped,
           persisted,
           mem0AddAttempts,
           mem0AddWithId,

package/src/mem0-client.ts

@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
 import { createRequire } from "node:module";
 import os from "node:os";
 import path from "node:path";
-import { normalizeForHash } from "./chunking.js";
+import { normalizeForHash, sha256 } from "./chunking.js";
 import type { MemoryBraidConfig } from "./config.js";
 import { MemoryBraidLogger } from "./logger.js";
 import type { MemoryBraidResult, ScopeKey } from "./types.js";
@@ -403,6 +403,9 @@ type Mem0AdapterOptions = {
   stateDir?: string;
 };
 
+const SEMANTIC_SEARCH_CACHE_TTL_MS = 30_000;
+const SEMANTIC_SEARCH_CACHE_MAX_ENTRIES = 256;
+
 export class Mem0Adapter {
   private cloudClient: CloudClientLike | null = null;
   private ossClient: OssClientLike | null = null;
@@ -410,6 +413,13 @@ export class Mem0Adapter {
   private readonly log: MemoryBraidLogger;
   private readonly pluginDir?: string;
   private stateDir?: string;
+  private readonly semanticSearchCache = new Map<
+    string,
+    {
+      expiresAt: number;
+      results: MemoryBraidResult[];
+    }
+  >();
 
   constructor(cfg: MemoryBraidConfig, log: MemoryBraidLogger, options?: Mem0AdapterOptions) {
     this.cfg = cfg;
@@ -425,6 +435,7 @@ export class Mem0Adapter {
     }
     this.stateDir = next;
     this.ossClient = null;
+    this.semanticSearchCache.clear();
   }
 
   private async ensureCloudClient(): Promise<CloudClientLike | null> {
@@ -844,12 +855,38 @@ export class Mem0Adapter {
     runId?: string;
   }): Promise<number | undefined> {
     const rightHash = normalizeForHash(params.rightText);
-    const results = await this.searchMemories({
-      query: params.leftText,
-      maxResults: 5,
-      scope: params.scope,
-      runId: params.runId,
-    });
+    if (!rightHash) {
+      return undefined;
+    }
+
+    const leftHash = normalizeForHash(params.leftText);
+    if (!leftHash) {
+      return undefined;
+    }
+
+    const now = Date.now();
+    this.pruneSemanticSearchCache(now);
+    const scopeSession = params.scope.sessionKey ?? "";
+    const cacheKey = `${params.scope.workspaceHash}|${params.scope.agentId}|${scopeSession}|${sha256(leftHash)}`;
+    const cached = this.semanticSearchCache.get(cacheKey);
+    const results =
+      cached && cached.expiresAt > now
+        ? cached.results
+        : await this.searchMemories({
+            query: params.leftText,
+            maxResults: 5,
+            scope: params.scope,
+            runId: params.runId,
+          });
+
+    if (!cached || cached.expiresAt <= now) {
+      this.semanticSearchCache.set(cacheKey, {
+        expiresAt: now + SEMANTIC_SEARCH_CACHE_TTL_MS,
+        results,
+      });
+      this.pruneSemanticSearchCache(now);
+    }
+
     for (const result of results) {
       if (normalizeForHash(result.snippet) === rightHash) {
         return result.score;
@@ -857,4 +894,20 @@ export class Mem0Adapter {
     }
     return undefined;
   }
+
+  private pruneSemanticSearchCache(now = Date.now()): void {
+    for (const [key, entry] of this.semanticSearchCache.entries()) {
+      if (entry.expiresAt <= now) {
+        this.semanticSearchCache.delete(key);
+      }
+    }
+
+    while (this.semanticSearchCache.size > SEMANTIC_SEARCH_CACHE_MAX_ENTRIES) {
+      const oldest = this.semanticSearchCache.keys().next().value as string | undefined;
+      if (!oldest) {
+        break;
+      }
+      this.semanticSearchCache.delete(oldest);
+    }
+  }
 }

package/src/state.ts

@@ -77,7 +77,7 @@ export async function readCaptureDedupeState(paths: StatePaths): Promise<Capture
   const value = await readJsonFile(paths.captureDedupeFile, DEFAULT_CAPTURE_DEDUPE);
   return {
     version: 1,
-    seen: value.seen ?? {},
+    seen: { ...(value.seen ?? {}) },
   };
 }
 
@@ -92,7 +92,7 @@ export async function readLifecycleState(paths: StatePaths): Promise<LifecycleSt
   const value = await readJsonFile(paths.lifecycleFile, DEFAULT_LIFECYCLE);
   return {
     version: 1,
-    entries: value.entries ?? {},
+    entries: { ...(value.entries ?? {}) },
     lastCleanupAt: value.lastCleanupAt,
     lastCleanupReason: value.lastCleanupReason,
     lastCleanupScanned: value.lastCleanupScanned,