@joshuaswarren/openclaw-engram 9.1.6 → 9.1.8

package/README.md

@@ -266,6 +266,8 @@ These capabilities can be enabled progressively:
 - **Shared Context** — Cross-agent memory sharing for multi-agent setups
 - **Identity Continuity** — Consistent agent personality across sessions
 - **Hot/Cold Tiering** — Automatic migration of aging memories to cold storage
+- **Memory Cache** — Process-level singleton cache for `readAllMemories()` — turns 15s disk scans into <100ms cache hits, shared across all sessions
+- **Semantic Consolidation** — Finds clusters of semantically similar memories, synthesizes canonical versions via LLM, archives originals to reduce bloat
 - **Native Knowledge** — Search curated markdown (workspace docs, Obsidian vaults) without extracting into memory
 - **Behavior Loop Tuning** — Runtime self-tuning of extraction and recall parameters
 
@@ -334,6 +336,45 @@ Enable it in your `openclaw.json`:
 
 Set `parallelMaxResultsPerAgent: 0` to disable an individual agent's results without disabling the feature entirely.
 
+### Semantic Consolidation (opt-in)
+
+Over time, memory stores accumulate redundant facts — the same information extracted multiple times across sessions, expressed slightly differently. Semantic consolidation finds clusters of similar memories using token overlap, synthesizes a single canonical version via LLM, and archives the originals. This reduces storage bloat, speeds up recall, and improves memory quality.
+
+- **Conservative by default** — Only merges when 80%+ token overlap is detected across 3+ memories
+- **LLM synthesis** — Uses your configured model to combine unique information from all cluster members
+- **Safe archival** — Originals are archived (not deleted) with full provenance tracking
+- **Configurable** — Adjust threshold, cluster size, excluded categories, model, and schedule
+- **Excluded categories** — Corrections and commitments are never consolidated (configurable)
+
+Enable it in your `openclaw.json`:
+
+```jsonc
+{
+  "plugins": {
+    "entries": {
+      "openclaw-engram": {
+        "config": {
+          "semanticConsolidationEnabled": true
+          // Optional tuning:
+          // "semanticConsolidationThreshold": 0.8,    // 0.8=conservative, 0.6=aggressive
+          // "semanticConsolidationModel": "fast",      // "auto", "fast", or specific model
+          // "semanticConsolidationIntervalHours": 168, // weekly (default)
+          // "semanticConsolidationMaxPerRun": 100
+        }
+      }
+    }
+  }
+}
+```
+
+Run manually from the CLI:
+
+```bash
+openclaw engram semantic-consolidate --dry-run    # Preview what would be merged
+openclaw engram semantic-consolidate --verbose     # Run with detailed output
+openclaw engram semantic-consolidate --threshold 0.6  # Override threshold
+```
+
 ### Advanced (opt-in)
 
 - **Objective-State Recall** — Surfaces file/process/tool state snapshots alongside semantic memory
@@ -417,6 +458,11 @@ openclaw engram review-disposition <id> --status rejected  # Operator review
 openclaw engram benchmark recall   # Benchmark status and validation
 openclaw engram benchmark-ci-gate  # CI gate for regressions
 
+# Memory maintenance
+openclaw engram consolidate                  # Run standard consolidation
+openclaw engram semantic-consolidate         # Run semantic dedup consolidation
+openclaw engram semantic-consolidate --dry-run  # Preview without changes
+
 # Access layer
 openclaw engram access http-serve --token "$TOKEN"  # Start HTTP API
 openclaw engram access mcp-serve   # Start stdio MCP server
@@ -441,6 +487,9 @@ All settings live in `openclaw.json` under `plugins.entries.openclaw-engram.conf
 | `memoryDir` | `~/.openclaw/workspace/memory/local` | Memory storage root |
 | `memoryOsPreset` | unset | Quick config: `conservative`, `balanced`, `research-max`, `local-llm-heavy` |
 | `lcmEnabled` | `false` | Enable Lossless Context Management (proactive session archive + summary DAG) |
+| `semanticConsolidationEnabled` | `false` | Enable periodic semantic dedup of similar memories |
+| `semanticConsolidationThreshold` | `0.8` | Token overlap threshold (0.8=conservative, 0.6=aggressive) |
+| `semanticConsolidationModel` | `"auto"` | LLM for synthesis: `"auto"`, `"fast"`, or specific model |
 
 **[See the full config reference for all 60+ settings](docs/config-reference.md)** including search backend configuration, namespace policies, Memory OS features, governance, evaluation harness, trust zones, causal trajectories, and more.
 

package/dist/access-cli.js

@@ -3,13 +3,13 @@ import {
   EngramAccessService,
   Orchestrator,
   parseConfig
-} from "./chunk-HPTDE747.js";
-import "./chunk-BIVUV2GJ.js";
+} from "./chunk-5HN5NZMV.js";
+import "./chunk-UO4TIAAW.js";
 import "./chunk-IMMYYNXG.js";
-import "./chunk-QX3ELIKM.js";
+import "./chunk-YKWJUR3I.js";
 import "./chunk-6KX4XLQJ.js";
+import "./chunk-XCAYYSI7.js";
 import "./chunk-MKM2BCQH.js";
-import "./chunk-V2E2ORE5.js";
 import "./chunk-DEIBZP3O.js";
 import "./chunk-SSIIJJKA.js";
 

package/dist/{calibration-LBE4XTN4.js → calibration-QBVYTFTQ.js}

@@ -1,7 +1,7 @@
 // openclaw-engram: Local-first memory plugin
 import {
   FallbackLlmClient
-} from "./chunk-V2E2ORE5.js";
+} from "./chunk-XCAYYSI7.js";
 import {
   listJsonFiles
 } from "./chunk-DEIBZP3O.js";
@@ -233,4 +233,4 @@ export {
   runCalibrationIfEnabled,
   synthesizeCalibrationRules
 };
-//# sourceMappingURL=calibration-LBE4XTN4.js.map
+//# sourceMappingURL=calibration-QBVYTFTQ.js.map

package/dist/{causal-consolidation-5VSMA2L5.js → causal-consolidation-T7EEYKPA.js}

@@ -1,4 +1,7 @@
 // openclaw-engram: Local-first memory plugin
+import {
+  FallbackLlmClient
+} from "./chunk-XCAYYSI7.js";
 import {
   readChainIndex,
   resolveChainsDir
@@ -6,9 +9,6 @@ import {
 import {
   isRecord
 } from "./chunk-MKM2BCQH.js";
-import {
-  FallbackLlmClient
-} from "./chunk-V2E2ORE5.js";
 import {
   listJsonFiles,
   readJsonFile
@@ -203,4 +203,4 @@ export {
   deriveCausalPromotionCandidates,
   synthesizeCausalPreferencesViaLlm
 };
-//# sourceMappingURL=causal-consolidation-5VSMA2L5.js.map
+//# sourceMappingURL=causal-consolidation-T7EEYKPA.js.map

package/dist/{chunk-HPTDE747.js → chunk-5HN5NZMV.js}

@@ -3,7 +3,7 @@ import {
   CompoundingEngine,
   SharedContextManager,
   defaultTierMigrationCycleBudget
-} from "./chunk-BIVUV2GJ.js";
+} from "./chunk-UO4TIAAW.js";
 import {
   searchCausalTrajectories
 } from "./chunk-IMMYYNXG.js";
@@ -21,10 +21,16 @@ import {
   rotateMarkdownFileToArchive,
   sanitizeMemoryContent,
   toMemoryPathRel
-} from "./chunk-QX3ELIKM.js";
+} from "./chunk-YKWJUR3I.js";
 import {
   GraphIndex
 } from "./chunk-6KX4XLQJ.js";
+import {
+  FallbackLlmClient,
+  buildChatCompletionTokenLimit,
+  extractJsonCandidates,
+  shouldAssumeOpenAiChatCompletions
+} from "./chunk-XCAYYSI7.js";
 import {
   BoxBuilder,
   assertIsoRecordedAt,
@@ -38,12 +44,6 @@ import {
   recordStoreDay,
   validateStringRecord
 } from "./chunk-MKM2BCQH.js";
-import {
-  FallbackLlmClient,
-  buildChatCompletionTokenLimit,
-  extractJsonCandidates,
-  shouldAssumeOpenAiChatCompletions
-} from "./chunk-V2E2ORE5.js";
 import {
   listJsonFiles,
   listNamedFiles,
@@ -545,6 +545,15 @@ function parseConfig(raw) {
     verifiedRecallEnabled: cfg.verifiedRecallEnabled === true,
     semanticRulePromotionEnabled: cfg.semanticRulePromotionEnabled === true,
     semanticRuleVerificationEnabled: cfg.semanticRuleVerificationEnabled === true,
+    semanticConsolidationEnabled: cfg.semanticConsolidationEnabled === true,
+    semanticConsolidationModel: typeof cfg.semanticConsolidationModel === "string" && cfg.semanticConsolidationModel.length > 0 ? cfg.semanticConsolidationModel : "auto",
+    semanticConsolidationThreshold: typeof cfg.semanticConsolidationThreshold === "number" ? cfg.semanticConsolidationThreshold : 0.8,
+    semanticConsolidationMinClusterSize: typeof cfg.semanticConsolidationMinClusterSize === "number" ? Math.max(2, Math.floor(cfg.semanticConsolidationMinClusterSize)) : 3,
+    semanticConsolidationExcludeCategories: Array.isArray(cfg.semanticConsolidationExcludeCategories) ? cfg.semanticConsolidationExcludeCategories.filter(
+      (c) => typeof c === "string" && c.length > 0
+    ) : ["correction", "commitment"],
+    semanticConsolidationIntervalHours: typeof cfg.semanticConsolidationIntervalHours === "number" ? Math.max(1, Math.floor(cfg.semanticConsolidationIntervalHours)) : 168,
+    semanticConsolidationMaxPerRun: typeof cfg.semanticConsolidationMaxPerRun === "number" ? Math.max(0, Math.floor(cfg.semanticConsolidationMaxPerRun)) : 100,
     creationMemoryEnabled: cfg.creationMemoryEnabled === true,
     memoryUtilityLearningEnabled: cfg.memoryUtilityLearningEnabled === true,
     promotionByOutcomeEnabled: cfg.promotionByOutcomeEnabled === true,
@@ -18750,6 +18759,82 @@ function validateReplayTurn(turn, index) {
   return issues;
 }
 
+// src/semantic-consolidation.ts
+function findSimilarClusters(memories, config) {
+  const excluded = new Set(config.excludeCategories);
+  const byCategory = /* @__PURE__ */ new Map();
+  for (const m of memories) {
+    const cat = m.frontmatter.category;
+    if (excluded.has(cat)) continue;
+    if (m.frontmatter.status && m.frontmatter.status !== "active") continue;
+    const list = byCategory.get(cat) ?? [];
+    list.push(m);
+    byCategory.set(cat, list);
+  }
+  const clusters = [];
+  let totalCandidates = 0;
+  for (const [category, mems] of byCategory) {
+    if (totalCandidates >= config.maxPerRun) break;
+    const tokenized = mems.map((m) => ({
+      memory: m,
+      tokens: new Set(normalizeRecallTokens(m.content, []))
+    }));
+    const clustered = /* @__PURE__ */ new Set();
+    for (let i = 0; i < tokenized.length && totalCandidates < config.maxPerRun; i++) {
+      if (clustered.has(tokenized[i].memory.frontmatter.id)) continue;
+      const cluster = [tokenized[i].memory];
+      let totalOverlap = 0;
+      let comparisons = 0;
+      for (let j = i + 1; j < tokenized.length; j++) {
+        if (clustered.has(tokenized[j].memory.frontmatter.id)) continue;
+        const aTokens = tokenized[i].tokens;
+        const bTokens = tokenized[j].tokens;
+        if (aTokens.size === 0 || bTokens.size === 0) continue;
+        const overlap = countRecallTokenOverlap(aTokens, [...bTokens].join(" "));
+        const maxTokens = Math.max(aTokens.size, bTokens.size);
+        const score = maxTokens > 0 ? overlap / maxTokens : 0;
+        if (score >= config.threshold) {
+          cluster.push(tokenized[j].memory);
+          totalOverlap += score;
+          comparisons++;
+          if (totalCandidates + cluster.length >= config.maxPerRun) break;
+        }
+      }
+      if (cluster.length >= config.minClusterSize) {
+        for (const m of cluster) clustered.add(m.frontmatter.id);
+        clusters.push({
+          category,
+          memories: cluster,
+          overlapScore: comparisons > 0 ? totalOverlap / comparisons : 0
+        });
+        totalCandidates += cluster.length;
+      }
+    }
+  }
+  return clusters;
+}
+function buildConsolidationPrompt(cluster) {
+  const memoryTexts = cluster.memories.map(
+    (m, i) => `Memory ${i + 1} (${m.frontmatter.id}, created ${m.frontmatter.created}):
+${m.content}`
+  ).join("\n\n");
+  return `You are a memory consolidation system. The following ${cluster.memories.length} memories in the "${cluster.category}" category contain overlapping information.
+
+Synthesize them into ONE canonical memory that:
+1. Preserves ALL unique information from every source memory
+2. Removes redundancy and repetition
+3. Uses clear, concise language
+4. Maintains the same category and tone
+5. Does NOT add information that isn't in the sources
+
+${memoryTexts}
+
+Write ONLY the consolidated memory content (no metadata, no explanation, no preamble):`;
+}
+function parseConsolidationResponse(response) {
+  return response.trim();
+}
+
 // src/conversation-index/chunker.ts
 function chunkTranscriptEntries(sessionKey, entries, opts) {
   const maxChars = Math.max(500, opts.maxChars);
@@ -21098,6 +21183,140 @@ var Orchestrator = class _Orchestrator {
   async runConsolidationNow() {
     return this.runConsolidation();
   }
+  async runSemanticConsolidationNow(options) {
+    return this.runSemanticConsolidation({ ...options, force: true });
+  }
+  async runSemanticConsolidation(options) {
+    const result = {
+      clustersFound: 0,
+      memoriesConsolidated: 0,
+      memoriesArchived: 0,
+      errors: 0,
+      clusters: []
+    };
+    if (!this.config.semanticConsolidationEnabled && !options?.force) {
+      log.debug("[semantic-consolidation] disabled in config");
+      return result;
+    }
+    log.info("[semantic-consolidation] starting run");
+    const allMemories = await this.storage.readAllMemories();
+    if (allMemories.length < 10) {
+      log.debug("[semantic-consolidation] too few memories, skipping");
+      return result;
+    }
+    const threshold = options?.thresholdOverride ?? this.config.semanticConsolidationThreshold;
+    const clusters = findSimilarClusters(allMemories, {
+      threshold,
+      minClusterSize: this.config.semanticConsolidationMinClusterSize,
+      excludeCategories: this.config.semanticConsolidationExcludeCategories,
+      maxPerRun: this.config.semanticConsolidationMaxPerRun
+    });
+    result.clustersFound = clusters.length;
+    result.clusters = clusters;
+    if (clusters.length === 0) {
+      log.info("[semantic-consolidation] no clusters found");
+      return result;
+    }
+    log.info(`[semantic-consolidation] found ${clusters.length} cluster(s)`);
+    if (options?.dryRun) {
+      log.info("[semantic-consolidation] dry run \u2014 skipping LLM synthesis and archival");
+      return result;
+    }
+    const { FallbackLlmClient: FallbackLlmClient2 } = await import("./fallback-llm-KV6HAJ2N.js");
+    const modelSetting = this.config.semanticConsolidationModel;
+    if (modelSetting === "fast" && this.fastLlm) {
+      log.info("[semantic-consolidation] using fast local LLM for synthesis");
+    }
+    const llm = new FallbackLlmClient2(this.config.gatewayConfig);
+    if (!llm.isAvailable() && !(modelSetting === "fast" && this.fastLlm)) {
+      log.warn("[semantic-consolidation] no LLM available \u2014 skipping synthesis");
+      return result;
+    }
+    for (const cluster of clusters) {
+      try {
+        const prompt = buildConsolidationPrompt(cluster);
+        const messages = [
+          { role: "system", content: "You are a memory consolidation system. Output only the consolidated memory text." },
+          { role: "user", content: prompt }
+        ];
+        const llmOpts = { temperature: 0.2, maxTokens: 2e3 };
+        let response = null;
+        if (modelSetting === "fast" && this.fastLlm) {
+          const fastResult = await this.fastLlm.chatCompletion(messages, {
+            operation: "semantic-consolidation",
+            maxTokens: llmOpts.maxTokens,
+            temperature: llmOpts.temperature
+          });
+          response = fastResult ? { content: fastResult.content } : null;
+        } else {
+          response = await llm.chatCompletion(messages, llmOpts);
+        }
+        if (!response?.content) {
+          log.warn(`[semantic-consolidation] empty LLM response for cluster in "${cluster.category}"`);
+          result.errors++;
+          continue;
+        }
+        const canonicalContent = parseConsolidationResponse(response.content);
+        cluster.canonicalContent = canonicalContent;
+        const sorted = [...cluster.memories].sort(
+          (a, b) => new Date(b.frontmatter.created).getTime() - new Date(a.frontmatter.created).getTime()
+        );
+        const newest = sorted[0];
+        const lineageIds = cluster.memories.map((m) => m.frontmatter.id);
+        const canonicalId = await this.storage.writeMemory(
+          newest.frontmatter.category,
+          canonicalContent,
+          {
+            actor: "semantic-consolidation",
+            confidence: newest.frontmatter.confidence,
+            tags: [...new Set(cluster.memories.flatMap((m) => m.frontmatter.tags ?? []))],
+            source: "semantic-consolidation",
+            lineage: lineageIds
+          }
+        );
+        result.memoriesConsolidated++;
+        for (const m of cluster.memories) {
+          const archiveResult = await this.storage.archiveMemory(m, {
+            actor: "semantic-consolidation",
+            reasonCode: "semantic-consolidation",
+            relatedMemoryIds: [canonicalId]
+          });
+          if (archiveResult) {
+            if (this.contentHashIndex) {
+              this.contentHashIndex.remove(m.content);
+            }
+            await this.embeddingFallback.removeFromIndex(m.frontmatter.id);
+            if (this.config.queryAwareIndexingEnabled && m.path && m.frontmatter?.created) {
+              deindexMemory(
+                this.config.memoryDir,
+                m.path,
+                m.frontmatter.created,
+                m.frontmatter.tags ?? []
+              );
+            }
+            result.memoriesArchived++;
+          }
+        }
+        log.info(
+          `[semantic-consolidation] consolidated ${cluster.memories.length} memories \u2192 ${canonicalId}`
+        );
+      } catch (err) {
+        log.warn(
+          `[semantic-consolidation] cluster processing failed: ${err instanceof Error ? err.message : String(err)}`
+        );
+        result.errors++;
+      }
+    }
+    if (result.memoriesArchived > 0 && this.contentHashIndex) {
+      await this.contentHashIndex.save().catch(
+        (err) => log.warn(`[semantic-consolidation] content-hash index save failed: ${err}`)
+      );
+    }
+    log.info(
+      `[semantic-consolidation] complete: clusters=${result.clustersFound}, consolidated=${result.memoriesConsolidated}, archived=${result.memoriesArchived}, errors=${result.errors}`
+    );
+    return result;
+  }
   async waitForExtractionIdle(timeoutMs = 6e4) {
     const started = Date.now();
     while (this.queueProcessing || this.extractionQueue.length > 0) {
@@ -22823,7 +23042,7 @@ ${trimmedBody}`;
         return null;
       }
       try {
-        const { getCalibrationRulesForRecall, buildCalibrationRecallSection } = await import("./calibration-LBE4XTN4.js");
+        const { getCalibrationRulesForRecall, buildCalibrationRecallSection } = await import("./calibration-QBVYTFTQ.js");
         const rules = await getCalibrationRulesForRecall(this.config.memoryDir);
         if (rules.length === 0) {
           timings.calibrationRules = "skip(no-rules)";
@@ -25368,6 +25587,39 @@ _Context: ${topQuestion.context}_`
         log.info(`archived ${archived} old low-importance facts`);
       }
     }
+    if (this.config.semanticConsolidationEnabled) {
+      try {
+        const stateFilePath = path40.join(this.config.memoryDir, "state", "semantic-consolidation-last-run.json");
+        let shouldRun = true;
+        try {
+          const stateRaw = await readFile22(stateFilePath, "utf-8");
+          const stateData = JSON.parse(stateRaw);
+          if (stateData.lastRunAt) {
+            const lastRunMs = new Date(stateData.lastRunAt).getTime();
+            const intervalMs = this.config.semanticConsolidationIntervalHours * 60 * 60 * 1e3;
+            if (Date.now() - lastRunMs < intervalMs) {
+              shouldRun = false;
+              log.debug("[semantic-consolidation] skipping \u2014 not enough time since last run");
+            }
+          }
+        } catch {
+        }
+        if (shouldRun) {
+          const semResult = await this.runSemanticConsolidation();
+          if (semResult.memoriesArchived > 0) {
+            log.info(`[semantic-consolidation] archived ${semResult.memoriesArchived} memories during maintenance`);
+            allMemories = await this.storage.readAllMemories();
+          }
+          if (semResult.errors === 0 || semResult.memoriesArchived > 0) {
+            const stateDir2 = path40.join(this.config.memoryDir, "state");
+            await mkdir27(stateDir2, { recursive: true });
+            await writeFile26(stateFilePath, JSON.stringify({ lastRunAt: (/* @__PURE__ */ new Date()).toISOString() }), "utf-8");
+          }
+        }
+      } catch (err) {
+        log.warn(`[semantic-consolidation] maintenance pass failed (non-fatal): ${err}`);
+      }
+    }
     if (this.config.identityEnabled) {
       await this.autoConsolidateIdentity();
     }
@@ -28558,8 +28810,15 @@ var EngramAccessService = class {
     const lcmSessionKey = namespace !== this.orchestrator.config.defaultNamespace ? `${namespace}:${request.sessionKey}` : request.sessionKey;
     let lcmArchived = false;
     if (this.orchestrator.lcmEngine && this.orchestrator.lcmEngine.enabled) {
-      await this.orchestrator.lcmEngine.observeMessages(lcmSessionKey, request.messages);
-      lcmArchived = true;
+      try {
+        const lcmPromise = this.orchestrator.lcmEngine.observeMessages(lcmSessionKey, request.messages);
+        lcmPromise.catch((err) => {
+          log.error(`access-observe LCM archival failed: ${err}`);
+        });
+        lcmArchived = true;
+      } catch (err) {
+        log.error(`access-observe LCM enqueue failed: ${err}`);
+      }
     }
     let extractionQueued = false;
     if (request.skipExtraction !== true) {
@@ -28726,4 +28985,4 @@ export {
   EngramAccessInputError,
   EngramAccessService
 };
-//# sourceMappingURL=chunk-HPTDE747.js.map
+//# sourceMappingURL=chunk-5HN5NZMV.js.map