cipher-security 2.0.0

package/lib/memory/synthesizer.js

@@ -0,0 +1,333 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * CIPHER Memory — Stage 2: Online Semantic Synthesis
+ *
+ * Performs intra-session consolidation of memory entries:
+ * - Detects near-duplicate and semantically related entries
+ * - Merges related entries into unified abstract representations
+ * - Maintains compact, coherent memory topology
+ * - Eliminates redundancy while preserving information
+ *
+ * Ported from Python memory/core/synthesizer.py.
+ */
+
+import { createHash } from 'node:crypto';
+
+// ---------------------------------------------------------------------------
+// Helpers (standalone — separate from MemoryConsolidator in engine.js)
+// ---------------------------------------------------------------------------
+
+/**
+ * Generate word trigrams for similarity comparison.
+ * @param {string} text
+ * @returns {Set<string>}
+ */
+function _trigramSet(text) {
+  const words = text.toLowerCase().split(/\s+/).filter(Boolean);
+  if (words.length < 3) return new Set(words);
+  const trigrams = new Set();
+  for (let i = 0; i <= words.length - 3; i++) {
+    trigrams.add(`${words[i]} ${words[i + 1]} ${words[i + 2]}`);
+  }
+  return trigrams;
+}
+
+/**
+ * Jaccard similarity on word trigrams.
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+function _jaccardSimilarity(a, b) {
+  const tgA = _trigramSet(a);
+  const tgB = _trigramSet(b);
+  if (tgA.size === 0 || tgB.size === 0) return 0.0;
+
+  let intersectionSize = 0;
+  for (const t of tgA) {
+    if (tgB.has(t)) intersectionSize++;
+  }
+
+  const unionSize = tgA.size + tgB.size - intersectionSize;
+  return unionSize > 0 ? intersectionSize / unionSize : 0.0;
+}
+
+/**
+ * Generate content fingerprint for near-duplicate detection.
+ * Normalize text, MD5 hash (first 16 chars).
+ * @param {string} text
+ * @returns {string}
+ */
+function _contentFingerprint(text) {
+  // Normalize: lowercase, collapse whitespace, remove punctuation
+  let normalized = text.toLowerCase().replace(/[^\w\s]/g, '');
+  normalized = normalized.replace(/\s+/g, ' ').trim();
+  return createHash('md5').update(normalized).digest('hex').slice(0, 16);
+}
+
+// ---------------------------------------------------------------------------
+// SynthesisResult
+// ---------------------------------------------------------------------------
+
+/**
+ * Result of a synthesis operation.
+ */
+class SynthesisResult {
+  /**
+   * @param {{ mergedEntries?: any[], removedEntryIds?: string[], newEntries?: any[], stats?: Record<string, number> }} opts
+   */
+  constructor(opts = {}) {
+    this.mergedEntries = opts.mergedEntries ?? [];
+    this.removedEntryIds = opts.removedEntryIds ?? [];
+    this.newEntries = opts.newEntries ?? [];
+    this.stats = opts.stats ?? {};
+  }
+}
+
+// ---------------------------------------------------------------------------
+// SemanticSynthesizer
+// ---------------------------------------------------------------------------
+
+// Severity ordering for merge comparison
+const _SEVERITY_ORDER = {
+  critical: 5,
+  high: 4,
+  medium: 3,
+  low: 2,
+  info: 1,
+  '': 0,
+};
+
+// Confidence ordering for merge comparison
+const _CONF_ORDER = {
+  confirmed: 3,
+  inferred: 2,
+  uncertain: 1,
+};
+
+/**
+ * Stage 2: Online Semantic Synthesis.
+ *
+ * Consolidates memory entries during the session:
+ * 1. Deduplication — Remove near-identical entries (fingerprint match)
+ * 2. Merging — Combine highly similar entries (Jaccard > threshold)
+ * 3. Linking — Connect related entries via shared entities/topics
+ */
+class SemanticSynthesizer {
+  /**
+   * @param {{ mergeThreshold?: number, dedupThreshold?: number }} opts
+   */
+  constructor(opts = {}) {
+    this.mergeThreshold = opts.mergeThreshold ?? 0.7;
+    this.dedupThreshold = opts.dedupThreshold ?? 0.95;
+    /** @type {Map<string, string>} fingerprint → entry_id */
+    this._fingerprints = new Map();
+  }
+
+  /**
+   * Synthesize new entries with existing memory.
+   * @param {import('./compressor.js').CompressedEntry[]} newEntries
+   * @param {import('./compressor.js').CompressedEntry[]} existingEntries
+   * @returns {SynthesisResult}
+   */
+  synthesize(newEntries, existingEntries) {
+    // Build fingerprint index for existing entries
+    for (const entry of existingEntries) {
+      const fp = _contentFingerprint(entry.losslessRestatement);
+      this._fingerprints.set(fp, entry.entryId);
+    }
+
+    const merged = [];
+    const removedIds = [];
+    const trulyNew = [];
+    const stats = { deduplicated: 0, merged: 0, linked: 0, new: 0 };
+
+    for (const newEntry of newEntries) {
+      const fp = _contentFingerprint(newEntry.losslessRestatement);
+
+      // Step 1: Exact deduplication
+      if (this._fingerprints.has(fp)) {
+        stats.deduplicated++;
+        continue;
+      }
+
+      // Step 2: Find merge candidates
+      let bestMatch = null;
+      let bestSim = 0.0;
+
+      for (const existing of existingEntries) {
+        if (removedIds.includes(existing.entryId)) continue;
+
+        const sim = _jaccardSimilarity(
+          newEntry.losslessRestatement,
+          existing.losslessRestatement,
+        );
+        if (sim > bestSim) {
+          bestSim = sim;
+          bestMatch = existing;
+        }
+      }
+
+      if (bestMatch && bestSim >= this.mergeThreshold) {
+        const mergedEntry = this._mergeEntries(bestMatch, newEntry);
+        merged.push(mergedEntry);
+        stats.merged++;
+      } else {
+        // Step 3: Link to related entries via shared entities
+        const linked = this._linkEntry(newEntry, existingEntries);
+        if (linked) stats.linked++;
+
+        trulyNew.push(newEntry);
+        this._fingerprints.set(fp, newEntry.entryId);
+        stats.new++;
+      }
+    }
+
+    return new SynthesisResult({
+      mergedEntries: merged,
+      removedEntryIds: removedIds,
+      newEntries: trulyNew,
+      stats,
+    });
+  }
+
+  /**
+   * Merge two similar entries, keeping the richer content.
+   * @private
+   */
+  _mergeEntries(existing, newEntry) {
+    // Keep the longer/richer restatement
+    if (newEntry.losslessRestatement.length > existing.losslessRestatement.length) {
+      existing.losslessRestatement = newEntry.losslessRestatement;
+    }
+
+    // Union all list fields
+    existing.targets = [...new Set([...existing.targets, ...newEntry.targets])];
+    existing.cveIds = [...new Set([...existing.cveIds, ...newEntry.cveIds])];
+    existing.mitreAttack = [...new Set([...existing.mitreAttack, ...newEntry.mitreAttack])];
+    existing.keywords = [...new Set([...existing.keywords, ...newEntry.keywords])].slice(0, 15);
+    existing.toolsUsed = [...new Set([...existing.toolsUsed, ...newEntry.toolsUsed])];
+    existing.hashes = [...new Set([...existing.hashes, ...newEntry.hashes])];
+    existing.persons = [...new Set([...existing.persons, ...newEntry.persons])];
+    existing.entities = [...new Set([...existing.entities, ...newEntry.entities])];
+    existing.sourceTurns = [...new Set([...existing.sourceTurns, ...newEntry.sourceTurns])];
+
+    // Keep higher severity
+    if ((_SEVERITY_ORDER[newEntry.severity] ?? 0) > (_SEVERITY_ORDER[existing.severity] ?? 0)) {
+      existing.severity = newEntry.severity;
+    }
+
+    // Keep higher confidence
+    if ((_CONF_ORDER[newEntry.confidence] ?? 0) > (_CONF_ORDER[existing.confidence] ?? 0)) {
+      existing.confidence = newEntry.confidence;
+    }
+
+    return existing;
+  }
+
+  /**
+   * Link entry to related existing entries via shared entities.
+   * @private
+   */
+  _linkEntry(newEntry, existingEntries) {
+    let linked = false;
+    const newEntities = new Set([
+      ...newEntry.targets,
+      ...newEntry.cveIds,
+      ...newEntry.mitreAttack,
+    ]);
+
+    if (newEntities.size === 0) return false;
+
+    for (const existing of existingEntries) {
+      const existingEntities = new Set([
+        ...existing.targets,
+        ...existing.cveIds,
+        ...existing.mitreAttack,
+      ]);
+
+      // Check for shared entities
+      let hasShared = false;
+      for (const e of newEntities) {
+        if (existingEntities.has(e)) {
+          hasShared = true;
+          break;
+        }
+      }
+
+      if (hasShared) {
+        linked = true;
+        const linkKeywords = [`related:${existing.entryId.slice(0, 8)}`];
+        newEntry.keywords = [...new Set([...newEntry.keywords, ...linkKeywords])];
+      }
+    }
+
+    return linked;
+  }
+
+  /**
+   * Consolidate a batch of entries among themselves.
+   * Groups entries by shared entities and merges within groups.
+   * @param {import('./compressor.js').CompressedEntry[]} entries
+   * @returns {import('./compressor.js').CompressedEntry[]}
+   */
+  consolidateBatch(entries) {
+    if (entries.length <= 1) return entries;
+
+    // Group by shared entities
+    const groups = new Map();
+    const ungrouped = [];
+
+    for (const entry of entries) {
+      const keyEntities = [...entry.targets, ...entry.cveIds];
+      if (keyEntities.length > 0) {
+        const groupKey = [...keyEntities].sort()[0];
+        if (!groups.has(groupKey)) groups.set(groupKey, []);
+        groups.get(groupKey).push(entry);
+      } else {
+        ungrouped.push(entry);
+      }
+    }
+
+    // Merge within groups
+    const consolidated = [...ungrouped];
+    for (const [, groupEntries] of groups) {
+      if (groupEntries.length === 1) {
+        consolidated.push(groupEntries[0]);
+        continue;
+      }
+
+      // Sort by content length (richest first)
+      groupEntries.sort(
+        (a, b) => b.losslessRestatement.length - a.losslessRestatement.length,
+      );
+      let base = groupEntries[0];
+      for (let i = 1; i < groupEntries.length; i++) {
+        const other = groupEntries[i];
+        const sim = _jaccardSimilarity(
+          base.losslessRestatement,
+          other.losslessRestatement,
+        );
+        if (sim >= this.mergeThreshold) {
+          base = this._mergeEntries(base, other);
+        } else {
+          consolidated.push(other);
+        }
+      }
+      consolidated.push(base);
+    }
+
+    return consolidated;
+  }
+}
+
+export {
+  _trigramSet,
+  _jaccardSimilarity,
+  _contentFingerprint,
+  SynthesisResult,
+  SemanticSynthesizer,
+};