cipher-security 5.0.0

package/lib/memory/compressor.js

@@ -0,0 +1,425 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * CIPHER Memory — Stage 1: Semantic Structured Compression
+ *
+ * Converts raw security engagement dialogues into atomic, self-contained
+ * memory entries through extraction:
+ *
+ * - Security-specific entity extraction (IPs, CVEs, MITRE ATT&CK, tools)
+ * - Information density gating (skip low-information exchanges)
+ * - Heuristic compression (always works, no LLM required)
+ * - LLM compression (stubbed — S03 provides the LLM client)
+ *
+ * Ported from Python memory/core/compressor.py.
+ */
+
+import { randomUUID } from 'node:crypto';
+
+// ---------------------------------------------------------------------------
+// Security-specific extraction patterns
+// ---------------------------------------------------------------------------
+
+const _IP_RE = /\b(?:\d{1,3}\.){3}\d{1,3}\b/g;
+const _CVE_RE = /CVE-\d{4}-\d{4,}/g;
+const _MITRE_RE = /T\d{4}(?:\.\d{3})?/g;
+const _PORT_RE = /\b(?:port\s+)?(\d{1,5})(?:\/(?:tcp|udp))?\b/gi;
+const _DOMAIN_RE =
+  /\b(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}\b/g;
+const _HASH_MD5_RE = /\b[a-fA-F0-9]{32}\b/g;
+const _HASH_SHA256_RE = /\b[a-fA-F0-9]{64}\b/g;
+
+// File extensions to filter from domain matches
+const _FILE_EXTENSIONS = ['.py', '.js', '.md', '.yaml', '.json', '.txt'];
+
+// ---------------------------------------------------------------------------
+// Entity Extraction
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract security-relevant entities from text using regex patterns.
+ * @param {string} text
+ * @returns {{ ips: string[], cves: string[], mitre: string[], domains: string[], hashes_sha256: string[], hashes_md5: string[] }}
+ */
+function extractSecurityEntities(text) {
+  // Reset lastIndex on all regexes (they have /g flag)
+  const matchAll = (re, str) => [...str.matchAll(new RegExp(re.source, re.flags))].map((m) => m[0]);
+
+  const ips = [...new Set(matchAll(_IP_RE, text))];
+  const cves = [...new Set(matchAll(_CVE_RE, text))];
+  const mitre = [...new Set(matchAll(_MITRE_RE, text))];
+  const hashes_sha256 = [...new Set(matchAll(_HASH_SHA256_RE, text))];
+  const hashes_md5 = [...new Set(matchAll(_HASH_MD5_RE, text))];
+
+  // Filter hashes: SHA256 matches are 64 chars, MD5 are 32 chars.
+  // If a 64-char hash also matches the 32-char pattern, remove from MD5.
+  const sha256Set = new Set(hashes_sha256);
+  const filteredMd5 = hashes_md5.filter(
+    (h) => !hashes_sha256.some((s) => s.includes(h)),
+  );
+
+  const rawDomains = [...new Set(matchAll(_DOMAIN_RE, text))];
+  const domains = rawDomains.filter(
+    (d) => !_FILE_EXTENSIONS.some((ext) => d.endsWith(ext)),
+  );
+
+  return { ips, cves, mitre, domains, hashes_sha256, hashes_md5: filteredMd5 };
+}
+
+// ---------------------------------------------------------------------------
+// DialogueTurn
+// ---------------------------------------------------------------------------
+
+/**
+ * Single turn in a security engagement dialogue.
+ */
+class DialogueTurn {
+  /**
+   * @param {{ turnId: number, role: string, content: string, timestamp?: string, toolName?: string, toolOutput?: string }} opts
+   */
+  constructor(opts = {}) {
+    this.turnId = opts.turnId ?? 0;
+    this.role = opts.role ?? 'user';
+    this.content = opts.content ?? '';
+    this.timestamp = opts.timestamp ?? '';
+    this.toolName = opts.toolName ?? '';
+    this.toolOutput = opts.toolOutput ?? '';
+  }
+}
+
+// ---------------------------------------------------------------------------
+// CompressedEntry
+// ---------------------------------------------------------------------------
+
+/**
+ * Atomic memory unit produced by Stage 1 compression.
+ */
+class CompressedEntry {
+  constructor(opts = {}) {
+    this.entryId = opts.entryId ?? randomUUID();
+    this.losslessRestatement = opts.losslessRestatement ?? '';
+    this.memoryType = opts.memoryType ?? 'note';
+    this.confidence = opts.confidence ?? 'confirmed';
+
+    // Security-specific extracted metadata
+    this.targets = opts.targets ?? [];
+    this.cveIds = opts.cveIds ?? [];
+    this.mitreAttack = opts.mitreAttack ?? [];
+    this.severity = opts.severity ?? '';
+    this.toolsUsed = opts.toolsUsed ?? [];
+    this.hashes = opts.hashes ?? [];
+
+    // General metadata
+    this.keywords = opts.keywords ?? [];
+    this.persons = opts.persons ?? [];
+    this.entities = opts.entities ?? [];
+    this.timestamp = opts.timestamp ?? '';
+    this.topic = opts.topic ?? '';
+
+    // Source tracking
+    this.sourceTurns = opts.sourceTurns ?? [];
+    this.engagementId = opts.engagementId ?? '';
+    this.sourceSkill = opts.sourceSkill ?? '';
+
+    this.createdAt = opts.createdAt ?? new Date().toISOString();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// DensityGate
+// ---------------------------------------------------------------------------
+
+/**
+ * Information density gate — filters out low-information dialogue turns.
+ *
+ * Scores based on: technical content, content length, lexical diversity.
+ */
+class DensityGate {
+  static LOW_INFO_PATTERNS = [
+    /^(?:ok|okay|sure|yes|no|thanks|thank you|got it|understood)[.!?]?$/i,
+    /^(?:let me|i'll|i will|going to)\s/i,
+    /^(?:here (?:is|are)|sure,?\s+here)/i,
+  ];
+
+  /**
+   * @param {number} minScore — minimum density score to include a turn (default 0.3)
+   */
+  constructor(minScore = 0.3) {
+    this.minScore = minScore;
+  }
+
+  /**
+   * Score information density of a dialogue turn (0.0 - 1.0).
+   * @param {DialogueTurn} turn
+   * @returns {number}
+   */
+  score(turn) {
+    const text = (turn.content || '').trim();
+    if (!text) return 0.0;
+
+    let score = 0.0;
+
+    // Length score
+    const length = text.length;
+    if (length > 200) {
+      score += 0.3;
+    } else if (length > 50) {
+      score += 0.15;
+    }
+
+    // Technical content indicators
+    if (_IP_RE.test(text)) score += 0.15;
+    _IP_RE.lastIndex = 0;
+    if (_CVE_RE.test(text)) score += 0.2;
+    _CVE_RE.lastIndex = 0;
+    if (_MITRE_RE.test(text)) score += 0.15;
+    _MITRE_RE.lastIndex = 0;
+    if (text.includes('```')) score += 0.2;
+    if (_HASH_SHA256_RE.test(text) || _HASH_MD5_RE.test(text)) score += 0.1;
+    _HASH_SHA256_RE.lastIndex = 0;
+    _HASH_MD5_RE.lastIndex = 0;
+
+    // Unique term ratio (lexical diversity)
+    const words = text.toLowerCase().split(/\s+/);
+    if (words.length > 0) {
+      const uniqueRatio = new Set(words).size / words.length;
+      score += uniqueRatio * 0.2;
+    }
+
+    // Penalize boilerplate
+    for (const pat of DensityGate.LOW_INFO_PATTERNS) {
+      if (pat.test(text)) {
+        score *= 0.3;
+        break;
+      }
+    }
+
+    // Tool output is almost always high-info
+    if (turn.role === 'tool' && turn.toolOutput) {
+      score = Math.max(score, 0.5);
+    }
+
+    return Math.min(score, 1.0);
+  }
+
+  /**
+   * @param {DialogueTurn} turn
+   * @returns {boolean}
+   */
+  shouldInclude(turn) {
+    return this.score(turn) >= this.minScore;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// SemanticCompressor
+// ---------------------------------------------------------------------------
+
+// Stop words for keyword extraction
+const _STOP_WORDS = new Set([
+  'the', 'and', 'for', 'that', 'this', 'with', 'from', 'have', 'has',
+  'been', 'were', 'was', 'are', 'will', 'would', 'could', 'should',
+  'into', 'about', 'which',
+]);
+
+/**
+ * Stage 1: Semantic Structured Compression.
+ *
+ * Processes dialogue windows through:
+ * 1. Density gating — filter low-info turns
+ * 2. Entity extraction — security-specific pattern matching
+ * 3. LLM compression (stubbed) or heuristic extraction
+ * 4. Atomic entry creation — self-contained memory units
+ */
+class SemanticCompressor {
+  /**
+   * @param {{ llmClient?: any, windowSize?: number, overlapSize?: number, densityThreshold?: number, model?: string }} opts
+   */
+  constructor(opts = {}) {
+    this.llmClient = opts.llmClient ?? null;
+    this.windowSize = opts.windowSize ?? 20;
+    this.overlapSize = opts.overlapSize ?? 2;
+    this.densityGate = new DensityGate(opts.densityThreshold ?? 0.3);
+    this.model = opts.model ?? 'claude-sonnet-4-20250514';
+    this._previousContext = '';
+  }
+
+  /**
+   * Compress dialogue turns into atomic memory entries.
+   * @param {DialogueTurn[]} turns
+   * @param {string} engagementId
+   * @param {string} sourceSkill
+   * @returns {Promise<CompressedEntry[]>}
+   */
+  async compress(turns, engagementId = '', sourceSkill = '') {
+    // Step 1: Density gating
+    const filtered = turns.filter((t) => this.densityGate.shouldInclude(t));
+    if (filtered.length === 0) return [];
+
+    // Step 2: Window processing
+    const allEntries = [];
+    const step = this.windowSize - this.overlapSize;
+    for (let i = 0; i < filtered.length; i += step) {
+      const window = filtered.slice(i, i + this.windowSize);
+      const entries = await this._processWindow(window, engagementId, sourceSkill);
+      allEntries.push(...entries);
+    }
+
+    return allEntries;
+  }
+
+  /**
+   * Process a single window of dialogue turns.
+   * @private
+   */
+  async _processWindow(window, engagementId, sourceSkill) {
+    const combinedText = window.map((t) => t.content).join(' ');
+    const entities = extractSecurityEntities(combinedText);
+
+    let entries;
+    if (this.llmClient !== null) {
+      entries = await this._llmCompress(window, entities);
+    } else {
+      entries = this._heuristicCompress(window, entities);
+    }
+
+    // Annotate all entries with engagement context
+    for (const entry of entries) {
+      entry.engagementId = engagementId;
+      entry.sourceSkill = sourceSkill;
+      entry.targets = entities.ips.concat(entities.domains);
+      entry.cveIds = entities.cves;
+      entry.mitreAttack = entities.mitre;
+      entry.hashes = entities.hashes_sha256.concat(entities.hashes_md5);
+    }
+
+    this._previousContext = this._buildContextSummary(entries);
+    return entries;
+  }
+
+  /**
+   * LLM compression — stubbed for S03 integration.
+   * Falls back to heuristic compression.
+   * @private
+   */
+  async _llmCompress(window, entities) {
+    // TODO: S03 LLM integration — call this.llmClient with extraction prompt
+    // For now, fall back to heuristic compression
+    return this._heuristicCompress(window, entities);
+  }
+
+  /**
+   * Fallback heuristic compression when no LLM is available.
+   * @private
+   */
+  _heuristicCompress(window, entities) {
+    const entries = [];
+
+    for (const turn of window) {
+      const sentences = turn.content.split(/[.!?\n]+/);
+      for (let sent of sentences) {
+        sent = sent.trim();
+        if (sent.length < 20) continue;
+
+        const sentEntities = extractSecurityEntities(sent);
+        const hasSecurityContent = Object.values(sentEntities).some(
+          (arr) => arr.length > 0,
+        );
+
+        if (
+          hasSecurityContent ||
+          this.densityGate.score(new DialogueTurn({ turnId: 0, role: turn.role, content: sent })) > 0.5
+        ) {
+          // Determine memory type from content
+          let memType = 'note';
+          if (sentEntities.cves.length > 0) {
+            memType = 'finding';
+          } else if (sentEntities.ips.length > 0 || sentEntities.domains.length > 0) {
+            memType = 'ioc';
+          } else if (sentEntities.mitre.length > 0) {
+            memType = 'ttp';
+          }
+
+          // Extract keywords (content words > 4 chars)
+          const words = sent.toLowerCase().split(/\s+/);
+          const keywords = words
+            .map((w) => w.replace(/[.,;:!?()[\]{}"']/g, ''))
+            .filter((w) => w.length > 4 && !_STOP_WORDS.has(w))
+            .slice(0, 10);
+
+          entries.push(
+            new CompressedEntry({
+              losslessRestatement: sent,
+              memoryType: memType,
+              keywords,
+              sourceTurns: [turn.turnId],
+              timestamp: turn.timestamp || new Date().toISOString(),
+            }),
+          );
+        }
+      }
+    }
+
+    return entries;
+  }
+
+  /**
+   * Format a window of turns into human-readable text.
+   * @private
+   */
+  _formatWindow(window) {
+    const lines = [];
+    for (const turn of window) {
+      const ts = turn.timestamp ? ` [${turn.timestamp}]` : '';
+      let prefix = turn.role.toUpperCase();
+      if (turn.toolName) prefix = `TOOL(${turn.toolName})`;
+      lines.push(`${prefix}${ts}: ${turn.content}`);
+      if (turn.toolOutput) {
+        lines.push(`  OUTPUT: ${turn.toolOutput.slice(0, 500)}`);
+      }
+    }
+    return lines.join('\n');
+  }
+
+  /**
+   * Build extraction prompt for LLM compression.
+   * @private
+   */
+  _buildExtractionPrompt(dialogueText, entities) {
+    let contextSection = '';
+    if (this._previousContext) {
+      contextSection = `## Previous Context (avoid duplication):\n${this._previousContext}\n\n---\n`;
+    }
+
+    let entityHints = '';
+    if (Object.values(entities).some((arr) => arr.length > 0)) {
+      const parts = [];
+      if (entities.ips.length > 0) parts.push(`IPs: ${entities.ips.join(', ')}`);
+      if (entities.cves.length > 0) parts.push(`CVEs: ${entities.cves.join(', ')}`);
+      if (entities.mitre.length > 0) parts.push(`MITRE: ${entities.mitre.join(', ')}`);
+      if (entities.domains.length > 0) parts.push(`Domains: ${entities.domains.slice(0, 10).join(', ')}`);
+      entityHints = `\n## Detected Entities:\n${parts.join('\n')}\n`;
+    }
+
+    return `${contextSection}${entityHints}\n## Engagement Dialogue:\n${dialogueText}`;
+  }
+
+  /**
+   * Build summary of recently compressed entries for context tracking.
+   * @private
+   */
+  _buildContextSummary(entries) {
+    if (entries.length === 0) return '';
+    return entries.slice(-5).map((e) => `- ${e.losslessRestatement}`).join('\n');
+  }
+}
+
+export {
+  extractSecurityEntities,
+  DialogueTurn,
+  CompressedEntry,
+  DensityGate,
+  SemanticCompressor,
+};