cipher-security 2.0.0

package/lib/memory/retriever.js

@@ -0,0 +1,515 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * CIPHER Memory — Stage 3: Intent-Aware Retrieval Planning
+ *
+ * Intelligent retrieval that understands query intent and adapts strategy:
+ * - Query complexity analysis → determines retrieval depth
+ * - Multi-query decomposition → sub-queries for complex questions
+ * - Hybrid search fusion → lexical + symbolic via RRF
+ * - Reflection-based refinement → iterative retrieval for complex queries
+ *
+ * Ported from Python memory/core/retriever.py.
+ */
+
+import { extractSecurityEntities } from './compressor.js';
+
+// ---------------------------------------------------------------------------
+// Query Intent Classification
+// ---------------------------------------------------------------------------
+
+/**
+ * Security query intent categories with retrieval strategy mapping.
+ */
+const QueryIntent = {
+  // Intent → [description, searchDepth, requiresSymbolic]
+  INTENTS: {
+    finding_lookup: ['Looking up specific vulnerability or finding', 'shallow', true],
+    ioc_search: ['Searching for indicators of compromise', 'shallow', true],
+    ttp_mapping: ['Mapping techniques, tactics, procedures', 'medium', true],
+    engagement_context: ['Getting context for an engagement', 'deep', true],
+    temporal_query: ['Time-based query about when something happened', 'medium', true],
+    comparative: ['Comparing findings, hosts, or engagements', 'deep', false],
+    causal: ['Understanding why or how something happened', 'deep', false],
+    general: ['General security knowledge query', 'shallow', false],
+  },
+
+  // Keyword patterns for intent detection
+  PATTERNS: {
+    finding_lookup: [
+      'vulnerability', 'vuln', 'finding', 'cve-', 'exploit', 'weakness',
+      'injection', 'xss', 'sqli', 'rce', 'ssrf', 'csrf',
+    ],
+    ioc_search: [
+      'ioc', 'indicator', 'hash', 'ip address', 'domain', 'c2', 'beacon',
+      'callback', 'malware', 'sample',
+    ],
+    ttp_mapping: [
+      'technique', 'tactic', 'procedure', 'mitre', 'att&ck', 't1',
+      'lateral', 'persistence', 'privilege',
+    ],
+    engagement_context: [
+      'engagement', 'pentest', 'assessment', 'audit', 'scope', 'client',
+      'target', 'progress', 'status',
+    ],
+    temporal_query: [
+      'when', 'timeline', 'first', 'last', 'before', 'after', 'during',
+      'recently', 'latest', 'oldest',
+    ],
+    comparative: [
+      'compare', 'difference', 'between', 'versus', 'vs', 'similar',
+      'different', 'common',
+    ],
+    causal: [
+      'why', 'how', 'because', 'cause', 'root cause', 'reason', 'led to',
+      'resulted in', 'impact',
+    ],
+  },
+};
+
+// ---------------------------------------------------------------------------
+// RetrievalPlan
+// ---------------------------------------------------------------------------
+
+/**
+ * Structured retrieval plan for a query.
+ */
+class RetrievalPlan {
+  constructor(opts = {}) {
+    this.intent = opts.intent ?? 'general';
+    this.complexity = opts.complexity ?? 0.3;
+    this.subQueries = opts.subQueries ?? [];
+    this.requiredFilters = opts.requiredFilters ?? {};
+    this.searchDepth = opts.searchDepth ?? 'shallow';
+    this.needsReflection = opts.needsReflection ?? false;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// IntentClassifier
+// ---------------------------------------------------------------------------
+
+/**
+ * Classify security query intent using keyword matching.
+ */
+class IntentClassifier {
+  /**
+   * Return the most likely intent for a query.
+   * @param {string} query
+   * @returns {string}
+   */
+  classify(query) {
+    const queryLower = query.toLowerCase();
+    const scores = {};
+
+    for (const [intent, keywords] of Object.entries(QueryIntent.PATTERNS)) {
+      let score = 0;
+      for (const kw of keywords) {
+        if (queryLower.includes(kw)) score++;
+      }
+      if (score > 0) scores[intent] = score;
+    }
+
+    if (Object.keys(scores).length === 0) return 'general';
+
+    // Return intent with highest score
+    let maxIntent = 'general';
+    let maxScore = 0;
+    for (const [intent, score] of Object.entries(scores)) {
+      if (score > maxScore) {
+        maxScore = score;
+        maxIntent = intent;
+      }
+    }
+    return maxIntent;
+  }
+
+  /**
+   * Estimate query complexity (0.0 = trivial, 1.0 = very complex).
+   * @param {string} query
+   * @returns {number}
+   */
+  estimateComplexity(query) {
+    let score = 0.0;
+
+    // Length-based
+    const words = query.split(/\s+/);
+    if (words.length > 20) {
+      score += 0.2;
+    } else if (words.length > 10) {
+      score += 0.1;
+    }
+
+    // Multi-hop indicators
+    const multiHop = ['and', 'also', 'then', 'after that', 'in addition'];
+    const queryLower = query.toLowerCase();
+    for (const kw of multiHop) {
+      if (queryLower.includes(kw)) score += 0.1;
+    }
+
+    // Comparative/causal is inherently complex
+    if (['compare', 'why', 'how did'].some((w) => queryLower.includes(w))) {
+      score += 0.3;
+    }
+
+    // Multiple entity references
+    const entities = extractSecurityEntities(query);
+    const entityCount = Object.values(entities).reduce((sum, arr) => sum + arr.length, 0);
+    if (entityCount > 3) {
+      score += 0.2;
+    } else if (entityCount > 1) {
+      score += 0.1;
+    }
+
+    return Math.min(score, 1.0);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// QueryDecomposer
+// ---------------------------------------------------------------------------
+
+/**
+ * Decompose complex queries into targeted sub-queries.
+ */
+class QueryDecomposer {
+  /**
+   * Break a complex query into focused sub-queries.
+   * @param {string} query
+   * @param {string} intent
+   * @returns {string[]}
+   */
+  decompose(query, intent) {
+    let subQueries = [query];
+
+    // Split on conjunctions
+    if (/\s+and\s+/i.test(query)) {
+      const parts = query.split(/\s+and\s+/i);
+      if (parts.length > 1 && parts.every((p) => p.split(/\s+/).length > 3)) {
+        subQueries = parts;
+      }
+    } else if (intent === 'comparative') {
+      // For comparative queries, search each entity separately
+      const entities = extractSecurityEntities(query);
+      const targets = [
+        ...entities.ips,
+        ...entities.domains,
+        ...entities.cves,
+      ];
+      if (targets.length >= 2) {
+        subQueries = targets.slice(0, 3).map((t) => `${query} ${t}`);
+      }
+    } else if (intent === 'causal') {
+      // For causal queries, search for both the event and the cause
+      subQueries = [query];
+      const stripped = query
+        .replace(/\b(why|how|because|cause)\b/gi, '')
+        .trim();
+      if (stripped !== query && stripped.length > 10) {
+        subQueries.push(stripped);
+      }
+    }
+
+    return subQueries.slice(0, 4); // Max 4 sub-queries
+  }
+}
+
+// ---------------------------------------------------------------------------
+// AdaptiveRetriever
+// ---------------------------------------------------------------------------
+
+/** Depth → top_k mapping */
+const DEPTH_MAP = { shallow: 5, medium: 15, deep: 30 };
+
+/**
+ * Stage 3: Intent-Aware Adaptive Retrieval.
+ *
+ * Combines intent classification, query decomposition, and hybrid search
+ * with optional reflection-based iterative refinement.
+ */
+class AdaptiveRetriever {
+  /**
+   * @param {import('./engine.js').CipherMemory} memory
+   * @param {{ enableReflection?: boolean, maxReflectionRounds?: number }} opts
+   */
+  constructor(memory, opts = {}) {
+    this.memory = memory;
+    this.classifier = new IntentClassifier();
+    this.decomposer = new QueryDecomposer();
+    this.enableReflection = opts.enableReflection ?? true;
+    this.maxReflectionRounds = opts.maxReflectionRounds ?? 2;
+  }
+
+  /**
+   * Create a retrieval plan for the query.
+   * @param {string} query
+   * @returns {RetrievalPlan}
+   */
+  plan(query) {
+    const intent = this.classifier.classify(query);
+    const complexity = this.classifier.estimate_complexity
+      ? this.classifier.estimateComplexity(query)
+      : this.classifier.estimateComplexity(query);
+
+    const intentInfo = QueryIntent.INTENTS[intent] ?? ['', 'shallow', false];
+    const [, depth] = intentInfo;
+
+    const subQueries =
+      complexity > 0.5
+        ? this.decomposer.decompose(query, intent)
+        : [query];
+
+    // Extract filter hints from query
+    const filters = {};
+    const entities = extractSecurityEntities(query);
+    if (entities.cves.length > 0) filters.cve_id = entities.cves[0];
+    if (entities.mitre.length > 0) filters.mitre_technique = entities.mitre[0];
+
+    // Engagement ID extraction (e.g., "engagement ENG-001")
+    const engMatch = query.match(/engagement\s+([A-Za-z0-9-]+)/i);
+    if (engMatch) filters.engagement_id = engMatch[1];
+
+    return new RetrievalPlan({
+      intent,
+      complexity,
+      subQueries,
+      requiredFilters: filters,
+      searchDepth: depth,
+      needsReflection: complexity > 0.6 && this.enableReflection,
+    });
+  }
+
+  /**
+   * Retrieve relevant memories with intent-aware planning.
+   * @param {string} query
+   * @param {string} engagementId
+   * @param {number} [limit]
+   * @returns {import('./engine.js').MemoryEntry[]}
+   */
+  retrieve(query, engagementId = '', limit = null) {
+    const plan = this.plan(query);
+
+    if (engagementId) {
+      plan.requiredFilters.engagement_id = engagementId;
+    }
+
+    const topK = limit ?? DEPTH_MAP[plan.searchDepth] ?? 10;
+
+    // Execute sub-queries and merge results
+    const allResults = [];
+    const seenIds = new Set();
+
+    for (const subQuery of plan.subQueries) {
+      const results = this.memory.search(subQuery, {
+        engagementId: plan.requiredFilters.engagement_id ?? '',
+        memoryType: plan.requiredFilters.memory_type ?? '',
+        severity: plan.requiredFilters.severity ?? '',
+        mitreTechnique: plan.requiredFilters.mitre_technique ?? '',
+      }, topK);
+
+      for (const entry of results) {
+        if (!seenIds.has(entry.entryId)) {
+          seenIds.add(entry.entryId);
+          allResults.push(entry);
+        }
+      }
+    }
+
+    // Reflection: check if we have enough relevant results
+    if (plan.needsReflection && allResults.length < 3) {
+      return this._reflectAndRetry(query, plan, allResults, seenIds, topK);
+    }
+
+    return allResults.slice(0, topK);
+  }
+
+  /**
+   * Iterative refinement when initial results are insufficient.
+   * @private
+   */
+  _reflectAndRetry(query, plan, currentResults, seenIds, topK) {
+    for (let round = 0; round < this.maxReflectionRounds; round++) {
+      const altQueries = this._generateAlternativeQueries(query, currentResults);
+
+      for (const altQuery of altQueries) {
+        const results = this.memory.search(altQuery, {
+          engagementId: plan.requiredFilters.engagement_id ?? '',
+        }, topK);
+
+        for (const entry of results) {
+          if (!seenIds.has(entry.entryId)) {
+            seenIds.add(entry.entryId);
+            currentResults.push(entry);
+          }
+        }
+      }
+
+      if (currentResults.length >= 3) break;
+    }
+
+    return currentResults;
+  }
+
+  /**
+   * Generate alternative search queries based on current results.
+   * @private
+   */
+  _generateAlternativeQueries(originalQuery, currentResults) {
+    const alternatives = [];
+
+    // Try broader query (remove specifics)
+    let broader = originalQuery.replace(/CVE-\d+-\d+/g, '').trim();
+    broader = broader.replace(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, '').trim();
+    if (broader && broader !== originalQuery) {
+      alternatives.push(broader);
+    }
+
+    // Try extracting key terms
+    const words = originalQuery.split(/\s+/);
+    const skipWords = new Set([
+      'what', 'where', 'when', 'which', 'about', 'find', 'search', 'look',
+      'show', 'there',
+    ]);
+    const contentWords = words.filter(
+      (w) => w.length > 4 && !skipWords.has(w.toLowerCase()),
+    );
+    if (contentWords.length > 0) {
+      alternatives.push(contentWords.slice(0, 5).join(' '));
+    }
+
+    // Try using tags from current results
+    if (currentResults.length > 0) {
+      for (const entry of currentResults.slice(0, 2)) {
+        if (entry.tags && entry.tags.length > 0) {
+          alternatives.push(entry.tags.slice(0, 3).join(' '));
+        }
+      }
+    }
+
+    return alternatives.slice(0, 3);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// ContextBuilder — Token-budgeted context assembly
+// ---------------------------------------------------------------------------
+
+/**
+ * Build token-budgeted context from retrieved memories.
+ *
+ * Assembles memories into a structured context block suitable for
+ * injection into LLM prompts, respecting token budget constraints.
+ */
+class ContextBuilder {
+  /**
+   * @param {{ maxTokens?: number, charsPerToken?: number }} opts
+   */
+  constructor(opts = {}) {
+    this.maxTokens = opts.maxTokens ?? 4000;
+    this.charsPerToken = opts.charsPerToken ?? 4.0;
+  }
+
+  /**
+   * Build structured context from memory entries.
+   * Returns a formatted string suitable for system prompt injection.
+   * @param {import('./engine.js').MemoryEntry[]} entries
+   * @param {string} query
+   * @returns {string}
+   */
+  build(entries, query = '') {
+    if (!entries || entries.length === 0) return '';
+
+    const maxChars = Math.floor(this.maxTokens * this.charsPerToken);
+    const sections = {
+      findings: [],
+      iocs: [],
+      ttps: [],
+      decisions: [],
+      context: [],
+    };
+
+    for (const entry of entries) {
+      let line = `- ${entry.content}`;
+      if (entry.severity) {
+        line = `- [${entry.severity.toUpperCase()}] ${entry.content}`;
+      }
+      if (entry.mitreAttack && entry.mitreAttack.length > 0) {
+        const attacks = entry.mitreAttack.slice(0, 3).join(', ');
+        line += ` (${attacks})`;
+      }
+
+      // Route to section based on memoryType
+      // memoryType is a string value from MemoryType enum
+      const memType = typeof entry.memoryType === 'object' && entry.memoryType.value
+        ? entry.memoryType.value
+        : entry.memoryType;
+
+      if (memType === 'finding') {
+        sections.findings.push(line);
+      } else if (memType === 'ioc') {
+        sections.iocs.push(line);
+      } else if (memType === 'ttp') {
+        sections.ttps.push(line);
+      } else if (memType === 'decision') {
+        sections.decisions.push(line);
+      } else {
+        sections.context.push(line);
+      }
+    }
+
+    // Assemble within budget
+    const parts = ['## Engagement Memory'];
+    let currentChars = parts[0].length;
+
+    const sectionLabels = {
+      findings: '### Findings',
+      iocs: '### Indicators of Compromise',
+      ttps: '### Techniques & Procedures',
+      decisions: '### Decisions',
+      context: '### Context',
+    };
+
+    for (const [key, label] of Object.entries(sectionLabels)) {
+      const items = sections[key];
+      if (items.length === 0) continue;
+
+      const sectionText = `\n${label}\n${items.join('\n')}`;
+      if (currentChars + sectionText.length > maxChars) {
+        // Truncate items to fit
+        const remaining = maxChars - currentChars - (`\n${label}\n`).length;
+        if (remaining > 100) {
+          const truncated = [];
+          let charsUsed = 0;
+          for (const item of items) {
+            if (charsUsed + item.length + 1 > remaining) break;
+            truncated.push(item);
+            charsUsed += item.length + 1;
+          }
+          if (truncated.length > 0) {
+            parts.push(`\n${label}`);
+            parts.push(...truncated);
+          }
+        }
+        break;
+      }
+
+      parts.push(`\n${label}`);
+      parts.push(...items);
+      currentChars += sectionText.length;
+    }
+
+    return parts.join('\n');
+  }
+}
+
+export {
+  QueryIntent,
+  RetrievalPlan,
+  IntentClassifier,
+  QueryDecomposer,
+  AdaptiveRetriever,
+  ContextBuilder,
+  DEPTH_MAP,
+};