cipher-security 2.0.0

package/lib/memory/evolution.js

@@ -0,0 +1,668 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * CIPHER Evolution Engine — Self-Improving Skill System
+ *
+ * Inspired by MetaClaw's RL-driven skill evolution:
+ * - PRM Scorer: Evaluates CIPHER outputs for quality (+1/-1/0)
+ * - Skill Evolver: Generates new skills from failed engagements
+ * - Failure Pipeline: Captures failures → analyzes patterns → creates skills
+ *
+ * Ported from Python memory/core/evolution.py.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, writeFileSync, appendFileSync, chmodSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { extractSecurityEntities } from './compressor.js';
+
+// ---------------------------------------------------------------------------
+// ScoredResponse
+// ---------------------------------------------------------------------------
+
+/**
+ * A scored engagement response.
+ */
+class ScoredResponse {
+  constructor(opts = {}) {
+    this.responseId = opts.responseId ?? randomUUID();
+    this.query = opts.query ?? '';
+    this.response = opts.response ?? '';
+    this.mode = opts.mode ?? '';
+    this.skillUsed = opts.skillUsed ?? '';
+    this.score = opts.score ?? 0.0;
+    this.votes = opts.votes ?? [];
+    this.feedback = opts.feedback ?? '';
+    this.timestamp = opts.timestamp ?? new Date().toISOString();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// ResponseScorer
+// ---------------------------------------------------------------------------
+
+/**
+ * Process Reward Model (PRM) scorer for CIPHER outputs.
+ *
+ * Evaluates response quality using:
+ * 1. Structural checks (required sections present, proper format)
+ * 2. Content checks (actionable, specific, not generic)
+ * 3. Domain checks (correct ATT&CK refs, valid CVE format, proper severity)
+ *
+ * Returns score: +1 (helpful), -1 (unhelpful), 0 (ambiguous)
+ */
+class ResponseScorer {
+  // Required structural elements by mode
+  static MODE_REQUIREMENTS = {
+    RED: ['ATT&CK', 'DETECTION OPPORTUNITIES'],
+    BLUE: ['Sigma', 'detection', 'rule'],
+    INCIDENT: ['Triage', 'Containment', 'Evidence'],
+    PRIVACY: ['GDPR', 'data flow', 'DPIA'],
+    RECON: ['source', 'confidence'],
+    ARCHITECT: ['threat model', 'trust boundary'],
+    PURPLE: ['detection coverage', 'gap'],
+  };
+
+  // Quality signals
+  static POSITIVE_SIGNALS = [
+    /CVE-\d{4}-\d{4,}/,           // Specific CVE reference
+    /T\d{4}(?:\.\d{3})?/,         // MITRE ATT&CK ID
+    /CIS\s+\d+\.\d+/,             // CIS Control reference
+    /NIST\s+\d{3}-\d{2,3}/,       // NIST SP reference
+    /```/,                         // Code blocks (showing work)
+    /\[CONFIRMED\]|\[INFERRED\]/, // Confidence tags
+    /Severity\s*:\s*\w+/,         // Finding format
+  ];
+
+  static NEGATIVE_SIGNALS = [
+    /I (?:can't|cannot|am unable to)/i,             // Refusals
+    /as an AI/i,                                     // Disclaimers
+    /I don't have (?:access|information)/i,          // Hedging
+    /consult (?:a |your )(?:professional|expert)/i,  // Deflection
+  ];
+
+  /**
+   * @param {{ llmClient?: any, prmVotes?: number }} opts
+   */
+  constructor(opts = {}) {
+    this.llmClient = opts.llmClient ?? null;
+    this.prmVotes = opts.prmVotes ?? 3;
+  }
+
+  /**
+   * Score a CIPHER response.
+   * Uses heuristic scoring (always available) + optional LLM judge.
+   * @param {string} query
+   * @param {string} response
+   * @param {string} mode
+   * @param {string} skillUsed
+   * @returns {ScoredResponse}
+   */
+  score(query, response, mode = '', skillUsed = '') {
+    const votes = [];
+
+    // Vote 1: Structural quality
+    votes.push(this._scoreStructural(response, mode));
+
+    // Vote 2: Content quality
+    votes.push(this._scoreContent(response, query));
+
+    // Vote 3: Domain accuracy
+    votes.push(this._scoreDomain(response, mode));
+
+    // Majority vote
+    const final = ResponseScorer._majorityVote(votes);
+
+    return new ScoredResponse({
+      query,
+      response,
+      mode,
+      skillUsed,
+      score: final,
+      votes,
+      feedback: this._generateFeedback(votes, response, mode),
+    });
+  }
+
+  /**
+   * Check if response has required structural elements.
+   * @private
+   */
+  _scoreStructural(response, mode) {
+    if (!mode || !(mode in ResponseScorer.MODE_REQUIREMENTS)) {
+      // General structural check
+      const hasStructure = ['```', '##', '- ', '1.', '|'].some(
+        (marker) => response.includes(marker),
+      );
+      return hasStructure && response.length > 100 ? 1.0 : 0.0;
+    }
+
+    const requirements = ResponseScorer.MODE_REQUIREMENTS[mode];
+    let matched = 0;
+    for (const req of requirements) {
+      if (response.toLowerCase().includes(req.toLowerCase())) matched++;
+    }
+    const ratio = matched / requirements.length;
+
+    if (ratio >= 0.7) return 1.0;
+    if (ratio >= 0.3) return 0.0;
+    return -1.0;
+  }
+
+  /**
+   * Check if response is actionable and specific.
+   * @private
+   */
+  _scoreContent(response, query) {
+    let score = 0.0;
+
+    // Positive signals
+    for (const pattern of ResponseScorer.POSITIVE_SIGNALS) {
+      if (pattern.test(response)) score += 0.3;
+    }
+
+    // Negative signals
+    for (const pattern of ResponseScorer.NEGATIVE_SIGNALS) {
+      if (pattern.test(response)) score -= 0.5;
+    }
+
+    // Length check
+    if (response.length < 50) {
+      score -= 0.5;
+    } else if (response.length > 200) {
+      score += 0.1;
+    }
+
+    // Specificity: contains concrete values
+    const entities = extractSecurityEntities(response);
+    const entityCount = Object.values(entities).reduce(
+      (sum, arr) => sum + arr.length,
+      0,
+    );
+    if (entityCount > 0) score += 0.2;
+
+    if (score > 0.3) return 1.0;
+    if (score < -0.3) return -1.0;
+    return 0.0;
+  }
+
+  /**
+   * Check domain-specific accuracy.
+   * @private
+   */
+  _scoreDomain(response, mode) {
+    if (!mode) return 0.0;
+
+    let score = 0.0;
+
+    // Check for valid ATT&CK references
+    const mitreRefs = response.match(/T\d{4}(?:\.\d{3})?/g) || [];
+    if (mitreRefs.length > 0) {
+      const valid = mitreRefs.every((ref) => {
+        const numMatch = ref.match(/(\d{4})/);
+        if (!numMatch) return false;
+        const num = parseInt(numMatch[1], 10);
+        return num >= 1001 && num <= 1999;
+      });
+      score += valid ? 0.3 : -0.2;
+    }
+
+    // Check for valid CVE format
+    const cves = [...response.matchAll(/CVE-(\d{4})-(\d+)/g)];
+    if (cves.length > 0) {
+      const valid = cves.every(([, year]) => {
+        const y = parseInt(year, 10);
+        return y >= 1999 && y <= 2030;
+      });
+      score += valid ? 0.2 : -0.3;
+    }
+
+    // Check Sigma rule format (if BLUE mode)
+    if (mode === 'BLUE' && response.toLowerCase().includes('detection:')) {
+      const hasTitle = /title:/i.test(response);
+      const hasLogsource = /logsource:/i.test(response);
+      const hasCondition = /condition:/i.test(response);
+      if (hasTitle && hasLogsource && hasCondition) {
+        score += 0.5;
+      }
+    }
+
+    if (score > 0.2) return 1.0;
+    if (score < -0.2) return -1.0;
+    return 0.0;
+  }
+
+  /**
+   * Majority vote from multiple scores.
+   * @param {number[]} votes
+   * @returns {number}
+   */
+  static _majorityVote(votes) {
+    if (!votes || votes.length === 0) return 0.0;
+
+    // Count occurrences
+    const counter = new Map();
+    for (const v of votes) {
+      counter.set(v, (counter.get(v) ?? 0) + 1);
+    }
+
+    // Find most common
+    let topValue = 0.0;
+    let topCount = 0;
+    for (const [value, count] of counter) {
+      if (count > topCount) {
+        topCount = count;
+        topValue = value;
+      }
+    }
+
+    // If tied, return 0.0 (ambiguous)
+    let tiedCount = 0;
+    for (const count of counter.values()) {
+      if (count === topCount) tiedCount++;
+    }
+    if (tiedCount > 1) return 0.0;
+
+    return topValue;
+  }
+
+  /**
+   * Generate human-readable feedback from scoring.
+   * @private
+   */
+  _generateFeedback(votes, response, mode) {
+    const issues = [];
+
+    if (votes[0] < 0) {
+      issues.push(`Missing required structural elements for ${mode} mode`);
+    }
+    if (votes[1] < 0) {
+      issues.push('Response lacks specificity or contains deflections');
+    }
+    if (votes.length > 2 && votes[2] < 0) {
+      issues.push('Domain-specific references may be inaccurate');
+    }
+
+    if (issues.length === 0) return 'Response meets quality standards';
+    return issues.join('; ');
+  }
+}
+
+// ---------------------------------------------------------------------------
+// EvolutionRecord
+// ---------------------------------------------------------------------------
+
+class EvolutionRecord {
+  constructor(opts = {}) {
+    this.timestamp = opts.timestamp ?? '';
+    this.failuresAnalyzed = opts.failuresAnalyzed ?? 0;
+    this.skillsGenerated = opts.skillsGenerated ?? 0;
+    this.skillNames = opts.skillNames ?? [];
+    this.failurePatterns = opts.failurePatterns ?? [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// SkillEvolver
+// ---------------------------------------------------------------------------
+
+/**
+ * Generates new CIPHER skills from failed engagement patterns.
+ *
+ * When CIPHER fails a task (score <= 0), the Evolver:
+ * 1. Collects failed responses
+ * 2. Analyzes failure patterns
+ * 3. Generates new skill files to handle similar cases
+ * 4. Writes to skills directory for immediate use
+ */
+class SkillEvolver {
+  // Common failure patterns → skill templates
+  static FAILURE_TEMPLATES = {
+    missing_detection_rule: {
+      name: 'writing-detection-rules-for-{technique}',
+      domain: 'detection-engineering',
+      description: 'Generate Sigma/KQL/SPL detection rules for {technique}',
+      template: 'detection',
+    },
+    incomplete_finding: {
+      name: 'complete-vulnerability-assessment-{target_type}',
+      domain: 'vulnerability-management',
+      description: 'Thorough vulnerability assessment with proof-of-concept for {target_type}',
+      template: 'finding',
+    },
+    missing_mitre_mapping: {
+      name: 'mapping-{technique}-to-attack',
+      domain: 'purple-team',
+      description: 'Map observed technique to MITRE ATT&CK framework with detection',
+      template: 'mapping',
+    },
+    weak_remediation: {
+      name: 'remediation-guidance-{vuln_class}',
+      domain: 'vulnerability-management',
+      description: 'Specific remediation steps with verification for {vuln_class}',
+      template: 'remediation',
+    },
+  };
+
+  /**
+   * @param {{ skillsDir: string, llmClient?: any, maxNewSkills?: number, historyPath?: string }} opts
+   */
+  constructor(opts = {}) {
+    this.skillsDir = opts.skillsDir ?? '';
+    this.llmClient = opts.llmClient ?? null;
+    this.maxNewSkills = opts.maxNewSkills ?? 3;
+    this.historyPath = opts.historyPath ?? null;
+    /** @type {EvolutionRecord[]} */
+    this.evolutionHistory = [];
+  }
+
+  /**
+   * Check if failure rate warrants skill evolution.
+   * @param {ScoredResponse[]} scoredResponses
+   * @param {number} threshold
+   * @returns {boolean}
+   */
+  shouldEvolve(scoredResponses, threshold = 0.4) {
+    if (!scoredResponses || scoredResponses.length === 0) return false;
+    const successes = scoredResponses.filter((s) => s.score > 0).length;
+    const rate = successes / scoredResponses.length;
+    return rate < threshold;
+  }
+
+  /**
+   * Analyze failures and generate new skills.
+   * @param {ScoredResponse[]} failedResponses
+   * @param {boolean} dryRun — If true, don't write files
+   * @returns {object[]} Generated skill metadata
+   */
+  evolve(failedResponses, dryRun = false) {
+    if (!failedResponses || failedResponses.length === 0) return [];
+
+    // Analyze failure patterns
+    const patterns = this._analyzeFailures(failedResponses);
+
+    // Generate skills for each pattern
+    const generated = [];
+    for (const [pattern, context] of patterns.slice(0, this.maxNewSkills)) {
+      const skillMeta = this._generateSkill(pattern, context);
+      if (skillMeta && !dryRun) {
+        this._writeSkill(skillMeta);
+      }
+      if (skillMeta) generated.push(skillMeta);
+    }
+
+    // Record evolution
+    const record = new EvolutionRecord({
+      timestamp: new Date().toISOString(),
+      failuresAnalyzed: failedResponses.length,
+      skillsGenerated: generated.length,
+      skillNames: generated.map((s) => s.name || ''),
+      failurePatterns: patterns.map(([p]) => p),
+    });
+    this.evolutionHistory.push(record);
+
+    if (this.historyPath) {
+      this._persistHistory(record);
+    }
+
+    return generated;
+  }
+
+  /**
+   * Identify failure patterns from scored responses.
+   * @private
+   */
+  _analyzeFailures(responses) {
+    const patterns = [];
+
+    for (const resp of responses) {
+      const context = {
+        query: resp.query.slice(0, 200),
+        mode: resp.mode,
+        feedback: resp.feedback,
+        skill_used: resp.skillUsed,
+      };
+
+      // Pattern detection
+      if (resp.mode === 'BLUE' && resp.feedback.toLowerCase().includes('detection')) {
+        patterns.push(['missing_detection_rule', context]);
+      } else if (resp.feedback.toLowerCase().includes('specificity')) {
+        patterns.push(['incomplete_finding', context]);
+      } else if (resp.feedback.toLowerCase().includes('structural') && resp.mode) {
+        patterns.push(['missing_mitre_mapping', context]);
+      } else {
+        patterns.push(['weak_remediation', context]);
+      }
+    }
+
+    // Deduplicate patterns
+    const seen = new Set();
+    const unique = [];
+    for (const [pattern, ctx] of patterns) {
+      const key = `${pattern}:${ctx.mode || ''}`;
+      if (!seen.has(key)) {
+        seen.add(key);
+        unique.push([pattern, ctx]);
+      }
+    }
+
+    return unique;
+  }
+
+  /**
+   * Generate a skill from a failure pattern.
+   * @private
+   */
+  _generateSkill(pattern, context) {
+    const template = SkillEvolver.FAILURE_TEMPLATES[pattern];
+    if (!template) return null;
+
+    const technique = this._extractTechnique(context);
+    const name = template.name
+      .replace('{technique}', technique)
+      .replace('{target_type}', technique)
+      .replace('{vuln_class}', technique);
+    const description = template.description
+      .replace('{technique}', technique)
+      .replace('{target_type}', technique)
+      .replace('{vuln_class}', technique);
+
+    return {
+      name,
+      domain: template.domain,
+      description,
+      mode: context.mode || '',
+      generated_from: pattern,
+      source_query: context.query || '',
+    };
+  }
+
+  /**
+   * Extract technique name from failure context.
+   * @private
+   */
+  _extractTechnique(context) {
+    const query = context.query || '';
+
+    // Try to find MITRE technique
+    const mitre = query.match(/T\d{4}(?:\.\d{3})?/g);
+    if (mitre) return mitre[0].toLowerCase();
+
+    // Try to find CVE
+    const cves = query.match(/CVE-\d{4}-\d+/g);
+    if (cves) return cves[0].toLowerCase();
+
+    // Fall back to mode-specific default
+    const modeDefaults = {
+      RED: 'attack-technique',
+      BLUE: 'detection-gap',
+      INCIDENT: 'incident-pattern',
+      PRIVACY: 'data-exposure',
+    };
+    return modeDefaults[context.mode] || 'security-pattern';
+  }
+
+  /**
+   * Write generated skill to filesystem.
+   * @private
+   */
+  _writeSkill(meta) {
+    const name = meta.name;
+    const domain = meta.domain;
+    const skillDir = join(this.skillsDir, domain, 'techniques', name);
+    mkdirSync(skillDir, { recursive: true });
+
+    // SKILL.md
+    const skillMd = `<!-- Copyright (c) 2026 defconxt. All rights reserved. -->
+<!-- Licensed under AGPL-3.0 — see LICENSE file for details. -->
+<!-- CIPHER is a trademark of defconxt. -->
+
+---
+name: ${name}
+description: ${meta.description}
+domain: cybersecurity
+subdomain: ${domain}
+tags:
+  - auto-generated
+  - evolution-engine
+version: "1.0"
+---
+
+# ${name.replace(/-/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())}
+
+> Auto-generated by CIPHER Evolution Engine from engagement failure analysis.
+> Pattern: ${meta.generated_from || 'unknown'}
+> Source: ${(meta.source_query || 'N/A').slice(0, 100)}
+
+## Quick Reference
+
+\`\`\`bash
+node scripts/agent.js analyze --target <scope>
+\`\`\`
+
+## Workflow
+
+1. **Identify** — Recognize the pattern: ${meta.description}
+2. **Analyze** — Gather context and assess scope
+3. **Execute** — Apply technique with proper tooling
+4. **Verify** — Confirm results and document findings
+5. **Report** — Generate structured output with evidence
+
+## Verification
+
+- [ ] Output matches expected format
+- [ ] All references are valid (CVE, MITRE ATT&CK)
+- [ ] Remediation steps are actionable
+- [ ] Detection opportunities documented
+
+## References
+
+- MITRE ATT&CK Framework
+- NIST Cybersecurity Framework
+- OWASP Testing Guide
+`;
+    writeFileSync(join(skillDir, 'SKILL.md'), skillMd);
+
+    // scripts/agent.js
+    const scriptsDir = join(skillDir, 'scripts');
+    mkdirSync(scriptsDir, { recursive: true });
+    const agentJs = `#!/usr/bin/env node
+/**
+ * Agent script for ${name} — auto-generated by CIPHER Evolution Engine.
+ */
+
+import { parseArgs } from 'node:util';
+
+const { values } = parseArgs({
+  options: {
+    target: { type: 'string', short: 't' },
+    command: { type: 'string', short: 'c', default: 'analyze' },
+  },
+});
+
+if (values.command === 'analyze') {
+  const result = {
+    skill: '${name}',
+    domain: '${domain}',
+    target: values.target || '',
+    status: 'analyzed',
+    auto_generated: true,
+    findings: [],
+    recommendations: [],
+  };
+  console.log(JSON.stringify(result, null, 2));
+} else {
+  console.error('Usage: agent.js --command analyze --target <scope>');
+  process.exit(1);
+}
+`;
+    writeFileSync(join(scriptsDir, 'agent.js'), agentJs);
+    chmodSync(join(scriptsDir, 'agent.js'), 0o755);
+
+    // references/api-reference.md
+    const refsDir = join(skillDir, 'references');
+    mkdirSync(refsDir, { recursive: true });
+    const refMd = `<!-- Copyright (c) 2026 defconxt. All rights reserved. -->
+<!-- Licensed under AGPL-3.0 — see LICENSE file for details. -->
+<!-- CIPHER is a trademark of defconxt. -->
+
+# API Reference — ${name}
+
+| Command | Description |
+|---------|-------------|
+| \`analyze --target <scope>\` | Run analysis on target |
+
+## Auto-Generated
+
+This skill was generated by the CIPHER Evolution Engine.
+Pattern: ${meta.generated_from || 'unknown'}
+`;
+    writeFileSync(join(refsDir, 'api-reference.md'), refMd);
+  }
+
+  /**
+   * Persist evolution history to JSONL file.
+   * @private
+   */
+  _persistHistory(record) {
+    if (!this.historyPath) return;
+    try {
+      mkdirSync(dirname(this.historyPath), { recursive: true });
+      const line = JSON.stringify({
+        timestamp: record.timestamp,
+        failures_analyzed: record.failuresAnalyzed,
+        skills_generated: record.skillsGenerated,
+        skill_names: record.skillNames,
+        failure_patterns: record.failurePatterns,
+      }) + '\n';
+      appendFileSync(this.historyPath, line, 'utf-8');
+    } catch (e) {
+      // Non-fatal: log and continue
+    }
+  }
+
+  /**
+   * Get evolution engine summary.
+   * @returns {object}
+   */
+  getSummary() {
+    return {
+      total_evolutions: this.evolutionHistory.length,
+      total_skills_generated: this.evolutionHistory.reduce(
+        (sum, r) => sum + r.skillsGenerated,
+        0,
+      ),
+      all_skill_names: this.evolutionHistory.flatMap((r) => r.skillNames),
+      failure_patterns: this.evolutionHistory.flatMap((r) => r.failurePatterns),
+    };
+  }
+}
+
+export {
+  ScoredResponse,
+  ResponseScorer,
+  EvolutionRecord,
+  SkillEvolver,
+};