cipher-security 2.0.0

package/lib/autonomous/researcher.js

@@ -0,0 +1,583 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * Autonomous security research engine for self-improving skill generation.
+ *
+ * @module autonomous/researcher
+ */
+
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync, statSync, chmodSync } from 'node:fs';
+import { join, resolve, isAbsolute } from 'node:path';
+
+// ---------------------------------------------------------------------------
+// Data classes
+// ---------------------------------------------------------------------------
+
+export class ResearchHypothesis {
+  constructor({
+    hypothesisId = randomUUID(),
+    domain = '',
+    techniqueName = '',
+    description = '',
+    skillContent = '',
+    agentScript = '',
+    referenceContent = '',
+    confidence = 0,
+    status = 'proposed',
+    testResults = {},
+    createdAt = new Date().toISOString(),
+  } = {}) {
+    this.hypothesisId = hypothesisId;
+    this.domain = domain;
+    this.techniqueName = techniqueName;
+    this.description = description;
+    this.skillContent = skillContent;
+    this.agentScript = agentScript;
+    this.referenceContent = referenceContent;
+    this.confidence = confidence;
+    this.status = status;
+    this.testResults = testResults;
+    this.createdAt = createdAt;
+  }
+}
+
+export class ResearchExperiment {
+  constructor({
+    experimentId = randomUUID(),
+    hypothesis = new ResearchHypothesis(),
+    validationChecks = [],
+    checkResults = {},
+    overallPass = false,
+    createdAt = new Date().toISOString(),
+  } = {}) {
+    this.experimentId = experimentId;
+    this.hypothesis = hypothesis;
+    this.validationChecks = validationChecks;
+    this.checkResults = checkResults;
+    this.overallPass = overallPass;
+    this.createdAt = createdAt;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// MITRE ATT&CK tactics mapped to technique IDs
+// ---------------------------------------------------------------------------
+
+const _MITRE_TACTICS = {
+  'reconnaissance': ['T1595', 'T1592', 'T1589', 'T1590', 'T1591', 'T1596', 'T1597', 'T1598'],
+  'resource-development': ['T1583', 'T1584', 'T1585', 'T1586', 'T1587', 'T1588'],
+  'initial-access': ['T1189', 'T1190', 'T1133', 'T1200', 'T1566', 'T1195', 'T1199', 'T1078'],
+  'execution': ['T1059', 'T1203', 'T1047', 'T1053', 'T1129', 'T1106', 'T1204'],
+  'persistence': ['T1098', 'T1197', 'T1547', 'T1136', 'T1543', 'T1546', 'T1053', 'T1078'],
+  'privilege-escalation': ['T1548', 'T1134', 'T1547', 'T1484', 'T1055', 'T1053', 'T1078'],
+  'defense-evasion': ['T1140', 'T1027', 'T1036', 'T1070', 'T1562', 'T1202', 'T1218', 'T1222'],
+  'credential-access': ['T1110', 'T1003', 'T1555', 'T1552', 'T1187', 'T1557', 'T1558'],
+  'discovery': ['T1087', 'T1482', 'T1083', 'T1135', 'T1046', 'T1057', 'T1018', 'T1082'],
+  'lateral-movement': ['T1021', 'T1210', 'T1534', 'T1570', 'T1563', 'T1080'],
+  'collection': ['T1560', 'T1123', 'T1119', 'T1005', 'T1039', 'T1025', 'T1074', 'T1114'],
+  'command-and-control': ['T1071', 'T1132', 'T1001', 'T1568', 'T1573', 'T1095', 'T1572', 'T1090'],
+  'exfiltration': ['T1041', 'T1048', 'T1567', 'T1029', 'T1030', 'T1537'],
+  'impact': ['T1485', 'T1486', 'T1565', 'T1491', 'T1561', 'T1499', 'T1529'],
+};
+
+// ---------------------------------------------------------------------------
+// SkillGapAnalyzer
+// ---------------------------------------------------------------------------
+
+export class SkillGapAnalyzer {
+  /**
+   * @param {string} skillsDir
+   */
+  constructor(skillsDir) {
+    this.skillsDir = resolve(String(skillsDir));
+  }
+
+  /** @returns {string[]} */
+  _listDomains() {
+    if (!existsSync(this.skillsDir)) return [];
+    try {
+      return readdirSync(this.skillsDir)
+        .filter(d => !d.startsWith('.') && statSync(join(this.skillsDir, d)).isDirectory())
+        .sort();
+    } catch { return []; }
+  }
+
+  /**
+   * @param {string} domain
+   * @returns {string[]}
+   */
+  _listTechniques(domain) {
+    const techDir = join(this.skillsDir, domain, 'techniques');
+    if (!existsSync(techDir)) return [];
+    try {
+      return readdirSync(techDir)
+        .filter(d => !d.startsWith('.') && statSync(join(techDir, d)).isDirectory())
+        .sort();
+    } catch { return []; }
+  }
+
+  /**
+   * Parse YAML frontmatter from a SKILL.md file.
+   * @param {string} skillMdPath
+   * @returns {object}
+   */
+  _parseSkillFrontmatter(skillMdPath) {
+    if (!existsSync(skillMdPath)) return {};
+    try {
+      const text = readFileSync(skillMdPath, 'utf-8');
+      const parts = text.split('---');
+      for (let i = 1; i < parts.length; i++) {
+        const candidate = parts[i].trim();
+        if (!candidate) continue;
+        // Lazy import yaml
+        try {
+          const YAML = require('yaml');
+          const data = YAML.parse(candidate);
+          if (data && typeof data === 'object') return data;
+        } catch { continue; }
+      }
+    } catch { /* ok */ }
+    return {};
+  }
+
+  /**
+   * Map MITRE ATT&CK tactics to covered/uncovered techniques.
+   */
+  analyzeCoverage() {
+    const domains = this._listDomains();
+    const coveredIds = new Set();
+
+    for (const domain of domains) {
+      const domainSkill = join(this.skillsDir, domain, 'SKILL.md');
+      const meta = this._parseSkillFrontmatter(domainSkill);
+      const mitre = meta.metadata || {};
+      if (typeof mitre === 'object' && Array.isArray(mitre['mitre-attack'])) {
+        for (const id of mitre['mitre-attack']) coveredIds.add(String(id));
+      }
+      for (const tech of this._listTechniques(domain)) {
+        const techSkill = join(this.skillsDir, domain, 'techniques', tech, 'SKILL.md');
+        const techMeta = this._parseSkillFrontmatter(techSkill);
+        const techMitre = techMeta.metadata || {};
+        if (typeof techMitre === 'object' && Array.isArray(techMitre['mitre-attack'])) {
+          for (const id of techMitre['mitre-attack']) coveredIds.add(String(id));
+        }
+      }
+    }
+
+    const coverage = {};
+    for (const [tactic, techniqueIds] of Object.entries(_MITRE_TACTICS)) {
+      const covered = techniqueIds.filter(t => coveredIds.has(t));
+      const uncovered = techniqueIds.filter(t => !coveredIds.has(t));
+      coverage[tactic] = {
+        total: techniqueIds.length,
+        covered,
+        uncovered,
+        pct: Math.round((covered.length / Math.max(techniqueIds.length, 1)) * 1000) / 10,
+      };
+    }
+    return coverage;
+  }
+
+  /** Identify domains with fewer than 24 techniques. */
+  findGaps() {
+    const gaps = [];
+    for (const domain of this._listDomains()) {
+      const techniques = this._listTechniques(domain);
+      if (techniques.length < 24) {
+        gaps.push({
+          domain,
+          techniqueCount: techniques.length,
+          deficit: 24 - techniques.length,
+          existing: techniques,
+        });
+      }
+    }
+    return gaps;
+  }
+
+  /**
+   * Suggest new technique names based on existing patterns.
+   * @param {string} domain
+   * @param {number} [count=5]
+   */
+  suggestNewTechniques(domain, count = 5) {
+    const existing = this._listTechniques(domain);
+    const verbCounts = {};
+    for (const d of this._listDomains()) {
+      for (const tech of this._listTechniques(d)) {
+        const verb = tech.includes('-') ? tech.split('-')[0] : tech;
+        verbCounts[verb] = (verbCounts[verb] || 0) + 1;
+      }
+    }
+    const popularVerbs = Object.keys(verbCounts).sort((a, b) => verbCounts[b] - verbCounts[a]);
+    const domainWords = domain.replace(/-/g, ' ').split(' ');
+    const suggestions = [];
+    const existingSet = new Set(existing);
+
+    for (const verb of popularVerbs) {
+      if (suggestions.length >= count) break;
+      for (const kw of domainWords) {
+        const candidate = `${verb}-${kw}-operations`;
+        if (!existingSet.has(candidate) && !suggestions.includes(candidate)) {
+          suggestions.push(candidate);
+          if (suggestions.length >= count) break;
+        }
+      }
+      const candidateAlt = `${verb}-${domain}-analysis`;
+      if (!existingSet.has(candidateAlt) && !suggestions.includes(candidateAlt)) {
+        suggestions.push(candidateAlt);
+        if (suggestions.length >= count) break;
+      }
+    }
+    return suggestions.slice(0, count);
+  }
+
+  /** Get full coverage report. */
+  getCoverageReport() {
+    const domains = this._listDomains();
+    let totalTechniques = 0;
+    const domainStats = [];
+    for (const domain of domains) {
+      const techs = this._listTechniques(domain);
+      totalTechniques += techs.length;
+      domainStats.push({ domain, techniqueCount: techs.length });
+    }
+    return {
+      totalDomains: domains.length,
+      totalTechniques,
+      domainStats,
+      mitreCoverage: this.analyzeCoverage(),
+      gapCount: this.findGaps().length,
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Content generators
+// ---------------------------------------------------------------------------
+
+function _titleCase(slug) {
+  return slug.split('-').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+}
+
+function _generateSkillMd(domain, techniqueName, description) {
+  return `<!-- Copyright (c) 2026 defconxt. All rights reserved. -->
+<!-- Licensed under AGPL-3.0 — see LICENSE file for details. -->
+---
+name: ${techniqueName}
+description: >-
+  ${description}
+domain: cybersecurity
+subdomain: ${domain}
+tags:
+  - ${domain}
+  - ${techniqueName.split('-')[0]}
+  - security-automation
+version: "1.0"
+author: defconxt
+license: AGPL-3.0
+---
+
+# ${_titleCase(techniqueName)}
+
+## Quick Reference
+
+| Item | Detail |
+|---|---|
+| Domain | ${domain} |
+| Technique | ${techniqueName} |
+| ATT&CK | See metadata |
+| Difficulty | Intermediate |
+
+## Overview
+
+${description}
+
+## Workflow
+
+1. **Preparation** — Gather required tooling and access.
+2. **Execution** — Run the technique using the agent script.
+3. **Analysis** — Review output and correlate findings.
+4. **Reporting** — Document results and remediation steps.
+
+## Verification
+
+\`\`\`bash
+python3 scripts/agent.py analyze --target example
+python3 scripts/agent.py report --format json
+\`\`\`
+
+## References
+
+- MITRE ATT&CK: https://attack.mitre.org/
+- See \`references/api-reference.md\` for command details.
+`;
+}
+
+function _generateAgentPy(domain, techniqueName) {
+  const funcPrefix = techniqueName.replace(/-/g, '_').slice(0, 20);
+  return `#!/usr/bin/env python3
+"""Agent tooling for ${_titleCase(techniqueName).toLowerCase()}."""
+
+import argparse
+import json
+import sys
+
+
+def ${funcPrefix}_analyze(target, verbose=False):
+    """Analyze a target for ${techniqueName} indicators."""
+    results = {
+        "action": "analyze",
+        "technique": "${techniqueName}",
+        "domain": "${domain}",
+        "target": target,
+        "findings": [],
+        "risk_level": "medium",
+    }
+    if verbose:
+        results["debug"] = {"verbose": True}
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="${techniqueName} agent")
+    parser.add_argument("--target", required=True)
+    parser.add_argument("--verbose", action="store_true")
+    parser.add_argument("--format", choices=["json", "text"], default="json")
+    args = parser.parse_args()
+
+    result = ${funcPrefix}_analyze(args.target, args.verbose)
+    if args.format == "json":
+        print(json.dumps(result, indent=2))
+    else:
+        for k, v in result.items():
+            print(f"{k}: {v}")
+
+
+if __name__ == "__main__":
+    main()
+`;
+}
+
+function _generateApiReference(domain, techniqueName) {
+  const title = _titleCase(techniqueName);
+  return `<!-- Copyright (c) 2026 defconxt. All rights reserved. -->
+<!-- Licensed under AGPL-3.0 — see LICENSE file for details. -->
+
+# API Reference — ${title}
+
+## Agent Script
+
+| Command | Description |
+|---|---|
+| \`python3 agent.py analyze --target <t>\` | Analyze a target for ${techniqueName} indicators |
+
+## Options
+
+| Flag | Type | Default | Description |
+|---|---|---|---|
+| \`--target\` | str | *(required)* | Target identifier for analysis |
+| \`--verbose\` | flag | false | Enable verbose output |
+| \`--format\` | str | json | Output format |
+
+## Output Format
+
+All commands return JSON.
+`;
+}
+
+// ---------------------------------------------------------------------------
+// AutonomousResearcher
+// ---------------------------------------------------------------------------
+
+export class AutonomousResearcher {
+  /**
+   * @param {string} skillsDir
+   * @param {object} [memory] - CipherMemory instance (optional, lazy-loaded)
+   */
+  constructor(skillsDir, memory = null) {
+    this.skillsDir = resolve(String(skillsDir));
+    this.memory = memory;
+    this._history = [];
+    this._analyzer = new SkillGapAnalyzer(this.skillsDir);
+  }
+
+  /**
+   * Create a complete hypothesis with SKILL.md, agent.py, and api-reference.md.
+   * @param {string} domain
+   * @param {string} techniqueName
+   * @returns {ResearchHypothesis}
+   */
+  generateHypothesis(domain, techniqueName) {
+    const description =
+      `Automated security technique for ${_titleCase(techniqueName).toLowerCase()} ` +
+      `within the ${domain} domain. Provides analysis, scanning, and reporting ` +
+      `capabilities for defensive and offensive security workflows.`;
+
+    return new ResearchHypothesis({
+      domain,
+      techniqueName,
+      description,
+      skillContent: _generateSkillMd(domain, techniqueName, description),
+      agentScript: _generateAgentPy(domain, techniqueName),
+      referenceContent: _generateApiReference(domain, techniqueName),
+      confidence: 0.75,
+      status: 'proposed',
+    });
+  }
+
+  /**
+   * Run validation checks against a hypothesis.
+   * @param {ResearchHypothesis} hypothesis
+   * @returns {ResearchExperiment}
+   */
+  validateHypothesis(hypothesis) {
+    const checks = ['yaml_valid', 'script_compiles', 'references_exist', 'skill_has_sections', 'content_length'];
+    const results = {};
+
+    results.yaml_valid = AutonomousResearcher._checkYaml(hypothesis.skillContent);
+    results.script_compiles = AutonomousResearcher._checkCompiles(hypothesis.agentScript);
+    results.references_exist = AutonomousResearcher._checkReferences(hypothesis.referenceContent);
+    results.skill_has_sections = AutonomousResearcher._checkSkillSections(hypothesis.skillContent);
+    results.content_length = AutonomousResearcher._checkContentLength(
+      hypothesis.skillContent, hypothesis.agentScript, hypothesis.referenceContent
+    );
+
+    const overall = Object.values(results).every(v => v);
+    hypothesis.testResults = { ...results };
+    hypothesis.status = 'testing';
+
+    return new ResearchExperiment({
+      hypothesis,
+      validationChecks: checks,
+      checkResults: results,
+      overallPass: overall,
+    });
+  }
+
+  static _checkYaml(skillContent) {
+    const parts = skillContent.split('---');
+    for (let i = 1; i < parts.length; i++) {
+      const candidate = parts[i].trim();
+      if (!candidate) continue;
+      try {
+        const YAML = require('yaml');
+        const data = YAML.parse(candidate);
+        return data && typeof data === 'object' && 'name' in data;
+      } catch { continue; }
+    }
+    return false;
+  }
+
+  static _checkCompiles(script) {
+    // For JS port: just verify it's a non-empty string with valid Python syntax indicators
+    // We can't actually parse Python in Node.js, but we can check for structure
+    return typeof script === 'string' && script.length > 100 && script.includes('def ') && script.includes('import ');
+  }
+
+  static _checkReferences(refContent) {
+    return !!refContent && (refContent.trim().startsWith('#') || refContent.includes('# '));
+  }
+
+  static _checkSkillSections(skillContent) {
+    const required = ['Quick Reference', 'Workflow', 'Verification', 'References'];
+    const lower = skillContent.toLowerCase();
+    return required.every(section => lower.includes(section.toLowerCase()));
+  }
+
+  static _checkContentLength(skill, script, reference) {
+    return skill.length >= 200 && script.length >= 200 && reference.length >= 100;
+  }
+
+  /**
+   * Write hypothesis files to disk.
+   * @param {ResearchHypothesis} hypothesis
+   * @returns {boolean}
+   */
+  applyHypothesis(hypothesis) {
+    try {
+      const base = join(this.skillsDir, hypothesis.domain, 'techniques', hypothesis.techniqueName);
+      const scriptsDir = join(base, 'scripts');
+      const refsDir = join(base, 'references');
+
+      mkdirSync(scriptsDir, { recursive: true });
+      mkdirSync(refsDir, { recursive: true });
+
+      writeFileSync(join(base, 'SKILL.md'), hypothesis.skillContent, 'utf-8');
+      const agentPath = join(scriptsDir, 'agent.py');
+      writeFileSync(agentPath, hypothesis.agentScript, 'utf-8');
+      try { chmodSync(agentPath, 0o755); } catch { /* ok */ }
+      writeFileSync(join(refsDir, 'api-reference.md'), hypothesis.referenceContent, 'utf-8');
+
+      hypothesis.status = 'accepted';
+
+      // Store in memory if available
+      if (this.memory) {
+        try {
+          // Lazy import to avoid circular deps
+          this.memory.store({
+            content: `Generated technique ${hypothesis.techniqueName} in ${hypothesis.domain}`,
+            keywords: [hypothesis.domain, hypothesis.techniqueName, hypothesis.hypothesisId],
+          });
+        } catch { /* memory is optional */ }
+      }
+
+      return true;
+    } catch {
+      hypothesis.status = 'rejected';
+      return false;
+    }
+  }
+
+  /**
+   * Full cycle: generate → validate → apply if passing.
+   * @param {string} domain
+   * @param {string} techniqueName
+   * @returns {ResearchExperiment}
+   */
+  runExperiment(domain, techniqueName) {
+    const hypothesis = this.generateHypothesis(domain, techniqueName);
+    const experiment = this.validateHypothesis(hypothesis);
+
+    if (experiment.overallPass) {
+      const applied = this.applyHypothesis(hypothesis);
+      if (!applied) {
+        hypothesis.status = 'rejected';
+        experiment.overallPass = false;
+      }
+    } else {
+      hypothesis.status = 'rejected';
+    }
+
+    this._history.push({
+      experimentId: experiment.experimentId,
+      hypothesisId: hypothesis.hypothesisId,
+      domain,
+      techniqueName,
+      overallPass: experiment.overallPass,
+      status: hypothesis.status,
+      checks: { ...experiment.checkResults },
+      createdAt: experiment.createdAt,
+    });
+
+    return experiment;
+  }
+
+  /**
+   * Run experiments for multiple techniques in a domain.
+   * @param {string} domain
+   * @param {string[]} techniques
+   * @returns {ResearchExperiment[]}
+   */
+  batchExpand(domain, techniques) {
+    return techniques.map(tech => this.runExperiment(domain, tech));
+  }
+
+  /** Return all experiments with results. */
+  getResearchHistory() {
+    return [...this._history];
+  }
+}