cipher-security 5.0.0

package/lib/autonomous/feedback-loop.js

@@ -0,0 +1,554 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * Leaderboard → Researcher Feedback Loop.
+ *
+ * Connects skill performance metrics to autonomous improvement:
+ * - Identifies bottom-performing skills from leaderboard
+ * - Triggers skill regeneration via quality analysis
+ * - Tracks improvement cycles and validates quality gains
+ *
+ * @module autonomous/feedback-loop
+ */
+
+import { randomBytes } from 'node:crypto';
+import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync, statSync, chmodSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+// ---------------------------------------------------------------------------
+// ImprovementStatus
+// ---------------------------------------------------------------------------
+
+export const ImprovementStatus = Object.freeze({
+  IDENTIFIED: 'identified',
+  ANALYZING: 'analyzing',
+  REGENERATING: 'regenerating',
+  VALIDATING: 'validating',
+  COMPLETED: 'completed',
+  FAILED: 'failed',
+  SKIPPED: 'skipped',
+});
+
+// ---------------------------------------------------------------------------
+// ImprovementCandidate
+// ---------------------------------------------------------------------------
+
+export class ImprovementCandidate {
+  constructor({
+    skillPath = '',
+    domain = '',
+    currentScore = 0,
+    invocations = 0,
+    trend = '',
+    issues = [],
+    status = ImprovementStatus.IDENTIFIED,
+    improvementScore = 0,
+    cycleId = '',
+    startedAt = 0,
+    completedAt = 0,
+  } = {}) {
+    this.skillPath = skillPath;
+    this.domain = domain;
+    this.currentScore = currentScore;
+    this.invocations = invocations;
+    this.trend = trend;
+    this.issues = [...issues];
+    this.status = status;
+    this.improvementScore = improvementScore;
+    this.cycleId = cycleId;
+    this.startedAt = startedAt;
+    this.completedAt = completedAt;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// ImprovementCycle
+// ---------------------------------------------------------------------------
+
+export class ImprovementCycle {
+  constructor({
+    cycleId = '',
+    startedAt = 0,
+    candidates = [],
+    completedAt = 0,
+    skillsImproved = 0,
+    skillsFailed = 0,
+    skillsSkipped = 0,
+    avgScoreBefore = 0,
+    avgScoreAfter = 0,
+  } = {}) {
+    this.cycleId = cycleId;
+    this.startedAt = startedAt;
+    this.candidates = candidates;
+    this.completedAt = completedAt;
+    this.skillsImproved = skillsImproved;
+    this.skillsFailed = skillsFailed;
+    this.skillsSkipped = skillsSkipped;
+    this.avgScoreBefore = avgScoreBefore;
+    this.avgScoreAfter = avgScoreAfter;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// SkillQualityAnalyzer
+// ---------------------------------------------------------------------------
+
+export class SkillQualityAnalyzer {
+  static MIN_SKILL_MD_LINES = 20;
+  static MIN_AGENT_PY_LINES = 15;
+  static MIN_CODE_BLOCKS = 1;
+  static REQUIRED_SECTIONS = new Set(['overview', 'techniques', 'best practices', 'references']);
+
+  /**
+   * @param {string} [skillsDir='skills']
+   */
+  constructor(skillsDir = 'skills') {
+    this.skillsDir = resolve(String(skillsDir));
+  }
+
+  /**
+   * Analyze a skill's quality and return findings.
+   * @param {string} skillPath
+   * @returns {object}
+   */
+  analyzeSkill(skillPath) {
+    let fullPath;
+    if (isAbsolute(skillPath)) {
+      fullPath = skillPath;
+    } else {
+      // Try relative to parent of skillsDir
+      fullPath = resolve(join(this.skillsDir, '..'), skillPath);
+      if (!existsSync(fullPath)) {
+        const cleaned = skillPath.startsWith('skills/') ? skillPath.slice(7) : skillPath;
+        fullPath = join(this.skillsDir, cleaned);
+      }
+    }
+
+    const issues = [];
+    const scores = {};
+
+    // Check SKILL.md
+    const skillMd = join(fullPath, 'SKILL.md');
+    if (existsSync(skillMd)) {
+      const content = readFileSync(skillMd, 'utf-8');
+      const lines = content.trim().split('\n');
+
+      if (lines.length < SkillQualityAnalyzer.MIN_SKILL_MD_LINES) {
+        issues.push(`SKILL.md too short (${lines.length} lines, min ${SkillQualityAnalyzer.MIN_SKILL_MD_LINES})`);
+      }
+      scores.content_length = Math.min(lines.length / 50, 1.0);
+
+      const lower = content.toLowerCase();
+      const foundSections = new Set();
+      for (const s of SkillQualityAnalyzer.REQUIRED_SECTIONS) {
+        if (lower.includes(`## ${s}`) || lower.includes(`# ${s}`)) {
+          foundSections.add(s);
+        }
+      }
+      const missing = [...SkillQualityAnalyzer.REQUIRED_SECTIONS].filter(s => !foundSections.has(s));
+      if (missing.length > 0) {
+        issues.push(`Missing sections: ${missing.sort().join(', ')}`);
+      }
+      scores.section_coverage = foundSections.size / SkillQualityAnalyzer.REQUIRED_SECTIONS.size;
+
+      const codeBlocks = (content.match(/```/g) || []).length;
+      if (codeBlocks < SkillQualityAnalyzer.MIN_CODE_BLOCKS * 2) {
+        issues.push('No code blocks found');
+      }
+      scores.code_examples = Math.min(codeBlocks / 6, 1.0);
+
+      if (!content.slice(0, 200).includes('Copyright')) {
+        issues.push('Missing copyright header');
+      }
+      scores.frontmatter = content.slice(0, 10).includes('---') ? 1.0 : 0.0;
+      if (!content.slice(0, 10).includes('---')) {
+        issues.push('Missing YAML frontmatter');
+      }
+    } else {
+      issues.push('SKILL.md missing');
+      scores.content_length = 0;
+      scores.section_coverage = 0;
+    }
+
+    // Check agent.py
+    const agentPy = join(fullPath, 'scripts', 'agent.py');
+    if (existsSync(agentPy)) {
+      const agentContent = readFileSync(agentPy, 'utf-8');
+      const agentLines = agentContent.trim().split('\n');
+      if (agentLines.length < SkillQualityAnalyzer.MIN_AGENT_PY_LINES) {
+        issues.push(`agent.py too short (${agentLines.length} lines)`);
+      }
+      scores.agent_quality = Math.min(agentLines.length / 50, 1.0);
+      if (!agentContent.includes('argparse')) issues.push('agent.py missing argparse');
+      if (!agentContent.includes('json')) issues.push('agent.py missing JSON output');
+      if (!agentContent.includes('__main__')) issues.push('agent.py missing __main__ guard');
+    } else {
+      issues.push('scripts/agent.py missing');
+      scores.agent_quality = 0;
+    }
+
+    // Check references
+    const refDir = join(fullPath, 'references');
+    if (existsSync(refDir)) {
+      try {
+        const refs = readdirSync(refDir).filter(f => f.endsWith('.md'));
+        scores.references = refs.length > 0 ? 1.0 : 0.0;
+        if (refs.length === 0) issues.push('No reference documentation');
+      } catch {
+        scores.references = 0;
+        issues.push('No reference documentation');
+      }
+    } else {
+      issues.push('No reference documentation');
+      scores.references = 0;
+    }
+
+    const vals = Object.values(scores);
+    const overall = vals.length > 0 ? vals.reduce((a, b) => a + b, 0) / vals.length : 0;
+
+    return {
+      path: fullPath,
+      issues,
+      scores,
+      overallQuality: Math.round(overall * 1000) / 1000,
+      needsImprovement: issues.length > 2 || overall < 0.6,
+    };
+  }
+}
+
+/** Helper to check if a path is absolute. */
+function isAbsolute(p) {
+  return p.startsWith('/');
+}
+
+// ---------------------------------------------------------------------------
+// FeedbackLoop
+// ---------------------------------------------------------------------------
+
+export class FeedbackLoop {
+  static DEFAULT_BATCH_SIZE = 10;
+  static MIN_INVOCATIONS_FOR_FEEDBACK = 1;
+  static SCORE_THRESHOLD = 0.5;
+
+  /**
+   * @param {object} [opts]
+   * @param {string} [opts.skillsDir='skills']
+   * @param {number} [opts.maxImprovementsPerCycle=10]
+   */
+  constructor({ skillsDir = 'skills', maxImprovementsPerCycle = 10 } = {}) {
+    this.skillsDir = resolve(String(skillsDir));
+    this.maxPerCycle = maxImprovementsPerCycle;
+    this.analyzer = new SkillQualityAnalyzer(this.skillsDir);
+    /** @type {ImprovementCycle[]} */
+    this._history = [];
+  }
+
+  /**
+   * Identify skills that need improvement.
+   * @param {object} [opts]
+   * @param {object[]|null} [opts.bottomSkills]
+   * @param {number} [opts.scoreThreshold]
+   * @returns {ImprovementCandidate[]}
+   */
+  identifyCandidates({ bottomSkills = null, scoreThreshold } = {}) {
+    const threshold = scoreThreshold ?? FeedbackLoop.SCORE_THRESHOLD;
+    const candidates = [];
+
+    if (!bottomSkills) {
+      return this._identifyByQuality();
+    }
+
+    for (const skill of bottomSkills) {
+      const path = skill.skillPath || skill.path || '';
+      const score = skill.score ?? 0;
+      const invocations = skill.invocations ?? 0;
+      const trend = skill.trend ?? 'stable';
+
+      if (score >= threshold && trend !== 'declining') continue;
+
+      const analysis = this.analyzer.analyzeSkill(path);
+      candidates.push(new ImprovementCandidate({
+        skillPath: path,
+        domain: FeedbackLoop._extractDomain(path),
+        currentScore: score,
+        invocations,
+        trend,
+        issues: analysis.issues || [],
+      }));
+    }
+
+    // Sort: declining first, then lowest score, then most used
+    candidates.sort((a, b) => {
+      const aDecl = a.trend === 'declining' ? 1 : 0;
+      const bDecl = b.trend === 'declining' ? 1 : 0;
+      if (bDecl !== aDecl) return bDecl - aDecl;
+      if (a.currentScore !== b.currentScore) return a.currentScore - b.currentScore;
+      return b.invocations - a.invocations;
+    });
+
+    return candidates.slice(0, this.maxPerCycle);
+  }
+
+  /** @private */
+  _identifyByQuality() {
+    const candidates = [];
+    if (!existsSync(this.skillsDir)) return candidates;
+
+    try {
+      for (const domainName of readdirSync(this.skillsDir).sort()) {
+        const domainDir = join(this.skillsDir, domainName);
+        if (domainName.startsWith('.') || !statSync(domainDir).isDirectory()) continue;
+        const techniquesDir = join(domainDir, 'techniques');
+        if (!existsSync(techniquesDir)) continue;
+
+        for (const techName of readdirSync(techniquesDir).sort()) {
+          const techDir = join(techniquesDir, techName);
+          if (!statSync(techDir).isDirectory()) continue;
+          const relPath = `skills/${domainName}/techniques/${techName}`;
+          const analysis = this.analyzer.analyzeSkill(relPath);
+          if (analysis.needsImprovement) {
+            candidates.push(new ImprovementCandidate({
+              skillPath: relPath,
+              domain: domainName,
+              currentScore: analysis.overallQuality,
+              invocations: 0,
+              trend: 'unknown',
+              issues: analysis.issues,
+            }));
+          }
+        }
+      }
+    } catch { /* ok */ }
+
+    candidates.sort((a, b) => a.currentScore - b.currentScore);
+    return candidates.slice(0, this.maxPerCycle);
+  }
+
+  /**
+   * Run a full improvement cycle.
+   * @param {object} [opts]
+   * @param {ImprovementCandidate[]|null} [opts.candidates]
+   * @param {boolean} [opts.dryRun=false]
+   * @returns {ImprovementCycle}
+   */
+  runImprovementCycle({ candidates = null, dryRun = false } = {}) {
+    const cycleId = `cycle-${randomBytes(4).toString('hex')}`;
+    const cycle = new ImprovementCycle({
+      cycleId,
+      startedAt: Date.now() / 1000,
+    });
+
+    if (!candidates) {
+      candidates = this.identifyCandidates();
+    }
+
+    cycle.candidates = candidates;
+    const scoresBefore = [];
+
+    for (const candidate of candidates) {
+      candidate.cycleId = cycleId;
+      candidate.startedAt = Date.now() / 1000;
+      scoresBefore.push(candidate.currentScore);
+
+      if (dryRun) {
+        candidate.status = ImprovementStatus.SKIPPED;
+        cycle.skillsSkipped += 1;
+        continue;
+      }
+
+      try {
+        candidate.status = ImprovementStatus.ANALYZING;
+        const analysis = this.analyzer.analyzeSkill(candidate.skillPath);
+
+        if (!analysis.needsImprovement) {
+          candidate.status = ImprovementStatus.SKIPPED;
+          cycle.skillsSkipped += 1;
+          continue;
+        }
+
+        candidate.status = ImprovementStatus.REGENERATING;
+        const improved = this._improveSkill(candidate, analysis);
+
+        if (improved) {
+          candidate.status = ImprovementStatus.VALIDATING;
+          const validation = this.analyzer.analyzeSkill(candidate.skillPath);
+          candidate.improvementScore = validation.overallQuality;
+          candidate.status = ImprovementStatus.COMPLETED;
+          cycle.skillsImproved += 1;
+        } else {
+          candidate.status = ImprovementStatus.FAILED;
+          cycle.skillsFailed += 1;
+        }
+      } catch {
+        candidate.status = ImprovementStatus.FAILED;
+        cycle.skillsFailed += 1;
+      }
+
+      candidate.completedAt = Date.now() / 1000;
+    }
+
+    cycle.completedAt = Date.now() / 1000;
+    cycle.avgScoreBefore = scoresBefore.length > 0
+      ? scoresBefore.reduce((a, b) => a + b, 0) / scoresBefore.length : 0;
+    const scoresAfter = candidates
+      .filter(c => c.improvementScore > 0)
+      .map(c => c.improvementScore);
+    cycle.avgScoreAfter = scoresAfter.length > 0
+      ? scoresAfter.reduce((a, b) => a + b, 0) / scoresAfter.length : 0;
+
+    this._history.push(cycle);
+    return cycle;
+  }
+
+  /**
+   * Improve a single skill based on analysis findings.
+   * @private
+   * @param {ImprovementCandidate} candidate
+   * @param {object} analysis
+   * @returns {boolean}
+   */
+  _improveSkill(candidate, analysis) {
+    let skillDir = candidate.skillPath;
+    if (!isAbsolute(skillDir)) {
+      skillDir = resolve(join(this.skillsDir, '..'), candidate.skillPath);
+    }
+    if (!existsSync(skillDir)) return false;
+
+    let improved = false;
+
+    // Fix missing sections in SKILL.md
+    const skillMd = join(skillDir, 'SKILL.md');
+    if (existsSync(skillMd)) {
+      let content = readFileSync(skillMd, 'utf-8');
+      const lower = content.toLowerCase();
+      const additions = [];
+
+      if (!lower.includes('## overview') && !lower.includes('# overview')) {
+        additions.push('\n## Overview\nThis technique provides security analysis capabilities.\n');
+      }
+      if (!lower.includes('## techniques') && !lower.includes('# techniques')) {
+        additions.push('\n## Techniques\n- Analysis and detection methods\n- Tool configuration and usage\n- Threat indicator identification\n');
+      }
+      if (!lower.includes('## best practices') && !lower.includes('# best practices')) {
+        additions.push('\n## Best Practices\n- Follow defense-in-depth principles\n- Validate against known-good baselines\n- Document all findings with evidence\n');
+      }
+      if (!lower.includes('## references') && !lower.includes('# references')) {
+        additions.push('\n## References\n- MITRE ATT&CK Framework\n- CIS Controls v8\n- NIST SP 800-53 Rev. 5\n');
+      }
+      if (!content.includes('```')) {
+        additions.push('\n## Example\n\n```yaml\n# Configuration example\nenabled: true\nlevel: high\n```\n');
+      }
+
+      if (additions.length > 0) {
+        content += additions.join('');
+        writeFileSync(skillMd, content, 'utf-8');
+        improved = true;
+      }
+    }
+
+    return improved;
+  }
+
+  /**
+   * Generate a report for an improvement cycle.
+   * @param {ImprovementCycle} cycle
+   * @returns {object}
+   */
+  getCycleReport(cycle) {
+    return {
+      cycleId: cycle.cycleId,
+      durationS: cycle.completedAt
+        ? Math.round((cycle.completedAt - cycle.startedAt) * 100) / 100
+        : 0,
+      totalCandidates: cycle.candidates.length,
+      improved: cycle.skillsImproved,
+      failed: cycle.skillsFailed,
+      skipped: cycle.skillsSkipped,
+      avgScoreBefore: Math.round(cycle.avgScoreBefore * 1000) / 1000,
+      avgScoreAfter: Math.round(cycle.avgScoreAfter * 1000) / 1000,
+      candidates: cycle.candidates.map(c => ({
+        path: c.skillPath,
+        domain: c.domain,
+        status: c.status,
+        scoreBefore: c.currentScore,
+        scoreAfter: c.improvementScore,
+        issues: c.issues,
+      })),
+    };
+  }
+
+  /** Return all cycle reports. */
+  getHistory() {
+    return this._history.map(c => this.getCycleReport(c));
+  }
+
+  /**
+   * Extract domain name from skill path.
+   * @param {string} skillPath
+   * @returns {string}
+   */
+  static _extractDomain(skillPath) {
+    const parts = skillPath.split('/').filter(Boolean);
+    for (let i = 0; i < parts.length; i++) {
+      if (parts[i] === 'skills' && i + 1 < parts.length) {
+        return parts[i + 1];
+      }
+    }
+    if (parts.length >= 2) {
+      return parts[0] === 'skills' ? parts[1] : parts[0];
+    }
+    return 'unknown';
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Convenience function
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a complete feedback cycle: leaderboard → analysis → improvement.
+ * @param {object} [opts]
+ * @param {string} [opts.skillsDir='skills']
+ * @param {number} [opts.maxImprovements=10]
+ * @param {number} [opts.scoreThreshold=0.5]
+ * @param {boolean} [opts.dryRun=false]
+ * @returns {object}
+ */
+export function connectLeaderboardToResearcher({
+  skillsDir = 'skills',
+  maxImprovements = 10,
+  scoreThreshold = 0.5,
+  dryRun = false,
+} = {}) {
+  const loop = new FeedbackLoop({
+    skillsDir,
+    maxImprovementsPerCycle: maxImprovements,
+  });
+
+  // Try to get leaderboard data
+  let bottomSkills = null;
+  try {
+    const { SkillLeaderboard } = require('./leaderboard.js');
+    const lb = new SkillLeaderboard();
+    const bottomEntries = lb.getBottomSkills(maxImprovements * 2);
+    bottomSkills = bottomEntries.map(e => ({
+      skillPath: e.skillPath,
+      score: e.score,
+      invocations: e.invocations,
+      trend: e.trend,
+    }));
+    lb.close();
+  } catch {
+    // Leaderboard not available — quality-only analysis
+  }
+
+  const candidates = loop.identifyCandidates({
+    bottomSkills,
+    scoreThreshold,
+  });
+
+  const cycle = loop.runImprovementCycle({ candidates, dryRun });
+  return loop.getCycleReport(cycle);
+}