cipher-security 5.0.0

package/lib/autonomous/leaderboard.js

@@ -0,0 +1,594 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * Skill effectiveness tracking and metrics system.
+ *
+ * SQLite-backed leaderboard that records skill invocations, computes
+ * performance trends, and surfaces domain-level coverage insights.
+ *
+ * @module autonomous/leaderboard
+ */
+
+import { mkdirSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { homedir } from 'node:os';
+
+// ---------------------------------------------------------------------------
+// Data classes
+// ---------------------------------------------------------------------------
+
+export class SkillMetric {
+  constructor({
+    skillPath = '',
+    domain = '',
+    technique = '',
+    invocationCount = 0,
+    successCount = 0,
+    failureCount = 0,
+    avgScore = 0,
+    lastUsed = '',
+    lastScore = 0,
+    trend = '',
+    scoresHistory = [],
+  } = {}) {
+    this.skillPath = skillPath;
+    this.domain = domain;
+    this.technique = technique;
+    this.invocationCount = invocationCount;
+    this.successCount = successCount;
+    this.failureCount = failureCount;
+    this.avgScore = avgScore;
+    this.lastUsed = lastUsed;
+    this.lastScore = lastScore;
+    this.trend = trend;
+    this.scoresHistory = scoresHistory;
+  }
+}
+
+export class DomainMetric {
+  constructor({
+    domain = '',
+    techniqueCount = 0,
+    totalInvocations = 0,
+    avgSuccessRate = 0,
+    avgScore = 0,
+    coverageScore = 0,
+    strongestTechnique = '',
+    weakestTechnique = '',
+  } = {}) {
+    this.domain = domain;
+    this.techniqueCount = techniqueCount;
+    this.totalInvocations = totalInvocations;
+    this.avgSuccessRate = avgSuccessRate;
+    this.avgScore = avgScore;
+    this.coverageScore = coverageScore;
+    this.strongestTechnique = strongestTechnique;
+    this.weakestTechnique = weakestTechnique;
+  }
+}
+
+export class LeaderboardEntry {
+  constructor({
+    rank = 0,
+    skillPath = '',
+    domain = '',
+    score = 0,
+    invocations = 0,
+    successRate = 0,
+    trend = '',
+  } = {}) {
+    this.rank = rank;
+    this.skillPath = skillPath;
+    this.domain = domain;
+    this.score = score;
+    this.invocations = invocations;
+    this.successRate = successRate;
+    this.trend = trend;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// MITRE ATT&CK reference counts per domain
+// ---------------------------------------------------------------------------
+
+const _MITRE_TECHNIQUE_COUNTS = {
+  'initial-access': 10,
+  'execution': 14,
+  'persistence': 20,
+  'privilege-escalation': 14,
+  'defense-evasion': 43,
+  'credential-access': 17,
+  'discovery': 32,
+  'lateral-movement': 9,
+  'collection': 17,
+  'command-and-control': 16,
+  'exfiltration': 9,
+  'impact': 14,
+  'reconnaissance': 10,
+  'resource-development': 8,
+};
+
+// ---------------------------------------------------------------------------
+// SkillLeaderboard
+// ---------------------------------------------------------------------------
+
+export class SkillLeaderboard {
+  /**
+   * @param {string} [dbPath='~/.cipher/leaderboard.db']
+   */
+  constructor(dbPath = '~/.cipher/leaderboard.db') {
+    // Resolve ~ to home directory
+    const resolved = dbPath.startsWith('~')
+      ? resolve(homedir(), dbPath.slice(2))
+      : resolve(dbPath);
+
+    // Ensure parent directory exists
+    try { mkdirSync(dirname(resolved), { recursive: true }); } catch { /* ok */ }
+
+    // Lazy-load better-sqlite3 to avoid import-time cost
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const Database = require('better-sqlite3');
+    this._dbPath = resolved;
+
+    // ':memory:' for in-memory databases
+    if (dbPath === ':memory:') {
+      this._db = new Database(':memory:');
+    } else {
+      this._db = new Database(resolved);
+    }
+
+    this._db.pragma('journal_mode = WAL');
+    this._initDb();
+  }
+
+  // ------------------------------------------------------------------
+  // Schema
+  // ------------------------------------------------------------------
+
+  _initDb() {
+    this._db.exec(`
+      CREATE TABLE IF NOT EXISTS skill_metrics (
+        skill_path    TEXT PRIMARY KEY,
+        domain        TEXT NOT NULL,
+        technique     TEXT NOT NULL DEFAULT '',
+        invocation_count INTEGER NOT NULL DEFAULT 0,
+        success_count INTEGER NOT NULL DEFAULT 0,
+        failure_count INTEGER NOT NULL DEFAULT 0,
+        avg_score     REAL NOT NULL DEFAULT 0.0,
+        last_used     TEXT NOT NULL DEFAULT '',
+        last_score    REAL NOT NULL DEFAULT 0.0,
+        trend         TEXT NOT NULL DEFAULT '',
+        scores_history TEXT NOT NULL DEFAULT '[]'
+      );
+
+      CREATE TABLE IF NOT EXISTS invocation_log (
+        id         INTEGER PRIMARY KEY AUTOINCREMENT,
+        skill_path TEXT NOT NULL,
+        timestamp  TEXT NOT NULL,
+        success    INTEGER NOT NULL,
+        score      REAL NOT NULL,
+        context    TEXT NOT NULL DEFAULT '{}'
+      );
+
+      CREATE TABLE IF NOT EXISTS domain_metrics (
+        domain              TEXT PRIMARY KEY,
+        technique_count     INTEGER NOT NULL DEFAULT 0,
+        total_invocations   INTEGER NOT NULL DEFAULT 0,
+        avg_success_rate    REAL NOT NULL DEFAULT 0.0,
+        avg_score           REAL NOT NULL DEFAULT 0.0,
+        coverage_score      REAL NOT NULL DEFAULT 0.0,
+        strongest_technique TEXT NOT NULL DEFAULT '',
+        weakest_technique   TEXT NOT NULL DEFAULT ''
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_invlog_skill ON invocation_log(skill_path);
+      CREATE INDEX IF NOT EXISTS idx_invlog_ts ON invocation_log(timestamp);
+      CREATE INDEX IF NOT EXISTS idx_skill_domain ON skill_metrics(domain);
+    `);
+  }
+
+  // ------------------------------------------------------------------
+  // Recording
+  // ------------------------------------------------------------------
+
+  /**
+   * Record a single skill invocation and update aggregates.
+   * @param {string} skillPath
+   * @param {boolean} success
+   * @param {number} score
+   * @param {object} [context]
+   */
+  recordInvocation(skillPath, success, score, context = {}) {
+    const now = new Date().toISOString();
+    const ctxJson = JSON.stringify(context);
+
+    const parts = skillPath.replace(/^\//, '').split('/');
+    const domain = parts[0] || 'unknown';
+    const technique = parts.length > 1 ? parts[parts.length - 1] : '';
+
+    // Sanitize NaN/Infinity
+    if (!Number.isFinite(score)) score = 0;
+
+    // Insert invocation log
+    this._db.prepare(
+      'INSERT INTO invocation_log (skill_path, timestamp, success, score, context) VALUES (?, ?, ?, ?, ?)'
+    ).run(skillPath, now, success ? 1 : 0, score, ctxJson);
+
+    // Upsert skill_metrics
+    const row = this._db.prepare('SELECT * FROM skill_metrics WHERE skill_path = ?').get(skillPath);
+
+    if (!row) {
+      const history = [score];
+      this._db.prepare(
+        'INSERT INTO skill_metrics (skill_path, domain, technique, invocation_count, success_count, failure_count, avg_score, last_used, last_score, trend, scores_history) VALUES (?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?)'
+      ).run(
+        skillPath, domain, technique,
+        success ? 1 : 0, success ? 0 : 1,
+        score, now, score, '', JSON.stringify(history)
+      );
+    } else {
+      const inv = row.invocation_count + 1;
+      const sc = row.success_count + (success ? 1 : 0);
+      const fc = row.failure_count + (success ? 0 : 1);
+      let prevHistory = JSON.parse(row.scores_history);
+      prevHistory.push(score);
+      if (prevHistory.length > 100) prevHistory = prevHistory.slice(-100);
+      const avg = prevHistory.reduce((a, b) => a + b, 0) / prevHistory.length;
+      this._db.prepare(
+        'UPDATE skill_metrics SET invocation_count = ?, success_count = ?, failure_count = ?, avg_score = ?, last_used = ?, last_score = ?, scores_history = ? WHERE skill_path = ?'
+      ).run(inv, sc, fc, avg, now, score, JSON.stringify(prevHistory), skillPath);
+    }
+
+    this._updateDomainAggregates(domain);
+  }
+
+  // ------------------------------------------------------------------
+  // Queries — single skill
+  // ------------------------------------------------------------------
+
+  /**
+   * Return the metric snapshot for a single skill.
+   * @param {string} skillPath
+   * @returns {SkillMetric}
+   */
+  getSkillMetric(skillPath) {
+    const row = this._db.prepare('SELECT * FROM skill_metrics WHERE skill_path = ?').get(skillPath);
+    if (!row) {
+      const parts = skillPath.replace(/^\//, '').split('/');
+      return new SkillMetric({
+        skillPath,
+        domain: parts[0] || 'unknown',
+        technique: parts.length > 1 ? parts[parts.length - 1] : '',
+      });
+    }
+    return new SkillMetric({
+      skillPath: row.skill_path,
+      domain: row.domain,
+      technique: row.technique,
+      invocationCount: row.invocation_count,
+      successCount: row.success_count,
+      failureCount: row.failure_count,
+      avgScore: row.avg_score,
+      lastUsed: row.last_used,
+      lastScore: row.last_score,
+      trend: row.trend,
+      scoresHistory: JSON.parse(row.scores_history),
+    });
+  }
+
+  // ------------------------------------------------------------------
+  // Queries — domain
+  // ------------------------------------------------------------------
+
+  /**
+   * Return aggregate metrics for a security domain.
+   * @param {string} domain
+   * @returns {DomainMetric}
+   */
+  getDomainMetrics(domain) {
+    const row = this._db.prepare('SELECT * FROM domain_metrics WHERE domain = ?').get(domain);
+    if (!row) {
+      return new DomainMetric({ domain });
+    }
+    return new DomainMetric({
+      domain: row.domain,
+      techniqueCount: row.technique_count,
+      totalInvocations: row.total_invocations,
+      avgSuccessRate: row.avg_success_rate,
+      avgScore: row.avg_score,
+      coverageScore: row.coverage_score,
+      strongestTechnique: row.strongest_technique,
+      weakestTechnique: row.weakest_technique,
+    });
+  }
+
+  // ------------------------------------------------------------------
+  // Leaderboard queries
+  // ------------------------------------------------------------------
+
+  /** @private */
+  _rowsToEntries(rows, startRank = 1) {
+    return rows.map((r, i) => {
+      const inv = r.invocation_count || 1;
+      return new LeaderboardEntry({
+        rank: startRank + i,
+        skillPath: r.skill_path,
+        domain: r.domain,
+        score: r.avg_score,
+        invocations: r.invocation_count,
+        successRate: Math.round((r.success_count / inv) * 10000) / 10000,
+        trend: r.trend,
+      });
+    });
+  }
+
+  /**
+   * Return the top-N skills by avg_score.
+   * @param {number} [n=20]
+   * @param {string} [domain]
+   * @returns {LeaderboardEntry[]}
+   */
+  getTopSkills(n = 20, domain) {
+    let rows;
+    if (domain) {
+      rows = this._db.prepare(
+        'SELECT * FROM skill_metrics WHERE domain = ? ORDER BY avg_score DESC LIMIT ?'
+      ).all(domain, n);
+    } else {
+      rows = this._db.prepare(
+        'SELECT * FROM skill_metrics ORDER BY avg_score DESC LIMIT ?'
+      ).all(n);
+    }
+    return this._rowsToEntries(rows);
+  }
+
+  /**
+   * Return the bottom-N skills by avg_score.
+   * @param {number} [n=20]
+   * @param {string} [domain]
+   * @returns {LeaderboardEntry[]}
+   */
+  getBottomSkills(n = 20, domain) {
+    let rows;
+    if (domain) {
+      rows = this._db.prepare(
+        'SELECT * FROM skill_metrics WHERE domain = ? ORDER BY avg_score ASC LIMIT ?'
+      ).all(domain, n);
+    } else {
+      rows = this._db.prepare(
+        'SELECT * FROM skill_metrics ORDER BY avg_score ASC LIMIT ?'
+      ).all(n);
+    }
+    return this._rowsToEntries(rows);
+  }
+
+  /**
+   * Return skills trending in the given direction.
+   * @param {string} [direction='up']
+   * @param {number} [n=10]
+   * @returns {LeaderboardEntry[]}
+   */
+  getTrending(direction = 'up', n = 10) {
+    const trendVal = direction === 'up' ? 'improving' : 'declining';
+    const rows = this._db.prepare(
+      'SELECT * FROM skill_metrics WHERE trend = ? ORDER BY avg_score DESC LIMIT ?'
+    ).all(trendVal, n);
+    return this._rowsToEntries(rows);
+  }
+
+  /**
+   * Return skill paths not invoked within the last N days.
+   * @param {number} [days=30]
+   * @returns {string[]}
+   */
+  getUnusedSkills(days = 30) {
+    const cutoff = new Date(Date.now() - days * 86400000).toISOString();
+    const rows = this._db.prepare(
+      "SELECT skill_path FROM skill_metrics WHERE last_used < ? OR last_used = ''"
+    ).all(cutoff);
+    return rows.map(r => r.skill_path);
+  }
+
+  // ------------------------------------------------------------------
+  // Trend computation
+  // ------------------------------------------------------------------
+
+  /**
+   * Determine trend via linear regression slope.
+   * @param {number[]} scores
+   * @returns {string}
+   */
+  static calculateTrend(scores) {
+    if (scores.length < 3) return 'stable';
+    const n = scores.length;
+    const xMean = (n - 1) / 2;
+    const yMean = scores.reduce((a, b) => a + b, 0) / n;
+    let numerator = 0;
+    let denominator = 0;
+    for (let i = 0; i < n; i++) {
+      numerator += (i - xMean) * (scores[i] - yMean);
+      denominator += (i - xMean) ** 2;
+    }
+    if (denominator === 0) return 'stable';
+    const slope = numerator / denominator;
+    if (slope > 0.02) return 'improving';
+    if (slope < -0.02) return 'declining';
+    return 'stable';
+  }
+
+  /**
+   * Recompute trend field for all skills based on recent scores.
+   * @param {number} [window=10]
+   */
+  computeTrends(window = 10) {
+    const rows = this._db.prepare('SELECT skill_path, scores_history FROM skill_metrics').all();
+    for (const r of rows) {
+      const history = JSON.parse(r.scores_history);
+      const recent = history.length > window ? history.slice(-window) : history;
+      const trend = SkillLeaderboard.calculateTrend(recent);
+      this._db.prepare('UPDATE skill_metrics SET trend = ? WHERE skill_path = ?').run(trend, r.skill_path);
+    }
+  }
+
+  // ------------------------------------------------------------------
+  // Domain aggregation
+  // ------------------------------------------------------------------
+
+  /** @private */
+  _updateDomainAggregates(domain) {
+    const rows = this._db.prepare('SELECT * FROM skill_metrics WHERE domain = ?').all(domain);
+
+    if (rows.length === 0) {
+      this._db.prepare('DELETE FROM domain_metrics WHERE domain = ?').run(domain);
+      return;
+    }
+
+    const techniqueCount = rows.length;
+    const totalInv = rows.reduce((s, r) => s + r.invocation_count, 0);
+    const totalSuccess = rows.reduce((s, r) => s + r.success_count, 0);
+    const avgSuccessRate = totalInv > 0 ? totalSuccess / totalInv : 0;
+    const scores = rows.map(r => r.avg_score).filter(s => Number.isFinite(s));
+    const avgScore = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : 0;
+
+    const mitreTotal = _MITRE_TECHNIQUE_COUNTS[domain] || 0;
+    let coverage = mitreTotal > 0 ? techniqueCount / mitreTotal : 0;
+    coverage = Math.min(coverage, 1.0);
+
+    const best = rows.reduce((a, b) => a.avg_score >= b.avg_score ? a : b);
+    const worst = rows.reduce((a, b) => a.avg_score <= b.avg_score ? a : b);
+
+    this._db.prepare(`
+      INSERT INTO domain_metrics
+        (domain, technique_count, total_invocations, avg_success_rate, avg_score, coverage_score, strongest_technique, weakest_technique)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(domain) DO UPDATE SET
+        technique_count = excluded.technique_count,
+        total_invocations = excluded.total_invocations,
+        avg_success_rate = excluded.avg_success_rate,
+        avg_score = excluded.avg_score,
+        coverage_score = excluded.coverage_score,
+        strongest_technique = excluded.strongest_technique,
+        weakest_technique = excluded.weakest_technique
+    `).run(
+      domain,
+      techniqueCount,
+      totalInv,
+      Math.round(avgSuccessRate * 10000) / 10000,
+      Math.round(avgScore * 10000) / 10000,
+      Math.round(coverage * 10000) / 10000,
+      best.technique,
+      worst.technique,
+    );
+  }
+
+  // ------------------------------------------------------------------
+  // Dashboard & reporting
+  // ------------------------------------------------------------------
+
+  getDashboard() {
+    const skills = this._db.prepare('SELECT * FROM skill_metrics').all();
+    const domains = this._db.prepare('SELECT * FROM domain_metrics ORDER BY avg_score DESC').all();
+
+    if (skills.length === 0) {
+      return {
+        totalSkills: 0,
+        avgScore: 0,
+        totalInvocations: 0,
+        topDomains: [],
+        worstDomains: [],
+        unusedCount: 0,
+        trendSummary: { improving: 0, stable: 0, declining: 0 },
+      };
+    }
+
+    const avgScore = skills.reduce((s, r) => s + r.avg_score, 0) / skills.length;
+    const totalInv = skills.reduce((s, r) => s + r.invocation_count, 0);
+    const unused = this.getUnusedSkills(30).length;
+
+    const trendCounts = { improving: 0, stable: 0, declining: 0 };
+    for (const r of skills) {
+      const t = r.trend in trendCounts ? r.trend : 'stable';
+      trendCounts[t] += 1;
+    }
+
+    const topDomains = domains.slice(0, 5).map(d => ({
+      domain: d.domain,
+      avgScore: d.avg_score,
+      coverage: d.coverage_score,
+    }));
+    const worstDomains = domains.slice(-5).reverse().map(d => ({
+      domain: d.domain,
+      avgScore: d.avg_score,
+      coverage: d.coverage_score,
+    }));
+
+    return {
+      totalSkills: skills.length,
+      avgScore: Math.round(avgScore * 10000) / 10000,
+      totalInvocations: totalInv,
+      topDomains,
+      worstDomains,
+      unusedCount: unused,
+      trendSummary: trendCounts,
+    };
+  }
+
+  exportReport() {
+    const skills = this._db.prepare('SELECT * FROM skill_metrics ORDER BY avg_score DESC').all();
+    const entries = skills.map((r, i) => {
+      const inv = r.invocation_count || 1;
+      return {
+        rank: i + 1,
+        skillPath: r.skill_path,
+        domain: r.domain,
+        technique: r.technique,
+        invocationCount: r.invocation_count,
+        successCount: r.success_count,
+        failureCount: r.failure_count,
+        avgScore: r.avg_score,
+        lastUsed: r.last_used,
+        lastScore: r.last_score,
+        trend: r.trend,
+        successRate: Math.round((r.success_count / inv) * 10000) / 10000,
+      };
+    });
+
+    return JSON.stringify({
+      generatedAt: new Date().toISOString(),
+      totalSkills: entries.length,
+      dashboard: this.getDashboard(),
+      skills: entries,
+    }, null, 2);
+  }
+
+  // ------------------------------------------------------------------
+  // EWA / ACE-style Online Adaptation
+  // ------------------------------------------------------------------
+
+  /**
+   * Compute exponentially-decayed score.
+   * @param {string} skillPath
+   * @param {number} [decayFactor=0.95]
+   * @param {number} [window=50]
+   * @returns {number}
+   */
+  decayedScore(skillPath, decayFactor = 0.95, window = 50) {
+    const row = this._db.prepare('SELECT scores_history FROM skill_metrics WHERE skill_path = ?').get(skillPath);
+    if (!row) return 0;
+    const history = JSON.parse(row.scores_history);
+    if (history.length === 0) return 0;
+    const recent = history.slice(-window);
+    const n = recent.length;
+    const weights = Array.from({ length: n }, (_, i) => decayFactor ** (n - 1 - i));
+    const totalWeight = weights.reduce((a, b) => a + b, 0);
+    if (totalWeight === 0) return 0;
+    return weights.reduce((sum, w, i) => sum + w * recent[i], 0) / totalWeight;
+  }
+
+  /** Close the database connection. */
+  close() {
+    this._db.close();
+  }
+}