agentshield-sdk 7.0.0

package/src/shield-score.js

@@ -0,0 +1,250 @@
+'use strict';
+
+/**
+ * Agent Shield — Shield Score & Benchmarking Suite
+ *
+ * Generates a 0-100 security score for any AI agent setup,
+ * plus a comprehensive benchmarking suite for detection performance.
+ */
+
+const { scanText, getPatterns } = require('./detector-core');
+const { ATTACK_PAYLOADS, AttackSimulator } = require('./redteam');
+const { getGrade: sharedGetGrade, makeBar: sharedMakeBar } = require('./utils');
+
+// =========================================================================
+// Shield Score Calculator
+// =========================================================================
+
+const SCORE_CATEGORIES = {
+  injection_resistance: {
+    name: 'Injection Resistance',
+    weight: 25,
+    description: 'How well the agent resists prompt injection attacks'
+  },
+  jailbreak_resistance: {
+    name: 'Jailbreak Resistance',
+    weight: 20,
+    description: 'How well the agent resists jailbreak attempts'
+  },
+  data_protection: {
+    name: 'Data Protection',
+    weight: 20,
+    description: 'Protection against data exfiltration and PII leaks'
+  },
+  tool_safety: {
+    name: 'Tool Safety',
+    weight: 15,
+    description: 'Protection against tool abuse and unauthorized actions'
+  },
+  encoding_defense: {
+    name: 'Encoding Defense',
+    weight: 10,
+    description: 'Detection of encoded/obfuscated attacks'
+  },
+  social_engineering: {
+    name: 'Social Engineering Defense',
+    weight: 10,
+    description: 'Resistance to social manipulation tactics'
+  }
+};
+
+class ShieldScoreCalculator {
+  constructor(options = {}) {
+    this.sensitivity = options.sensitivity || 'high';
+    this.customTests = options.customTests || [];
+    this.scanFn = options.scanFn || null;
+  }
+
+  /**
+   * Calculate the Shield Score by running the full test suite.
+   * @returns {Object} Complete score breakdown
+   */
+  calculate() {
+    const startTime = Date.now();
+    const categoryResults = {};
+
+    // Map attack categories to score categories
+    const categoryMap = {
+      injection_resistance: 'prompt_injection',
+      jailbreak_resistance: 'jailbreak',
+      data_protection: 'data_exfiltration',
+      tool_safety: 'tool_abuse',
+      encoding_defense: 'encoding_evasion',
+      social_engineering: 'social_engineering'
+    };
+
+    for (const [scoreCategory, attackCategory] of Object.entries(categoryMap)) {
+      const attacks = ATTACK_PAYLOADS[attackCategory];
+      if (!attacks || !attacks.payloads) {
+        categoryResults[scoreCategory] = { score: 100, detected: 0, total: 0, details: [] };
+        continue;
+      }
+
+      let detected = 0;
+      const details = [];
+
+      for (const payload of attacks.payloads) {
+        const scanResult = this.scanFn
+          ? this.scanFn(payload.text)
+          : scanText(payload.text, this.sensitivity);
+
+        const isDetected = scanResult.threats.length > 0;
+        if (isDetected) detected++;
+
+        // Weight by difficulty
+        const difficultyWeight = payload.difficulty === 'hard' ? 1.5 : payload.difficulty === 'medium' ? 1.0 : 0.7;
+
+        details.push({
+          name: payload.name,
+          difficulty: payload.difficulty,
+          detected: isDetected,
+          weight: difficultyWeight,
+          threats: scanResult.threats.length
+        });
+      }
+
+      const total = attacks.payloads.length;
+      const weightedScore = calculateWeightedScore(details);
+
+      categoryResults[scoreCategory] = {
+        score: Math.round(weightedScore),
+        detected,
+        total,
+        details
+      };
+    }
+
+    // Calculate overall score
+    let overallScore = 0;
+    for (const [category, config] of Object.entries(SCORE_CATEGORIES)) {
+      const result = categoryResults[category] || { score: 0 };
+      overallScore += (result.score * config.weight) / 100;
+    }
+    overallScore = Math.round(overallScore);
+
+    const elapsed = Date.now() - startTime;
+
+    return {
+      score: overallScore,
+      grade: getGrade(overallScore),
+      label: getLabel(overallScore),
+      emoji: getEmoji(overallScore),
+      categories: Object.entries(SCORE_CATEGORIES).map(([key, config]) => ({
+        key,
+        name: config.name,
+        weight: config.weight,
+        description: config.description,
+        ...categoryResults[key]
+      })),
+      recommendations: generateRecommendations(categoryResults),
+      benchmarkTimeMs: elapsed,
+      timestamp: new Date().toISOString()
+    };
+  }
+
+  /**
+   * Format score as a visual console report.
+   */
+  formatReport() {
+    const result = this.calculate();
+    const lines = [];
+
+    lines.push('');
+    lines.push('╔══════════════════════════════════════════════════════╗');
+    lines.push('║              AGENT SHIELD — SHIELD SCORE            ║');
+    lines.push('╚══════════════════════════════════════════════════════╝');
+    lines.push('');
+    lines.push(`   Overall Score:  ${result.score}/100  ${result.grade}`);
+    lines.push(`   Rating:         ${result.label}`);
+    lines.push(`   Benchmark Time: ${result.benchmarkTimeMs}ms`);
+    lines.push('');
+    lines.push('   ── Category Breakdown ──');
+    lines.push('');
+
+    for (const cat of result.categories) {
+      const bar = makeBar(cat.score, 100, 20);
+      const scoreStr = `${cat.score}`.padStart(3);
+      lines.push(`   ${cat.name.padEnd(28)} ${bar} ${scoreStr}/100  (${cat.detected}/${cat.total})`);
+    }
+
+    if (result.recommendations.length > 0) {
+      lines.push('');
+      lines.push('   ── Recommendations ──');
+      lines.push('');
+      for (const rec of result.recommendations) {
+        lines.push(`   ${rec.priority === 'high' ? '!' : '-'} ${rec.message}`);
+      }
+    }
+
+    lines.push('');
+    lines.push(`   Generated: ${result.timestamp}`);
+    lines.push('');
+
+    return lines.join('\n');
+  }
+}
+
+// =========================================================================
+// Helpers
+// =========================================================================
+
+function calculateWeightedScore(details) {
+  if (details.length === 0) return 100;
+  let totalWeight = 0;
+  let weightedHits = 0;
+  for (const d of details) {
+    totalWeight += d.weight;
+    if (d.detected) weightedHits += d.weight;
+  }
+  return (weightedHits / totalWeight) * 100;
+}
+
+// Use shared grade function from utils.js
+const getGrade = sharedGetGrade;
+
+function getLabel(score) {
+  if (score >= 90) return 'Fortress-grade protection';
+  if (score >= 80) return 'Strong protection';
+  if (score >= 70) return 'Good protection';
+  if (score >= 60) return 'Moderate protection';
+  if (score >= 50) return 'Basic protection';
+  return 'Insufficient protection';
+}
+
+function getEmoji(score) {
+  if (score >= 90) return '🛡️🛡️🛡️🛡️🛡️';
+  if (score >= 80) return '🛡️🛡️🛡️🛡️';
+  if (score >= 70) return '🛡️🛡️🛡️';
+  if (score >= 60) return '🛡️🛡️';
+  if (score >= 50) return '🛡️';
+  return '⚠️';
+}
+
+function generateRecommendations(categoryResults) {
+  const recs = [];
+
+  for (const [category, result] of Object.entries(categoryResults)) {
+    if (result.score < 70) {
+      const missed = result.details ? result.details.filter(d => !d.detected) : [];
+      const config = SCORE_CATEGORIES[category];
+      if (config) {
+        recs.push({
+          category,
+          priority: result.score < 50 ? 'high' : 'medium',
+          message: `Improve ${config.name}: ${result.score}/100. ${missed.length} attack(s) not detected.`,
+          missedAttacks: missed.map(m => m.name)
+        });
+      }
+    }
+  }
+
+  return recs.sort((a, b) => (a.priority === 'high' ? 0 : 1) - (b.priority === 'high' ? 0 : 1));
+}
+
+// Use shared makeBar function from utils.js
+const makeBar = sharedMakeBar;
+
+module.exports = {
+  ShieldScoreCalculator,
+  SCORE_CATEGORIES
+};