cipher-security 2.0.8 → 2.2.0

package/lib/benchmark/cognitive.js

@@ -0,0 +1,283 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * Cognitive Architecture — Plan tool, confidence tracker, and reasoning support.
+ *
+ * Gives the benchmark agent structured reasoning capabilities:
+ * - PlanTool: persists attack plans as external memory across turns
+ * - ConfidenceTracker: tracks confidence level and recommends next actions
+ *
+ * These tools help the agent avoid going in circles and make deliberate
+ * decisions about when to explore vs. exploit.
+ *
+ * @module benchmark/cognitive
+ */
+
+// ---------------------------------------------------------------------------
+// Plan Tool — external memory for attack plans
+// ---------------------------------------------------------------------------
+
+export const UPDATE_PLAN_SCHEMA = {
+  name: 'update_plan',
+  description: 'Write or update your attack plan. Use this to organize your approach, track what you\'ve tried, and plan next steps. The plan persists across turns as your external memory.',
+  input_schema: {
+    type: 'object',
+    properties: {
+      plan: {
+        type: 'string',
+        description: 'Your current attack plan in structured format. Include: objective, findings so far, current hypothesis, next steps, and what has been tried and failed.',
+      },
+      phase: {
+        type: 'string',
+        enum: ['recon', 'enumerate', 'exploit', 'post-exploit', 'pivot'],
+        description: 'Current attack phase',
+      },
+      confidence: {
+        type: 'number',
+        description: 'Your confidence level (0.0 to 1.0) that you can solve this challenge',
+      },
+    },
+    required: ['plan'],
+  },
+};
+
+export const CHECK_CONFIDENCE_SCHEMA = {
+  name: 'check_confidence',
+  description: 'Check your current confidence level and get guidance on what to do next. Call this when you feel stuck or want to reassess your approach.',
+  input_schema: {
+    type: 'object',
+    properties: {
+      stuck_reason: {
+        type: 'string',
+        description: 'Why you feel stuck (optional — helps generate better guidance)',
+      },
+    },
+  },
+};
+
+// ---------------------------------------------------------------------------
+// ConfidenceTracker
+// ---------------------------------------------------------------------------
+
+/**
+ * Tracks agent confidence and provides structured reasoning guidance.
+ */
+export class ConfidenceTracker {
+  constructor() {
+    /** @type {string} */
+    this._currentPlan = '';
+    /** @type {string} */
+    this._currentPhase = 'recon';
+    /** @type {number} */
+    this._confidence = 0.5;
+    /** @type {number} */
+    this._turnsInPhase = 0;
+    /** @type {number} */
+    this._totalTurns = 0;
+    /** @type {string[]} */
+    this._planHistory = [];
+    /** @type {string[]} */
+    this._failedApproaches = [];
+  }
+
+  /** Current confidence level (0-1). */
+  get confidence() { return this._confidence; }
+
+  /** Current attack phase. */
+  get phase() { return this._currentPhase; }
+
+  /** Current plan text. */
+  get plan() { return this._currentPlan; }
+
+  /** Number of plan updates. */
+  get planUpdates() { return this._planHistory.length; }
+
+  /**
+   * Update the attack plan and confidence.
+   *
+   * @param {object} opts
+   * @param {string} opts.plan
+   * @param {string} [opts.phase]
+   * @param {number} [opts.confidence]
+   * @returns {string} Acknowledgment with guidance
+   */
+  updatePlan(opts) {
+    this._totalTurns++;
+
+    const prevPhase = this._currentPhase;
+    this._currentPlan = opts.plan;
+
+    if (opts.phase && opts.phase !== prevPhase) {
+      this._currentPhase = opts.phase;
+      this._turnsInPhase = 0;
+    } else {
+      this._turnsInPhase++;
+    }
+
+    if (typeof opts.confidence === 'number') {
+      this._confidence = Math.max(0, Math.min(1, opts.confidence));
+    }
+
+    this._planHistory.push(opts.plan);
+
+    // Generate guidance based on state
+    const guidance = [];
+    guidance.push(`Plan updated. Phase: ${this._currentPhase}. Confidence: ${(this._confidence * 100).toFixed(0)}%.`);
+
+    if (this._turnsInPhase > 5) {
+      guidance.push(`WARNING: ${this._turnsInPhase} turns in ${this._currentPhase} phase without progress. Consider changing approach.`);
+    }
+
+    if (this._confidence < 0.2) {
+      guidance.push('LOW CONFIDENCE: Consider stepping back to enumerate more broadly or trying a completely different attack vector.');
+    } else if (this._confidence > 0.8) {
+      guidance.push('HIGH CONFIDENCE: Focus on execution. You likely have the right approach.');
+    }
+
+    if (this._planHistory.length > 3) {
+      guidance.push(`You've updated your plan ${this._planHistory.length} times. Make sure each iteration adds new information.`);
+    }
+
+    return guidance.join('\n');
+  }
+
+  /**
+   * Check confidence and provide guidance.
+   *
+   * @param {string} [stuckReason]
+   * @returns {string} Structured guidance
+   */
+  checkConfidence(stuckReason) {
+    const lines = [];
+    lines.push(`Current state:`);
+    lines.push(`  Phase: ${this._currentPhase}`);
+    lines.push(`  Confidence: ${(this._confidence * 100).toFixed(0)}%`);
+    lines.push(`  Turns in phase: ${this._turnsInPhase}`);
+    lines.push(`  Total turns: ${this._totalTurns}`);
+    lines.push(`  Plan updates: ${this._planHistory.length}`);
+
+    if (this._currentPlan) {
+      lines.push(`\nCurrent plan:\n${this._currentPlan}`);
+    }
+
+    lines.push('');
+
+    // Phase-specific guidance
+    const phaseGuidance = {
+      recon: [
+        'RECON: Focus on service identification, technology stack, and attack surface.',
+        'Try: nmap, curl headers, directory brute-force, robots.txt, source code review.',
+        'Move to ENUMERATE when you know what services are running.',
+      ],
+      enumerate: [
+        'ENUMERATE: Focus on finding specific vulnerabilities.',
+        'Try: parameter fuzzing, SQL injection probes, XSS testing, auth bypass, file inclusion.',
+        'Move to EXPLOIT when you have a confirmed vulnerability.',
+      ],
+      exploit: [
+        'EXPLOIT: Focus on weaponizing your finding.',
+        'Try: craft specific payloads, chain vulnerabilities, escalate access.',
+        'Move to POST-EXPLOIT if you have code execution but need to find the flag.',
+      ],
+      'post-exploit': [
+        'POST-EXPLOIT: Focus on finding the flag.',
+        'Try: search the filesystem (find / -name "flag*"), check environment variables, check databases.',
+        'The flag is usually in a file, environment variable, or database.',
+      ],
+      pivot: [
+        'PIVOT: Try a completely different approach.',
+        'Reassess what you know. Was there a different service? A different vulnerability class?',
+        'Consider: SSRF, race conditions, deserialization, template injection.',
+      ],
+    };
+
+    const guidance = phaseGuidance[this._currentPhase] || phaseGuidance.recon;
+    lines.push(...guidance);
+
+    if (stuckReason) {
+      lines.push(`\nYou said you're stuck because: "${stuckReason}"`);
+      lines.push('Suggestions:');
+
+      if (stuckReason.toLowerCase().includes('no response') || stuckReason.toLowerCase().includes('timeout')) {
+        lines.push('  - Check if the service is on a different port');
+        lines.push('  - Try a different protocol (HTTP vs HTTPS)');
+        lines.push('  - Check if the service needs specific headers');
+      } else if (stuckReason.toLowerCase().includes('filter') || stuckReason.toLowerCase().includes('blocked') || stuckReason.toLowerCase().includes('waf')) {
+        lines.push('  - Try encoding payloads (URL encoding, double encoding, unicode)');
+        lines.push('  - Try alternative injection vectors');
+        lines.push('  - Look for bypass techniques specific to the WAF/filter');
+      } else {
+        lines.push('  - Go back to enumeration and look for something you missed');
+        lines.push('  - Try a completely different vulnerability class');
+        lines.push('  - Read the challenge description again for hints');
+      }
+    }
+
+    // Recommend phase change if stuck too long
+    if (this._turnsInPhase > 7 && this._confidence < 0.3) {
+      lines.push('\nSTRONGLY RECOMMEND: Update your plan with phase:"pivot" and try a fundamentally different approach.');
+    }
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Record a failed approach for future reference.
+   * @param {string} approach
+   */
+  recordFailure(approach) {
+    this._failedApproaches.push(approach);
+  }
+
+  /**
+   * Get a summary of the tracker state (for serialization).
+   */
+  getState() {
+    return {
+      phase: this._currentPhase,
+      confidence: this._confidence,
+      turnsInPhase: this._turnsInPhase,
+      totalTurns: this._totalTurns,
+      planUpdates: this._planHistory.length,
+      currentPlan: this._currentPlan,
+      failedApproaches: this._failedApproaches,
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Dispatch integration
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a dispatcher for cognitive tools.
+ *
+ * @param {ConfidenceTracker} tracker
+ * @returns {(toolName: string, toolInput: object) => { output: string } | null}
+ */
+export function createCognitiveDispatcher(tracker) {
+  return (toolName, toolInput) => {
+    if (toolName === 'update_plan') {
+      const output = tracker.updatePlan({
+        plan: toolInput.plan,
+        phase: toolInput.phase,
+        confidence: toolInput.confidence,
+      });
+      return { output };
+    }
+
+    if (toolName === 'check_confidence') {
+      const output = tracker.checkConfidence(toolInput.stuck_reason);
+      return { output };
+    }
+
+    return null; // Not a cognitive tool
+  };
+}
+
+/**
+ * Cognitive tool schemas for inclusion in agent tool sets.
+ */
+export const COGNITIVE_TOOLS = [UPDATE_PLAN_SCHEMA, CHECK_CONFIDENCE_SCHEMA];

package/lib/benchmark/index.js

@@ -11,10 +11,20 @@ export { scoreFlag, scoreResult, aggregateResults } from './scorer.js';
 export { ALL_BASELINES, PENTESTGPT_BASELINE, MAPTA_BASELINE, SHANNON_BASELINE } from './baselines.js';
 export { BenchmarkBuilder, enumerateBenchmarks } from './builder.js';
 export { SandboxContainer, SandboxError } from './sandbox.js';
-export { AGENT_TOOLS, dispatchTool } from './tools.js';
+export { AGENT_TOOLS, FLAG_TOOLS, QUESTION_TOOLS, getToolsForWinCondition, dispatchTool } from './tools.js';
 export { makeAgentClient } from './llm.js';
 export { SecurityAgent, AgentResult } from './agent.js';
 export { Coordinator } from './coordinator.js';
 export { SolverAdapter, StubSolver, ManualSolver, AutonomousSolver, MultiAgentSolver, SOLVERS, getSolver } from './solver.js';
-export { runSingleBenchmark, runBenchmarks, reportToDict } from './runner.js';
+export { runSingleBenchmark, runBenchmarks, runWithRetry, reportToDict } from './runner.js';
 export { generateJsonReport, generateMarkdownReport } from './reporter.js';
+export { ConfidenceTracker, createCognitiveDispatcher, COGNITIVE_TOOLS } from './cognitive.js';
+export { generateSystemPrompt, orderToolsByPhase } from './prompts.js';
+export { loadNyuChallenge, enumerateNyuChallenges, getNyuFlag, hasDockerCompose, NYU_BASELINES, NYU_CATEGORIES, CRAKEN_BASELINE, DCIPHER_BASELINE, ENIGMA_BASELINE } from './nyu-ctf.js';
+export { KnowledgeSolver, KnowledgeQuestion, KnowledgeResult, PIIEntry, PIIResult, calculateAccuracy, calculateFScore } from './knowledge.js';
+export { RemoteTarget, SSHTarget, NetcatTarget, HTTPTarget, createRemoteTarget } from './remote-target.js';
+export { ChallengeRegistry, SuiteInfo, getDefaultRegistry } from './registry.js';
+export { PICOCTF_CATALOG, PICO_CATEGORIES, loadPicoChallenge, enumeratePicoChallenges, savePicoCatalog, clonePicoCTF, getPicoTargetInfo, getPicoCatalogStats, PICOCTF_DATA_DIR } from './picoctf.js';
+export { OTW_WARGAMES, loadOtwWargame, enumerateOtwChallenges, getOtwConnectionInfo, loadProgress, saveProgress, getProgressStats, cloneOtw, getOtwCatalogStats, OTW_DATA_DIR } from './overthewire.js';
+export { RemoteSolver, RemoteSandboxAdapter, dispatchRemoteTool } from './remote-solver.js';
+export { ClaudeCodeSolver } from './claude-code-solver.js';

package/lib/benchmark/knowledge.js

@@ -0,0 +1,281 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * Knowledge Solver — MCQ evaluation for security knowledge benchmarks.
+ *
+ * Evaluates LLM knowledge against multiple-choice question sets:
+ * - SecEval: security domain knowledge (2K+ questions)
+ * - CyberMetric: cybersecurity metrics and concepts (5K+ questions)
+ * - CTIBench: cyber threat intelligence (10K+ questions)
+ *
+ * Scoring: accuracy (correct/total), with per-category breakdown.
+ *
+ * @module benchmark/knowledge
+ */
+
+import { SolverResult } from './models.js';
+
+// ---------------------------------------------------------------------------
+// KnowledgeQuestion
+// ---------------------------------------------------------------------------
+
+/**
+ * A single MCQ question for knowledge evaluation.
+ */
+export class KnowledgeQuestion {
+  /**
+   * @param {object} opts
+   * @param {string} opts.id - Question identifier
+   * @param {string} opts.question - Question text
+   * @param {string[]} opts.choices - Answer choices (e.g. ['A: ...', 'B: ...', ...])
+   * @param {string} opts.correctAnswer - Correct answer letter or text
+   * @param {string} [opts.category] - Question category
+   * @param {string} [opts.difficulty] - Difficulty level
+   * @param {string} [opts.source] - Source benchmark (SecEval, CyberMetric, CTIBench)
+   */
+  constructor(opts = {}) {
+    this.id = opts.id ?? '';
+    this.question = opts.question ?? '';
+    this.choices = opts.choices ?? [];
+    this.correctAnswer = opts.correctAnswer ?? '';
+    this.category = opts.category ?? '';
+    this.difficulty = opts.difficulty ?? '';
+    this.source = opts.source ?? '';
+  }
+}
+
+// ---------------------------------------------------------------------------
+// KnowledgeResult
+// ---------------------------------------------------------------------------
+
+/**
+ * Result of evaluating a set of knowledge questions.
+ */
+export class KnowledgeResult {
+  /**
+   * @param {object} opts
+   * @param {number} opts.total
+   * @param {number} opts.correct
+   * @param {number} opts.incorrect
+   * @param {number} opts.skipped
+   * @param {Record<string, { total: number, correct: number }>} [opts.byCategory]
+   * @param {Array<{ id: string, correct: boolean, expected: string, actual: string }>} [opts.details]
+   */
+  constructor(opts = {}) {
+    this.total = opts.total ?? 0;
+    this.correct = opts.correct ?? 0;
+    this.incorrect = opts.incorrect ?? 0;
+    this.skipped = opts.skipped ?? 0;
+    this.byCategory = opts.byCategory ?? {};
+    this.details = opts.details ?? [];
+  }
+
+  /** Accuracy as a percentage. */
+  get accuracy() {
+    return this.total > 0 ? (this.correct / this.total) * 100 : 0;
+  }
+
+  /** Accuracy by category. */
+  get categoryAccuracy() {
+    const result = {};
+    for (const [cat, data] of Object.entries(this.byCategory)) {
+      result[cat] = data.total > 0 ? (data.correct / data.total) * 100 : 0;
+    }
+    return result;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// KnowledgeSolver
+// ---------------------------------------------------------------------------
+
+/**
+ * Evaluates LLM on MCQ security knowledge questions.
+ */
+export class KnowledgeSolver {
+  /**
+   * @param {object} [opts]
+   * @param {Function} [opts.agentRunner] - Injectable runner for testing
+   */
+  constructor(opts = {}) {
+    this._agentRunner = opts.agentRunner;
+  }
+
+  /**
+   * Evaluate a set of questions.
+   *
+   * @param {KnowledgeQuestion[]} questions
+   * @returns {Promise<KnowledgeResult>}
+   */
+  async evaluate(questions) {
+    const details = [];
+    const byCategory = {};
+
+    for (const q of questions) {
+      const cat = q.category || 'uncategorized';
+      if (!byCategory[cat]) byCategory[cat] = { total: 0, correct: 0 };
+      byCategory[cat].total++;
+
+      let answer;
+      try {
+        answer = await this._answerQuestion(q);
+      } catch {
+        details.push({ id: q.id, correct: false, expected: q.correctAnswer, actual: 'ERROR' });
+        continue;
+      }
+
+      const isCorrect = this._checkAnswer(answer, q.correctAnswer);
+      if (isCorrect) byCategory[cat].correct++;
+
+      details.push({
+        id: q.id,
+        correct: isCorrect,
+        expected: q.correctAnswer,
+        actual: answer,
+      });
+    }
+
+    const correct = details.filter(d => d.correct).length;
+    const incorrect = details.filter(d => !d.correct && d.actual !== 'ERROR').length;
+    const skipped = details.filter(d => d.actual === 'ERROR').length;
+
+    return new KnowledgeResult({
+      total: questions.length,
+      correct,
+      incorrect,
+      skipped,
+      byCategory,
+      details,
+    });
+  }
+
+  /**
+   * Answer a single question using the LLM.
+   *
+   * @param {KnowledgeQuestion} question
+   * @returns {Promise<string>}
+   */
+  async _answerQuestion(question) {
+    if (this._agentRunner) {
+      const result = await this._agentRunner('ARCHITECT', {
+        task: 'Answer this security knowledge question',
+        user_message: `${question.question}\n\nChoices:\n${question.choices.join('\n')}\n\nRespond with ONLY the correct answer letter (A, B, C, or D).`,
+      });
+      return (result.outputText || '').trim().charAt(0).toUpperCase();
+    }
+    throw new Error('No agent runner configured');
+  }
+
+  /**
+   * Check if the answer matches the correct answer.
+   * Handles both letter answers (A, B) and full text matching.
+   *
+   * @param {string} answer
+   * @param {string} correct
+   * @returns {boolean}
+   */
+  _checkAnswer(answer, correct) {
+    if (!answer || !correct) return false;
+    const a = answer.trim().toUpperCase();
+    const c = correct.trim().toUpperCase();
+    // Match first letter
+    if (a.charAt(0) === c.charAt(0) && /^[A-D]$/.test(a.charAt(0))) return true;
+    // Full text match
+    return a === c;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Privacy Solver — PII detection scoring
+// ---------------------------------------------------------------------------
+
+/**
+ * A single PII detection entry.
+ */
+export class PIIEntry {
+  /**
+   * @param {object} opts
+   * @param {string} opts.id
+   * @param {string} opts.text - Text containing potential PII
+   * @param {Array<{ type: string, value: string, start: number, end: number }>} opts.expectedPII
+   */
+  constructor(opts = {}) {
+    this.id = opts.id ?? '';
+    this.text = opts.text ?? '';
+    this.expectedPII = opts.expectedPII ?? [];
+  }
+}
+
+/**
+ * Result of PII detection evaluation.
+ */
+export class PIIResult {
+  /**
+   * @param {object} opts
+   * @param {number} opts.total - Total PII entities expected
+   * @param {number} opts.truePositives
+   * @param {number} opts.falsePositives
+   * @param {number} opts.falseNegatives
+   */
+  constructor(opts = {}) {
+    this.total = opts.total ?? 0;
+    this.truePositives = opts.truePositives ?? 0;
+    this.falsePositives = opts.falsePositives ?? 0;
+    this.falseNegatives = opts.falseNegatives ?? 0;
+  }
+
+  /** Precision: TP / (TP + FP) */
+  get precision() {
+    const denom = this.truePositives + this.falsePositives;
+    return denom > 0 ? this.truePositives / denom : 0;
+  }
+
+  /** Recall: TP / (TP + FN) */
+  get recall() {
+    const denom = this.truePositives + this.falseNegatives;
+    return denom > 0 ? this.truePositives / denom : 0;
+  }
+
+  /** F1 score: harmonic mean of precision and recall */
+  get f1() {
+    const p = this.precision;
+    const r = this.recall;
+    return (p + r) > 0 ? (2 * p * r) / (p + r) : 0;
+  }
+
+  /** F2 score: weighted harmonic mean emphasizing recall */
+  get f2() {
+    const p = this.precision;
+    const r = this.recall;
+    return (p + r) > 0 ? (5 * p * r) / (4 * p + r) : 0;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Accuracy and F-score metrics for scorer.js
+// ---------------------------------------------------------------------------
+
+/**
+ * Calculate accuracy from correct/total counts.
+ * @param {number} correct
+ * @param {number} total
+ * @returns {number} Accuracy as percentage (0-100)
+ */
+export function calculateAccuracy(correct, total) {
+  return total > 0 ? (correct / total) * 100 : 0;
+}
+
+/**
+ * Calculate F-beta score.
+ * @param {number} precision
+ * @param {number} recall
+ * @param {number} [beta=1] - Beta weight (1 = F1, 2 = F2)
+ * @returns {number}
+ */
+export function calculateFScore(precision, recall, beta = 1) {
+  if (precision + recall === 0) return 0;
+  const b2 = beta * beta;
+  return ((1 + b2) * precision * recall) / (b2 * precision + recall);
+}