cipher-security 2.0.8 → 2.2.0

package/lib/execution/council.js

@@ -0,0 +1,434 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * LLM Council — Multi-model consensus via parallel evaluation and synthesis.
+ *
+ * 3-stage pipeline:
+ *   1. Parallel evaluation: N members independently evaluate the task
+ *   2. Cross-ranking: each member scores the others' responses
+ *   3. Synthesis: highest-ranked response used as basis for final output
+ *
+ * Designed for high-stakes decisions where single-model bias is a risk:
+ * exploit/no-exploit, severity ratings, compliance findings, threat models.
+ *
+ * Cost: 2N+1 API calls per invocation (N evaluate + N cross-rank + 1 synthesize).
+ *
+ * @module execution/council
+ */
+
+import { ModeAgentResult } from '../autonomous/framework.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+/**
+ * Result from a council evaluation.
+ */
+export class CouncilResult {
+  /**
+   * @param {object} opts
+   * @param {string} opts.task - Original task
+   * @param {CouncilMemberResponse[]} [opts.responses] - Individual member responses
+   * @param {number[][]} [opts.rankings] - Cross-ranking matrix (member × member scores)
+   * @param {string} [opts.synthesis] - Final synthesized output
+   * @param {number} [opts.confidence] - Consensus confidence 0-1
+   * @param {number} [opts.totalTokensIn] - Total input tokens
+   * @param {number} [opts.totalTokensOut] - Total output tokens
+   * @param {number} [opts.totalDurationS] - Wall-clock seconds
+   * @param {number} [opts.estimatedCostUSD] - Estimated cost
+   * @param {string|null} [opts.error] - Top-level error
+   */
+  constructor(opts = {}) {
+    this.task = opts.task ?? '';
+    /** @type {CouncilMemberResponse[]} */
+    this.responses = opts.responses ?? [];
+    /** @type {number[][]} */
+    this.rankings = opts.rankings ?? [];
+    this.synthesis = opts.synthesis ?? '';
+    this.confidence = opts.confidence ?? 0;
+    this.totalTokensIn = opts.totalTokensIn ?? 0;
+    this.totalTokensOut = opts.totalTokensOut ?? 0;
+    this.totalDurationS = opts.totalDurationS ?? 0;
+    this.estimatedCostUSD = opts.estimatedCostUSD ?? 0;
+    this.error = opts.error ?? null;
+  }
+
+  /** Number of council members. */
+  get memberCount() {
+    return this.responses.length;
+  }
+}
+
+/**
+ * Individual council member response.
+ */
+export class CouncilMemberResponse {
+  /**
+   * @param {object} opts
+   * @param {number} opts.memberId - 0-indexed member number
+   * @param {string} opts.response - Member's evaluation text
+   * @param {number} opts.tokensIn
+   * @param {number} opts.tokensOut
+   * @param {number} opts.durationS
+   * @param {string|null} [opts.error]
+   */
+  constructor(opts = {}) {
+    this.memberId = opts.memberId ?? 0;
+    this.response = opts.response ?? '';
+    this.tokensIn = opts.tokensIn ?? 0;
+    this.tokensOut = opts.tokensOut ?? 0;
+    this.durationS = opts.durationS ?? 0;
+    this.error = opts.error ?? null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Cost estimation
+// ---------------------------------------------------------------------------
+
+/** Rough token costs per 1M tokens (input/output) by model family. */
+const MODEL_COSTS = {
+  'claude': { input: 3.0, output: 15.0 },   // Claude Sonnet
+  'gpt-4': { input: 5.0, output: 15.0 },
+  'ollama': { input: 0, output: 0 },         // Local, no API cost
+  'default': { input: 3.0, output: 15.0 },
+};
+
+/**
+ * Estimate council cost before execution.
+ *
+ * @param {string} task - Task text
+ * @param {object} [opts]
+ * @param {number} [opts.members=3] - Number of council members
+ * @param {string} [opts.backend='default'] - Backend for cost lookup
+ * @returns {{ estimatedTokensIn: number, estimatedTokensOut: number, estimatedCostUSD: number, apiCalls: number }}
+ */
+export function estimateCouncilCost(task, opts = {}) {
+  const members = opts.members ?? 3;
+  const backend = opts.backend ?? 'default';
+
+  // Rough heuristic: task tokens + system prompt ≈ 500 tokens per call
+  const taskTokens = Math.ceil(task.length / 4);
+  const perCallIn = taskTokens + 500;
+  const perCallOut = 800; // average response length
+
+  // 2N+1 calls: N evaluate + N cross-rank + 1 synthesize
+  const apiCalls = 2 * members + 1;
+  const totalIn = perCallIn * apiCalls;
+  const totalOut = perCallOut * apiCalls;
+
+  const costKey = Object.keys(MODEL_COSTS).find(k => backend.includes(k)) || 'default';
+  const costs = MODEL_COSTS[costKey];
+  const costUSD = (totalIn / 1_000_000) * costs.input + (totalOut / 1_000_000) * costs.output;
+
+  return {
+    estimatedTokensIn: totalIn,
+    estimatedTokensOut: totalOut,
+    estimatedCostUSD: Math.round(costUSD * 10000) / 10000,
+    apiCalls,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// LLMCouncil
+// ---------------------------------------------------------------------------
+
+/**
+ * Multi-model consensus engine.
+ */
+export class LLMCouncil {
+  /**
+   * @param {object} opts
+   * @param {number} [opts.members=3] - Number of council members
+   * @param {Function} [opts.agentRunner] - Injectable runner for evaluation calls
+   */
+  constructor(opts = {}) {
+    this._members = opts.members ?? 3;
+    this._agentRunner = opts.agentRunner ?? null;
+  }
+
+  /**
+   * Run full council evaluation: evaluate → cross-rank → synthesize.
+   *
+   * @param {string} task
+   * @param {object} [opts]
+   * @param {string|null} [opts.backend] - LLM backend override
+   * @param {boolean} [opts.dryRun=false] - Return cost estimate only
+   * @returns {Promise<CouncilResult>}
+   */
+  async evaluate(task, opts = {}) {
+    if (opts.dryRun) {
+      const estimate = estimateCouncilCost(task, {
+        members: this._members,
+        backend: opts.backend || 'default',
+      });
+      return new CouncilResult({
+        task,
+        estimatedCostUSD: estimate.estimatedCostUSD,
+        totalTokensIn: estimate.estimatedTokensIn,
+        totalTokensOut: estimate.estimatedTokensOut,
+      });
+    }
+
+    const startTime = performance.now() / 1000;
+    const runner = this._agentRunner || (await import('../autonomous/runner.js')).runAutonomous;
+
+    // Stage 1: Parallel evaluation
+    const evaluationPromises = [];
+    for (let i = 0; i < this._members; i++) {
+      evaluationPromises.push(
+        this._evaluateMember(i, task, runner, opts.backend)
+      );
+    }
+    const responses = await Promise.allSettled(evaluationPromises);
+    const memberResponses = responses.map((settled, i) => {
+      if (settled.status === 'fulfilled') return settled.value;
+      return new CouncilMemberResponse({
+        memberId: i,
+        error: settled.reason?.message || 'Unknown error',
+      });
+    });
+
+    // Check if we have enough successful responses
+    const successfulResponses = memberResponses.filter(r => !r.error);
+    if (successfulResponses.length === 0) {
+      return new CouncilResult({
+        task,
+        responses: memberResponses,
+        error: 'All council members failed',
+        totalDurationS: performance.now() / 1000 - startTime,
+      });
+    }
+
+    // Stage 2: Cross-ranking
+    const rankings = await this._crossRank(successfulResponses, task, runner, opts.backend);
+
+    // Stage 3: Synthesis
+    const synthesis = await this._synthesize(successfulResponses, rankings, task, runner, opts.backend);
+
+    // Aggregate metrics
+    let totalIn = 0, totalOut = 0;
+    for (const r of memberResponses) {
+      totalIn += r.tokensIn;
+      totalOut += r.tokensOut;
+    }
+    // Add ranking + synthesis tokens (estimated)
+    totalIn += synthesis.tokensIn || 0;
+    totalOut += synthesis.tokensOut || 0;
+
+    const confidence = this._computeConfidence(rankings, successfulResponses.length);
+
+    return new CouncilResult({
+      task,
+      responses: memberResponses,
+      rankings,
+      synthesis: synthesis.text,
+      confidence,
+      totalTokensIn: totalIn,
+      totalTokensOut: totalOut,
+      totalDurationS: performance.now() / 1000 - startTime,
+      estimatedCostUSD: estimateCouncilCost(task, {
+        members: this._members,
+        backend: opts.backend || 'default',
+      }).estimatedCostUSD,
+    });
+  }
+
+  // -- internal -----------------------------------------------------------
+
+  /**
+   * Evaluate a single member.
+   * @param {number} memberId
+   * @param {string} task
+   * @param {Function} runner
+   * @param {string|null} backend
+   * @returns {Promise<CouncilMemberResponse>}
+   */
+  async _evaluateMember(memberId, task, runner, backend) {
+    const startTime = performance.now() / 1000;
+
+    const taskInput = {
+      task,
+      user_message: `[Council Member ${memberId + 1}] Evaluate the following independently:\n\n${task}`,
+    };
+
+    const result = await runner('ARCHITECT', taskInput, backend, null);
+
+    return new CouncilMemberResponse({
+      memberId,
+      response: result.outputText,
+      tokensIn: result.tokensIn,
+      tokensOut: result.tokensOut,
+      durationS: performance.now() / 1000 - startTime,
+      error: result.error,
+    });
+  }
+
+  /**
+   * Cross-rank: each member scores the others' responses.
+   * Returns a matrix where rankings[i][j] = member i's score for response j.
+   *
+   * @param {CouncilMemberResponse[]} responses
+   * @param {string} task
+   * @param {Function} runner
+   * @param {string|null} backend
+   * @returns {Promise<number[][]>}
+   */
+  async _crossRank(responses, task, runner, backend) {
+    const n = responses.length;
+    const rankings = Array.from({ length: n }, () => Array(n).fill(0));
+
+    const rankingPromises = [];
+    for (let i = 0; i < n; i++) {
+      rankingPromises.push(
+        this._rankByMember(i, responses, task, runner, backend)
+      );
+    }
+
+    const results = await Promise.allSettled(rankingPromises);
+    for (let i = 0; i < results.length; i++) {
+      if (results[i].status === 'fulfilled') {
+        rankings[i] = results[i].value;
+      }
+    }
+
+    return rankings;
+  }
+
+  /**
+   * Single member ranks all responses.
+   * @param {number} rankerId
+   * @param {CouncilMemberResponse[]} responses
+   * @param {string} task
+   * @param {Function} runner
+   * @param {string|null} backend
+   * @returns {Promise<number[]>}
+   */
+  async _rankByMember(rankerId, responses, task, runner, backend) {
+    const responseTexts = responses.map((r, j) =>
+      `Response ${j + 1}:\n${r.response}\n`
+    ).join('\n---\n\n');
+
+    const taskInput = {
+      task: `Rank these ${responses.length} responses to: "${task}"`,
+      user_message: `Score each response from 0 to 10 based on accuracy, completeness, and actionability.\n\n${responseTexts}\n\nReturn ONLY a JSON array of scores, e.g. [8, 6, 9]`,
+    };
+
+    const result = await runner('ARCHITECT', taskInput, backend, null);
+
+    // Parse scores from output
+    try {
+      const match = result.outputText.match(/\[[\d\s,\.]+\]/);
+      if (match) {
+        const scores = JSON.parse(match[0]);
+        return scores.map(s => Math.min(10, Math.max(0, Number(s) || 0)));
+      }
+    } catch { /* fall through */ }
+
+    // Fallback: equal scores
+    return Array(responses.length).fill(5);
+  }
+
+  /**
+   * Synthesize final output from the highest-ranked response.
+   *
+   * @param {CouncilMemberResponse[]} responses
+   * @param {number[][]} rankings
+   * @param {string} task
+   * @param {Function} runner
+   * @param {string|null} backend
+   * @returns {Promise<{ text: string, tokensIn: number, tokensOut: number }>}
+   */
+  async _synthesize(responses, rankings, task, runner, backend) {
+    // Compute aggregate score per response
+    const n = responses.length;
+    const aggregateScores = Array(n).fill(0);
+    for (let i = 0; i < rankings.length; i++) {
+      for (let j = 0; j < rankings[i].length; j++) {
+        aggregateScores[j] += rankings[i][j];
+      }
+    }
+
+    // Find highest-scored response
+    let bestIdx = 0;
+    let bestScore = aggregateScores[0];
+    for (let j = 1; j < n; j++) {
+      if (aggregateScores[j] > bestScore) {
+        bestScore = aggregateScores[j];
+        bestIdx = j;
+      }
+    }
+
+    const taskInput = {
+      task: `Synthesize a consensus from council evaluation`,
+      user_message: `Original task: "${task}"\n\nBest-ranked response (score ${bestScore}):\n${responses[bestIdx].response}\n\nProvide a refined, authoritative response incorporating the strongest analysis.`,
+    };
+
+    const result = await runner('ARCHITECT', taskInput, backend, null);
+
+    return {
+      text: result.outputText || responses[bestIdx].response,
+      tokensIn: result.tokensIn,
+      tokensOut: result.tokensOut,
+    };
+  }
+
+  /**
+   * Compute consensus confidence from cross-rankings.
+   * Higher agreement between rankers → higher confidence.
+   *
+   * @param {number[][]} rankings
+   * @param {number} responseCount
+   * @returns {number} 0-1 confidence score
+   */
+  _computeConfidence(rankings, responseCount) {
+    if (rankings.length === 0 || responseCount === 0) return 0;
+
+    // Compute per-response aggregate scores
+    const aggregates = Array(responseCount).fill(0);
+    let totalScores = 0;
+    for (const row of rankings) {
+      for (let j = 0; j < row.length; j++) {
+        aggregates[j] += row[j];
+        totalScores += row[j];
+      }
+    }
+
+    if (totalScores === 0) return 0;
+
+    // Find the spread: if one response dominates, confidence is high
+    const maxScore = Math.max(...aggregates);
+    const avgScore = totalScores / responseCount;
+
+    // Confidence = how much the best response stands out
+    if (avgScore === 0) return 0;
+    const dominance = maxScore / (totalScores || 1);
+    return Math.min(1, Math.max(0, dominance * responseCount));
+  }
+}
+
+// ---------------------------------------------------------------------------
+// runCouncil — convenience entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a council evaluation.
+ *
+ * @param {string} task
+ * @param {object} [opts]
+ * @param {number} [opts.members=3]
+ * @param {string|null} [opts.backend]
+ * @param {boolean} [opts.dryRun=false]
+ * @param {Function} [opts.agentRunner]
+ * @returns {Promise<CouncilResult>}
+ */
+export async function runCouncil(task, opts = {}) {
+  const council = new LLMCouncil({
+    members: opts.members ?? 3,
+    agentRunner: opts.agentRunner,
+  });
+  return council.evaluate(task, opts);
+}