@consensus-tools/universal 0.9.0

package/src/persona-reviewer-factory.ts

@@ -0,0 +1,387 @@
+import crypto from "node:crypto";
+import { resolveConsensus } from "@consensus-tools/core";
+import { createGuardTemplate, GUARD_CONFIGS } from "@consensus-tools/guards";
+import type { PersonaConfig, EvalPersonaConfig } from "@consensus-tools/personas";
+import type { ConsensusInput, ConsensusResult } from "@consensus-tools/schemas";
+import type { ModelAdapter, ModelMessage, LlmDecisionResult } from "./types.js";
+import type { ReputationManager } from "./reputation-manager.js";
+import { classifyTool } from "./risk-tiers.js";
+import type { RiskTierMap } from "./types.js";
+
+// ── Persona Reviewer Factory ─────────────────────────────────────────
+// Creates LLM-backed persona reviewers that use resolveConsensus()
+// for multi-model deliberation with reputation-weighted voting.
+//
+// Architecture:
+//   1. Regex pre-screen (sub-ms, deterministic)
+//   2. Risk tier check (low = fast-path regex only)
+//   3. Parallel LLM calls per persona (with timeout + fallback)
+//   4. Parse votes from LLM responses
+//   5. Synthesize ConsensusInput (Job, Submissions, Votes)
+//   6. Call resolveConsensus() with the configured policy
+//   7. Return LlmDecisionResult
+
+// ── Vote Parsing ─────────────────────────────────────────────────────
+
+interface ParsedVote {
+  vote: "YES" | "NO" | "REWRITE";
+  confidence: number;
+  rationale: string;
+}
+
+const VOTE_PATTERN = /\b(YES|NO|REWRITE)\b/i;
+const CONFIDENCE_PATTERN = /confidence[:\s]*([0-9]*\.?[0-9]+)/i;
+
+function parseVoteFromLlm(response: string): ParsedVote | null {
+  const voteMatch = response.match(VOTE_PATTERN);
+  if (!voteMatch) return null;
+
+  const vote = voteMatch[1]!.toUpperCase() as "YES" | "NO" | "REWRITE";
+  const confMatch = response.match(CONFIDENCE_PATTERN);
+  const confidence = confMatch?.[1] ? Math.min(1, Math.max(0, parseFloat(confMatch[1]))) : 0.5;
+
+  // Use the full response as rationale (stripped of vote/confidence lines)
+  const rationale = response
+    .replace(/^.*\b(YES|NO|REWRITE)\b.*$/im, "")
+    .replace(/^.*confidence.*$/im, "")
+    .trim()
+    .slice(0, 500) || "No rationale provided";
+
+  return { vote, confidence, rationale };
+}
+
+// ── Prompt Construction ──────────────────────────────────────────────
+
+function buildPersonaPrompt(
+  persona: PersonaConfig,
+  toolName: string,
+  args: Record<string, unknown>,
+  regexSignals: string[],
+): ModelMessage[] {
+  // Use systemPrompt if available (EvalPersonaConfig), otherwise construct from role/bias
+  const evalPersona = persona as Partial<EvalPersonaConfig>;
+  const systemPrompt = evalPersona.systemPrompt ??
+    `You are a ${persona.role} reviewer named ${persona.name}.` +
+    (persona.bias ? ` Your bias: ${persona.bias}.` : "") +
+    (persona.non_negotiables?.length
+      ? ` Non-negotiables: ${persona.non_negotiables.join(", ")}.`
+      : "") +
+    (persona.failure_modes?.length
+      ? ` Watch for: ${persona.failure_modes.join(", ")}.`
+      : "");
+
+  const regexContext = regexSignals.length > 0
+    ? `\n\nRegex pre-screen detected these signals:\n${regexSignals.map((s) => `- ${s}`).join("\n")}`
+    : "\n\nRegex pre-screen: no pattern matches detected.";
+
+  const userPrompt =
+    `Evaluate this tool call for risks:\n\n` +
+    `Tool: ${toolName}\n` +
+    `Arguments: ${JSON.stringify(args, null, 2)}\n` +
+    regexContext +
+    `\n\nRespond with:\n` +
+    `VOTE: YES (safe to proceed), NO (block this action), or REWRITE (needs modification)\n` +
+    `CONFIDENCE: 0.0 to 1.0\n` +
+    `RATIONALE: Brief explanation of your decision`;
+
+  return [
+    { role: "system" as const, content: systemPrompt },
+    { role: "user" as const, content: userPrompt },
+  ];
+}
+
+// ── Regex Pre-Screen ─────────────────────────────────────────────────
+
+function runRegexPreScreen(
+  toolName: string,
+  args: Record<string, unknown>,
+  guards: string[],
+): string[] {
+  const signals: string[] = [];
+
+  for (const domain of guards) {
+    const config = GUARD_CONFIGS[domain];
+    if (!config) continue;
+
+    try {
+      const template = createGuardTemplate(domain, config);
+      const votes = template.evaluate({
+        boardId: "facade",
+        action: { type: toolName, payload: args },
+      });
+
+      for (const vote of votes) {
+        if (vote.vote === "NO" || (vote.risk && vote.risk > 0.5)) {
+          signals.push(`[${domain}] ${vote.reason} (risk: ${vote.risk ?? "unknown"})`);
+        }
+      }
+    } catch {
+      // Regex pre-screen failure is non-fatal
+    }
+  }
+
+  return signals;
+}
+
+// ── LLM Call with Timeout ────────────────────────────────────────────
+
+async function callLlmWithTimeout(
+  model: ModelAdapter,
+  messages: ModelMessage[],
+  timeoutMs: number,
+): Promise<string> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const result = await Promise.race([
+      model(messages),
+      new Promise<never>((_, reject) => {
+        controller.signal.addEventListener("abort", () =>
+          reject(new Error("LLM call timed out")),
+        );
+      }),
+    ]);
+    return result;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// ── Regex Fallback Vote ──────────────────────────────────────────────
+
+function regexFallbackVote(
+  persona: PersonaConfig,
+  toolName: string,
+  args: Record<string, unknown>,
+  guards: string[],
+): ParsedVote {
+  const signals = runRegexPreScreen(toolName, args, guards);
+  if (signals.length > 0) {
+    return {
+      vote: "NO",
+      confidence: 0.6,
+      rationale: `Regex fallback: ${signals.join("; ")}`,
+    };
+  }
+  return {
+    vote: "YES",
+    confidence: 0.4,
+    rationale: "Regex fallback: no pattern matches (LLM unavailable)",
+  };
+}
+
+// ── Factory ──────────────────────────────────────────────────────────
+
+export interface PersonaReviewerConfig {
+  model: ModelAdapter;
+  pack?: string;
+  personas?: PersonaConfig[];
+  guards?: string[];
+  policyType: string;
+  riskTiers?: RiskTierMap;
+  reputationManager: ReputationManager;
+  timeoutMs: number;
+}
+
+/**
+ * Run LLM persona deliberation on a tool call.
+ *
+ * Returns an LlmDecisionResult with per-persona votes, consensus trace,
+ * and final action (allow/block/escalate).
+ */
+export async function deliberate(
+  config: PersonaReviewerConfig,
+  toolName: string,
+  args: Record<string, unknown>,
+): Promise<LlmDecisionResult> {
+  const decisionId = `dec_${crypto.randomUUID().slice(0, 12)}`;
+  const personas = config.reputationManager.getPersonas();
+  const guards = config.guards ?? ["security", "compliance", "user-impact"];
+
+  // 1. Regex pre-screen
+  const regexSignals = runRegexPreScreen(toolName, args, guards);
+
+  // 2. Risk tier check
+  const tier = classifyTool(toolName, config.riskTiers);
+  if (tier === "low") {
+    // Fast-path: regex only, no LLM calls
+    const hasRisk = regexSignals.length > 0;
+    return {
+      decisionId,
+      action: hasRisk ? "block" : "allow",
+      votes: personas.map((p) => ({
+        personaId: p.id,
+        personaName: p.name,
+        vote: hasRisk ? ("NO" as const) : ("YES" as const),
+        confidence: hasRisk ? 0.7 : 0.3,
+        rationale: hasRisk
+          ? `Fast-path regex: ${regexSignals.join("; ")}`
+          : "Fast-path: low-risk tool, no regex signals",
+        source: "regex_fallback" as const,
+      })),
+      policy: "fast_path",
+      consensusTrace: { tier: "low", regexSignals },
+      aggregateScore: hasRisk ? 0.0 : 1.0,
+    };
+  }
+
+  // 3. Parallel LLM calls per persona (with timeout + fallback)
+  const voteResults = await Promise.all(
+    personas.map(async (persona) => {
+      const messages = buildPersonaPrompt(persona, toolName, args, regexSignals);
+
+      try {
+        const response = await callLlmWithTimeout(config.model, messages, config.timeoutMs);
+        const parsed = parseVoteFromLlm(response);
+
+        if (parsed) {
+          return {
+            personaId: persona.id,
+            personaName: persona.name,
+            ...parsed,
+            source: "llm" as const,
+          };
+        }
+
+        // Unparseable response, fall back to regex
+        const fallback = regexFallbackVote(persona, toolName, args, guards);
+        return {
+          personaId: persona.id,
+          personaName: persona.name,
+          ...fallback,
+          source: "regex_fallback" as const,
+        };
+      } catch {
+        // LLM failure, fall back to regex
+        const fallback = regexFallbackVote(persona, toolName, args, guards);
+        return {
+          personaId: persona.id,
+          personaName: persona.name,
+          ...fallback,
+          source: "regex_fallback" as const,
+        };
+      }
+    }),
+  );
+
+  // 4. Synthesize ConsensusInput for resolveConsensus()
+  //    Each persona creates a "submission" (their evaluation) and votes for it
+  const now = new Date().toISOString();
+  const jobId = `job_facade_${decisionId}`;
+
+  // Create a minimal Job with the configured policy
+  const job = {
+    id: jobId,
+    boardId: "",
+    status: "SUBMITTED" as const,
+    title: `Deliberation: ${toolName}`,
+    description: JSON.stringify(args),
+    createdByAgentId: "facade",
+    createdAt: now,
+    updatedAt: now,
+    mode: "VOTING" as const,
+    consensusPolicy: { type: config.policyType as any },
+    stakeRequired: 0,
+    reward: 0,
+    maxParticipants: personas.length,
+    minParticipants: 1,
+  };
+
+  // Each persona submits their evaluation
+  const submissions = voteResults.map((v, i) => ({
+    id: `sub_${decisionId}_${i}`,
+    jobId,
+    agentId: v.personaId,
+    submittedAt: now,
+    summary: v.rationale,
+    artifacts: { vote: v.vote, confidence: v.confidence, source: v.source },
+    confidence: v.confidence,
+    requestedPayout: 0,
+    status: "SUBMITTED" as const,
+  }));
+
+  // Each persona votes YES (+1) on their own submission
+  // and scores based on their confidence
+  const votes = voteResults.map((v, i) => ({
+    id: `vote_${decisionId}_${i}`,
+    jobId,
+    agentId: v.personaId,
+    submissionId: `sub_${decisionId}_${i}`,
+    score: v.vote === "YES" ? 1 : v.vote === "NO" ? -1 : 0,
+    weight: v.confidence,
+    rationale: v.rationale,
+    createdAt: now,
+  }));
+
+  // Reputation function from the manager
+  const reputation = (agentId: string) =>
+    config.reputationManager.getReputation(agentId);
+
+  // 5. Resolve consensus
+  const consensusInput: ConsensusInput = {
+    job: job as any,
+    submissions: submissions as any[],
+    votes: votes as any[],
+    reputation,
+  };
+
+  let consensusResult: ConsensusResult;
+  try {
+    consensusResult = resolveConsensus(consensusInput);
+  } catch {
+    // If resolution fails, fall back to simple majority
+    const yesCount = voteResults.filter((v) => v.vote === "YES").length;
+    const majority = yesCount > voteResults.length / 2;
+    consensusResult = {
+      winners: majority ? ["allow"] : ["block"],
+      winningSubmissionIds: [],
+      consensusTrace: { policy: "fallback_majority", reason: "resolve_error" },
+      finalArtifact: null,
+    };
+  }
+
+  // 6. Determine final action
+  const winnerIds = new Set(consensusResult.winners);
+  const winningVotes = voteResults.filter((v) => winnerIds.has(v.personaId));
+  const dominantVote = winningVotes.length > 0
+    ? winningVotes[0]!.vote
+    : voteResults[0]?.vote ?? "YES";
+
+  let action: "allow" | "block" | "escalate";
+  if (dominantVote === "YES") {
+    action = "allow";
+  } else if (dominantVote === "NO") {
+    action = "block";
+  } else {
+    action = "escalate";
+  }
+
+  // If no clear winner (empty winners), use simple vote counting
+  if (consensusResult.winners.length === 0) {
+    const yesCount = voteResults.filter((v) => v.vote === "YES").length;
+    const noCount = voteResults.filter((v) => v.vote === "NO").length;
+    action = yesCount >= noCount ? "allow" : "block";
+  }
+
+  // Compute aggregate score (0-1 based on vote distribution)
+  const totalConfidence = voteResults.reduce((s, v) => s + v.confidence, 0);
+  const yesConfidence = voteResults
+    .filter((v) => v.vote === "YES")
+    .reduce((s, v) => s + v.confidence, 0);
+  const aggregateScore = totalConfidence > 0 ? yesConfidence / totalConfidence : 0.5;
+
+  const result: LlmDecisionResult = {
+    decisionId,
+    action,
+    votes: voteResults,
+    policy: config.policyType,
+    consensusTrace: consensusResult.consensusTrace,
+    aggregateScore,
+  };
+
+  // 7. Record decision for reputation tracking
+  config.reputationManager.recordDecision(result);
+
+  return result;
+}

package/src/reputation-manager.test.ts

@@ -0,0 +1,131 @@
+import { describe, it, expect, vi } from "vitest";
+import { ReputationManager } from "./reputation-manager.js";
+import type { PersonaConfig } from "@consensus-tools/personas";
+import type { LlmDecisionResult } from "./types.js";
+
+function makePersonas(): PersonaConfig[] {
+  return [
+    { id: "p1", name: "Security", role: "security", reputation: 0.55 },
+    { id: "p2", name: "Compliance", role: "compliance", reputation: 0.55 },
+    { id: "p3", name: "Operations", role: "operations", reputation: 0.55 },
+  ];
+}
+
+function makeDecision(overrides?: Partial<LlmDecisionResult>): LlmDecisionResult {
+  return {
+    decisionId: "dec_test",
+    action: "block",
+    votes: [
+      { personaId: "p1", personaName: "Security", vote: "NO", confidence: 0.9, rationale: "risky", source: "llm" },
+      { personaId: "p2", personaName: "Compliance", vote: "YES", confidence: 0.7, rationale: "ok", source: "llm" },
+      { personaId: "p3", personaName: "Operations", vote: "NO", confidence: 0.8, rationale: "risky", source: "llm" },
+    ],
+    policy: "MAJORITY_VOTE",
+    consensusTrace: {},
+    aggregateScore: 0.3,
+    ...overrides,
+  };
+}
+
+describe("ReputationManager", () => {
+  it("initializes all personas at 0.55", () => {
+    const mgr = new ReputationManager(makePersonas());
+    expect(mgr.getReputation("p1")).toBe(0.55);
+    expect(mgr.getReputation("p2")).toBe(0.55);
+    expect(mgr.getReputation("p3")).toBe(0.55);
+  });
+
+  it("returns 0.55 for unknown persona IDs", () => {
+    const mgr = new ReputationManager(makePersonas());
+    expect(mgr.getReputation("unknown")).toBe(0.55);
+  });
+
+  it("records decisions for feedback correlation", () => {
+    const mgr = new ReputationManager(makePersonas());
+    const decision = makeDecision();
+    mgr.recordDecision(decision);
+
+    // processFeedback should find this decision
+    const changes = mgr.processFeedback({
+      decisionId: "dec_test",
+      type: "override_block",
+    });
+    expect(changes.length).toBe(3);
+  });
+
+  it("updates reputation on override_block feedback", () => {
+    const mgr = new ReputationManager(makePersonas());
+    const decision = makeDecision();
+    mgr.recordDecision(decision);
+
+    // override_block means the block was wrong, ground truth is ALLOW
+    // p1 voted NO (misaligned with ALLOW), p2 voted YES (aligned), p3 voted NO (misaligned)
+    const changes = mgr.processFeedback({
+      decisionId: "dec_test",
+      type: "override_block",
+    });
+
+    // p2 (voted YES, aligned with ALLOW) should gain reputation
+    const p2Change = changes.find((c) => c.persona_id === "p2");
+    expect(p2Change!.delta).toBeGreaterThan(0);
+
+    // p1 (voted NO, misaligned with ALLOW) should lose reputation
+    const p1Change = changes.find((c) => c.persona_id === "p1");
+    expect(p1Change!.delta).toBeLessThan(0);
+  });
+
+  it("updates reputation on flag_miss feedback", () => {
+    const mgr = new ReputationManager(makePersonas());
+    const decision = makeDecision({ action: "allow" });
+    mgr.recordDecision(decision);
+
+    // flag_miss means the allow was wrong, ground truth is BLOCK
+    const changes = mgr.processFeedback({
+      decisionId: "dec_test",
+      type: "flag_miss",
+    });
+
+    // p1 and p3 (voted NO, aligned with BLOCK) should gain reputation
+    const p1Change = changes.find((c) => c.persona_id === "p1");
+    expect(p1Change!.delta).toBeGreaterThan(0);
+  });
+
+  it("returns empty changes for unknown decision IDs", () => {
+    const mgr = new ReputationManager(makePersonas());
+    const changes = mgr.processFeedback({
+      decisionId: "nonexistent",
+      type: "override_block",
+    });
+    expect(changes).toEqual([]);
+  });
+
+  it("triggers respawn when reputation drops below threshold", () => {
+    const personas = makePersonas();
+    personas[0]!.reputation = 0.10; // Already below default threshold of 0.15
+    const mgr = new ReputationManager(personas, 0.15);
+
+    const respawnHandler = vi.fn();
+    mgr.setRespawnHandler(respawnHandler);
+
+    // Record + feedback to trigger checkRespawn
+    const decision = makeDecision();
+    mgr.recordDecision(decision);
+    mgr.processFeedback({ decisionId: "dec_test", type: "override_block" });
+
+    // p1 started at 0.10, misaligned feedback pushes it further down
+    // Respawn should have been triggered
+    if (mgr.getReputation("p1") < 0.15 || respawnHandler.mock.calls.length === 0) {
+      // Persona was replaced, check new personas list
+      const currentPersonas = mgr.getPersonas();
+      // Either the original was respawned or it's below threshold
+      expect(currentPersonas.length).toBe(3);
+    }
+  });
+
+  it("getPersonas returns copies", () => {
+    const mgr = new ReputationManager(makePersonas());
+    const p1 = mgr.getPersonas();
+    const p2 = mgr.getPersonas();
+    expect(p1).not.toBe(p2);
+  });
+});

package/src/reputation-manager.ts

@@ -0,0 +1,168 @@
+import { updateReputation } from "@consensus-tools/personas";
+import { buildLearningSummary, mutatePersona } from "@consensus-tools/personas";
+import type { PersonaConfig, ReputationChange } from "@consensus-tools/personas";
+import type { IStorage } from "@consensus-tools/storage";
+import type { FeedbackSignal, LlmDecisionResult } from "./types.js";
+
+// ── Reputation Manager ───────────────────────────────────────────────
+// In-memory reputation tracking with optional persistence.
+// Updates from human feedback signals (onFeedback), not self-consensus.
+// Triggers persona respawn when reputation drops below threshold.
+
+export interface RespawnEvent {
+  oldPersona: PersonaConfig;
+  newPersona: PersonaConfig;
+  reputation: number;
+  reason: string;
+}
+
+export class ReputationManager {
+  private scores: Map<string, number> = new Map();
+  private decisions: Map<string, LlmDecisionResult> = new Map();
+  private decisionHistory: LlmDecisionResult[] = [];
+  private onRespawn?: (event: RespawnEvent) => void;
+
+  constructor(
+    private personas: PersonaConfig[],
+    private threshold: number = 0.15,
+    private store?: IStorage,
+  ) {
+    for (const p of personas) {
+      this.scores.set(p.id, p.reputation ?? 0.55);
+    }
+  }
+
+  /** Set callback for respawn events. */
+  setRespawnHandler(handler: (event: RespawnEvent) => void): void {
+    this.onRespawn = handler;
+  }
+
+  /** Get current reputation for a persona. */
+  getReputation(personaId: string): number {
+    return this.scores.get(personaId) ?? 0.55;
+  }
+
+  /** Get all current reputation scores. */
+  getAllReputations(): Map<string, number> {
+    return new Map(this.scores);
+  }
+
+  /** Record a decision for later feedback correlation. */
+  recordDecision(result: LlmDecisionResult): void {
+    this.decisions.set(result.decisionId, result);
+    this.decisionHistory.push(result);
+    // Keep last 100 decisions for respawn analysis
+    if (this.decisionHistory.length > 100) {
+      this.decisionHistory.shift();
+    }
+  }
+
+  /** Process human feedback signal and update reputation. */
+  processFeedback(signal: FeedbackSignal): ReputationChange[] {
+    const decision = this.decisions.get(signal.decisionId);
+    if (!decision) return [];
+
+    // Map feedback to a "ground truth" final decision
+    // override_block = human says the block was wrong, action should have been ALLOW
+    // flag_miss = human says the allow was wrong, action should have been BLOCK
+    const groundTruth = signal.type === "override_block" ? "ALLOW" : "BLOCK";
+
+    const votes = decision.votes.map((v) => ({
+      persona_id: v.personaId,
+      vote: v.vote,
+      confidence: v.confidence,
+    }));
+
+    const personaConfigs = this.personas.map((p) => ({
+      ...p,
+      reputation: this.scores.get(p.id) ?? 0.55,
+    }));
+
+    const result = updateReputation(votes, groundTruth, personaConfigs);
+
+    // Apply changes
+    for (const change of result.changes) {
+      this.scores.set(change.persona_id, change.reputation_after);
+    }
+
+    // Check for respawn
+    this.checkRespawn();
+
+    // Persist if store configured
+    this.persist();
+
+    return result.changes;
+  }
+
+  /** Check if any persona needs respawn. */
+  private checkRespawn(): void {
+    for (let i = 0; i < this.personas.length; i++) {
+      const persona = this.personas[i]!;
+      const rep = this.scores.get(persona.id) ?? 0.55;
+
+      if (rep < this.threshold) {
+        // Build learning summary from decision history
+        const decisionRecords = this.decisionHistory.map((d) => ({
+          final_decision: d.action === "allow" ? "ALLOW" : "BLOCK",
+          votes: d.votes.map((v) => ({
+            persona_id: v.personaId,
+            vote: v.vote,
+            confidence: v.confidence,
+          })),
+        }));
+
+        const learning = buildLearningSummary(persona.id, decisionRecords);
+        const successor = mutatePersona(persona, learning);
+
+        // Replace persona
+        this.personas[i] = successor;
+        this.scores.delete(persona.id);
+        this.scores.set(successor.id, successor.reputation ?? 0.55);
+
+        this.onRespawn?.({
+          oldPersona: persona,
+          newPersona: successor,
+          reputation: rep,
+          reason: `Reputation ${rep.toFixed(3)} below threshold ${this.threshold}`,
+        });
+      }
+    }
+  }
+
+  /** Get current persona list (may include respawned successors). */
+  getPersonas(): PersonaConfig[] {
+    return [...this.personas];
+  }
+
+  /** Persist reputation scores to store (if configured). */
+  private persist(): void {
+    if (!this.store) return;
+    const data: Record<string, number> = {};
+    for (const [id, score] of this.scores) {
+      data[id] = score;
+    }
+    this.store.update((state) => {
+      (state as any).reputation = data;
+    }).catch(() => {
+      // Persistence failure is non-fatal
+    });
+  }
+
+  /** Load reputation scores from store (if configured). */
+  async load(): Promise<void> {
+    if (!this.store) return;
+    try {
+      const state = await this.store.getState();
+      const saved = (state as any)?.reputation as Record<string, number> | undefined;
+      if (saved) {
+        for (const [id, score] of Object.entries(saved)) {
+          if (this.scores.has(id)) {
+            this.scores.set(id, score);
+          }
+        }
+      }
+    } catch {
+      // Load failure is non-fatal, start with defaults
+    }
+  }
+}