@consensus-tools/universal 0.9.0

package/src/consensus-llm.test.ts

@@ -0,0 +1,260 @@
+import { describe, it, expect, vi } from "vitest";
+import { consensus } from "./index.js";
+import { ConsensusBlockedError } from "./errors.js";
+import type { ModelAdapter, ModelMessage, LlmDecisionResult } from "./types.js";
+
+// ── Mock Model Adapter ──────────────────────────────────────────────
+
+function createMockModel(response: string): ModelAdapter {
+  return async (_messages: ModelMessage[]) => response;
+}
+
+function createAllowModel(): ModelAdapter {
+  return createMockModel("VOTE: YES\nCONFIDENCE: 0.9\nRATIONALE: Looks safe to proceed.");
+}
+
+function createBlockModel(): ModelAdapter {
+  return createMockModel("VOTE: NO\nCONFIDENCE: 0.95\nRATIONALE: This action is dangerous.");
+}
+
+function createFailingModel(): ModelAdapter {
+  return async () => { throw new Error("LLM unavailable"); };
+}
+
+function createSlowModel(delayMs: number): ModelAdapter {
+  return async () => {
+    await new Promise((r) => setTimeout(r, delayMs));
+    return "VOTE: YES\nCONFIDENCE: 0.8\nRATIONALE: ok";
+  };
+}
+
+// ── Mock Executor ────────────────────────────────────────────────────
+
+function createMockExecutor() {
+  return vi.fn(async (toolName: string, args: Record<string, unknown>) => {
+    return { tool: toolName, result: "executed", args };
+  });
+}
+
+// ── Tests ────────────────────────────────────────────────────────────
+
+describe("consensus.wrap with LLM persona mode", () => {
+  describe("backward compatibility", () => {
+    it("works without model (regex-only mode)", async () => {
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor);
+      const result = await safe("get_weather", { city: "SF" });
+      expect(executor).toHaveBeenCalledWith("get_weather", { city: "SF" });
+      expect(result).toEqual({ tool: "get_weather", result: "executed", args: { city: "SF" } });
+    });
+  });
+
+  describe("LLM persona mode - allow", () => {
+    it("allows safe tool calls when all personas vote YES", async () => {
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model: createAllowModel(),
+        logger: false,
+      });
+      const result = await safe("send_email", { to: "user@test.com", body: "hello" });
+      expect(executor).toHaveBeenCalled();
+      expect(result).toBeDefined();
+    });
+  });
+
+  describe("LLM persona mode - block", () => {
+    it("blocks dangerous tool calls when personas vote NO", async () => {
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model: createBlockModel(),
+        failPolicy: "closed",
+        logger: false,
+      });
+      await expect(
+        safe("delete_database", { target: "production" }),
+      ).rejects.toThrow(ConsensusBlockedError);
+      expect(executor).not.toHaveBeenCalled();
+    });
+  });
+
+  describe("shadow mode", () => {
+    it("never blocks in shadow mode, even when personas vote NO", async () => {
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model: createBlockModel(),
+        mode: "shadow",
+        logger: false,
+      });
+      const result = await safe("delete_database", { target: "production" });
+      expect(executor).toHaveBeenCalled();
+      expect(result).toBeDefined();
+    });
+
+    it("still fires onDecision in shadow mode", async () => {
+      const onDecision = vi.fn();
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model: createBlockModel(),
+        mode: "shadow",
+        onDecision,
+        logger: false,
+      });
+      await safe("delete_database", {});
+      expect(onDecision).toHaveBeenCalled();
+      const decision = onDecision.mock.calls[0]![0] as LlmDecisionResult;
+      expect(decision.action).toBe("block");
+    });
+  });
+
+  describe("fast-path (low-risk tools)", () => {
+    it("skips LLM calls for low-risk tools", async () => {
+      const modelCalls: ModelMessage[][] = [];
+      const model: ModelAdapter = async (msgs) => {
+        modelCalls.push(msgs);
+        return "VOTE: YES\nCONFIDENCE: 0.9\nRATIONALE: ok";
+      };
+
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, { model, logger: false });
+      await safe("get_weather", { city: "SF" });
+
+      // Low-risk tool should not trigger LLM calls
+      expect(modelCalls.length).toBe(0);
+      expect(executor).toHaveBeenCalled();
+    });
+
+    it("runs full LLM deliberation for high-risk tools", async () => {
+      const modelCalls: ModelMessage[][] = [];
+      const model: ModelAdapter = async (msgs) => {
+        modelCalls.push(msgs);
+        return "VOTE: YES\nCONFIDENCE: 0.9\nRATIONALE: ok";
+      };
+
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, { model, logger: false });
+      await safe("send_email", { to: "user@test.com" });
+
+      // High-risk tool should trigger 3 LLM calls (one per persona)
+      expect(modelCalls.length).toBe(3);
+    });
+  });
+
+  describe("LLM failure fallback", () => {
+    it("falls back to regex when all LLM calls fail", async () => {
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model: createFailingModel(),
+        failPolicy: "open",
+        logger: false,
+      });
+      // Should not throw, should fall back to regex and likely allow
+      const result = await safe("send_email", { to: "user@test.com", body: "hello" });
+      expect(result).toBeDefined();
+    });
+  });
+
+  describe("per-persona timeout", () => {
+    it("falls back to regex for timed-out personas", async () => {
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model: createSlowModel(5000), // 5 second delay, will exceed default 3s timeout
+        personaTimeout: 100, // 100ms timeout for fast test
+        logger: false,
+      });
+      // Should not hang, should fall back to regex
+      const result = await safe("send_email", { to: "user@test.com" });
+      expect(result).toBeDefined();
+    });
+  });
+
+  describe("onDecision callback", () => {
+    it("receives LlmDecisionResult with vote breakdown", async () => {
+      const onDecision = vi.fn();
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model: createAllowModel(),
+        onDecision,
+        logger: false,
+      });
+      await safe("send_email", { to: "test@test.com" });
+
+      expect(onDecision).toHaveBeenCalledTimes(1);
+      const decision = onDecision.mock.calls[0]![0] as LlmDecisionResult;
+      expect(decision.decisionId).toBeDefined();
+      expect(decision.action).toBe("allow");
+      expect(decision.votes.length).toBe(3);
+      expect(decision.votes[0]).toHaveProperty("personaId");
+      expect(decision.votes[0]).toHaveProperty("personaName");
+      expect(decision.votes[0]).toHaveProperty("vote");
+      expect(decision.votes[0]).toHaveProperty("confidence");
+      expect(decision.votes[0]).toHaveProperty("rationale");
+      expect(decision.votes[0]).toHaveProperty("source");
+    });
+  });
+
+  describe("persona pack selection", () => {
+    it("uses governance pack when configured", async () => {
+      const executor = createMockExecutor();
+      // governance pack has 5 personas, should trigger 5 LLM calls
+      const modelCalls: ModelMessage[][] = [];
+      const model: ModelAdapter = async (msgs) => {
+        modelCalls.push(msgs);
+        return "VOTE: YES\nCONFIDENCE: 0.8\nRATIONALE: ok";
+      };
+
+      const safe = consensus.wrap(executor, {
+        model,
+        pack: "governance",
+        logger: false,
+      });
+      await safe("send_email", { to: "test@test.com" });
+
+      expect(modelCalls.length).toBe(5); // governance pack has 5 personas
+    });
+  });
+
+  describe("custom risk tiers", () => {
+    it("respects user-defined risk tier overrides", async () => {
+      const modelCalls: ModelMessage[][] = [];
+      const model: ModelAdapter = async (msgs) => {
+        modelCalls.push(msgs);
+        return "VOTE: YES\nCONFIDENCE: 0.9\nRATIONALE: ok";
+      };
+
+      const executor = createMockExecutor();
+      const safe = consensus.wrap(executor, {
+        model,
+        riskTiers: { get_weather: "high" }, // Override: treat read-only as high-risk
+        logger: false,
+      });
+      await safe("get_weather", { city: "SF" });
+
+      // Should have triggered LLM calls because we overrode to high-risk
+      expect(modelCalls.length).toBe(3);
+    });
+  });
+});
+
+describe("consensus.wrap with LLM - policy support", () => {
+  it("accepts core policy names", async () => {
+    const executor = createMockExecutor();
+    const safe = consensus.wrap(executor, {
+      model: createAllowModel(),
+      policy: "WEIGHTED_REPUTATION",
+      logger: false,
+    });
+    const result = await safe("send_email", { to: "test@test.com" });
+    expect(result).toBeDefined();
+  });
+
+  it("accepts friendly policy names", async () => {
+    const executor = createMockExecutor();
+    const safe = consensus.wrap(executor, {
+      model: createAllowModel(),
+      policy: "weighted_reputation",
+      logger: false,
+    });
+    const result = await safe("send_email", { to: "test@test.com" });
+    expect(result).toBeDefined();
+  });
+});

package/src/defaults.ts

@@ -0,0 +1,124 @@
+import type { StrategyConfig } from "@consensus-tools/wrapper";
+import type { UniversalConfig } from "./types.js";
+import { ConfigError } from "./errors.js";
+export { DEFAULT_PERSONA_TRIO } from "@consensus-tools/guards";
+
+// ── Default Configuration ────────────────────────────────────────────
+
+export const DEFAULT_GUARD = "agent_action";
+export const DEFAULT_POLICY = "majority";
+export const DEFAULT_PERSONA_COUNT = 3;
+export const DEFAULT_PACK = "default";
+export const DEFAULT_PERSONA_TIMEOUT_MS = 3000;
+export const DEFAULT_RESPAWN_THRESHOLD = 0.15;
+
+export const DEFAULTS: Required<
+  Pick<UniversalConfig, "policy" | "guards" | "failPolicy" | "storage" | "logger">
+> = {
+  policy: DEFAULT_POLICY,
+  guards: [DEFAULT_GUARD],
+  failPolicy: "closed",
+  storage: "memory",
+  logger: true,
+};
+
+// ── Core Policy Type Names ───────────────────────────────────────────
+// All 9 policies supported by resolveConsensus() in @consensus-tools/core.
+// These are used in LLM mode where we bypass the wrapper and call
+// resolveConsensus() directly.
+
+export const CORE_POLICY_TYPES = new Set([
+  "FIRST_SUBMISSION_WINS",
+  "HIGHEST_CONFIDENCE_SINGLE",
+  "APPROVAL_VOTE",
+  "OWNER_PICK",
+  "TOP_K_SPLIT",
+  "MAJORITY_VOTE",
+  "WEIGHTED_VOTE_SIMPLE",
+  "WEIGHTED_REPUTATION",
+  "TRUSTED_ARBITER",
+]);
+
+// ── Friendly → Core Policy Mapping ───────────────────────────────────
+// Maps friendly names (used in facade config) to core policy type names.
+
+const FRIENDLY_TO_CORE: Record<string, string> = {
+  majority: "MAJORITY_VOTE",
+  supermajority: "APPROVAL_VOTE",
+  unanimous: "APPROVAL_VOTE",
+  weighted_reputation: "WEIGHTED_REPUTATION",
+  first_wins: "FIRST_SUBMISSION_WINS",
+  highest_confidence: "HIGHEST_CONFIDENCE_SINGLE",
+  top_k: "TOP_K_SPLIT",
+  owner_pick: "OWNER_PICK",
+  arbiter: "TRUSTED_ARBITER",
+};
+
+/**
+ * Resolve a policy string to a core policy type name for LLM mode.
+ * Accepts friendly names, core names, and threshold:X patterns.
+ */
+export function resolvePolicyType(policy: string): string {
+  // Direct core policy name
+  if (CORE_POLICY_TYPES.has(policy)) return policy;
+
+  // Friendly name mapping
+  const mapped = FRIENDLY_TO_CORE[policy];
+  if (mapped) return mapped;
+
+  // threshold:X -> APPROVAL_VOTE with custom config
+  if (policy.startsWith("threshold:")) return "APPROVAL_VOTE";
+
+  // Default to MAJORITY_VOTE
+  return "MAJORITY_VOTE";
+}
+
+// ── Policy-to-Strategy Mapping (regex mode) ──────────────────────────
+
+/**
+ * Maps a user-facing policy name to a wrapper StrategyConfig.
+ * Used in regex-only mode (no model provided).
+ *
+ * Supported names:
+ *   'majority'        -> { strategy: 'majority' }
+ *   'supermajority'   -> { strategy: 'threshold', threshold: 0.67 }
+ *   'unanimous'       -> { strategy: 'unanimous' }
+ *   'threshold:X'     -> { strategy: 'threshold', threshold: X }
+ *
+ * @throws ConfigError for unrecognized policy names.
+ */
+export function policyToStrategy(policy: string): StrategyConfig {
+  switch (policy) {
+    case "majority":
+      return { strategy: "majority" };
+    case "supermajority":
+      return { strategy: "threshold", threshold: 0.67 };
+    case "unanimous":
+      return { strategy: "unanimous" };
+    default: {
+      // Handle 'threshold:X' pattern
+      if (policy.startsWith("threshold:")) {
+        const value = Number(policy.slice("threshold:".length));
+        if (Number.isNaN(value) || value < 0 || value > 1) {
+          throw new ConfigError(
+            `Invalid threshold value in policy "${policy}". Expected a number between 0 and 1.`,
+          );
+        }
+        return { strategy: "threshold", threshold: value };
+      }
+
+      // In LLM mode, core policy names are valid. In regex mode, they're not.
+      // Let the caller decide — we just throw for truly unknown strings.
+      if (CORE_POLICY_TYPES.has(policy) || FRIENDLY_TO_CORE[policy]) {
+        // Valid LLM-mode policy used in regex mode — fall back to majority
+        return { strategy: "majority" };
+      }
+
+      throw new ConfigError(
+        `Unknown policy "${policy}". ` +
+        `Supported: 'majority', 'supermajority', 'unanimous', 'threshold:X' (where X is 0-1).` +
+        ` For LLM mode, also: ${[...CORE_POLICY_TYPES].join(", ")}.`,
+      );
+    }
+  }
+}

package/src/errors.ts

@@ -0,0 +1,35 @@
+/**
+ * Thrown when consensus blocks an action and failPolicy is 'closed'.
+ */
+export class ConsensusBlockedError extends Error {
+  override name = "ConsensusBlockedError";
+
+  constructor(message: string, public readonly cause?: Error) {
+    super(message);
+  }
+}
+
+/**
+ * Thrown when an optional adapter package is not installed.
+ */
+export class MissingDependencyError extends Error {
+  override name = "MissingDependencyError";
+
+  constructor(packageName: string) {
+    super(
+      `Package "${packageName}" is required but not installed. ` +
+      `Install it with: pnpm add ${packageName}`,
+    );
+  }
+}
+
+/**
+ * Thrown for invalid configuration values.
+ */
+export class ConfigError extends Error {
+  override name = "ConfigError";
+
+  constructor(message: string) {
+    super(message);
+  }
+}