@vibecheckai/cli 3.2.5 → 3.3.0

package/bin/runners/lib/agent-firewall/critic/judge.js

@@ -0,0 +1,432 @@
+/**
+ * Critic LLM Judge
+ * 
+ * The "savage" judge that evaluates proposal quality.
+ * Detects hand-waving, vague intent, and unverified assumptions.
+ * 
+ * Philosophy: "If this change cannot be proven safe by the repository, block it."
+ */
+
+"use strict";
+
+const {
+  CRITIC_SYSTEM_PROMPT,
+  buildEvaluationPrompt,
+  buildVaguenessPrompt,
+  buildVerificationPrompt,
+  parseCriticResponse,
+} = require("./prompts");
+
+/**
+ * @typedef {Object} CriticVerdict
+ * @property {string} verdict - ALLOW, BLOCK, or REQUIRE_CONFIRMATION
+ * @property {number} confidence - Confidence in verdict (0-1)
+ * @property {Array} reasoning - Reasons for the verdict
+ * @property {Array} violations - Specific violations found
+ * @property {Array} recommendations - Suggestions for improvement
+ */
+
+/**
+ * Default LLM client configuration
+ */
+const DEFAULT_CONFIG = {
+  model: "gpt-4-turbo-preview",
+  temperature: 0.1, // Low temperature for consistent judgments
+  maxTokens: 1000,
+  timeout: 30000,
+};
+
+/**
+ * Critic Judge class
+ */
+class CriticJudge {
+  constructor(options = {}) {
+    this.config = { ...DEFAULT_CONFIG, ...options };
+    this.llmClient = options.llmClient || null;
+    this.enabled = options.enabled !== false;
+    this.fallbackMode = options.fallbackMode || "conservative";
+  }
+  
+  /**
+   * Set the LLM client
+   * @param {Function} client - LLM client function
+   */
+  setClient(client) {
+    this.llmClient = client;
+  }
+  
+  /**
+   * Check if critic is available
+   * @returns {boolean} Is available
+   */
+  isAvailable() {
+    return this.enabled && this.llmClient !== null;
+  }
+  
+  /**
+   * Evaluate a proposal
+   * @param {Object} params - Evaluation parameters
+   * @returns {Promise<CriticVerdict>} Critic verdict
+   */
+  async evaluate(params) {
+    const {
+      proposal,
+      validationResults = {},
+      riskScore = {},
+      simulationResult = {},
+      realityState = {},
+    } = params;
+    
+    // If critic is disabled, use rule-based evaluation
+    if (!this.isAvailable()) {
+      return this.ruleBasedEvaluation(params);
+    }
+    
+    try {
+      // Build the evaluation prompt
+      const prompt = buildEvaluationPrompt({
+        proposal,
+        validationResults,
+        riskScore,
+        simulationResult,
+        realityState,
+      });
+      
+      // Call LLM
+      const response = await this.callLLM(prompt);
+      
+      // Parse response
+      const verdict = parseCriticResponse(response);
+      
+      // Validate verdict
+      return this.validateVerdict(verdict);
+    } catch (error) {
+      console.warn(`Critic LLM evaluation failed: ${error.message}`);
+      
+      // Fall back to rule-based evaluation
+      if (this.fallbackMode === "conservative") {
+        return this.ruleBasedEvaluation(params);
+      } else {
+        return {
+          verdict: "ALLOW",
+          confidence: 0.3,
+          reasoning: ["Critic unavailable, using permissive fallback"],
+          violations: [],
+          recommendations: ["Consider manual review"],
+        };
+      }
+    }
+  }
+  
+  /**
+   * Check for vagueness in proposal
+   * @param {Object} proposal - Proposal to check
+   * @returns {Promise<Object>} Vagueness analysis
+   */
+  async checkVagueness(proposal) {
+    if (!this.isAvailable()) {
+      return this.ruleBasedVaguenessCheck(proposal);
+    }
+    
+    try {
+      const prompt = buildVaguenessPrompt(proposal);
+      const response = await this.callLLM(prompt);
+      return parseCriticResponse(response);
+    } catch (error) {
+      return this.ruleBasedVaguenessCheck(proposal);
+    }
+  }
+  
+  /**
+   * Verify assumptions against reality
+   * @param {Array} assumptions - Assumptions to verify
+   * @param {Object} realityState - Repository state
+   * @returns {Promise<Object>} Verification results
+   */
+  async verifyAssumptions(assumptions, realityState) {
+    if (!this.isAvailable()) {
+      return this.ruleBasedAssumptionVerification(assumptions, realityState);
+    }
+    
+    try {
+      const prompt = buildVerificationPrompt(assumptions, realityState);
+      const response = await this.callLLM(prompt);
+      return parseCriticResponse(response);
+    } catch (error) {
+      return this.ruleBasedAssumptionVerification(assumptions, realityState);
+    }
+  }
+  
+  /**
+   * Call the LLM
+   * @param {string} prompt - User prompt
+   * @returns {Promise<string>} LLM response
+   */
+  async callLLM(prompt) {
+    if (!this.llmClient) {
+      throw new Error("LLM client not configured");
+    }
+    
+    return this.llmClient({
+      systemPrompt: CRITIC_SYSTEM_PROMPT,
+      userPrompt: prompt,
+      model: this.config.model,
+      temperature: this.config.temperature,
+      maxTokens: this.config.maxTokens,
+    });
+  }
+  
+  /**
+   * Validate and normalize verdict
+   * @param {Object} verdict - Raw verdict
+   * @returns {CriticVerdict} Validated verdict
+   */
+  validateVerdict(verdict) {
+    const validVerdicts = ["ALLOW", "BLOCK", "REQUIRE_CONFIRMATION"];
+    
+    return {
+      verdict: validVerdicts.includes(verdict.verdict) ? verdict.verdict : "BLOCK",
+      confidence: typeof verdict.confidence === "number" 
+        ? Math.max(0, Math.min(1, verdict.confidence))
+        : 0.5,
+      reasoning: Array.isArray(verdict.reasoning) ? verdict.reasoning : [],
+      violations: Array.isArray(verdict.violations) ? verdict.violations : [],
+      recommendations: Array.isArray(verdict.recommendations) ? verdict.recommendations : [],
+    };
+  }
+  
+  /**
+   * Rule-based evaluation fallback
+   * @param {Object} params - Evaluation parameters
+   * @returns {CriticVerdict} Verdict
+   */
+  ruleBasedEvaluation(params) {
+    const { proposal, validationResults, riskScore, simulationResult } = params;
+    
+    const violations = [];
+    const reasoning = [];
+    let verdict = "ALLOW";
+    let confidence = 0.7;
+    
+    // Check simulation result
+    if (simulationResult && !simulationResult.passed) {
+      violations.push("Simulation failed");
+      reasoning.push("Change would break imports or routes");
+      verdict = "BLOCK";
+      confidence = 0.9;
+    }
+    
+    // Check risk score
+    if (riskScore?.total >= 80) {
+      violations.push(`High risk score: ${riskScore.total}`);
+      reasoning.push("Risk score exceeds safe threshold");
+      verdict = verdict === "BLOCK" ? "BLOCK" : "REQUIRE_CONFIRMATION";
+      confidence = Math.max(confidence, 0.8);
+    }
+    
+    // Check unverified assumptions
+    if (validationResults?.invalid?.length > 0) {
+      for (const invalid of validationResults.invalid) {
+        violations.push(`Unverified assumption: ${invalid.assumption?.key || invalid.assumption?.type}`);
+      }
+      reasoning.push(`${validationResults.invalid.length} assumptions could not be verified`);
+      verdict = "BLOCK";
+      confidence = 0.85;
+    }
+    
+    // Check vagueness
+    const vaguenessCheck = this.ruleBasedVaguenessCheck(proposal);
+    if (vaguenessCheck.specificityScore < 4) {
+      violations.push("Proposal is too vague");
+      reasoning.push(`Specificity score: ${vaguenessCheck.specificityScore}/10`);
+      if (verdict === "ALLOW") verdict = "REQUIRE_CONFIRMATION";
+    }
+    
+    // Check for sensitive domains without explicit acknowledgment
+    const sensitiveDomains = ["auth", "payments", "database"];
+    const touchesSensitive = (proposal.operations || []).some(op => {
+      const path = (op.path || "").toLowerCase();
+      return sensitiveDomains.some(d => path.includes(d));
+    });
+    
+    if (touchesSensitive && !proposal.riskAcknowledgment) {
+      violations.push("Touches sensitive domains without risk acknowledgment");
+      reasoning.push("Changes to auth/payments/database require explicit acknowledgment");
+      if (verdict === "ALLOW") verdict = "REQUIRE_CONFIRMATION";
+    }
+    
+    // No violations = allow
+    if (violations.length === 0) {
+      reasoning.push("No violations detected");
+    }
+    
+    return {
+      verdict,
+      confidence,
+      reasoning,
+      violations,
+      recommendations: violations.length > 0 
+        ? ["Address violations before proceeding", "Add missing assumptions"]
+        : [],
+    };
+  }
+  
+  /**
+   * Rule-based vagueness check
+   * @param {Object} proposal - Proposal to check
+   * @returns {Object} Vagueness analysis
+   */
+  ruleBasedVaguenessCheck(proposal) {
+    const vagueTerms = [];
+    let specificityScore = 10;
+    
+    // Check intent
+    const vagueIntents = ["fix", "update", "change", "modify", "improve", "refactor", "adjust"];
+    const intentWords = (proposal.intent || "").toLowerCase().split("_");
+    
+    if (intentWords.length === 1 && vagueIntents.includes(intentWords[0])) {
+      vagueTerms.push(proposal.intent);
+      specificityScore -= 3;
+    }
+    
+    // Check summary
+    if (!proposal.summary) {
+      specificityScore -= 2;
+    } else if (proposal.summary.length < 20) {
+      specificityScore -= 1;
+    }
+    
+    // Check assumptions
+    if (!proposal.assumptions || proposal.assumptions.length === 0) {
+      specificityScore -= 2;
+    }
+    
+    // Check operation count vs explanation
+    const opCount = (proposal.operations || []).length;
+    if (opCount > 3 && (!proposal.summary || proposal.summary.length < 50)) {
+      specificityScore -= 2;
+    }
+    
+    // Check for vague words in summary
+    const vagueWords = ["some", "various", "etc", "stuff", "things", "somehow"];
+    if (proposal.summary) {
+      for (const word of vagueWords) {
+        if (proposal.summary.toLowerCase().includes(word)) {
+          vagueTerms.push(word);
+          specificityScore -= 1;
+        }
+      }
+    }
+    
+    specificityScore = Math.max(1, specificityScore);
+    
+    return {
+      specificityScore,
+      vagueTerms,
+      suggestions: vagueTerms.length > 0 
+        ? [`Replace vague terms: ${vagueTerms.join(", ")}`, "Add specific details"]
+        : [],
+    };
+  }
+  
+  /**
+   * Rule-based assumption verification
+   * @param {Array} assumptions - Assumptions to verify
+   * @param {Object} realityState - Repository state
+   * @returns {Object} Verification results
+   */
+  ruleBasedAssumptionVerification(assumptions, realityState) {
+    const results = [];
+    let verifiedCount = 0;
+    
+    for (const assumption of assumptions) {
+      let verified = false;
+      let evidence = null;
+      let reason = "";
+      
+      switch (assumption.type) {
+        case "env":
+          if (realityState?.envVars?.has(assumption.key)) {
+            verified = true;
+            evidence = `Found in env vars: ${assumption.key}`;
+          } else {
+            reason = `Env var '${assumption.key}' not found in declared variables`;
+          }
+          break;
+          
+        case "route":
+          const routeExists = realityState?.routes?.some(r => 
+            r.path === assumption.path && 
+            (r.method === assumption.method || !assumption.method)
+          );
+          if (routeExists) {
+            verified = true;
+            evidence = `Route ${assumption.method || "GET"} ${assumption.path} is registered`;
+          } else {
+            reason = `Route ${assumption.path} not found in registered routes`;
+          }
+          break;
+          
+        case "service":
+          const serviceExists = realityState?.services?.some(s => 
+            s.name === assumption.key || s.name === assumption.name
+          );
+          if (serviceExists) {
+            verified = true;
+            evidence = `Service ${assumption.key || assumption.name} is registered`;
+          } else {
+            reason = `Service ${assumption.key || assumption.name} not found`;
+          }
+          break;
+          
+        case "file":
+          if (realityState?.files?.has(assumption.path?.replace(/\\/g, "/"))) {
+            verified = true;
+            evidence = `File ${assumption.path} exists`;
+          } else {
+            reason = `File ${assumption.path} not found`;
+          }
+          break;
+          
+        default:
+          reason = `Unknown assumption type: ${assumption.type}`;
+      }
+      
+      if (verified) verifiedCount++;
+      
+      results.push({
+        assumption: assumption.key || assumption.path || assumption.type,
+        verified,
+        evidence,
+        reason,
+      });
+    }
+    
+    return {
+      results,
+      overallVerificationRate: assumptions.length > 0 
+        ? verifiedCount / assumptions.length 
+        : 1,
+    };
+  }
+}
+
+/**
+ * Create a critic judge instance
+ * @param {Object} options - Configuration options
+ * @returns {CriticJudge} Judge instance
+ */
+function createJudge(options = {}) {
+  return new CriticJudge(options);
+}
+
+/**
+ * Default judge instance
+ */
+const defaultJudge = createJudge();
+
+module.exports = {
+  CriticJudge,
+  createJudge,
+  defaultJudge,
+};