@vibecheckai/cli 3.5.0 → 3.5.1

package/bin/runners/lib/agent-firewall/critic/judge.js

@@ -1,432 +0,0 @@
-/**
- * Critic LLM Judge
- * 
- * The "savage" judge that evaluates proposal quality.
- * Detects hand-waving, vague intent, and unverified assumptions.
- * 
- * Philosophy: "If this change cannot be proven safe by the repository, block it."
- */
-
-"use strict";
-
-const {
-  CRITIC_SYSTEM_PROMPT,
-  buildEvaluationPrompt,
-  buildVaguenessPrompt,
-  buildVerificationPrompt,
-  parseCriticResponse,
-} = require("./prompts");
-
-/**
- * @typedef {Object} CriticVerdict
- * @property {string} verdict - ALLOW, BLOCK, or REQUIRE_CONFIRMATION
- * @property {number} confidence - Confidence in verdict (0-1)
- * @property {Array} reasoning - Reasons for the verdict
- * @property {Array} violations - Specific violations found
- * @property {Array} recommendations - Suggestions for improvement
- */
-
-/**
- * Default LLM client configuration
- */
-const DEFAULT_CONFIG = {
-  model: "gpt-4-turbo-preview",
-  temperature: 0.1, // Low temperature for consistent judgments
-  maxTokens: 1000,
-  timeout: 30000,
-};
-
-/**
- * Critic Judge class
- */
-class CriticJudge {
-  constructor(options = {}) {
-    this.config = { ...DEFAULT_CONFIG, ...options };
-    this.llmClient = options.llmClient || null;
-    this.enabled = options.enabled !== false;
-    this.fallbackMode = options.fallbackMode || "conservative";
-  }
-  
-  /**
-   * Set the LLM client
-   * @param {Function} client - LLM client function
-   */
-  setClient(client) {
-    this.llmClient = client;
-  }
-  
-  /**
-   * Check if critic is available
-   * @returns {boolean} Is available
-   */
-  isAvailable() {
-    return this.enabled && this.llmClient !== null;
-  }
-  
-  /**
-   * Evaluate a proposal
-   * @param {Object} params - Evaluation parameters
-   * @returns {Promise<CriticVerdict>} Critic verdict
-   */
-  async evaluate(params) {
-    const {
-      proposal,
-      validationResults = {},
-      riskScore = {},
-      simulationResult = {},
-      realityState = {},
-    } = params;
-    
-    // If critic is disabled, use rule-based evaluation
-    if (!this.isAvailable()) {
-      return this.ruleBasedEvaluation(params);
-    }
-    
-    try {
-      // Build the evaluation prompt
-      const prompt = buildEvaluationPrompt({
-        proposal,
-        validationResults,
-        riskScore,
-        simulationResult,
-        realityState,
-      });
-      
-      // Call LLM
-      const response = await this.callLLM(prompt);
-      
-      // Parse response
-      const verdict = parseCriticResponse(response);
-      
-      // Validate verdict
-      return this.validateVerdict(verdict);
-    } catch (error) {
-      console.warn(`Critic LLM evaluation failed: ${error.message}`);
-      
-      // Fall back to rule-based evaluation
-      if (this.fallbackMode === "conservative") {
-        return this.ruleBasedEvaluation(params);
-      } else {
-        return {
-          verdict: "ALLOW",
-          confidence: 0.3,
-          reasoning: ["Critic unavailable, using permissive fallback"],
-          violations: [],
-          recommendations: ["Consider manual review"],
-        };
-      }
-    }
-  }
-  
-  /**
-   * Check for vagueness in proposal
-   * @param {Object} proposal - Proposal to check
-   * @returns {Promise<Object>} Vagueness analysis
-   */
-  async checkVagueness(proposal) {
-    if (!this.isAvailable()) {
-      return this.ruleBasedVaguenessCheck(proposal);
-    }
-    
-    try {
-      const prompt = buildVaguenessPrompt(proposal);
-      const response = await this.callLLM(prompt);
-      return parseCriticResponse(response);
-    } catch (error) {
-      return this.ruleBasedVaguenessCheck(proposal);
-    }
-  }
-  
-  /**
-   * Verify assumptions against reality
-   * @param {Array} assumptions - Assumptions to verify
-   * @param {Object} realityState - Repository state
-   * @returns {Promise<Object>} Verification results
-   */
-  async verifyAssumptions(assumptions, realityState) {
-    if (!this.isAvailable()) {
-      return this.ruleBasedAssumptionVerification(assumptions, realityState);
-    }
-    
-    try {
-      const prompt = buildVerificationPrompt(assumptions, realityState);
-      const response = await this.callLLM(prompt);
-      return parseCriticResponse(response);
-    } catch (error) {
-      return this.ruleBasedAssumptionVerification(assumptions, realityState);
-    }
-  }
-  
-  /**
-   * Call the LLM
-   * @param {string} prompt - User prompt
-   * @returns {Promise<string>} LLM response
-   */
-  async callLLM(prompt) {
-    if (!this.llmClient) {
-      throw new Error("LLM client not configured");
-    }
-    
-    return this.llmClient({
-      systemPrompt: CRITIC_SYSTEM_PROMPT,
-      userPrompt: prompt,
-      model: this.config.model,
-      temperature: this.config.temperature,
-      maxTokens: this.config.maxTokens,
-    });
-  }
-  
-  /**
-   * Validate and normalize verdict
-   * @param {Object} verdict - Raw verdict
-   * @returns {CriticVerdict} Validated verdict
-   */
-  validateVerdict(verdict) {
-    const validVerdicts = ["ALLOW", "BLOCK", "REQUIRE_CONFIRMATION"];
-    
-    return {
-      verdict: validVerdicts.includes(verdict.verdict) ? verdict.verdict : "BLOCK",
-      confidence: typeof verdict.confidence === "number" 
-        ? Math.max(0, Math.min(1, verdict.confidence))
-        : 0.5,
-      reasoning: Array.isArray(verdict.reasoning) ? verdict.reasoning : [],
-      violations: Array.isArray(verdict.violations) ? verdict.violations : [],
-      recommendations: Array.isArray(verdict.recommendations) ? verdict.recommendations : [],
-    };
-  }
-  
-  /**
-   * Rule-based evaluation fallback
-   * @param {Object} params - Evaluation parameters
-   * @returns {CriticVerdict} Verdict
-   */
-  ruleBasedEvaluation(params) {
-    const { proposal, validationResults, riskScore, simulationResult } = params;
-    
-    const violations = [];
-    const reasoning = [];
-    let verdict = "ALLOW";
-    let confidence = 0.7;
-    
-    // Check simulation result
-    if (simulationResult && !simulationResult.passed) {
-      violations.push("Simulation failed");
-      reasoning.push("Change would break imports or routes");
-      verdict = "BLOCK";
-      confidence = 0.9;
-    }
-    
-    // Check risk score
-    if (riskScore?.total >= 80) {
-      violations.push(`High risk score: ${riskScore.total}`);
-      reasoning.push("Risk score exceeds safe threshold");
-      verdict = verdict === "BLOCK" ? "BLOCK" : "REQUIRE_CONFIRMATION";
-      confidence = Math.max(confidence, 0.8);
-    }
-    
-    // Check unverified assumptions
-    if (validationResults?.invalid?.length > 0) {
-      for (const invalid of validationResults.invalid) {
-        violations.push(`Unverified assumption: ${invalid.assumption?.key || invalid.assumption?.type}`);
-      }
-      reasoning.push(`${validationResults.invalid.length} assumptions could not be verified`);
-      verdict = "BLOCK";
-      confidence = 0.85;
-    }
-    
-    // Check vagueness
-    const vaguenessCheck = this.ruleBasedVaguenessCheck(proposal);
-    if (vaguenessCheck.specificityScore < 4) {
-      violations.push("Proposal is too vague");
-      reasoning.push(`Specificity score: ${vaguenessCheck.specificityScore}/10`);
-      if (verdict === "ALLOW") verdict = "REQUIRE_CONFIRMATION";
-    }
-    
-    // Check for sensitive domains without explicit acknowledgment
-    const sensitiveDomains = ["auth", "payments", "database"];
-    const touchesSensitive = (proposal.operations || []).some(op => {
-      const path = (op.path || "").toLowerCase();
-      return sensitiveDomains.some(d => path.includes(d));
-    });
-    
-    if (touchesSensitive && !proposal.riskAcknowledgment) {
-      violations.push("Touches sensitive domains without risk acknowledgment");
-      reasoning.push("Changes to auth/payments/database require explicit acknowledgment");
-      if (verdict === "ALLOW") verdict = "REQUIRE_CONFIRMATION";
-    }
-    
-    // No violations = allow
-    if (violations.length === 0) {
-      reasoning.push("No violations detected");
-    }
-    
-    return {
-      verdict,
-      confidence,
-      reasoning,
-      violations,
-      recommendations: violations.length > 0 
-        ? ["Address violations before proceeding", "Add missing assumptions"]
-        : [],
-    };
-  }
-  
-  /**
-   * Rule-based vagueness check
-   * @param {Object} proposal - Proposal to check
-   * @returns {Object} Vagueness analysis
-   */
-  ruleBasedVaguenessCheck(proposal) {
-    const vagueTerms = [];
-    let specificityScore = 10;
-    
-    // Check intent
-    const vagueIntents = ["fix", "update", "change", "modify", "improve", "refactor", "adjust"];
-    const intentWords = (proposal.intent || "").toLowerCase().split("_");
-    
-    if (intentWords.length === 1 && vagueIntents.includes(intentWords[0])) {
-      vagueTerms.push(proposal.intent);
-      specificityScore -= 3;
-    }
-    
-    // Check summary
-    if (!proposal.summary) {
-      specificityScore -= 2;
-    } else if (proposal.summary.length < 20) {
-      specificityScore -= 1;
-    }
-    
-    // Check assumptions
-    if (!proposal.assumptions || proposal.assumptions.length === 0) {
-      specificityScore -= 2;
-    }
-    
-    // Check operation count vs explanation
-    const opCount = (proposal.operations || []).length;
-    if (opCount > 3 && (!proposal.summary || proposal.summary.length < 50)) {
-      specificityScore -= 2;
-    }
-    
-    // Check for vague words in summary
-    const vagueWords = ["some", "various", "etc", "stuff", "things", "somehow"];
-    if (proposal.summary) {
-      for (const word of vagueWords) {
-        if (proposal.summary.toLowerCase().includes(word)) {
-          vagueTerms.push(word);
-          specificityScore -= 1;
-        }
-      }
-    }
-    
-    specificityScore = Math.max(1, specificityScore);
-    
-    return {
-      specificityScore,
-      vagueTerms,
-      suggestions: vagueTerms.length > 0 
-        ? [`Replace vague terms: ${vagueTerms.join(", ")}`, "Add specific details"]
-        : [],
-    };
-  }
-  
-  /**
-   * Rule-based assumption verification
-   * @param {Array} assumptions - Assumptions to verify
-   * @param {Object} realityState - Repository state
-   * @returns {Object} Verification results
-   */
-  ruleBasedAssumptionVerification(assumptions, realityState) {
-    const results = [];
-    let verifiedCount = 0;
-    
-    for (const assumption of assumptions) {
-      let verified = false;
-      let evidence = null;
-      let reason = "";
-      
-      switch (assumption.type) {
-        case "env":
-          if (realityState?.envVars?.has(assumption.key)) {
-            verified = true;
-            evidence = `Found in env vars: ${assumption.key}`;
-          } else {
-            reason = `Env var '${assumption.key}' not found in declared variables`;
-          }
-          break;
-          
-        case "route":
-          const routeExists = realityState?.routes?.some(r => 
-            r.path === assumption.path && 
-            (r.method === assumption.method || !assumption.method)
-          );
-          if (routeExists) {
-            verified = true;
-            evidence = `Route ${assumption.method || "GET"} ${assumption.path} is registered`;
-          } else {
-            reason = `Route ${assumption.path} not found in registered routes`;
-          }
-          break;
-          
-        case "service":
-          const serviceExists = realityState?.services?.some(s => 
-            s.name === assumption.key || s.name === assumption.name
-          );
-          if (serviceExists) {
-            verified = true;
-            evidence = `Service ${assumption.key || assumption.name} is registered`;
-          } else {
-            reason = `Service ${assumption.key || assumption.name} not found`;
-          }
-          break;
-          
-        case "file":
-          if (realityState?.files?.has(assumption.path?.replace(/\\/g, "/"))) {
-            verified = true;
-            evidence = `File ${assumption.path} exists`;
-          } else {
-            reason = `File ${assumption.path} not found`;
-          }
-          break;
-          
-        default:
-          reason = `Unknown assumption type: ${assumption.type}`;
-      }
-      
-      if (verified) verifiedCount++;
-      
-      results.push({
-        assumption: assumption.key || assumption.path || assumption.type,
-        verified,
-        evidence,
-        reason,
-      });
-    }
-    
-    return {
-      results,
-      overallVerificationRate: assumptions.length > 0 
-        ? verifiedCount / assumptions.length 
-        : 1,
-    };
-  }
-}
-
-/**
- * Create a critic judge instance
- * @param {Object} options - Configuration options
- * @returns {CriticJudge} Judge instance
- */
-function createJudge(options = {}) {
-  return new CriticJudge(options);
-}
-
-/**
- * Default judge instance
- */
-const defaultJudge = createJudge();
-
-module.exports = {
-  CriticJudge,
-  createJudge,
-  defaultJudge,
-};