@howlil/ez-agents 3.5.0 → 4.0.0

package/ez-agents/bin/guards/hallucination-guard.cjs

@@ -0,0 +1,271 @@
+/**
+ * EDGE-01: Hallucination Guard
+ * 
+ * Detects AI hallucinations by requiring citations and flagging uncertainty.
+ * Verifies library claims against package.json or Context7.
+ */
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Search codebase for claim evidence
+ * @param {string} claim - The claim to verify
+ * @param {string} searchDir - Directory to search in
+ * @returns {object} { cited: boolean, citations: array, uncertainty: boolean }
+ */
+function checkCitation(claim, searchDir = process.cwd()) {
+  const citations = [];
+  
+  try {
+    // Search in codebase using grep
+    const grepCmd = process.platform === 'win32' 
+      ? `findstr /s /i /c:"${claim}" * 2>nul`
+      : `grep -ri "${claim}" . 2>/dev/null | head -20`;
+    
+    const result = execSync(grepCmd, {
+      cwd: searchDir,
+      encoding: 'utf8',
+      stdio: ['pipe', 'pipe', 'ignore']
+    });
+    
+    if (result.trim()) {
+      const lines = result.trim().split('\n').slice(0, 5);
+      lines.forEach(line => {
+        citations.push({
+          source: 'codebase',
+          evidence: line.trim()
+        });
+      });
+    }
+  } catch (e) {
+    // No matches found - not an error, just no citation
+  }
+  
+  return {
+    cited: citations.length > 0,
+    citations,
+    uncertainty: citations.length === 0
+  };
+}
+
+/**
+ * Verify a library/dependency claim
+ * @param {string} claim - Library name or claim
+ * @param {string} projectDir - Project directory
+ * @returns {object} { verified: boolean, source: string, details: object }
+ */
+function verifyClaim(claim, projectDir = process.cwd()) {
+  // Extract library name from claim
+  const libMatch = claim.match(/(?:library|package|module|dependency)[:\s]+(['"]?)([^'"\s]+)\1/i);
+  const libName = libMatch ? libMatch[2] : claim.split(/\s+/).pop();
+  
+  // Check package.json
+  const packageJsonPath = path.join(projectDir, 'package.json');
+  if (fs.existsSync(packageJsonPath)) {
+    try {
+      const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+      const allDeps = {
+        ...pkg.dependencies,
+        ...pkg.devDependencies,
+        ...pkg.peerDependencies
+      };
+      
+      // Check if library is in dependencies
+      for (const [name, version] of Object.entries(allDeps)) {
+        if (name.toLowerCase().includes(libName.toLowerCase()) || 
+            libName.toLowerCase().includes(name.toLowerCase())) {
+          return {
+            verified: true,
+            source: 'package.json',
+            details: { name, version }
+          };
+        }
+      }
+    } catch (e) {
+      // Invalid package.json
+    }
+  }
+  
+  // Check if it's a known library (would verify with Context7 in real implementation)
+  const knownLibraries = [
+    'vitest', 'jest', 'playwright', 'cypress', 'react', 'vue', 'angular',
+    'express', 'fastify', 'nest', 'next', 'nuxt', 'sveltekit',
+    'prisma', 'mongoose', 'sequelize', 'typeorm',
+    'typescript', 'eslint', 'prettier'
+  ];
+  
+  if (knownLibraries.some(lib => libName.toLowerCase().includes(lib))) {
+    return {
+      verified: true,
+      source: 'known-library',
+      details: { name: libName, note: 'Would verify with Context7' }
+    };
+  }
+  
+  return {
+    verified: false,
+    source: 'none',
+    details: { name: libName, note: 'Library not found in dependencies or known libraries' }
+  };
+}
+
+/**
+ * Flag uncertainty in AI output
+ * @param {string} output - AI generated text
+ * @returns {object} { flagged: boolean, uncertainPhrases: array, confidence: string }
+ */
+function flagUncertainty(output) {
+  const uncertaintyMarkers = [
+    'might', 'may', 'could', 'possibly', 'perhaps',
+    'i think', 'i believe', 'probably', 'likely',
+    'not sure', 'uncertain', 'approximate',
+    'should work', 'might work', 'could work',
+    'as far as i know', 'to the best of my knowledge',
+    'it seems', 'appears to'
+  ];
+  
+  const lowerOutput = output.toLowerCase();
+  const foundMarkers = [];
+  
+  for (const marker of uncertaintyMarkers) {
+    if (lowerOutput.includes(marker)) {
+      foundMarkers.push(marker);
+    }
+  }
+  
+  // Check for citation needed patterns
+  const citationPatterns = [
+    /according to.*documentation/i,
+    /the documentation says/i,
+    /as documented in/i,
+    /the official.*states/i
+  ];
+  
+  for (const pattern of citationPatterns) {
+    if (pattern.test(output)) {
+      foundMarkers.push(`citation claim: ${pattern.source}`);
+    }
+  }
+  
+  return {
+    flagged: foundMarkers.length > 0,
+    uncertainPhrases: foundMarkers,
+    confidence: foundMarkers.length === 0 ? 'high' : 
+                foundMarkers.length <= 2 ? 'medium' : 'low'
+  };
+}
+
+/**
+ * Full hallucination check
+ * @param {string} output - AI generated text
+ * @param {string} projectDir - Project directory
+ * @returns {object} Comprehensive hallucination check result
+ */
+function checkHallucination(output, projectDir = process.cwd()) {
+  const uncertaintyResult = flagUncertainty(output);
+  
+  // Extract claims that need verification
+  const claimsToVerify = [];
+  const claimPatterns = [
+    /(?:library|package|module|dependency)[:\s]+['"]?([^'"\s,]+)['"]?/gi,
+    /(?:uses?|requires?|needs?|installs?)\s+(?:the\s+)?(['"]?)([^'"\s,]+)\1(?:\s+library)?/gi
+  ];
+  
+  for (const pattern of claimPatterns) {
+    let match;
+    while ((match = pattern.exec(output)) !== null) {
+      claimsToVerify.push(match[0]);
+    }
+  }
+  
+  // Verify each claim
+  const verifiedClaims = [];
+  const unverifiedClaims = [];
+  
+  for (const claim of claimsToVerify) {
+    const verification = verifyClaim(claim, projectDir);
+    if (verification.verified) {
+      verifiedClaims.push({ claim, verification });
+    } else {
+      unverifiedClaims.push({ claim, verification });
+    }
+  }
+  
+  return {
+    hallucinationRisk: unverifiedClaims.length > 0 || uncertaintyResult.flagged,
+    confidence: uncertaintyResult.confidence,
+    uncertainty: uncertaintyResult,
+    verifiedClaims,
+    unverifiedClaims,
+    recommendations: unverifiedClaims.length > 0 
+      ? ['Verify unverified claims against official documentation', 'Add citations for external references']
+      : uncertaintyResult.flagged
+        ? ['Reduce uncertainty language', 'Add specific citations']
+        : []
+  };
+}
+
+/**
+ * CLI entry point
+ */
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const command = args[0];
+  
+  if (command === 'check' && args[1]) {
+    const claim = args.slice(1).join(' ');
+    console.log(`Checking claim: "${claim}"`);
+    const result = checkCitation(claim);
+    
+    if (result.cited) {
+      console.log('✅ Claim has citations');
+      result.citations.forEach(c => console.log(`   Source: ${c.source}`));
+      console.log(`   Evidence: ${c.citations[0]?.evidence?.substring(0, 100)}...`);
+    } else {
+      console.log('⚠️  Claim has no citations - potential hallucination');
+    }
+    process.exit(result.cited ? 0 : 1);
+  } else if (command === 'verify' && args[1]) {
+    const claim = args.slice(1).join(' ');
+    console.log(`Verifying: "${claim}"`);
+    const result = verifyClaim(claim);
+    
+    if (result.verified) {
+      console.log(`✅ Verified via ${result.source}`);
+      console.log(`   Details: ${JSON.stringify(result.details)}`);
+    } else {
+      console.log('❌ Could not verify claim');
+      console.log(`   Details: ${result.details.note}`);
+    }
+    process.exit(result.verified ? 0 : 1);
+  } else if (command === 'uncertainty' && args[1]) {
+    const text = args.slice(1).join(' ');
+    console.log('Checking for uncertainty markers...');
+    const result = flagUncertainty(text);
+    
+    if (result.flagged) {
+      console.log(`⚠️  Found ${result.uncertainPhrases.length} uncertainty markers:`);
+      result.uncertainPhrases.forEach(p => console.log(`   - "${p}"`));
+      console.log(`Confidence: ${result.confidence}`);
+    } else {
+      console.log('✅ No uncertainty markers found');
+    }
+    process.exit(0);
+  } else {
+    console.log('Usage: node hallucination-guard.cjs <command> [text]');
+    console.log('Commands:');
+    console.log('  check <claim>      - Check if claim has citations');
+    console.log('  verify <claim>     - Verify library/dependency claim');
+    console.log('  uncertainty <text> - Check for uncertainty markers');
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  checkCitation,
+  verifyClaim,
+  flagUncertainty,
+  checkHallucination
+};

package/ez-agents/bin/guards/hidden-state-guard.cjs

@@ -0,0 +1,182 @@
+/**
+ * EDGE-03: Hidden State Guard
+ * 
+ * Detects state not persisted to .planning/ files.
+ * Ensures all state is explicitly persisted for auditability.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * List all state files in .planning/ directory
+ * @param {string} phaseDir - Phase directory
+ * @returns {string[]} Array of state file paths
+ */
+function listStateFiles(phaseDir) {
+  const stateFiles = [];
+  const planningDir = path.join(phaseDir, '.planning');
+  
+  if (!fs.existsSync(planningDir)) {
+    return stateFiles;
+  }
+  
+  // Recursively find all .md files in .planning/
+  function scanDir(dir) {
+    try {
+      const entries = fs.readdirSync(dir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory() && !entry.name.startsWith('.')) {
+          scanDir(fullPath);
+        } else if (entry.isFile() && entry.name.endsWith('.md')) {
+          stateFiles.push(fullPath);
+        }
+      }
+    } catch (e) {
+      // Ignore permission errors
+    }
+  }
+  
+  scanDir(planningDir);
+  return stateFiles;
+}
+
+/**
+ * Extract state references from text
+ * @param {string} text - Text to analyze
+ * @returns {string[]} Array of state references
+ */
+function extractStateReferences(text) {
+  const statePatterns = [
+    /(?:state|status|progress|phase|step|current)[\s:]+([A-Z_]+\d*(?:\.\d+)?)/gi,
+    /(?:task|plan|gate|edge)[\s:]+(\d+(?:-\d+)?)/gi,
+    /(?:completed|finished|done|pending|blocked)[\s:]+([A-Z_]+)/gi
+  ];
+  
+  const references = new Set();
+  
+  for (const pattern of statePatterns) {
+    let match;
+    while ((match = pattern.exec(text)) !== null) {
+      references.add(match[0].toLowerCase());
+    }
+  }
+  
+  return Array.from(references);
+}
+
+/**
+ * Check for hidden state (state not persisted to files)
+ * @param {string} output - AI generated output
+ * @param {string} phaseDir - Phase directory
+ * @returns {object} { hasHiddenState: boolean, stateFiles: array, missing: array }
+ */
+function checkHiddenState(output, phaseDir) {
+  const stateFiles = listStateFiles(phaseDir);
+  const stateReferences = extractStateReferences(output);
+  
+  // Read all state file contents
+  const stateContents = stateFiles.map(file => {
+    try {
+      return fs.readFileSync(file, 'utf8').toLowerCase();
+    } catch (e) {
+      return '';
+    }
+  }).join(' ');
+  
+  // Check which state references are not in persisted files
+  const missing = stateReferences.filter(ref => {
+    // Skip common false positives
+    if (['state', 'status', 'phase', 'plan'].includes(ref)) return false;
+    return !stateContents.includes(ref);
+  });
+  
+  return {
+    hasHiddenState: missing.length > 0,
+    stateFiles,
+    stateReferences,
+    missing,
+    recommendation: missing.length > 0 
+      ? `Persist the following state to .planning/ files: ${missing.join(', ')}`
+      : 'All state appears to be persisted'
+  };
+}
+
+/**
+ * Validate that all state is persisted
+ * @param {string} output - AI generated output
+ * @param {string} phaseDir - Phase directory
+ * @returns {object} Validation result
+ */
+function validatePersistence(output, phaseDir) {
+  const result = checkHiddenState(output, phaseDir);
+  
+  return {
+    valid: !result.hasHiddenState,
+    ...result
+  };
+}
+
+/**
+ * CLI entry point
+ */
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const command = args[0];
+  
+  if (command === 'check' && args[1]) {
+    const phaseDir = args[1];
+    console.log(`Checking for hidden state in ${phaseDir}`);
+    
+    const stateFiles = listStateFiles(phaseDir);
+    console.log(`Found ${stateFiles.length} state files in .planning/`);
+    
+    if (stateFiles.length > 0) {
+      console.log('State files:');
+      stateFiles.slice(0, 10).forEach(f => console.log(`  - ${path.relative(phaseDir, f)}`));
+      if (stateFiles.length > 10) {
+        console.log(`  ... and ${stateFiles.length - 10} more`);
+      }
+    }
+    
+    process.exit(0);
+    
+  } else if (command === 'validate' && args[1]) {
+    const phaseDir = args[1];
+    const outputFile = args[2];
+    
+    if (!outputFile || !fs.existsSync(outputFile)) {
+      console.error('Usage: node hidden-state-guard.cjs validate <phaseDir> <outputFile>');
+      process.exit(1);
+    }
+    
+    const output = fs.readFileSync(outputFile, 'utf8');
+    const result = validatePersistence(output, phaseDir);
+    
+    if (result.valid) {
+      console.log('✅ All state is persisted');
+      console.log(`   State files: ${result.stateFiles.length}`);
+      process.exit(0);
+    } else {
+      console.log('⚠️  Hidden state detected');
+      console.log(`   Missing: ${result.missing.join(', ')}`);
+      console.log(`   ${result.recommendation}`);
+      process.exit(1);
+    }
+    
+  } else {
+    console.log('Usage: node hidden-state-guard.cjs <command> [args]');
+    console.log('Commands:');
+    console.log('  check <phaseDir>            - List state files');
+    console.log('  validate <phaseDir> <file>  - Validate output persistence');
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  listStateFiles,
+  extractStateReferences,
+  checkHiddenState,
+  validatePersistence
+};