@hailer/mcp 0.0.5 → 0.1.0

package/.claude/hooks/agent-failure-detector.cjs

@@ -0,0 +1,286 @@
+#!/usr/bin/env node
+/**
+ * Agent Failure Detector Hook
+ *
+ * PostToolUse hook for Task tool - detects agent failures, categorizes them,
+ * and suggests appropriate fixes (skill creation vs agent update).
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const TRACKER_FILE = path.join(os.tmpdir(), '.claude-agent-failures.json');
+const FAILURE_THRESHOLD = 2;
+
+let input = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', chunk => input += chunk);
+process.stdin.on('end', () => {
+  try {
+    const data = JSON.parse(input);
+    processHook(data);
+  } catch {
+    process.exit(0);
+  }
+});
+
+function loadTracker() {
+  try {
+    if (fs.existsSync(TRACKER_FILE)) {
+      return JSON.parse(fs.readFileSync(TRACKER_FILE, 'utf8'));
+    }
+  } catch {}
+  return { failures: {}, prompted: {}, errorTypes: {} };
+}
+
+function saveTracker(tracker) {
+  fs.writeFileSync(TRACKER_FILE, JSON.stringify(tracker, null, 2));
+}
+
+// Error categorization - determines fix type
+const ERROR_CATEGORIES = {
+  // API/Schema errors -> Create skill (reusable knowledge)
+  skill: {
+    patterns: [
+      'validation error',
+      'invalid field',
+      'schema',
+      'required parameter',
+      'type mismatch',
+      'format error',
+      'join syntax',
+      'sql error',
+      'query error',
+      'activitylink',
+      'workflow.*not found',
+      'phase.*not found',
+      'field.*not found'
+    ],
+    suggestion: 'Create a skill documenting the correct pattern',
+    action: 'create_skill'
+  },
+  // Agent behavior errors -> Update agent
+  agent: {
+    patterns: [
+      'wrong approach',
+      'misunderstood',
+      'incorrect assumption',
+      'should have',
+      'forgot to',
+      'didn\'t check',
+      'missing context',
+      'needs to first',
+      'order of operations'
+    ],
+    suggestion: 'Update agent definition with better instructions',
+    action: 'update_agent'
+  },
+  // Permission/Auth errors -> Neither (user issue)
+  user_issue: {
+    patterns: [
+      'permission denied',
+      'unauthorized',
+      'access denied',
+      'not allowed',
+      'admin required'
+    ],
+    suggestion: 'User permission issue - not an agent problem',
+    action: 'ignore'
+  },
+  // Connection/Transient errors -> Neither (retry)
+  transient: {
+    patterns: [
+      'timeout',
+      'connection',
+      'network',
+      'unavailable',
+      'try again',
+      'rate limit'
+    ],
+    suggestion: 'Transient error - retry may work',
+    action: 'ignore'
+  }
+};
+
+function categorizeError(responseText) {
+  const lower = responseText.toLowerCase();
+
+  for (const [category, config] of Object.entries(ERROR_CATEGORIES)) {
+    for (const pattern of config.patterns) {
+      const regex = new RegExp(pattern, 'i');
+      if (regex.test(lower)) {
+        return {
+          category,
+          ...config,
+          matchedPattern: pattern
+        };
+      }
+    }
+  }
+
+  // Default: likely needs skill (documentation)
+  return {
+    category: 'unknown',
+    suggestion: 'Analyze error to determine fix type',
+    action: 'analyze'
+  };
+}
+
+function detectFailure(responseText) {
+  const lower = responseText.toLowerCase();
+  const failurePatterns = [
+    '"status":"error"',
+    '"status": "error"',
+    'failed to',
+    'error:',
+    'could not',
+    'unable to',
+    'invalid',
+    'not found',
+    'exception'
+  ];
+
+  for (const pattern of failurePatterns) {
+    if (lower.includes(pattern)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function extractErrorMessage(responseText) {
+  // Try to extract meaningful error message
+  const patterns = [
+    /"error":\s*"([^"]+)"/i,
+    /"message":\s*"([^"]+)"/i,
+    /Error:\s*([^\n]+)/i,
+    /failed to ([^\n.]+)/i
+  ];
+
+  for (const pattern of patterns) {
+    const match = responseText.match(pattern);
+    if (match) {
+      return match[1].substring(0, 100); // Limit length
+    }
+  }
+
+  return 'Unknown error';
+}
+
+function processHook(data) {
+  const { tool_name, tool_input, tool_response } = data;
+
+  if (tool_name !== 'Task') {
+    process.exit(0);
+  }
+
+  const agentName = tool_input?.subagent_type;
+  if (!agentName || agentName === 'ada') {
+    process.exit(0);
+  }
+
+  const responseText = typeof tool_response === 'string'
+    ? tool_response
+    : JSON.stringify(tool_response || '');
+
+  if (!detectFailure(responseText)) {
+    process.exit(0);
+  }
+
+  // Categorize the error
+  const errorInfo = categorizeError(responseText);
+  const errorMessage = extractErrorMessage(responseText);
+
+  // Skip if it's a user issue or transient
+  if (errorInfo.action === 'ignore') {
+    process.exit(0);
+  }
+
+  // Track the failure with category
+  const tracker = loadTracker();
+  const key = `${agentName}:${errorInfo.category}`;
+  tracker.failures[key] = (tracker.failures[key] || 0) + 1;
+  tracker.errorTypes[key] = errorInfo;
+  saveTracker(tracker);
+
+  const failureCount = tracker.failures[key];
+
+  if (failureCount >= FAILURE_THRESHOLD && !tracker.prompted[key]) {
+    tracker.prompted[key] = true;
+    saveTracker(tracker);
+
+    // Build suggestion based on category
+    let fixOptions;
+    let recommendation;
+
+    if (errorInfo.action === 'create_skill') {
+      recommendation = `📚 RECOMMENDED: Create a skill to document this pattern`;
+      fixOptions = [
+        { label: 'Create skill', description: `Ada creates docs/skills/ entry for ${errorInfo.matchedPattern || 'this pattern'}` },
+        { label: 'Update agent instead', description: `Add knowledge directly to ${agentName}'s definition` },
+        { label: 'Ignore', description: 'Continue without changes' }
+      ];
+    } else if (errorInfo.action === 'update_agent') {
+      recommendation = `🤖 RECOMMENDED: Update ${agentName}'s agent definition`;
+      fixOptions = [
+        { label: 'Update agent', description: `Ada adds better instructions to ${agentName}.md` },
+        { label: 'Create skill instead', description: 'Document as reusable skill for all agents' },
+        { label: 'Ignore', description: 'Continue without changes' }
+      ];
+    } else {
+      recommendation = `🔍 ANALYZE: Let Ada determine the best fix`;
+      fixOptions = [
+        { label: 'Let Ada analyze', description: 'Ada will determine if skill or agent update is better' },
+        { label: 'Ignore', description: 'Continue without changes' }
+      ];
+    }
+
+    const output = `
+${'='.repeat(60)}
+🔧 AGENT IMPROVEMENT SUGGESTION
+${'='.repeat(60)}
+
+Agent "${agentName}" has failed ${failureCount} times with similar errors.
+
+**Error type:** ${errorInfo.category}
+**Error:** ${errorMessage}
+**Pattern:** ${errorInfo.matchedPattern || 'general failure'}
+
+${recommendation}
+
+USE THIS AskUserQuestion CALL:
+
+\`\`\`json
+{
+  "questions": [{
+    "question": "${agentName} failed ${failureCount}x (${errorInfo.category}). How should Ada fix this?",
+    "header": "Fix type",
+    "options": ${JSON.stringify(fixOptions)},
+    "multiSelect": false
+  }]
+}
+\`\`\`
+
+After user responds, invoke Ada with context:
+\`\`\`
+Task(subagent_type="ada", prompt=\`{
+  "task": "${errorInfo.action === 'create_skill' ? 'create_skill' : 'update_agent'}",
+  "context": {
+    "agent": "${agentName}",
+    "error_type": "${errorInfo.category}",
+    "error_message": "${errorMessage.replace(/"/g, '\\"')}",
+    "failure_count": ${failureCount}
+  },
+  "output": ["action_taken", "file_path"]
+}\`)
+\`\`\`
+
+${'='.repeat(60)}
+`;
+
+    console.error(output);
+  }
+
+  process.exit(0);
+}