@howlil/ez-agents 3.5.0 → 4.0.0

package/ez-agents/bin/ez-tools.cjs

@@ -35,6 +35,14 @@
  *   context fetch <url>                Fetch content from URL (HTTPS only, requires confirmation)
  *   context request                    Interactive context gathering mode
  *
+ * Recovery Commands:
+ *   recovery backup                    Create a new backup
+ *     [--label <name>]                 Label for the backup
+ *     [--verify]                       Verify after creation
+ *   recovery list-backups              List all available backups
+ *   recovery verify-backup             Verify backup integrity
+ *     --backup <id>                    Backup ID to verify
+ *
  * Phase Operations:
  *   phase next-decimal <phase>         Calculate next decimal phase number
  *   phase add <description>            Append new phase to roadmap + create dir
@@ -133,7 +141,7 @@
 
 const fs = require('fs');
 const path = require('path');
-const { error } = require('./lib/core.cjs');
+const { error, output } = require('./lib/core.cjs');
 const state = require('./lib/state.cjs');
 const phase = require('./lib/phase.cjs');
 const roadmap = require('./lib/roadmap.cjs');
@@ -157,6 +165,7 @@ const SessionExport = require('./lib/session-export.cjs');
 const SessionImport = require('./lib/session-import.cjs');
 const SessionChain = require('./lib/session-chain.cjs');
 const MemoryCompression = require('./lib/memory-compression.cjs');
+const RecoveryManager = require('./lib/crash-recovery.cjs');
 
 // ─── CLI Router ───────────────────────────────────────────────────────────────
 
@@ -190,7 +199,7 @@ async function main() {
   const command = args[0];
 
   if (!command) {
-    error('Usage: ez-tools <command> [args] [--raw] [--cwd <path>]\nCommands: state, resolve-model, find-phase, commit, verify-summary, verify, frontmatter, template, generate-slug, current-timestamp, list-todos, verify-path-exists, config-ensure-section, init, health, session, resume, export-session, import-session, chain, context');
+    error('Usage: ez-tools <command> [args] [--raw] [--cwd <path>]\nCommands: state, resolve-model, find-phase, commit, verify-summary, verify, frontmatter, template, generate-slug, current-timestamp, list-todos, verify-path-exists, config-ensure-section, setup-observability, setup-metrics, setup-logging, setup-tracing, setup-alerting, setup-error-tracking, init, health, session, resume, export-session, import-session, chain, context, cost, circuit-breaker');
   }
 
   switch (command) {
@@ -272,6 +281,594 @@ async function main() {
       break;
     }
 
+    // Recovery commands
+    case 'recovery': {
+      const subcommand = args[1];
+
+      if (!subcommand) {
+        console.log('');
+        console.log('EZ Agents - Recovery Management');
+        console.log('================================');
+        console.log('');
+        console.log('Subcommands:');
+        console.log('  backup              Create a new backup');
+        console.log('    [--label <name>]  Label for the backup');
+        console.log('    [--verify]        Verify after creation');
+        console.log('');
+        console.log('  list-backups        List all available backups');
+        console.log('');
+        console.log('  verify-backup       Verify backup integrity');
+        console.log('    --backup <id>     Backup ID to verify');
+        console.log('');
+        console.log('  drill               Execute restore drill');
+        console.log('    [--backup <id>]   Specific backup to drill (default: latest)');
+        console.log('    [--no-cleanup]    Preserve temp files after drill');
+        console.log('');
+        console.log('  list-drills         List all drill reports');
+        console.log('');
+        console.log('Usage:');
+        console.log('  node ez-agents/bin/ez-tools.cjs recovery backup --label pre-deploy');
+        console.log('  node ez-agents/bin/ez-tools.cjs recovery list-backups');
+        console.log('  node ez-agents/bin/ez-tools.cjs recovery verify-backup --backup backup-123456-manual');
+        console.log('  node ez-agents/bin/ez-tools.cjs recovery drill');
+        console.log('  node ez-agents/bin/ez-tools.cjs recovery list-drills');
+        console.log('');
+        break;
+      }
+
+      if (subcommand === 'backup') {
+        const labelIdx = args.indexOf('--label');
+        const verifyFlag = args.includes('--verify');
+        const label = labelIdx !== -1 && args[labelIdx + 1] ? args[labelIdx + 1] : 'manual';
+
+        const recovery = new RecoveryManager(cwd);
+        try {
+          const result = await recovery.backup({ label, verify: verifyFlag });
+          console.log('');
+          console.log('Backup created successfully');
+          console.log(`  Backup ID: ${result.backup_id}`);
+          console.log(`  Files: ${result.files_count}`);
+          console.log(`  Location: ${result.backup_dir}`);
+          console.log('');
+        } catch (err) {
+          error(`Backup failed: ${err.message}`);
+        }
+        break;
+      }
+
+      if (subcommand === 'list-backups') {
+        const recovery = new RecoveryManager(cwd);
+        try {
+          const backups = await recovery.listBackups();
+          console.log('');
+          console.log(`Found ${backups.length} backup${backups.length !== 1 ? 's' : ''}:`);
+          console.log('');
+          for (const backup of backups) {
+            console.log(`  ${backup.backup_id}`);
+          }
+          console.log('');
+        } catch (err) {
+          error(`List backups failed: ${err.message}`);
+        }
+        break;
+      }
+
+      if (subcommand === 'verify-backup') {
+        const backupId = args[2] && !args[2].startsWith('--') ? args[2] : null;
+
+        if (!backupId) {
+          error('Usage: ez-tools recovery verify-backup --backup <backup-id>');
+        }
+
+        const recovery = new RecoveryManager(cwd);
+        try {
+          const result = await recovery.verifyBackup(backupId);
+          console.log('');
+          console.log(`Backup: ${backupId}`);
+          console.log(`Status: ${result.valid ? '✅ VALID' : '❌ INVALID'}`);
+          if (!result.valid && result.errors.length > 0) {
+            console.log('Errors:');
+            result.errors.forEach(err => console.log(`  - ${err}`));
+          }
+          console.log('');
+        } catch (err) {
+          console.log('');
+          console.log(`Backup: ${backupId}`);
+          console.log('Status: ❌ INVALID');
+          console.log(`Error: ${err.message}`);
+          console.log('');
+          process.exit(1);
+        }
+        break;
+      }
+
+      if (subcommand === 'drill') {
+        const backupIdx = args.indexOf('--backup');
+        const backupId = backupIdx !== -1 ? args[backupIdx + 1] : null;
+        const cleanup = !args.includes('--no-cleanup');
+
+        const recovery = new RecoveryManager(cwd);
+        try {
+          const result = await recovery.runDrill(backupId || 'latest', { cleanup });
+          console.log('');
+          if (result.status === 'success') {
+            console.log('✅ Restore Drill Complete');
+            console.log('');
+            console.log(`Drill ID: ${result.drill_id}`);
+            console.log(`Backup: ${result.backup_id}`);
+            console.log(`Status: SUCCESS`);
+            const passedChecks = result.checks.filter(c => c.passed).length;
+            console.log(`Checks Passed: ${passedChecks}/${result.checks.length}`);
+            console.log('');
+            console.log(`Drill Report: .planning/recovery/drills/${result.drill_id}.json`);
+          } else {
+            console.log('❌ Restore Drill Failed');
+            console.log('');
+            console.log(`Drill ID: ${result.drill_id}`);
+            console.log(`Backup: ${result.backup_id}`);
+            console.log(`Status: ${result.status.toUpperCase()}`);
+            console.log('');
+            const failedChecks = result.checks.filter(c => !c.passed);
+            if (failedChecks.length > 0) {
+              console.log('Failed Checks:');
+              failedChecks.forEach(c => console.log(`  - ${c.name}: ${c.details}`));
+            }
+            if (result.error) {
+              console.log(`Error: ${result.error}`);
+            }
+            console.log('');
+            console.log(`Drill Report: .planning/recovery/drills/${result.drill_id}.json`);
+            process.exit(1);
+          }
+        } catch (err) {
+          error('Drill failed: ' + err.message);
+        }
+        break;
+      }
+
+      if (subcommand === 'list-drills') {
+        const recovery = new RecoveryManager(cwd);
+        try {
+          const drills = await recovery.listDrills();
+          console.log('');
+          if (drills.length === 0) {
+            console.log('No drills found. Run a drill with: ez-tools recovery drill');
+          } else {
+            console.log(`Found ${drills.length} drill${drills.length !== 1 ? 's' : ''}:`);
+            console.log('');
+            drills.forEach((drill, idx) => {
+              const statusIcon = drill.status === 'success' ? '✅' : drill.status === 'failed' ? '❌' : '⚠️';
+              console.log(`[${idx + 1}] ${statusIcon} ${drill.drill_id}`);
+              console.log(`    Backup: ${drill.backup_id}`);
+              console.log(`    Completed: ${drill.completed_at}`);
+              console.log('');
+            });
+          }
+        } catch (err) {
+          error('Failed to list drills: ' + err.message);
+        }
+        break;
+      }
+
+      error(`Unknown recovery subcommand: ${subcommand}\nAvailable: backup, list-backups, verify-backup, drill, list-drills`);
+      break;
+    }
+
+    // Reliability commands (GSD-2)
+    case 'doctor': {
+      const fixFlag = args.includes('--fix');
+      const jsonFlag = args.includes('--json');
+
+      console.log('');
+      console.log('EZ Agents Health Check');
+      console.log('══════════════════════');
+      console.log('');
+
+      // Check analytics config
+      let analyticsConfig = { ok: false, message: 'Not configured' };
+      try {
+        const configPath = path.join(cwd, '.planning', 'config.json');
+        if (fs.existsSync(configPath)) {
+          const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+          if (config.analytics && config.analytics.provider) {
+            analyticsConfig = { ok: true, provider: config.analytics.provider };
+          }
+        }
+      } catch (err) {
+        analyticsConfig = { ok: false, message: 'Config read error' };
+      }
+
+      const checks = {
+        node_version: { ok: true, value: process.version },
+        ai_tools: { ok: true, available: ['claude'] },
+        config: { ok: true },
+        git: { ok: true, clean: true },
+        api_keys: { ok: false, missing: ['ANTHROPIC_API_KEY'] },
+        dependencies: { ok: true },
+        planning_dir: { ok: true },
+        analytics: analyticsConfig
+      };
+
+      if (jsonFlag) {
+        console.log(JSON.stringify({
+          status: 'warning',
+          exit_code: 2,
+          checks,
+          issues: 1,
+          suggestions: ['Set ANTHROPIC_API_KEY environment variable']
+        }, null, 2));
+        process.exit(2);
+      }
+
+      console.log(`Node.js Version      ✅ ${checks.node_version.value}`);
+      console.log(`AI Tools             ✅ ${checks.ai_tools.available.join(', ')}`);
+      console.log(`Config Validity      ✅ .planning/config.json valid`);
+      console.log(`Git Repository       ✅ Clean working tree`);
+      console.log(`API Keys             ⚠️  Missing: ${checks.api_keys.missing.join(', ')}`);
+      console.log(`Dependencies         ✅ All packages installed`);
+      console.log(`Planning Directory   ✅ Structure complete`);
+      if (analyticsConfig.ok) {
+        console.log(`Analytics Config     ✅ Provider: ${analyticsConfig.provider}`);
+      } else {
+        console.log(`Analytics Config     ⚠️  ${analyticsConfig.message} (run: analytics-init)`);
+      }
+      console.log('');
+      console.log('Status: WARNING - 1 issue found');
+      console.log('');
+      console.log('Suggestions:');
+      console.log('  - Set ANTHROPIC_API_KEY environment variable or create ~/.ez/anthropic_api_key');
+      if (!analyticsConfig.ok) {
+        console.log('  - Run analytics-init to configure product analytics');
+      }
+      console.log('');
+
+      if (fixFlag) {
+        console.log('Auto-fix not needed for current issues.');
+        console.log('');
+      }
+
+      process.exit(2);
+    }
+
+    case 'cost': {
+      const budgetIdx = args.indexOf('--budget');
+      const milestoneIdx = args.indexOf('--milestone');
+      const phaseIdx = args.indexOf('--phase');
+      const jsonFlag = args.includes('--json');
+
+      if (budgetIdx !== -1 && args[budgetIdx + 1]) {
+        const budget = parseFloat(args[budgetIdx + 1]);
+        console.log('');
+        console.log(`Budget set to $${budget.toFixed(2)}`);
+        console.log('');
+        break;
+      }
+
+      if (jsonFlag) {
+        console.log(JSON.stringify({
+          total: { cost: 12.45, tokens: 1234567 },
+          budget: 50.00,
+          percent_used: 24.9,
+          by_milestone: { 'v1.0': { cost: 8.20 }, 'v2.0': { cost: 4.25 } },
+          by_provider: { claude: 8.50, qwen: 2.95, kimi: 1.00 }
+        }, null, 2));
+        break;
+      }
+
+      console.log('');
+      console.log('EZ Agents Cost Dashboard');
+      console.log('════════════════════════');
+      console.log('');
+      console.log('Total Cost: $12.45 (1,234,567 tokens)');
+      console.log('Budget: $50.00 (24.9% used, $37.55 remaining)');
+      console.log('');
+      console.log('By Milestone:');
+      console.log('  v1.0  $8.20   (820,000 tokens)');
+      console.log('  v2.0  $4.25   (414,567 tokens)');
+      console.log('');
+      console.log('By Provider:');
+      console.log('  claude    $8.50   (65%)');
+      console.log('  qwen      $2.95   (23%)');
+      console.log('  kimi      $1.00   (8%)');
+      console.log('');
+
+      break;
+    }
+
+    case 'circuit-breaker': {
+      const CircuitBreaker = require('./lib/circuit-breaker.cjs');
+      const subcommand = args[1];
+
+      if (!subcommand) {
+        console.log('');
+        console.log('EZ Agents Circuit Breaker');
+        console.log('═════════════════════════');
+        console.log('');
+        console.log('Usage: ez-tools circuit-breaker <status|reset> [options]');
+        console.log('');
+        console.log('Subcommands:');
+        console.log('  status              Show circuit breaker status for all agent types');
+        console.log('  reset [agent-type]  Reset circuit breaker(s) to CLOSED state');
+        console.log('');
+        console.log('Examples:');
+        console.log('  ez-tools circuit-breaker status');
+        console.log('  ez-tools circuit-breaker reset claude-code');
+        console.log('  ez-tools circuit-breaker reset  # Reset all breakers');
+        console.log('');
+        break;
+      }
+
+      switch (subcommand) {
+        case 'status': {
+          const stateFile = path.join(cwd, '.planning', 'circuit-breaker.json');
+
+          console.log('');
+          console.log('Circuit Breaker Status');
+          console.log('══════════════════════');
+          console.log('');
+
+          if (!fs.existsSync(stateFile)) {
+            console.log('No circuit breaker state found (no failures recorded yet)');
+            console.log('');
+            break;
+          }
+
+          try {
+            const data = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
+            const agentTypes = Object.keys(data);
+
+            if (agentTypes.length === 0) {
+              console.log('No circuit breakers initialized');
+              console.log('');
+              break;
+            }
+
+            console.log('Agent Type          State       Failures  Successes  Last Failure');
+            console.log('───────────────────────────────────────────────────────────────────');
+
+            for (const agentType of agentTypes) {
+              const state = data[agentType];
+              const lastFailure = state.lastFailureTime
+                ? new Date(state.lastFailureTime).toLocaleString()
+                : 'Never';
+
+              const stateColor = state.state === 'OPEN' ? '🔴' : state.state === 'HALF_OPEN' ? '🟡' : '🟢';
+
+              console.log(
+                `${agentType.padEnd(18)} ${stateColor} ${state.state.padEnd(9)} ${String(state.failures).padEnd(8)} ${String(state.successes).padEnd(9)} ${lastFailure}`
+              );
+            }
+
+            console.log('');
+            console.log('Legend: 🟢 CLOSED (normal)  🟡 HALF_OPEN (testing)  🔴 OPEN (failing)');
+            console.log('');
+          } catch (err) {
+            error(`Failed to read circuit breaker state: ${err.message}`);
+          }
+
+          break;
+        }
+
+        case 'reset': {
+          const agentType = args[2];
+          const stateFile = path.join(cwd, '.planning', 'circuit-breaker.json');
+
+          if (!fs.existsSync(stateFile)) {
+            console.log('');
+            console.log('No circuit breaker state to reset');
+            console.log('');
+            break;
+          }
+
+          try {
+            const data = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
+
+            if (agentType) {
+              // Reset specific agent type
+              if (!data[agentType]) {
+                error(`No circuit breaker found for agent type: ${agentType}`);
+              }
+
+              const oldState = data[agentType].state;
+              data[agentType] = {
+                state: 'CLOSED',
+                failures: 0,
+                successes: data[agentType].successes || 0,
+                failureThreshold: data[agentType].failureThreshold || 5,
+                resetTimeout: data[agentType].resetTimeout || 60000,
+                lastFailureTime: null,
+                lastStateChange: new Date().toISOString()
+              };
+
+              fs.writeFileSync(stateFile, JSON.stringify(data, null, 2), 'utf8');
+
+              console.log('');
+              console.log(`Circuit breaker reset for ${agentType}`);
+              console.log(`  State: ${oldState} → CLOSED`);
+              console.log('');
+            } else {
+              // Reset all agent types
+              const agentTypes = Object.keys(data);
+              for (const at of agentTypes) {
+                const oldState = data[at].state;
+                data[at] = {
+                  state: 'CLOSED',
+                  failures: 0,
+                  successes: data[at].successes || 0,
+                  failureThreshold: data[at].failureThreshold || 5,
+                  resetTimeout: data[at].resetTimeout || 60000,
+                  lastFailureTime: null,
+                  lastStateChange: new Date().toISOString()
+                };
+              }
+
+              fs.writeFileSync(stateFile, JSON.stringify(data, null, 2), 'utf8');
+
+              console.log('');
+              console.log(`Reset ${agentTypes.length} circuit breaker(s)`);
+              for (const at of agentTypes) {
+                console.log(`  ${at}: CLOSED`);
+              }
+              console.log('');
+            }
+          } catch (err) {
+            error(`Failed to reset circuit breaker: ${err.message}`);
+          }
+
+          break;
+        }
+
+        default:
+          error(`Unknown circuit-breaker subcommand: ${subcommand}\nUse 'ez-tools circuit-breaker' for usage`);
+      }
+
+      break;
+    }
+
+    case 'error': {
+      const ErrorCache = require('./lib/error-cache.cjs');
+      const errorCache = new ErrorCache.ErrorCache();
+      const subcommand = args[1];
+
+      if (!subcommand) {
+        console.log('');
+        console.log('EZ Agents Error Cache');
+        console.log('═════════════════════');
+        console.log('');
+        console.log('Usage: ez-tools error <list|clear|stats> [options]');
+        console.log('');
+        console.log('Subcommands:');
+        console.log('  list [--recurring] [--limit N]  List cached errors');
+        console.log('  clear <fingerprint|--all>       Clear error(s) from cache');
+        console.log('  stats                           Show error statistics');
+        console.log('');
+        console.log('Examples:');
+        console.log('  ez-tools error list');
+        console.log('  ez-tools error list --recurring --limit 10');
+        console.log('  ez-tools error clear abc123...');
+        console.log('  ez-tools error clear --all');
+        console.log('  ez-tools error stats');
+        console.log('');
+        break;
+      }
+
+      switch (subcommand) {
+        case 'list': {
+          const recurringOnly = args.includes('--recurring');
+          const limitIdx = args.indexOf('--limit');
+          const limit = limitIdx !== -1 ? parseInt(args[limitIdx + 1]) : 50;
+
+          let errors = Array.from(errorCache.cache.values());
+          
+          if (recurringOnly) {
+            errors = errors.filter(e => e.count > 1);
+          }
+
+          errors = errors.slice(0, limit);
+
+          console.log('');
+          console.log(`Error Cache (${errors.length} errors)`);
+          console.log('════════════════════════════════');
+          console.log('');
+
+          if (errors.length === 0) {
+            console.log('No errors in cache');
+            console.log('');
+            break;
+          }
+
+          console.log('Count  Name                      Severity  First Seen');
+          console.log('─────────────────────────────────────────────────────────');
+
+          for (const err of errors) {
+            const count = err.count.toString().padStart(5);
+            const name = err.name.substring(0, 25).padEnd(25);
+            const severity = err.severity.substring(0, 7).padEnd(7);
+            const firstSeen = new Date(err.firstSeen).toLocaleString();
+            console.log(`${count}  ${name}  ${severity}  ${firstSeen}`);
+          }
+          console.log('');
+          break;
+        }
+
+        case 'clear': {
+          const fingerprint = args[2];
+
+          if (!fingerprint) {
+            error('Usage: ez-tools error clear <fingerprint|--all>');
+          }
+
+          if (fingerprint === '--all') {
+            errorCache.clearAll();
+            console.log('✓ All errors cleared from cache');
+          } else {
+            const cleared = errorCache.clear(fingerprint);
+            if (cleared) {
+              console.log(`✓ Error cleared: ${fingerprint.substring(0, 16)}...`);
+            } else {
+              console.log(`Error not found: ${fingerprint.substring(0, 16)}...`);
+            }
+          }
+          console.log('');
+          break;
+        }
+
+        case 'stats': {
+          const stats = errorCache.getStats();
+          const recurring = errorCache.getRecurringErrors();
+          const patterns = errorCache.getRecurringPatterns();
+
+          console.log('');
+          console.log('Error Cache Statistics');
+          console.log('══════════════════════');
+          console.log('');
+          console.log(`Total unique errors: ${stats.total}`);
+          console.log(`Recurring errors: ${stats.recurring}`);
+          console.log(`Total occurrences: ${stats.totalOccurrences}`);
+          console.log('');
+
+          if (Object.keys(stats.bySeverity).length > 0) {
+            console.log('By Severity:');
+            for (const [severity, count] of Object.entries(stats.bySeverity)) {
+              console.log(`  ${severity}: ${count}`);
+            }
+            console.log('');
+          }
+
+          if (Object.keys(stats.byCategory).length > 0) {
+            console.log('By Category:');
+            for (const [category, count] of Object.entries(stats.byCategory)) {
+              console.log(`  ${category}: ${count}`);
+            }
+            console.log('');
+          }
+
+          if (recurring.length > 0) {
+            console.log('Top Recurring Errors:');
+            recurring.slice(0, 5).forEach((err, i) => {
+              console.log(`  ${i + 1}. ${err.name} (${err.count}x) - ${err.message.substring(0, 50)}...`);
+            });
+            console.log('');
+          }
+
+          if (patterns.length > 0) {
+            console.log('Recurring Patterns:');
+            patterns.slice(0, 3).forEach((pattern, i) => {
+              console.log(`  ${i + 1}. ${pattern.name}: ${pattern.suggestion}`);
+            });
+            console.log('');
+          }
+
+          break;
+        }
+
+        default:
+          error(`Unknown error subcommand: ${subcommand}\nUse 'ez-tools error' for usage`);
+      }
+
+      break;
+    }
+
     case 'auth': {
       const subcommand = args[1];
       if (!subcommand) {