aidp 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ceebdfcb742085ac8d9ea6b5db63557d5fe999f498e97699c199fc9651cf4654
-  data.tar.gz: f1b230dcc33692c4c0391464beaf305b76993dbe02964d8c80198a7a50294a5c
+  metadata.gz: 26282ecc76062d41d599eb9bc8190303bcef16257fb98fa25250eecaf894270a
+  data.tar.gz: ee50c920a5aa61d2883f906c7bea5d09c6835725c27c359bd2bce7d698f594e2
 SHA512:
-  metadata.gz: d20547fa251feb09a97e6795b04fc09dd13d0f0fd801c5abdc994bb27a20898a604692eb04b7ed8a370ac7619bcbaf89a17af60dc34c844e782e8b68087ef67a
-  data.tar.gz: cb8503f9b0a85d6113d9ab2dada1cb0087b9b9e6ab1abb44894aa5152d3596c782c08f4708e3f03f9b969a94736acb7d882350b6f0cb869787069c8b12ddde85
+  metadata.gz: 9fa312b72678f5bba517ee9add8c9fcd1bc06237f65781ddf429bb687b7f3a85fb3e031ebe992cb768d53885a09648d74c6f9d57282a6f988eeada7994ff534f
+  data.tar.gz: e15204813d179e130dea804b89dc0a5225e1410af19fc5420ed2e82a3992d19b7a6e9e2054dce0269dce957f1ebee42e6c881f301e8554643f0eff1167a6fd61

data/README.md

@@ -76,6 +76,7 @@ The pipeline includes 15 steps total:
 aidp status                   # Show progress of all steps
 aidp execute next             # Run next pending step
 aidp approve current          # Approve current gate step
+aidp jobs                     # Monitor background jobs (real-time)
 aidp detect                   # See which AI provider will be used
 aidp execute <step>           # Run specific step (e.g., prd, arch, tasks)
 aidp approve <step>           # Approve specific step
@@ -97,6 +98,56 @@ AIDP_PROVIDER=anthropic aidp execute next
 AIDP_LLM_CMD=/usr/local/bin/claude aidp execute next
 ```
 
+## Background Jobs
+
+AIDP uses background jobs to handle all AI provider executions, providing better reliability and real-time monitoring capabilities.
+
+### Job Monitoring
+
+Monitor running and completed jobs in real-time:
+
+```bash
+aidp jobs                     # Show job status with real-time updates
+```
+
+The jobs view displays:
+
+- **Running jobs** with live progress updates
+- **Queued jobs** waiting to be processed
+- **Completed jobs** with execution results
+- **Failed jobs** with error details
+
+### Job Controls
+
+From the jobs view, you can:
+
+- **Retry failed jobs** by pressing `r` on a failed job
+- **View job details** by pressing `d` on any job
+- **Exit monitoring** by pressing `q`
+
+### Job Persistence
+
+- Jobs persist across CLI restarts
+- Job history is preserved for analysis
+- Failed jobs can be retried at any time
+- All job metadata and logs are stored
+
+### Database Setup
+
+AIDP uses PostgreSQL for job management. Ensure PostgreSQL is installed and running:
+
+```bash
+# macOS (using Homebrew)
+brew install postgresql
+brew services start postgresql
+
+# Ubuntu/Debian
+sudo apt-get install postgresql postgresql-contrib
+sudo systemctl start postgresql
+
+# The database will be created automatically on first use
+```
+
 ## File-Based Interaction
 
 At gate steps, the AI creates files for interaction instead of requiring real-time chat:
@@ -137,20 +188,24 @@ Here's a typical session:
 aidp execute next
 # → Creates docs/PRD.md and PRD_QUESTIONS.md
 
-# 2. Review the questions (if any)
+# 2. Monitor job progress (optional)
+aidp jobs
+# → Shows real-time job status and progress
+
+# 3. Review the questions (if any)
 cat PRD_QUESTIONS.md
 # → If questions exist, edit the file with your answers, then re-run
 
-# 3. Review the PRD
+# 4. Review the PRD
 cat docs/PRD.md
 # → Edit if needed
 
-# 4. Approve and continue
+# 5. Approve and continue
 aidp approve current
 aidp execute next
 # → Creates docs/NFRs.md automatically
 
-# 5. Continue through gates
+# 6. Continue through gates
 aidp execute next
 # → Creates docs/Architecture.md and ARCH_QUESTIONS.md
 # → Repeat review/approve cycle

data/lib/aidp/analyze/agent_personas.rb

@@ -9,7 +9,7 @@ module Aidp
           "name" => "Repository Analyst",
           "description" => "Expert in analyzing version control data, code evolution patterns, and repository metrics. Specializes in identifying hotspots, technical debt, and code quality trends over time.",
           "expertise" => ["Git analysis", "Code metrics", "Temporal patterns", "Hotspot identification"],
-          "tools" => ["Code Maat", "Git log analysis", "Statistical analysis"]
+          "tools" => ["ruby-maat", "Git log analysis", "Statistical analysis"]
         },
         "Architecture Analyst" => {
           "name" => "Architecture Analyst",

data/lib/aidp/analyze/database.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require "sqlite3"
+require "pg"
 require "json"
 require "fileutils"
 
@@ -8,7 +8,6 @@ module Aidp
   class AnalysisDatabase
     def initialize(project_dir = Dir.pwd)
       @project_dir = project_dir
-      @db_path = File.join(project_dir, ".aidp-analysis.db")
       ensure_database_exists
     end
 
@@ -17,15 +16,21 @@ module Aidp
       db = connect
 
       # Store the main analysis result
-      db.execute(
-        "INSERT OR REPLACE INTO analysis_results (step_name, data, metadata, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
-        [step_name, data.to_json, metadata.to_json, Time.now.iso8601, Time.now.iso8601]
+      db.exec_params(
+        <<~SQL,
+          INSERT INTO analysis_results (step_name, data, metadata, created_at, updated_at)
+          VALUES ($1, $2, $3, $4, $5)
+          ON CONFLICT (step_name)
+          DO UPDATE SET
+            data = EXCLUDED.data,
+            metadata = EXCLUDED.metadata,
+            updated_at = EXCLUDED.updated_at
+        SQL
+        [step_name, data.to_json, metadata.to_json, Time.now, Time.now]
       )
 
       # Store metrics for indefinite retention
       store_metrics(step_name, metadata[:metrics]) if metadata[:metrics]
-
-      db.close
     end
 
     # Store metrics that should be retained indefinitely
@@ -33,57 +38,66 @@ module Aidp
       db = connect
 
       metrics.each do |metric_name, value|
-        db.execute(
-          "INSERT OR REPLACE INTO analysis_metrics (step_name, metric_name, value, recorded_at) VALUES (?, ?, ?, ?)",
-          [step_name, metric_name.to_s, value.to_json, Time.now.iso8601]
+        db.exec_params(
+          <<~SQL,
+            INSERT INTO analysis_metrics (step_name, metric_name, value, recorded_at)
+            VALUES ($1, $2, $3, $4)
+            ON CONFLICT (step_name, metric_name, recorded_at)
+            DO UPDATE SET value = EXCLUDED.value
+          SQL
+          [step_name, metric_name.to_s, value.to_json, Time.now]
         )
       end
-
-      db.close
     end
 
     # Store embedding vectors for future semantic analysis
     def store_embeddings(step_name, embeddings_data)
       db = connect
 
-      db.execute(
-        "INSERT OR REPLACE INTO embeddings (step_name, embeddings_data, created_at) VALUES (?, ?, ?)",
-        [step_name, embeddings_data.to_json, Time.now.iso8601]
+      db.exec_params(
+        <<~SQL,
+          INSERT INTO embeddings (step_name, embeddings_data, created_at)
+          VALUES ($1, $2, $3)
+          ON CONFLICT (step_name)
+          DO UPDATE SET
+            embeddings_data = EXCLUDED.embeddings_data,
+            created_at = EXCLUDED.created_at
+        SQL
+        [step_name, embeddings_data.to_json, Time.now]
       )
-
-      db.close
     end
 
     # Retrieve analysis results
     def get_analysis_result(step_name)
       db = connect
-      result = db.execute("SELECT data, metadata, created_at, updated_at FROM analysis_results WHERE step_name = ?",
-        [step_name]).first
-      db.close
+      result = db.exec_params(
+        "SELECT data, metadata, created_at, updated_at FROM analysis_results WHERE step_name = $1",
+        [step_name]
+      ).first
 
       return nil unless result
 
       {
-        data: JSON.parse(result[0]),
-        metadata: JSON.parse(result[1]),
-        created_at: result[2],
-        updated_at: result[3]
+        data: JSON.parse(result["data"]),
+        metadata: JSON.parse(result["metadata"]),
+        created_at: result["created_at"],
+        updated_at: result["updated_at"]
       }
     end
 
     # Retrieve metrics for a step
     def get_metrics(step_name)
       db = connect
-      results = db.execute(
-        "SELECT metric_name, value, recorded_at FROM analysis_metrics WHERE step_name = ? ORDER BY recorded_at DESC", [step_name]
+      results = db.exec_params(
+        "SELECT metric_name, value, recorded_at FROM analysis_metrics WHERE step_name = $1 ORDER BY recorded_at DESC",
+        [step_name]
       )
-      db.close
 
       results.map do |row|
         {
-          metric_name: row[0],
-          value: JSON.parse(row[1]),
-          recorded_at: row[2]
+          metric_name: row["metric_name"],
+          value: JSON.parse(row["value"]),
+          recorded_at: row["recorded_at"]
         }
       end
     end
@@ -91,15 +105,14 @@ module Aidp
     # Get all metrics for trend analysis
     def get_all_metrics
       db = connect
-      results = db.execute("SELECT step_name, metric_name, value, recorded_at FROM analysis_metrics ORDER BY recorded_at DESC")
-      db.close
+      results = db.exec("SELECT step_name, metric_name, value, recorded_at FROM analysis_metrics ORDER BY recorded_at DESC")
 
       results.map do |row|
         {
-          step_name: row[0],
-          metric_name: row[1],
-          value: JSON.parse(row[2]),
-          recorded_at: row[2]
+          step_name: row["step_name"],
+          metric_name: row["metric_name"],
+          value: JSON.parse(row["value"]),
+          recorded_at: row["recorded_at"]
         }
       end
     end
@@ -109,28 +122,28 @@ module Aidp
       db = connect
 
       # Delete existing data
-      db.execute("DELETE FROM analysis_results WHERE step_name = ?", [step_name])
-      db.execute("DELETE FROM embeddings WHERE step_name = ?", [step_name])
+      db.exec_params("DELETE FROM analysis_results WHERE step_name = $1", [step_name])
+      db.exec_params("DELETE FROM embeddings WHERE step_name = $1", [step_name])
 
       # Store new data
-      db.execute(
-        "INSERT INTO analysis_results (step_name, data, metadata, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
-        [step_name, data.to_json, metadata.to_json, Time.now.iso8601, Time.now.iso8601]
+      db.exec_params(
+        <<~SQL,
+          INSERT INTO analysis_results (step_name, data, metadata, created_at, updated_at)
+          VALUES ($1, $2, $3, $4, $5)
+        SQL
+        [step_name, data.to_json, metadata.to_json, Time.now, Time.now]
       )
 
       # Store metrics (these are retained indefinitely)
       store_metrics(step_name, metadata[:metrics]) if metadata[:metrics]
-
-      db.close
     end
 
     # Delete analysis data (for user cleanup)
     def delete_analysis_data(step_name)
       db = connect
-      db.execute("DELETE FROM analysis_results WHERE step_name = ?", [step_name])
-      db.execute("DELETE FROM embeddings WHERE step_name = ?", [step_name])
+      db.exec_params("DELETE FROM analysis_results WHERE step_name = $1", [step_name])
+      db.exec_params("DELETE FROM embeddings WHERE step_name = $1", [step_name])
       # NOTE: metrics are NOT deleted as they should be retained indefinitely
-      db.close
     end
 
     # Export data in different formats
@@ -152,68 +165,72 @@ module Aidp
     def get_statistics
       db = connect
 
-      stats = {
-        total_analysis_results: db.execute("SELECT COUNT(*) FROM analysis_results").first[0],
-        total_metrics: db.execute("SELECT COUNT(*) FROM analysis_metrics").first[0],
-        total_embeddings: db.execute("SELECT COUNT(*) FROM embeddings").first[0],
-        steps_analyzed: db.execute("SELECT DISTINCT step_name FROM analysis_results").map { |row| row[0] },
-        oldest_metric: db.execute("SELECT MIN(recorded_at) FROM analysis_metrics").first[0],
-        newest_metric: db.execute("SELECT MAX(recorded_at) FROM analysis_metrics").first[0]
+      {
+        total_analysis_results: db.exec("SELECT COUNT(*) FROM analysis_results").first["count"].to_i,
+        total_metrics: db.exec("SELECT COUNT(*) FROM analysis_metrics").first["count"].to_i,
+        total_embeddings: db.exec("SELECT COUNT(*) FROM embeddings").first["count"].to_i,
+        steps_analyzed: db.exec("SELECT DISTINCT step_name FROM analysis_results").map { |row| row["step_name"] },
+        oldest_metric: db.exec("SELECT MIN(recorded_at) FROM analysis_metrics").first["min"],
+        newest_metric: db.exec("SELECT MAX(recorded_at) FROM analysis_metrics").first["max"]
       }
-
-      db.close
-      stats
     end
 
     private
 
     def ensure_database_exists
-      return if File.exist?(@db_path)
+      db = connect
+      create_schema(db)
+    end
 
-      db = SQLite3::Database.new(@db_path)
+    def connect
+      @db ||= PG.connect(
+        host: ENV["AIDP_DB_HOST"] || "localhost",
+        port: ENV["AIDP_DB_PORT"] || 5432,
+        dbname: ENV["AIDP_DB_NAME"] || "aidp",
+        user: ENV["AIDP_DB_USER"] || ENV["USER"],
+        password: ENV["AIDP_DB_PASSWORD"]
+      )
+      @db.type_map_for_results = PG::BasicTypeMapForResults.new(@db)
+      @db
+    end
 
+    def create_schema(db)
       # Create analysis_results table
-      db.execute(<<~SQL)
-        CREATE TABLE analysis_results (
+      db.exec(<<~SQL)
+        CREATE TABLE IF NOT EXISTS analysis_results (
           step_name TEXT PRIMARY KEY,
-          data TEXT NOT NULL,
-          metadata TEXT,
-          created_at TEXT NOT NULL,
-          updated_at TEXT NOT NULL
+          data JSONB NOT NULL,
+          metadata JSONB,
+          created_at TIMESTAMP WITH TIME ZONE NOT NULL,
+          updated_at TIMESTAMP WITH TIME ZONE NOT NULL
         )
       SQL
 
       # Create analysis_metrics table (indefinite retention)
-      db.execute(<<~SQL)
-        CREATE TABLE analysis_metrics (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
+      db.exec(<<~SQL)
+        CREATE TABLE IF NOT EXISTS analysis_metrics (
+          id SERIAL PRIMARY KEY,
           step_name TEXT NOT NULL,
           metric_name TEXT NOT NULL,
-          value TEXT NOT NULL,
-          recorded_at TEXT NOT NULL,
+          value JSONB NOT NULL,
+          recorded_at TIMESTAMP WITH TIME ZONE NOT NULL,
           UNIQUE(step_name, metric_name, recorded_at)
         )
       SQL
 
       # Create embeddings table (for future semantic analysis)
-      db.execute(<<~SQL)
-        CREATE TABLE embeddings (
+      db.exec(<<~SQL)
+        CREATE TABLE IF NOT EXISTS embeddings (
           step_name TEXT PRIMARY KEY,
-          embeddings_data TEXT NOT NULL,
-          created_at TEXT NOT NULL
+          embeddings_data JSONB NOT NULL,
+          created_at TIMESTAMP WITH TIME ZONE NOT NULL
         )
       SQL
 
       # Create indexes for better performance
-      db.execute("CREATE INDEX idx_analysis_metrics_step_name ON analysis_metrics(step_name)")
-      db.execute("CREATE INDEX idx_analysis_metrics_recorded_at ON analysis_metrics(recorded_at)")
-      db.execute("CREATE INDEX idx_analysis_results_updated_at ON analysis_results(updated_at)")
-
-      db.close
-    end
-
-    def connect
-      SQLite3::Database.new(@db_path)
+      db.exec("CREATE INDEX IF NOT EXISTS idx_analysis_metrics_step_name ON analysis_metrics(step_name)")
+      db.exec("CREATE INDEX IF NOT EXISTS idx_analysis_metrics_recorded_at ON analysis_metrics(recorded_at)")
+      db.exec("CREATE INDEX IF NOT EXISTS idx_analysis_results_updated_at ON analysis_results(updated_at)")
     end
 
     def export_to_csv(data)

data/lib/aidp/analyze/error_handler.rb

@@ -105,13 +105,6 @@ module Aidp
         end
       end
 
-      def fallback_to_mock_data(operation, fallback_data)
-        operation.call
-      rescue => e
-        logger.warn("Operation failed, using fallback data: #{e.message}")
-        fallback_data
-      end
-
       def skip_step_with_warning(step_name, error)
         logger.warn("Skipping step '#{step_name}' due to error: #{error.message}")
         {
@@ -190,7 +183,7 @@ module Aidp
           SQLite3::CorruptException => :critical_error,
           AnalysisTimeoutError => :chunk_and_retry,
           AnalysisDataError => :continue_with_partial_data,
-          AnalysisToolError => :fallback_to_mock_data
+          AnalysisToolError => :log_and_continue
         }
       end
 
@@ -236,8 +229,6 @@ module Aidp
           chunk_and_retry(error_info)
         when :continue_with_partial_data
           continue_with_partial(error_info)
-        when :fallback_to_mock_data
-          fallback_to_mock(error_info)
         when :log_and_continue
           log_and_continue(error_info)
         else
@@ -273,7 +264,8 @@ module Aidp
         if context[:network_required]
           raise_critical_error({error: error, context: context})
         else
-          fallback_to_mock_data(-> { context[:operation].call }, context[:fallback_data])
+          logger.error("Network connection error: #{error.message}")
+          raise error
         end
       end
 
@@ -329,11 +321,15 @@ module Aidp
       end
 
       def handle_analysis_tool_error(error, context)
-        logger.warn("Analysis tool error: #{error.message}")
-        fallback_to_mock_data(
-          -> { context[:operation].call },
-          context[:mock_data] || generate_mock_data(context)
-        )
+        logger.error("Analysis tool error: #{error.message}")
+        tool_name = context[:tool_name] || "analysis tool"
+        error_msg = "#{tool_name} failed: #{error.message}"
+
+        if context[:installation_guide]
+          error_msg += "\n\nTo install #{tool_name}:\n#{context[:installation_guide]}"
+        end
+
+        raise AnalysisToolError.new(error_msg)
       end
 
       # Recovery strategy implementations
@@ -390,72 +386,12 @@ module Aidp
         continue_with_partial_data(operation, partial_handler)
       end
 
-      def fallback_to_mock(error_info)
-        context = error_info[:context]
-        operation = context[:operation]
-        mock_data = context[:mock_data] || generate_mock_data(context)
-
-        fallback_to_mock_data(operation, mock_data)
-      end
-
       def log_and_continue(error_info)
         error = error_info[:error]
         logger.warn("Continuing after error: #{error.message}")
         {status: "continued_with_error", error: error.message}
       end
 
-      def generate_mock_data(context)
-        case context[:analysis_type]
-        when "repository"
-          generate_mock_repository_data
-        when "architecture"
-          generate_mock_architecture_data
-        when "test_coverage"
-          generate_mock_test_data
-        else
-          {status: "mock_data", message: "Mock data generated due to error"}
-        end
-      end
-
-      def generate_mock_repository_data
-        {
-          analysis_type: "repository",
-          status: "completed",
-          data: [
-            {entity: "mock_file.rb", nrev: 5, nloc: 100, churn: 20}
-          ],
-          statistics: {
-            total_files: 1,
-            total_commits: 5,
-            total_lines: 100
-          }
-        }
-      end
-
-      def generate_mock_architecture_data
-        {
-          analysis_type: "architecture",
-          status: "completed",
-          data: {
-            pattern: "monolithic",
-            components: ["mock_component"],
-            dependencies: []
-          }
-        }
-      end
-
-      def generate_mock_test_data
-        {
-          analysis_type: "test_coverage",
-          status: "completed",
-          data: {
-            coverage: 75.0,
-            tests: 10,
-            files: 5
-          }
-        }
-      end
-
       def calculate_recovery_success_rate
         return 0.0 if @error_history.empty?
 

data/lib/aidp/analyze/focus_guidance.rb

@@ -53,7 +53,7 @@ module Aidp
         generate_focused_plan(selected_areas, recommendations)
       end
 
-      # Get focus areas based on Code Maat analysis
+      # Get focus areas based on ruby-maat analysis
       def get_code_maat_focus_areas
         @code_maat.run_comprehensive_analysis
 
@@ -490,7 +490,7 @@ module Aidp
 
           ### Phase 1: Baseline Analysis
           - Repository analysis to establish current state
-          - Code Maat analysis for historical patterns
+          - Ruby-maat analysis for historical patterns
           - Feature analysis for current structure
 
           ### Phase 2: Focused Analysis