aidp 0.1.0

data/lib/aidp/analyze/data_retention_manager.rb

@@ -0,0 +1,426 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+require_relative "storage"
+
+module Aidp
+  module Analyze
+    class DataRetentionManager
+      # Retention policies
+      RETENTION_POLICIES = {
+        "metrics" => "indefinite",
+        "analysis_results" => "configurable",
+        "execution_logs" => "configurable",
+        "temporary_data" => "immediate",
+        "embeddings" => "indefinite"
+      }.freeze
+
+      # Default retention periods (in days)
+      DEFAULT_RETENTION_PERIODS = {
+        "analysis_results" => 90, # 3 months
+        "execution_logs" => 30,   # 1 month
+        "temporary_data" => 1     # 1 day
+      }.freeze
+
+      def initialize(project_dir = Dir.pwd, config = {})
+        @project_dir = project_dir
+        @config = config
+        @storage = config[:storage] || Aidp::AnalysisStorage.new(project_dir)
+        @retention_config = load_retention_config
+      end
+
+      # Apply retention policies
+      def apply_retention_policies(options = {})
+        dry_run = options[:dry_run] || false
+        options[:force] || false
+
+        results = {
+          cleaned_data: {},
+          retained_data: {},
+          errors: []
+        }
+
+        # Process each data type
+        RETENTION_POLICIES.each do |data_type, policy|
+          case policy
+          when "indefinite"
+            results[:retained_data][data_type] = retain_indefinitely(data_type)
+          when "configurable"
+            results[:cleaned_data][data_type] = apply_configurable_retention(data_type, dry_run)
+          when "immediate"
+            results[:cleaned_data][data_type] = clean_immediately(data_type, dry_run)
+          end
+        rescue => e
+          results[:errors] << {
+            data_type: data_type,
+            error: e.message
+          }
+        end
+
+        results
+      end
+
+      # Handle force/rerun operations
+      def handle_force_rerun(execution_id, step_name, options = {})
+        operation = options[:operation] || "force"
+
+        case operation
+        when "force"
+          handle_force_operation(execution_id, step_name, options)
+        when "rerun"
+          handle_rerun_operation(execution_id, step_name, options)
+        else
+          raise "Unknown operation: #{operation}"
+        end
+      end
+
+      # Clean old data based on retention policies
+      def clean_old_data(options = {})
+        dry_run = options[:dry_run] || false
+        data_types = options[:data_types] || RETENTION_POLICIES.keys
+
+        results = {
+          cleaned: {},
+          errors: []
+        }
+
+        data_types.each do |data_type|
+          next unless RETENTION_POLICIES[data_type] == "configurable"
+
+          begin
+            cleaned = clean_data_by_type(data_type, dry_run)
+            results[:cleaned][data_type] = cleaned
+          rescue => e
+            results[:errors] << {
+              data_type: data_type,
+              error: e.message
+            }
+          end
+        end
+
+        results
+      end
+
+      # Get retention statistics
+      def get_retention_statistics
+        stats = {
+          policies: RETENTION_POLICIES,
+          config: @retention_config,
+          data_sizes: {},
+          retention_status: {}
+        }
+
+        # Calculate data sizes
+        RETENTION_POLICIES.keys.each do |data_type|
+          stats[:data_sizes][data_type] = calculate_data_size(data_type)
+          stats[:retention_status][data_type] = get_retention_status(data_type)
+        end
+
+        stats
+      end
+
+      # Export data with retention metadata
+      def export_data_with_retention(data_type, options = {})
+        include_retention_info = options[:include_retention_info] || true
+
+        data = export_data(data_type, options)
+
+        if include_retention_info
+          data[:retention_info] = {
+            policy: RETENTION_POLICIES[data_type],
+            retention_period: @retention_config[data_type],
+            last_cleaned: get_last_cleaned_date(data_type),
+            next_cleanup: calculate_next_cleanup(data_type)
+          }
+        end
+
+        data
+      end
+
+      # Set retention policy for a data type
+      def set_retention_policy(data_type, policy, options = {})
+        raise "Unknown data type: #{data_type}" unless RETENTION_POLICIES.key?(data_type)
+
+        case policy
+        when "indefinite"
+          @retention_config[data_type] = {policy: "indefinite"}
+        when "configurable"
+          retention_period = options[:retention_period] || DEFAULT_RETENTION_PERIODS[data_type]
+          @retention_config[data_type] = {
+            policy: "configurable",
+            retention_period: retention_period
+          }
+        when "immediate"
+          @retention_config[data_type] = {policy: "immediate"}
+        else
+          raise "Unknown retention policy: #{policy}"
+        end
+
+        save_retention_config
+
+        {
+          data_type: data_type,
+          policy: policy,
+          config: @retention_config[data_type]
+        }
+      end
+
+      # Get data retention status
+      def get_data_retention_status(data_type)
+        {
+          data_type: data_type,
+          policy: RETENTION_POLICIES[data_type],
+          config: @retention_config[data_type],
+          data_size: calculate_data_size(data_type),
+          last_cleaned: get_last_cleaned_date(data_type),
+          next_cleanup: calculate_next_cleanup(data_type),
+          retention_status: get_retention_status(data_type)
+        }
+      end
+
+      private
+
+      def load_retention_config
+        config_file = File.join(@project_dir, ".aidp-retention-config.yml")
+
+        if File.exist?(config_file)
+          YAML.load_file(config_file) || DEFAULT_RETENTION_PERIODS
+        else
+          DEFAULT_RETENTION_PERIODS
+        end
+      end
+
+      def save_retention_config
+        config_file = File.join(@project_dir, ".aidp-retention-config.yml")
+        File.write(config_file, YAML.dump(@retention_config))
+      end
+
+      def retain_indefinitely(data_type)
+        {
+          data_type: data_type,
+          action: "retained",
+          reason: "Indefinite retention policy",
+          timestamp: Time.now
+        }
+      end
+
+      def apply_configurable_retention(data_type, dry_run)
+        retention_period = @retention_config[data_type] || DEFAULT_RETENTION_PERIODS[data_type]
+        cutoff_date = Time.now - (retention_period * 24 * 60 * 60)
+
+        # Get old data
+        old_data = get_old_data(data_type, cutoff_date)
+
+        if dry_run
+          {
+            data_type: data_type,
+            action: "would_clean",
+            records_to_clean: old_data.length,
+            cutoff_date: cutoff_date,
+            dry_run: true
+          }
+        else
+          # Actually clean the data
+          cleaned_count = clean_data(data_type, old_data)
+
+          {
+            data_type: data_type,
+            action: "cleaned",
+            records_cleaned: cleaned_count,
+            cutoff_date: cutoff_date,
+            timestamp: Time.now
+          }
+        end
+      end
+
+      def clean_immediately(data_type, dry_run)
+        if dry_run
+          {
+            data_type: data_type,
+            action: "would_clean_immediately",
+            dry_run: true
+          }
+        else
+          # Clean all data of this type
+          cleaned_count = clean_all_data(data_type)
+
+          {
+            data_type: data_type,
+            action: "cleaned_immediately",
+            records_cleaned: cleaned_count,
+            timestamp: Time.now
+          }
+        end
+      end
+
+      def handle_force_operation(execution_id, step_name, options)
+        # Force operation: overwrite main data, retain metrics
+        results = {
+          operation: "force",
+          execution_id: execution_id,
+          step_name: step_name,
+          actions: []
+        }
+
+        # Overwrite analysis results
+        if options[:analysis_data]
+          @storage.force_overwrite(execution_id, step_name, options[:analysis_data])
+          results[:actions] << "overwrote_analysis_data"
+        end
+
+        # Retain metrics (indefinite retention)
+        results[:actions] << "retained_metrics"
+
+        results
+      end
+
+      def handle_rerun_operation(execution_id, step_name, options)
+        # Rerun operation: overwrite main data, retain metrics
+        results = {
+          operation: "rerun",
+          execution_id: execution_id,
+          step_name: step_name,
+          actions: []
+        }
+
+        # Overwrite analysis results
+        if options[:analysis_data]
+          @storage.force_overwrite(execution_id, step_name, options[:analysis_data])
+          results[:actions] << "overwrote_analysis_data"
+        end
+
+        # Retain metrics (indefinite retention)
+        results[:actions] << "retained_metrics"
+
+        results
+      end
+
+      def get_old_data(data_type, cutoff_date)
+        case data_type
+        when "analysis_results"
+          get_old_analysis_results(cutoff_date)
+        when "execution_logs"
+          get_old_execution_logs(cutoff_date)
+        when "temporary_data"
+          get_old_temporary_data(cutoff_date)
+        else
+          []
+        end
+      end
+
+      def get_old_analysis_results(cutoff_date)
+        # Get analysis results older than cutoff date
+        # This would query the database for old records
+        []
+      end
+
+      def get_old_execution_logs(cutoff_date)
+        # Get execution logs older than cutoff date
+        []
+      end
+
+      def get_old_temporary_data(cutoff_date)
+        # Get temporary data older than cutoff date
+        []
+      end
+
+      def clean_data(data_type, old_data)
+        case data_type
+        when "analysis_results"
+          clean_analysis_results(old_data)
+        when "execution_logs"
+          clean_execution_logs(old_data)
+        when "temporary_data"
+          clean_temporary_data(old_data)
+        else
+          0
+        end
+      end
+
+      def clean_all_data(data_type)
+        case data_type
+        when "temporary_data"
+          clean_all_temporary_data
+        else
+          0
+        end
+      end
+
+      def clean_analysis_results(old_data)
+        # Clean old analysis results from database
+        # This would delete records from the database
+        old_data.length
+      end
+
+      def clean_execution_logs(old_data)
+        # Clean old execution logs
+        old_data.length
+      end
+
+      def clean_temporary_data(old_data)
+        # Clean old temporary data
+        old_data.length
+      end
+
+      def clean_all_temporary_data
+        # Clean all temporary data
+        0
+      end
+
+      def calculate_data_size(data_type)
+        case data_type
+        when "metrics"
+          @storage.get_analysis_statistics[:total_metrics] || 0
+        when "analysis_results"
+          @storage.get_analysis_statistics[:total_steps] || 0
+        when "execution_logs"
+          @storage.get_execution_history.length
+        else
+          0
+        end
+      end
+
+      def get_retention_status(data_type)
+        policy = RETENTION_POLICIES[data_type]
+        config = @retention_config[data_type]
+
+        {
+          policy: policy,
+          config: config,
+          status: "active"
+        }
+      end
+
+      def get_last_cleaned_date(data_type)
+        # This would be stored in a metadata table
+        nil
+      end
+
+      def calculate_next_cleanup(data_type)
+        policy = RETENTION_POLICIES[data_type]
+
+        case policy
+        when "indefinite"
+          nil
+        when "configurable"
+          retention_period = @retention_config[data_type] || DEFAULT_RETENTION_PERIODS[data_type]
+          Time.now + (retention_period * 24 * 60 * 60)
+        when "immediate"
+          Time.now
+        end
+      end
+
+      def export_data(data_type, options)
+        case data_type
+        when "metrics"
+          @storage.export_data("json", options)
+        when "analysis_results"
+          @storage.export_data("json", options)
+        else
+          {error: "Unknown data type: #{data_type}"}
+        end
+      end
+    end
+  end
+end

data/lib/aidp/analyze/database.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+require "sqlite3"
+require "json"
+require "fileutils"
+
+module Aidp
+  class AnalysisDatabase
+    def initialize(project_dir = Dir.pwd)
+      @project_dir = project_dir
+      @db_path = File.join(project_dir, ".aidp-analysis.db")
+      ensure_database_exists
+    end
+
+    # Store analysis results with retention policies
+    def store_analysis_result(step_name, data, metadata = {})
+      db = connect
+
+      # Store the main analysis result
+      db.execute(
+        "INSERT OR REPLACE INTO analysis_results (step_name, data, metadata, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
+        [step_name, data.to_json, metadata.to_json, Time.now.iso8601, Time.now.iso8601]
+      )
+
+      # Store metrics for indefinite retention
+      store_metrics(step_name, metadata[:metrics]) if metadata[:metrics]
+
+      db.close
+    end
+
+    # Store metrics that should be retained indefinitely
+    def store_metrics(step_name, metrics)
+      db = connect
+
+      metrics.each do |metric_name, value|
+        db.execute(
+          "INSERT OR REPLACE INTO analysis_metrics (step_name, metric_name, value, recorded_at) VALUES (?, ?, ?, ?)",
+          [step_name, metric_name.to_s, value.to_json, Time.now.iso8601]
+        )
+      end
+
+      db.close
+    end
+
+    # Store embedding vectors for future semantic analysis
+    def store_embeddings(step_name, embeddings_data)
+      db = connect
+
+      db.execute(
+        "INSERT OR REPLACE INTO embeddings (step_name, embeddings_data, created_at) VALUES (?, ?, ?)",
+        [step_name, embeddings_data.to_json, Time.now.iso8601]
+      )
+
+      db.close
+    end
+
+    # Retrieve analysis results
+    def get_analysis_result(step_name)
+      db = connect
+      result = db.execute("SELECT data, metadata, created_at, updated_at FROM analysis_results WHERE step_name = ?",
+        [step_name]).first
+      db.close
+
+      return nil unless result
+
+      {
+        data: JSON.parse(result[0]),
+        metadata: JSON.parse(result[1]),
+        created_at: result[2],
+        updated_at: result[3]
+      }
+    end
+
+    # Retrieve metrics for a step
+    def get_metrics(step_name)
+      db = connect
+      results = db.execute(
+        "SELECT metric_name, value, recorded_at FROM analysis_metrics WHERE step_name = ? ORDER BY recorded_at DESC", [step_name]
+      )
+      db.close
+
+      results.map do |row|
+        {
+          metric_name: row[0],
+          value: JSON.parse(row[1]),
+          recorded_at: row[2]
+        }
+      end
+    end
+
+    # Get all metrics for trend analysis
+    def get_all_metrics
+      db = connect
+      results = db.execute("SELECT step_name, metric_name, value, recorded_at FROM analysis_metrics ORDER BY recorded_at DESC")
+      db.close
+
+      results.map do |row|
+        {
+          step_name: row[0],
+          metric_name: row[1],
+          value: JSON.parse(row[2]),
+          recorded_at: row[2]
+        }
+      end
+    end
+
+    # Force overwrite analysis data (for --force/--rerun flags)
+    def force_overwrite(step_name, data, metadata = {})
+      db = connect
+
+      # Delete existing data
+      db.execute("DELETE FROM analysis_results WHERE step_name = ?", [step_name])
+      db.execute("DELETE FROM embeddings WHERE step_name = ?", [step_name])
+
+      # Store new data
+      db.execute(
+        "INSERT INTO analysis_results (step_name, data, metadata, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
+        [step_name, data.to_json, metadata.to_json, Time.now.iso8601, Time.now.iso8601]
+      )
+
+      # Store metrics (these are retained indefinitely)
+      store_metrics(step_name, metadata[:metrics]) if metadata[:metrics]
+
+      db.close
+    end
+
+    # Delete analysis data (for user cleanup)
+    def delete_analysis_data(step_name)
+      db = connect
+      db.execute("DELETE FROM analysis_results WHERE step_name = ?", [step_name])
+      db.execute("DELETE FROM embeddings WHERE step_name = ?", [step_name])
+      # NOTE: metrics are NOT deleted as they should be retained indefinitely
+      db.close
+    end
+
+    # Export data in different formats
+    def export_data(step_name, format = "json")
+      result = get_analysis_result(step_name)
+      return nil unless result
+
+      case format.downcase
+      when "json"
+        result.to_json
+      when "csv"
+        export_to_csv(result)
+      else
+        raise "Unsupported export format: #{format}"
+      end
+    end
+
+    # Get database statistics
+    def get_statistics
+      db = connect
+
+      stats = {
+        total_analysis_results: db.execute("SELECT COUNT(*) FROM analysis_results").first[0],
+        total_metrics: db.execute("SELECT COUNT(*) FROM analysis_metrics").first[0],
+        total_embeddings: db.execute("SELECT COUNT(*) FROM embeddings").first[0],
+        steps_analyzed: db.execute("SELECT DISTINCT step_name FROM analysis_results").map { |row| row[0] },
+        oldest_metric: db.execute("SELECT MIN(recorded_at) FROM analysis_metrics").first[0],
+        newest_metric: db.execute("SELECT MAX(recorded_at) FROM analysis_metrics").first[0]
+      }
+
+      db.close
+      stats
+    end
+
+    private
+
+    def ensure_database_exists
+      return if File.exist?(@db_path)
+
+      db = SQLite3::Database.new(@db_path)
+
+      # Create analysis_results table
+      db.execute(<<~SQL)
+        CREATE TABLE analysis_results (
+          step_name TEXT PRIMARY KEY,
+          data TEXT NOT NULL,
+          metadata TEXT,
+          created_at TEXT NOT NULL,
+          updated_at TEXT NOT NULL
+        )
+      SQL
+
+      # Create analysis_metrics table (indefinite retention)
+      db.execute(<<~SQL)
+        CREATE TABLE analysis_metrics (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          step_name TEXT NOT NULL,
+          metric_name TEXT NOT NULL,
+          value TEXT NOT NULL,
+          recorded_at TEXT NOT NULL,
+          UNIQUE(step_name, metric_name, recorded_at)
+        )
+      SQL
+
+      # Create embeddings table (for future semantic analysis)
+      db.execute(<<~SQL)
+        CREATE TABLE embeddings (
+          step_name TEXT PRIMARY KEY,
+          embeddings_data TEXT NOT NULL,
+          created_at TEXT NOT NULL
+        )
+      SQL
+
+      # Create indexes for better performance
+      db.execute("CREATE INDEX idx_analysis_metrics_step_name ON analysis_metrics(step_name)")
+      db.execute("CREATE INDEX idx_analysis_metrics_recorded_at ON analysis_metrics(recorded_at)")
+      db.execute("CREATE INDEX idx_analysis_results_updated_at ON analysis_results(updated_at)")
+
+      db.close
+    end
+
+    def connect
+      SQLite3::Database.new(@db_path)
+    end
+
+    def export_to_csv(data)
+      require "csv"
+
+      # For now, export a simplified CSV format
+      # This can be enhanced based on specific data structures
+      CSV.generate do |csv|
+        csv << %w[Field Value]
+        csv << ["created_at", data[:created_at]]
+        csv << ["updated_at", data[:updated_at]]
+
+        # Add metadata fields
+        data[:metadata].each do |key, value|
+          csv << ["metadata_#{key}", value]
+        end
+
+        # Add data summary
+        if data[:data].is_a?(Hash)
+          data[:data].each do |key, value|
+            csv << ["data_#{key}", value.to_s[0..100]] # Truncate long values
+          end
+        end
+      end
+    end
+  end
+end