aidp 0.47.1 → 0.48.0

data/lib/aidp/database.rb

@@ -3,6 +3,7 @@
 require "sqlite3"
 require "json"
 require "fileutils"
+Kernel.require("set") unless defined?(Set)
 
 module Aidp
   # Database module for SQLite-based storage
@@ -15,6 +16,8 @@ module Aidp
     # Thread-safe connection cache
     @connections = {}
     @mutex = Mutex.new
+    @migrated_projects = Set.new
+    @migration_mutex = Mutex.new
 
     class << self
       # Get or create a database connection for the given project directory
@@ -26,6 +29,8 @@ module Aidp
         db_path = ConfigPaths.database_file(project_dir)
 
         @mutex.synchronize do
+          invalidate_missing_connection!(db_path)
+
           # Return cached connection if valid
           if @connections[db_path]&.closed? == false
             return @connections[db_path]
@@ -53,6 +58,23 @@ module Aidp
         Migrations.run!(project_dir)
       end
 
+      # Run pending migrations at most once per process for each project directory.
+      #
+      # @param project_dir [String] Project directory path
+      # @return [Array<Integer>] List of applied migration versions
+      def migrate_once!(project_dir = Dir.pwd)
+        expanded_project_dir = File.expand_path(project_dir)
+        require_relative "database/migrations"
+
+        @migration_mutex.synchronize do
+          return [] if migration_current?(expanded_project_dir)
+
+          migrate!(expanded_project_dir).tap do
+            @migrated_projects << expanded_project_dir
+          end
+        end
+      end
+
       # Check if database exists and is initialized
       #
       # @param project_dir [String] Project directory path
@@ -134,6 +156,22 @@ module Aidp
         # Set busy timeout to 5 seconds
         db.busy_timeout = 5000
       end
+
+      def invalidate_missing_connection!(db_path)
+        db = @connections[db_path]
+        return unless db
+        return if db.closed?
+        return if File.exist?(db_path)
+
+        db.close
+        @connections.delete(db_path)
+      end
+
+      def migration_current?(project_dir)
+        @migrated_projects.include?(project_dir) &&
+          exists?(project_dir) &&
+          !Migrations.pending?(project_dir)
+      end
     end
   end
 end

data/lib/aidp/strategy_execution/cli_protocol.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require "shellwords"
+require "tmpdir"
+require "fileutils"
+require "pathname"
+
+module Aidp
+  module StrategyExecution
+    module CliProtocol
+      class ProtocolError < StandardError; end
+
+      class Runner
+        PROTOCOL_VERSION = "1.0"
+
+        def initialize(project_dir: Dir.pwd)
+          @project_dir = project_dir
+        end
+
+        def execute(command:, role:, request:)
+          parts = command_parts(command)
+          raise ProtocolError, "command must not be blank" if parts.empty?
+
+          artifact_dir = create_artifact_dir(role)
+          payload = request.merge(
+            protocol_version: PROTOCOL_VERSION,
+            role: role,
+            artifact_dir: artifact_dir
+          )
+
+          Aidp.log_debug("cli_protocol", "executing",
+            role: role,
+            command: parts.first,
+            task_id: request[:task]&.dig(:id),
+            artifact_dir: artifact_dir)
+
+          stdout, stderr, status = Open3.capture3(*parts, stdin_data: JSON.generate(payload), chdir: @project_dir)
+          unless status.success?
+            Aidp.log_error("cli_protocol", "execution_failed",
+              role: role,
+              command: parts.first,
+              exit_status: status.exitstatus,
+              stderr: stderr.to_s.strip)
+            raise ProtocolError, stderr.strip
+          end
+
+          response = JSON.parse(stdout, symbolize_names: true)
+          validate_response!(response, role)
+
+          response.merge(
+            artifacts: normalize_artifacts(response[:artifacts], artifact_dir),
+            artifact_dir: artifact_dir,
+            stderr: stderr
+          )
+        rescue JSON::ParserError => e
+          Aidp.log_error("cli_protocol", "invalid_response",
+            role: role,
+            error: e.message)
+          raise ProtocolError, "Invalid JSON response: #{e.message}"
+        rescue ProtocolError
+          raise
+        rescue Errno::ENOENT, Errno::EACCES => e
+          Aidp.log_error("cli_protocol", "execution_failed",
+            role: role,
+            command: parts.first,
+            error: e.message,
+            error_class: e.class.name)
+          raise ProtocolError, "Command failed: #{e.message}"
+        end
+
+        private
+
+        def command_parts(command)
+          case command
+          when Array then command
+          else Shellwords.split(command.to_s)
+          end
+        end
+
+        def create_artifact_dir(role)
+          base_dir = File.join(@project_dir, ".aidp")
+          FileUtils.mkdir_p(base_dir)
+          dir = Dir.mktmpdir("aidp-#{role}-", base_dir)
+          FileUtils.mkdir_p(dir)
+          dir
+        end
+
+        def normalize_artifacts(artifacts, artifact_dir)
+          expanded_artifact_dir = File.expand_path(artifact_dir)
+
+          Array(artifacts).map do |artifact|
+            next artifact unless artifact.is_a?(String)
+
+            expanded_path = File.expand_path(artifact, expanded_artifact_dir)
+            validate_artifact_path!(artifact, expanded_path, expanded_artifact_dir)
+            expanded_path
+          end
+        end
+
+        def validate_artifact_path!(artifact, expanded_path, artifact_dir)
+          return if expanded_path == artifact_dir
+          return if expanded_path.start_with?("#{artifact_dir}#{File::SEPARATOR}")
+
+          raise ProtocolError, "artifact path escapes artifact directory: #{artifact}"
+        end
+
+        def validate_response!(response, role)
+          raise ProtocolError, "response must include success" unless response.key?(:success)
+          return unless role == "evaluator"
+          return if response.key?(:score) || response.key?(:passed)
+
+          raise ProtocolError, "evaluator response must include score or passed"
+        end
+      end
+    end
+  end
+end

data/lib/aidp/strategy_execution/experience_store.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../database"
+require_relative "../database/repositories/strategy_repository"
+require_relative "../database/repositories/experience_task_repository"
+require_relative "../database/repositories/experience_run_repository"
+require_relative "../database/repositories/experience_evaluation_repository"
+require_relative "../database/repositories/experience_artifact_repository"
+
+module Aidp
+  module StrategyExecution
+    class ExperienceStore
+      def initialize(project_dir: Dir.pwd)
+        @project_dir = project_dir
+        Aidp::Database.migrate_once!(@project_dir)
+      end
+
+      def register_strategy(strategy_spec)
+        record = strategy_repository.register(
+          name: strategy_spec.name,
+          spec: JSON.generate(strategy_spec.to_h)
+        )
+        Aidp.log_debug("experience_store", "strategy_registered",
+          strategy_id: record[:id], name: strategy_spec.name)
+        record
+      end
+
+      def create_task(description:, title: nil, input_payload: {}, context: {}, source_run_id: nil)
+        task = task_repository.create(
+          description: description,
+          title: title,
+          input_payload: input_payload,
+          context: context,
+          source_run_id: source_run_id
+        )
+        Aidp.log_debug("experience_store", "task_created",
+          task_id: task[:id], source_run_id: source_run_id)
+        task
+      end
+
+      def start_run(task_id:, strategy_id:, workflow_id:, depth:, input_payload:, parent_run_id: nil, branch_key: nil, metadata: {})
+        run = run_repository.start(
+          task_id: task_id,
+          strategy_id: strategy_id,
+          workflow_id: workflow_id,
+          depth: depth,
+          input_payload: input_payload,
+          parent_run_id: parent_run_id,
+          branch_key: branch_key,
+          metadata: metadata
+        )
+        Aidp.log_debug("experience_store", "run_started",
+          run_id: run[:id],
+          task_id: task_id,
+          strategy_id: strategy_id,
+          branch_key: branch_key,
+          depth: depth)
+        run
+      end
+
+      def complete_run(run_id:, status:, output_payload:, metadata: {})
+        run = run_repository.complete(
+          run_id: run_id,
+          status: status,
+          output_payload: output_payload,
+          metadata: metadata
+        )
+        Aidp.log_debug("experience_store", "run_completed",
+          run_id: run_id, status: status)
+        run
+      end
+
+      def record_evaluation(run_id:, evaluator_name:, score:, passed:, summary: nil, metadata: {})
+        evaluation_repository.record(
+          run_id: run_id,
+          evaluator_name: evaluator_name,
+          score: score,
+          passed: passed,
+          summary: summary,
+          metadata: metadata
+        )
+        Aidp.log_debug("experience_store", "evaluation_recorded",
+          run_id: run_id,
+          evaluator_name: evaluator_name,
+          score: score,
+          passed: passed)
+      end
+
+      def record_artifact(run_id:, role:, path:, metadata: {})
+        artifact_repository.record(
+          run_id: run_id,
+          role: role,
+          path: path,
+          metadata: metadata
+        )
+        Aidp.log_debug("experience_store", "artifact_recorded",
+          run_id: run_id, role: role, path: path)
+      end
+
+      def replay_bundle(run_id)
+        bundle = run_repository.bundle_for_replay(run_id)
+        Aidp.log_debug("experience_store", "replay_bundle_loaded",
+          run_id: run_id, found: !bundle.nil?)
+        bundle
+      end
+
+      def task_details(task_id)
+        task = task_repository.load(task_id)
+        Aidp.log_debug("experience_store", "task_details_loaded",
+          task_id: task_id, found: !task.nil?)
+        task
+      end
+
+      def run_details(run_id)
+        run = run_repository.load(run_id)
+        return nil unless run
+
+        details = run.merge(
+          evaluations: evaluation_repository.list_for_run(run_id),
+          artifacts: artifact_repository.list_for_run(run_id),
+          children: run_repository.children(run_id)
+        )
+        Aidp.log_debug("experience_store", "run_details_loaded",
+          run_id: run_id,
+          evaluation_count: details[:evaluations].length,
+          artifact_count: details[:artifacts].length,
+          child_count: details[:children].length)
+        details
+      end
+
+      private
+
+      def strategy_repository
+        @strategy_repository ||= Aidp::Database::Repositories::StrategyRepository.new(project_dir: @project_dir)
+      end
+
+      def task_repository
+        @task_repository ||= Aidp::Database::Repositories::ExperienceTaskRepository.new(project_dir: @project_dir)
+      end
+
+      def run_repository
+        @run_repository ||= Aidp::Database::Repositories::ExperienceRunRepository.new(project_dir: @project_dir)
+      end
+
+      def evaluation_repository
+        @evaluation_repository ||= Aidp::Database::Repositories::ExperienceEvaluationRepository.new(project_dir: @project_dir)
+      end
+
+      def artifact_repository
+        @artifact_repository ||= Aidp::Database::Repositories::ExperienceArtifactRepository.new(project_dir: @project_dir)
+      end
+    end
+  end
+end

data/lib/aidp/strategy_execution/strategy_loader.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "date"
+require "yaml"
+require_relative "strategy_spec"
+
+module Aidp
+  module StrategyExecution
+    class StrategyLoader
+      def initialize(project_dir: Dir.pwd)
+        @project_dir = project_dir
+      end
+
+      def load_file(path)
+        raw = YAML.safe_load_file(expand_path(path), permitted_classes: [Date, Time, Symbol], aliases: true) || {}
+        StrategySpec.from_hash(raw)
+      end
+
+      private
+
+      def expand_path(path)
+        File.expand_path(path, @project_dir)
+      end
+    end
+  end
+end

data/lib/aidp/strategy_execution/strategy_spec.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Aidp
+  module StrategyExecution
+    class StrategySpec
+      DEFAULT_MAX_DEPTH = 1
+      DEFAULT_FANOUT = 1
+      DEFAULT_MERGE_POLICY = "highest_score"
+
+      attr_reader :name, :max_depth, :fanout, :agents, :evaluators, :merge_policy, :constraints, :timeouts
+
+      def self.from_hash(data)
+        new(data.transform_keys(&:to_sym))
+      end
+
+      def initialize(data)
+        @name = fetch_name(data)
+        @max_depth = positive_integer(data[:max_depth], DEFAULT_MAX_DEPTH)
+        @fanout = positive_integer(data[:fanout], DEFAULT_FANOUT)
+        @agents = normalize_agents(data[:agents] || {})
+        @evaluators = normalize_evaluators(data[:evaluators] || [])
+        @merge_policy = (data[:merge_policy] || DEFAULT_MERGE_POLICY).to_s
+        @constraints = (data[:constraints] || {}).transform_keys(&:to_sym)
+        @timeouts = (data[:timeouts] || {}).transform_keys(&:to_sym)
+        ensure_runnable!
+      end
+
+      def to_h
+        {
+          name: name,
+          max_depth: max_depth,
+          fanout: fanout,
+          agents: agents,
+          evaluators: evaluators,
+          merge_policy: merge_policy,
+          constraints: constraints,
+          timeouts: timeouts
+        }
+      end
+
+      def branch_commands
+        commands = candidate_agent_commands
+        Array.new(fanout) do |index|
+          command = commands[index % commands.length]
+          {
+            key: "branch_#{index}",
+            name: command[:name],
+            command: command[:command]
+          }
+        end
+      end
+
+      def evaluator_definitions
+        evaluators
+      end
+
+      private
+
+      def fetch_name(data)
+        value = data[:name].to_s.strip
+        raise ArgumentError, "strategy name is required" if value.empty?
+
+        value
+      end
+
+      def positive_integer(value, fallback)
+        number = value.to_i
+        number.positive? ? number : fallback
+      end
+
+      def normalize_agents(agents)
+        agents.each_with_object({}) do |(role, value), normalized|
+          normalized[role.to_sym] = case value
+          when Array
+            value.map { |item| normalize_agent_entry(role, item) }
+          else
+            normalize_agent_entry(role, value)
+          end
+        end
+      end
+
+      def normalize_agent_entry(role, value)
+        case value
+        when Hash
+          {
+            name: (value[:name] || value["name"] || role).to_s,
+            command: (value[:command] || value["command"]).to_s
+          }
+        else
+          {
+            name: role.to_s,
+            command: value.to_s
+          }
+        end
+      end
+
+      def normalize_evaluators(evaluators)
+        Array(evaluators).map do |entry|
+          case entry
+          when Hash
+            {
+              name: (entry[:name] || entry["name"]).to_s,
+              command: (entry[:command] || entry["command"] || entry[:name] || entry["name"]).to_s
+            }
+          else
+            {
+              name: entry.to_s,
+              command: entry.to_s
+            }
+          end
+        end
+      end
+
+      def candidate_agent_commands
+        raw = agents[:coder] || agents[:execution] || agents.values
+        commands = raw.is_a?(Hash) ? [raw] : Array(raw).flatten
+        commands.reject { |entry| entry[:command].to_s.empty? }
+      end
+
+      def ensure_runnable!
+        return unless candidate_agent_commands.empty?
+
+        raise ArgumentError, "strategy '#{name}' requires at least one agent with a non-empty command"
+      end
+    end
+  end
+end

data/lib/aidp/temporal/activities/execute_cli_command_activity.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "base_activity"
+require_relative "../../strategy_execution/cli_protocol"
+
+module Aidp
+  module Temporal
+    module Activities
+      class ExecuteCliCommandActivity < BaseActivity
+        def execute(input)
+          with_activity_context do
+            project_dir = input[:project_dir]
+            command = input[:command]
+            role = input[:role]
+            request = input[:request] || {}
+
+            runner = Aidp::StrategyExecution::CliProtocol::Runner.new(project_dir: project_dir)
+
+            # The CLI invocation blocks for the full duration of the external
+            # agent/evaluator (often minutes). Heartbeat on a background thread so
+            # Temporal does not mark the activity timed out and retry the
+            # side-effecting command while the original subprocess is still running.
+            heartbeat_thread = start_heartbeat_thread(role: role)
+            begin
+              runner.execute(command: command, role: role, request: request)
+            ensure
+              heartbeat_thread&.kill
+            end
+          end
+        end
+
+        private
+
+        # Interval between heartbeats; kept well below the workflow's
+        # heartbeat_timeout so a missed beat does not time out the activity.
+        def heartbeat_interval_seconds
+          30
+        end
+
+        def start_heartbeat_thread(role:)
+          Thread.new do
+            loop do
+              sleep heartbeat_interval_seconds
+              heartbeat(role: role, status: "running")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/aidp/temporal/activities/manage_experience_store_activity.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "base_activity"
+require_relative "../../strategy_execution/experience_store"
+require_relative "../../strategy_execution/strategy_spec"
+
+module Aidp
+  module Temporal
+    module Activities
+      class ManageExperienceStoreActivity < BaseActivity
+        def execute(input)
+          with_activity_context do
+            project_dir = input[:project_dir]
+            operation = input[:operation]
+            payload = input[:payload] || {}
+            store = Aidp::StrategyExecution::ExperienceStore.new(project_dir: project_dir)
+
+            Aidp.log_debug("manage_experience_store_activity", "dispatching",
+              operation: operation,
+              run_id: payload[:run_id],
+              task_id: payload[:task_id],
+              strategy_id: payload[:strategy_id])
+
+            case operation
+            when "register_strategy"
+              strategy = Aidp::StrategyExecution::StrategySpec.from_hash(payload[:strategy])
+              store.register_strategy(strategy)
+            when "create_task"
+              store.create_task(**payload)
+            when "start_run"
+              store.start_run(**payload)
+            when "complete_run"
+              store.complete_run(**payload)
+            when "record_evaluation"
+              store.record_evaluation(**payload)
+              success_result
+            when "record_artifact"
+              store.record_artifact(**payload)
+              success_result
+            when "task_details"
+              store.task_details(payload[:task_id])
+            when "replay_bundle"
+              store.replay_bundle(payload[:run_id])
+            when "run_details"
+              store.run_details(payload[:run_id])
+            else
+              Aidp.log_error("manage_experience_store_activity", "unknown_operation",
+                operation: operation)
+              error_result("Unknown operation: #{operation}")
+            end
+          end
+        end
+      end
+    end
+  end
+end