aidp 0.47.1 → 0.48.0

data/lib/aidp/temporal/workflows/strategy_branch_workflow.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+
+require "temporalio/workflow"
+require_relative "base_workflow"
+require_relative "../../strategy_execution/strategy_spec"
+require_relative "../../strategy_execution/cli_protocol"
+
+module Aidp
+  module Temporal
+    module Workflows
+      class StrategyBranchWorkflow < BaseWorkflow
+        # Must exceed the 30s heartbeat interval in ExecuteCliCommandActivity
+        # while staying below the agent/evaluator start_to_close budgets, so a
+        # missed beat does not time out a long-running CLI invocation.
+        HEARTBEAT_TIMEOUT = 120
+
+        def execute(input)
+          initialize_state(input)
+          log_workflow("execute_started",
+            branch_key: @branch&.dig(:key),
+            strategy_id: @strategy_id,
+            depth: @depth)
+
+          @run = start_run
+          log_workflow("run_started",
+            run_id: @run[:id],
+            branch_key: @branch&.dig(:key))
+
+          agent_result = execute_agent
+          fail_branch!(agent_result) unless agent_result[:success]
+          log_workflow("agent_completed",
+            run_id: @run[:id],
+            branch_key: @branch&.dig(:key),
+            artifact_count: Array(agent_result[:artifacts]).length)
+
+          evaluations = execute_evaluators(agent_result)
+          log_workflow("evaluator_completed",
+            run_id: @run[:id],
+            branch_key: @branch&.dig(:key),
+            evaluator_count: evaluations.length,
+            aggregate_score: aggregate_score(
+              evaluations.filter_map { |evaluation| evaluation[:score]&.to_f },
+              evaluations
+            ))
+
+          artifacts = record_artifacts(agent_result, evaluations)
+          log_workflow("artifacts_recorded",
+            run_id: @run[:id],
+            branch_key: @branch&.dig(:key),
+            artifact_count: artifacts.length)
+
+          output = build_output(agent_result, evaluations, artifacts)
+          complete_run("completed", output)
+
+          log_workflow("run_completed",
+            run_id: @run[:id],
+            branch_key: @branch&.dig(:key),
+            status: "completed",
+            aggregate_score: output[:aggregate_score])
+
+          output
+        rescue Temporalio::Error::CanceledError
+          raise
+        rescue Aidp::StrategyExecution::CliProtocol::ProtocolError,
+          Aidp::Database::Error => e
+          Aidp.log_error("strategy_branch_workflow", "execute_failed",
+            run_id: @run&.dig(:id),
+            branch_key: @branch&.dig(:key),
+            error: e.message,
+            error_class: e.class.name)
+          complete_run("failed", {error: e.message}) if @run && !run_completed?
+          raise
+        end
+
+        private
+
+        def initialize_state(input)
+          @project_dir = input[:project_dir]
+          @task = input[:task]
+          @task_id = input[:task_id]
+          @strategy = Aidp::StrategyExecution::StrategySpec.from_hash(input[:strategy])
+          @strategy_id = input[:strategy_id]
+          @branch = input[:branch]
+          @depth = input[:depth] || 0
+          @parent_run_id = input[:parent_run_id]
+          @run_completed = false
+        end
+
+        def start_run
+          execute_activity(
+            "start_run",
+            task_id: @task_id,
+            strategy_id: @strategy_id,
+            workflow_id: workflow_info.workflow_id,
+            depth: @depth,
+            input_payload: {task: @task, branch: @branch},
+            parent_run_id: @parent_run_id,
+            branch_key: @branch[:key],
+            metadata: {workflow_type: self.class.name}
+          )
+        end
+
+        def execute_agent
+          log_workflow("agent_started",
+            run_id: @run[:id],
+            branch_key: @branch&.dig(:key))
+
+          Temporalio::Workflow.execute_activity(
+            Activities::ExecuteCliCommandActivity,
+            {
+              project_dir: @project_dir,
+              command: @branch[:command],
+              role: "agent",
+              request: {
+                task: @task,
+                branch: @branch,
+                depth: @depth,
+                strategy: @strategy.to_h
+              }
+            },
+            **cli_activity_options(
+              start_to_close_timeout: activity_timeout(:agent, 900),
+              heartbeat_timeout: activity_heartbeat_timeout
+            )
+          )
+        end
+
+        def fail_branch!(agent_result)
+          message = agent_result[:error] || agent_result[:summary] || "agent execution reported failure"
+          complete_run("failed", {error: message, agent_result: agent_result})
+          raise Aidp::StrategyExecution::CliProtocol::ProtocolError, message
+        end
+
+        def execute_evaluators(agent_result)
+          @strategy.evaluator_definitions.map do |evaluator|
+            log_workflow("evaluator_started",
+              run_id: @run[:id],
+              branch_key: @branch&.dig(:key),
+              evaluator_name: evaluator[:name])
+
+            result = Temporalio::Workflow.execute_activity(
+              Activities::ExecuteCliCommandActivity,
+              {
+                project_dir: @project_dir,
+                command: evaluator[:command],
+                role: "evaluator",
+                request: {
+                  task: @task,
+                  candidate: agent_result,
+                  evaluator: evaluator,
+                  depth: @depth,
+                  strategy: @strategy.to_h
+                }
+              },
+              **cli_activity_options(
+                start_to_close_timeout: activity_timeout(:evaluator, 300),
+                heartbeat_timeout: activity_heartbeat_timeout
+              )
+            )
+
+            execute_activity(
+              "record_evaluation",
+              run_id: @run[:id],
+              evaluator_name: evaluator[:name],
+              score: result[:score],
+              passed: result[:passed],
+              summary: result[:summary],
+              metadata: result[:metadata] || {}
+            )
+
+            result.merge(name: evaluator[:name])
+          end
+        end
+
+        def record_artifacts(agent_result, evaluations)
+          records = []
+
+          Array(agent_result[:artifacts]).each do |path|
+            execute_activity(
+              "record_artifact",
+              run_id: @run[:id],
+              role: "agent",
+              path: path,
+              metadata: {branch: @branch[:key]}
+            )
+            records << {role: "agent", path: path}
+          end
+
+          evaluations.each do |evaluation|
+            Array(evaluation[:artifacts]).each do |path|
+              execute_activity(
+                "record_artifact",
+                run_id: @run[:id],
+                role: evaluation[:name],
+                path: path,
+                metadata: {branch: @branch[:key]}
+              )
+              records << {role: evaluation[:name], path: path}
+            end
+          end
+
+          records
+        end
+
+        def build_output(agent_result, evaluations, artifacts)
+          scores = evaluations.filter_map { |evaluation| evaluation[:score]&.to_f }
+
+          {
+            status: "completed",
+            run_id: @run[:id],
+            branch_key: @branch[:key],
+            branch_name: @branch[:name],
+            output: agent_result[:output] || agent_result[:content],
+            summary: agent_result[:summary],
+            metadata: agent_result[:metadata] || {},
+            subtasks: agent_result[:subtasks] || [],
+            evaluations: evaluations,
+            artifacts: artifacts,
+            aggregate_score: aggregate_score(scores, evaluations)
+          }
+        end
+
+        def aggregate_score(scores, evaluations)
+          normalized = if scores.empty?
+            pass_ratio(evaluations)
+          else
+            scores.sum / scores.length
+          end
+          bonus = evaluations.count { |evaluation| evaluation[:passed] } * 0.1
+          normalized + bonus
+        end
+
+        def pass_ratio(evaluations)
+          return 0.0 if evaluations.empty?
+
+          evaluations.count { |evaluation| evaluation[:passed] }.to_f / evaluations.length
+        end
+
+        def complete_run(status, output)
+          execute_activity(
+            "complete_run",
+            run_id: @run[:id],
+            status: status,
+            output_payload: output,
+            metadata: {branch_key: @branch[:key]}
+          )
+          @run_completed = true
+        end
+
+        def run_completed?
+          @run_completed
+        end
+
+        def execute_activity(operation, payload)
+          Temporalio::Workflow.execute_activity(
+            Activities::ManageExperienceStoreActivity,
+            {
+              project_dir: @project_dir,
+              operation: operation,
+              payload: payload
+            },
+            **store_activity_options(operation)
+          )
+        end
+
+        def activity_timeout(role, fallback)
+          value = @strategy.timeouts[role]
+          number = value.to_i
+          number.positive? ? number : fallback
+        end
+
+        def cli_activity_options(overrides = {})
+          options = activity_options(overrides)
+          options.merge(retry_policy: options.fetch(:retry_policy, {}).merge(maximum_attempts: 1))
+        end
+
+        def store_activity_options(operation)
+          options = activity_options(start_to_close_timeout: 60)
+          return options unless %w[start_run record_evaluation record_artifact].include?(operation)
+
+          options.merge(retry_policy: options.fetch(:retry_policy, {}).merge(maximum_attempts: 1))
+        end
+
+        def activity_heartbeat_timeout
+          HEARTBEAT_TIMEOUT
+        end
+
+        def activity_options(overrides = {})
+          self.class.activity_options(overrides)
+        end
+      end
+    end
+  end
+end

data/lib/aidp/temporal/workflows/strategy_execution_workflow.rb

@@ -0,0 +1,294 @@
+# frozen_string_literal: true
+
+require "temporalio/workflow"
+require_relative "base_workflow"
+require_relative "../../strategy_execution/strategy_spec"
+require_relative "../../strategy_execution/cli_protocol"
+
+module Aidp
+  module Temporal
+    module Workflows
+      class StrategyExecutionWorkflow < BaseWorkflow
+        workflow_query
+        def progress
+          {
+            state: @state,
+            depth: @depth,
+            strategy: @strategy&.name,
+            task: @task&.fetch(:description, nil),
+            run_id: @run&.fetch(:id, nil),
+            winning_branch: @winning_branch&.fetch(:branch_key, nil)
+          }
+        end
+
+        def execute(input)
+          initialize_state(input)
+          log_workflow("execute_started",
+            strategy: @strategy.name,
+            depth: @depth,
+            task_id: @task[:id])
+
+          @strategy_record = register_strategy
+          log_workflow("strategy_registered", strategy_id: @strategy_record[:id])
+
+          @task_record = ensure_task
+          log_workflow("task_ensured", task_id: @task_record[:id])
+
+          @run = start_run
+          log_workflow("run_started", run_id: @run[:id])
+
+          transition_to(:fanout)
+          branches = launch_branches
+          log_workflow("branches_launched", count: branches.length)
+
+          @winning_branch = select_winning_branch(branches)
+          log_workflow("winning_branch_selected",
+            branch_key: @winning_branch&.fetch(:branch_key, nil),
+            aggregate_score: @winning_branch&.fetch(:aggregate_score, nil))
+
+          transition_to(:merge)
+          merged_children = execute_subtasks(@winning_branch)
+          log_workflow("subtasks_completed", count: merged_children.length)
+
+          result = build_result(branches, merged_children)
+
+          complete_run("completed", result)
+          log_workflow("execute_completed", run_id: @run[:id])
+          result
+        rescue Temporalio::Error::CanceledError
+          raise
+        rescue Aidp::StrategyExecution::CliProtocol::ProtocolError,
+          Aidp::Database::Error => e
+          Aidp.log_error("strategy_execution_workflow", "execute_failed",
+            run_id: @run&.fetch(:id, nil),
+            depth: @depth,
+            error: e.message,
+            error_class: e.class.name)
+          complete_run("failed", {error: e.message}) if @run
+          raise
+        end
+
+        private
+
+        def initialize_state(input)
+          @project_dir = input[:project_dir]
+          @strategy = Aidp::StrategyExecution::StrategySpec.from_hash(input[:strategy])
+          @task = normalize_task(input[:task])
+          @depth = input[:depth] || 0
+          @parent_run_id = input[:parent_run_id]
+          @state = :init
+          @winning_branch = nil
+          @strategy_record = nil
+          @task_record = nil
+          @run = nil
+        end
+
+        def register_strategy
+          execute_store(
+            "register_strategy",
+            strategy: @strategy.to_h
+          )
+        end
+
+        def ensure_task
+          existing_task = existing_task_record
+          return existing_task if existing_task
+
+          execute_store(
+            "create_task",
+            title: @task[:title],
+            description: @task[:description],
+            input_payload: @task[:input_payload] || {},
+            context: @task[:context] || {},
+            source_run_id: @task[:source_run_id]
+          )
+        end
+
+        def existing_task_record
+          task_id = @task[:id]
+          return nil unless task_id
+
+          execute_store("task_details", task_id: task_id)
+        end
+
+        def start_run
+          execute_store(
+            "start_run",
+            task_id: @task_record[:id],
+            strategy_id: @strategy_record[:id],
+            workflow_id: workflow_info.workflow_id,
+            depth: @depth,
+            input_payload: {task: @task, strategy: @strategy.to_h},
+            parent_run_id: @parent_run_id,
+            metadata: {workflow_type: self.class.name}
+          )
+        end
+
+        def launch_branches
+          @strategy.branch_commands.map.with_index do |branch, index|
+            log_workflow("branch_launching",
+              branch_key: branch[:key],
+              branch_index: index,
+              run_id: @run[:id])
+
+            Temporalio::Workflow.execute_child_workflow(
+              StrategyBranchWorkflow,
+              {
+                project_dir: @project_dir,
+                task: @task_record,
+                task_id: @task_record[:id],
+                strategy: @strategy.to_h,
+                strategy_id: @strategy_record[:id],
+                branch: branch,
+                depth: @depth,
+                parent_run_id: @run[:id]
+              },
+              id: "#{workflow_info.workflow_id}-branch-#{index}",
+              **child_workflow_options
+            )
+          end.map(&:result)
+        end
+
+        def select_winning_branch(branches)
+          winning = case @strategy.merge_policy
+          when "highest_score", "critic_vote_then_merge"
+            branches.max_by { |branch| branch[:aggregate_score].to_f }
+          else
+            branches.first
+          end
+
+          log_workflow("merge_policy_applied",
+            merge_policy: @strategy.merge_policy,
+            candidates: branches.length,
+            winning_branch_key: winning&.fetch(:branch_key, nil),
+            run_id: @run[:id],
+            depth: @depth)
+
+          winning
+        end
+
+        def execute_subtasks(winning_branch)
+          subtasks = Array(winning_branch[:subtasks])
+          return [] if subtasks.empty?
+          return [] if @depth >= @strategy.max_depth
+
+          log_workflow("recursive_fanout_starting",
+            subtask_count: subtasks.length,
+            run_id: @run[:id],
+            depth: @depth)
+
+          transition_to(:recursive_fanout)
+
+          subtasks.map.with_index do |subtask, index|
+            Temporalio::Workflow.execute_child_workflow(
+              StrategyExecutionWorkflow,
+              {
+                project_dir: @project_dir,
+                strategy: @strategy.to_h,
+                task: normalize_subtask(subtask),
+                depth: @depth + 1,
+                parent_run_id: @run[:id]
+              },
+              id: "#{workflow_info.workflow_id}-subtask-#{index}",
+              **child_workflow_options
+            )
+          end.map(&:result)
+        end
+
+        def build_result(branches, merged_children)
+          {
+            status: "completed",
+            run_id: @run[:id],
+            task_id: @task_record[:id],
+            strategy: @strategy.name,
+            depth: @depth,
+            winning_branch: @winning_branch,
+            branch_count: branches.length,
+            branches: branches,
+            merged_children: merged_children
+          }
+        end
+
+        def complete_run(status, output)
+          execute_store(
+            "complete_run",
+            run_id: @run[:id],
+            status: status,
+            output_payload: output,
+            metadata: {
+              state: @state,
+              winning_branch: @winning_branch&.fetch(:branch_key, nil)
+            }
+          )
+          log_workflow("run_completed",
+            run_id: @run[:id],
+            status: status,
+            depth: @depth,
+            branch_key: @winning_branch&.fetch(:branch_key, nil))
+        end
+
+        def execute_store(operation, payload)
+          Temporalio::Workflow.execute_activity(
+            Activities::ManageExperienceStoreActivity,
+            {
+              project_dir: @project_dir,
+              operation: operation,
+              payload: payload
+            },
+            **store_activity_options(operation)
+          )
+        end
+
+        def child_workflow_options
+          {
+            task_queue: workflow_info.task_queue,
+            execution_timeout: 3600
+          }
+        end
+
+        def store_activity_options(operation)
+          options = activity_options(start_to_close_timeout: 60)
+          return options unless %w[create_task start_run].include?(operation)
+
+          options.merge(retry_policy: options.fetch(:retry_policy, {}).merge(maximum_attempts: 1))
+        end
+
+        def normalize_task(task)
+          normalized = (task || {}).transform_keys(&:to_sym)
+          description = normalized[:description].to_s
+          raise ArgumentError, "task description is required" if description.empty?
+
+          normalized
+        end
+
+        def normalize_subtask(subtask)
+          data = case subtask
+          when Hash then subtask.transform_keys(&:to_sym)
+          else {description: subtask.to_s}
+          end
+
+          {
+            title: data[:title],
+            description: data[:description].to_s,
+            input_payload: data[:input_payload] || {},
+            context: data[:context] || {}
+          }
+        end
+
+        def transition_to(state)
+          previous = @state
+          @state = state
+          log_workflow("state_transition",
+            from: previous,
+            to: state,
+            run_id: @run&.fetch(:id, nil),
+            depth: @depth)
+        end
+
+        def activity_options(overrides = {})
+          self.class.activity_options(overrides)
+        end
+      end
+    end
+  end
+end

data/lib/aidp/temporal.rb

@@ -13,6 +13,8 @@
 # - IssueToPrWorkflow: Full issue analysis → implementation → PR pipeline
 # - WorkLoopWorkflow: Fix-forward iteration pattern
 # - SubIssueWorkflow: Child workflow for decomposed tasks
+# - StrategyExecutionWorkflow: Generic strategy-as-data execution
+# - StrategyBranchWorkflow: Speculative branch execution for strategies
 #
 # Activities:
 # - RunAgentActivity: Execute AI agent
@@ -24,6 +26,8 @@
 # - PrepareNextIterationActivity: Set up next iteration
 # - RecordCheckpointActivity: Save progress
 # - CreatePrActivity: Create GitHub PR
+# - ExecuteCliCommandActivity: Generic agent/evaluator CLI protocol runner
+# - ManageExperienceStoreActivity: Persist strategy experience data
 
 module Aidp
   module Temporal
@@ -104,6 +108,8 @@ require_relative "temporal/workflows/base_workflow"
 require_relative "temporal/workflows/issue_to_pr_workflow"
 require_relative "temporal/workflows/work_loop_workflow"
 require_relative "temporal/workflows/sub_issue_workflow"
+require_relative "temporal/workflows/strategy_branch_workflow"
+require_relative "temporal/workflows/strategy_execution_workflow"
 
 # Require activities
 require_relative "temporal/activities/base_activity"
@@ -118,3 +124,5 @@ require_relative "temporal/activities/record_checkpoint_activity"
 require_relative "temporal/activities/create_pr_activity"
 require_relative "temporal/activities/run_work_loop_iteration_activity"
 require_relative "temporal/activities/analyze_sub_task_activity"
+require_relative "temporal/activities/execute_cli_command_activity"
+require_relative "temporal/activities/manage_experience_store_activity"

data/lib/aidp/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aidp
-  VERSION = "0.47.1"
+  VERSION = "0.48.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aidp
 version: !ruby/object:Gem::Version
-  version: 0.47.1
+  version: 0.48.0
 platform: ruby
 authors:
 - Bart Agapinan
@@ -374,6 +374,10 @@ files:
 - lib/aidp/database/repositories/checkpoint_repository.rb
 - lib/aidp/database/repositories/deprecated_models_repository.rb
 - lib/aidp/database/repositories/evaluation_repository.rb
+- lib/aidp/database/repositories/experience_artifact_repository.rb
+- lib/aidp/database/repositories/experience_evaluation_repository.rb
+- lib/aidp/database/repositories/experience_run_repository.rb
+- lib/aidp/database/repositories/experience_task_repository.rb
 - lib/aidp/database/repositories/harness_state_repository.rb
 - lib/aidp/database/repositories/job_repository.rb
 - lib/aidp/database/repositories/model_cache_repository.rb
@@ -383,6 +387,7 @@ files:
 - lib/aidp/database/repositories/provider_info_cache_repository.rb
 - lib/aidp/database/repositories/provider_metrics_repository.rb
 - lib/aidp/database/repositories/secrets_repository.rb
+- lib/aidp/database/repositories/strategy_repository.rb
 - lib/aidp/database/repositories/task_repository.rb
 - lib/aidp/database/repositories/template_version_repository.rb
 - lib/aidp/database/repositories/watch_state_repository.rb
@@ -574,6 +579,10 @@ files:
 - lib/aidp/storage/csv_storage.rb
 - lib/aidp/storage/file_manager.rb
 - lib/aidp/storage/json_storage.rb
+- lib/aidp/strategy_execution/cli_protocol.rb
+- lib/aidp/strategy_execution/experience_store.rb
+- lib/aidp/strategy_execution/strategy_loader.rb
+- lib/aidp/strategy_execution/strategy_spec.rb
 - lib/aidp/style_guide/selector.rb
 - lib/aidp/temporal.rb
 - lib/aidp/temporal/activities/analyze_issue_activity.rb
@@ -583,6 +592,8 @@ files:
 - lib/aidp/temporal/activities/create_pr_activity.rb
 - lib/aidp/temporal/activities/create_prompt_activity.rb
 - lib/aidp/temporal/activities/diagnose_failure_activity.rb
+- lib/aidp/temporal/activities/execute_cli_command_activity.rb
+- lib/aidp/temporal/activities/manage_experience_store_activity.rb
 - lib/aidp/temporal/activities/prepare_next_iteration_activity.rb
 - lib/aidp/temporal/activities/record_checkpoint_activity.rb
 - lib/aidp/temporal/activities/run_agent_activity.rb
@@ -594,6 +605,8 @@ files:
 - lib/aidp/temporal/workflow_client.rb
 - lib/aidp/temporal/workflows/base_workflow.rb
 - lib/aidp/temporal/workflows/issue_to_pr_workflow.rb
+- lib/aidp/temporal/workflows/strategy_branch_workflow.rb
+- lib/aidp/temporal/workflows/strategy_execution_workflow.rb
 - lib/aidp/temporal/workflows/sub_issue_workflow.rb
 - lib/aidp/temporal/workflows/work_loop_workflow.rb
 - lib/aidp/tooling_detector.rb