aidp 0.17.1 → 0.19.0

data/lib/aidp/execute/work_loop_runner.rb

@@ -46,11 +46,12 @@ module Aidp
         @project_dir = project_dir
         @provider_manager = provider_manager
         @config = config
-        @prompt_manager = PromptManager.new(project_dir)
+        @prompt_manager = PromptManager.new(project_dir, config: config)
         @test_runner = Aidp::Harness::TestRunner.new(project_dir, config)
         @checkpoint = Checkpoint.new(project_dir)
         @checkpoint_display = CheckpointDisplay.new
         @guard_policy = GuardPolicy.new(project_dir, config.guards_config)
+        @persistent_tasklist = PersistentTasklist.new(project_dir)
         @iteration_count = 0
         @step_name = nil
         @options = options
@@ -74,6 +75,7 @@ module Aidp
         display_message("  Flow: Deterministic ↔ Agentic with fix-forward core", type: :info)
 
         display_guard_policy_status
+        display_pending_tasks
 
         @unit_scheduler = WorkLoopUnitScheduler.new(units_config)
         base_context = context.dup
@@ -148,6 +150,9 @@ module Aidp
           transition_to(:apply_patch)
           agent_result = apply_patch
 
+          # Process agent output for task filing signals
+          process_task_filing(agent_result)
+
           transition_to(:test)
           test_results = @test_runner.run_tests
           lint_results = @test_runner.run_linters
@@ -343,6 +348,16 @@ module Aidp
 
       # Create initial PROMPT.md with all context
       def create_initial_prompt(step_spec, context)
+        # Try intelligent prompt optimization first (ZFC-powered)
+        if @prompt_manager.optimization_enabled?
+          if create_optimized_prompt(step_spec, context)
+            return
+          end
+          # Fallback to traditional prompt on optimization failure
+          display_message("  ⚠️  Prompt optimization failed, using traditional approach", type: :warning)
+        end
+
+        # Traditional prompt building (fallback or when optimization disabled)
         template_content = load_template(step_spec["templates"]&.first)
         prd_content = load_prd
         style_guide = load_style_guide
@@ -364,6 +379,103 @@ module Aidp
         display_message("  Created PROMPT.md (#{initial_prompt.length} chars)", type: :info)
       end
 
+      # Create prompt using intelligent optimization (Zero Framework Cognition)
+      # Selects only the most relevant fragments from style guide, templates, and code
+      def create_optimized_prompt(step_spec, context)
+        user_input = format_user_input(context[:user_input])
+
+        # Infer task type from step name
+        task_type = infer_task_type(step_spec, user_input)
+
+        # Extract affected files from context or PRD
+        affected_files = extract_affected_files(context, user_input)
+
+        # Build task context for optimizer
+        task_context = {
+          task_type: task_type,
+          description: build_task_description(user_input, context),
+          affected_files: affected_files,
+          step_name: @step_name,
+          tags: extract_tags(user_input, step_spec)
+        }
+
+        # Use optimizer to create prompt
+        success = @prompt_manager.write_optimized(
+          task_context,
+          include_metadata: @config.prompt_log_fragments?
+        )
+
+        if success
+          stats = @prompt_manager.last_optimization_stats
+          display_message("  ✨ Created optimized PROMPT.md", type: :success)
+          display_message("     Selected: #{stats.selected_count} fragments, Excluded: #{stats.excluded_count}", type: :info)
+          display_message("     Tokens: #{stats.total_tokens} (#{stats.budget_utilization.round(1)}% of budget)", type: :info)
+          display_message("     Avg relevance: #{(stats.average_score * 100).round(1)}%", type: :info)
+        end
+
+        success
+      end
+
+      # Infer task type from step name and context
+      def infer_task_type(step_spec, user_input)
+        step_name = @step_name.to_s.downcase
+        input_lower = user_input.to_s.downcase
+
+        return :test if step_name.include?("test") || input_lower.include?("test")
+        return :bugfix if step_name.include?("fix") || input_lower.include?("fix") || input_lower.include?("bug")
+        return :refactor if step_name.include?("refactor") || input_lower.include?("refactor")
+        return :analysis if step_name.include?("analyz") || step_name.include?("review")
+
+        :feature # Default to feature
+      end
+
+      # Extract files that will be affected by this work
+      def extract_affected_files(context, user_input)
+        files = []
+
+        # From user input (e.g., "update lib/user.rb")
+        user_input&.scan(/[\w\/]+\.rb/)&.each do |file|
+          files << file
+        end
+
+        # From deterministic outputs
+        context[:deterministic_outputs]&.each do |output|
+          if output[:output_path]&.end_with?(".rb")
+            files << output[:output_path]
+          end
+        end
+
+        files.uniq
+      end
+
+      # Build task description from context
+      def build_task_description(user_input, context)
+        parts = []
+        parts << user_input if user_input && !user_input.empty?
+        parts << context[:previous_agent_summary] if context[:previous_agent_summary]
+        parts.join("\n\n")
+      end
+
+      # Extract relevant tags from input and spec
+      def extract_tags(user_input, step_spec)
+        tags = []
+        input_lower = user_input.to_s.downcase
+
+        # Common tags from content
+        tags << "testing" if input_lower.include?("test")
+        tags << "security" if input_lower.include?("security") || input_lower.include?("auth")
+        tags << "api" if input_lower.include?("api") || input_lower.include?("endpoint")
+        tags << "database" if input_lower.include?("database") || input_lower.include?("migration")
+        tags << "performance" if input_lower.include?("performance") || input_lower.include?("optim")
+
+        # Tags from step spec
+        if step_spec["tags"]
+          tags.concat(Array(step_spec["tags"]))
+        end
+
+        tags.uniq
+      end
+
       def build_initial_prompt_content(template:, prd:, style_guide:, user_input:, step_name:, deterministic_outputs:, previous_agent_summary:)
         parts = []
 
@@ -669,6 +781,55 @@ module Aidp
         display_message("")
       end
 
+      # Display pending tasks from persistent tasklist
+      def display_pending_tasks
+        pending_tasks = @persistent_tasklist.pending
+        return if pending_tasks.empty?
+
+        display_message("\n📋 Pending Tasks from Previous Sessions:", type: :info)
+
+        # Show up to 5 most recent pending tasks
+        pending_tasks.take(5).each do |task|
+          priority_icon = case task.priority
+          when :high then "⚠️ "
+          when :medium then "○ "
+          when :low then "· "
+          end
+
+          age = ((Time.now - task.created_at) / 86400).to_i
+          age_str = (age > 0) ? " (#{age}d ago)" : " (today)"
+
+          display_message("  #{priority_icon}#{task.description}#{age_str}", type: :info)
+        end
+
+        if pending_tasks.size > 5
+          display_message("  ... and #{pending_tasks.size - 5} more. Use /tasks list to see all", type: :info)
+        end
+
+        display_message("")
+      end
+
+      # Process agent output for task filing signals
+      def process_task_filing(agent_result)
+        return unless agent_result && agent_result[:output]
+
+        filed_tasks = AgentSignalParser.parse_task_filing(agent_result[:output])
+        return if filed_tasks.empty?
+
+        filed_tasks.each do |task_data|
+          task = @persistent_tasklist.create(
+            task_data[:description],
+            priority: task_data[:priority],
+            session: @step_name,
+            discovered_during: "#{@step_name} iteration #{@iteration_count}",
+            tags: task_data[:tags]
+          )
+
+          Aidp.log_info("tasklist", "Filed new task from agent", task_id: task.id, description: task.description)
+          display_message("📋 Filed task: #{task.description} (#{task.id})", type: :info)
+        end
+      end
+
       # Validate changes against guard policy
       # Returns validation result with errors if any
       def validate_guard_policy(changed_files = [])

data/lib/aidp/harness/ai_decision_engine.rb

@@ -0,0 +1,376 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "provider_factory"
+require_relative "thinking_depth_manager"
+
+module Aidp
+  module Harness
+    # Zero Framework Cognition (ZFC) Decision Engine
+    #
+    # Delegates semantic analysis and decision-making to AI models instead of
+    # using brittle pattern matching, scoring formulas, or heuristic thresholds.
+    #
+    # @example Basic usage
+    #   engine = AIDecisionEngine.new(config, provider_manager)
+    #   result = engine.decide(:condition_detection,
+    #     context: { error: "Rate limit exceeded" },
+    #     schema: ConditionSchema,
+    #     tier: "mini"
+    #   )
+    #   # => { condition: "rate_limit", confidence: 0.95, reasoning: "..." }
+    #
+    # @see docs/ZFC_COMPLIANCE_ASSESSMENT.md
+    # @see docs/ZFC_IMPLEMENTATION_PLAN.md
+    class AIDecisionEngine
+      # Decision templates define prompts, schemas, and defaults for each decision type
+      DECISION_TEMPLATES = {
+        condition_detection: {
+          prompt_template: <<~PROMPT,
+            Analyze the following API response or error message and classify the condition.
+
+            Response/Error:
+            {{response}}
+
+            Classify this into one of the following conditions:
+            - rate_limit: API rate limiting or quota exceeded
+            - auth_error: Authentication or authorization failure
+            - timeout: Request timeout or network timeout
+            - completion_marker: Work is complete or done
+            - user_feedback_needed: AI is asking for user input/clarification
+            - api_error: General API error (not rate limit/auth)
+            - success: Successful response
+            - other: None of the above
+
+            Provide your classification with a confidence score (0.0 to 1.0) and brief reasoning.
+          PROMPT
+          schema: {
+            type: "object",
+            properties: {
+              condition: {
+                type: "string",
+                enum: [
+                  "rate_limit",
+                  "auth_error",
+                  "timeout",
+                  "completion_marker",
+                  "user_feedback_needed",
+                  "api_error",
+                  "success",
+                  "other"
+                ]
+              },
+              confidence: {
+                type: "number",
+                minimum: 0.0,
+                maximum: 1.0
+              },
+              reasoning: {
+                type: "string"
+              }
+            },
+            required: ["condition", "confidence"]
+          },
+          default_tier: "mini",
+          cache_ttl: nil  # Each response is unique
+        },
+
+        error_classification: {
+          prompt_template: <<~PROMPT,
+            Classify the following error and determine if it's retryable.
+
+            Error:
+            {{error_message}}
+
+            Context:
+            {{context}}
+
+            Determine:
+            1. Error type (rate_limit, auth, timeout, network, api_bug, other)
+            2. Whether it's retryable (transient vs permanent)
+            3. Recommended action (retry, switch_provider, escalate, fail)
+
+            Provide classification with confidence and reasoning.
+          PROMPT
+          schema: {
+            type: "object",
+            properties: {
+              error_type: {
+                type: "string",
+                enum: ["rate_limit", "auth", "timeout", "network", "api_bug", "other"]
+              },
+              retryable: {
+                type: "boolean"
+              },
+              recommended_action: {
+                type: "string",
+                enum: ["retry", "switch_provider", "escalate", "fail"]
+              },
+              confidence: {
+                type: "number",
+                minimum: 0.0,
+                maximum: 1.0
+              },
+              reasoning: {
+                type: "string"
+              }
+            },
+            required: ["error_type", "retryable", "recommended_action", "confidence"]
+          },
+          default_tier: "mini",
+          cache_ttl: nil
+        },
+
+        completion_detection: {
+          prompt_template: <<~PROMPT,
+            Determine if the work described is complete based on the AI response.
+
+            Task:
+            {{task_description}}
+
+            AI Response:
+            {{response}}
+
+            Is the work complete? Consider:
+            - Explicit completion markers ("done", "finished", etc.)
+            - Implicit indicators (results provided, no follow-up questions)
+            - Requests for more information (incomplete)
+
+            Provide boolean completion status with confidence and reasoning.
+          PROMPT
+          schema: {
+            type: "object",
+            properties: {
+              complete: {
+                type: "boolean"
+              },
+              confidence: {
+                type: "number",
+                minimum: 0.0,
+                maximum: 1.0
+              },
+              reasoning: {
+                type: "string"
+              }
+            },
+            required: ["complete", "confidence"]
+          },
+          default_tier: "mini",
+          cache_ttl: nil
+        }
+      }.freeze
+
+      attr_reader :config, :provider_factory, :cache
+
+      # Initialize the AI Decision Engine
+      #
+      # @param config [Configuration] AIDP configuration object
+      # @param provider_factory [ProviderFactory] Factory for creating provider instances
+      def initialize(config, provider_factory: nil)
+        @config = config
+        @provider_factory = provider_factory || ProviderFactory.new(config)
+        @cache = {}
+        @cache_timestamps = {}
+      end
+
+      # Make an AI-powered decision
+      #
+      # @param decision_type [Symbol] Type of decision (:condition_detection, :error_classification, etc.)
+      # @param context [Hash] Context data for the decision
+      # @param schema [Hash, nil] JSON schema for response validation (overrides default)
+      # @param tier [String, nil] Thinking depth tier (overrides default)
+      # @param cache_ttl [Integer, nil] Cache TTL in seconds (overrides default)
+      # @return [Hash] Validated decision result
+      # @raise [ArgumentError] If decision_type is unknown
+      # @raise [ValidationError] If response doesn't match schema
+      def decide(decision_type, context:, schema: nil, tier: nil, cache_ttl: nil)
+        template = DECISION_TEMPLATES[decision_type]
+        raise ArgumentError, "Unknown decision type: #{decision_type}" unless template
+
+        # Check cache if TTL specified
+        cache_key = build_cache_key(decision_type, context)
+        ttl = cache_ttl || template[:cache_ttl]
+        if ttl && (cached_result = get_cached(cache_key, ttl))
+          Aidp.log_debug("ai_decision_engine", "Cache hit for #{decision_type}", {
+            cache_key: cache_key,
+            ttl: ttl
+          })
+          return cached_result
+        end
+
+        # Build prompt from template
+        prompt = build_prompt(template[:prompt_template], context)
+
+        # Select tier
+        selected_tier = tier || template[:default_tier]
+
+        # Get model for tier
+        thinking_manager = ThinkingDepthManager.new(config)
+        provider_name, model_name, _model_data = thinking_manager.select_model_for_tier(selected_tier)
+
+        Aidp.log_debug("ai_decision_engine", "Making AI decision", {
+          decision_type: decision_type,
+          tier: selected_tier,
+          provider: provider_name,
+          model: model_name,
+          cache_ttl: ttl
+        })
+
+        # Call AI with schema validation
+        response_schema = schema || template[:schema]
+        result = call_ai_with_schema(provider_name, model_name, prompt, response_schema)
+
+        # Validate result
+        validate_schema(result, response_schema)
+
+        # Cache if TTL specified
+        if ttl
+          set_cached(cache_key, result)
+        end
+
+        result
+      end
+
+      private
+
+      # Build cache key from decision type and context
+      def build_cache_key(decision_type, context)
+        # Simple hash-based key
+        "#{decision_type}:#{context.hash}"
+      end
+
+      # Get cached result if still valid
+      def get_cached(key, ttl)
+        return nil unless @cache.key?(key)
+        return nil if Time.now - @cache_timestamps[key] > ttl
+        @cache[key]
+      end
+
+      # Store result in cache
+      def set_cached(key, value)
+        @cache[key] = value
+        @cache_timestamps[key] = Time.now
+      end
+
+      # Build prompt from template with context substitution
+      def build_prompt(template, context)
+        prompt = template.dup
+        context.each do |key, value|
+          prompt.gsub!("{{#{key}}}", value.to_s)
+        end
+        prompt
+      end
+
+      # Call AI with schema validation using structured output
+      def call_ai_with_schema(provider_name, model_name, prompt, schema)
+        # Create provider instance
+        provider_options = {
+          model: model_name,
+          output: nil,  # No output for background decisions
+          prompt: nil   # No TTY prompt needed
+        }
+
+        provider = @provider_factory.create_provider(provider_name, provider_options)
+
+        # Build enhanced prompt requesting JSON output
+        enhanced_prompt = <<~PROMPT
+          #{prompt}
+
+          IMPORTANT: Respond with ONLY valid JSON. No additional text or explanation.
+          The JSON must match this structure: #{JSON.generate(schema[:properties].keys)}
+        PROMPT
+
+        # Call provider
+        response = provider.send_message(prompt: enhanced_prompt, session: nil)
+
+        # Parse JSON response
+        begin
+          # Response might be a string or already structured
+          response_text = response.is_a?(String) ? response : response.to_s
+
+          # Try to extract JSON if there's extra text
+          # Use non-greedy match and handle nested braces
+          json_match = response_text.match(/\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/m) || response_text.match(/\{.*\}/m)
+          json_text = json_match ? json_match[0] : response_text
+
+          result = JSON.parse(json_text, symbolize_names: true)
+
+          Aidp.log_debug("ai_decision_engine", "Parsed JSON successfully", {
+            response_length: response_text.length,
+            json_length: json_text.length,
+            result_keys: result.keys,
+            provider: provider_name
+          })
+
+          result
+        rescue JSON::ParserError => e
+          Aidp.log_error("ai_decision_engine", "Failed to parse AI response as JSON", {
+            error: e.message,
+            response: response_text&.slice(0, 200),
+            provider: provider_name,
+            model: model_name
+          })
+          raise ValidationError, "AI response is not valid JSON: #{e.message}"
+        end
+      rescue => e
+        Aidp.log_error("ai_decision_engine", "Error calling AI provider", {
+          error: e.message,
+          provider: provider_name,
+          model: model_name,
+          error_class: e.class.name
+        })
+        raise
+      end
+
+      # Validate response against JSON schema
+      def validate_schema(result, schema)
+        # Basic validation of required fields and types
+        # Schema uses string keys, but our result uses symbol keys from JSON parsing
+        schema[:required]&.each do |field|
+          field_sym = field.to_sym
+          unless result.key?(field_sym)
+            raise ValidationError, "Missing required field: #{field}"
+          end
+        end
+
+        schema[:properties]&.each do |field, constraints|
+          field_sym = field.to_sym
+          next unless result.key?(field_sym)
+          value = result[field_sym]
+
+          # Type validation
+          case constraints[:type]
+          when "string"
+            unless value.is_a?(String)
+              raise ValidationError, "Field #{field} must be string, got #{value.class}"
+            end
+            # Enum validation
+            if constraints[:enum] && !constraints[:enum].include?(value)
+              raise ValidationError, "Field #{field} must be one of #{constraints[:enum]}, got #{value}"
+            end
+          when "number"
+            unless value.is_a?(Numeric)
+              raise ValidationError, "Field #{field} must be number, got #{value.class}"
+            end
+            # Range validation
+            if constraints[:minimum] && value < constraints[:minimum]
+              raise ValidationError, "Field #{field} must be >= #{constraints[:minimum]}"
+            end
+            if constraints[:maximum] && value > constraints[:maximum]
+              raise ValidationError, "Field #{field} must be <= #{constraints[:maximum]}"
+            end
+          when "boolean"
+            unless [true, false].include?(value)
+              raise ValidationError, "Field #{field} must be boolean, got #{value.class}"
+            end
+          end
+        end
+
+        true
+      end
+    end
+
+    # Validation error for schema violations
+    class ValidationError < StandardError; end
+  end
+end