aidp 0.21.1 → 0.23.0

data/lib/aidp/execute/prompt_manager.rb

@@ -19,12 +19,16 @@ module Aidp
 
       def initialize(project_dir, config: nil)
         @project_dir = project_dir
-        @prompt_path = File.join(project_dir, PROMPT_FILENAME)
+        @aidp_dir = File.join(project_dir, ".aidp")
+        @prompt_path = File.join(@aidp_dir, PROMPT_FILENAME)
         @archive_dir = File.join(project_dir, ARCHIVE_DIR)
         @config = config
         @optimizer = nil
         @last_optimization_stats = nil
 
+        # Ensure .aidp directory exists
+        FileUtils.mkdir_p(@aidp_dir)
+
         # Initialize optimizer if enabled
         if config&.respond_to?(:prompt_optimization_enabled?) && config.prompt_optimization_enabled?
           @optimizer = Aidp::PromptOptimization::Optimizer.new(
@@ -37,8 +41,15 @@ module Aidp
       # Write content to PROMPT.md
       # If optimization is enabled, stores the content but doesn't write yet
       # (use write_optimized instead)
-      def write(content)
+      #
+      # @param content [String] The prompt content to write
+      # @param step_name [String, nil] Optional step name for immediate archiving
+      # @return [String, nil] Archive path if archived, nil otherwise
+      def write(content, step_name: nil)
         File.write(@prompt_path, content)
+
+        # Archive immediately if step_name provided (issue #224)
+        archive(step_name) if step_name
       end
 
       # Write optimized prompt using intelligent fragment selection
@@ -88,6 +99,9 @@ module Aidp
             budget_utilization: result.composition_result.budget_utilization
           )
 
+          # Archive immediately if step_name provided (issue #224)
+          archive(task_context[:step_name]) if task_context[:step_name]
+
           true
         rescue => e
           Aidp.logger.error("prompt_manager", "Optimization failed, using fallback", error: e.message)

data/lib/aidp/execute/runner.rb

@@ -308,12 +308,17 @@ module Aidp
       end
 
       def template_search_paths
+        gem_root = File.expand_path("../../../..", __dir__)
+        # We search:
+        # 1. Project root templates (supports entries like "planning/create_prd.md" since name carries subfolder)
+        # 2. Project COMMON templates (supports direct filename lookups like "00_PRD.md")
+        # 3. Gem root templates (fallback when project lacks local copies)
+        # 4. Gem COMMON templates (fallback for filename-only lookups)
         [
-          File.join(@project_dir, "templates"),        # Root templates folder
-          File.join(@project_dir, "templates", "planning"),
-          File.join(@project_dir, "templates", "analysis"),
-          File.join(@project_dir, "templates", "implementation"),
-          File.join(@project_dir, "templates", "COMMON")
+          File.join(@project_dir, "templates"),
+          File.join(@project_dir, "templates", "COMMON"),
+          File.join(gem_root, "templates"),
+          File.join(gem_root, "templates", "COMMON")
         ]
       end
 

data/lib/aidp/execute/work_loop_runner.rb

@@ -375,7 +375,7 @@ module Aidp
           previous_agent_summary: previous_summary
         )
 
-        @prompt_manager.write(initial_prompt)
+        @prompt_manager.write(initial_prompt, step_name: @step_name)
         display_message("  Created PROMPT.md (#{initial_prompt.length} chars)", type: :info)
       end
 
@@ -603,10 +603,10 @@ module Aidp
 
         return if test_results[:success] && lint_results[:success]
 
-        # Append failures to PROMPT.md
+        # Append failures to PROMPT.md and archive immediately (issue #224)
         current_prompt = @prompt_manager.read
         updated_prompt = current_prompt + "\n\n---\n\n" + failures.join("\n")
-        @prompt_manager.write(updated_prompt)
+        @prompt_manager.write(updated_prompt, step_name: @step_name)
 
         display_message("  [NEXT_PATCH] Added failure reports and diagnostic to PROMPT.md", type: :warning)
       end

data/lib/aidp/harness/runner.rb

@@ -27,11 +27,12 @@ module Aidp
         waiting_for_rate_limit: "waiting_for_rate_limit",
         stopped: "stopped",
         completed: "completed",
-        error: "error"
+        error: "error",
+        needs_clarification: "needs_clarification"
       }.freeze
 
       # Public accessors for testing and integration
-      attr_reader :current_provider, :current_step, :user_input, :execution_log, :provider_manager
+      attr_reader :current_provider, :current_step, :user_input, :execution_log, :provider_manager, :clarification_questions
 
       def initialize(project_dir, mode = :analyze, options = {})
         @project_dir = project_dir
@@ -132,7 +133,9 @@ module Aidp
           cleanup
         end
 
-        {status: @state, message: get_completion_message}
+        result = {status: @state, message: get_completion_message}
+        result[:clarification_questions] = @clarification_questions if @clarification_questions
+        result
       end
 
       # Pause the harness execution
@@ -248,12 +251,23 @@ module Aidp
       end
 
       def handle_user_feedback_request(result)
-        @state = STATES[:waiting_for_user]
-        log_execution("Waiting for user feedback")
-
         # Extract questions from result
         questions = @condition_detector.extract_questions(result)
 
+        # Check if we're in watch mode (non-interactive)
+        if @options[:workflow_type] == :watch_mode
+          # Store questions for later retrieval and set state to needs_clarification
+          @clarification_questions = questions
+          @state = STATES[:needs_clarification]
+          log_execution("Clarification needed in watch mode", {question_count: questions.size})
+          # Don't continue - exit the loop so we can return this status
+          return
+        end
+
+        # Interactive mode: collect feedback from user
+        @state = STATES[:waiting_for_user]
+        log_execution("Waiting for user feedback")
+
         # Collect user input
         user_responses = @user_interface.collect_feedback(questions)
 

data/lib/aidp/harness/state/persistence.rb

@@ -2,12 +2,15 @@
 
 require "json"
 require "fileutils"
+require_relative "../../safe_directory"
 
 module Aidp
   module Harness
     module State
       # Handles file I/O and persistence for state management
       class Persistence
+        include Aidp::SafeDirectory
+
         def initialize(project_dir, mode, skip_persistence: false)
           @project_dir = project_dir
           @mode = mode
@@ -83,7 +86,15 @@ module Aidp
         end
 
         def ensure_state_directory
-          FileUtils.mkdir_p(@state_dir) unless Dir.exist?(@state_dir)
+          return if @skip_persistence  # Don't create directories when persistence is disabled
+
+          original_dir = @state_dir
+          @state_dir = safe_mkdir_p(@state_dir, component_name: "State::Persistence")
+
+          # If fallback occurred, switch to in-memory mode
+          if @state_dir != original_dir
+            @skip_persistence = true
+          end
         end
 
         def with_lock(&block)

data/lib/aidp/harness/state_manager.rb

@@ -6,11 +6,14 @@ require_relative "../execute/progress"
 require_relative "../analyze/progress"
 require_relative "../execute/steps"
 require_relative "../analyze/steps"
+require_relative "../safe_directory"
 
 module Aidp
   module Harness
     # Manages harness-specific state and persistence, extending existing progress tracking
     class StateManager
+      include Aidp::SafeDirectory
+
       def initialize(project_dir, mode, skip_persistence: false)
         @project_dir = project_dir
         @mode = mode
@@ -576,7 +579,16 @@ module Aidp
       end
 
       def ensure_state_directory
-        FileUtils.mkdir_p(@state_dir) unless Dir.exist?(@state_dir)
+        return if @skip_persistence  # Don't create directories when persistence is disabled
+
+        original_dir = @state_dir
+        @state_dir = safe_mkdir_p(@state_dir, component_name: "StateManager")
+
+        # If fallback occurred, switch to in-memory mode
+        if @state_dir != original_dir
+          @skip_persistence = true
+          @memory_state ||= {}
+        end
       end
 
       def with_lock(&_block)

data/lib/aidp/jobs/background_runner.rb

@@ -6,6 +6,7 @@ require "fileutils"
 require "time"
 require_relative "../rescue_logging"
 require_relative "../concurrency"
+require_relative "../safe_directory"
 
 module Aidp
   module Jobs
@@ -14,6 +15,7 @@ module Aidp
     class BackgroundRunner
       include Aidp::MessageDisplay
       include Aidp::RescueLogging
+      include Aidp::SafeDirectory
 
       attr_reader :project_dir, :jobs_dir
 
@@ -190,7 +192,7 @@ module Aidp
       private
 
       def ensure_jobs_directory
-        FileUtils.mkdir_p(@jobs_dir) unless Dir.exist?(@jobs_dir)
+        @jobs_dir = safe_mkdir_p(@jobs_dir, component_name: "BackgroundRunner")
       end
 
       def generate_job_id

data/lib/aidp/logger.rb

@@ -39,6 +39,7 @@ module Aidp
       @json_format = config[:json] || false
       @max_size = config[:max_size_mb] ? config[:max_size_mb] * 1024 * 1024 : DEFAULT_MAX_SIZE
       @max_files = config[:max_backups] || DEFAULT_MAX_FILES
+      @instrument_internal = config.key?(:instrument) ? config[:instrument] : (ENV["AIDP_LOG_INSTRUMENT"] == "1")
 
       ensure_log_directory
       setup_logger
@@ -95,12 +96,24 @@ module Aidp
 
     def ensure_log_directory
       log_dir = File.join(@project_dir, LOG_DIR)
-      FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir)
+      return if Dir.exist?(log_dir)
+      begin
+        FileUtils.mkdir_p(log_dir)
+      rescue SystemCallError => e
+        Kernel.warn "[AIDP Logger] Cannot create log directory #{log_dir}: #{e.class}: #{e.message}. Falling back to STDERR logging."
+        # We intentionally do not re-raise; file logger setup will attempt and then fall back itself.
+      end
     end
 
     def setup_logger
       info_path = File.join(@project_dir, INFO_LOG)
       @logger = create_logger(info_path)
+      # Emit instrumentation after logger is available (avoid recursive Aidp.log_* calls during bootstrap)
+      return unless @instrument_internal
+      if defined?(@root_fallback) && @root_fallback
+        debug("logger", "root_fallback_applied", effective_dir: @root_fallback)
+      end
+      debug("logger", "logger_initialized", path: info_path, project_dir: @project_dir)
     end
 
     def create_logger(path)
@@ -200,12 +213,35 @@ module Aidp
     # that would create odd top-level directories like "<STDERR>".
     def sanitize_project_dir(dir)
       return Dir.pwd if dir.nil?
-      str = dir.to_s
-      if str.empty? || str.match?(/[<>|]/) || str.match?(/[\x00-\x1F]/)
-        Kernel.warn "[AIDP Logger] Invalid project_dir '#{str}' - falling back to #{Dir.pwd}"
+      raw_input = dir.to_s
+      raw_invalid = raw_input.empty? || raw_input.match?(/[<>|]/) || raw_input.match?(/[\x00-\x1F]/)
+      if raw_invalid
+        Kernel.warn "[AIDP Logger] Invalid project_dir '#{raw_input}' - falling back to #{Dir.pwd}"
+        if Dir.pwd == File::SEPARATOR
+          fallback = begin
+            home = Dir.home
+            (home && !home.empty?) ? home : Dir.tmpdir
+          rescue
+            Dir.tmpdir
+          end
+          @root_fallback = fallback
+          Kernel.warn "[AIDP Logger] Root directory detected - using #{fallback} for logging instead of '#{Dir.pwd}'"
+          return fallback
+        end
         return Dir.pwd
       end
-      str
+      if raw_input == File::SEPARATOR
+        fallback = begin
+          home = Dir.home
+          (home && !home.empty?) ? home : Dir.tmpdir
+        rescue
+          Dir.tmpdir
+        end
+        @root_fallback = fallback
+        Kernel.warn "[AIDP Logger] Root directory detected - using #{fallback} for logging instead of '#{raw_input}'"
+        return fallback
+      end
+      raw_input
     end
   end
 

data/lib/aidp/safe_directory.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+
+module Aidp
+  # Provides safe directory creation with automatic fallback for permission errors
+  # Used across AIDP components to handle CI environments and restricted filesystems
+  module SafeDirectory
+    # Safely create a directory with fallback to temp/home on permission errors
+    #
+    # @param path [String] The directory path to create
+    # @param component_name [String] Name of the component for logging (default: "AIDP")
+    # @param skip_creation [Boolean] If true, skip directory creation entirely (default: false)
+    # @return [String] The actual directory path (may differ from input if fallback occurred)
+    def safe_mkdir_p(path, component_name: "AIDP", skip_creation: false)
+      return path if skip_creation
+      return path if Dir.exist?(path)
+
+      begin
+        FileUtils.mkdir_p(path)
+        path
+      rescue SystemCallError => e
+        fallback = determine_fallback_path(path)
+        Kernel.warn "[#{component_name}] Cannot create directory #{path}: #{e.class}: #{e.message}"
+        Kernel.warn "[#{component_name}] Using fallback directory: #{fallback}"
+
+        # Try to create fallback directory
+        begin
+          FileUtils.mkdir_p(fallback) unless Dir.exist?(fallback)
+        rescue SystemCallError => e2
+          Kernel.warn "[#{component_name}] Fallback directory creation also failed: #{e2.class}: #{e2.message}"
+        end
+
+        fallback
+      end
+    end
+
+    private
+
+    # Determine a fallback directory path when primary creation fails
+    # Tries: $HOME/.aidp -> /tmp/aidp_<basename>
+    #
+    # @param original_path [String] The original path that failed
+    # @return [String] A fallback directory path
+    def determine_fallback_path(original_path)
+      # Extract meaningful name from path (e.g., ".aidp/jobs" -> "aidp_jobs")
+      base_name = extract_base_name(original_path)
+
+      # Try home directory first
+      begin
+        home = Dir.home
+        if home && !home.empty? && File.writable?(home)
+          return File.join(home, base_name)
+        end
+      rescue
+        # Ignore home directory errors, fall through to temp
+      end
+
+      # Fall back to temp directory
+      File.join(Dir.tmpdir, base_name)
+    end
+
+    # Extract a meaningful base name from a path for fallback naming
+    #
+    # @param path [String] The original path
+    # @return [String] A sanitized base name
+    def extract_base_name(path)
+      # Handle paths like "/project/.aidp/jobs" -> "aidp_jobs"
+      # or "/.aidp" -> ".aidp"
+      parts = path.split(File::SEPARATOR).reject(&:empty?)
+
+      if parts.include?(".aidp")
+        # If path contains .aidp, use .aidp and subdirectory
+        idx = parts.index(".aidp")
+        if idx && parts[idx + 1]
+          return "aidp_#{parts[idx + 1]}"
+        else
+          return ".aidp"
+        end
+      end
+
+      # Fallback: use last directory name
+      parts.last || "aidp_storage"
+    end
+  end
+end

data/lib/aidp/setup/devcontainer/backup_manager.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "time"
+
+module Aidp
+  module Setup
+    module Devcontainer
+      # Manages backups of devcontainer.json before modifications
+      class BackupManager
+        class BackupError < StandardError; end
+
+        def initialize(project_dir, clock: Time)
+          @project_dir = project_dir
+          @backup_dir = File.join(project_dir, ".aidp", "backups", "devcontainer")
+          @clock = clock
+        end
+
+        # Create a backup of the devcontainer file
+        # @param source_path [String] Path to devcontainer.json to backup
+        # @param metadata [Hash] Optional metadata to store with backup
+        # @return [String] Path to backup file
+        def create_backup(source_path, metadata = {})
+          unless File.exist?(source_path)
+            raise BackupError, "Source file does not exist: #{source_path}"
+          end
+
+          ensure_backup_directory_exists
+
+          timestamp = current_time.utc.strftime("%Y%m%d_%H%M%S")
+          backup_filename = "devcontainer-#{timestamp}.json"
+          backup_path = File.join(@backup_dir, backup_filename)
+
+          FileUtils.cp(source_path, backup_path)
+
+          # Create metadata file
+          if metadata.any?
+            metadata_path = "#{backup_path}.meta"
+            File.write(metadata_path, JSON.pretty_generate(metadata))
+          end
+
+          Aidp.log_info("backup_manager", "created backup",
+            source: source_path,
+            backup: backup_path)
+
+          backup_path
+        rescue => e
+          raise BackupError, "Failed to create backup: #{e.message}"
+        end
+
+        # List all available backups
+        # @return [Array<Hash>] Array of backup info hashes
+        def list_backups
+          return [] unless File.directory?(@backup_dir)
+
+          Dir.glob(File.join(@backup_dir, "devcontainer-*.json"))
+            .reject { |f| f.end_with?(".meta") }
+            .map { |path| backup_info(path) }
+            .sort_by { |info| info[:timestamp] }
+            .reverse
+        end
+
+        # Restore a backup to the specified location
+        # @param backup_path [String] Path to backup file
+        # @param target_path [String] Where to restore the backup
+        # @param create_backup [Boolean] Create backup of target before restoring
+        # @return [Boolean] true if successful
+        def restore_backup(backup_path, target_path, create_backup: true)
+          unless File.exist?(backup_path)
+            raise BackupError, "Backup file does not exist: #{backup_path}"
+          end
+
+          # Backup current file before restoring
+          if create_backup && File.exist?(target_path)
+            create_backup(target_path, {
+              reason: "pre_restore",
+              restoring_from: backup_path
+            })
+          end
+
+          FileUtils.mkdir_p(File.dirname(target_path))
+          FileUtils.cp(backup_path, target_path)
+
+          Aidp.log_info("backup_manager", "restored backup",
+            backup: backup_path,
+            target: target_path)
+
+          true
+        rescue => e
+          raise BackupError, "Failed to restore backup: #{e.message}"
+        end
+
+        # Delete old backups, keeping only the N most recent
+        # @param keep_count [Integer] Number of backups to keep
+        # @return [Integer] Number of backups deleted
+        def cleanup_old_backups(keep_count = 10)
+          backups = list_backups
+          return 0 if backups.size <= keep_count
+
+          to_delete = backups[keep_count..]
+          deleted_count = 0
+
+          to_delete.each do |backup|
+            File.delete(backup[:path]) if File.exist?(backup[:path])
+
+            metadata_path = "#{backup[:path]}.meta"
+            File.delete(metadata_path) if File.exist?(metadata_path)
+
+            deleted_count += 1
+          end
+
+          Aidp.log_info("backup_manager", "cleaned up old backups",
+            deleted: deleted_count,
+            kept: keep_count)
+
+          deleted_count
+        end
+
+        # Get the most recent backup
+        # @return [Hash, nil] Backup info or nil if no backups
+        def latest_backup
+          list_backups.first
+        end
+
+        # Calculate total size of all backups
+        # @return [Integer] Total size in bytes
+        def total_backup_size
+          return 0 unless File.directory?(@backup_dir)
+
+          Dir.glob(File.join(@backup_dir, "devcontainer-*.{json,meta}"))
+            .sum { |f| File.size(f) }
+        end
+
+        private
+
+        def ensure_backup_directory_exists
+          FileUtils.mkdir_p(@backup_dir) unless File.directory?(@backup_dir)
+        end
+
+        def backup_info(path)
+          filename = File.basename(path)
+          timestamp_str = filename[/\d{8}_\d{6}/]
+
+          info = {
+            path: path,
+            filename: filename,
+            size: File.size(path),
+            created_at: File.mtime(path),
+            timestamp: parse_timestamp(timestamp_str)
+          }
+
+          # Load metadata if available
+          metadata_path = "#{path}.meta"
+          if File.exist?(metadata_path)
+            begin
+              metadata = JSON.parse(File.read(metadata_path))
+              info[:metadata] = metadata
+            rescue JSON::ParserError
+              # Ignore invalid metadata
+            end
+          end
+
+          info
+        end
+
+        def parse_timestamp(timestamp_str)
+          return current_time if timestamp_str.nil?
+
+          Time.strptime(timestamp_str, "%Y%m%d_%H%M%S")
+        rescue ArgumentError
+          current_time
+        end
+
+        attr_reader :clock
+
+        def current_time
+          clock.respond_to?(:call) ? clock.call : clock.now
+        end
+      end
+    end
+  end
+end