aidp 0.40.0 → 0.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2414ec5ab6a21100ba8ccb1d6ae8ddc6bb137a5403b069ff941707c17c9ce9fa
-  data.tar.gz: fb0401b3daf671719690b5c5347418eab335e8090dac38ebe08aef43a921430d
+  metadata.gz: f44b47ad0d31015794ec7b2286dc659e85061eb3e6dad8c89ae018976e0d9b1d
+  data.tar.gz: 6d8b28a431c3d55e037e32d0e54e583313af4d495982578b72ab3374155323c5
 SHA512:
-  metadata.gz: c033c034fed453eb4daa2108bb653f3da5007e0d49a792d4a3d9b4f976d63de3c796647da98d3131ea4b81d41425b7aea19ab4e94e743b2eb59936eea9dcad59
-  data.tar.gz: 70ebc3d5e2f809e21398321b7b26c315309a849a35ab4be5742d58650a123a4826c69e64c0b9e8fbc0aef7402da7a99a9970c0d49e83e516aab436e57187d974
+  metadata.gz: d2f838a10c7789f69a253858d0d71aee754a24020722a845eba38cd29eb764be247fee615427afe67f13618cec384565c04cf0f25a25ddd2493962df0df3e0af
+  data.tar.gz: 616bf1a6910461b907c5ab99bccb8dddc3312ee7816550d22d31c395b9faecb90e97bd4e5a032ba594a5d78538d9199a0ffbee203a8fe5c08ae8f784ae47b84e

data/lib/aidp/execute/project_knowledge_manager.rb

@@ -0,0 +1,300 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "shellwords"
+
+module Aidp
+  module Execute
+    # Creates and refreshes concise feature/tool notes under docs/
+    class ProjectKnowledgeManager
+      MAX_TOOL_NOTES = 5
+      FEATURE_ROOT_SEGMENTS = %w[app client lib packages server src].freeze
+
+      def initialize(project_dir)
+        @project_dir = project_dir
+      end
+
+      def sync!(step_name:, task_description:, affected_files:, tool_commands:, feature_identifier: nil)
+        sync_feature_note(
+          feature_identifier: feature_identifier,
+          task_description: task_description,
+          affected_files: affected_files
+        )
+        sync_tool_notes(step_name: step_name, affected_files: affected_files, tool_commands: tool_commands)
+      end
+
+      private
+
+      def sync_feature_note(feature_identifier:, task_description:, affected_files:)
+        return if affected_files.empty?
+
+        feature_id = resolve_feature_id(
+          feature_identifier: feature_identifier,
+          affected_files: affected_files
+        )
+        return if feature_id.empty?
+
+        path = File.join(@project_dir, "docs", "features", "#{feature_id}.md")
+        markers = affected_files.map { |file| "<#{feature_id}:#{file}>" }
+        learning = bullet_line("#{Time.now.utc.iso8601}: #{learning_summary(task_description)}")
+
+        content = if File.exist?(path)
+          update_existing_note(File.read(path, encoding: "UTF-8"), markers: markers, learning: learning)
+        else
+          build_feature_note(feature_id: feature_id, markers: markers, learning: learning)
+        end
+
+        write_note(path, content)
+      end
+
+      def sync_tool_notes(step_name:, affected_files:, tool_commands:)
+        tool_commands.first(MAX_TOOL_NOTES).each do |command|
+          tool_id = tool_identifier(command)
+          next if tool_id.empty?
+
+          path = File.join(@project_dir, "docs", "tools", "#{tool_id}.md")
+          commands = [bullet_line("`#{command}`")]
+          markers = affected_files.map { |file| "<#{tool_id}:#{file}>" }
+          learning = bullet_line("#{Time.now.utc.iso8601}: Used `#{command}` during `#{step_name}`.")
+
+          content = if File.exist?(path)
+            update_existing_note(File.read(path, encoding: "UTF-8"), markers: markers, learning: learning, commands: commands)
+          else
+            build_tool_note(commands: commands, tool_id: tool_id, markers: markers, learning: learning)
+          end
+
+          write_note(path, content)
+        end
+      end
+
+      def build_feature_note(feature_id:, markers:, learning:)
+        <<~MARKDOWN
+          # #{humanize(feature_id)}
+
+          ## Paths
+          #{markers.join("\n")}
+
+          ## Purpose
+          - Concise feature context for work touching these paths.
+
+          ## Users
+          - Add the end users or personas when they become clear.
+
+          ## Known Gaps
+          - Add concrete gaps when discovered.
+
+          ## Key Decisions
+          - Add behavior-affecting decisions when they become clear.
+
+          ## Recent Learnings
+          #{learning}
+        MARKDOWN
+      end
+
+      def build_tool_note(commands:, tool_id:, markers:, learning:)
+        paths_section = markers.empty? ? "- Add affected paths as they become known." : markers.join("\n")
+
+        <<~MARKDOWN
+          # #{humanize(tool_id)}
+
+          ## Commands
+          #{commands.join("\n")}
+
+          ## Paths
+          #{paths_section}
+
+          ## Intended Use
+          - Run this tool while validating or updating related paths.
+
+          ## Known Gaps
+          - Add environment quirks or failure modes when discovered.
+
+          ## Recent Learnings
+          #{learning}
+        MARKDOWN
+      end
+
+      def update_existing_note(content, markers:, learning:, commands: [])
+        updated = ensure_section_entries(content, "Commands", commands)
+        updated = ensure_section_entries(updated, "Paths", markers)
+        ensure_section_entries(updated, "Recent Learnings", [learning], normalizer: method(:normalize_learning_entry))
+      end
+
+      def ensure_section_entries(content, section_name, entries, normalizer: nil)
+        return content if entries.empty?
+
+        existing_section = content.match(/(## #{Regexp.escape(section_name)}\n)(.*?)(?=\n## |\z)/m)
+        return append_new_section(content, section_name, entries) unless existing_section
+
+        body = existing_section[2].to_s
+        existing_entries = body.lines.map(&:strip).reject(&:empty?)
+        normalized_existing = normalize_section_entries(existing_entries, normalizer)
+        missing = entries.reject do |entry|
+          normalized_existing.include?(normalize_section_entry(entry, normalizer))
+        end
+        return content if missing.empty?
+
+        replacement = "#{existing_section[1]}#{body.rstrip}\n#{missing.join("\n")}\n"
+        content.sub(existing_section[0], replacement)
+      end
+
+      def append_new_section(content, section_name, entries)
+        [content.rstrip, "", "## #{section_name}", entries.join("\n"), ""].join("\n")
+      end
+
+      def write_note(path, content)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, content)
+      end
+
+      def resolve_feature_id(feature_identifier:, affected_files:)
+        explicit_id = normalize_id(feature_identifier)
+        return explicit_id unless explicit_id.empty?
+
+        derived_id = normalize_id(feature_path_identifier(affected_files))
+        return derived_id unless derived_id.empty?
+
+        ""
+      end
+
+      def feature_path_identifier(affected_files)
+        feature_files = affected_files.select { |file| feature_path?(file) }
+        segments = feature_files.map { |file| trimmed_feature_segments(file) }
+        return "" if segments.empty?
+
+        shared_segments = meaningful_segments(common_path_segments(segments))
+        return shared_segments.join("_") unless shared_segments.empty?
+
+        meaningful_segments(segments.first).join("_")
+      end
+
+      def feature_path?(file)
+        path_segments = file.to_s.split("/")
+        FEATURE_ROOT_SEGMENTS.include?(path_segments.first) && path_segments.length > 1
+      end
+
+      def common_path_segments(all_segments)
+        shared = []
+        max_length = all_segments.map(&:length).min
+
+        max_length.times do |index|
+          segment = all_segments.first[index]
+          break unless all_segments.all? { |segments| segments[index] == segment }
+
+          shared << segment
+        end
+
+        shared
+      end
+
+      def meaningful_segments(path_segments)
+        path_segments.map { |segment| segment.sub(/\.[^.]+\z/, "") }
+          .reject { |segment| generic_path_segment?(segment) }
+          .reject(&:empty?)
+      end
+
+      def trimmed_feature_segments(file)
+        segments = file.to_s.split("/")
+        return segments unless generic_path_segment?(segments.first)
+
+        segments.drop(1)
+      end
+
+      def generic_path_segment?(segment)
+        %w[app client docs lib packages server spec src test].include?(segment)
+      end
+
+      def normalize_id(value)
+        value.to_s.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/\A_|_\z/, "")
+      end
+
+      def learning_summary(task_description)
+        single_line = task_description.to_s.encode("UTF-8", invalid: :replace, undef: :replace)
+          .lines
+          .map(&:strip)
+          .reject(&:empty?)
+          .join(" ")
+          .gsub(/\s+/, " ")
+
+        return "Updated feature context." if single_line.empty?
+        return single_line if single_line.length <= 200
+
+        "#{single_line[0, 197].rstrip}..."
+      end
+
+      def normalize_section_entries(entries, normalizer)
+        entries.map { |entry| normalize_section_entry(entry, normalizer) }
+      end
+
+      def normalize_section_entry(entry, normalizer)
+        return entry unless normalizer
+
+        normalizer.call(entry)
+      end
+
+      def normalize_learning_entry(entry)
+        entry.to_s.sub(/\A-\s*/, "").sub(/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z:\s*/, "")
+      end
+
+      def tool_identifier(command)
+        raw_command = command.to_s.strip
+        tokens = Shellwords.split(raw_command)
+        executable = unwrap_tool_command(strip_env_assignments(tokens))
+        normalize_id(executable)
+      rescue ArgumentError
+        normalize_id(raw_command.split(/\s+/).first)
+      end
+
+      def unwrap_tool_command(tokens)
+        command_tokens = tokens.dup
+
+        loop do
+          return "" if command_tokens.empty?
+
+          case command_tokens.first
+          when "mise"
+            command_tokens = unwrap_mise_exec(command_tokens)
+          when "env", "/usr/bin/env"
+            command_tokens = unwrap_env(command_tokens)
+          when "bundle"
+            return File.basename(command_tokens.first, ".rb") unless command_tokens[1] == "exec"
+
+            command_tokens = command_tokens.drop(2)
+          when "ruby"
+            return File.basename(command_tokens[1], ".rb") if command_tokens[1]
+
+            command_tokens = []
+          else
+            return File.basename(command_tokens.first, ".rb")
+          end
+        end
+      end
+
+      def strip_env_assignments(tokens)
+        tokens.drop_while { |token| token.include?("=") && !token.start_with?("/", "./", "../") }
+      end
+
+      def unwrap_mise_exec(tokens)
+        return tokens unless tokens[1] == "exec"
+
+        separator_index = tokens.index("--")
+        remaining = separator_index ? tokens.drop(separator_index + 1) : tokens.drop(2)
+        strip_env_assignments(remaining)
+      end
+
+      def unwrap_env(tokens)
+        remaining = tokens.drop(1)
+        remaining = remaining.drop_while { |token| token.start_with?("-") }
+        strip_env_assignments(remaining)
+      end
+
+      def humanize(value)
+        value.to_s.split(/[_\s]+/).map(&:capitalize).join(" ")
+      end
+
+      def bullet_line(value)
+        "- #{value}"
+      end
+    end
+  end
+end

data/lib/aidp/execute/work_loop_runner.rb

@@ -8,9 +8,11 @@ require_relative "guard_policy"
 require_relative "work_loop_unit_scheduler"
 require_relative "deterministic_unit"
 require_relative "agent_signal_parser"
+require_relative "project_knowledge_manager"
 require_relative "steps"
 require_relative "../harness/test_runner"
 require_relative "../errors"
+require_relative "../prompts/prompt_template_manager"
 require_relative "../style_guide/selector"
 require_relative "../security"
 
@@ -44,7 +46,7 @@ module Aidp
       # Expose state for testability
       attr_accessor :iteration_count, :step_name, :options, :persistent_tasklist
       attr_reader :project_dir, :current_state, :state_history, :test_runner, :prompt_manager, :checkpoint
-      attr_writer :guard_policy, :prompt_manager, :style_guide_selector
+      attr_writer :guard_policy, :prompt_manager, :prompt_template_manager, :style_guide_selector
 
       MAX_ITERATIONS = 50 # Safety limit
       CHECKPOINT_INTERVAL = 5 # Record checkpoint every N iterations
@@ -60,7 +62,10 @@ module Aidp
         @checkpoint = Checkpoint.new(project_dir)
         @checkpoint_display = CheckpointDisplay.new(prompt: @prompt)
         @guard_policy = GuardPolicy.new(project_dir, config.guards_config)
+        @project_knowledge_manager = options[:project_knowledge_manager] || ProjectKnowledgeManager.new(project_dir)
         @work_context = {}
+        @executed_tool_commands = []
+        @agentic_execution_results = []
         @persistent_tasklist = PersistentTasklist.new(project_dir)
         @iteration_count = 0
         @step_name = nil
@@ -87,6 +92,7 @@ module Aidp
 
         # FIX for issue #391: Initialize prompt evaluator for iteration threshold assessment
         @prompt_evaluator = options[:prompt_evaluator] || PromptEvaluator.new(config)
+        @prompt_template_manager = options[:prompt_template_manager] || Prompts::PromptTemplateManager.new(project_dir: project_dir)
 
         # Initialize security adapter for Rule of Two enforcement
         @security_adapter = options[:security_adapter] || Aidp::Security::WorkLoopAdapter.new(project_dir: project_dir)
@@ -98,6 +104,8 @@ module Aidp
       def execute_step(step_name, step_spec, context = {})
         @step_name = step_name
         @work_context = context
+        @executed_tool_commands = []
+        @agentic_execution_results = []
         @iteration_count = 0
         transition_to(:ready)
 
@@ -570,6 +578,7 @@ module Aidp
             tier: @thinking_depth_manager.current_tier
           }
         )
+        record_agentic_execution_result(agent_result)
 
         requested = AgentSignalParser.extract_next_unit(agent_result[:output])
 
@@ -603,6 +612,7 @@ module Aidp
             tier: @thinking_depth_manager.current_tier
           }
         )
+        record_agentic_execution_result(agent_result)
 
         requested = AgentSignalParser.extract_next_unit(agent_result[:output])
 
@@ -776,11 +786,13 @@ module Aidp
         if @config.respond_to?(:commands) && @config.commands.any?
           # New phase-based execution
           each_unit_results = @test_runner.run_commands_for_phase(:each_unit)
+          record_phase_commands(:each_unit, each_unit_results)
           all_results[:each_unit] = each_unit_results
 
           # Run on_completion commands only if agent marked work complete
           if agent_marked_complete?(agent_result)
             on_completion_results = @test_runner.run_commands_for_phase(:on_completion)
+            record_phase_commands(:on_completion, on_completion_results)
             all_results[:on_completion] = on_completion_results
           else
             all_results[:on_completion] = {
@@ -808,6 +820,7 @@ module Aidp
         return {success: true, output: "", failures: [], required_failures: []} unless @config.respond_to?(:commands)
 
         full_loop_results = @test_runner.run_commands_for_phase(:full_loop)
+        record_phase_commands(:full_loop, full_loop_results)
 
         Aidp.log_debug("work_loop", "ran_full_loop_commands",
           success: full_loop_results[:success],
@@ -819,13 +832,17 @@ module Aidp
       # Run commands using legacy category-based approach (backwards compatibility)
       def run_legacy_category_commands(agent_result)
         test_results = @test_runner.run_tests
+        record_legacy_category_commands(:tests)
         lint_results = @test_runner.run_linters
+        record_legacy_category_commands(:lints)
         build_results = @test_runner.run_builds
+        record_legacy_category_commands(:builds)
         doc_results = @test_runner.run_documentation
+        record_legacy_category_commands(:docs)
 
         # Run formatters only if agent marked work complete (per issue #234)
         formatter_results = if agent_marked_complete?(agent_result)
-          @test_runner.run_formatters
+          @test_runner.run_formatters.tap { record_legacy_category_commands(:formatters) }
         else
           {success: true, output: "Formatters: Skipped (work not complete)", failures: [], required_failures: []}
         end
@@ -957,20 +974,8 @@ module Aidp
 
       # Extract files that will be affected by this work
       def extract_affected_files(context, user_input)
-        files = []
-
-        # From user input (e.g., "update lib/user.rb")
-        user_input&.scan(/[\w\/]+\.rb/)&.each do |file|
-          files << file
-        end
-
-        # From deterministic outputs
-        context[:deterministic_outputs]&.each do |output|
-          if output[:output_path]&.end_with?(".rb")
-            files << output[:output_path]
-          end
-        end
-
+        files = extract_paths_from_text(user_input)
+        files.concat(extract_output_paths(context[:deterministic_outputs]))
         files.uniq
       end
 
@@ -1039,71 +1044,20 @@ module Aidp
       end
 
       def build_initial_prompt_content(template:, prd:, style_guide:, user_input:, step_name:, deterministic_outputs:, previous_agent_summary:, task_description:, additional_context:)
-        parts = []
-
-        parts << "# Work Loop: #{step_name}"
-        parts << ""
-        parts << "## Instructions"
-        parts << "You are working in a work loop. Your responsibilities:"
-        parts << "1. Read this file (`.aidp/PROMPT.md`) to understand what needs to be done"
-        parts << "2. Complete the work described below"
-        parts << "3. **IMPORTANT**: Edit this `.aidp/PROMPT.md` file to:"
-        parts << "   - Remove completed items"
-        parts << "   - Update with current status"
-        parts << "   - Keep it concise (remove unnecessary context)"
-        parts << "   - Mark the step COMPLETE when 100% done"
-        parts << "4. After you finish, tests and linters will run automatically"
-        parts << "5. If tests/linters fail, you'll see the errors in the next iteration"
-        parts << ""
-        parts << "## Completion Criteria"
-        parts << "Mark this step COMPLETE by adding this line to `.aidp/PROMPT.md`:"
-        parts << "```"
-        parts << "STATUS: COMPLETE"
-        parts << "```"
-        parts << ""
-
-        if user_input && !user_input.empty?
-          parts << "## User Input"
-          parts << user_input
-          parts << ""
-        end
-
-        if previous_agent_summary && !previous_agent_summary.empty?
-          parts << "## Previous Agent Summary"
-          parts << previous_agent_summary
-          parts << ""
-        end
-
-        unless deterministic_outputs.empty?
-          parts << "## Recent Deterministic Outputs"
-          deterministic_outputs.each do |entry|
-            parts << "- #{entry[:name]} (status: #{entry[:status]})"
-            parts << "  Output: #{entry[:output_path] || "n/a"}"
-          end
-          parts << ""
-        end
-
-        if style_guide
-          parts << "## LLM Style Guide"
-          parts << style_guide
-          parts << ""
-        end
-
-        if prd
-          parts << "## Product Requirements (PRD)"
-          parts << prd
-          parts << ""
-        end
-
-        parts << "## Task Template"
-        parts << interpolate_task_template(
-          template,
-          task_description: task_description,
-          additional_context: additional_context
+        @prompt_template_manager.render(
+          "work_loop/initial_prompt",
+          STEP_NAME: step_name,
+          USER_INPUT_SECTION: optional_section("## User Input", user_input),
+          PREVIOUS_AGENT_SUMMARY_SECTION: optional_section("## Previous Agent Summary", previous_agent_summary),
+          DETERMINISTIC_OUTPUTS_SECTION: deterministic_outputs_section(deterministic_outputs),
+          STYLE_GUIDE_SECTION: optional_section("## LLM Style Guide", style_guide),
+          PRD_SECTION: optional_section("## Product Requirements (PRD)", prd),
+          TASK_TEMPLATE: interpolate_task_template(
+            template,
+            task_description: task_description,
+            additional_context: additional_context
+          )
         )
-        parts << ""
-
-        parts.join("\n")
       end
 
       def send_to_agent(selected_provider: nil, selected_model: nil)
@@ -1159,6 +1113,7 @@ module Aidp
             execute_block.call
           end
         end
+          .tap { |result| record_agentic_execution_result(result) }
       end
 
       def display_iteration_overview(provider_name, model_name, prompt_length, checks_summary = nil)
@@ -1325,88 +1280,14 @@ module Aidp
 
       # FIX for issue #391: Enhanced work loop header with upfront task filing requirements
       def build_work_loop_header(step_name, iteration)
-        parts = []
-        parts << "# Work Loop: #{step_name} (Iteration #{iteration})"
-        parts << ""
-        parts << "## Instructions"
-        parts << "You are working in a work loop. Your responsibilities:"
-        parts << "1. **FIRST**: File tasks for all work items (see Task Filing section below)"
-        parts << "2. Read the task description below to understand what needs to be done"
-        parts << "3. **Write/edit CODE files** to implement the required changes"
-        parts << "4. Run tests to verify your changes work correctly"
-        parts << "5. Update task status as you complete items"
-        parts << "6. When ALL tasks are complete and tests pass, mark the step COMPLETE"
-        parts << ""
-        parts << "## Important Notes"
-        parts << "- You have full file system access - create and edit files as needed"
-        parts << "- The working directory is: #{@project_dir}"
-        parts << "- After you finish, tests and linters will run automatically"
-        parts << "- If tests/linters fail, you'll see the errors in the next iteration and can fix them"
-        parts << ""
-        parts << "## ⚠️  Code Changes Required"
-        parts << "**IMPORTANT**: This implementation requires actual code changes."
-        parts << "- Documentation-only changes will NOT be accepted as complete"
-        parts << "- Configuration-only changes will NOT be accepted as complete"
-        parts << "- You must modify/create code files (.rb, .py, .js, etc.) to implement the feature/fix"
-        parts << "- Tests should accompany code changes"
-        parts << ""
-
-        if @config.task_completion_required?
-          parts << "## Task Filing (REQUIRED - DO THIS FIRST)"
-          parts << "**CRITICAL**: This work loop requires task tracking. You MUST file tasks before implementation."
-          parts << ""
-          parts << "### Step 1: File Tasks Immediately"
-          parts << "In your FIRST iteration, analyze the requirements and file tasks for ALL work:"
-          parts << ""
-          parts << "```text"
-          parts << "File task: \"Implement [feature/fix description]\" priority: high tags: implementation"
-          parts << "File task: \"Add unit tests for [feature]\" priority: high tags: testing"
-          parts << "File task: \"Add integration tests if needed\" priority: medium tags: testing"
-          parts << "```"
-          parts << ""
-          parts << "### Step 2: Work Through Tasks"
-          parts << "- Pick the highest priority pending task"
-          parts << "- Implement it completely"
-          parts << "- Mark it done: `Update task: task_id status: done`"
-          parts << "- Repeat until all tasks are complete"
-          parts << ""
-          parts << "### Step 3: Complete the Work Loop"
-          parts << "Only after ALL tasks are done:"
-          parts << "- Verify tests pass"
-          parts << "- Add STATUS: COMPLETE to `.aidp/PROMPT.md`"
-          parts << ""
-          parts << "### Task Rules"
-          parts << "- **At least ONE task must be filed** - completion blocked without tasks"
-          parts << "- **At least ONE task must be DONE** - completion blocked if all abandoned"
-          parts << "- **Substantive work required** - doc-only changes rejected"
-          parts << ""
-          parts << "**Important**: Tasks exist due to careful planning. Do NOT abandon tasks due to"
-          parts << "perceived complexity - these factors were considered during planning. Only abandon"
-          parts << "when truly obsolete (requirements changed, duplicate, external blockers)."
-          parts << ""
-          parts << "### Task Filing Examples"
-          parts << "- `File task: \"Implement user authentication\" priority: high tags: security,auth`"
-          parts << "- `File task: \"Add tests for login flow\" priority: medium tags: testing`"
-          parts << "- `File task: \"Update documentation\" priority: low tags: docs`"
-          parts << ""
-          parts << "### Task Status Update Examples"
-          parts << "- `Update task: task_123_abc status: in_progress`"
-          parts << "- `Update task: task_456_def status: done`"
-          parts << "- `Update task: task_789_ghi status: abandoned reason: \"Requirements changed\"`"
-          parts << ""
-        end
-
-        parts << "## Completion Criteria"
-        parts << "Mark this step COMPLETE by adding these lines to `.aidp/PROMPT.md`:"
-        parts << "```"
-        parts << "STATUS: COMPLETE"
-        if @config.task_completion_required?
-          parts << ""
-          parts << "Update task: task_xxx_yyy status: done  # Mark ALL your tasks as done"
-        end
-        parts << "```"
-        parts << ""
-        parts.join("\n")
+        @prompt_template_manager.render(
+          "work_loop/header",
+          STEP_NAME: step_name,
+          ITERATION: iteration,
+          PROJECT_DIR: @project_dir,
+          TASK_FILING_SECTION: task_filing_section,
+          TASK_COMPLETION_LINE: task_completion_line
+        )
       end
 
       def iteration_context_metadata
@@ -1638,10 +1519,283 @@ module Aidp
       end
 
       def archive_and_cleanup
+        sync_project_knowledge
         @prompt_manager.archive(@step_name)
         @prompt_manager.delete
       end
 
+      def sync_project_knowledge
+        affected_files = knowledge_affected_files
+        return if affected_files.empty?
+
+        @project_knowledge_manager.sync!(
+          step_name: @step_name,
+          feature_identifier: knowledge_feature_identifier,
+          task_description: build_task_description(format_user_input(@work_context[:user_input]), @work_context),
+          affected_files: affected_files,
+          tool_commands: knowledge_tool_commands
+        )
+      rescue => e
+        Aidp.logger.warn("work_loop", "project_knowledge_sync_failed",
+          step: @step_name,
+          error: e.message)
+      end
+
+      def knowledge_affected_files
+        files = Array(@work_context[:affected_files])
+        files.concat(extract_affected_files(@work_context, format_user_input(@work_context[:user_input])))
+        files.concat(metadata_affected_files)
+        files.concat(get_changed_files) if files.empty?
+        files.map(&:to_s).select { |path| knowledge_path_candidate?(path) }.uniq
+      end
+
+      def extract_paths_from_text(text)
+        return [] if text.nil?
+
+        text.to_s.split.filter_map do |token|
+          path = trim_path_token(token)
+          path if valid_path_candidate?(path)
+        end
+      end
+
+      def extract_output_paths(outputs)
+        Array(outputs).filter_map do |output|
+          path = output[:output_path].to_s.strip
+          path unless path.empty?
+        end
+      end
+
+      def knowledge_tool_commands
+        metadata_tool_commands.concat(@executed_tool_commands).uniq
+      end
+
+      def record_agentic_execution_result(result)
+        return unless result.is_a?(Hash)
+
+        @agentic_execution_results << result
+      end
+
+      def knowledge_feature_identifier
+        user_input = @work_context[:user_input]
+
+        @work_context[:feature_identifier] ||
+          @work_context[:feature_id] ||
+          extract_feature_identifier(user_input)
+      end
+
+      def record_phase_commands(phase, results)
+        commands_by_name = Array(@config.commands_for_phase(phase)).each_with_object({}) do |entry, indexed|
+          indexed[entry[:name]] = entry
+        end
+        executed = results.fetch(:results_by_command, {}).keys.filter_map do |name|
+          commands_by_name[name]&.fetch(:command, nil)
+        end
+        executed = commands_by_name.values.map { |entry| entry[:command] } if executed.empty? && commands_by_name.any?
+        @executed_tool_commands.concat(executed.compact)
+      end
+
+      def record_legacy_category_commands(category)
+        return unless @test_runner.respond_to?(:planned_commands)
+
+        commands = Array(@test_runner.planned_commands[category]).filter_map do |entry|
+          entry.is_a?(String) ? entry : entry[:command]
+        end
+        @executed_tool_commands.concat(commands)
+      end
+
+      def extract_feature_identifier(user_input)
+        return unless user_input.is_a?(Hash)
+
+        user_input["feature_identifier"] ||
+          user_input[:feature_identifier] ||
+          user_input["feature_id"] ||
+          user_input[:feature_id]
+      end
+
+      def metadata_affected_files
+        metadata_entries = metadata_value_objects
+        files = extract_file_entries(metadata_entries, :affected_files)
+        files.concat(extract_file_entries(metadata_entries, :changed_files))
+        files.concat(extract_file_entries(metadata_entries, :edited_files))
+        files.concat(extract_file_entries(metadata_entries, :files_changed))
+        files.concat(extract_file_entries(metadata_entries, :modified_files))
+        files.concat(extract_file_entries(metadata_entries, :touched_files))
+        files.uniq
+      end
+
+      def metadata_tool_commands
+        metadata_entries = metadata_value_objects
+        commands = extract_command_entries(metadata_entries, :tool_commands)
+        commands.concat(extract_command_entries(metadata_entries, :tool_usage))
+        commands.concat(extract_command_entries(metadata_entries, :tool_calls))
+        commands.concat(extract_command_entries(metadata_entries, :tools_used))
+        commands.concat(extract_command_entries(metadata_entries, :commands))
+        commands.uniq
+      end
+
+      def metadata_value_objects
+        Array(@agentic_execution_results).flat_map do |result|
+          [
+            result,
+            indifferent_hash_value(result, :metadata),
+            indifferent_hash_value(result, :output)
+          ]
+        end.compact.select { |entry| entry.is_a?(Hash) }
+      end
+
+      def extract_file_entries(entries, key)
+        entries.flat_map do |entry|
+          Array(indifferent_hash_value(entry, key)).flat_map do |value|
+            case value
+            when String
+              [value]
+            when Hash
+              candidate = file_candidate_from_hash(value)
+              candidate ? [candidate] : []
+            else
+              []
+            end
+          end
+        end
+      end
+
+      def extract_command_entries(entries, key)
+        entries.flat_map do |entry|
+          Array(indifferent_hash_value(entry, key)).flat_map do |value|
+            case value
+            when String
+              [value]
+            when Hash
+              candidate = command_candidate_from_hash(value)
+              candidate ? [candidate] : []
+            else
+              []
+            end
+          end
+        end.reject(&:empty?)
+      end
+
+      def indifferent_hash_value(hash, key)
+        hash[key] || hash[key.to_s]
+      end
+
+      def file_candidate_from_hash(hash)
+        %i[path file file_path output_path relative_path].each do |key|
+          candidate = indifferent_hash_value(hash, key).to_s.strip
+          return candidate unless candidate.empty?
+        end
+
+        nil
+      end
+
+      def command_candidate_from_hash(hash)
+        %i[command tool name identifier].each do |key|
+          candidate = indifferent_hash_value(hash, key).to_s.strip
+          return candidate unless candidate.empty?
+        end
+
+        nil
+      end
+
+      def trim_path_token(token)
+        text = token.to_s
+        start_index = 0
+        end_index = text.length - 1
+
+        start_index += 1 while start_index <= end_index && !path_character?(text[start_index])
+        end_index -= 1 while end_index >= start_index && !path_character?(text[end_index])
+
+        return "" if start_index > end_index
+
+        text[start_index..end_index]
+      end
+
+      def knowledge_path_candidate?(path)
+        return false if path.empty?
+        return false if internal_generated_path?(path)
+
+        true
+      end
+
+      def internal_generated_path?(path)
+        path == ".aidp" || path.start_with?(".aidp/")
+      end
+
+      def path_character?(char)
+        alphanumeric_character?(char) || ["_", ".", "/", "-"].include?(char)
+      end
+
+      def valid_path_candidate?(path)
+        return false if path.empty?
+        return false if standalone_version_token?(path)
+
+        segments = path.split("/")
+        return false if segments.empty? || segments.any?(&:empty?)
+
+        filename = segments.last
+        return false unless valid_filename_candidate?(filename, path.include?("/"))
+
+        segments.all? { |segment| valid_path_segment?(segment) }
+      end
+
+      def standalone_version_token?(path)
+        return false if path.include?("/")
+
+        components = path.split(".")
+        return false if components.length < 2
+
+        components.all? { |component| version_component?(component) }
+      end
+
+      def version_component?(component)
+        component.match?(/\Av?\d+\z/i)
+      end
+
+      def valid_path_segment?(segment)
+        return false if segment == "." || segment == ".."
+
+        segment.each_char.all? do |char|
+          alphanumeric_character?(char) || ["_", ".", "-"].include?(char)
+        end
+      end
+
+      def valid_filename_candidate?(filename, nested_path)
+        return valid_extension_filename?(filename) if filename.include?(".")
+
+        return true if nested_path
+
+        known_extensionless_filename?(filename)
+      end
+
+      def valid_extension_filename?(filename)
+        name, _separator, extension = filename.rpartition(".")
+        return false if name.to_s.empty? || extension.to_s.empty?
+
+        extension.each_char.all? { |char| alphanumeric_character?(char) }
+      end
+
+      def known_extensionless_filename?(filename)
+        %w[
+          Dockerfile
+          Gemfile
+          Procfile
+          Rakefile
+          Makefile
+          Brewfile
+          Podfile
+          Fastfile
+          Appfile
+          Guardfile
+          Vagrantfile
+        ].include?(filename)
+      end
+
+      def alphanumeric_character?(char)
+        char.between?("a", "z") ||
+          char.between?("A", "Z") ||
+          char.between?("0", "9")
+      end
+
       def load_template(template_name)
         return "" unless template_name
 
@@ -2131,51 +2285,48 @@ module Aidp
 
       # Append task completion requirement to PROMPT.md
       def append_task_requirement_to_prompt(message)
-        task_requirement = []
-
-        task_requirement << "## Task Completion Requirement"
-        task_requirement << ""
-        task_requirement << "**CRITICAL**: #{message}"
-        task_requirement << ""
-        task_requirement << "### How to Complete Tasks"
-        task_requirement << ""
-        task_requirement << "Update task status using these signals in your output:"
-        task_requirement << ""
-        task_requirement << "**Creating tasks:**"
-        task_requirement << "```"
-        task_requirement << 'File task: "Implement feature X" priority: high tags: feature,backend'
-        task_requirement << 'File task: "Add tests for feature X" priority: medium tags: testing'
-        task_requirement << "```"
-        task_requirement << ""
-        task_requirement << "**Updating task status:**"
-        task_requirement << "```"
-        task_requirement << "Update task: task_123_abc status: in_progress"
-        task_requirement << "Update task: task_123_abc status: done"
-        task_requirement << 'Update task: task_456_def status: abandoned reason: "Requirements changed"'
-        task_requirement << "```"
-        task_requirement << ""
-        task_requirement << "**Task states:**"
-        task_requirement << "- ⏳ **pending** - Not started yet"
-        task_requirement << "- 🚧 **in_progress** - Currently working on it"
-        task_requirement << "- ✅ **done** - Completed successfully"
-        task_requirement << "- ❌ **abandoned** - Not doing this (requires reason)"
-        task_requirement << ""
-        task_requirement << "**Completion requirement:**"
-        task_requirement << "All tasks for this session must be marked as DONE or ABANDONED (with reason) before the work loop can complete."
-        task_requirement << ""
-        task_requirement << "**Action Required**: Review the current task list and update status for all tasks."
-        task_requirement << ""
-
         # Append to PROMPT.md - ensure directory exists
-        begin
-          current_prompt = @prompt_manager.read || ""
-          updated_prompt = current_prompt + "\n\n---\n\n" + task_requirement.join("\n")
-          @prompt_manager.write(updated_prompt, step_name: @step_name)
-          display_message("  [TASK_REQ] Added task completion requirement to PROMPT.md", type: :warning)
-        rescue => e
-          Aidp.log_warn("work_loop", "Failed to append task requirement to PROMPT.md", error: e.message)
-          display_message("  [TASK_REQ] Warning: Could not update PROMPT.md: #{e.message}", type: :warning)
+        current_prompt = @prompt_manager.read || ""
+        task_requirement = @prompt_template_manager.render(
+          "work_loop/task_completion_requirement",
+          MESSAGE: message
+        )
+        updated_prompt = current_prompt + "\n\n---\n\n" + task_requirement
+        @prompt_manager.write(updated_prompt, step_name: @step_name)
+        display_message("  [TASK_REQ] Added task completion requirement to PROMPT.md", type: :warning)
+      rescue => e
+        Aidp.log_warn("work_loop", "Failed to append task requirement to PROMPT.md", error: e.message)
+        display_message("  [TASK_REQ] Warning: Could not update PROMPT.md: #{e.message}", type: :warning)
+      end
+
+      def optional_section(title, body)
+        return "" if body.nil? || body.empty?
+
+        [title, body, ""].join("\n")
+      end
+
+      def deterministic_outputs_section(deterministic_outputs)
+        return "" if deterministic_outputs.empty?
+
+        lines = ["## Recent Deterministic Outputs"]
+        deterministic_outputs.each do |entry|
+          lines << "- #{entry[:name]} (status: #{entry[:status]})"
+          lines << "  Output: #{entry[:output_path] || "n/a"}"
         end
+        lines << ""
+        lines.join("\n")
+      end
+
+      def task_filing_section
+        return "" unless @config.task_completion_required?
+
+        @prompt_template_manager.render("work_loop/task_filing")
+      end
+
+      def task_completion_line
+        return "" unless @config.task_completion_required?
+
+        "Update task: task_xxx_yyy status: done  # Mark ALL your tasks as done\n"
       end
 
       # Validate changes against guard policy

data/lib/aidp/prompt_optimization/context_composer.rb

@@ -115,6 +115,8 @@ module Aidp
       def meets_threshold?(fragment, score, thresholds)
         threshold = if fragment.class.name.include?("Fragment") && fragment.respond_to?(:heading)
           thresholds[:style_guide] || 0.75
+        elsif fragment.respond_to?(:linked_paths)
+          thresholds[:knowledge] || 0.72
         elsif fragment.respond_to?(:category)
           thresholds[:templates] || 0.8
         elsif fragment.respond_to?(:file_path) && fragment.respond_to?(:type)
@@ -248,6 +250,8 @@ module Aidp
           case type
           when :style_guide
             item[:fragment].class.name.include?("Fragment") && item[:fragment].respond_to?(:heading)
+          when :project_knowledge
+            item[:fragment].respond_to?(:linked_paths)
           when :template
             item[:fragment].respond_to?(:category) && !item[:fragment].respond_to?(:type)
           when :code
@@ -272,6 +276,7 @@ module Aidp
           over_budget: over_budget?,
           by_type: {
             style_guide: fragments_by_type(:style_guide).count,
+            project_knowledge: fragments_by_type(:project_knowledge).count,
             templates: fragments_by_type(:template).count,
             code: fragments_by_type(:code).count
           }

data/lib/aidp/prompt_optimization/optimizer.rb

@@ -6,6 +6,7 @@ require_relative "source_code_fragmenter"
 require_relative "relevance_scorer"
 require_relative "context_composer"
 require_relative "prompt_builder"
+require_relative "project_knowledge_indexer"
 
 module Aidp
   module PromptOptimization
@@ -38,6 +39,7 @@ module Aidp
         @style_guide_indexer = nil
         @template_indexer = nil
         @fragmenter = nil
+        @project_knowledge_indexer = nil
         @scorer = nil
         @composer = nil
         @builder = nil
@@ -100,6 +102,7 @@ module Aidp
         @style_guide_indexer = nil
         @template_indexer = nil
         @fragmenter = nil
+        @project_knowledge_indexer = nil
         @stats.reset!
       end
 
@@ -127,6 +130,10 @@ module Aidp
         template_indexer.index!
         fragments.concat(template_indexer.templates)
 
+        # Project knowledge fragments
+        project_knowledge_indexer.index!
+        fragments.concat(project_knowledge_indexer.fragments)
+
         # Source code fragments (only for affected files)
         if affected_files && !affected_files.empty?
           code_fragments = fragmenter.fragment_files(affected_files)
@@ -180,6 +187,10 @@ module Aidp
         @fragmenter ||= SourceCodeFragmenter.new(project_dir: @project_dir)
       end
 
+      def project_knowledge_indexer
+        @project_knowledge_indexer ||= ProjectKnowledgeIndexer.new(project_dir: @project_dir)
+      end
+
       # Get or create relevance scorer (cached)
       def scorer
         @scorer ||= RelevanceScorer.new

data/lib/aidp/prompt_optimization/project_knowledge_indexer.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Aidp
+  module PromptOptimization
+    # Indexes concise feature and tool notes stored under docs/
+    #
+    # These notes use inline path markers like <feature_id:lib/file.rb> so
+    # they can be found quickly by ripgrep and loaded for matching files.
+    class ProjectKnowledgeIndexer
+      KNOWLEDGE_DIRS = {
+        feature: File.join("docs", "features"),
+        tool: File.join("docs", "tools")
+      }.freeze
+
+      MARKER_PATTERN = /<([a-z0-9_]+):([^>\n]+)>/
+
+      attr_reader :fragments, :project_dir
+
+      def initialize(project_dir:)
+        @project_dir = project_dir
+        @fragments = []
+      end
+
+      def index!
+        @fragments = []
+
+        KNOWLEDGE_DIRS.each do |note_type, relative_dir|
+          index_directory(note_type, File.join(project_dir, relative_dir))
+        end
+
+        @fragments
+      end
+
+      private
+
+      def index_directory(note_type, directory)
+        return unless Dir.exist?(directory)
+
+        Dir.glob(File.join(directory, "**", "*.md")).sort.each do |file_path|
+          @fragments << parse_file(file_path, note_type)
+        end
+      end
+
+      def parse_file(file_path, note_type)
+        content = File.read(file_path, encoding: "UTF-8")
+        title = extract_title(content, file_path)
+        markers = content.scan(MARKER_PATTERN)
+        note_id = markers.first&.first || File.basename(file_path, ".md")
+
+        ProjectKnowledgeFragment.new(
+          id: note_id,
+          title: title,
+          note_type: note_type,
+          file_path: file_path,
+          content: content,
+          linked_paths: markers.map { |(_, path)| path.strip }.uniq,
+          tags: extract_tags(note_type, content)
+        )
+      end
+
+      def extract_title(content, file_path)
+        heading = content.lines.find { |line| line.start_with?("# ") }
+        return heading.sub(/^#\s+/, "").strip if heading
+
+        File.basename(file_path, ".md").split("_").map(&:capitalize).join(" ")
+      end
+
+      def extract_tags(note_type, content)
+        text = content.downcase
+        tags = [note_type.to_s, "project_knowledge"]
+        tags << "testing" if text.match?(/rspec|rubocop|test|lint|spec/)
+        tags << "documentation" if text.match?(/doc|guide|markdown/)
+        tags << "implementation" if text.match?(/implement|feature|behavior/)
+        tags << "filesystem" if text.match?(/file|path|directory|workspace/)
+        tags << "external_service" if text.match?(/api|provider|service|github|slack/)
+        tags.uniq
+      end
+    end
+
+    class ProjectKnowledgeFragment
+      attr_reader :id, :title, :note_type, :file_path, :content, :linked_paths, :tags
+
+      def initialize(id:, title:, note_type:, file_path:, content:, linked_paths:, tags:)
+        @id = id
+        @title = title
+        @note_type = note_type
+        @file_path = file_path
+        @content = content
+        @linked_paths = linked_paths
+        @tags = tags
+      end
+
+      def estimated_tokens
+        (content.length / 4.0).ceil
+      end
+    end
+  end
+end

data/lib/aidp/prompt_optimization/prompt_builder.rb

@@ -31,11 +31,13 @@ module Aidp
 
         # Group fragments by type
         style_guide_fragments = composition_result.fragments_by_type(:style_guide)
+        project_knowledge_fragments = composition_result.fragments_by_type(:project_knowledge)
         template_fragments = composition_result.fragments_by_type(:template)
         code_fragments = composition_result.fragments_by_type(:code)
 
         # Add relevant sections if they have content
         sections << build_style_guide_section(style_guide_fragments) unless style_guide_fragments.empty?
+        sections << build_project_knowledge_section(project_knowledge_fragments) unless project_knowledge_fragments.empty?
         sections << build_template_section(template_fragments) unless template_fragments.empty?
         sections << build_code_section(code_fragments) unless code_fragments.empty?
 
@@ -139,6 +141,25 @@ module Aidp
         lines.join("\n")
       end
 
+      def build_project_knowledge_section(fragments)
+        lines = ["# Project Knowledge"]
+        lines << ""
+        lines << "Concise notes associated with the affected files:"
+        lines << ""
+
+        fragments.each do |item|
+          fragment = item[:fragment]
+          lines << "## #{fragment.title}"
+          lines << ""
+          lines << "**Type**: #{fragment.note_type}"
+          lines << ""
+          lines << fragment.content
+          lines << ""
+        end
+
+        lines.join("\n")
+      end
+
       # Build code context section
       #
       # @param fragments [Array<Hash>] Code fragments
@@ -198,6 +219,7 @@ module Aidp
         lines << "## Fragments by Type"
         lines << ""
         lines << "- **Style Guide Sections**: #{summary[:by_type][:style_guide]}"
+        lines << "- **Project Knowledge Notes**: #{summary[:by_type][:project_knowledge]}"
         lines << "- **Templates**: #{summary[:by_type][:templates]}"
         lines << "- **Code Fragments**: #{summary[:by_type][:code]}"
         lines << ""

data/lib/aidp/prompt_optimization/relevance_scorer.rb

@@ -105,6 +105,13 @@ module Aidp
       def score_file_location_match(fragment, context)
         return 0.5 unless context.affected_files && !context.affected_files.empty?
 
+        if fragment.respond_to?(:linked_paths) && fragment.linked_paths.any?
+          matches_affected_file = context.affected_files.any? do |affected_file|
+            knowledge_path_match?(fragment.linked_paths, affected_file)
+          end
+          return matches_affected_file ? 1.0 : 0.1
+        end
+
         # Only code fragments have file_path
         return 0.5 unless fragment.respond_to?(:file_path)
 
@@ -114,6 +121,16 @@ module Aidp
         end) ? 1.0 : 0.1
       end
 
+      def knowledge_path_match?(linked_paths, affected_file)
+        linked_paths.any? do |linked_path|
+          normalized_link = linked_path.sub(%r{\A\./}, "")
+          normalized_file = affected_file.to_s.sub(%r{\A\./}, "")
+          normalized_file == normalized_link ||
+            normalized_file.start_with?("#{normalized_link}/") ||
+            normalized_link.start_with?("#{normalized_file}/")
+        end
+      end
+
       # Score based on work loop step
       #
       # @param fragment [Fragment] Fragment to score

data/lib/aidp/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Aidp
-  VERSION = "0.40.0"
+  VERSION = "0.42.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aidp
 version: !ruby/object:Gem::Version
-  version: 0.40.0
+  version: 0.42.0
 platform: ruby
 authors:
 - Bart Agapinan
@@ -407,6 +407,7 @@ files:
 - lib/aidp/execute/interactive_repl.rb
 - lib/aidp/execute/persistent_tasklist.rb
 - lib/aidp/execute/progress.rb
+- lib/aidp/execute/project_knowledge_manager.rb
 - lib/aidp/execute/prompt_evaluator.rb
 - lib/aidp/execute/prompt_manager.rb
 - lib/aidp/execute/repl_macros.rb
@@ -526,6 +527,7 @@ files:
 - lib/aidp/pr_worktree_manager.rb
 - lib/aidp/prompt_optimization/context_composer.rb
 - lib/aidp/prompt_optimization/optimizer.rb
+- lib/aidp/prompt_optimization/project_knowledge_indexer.rb
 - lib/aidp/prompt_optimization/prompt_builder.rb
 - lib/aidp/prompt_optimization/relevance_scorer.rb
 - lib/aidp/prompt_optimization/source_code_fragmenter.rb