aidp 0.28.0 → 0.30.0

data/lib/aidp/harness/thinking_depth_manager.rb

@@ -160,11 +160,53 @@ module Aidp
           if configured_models.any?
             # Use first configured model for this provider and tier
             model_name = configured_models.first
-            Aidp.log_debug("thinking_depth_manager", "Selected model from user config",
-              tier: tier,
-              provider: provider,
-              model: model_name)
-            return [provider, model_name, {}]
+
+            # Check if model is deprecated and try to upgrade
+            require_relative "ruby_llm_registry" unless defined?(Aidp::Harness::RubyLLMRegistry)
+            llm_registry = Aidp::Harness::RubyLLMRegistry.new
+
+            if llm_registry.model_deprecated?(model_name, provider)
+              Aidp.log_warn("thinking_depth_manager", "Configured model is deprecated",
+                tier: tier,
+                provider: provider,
+                model: model_name)
+
+              # Try to find replacement
+              replacement = llm_registry.find_replacement_model(model_name, provider: provider)
+              if replacement
+                Aidp.log_info("thinking_depth_manager", "Auto-upgrading to non-deprecated model",
+                  tier: tier,
+                  provider: provider,
+                  old_model: model_name,
+                  new_model: replacement)
+                model_name = replacement
+              else
+                # Try next model in config list
+                non_deprecated = configured_models.find { |m| !llm_registry.model_deprecated?(m, provider) }
+                if non_deprecated
+                  Aidp.log_info("thinking_depth_manager", "Using alternate configured model",
+                    tier: tier,
+                    provider: provider,
+                    skipped: model_name,
+                    selected: non_deprecated)
+                  model_name = non_deprecated
+                else
+                  Aidp.log_warn("thinking_depth_manager", "All configured models deprecated, falling back to catalog",
+                    tier: tier,
+                    provider: provider)
+                  # Fall through to catalog selection
+                  model_name = nil
+                end
+              end
+            end
+
+            if model_name
+              Aidp.log_debug("thinking_depth_manager", "Selected model from user config",
+                tier: tier,
+                provider: provider,
+                model: model_name)
+              return [provider, model_name, {}]
+            end
           end
 
           # Provider specified but has no models for this tier in config

data/lib/aidp/metadata/cache.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require_relative "../errors"
+require_relative "scanner"
+require_relative "compiler"
+
+module Aidp
+  module Metadata
+    # Manages cached tool directory with automatic invalidation
+    #
+    # Loads compiled tool directory from cache, checks for file changes,
+    # and regenerates cache when needed.
+    #
+    # @example Loading from cache
+    #   cache = Cache.new(
+    #     cache_path: ".aidp/cache/tool_directory.json",
+    #     directories: [".aidp/skills", ".aidp/templates"]
+    #   )
+    #   directory = cache.load
+    class Cache
+      # Default cache TTL (24 hours)
+      DEFAULT_TTL = 86400
+
+      # Initialize cache
+      #
+      # @param cache_path [String] Path to cache file
+      # @param directories [Array<String>] Directories to monitor
+      # @param ttl [Integer] Cache TTL in seconds (default: 24 hours)
+      # @param strict [Boolean] Whether to fail on validation errors
+      def initialize(cache_path:, directories: [], ttl: DEFAULT_TTL, strict: false)
+        @cache_path = cache_path
+        @directories = Array(directories)
+        @ttl = ttl
+        @strict = strict
+        @file_hashes_path = "#{cache_path}.hashes"
+      end
+
+      # Load tool directory from cache or regenerate
+      #
+      # @return [Hash] Tool directory structure
+      def load
+        Aidp.log_debug("metadata", "Loading cache", path: @cache_path)
+
+        if cache_valid?
+          Aidp.log_debug("metadata", "Using cached directory")
+          load_from_cache
+        else
+          Aidp.log_info("metadata", "Cache invalid, regenerating")
+          regenerate
+        end
+      end
+
+      # Regenerate cache from source files
+      #
+      # @return [Hash] Tool directory structure
+      def regenerate
+        Aidp.log_info("metadata", "Regenerating tool directory", directories: @directories)
+
+        # Compile directory
+        compiler = Compiler.new(directories: @directories, strict: @strict)
+        directory = compiler.compile(output_path: @cache_path)
+
+        # Save file hashes for change detection
+        save_file_hashes
+
+        directory
+      end
+
+      # Force reload cache
+      #
+      # @return [Hash] Tool directory structure
+      def reload
+        Aidp.log_info("metadata", "Force reloading cache")
+        regenerate
+      end
+
+      # Check if cache is valid
+      #
+      # @return [Boolean] True if cache exists and is not stale
+      def cache_valid?
+        return false unless File.exist?(@cache_path)
+        return false if cache_expired?
+        return false if files_changed?
+
+        true
+      end
+
+      # Check if cache has expired based on TTL
+      #
+      # @return [Boolean] True if cache is expired
+      def cache_expired?
+        return true unless File.exist?(@cache_path)
+
+        cache_age = Time.now - File.mtime(@cache_path)
+        expired = cache_age > @ttl
+
+        if expired
+          Aidp.log_debug(
+            "metadata",
+            "Cache expired",
+            age_seconds: cache_age.to_i,
+            ttl: @ttl
+          )
+        end
+
+        expired
+      end
+
+      # Check if source files have changed
+      #
+      # @return [Boolean] True if any source files have changed
+      def files_changed?
+        previous_hashes = load_file_hashes
+        current_hashes = compute_current_hashes
+
+        changed = previous_hashes != current_hashes
+
+        if changed
+          Aidp.log_debug(
+            "metadata",
+            "Source files changed",
+            previous_count: previous_hashes.size,
+            current_count: current_hashes.size
+          )
+        end
+
+        changed
+      end
+
+      # Load directory from cache file
+      #
+      # @return [Hash] Cached directory structure
+      # @raise [Aidp::Errors::ConfigurationError] if cache is invalid
+      def load_from_cache
+        content = File.read(@cache_path, encoding: "UTF-8")
+        directory = JSON.parse(content)
+
+        Aidp.log_debug(
+          "metadata",
+          "Loaded from cache",
+          tools: directory["statistics"]["total_tools"],
+          compiled_at: directory["compiled_at"]
+        )
+
+        directory
+      rescue JSON::ParserError => e
+        Aidp.log_error("metadata", "Invalid cache JSON", error: e.message)
+        raise Aidp::Errors::ConfigurationError, "Invalid tool directory cache: #{e.message}"
+      end
+
+      # Compute current file hashes for all source files
+      #
+      # @return [Hash<String, String>] Map of file_path => file_hash
+      def compute_current_hashes
+        hashes = {}
+
+        @directories.each do |dir|
+          next unless Dir.exist?(dir)
+
+          scanner = Scanner.new([dir])
+          md_files = scanner.find_markdown_files(dir)
+
+          md_files.each do |file_path|
+            content = File.read(file_path, encoding: "UTF-8")
+            hashes[file_path] = Parser.compute_file_hash(content)
+          end
+        end
+
+        hashes
+      end
+
+      # Load saved file hashes
+      #
+      # @return [Hash<String, String>] Saved file hashes
+      def load_file_hashes
+        return {} unless File.exist?(@file_hashes_path)
+
+        content = File.read(@file_hashes_path, encoding: "UTF-8")
+        JSON.parse(content)
+      rescue JSON::ParserError
+        Aidp.log_warn("metadata", "Invalid file hashes cache, regenerating")
+        {}
+      end
+
+      # Save current file hashes
+      def save_file_hashes
+        hashes = compute_current_hashes
+
+        # Ensure directory exists
+        dir = File.dirname(@file_hashes_path)
+        FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+
+        File.write(@file_hashes_path, JSON.pretty_generate(hashes))
+
+        Aidp.log_debug("metadata", "Saved file hashes", count: hashes.size, path: @file_hashes_path)
+      end
+    end
+  end
+end

data/lib/aidp/metadata/compiler.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../errors"
+require_relative "scanner"
+require_relative "validator"
+
+module Aidp
+  module Metadata
+    # Compiles tool metadata into a cached directory structure
+    #
+    # Aggregates metadata from all tool files, builds indexes, resolves dependencies,
+    # and generates a cached tool_directory.json for fast lookups.
+    #
+    # @example Compiling the tool directory
+    #   compiler = Compiler.new(directories: [".aidp/skills", ".aidp/templates"])
+    #   compiler.compile(output_path: ".aidp/cache/tool_directory.json")
+    class Compiler
+      # Compiled directory structure
+      attr_reader :tools, :indexes, :dependency_graph
+
+      # Initialize compiler
+      #
+      # @param directories [Array<String>] Directories to scan
+      # @param strict [Boolean] Whether to fail on validation errors
+      def initialize(directories: [], strict: false)
+        @directories = Array(directories)
+        @strict = strict
+        @tools = []
+        @indexes = {}
+        @dependency_graph = {}
+      end
+
+      # Compile tool directory
+      #
+      # @param output_path [String] Path to output JSON file
+      # @return [Hash] Compiled directory structure
+      def compile(output_path:)
+        Aidp.log_info("metadata", "Compiling tool directory", directories: @directories, output: output_path)
+
+        # Scan all directories
+        scanner = Scanner.new(@directories)
+        @tools = scanner.scan_all
+
+        # Validate tools
+        validator = Validator.new(@tools)
+        validation_results = validator.validate_all
+
+        # Handle validation failures
+        handle_validation_results(validation_results)
+
+        # Build indexes and graphs
+        build_indexes
+        build_dependency_graph
+
+        # Create directory structure
+        directory = create_directory_structure
+
+        # Write to file
+        write_directory(directory, output_path)
+
+        Aidp.log_info(
+          "metadata",
+          "Compilation complete",
+          tools: @tools.size,
+          output: output_path
+        )
+
+        directory
+      end
+
+      # Build indexes for fast lookups
+      def build_indexes
+        Aidp.log_debug("metadata", "Building indexes")
+
+        @indexes = {
+          by_id: {},
+          by_type: {},
+          by_tag: {},
+          by_work_unit_type: {}
+        }
+
+        @tools.each do |tool|
+          # Index by ID
+          @indexes[:by_id][tool.id] = tool
+
+          # Index by type
+          @indexes[:by_type][tool.type] ||= []
+          @indexes[:by_type][tool.type] << tool
+
+          # Index by tags
+          tool.applies_to.each do |tag|
+            @indexes[:by_tag][tag] ||= []
+            @indexes[:by_tag][tag] << tool
+          end
+
+          # Index by work unit types
+          tool.work_unit_types.each do |wut|
+            @indexes[:by_work_unit_type][wut] ||= []
+            @indexes[:by_work_unit_type][wut] << tool
+          end
+        end
+
+        Aidp.log_debug(
+          "metadata",
+          "Indexes built",
+          types: @indexes[:by_type].keys,
+          tags: @indexes[:by_tag].keys.size,
+          work_unit_types: @indexes[:by_work_unit_type].keys
+        )
+      end
+
+      # Build dependency graph
+      def build_dependency_graph
+        Aidp.log_debug("metadata", "Building dependency graph")
+
+        @dependency_graph = {}
+
+        @tools.each do |tool|
+          @dependency_graph[tool.id] = {
+            dependencies: tool.dependencies,
+            dependents: []
+          }
+        end
+
+        # Build reverse dependencies (dependents)
+        @tools.each do |tool|
+          tool.dependencies.each do |dep_id|
+            next unless @dependency_graph[dep_id]
+
+            @dependency_graph[dep_id][:dependents] << tool.id
+          end
+        end
+
+        Aidp.log_debug(
+          "metadata",
+          "Dependency graph built",
+          nodes: @dependency_graph.size
+        )
+      end
+
+      # Create directory structure for serialization
+      #
+      # @return [Hash] Directory structure
+      def create_directory_structure
+        {
+          version: "1.0.0",
+          compiled_at: Time.now.iso8601,
+          tools: @tools.map(&:to_h),
+          indexes: {
+            by_type: @indexes[:by_type].transform_values { |tools| tools.map(&:id) },
+            by_tag: @indexes[:by_tag].transform_values { |tools| tools.map(&:id) },
+            by_work_unit_type: @indexes[:by_work_unit_type].transform_values { |tools| tools.map(&:id) }
+          },
+          dependency_graph: @dependency_graph,
+          statistics: {
+            total_tools: @tools.size,
+            by_type: @tools.group_by(&:type).transform_values(&:size),
+            total_tags: @indexes[:by_tag].size,
+            total_work_unit_types: @indexes[:by_work_unit_type].size
+          }
+        }
+      end
+
+      # Write directory to JSON file
+      #
+      # @param directory [Hash] Directory structure
+      # @param output_path [String] Output file path
+      def write_directory(directory, output_path)
+        # Ensure output directory exists
+        output_dir = File.dirname(output_path)
+        FileUtils.mkdir_p(output_dir) unless Dir.exist?(output_dir)
+
+        # Write with pretty formatting
+        File.write(output_path, JSON.pretty_generate(directory))
+
+        Aidp.log_debug("metadata", "Wrote directory", path: output_path, size: File.size(output_path))
+      end
+
+      # Handle validation results
+      #
+      # @param results [Array<ValidationResult>] Validation results
+      # @raise [Aidp::Errors::ValidationError] if strict mode and errors found
+      def handle_validation_results(results)
+        invalid_results = results.reject(&:valid)
+
+        if invalid_results.any?
+          Aidp.log_warn(
+            "metadata",
+            "Validation errors found",
+            count: invalid_results.size
+          )
+
+          invalid_results.each do |result|
+            Aidp.log_error(
+              "metadata",
+              "Tool validation failed",
+              tool_id: result.tool_id,
+              file: result.file_path,
+              errors: result.errors
+            )
+          end
+
+          if @strict
+            raise Aidp::Errors::ValidationError,
+              "#{invalid_results.size} tool(s) failed validation (strict mode enabled)"
+          end
+
+          # Remove invalid tools from compilation
+          invalid_ids = invalid_results.map(&:tool_id)
+          @tools.reject! { |tool| invalid_ids.include?(tool.id) }
+        end
+
+        # Log warnings
+        results.each do |result|
+          next if result.warnings.empty?
+
+          Aidp.log_warn(
+            "metadata",
+            "Tool validation warnings",
+            tool_id: result.tool_id,
+            file: result.file_path,
+            warnings: result.warnings
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aidp/metadata/parser.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "digest"
+require_relative "../errors"
+require_relative "tool_metadata"
+
+module Aidp
+  module Metadata
+    # Parses tool metadata from markdown files with YAML frontmatter
+    #
+    # Extracts metadata headers from skill, persona, and template files.
+    # Supports both new metadata format and legacy skill format.
+    #
+    # @example Parsing a file
+    #   metadata = Parser.parse_file("/path/to/tool.md", type: "skill")
+    #
+    # @example Parsing with auto-detection
+    #   metadata = Parser.parse_file("/path/to/SKILL.md")
+    class Parser
+      # Parse metadata from a file
+      #
+      # @param file_path [String] Path to .md file
+      # @param type [String, nil] Tool type ("skill", "persona", "template") or nil to auto-detect
+      # @return [ToolMetadata] Parsed metadata
+      # @raise [Aidp::Errors::ValidationError] if file format is invalid
+      def self.parse_file(file_path, type: nil)
+        Aidp.log_debug("metadata", "Parsing file", file: file_path, type: type)
+
+        unless File.exist?(file_path)
+          raise Aidp::Errors::ValidationError, "File not found: #{file_path}"
+        end
+
+        content = File.read(file_path, encoding: "UTF-8")
+        file_hash = compute_file_hash(content)
+
+        # Auto-detect type from filename or path if not specified
+        type ||= detect_type(file_path)
+
+        parse_string(content, source_path: file_path, file_hash: file_hash, type: type)
+      end
+
+      # Parse metadata from a string
+      #
+      # @param content [String] File content with frontmatter
+      # @param source_path [String] Source file path for reference
+      # @param file_hash [String] SHA256 hash of content
+      # @param type [String] Tool type ("skill", "persona", "template")
+      # @return [ToolMetadata] Parsed metadata
+      # @raise [Aidp::Errors::ValidationError] if format is invalid
+      def self.parse_string(content, source_path:, file_hash:, type:)
+        metadata_hash, markdown = parse_frontmatter(content, source_path: source_path)
+
+        # Map legacy skill fields to new metadata schema
+        normalized = normalize_metadata(metadata_hash, type: type)
+
+        ToolMetadata.new(
+          type: type,
+          id: normalized["id"],
+          title: normalized["title"],
+          summary: normalized["summary"],
+          version: normalized["version"],
+          applies_to: normalized["applies_to"] || [],
+          work_unit_types: normalized["work_unit_types"] || [],
+          priority: normalized["priority"] || ToolMetadata::DEFAULT_PRIORITY,
+          capabilities: normalized["capabilities"] || [],
+          dependencies: normalized["dependencies"] || [],
+          experimental: normalized["experimental"] || false,
+          content: markdown,
+          source_path: source_path,
+          file_hash: file_hash
+        )
+      rescue Aidp::Errors::ValidationError => e
+        Aidp.log_error("metadata", "Metadata validation failed", error: e.message, file: source_path)
+        raise
+      end
+
+      # Compute SHA256 hash of file content
+      #
+      # @param content [String] File content
+      # @return [String] SHA256 hex string
+      def self.compute_file_hash(content)
+        Digest::SHA256.hexdigest(content)
+      end
+
+      # Detect tool type from file path
+      #
+      # @param file_path [String] File path
+      # @return [String] Detected type ("skill", "persona", or "template")
+      def self.detect_type(file_path)
+        case file_path
+        when %r{/skills/}
+          "skill"
+        when %r{/personas/}
+          "persona"
+        when /SKILL\.md$/
+          "skill"
+        else
+          "template"
+        end
+      end
+
+      # Parse YAML frontmatter from content
+      #
+      # @param content [String] File content with frontmatter
+      # @param source_path [String] Source path for error messages
+      # @return [Array(Hash, String)] Tuple of [metadata, markdown_content]
+      # @raise [Aidp::Errors::ValidationError] if frontmatter is invalid
+      def self.parse_frontmatter(content, source_path:)
+        # Ensure content is UTF-8 encoded
+        content = content.encode("UTF-8", invalid: :replace, undef: :replace) unless content.encoding == Encoding::UTF_8
+        lines = content.lines
+
+        unless lines.first&.strip == "---"
+          raise Aidp::Errors::ValidationError,
+            "Invalid format: missing YAML frontmatter in #{source_path}"
+        end
+
+        frontmatter_lines = []
+        body_start_index = nil
+
+        lines[1..].each_with_index do |line, index|
+          if line.strip == "---"
+            body_start_index = index + 2
+            break
+          end
+
+          frontmatter_lines << line
+        end
+
+        unless body_start_index
+          raise Aidp::Errors::ValidationError,
+            "Invalid format: missing closing frontmatter delimiter in #{source_path}"
+        end
+
+        markdown_content = lines[body_start_index..]&.join.to_s.strip
+        frontmatter_yaml = frontmatter_lines.join
+
+        begin
+          metadata = YAML.safe_load(frontmatter_yaml, permitted_classes: [Symbol])
+        rescue Psych::SyntaxError => e
+          raise Aidp::Errors::ValidationError,
+            "Invalid YAML frontmatter in #{source_path}: #{e.message}"
+        end
+
+        unless metadata.is_a?(Hash)
+          raise Aidp::Errors::ValidationError,
+            "YAML frontmatter must be a hash in #{source_path}"
+        end
+
+        [metadata, markdown_content]
+      end
+
+      # Normalize metadata from various formats to unified schema
+      #
+      # Handles both legacy skill format and new metadata format.
+      #
+      # @param metadata [Hash] Raw metadata from frontmatter
+      # @param type [String] Tool type
+      # @return [Hash] Normalized metadata
+      def self.normalize_metadata(metadata, type:)
+        normalized = {}
+
+        # Required fields (map from legacy names)
+        normalized["id"] = metadata["id"]
+        normalized["title"] = metadata["title"] || metadata["name"]
+        normalized["summary"] = metadata["summary"] || metadata["description"]
+        normalized["version"] = metadata["version"]
+
+        # Optional fields (new schema)
+        normalized["applies_to"] = extract_applies_to(metadata)
+        normalized["work_unit_types"] = metadata["work_unit_types"] || []
+        normalized["priority"] = metadata["priority"]&.to_i
+        normalized["capabilities"] = metadata["capabilities"] || []
+        normalized["dependencies"] = metadata["dependencies"] || []
+        normalized["experimental"] = metadata["experimental"] || false
+
+        normalized
+      end
+
+      # Extract applies_to tags from various metadata fields
+      #
+      # Combines keywords, tags, expertise areas, etc. into unified applies_to list
+      #
+      # @param metadata [Hash] Raw metadata
+      # @return [Array<String>] Combined applies_to tags
+      def self.extract_applies_to(metadata)
+        applies_to = []
+
+        # New schema
+        applies_to.concat(metadata["applies_to"] || [])
+
+        # Legacy skill schema
+        applies_to.concat(metadata["keywords"] || [])
+        applies_to.concat(metadata["tags"] || [])
+
+        # Flatten and deduplicate
+        applies_to.flatten.compact.uniq
+      end
+
+      private_class_method :parse_frontmatter, :normalize_metadata, :extract_applies_to
+    end
+  end
+end