aidp 0.28.0 → 0.29.0

data/lib/aidp/metadata/compiler.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../errors"
+require_relative "scanner"
+require_relative "validator"
+
+module Aidp
+  module Metadata
+    # Compiles tool metadata into a cached directory structure
+    #
+    # Aggregates metadata from all tool files, builds indexes, resolves dependencies,
+    # and generates a cached tool_directory.json for fast lookups.
+    #
+    # @example Compiling the tool directory
+    #   compiler = Compiler.new(directories: [".aidp/skills", ".aidp/templates"])
+    #   compiler.compile(output_path: ".aidp/cache/tool_directory.json")
+    class Compiler
+      # Compiled directory structure
+      attr_reader :tools, :indexes, :dependency_graph
+
+      # Initialize compiler
+      #
+      # @param directories [Array<String>] Directories to scan
+      # @param strict [Boolean] Whether to fail on validation errors
+      def initialize(directories: [], strict: false)
+        @directories = Array(directories)
+        @strict = strict
+        @tools = []
+        @indexes = {}
+        @dependency_graph = {}
+      end
+
+      # Compile tool directory
+      #
+      # @param output_path [String] Path to output JSON file
+      # @return [Hash] Compiled directory structure
+      def compile(output_path:)
+        Aidp.log_info("metadata", "Compiling tool directory", directories: @directories, output: output_path)
+
+        # Scan all directories
+        scanner = Scanner.new(@directories)
+        @tools = scanner.scan_all
+
+        # Validate tools
+        validator = Validator.new(@tools)
+        validation_results = validator.validate_all
+
+        # Handle validation failures
+        handle_validation_results(validation_results)
+
+        # Build indexes and graphs
+        build_indexes
+        build_dependency_graph
+
+        # Create directory structure
+        directory = create_directory_structure
+
+        # Write to file
+        write_directory(directory, output_path)
+
+        Aidp.log_info(
+          "metadata",
+          "Compilation complete",
+          tools: @tools.size,
+          output: output_path
+        )
+
+        directory
+      end
+
+      # Build indexes for fast lookups
+      def build_indexes
+        Aidp.log_debug("metadata", "Building indexes")
+
+        @indexes = {
+          by_id: {},
+          by_type: {},
+          by_tag: {},
+          by_work_unit_type: {}
+        }
+
+        @tools.each do |tool|
+          # Index by ID
+          @indexes[:by_id][tool.id] = tool
+
+          # Index by type
+          @indexes[:by_type][tool.type] ||= []
+          @indexes[:by_type][tool.type] << tool
+
+          # Index by tags
+          tool.applies_to.each do |tag|
+            @indexes[:by_tag][tag] ||= []
+            @indexes[:by_tag][tag] << tool
+          end
+
+          # Index by work unit types
+          tool.work_unit_types.each do |wut|
+            @indexes[:by_work_unit_type][wut] ||= []
+            @indexes[:by_work_unit_type][wut] << tool
+          end
+        end
+
+        Aidp.log_debug(
+          "metadata",
+          "Indexes built",
+          types: @indexes[:by_type].keys,
+          tags: @indexes[:by_tag].keys.size,
+          work_unit_types: @indexes[:by_work_unit_type].keys
+        )
+      end
+
+      # Build dependency graph
+      def build_dependency_graph
+        Aidp.log_debug("metadata", "Building dependency graph")
+
+        @dependency_graph = {}
+
+        @tools.each do |tool|
+          @dependency_graph[tool.id] = {
+            dependencies: tool.dependencies,
+            dependents: []
+          }
+        end
+
+        # Build reverse dependencies (dependents)
+        @tools.each do |tool|
+          tool.dependencies.each do |dep_id|
+            next unless @dependency_graph[dep_id]
+
+            @dependency_graph[dep_id][:dependents] << tool.id
+          end
+        end
+
+        Aidp.log_debug(
+          "metadata",
+          "Dependency graph built",
+          nodes: @dependency_graph.size
+        )
+      end
+
+      # Create directory structure for serialization
+      #
+      # @return [Hash] Directory structure
+      def create_directory_structure
+        {
+          version: "1.0.0",
+          compiled_at: Time.now.iso8601,
+          tools: @tools.map(&:to_h),
+          indexes: {
+            by_type: @indexes[:by_type].transform_values { |tools| tools.map(&:id) },
+            by_tag: @indexes[:by_tag].transform_values { |tools| tools.map(&:id) },
+            by_work_unit_type: @indexes[:by_work_unit_type].transform_values { |tools| tools.map(&:id) }
+          },
+          dependency_graph: @dependency_graph,
+          statistics: {
+            total_tools: @tools.size,
+            by_type: @tools.group_by(&:type).transform_values(&:size),
+            total_tags: @indexes[:by_tag].size,
+            total_work_unit_types: @indexes[:by_work_unit_type].size
+          }
+        }
+      end
+
+      # Write directory to JSON file
+      #
+      # @param directory [Hash] Directory structure
+      # @param output_path [String] Output file path
+      def write_directory(directory, output_path)
+        # Ensure output directory exists
+        output_dir = File.dirname(output_path)
+        FileUtils.mkdir_p(output_dir) unless Dir.exist?(output_dir)
+
+        # Write with pretty formatting
+        File.write(output_path, JSON.pretty_generate(directory))
+
+        Aidp.log_debug("metadata", "Wrote directory", path: output_path, size: File.size(output_path))
+      end
+
+      # Handle validation results
+      #
+      # @param results [Array<ValidationResult>] Validation results
+      # @raise [Aidp::Errors::ValidationError] if strict mode and errors found
+      def handle_validation_results(results)
+        invalid_results = results.reject(&:valid)
+
+        if invalid_results.any?
+          Aidp.log_warn(
+            "metadata",
+            "Validation errors found",
+            count: invalid_results.size
+          )
+
+          invalid_results.each do |result|
+            Aidp.log_error(
+              "metadata",
+              "Tool validation failed",
+              tool_id: result.tool_id,
+              file: result.file_path,
+              errors: result.errors
+            )
+          end
+
+          if @strict
+            raise Aidp::Errors::ValidationError,
+              "#{invalid_results.size} tool(s) failed validation (strict mode enabled)"
+          end
+
+          # Remove invalid tools from compilation
+          invalid_ids = invalid_results.map(&:tool_id)
+          @tools.reject! { |tool| invalid_ids.include?(tool.id) }
+        end
+
+        # Log warnings
+        results.each do |result|
+          next if result.warnings.empty?
+
+          Aidp.log_warn(
+            "metadata",
+            "Tool validation warnings",
+            tool_id: result.tool_id,
+            file: result.file_path,
+            warnings: result.warnings
+          )
+        end
+      end
+    end
+  end
+end

data/lib/aidp/metadata/parser.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "digest"
+require_relative "../errors"
+require_relative "tool_metadata"
+
+module Aidp
+  module Metadata
+    # Parses tool metadata from markdown files with YAML frontmatter
+    #
+    # Extracts metadata headers from skill, persona, and template files.
+    # Supports both new metadata format and legacy skill format.
+    #
+    # @example Parsing a file
+    #   metadata = Parser.parse_file("/path/to/tool.md", type: "skill")
+    #
+    # @example Parsing with auto-detection
+    #   metadata = Parser.parse_file("/path/to/SKILL.md")
+    class Parser
+      # Parse metadata from a file
+      #
+      # @param file_path [String] Path to .md file
+      # @param type [String, nil] Tool type ("skill", "persona", "template") or nil to auto-detect
+      # @return [ToolMetadata] Parsed metadata
+      # @raise [Aidp::Errors::ValidationError] if file format is invalid
+      def self.parse_file(file_path, type: nil)
+        Aidp.log_debug("metadata", "Parsing file", file: file_path, type: type)
+
+        unless File.exist?(file_path)
+          raise Aidp::Errors::ValidationError, "File not found: #{file_path}"
+        end
+
+        content = File.read(file_path, encoding: "UTF-8")
+        file_hash = compute_file_hash(content)
+
+        # Auto-detect type from filename or path if not specified
+        type ||= detect_type(file_path)
+
+        parse_string(content, source_path: file_path, file_hash: file_hash, type: type)
+      end
+
+      # Parse metadata from a string
+      #
+      # @param content [String] File content with frontmatter
+      # @param source_path [String] Source file path for reference
+      # @param file_hash [String] SHA256 hash of content
+      # @param type [String] Tool type ("skill", "persona", "template")
+      # @return [ToolMetadata] Parsed metadata
+      # @raise [Aidp::Errors::ValidationError] if format is invalid
+      def self.parse_string(content, source_path:, file_hash:, type:)
+        metadata_hash, markdown = parse_frontmatter(content, source_path: source_path)
+
+        # Map legacy skill fields to new metadata schema
+        normalized = normalize_metadata(metadata_hash, type: type)
+
+        ToolMetadata.new(
+          type: type,
+          id: normalized["id"],
+          title: normalized["title"],
+          summary: normalized["summary"],
+          version: normalized["version"],
+          applies_to: normalized["applies_to"] || [],
+          work_unit_types: normalized["work_unit_types"] || [],
+          priority: normalized["priority"] || ToolMetadata::DEFAULT_PRIORITY,
+          capabilities: normalized["capabilities"] || [],
+          dependencies: normalized["dependencies"] || [],
+          experimental: normalized["experimental"] || false,
+          content: markdown,
+          source_path: source_path,
+          file_hash: file_hash
+        )
+      rescue Aidp::Errors::ValidationError => e
+        Aidp.log_error("metadata", "Metadata validation failed", error: e.message, file: source_path)
+        raise
+      end
+
+      # Compute SHA256 hash of file content
+      #
+      # @param content [String] File content
+      # @return [String] SHA256 hex string
+      def self.compute_file_hash(content)
+        Digest::SHA256.hexdigest(content)
+      end
+
+      # Detect tool type from file path
+      #
+      # @param file_path [String] File path
+      # @return [String] Detected type ("skill", "persona", or "template")
+      def self.detect_type(file_path)
+        case file_path
+        when %r{/skills/}
+          "skill"
+        when %r{/personas/}
+          "persona"
+        when /SKILL\.md$/
+          "skill"
+        else
+          "template"
+        end
+      end
+
+      # Parse YAML frontmatter from content
+      #
+      # @param content [String] File content with frontmatter
+      # @param source_path [String] Source path for error messages
+      # @return [Array(Hash, String)] Tuple of [metadata, markdown_content]
+      # @raise [Aidp::Errors::ValidationError] if frontmatter is invalid
+      def self.parse_frontmatter(content, source_path:)
+        # Ensure content is UTF-8 encoded
+        content = content.encode("UTF-8", invalid: :replace, undef: :replace) unless content.encoding == Encoding::UTF_8
+        lines = content.lines
+
+        unless lines.first&.strip == "---"
+          raise Aidp::Errors::ValidationError,
+            "Invalid format: missing YAML frontmatter in #{source_path}"
+        end
+
+        frontmatter_lines = []
+        body_start_index = nil
+
+        lines[1..].each_with_index do |line, index|
+          if line.strip == "---"
+            body_start_index = index + 2
+            break
+          end
+
+          frontmatter_lines << line
+        end
+
+        unless body_start_index
+          raise Aidp::Errors::ValidationError,
+            "Invalid format: missing closing frontmatter delimiter in #{source_path}"
+        end
+
+        markdown_content = lines[body_start_index..]&.join.to_s.strip
+        frontmatter_yaml = frontmatter_lines.join
+
+        begin
+          metadata = YAML.safe_load(frontmatter_yaml, permitted_classes: [Symbol])
+        rescue Psych::SyntaxError => e
+          raise Aidp::Errors::ValidationError,
+            "Invalid YAML frontmatter in #{source_path}: #{e.message}"
+        end
+
+        unless metadata.is_a?(Hash)
+          raise Aidp::Errors::ValidationError,
+            "YAML frontmatter must be a hash in #{source_path}"
+        end
+
+        [metadata, markdown_content]
+      end
+
+      # Normalize metadata from various formats to unified schema
+      #
+      # Handles both legacy skill format and new metadata format.
+      #
+      # @param metadata [Hash] Raw metadata from frontmatter
+      # @param type [String] Tool type
+      # @return [Hash] Normalized metadata
+      def self.normalize_metadata(metadata, type:)
+        normalized = {}
+
+        # Required fields (map from legacy names)
+        normalized["id"] = metadata["id"]
+        normalized["title"] = metadata["title"] || metadata["name"]
+        normalized["summary"] = metadata["summary"] || metadata["description"]
+        normalized["version"] = metadata["version"]
+
+        # Optional fields (new schema)
+        normalized["applies_to"] = extract_applies_to(metadata)
+        normalized["work_unit_types"] = metadata["work_unit_types"] || []
+        normalized["priority"] = metadata["priority"]&.to_i
+        normalized["capabilities"] = metadata["capabilities"] || []
+        normalized["dependencies"] = metadata["dependencies"] || []
+        normalized["experimental"] = metadata["experimental"] || false
+
+        normalized
+      end
+
+      # Extract applies_to tags from various metadata fields
+      #
+      # Combines keywords, tags, expertise areas, etc. into unified applies_to list
+      #
+      # @param metadata [Hash] Raw metadata
+      # @return [Array<String>] Combined applies_to tags
+      def self.extract_applies_to(metadata)
+        applies_to = []
+
+        # New schema
+        applies_to.concat(metadata["applies_to"] || [])
+
+        # Legacy skill schema
+        applies_to.concat(metadata["keywords"] || [])
+        applies_to.concat(metadata["tags"] || [])
+
+        # Flatten and deduplicate
+        applies_to.flatten.compact.uniq
+      end
+
+      private_class_method :parse_frontmatter, :normalize_metadata, :extract_applies_to
+    end
+  end
+end

data/lib/aidp/metadata/query.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+require_relative "tool_metadata"
+require_relative "cache"
+
+module Aidp
+  module Metadata
+    # Query interface for tool-choosing agent
+    #
+    # Provides filtering, ranking, and dependency resolution for tools
+    # based on metadata criteria.
+    #
+    # @example Querying for tools
+    #   query = Query.new(cache: cache)
+    #   tools = query.find_by_tags(["ruby", "testing"])
+    #   ranked = query.rank_by_priority(tools)
+    class Query
+      # Initialize query interface
+      #
+      # @param cache [Cache] Metadata cache instance
+      def initialize(cache:)
+        @cache = cache
+        @directory = nil
+      end
+
+      # Load directory (lazy)
+      #
+      # @return [Hash] Tool directory
+      def directory
+        @directory ||= @cache.load
+      end
+
+      # Reload directory
+      def reload
+        @directory = @cache.reload
+      end
+
+      # Find tool by ID
+      #
+      # @param id [String] Tool ID
+      # @return [Hash, nil] Tool metadata or nil
+      def find_by_id(id)
+        tools = directory["tools"]
+        tools.find { |tool| tool["id"] == id }
+      end
+
+      # Find tools by type
+      #
+      # @param type [String] Tool type ("skill", "persona", "template")
+      # @return [Array<Hash>] Matching tools
+      def find_by_type(type)
+        indexes = directory["indexes"]["by_type"]
+        tool_ids = indexes[type] || []
+        tools_by_ids(tool_ids)
+      end
+
+      # Find tools by applies_to tags
+      #
+      # @param tags [Array<String>] Tags to filter by
+      # @param match_all [Boolean] Whether to match all tags (AND) or any tag (OR)
+      # @return [Array<Hash>] Matching tools
+      def find_by_tags(tags, match_all: false)
+        Aidp.log_debug("metadata", "Finding by tags", tags: tags, match_all: match_all)
+
+        tags = Array(tags).map(&:downcase)
+        indexes = directory["indexes"]["by_tag"]
+
+        if match_all
+          # Find tools that have ALL specified tags
+          tool_ids_sets = tags.map { |tag| indexes[tag] || [] }
+          tool_ids = tool_ids_sets.reduce(&:&) || []
+        else
+          # Find tools that have ANY specified tag
+          tool_ids = tags.flat_map { |tag| indexes[tag] || [] }.uniq
+        end
+
+        tools = tools_by_ids(tool_ids)
+
+        Aidp.log_debug("metadata", "Found tools by tags", count: tools.size)
+
+        tools
+      end
+
+      # Find tools by work unit type
+      #
+      # @param work_unit_type [String] Work unit type (e.g., "implementation", "analysis")
+      # @return [Array<Hash>] Matching tools
+      def find_by_work_unit_type(work_unit_type)
+        Aidp.log_debug("metadata", "Finding by work unit type", type: work_unit_type)
+
+        indexes = directory["indexes"]["by_work_unit_type"]
+        tool_ids = indexes[work_unit_type.downcase] || []
+        tools = tools_by_ids(tool_ids)
+
+        Aidp.log_debug("metadata", "Found tools by work unit type", count: tools.size)
+
+        tools
+      end
+
+      # Filter tools by multiple criteria
+      #
+      # @param type [String, nil] Tool type filter
+      # @param tags [Array<String>, nil] Tag filter
+      # @param work_unit_type [String, nil] Work unit type filter
+      # @param experimental [Boolean, nil] Experimental filter (true/false/nil for all)
+      # @return [Array<Hash>] Filtered tools
+      def filter(type: nil, tags: nil, work_unit_type: nil, experimental: nil)
+        Aidp.log_debug(
+          "metadata",
+          "Filtering tools",
+          type: type,
+          tags: tags,
+          work_unit_type: work_unit_type,
+          experimental: experimental
+        )
+
+        tools = directory["tools"]
+
+        # Filter by type
+        tools = tools.select { |t| t["type"] == type } if type
+
+        # Filter by tags
+        if tags && !tags.empty?
+          tags = Array(tags).map(&:downcase)
+          tools = tools.select do |t|
+            tool_tags = (t["applies_to"] || []).map(&:downcase)
+            (tool_tags & tags).any?
+          end
+        end
+
+        # Filter by work unit type
+        if work_unit_type
+          wut_lower = work_unit_type.downcase
+          tools = tools.select do |t|
+            (t["work_unit_types"] || []).map(&:downcase).include?(wut_lower)
+          end
+        end
+
+        # Filter by experimental flag
+        tools = tools.select { |t| t["experimental"] == experimental } unless experimental.nil?
+
+        Aidp.log_debug("metadata", "Filtered tools", count: tools.size)
+
+        tools
+      end
+
+      # Rank tools by priority (highest first)
+      #
+      # @param tools [Array<Hash>] Tools to rank
+      # @return [Array<Hash>] Ranked tools
+      def rank_by_priority(tools)
+        tools.sort_by { |t| -(t["priority"] || ToolMetadata::DEFAULT_PRIORITY) }
+      end
+
+      # Resolve dependencies for a tool
+      #
+      # Returns all dependencies recursively in dependency order (topological sort)
+      #
+      # @param tool_id [String] Tool ID
+      # @return [Array<String>] Ordered list of dependency IDs
+      # @raise [Aidp::Errors::ValidationError] if circular dependency detected
+      def resolve_dependencies(tool_id)
+        Aidp.log_debug("metadata", "Resolving dependencies", tool_id: tool_id)
+
+        graph = directory["dependency_graph"]
+        resolved = []
+        seen = Set.new
+
+        resolve_recursive(tool_id, graph, resolved, seen)
+
+        Aidp.log_debug("metadata", "Dependencies resolved", tool_id: tool_id, dependencies: resolved)
+
+        resolved
+      end
+
+      # Find dependents of a tool
+      #
+      # Returns all tools that depend on this tool (directly or indirectly)
+      #
+      # @param tool_id [String] Tool ID
+      # @return [Array<String>] List of dependent tool IDs
+      def find_dependents(tool_id)
+        graph = directory["dependency_graph"]
+        return [] unless graph[tool_id]
+
+        graph[tool_id]["dependents"] || []
+      end
+
+      # Get statistics about the tool directory
+      #
+      # @return [Hash] Statistics
+      def statistics
+        directory["statistics"]
+      end
+
+      private
+
+      # Get tools by IDs
+      #
+      # @param tool_ids [Array<String>] Tool IDs
+      # @return [Array<Hash>] Tools
+      def tools_by_ids(tool_ids)
+        all_tools = directory["tools"]
+        tool_ids.map { |id| all_tools.find { |t| t["id"] == id } }.compact
+      end
+
+      # Resolve dependencies recursively (topological sort)
+      #
+      # @param tool_id [String] Tool ID
+      # @param graph [Hash] Dependency graph
+      # @param resolved [Array<String>] Resolved dependencies (output)
+      # @param seen [Set<String>] Seen tools (for cycle detection)
+      # @raise [Aidp::Errors::ValidationError] if circular dependency detected
+      def resolve_recursive(tool_id, graph, resolved, seen)
+        return if resolved.include?(tool_id)
+
+        if seen.include?(tool_id)
+          raise Aidp::Errors::ValidationError, "Circular dependency detected: #{tool_id}"
+        end
+
+        seen.add(tool_id)
+
+        node = graph[tool_id]
+        if node
+          dependencies = node["dependencies"] || []
+          dependencies.each do |dep_id|
+            resolve_recursive(dep_id, graph, resolved, seen)
+          end
+        end
+
+        resolved << tool_id unless resolved.include?(tool_id)
+        seen.delete(tool_id)
+      end
+    end
+  end
+end