ruby-skill-bench 0.1.0

data/lib/skill_bench/services/skill_resolver.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require_relative '../models/skill'
+
+module SkillBench
+  module Services
+    # Resolves a skill identifier to a Skill model instance.
+    # Supports both direct paths (containing '/') and skill names (searched recursively).
+    class SkillResolver
+      # Resolves a skill identifier to a Skill instance.
+      #
+      # @param identifier [String] Skill path or name
+      # @param base_path [String] Base directory for skill discovery (default: 'skills/')
+      # @return [SkillBench::Models::Skill] The resolved skill
+      # @raise [ArgumentError] if skill not found
+      def self.call(identifier, base_path = 'skills/')
+        new(identifier, base_path).call
+      end
+
+      # @param identifier [String] Skill path or name
+      # @param base_path [String] Base directory for skill discovery
+      def initialize(identifier, base_path = 'skills/')
+        @identifier = identifier
+        @base_path = base_path
+      end
+
+      # Resolves the skill identifier.
+      #
+      # @return [SkillBench::Models::Skill] The resolved skill
+      # @raise [ArgumentError] if skill not found
+      def call
+        return resolve_by_path if identifier.include?('/')
+
+        resolve_by_name
+      end
+
+      private
+
+      attr_reader :identifier, :base_path
+
+      # Resolves a skill by direct file path.
+      #
+      # @return [SkillBench::Models::Skill] The resolved skill
+      # @raise [ArgumentError] if skill file not found at path or path escapes project boundary
+      def resolve_by_path
+        normalized_path = identifier.end_with?('SKILL.md') ? File.dirname(identifier) : identifier
+        absolute_path = File.expand_path(normalized_path)
+        cwd = File.expand_path(Dir.pwd)
+        cwd_with_sep = cwd + File::SEPARATOR
+
+        raise(ArgumentError, "Skill path escapes project boundary: #{identifier}") unless absolute_path == cwd || absolute_path.start_with?(cwd_with_sep)
+
+        skill_md = File.join(normalized_path, 'SKILL.md')
+
+        return Models::Skill.new(name: File.basename(normalized_path), path: normalized_path) if File.exist?(skill_md)
+
+        raise(ArgumentError, "Skill not found: #{identifier}")
+      end
+
+      # Resolves a skill by name using recursive discovery.
+      #
+      # @return [SkillBench::Models::Skill] The resolved skill
+      # @raise [ArgumentError] if no skill with matching name found
+      def resolve_by_name
+        skills = Models::Skill.discover(base_path)
+        matches = skills.select { |skill| skill.name == identifier }
+
+        if matches.empty?
+          raise(ArgumentError, "Skill not found: #{identifier}")
+        elsif matches.size > 1
+          raise(ArgumentError, "Multiple skills found with name '#{identifier}': #{matches.map(&:path).join(', ')}")
+        end
+
+        matches.first
+      end
+    end
+  end
+end

data/lib/skill_bench/services/template_registry/category_data.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    class TemplateRegistry
+      # Value object holding all template data for a single category.
+      CategoryData = Data.define(:requirements, :criteria, :pattern, :code_template)
+
+      REGISTRY = {
+        crud: CategoryData.new(
+          requirements: "- Implement Create, Read, Update, Delete operations\n- Use Service Object pattern with `.call`\n- Include input validation",
+          criteria: { focus: 'data integrity', required_tests: %w[create read update delete] },
+          pattern: 'Service Object implementing Create, Read, Update, Delete operations.',
+          code_template: "class {{skill_name}}\n  def self.call(params)\n    new(params).call\n  end\nend"
+        ),
+        api: CategoryData.new(
+          requirements: "- Implement API client with proper error handling\n- Use Faraday or Net::HTTP\n- Handle authentication and retries",
+          criteria: { focus: 'error handling', required_tests: %w[success failure timeout] },
+          pattern: 'Layered API client with Auth, Client, Fetcher, Builder, and Entity layers.',
+          code_template: "class {{skill_name}}\n  def self.call(endpoint, params = {})\n    new(endpoint, params).call\n  end\nend"
+        ),
+        background_job: CategoryData.new(
+          requirements: "- Implement as an ActiveJob or Sidekiq worker\n- Include retry logic and error handling\n- Ensure idempotency",
+          criteria: { focus: 'reliability', required_tests: %w[perform retry failure] },
+          pattern: 'Background job with retry logic, error handling, and idempotency.',
+          code_template: "class {{skill_name}} < ApplicationJob\n  def perform(*args)\n    # job logic\n  end\nend"
+        ),
+        controller: CategoryData.new(
+          requirements: "- Follow RESTful conventions\n- Use strong parameters\n- Include proper error responses",
+          criteria: { focus: 'REST compliance', required_tests: %w[index show create update destroy] },
+          pattern: 'RESTful controller with strong parameters and proper error responses.',
+          code_template: "class {{skill_name}}Controller < ApplicationController\n  def index; end\n  def show; end\nend"
+        ),
+        model: CategoryData.new(
+          requirements: "- Define validations and associations\n- Add scopes for common queries\n- Include callback hooks where appropriate",
+          criteria: { focus: 'data modeling', required_tests: %w[validations associations scopes] },
+          pattern: 'ActiveRecord model with validations, associations, and scopes.',
+          code_template: "class {{skill_name}} < ApplicationRecord\n  validates :name, presence: true\nend"
+        ),
+        migration: CategoryData.new(
+          requirements: "- Write reversible migration\n- Include index definitions\n- Handle data migration if needed",
+          criteria: { focus: 'reversibility', required_tests: %w[up down] },
+          pattern: 'Reversible database migration with indexes and data handling.',
+          code_template: "class {{skill_name}} < ActiveRecord::Migration[7.1]\n  def change\n    # migration logic\n  end\nend"
+        ),
+        concern: CategoryData.new(
+          requirements: "- Extract shared behavior into a module\n- Use ActiveSupport::Concern\n- Keep interface minimal",
+          criteria: { focus: 'reusability', required_tests: %w[inclusion behavior] },
+          pattern: 'ActiveSupport::Concern extracting shared behavior.',
+          code_template: "module {{skill_name}}\n  extend ActiveSupport::Concern\nend"
+        ),
+        policy: CategoryData.new(
+          requirements: "- Implement authorization checks\n- Follow Pundit or similar patterns\n- Cover all CRUD actions",
+          criteria: { focus: 'authorization', required_tests: %w[permitted denied] },
+          pattern: 'Authorization policy covering all CRUD actions.',
+          code_template: "class {{skill_name}}Policy\n  def initialize(user, record)\n    @user = user\n    @record = record\n  end\nend"
+        ),
+        form_object: CategoryData.new(
+          requirements: "- Encapsulate form logic outside the model\n- Include ActiveModel validations\n- Handle nested attributes",
+          criteria: { focus: 'validation', required_tests: %w[valid invalid submit] },
+          pattern: 'Form object encapsulating validation and persistence logic.',
+          code_template: "class {{skill_name}}\n  include ActiveModel::Model\n  include ActiveModel::Attributes\nend"
+        ),
+        view_component: CategoryData.new(
+          requirements: "- Create a reusable view component\n- Include preview support\n- Add unit tests for rendering",
+          criteria: { focus: 'rendering', required_tests: %w[render slots preview] },
+          pattern: 'Reusable view component with previews and unit tests.',
+          code_template: "class {{skill_name}} < ViewComponent::Base\n  def initialize(title:)\n    @title = title\n  end\nend"
+        )
+      }.freeze
+    end
+  end
+end

data/lib/skill_bench/services/template_registry.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'template_registry/category_data'
+
+module SkillBench
+  module Services
+    # Resolves and renders evaluation templates by type and category.
+    #
+    # Provides a registry of template strings for generating eval scaffolding
+    # (task descriptions, scoring criteria, and skill instructions) across
+    # supported Rails pattern categories. Supports variable interpolation
+    # using +{{variable_name}}+ syntax.
+    #
+    # @example Resolve a task template with variables
+    #   TemplateRegistry.call(:task_md, :crud, skill_name: "UserCreator")
+    #
+    # @example Resolve criteria JSON
+    #   TemplateRegistry.call(:criteria_json, :api)
+    class TemplateRegistry
+      TEMPLATE_TYPES = %i[task_md criteria_json skill_md].freeze
+      CATEGORIES = REGISTRY.keys.freeze
+
+      # @param template_type [Symbol, String] Template type (:task_md, :criteria_json, :skill_md)
+      # @param category [Symbol, String] Category (:crud, :api, :background_job, etc.)
+      # @param variables [Hash{Symbol, String => String}] Variables for interpolation
+      # @return [String] The rendered template content
+      # @raise [ArgumentError] if template_type or category is invalid
+      def self.call(template_type, category, variables = {})
+        new(template_type, category, variables).call
+      end
+
+      # @param template_type [Symbol, String] Template type
+      # @param category [Symbol, String] Category
+      # @param variables [Hash{Symbol, String => String}] Variables for interpolation
+      def initialize(template_type, category, variables = {})
+        @template_type = template_type.to_sym
+        @category = category.to_sym
+        @variables = variables
+      end
+
+      # Resolves the template and applies variable interpolation.
+      #
+      # @return [String] The rendered template content
+      # @raise [ArgumentError] if template_type or category is invalid
+      def call
+        validate_template_type!
+        validate_category!
+
+        interpolate(build_template)
+      end
+
+      private
+
+      attr_reader :template_type, :category, :variables
+
+      def validate_template_type!
+        return if TEMPLATE_TYPES.include?(template_type)
+
+        raise ArgumentError, "Invalid template type: #{template_type}. Valid types: #{TEMPLATE_TYPES.join(', ')}"
+      end
+
+      def validate_category!
+        return if CATEGORIES.include?(category)
+
+        raise ArgumentError, "Invalid category: #{category}. Valid categories: #{CATEGORIES.join(', ')}"
+      end
+
+      def category_data
+        REGISTRY.fetch(category)
+      end
+
+      def build_template
+        case template_type
+        when :task_md then build_task_md
+        when :criteria_json then build_criteria_json
+        when :skill_md then build_skill_md
+        end
+      end
+
+      def interpolate(template)
+        variables.reduce(template.dup) do |result, (key, value)|
+          result.gsub("{{#{key}}}", value.to_s)
+        end
+      end
+
+      def build_task_md
+        <<~MARKDOWN
+          # Task: Implement {{skill_name}} (#{category})
+
+          ## Objective
+
+          Implement a #{category.to_s.tr('_', ' ')} following Rails best practices and the project's established patterns.
+
+          ## Requirements
+
+          #{category_data.requirements}
+
+          ## Acceptance Criteria
+
+          - All tests pass (`bundle exec rake test`)
+          - Code follows project conventions
+          - YARD documentation for all public methods
+          - No rubocop or reek offenses
+        MARKDOWN
+      end
+
+      def build_criteria_json
+        JSON.pretty_generate(
+          category: category.to_s,
+          dimensions: [
+            { name: 'correctness', weight: 30, pass_threshold: 70 },
+            { name: 'adherence',   weight: 25, pass_threshold: 60 },
+            { name: 'quality',     weight: 20, pass_threshold: 60 },
+            { name: 'tests',       weight: 15, pass_threshold: 80 },
+            { name: 'docs',        weight: 10, pass_threshold: 50 }
+          ],
+          minimum_delta: 5,
+          category_specific: category_data.criteria
+        )
+      end
+
+      def build_skill_md
+        <<~MARKDOWN
+          # Skill: {{skill_name}} (#{category})
+
+          ## Pattern
+
+          #{category_data.pattern}
+
+          ## Hard Rules
+
+          1. Follow TDD — write failing test first, then implement.
+          2. Use `.call` class method as entry point (Service Object pattern).
+          3. Each class has one responsibility (SRP).
+          4. YARD documentation on all public methods.
+          5. `rubocop -A` and `reek` must pass.
+
+          ## Template
+
+          ```ruby
+          #{category_data.code_template}
+          ```
+        MARKDOWN
+      end
+    end
+  end
+end

data/lib/skill_bench/task/evaluator.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require_relative 'file_reader'
+require_relative '../execution/context_hydrator'
+require_relative '../agent/react_agent'
+require_relative '../execution/sandbox'
+require_relative '../judge/judge'
+require_relative '../agent/runner'
+require_relative '../execution/source_path_resolver'
+require_relative '../error_logger'
+
+module SkillBench
+  module Task
+    # Evaluates a single task by running baseline and context-hydrated evaluations.
+    # Orchestrates Agent::Runner calls and Judge::Judge scoring.
+    # @deprecated Use {SkillBench::Evaluation::Runner} instead.
+    class Evaluator
+      # Evaluates a single task.
+      #
+      # @param full_eval_path [Pathname] The path to the evaluation directory.
+      # @param base_path [Pathname] The base path for relative file resolution.
+      # @param skill_path [String, nil] Optional override for the source directory.
+      # @param client_params [Hash] Parameters to pass to the LLM client.
+      # @return [Hash] The result of the task evaluation.
+      def self.call(full_eval_path:, base_path:, skill_path: nil, client_params: {})
+        new(full_eval_path:, base_path:, skill_path:, client_params:).call
+      end
+
+      # @param full_eval_path [Pathname] The path to the evaluation directory.
+      # @param base_path [Pathname] The base path for relative file resolution.
+      # @param skill_path [String, nil] Optional override for the source directory.
+      # @param client_params [Hash] Parameters to pass to the LLM client.
+      def initialize(full_eval_path:, base_path:, skill_path:, client_params:)
+        @full_eval_path = full_eval_path
+        @base_path = base_path
+        @skill_path = skill_path
+        @client_params = client_params
+      end
+
+      # Executes the task evaluation.
+      #
+      # @return [Hash] The result of the task evaluation.
+      def call
+        relative_path = @full_eval_path.relative_path_from(@base_path)
+        relative_path_str = relative_path.to_s
+
+        files_result = FileReader.call(@full_eval_path)
+        return files_result unless files_result[:success]
+
+        files_response = files_result[:response]
+        task_content = files_response[:task]
+        criteria_content = files_response[:criteria]
+
+        source_path = Execution::SourcePathResolver.call(
+          eval_folder_path: relative_path_str,
+          skill_path: @skill_path
+        )
+
+        return { success: false, response: { error: { message: 'No source path inferred' } } } unless source_path
+
+        baseline_result, baseline_code_diff = Agent::Runner.call(
+          mode: :baseline,
+          full_eval_path: @full_eval_path,
+          task_content: task_content,
+          client_params: @client_params
+        )
+
+        context_result, context_code_diff = Agent::Runner.call(
+          mode: :context,
+          full_eval_path: @full_eval_path,
+          task_content: task_content,
+          client_params: @client_params,
+          source_path: source_path,
+          base_path: @base_path
+        )
+
+        judge_score = Judge::Judge.call(task_content, criteria_content, baseline_code_diff, context_code_diff, @client_params)
+        return judge_score unless judge_score[:success]
+
+        {
+          path: relative_path_str,
+          baseline: baseline_result,
+          baseline_diff: baseline_code_diff,
+          with_context: context_result,
+          context_diff: context_code_diff,
+          judge_score: judge_score
+        }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Task::Evaluator Error')
+        { success: false, response: { error: { message: "Error evaluating task: #{e.message}" } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/task/file_reader.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require_relative '../error_logger'
+
+module SkillBench
+  module Task
+    # Reads task.md and criteria.json files for an evaluation task.
+    # Returns structured responses following service object contract.
+    class FileReader
+      # Reads the task and criteria files from the given evaluation path.
+      #
+      # @param full_eval_path [Pathname] The path to the evaluation directory.
+      # @return [Hash] with :success [Boolean] and :response containing file contents or error.
+      def self.call(full_eval_path)
+        new(full_eval_path).call
+      end
+
+      # @param full_eval_path [Pathname] The path to the evaluation directory.
+      def initialize(full_eval_path)
+        @full_eval_path = full_eval_path
+      end
+
+      # Reads task.md and criteria.json files.
+      #
+      # @return [Hash] with :success [Boolean] and :response containing file contents or error.
+      def call
+        task_content = read_file('task.md')
+        return task_content unless task_content[:success]
+
+        criteria_content = read_file('criteria.json')
+        return criteria_content unless criteria_content[:success]
+
+        {
+          success: true,
+          response: {
+            task: task_content[:response][:content],
+            criteria: criteria_content[:response][:content]
+          }
+        }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Task::FileReader Error')
+        { success: false, response: { error: { message: "Error reading task files: #{e.message}" } } }
+      end
+
+      private
+
+      # Reads a single file from the evaluation path.
+      #
+      # @param filename [String] The name of the file to read.
+      # @return [Hash] with :success [Boolean] and :response containing content or error.
+      def read_file(filename)
+        file_path = @full_eval_path.join(filename)
+        unless file_path.exist?
+          return {
+            success: false,
+            response: { error: { message: "File not found: #{file_path}" } }
+          }
+        end
+
+        content = File.read(file_path)
+        { success: true, response: { content: content } }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, "Task::FileReader##{filename} Error")
+        { success: false, response: { error: { message: "Error reading #{filename}: #{e.message}" } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/task.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Namespace for the task subsystem.
+  #
+  # The task subsystem manages individual evaluation tasks,
+  # including file reading and task evaluation orchestration.
+  module Task
+  end
+end

data/lib/skill_bench/tools/argument_parser.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  module Tools
+    # Parses JSON arguments for tools, handling format errors gracefully.
+    class ArgumentParser
+      # Parses a JSON string of arguments.
+      #
+      # @param arguments [String] The JSON string to parse.
+      # @return [Hash, String] The parsed arguments hash, or an error message string.
+      def self.call(arguments)
+        JSON.parse(arguments)
+      rescue JSON::ParserError => e
+        "Error executing tool: Invalid JSON format for arguments. Please correct it. Details: #{e.message}"
+      end
+    end
+  end
+end

data/lib/skill_bench/tools/base.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module SkillBench
+  module Tools
+    # Base functionality for tools, providing common utilities like secure path resolution.
+    class Base
+      class << self
+        protected
+
+        # Sanitizes and resolves a relative path against the working directory.
+        # Ensures the resulting path stays within the boundaries of the working directory.
+        #
+        # @param path [String] The relative path to resolve.
+        # @param working_dir_path [Pathname, String] The pathname of the working directory.
+        # @return [Pathname] The fully expanded and secure path.
+        # @raise [ArgumentError] If path is invalid or attempts traversal.
+        def secure_path(path, working_dir_path)
+          validate_input!(path, working_dir_path)
+
+          working_dir = Pathname(working_dir_path).realpath
+          full_path = working_dir.join(path).cleanpath
+          working_dir_str = working_dir.to_s
+
+          # Ensure the path is still within the working directory
+          # We check against the string representation and ensure it's not escaping
+          # by adding the separator to the prefix check.
+          raise ArgumentError, "Path traversal attempt: #{path}" unless inside_dir?(full_path.to_s, working_dir_str)
+
+          verify_symlink_safety!(full_path, working_dir, working_dir_str, path)
+
+          full_path
+        end
+
+        private
+
+        def validate_input!(path, working_dir_path)
+          raise ArgumentError, 'Path must be a string' unless path.is_a?(String)
+          raise ArgumentError, 'Working directory must be provided' unless working_dir_path
+          raise ArgumentError, 'Path cannot be empty' if path.strip.empty?
+          raise ArgumentError, 'Absolute paths are not allowed' if path.start_with?('/')
+        end
+
+        def inside_dir?(path_str, dir_str)
+          path_str == dir_str || path_str.start_with?(dir_str + File::SEPARATOR)
+        end
+
+        def verify_symlink_safety!(full_path, working_dir, working_dir_str, original_path)
+          # Check every component of the path to prevent escaping via intermediate symlinks
+          current = full_path
+          while current != working_dir && current.to_s.length > working_dir_str.length
+            verify_component_safety!(current, working_dir_str, original_path)
+            current = current.dirname
+          end
+        end
+
+        def verify_component_safety!(component, working_dir_str, original_path)
+          is_symlink = component.symlink?
+          return unless component.exist? || is_symlink
+
+          begin
+            real = component.realpath
+            raise ArgumentError, "Symlink escapes sandbox: #{original_path}" unless inside_dir?(real.to_s, working_dir_str)
+          rescue Errno::ENOENT
+            # Re-check symlink status to avoid TOCTOU if the file was replaced between initial check and realpath
+            raise ArgumentError, "Dangling symlink: #{original_path}" if component.symlink?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/tools/dispatcher.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require_relative 'read_file'
+require_relative 'write_file'
+require_relative 'run_command'
+require_relative 'argument_parser'
+
+module SkillBench
+  module Tools
+    # Dispatches tool execution based on the tool name, coordinating parsing and invocation.
+    class Dispatcher
+      # Executes a specified tool with the given arguments within a working directory.
+      #
+      # @param name [String] The name of the tool to execute (e.g., 'read_file').
+      # @param arguments [String] A JSON string containing the arguments for the tool.
+      # @param working_dir [String] The base directory in which the tool should operate.
+      # @param container_id [String, nil] The Docker container ID for isolated execution.
+      # @return tool execution result or raises exception.
+      # @raise [StandardError] when execution or argument parsing fails
+      def self.call(name, arguments, working_dir, container_id = nil)
+        args = ArgumentParser.call(arguments)
+        return args if args.is_a?(Hash) && args[:success] == false
+
+        working_dir_path = Pathname.new(working_dir).expand_path
+
+        execute_tool(name, args, working_dir_path, container_id)
+      rescue StandardError => e
+        log_error(e)
+        raise
+      end
+
+      class << self
+        private
+
+        def log_error(exception)
+          msg = "#{exception.message}\n#{exception.backtrace.first(5).join("\n")}"
+          if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+            Rails.logger.error(msg)
+          elsif !defined?(Minitest)
+            warn("Dispatcher Error: #{msg}")
+          end
+        end
+
+        def execute_tool(name, args, working_dir_path, container_id)
+          path = args['path']
+          case name
+          when 'read_file'
+            ReadFile.call(path, working_dir_path)
+          when 'write_file'
+            WriteFile.call(path, args['content'], working_dir_path)
+          when 'run_command'
+            RunCommand.call(args['command'], working_dir_path, container_id)
+          else
+            raise StandardError, "Unknown tool '#{name}'"
+          end
+        end
+      end
+    end
+  end
+end