ruby-skill-bench 0.1.0

data/lib/skill_bench/agent/runner.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative '../execution/sandbox'
+require_relative '../execution/context_hydrator'
+require_relative 'react_agent'
+
+module SkillBench
+  module Agent
+    # Responsible for executing a specific scenario (baseline or context-hydrated)
+    # within an isolated sandbox. Handles the system prompt generation and agent execution.
+    class Runner
+      # Executes the agent run scenario.
+      #
+      # @param params [Hash] The configuration parameters for the run.
+      # @option params [Symbol] :mode The mode to run in (`:baseline` or `:context`).
+      # @option params [Pathname] :full_eval_path The path to the evaluation directory.
+      # @option params [String] :task_content The task description.
+      # @option params [Hash] :client_params Parameters for the LLM client.
+      # @option params [String] :source_path Required if mode is `:context`.
+      # @option params [Pathname] :base_path Required if mode is `:context`.
+      # @return [Array<String, String>] The agent's final answer and the git diff.
+      def self.call(params)
+        new(params).call
+      end
+
+      # @param params [Hash] The configuration parameters for the run.
+      def initialize(params)
+        @mode = validate_mode(params.fetch(:mode))
+        @full_eval_path = params.fetch(:full_eval_path)
+        @task_content = params.fetch(:task_content)
+        @client_params = params.fetch(:client_params, {})
+
+        @source_path = params[:source_path]
+        @base_path = params[:base_path]
+      end
+
+      # Runs the evaluation scenario and captures the results.
+      #
+      # @return [Array<String, String>] A tuple containing the final answer and the diff.
+      def call
+        Execution::Sandbox.run(@full_eval_path) do |sandbox|
+          working_dir = sandbox.path
+          agent_result = ReactAgent.call(
+            client_params: @client_params,
+            working_dir: working_dir,
+            container_id: sandbox.container_id,
+            system_prompt: build_system_prompt,
+            initial_prompt: @task_content
+          )
+
+          response = agent_result[:response]
+          final_answer = if agent_result[:success]
+                           response&.dig(:content) || 'Error: Empty response from agent'
+                         else
+                           error_msg = response&.dig(:error, :message) || 'Unknown error'
+                           "Error: #{error_msg}"
+                         end
+          [final_answer, Execution::Sandbox.capture_diff(working_dir)]
+        end
+      end
+
+      private
+
+      # Builds the appropriate system prompt based on the execution mode.
+      #
+      # @return [String] The system prompt for the agent.
+      # @raise [RuntimeError] when context hydration fails in :context mode.
+      def build_system_prompt
+        case @mode
+        when :baseline
+          baseline_system_prompt
+        when :context
+          context_system_prompt
+        end
+      end
+
+      def baseline_system_prompt
+        <<~PROMPT
+          You are an expert Ruby on Rails developer.#{' '}
+          Your job is to read the task, modify the codebase using the tools provided to meet the requirements, and then explain what you did.
+        PROMPT
+      end
+
+      def context_system_prompt
+        hydrator_result = Execution::ContextHydrator.call(source_path: @source_path, base_path: @base_path)
+        raise "Context hydration failed: #{hydrator_result.dig(:response, :error, :message)}" unless hydrator_result[:success]
+
+        context_xml = hydrator_result[:response][:context]
+
+        <<~PROMPT
+          You are an expert Ruby on Rails developer.
+          You have access to specific skill files wrapped in <agent_context> tags.
+          Use these skills exactly as instructed to solve the user's task.
+          Modify the codebase using the tools provided to meet the requirements, and then explain what you did.
+
+          #{context_xml}
+        PROMPT
+      end
+
+      def validate_mode(mode)
+        allowed = %i[baseline context]
+        return mode if allowed.include?(mode)
+
+        raise ArgumentError, "Invalid mode: #{mode.inspect}. Allowed: #{allowed.join(', ')}"
+      end
+    end
+  end
+end

data/lib/skill_bench/agent/summary.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Agent
+    # Value object capturing sandbox execution metadata.
+    #
+    # Holds files changed, commands run, and agent reasoning extracted
+    # from an evaluation run for delivery to the judge.
+    class Summary
+      attr_reader :files_changed, :commands_run, :agent_reasoning
+
+      # Creates an Agent::Summary from execution data.
+      #
+      # @param files_changed [Array<String>] List of file paths modified.
+      # @param commands_run [Array<String>] List of shell commands executed.
+      # @param agent_reasoning [String] Excerpt of agent reasoning.
+      # @return [Hash] Service response with agent_summary or error.
+      def self.call(files_changed: [], commands_run: [], agent_reasoning: '')
+        new(files_changed:, commands_run:, agent_reasoning:).call
+      end
+
+      # @param files_changed [Array<String>] Modified file paths.
+      # @param commands_run [Array<String>] Executed commands.
+      # @param agent_reasoning [String] Agent reasoning excerpt.
+      def initialize(files_changed:, commands_run:, agent_reasoning:)
+        @files_changed = files_changed
+        @commands_run = commands_run
+        @agent_reasoning = agent_reasoning
+      end
+
+      # Returns the agent summary in the service response format.
+      #
+      # @return [Hash] Service response with agent_summary.
+      def call
+        { success: true, response: { agent_summary: self } }
+      end
+    end
+  end
+end

data/lib/skill_bench/agent.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Namespace for the agent subsystem.
+  #
+  # The agent subsystem executes AI agents within isolated sandboxes,
+  # following the ReAct (Reasoning and Acting) loop pattern.
+  module Agent
+  end
+end

data/lib/skill_bench/cli/eval/eval_command_registry.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Cli
+    module Eval
+      # Registry for eval command handlers
+      class EvalCommandRegistry
+        # @api private
+        # Maps eval action names to their handler classes
+        COMMANDS = {
+          'new' => NewEvalCommand,
+          'generate' => GenerateEvalCommand,
+          'help' => HelpEvalCommand
+        }.freeze
+
+        # Gets command class for action
+        #
+        # @param action [String] Command action name
+        # @return [Class<BaseEvalCommand>, nil] Command class or nil if not found
+        def self.get_command(action)
+          return COMMANDS['help'] if action.nil? || %w[-h --help help].include?(action)
+
+          COMMANDS[action]
+        end
+
+        # Lists all available actions
+        #
+        # @return [Array<String>] Available action names
+        def self.available_actions
+          COMMANDS.keys
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/cli/eval/eval_commands.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require_relative '../../evaluation/generator'
+require_relative '../../commands/eval_new'
+
+module SkillBench
+  module Cli
+    module Eval
+      # Base class for eval command handlers
+      class BaseEvalCommand
+        # Executes command.
+        #
+        # @param argv [Array<String>] Command line arguments
+        # @return [Integer] Exit code
+        # @raise [NotImplementedError] always — subclasses must override
+        def call(argv)
+          raise NotImplementedError, 'Subclasses must implement #call'
+        end
+
+        protected
+
+        # Wraps a command block with standard rescue handling for HelpRequested
+        # and generic StandardError.
+        #
+        # @yield Block that implements the command logic
+        # @return [Integer] Exit code from the block, 0 for help, or 1 on error
+        # @raise [HelpRequested] caught internally, returns 0
+        # @raise [StandardError] caught internally, prints to stderr and returns 1
+        def run_with_rescue
+          yield
+        rescue HelpRequested
+          0
+        rescue StandardError => e
+          warn "Error: #{e.message}"
+          1
+        end
+
+        # Returns error response for missing required argument
+        #
+        # @param message [String] Error message
+        # @return [Integer] Exit code 1
+        def error_missing(message)
+          warn "Error: #{message}"
+          1
+        end
+      end
+
+      # Handles 'eval new' command
+      class NewEvalCommand < BaseEvalCommand
+        # Creates a new evaluation
+        #
+        # @param argv [Array<String>] Command line arguments
+        # @return [Integer] Exit code
+        def call(argv)
+          run_with_rescue do
+            options_parser = NewEvalOptions.new
+            options_parser.parse!(argv)
+
+            name = argv.shift
+            return error_missing('eval name is required') unless name
+
+            Commands::EvalNew.run(name: name, **options_parser.options)
+            puts "Created eval: #{name}"
+            0
+          end
+        end
+      end
+
+      # Handles 'eval generate' command
+      class GenerateEvalCommand < BaseEvalCommand
+        # Generates an evaluation from a skill
+        #
+        # @param argv [Array<String>] Command line arguments
+        # @return [Integer] Exit code
+        def call(argv)
+          run_with_rescue do
+            options_parser = GenerateEvalOptions.new
+            options_parser.parse!(argv)
+
+            skill_name = argv.shift
+            return error_missing('skill name is required') unless skill_name
+
+            eval_name = options_parser.options[:eval_name] || "#{skill_name}-eval"
+            result = Evaluation::Generator.new(skill_name: skill_name, eval_name: eval_name).call
+
+            if result[:success]
+              puts "Generated eval: #{eval_name} from skill: #{skill_name}"
+              0
+            else
+              warn "Error: #{result[:response][:error][:message]}"
+              1
+            end
+          end
+        end
+      end
+
+      # Handles help display for eval commands
+      class HelpEvalCommand < BaseEvalCommand
+        # Shows help information
+        #
+        # @param _argv [Array<String>] Unused arguments
+        # @return [Integer] Exit code 0
+        def call(_argv)
+          puts 'Usage: skill-bench eval new <name> [options]'
+          puts '  --runtime TYPE  rails, ruby, etc. (default: ruby)'
+          puts 'Usage: skill-bench eval generate <skill-name> [--name <eval-name>]'
+          0
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/cli/eval/eval_options.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module SkillBench
+  module Cli
+    module Eval
+      # Base class for eval command option parsing
+      class BaseEvalOptions
+        attr_reader :options, :parser
+
+        def initialize
+          @options = default_options
+          @parser = create_parser
+        end
+
+        # Parses command line arguments
+        #
+        # @param argv [Array<String>] Command line arguments
+        # @return [Array<String>] Remaining arguments after parsing options
+        def parse!(argv)
+          parser.parse!(argv)
+        end
+
+        protected
+
+        # Override in subclasses to define default options
+        def default_options
+          {}
+        end
+
+        # Override in subclasses to configure OptionParser
+        def create_parser
+          OptionParser.new
+        end
+      end
+
+      # Options parser for 'eval new' command
+      class NewEvalOptions < BaseEvalOptions
+        protected
+
+        def default_options
+          { runtime: 'ruby' }
+        end
+
+        def create_parser
+          OptionParser.new do |opts|
+            opts.banner = 'Usage: skill-bench eval new <name> [options]'
+            opts.on('--runtime TYPE', 'rails, ruby, etc.') { |v| @options[:runtime] = v }
+            opts.on('-h', '--help', 'Prints this help') do
+              puts opts
+              raise HelpRequested
+            end
+          end
+        end
+      end
+
+      # Options parser for 'eval generate' command
+      class GenerateEvalOptions < BaseEvalOptions
+        protected
+
+        def create_parser
+          OptionParser.new do |opts|
+            opts.banner = 'Usage: skill-bench eval generate <skill-name> [options]'
+            opts.on('--name NAME', 'Name for generated eval') { |v| @options[:eval_name] = v }
+            opts.on('-h', '--help', 'Prints this help') do
+              puts opts
+              raise HelpRequested
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/cli/eval_command.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative 'eval/eval_options'
+require_relative 'eval/eval_commands'
+require_relative 'eval/eval_command_registry'
+
+module SkillBench
+  module Cli
+    # Handles the `skill-bench eval` subcommand.
+    # Dispatches to appropriate command handlers.
+    class EvalCommand
+      # Parses argv and executes eval command.
+      #
+      # @param argv [Array<String>] Raw CLI arguments
+      # @return [Integer] Exit code
+      def self.call(argv)
+        new(argv).call
+      end
+
+      # @param argv [Array<String>] Raw CLI arguments
+      def initialize(argv)
+        @argv = argv
+      end
+
+      # Dispatches to appropriate eval action.
+      #
+      def call
+        action = @argv.shift
+        command_class = Eval::EvalCommandRegistry.get_command(action)
+
+        if command_class
+          command_class.new.call(@argv)
+        else
+          warn "Unknown eval action: #{action}"
+          1
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/cli/help_printer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Cli
+    # Prints the CLI help/usage message.
+    class HelpPrinter
+      # Prints the help message and returns exit code 0.
+      #
+      # @return [Integer] Exit code (always 0)
+      def self.call
+        providers = SkillBench::Clients::ProviderSchemas.names.map { |name| "--#{name}" }.join(', ')
+
+        puts <<~USAGE
+          Usage: skill-bench <subcommand> [options]
+
+          Subcommands:
+            init --<provider> [--force]
+              Generate configuration file
+              Providers: #{providers}
+              --force    Overwrite existing config file
+
+            run <eval> --skill <name> [--skill <name>] [--format FORMAT]
+              Run an evaluation
+              --skill    Skill to use (can be specified multiple times)
+              --format   Output format: human, json, junit (default: human)
+
+            skill new <name> [--mode MODE] [--template TYPE]
+              Create a new skill
+              --mode     simple, advanced, or rails (default: simple)
+              --template service_object, concern, active_record_model (default: service_object)
+
+            eval new <name> [--runtime TYPE]
+              Create a new eval
+              --runtime  rails, ruby, etc. (default: ruby)
+
+            eval generate <skill-name> [--name <eval-name>]
+              Auto-generate an eval from a skill
+              --name     Name for the generated eval (optional)
+
+          Global Options:
+            -h, --help        Show this help message
+        USAGE
+        0
+      end
+    end
+  end
+end

data/lib/skill_bench/cli/init_command.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module SkillBench
+  module Cli
+    # Handles the `skill-bench init` subcommand.
+    # Parses options and delegates to Commands::Init.
+    class InitCommand
+      #
+      # @param argv [Array<String>] Raw CLI arguments
+      # @return [Integer] Exit code
+      def self.call(argv)
+        new(argv).call
+      end
+
+      # @param argv [Array<String>] Raw CLI arguments
+      def initialize(argv)
+        @argv = argv
+      end
+
+      # Parses options and runs init.
+      #
+      # @return [Integer] Exit code
+      def call
+        options = { force: false, provider: nil }
+        parser = build_parser(options)
+        parser.parse!(@argv)
+
+        return error_missing_provider unless options[:provider]
+
+        Commands::Init.run(**options)
+        puts "Created #{SkillBench::Config::CONFIG_FILENAME}"
+        0
+      rescue SkillBench::HelpRequested
+        0
+      rescue StandardError => e
+        warn "Error: #{e.message}"
+        1
+      end
+
+      private
+
+      def build_parser(options)
+        OptionParser.new do |opts|
+          opts.banner = 'Usage: skill-bench init --<provider> [options]'
+          register_provider_options(opts, options)
+          opts.on('--force', 'Overwrite existing config file') { options[:force] = true }
+          opts.on('-h', '--help', 'Prints this help') do
+            puts opts
+            raise SkillBench::HelpRequested
+          end
+        end
+      end
+
+      def register_provider_options(parser, options)
+        SkillBench::Clients::ProviderSchemas.names.each do |name|
+          parser.on("--#{name}", "Generate config for #{name.to_s.capitalize}") { options[:provider] = name }
+        end
+      end
+
+      def error_missing_provider
+        providers = SkillBench::Clients::ProviderSchemas.names.map { |provider_name| "--#{provider_name}" }.join(', ')
+        warn "Error: provider is required. Use one of: #{providers}"
+        1
+      end
+    end
+  end
+end

data/lib/skill_bench/cli/result_printer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative '../output_formatter'
+
+module SkillBench
+  module Cli
+    # Prints the result of a `skill-bench run` command.
+    class ResultPrinter
+      # Prints the result and returns the appropriate exit code.
+      #
+      # @param result [Hash] Result from ScoringService
+      # @param format [Symbol] Output format (:human, :json, :junit)
+      # @return [Integer] Exit code (0 for pass, 1 for fail)
+      def self.call(result, format: :human)
+        puts OutputFormatter.format(result, format: format)
+        OutputFormatter.exit_code(result)
+      end
+    end
+  end
+end

data/lib/skill_bench/cli/run_command.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module SkillBench
+  module Cli
+    # Handles the `skill-bench run` subcommand.
+    # Parses options and delegates to Commands::Run.
+    class RunCommand
+      #
+      # @param argv [Array<String>] Raw CLI arguments
+      # @return [Integer] Exit code
+      def self.call(argv)
+        new(argv).call
+      end
+
+      # @param argv [Array<String>] Raw CLI arguments
+      def initialize(argv)
+        @argv = argv
+      end
+
+      # Parses options and runs the eval.
+      #
+      # @return [Integer] Exit code
+      def call
+        options = { skill_names: [] }
+        parser = build_parser(options)
+        parser.parse!(@argv)
+
+        eval_name = @argv.shift
+        return error_missing_eval unless eval_name
+        return error_missing_skill if options[:skill_names].empty?
+
+        options[:eval_name] = eval_name
+        exec_options = options.reject { |key| key == :format }
+        result = Commands::Run.run(**exec_options)
+        ResultPrinter.call(result, format: options[:format] || :human)
+      rescue HelpRequested
+        0
+      rescue StandardError => e
+        warn "Error: #{e.message}"
+        1
+      end
+
+      private
+
+      def build_parser(options)
+        OptionParser.new do |opts|
+          opts.banner = 'Usage: skill-bench run <eval> [options]'
+          opts.on('--skill NAME', 'Skill to use (can be specified multiple times)') { |v| options[:skill_names] << v }
+          opts.on('--format FORMAT', 'Output format (human, json, junit)') { |v| options[:format] = v.to_sym }
+          opts.on('-h', '--help', 'Prints this help') do
+            puts opts
+            raise SkillBench::HelpRequested
+          end
+        end
+      end
+
+      def error_missing_eval
+        warn 'Error: eval name is required'
+        warn 'Usage: skill-bench run <eval> --skill <name>'
+        1
+      end
+
+      def error_missing_skill
+        warn 'Error: skill name is required'
+        warn 'Usage: skill-bench run <eval> --skill <name>'
+        1
+      end
+    end
+  end
+end