ruby-skill-bench 0.1.0 → 1.1.0

data/lib/skill_bench/services/agent_spawner_service.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative '../execution/sandbox'
+require_relative '../agent/react_agent'
+
+module SkillBench
+  module Services
+    # Spawns and executes LLM agents for evaluation.
+    class AgentSpawnerService
+      # Spawns the LLM agent with the given system prompt.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param system_prompt [String] The system prompt for the agent
+      # @param provider [Object] The resolved provider
+      # @param config [Hash, nil] Provider config
+      # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations
+      def self.call(evaluation, system_prompt, provider, config)
+        new(evaluation, system_prompt, provider, config).call
+      end
+
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param system_prompt [String] The system prompt for the agent
+      # @param provider [Object] The resolved provider
+      # @param config [Hash, nil] Provider config
+      def initialize(evaluation, system_prompt, provider, config)
+        @evaluation = evaluation
+        @system_prompt = system_prompt
+        @provider = provider
+        @config = config
+      end
+
+      # Spawns the LLM agent with the given system prompt.
+      #
+      # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations
+      def call
+        return { result: 'mock result', status: :success, iterations: [] } if @provider.name == 'mock'
+
+        client_params = build_client_params
+        max_iterations = @config&.[](:max_iterations) || @config&.[]('max_iterations') || 25
+
+        run_agent(client_params, max_iterations)
+      end
+
+      private
+
+      # Runs the agent in a sandbox with error handling.
+      #
+      # @param client_params [Hash] Client parameters for the agent
+      # @param max_iterations [Integer] Maximum iterations for the agent
+      # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations
+      def run_agent(client_params, max_iterations)
+        Execution::Sandbox.run(@evaluation.path) do |sandbox|
+          agent_result = Agent::ReactAgent.call(
+            system_prompt: @system_prompt,
+            initial_prompt: @evaluation.task,
+            working_dir: sandbox.path,
+            container_id: sandbox.container_id,
+            client_params: client_params,
+            max_iterations: max_iterations
+          )
+
+          status = agent_result[:success] ? :success : :error
+          final_answer = agent_result.dig(:response, :content) || ''
+          diff = Execution::Sandbox.capture_diff(sandbox.path)
+          iterations = agent_result.dig(:response, :iterations) || []
+
+          output = [final_answer, diff].reject(&:empty?).join("\n\n")
+
+          {
+            result: output,
+            status: status,
+            runtime: @provider.runtime,
+            usage: {},
+            raw_response: agent_result,
+            iterations: iterations
+          }
+        end
+      rescue StandardError => e
+        {
+          result: "Error: #{e.message}",
+          status: :error,
+          runtime: @provider.runtime,
+          usage: {},
+          raw_response: { error: e.message, backtrace: e.backtrace },
+          iterations: []
+        }
+      end
+
+      # Builds client parameters for the ReactAgent.
+      #
+      # @return [Hash] Client parameters
+      def build_client_params
+        config = @config || safe_merged_config
+        return {} unless config
+
+        params = config.dup
+        params[:model] ||= @provider.llm
+        params[:provider] = @provider.runtime.to_sym
+        params
+      rescue StandardError
+        {}
+      end
+
+      # Safely calls merged_config, returning nil on any error.
+      #
+      # @return [Hash, nil] The merged config or nil
+      def safe_merged_config
+        @provider.merged_config
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/skill_bench/services/compare_option_parser.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module SkillBench
+  module Services
+    # Parses CLI options for the compare command.
+    class CompareOptionParser
+      # Parses the given argv and returns the options hash.
+      #
+      # @param argv [Array<String>] Raw CLI arguments
+      # @return [Hash] Parsed options with keys: :variant_a, :variant_b, :eval, :format
+      # @raise [OptionParser::ParseError] when option parsing fails
+      def self.call(argv)
+        new(argv).call
+      end
+
+      # @param argv [Array<String>] Raw CLI arguments
+      def initialize(argv)
+        @argv = argv
+      end
+
+      # Parses options from argv.
+      #
+      # @return [Hash] Parsed options with keys: :variant_a, :variant_b, :eval, :format
+      # @raise [OptionParser::ParseError] when option parsing fails
+      def call
+        options = { format: :human }
+        parser = build_parser(options)
+        parser.parse!(@argv)
+        options
+      end
+
+      private
+
+      # Builds the OptionParser instance.
+      #
+      # @param options [Hash] Options hash to populate
+      # @return [OptionParser] Configured parser
+      def build_parser(options)
+        OptionParser.new do |opts|
+          opts.banner = 'Usage: skill-bench compare <skill-name> [options]'
+          opts.on('--variant-a SPEC', 'First variant (e.g., "pack:rails" or "/path/to/skill")') { |v| options[:variant_a] = v }
+          opts.on('--variant-b SPEC', 'Second variant (e.g., "pack:hanami" or "/path/to/skill")') { |v| options[:variant_b] = v }
+          opts.on('--eval PATH', 'Path to the eval directory') { |v| options[:eval] = v }
+          opts.on('--format FORMAT', 'Output format (human, json)') { |v| options[:format] = v.to_sym }
+          opts.on('-h', '--help', 'Prints this help') do
+            puts opts
+            raise SkillBench::HelpRequested
+          end
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/services/comparison_reporter.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Prints a formatted comparison report for two evaluation results.
+    class ComparisonReporter
+      # Prints the comparison report to stdout.
+      #
+      # @param result_a [Hash] First evaluation result
+      # @param result_b [Hash] Second evaluation result
+      # @param label_a [String] Label for first variant
+      # @param label_b [String] Label for second variant
+      # @return [nil]
+      def self.call(result_a, result_b, label_a, label_b)
+        new(result_a, result_b, label_a, label_b).call
+      end
+
+      # @param result_a [Hash] First evaluation result
+      # @param result_b [Hash] Second evaluation result
+      # @param label_a [String] Label for first variant
+      # @param label_b [String] Label for second variant
+      def initialize(result_a, result_b, label_a, label_b)
+        @result_a = result_a
+        @result_b = result_b
+        @label_a = label_a
+        @label_b = label_b
+      end
+
+      # Prints the comparison report to stdout.
+      #
+      # @return [nil]
+      def call
+        puts "\n=== Comparison Report ==="
+        puts "| Dimension | #{@label_a} | #{@label_b} | Delta |"
+        puts '|-----------|----------|----------|-------|'
+
+        report_a = @result_a.dig(:response, :report)
+        report_b = @result_b.dig(:response, :report)
+        return unless report_a && report_b
+
+        print_dimension_scores(report_a, report_b)
+        print_total_scores(report_a, report_b)
+        print_verdicts(report_a, report_b)
+      end
+
+      private
+
+      # Prints dimension score comparison.
+      #
+      # @param report_a [Object] First evaluation report
+      # @param report_b [Object] Second evaluation report
+      def print_dimension_scores(report_a, report_b)
+        report_b_by_name = report_b.dimensions.to_h { |d| [d.name, d] }
+
+        report_a.dimensions.each do |dim|
+          score_a = dim.score
+          score_b = report_b_by_name[dim.name]&.score || 0
+          delta = score_a - score_b
+          puts format('| %<name>-9s | %<a>8.1f | %<b>8.1f | %<delta>+5.1f |',
+                      name: dim.name, a: score_a, b: score_b, delta: delta.to_f)
+        end
+      end
+
+      # Prints total score comparison.
+      #
+      # @param report_a [Object] First evaluation report
+      # @param report_b [Object] Second evaluation report
+      def print_total_scores(report_a, report_b)
+        total_a = report_a.total
+        total_b = report_b.total
+        return unless total_a && total_b
+
+        delta = total_a - total_b
+        puts format('| %<name>-9s | %<a>8.1f | %<b>8.1f | %<delta>+5.1f |',
+                    name: 'TOTAL', a: total_a.to_f, b: total_b.to_f, delta: delta.to_f)
+      end
+
+      # Prints verdict comparison.
+      #
+      # @param report_a [Object] First evaluation report
+      # @param report_b [Object] Second evaluation report
+      def print_verdicts(report_a, report_b)
+        verdict_a = format_verdict(report_a.verdict)
+        verdict_b = format_verdict(report_b.verdict)
+        puts "| A: #{verdict_a} | B: #{verdict_b} |"
+      end
+
+      def format_verdict(verdict)
+        case verdict
+        when true then 'PASS'
+        when false then 'FAIL'
+        else verdict.to_s
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/services/comparison_runner.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative 'variant_resolver'
+require_relative 'runner_service'
+
+module SkillBench
+  module Services
+    # Runs both variants of a skill comparison.
+    class ComparisonRunner
+      # Runs both variants and returns their results.
+      #
+      # @param variant_a [Hash] First variant specification
+      # @param variant_b [Hash] Second variant specification
+      # @param skill_name [String] Name of the skill to compare
+      # @param eval_path [String] Path to the eval directory
+      # @param manifest_path [String, nil] Optional path to registry manifest
+      # @return [Hash] Hash with :result_a and :result_b keys
+      def self.call(variant_a, variant_b, skill_name, eval_path, manifest_path: nil)
+        new(variant_a, variant_b, skill_name, eval_path, manifest_path: manifest_path).call
+      end
+
+      # @param variant_a [Hash] First variant specification
+      # @param variant_b [Hash] Second variant specification
+      # @param skill_name [String] Name of the skill to compare
+      # @param eval_path [String] Path to the eval directory
+      # @param manifest_path [String, nil] Optional path to registry manifest
+      def initialize(variant_a, variant_b, skill_name, eval_path, manifest_path: nil)
+        @variant_a = variant_a
+        @variant_b = variant_b
+        @skill_name = skill_name
+        @eval_path = eval_path
+        @manifest_path = manifest_path
+      end
+
+      # Runs both variants and returns their results.
+      #
+      # @return [Hash] Hash with :result_a and :result_b keys
+      def call
+        skill_paths_a = VariantResolver.call(@variant_a, @skill_name, manifest_path: @manifest_path)
+        skill_paths_b = VariantResolver.call(@variant_b, @skill_name, manifest_path: @manifest_path)
+
+        result_a = RunnerService.call(eval_name: @eval_path, skill_names: skill_paths_a)
+        result_b = RunnerService.call(eval_name: @eval_path, skill_names: skill_paths_b)
+
+        { result_a: result_a, result_b: result_b }
+      end
+    end
+  end
+end

data/lib/skill_bench/services/context_loader_service.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Loads and combines skill context from SKILL.md files.
+    class ContextLoaderService
+      # Loads and combines skill context from SKILL.md files.
+      #
+      # @param skills [Array<SkillBench::Models::Skill>] The skills to load context from
+      # @return [String] The combined skill context
+      def self.call(skills)
+        new(skills).call
+      end
+
+      # @param skills [Array<SkillBench::Models::Skill>] The skills to load context from
+      def initialize(skills)
+        @skills = skills
+      end
+
+      # Loads and combines skill context from SKILL.md files.
+      #
+      # @return [String] The combined skill context
+      def call
+        return '' if @skills.nil? || @skills.empty?
+
+        contexts = @skills.map { |skill| load_skill_context(skill) }
+        contexts.reject(&:empty?).join("\n\n#{'=' * 40}\n\n")
+      end
+
+      private
+
+      # Loads the skill context from a single skill's SKILL.md file.
+      #
+      # @param skill [SkillBench::Models::Skill] The skill to load context from
+      # @return [String] The skill context or empty string if not found
+      def load_skill_context(skill)
+        skill_md = File.join(skill.path, 'SKILL.md')
+        File.exist?(skill_md) ? File.read(skill_md) : ''
+      end
+    end
+  end
+end

data/lib/skill_bench/services/error_response_builder.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Builds standardized error responses with metadata.
+    class ErrorResponseBuilder
+      # Builds a configuration error response.
+      #
+      # @param error [Exception] The configuration error
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param provider [Object] The resolved provider
+      # @param skill_names [Array<String>] Names of the skills
+      # @return [Hash] Error result with metadata
+      def self.config_error(error, evaluation, provider, skill_names)
+        new(evaluation, provider, skill_names).config_error(error)
+      end
+
+      # Builds an agent error response.
+      #
+      # @param result [Hash] The agent result containing the error
+      # @param phase [String] The phase that failed (e.g., 'baseline', 'context')
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param provider [Object] The resolved provider
+      # @param skill_names [Array<String>] Names of the skills
+      # @return [Hash] Error result with metadata
+      def self.agent_error(result, phase, evaluation, provider, skill_names)
+        new(evaluation, provider, skill_names).agent_error(result, phase)
+      end
+
+      # Builds an empty context error response.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param provider [Object] The resolved provider
+      # @param skill_names [Array<String>] Names of the skills
+      # @return [Hash] Error result with metadata
+      def self.empty_context_error(evaluation, provider, skill_names)
+        new(evaluation, provider, skill_names).empty_context_error
+      end
+
+      # Enriches an existing error result with metadata.
+      #
+      # @param result [Hash] The existing error result
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param provider [Object] The resolved provider
+      # @param skill_names [Array<String>] Names of the skills
+      # @return [Hash] Enriched error result with metadata
+      def self.enrich_error(result, evaluation, provider, skill_names)
+        new(evaluation, provider, skill_names).enrich_error(result)
+      end
+
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param provider [Object] The resolved provider
+      # @param skill_names [Array<String>] Names of the skills
+      def initialize(evaluation, provider, skill_names)
+        @evaluation = evaluation
+        @provider = provider
+        @skill_names = skill_names
+      end
+
+      # Builds a configuration error response.
+      #
+      # @param error [Exception] The configuration error
+      # @return [Hash] Error result with metadata
+      def config_error(error)
+        base_error_result("Configuration error: #{error.message}")
+      end
+
+      # Builds an agent error response.
+      #
+      # @param result [Hash] The agent result containing the error
+      # @param phase [String] The phase that failed (e.g., 'baseline', 'context')
+      # @return [Hash] Error result with metadata
+      def agent_error(result, phase)
+        raw = result[:raw_response]
+        error_msg = raw&.dig(:response, :error, :message) || raw&.dig(:error, :message) || 'unknown error'
+        base_error_result("#{phase.capitalize} agent failed: #{error_msg}")
+      end
+
+      # Builds an empty context error response.
+      #
+      # @return [Hash] Error result with metadata
+      def empty_context_error
+        base_error_result('Skill context is empty. Ensure SKILL.md exists and has content.')
+      end
+
+      # Enriches an existing error result with metadata.
+      #
+      # @param result [Hash] The existing error result
+      # @return [Hash] Enriched error result with metadata
+      def enrich_error(result)
+        result.merge(
+          eval_name: @evaluation.name,
+          skill_name: @skill_names.join(', '),
+          provider_name: @provider.name
+        )
+      end
+
+      private
+
+      # Builds a base error result with metadata.
+      #
+      # @param message [String] The error message
+      # @return [Hash] Error result with metadata
+      def base_error_result(message)
+        {
+          success: false,
+          response: {
+            error: {
+              message: message
+            }
+          },
+          eval_name: @evaluation.name,
+          skill_name: @skill_names.join(', '),
+          provider_name: @provider.name
+        }
+      end
+    end
+  end
+end

data/lib/skill_bench/services/eval_resolver.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative '../models/eval'
+
+module SkillBench
+  module Services
+    # Resolves an eval from a name or path.
+    class EvalResolver
+      # Resolves an eval from a name or path.
+      #
+      # @param eval_name [String] Name or path of the eval to resolve
+      # @return [SkillBench::Models::Eval] The resolved eval
+      # @raise [Errno::ENOENT] when the eval directory does not exist
+      def self.call(eval_name)
+        new(eval_name).call
+      end
+
+      # @param eval_name [String] Name or path of the eval
+      def initialize(eval_name)
+        @eval_name = eval_name
+      end
+
+      # Resolves the eval from the name or path.
+      #
+      # @return [SkillBench::Models::Eval] The resolved eval
+      # @raise [Errno::ENOENT] when the eval directory does not exist
+      def call
+        eval_path = @eval_name.include?('/') ? @eval_name : "evals/#{@eval_name}"
+        SkillBench::Models::Eval.load(eval_path)
+      end
+    end
+  end
+end

data/lib/skill_bench/services/exit_code_calculator.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Calculates the exit code based on comparison results.
+    class ExitCodeCalculator
+      # Calculates the exit code from comparison results.
+      #
+      # @param result_a [Hash] First evaluation result
+      # @param result_b [Hash] Second evaluation result
+      # @return [Integer] 0 if both pass, 1 otherwise
+      def self.call(result_a, result_b)
+        new(result_a, result_b).call
+      end
+
+      # @param result_a [Hash] First evaluation result
+      # @param result_b [Hash] Second evaluation result
+      def initialize(result_a, result_b)
+        @result_a = result_a
+        @result_b = result_b
+      end
+
+      # Calculates the exit code from comparison results.
+      #
+      # @return [Integer] 0 if both pass, 1 otherwise
+      def call
+        report_a = @result_a.dig(:response, :report)
+        report_b = @result_b.dig(:response, :report)
+
+        verdict_a = report_a.is_a?(Hash) ? report_a[:verdict] : report_a&.verdict
+        verdict_b = report_b.is_a?(Hash) ? report_b[:verdict] : report_b&.verdict
+
+        passed_a = verdict_a == 'PASS'
+        passed_b = verdict_b == 'PASS'
+        passed_a && passed_b ? 0 : 1
+      end
+    end
+  end
+end

data/lib/skill_bench/services/judge_params_builder.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Builds judge parameters from provider configuration.
+    class JudgeParamsBuilder
+      # Builds judge parameters from provider configuration.
+      #
+      # @param provider [Object] The resolved provider
+      # @param config [Hash, nil] Provider config
+      # @return [Hash] Judge parameters with api_key, model, and provider
+      def self.call(provider, config)
+        new(provider, config).call
+      end
+
+      # @param provider [Object] The resolved provider
+      # @param config [Hash, nil] Provider config
+      def initialize(provider, config)
+        @provider = provider
+        @config = config
+      end
+
+      # Builds judge parameters from provider configuration.
+      #
+      # @return [Hash] Judge parameters with api_key, model, and provider
+      def call
+        return { provider: :mock } if @provider.name == 'mock'
+
+        config = @config || safe_merged_config
+        return {} unless config
+
+        {
+          api_key: config[:api_key],
+          model: config[:model] || @provider.llm,
+          provider: @provider.runtime.to_sym
+        }
+      rescue KeyError, NoMethodError
+        # Expected errors from missing config keys or nil config
+        {}
+      end
+
+      private
+
+      # Safely calls merged_config, returning nil on any error.
+      #
+      # @return [Hash, nil] The merged config or nil
+      def safe_merged_config
+        @provider.merged_config
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/skill_bench/services/manifest_finder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Finds the registry manifest file path.
+    class ManifestFinder
+      # Default path relative to current working directory.
+      DEFAULT_PATH = '../agent-mcp-runtime/registry.json'
+
+      # Finds the registry manifest file.
+      #
+      # @param path [String, nil] Optional custom path to the manifest
+      # @return [String] Absolute path to the registry manifest
+      # @raise [ArgumentError] when the manifest file is not found
+      def self.call(path: nil)
+        new(path: path).call
+      end
+
+      # @param path [String, nil] Optional custom path to the manifest
+      def initialize(path: nil)
+        @path = path
+      end
+
+      # Finds the registry manifest file.
+      #
+      # @return [String] Absolute path to the registry manifest
+      # @raise [ArgumentError] when the manifest file is not found
+      def call
+        manifest_path = @path || File.expand_path(DEFAULT_PATH, Dir.pwd)
+        raise ArgumentError, "Registry manifest not found: #{manifest_path}" unless File.exist?(manifest_path)
+
+        manifest_path
+      end
+    end
+  end
+end

data/lib/skill_bench/services/output_formatter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Formats agent output for evaluation.
+    class OutputFormatter
+      # Formats agent output for evaluation.
+      #
+      # @param agent_result [Hash] The agent result containing the output
+      # @return [String] The formatted output
+      def self.call(agent_result)
+        new(agent_result).call
+      end
+
+      # @param agent_result [Hash] The agent result containing the output
+      def initialize(agent_result)
+        @agent_result = agent_result
+      end
+
+      # Formats agent output for evaluation.
+      #
+      # @return [String] The formatted output
+      def call
+        @agent_result[:result].to_s
+      end
+    end
+  end
+end