ruby-skill-bench 0.1.0

data/lib/skill_bench/services/option_parser_service.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module SkillBench
+  module Services
+    # Parses CLI arguments for the EvaluateCommand using Ruby's OptionParser.
+    # Provides standardized error handling for invalid flags and missing arguments.
+    # @deprecated Use {SkillBench::Cli::RunCommand} option parsing instead.
+    class OptionParserService
+      # Parses command-line options into a hash.
+      #
+      # @param argv [Array<String>] Raw CLI arguments.
+      # @return [Hash] Result envelope with parsed options or error message.
+      def self.call(argv)
+        new(argv).call
+      end
+
+      # @param argv [Array<String>] Raw CLI arguments.
+      def initialize(argv)
+        @argv = argv
+      end
+
+      # Parses the arguments and returns a result hash.
+      #
+      # @return [Hash] Result envelope with parsed options or error message.
+      def call
+        options = {}
+
+        parser(options).parse!(@argv)
+
+        { success: true, response: options }
+      rescue OptionParser::ParseError => e
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      def parser(options)
+        OptionParser.new do |opts|
+          opts.banner = 'Usage: skill-bench [options]'
+
+          opts.on('-e', '--eval FOLDER', 'Path to the eval folder') do |eval_path|
+            options[:eval] = eval_path
+          end
+
+          opts.on('-s', '--skill FOLDER', 'Optional override for the source skill folder') do |skill_path|
+            options[:skill] = skill_path
+          end
+
+          opts.on('-o', '--output FILE', 'Path to save the JSON report') do |output_path|
+            options[:output] = output_path
+          end
+
+          opts.on('-h', '--help', 'Prints this help') do
+            puts opts
+            raise SkillBench::HelpRequested
+          end
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/services/output_persistence_service.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+
+module SkillBench
+  module Services
+    # Service object for persisting evaluation results to JSON files.
+    # Handles file I/O, JSON serialization, and provides standardized error responses
+    # for filesystem operations.
+    # @deprecated Use {SkillBench::Cli::RunCommand} output handling instead.
+    class OutputPersistenceService
+      WRITE_ERROR = 'Failed to write output file'
+
+      # Persists evaluation results to a JSON file with proper formatting.
+      #
+      # @param result [Hash] Evaluation result hash containing all evaluation data
+      # @param output_path [String, nil] Path to save the JSON report. If nil or empty, no action is taken
+      # @return [Hash] Standardized response hash with format:
+      #   - { success: true, response: { message: String } } on success
+      #   - { success: true, response: {} } when no output path is provided
+      #   - { success: false, response: { error: { message: String } } } on failure
+      # @example Save to file
+      #   result = OutputPersistenceService.call(evaluation_result, output_path: 'output.json')
+      #   # => { success: true, response: { message: 'Report saved to output.json' } }
+      # @example No output path
+      #   result = OutputPersistenceService.call(evaluation_result, output_path: nil)
+      #   # => { success: true, response: {} }
+      def self.call(result, output_path:)
+        new(result, output_path: output_path).call
+      end
+
+      # Initializes a new persistence service instance.
+      #
+      # @param result [Hash] Evaluation result hash containing all evaluation data
+      # @param output_path [String, nil] Path to save the JSON report
+      def initialize(result, output_path:)
+        @result = result
+        @output_path = output_path
+      end
+
+      # Persists the evaluation result to the specified output path.
+      #
+      # @return [Hash] Standardized response hash with format:
+      #   - { success: true, response: { message: String } } on success
+      #   - { success: true, response: {} } when no output path is provided
+      #   - { success: false, response: { error: { message: String } } } on failure
+      # @raise [SystemCallError] when file system operations fail (handled internally)
+      def call
+        return { success: true, response: {} } if @output_path.to_s.empty?
+
+        ensure_directory_exists
+        write_json_file
+
+        { success: true, response: { message: "Report saved to #{@output_path}" } }
+      rescue SystemCallError, JSON::GeneratorError => e
+        { success: false, response: { error: { message: "#{WRITE_ERROR}: #{e.message}" } } }
+      end
+
+      private
+
+      # Ensures the parent directory for the output file exists.
+      # Creates the directory structure if it doesn't exist.
+      def ensure_directory_exists
+        directory = File.dirname(@output_path)
+        FileUtils.mkdir_p(directory) unless File.directory?(directory)
+      end
+
+      # Writes the evaluation result as a formatted JSON file.
+      #
+      # @raise [SystemCallError] when file write operation fails
+      def write_json_file
+        File.write(@output_path, JSON.generate(@result, pretty: true))
+      end
+    end
+  end
+end

data/lib/skill_bench/services/result_printer_service.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require_relative 'judge_score_parser_service'
+
+module SkillBench
+  module Services
+    # Service object for printing formatted evaluation results to stdout.
+    # Handles result formatting, score parsing, and provides standardized output for
+    # both successful and failed evaluations.
+    # @deprecated Use {SkillBench::Cli::ResultPrinter} instead.
+    class ResultPrinterService
+      RESULTS_BANNER = "\n=========================================\n              " \
+                       "RESULTS                    \n" \
+                       "=========================================\n"
+
+      # Prints formatted evaluation results to the specified output stream.
+      #
+      # @param result [Hash] Evaluation result hash containing success status and task data
+      # @param stdout [#puts, #write] Output stream for user-visible messages. Defaults to $stdout
+      # @return [Hash] Standardized response hash with format:
+      #   - { success: true, response: {} } on successful printing
+      # @example Print successful results
+      #   result = ResultPrinterService.call(evaluation_result)
+      #   # => { success: true, response: {} }
+      # @example Print to custom stream
+      #   result = ResultPrinterService.call(evaluation_result, stdout: string_io)
+      #   # => { success: true, response: {} }
+      def self.call(result, stdout: $stdout)
+        new(result, stdout: stdout).call
+      end
+
+      # Initializes a new result printer instance.
+      #
+      # @param result [Hash] Evaluation result hash containing success status and task data
+      # @param stdout [#puts, #write] Output stream for user-visible messages. Defaults to $stdout
+      def initialize(result, stdout: $stdout)
+        @result = result
+        @stdout = stdout
+      end
+
+      # Prints the evaluation results in a formatted, user-friendly manner.
+      # Handles both successful evaluations and error cases.
+      #
+      # @return [Hash] Standardized response hash with format:
+      #   - { success: true, response: {} } on successful printing
+      def call
+        @stdout.puts RESULTS_BANNER
+
+        unless @result[:success]
+          error_msg = @result.dig(:response, :error, :message) || 'Unknown error'
+          @stdout.puts "Evaluation failed: #{error_msg}"
+          return { success: true, response: {} }
+        end
+
+        @result[:tasks]&.each do |task_result|
+          @stdout.puts "\n========================================="
+          @stdout.puts "       RESULTS: #{task_result[:path]}    "
+          @stdout.puts "=========================================\n"
+          print_task_result(task_result)
+        end
+
+        { success: true, response: {} }
+      end
+
+      private
+
+      # Prints the result for a single task, including scores and diffs.
+      #
+      # @param task_result [Hash] Individual task result containing judge scores and diffs
+      def print_task_result(task_result)
+        score_payload = task_result[:judge_score]
+        parser_class = SkillBench::Services::JudgeScoreParserService
+        parsed_judge = parser_class.call(score_payload)
+
+        unless parsed_judge[:success]
+          print_parse_error
+          @stdout.puts(score_payload || 'nil')
+          return
+        end
+
+        print_judge_summary(parsed_judge[:response])
+        print_task_diffs(task_result[:path], task_result[:baseline_diff], task_result[:context_diff])
+      end
+
+      # Prints an error message when judge score parsing fails.
+      def print_parse_error
+        @stdout.puts 'Could not parse judge JSON response. Raw output:'
+      end
+
+      # Prints the judge score summary including baseline and context scores.
+      #
+      # @param parsed_judge [Hash] Parsed judge score data containing scores and reasoning
+      def print_judge_summary(parsed_judge)
+        @stdout.puts "Baseline Score: #{parsed_judge['baseline_score']}/100"
+        @stdout.puts "Context Score:  #{parsed_judge['context_score']}/100"
+        @stdout.puts "\nReasoning:"
+        @stdout.puts parsed_judge['reasoning']
+      end
+
+      # Prints the baseline and context diffs for a task.
+      #
+      # @param path [String] The file path associated with the diff
+      # @param baseline_diff [String] The diff content for the baseline
+      # @param context_diff [String] The diff content for the context
+      def print_task_diffs(path, baseline_diff, context_diff)
+        print_diff_section('BASELINE CHANGES', path, baseline_diff)
+        print_diff_section('CONTEXT CHANGES', path, context_diff)
+      end
+
+      # Prints a formatted diff section with a banner.
+      #
+      # @param title [String] The title for the diff section (e.g., 'BASELINE CHANGES')
+      # @param path [String] The file path associated with the diff
+      # @param diff [String] The diff content to print
+      def print_diff_section(title, path, diff)
+        sep_newline = "\n========================================="
+        sep_plain = "=========================================\n"
+
+        @stdout.puts sep_newline
+        @stdout.puts "  #{title}: #{path}  "
+        @stdout.puts sep_plain
+        @stdout.puts diff
+      end
+    end
+  end
+end

data/lib/skill_bench/services/runner_service.rb

@@ -0,0 +1,381 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'pathname'
+require_relative '../models/eval'
+require_relative '../models/skill'
+require_relative '../models/config'
+require_relative '../models/provider'
+require_relative '../clients/all'
+require_relative 'skill_resolver'
+require_relative '../trend_tracker'
+require_relative '../execution/sandbox'
+require_relative '../execution/context_hydrator'
+require_relative '../execution/source_path_resolver'
+require_relative '../agent/react_agent'
+
+module SkillBench
+  module Services
+    # Orchestrates the execution of an eval with baseline and context runs.
+    # rubocop:disable Metrics/ClassLength
+    class RunnerService
+      # Stand-in provider when no LLM config is available.
+      MOCK_PROVIDER = Struct.new(:name, :runtime, :llm, :merged_config)
+      private_constant :MOCK_PROVIDER
+
+      # Runs an eval with the given parameters.
+      #
+      # @param eval_name [String] Name or path of the eval to run
+      # @param skill_names [Array<String>] Names of the skills to use
+      # @return [Hash] Result from EvaluationRunner
+      def self.call(eval_name:, skill_names:)
+        new(eval_name: eval_name, skill_names: skill_names).call
+      end
+
+      # @param eval_name [String] Name or path of the eval
+      # @param skill_names [Array<String>] Names of the skills
+      def initialize(eval_name:, skill_names:)
+        @eval_name = eval_name
+        @skill_names = skill_names
+      end
+
+      # Executes the eval: resolves entities, runs baseline and context, evaluates.
+      #
+      # @return [Hash] Evaluation result with deltas and verdict.
+      # @raise [Errno::ENOENT] when the eval directory does not exist.
+      # @raise [ArgumentError] when a skill cannot be resolved.
+      def call
+        evaluation = resolve_eval
+        skills = resolve_skills
+        provider = resolve_provider
+
+        config_result = resolve_provider_config(provider)
+        return config_error_result(config_result[:error], evaluation, provider) unless config_result[:success]
+
+        config = config_result[:config]
+        baseline_prompt = build_baseline_system_prompt
+
+        baseline_output = spawn_agent(evaluation, baseline_prompt, provider, config)
+        return agent_error_result(baseline_output, 'baseline', evaluation, provider) if baseline_output[:status] == :error
+
+        skill_context = load_combined_skill_context(skills)
+        return empty_context_error_result(evaluation, provider) if skill_context.strip.empty?
+
+        context_prompt = build_context_system_prompt(evaluation, skills)
+        context_output = spawn_agent(evaluation, context_prompt, provider, config)
+        return agent_error_result(context_output, 'context', evaluation, provider) if context_output[:status] == :error
+
+        criteria = evaluation.criteria
+
+        judge_params = build_judge_params(provider, config)
+
+        result = Evaluation::Runner.call(
+          task: evaluation.task,
+          criteria: criteria,
+          skill_context: skill_context,
+          baseline_output: format_output(baseline_output),
+          context_output: format_output(context_output),
+          judge_params: judge_params
+        )
+
+        return enrich_error_result(result, evaluation, provider) unless result[:success]
+
+        trend_result = record_and_compute_trend(result)
+        return enrich_error_result(trend_result, evaluation, provider) unless trend_result[:success]
+
+        {
+          success: true,
+          eval_name: eval_name,
+          skill_name: skill_names.join(', '),
+          provider_name: provider.name,
+          response: result[:response].merge(
+            trend: trend_result[:trend],
+            baseline_iterations: baseline_output[:iterations] || [],
+            context_iterations: context_output[:iterations] || []
+          )
+        }
+      end
+
+      private
+
+      attr_reader :eval_name, :skill_names
+
+      def resolve_eval
+        eval_path = eval_name.include?('/') ? eval_name : "evals/#{eval_name}"
+        SkillBench::Models::Eval.load(eval_path)
+      end
+
+      def resolve_skills
+        skill_names.map { |name| Services::SkillResolver.call(name) }
+      end
+
+      def resolve_provider_config(provider)
+        { success: true, config: provider.merged_config }
+      rescue ArgumentError => e
+        { success: false, error: e }
+      end
+
+      # Safely calls merged_config, returning nil on any error.
+      #
+      # @param provider [Object] The provider to query.
+      # @return [Hash, nil] The merged config or nil.
+      def safe_merged_config(provider)
+        provider.merged_config
+      rescue StandardError
+        nil
+      end
+
+      def resolve_provider
+        config = SkillBench::Models::Config.load
+        provider = config.to_provider
+        return provider if provider
+
+        warn 'Config load failed, using mock provider'
+        MOCK_PROVIDER.new('mock', 'mock', 'mock', {})
+      end
+
+      # Spawns the LLM agent with the given system prompt.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run.
+      # @param system_prompt [String] The system prompt for the agent.
+      # @param provider [Object] The resolved provider.
+      # @param config [Hash, nil] Provider config.
+      # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations.
+      def spawn_agent(evaluation, system_prompt, provider, config)
+        return { result: 'mock result', status: :success, iterations: [] } if provider.name == 'mock'
+
+        client_params = build_client_params(provider, config)
+
+        max_iterations = config&.[](:max_iterations) || config&.[]('max_iterations') || 25
+
+        Execution::Sandbox.run(evaluation.path) do |sandbox|
+          agent_result = Agent::ReactAgent.call(
+            system_prompt: system_prompt,
+            initial_prompt: evaluation.task,
+            working_dir: sandbox.path,
+            container_id: sandbox.container_id,
+            client_params: client_params,
+            max_iterations: max_iterations
+          )
+
+          status = agent_result[:success] ? :success : :error
+          final_answer = agent_result.dig(:response, :content) || ''
+          diff = Execution::Sandbox.capture_diff(sandbox.path)
+          iterations = agent_result.dig(:response, :iterations) || []
+
+          output = [final_answer, diff].reject(&:empty?).join("\n\n")
+
+          {
+            result: output,
+            status: status,
+            runtime: provider.runtime,
+            usage: {},
+            raw_response: agent_result,
+            iterations: iterations
+          }
+        end
+      end
+
+      # Builds client parameters for the ReactAgent.
+      #
+      # @param provider [Object] The resolved provider.
+      # @param config [Hash, nil] Provider config.
+      # @return [Hash] Client parameters.
+      def build_client_params(provider, config)
+        config ||= safe_merged_config(provider)
+        return {} unless config
+
+        params = config.dup
+        params[:model] ||= provider.llm
+        params[:provider] = provider.runtime.to_sym
+        params
+      rescue StandardError
+        {}
+      end
+
+      # Builds the baseline system prompt (no skill context).
+      #
+      # @return [String] The baseline system prompt.
+      def build_baseline_system_prompt
+        <<~PROMPT
+          You are an expert Ruby on Rails developer. Your job is to read the task,
+          modify the codebase using the tools provided to meet the requirements,
+          and then explain what you did.
+        PROMPT
+      end
+
+      # Builds the context-aware system prompt based on eval metadata.
+      #
+      # For `skill_bundle_xml` context mode, combines SKILL.md with source code
+      # via ContextHydrator. Falls back to SKILL.md-only if source is unavailable.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run.
+      # @param skills [Array<SkillBench::Models::Skill>] Resolved skills.
+      # @return [String] The context system prompt.
+      def build_context_system_prompt(evaluation, skills)
+        skill_md_content = load_combined_skill_context(skills)
+        return skill_md_content unless evaluation.metadata['context_mode'] == 'skill_bundle_xml'
+
+        source_path = resolve_source_path(evaluation)
+        return skill_md_content unless source_path
+
+        xml_result = Execution::ContextHydrator.call(source_path: source_path, base_path: Pathname.new(Dir.pwd))
+        hydrator_response = xml_result[:response]
+        xml_context = hydrator_response[:context]
+        return skill_md_content unless xml_result[:success] && !xml_context.empty?
+
+        <<~PROMPT
+          You are an expert Ruby on Rails developer.
+          You have access to a skill file and source code wrapped in <agent_context> tags.
+          Use the skill instructions and the provided source code to solve the task.
+
+          ## Skill Instructions
+          #{skill_md_content}
+
+          ## Source Code
+          #{xml_context}
+        PROMPT
+      end
+
+      # Resolves the source path for context hydration.
+      #
+      # Tries the eval's `source/` subdirectory first, then falls back to
+      # SourcePathResolver inference.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run.
+      # @return [String, nil] The resolved source path, or nil if not found.
+      def resolve_source_path(evaluation)
+        eval_path = evaluation.path
+        eval_source = File.join(eval_path, 'source')
+        return eval_source if Dir.exist?(eval_source)
+
+        inferred = Execution::SourcePathResolver.call(eval_folder_path: eval_path.to_s)
+        inferred if inferred && Dir.exist?(inferred)
+      end
+
+      # Returns an error result when skill context is empty.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run.
+      # @param provider [Object] The resolved provider.
+      # @return [Hash] Error result with metadata.
+      def empty_context_error_result(evaluation, provider)
+        {
+          success: false,
+          response: {
+            error: {
+              message: 'Skill context is empty. Ensure SKILL.md exists and has content.'
+            }
+          },
+          eval_name: evaluation.name,
+          skill_name: skill_names.join(', '),
+          provider_name: provider.name
+        }
+      end
+
+      def load_combined_skill_context(skills)
+        return '' if skills.nil? || skills.empty?
+
+        contexts = skills.map { |skill| load_skill_context(skill) }
+        contexts.reject(&:empty?).join("\n\n#{'=' * 40}\n\n")
+      end
+
+      def load_skill_context(skill)
+        skill_md = File.join(skill.path, 'SKILL.md')
+        File.exist?(skill_md) ? File.read(skill_md) : ''
+      end
+
+      def build_judge_params(provider, config)
+        return {} if provider.name == 'mock'
+
+        config ||= safe_merged_config(provider)
+        return {} unless config
+
+        {
+          api_key: config[:api_key],
+          model: config[:model] || provider.llm,
+          provider: provider.runtime.to_sym
+        }
+      rescue StandardError
+        {}
+      end
+
+      def format_output(agent_result)
+        agent_result[:result].to_s
+      end
+
+      def agent_error_result(result, phase, evaluation, provider)
+        raw = result[:raw_response]
+        error_msg = raw&.dig(:response, :error, :message) || raw&.dig(:error, :message) || 'unknown error'
+        {
+          success: false,
+          response: {
+            error: {
+              message: "#{phase.capitalize} agent failed: #{error_msg}"
+            }
+          },
+          eval_name: evaluation.name,
+          skill_name: skill_names.join(', '),
+          provider_name: provider.name
+        }
+      end
+
+      def config_error_result(error, evaluation, provider)
+        {
+          success: false,
+          response: {
+            error: {
+              message: "Configuration error: #{error.message}"
+            }
+          },
+          eval_name: evaluation.name,
+          skill_name: skill_names.join(', '),
+          provider_name: provider.name
+        }
+      end
+
+      def enrich_error_result(result, evaluation, provider)
+        result.merge(
+          eval_name: evaluation.name,
+          skill_name: skill_names.join(', '),
+          provider_name: provider.name
+        )
+      end
+
+      def record_and_compute_trend(result)
+        tracker = TrendTracker.new
+        enriched = result.merge(eval_name: eval_name, skill_names: skill_names)
+        trend = tracker.trend_for(enriched)
+        record_result = tracker.record(enriched)
+
+        record_success = record_result.is_a?(Hash) && record_result[:success]
+        unless record_success
+          message = if record_result.is_a?(Hash)
+                      record_result.dig(:response, :error, :message) ||
+                        record_result.dig(:error, :message) ||
+                        'Unknown error'
+                    else
+                      'Unexpected record response'
+                    end
+          SkillBench::ErrorLogger.log_error(
+            StandardError.new(message),
+            "Trend tracking record failed for eval #{eval_name}"
+          )
+          return {
+            success: false,
+            response: {
+              error: {
+                message: "Trend tracking record failed: #{message}",
+                record_result: record_result
+              }
+            }
+          }
+        end
+        { success: true, trend: trend }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Trend tracking failed')
+        { success: false, response: { error: { message: e.message } } }
+      end
+      # rubocop:enable Metrics/ClassLength
+    end
+  end
+end