ruby-skill-bench 0.1.0

data/lib/skill_bench/judge/response.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  module Judge
+    # Parses and validates structured JSON responses from the LLM judge.
+    #
+    # Expects a JSON object with a 'dimensions' key mapping dimension names
+    # to score hashes, and an optional 'overall_reasoning' string.
+    class Response
+      attr_reader :dimensions, :overall_reasoning
+
+      # Parses a judge JSON string.
+      #
+      # @param json [String] The raw JSON string from the judge.
+      # @return [Hash] Service response with parsed judge response or error.
+      def self.call(json:)
+        new(json:).call
+      end
+
+      # @param json [String] The raw JSON string from the judge.
+      def initialize(json:)
+        @json = json
+      end
+
+      # Parses and validates the judge JSON.
+      #
+      # @return [Hash] Service response with judge response or error.
+      def call
+        data = parse_json
+        return data unless data[:success]
+
+        payload = data[:response][:data]
+        validation = validate_structure(payload)
+        return validation unless validation[:success]
+
+        dims = payload['dimensions'] || payload[:dimensions]
+        extracted = extract_dimensions(dims)
+        return extracted unless extracted[:success]
+
+        @dimensions = extracted[:response][:dimensions]
+        @overall_reasoning = payload['overall_reasoning'] || payload[:overall_reasoning] || ''
+
+        { success: true, response: { judge_response: self } }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Judge::Response Parse Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      attr_reader :json
+
+      def parse_json
+        stripped = strip_markdown_fences(json)
+        data = JSON.parse(stripped)
+        { success: true, response: { data: data } }
+      rescue JSON::ParserError => e
+        { success: false, response: { error: { message: "Invalid JSON: #{e.message}" } } }
+      end
+
+      def strip_markdown_fences(text)
+        return text unless text.start_with?('```')
+
+        lines = text.each_line.to_a
+        lines.shift if lines.first&.strip&.start_with?('```')
+        lines.pop if lines.last&.strip == '```'
+        lines.join.strip
+      end
+
+      def validate_structure(payload)
+        dims = payload['dimensions'] || payload[:dimensions]
+
+        return missing_dimensions_result if dims.nil?
+        return empty_dimensions_result if dims.empty?
+
+        { success: true, response: {} }
+      end
+
+      def missing_dimensions_result
+        { success: false, response: { error: { message: "Judge response missing 'dimensions' key" } } }
+      end
+
+      def empty_dimensions_result
+        { success: false, response: { error: { message: "Judge response 'dimensions' is empty" } } }
+      end
+
+      def extract_dimensions(dims)
+        dimensions = {}
+
+        dims.each do |name, dim|
+          validated = validate_dimension(name, dim)
+          return validated unless validated[:success]
+
+          dimensions[name] = validated[:response][:dimension]
+        end
+
+        { success: true, response: { dimensions: dimensions } }
+      end
+
+      def validate_dimension(name, dim)
+        score = dim['score'] || dim[:score]
+        return missing_score_result(name) if score.nil?
+
+        numeric_score = parse_numeric(score)
+        return invalid_score_result(name, score) if numeric_score.nil?
+
+        max_score = dim['max_score'] || dim[:max_score]
+        max_score_result = validate_max_score(name, numeric_score, max_score)
+        return max_score_result unless max_score_result[:success]
+
+        {
+          success: true,
+          response: {
+            dimension: {
+              score: numeric_score,
+              max_score: max_score,
+              reasoning: dim['reasoning'] || dim[:reasoning] || ''
+            }
+          }
+        }
+      end
+
+      def validate_max_score(name, numeric_score, max_score)
+        return { success: true, response: {} } unless max_score
+        return invalid_max_score_result(name, max_score) unless max_score.is_a?(Numeric)
+        return out_of_bounds_result(name, numeric_score, max_score) if numeric_score.negative? || numeric_score > max_score
+
+        { success: true, response: {} }
+      end
+
+      def parse_numeric(value)
+        return value if value.is_a?(Numeric)
+
+        Float(value)
+      rescue ArgumentError, TypeError
+        nil
+      end
+
+      def missing_score_result(name)
+        { success: false, response: { error: { message: "Judge dimension '#{name}' missing score" } } }
+      end
+
+      def invalid_score_result(name, score)
+        { success: false, response: { error: { message: "Judge dimension '#{name}' has invalid score: #{score.inspect}" } } }
+      end
+
+      def out_of_bounds_result(name, score, max_score)
+        { success: false, response: { error: { message: "Judge dimension '#{name}' score #{score} out of bounds (0..#{max_score})" } } }
+      end
+
+      def invalid_max_score_result(name, max_score)
+        { success: false, response: { error: { message: "Judge dimension '#{name}' has invalid max_score: #{max_score.inspect} (must be numeric)" } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/judge.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Namespace for the judge subsystem.
+  #
+  # The judge evaluates AI-generated code modifications by calling an LLM
+  # and parsing structured JSON responses.
+  module Judge
+  end
+end

data/lib/skill_bench/migration/provider_migrator.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'json'
+
+module SkillBench
+  module Migration
+    # Migrates old provider classes to new YAML-based configuration
+    class ProviderMigrator
+      # Migrate providers to YAML config file
+      # @param providers [Hash] Providers to migrate (name => config hash)
+      # @param yaml_path [String] Path to YAML config file (default: .agent-eval.yml)
+      def self.migrate(providers, yaml_path = '.agent-eval.yml')
+        existing = if File.exist?(yaml_path)
+                     YAML.safe_load_file(yaml_path, permitted_classes: [], aliases: false) || {}
+                   else
+                     {}
+                   end
+
+        existing['providers'] ||= {}
+
+        providers.each do |name, config|
+          existing['providers'][name.to_s] = config.transform_keys(&:to_s)
+        end
+
+        File.write(yaml_path, existing.to_yaml)
+      end
+    end
+  end
+end

data/lib/skill_bench/models/config.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'provider'
+
+module SkillBench
+  module Models
+    # Represents the skill-bench configuration loaded from skill-bench.json
+    class Config
+      # @param data [Hash] Raw configuration data
+      # @raise [ArgumentError] if data is not a Hash
+      def initialize(data = {})
+        raise ArgumentError, 'Config-data must be a Hash' unless data.is_a?(Hash)
+
+        @data = data
+      end
+
+      # Load configuration from a JSON file
+      # @param path [String] Path to config file (default: skill-bench.json)
+      # @return [SkillBench::Models::Config] Loaded config instance
+      # @raise [Errno::ENOENT] if config file not found
+      def self.load(path = 'skill-bench.json')
+        raw_data = JSON.parse(File.read(path), symbolize_names: true)
+        new(raw_data)
+      end
+
+      # Returns the configured provider name
+      # @return [String, nil] Provider name
+      def provider_name
+        @data[:provider]
+      end
+
+      # Returns the provider configuration
+      # @return [Hash] Provider configuration
+      def provider_config
+        @data[:config] || {}
+      end
+
+      # Returns max execution time
+      # @return [Integer] Max execution time in seconds
+      def max_execution_time
+        @data[:max_execution_time] || 30
+      end
+
+      # Builds a Provider model from the current configuration.
+      # Returns a mock provider if provider name is 'mock'.
+      #
+      # @return [SkillBench::Models::Provider] The configured provider
+      def to_provider
+        return nil if provider_name.nil? || provider_name == 'mock'
+
+        Provider.new(
+          name: provider_name,
+          runtime: provider_name,
+          llm: provider_name,
+          config: provider_config
+        )
+      end
+    end
+  end
+end

data/lib/skill_bench/models/criteria_validator.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  module Models
+    # Validates evaluation criteria JSON structure without building rich objects.
+    class CriteriaValidator
+      # Validates criteria from a JSON file
+      #
+      # @param path [String] Path to criteria JSON file
+      # @return [Hash] Validation result with success status and criteria data
+      def self.call(path:)
+        new(path).call
+      end
+
+      # @param path [String] Path to criteria JSON file
+      def initialize(path)
+        @path = path
+      end
+
+      # Validates the criteria file.
+      #
+      # @return [Hash] Validation result.
+      def call
+        return file_not_found_result unless File.exist?(path)
+
+        data = parse_json(path)
+        return data unless data[:success]
+
+        parsed = data[:response][:data]
+        validation = validate(parsed)
+        return validation unless validation[:success]
+
+        { success: true, response: { criteria: parsed } }
+      end
+
+      private
+
+      attr_reader :path
+
+      def file_not_found_result
+        { success: false, response: { error: { message: "File not found: #{path}" } } }
+      end
+
+      def parse_json(file_path)
+        parsed = JSON.parse(File.read(file_path), symbolize_names: true)
+        { success: true, response: { data: parsed } }
+      rescue JSON::ParserError => e
+        { success: false, response: { error: { message: "Invalid JSON: #{e.message}" } } }
+      end
+
+      def validate(data)
+        dim_result = validate_dimensions(data.fetch(:dimensions, []))
+        return dim_result unless dim_result[:success]
+
+        field_result = validate_required_fields(data)
+        return field_result unless field_result[:success]
+
+        threshold_result = validate_pass_threshold(data[:pass_threshold])
+        return threshold_result unless threshold_result[:success]
+
+        validate_minimum_delta(data[:minimum_delta])
+      end
+
+      def validate_dimensions(dimensions)
+        return invalid_dimensions_result unless dimensions.is_a?(Array)
+        return invalid_dimensions_result unless dimensions.all? do |dim|
+          dim.is_a?(Hash) && dim[:name] && dim[:max_score].is_a?(Numeric)
+        end
+
+        total = dimensions.sum { |dim| dim[:max_score] || 0 }
+        return score_sum_result(total) unless total == 100
+
+        { success: true, response: {} }
+      end
+
+      def invalid_dimensions_result
+        { success: false, response: { error: { message: 'Invalid dimensions format' } } }
+      end
+
+      def score_sum_result(total)
+        { success: false, response: { error: { message: "Dimension scores must sum to 100, got #{total}" } } }
+      end
+
+      def validate_required_fields(data)
+        missing = %i[pass_threshold minimum_delta].select { |field| data[field].nil? }
+        return { success: true, response: {} } if missing.empty?
+
+        { success: false, response: { error: { message: "Missing required fields: #{missing.join(', ')}" } } }
+      end
+
+      def validate_pass_threshold(value)
+        return { success: true, response: {} } if value.is_a?(Integer) && value.between?(0, 100)
+
+        { success: false, response: { error: { message: 'Pass threshold must be between 0 and 100' } } }
+      end
+
+      def validate_minimum_delta(value)
+        return { success: true, response: {} } if value.is_a?(Integer) && value >= 0
+
+        { success: false, response: { error: { message: 'Minimum delta must be non-negative integer' } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/models/eval.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'pathname'
+require_relative '../criteria'
+
+module SkillBench
+  module Models
+    # Represents an evaluation scenario
+    class Eval
+      attr_reader :name, :path, :task, :criteria, :source_code, :metadata
+
+      # @param name [String] Eval name
+      # @param path [String] Path to eval directory
+      # @param task [String] Task description from task.md
+      # @param criteria [Hash] Criteria from criteria.json
+      # @param source_code [String] Source code to evaluate
+      # @param metadata [Hash] Metadata from metadata.json
+      def initialize(name:, path:, task: '', criteria: {}, source_code: '', metadata: {})
+        @name = name
+        @path = path
+        @task = task
+        @criteria = criteria
+        @source_code = source_code
+        @metadata = metadata
+      end
+
+      # Load an eval from a directory
+      # @param dir_path [String] Path to eval directory
+      # @return [SkillBench::Models::Eval] Loaded eval instance
+      # @raise [Errno::ENOENT] if eval directory does not exist
+      def self.load(dir_path)
+        path = Pathname.new(dir_path)
+        raise Errno::ENOENT, "Eval directory not found: #{dir_path}" unless path.exist?
+
+        name = path.basename.to_s
+        task = load_task(path)
+        criteria = load_criteria(path)
+        metadata = load_metadata(path)
+
+        new(name: name, path: dir_path, task: task, criteria: criteria, metadata: metadata)
+      end
+
+      # Load task description from task.md
+      # @param path [Pathname] Path to eval directory
+      # @return [String] Task description or empty string if file doesn't exist
+      def self.load_task(path)
+        task_md = path.join('task.md')
+        task_md.exist? ? File.read(task_md) : ''
+      end
+
+      # Load evaluation criteria from criteria.json
+      # @param path [Pathname] Path to eval directory
+      # @return [SkillBench::Criteria] Parsed criteria or empty criteria if file doesn't exist
+      # @raise [RuntimeError] if JSON is malformed or criteria validation fails
+      def self.load_criteria(path)
+        criteria_json = path.join('criteria.json')
+        return SkillBench::Criteria.empty unless criteria_json.exist?
+
+        result = SkillBench::Criteria.call(path: criteria_json.to_s)
+        response = result[:response]
+        return response[:criteria] if result[:success]
+
+        raise "Failed to load criteria: #{response[:error][:message]}"
+      end
+
+      # Load metadata from metadata.json
+      # @param path [Pathname] Path to eval directory
+      # @return [Hash] Parsed metadata or empty hash if file doesn't exist
+      # @raise [JSON::ParserError] if JSON is malformed
+      def self.load_metadata(path)
+        metadata_file = path.join('metadata.json')
+        return {} unless metadata_file.exist?
+
+        JSON.parse(File.read(metadata_file))
+      end
+
+      private_class_method :load_task, :load_criteria, :load_metadata
+    end
+  end
+end

data/lib/skill_bench/models/provider.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../clients/provider_schemas'
+
+module SkillBench
+  module Models
+    # Represents an agent runtime + LLM provider
+    class Provider
+      attr_reader :name, :runtime, :llm, :config
+
+      ALLOWED_PROVIDERS = (Clients::ProviderSchemas.names.map(&:to_s) + %w[mock]).freeze
+
+      # Settings that can be overridden via environment variables.
+      ENV_OVERRIDABLE_SETTINGS = %i[api_key model base_url endpoint location project_id api_version].freeze
+
+      # Initialize a new Provider
+      # @param name [String] Provider name (e.g., "openai")
+      # @param runtime [String] Agent runtime (e.g., "opencode")
+      # @param llm [String] LLM provider (e.g., "openai")
+      # @param config [Hash] Provider-specific configuration
+      def initialize(name:, runtime:, llm:, config: {})
+        @name = name
+        @runtime = runtime
+        @llm = llm
+        @config = config.is_a?(Hash) ? config.transform_keys(&:to_sym) : {}
+      end
+
+      # Returns merged config with environment variable fallbacks.
+      # Checks both `SKILL_BENCH_<PROVIDER>_<SETTING>` (documented standard)
+      # and `<PROVIDER>_<SETTING>` (legacy) naming conventions.
+      #
+      # @return [Hash] Merged configuration
+      # @raise [ArgumentError] if provider name is invalid or API key is missing
+      def merged_config
+        raise ArgumentError, "Invalid provider name: #{name}" unless ALLOWED_PROVIDERS.include?(name)
+
+        merged = config.dup
+        ENV_OVERRIDABLE_SETTINGS.each do |setting|
+          merged[setting] = resolve_env_setting(setting)
+        end
+
+        api_key = merged[:api_key]
+        raise ArgumentError, "API key not found for provider '#{name}'. Set SKILL_BENCH_#{name.upcase}_API_KEY environment variable or provide in config." if api_key.nil? || api_key.to_s.empty?
+
+        merged
+      end
+
+      private
+
+      # Resolves a single setting from environment variables.
+      # Prefers `SKILL_BENCH_<PROVIDER>_<SETTING>`, falls back to
+      # `<PROVIDER>_<SETTING>`, then to the config file value.
+      #
+      # @param setting [Symbol] The setting name (e.g., :api_key)
+      # @return [String, nil] The resolved value
+      def resolve_env_setting(setting)
+        provider_name = name.upcase
+        setting_name = setting.upcase
+
+        prefixed = ENV.fetch("SKILL_BENCH_#{provider_name}_#{setting_name}", nil)
+        return prefixed if prefixed && !prefixed.to_s.empty?
+
+        legacy = ENV.fetch("#{provider_name}_#{setting_name}", nil)
+        return legacy if legacy && !legacy.to_s.empty?
+
+        config[setting]
+      end
+    end
+  end
+end

data/lib/skill_bench/models/skill.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'pathname'
+
+module SkillBench
+  module Models
+    # Represents a reusable skill for agent evaluation
+    class Skill
+      attr_reader :name, :path
+
+      # Initialize a new Skill
+      # @param name [String] Skill name
+      # @param path [String] Path to skill directory
+      def initialize(name:, path:)
+        @name = name
+        @path = path
+      end
+
+      # Discover skills from a directory recursively
+      # @param base_path [String] Directory to search (default: "skills/")
+      # @return [Array<SkillBench::Models::Skill>] Discovered skills
+      def self.discover(base_path = 'skills/')
+        return [] unless Dir.exist?(base_path)
+
+        Dir.glob(File.join(base_path, '**', 'SKILL.md')).map do |skill_md_path|
+          skill_dir = File.dirname(skill_md_path)
+          new(name: File.basename(skill_dir), path: skill_dir)
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/output_formatter.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require_relative 'services/iteration_formatter'
+require_relative 'services/delta_table_formatter'
+require_relative 'services/feedback_generator'
+require_relative 'services/json_formatter'
+require_relative 'services/junit_formatter'
+
+module SkillBench
+  # Handles formatting output for different use cases (human, CI, etc.).
+  # Delegates all presentation logic to focused service objects under
+  # {SkillBench::Services}.
+  class OutputFormatter
+    # Format the eval result for output.
+    #
+    # @param result [Hash] Eval result with keys like :eval_name, :pass, :score, etc.
+    # @param format [Symbol] Output format (:human, :json, :junit)
+    # @return [String] Formatted output string
+    def self.format(result, format: :human)
+      case format
+      when :json
+        Services::JsonFormatter.format(result)
+      when :junit
+        Services::JUnitFormatter.format(result)
+      else
+        format_human(result)
+      end
+    end
+
+    # Determine exit code based on eval result.
+    #
+    # @param result [Hash] Eval result with :pass or :success/:response keys.
+    # @return [Integer] 0 if passed, 1 if failed
+    def self.exit_code(result)
+      return 0 if result[:pass]
+      return 1 unless result[:success]
+
+      report = result.dig(:response, :report)
+      report&.verdict ? 0 : 1
+    end
+
+    # Format result as human-readable text.
+    #
+    # @param result [Hash] Eval result in old or new format.
+    # @return [String] Human-readable formatted string.
+    def self.format_human(result)
+      report = result.dig(:response, :report)
+      return format_legacy_human(result) unless delta_report?(report)
+
+      format_delta_report(result, report)
+    end
+    private_class_method :format_human
+
+    # Checks whether a report object is a DeltaReport.
+    #
+    # @param report [Object] The report to inspect.
+    # @return [Boolean] true when the report has DeltaReport attributes.
+    def self.delta_report?(report)
+      report.respond_to?(:deltas) && report.respond_to?(:criteria) &&
+        report.respond_to?(:baseline_scores) && report.respond_to?(:context_scores)
+    end
+    private_class_method :delta_report?
+
+    # Formats a legacy result hash.
+    #
+    # @param result [Hash] Legacy eval result.
+    # @return [String] Human-readable formatted string.
+    def self.format_legacy_human(result)
+      status = result[:pass] ? 'PASSED' : 'FAILED'
+      lines = [
+        '=' * 60,
+        "Eval: #{result[:eval_name] || ''}",
+        "Skill: #{result[:skill_name] || ''}",
+        "Provider: #{result[:provider_name] || ''}",
+        "Status: #{status}",
+        "Score: #{result[:score]&.round(2) || 'N/A'}"
+      ]
+      error_msg = result.dig(:response, :error, :message)
+      lines << "Error: #{error_msg}" if error_msg
+      lines << ('=' * 60)
+      lines.join("\n")
+    end
+    private_class_method :format_legacy_human
+
+    # Formats a DeltaReport as a human-readable report.
+    #
+    # @param result [Hash] Eval result envelope.
+    # @param report [SkillBench::DeltaReport] The delta report.
+    # @return [String] Formatted report string.
+    def self.format_delta_report(result, report)
+      lines = [
+        ('═' * 55),
+        "  Eval: #{result[:eval_name] || ''}",
+        "  Skill: #{result[:skill_name] || ''}",
+        "  Provider: #{result[:provider_name] || ''}",
+        ('═' * 55),
+        ''
+      ]
+
+      lines.concat(build_iteration_lines(result))
+      lines << Services::DeltaTableFormatter.format(report, result)
+
+      feedback_result = Services::FeedbackGenerator.call(report)
+      if feedback_result[:success]
+        output = feedback_result.dig(:response, :output)
+        lines << output unless output.empty?
+      end
+
+      lines.join("\n")
+    end
+    private_class_method :format_delta_report
+
+    # Builds iteration timeline lines from the result response.
+    #
+    # @param result [Hash] Eval result envelope.
+    # @return [Array<String>] Lines to append, or empty array.
+    def self.build_iteration_lines(result)
+      baseline = result.dig(:response, :baseline_iterations) || []
+      context = result.dig(:response, :context_iterations) || []
+      baseline_empty = baseline.empty?
+      context_empty = context.empty?
+      lines = []
+
+      lines << Services::IterationFormatter.format('BASELINE ITERATIONS', baseline) unless baseline_empty
+      lines << Services::IterationFormatter.format('CONTEXT ITERATIONS', context) unless context_empty
+      lines << '' unless baseline_empty && context_empty
+
+      lines
+    end
+    private_class_method :build_iteration_lines
+  end
+end