ruby-skill-bench 0.1.0

data/lib/skill_bench/config.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require_relative 'config/defaults'
+require_relative 'config/store'
+require_relative 'config/applier'
+require_relative 'config/json_loader'
+require_relative 'config/env_overrides'
+require_relative 'config/facade_readers'
+require_relative 'config/facade_writers'
+
+module SkillBench
+  # Centralized configuration for the SkillBench system.
+  # Supports hierarchical loading: Defaults < Home JSON < Local JSON < ENV Variables.
+  class Config
+    # File name used for local and home evaluator configuration.
+    CONFIG_FILENAME = 'skill-bench.json'
+
+    class << self
+      include Config::FacadeReaders
+      include Config::FacadeWriters
+
+      # Returns the mutable configuration store behind the facade.
+      # Lazily initializes configuration on first access.
+      #
+      # @return [Config::Store] configuration state store
+      def store
+        @store ||= Config::Store.new
+      end
+
+      # Returns the default configuration.
+      #
+      # @return [Hash] default configuration hash
+      def defaults
+        Config::Defaults.call
+      end
+
+      # Applies configuration from the store.
+      #
+      # @return [Hash] applied configuration
+      def apply
+        Config::Applier.call(store.to_h)
+      end
+
+      # Loads configuration from a JSON file.
+      #
+      # @param path [String] Path to JSON file
+      # @return [Hash] loaded configuration
+      def load_from_file(path)
+        Config::JsonLoader.call(path)
+      end
+
+      # Saves configuration to a JSON file.
+      #
+      # @param path [String] Path to JSON file
+      # @param config [Hash] Configuration to save
+      # @return [void]
+      def save_to_file(path, config)
+        Config::FacadeWriters.save_to_file(path, config)
+      end
+
+      # Returns configuration overrides from environment variables.
+      #
+      # @return [Hash] environment-based overrides
+      def env_overrides
+        Config::EnvOverrides.call
+      end
+
+      # Resets and reloads configuration from all sources.
+      # Pipeline: Defaults → Home JSON → Local JSON → ENV overrides.
+      #
+      # @return [void]
+      def reset
+        @store = Config::Store.new
+        apply_defaults
+        apply_json_config(home_config_path)
+        apply_json_config(Pathname.new(Dir.pwd).join(CONFIG_FILENAME))
+        apply_env_overrides
+      end
+
+      # Sets up configuration with a block.
+      #
+      # @yieldparam config [Config::Store] Configuration store for modification
+      # @return [void]
+      def setup
+        yield store
+      end
+
+      # Returns allowed commands from configuration.
+      #
+      # @return [Array<String>, nil] List of allowed commands
+      def allowed_commands
+        store.allowed_commands
+      end
+
+      # Returns max execution time from configuration.
+      #
+      # @return [Integer] Maximum execution time in seconds
+      def max_execution_time
+        store.max_execution_time || 30
+      end
+
+      # Returns the current LLM provider name.
+      #
+      # @return [Symbol] Current provider name
+      def current_llm_provider
+        store.current_llm_provider || :openai
+      end
+
+      # Sets the current LLM provider.
+      #
+      # @param provider [Symbol] Provider name
+      # @return [void]
+      def current_llm_provider=(provider)
+        store.assign_current_llm_provider(provider)
+      end
+
+      # Returns LLM providers configuration.
+      #
+      # @return [Hash] Providers configuration
+      def llm_providers_config
+        store.llm_providers_config || {}
+      end
+
+      # Returns API key from configuration.
+      #
+      # @return [String, nil] API key
+      def api_key
+        store.api_key
+      end
+
+      # Returns model from configuration.
+      #
+      # @return [String, nil] Model name
+      def model
+        store.model
+      end
+
+      private
+
+      def home_config_path
+        Pathname.new(Dir.home).join(CONFIG_FILENAME)
+      rescue ArgumentError
+        nil
+      end
+
+      def apply_defaults
+        result = Config::Defaults.call
+        return unless result[:success]
+
+        Config::Applier.call(store: store, data: result[:response][:config])
+      end
+
+      def apply_json_config(path)
+        return unless path
+        return unless File.exist?(path)
+
+        result = Config::JsonLoader.call(path)
+        return unless result[:success]
+
+        Config::Applier.call(store: store, data: result[:response][:config])
+      end
+
+      def apply_env_overrides
+        result = Config::EnvOverrides.call
+        return unless result[:success]
+
+        store.apply_provider_config(result[:response][:overrides])
+      end
+    end
+  end
+end

data/lib/skill_bench/criteria.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  # Loads, validates, and represents evaluation criteria from criteria.json.
+  #
+  # Merges eval-specific dimension overrides with built-in default descriptions
+  # and validates that dimension weights sum to exactly 100.
+  class Criteria
+    attr_reader :dimensions, :context, :pass_threshold, :minimum_delta
+
+    # Loads criteria from a JSON file.
+    #
+    # @param path [String] Path to the criteria.json file.
+    # @return [Hash] Service response with :success and :response keys.
+    # @raise [TypeError] when the provided path is not a string.
+    def self.call(path:)
+      new(path:).call
+    end
+
+    # Returns an empty criteria with default thresholds and no dimensions.
+    #
+    # @return [SkillBench::Criteria] An empty criteria instance.
+    def self.empty
+      new(path: '').tap do |criteria|
+        criteria.instance_variable_set(:@context, '')
+        criteria.instance_variable_set(:@pass_threshold, 70)
+        criteria.instance_variable_set(:@minimum_delta, 10)
+        criteria.instance_variable_set(:@dimensions, [])
+      end
+    end
+
+    # @param path [String] Path to the criteria.json file.
+    def initialize(path:)
+      @path = path
+    end
+
+    # Loads and validates the criteria file.
+    #
+    # @return [Hash] Service response with criteria or error.
+    def call
+      raw = load_json
+      return raw unless raw[:success]
+
+      data = raw[:response][:data]
+      raw_dimensions = data['dimensions'] || data[:dimensions] || []
+      dimensions = build_dimensions(raw_dimensions)
+
+      core_validation = validate_core_dimensions(dimensions)
+      return core_validation unless core_validation[:success]
+
+      validation = validate_dimensions(dimensions)
+      return validation unless validation[:success]
+
+      assign_attributes(data, dimensions)
+
+      { success: true, response: { criteria: self } }
+    rescue StandardError => e
+      SkillBench::ErrorLogger.log_error(e, 'Criteria Load Error')
+      { success: false, response: { error: { message: e.message } } }
+    end
+
+    private
+
+    attr_reader :path
+
+    def load_json
+      return missing_file_result unless File.exist?(path)
+
+      data = JSON.parse(File.read(path))
+      { success: true, response: { data: data } }
+    rescue JSON::ParserError => e
+      { success: false, response: { error: { message: "Invalid JSON: #{e.message}" } } }
+    end
+
+    def missing_file_result
+      { success: false, response: { error: { message: "Criteria file #{path} does not exist" } } }
+    end
+
+    def build_dimensions(raw_dimensions)
+      defaults = DEFAULT_DIMENSIONS.to_h { |d| [d.name, d] }
+
+      raw_dimensions.map do |raw|
+        name = raw['name'] || raw[:name]
+        default = defaults[name]
+        description = raw['description'] || raw[:description] || default&.description || ''
+
+        Dimension.new(
+          name: name,
+          description: description,
+          max_score: raw['max_score'] || raw[:max_score]
+        )
+      end
+    end
+
+    def validate_dimensions(dimensions)
+      invalid = invalid_dimensions(dimensions)
+      return invalid_max_score_result(invalid) unless invalid.empty?
+
+      total = dimensions.sum(&:max_score)
+      return { success: true, response: {} } if total == 100
+
+      {
+        success: false,
+        response: { error: { message: "Dimension max_scores must sum to 100, got #{total}" } }
+      }
+    end
+
+    def invalid_dimensions(dimensions)
+      dimensions.reject { |d| d.max_score.is_a?(Numeric) }
+    end
+
+    def invalid_max_score_result(invalid)
+      names = invalid.map(&:name).join(', ')
+      {
+        success: false,
+        response: { error: { message: "Dimensions missing or invalid max_score: #{names}" } }
+      }
+    end
+
+    def validate_core_dimensions(dimensions)
+      core_names = DEFAULT_DIMENSIONS.map(&:name)
+      present_names = dimensions.map(&:name)
+      missing = core_names - present_names
+      return { success: true, response: {} } if missing.empty?
+
+      {
+        success: false,
+        response: { error: { message: "missing required core dimensions: #{missing.join(', ')}" } }
+      }
+    end
+
+    def assign_attributes(data, dimensions)
+      @context = data['context'] || data[:context] || ''
+      @pass_threshold = [data['pass_threshold'], data[:pass_threshold]].compact.first || 70
+      @minimum_delta = [data['minimum_delta'], data[:minimum_delta]].compact.first || 10
+      @dimensions = dimensions
+    end
+  end
+end

data/lib/skill_bench/delta_report.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Computes baseline vs context deltas per dimension and determines verdict.
+  #
+  # Verdict is true when context score meets pass_threshold AND
+  # the total delta meets minimum_delta.
+  class DeltaReport
+    attr_reader :deltas, :baseline_total, :context_total, :verdict, :baseline_scores, :context_scores, :criteria,
+                :baseline_dimensions, :context_dimensions
+
+    # Computes deltas and verdict from baseline and context judge responses.
+    #
+    # @param baseline [Hash] Baseline judge dimensions hash.
+    # @param context [Hash] Context judge dimensions hash.
+    # @param criteria [SkillBench::Criteria] The eval criteria with thresholds.
+    # @return [Hash] Service response with delta_report or error.
+    def self.call(baseline:, context:, criteria:)
+      new(baseline:, context:, criteria:).call
+    end
+
+    # @param baseline [Hash] Baseline dimensions.
+    # @param context [Hash] Context dimensions.
+    # @param criteria [SkillBench::Criteria] Eval criteria.
+    def initialize(baseline:, context:, criteria:)
+      @baseline = baseline
+      @context = context
+      @criteria = criteria
+      @deltas = {}
+    end
+
+    # Computes deltas and determines verdict.
+    #
+    # @return [Hash] Service response with delta_report or error.
+    def call
+      return mismatch_result unless dimensions_match?
+
+      @baseline_dimensions = deep_copy_dimensions(baseline)
+      @context_dimensions = deep_copy_dimensions(context)
+      @baseline_scores = extract_scores(baseline)
+      @context_scores = extract_scores(context)
+      compute_totals
+      compute_deltas
+      determine_verdict
+
+      { success: true, response: { delta_report: self } }
+    rescue StandardError => e
+      SkillBench::ErrorLogger.log_error(e, 'DeltaReport Error')
+      { success: false, response: { error: { message: e.message } } }
+    end
+
+    private
+
+    attr_reader :baseline, :context
+
+    def dimensions_match?
+      baseline.keys.sort == context.keys.sort
+    end
+
+    def mismatch_result
+      { success: false, response: { error: { message: 'Baseline and context dimension names mismatch' } } }
+    end
+
+    def compute_totals
+      @baseline_total = baseline.values.sum { |v| extract_score(v) }
+      @context_total = context.values.sum { |v| extract_score(v) }
+    end
+
+    def compute_deltas
+      baseline.each do |name, base|
+        base_score = extract_score(base)
+        context_score = extract_score(context[name])
+        @deltas[name] = context_score - base_score
+      end
+    end
+
+    def extract_score(dim)
+      dim[:score] || dim['score']
+    end
+
+    def extract_scores(dimensions)
+      dimensions.transform_values { |dim| extract_score(dim) }
+    end
+
+    def deep_copy_dimensions(dimensions)
+      dimensions.transform_values(&:dup)
+    end
+
+    def determine_verdict
+      @verdict = context_total >= criteria.pass_threshold && total_delta >= criteria.minimum_delta
+    end
+
+    def total_delta
+      context_total - baseline_total
+    end
+  end
+end

data/lib/skill_bench/dimension.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Value object representing a scoring dimension for evaluation.
+  #
+  # Dimensions are used by the judge to score agent output across
+  # different aspects such as correctness, code quality, and skill adherence.
+  class Dimension
+    attr_reader :name, :description, :max_score
+
+    # @param name [String] The machine-friendly identifier for the dimension.
+    # @param description [String] Human-readable explanation of what the dimension measures.
+    # @param max_score [Integer, nil] Maximum score this dimension can contribute. Nil in defaults.
+    def initialize(name:, description:, max_score:)
+      @name = name
+      @description = description
+      @max_score = max_score
+    end
+
+    # Compares two Dimension instances for equality.
+    #
+    # @param other [Object] The object to compare against.
+    # @return [Boolean] true when all attributes match.
+    def ==(other)
+      other.is_a?(Dimension) &&
+        name == other.name &&
+        description == other.description &&
+        max_score == other.max_score
+    end
+    alias eql? ==
+
+    # Computes a hash code based on attributes.
+    #
+    # @return [Integer] The hash code.
+    def hash
+      [name, description, max_score].hash
+    end
+  end
+
+  # Canonical dimensions used when eval authors do not override descriptions.
+  # Weights (max_score) are nil here; the eval's criteria.json provides them.
+  DEFAULT_DIMENSIONS = [
+    Dimension.new(
+      name: 'correctness',
+      description: 'Does the output fulfill the task requirements? Are all specified behaviors present and correct?',
+      max_score: nil
+    ),
+    Dimension.new(
+      name: 'skill_adherence',
+      description: 'Did the agent follow the specific patterns, hard gates, and workflows defined in the skill?',
+      max_score: nil
+    ),
+    Dimension.new(
+      name: 'code_quality',
+      description: 'Is the code clean, well-structured, free of smells, follows SRP, and avoids duplication?',
+      max_score: nil
+    ),
+    Dimension.new(
+      name: 'test_coverage',
+      description: 'Are there meaningful tests? Do they test the right things? Are they following TDD/best practices from the skill?',
+      max_score: nil
+    ),
+    Dimension.new(
+      name: 'documentation',
+      description: 'Is there adequate YARD documentation, clear intent, and helpful inline comments where needed?',
+      max_score: nil
+    )
+  ].freeze
+end

data/lib/skill_bench/error_logger.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Shared error logging mixin for service objects.
+  # Logs error message and backtrace to Rails.logger or stderr.
+  module ErrorLogger
+    # Logs an error with message and backtrace.
+    #
+    # @param error [StandardError] The exception to log
+    # @param prefix [String] Optional prefix for the log message
+    # @return [void]
+    def log_error(error, prefix = nil)
+      message = prefix ? "#{prefix}: #{error.message}" : error.message
+      backtrace = error.backtrace&.first(5)&.join("\n") || '(no backtrace)'
+
+      return if skip_stderr_output?
+
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger.error(message)
+        Rails.logger.error(backtrace)
+      else
+        warn(message)
+        warn(backtrace)
+      end
+    end
+
+    # @return [Boolean] true when stderr should be skipped (test mode without explicit capture).
+    def skip_stderr_output?
+      defined?(Minitest) && !$stderr.is_a?(StringIO)
+    end
+
+    module_function :log_error
+    module_function :skip_stderr_output?
+  end
+end

data/lib/skill_bench/evaluate_command.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require_relative 'runner'
+require_relative 'services/option_parser_service'
+require_relative 'services/result_printer_service'
+require_relative 'services/output_persistence_service'
+
+module SkillBench
+  # Implements the `skill-bench run` CLI command.
+  # Orchestrates option parsing, evaluation execution, result printing, and output persistence.
+  # @deprecated Use {SkillBench::Cli::RunCommand} and {SkillBench::Services::RunnerService} instead.
+  class EvaluateCommand
+    # Parses arguments, runs the evaluator, prints the report, and records history.
+    #
+    # @param argv [Array<String>] Raw CLI arguments.
+    # @param stdout [#puts, #write] Output stream for user-visible messages.
+    # @return [Integer] Shell-compatible exit code.
+    # @raise [OptionParser::ParseError] when invalid CLI flags are provided.
+    # @raise [SystemCallError] if writing output fails.
+    def self.call(argv, stdout: $stdout)
+      new(argv, stdout: stdout).call
+    end
+
+    # @param argv [Array<String>] Raw CLI arguments.
+    # @param stdout [#puts, #write] Output stream for user-visible messages.
+    def initialize(argv, stdout:)
+      @argv = argv
+      @stdout = stdout
+      @options = nil
+    end
+
+    # Executes the command by orchestrating service objects.
+    #
+    # @return [Integer] Shell-compatible exit code.
+    # @raise [OptionParser::ParseError] when invalid CLI flags are provided.
+    # @raise [SystemCallError] when the optional JSON output file cannot be written.
+    def call
+      return 1 unless parse_options? && validate_options?
+
+      result = run_evaluation
+      return 1 unless result[:success]
+
+      return 1 unless persist_output?(result)
+
+      SkillBench::HistoryRecorder.record(
+        result,
+        source_path: result[:source_path],
+        model: SkillBench::Config.model
+      )
+
+      0
+    end
+
+    private
+
+    def parse_options?
+      options_result = Services::OptionParserService.call(@argv)
+      @options = options_result[:response]
+
+      unless options_result[:success]
+        @stdout.puts "Error: #{@options[:error][:message]}"
+        return false
+      end
+
+      true
+    end
+
+    def validate_options?
+      eval_path = @options[:eval]
+      return true if eval_path
+
+      @stdout.puts 'Error: The --eval option is required.'
+      @stdout.puts 'Example: bin/evaluate -e evals/skills/infrastructure/rails-api-versioning/api-versioning-with-controller-inheritan'
+      false
+    end
+
+    def run_evaluation
+      skill_option = @options[:skill]
+      eval_path = safe_expand_path(@options[:eval])
+      skill_path = skill_option ? safe_expand_path(skill_option) : nil
+
+      result = SkillBench::Runner.call(
+        eval_folder_path: eval_path,
+        skill_path: skill_path
+      )
+      Services::ResultPrinterService.call(result, stdout: @stdout)
+      result
+    end
+
+    def persist_output?(result)
+      output_result = Services::OutputPersistenceService.call(result, output_path: @options[:output])
+      output_response = output_result[:response]
+      message = output_response[:message]
+
+      if output_result[:success]
+        @stdout.puts(message) if message
+        true
+      else
+        @stdout.puts "Error saving report: #{output_response[:error][:message]}"
+        false
+      end
+    end
+
+    def safe_expand_path(path)
+      expanded = File.expand_path(path)
+      base = File.expand_path(Dir.pwd)
+
+      real_expanded = File.exist?(expanded) ? File.realpath(expanded) : expanded
+      real_base = File.realpath(base)
+
+      relative = Pathname.new(real_expanded).relative_path_from(Pathname.new(real_base)).to_s
+      raise ArgumentError, "Path '#{path}' resolves outside the current working directory" if relative.start_with?('..')
+
+      expanded
+    rescue Errno::ENOENT, Errno::EACCES => e
+      raise ArgumentError, "Path '#{path}' is not accessible: #{e.message}"
+    end
+  end
+end