ruby-skill-bench 0.1.0

data/lib/skill_bench/clients/request_builder.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+module SkillBench
+  module Clients
+    # Builds and executes HTTP requests to LLM provider APIs.
+    # Encapsulates Faraday connection setup and request execution.
+    class RequestBuilder
+      DEFAULT_OPEN_TIMEOUT = 10
+      DEFAULT_TIMEOUT = 120
+
+      # Creates a Faraday connection with JSON middleware.
+      #
+      # @param base_url [String] The API base URL
+      # @param open_timeout [Integer] Connection open timeout in seconds
+      # @param timeout [Integer] Request timeout in seconds
+      # @return [Faraday::Connection] Configured Faraday connection
+      def self.build_connection(base_url, open_timeout: DEFAULT_OPEN_TIMEOUT, timeout: DEFAULT_TIMEOUT)
+        Faraday.new(url: base_url) do |f|
+          f.request :json
+          f.response :json
+          f.options.open_timeout = open_timeout
+          f.options.timeout = timeout
+        end
+      end
+
+      # Executes a POST request to the LLM API.
+      #
+      # @param connection [Faraday::Connection] The Faraday connection
+      # @param path [String] The request path
+      # @param headers [Hash] Request headers
+      # @param body [Hash] Request body
+      # @return [Faraday::Response] The HTTP response
+      def self.execute(connection, path, headers:, body:)
+        connection.post(path) do |req|
+          req.headers.update(headers)
+          req.body = body.to_json
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/response_error_handler.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Clients
+    # Handles error responses and logging for LLM provider clients.
+    # Encapsulates error formatting, logging, and exception handling.
+    class ResponseErrorHandler
+      API_FAILED = 'API Request failed'
+
+      # Creates an error response for failed HTTP requests.
+      #
+      # @param response [Faraday::Response] The HTTP response
+      # @param parsed [Hash] Parsed response body
+      # @param usage_extractor [Proc] Block to extract usage data
+      # @return [Hash] Standardized error response
+      def self.failure_response(response, parsed, &usage_extractor)
+        error_msg = "#{API_FAILED}: #{response.status}"
+        detail = parsed.is_a?(Hash) ? (parsed[:error] || parsed['error'] || parsed) : parsed
+
+        if detail.is_a?(Hash) && (detail[:message] || detail['message'])
+          error_msg += " - #{detail[:message] || detail['message']}"
+        elsif !detail.to_s.empty?
+          error_msg += " - #{detail}"
+        end
+
+        {
+          success: false,
+          result: error_msg,
+          usage: usage_extractor.call(parsed),
+          response: { error: { message: error_msg } },
+          status: 'error',
+          code: response.status
+        }
+      end
+
+      # Creates an error response when the LLM response has no message content.
+      #
+      # @param response [Faraday::Response] The HTTP response
+      # @param parsed [Hash] Parsed response body
+      # @param usage_extractor [Proc] Block to extract usage data
+      # @return [Hash] Standardized error response
+      def self.missing_message_response(response, parsed, &usage_extractor)
+        error_msg = 'LLM response missing message content'
+        {
+          success: false,
+          result: error_msg,
+          usage: usage_extractor.call(parsed),
+          response: { error: { message: error_msg } },
+          status: 'error',
+          code: response.status
+        }
+      end
+
+      # Handles an exception by logging and returning a standardized error response.
+      #
+      # @param error [StandardError] The exception that occurred
+      # @param type [String] The error type label
+      # @return [Hash] Standardized error response
+      def self.handle_exception(error, type)
+        log_error(error)
+        { success: false, result: "#{type}: #{error.message}", status: 'error' }
+      end
+
+      # Logs an error message and backtrace to Rails.logger or stderr.
+      #
+      # @param error [StandardError] The exception to log
+      # @return [void]
+      def self.log_error(error)
+        SkillBench::ErrorLogger.log_error(error)
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/response_parser.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  module Clients
+    # Parses LLM provider responses and extracts messages and usage data.
+    # Handles JSON parsing, message extraction, and validation.
+    class ResponseParser
+      # Parses the response body into a Hash.
+      #
+      # @param response [Faraday::Response] The HTTP response
+      # @return [Hash] Parsed response body
+      def self.parse_body(response)
+        return response.body if response.body.is_a?(Hash)
+        return { error: { message: response.body.to_s } } if response.body.is_a?(Array)
+
+        JSON.parse(response.body, symbolize_names: true)
+      rescue JSON::ParserError
+        { error: { message: response.body.to_s } }
+      end
+
+      # Strips markdown code fences from a string if present.
+      #
+      # @param text [String] The text to clean
+      # @return [String] Cleaned text
+      def self.strip_markdown_fences(text)
+        return text unless text.is_a?(String)
+
+        if text.start_with?('```')
+          lines = text.each_line.to_a
+          lines.shift if lines.first&.strip&.start_with?('```')
+          lines.pop if lines.last&.strip == '```'
+          lines.join.strip
+        else
+          text
+        end
+      end
+
+      # Checks if a message is valid (has content or tool calls).
+      #
+      # @param message [Hash, String, nil] The message to validate
+      # @return [Boolean] True if the message is valid
+      def self.valid_message?(message)
+        return false if message.nil?
+
+        content = extract_content(message)
+        tool_calls = extract_tool_calls(message)
+
+        !content.nil? || !Array(tool_calls).empty?
+      end
+
+      # Extracts the content from a message.
+      #
+      # @param message [Hash, String] The message
+      # @return [String, nil] The content or nil
+      def self.extract_content(message)
+        return message unless message.is_a?(Hash)
+
+        message[:content] || message['content']
+      end
+
+      # Extracts tool calls from a message.
+      #
+      # @param message [Hash] The message
+      # @return [Array, nil] The tool calls or nil
+      def self.extract_tool_calls(message)
+        return nil unless message.is_a?(Hash)
+
+        message[:tool_calls] || message['tool_calls']
+      end
+
+      # Extracts the message from an OpenAI-compatible response body.
+      #
+      # @param body [Hash] The parsed response body
+      # @return [Hash, nil] The message or nil
+      def self.extract_openai_message(body)
+        choices = body[:choices] || body['choices']
+        return nil unless choices&.any?
+
+        choices.first[:message] || choices.first['message']
+      end
+
+      # Extracts usage data from an OpenAI-compatible response.
+      #
+      # @param body [Hash] The parsed response body
+      # @return [Hash] Usage data
+      def self.extract_openai_usage(body)
+        body[:usage] || body['usage'] || {}
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/retry_handler.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require_relative '../error_logger'
+
+module SkillBench
+  module Clients
+    # Service object for retrying HTTP requests with exponential backoff.
+    # Retries on transient errors (429, 503). Raises permanent errors immediately.
+    # Returns the block result on success.
+    class RetryHandler
+      RETRYABLE_STATUSES = [429, 503].freeze
+
+      MAX_DELAY = 30 # Maximum delay cap in seconds
+
+      # Executes the given block with retry logic.
+      #
+      # @param max_attempts [Integer] Maximum number of attempts (default: 3).
+      # @param base_delay [Numeric] Base delay in seconds before first retry (doubles each attempt).
+      # @yield The request block to execute.
+      # @return [Object] The block's return value on success.
+      # @raise [Faraday::Error] On non-retryable errors or after exhausting retries.
+      # @raise [ArgumentError] if no block is given or max_attempts < 1.
+      def self.call(max_attempts: 3, base_delay: 1, &block)
+        raise ArgumentError, 'RetryHandler requires a block' unless block
+        raise ArgumentError, 'max_attempts must be >= 1' if max_attempts < 1
+
+        new(max_attempts:, base_delay:, block:).call
+      end
+
+      # @param max_attempts [Integer] Maximum number of attempts.
+      # @param base_delay [Numeric] Base delay before first retry.
+      # @param block [Proc] The request block to execute.
+      def initialize(max_attempts:, base_delay:, block:)
+        @max_attempts = max_attempts
+        @base_delay = base_delay
+        @block = block
+      end
+
+      # Executes the block with retry logic.
+      #
+      # @return [Object] The block's return value on success.
+      # @raise [Faraday::Error] On non-retryable errors or after exhausting retries.
+      def call
+        attempt = 0
+
+        loop do
+          attempt += 1
+          return @block.call
+        rescue Faraday::Error => e
+          status = extract_status(e)
+          raise e unless retryable?(status, attempt)
+
+          delay = compute_delay(attempt)
+          wait(delay)
+        end
+      end
+
+      private
+
+      def retryable?(status, attempt)
+        RETRYABLE_STATUSES.include?(status) && attempt < @max_attempts
+      end
+
+      def compute_delay(attempt)
+        [@base_delay * (2**(attempt - 1)), MAX_DELAY].min
+      end
+
+      def extract_status(error)
+        error.respond_to?(:response_status) ? error.response_status : 0
+      end
+
+      def wait(delay)
+        sleep(delay)
+      end
+    end
+  end
+end

data/lib/skill_bench/commands/eval_new.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module SkillBench
+  module Commands
+    # Handles the `skill-bench eval new` command
+    class EvalNew
+      # Allowed runtime values for eval scaffolding.
+      ALLOWED_RUNTIMES = %w[ruby rails].freeze
+
+      # Run the eval new command
+      # @param name [String] Eval name
+      # @param runtime [String] "ruby" or "rails" (default: ruby)
+      # @return [void]
+      # @raise [ArgumentError] if runtime is not in ALLOWED_RUNTIMES.
+      def self.run(name:, runtime: 'ruby')
+        raise ArgumentError, "Unsupported runtime '#{runtime}'. Allowed: #{ALLOWED_RUNTIMES.join(', ')}" unless ALLOWED_RUNTIMES.include?(runtime)
+
+        eval_path = File.join('evals', name)
+        FileUtils.mkdir_p(eval_path)
+
+        create_task_md(eval_path, name)
+        create_criteria_json(eval_path, runtime)
+        create_rails_files(eval_path, name) if runtime == 'rails'
+      end
+
+      # Create task.md for the eval
+      # @param path [String] Eval directory path
+      # @param name [String] Eval name
+      # @return [void]
+      def self.create_task_md(path, name)
+        File.write(File.join(path, 'task.md'), task_template(name))
+      end
+
+      # Create criteria.json for the eval
+      # @param path [String] Eval directory path
+      # @param runtime [String] Runtime type
+      # @return [void]
+      def self.create_criteria_json(path, runtime)
+        criteria = default_criteria(runtime)
+        File.write(File.join(path, 'criteria.json'), JSON.pretty_generate(criteria))
+      end
+
+      # Generate task.md template
+      # @param name [String] Eval name
+      # @return [String] Markdown template
+      def self.task_template(name)
+        <<~MARKDOWN
+          # Eval: #{name}
+
+          ## Task
+          Describe the task for the agent here.
+
+          ## Success Criteria
+          Define what constitutes a successful completion.
+        MARKDOWN
+      end
+
+      # Generate default criteria hash.
+      #
+      # @param runtime [String] Runtime type.
+      # @return [Hash] Criteria configuration in the new format.
+      def self.default_criteria(runtime)
+        {
+          context: "Evaluate #{runtime} task",
+          dimensions: [
+            { name: 'correctness', max_score: 30 },
+            { name: 'skill_adherence', max_score: 25 },
+            { name: 'code_quality', max_score: 20 },
+            { name: 'test_coverage', max_score: 15 },
+            { name: 'documentation', max_score: 10 }
+          ],
+          pass_threshold: 70,
+          minimum_delta: 10
+        }
+      end
+
+      # Create Rails-specific files for the eval
+      # @param path [String] Eval directory path
+      # @param _name [String] Eval name
+      # @return [void]
+      def self.create_rails_files(path, _name)
+        File.write(File.join(path, 'rails_helper.rb'), "require 'rails_helper'\n")
+      end
+    end
+  end
+end

data/lib/skill_bench/commands/init.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../clients/provider_schemas'
+
+module SkillBench
+  module Commands
+    # Handles the `skill-bench init` command.
+    # Generates a skill-bench.json config file with single-provider settings.
+    class Init
+      # Run the init command to generate config.
+      #
+      # @param provider [Symbol] LLM provider name (e.g., :openai, :gemini)
+      # @param force [Boolean] Whether to overwrite an existing config file.
+      # @return [void]
+      # @raise [RuntimeError] if config file exists and force is false
+      # @raise [ArgumentError] if provider is not registered
+      def self.run(provider:, force: false)
+        raise "Config file '#{SkillBench::Config::CONFIG_FILENAME}' already exists. Use --force to overwrite." if File.exist?(SkillBench::Config::CONFIG_FILENAME) && !force
+
+        config = config_for_provider(provider)
+        File.write(SkillBench::Config::CONFIG_FILENAME, JSON.pretty_generate(config))
+      end
+
+      # Generates configuration hash for a specific provider.
+      #
+      # @param provider [Symbol] LLM provider name
+      # @return [Hash] Single-provider configuration
+      # @raise [ArgumentError] if provider is not registered
+      def self.config_for_provider(provider)
+        {
+          provider: provider,
+          max_execution_time: 30,
+          config: SkillBench::Clients::ProviderSchemas.for(provider)
+        }
+      end
+    end
+  end
+end

data/lib/skill_bench/commands/run.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative '../services/runner_service'
+
+module SkillBench
+  module Commands
+    # Handles the `skill-bench run` command
+    class Run
+      # Run an eval with specified skill(s)
+      # @param eval_name [String] Name of eval to run (e.g., 'test-eval' or 'evals/test-eval')
+      # @param skill_names [Array<String>] Names of skills to use
+      # @return [Hash] Result with pass/fail and score
+      def self.run(eval_name:, skill_names:)
+        Services::RunnerService.call(
+          eval_name: eval_name,
+          skill_names: skill_names
+        )
+      end
+    end
+  end
+end

data/lib/skill_bench/commands/skill_new.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require_relative '../rails/skill_templates'
+
+module SkillBench
+  module Commands
+    # Handles the `skill-bench skill new` command
+    class SkillNew
+      # Run the skill new command
+      # @param name [String] Skill name
+      # @param mode [String] "simple", "advanced", or "rails"
+      # @param template [String] Rails template type (service_object, concern, active_record_model)
+      # @return [void]
+      # @raise [ArgumentError] if mode is invalid
+      def self.run(name:, mode: 'simple', template: 'service_object')
+        skill_path = File.join('skills', name)
+        FileUtils.mkdir_p(skill_path)
+
+        case mode
+        when 'simple'
+          create_simple_skill(skill_path, name)
+        when 'advanced'
+          create_advanced_skill(skill_path, name)
+        when 'rails'
+          create_rails_skill(skill_path, name, template)
+        else
+          raise ArgumentError, "Invalid mode: #{mode}. Use 'simple', 'advanced', or 'rails'."
+        end
+      end
+
+      # Create a simple skill with SKILL.md
+      # @param path [String] Skill directory path
+      # @param name [String] Skill name
+      # @return [void]
+      def self.create_simple_skill(path, name)
+        File.write(File.join(path, 'SKILL.md'), simple_skill_template(name))
+      end
+
+      # Create an advanced skill with Ruby class
+      # @param path [String] Skill directory path
+      # @param name [String] Skill name
+      # @return [void]
+      def self.create_advanced_skill(path, name)
+        File.write(File.join(path, 'skill.rb'), advanced_skill_template(name))
+      end
+
+      # Generate simple skill template
+      # @param name [String] Skill name
+      # @return [String] Markdown template
+      def self.simple_skill_template(name)
+        <<~MARKDOWN
+          # Skill: #{name}
+
+          ## Description
+          Add skill description here.
+
+          ## Context
+          Add context injection content here.
+
+          ## Workflow
+          Add workflow steps here.
+        MARKDOWN
+      end
+
+      # Convert snake_case to CamelCase
+      # @param string [String] String to convert
+      # @return [String] CamelCase string
+      def self.camelize(string)
+        string.split(/[_\s]+/).map(&:capitalize).join
+      end
+
+      # Generate advanced skill template
+      # @param name [String] Skill name
+      # @return [String] Ruby class template
+      def self.advanced_skill_template(name)
+        class_name = camelize(name)
+        <<~RUBY
+          # frozen_string_literal: true
+
+          module SkillBench
+            module Skills
+              class #{class_name}
+                def initialize; end
+
+                def call
+                  # Implement skill logic here
+                end
+              end
+            end
+          end
+        RUBY
+      end
+
+      RAILS_TEMPLATES = {
+        'service_object' => 'service.rb',
+        'concern' => 'concern.rb',
+        'active_record_model' => 'model.rb'
+      }.freeze
+
+      # Create a Rails skill using templates
+      # @param path [String] Skill directory path
+      # @param name [String] Skill name
+      # @param template [String] Template type (service_object, concern, active_record_model)
+      # @return [void]
+      def self.create_rails_skill(path, name, template)
+        file_name = RAILS_TEMPLATES[template]
+        raise ArgumentError, "Invalid template: #{template}. Use one of: #{RAILS_TEMPLATES.keys.join(', ')}." unless file_name
+
+        content = Rails::SkillTemplates.public_send(template.to_sym, name)
+        File.write(File.join(path, file_name), content)
+      end
+    end
+  end
+end

data/lib/skill_bench/config/applier.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class Config
+    # Applies normalized configuration hashes to a mutable store.
+    class Applier
+      # Applies configuration values to a store.
+      #
+      # @param store [Store] mutable configuration store
+      # @param data [Hash] normalized configuration values
+      # @return [Hash] result envelope with applied status
+      def self.call(store:, data:)
+        new(store:, data:).call
+      end
+
+      # Initializes the applier.
+      #
+      # @param store [Store] mutable configuration store
+      # @param data [Hash] normalized configuration values
+      # @return [Applier] an applier instance
+      def initialize(store:, data:)
+        @store = store
+        @data = data
+      end
+
+      # Applies configuration values to the configured store.
+      #
+      # @return [Hash] result envelope with applied status
+      def call
+        apply_scalar_values
+        apply_provider_values
+        { success: true, response: { applied: true } }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Applier Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      def apply_scalar_values
+        assign_current_provider
+        @store.assign_max_execution_time(@data[:max_execution_time]) if @data.key?(:max_execution_time)
+        @store.assign_allowed_commands(@data[:allowed_commands]) if @data.key?(:allowed_commands)
+      end
+
+      def apply_provider_values
+        if @data.key?(:llm_providers_config)
+          @store.replace_provider_config(copied_provider_config)
+        else
+          @store.apply_provider_config(@data[:providers] || {})
+        end
+      end
+
+      def assign_current_provider
+        provider = @data.fetch(:current_llm_provider) { return }
+        provider_name = provider.to_s.strip
+        return if provider_name.empty?
+
+        @store.assign_current_llm_provider(provider_name.to_sym)
+      end
+
+      def copied_provider_config
+        @data[:llm_providers_config].transform_values(&:dup)
+      end
+    end
+  end
+end

data/lib/skill_bench/config/defaults.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class Config
+    # Builds the default evaluator configuration state.
+    class Defaults
+      # Returns the default configuration values.
+      #
+      # @return [Hash] result envelope with default provider, timeout, command, and provider settings
+      def self.call
+        { success: true, response: { config: config } }
+      end
+
+      # Builds the raw default configuration hash.
+      #
+      # @return [Hash] default provider, timeout, command, and provider settings
+      def self.config
+        {
+          current_llm_provider: :openai,
+          max_execution_time: 30,
+          allowed_commands: nil,
+          llm_providers_config: {
+            openai: { api_key: nil, model: 'gpt-4o' },
+            anthropic: { api_key: nil, model: 'claude-sonnet-4-20250514' },
+            gemini: {
+              api_key: nil,
+              model: 'gemini-1.5-flash-latest',
+              location: 'us-central1',
+              project_id: nil
+            },
+            ollama: { api_key: nil, model: 'qwen:7b', base_url: nil },
+            azure: { api_key: nil, model: 'gpt-4', endpoint: nil, api_version: nil },
+            groq: { api_key: nil, model: 'llama-3.3-70b-versatile' },
+            deepseek: { api_key: nil, model: 'deepseek-chat' },
+            opencode: { api_key: nil, model: 'opencode-model', base_url: nil },
+            openrouter: { api_key: nil, model: 'anthropic/claude-3.5-sonnet' }
+          }
+        }
+      end
+    end
+  end
+end